diff --git a/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/hf_model_metrics.json b/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..5a1dce2d06a8404cbf079553671ed2a38d873943 --- /dev/null +++ b/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/hf_model_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.9917178819444444, + "avg_response_len": 180.0242638888889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9917178819444444, + "pass@2": 0.9949338910761112, + "pass@4": 0.996517369245511, + "pass@8": 0.997597984873445, + "pass@16": 0.998319874147491, + "pass@32": 0.9987599375664495, + "pass@64": 0.9990436401384422, + "pass@128": 0.9992222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9699453125, + "pass@2": 0.9797700541338586, + "pass@4": 0.9846088579865014, + "pass@8": 0.9883971297589192, + "pass@16": 0.991548693276063, + "pass@32": 0.9939062487742031, + "pass@64": 0.9956849154464417, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99759375, + "pass@2": 0.9991465305118111, + "pass@4": 0.9998152384701912, + "pass@8": 0.9999862821828273, + "pass@16": 0.9999999227084759, + "pass@32": 0.9999999999993933, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99834375, + "pass@2": 0.9994431594488189, + "pass@4": 0.9998681761342333, + "pass@8": 0.9999880140745279, + "pass@16": 0.9999999083912323, + "pass@32": 0.999999999998944, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9933515625, + "pass@2": 0.9955663139763777, + "pass@4": 0.9974079118860144, + "pass@8": 0.9986958248958838, + "pass@16": 0.9994353865491737, + "pass@32": 0.9998235513802302, + "pass@64": 0.9999861740539754, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.995625, + "pass@2": 0.99801156496063, + "pass@4": 0.9991621754780652, + "pass@8": 0.9997905041568963, + "pass@16": 0.9999826485802666, + "pass@32": 0.999999900717592, + "pass@64": 0.9999999999999175, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.991, + "pass@2": 0.9933371062992128, + "pass@4": 0.9947224583802023, + "pass@8": 0.9958695873510292, + "pass@16": 0.9966041585170369, + "pass@32": 0.9969280491355603, + "pass@64": 0.9969985543119962, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.9948046875, + "pass@2": 0.99785125492126, + "pass@4": 0.998795172384702, + "pass@8": 0.9994316174246192, + "pass@16": 0.9998579688324685, + "pass@32": 0.9999931255781963, + "pass@64": 0.9999999975930302, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.984796875, + "pass@2": 0.9912790354330716, + "pass@4": 0.9942763324896889, + "pass@8": 0.996222904016301, + "pass@16": 0.9974501804727003, + "pass@32": 0.9981885625139278, + "pass@64": 0.998723119840615, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.9699453125, + "2": 1.0, + "3": 0.99759375, + "4": 0.99834375, + "5": 0.9933515625, + "6": 0.995625, + "7": 0.991, + "8": 0.9948046875, + "9": 0.984796875 + }, + "per_op_avg_response_len": { + "10": 251.330375, + "2": 104.7425390625, + "3": 124.3257890625, + "4": 144.4909296875, + "5": 167.9944296875, + "6": 178.6954140625, + "7": 194.371984375, + "8": 218.1445625, + "9": 236.1223515625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 251.330375, + "2": 104.7425390625, + "3": 124.3257890625, + "4": 144.4909296875, + "5": 167.9944296875, + "6": 178.6954140625, + "7": 194.371984375, + "8": 218.1445625, + "9": 236.1223515625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 1142459, + "answer_accuracy": 0.9917178819444444, + "avg_response_len": 180.0242638888889, + "resp_tokens_sum": 207387952, + "pass_at_k": { + "pass@1": 0.9917178819444444, + "pass@2": 0.9949338910761112, + "pass@4": 0.996517369245511, + "pass@8": 0.997597984873445, + "pass@16": 0.998319874147491, + "pass@32": 0.9987599375664495, + "pass@64": 0.9990436401384422, + "pass@128": 0.9992222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9699453125, + "pass@2": 0.9797700541338586, + "pass@4": 0.9846088579865014, + "pass@8": 0.9883971297589192, + "pass@16": 0.991548693276063, + "pass@32": 0.9939062487742031, + "pass@64": 0.9956849154464417, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99759375, + "pass@2": 0.9991465305118111, + "pass@4": 0.9998152384701912, + "pass@8": 0.9999862821828273, + "pass@16": 0.9999999227084759, + "pass@32": 0.9999999999993933, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99834375, + "pass@2": 0.9994431594488189, + "pass@4": 0.9998681761342333, + "pass@8": 0.9999880140745279, + "pass@16": 0.9999999083912323, + "pass@32": 0.999999999998944, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9933515625, + "pass@2": 0.9955663139763777, + "pass@4": 0.9974079118860144, + "pass@8": 0.9986958248958838, + "pass@16": 0.9994353865491737, + "pass@32": 0.9998235513802302, + "pass@64": 0.9999861740539754, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.995625, + "pass@2": 0.99801156496063, + "pass@4": 0.9991621754780652, + "pass@8": 0.9997905041568963, + "pass@16": 0.9999826485802666, + "pass@32": 0.999999900717592, + "pass@64": 0.9999999999999175, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.991, + "pass@2": 0.9933371062992128, + "pass@4": 0.9947224583802023, + "pass@8": 0.9958695873510292, + "pass@16": 0.9966041585170369, + "pass@32": 0.9969280491355603, + "pass@64": 0.9969985543119962, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.9948046875, + "pass@2": 0.99785125492126, + "pass@4": 0.998795172384702, + "pass@8": 0.9994316174246192, + "pass@16": 0.9998579688324685, + "pass@32": 0.9999931255781963, + "pass@64": 0.9999999975930302, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.984796875, + "pass@2": 0.9912790354330716, + "pass@4": 0.9942763324896889, + "pass@8": 0.996222904016301, + "pass@16": 0.9974501804727003, + "pass@32": 0.9981885625139278, + "pass@64": 0.998723119840615, + "pass@128": 0.999 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.90478203125, + "avg_response_len": 346.5120359375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.90478203125, + "pass@2": 0.9390087844487882, + "pass@4": 0.956556485714285, + "pass@8": 0.9671536617625254, + "pass@16": 0.9742336997731345, + "pass@32": 0.9795612719017148, + "pass@64": 0.9838756366040737, + "pass@128": 0.9875 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9512578125, + "pass@2": 0.9658013041338588, + "pass@4": 0.9731058290213717, + "pass@8": 0.9777503786832513, + "pass@16": 0.9812478554053372, + "pass@32": 0.9844936593333612, + "pass@64": 0.9875782128535292, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.9279609375, + "pass@2": 0.9452753444881897, + "pass@4": 0.9557707446569181, + "pass@8": 0.9629128336731604, + "pass@16": 0.9684001319561727, + "pass@32": 0.9734901999432878, + "pass@64": 0.9785937143414183, + "pass@128": 0.984 + }, + "13": { + "pass@1": 0.9148515625, + "pass@2": 0.9373233267716538, + "pass@4": 0.9503126651668536, + "pass@8": 0.9590150667585761, + "pass@16": 0.9658224358162538, + "pass@32": 0.9720045806193894, + "pass@64": 0.9776870791192475, + "pass@128": 0.983 + }, + "14": { + "pass@1": 0.9195390625, + "pass@2": 0.9461455462598434, + "pass@4": 0.9610489086989125, + "pass@8": 0.9705652801128191, + "pass@16": 0.976617495691548, + "pass@32": 0.9807539224865852, + "pass@64": 0.9838434201680727, + "pass@128": 0.986 + }, + "15": { + "pass@1": 0.9212421875, + "pass@2": 0.9496477608267725, + "pass@4": 0.9639471972253464, + "pass@8": 0.9731272495996897, + "pass@16": 0.9795135593649998, + "pass@32": 0.9843676266202894, + "pass@64": 0.9887092071015339, + "pass@128": 0.993 + }, + "16": { + "pass@1": 0.9186484375, + "pass@2": 0.9507684547244115, + "pass@4": 0.9659008573303339, + "pass@8": 0.974618462414321, + "pass@16": 0.9800429740397889, + "pass@32": 0.9841030640501913, + "pass@64": 0.9876616774077218, + "pass@128": 0.99 + }, + "17": { + "pass@1": 0.9137890625, + "pass@2": 0.9511459153543321, + "pass@4": 0.9683324656917881, + "pass@8": 0.9782519917436393, + "pass@16": 0.9845447476466959, + "pass@32": 0.9884888034847372, + "pass@64": 0.9909003607208057, + "pass@128": 0.993 + }, + "18": { + "pass@1": 0.8819921875, + "pass@2": 0.9288923474409468, + "pass@4": 0.9518024659730036, + "pass@8": 0.9643686704288694, + "pass@16": 0.9725036453284903, + "pass@32": 0.9785188430691222, + "pass@64": 0.9827390755534815, + "pass@128": 0.985 + }, + "19": { + "pass@1": 0.8668359375, + "pass@2": 0.9196662155511819, + "pass@4": 0.9460197001312345, + "pass@8": 0.9618559891779669, + "pass@16": 0.9724486989570319, + "pass@32": 0.9796740074314165, + "pass@64": 0.9850677359090562, + "pass@128": 0.99 + }, + "20": { + "pass@1": 0.831703125, + "pass@2": 0.8954216289370094, + "pass@4": 0.9293240232470946, + "pass@8": 0.9490706950329557, + "pass@16": 0.9611954535250025, + "pass@32": 0.9697180119787673, + "pass@64": 0.9759758828658656, + "pass@128": 0.981 + } + }, + "per_op_accuracy": { + "11": 0.9512578125, + "12": 0.9279609375, + "13": 0.9148515625, + "14": 0.9195390625, + "15": 0.9212421875, + "16": 0.9186484375, + "17": 0.9137890625, + "18": 0.8819921875, + "19": 0.8668359375, + "20": 0.831703125 + }, + "per_op_avg_response_len": { + "11": 271.009546875, + "12": 283.33071875, + "13": 294.0766015625, + "14": 323.366203125, + "15": 339.5786640625, + "16": 364.742375, + "17": 376.832859375, + "18": 390.7912109375, + "19": 401.32259375, + "20": 420.0695859375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 271.009546875, + "12": 283.33071875, + "13": 294.0766015625, + "14": 323.366203125, + "15": 339.5786640625, + "16": 364.742375, + "17": 376.832859375, + "18": 390.7912109375, + "19": 401.32259375, + "20": 420.0695859375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 1158121, + "answer_accuracy": 0.90478203125, + "avg_response_len": 346.5120359375, + "resp_tokens_sum": 443535406, + "pass_at_k": { + "pass@1": 0.90478203125, + "pass@2": 0.9390087844487882, + "pass@4": 0.956556485714285, + "pass@8": 0.9671536617625254, + "pass@16": 0.9742336997731345, + "pass@32": 0.9795612719017148, + "pass@64": 0.9838756366040737, + "pass@128": 0.9875 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9512578125, + "pass@2": 0.9658013041338588, + "pass@4": 0.9731058290213717, + "pass@8": 0.9777503786832513, + "pass@16": 0.9812478554053372, + "pass@32": 0.9844936593333612, + "pass@64": 0.9875782128535292, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.9279609375, + "pass@2": 0.9452753444881897, + "pass@4": 0.9557707446569181, + "pass@8": 0.9629128336731604, + "pass@16": 0.9684001319561727, + "pass@32": 0.9734901999432878, + "pass@64": 0.9785937143414183, + "pass@128": 0.984 + }, + "13": { + "pass@1": 0.9148515625, + "pass@2": 0.9373233267716538, + "pass@4": 0.9503126651668536, + "pass@8": 0.9590150667585761, + "pass@16": 0.9658224358162538, + "pass@32": 0.9720045806193894, + "pass@64": 0.9776870791192475, + "pass@128": 0.983 + }, + "14": { + "pass@1": 0.9195390625, + "pass@2": 0.9461455462598434, + "pass@4": 0.9610489086989125, + "pass@8": 0.9705652801128191, + "pass@16": 0.976617495691548, + "pass@32": 0.9807539224865852, + "pass@64": 0.9838434201680727, + "pass@128": 0.986 + }, + "15": { + "pass@1": 0.9212421875, + "pass@2": 0.9496477608267725, + "pass@4": 0.9639471972253464, + "pass@8": 0.9731272495996897, + "pass@16": 0.9795135593649998, + "pass@32": 0.9843676266202894, + "pass@64": 0.9887092071015339, + "pass@128": 0.993 + }, + "16": { + "pass@1": 0.9186484375, + "pass@2": 0.9507684547244115, + "pass@4": 0.9659008573303339, + "pass@8": 0.974618462414321, + "pass@16": 0.9800429740397889, + "pass@32": 0.9841030640501913, + "pass@64": 0.9876616774077218, + "pass@128": 0.99 + }, + "17": { + "pass@1": 0.9137890625, + "pass@2": 0.9511459153543321, + "pass@4": 0.9683324656917881, + "pass@8": 0.9782519917436393, + "pass@16": 0.9845447476466959, + "pass@32": 0.9884888034847372, + "pass@64": 0.9909003607208057, + "pass@128": 0.993 + }, + "18": { + "pass@1": 0.8819921875, + "pass@2": 0.9288923474409468, + "pass@4": 0.9518024659730036, + "pass@8": 0.9643686704288694, + "pass@16": 0.9725036453284903, + "pass@32": 0.9785188430691222, + "pass@64": 0.9827390755534815, + "pass@128": 0.985 + }, + "19": { + "pass@1": 0.8668359375, + "pass@2": 0.9196662155511819, + "pass@4": 0.9460197001312345, + "pass@8": 0.9618559891779669, + "pass@16": 0.9724486989570319, + "pass@32": 0.9796740074314165, + "pass@64": 0.9850677359090562, + "pass@128": 0.99 + }, + "20": { + "pass@1": 0.831703125, + "pass@2": 0.8954216289370094, + "pass@4": 0.9293240232470946, + "pass@8": 0.9490706950329557, + "pass@16": 0.9611954535250025, + "pass@32": 0.9697180119787673, + "pass@64": 0.9759758828658656, + "pass@128": 0.981 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.9459621710526316, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.9459621710526316, + "pass@2": 0.965499624430164, + "pass@4": 0.9754853252816995, + "pass@8": 0.9815746569203379, + "pass@16": 0.9856429402662464, + "pass@32": 0.9886553766902775, + "pass@64": 0.9910604803835107, + "pass@128": 0.9930526315789474 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9699453125, + "pass@2": 0.9797700541338586, + "pass@4": 0.9846088579865014, + "pass@8": 0.9883971297589192, + "pass@16": 0.991548693276063, + "pass@32": 0.9939062487742031, + "pass@64": 0.9956849154464417, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99759375, + "pass@2": 0.9991465305118111, + "pass@4": 0.9998152384701912, + "pass@8": 0.9999862821828273, + "pass@16": 0.9999999227084759, + "pass@32": 0.9999999999993933, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99834375, + "pass@2": 0.9994431594488189, + "pass@4": 0.9998681761342333, + "pass@8": 0.9999880140745279, + "pass@16": 0.9999999083912323, + "pass@32": 0.999999999998944, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9933515625, + "pass@2": 0.9955663139763777, + "pass@4": 0.9974079118860144, + "pass@8": 0.9986958248958838, + "pass@16": 0.9994353865491737, + "pass@32": 0.9998235513802302, + "pass@64": 0.9999861740539754, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.995625, + "pass@2": 0.99801156496063, + "pass@4": 0.9991621754780652, + "pass@8": 0.9997905041568963, + "pass@16": 0.9999826485802666, + "pass@32": 0.999999900717592, + "pass@64": 0.9999999999999175, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.991, + "pass@2": 0.9933371062992128, + "pass@4": 0.9947224583802023, + "pass@8": 0.9958695873510292, + "pass@16": 0.9966041585170369, + "pass@32": 0.9969280491355603, + "pass@64": 0.9969985543119962, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.9948046875, + "pass@2": 0.99785125492126, + "pass@4": 0.998795172384702, + "pass@8": 0.9994316174246192, + "pass@16": 0.9998579688324685, + "pass@32": 0.9999931255781963, + "pass@64": 0.9999999975930302, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.984796875, + "pass@2": 0.9912790354330716, + "pass@4": 0.9942763324896889, + "pass@8": 0.996222904016301, + "pass@16": 0.9974501804727003, + "pass@32": 0.9981885625139278, + "pass@64": 0.998723119840615, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9512578125, + "pass@2": 0.9658013041338588, + "pass@4": 0.9731058290213717, + "pass@8": 0.9777503786832513, + "pass@16": 0.9812478554053372, + "pass@32": 0.9844936593333612, + "pass@64": 0.9875782128535292, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.9279609375, + "pass@2": 0.9452753444881897, + "pass@4": 0.9557707446569181, + "pass@8": 0.9629128336731604, + "pass@16": 0.9684001319561727, + "pass@32": 0.9734901999432878, + "pass@64": 0.9785937143414183, + "pass@128": 0.984 + }, + "13": { + "pass@1": 0.9148515625, + "pass@2": 0.9373233267716538, + "pass@4": 0.9503126651668536, + "pass@8": 0.9590150667585761, + "pass@16": 0.9658224358162538, + "pass@32": 0.9720045806193894, + "pass@64": 0.9776870791192475, + "pass@128": 0.983 + }, + "14": { + "pass@1": 0.9195390625, + "pass@2": 0.9461455462598434, + "pass@4": 0.9610489086989125, + "pass@8": 0.9705652801128191, + "pass@16": 0.976617495691548, + "pass@32": 0.9807539224865852, + "pass@64": 0.9838434201680727, + "pass@128": 0.986 + }, + "15": { + "pass@1": 0.9212421875, + "pass@2": 0.9496477608267725, + "pass@4": 0.9639471972253464, + "pass@8": 0.9731272495996897, + "pass@16": 0.9795135593649998, + "pass@32": 0.9843676266202894, + "pass@64": 0.9887092071015339, + "pass@128": 0.993 + }, + "16": { + "pass@1": 0.9186484375, + "pass@2": 0.9507684547244115, + "pass@4": 0.9659008573303339, + "pass@8": 0.974618462414321, + "pass@16": 0.9800429740397889, + "pass@32": 0.9841030640501913, + "pass@64": 0.9876616774077218, + "pass@128": 0.99 + }, + "17": { + "pass@1": 0.9137890625, + "pass@2": 0.9511459153543321, + "pass@4": 0.9683324656917881, + "pass@8": 0.9782519917436393, + "pass@16": 0.9845447476466959, + "pass@32": 0.9884888034847372, + "pass@64": 0.9909003607208057, + "pass@128": 0.993 + }, + "18": { + "pass@1": 0.8819921875, + "pass@2": 0.9288923474409468, + "pass@4": 0.9518024659730036, + "pass@8": 0.9643686704288694, + "pass@16": 0.9725036453284903, + "pass@32": 0.9785188430691222, + "pass@64": 0.9827390755534815, + "pass@128": 0.985 + }, + "19": { + "pass@1": 0.8668359375, + "pass@2": 0.9196662155511819, + "pass@4": 0.9460197001312345, + "pass@8": 0.9618559891779669, + "pass@16": 0.9724486989570319, + "pass@32": 0.9796740074314165, + "pass@64": 0.9850677359090562, + "pass@128": 0.99 + }, + "20": { + "pass@1": 0.831703125, + "pass@2": 0.8954216289370094, + "pass@4": 0.9293240232470946, + "pass@8": 0.9490706950329557, + "pass@16": 0.9611954535250025, + "pass@32": 0.9697180119787673, + "pass@64": 0.9759758828658656, + "pass@128": 0.981 + } + }, + "per_op_accuracy": { + "10": 0.9699453125, + "2": 1.0, + "3": 0.99759375, + "4": 0.99834375, + "5": 0.9933515625, + "6": 0.995625, + "7": 0.991, + "8": 0.9948046875, + "9": 0.984796875, + "11": 0.9512578125, + "12": 0.9279609375, + "13": 0.9148515625, + "14": 0.9195390625, + "15": 0.9212421875, + "16": 0.9186484375, + "17": 0.9137890625, + "18": 0.8819921875, + "19": 0.8668359375, + "20": 0.831703125 + }, + "per_op_avg_response_len": { + "10": 251.330375, + "2": 104.7425390625, + "3": 124.3257890625, + "4": 144.4909296875, + "5": 167.9944296875, + "6": 178.6954140625, + "7": 194.371984375, + "8": 218.1445625, + "9": 236.1223515625, + "11": 271.009546875, + "12": 283.33071875, + "13": 294.0766015625, + "14": 323.366203125, + "15": 339.5786640625, + "16": 364.742375, + "17": 376.832859375, + "18": 390.7912109375, + "19": 401.32259375, + "20": 420.0695859375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 251.330375, + "2": 104.7425390625, + "3": 124.3257890625, + "4": 144.4909296875, + "5": 167.9944296875, + "6": 178.6954140625, + "7": 194.371984375, + "8": 218.1445625, + "9": 236.1223515625, + "11": 271.009546875, + "12": 283.33071875, + "13": 294.0766015625, + "14": 323.366203125, + "15": 339.5786640625, + "16": 364.742375, + "17": 376.832859375, + "18": 390.7912109375, + "19": 401.32259375, + "20": 420.0695859375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 2300580, + "answer_accuracy": 0.9459621710526316, + "avg_response_len": 267.64940707236843, + "resp_tokens_sum": 650923358, + "pass_at_k": { + "pass@1": 0.9459621710526316, + "pass@2": 0.965499624430164, + "pass@4": 0.9754853252816995, + "pass@8": 0.9815746569203379, + "pass@16": 0.9856429402662464, + "pass@32": 0.9886553766902775, + "pass@64": 0.9910604803835107, + "pass@128": 0.9930526315789474 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9699453125, + "pass@2": 0.9797700541338586, + "pass@4": 0.9846088579865014, + "pass@8": 0.9883971297589192, + "pass@16": 0.991548693276063, + "pass@32": 0.9939062487742031, + "pass@64": 0.9956849154464417, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99759375, + "pass@2": 0.9991465305118111, + "pass@4": 0.9998152384701912, + "pass@8": 0.9999862821828273, + "pass@16": 0.9999999227084759, + "pass@32": 0.9999999999993933, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99834375, + "pass@2": 0.9994431594488189, + "pass@4": 0.9998681761342333, + "pass@8": 0.9999880140745279, + "pass@16": 0.9999999083912323, + "pass@32": 0.999999999998944, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9933515625, + "pass@2": 0.9955663139763777, + "pass@4": 0.9974079118860144, + "pass@8": 0.9986958248958838, + "pass@16": 0.9994353865491737, + "pass@32": 0.9998235513802302, + "pass@64": 0.9999861740539754, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.995625, + "pass@2": 0.99801156496063, + "pass@4": 0.9991621754780652, + "pass@8": 0.9997905041568963, + "pass@16": 0.9999826485802666, + "pass@32": 0.999999900717592, + "pass@64": 0.9999999999999175, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.991, + "pass@2": 0.9933371062992128, + "pass@4": 0.9947224583802023, + "pass@8": 0.9958695873510292, + "pass@16": 0.9966041585170369, + "pass@32": 0.9969280491355603, + "pass@64": 0.9969985543119962, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.9948046875, + "pass@2": 0.99785125492126, + "pass@4": 0.998795172384702, + "pass@8": 0.9994316174246192, + "pass@16": 0.9998579688324685, + "pass@32": 0.9999931255781963, + "pass@64": 0.9999999975930302, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.984796875, + "pass@2": 0.9912790354330716, + "pass@4": 0.9942763324896889, + "pass@8": 0.996222904016301, + "pass@16": 0.9974501804727003, + "pass@32": 0.9981885625139278, + "pass@64": 0.998723119840615, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9512578125, + "pass@2": 0.9658013041338588, + "pass@4": 0.9731058290213717, + "pass@8": 0.9777503786832513, + "pass@16": 0.9812478554053372, + "pass@32": 0.9844936593333612, + "pass@64": 0.9875782128535292, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.9279609375, + "pass@2": 0.9452753444881897, + "pass@4": 0.9557707446569181, + "pass@8": 0.9629128336731604, + "pass@16": 0.9684001319561727, + "pass@32": 0.9734901999432878, + "pass@64": 0.9785937143414183, + "pass@128": 0.984 + }, + "13": { + "pass@1": 0.9148515625, + "pass@2": 0.9373233267716538, + "pass@4": 0.9503126651668536, + "pass@8": 0.9590150667585761, + "pass@16": 0.9658224358162538, + "pass@32": 0.9720045806193894, + "pass@64": 0.9776870791192475, + "pass@128": 0.983 + }, + "14": { + "pass@1": 0.9195390625, + "pass@2": 0.9461455462598434, + "pass@4": 0.9610489086989125, + "pass@8": 0.9705652801128191, + "pass@16": 0.976617495691548, + "pass@32": 0.9807539224865852, + "pass@64": 0.9838434201680727, + "pass@128": 0.986 + }, + "15": { + "pass@1": 0.9212421875, + "pass@2": 0.9496477608267725, + "pass@4": 0.9639471972253464, + "pass@8": 0.9731272495996897, + "pass@16": 0.9795135593649998, + "pass@32": 0.9843676266202894, + "pass@64": 0.9887092071015339, + "pass@128": 0.993 + }, + "16": { + "pass@1": 0.9186484375, + "pass@2": 0.9507684547244115, + "pass@4": 0.9659008573303339, + "pass@8": 0.974618462414321, + "pass@16": 0.9800429740397889, + "pass@32": 0.9841030640501913, + "pass@64": 0.9876616774077218, + "pass@128": 0.99 + }, + "17": { + "pass@1": 0.9137890625, + "pass@2": 0.9511459153543321, + "pass@4": 0.9683324656917881, + "pass@8": 0.9782519917436393, + "pass@16": 0.9845447476466959, + "pass@32": 0.9884888034847372, + "pass@64": 0.9909003607208057, + "pass@128": 0.993 + }, + "18": { + "pass@1": 0.8819921875, + "pass@2": 0.9288923474409468, + "pass@4": 0.9518024659730036, + "pass@8": 0.9643686704288694, + "pass@16": 0.9725036453284903, + "pass@32": 0.9785188430691222, + "pass@64": 0.9827390755534815, + "pass@128": 0.985 + }, + "19": { + "pass@1": 0.8668359375, + "pass@2": 0.9196662155511819, + "pass@4": 0.9460197001312345, + "pass@8": 0.9618559891779669, + "pass@16": 0.9724486989570319, + "pass@32": 0.9796740074314165, + "pass@64": 0.9850677359090562, + "pass@128": 0.99 + }, + "20": { + "pass@1": 0.831703125, + "pass@2": 0.8954216289370094, + "pass@4": 0.9293240232470946, + "pass@8": 0.9490706950329557, + "pass@16": 0.9611954535250025, + "pass@32": 0.9697180119787673, + "pass@64": 0.9759758828658656, + "pass@128": 0.981 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0dc59402575a4093b25f6c103c8aa737a7e842fb --- /dev/null +++ b/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive", "results_dir": "results/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/hf_model_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1024664, "value_mismatch": 8805, "dependency_mismatch": 125514, "answer_mismatch": 7776, "extra_nodes": 10362, "missing_nodes": 8974, "perfect_with_extra": 643, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337990, "value_mismatch": 2543, "answer_mismatch": 2145, "dependency_mismatch": 40544, "extra_nodes": 3217, "perfect_with_extra": 109, "missing_nodes": 2826}, "teachers_in_school": {"total": 390656, "perfect": 347108, "value_mismatch": 3320, "dependency_mismatch": 42907, "answer_mismatch": 3093, "missing_nodes": 3426, "extra_nodes": 3661, "perfect_with_extra": 174}, "movie_festival_awards": {"total": 382336, "perfect": 339566, "value_mismatch": 2942, "answer_mismatch": 2538, "dependency_mismatch": 42063, "extra_nodes": 3484, "missing_nodes": 2722, "perfect_with_extra": 360}}, "results_path": "results/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/graphs/hf_model_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/hf_model_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 453513, "value_mismatch": 528235, "dependency_mismatch": 741943, "answer_mismatch": 422750, "extra_nodes": 45823, "missing_nodes": 195819, "perfect_with_extra": 304, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 151972, "value_mismatch": 178323, "dependency_mismatch": 249517, "answer_mismatch": 142111, "extra_nodes": 16393, "missing_nodes": 69447, "perfect_with_extra": 73}, "movie_festival_awards": {"total": 418816, "perfect": 145534, "value_mismatch": 179604, "dependency_mismatch": 245458, "answer_mismatch": 144161, "extra_nodes": 15235, "missing_nodes": 63417, "perfect_with_extra": 175}, "teachers_in_school": {"total": 430464, "perfect": 156007, "dependency_mismatch": 246968, "missing_nodes": 62955, "value_mismatch": 170308, "answer_mismatch": 136478, "extra_nodes": 14195, "perfect_with_extra": 56}}, "results_path": "results/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/graphs/hf_model_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1478177, "value_mismatch": 537040, "dependency_mismatch": 867457, "answer_mismatch": 430526, "extra_nodes": 56185, "missing_nodes": 204793, "perfect_with_extra": 947, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 489962, "value_mismatch": 180866, "answer_mismatch": 144256, "dependency_mismatch": 290061, "extra_nodes": 19610, "perfect_with_extra": 182, "missing_nodes": 72273}, "teachers_in_school": {"total": 821120, "perfect": 503115, "value_mismatch": 173628, "dependency_mismatch": 289875, "answer_mismatch": 139571, "missing_nodes": 66381, "extra_nodes": 17856, "perfect_with_extra": 230}, "movie_festival_awards": {"total": 801152, "perfect": 485100, "value_mismatch": 182546, "answer_mismatch": 146699, "dependency_mismatch": 287521, "extra_nodes": 18719, "missing_nodes": 66139, "perfect_with_extra": 535}}}} diff --git a/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/graphs/results.json b/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..4212c0af059e965e46e33133a19e879c9cbb4cc5 --- /dev/null +++ b/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "hf_model_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9619047619047619, + "pass@2": 0.9904761904761905, + "pass@4": 0.9968253968253968, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9520958083832335, + "pass@2": 0.9580838323353293, + "pass@4": 0.9730538922155688, + "pass@8": 0.9880239520958084, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8905325443786982, + "pass@2": 0.9408284023668639, + "pass@4": 0.9585798816568047, + "pass@8": 0.985207100591716, + "pass@16": 0.9970414201183432, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8205128205128205, + "pass@2": 0.8846153846153846, + "pass@4": 0.9551282051282052, + "pass@8": 0.9871794871794872, + "pass@16": 0.9967948717948718, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7577464788732394, + "pass@2": 0.8394366197183099, + "pass@4": 0.9154929577464789, + "pass@8": 0.9577464788732394, + "pass@16": 0.9830985915492958, + "pass@32": 0.9915492957746479, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6666666666666666, + "pass@2": 0.7286135693215339, + "pass@4": 0.799410029498525, + "pass@8": 0.8820058997050148, + "pass@16": 0.9262536873156342, + "pass@32": 0.9469026548672567, + "pass@64": 0.9646017699115044, + "pass@128": 0.9734513274336283 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9619047619047619, + "6": 0.9520958083832335, + "7": 0.8905325443786982, + "8": 0.8205128205128205, + "9": 0.7577464788732394, + "10": 0.6666666666666666 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9904761904761905, + "6": 0.9580838323353293, + "7": 0.9408284023668639, + "8": 0.8846153846153846, + "9": 0.8394366197183099, + "10": 0.7286135693215339 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 0.9730538922155688, + "7": 0.9585798816568047, + "8": 0.9551282051282052, + "9": 0.9154929577464789, + "10": 0.799410029498525 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9880239520958084, + "7": 0.985207100591716, + "8": 0.9871794871794872, + "9": 0.9577464788732394, + "10": 0.8820058997050148 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 0.9830985915492958, + "10": 0.9262536873156342 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9915492957746479, + "10": 0.9469026548672567 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9646017699115044 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9734513274336283 + } + }, + "total_pass_at_k": { + "pass@1": 0.8912529550827423, + "pass@2": 0.9250253292806484, + "pass@4": 0.9540695710908477, + "pass@8": 0.9770347855454239, + "pass@16": 0.9888551165146909, + "pass@32": 0.9929078014184397, + "pass@64": 0.9959473150962512, + "pass@128": 0.9969604863221885 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9853801169590644, + "pass@2": 0.9912280701754386, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.95, + "pass@2": 0.9794117647058823, + "pass@4": 0.9911764705882353, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9716088328075709, + "pass@2": 0.9842271293375394, + "pass@4": 0.9873817034700315, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8829113924050633, + "pass@2": 0.9177215189873418, + "pass@4": 0.9430379746835443, + "pass@8": 0.9746835443037974, + "pass@16": 0.9936708860759493, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8048048048048048, + "pass@2": 0.8648648648648649, + "pass@4": 0.924924924924925, + "pass@8": 0.978978978978979, + "pass@16": 0.990990990990991, + "pass@32": 0.996996996996997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7684887459807074, + "pass@2": 0.8360128617363344, + "pass@4": 0.909967845659164, + "pass@8": 0.9710610932475884, + "pass@16": 0.9967845659163987, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6119402985074627, + "pass@2": 0.6805970149253732, + "pass@4": 0.7910447761194029, + "pass@8": 0.8716417910447761, + "pass@16": 0.9194029850746268, + "pass@32": 0.9492537313432836, + "pass@64": 0.9671641791044776, + "pass@128": 0.9701492537313433 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9853801169590644, + "5": 0.95, + "6": 0.9716088328075709, + "7": 0.8829113924050633, + "8": 0.8048048048048048, + "9": 0.7684887459807074, + "10": 0.6119402985074627 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9794117647058823, + "6": 0.9842271293375394, + "7": 0.9177215189873418, + "8": 0.8648648648648649, + "9": 0.8360128617363344, + "10": 0.6805970149253732 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9911764705882353, + "6": 0.9873817034700315, + "7": 0.9430379746835443, + "8": 0.924924924924925, + "9": 0.909967845659164, + "10": 0.7910447761194029 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9746835443037974, + "8": 0.978978978978979, + "9": 0.9710610932475884, + "10": 0.8716417910447761 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.990990990990991, + "9": 0.9967845659163987, + "10": 0.9194029850746268 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9492537313432836 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9671641791044776 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9701492537313433 + } + }, + "total_pass_at_k": { + "pass@1": 0.887847338466689, + "pass@2": 0.9183126883160362, + "pass@4": 0.950451958486776, + "pass@8": 0.9775694676933377, + "pass@16": 0.9889521258788082, + "pass@32": 0.9939738868429863, + "pass@64": 0.9963173752929361, + "pass@128": 0.9966521593572146 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9763313609467456, + "pass@2": 0.9911242603550295, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9536231884057971, + "pass@2": 0.9826086956521739, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9770773638968482, + "pass@2": 0.9770773638968482, + "pass@4": 0.9914040114613181, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.869942196531792, + "pass@2": 0.9277456647398844, + "pass@4": 0.953757225433526, + "pass@8": 0.9826589595375722, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8225352112676056, + "pass@2": 0.8788732394366198, + "pass@4": 0.9380281690140845, + "pass@8": 0.9690140845070423, + "pass@16": 0.9915492957746479, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7784431137724551, + "pass@2": 0.8383233532934131, + "pass@4": 0.907185628742515, + "pass@8": 0.9520958083832335, + "pass@16": 0.9850299401197605, + "pass@32": 0.9940119760479041, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6503067484662577, + "pass@2": 0.7208588957055214, + "pass@4": 0.7822085889570553, + "pass@8": 0.8650306748466258, + "pass@16": 0.9079754601226994, + "pass@32": 0.941717791411043, + "pass@64": 0.9570552147239264, + "pass@128": 0.9631901840490797 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9763313609467456, + "5": 0.9536231884057971, + "6": 0.9770773638968482, + "7": 0.869942196531792, + "8": 0.8225352112676056, + "9": 0.7784431137724551, + "10": 0.6503067484662577 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9911242603550295, + "5": 0.9826086956521739, + "6": 0.9770773638968482, + "7": 0.9277456647398844, + "8": 0.8788732394366198, + "9": 0.8383233532934131, + "10": 0.7208588957055214 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9914040114613181, + "7": 0.953757225433526, + "8": 0.9380281690140845, + "9": 0.907185628742515, + "10": 0.7822085889570553 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9826589595375722, + "8": 0.9690140845070423, + "9": 0.9520958083832335, + "10": 0.8650306748466258 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9915492957746479, + "9": 0.9850299401197605, + "10": 0.9079754601226994 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9940119760479041, + "10": 0.941717791411043 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9570552147239264 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9631901840490797 + } + }, + "total_pass_at_k": { + "pass@1": 0.8925294888597641, + "pass@2": 0.9246395806028833, + "pass@4": 0.9531454783748362, + "pass@8": 0.9747706422018348, + "pass@16": 0.9875491480996068, + "pass@32": 0.9927916120576671, + "pass@64": 0.9954128440366973, + "pass@128": 0.9960681520314548 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985, + "pass@2": 0.994, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.955, + "pass@2": 0.984, + "pass@4": 0.996, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.967, + "pass@2": 0.973, + "pass@4": 0.984, + "pass@8": 0.996, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.881, + "pass@2": 0.929, + "pass@4": 0.952, + "pass@8": 0.981, + "pass@16": 0.997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.816, + "pass@2": 0.876, + "pass@4": 0.939, + "pass@8": 0.978, + "pass@16": 0.993, + "pass@32": 0.998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.768, + "pass@2": 0.838, + "pass@4": 0.911, + "pass@8": 0.96, + "pass@16": 0.988, + "pass@32": 0.995, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.643, + "pass@2": 0.71, + "pass@4": 0.791, + "pass@8": 0.873, + "pass@16": 0.918, + "pass@32": 0.946, + "pass@64": 0.963, + "pass@128": 0.969 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.985, + "5": 0.955, + "6": 0.967, + "7": 0.881, + "8": 0.816, + "9": 0.768, + "10": 0.643 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.994, + "5": 0.984, + "6": 0.973, + "7": 0.929, + "8": 0.876, + "9": 0.838, + "10": 0.71 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.996, + "6": 0.984, + "7": 0.952, + "8": 0.939, + "9": 0.911, + "10": 0.791 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.996, + "7": 0.981, + "8": 0.978, + "9": 0.96, + "10": 0.873 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.997, + "8": 0.993, + "9": 0.988, + "10": 0.918 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.998, + "9": 0.995, + "10": 0.946 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.963 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.969 + } + }, + "total_pass_at_k": { + "pass@1": 0.8905555555555555, + "pass@2": 0.9226666666666666, + "pass@4": 0.9525555555555556, + "pass@8": 0.9764444444444444, + "pass@16": 0.9884444444444445, + "pass@32": 0.9932222222222222, + "pass@64": 0.9958888888888889, + "pass@128": 0.9965555555555555 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "hf_model_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5779036827195467, + "pass@2": 0.660056657223796, + "pass@4": 0.7308781869688386, + "pass@8": 0.830028328611898, + "pass@16": 0.8640226628895185, + "pass@32": 0.9065155807365439, + "pass@64": 0.9348441926345609, + "pass@128": 0.9461756373937678 + }, + "12": { + "pass@1": 0.5903954802259888, + "pass@2": 0.6440677966101694, + "pass@4": 0.7090395480225988, + "pass@8": 0.7598870056497176, + "pass@16": 0.8050847457627118, + "pass@32": 0.8418079096045198, + "pass@64": 0.8531073446327684, + "pass@128": 0.8700564971751412 + }, + "13": { + "pass@1": 0.47962382445141066, + "pass@2": 0.5266457680250783, + "pass@4": 0.5893416927899686, + "pass@8": 0.6332288401253918, + "pass@16": 0.6927899686520376, + "pass@32": 0.7586206896551724, + "pass@64": 0.7962382445141066, + "pass@128": 0.8150470219435737 + }, + "14": { + "pass@1": 0.4392523364485981, + "pass@2": 0.5015576323987538, + "pass@4": 0.5545171339563862, + "pass@8": 0.5856697819314641, + "pass@16": 0.6292834890965732, + "pass@32": 0.7009345794392523, + "pass@64": 0.7570093457943925, + "pass@128": 0.7975077881619937 + }, + "15": { + "pass@1": 0.3930635838150289, + "pass@2": 0.43641618497109824, + "pass@4": 0.5, + "pass@8": 0.5635838150289018, + "pass@16": 0.630057803468208, + "pass@32": 0.7023121387283237, + "pass@64": 0.7601156069364162, + "pass@128": 0.8265895953757225 + }, + "16": { + "pass@1": 0.26129032258064516, + "pass@2": 0.33225806451612905, + "pass@4": 0.36774193548387096, + "pass@8": 0.4032258064516129, + "pass@16": 0.4612903225806452, + "pass@32": 0.5161290322580645, + "pass@64": 0.5774193548387097, + "pass@128": 0.6129032258064516 + }, + "17": { + "pass@1": 0.2514450867052023, + "pass@2": 0.3265895953757225, + "pass@4": 0.38439306358381503, + "pass@8": 0.42485549132947975, + "pass@16": 0.4797687861271676, + "pass@32": 0.5057803468208093, + "pass@64": 0.5722543352601156, + "pass@128": 0.6271676300578035 + }, + "18": { + "pass@1": 0.21714285714285714, + "pass@2": 0.26571428571428574, + "pass@4": 0.31142857142857144, + "pass@8": 0.34285714285714286, + "pass@16": 0.38285714285714284, + "pass@32": 0.4228571428571429, + "pass@64": 0.47714285714285715, + "pass@128": 0.5457142857142857 + }, + "19": { + "pass@1": 0.12312312312312312, + "pass@2": 0.16216216216216217, + "pass@4": 0.2072072072072072, + "pass@8": 0.22822822822822822, + "pass@16": 0.25225225225225223, + "pass@32": 0.3063063063063063, + "pass@64": 0.34234234234234234, + "pass@128": 0.3993993993993994 + }, + "20": { + "pass@1": 0.16516516516516516, + "pass@2": 0.2132132132132132, + "pass@4": 0.23723723723723725, + "pass@8": 0.27627627627627627, + "pass@16": 0.2972972972972973, + "pass@32": 0.32732732732732733, + "pass@64": 0.35435435435435436, + "pass@128": 0.3993993993993994 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5779036827195467, + "12": 0.5903954802259888, + "13": 0.47962382445141066, + "14": 0.4392523364485981, + "15": 0.3930635838150289, + "16": 0.26129032258064516, + "17": 0.2514450867052023, + "18": 0.21714285714285714, + "19": 0.12312312312312312, + "20": 0.16516516516516516 + }, + "pass@2": { + "11": 0.660056657223796, + "12": 0.6440677966101694, + "13": 0.5266457680250783, + "14": 0.5015576323987538, + "15": 0.43641618497109824, + "16": 0.33225806451612905, + "17": 0.3265895953757225, + "18": 0.26571428571428574, + "19": 0.16216216216216217, + "20": 0.2132132132132132 + }, + "pass@4": { + "11": 0.7308781869688386, + "12": 0.7090395480225988, + "13": 0.5893416927899686, + "14": 0.5545171339563862, + "15": 0.5, + "16": 0.36774193548387096, + "17": 0.38439306358381503, + "18": 0.31142857142857144, + "19": 0.2072072072072072, + "20": 0.23723723723723725 + }, + "pass@8": { + "11": 0.830028328611898, + "12": 0.7598870056497176, + "13": 0.6332288401253918, + "14": 0.5856697819314641, + "15": 0.5635838150289018, + "16": 0.4032258064516129, + "17": 0.42485549132947975, + "18": 0.34285714285714286, + "19": 0.22822822822822822, + "20": 0.27627627627627627 + }, + "pass@16": { + "11": 0.8640226628895185, + "12": 0.8050847457627118, + "13": 0.6927899686520376, + "14": 0.6292834890965732, + "15": 0.630057803468208, + "16": 0.4612903225806452, + "17": 0.4797687861271676, + "18": 0.38285714285714284, + "19": 0.25225225225225223, + "20": 0.2972972972972973 + }, + "pass@32": { + "11": 0.9065155807365439, + "12": 0.8418079096045198, + "13": 0.7586206896551724, + "14": 0.7009345794392523, + "15": 0.7023121387283237, + "16": 0.5161290322580645, + "17": 0.5057803468208093, + "18": 0.4228571428571429, + "19": 0.3063063063063063, + "20": 0.32732732732732733 + }, + "pass@64": { + "11": 0.9348441926345609, + "12": 0.8531073446327684, + "13": 0.7962382445141066, + "14": 0.7570093457943925, + "15": 0.7601156069364162, + "16": 0.5774193548387097, + "17": 0.5722543352601156, + "18": 0.47714285714285715, + "19": 0.34234234234234234, + "20": 0.35435435435435436 + }, + "pass@128": { + "11": 0.9461756373937678, + "12": 0.8700564971751412, + "13": 0.8150470219435737, + "14": 0.7975077881619937, + "15": 0.8265895953757225, + "16": 0.6129032258064516, + "17": 0.6271676300578035, + "18": 0.5457142857142857, + "19": 0.3993993993993994, + "20": 0.3993993993993994 + } + }, + "total_pass_at_k": { + "pass@1": 0.3515601783060921, + "pass@2": 0.4086181277860327, + "pass@4": 0.4612184249628529, + "pass@8": 0.5072808320950966, + "pass@16": 0.5518573551263002, + "pass@32": 0.600891530460624, + "pass@64": 0.6442793462109956, + "pass@128": 0.6858841010401189 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.580952380952381, + "pass@2": 0.6761904761904762, + "pass@4": 0.8031746031746032, + "pass@8": 0.8888888888888888, + "pass@16": 0.9365079365079365, + "pass@32": 0.9619047619047619, + "pass@64": 0.9777777777777777, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.6238532110091743, + "pass@2": 0.6605504587155964, + "pass@4": 0.7370030581039755, + "pass@8": 0.8103975535168195, + "pass@16": 0.8685015290519877, + "pass@32": 0.8776758409785933, + "pass@64": 0.9051987767584098, + "pass@128": 0.9174311926605505 + }, + "13": { + "pass@1": 0.48148148148148145, + "pass@2": 0.5524691358024691, + "pass@4": 0.6234567901234568, + "pass@8": 0.6574074074074074, + "pass@16": 0.7314814814814815, + "pass@32": 0.7716049382716049, + "pass@64": 0.7870370370370371, + "pass@128": 0.7993827160493827 + }, + "14": { + "pass@1": 0.4332344213649852, + "pass@2": 0.5014836795252225, + "pass@4": 0.5459940652818991, + "pass@8": 0.5964391691394659, + "pass@16": 0.6735905044510386, + "pass@32": 0.7477744807121661, + "pass@64": 0.798219584569733, + "pass@128": 0.8338278931750742 + }, + "15": { + "pass@1": 0.3425925925925926, + "pass@2": 0.4074074074074074, + "pass@4": 0.4660493827160494, + "pass@8": 0.5061728395061729, + "pass@16": 0.595679012345679, + "pass@32": 0.6820987654320988, + "pass@64": 0.7685185185185185, + "pass@128": 0.8055555555555556 + }, + "16": { + "pass@1": 0.2949852507374631, + "pass@2": 0.3805309734513274, + "pass@4": 0.4247787610619469, + "pass@8": 0.471976401179941, + "pass@16": 0.5014749262536873, + "pass@32": 0.5575221238938053, + "pass@64": 0.6047197640117994, + "pass@128": 0.6489675516224189 + }, + "17": { + "pass@1": 0.25, + "pass@2": 0.3042168674698795, + "pass@4": 0.3433734939759036, + "pass@8": 0.4066265060240964, + "pass@16": 0.4427710843373494, + "pass@32": 0.48493975903614456, + "pass@64": 0.5271084337349398, + "pass@128": 0.5783132530120482 + }, + "18": { + "pass@1": 0.22903225806451613, + "pass@2": 0.27419354838709675, + "pass@4": 0.3032258064516129, + "pass@8": 0.33225806451612905, + "pass@16": 0.3741935483870968, + "pass@32": 0.4129032258064516, + "pass@64": 0.45483870967741935, + "pass@128": 0.5032258064516129 + }, + "19": { + "pass@1": 0.11782477341389729, + "pass@2": 0.1419939577039275, + "pass@4": 0.17522658610271905, + "pass@8": 0.2175226586102719, + "pass@16": 0.256797583081571, + "pass@32": 0.2809667673716012, + "pass@64": 0.3081570996978852, + "pass@128": 0.3323262839879154 + }, + "20": { + "pass@1": 0.12612612612612611, + "pass@2": 0.15915915915915915, + "pass@4": 0.1921921921921922, + "pass@8": 0.21921921921921922, + "pass@16": 0.24024024024024024, + "pass@32": 0.2672672672672673, + "pass@64": 0.2912912912912913, + "pass@128": 0.30930930930930933 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.580952380952381, + "12": 0.6238532110091743, + "13": 0.48148148148148145, + "14": 0.4332344213649852, + "15": 0.3425925925925926, + "16": 0.2949852507374631, + "17": 0.25, + "18": 0.22903225806451613, + "19": 0.11782477341389729, + "20": 0.12612612612612611 + }, + "pass@2": { + "11": 0.6761904761904762, + "12": 0.6605504587155964, + "13": 0.5524691358024691, + "14": 0.5014836795252225, + "15": 0.4074074074074074, + "16": 0.3805309734513274, + "17": 0.3042168674698795, + "18": 0.27419354838709675, + "19": 0.1419939577039275, + "20": 0.15915915915915915 + }, + "pass@4": { + "11": 0.8031746031746032, + "12": 0.7370030581039755, + "13": 0.6234567901234568, + "14": 0.5459940652818991, + "15": 0.4660493827160494, + "16": 0.4247787610619469, + "17": 0.3433734939759036, + "18": 0.3032258064516129, + "19": 0.17522658610271905, + "20": 0.1921921921921922 + }, + "pass@8": { + "11": 0.8888888888888888, + "12": 0.8103975535168195, + "13": 0.6574074074074074, + "14": 0.5964391691394659, + "15": 0.5061728395061729, + "16": 0.471976401179941, + "17": 0.4066265060240964, + "18": 0.33225806451612905, + "19": 0.2175226586102719, + "20": 0.21921921921921922 + }, + "pass@16": { + "11": 0.9365079365079365, + "12": 0.8685015290519877, + "13": 0.7314814814814815, + "14": 0.6735905044510386, + "15": 0.595679012345679, + "16": 0.5014749262536873, + "17": 0.4427710843373494, + "18": 0.3741935483870968, + "19": 0.256797583081571, + "20": 0.24024024024024024 + }, + "pass@32": { + "11": 0.9619047619047619, + "12": 0.8776758409785933, + "13": 0.7716049382716049, + "14": 0.7477744807121661, + "15": 0.6820987654320988, + "16": 0.5575221238938053, + "17": 0.48493975903614456, + "18": 0.4129032258064516, + "19": 0.2809667673716012, + "20": 0.2672672672672673 + }, + "pass@64": { + "11": 0.9777777777777777, + "12": 0.9051987767584098, + "13": 0.7870370370370371, + "14": 0.798219584569733, + "15": 0.7685185185185185, + "16": 0.6047197640117994, + "17": 0.5271084337349398, + "18": 0.45483870967741935, + "19": 0.3081570996978852, + "20": 0.2912912912912913 + }, + "pass@128": { + "11": 0.9841269841269841, + "12": 0.9174311926605505, + "13": 0.7993827160493827, + "14": 0.8338278931750742, + "15": 0.8055555555555556, + "16": 0.6489675516224189, + "17": 0.5783132530120482, + "18": 0.5032258064516129, + "19": 0.3323262839879154, + "20": 0.30930930930930933 + } + }, + "total_pass_at_k": { + "pass@1": 0.3468826405867971, + "pass@2": 0.40464547677261614, + "pass@4": 0.45996332518337407, + "pass@8": 0.5091687041564792, + "pass@16": 0.5605134474327629, + "pass@32": 0.6029951100244498, + "pass@64": 0.6408924205378973, + "pass@128": 0.6699266503667481 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6355421686746988, + "pass@2": 0.6897590361445783, + "pass@4": 0.7710843373493976, + "pass@8": 0.8433734939759037, + "pass@16": 0.9126506024096386, + "pass@32": 0.9397590361445783, + "pass@64": 0.9608433734939759, + "pass@128": 0.9668674698795181 + }, + "12": { + "pass@1": 0.5391849529780565, + "pass@2": 0.6018808777429467, + "pass@4": 0.6833855799373041, + "pass@8": 0.7398119122257053, + "pass@16": 0.8181818181818182, + "pass@32": 0.8557993730407524, + "pass@64": 0.8714733542319749, + "pass@128": 0.8840125391849529 + }, + "13": { + "pass@1": 0.49859943977591037, + "pass@2": 0.5770308123249299, + "pass@4": 0.6274509803921569, + "pass@8": 0.7142857142857143, + "pass@16": 0.773109243697479, + "pass@32": 0.8011204481792717, + "pass@64": 0.8151260504201681, + "pass@128": 0.8319327731092437 + }, + "14": { + "pass@1": 0.4649122807017544, + "pass@2": 0.5029239766081871, + "pass@4": 0.5497076023391813, + "pass@8": 0.6286549707602339, + "pass@16": 0.7017543859649122, + "pass@32": 0.7602339181286549, + "pass@64": 0.8245614035087719, + "pass@128": 0.8567251461988304 + }, + "15": { + "pass@1": 0.4121212121212121, + "pass@2": 0.4666666666666667, + "pass@4": 0.5121212121212121, + "pass@8": 0.5606060606060606, + "pass@16": 0.6272727272727273, + "pass@32": 0.706060606060606, + "pass@64": 0.7666666666666667, + "pass@128": 0.8212121212121212 + }, + "16": { + "pass@1": 0.32193732193732194, + "pass@2": 0.396011396011396, + "pass@4": 0.43304843304843305, + "pass@8": 0.48717948717948717, + "pass@16": 0.5299145299145299, + "pass@32": 0.6011396011396012, + "pass@64": 0.6438746438746439, + "pass@128": 0.6723646723646723 + }, + "17": { + "pass@1": 0.2453416149068323, + "pass@2": 0.3105590062111801, + "pass@4": 0.36645962732919257, + "pass@8": 0.40993788819875776, + "pass@16": 0.46273291925465837, + "pass@32": 0.5186335403726708, + "pass@64": 0.593167701863354, + "pass@128": 0.6428571428571429 + }, + "18": { + "pass@1": 0.2411764705882353, + "pass@2": 0.27647058823529413, + "pass@4": 0.31470588235294117, + "pass@8": 0.34705882352941175, + "pass@16": 0.37941176470588234, + "pass@32": 0.4323529411764706, + "pass@64": 0.5029411764705882, + "pass@128": 0.55 + }, + "19": { + "pass@1": 0.12797619047619047, + "pass@2": 0.17261904761904762, + "pass@4": 0.2113095238095238, + "pass@8": 0.23809523809523808, + "pass@16": 0.27380952380952384, + "pass@32": 0.30654761904761907, + "pass@64": 0.35119047619047616, + "pass@128": 0.3869047619047619 + }, + "20": { + "pass@1": 0.15269461077844312, + "pass@2": 0.19760479041916168, + "pass@4": 0.2275449101796407, + "pass@8": 0.26646706586826346, + "pass@16": 0.2904191616766467, + "pass@32": 0.32335329341317365, + "pass@64": 0.3413173652694611, + "pass@128": 0.3652694610778443 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6355421686746988, + "12": 0.5391849529780565, + "13": 0.49859943977591037, + "14": 0.4649122807017544, + "15": 0.4121212121212121, + "16": 0.32193732193732194, + "17": 0.2453416149068323, + "18": 0.2411764705882353, + "19": 0.12797619047619047, + "20": 0.15269461077844312 + }, + "pass@2": { + "11": 0.6897590361445783, + "12": 0.6018808777429467, + "13": 0.5770308123249299, + "14": 0.5029239766081871, + "15": 0.4666666666666667, + "16": 0.396011396011396, + "17": 0.3105590062111801, + "18": 0.27647058823529413, + "19": 0.17261904761904762, + "20": 0.19760479041916168 + }, + "pass@4": { + "11": 0.7710843373493976, + "12": 0.6833855799373041, + "13": 0.6274509803921569, + "14": 0.5497076023391813, + "15": 0.5121212121212121, + "16": 0.43304843304843305, + "17": 0.36645962732919257, + "18": 0.31470588235294117, + "19": 0.2113095238095238, + "20": 0.2275449101796407 + }, + "pass@8": { + "11": 0.8433734939759037, + "12": 0.7398119122257053, + "13": 0.7142857142857143, + "14": 0.6286549707602339, + "15": 0.5606060606060606, + "16": 0.48717948717948717, + "17": 0.40993788819875776, + "18": 0.34705882352941175, + "19": 0.23809523809523808, + "20": 0.26646706586826346 + }, + "pass@16": { + "11": 0.9126506024096386, + "12": 0.8181818181818182, + "13": 0.773109243697479, + "14": 0.7017543859649122, + "15": 0.6272727272727273, + "16": 0.5299145299145299, + "17": 0.46273291925465837, + "18": 0.37941176470588234, + "19": 0.27380952380952384, + "20": 0.2904191616766467 + }, + "pass@32": { + "11": 0.9397590361445783, + "12": 0.8557993730407524, + "13": 0.8011204481792717, + "14": 0.7602339181286549, + "15": 0.706060606060606, + "16": 0.6011396011396012, + "17": 0.5186335403726708, + "18": 0.4323529411764706, + "19": 0.30654761904761907, + "20": 0.32335329341317365 + }, + "pass@64": { + "11": 0.9608433734939759, + "12": 0.8714733542319749, + "13": 0.8151260504201681, + "14": 0.8245614035087719, + "15": 0.7666666666666667, + "16": 0.6438746438746439, + "17": 0.593167701863354, + "18": 0.5029411764705882, + "19": 0.35119047619047616, + "20": 0.3413173652694611 + }, + "pass@128": { + "11": 0.9668674698795181, + "12": 0.8840125391849529, + "13": 0.8319327731092437, + "14": 0.8567251461988304, + "15": 0.8212121212121212, + "16": 0.6723646723646723, + "17": 0.6428571428571429, + "18": 0.55, + "19": 0.3869047619047619, + "20": 0.3652694610778443 + } + }, + "total_pass_at_k": { + "pass@1": 0.3639607493309545, + "pass@2": 0.4192685102586976, + "pass@4": 0.4695212607790663, + "pass@8": 0.5236396074933095, + "pass@16": 0.5768658935474279, + "pass@32": 0.6244424620874219, + "pass@64": 0.6669640202200416, + "pass@128": 0.6975914362176628 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.598, + "pass@2": 0.675, + "pass@4": 0.767, + "pass@8": 0.853, + "pass@16": 0.903, + "pass@32": 0.935, + "pass@64": 0.957, + "pass@128": 0.965 + }, + "12": { + "pass@1": 0.585, + "pass@2": 0.636, + "pass@4": 0.71, + "pass@8": 0.77, + "pass@16": 0.83, + "pass@32": 0.858, + "pass@64": 0.876, + "pass@128": 0.89 + }, + "13": { + "pass@1": 0.487, + "pass@2": 0.553, + "pass@4": 0.614, + "pass@8": 0.67, + "pass@16": 0.734, + "pass@32": 0.778, + "pass@64": 0.8, + "pass@128": 0.816 + }, + "14": { + "pass@1": 0.446, + "pass@2": 0.502, + "pass@4": 0.55, + "pass@8": 0.604, + "pass@16": 0.669, + "pass@32": 0.737, + "pass@64": 0.794, + "pass@128": 0.83 + }, + "15": { + "pass@1": 0.383, + "pass@2": 0.437, + "pass@4": 0.493, + "pass@8": 0.544, + "pass@16": 0.618, + "pass@32": 0.697, + "pass@64": 0.765, + "pass@128": 0.818 + }, + "16": { + "pass@1": 0.294, + "pass@2": 0.371, + "pass@4": 0.41, + "pass@8": 0.456, + "pass@16": 0.499, + "pass@32": 0.56, + "pass@64": 0.61, + "pass@128": 0.646 + }, + "17": { + "pass@1": 0.249, + "pass@2": 0.314, + "pass@4": 0.365, + "pass@8": 0.414, + "pass@16": 0.462, + "pass@32": 0.503, + "pass@64": 0.564, + "pass@128": 0.616 + }, + "18": { + "pass@1": 0.229, + "pass@2": 0.272, + "pass@4": 0.31, + "pass@8": 0.341, + "pass@16": 0.379, + "pass@32": 0.423, + "pass@64": 0.479, + "pass@128": 0.534 + }, + "19": { + "pass@1": 0.123, + "pass@2": 0.159, + "pass@4": 0.198, + "pass@8": 0.228, + "pass@16": 0.261, + "pass@32": 0.298, + "pass@64": 0.334, + "pass@128": 0.373 + }, + "20": { + "pass@1": 0.148, + "pass@2": 0.19, + "pass@4": 0.219, + "pass@8": 0.254, + "pass@16": 0.276, + "pass@32": 0.306, + "pass@64": 0.329, + "pass@128": 0.358 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.598, + "12": 0.585, + "13": 0.487, + "14": 0.446, + "15": 0.383, + "16": 0.294, + "17": 0.249, + "18": 0.229, + "19": 0.123, + "20": 0.148 + }, + "pass@2": { + "11": 0.675, + "12": 0.636, + "13": 0.553, + "14": 0.502, + "15": 0.437, + "16": 0.371, + "17": 0.314, + "18": 0.272, + "19": 0.159, + "20": 0.19 + }, + "pass@4": { + "11": 0.767, + "12": 0.71, + "13": 0.614, + "14": 0.55, + "15": 0.493, + "16": 0.41, + "17": 0.365, + "18": 0.31, + "19": 0.198, + "20": 0.219 + }, + "pass@8": { + "11": 0.853, + "12": 0.77, + "13": 0.67, + "14": 0.604, + "15": 0.544, + "16": 0.456, + "17": 0.414, + "18": 0.341, + "19": 0.228, + "20": 0.254 + }, + "pass@16": { + "11": 0.903, + "12": 0.83, + "13": 0.734, + "14": 0.669, + "15": 0.618, + "16": 0.499, + "17": 0.462, + "18": 0.379, + "19": 0.261, + "20": 0.276 + }, + "pass@32": { + "11": 0.935, + "12": 0.858, + "13": 0.778, + "14": 0.737, + "15": 0.697, + "16": 0.56, + "17": 0.503, + "18": 0.423, + "19": 0.298, + "20": 0.306 + }, + "pass@64": { + "11": 0.957, + "12": 0.876, + "13": 0.8, + "14": 0.794, + "15": 0.765, + "16": 0.61, + "17": 0.564, + "18": 0.479, + "19": 0.334, + "20": 0.329 + }, + "pass@128": { + "11": 0.965, + "12": 0.89, + "13": 0.816, + "14": 0.83, + "15": 0.818, + "16": 0.646, + "17": 0.616, + "18": 0.534, + "19": 0.373, + "20": 0.358 + } + }, + "total_pass_at_k": { + "pass@1": 0.3542, + "pass@2": 0.4109, + "pass@4": 0.4636, + "pass@8": 0.5134, + "pass@16": 0.5631, + "pass@32": 0.6095, + "pass@64": 0.6508, + "pass@128": 0.6846 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/hf_model_metrics.json b/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..be09cd714549b81e9dfd647ee3fa321479b3b719 --- /dev/null +++ b/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/hf_model_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.99325, + "avg_response_len": 199.50800086805555, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.99325, + "pass@2": 0.9954187445319325, + "pass@4": 0.9966956619797519, + "pass@8": 0.9975696547937937, + "pass@16": 0.9982506538665, + "pass@32": 0.9988334672854507, + "pass@64": 0.9992864592247551, + "pass@128": 0.9995555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9552421875, + "pass@2": 0.9663871801181106, + "pass@4": 0.9742945933633294, + "pass@8": 0.9803053180219854, + "pass@16": 0.9853246417483904, + "pass@32": 0.9899114691529983, + "pass@64": 0.9936407902121531, + "pass@128": 0.996 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999296875, + "pass@2": 0.9999974163385827, + "pass@4": 0.9999999967191601, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997734375, + "pass@2": 0.999956815944882, + "pass@4": 0.9999983548931384, + "pass@8": 0.999999998447177, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9965859375, + "pass@2": 0.9985543799212597, + "pass@4": 0.9995019312898388, + "pass@8": 0.999874220375141, + "pass@16": 0.9999878760406309, + "pass@32": 0.9999999341940324, + "pass@64": 0.9999999999999704, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999359375, + "pass@2": 0.999857283464567, + "pass@4": 0.9999880380577427, + "pass@8": 0.99999989854835, + "pass@16": 0.999999999997159, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9983046875, + "pass@2": 0.9994753937007874, + "pass@4": 0.9999047407199099, + "pass@8": 0.9999959162473189, + "pass@16": 0.9999999929462106, + "pass@32": 0.9999999999999974, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9946875, + "pass@2": 0.9970856299212598, + "pass@4": 0.9983911218597675, + "pass@8": 0.9993263139458172, + "pass@16": 0.9998583304665516, + "pass@32": 0.9999939443218611, + "pass@64": 0.999999997730069, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9953671875, + "pass@2": 0.9974546013779528, + "pass@4": 0.9981821809148856, + "pass@8": 0.9986252275583559, + "pass@16": 0.9990850435995567, + "pass@32": 0.9995958579001671, + "pass@64": 0.9999373450806051, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9552421875, + "2": 1.0, + "3": 0.9999296875, + "4": 0.9997734375, + "5": 0.9965859375, + "6": 0.999359375, + "7": 0.9983046875, + "8": 0.9946875, + "9": 0.9953671875 + }, + "per_op_avg_response_len": { + "10": 280.122890625, + "2": 115.4205078125, + "3": 144.5612890625, + "4": 159.5238828125, + "5": 172.4870625, + "6": 198.431640625, + "7": 223.4593671875, + "8": 242.9427578125, + "9": 258.622609375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.122890625, + "2": 115.4205078125, + "3": 144.5612890625, + "4": 159.5238828125, + "5": 172.4870625, + "6": 198.431640625, + "7": 223.4593671875, + "8": 242.9427578125, + "9": 258.622609375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376863, + "answer_accuracy": 0.9943404888551165, + "avg_response_len": 189.97565750591016, + "resp_tokens_sum": 72002294, + "pass_at_k": { + "pass@1": 0.9943404888551165, + "pass@2": 0.9962871579217494, + "pass@4": 0.9973418919456246, + "pass@8": 0.9980631263546405, + "pass@16": 0.9986508555329557, + "pass@32": 0.9991843205325543, + "pass@64": 0.9996166649000169, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9615597345132744, + "pass@2": 0.9727621333008151, + "pass@4": 0.9804466844852356, + "pass@8": 0.9857071172020305, + "pass@16": 0.9899689862028797, + "pass@32": 0.9937941997890816, + "pass@64": 0.9968274449088425, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99931640625, + "pass@2": 0.9998650498277559, + "pass@4": 0.9999948590410573, + "pass@8": 0.9999999951474285, + "pass@16": 0.9999999999999997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996453373015873, + "pass@2": 0.9979561148606424, + "pass@4": 0.9988745466340517, + "pass@8": 0.999619849729349, + "pass@16": 0.9999615508301392, + "pass@32": 0.9999997910921954, + "pass@64": 0.9999999999999056, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9982433431952663, + "pass@2": 0.9996964252434422, + "pass@4": 0.9999810179977504, + "pass@8": 0.9999999200555205, + "pass@16": 0.999999999999592, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9984224759615384, + "pass@2": 0.9999282316272965, + "pass@4": 0.9999997100722985, + "pass@8": 0.9999999999970139, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.996368838028169, + "pass@2": 0.9973293501164467, + "pass@4": 0.9975227754981333, + "pass@8": 0.9978309304643527, + "pass@16": 0.9983600236590144, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387563, + "answer_accuracy": 0.9920825483289646, + "avg_response_len": 197.48116245494757, + "resp_tokens_sum": 77147201, + "pass_at_k": { + "pass@1": 0.9920825483289646, + "pass@2": 0.9942498497177532, + "pass@4": 0.9956616111571404, + "pass@8": 0.996733950227735, + "pass@16": 0.9976424292146799, + "pass@32": 0.9984078129244492, + "pass@64": 0.9989045403750856, + "pass@128": 0.9990170380078637 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9413822852760736, + "pass@2": 0.9538937581517802, + "pass@4": 0.9636568040191295, + "pass@8": 0.9718906525426227, + "pass@16": 0.979069899257666, + "pass@32": 0.9853812376219401, + "pass@64": 0.9897538439137533, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999705615942029, + "pass@2": 0.9999932243523908, + "pass@4": 0.9999999983697689, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991717406876791, + "pass@2": 0.999909400875392, + "pass@4": 0.9999988719175146, + "pass@8": 0.9999999999254859, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9971098265895953, + "pass@2": 0.998783552751354, + "pass@4": 0.9997432275445338, + "pass@8": 0.9999882753426383, + "pass@16": 0.9999999796137242, + "pass@32": 0.9999999999999929, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.994806338028169, + "pass@2": 0.9972243401353, + "pass@4": 0.9985757458134635, + "pass@8": 0.9994029807049142, + "pass@16": 0.9998887166365205, + "pass@32": 0.9999974897170781, + "pass@64": 0.9999999999243792, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9945967440119761, + "pass@2": 0.9967706074072328, + "pass@4": 0.997610679372414, + "pass@8": 0.9982385089022755, + "pass@16": 0.9990042498589532, + "pass@32": 0.9997223135661869, + "pass@64": 0.999990730945279, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379798, + "answer_accuracy": 0.9933618597254771, + "avg_response_len": 211.02831540843655, + "resp_tokens_sum": 80683722, + "pass_at_k": { + "pass@1": 0.9933618597254771, + "pass@2": 0.9957522212395445, + "pass@4": 0.997111609814257, + "pass@8": 0.9979343689029155, + "pass@16": 0.9984753959163758, + "pass@32": 0.9989205850106277, + "pass@64": 0.9993493575591181, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9623367537313433, + "pass@2": 0.972093885885533, + "pass@4": 0.978421042425667, + "pass@8": 0.983027619586872, + "pass@16": 0.9867115470675084, + "pass@32": 0.9903909013723802, + "pass@64": 0.9941986067826024, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998177842565598, + "pass@2": 0.9999924674594246, + "pass@4": 0.9999999904348691, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9935431985294118, + "pass@2": 0.9976486509958314, + "pass@4": 0.9995777931251238, + "pass@8": 0.9999822579717529, + "pass@16": 0.9999999633209915, + "pass@32": 0.9999999999999732, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9989402602523659, + "pass@2": 0.9996495348866092, + "pass@4": 0.9999635071247008, + "pass@8": 0.9999996800452854, + "pass@16": 0.9999999999910381, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9996785996835443, + "pass@2": 0.9999964959384032, + "pass@4": 0.99999999970336, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9910613738738738, + "pass@2": 0.9942744171336693, + "pass@4": 0.9966871548151077, + "pass@8": 0.9986133807702835, + "pass@16": 0.999693201383144, + "pass@32": 0.9999844909078028, + "pass@64": 0.999999993264007, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9950512459807074, + "pass@2": 0.9983321518089983, + "pass@4": 0.9995486453461805, + "pass@8": 0.9999472195181695, + "pass@16": 0.9999994075489268, + "pass@32": 0.9999999999829826, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.6697265625, + "avg_response_len": 373.5406, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.6697265625, + "pass@2": 0.7438719119094395, + "pass@4": 0.7987249444975563, + "pass@8": 0.8387653816413493, + "pass@16": 0.8677611874341606, + "pass@32": 0.8895298282765596, + "pass@64": 0.9064768389238952, + "pass@128": 0.9202 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9481015625, + "pass@2": 0.9605262057086615, + "pass@4": 0.9688533562992127, + "pass@8": 0.9748427569190922, + "pass@16": 0.9790820613125144, + "pass@32": 0.9824731447427597, + "pass@64": 0.9854467859862046, + "pass@128": 0.989 + }, + "12": { + "pass@1": 0.895625, + "pass@2": 0.9198941929133861, + "pass@4": 0.9340361457630291, + "pass@8": 0.9439423079108554, + "pass@16": 0.9513297574307273, + "pass@32": 0.9579614693787893, + "pass@64": 0.9645552618183556, + "pass@128": 0.969 + }, + "13": { + "pass@1": 0.7983203125, + "pass@2": 0.8467202263779529, + "pass@4": 0.8772908443007121, + "pass@8": 0.8986300217806333, + "pass@16": 0.9146204534384816, + "pass@32": 0.927608372739694, + "pass@64": 0.9391455705545309, + "pass@128": 0.948 + }, + "14": { + "pass@1": 0.81715625, + "pass@2": 0.8702081692913392, + "pass@4": 0.9016159673790778, + "pass@8": 0.9205296499527171, + "pass@16": 0.9330841704378324, + "pass@32": 0.9436380650641849, + "pass@64": 0.9526898770949507, + "pass@128": 0.959 + }, + "15": { + "pass@1": 0.7866328125, + "pass@2": 0.8491219242125994, + "pass@4": 0.8887512251593543, + "pass@8": 0.9160354515018363, + "pass@16": 0.934821020226888, + "pass@32": 0.9477046447229035, + "pass@64": 0.9568601061256404, + "pass@128": 0.964 + }, + "16": { + "pass@1": 0.6070625, + "pass@2": 0.7074254429133862, + "pass@4": 0.7787150926134232, + "pass@8": 0.8298943330312608, + "pass@16": 0.8670571480724663, + "pass@32": 0.8944302250263362, + "pass@64": 0.9145507940699726, + "pass@128": 0.929 + }, + "17": { + "pass@1": 0.5728203125, + "pass@2": 0.6743172982283463, + "pass@4": 0.7482087569366332, + "pass@8": 0.798978663640173, + "pass@16": 0.8335803442405866, + "pass@32": 0.8595010399973773, + "pass@64": 0.8810828668926628, + "pass@128": 0.9 + }, + "18": { + "pass@1": 0.5366796875, + "pass@2": 0.6374736712598427, + "pass@4": 0.7099372144731906, + "pass@8": 0.7635124180379688, + "pass@16": 0.8049621574437177, + "pass@32": 0.8368488830785117, + "pass@64": 0.8601014204798583, + "pass@128": 0.879 + }, + "19": { + "pass@1": 0.3674296875, + "pass@2": 0.48628850885826785, + "pass@4": 0.5898773649231349, + "pass@8": 0.6715228621949131, + "pass@16": 0.7321707789248942, + "pass@32": 0.7765155934680733, + "pass@64": 0.8089260929574748, + "pass@128": 0.833 + }, + "20": { + "pass@1": 0.3674375, + "pass@2": 0.4867434793307089, + "pass@4": 0.58996347712786, + "pass@8": 0.6697653514440781, + "pass@16": 0.7269039828133871, + "pass@32": 0.7686168445469246, + "pass@64": 0.8014096132592516, + "pass@128": 0.832 + } + }, + "per_op_accuracy": { + "11": 0.9481015625, + "12": 0.895625, + "13": 0.7983203125, + "14": 0.81715625, + "15": 0.7866328125, + "16": 0.6070625, + "17": 0.5728203125, + "18": 0.5366796875, + "19": 0.3674296875, + "20": 0.3674375 + }, + "per_op_avg_response_len": { + "11": 297.879796875, + "12": 308.1949140625, + "13": 326.3543671875, + "14": 354.147171875, + "15": 373.895859375, + "16": 380.260359375, + "17": 404.2368984375, + "18": 419.33965625, + "19": 424.981359375, + "20": 446.1156171875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.879796875, + "12": 308.1949140625, + "13": 326.3543671875, + "14": 354.147171875, + "15": 373.895859375, + "16": 380.260359375, + "17": 404.2368984375, + "18": 419.33965625, + "19": 424.981359375, + "20": 446.1156171875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 288609, + "answer_accuracy": 0.6700617570579495, + "avg_response_len": 355.29167672734025, + "resp_tokens_sum": 153031231, + "pass_at_k": { + "pass@1": 0.6700617570579495, + "pass@2": 0.743270634191719, + "pass@4": 0.796685554610284, + "pass@8": 0.835245048744268, + "pass@16": 0.8632444331367674, + "pass@32": 0.8842794450752608, + "pass@64": 0.9005907241329918, + "pass@128": 0.9144130757800891 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9355524079320113, + "pass@2": 0.9494138403113916, + "pass@4": 0.9587174374980745, + "pass@8": 0.9654747650936155, + "pass@16": 0.9708917986847155, + "pass@32": 0.9757436548795662, + "pass@64": 0.9793999690641447, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.8806497175141242, + "pass@2": 0.9054259670136573, + "pass@4": 0.9203959054906274, + "pass@8": 0.9309233567968177, + "pass@16": 0.9386696115032847, + "pass@32": 0.9454865056893909, + "pass@64": 0.9526492494418823, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8020914968652038, + "pass@2": 0.8500751302051195, + "pass@4": 0.8829129885410095, + "pass@8": 0.9072514585026777, + "pass@16": 0.9247382338224696, + "pass@32": 0.9368263153026625, + "pass@64": 0.9448406529194303, + "pass@128": 0.9498432601880877 + }, + "14": { + "pass@1": 0.8056853582554517, + "pass@2": 0.8563524879191502, + "pass@4": 0.8858799866722263, + "pass@8": 0.9025076053682433, + "pass@16": 0.913900500083965, + "pass@32": 0.925047708329376, + "pass@64": 0.9367271813806438, + "pass@128": 0.9470404984423676 + }, + "15": { + "pass@1": 0.7983878251445087, + "pass@2": 0.859632797209959, + "pass@4": 0.8984884428283165, + "pass@8": 0.9249886345537007, + "pass@16": 0.9410397807831975, + "pass@32": 0.949840483402603, + "pass@64": 0.956831430393292, + "pass@128": 0.9653179190751445 + }, + "16": { + "pass@1": 0.6071824596774194, + "pass@2": 0.7037088042926081, + "pass@4": 0.768007695308247, + "pass@8": 0.8124516737316596, + "pass@16": 0.8460015212662262, + "pass@32": 0.8723927091205886, + "pass@64": 0.8951103234664651, + "pass@128": 0.9161290322580645 + }, + "17": { + "pass@1": 0.5784185332369942, + "pass@2": 0.6753856646260984, + "pass@4": 0.7448697205840593, + "pass@8": 0.7936231621801595, + "pass@16": 0.827266712612159, + "pass@32": 0.8520895959798461, + "pass@64": 0.8715641230767441, + "pass@128": 0.8872832369942196 + }, + "18": { + "pass@1": 0.5097991071428571, + "pass@2": 0.6145180680539934, + "pass@4": 0.6914994332851245, + "pass@8": 0.74884747793879, + "pass@16": 0.7939464107956918, + "pass@32": 0.8293121568507497, + "pass@64": 0.855726650165124, + "pass@128": 0.8771428571428571 + }, + "19": { + "pass@1": 0.3723958333333333, + "pass@2": 0.49274779208342234, + "pass@4": 0.5936755563212254, + "pass@8": 0.6701203130156419, + "pass@16": 0.7265665300507005, + "pass@32": 0.7676342272925999, + "pass@64": 0.7945744080866722, + "pass@128": 0.8108108108108109 + }, + "20": { + "pass@1": 0.39407376126126126, + "pass@2": 0.5133520577664276, + "pass@4": 0.6131522349120772, + "pass@8": 0.6888155856314068, + "pass@16": 0.743326415580538, + "pass@32": 0.7834285004700964, + "pass@64": 0.8144789867720131, + "pass@128": 0.8468468468468469 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 293986, + "answer_accuracy": 0.6829514198632174, + "avg_response_len": 368.8295862139459, + "resp_tokens_sum": 158767859, + "pass_at_k": { + "pass@1": 0.6829514198632174, + "pass@2": 0.7556454006195314, + "pass@4": 0.8087300251328982, + "pass@8": 0.8479436632699104, + "pass@16": 0.8768762077279405, + "pass@32": 0.8988522042405073, + "pass@64": 0.9157729162613696, + "pass@128": 0.9283377936366339 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9588667168674698, + "pass@2": 0.9688070688264869, + "pass@4": 0.9738426495633828, + "pass@8": 0.9767944886728196, + "pass@16": 0.9796148083018154, + "pass@32": 0.982822830027572, + "pass@64": 0.9855982198956623, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8903800940438872, + "pass@2": 0.9155861235899592, + "pass@4": 0.9309091514422766, + "pass@8": 0.9412217906007945, + "pass@16": 0.9481690271077153, + "pass@32": 0.9544341226634312, + "pass@64": 0.9611133242427224, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.8157169117647058, + "pass@2": 0.8605739539910452, + "pass@4": 0.8872929843853552, + "pass@8": 0.905276867908199, + "pass@16": 0.9190960959975872, + "pass@32": 0.9314763224458966, + "pass@64": 0.9439972488605914, + "pass@128": 0.9551820728291317 + }, + "14": { + "pass@1": 0.8373309576023392, + "pass@2": 0.8934615594925635, + "pass@4": 0.9265919024376331, + "pass@8": 0.946333115242231, + "pass@16": 0.9589428298211253, + "pass@32": 0.9680942410488826, + "pass@64": 0.9739592473038264, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.7923532196969697, + "pass@2": 0.8550170752803624, + "pass@4": 0.8956887607799019, + "pass@8": 0.9255279916761899, + "pass@16": 0.9489574497373325, + "pass@32": 0.9662421289695058, + "pass@64": 0.9762515985143063, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.6154736467236467, + "pass@2": 0.7142381441101907, + "pass@4": 0.7865660093770329, + "pass@8": 0.8398169209514765, + "pass@16": 0.878125499005597, + "pass@32": 0.9055294390113082, + "pass@64": 0.9247983094349499, + "pass@128": 0.9373219373219374 + }, + "17": { + "pass@1": 0.6003493788819876, + "pass@2": 0.7049765858072087, + "pass@4": 0.7759227191903806, + "pass@8": 0.8225132499570565, + "pass@16": 0.8555351979846405, + "pass@32": 0.8822355510888512, + "pass@64": 0.9047357050959228, + "pass@128": 0.9192546583850931 + }, + "18": { + "pass@1": 0.5742876838235295, + "pass@2": 0.6695149664196381, + "pass@4": 0.7379379058316233, + "pass@8": 0.7895887044016283, + "pass@16": 0.8302944179161645, + "pass@32": 0.8623677634837246, + "pass@64": 0.8845639909682449, + "pass@128": 0.9 + }, + "19": { + "pass@1": 0.37486049107142855, + "pass@2": 0.48710293049306325, + "pass@4": 0.5855916665550731, + "pass@8": 0.6672813754848781, + "pass@16": 0.7282600529581397, + "pass@32": 0.7711494737197626, + "pass@64": 0.804945509850621, + "pass@128": 0.8333333333333334 + }, + "20": { + "pass@1": 0.37359655688622756, + "pass@2": 0.4897408676741004, + "pass@4": 0.5887979210308285, + "pass@8": 0.6662973240324496, + "pass@16": 0.7224702823795047, + "pass@32": 0.7645544154261024, + "pass@64": 0.797975742891563, + "pass@128": 0.8263473053892215 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 274655, + "answer_accuracy": 0.6557891771088019, + "avg_response_len": 397.15024736399755, + "resp_tokens_sum": 166332878, + "pass_at_k": { + "pass@1": 0.6557891771088019, + "pass@2": 0.7323893498642778, + "pass@4": 0.7905389606326764, + "pass@8": 0.8329522273265275, + "pass@16": 0.8630377965303354, + "pass@32": 0.8853477956071074, + "pass@64": 0.9029756372690593, + "pass@128": 0.9177872860635696 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9508184523809524, + "pass@2": 0.9642513435820521, + "pass@4": 0.9749534641503145, + "pass@8": 0.9832837923862544, + "pass@16": 0.9876988572082758, + "pass@32": 0.9896458888924424, + "pass@64": 0.9920634536546087, + "pass@128": 0.9968253968253968 + }, + "12": { + "pass@1": 0.9169533639143731, + "pass@2": 0.9397597161020009, + "pass@4": 0.9518531373371902, + "pass@8": 0.9606901786028378, + "pass@16": 0.9681186400954256, + "pass@32": 0.9749075267128768, + "pass@64": 0.9808020705886862, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.775438850308642, + "pass@2": 0.8281523403324585, + "pass@4": 0.8607345849708593, + "pass@8": 0.8828177922069514, + "pass@16": 0.8997273165986912, + "pass@32": 0.9142707748301222, + "pass@64": 0.9281925445370405, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.8076084940652819, + "pass@2": 0.8598076181686491, + "pass@4": 0.891258341316386, + "pass@8": 0.9115097424826967, + "pass@16": 0.9251147243681173, + "pass@32": 0.9365267663256304, + "pass@64": 0.9463097605158902, + "pass@128": 0.9525222551928784 + }, + "15": { + "pass@1": 0.7682532793209876, + "pass@2": 0.8318930294303489, + "pass@4": 0.8712868298870047, + "pass@8": 0.8968060083120799, + "pass@16": 0.9137817829092058, + "pass@32": 0.9265430089681044, + "pass@64": 0.9371401348760496, + "pass@128": 0.9444444444444444 + }, + "16": { + "pass@1": 0.5982439159292036, + "pass@2": 0.703770280235988, + "pass@4": 0.7803776925561295, + "pass@8": 0.8355710174645408, + "pass@16": 0.8748514051002123, + "pass@32": 0.9030904191916943, + "pass@64": 0.9217179562941028, + "pass@128": 0.9321533923303835 + }, + "17": { + "pass@1": 0.5402861445783133, + "pass@2": 0.6434680651740823, + "pass@4": 0.724809391672426, + "pass@8": 0.7817342862640553, + "pass@16": 0.8188666503786926, + "pass@32": 0.8451752781257261, + "pass@64": 0.8680626002024731, + "pass@128": 0.8945783132530121 + }, + "18": { + "pass@1": 0.52578125, + "pass@2": 0.6282492221234439, + "pass@4": 0.700043628518209, + "pass@8": 0.7514698105252829, + "pass@16": 0.7896155212055802, + "pass@32": 0.8173696406331719, + "pass@64": 0.8382107612672934, + "pass@128": 0.8580645161290322 + }, + "19": { + "pass@1": 0.3548904833836858, + "pass@2": 0.478963472655042, + "pass@4": 0.5904066607421803, + "pass@8": 0.6772394434978418, + "pass@16": 0.7417786907071799, + "pass@32": 0.7908978024465173, + "pass@64": 0.8274052077184393, + "pass@128": 0.8549848942598187 + }, + "20": { + "pass@1": 0.3346236861861862, + "pass@2": 0.4571285113854011, + "pass@4": 0.56794377560913, + "pass@8": 0.6541935591651141, + "pass@16": 0.7149285648958363, + "pass@32": 0.7578798172314243, + "pass@64": 0.7917844220372017, + "pass@128": 0.8228228228228228 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.8229745065789473, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8229745065789473, + "pass@2": 0.8630256747306098, + "pass@4": 0.8925005475154291, + "pass@8": 0.9139884583977791, + "pass@16": 0.9295719873231518, + "pass@32": 0.9413052362281488, + "pass@64": 0.9504392906453538, + "pass@128": 0.9577894736842105 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9552421875, + "pass@2": 0.9663871801181106, + "pass@4": 0.9742945933633294, + "pass@8": 0.9803053180219854, + "pass@16": 0.9853246417483904, + "pass@32": 0.9899114691529983, + "pass@64": 0.9936407902121531, + "pass@128": 0.996 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999296875, + "pass@2": 0.9999974163385827, + "pass@4": 0.9999999967191601, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997734375, + "pass@2": 0.999956815944882, + "pass@4": 0.9999983548931384, + "pass@8": 0.999999998447177, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9965859375, + "pass@2": 0.9985543799212597, + "pass@4": 0.9995019312898388, + "pass@8": 0.999874220375141, + "pass@16": 0.9999878760406309, + "pass@32": 0.9999999341940324, + "pass@64": 0.9999999999999704, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999359375, + "pass@2": 0.999857283464567, + "pass@4": 0.9999880380577427, + "pass@8": 0.99999989854835, + "pass@16": 0.999999999997159, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9983046875, + "pass@2": 0.9994753937007874, + "pass@4": 0.9999047407199099, + "pass@8": 0.9999959162473189, + "pass@16": 0.9999999929462106, + "pass@32": 0.9999999999999974, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9946875, + "pass@2": 0.9970856299212598, + "pass@4": 0.9983911218597675, + "pass@8": 0.9993263139458172, + "pass@16": 0.9998583304665516, + "pass@32": 0.9999939443218611, + "pass@64": 0.999999997730069, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9953671875, + "pass@2": 0.9974546013779528, + "pass@4": 0.9981821809148856, + "pass@8": 0.9986252275583559, + "pass@16": 0.9990850435995567, + "pass@32": 0.9995958579001671, + "pass@64": 0.9999373450806051, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9481015625, + "pass@2": 0.9605262057086615, + "pass@4": 0.9688533562992127, + "pass@8": 0.9748427569190922, + "pass@16": 0.9790820613125144, + "pass@32": 0.9824731447427597, + "pass@64": 0.9854467859862046, + "pass@128": 0.989 + }, + "12": { + "pass@1": 0.895625, + "pass@2": 0.9198941929133861, + "pass@4": 0.9340361457630291, + "pass@8": 0.9439423079108554, + "pass@16": 0.9513297574307273, + "pass@32": 0.9579614693787893, + "pass@64": 0.9645552618183556, + "pass@128": 0.969 + }, + "13": { + "pass@1": 0.7983203125, + "pass@2": 0.8467202263779529, + "pass@4": 0.8772908443007121, + "pass@8": 0.8986300217806333, + "pass@16": 0.9146204534384816, + "pass@32": 0.927608372739694, + "pass@64": 0.9391455705545309, + "pass@128": 0.948 + }, + "14": { + "pass@1": 0.81715625, + "pass@2": 0.8702081692913392, + "pass@4": 0.9016159673790778, + "pass@8": 0.9205296499527171, + "pass@16": 0.9330841704378324, + "pass@32": 0.9436380650641849, + "pass@64": 0.9526898770949507, + "pass@128": 0.959 + }, + "15": { + "pass@1": 0.7866328125, + "pass@2": 0.8491219242125994, + "pass@4": 0.8887512251593543, + "pass@8": 0.9160354515018363, + "pass@16": 0.934821020226888, + "pass@32": 0.9477046447229035, + "pass@64": 0.9568601061256404, + "pass@128": 0.964 + }, + "16": { + "pass@1": 0.6070625, + "pass@2": 0.7074254429133862, + "pass@4": 0.7787150926134232, + "pass@8": 0.8298943330312608, + "pass@16": 0.8670571480724663, + "pass@32": 0.8944302250263362, + "pass@64": 0.9145507940699726, + "pass@128": 0.929 + }, + "17": { + "pass@1": 0.5728203125, + "pass@2": 0.6743172982283463, + "pass@4": 0.7482087569366332, + "pass@8": 0.798978663640173, + "pass@16": 0.8335803442405866, + "pass@32": 0.8595010399973773, + "pass@64": 0.8810828668926628, + "pass@128": 0.9 + }, + "18": { + "pass@1": 0.5366796875, + "pass@2": 0.6374736712598427, + "pass@4": 0.7099372144731906, + "pass@8": 0.7635124180379688, + "pass@16": 0.8049621574437177, + "pass@32": 0.8368488830785117, + "pass@64": 0.8601014204798583, + "pass@128": 0.879 + }, + "19": { + "pass@1": 0.3674296875, + "pass@2": 0.48628850885826785, + "pass@4": 0.5898773649231349, + "pass@8": 0.6715228621949131, + "pass@16": 0.7321707789248942, + "pass@32": 0.7765155934680733, + "pass@64": 0.8089260929574748, + "pass@128": 0.833 + }, + "20": { + "pass@1": 0.3674375, + "pass@2": 0.4867434793307089, + "pass@4": 0.58996347712786, + "pass@8": 0.6697653514440781, + "pass@16": 0.7269039828133871, + "pass@32": 0.7686168445469246, + "pass@64": 0.8014096132592516, + "pass@128": 0.832 + } + }, + "per_op_accuracy": { + "10": 0.9552421875, + "2": 1.0, + "3": 0.9999296875, + "4": 0.9997734375, + "5": 0.9965859375, + "6": 0.999359375, + "7": 0.9983046875, + "8": 0.9946875, + "9": 0.9953671875, + "11": 0.9481015625, + "12": 0.895625, + "13": 0.7983203125, + "14": 0.81715625, + "15": 0.7866328125, + "16": 0.6070625, + "17": 0.5728203125, + "18": 0.5366796875, + "19": 0.3674296875, + "20": 0.3674375 + }, + "per_op_avg_response_len": { + "10": 280.122890625, + "2": 115.4205078125, + "3": 144.5612890625, + "4": 159.5238828125, + "5": 172.4870625, + "6": 198.431640625, + "7": 223.4593671875, + "8": 242.9427578125, + "9": 258.622609375, + "11": 297.879796875, + "12": 308.1949140625, + "13": 326.3543671875, + "14": 354.147171875, + "15": 373.895859375, + "16": 380.260359375, + "17": 404.2368984375, + "18": 419.33965625, + "19": 424.981359375, + "20": 446.1156171875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.122890625, + "2": 115.4205078125, + "3": 144.5612890625, + "4": 159.5238828125, + "5": 172.4870625, + "6": 198.431640625, + "7": 223.4593671875, + "8": 242.9427578125, + "9": 258.622609375, + "11": 297.879796875, + "12": 308.1949140625, + "13": 326.3543671875, + "14": 354.147171875, + "15": 373.895859375, + "16": 380.260359375, + "17": 404.2368984375, + "18": 419.33965625, + "19": 424.981359375, + "20": 446.1156171875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 665472, + "answer_accuracy": 0.8218463484034145, + "avg_response_len": 277.9124903671356, + "resp_tokens_sum": 225033525, + "pass_at_k": { + "pass@1": 0.8218463484034145, + "pass@2": 0.8616996456941791, + "pass@4": 0.8906064232239269, + "pass@8": 0.9114550278470633, + "pass@16": 0.9266238856684035, + "pass@32": 0.9380627735970853, + "pass@64": 0.9469416268536985, + "pass@128": 0.9544736010116978 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9615597345132744, + "pass@2": 0.9727621333008151, + "pass@4": 0.9804466844852356, + "pass@8": 0.9857071172020305, + "pass@16": 0.9899689862028797, + "pass@32": 0.9937941997890816, + "pass@64": 0.9968274449088425, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99931640625, + "pass@2": 0.9998650498277559, + "pass@4": 0.9999948590410573, + "pass@8": 0.9999999951474285, + "pass@16": 0.9999999999999997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996453373015873, + "pass@2": 0.9979561148606424, + "pass@4": 0.9988745466340517, + "pass@8": 0.999619849729349, + "pass@16": 0.9999615508301392, + "pass@32": 0.9999997910921954, + "pass@64": 0.9999999999999056, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9982433431952663, + "pass@2": 0.9996964252434422, + "pass@4": 0.9999810179977504, + "pass@8": 0.9999999200555205, + "pass@16": 0.999999999999592, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9984224759615384, + "pass@2": 0.9999282316272965, + "pass@4": 0.9999997100722985, + "pass@8": 0.9999999999970139, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.996368838028169, + "pass@2": 0.9973293501164467, + "pass@4": 0.9975227754981333, + "pass@8": 0.9978309304643527, + "pass@16": 0.9983600236590144, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9355524079320113, + "pass@2": 0.9494138403113916, + "pass@4": 0.9587174374980745, + "pass@8": 0.9654747650936155, + "pass@16": 0.9708917986847155, + "pass@32": 0.9757436548795662, + "pass@64": 0.9793999690641447, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.8806497175141242, + "pass@2": 0.9054259670136573, + "pass@4": 0.9203959054906274, + "pass@8": 0.9309233567968177, + "pass@16": 0.9386696115032847, + "pass@32": 0.9454865056893909, + "pass@64": 0.9526492494418823, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8020914968652038, + "pass@2": 0.8500751302051195, + "pass@4": 0.8829129885410095, + "pass@8": 0.9072514585026777, + "pass@16": 0.9247382338224696, + "pass@32": 0.9368263153026625, + "pass@64": 0.9448406529194303, + "pass@128": 0.9498432601880877 + }, + "14": { + "pass@1": 0.8056853582554517, + "pass@2": 0.8563524879191502, + "pass@4": 0.8858799866722263, + "pass@8": 0.9025076053682433, + "pass@16": 0.913900500083965, + "pass@32": 0.925047708329376, + "pass@64": 0.9367271813806438, + "pass@128": 0.9470404984423676 + }, + "15": { + "pass@1": 0.7983878251445087, + "pass@2": 0.859632797209959, + "pass@4": 0.8984884428283165, + "pass@8": 0.9249886345537007, + "pass@16": 0.9410397807831975, + "pass@32": 0.949840483402603, + "pass@64": 0.956831430393292, + "pass@128": 0.9653179190751445 + }, + "16": { + "pass@1": 0.6071824596774194, + "pass@2": 0.7037088042926081, + "pass@4": 0.768007695308247, + "pass@8": 0.8124516737316596, + "pass@16": 0.8460015212662262, + "pass@32": 0.8723927091205886, + "pass@64": 0.8951103234664651, + "pass@128": 0.9161290322580645 + }, + "17": { + "pass@1": 0.5784185332369942, + "pass@2": 0.6753856646260984, + "pass@4": 0.7448697205840593, + "pass@8": 0.7936231621801595, + "pass@16": 0.827266712612159, + "pass@32": 0.8520895959798461, + "pass@64": 0.8715641230767441, + "pass@128": 0.8872832369942196 + }, + "18": { + "pass@1": 0.5097991071428571, + "pass@2": 0.6145180680539934, + "pass@4": 0.6914994332851245, + "pass@8": 0.74884747793879, + "pass@16": 0.7939464107956918, + "pass@32": 0.8293121568507497, + "pass@64": 0.855726650165124, + "pass@128": 0.8771428571428571 + }, + "19": { + "pass@1": 0.3723958333333333, + "pass@2": 0.49274779208342234, + "pass@4": 0.5936755563212254, + "pass@8": 0.6701203130156419, + "pass@16": 0.7265665300507005, + "pass@32": 0.7676342272925999, + "pass@64": 0.7945744080866722, + "pass@128": 0.8108108108108109 + }, + "20": { + "pass@1": 0.39407376126126126, + "pass@2": 0.5133520577664276, + "pass@4": 0.6131522349120772, + "pass@8": 0.6888155856314068, + "pass@16": 0.743326415580538, + "pass@32": 0.7834285004700964, + "pass@64": 0.8144789867720131, + "pass@128": 0.8468468468468469 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 681549, + "answer_accuracy": 0.8300236262665628, + "avg_response_len": 287.3088708106002, + "resp_tokens_sum": 235915060, + "pass_at_k": { + "pass@1": 0.8300236262665628, + "pass@2": 0.869163838444581, + "pass@4": 0.8976645848438825, + "pass@8": 0.918732120915311, + "pass@16": 0.9343319377322344, + "pass@32": 0.9462167744203065, + "pass@64": 0.9553236125661387, + "pass@128": 0.9619641465315666 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9413822852760736, + "pass@2": 0.9538937581517802, + "pass@4": 0.9636568040191295, + "pass@8": 0.9718906525426227, + "pass@16": 0.979069899257666, + "pass@32": 0.9853812376219401, + "pass@64": 0.9897538439137533, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999705615942029, + "pass@2": 0.9999932243523908, + "pass@4": 0.9999999983697689, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991717406876791, + "pass@2": 0.999909400875392, + "pass@4": 0.9999988719175146, + "pass@8": 0.9999999999254859, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9971098265895953, + "pass@2": 0.998783552751354, + "pass@4": 0.9997432275445338, + "pass@8": 0.9999882753426383, + "pass@16": 0.9999999796137242, + "pass@32": 0.9999999999999929, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.994806338028169, + "pass@2": 0.9972243401353, + "pass@4": 0.9985757458134635, + "pass@8": 0.9994029807049142, + "pass@16": 0.9998887166365205, + "pass@32": 0.9999974897170781, + "pass@64": 0.9999999999243792, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9945967440119761, + "pass@2": 0.9967706074072328, + "pass@4": 0.997610679372414, + "pass@8": 0.9982385089022755, + "pass@16": 0.9990042498589532, + "pass@32": 0.9997223135661869, + "pass@64": 0.999990730945279, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9588667168674698, + "pass@2": 0.9688070688264869, + "pass@4": 0.9738426495633828, + "pass@8": 0.9767944886728196, + "pass@16": 0.9796148083018154, + "pass@32": 0.982822830027572, + "pass@64": 0.9855982198956623, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8903800940438872, + "pass@2": 0.9155861235899592, + "pass@4": 0.9309091514422766, + "pass@8": 0.9412217906007945, + "pass@16": 0.9481690271077153, + "pass@32": 0.9544341226634312, + "pass@64": 0.9611133242427224, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.8157169117647058, + "pass@2": 0.8605739539910452, + "pass@4": 0.8872929843853552, + "pass@8": 0.905276867908199, + "pass@16": 0.9190960959975872, + "pass@32": 0.9314763224458966, + "pass@64": 0.9439972488605914, + "pass@128": 0.9551820728291317 + }, + "14": { + "pass@1": 0.8373309576023392, + "pass@2": 0.8934615594925635, + "pass@4": 0.9265919024376331, + "pass@8": 0.946333115242231, + "pass@16": 0.9589428298211253, + "pass@32": 0.9680942410488826, + "pass@64": 0.9739592473038264, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.7923532196969697, + "pass@2": 0.8550170752803624, + "pass@4": 0.8956887607799019, + "pass@8": 0.9255279916761899, + "pass@16": 0.9489574497373325, + "pass@32": 0.9662421289695058, + "pass@64": 0.9762515985143063, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.6154736467236467, + "pass@2": 0.7142381441101907, + "pass@4": 0.7865660093770329, + "pass@8": 0.8398169209514765, + "pass@16": 0.878125499005597, + "pass@32": 0.9055294390113082, + "pass@64": 0.9247983094349499, + "pass@128": 0.9373219373219374 + }, + "17": { + "pass@1": 0.6003493788819876, + "pass@2": 0.7049765858072087, + "pass@4": 0.7759227191903806, + "pass@8": 0.8225132499570565, + "pass@16": 0.8555351979846405, + "pass@32": 0.8822355510888512, + "pass@64": 0.9047357050959228, + "pass@128": 0.9192546583850931 + }, + "18": { + "pass@1": 0.5742876838235295, + "pass@2": 0.6695149664196381, + "pass@4": 0.7379379058316233, + "pass@8": 0.7895887044016283, + "pass@16": 0.8302944179161645, + "pass@32": 0.8623677634837246, + "pass@64": 0.8845639909682449, + "pass@128": 0.9 + }, + "19": { + "pass@1": 0.37486049107142855, + "pass@2": 0.48710293049306325, + "pass@4": 0.5855916665550731, + "pass@8": 0.6672813754848781, + "pass@16": 0.7282600529581397, + "pass@32": 0.7711494737197626, + "pass@64": 0.804945509850621, + "pass@128": 0.8333333333333334 + }, + "20": { + "pass@1": 0.37359655688622756, + "pass@2": 0.4897408676741004, + "pass@4": 0.5887979210308285, + "pass@8": 0.6662973240324496, + "pass@16": 0.7224702823795047, + "pass@32": 0.7645544154261024, + "pass@64": 0.797975742891563, + "pass@128": 0.8263473053892215 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 654453, + "answer_accuracy": 0.8168899285029557, + "avg_response_len": 308.32675946636846, + "resp_tokens_sum": 247016600, + "pass_at_k": { + "pass@1": 0.8168899285029557, + "pass@2": 0.8580747463809546, + "pass@4": 0.8891222012630242, + "pass@8": 0.911687114191622, + "pass@16": 0.9276730592505993, + "pass@32": 0.939548454170509, + "pass@64": 0.948968336183653, + "pass@128": 0.956862118549289 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9623367537313433, + "pass@2": 0.972093885885533, + "pass@4": 0.978421042425667, + "pass@8": 0.983027619586872, + "pass@16": 0.9867115470675084, + "pass@32": 0.9903909013723802, + "pass@64": 0.9941986067826024, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998177842565598, + "pass@2": 0.9999924674594246, + "pass@4": 0.9999999904348691, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9935431985294118, + "pass@2": 0.9976486509958314, + "pass@4": 0.9995777931251238, + "pass@8": 0.9999822579717529, + "pass@16": 0.9999999633209915, + "pass@32": 0.9999999999999732, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9989402602523659, + "pass@2": 0.9996495348866092, + "pass@4": 0.9999635071247008, + "pass@8": 0.9999996800452854, + "pass@16": 0.9999999999910381, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9996785996835443, + "pass@2": 0.9999964959384032, + "pass@4": 0.99999999970336, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9910613738738738, + "pass@2": 0.9942744171336693, + "pass@4": 0.9966871548151077, + "pass@8": 0.9986133807702835, + "pass@16": 0.999693201383144, + "pass@32": 0.9999844909078028, + "pass@64": 0.999999993264007, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9950512459807074, + "pass@2": 0.9983321518089983, + "pass@4": 0.9995486453461805, + "pass@8": 0.9999472195181695, + "pass@16": 0.9999994075489268, + "pass@32": 0.9999999999829826, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9508184523809524, + "pass@2": 0.9642513435820521, + "pass@4": 0.9749534641503145, + "pass@8": 0.9832837923862544, + "pass@16": 0.9876988572082758, + "pass@32": 0.9896458888924424, + "pass@64": 0.9920634536546087, + "pass@128": 0.9968253968253968 + }, + "12": { + "pass@1": 0.9169533639143731, + "pass@2": 0.9397597161020009, + "pass@4": 0.9518531373371902, + "pass@8": 0.9606901786028378, + "pass@16": 0.9681186400954256, + "pass@32": 0.9749075267128768, + "pass@64": 0.9808020705886862, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.775438850308642, + "pass@2": 0.8281523403324585, + "pass@4": 0.8607345849708593, + "pass@8": 0.8828177922069514, + "pass@16": 0.8997273165986912, + "pass@32": 0.9142707748301222, + "pass@64": 0.9281925445370405, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.8076084940652819, + "pass@2": 0.8598076181686491, + "pass@4": 0.891258341316386, + "pass@8": 0.9115097424826967, + "pass@16": 0.9251147243681173, + "pass@32": 0.9365267663256304, + "pass@64": 0.9463097605158902, + "pass@128": 0.9525222551928784 + }, + "15": { + "pass@1": 0.7682532793209876, + "pass@2": 0.8318930294303489, + "pass@4": 0.8712868298870047, + "pass@8": 0.8968060083120799, + "pass@16": 0.9137817829092058, + "pass@32": 0.9265430089681044, + "pass@64": 0.9371401348760496, + "pass@128": 0.9444444444444444 + }, + "16": { + "pass@1": 0.5982439159292036, + "pass@2": 0.703770280235988, + "pass@4": 0.7803776925561295, + "pass@8": 0.8355710174645408, + "pass@16": 0.8748514051002123, + "pass@32": 0.9030904191916943, + "pass@64": 0.9217179562941028, + "pass@128": 0.9321533923303835 + }, + "17": { + "pass@1": 0.5402861445783133, + "pass@2": 0.6434680651740823, + "pass@4": 0.724809391672426, + "pass@8": 0.7817342862640553, + "pass@16": 0.8188666503786926, + "pass@32": 0.8451752781257261, + "pass@64": 0.8680626002024731, + "pass@128": 0.8945783132530121 + }, + "18": { + "pass@1": 0.52578125, + "pass@2": 0.6282492221234439, + "pass@4": 0.700043628518209, + "pass@8": 0.7514698105252829, + "pass@16": 0.7896155212055802, + "pass@32": 0.8173696406331719, + "pass@64": 0.8382107612672934, + "pass@128": 0.8580645161290322 + }, + "19": { + "pass@1": 0.3548904833836858, + "pass@2": 0.478963472655042, + "pass@4": 0.5904066607421803, + "pass@8": 0.6772394434978418, + "pass@16": 0.7417786907071799, + "pass@32": 0.7908978024465173, + "pass@64": 0.8274052077184393, + "pass@128": 0.8549848942598187 + }, + "20": { + "pass@1": 0.3346236861861862, + "pass@2": 0.4571285113854011, + "pass@4": 0.56794377560913, + "pass@8": 0.6541935591651141, + "pass@16": 0.7149285648958363, + "pass@32": 0.7578798172314243, + "pass@64": 0.7917844220372017, + "pass@128": 0.8228228228228228 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/summary.csv b/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..e64daa3c8ee7d8b9ee01b23539a1889785993531 --- /dev/null +++ b/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.99325,0.6697265625,0.8229745065789473,0.0,0.0,0.0,199.50800086805555,373.5406,1152000,1280000,2432000 diff --git a/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/summary.json b/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4737d41d8ab22cf88f086f9998e4803ca414c7d3 --- /dev/null +++ b/SDAR-100M-ep4-block4-RL-op17-20/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.99325, + "ood_acc": 0.6697265625, + "total_acc": 0.8229745065789473, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.50800086805555, + "ood_avg_resp_len": 373.5406, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 665472, + "answer_accuracy": 0.8218463484034145, + "avg_response_len": 277.9124903671356, + "resp_tokens_sum": 225033525, + "pass_at_k": { + "pass@1": 0.8218463484034145, + "pass@2": 0.8616996456941791, + "pass@4": 0.8906064232239269, + "pass@8": 0.9114550278470633, + "pass@16": 0.9266238856684035, + "pass@32": 0.9380627735970853, + "pass@64": 0.9469416268536985, + "pass@128": 0.9544736010116978 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9615597345132744, + "pass@2": 0.9727621333008151, + "pass@4": 0.9804466844852356, + "pass@8": 0.9857071172020305, + "pass@16": 0.9899689862028797, + "pass@32": 0.9937941997890816, + "pass@64": 0.9968274449088425, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99931640625, + "pass@2": 0.9998650498277559, + "pass@4": 0.9999948590410573, + "pass@8": 0.9999999951474285, + "pass@16": 0.9999999999999997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996453373015873, + "pass@2": 0.9979561148606424, + "pass@4": 0.9988745466340517, + "pass@8": 0.999619849729349, + "pass@16": 0.9999615508301392, + "pass@32": 0.9999997910921954, + "pass@64": 0.9999999999999056, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9982433431952663, + "pass@2": 0.9996964252434422, + "pass@4": 0.9999810179977504, + "pass@8": 0.9999999200555205, + "pass@16": 0.999999999999592, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9984224759615384, + "pass@2": 0.9999282316272965, + "pass@4": 0.9999997100722985, + "pass@8": 0.9999999999970139, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.996368838028169, + "pass@2": 0.9973293501164467, + "pass@4": 0.9975227754981333, + "pass@8": 0.9978309304643527, + "pass@16": 0.9983600236590144, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9355524079320113, + "pass@2": 0.9494138403113916, + "pass@4": 0.9587174374980745, + "pass@8": 0.9654747650936155, + "pass@16": 0.9708917986847155, + "pass@32": 0.9757436548795662, + "pass@64": 0.9793999690641447, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.8806497175141242, + "pass@2": 0.9054259670136573, + "pass@4": 0.9203959054906274, + "pass@8": 0.9309233567968177, + "pass@16": 0.9386696115032847, + "pass@32": 0.9454865056893909, + "pass@64": 0.9526492494418823, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8020914968652038, + "pass@2": 0.8500751302051195, + "pass@4": 0.8829129885410095, + "pass@8": 0.9072514585026777, + "pass@16": 0.9247382338224696, + "pass@32": 0.9368263153026625, + "pass@64": 0.9448406529194303, + "pass@128": 0.9498432601880877 + }, + "14": { + "pass@1": 0.8056853582554517, + "pass@2": 0.8563524879191502, + "pass@4": 0.8858799866722263, + "pass@8": 0.9025076053682433, + "pass@16": 0.913900500083965, + "pass@32": 0.925047708329376, + "pass@64": 0.9367271813806438, + "pass@128": 0.9470404984423676 + }, + "15": { + "pass@1": 0.7983878251445087, + "pass@2": 0.859632797209959, + "pass@4": 0.8984884428283165, + "pass@8": 0.9249886345537007, + "pass@16": 0.9410397807831975, + "pass@32": 0.949840483402603, + "pass@64": 0.956831430393292, + "pass@128": 0.9653179190751445 + }, + "16": { + "pass@1": 0.6071824596774194, + "pass@2": 0.7037088042926081, + "pass@4": 0.768007695308247, + "pass@8": 0.8124516737316596, + "pass@16": 0.8460015212662262, + "pass@32": 0.8723927091205886, + "pass@64": 0.8951103234664651, + "pass@128": 0.9161290322580645 + }, + "17": { + "pass@1": 0.5784185332369942, + "pass@2": 0.6753856646260984, + "pass@4": 0.7448697205840593, + "pass@8": 0.7936231621801595, + "pass@16": 0.827266712612159, + "pass@32": 0.8520895959798461, + "pass@64": 0.8715641230767441, + "pass@128": 0.8872832369942196 + }, + "18": { + "pass@1": 0.5097991071428571, + "pass@2": 0.6145180680539934, + "pass@4": 0.6914994332851245, + "pass@8": 0.74884747793879, + "pass@16": 0.7939464107956918, + "pass@32": 0.8293121568507497, + "pass@64": 0.855726650165124, + "pass@128": 0.8771428571428571 + }, + "19": { + "pass@1": 0.3723958333333333, + "pass@2": 0.49274779208342234, + "pass@4": 0.5936755563212254, + "pass@8": 0.6701203130156419, + "pass@16": 0.7265665300507005, + "pass@32": 0.7676342272925999, + "pass@64": 0.7945744080866722, + "pass@128": 0.8108108108108109 + }, + "20": { + "pass@1": 0.39407376126126126, + "pass@2": 0.5133520577664276, + "pass@4": 0.6131522349120772, + "pass@8": 0.6888155856314068, + "pass@16": 0.743326415580538, + "pass@32": 0.7834285004700964, + "pass@64": 0.8144789867720131, + "pass@128": 0.8468468468468469 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 681549, + "answer_accuracy": 0.8300236262665628, + "avg_response_len": 287.3088708106002, + "resp_tokens_sum": 235915060, + "pass_at_k": { + "pass@1": 0.8300236262665628, + "pass@2": 0.869163838444581, + "pass@4": 0.8976645848438825, + "pass@8": 0.918732120915311, + "pass@16": 0.9343319377322344, + "pass@32": 0.9462167744203065, + "pass@64": 0.9553236125661387, + "pass@128": 0.9619641465315666 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9413822852760736, + "pass@2": 0.9538937581517802, + "pass@4": 0.9636568040191295, + "pass@8": 0.9718906525426227, + "pass@16": 0.979069899257666, + "pass@32": 0.9853812376219401, + "pass@64": 0.9897538439137533, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999705615942029, + "pass@2": 0.9999932243523908, + "pass@4": 0.9999999983697689, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991717406876791, + "pass@2": 0.999909400875392, + "pass@4": 0.9999988719175146, + "pass@8": 0.9999999999254859, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9971098265895953, + "pass@2": 0.998783552751354, + "pass@4": 0.9997432275445338, + "pass@8": 0.9999882753426383, + "pass@16": 0.9999999796137242, + "pass@32": 0.9999999999999929, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.994806338028169, + "pass@2": 0.9972243401353, + "pass@4": 0.9985757458134635, + "pass@8": 0.9994029807049142, + "pass@16": 0.9998887166365205, + "pass@32": 0.9999974897170781, + "pass@64": 0.9999999999243792, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9945967440119761, + "pass@2": 0.9967706074072328, + "pass@4": 0.997610679372414, + "pass@8": 0.9982385089022755, + "pass@16": 0.9990042498589532, + "pass@32": 0.9997223135661869, + "pass@64": 0.999990730945279, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9588667168674698, + "pass@2": 0.9688070688264869, + "pass@4": 0.9738426495633828, + "pass@8": 0.9767944886728196, + "pass@16": 0.9796148083018154, + "pass@32": 0.982822830027572, + "pass@64": 0.9855982198956623, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8903800940438872, + "pass@2": 0.9155861235899592, + "pass@4": 0.9309091514422766, + "pass@8": 0.9412217906007945, + "pass@16": 0.9481690271077153, + "pass@32": 0.9544341226634312, + "pass@64": 0.9611133242427224, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.8157169117647058, + "pass@2": 0.8605739539910452, + "pass@4": 0.8872929843853552, + "pass@8": 0.905276867908199, + "pass@16": 0.9190960959975872, + "pass@32": 0.9314763224458966, + "pass@64": 0.9439972488605914, + "pass@128": 0.9551820728291317 + }, + "14": { + "pass@1": 0.8373309576023392, + "pass@2": 0.8934615594925635, + "pass@4": 0.9265919024376331, + "pass@8": 0.946333115242231, + "pass@16": 0.9589428298211253, + "pass@32": 0.9680942410488826, + "pass@64": 0.9739592473038264, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.7923532196969697, + "pass@2": 0.8550170752803624, + "pass@4": 0.8956887607799019, + "pass@8": 0.9255279916761899, + "pass@16": 0.9489574497373325, + "pass@32": 0.9662421289695058, + "pass@64": 0.9762515985143063, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.6154736467236467, + "pass@2": 0.7142381441101907, + "pass@4": 0.7865660093770329, + "pass@8": 0.8398169209514765, + "pass@16": 0.878125499005597, + "pass@32": 0.9055294390113082, + "pass@64": 0.9247983094349499, + "pass@128": 0.9373219373219374 + }, + "17": { + "pass@1": 0.6003493788819876, + "pass@2": 0.7049765858072087, + "pass@4": 0.7759227191903806, + "pass@8": 0.8225132499570565, + "pass@16": 0.8555351979846405, + "pass@32": 0.8822355510888512, + "pass@64": 0.9047357050959228, + "pass@128": 0.9192546583850931 + }, + "18": { + "pass@1": 0.5742876838235295, + "pass@2": 0.6695149664196381, + "pass@4": 0.7379379058316233, + "pass@8": 0.7895887044016283, + "pass@16": 0.8302944179161645, + "pass@32": 0.8623677634837246, + "pass@64": 0.8845639909682449, + "pass@128": 0.9 + }, + "19": { + "pass@1": 0.37486049107142855, + "pass@2": 0.48710293049306325, + "pass@4": 0.5855916665550731, + "pass@8": 0.6672813754848781, + "pass@16": 0.7282600529581397, + "pass@32": 0.7711494737197626, + "pass@64": 0.804945509850621, + "pass@128": 0.8333333333333334 + }, + "20": { + "pass@1": 0.37359655688622756, + "pass@2": 0.4897408676741004, + "pass@4": 0.5887979210308285, + "pass@8": 0.6662973240324496, + "pass@16": 0.7224702823795047, + "pass@32": 0.7645544154261024, + "pass@64": 0.797975742891563, + "pass@128": 0.8263473053892215 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 654453, + "answer_accuracy": 0.8168899285029557, + "avg_response_len": 308.32675946636846, + "resp_tokens_sum": 247016600, + "pass_at_k": { + "pass@1": 0.8168899285029557, + "pass@2": 0.8580747463809546, + "pass@4": 0.8891222012630242, + "pass@8": 0.911687114191622, + "pass@16": 0.9276730592505993, + "pass@32": 0.939548454170509, + "pass@64": 0.948968336183653, + "pass@128": 0.956862118549289 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9623367537313433, + "pass@2": 0.972093885885533, + "pass@4": 0.978421042425667, + "pass@8": 0.983027619586872, + "pass@16": 0.9867115470675084, + "pass@32": 0.9903909013723802, + "pass@64": 0.9941986067826024, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998177842565598, + "pass@2": 0.9999924674594246, + "pass@4": 0.9999999904348691, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9935431985294118, + "pass@2": 0.9976486509958314, + "pass@4": 0.9995777931251238, + "pass@8": 0.9999822579717529, + "pass@16": 0.9999999633209915, + "pass@32": 0.9999999999999732, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9989402602523659, + "pass@2": 0.9996495348866092, + "pass@4": 0.9999635071247008, + "pass@8": 0.9999996800452854, + "pass@16": 0.9999999999910381, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9996785996835443, + "pass@2": 0.9999964959384032, + "pass@4": 0.99999999970336, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9910613738738738, + "pass@2": 0.9942744171336693, + "pass@4": 0.9966871548151077, + "pass@8": 0.9986133807702835, + "pass@16": 0.999693201383144, + "pass@32": 0.9999844909078028, + "pass@64": 0.999999993264007, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9950512459807074, + "pass@2": 0.9983321518089983, + "pass@4": 0.9995486453461805, + "pass@8": 0.9999472195181695, + "pass@16": 0.9999994075489268, + "pass@32": 0.9999999999829826, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9508184523809524, + "pass@2": 0.9642513435820521, + "pass@4": 0.9749534641503145, + "pass@8": 0.9832837923862544, + "pass@16": 0.9876988572082758, + "pass@32": 0.9896458888924424, + "pass@64": 0.9920634536546087, + "pass@128": 0.9968253968253968 + }, + "12": { + "pass@1": 0.9169533639143731, + "pass@2": 0.9397597161020009, + "pass@4": 0.9518531373371902, + "pass@8": 0.9606901786028378, + "pass@16": 0.9681186400954256, + "pass@32": 0.9749075267128768, + "pass@64": 0.9808020705886862, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.775438850308642, + "pass@2": 0.8281523403324585, + "pass@4": 0.8607345849708593, + "pass@8": 0.8828177922069514, + "pass@16": 0.8997273165986912, + "pass@32": 0.9142707748301222, + "pass@64": 0.9281925445370405, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.8076084940652819, + "pass@2": 0.8598076181686491, + "pass@4": 0.891258341316386, + "pass@8": 0.9115097424826967, + "pass@16": 0.9251147243681173, + "pass@32": 0.9365267663256304, + "pass@64": 0.9463097605158902, + "pass@128": 0.9525222551928784 + }, + "15": { + "pass@1": 0.7682532793209876, + "pass@2": 0.8318930294303489, + "pass@4": 0.8712868298870047, + "pass@8": 0.8968060083120799, + "pass@16": 0.9137817829092058, + "pass@32": 0.9265430089681044, + "pass@64": 0.9371401348760496, + "pass@128": 0.9444444444444444 + }, + "16": { + "pass@1": 0.5982439159292036, + "pass@2": 0.703770280235988, + "pass@4": 0.7803776925561295, + "pass@8": 0.8355710174645408, + "pass@16": 0.8748514051002123, + "pass@32": 0.9030904191916943, + "pass@64": 0.9217179562941028, + "pass@128": 0.9321533923303835 + }, + "17": { + "pass@1": 0.5402861445783133, + "pass@2": 0.6434680651740823, + "pass@4": 0.724809391672426, + "pass@8": 0.7817342862640553, + "pass@16": 0.8188666503786926, + "pass@32": 0.8451752781257261, + "pass@64": 0.8680626002024731, + "pass@128": 0.8945783132530121 + }, + "18": { + "pass@1": 0.52578125, + "pass@2": 0.6282492221234439, + "pass@4": 0.700043628518209, + "pass@8": 0.7514698105252829, + "pass@16": 0.7896155212055802, + "pass@32": 0.8173696406331719, + "pass@64": 0.8382107612672934, + "pass@128": 0.8580645161290322 + }, + "19": { + "pass@1": 0.3548904833836858, + "pass@2": 0.478963472655042, + "pass@4": 0.5904066607421803, + "pass@8": 0.6772394434978418, + "pass@16": 0.7417786907071799, + "pass@32": 0.7908978024465173, + "pass@64": 0.8274052077184393, + "pass@128": 0.8549848942598187 + }, + "20": { + "pass@1": 0.3346236861861862, + "pass@2": 0.4571285113854011, + "pass@4": 0.56794377560913, + "pass@8": 0.6541935591651141, + "pass@16": 0.7149285648958363, + "pass@32": 0.7578798172314243, + "pass@64": 0.7917844220372017, + "pass@128": 0.8228228228228228 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep4-block4-op11-14/composition/autoregressive/hf_model_metrics.json b/SDAR-100M-ep4-block4-op11-14/composition/autoregressive/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3b3f3ee5b8d49f397d7a6bfeb152b2b88ade7434 --- /dev/null +++ b/SDAR-100M-ep4-block4-op11-14/composition/autoregressive/hf_model_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.9936449652777778, + "avg_response_len": 199.58314496527777, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9936449652777778, + "pass@2": 0.9955745160761146, + "pass@4": 0.9967351979440073, + "pass@8": 0.9975188392901952, + "pass@16": 0.9981103375830034, + "pass@32": 0.9986016114425479, + "pass@64": 0.9989544521590071, + "pass@128": 0.9991111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9566953125, + "pass@2": 0.9670733267716537, + "pass@4": 0.9742653679227598, + "pass@8": 0.9794420704275232, + "pass@16": 0.98370404958131, + "pass@32": 0.9876131313445727, + "pass@64": 0.9906040315832808, + "pass@128": 0.992 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99990625, + "pass@2": 0.9999918799212598, + "pass@4": 0.9999999535995501, + "pass@8": 0.9999999999996538, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996890625, + "pass@2": 0.9987973671259842, + "pass@4": 0.9996501834458194, + "pass@8": 0.9999408420930652, + "pass@16": 0.9999976881673509, + "pass@32": 0.9999999983745367, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991484375, + "pass@2": 0.9997263779527559, + "pass@4": 0.9999574376640419, + "pass@8": 0.9999986551927124, + "pass@16": 0.999999999143371, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9990546875, + "pass@2": 0.9998537155511811, + "pass@4": 0.9999935515560555, + "pass@8": 0.9999999859843178, + "pass@16": 0.9999999999999764, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9951640625, + "pass@2": 0.9970433070866144, + "pass@4": 0.9982334005436821, + "pass@8": 0.9991791093699898, + "pass@16": 0.9997618776815873, + "pass@32": 0.9999727342955712, + "pass@64": 0.9999998615508054, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9959453125, + "pass@2": 0.9976846702755906, + "pass@4": 0.9985168867641545, + "pass@8": 0.9991088905444924, + "pass@16": 0.999529423673436, + "pass@32": 0.9998286389682483, + "pass@64": 0.9999861762969797, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9566953125, + "2": 1.0, + "3": 1.0, + "4": 0.99990625, + "5": 0.996890625, + "6": 0.9991484375, + "7": 0.9990546875, + "8": 0.9951640625, + "9": 0.9959453125 + }, + "per_op_avg_response_len": { + "10": 280.2356640625, + "2": 115.461328125, + "3": 144.5140859375, + "4": 159.5963828125, + "5": 172.538421875, + "6": 198.58234375, + "7": 223.48178125, + "8": 243.0421875, + "9": 258.796109375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.2356640625, + "2": 115.461328125, + "3": 144.5140859375, + "4": 159.5963828125, + "5": 172.538421875, + "6": 198.58234375, + "7": 223.48178125, + "8": 243.0421875, + "9": 258.796109375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376748, + "answer_accuracy": 0.9940370651806822, + "avg_response_len": 190.07905374029045, + "resp_tokens_sum": 72041482, + "pass_at_k": { + "pass@1": 0.9940370651806822, + "pass@2": 0.9958554465931122, + "pass@4": 0.9969563209677312, + "pass@8": 0.9977754887819017, + "pass@16": 0.9984205836215371, + "pass@32": 0.9989076831039293, + "pass@64": 0.9992136337937035, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9571580014749262, + "pass@2": 0.9677603041599888, + "pass@4": 0.975415939434562, + "pass@8": 0.9812614083839927, + "pass@16": 0.9862952176965449, + "pass@32": 0.9904604242490184, + "pass@64": 0.9931314739448671, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99970703125, + "pass@2": 0.9999746247539371, + "pass@4": 0.9999998549985939, + "pass@8": 0.999999999998918, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966269841269841, + "pass@2": 0.9982509998750156, + "pass@4": 0.9992314291070759, + "pass@8": 0.9998278665841752, + "pass@16": 0.999992699151632, + "pass@32": 0.9999999948398277, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999766092814372, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9983357988165681, + "pass@2": 0.9997022492195872, + "pass@4": 0.9999851042103463, + "pass@8": 0.9999999617664239, + "pass@16": 0.9999999999999302, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.998848157051282, + "pass@2": 0.9999357239299413, + "pass@4": 0.999999570065761, + "pass@8": 0.999999999985574, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9970510563380282, + "pass@2": 0.9981323472330043, + "pass@4": 0.9987858537049066, + "pass@8": 0.9994925064642773, + "pass@16": 0.9999199128774705, + "pass@32": 0.9999987816218918, + "pass@64": 0.9999999999883001, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387842, + "answer_accuracy": 0.9927967316513762, + "avg_response_len": 197.56825954292268, + "resp_tokens_sum": 77181226, + "pass_at_k": { + "pass@1": 0.9927967316513762, + "pass@2": 0.9946963427880001, + "pass@4": 0.9957899767361972, + "pass@8": 0.9965292521776954, + "pass@16": 0.9971205358109471, + "pass@32": 0.9976769284102713, + "pass@64": 0.9981197974458447, + "pass@128": 0.9983617300131061 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9449290644171779, + "pass@2": 0.9567034805081881, + "pass@4": 0.9647007856763299, + "pass@8": 0.9702237713814276, + "pass@16": 0.9745168430378548, + "pass@32": 0.9787785426264238, + "pass@64": 0.9824400169645098, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994338768115942, + "pass@2": 0.999957563049184, + "pass@4": 0.9999997090037658, + "pass@8": 0.9999999999939099, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9989255014326648, + "pass@2": 0.999798707781513, + "pass@4": 0.9999920000150412, + "pass@8": 0.9999999882211767, + "pass@16": 0.9999999999999954, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9990742413294798, + "pass@2": 0.9998716347685586, + "pass@4": 0.9999959156117046, + "pass@8": 0.9999999968418107, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954885563380281, + "pass@2": 0.9974222302317843, + "pass@4": 0.9985825871237926, + "pass@8": 0.999402550678006, + "pass@16": 0.9998887161460439, + "pass@32": 0.9999974897170781, + "pass@64": 0.9999999999243792, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9953920284431138, + "pass@2": 0.9969231075251073, + "pass@4": 0.9975031138510381, + "pass@8": 0.9979833170387231, + "pass@16": 0.998679312074329, + "pass@32": 0.9994882388094946, + "pass@64": 0.9999586116800393, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380089, + "answer_accuracy": 0.9941229703716103, + "avg_response_len": 211.0632401866421, + "resp_tokens_sum": 80697075, + "pass_at_k": { + "pass@1": 0.9941229703716103, + "pass@2": 0.9961933140722659, + "pass@4": 0.9974817897930154, + "pass@8": 0.9982755452568516, + "pass@16": 0.9988141328586038, + "pass@32": 0.9992430089735673, + "pass@64": 0.9995503441456951, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9676772388059701, + "pass@2": 0.9764694000470092, + "pass@4": 0.9824086815640581, + "pass@8": 0.9865716522835937, + "pass@16": 0.9900223282144495, + "pass@32": 0.9933290824714656, + "pass@64": 0.9959910936881807, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9945542279411764, + "pass@2": 0.9981263026864289, + "pass@4": 0.9996834813846798, + "pass@8": 0.9999854827092686, + "pass@16": 0.9999999645134902, + "pass@32": 0.9999999999999732, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985212933753943, + "pass@2": 0.9993584509053877, + "pass@4": 0.9998745415103868, + "pass@8": 0.9999957706735703, + "pass@16": 0.9999999972977055, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998022151898734, + "pass@2": 0.999996106598226, + "pass@4": 0.9999999985167994, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9913663663663663, + "pass@2": 0.9939293378023693, + "pass@4": 0.9962063551177722, + "pass@8": 0.9981717834348306, + "pass@16": 0.9994035539031283, + "pass@32": 0.9999207971351605, + "pass@64": 0.99999958431727, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9952773311897106, + "pass@2": 0.9979915404967465, + "pass@4": 0.9992986130953887, + "pass@8": 0.9998798162660464, + "pass@16": 0.9999966699971327, + "pass@32": 0.9999999991321724, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.63034765625, + "avg_response_len": 365.1622640625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.63034765625, + "pass@2": 0.7048224901574739, + "pass@4": 0.7641028307742694, + "pass@8": 0.8094828831919582, + "pass@16": 0.8429851350989258, + "pass@32": 0.8683509720938262, + "pass@64": 0.8885767226356113, + "pass@128": 0.9043 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.954015625, + "pass@2": 0.9644905265748035, + "pass@4": 0.9711644761904762, + "pass@8": 0.9757205330964192, + "pass@16": 0.979031344222367, + "pass@32": 0.9815967967980977, + "pass@64": 0.9838816261970059, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.9025, + "pass@2": 0.9237837106299214, + "pass@4": 0.9366584860329956, + "pass@8": 0.9459949739746121, + "pass@16": 0.9536564122340577, + "pass@32": 0.9606255820011785, + "pass@64": 0.966712341245886, + "pass@128": 0.972 + }, + "13": { + "pass@1": 0.8218515625, + "pass@2": 0.8634689960629935, + "pass@4": 0.8884614780652423, + "pass@8": 0.9052056008326004, + "pass@16": 0.9178599354690884, + "pass@32": 0.9289051742386563, + "pass@64": 0.9387523105805766, + "pass@128": 0.947 + }, + "14": { + "pass@1": 0.83196875, + "pass@2": 0.8842485236220485, + "pass@4": 0.9123400595238086, + "pass@8": 0.9274922279181592, + "pass@16": 0.9369694871852076, + "pass@32": 0.9448955169085047, + "pass@64": 0.9519450244071079, + "pass@128": 0.957 + }, + "15": { + "pass@1": 0.76134375, + "pass@2": 0.8312776820866143, + "pass@4": 0.8747924567866506, + "pass@8": 0.9042052814668986, + "pass@16": 0.9257042732151205, + "pass@32": 0.9409176804179538, + "pass@64": 0.9504559175126083, + "pass@128": 0.956 + }, + "16": { + "pass@1": 0.5898046875, + "pass@2": 0.6973601131889768, + "pass@4": 0.7734774335395578, + "pass@8": 0.8257421433039629, + "pass@16": 0.8615484479408064, + "pass@32": 0.8880162983132416, + "pass@64": 0.9096919211793338, + "pass@128": 0.927 + }, + "17": { + "pass@1": 0.5148046875, + "pass@2": 0.6294880659448818, + "pass@4": 0.7171008025871763, + "pass@8": 0.7792052967778853, + "pass@16": 0.8216423844459994, + "pass@32": 0.8518329273965232, + "pass@64": 0.8754075200291189, + "pass@128": 0.892 + }, + "18": { + "pass@1": 0.4244921875, + "pass@2": 0.540060900590551, + "pass@4": 0.6354181049868758, + "pass@8": 0.7075168907678558, + "pass@16": 0.7601426954402293, + "pass@32": 0.8004978277335945, + "pass@64": 0.832602069078636, + "pass@128": 0.855 + }, + "19": { + "pass@1": 0.266953125, + "pass@2": 0.3751668307086617, + "pass@4": 0.4843383939820021, + "pass@8": 0.5798556956198491, + "pass@16": 0.6542798459127458, + "pass@32": 0.7112011775890043, + "pass@64": 0.7590658635954, + "pass@128": 0.802 + }, + "20": { + "pass@1": 0.2357421875, + "pass@2": 0.33887955216535465, + "pass@4": 0.44727661604799396, + "pass@8": 0.5438901881613861, + "pass@16": 0.619016524923539, + "pass@32": 0.6750207395414648, + "pass@64": 0.7172526325303729, + "pass@128": 0.749 + } + }, + "per_op_accuracy": { + "11": 0.954015625, + "12": 0.9025, + "13": 0.8218515625, + "14": 0.83196875, + "15": 0.76134375, + "16": 0.5898046875, + "17": 0.5148046875, + "18": 0.4244921875, + "19": 0.266953125, + "20": 0.2357421875 + }, + "per_op_avg_response_len": { + "11": 297.94484375, + "12": 308.3263828125, + "13": 327.143078125, + "14": 354.34553125, + "15": 371.7521640625, + "16": 378.26515625, + "17": 396.573625, + "18": 402.8715703125, + "19": 404.6708515625, + "20": 409.7294375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.94484375, + "12": 308.3263828125, + "13": 327.143078125, + "14": 354.34553125, + "15": 371.7521640625, + "16": 378.26515625, + "17": 396.573625, + "18": 402.8715703125, + "19": 404.6708515625, + "20": 409.7294375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 271868, + "answer_accuracy": 0.631194279346211, + "avg_response_len": 347.3222395059435, + "resp_tokens_sum": 149598635, + "pass_at_k": { + "pass@1": 0.631194279346211, + "pass@2": 0.7045183454037097, + "pass@4": 0.7622331483638876, + "pass@8": 0.8062977828635046, + "pass@16": 0.8387674652409794, + "pass@32": 0.8628060139243993, + "pass@64": 0.8814646831954202, + "pass@128": 0.8965824665676078 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9444050991501416, + "pass@2": 0.9561910285293657, + "pass@4": 0.9641967786958638, + "pass@8": 0.9697560155386075, + "pass@16": 0.9741350032582587, + "pass@32": 0.9779370472555533, + "pass@64": 0.9815867813603899, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.887601518361582, + "pass@2": 0.9092204807820631, + "pass@4": 0.9226452011295195, + "pass@8": 0.932021504510786, + "pass@16": 0.9383912783274672, + "pass@32": 0.9442462997372086, + "pass@64": 0.9515706040014275, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.8336843652037618, + "pass@2": 0.8732879723792364, + "pass@4": 0.8977824343273704, + "pass@8": 0.9148135688866712, + "pass@16": 0.9285541397791447, + "pass@32": 0.9400278370501557, + "pass@64": 0.9490071310096556, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.8212130062305296, + "pass@2": 0.8690642094095712, + "pass@4": 0.8946342282565142, + "pass@8": 0.9095416152396502, + "pass@16": 0.9205671977126164, + "pass@32": 0.9299201530908431, + "pass@64": 0.9376412005954143, + "pass@128": 0.9439252336448598 + }, + "15": { + "pass@1": 0.7766437861271677, + "pass@2": 0.8473861851986711, + "pass@4": 0.8899432666653658, + "pass@8": 0.9196497067291849, + "pass@16": 0.9396602187962684, + "pass@32": 0.950786723015623, + "pass@64": 0.9560000188004241, + "pass@128": 0.9595375722543352 + }, + "16": { + "pass@1": 0.5931451612903226, + "pass@2": 0.6976770859791713, + "pass@4": 0.7679156642718047, + "pass@8": 0.8140579344020908, + "pass@16": 0.8454150252129901, + "pass@32": 0.8705633373440882, + "pass@64": 0.8931359432887487, + "pass@128": 0.9129032258064517 + }, + "17": { + "pass@1": 0.5168442919075145, + "pass@2": 0.6315512493741748, + "pass@4": 0.7201531642901571, + "pass@8": 0.782651914043665, + "pass@16": 0.8222871470379363, + "pass@32": 0.8481959011248583, + "pass@64": 0.8685797265002355, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.4018526785714286, + "pass@2": 0.5119948678290213, + "pass@4": 0.6041472810541539, + "pass@8": 0.676469614184862, + "pass@16": 0.7335479686894587, + "pass@32": 0.7793197198583286, + "pass@64": 0.8148807734554173, + "pass@128": 0.8371428571428572 + }, + "19": { + "pass@1": 0.2663522897897898, + "pass@2": 0.37801704263318436, + "pass@4": 0.48845757090048425, + "pass@8": 0.5820235208190736, + "pass@16": 0.6525778863987478, + "pass@32": 0.7028863331400599, + "pass@64": 0.7412004878213567, + "pass@128": 0.7747747747747747 + }, + "20": { + "pass@1": 0.25844594594594594, + "pass@2": 0.3624208608214512, + "pass@4": 0.4657998673589216, + "pass@8": 0.5561911364487827, + "pass@16": 0.6273839014887124, + "pass@32": 0.6797247642370513, + "pass@64": 0.7172722633189874, + "pass@128": 0.7477477477477478 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 276419, + "answer_accuracy": 0.6421419677371395, + "avg_response_len": 360.9209875854892, + "resp_tokens_sum": 155363492, + "pass_at_k": { + "pass@1": 0.6421419677371395, + "pass@2": 0.716881369980407, + "pass@4": 0.7752087656081385, + "pass@8": 0.819429469555494, + "pass@16": 0.8524098883881526, + "pass@32": 0.8779363076254243, + "pass@64": 0.8990693084725377, + "pass@128": 0.9161462979482605 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9630788780120482, + "pass@2": 0.9706936752917183, + "pass@4": 0.9737727306865254, + "pass@8": 0.9753456810706799, + "pass@16": 0.9769636060644346, + "pass@32": 0.9788680562854182, + "pass@64": 0.9808120954397608, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8993681426332288, + "pass@2": 0.9229849832152642, + "pass@4": 0.9366999725543712, + "pass@8": 0.9458536190031113, + "pass@16": 0.9535302205560833, + "pass@32": 0.9602898879824436, + "pass@64": 0.9661108029867137, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.8356530112044818, + "pass@2": 0.8733330438474605, + "pass@4": 0.8943991720047597, + "pass@8": 0.9074957916972102, + "pass@16": 0.9178935997150323, + "pass@32": 0.9280352165055543, + "pass@64": 0.9380920225452458, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.8555601242690059, + "pass@2": 0.9101467168577612, + "pass@4": 0.940181309847234, + "pass@8": 0.9555749876630905, + "pass@16": 0.9629698546320736, + "pass@32": 0.967437763855752, + "pass@64": 0.9711078593956024, + "pass@128": 0.9736842105263158 + }, + "15": { + "pass@1": 0.7712121212121212, + "pass@2": 0.8411432235743264, + "pass@4": 0.8843673654429557, + "pass@8": 0.9130173513881795, + "pass@16": 0.9363332414054601, + "pass@32": 0.9546181905381521, + "pass@64": 0.9668377534638671, + "pass@128": 0.9727272727272728 + }, + "16": { + "pass@1": 0.5882300569800569, + "pass@2": 0.69388572021446, + "pass@4": 0.7695331761414431, + "pass@8": 0.8236199139050931, + "pass@16": 0.8618354777037932, + "pass@32": 0.8898731315172652, + "pass@64": 0.9135394483921452, + "pass@128": 0.9344729344729344 + }, + "17": { + "pass@1": 0.5468507375776398, + "pass@2": 0.6665892306939891, + "pass@4": 0.7522262501542892, + "pass@8": 0.8093810436102301, + "pass@16": 0.8482521259212884, + "pass@32": 0.8764311748934481, + "pass@64": 0.898967766392957, + "pass@128": 0.9130434782608695 + }, + "18": { + "pass@1": 0.44682904411764707, + "pass@2": 0.5680378647522, + "pass@4": 0.666828606442577, + "pass@8": 0.7401726878412729, + "pass@16": 0.792571136084111, + "pass@32": 0.8324196559563581, + "pass@64": 0.8620981240838929, + "pass@128": 0.8794117647058823 + }, + "19": { + "pass@1": 0.28848121279761907, + "pass@2": 0.39275058879358826, + "pass@4": 0.495936091414912, + "pass@8": 0.5880557806590099, + "pass@16": 0.6609011227155269, + "pass@32": 0.717875181912308, + "pass@64": 0.7699961963434331, + "pass@128": 0.8244047619047619 + }, + "20": { + "pass@1": 0.22768525449101795, + "pass@2": 0.33008834692819083, + "pass@4": 0.43881511274414076, + "pass@8": 0.5364195334497773, + "pass@16": 0.6135758123072342, + "pass@32": 0.6742568609474818, + "pass@64": 0.7237271635487402, + "pass@128": 0.7634730538922155 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 258558, + "answer_accuracy": 0.6173546378361858, + "avg_response_len": 387.8685890701406, + "resp_tokens_sum": 162445571, + "pass_at_k": { + "pass@1": 0.6173546378361858, + "pass@2": 0.6927410214080855, + "pass@4": 0.7546108449749891, + "pass@8": 0.8025352953755706, + "pass@16": 0.8376358116760279, + "pass@32": 0.8642016141620715, + "pass@64": 0.8851065045874899, + "pass@128": 0.9000611246943765 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9552331349206349, + "pass@2": 0.9672533120859892, + "pass@4": 0.9762237038822525, + "pass@8": 0.9827996618915072, + "pass@16": 0.9866976852660617, + "pass@32": 0.9885740458099326, + "pass@64": 0.9896884974310719, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9216838685015291, + "pass@2": 0.9403285956801272, + "pass@4": 0.9517883595972522, + "pass@8": 0.9612601128923591, + "pass@16": 0.970305074460992, + "pass@32": 0.978684763387789, + "pass@64": 0.9836911354025035, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7949942129629629, + "pass@2": 0.8429328278409639, + "pass@4": 0.8727419046577508, + "pass@8": 0.8932224528452122, + "pass@16": 0.9072936721644289, + "pass@32": 0.9189127219974499, + "pass@64": 0.9293832831476638, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.818272440652819, + "pass@2": 0.8724294814131179, + "pass@4": 0.9009509325985585, + "pass@8": 0.9160911680874009, + "pass@16": 0.9262069033688106, + "pass@32": 0.9362831825747088, + "pass@64": 0.9461225848744321, + "pass@128": 0.9525222551928784 + }, + "15": { + "pass@1": 0.7349537037037037, + "pass@2": 0.8040271550257607, + "pass@4": 0.8488606664328994, + "pass@8": 0.8787369042608045, + "pass@16": 0.8999749007648479, + "pass@32": 0.916424294373327, + "pass@64": 0.9278501616190906, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5883803466076696, + "pass@2": 0.7006676364016443, + "pass@4": 0.7826472943647531, + "pass@8": 0.8386241706744149, + "pass@16": 0.8760045352529456, + "pass@32": 0.902053671309776, + "pass@64": 0.9208478831096698, + "pass@128": 0.9321533923303835 + }, + "17": { + "pass@1": 0.48159826807228917, + "pass@2": 0.5913542210179296, + "pass@4": 0.6798522746780143, + "pass@8": 0.7463465014345855, + "pass@16": 0.7951621898319864, + "pass@32": 0.8317659858181697, + "pass@64": 0.8596726321732067, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.42555443548387095, + "pass@2": 0.5410639446278892, + "pass@4": 0.6362736465401501, + "pass@8": 0.7067542320552301, + "pass@16": 0.7546023230010389, + "pass@32": 0.7893975572839265, + "pass@64": 0.8202594715506963, + "pass@128": 0.8483870967741935 + }, + "19": { + "pass@1": 0.24570430513595165, + "pass@2": 0.354450023193853, + "pass@4": 0.468421438539971, + "pass@8": 0.5693508183857078, + "pass@16": 0.6492707930801992, + "pass@32": 0.7127914426913281, + "pass@64": 0.7659437437446937, + "pass@128": 0.8066465256797583 + }, + "20": { + "pass@1": 0.22111955705705705, + "pass@2": 0.32415584876214804, + "pass@4": 0.4372402779607506, + "pass@8": 0.5390823289841321, + "pass@16": 0.6161061994509991, + "pass@32": 0.6710828873695738, + "pass@64": 0.7107390276872974, + "pass@128": 0.7357357357357357 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.8024358552631579, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8024358552631579, + "pass@2": 0.8425471340136529, + "pass@4": 0.8742971099599208, + "pass@8": 0.8985525466069237, + "pass@16": 0.9164654941703156, + "pass@32": 0.9300486433642833, + "pass@64": 0.9408609103045854, + "pass@128": 0.9492105263157895 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9566953125, + "pass@2": 0.9670733267716537, + "pass@4": 0.9742653679227598, + "pass@8": 0.9794420704275232, + "pass@16": 0.98370404958131, + "pass@32": 0.9876131313445727, + "pass@64": 0.9906040315832808, + "pass@128": 0.992 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99990625, + "pass@2": 0.9999918799212598, + "pass@4": 0.9999999535995501, + "pass@8": 0.9999999999996538, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996890625, + "pass@2": 0.9987973671259842, + "pass@4": 0.9996501834458194, + "pass@8": 0.9999408420930652, + "pass@16": 0.9999976881673509, + "pass@32": 0.9999999983745367, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991484375, + "pass@2": 0.9997263779527559, + "pass@4": 0.9999574376640419, + "pass@8": 0.9999986551927124, + "pass@16": 0.999999999143371, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9990546875, + "pass@2": 0.9998537155511811, + "pass@4": 0.9999935515560555, + "pass@8": 0.9999999859843178, + "pass@16": 0.9999999999999764, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9951640625, + "pass@2": 0.9970433070866144, + "pass@4": 0.9982334005436821, + "pass@8": 0.9991791093699898, + "pass@16": 0.9997618776815873, + "pass@32": 0.9999727342955712, + "pass@64": 0.9999998615508054, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9959453125, + "pass@2": 0.9976846702755906, + "pass@4": 0.9985168867641545, + "pass@8": 0.9991088905444924, + "pass@16": 0.999529423673436, + "pass@32": 0.9998286389682483, + "pass@64": 0.9999861762969797, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.954015625, + "pass@2": 0.9644905265748035, + "pass@4": 0.9711644761904762, + "pass@8": 0.9757205330964192, + "pass@16": 0.979031344222367, + "pass@32": 0.9815967967980977, + "pass@64": 0.9838816261970059, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.9025, + "pass@2": 0.9237837106299214, + "pass@4": 0.9366584860329956, + "pass@8": 0.9459949739746121, + "pass@16": 0.9536564122340577, + "pass@32": 0.9606255820011785, + "pass@64": 0.966712341245886, + "pass@128": 0.972 + }, + "13": { + "pass@1": 0.8218515625, + "pass@2": 0.8634689960629935, + "pass@4": 0.8884614780652423, + "pass@8": 0.9052056008326004, + "pass@16": 0.9178599354690884, + "pass@32": 0.9289051742386563, + "pass@64": 0.9387523105805766, + "pass@128": 0.947 + }, + "14": { + "pass@1": 0.83196875, + "pass@2": 0.8842485236220485, + "pass@4": 0.9123400595238086, + "pass@8": 0.9274922279181592, + "pass@16": 0.9369694871852076, + "pass@32": 0.9448955169085047, + "pass@64": 0.9519450244071079, + "pass@128": 0.957 + }, + "15": { + "pass@1": 0.76134375, + "pass@2": 0.8312776820866143, + "pass@4": 0.8747924567866506, + "pass@8": 0.9042052814668986, + "pass@16": 0.9257042732151205, + "pass@32": 0.9409176804179538, + "pass@64": 0.9504559175126083, + "pass@128": 0.956 + }, + "16": { + "pass@1": 0.5898046875, + "pass@2": 0.6973601131889768, + "pass@4": 0.7734774335395578, + "pass@8": 0.8257421433039629, + "pass@16": 0.8615484479408064, + "pass@32": 0.8880162983132416, + "pass@64": 0.9096919211793338, + "pass@128": 0.927 + }, + "17": { + "pass@1": 0.5148046875, + "pass@2": 0.6294880659448818, + "pass@4": 0.7171008025871763, + "pass@8": 0.7792052967778853, + "pass@16": 0.8216423844459994, + "pass@32": 0.8518329273965232, + "pass@64": 0.8754075200291189, + "pass@128": 0.892 + }, + "18": { + "pass@1": 0.4244921875, + "pass@2": 0.540060900590551, + "pass@4": 0.6354181049868758, + "pass@8": 0.7075168907678558, + "pass@16": 0.7601426954402293, + "pass@32": 0.8004978277335945, + "pass@64": 0.832602069078636, + "pass@128": 0.855 + }, + "19": { + "pass@1": 0.266953125, + "pass@2": 0.3751668307086617, + "pass@4": 0.4843383939820021, + "pass@8": 0.5798556956198491, + "pass@16": 0.6542798459127458, + "pass@32": 0.7112011775890043, + "pass@64": 0.7590658635954, + "pass@128": 0.802 + }, + "20": { + "pass@1": 0.2357421875, + "pass@2": 0.33887955216535465, + "pass@4": 0.44727661604799396, + "pass@8": 0.5438901881613861, + "pass@16": 0.619016524923539, + "pass@32": 0.6750207395414648, + "pass@64": 0.7172526325303729, + "pass@128": 0.749 + } + }, + "per_op_accuracy": { + "10": 0.9566953125, + "2": 1.0, + "3": 1.0, + "4": 0.99990625, + "5": 0.996890625, + "6": 0.9991484375, + "7": 0.9990546875, + "8": 0.9951640625, + "9": 0.9959453125, + "11": 0.954015625, + "12": 0.9025, + "13": 0.8218515625, + "14": 0.83196875, + "15": 0.76134375, + "16": 0.5898046875, + "17": 0.5148046875, + "18": 0.4244921875, + "19": 0.266953125, + "20": 0.2357421875 + }, + "per_op_avg_response_len": { + "10": 280.2356640625, + "2": 115.461328125, + "3": 144.5140859375, + "4": 159.5963828125, + "5": 172.538421875, + "6": 198.58234375, + "7": 223.48178125, + "8": 243.0421875, + "9": 258.796109375, + "11": 297.94484375, + "12": 308.3263828125, + "13": 327.143078125, + "14": 354.34553125, + "15": 371.7521640625, + "16": 378.26515625, + "17": 396.573625, + "18": 402.8715703125, + "19": 404.6708515625, + "20": 409.7294375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.2356640625, + "2": 115.461328125, + "3": 144.5140859375, + "4": 159.5963828125, + "5": 172.538421875, + "6": 198.58234375, + "7": 223.48178125, + "8": 243.0421875, + "9": 258.796109375, + "11": 297.94484375, + "12": 308.3263828125, + "13": 327.143078125, + "14": 354.34553125, + "15": 371.7521640625, + "16": 378.26515625, + "17": 396.573625, + "18": 402.8715703125, + "19": 404.6708515625, + "20": 409.7294375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 648616, + "answer_accuracy": 0.8010294815049004, + "avg_response_len": 273.72169049359786, + "resp_tokens_sum": 221640117, + "pass_at_k": { + "pass@1": 0.8010294815049004, + "pass@2": 0.8408840040540062, + "pass@4": 0.8720996222936916, + "pass@8": 0.8959224251689653, + "pass@16": 0.9134960272904331, + "pass@32": 0.9265108894287637, + "pass@64": 0.9365792331039818, + "pass@128": 0.9446727790072715 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9571580014749262, + "pass@2": 0.9677603041599888, + "pass@4": 0.975415939434562, + "pass@8": 0.9812614083839927, + "pass@16": 0.9862952176965449, + "pass@32": 0.9904604242490184, + "pass@64": 0.9931314739448671, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99970703125, + "pass@2": 0.9999746247539371, + "pass@4": 0.9999998549985939, + "pass@8": 0.999999999998918, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966269841269841, + "pass@2": 0.9982509998750156, + "pass@4": 0.9992314291070759, + "pass@8": 0.9998278665841752, + "pass@16": 0.999992699151632, + "pass@32": 0.9999999948398277, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999766092814372, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9983357988165681, + "pass@2": 0.9997022492195872, + "pass@4": 0.9999851042103463, + "pass@8": 0.9999999617664239, + "pass@16": 0.9999999999999302, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.998848157051282, + "pass@2": 0.9999357239299413, + "pass@4": 0.999999570065761, + "pass@8": 0.999999999985574, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9970510563380282, + "pass@2": 0.9981323472330043, + "pass@4": 0.9987858537049066, + "pass@8": 0.9994925064642773, + "pass@16": 0.9999199128774705, + "pass@32": 0.9999987816218918, + "pass@64": 0.9999999999883001, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9444050991501416, + "pass@2": 0.9561910285293657, + "pass@4": 0.9641967786958638, + "pass@8": 0.9697560155386075, + "pass@16": 0.9741350032582587, + "pass@32": 0.9779370472555533, + "pass@64": 0.9815867813603899, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.887601518361582, + "pass@2": 0.9092204807820631, + "pass@4": 0.9226452011295195, + "pass@8": 0.932021504510786, + "pass@16": 0.9383912783274672, + "pass@32": 0.9442462997372086, + "pass@64": 0.9515706040014275, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.8336843652037618, + "pass@2": 0.8732879723792364, + "pass@4": 0.8977824343273704, + "pass@8": 0.9148135688866712, + "pass@16": 0.9285541397791447, + "pass@32": 0.9400278370501557, + "pass@64": 0.9490071310096556, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.8212130062305296, + "pass@2": 0.8690642094095712, + "pass@4": 0.8946342282565142, + "pass@8": 0.9095416152396502, + "pass@16": 0.9205671977126164, + "pass@32": 0.9299201530908431, + "pass@64": 0.9376412005954143, + "pass@128": 0.9439252336448598 + }, + "15": { + "pass@1": 0.7766437861271677, + "pass@2": 0.8473861851986711, + "pass@4": 0.8899432666653658, + "pass@8": 0.9196497067291849, + "pass@16": 0.9396602187962684, + "pass@32": 0.950786723015623, + "pass@64": 0.9560000188004241, + "pass@128": 0.9595375722543352 + }, + "16": { + "pass@1": 0.5931451612903226, + "pass@2": 0.6976770859791713, + "pass@4": 0.7679156642718047, + "pass@8": 0.8140579344020908, + "pass@16": 0.8454150252129901, + "pass@32": 0.8705633373440882, + "pass@64": 0.8931359432887487, + "pass@128": 0.9129032258064517 + }, + "17": { + "pass@1": 0.5168442919075145, + "pass@2": 0.6315512493741748, + "pass@4": 0.7201531642901571, + "pass@8": 0.782651914043665, + "pass@16": 0.8222871470379363, + "pass@32": 0.8481959011248583, + "pass@64": 0.8685797265002355, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.4018526785714286, + "pass@2": 0.5119948678290213, + "pass@4": 0.6041472810541539, + "pass@8": 0.676469614184862, + "pass@16": 0.7335479686894587, + "pass@32": 0.7793197198583286, + "pass@64": 0.8148807734554173, + "pass@128": 0.8371428571428572 + }, + "19": { + "pass@1": 0.2663522897897898, + "pass@2": 0.37801704263318436, + "pass@4": 0.48845757090048425, + "pass@8": 0.5820235208190736, + "pass@16": 0.6525778863987478, + "pass@32": 0.7028863331400599, + "pass@64": 0.7412004878213567, + "pass@128": 0.7747747747747747 + }, + "20": { + "pass@1": 0.25844594594594594, + "pass@2": 0.3624208608214512, + "pass@4": 0.4657998673589216, + "pass@8": 0.5561911364487827, + "pass@16": 0.6273839014887124, + "pass@32": 0.6797247642370513, + "pass@64": 0.7172722633189874, + "pass@128": 0.7477477477477478 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 664261, + "answer_accuracy": 0.8089694563522993, + "avg_response_len": 283.20430387763054, + "resp_tokens_sum": 232544718, + "pass_at_k": { + "pass@1": 0.8089694563522993, + "pass@2": 0.8490546041205046, + "pass@4": 0.880152468860326, + "pass@8": 0.9036864510929692, + "pass@16": 0.9212574169827573, + "pass@32": 0.934904097903735, + "pass@64": 0.9461935629303051, + "pass@128": 0.9552611067809821 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9449290644171779, + "pass@2": 0.9567034805081881, + "pass@4": 0.9647007856763299, + "pass@8": 0.9702237713814276, + "pass@16": 0.9745168430378548, + "pass@32": 0.9787785426264238, + "pass@64": 0.9824400169645098, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994338768115942, + "pass@2": 0.999957563049184, + "pass@4": 0.9999997090037658, + "pass@8": 0.9999999999939099, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9989255014326648, + "pass@2": 0.999798707781513, + "pass@4": 0.9999920000150412, + "pass@8": 0.9999999882211767, + "pass@16": 0.9999999999999954, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9990742413294798, + "pass@2": 0.9998716347685586, + "pass@4": 0.9999959156117046, + "pass@8": 0.9999999968418107, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954885563380281, + "pass@2": 0.9974222302317843, + "pass@4": 0.9985825871237926, + "pass@8": 0.999402550678006, + "pass@16": 0.9998887161460439, + "pass@32": 0.9999974897170781, + "pass@64": 0.9999999999243792, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9953920284431138, + "pass@2": 0.9969231075251073, + "pass@4": 0.9975031138510381, + "pass@8": 0.9979833170387231, + "pass@16": 0.998679312074329, + "pass@32": 0.9994882388094946, + "pass@64": 0.9999586116800393, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9630788780120482, + "pass@2": 0.9706936752917183, + "pass@4": 0.9737727306865254, + "pass@8": 0.9753456810706799, + "pass@16": 0.9769636060644346, + "pass@32": 0.9788680562854182, + "pass@64": 0.9808120954397608, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8993681426332288, + "pass@2": 0.9229849832152642, + "pass@4": 0.9366999725543712, + "pass@8": 0.9458536190031113, + "pass@16": 0.9535302205560833, + "pass@32": 0.9602898879824436, + "pass@64": 0.9661108029867137, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.8356530112044818, + "pass@2": 0.8733330438474605, + "pass@4": 0.8943991720047597, + "pass@8": 0.9074957916972102, + "pass@16": 0.9178935997150323, + "pass@32": 0.9280352165055543, + "pass@64": 0.9380920225452458, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.8555601242690059, + "pass@2": 0.9101467168577612, + "pass@4": 0.940181309847234, + "pass@8": 0.9555749876630905, + "pass@16": 0.9629698546320736, + "pass@32": 0.967437763855752, + "pass@64": 0.9711078593956024, + "pass@128": 0.9736842105263158 + }, + "15": { + "pass@1": 0.7712121212121212, + "pass@2": 0.8411432235743264, + "pass@4": 0.8843673654429557, + "pass@8": 0.9130173513881795, + "pass@16": 0.9363332414054601, + "pass@32": 0.9546181905381521, + "pass@64": 0.9668377534638671, + "pass@128": 0.9727272727272728 + }, + "16": { + "pass@1": 0.5882300569800569, + "pass@2": 0.69388572021446, + "pass@4": 0.7695331761414431, + "pass@8": 0.8236199139050931, + "pass@16": 0.8618354777037932, + "pass@32": 0.8898731315172652, + "pass@64": 0.9135394483921452, + "pass@128": 0.9344729344729344 + }, + "17": { + "pass@1": 0.5468507375776398, + "pass@2": 0.6665892306939891, + "pass@4": 0.7522262501542892, + "pass@8": 0.8093810436102301, + "pass@16": 0.8482521259212884, + "pass@32": 0.8764311748934481, + "pass@64": 0.898967766392957, + "pass@128": 0.9130434782608695 + }, + "18": { + "pass@1": 0.44682904411764707, + "pass@2": 0.5680378647522, + "pass@4": 0.666828606442577, + "pass@8": 0.7401726878412729, + "pass@16": 0.792571136084111, + "pass@32": 0.8324196559563581, + "pass@64": 0.8620981240838929, + "pass@128": 0.8794117647058823 + }, + "19": { + "pass@1": 0.28848121279761907, + "pass@2": 0.39275058879358826, + "pass@4": 0.495936091414912, + "pass@8": 0.5880557806590099, + "pass@16": 0.6609011227155269, + "pass@32": 0.717875181912308, + "pass@64": 0.7699961963434331, + "pass@128": 0.8244047619047619 + }, + "20": { + "pass@1": 0.22768525449101795, + "pass@2": 0.33008834692819083, + "pass@4": 0.43881511274414076, + "pass@8": 0.5364195334497773, + "pass@16": 0.6135758123072342, + "pass@32": 0.6742568609474818, + "pass@64": 0.7237271635487402, + "pass@128": 0.7634730538922155 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 638647, + "answer_accuracy": 0.7971608383927145, + "avg_response_len": 303.4912800567183, + "resp_tokens_sum": 243142646, + "pass_at_k": { + "pass@1": 0.7971608383927145, + "pass@2": 0.8375584040870843, + "pass@4": 0.8705168223150448, + "pass@8": 0.8959489599218805, + "pass@16": 0.9145553907417542, + "pass@32": 0.9286477950698765, + "pass@64": 0.9397228568419085, + "pass@128": 0.9475954625339511 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9676772388059701, + "pass@2": 0.9764694000470092, + "pass@4": 0.9824086815640581, + "pass@8": 0.9865716522835937, + "pass@16": 0.9900223282144495, + "pass@32": 0.9933290824714656, + "pass@64": 0.9959910936881807, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9945542279411764, + "pass@2": 0.9981263026864289, + "pass@4": 0.9996834813846798, + "pass@8": 0.9999854827092686, + "pass@16": 0.9999999645134902, + "pass@32": 0.9999999999999732, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985212933753943, + "pass@2": 0.9993584509053877, + "pass@4": 0.9998745415103868, + "pass@8": 0.9999957706735703, + "pass@16": 0.9999999972977055, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998022151898734, + "pass@2": 0.999996106598226, + "pass@4": 0.9999999985167994, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9913663663663663, + "pass@2": 0.9939293378023693, + "pass@4": 0.9962063551177722, + "pass@8": 0.9981717834348306, + "pass@16": 0.9994035539031283, + "pass@32": 0.9999207971351605, + "pass@64": 0.99999958431727, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9952773311897106, + "pass@2": 0.9979915404967465, + "pass@4": 0.9992986130953887, + "pass@8": 0.9998798162660464, + "pass@16": 0.9999966699971327, + "pass@32": 0.9999999991321724, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9552331349206349, + "pass@2": 0.9672533120859892, + "pass@4": 0.9762237038822525, + "pass@8": 0.9827996618915072, + "pass@16": 0.9866976852660617, + "pass@32": 0.9885740458099326, + "pass@64": 0.9896884974310719, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9216838685015291, + "pass@2": 0.9403285956801272, + "pass@4": 0.9517883595972522, + "pass@8": 0.9612601128923591, + "pass@16": 0.970305074460992, + "pass@32": 0.978684763387789, + "pass@64": 0.9836911354025035, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7949942129629629, + "pass@2": 0.8429328278409639, + "pass@4": 0.8727419046577508, + "pass@8": 0.8932224528452122, + "pass@16": 0.9072936721644289, + "pass@32": 0.9189127219974499, + "pass@64": 0.9293832831476638, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.818272440652819, + "pass@2": 0.8724294814131179, + "pass@4": 0.9009509325985585, + "pass@8": 0.9160911680874009, + "pass@16": 0.9262069033688106, + "pass@32": 0.9362831825747088, + "pass@64": 0.9461225848744321, + "pass@128": 0.9525222551928784 + }, + "15": { + "pass@1": 0.7349537037037037, + "pass@2": 0.8040271550257607, + "pass@4": 0.8488606664328994, + "pass@8": 0.8787369042608045, + "pass@16": 0.8999749007648479, + "pass@32": 0.916424294373327, + "pass@64": 0.9278501616190906, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5883803466076696, + "pass@2": 0.7006676364016443, + "pass@4": 0.7826472943647531, + "pass@8": 0.8386241706744149, + "pass@16": 0.8760045352529456, + "pass@32": 0.902053671309776, + "pass@64": 0.9208478831096698, + "pass@128": 0.9321533923303835 + }, + "17": { + "pass@1": 0.48159826807228917, + "pass@2": 0.5913542210179296, + "pass@4": 0.6798522746780143, + "pass@8": 0.7463465014345855, + "pass@16": 0.7951621898319864, + "pass@32": 0.8317659858181697, + "pass@64": 0.8596726321732067, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.42555443548387095, + "pass@2": 0.5410639446278892, + "pass@4": 0.6362736465401501, + "pass@8": 0.7067542320552301, + "pass@16": 0.7546023230010389, + "pass@32": 0.7893975572839265, + "pass@64": 0.8202594715506963, + "pass@128": 0.8483870967741935 + }, + "19": { + "pass@1": 0.24570430513595165, + "pass@2": 0.354450023193853, + "pass@4": 0.468421438539971, + "pass@8": 0.5693508183857078, + "pass@16": 0.6492707930801992, + "pass@32": 0.7127914426913281, + "pass@64": 0.7659437437446937, + "pass@128": 0.8066465256797583 + }, + "20": { + "pass@1": 0.22111955705705705, + "pass@2": 0.32415584876214804, + "pass@4": 0.4372402779607506, + "pass@8": 0.5390823289841321, + "pass@16": 0.6161061994509991, + "pass@32": 0.6710828873695738, + "pass@64": 0.7107390276872974, + "pass@128": 0.7357357357357357 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep4-block4-op11-14/composition/autoregressive/summary.csv b/SDAR-100M-ep4-block4-op11-14/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..98d6a9cfa02f93d1a01e96c982dbb7123209efac --- /dev/null +++ b/SDAR-100M-ep4-block4-op11-14/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.9936449652777778,0.63034765625,0.8024358552631579,0.0,0.0,0.0,199.58314496527777,365.1622640625,1152000,1280000,2432000 diff --git a/SDAR-100M-ep4-block4-op11-14/composition/autoregressive/summary.json b/SDAR-100M-ep4-block4-op11-14/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..95258b085681842fe6c4de00cf5b770ebe4d6aaf --- /dev/null +++ b/SDAR-100M-ep4-block4-op11-14/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.9936449652777778, + "ood_acc": 0.63034765625, + "total_acc": 0.8024358552631579, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.58314496527777, + "ood_avg_resp_len": 365.1622640625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 648616, + "answer_accuracy": 0.8010294815049004, + "avg_response_len": 273.72169049359786, + "resp_tokens_sum": 221640117, + "pass_at_k": { + "pass@1": 0.8010294815049004, + "pass@2": 0.8408840040540062, + "pass@4": 0.8720996222936916, + "pass@8": 0.8959224251689653, + "pass@16": 0.9134960272904331, + "pass@32": 0.9265108894287637, + "pass@64": 0.9365792331039818, + "pass@128": 0.9446727790072715 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9571580014749262, + "pass@2": 0.9677603041599888, + "pass@4": 0.975415939434562, + "pass@8": 0.9812614083839927, + "pass@16": 0.9862952176965449, + "pass@32": 0.9904604242490184, + "pass@64": 0.9931314739448671, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99970703125, + "pass@2": 0.9999746247539371, + "pass@4": 0.9999998549985939, + "pass@8": 0.999999999998918, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966269841269841, + "pass@2": 0.9982509998750156, + "pass@4": 0.9992314291070759, + "pass@8": 0.9998278665841752, + "pass@16": 0.999992699151632, + "pass@32": 0.9999999948398277, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999766092814372, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9983357988165681, + "pass@2": 0.9997022492195872, + "pass@4": 0.9999851042103463, + "pass@8": 0.9999999617664239, + "pass@16": 0.9999999999999302, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.998848157051282, + "pass@2": 0.9999357239299413, + "pass@4": 0.999999570065761, + "pass@8": 0.999999999985574, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9970510563380282, + "pass@2": 0.9981323472330043, + "pass@4": 0.9987858537049066, + "pass@8": 0.9994925064642773, + "pass@16": 0.9999199128774705, + "pass@32": 0.9999987816218918, + "pass@64": 0.9999999999883001, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9444050991501416, + "pass@2": 0.9561910285293657, + "pass@4": 0.9641967786958638, + "pass@8": 0.9697560155386075, + "pass@16": 0.9741350032582587, + "pass@32": 0.9779370472555533, + "pass@64": 0.9815867813603899, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.887601518361582, + "pass@2": 0.9092204807820631, + "pass@4": 0.9226452011295195, + "pass@8": 0.932021504510786, + "pass@16": 0.9383912783274672, + "pass@32": 0.9442462997372086, + "pass@64": 0.9515706040014275, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.8336843652037618, + "pass@2": 0.8732879723792364, + "pass@4": 0.8977824343273704, + "pass@8": 0.9148135688866712, + "pass@16": 0.9285541397791447, + "pass@32": 0.9400278370501557, + "pass@64": 0.9490071310096556, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.8212130062305296, + "pass@2": 0.8690642094095712, + "pass@4": 0.8946342282565142, + "pass@8": 0.9095416152396502, + "pass@16": 0.9205671977126164, + "pass@32": 0.9299201530908431, + "pass@64": 0.9376412005954143, + "pass@128": 0.9439252336448598 + }, + "15": { + "pass@1": 0.7766437861271677, + "pass@2": 0.8473861851986711, + "pass@4": 0.8899432666653658, + "pass@8": 0.9196497067291849, + "pass@16": 0.9396602187962684, + "pass@32": 0.950786723015623, + "pass@64": 0.9560000188004241, + "pass@128": 0.9595375722543352 + }, + "16": { + "pass@1": 0.5931451612903226, + "pass@2": 0.6976770859791713, + "pass@4": 0.7679156642718047, + "pass@8": 0.8140579344020908, + "pass@16": 0.8454150252129901, + "pass@32": 0.8705633373440882, + "pass@64": 0.8931359432887487, + "pass@128": 0.9129032258064517 + }, + "17": { + "pass@1": 0.5168442919075145, + "pass@2": 0.6315512493741748, + "pass@4": 0.7201531642901571, + "pass@8": 0.782651914043665, + "pass@16": 0.8222871470379363, + "pass@32": 0.8481959011248583, + "pass@64": 0.8685797265002355, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.4018526785714286, + "pass@2": 0.5119948678290213, + "pass@4": 0.6041472810541539, + "pass@8": 0.676469614184862, + "pass@16": 0.7335479686894587, + "pass@32": 0.7793197198583286, + "pass@64": 0.8148807734554173, + "pass@128": 0.8371428571428572 + }, + "19": { + "pass@1": 0.2663522897897898, + "pass@2": 0.37801704263318436, + "pass@4": 0.48845757090048425, + "pass@8": 0.5820235208190736, + "pass@16": 0.6525778863987478, + "pass@32": 0.7028863331400599, + "pass@64": 0.7412004878213567, + "pass@128": 0.7747747747747747 + }, + "20": { + "pass@1": 0.25844594594594594, + "pass@2": 0.3624208608214512, + "pass@4": 0.4657998673589216, + "pass@8": 0.5561911364487827, + "pass@16": 0.6273839014887124, + "pass@32": 0.6797247642370513, + "pass@64": 0.7172722633189874, + "pass@128": 0.7477477477477478 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 664261, + "answer_accuracy": 0.8089694563522993, + "avg_response_len": 283.20430387763054, + "resp_tokens_sum": 232544718, + "pass_at_k": { + "pass@1": 0.8089694563522993, + "pass@2": 0.8490546041205046, + "pass@4": 0.880152468860326, + "pass@8": 0.9036864510929692, + "pass@16": 0.9212574169827573, + "pass@32": 0.934904097903735, + "pass@64": 0.9461935629303051, + "pass@128": 0.9552611067809821 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9449290644171779, + "pass@2": 0.9567034805081881, + "pass@4": 0.9647007856763299, + "pass@8": 0.9702237713814276, + "pass@16": 0.9745168430378548, + "pass@32": 0.9787785426264238, + "pass@64": 0.9824400169645098, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994338768115942, + "pass@2": 0.999957563049184, + "pass@4": 0.9999997090037658, + "pass@8": 0.9999999999939099, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9989255014326648, + "pass@2": 0.999798707781513, + "pass@4": 0.9999920000150412, + "pass@8": 0.9999999882211767, + "pass@16": 0.9999999999999954, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9990742413294798, + "pass@2": 0.9998716347685586, + "pass@4": 0.9999959156117046, + "pass@8": 0.9999999968418107, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954885563380281, + "pass@2": 0.9974222302317843, + "pass@4": 0.9985825871237926, + "pass@8": 0.999402550678006, + "pass@16": 0.9998887161460439, + "pass@32": 0.9999974897170781, + "pass@64": 0.9999999999243792, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9953920284431138, + "pass@2": 0.9969231075251073, + "pass@4": 0.9975031138510381, + "pass@8": 0.9979833170387231, + "pass@16": 0.998679312074329, + "pass@32": 0.9994882388094946, + "pass@64": 0.9999586116800393, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9630788780120482, + "pass@2": 0.9706936752917183, + "pass@4": 0.9737727306865254, + "pass@8": 0.9753456810706799, + "pass@16": 0.9769636060644346, + "pass@32": 0.9788680562854182, + "pass@64": 0.9808120954397608, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8993681426332288, + "pass@2": 0.9229849832152642, + "pass@4": 0.9366999725543712, + "pass@8": 0.9458536190031113, + "pass@16": 0.9535302205560833, + "pass@32": 0.9602898879824436, + "pass@64": 0.9661108029867137, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.8356530112044818, + "pass@2": 0.8733330438474605, + "pass@4": 0.8943991720047597, + "pass@8": 0.9074957916972102, + "pass@16": 0.9178935997150323, + "pass@32": 0.9280352165055543, + "pass@64": 0.9380920225452458, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.8555601242690059, + "pass@2": 0.9101467168577612, + "pass@4": 0.940181309847234, + "pass@8": 0.9555749876630905, + "pass@16": 0.9629698546320736, + "pass@32": 0.967437763855752, + "pass@64": 0.9711078593956024, + "pass@128": 0.9736842105263158 + }, + "15": { + "pass@1": 0.7712121212121212, + "pass@2": 0.8411432235743264, + "pass@4": 0.8843673654429557, + "pass@8": 0.9130173513881795, + "pass@16": 0.9363332414054601, + "pass@32": 0.9546181905381521, + "pass@64": 0.9668377534638671, + "pass@128": 0.9727272727272728 + }, + "16": { + "pass@1": 0.5882300569800569, + "pass@2": 0.69388572021446, + "pass@4": 0.7695331761414431, + "pass@8": 0.8236199139050931, + "pass@16": 0.8618354777037932, + "pass@32": 0.8898731315172652, + "pass@64": 0.9135394483921452, + "pass@128": 0.9344729344729344 + }, + "17": { + "pass@1": 0.5468507375776398, + "pass@2": 0.6665892306939891, + "pass@4": 0.7522262501542892, + "pass@8": 0.8093810436102301, + "pass@16": 0.8482521259212884, + "pass@32": 0.8764311748934481, + "pass@64": 0.898967766392957, + "pass@128": 0.9130434782608695 + }, + "18": { + "pass@1": 0.44682904411764707, + "pass@2": 0.5680378647522, + "pass@4": 0.666828606442577, + "pass@8": 0.7401726878412729, + "pass@16": 0.792571136084111, + "pass@32": 0.8324196559563581, + "pass@64": 0.8620981240838929, + "pass@128": 0.8794117647058823 + }, + "19": { + "pass@1": 0.28848121279761907, + "pass@2": 0.39275058879358826, + "pass@4": 0.495936091414912, + "pass@8": 0.5880557806590099, + "pass@16": 0.6609011227155269, + "pass@32": 0.717875181912308, + "pass@64": 0.7699961963434331, + "pass@128": 0.8244047619047619 + }, + "20": { + "pass@1": 0.22768525449101795, + "pass@2": 0.33008834692819083, + "pass@4": 0.43881511274414076, + "pass@8": 0.5364195334497773, + "pass@16": 0.6135758123072342, + "pass@32": 0.6742568609474818, + "pass@64": 0.7237271635487402, + "pass@128": 0.7634730538922155 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 638647, + "answer_accuracy": 0.7971608383927145, + "avg_response_len": 303.4912800567183, + "resp_tokens_sum": 243142646, + "pass_at_k": { + "pass@1": 0.7971608383927145, + "pass@2": 0.8375584040870843, + "pass@4": 0.8705168223150448, + "pass@8": 0.8959489599218805, + "pass@16": 0.9145553907417542, + "pass@32": 0.9286477950698765, + "pass@64": 0.9397228568419085, + "pass@128": 0.9475954625339511 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9676772388059701, + "pass@2": 0.9764694000470092, + "pass@4": 0.9824086815640581, + "pass@8": 0.9865716522835937, + "pass@16": 0.9900223282144495, + "pass@32": 0.9933290824714656, + "pass@64": 0.9959910936881807, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9945542279411764, + "pass@2": 0.9981263026864289, + "pass@4": 0.9996834813846798, + "pass@8": 0.9999854827092686, + "pass@16": 0.9999999645134902, + "pass@32": 0.9999999999999732, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985212933753943, + "pass@2": 0.9993584509053877, + "pass@4": 0.9998745415103868, + "pass@8": 0.9999957706735703, + "pass@16": 0.9999999972977055, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998022151898734, + "pass@2": 0.999996106598226, + "pass@4": 0.9999999985167994, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9913663663663663, + "pass@2": 0.9939293378023693, + "pass@4": 0.9962063551177722, + "pass@8": 0.9981717834348306, + "pass@16": 0.9994035539031283, + "pass@32": 0.9999207971351605, + "pass@64": 0.99999958431727, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9952773311897106, + "pass@2": 0.9979915404967465, + "pass@4": 0.9992986130953887, + "pass@8": 0.9998798162660464, + "pass@16": 0.9999966699971327, + "pass@32": 0.9999999991321724, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9552331349206349, + "pass@2": 0.9672533120859892, + "pass@4": 0.9762237038822525, + "pass@8": 0.9827996618915072, + "pass@16": 0.9866976852660617, + "pass@32": 0.9885740458099326, + "pass@64": 0.9896884974310719, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9216838685015291, + "pass@2": 0.9403285956801272, + "pass@4": 0.9517883595972522, + "pass@8": 0.9612601128923591, + "pass@16": 0.970305074460992, + "pass@32": 0.978684763387789, + "pass@64": 0.9836911354025035, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7949942129629629, + "pass@2": 0.8429328278409639, + "pass@4": 0.8727419046577508, + "pass@8": 0.8932224528452122, + "pass@16": 0.9072936721644289, + "pass@32": 0.9189127219974499, + "pass@64": 0.9293832831476638, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.818272440652819, + "pass@2": 0.8724294814131179, + "pass@4": 0.9009509325985585, + "pass@8": 0.9160911680874009, + "pass@16": 0.9262069033688106, + "pass@32": 0.9362831825747088, + "pass@64": 0.9461225848744321, + "pass@128": 0.9525222551928784 + }, + "15": { + "pass@1": 0.7349537037037037, + "pass@2": 0.8040271550257607, + "pass@4": 0.8488606664328994, + "pass@8": 0.8787369042608045, + "pass@16": 0.8999749007648479, + "pass@32": 0.916424294373327, + "pass@64": 0.9278501616190906, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5883803466076696, + "pass@2": 0.7006676364016443, + "pass@4": 0.7826472943647531, + "pass@8": 0.8386241706744149, + "pass@16": 0.8760045352529456, + "pass@32": 0.902053671309776, + "pass@64": 0.9208478831096698, + "pass@128": 0.9321533923303835 + }, + "17": { + "pass@1": 0.48159826807228917, + "pass@2": 0.5913542210179296, + "pass@4": 0.6798522746780143, + "pass@8": 0.7463465014345855, + "pass@16": 0.7951621898319864, + "pass@32": 0.8317659858181697, + "pass@64": 0.8596726321732067, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.42555443548387095, + "pass@2": 0.5410639446278892, + "pass@4": 0.6362736465401501, + "pass@8": 0.7067542320552301, + "pass@16": 0.7546023230010389, + "pass@32": 0.7893975572839265, + "pass@64": 0.8202594715506963, + "pass@128": 0.8483870967741935 + }, + "19": { + "pass@1": 0.24570430513595165, + "pass@2": 0.354450023193853, + "pass@4": 0.468421438539971, + "pass@8": 0.5693508183857078, + "pass@16": 0.6492707930801992, + "pass@32": 0.7127914426913281, + "pass@64": 0.7659437437446937, + "pass@128": 0.8066465256797583 + }, + "20": { + "pass@1": 0.22111955705705705, + "pass@2": 0.32415584876214804, + "pass@4": 0.4372402779607506, + "pass@8": 0.5390823289841321, + "pass@16": 0.6161061994509991, + "pass@32": 0.6710828873695738, + "pass@64": 0.7107390276872974, + "pass@128": 0.7357357357357357 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep4-block4-op7-10/composition/autoregressive/hf_model_metrics.json b/SDAR-100M-ep4-block4-op7-10/composition/autoregressive/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..42aa6b00c04965721f712701b3c177565a11da1e --- /dev/null +++ b/SDAR-100M-ep4-block4-op7-10/composition/autoregressive/hf_model_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.9936180555555556, + "avg_response_len": 200.06038802083333, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9936180555555556, + "pass@2": 0.9955690480096231, + "pass@4": 0.9968625053534975, + "pass@8": 0.9977628403418239, + "pass@16": 0.9984271892277738, + "pass@32": 0.9989105707376555, + "pass@64": 0.9991623810379424, + "pass@128": 0.9992222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9551484375, + "pass@2": 0.9663884104330706, + "pass@4": 0.9748713232095988, + "pass@8": 0.9815256299919862, + "pass@16": 0.9870153172800666, + "pass@32": 0.9912142150975694, + "pass@64": 0.9934614913928741, + "pass@128": 0.994 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9975234375, + "pass@2": 0.9989872047244095, + "pass@4": 0.9996471928196476, + "pass@8": 0.999929016754186, + "pass@16": 0.9999966016697285, + "pass@32": 0.9999999961558147, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993828125, + "pass@2": 0.9998635580708661, + "pass@4": 0.9999914046681665, + "pass@8": 0.9999999642417031, + "pass@16": 0.9999999999997616, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99909375, + "pass@2": 0.9997829724409449, + "pass@4": 0.9999779075740532, + "pass@8": 0.9999996831294576, + "pass@16": 0.9999999999641311, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995859375, + "pass@2": 0.9979446358267716, + "pass@4": 0.9992358772965879, + "pass@8": 0.9998611370304065, + "pass@16": 0.9999938475734018, + "pass@32": 0.9999999908391373, + "pass@64": 0.9999999999999999, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9955703125, + "pass@2": 0.9971546505905512, + "pass@4": 0.9980388426134232, + "pass@8": 0.9985501319286774, + "pass@16": 0.998838936562875, + "pass@32": 0.9989809345463803, + "pass@64": 0.9989999379486069, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.9551484375, + "2": 1.0, + "3": 0.9999921875, + "4": 0.9999921875, + "5": 0.9975234375, + "6": 0.9993828125, + "7": 0.99909375, + "8": 0.995859375, + "9": 0.9955703125 + }, + "per_op_avg_response_len": { + "10": 281.0439375, + "2": 115.6566484375, + "3": 144.7500078125, + "4": 159.92284375, + "5": 172.8460625, + "6": 199.080953125, + "7": 224.0167421875, + "8": 243.6860234375, + "9": 259.5402734375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.0439375, + "2": 115.6566484375, + "3": 144.7500078125, + "4": 159.92284375, + "5": 172.8460625, + "6": 199.080953125, + "7": 224.0167421875, + "8": 243.6860234375, + "9": 259.5402734375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 377071, + "answer_accuracy": 0.9948892899358325, + "avg_response_len": 190.60182898514014, + "resp_tokens_sum": 72239618, + "pass_at_k": { + "pass@1": 0.9948892899358325, + "pass@2": 0.9965707411573554, + "pass@4": 0.997470881141124, + "pass@8": 0.9980432608401829, + "pass@16": 0.9985302799834912, + "pass@32": 0.9989539595034274, + "pass@64": 0.9992181992422777, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9642791297935103, + "pass@2": 0.9752165935010332, + "pass@4": 0.9816987470039695, + "pass@8": 0.9860516756202101, + "pass@16": 0.9901225277302209, + "pass@32": 0.9938132092442535, + "pass@64": 0.9961212034111633, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999755859375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996875, + "pass@2": 0.9982318616422946, + "pass@4": 0.9991640568738431, + "pass@8": 0.9997933137781767, + "pass@16": 0.9999892767188607, + "pass@32": 0.9999999877963381, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9980815458579881, + "pass@2": 0.9994521822438617, + "pass@4": 0.9999370938469968, + "pass@8": 0.9999990635278279, + "pass@16": 0.9999999998938789, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9993739983974359, + "pass@2": 0.9999842267312739, + "pass@4": 0.9999999876818282, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966549295774648, + "pass@2": 0.9971678496173894, + "pass@4": 0.9971830761471718, + "pass@8": 0.9971830985915474, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387733, + "answer_accuracy": 0.9925177137942333, + "avg_response_len": 198.0554579988532, + "resp_tokens_sum": 77371553, + "pass_at_k": { + "pass@1": 0.9925177137942333, + "pass@2": 0.9945164313964767, + "pass@4": 0.9960538279803, + "pass@8": 0.9972449708430466, + "pass@16": 0.9981262094351478, + "pass@32": 0.9987091289508245, + "pass@64": 0.9989753851362442, + "pass@128": 0.9990170380078637 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9419814033742331, + "pass@2": 0.9546104354862087, + "pass@4": 0.9660200076600852, + "pass@8": 0.9756070749670658, + "pass@16": 0.9829531280350566, + "pass@32": 0.987973396368052, + "pass@64": 0.9904077837030997, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995471014492754, + "pass@2": 0.9999711143444027, + "pass@4": 0.9999999087070638, + "pass@8": 0.9999999999998904, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991045845272206, + "pass@2": 0.9998568745346659, + "pass@4": 0.9999951975917051, + "pass@8": 0.9999999955503199, + "pass@16": 0.9999999999999997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9994355130057804, + "pass@2": 0.9999100376632835, + "pass@4": 0.9999976010043543, + "pass@8": 0.9999999990088202, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9979289120550073, + "pass@4": 0.9992789157517282, + "pass@8": 0.9998878246175227, + "pass@16": 0.9999970717408715, + "pass@32": 0.9999999992397323, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947838697604791, + "pass@2": 0.9966689406619832, + "pass@4": 0.9978810868139987, + "pass@8": 0.998753199791672, + "pass@16": 0.9995194939779444, + "pass@32": 0.9999429179994794, + "pass@64": 0.9999998142173857, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379844, + "answer_accuracy": 0.9934821727485772, + "avg_response_len": 211.48517534315366, + "resp_tokens_sum": 80858396, + "pass_at_k": { + "pass@1": 0.9934821727485772, + "pass@2": 0.9956515965509334, + "pass@4": 0.997085700075907, + "pass@8": 0.9980139995700226, + "pass@16": 0.9986325252168654, + "pass@32": 0.9990733849987735, + "pass@64": 0.9992981138096013, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9587220149253731, + "pass@2": 0.9689163679633328, + "pass@4": 0.9765758969494485, + "pass@8": 0.9827050910969309, + "pass@16": 0.9878240616122193, + "pass@32": 0.9917379401426335, + "pass@64": 0.9937416894008326, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9960707720588236, + "pass@2": 0.9986886289949051, + "pass@4": 0.9997369011777938, + "pass@8": 0.9999827144532304, + "pass@16": 0.9999999397155509, + "pass@32": 0.9999999999999061, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990388406940063, + "pass@2": 0.9997271572816017, + "pass@4": 0.9999781725825281, + "pass@8": 0.9999998920966604, + "pass@16": 0.9999999999992483, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998022151898734, + "pass@2": 0.9999976639589356, + "pass@4": 0.99999999970336, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9926801801801802, + "pass@2": 0.9960504303516113, + "pass@4": 0.9984740722161982, + "pass@8": 0.9997025804540118, + "pass@16": 0.9999846459621398, + "pass@32": 0.9999999733003972, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9951768488745981, + "pass@2": 0.9976612147758058, + "pass@4": 0.9991851047758901, + "pass@8": 0.9998924861680374, + "pass@16": 0.9999981529718375, + "pass@32": 0.9999999997895631, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.49039140625, + "avg_response_len": 341.833584375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.49039140625, + "pass@2": 0.565116584645663, + "pass@4": 0.629092688526423, + "pass@8": 0.6824067105341796, + "pass@16": 0.7258529843659807, + "pass@32": 0.7613167049767667, + "pass@64": 0.7909956507170707, + "pass@128": 0.8167 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.93753125, + "pass@2": 0.952107652559056, + "pass@4": 0.9608752785901759, + "pass@8": 0.966837090293199, + "pass@16": 0.9712662520099715, + "pass@32": 0.9750985201671579, + "pass@64": 0.9787849493437966, + "pass@128": 0.983 + }, + "12": { + "pass@1": 0.8597734375, + "pass@2": 0.892402066929135, + "pass@4": 0.9108887488751395, + "pass@8": 0.9240386006775467, + "pass@16": 0.9349088312957975, + "pass@32": 0.9441167740981736, + "pass@64": 0.9519940852790811, + "pass@128": 0.959 + }, + "13": { + "pass@1": 0.717328125, + "pass@2": 0.782937376968504, + "pass@4": 0.8259061431383582, + "pass@8": 0.8565334879096712, + "pass@16": 0.8789070749207669, + "pass@32": 0.8966054803216155, + "pass@64": 0.9114685642450719, + "pass@128": 0.923 + }, + "14": { + "pass@1": 0.66265625, + "pass@2": 0.7544363927165362, + "pass@4": 0.8163625853018365, + "pass@8": 0.8568341070871195, + "pass@16": 0.8846559648793955, + "pass@32": 0.904967434710607, + "pass@64": 0.919711784694804, + "pass@128": 0.931 + }, + "15": { + "pass@1": 0.5335, + "pass@2": 0.6419622293307092, + "pass@4": 0.7254633570491189, + "pass@8": 0.7885797938158986, + "pass@16": 0.8338857129014781, + "pass@32": 0.864596193927383, + "pass@64": 0.8863006035044457, + "pass@128": 0.904 + }, + "16": { + "pass@1": 0.3466796875, + "pass@2": 0.45398905019685043, + "pass@4": 0.5483833412073488, + "pass@8": 0.6254342073768804, + "pass@16": 0.6870406243039331, + "pass@32": 0.7367142515881637, + "pass@64": 0.7788045524578896, + "pass@128": 0.815 + }, + "17": { + "pass@1": 0.2726953125, + "pass@2": 0.37228063484252, + "pass@4": 0.46829908867641445, + "pass@8": 0.5509332255558134, + "pass@16": 0.6186111522725348, + "pass@32": 0.6761452165748015, + "pass@64": 0.7280746231437759, + "pass@128": 0.774 + }, + "18": { + "pass@1": 0.2377109375, + "pass@2": 0.3227122293307086, + "pass@4": 0.40548861529808733, + "pass@8": 0.48134990382786125, + "pass@16": 0.5476120764861299, + "pass@32": 0.6033783062484257, + "pass@64": 0.649587967249864, + "pass@128": 0.69 + }, + "19": { + "pass@1": 0.17771875, + "pass@2": 0.25335076279527563, + "pass@4": 0.33247645331833486, + "pass@8": 0.4060979696332052, + "pass@16": 0.4702402478312121, + "pass@32": 0.5240652097765713, + "pass@64": 0.5687493735277911, + "pass@128": 0.61 + }, + "20": { + "pass@1": 0.1583203125, + "pass@2": 0.2249874507874015, + "pass@4": 0.29678327380952363, + "pass@8": 0.36742871916460595, + "pass@16": 0.43140190675843637, + "pass@32": 0.4874796623547377, + "pass@64": 0.5364800037240876, + "pass@128": 0.578 + } + }, + "per_op_accuracy": { + "11": 0.93753125, + "12": 0.8597734375, + "13": 0.717328125, + "14": 0.66265625, + "15": 0.5335, + "16": 0.3466796875, + "17": 0.2726953125, + "18": 0.2377109375, + "19": 0.17771875, + "20": 0.1583203125 + }, + "per_op_avg_response_len": { + "11": 298.1637421875, + "12": 307.7665625, + "13": 323.6607578125, + "14": 345.5516328125, + "15": 354.272921875, + "16": 355.5277890625, + "17": 362.3539140625, + "18": 358.38421875, + "19": 357.370734375, + "20": 355.2835703125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.1637421875, + "12": 307.7665625, + "13": 323.6607578125, + "14": 345.5516328125, + "15": 354.272921875, + "16": 355.5277890625, + "17": 362.3539140625, + "18": 358.38421875, + "19": 357.370734375, + "20": 355.2835703125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 210840, + "answer_accuracy": 0.48950594353640414, + "avg_response_len": 325.2948435178306, + "resp_tokens_sum": 140110995, + "pass_at_k": { + "pass@1": 0.48950594353640414, + "pass@2": 0.5637347389757943, + "pass@4": 0.6270237990217811, + "pass@8": 0.6798324878047404, + "pass@16": 0.7234665203348808, + "pass@32": 0.7593779810186374, + "pass@64": 0.7894920023801919, + "pass@128": 0.8154531946508172 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9173158640226629, + "pass@2": 0.9321434247507305, + "pass@4": 0.9426232079524052, + "pass@8": 0.9504727221846965, + "pass@16": 0.9559958355681684, + "pass@32": 0.9610692250138667, + "pass@64": 0.9662962638202608, + "pass@128": 0.9716713881019831 + }, + "12": { + "pass@1": 0.8502604166666666, + "pass@2": 0.8822195855020245, + "pass@4": 0.8992252803674966, + "pass@8": 0.9110571323450316, + "pass@16": 0.9214628163570127, + "pass@32": 0.930887824277958, + "pass@64": 0.9398831538043408, + "pass@128": 0.9491525423728814 + }, + "13": { + "pass@1": 0.726391065830721, + "pass@2": 0.7912818879125214, + "pass@4": 0.8335420056231218, + "pass@8": 0.8626137561834727, + "pass@16": 0.8832100537971319, + "pass@32": 0.8983327245211211, + "pass@64": 0.9107522425913621, + "pass@128": 0.9216300940438872 + }, + "14": { + "pass@1": 0.6628455996884736, + "pass@2": 0.7529335921210779, + "pass@4": 0.8138323538996873, + "pass@8": 0.8543450805621458, + "pass@16": 0.8849860069383152, + "pass@32": 0.9078095134477392, + "pass@64": 0.9224300735729083, + "pass@128": 0.9283489096573209 + }, + "15": { + "pass@1": 0.5419978323699421, + "pass@2": 0.6499010769878473, + "pass@4": 0.7312498187545919, + "pass@8": 0.7944006265564277, + "pass@16": 0.8412181026930328, + "pass@32": 0.8731671020460094, + "pass@64": 0.8965896514172272, + "pass@128": 0.9161849710982659 + }, + "16": { + "pass@1": 0.3278225806451613, + "pass@2": 0.43879222758445535, + "pass@4": 0.5367525481693823, + "pass@8": 0.6151239749052684, + "pass@16": 0.6757731000120865, + "pass@32": 0.7249341757312748, + "pass@64": 0.7694531357122129, + "pass@128": 0.8064516129032258 + }, + "17": { + "pass@1": 0.2621703395953757, + "pass@2": 0.3600630518638202, + "pass@4": 0.456792897455737, + "pass@8": 0.5437621259982236, + "pass@16": 0.6180605029211054, + "pass@32": 0.6797477330429214, + "pass@64": 0.7305089357101668, + "pass@128": 0.7716763005780347 + }, + "18": { + "pass@1": 0.226875, + "pass@2": 0.3124177446569177, + "pass@4": 0.3960834297498527, + "pass@8": 0.4720975109117748, + "pass@16": 0.5387142011473198, + "pass@32": 0.5955129112414216, + "pass@64": 0.6430159881995816, + "pass@128": 0.6857142857142857 + }, + "19": { + "pass@1": 0.178490990990991, + "pass@2": 0.2551950326310563, + "pass@4": 0.33301340795238443, + "pass@8": 0.40505760318840506, + "pass@16": 0.4697162585750322, + "pass@32": 0.5253325257449054, + "pass@64": 0.57041801098719, + "pass@128": 0.6096096096096096 + }, + "20": { + "pass@1": 0.17771677927927929, + "pass@2": 0.24536064115296385, + "pass@4": 0.3145481130737036, + "pass@8": 0.3795063402799484, + "pass@16": 0.43726851513255144, + "pass@32": 0.48990032894731006, + "pass@64": 0.5395734975527527, + "pass@128": 0.5885885885885885 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 214389, + "answer_accuracy": 0.4980416480820696, + "avg_response_len": 338.38314702274755, + "resp_tokens_sum": 145661763, + "pass_at_k": { + "pass@1": 0.4980416480820696, + "pass@2": 0.5737524613616011, + "pass@4": 0.6379431487489151, + "pass@8": 0.6906560114236137, + "pass@16": 0.7328493523618609, + "pass@32": 0.7669850979851068, + "pass@64": 0.7955553725669628, + "pass@128": 0.8206958073148974 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9532661897590361, + "pass@2": 0.9668066952850776, + "pass@4": 0.9722518120852364, + "pass@8": 0.9737597100924331, + "pass@16": 0.9747293207783853, + "pass@32": 0.9764728204139039, + "pass@64": 0.9796746039275211, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8565585815047022, + "pass@2": 0.8891883469503618, + "pass@4": 0.9085116546952007, + "pass@8": 0.9209451718338288, + "pass@16": 0.9292786810971159, + "pass@32": 0.9355307499078754, + "pass@64": 0.9412521735186443, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.7366509103641457, + "pass@2": 0.8000004135512471, + "pass@4": 0.840610521899048, + "pass@8": 0.8703119785009618, + "pass@16": 0.8914338120219932, + "pass@32": 0.9068159095678788, + "pass@64": 0.9186600691718783, + "pass@128": 0.927170868347339 + }, + "14": { + "pass@1": 0.6795047514619883, + "pass@2": 0.774193316295989, + "pass@4": 0.8374891814510029, + "pass@8": 0.8772687353922201, + "pass@16": 0.9008876424388722, + "pass@32": 0.915532908772454, + "pass@64": 0.9256890720131518, + "pass@128": 0.935672514619883 + }, + "15": { + "pass@1": 0.5189867424242425, + "pass@2": 0.6329478346456691, + "pass@4": 0.7234800828873658, + "pass@8": 0.7914761856946387, + "pass@16": 0.8394843276261118, + "pass@32": 0.8714227086885048, + "pass@64": 0.8933370166253926, + "pass@128": 0.9121212121212121 + }, + "16": { + "pass@1": 0.36509526353276356, + "pass@2": 0.47117557821297973, + "pass@4": 0.5631385563983992, + "pass@8": 0.6375508753238303, + "pass@16": 0.6968728466572433, + "pass@32": 0.7434973553320483, + "pass@64": 0.7808550723883162, + "pass@128": 0.8148148148148148 + }, + "17": { + "pass@1": 0.29258055124223603, + "pass@2": 0.3973844726610258, + "pass@4": 0.4972612846918981, + "pass@8": 0.5823525736053493, + "pass@16": 0.650527208550206, + "pass@32": 0.7072938775670724, + "pass@64": 0.7591546374780846, + "pass@128": 0.8043478260869565 + }, + "18": { + "pass@1": 0.2530560661764706, + "pass@2": 0.34292894569244997, + "pass@4": 0.42779535223538234, + "pass@8": 0.5040166999918608, + "pass@16": 0.5706232825257309, + "pass@32": 0.6272652089469972, + "pass@64": 0.6715140041159122, + "pass@128": 0.7058823529411765 + }, + "19": { + "pass@1": 0.18085007440476192, + "pass@2": 0.25281654246344204, + "pass@4": 0.32825940005267196, + "pass@8": 0.397133814087942, + "pass@16": 0.4569411169793784, + "pass@32": 0.5108194348881472, + "pass@64": 0.5608846632509611, + "pass@128": 0.6101190476190477 + }, + "20": { + "pass@1": 0.14785273203592814, + "pass@2": 0.21188307440237614, + "pass@4": 0.28145796679980883, + "pass@8": 0.3520703062075992, + "pass@16": 0.4177568976295808, + "pass@32": 0.4751822427551991, + "pass@64": 0.5248288962548115, + "pass@128": 0.5658682634730539 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 202472, + "answer_accuracy": 0.4834390281173594, + "avg_response_len": 362.3888055852689, + "resp_tokens_sum": 151774230, + "pass_at_k": { + "pass@1": 0.4834390281173594, + "pass@2": 0.5576616510525979, + "pass@4": 0.622123775156928, + "pass@8": 0.6765753720847417, + "pass@16": 0.7211163296882313, + "pass@32": 0.7574844923948586, + "pass@64": 0.7878555015949061, + "pass@128": 0.8138753056234719 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9436011904761905, + "pass@2": 0.9589879390076239, + "pass@4": 0.9693386811767576, + "pass@8": 0.977879351147027, + "pass@16": 0.9847288493840766, + "pass@32": 0.9893717693963389, + "pass@64": 0.9918425387660565, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8732081422018348, + "pass@2": 0.9065604005634615, + "pass@4": 0.9258341950375469, + "pass@8": 0.9411096819951471, + "pass@16": 0.9549574771725867, + "pass@32": 0.9668139910800122, + "pass@64": 0.9755841146174223, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.6871141975308642, + "pass@2": 0.7559207008846116, + "pass@4": 0.8021860710698199, + "pass@8": 0.8353651955626558, + "pass@16": 0.8608678915667602, + "pass@32": 0.8836545107519911, + "pass@64": 0.9042498585310702, + "pass@128": 0.9197530864197531 + }, + "14": { + "pass@1": 0.6453774109792285, + "pass@2": 0.7358177907194096, + "pass@4": 0.7973326398925645, + "pass@8": 0.8384671475446017, + "pass@16": 0.8678690888370998, + "pass@32": 0.8915380596252334, + "pass@64": 0.9110565829359114, + "pass@128": 0.9287833827893175 + }, + "15": { + "pass@1": 0.5392071759259259, + "pass@2": 0.642665652036551, + "pass@4": 0.7213039889111081, + "pass@8": 0.7794136905806901, + "pass@16": 0.820353152015657, + "pass@32": 0.8484903788649915, + "pass@64": 0.8681461994682231, + "pass@128": 0.8827160493827161 + }, + "16": { + "pass@1": 0.34485619469026546, + "pass@2": 0.45009094894664675, + "pass@4": 0.5437416459557597, + "pass@8": 0.6223168670135193, + "pass@16": 0.6871639944645871, + "pass@32": 0.7404633787313277, + "pass@64": 0.7852328907929348, + "pass@128": 0.8230088495575221 + }, + "17": { + "pass@1": 0.2643778237951807, + "pass@2": 0.36066571897827526, + "pass@4": 0.4522006400178891, + "pass@8": 0.5279337387334523, + "pass@16": 0.5882302924959857, + "pass@32": 0.6421803384498621, + "pass@64": 0.695393789518416, + "pass@128": 0.7469879518072289 + }, + "18": { + "pass@1": 0.23311491935483872, + "pass@2": 0.3121618618237238, + "pass@4": 0.39164191976002993, + "pass@8": 0.46693579681131336, + "pass@16": 0.5324199678252257, + "pass@32": 0.5860600524901607, + "pass@64": 0.6329600321954819, + "pass@128": 0.6774193548387096 + }, + "19": { + "pass@1": 0.1737632175226586, + "pass@2": 0.25203763946047536, + "pass@4": 0.3362170092220345, + "pass@8": 0.4162441880299632, + "pass@16": 0.48426742725273664, + "pass@32": 0.5362360984323276, + "pass@64": 0.5750541662136981, + "pass@128": 0.6102719033232629 + }, + "20": { + "pass@1": 0.14942286036036037, + "pass@2": 0.21775798928850104, + "pass@4": 0.29438976349803125, + "pass@8": 0.3707556323665024, + "pass@16": 0.439221283516568, + "pass@32": 0.4973933445496916, + "pass@64": 0.5450726056754167, + "pass@128": 0.5795795795795796 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7287619243421053, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7287619243421053, + "pass@2": 0.7690151199233136, + "pass@4": 0.80329944386555, + "pass@8": 0.8317859299167593, + "pass@16": 0.8549670814057816, + "pass@32": 0.87386116770562, + "pass@64": 0.8896009440269548, + "pass@128": 0.9031578947368422 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9551484375, + "pass@2": 0.9663884104330706, + "pass@4": 0.9748713232095988, + "pass@8": 0.9815256299919862, + "pass@16": 0.9870153172800666, + "pass@32": 0.9912142150975694, + "pass@64": 0.9934614913928741, + "pass@128": 0.994 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9975234375, + "pass@2": 0.9989872047244095, + "pass@4": 0.9996471928196476, + "pass@8": 0.999929016754186, + "pass@16": 0.9999966016697285, + "pass@32": 0.9999999961558147, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993828125, + "pass@2": 0.9998635580708661, + "pass@4": 0.9999914046681665, + "pass@8": 0.9999999642417031, + "pass@16": 0.9999999999997616, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99909375, + "pass@2": 0.9997829724409449, + "pass@4": 0.9999779075740532, + "pass@8": 0.9999996831294576, + "pass@16": 0.9999999999641311, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995859375, + "pass@2": 0.9979446358267716, + "pass@4": 0.9992358772965879, + "pass@8": 0.9998611370304065, + "pass@16": 0.9999938475734018, + "pass@32": 0.9999999908391373, + "pass@64": 0.9999999999999999, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9955703125, + "pass@2": 0.9971546505905512, + "pass@4": 0.9980388426134232, + "pass@8": 0.9985501319286774, + "pass@16": 0.998838936562875, + "pass@32": 0.9989809345463803, + "pass@64": 0.9989999379486069, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.93753125, + "pass@2": 0.952107652559056, + "pass@4": 0.9608752785901759, + "pass@8": 0.966837090293199, + "pass@16": 0.9712662520099715, + "pass@32": 0.9750985201671579, + "pass@64": 0.9787849493437966, + "pass@128": 0.983 + }, + "12": { + "pass@1": 0.8597734375, + "pass@2": 0.892402066929135, + "pass@4": 0.9108887488751395, + "pass@8": 0.9240386006775467, + "pass@16": 0.9349088312957975, + "pass@32": 0.9441167740981736, + "pass@64": 0.9519940852790811, + "pass@128": 0.959 + }, + "13": { + "pass@1": 0.717328125, + "pass@2": 0.782937376968504, + "pass@4": 0.8259061431383582, + "pass@8": 0.8565334879096712, + "pass@16": 0.8789070749207669, + "pass@32": 0.8966054803216155, + "pass@64": 0.9114685642450719, + "pass@128": 0.923 + }, + "14": { + "pass@1": 0.66265625, + "pass@2": 0.7544363927165362, + "pass@4": 0.8163625853018365, + "pass@8": 0.8568341070871195, + "pass@16": 0.8846559648793955, + "pass@32": 0.904967434710607, + "pass@64": 0.919711784694804, + "pass@128": 0.931 + }, + "15": { + "pass@1": 0.5335, + "pass@2": 0.6419622293307092, + "pass@4": 0.7254633570491189, + "pass@8": 0.7885797938158986, + "pass@16": 0.8338857129014781, + "pass@32": 0.864596193927383, + "pass@64": 0.8863006035044457, + "pass@128": 0.904 + }, + "16": { + "pass@1": 0.3466796875, + "pass@2": 0.45398905019685043, + "pass@4": 0.5483833412073488, + "pass@8": 0.6254342073768804, + "pass@16": 0.6870406243039331, + "pass@32": 0.7367142515881637, + "pass@64": 0.7788045524578896, + "pass@128": 0.815 + }, + "17": { + "pass@1": 0.2726953125, + "pass@2": 0.37228063484252, + "pass@4": 0.46829908867641445, + "pass@8": 0.5509332255558134, + "pass@16": 0.6186111522725348, + "pass@32": 0.6761452165748015, + "pass@64": 0.7280746231437759, + "pass@128": 0.774 + }, + "18": { + "pass@1": 0.2377109375, + "pass@2": 0.3227122293307086, + "pass@4": 0.40548861529808733, + "pass@8": 0.48134990382786125, + "pass@16": 0.5476120764861299, + "pass@32": 0.6033783062484257, + "pass@64": 0.649587967249864, + "pass@128": 0.69 + }, + "19": { + "pass@1": 0.17771875, + "pass@2": 0.25335076279527563, + "pass@4": 0.33247645331833486, + "pass@8": 0.4060979696332052, + "pass@16": 0.4702402478312121, + "pass@32": 0.5240652097765713, + "pass@64": 0.5687493735277911, + "pass@128": 0.61 + }, + "20": { + "pass@1": 0.1583203125, + "pass@2": 0.2249874507874015, + "pass@4": 0.29678327380952363, + "pass@8": 0.36742871916460595, + "pass@16": 0.43140190675843637, + "pass@32": 0.4874796623547377, + "pass@64": 0.5364800037240876, + "pass@128": 0.578 + } + }, + "per_op_accuracy": { + "10": 0.9551484375, + "2": 1.0, + "3": 0.9999921875, + "4": 0.9999921875, + "5": 0.9975234375, + "6": 0.9993828125, + "7": 0.99909375, + "8": 0.995859375, + "9": 0.9955703125, + "11": 0.93753125, + "12": 0.8597734375, + "13": 0.717328125, + "14": 0.66265625, + "15": 0.5335, + "16": 0.3466796875, + "17": 0.2726953125, + "18": 0.2377109375, + "19": 0.17771875, + "20": 0.1583203125 + }, + "per_op_avg_response_len": { + "10": 281.0439375, + "2": 115.6566484375, + "3": 144.7500078125, + "4": 159.92284375, + "5": 172.8460625, + "6": 199.080953125, + "7": 224.0167421875, + "8": 243.6860234375, + "9": 259.5402734375, + "11": 298.1637421875, + "12": 307.7665625, + "13": 323.6607578125, + "14": 345.5516328125, + "15": 354.272921875, + "16": 355.5277890625, + "17": 362.3539140625, + "18": 358.38421875, + "19": 357.370734375, + "20": 355.2835703125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.0439375, + "2": 115.6566484375, + "3": 144.7500078125, + "4": 159.92284375, + "5": 172.8460625, + "6": 199.080953125, + "7": 224.0167421875, + "8": 243.6860234375, + "9": 259.5402734375, + "11": 298.1637421875, + "12": 307.7665625, + "13": 323.6607578125, + "14": 345.5516328125, + "15": 354.272921875, + "16": 355.5277890625, + "17": 362.3539140625, + "18": 358.38421875, + "19": 357.370734375, + "20": 355.2835703125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 587911, + "answer_accuracy": 0.7260598620771419, + "avg_response_len": 262.2493145846507, + "resp_tokens_sum": 212350613, + "pass_at_k": { + "pass@1": 0.7260598620771419, + "pass@2": 0.7663315461935566, + "pass@4": 0.8004183311361215, + "pass@8": 0.8287768600712482, + "pass@16": 0.8522151438441364, + "pass@32": 0.8715158994968991, + "pass@64": 0.8876581846294301, + "pass@128": 0.9015175466329434 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9642791297935103, + "pass@2": 0.9752165935010332, + "pass@4": 0.9816987470039695, + "pass@8": 0.9860516756202101, + "pass@16": 0.9901225277302209, + "pass@32": 0.9938132092442535, + "pass@64": 0.9961212034111633, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999755859375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996875, + "pass@2": 0.9982318616422946, + "pass@4": 0.9991640568738431, + "pass@8": 0.9997933137781767, + "pass@16": 0.9999892767188607, + "pass@32": 0.9999999877963381, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9980815458579881, + "pass@2": 0.9994521822438617, + "pass@4": 0.9999370938469968, + "pass@8": 0.9999990635278279, + "pass@16": 0.9999999998938789, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9993739983974359, + "pass@2": 0.9999842267312739, + "pass@4": 0.9999999876818282, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966549295774648, + "pass@2": 0.9971678496173894, + "pass@4": 0.9971830761471718, + "pass@8": 0.9971830985915474, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9173158640226629, + "pass@2": 0.9321434247507305, + "pass@4": 0.9426232079524052, + "pass@8": 0.9504727221846965, + "pass@16": 0.9559958355681684, + "pass@32": 0.9610692250138667, + "pass@64": 0.9662962638202608, + "pass@128": 0.9716713881019831 + }, + "12": { + "pass@1": 0.8502604166666666, + "pass@2": 0.8822195855020245, + "pass@4": 0.8992252803674966, + "pass@8": 0.9110571323450316, + "pass@16": 0.9214628163570127, + "pass@32": 0.930887824277958, + "pass@64": 0.9398831538043408, + "pass@128": 0.9491525423728814 + }, + "13": { + "pass@1": 0.726391065830721, + "pass@2": 0.7912818879125214, + "pass@4": 0.8335420056231218, + "pass@8": 0.8626137561834727, + "pass@16": 0.8832100537971319, + "pass@32": 0.8983327245211211, + "pass@64": 0.9107522425913621, + "pass@128": 0.9216300940438872 + }, + "14": { + "pass@1": 0.6628455996884736, + "pass@2": 0.7529335921210779, + "pass@4": 0.8138323538996873, + "pass@8": 0.8543450805621458, + "pass@16": 0.8849860069383152, + "pass@32": 0.9078095134477392, + "pass@64": 0.9224300735729083, + "pass@128": 0.9283489096573209 + }, + "15": { + "pass@1": 0.5419978323699421, + "pass@2": 0.6499010769878473, + "pass@4": 0.7312498187545919, + "pass@8": 0.7944006265564277, + "pass@16": 0.8412181026930328, + "pass@32": 0.8731671020460094, + "pass@64": 0.8965896514172272, + "pass@128": 0.9161849710982659 + }, + "16": { + "pass@1": 0.3278225806451613, + "pass@2": 0.43879222758445535, + "pass@4": 0.5367525481693823, + "pass@8": 0.6151239749052684, + "pass@16": 0.6757731000120865, + "pass@32": 0.7249341757312748, + "pass@64": 0.7694531357122129, + "pass@128": 0.8064516129032258 + }, + "17": { + "pass@1": 0.2621703395953757, + "pass@2": 0.3600630518638202, + "pass@4": 0.456792897455737, + "pass@8": 0.5437621259982236, + "pass@16": 0.6180605029211054, + "pass@32": 0.6797477330429214, + "pass@64": 0.7305089357101668, + "pass@128": 0.7716763005780347 + }, + "18": { + "pass@1": 0.226875, + "pass@2": 0.3124177446569177, + "pass@4": 0.3960834297498527, + "pass@8": 0.4720975109117748, + "pass@16": 0.5387142011473198, + "pass@32": 0.5955129112414216, + "pass@64": 0.6430159881995816, + "pass@128": 0.6857142857142857 + }, + "19": { + "pass@1": 0.178490990990991, + "pass@2": 0.2551950326310563, + "pass@4": 0.33301340795238443, + "pass@8": 0.40505760318840506, + "pass@16": 0.4697162585750322, + "pass@32": 0.5253325257449054, + "pass@64": 0.57041801098719, + "pass@128": 0.6096096096096096 + }, + "20": { + "pass@1": 0.17771677927927929, + "pass@2": 0.24536064115296385, + "pass@4": 0.3145481130737036, + "pass@8": 0.3795063402799484, + "pass@16": 0.43726851513255144, + "pass@32": 0.48990032894731006, + "pass@64": 0.5395734975527527, + "pass@128": 0.5885885885885885 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 602122, + "answer_accuracy": 0.7332935502727981, + "avg_response_len": 271.6208544427124, + "resp_tokens_sum": 223033316, + "pass_at_k": { + "pass@1": 0.7332935502727981, + "pass@2": 0.7739351015091283, + "pass@4": 0.808317863170449, + "pass@8": 0.8365187556399943, + "pass@16": 0.8590574533420197, + "pass@32": 0.877230108508473, + "pass@64": 0.8923344650629088, + "pass@128": 0.9055339049103663 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9419814033742331, + "pass@2": 0.9546104354862087, + "pass@4": 0.9660200076600852, + "pass@8": 0.9756070749670658, + "pass@16": 0.9829531280350566, + "pass@32": 0.987973396368052, + "pass@64": 0.9904077837030997, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995471014492754, + "pass@2": 0.9999711143444027, + "pass@4": 0.9999999087070638, + "pass@8": 0.9999999999998904, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991045845272206, + "pass@2": 0.9998568745346659, + "pass@4": 0.9999951975917051, + "pass@8": 0.9999999955503199, + "pass@16": 0.9999999999999997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9994355130057804, + "pass@2": 0.9999100376632835, + "pass@4": 0.9999976010043543, + "pass@8": 0.9999999990088202, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9979289120550073, + "pass@4": 0.9992789157517282, + "pass@8": 0.9998878246175227, + "pass@16": 0.9999970717408715, + "pass@32": 0.9999999992397323, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947838697604791, + "pass@2": 0.9966689406619832, + "pass@4": 0.9978810868139987, + "pass@8": 0.998753199791672, + "pass@16": 0.9995194939779444, + "pass@32": 0.9999429179994794, + "pass@64": 0.9999998142173857, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9532661897590361, + "pass@2": 0.9668066952850776, + "pass@4": 0.9722518120852364, + "pass@8": 0.9737597100924331, + "pass@16": 0.9747293207783853, + "pass@32": 0.9764728204139039, + "pass@64": 0.9796746039275211, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8565585815047022, + "pass@2": 0.8891883469503618, + "pass@4": 0.9085116546952007, + "pass@8": 0.9209451718338288, + "pass@16": 0.9292786810971159, + "pass@32": 0.9355307499078754, + "pass@64": 0.9412521735186443, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.7366509103641457, + "pass@2": 0.8000004135512471, + "pass@4": 0.840610521899048, + "pass@8": 0.8703119785009618, + "pass@16": 0.8914338120219932, + "pass@32": 0.9068159095678788, + "pass@64": 0.9186600691718783, + "pass@128": 0.927170868347339 + }, + "14": { + "pass@1": 0.6795047514619883, + "pass@2": 0.774193316295989, + "pass@4": 0.8374891814510029, + "pass@8": 0.8772687353922201, + "pass@16": 0.9008876424388722, + "pass@32": 0.915532908772454, + "pass@64": 0.9256890720131518, + "pass@128": 0.935672514619883 + }, + "15": { + "pass@1": 0.5189867424242425, + "pass@2": 0.6329478346456691, + "pass@4": 0.7234800828873658, + "pass@8": 0.7914761856946387, + "pass@16": 0.8394843276261118, + "pass@32": 0.8714227086885048, + "pass@64": 0.8933370166253926, + "pass@128": 0.9121212121212121 + }, + "16": { + "pass@1": 0.36509526353276356, + "pass@2": 0.47117557821297973, + "pass@4": 0.5631385563983992, + "pass@8": 0.6375508753238303, + "pass@16": 0.6968728466572433, + "pass@32": 0.7434973553320483, + "pass@64": 0.7808550723883162, + "pass@128": 0.8148148148148148 + }, + "17": { + "pass@1": 0.29258055124223603, + "pass@2": 0.3973844726610258, + "pass@4": 0.4972612846918981, + "pass@8": 0.5823525736053493, + "pass@16": 0.650527208550206, + "pass@32": 0.7072938775670724, + "pass@64": 0.7591546374780846, + "pass@128": 0.8043478260869565 + }, + "18": { + "pass@1": 0.2530560661764706, + "pass@2": 0.34292894569244997, + "pass@4": 0.42779535223538234, + "pass@8": 0.5040166999918608, + "pass@16": 0.5706232825257309, + "pass@32": 0.6272652089469972, + "pass@64": 0.6715140041159122, + "pass@128": 0.7058823529411765 + }, + "19": { + "pass@1": 0.18085007440476192, + "pass@2": 0.25281654246344204, + "pass@4": 0.32825940005267196, + "pass@8": 0.397133814087942, + "pass@16": 0.4569411169793784, + "pass@32": 0.5108194348881472, + "pass@64": 0.5608846632509611, + "pass@128": 0.6101190476190477 + }, + "20": { + "pass@1": 0.14785273203592814, + "pass@2": 0.21188307440237614, + "pass@4": 0.28145796679980883, + "pass@8": 0.3520703062075992, + "pass@16": 0.4177568976295808, + "pass@32": 0.4751822427551991, + "pass@64": 0.5248288962548115, + "pass@128": 0.5658682634730539 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 582316, + "answer_accuracy": 0.7268483383927145, + "avg_response_len": 290.37264588991854, + "resp_tokens_sum": 232632626, + "pass_at_k": { + "pass@1": 0.7268483383927145, + "pass@2": 0.7666848124527403, + "pass@4": 0.801067898776187, + "pass@8": 0.8299764234185844, + "pass@16": 0.8535561565046663, + "pass@32": 0.872778632386536, + "pass@64": 0.8887628482453852, + "pass@128": 0.902380571976354 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9587220149253731, + "pass@2": 0.9689163679633328, + "pass@4": 0.9765758969494485, + "pass@8": 0.9827050910969309, + "pass@16": 0.9878240616122193, + "pass@32": 0.9917379401426335, + "pass@64": 0.9937416894008326, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9960707720588236, + "pass@2": 0.9986886289949051, + "pass@4": 0.9997369011777938, + "pass@8": 0.9999827144532304, + "pass@16": 0.9999999397155509, + "pass@32": 0.9999999999999061, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990388406940063, + "pass@2": 0.9997271572816017, + "pass@4": 0.9999781725825281, + "pass@8": 0.9999998920966604, + "pass@16": 0.9999999999992483, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998022151898734, + "pass@2": 0.9999976639589356, + "pass@4": 0.99999999970336, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9926801801801802, + "pass@2": 0.9960504303516113, + "pass@4": 0.9984740722161982, + "pass@8": 0.9997025804540118, + "pass@16": 0.9999846459621398, + "pass@32": 0.9999999733003972, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9951768488745981, + "pass@2": 0.9976612147758058, + "pass@4": 0.9991851047758901, + "pass@8": 0.9998924861680374, + "pass@16": 0.9999981529718375, + "pass@32": 0.9999999997895631, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9436011904761905, + "pass@2": 0.9589879390076239, + "pass@4": 0.9693386811767576, + "pass@8": 0.977879351147027, + "pass@16": 0.9847288493840766, + "pass@32": 0.9893717693963389, + "pass@64": 0.9918425387660565, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8732081422018348, + "pass@2": 0.9065604005634615, + "pass@4": 0.9258341950375469, + "pass@8": 0.9411096819951471, + "pass@16": 0.9549574771725867, + "pass@32": 0.9668139910800122, + "pass@64": 0.9755841146174223, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.6871141975308642, + "pass@2": 0.7559207008846116, + "pass@4": 0.8021860710698199, + "pass@8": 0.8353651955626558, + "pass@16": 0.8608678915667602, + "pass@32": 0.8836545107519911, + "pass@64": 0.9042498585310702, + "pass@128": 0.9197530864197531 + }, + "14": { + "pass@1": 0.6453774109792285, + "pass@2": 0.7358177907194096, + "pass@4": 0.7973326398925645, + "pass@8": 0.8384671475446017, + "pass@16": 0.8678690888370998, + "pass@32": 0.8915380596252334, + "pass@64": 0.9110565829359114, + "pass@128": 0.9287833827893175 + }, + "15": { + "pass@1": 0.5392071759259259, + "pass@2": 0.642665652036551, + "pass@4": 0.7213039889111081, + "pass@8": 0.7794136905806901, + "pass@16": 0.820353152015657, + "pass@32": 0.8484903788649915, + "pass@64": 0.8681461994682231, + "pass@128": 0.8827160493827161 + }, + "16": { + "pass@1": 0.34485619469026546, + "pass@2": 0.45009094894664675, + "pass@4": 0.5437416459557597, + "pass@8": 0.6223168670135193, + "pass@16": 0.6871639944645871, + "pass@32": 0.7404633787313277, + "pass@64": 0.7852328907929348, + "pass@128": 0.8230088495575221 + }, + "17": { + "pass@1": 0.2643778237951807, + "pass@2": 0.36066571897827526, + "pass@4": 0.4522006400178891, + "pass@8": 0.5279337387334523, + "pass@16": 0.5882302924959857, + "pass@32": 0.6421803384498621, + "pass@64": 0.695393789518416, + "pass@128": 0.7469879518072289 + }, + "18": { + "pass@1": 0.23311491935483872, + "pass@2": 0.3121618618237238, + "pass@4": 0.39164191976002993, + "pass@8": 0.46693579681131336, + "pass@16": 0.5324199678252257, + "pass@32": 0.5860600524901607, + "pass@64": 0.6329600321954819, + "pass@128": 0.6774193548387096 + }, + "19": { + "pass@1": 0.1737632175226586, + "pass@2": 0.25203763946047536, + "pass@4": 0.3362170092220345, + "pass@8": 0.4162441880299632, + "pass@16": 0.48426742725273664, + "pass@32": 0.5362360984323276, + "pass@64": 0.5750541662136981, + "pass@128": 0.6102719033232629 + }, + "20": { + "pass@1": 0.14942286036036037, + "pass@2": 0.21775798928850104, + "pass@4": 0.29438976349803125, + "pass@8": 0.3707556323665024, + "pass@16": 0.439221283516568, + "pass@32": 0.4973933445496916, + "pass@64": 0.5450726056754167, + "pass@128": 0.5795795795795796 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep4-block4-op7-10/composition/autoregressive/summary.csv b/SDAR-100M-ep4-block4-op7-10/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..1ab7e775304c58291512cdf195bd8c7514031666 --- /dev/null +++ b/SDAR-100M-ep4-block4-op7-10/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.9936180555555556,0.49039140625,0.7287619243421053,0.0,0.0,0.0,200.06038802083333,341.833584375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep4-block4-op7-10/composition/autoregressive/summary.json b/SDAR-100M-ep4-block4-op7-10/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..deae11dd55bad58e70381cb16374943a73b3d5f8 --- /dev/null +++ b/SDAR-100M-ep4-block4-op7-10/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.9936180555555556, + "ood_acc": 0.49039140625, + "total_acc": 0.7287619243421053, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.06038802083333, + "ood_avg_resp_len": 341.833584375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 587911, + "answer_accuracy": 0.7260598620771419, + "avg_response_len": 262.2493145846507, + "resp_tokens_sum": 212350613, + "pass_at_k": { + "pass@1": 0.7260598620771419, + "pass@2": 0.7663315461935566, + "pass@4": 0.8004183311361215, + "pass@8": 0.8287768600712482, + "pass@16": 0.8522151438441364, + "pass@32": 0.8715158994968991, + "pass@64": 0.8876581846294301, + "pass@128": 0.9015175466329434 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9642791297935103, + "pass@2": 0.9752165935010332, + "pass@4": 0.9816987470039695, + "pass@8": 0.9860516756202101, + "pass@16": 0.9901225277302209, + "pass@32": 0.9938132092442535, + "pass@64": 0.9961212034111633, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999755859375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996875, + "pass@2": 0.9982318616422946, + "pass@4": 0.9991640568738431, + "pass@8": 0.9997933137781767, + "pass@16": 0.9999892767188607, + "pass@32": 0.9999999877963381, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9980815458579881, + "pass@2": 0.9994521822438617, + "pass@4": 0.9999370938469968, + "pass@8": 0.9999990635278279, + "pass@16": 0.9999999998938789, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9993739983974359, + "pass@2": 0.9999842267312739, + "pass@4": 0.9999999876818282, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966549295774648, + "pass@2": 0.9971678496173894, + "pass@4": 0.9971830761471718, + "pass@8": 0.9971830985915474, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9173158640226629, + "pass@2": 0.9321434247507305, + "pass@4": 0.9426232079524052, + "pass@8": 0.9504727221846965, + "pass@16": 0.9559958355681684, + "pass@32": 0.9610692250138667, + "pass@64": 0.9662962638202608, + "pass@128": 0.9716713881019831 + }, + "12": { + "pass@1": 0.8502604166666666, + "pass@2": 0.8822195855020245, + "pass@4": 0.8992252803674966, + "pass@8": 0.9110571323450316, + "pass@16": 0.9214628163570127, + "pass@32": 0.930887824277958, + "pass@64": 0.9398831538043408, + "pass@128": 0.9491525423728814 + }, + "13": { + "pass@1": 0.726391065830721, + "pass@2": 0.7912818879125214, + "pass@4": 0.8335420056231218, + "pass@8": 0.8626137561834727, + "pass@16": 0.8832100537971319, + "pass@32": 0.8983327245211211, + "pass@64": 0.9107522425913621, + "pass@128": 0.9216300940438872 + }, + "14": { + "pass@1": 0.6628455996884736, + "pass@2": 0.7529335921210779, + "pass@4": 0.8138323538996873, + "pass@8": 0.8543450805621458, + "pass@16": 0.8849860069383152, + "pass@32": 0.9078095134477392, + "pass@64": 0.9224300735729083, + "pass@128": 0.9283489096573209 + }, + "15": { + "pass@1": 0.5419978323699421, + "pass@2": 0.6499010769878473, + "pass@4": 0.7312498187545919, + "pass@8": 0.7944006265564277, + "pass@16": 0.8412181026930328, + "pass@32": 0.8731671020460094, + "pass@64": 0.8965896514172272, + "pass@128": 0.9161849710982659 + }, + "16": { + "pass@1": 0.3278225806451613, + "pass@2": 0.43879222758445535, + "pass@4": 0.5367525481693823, + "pass@8": 0.6151239749052684, + "pass@16": 0.6757731000120865, + "pass@32": 0.7249341757312748, + "pass@64": 0.7694531357122129, + "pass@128": 0.8064516129032258 + }, + "17": { + "pass@1": 0.2621703395953757, + "pass@2": 0.3600630518638202, + "pass@4": 0.456792897455737, + "pass@8": 0.5437621259982236, + "pass@16": 0.6180605029211054, + "pass@32": 0.6797477330429214, + "pass@64": 0.7305089357101668, + "pass@128": 0.7716763005780347 + }, + "18": { + "pass@1": 0.226875, + "pass@2": 0.3124177446569177, + "pass@4": 0.3960834297498527, + "pass@8": 0.4720975109117748, + "pass@16": 0.5387142011473198, + "pass@32": 0.5955129112414216, + "pass@64": 0.6430159881995816, + "pass@128": 0.6857142857142857 + }, + "19": { + "pass@1": 0.178490990990991, + "pass@2": 0.2551950326310563, + "pass@4": 0.33301340795238443, + "pass@8": 0.40505760318840506, + "pass@16": 0.4697162585750322, + "pass@32": 0.5253325257449054, + "pass@64": 0.57041801098719, + "pass@128": 0.6096096096096096 + }, + "20": { + "pass@1": 0.17771677927927929, + "pass@2": 0.24536064115296385, + "pass@4": 0.3145481130737036, + "pass@8": 0.3795063402799484, + "pass@16": 0.43726851513255144, + "pass@32": 0.48990032894731006, + "pass@64": 0.5395734975527527, + "pass@128": 0.5885885885885885 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 602122, + "answer_accuracy": 0.7332935502727981, + "avg_response_len": 271.6208544427124, + "resp_tokens_sum": 223033316, + "pass_at_k": { + "pass@1": 0.7332935502727981, + "pass@2": 0.7739351015091283, + "pass@4": 0.808317863170449, + "pass@8": 0.8365187556399943, + "pass@16": 0.8590574533420197, + "pass@32": 0.877230108508473, + "pass@64": 0.8923344650629088, + "pass@128": 0.9055339049103663 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9419814033742331, + "pass@2": 0.9546104354862087, + "pass@4": 0.9660200076600852, + "pass@8": 0.9756070749670658, + "pass@16": 0.9829531280350566, + "pass@32": 0.987973396368052, + "pass@64": 0.9904077837030997, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995471014492754, + "pass@2": 0.9999711143444027, + "pass@4": 0.9999999087070638, + "pass@8": 0.9999999999998904, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991045845272206, + "pass@2": 0.9998568745346659, + "pass@4": 0.9999951975917051, + "pass@8": 0.9999999955503199, + "pass@16": 0.9999999999999997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9994355130057804, + "pass@2": 0.9999100376632835, + "pass@4": 0.9999976010043543, + "pass@8": 0.9999999990088202, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9979289120550073, + "pass@4": 0.9992789157517282, + "pass@8": 0.9998878246175227, + "pass@16": 0.9999970717408715, + "pass@32": 0.9999999992397323, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947838697604791, + "pass@2": 0.9966689406619832, + "pass@4": 0.9978810868139987, + "pass@8": 0.998753199791672, + "pass@16": 0.9995194939779444, + "pass@32": 0.9999429179994794, + "pass@64": 0.9999998142173857, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9532661897590361, + "pass@2": 0.9668066952850776, + "pass@4": 0.9722518120852364, + "pass@8": 0.9737597100924331, + "pass@16": 0.9747293207783853, + "pass@32": 0.9764728204139039, + "pass@64": 0.9796746039275211, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8565585815047022, + "pass@2": 0.8891883469503618, + "pass@4": 0.9085116546952007, + "pass@8": 0.9209451718338288, + "pass@16": 0.9292786810971159, + "pass@32": 0.9355307499078754, + "pass@64": 0.9412521735186443, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.7366509103641457, + "pass@2": 0.8000004135512471, + "pass@4": 0.840610521899048, + "pass@8": 0.8703119785009618, + "pass@16": 0.8914338120219932, + "pass@32": 0.9068159095678788, + "pass@64": 0.9186600691718783, + "pass@128": 0.927170868347339 + }, + "14": { + "pass@1": 0.6795047514619883, + "pass@2": 0.774193316295989, + "pass@4": 0.8374891814510029, + "pass@8": 0.8772687353922201, + "pass@16": 0.9008876424388722, + "pass@32": 0.915532908772454, + "pass@64": 0.9256890720131518, + "pass@128": 0.935672514619883 + }, + "15": { + "pass@1": 0.5189867424242425, + "pass@2": 0.6329478346456691, + "pass@4": 0.7234800828873658, + "pass@8": 0.7914761856946387, + "pass@16": 0.8394843276261118, + "pass@32": 0.8714227086885048, + "pass@64": 0.8933370166253926, + "pass@128": 0.9121212121212121 + }, + "16": { + "pass@1": 0.36509526353276356, + "pass@2": 0.47117557821297973, + "pass@4": 0.5631385563983992, + "pass@8": 0.6375508753238303, + "pass@16": 0.6968728466572433, + "pass@32": 0.7434973553320483, + "pass@64": 0.7808550723883162, + "pass@128": 0.8148148148148148 + }, + "17": { + "pass@1": 0.29258055124223603, + "pass@2": 0.3973844726610258, + "pass@4": 0.4972612846918981, + "pass@8": 0.5823525736053493, + "pass@16": 0.650527208550206, + "pass@32": 0.7072938775670724, + "pass@64": 0.7591546374780846, + "pass@128": 0.8043478260869565 + }, + "18": { + "pass@1": 0.2530560661764706, + "pass@2": 0.34292894569244997, + "pass@4": 0.42779535223538234, + "pass@8": 0.5040166999918608, + "pass@16": 0.5706232825257309, + "pass@32": 0.6272652089469972, + "pass@64": 0.6715140041159122, + "pass@128": 0.7058823529411765 + }, + "19": { + "pass@1": 0.18085007440476192, + "pass@2": 0.25281654246344204, + "pass@4": 0.32825940005267196, + "pass@8": 0.397133814087942, + "pass@16": 0.4569411169793784, + "pass@32": 0.5108194348881472, + "pass@64": 0.5608846632509611, + "pass@128": 0.6101190476190477 + }, + "20": { + "pass@1": 0.14785273203592814, + "pass@2": 0.21188307440237614, + "pass@4": 0.28145796679980883, + "pass@8": 0.3520703062075992, + "pass@16": 0.4177568976295808, + "pass@32": 0.4751822427551991, + "pass@64": 0.5248288962548115, + "pass@128": 0.5658682634730539 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 582316, + "answer_accuracy": 0.7268483383927145, + "avg_response_len": 290.37264588991854, + "resp_tokens_sum": 232632626, + "pass_at_k": { + "pass@1": 0.7268483383927145, + "pass@2": 0.7666848124527403, + "pass@4": 0.801067898776187, + "pass@8": 0.8299764234185844, + "pass@16": 0.8535561565046663, + "pass@32": 0.872778632386536, + "pass@64": 0.8887628482453852, + "pass@128": 0.902380571976354 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9587220149253731, + "pass@2": 0.9689163679633328, + "pass@4": 0.9765758969494485, + "pass@8": 0.9827050910969309, + "pass@16": 0.9878240616122193, + "pass@32": 0.9917379401426335, + "pass@64": 0.9937416894008326, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9960707720588236, + "pass@2": 0.9986886289949051, + "pass@4": 0.9997369011777938, + "pass@8": 0.9999827144532304, + "pass@16": 0.9999999397155509, + "pass@32": 0.9999999999999061, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990388406940063, + "pass@2": 0.9997271572816017, + "pass@4": 0.9999781725825281, + "pass@8": 0.9999998920966604, + "pass@16": 0.9999999999992483, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998022151898734, + "pass@2": 0.9999976639589356, + "pass@4": 0.99999999970336, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9926801801801802, + "pass@2": 0.9960504303516113, + "pass@4": 0.9984740722161982, + "pass@8": 0.9997025804540118, + "pass@16": 0.9999846459621398, + "pass@32": 0.9999999733003972, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9951768488745981, + "pass@2": 0.9976612147758058, + "pass@4": 0.9991851047758901, + "pass@8": 0.9998924861680374, + "pass@16": 0.9999981529718375, + "pass@32": 0.9999999997895631, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9436011904761905, + "pass@2": 0.9589879390076239, + "pass@4": 0.9693386811767576, + "pass@8": 0.977879351147027, + "pass@16": 0.9847288493840766, + "pass@32": 0.9893717693963389, + "pass@64": 0.9918425387660565, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8732081422018348, + "pass@2": 0.9065604005634615, + "pass@4": 0.9258341950375469, + "pass@8": 0.9411096819951471, + "pass@16": 0.9549574771725867, + "pass@32": 0.9668139910800122, + "pass@64": 0.9755841146174223, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.6871141975308642, + "pass@2": 0.7559207008846116, + "pass@4": 0.8021860710698199, + "pass@8": 0.8353651955626558, + "pass@16": 0.8608678915667602, + "pass@32": 0.8836545107519911, + "pass@64": 0.9042498585310702, + "pass@128": 0.9197530864197531 + }, + "14": { + "pass@1": 0.6453774109792285, + "pass@2": 0.7358177907194096, + "pass@4": 0.7973326398925645, + "pass@8": 0.8384671475446017, + "pass@16": 0.8678690888370998, + "pass@32": 0.8915380596252334, + "pass@64": 0.9110565829359114, + "pass@128": 0.9287833827893175 + }, + "15": { + "pass@1": 0.5392071759259259, + "pass@2": 0.642665652036551, + "pass@4": 0.7213039889111081, + "pass@8": 0.7794136905806901, + "pass@16": 0.820353152015657, + "pass@32": 0.8484903788649915, + "pass@64": 0.8681461994682231, + "pass@128": 0.8827160493827161 + }, + "16": { + "pass@1": 0.34485619469026546, + "pass@2": 0.45009094894664675, + "pass@4": 0.5437416459557597, + "pass@8": 0.6223168670135193, + "pass@16": 0.6871639944645871, + "pass@32": 0.7404633787313277, + "pass@64": 0.7852328907929348, + "pass@128": 0.8230088495575221 + }, + "17": { + "pass@1": 0.2643778237951807, + "pass@2": 0.36066571897827526, + "pass@4": 0.4522006400178891, + "pass@8": 0.5279337387334523, + "pass@16": 0.5882302924959857, + "pass@32": 0.6421803384498621, + "pass@64": 0.695393789518416, + "pass@128": 0.7469879518072289 + }, + "18": { + "pass@1": 0.23311491935483872, + "pass@2": 0.3121618618237238, + "pass@4": 0.39164191976002993, + "pass@8": 0.46693579681131336, + "pass@16": 0.5324199678252257, + "pass@32": 0.5860600524901607, + "pass@64": 0.6329600321954819, + "pass@128": 0.6774193548387096 + }, + "19": { + "pass@1": 0.1737632175226586, + "pass@2": 0.25203763946047536, + "pass@4": 0.3362170092220345, + "pass@8": 0.4162441880299632, + "pass@16": 0.48426742725273664, + "pass@32": 0.5362360984323276, + "pass@64": 0.5750541662136981, + "pass@128": 0.6102719033232629 + }, + "20": { + "pass@1": 0.14942286036036037, + "pass@2": 0.21775798928850104, + "pass@4": 0.29438976349803125, + "pass@8": 0.3707556323665024, + "pass@16": 0.439221283516568, + "pass@32": 0.4973933445496916, + "pass@64": 0.5450726056754167, + "pass@128": 0.5795795795795796 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep4-block4-op9-12/composition/autoregressive/hf_model_metrics.json b/SDAR-100M-ep4-block4-op9-12/composition/autoregressive/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..da073cbf255e28619dbfdde4efccb4a7bed79f17 --- /dev/null +++ b/SDAR-100M-ep4-block4-op9-12/composition/autoregressive/hf_model_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.9940303819444445, + "avg_response_len": 199.72603993055554, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9940303819444445, + "pass@2": 0.9958945346675407, + "pass@4": 0.9970733745156856, + "pass@8": 0.9978977386889625, + "pass@16": 0.9985355110471349, + "pass@32": 0.9990374703606685, + "pass@64": 0.9994152792875225, + "pass@128": 0.9996666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9592109375, + "pass@2": 0.9698198818897638, + "pass@4": 0.9775970624296962, + "pass@8": 0.9833988470253193, + "pass@16": 0.9880673774247114, + "pass@32": 0.9919647338892076, + "pass@64": 0.9949862407389606, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999609375, + "pass@2": 0.9999997539370079, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9973125, + "pass@2": 0.9988038877952756, + "pass@4": 0.9994920357142858, + "pass@8": 0.9998169993777086, + "pass@16": 0.9999715651432676, + "pass@32": 0.9999995674757711, + "pass@64": 0.9999999999958465, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999359375, + "pass@2": 0.9998471948818897, + "pass@4": 0.9999893286464192, + "pass@8": 0.9999999514497075, + "pass@16": 0.9999999999995928, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99915625, + "pass@2": 0.9998037647637795, + "pass@4": 0.9999841450131234, + "pass@8": 0.9999998717991873, + "pass@16": 0.999999999995534, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9951953125, + "pass@2": 0.9971200787401576, + "pass@4": 0.9982833751406074, + "pass@8": 0.9991360598170629, + "pass@16": 0.9996836215015343, + "pass@32": 0.9999477157399603, + "pass@64": 0.9999993318674038, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.996078125, + "pass@2": 0.9976562500000001, + "pass@4": 0.9983144236970378, + "pass@8": 0.9987279187316727, + "pass@16": 0.9990970353595758, + "pass@32": 0.9994252161410782, + "pass@64": 0.9997519409854931, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9592109375, + "2": 1.0, + "3": 1.0, + "4": 0.9999609375, + "5": 0.9973125, + "6": 0.999359375, + "7": 0.99915625, + "8": 0.9951953125, + "9": 0.996078125 + }, + "per_op_avg_response_len": { + "10": 280.5589140625, + "2": 115.5463046875, + "3": 144.54734375, + "4": 159.7105390625, + "5": 172.6188515625, + "6": 198.7573515625, + "7": 223.5936328125, + "8": 243.207015625, + "9": 258.99440625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.5589140625, + "2": 115.5463046875, + "3": 144.54734375, + "4": 159.7105390625, + "5": 172.6188515625, + "6": 198.7573515625, + "7": 223.5936328125, + "8": 243.207015625, + "9": 258.99440625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 377034, + "answer_accuracy": 0.9947916666666666, + "avg_response_len": 190.24573887622424, + "resp_tokens_sum": 72104657, + "pass_at_k": { + "pass@1": 0.9947916666666666, + "pass@2": 0.9965351322441078, + "pass@4": 0.997525146331126, + "pass@8": 0.998166570143595, + "pass@16": 0.9986886899570382, + "pass@32": 0.9991392438051735, + "pass@64": 0.999482545219604, + "pass@128": 0.999662276258021 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9638873525073747, + "pass@2": 0.9752224002973081, + "pass@4": 0.9824726737697609, + "pass@8": 0.9871089937889923, + "pass@16": 0.9908861524344581, + "pass@32": 0.9941379264111941, + "pass@64": 0.9962119406945845, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99990234375, + "pass@2": 0.9999992310531496, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966517857142857, + "pass@2": 0.9977686382952132, + "pass@4": 0.9986264582998554, + "pass@8": 0.999428057911097, + "pass@16": 0.999909743084133, + "pass@32": 0.9999986269072114, + "pass@64": 0.9999999999868145, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999766092814372, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9981508875739645, + "pass@2": 0.9994958620649489, + "pass@4": 0.9999547515880042, + "pass@8": 0.9999996211287503, + "pass@16": 0.9999999999867872, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9990985576923077, + "pass@2": 0.9999593838330306, + "pass@4": 0.9999997803760107, + "pass@8": 0.9999999999971149, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9972572640567817, + "pass@4": 0.9973570422535211, + "pass@8": 0.9975255073749407, + "pass@16": 0.9978457358323166, + "pass@32": 0.998419651768881, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 388058, + "answer_accuracy": 0.9933496477719528, + "avg_response_len": 197.72181919642858, + "resp_tokens_sum": 77241215, + "pass_at_k": { + "pass@1": 0.9933496477719528, + "pass@2": 0.9952137039349439, + "pass@4": 0.9964969655148774, + "pass@8": 0.9974429429895413, + "pass@16": 0.9981976053217531, + "pass@32": 0.9987959103538933, + "pass@64": 0.9992474112646134, + "pass@128": 0.9996723460026212 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9486435966257669, + "pass@2": 0.9604827363412394, + "pass@4": 0.9700275196620363, + "pass@8": 0.9774135724453075, + "pass@16": 0.983556900424375, + "pass@32": 0.988769571356498, + "pass@64": 0.9929543763743685, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995697463768116, + "pass@2": 0.9999825259614288, + "pass@4": 0.9999999766333555, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990598137535817, + "pass@2": 0.9998233845858808, + "pass@4": 0.9999915149395725, + "pass@8": 0.9999999841898438, + "pass@16": 0.9999999999999909, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9994129335260116, + "pass@2": 0.9999253277047017, + "pass@4": 0.9999983785444451, + "pass@8": 0.9999999995886407, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956205985915493, + "pass@2": 0.9978589054009094, + "pass@4": 0.9991935824923294, + "pass@8": 0.9998422314755744, + "pass@16": 0.9999935172675638, + "pass@32": 0.9999999954212553, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9960469685628742, + "pass@2": 0.9973905535385922, + "pass@4": 0.9981125027784701, + "pass@8": 0.9988474577040616, + "pass@16": 0.9995863571067751, + "pass@32": 0.999958801686582, + "pass@64": 0.9999999176094492, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380031, + "answer_accuracy": 0.9939712713424841, + "avg_response_len": 211.1716552979578, + "resp_tokens_sum": 80738526, + "pass_at_k": { + "pass@1": 0.9939712713424841, + "pass@2": 0.9959551593651232, + "pass@4": 0.9972144873127887, + "pass@8": 0.9980959397393345, + "pass@16": 0.9987289243453078, + "pass@32": 0.999183399376905, + "pass@64": 0.999520119856999, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9647621268656716, + "pass@2": 0.9734391526618872, + "pass@4": 0.9800294167520107, + "pass@8": 0.9854688761602408, + "pass@16": 0.98960424540622, + "pass@32": 0.9928749151450584, + "pass@64": 0.9957231825237381, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956341911764706, + "pass@2": 0.9985670449282076, + "pass@4": 0.999778557092128, + "pass@8": 0.9999916503991577, + "pass@16": 0.9999999884463692, + "pass@32": 0.9999999999999981, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990388406940063, + "pass@2": 0.9997124090265532, + "pass@4": 0.9999756780205314, + "pass@8": 0.9999998642506372, + "pass@16": 0.9999999999987255, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999505537974683, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9910848348348348, + "pass@2": 0.993672190891679, + "pass@4": 0.9957048705060514, + "pass@8": 0.9975737767091117, + "pass@16": 0.9990568254400874, + "pass@32": 0.9998429950913355, + "pass@64": 0.999997993595807, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954280546623794, + "pass@2": 0.998397030154189, + "pass@4": 0.9996241085820865, + "pass@8": 0.9999720666894278, + "pass@16": 0.9999998580882329, + "pass@32": 0.9999999999993795, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.55558671875, + "avg_response_len": 351.02036953125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.55558671875, + "pass@2": 0.6285419783464454, + "pass@4": 0.6892563927446473, + "pass@8": 0.7387642739442581, + "pass@16": 0.7783981090494771, + "pass@32": 0.8102095441289227, + "pass@64": 0.8363550543496012, + "pass@128": 0.8585 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.950765625, + "pass@2": 0.9614740403543309, + "pass@4": 0.9687201411698537, + "pass@8": 0.974386017494287, + "pass@16": 0.978951297963125, + "pass@32": 0.9826278574459738, + "pass@64": 0.9854942292520122, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.8991484375, + "pass@2": 0.920307086614174, + "pass@4": 0.9330339433820775, + "pass@8": 0.9424397631143605, + "pass@16": 0.950055418886693, + "pass@32": 0.9568861029273782, + "pass@64": 0.9631518934285513, + "pass@128": 0.969 + }, + "13": { + "pass@1": 0.797796875, + "pass@2": 0.8472956446850397, + "pass@4": 0.8769101734158229, + "pass@8": 0.8972971475668615, + "pass@16": 0.9131068623816517, + "pass@32": 0.9267433357298904, + "pass@64": 0.9388567248399201, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.7676796875, + "pass@2": 0.8373948080708675, + "pass@4": 0.877752200693663, + "pass@8": 0.9021086412190684, + "pass@16": 0.9186930996933489, + "pass@32": 0.9307486366834251, + "pass@64": 0.9396032011202833, + "pass@128": 0.947 + }, + "15": { + "pass@1": 0.6468671875, + "pass@2": 0.7423140994094497, + "pass@4": 0.8082801520434945, + "pass@8": 0.8532897777630116, + "pass@16": 0.8838873036221627, + "pass@32": 0.9052644380269023, + "pass@64": 0.9206570299508262, + "pass@128": 0.933 + }, + "16": { + "pass@1": 0.469984375, + "pass@2": 0.5875038139763786, + "pass@4": 0.6800288287401575, + "pass@8": 0.7495063421768612, + "pass@16": 0.8020241155596487, + "pass@32": 0.841469117706701, + "pass@64": 0.8708999688448504, + "pass@128": 0.891 + }, + "17": { + "pass@1": 0.36625, + "pass@2": 0.4789491879921264, + "pass@4": 0.5784904901574796, + "pass@8": 0.6586822957108679, + "pass@16": 0.7202093202474215, + "pass@32": 0.7665917795921545, + "pass@64": 0.80311440631741, + "pass@128": 0.833 + }, + "18": { + "pass@1": 0.2888828125, + "pass@2": 0.38305155019685017, + "pass@4": 0.471190809242594, + "pass@8": 0.548671447428103, + "pass@16": 0.6143818676232548, + "pass@32": 0.6696475014396696, + "pass@64": 0.717475526520711, + "pass@128": 0.762 + }, + "19": { + "pass@1": 0.1931328125, + "pass@2": 0.27672010334645697, + "pass@4": 0.366691223284589, + "pass@8": 0.4521304833092716, + "pass@16": 0.525335973902767, + "pass@32": 0.5869952553452574, + "pass@64": 0.6413526713944006, + "pass@128": 0.691 + }, + "20": { + "pass@1": 0.175359375, + "pass@2": 0.25040944881889754, + "pass@4": 0.33146596531683503, + "pass@8": 0.40913082365990827, + "pass@16": 0.4773358306145655, + "pass@32": 0.5351214163918355, + "pass@64": 0.5829448918269658, + "pass@128": 0.622 + } + }, + "per_op_accuracy": { + "11": 0.950765625, + "12": 0.8991484375, + "13": 0.797796875, + "14": 0.7676796875, + "15": 0.6468671875, + "16": 0.469984375, + "17": 0.36625, + "18": 0.2888828125, + "19": 0.1931328125, + "20": 0.175359375 + }, + "per_op_avg_response_len": { + "11": 297.8854921875, + "12": 307.992015625, + "13": 325.7980546875, + "14": 350.7086640625, + "15": 362.6583359375, + "16": 367.1412265625, + "17": 376.8088203125, + "18": 373.2311484375, + "19": 374.4705546875, + "20": 373.5093828125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.8854921875, + "12": 307.992015625, + "13": 325.7980546875, + "14": 350.7086640625, + "15": 362.6583359375, + "16": 367.1412265625, + "17": 376.8088203125, + "18": 373.2311484375, + "19": 374.4705546875, + "20": 373.5093828125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 238948, + "answer_accuracy": 0.5547641158989599, + "avg_response_len": 333.8713572622585, + "resp_tokens_sum": 143805071, + "pass_at_k": { + "pass@1": 0.5547641158989599, + "pass@2": 0.6264341487171133, + "pass@4": 0.6857473474433831, + "pass@8": 0.7341518732755067, + "pass@16": 0.7731272066178299, + "pass@32": 0.8051157256009613, + "pass@64": 0.8320628101880153, + "pass@128": 0.8549777117384844 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9381196883852692, + "pass@2": 0.949643870870603, + "pass@4": 0.9578818050541134, + "pass@8": 0.9649502869843438, + "pass@16": 0.9710883897433248, + "pass@32": 0.9766777608768118, + "pass@64": 0.9808954606297404, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.8818193855932204, + "pass@2": 0.9022261054762225, + "pass@4": 0.914209971370104, + "pass@8": 0.9234397332635903, + "pass@16": 0.9316021266218226, + "pass@32": 0.9400656932927539, + "pass@64": 0.9486927052478563, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8077243338557993, + "pass@2": 0.8541556105447636, + "pass@4": 0.8810285076042608, + "pass@8": 0.8995733119234237, + "pass@16": 0.9142749359866263, + "pass@32": 0.9276676679813841, + "pass@64": 0.9396258281637503, + "pass@128": 0.9467084639498433 + }, + "14": { + "pass@1": 0.7615362149532711, + "pass@2": 0.8287777184978043, + "pass@4": 0.8663982992196069, + "pass@8": 0.8900014538631619, + "pass@16": 0.9092698366197164, + "pass@32": 0.9263179901559615, + "pass@64": 0.9397190795663035, + "pass@128": 0.9501557632398754 + }, + "15": { + "pass@1": 0.6521179552023122, + "pass@2": 0.7485534909653632, + "pass@4": 0.8158557660422503, + "pass@8": 0.8616490020826003, + "pass@16": 0.8918965274163206, + "pass@32": 0.912556561872447, + "pass@64": 0.9272614909305799, + "pass@128": 0.9393063583815029 + }, + "16": { + "pass@1": 0.47439516129032255, + "pass@2": 0.5907647002794009, + "pass@4": 0.6795826281795416, + "pass@8": 0.7443972792413011, + "pass@16": 0.7924224915481661, + "pass@32": 0.8295373414305435, + "pass@64": 0.8601997003236361, + "pass@128": 0.8838709677419355 + }, + "17": { + "pass@1": 0.3637102601156069, + "pass@2": 0.4765169854581037, + "pass@4": 0.5759331201735617, + "pass@8": 0.657336958398864, + "pass@16": 0.7201510019520492, + "pass@32": 0.765166904477689, + "pass@64": 0.7973211080359744, + "pass@128": 0.8208092485549133 + }, + "18": { + "pass@1": 0.2716294642857143, + "pass@2": 0.3653726096737904, + "pass@4": 0.4531094445337191, + "pass@8": 0.5287128623379376, + "pass@16": 0.5931104674774232, + "pass@32": 0.651151526924924, + "pass@64": 0.7066112942950599, + "pass@128": 0.7657142857142857 + }, + "19": { + "pass@1": 0.190503003003003, + "pass@2": 0.2718482508098649, + "pass@4": 0.35988050423877216, + "pass@8": 0.44355583224858297, + "pass@16": 0.5153708745650099, + "pass@32": 0.5764999950100734, + "pass@64": 0.6307394057060325, + "pass@128": 0.6786786786786787 + }, + "20": { + "pass@1": 0.1930837087087087, + "pass@2": 0.2684924097325671, + "pass@4": 0.34821977855245556, + "pass@8": 0.4238909629104684, + "pass@16": 0.4889963703968851, + "pass@32": 0.5431334016314513, + "pass@64": 0.5876574049594343, + "pass@128": 0.6216216216216216 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 242269, + "answer_accuracy": 0.5628089689265536, + "avg_response_len": 347.32652440157597, + "resp_tokens_sum": 149511565, + "pass_at_k": { + "pass@1": 0.5628089689265536, + "pass@2": 0.6362060364527389, + "pass@4": 0.6964946479705222, + "pass@8": 0.7448835029089924, + "pass@16": 0.783638754863171, + "pass@32": 0.8147834298902633, + "pass@64": 0.8407808844542585, + "pass@128": 0.864109426107642 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9620905496987951, + "pass@2": 0.9702171135328715, + "pass@4": 0.9740242053704128, + "pass@8": 0.9761798868262478, + "pass@16": 0.978471201018388, + "pass@32": 0.9812143276201, + "pass@64": 0.9841708700183511, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8975068573667712, + "pass@2": 0.9199666619356751, + "pass@4": 0.9324793790470546, + "pass@8": 0.9407002933796882, + "pass@16": 0.9474030994992948, + "pass@32": 0.9534276316888352, + "pass@64": 0.9596213973958759, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.8133315826330533, + "pass@2": 0.8619683109464253, + "pass@4": 0.8907345877983739, + "pass@8": 0.9097965719918113, + "pass@16": 0.9240904902930944, + "pass@32": 0.9352971856676536, + "pass@64": 0.9442273930544399, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7862070540935673, + "pass@2": 0.8562384163328274, + "pass@4": 0.8968682741411707, + "pass@8": 0.9215926852420111, + "pass@16": 0.9376229800783108, + "pass@32": 0.9472395252470556, + "pass@64": 0.9533718603838929, + "pass@128": 0.9590643274853801 + }, + "15": { + "pass@1": 0.6526751893939394, + "pass@2": 0.7521571522309706, + "pass@4": 0.8187771744441035, + "pass@8": 0.8622644853436704, + "pass@16": 0.8930507885397345, + "pass@32": 0.9162751773331523, + "pass@64": 0.9328910725136462, + "pass@128": 0.9454545454545454 + }, + "16": { + "pass@1": 0.4648548789173789, + "pass@2": 0.5819018215671757, + "pass@4": 0.6748660992696419, + "pass@8": 0.7450755200307121, + "pass@16": 0.799403052133679, + "pass@32": 0.8416186844245477, + "pass@64": 0.8721309045304662, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.3935607531055901, + "pass@2": 0.509288877952756, + "pass@4": 0.6092241197917025, + "pass@8": 0.6847584806406737, + "pass@16": 0.7389377235622425, + "pass@32": 0.7802741974544923, + "pass@64": 0.8185235964385869, + "pass@128": 0.8571428571428571 + }, + "18": { + "pass@1": 0.29558823529411765, + "pass@2": 0.39329840203798055, + "pass@4": 0.48604767501488755, + "pass@8": 0.5690211737279396, + "pass@16": 0.6398204399444749, + "pass@32": 0.6954694440103872, + "pass@64": 0.7373755892292073, + "pass@128": 0.7705882352941177 + }, + "19": { + "pass@1": 0.20124162946428573, + "pass@2": 0.2826659607002248, + "pass@4": 0.36939328119699344, + "pass@8": 0.45250315351747245, + "pass@16": 0.5246492489077819, + "pass@32": 0.5865849417253087, + "pass@64": 0.6441483178327181, + "pass@128": 0.7023809523809523 + }, + "20": { + "pass@1": 0.1627994011976048, + "pass@2": 0.23435805554245845, + "pass@4": 0.3117855043568655, + "pass@8": 0.3856789000143374, + "pass@16": 0.45113612331383535, + "pass@32": 0.5083044335567928, + "pass@64": 0.5594660767363314, + "pass@128": 0.6077844311377245 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 229934, + "answer_accuracy": 0.5490095889364304, + "avg_response_len": 372.45338525748775, + "resp_tokens_sum": 155989437, + "pass_at_k": { + "pass@1": 0.5490095889364304, + "pass@2": 0.6228325099147168, + "pass@4": 0.6854256119115953, + "pass@8": 0.7372183574534321, + "pass@16": 0.7784324289794793, + "pass@32": 0.8107470507093448, + "pass@64": 0.8362203767706432, + "pass@128": 0.8563569682151589 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9530009920634921, + "pass@2": 0.9655164198225222, + "pass@4": 0.9752756438183323, + "pass@8": 0.9830693452777767, + "pass@16": 0.9882687544305585, + "pass@32": 0.9907855590050356, + "pass@64": 0.9920425485829253, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.919509747706422, + "pass@2": 0.9402130890462086, + "pass@4": 0.9539531853243111, + "pass@8": 0.9647055166695085, + "pass@16": 0.9726198083250552, + "pass@32": 0.978469214045886, + "pass@64": 0.9822490825734729, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7709056712962963, + "pass@2": 0.8243744380042777, + "pass@4": 0.8576229371791491, + "pass@8": 0.8812835952537411, + "pass@16": 0.8998544838496414, + "pass@32": 0.9164081924706067, + "pass@64": 0.9321818096766924, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.7547292284866469, + "pass@2": 0.8264795906446407, + "pass@4": 0.8691673498156945, + "pass@8": 0.8938678818285647, + "pass@16": 0.9084582283431383, + "pass@32": 0.9182333656049506, + "pass@64": 0.9255198822795507, + "pass@128": 0.9317507418397626 + }, + "15": { + "pass@1": 0.6353443287037037, + "pass@2": 0.7256257138864582, + "pass@4": 0.7894987327972884, + "pass@8": 0.8352220459229024, + "pass@16": 0.8660010645617393, + "pass@32": 0.8862625280836279, + "pass@64": 0.9011435190103152, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.4712619837758112, + "pass@2": 0.5903221755742922, + "pass@4": 0.6857823397075362, + "pass@8": 0.758766012039169, + "pass@16": 0.8135182061380419, + "pass@32": 0.8522253204431165, + "pass@64": 0.8794103665319446, + "pass@128": 0.8967551622418879 + }, + "17": { + "pass@1": 0.3424086972891566, + "pass@2": 0.4520581094061286, + "pass@4": 0.5513477831460822, + "pass@8": 0.6347936064414588, + "pass@16": 0.7021058029667845, + "pass@32": 0.7548064429594218, + "pass@64": 0.7942069424208933, + "pass@128": 0.822289156626506 + }, + "18": { + "pass@1": 0.30100806451612905, + "pass@2": 0.39177316167132326, + "pass@4": 0.4753106262926812, + "pass@8": 0.548886279168791, + "pass@16": 0.6104975949194693, + "pass@32": 0.6622092130723058, + "pass@64": 0.7079157199338997, + "pass@128": 0.7483870967741936 + }, + "19": { + "pass@1": 0.18754720543806647, + "pass@2": 0.27558571912838686, + "pass@4": 0.3708002202141651, + "pass@8": 0.4603786452828474, + "pass@16": 0.5360583838054499, + "pass@32": 0.5979704428616274, + "pass@64": 0.6491921888292985, + "pass@128": 0.6918429003021148 + }, + "20": { + "pass@1": 0.17023273273273273, + "pass@2": 0.24842608356387852, + "pass@4": 0.3344517135245482, + "pass@8": 0.4178930342520521, + "pass@16": 0.4919536759326808, + "pass@32": 0.5540069454672477, + "pass@64": 0.6017817007373545, + "pass@128": 0.6366366366366366 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7632705592105263, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7632705592105263, + "pass@2": 0.8025510839722155, + "pass@4": 0.8350644367414456, + "pass@8": 0.8615117046128136, + "pass@16": 0.8826737205220483, + "pass@32": 0.8996543512913374, + "pass@64": 0.9135941082675603, + "pass@128": 0.9253684210526316 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9592109375, + "pass@2": 0.9698198818897638, + "pass@4": 0.9775970624296962, + "pass@8": 0.9833988470253193, + "pass@16": 0.9880673774247114, + "pass@32": 0.9919647338892076, + "pass@64": 0.9949862407389606, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999609375, + "pass@2": 0.9999997539370079, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9973125, + "pass@2": 0.9988038877952756, + "pass@4": 0.9994920357142858, + "pass@8": 0.9998169993777086, + "pass@16": 0.9999715651432676, + "pass@32": 0.9999995674757711, + "pass@64": 0.9999999999958465, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999359375, + "pass@2": 0.9998471948818897, + "pass@4": 0.9999893286464192, + "pass@8": 0.9999999514497075, + "pass@16": 0.9999999999995928, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99915625, + "pass@2": 0.9998037647637795, + "pass@4": 0.9999841450131234, + "pass@8": 0.9999998717991873, + "pass@16": 0.999999999995534, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9951953125, + "pass@2": 0.9971200787401576, + "pass@4": 0.9982833751406074, + "pass@8": 0.9991360598170629, + "pass@16": 0.9996836215015343, + "pass@32": 0.9999477157399603, + "pass@64": 0.9999993318674038, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.996078125, + "pass@2": 0.9976562500000001, + "pass@4": 0.9983144236970378, + "pass@8": 0.9987279187316727, + "pass@16": 0.9990970353595758, + "pass@32": 0.9994252161410782, + "pass@64": 0.9997519409854931, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.950765625, + "pass@2": 0.9614740403543309, + "pass@4": 0.9687201411698537, + "pass@8": 0.974386017494287, + "pass@16": 0.978951297963125, + "pass@32": 0.9826278574459738, + "pass@64": 0.9854942292520122, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.8991484375, + "pass@2": 0.920307086614174, + "pass@4": 0.9330339433820775, + "pass@8": 0.9424397631143605, + "pass@16": 0.950055418886693, + "pass@32": 0.9568861029273782, + "pass@64": 0.9631518934285513, + "pass@128": 0.969 + }, + "13": { + "pass@1": 0.797796875, + "pass@2": 0.8472956446850397, + "pass@4": 0.8769101734158229, + "pass@8": 0.8972971475668615, + "pass@16": 0.9131068623816517, + "pass@32": 0.9267433357298904, + "pass@64": 0.9388567248399201, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.7676796875, + "pass@2": 0.8373948080708675, + "pass@4": 0.877752200693663, + "pass@8": 0.9021086412190684, + "pass@16": 0.9186930996933489, + "pass@32": 0.9307486366834251, + "pass@64": 0.9396032011202833, + "pass@128": 0.947 + }, + "15": { + "pass@1": 0.6468671875, + "pass@2": 0.7423140994094497, + "pass@4": 0.8082801520434945, + "pass@8": 0.8532897777630116, + "pass@16": 0.8838873036221627, + "pass@32": 0.9052644380269023, + "pass@64": 0.9206570299508262, + "pass@128": 0.933 + }, + "16": { + "pass@1": 0.469984375, + "pass@2": 0.5875038139763786, + "pass@4": 0.6800288287401575, + "pass@8": 0.7495063421768612, + "pass@16": 0.8020241155596487, + "pass@32": 0.841469117706701, + "pass@64": 0.8708999688448504, + "pass@128": 0.891 + }, + "17": { + "pass@1": 0.36625, + "pass@2": 0.4789491879921264, + "pass@4": 0.5784904901574796, + "pass@8": 0.6586822957108679, + "pass@16": 0.7202093202474215, + "pass@32": 0.7665917795921545, + "pass@64": 0.80311440631741, + "pass@128": 0.833 + }, + "18": { + "pass@1": 0.2888828125, + "pass@2": 0.38305155019685017, + "pass@4": 0.471190809242594, + "pass@8": 0.548671447428103, + "pass@16": 0.6143818676232548, + "pass@32": 0.6696475014396696, + "pass@64": 0.717475526520711, + "pass@128": 0.762 + }, + "19": { + "pass@1": 0.1931328125, + "pass@2": 0.27672010334645697, + "pass@4": 0.366691223284589, + "pass@8": 0.4521304833092716, + "pass@16": 0.525335973902767, + "pass@32": 0.5869952553452574, + "pass@64": 0.6413526713944006, + "pass@128": 0.691 + }, + "20": { + "pass@1": 0.175359375, + "pass@2": 0.25040944881889754, + "pass@4": 0.33146596531683503, + "pass@8": 0.40913082365990827, + "pass@16": 0.4773358306145655, + "pass@32": 0.5351214163918355, + "pass@64": 0.5829448918269658, + "pass@128": 0.622 + } + }, + "per_op_accuracy": { + "10": 0.9592109375, + "2": 1.0, + "3": 1.0, + "4": 0.9999609375, + "5": 0.9973125, + "6": 0.999359375, + "7": 0.99915625, + "8": 0.9951953125, + "9": 0.996078125, + "11": 0.950765625, + "12": 0.8991484375, + "13": 0.797796875, + "14": 0.7676796875, + "15": 0.6468671875, + "16": 0.469984375, + "17": 0.36625, + "18": 0.2888828125, + "19": 0.1931328125, + "20": 0.175359375 + }, + "per_op_avg_response_len": { + "10": 280.5589140625, + "2": 115.5463046875, + "3": 144.54734375, + "4": 159.7105390625, + "5": 172.6188515625, + "6": 198.7573515625, + "7": 223.5936328125, + "8": 243.207015625, + "9": 258.99440625, + "11": 297.8854921875, + "12": 307.992015625, + "13": 325.7980546875, + "14": 350.7086640625, + "15": 362.6583359375, + "16": 367.1412265625, + "17": 376.8088203125, + "18": 373.2311484375, + "19": 374.4705546875, + "20": 373.5093828125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.5589140625, + "2": 115.5463046875, + "3": 144.54734375, + "4": 159.7105390625, + "5": 172.6188515625, + "6": 198.7573515625, + "7": 223.5936328125, + "8": 243.207015625, + "9": 258.99440625, + "11": 297.8854921875, + "12": 307.992015625, + "13": 325.7980546875, + "14": 350.7086640625, + "15": 362.6583359375, + "16": 367.1412265625, + "17": 376.8088203125, + "18": 373.2311484375, + "19": 374.4705546875, + "20": 373.5093828125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 615982, + "answer_accuracy": 0.7607270589630099, + "avg_response_len": 266.6447597217831, + "resp_tokens_sum": 215909728, + "pass_at_k": { + "pass@1": 0.7607270589630099, + "pass@2": 0.799666683055299, + "pass@4": 0.8316806485035426, + "pass@8": 0.8577287808674091, + "pass@16": 0.8787053843237128, + "pass@32": 0.8959319819086902, + "pass@64": 0.9104266792092892, + "pass@128": 0.9226999683844451 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9638873525073747, + "pass@2": 0.9752224002973081, + "pass@4": 0.9824726737697609, + "pass@8": 0.9871089937889923, + "pass@16": 0.9908861524344581, + "pass@32": 0.9941379264111941, + "pass@64": 0.9962119406945845, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99990234375, + "pass@2": 0.9999992310531496, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966517857142857, + "pass@2": 0.9977686382952132, + "pass@4": 0.9986264582998554, + "pass@8": 0.999428057911097, + "pass@16": 0.999909743084133, + "pass@32": 0.9999986269072114, + "pass@64": 0.9999999999868145, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999766092814372, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9981508875739645, + "pass@2": 0.9994958620649489, + "pass@4": 0.9999547515880042, + "pass@8": 0.9999996211287503, + "pass@16": 0.9999999999867872, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9990985576923077, + "pass@2": 0.9999593838330306, + "pass@4": 0.9999997803760107, + "pass@8": 0.9999999999971149, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9972572640567817, + "pass@4": 0.9973570422535211, + "pass@8": 0.9975255073749407, + "pass@16": 0.9978457358323166, + "pass@32": 0.998419651768881, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9381196883852692, + "pass@2": 0.949643870870603, + "pass@4": 0.9578818050541134, + "pass@8": 0.9649502869843438, + "pass@16": 0.9710883897433248, + "pass@32": 0.9766777608768118, + "pass@64": 0.9808954606297404, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.8818193855932204, + "pass@2": 0.9022261054762225, + "pass@4": 0.914209971370104, + "pass@8": 0.9234397332635903, + "pass@16": 0.9316021266218226, + "pass@32": 0.9400656932927539, + "pass@64": 0.9486927052478563, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8077243338557993, + "pass@2": 0.8541556105447636, + "pass@4": 0.8810285076042608, + "pass@8": 0.8995733119234237, + "pass@16": 0.9142749359866263, + "pass@32": 0.9276676679813841, + "pass@64": 0.9396258281637503, + "pass@128": 0.9467084639498433 + }, + "14": { + "pass@1": 0.7615362149532711, + "pass@2": 0.8287777184978043, + "pass@4": 0.8663982992196069, + "pass@8": 0.8900014538631619, + "pass@16": 0.9092698366197164, + "pass@32": 0.9263179901559615, + "pass@64": 0.9397190795663035, + "pass@128": 0.9501557632398754 + }, + "15": { + "pass@1": 0.6521179552023122, + "pass@2": 0.7485534909653632, + "pass@4": 0.8158557660422503, + "pass@8": 0.8616490020826003, + "pass@16": 0.8918965274163206, + "pass@32": 0.912556561872447, + "pass@64": 0.9272614909305799, + "pass@128": 0.9393063583815029 + }, + "16": { + "pass@1": 0.47439516129032255, + "pass@2": 0.5907647002794009, + "pass@4": 0.6795826281795416, + "pass@8": 0.7443972792413011, + "pass@16": 0.7924224915481661, + "pass@32": 0.8295373414305435, + "pass@64": 0.8601997003236361, + "pass@128": 0.8838709677419355 + }, + "17": { + "pass@1": 0.3637102601156069, + "pass@2": 0.4765169854581037, + "pass@4": 0.5759331201735617, + "pass@8": 0.657336958398864, + "pass@16": 0.7201510019520492, + "pass@32": 0.765166904477689, + "pass@64": 0.7973211080359744, + "pass@128": 0.8208092485549133 + }, + "18": { + "pass@1": 0.2716294642857143, + "pass@2": 0.3653726096737904, + "pass@4": 0.4531094445337191, + "pass@8": 0.5287128623379376, + "pass@16": 0.5931104674774232, + "pass@32": 0.651151526924924, + "pass@64": 0.7066112942950599, + "pass@128": 0.7657142857142857 + }, + "19": { + "pass@1": 0.190503003003003, + "pass@2": 0.2718482508098649, + "pass@4": 0.35988050423877216, + "pass@8": 0.44355583224858297, + "pass@16": 0.5153708745650099, + "pass@32": 0.5764999950100734, + "pass@64": 0.6307394057060325, + "pass@128": 0.6786786786786787 + }, + "20": { + "pass@1": 0.1930837087087087, + "pass@2": 0.2684924097325671, + "pass@4": 0.34821977855245556, + "pass@8": 0.4238909629104684, + "pass@16": 0.4889963703968851, + "pass@32": 0.5431334016314513, + "pass@64": 0.5876574049594343, + "pass@128": 0.6216216216216216 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 630327, + "answer_accuracy": 0.7676429754481684, + "avg_response_len": 276.1505991816056, + "resp_tokens_sum": 226752780, + "pass_at_k": { + "pass@1": 0.7676429754481684, + "pass@2": 0.8070075019485474, + "pass@4": 0.8392237318591167, + "pass@8": 0.8650411663736571, + "pass@16": 0.8857172601787794, + "pass@32": 0.9023291961217534, + "pass@64": 0.9161729093685608, + "pass@128": 0.9286048324240063 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9486435966257669, + "pass@2": 0.9604827363412394, + "pass@4": 0.9700275196620363, + "pass@8": 0.9774135724453075, + "pass@16": 0.983556900424375, + "pass@32": 0.988769571356498, + "pass@64": 0.9929543763743685, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995697463768116, + "pass@2": 0.9999825259614288, + "pass@4": 0.9999999766333555, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990598137535817, + "pass@2": 0.9998233845858808, + "pass@4": 0.9999915149395725, + "pass@8": 0.9999999841898438, + "pass@16": 0.9999999999999909, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9994129335260116, + "pass@2": 0.9999253277047017, + "pass@4": 0.9999983785444451, + "pass@8": 0.9999999995886407, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956205985915493, + "pass@2": 0.9978589054009094, + "pass@4": 0.9991935824923294, + "pass@8": 0.9998422314755744, + "pass@16": 0.9999935172675638, + "pass@32": 0.9999999954212553, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9960469685628742, + "pass@2": 0.9973905535385922, + "pass@4": 0.9981125027784701, + "pass@8": 0.9988474577040616, + "pass@16": 0.9995863571067751, + "pass@32": 0.999958801686582, + "pass@64": 0.9999999176094492, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9620905496987951, + "pass@2": 0.9702171135328715, + "pass@4": 0.9740242053704128, + "pass@8": 0.9761798868262478, + "pass@16": 0.978471201018388, + "pass@32": 0.9812143276201, + "pass@64": 0.9841708700183511, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8975068573667712, + "pass@2": 0.9199666619356751, + "pass@4": 0.9324793790470546, + "pass@8": 0.9407002933796882, + "pass@16": 0.9474030994992948, + "pass@32": 0.9534276316888352, + "pass@64": 0.9596213973958759, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.8133315826330533, + "pass@2": 0.8619683109464253, + "pass@4": 0.8907345877983739, + "pass@8": 0.9097965719918113, + "pass@16": 0.9240904902930944, + "pass@32": 0.9352971856676536, + "pass@64": 0.9442273930544399, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7862070540935673, + "pass@2": 0.8562384163328274, + "pass@4": 0.8968682741411707, + "pass@8": 0.9215926852420111, + "pass@16": 0.9376229800783108, + "pass@32": 0.9472395252470556, + "pass@64": 0.9533718603838929, + "pass@128": 0.9590643274853801 + }, + "15": { + "pass@1": 0.6526751893939394, + "pass@2": 0.7521571522309706, + "pass@4": 0.8187771744441035, + "pass@8": 0.8622644853436704, + "pass@16": 0.8930507885397345, + "pass@32": 0.9162751773331523, + "pass@64": 0.9328910725136462, + "pass@128": 0.9454545454545454 + }, + "16": { + "pass@1": 0.4648548789173789, + "pass@2": 0.5819018215671757, + "pass@4": 0.6748660992696419, + "pass@8": 0.7450755200307121, + "pass@16": 0.799403052133679, + "pass@32": 0.8416186844245477, + "pass@64": 0.8721309045304662, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.3935607531055901, + "pass@2": 0.509288877952756, + "pass@4": 0.6092241197917025, + "pass@8": 0.6847584806406737, + "pass@16": 0.7389377235622425, + "pass@32": 0.7802741974544923, + "pass@64": 0.8185235964385869, + "pass@128": 0.8571428571428571 + }, + "18": { + "pass@1": 0.29558823529411765, + "pass@2": 0.39329840203798055, + "pass@4": 0.48604767501488755, + "pass@8": 0.5690211737279396, + "pass@16": 0.6398204399444749, + "pass@32": 0.6954694440103872, + "pass@64": 0.7373755892292073, + "pass@128": 0.7705882352941177 + }, + "19": { + "pass@1": 0.20124162946428573, + "pass@2": 0.2826659607002248, + "pass@4": 0.36939328119699344, + "pass@8": 0.45250315351747245, + "pass@16": 0.5246492489077819, + "pass@32": 0.5865849417253087, + "pass@64": 0.6441483178327181, + "pass@128": 0.7023809523809523 + }, + "20": { + "pass@1": 0.1627994011976048, + "pass@2": 0.23435805554245845, + "pass@4": 0.3117855043568655, + "pass@8": 0.3856789000143374, + "pass@16": 0.45113612331383535, + "pass@32": 0.5083044335567928, + "pass@64": 0.5594660767363314, + "pass@128": 0.6077844311377245 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 609965, + "answer_accuracy": 0.7613598917558716, + "avg_response_len": 295.48445613316824, + "resp_tokens_sum": 236727963, + "pass_at_k": { + "pass@1": 0.7613598917558716, + "pass@2": 0.8008988709801111, + "pass@4": 0.8342214851858113, + "pass@8": 0.8617176925369859, + "pass@16": 0.8835651389423725, + "pass@32": 0.9006750541396102, + "pass@64": 0.9141523679192285, + "pass@128": 0.9247483623582042 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9647621268656716, + "pass@2": 0.9734391526618872, + "pass@4": 0.9800294167520107, + "pass@8": 0.9854688761602408, + "pass@16": 0.98960424540622, + "pass@32": 0.9928749151450584, + "pass@64": 0.9957231825237381, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956341911764706, + "pass@2": 0.9985670449282076, + "pass@4": 0.999778557092128, + "pass@8": 0.9999916503991577, + "pass@16": 0.9999999884463692, + "pass@32": 0.9999999999999981, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990388406940063, + "pass@2": 0.9997124090265532, + "pass@4": 0.9999756780205314, + "pass@8": 0.9999998642506372, + "pass@16": 0.9999999999987255, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999505537974683, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9910848348348348, + "pass@2": 0.993672190891679, + "pass@4": 0.9957048705060514, + "pass@8": 0.9975737767091117, + "pass@16": 0.9990568254400874, + "pass@32": 0.9998429950913355, + "pass@64": 0.999997993595807, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954280546623794, + "pass@2": 0.998397030154189, + "pass@4": 0.9996241085820865, + "pass@8": 0.9999720666894278, + "pass@16": 0.9999998580882329, + "pass@32": 0.9999999999993795, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9530009920634921, + "pass@2": 0.9655164198225222, + "pass@4": 0.9752756438183323, + "pass@8": 0.9830693452777767, + "pass@16": 0.9882687544305585, + "pass@32": 0.9907855590050356, + "pass@64": 0.9920425485829253, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.919509747706422, + "pass@2": 0.9402130890462086, + "pass@4": 0.9539531853243111, + "pass@8": 0.9647055166695085, + "pass@16": 0.9726198083250552, + "pass@32": 0.978469214045886, + "pass@64": 0.9822490825734729, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7709056712962963, + "pass@2": 0.8243744380042777, + "pass@4": 0.8576229371791491, + "pass@8": 0.8812835952537411, + "pass@16": 0.8998544838496414, + "pass@32": 0.9164081924706067, + "pass@64": 0.9321818096766924, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.7547292284866469, + "pass@2": 0.8264795906446407, + "pass@4": 0.8691673498156945, + "pass@8": 0.8938678818285647, + "pass@16": 0.9084582283431383, + "pass@32": 0.9182333656049506, + "pass@64": 0.9255198822795507, + "pass@128": 0.9317507418397626 + }, + "15": { + "pass@1": 0.6353443287037037, + "pass@2": 0.7256257138864582, + "pass@4": 0.7894987327972884, + "pass@8": 0.8352220459229024, + "pass@16": 0.8660010645617393, + "pass@32": 0.8862625280836279, + "pass@64": 0.9011435190103152, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.4712619837758112, + "pass@2": 0.5903221755742922, + "pass@4": 0.6857823397075362, + "pass@8": 0.758766012039169, + "pass@16": 0.8135182061380419, + "pass@32": 0.8522253204431165, + "pass@64": 0.8794103665319446, + "pass@128": 0.8967551622418879 + }, + "17": { + "pass@1": 0.3424086972891566, + "pass@2": 0.4520581094061286, + "pass@4": 0.5513477831460822, + "pass@8": 0.6347936064414588, + "pass@16": 0.7021058029667845, + "pass@32": 0.7548064429594218, + "pass@64": 0.7942069424208933, + "pass@128": 0.822289156626506 + }, + "18": { + "pass@1": 0.30100806451612905, + "pass@2": 0.39177316167132326, + "pass@4": 0.4753106262926812, + "pass@8": 0.548886279168791, + "pass@16": 0.6104975949194693, + "pass@32": 0.6622092130723058, + "pass@64": 0.7079157199338997, + "pass@128": 0.7483870967741936 + }, + "19": { + "pass@1": 0.18754720543806647, + "pass@2": 0.27558571912838686, + "pass@4": 0.3708002202141651, + "pass@8": 0.4603786452828474, + "pass@16": 0.5360583838054499, + "pass@32": 0.5979704428616274, + "pass@64": 0.6491921888292985, + "pass@128": 0.6918429003021148 + }, + "20": { + "pass@1": 0.17023273273273273, + "pass@2": 0.24842608356387852, + "pass@4": 0.3344517135245482, + "pass@8": 0.4178930342520521, + "pass@16": 0.4919536759326808, + "pass@32": 0.5540069454672477, + "pass@64": 0.6017817007373545, + "pass@128": 0.6366366366366366 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep4-block4-op9-12/composition/autoregressive/summary.csv b/SDAR-100M-ep4-block4-op9-12/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..48019b23453a942435f8252576b7a7e40f626b6e --- /dev/null +++ b/SDAR-100M-ep4-block4-op9-12/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.9940303819444445,0.55558671875,0.7632705592105263,0.0,0.0,0.0,199.72603993055554,351.02036953125,1152000,1280000,2432000 diff --git a/SDAR-100M-ep4-block4-op9-12/composition/autoregressive/summary.json b/SDAR-100M-ep4-block4-op9-12/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0f60edd9908a27870b7ede2fd487871c0e49b91f --- /dev/null +++ b/SDAR-100M-ep4-block4-op9-12/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.9940303819444445, + "ood_acc": 0.55558671875, + "total_acc": 0.7632705592105263, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.72603993055554, + "ood_avg_resp_len": 351.02036953125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 615982, + "answer_accuracy": 0.7607270589630099, + "avg_response_len": 266.6447597217831, + "resp_tokens_sum": 215909728, + "pass_at_k": { + "pass@1": 0.7607270589630099, + "pass@2": 0.799666683055299, + "pass@4": 0.8316806485035426, + "pass@8": 0.8577287808674091, + "pass@16": 0.8787053843237128, + "pass@32": 0.8959319819086902, + "pass@64": 0.9104266792092892, + "pass@128": 0.9226999683844451 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9638873525073747, + "pass@2": 0.9752224002973081, + "pass@4": 0.9824726737697609, + "pass@8": 0.9871089937889923, + "pass@16": 0.9908861524344581, + "pass@32": 0.9941379264111941, + "pass@64": 0.9962119406945845, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99990234375, + "pass@2": 0.9999992310531496, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966517857142857, + "pass@2": 0.9977686382952132, + "pass@4": 0.9986264582998554, + "pass@8": 0.999428057911097, + "pass@16": 0.999909743084133, + "pass@32": 0.9999986269072114, + "pass@64": 0.9999999999868145, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999766092814372, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9981508875739645, + "pass@2": 0.9994958620649489, + "pass@4": 0.9999547515880042, + "pass@8": 0.9999996211287503, + "pass@16": 0.9999999999867872, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9990985576923077, + "pass@2": 0.9999593838330306, + "pass@4": 0.9999997803760107, + "pass@8": 0.9999999999971149, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9972572640567817, + "pass@4": 0.9973570422535211, + "pass@8": 0.9975255073749407, + "pass@16": 0.9978457358323166, + "pass@32": 0.998419651768881, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9381196883852692, + "pass@2": 0.949643870870603, + "pass@4": 0.9578818050541134, + "pass@8": 0.9649502869843438, + "pass@16": 0.9710883897433248, + "pass@32": 0.9766777608768118, + "pass@64": 0.9808954606297404, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.8818193855932204, + "pass@2": 0.9022261054762225, + "pass@4": 0.914209971370104, + "pass@8": 0.9234397332635903, + "pass@16": 0.9316021266218226, + "pass@32": 0.9400656932927539, + "pass@64": 0.9486927052478563, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8077243338557993, + "pass@2": 0.8541556105447636, + "pass@4": 0.8810285076042608, + "pass@8": 0.8995733119234237, + "pass@16": 0.9142749359866263, + "pass@32": 0.9276676679813841, + "pass@64": 0.9396258281637503, + "pass@128": 0.9467084639498433 + }, + "14": { + "pass@1": 0.7615362149532711, + "pass@2": 0.8287777184978043, + "pass@4": 0.8663982992196069, + "pass@8": 0.8900014538631619, + "pass@16": 0.9092698366197164, + "pass@32": 0.9263179901559615, + "pass@64": 0.9397190795663035, + "pass@128": 0.9501557632398754 + }, + "15": { + "pass@1": 0.6521179552023122, + "pass@2": 0.7485534909653632, + "pass@4": 0.8158557660422503, + "pass@8": 0.8616490020826003, + "pass@16": 0.8918965274163206, + "pass@32": 0.912556561872447, + "pass@64": 0.9272614909305799, + "pass@128": 0.9393063583815029 + }, + "16": { + "pass@1": 0.47439516129032255, + "pass@2": 0.5907647002794009, + "pass@4": 0.6795826281795416, + "pass@8": 0.7443972792413011, + "pass@16": 0.7924224915481661, + "pass@32": 0.8295373414305435, + "pass@64": 0.8601997003236361, + "pass@128": 0.8838709677419355 + }, + "17": { + "pass@1": 0.3637102601156069, + "pass@2": 0.4765169854581037, + "pass@4": 0.5759331201735617, + "pass@8": 0.657336958398864, + "pass@16": 0.7201510019520492, + "pass@32": 0.765166904477689, + "pass@64": 0.7973211080359744, + "pass@128": 0.8208092485549133 + }, + "18": { + "pass@1": 0.2716294642857143, + "pass@2": 0.3653726096737904, + "pass@4": 0.4531094445337191, + "pass@8": 0.5287128623379376, + "pass@16": 0.5931104674774232, + "pass@32": 0.651151526924924, + "pass@64": 0.7066112942950599, + "pass@128": 0.7657142857142857 + }, + "19": { + "pass@1": 0.190503003003003, + "pass@2": 0.2718482508098649, + "pass@4": 0.35988050423877216, + "pass@8": 0.44355583224858297, + "pass@16": 0.5153708745650099, + "pass@32": 0.5764999950100734, + "pass@64": 0.6307394057060325, + "pass@128": 0.6786786786786787 + }, + "20": { + "pass@1": 0.1930837087087087, + "pass@2": 0.2684924097325671, + "pass@4": 0.34821977855245556, + "pass@8": 0.4238909629104684, + "pass@16": 0.4889963703968851, + "pass@32": 0.5431334016314513, + "pass@64": 0.5876574049594343, + "pass@128": 0.6216216216216216 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 630327, + "answer_accuracy": 0.7676429754481684, + "avg_response_len": 276.1505991816056, + "resp_tokens_sum": 226752780, + "pass_at_k": { + "pass@1": 0.7676429754481684, + "pass@2": 0.8070075019485474, + "pass@4": 0.8392237318591167, + "pass@8": 0.8650411663736571, + "pass@16": 0.8857172601787794, + "pass@32": 0.9023291961217534, + "pass@64": 0.9161729093685608, + "pass@128": 0.9286048324240063 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9486435966257669, + "pass@2": 0.9604827363412394, + "pass@4": 0.9700275196620363, + "pass@8": 0.9774135724453075, + "pass@16": 0.983556900424375, + "pass@32": 0.988769571356498, + "pass@64": 0.9929543763743685, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995697463768116, + "pass@2": 0.9999825259614288, + "pass@4": 0.9999999766333555, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990598137535817, + "pass@2": 0.9998233845858808, + "pass@4": 0.9999915149395725, + "pass@8": 0.9999999841898438, + "pass@16": 0.9999999999999909, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9994129335260116, + "pass@2": 0.9999253277047017, + "pass@4": 0.9999983785444451, + "pass@8": 0.9999999995886407, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956205985915493, + "pass@2": 0.9978589054009094, + "pass@4": 0.9991935824923294, + "pass@8": 0.9998422314755744, + "pass@16": 0.9999935172675638, + "pass@32": 0.9999999954212553, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9960469685628742, + "pass@2": 0.9973905535385922, + "pass@4": 0.9981125027784701, + "pass@8": 0.9988474577040616, + "pass@16": 0.9995863571067751, + "pass@32": 0.999958801686582, + "pass@64": 0.9999999176094492, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9620905496987951, + "pass@2": 0.9702171135328715, + "pass@4": 0.9740242053704128, + "pass@8": 0.9761798868262478, + "pass@16": 0.978471201018388, + "pass@32": 0.9812143276201, + "pass@64": 0.9841708700183511, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8975068573667712, + "pass@2": 0.9199666619356751, + "pass@4": 0.9324793790470546, + "pass@8": 0.9407002933796882, + "pass@16": 0.9474030994992948, + "pass@32": 0.9534276316888352, + "pass@64": 0.9596213973958759, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.8133315826330533, + "pass@2": 0.8619683109464253, + "pass@4": 0.8907345877983739, + "pass@8": 0.9097965719918113, + "pass@16": 0.9240904902930944, + "pass@32": 0.9352971856676536, + "pass@64": 0.9442273930544399, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7862070540935673, + "pass@2": 0.8562384163328274, + "pass@4": 0.8968682741411707, + "pass@8": 0.9215926852420111, + "pass@16": 0.9376229800783108, + "pass@32": 0.9472395252470556, + "pass@64": 0.9533718603838929, + "pass@128": 0.9590643274853801 + }, + "15": { + "pass@1": 0.6526751893939394, + "pass@2": 0.7521571522309706, + "pass@4": 0.8187771744441035, + "pass@8": 0.8622644853436704, + "pass@16": 0.8930507885397345, + "pass@32": 0.9162751773331523, + "pass@64": 0.9328910725136462, + "pass@128": 0.9454545454545454 + }, + "16": { + "pass@1": 0.4648548789173789, + "pass@2": 0.5819018215671757, + "pass@4": 0.6748660992696419, + "pass@8": 0.7450755200307121, + "pass@16": 0.799403052133679, + "pass@32": 0.8416186844245477, + "pass@64": 0.8721309045304662, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.3935607531055901, + "pass@2": 0.509288877952756, + "pass@4": 0.6092241197917025, + "pass@8": 0.6847584806406737, + "pass@16": 0.7389377235622425, + "pass@32": 0.7802741974544923, + "pass@64": 0.8185235964385869, + "pass@128": 0.8571428571428571 + }, + "18": { + "pass@1": 0.29558823529411765, + "pass@2": 0.39329840203798055, + "pass@4": 0.48604767501488755, + "pass@8": 0.5690211737279396, + "pass@16": 0.6398204399444749, + "pass@32": 0.6954694440103872, + "pass@64": 0.7373755892292073, + "pass@128": 0.7705882352941177 + }, + "19": { + "pass@1": 0.20124162946428573, + "pass@2": 0.2826659607002248, + "pass@4": 0.36939328119699344, + "pass@8": 0.45250315351747245, + "pass@16": 0.5246492489077819, + "pass@32": 0.5865849417253087, + "pass@64": 0.6441483178327181, + "pass@128": 0.7023809523809523 + }, + "20": { + "pass@1": 0.1627994011976048, + "pass@2": 0.23435805554245845, + "pass@4": 0.3117855043568655, + "pass@8": 0.3856789000143374, + "pass@16": 0.45113612331383535, + "pass@32": 0.5083044335567928, + "pass@64": 0.5594660767363314, + "pass@128": 0.6077844311377245 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 609965, + "answer_accuracy": 0.7613598917558716, + "avg_response_len": 295.48445613316824, + "resp_tokens_sum": 236727963, + "pass_at_k": { + "pass@1": 0.7613598917558716, + "pass@2": 0.8008988709801111, + "pass@4": 0.8342214851858113, + "pass@8": 0.8617176925369859, + "pass@16": 0.8835651389423725, + "pass@32": 0.9006750541396102, + "pass@64": 0.9141523679192285, + "pass@128": 0.9247483623582042 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9647621268656716, + "pass@2": 0.9734391526618872, + "pass@4": 0.9800294167520107, + "pass@8": 0.9854688761602408, + "pass@16": 0.98960424540622, + "pass@32": 0.9928749151450584, + "pass@64": 0.9957231825237381, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956341911764706, + "pass@2": 0.9985670449282076, + "pass@4": 0.999778557092128, + "pass@8": 0.9999916503991577, + "pass@16": 0.9999999884463692, + "pass@32": 0.9999999999999981, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990388406940063, + "pass@2": 0.9997124090265532, + "pass@4": 0.9999756780205314, + "pass@8": 0.9999998642506372, + "pass@16": 0.9999999999987255, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999505537974683, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9910848348348348, + "pass@2": 0.993672190891679, + "pass@4": 0.9957048705060514, + "pass@8": 0.9975737767091117, + "pass@16": 0.9990568254400874, + "pass@32": 0.9998429950913355, + "pass@64": 0.999997993595807, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954280546623794, + "pass@2": 0.998397030154189, + "pass@4": 0.9996241085820865, + "pass@8": 0.9999720666894278, + "pass@16": 0.9999998580882329, + "pass@32": 0.9999999999993795, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9530009920634921, + "pass@2": 0.9655164198225222, + "pass@4": 0.9752756438183323, + "pass@8": 0.9830693452777767, + "pass@16": 0.9882687544305585, + "pass@32": 0.9907855590050356, + "pass@64": 0.9920425485829253, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.919509747706422, + "pass@2": 0.9402130890462086, + "pass@4": 0.9539531853243111, + "pass@8": 0.9647055166695085, + "pass@16": 0.9726198083250552, + "pass@32": 0.978469214045886, + "pass@64": 0.9822490825734729, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7709056712962963, + "pass@2": 0.8243744380042777, + "pass@4": 0.8576229371791491, + "pass@8": 0.8812835952537411, + "pass@16": 0.8998544838496414, + "pass@32": 0.9164081924706067, + "pass@64": 0.9321818096766924, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.7547292284866469, + "pass@2": 0.8264795906446407, + "pass@4": 0.8691673498156945, + "pass@8": 0.8938678818285647, + "pass@16": 0.9084582283431383, + "pass@32": 0.9182333656049506, + "pass@64": 0.9255198822795507, + "pass@128": 0.9317507418397626 + }, + "15": { + "pass@1": 0.6353443287037037, + "pass@2": 0.7256257138864582, + "pass@4": 0.7894987327972884, + "pass@8": 0.8352220459229024, + "pass@16": 0.8660010645617393, + "pass@32": 0.8862625280836279, + "pass@64": 0.9011435190103152, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.4712619837758112, + "pass@2": 0.5903221755742922, + "pass@4": 0.6857823397075362, + "pass@8": 0.758766012039169, + "pass@16": 0.8135182061380419, + "pass@32": 0.8522253204431165, + "pass@64": 0.8794103665319446, + "pass@128": 0.8967551622418879 + }, + "17": { + "pass@1": 0.3424086972891566, + "pass@2": 0.4520581094061286, + "pass@4": 0.5513477831460822, + "pass@8": 0.6347936064414588, + "pass@16": 0.7021058029667845, + "pass@32": 0.7548064429594218, + "pass@64": 0.7942069424208933, + "pass@128": 0.822289156626506 + }, + "18": { + "pass@1": 0.30100806451612905, + "pass@2": 0.39177316167132326, + "pass@4": 0.4753106262926812, + "pass@8": 0.548886279168791, + "pass@16": 0.6104975949194693, + "pass@32": 0.6622092130723058, + "pass@64": 0.7079157199338997, + "pass@128": 0.7483870967741936 + }, + "19": { + "pass@1": 0.18754720543806647, + "pass@2": 0.27558571912838686, + "pass@4": 0.3708002202141651, + "pass@8": 0.4603786452828474, + "pass@16": 0.5360583838054499, + "pass@32": 0.5979704428616274, + "pass@64": 0.6491921888292985, + "pass@128": 0.6918429003021148 + }, + "20": { + "pass@1": 0.17023273273273273, + "pass@2": 0.24842608356387852, + "pass@4": 0.3344517135245482, + "pass@8": 0.4178930342520521, + "pass@16": 0.4919536759326808, + "pass@32": 0.5540069454672477, + "pass@64": 0.6017817007373545, + "pass@128": 0.6366366366366366 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep4-block8/composition/autoregressive/checkpoint-86183_metrics.json b/SDAR-100M-ep4-block8/composition/autoregressive/checkpoint-86183_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f0a00e14f9173267fb5a7ca3a7dfd567a4f866eb --- /dev/null +++ b/SDAR-100M-ep4-block8/composition/autoregressive/checkpoint-86183_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-86183", + "id": { + "answer_accuracy": 0.9919913194444444, + "avg_response_len": 199.9115217013889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9919913194444444, + "pass@2": 0.9943843503936993, + "pass@4": 0.9957170507332416, + "pass@8": 0.9965522306033602, + "pass@16": 0.9972218809753969, + "pass@32": 0.9978707243501875, + "pass@64": 0.9984907174262098, + "pass@128": 0.9991111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.943203125, + "pass@2": 0.9570258366141737, + "pass@4": 0.9660319163854519, + "pass@8": 0.9719589205539898, + "pass@16": 0.9765988960903255, + "pass@32": 0.9813721614164168, + "pass@64": 0.9864823317329837, + "pass@128": 0.992 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999765625, + "pass@2": 0.9999996309055118, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99978125, + "pass@2": 0.9999929872047245, + "pass@4": 0.9999999867829021, + "pass@8": 0.9999999999999938, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.998640625, + "pass@2": 0.9995049212598426, + "pass@4": 0.9998769653168353, + "pass@8": 0.9999881560066616, + "pass@16": 0.9999999084014511, + "pass@32": 0.999999999998944, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.999203125, + "pass@2": 0.999855561023622, + "pass@4": 0.9999846433258343, + "pass@8": 0.9999998174912025, + "pass@16": 0.9999999999893779, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9948125, + "pass@2": 0.9979717027559054, + "pass@4": 0.9990769505061868, + "pass@8": 0.9994078210140124, + "pass@16": 0.9996674141841217, + "pass@32": 0.9999072527311053, + "pass@64": 0.9999969041357232, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9923046875, + "pass@2": 0.9951085137795277, + "pass@4": 0.9964829942819646, + "pass@8": 0.9976153603643774, + "pass@16": 0.998730710113297, + "pass@32": 0.9995571050052217, + "pass@64": 0.999937220967182, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.943203125, + "2": 1.0, + "3": 1.0, + "4": 0.9999765625, + "5": 0.99978125, + "6": 0.998640625, + "7": 0.999203125, + "8": 0.9948125, + "9": 0.9923046875 + }, + "per_op_avg_response_len": { + "10": 280.4438671875, + "2": 115.64675, + "3": 144.73703125, + "4": 159.913046875, + "5": 172.7726796875, + "6": 198.950328125, + "7": 223.8366015625, + "8": 243.5049921875, + "9": 259.3983984375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.4438671875, + "2": 115.64675, + "3": 144.73703125, + "4": 159.913046875, + "5": 172.7726796875, + "6": 198.950328125, + "7": 223.8366015625, + "8": 243.5049921875, + "9": 259.3983984375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 375862, + "answer_accuracy": 0.9916993836541709, + "avg_response_len": 190.44622013255656, + "resp_tokens_sum": 72180641, + "pass_at_k": { + "pass@1": 0.9916993836541709, + "pass@2": 0.9941074935446902, + "pass@4": 0.9955758542783472, + "pass@8": 0.9964259801578738, + "pass@16": 0.9970286710529218, + "pass@32": 0.9975658381500544, + "pass@64": 0.9980044904084722, + "pass@128": 0.9983113812901047 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9424317846607669, + "pass@2": 0.9554277576475513, + "pass@4": 0.9645232338214357, + "pass@8": 0.9705729284703094, + "pass@16": 0.9750282528248507, + "pass@32": 0.979012371776588, + "pass@64": 0.9825793273267351, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9998511904761904, + "pass@2": 0.9999992188476441, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9982456961077845, + "pass@2": 0.9989778071809138, + "pass@4": 0.9996588796535164, + "pass@8": 0.9999646694808768, + "pass@16": 0.9999997257540574, + "pass@32": 0.9999999999968382, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997457470414202, + "pass@2": 0.9999949040208731, + "pass@4": 0.9999999986133391, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9944411057692307, + "pass@2": 0.9981253470119119, + "pass@4": 0.9996946421360792, + "pass@8": 0.9999886513123039, + "pass@16": 0.9999999830961206, + "pass@32": 0.999999999999996, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9926496478873239, + "pass@2": 0.996029721636908, + "pass@4": 0.9975660128751512, + "pass@8": 0.9983336498027551, + "pass@16": 0.999063138456991, + "pass@32": 0.9997387400876202, + "pass@64": 0.9999912792555583, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387425, + "answer_accuracy": 0.9917292963630406, + "avg_response_len": 197.95808332650722, + "resp_tokens_sum": 77333513, + "pass_at_k": { + "pass@1": 0.9917292963630406, + "pass@2": 0.9939390863484381, + "pass@4": 0.9952053170000211, + "pass@8": 0.996057012438397, + "pass@16": 0.9967853721291523, + "pass@32": 0.9975510326815084, + "pass@64": 0.9983849740574036, + "pass@128": 0.9993446920052425 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9370446702453987, + "pass@2": 0.9524521007439255, + "pass@4": 0.9622697201193403, + "pass@8": 0.9685217512597472, + "pass@16": 0.9732031072775292, + "pass@32": 0.9783710405726299, + "pass@64": 0.985072569920195, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999094202898551, + "pass@2": 0.9999989301609038, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994179799426934, + "pass@2": 0.9999027028856351, + "pass@4": 0.9999971459513121, + "pass@8": 0.9999999985260104, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998419436416185, + "pass@2": 0.9999996444176414, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954225352112676, + "pass@2": 0.9971879505378729, + "pass@4": 0.9978147124215107, + "pass@8": 0.9983444096217255, + "pass@16": 0.9990631538362801, + "pass@32": 0.9997387400876238, + "pass@64": 0.9999912792555583, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9916729041916168, + "pass@2": 0.9941191681597434, + "pass@4": 0.9953397361632191, + "pass@8": 0.9964540303892805, + "pass@16": 0.9977764166281999, + "pass@32": 0.9990105981622154, + "pass@64": 0.99982149369308, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379487, + "answer_accuracy": 0.9925484390693003, + "avg_response_len": 211.2903807122531, + "resp_tokens_sum": 80783919, + "pass_at_k": { + "pass@1": 0.9925484390693003, + "pass@2": 0.9951137507941235, + "pass@4": 0.9963798877123959, + "pass@8": 0.9971833767059872, + "pass@16": 0.9978594168238033, + "pass@32": 0.998499605170878, + "pass@64": 0.9990807565829283, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9499766791044776, + "pass@2": 0.963093856504877, + "pass@4": 0.9712197354062085, + "pass@8": 0.9767062892294843, + "pass@16": 0.9814928519708268, + "pass@32": 0.9866806213656, + "pass@64": 0.9918038267917523, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995863970588236, + "pass@2": 0.9999811834182493, + "pass@4": 0.9999999611261827, + "pass@8": 0.9999999999999815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9982009069400631, + "pass@2": 0.9996223670483617, + "pass@4": 0.999974433992281, + "pass@8": 0.999999864083505, + "pass@16": 0.9999999999987256, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9979232594936709, + "pass@2": 0.9995487547343765, + "pass@4": 0.9999514044130559, + "pass@8": 0.9999994224405141, + "pass@16": 0.9999999999663859, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9945101351351351, + "pass@2": 0.9986632794211536, + "pass@4": 0.9998438415918729, + "pass@8": 0.9999972864530364, + "pass@16": 0.9999999995382962, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925894292604501, + "pass@2": 0.9951194869230575, + "pass@4": 0.9964745589478163, + "pass@8": 0.998042664097683, + "pass@16": 0.9993761183512748, + "pass@32": 0.9999366961026899, + "pass@64": 0.9999998004436335, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.4688859375, + "avg_response_len": 336.13270703125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.4688859375, + "pass@2": 0.5431404773622015, + "pass@4": 0.6047769180914772, + "pass@8": 0.6552859413249879, + "pass@16": 0.6965352992999928, + "pass@32": 0.7306665448666592, + "pass@64": 0.7602630764121265, + "pass@128": 0.7872 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9265625, + "pass@2": 0.9490615157480323, + "pass@4": 0.9621242044431944, + "pass@8": 0.9706536825186178, + "pass@16": 0.9764014129097319, + "pass@32": 0.9805757997942465, + "pass@64": 0.9837661955117719, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.831796875, + "pass@2": 0.8833303395669297, + "pass@4": 0.9081133770153732, + "pass@8": 0.9225558972649252, + "pass@16": 0.9325012437351446, + "pass@32": 0.9399688992094204, + "pass@64": 0.9466652116462047, + "pass@128": 0.953 + }, + "13": { + "pass@1": 0.6628203125, + "pass@2": 0.7440194389763795, + "pass@4": 0.7990956918822645, + "pass@8": 0.8372189758495833, + "pass@16": 0.8651134991100339, + "pass@32": 0.8869837329021945, + "pass@64": 0.9036187855482851, + "pass@128": 0.915 + }, + "14": { + "pass@1": 0.6339609375, + "pass@2": 0.7229564468503937, + "pass@4": 0.784271693100862, + "pass@8": 0.8255480480457074, + "pass@16": 0.8544655156226874, + "pass@32": 0.8773881477144116, + "pass@64": 0.8975452164937121, + "pass@128": 0.915 + }, + "15": { + "pass@1": 0.5383671875, + "pass@2": 0.6481603100393698, + "pass@4": 0.7320636483877012, + "pass@8": 0.7919102912159756, + "pass@16": 0.8331286840231188, + "pass@32": 0.8623839139049914, + "pass@64": 0.8848044315661904, + "pass@128": 0.903 + }, + "16": { + "pass@1": 0.297484375, + "pass@2": 0.38550492125984287, + "pass@4": 0.4632323821709782, + "pass@8": 0.529759855488038, + "pass@16": 0.587033251455353, + "pass@32": 0.6372908748376629, + "pass@64": 0.6852063498779605, + "pass@128": 0.733 + }, + "17": { + "pass@1": 0.25428125, + "pass@2": 0.34081360728346466, + "pass@4": 0.4238608893888262, + "pass@8": 0.4981086284140166, + "pass@16": 0.5618750652808251, + "pass@32": 0.6151331140023338, + "pass@64": 0.6610466759237624, + "pass@128": 0.707 + }, + "18": { + "pass@1": 0.2114375, + "pass@2": 0.2884830216535432, + "pass@4": 0.3670773106486688, + "pass@8": 0.44195859262711745, + "pass@16": 0.509527145959014, + "pass@32": 0.566575110484869, + "pass@64": 0.613543926642942, + "pass@128": 0.656 + }, + "19": { + "pass@1": 0.1746953125, + "pass@2": 0.24318331692913395, + "pass@4": 0.310761648293963, + "pass@8": 0.3713671039645726, + "pass@16": 0.424071297515324, + "pass@32": 0.47117190176072044, + "pass@64": 0.5152217284234921, + "pass@128": 0.555 + }, + "20": { + "pass@1": 0.157453125, + "pass@2": 0.22589185531496073, + "pass@4": 0.29716833558305206, + "pass@8": 0.36377833786132596, + "pass@16": 0.42123587738855095, + "pass@32": 0.4691939540557233, + "pass@64": 0.5112122424868704, + "pass@128": 0.549 + } + }, + "per_op_accuracy": { + "11": 0.9265625, + "12": 0.831796875, + "13": 0.6628203125, + "14": 0.6339609375, + "15": 0.5383671875, + "16": 0.297484375, + "17": 0.25428125, + "18": 0.2114375, + "19": 0.1746953125, + "20": 0.157453125 + }, + "per_op_avg_response_len": { + "11": 297.794328125, + "12": 305.443828125, + "13": 319.6149296875, + "14": 344.0471875, + "15": 355.1386875, + "16": 347.4819609375, + "17": 352.097546875, + "18": 353.871234375, + "19": 339.7157421875, + "20": 346.121625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.794328125, + "12": 305.443828125, + "13": 319.6149296875, + "14": 344.0471875, + "15": 355.1386875, + "16": 347.4819609375, + "17": 352.097546875, + "18": 353.871234375, + "19": 339.7157421875, + "20": 346.121625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 204603, + "answer_accuracy": 0.47502553863298663, + "avg_response_len": 319.6210507986627, + "resp_tokens_sum": 137667179, + "pass_at_k": { + "pass@1": 0.47502553863298663, + "pass@2": 0.5470776272068906, + "pass@4": 0.6074450415373427, + "pass@8": 0.6572696490077963, + "pass@16": 0.6981663136936618, + "pass@32": 0.731609692334189, + "pass@64": 0.759917735678695, + "pass@128": 0.7869242199108469 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9241988314447592, + "pass@2": 0.9451147085721935, + "pass@4": 0.9583057509100311, + "pass@8": 0.9683471315113082, + "pass@16": 0.9758373276009441, + "pass@32": 0.9813627292403387, + "pass@64": 0.9854111072441946, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.832229872881356, + "pass@2": 0.8820969015525602, + "pass@4": 0.9064526634912154, + "pass@8": 0.9210831491573624, + "pass@16": 0.9322645619669031, + "pass@32": 0.9417992939933498, + "pass@64": 0.9503938639071714, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.6866918103448276, + "pass@2": 0.7621581344259865, + "pass@4": 0.8154033129283599, + "pass@8": 0.8542870664981383, + "pass@16": 0.8825616429284143, + "pass@32": 0.9031559569013347, + "pass@64": 0.9156100331712211, + "pass@128": 0.9184952978056427 + }, + "14": { + "pass@1": 0.6419635903426791, + "pass@2": 0.724484570853877, + "pass@4": 0.77979786434406, + "pass@8": 0.8168612902209831, + "pass@16": 0.8439180312348161, + "pass@32": 0.8659116440785112, + "pass@64": 0.8836119795991179, + "pass@128": 0.897196261682243 + }, + "15": { + "pass@1": 0.5370077673410405, + "pass@2": 0.6450683571526107, + "pass@4": 0.7338411369099088, + "pass@8": 0.7986025998546395, + "pass@16": 0.8436119684412422, + "pass@32": 0.8759949765481339, + "pass@64": 0.8985630011957301, + "pass@128": 0.9161849710982659 + }, + "16": { + "pass@1": 0.2983618951612903, + "pass@2": 0.3907174720599443, + "pass@4": 0.4704105074567288, + "pass@8": 0.535630716107639, + "pass@16": 0.5907458646622782, + "pass@32": 0.6372065651541644, + "pass@64": 0.6795116518435073, + "pass@128": 0.7225806451612903 + }, + "17": { + "pass@1": 0.2517837789017341, + "pass@2": 0.3336255035046197, + "pass@4": 0.41323942626752563, + "pass@8": 0.48810200863925596, + "pass@16": 0.5547621412716494, + "pass@32": 0.6106711542212494, + "pass@64": 0.6590666679799665, + "pass@128": 0.708092485549133 + }, + "18": { + "pass@1": 0.19428571428571428, + "pass@2": 0.2676690804274464, + "pass@4": 0.34212716508650715, + "pass@8": 0.4139243368536438, + "pass@16": 0.4805783850514051, + "pass@32": 0.5388344572164389, + "pass@64": 0.5889168734491658, + "pass@128": 0.6371428571428571 + }, + "19": { + "pass@1": 0.18327702702702703, + "pass@2": 0.2533107664751364, + "pass@4": 0.3211475995117731, + "pass@8": 0.3820281677250221, + "pass@16": 0.4348139659769961, + "pass@32": 0.4792898587653093, + "pass@64": 0.5180948362733667, + "pass@128": 0.5585585585585585 + }, + "20": { + "pass@1": 0.17429147897897898, + "pass@2": 0.2449047226360218, + "pass@4": 0.3154769668431085, + "pass@8": 0.3776520541403074, + "pass@16": 0.4277429922970302, + "pass@32": 0.46807247378301114, + "pass@64": 0.5071460121712338, + "pass@128": 0.5525525525525525 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 199602, + "answer_accuracy": 0.46369034344335414, + "avg_response_len": 331.5233167001189, + "resp_tokens_sum": 142708853, + "pass_at_k": { + "pass@1": 0.46369034344335414, + "pass@2": 0.5385155531127309, + "pass@4": 0.6001277824772377, + "pass@8": 0.6511879310832249, + "pass@16": 0.6938440441234837, + "pass@32": 0.7291873344897845, + "pass@64": 0.7592672184182168, + "pass@128": 0.7859054415700267 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.932652484939759, + "pass@2": 0.9551909656341906, + "pass@4": 0.967504908272008, + "pass@8": 0.9743332008558964, + "pass@16": 0.9776944087377336, + "pass@32": 0.9797107289052188, + "pass@64": 0.981380697080861, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8275127351097179, + "pass@2": 0.8741306802754675, + "pass@4": 0.8940140824873376, + "pass@8": 0.9059417569605126, + "pass@16": 0.9149301720126192, + "pass@32": 0.9210451859792664, + "pass@64": 0.9267178933506086, + "pass@128": 0.9341692789968652 + }, + "13": { + "pass@1": 0.6466211484593838, + "pass@2": 0.7352568980348043, + "pass@4": 0.7941474946934152, + "pass@8": 0.8322874017240125, + "pass@16": 0.8593893686275413, + "pass@32": 0.8812230108196569, + "pass@64": 0.8990334067498752, + "pass@128": 0.9131652661064426 + }, + "14": { + "pass@1": 0.6178956505847953, + "pass@2": 0.7086092548464337, + "pass@4": 0.7747260120445467, + "pass@8": 0.8222148893486398, + "pass@16": 0.856872436091234, + "pass@32": 0.8836770532613494, + "pass@64": 0.9065420227864525, + "pass@128": 0.9269005847953217 + }, + "15": { + "pass@1": 0.5457149621212121, + "pass@2": 0.661665995585779, + "pass@4": 0.7447602410494136, + "pass@8": 0.8003608834636073, + "pass@16": 0.837934698476195, + "pass@32": 0.8649150245288428, + "pass@64": 0.8867527597855025, + "pass@128": 0.906060606060606 + }, + "16": { + "pass@1": 0.3033297720797721, + "pass@2": 0.38540860477376226, + "pass@4": 0.4561851985382169, + "pass@8": 0.5197053256135613, + "pass@16": 0.5790541044663001, + "pass@32": 0.6333955577098603, + "pass@64": 0.6845196910184368, + "pass@128": 0.7321937321937322 + }, + "17": { + "pass@1": 0.2654794254658385, + "pass@2": 0.3596355822370031, + "pass@4": 0.4481225438823254, + "pass@8": 0.5224691510802295, + "pass@16": 0.580719897610317, + "pass@32": 0.6246834181217727, + "pass@64": 0.6622791915081849, + "pass@128": 0.7080745341614907 + }, + "18": { + "pass@1": 0.20091911764705883, + "pass@2": 0.28180508916164887, + "pass@4": 0.3687975969915525, + "pass@8": 0.4557692555372953, + "pass@16": 0.5351237386501738, + "pass@32": 0.6004961265979195, + "pass@64": 0.6507879587120721, + "pass@128": 0.6911764705882353 + }, + "19": { + "pass@1": 0.16022600446428573, + "pass@2": 0.22008833368485176, + "pass@4": 0.2805208366811292, + "pass@8": 0.3395545181912359, + "pass@16": 0.3959618657365702, + "pass@32": 0.4475826196030161, + "pass@64": 0.49460998571116155, + "pass@128": 0.5327380952380952 + }, + "20": { + "pass@1": 0.14796968562874252, + "pass@2": 0.2126687815314254, + "pass@4": 0.2800213515600969, + "pass@8": 0.3452822815254244, + "pass@16": 0.40522265822512654, + "pass@32": 0.4578358076989143, + "pass@64": 0.5011268713315313, + "pass@128": 0.5329341317365269 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 195969, + "answer_accuracy": 0.4679119231356968, + "avg_response_len": 357.8512592642115, + "resp_tokens_sum": 149873833, + "pass_at_k": { + "pass@1": 0.4679119231356968, + "pass@2": 0.5438449733963205, + "pass@4": 0.6068113947649143, + "pass@8": 0.6574578337731608, + "pass@16": 0.6976240302669221, + "pass@32": 0.7312169401200675, + "pass@64": 0.7616417872927552, + "pass@128": 0.7888141809290954 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9227926587301587, + "pass@2": 0.9470242000999873, + "pass@4": 0.960732205795704, + "pass@8": 0.9693603886062503, + "pass@16": 0.9756707700497491, + "pass@32": 0.9806056964313475, + "pass@64": 0.9844370895991283, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8355074541284404, + "pass@2": 0.8936402122613109, + "pass@4": 0.9236655713333995, + "pass@8": 0.9403579265835318, + "pass@16": 0.9498986664429232, + "pass@32": 0.9564481186800877, + "pass@64": 0.9620879995236127, + "pass@128": 0.9663608562691132 + }, + "13": { + "pass@1": 0.6571662808641975, + "pass@2": 0.7358156836298242, + "pass@4": 0.7884919118906429, + "pass@8": 0.8258481457444613, + "pass@16": 0.854241760542711, + "pass@32": 0.8774085425557125, + "pass@64": 0.8968649961634577, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.6426418768545994, + "pass@2": 0.7360609330825484, + "pass@4": 0.7982204229293296, + "pass@8": 0.8372049902894257, + "pass@16": 0.8620695977837106, + "pass@32": 0.8819375600885098, + "pass@64": 0.901686644657056, + "pass@128": 0.9198813056379822 + }, + "15": { + "pass@1": 0.5323350694444444, + "pass@2": 0.6377064195100618, + "pass@4": 0.7172337514523645, + "pass@8": 0.7761564818619724, + "pass@16": 0.8170385569299585, + "pass@32": 0.8452706602617858, + "pass@64": 0.8681272297013932, + "pass@128": 0.8858024691358025 + }, + "16": { + "pass@1": 0.2906296091445428, + "pass@2": 0.3808380078043341, + "pass@4": 0.4639649562609984, + "pass@8": 0.5348016643194988, + "pass@16": 0.5918998310984556, + "pass@32": 0.6414011766481142, + "pass@64": 0.6911248562212414, + "pass@128": 0.7433628318584071 + }, + "17": { + "pass@1": 0.24602315512048192, + "pass@2": 0.33004977587515394, + "pass@4": 0.41139936376552333, + "pass@8": 0.48491044209939826, + "pass@16": 0.5510107149708241, + "pass@32": 0.6105205843571422, + "pass@64": 0.6619147865574031, + "pass@128": 0.7048192771084337 + }, + "18": { + "pass@1": 0.24233870967741936, + "pass@2": 0.31930681673863354, + "pass@4": 0.3933600641653665, + "pass@8": 0.4584629930505219, + "pass@16": 0.5141375485482685, + "pass@32": 0.5606915077929782, + "pass@64": 0.6005003708826756, + "pass@128": 0.6387096774193548 + }, + "19": { + "pass@1": 0.18074962235649547, + "pass@2": 0.2564385243951755, + "pass@4": 0.33101056353982955, + "pass@8": 0.39293476132895816, + "pass@16": 0.4417977642220465, + "pass@32": 0.486950509381447, + "pass@64": 0.5332543587477647, + "pass@128": 0.5740181268882175 + }, + "20": { + "pass@1": 0.1501266891891892, + "pass@2": 0.22014177070771548, + "pass@4": 0.29605818079046403, + "pass@8": 0.36845622163096464, + "pass@16": 0.4307900693887324, + "pass@32": 0.4817076892328625, + "pass@64": 0.5253941303577103, + "pass@128": 0.5615615615615616 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7166726973684211, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7166726973684211, + "pass@2": 0.7568875751139458, + "pass@4": 0.7899590861849313, + "pass@8": 0.8169383941410714, + "pass@16": 0.8389657853567667, + "pass@32": 0.8572369456746595, + "pass@64": 0.873107748471427, + "pass@128": 0.887578947368421 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.943203125, + "pass@2": 0.9570258366141737, + "pass@4": 0.9660319163854519, + "pass@8": 0.9719589205539898, + "pass@16": 0.9765988960903255, + "pass@32": 0.9813721614164168, + "pass@64": 0.9864823317329837, + "pass@128": 0.992 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999765625, + "pass@2": 0.9999996309055118, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99978125, + "pass@2": 0.9999929872047245, + "pass@4": 0.9999999867829021, + "pass@8": 0.9999999999999938, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.998640625, + "pass@2": 0.9995049212598426, + "pass@4": 0.9998769653168353, + "pass@8": 0.9999881560066616, + "pass@16": 0.9999999084014511, + "pass@32": 0.999999999998944, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.999203125, + "pass@2": 0.999855561023622, + "pass@4": 0.9999846433258343, + "pass@8": 0.9999998174912025, + "pass@16": 0.9999999999893779, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9948125, + "pass@2": 0.9979717027559054, + "pass@4": 0.9990769505061868, + "pass@8": 0.9994078210140124, + "pass@16": 0.9996674141841217, + "pass@32": 0.9999072527311053, + "pass@64": 0.9999969041357232, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9923046875, + "pass@2": 0.9951085137795277, + "pass@4": 0.9964829942819646, + "pass@8": 0.9976153603643774, + "pass@16": 0.998730710113297, + "pass@32": 0.9995571050052217, + "pass@64": 0.999937220967182, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9265625, + "pass@2": 0.9490615157480323, + "pass@4": 0.9621242044431944, + "pass@8": 0.9706536825186178, + "pass@16": 0.9764014129097319, + "pass@32": 0.9805757997942465, + "pass@64": 0.9837661955117719, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.831796875, + "pass@2": 0.8833303395669297, + "pass@4": 0.9081133770153732, + "pass@8": 0.9225558972649252, + "pass@16": 0.9325012437351446, + "pass@32": 0.9399688992094204, + "pass@64": 0.9466652116462047, + "pass@128": 0.953 + }, + "13": { + "pass@1": 0.6628203125, + "pass@2": 0.7440194389763795, + "pass@4": 0.7990956918822645, + "pass@8": 0.8372189758495833, + "pass@16": 0.8651134991100339, + "pass@32": 0.8869837329021945, + "pass@64": 0.9036187855482851, + "pass@128": 0.915 + }, + "14": { + "pass@1": 0.6339609375, + "pass@2": 0.7229564468503937, + "pass@4": 0.784271693100862, + "pass@8": 0.8255480480457074, + "pass@16": 0.8544655156226874, + "pass@32": 0.8773881477144116, + "pass@64": 0.8975452164937121, + "pass@128": 0.915 + }, + "15": { + "pass@1": 0.5383671875, + "pass@2": 0.6481603100393698, + "pass@4": 0.7320636483877012, + "pass@8": 0.7919102912159756, + "pass@16": 0.8331286840231188, + "pass@32": 0.8623839139049914, + "pass@64": 0.8848044315661904, + "pass@128": 0.903 + }, + "16": { + "pass@1": 0.297484375, + "pass@2": 0.38550492125984287, + "pass@4": 0.4632323821709782, + "pass@8": 0.529759855488038, + "pass@16": 0.587033251455353, + "pass@32": 0.6372908748376629, + "pass@64": 0.6852063498779605, + "pass@128": 0.733 + }, + "17": { + "pass@1": 0.25428125, + "pass@2": 0.34081360728346466, + "pass@4": 0.4238608893888262, + "pass@8": 0.4981086284140166, + "pass@16": 0.5618750652808251, + "pass@32": 0.6151331140023338, + "pass@64": 0.6610466759237624, + "pass@128": 0.707 + }, + "18": { + "pass@1": 0.2114375, + "pass@2": 0.2884830216535432, + "pass@4": 0.3670773106486688, + "pass@8": 0.44195859262711745, + "pass@16": 0.509527145959014, + "pass@32": 0.566575110484869, + "pass@64": 0.613543926642942, + "pass@128": 0.656 + }, + "19": { + "pass@1": 0.1746953125, + "pass@2": 0.24318331692913395, + "pass@4": 0.310761648293963, + "pass@8": 0.3713671039645726, + "pass@16": 0.424071297515324, + "pass@32": 0.47117190176072044, + "pass@64": 0.5152217284234921, + "pass@128": 0.555 + }, + "20": { + "pass@1": 0.157453125, + "pass@2": 0.22589185531496073, + "pass@4": 0.29716833558305206, + "pass@8": 0.36377833786132596, + "pass@16": 0.42123587738855095, + "pass@32": 0.4691939540557233, + "pass@64": 0.5112122424868704, + "pass@128": 0.549 + } + }, + "per_op_accuracy": { + "10": 0.943203125, + "2": 1.0, + "3": 1.0, + "4": 0.9999765625, + "5": 0.99978125, + "6": 0.998640625, + "7": 0.999203125, + "8": 0.9948125, + "9": 0.9923046875, + "11": 0.9265625, + "12": 0.831796875, + "13": 0.6628203125, + "14": 0.6339609375, + "15": 0.5383671875, + "16": 0.297484375, + "17": 0.25428125, + "18": 0.2114375, + "19": 0.1746953125, + "20": 0.157453125 + }, + "per_op_avg_response_len": { + "10": 280.4438671875, + "2": 115.64675, + "3": 144.73703125, + "4": 159.913046875, + "5": 172.7726796875, + "6": 198.950328125, + "7": 223.8366015625, + "8": 243.5049921875, + "9": 259.3983984375, + "11": 297.794328125, + "12": 305.443828125, + "13": 319.6149296875, + "14": 344.0471875, + "15": 355.1386875, + "16": 347.4819609375, + "17": 352.097546875, + "18": 353.871234375, + "19": 339.7157421875, + "20": 346.121625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.4438671875, + "2": 115.64675, + "3": 144.73703125, + "4": 159.913046875, + "5": 172.7726796875, + "6": 198.950328125, + "7": 223.8366015625, + "8": 243.5049921875, + "9": 259.3983984375, + "11": 297.794328125, + "12": 305.443828125, + "13": 319.6149296875, + "14": 344.0471875, + "15": 355.1386875, + "16": 347.4819609375, + "17": 352.097546875, + "18": 353.871234375, + "19": 339.7157421875, + "20": 346.121625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 580465, + "answer_accuracy": 0.7168641815523238, + "avg_response_len": 259.1584087496048, + "resp_tokens_sum": 209847820, + "pass_at_k": { + "pass@1": 0.7168641815523238, + "pass@2": 0.7563181321430583, + "pass@4": 0.7891167671974877, + "pass@8": 0.8160179728357079, + "pass@16": 0.8380543061281882, + "pass@32": 0.8560953306144278, + "pass@64": 0.8713585957411207, + "pass@128": 0.8858678469807145 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9424317846607669, + "pass@2": 0.9554277576475513, + "pass@4": 0.9645232338214357, + "pass@8": 0.9705729284703094, + "pass@16": 0.9750282528248507, + "pass@32": 0.979012371776588, + "pass@64": 0.9825793273267351, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9998511904761904, + "pass@2": 0.9999992188476441, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9982456961077845, + "pass@2": 0.9989778071809138, + "pass@4": 0.9996588796535164, + "pass@8": 0.9999646694808768, + "pass@16": 0.9999997257540574, + "pass@32": 0.9999999999968382, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997457470414202, + "pass@2": 0.9999949040208731, + "pass@4": 0.9999999986133391, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9944411057692307, + "pass@2": 0.9981253470119119, + "pass@4": 0.9996946421360792, + "pass@8": 0.9999886513123039, + "pass@16": 0.9999999830961206, + "pass@32": 0.999999999999996, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9926496478873239, + "pass@2": 0.996029721636908, + "pass@4": 0.9975660128751512, + "pass@8": 0.9983336498027551, + "pass@16": 0.999063138456991, + "pass@32": 0.9997387400876202, + "pass@64": 0.9999912792555583, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9241988314447592, + "pass@2": 0.9451147085721935, + "pass@4": 0.9583057509100311, + "pass@8": 0.9683471315113082, + "pass@16": 0.9758373276009441, + "pass@32": 0.9813627292403387, + "pass@64": 0.9854111072441946, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.832229872881356, + "pass@2": 0.8820969015525602, + "pass@4": 0.9064526634912154, + "pass@8": 0.9210831491573624, + "pass@16": 0.9322645619669031, + "pass@32": 0.9417992939933498, + "pass@64": 0.9503938639071714, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.6866918103448276, + "pass@2": 0.7621581344259865, + "pass@4": 0.8154033129283599, + "pass@8": 0.8542870664981383, + "pass@16": 0.8825616429284143, + "pass@32": 0.9031559569013347, + "pass@64": 0.9156100331712211, + "pass@128": 0.9184952978056427 + }, + "14": { + "pass@1": 0.6419635903426791, + "pass@2": 0.724484570853877, + "pass@4": 0.77979786434406, + "pass@8": 0.8168612902209831, + "pass@16": 0.8439180312348161, + "pass@32": 0.8659116440785112, + "pass@64": 0.8836119795991179, + "pass@128": 0.897196261682243 + }, + "15": { + "pass@1": 0.5370077673410405, + "pass@2": 0.6450683571526107, + "pass@4": 0.7338411369099088, + "pass@8": 0.7986025998546395, + "pass@16": 0.8436119684412422, + "pass@32": 0.8759949765481339, + "pass@64": 0.8985630011957301, + "pass@128": 0.9161849710982659 + }, + "16": { + "pass@1": 0.2983618951612903, + "pass@2": 0.3907174720599443, + "pass@4": 0.4704105074567288, + "pass@8": 0.535630716107639, + "pass@16": 0.5907458646622782, + "pass@32": 0.6372065651541644, + "pass@64": 0.6795116518435073, + "pass@128": 0.7225806451612903 + }, + "17": { + "pass@1": 0.2517837789017341, + "pass@2": 0.3336255035046197, + "pass@4": 0.41323942626752563, + "pass@8": 0.48810200863925596, + "pass@16": 0.5547621412716494, + "pass@32": 0.6106711542212494, + "pass@64": 0.6590666679799665, + "pass@128": 0.708092485549133 + }, + "18": { + "pass@1": 0.19428571428571428, + "pass@2": 0.2676690804274464, + "pass@4": 0.34212716508650715, + "pass@8": 0.4139243368536438, + "pass@16": 0.4805783850514051, + "pass@32": 0.5388344572164389, + "pass@64": 0.5889168734491658, + "pass@128": 0.6371428571428571 + }, + "19": { + "pass@1": 0.18327702702702703, + "pass@2": 0.2533107664751364, + "pass@4": 0.3211475995117731, + "pass@8": 0.3820281677250221, + "pass@16": 0.4348139659769961, + "pass@32": 0.4792898587653093, + "pass@64": 0.5180948362733667, + "pass@128": 0.5585585585585585 + }, + "20": { + "pass@1": 0.17429147897897898, + "pass@2": 0.2449047226360218, + "pass@4": 0.3154769668431085, + "pass@8": 0.3776520541403074, + "pass@16": 0.4277429922970302, + "pass@32": 0.46807247378301114, + "pass@64": 0.5071460121712338, + "pass@128": 0.5525525525525525 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 587027, + "answer_accuracy": 0.7149101227591582, + "avg_response_len": 267.97832959859704, + "resp_tokens_sum": 220042366, + "pass_at_k": { + "pass@1": 0.7149101227591582, + "pass@2": 0.7551878248875328, + "pass@4": 0.788089845667179, + "pass@8": 0.8152628237248414, + "pass@16": 0.8379713914459063, + "pass@32": 0.8568640308079691, + "pass@64": 0.8730298669312079, + "pass@128": 0.8874512860483242 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9370446702453987, + "pass@2": 0.9524521007439255, + "pass@4": 0.9622697201193403, + "pass@8": 0.9685217512597472, + "pass@16": 0.9732031072775292, + "pass@32": 0.9783710405726299, + "pass@64": 0.985072569920195, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999094202898551, + "pass@2": 0.9999989301609038, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994179799426934, + "pass@2": 0.9999027028856351, + "pass@4": 0.9999971459513121, + "pass@8": 0.9999999985260104, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998419436416185, + "pass@2": 0.9999996444176414, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954225352112676, + "pass@2": 0.9971879505378729, + "pass@4": 0.9978147124215107, + "pass@8": 0.9983444096217255, + "pass@16": 0.9990631538362801, + "pass@32": 0.9997387400876238, + "pass@64": 0.9999912792555583, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9916729041916168, + "pass@2": 0.9941191681597434, + "pass@4": 0.9953397361632191, + "pass@8": 0.9964540303892805, + "pass@16": 0.9977764166281999, + "pass@32": 0.9990105981622154, + "pass@64": 0.99982149369308, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.932652484939759, + "pass@2": 0.9551909656341906, + "pass@4": 0.967504908272008, + "pass@8": 0.9743332008558964, + "pass@16": 0.9776944087377336, + "pass@32": 0.9797107289052188, + "pass@64": 0.981380697080861, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8275127351097179, + "pass@2": 0.8741306802754675, + "pass@4": 0.8940140824873376, + "pass@8": 0.9059417569605126, + "pass@16": 0.9149301720126192, + "pass@32": 0.9210451859792664, + "pass@64": 0.9267178933506086, + "pass@128": 0.9341692789968652 + }, + "13": { + "pass@1": 0.6466211484593838, + "pass@2": 0.7352568980348043, + "pass@4": 0.7941474946934152, + "pass@8": 0.8322874017240125, + "pass@16": 0.8593893686275413, + "pass@32": 0.8812230108196569, + "pass@64": 0.8990334067498752, + "pass@128": 0.9131652661064426 + }, + "14": { + "pass@1": 0.6178956505847953, + "pass@2": 0.7086092548464337, + "pass@4": 0.7747260120445467, + "pass@8": 0.8222148893486398, + "pass@16": 0.856872436091234, + "pass@32": 0.8836770532613494, + "pass@64": 0.9065420227864525, + "pass@128": 0.9269005847953217 + }, + "15": { + "pass@1": 0.5457149621212121, + "pass@2": 0.661665995585779, + "pass@4": 0.7447602410494136, + "pass@8": 0.8003608834636073, + "pass@16": 0.837934698476195, + "pass@32": 0.8649150245288428, + "pass@64": 0.8867527597855025, + "pass@128": 0.906060606060606 + }, + "16": { + "pass@1": 0.3033297720797721, + "pass@2": 0.38540860477376226, + "pass@4": 0.4561851985382169, + "pass@8": 0.5197053256135613, + "pass@16": 0.5790541044663001, + "pass@32": 0.6333955577098603, + "pass@64": 0.6845196910184368, + "pass@128": 0.7321937321937322 + }, + "17": { + "pass@1": 0.2654794254658385, + "pass@2": 0.3596355822370031, + "pass@4": 0.4481225438823254, + "pass@8": 0.5224691510802295, + "pass@16": 0.580719897610317, + "pass@32": 0.6246834181217727, + "pass@64": 0.6622791915081849, + "pass@128": 0.7080745341614907 + }, + "18": { + "pass@1": 0.20091911764705883, + "pass@2": 0.28180508916164887, + "pass@4": 0.3687975969915525, + "pass@8": 0.4557692555372953, + "pass@16": 0.5351237386501738, + "pass@32": 0.6004961265979195, + "pass@64": 0.6507879587120721, + "pass@128": 0.6911764705882353 + }, + "19": { + "pass@1": 0.16022600446428573, + "pass@2": 0.22008833368485176, + "pass@4": 0.2805208366811292, + "pass@8": 0.3395545181912359, + "pass@16": 0.3959618657365702, + "pass@32": 0.4475826196030161, + "pass@64": 0.49460998571116155, + "pass@128": 0.5327380952380952 + }, + "20": { + "pass@1": 0.14796968562874252, + "pass@2": 0.2126687815314254, + "pass@4": 0.2800213515600969, + "pass@8": 0.3452822815254244, + "pass@16": 0.40522265822512654, + "pass@32": 0.4578358076989143, + "pass@64": 0.5011268713315313, + "pass@128": 0.5329341317365269 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 575456, + "answer_accuracy": 0.7182856686371625, + "avg_response_len": 287.9076030516057, + "resp_tokens_sum": 230657752, + "pass_at_k": { + "pass@1": 0.7182856686371625, + "pass@2": 0.7592052287226031, + "pass@4": 0.7927262515206405, + "pass@8": 0.8195860006912514, + "pass@16": 0.8409061998859412, + "pass@32": 0.8587729906883347, + "pass@64": 0.874955447824755, + "pass@128": 0.8894392075411408 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9499766791044776, + "pass@2": 0.963093856504877, + "pass@4": 0.9712197354062085, + "pass@8": 0.9767062892294843, + "pass@16": 0.9814928519708268, + "pass@32": 0.9866806213656, + "pass@64": 0.9918038267917523, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995863970588236, + "pass@2": 0.9999811834182493, + "pass@4": 0.9999999611261827, + "pass@8": 0.9999999999999815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9982009069400631, + "pass@2": 0.9996223670483617, + "pass@4": 0.999974433992281, + "pass@8": 0.999999864083505, + "pass@16": 0.9999999999987256, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9979232594936709, + "pass@2": 0.9995487547343765, + "pass@4": 0.9999514044130559, + "pass@8": 0.9999994224405141, + "pass@16": 0.9999999999663859, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9945101351351351, + "pass@2": 0.9986632794211536, + "pass@4": 0.9998438415918729, + "pass@8": 0.9999972864530364, + "pass@16": 0.9999999995382962, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925894292604501, + "pass@2": 0.9951194869230575, + "pass@4": 0.9964745589478163, + "pass@8": 0.998042664097683, + "pass@16": 0.9993761183512748, + "pass@32": 0.9999366961026899, + "pass@64": 0.9999998004436335, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9227926587301587, + "pass@2": 0.9470242000999873, + "pass@4": 0.960732205795704, + "pass@8": 0.9693603886062503, + "pass@16": 0.9756707700497491, + "pass@32": 0.9806056964313475, + "pass@64": 0.9844370895991283, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8355074541284404, + "pass@2": 0.8936402122613109, + "pass@4": 0.9236655713333995, + "pass@8": 0.9403579265835318, + "pass@16": 0.9498986664429232, + "pass@32": 0.9564481186800877, + "pass@64": 0.9620879995236127, + "pass@128": 0.9663608562691132 + }, + "13": { + "pass@1": 0.6571662808641975, + "pass@2": 0.7358156836298242, + "pass@4": 0.7884919118906429, + "pass@8": 0.8258481457444613, + "pass@16": 0.854241760542711, + "pass@32": 0.8774085425557125, + "pass@64": 0.8968649961634577, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.6426418768545994, + "pass@2": 0.7360609330825484, + "pass@4": 0.7982204229293296, + "pass@8": 0.8372049902894257, + "pass@16": 0.8620695977837106, + "pass@32": 0.8819375600885098, + "pass@64": 0.901686644657056, + "pass@128": 0.9198813056379822 + }, + "15": { + "pass@1": 0.5323350694444444, + "pass@2": 0.6377064195100618, + "pass@4": 0.7172337514523645, + "pass@8": 0.7761564818619724, + "pass@16": 0.8170385569299585, + "pass@32": 0.8452706602617858, + "pass@64": 0.8681272297013932, + "pass@128": 0.8858024691358025 + }, + "16": { + "pass@1": 0.2906296091445428, + "pass@2": 0.3808380078043341, + "pass@4": 0.4639649562609984, + "pass@8": 0.5348016643194988, + "pass@16": 0.5918998310984556, + "pass@32": 0.6414011766481142, + "pass@64": 0.6911248562212414, + "pass@128": 0.7433628318584071 + }, + "17": { + "pass@1": 0.24602315512048192, + "pass@2": 0.33004977587515394, + "pass@4": 0.41139936376552333, + "pass@8": 0.48491044209939826, + "pass@16": 0.5510107149708241, + "pass@32": 0.6105205843571422, + "pass@64": 0.6619147865574031, + "pass@128": 0.7048192771084337 + }, + "18": { + "pass@1": 0.24233870967741936, + "pass@2": 0.31930681673863354, + "pass@4": 0.3933600641653665, + "pass@8": 0.4584629930505219, + "pass@16": 0.5141375485482685, + "pass@32": 0.5606915077929782, + "pass@64": 0.6005003708826756, + "pass@128": 0.6387096774193548 + }, + "19": { + "pass@1": 0.18074962235649547, + "pass@2": 0.2564385243951755, + "pass@4": 0.33101056353982955, + "pass@8": 0.39293476132895816, + "pass@16": 0.4417977642220465, + "pass@32": 0.486950509381447, + "pass@64": 0.5332543587477647, + "pass@128": 0.5740181268882175 + }, + "20": { + "pass@1": 0.1501266891891892, + "pass@2": 0.22014177070771548, + "pass@4": 0.29605818079046403, + "pass@8": 0.36845622163096464, + "pass@16": 0.4307900693887324, + "pass@32": 0.4817076892328625, + "pass@64": 0.5253941303577103, + "pass@128": 0.5615615615615616 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep4-block8/composition/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-ep4-block8/composition/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..56962b6fa1bd0311eb9587c5a5312c71cc1a24f5 --- /dev/null +++ b/SDAR-100M-ep4-block8/composition/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-ep4-block8/composition/autoregressive", "results_dir": "results/SDAR-100M-ep4-block8/composition/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-ep4-block8/composition/autoregressive/checkpoint-86183_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1019571, "value_mismatch": 10168, "dependency_mismatch": 130229, "answer_mismatch": 9226, "extra_nodes": 13098, "missing_nodes": 13188, "perfect_with_extra": 354, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 336842, "value_mismatch": 3506, "answer_mismatch": 3146, "dependency_mismatch": 41358, "extra_nodes": 3295, "missing_nodes": 3427, "perfect_with_extra": 126}, "teachers_in_school": {"total": 390656, "perfect": 345485, "dependency_mismatch": 44406, "value_mismatch": 3386, "answer_mismatch": 3231, "extra_nodes": 4614, "missing_nodes": 4592, "perfect_with_extra": 115}, "movie_festival_awards": {"total": 382336, "perfect": 337244, "dependency_mismatch": 44465, "missing_nodes": 5169, "extra_nodes": 5189, "value_mismatch": 3276, "answer_mismatch": 2849, "perfect_with_extra": 113}}, "results_path": "results/SDAR-100M-ep4-block8/composition/autoregressive/graphs/checkpoint-86183_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-ep4-block8/composition/autoregressive/checkpoint-86183_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 270135, "value_mismatch": 780671, "dependency_mismatch": 935556, "answer_mismatch": 679827, "extra_nodes": 55883, "missing_nodes": 614937, "perfect_with_extra": 30, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 261475, "dependency_mismatch": 316065, "answer_mismatch": 226113, "extra_nodes": 19489, "missing_nodes": 210633, "perfect": 91974, "perfect_with_extra": 19}, "movie_festival_awards": {"total": 418816, "perfect": 87603, "value_mismatch": 254202, "dependency_mismatch": 305307, "answer_mismatch": 222851, "extra_nodes": 18790, "missing_nodes": 196466, "perfect_with_extra": 10}, "teachers_in_school": {"total": 430464, "perfect": 90558, "value_mismatch": 264994, "dependency_mismatch": 314184, "answer_mismatch": 230863, "extra_nodes": 17604, "missing_nodes": 207838, "perfect_with_extra": 1}}, "results_path": "results/SDAR-100M-ep4-block8/composition/autoregressive/graphs/checkpoint-86183_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1289706, "value_mismatch": 790839, "dependency_mismatch": 1065785, "answer_mismatch": 689053, "extra_nodes": 68981, "missing_nodes": 628125, "perfect_with_extra": 384, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-ep4-block8/composition/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 428816, "value_mismatch": 264981, "answer_mismatch": 229259, "dependency_mismatch": 357423, "extra_nodes": 22784, "missing_nodes": 214060, "perfect_with_extra": 145}, "teachers_in_school": {"total": 821120, "perfect": 436043, "dependency_mismatch": 358590, "value_mismatch": 268380, "answer_mismatch": 234094, "extra_nodes": 22218, "missing_nodes": 212430, "perfect_with_extra": 116}, "movie_festival_awards": {"total": 801152, "perfect": 424847, "dependency_mismatch": 349772, "missing_nodes": 201635, "extra_nodes": 23979, "value_mismatch": 257478, "answer_mismatch": 225700, "perfect_with_extra": 123}}}} diff --git a/SDAR-100M-ep4-block8/composition/autoregressive/graphs/results.json b/SDAR-100M-ep4-block8/composition/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..650b73cd645ef6607557dbae0a1dd382f325165f --- /dev/null +++ b/SDAR-100M-ep4-block8/composition/autoregressive/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-86183_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.996875, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9587301587301588, + "pass@2": 0.9809523809523809, + "pass@4": 0.9904761904761905, + "pass@8": 0.9968253968253968, + "pass@16": 0.9968253968253968, + "pass@32": 0.9968253968253968, + "pass@64": 0.9968253968253968, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9580838323353293, + "pass@2": 0.9760479041916168, + "pass@4": 0.9820359281437125, + "pass@8": 0.9940119760479041, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8905325443786982, + "pass@2": 0.9112426035502958, + "pass@4": 0.9556213017751479, + "pass@8": 0.9881656804733728, + "pass@16": 0.9970414201183432, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8173076923076923, + "pass@2": 0.8942307692307693, + "pass@4": 0.9583333333333334, + "pass@8": 0.9743589743589743, + "pass@16": 0.9903846153846154, + "pass@32": 0.9967948717948718, + "pass@64": 0.9967948717948718, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7633802816901408, + "pass@2": 0.8394366197183099, + "pass@4": 0.9098591549295775, + "pass@8": 0.9774647887323944, + "pass@16": 0.9943661971830986, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6519174041297935, + "pass@2": 0.7079646017699115, + "pass@4": 0.799410029498525, + "pass@8": 0.8702064896755162, + "pass@16": 0.9085545722713865, + "pass@32": 0.9292035398230089, + "pass@64": 0.943952802359882, + "pass@128": 0.9469026548672567 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9587301587301588, + "6": 0.9580838323353293, + "7": 0.8905325443786982, + "8": 0.8173076923076923, + "9": 0.7633802816901408, + "10": 0.6519174041297935 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9809523809523809, + "6": 0.9760479041916168, + "7": 0.9112426035502958, + "8": 0.8942307692307693, + "9": 0.8394366197183099, + "10": 0.7079646017699115 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9904761904761905, + "6": 0.9820359281437125, + "7": 0.9556213017751479, + "8": 0.9583333333333334, + "9": 0.9098591549295775, + "10": 0.799410029498525 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 0.9940119760479041, + "7": 0.9881656804733728, + "8": 0.9743589743589743, + "9": 0.9774647887323944, + "10": 0.8702064896755162 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9903846153846154, + "9": 0.9943661971830986, + "10": 0.9085545722713865 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9292035398230089 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.943952802359882 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9469026548672567 + } + }, + "total_pass_at_k": { + "pass@1": 0.8902397838568051, + "pass@2": 0.9209726443768997, + "pass@4": 0.9537318473488686, + "pass@8": 0.9773725092874029, + "pass@16": 0.9871664978047957, + "pass@32": 0.9912191827085444, + "pass@64": 0.9929078014184397, + "pass@128": 0.9932455251604188 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9853801169590644, + "pass@2": 0.9941520467836257, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9588235294117647, + "pass@2": 0.9823529411764705, + "pass@4": 0.9970588235294118, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9369085173501577, + "pass@2": 0.9589905362776026, + "pass@4": 0.9842271293375394, + "pass@8": 0.9936908517350158, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8291139240506329, + "pass@2": 0.8987341772151899, + "pass@4": 0.9493670886075949, + "pass@8": 0.9810126582278481, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8018018018018018, + "pass@2": 0.8648648648648649, + "pass@4": 0.9279279279279279, + "pass@8": 0.978978978978979, + "pass@16": 0.996996996996997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.752411575562701, + "pass@2": 0.819935691318328, + "pass@4": 0.8906752411575563, + "pass@8": 0.954983922829582, + "pass@16": 0.9807073954983923, + "pass@32": 0.9967845659163987, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.608955223880597, + "pass@2": 0.6835820895522388, + "pass@4": 0.7671641791044777, + "pass@8": 0.844776119402985, + "pass@16": 0.8895522388059701, + "pass@32": 0.9462686567164179, + "pass@64": 0.9641791044776119, + "pass@128": 0.9641791044776119 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9853801169590644, + "5": 0.9588235294117647, + "6": 0.9369085173501577, + "7": 0.8291139240506329, + "8": 0.8018018018018018, + "9": 0.752411575562701, + "10": 0.608955223880597 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9823529411764705, + "6": 0.9589905362776026, + "7": 0.8987341772151899, + "8": 0.8648648648648649, + "9": 0.819935691318328, + "10": 0.6835820895522388 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9842271293375394, + "7": 0.9493670886075949, + "8": 0.9279279279279279, + "9": 0.8906752411575563, + "10": 0.7671641791044777 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9936908517350158, + "7": 0.9810126582278481, + "8": 0.978978978978979, + "9": 0.954983922829582, + "10": 0.844776119402985 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 0.9807073954983923, + "10": 0.8895522388059701 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9967845659163987, + "10": 0.9462686567164179 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9641791044776119 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9641791044776119 + } + }, + "total_pass_at_k": { + "pass@1": 0.8771342484097757, + "pass@2": 0.9129561432875795, + "pass@4": 0.9471041178439906, + "pass@8": 0.9728824907934382, + "pass@16": 0.9852695011717443, + "pass@32": 0.9936391027787077, + "pass@64": 0.9959825912286575, + "pass@128": 0.9959825912286575 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9763313609467456, + "pass@2": 0.9822485207100592, + "pass@4": 0.9970414201183432, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9594202898550724, + "pass@2": 0.9797101449275363, + "pass@4": 0.9971014492753624, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9541547277936963, + "pass@2": 0.9770773638968482, + "pass@4": 0.9914040114613181, + "pass@8": 0.997134670487106, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8554913294797688, + "pass@2": 0.8959537572254336, + "pass@4": 0.9421965317919075, + "pass@8": 0.9739884393063584, + "pass@16": 0.9971098265895953, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7943661971830986, + "pass@2": 0.8647887323943662, + "pass@4": 0.9183098591549296, + "pass@8": 0.9859154929577465, + "pass@16": 0.9971830985915493, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7155688622754491, + "pass@2": 0.7934131736526946, + "pass@4": 0.8862275449101796, + "pass@8": 0.9491017964071856, + "pass@16": 0.9850299401197605, + "pass@32": 0.9970059880239521, + "pass@64": 0.9970059880239521, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6349693251533742, + "pass@2": 0.7055214723926381, + "pass@4": 0.7730061349693251, + "pass@8": 0.8466257668711656, + "pass@16": 0.8926380368098159, + "pass@32": 0.9263803680981595, + "pass@64": 0.9386503067484663, + "pass@128": 0.941717791411043 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9763313609467456, + "5": 0.9594202898550724, + "6": 0.9541547277936963, + "7": 0.8554913294797688, + "8": 0.7943661971830986, + "9": 0.7155688622754491, + "10": 0.6349693251533742 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9822485207100592, + "5": 0.9797101449275363, + "6": 0.9770773638968482, + "7": 0.8959537572254336, + "8": 0.8647887323943662, + "9": 0.7934131736526946, + "10": 0.7055214723926381 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.9971014492753624, + "6": 0.9914040114613181, + "7": 0.9421965317919075, + "8": 0.9183098591549296, + "9": 0.8862275449101796, + "10": 0.7730061349693251 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9739884393063584, + "8": 0.9859154929577465, + "9": 0.9491017964071856, + "10": 0.8466257668711656 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9971830985915493, + "9": 0.9850299401197605, + "10": 0.8926380368098159 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.9263803680981595 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.9386503067484663 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.941717791411043 + } + }, + "total_pass_at_k": { + "pass@1": 0.877129750982962, + "pass@2": 0.9115334207077327, + "pass@4": 0.9456094364351245, + "pass@8": 0.973132372214941, + "pass@16": 0.9862385321100917, + "pass@32": 0.9918086500655308, + "pass@64": 0.9931192660550459, + "pass@128": 0.9937745740498034 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985, + "pass@2": 0.991, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.959, + "pass@2": 0.981, + "pass@4": 0.995, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.95, + "pass@2": 0.971, + "pass@4": 0.986, + "pass@8": 0.995, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.859, + "pass@2": 0.902, + "pass@4": 0.949, + "pass@8": 0.981, + "pass@16": 0.998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.804, + "pass@2": 0.874, + "pass@4": 0.934, + "pass@8": 0.98, + "pass@16": 0.995, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.744, + "pass@2": 0.818, + "pass@4": 0.896, + "pass@8": 0.961, + "pass@16": 0.987, + "pass@32": 0.998, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.632, + "pass@2": 0.699, + "pass@4": 0.78, + "pass@8": 0.854, + "pass@16": 0.897, + "pass@32": 0.934, + "pass@64": 0.949, + "pass@128": 0.951 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.985, + "5": 0.959, + "6": 0.95, + "7": 0.859, + "8": 0.804, + "9": 0.744, + "10": 0.632 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.991, + "5": 0.981, + "6": 0.971, + "7": 0.902, + "8": 0.874, + "9": 0.818, + "10": 0.699 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.995, + "6": 0.986, + "7": 0.949, + "8": 0.934, + "9": 0.896, + "10": 0.78 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 0.995, + "7": 0.981, + "8": 0.98, + "9": 0.961, + "10": 0.854 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.998, + "8": 0.995, + "9": 0.987, + "10": 0.897 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.998, + "10": 0.934 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.999, + "10": 0.949 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 1.0, + "10": 0.951 + } + }, + "total_pass_at_k": { + "pass@1": 0.8814444444444445, + "pass@2": 0.9151111111111111, + "pass@4": 0.9487777777777778, + "pass@8": 0.9744444444444444, + "pass@16": 0.9862222222222222, + "pass@32": 0.9922222222222222, + "pass@64": 0.994, + "pass@128": 0.9943333333333333 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-86183_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5694050991501416, + "pass@2": 0.6515580736543909, + "pass@4": 0.7195467422096318, + "pass@8": 0.8045325779036827, + "pass@16": 0.8725212464589235, + "pass@32": 0.9093484419263456, + "pass@64": 0.9235127478753541, + "pass@128": 0.9291784702549575 + }, + "12": { + "pass@1": 0.5480225988700564, + "pass@2": 0.615819209039548, + "pass@4": 0.6779661016949152, + "pass@8": 0.731638418079096, + "pass@16": 0.7768361581920904, + "pass@32": 0.8192090395480226, + "pass@64": 0.8389830508474576, + "pass@128": 0.844632768361582 + }, + "13": { + "pass@1": 0.3448275862068966, + "pass@2": 0.40752351097178685, + "pass@4": 0.47962382445141066, + "pass@8": 0.5517241379310345, + "pass@16": 0.6238244514106583, + "pass@32": 0.6990595611285266, + "pass@64": 0.7210031347962382, + "pass@128": 0.7304075235109718 + }, + "14": { + "pass@1": 0.308411214953271, + "pass@2": 0.3894080996884735, + "pass@4": 0.42990654205607476, + "pass@8": 0.48286604361370716, + "pass@16": 0.5295950155763239, + "pass@32": 0.6012461059190031, + "pass@64": 0.6510903426791277, + "pass@128": 0.7071651090342679 + }, + "15": { + "pass@1": 0.23121387283236994, + "pass@2": 0.2832369942196532, + "pass@4": 0.3468208092485549, + "pass@8": 0.4190751445086705, + "pass@16": 0.4653179190751445, + "pass@32": 0.5028901734104047, + "pass@64": 0.5635838150289018, + "pass@128": 0.6242774566473989 + }, + "16": { + "pass@1": 0.06451612903225806, + "pass@2": 0.08709677419354839, + "pass@4": 0.1032258064516129, + "pass@8": 0.11612903225806452, + "pass@16": 0.13548387096774195, + "pass@32": 0.18064516129032257, + "pass@64": 0.2064516129032258, + "pass@128": 0.20967741935483872 + }, + "17": { + "pass@1": 0.049132947976878616, + "pass@2": 0.06647398843930635, + "pass@4": 0.08092485549132948, + "pass@8": 0.09826589595375723, + "pass@16": 0.12427745664739884, + "pass@32": 0.14739884393063585, + "pass@64": 0.1676300578034682, + "pass@128": 0.19653179190751446 + }, + "18": { + "pass@1": 0.017142857142857144, + "pass@2": 0.022857142857142857, + "pass@4": 0.02857142857142857, + "pass@8": 0.04, + "pass@16": 0.04857142857142857, + "pass@32": 0.06571428571428571, + "pass@64": 0.07428571428571429, + "pass@128": 0.08 + }, + "19": { + "pass@1": 0.003003003003003003, + "pass@2": 0.009009009009009009, + "pass@4": 0.012012012012012012, + "pass@8": 0.012012012012012012, + "pass@16": 0.012012012012012012, + "pass@32": 0.012012012012012012, + "pass@64": 0.012012012012012012, + "pass@128": 0.012012012012012012 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.003003003003003003, + "pass@8": 0.003003003003003003, + "pass@16": 0.003003003003003003, + "pass@32": 0.006006006006006006, + "pass@64": 0.012012012012012012, + "pass@128": 0.015015015015015015 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5694050991501416, + "12": 0.5480225988700564, + "13": 0.3448275862068966, + "14": 0.308411214953271, + "15": 0.23121387283236994, + "16": 0.06451612903225806, + "17": 0.049132947976878616, + "18": 0.017142857142857144, + "19": 0.003003003003003003, + "20": 0.0 + }, + "pass@2": { + "11": 0.6515580736543909, + "12": 0.615819209039548, + "13": 0.40752351097178685, + "14": 0.3894080996884735, + "15": 0.2832369942196532, + "16": 0.08709677419354839, + "17": 0.06647398843930635, + "18": 0.022857142857142857, + "19": 0.009009009009009009, + "20": 0.0 + }, + "pass@4": { + "11": 0.7195467422096318, + "12": 0.6779661016949152, + "13": 0.47962382445141066, + "14": 0.42990654205607476, + "15": 0.3468208092485549, + "16": 0.1032258064516129, + "17": 0.08092485549132948, + "18": 0.02857142857142857, + "19": 0.012012012012012012, + "20": 0.003003003003003003 + }, + "pass@8": { + "11": 0.8045325779036827, + "12": 0.731638418079096, + "13": 0.5517241379310345, + "14": 0.48286604361370716, + "15": 0.4190751445086705, + "16": 0.11612903225806452, + "17": 0.09826589595375723, + "18": 0.04, + "19": 0.012012012012012012, + "20": 0.003003003003003003 + }, + "pass@16": { + "11": 0.8725212464589235, + "12": 0.7768361581920904, + "13": 0.6238244514106583, + "14": 0.5295950155763239, + "15": 0.4653179190751445, + "16": 0.13548387096774195, + "17": 0.12427745664739884, + "18": 0.04857142857142857, + "19": 0.012012012012012012, + "20": 0.003003003003003003 + }, + "pass@32": { + "11": 0.9093484419263456, + "12": 0.8192090395480226, + "13": 0.6990595611285266, + "14": 0.6012461059190031, + "15": 0.5028901734104047, + "16": 0.18064516129032257, + "17": 0.14739884393063585, + "18": 0.06571428571428571, + "19": 0.012012012012012012, + "20": 0.006006006006006006 + }, + "pass@64": { + "11": 0.9235127478753541, + "12": 0.8389830508474576, + "13": 0.7210031347962382, + "14": 0.6510903426791277, + "15": 0.5635838150289018, + "16": 0.2064516129032258, + "17": 0.1676300578034682, + "18": 0.07428571428571429, + "19": 0.012012012012012012, + "20": 0.012012012012012012 + }, + "pass@128": { + "11": 0.9291784702549575, + "12": 0.844632768361582, + "13": 0.7304075235109718, + "14": 0.7071651090342679, + "15": 0.6242774566473989, + "16": 0.20967741935483872, + "17": 0.19653179190751446, + "18": 0.08, + "19": 0.012012012012012012, + "20": 0.015015015015015015 + } + }, + "total_pass_at_k": { + "pass@1": 0.21634472511144132, + "pass@2": 0.25616641901931647, + "pass@4": 0.2912332838038633, + "pass@8": 0.32927191679049034, + "pass@16": 0.362555720653789, + "pass@32": 0.39732540861812776, + "pass@64": 0.4199108469539376, + "pass@128": 0.437741456166419 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5492063492063493, + "pass@2": 0.6190476190476191, + "pass@4": 0.707936507936508, + "pass@8": 0.8063492063492064, + "pass@16": 0.8761904761904762, + "pass@32": 0.9333333333333333, + "pass@64": 0.9428571428571428, + "pass@128": 0.9619047619047619 + }, + "12": { + "pass@1": 0.5229357798165137, + "pass@2": 0.5963302752293578, + "pass@4": 0.6605504587155964, + "pass@8": 0.7308868501529052, + "pass@16": 0.7828746177370031, + "pass@32": 0.8226299694189603, + "pass@64": 0.8470948012232415, + "pass@128": 0.8562691131498471 + }, + "13": { + "pass@1": 0.37037037037037035, + "pass@2": 0.4444444444444444, + "pass@4": 0.5, + "pass@8": 0.5709876543209876, + "pass@16": 0.6111111111111112, + "pass@32": 0.6512345679012346, + "pass@64": 0.6759259259259259, + "pass@128": 0.7006172839506173 + }, + "14": { + "pass@1": 0.29080118694362017, + "pass@2": 0.3264094955489614, + "pass@4": 0.40059347181008903, + "pass@8": 0.4599406528189911, + "pass@16": 0.5103857566765578, + "pass@32": 0.5845697329376854, + "pass@64": 0.6617210682492581, + "pass@128": 0.7151335311572701 + }, + "15": { + "pass@1": 0.23148148148148148, + "pass@2": 0.2716049382716049, + "pass@4": 0.32407407407407407, + "pass@8": 0.3765432098765432, + "pass@16": 0.43209876543209874, + "pass@32": 0.5061728395061729, + "pass@64": 0.5679012345679012, + "pass@128": 0.6327160493827161 + }, + "16": { + "pass@1": 0.06489675516224189, + "pass@2": 0.09144542772861357, + "pass@4": 0.11799410029498525, + "pass@8": 0.14749262536873156, + "pass@16": 0.1710914454277286, + "pass@32": 0.20058997050147492, + "pass@64": 0.23303834808259588, + "pass@128": 0.2743362831858407 + }, + "17": { + "pass@1": 0.03614457831325301, + "pass@2": 0.0572289156626506, + "pass@4": 0.06626506024096386, + "pass@8": 0.08433734939759036, + "pass@16": 0.10542168674698796, + "pass@32": 0.11746987951807229, + "pass@64": 0.14156626506024098, + "pass@128": 0.18072289156626506 + }, + "18": { + "pass@1": 0.035483870967741936, + "pass@2": 0.05161290322580645, + "pass@4": 0.06451612903225806, + "pass@8": 0.08387096774193549, + "pass@16": 0.1, + "pass@32": 0.11612903225806452, + "pass@64": 0.12258064516129032, + "pass@128": 0.14193548387096774 + }, + "19": { + "pass@1": 0.0030211480362537764, + "pass@2": 0.006042296072507553, + "pass@4": 0.012084592145015106, + "pass@8": 0.012084592145015106, + "pass@16": 0.015105740181268883, + "pass@32": 0.021148036253776436, + "pass@64": 0.02416918429003021, + "pass@128": 0.027190332326283987 + }, + "20": { + "pass@1": 0.006006006006006006, + "pass@2": 0.006006006006006006, + "pass@4": 0.009009009009009009, + "pass@8": 0.012012012012012012, + "pass@16": 0.018018018018018018, + "pass@32": 0.02702702702702703, + "pass@64": 0.03303303303303303, + "pass@128": 0.03303303303303303 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5492063492063493, + "12": 0.5229357798165137, + "13": 0.37037037037037035, + "14": 0.29080118694362017, + "15": 0.23148148148148148, + "16": 0.06489675516224189, + "17": 0.03614457831325301, + "18": 0.035483870967741936, + "19": 0.0030211480362537764, + "20": 0.006006006006006006 + }, + "pass@2": { + "11": 0.6190476190476191, + "12": 0.5963302752293578, + "13": 0.4444444444444444, + "14": 0.3264094955489614, + "15": 0.2716049382716049, + "16": 0.09144542772861357, + "17": 0.0572289156626506, + "18": 0.05161290322580645, + "19": 0.006042296072507553, + "20": 0.006006006006006006 + }, + "pass@4": { + "11": 0.707936507936508, + "12": 0.6605504587155964, + "13": 0.5, + "14": 0.40059347181008903, + "15": 0.32407407407407407, + "16": 0.11799410029498525, + "17": 0.06626506024096386, + "18": 0.06451612903225806, + "19": 0.012084592145015106, + "20": 0.009009009009009009 + }, + "pass@8": { + "11": 0.8063492063492064, + "12": 0.7308868501529052, + "13": 0.5709876543209876, + "14": 0.4599406528189911, + "15": 0.3765432098765432, + "16": 0.14749262536873156, + "17": 0.08433734939759036, + "18": 0.08387096774193549, + "19": 0.012084592145015106, + "20": 0.012012012012012012 + }, + "pass@16": { + "11": 0.8761904761904762, + "12": 0.7828746177370031, + "13": 0.6111111111111112, + "14": 0.5103857566765578, + "15": 0.43209876543209874, + "16": 0.1710914454277286, + "17": 0.10542168674698796, + "18": 0.1, + "19": 0.015105740181268883, + "20": 0.018018018018018018 + }, + "pass@32": { + "11": 0.9333333333333333, + "12": 0.8226299694189603, + "13": 0.6512345679012346, + "14": 0.5845697329376854, + "15": 0.5061728395061729, + "16": 0.20058997050147492, + "17": 0.11746987951807229, + "18": 0.11612903225806452, + "19": 0.021148036253776436, + "20": 0.02702702702702703 + }, + "pass@64": { + "11": 0.9428571428571428, + "12": 0.8470948012232415, + "13": 0.6759259259259259, + "14": 0.6617210682492581, + "15": 0.5679012345679012, + "16": 0.23303834808259588, + "17": 0.14156626506024098, + "18": 0.12258064516129032, + "19": 0.02416918429003021, + "20": 0.03303303303303303 + }, + "pass@128": { + "11": 0.9619047619047619, + "12": 0.8562691131498471, + "13": 0.7006172839506173, + "14": 0.7151335311572701, + "15": 0.6327160493827161, + "16": 0.2743362831858407, + "17": 0.18072289156626506, + "18": 0.14193548387096774, + "19": 0.027190332326283987, + "20": 0.03303303303303303 + } + }, + "total_pass_at_k": { + "pass@1": 0.2093520782396088, + "pass@2": 0.24511002444987776, + "pass@4": 0.28422982885085574, + "pass@8": 0.3261002444987775, + "pass@16": 0.359718826405868, + "pass@32": 0.3954767726161369, + "pass@64": 0.42267726161369196, + "pass@128": 0.4501833740831296 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6054216867469879, + "pass@2": 0.6626506024096386, + "pass@4": 0.7379518072289156, + "pass@8": 0.8102409638554217, + "pass@16": 0.8885542168674698, + "pass@32": 0.927710843373494, + "pass@64": 0.9427710843373494, + "pass@128": 0.9548192771084337 + }, + "12": { + "pass@1": 0.47335423197492166, + "pass@2": 0.5579937304075235, + "pass@4": 0.6332288401253918, + "pass@8": 0.7021943573667712, + "pass@16": 0.7648902821316614, + "pass@32": 0.7899686520376176, + "pass@64": 0.8119122257053292, + "pass@128": 0.8244514106583072 + }, + "13": { + "pass@1": 0.35014005602240894, + "pass@2": 0.42296918767507, + "pass@4": 0.5154061624649859, + "pass@8": 0.5994397759103641, + "pass@16": 0.6582633053221288, + "pass@32": 0.7030812324929971, + "pass@64": 0.7282913165266106, + "pass@128": 0.7507002801120448 + }, + "14": { + "pass@1": 0.2894736842105263, + "pass@2": 0.3654970760233918, + "pass@4": 0.40350877192982454, + "pass@8": 0.4649122807017544, + "pass@16": 0.5233918128654971, + "pass@32": 0.5701754385964912, + "pass@64": 0.6257309941520468, + "pass@128": 0.6754385964912281 + }, + "15": { + "pass@1": 0.24242424242424243, + "pass@2": 0.2878787878787879, + "pass@4": 0.3515151515151515, + "pass@8": 0.4, + "pass@16": 0.44545454545454544, + "pass@32": 0.5, + "pass@64": 0.5606060606060606, + "pass@128": 0.6272727272727273 + }, + "16": { + "pass@1": 0.06837606837606838, + "pass@2": 0.08547008547008547, + "pass@4": 0.10826210826210826, + "pass@8": 0.1452991452991453, + "pass@16": 0.1737891737891738, + "pass@32": 0.18233618233618235, + "pass@64": 0.21082621082621084, + "pass@128": 0.23931623931623933 + }, + "17": { + "pass@1": 0.040372670807453416, + "pass@2": 0.055900621118012424, + "pass@4": 0.07763975155279502, + "pass@8": 0.08695652173913043, + "pass@16": 0.09627329192546584, + "pass@32": 0.12732919254658384, + "pass@64": 0.14285714285714285, + "pass@128": 0.15838509316770186 + }, + "18": { + "pass@1": 0.008823529411764706, + "pass@2": 0.023529411764705882, + "pass@4": 0.03235294117647059, + "pass@8": 0.05, + "pass@16": 0.07647058823529412, + "pass@32": 0.08823529411764706, + "pass@64": 0.10294117647058823, + "pass@128": 0.12058823529411765 + }, + "19": { + "pass@1": 0.002976190476190476, + "pass@2": 0.002976190476190476, + "pass@4": 0.002976190476190476, + "pass@8": 0.008928571428571428, + "pass@16": 0.01488095238095238, + "pass@32": 0.01488095238095238, + "pass@64": 0.020833333333333332, + "pass@128": 0.020833333333333332 + }, + "20": { + "pass@1": 0.0029940119760479044, + "pass@2": 0.005988023952095809, + "pass@4": 0.008982035928143712, + "pass@8": 0.011976047904191617, + "pass@16": 0.017964071856287425, + "pass@32": 0.020958083832335328, + "pass@64": 0.023952095808383235, + "pass@128": 0.029940119760479042 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6054216867469879, + "12": 0.47335423197492166, + "13": 0.35014005602240894, + "14": 0.2894736842105263, + "15": 0.24242424242424243, + "16": 0.06837606837606838, + "17": 0.040372670807453416, + "18": 0.008823529411764706, + "19": 0.002976190476190476, + "20": 0.0029940119760479044 + }, + "pass@2": { + "11": 0.6626506024096386, + "12": 0.5579937304075235, + "13": 0.42296918767507, + "14": 0.3654970760233918, + "15": 0.2878787878787879, + "16": 0.08547008547008547, + "17": 0.055900621118012424, + "18": 0.023529411764705882, + "19": 0.002976190476190476, + "20": 0.005988023952095809 + }, + "pass@4": { + "11": 0.7379518072289156, + "12": 0.6332288401253918, + "13": 0.5154061624649859, + "14": 0.40350877192982454, + "15": 0.3515151515151515, + "16": 0.10826210826210826, + "17": 0.07763975155279502, + "18": 0.03235294117647059, + "19": 0.002976190476190476, + "20": 0.008982035928143712 + }, + "pass@8": { + "11": 0.8102409638554217, + "12": 0.7021943573667712, + "13": 0.5994397759103641, + "14": 0.4649122807017544, + "15": 0.4, + "16": 0.1452991452991453, + "17": 0.08695652173913043, + "18": 0.05, + "19": 0.008928571428571428, + "20": 0.011976047904191617 + }, + "pass@16": { + "11": 0.8885542168674698, + "12": 0.7648902821316614, + "13": 0.6582633053221288, + "14": 0.5233918128654971, + "15": 0.44545454545454544, + "16": 0.1737891737891738, + "17": 0.09627329192546584, + "18": 0.07647058823529412, + "19": 0.01488095238095238, + "20": 0.017964071856287425 + }, + "pass@32": { + "11": 0.927710843373494, + "12": 0.7899686520376176, + "13": 0.7030812324929971, + "14": 0.5701754385964912, + "15": 0.5, + "16": 0.18233618233618235, + "17": 0.12732919254658384, + "18": 0.08823529411764706, + "19": 0.01488095238095238, + "20": 0.020958083832335328 + }, + "pass@64": { + "11": 0.9427710843373494, + "12": 0.8119122257053292, + "13": 0.7282913165266106, + "14": 0.6257309941520468, + "15": 0.5606060606060606, + "16": 0.21082621082621084, + "17": 0.14285714285714285, + "18": 0.10294117647058823, + "19": 0.020833333333333332, + "20": 0.023952095808383235 + }, + "pass@128": { + "11": 0.9548192771084337, + "12": 0.8244514106583072, + "13": 0.7507002801120448, + "14": 0.6754385964912281, + "15": 0.6272727272727273, + "16": 0.23931623931623933, + "17": 0.15838509316770186, + "18": 0.12058823529411765, + "19": 0.020833333333333332, + "20": 0.029940119760479042 + } + }, + "total_pass_at_k": { + "pass@1": 0.20755278025572405, + "pass@2": 0.24620874219446923, + "pass@4": 0.2863514719000892, + "pass@8": 0.32738626226583406, + "pass@16": 0.3654475170978293, + "pass@32": 0.391911983348201, + "pass@64": 0.41659232827832293, + "pass@128": 0.43978590544157004 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.575, + "pass@2": 0.645, + "pass@4": 0.722, + "pass@8": 0.807, + "pass@16": 0.879, + "pass@32": 0.923, + "pass@64": 0.936, + "pass@128": 0.948 + }, + "12": { + "pass@1": 0.516, + "pass@2": 0.591, + "pass@4": 0.658, + "pass@8": 0.722, + "pass@16": 0.775, + "pass@32": 0.811, + "pass@64": 0.833, + "pass@128": 0.842 + }, + "13": { + "pass@1": 0.355, + "pass@2": 0.425, + "pass@4": 0.499, + "pass@8": 0.575, + "pass@16": 0.632, + "pass@32": 0.685, + "pass@64": 0.709, + "pass@128": 0.728 + }, + "14": { + "pass@1": 0.296, + "pass@2": 0.36, + "pass@4": 0.411, + "pass@8": 0.469, + "pass@16": 0.521, + "pass@32": 0.585, + "pass@64": 0.646, + "pass@128": 0.699 + }, + "15": { + "pass@1": 0.235, + "pass@2": 0.281, + "pass@4": 0.341, + "pass@8": 0.399, + "pass@16": 0.448, + "pass@32": 0.503, + "pass@64": 0.564, + "pass@128": 0.628 + }, + "16": { + "pass@1": 0.066, + "pass@2": 0.088, + "pass@4": 0.11, + "pass@8": 0.137, + "pass@16": 0.161, + "pass@32": 0.188, + "pass@64": 0.217, + "pass@128": 0.242 + }, + "17": { + "pass@1": 0.042, + "pass@2": 0.06, + "pass@4": 0.075, + "pass@8": 0.09, + "pass@16": 0.109, + "pass@32": 0.131, + "pass@64": 0.151, + "pass@128": 0.179 + }, + "18": { + "pass@1": 0.02, + "pass@2": 0.032, + "pass@4": 0.041, + "pass@8": 0.057, + "pass@16": 0.074, + "pass@32": 0.089, + "pass@64": 0.099, + "pass@128": 0.113 + }, + "19": { + "pass@1": 0.003, + "pass@2": 0.006, + "pass@4": 0.009, + "pass@8": 0.011, + "pass@16": 0.014, + "pass@32": 0.016, + "pass@64": 0.019, + "pass@128": 0.02 + }, + "20": { + "pass@1": 0.003, + "pass@2": 0.004, + "pass@4": 0.007, + "pass@8": 0.009, + "pass@16": 0.013, + "pass@32": 0.018, + "pass@64": 0.023, + "pass@128": 0.026 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.575, + "12": 0.516, + "13": 0.355, + "14": 0.296, + "15": 0.235, + "16": 0.066, + "17": 0.042, + "18": 0.02, + "19": 0.003, + "20": 0.003 + }, + "pass@2": { + "11": 0.645, + "12": 0.591, + "13": 0.425, + "14": 0.36, + "15": 0.281, + "16": 0.088, + "17": 0.06, + "18": 0.032, + "19": 0.006, + "20": 0.004 + }, + "pass@4": { + "11": 0.722, + "12": 0.658, + "13": 0.499, + "14": 0.411, + "15": 0.341, + "16": 0.11, + "17": 0.075, + "18": 0.041, + "19": 0.009, + "20": 0.007 + }, + "pass@8": { + "11": 0.807, + "12": 0.722, + "13": 0.575, + "14": 0.469, + "15": 0.399, + "16": 0.137, + "17": 0.09, + "18": 0.057, + "19": 0.011, + "20": 0.009 + }, + "pass@16": { + "11": 0.879, + "12": 0.775, + "13": 0.632, + "14": 0.521, + "15": 0.448, + "16": 0.161, + "17": 0.109, + "18": 0.074, + "19": 0.014, + "20": 0.013 + }, + "pass@32": { + "11": 0.923, + "12": 0.811, + "13": 0.685, + "14": 0.585, + "15": 0.503, + "16": 0.188, + "17": 0.131, + "18": 0.089, + "19": 0.016, + "20": 0.018 + }, + "pass@64": { + "11": 0.936, + "12": 0.833, + "13": 0.709, + "14": 0.646, + "15": 0.564, + "16": 0.217, + "17": 0.151, + "18": 0.099, + "19": 0.019, + "20": 0.023 + }, + "pass@128": { + "11": 0.948, + "12": 0.842, + "13": 0.728, + "14": 0.699, + "15": 0.628, + "16": 0.242, + "17": 0.179, + "18": 0.113, + "19": 0.02, + "20": 0.026 + } + }, + "total_pass_at_k": { + "pass@1": 0.2111, + "pass@2": 0.2492, + "pass@4": 0.2873, + "pass@8": 0.3276, + "pass@16": 0.3626, + "pass@32": 0.3949, + "pass@64": 0.4197, + "pass@128": 0.4425 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-ep4-block8/composition/autoregressive/summary.csv b/SDAR-100M-ep4-block8/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..75ef654e667a0b9123d0a26c4b5ab6622dc9d0c8 --- /dev/null +++ b/SDAR-100M-ep4-block8/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-86183,0.9919913194444444,0.4688859375,0.7166726973684211,0.0,0.0,0.0,199.9115217013889,336.13270703125,1152000,1280000,2432000 diff --git a/SDAR-100M-ep4-block8/composition/autoregressive/summary.json b/SDAR-100M-ep4-block8/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d3fee84fc0064122ca0bb312e3e63dfd670390e2 --- /dev/null +++ b/SDAR-100M-ep4-block8/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-86183", + "id_acc": 0.9919913194444444, + "ood_acc": 0.4688859375, + "total_acc": 0.7166726973684211, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.9115217013889, + "ood_avg_resp_len": 336.13270703125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 580465, + "answer_accuracy": 0.7168641815523238, + "avg_response_len": 259.1584087496048, + "resp_tokens_sum": 209847820, + "pass_at_k": { + "pass@1": 0.7168641815523238, + "pass@2": 0.7563181321430583, + "pass@4": 0.7891167671974877, + "pass@8": 0.8160179728357079, + "pass@16": 0.8380543061281882, + "pass@32": 0.8560953306144278, + "pass@64": 0.8713585957411207, + "pass@128": 0.8858678469807145 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9424317846607669, + "pass@2": 0.9554277576475513, + "pass@4": 0.9645232338214357, + "pass@8": 0.9705729284703094, + "pass@16": 0.9750282528248507, + "pass@32": 0.979012371776588, + "pass@64": 0.9825793273267351, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9998511904761904, + "pass@2": 0.9999992188476441, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9982456961077845, + "pass@2": 0.9989778071809138, + "pass@4": 0.9996588796535164, + "pass@8": 0.9999646694808768, + "pass@16": 0.9999997257540574, + "pass@32": 0.9999999999968382, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997457470414202, + "pass@2": 0.9999949040208731, + "pass@4": 0.9999999986133391, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9944411057692307, + "pass@2": 0.9981253470119119, + "pass@4": 0.9996946421360792, + "pass@8": 0.9999886513123039, + "pass@16": 0.9999999830961206, + "pass@32": 0.999999999999996, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9926496478873239, + "pass@2": 0.996029721636908, + "pass@4": 0.9975660128751512, + "pass@8": 0.9983336498027551, + "pass@16": 0.999063138456991, + "pass@32": 0.9997387400876202, + "pass@64": 0.9999912792555583, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9241988314447592, + "pass@2": 0.9451147085721935, + "pass@4": 0.9583057509100311, + "pass@8": 0.9683471315113082, + "pass@16": 0.9758373276009441, + "pass@32": 0.9813627292403387, + "pass@64": 0.9854111072441946, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.832229872881356, + "pass@2": 0.8820969015525602, + "pass@4": 0.9064526634912154, + "pass@8": 0.9210831491573624, + "pass@16": 0.9322645619669031, + "pass@32": 0.9417992939933498, + "pass@64": 0.9503938639071714, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.6866918103448276, + "pass@2": 0.7621581344259865, + "pass@4": 0.8154033129283599, + "pass@8": 0.8542870664981383, + "pass@16": 0.8825616429284143, + "pass@32": 0.9031559569013347, + "pass@64": 0.9156100331712211, + "pass@128": 0.9184952978056427 + }, + "14": { + "pass@1": 0.6419635903426791, + "pass@2": 0.724484570853877, + "pass@4": 0.77979786434406, + "pass@8": 0.8168612902209831, + "pass@16": 0.8439180312348161, + "pass@32": 0.8659116440785112, + "pass@64": 0.8836119795991179, + "pass@128": 0.897196261682243 + }, + "15": { + "pass@1": 0.5370077673410405, + "pass@2": 0.6450683571526107, + "pass@4": 0.7338411369099088, + "pass@8": 0.7986025998546395, + "pass@16": 0.8436119684412422, + "pass@32": 0.8759949765481339, + "pass@64": 0.8985630011957301, + "pass@128": 0.9161849710982659 + }, + "16": { + "pass@1": 0.2983618951612903, + "pass@2": 0.3907174720599443, + "pass@4": 0.4704105074567288, + "pass@8": 0.535630716107639, + "pass@16": 0.5907458646622782, + "pass@32": 0.6372065651541644, + "pass@64": 0.6795116518435073, + "pass@128": 0.7225806451612903 + }, + "17": { + "pass@1": 0.2517837789017341, + "pass@2": 0.3336255035046197, + "pass@4": 0.41323942626752563, + "pass@8": 0.48810200863925596, + "pass@16": 0.5547621412716494, + "pass@32": 0.6106711542212494, + "pass@64": 0.6590666679799665, + "pass@128": 0.708092485549133 + }, + "18": { + "pass@1": 0.19428571428571428, + "pass@2": 0.2676690804274464, + "pass@4": 0.34212716508650715, + "pass@8": 0.4139243368536438, + "pass@16": 0.4805783850514051, + "pass@32": 0.5388344572164389, + "pass@64": 0.5889168734491658, + "pass@128": 0.6371428571428571 + }, + "19": { + "pass@1": 0.18327702702702703, + "pass@2": 0.2533107664751364, + "pass@4": 0.3211475995117731, + "pass@8": 0.3820281677250221, + "pass@16": 0.4348139659769961, + "pass@32": 0.4792898587653093, + "pass@64": 0.5180948362733667, + "pass@128": 0.5585585585585585 + }, + "20": { + "pass@1": 0.17429147897897898, + "pass@2": 0.2449047226360218, + "pass@4": 0.3154769668431085, + "pass@8": 0.3776520541403074, + "pass@16": 0.4277429922970302, + "pass@32": 0.46807247378301114, + "pass@64": 0.5071460121712338, + "pass@128": 0.5525525525525525 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 587027, + "answer_accuracy": 0.7149101227591582, + "avg_response_len": 267.97832959859704, + "resp_tokens_sum": 220042366, + "pass_at_k": { + "pass@1": 0.7149101227591582, + "pass@2": 0.7551878248875328, + "pass@4": 0.788089845667179, + "pass@8": 0.8152628237248414, + "pass@16": 0.8379713914459063, + "pass@32": 0.8568640308079691, + "pass@64": 0.8730298669312079, + "pass@128": 0.8874512860483242 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9370446702453987, + "pass@2": 0.9524521007439255, + "pass@4": 0.9622697201193403, + "pass@8": 0.9685217512597472, + "pass@16": 0.9732031072775292, + "pass@32": 0.9783710405726299, + "pass@64": 0.985072569920195, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999094202898551, + "pass@2": 0.9999989301609038, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994179799426934, + "pass@2": 0.9999027028856351, + "pass@4": 0.9999971459513121, + "pass@8": 0.9999999985260104, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998419436416185, + "pass@2": 0.9999996444176414, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954225352112676, + "pass@2": 0.9971879505378729, + "pass@4": 0.9978147124215107, + "pass@8": 0.9983444096217255, + "pass@16": 0.9990631538362801, + "pass@32": 0.9997387400876238, + "pass@64": 0.9999912792555583, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9916729041916168, + "pass@2": 0.9941191681597434, + "pass@4": 0.9953397361632191, + "pass@8": 0.9964540303892805, + "pass@16": 0.9977764166281999, + "pass@32": 0.9990105981622154, + "pass@64": 0.99982149369308, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.932652484939759, + "pass@2": 0.9551909656341906, + "pass@4": 0.967504908272008, + "pass@8": 0.9743332008558964, + "pass@16": 0.9776944087377336, + "pass@32": 0.9797107289052188, + "pass@64": 0.981380697080861, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8275127351097179, + "pass@2": 0.8741306802754675, + "pass@4": 0.8940140824873376, + "pass@8": 0.9059417569605126, + "pass@16": 0.9149301720126192, + "pass@32": 0.9210451859792664, + "pass@64": 0.9267178933506086, + "pass@128": 0.9341692789968652 + }, + "13": { + "pass@1": 0.6466211484593838, + "pass@2": 0.7352568980348043, + "pass@4": 0.7941474946934152, + "pass@8": 0.8322874017240125, + "pass@16": 0.8593893686275413, + "pass@32": 0.8812230108196569, + "pass@64": 0.8990334067498752, + "pass@128": 0.9131652661064426 + }, + "14": { + "pass@1": 0.6178956505847953, + "pass@2": 0.7086092548464337, + "pass@4": 0.7747260120445467, + "pass@8": 0.8222148893486398, + "pass@16": 0.856872436091234, + "pass@32": 0.8836770532613494, + "pass@64": 0.9065420227864525, + "pass@128": 0.9269005847953217 + }, + "15": { + "pass@1": 0.5457149621212121, + "pass@2": 0.661665995585779, + "pass@4": 0.7447602410494136, + "pass@8": 0.8003608834636073, + "pass@16": 0.837934698476195, + "pass@32": 0.8649150245288428, + "pass@64": 0.8867527597855025, + "pass@128": 0.906060606060606 + }, + "16": { + "pass@1": 0.3033297720797721, + "pass@2": 0.38540860477376226, + "pass@4": 0.4561851985382169, + "pass@8": 0.5197053256135613, + "pass@16": 0.5790541044663001, + "pass@32": 0.6333955577098603, + "pass@64": 0.6845196910184368, + "pass@128": 0.7321937321937322 + }, + "17": { + "pass@1": 0.2654794254658385, + "pass@2": 0.3596355822370031, + "pass@4": 0.4481225438823254, + "pass@8": 0.5224691510802295, + "pass@16": 0.580719897610317, + "pass@32": 0.6246834181217727, + "pass@64": 0.6622791915081849, + "pass@128": 0.7080745341614907 + }, + "18": { + "pass@1": 0.20091911764705883, + "pass@2": 0.28180508916164887, + "pass@4": 0.3687975969915525, + "pass@8": 0.4557692555372953, + "pass@16": 0.5351237386501738, + "pass@32": 0.6004961265979195, + "pass@64": 0.6507879587120721, + "pass@128": 0.6911764705882353 + }, + "19": { + "pass@1": 0.16022600446428573, + "pass@2": 0.22008833368485176, + "pass@4": 0.2805208366811292, + "pass@8": 0.3395545181912359, + "pass@16": 0.3959618657365702, + "pass@32": 0.4475826196030161, + "pass@64": 0.49460998571116155, + "pass@128": 0.5327380952380952 + }, + "20": { + "pass@1": 0.14796968562874252, + "pass@2": 0.2126687815314254, + "pass@4": 0.2800213515600969, + "pass@8": 0.3452822815254244, + "pass@16": 0.40522265822512654, + "pass@32": 0.4578358076989143, + "pass@64": 0.5011268713315313, + "pass@128": 0.5329341317365269 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 575456, + "answer_accuracy": 0.7182856686371625, + "avg_response_len": 287.9076030516057, + "resp_tokens_sum": 230657752, + "pass_at_k": { + "pass@1": 0.7182856686371625, + "pass@2": 0.7592052287226031, + "pass@4": 0.7927262515206405, + "pass@8": 0.8195860006912514, + "pass@16": 0.8409061998859412, + "pass@32": 0.8587729906883347, + "pass@64": 0.874955447824755, + "pass@128": 0.8894392075411408 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9499766791044776, + "pass@2": 0.963093856504877, + "pass@4": 0.9712197354062085, + "pass@8": 0.9767062892294843, + "pass@16": 0.9814928519708268, + "pass@32": 0.9866806213656, + "pass@64": 0.9918038267917523, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995863970588236, + "pass@2": 0.9999811834182493, + "pass@4": 0.9999999611261827, + "pass@8": 0.9999999999999815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9982009069400631, + "pass@2": 0.9996223670483617, + "pass@4": 0.999974433992281, + "pass@8": 0.999999864083505, + "pass@16": 0.9999999999987256, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9979232594936709, + "pass@2": 0.9995487547343765, + "pass@4": 0.9999514044130559, + "pass@8": 0.9999994224405141, + "pass@16": 0.9999999999663859, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9945101351351351, + "pass@2": 0.9986632794211536, + "pass@4": 0.9998438415918729, + "pass@8": 0.9999972864530364, + "pass@16": 0.9999999995382962, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925894292604501, + "pass@2": 0.9951194869230575, + "pass@4": 0.9964745589478163, + "pass@8": 0.998042664097683, + "pass@16": 0.9993761183512748, + "pass@32": 0.9999366961026899, + "pass@64": 0.9999998004436335, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9227926587301587, + "pass@2": 0.9470242000999873, + "pass@4": 0.960732205795704, + "pass@8": 0.9693603886062503, + "pass@16": 0.9756707700497491, + "pass@32": 0.9806056964313475, + "pass@64": 0.9844370895991283, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8355074541284404, + "pass@2": 0.8936402122613109, + "pass@4": 0.9236655713333995, + "pass@8": 0.9403579265835318, + "pass@16": 0.9498986664429232, + "pass@32": 0.9564481186800877, + "pass@64": 0.9620879995236127, + "pass@128": 0.9663608562691132 + }, + "13": { + "pass@1": 0.6571662808641975, + "pass@2": 0.7358156836298242, + "pass@4": 0.7884919118906429, + "pass@8": 0.8258481457444613, + "pass@16": 0.854241760542711, + "pass@32": 0.8774085425557125, + "pass@64": 0.8968649961634577, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.6426418768545994, + "pass@2": 0.7360609330825484, + "pass@4": 0.7982204229293296, + "pass@8": 0.8372049902894257, + "pass@16": 0.8620695977837106, + "pass@32": 0.8819375600885098, + "pass@64": 0.901686644657056, + "pass@128": 0.9198813056379822 + }, + "15": { + "pass@1": 0.5323350694444444, + "pass@2": 0.6377064195100618, + "pass@4": 0.7172337514523645, + "pass@8": 0.7761564818619724, + "pass@16": 0.8170385569299585, + "pass@32": 0.8452706602617858, + "pass@64": 0.8681272297013932, + "pass@128": 0.8858024691358025 + }, + "16": { + "pass@1": 0.2906296091445428, + "pass@2": 0.3808380078043341, + "pass@4": 0.4639649562609984, + "pass@8": 0.5348016643194988, + "pass@16": 0.5918998310984556, + "pass@32": 0.6414011766481142, + "pass@64": 0.6911248562212414, + "pass@128": 0.7433628318584071 + }, + "17": { + "pass@1": 0.24602315512048192, + "pass@2": 0.33004977587515394, + "pass@4": 0.41139936376552333, + "pass@8": 0.48491044209939826, + "pass@16": 0.5510107149708241, + "pass@32": 0.6105205843571422, + "pass@64": 0.6619147865574031, + "pass@128": 0.7048192771084337 + }, + "18": { + "pass@1": 0.24233870967741936, + "pass@2": 0.31930681673863354, + "pass@4": 0.3933600641653665, + "pass@8": 0.4584629930505219, + "pass@16": 0.5141375485482685, + "pass@32": 0.5606915077929782, + "pass@64": 0.6005003708826756, + "pass@128": 0.6387096774193548 + }, + "19": { + "pass@1": 0.18074962235649547, + "pass@2": 0.2564385243951755, + "pass@4": 0.33101056353982955, + "pass@8": 0.39293476132895816, + "pass@16": 0.4417977642220465, + "pass@32": 0.486950509381447, + "pass@64": 0.5332543587477647, + "pass@128": 0.5740181268882175 + }, + "20": { + "pass@1": 0.1501266891891892, + "pass@2": 0.22014177070771548, + "pass@4": 0.29605818079046403, + "pass@8": 0.36845622163096464, + "pass@16": 0.4307900693887324, + "pass@32": 0.4817076892328625, + "pass@64": 0.5253941303577103, + "pass@128": 0.5615615615615616 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch1-RL-op11-14/composition/graphs/composition_score_report.jsonl b/composition/Qwen3-100M-epoch1-RL-op11-14/composition/graphs/composition_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8b1e930fa3b2749837d6124e1fd594bb13c46968 --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op11-14/composition/graphs/composition_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/Qwen3-100M-epoch1-RL-op11-14/composition", "results_dir": "results/composition/Qwen3-100M-epoch1-RL-op11-14/composition/graphs", "per_file": [{"file": "results/composition/Qwen3-100M-epoch1-RL-op11-14/composition/hf_model_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1006012, "value_mismatch": 25592, "dependency_mismatch": 143382, "answer_mismatch": 22114, "extra_nodes": 16782, "missing_nodes": 9150, "perfect_with_extra": 855, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 331338, "dependency_mismatch": 46868, "value_mismatch": 8915, "answer_mismatch": 7596, "extra_nodes": 5364, "missing_nodes": 2886, "perfect_with_extra": 278}, "teachers_in_school": {"total": 390656, "perfect": 341769, "dependency_mismatch": 47890, "value_mismatch": 8193, "answer_mismatch": 7232, "extra_nodes": 5215, "missing_nodes": 3298, "perfect_with_extra": 109}, "movie_festival_awards": {"total": 382336, "perfect": 332905, "dependency_mismatch": 48624, "value_mismatch": 8484, "answer_mismatch": 7286, "extra_nodes": 6203, "missing_nodes": 2966, "perfect_with_extra": 468}}, "results_path": "results/composition/Qwen3-100M-epoch1-RL-op11-14/composition/graphs/hf_model_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/Qwen3-100M-epoch1-RL-op11-14/composition/hf_model_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 311307, "value_mismatch": 715946, "dependency_mismatch": 912759, "answer_mismatch": 570238, "extra_nodes": 52915, "missing_nodes": 205554, "perfect_with_extra": 113, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 103509, "value_mismatch": 240205, "dependency_mismatch": 308353, "answer_mismatch": 191084, "extra_nodes": 20859, "missing_nodes": 72860, "perfect_with_extra": 13}, "movie_festival_awards": {"total": 418816, "value_mismatch": 237754, "dependency_mismatch": 299810, "answer_mismatch": 189423, "perfect": 101060, "missing_nodes": 67744, "extra_nodes": 16825, "perfect_with_extra": 2}, "teachers_in_school": {"total": 430464, "perfect": 106738, "dependency_mismatch": 304596, "value_mismatch": 237987, "answer_mismatch": 189731, "missing_nodes": 64950, "extra_nodes": 15231, "perfect_with_extra": 98}}, "results_path": "results/composition/Qwen3-100M-epoch1-RL-op11-14/composition/graphs/hf_model_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1317319, "value_mismatch": 741538, "dependency_mismatch": 1056141, "answer_mismatch": 592352, "extra_nodes": 69697, "missing_nodes": 214704, "perfect_with_extra": 968, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/Qwen3-100M-epoch1-RL-op11-14/composition/graphs/composition_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 434847, "dependency_mismatch": 355221, "value_mismatch": 249120, "answer_mismatch": 198680, "extra_nodes": 26223, "missing_nodes": 75746, "perfect_with_extra": 291}, "teachers_in_school": {"total": 821120, "perfect": 448507, "dependency_mismatch": 352486, "value_mismatch": 246180, "answer_mismatch": 196963, "extra_nodes": 20446, "missing_nodes": 68248, "perfect_with_extra": 207}, "movie_festival_awards": {"total": 801152, "perfect": 433965, "dependency_mismatch": 348434, "value_mismatch": 246238, "answer_mismatch": 196709, "extra_nodes": 23028, "missing_nodes": 70710, "perfect_with_extra": 470}}}} diff --git a/composition/Qwen3-100M-epoch1-RL-op11-14/composition/graphs/results.json b/composition/Qwen3-100M-epoch1-RL-op11-14/composition/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..7f11d18b37cbf34a55c95a12ae867b259b0a84f3 --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op11-14/composition/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "hf_model_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990625, + "pass@2": 0.990625, + "pass@4": 0.99375, + "pass@8": 0.996875, + "pass@16": 0.996875, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9682539682539683, + "pass@2": 0.9809523809523809, + "pass@4": 0.9904761904761905, + "pass@8": 0.9968253968253968, + "pass@16": 0.9968253968253968, + "pass@32": 0.9968253968253968, + "pass@64": 0.9968253968253968, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9700598802395209, + "pass@2": 0.9820359281437125, + "pass@4": 0.9910179640718563, + "pass@8": 0.9970059880239521, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8846153846153846, + "pass@2": 0.9408284023668639, + "pass@4": 0.9704142011834319, + "pass@8": 0.985207100591716, + "pass@16": 0.9940828402366864, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.7788461538461539, + "pass@2": 0.8525641025641025, + "pass@4": 0.9262820512820513, + "pass@8": 0.9775641025641025, + "pass@16": 0.9871794871794872, + "pass@32": 0.9871794871794872, + "pass@64": 0.9935897435897436, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.704225352112676, + "pass@2": 0.7971830985915493, + "pass@4": 0.8901408450704226, + "pass@8": 0.9408450704225352, + "pass@16": 0.9746478873239437, + "pass@32": 0.9915492957746479, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "10": { + "pass@1": 0.5929203539823009, + "pass@2": 0.6873156342182891, + "pass@4": 0.7669616519174042, + "pass@8": 0.8407079646017699, + "pass@16": 0.8820058997050148, + "pass@32": 0.8938053097345132, + "pass@64": 0.9056047197640118, + "pass@128": 0.9085545722713865 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9682539682539683, + "6": 0.9700598802395209, + "7": 0.8846153846153846, + "8": 0.7788461538461539, + "9": 0.704225352112676, + "10": 0.5929203539823009 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9809523809523809, + "6": 0.9820359281437125, + "7": 0.9408284023668639, + "8": 0.8525641025641025, + "9": 0.7971830985915493, + "10": 0.6873156342182891 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9904761904761905, + "6": 0.9910179640718563, + "7": 0.9704142011834319, + "8": 0.9262820512820513, + "9": 0.8901408450704226, + "10": 0.7669616519174042 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9968253968253968, + "6": 0.9970059880239521, + "7": 0.985207100591716, + "8": 0.9775641025641025, + "9": 0.9408450704225352, + "10": 0.8407079646017699 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9940828402366864, + "8": 0.9871794871794872, + "9": 0.9746478873239437, + "10": 0.8820058997050148 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9871794871794872, + "9": 0.9915492957746479, + "10": 0.8938053097345132 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9935897435897436, + "9": 0.9971830985915493, + "10": 0.9056047197640118 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9935897435897436, + "9": 0.9971830985915493, + "10": 0.9085545722713865 + } + }, + "total_pass_at_k": { + "pass@1": 0.8736913204998311, + "pass@2": 0.9125295508274232, + "pass@4": 0.9463019250253293, + "pass@8": 0.9696048632218845, + "pass@16": 0.9807497467071935, + "pass@32": 0.9848024316109423, + "pass@64": 0.9875042215467748, + "pass@128": 0.9878419452887538 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9941520467836257, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9411764705882353, + "pass@2": 0.9647058823529412, + "pass@4": 0.9794117647058823, + "pass@8": 0.9882352941176471, + "pass@16": 0.9941176470588236, + "pass@32": 0.9970588235294118, + "pass@64": 0.9970588235294118, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9558359621451105, + "pass@2": 0.9810725552050473, + "pass@4": 0.9873817034700315, + "pass@8": 0.9936908517350158, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8354430379746836, + "pass@2": 0.8860759493670886, + "pass@4": 0.9367088607594937, + "pass@8": 0.9683544303797469, + "pass@16": 0.990506329113924, + "pass@32": 0.9936708860759493, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.8078078078078078, + "pass@2": 0.8708708708708709, + "pass@4": 0.9369369369369369, + "pass@8": 0.96996996996997, + "pass@16": 0.987987987987988, + "pass@32": 0.993993993993994, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.7138263665594855, + "pass@2": 0.7877813504823151, + "pass@4": 0.8713826366559485, + "pass@8": 0.9453376205787781, + "pass@16": 0.9742765273311897, + "pass@32": 0.9871382636655949, + "pass@64": 0.9903536977491961, + "pass@128": 0.9935691318327974 + }, + "10": { + "pass@1": 0.5492537313432836, + "pass@2": 0.6298507462686567, + "pass@4": 0.7194029850746269, + "pass@8": 0.7910447761194029, + "pass@16": 0.8507462686567164, + "pass@32": 0.8776119402985074, + "pass@64": 0.8835820895522388, + "pass@128": 0.8925373134328358 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9411764705882353, + "6": 0.9558359621451105, + "7": 0.8354430379746836, + "8": 0.8078078078078078, + "9": 0.7138263665594855, + "10": 0.5492537313432836 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9647058823529412, + "6": 0.9810725552050473, + "7": 0.8860759493670886, + "8": 0.8708708708708709, + "9": 0.7877813504823151, + "10": 0.6298507462686567 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9794117647058823, + "6": 0.9873817034700315, + "7": 0.9367088607594937, + "8": 0.9369369369369369, + "9": 0.8713826366559485, + "10": 0.7194029850746269 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9882352941176471, + "6": 0.9936908517350158, + "7": 0.9683544303797469, + "8": 0.96996996996997, + "9": 0.9453376205787781, + "10": 0.7910447761194029 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9941176470588236, + "6": 1.0, + "7": 0.990506329113924, + "8": 0.987987987987988, + "9": 0.9742765273311897, + "10": 0.8507462686567164 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.993993993993994, + "9": 0.9871382636655949, + "10": 0.8776119402985074 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.996996996996997, + "9": 0.9903536977491961, + "10": 0.8835820895522388 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.996996996996997, + "9": 0.9935691318327974, + "10": 0.8925373134328358 + } + }, + "total_pass_at_k": { + "pass@1": 0.8687646468028122, + "pass@2": 0.9039169735520589, + "pass@4": 0.9377301640441915, + "pass@8": 0.9621694007365249, + "pass@16": 0.9775694676933377, + "pass@32": 0.983260796786073, + "pass@64": 0.9845999330431872, + "pass@128": 0.9862738533645798 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985207100591716, + "pass@2": 0.9911242603550295, + "pass@4": 0.9970414201183432, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9449275362318841, + "pass@2": 0.9768115942028985, + "pass@4": 0.991304347826087, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9742120343839542, + "pass@2": 0.9856733524355301, + "pass@4": 0.9914040114613181, + "pass@8": 0.997134670487106, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8641618497109826, + "pass@2": 0.9075144508670521, + "pass@4": 0.953757225433526, + "pass@8": 0.9797687861271677, + "pass@16": 0.9942196531791907, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7887323943661971, + "pass@2": 0.8873239436619719, + "pass@4": 0.9464788732394366, + "pass@8": 0.9661971830985916, + "pass@16": 0.9915492957746479, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.6976047904191617, + "pass@2": 0.7664670658682635, + "pass@4": 0.8353293413173652, + "pass@8": 0.9041916167664671, + "pass@16": 0.9700598802395209, + "pass@32": 0.9880239520958084, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "10": { + "pass@1": 0.6134969325153374, + "pass@2": 0.6932515337423313, + "pass@4": 0.7576687116564417, + "pass@8": 0.803680981595092, + "pass@16": 0.8466257668711656, + "pass@32": 0.8588957055214724, + "pass@64": 0.8773006134969326, + "pass@128": 0.8773006134969326 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.985207100591716, + "5": 0.9449275362318841, + "6": 0.9742120343839542, + "7": 0.8641618497109826, + "8": 0.7887323943661971, + "9": 0.6976047904191617, + "10": 0.6134969325153374 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9911242603550295, + "5": 0.9768115942028985, + "6": 0.9856733524355301, + "7": 0.9075144508670521, + "8": 0.8873239436619719, + "9": 0.7664670658682635, + "10": 0.6932515337423313 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.991304347826087, + "6": 0.9914040114613181, + "7": 0.953757225433526, + "8": 0.9464788732394366, + "9": 0.8353293413173652, + "10": 0.7576687116564417 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.997134670487106, + "7": 0.9797687861271677, + "8": 0.9661971830985916, + "9": 0.9041916167664671, + "10": 0.803680981595092 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.997134670487106, + "7": 0.9942196531791907, + "8": 0.9915492957746479, + "9": 0.9700598802395209, + "10": 0.8466257668711656 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 1.0, + "8": 1.0, + "9": 0.9880239520958084, + "10": 0.8588957055214724 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 1.0, + "8": 1.0, + "9": 0.9940119760479041, + "10": 0.8773006134969326 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9940119760479041, + "10": 0.8773006134969326 + } + }, + "total_pass_at_k": { + "pass@1": 0.8748361730013107, + "pass@2": 0.9128440366972477, + "pass@4": 0.9423328964613368, + "pass@8": 0.9616644823066841, + "pass@16": 0.9780471821756226, + "pass@32": 0.9832896461336829, + "pass@64": 0.9859108781127129, + "pass@128": 0.9862385321100917 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99, + "pass@2": 0.994, + "pass@4": 0.997, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.951, + "pass@2": 0.974, + "pass@4": 0.987, + "pass@8": 0.994, + "pass@16": 0.996, + "pass@32": 0.998, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.967, + "pass@2": 0.983, + "pass@4": 0.99, + "pass@8": 0.996, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.862, + "pass@2": 0.912, + "pass@4": 0.954, + "pass@8": 0.978, + "pass@16": 0.993, + "pass@32": 0.997, + "pass@64": 0.997, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.792, + "pass@2": 0.871, + "pass@4": 0.937, + "pass@8": 0.971, + "pass@16": 0.989, + "pass@32": 0.994, + "pass@64": 0.997, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.705, + "pass@2": 0.784, + "pass@4": 0.866, + "pass@8": 0.93, + "pass@16": 0.973, + "pass@32": 0.989, + "pass@64": 0.994, + "pass@128": 0.995 + }, + "10": { + "pass@1": 0.585, + "pass@2": 0.67, + "pass@4": 0.748, + "pass@8": 0.812, + "pass@16": 0.86, + "pass@32": 0.877, + "pass@64": 0.889, + "pass@128": 0.893 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99, + "5": 0.951, + "6": 0.967, + "7": 0.862, + "8": 0.792, + "9": 0.705, + "10": 0.585 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.994, + "5": 0.974, + "6": 0.983, + "7": 0.912, + "8": 0.871, + "9": 0.784, + "10": 0.67 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.997, + "5": 0.987, + "6": 0.99, + "7": 0.954, + "8": 0.937, + "9": 0.866, + "10": 0.748 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.994, + "6": 0.996, + "7": 0.978, + "8": 0.971, + "9": 0.93, + "10": 0.812 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.996, + "6": 0.999, + "7": 0.993, + "8": 0.989, + "9": 0.973, + "10": 0.86 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 0.999, + "7": 0.997, + "8": 0.994, + "9": 0.989, + "10": 0.877 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 0.999, + "7": 0.997, + "8": 0.997, + "9": 0.994, + "10": 0.889 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.997, + "8": 0.997, + "9": 0.995, + "10": 0.893 + } + }, + "total_pass_at_k": { + "pass@1": 0.8724444444444445, + "pass@2": 0.9097777777777778, + "pass@4": 0.9421111111111111, + "pass@8": 0.9644444444444444, + "pass@16": 0.9787777777777777, + "pass@32": 0.9837777777777778, + "pass@64": 0.986, + "pass@128": 0.9867777777777778 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "hf_model_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5184135977337111, + "pass@2": 0.6118980169971672, + "pass@4": 0.6855524079320113, + "pass@8": 0.8045325779036827, + "pass@16": 0.8781869688385269, + "pass@32": 0.9065155807365439, + "pass@64": 0.9178470254957507, + "pass@128": 0.9263456090651558 + }, + "12": { + "pass@1": 0.556497175141243, + "pass@2": 0.6440677966101694, + "pass@4": 0.7062146892655368, + "pass@8": 0.7655367231638418, + "pass@16": 0.8135593220338984, + "pass@32": 0.8305084745762712, + "pass@64": 0.8502824858757062, + "pass@128": 0.864406779661017 + }, + "13": { + "pass@1": 0.42946708463949845, + "pass@2": 0.5047021943573667, + "pass@4": 0.5768025078369906, + "pass@8": 0.6300940438871473, + "pass@16": 0.6927899686520376, + "pass@32": 0.7335423197492164, + "pass@64": 0.774294670846395, + "pass@128": 0.8056426332288401 + }, + "14": { + "pass@1": 0.3146417445482866, + "pass@2": 0.3956386292834891, + "pass@4": 0.4735202492211838, + "pass@8": 0.5482866043613707, + "pass@16": 0.5981308411214953, + "pass@32": 0.6542056074766355, + "pass@64": 0.7071651090342679, + "pass@128": 0.7570093457943925 + }, + "15": { + "pass@1": 0.24566473988439305, + "pass@2": 0.27167630057803466, + "pass@4": 0.3352601156069364, + "pass@8": 0.3988439306358382, + "pass@16": 0.47398843930635837, + "pass@32": 0.523121387283237, + "pass@64": 0.6127167630057804, + "pass@128": 0.6676300578034682 + }, + "16": { + "pass@1": 0.16451612903225807, + "pass@2": 0.21935483870967742, + "pass@4": 0.2645161290322581, + "pass@8": 0.3064516129032258, + "pass@16": 0.3709677419354839, + "pass@32": 0.42258064516129035, + "pass@64": 0.4806451612903226, + "pass@128": 0.5225806451612903 + }, + "17": { + "pass@1": 0.11271676300578035, + "pass@2": 0.17052023121387283, + "pass@4": 0.1907514450867052, + "pass@8": 0.23121387283236994, + "pass@16": 0.2832369942196532, + "pass@32": 0.3063583815028902, + "pass@64": 0.3583815028901734, + "pass@128": 0.41040462427745666 + }, + "18": { + "pass@1": 0.06857142857142857, + "pass@2": 0.09714285714285714, + "pass@4": 0.11428571428571428, + "pass@8": 0.15428571428571428, + "pass@16": 0.17142857142857143, + "pass@32": 0.19714285714285715, + "pass@64": 0.24285714285714285, + "pass@128": 0.29714285714285715 + }, + "19": { + "pass@1": 0.02702702702702703, + "pass@2": 0.036036036036036036, + "pass@4": 0.05405405405405406, + "pass@8": 0.06606606606606606, + "pass@16": 0.0960960960960961, + "pass@32": 0.11711711711711711, + "pass@64": 0.14714714714714713, + "pass@128": 0.17117117117117117 + }, + "20": { + "pass@1": 0.02702702702702703, + "pass@2": 0.03303303303303303, + "pass@4": 0.04504504504504504, + "pass@8": 0.06306306306306306, + "pass@16": 0.09309309309309309, + "pass@32": 0.11711711711711711, + "pass@64": 0.13513513513513514, + "pass@128": 0.16816816816816818 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5184135977337111, + "12": 0.556497175141243, + "13": 0.42946708463949845, + "14": 0.3146417445482866, + "15": 0.24566473988439305, + "16": 0.16451612903225807, + "17": 0.11271676300578035, + "18": 0.06857142857142857, + "19": 0.02702702702702703, + "20": 0.02702702702702703 + }, + "pass@2": { + "11": 0.6118980169971672, + "12": 0.6440677966101694, + "13": 0.5047021943573667, + "14": 0.3956386292834891, + "15": 0.27167630057803466, + "16": 0.21935483870967742, + "17": 0.17052023121387283, + "18": 0.09714285714285714, + "19": 0.036036036036036036, + "20": 0.03303303303303303 + }, + "pass@4": { + "11": 0.6855524079320113, + "12": 0.7062146892655368, + "13": 0.5768025078369906, + "14": 0.4735202492211838, + "15": 0.3352601156069364, + "16": 0.2645161290322581, + "17": 0.1907514450867052, + "18": 0.11428571428571428, + "19": 0.05405405405405406, + "20": 0.04504504504504504 + }, + "pass@8": { + "11": 0.8045325779036827, + "12": 0.7655367231638418, + "13": 0.6300940438871473, + "14": 0.5482866043613707, + "15": 0.3988439306358382, + "16": 0.3064516129032258, + "17": 0.23121387283236994, + "18": 0.15428571428571428, + "19": 0.06606606606606606, + "20": 0.06306306306306306 + }, + "pass@16": { + "11": 0.8781869688385269, + "12": 0.8135593220338984, + "13": 0.6927899686520376, + "14": 0.5981308411214953, + "15": 0.47398843930635837, + "16": 0.3709677419354839, + "17": 0.2832369942196532, + "18": 0.17142857142857143, + "19": 0.0960960960960961, + "20": 0.09309309309309309 + }, + "pass@32": { + "11": 0.9065155807365439, + "12": 0.8305084745762712, + "13": 0.7335423197492164, + "14": 0.6542056074766355, + "15": 0.523121387283237, + "16": 0.42258064516129035, + "17": 0.3063583815028902, + "18": 0.19714285714285715, + "19": 0.11711711711711711, + "20": 0.11711711711711711 + }, + "pass@64": { + "11": 0.9178470254957507, + "12": 0.8502824858757062, + "13": 0.774294670846395, + "14": 0.7071651090342679, + "15": 0.6127167630057804, + "16": 0.4806451612903226, + "17": 0.3583815028901734, + "18": 0.24285714285714285, + "19": 0.14714714714714713, + "20": 0.13513513513513514 + }, + "pass@128": { + "11": 0.9263456090651558, + "12": 0.864406779661017, + "13": 0.8056426332288401, + "14": 0.7570093457943925, + "15": 0.6676300578034682, + "16": 0.5225806451612903, + "17": 0.41040462427745666, + "18": 0.29714285714285715, + "19": 0.17117117117117117, + "20": 0.16816816816816818 + } + }, + "total_pass_at_k": { + "pass@1": 0.24814264487369986, + "pass@2": 0.300148588410104, + "pass@4": 0.3462109955423477, + "pass@8": 0.39881129271916793, + "pass@16": 0.44903417533432394, + "pass@32": 0.4823179791976226, + "pass@64": 0.5239227340267459, + "pass@128": 0.5601783060921248 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5047619047619047, + "pass@2": 0.6285714285714286, + "pass@4": 0.7333333333333333, + "pass@8": 0.8031746031746032, + "pass@16": 0.873015873015873, + "pass@32": 0.9238095238095239, + "pass@64": 0.9396825396825397, + "pass@128": 0.9523809523809523 + }, + "12": { + "pass@1": 0.5321100917431193, + "pass@2": 0.6238532110091743, + "pass@4": 0.7186544342507645, + "pass@8": 0.7767584097859327, + "pass@16": 0.8256880733944955, + "pass@32": 0.8501529051987767, + "pass@64": 0.8623853211009175, + "pass@128": 0.8776758409785933 + }, + "13": { + "pass@1": 0.37962962962962965, + "pass@2": 0.46296296296296297, + "pass@4": 0.5462962962962963, + "pass@8": 0.6265432098765432, + "pass@16": 0.7006172839506173, + "pass@32": 0.7283950617283951, + "pass@64": 0.7530864197530864, + "pass@128": 0.7746913580246914 + }, + "14": { + "pass@1": 0.27002967359050445, + "pass@2": 0.35311572700296734, + "pass@4": 0.42433234421364985, + "pass@8": 0.47181008902077154, + "pass@16": 0.543026706231454, + "pass@32": 0.6172106824925816, + "pass@64": 0.6795252225519288, + "pass@128": 0.7299703264094956 + }, + "15": { + "pass@1": 0.24691358024691357, + "pass@2": 0.2993827160493827, + "pass@4": 0.3487654320987654, + "pass@8": 0.3950617283950617, + "pass@16": 0.4537037037037037, + "pass@32": 0.5123456790123457, + "pass@64": 0.595679012345679, + "pass@128": 0.6697530864197531 + }, + "16": { + "pass@1": 0.19174041297935104, + "pass@2": 0.26843657817109146, + "pass@4": 0.3215339233038348, + "pass@8": 0.3775811209439528, + "pass@16": 0.415929203539823, + "pass@32": 0.4631268436578171, + "pass@64": 0.504424778761062, + "pass@128": 0.5486725663716814 + }, + "17": { + "pass@1": 0.10542168674698796, + "pass@2": 0.14457831325301204, + "pass@4": 0.1686746987951807, + "pass@8": 0.20783132530120482, + "pass@16": 0.25301204819277107, + "pass@32": 0.286144578313253, + "pass@64": 0.3373493975903614, + "pass@128": 0.4036144578313253 + }, + "18": { + "pass@1": 0.08064516129032258, + "pass@2": 0.1064516129032258, + "pass@4": 0.13870967741935483, + "pass@8": 0.16451612903225807, + "pass@16": 0.1870967741935484, + "pass@32": 0.2129032258064516, + "pass@64": 0.23548387096774193, + "pass@128": 0.2709677419354839 + }, + "19": { + "pass@1": 0.03323262839879154, + "pass@2": 0.045317220543806644, + "pass@4": 0.05740181268882175, + "pass@8": 0.06948640483383686, + "pass@16": 0.09063444108761329, + "pass@32": 0.09667673716012085, + "pass@64": 0.1268882175226586, + "pass@128": 0.1540785498489426 + }, + "20": { + "pass@1": 0.03303303303303303, + "pass@2": 0.036036036036036036, + "pass@4": 0.04504504504504504, + "pass@8": 0.06606606606606606, + "pass@16": 0.07507507507507508, + "pass@32": 0.09309309309309309, + "pass@64": 0.10810810810810811, + "pass@128": 0.12312312312312312 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5047619047619047, + "12": 0.5321100917431193, + "13": 0.37962962962962965, + "14": 0.27002967359050445, + "15": 0.24691358024691357, + "16": 0.19174041297935104, + "17": 0.10542168674698796, + "18": 0.08064516129032258, + "19": 0.03323262839879154, + "20": 0.03303303303303303 + }, + "pass@2": { + "11": 0.6285714285714286, + "12": 0.6238532110091743, + "13": 0.46296296296296297, + "14": 0.35311572700296734, + "15": 0.2993827160493827, + "16": 0.26843657817109146, + "17": 0.14457831325301204, + "18": 0.1064516129032258, + "19": 0.045317220543806644, + "20": 0.036036036036036036 + }, + "pass@4": { + "11": 0.7333333333333333, + "12": 0.7186544342507645, + "13": 0.5462962962962963, + "14": 0.42433234421364985, + "15": 0.3487654320987654, + "16": 0.3215339233038348, + "17": 0.1686746987951807, + "18": 0.13870967741935483, + "19": 0.05740181268882175, + "20": 0.04504504504504504 + }, + "pass@8": { + "11": 0.8031746031746032, + "12": 0.7767584097859327, + "13": 0.6265432098765432, + "14": 0.47181008902077154, + "15": 0.3950617283950617, + "16": 0.3775811209439528, + "17": 0.20783132530120482, + "18": 0.16451612903225807, + "19": 0.06948640483383686, + "20": 0.06606606606606606 + }, + "pass@16": { + "11": 0.873015873015873, + "12": 0.8256880733944955, + "13": 0.7006172839506173, + "14": 0.543026706231454, + "15": 0.4537037037037037, + "16": 0.415929203539823, + "17": 0.25301204819277107, + "18": 0.1870967741935484, + "19": 0.09063444108761329, + "20": 0.07507507507507508 + }, + "pass@32": { + "11": 0.9238095238095239, + "12": 0.8501529051987767, + "13": 0.7283950617283951, + "14": 0.6172106824925816, + "15": 0.5123456790123457, + "16": 0.4631268436578171, + "17": 0.286144578313253, + "18": 0.2129032258064516, + "19": 0.09667673716012085, + "20": 0.09309309309309309 + }, + "pass@64": { + "11": 0.9396825396825397, + "12": 0.8623853211009175, + "13": 0.7530864197530864, + "14": 0.6795252225519288, + "15": 0.595679012345679, + "16": 0.504424778761062, + "17": 0.3373493975903614, + "18": 0.23548387096774193, + "19": 0.1268882175226586, + "20": 0.10810810810810811 + }, + "pass@128": { + "11": 0.9523809523809523, + "12": 0.8776758409785933, + "13": 0.7746913580246914, + "14": 0.7299703264094956, + "15": 0.6697530864197531, + "16": 0.5486725663716814, + "17": 0.4036144578313253, + "18": 0.2709677419354839, + "19": 0.1540785498489426, + "20": 0.12312312312312312 + } + }, + "total_pass_at_k": { + "pass@1": 0.2365525672371638, + "pass@2": 0.29553789731051344, + "pass@4": 0.3487163814180929, + "pass@8": 0.39425427872860636, + "pass@16": 0.4400977995110024, + "pass@32": 0.4767726161369193, + "pass@64": 0.5128361858190709, + "pass@128": 0.5492053789731052 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6054216867469879, + "pass@2": 0.6506024096385542, + "pass@4": 0.7108433734939759, + "pass@8": 0.8042168674698795, + "pass@16": 0.8524096385542169, + "pass@32": 0.9006024096385542, + "pass@64": 0.927710843373494, + "pass@128": 0.9578313253012049 + }, + "12": { + "pass@1": 0.5078369905956113, + "pass@2": 0.5799373040752351, + "pass@4": 0.6426332288401254, + "pass@8": 0.7366771159874608, + "pass@16": 0.780564263322884, + "pass@32": 0.8087774294670846, + "pass@64": 0.8401253918495298, + "pass@128": 0.8557993730407524 + }, + "13": { + "pass@1": 0.42296918767507, + "pass@2": 0.5126050420168067, + "pass@4": 0.6078431372549019, + "pass@8": 0.6974789915966386, + "pass@16": 0.7366946778711485, + "pass@32": 0.7759103641456583, + "pass@64": 0.8067226890756303, + "pass@128": 0.8291316526610645 + }, + "14": { + "pass@1": 0.31871345029239767, + "pass@2": 0.3830409356725146, + "pass@4": 0.4444444444444444, + "pass@8": 0.5146198830409356, + "pass@16": 0.5818713450292398, + "pass@32": 0.6432748538011696, + "pass@64": 0.7017543859649122, + "pass@128": 0.7485380116959064 + }, + "15": { + "pass@1": 0.21515151515151515, + "pass@2": 0.296969696969697, + "pass@4": 0.34545454545454546, + "pass@8": 0.4121212121212121, + "pass@16": 0.47575757575757577, + "pass@32": 0.5454545454545454, + "pass@64": 0.6393939393939394, + "pass@128": 0.6909090909090909 + }, + "16": { + "pass@1": 0.19658119658119658, + "pass@2": 0.24786324786324787, + "pass@4": 0.2905982905982906, + "pass@8": 0.3475783475783476, + "pass@16": 0.396011396011396, + "pass@32": 0.4444444444444444, + "pass@64": 0.49572649572649574, + "pass@128": 0.5612535612535613 + }, + "17": { + "pass@1": 0.09316770186335403, + "pass@2": 0.14596273291925466, + "pass@4": 0.20496894409937888, + "pass@8": 0.2515527950310559, + "pass@16": 0.2857142857142857, + "pass@32": 0.3167701863354037, + "pass@64": 0.37577639751552794, + "pass@128": 0.4254658385093168 + }, + "18": { + "pass@1": 0.1, + "pass@2": 0.13529411764705881, + "pass@4": 0.15588235294117647, + "pass@8": 0.18235294117647058, + "pass@16": 0.2323529411764706, + "pass@32": 0.24705882352941178, + "pass@64": 0.28823529411764703, + "pass@128": 0.3323529411764706 + }, + "19": { + "pass@1": 0.01488095238095238, + "pass@2": 0.03273809523809524, + "pass@4": 0.041666666666666664, + "pass@8": 0.05357142857142857, + "pass@16": 0.08928571428571429, + "pass@32": 0.1130952380952381, + "pass@64": 0.15476190476190477, + "pass@128": 0.18154761904761904 + }, + "20": { + "pass@1": 0.029940119760479042, + "pass@2": 0.038922155688622756, + "pass@4": 0.05089820359281437, + "pass@8": 0.0748502994011976, + "pass@16": 0.08383233532934131, + "pass@32": 0.10179640718562874, + "pass@64": 0.12574850299401197, + "pass@128": 0.1437125748502994 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6054216867469879, + "12": 0.5078369905956113, + "13": 0.42296918767507, + "14": 0.31871345029239767, + "15": 0.21515151515151515, + "16": 0.19658119658119658, + "17": 0.09316770186335403, + "18": 0.1, + "19": 0.01488095238095238, + "20": 0.029940119760479042 + }, + "pass@2": { + "11": 0.6506024096385542, + "12": 0.5799373040752351, + "13": 0.5126050420168067, + "14": 0.3830409356725146, + "15": 0.296969696969697, + "16": 0.24786324786324787, + "17": 0.14596273291925466, + "18": 0.13529411764705881, + "19": 0.03273809523809524, + "20": 0.038922155688622756 + }, + "pass@4": { + "11": 0.7108433734939759, + "12": 0.6426332288401254, + "13": 0.6078431372549019, + "14": 0.4444444444444444, + "15": 0.34545454545454546, + "16": 0.2905982905982906, + "17": 0.20496894409937888, + "18": 0.15588235294117647, + "19": 0.041666666666666664, + "20": 0.05089820359281437 + }, + "pass@8": { + "11": 0.8042168674698795, + "12": 0.7366771159874608, + "13": 0.6974789915966386, + "14": 0.5146198830409356, + "15": 0.4121212121212121, + "16": 0.3475783475783476, + "17": 0.2515527950310559, + "18": 0.18235294117647058, + "19": 0.05357142857142857, + "20": 0.0748502994011976 + }, + "pass@16": { + "11": 0.8524096385542169, + "12": 0.780564263322884, + "13": 0.7366946778711485, + "14": 0.5818713450292398, + "15": 0.47575757575757577, + "16": 0.396011396011396, + "17": 0.2857142857142857, + "18": 0.2323529411764706, + "19": 0.08928571428571429, + "20": 0.08383233532934131 + }, + "pass@32": { + "11": 0.9006024096385542, + "12": 0.8087774294670846, + "13": 0.7759103641456583, + "14": 0.6432748538011696, + "15": 0.5454545454545454, + "16": 0.4444444444444444, + "17": 0.3167701863354037, + "18": 0.24705882352941178, + "19": 0.1130952380952381, + "20": 0.10179640718562874 + }, + "pass@64": { + "11": 0.927710843373494, + "12": 0.8401253918495298, + "13": 0.8067226890756303, + "14": 0.7017543859649122, + "15": 0.6393939393939394, + "16": 0.49572649572649574, + "17": 0.37577639751552794, + "18": 0.28823529411764703, + "19": 0.15476190476190477, + "20": 0.12574850299401197 + }, + "pass@128": { + "11": 0.9578313253012049, + "12": 0.8557993730407524, + "13": 0.8291316526610645, + "14": 0.7485380116959064, + "15": 0.6909090909090909, + "16": 0.5612535612535613, + "17": 0.4254658385093168, + "18": 0.3323529411764706, + "19": 0.18154761904761904, + "20": 0.1437125748502994 + } + }, + "total_pass_at_k": { + "pass@1": 0.2503716919417187, + "pass@2": 0.3024085637823372, + "pass@4": 0.3496877787689563, + "pass@8": 0.40767172167707405, + "pass@16": 0.45168004757656854, + "pass@32": 0.49003865596193874, + "pass@64": 0.535831103181683, + "pass@128": 0.5730002973535534 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.543, + "pass@2": 0.63, + "pass@4": 0.709, + "pass@8": 0.804, + "pass@16": 0.868, + "pass@32": 0.91, + "pass@64": 0.928, + "pass@128": 0.945 + }, + "12": { + "pass@1": 0.533, + "pass@2": 0.617, + "pass@4": 0.69, + "pass@8": 0.76, + "pass@16": 0.807, + "pass@32": 0.83, + "pass@64": 0.851, + "pass@128": 0.866 + }, + "13": { + "pass@1": 0.411, + "pass@2": 0.494, + "pass@4": 0.578, + "pass@8": 0.653, + "pass@16": 0.711, + "pass@32": 0.747, + "pass@64": 0.779, + "pass@128": 0.804 + }, + "14": { + "pass@1": 0.301, + "pass@2": 0.377, + "pass@4": 0.447, + "pass@8": 0.511, + "pass@16": 0.574, + "pass@32": 0.638, + "pass@64": 0.696, + "pass@128": 0.745 + }, + "15": { + "pass@1": 0.236, + "pass@2": 0.289, + "pass@4": 0.343, + "pass@8": 0.402, + "pass@16": 0.468, + "pass@32": 0.527, + "pass@64": 0.616, + "pass@128": 0.676 + }, + "16": { + "pass@1": 0.185, + "pass@2": 0.246, + "pass@4": 0.293, + "pass@8": 0.345, + "pass@16": 0.395, + "pass@32": 0.444, + "pass@64": 0.494, + "pass@128": 0.545 + }, + "17": { + "pass@1": 0.104, + "pass@2": 0.154, + "pass@4": 0.188, + "pass@8": 0.23, + "pass@16": 0.274, + "pass@32": 0.303, + "pass@64": 0.357, + "pass@128": 0.413 + }, + "18": { + "pass@1": 0.083, + "pass@2": 0.113, + "pass@4": 0.136, + "pass@8": 0.167, + "pass@16": 0.197, + "pass@32": 0.219, + "pass@64": 0.256, + "pass@128": 0.301 + }, + "19": { + "pass@1": 0.025, + "pass@2": 0.038, + "pass@4": 0.051, + "pass@8": 0.063, + "pass@16": 0.092, + "pass@32": 0.109, + "pass@64": 0.143, + "pass@128": 0.169 + }, + "20": { + "pass@1": 0.03, + "pass@2": 0.036, + "pass@4": 0.047, + "pass@8": 0.068, + "pass@16": 0.084, + "pass@32": 0.104, + "pass@64": 0.123, + "pass@128": 0.145 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.543, + "12": 0.533, + "13": 0.411, + "14": 0.301, + "15": 0.236, + "16": 0.185, + "17": 0.104, + "18": 0.083, + "19": 0.025, + "20": 0.03 + }, + "pass@2": { + "11": 0.63, + "12": 0.617, + "13": 0.494, + "14": 0.377, + "15": 0.289, + "16": 0.246, + "17": 0.154, + "18": 0.113, + "19": 0.038, + "20": 0.036 + }, + "pass@4": { + "11": 0.709, + "12": 0.69, + "13": 0.578, + "14": 0.447, + "15": 0.343, + "16": 0.293, + "17": 0.188, + "18": 0.136, + "19": 0.051, + "20": 0.047 + }, + "pass@8": { + "11": 0.804, + "12": 0.76, + "13": 0.653, + "14": 0.511, + "15": 0.402, + "16": 0.345, + "17": 0.23, + "18": 0.167, + "19": 0.063, + "20": 0.068 + }, + "pass@16": { + "11": 0.868, + "12": 0.807, + "13": 0.711, + "14": 0.574, + "15": 0.468, + "16": 0.395, + "17": 0.274, + "18": 0.197, + "19": 0.092, + "20": 0.084 + }, + "pass@32": { + "11": 0.91, + "12": 0.83, + "13": 0.747, + "14": 0.638, + "15": 0.527, + "16": 0.444, + "17": 0.303, + "18": 0.219, + "19": 0.109, + "20": 0.104 + }, + "pass@64": { + "11": 0.928, + "12": 0.851, + "13": 0.779, + "14": 0.696, + "15": 0.616, + "16": 0.494, + "17": 0.357, + "18": 0.256, + "19": 0.143, + "20": 0.123 + }, + "pass@128": { + "11": 0.945, + "12": 0.866, + "13": 0.804, + "14": 0.745, + "15": 0.676, + "16": 0.545, + "17": 0.413, + "18": 0.301, + "19": 0.169, + "20": 0.145 + } + }, + "total_pass_at_k": { + "pass@1": 0.2451, + "pass@2": 0.2994, + "pass@4": 0.3482, + "pass@8": 0.4003, + "pass@16": 0.447, + "pass@32": 0.4831, + "pass@64": 0.5243, + "pass@128": 0.5609 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/Qwen3-100M-epoch1-RL-op11-14/composition/hf_model_metrics.json b/composition/Qwen3-100M-epoch1-RL-op11-14/composition/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..71edac3c4f584378d7537e860521c668e24b3f0a --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op11-14/composition/hf_model_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.9808038194444444, + "avg_response_len": 198.925859375, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9808038194444444, + "pass@2": 0.9868979658792594, + "pass@4": 0.9900498776923711, + "pass@8": 0.9920821935527284, + "pass@16": 0.9934504581534681, + "pass@32": 0.9943820118674019, + "pass@64": 0.9950371407354888, + "pass@128": 0.9955555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.918921875, + "pass@2": 0.9345583169291337, + "pass@4": 0.943269286276715, + "pass@8": 0.9493995987750118, + "pass@16": 0.9543911543654843, + "pass@32": 0.9587902086599787, + "pass@64": 0.9627103821823005, + "pass@128": 0.966 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999265625, + "pass@2": 0.999923474409449, + "pass@4": 0.9999989649418821, + "pass@8": 0.9999999998733826, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998375, + "pass@2": 0.9991230314960631, + "pass@4": 0.9996207719347582, + "pass@8": 0.9998968349840596, + "pass@16": 0.9999915015594042, + "pass@32": 0.9999999700068078, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.98590625, + "pass@2": 0.990609375, + "pass@4": 0.9941187244094487, + "pass@8": 0.9967705762096115, + "pass@16": 0.9986236324272478, + "pass@32": 0.9996398839845977, + "pass@64": 0.999969899907236, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9965234375, + "pass@2": 0.9980569635826774, + "pass@4": 0.9988050461192352, + "pass@8": 0.9991801968316429, + "pass@16": 0.9994162453640056, + "pass@32": 0.9996886045034639, + "pass@64": 0.9999404409448819, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9910390625, + "pass@2": 0.9944616141732285, + "pass@4": 0.9960210366516683, + "pass@8": 0.997143365187022, + "pass@16": 0.9978839411380321, + "pass@32": 0.9982329183934826, + "pass@64": 0.9984999826968121, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.9800390625, + "pass@2": 0.989688853346457, + "pass@4": 0.9935460817397821, + "pass@8": 0.9959264229266411, + "pass@16": 0.9976315460599925, + "pass@32": 0.9986283768487406, + "pass@64": 0.9989690357139468, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.9571640625, + "pass@2": 0.9756600639763789, + "pass@4": 0.9850689871578558, + "pass@8": 0.9904227471871823, + "pass@16": 0.9931161024670461, + "pass@32": 0.9944581444095479, + "pass@64": 0.9952445251742235, + "pass@128": 0.996 + } + }, + "per_op_accuracy": { + "10": 0.918921875, + "2": 1.0, + "3": 0.999265625, + "4": 0.998375, + "5": 0.98590625, + "6": 0.9965234375, + "7": 0.9910390625, + "8": 0.9800390625, + "9": 0.9571640625 + }, + "per_op_avg_response_len": { + "10": 279.6280078125, + "2": 115.096359375, + "3": 144.0779921875, + "4": 159.11890625, + "5": 173.1603515625, + "6": 197.6270703125, + "7": 222.665234375, + "8": 241.5987421875, + "9": 257.3600703125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.6280078125, + "2": 115.096359375, + "3": 144.0779921875, + "4": 159.11890625, + "5": 173.1603515625, + "6": 197.6270703125, + "7": 222.665234375, + "8": 241.5987421875, + "9": 257.3600703125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 371412, + "answer_accuracy": 0.9799582066869301, + "avg_response_len": 189.36178655859507, + "resp_tokens_sum": 71769632, + "pass_at_k": { + "pass@1": 0.9799582066869301, + "pass@2": 0.9870204472446273, + "pass@4": 0.9906200951388172, + "pass@8": 0.9927423437080808, + "pass@16": 0.9939477788475964, + "pass@32": 0.994656438886884, + "pass@64": 0.9951558728063505, + "pass@128": 0.9956095913542722 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9221054572271387, + "pass@2": 0.9400673298028014, + "pass@4": 0.949361465878712, + "pass@8": 0.9547948042979221, + "pass@16": 0.9585332135653881, + "pass@32": 0.9613178358810445, + "pass@64": 0.9636093956539674, + "pass@128": 0.9646017699115044 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9991637996941896, + "pass@2": 0.9998976618748345, + "pass@4": 0.9999986354916645, + "pass@8": 0.9999999998960027, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99658203125, + "pass@2": 0.9979638287401574, + "pass@4": 0.9989677803946382, + "pass@8": 0.999684046038967, + "pass@16": 0.9999734491626946, + "pass@32": 0.9999999062712751, + "pass@64": 0.9999999999999887, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9881696428571428, + "pass@2": 0.9919502249718786, + "pass@4": 0.9951166458359371, + "pass@8": 0.9975764494654312, + "pass@16": 0.9991626286735529, + "pass@32": 0.999880913313676, + "pass@64": 0.999999029712244, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992982784431138, + "pass@2": 0.9999756883870056, + "pass@4": 0.9999999393788351, + "pass@8": 0.9999999999999057, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9931120562130178, + "pass@2": 0.9952294355402319, + "pass@4": 0.9961295363227524, + "pass@8": 0.9969006813652405, + "pass@16": 0.9973812009579224, + "pass@32": 0.9977809333273798, + "pass@64": 0.9985207100591404, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9768880208333334, + "pass@2": 0.989932711235615, + "pass@4": 0.9950610605765626, + "pass@8": 0.9979172271283807, + "pass@16": 0.9994694376766202, + "pass@32": 0.9999552999134591, + "pass@64": 0.9999999117991033, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9490977112676057, + "pass@2": 0.9714563463457915, + "pass@4": 0.9834106024599026, + "pass@8": 0.9898494637100985, + "pass@16": 0.9928438634533845, + "pass@32": 0.994626767442593, + "pass@64": 0.9957557391894256, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 383424, + "answer_accuracy": 0.9814875491480997, + "avg_response_len": 196.8429795011468, + "resp_tokens_sum": 76897891, + "pass_at_k": { + "pass@1": 0.9814875491480997, + "pass@2": 0.9865376856920985, + "pass@4": 0.989061953449299, + "pass@8": 0.9907559452985748, + "pass@16": 0.9920464649375225, + "pass@32": 0.9930869871600271, + "pass@64": 0.9938976218619472, + "pass@128": 0.994429882044561 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9113065567484663, + "pass@2": 0.9251266545094442, + "pass@4": 0.9318315537436652, + "pass@8": 0.9359810259140621, + "pass@16": 0.9398201281022645, + "pass@32": 0.9447125150339829, + "pass@64": 0.9499553739007809, + "pass@128": 0.9539877300613497 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9988162878787878, + "pass@2": 0.9998777141493678, + "pass@4": 0.9999982255059028, + "pass@8": 0.999999999719363, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998613165680473, + "pass@2": 0.9999992720029819, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.989447463768116, + "pass@2": 0.9926609038000683, + "pass@4": 0.9946517704308702, + "pass@8": 0.9966518367905446, + "pass@16": 0.998582050313881, + "pass@32": 0.9997499573766314, + "pass@64": 0.9999971788090677, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9930157593123209, + "pass@2": 0.9951002442298582, + "pass@4": 0.9966461205243328, + "pass@8": 0.9976517473074589, + "pass@16": 0.9983273506604989, + "pass@32": 0.9991077492935924, + "pass@64": 0.99982934368161, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.993135838150289, + "pass@2": 0.9963200781712257, + "pass@4": 0.9979675979808883, + "pass@8": 0.9990151743820411, + "pass@16": 0.9997052474997874, + "pass@32": 0.9999794947349081, + "pass@64": 0.9999999848737747, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9828125, + "pass@2": 0.9905698957524677, + "pass@4": 0.9942337766758024, + "pass@8": 0.9968760475883579, + "pass@16": 0.9988561668827354, + "pass@32": 0.9998348548763891, + "pass@64": 0.9999981038504987, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9622941616766467, + "pass@2": 0.9767223672733275, + "pass@4": 0.9838516302602888, + "pass@8": 0.9882686825233068, + "pass@16": 0.990794735361916, + "pass@32": 0.9921813965674419, + "pass@64": 0.9932673675561231, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 375050, + "answer_accuracy": 0.980943463341145, + "avg_response_len": 210.53488816119852, + "resp_tokens_sum": 80495067, + "pass_at_k": { + "pass@1": 0.980943463341145, + "pass@2": 0.9871446708703601, + "pass@4": 0.9904940460656353, + "pass@8": 0.99278289829383, + "pass@16": 0.9943920117924884, + "pass@32": 0.9954331792601782, + "pass@64": 0.9960837580572915, + "pass@128": 0.9966521593572146 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9231110074626866, + "pass@2": 0.9381617992713601, + "pass@4": 0.9482348143982001, + "pass@8": 0.9569980467762143, + "pass@16": 0.9643792035984474, + "pass@32": 0.9699318877469423, + "pass@64": 0.974212970638659, + "pass@128": 0.9791044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997950072886297, + "pass@2": 0.9999921087670163, + "pass@4": 0.9999999904348691, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9985836988304093, + "pass@2": 0.9993416735736981, + "pass@4": 0.9998569655218537, + "pass@8": 0.9999939773438304, + "pass@16": 0.9999999936471988, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9802159926470588, + "pass@2": 0.9872854185965726, + "pass@4": 0.9926532946249366, + "pass@8": 0.9961444439272436, + "pass@16": 0.9981664618729108, + "pass@32": 0.9993048852819176, + "pass@64": 0.9999152319375018, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9974615536277602, + "pass@2": 0.9992905313097694, + "pass@4": 0.9999229347120254, + "pass@8": 0.9999991704144204, + "pass@16": 0.999999999947923, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9865259098101266, + "pass@2": 0.9916054364347653, + "pass@4": 0.9937736217506039, + "pass@8": 0.9953534321183053, + "pass@16": 0.9964274670231903, + "pass@32": 0.9968039802232598, + "pass@64": 0.9968354048433435, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9800347222222222, + "pass@2": 0.988521124470928, + "pass@4": 0.9913935138738287, + "pass@8": 0.9930488023084659, + "pass@16": 0.9946040308153634, + "pass@32": 0.9960989483322017, + "pass@64": 0.9969061152123114, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9608621382636656, + "pass@2": 0.9793176538724461, + "pass@4": 0.9882693851371472, + "pass@8": 0.9933905067759253, + "pass@16": 0.9959199013833268, + "pass@32": 0.9967107894337671, + "pass@64": 0.9967843633383671, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.55450703125, + "avg_response_len": 370.0711171875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.55450703125, + "pass@2": 0.6366033341535341, + "pass@4": 0.7048246399699933, + "pass@8": 0.7597000936417586, + "pass@16": 0.8026905168429185, + "pass@32": 0.836301924534803, + "pass@64": 0.8633132687974364, + "pass@128": 0.8859 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9137734375, + "pass@2": 0.942595226377953, + "pass@4": 0.9578507694038247, + "pass@8": 0.9668308275856704, + "pass@16": 0.9730883010977072, + "pass@32": 0.9780950758725304, + "pass@64": 0.9821233558936551, + "pass@128": 0.984 + }, + "12": { + "pass@1": 0.8790625, + "pass@2": 0.9087434793307092, + "pass@4": 0.9259832760592426, + "pass@8": 0.9370632453017427, + "pass@16": 0.9443895108125457, + "pass@32": 0.9496765725364011, + "pass@64": 0.9540170947758683, + "pass@128": 0.958 + }, + "13": { + "pass@1": 0.7547890625, + "pass@2": 0.8164760088582679, + "pass@4": 0.8583099431946, + "pass@8": 0.8878386948117137, + "pass@16": 0.9083981828690031, + "pass@32": 0.923706693953936, + "pass@64": 0.9365819375020756, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.698203125, + "pass@2": 0.781750984251969, + "pass@4": 0.8382186915073125, + "pass@8": 0.8785815165927703, + "pass@16": 0.9079747469017242, + "pass@32": 0.9282146224392507, + "pass@64": 0.9426946900784411, + "pass@128": 0.953 + }, + "15": { + "pass@1": 0.5978671875, + "pass@2": 0.7011580954724412, + "pass@4": 0.7782147362204725, + "pass@8": 0.8310367837625963, + "pass@16": 0.8669843280922963, + "pass@32": 0.8936748922101493, + "pass@64": 0.9157006424936757, + "pass@128": 0.933 + }, + "16": { + "pass@1": 0.5058359375, + "pass@2": 0.6168774606299207, + "pass@4": 0.7040999335395578, + "pass@8": 0.7712715932673241, + "pass@16": 0.8218927296378162, + "pass@32": 0.8599485271434877, + "pass@64": 0.8886993736285085, + "pass@128": 0.909 + }, + "17": { + "pass@1": 0.388140625, + "pass@2": 0.4966109744094487, + "pass@4": 0.5918282779340087, + "pass@8": 0.6694251506734931, + "pass@16": 0.7294898529588945, + "pass@32": 0.7750607540814882, + "pass@64": 0.8107231357221262, + "pass@128": 0.841 + }, + "18": { + "pass@1": 0.346265625, + "pass@2": 0.45340501968504016, + "pass@4": 0.5526364272590927, + "pass@8": 0.6361182517701623, + "pass@16": 0.7022117723066393, + "pass@32": 0.7531087664434932, + "pass@64": 0.7917766012254918, + "pass@128": 0.824 + }, + "19": { + "pass@1": 0.2376796875, + "pass@2": 0.3305230068897632, + "pass@4": 0.42503590635545546, + "pass@8": 0.5120922967380384, + "pass@16": 0.5878514521220423, + "pass@32": 0.6524551699692902, + "pass@64": 0.7077525196805154, + "pass@128": 0.756 + }, + "20": { + "pass@1": 0.223453125, + "pass@2": 0.31789308562992113, + "pass@4": 0.4160684382264714, + "pass@8": 0.506742575914105, + "pass@16": 0.5846242916303807, + "pass@32": 0.6490781706979735, + "pass@64": 0.703063336973903, + "pass@128": 0.752 + } + }, + "per_op_accuracy": { + "11": 0.9137734375, + "12": 0.8790625, + "13": 0.7547890625, + "14": 0.698203125, + "15": 0.5978671875, + "16": 0.5058359375, + "17": 0.388140625, + "18": 0.346265625, + "19": 0.2376796875, + "20": 0.223453125 + }, + "per_op_avg_response_len": { + "11": 297.1982265625, + "12": 307.9220546875, + "13": 327.613, + "14": 354.604125, + "15": 372.8530703125, + "16": 380.6671953125, + "17": 401.2857890625, + "18": 414.542578125, + "19": 412.855484375, + "20": 431.1696484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.1982265625, + "12": 307.9220546875, + "13": 327.613, + "14": 354.604125, + "15": 372.8530703125, + "16": 380.6671953125, + "17": 401.2857890625, + "18": 414.542578125, + "19": 412.855484375, + "20": 431.1696484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 239640, + "answer_accuracy": 0.5563707280832095, + "avg_response_len": 351.66095375185733, + "resp_tokens_sum": 151467406, + "pass_at_k": { + "pass@1": 0.5563707280832095, + "pass@2": 0.6360270223233648, + "pass@4": 0.7016119429257841, + "pass@8": 0.7547548210892957, + "pass@16": 0.7974603754849667, + "pass@32": 0.8319590771613706, + "pass@64": 0.8602854520440317, + "pass@128": 0.8849925705794948 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9136641288951841, + "pass@2": 0.9448533102094537, + "pass@4": 0.9621818766988403, + "pass@8": 0.9722579032844575, + "pass@16": 0.9783576572544549, + "pass@32": 0.9822485930790287, + "pass@64": 0.9849666200815427, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8903160310734464, + "pass@2": 0.9213394306908673, + "pass@4": 0.9376138443499642, + "pass@8": 0.9477520057884693, + "pass@16": 0.9550494812253661, + "pass@32": 0.9613654075465042, + "pass@64": 0.9670326878601943, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.7568818573667712, + "pass@2": 0.8172446190111813, + "pass@4": 0.8549627382979937, + "pass@8": 0.8809640566371564, + "pass@16": 0.9017800586952615, + "pass@32": 0.9209144214036475, + "pass@64": 0.9381434303380467, + "pass@128": 0.9529780564263323 + }, + "14": { + "pass@1": 0.7071651090342679, + "pass@2": 0.7924351344224491, + "pass@4": 0.8502455995570642, + "pass@8": 0.8910067656860042, + "pass@16": 0.9189231525733634, + "pass@32": 0.9373696512731893, + "pass@64": 0.9506183186383255, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.5974530346820809, + "pass@2": 0.7007393979563971, + "pass@4": 0.7801067936414015, + "pass@8": 0.8354042806919619, + "pass@16": 0.8730541335942021, + "pass@32": 0.901120904000133, + "pass@64": 0.9254484074093287, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5054939516129032, + "pass@2": 0.6082335852171704, + "pass@4": 0.6902914855401138, + "pass@8": 0.7560433401649518, + "pass@16": 0.807505375306526, + "pass@32": 0.8481175391774843, + "pass@64": 0.8783173741315493, + "pass@128": 0.896774193548387 + }, + "17": { + "pass@1": 0.3838059971098266, + "pass@2": 0.4859715647899499, + "pass@4": 0.575622420300786, + "pass@8": 0.655213423703959, + "pass@16": 0.7220266494245524, + "pass@32": 0.7722586027005358, + "pass@64": 0.8081715186093134, + "pass@128": 0.8352601156069365 + }, + "18": { + "pass@1": 0.32388392857142856, + "pass@2": 0.4244576068616422, + "pass@4": 0.5181791052011352, + "pass@8": 0.5980358612928792, + "pass@16": 0.6629633073108419, + "pass@32": 0.7142480405100232, + "pass@64": 0.7544531085968116, + "pass@128": 0.7942857142857143 + }, + "19": { + "pass@1": 0.2381522147147147, + "pass@2": 0.3297909129602043, + "pass@4": 0.4215563141882039, + "pass@8": 0.5039769605762625, + "pass@16": 0.5763823880150785, + "pass@32": 0.6406778469014253, + "pass@64": 0.6984631506742155, + "pass@128": 0.7567567567567568 + }, + "20": { + "pass@1": 0.2317239114114114, + "pass@2": 0.32413183360525893, + "pass@4": 0.41706644327116754, + "pass@8": 0.5007026327565921, + "pass@16": 0.5740754156310071, + "pass@32": 0.638377011630852, + "pass@64": 0.6956871639925536, + "pass@128": 0.7477477477477478 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 240735, + "answer_accuracy": 0.5592453724353256, + "avg_response_len": 365.8424583705025, + "resp_tokens_sum": 157482008, + "pass_at_k": { + "pass@1": 0.5592453724353256, + "pass@2": 0.6419249340320007, + "pass@4": 0.7112396518008854, + "pass@8": 0.7672510615311416, + "pass@16": 0.81059469485332, + "pass@32": 0.8434025378608513, + "pass@64": 0.8693494871617078, + "pass@128": 0.8908712459113887 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9145566641566265, + "pass@2": 0.9395704125557349, + "pass@4": 0.9531212250577107, + "pass@8": 0.9619663334454763, + "pass@16": 0.968723135248245, + "pass@32": 0.9746702926505119, + "pass@64": 0.9795887846110832, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8619954937304075, + "pass@2": 0.8864180170562533, + "pass@4": 0.9021936098113125, + "pass@8": 0.91375233313683, + "pass@16": 0.9201638977773448, + "pass@32": 0.9233086460149744, + "pass@64": 0.9255429242327472, + "pass@128": 0.9278996865203761 + }, + "13": { + "pass@1": 0.7585127801120448, + "pass@2": 0.8136451923288999, + "pass@4": 0.852209066828831, + "pass@8": 0.8806241733453731, + "pass@16": 0.9005599569445452, + "pass@32": 0.9153669801046609, + "pass@64": 0.9288086017030489, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.7070084064327485, + "pass@2": 0.7881876093613296, + "pass@4": 0.8460380092291095, + "pass@8": 0.8907571809918315, + "pass@16": 0.9247672939952509, + "pass@32": 0.9464667748863145, + "pass@64": 0.9590041317766209, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.5989109848484848, + "pass@2": 0.706179909329516, + "pass@4": 0.7829212430832506, + "pass@8": 0.8339317426580697, + "pass@16": 0.8695270739047846, + "pass@32": 0.8969910152454786, + "pass@64": 0.9193853522624728, + "pass@128": 0.9333333333333333 + }, + "16": { + "pass@1": 0.5016693376068376, + "pass@2": 0.6151490679049731, + "pass@4": 0.7033438189350254, + "pass@8": 0.769372720072636, + "pass@16": 0.8183349225214916, + "pass@32": 0.8549634298764497, + "pass@64": 0.8840951888891482, + "pass@128": 0.9088319088319088 + }, + "17": { + "pass@1": 0.41702251552795033, + "pass@2": 0.5324107754193774, + "pass@4": 0.6294041246591063, + "pass@8": 0.7014292101498082, + "pass@16": 0.7530650147190665, + "pass@32": 0.7912314076196781, + "pass@64": 0.8222645098320329, + "pass@128": 0.8478260869565217 + }, + "18": { + "pass@1": 0.3754825367647059, + "pass@2": 0.48911641674386275, + "pass@4": 0.5957133086305385, + "pass@8": 0.6875344124222568, + "pass@16": 0.7598632015899763, + "pass@32": 0.8117664354039354, + "pass@64": 0.8464364058423706, + "pass@128": 0.8676470588235294 + }, + "19": { + "pass@1": 0.23209635416666666, + "pass@2": 0.32691116247187857, + "pass@4": 0.4285349596367417, + "pass@8": 0.5260741360548356, + "pass@16": 0.6098688407896331, + "pass@32": 0.6770059830627879, + "pass@64": 0.7314934121455681, + "pass@128": 0.7767857142857143 + }, + "20": { + "pass@1": 0.22721744011976047, + "pass@2": 0.32192406525531636, + "pass@4": 0.4180934567872128, + "pass@8": 0.5052503046175902, + "pass@16": 0.5783695489331038, + "pass@32": 0.6389484435964482, + "pass@64": 0.6932115507430644, + "pass@128": 0.7485029940119761 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 229394, + "answer_accuracy": 0.5477202399144254, + "avg_response_len": 393.3508175427873, + "resp_tokens_sum": 164741616, + "pass_at_k": { + "pass@1": 0.5477202399144254, + "pass@2": 0.6317264236759547, + "pass@4": 0.7015352270013488, + "pass@8": 0.7570249521769413, + "pass@16": 0.7999453074970916, + "pass@32": 0.8334701148758902, + "pass@64": 0.8602230490590537, + "pass@128": 0.8817237163814181 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9130704365079365, + "pass@2": 0.9432527965254344, + "pass@4": 0.9579819689205518, + "pass@8": 0.9658760794360599, + "pass@16": 0.9717840228078343, + "pass@32": 0.9770501122402013, + "pass@64": 0.9816084524253677, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8835292431192661, + "pass@2": 0.916886768884394, + "pass@4": 0.9366000416805697, + "pass@8": 0.948232541229283, + "pass@16": 0.9564822968434049, + "pass@32": 0.9627454439943796, + "pass@64": 0.9677043744437711, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.7486255787037037, + "pass@2": 0.8188384004082823, + "pass@4": 0.8683277556161959, + "pass@8": 0.9025565767288988, + "pass@16": 0.9235507392469514, + "pass@32": 0.9356450050271856, + "pass@64": 0.9436096062229937, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.6807307121661721, + "pass@2": 0.7650419694385382, + "pass@4": 0.8188274625909145, + "pass@8": 0.8543898780663406, + "pass@16": 0.880504452164093, + "pass@32": 0.900971297832159, + "pass@64": 0.9185958359879355, + "pass@128": 0.9347181008902077 + }, + "15": { + "pass@1": 0.5972463348765432, + "pass@2": 0.6964904126567509, + "pass@4": 0.7714005413675139, + "pass@8": 0.8234241591543643, + "pass@16": 0.8579125416053235, + "pass@32": 0.8823457543058479, + "pass@64": 0.9015380471710855, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.5104627581120944, + "pass@2": 0.6265714642417487, + "pass@4": 0.7175100075986074, + "pass@8": 0.7871632243973276, + "pass@16": 0.8387330545361368, + "pass@32": 0.8759289855806293, + "pass@64": 0.9029604022054188, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.3646460843373494, + "pass@2": 0.47297754008158627, + "pass@4": 0.5722734709930832, + "pass@8": 0.6531960855532634, + "pass@16": 0.714402703368858, + "pass@32": 0.7622974828119503, + "pass@64": 0.8021886087873138, + "pass@128": 0.8403614457831325 + }, + "18": { + "pass@1": 0.3394909274193548, + "pass@2": 0.44692024384048745, + "pass@4": 0.5442942435622966, + "pass@8": 0.6227225809486652, + "pass@16": 0.6832939555072691, + "pass@32": 0.7326495620246682, + "pass@64": 0.7739665652587125, + "pass@128": 0.8096774193548387 + }, + "19": { + "pass@1": 0.24287197885196374, + "pass@2": 0.3349259283488355, + "pass@4": 0.42498461417775973, + "pass@8": 0.5060636228148577, + "pass@16": 0.57703983808974, + "pass@32": 0.6393819838127462, + "pass@64": 0.6929985015863778, + "pass@128": 0.7341389728096677 + }, + "20": { + "pass@1": 0.21140671921921922, + "pass@2": 0.30761125298526865, + "pass@4": 0.41303933348421523, + "pass@8": 0.5142792716633172, + "pass@16": 0.6014466933381385, + "pass@32": 0.669939476467524, + "pass@64": 0.7203208811297246, + "pass@128": 0.7597597597597597 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7564370888157895, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7564370888157895, + "pass@2": 0.8025323702341295, + "pass@4": 0.8399313315226856, + "pass@8": 0.869775825178544, + "pass@16": 0.8930504890426428, + "pass@32": 0.9111819659028908, + "pass@64": 0.9257087870838782, + "pass@128": 0.9378421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.918921875, + "pass@2": 0.9345583169291337, + "pass@4": 0.943269286276715, + "pass@8": 0.9493995987750118, + "pass@16": 0.9543911543654843, + "pass@32": 0.9587902086599787, + "pass@64": 0.9627103821823005, + "pass@128": 0.966 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999265625, + "pass@2": 0.999923474409449, + "pass@4": 0.9999989649418821, + "pass@8": 0.9999999998733826, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998375, + "pass@2": 0.9991230314960631, + "pass@4": 0.9996207719347582, + "pass@8": 0.9998968349840596, + "pass@16": 0.9999915015594042, + "pass@32": 0.9999999700068078, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.98590625, + "pass@2": 0.990609375, + "pass@4": 0.9941187244094487, + "pass@8": 0.9967705762096115, + "pass@16": 0.9986236324272478, + "pass@32": 0.9996398839845977, + "pass@64": 0.999969899907236, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9965234375, + "pass@2": 0.9980569635826774, + "pass@4": 0.9988050461192352, + "pass@8": 0.9991801968316429, + "pass@16": 0.9994162453640056, + "pass@32": 0.9996886045034639, + "pass@64": 0.9999404409448819, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9910390625, + "pass@2": 0.9944616141732285, + "pass@4": 0.9960210366516683, + "pass@8": 0.997143365187022, + "pass@16": 0.9978839411380321, + "pass@32": 0.9982329183934826, + "pass@64": 0.9984999826968121, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.9800390625, + "pass@2": 0.989688853346457, + "pass@4": 0.9935460817397821, + "pass@8": 0.9959264229266411, + "pass@16": 0.9976315460599925, + "pass@32": 0.9986283768487406, + "pass@64": 0.9989690357139468, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.9571640625, + "pass@2": 0.9756600639763789, + "pass@4": 0.9850689871578558, + "pass@8": 0.9904227471871823, + "pass@16": 0.9931161024670461, + "pass@32": 0.9944581444095479, + "pass@64": 0.9952445251742235, + "pass@128": 0.996 + }, + "11": { + "pass@1": 0.9137734375, + "pass@2": 0.942595226377953, + "pass@4": 0.9578507694038247, + "pass@8": 0.9668308275856704, + "pass@16": 0.9730883010977072, + "pass@32": 0.9780950758725304, + "pass@64": 0.9821233558936551, + "pass@128": 0.984 + }, + "12": { + "pass@1": 0.8790625, + "pass@2": 0.9087434793307092, + "pass@4": 0.9259832760592426, + "pass@8": 0.9370632453017427, + "pass@16": 0.9443895108125457, + "pass@32": 0.9496765725364011, + "pass@64": 0.9540170947758683, + "pass@128": 0.958 + }, + "13": { + "pass@1": 0.7547890625, + "pass@2": 0.8164760088582679, + "pass@4": 0.8583099431946, + "pass@8": 0.8878386948117137, + "pass@16": 0.9083981828690031, + "pass@32": 0.923706693953936, + "pass@64": 0.9365819375020756, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.698203125, + "pass@2": 0.781750984251969, + "pass@4": 0.8382186915073125, + "pass@8": 0.8785815165927703, + "pass@16": 0.9079747469017242, + "pass@32": 0.9282146224392507, + "pass@64": 0.9426946900784411, + "pass@128": 0.953 + }, + "15": { + "pass@1": 0.5978671875, + "pass@2": 0.7011580954724412, + "pass@4": 0.7782147362204725, + "pass@8": 0.8310367837625963, + "pass@16": 0.8669843280922963, + "pass@32": 0.8936748922101493, + "pass@64": 0.9157006424936757, + "pass@128": 0.933 + }, + "16": { + "pass@1": 0.5058359375, + "pass@2": 0.6168774606299207, + "pass@4": 0.7040999335395578, + "pass@8": 0.7712715932673241, + "pass@16": 0.8218927296378162, + "pass@32": 0.8599485271434877, + "pass@64": 0.8886993736285085, + "pass@128": 0.909 + }, + "17": { + "pass@1": 0.388140625, + "pass@2": 0.4966109744094487, + "pass@4": 0.5918282779340087, + "pass@8": 0.6694251506734931, + "pass@16": 0.7294898529588945, + "pass@32": 0.7750607540814882, + "pass@64": 0.8107231357221262, + "pass@128": 0.841 + }, + "18": { + "pass@1": 0.346265625, + "pass@2": 0.45340501968504016, + "pass@4": 0.5526364272590927, + "pass@8": 0.6361182517701623, + "pass@16": 0.7022117723066393, + "pass@32": 0.7531087664434932, + "pass@64": 0.7917766012254918, + "pass@128": 0.824 + }, + "19": { + "pass@1": 0.2376796875, + "pass@2": 0.3305230068897632, + "pass@4": 0.42503590635545546, + "pass@8": 0.5120922967380384, + "pass@16": 0.5878514521220423, + "pass@32": 0.6524551699692902, + "pass@64": 0.7077525196805154, + "pass@128": 0.756 + }, + "20": { + "pass@1": 0.223453125, + "pass@2": 0.31789308562992113, + "pass@4": 0.4160684382264714, + "pass@8": 0.506742575914105, + "pass@16": 0.5846242916303807, + "pass@32": 0.6490781706979735, + "pass@64": 0.703063336973903, + "pass@128": 0.752 + } + }, + "per_op_accuracy": { + "10": 0.918921875, + "2": 1.0, + "3": 0.999265625, + "4": 0.998375, + "5": 0.98590625, + "6": 0.9965234375, + "7": 0.9910390625, + "8": 0.9800390625, + "9": 0.9571640625, + "11": 0.9137734375, + "12": 0.8790625, + "13": 0.7547890625, + "14": 0.698203125, + "15": 0.5978671875, + "16": 0.5058359375, + "17": 0.388140625, + "18": 0.346265625, + "19": 0.2376796875, + "20": 0.223453125 + }, + "per_op_avg_response_len": { + "10": 279.6280078125, + "2": 115.096359375, + "3": 144.0779921875, + "4": 159.11890625, + "5": 173.1603515625, + "6": 197.6270703125, + "7": 222.665234375, + "8": 241.5987421875, + "9": 257.3600703125, + "11": 297.1982265625, + "12": 307.9220546875, + "13": 327.613, + "14": 354.604125, + "15": 372.8530703125, + "16": 380.6671953125, + "17": 401.2857890625, + "18": 414.542578125, + "19": 412.855484375, + "20": 431.1696484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.6280078125, + "2": 115.096359375, + "3": 144.0779921875, + "4": 159.11890625, + "5": 173.1603515625, + "6": 197.6270703125, + "7": 222.665234375, + "8": 241.5987421875, + "9": 257.3600703125, + "11": 297.1982265625, + "12": 307.9220546875, + "13": 327.613, + "14": 354.604125, + "15": 372.8530703125, + "16": 380.6671953125, + "17": 401.2857890625, + "18": 414.542578125, + "19": 412.855484375, + "20": 431.1696484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 611052, + "answer_accuracy": 0.7546385946885867, + "avg_response_len": 275.69386016044893, + "resp_tokens_sum": 223237038, + "pass_at_k": { + "pass@1": 0.7546385946885867, + "pass@2": 0.8003159143865632, + "pass@4": 0.8368874944121506, + "pass@8": 0.8661492337472454, + "pass@16": 0.8894298983045624, + "pass@32": 0.9081125529864221, + "pass@64": 0.9234140192076873, + "pass@128": 0.9367688902940247 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9221054572271387, + "pass@2": 0.9400673298028014, + "pass@4": 0.949361465878712, + "pass@8": 0.9547948042979221, + "pass@16": 0.9585332135653881, + "pass@32": 0.9613178358810445, + "pass@64": 0.9636093956539674, + "pass@128": 0.9646017699115044 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9991637996941896, + "pass@2": 0.9998976618748345, + "pass@4": 0.9999986354916645, + "pass@8": 0.9999999998960027, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99658203125, + "pass@2": 0.9979638287401574, + "pass@4": 0.9989677803946382, + "pass@8": 0.999684046038967, + "pass@16": 0.9999734491626946, + "pass@32": 0.9999999062712751, + "pass@64": 0.9999999999999887, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9881696428571428, + "pass@2": 0.9919502249718786, + "pass@4": 0.9951166458359371, + "pass@8": 0.9975764494654312, + "pass@16": 0.9991626286735529, + "pass@32": 0.999880913313676, + "pass@64": 0.999999029712244, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992982784431138, + "pass@2": 0.9999756883870056, + "pass@4": 0.9999999393788351, + "pass@8": 0.9999999999999057, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9931120562130178, + "pass@2": 0.9952294355402319, + "pass@4": 0.9961295363227524, + "pass@8": 0.9969006813652405, + "pass@16": 0.9973812009579224, + "pass@32": 0.9977809333273798, + "pass@64": 0.9985207100591404, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9768880208333334, + "pass@2": 0.989932711235615, + "pass@4": 0.9950610605765626, + "pass@8": 0.9979172271283807, + "pass@16": 0.9994694376766202, + "pass@32": 0.9999552999134591, + "pass@64": 0.9999999117991033, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9490977112676057, + "pass@2": 0.9714563463457915, + "pass@4": 0.9834106024599026, + "pass@8": 0.9898494637100985, + "pass@16": 0.9928438634533845, + "pass@32": 0.994626767442593, + "pass@64": 0.9957557391894256, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9136641288951841, + "pass@2": 0.9448533102094537, + "pass@4": 0.9621818766988403, + "pass@8": 0.9722579032844575, + "pass@16": 0.9783576572544549, + "pass@32": 0.9822485930790287, + "pass@64": 0.9849666200815427, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8903160310734464, + "pass@2": 0.9213394306908673, + "pass@4": 0.9376138443499642, + "pass@8": 0.9477520057884693, + "pass@16": 0.9550494812253661, + "pass@32": 0.9613654075465042, + "pass@64": 0.9670326878601943, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.7568818573667712, + "pass@2": 0.8172446190111813, + "pass@4": 0.8549627382979937, + "pass@8": 0.8809640566371564, + "pass@16": 0.9017800586952615, + "pass@32": 0.9209144214036475, + "pass@64": 0.9381434303380467, + "pass@128": 0.9529780564263323 + }, + "14": { + "pass@1": 0.7071651090342679, + "pass@2": 0.7924351344224491, + "pass@4": 0.8502455995570642, + "pass@8": 0.8910067656860042, + "pass@16": 0.9189231525733634, + "pass@32": 0.9373696512731893, + "pass@64": 0.9506183186383255, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.5974530346820809, + "pass@2": 0.7007393979563971, + "pass@4": 0.7801067936414015, + "pass@8": 0.8354042806919619, + "pass@16": 0.8730541335942021, + "pass@32": 0.901120904000133, + "pass@64": 0.9254484074093287, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5054939516129032, + "pass@2": 0.6082335852171704, + "pass@4": 0.6902914855401138, + "pass@8": 0.7560433401649518, + "pass@16": 0.807505375306526, + "pass@32": 0.8481175391774843, + "pass@64": 0.8783173741315493, + "pass@128": 0.896774193548387 + }, + "17": { + "pass@1": 0.3838059971098266, + "pass@2": 0.4859715647899499, + "pass@4": 0.575622420300786, + "pass@8": 0.655213423703959, + "pass@16": 0.7220266494245524, + "pass@32": 0.7722586027005358, + "pass@64": 0.8081715186093134, + "pass@128": 0.8352601156069365 + }, + "18": { + "pass@1": 0.32388392857142856, + "pass@2": 0.4244576068616422, + "pass@4": 0.5181791052011352, + "pass@8": 0.5980358612928792, + "pass@16": 0.6629633073108419, + "pass@32": 0.7142480405100232, + "pass@64": 0.7544531085968116, + "pass@128": 0.7942857142857143 + }, + "19": { + "pass@1": 0.2381522147147147, + "pass@2": 0.3297909129602043, + "pass@4": 0.4215563141882039, + "pass@8": 0.5039769605762625, + "pass@16": 0.5763823880150785, + "pass@32": 0.6406778469014253, + "pass@64": 0.6984631506742155, + "pass@128": 0.7567567567567568 + }, + "20": { + "pass@1": 0.2317239114114114, + "pass@2": 0.32413183360525893, + "pass@4": 0.41706644327116754, + "pass@8": 0.5007026327565921, + "pass@16": 0.5740754156310071, + "pass@32": 0.638377011630852, + "pass@64": 0.6956871639925536, + "pass@128": 0.7477477477477478 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 624159, + "answer_accuracy": 0.7601312840997662, + "avg_response_len": 285.43927684138737, + "resp_tokens_sum": 234379899, + "pass_at_k": { + "pass@1": 0.7601312840997662, + "pass@2": 0.8058778752738638, + "pass@4": 0.8434163727098335, + "pass@8": 0.8735857310959408, + "pass@16": 0.8969221776743685, + "pass@32": 0.9146164021260262, + "pass@64": 0.9286044999606458, + "pass@128": 0.9401402961808262 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9113065567484663, + "pass@2": 0.9251266545094442, + "pass@4": 0.9318315537436652, + "pass@8": 0.9359810259140621, + "pass@16": 0.9398201281022645, + "pass@32": 0.9447125150339829, + "pass@64": 0.9499553739007809, + "pass@128": 0.9539877300613497 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9988162878787878, + "pass@2": 0.9998777141493678, + "pass@4": 0.9999982255059028, + "pass@8": 0.999999999719363, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998613165680473, + "pass@2": 0.9999992720029819, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.989447463768116, + "pass@2": 0.9926609038000683, + "pass@4": 0.9946517704308702, + "pass@8": 0.9966518367905446, + "pass@16": 0.998582050313881, + "pass@32": 0.9997499573766314, + "pass@64": 0.9999971788090677, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9930157593123209, + "pass@2": 0.9951002442298582, + "pass@4": 0.9966461205243328, + "pass@8": 0.9976517473074589, + "pass@16": 0.9983273506604989, + "pass@32": 0.9991077492935924, + "pass@64": 0.99982934368161, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.993135838150289, + "pass@2": 0.9963200781712257, + "pass@4": 0.9979675979808883, + "pass@8": 0.9990151743820411, + "pass@16": 0.9997052474997874, + "pass@32": 0.9999794947349081, + "pass@64": 0.9999999848737747, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9828125, + "pass@2": 0.9905698957524677, + "pass@4": 0.9942337766758024, + "pass@8": 0.9968760475883579, + "pass@16": 0.9988561668827354, + "pass@32": 0.9998348548763891, + "pass@64": 0.9999981038504987, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9622941616766467, + "pass@2": 0.9767223672733275, + "pass@4": 0.9838516302602888, + "pass@8": 0.9882686825233068, + "pass@16": 0.990794735361916, + "pass@32": 0.9921813965674419, + "pass@64": 0.9932673675561231, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9145566641566265, + "pass@2": 0.9395704125557349, + "pass@4": 0.9531212250577107, + "pass@8": 0.9619663334454763, + "pass@16": 0.968723135248245, + "pass@32": 0.9746702926505119, + "pass@64": 0.9795887846110832, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8619954937304075, + "pass@2": 0.8864180170562533, + "pass@4": 0.9021936098113125, + "pass@8": 0.91375233313683, + "pass@16": 0.9201638977773448, + "pass@32": 0.9233086460149744, + "pass@64": 0.9255429242327472, + "pass@128": 0.9278996865203761 + }, + "13": { + "pass@1": 0.7585127801120448, + "pass@2": 0.8136451923288999, + "pass@4": 0.852209066828831, + "pass@8": 0.8806241733453731, + "pass@16": 0.9005599569445452, + "pass@32": 0.9153669801046609, + "pass@64": 0.9288086017030489, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.7070084064327485, + "pass@2": 0.7881876093613296, + "pass@4": 0.8460380092291095, + "pass@8": 0.8907571809918315, + "pass@16": 0.9247672939952509, + "pass@32": 0.9464667748863145, + "pass@64": 0.9590041317766209, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.5989109848484848, + "pass@2": 0.706179909329516, + "pass@4": 0.7829212430832506, + "pass@8": 0.8339317426580697, + "pass@16": 0.8695270739047846, + "pass@32": 0.8969910152454786, + "pass@64": 0.9193853522624728, + "pass@128": 0.9333333333333333 + }, + "16": { + "pass@1": 0.5016693376068376, + "pass@2": 0.6151490679049731, + "pass@4": 0.7033438189350254, + "pass@8": 0.769372720072636, + "pass@16": 0.8183349225214916, + "pass@32": 0.8549634298764497, + "pass@64": 0.8840951888891482, + "pass@128": 0.9088319088319088 + }, + "17": { + "pass@1": 0.41702251552795033, + "pass@2": 0.5324107754193774, + "pass@4": 0.6294041246591063, + "pass@8": 0.7014292101498082, + "pass@16": 0.7530650147190665, + "pass@32": 0.7912314076196781, + "pass@64": 0.8222645098320329, + "pass@128": 0.8478260869565217 + }, + "18": { + "pass@1": 0.3754825367647059, + "pass@2": 0.48911641674386275, + "pass@4": 0.5957133086305385, + "pass@8": 0.6875344124222568, + "pass@16": 0.7598632015899763, + "pass@32": 0.8117664354039354, + "pass@64": 0.8464364058423706, + "pass@128": 0.8676470588235294 + }, + "19": { + "pass@1": 0.23209635416666666, + "pass@2": 0.32691116247187857, + "pass@4": 0.4285349596367417, + "pass@8": 0.5260741360548356, + "pass@16": 0.6098688407896331, + "pass@32": 0.6770059830627879, + "pass@64": 0.7314934121455681, + "pass@128": 0.7767857142857143 + }, + "20": { + "pass@1": 0.22721744011976047, + "pass@2": 0.32192406525531636, + "pass@4": 0.4180934567872128, + "pass@8": 0.5052503046175902, + "pass@16": 0.5783695489331038, + "pass@32": 0.6389484435964482, + "pass@64": 0.6932115507430644, + "pass@128": 0.7485029940119761 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 604444, + "answer_accuracy": 0.7544685652660169, + "avg_response_len": 306.1050624600575, + "resp_tokens_sum": 245236683, + "pass_at_k": { + "pass@1": 0.7544685652660169, + "pass@2": 0.8013436635496822, + "pass@4": 0.8394358489129932, + "pass@8": 0.8695363733386458, + "pass@16": 0.8927416496812058, + "pass@32": 0.9107641991251118, + "pass@64": 0.9250602335578206, + "pass@128": 0.936571337274325 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9231110074626866, + "pass@2": 0.9381617992713601, + "pass@4": 0.9482348143982001, + "pass@8": 0.9569980467762143, + "pass@16": 0.9643792035984474, + "pass@32": 0.9699318877469423, + "pass@64": 0.974212970638659, + "pass@128": 0.9791044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997950072886297, + "pass@2": 0.9999921087670163, + "pass@4": 0.9999999904348691, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9985836988304093, + "pass@2": 0.9993416735736981, + "pass@4": 0.9998569655218537, + "pass@8": 0.9999939773438304, + "pass@16": 0.9999999936471988, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9802159926470588, + "pass@2": 0.9872854185965726, + "pass@4": 0.9926532946249366, + "pass@8": 0.9961444439272436, + "pass@16": 0.9981664618729108, + "pass@32": 0.9993048852819176, + "pass@64": 0.9999152319375018, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9974615536277602, + "pass@2": 0.9992905313097694, + "pass@4": 0.9999229347120254, + "pass@8": 0.9999991704144204, + "pass@16": 0.999999999947923, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9865259098101266, + "pass@2": 0.9916054364347653, + "pass@4": 0.9937736217506039, + "pass@8": 0.9953534321183053, + "pass@16": 0.9964274670231903, + "pass@32": 0.9968039802232598, + "pass@64": 0.9968354048433435, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9800347222222222, + "pass@2": 0.988521124470928, + "pass@4": 0.9913935138738287, + "pass@8": 0.9930488023084659, + "pass@16": 0.9946040308153634, + "pass@32": 0.9960989483322017, + "pass@64": 0.9969061152123114, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9608621382636656, + "pass@2": 0.9793176538724461, + "pass@4": 0.9882693851371472, + "pass@8": 0.9933905067759253, + "pass@16": 0.9959199013833268, + "pass@32": 0.9967107894337671, + "pass@64": 0.9967843633383671, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9130704365079365, + "pass@2": 0.9432527965254344, + "pass@4": 0.9579819689205518, + "pass@8": 0.9658760794360599, + "pass@16": 0.9717840228078343, + "pass@32": 0.9770501122402013, + "pass@64": 0.9816084524253677, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8835292431192661, + "pass@2": 0.916886768884394, + "pass@4": 0.9366000416805697, + "pass@8": 0.948232541229283, + "pass@16": 0.9564822968434049, + "pass@32": 0.9627454439943796, + "pass@64": 0.9677043744437711, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.7486255787037037, + "pass@2": 0.8188384004082823, + "pass@4": 0.8683277556161959, + "pass@8": 0.9025565767288988, + "pass@16": 0.9235507392469514, + "pass@32": 0.9356450050271856, + "pass@64": 0.9436096062229937, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.6807307121661721, + "pass@2": 0.7650419694385382, + "pass@4": 0.8188274625909145, + "pass@8": 0.8543898780663406, + "pass@16": 0.880504452164093, + "pass@32": 0.900971297832159, + "pass@64": 0.9185958359879355, + "pass@128": 0.9347181008902077 + }, + "15": { + "pass@1": 0.5972463348765432, + "pass@2": 0.6964904126567509, + "pass@4": 0.7714005413675139, + "pass@8": 0.8234241591543643, + "pass@16": 0.8579125416053235, + "pass@32": 0.8823457543058479, + "pass@64": 0.9015380471710855, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.5104627581120944, + "pass@2": 0.6265714642417487, + "pass@4": 0.7175100075986074, + "pass@8": 0.7871632243973276, + "pass@16": 0.8387330545361368, + "pass@32": 0.8759289855806293, + "pass@64": 0.9029604022054188, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.3646460843373494, + "pass@2": 0.47297754008158627, + "pass@4": 0.5722734709930832, + "pass@8": 0.6531960855532634, + "pass@16": 0.714402703368858, + "pass@32": 0.7622974828119503, + "pass@64": 0.8021886087873138, + "pass@128": 0.8403614457831325 + }, + "18": { + "pass@1": 0.3394909274193548, + "pass@2": 0.44692024384048745, + "pass@4": 0.5442942435622966, + "pass@8": 0.6227225809486652, + "pass@16": 0.6832939555072691, + "pass@32": 0.7326495620246682, + "pass@64": 0.7739665652587125, + "pass@128": 0.8096774193548387 + }, + "19": { + "pass@1": 0.24287197885196374, + "pass@2": 0.3349259283488355, + "pass@4": 0.42498461417775973, + "pass@8": 0.5060636228148577, + "pass@16": 0.57703983808974, + "pass@32": 0.6393819838127462, + "pass@64": 0.6929985015863778, + "pass@128": 0.7341389728096677 + }, + "20": { + "pass@1": 0.21140671921921922, + "pass@2": 0.30761125298526865, + "pass@4": 0.41303933348421523, + "pass@8": 0.5142792716633172, + "pass@16": 0.6014466933381385, + "pass@32": 0.669939476467524, + "pass@64": 0.7203208811297246, + "pass@128": 0.7597597597597597 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch1-RL-op11-14/composition/summary.csv b/composition/Qwen3-100M-epoch1-RL-op11-14/composition/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..f5b6c47ef101a87eb0dc838aa5544df0d69f243a --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op11-14/composition/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.9808038194444444,0.55450703125,0.7564370888157895,0.0,0.0,0.0,198.925859375,370.0711171875,1152000,1280000,2432000 diff --git a/composition/Qwen3-100M-epoch1-RL-op11-14/composition/summary.json b/composition/Qwen3-100M-epoch1-RL-op11-14/composition/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..531e75c3257315fa01829eb50c0b340f3c2f5f17 --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op11-14/composition/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.9808038194444444, + "ood_acc": 0.55450703125, + "total_acc": 0.7564370888157895, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 198.925859375, + "ood_avg_resp_len": 370.0711171875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 611052, + "answer_accuracy": 0.7546385946885867, + "avg_response_len": 275.69386016044893, + "resp_tokens_sum": 223237038, + "pass_at_k": { + "pass@1": 0.7546385946885867, + "pass@2": 0.8003159143865632, + "pass@4": 0.8368874944121506, + "pass@8": 0.8661492337472454, + "pass@16": 0.8894298983045624, + "pass@32": 0.9081125529864221, + "pass@64": 0.9234140192076873, + "pass@128": 0.9367688902940247 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9221054572271387, + "pass@2": 0.9400673298028014, + "pass@4": 0.949361465878712, + "pass@8": 0.9547948042979221, + "pass@16": 0.9585332135653881, + "pass@32": 0.9613178358810445, + "pass@64": 0.9636093956539674, + "pass@128": 0.9646017699115044 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9991637996941896, + "pass@2": 0.9998976618748345, + "pass@4": 0.9999986354916645, + "pass@8": 0.9999999998960027, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99658203125, + "pass@2": 0.9979638287401574, + "pass@4": 0.9989677803946382, + "pass@8": 0.999684046038967, + "pass@16": 0.9999734491626946, + "pass@32": 0.9999999062712751, + "pass@64": 0.9999999999999887, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9881696428571428, + "pass@2": 0.9919502249718786, + "pass@4": 0.9951166458359371, + "pass@8": 0.9975764494654312, + "pass@16": 0.9991626286735529, + "pass@32": 0.999880913313676, + "pass@64": 0.999999029712244, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992982784431138, + "pass@2": 0.9999756883870056, + "pass@4": 0.9999999393788351, + "pass@8": 0.9999999999999057, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9931120562130178, + "pass@2": 0.9952294355402319, + "pass@4": 0.9961295363227524, + "pass@8": 0.9969006813652405, + "pass@16": 0.9973812009579224, + "pass@32": 0.9977809333273798, + "pass@64": 0.9985207100591404, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9768880208333334, + "pass@2": 0.989932711235615, + "pass@4": 0.9950610605765626, + "pass@8": 0.9979172271283807, + "pass@16": 0.9994694376766202, + "pass@32": 0.9999552999134591, + "pass@64": 0.9999999117991033, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9490977112676057, + "pass@2": 0.9714563463457915, + "pass@4": 0.9834106024599026, + "pass@8": 0.9898494637100985, + "pass@16": 0.9928438634533845, + "pass@32": 0.994626767442593, + "pass@64": 0.9957557391894256, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9136641288951841, + "pass@2": 0.9448533102094537, + "pass@4": 0.9621818766988403, + "pass@8": 0.9722579032844575, + "pass@16": 0.9783576572544549, + "pass@32": 0.9822485930790287, + "pass@64": 0.9849666200815427, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8903160310734464, + "pass@2": 0.9213394306908673, + "pass@4": 0.9376138443499642, + "pass@8": 0.9477520057884693, + "pass@16": 0.9550494812253661, + "pass@32": 0.9613654075465042, + "pass@64": 0.9670326878601943, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.7568818573667712, + "pass@2": 0.8172446190111813, + "pass@4": 0.8549627382979937, + "pass@8": 0.8809640566371564, + "pass@16": 0.9017800586952615, + "pass@32": 0.9209144214036475, + "pass@64": 0.9381434303380467, + "pass@128": 0.9529780564263323 + }, + "14": { + "pass@1": 0.7071651090342679, + "pass@2": 0.7924351344224491, + "pass@4": 0.8502455995570642, + "pass@8": 0.8910067656860042, + "pass@16": 0.9189231525733634, + "pass@32": 0.9373696512731893, + "pass@64": 0.9506183186383255, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.5974530346820809, + "pass@2": 0.7007393979563971, + "pass@4": 0.7801067936414015, + "pass@8": 0.8354042806919619, + "pass@16": 0.8730541335942021, + "pass@32": 0.901120904000133, + "pass@64": 0.9254484074093287, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5054939516129032, + "pass@2": 0.6082335852171704, + "pass@4": 0.6902914855401138, + "pass@8": 0.7560433401649518, + "pass@16": 0.807505375306526, + "pass@32": 0.8481175391774843, + "pass@64": 0.8783173741315493, + "pass@128": 0.896774193548387 + }, + "17": { + "pass@1": 0.3838059971098266, + "pass@2": 0.4859715647899499, + "pass@4": 0.575622420300786, + "pass@8": 0.655213423703959, + "pass@16": 0.7220266494245524, + "pass@32": 0.7722586027005358, + "pass@64": 0.8081715186093134, + "pass@128": 0.8352601156069365 + }, + "18": { + "pass@1": 0.32388392857142856, + "pass@2": 0.4244576068616422, + "pass@4": 0.5181791052011352, + "pass@8": 0.5980358612928792, + "pass@16": 0.6629633073108419, + "pass@32": 0.7142480405100232, + "pass@64": 0.7544531085968116, + "pass@128": 0.7942857142857143 + }, + "19": { + "pass@1": 0.2381522147147147, + "pass@2": 0.3297909129602043, + "pass@4": 0.4215563141882039, + "pass@8": 0.5039769605762625, + "pass@16": 0.5763823880150785, + "pass@32": 0.6406778469014253, + "pass@64": 0.6984631506742155, + "pass@128": 0.7567567567567568 + }, + "20": { + "pass@1": 0.2317239114114114, + "pass@2": 0.32413183360525893, + "pass@4": 0.41706644327116754, + "pass@8": 0.5007026327565921, + "pass@16": 0.5740754156310071, + "pass@32": 0.638377011630852, + "pass@64": 0.6956871639925536, + "pass@128": 0.7477477477477478 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 624159, + "answer_accuracy": 0.7601312840997662, + "avg_response_len": 285.43927684138737, + "resp_tokens_sum": 234379899, + "pass_at_k": { + "pass@1": 0.7601312840997662, + "pass@2": 0.8058778752738638, + "pass@4": 0.8434163727098335, + "pass@8": 0.8735857310959408, + "pass@16": 0.8969221776743685, + "pass@32": 0.9146164021260262, + "pass@64": 0.9286044999606458, + "pass@128": 0.9401402961808262 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9113065567484663, + "pass@2": 0.9251266545094442, + "pass@4": 0.9318315537436652, + "pass@8": 0.9359810259140621, + "pass@16": 0.9398201281022645, + "pass@32": 0.9447125150339829, + "pass@64": 0.9499553739007809, + "pass@128": 0.9539877300613497 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9988162878787878, + "pass@2": 0.9998777141493678, + "pass@4": 0.9999982255059028, + "pass@8": 0.999999999719363, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998613165680473, + "pass@2": 0.9999992720029819, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.989447463768116, + "pass@2": 0.9926609038000683, + "pass@4": 0.9946517704308702, + "pass@8": 0.9966518367905446, + "pass@16": 0.998582050313881, + "pass@32": 0.9997499573766314, + "pass@64": 0.9999971788090677, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9930157593123209, + "pass@2": 0.9951002442298582, + "pass@4": 0.9966461205243328, + "pass@8": 0.9976517473074589, + "pass@16": 0.9983273506604989, + "pass@32": 0.9991077492935924, + "pass@64": 0.99982934368161, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.993135838150289, + "pass@2": 0.9963200781712257, + "pass@4": 0.9979675979808883, + "pass@8": 0.9990151743820411, + "pass@16": 0.9997052474997874, + "pass@32": 0.9999794947349081, + "pass@64": 0.9999999848737747, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9828125, + "pass@2": 0.9905698957524677, + "pass@4": 0.9942337766758024, + "pass@8": 0.9968760475883579, + "pass@16": 0.9988561668827354, + "pass@32": 0.9998348548763891, + "pass@64": 0.9999981038504987, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9622941616766467, + "pass@2": 0.9767223672733275, + "pass@4": 0.9838516302602888, + "pass@8": 0.9882686825233068, + "pass@16": 0.990794735361916, + "pass@32": 0.9921813965674419, + "pass@64": 0.9932673675561231, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9145566641566265, + "pass@2": 0.9395704125557349, + "pass@4": 0.9531212250577107, + "pass@8": 0.9619663334454763, + "pass@16": 0.968723135248245, + "pass@32": 0.9746702926505119, + "pass@64": 0.9795887846110832, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8619954937304075, + "pass@2": 0.8864180170562533, + "pass@4": 0.9021936098113125, + "pass@8": 0.91375233313683, + "pass@16": 0.9201638977773448, + "pass@32": 0.9233086460149744, + "pass@64": 0.9255429242327472, + "pass@128": 0.9278996865203761 + }, + "13": { + "pass@1": 0.7585127801120448, + "pass@2": 0.8136451923288999, + "pass@4": 0.852209066828831, + "pass@8": 0.8806241733453731, + "pass@16": 0.9005599569445452, + "pass@32": 0.9153669801046609, + "pass@64": 0.9288086017030489, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.7070084064327485, + "pass@2": 0.7881876093613296, + "pass@4": 0.8460380092291095, + "pass@8": 0.8907571809918315, + "pass@16": 0.9247672939952509, + "pass@32": 0.9464667748863145, + "pass@64": 0.9590041317766209, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.5989109848484848, + "pass@2": 0.706179909329516, + "pass@4": 0.7829212430832506, + "pass@8": 0.8339317426580697, + "pass@16": 0.8695270739047846, + "pass@32": 0.8969910152454786, + "pass@64": 0.9193853522624728, + "pass@128": 0.9333333333333333 + }, + "16": { + "pass@1": 0.5016693376068376, + "pass@2": 0.6151490679049731, + "pass@4": 0.7033438189350254, + "pass@8": 0.769372720072636, + "pass@16": 0.8183349225214916, + "pass@32": 0.8549634298764497, + "pass@64": 0.8840951888891482, + "pass@128": 0.9088319088319088 + }, + "17": { + "pass@1": 0.41702251552795033, + "pass@2": 0.5324107754193774, + "pass@4": 0.6294041246591063, + "pass@8": 0.7014292101498082, + "pass@16": 0.7530650147190665, + "pass@32": 0.7912314076196781, + "pass@64": 0.8222645098320329, + "pass@128": 0.8478260869565217 + }, + "18": { + "pass@1": 0.3754825367647059, + "pass@2": 0.48911641674386275, + "pass@4": 0.5957133086305385, + "pass@8": 0.6875344124222568, + "pass@16": 0.7598632015899763, + "pass@32": 0.8117664354039354, + "pass@64": 0.8464364058423706, + "pass@128": 0.8676470588235294 + }, + "19": { + "pass@1": 0.23209635416666666, + "pass@2": 0.32691116247187857, + "pass@4": 0.4285349596367417, + "pass@8": 0.5260741360548356, + "pass@16": 0.6098688407896331, + "pass@32": 0.6770059830627879, + "pass@64": 0.7314934121455681, + "pass@128": 0.7767857142857143 + }, + "20": { + "pass@1": 0.22721744011976047, + "pass@2": 0.32192406525531636, + "pass@4": 0.4180934567872128, + "pass@8": 0.5052503046175902, + "pass@16": 0.5783695489331038, + "pass@32": 0.6389484435964482, + "pass@64": 0.6932115507430644, + "pass@128": 0.7485029940119761 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 604444, + "answer_accuracy": 0.7544685652660169, + "avg_response_len": 306.1050624600575, + "resp_tokens_sum": 245236683, + "pass_at_k": { + "pass@1": 0.7544685652660169, + "pass@2": 0.8013436635496822, + "pass@4": 0.8394358489129932, + "pass@8": 0.8695363733386458, + "pass@16": 0.8927416496812058, + "pass@32": 0.9107641991251118, + "pass@64": 0.9250602335578206, + "pass@128": 0.936571337274325 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9231110074626866, + "pass@2": 0.9381617992713601, + "pass@4": 0.9482348143982001, + "pass@8": 0.9569980467762143, + "pass@16": 0.9643792035984474, + "pass@32": 0.9699318877469423, + "pass@64": 0.974212970638659, + "pass@128": 0.9791044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997950072886297, + "pass@2": 0.9999921087670163, + "pass@4": 0.9999999904348691, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9985836988304093, + "pass@2": 0.9993416735736981, + "pass@4": 0.9998569655218537, + "pass@8": 0.9999939773438304, + "pass@16": 0.9999999936471988, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9802159926470588, + "pass@2": 0.9872854185965726, + "pass@4": 0.9926532946249366, + "pass@8": 0.9961444439272436, + "pass@16": 0.9981664618729108, + "pass@32": 0.9993048852819176, + "pass@64": 0.9999152319375018, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9974615536277602, + "pass@2": 0.9992905313097694, + "pass@4": 0.9999229347120254, + "pass@8": 0.9999991704144204, + "pass@16": 0.999999999947923, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9865259098101266, + "pass@2": 0.9916054364347653, + "pass@4": 0.9937736217506039, + "pass@8": 0.9953534321183053, + "pass@16": 0.9964274670231903, + "pass@32": 0.9968039802232598, + "pass@64": 0.9968354048433435, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9800347222222222, + "pass@2": 0.988521124470928, + "pass@4": 0.9913935138738287, + "pass@8": 0.9930488023084659, + "pass@16": 0.9946040308153634, + "pass@32": 0.9960989483322017, + "pass@64": 0.9969061152123114, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9608621382636656, + "pass@2": 0.9793176538724461, + "pass@4": 0.9882693851371472, + "pass@8": 0.9933905067759253, + "pass@16": 0.9959199013833268, + "pass@32": 0.9967107894337671, + "pass@64": 0.9967843633383671, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9130704365079365, + "pass@2": 0.9432527965254344, + "pass@4": 0.9579819689205518, + "pass@8": 0.9658760794360599, + "pass@16": 0.9717840228078343, + "pass@32": 0.9770501122402013, + "pass@64": 0.9816084524253677, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8835292431192661, + "pass@2": 0.916886768884394, + "pass@4": 0.9366000416805697, + "pass@8": 0.948232541229283, + "pass@16": 0.9564822968434049, + "pass@32": 0.9627454439943796, + "pass@64": 0.9677043744437711, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.7486255787037037, + "pass@2": 0.8188384004082823, + "pass@4": 0.8683277556161959, + "pass@8": 0.9025565767288988, + "pass@16": 0.9235507392469514, + "pass@32": 0.9356450050271856, + "pass@64": 0.9436096062229937, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.6807307121661721, + "pass@2": 0.7650419694385382, + "pass@4": 0.8188274625909145, + "pass@8": 0.8543898780663406, + "pass@16": 0.880504452164093, + "pass@32": 0.900971297832159, + "pass@64": 0.9185958359879355, + "pass@128": 0.9347181008902077 + }, + "15": { + "pass@1": 0.5972463348765432, + "pass@2": 0.6964904126567509, + "pass@4": 0.7714005413675139, + "pass@8": 0.8234241591543643, + "pass@16": 0.8579125416053235, + "pass@32": 0.8823457543058479, + "pass@64": 0.9015380471710855, + "pass@128": 0.9135802469135802 + }, + "16": { + "pass@1": 0.5104627581120944, + "pass@2": 0.6265714642417487, + "pass@4": 0.7175100075986074, + "pass@8": 0.7871632243973276, + "pass@16": 0.8387330545361368, + "pass@32": 0.8759289855806293, + "pass@64": 0.9029604022054188, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.3646460843373494, + "pass@2": 0.47297754008158627, + "pass@4": 0.5722734709930832, + "pass@8": 0.6531960855532634, + "pass@16": 0.714402703368858, + "pass@32": 0.7622974828119503, + "pass@64": 0.8021886087873138, + "pass@128": 0.8403614457831325 + }, + "18": { + "pass@1": 0.3394909274193548, + "pass@2": 0.44692024384048745, + "pass@4": 0.5442942435622966, + "pass@8": 0.6227225809486652, + "pass@16": 0.6832939555072691, + "pass@32": 0.7326495620246682, + "pass@64": 0.7739665652587125, + "pass@128": 0.8096774193548387 + }, + "19": { + "pass@1": 0.24287197885196374, + "pass@2": 0.3349259283488355, + "pass@4": 0.42498461417775973, + "pass@8": 0.5060636228148577, + "pass@16": 0.57703983808974, + "pass@32": 0.6393819838127462, + "pass@64": 0.6929985015863778, + "pass@128": 0.7341389728096677 + }, + "20": { + "pass@1": 0.21140671921921922, + "pass@2": 0.30761125298526865, + "pass@4": 0.41303933348421523, + "pass@8": 0.5142792716633172, + "pass@16": 0.6014466933381385, + "pass@32": 0.669939476467524, + "pass@64": 0.7203208811297246, + "pass@128": 0.7597597597597597 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch1-RL-op17-20/composition/graphs/composition_score_report.jsonl b/composition/Qwen3-100M-epoch1-RL-op17-20/composition/graphs/composition_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3847fa0820353dacacbaaea39051ac6e19604433 --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op17-20/composition/graphs/composition_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/Qwen3-100M-epoch1-RL-op17-20/composition", "results_dir": "results/composition/Qwen3-100M-epoch1-RL-op17-20/composition/graphs", "per_file": [{"file": "results/composition/Qwen3-100M-epoch1-RL-op17-20/composition/hf_model_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 996376, "value_mismatch": 33306, "dependency_mismatch": 147021, "answer_mismatch": 30846, "extra_nodes": 15012, "missing_nodes": 10554, "perfect_with_extra": 820, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 328264, "value_mismatch": 11557, "dependency_mismatch": 48184, "answer_mismatch": 10615, "extra_nodes": 5033, "missing_nodes": 3440, "perfect_with_extra": 258}, "teachers_in_school": {"total": 390656, "perfect": 337697, "value_mismatch": 11238, "dependency_mismatch": 49626, "answer_mismatch": 10565, "extra_nodes": 5064, "missing_nodes": 3865, "perfect_with_extra": 125}, "movie_festival_awards": {"total": 382336, "perfect": 330415, "value_mismatch": 10511, "dependency_mismatch": 49211, "answer_mismatch": 9666, "extra_nodes": 4915, "missing_nodes": 3249, "perfect_with_extra": 437}}, "results_path": "results/composition/Qwen3-100M-epoch1-RL-op17-20/composition/graphs/hf_model_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/Qwen3-100M-epoch1-RL-op17-20/composition/hf_model_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 280869, "value_mismatch": 765484, "dependency_mismatch": 944765, "answer_mismatch": 610280, "extra_nodes": 47863, "missing_nodes": 180496, "perfect_with_extra": 168, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 258133, "dependency_mismatch": 319988, "answer_mismatch": 204689, "extra_nodes": 17156, "missing_nodes": 63128, "perfect": 93018, "perfect_with_extra": 40}, "movie_festival_awards": {"total": 418816, "perfect": 91074, "value_mismatch": 251135, "dependency_mismatch": 310281, "answer_mismatch": 200970, "extra_nodes": 15874, "missing_nodes": 59294, "perfect_with_extra": 4}, "teachers_in_school": {"total": 430464, "perfect": 96777, "value_mismatch": 256216, "dependency_mismatch": 314496, "answer_mismatch": 204621, "extra_nodes": 14833, "missing_nodes": 58074, "perfect_with_extra": 124}}, "results_path": "results/composition/Qwen3-100M-epoch1-RL-op17-20/composition/graphs/hf_model_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1277245, "value_mismatch": 798790, "dependency_mismatch": 1091786, "answer_mismatch": 641126, "extra_nodes": 62875, "missing_nodes": 191050, "perfect_with_extra": 988, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/Qwen3-100M-epoch1-RL-op17-20/composition/graphs/composition_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 421282, "value_mismatch": 269690, "dependency_mismatch": 368172, "answer_mismatch": 215304, "extra_nodes": 22189, "missing_nodes": 66568, "perfect_with_extra": 298}, "teachers_in_school": {"total": 821120, "perfect": 434474, "value_mismatch": 267454, "dependency_mismatch": 364122, "answer_mismatch": 215186, "extra_nodes": 19897, "missing_nodes": 61939, "perfect_with_extra": 249}, "movie_festival_awards": {"total": 801152, "perfect": 421489, "value_mismatch": 261646, "dependency_mismatch": 359492, "answer_mismatch": 210636, "extra_nodes": 20789, "missing_nodes": 62543, "perfect_with_extra": 441}}}} diff --git a/composition/Qwen3-100M-epoch1-RL-op17-20/composition/graphs/results.json b/composition/Qwen3-100M-epoch1-RL-op17-20/composition/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..a563839e2aeb6bc09266431e17c66850951e0c48 --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op17-20/composition/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "hf_model_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9938837920489296, + "pass@2": 0.9969418960244648, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.996875, + "pass@4": 0.996875, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9650793650793651, + "pass@2": 0.9841269841269841, + "pass@4": 0.9936507936507937, + "pass@8": 0.9936507936507937, + "pass@16": 0.9936507936507937, + "pass@32": 0.9968253968253968, + "pass@64": 0.9968253968253968, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9580838323353293, + "pass@2": 0.9730538922155688, + "pass@4": 0.9880239520958084, + "pass@8": 0.9940119760479041, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8757396449704142, + "pass@2": 0.9230769230769231, + "pass@4": 0.9615384615384616, + "pass@8": 0.985207100591716, + "pass@16": 0.9911242603550295, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.75, + "pass@2": 0.8589743589743589, + "pass@4": 0.9198717948717948, + "pass@8": 0.9711538461538461, + "pass@16": 0.9903846153846154, + "pass@32": 0.9967948717948718, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7070422535211267, + "pass@2": 0.8084507042253521, + "pass@4": 0.8788732394366198, + "pass@8": 0.9436619718309859, + "pass@16": 0.9746478873239437, + "pass@32": 0.9915492957746479, + "pass@64": 0.9943661971830986, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.5899705014749262, + "pass@2": 0.6548672566371682, + "pass@4": 0.7640117994100295, + "pass@8": 0.8525073746312685, + "pass@16": 0.887905604719764, + "pass@32": 0.9144542772861357, + "pass@64": 0.9351032448377581, + "pass@128": 0.9380530973451328 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9938837920489296, + "4": 0.99375, + "5": 0.9650793650793651, + "6": 0.9580838323353293, + "7": 0.8757396449704142, + "8": 0.75, + "9": 0.7070422535211267, + "10": 0.5899705014749262 + }, + "pass@2": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 0.996875, + "5": 0.9841269841269841, + "6": 0.9730538922155688, + "7": 0.9230769230769231, + "8": 0.8589743589743589, + "9": 0.8084507042253521, + "10": 0.6548672566371682 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 0.9880239520958084, + "7": 0.9615384615384616, + "8": 0.9198717948717948, + "9": 0.8788732394366198, + "10": 0.7640117994100295 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 0.9940119760479041, + "7": 0.985207100591716, + "8": 0.9711538461538461, + "9": 0.9436619718309859, + "10": 0.8525073746312685 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 1.0, + "7": 0.9911242603550295, + "8": 0.9903846153846154, + "9": 0.9746478873239437, + "10": 0.887905604719764 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 0.9915492957746479, + "10": 0.9144542772861357 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9970414201183432, + "8": 1.0, + "9": 0.9943661971830986, + "10": 0.9351032448377581 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9970414201183432, + "8": 1.0, + "9": 1.0, + "10": 0.9380530973451328 + } + }, + "total_pass_at_k": { + "pass@1": 0.8676122931442081, + "pass@2": 0.9084768659236744, + "pass@4": 0.9432624113475178, + "pass@8": 0.9702803107058426, + "pass@16": 0.9814251941911516, + "pass@32": 0.9881796690307328, + "pass@64": 0.9912191827085444, + "pass@128": 0.9922323539344816 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970845481049563, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 0.9941520467836257, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9470588235294117, + "pass@2": 0.9705882352941176, + "pass@4": 0.9852941176470589, + "pass@8": 0.9941176470588236, + "pass@16": 0.9970588235294118, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9652996845425867, + "pass@2": 0.9810725552050473, + "pass@4": 0.9968454258675079, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8512658227848101, + "pass@2": 0.8955696202531646, + "pass@4": 0.9335443037974683, + "pass@8": 0.9715189873417721, + "pass@16": 0.9936708860759493, + "pass@32": 0.9968354430379747, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7537537537537538, + "pass@2": 0.8498498498498499, + "pass@4": 0.9009009009009009, + "pass@8": 0.9519519519519519, + "pass@16": 0.978978978978979, + "pass@32": 0.996996996996997, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.7202572347266881, + "pass@2": 0.8070739549839229, + "pass@4": 0.8842443729903537, + "pass@8": 0.9196141479099679, + "pass@16": 0.9678456591639871, + "pass@32": 0.9967845659163987, + "pass@64": 0.9967845659163987, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.5253731343283582, + "pass@2": 0.6238805970149254, + "pass@4": 0.7223880597014926, + "pass@8": 0.8208955223880597, + "pass@16": 0.8925373134328358, + "pass@32": 0.9223880597014925, + "pass@64": 0.9343283582089552, + "pass@128": 0.9492537313432836 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9883040935672515, + "5": 0.9470588235294117, + "6": 0.9652996845425867, + "7": 0.8512658227848101, + "8": 0.7537537537537538, + "9": 0.7202572347266881, + "10": 0.5253731343283582 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9705882352941176, + "6": 0.9810725552050473, + "7": 0.8955696202531646, + "8": 0.8498498498498499, + "9": 0.8070739549839229, + "10": 0.6238805970149254 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9852941176470589, + "6": 0.9968454258675079, + "7": 0.9335443037974683, + "8": 0.9009009009009009, + "9": 0.8842443729903537, + "10": 0.7223880597014926 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9941176470588236, + "6": 1.0, + "7": 0.9715189873417721, + "8": 0.9519519519519519, + "9": 0.9196141479099679, + "10": 0.8208955223880597 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.978978978978979, + "9": 0.9678456591639871, + "10": 0.8925373134328358 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.996996996996997, + "9": 0.9967845659163987, + "10": 0.9223880597014925 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 0.9967845659163987, + "10": 0.9343283582089552 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9492537313432836 + } + }, + "total_pass_at_k": { + "pass@1": 0.863073317710077, + "pass@2": 0.9039169735520589, + "pass@4": 0.9367258118513558, + "pass@8": 0.9625041848008035, + "pass@16": 0.9812520924004018, + "pass@32": 0.9902912621359223, + "pass@64": 0.991965182457315, + "pass@128": 0.9939738868429863 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9969604863221885, + "pass@2": 0.9969604863221885, + "pass@4": 0.9969604863221885, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996969696969697, + "pass@2": 0.996969696969697, + "pass@4": 0.996969696969697, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9674556213017751, + "pass@2": 0.9822485207100592, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9536231884057971, + "pass@2": 0.9797101449275363, + "pass@4": 0.9884057971014493, + "pass@8": 0.9971014492753624, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9570200573065902, + "pass@2": 0.9713467048710601, + "pass@4": 0.9885386819484241, + "pass@8": 0.994269340974212, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8439306358381503, + "pass@2": 0.8959537572254336, + "pass@4": 0.9335260115606936, + "pass@8": 0.9826589595375722, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7690140845070422, + "pass@2": 0.828169014084507, + "pass@4": 0.9183098591549296, + "pass@8": 0.9549295774647887, + "pass@16": 0.9774647887323944, + "pass@32": 0.9830985915492958, + "pass@64": 0.9887323943661972, + "pass@128": 0.9887323943661972 + }, + "9": { + "pass@1": 0.6646706586826348, + "pass@2": 0.7544910179640718, + "pass@4": 0.8263473053892215, + "pass@8": 0.907185628742515, + "pass@16": 0.9610778443113772, + "pass@32": 0.9820359281437125, + "pass@64": 0.9910179640718563, + "pass@128": 0.9910179640718563 + }, + "10": { + "pass@1": 0.6196319018404908, + "pass@2": 0.6840490797546013, + "pass@4": 0.7668711656441718, + "pass@8": 0.8374233128834356, + "pass@16": 0.8834355828220859, + "pass@32": 0.901840490797546, + "pass@64": 0.911042944785276, + "pass@128": 0.9233128834355828 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9969604863221885, + "3": 0.996969696969697, + "4": 0.9674556213017751, + "5": 0.9536231884057971, + "6": 0.9570200573065902, + "7": 0.8439306358381503, + "8": 0.7690140845070422, + "9": 0.6646706586826348, + "10": 0.6196319018404908 + }, + "pass@2": { + "2": 0.9969604863221885, + "3": 0.996969696969697, + "4": 0.9822485207100592, + "5": 0.9797101449275363, + "6": 0.9713467048710601, + "7": 0.8959537572254336, + "8": 0.828169014084507, + "9": 0.7544910179640718, + "10": 0.6840490797546013 + }, + "pass@4": { + "2": 0.9969604863221885, + "3": 0.996969696969697, + "4": 1.0, + "5": 0.9884057971014493, + "6": 0.9885386819484241, + "7": 0.9335260115606936, + "8": 0.9183098591549296, + "9": 0.8263473053892215, + "10": 0.7668711656441718 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.994269340974212, + "7": 0.9826589595375722, + "8": 0.9549295774647887, + "9": 0.907185628742515, + "10": 0.8374233128834356 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9774647887323944, + "9": 0.9610778443113772, + "10": 0.8834355828220859 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9830985915492958, + "9": 0.9820359281437125, + "10": 0.901840490797546 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9887323943661972, + "9": 0.9910179640718563, + "10": 0.911042944785276 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9887323943661972, + "9": 0.9910179640718563, + "10": 0.9233128834355828 + } + }, + "total_pass_at_k": { + "pass@1": 0.8636959370904325, + "pass@2": 0.8994102228047182, + "pass@4": 0.9357798165137615, + "pass@8": 0.9642857142857143, + "pass@16": 0.9806684141546527, + "pass@32": 0.9855832241153342, + "pass@64": 0.9882044560943644, + "pass@128": 0.9895150720838795 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.999, + "pass@2": 0.999, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996, + "pass@2": 0.998, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.983, + "pass@2": 0.991, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.955, + "pass@2": 0.978, + "pass@4": 0.989, + "pass@8": 0.995, + "pass@16": 0.997, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.96, + "pass@2": 0.975, + "pass@4": 0.991, + "pass@8": 0.996, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.857, + "pass@2": 0.905, + "pass@4": 0.943, + "pass@8": 0.98, + "pass@16": 0.995, + "pass@32": 0.998, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.758, + "pass@2": 0.845, + "pass@4": 0.913, + "pass@8": 0.959, + "pass@16": 0.982, + "pass@32": 0.992, + "pass@64": 0.995, + "pass@128": 0.995 + }, + "9": { + "pass@1": 0.697, + "pass@2": 0.79, + "pass@4": 0.863, + "pass@8": 0.924, + "pass@16": 0.968, + "pass@32": 0.99, + "pass@64": 0.994, + "pass@128": 0.997 + }, + "10": { + "pass@1": 0.578, + "pass@2": 0.654, + "pass@4": 0.751, + "pass@8": 0.837, + "pass@16": 0.888, + "pass@32": 0.913, + "pass@64": 0.927, + "pass@128": 0.937 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.999, + "3": 0.996, + "4": 0.983, + "5": 0.955, + "6": 0.96, + "7": 0.857, + "8": 0.758, + "9": 0.697, + "10": 0.578 + }, + "pass@2": { + "2": 0.999, + "3": 0.998, + "4": 0.991, + "5": 0.978, + "6": 0.975, + "7": 0.905, + "8": 0.845, + "9": 0.79, + "10": 0.654 + }, + "pass@4": { + "2": 0.999, + "3": 0.999, + "4": 0.999, + "5": 0.989, + "6": 0.991, + "7": 0.943, + "8": 0.913, + "9": 0.863, + "10": 0.751 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.995, + "6": 0.996, + "7": 0.98, + "8": 0.959, + "9": 0.924, + "10": 0.837 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.997, + "6": 1.0, + "7": 0.995, + "8": 0.982, + "9": 0.968, + "10": 0.888 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.998, + "8": 0.992, + "9": 0.99, + "10": 0.913 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.999, + "8": 0.995, + "9": 0.994, + "10": 0.927 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.999, + "8": 0.995, + "9": 0.997, + "10": 0.937 + } + }, + "total_pass_at_k": { + "pass@1": 0.8647777777777778, + "pass@2": 0.9038888888888889, + "pass@4": 0.9385555555555556, + "pass@8": 0.9656666666666667, + "pass@16": 0.9811111111111112, + "pass@32": 0.988, + "pass@64": 0.9904444444444445, + "pass@128": 0.9918888888888889 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "hf_model_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.46175637393767704, + "pass@2": 0.56657223796034, + "pass@4": 0.6912181303116147, + "pass@8": 0.7847025495750708, + "pass@16": 0.8526912181303116, + "pass@32": 0.8895184135977338, + "pass@64": 0.9178470254957507, + "pass@128": 0.9291784702549575 + }, + "12": { + "pass@1": 0.4915254237288136, + "pass@2": 0.5790960451977402, + "pass@4": 0.6610169491525424, + "pass@8": 0.7062146892655368, + "pass@16": 0.7570621468926554, + "pass@32": 0.7966101694915254, + "pass@64": 0.8333333333333334, + "pass@128": 0.8502824858757062 + }, + "13": { + "pass@1": 0.32601880877742945, + "pass@2": 0.438871473354232, + "pass@4": 0.5015673981191222, + "pass@8": 0.5768025078369906, + "pass@16": 0.6394984326018809, + "pass@32": 0.6927899686520376, + "pass@64": 0.7272727272727273, + "pass@128": 0.7492163009404389 + }, + "14": { + "pass@1": 0.2834890965732087, + "pass@2": 0.35202492211838005, + "pass@4": 0.4236760124610592, + "pass@8": 0.49221183800623053, + "pass@16": 0.557632398753894, + "pass@32": 0.6261682242990654, + "pass@64": 0.6791277258566978, + "pass@128": 0.7289719626168224 + }, + "15": { + "pass@1": 0.21676300578034682, + "pass@2": 0.2658959537572254, + "pass@4": 0.3236994219653179, + "pass@8": 0.3786127167630058, + "pass@16": 0.4393063583815029, + "pass@32": 0.5144508670520231, + "pass@64": 0.6069364161849711, + "pass@128": 0.6820809248554913 + }, + "16": { + "pass@1": 0.16451612903225807, + "pass@2": 0.2129032258064516, + "pass@4": 0.2645161290322581, + "pass@8": 0.3, + "pass@16": 0.33225806451612905, + "pass@32": 0.38387096774193546, + "pass@64": 0.43548387096774194, + "pass@128": 0.4645161290322581 + }, + "17": { + "pass@1": 0.07803468208092486, + "pass@2": 0.12138728323699421, + "pass@4": 0.15606936416184972, + "pass@8": 0.19653179190751446, + "pass@16": 0.24566473988439305, + "pass@32": 0.30346820809248554, + "pass@64": 0.3468208092485549, + "pass@128": 0.407514450867052 + }, + "18": { + "pass@1": 0.07428571428571429, + "pass@2": 0.08857142857142856, + "pass@4": 0.12571428571428572, + "pass@8": 0.15714285714285714, + "pass@16": 0.2057142857142857, + "pass@32": 0.22285714285714286, + "pass@64": 0.2742857142857143, + "pass@128": 0.3142857142857143 + }, + "19": { + "pass@1": 0.02702702702702703, + "pass@2": 0.05105105105105105, + "pass@4": 0.06606606606606606, + "pass@8": 0.09309309309309309, + "pass@16": 0.12012012012012012, + "pass@32": 0.14114114114114115, + "pass@64": 0.16816816816816818, + "pass@128": 0.2042042042042042 + }, + "20": { + "pass@1": 0.03903903903903904, + "pass@2": 0.05405405405405406, + "pass@4": 0.07507507507507508, + "pass@8": 0.1021021021021021, + "pass@16": 0.12912912912912913, + "pass@32": 0.15015015015015015, + "pass@64": 0.18018018018018017, + "pass@128": 0.2012012012012012 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.46175637393767704, + "12": 0.4915254237288136, + "13": 0.32601880877742945, + "14": 0.2834890965732087, + "15": 0.21676300578034682, + "16": 0.16451612903225807, + "17": 0.07803468208092486, + "18": 0.07428571428571429, + "19": 0.02702702702702703, + "20": 0.03903903903903904 + }, + "pass@2": { + "11": 0.56657223796034, + "12": 0.5790960451977402, + "13": 0.438871473354232, + "14": 0.35202492211838005, + "15": 0.2658959537572254, + "16": 0.2129032258064516, + "17": 0.12138728323699421, + "18": 0.08857142857142856, + "19": 0.05105105105105105, + "20": 0.05405405405405406 + }, + "pass@4": { + "11": 0.6912181303116147, + "12": 0.6610169491525424, + "13": 0.5015673981191222, + "14": 0.4236760124610592, + "15": 0.3236994219653179, + "16": 0.2645161290322581, + "17": 0.15606936416184972, + "18": 0.12571428571428572, + "19": 0.06606606606606606, + "20": 0.07507507507507508 + }, + "pass@8": { + "11": 0.7847025495750708, + "12": 0.7062146892655368, + "13": 0.5768025078369906, + "14": 0.49221183800623053, + "15": 0.3786127167630058, + "16": 0.3, + "17": 0.19653179190751446, + "18": 0.15714285714285714, + "19": 0.09309309309309309, + "20": 0.1021021021021021 + }, + "pass@16": { + "11": 0.8526912181303116, + "12": 0.7570621468926554, + "13": 0.6394984326018809, + "14": 0.557632398753894, + "15": 0.4393063583815029, + "16": 0.33225806451612905, + "17": 0.24566473988439305, + "18": 0.2057142857142857, + "19": 0.12012012012012012, + "20": 0.12912912912912913 + }, + "pass@32": { + "11": 0.8895184135977338, + "12": 0.7966101694915254, + "13": 0.6927899686520376, + "14": 0.6261682242990654, + "15": 0.5144508670520231, + "16": 0.38387096774193546, + "17": 0.30346820809248554, + "18": 0.22285714285714286, + "19": 0.14114114114114115, + "20": 0.15015015015015015 + }, + "pass@64": { + "11": 0.9178470254957507, + "12": 0.8333333333333334, + "13": 0.7272727272727273, + "14": 0.6791277258566978, + "15": 0.6069364161849711, + "16": 0.43548387096774194, + "17": 0.3468208092485549, + "18": 0.2742857142857143, + "19": 0.16816816816816818, + "20": 0.18018018018018017 + }, + "pass@128": { + "11": 0.9291784702549575, + "12": 0.8502824858757062, + "13": 0.7492163009404389, + "14": 0.7289719626168224, + "15": 0.6820809248554913, + "16": 0.4645161290322581, + "17": 0.407514450867052, + "18": 0.3142857142857143, + "19": 0.2042042042042042, + "20": 0.2012012012012012 + } + }, + "total_pass_at_k": { + "pass@1": 0.21783060921248143, + "pass@2": 0.274591381872214, + "pass@4": 0.33075780089153045, + "pass@8": 0.38068350668647843, + "pass@16": 0.43001485884101043, + "pass@32": 0.4739970282317979, + "pass@64": 0.5188707280832096, + "pass@128": 0.5551263001485884 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.4857142857142857, + "pass@2": 0.5841269841269842, + "pass@4": 0.6825396825396826, + "pass@8": 0.7904761904761904, + "pass@16": 0.8476190476190476, + "pass@32": 0.8920634920634921, + "pass@64": 0.9365079365079365, + "pass@128": 0.9523809523809523 + }, + "12": { + "pass@1": 0.4923547400611621, + "pass@2": 0.5871559633027523, + "pass@4": 0.6819571865443425, + "pass@8": 0.7370030581039755, + "pass@16": 0.8042813455657493, + "pass@32": 0.8409785932721713, + "pass@64": 0.8685015290519877, + "pass@128": 0.8807339449541285 + }, + "13": { + "pass@1": 0.31790123456790126, + "pass@2": 0.39197530864197533, + "pass@4": 0.4876543209876543, + "pass@8": 0.5895061728395061, + "pass@16": 0.654320987654321, + "pass@32": 0.6882716049382716, + "pass@64": 0.7160493827160493, + "pass@128": 0.7314814814814815 + }, + "14": { + "pass@1": 0.22551928783382788, + "pass@2": 0.2997032640949555, + "pass@4": 0.37388724035608306, + "pass@8": 0.4599406528189911, + "pass@16": 0.5044510385756676, + "pass@32": 0.5845697329376854, + "pass@64": 0.6676557863501483, + "pass@128": 0.6943620178041543 + }, + "15": { + "pass@1": 0.2037037037037037, + "pass@2": 0.2808641975308642, + "pass@4": 0.3425925925925926, + "pass@8": 0.39814814814814814, + "pass@16": 0.4382716049382716, + "pass@32": 0.5154320987654321, + "pass@64": 0.5925925925925926, + "pass@128": 0.6759259259259259 + }, + "16": { + "pass@1": 0.168141592920354, + "pass@2": 0.22713864306784662, + "pass@4": 0.3008849557522124, + "pass@8": 0.336283185840708, + "pass@16": 0.3775811209439528, + "pass@32": 0.41887905604719766, + "pass@64": 0.43952802359882004, + "pass@128": 0.48672566371681414 + }, + "17": { + "pass@1": 0.09939759036144578, + "pass@2": 0.15060240963855423, + "pass@4": 0.19578313253012047, + "pass@8": 0.24096385542168675, + "pass@16": 0.27710843373493976, + "pass@32": 0.3253012048192771, + "pass@64": 0.3674698795180723, + "pass@128": 0.4066265060240964 + }, + "18": { + "pass@1": 0.08709677419354839, + "pass@2": 0.11612903225806452, + "pass@4": 0.13548387096774195, + "pass@8": 0.16451612903225807, + "pass@16": 0.1967741935483871, + "pass@32": 0.22903225806451613, + "pass@64": 0.27741935483870966, + "pass@128": 0.3193548387096774 + }, + "19": { + "pass@1": 0.030211480362537766, + "pass@2": 0.054380664652567974, + "pass@4": 0.06646525679758308, + "pass@8": 0.09063444108761329, + "pass@16": 0.11782477341389729, + "pass@32": 0.1419939577039275, + "pass@64": 0.16314199395770393, + "pass@128": 0.19939577039274925 + }, + "20": { + "pass@1": 0.03303303303303303, + "pass@2": 0.04504504504504504, + "pass@4": 0.06906906906906907, + "pass@8": 0.09309309309309309, + "pass@16": 0.1111111111111111, + "pass@32": 0.12612612612612611, + "pass@64": 0.13513513513513514, + "pass@128": 0.15315315315315314 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.4857142857142857, + "12": 0.4923547400611621, + "13": 0.31790123456790126, + "14": 0.22551928783382788, + "15": 0.2037037037037037, + "16": 0.168141592920354, + "17": 0.09939759036144578, + "18": 0.08709677419354839, + "19": 0.030211480362537766, + "20": 0.03303303303303303 + }, + "pass@2": { + "11": 0.5841269841269842, + "12": 0.5871559633027523, + "13": 0.39197530864197533, + "14": 0.2997032640949555, + "15": 0.2808641975308642, + "16": 0.22713864306784662, + "17": 0.15060240963855423, + "18": 0.11612903225806452, + "19": 0.054380664652567974, + "20": 0.04504504504504504 + }, + "pass@4": { + "11": 0.6825396825396826, + "12": 0.6819571865443425, + "13": 0.4876543209876543, + "14": 0.37388724035608306, + "15": 0.3425925925925926, + "16": 0.3008849557522124, + "17": 0.19578313253012047, + "18": 0.13548387096774195, + "19": 0.06646525679758308, + "20": 0.06906906906906907 + }, + "pass@8": { + "11": 0.7904761904761904, + "12": 0.7370030581039755, + "13": 0.5895061728395061, + "14": 0.4599406528189911, + "15": 0.39814814814814814, + "16": 0.336283185840708, + "17": 0.24096385542168675, + "18": 0.16451612903225807, + "19": 0.09063444108761329, + "20": 0.09309309309309309 + }, + "pass@16": { + "11": 0.8476190476190476, + "12": 0.8042813455657493, + "13": 0.654320987654321, + "14": 0.5044510385756676, + "15": 0.4382716049382716, + "16": 0.3775811209439528, + "17": 0.27710843373493976, + "18": 0.1967741935483871, + "19": 0.11782477341389729, + "20": 0.1111111111111111 + }, + "pass@32": { + "11": 0.8920634920634921, + "12": 0.8409785932721713, + "13": 0.6882716049382716, + "14": 0.5845697329376854, + "15": 0.5154320987654321, + "16": 0.41887905604719766, + "17": 0.3253012048192771, + "18": 0.22903225806451613, + "19": 0.1419939577039275, + "20": 0.12612612612612611 + }, + "pass@64": { + "11": 0.9365079365079365, + "12": 0.8685015290519877, + "13": 0.7160493827160493, + "14": 0.6676557863501483, + "15": 0.5925925925925926, + "16": 0.43952802359882004, + "17": 0.3674698795180723, + "18": 0.27741935483870966, + "19": 0.16314199395770393, + "20": 0.13513513513513514 + }, + "pass@128": { + "11": 0.9523809523809523, + "12": 0.8807339449541285, + "13": 0.7314814814814815, + "14": 0.6943620178041543, + "15": 0.6759259259259259, + "16": 0.48672566371681414, + "17": 0.4066265060240964, + "18": 0.3193548387096774, + "19": 0.19939577039274925, + "20": 0.15315315315315314 + } + }, + "total_pass_at_k": { + "pass@1": 0.21301955990220048, + "pass@2": 0.2723105134474328, + "pass@4": 0.3322127139364303, + "pass@8": 0.38844743276283616, + "pass@16": 0.43123471882640585, + "pass@32": 0.47463325183374083, + "pass@64": 0.5146699266503667, + "pass@128": 0.5482885085574573 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5391566265060241, + "pass@2": 0.6325301204819277, + "pass@4": 0.7108433734939759, + "pass@8": 0.8012048192771084, + "pass@16": 0.8524096385542169, + "pass@32": 0.9006024096385542, + "pass@64": 0.9246987951807228, + "pass@128": 0.9397590361445783 + }, + "12": { + "pass@1": 0.4670846394984326, + "pass@2": 0.5736677115987461, + "pass@4": 0.6551724137931034, + "pass@8": 0.7366771159874608, + "pass@16": 0.7774294670846394, + "pass@32": 0.8213166144200627, + "pass@64": 0.8338557993730408, + "pass@128": 0.8526645768025078 + }, + "13": { + "pass@1": 0.3557422969187675, + "pass@2": 0.4425770308123249, + "pass@4": 0.5042016806722689, + "pass@8": 0.5882352941176471, + "pass@16": 0.6582633053221288, + "pass@32": 0.7310924369747899, + "pass@64": 0.773109243697479, + "pass@128": 0.8067226890756303 + }, + "14": { + "pass@1": 0.30994152046783624, + "pass@2": 0.347953216374269, + "pass@4": 0.42105263157894735, + "pass@8": 0.47953216374269003, + "pass@16": 0.543859649122807, + "pass@32": 0.6111111111111112, + "pass@64": 0.695906432748538, + "pass@128": 0.7397660818713451 + }, + "15": { + "pass@1": 0.2, + "pass@2": 0.2606060606060606, + "pass@4": 0.33636363636363636, + "pass@8": 0.396969696969697, + "pass@16": 0.4484848484848485, + "pass@32": 0.5303030303030303, + "pass@64": 0.593939393939394, + "pass@128": 0.6666666666666666 + }, + "16": { + "pass@1": 0.16809116809116809, + "pass@2": 0.20797720797720798, + "pass@4": 0.25071225071225073, + "pass@8": 0.29914529914529914, + "pass@16": 0.3561253561253561, + "pass@32": 0.38461538461538464, + "pass@64": 0.45584045584045585, + "pass@128": 0.5071225071225072 + }, + "17": { + "pass@1": 0.11801242236024845, + "pass@2": 0.15527950310559005, + "pass@4": 0.19875776397515527, + "pass@8": 0.22981366459627328, + "pass@16": 0.2732919254658385, + "pass@32": 0.30745341614906835, + "pass@64": 0.35093167701863354, + "pass@128": 0.391304347826087 + }, + "18": { + "pass@1": 0.08823529411764706, + "pass@2": 0.11764705882352941, + "pass@4": 0.14705882352941177, + "pass@8": 0.17941176470588235, + "pass@16": 0.20588235294117646, + "pass@32": 0.26176470588235295, + "pass@64": 0.27941176470588236, + "pass@128": 0.31176470588235294 + }, + "19": { + "pass@1": 0.03273809523809524, + "pass@2": 0.044642857142857144, + "pass@4": 0.05952380952380952, + "pass@8": 0.08333333333333333, + "pass@16": 0.12797619047619047, + "pass@32": 0.13988095238095238, + "pass@64": 0.16666666666666666, + "pass@128": 0.20535714285714285 + }, + "20": { + "pass@1": 0.02694610778443114, + "pass@2": 0.04491017964071856, + "pass@4": 0.0688622754491018, + "pass@8": 0.08982035928143713, + "pass@16": 0.11676646706586827, + "pass@32": 0.1347305389221557, + "pass@64": 0.15269461077844312, + "pass@128": 0.17365269461077845 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5391566265060241, + "12": 0.4670846394984326, + "13": 0.3557422969187675, + "14": 0.30994152046783624, + "15": 0.2, + "16": 0.16809116809116809, + "17": 0.11801242236024845, + "18": 0.08823529411764706, + "19": 0.03273809523809524, + "20": 0.02694610778443114 + }, + "pass@2": { + "11": 0.6325301204819277, + "12": 0.5736677115987461, + "13": 0.4425770308123249, + "14": 0.347953216374269, + "15": 0.2606060606060606, + "16": 0.20797720797720798, + "17": 0.15527950310559005, + "18": 0.11764705882352941, + "19": 0.044642857142857144, + "20": 0.04491017964071856 + }, + "pass@4": { + "11": 0.7108433734939759, + "12": 0.6551724137931034, + "13": 0.5042016806722689, + "14": 0.42105263157894735, + "15": 0.33636363636363636, + "16": 0.25071225071225073, + "17": 0.19875776397515527, + "18": 0.14705882352941177, + "19": 0.05952380952380952, + "20": 0.0688622754491018 + }, + "pass@8": { + "11": 0.8012048192771084, + "12": 0.7366771159874608, + "13": 0.5882352941176471, + "14": 0.47953216374269003, + "15": 0.396969696969697, + "16": 0.29914529914529914, + "17": 0.22981366459627328, + "18": 0.17941176470588235, + "19": 0.08333333333333333, + "20": 0.08982035928143713 + }, + "pass@16": { + "11": 0.8524096385542169, + "12": 0.7774294670846394, + "13": 0.6582633053221288, + "14": 0.543859649122807, + "15": 0.4484848484848485, + "16": 0.3561253561253561, + "17": 0.2732919254658385, + "18": 0.20588235294117646, + "19": 0.12797619047619047, + "20": 0.11676646706586827 + }, + "pass@32": { + "11": 0.9006024096385542, + "12": 0.8213166144200627, + "13": 0.7310924369747899, + "14": 0.6111111111111112, + "15": 0.5303030303030303, + "16": 0.38461538461538464, + "17": 0.30745341614906835, + "18": 0.26176470588235295, + "19": 0.13988095238095238, + "20": 0.1347305389221557 + }, + "pass@64": { + "11": 0.9246987951807228, + "12": 0.8338557993730408, + "13": 0.773109243697479, + "14": 0.695906432748538, + "15": 0.593939393939394, + "16": 0.45584045584045585, + "17": 0.35093167701863354, + "18": 0.27941176470588236, + "19": 0.16666666666666666, + "20": 0.15269461077844312 + }, + "pass@128": { + "11": 0.9397590361445783, + "12": 0.8526645768025078, + "13": 0.8067226890756303, + "14": 0.7397660818713451, + "15": 0.6666666666666666, + "16": 0.5071225071225072, + "17": 0.391304347826087, + "18": 0.31176470588235294, + "19": 0.20535714285714285, + "20": 0.17365269461077845 + } + }, + "total_pass_at_k": { + "pass@1": 0.23015165031222123, + "pass@2": 0.28218852215283974, + "pass@4": 0.3345227475468332, + "pass@8": 0.38774903360095153, + "pass@16": 0.4356229556943205, + "pass@32": 0.48201011002081473, + "pass@64": 0.5227475468331847, + "pass@128": 0.55961938745168 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.495, + "pass@2": 0.594, + "pass@4": 0.695, + "pass@8": 0.792, + "pass@16": 0.851, + "pass@32": 0.894, + "pass@64": 0.926, + "pass@128": 0.94 + }, + "12": { + "pass@1": 0.484, + "pass@2": 0.58, + "pass@4": 0.666, + "pass@8": 0.726, + "pass@16": 0.779, + "pass@32": 0.819, + "pass@64": 0.845, + "pass@128": 0.861 + }, + "13": { + "pass@1": 0.334, + "pass@2": 0.425, + "pass@4": 0.498, + "pass@8": 0.585, + "pass@16": 0.651, + "pass@32": 0.705, + "pass@64": 0.74, + "pass@128": 0.764 + }, + "14": { + "pass@1": 0.273, + "pass@2": 0.333, + "pass@4": 0.406, + "pass@8": 0.477, + "pass@16": 0.535, + "pass@32": 0.607, + "pass@64": 0.681, + "pass@128": 0.721 + }, + "15": { + "pass@1": 0.207, + "pass@2": 0.269, + "pass@4": 0.334, + "pass@8": 0.391, + "pass@16": 0.442, + "pass@32": 0.52, + "pass@64": 0.598, + "pass@128": 0.675 + }, + "16": { + "pass@1": 0.167, + "pass@2": 0.216, + "pass@4": 0.272, + "pass@8": 0.312, + "pass@16": 0.356, + "pass@32": 0.396, + "pass@64": 0.444, + "pass@128": 0.487 + }, + "17": { + "pass@1": 0.098, + "pass@2": 0.142, + "pass@4": 0.183, + "pass@8": 0.222, + "pass@16": 0.265, + "pass@32": 0.312, + "pass@64": 0.355, + "pass@128": 0.402 + }, + "18": { + "pass@1": 0.083, + "pass@2": 0.107, + "pass@4": 0.136, + "pass@8": 0.167, + "pass@16": 0.203, + "pass@32": 0.238, + "pass@64": 0.277, + "pass@128": 0.315 + }, + "19": { + "pass@1": 0.03, + "pass@2": 0.05, + "pass@4": 0.064, + "pass@8": 0.089, + "pass@16": 0.122, + "pass@32": 0.141, + "pass@64": 0.166, + "pass@128": 0.203 + }, + "20": { + "pass@1": 0.033, + "pass@2": 0.048, + "pass@4": 0.071, + "pass@8": 0.095, + "pass@16": 0.119, + "pass@32": 0.137, + "pass@64": 0.156, + "pass@128": 0.176 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.495, + "12": 0.484, + "13": 0.334, + "14": 0.273, + "15": 0.207, + "16": 0.167, + "17": 0.098, + "18": 0.083, + "19": 0.03, + "20": 0.033 + }, + "pass@2": { + "11": 0.594, + "12": 0.58, + "13": 0.425, + "14": 0.333, + "15": 0.269, + "16": 0.216, + "17": 0.142, + "18": 0.107, + "19": 0.05, + "20": 0.048 + }, + "pass@4": { + "11": 0.695, + "12": 0.666, + "13": 0.498, + "14": 0.406, + "15": 0.334, + "16": 0.272, + "17": 0.183, + "18": 0.136, + "19": 0.064, + "20": 0.071 + }, + "pass@8": { + "11": 0.792, + "12": 0.726, + "13": 0.585, + "14": 0.477, + "15": 0.391, + "16": 0.312, + "17": 0.222, + "18": 0.167, + "19": 0.089, + "20": 0.095 + }, + "pass@16": { + "11": 0.851, + "12": 0.779, + "13": 0.651, + "14": 0.535, + "15": 0.442, + "16": 0.356, + "17": 0.265, + "18": 0.203, + "19": 0.122, + "20": 0.119 + }, + "pass@32": { + "11": 0.894, + "12": 0.819, + "13": 0.705, + "14": 0.607, + "15": 0.52, + "16": 0.396, + "17": 0.312, + "18": 0.238, + "19": 0.141, + "20": 0.137 + }, + "pass@64": { + "11": 0.926, + "12": 0.845, + "13": 0.74, + "14": 0.681, + "15": 0.598, + "16": 0.444, + "17": 0.355, + "18": 0.277, + "19": 0.166, + "20": 0.156 + }, + "pass@128": { + "11": 0.94, + "12": 0.861, + "13": 0.764, + "14": 0.721, + "15": 0.675, + "16": 0.487, + "17": 0.402, + "18": 0.315, + "19": 0.203, + "20": 0.176 + } + }, + "total_pass_at_k": { + "pass@1": 0.2204, + "pass@2": 0.2764, + "pass@4": 0.3325, + "pass@8": 0.3856, + "pass@16": 0.4323, + "pass@32": 0.4769, + "pass@64": 0.5188, + "pass@128": 0.5544 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/Qwen3-100M-epoch1-RL-op17-20/composition/hf_model_metrics.json b/composition/Qwen3-100M-epoch1-RL-op17-20/composition/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e80aa815dadd2b570b38394e2db2a59be0022f35 --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op17-20/composition/hf_model_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.9732239583333333, + "avg_response_len": 199.84788975694445, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9732239583333333, + "pass@2": 0.983938839676279, + "pass@4": 0.9892961796442103, + "pass@8": 0.9925038161016022, + "pass@16": 0.99464485612357, + "pass@32": 0.9961348022692114, + "pass@64": 0.9971513603785811, + "pass@128": 0.9977777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.909484375, + "pass@2": 0.9370409694881897, + "pass@4": 0.9536612372515937, + "pass@8": 0.9642115689414419, + "pass@16": 0.9715736424436983, + "pass@32": 0.9774854099391591, + "pass@64": 0.9823532796681838, + "pass@128": 0.986 + }, + "2": { + "pass@1": 0.9991015625, + "pass@2": 0.9991935285433071, + "pass@4": 0.9993519553805774, + "pass@8": 0.99958628732288, + "pass@16": 0.9998395149139014, + "pass@32": 0.9999809346118268, + "pass@64": 0.9999999379486069, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9985546875, + "pass@2": 0.9997164124015749, + "pass@4": 0.9999829086989127, + "pass@8": 0.999999906208338, + "pass@16": 0.9999999999982037, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999515625, + "pass@2": 0.9998928395669291, + "pass@4": 0.9999913394263217, + "pass@8": 0.9999999462072147, + "pass@16": 0.9999999999993266, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9824375, + "pass@2": 0.989911786417323, + "pass@4": 0.9938203699850016, + "pass@8": 0.9961333568534868, + "pass@16": 0.997853368947609, + "pass@32": 0.999169951833145, + "pass@64": 0.9998736956269246, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9941875, + "pass@2": 0.997507258858268, + "pass@4": 0.9988567658417696, + "pass@8": 0.9996239543447049, + "pass@16": 0.9999389855259089, + "pass@32": 0.9999981837531812, + "pass@64": 0.9999999998360332, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9815859375, + "pass@2": 0.9918844734251976, + "pass@4": 0.9959160579302584, + "pass@8": 0.9978769427377138, + "pass@16": 0.998771418075031, + "pass@32": 0.999196451029371, + "pass@64": 0.9994993313142227, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.952578125, + "pass@2": 0.9722695620078745, + "pass@4": 0.9819338375515552, + "pass@8": 0.9878083359489976, + "pass@16": 0.9916546302522348, + "pass@32": 0.9941162060704046, + "pass@64": 0.9955303943971681, + "pass@128": 0.996 + }, + "9": { + "pass@1": 0.9415703125, + "pass@2": 0.9680327263779536, + "pass@4": 0.9801511447319085, + "pass@8": 0.9872940463496322, + "pass@16": 0.9921721449562104, + "pass@32": 0.9952660831858092, + "pass@64": 0.9971056046160844, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.909484375, + "2": 0.9991015625, + "3": 0.9985546875, + "4": 0.999515625, + "5": 0.9824375, + "6": 0.9941875, + "7": 0.9815859375, + "8": 0.952578125, + "9": 0.9415703125 + }, + "per_op_avg_response_len": { + "10": 279.951265625, + "2": 115.5925703125, + "3": 144.8465, + "4": 159.5443515625, + "5": 174.0565859375, + "6": 198.5944609375, + "7": 223.8870859375, + "8": 243.2831015625, + "9": 258.8750859375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.951265625, + "2": 115.5925703125, + "3": 144.8465, + "4": 159.5443515625, + "5": 174.0565859375, + "6": 198.5944609375, + "7": 223.8870859375, + "8": 243.2831015625, + "9": 258.8750859375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 368393, + "answer_accuracy": 0.9719926756163458, + "avg_response_len": 190.36455694866598, + "resp_tokens_sum": 72149690, + "pass_at_k": { + "pass@1": 0.9719926756163458, + "pass@2": 0.9829686764553388, + "pass@4": 0.9884896767589975, + "pass@8": 0.9920497194277219, + "pass@16": 0.9947486021263566, + "pass@32": 0.9967390996817556, + "pass@64": 0.9980913420478178, + "pass@128": 0.9989868287740629 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9062269542772862, + "pass@2": 0.93377856943767, + "pass@4": 0.9505952383717517, + "pass@8": 0.9616645650151304, + "pass@16": 0.9708360396229119, + "pass@32": 0.979137717466014, + "pass@64": 0.9861165304911527, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9986381880733946, + "pass@2": 0.9996613812034965, + "pass@4": 0.9999678924882097, + "pass@8": 0.9999997475359087, + "pass@16": 0.9999999999945448, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999560546875, + "pass@2": 0.9999650129183071, + "pass@4": 0.9999997067749344, + "pass@8": 0.9999999999934361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9850198412698413, + "pass@2": 0.9903390982377203, + "pass@4": 0.9935438281524333, + "pass@8": 0.9957480083422041, + "pass@16": 0.997451178844889, + "pass@32": 0.9988963683428546, + "pass@64": 0.9998099535588568, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.997029378742515, + "pass@2": 0.9992923847187516, + "pass@4": 0.9998629397560335, + "pass@8": 0.9999946418627286, + "pass@16": 0.9999999952012559, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9821329511834319, + "pass@2": 0.9910419966919816, + "pass@4": 0.9939767165753687, + "pass@8": 0.9954793655356908, + "pass@16": 0.9966585838989757, + "pass@32": 0.9976339233964078, + "pass@64": 0.9985187345105551, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9526742788461539, + "pass@2": 0.9762446686351706, + "pass@4": 0.9875043777580687, + "pass@8": 0.9944048642135856, + "pass@16": 0.9981674442303922, + "pass@32": 0.999578041813267, + "pass@64": 0.9999789328776327, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9322843309859155, + "pass@2": 0.9601696794942888, + "pass@4": 0.9737766655048402, + "pass@8": 0.9832953184438525, + "pass@16": 0.9911020557032694, + "pass@32": 0.996326244380213, + "pass@64": 0.9989353951370694, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 380091, + "answer_accuracy": 0.972955746231979, + "avg_response_len": 197.78420912516384, + "resp_tokens_sum": 77265588, + "pass_at_k": { + "pass@1": 0.972955746231979, + "pass@2": 0.9830272165663944, + "pass@4": 0.9879440826572643, + "pass@8": 0.9907435293187308, + "pass@16": 0.9925269818183432, + "pass@32": 0.9937382279738785, + "pass@64": 0.9945664701724136, + "pass@128": 0.9950851900393185 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9080233895705522, + "pass@2": 0.9325764757741168, + "pass@4": 0.9461707905760242, + "pass@8": 0.953994960326316, + "pass@16": 0.9594360890980274, + "pass@32": 0.9644964902058345, + "pass@64": 0.9691135912953294, + "pass@128": 0.9723926380368099 + }, + "2": { + "pass@1": 0.9972691869300911, + "pass@2": 0.9975487189766172, + "pass@4": 0.9980302595154329, + "pass@8": 0.9987425146592096, + "pass@16": 0.9995122033857184, + "pass@32": 0.9999420504918746, + "pass@64": 0.9999998113939417, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977035984848485, + "pass@2": 0.9995783375089478, + "pass@4": 0.9999825490563681, + "pass@8": 0.9999999669910425, + "pass@16": 0.9999999999999619, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990292159763313, + "pass@2": 0.9997160811629315, + "pass@4": 0.9999746546104813, + "pass@8": 0.9999998408559615, + "pass@16": 0.9999999999980078, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9872735507246376, + "pass@2": 0.9934212027844346, + "pass@4": 0.9960166188465572, + "pass@8": 0.9977547571009451, + "pass@16": 0.9991579262409745, + "pass@32": 0.9998845013853158, + "pass@64": 0.9999991115258494, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9901728151862464, + "pass@2": 0.9940444295286872, + "pass@4": 0.9968823914918945, + "pass@8": 0.9989277166937187, + "pass@16": 0.9998251780193597, + "pass@32": 0.9999947958543877, + "pass@64": 0.9999999995301808, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9826138005780347, + "pass@2": 0.9935234229211234, + "pass@4": 0.9980839851665938, + "pass@8": 0.9997614186744043, + "pass@16": 0.9999954079465505, + "pass@32": 0.9999999992044007, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9533230633802817, + "pass@2": 0.9697533131862036, + "pass@4": 0.9778347227723291, + "pass@8": 0.9826861224766935, + "pass@16": 0.9861681574883515, + "pass@32": 0.9882344037134858, + "pass@64": 0.9887213879364309, + "pass@128": 0.9887323943661972 + }, + "9": { + "pass@1": 0.9396285553892215, + "pass@2": 0.9657106800179176, + "pass@4": 0.9772753267368522, + "pass@8": 0.9836481530198901, + "pass@16": 0.9875451790737914, + "pass@32": 0.990121950847974, + "pass@64": 0.9924853659444284, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 372670, + "answer_accuracy": 0.9747185721459658, + "avg_response_len": 211.35726429109474, + "resp_tokens_sum": 80809491, + "pass_at_k": { + "pass@1": 0.9747185721459658, + "pass@2": 0.9858320191301431, + "pass@4": 0.9914771823383159, + "pass@8": 0.9947525524633932, + "pass@16": 0.9967059744581199, + "pass@32": 0.9979844909571278, + "pass@64": 0.9988607005816629, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9142024253731343, + "pass@2": 0.944686875661065, + "pass@4": 0.9640530558680164, + "pass@8": 0.9767311174177122, + "pass@16": 0.9841315222852977, + "pass@32": 0.988453337047995, + "pass@64": 0.9914290897594195, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9992939139941691, + "pass@2": 0.999901718280113, + "pass@4": 0.9999975704567541, + "pass@8": 0.9999999990001509, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999543128654971, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9751378676470588, + "pass@2": 0.9859548691523854, + "pass@4": 0.991848031220362, + "pass@8": 0.9948451265466661, + "pass@16": 0.9969022442892146, + "pass@32": 0.9986983612565055, + "pass@64": 0.9998054905278431, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9956131703470031, + "pass@2": 0.9994387900842048, + "pass@4": 0.9999703133519979, + "pass@8": 0.9999999080014058, + "pass@16": 0.9999999999997884, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9798753955696202, + "pass@2": 0.9909910576348052, + "pass@4": 0.9956166736080455, + "pass@8": 0.9983780579914756, + "pass@16": 0.9996911568597177, + "pass@32": 0.9999879278375395, + "pass@64": 0.9999999969925794, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9516938813813813, + "pass@2": 0.9712276252630587, + "pass@4": 0.981084519840425, + "pass@8": 0.9870884229283264, + "pass@16": 0.9914014767086731, + "pass@32": 0.9952692003194525, + "pass@64": 0.9986212451108519, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9542554260450161, + "pass@2": 0.979502003316708, + "pass@4": 0.9905159786095866, + "pass@8": 0.9957740359917082, + "pass@16": 0.9983627825430983, + "pass@32": 0.9995804979022844, + "pass@64": 0.9999788653279279, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.52322109375, + "avg_response_len": 373.5542125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.52322109375, + "pass@2": 0.6205332062007808, + "pass@4": 0.7013960804836779, + "pass@8": 0.7652119499175566, + "pass@16": 0.8135733234896841, + "pass@32": 0.849761382301005, + "pass@64": 0.8777406566282842, + "pass@128": 0.8999 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8693203125, + "pass@2": 0.9169223671259847, + "pass@4": 0.9431539670978625, + "pass@8": 0.9576215014933498, + "pass@16": 0.9664858099196177, + "pass@32": 0.9735371340474053, + "pass@64": 0.9798610568122847, + "pass@128": 0.985 + }, + "12": { + "pass@1": 0.817671875, + "pass@2": 0.8758592519685052, + "pass@4": 0.9081501521372319, + "pass@8": 0.9281222782097398, + "pass@16": 0.9415562351128167, + "pass@32": 0.9518892216762417, + "pass@64": 0.9610060049791429, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.6533671875, + "pass@2": 0.7487383120078739, + "pass@4": 0.8200541114548177, + "pass@8": 0.8716508144618947, + "pass@16": 0.9073551500616253, + "pass@32": 0.9319625086738039, + "pass@64": 0.9508814928821682, + "pass@128": 0.965 + }, + "14": { + "pass@1": 0.636890625, + "pass@2": 0.7399282726377948, + "pass@4": 0.8155623213348322, + "pass@8": 0.8690980696578025, + "pass@16": 0.9054343065404875, + "pass@32": 0.928481837956825, + "pass@64": 0.9420224047748543, + "pass@128": 0.95 + }, + "15": { + "pass@1": 0.5613359375, + "pass@2": 0.6738644192913384, + "pass@4": 0.7629428038995129, + "pass@8": 0.8280016058186453, + "pass@16": 0.8710593312929621, + "pass@32": 0.8989863989669378, + "pass@64": 0.9203518982282868, + "pass@128": 0.94 + }, + "16": { + "pass@1": 0.4624765625, + "pass@2": 0.5842992125984253, + "pass@4": 0.683700046869141, + "pass@8": 0.7594630564179031, + "pass@16": 0.8170072028386846, + "pass@32": 0.8602825280176216, + "pass@64": 0.8920889039224658, + "pass@128": 0.915 + }, + "17": { + "pass@1": 0.377125, + "pass@2": 0.4947443405511814, + "pass@4": 0.5992732603112109, + "pass@8": 0.6840335303179859, + "pass@16": 0.7498341717811102, + "pass@32": 0.7998023694824413, + "pass@64": 0.8372466230763199, + "pass@128": 0.865 + }, + "18": { + "pass@1": 0.345359375, + "pass@2": 0.4591813484251975, + "pass@4": 0.5645886353580799, + "pass@8": 0.6516129215361331, + "pass@16": 0.7197269212461064, + "pass@32": 0.7716237318520149, + "pass@64": 0.8104916475549201, + "pass@128": 0.84 + }, + "19": { + "pass@1": 0.25065625, + "pass@2": 0.35162524606299206, + "pass@4": 0.45515188395200556, + "pass@8": 0.5510033904285595, + "pass@16": 0.631991305686842, + "pass@32": 0.6967430789909519, + "pass@64": 0.7489312211789294, + "pass@128": 0.792 + }, + "20": { + "pass@1": 0.2580078125, + "pass@2": 0.3601692913385827, + "pass@4": 0.4613836224221967, + "pass@8": 0.5515123308335779, + "pass@16": 0.6252828004163944, + "pass@32": 0.6843050133457697, + "pass@64": 0.7345253128733888, + "pass@128": 0.779 + } + }, + "per_op_accuracy": { + "11": 0.8693203125, + "12": 0.817671875, + "13": 0.6533671875, + "14": 0.636890625, + "15": 0.5613359375, + "16": 0.4624765625, + "17": 0.377125, + "18": 0.345359375, + "19": 0.25065625, + "20": 0.2580078125 + }, + "per_op_avg_response_len": { + "11": 298.56009375, + "12": 307.4940546875, + "13": 325.3804609375, + "14": 353.95775, + "15": 375.143015625, + "16": 379.0198515625, + "17": 404.4674453125, + "18": 421.4213359375, + "19": 422.1345546875, + "20": 447.9635625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.56009375, + "12": 307.4940546875, + "13": 325.3804609375, + "14": 353.95775, + "15": 375.143015625, + "16": 379.0198515625, + "17": 404.4674453125, + "18": 421.4213359375, + "19": 422.1345546875, + "20": 447.9635625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 226031, + "answer_accuracy": 0.5247747956909361, + "avg_response_len": 355.2154183692422, + "resp_tokens_sum": 152998385, + "pass_at_k": { + "pass@1": 0.5247747956909361, + "pass@2": 0.6195935244117905, + "pass@4": 0.6971609348979979, + "pass@8": 0.7592476818686267, + "pass@16": 0.8080896719120051, + "pass@32": 0.8460049894939322, + "pass@64": 0.8763567840346517, + "pass@128": 0.9013372956909361 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8724327195467422, + "pass@2": 0.923634385246816, + "pass@4": 0.9503822968481626, + "pass@8": 0.9648353122583307, + "pass@16": 0.973952347913156, + "pass@32": 0.9806782677294469, + "pass@64": 0.9856572373418709, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8221883827683616, + "pass@2": 0.8813051903999292, + "pass@4": 0.9115400233339476, + "pass@8": 0.9287599940682195, + "pass@16": 0.9401571241780109, + "pass@32": 0.9498534527546343, + "pass@64": 0.9593712803795826, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.6445434952978056, + "pass@2": 0.7376050588699922, + "pass@4": 0.8092676895011944, + "pass@8": 0.8633510798804404, + "pass@16": 0.9027293938044151, + "pass@32": 0.93160249813217, + "pass@64": 0.9544420983735626, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.6434725467289719, + "pass@2": 0.7494752955822113, + "pass@4": 0.8239008923534091, + "pass@8": 0.8768464903393149, + "pass@16": 0.9131865701372338, + "pass@32": 0.9355432517352564, + "pass@64": 0.948839701524294, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.5680771315028902, + "pass@2": 0.6790506519958127, + "pass@4": 0.7673308419540041, + "pass@8": 0.8321651360027117, + "pass@16": 0.8752393426050006, + "pass@32": 0.9046743185604588, + "pass@64": 0.926849629105118, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.4576360887096774, + "pass@2": 0.5696969456438907, + "pass@4": 0.6595860659554652, + "pass@8": 0.730592961465299, + "pass@16": 0.789544236735948, + "pass@32": 0.8369620391949317, + "pass@64": 0.8752227021714117, + "pass@128": 0.9096774193548387 + }, + "17": { + "pass@1": 0.3633941473988439, + "pass@2": 0.4766912208137999, + "pass@4": 0.5772174451820685, + "pass@8": 0.6598159310844616, + "pass@16": 0.7250069898614739, + "pass@32": 0.7758477060303068, + "pass@64": 0.8176952910565459, + "pass@128": 0.8554913294797688 + }, + "18": { + "pass@1": 0.32930803571428574, + "pass@2": 0.43295451349831254, + "pass@4": 0.5302455525202205, + "pass@8": 0.6154654950714722, + "pass@16": 0.6884315150553474, + "pass@32": 0.7453938186323203, + "pass@64": 0.7856894486479148, + "pass@128": 0.8142857142857143 + }, + "19": { + "pass@1": 0.25736674174174173, + "pass@2": 0.3567418304130902, + "pass@4": 0.4550597968609781, + "pass@8": 0.5463556456220624, + "pass@16": 0.624890768528339, + "pass@32": 0.6885183056794916, + "pass@64": 0.7422557482331051, + "pass@128": 0.7897897897897898 + }, + "20": { + "pass@1": 0.26895645645645644, + "pass@2": 0.3729963969875385, + "pass@4": 0.4755003121794459, + "pass@8": 0.5662618857243946, + "pass@16": 0.6426840209099434, + "pass@32": 0.7081242437863503, + "pass@64": 0.766509055353242, + "pass@128": 0.8168168168168168 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 225846, + "answer_accuracy": 0.5246571141837645, + "avg_response_len": 369.03983376077906, + "resp_tokens_sum": 158858363, + "pass_at_k": { + "pass@1": 0.5246571141837645, + "pass@2": 0.6225992212614847, + "pass@4": 0.704608004870267, + "pass@8": 0.7695048622073701, + "pass@16": 0.8185040611244835, + "pass@32": 0.8550338630233388, + "pass@64": 0.8831796020410525, + "pass@128": 0.9048468629200119 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8694700677710844, + "pass@2": 0.909607055189261, + "pass@4": 0.9317818410876343, + "pass@8": 0.9454585632863172, + "pass@16": 0.9556582000789117, + "pass@32": 0.9649759331729336, + "pass@64": 0.9740005463669225, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8088753918495298, + "pass@2": 0.8641022634709847, + "pass@4": 0.8943006227865717, + "pass@8": 0.9132144665068072, + "pass@16": 0.9271130674479252, + "pass@32": 0.9389810679709106, + "pass@64": 0.9492095408011472, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6670386904761905, + "pass@2": 0.7605552063345015, + "pass@4": 0.8290062037938528, + "pass@8": 0.8782387659005888, + "pass@16": 0.9117272202364387, + "pass@32": 0.9333991943128012, + "pass@64": 0.9489098468813019, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.636421783625731, + "pass@2": 0.7366485644886493, + "pass@4": 0.8129156654212081, + "pass@8": 0.8685094524738564, + "pass@16": 0.9069490253899266, + "pass@32": 0.9322473949964639, + "pass@64": 0.9468054042889353, + "pass@128": 0.9532163742690059 + }, + "15": { + "pass@1": 0.5566761363636363, + "pass@2": 0.6740545215938918, + "pass@4": 0.7655083620229282, + "pass@8": 0.830314977109201, + "pass@16": 0.8728943261197528, + "pass@32": 0.9007579003977949, + "pass@64": 0.9218918824899519, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.4544827279202279, + "pass@2": 0.5782294540906747, + "pass@4": 0.6807875420486968, + "pass@8": 0.7586801919280979, + "pass@16": 0.8155312114186396, + "pass@32": 0.8572814405727026, + "pass@64": 0.8900660067538252, + "pass@128": 0.9145299145299145 + }, + "17": { + "pass@1": 0.4038480201863354, + "pass@2": 0.5302863806426372, + "pass@4": 0.6393859222566122, + "pass@8": 0.7230418188024726, + "pass@16": 0.7829436866294603, + "pass@32": 0.8259523452426276, + "pass@64": 0.8575249008903864, + "pass@128": 0.8757763975155279 + }, + "18": { + "pass@1": 0.35723805147058824, + "pass@2": 0.48015357225567384, + "pass@4": 0.5944133659763114, + "pass@8": 0.686807116888759, + "pass@16": 0.7560314779655988, + "pass@32": 0.8086900156791244, + "pass@64": 0.8510250714257173, + "pass@128": 0.888235294117647 + }, + "19": { + "pass@1": 0.24251302083333334, + "pass@2": 0.34276867735283073, + "pass@4": 0.45104315531987044, + "pass@8": 0.555444460436145, + "pass@16": 0.646163410551263, + "pass@32": 0.7183930552407676, + "pass@64": 0.7730795118522583, + "pass@128": 0.8125 + }, + "20": { + "pass@1": 0.25666635479041916, + "pass@2": 0.35371186760337586, + "pass@4": 0.44875263421413675, + "pass@8": 0.5353646936771068, + "pass@16": 0.6088692333415894, + "pass@32": 0.6678819447924638, + "pass@64": 0.7171705545064891, + "pass@128": 0.7604790419161677 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 217846, + "answer_accuracy": 0.5201472723105135, + "avg_response_len": 397.0541813111247, + "resp_tokens_sum": 166292644, + "pass_at_k": { + "pass@1": 0.5201472723105135, + "pass@2": 0.6193761220231813, + "pass@4": 0.7024503479604048, + "pass@8": 0.76693343462235, + "pass@16": 0.8141449667943694, + "pass@32": 0.8482054254325866, + "pass@64": 0.8735736510825346, + "pass@128": 0.8933374083129584 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8656746031746032, + "pass@2": 0.9171107517810279, + "pass@4": 0.9470395081567186, + "pass@8": 0.9623568039844478, + "pass@16": 0.9695305037462729, + "pass@32": 0.9745577642079248, + "pass@64": 0.9795424464659857, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8213637232415902, + "pass@2": 0.8814330046473546, + "pass@4": 0.9179910862632998, + "pass@8": 0.9419750015104567, + "pass@16": 0.9571606869660971, + "pass@32": 0.9666854395057514, + "pass@64": 0.9742835725052125, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.6469907407407407, + "pass@2": 0.7466792887868181, + "pass@4": 0.8208101967578123, + "pass@8": 0.8725635202269273, + "pass@16": 0.9070921475728697, + "pass@32": 0.930733948765345, + "pass@64": 0.9495482968036301, + "pass@128": 0.9629629629629629 + }, + "14": { + "pass@1": 0.6310969955489614, + "pass@2": 0.734162889319844, + "pass@4": 0.8103055706686509, + "pass@8": 0.8623148768926493, + "pass@16": 0.8965129105135972, + "pass@32": 0.9179342583412952, + "pass@64": 0.9306748140021945, + "pass@128": 0.9406528189910979 + }, + "15": { + "pass@1": 0.5588831018518519, + "pass@2": 0.6681324122679111, + "pass@4": 0.7556437441847542, + "pass@8": 0.8211991552952789, + "pass@16": 0.864726515839861, + "pass@32": 0.8911079555979848, + "pass@64": 0.9118444609142939, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.47517975663716816, + "pass@2": 0.6039369352890623, + "pass@4": 0.7087667821942608, + "pass@8": 0.786674014740112, + "pass@16": 0.8436490685622383, + "pass@32": 0.884715369398746, + "pass@64": 0.9096067784033466, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.3655167545180723, + "pass@2": 0.4790872397068589, + "pass@4": 0.5833546906975483, + "pass@8": 0.671439013579416, + "pass@16": 0.7435960427540855, + "pass@32": 0.7994048434573172, + "pass@64": 0.8379549225423192, + "pass@128": 0.8644578313253012 + }, + "18": { + "pass@1": 0.3504536290322581, + "pass@2": 0.46579049720599447, + "pass@4": 0.5706520888517969, + "pass@8": 0.6538244468352877, + "pass@16": 0.7152425434465555, + "pass@32": 0.7605848064509636, + "pass@64": 0.7940381169142109, + "pass@128": 0.8161290322580645 + }, + "19": { + "pass@1": 0.2521714501510574, + "pass@2": 0.3554680995313651, + "pass@4": 0.45941532147983016, + "pass@8": 0.5511710626280015, + "pass@16": 0.6247485613948076, + "pass@32": 0.6830405336519113, + "pass@64": 0.7311339608306535, + "pass@128": 0.7734138972809668 + }, + "20": { + "pass@1": 0.24840465465465467, + "pass@2": 0.35381900108770165, + "pass@4": 0.45993585176852897, + "pass@8": 0.5529589045021037, + "pass@16": 0.6243444369888672, + "pass@32": 0.6769581699826803, + "pass@64": 0.7199484451519275, + "pass@128": 0.7597597597597597 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7363803453947368, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7363803453947368, + "pass@2": 0.7926727167944182, + "pass@4": 0.8377698116649664, + "pass@8": 0.8728765181100161, + "pass@16": 0.8993440494741402, + "pass@32": 0.9190961601806963, + "pass@64": 0.9343036215626328, + "pass@128": 0.9462631578947368 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.909484375, + "pass@2": 0.9370409694881897, + "pass@4": 0.9536612372515937, + "pass@8": 0.9642115689414419, + "pass@16": 0.9715736424436983, + "pass@32": 0.9774854099391591, + "pass@64": 0.9823532796681838, + "pass@128": 0.986 + }, + "2": { + "pass@1": 0.9991015625, + "pass@2": 0.9991935285433071, + "pass@4": 0.9993519553805774, + "pass@8": 0.99958628732288, + "pass@16": 0.9998395149139014, + "pass@32": 0.9999809346118268, + "pass@64": 0.9999999379486069, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9985546875, + "pass@2": 0.9997164124015749, + "pass@4": 0.9999829086989127, + "pass@8": 0.999999906208338, + "pass@16": 0.9999999999982037, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999515625, + "pass@2": 0.9998928395669291, + "pass@4": 0.9999913394263217, + "pass@8": 0.9999999462072147, + "pass@16": 0.9999999999993266, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9824375, + "pass@2": 0.989911786417323, + "pass@4": 0.9938203699850016, + "pass@8": 0.9961333568534868, + "pass@16": 0.997853368947609, + "pass@32": 0.999169951833145, + "pass@64": 0.9998736956269246, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9941875, + "pass@2": 0.997507258858268, + "pass@4": 0.9988567658417696, + "pass@8": 0.9996239543447049, + "pass@16": 0.9999389855259089, + "pass@32": 0.9999981837531812, + "pass@64": 0.9999999998360332, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9815859375, + "pass@2": 0.9918844734251976, + "pass@4": 0.9959160579302584, + "pass@8": 0.9978769427377138, + "pass@16": 0.998771418075031, + "pass@32": 0.999196451029371, + "pass@64": 0.9994993313142227, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.952578125, + "pass@2": 0.9722695620078745, + "pass@4": 0.9819338375515552, + "pass@8": 0.9878083359489976, + "pass@16": 0.9916546302522348, + "pass@32": 0.9941162060704046, + "pass@64": 0.9955303943971681, + "pass@128": 0.996 + }, + "9": { + "pass@1": 0.9415703125, + "pass@2": 0.9680327263779536, + "pass@4": 0.9801511447319085, + "pass@8": 0.9872940463496322, + "pass@16": 0.9921721449562104, + "pass@32": 0.9952660831858092, + "pass@64": 0.9971056046160844, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.8693203125, + "pass@2": 0.9169223671259847, + "pass@4": 0.9431539670978625, + "pass@8": 0.9576215014933498, + "pass@16": 0.9664858099196177, + "pass@32": 0.9735371340474053, + "pass@64": 0.9798610568122847, + "pass@128": 0.985 + }, + "12": { + "pass@1": 0.817671875, + "pass@2": 0.8758592519685052, + "pass@4": 0.9081501521372319, + "pass@8": 0.9281222782097398, + "pass@16": 0.9415562351128167, + "pass@32": 0.9518892216762417, + "pass@64": 0.9610060049791429, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.6533671875, + "pass@2": 0.7487383120078739, + "pass@4": 0.8200541114548177, + "pass@8": 0.8716508144618947, + "pass@16": 0.9073551500616253, + "pass@32": 0.9319625086738039, + "pass@64": 0.9508814928821682, + "pass@128": 0.965 + }, + "14": { + "pass@1": 0.636890625, + "pass@2": 0.7399282726377948, + "pass@4": 0.8155623213348322, + "pass@8": 0.8690980696578025, + "pass@16": 0.9054343065404875, + "pass@32": 0.928481837956825, + "pass@64": 0.9420224047748543, + "pass@128": 0.95 + }, + "15": { + "pass@1": 0.5613359375, + "pass@2": 0.6738644192913384, + "pass@4": 0.7629428038995129, + "pass@8": 0.8280016058186453, + "pass@16": 0.8710593312929621, + "pass@32": 0.8989863989669378, + "pass@64": 0.9203518982282868, + "pass@128": 0.94 + }, + "16": { + "pass@1": 0.4624765625, + "pass@2": 0.5842992125984253, + "pass@4": 0.683700046869141, + "pass@8": 0.7594630564179031, + "pass@16": 0.8170072028386846, + "pass@32": 0.8602825280176216, + "pass@64": 0.8920889039224658, + "pass@128": 0.915 + }, + "17": { + "pass@1": 0.377125, + "pass@2": 0.4947443405511814, + "pass@4": 0.5992732603112109, + "pass@8": 0.6840335303179859, + "pass@16": 0.7498341717811102, + "pass@32": 0.7998023694824413, + "pass@64": 0.8372466230763199, + "pass@128": 0.865 + }, + "18": { + "pass@1": 0.345359375, + "pass@2": 0.4591813484251975, + "pass@4": 0.5645886353580799, + "pass@8": 0.6516129215361331, + "pass@16": 0.7197269212461064, + "pass@32": 0.7716237318520149, + "pass@64": 0.8104916475549201, + "pass@128": 0.84 + }, + "19": { + "pass@1": 0.25065625, + "pass@2": 0.35162524606299206, + "pass@4": 0.45515188395200556, + "pass@8": 0.5510033904285595, + "pass@16": 0.631991305686842, + "pass@32": 0.6967430789909519, + "pass@64": 0.7489312211789294, + "pass@128": 0.792 + }, + "20": { + "pass@1": 0.2580078125, + "pass@2": 0.3601692913385827, + "pass@4": 0.4613836224221967, + "pass@8": 0.5515123308335779, + "pass@16": 0.6252828004163944, + "pass@32": 0.6843050133457697, + "pass@64": 0.7345253128733888, + "pass@128": 0.779 + } + }, + "per_op_accuracy": { + "10": 0.909484375, + "2": 0.9991015625, + "3": 0.9985546875, + "4": 0.999515625, + "5": 0.9824375, + "6": 0.9941875, + "7": 0.9815859375, + "8": 0.952578125, + "9": 0.9415703125, + "11": 0.8693203125, + "12": 0.817671875, + "13": 0.6533671875, + "14": 0.636890625, + "15": 0.5613359375, + "16": 0.4624765625, + "17": 0.377125, + "18": 0.345359375, + "19": 0.25065625, + "20": 0.2580078125 + }, + "per_op_avg_response_len": { + "10": 279.951265625, + "2": 115.5925703125, + "3": 144.8465, + "4": 159.5443515625, + "5": 174.0565859375, + "6": 198.5944609375, + "7": 223.8870859375, + "8": 243.2831015625, + "9": 258.8750859375, + "11": 298.56009375, + "12": 307.4940546875, + "13": 325.3804609375, + "14": 353.95775, + "15": 375.143015625, + "16": 379.0198515625, + "17": 404.4674453125, + "18": 421.4213359375, + "19": 422.1345546875, + "20": 447.9635625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.951265625, + "2": 115.5925703125, + "3": 144.8465, + "4": 159.5443515625, + "5": 174.0565859375, + "6": 198.5944609375, + "7": 223.8870859375, + "8": 243.2831015625, + "9": 258.8750859375, + "11": 298.56009375, + "12": 307.4940546875, + "13": 325.3804609375, + "14": 353.95775, + "15": 375.143015625, + "16": 379.0198515625, + "17": 404.4674453125, + "18": 421.4213359375, + "19": 422.1345546875, + "20": 447.9635625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 594424, + "answer_accuracy": 0.7341033038254822, + "avg_response_len": 278.0539576252766, + "resp_tokens_sum": 225148075, + "pass_at_k": { + "pass@1": 0.7341033038254822, + "pass@2": 0.7896779103113946, + "pass@4": 0.833522680811745, + "pass@8": 0.8682149334039494, + "pass@16": 0.8954587981157262, + "pass@32": 0.9165588466336974, + "pass@64": 0.9333368703889094, + "pass@128": 0.9470439456212456 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9062269542772862, + "pass@2": 0.93377856943767, + "pass@4": 0.9505952383717517, + "pass@8": 0.9616645650151304, + "pass@16": 0.9708360396229119, + "pass@32": 0.979137717466014, + "pass@64": 0.9861165304911527, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9986381880733946, + "pass@2": 0.9996613812034965, + "pass@4": 0.9999678924882097, + "pass@8": 0.9999997475359087, + "pass@16": 0.9999999999945448, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999560546875, + "pass@2": 0.9999650129183071, + "pass@4": 0.9999997067749344, + "pass@8": 0.9999999999934361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9850198412698413, + "pass@2": 0.9903390982377203, + "pass@4": 0.9935438281524333, + "pass@8": 0.9957480083422041, + "pass@16": 0.997451178844889, + "pass@32": 0.9988963683428546, + "pass@64": 0.9998099535588568, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.997029378742515, + "pass@2": 0.9992923847187516, + "pass@4": 0.9998629397560335, + "pass@8": 0.9999946418627286, + "pass@16": 0.9999999952012559, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9821329511834319, + "pass@2": 0.9910419966919816, + "pass@4": 0.9939767165753687, + "pass@8": 0.9954793655356908, + "pass@16": 0.9966585838989757, + "pass@32": 0.9976339233964078, + "pass@64": 0.9985187345105551, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9526742788461539, + "pass@2": 0.9762446686351706, + "pass@4": 0.9875043777580687, + "pass@8": 0.9944048642135856, + "pass@16": 0.9981674442303922, + "pass@32": 0.999578041813267, + "pass@64": 0.9999789328776327, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9322843309859155, + "pass@2": 0.9601696794942888, + "pass@4": 0.9737766655048402, + "pass@8": 0.9832953184438525, + "pass@16": 0.9911020557032694, + "pass@32": 0.996326244380213, + "pass@64": 0.9989353951370694, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8724327195467422, + "pass@2": 0.923634385246816, + "pass@4": 0.9503822968481626, + "pass@8": 0.9648353122583307, + "pass@16": 0.973952347913156, + "pass@32": 0.9806782677294469, + "pass@64": 0.9856572373418709, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8221883827683616, + "pass@2": 0.8813051903999292, + "pass@4": 0.9115400233339476, + "pass@8": 0.9287599940682195, + "pass@16": 0.9401571241780109, + "pass@32": 0.9498534527546343, + "pass@64": 0.9593712803795826, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.6445434952978056, + "pass@2": 0.7376050588699922, + "pass@4": 0.8092676895011944, + "pass@8": 0.8633510798804404, + "pass@16": 0.9027293938044151, + "pass@32": 0.93160249813217, + "pass@64": 0.9544420983735626, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.6434725467289719, + "pass@2": 0.7494752955822113, + "pass@4": 0.8239008923534091, + "pass@8": 0.8768464903393149, + "pass@16": 0.9131865701372338, + "pass@32": 0.9355432517352564, + "pass@64": 0.948839701524294, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.5680771315028902, + "pass@2": 0.6790506519958127, + "pass@4": 0.7673308419540041, + "pass@8": 0.8321651360027117, + "pass@16": 0.8752393426050006, + "pass@32": 0.9046743185604588, + "pass@64": 0.926849629105118, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.4576360887096774, + "pass@2": 0.5696969456438907, + "pass@4": 0.6595860659554652, + "pass@8": 0.730592961465299, + "pass@16": 0.789544236735948, + "pass@32": 0.8369620391949317, + "pass@64": 0.8752227021714117, + "pass@128": 0.9096774193548387 + }, + "17": { + "pass@1": 0.3633941473988439, + "pass@2": 0.4766912208137999, + "pass@4": 0.5772174451820685, + "pass@8": 0.6598159310844616, + "pass@16": 0.7250069898614739, + "pass@32": 0.7758477060303068, + "pass@64": 0.8176952910565459, + "pass@128": 0.8554913294797688 + }, + "18": { + "pass@1": 0.32930803571428574, + "pass@2": 0.43295451349831254, + "pass@4": 0.5302455525202205, + "pass@8": 0.6154654950714722, + "pass@16": 0.6884315150553474, + "pass@32": 0.7453938186323203, + "pass@64": 0.7856894486479148, + "pass@128": 0.8142857142857143 + }, + "19": { + "pass@1": 0.25736674174174173, + "pass@2": 0.3567418304130902, + "pass@4": 0.4550597968609781, + "pass@8": 0.5463556456220624, + "pass@16": 0.624890768528339, + "pass@32": 0.6885183056794916, + "pass@64": 0.7422557482331051, + "pass@128": 0.7897897897897898 + }, + "20": { + "pass@1": 0.26895645645645644, + "pass@2": 0.3729963969875385, + "pass@4": 0.4755003121794459, + "pass@8": 0.5662618857243946, + "pass@16": 0.6426840209099434, + "pass@32": 0.7081242437863503, + "pass@64": 0.766509055353242, + "pass@128": 0.8168168168168168 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 605937, + "answer_accuracy": 0.7379396434138737, + "avg_response_len": 287.5632684625877, + "resp_tokens_sum": 236123951, + "pass_at_k": { + "pass@1": 0.7379396434138737, + "pass@2": 0.7940764218336671, + "pass@4": 0.8394079595711046, + "pass@8": 0.8747613566771798, + "pass@16": 0.9012971950227997, + "pass@32": 0.921023843043457, + "pass@64": 0.9361730114778367, + "pass@128": 0.9477786438035853 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9080233895705522, + "pass@2": 0.9325764757741168, + "pass@4": 0.9461707905760242, + "pass@8": 0.953994960326316, + "pass@16": 0.9594360890980274, + "pass@32": 0.9644964902058345, + "pass@64": 0.9691135912953294, + "pass@128": 0.9723926380368099 + }, + "2": { + "pass@1": 0.9972691869300911, + "pass@2": 0.9975487189766172, + "pass@4": 0.9980302595154329, + "pass@8": 0.9987425146592096, + "pass@16": 0.9995122033857184, + "pass@32": 0.9999420504918746, + "pass@64": 0.9999998113939417, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977035984848485, + "pass@2": 0.9995783375089478, + "pass@4": 0.9999825490563681, + "pass@8": 0.9999999669910425, + "pass@16": 0.9999999999999619, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990292159763313, + "pass@2": 0.9997160811629315, + "pass@4": 0.9999746546104813, + "pass@8": 0.9999998408559615, + "pass@16": 0.9999999999980078, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9872735507246376, + "pass@2": 0.9934212027844346, + "pass@4": 0.9960166188465572, + "pass@8": 0.9977547571009451, + "pass@16": 0.9991579262409745, + "pass@32": 0.9998845013853158, + "pass@64": 0.9999991115258494, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9901728151862464, + "pass@2": 0.9940444295286872, + "pass@4": 0.9968823914918945, + "pass@8": 0.9989277166937187, + "pass@16": 0.9998251780193597, + "pass@32": 0.9999947958543877, + "pass@64": 0.9999999995301808, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9826138005780347, + "pass@2": 0.9935234229211234, + "pass@4": 0.9980839851665938, + "pass@8": 0.9997614186744043, + "pass@16": 0.9999954079465505, + "pass@32": 0.9999999992044007, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9533230633802817, + "pass@2": 0.9697533131862036, + "pass@4": 0.9778347227723291, + "pass@8": 0.9826861224766935, + "pass@16": 0.9861681574883515, + "pass@32": 0.9882344037134858, + "pass@64": 0.9887213879364309, + "pass@128": 0.9887323943661972 + }, + "9": { + "pass@1": 0.9396285553892215, + "pass@2": 0.9657106800179176, + "pass@4": 0.9772753267368522, + "pass@8": 0.9836481530198901, + "pass@16": 0.9875451790737914, + "pass@32": 0.990121950847974, + "pass@64": 0.9924853659444284, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.8694700677710844, + "pass@2": 0.909607055189261, + "pass@4": 0.9317818410876343, + "pass@8": 0.9454585632863172, + "pass@16": 0.9556582000789117, + "pass@32": 0.9649759331729336, + "pass@64": 0.9740005463669225, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8088753918495298, + "pass@2": 0.8641022634709847, + "pass@4": 0.8943006227865717, + "pass@8": 0.9132144665068072, + "pass@16": 0.9271130674479252, + "pass@32": 0.9389810679709106, + "pass@64": 0.9492095408011472, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6670386904761905, + "pass@2": 0.7605552063345015, + "pass@4": 0.8290062037938528, + "pass@8": 0.8782387659005888, + "pass@16": 0.9117272202364387, + "pass@32": 0.9333991943128012, + "pass@64": 0.9489098468813019, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.636421783625731, + "pass@2": 0.7366485644886493, + "pass@4": 0.8129156654212081, + "pass@8": 0.8685094524738564, + "pass@16": 0.9069490253899266, + "pass@32": 0.9322473949964639, + "pass@64": 0.9468054042889353, + "pass@128": 0.9532163742690059 + }, + "15": { + "pass@1": 0.5566761363636363, + "pass@2": 0.6740545215938918, + "pass@4": 0.7655083620229282, + "pass@8": 0.830314977109201, + "pass@16": 0.8728943261197528, + "pass@32": 0.9007579003977949, + "pass@64": 0.9218918824899519, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.4544827279202279, + "pass@2": 0.5782294540906747, + "pass@4": 0.6807875420486968, + "pass@8": 0.7586801919280979, + "pass@16": 0.8155312114186396, + "pass@32": 0.8572814405727026, + "pass@64": 0.8900660067538252, + "pass@128": 0.9145299145299145 + }, + "17": { + "pass@1": 0.4038480201863354, + "pass@2": 0.5302863806426372, + "pass@4": 0.6393859222566122, + "pass@8": 0.7230418188024726, + "pass@16": 0.7829436866294603, + "pass@32": 0.8259523452426276, + "pass@64": 0.8575249008903864, + "pass@128": 0.8757763975155279 + }, + "18": { + "pass@1": 0.35723805147058824, + "pass@2": 0.48015357225567384, + "pass@4": 0.5944133659763114, + "pass@8": 0.686807116888759, + "pass@16": 0.7560314779655988, + "pass@32": 0.8086900156791244, + "pass@64": 0.8510250714257173, + "pass@128": 0.888235294117647 + }, + "19": { + "pass@1": 0.24251302083333334, + "pass@2": 0.34276867735283073, + "pass@4": 0.45104315531987044, + "pass@8": 0.555444460436145, + "pass@16": 0.646163410551263, + "pass@32": 0.7183930552407676, + "pass@64": 0.7730795118522583, + "pass@128": 0.8125 + }, + "20": { + "pass@1": 0.25666635479041916, + "pass@2": 0.35371186760337586, + "pass@4": 0.44875263421413675, + "pass@8": 0.5353646936771068, + "pass@16": 0.6088692333415894, + "pass@32": 0.6678819447924638, + "pass@64": 0.7171705545064891, + "pass@128": 0.7604790419161677 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 590516, + "answer_accuracy": 0.7370835996165521, + "avg_response_len": 308.4335244747563, + "resp_tokens_sum": 247102135, + "pass_at_k": { + "pass@1": 0.7370835996165521, + "pass@2": 0.7942608903022133, + "pass@4": 0.8403834290095774, + "pass@8": 0.8756561866580052, + "pass@16": 0.9012690648757998, + "pass@32": 0.9196849059760968, + "pass@64": 0.9333647386131214, + "pass@128": 0.9439207541140757 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9142024253731343, + "pass@2": 0.944686875661065, + "pass@4": 0.9640530558680164, + "pass@8": 0.9767311174177122, + "pass@16": 0.9841315222852977, + "pass@32": 0.988453337047995, + "pass@64": 0.9914290897594195, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9992939139941691, + "pass@2": 0.999901718280113, + "pass@4": 0.9999975704567541, + "pass@8": 0.9999999990001509, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999543128654971, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9751378676470588, + "pass@2": 0.9859548691523854, + "pass@4": 0.991848031220362, + "pass@8": 0.9948451265466661, + "pass@16": 0.9969022442892146, + "pass@32": 0.9986983612565055, + "pass@64": 0.9998054905278431, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9956131703470031, + "pass@2": 0.9994387900842048, + "pass@4": 0.9999703133519979, + "pass@8": 0.9999999080014058, + "pass@16": 0.9999999999997884, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9798753955696202, + "pass@2": 0.9909910576348052, + "pass@4": 0.9956166736080455, + "pass@8": 0.9983780579914756, + "pass@16": 0.9996911568597177, + "pass@32": 0.9999879278375395, + "pass@64": 0.9999999969925794, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9516938813813813, + "pass@2": 0.9712276252630587, + "pass@4": 0.981084519840425, + "pass@8": 0.9870884229283264, + "pass@16": 0.9914014767086731, + "pass@32": 0.9952692003194525, + "pass@64": 0.9986212451108519, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9542554260450161, + "pass@2": 0.979502003316708, + "pass@4": 0.9905159786095866, + "pass@8": 0.9957740359917082, + "pass@16": 0.9983627825430983, + "pass@32": 0.9995804979022844, + "pass@64": 0.9999788653279279, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8656746031746032, + "pass@2": 0.9171107517810279, + "pass@4": 0.9470395081567186, + "pass@8": 0.9623568039844478, + "pass@16": 0.9695305037462729, + "pass@32": 0.9745577642079248, + "pass@64": 0.9795424464659857, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8213637232415902, + "pass@2": 0.8814330046473546, + "pass@4": 0.9179910862632998, + "pass@8": 0.9419750015104567, + "pass@16": 0.9571606869660971, + "pass@32": 0.9666854395057514, + "pass@64": 0.9742835725052125, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.6469907407407407, + "pass@2": 0.7466792887868181, + "pass@4": 0.8208101967578123, + "pass@8": 0.8725635202269273, + "pass@16": 0.9070921475728697, + "pass@32": 0.930733948765345, + "pass@64": 0.9495482968036301, + "pass@128": 0.9629629629629629 + }, + "14": { + "pass@1": 0.6310969955489614, + "pass@2": 0.734162889319844, + "pass@4": 0.8103055706686509, + "pass@8": 0.8623148768926493, + "pass@16": 0.8965129105135972, + "pass@32": 0.9179342583412952, + "pass@64": 0.9306748140021945, + "pass@128": 0.9406528189910979 + }, + "15": { + "pass@1": 0.5588831018518519, + "pass@2": 0.6681324122679111, + "pass@4": 0.7556437441847542, + "pass@8": 0.8211991552952789, + "pass@16": 0.864726515839861, + "pass@32": 0.8911079555979848, + "pass@64": 0.9118444609142939, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.47517975663716816, + "pass@2": 0.6039369352890623, + "pass@4": 0.7087667821942608, + "pass@8": 0.786674014740112, + "pass@16": 0.8436490685622383, + "pass@32": 0.884715369398746, + "pass@64": 0.9096067784033466, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.3655167545180723, + "pass@2": 0.4790872397068589, + "pass@4": 0.5833546906975483, + "pass@8": 0.671439013579416, + "pass@16": 0.7435960427540855, + "pass@32": 0.7994048434573172, + "pass@64": 0.8379549225423192, + "pass@128": 0.8644578313253012 + }, + "18": { + "pass@1": 0.3504536290322581, + "pass@2": 0.46579049720599447, + "pass@4": 0.5706520888517969, + "pass@8": 0.6538244468352877, + "pass@16": 0.7152425434465555, + "pass@32": 0.7605848064509636, + "pass@64": 0.7940381169142109, + "pass@128": 0.8161290322580645 + }, + "19": { + "pass@1": 0.2521714501510574, + "pass@2": 0.3554680995313651, + "pass@4": 0.45941532147983016, + "pass@8": 0.5511710626280015, + "pass@16": 0.6247485613948076, + "pass@32": 0.6830405336519113, + "pass@64": 0.7311339608306535, + "pass@128": 0.7734138972809668 + }, + "20": { + "pass@1": 0.24840465465465467, + "pass@2": 0.35381900108770165, + "pass@4": 0.45993585176852897, + "pass@8": 0.5529589045021037, + "pass@16": 0.6243444369888672, + "pass@32": 0.6769581699826803, + "pass@64": 0.7199484451519275, + "pass@128": 0.7597597597597597 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch1-RL-op17-20/composition/summary.csv b/composition/Qwen3-100M-epoch1-RL-op17-20/composition/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..e994c2e4fe484d0d650a4970bfaa0c6064d07cca --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op17-20/composition/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.9732239583333333,0.52322109375,0.7363803453947368,0.0,0.0,0.0,199.84788975694445,373.5542125,1152000,1280000,2432000 diff --git a/composition/Qwen3-100M-epoch1-RL-op17-20/composition/summary.json b/composition/Qwen3-100M-epoch1-RL-op17-20/composition/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..cae08d9cd307f8b347f13bb829f4312c65bb5c3d --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op17-20/composition/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.9732239583333333, + "ood_acc": 0.52322109375, + "total_acc": 0.7363803453947368, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.84788975694445, + "ood_avg_resp_len": 373.5542125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 594424, + "answer_accuracy": 0.7341033038254822, + "avg_response_len": 278.0539576252766, + "resp_tokens_sum": 225148075, + "pass_at_k": { + "pass@1": 0.7341033038254822, + "pass@2": 0.7896779103113946, + "pass@4": 0.833522680811745, + "pass@8": 0.8682149334039494, + "pass@16": 0.8954587981157262, + "pass@32": 0.9165588466336974, + "pass@64": 0.9333368703889094, + "pass@128": 0.9470439456212456 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9062269542772862, + "pass@2": 0.93377856943767, + "pass@4": 0.9505952383717517, + "pass@8": 0.9616645650151304, + "pass@16": 0.9708360396229119, + "pass@32": 0.979137717466014, + "pass@64": 0.9861165304911527, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9986381880733946, + "pass@2": 0.9996613812034965, + "pass@4": 0.9999678924882097, + "pass@8": 0.9999997475359087, + "pass@16": 0.9999999999945448, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999560546875, + "pass@2": 0.9999650129183071, + "pass@4": 0.9999997067749344, + "pass@8": 0.9999999999934361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9850198412698413, + "pass@2": 0.9903390982377203, + "pass@4": 0.9935438281524333, + "pass@8": 0.9957480083422041, + "pass@16": 0.997451178844889, + "pass@32": 0.9988963683428546, + "pass@64": 0.9998099535588568, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.997029378742515, + "pass@2": 0.9992923847187516, + "pass@4": 0.9998629397560335, + "pass@8": 0.9999946418627286, + "pass@16": 0.9999999952012559, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9821329511834319, + "pass@2": 0.9910419966919816, + "pass@4": 0.9939767165753687, + "pass@8": 0.9954793655356908, + "pass@16": 0.9966585838989757, + "pass@32": 0.9976339233964078, + "pass@64": 0.9985187345105551, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9526742788461539, + "pass@2": 0.9762446686351706, + "pass@4": 0.9875043777580687, + "pass@8": 0.9944048642135856, + "pass@16": 0.9981674442303922, + "pass@32": 0.999578041813267, + "pass@64": 0.9999789328776327, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9322843309859155, + "pass@2": 0.9601696794942888, + "pass@4": 0.9737766655048402, + "pass@8": 0.9832953184438525, + "pass@16": 0.9911020557032694, + "pass@32": 0.996326244380213, + "pass@64": 0.9989353951370694, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8724327195467422, + "pass@2": 0.923634385246816, + "pass@4": 0.9503822968481626, + "pass@8": 0.9648353122583307, + "pass@16": 0.973952347913156, + "pass@32": 0.9806782677294469, + "pass@64": 0.9856572373418709, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8221883827683616, + "pass@2": 0.8813051903999292, + "pass@4": 0.9115400233339476, + "pass@8": 0.9287599940682195, + "pass@16": 0.9401571241780109, + "pass@32": 0.9498534527546343, + "pass@64": 0.9593712803795826, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.6445434952978056, + "pass@2": 0.7376050588699922, + "pass@4": 0.8092676895011944, + "pass@8": 0.8633510798804404, + "pass@16": 0.9027293938044151, + "pass@32": 0.93160249813217, + "pass@64": 0.9544420983735626, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.6434725467289719, + "pass@2": 0.7494752955822113, + "pass@4": 0.8239008923534091, + "pass@8": 0.8768464903393149, + "pass@16": 0.9131865701372338, + "pass@32": 0.9355432517352564, + "pass@64": 0.948839701524294, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.5680771315028902, + "pass@2": 0.6790506519958127, + "pass@4": 0.7673308419540041, + "pass@8": 0.8321651360027117, + "pass@16": 0.8752393426050006, + "pass@32": 0.9046743185604588, + "pass@64": 0.926849629105118, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.4576360887096774, + "pass@2": 0.5696969456438907, + "pass@4": 0.6595860659554652, + "pass@8": 0.730592961465299, + "pass@16": 0.789544236735948, + "pass@32": 0.8369620391949317, + "pass@64": 0.8752227021714117, + "pass@128": 0.9096774193548387 + }, + "17": { + "pass@1": 0.3633941473988439, + "pass@2": 0.4766912208137999, + "pass@4": 0.5772174451820685, + "pass@8": 0.6598159310844616, + "pass@16": 0.7250069898614739, + "pass@32": 0.7758477060303068, + "pass@64": 0.8176952910565459, + "pass@128": 0.8554913294797688 + }, + "18": { + "pass@1": 0.32930803571428574, + "pass@2": 0.43295451349831254, + "pass@4": 0.5302455525202205, + "pass@8": 0.6154654950714722, + "pass@16": 0.6884315150553474, + "pass@32": 0.7453938186323203, + "pass@64": 0.7856894486479148, + "pass@128": 0.8142857142857143 + }, + "19": { + "pass@1": 0.25736674174174173, + "pass@2": 0.3567418304130902, + "pass@4": 0.4550597968609781, + "pass@8": 0.5463556456220624, + "pass@16": 0.624890768528339, + "pass@32": 0.6885183056794916, + "pass@64": 0.7422557482331051, + "pass@128": 0.7897897897897898 + }, + "20": { + "pass@1": 0.26895645645645644, + "pass@2": 0.3729963969875385, + "pass@4": 0.4755003121794459, + "pass@8": 0.5662618857243946, + "pass@16": 0.6426840209099434, + "pass@32": 0.7081242437863503, + "pass@64": 0.766509055353242, + "pass@128": 0.8168168168168168 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 605937, + "answer_accuracy": 0.7379396434138737, + "avg_response_len": 287.5632684625877, + "resp_tokens_sum": 236123951, + "pass_at_k": { + "pass@1": 0.7379396434138737, + "pass@2": 0.7940764218336671, + "pass@4": 0.8394079595711046, + "pass@8": 0.8747613566771798, + "pass@16": 0.9012971950227997, + "pass@32": 0.921023843043457, + "pass@64": 0.9361730114778367, + "pass@128": 0.9477786438035853 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9080233895705522, + "pass@2": 0.9325764757741168, + "pass@4": 0.9461707905760242, + "pass@8": 0.953994960326316, + "pass@16": 0.9594360890980274, + "pass@32": 0.9644964902058345, + "pass@64": 0.9691135912953294, + "pass@128": 0.9723926380368099 + }, + "2": { + "pass@1": 0.9972691869300911, + "pass@2": 0.9975487189766172, + "pass@4": 0.9980302595154329, + "pass@8": 0.9987425146592096, + "pass@16": 0.9995122033857184, + "pass@32": 0.9999420504918746, + "pass@64": 0.9999998113939417, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977035984848485, + "pass@2": 0.9995783375089478, + "pass@4": 0.9999825490563681, + "pass@8": 0.9999999669910425, + "pass@16": 0.9999999999999619, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990292159763313, + "pass@2": 0.9997160811629315, + "pass@4": 0.9999746546104813, + "pass@8": 0.9999998408559615, + "pass@16": 0.9999999999980078, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9872735507246376, + "pass@2": 0.9934212027844346, + "pass@4": 0.9960166188465572, + "pass@8": 0.9977547571009451, + "pass@16": 0.9991579262409745, + "pass@32": 0.9998845013853158, + "pass@64": 0.9999991115258494, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9901728151862464, + "pass@2": 0.9940444295286872, + "pass@4": 0.9968823914918945, + "pass@8": 0.9989277166937187, + "pass@16": 0.9998251780193597, + "pass@32": 0.9999947958543877, + "pass@64": 0.9999999995301808, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9826138005780347, + "pass@2": 0.9935234229211234, + "pass@4": 0.9980839851665938, + "pass@8": 0.9997614186744043, + "pass@16": 0.9999954079465505, + "pass@32": 0.9999999992044007, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9533230633802817, + "pass@2": 0.9697533131862036, + "pass@4": 0.9778347227723291, + "pass@8": 0.9826861224766935, + "pass@16": 0.9861681574883515, + "pass@32": 0.9882344037134858, + "pass@64": 0.9887213879364309, + "pass@128": 0.9887323943661972 + }, + "9": { + "pass@1": 0.9396285553892215, + "pass@2": 0.9657106800179176, + "pass@4": 0.9772753267368522, + "pass@8": 0.9836481530198901, + "pass@16": 0.9875451790737914, + "pass@32": 0.990121950847974, + "pass@64": 0.9924853659444284, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.8694700677710844, + "pass@2": 0.909607055189261, + "pass@4": 0.9317818410876343, + "pass@8": 0.9454585632863172, + "pass@16": 0.9556582000789117, + "pass@32": 0.9649759331729336, + "pass@64": 0.9740005463669225, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8088753918495298, + "pass@2": 0.8641022634709847, + "pass@4": 0.8943006227865717, + "pass@8": 0.9132144665068072, + "pass@16": 0.9271130674479252, + "pass@32": 0.9389810679709106, + "pass@64": 0.9492095408011472, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6670386904761905, + "pass@2": 0.7605552063345015, + "pass@4": 0.8290062037938528, + "pass@8": 0.8782387659005888, + "pass@16": 0.9117272202364387, + "pass@32": 0.9333991943128012, + "pass@64": 0.9489098468813019, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.636421783625731, + "pass@2": 0.7366485644886493, + "pass@4": 0.8129156654212081, + "pass@8": 0.8685094524738564, + "pass@16": 0.9069490253899266, + "pass@32": 0.9322473949964639, + "pass@64": 0.9468054042889353, + "pass@128": 0.9532163742690059 + }, + "15": { + "pass@1": 0.5566761363636363, + "pass@2": 0.6740545215938918, + "pass@4": 0.7655083620229282, + "pass@8": 0.830314977109201, + "pass@16": 0.8728943261197528, + "pass@32": 0.9007579003977949, + "pass@64": 0.9218918824899519, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.4544827279202279, + "pass@2": 0.5782294540906747, + "pass@4": 0.6807875420486968, + "pass@8": 0.7586801919280979, + "pass@16": 0.8155312114186396, + "pass@32": 0.8572814405727026, + "pass@64": 0.8900660067538252, + "pass@128": 0.9145299145299145 + }, + "17": { + "pass@1": 0.4038480201863354, + "pass@2": 0.5302863806426372, + "pass@4": 0.6393859222566122, + "pass@8": 0.7230418188024726, + "pass@16": 0.7829436866294603, + "pass@32": 0.8259523452426276, + "pass@64": 0.8575249008903864, + "pass@128": 0.8757763975155279 + }, + "18": { + "pass@1": 0.35723805147058824, + "pass@2": 0.48015357225567384, + "pass@4": 0.5944133659763114, + "pass@8": 0.686807116888759, + "pass@16": 0.7560314779655988, + "pass@32": 0.8086900156791244, + "pass@64": 0.8510250714257173, + "pass@128": 0.888235294117647 + }, + "19": { + "pass@1": 0.24251302083333334, + "pass@2": 0.34276867735283073, + "pass@4": 0.45104315531987044, + "pass@8": 0.555444460436145, + "pass@16": 0.646163410551263, + "pass@32": 0.7183930552407676, + "pass@64": 0.7730795118522583, + "pass@128": 0.8125 + }, + "20": { + "pass@1": 0.25666635479041916, + "pass@2": 0.35371186760337586, + "pass@4": 0.44875263421413675, + "pass@8": 0.5353646936771068, + "pass@16": 0.6088692333415894, + "pass@32": 0.6678819447924638, + "pass@64": 0.7171705545064891, + "pass@128": 0.7604790419161677 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 590516, + "answer_accuracy": 0.7370835996165521, + "avg_response_len": 308.4335244747563, + "resp_tokens_sum": 247102135, + "pass_at_k": { + "pass@1": 0.7370835996165521, + "pass@2": 0.7942608903022133, + "pass@4": 0.8403834290095774, + "pass@8": 0.8756561866580052, + "pass@16": 0.9012690648757998, + "pass@32": 0.9196849059760968, + "pass@64": 0.9333647386131214, + "pass@128": 0.9439207541140757 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9142024253731343, + "pass@2": 0.944686875661065, + "pass@4": 0.9640530558680164, + "pass@8": 0.9767311174177122, + "pass@16": 0.9841315222852977, + "pass@32": 0.988453337047995, + "pass@64": 0.9914290897594195, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9992939139941691, + "pass@2": 0.999901718280113, + "pass@4": 0.9999975704567541, + "pass@8": 0.9999999990001509, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999543128654971, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9751378676470588, + "pass@2": 0.9859548691523854, + "pass@4": 0.991848031220362, + "pass@8": 0.9948451265466661, + "pass@16": 0.9969022442892146, + "pass@32": 0.9986983612565055, + "pass@64": 0.9998054905278431, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9956131703470031, + "pass@2": 0.9994387900842048, + "pass@4": 0.9999703133519979, + "pass@8": 0.9999999080014058, + "pass@16": 0.9999999999997884, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9798753955696202, + "pass@2": 0.9909910576348052, + "pass@4": 0.9956166736080455, + "pass@8": 0.9983780579914756, + "pass@16": 0.9996911568597177, + "pass@32": 0.9999879278375395, + "pass@64": 0.9999999969925794, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9516938813813813, + "pass@2": 0.9712276252630587, + "pass@4": 0.981084519840425, + "pass@8": 0.9870884229283264, + "pass@16": 0.9914014767086731, + "pass@32": 0.9952692003194525, + "pass@64": 0.9986212451108519, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9542554260450161, + "pass@2": 0.979502003316708, + "pass@4": 0.9905159786095866, + "pass@8": 0.9957740359917082, + "pass@16": 0.9983627825430983, + "pass@32": 0.9995804979022844, + "pass@64": 0.9999788653279279, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8656746031746032, + "pass@2": 0.9171107517810279, + "pass@4": 0.9470395081567186, + "pass@8": 0.9623568039844478, + "pass@16": 0.9695305037462729, + "pass@32": 0.9745577642079248, + "pass@64": 0.9795424464659857, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8213637232415902, + "pass@2": 0.8814330046473546, + "pass@4": 0.9179910862632998, + "pass@8": 0.9419750015104567, + "pass@16": 0.9571606869660971, + "pass@32": 0.9666854395057514, + "pass@64": 0.9742835725052125, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.6469907407407407, + "pass@2": 0.7466792887868181, + "pass@4": 0.8208101967578123, + "pass@8": 0.8725635202269273, + "pass@16": 0.9070921475728697, + "pass@32": 0.930733948765345, + "pass@64": 0.9495482968036301, + "pass@128": 0.9629629629629629 + }, + "14": { + "pass@1": 0.6310969955489614, + "pass@2": 0.734162889319844, + "pass@4": 0.8103055706686509, + "pass@8": 0.8623148768926493, + "pass@16": 0.8965129105135972, + "pass@32": 0.9179342583412952, + "pass@64": 0.9306748140021945, + "pass@128": 0.9406528189910979 + }, + "15": { + "pass@1": 0.5588831018518519, + "pass@2": 0.6681324122679111, + "pass@4": 0.7556437441847542, + "pass@8": 0.8211991552952789, + "pass@16": 0.864726515839861, + "pass@32": 0.8911079555979848, + "pass@64": 0.9118444609142939, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.47517975663716816, + "pass@2": 0.6039369352890623, + "pass@4": 0.7087667821942608, + "pass@8": 0.786674014740112, + "pass@16": 0.8436490685622383, + "pass@32": 0.884715369398746, + "pass@64": 0.9096067784033466, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.3655167545180723, + "pass@2": 0.4790872397068589, + "pass@4": 0.5833546906975483, + "pass@8": 0.671439013579416, + "pass@16": 0.7435960427540855, + "pass@32": 0.7994048434573172, + "pass@64": 0.8379549225423192, + "pass@128": 0.8644578313253012 + }, + "18": { + "pass@1": 0.3504536290322581, + "pass@2": 0.46579049720599447, + "pass@4": 0.5706520888517969, + "pass@8": 0.6538244468352877, + "pass@16": 0.7152425434465555, + "pass@32": 0.7605848064509636, + "pass@64": 0.7940381169142109, + "pass@128": 0.8161290322580645 + }, + "19": { + "pass@1": 0.2521714501510574, + "pass@2": 0.3554680995313651, + "pass@4": 0.45941532147983016, + "pass@8": 0.5511710626280015, + "pass@16": 0.6247485613948076, + "pass@32": 0.6830405336519113, + "pass@64": 0.7311339608306535, + "pass@128": 0.7734138972809668 + }, + "20": { + "pass@1": 0.24840465465465467, + "pass@2": 0.35381900108770165, + "pass@4": 0.45993585176852897, + "pass@8": 0.5529589045021037, + "pass@16": 0.6243444369888672, + "pass@32": 0.6769581699826803, + "pass@64": 0.7199484451519275, + "pass@128": 0.7597597597597597 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch1-RL-op7-10/composition/hf_model_metrics.json b/composition/Qwen3-100M-epoch1-RL-op7-10/composition/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..afc17d72c1def2649c0e35cb101ecb59144b0b99 --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op7-10/composition/hf_model_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.9859791666666666, + "avg_response_len": 198.56494618055555, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9859791666666666, + "pass@2": 0.9908360263560759, + "pass@4": 0.9936637789338826, + "pass@8": 0.9955060493003168, + "pass@16": 0.9966856802457575, + "pass@32": 0.9974806767559844, + "pass@64": 0.9980905700065218, + "pass@128": 0.9986666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9374609375, + "pass@2": 0.9568209891732286, + "pass@4": 0.969609387326584, + "pass@8": 0.9780410694302417, + "pass@16": 0.9835916232526909, + "pass@32": 0.9875500212448162, + "pass@64": 0.9904628626561853, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999953125, + "pass@2": 0.9999987696850394, + "pass@4": 0.9999999995313085, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9916328125, + "pass@2": 0.9935068897637795, + "pass@4": 0.9949222595613048, + "pass@8": 0.9958238935603797, + "pass@16": 0.9963295794851466, + "pass@32": 0.9967494938802602, + "pass@64": 0.9974999999977977, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.9995703125, + "pass@2": 0.9999409448818898, + "pass@4": 0.999997050056243, + "pass@8": 0.9999999944822617, + "pass@16": 0.9999999999999968, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.995484375, + "pass@2": 0.9976993110236222, + "pass@4": 0.9988875447131609, + "pass@8": 0.9995084998310533, + "pass@16": 0.9998131921683332, + "pass@32": 0.9999736715410081, + "pass@64": 0.9999998615776614, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.985671875, + "pass@2": 0.991946973425197, + "pass@4": 0.9946909814398197, + "pass@8": 0.9964191371747406, + "pass@16": 0.9977549650714689, + "pass@32": 0.9986390182664511, + "pass@64": 0.9989690611535798, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.9640546875, + "pass@2": 0.9776104822834655, + "pass@4": 0.9848667877765283, + "pass@8": 0.989761849224164, + "pass@16": 0.9926817622341847, + "pass@32": 0.9944138858713218, + "pass@64": 0.9958833446734731, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.9374609375, + "2": 1.0, + "3": 0.999984375, + "4": 0.999953125, + "5": 0.9916328125, + "6": 0.9995703125, + "7": 0.995484375, + "8": 0.985671875, + "9": 0.9640546875 + }, + "per_op_avg_response_len": { + "10": 278.8289140625, + "2": 115.11125, + "3": 143.84865625, + "4": 158.940453125, + "5": 171.911078125, + "6": 197.4796953125, + "7": 222.2320390625, + "8": 241.496875, + "9": 257.2355546875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.8289140625, + "2": 115.11125, + "3": 143.84865625, + "4": 158.940453125, + "5": 171.911078125, + "6": 197.4796953125, + "7": 222.2320390625, + "8": 241.496875, + "9": 257.2355546875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 373201, + "answer_accuracy": 0.9846784236744344, + "avg_response_len": 189.06281397754137, + "resp_tokens_sum": 71656319, + "pass_at_k": { + "pass@1": 0.9846784236744344, + "pass@2": 0.9902760243400436, + "pass@4": 0.9934085639877593, + "pass@8": 0.9953909834112721, + "pass@16": 0.9965699866164012, + "pass@32": 0.9973700878530838, + "pass@64": 0.9981103904443656, + "pass@128": 0.9986491050320837 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.935126290560472, + "pass@2": 0.9578778627505631, + "pass@4": 0.9718444243142172, + "pass@8": 0.979102374498778, + "pass@16": 0.9830894960513075, + "pass@32": 0.9864112036490584, + "pass@64": 0.9903896215785958, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.9999961552657479, + "pass@4": 0.9999999985353394, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9935515873015873, + "pass@2": 0.9958173197100364, + "pass@4": 0.9979374661500646, + "pass@8": 0.999387736990032, + "pass@16": 0.9999321953197198, + "pass@32": 0.9999993438733693, + "pass@64": 0.9999999999980798, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999064371257484, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9965329142011834, + "pass@2": 0.9975601179937568, + "pass@4": 0.9980142238137393, + "pass@8": 0.9986853311816261, + "pass@16": 0.9994492227814368, + "pass@32": 0.9999221053455868, + "pass@64": 0.9999995904664537, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9855018028846154, + "pass@2": 0.9916890647082576, + "pass@4": 0.9938409859224325, + "pass@8": 0.9956982672073414, + "pass@16": 0.9975426088063732, + "pass@32": 0.9990999858579014, + "pass@64": 0.9999054915606393, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9561399647887324, + "pass@2": 0.9724600060995899, + "pass@4": 0.9810422577459504, + "pass@8": 0.9870883337063461, + "pass@16": 0.9902834322150555, + "pass@32": 0.9919063988207483, + "pass@64": 0.9934997449201314, + "pass@128": 0.9943661971830986 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 385441, + "answer_accuracy": 0.9866506594036697, + "avg_response_len": 196.57071950769986, + "resp_tokens_sum": 76791531, + "pass_at_k": { + "pass@1": 0.9866506594036697, + "pass@2": 0.9912304181587394, + "pass@4": 0.9938927370214861, + "pass@8": 0.9954915370529469, + "pass@16": 0.9965414415099906, + "pass@32": 0.997352569155673, + "pass@64": 0.9980118803965365, + "pass@128": 0.9986893840104849 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9348399156441718, + "pass@2": 0.9528630863243323, + "pass@4": 0.9651524198853975, + "pass@8": 0.9735246936336497, + "pass@16": 0.9791935962267696, + "pass@32": 0.983532318769681, + "pass@64": 0.9869365859226864, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 0.9999996271772846, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9983242753623188, + "pass@2": 0.9996569382631519, + "pass@4": 0.9999822019530168, + "pass@8": 0.9999999500901793, + "pass@16": 0.999999999999874, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990821991404012, + "pass@2": 0.9998346654107347, + "pass@4": 0.9999915487820469, + "pass@8": 0.9999999841898617, + "pass@16": 0.9999999999999909, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9943325505780347, + "pass@2": 0.997179520731874, + "pass@4": 0.9990721668281349, + "pass@8": 0.9998892341279048, + "pass@16": 0.9999982631322548, + "pass@32": 0.9999999998103838, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9881161971830986, + "pass@2": 0.9947415298879895, + "pass@4": 0.9977329795043224, + "pass@8": 0.9990231622731005, + "pass@16": 0.9997126319909666, + "pass@32": 0.9999800145860199, + "pass@64": 0.9999999852572564, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9628555389221557, + "pass@2": 0.9749122571785567, + "pass@4": 0.9816043618612045, + "pass@8": 0.9857971439594125, + "pass@16": 0.9890119272995747, + "pass@32": 0.9919029941070923, + "pass@64": 0.9945836442907414, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 377206, + "answer_accuracy": 0.9865824824238366, + "avg_response_len": 210.02199112822228, + "resp_tokens_sum": 80298968, + "pass_at_k": { + "pass@1": 0.9865824824238366, + "pass@2": 0.9909881797500466, + "pass@4": 0.9936828319543419, + "pass@8": 0.9956349416593467, + "pass@16": 0.9969477443428747, + "pass@32": 0.997721198395635, + "pass@64": 0.9981513240651828, + "pass@128": 0.9986608637428859 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9423740671641792, + "pass@2": 0.959603067340463, + "pass@4": 0.9716848913475367, + "pass@8": 0.9813621323898088, + "pass@16": 0.9883796170787192, + "pass@32": 0.9926122008622965, + "pass@64": 0.9939685191947623, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9830652573529411, + "pass@2": 0.9851258539833256, + "pass@4": 0.9869944060301287, + "pass@8": 0.9882846283159047, + "pass@16": 0.9892674645867016, + "pass@32": 0.9904402957651445, + "pass@64": 0.992647058818831, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9997535488958991, + "pass@2": 0.9999961188802505, + "pass@4": 0.9999999985214783, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956240110759493, + "pass@2": 0.9984173321788099, + "pass@4": 0.9996195169037414, + "pass@8": 0.9999720977323062, + "pass@16": 0.9999998602039476, + "pass@32": 0.9999999999993893, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9832254129129129, + "pass@2": 0.9892094358137666, + "pass@4": 0.9922444027266861, + "pass@8": 0.9943184840813796, + "pass@16": 0.9958669272284904, + "pass@32": 0.996777530032279, + "pass@64": 0.9969926517127159, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9743770096463023, + "pass@2": 0.9863874154999117, + "pass@4": 0.9927361074439651, + "pass@8": 0.9970715262892839, + "pass@16": 0.9993606433433512, + "pass@32": 0.9999727146565515, + "pass@64": 0.9999999936775518, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.40523515625, + "avg_response_len": 329.831925, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.40523515625, + "pass@2": 0.4756683932086604, + "pass@4": 0.5370877914885598, + "pass@8": 0.5897599180776594, + "pass@16": 0.6348370319957497, + "pass@32": 0.6740292861921086, + "pass@64": 0.7090949625642076, + "pass@128": 0.7411 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8928828125, + "pass@2": 0.9304996309055115, + "pass@4": 0.9517802391263591, + "pass@8": 0.9646634253663902, + "pass@16": 0.9727097527282708, + "pass@32": 0.9780033779449054, + "pass@64": 0.9816839060885125, + "pass@128": 0.985 + }, + "12": { + "pass@1": 0.8229296875, + "pass@2": 0.8794469734251978, + "pass@4": 0.91197360423697, + "pass@8": 0.9329927197607489, + "pass@16": 0.9475447037674732, + "pass@32": 0.9577132375765316, + "pass@64": 0.9647098494024046, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.59034375, + "pass@2": 0.6834507874015745, + "pass@4": 0.7525489787214105, + "pass@8": 0.8037467958544191, + "pass@16": 0.8420582580595548, + "pass@32": 0.871225002739347, + "pass@64": 0.8942999913048029, + "pass@128": 0.912 + }, + "14": { + "pass@1": 0.4938125, + "pass@2": 0.593524360236221, + "pass@4": 0.672868000656167, + "pass@8": 0.7339828959553184, + "pass@16": 0.7823024545523954, + "pass@32": 0.8216617138740999, + "pass@64": 0.8549526964970245, + "pass@128": 0.884 + }, + "15": { + "pass@1": 0.3363125, + "pass@2": 0.43280684055118135, + "pass@4": 0.5218394927821517, + "pass@8": 0.5989764468164341, + "pass@16": 0.6634686127535178, + "pass@32": 0.7175496425757314, + "pass@64": 0.7629674502846845, + "pass@128": 0.804 + }, + "16": { + "pass@1": 0.2378671875, + "pass@2": 0.31033796751968495, + "pass@4": 0.3802311455755525, + "pass@8": 0.4448543139638245, + "pass@16": 0.503592993748637, + "pass@32": 0.5582595364270498, + "pass@64": 0.6113511394585666, + "pass@128": 0.66 + }, + "17": { + "pass@1": 0.19515625, + "pass@2": 0.26501722440944897, + "pass@4": 0.3358142904949378, + "pass@8": 0.4028738936800769, + "pass@16": 0.463582430082757, + "pass@32": 0.5181274567245612, + "pass@64": 0.56847235268569, + "pass@128": 0.62 + }, + "18": { + "pass@1": 0.1873125, + "pass@2": 0.2518336614173228, + "pass@4": 0.31633660601799735, + "pass@8": 0.37678362942783716, + "pass@16": 0.4330532993922898, + "pass@32": 0.4864742309003656, + "pass@64": 0.5378284053318653, + "pass@128": 0.586 + }, + "19": { + "pass@1": 0.160859375, + "pass@2": 0.2180234990157482, + "pass@4": 0.2736183600487438, + "pass@8": 0.3242448631843231, + "pass@16": 0.3698350069177702, + "pass@32": 0.41086370085369817, + "pass@64": 0.44953961912830304, + "pass@128": 0.488 + }, + "20": { + "pass@1": 0.134875, + "pass@2": 0.19174298720472446, + "pass@4": 0.25386719722534645, + "pass@8": 0.31448019676719036, + "pass@16": 0.37022280795466495, + "pass@32": 0.4204149623047817, + "pass@64": 0.4651442154601655, + "pass@128": 0.504 + } + }, + "per_op_accuracy": { + "11": 0.8928828125, + "12": 0.8229296875, + "13": 0.59034375, + "14": 0.4938125, + "15": 0.3363125, + "16": 0.2378671875, + "17": 0.19515625, + "18": 0.1873125, + "19": 0.160859375, + "20": 0.134875 + }, + "per_op_avg_response_len": { + "11": 295.3559765625, + "12": 304.267171875, + "13": 317.0078828125, + "14": 334.990734375, + "15": 340.2494453125, + "16": 335.35834375, + "17": 344.936578125, + "18": 344.8016171875, + "19": 337.8381484375, + "20": 343.5133515625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 295.3559765625, + "12": 304.267171875, + "13": 317.0078828125, + "14": 334.990734375, + "15": 340.2494453125, + "16": 335.35834375, + "17": 344.936578125, + "18": 344.8016171875, + "19": 337.8381484375, + "20": 343.5133515625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 174318, + "answer_accuracy": 0.40471303863298663, + "avg_response_len": 311.8808901374443, + "resp_tokens_sum": 134333337, + "pass_at_k": { + "pass@1": 0.40471303863298663, + "pass@2": 0.4742014835441279, + "pass@4": 0.5340066809349432, + "pass@8": 0.5852126039613251, + "pass@16": 0.630453081750082, + "pass@32": 0.6713997167337046, + "pass@64": 0.7079712875825656, + "pass@128": 0.7411589895988113 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8831223441926346, + "pass@2": 0.9245008336865118, + "pass@4": 0.9477521931571583, + "pass@8": 0.9607035690298851, + "pass@16": 0.968703861284288, + "pass@32": 0.9744709138233744, + "pass@64": 0.9794031631325362, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8216366525423728, + "pass@2": 0.8857683838249031, + "pass@4": 0.92275589942147, + "pass@8": 0.946291937886921, + "pass@16": 0.9618294686978542, + "pass@32": 0.9713007305324585, + "pass@64": 0.9763362592837835, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.5992848746081505, + "pass@2": 0.6910794837459577, + "pass@4": 0.7593168583159077, + "pass@8": 0.8106964463013567, + "pass@16": 0.8493344595262924, + "pass@32": 0.8775505709931961, + "pass@64": 0.8993150970974295, + "pass@128": 0.9153605015673981 + }, + "14": { + "pass@1": 0.47748734423676015, + "pass@2": 0.5846632999730172, + "pass@4": 0.6713725787197162, + "pass@8": 0.7362450922488023, + "pass@16": 0.7866651958227615, + "pass@32": 0.8300273996288141, + "pass@64": 0.8656000028744902, + "pass@128": 0.8940809968847352 + }, + "15": { + "pass@1": 0.3230897760115607, + "pass@2": 0.4112555328614995, + "pass@4": 0.4971151013998969, + "pass@8": 0.5771504498506533, + "pass@16": 0.6481016179810867, + "pass@32": 0.707089445007043, + "pass@64": 0.7521345054468381, + "pass@128": 0.7890173410404624 + }, + "16": { + "pass@1": 0.24778225806451612, + "pass@2": 0.321448358521717, + "pass@4": 0.3901999225903216, + "pass@8": 0.4514741921201661, + "pass@16": 0.507778863363312, + "pass@32": 0.562700667039377, + "pass@64": 0.6167569408920534, + "pass@128": 0.6645161290322581 + }, + "17": { + "pass@1": 0.1856936416184971, + "pass@2": 0.2507627241591187, + "pass@4": 0.3154409294394557, + "pass@8": 0.37876531624118964, + "pass@16": 0.4407413021254028, + "pass@32": 0.4991258395178229, + "pass@64": 0.5514502079284902, + "pass@128": 0.6040462427745664 + }, + "18": { + "pass@1": 0.17433035714285713, + "pass@2": 0.23157761529808754, + "pass@4": 0.2889288510364776, + "pass@8": 0.34402642259043337, + "pass@16": 0.3994066535761887, + "pass@32": 0.45911863063083785, + "pass@64": 0.5206270692676888, + "pass@128": 0.5771428571428572 + }, + "19": { + "pass@1": 0.16211524024024024, + "pass@2": 0.22172455427868798, + "pass@4": 0.27695589121179676, + "pass@8": 0.3244718288012454, + "pass@16": 0.36720452860714353, + "pass@32": 0.40559504045048284, + "pass@64": 0.44125668250848554, + "pass@128": 0.4804804804804805 + }, + "20": { + "pass@1": 0.14102383633633633, + "pass@2": 0.1923488449078999, + "pass@4": 0.24783760025492316, + "pass@8": 0.3033502821196472, + "pass@16": 0.3584227324664222, + "pass@32": 0.4129476389624526, + "pass@64": 0.46498198625491477, + "pass@128": 0.5135135135135135 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 175780, + "answer_accuracy": 0.4083500594707107, + "avg_response_len": 326.0048668413619, + "resp_tokens_sum": 140333359, + "pass_at_k": { + "pass@1": 0.4083500594707107, + "pass@2": 0.4790235140517138, + "pass@4": 0.5405782210151935, + "pass@8": 0.5929402951685977, + "pass@16": 0.637231184674975, + "pass@32": 0.6757553235815628, + "pass@64": 0.7110051517857473, + "pass@128": 0.7430865298840321 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.900414156626506, + "pass@2": 0.9350123179726779, + "pass@4": 0.9550508309955227, + "pass@8": 0.9674511498873033, + "pass@16": 0.9746696385552203, + "pass@32": 0.9781687408964208, + "pass@64": 0.9789054319272765, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8177164968652038, + "pass@2": 0.8662809622837115, + "pass@4": 0.8951960123323143, + "pass@8": 0.9141750532759667, + "pass@16": 0.9279755431874404, + "pass@32": 0.9392592527593965, + "pass@64": 0.9479558030667862, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.5941001400560224, + "pass@2": 0.6847323082776416, + "pass@4": 0.7461941632295956, + "pass@8": 0.7901620120404012, + "pass@16": 0.8250701005723396, + "pass@32": 0.8536865506090808, + "pass@64": 0.876800327740759, + "pass@128": 0.8935574229691877 + }, + "14": { + "pass@1": 0.5205820540935673, + "pass@2": 0.6189182149928629, + "pass@4": 0.6981114899014816, + "pass@8": 0.7601463796827449, + "pass@16": 0.8069542276262767, + "pass@32": 0.840170709001981, + "pass@64": 0.8655683771981271, + "pass@128": 0.8888888888888888 + }, + "15": { + "pass@1": 0.3488162878787879, + "pass@2": 0.4543855136005727, + "pass@4": 0.5469166294554091, + "pass@8": 0.6202028758837862, + "pass@16": 0.6771987839186235, + "pass@32": 0.725887979805038, + "pass@64": 0.7719392599869858, + "pass@128": 0.8181818181818182 + }, + "16": { + "pass@1": 0.23317307692307693, + "pass@2": 0.30855002299392065, + "pass@4": 0.3825931414021966, + "pass@8": 0.44921159232600466, + "pass@16": 0.5063128891342469, + "pass@32": 0.5572324306688641, + "pass@64": 0.6074042418530502, + "pass@128": 0.6552706552706553 + }, + "17": { + "pass@1": 0.2060850155279503, + "pass@2": 0.2831642478114148, + "pass@4": 0.36248983929415635, + "pass@8": 0.43754655513862, + "pass@16": 0.5028174506846871, + "pass@32": 0.5573710122030797, + "pass@64": 0.6053850077444561, + "pass@128": 0.6521739130434783 + }, + "18": { + "pass@1": 0.19751838235294117, + "pass@2": 0.2667702784854099, + "pass@4": 0.335324297789982, + "pass@8": 0.39891717056702747, + "pass@16": 0.45777200500975923, + "pass@32": 0.5134400046716384, + "pass@64": 0.5670193664414482, + "pass@128": 0.6147058823529412 + }, + "19": { + "pass@1": 0.15252976190476192, + "pass@2": 0.20051534085583056, + "pass@4": 0.24895375047985074, + "pass@8": 0.29642610320817747, + "pass@16": 0.3433369906473041, + "pass@32": 0.3918855662817668, + "pass@64": 0.44317301534958015, + "pass@128": 0.49404761904761907 + }, + "20": { + "pass@1": 0.12467252994011976, + "pass@2": 0.18133737446367107, + "pass@4": 0.24257957369849725, + "pass@8": 0.3015672795538963, + "pass@16": 0.3555785376399858, + "pass@32": 0.4049003421017203, + "pass@64": 0.449425700433271, + "pass@128": 0.48502994011976047 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 168603, + "answer_accuracy": 0.4025705799205379, + "avg_response_len": 352.22667710880194, + "resp_tokens_sum": 147518168, + "pass_at_k": { + "pass@1": 0.4025705799205379, + "pass@2": 0.4737285642434991, + "pass@4": 0.5366689719637757, + "pass@8": 0.5911676515265034, + "pass@16": 0.6368848489628256, + "pass@32": 0.6749595543726539, + "pass@64": 0.7082872608408296, + "pass@128": 0.738997555012225 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8958829365079365, + "pass@2": 0.9324658636420451, + "pass@4": 0.9528471080995822, + "pass@8": 0.9661628055119237, + "pass@16": 0.9751332371256648, + "pass@32": 0.9817876932941041, + "pass@64": 0.987168209850384, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8294151376146789, + "pass@2": 0.8854475186014598, + "pass@4": 0.9166681587393313, + "pass@8": 0.9369526964946373, + "pass@16": 0.951170867191561, + "pass@32": 0.9610063527149639, + "pass@64": 0.9684676221334558, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.5774016203703703, + "pass@2": 0.6745277717021484, + "pass@4": 0.7528876069310777, + "pass@8": 0.8118728123020478, + "pass@16": 0.8536127764393275, + "pass@32": 0.8843218272378883, + "pass@64": 0.9086443158249113, + "pass@128": 0.9290123456790124 + }, + "14": { + "pass@1": 0.4821958456973294, + "pass@2": 0.5761940991611951, + "pass@4": 0.6486744016048438, + "pass@8": 0.7052764376615864, + "pass@16": 0.7531293200151414, + "pass@32": 0.7949096027138687, + "pass@64": 0.8340377168325014, + "pass@128": 0.8694362017804155 + }, + "15": { + "pass@1": 0.3376977237654321, + "pass@2": 0.43384323053368284, + "pass@4": 0.5227011727700702, + "pass@8": 0.6006649448347476, + "pass@16": 0.6658946118485112, + "pass@32": 0.7202273495914582, + "pass@64": 0.7653980111246035, + "pass@128": 0.8055555555555556 + }, + "16": { + "pass@1": 0.23366058259587022, + "pass@2": 0.30202925754302834, + "pass@4": 0.36866954849670325, + "pass@8": 0.4342892197644405, + "pass@16": 0.4969490325070487, + "pass@32": 0.5552617890267609, + "pass@64": 0.6104943920106468, + "pass@128": 0.6607669616519174 + }, + "17": { + "pass@1": 0.1944182981927711, + "pass@2": 0.26227239173228345, + "pass@4": 0.3311747007715901, + "pass@8": 0.39437079369274153, + "pass@16": 0.44933349526174327, + "pass@32": 0.49986882627109364, + "pass@64": 0.5504114706286666, + "pass@128": 0.6054216867469879 + }, + "18": { + "pass@1": 0.19077620967741934, + "pass@2": 0.2583212947675897, + "pass@4": 0.3264556351826988, + "pass@8": 0.38949207589805374, + "pass@16": 0.44393060947518126, + "pass@32": 0.48778422126198656, + "pass@64": 0.5252333758002631, + "pass@128": 0.5645161290322581 + }, + "19": { + "pass@1": 0.16805135951661632, + "pass@2": 0.2320727127292623, + "pass@4": 0.29529784928696595, + "pass@8": 0.35225550911045606, + "pass@16": 0.3993796677767297, + "pass@32": 0.43542900940487445, + "pass@64": 0.46433537974476874, + "pass@128": 0.48942598187311176 + }, + "20": { + "pass@1": 0.13895927177177178, + "pass@2": 0.20157399032891157, + "pass@4": 0.27121831448996814, + "pass@8": 0.33856180615719716, + "pass@16": 0.39671113054531904, + "pass@32": 0.44344349630123264, + "pass@64": 0.48107216244013823, + "pass@128": 0.5135135135135135 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6803244243421053, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6803244243421053, + "pass@2": 0.7196951668047853, + "pass@4": 0.7533606276468515, + "pass@8": 0.7819554539199778, + "pass@16": 0.8062390232720734, + "pass@32": 0.8272431027750123, + "pass@64": 0.8459876187210926, + "pass@128": 0.8631052631578947 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9374609375, + "pass@2": 0.9568209891732286, + "pass@4": 0.969609387326584, + "pass@8": 0.9780410694302417, + "pass@16": 0.9835916232526909, + "pass@32": 0.9875500212448162, + "pass@64": 0.9904628626561853, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999953125, + "pass@2": 0.9999987696850394, + "pass@4": 0.9999999995313085, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9916328125, + "pass@2": 0.9935068897637795, + "pass@4": 0.9949222595613048, + "pass@8": 0.9958238935603797, + "pass@16": 0.9963295794851466, + "pass@32": 0.9967494938802602, + "pass@64": 0.9974999999977977, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.9995703125, + "pass@2": 0.9999409448818898, + "pass@4": 0.999997050056243, + "pass@8": 0.9999999944822617, + "pass@16": 0.9999999999999968, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.995484375, + "pass@2": 0.9976993110236222, + "pass@4": 0.9988875447131609, + "pass@8": 0.9995084998310533, + "pass@16": 0.9998131921683332, + "pass@32": 0.9999736715410081, + "pass@64": 0.9999998615776614, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.985671875, + "pass@2": 0.991946973425197, + "pass@4": 0.9946909814398197, + "pass@8": 0.9964191371747406, + "pass@16": 0.9977549650714689, + "pass@32": 0.9986390182664511, + "pass@64": 0.9989690611535798, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.9640546875, + "pass@2": 0.9776104822834655, + "pass@4": 0.9848667877765283, + "pass@8": 0.989761849224164, + "pass@16": 0.9926817622341847, + "pass@32": 0.9944138858713218, + "pass@64": 0.9958833446734731, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.8928828125, + "pass@2": 0.9304996309055115, + "pass@4": 0.9517802391263591, + "pass@8": 0.9646634253663902, + "pass@16": 0.9727097527282708, + "pass@32": 0.9780033779449054, + "pass@64": 0.9816839060885125, + "pass@128": 0.985 + }, + "12": { + "pass@1": 0.8229296875, + "pass@2": 0.8794469734251978, + "pass@4": 0.91197360423697, + "pass@8": 0.9329927197607489, + "pass@16": 0.9475447037674732, + "pass@32": 0.9577132375765316, + "pass@64": 0.9647098494024046, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.59034375, + "pass@2": 0.6834507874015745, + "pass@4": 0.7525489787214105, + "pass@8": 0.8037467958544191, + "pass@16": 0.8420582580595548, + "pass@32": 0.871225002739347, + "pass@64": 0.8942999913048029, + "pass@128": 0.912 + }, + "14": { + "pass@1": 0.4938125, + "pass@2": 0.593524360236221, + "pass@4": 0.672868000656167, + "pass@8": 0.7339828959553184, + "pass@16": 0.7823024545523954, + "pass@32": 0.8216617138740999, + "pass@64": 0.8549526964970245, + "pass@128": 0.884 + }, + "15": { + "pass@1": 0.3363125, + "pass@2": 0.43280684055118135, + "pass@4": 0.5218394927821517, + "pass@8": 0.5989764468164341, + "pass@16": 0.6634686127535178, + "pass@32": 0.7175496425757314, + "pass@64": 0.7629674502846845, + "pass@128": 0.804 + }, + "16": { + "pass@1": 0.2378671875, + "pass@2": 0.31033796751968495, + "pass@4": 0.3802311455755525, + "pass@8": 0.4448543139638245, + "pass@16": 0.503592993748637, + "pass@32": 0.5582595364270498, + "pass@64": 0.6113511394585666, + "pass@128": 0.66 + }, + "17": { + "pass@1": 0.19515625, + "pass@2": 0.26501722440944897, + "pass@4": 0.3358142904949378, + "pass@8": 0.4028738936800769, + "pass@16": 0.463582430082757, + "pass@32": 0.5181274567245612, + "pass@64": 0.56847235268569, + "pass@128": 0.62 + }, + "18": { + "pass@1": 0.1873125, + "pass@2": 0.2518336614173228, + "pass@4": 0.31633660601799735, + "pass@8": 0.37678362942783716, + "pass@16": 0.4330532993922898, + "pass@32": 0.4864742309003656, + "pass@64": 0.5378284053318653, + "pass@128": 0.586 + }, + "19": { + "pass@1": 0.160859375, + "pass@2": 0.2180234990157482, + "pass@4": 0.2736183600487438, + "pass@8": 0.3242448631843231, + "pass@16": 0.3698350069177702, + "pass@32": 0.41086370085369817, + "pass@64": 0.44953961912830304, + "pass@128": 0.488 + }, + "20": { + "pass@1": 0.134875, + "pass@2": 0.19174298720472446, + "pass@4": 0.25386719722534645, + "pass@8": 0.31448019676719036, + "pass@16": 0.37022280795466495, + "pass@32": 0.4204149623047817, + "pass@64": 0.4651442154601655, + "pass@128": 0.504 + } + }, + "per_op_accuracy": { + "10": 0.9374609375, + "2": 1.0, + "3": 0.999984375, + "4": 0.999953125, + "5": 0.9916328125, + "6": 0.9995703125, + "7": 0.995484375, + "8": 0.985671875, + "9": 0.9640546875, + "11": 0.8928828125, + "12": 0.8229296875, + "13": 0.59034375, + "14": 0.4938125, + "15": 0.3363125, + "16": 0.2378671875, + "17": 0.19515625, + "18": 0.1873125, + "19": 0.160859375, + "20": 0.134875 + }, + "per_op_avg_response_len": { + "10": 278.8289140625, + "2": 115.11125, + "3": 143.84865625, + "4": 158.940453125, + "5": 171.911078125, + "6": 197.4796953125, + "7": 222.2320390625, + "8": 241.496875, + "9": 257.2355546875, + "11": 295.3559765625, + "12": 304.267171875, + "13": 317.0078828125, + "14": 334.990734375, + "15": 340.2494453125, + "16": 335.35834375, + "17": 344.936578125, + "18": 344.8016171875, + "19": 337.8381484375, + "20": 343.5133515625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.8289140625, + "2": 115.11125, + "3": 143.84865625, + "4": 158.940453125, + "5": 171.911078125, + "6": 197.4796953125, + "7": 222.2320390625, + "8": 241.496875, + "9": 257.2355546875, + "11": 295.3559765625, + "12": 304.267171875, + "13": 317.0078828125, + "14": 334.990734375, + "15": 340.2494453125, + "16": 335.35834375, + "17": 344.936578125, + "18": 344.8016171875, + "19": 337.8381484375, + "20": 343.5133515625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 547519, + "answer_accuracy": 0.6761764444356624, + "avg_response_len": 254.39364329750236, + "resp_tokens_sum": 205989656, + "pass_at_k": { + "pass@1": 0.6761764444356624, + "pass@2": 0.7157596111597878, + "pass@4": 0.7490381345737922, + "pass@8": 0.7772040964607381, + "pass@16": 0.8018207952039584, + "pass@32": 0.8239761108033385, + "pass@64": 0.8437762012047311, + "pass@128": 0.861681947518179 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.935126290560472, + "pass@2": 0.9578778627505631, + "pass@4": 0.9718444243142172, + "pass@8": 0.979102374498778, + "pass@16": 0.9830894960513075, + "pass@32": 0.9864112036490584, + "pass@64": 0.9903896215785958, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.9999961552657479, + "pass@4": 0.9999999985353394, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9935515873015873, + "pass@2": 0.9958173197100364, + "pass@4": 0.9979374661500646, + "pass@8": 0.999387736990032, + "pass@16": 0.9999321953197198, + "pass@32": 0.9999993438733693, + "pass@64": 0.9999999999980798, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999064371257484, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9965329142011834, + "pass@2": 0.9975601179937568, + "pass@4": 0.9980142238137393, + "pass@8": 0.9986853311816261, + "pass@16": 0.9994492227814368, + "pass@32": 0.9999221053455868, + "pass@64": 0.9999995904664537, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9855018028846154, + "pass@2": 0.9916890647082576, + "pass@4": 0.9938409859224325, + "pass@8": 0.9956982672073414, + "pass@16": 0.9975426088063732, + "pass@32": 0.9990999858579014, + "pass@64": 0.9999054915606393, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9561399647887324, + "pass@2": 0.9724600060995899, + "pass@4": 0.9810422577459504, + "pass@8": 0.9870883337063461, + "pass@16": 0.9902834322150555, + "pass@32": 0.9919063988207483, + "pass@64": 0.9934997449201314, + "pass@128": 0.9943661971830986 + }, + "11": { + "pass@1": 0.8831223441926346, + "pass@2": 0.9245008336865118, + "pass@4": 0.9477521931571583, + "pass@8": 0.9607035690298851, + "pass@16": 0.968703861284288, + "pass@32": 0.9744709138233744, + "pass@64": 0.9794031631325362, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8216366525423728, + "pass@2": 0.8857683838249031, + "pass@4": 0.92275589942147, + "pass@8": 0.946291937886921, + "pass@16": 0.9618294686978542, + "pass@32": 0.9713007305324585, + "pass@64": 0.9763362592837835, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.5992848746081505, + "pass@2": 0.6910794837459577, + "pass@4": 0.7593168583159077, + "pass@8": 0.8106964463013567, + "pass@16": 0.8493344595262924, + "pass@32": 0.8775505709931961, + "pass@64": 0.8993150970974295, + "pass@128": 0.9153605015673981 + }, + "14": { + "pass@1": 0.47748734423676015, + "pass@2": 0.5846632999730172, + "pass@4": 0.6713725787197162, + "pass@8": 0.7362450922488023, + "pass@16": 0.7866651958227615, + "pass@32": 0.8300273996288141, + "pass@64": 0.8656000028744902, + "pass@128": 0.8940809968847352 + }, + "15": { + "pass@1": 0.3230897760115607, + "pass@2": 0.4112555328614995, + "pass@4": 0.4971151013998969, + "pass@8": 0.5771504498506533, + "pass@16": 0.6481016179810867, + "pass@32": 0.707089445007043, + "pass@64": 0.7521345054468381, + "pass@128": 0.7890173410404624 + }, + "16": { + "pass@1": 0.24778225806451612, + "pass@2": 0.321448358521717, + "pass@4": 0.3901999225903216, + "pass@8": 0.4514741921201661, + "pass@16": 0.507778863363312, + "pass@32": 0.562700667039377, + "pass@64": 0.6167569408920534, + "pass@128": 0.6645161290322581 + }, + "17": { + "pass@1": 0.1856936416184971, + "pass@2": 0.2507627241591187, + "pass@4": 0.3154409294394557, + "pass@8": 0.37876531624118964, + "pass@16": 0.4407413021254028, + "pass@32": 0.4991258395178229, + "pass@64": 0.5514502079284902, + "pass@128": 0.6040462427745664 + }, + "18": { + "pass@1": 0.17433035714285713, + "pass@2": 0.23157761529808754, + "pass@4": 0.2889288510364776, + "pass@8": 0.34402642259043337, + "pass@16": 0.3994066535761887, + "pass@32": 0.45911863063083785, + "pass@64": 0.5206270692676888, + "pass@128": 0.5771428571428572 + }, + "19": { + "pass@1": 0.16211524024024024, + "pass@2": 0.22172455427868798, + "pass@4": 0.27695589121179676, + "pass@8": 0.3244718288012454, + "pass@16": 0.36720452860714353, + "pass@32": 0.40559504045048284, + "pass@64": 0.44125668250848554, + "pass@128": 0.4804804804804805 + }, + "20": { + "pass@1": 0.14102383633633633, + "pass@2": 0.1923488449078999, + "pass@4": 0.24783760025492316, + "pass@8": 0.3033502821196472, + "pass@16": 0.3584227324664222, + "pass@32": 0.4129476389624526, + "pass@64": 0.46498198625491477, + "pass@128": 0.5135135135135135 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 561221, + "answer_accuracy": 0.6834823168355417, + "avg_response_len": 264.4252849766173, + "resp_tokens_sum": 217124890, + "pass_at_k": { + "pass@1": 0.6834823168355417, + "pass@2": 0.722711038811589, + "pass@4": 0.7562471068844316, + "pass@8": 0.7844580489068709, + "pass@16": 0.808176610062431, + "pass@32": 0.8287584090830761, + "pass@64": 0.8475514550936448, + "pass@128": 0.8646921278254092 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9348399156441718, + "pass@2": 0.9528630863243323, + "pass@4": 0.9651524198853975, + "pass@8": 0.9735246936336497, + "pass@16": 0.9791935962267696, + "pass@32": 0.983532318769681, + "pass@64": 0.9869365859226864, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 0.9999996271772846, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9983242753623188, + "pass@2": 0.9996569382631519, + "pass@4": 0.9999822019530168, + "pass@8": 0.9999999500901793, + "pass@16": 0.999999999999874, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990821991404012, + "pass@2": 0.9998346654107347, + "pass@4": 0.9999915487820469, + "pass@8": 0.9999999841898617, + "pass@16": 0.9999999999999909, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9943325505780347, + "pass@2": 0.997179520731874, + "pass@4": 0.9990721668281349, + "pass@8": 0.9998892341279048, + "pass@16": 0.9999982631322548, + "pass@32": 0.9999999998103838, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9881161971830986, + "pass@2": 0.9947415298879895, + "pass@4": 0.9977329795043224, + "pass@8": 0.9990231622731005, + "pass@16": 0.9997126319909666, + "pass@32": 0.9999800145860199, + "pass@64": 0.9999999852572564, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9628555389221557, + "pass@2": 0.9749122571785567, + "pass@4": 0.9816043618612045, + "pass@8": 0.9857971439594125, + "pass@16": 0.9890119272995747, + "pass@32": 0.9919029941070923, + "pass@64": 0.9945836442907414, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.900414156626506, + "pass@2": 0.9350123179726779, + "pass@4": 0.9550508309955227, + "pass@8": 0.9674511498873033, + "pass@16": 0.9746696385552203, + "pass@32": 0.9781687408964208, + "pass@64": 0.9789054319272765, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8177164968652038, + "pass@2": 0.8662809622837115, + "pass@4": 0.8951960123323143, + "pass@8": 0.9141750532759667, + "pass@16": 0.9279755431874404, + "pass@32": 0.9392592527593965, + "pass@64": 0.9479558030667862, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.5941001400560224, + "pass@2": 0.6847323082776416, + "pass@4": 0.7461941632295956, + "pass@8": 0.7901620120404012, + "pass@16": 0.8250701005723396, + "pass@32": 0.8536865506090808, + "pass@64": 0.876800327740759, + "pass@128": 0.8935574229691877 + }, + "14": { + "pass@1": 0.5205820540935673, + "pass@2": 0.6189182149928629, + "pass@4": 0.6981114899014816, + "pass@8": 0.7601463796827449, + "pass@16": 0.8069542276262767, + "pass@32": 0.840170709001981, + "pass@64": 0.8655683771981271, + "pass@128": 0.8888888888888888 + }, + "15": { + "pass@1": 0.3488162878787879, + "pass@2": 0.4543855136005727, + "pass@4": 0.5469166294554091, + "pass@8": 0.6202028758837862, + "pass@16": 0.6771987839186235, + "pass@32": 0.725887979805038, + "pass@64": 0.7719392599869858, + "pass@128": 0.8181818181818182 + }, + "16": { + "pass@1": 0.23317307692307693, + "pass@2": 0.30855002299392065, + "pass@4": 0.3825931414021966, + "pass@8": 0.44921159232600466, + "pass@16": 0.5063128891342469, + "pass@32": 0.5572324306688641, + "pass@64": 0.6074042418530502, + "pass@128": 0.6552706552706553 + }, + "17": { + "pass@1": 0.2060850155279503, + "pass@2": 0.2831642478114148, + "pass@4": 0.36248983929415635, + "pass@8": 0.43754655513862, + "pass@16": 0.5028174506846871, + "pass@32": 0.5573710122030797, + "pass@64": 0.6053850077444561, + "pass@128": 0.6521739130434783 + }, + "18": { + "pass@1": 0.19751838235294117, + "pass@2": 0.2667702784854099, + "pass@4": 0.335324297789982, + "pass@8": 0.39891717056702747, + "pass@16": 0.45777200500975923, + "pass@32": 0.5134400046716384, + "pass@64": 0.5670193664414482, + "pass@128": 0.6147058823529412 + }, + "19": { + "pass@1": 0.15252976190476192, + "pass@2": 0.20051534085583056, + "pass@4": 0.24895375047985074, + "pass@8": 0.29642610320817747, + "pass@16": 0.3433369906473041, + "pass@32": 0.3918855662817668, + "pass@64": 0.44317301534958015, + "pass@128": 0.49404761904761907 + }, + "20": { + "pass@1": 0.12467252994011976, + "pass@2": 0.18133737446367107, + "pass@4": 0.24257957369849725, + "pass@8": 0.3015672795538963, + "pass@16": 0.3555785376399858, + "pass@32": 0.4049003421017203, + "pass@64": 0.449425700433271, + "pass@128": 0.48502994011976047 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 545809, + "answer_accuracy": 0.6812802065026362, + "avg_response_len": 284.3619388081163, + "resp_tokens_sum": 227817136, + "pass_at_k": { + "pass@1": 0.6812802065026362, + "pass@2": 0.7205818110110382, + "pass@4": 0.7547709690546537, + "pass@8": 0.7841927027530263, + "pass@16": 0.8087186672245711, + "pass@32": 0.8289919925731091, + "pass@64": 0.8466198949439094, + "pass@128": 0.8629173989455184 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9423740671641792, + "pass@2": 0.959603067340463, + "pass@4": 0.9716848913475367, + "pass@8": 0.9813621323898088, + "pass@16": 0.9883796170787192, + "pass@32": 0.9926122008622965, + "pass@64": 0.9939685191947623, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9830652573529411, + "pass@2": 0.9851258539833256, + "pass@4": 0.9869944060301287, + "pass@8": 0.9882846283159047, + "pass@16": 0.9892674645867016, + "pass@32": 0.9904402957651445, + "pass@64": 0.992647058818831, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9997535488958991, + "pass@2": 0.9999961188802505, + "pass@4": 0.9999999985214783, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956240110759493, + "pass@2": 0.9984173321788099, + "pass@4": 0.9996195169037414, + "pass@8": 0.9999720977323062, + "pass@16": 0.9999998602039476, + "pass@32": 0.9999999999993893, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9832254129129129, + "pass@2": 0.9892094358137666, + "pass@4": 0.9922444027266861, + "pass@8": 0.9943184840813796, + "pass@16": 0.9958669272284904, + "pass@32": 0.996777530032279, + "pass@64": 0.9969926517127159, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9743770096463023, + "pass@2": 0.9863874154999117, + "pass@4": 0.9927361074439651, + "pass@8": 0.9970715262892839, + "pass@16": 0.9993606433433512, + "pass@32": 0.9999727146565515, + "pass@64": 0.9999999936775518, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8958829365079365, + "pass@2": 0.9324658636420451, + "pass@4": 0.9528471080995822, + "pass@8": 0.9661628055119237, + "pass@16": 0.9751332371256648, + "pass@32": 0.9817876932941041, + "pass@64": 0.987168209850384, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8294151376146789, + "pass@2": 0.8854475186014598, + "pass@4": 0.9166681587393313, + "pass@8": 0.9369526964946373, + "pass@16": 0.951170867191561, + "pass@32": 0.9610063527149639, + "pass@64": 0.9684676221334558, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.5774016203703703, + "pass@2": 0.6745277717021484, + "pass@4": 0.7528876069310777, + "pass@8": 0.8118728123020478, + "pass@16": 0.8536127764393275, + "pass@32": 0.8843218272378883, + "pass@64": 0.9086443158249113, + "pass@128": 0.9290123456790124 + }, + "14": { + "pass@1": 0.4821958456973294, + "pass@2": 0.5761940991611951, + "pass@4": 0.6486744016048438, + "pass@8": 0.7052764376615864, + "pass@16": 0.7531293200151414, + "pass@32": 0.7949096027138687, + "pass@64": 0.8340377168325014, + "pass@128": 0.8694362017804155 + }, + "15": { + "pass@1": 0.3376977237654321, + "pass@2": 0.43384323053368284, + "pass@4": 0.5227011727700702, + "pass@8": 0.6006649448347476, + "pass@16": 0.6658946118485112, + "pass@32": 0.7202273495914582, + "pass@64": 0.7653980111246035, + "pass@128": 0.8055555555555556 + }, + "16": { + "pass@1": 0.23366058259587022, + "pass@2": 0.30202925754302834, + "pass@4": 0.36866954849670325, + "pass@8": 0.4342892197644405, + "pass@16": 0.4969490325070487, + "pass@32": 0.5552617890267609, + "pass@64": 0.6104943920106468, + "pass@128": 0.6607669616519174 + }, + "17": { + "pass@1": 0.1944182981927711, + "pass@2": 0.26227239173228345, + "pass@4": 0.3311747007715901, + "pass@8": 0.39437079369274153, + "pass@16": 0.44933349526174327, + "pass@32": 0.49986882627109364, + "pass@64": 0.5504114706286666, + "pass@128": 0.6054216867469879 + }, + "18": { + "pass@1": 0.19077620967741934, + "pass@2": 0.2583212947675897, + "pass@4": 0.3264556351826988, + "pass@8": 0.38949207589805374, + "pass@16": 0.44393060947518126, + "pass@32": 0.48778422126198656, + "pass@64": 0.5252333758002631, + "pass@128": 0.5645161290322581 + }, + "19": { + "pass@1": 0.16805135951661632, + "pass@2": 0.2320727127292623, + "pass@4": 0.29529784928696595, + "pass@8": 0.35225550911045606, + "pass@16": 0.3993796677767297, + "pass@32": 0.43542900940487445, + "pass@64": 0.46433537974476874, + "pass@128": 0.48942598187311176 + }, + "20": { + "pass@1": 0.13895927177177178, + "pass@2": 0.20157399032891157, + "pass@4": 0.27121831448996814, + "pass@8": 0.33856180615719716, + "pass@16": 0.39671113054531904, + "pass@32": 0.44344349630123264, + "pass@64": 0.48107216244013823, + "pass@128": 0.5135135135135135 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch1-RL-op7-10/composition/summary.csv b/composition/Qwen3-100M-epoch1-RL-op7-10/composition/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..f3bc4d71ee65df48c3c39cafe2910ff4a21eea1a --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op7-10/composition/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.9859791666666666,0.40523515625,0.6803244243421053,0.0,0.0,0.0,198.56494618055555,329.831925,1152000,1280000,2432000 diff --git a/composition/Qwen3-100M-epoch1-RL-op7-10/composition/summary.json b/composition/Qwen3-100M-epoch1-RL-op7-10/composition/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..eb79b12a794794e7293659bac2ff486b26d9942f --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op7-10/composition/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.9859791666666666, + "ood_acc": 0.40523515625, + "total_acc": 0.6803244243421053, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 198.56494618055555, + "ood_avg_resp_len": 329.831925, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 547519, + "answer_accuracy": 0.6761764444356624, + "avg_response_len": 254.39364329750236, + "resp_tokens_sum": 205989656, + "pass_at_k": { + "pass@1": 0.6761764444356624, + "pass@2": 0.7157596111597878, + "pass@4": 0.7490381345737922, + "pass@8": 0.7772040964607381, + "pass@16": 0.8018207952039584, + "pass@32": 0.8239761108033385, + "pass@64": 0.8437762012047311, + "pass@128": 0.861681947518179 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.935126290560472, + "pass@2": 0.9578778627505631, + "pass@4": 0.9718444243142172, + "pass@8": 0.979102374498778, + "pass@16": 0.9830894960513075, + "pass@32": 0.9864112036490584, + "pass@64": 0.9903896215785958, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.9999961552657479, + "pass@4": 0.9999999985353394, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9935515873015873, + "pass@2": 0.9958173197100364, + "pass@4": 0.9979374661500646, + "pass@8": 0.999387736990032, + "pass@16": 0.9999321953197198, + "pass@32": 0.9999993438733693, + "pass@64": 0.9999999999980798, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999064371257484, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9965329142011834, + "pass@2": 0.9975601179937568, + "pass@4": 0.9980142238137393, + "pass@8": 0.9986853311816261, + "pass@16": 0.9994492227814368, + "pass@32": 0.9999221053455868, + "pass@64": 0.9999995904664537, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9855018028846154, + "pass@2": 0.9916890647082576, + "pass@4": 0.9938409859224325, + "pass@8": 0.9956982672073414, + "pass@16": 0.9975426088063732, + "pass@32": 0.9990999858579014, + "pass@64": 0.9999054915606393, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9561399647887324, + "pass@2": 0.9724600060995899, + "pass@4": 0.9810422577459504, + "pass@8": 0.9870883337063461, + "pass@16": 0.9902834322150555, + "pass@32": 0.9919063988207483, + "pass@64": 0.9934997449201314, + "pass@128": 0.9943661971830986 + }, + "11": { + "pass@1": 0.8831223441926346, + "pass@2": 0.9245008336865118, + "pass@4": 0.9477521931571583, + "pass@8": 0.9607035690298851, + "pass@16": 0.968703861284288, + "pass@32": 0.9744709138233744, + "pass@64": 0.9794031631325362, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8216366525423728, + "pass@2": 0.8857683838249031, + "pass@4": 0.92275589942147, + "pass@8": 0.946291937886921, + "pass@16": 0.9618294686978542, + "pass@32": 0.9713007305324585, + "pass@64": 0.9763362592837835, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.5992848746081505, + "pass@2": 0.6910794837459577, + "pass@4": 0.7593168583159077, + "pass@8": 0.8106964463013567, + "pass@16": 0.8493344595262924, + "pass@32": 0.8775505709931961, + "pass@64": 0.8993150970974295, + "pass@128": 0.9153605015673981 + }, + "14": { + "pass@1": 0.47748734423676015, + "pass@2": 0.5846632999730172, + "pass@4": 0.6713725787197162, + "pass@8": 0.7362450922488023, + "pass@16": 0.7866651958227615, + "pass@32": 0.8300273996288141, + "pass@64": 0.8656000028744902, + "pass@128": 0.8940809968847352 + }, + "15": { + "pass@1": 0.3230897760115607, + "pass@2": 0.4112555328614995, + "pass@4": 0.4971151013998969, + "pass@8": 0.5771504498506533, + "pass@16": 0.6481016179810867, + "pass@32": 0.707089445007043, + "pass@64": 0.7521345054468381, + "pass@128": 0.7890173410404624 + }, + "16": { + "pass@1": 0.24778225806451612, + "pass@2": 0.321448358521717, + "pass@4": 0.3901999225903216, + "pass@8": 0.4514741921201661, + "pass@16": 0.507778863363312, + "pass@32": 0.562700667039377, + "pass@64": 0.6167569408920534, + "pass@128": 0.6645161290322581 + }, + "17": { + "pass@1": 0.1856936416184971, + "pass@2": 0.2507627241591187, + "pass@4": 0.3154409294394557, + "pass@8": 0.37876531624118964, + "pass@16": 0.4407413021254028, + "pass@32": 0.4991258395178229, + "pass@64": 0.5514502079284902, + "pass@128": 0.6040462427745664 + }, + "18": { + "pass@1": 0.17433035714285713, + "pass@2": 0.23157761529808754, + "pass@4": 0.2889288510364776, + "pass@8": 0.34402642259043337, + "pass@16": 0.3994066535761887, + "pass@32": 0.45911863063083785, + "pass@64": 0.5206270692676888, + "pass@128": 0.5771428571428572 + }, + "19": { + "pass@1": 0.16211524024024024, + "pass@2": 0.22172455427868798, + "pass@4": 0.27695589121179676, + "pass@8": 0.3244718288012454, + "pass@16": 0.36720452860714353, + "pass@32": 0.40559504045048284, + "pass@64": 0.44125668250848554, + "pass@128": 0.4804804804804805 + }, + "20": { + "pass@1": 0.14102383633633633, + "pass@2": 0.1923488449078999, + "pass@4": 0.24783760025492316, + "pass@8": 0.3033502821196472, + "pass@16": 0.3584227324664222, + "pass@32": 0.4129476389624526, + "pass@64": 0.46498198625491477, + "pass@128": 0.5135135135135135 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 561221, + "answer_accuracy": 0.6834823168355417, + "avg_response_len": 264.4252849766173, + "resp_tokens_sum": 217124890, + "pass_at_k": { + "pass@1": 0.6834823168355417, + "pass@2": 0.722711038811589, + "pass@4": 0.7562471068844316, + "pass@8": 0.7844580489068709, + "pass@16": 0.808176610062431, + "pass@32": 0.8287584090830761, + "pass@64": 0.8475514550936448, + "pass@128": 0.8646921278254092 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9348399156441718, + "pass@2": 0.9528630863243323, + "pass@4": 0.9651524198853975, + "pass@8": 0.9735246936336497, + "pass@16": 0.9791935962267696, + "pass@32": 0.983532318769681, + "pass@64": 0.9869365859226864, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 0.9999996271772846, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9983242753623188, + "pass@2": 0.9996569382631519, + "pass@4": 0.9999822019530168, + "pass@8": 0.9999999500901793, + "pass@16": 0.999999999999874, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990821991404012, + "pass@2": 0.9998346654107347, + "pass@4": 0.9999915487820469, + "pass@8": 0.9999999841898617, + "pass@16": 0.9999999999999909, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9943325505780347, + "pass@2": 0.997179520731874, + "pass@4": 0.9990721668281349, + "pass@8": 0.9998892341279048, + "pass@16": 0.9999982631322548, + "pass@32": 0.9999999998103838, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9881161971830986, + "pass@2": 0.9947415298879895, + "pass@4": 0.9977329795043224, + "pass@8": 0.9990231622731005, + "pass@16": 0.9997126319909666, + "pass@32": 0.9999800145860199, + "pass@64": 0.9999999852572564, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9628555389221557, + "pass@2": 0.9749122571785567, + "pass@4": 0.9816043618612045, + "pass@8": 0.9857971439594125, + "pass@16": 0.9890119272995747, + "pass@32": 0.9919029941070923, + "pass@64": 0.9945836442907414, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.900414156626506, + "pass@2": 0.9350123179726779, + "pass@4": 0.9550508309955227, + "pass@8": 0.9674511498873033, + "pass@16": 0.9746696385552203, + "pass@32": 0.9781687408964208, + "pass@64": 0.9789054319272765, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8177164968652038, + "pass@2": 0.8662809622837115, + "pass@4": 0.8951960123323143, + "pass@8": 0.9141750532759667, + "pass@16": 0.9279755431874404, + "pass@32": 0.9392592527593965, + "pass@64": 0.9479558030667862, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.5941001400560224, + "pass@2": 0.6847323082776416, + "pass@4": 0.7461941632295956, + "pass@8": 0.7901620120404012, + "pass@16": 0.8250701005723396, + "pass@32": 0.8536865506090808, + "pass@64": 0.876800327740759, + "pass@128": 0.8935574229691877 + }, + "14": { + "pass@1": 0.5205820540935673, + "pass@2": 0.6189182149928629, + "pass@4": 0.6981114899014816, + "pass@8": 0.7601463796827449, + "pass@16": 0.8069542276262767, + "pass@32": 0.840170709001981, + "pass@64": 0.8655683771981271, + "pass@128": 0.8888888888888888 + }, + "15": { + "pass@1": 0.3488162878787879, + "pass@2": 0.4543855136005727, + "pass@4": 0.5469166294554091, + "pass@8": 0.6202028758837862, + "pass@16": 0.6771987839186235, + "pass@32": 0.725887979805038, + "pass@64": 0.7719392599869858, + "pass@128": 0.8181818181818182 + }, + "16": { + "pass@1": 0.23317307692307693, + "pass@2": 0.30855002299392065, + "pass@4": 0.3825931414021966, + "pass@8": 0.44921159232600466, + "pass@16": 0.5063128891342469, + "pass@32": 0.5572324306688641, + "pass@64": 0.6074042418530502, + "pass@128": 0.6552706552706553 + }, + "17": { + "pass@1": 0.2060850155279503, + "pass@2": 0.2831642478114148, + "pass@4": 0.36248983929415635, + "pass@8": 0.43754655513862, + "pass@16": 0.5028174506846871, + "pass@32": 0.5573710122030797, + "pass@64": 0.6053850077444561, + "pass@128": 0.6521739130434783 + }, + "18": { + "pass@1": 0.19751838235294117, + "pass@2": 0.2667702784854099, + "pass@4": 0.335324297789982, + "pass@8": 0.39891717056702747, + "pass@16": 0.45777200500975923, + "pass@32": 0.5134400046716384, + "pass@64": 0.5670193664414482, + "pass@128": 0.6147058823529412 + }, + "19": { + "pass@1": 0.15252976190476192, + "pass@2": 0.20051534085583056, + "pass@4": 0.24895375047985074, + "pass@8": 0.29642610320817747, + "pass@16": 0.3433369906473041, + "pass@32": 0.3918855662817668, + "pass@64": 0.44317301534958015, + "pass@128": 0.49404761904761907 + }, + "20": { + "pass@1": 0.12467252994011976, + "pass@2": 0.18133737446367107, + "pass@4": 0.24257957369849725, + "pass@8": 0.3015672795538963, + "pass@16": 0.3555785376399858, + "pass@32": 0.4049003421017203, + "pass@64": 0.449425700433271, + "pass@128": 0.48502994011976047 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 545809, + "answer_accuracy": 0.6812802065026362, + "avg_response_len": 284.3619388081163, + "resp_tokens_sum": 227817136, + "pass_at_k": { + "pass@1": 0.6812802065026362, + "pass@2": 0.7205818110110382, + "pass@4": 0.7547709690546537, + "pass@8": 0.7841927027530263, + "pass@16": 0.8087186672245711, + "pass@32": 0.8289919925731091, + "pass@64": 0.8466198949439094, + "pass@128": 0.8629173989455184 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9423740671641792, + "pass@2": 0.959603067340463, + "pass@4": 0.9716848913475367, + "pass@8": 0.9813621323898088, + "pass@16": 0.9883796170787192, + "pass@32": 0.9926122008622965, + "pass@64": 0.9939685191947623, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9830652573529411, + "pass@2": 0.9851258539833256, + "pass@4": 0.9869944060301287, + "pass@8": 0.9882846283159047, + "pass@16": 0.9892674645867016, + "pass@32": 0.9904402957651445, + "pass@64": 0.992647058818831, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9997535488958991, + "pass@2": 0.9999961188802505, + "pass@4": 0.9999999985214783, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956240110759493, + "pass@2": 0.9984173321788099, + "pass@4": 0.9996195169037414, + "pass@8": 0.9999720977323062, + "pass@16": 0.9999998602039476, + "pass@32": 0.9999999999993893, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9832254129129129, + "pass@2": 0.9892094358137666, + "pass@4": 0.9922444027266861, + "pass@8": 0.9943184840813796, + "pass@16": 0.9958669272284904, + "pass@32": 0.996777530032279, + "pass@64": 0.9969926517127159, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9743770096463023, + "pass@2": 0.9863874154999117, + "pass@4": 0.9927361074439651, + "pass@8": 0.9970715262892839, + "pass@16": 0.9993606433433512, + "pass@32": 0.9999727146565515, + "pass@64": 0.9999999936775518, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8958829365079365, + "pass@2": 0.9324658636420451, + "pass@4": 0.9528471080995822, + "pass@8": 0.9661628055119237, + "pass@16": 0.9751332371256648, + "pass@32": 0.9817876932941041, + "pass@64": 0.987168209850384, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8294151376146789, + "pass@2": 0.8854475186014598, + "pass@4": 0.9166681587393313, + "pass@8": 0.9369526964946373, + "pass@16": 0.951170867191561, + "pass@32": 0.9610063527149639, + "pass@64": 0.9684676221334558, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.5774016203703703, + "pass@2": 0.6745277717021484, + "pass@4": 0.7528876069310777, + "pass@8": 0.8118728123020478, + "pass@16": 0.8536127764393275, + "pass@32": 0.8843218272378883, + "pass@64": 0.9086443158249113, + "pass@128": 0.9290123456790124 + }, + "14": { + "pass@1": 0.4821958456973294, + "pass@2": 0.5761940991611951, + "pass@4": 0.6486744016048438, + "pass@8": 0.7052764376615864, + "pass@16": 0.7531293200151414, + "pass@32": 0.7949096027138687, + "pass@64": 0.8340377168325014, + "pass@128": 0.8694362017804155 + }, + "15": { + "pass@1": 0.3376977237654321, + "pass@2": 0.43384323053368284, + "pass@4": 0.5227011727700702, + "pass@8": 0.6006649448347476, + "pass@16": 0.6658946118485112, + "pass@32": 0.7202273495914582, + "pass@64": 0.7653980111246035, + "pass@128": 0.8055555555555556 + }, + "16": { + "pass@1": 0.23366058259587022, + "pass@2": 0.30202925754302834, + "pass@4": 0.36866954849670325, + "pass@8": 0.4342892197644405, + "pass@16": 0.4969490325070487, + "pass@32": 0.5552617890267609, + "pass@64": 0.6104943920106468, + "pass@128": 0.6607669616519174 + }, + "17": { + "pass@1": 0.1944182981927711, + "pass@2": 0.26227239173228345, + "pass@4": 0.3311747007715901, + "pass@8": 0.39437079369274153, + "pass@16": 0.44933349526174327, + "pass@32": 0.49986882627109364, + "pass@64": 0.5504114706286666, + "pass@128": 0.6054216867469879 + }, + "18": { + "pass@1": 0.19077620967741934, + "pass@2": 0.2583212947675897, + "pass@4": 0.3264556351826988, + "pass@8": 0.38949207589805374, + "pass@16": 0.44393060947518126, + "pass@32": 0.48778422126198656, + "pass@64": 0.5252333758002631, + "pass@128": 0.5645161290322581 + }, + "19": { + "pass@1": 0.16805135951661632, + "pass@2": 0.2320727127292623, + "pass@4": 0.29529784928696595, + "pass@8": 0.35225550911045606, + "pass@16": 0.3993796677767297, + "pass@32": 0.43542900940487445, + "pass@64": 0.46433537974476874, + "pass@128": 0.48942598187311176 + }, + "20": { + "pass@1": 0.13895927177177178, + "pass@2": 0.20157399032891157, + "pass@4": 0.27121831448996814, + "pass@8": 0.33856180615719716, + "pass@16": 0.39671113054531904, + "pass@32": 0.44344349630123264, + "pass@64": 0.48107216244013823, + "pass@128": 0.5135135135135135 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch1-RL-op9-12/composition/hf_model_metrics.json b/composition/Qwen3-100M-epoch1-RL-op9-12/composition/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..ea321a4e39625e352242e73ce6402e8c89004c49 --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op9-12/composition/hf_model_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.9853706597222223, + "avg_response_len": 198.38533680555557, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9853706597222223, + "pass@2": 0.9905136018153927, + "pass@4": 0.9934687289817931, + "pass@8": 0.9955307882585264, + "pass@16": 0.9970308695228473, + "pass@32": 0.9980789211842607, + "pass@64": 0.9987252151537171, + "pass@128": 0.9991111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.937453125, + "pass@2": 0.9558822588582679, + "pass@4": 0.9680160880202473, + "pass@8": 0.9768610868196399, + "pass@16": 0.9834541239321481, + "pass@32": 0.9879920155201115, + "pass@64": 0.9909572141273251, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99975, + "pass@2": 0.9999859744094489, + "pass@4": 0.9999999306336709, + "pass@8": 0.9999999999990998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999734375, + "pass@2": 0.9999731791338583, + "pass@4": 0.9999996169853769, + "pass@8": 0.999999999947103, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9915234375, + "pass@2": 0.9945222687007875, + "pass@4": 0.9965733731721034, + "pass@8": 0.9980091055458946, + "pass@16": 0.9990016636755122, + "pass@32": 0.9996162045730839, + "pass@64": 0.9999389956470872, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9981640625, + "pass@2": 0.9992300688976377, + "pass@4": 0.9998092772778402, + "pass@8": 0.9999886283533156, + "pass@16": 0.9999999673324529, + "pass@32": 0.9999999999999404, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9935, + "pass@2": 0.9960433070866144, + "pass@4": 0.997461278871391, + "pass@8": 0.9986368243453445, + "pass@16": 0.9994574439865509, + "pass@32": 0.9998675346942224, + "pass@64": 0.999993426730079, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.983125, + "pass@2": 0.9905113188976382, + "pass@4": 0.9937524416947879, + "pass@8": 0.9958266836261157, + "pass@16": 0.9975393869690035, + "pass@32": 0.9989698342517158, + "pass@64": 0.9998174124680705, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9650859375, + "pass@2": 0.9784740403543307, + "pass@4": 0.9856065541807275, + "pass@8": 0.9904547656902203, + "pass@16": 0.9938252398099534, + "pass@32": 0.9962647016192685, + "pass@64": 0.9978198874108859, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.937453125, + "2": 1.0, + "3": 0.99975, + "4": 0.999734375, + "5": 0.9915234375, + "6": 0.9981640625, + "7": 0.9935, + "8": 0.983125, + "9": 0.9650859375 + }, + "per_op_avg_response_len": { + "10": 278.47321875, + "2": 115.028609375, + "3": 143.7909921875, + "4": 158.810578125, + "5": 171.9432890625, + "6": 197.24334375, + "7": 222.158796875, + "8": 241.1735, + "9": 256.845703125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.47321875, + "2": 115.028609375, + "3": 143.7909921875, + "4": 158.810578125, + "5": 171.9432890625, + "6": 197.24334375, + "7": 222.158796875, + "8": 241.1735, + "9": 256.845703125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 373096, + "answer_accuracy": 0.9844013846673421, + "avg_response_len": 188.876965657717, + "resp_tokens_sum": 71585881, + "pass_at_k": { + "pass@1": 0.9844013846673421, + "pass@2": 0.990004573896349, + "pass@4": 0.9931748364281224, + "pass@8": 0.9951691560115009, + "pass@16": 0.9962593956179643, + "pass@32": 0.9970017870308048, + "pass@64": 0.9978082484690183, + "pass@128": 0.9986491050320837 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9300792772861357, + "pass@2": 0.9506288034515598, + "pass@4": 0.9636359633696227, + "pass@8": 0.9717781371735056, + "pass@16": 0.976361752430648, + "pass@32": 0.9797711596050733, + "pass@64": 0.9841706667237161, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998088685015291, + "pass@2": 0.9999939801102844, + "pass@4": 0.9999999957000787, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9992919921875, + "pass@2": 0.9999169537401574, + "pass@4": 0.9999988030793026, + "pass@8": 0.9999999998346969, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.994171626984127, + "pass@2": 0.9966371391076116, + "pass@4": 0.998588466322662, + "pass@8": 0.9996704275760121, + "pass@16": 0.9999772476512871, + "pass@32": 0.9999999361932187, + "pass@64": 0.999999999999996, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998362649700598, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9945220044378699, + "pass@2": 0.9966806975958625, + "pass@4": 0.997982214963514, + "pass@8": 0.9990981813231818, + "pass@16": 0.9997780458447582, + "pass@32": 0.9999893290784243, + "pass@64": 0.9999999971901181, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9853265224358975, + "pass@2": 0.9928211910710678, + "pass@4": 0.9950341814244371, + "pass@8": 0.9964232589832497, + "pass@16": 0.9976202982400736, + "pass@32": 0.9986469609135363, + "pass@64": 0.999608819891556, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9609154929577465, + "pass@2": 0.9763100255073749, + "pass@4": 0.9853365164741729, + "pass@8": 0.990951142886426, + "pass@16": 0.9936960304062404, + "pass@32": 0.9955088545287316, + "pass@64": 0.9971786389887288, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 385261, + "answer_accuracy": 0.9861898959698558, + "avg_response_len": 196.3569150352228, + "resp_tokens_sum": 76708007, + "pass_at_k": { + "pass@1": 0.9861898959698558, + "pass@2": 0.9910280934278286, + "pass@4": 0.9936960531575915, + "pass@8": 0.9955100069634649, + "pass@16": 0.9969235323511221, + "pass@32": 0.9979012980493756, + "pass@64": 0.998429596404013, + "pass@128": 0.9986893840104849 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9367810582822086, + "pass@2": 0.9545342012463163, + "pass@4": 0.9656823695542656, + "pass@8": 0.9742888215981227, + "pass@16": 0.9816129882446868, + "pass@32": 0.986408904087696, + "pass@64": 0.9888729553346032, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9994318181818181, + "pass@2": 0.9999634633738963, + "pass@4": 0.9999997940598334, + "pass@8": 0.9999999999972722, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999992720029819, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9961050724637681, + "pass@2": 0.9988381547415269, + "pass@4": 0.9998410413372242, + "pass@8": 0.9999961121701334, + "pass@16": 0.999999998171191, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9966645773638968, + "pass@2": 0.9984552320465674, + "pass@4": 0.9995937385513057, + "pass@8": 0.9999733183431431, + "pass@16": 0.9999999126220942, + "pass@32": 0.9999999999998295, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9936100072254336, + "pass@2": 0.9963759046015201, + "pass@4": 0.9980907790247318, + "pass@8": 0.9992033281484878, + "pass@16": 0.999815155302214, + "pass@32": 0.9999926083157176, + "pass@64": 0.999999998852127, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9861355633802817, + "pass@2": 0.9926226156149497, + "pass@4": 0.9959982136915982, + "pass@8": 0.9980394885254082, + "pass@16": 0.9991635856551462, + "pass@32": 0.9998083508916638, + "pass@64": 0.999995930319257, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9650542664670658, + "pass@2": 0.9768406101183464, + "pass@4": 0.9827120368374614, + "pass@8": 0.987007989958256, + "pass@16": 0.9909152726094004, + "pass@32": 0.9942995564981604, + "pass@64": 0.9965149398796272, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 376790, + "answer_accuracy": 0.9854944342149313, + "avg_response_len": 209.88350560763308, + "resp_tokens_sum": 80246020, + "pass_at_k": { + "pass@1": 0.9854944342149313, + "pass@2": 0.990492511513146, + "pass@4": 0.9935277924122892, + "pass@8": 0.9959105062350794, + "pass@16": 0.9979053011534009, + "pass@32": 0.9993281679321845, + "pass@64": 0.9999362519054691, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9455690298507463, + "pass@2": 0.9625102832295216, + "pass@4": 0.9747195342186703, + "pass@8": 0.9845078879905481, + "pass@16": 0.9924228528071358, + "pass@32": 0.9978516109892621, + "pass@64": 0.9998530587131482, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9844209558823529, + "pass@2": 0.9881835485178323, + "pass@4": 0.9913907265268312, + "pass@8": 0.9944537122375137, + "pass@16": 0.9970847979302819, + "pass@32": 0.998871249035941, + "pass@64": 0.9998205754326136, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9980530362776026, + "pass@2": 0.9992722900469461, + "pass@4": 0.9998456231023172, + "pass@8": 0.999993502370847, + "pass@16": 0.9999999931461893, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9922863924050633, + "pass@2": 0.994997368060401, + "pass@4": 0.9962148122505257, + "pass@8": 0.9975230617681402, + "pass@16": 0.9987228504951157, + "pass@32": 0.9996003132546726, + "pass@64": 0.9999792027752626, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9778528528528528, + "pass@2": 0.9860963325530254, + "pass@4": 0.9901573910806193, + "pass@8": 0.9929087339243897, + "pass@16": 0.9957320751066778, + "pass@32": 0.9983784320725285, + "pass@64": 0.9998225387344413, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9698804260450161, + "pass@2": 0.9826984486163508, + "pass@4": 0.9890233779057358, + "pass@8": 0.9935898434710003, + "pass@16": 0.9970979034218608, + "pass@32": 0.9992379626726149, + "pass@64": 0.9999533139874329, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.492953125, + "avg_response_len": 353.6592453125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.492953125, + "pass@2": 0.5692891609251909, + "pass@4": 0.6348339537682659, + "pass@8": 0.6895096592773755, + "pass@16": 0.7346785527402441, + "pass@32": 0.7725330270544023, + "pass@64": 0.8051152595175582, + "pass@128": 0.8331 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.91471875, + "pass@2": 0.9433192667322837, + "pass@4": 0.9588780389951258, + "pass@8": 0.9677262498050817, + "pass@16": 0.9733190698121774, + "pass@32": 0.9771944777980829, + "pass@64": 0.9801448981522918, + "pass@128": 0.983 + }, + "12": { + "pass@1": 0.872140625, + "pass@2": 0.9067229330708668, + "pass@4": 0.9280201115485563, + "pass@8": 0.9418838968656523, + "pass@16": 0.95114741382113, + "pass@32": 0.9584214999119537, + "pass@64": 0.9645158985924684, + "pass@128": 0.969 + }, + "13": { + "pass@1": 0.7056640625, + "pass@2": 0.7780696358267724, + "pass@4": 0.8290499243532062, + "pass@8": 0.8668198148174304, + "pass@16": 0.8965971990440574, + "pass@32": 0.9204009169491684, + "pass@64": 0.9386225702011887, + "pass@128": 0.952 + }, + "14": { + "pass@1": 0.6382734375, + "pass@2": 0.728443897637796, + "pass@4": 0.7921607552493434, + "pass@8": 0.8387031765070113, + "pass@16": 0.8744282123442151, + "pass@32": 0.9004069871216294, + "pass@64": 0.9180989637432281, + "pass@128": 0.93 + }, + "15": { + "pass@1": 0.5125625, + "pass@2": 0.6179248277559062, + "pass@4": 0.702817684383202, + "pass@8": 0.7678852239404601, + "pass@16": 0.816122100716546, + "pass@32": 0.8515935888331901, + "pass@64": 0.8787718061994974, + "pass@128": 0.902 + }, + "16": { + "pass@1": 0.3863125, + "pass@2": 0.4879427903543308, + "pass@4": 0.5753028591113607, + "pass@8": 0.6478276793530026, + "pass@16": 0.7085184430353842, + "pass@32": 0.759646523617092, + "pass@64": 0.802706151441046, + "pass@128": 0.838 + }, + "17": { + "pass@1": 0.283140625, + "pass@2": 0.37741067913385873, + "pass@4": 0.4686248689538805, + "pass@8": 0.549468527134689, + "pass@16": 0.6168373940799338, + "pass@32": 0.6721576309756664, + "pass@64": 0.7197861549266963, + "pass@128": 0.76 + }, + "18": { + "pass@1": 0.2564375, + "pass@2": 0.3442957677165357, + "pass@4": 0.43100673978252685, + "pass@8": 0.5092056459704383, + "pass@16": 0.5778835218223048, + "pass@32": 0.640888246088884, + "pass@64": 0.6979854436036252, + "pass@128": 0.747 + }, + "19": { + "pass@1": 0.187328125, + "pass@2": 0.26102239173228325, + "pass@4": 0.33610365335583037, + "pass@8": 0.4054484365959564, + "pass@16": 0.46578777902787455, + "pass@32": 0.5199237213039392, + "pass@64": 0.5733993407899091, + "pass@128": 0.627 + }, + "20": { + "pass@1": 0.172953125, + "pass@2": 0.24773941929133864, + "pass@4": 0.32637490194975594, + "pass@8": 0.40012794178405553, + "pass@16": 0.4661443936986512, + "pass@32": 0.5246966779444031, + "pass@64": 0.5771213675255428, + "pass@128": 0.623 + } + }, + "per_op_accuracy": { + "11": 0.91471875, + "12": 0.872140625, + "13": 0.7056640625, + "14": 0.6382734375, + "15": 0.5125625, + "16": 0.3863125, + "17": 0.283140625, + "18": 0.2564375, + "19": 0.187328125, + "20": 0.172953125 + }, + "per_op_avg_response_len": { + "11": 296.288, + "12": 306.9627890625, + "13": 324.4796484375, + "14": 348.7399765625, + "15": 363.4608984375, + "16": 365.20496875, + "17": 380.63134375, + "18": 384.5688828125, + "19": 376.0870625, + "20": 390.1688828125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 296.288, + "12": 306.9627890625, + "13": 324.4796484375, + "14": 348.7399765625, + "15": 363.4608984375, + "16": 365.20496875, + "17": 380.63134375, + "18": 384.5688828125, + "19": 376.0870625, + "20": 390.1688828125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 211151, + "answer_accuracy": 0.49022799034175335, + "avg_response_len": 334.49431649331353, + "resp_tokens_sum": 144073392, + "pass_at_k": { + "pass@1": 0.49022799034175335, + "pass@2": 0.5647673991763309, + "pass@4": 0.6285583233187992, + "pass@8": 0.6814975823127939, + "pass@16": 0.7254144359604083, + "pass@32": 0.7628179078044346, + "pass@64": 0.7958919545831215, + "pass@128": 0.82555720653789 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9119157223796034, + "pass@2": 0.9433006039347777, + "pass@4": 0.9610636509494386, + "pass@8": 0.9707724177448829, + "pass@16": 0.9765900844469294, + "pass@32": 0.9804514883161396, + "pass@64": 0.9834657893820904, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8797007415254238, + "pass@2": 0.9152302565728009, + "pass@4": 0.9368644835709092, + "pass@8": 0.9510211852512445, + "pass@16": 0.9606753690881392, + "pass@32": 0.9682087869225152, + "pass@64": 0.9731223154982348, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.7011902429467085, + "pass@2": 0.7779242928195887, + "pass@4": 0.8325995192019487, + "pass@8": 0.8719230350813825, + "pass@16": 0.9001092072397098, + "pass@32": 0.9216947243084077, + "pass@64": 0.9398832301721639, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.6429857866043613, + "pass@2": 0.7402437173449115, + "pass@4": 0.8083627916837498, + "pass@8": 0.8560466917319087, + "pass@16": 0.8906758368356597, + "pass@32": 0.9150102873575595, + "pass@64": 0.9324748798290006, + "pass@128": 0.9470404984423676 + }, + "15": { + "pass@1": 0.490923049132948, + "pass@2": 0.593261785421692, + "pass@4": 0.6829506191603214, + "pass@8": 0.7569743288240068, + "pass@16": 0.8138482959427785, + "pass@32": 0.8554503362783789, + "pass@64": 0.888217070271757, + "pass@128": 0.9190751445086706 + }, + "16": { + "pass@1": 0.39375, + "pass@2": 0.4888827152654307, + "pass@4": 0.5712073339622871, + "pass@8": 0.6410362751077191, + "pass@16": 0.7001922566374801, + "pass@32": 0.7505681623959332, + "pass@64": 0.7936145904228401, + "pass@128": 0.8290322580645161 + }, + "17": { + "pass@1": 0.2819048049132948, + "pass@2": 0.37064873867825776, + "pass@4": 0.45060537727220507, + "pass@8": 0.5193154487599553, + "pass@16": 0.579726409811185, + "pass@32": 0.6343912297837558, + "pass@64": 0.6846328533391884, + "pass@128": 0.7283236994219653 + }, + "18": { + "pass@1": 0.22379464285714284, + "pass@2": 0.3027994938132731, + "pass@4": 0.3840521158069528, + "pass@8": 0.46058139945706233, + "pass@16": 0.5295414509443225, + "pass@32": 0.592193073420357, + "pass@64": 0.6488709025980459, + "pass@128": 0.7 + }, + "19": { + "pass@1": 0.17630912162162163, + "pass@2": 0.2495322586365894, + "pass@4": 0.3233865428983538, + "pass@8": 0.38856729069713164, + "pass@16": 0.44334964674116345, + "pass@32": 0.49201329032879687, + "pass@64": 0.5396058277950417, + "pass@128": 0.5885885885885885 + }, + "20": { + "pass@1": 0.17933558558558557, + "pass@2": 0.25089742793975073, + "pass@4": 0.32348033522836656, + "pass@8": 0.3905311088984681, + "pass@16": 0.453306389976786, + "pass@32": 0.5135998226509072, + "pass@64": 0.5716527441993955, + "pass@128": 0.6246246246246246 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 213587, + "answer_accuracy": 0.4961785422242046, + "avg_response_len": 349.7932812035385, + "resp_tokens_sum": 150573415, + "pass_at_k": { + "pass@1": 0.4961785422242046, + "pass@2": 0.5735818708572463, + "pass@4": 0.6398494478444458, + "pass@8": 0.6950043420320503, + "pass@16": 0.7407837579575262, + "pass@32": 0.7798180483362328, + "pass@64": 0.8134201946596387, + "pass@128": 0.8412132024977699 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9141566265060241, + "pass@2": 0.9404475678303766, + "pass@4": 0.9542405067401211, + "pass@8": 0.9626365274799337, + "pass@16": 0.9686866446791836, + "pass@32": 0.9731272196351315, + "pass@64": 0.9758453657740855, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8625342868338558, + "pass@2": 0.8947961148273393, + "pass@4": 0.9139933786685754, + "pass@8": 0.9260792047881582, + "pass@16": 0.9333329464384502, + "pass@32": 0.9390314182250337, + "pass@64": 0.9440685241712219, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.7135635504201681, + "pass@2": 0.7769363158649286, + "pass@4": 0.8207167215967748, + "pass@8": 0.8556100836801505, + "pass@16": 0.8865367253781443, + "pass@32": 0.9131969663245345, + "pass@64": 0.9320169721566308, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6453079312865497, + "pass@2": 0.7307074382971863, + "pass@4": 0.7917011412498877, + "pass@8": 0.8400084203242523, + "pass@16": 0.8792234387626376, + "pass@32": 0.9082357436128263, + "pass@64": 0.9277205938920358, + "pass@128": 0.9385964912280702 + }, + "15": { + "pass@1": 0.5251893939393939, + "pass@2": 0.6354136840849434, + "pass@4": 0.7167874464555566, + "pass@8": 0.7734315453385432, + "pass@16": 0.8150253555478885, + "pass@32": 0.8482512514698151, + "pass@64": 0.8749376454187987, + "pass@128": 0.896969696969697 + }, + "16": { + "pass@1": 0.37506677350427353, + "pass@2": 0.4839536784664738, + "pass@4": 0.5785922203100682, + "pass@8": 0.6535012956218443, + "pass@16": 0.7126001113590912, + "pass@32": 0.7583889583386475, + "pass@64": 0.7941181600631738, + "pass@128": 0.8233618233618234 + }, + "17": { + "pass@1": 0.3039353649068323, + "pass@2": 0.40960891267667615, + "pass@4": 0.512253928565606, + "pass@8": 0.6018165928946795, + "pass@16": 0.6724456620568161, + "pass@32": 0.7258528783479469, + "pass@64": 0.7670180711381164, + "pass@128": 0.7981366459627329 + }, + "18": { + "pass@1": 0.27863051470588235, + "pass@2": 0.37495693897637794, + "pass@4": 0.4687283861576127, + "pass@8": 0.5524870961538081, + "pass@16": 0.6272312718210945, + "pass@32": 0.6982727954187062, + "pass@64": 0.7639304114248799, + "pass@128": 0.8205882352941176 + }, + "19": { + "pass@1": 0.18629092261904762, + "pass@2": 0.2586107400637419, + "pass@4": 0.3343563700742765, + "pass@8": 0.4060499041206816, + "pass@16": 0.4692035735259686, + "pass@32": 0.527745944458945, + "pass@64": 0.5895246970799036, + "pass@128": 0.6488095238095238 + }, + "20": { + "pass@1": 0.1628929640718563, + "pass@2": 0.23410830897260604, + "pass@4": 0.3095850714207131, + "pass@8": 0.37986419480354333, + "pass@16": 0.4437456057573142, + "pass@32": 0.5050701914914746, + "pass@64": 0.5632387354539061, + "pass@128": 0.6137724550898204 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 206242, + "answer_accuracy": 0.4924405944376528, + "avg_response_len": 377.34238185742663, + "resp_tokens_sum": 158037027, + "pass_at_k": { + "pass@1": 0.4924405944376528, + "pass@2": 0.5695273469837141, + "pass@4": 0.6361329726815955, + "pass@8": 0.6921019639478836, + "pass@16": 0.7379309817800689, + "pass@32": 0.7750366486024804, + "pass@64": 0.8060648084849908, + "pass@128": 0.8325183374083129 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9184523809523809, + "pass@2": 0.9463668603924509, + "pass@4": 0.9613165776896933, + "pass@8": 0.9696770133898424, + "pass@16": 0.9745358856156271, + "pass@32": 0.977831319059118, + "pass@64": 0.9809549621060872, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8733275993883792, + "pass@2": 0.9091482006549643, + "pass@4": 0.9321290414959594, + "pass@8": 0.9474101864198438, + "pass@16": 0.9582113860858212, + "pass@32": 0.9667417948856186, + "pass@64": 0.9751459929525179, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7013647762345679, + "pass@2": 0.7794614865850105, + "pass@4": 0.8347370620917749, + "pass@8": 0.8741468109649858, + "pass@16": 0.9042245400450407, + "pass@32": 0.9270647620892849, + "pass@64": 0.9446597552973784, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.6266459569732937, + "pass@2": 0.7149071824108042, + "pass@4": 0.777194388164387, + "pass@8": 0.8208584828195092, + "pass@16": 0.854085616211117, + "pass@32": 0.8785521084993064, + "pass@64": 0.8946411994274263, + "pass@128": 0.9050445103857567 + }, + "15": { + "pass@1": 0.5228105709876543, + "pass@2": 0.6264497970739771, + "pass@4": 0.7098052864919656, + "pass@8": 0.773887951251961, + "pass@16": 0.8196673549059977, + "pass@32": 0.8508791959747911, + "pass@64": 0.8725903607940296, + "pass@128": 0.8888888888888888 + }, + "16": { + "pass@1": 0.39115505162241887, + "pass@2": 0.49121359138735954, + "pass@4": 0.5756422308384014, + "pass@8": 0.6481636557709216, + "pass@16": 0.7119062076422585, + "pass@32": 0.7692503507300534, + "pass@64": 0.8199119593740152, + "pass@128": 0.8613569321533924 + }, + "17": { + "pass@1": 0.2642601656626506, + "pass@2": 0.3532293544255763, + "pass@4": 0.4450892874083509, + "pass@8": 0.5301218040712592, + "pass@16": 0.6015799792258167, + "pass@32": 0.6594386706097841, + "pass@64": 0.71061255652067, + "pass@128": 0.7560240963855421 + }, + "18": { + "pass@1": 0.26895161290322583, + "pass@2": 0.3575180181610362, + "pass@4": 0.4426478966338885, + "pass@8": 0.5166340111876507, + "pass@16": 0.5783399405568437, + "pass@32": 0.6329287740303212, + "pass@64": 0.6811106058382247, + "pass@128": 0.7193548387096774 + }, + "19": { + "pass@1": 0.19946657854984895, + "pass@2": 0.27503003306610846, + "pass@4": 0.35067128164870626, + "pass@8": 0.4218210302394642, + "pass@16": 0.48489409655088384, + "pass@32": 0.5400624117409185, + "pass@64": 0.5910281024631786, + "pass@128": 0.6435045317220544 + }, + "20": { + "pass@1": 0.17666103603603603, + "pass@2": 0.25825345522688054, + "pass@4": 0.3461097191116876, + "pass@8": 0.43004937374319013, + "pass@16": 0.5014484489893055, + "pass@32": 0.5554789580885844, + "pass@64": 0.5965143125091258, + "pass@128": 0.6306306306306306 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7262035361842105, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7262035361842105, + "pass@2": 0.7688165276626383, + "pass@4": 0.8047135841325541, + "pass@8": 0.8344670361631966, + "pass@16": 0.8589507027951635, + "pass@32": 0.8793705558527696, + "pass@64": 0.896825238503104, + "pass@128": 0.9117368421052632 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.937453125, + "pass@2": 0.9558822588582679, + "pass@4": 0.9680160880202473, + "pass@8": 0.9768610868196399, + "pass@16": 0.9834541239321481, + "pass@32": 0.9879920155201115, + "pass@64": 0.9909572141273251, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99975, + "pass@2": 0.9999859744094489, + "pass@4": 0.9999999306336709, + "pass@8": 0.9999999999990998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999734375, + "pass@2": 0.9999731791338583, + "pass@4": 0.9999996169853769, + "pass@8": 0.999999999947103, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9915234375, + "pass@2": 0.9945222687007875, + "pass@4": 0.9965733731721034, + "pass@8": 0.9980091055458946, + "pass@16": 0.9990016636755122, + "pass@32": 0.9996162045730839, + "pass@64": 0.9999389956470872, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9981640625, + "pass@2": 0.9992300688976377, + "pass@4": 0.9998092772778402, + "pass@8": 0.9999886283533156, + "pass@16": 0.9999999673324529, + "pass@32": 0.9999999999999404, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9935, + "pass@2": 0.9960433070866144, + "pass@4": 0.997461278871391, + "pass@8": 0.9986368243453445, + "pass@16": 0.9994574439865509, + "pass@32": 0.9998675346942224, + "pass@64": 0.999993426730079, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.983125, + "pass@2": 0.9905113188976382, + "pass@4": 0.9937524416947879, + "pass@8": 0.9958266836261157, + "pass@16": 0.9975393869690035, + "pass@32": 0.9989698342517158, + "pass@64": 0.9998174124680705, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9650859375, + "pass@2": 0.9784740403543307, + "pass@4": 0.9856065541807275, + "pass@8": 0.9904547656902203, + "pass@16": 0.9938252398099534, + "pass@32": 0.9962647016192685, + "pass@64": 0.9978198874108859, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.91471875, + "pass@2": 0.9433192667322837, + "pass@4": 0.9588780389951258, + "pass@8": 0.9677262498050817, + "pass@16": 0.9733190698121774, + "pass@32": 0.9771944777980829, + "pass@64": 0.9801448981522918, + "pass@128": 0.983 + }, + "12": { + "pass@1": 0.872140625, + "pass@2": 0.9067229330708668, + "pass@4": 0.9280201115485563, + "pass@8": 0.9418838968656523, + "pass@16": 0.95114741382113, + "pass@32": 0.9584214999119537, + "pass@64": 0.9645158985924684, + "pass@128": 0.969 + }, + "13": { + "pass@1": 0.7056640625, + "pass@2": 0.7780696358267724, + "pass@4": 0.8290499243532062, + "pass@8": 0.8668198148174304, + "pass@16": 0.8965971990440574, + "pass@32": 0.9204009169491684, + "pass@64": 0.9386225702011887, + "pass@128": 0.952 + }, + "14": { + "pass@1": 0.6382734375, + "pass@2": 0.728443897637796, + "pass@4": 0.7921607552493434, + "pass@8": 0.8387031765070113, + "pass@16": 0.8744282123442151, + "pass@32": 0.9004069871216294, + "pass@64": 0.9180989637432281, + "pass@128": 0.93 + }, + "15": { + "pass@1": 0.5125625, + "pass@2": 0.6179248277559062, + "pass@4": 0.702817684383202, + "pass@8": 0.7678852239404601, + "pass@16": 0.816122100716546, + "pass@32": 0.8515935888331901, + "pass@64": 0.8787718061994974, + "pass@128": 0.902 + }, + "16": { + "pass@1": 0.3863125, + "pass@2": 0.4879427903543308, + "pass@4": 0.5753028591113607, + "pass@8": 0.6478276793530026, + "pass@16": 0.7085184430353842, + "pass@32": 0.759646523617092, + "pass@64": 0.802706151441046, + "pass@128": 0.838 + }, + "17": { + "pass@1": 0.283140625, + "pass@2": 0.37741067913385873, + "pass@4": 0.4686248689538805, + "pass@8": 0.549468527134689, + "pass@16": 0.6168373940799338, + "pass@32": 0.6721576309756664, + "pass@64": 0.7197861549266963, + "pass@128": 0.76 + }, + "18": { + "pass@1": 0.2564375, + "pass@2": 0.3442957677165357, + "pass@4": 0.43100673978252685, + "pass@8": 0.5092056459704383, + "pass@16": 0.5778835218223048, + "pass@32": 0.640888246088884, + "pass@64": 0.6979854436036252, + "pass@128": 0.747 + }, + "19": { + "pass@1": 0.187328125, + "pass@2": 0.26102239173228325, + "pass@4": 0.33610365335583037, + "pass@8": 0.4054484365959564, + "pass@16": 0.46578777902787455, + "pass@32": 0.5199237213039392, + "pass@64": 0.5733993407899091, + "pass@128": 0.627 + }, + "20": { + "pass@1": 0.172953125, + "pass@2": 0.24773941929133864, + "pass@4": 0.32637490194975594, + "pass@8": 0.40012794178405553, + "pass@16": 0.4661443936986512, + "pass@32": 0.5246966779444031, + "pass@64": 0.5771213675255428, + "pass@128": 0.623 + } + }, + "per_op_accuracy": { + "10": 0.937453125, + "2": 1.0, + "3": 0.99975, + "4": 0.999734375, + "5": 0.9915234375, + "6": 0.9981640625, + "7": 0.9935, + "8": 0.983125, + "9": 0.9650859375, + "11": 0.91471875, + "12": 0.872140625, + "13": 0.7056640625, + "14": 0.6382734375, + "15": 0.5125625, + "16": 0.3863125, + "17": 0.283140625, + "18": 0.2564375, + "19": 0.187328125, + "20": 0.172953125 + }, + "per_op_avg_response_len": { + "10": 278.47321875, + "2": 115.028609375, + "3": 143.7909921875, + "4": 158.810578125, + "5": 171.9432890625, + "6": 197.24334375, + "7": 222.158796875, + "8": 241.1735, + "9": 256.845703125, + "11": 296.288, + "12": 306.9627890625, + "13": 324.4796484375, + "14": 348.7399765625, + "15": 363.4608984375, + "16": 365.20496875, + "17": 380.63134375, + "18": 384.5688828125, + "19": 376.0870625, + "20": 390.1688828125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.47321875, + "2": 115.028609375, + "3": 143.7909921875, + "4": 158.810578125, + "5": 171.9432890625, + "6": 197.24334375, + "7": 222.158796875, + "8": 241.1735, + "9": 256.845703125, + "11": 296.288, + "12": 306.9627890625, + "13": 324.4796484375, + "14": 348.7399765625, + "15": 363.4608984375, + "16": 365.20496875, + "17": 380.63134375, + "18": 384.5688828125, + "19": 376.0870625, + "20": 390.1688828125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 584247, + "answer_accuracy": 0.7215348857888081, + "avg_response_len": 266.3354521518337, + "resp_tokens_sum": 215659273, + "pass_at_k": { + "pass@1": 0.7215348857888081, + "pass@2": 0.7638074362212142, + "pass@4": 0.7992237509692353, + "pass@8": 0.8283172993096083, + "pass@16": 0.8521883729736969, + "pass@32": 0.8724319556054612, + "pass@64": 0.8904025689043645, + "pass@128": 0.9065760354094214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9300792772861357, + "pass@2": 0.9506288034515598, + "pass@4": 0.9636359633696227, + "pass@8": 0.9717781371735056, + "pass@16": 0.976361752430648, + "pass@32": 0.9797711596050733, + "pass@64": 0.9841706667237161, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998088685015291, + "pass@2": 0.9999939801102844, + "pass@4": 0.9999999957000787, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9992919921875, + "pass@2": 0.9999169537401574, + "pass@4": 0.9999988030793026, + "pass@8": 0.9999999998346969, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.994171626984127, + "pass@2": 0.9966371391076116, + "pass@4": 0.998588466322662, + "pass@8": 0.9996704275760121, + "pass@16": 0.9999772476512871, + "pass@32": 0.9999999361932187, + "pass@64": 0.999999999999996, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998362649700598, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9945220044378699, + "pass@2": 0.9966806975958625, + "pass@4": 0.997982214963514, + "pass@8": 0.9990981813231818, + "pass@16": 0.9997780458447582, + "pass@32": 0.9999893290784243, + "pass@64": 0.9999999971901181, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9853265224358975, + "pass@2": 0.9928211910710678, + "pass@4": 0.9950341814244371, + "pass@8": 0.9964232589832497, + "pass@16": 0.9976202982400736, + "pass@32": 0.9986469609135363, + "pass@64": 0.999608819891556, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9609154929577465, + "pass@2": 0.9763100255073749, + "pass@4": 0.9853365164741729, + "pass@8": 0.990951142886426, + "pass@16": 0.9936960304062404, + "pass@32": 0.9955088545287316, + "pass@64": 0.9971786389887288, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9119157223796034, + "pass@2": 0.9433006039347777, + "pass@4": 0.9610636509494386, + "pass@8": 0.9707724177448829, + "pass@16": 0.9765900844469294, + "pass@32": 0.9804514883161396, + "pass@64": 0.9834657893820904, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8797007415254238, + "pass@2": 0.9152302565728009, + "pass@4": 0.9368644835709092, + "pass@8": 0.9510211852512445, + "pass@16": 0.9606753690881392, + "pass@32": 0.9682087869225152, + "pass@64": 0.9731223154982348, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.7011902429467085, + "pass@2": 0.7779242928195887, + "pass@4": 0.8325995192019487, + "pass@8": 0.8719230350813825, + "pass@16": 0.9001092072397098, + "pass@32": 0.9216947243084077, + "pass@64": 0.9398832301721639, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.6429857866043613, + "pass@2": 0.7402437173449115, + "pass@4": 0.8083627916837498, + "pass@8": 0.8560466917319087, + "pass@16": 0.8906758368356597, + "pass@32": 0.9150102873575595, + "pass@64": 0.9324748798290006, + "pass@128": 0.9470404984423676 + }, + "15": { + "pass@1": 0.490923049132948, + "pass@2": 0.593261785421692, + "pass@4": 0.6829506191603214, + "pass@8": 0.7569743288240068, + "pass@16": 0.8138482959427785, + "pass@32": 0.8554503362783789, + "pass@64": 0.888217070271757, + "pass@128": 0.9190751445086706 + }, + "16": { + "pass@1": 0.39375, + "pass@2": 0.4888827152654307, + "pass@4": 0.5712073339622871, + "pass@8": 0.6410362751077191, + "pass@16": 0.7001922566374801, + "pass@32": 0.7505681623959332, + "pass@64": 0.7936145904228401, + "pass@128": 0.8290322580645161 + }, + "17": { + "pass@1": 0.2819048049132948, + "pass@2": 0.37064873867825776, + "pass@4": 0.45060537727220507, + "pass@8": 0.5193154487599553, + "pass@16": 0.579726409811185, + "pass@32": 0.6343912297837558, + "pass@64": 0.6846328533391884, + "pass@128": 0.7283236994219653 + }, + "18": { + "pass@1": 0.22379464285714284, + "pass@2": 0.3027994938132731, + "pass@4": 0.3840521158069528, + "pass@8": 0.46058139945706233, + "pass@16": 0.5295414509443225, + "pass@32": 0.592193073420357, + "pass@64": 0.6488709025980459, + "pass@128": 0.7 + }, + "19": { + "pass@1": 0.17630912162162163, + "pass@2": 0.2495322586365894, + "pass@4": 0.3233865428983538, + "pass@8": 0.38856729069713164, + "pass@16": 0.44334964674116345, + "pass@32": 0.49201329032879687, + "pass@64": 0.5396058277950417, + "pass@128": 0.5885885885885885 + }, + "20": { + "pass@1": 0.17933558558558557, + "pass@2": 0.25089742793975073, + "pass@4": 0.32348033522836656, + "pass@8": 0.3905311088984681, + "pass@16": 0.453306389976786, + "pass@32": 0.5135998226509072, + "pass@64": 0.5716527441993955, + "pass@128": 0.6246246246246246 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 598848, + "answer_accuracy": 0.7293063133281372, + "avg_response_len": 276.79440520265, + "resp_tokens_sum": 227281422, + "pass_at_k": { + "pass@1": 0.7293063133281372, + "pass@2": 0.7721860596780367, + "pass@4": 0.8081954867245168, + "pass@8": 0.837972898442128, + "pass@16": 0.8626448010517291, + "pass@32": 0.8835733216214263, + "pass@64": 0.9014402560974997, + "pass@128": 0.9161340607950117 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9367810582822086, + "pass@2": 0.9545342012463163, + "pass@4": 0.9656823695542656, + "pass@8": 0.9742888215981227, + "pass@16": 0.9816129882446868, + "pass@32": 0.986408904087696, + "pass@64": 0.9888729553346032, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9994318181818181, + "pass@2": 0.9999634633738963, + "pass@4": 0.9999997940598334, + "pass@8": 0.9999999999972722, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999992720029819, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9961050724637681, + "pass@2": 0.9988381547415269, + "pass@4": 0.9998410413372242, + "pass@8": 0.9999961121701334, + "pass@16": 0.999999998171191, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9966645773638968, + "pass@2": 0.9984552320465674, + "pass@4": 0.9995937385513057, + "pass@8": 0.9999733183431431, + "pass@16": 0.9999999126220942, + "pass@32": 0.9999999999998295, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9936100072254336, + "pass@2": 0.9963759046015201, + "pass@4": 0.9980907790247318, + "pass@8": 0.9992033281484878, + "pass@16": 0.999815155302214, + "pass@32": 0.9999926083157176, + "pass@64": 0.999999998852127, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9861355633802817, + "pass@2": 0.9926226156149497, + "pass@4": 0.9959982136915982, + "pass@8": 0.9980394885254082, + "pass@16": 0.9991635856551462, + "pass@32": 0.9998083508916638, + "pass@64": 0.999995930319257, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9650542664670658, + "pass@2": 0.9768406101183464, + "pass@4": 0.9827120368374614, + "pass@8": 0.987007989958256, + "pass@16": 0.9909152726094004, + "pass@32": 0.9942995564981604, + "pass@64": 0.9965149398796272, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9141566265060241, + "pass@2": 0.9404475678303766, + "pass@4": 0.9542405067401211, + "pass@8": 0.9626365274799337, + "pass@16": 0.9686866446791836, + "pass@32": 0.9731272196351315, + "pass@64": 0.9758453657740855, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8625342868338558, + "pass@2": 0.8947961148273393, + "pass@4": 0.9139933786685754, + "pass@8": 0.9260792047881582, + "pass@16": 0.9333329464384502, + "pass@32": 0.9390314182250337, + "pass@64": 0.9440685241712219, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.7135635504201681, + "pass@2": 0.7769363158649286, + "pass@4": 0.8207167215967748, + "pass@8": 0.8556100836801505, + "pass@16": 0.8865367253781443, + "pass@32": 0.9131969663245345, + "pass@64": 0.9320169721566308, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6453079312865497, + "pass@2": 0.7307074382971863, + "pass@4": 0.7917011412498877, + "pass@8": 0.8400084203242523, + "pass@16": 0.8792234387626376, + "pass@32": 0.9082357436128263, + "pass@64": 0.9277205938920358, + "pass@128": 0.9385964912280702 + }, + "15": { + "pass@1": 0.5251893939393939, + "pass@2": 0.6354136840849434, + "pass@4": 0.7167874464555566, + "pass@8": 0.7734315453385432, + "pass@16": 0.8150253555478885, + "pass@32": 0.8482512514698151, + "pass@64": 0.8749376454187987, + "pass@128": 0.896969696969697 + }, + "16": { + "pass@1": 0.37506677350427353, + "pass@2": 0.4839536784664738, + "pass@4": 0.5785922203100682, + "pass@8": 0.6535012956218443, + "pass@16": 0.7126001113590912, + "pass@32": 0.7583889583386475, + "pass@64": 0.7941181600631738, + "pass@128": 0.8233618233618234 + }, + "17": { + "pass@1": 0.3039353649068323, + "pass@2": 0.40960891267667615, + "pass@4": 0.512253928565606, + "pass@8": 0.6018165928946795, + "pass@16": 0.6724456620568161, + "pass@32": 0.7258528783479469, + "pass@64": 0.7670180711381164, + "pass@128": 0.7981366459627329 + }, + "18": { + "pass@1": 0.27863051470588235, + "pass@2": 0.37495693897637794, + "pass@4": 0.4687283861576127, + "pass@8": 0.5524870961538081, + "pass@16": 0.6272312718210945, + "pass@32": 0.6982727954187062, + "pass@64": 0.7639304114248799, + "pass@128": 0.8205882352941176 + }, + "19": { + "pass@1": 0.18629092261904762, + "pass@2": 0.2586107400637419, + "pass@4": 0.3343563700742765, + "pass@8": 0.4060499041206816, + "pass@16": 0.4692035735259686, + "pass@32": 0.527745944458945, + "pass@64": 0.5895246970799036, + "pass@128": 0.6488095238095238 + }, + "20": { + "pass@1": 0.1628929640718563, + "pass@2": 0.23410830897260604, + "pass@4": 0.3095850714207131, + "pass@8": 0.37986419480354333, + "pass@16": 0.4437456057573142, + "pass@32": 0.5050701914914746, + "pass@64": 0.5632387354539061, + "pass@128": 0.6137724550898204 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 583032, + "answer_accuracy": 0.7277420514459179, + "avg_response_len": 297.42551600695, + "resp_tokens_sum": 238283047, + "pass_at_k": { + "pass@1": 0.7277420514459179, + "pass@2": 0.7704257247516288, + "pass@4": 0.8066934977711516, + "pass@8": 0.8370893606265589, + "pass@16": 0.8619992501884716, + "pass@32": 0.8820759149769569, + "pass@64": 0.8985866173197917, + "pass@128": 0.9124460776481866 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9455690298507463, + "pass@2": 0.9625102832295216, + "pass@4": 0.9747195342186703, + "pass@8": 0.9845078879905481, + "pass@16": 0.9924228528071358, + "pass@32": 0.9978516109892621, + "pass@64": 0.9998530587131482, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9844209558823529, + "pass@2": 0.9881835485178323, + "pass@4": 0.9913907265268312, + "pass@8": 0.9944537122375137, + "pass@16": 0.9970847979302819, + "pass@32": 0.998871249035941, + "pass@64": 0.9998205754326136, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9980530362776026, + "pass@2": 0.9992722900469461, + "pass@4": 0.9998456231023172, + "pass@8": 0.999993502370847, + "pass@16": 0.9999999931461893, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9922863924050633, + "pass@2": 0.994997368060401, + "pass@4": 0.9962148122505257, + "pass@8": 0.9975230617681402, + "pass@16": 0.9987228504951157, + "pass@32": 0.9996003132546726, + "pass@64": 0.9999792027752626, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9778528528528528, + "pass@2": 0.9860963325530254, + "pass@4": 0.9901573910806193, + "pass@8": 0.9929087339243897, + "pass@16": 0.9957320751066778, + "pass@32": 0.9983784320725285, + "pass@64": 0.9998225387344413, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9698804260450161, + "pass@2": 0.9826984486163508, + "pass@4": 0.9890233779057358, + "pass@8": 0.9935898434710003, + "pass@16": 0.9970979034218608, + "pass@32": 0.9992379626726149, + "pass@64": 0.9999533139874329, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9184523809523809, + "pass@2": 0.9463668603924509, + "pass@4": 0.9613165776896933, + "pass@8": 0.9696770133898424, + "pass@16": 0.9745358856156271, + "pass@32": 0.977831319059118, + "pass@64": 0.9809549621060872, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8733275993883792, + "pass@2": 0.9091482006549643, + "pass@4": 0.9321290414959594, + "pass@8": 0.9474101864198438, + "pass@16": 0.9582113860858212, + "pass@32": 0.9667417948856186, + "pass@64": 0.9751459929525179, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7013647762345679, + "pass@2": 0.7794614865850105, + "pass@4": 0.8347370620917749, + "pass@8": 0.8741468109649858, + "pass@16": 0.9042245400450407, + "pass@32": 0.9270647620892849, + "pass@64": 0.9446597552973784, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.6266459569732937, + "pass@2": 0.7149071824108042, + "pass@4": 0.777194388164387, + "pass@8": 0.8208584828195092, + "pass@16": 0.854085616211117, + "pass@32": 0.8785521084993064, + "pass@64": 0.8946411994274263, + "pass@128": 0.9050445103857567 + }, + "15": { + "pass@1": 0.5228105709876543, + "pass@2": 0.6264497970739771, + "pass@4": 0.7098052864919656, + "pass@8": 0.773887951251961, + "pass@16": 0.8196673549059977, + "pass@32": 0.8508791959747911, + "pass@64": 0.8725903607940296, + "pass@128": 0.8888888888888888 + }, + "16": { + "pass@1": 0.39115505162241887, + "pass@2": 0.49121359138735954, + "pass@4": 0.5756422308384014, + "pass@8": 0.6481636557709216, + "pass@16": 0.7119062076422585, + "pass@32": 0.7692503507300534, + "pass@64": 0.8199119593740152, + "pass@128": 0.8613569321533924 + }, + "17": { + "pass@1": 0.2642601656626506, + "pass@2": 0.3532293544255763, + "pass@4": 0.4450892874083509, + "pass@8": 0.5301218040712592, + "pass@16": 0.6015799792258167, + "pass@32": 0.6594386706097841, + "pass@64": 0.71061255652067, + "pass@128": 0.7560240963855421 + }, + "18": { + "pass@1": 0.26895161290322583, + "pass@2": 0.3575180181610362, + "pass@4": 0.4426478966338885, + "pass@8": 0.5166340111876507, + "pass@16": 0.5783399405568437, + "pass@32": 0.6329287740303212, + "pass@64": 0.6811106058382247, + "pass@128": 0.7193548387096774 + }, + "19": { + "pass@1": 0.19946657854984895, + "pass@2": 0.27503003306610846, + "pass@4": 0.35067128164870626, + "pass@8": 0.4218210302394642, + "pass@16": 0.48489409655088384, + "pass@32": 0.5400624117409185, + "pass@64": 0.5910281024631786, + "pass@128": 0.6435045317220544 + }, + "20": { + "pass@1": 0.17666103603603603, + "pass@2": 0.25825345522688054, + "pass@4": 0.3461097191116876, + "pass@8": 0.43004937374319013, + "pass@16": 0.5014484489893055, + "pass@32": 0.5554789580885844, + "pass@64": 0.5965143125091258, + "pass@128": 0.6306306306306306 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch1-RL-op9-12/composition/summary.csv b/composition/Qwen3-100M-epoch1-RL-op9-12/composition/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..5f9f8afed3e65bbb52834c50fd0352a55bd86973 --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op9-12/composition/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.9853706597222223,0.492953125,0.7262035361842105,0.0,0.0,0.0,198.38533680555557,353.6592453125,1152000,1280000,2432000 diff --git a/composition/Qwen3-100M-epoch1-RL-op9-12/composition/summary.json b/composition/Qwen3-100M-epoch1-RL-op9-12/composition/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4602811673f8da54190dda29a7c35365d545afba --- /dev/null +++ b/composition/Qwen3-100M-epoch1-RL-op9-12/composition/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.9853706597222223, + "ood_acc": 0.492953125, + "total_acc": 0.7262035361842105, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 198.38533680555557, + "ood_avg_resp_len": 353.6592453125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 584247, + "answer_accuracy": 0.7215348857888081, + "avg_response_len": 266.3354521518337, + "resp_tokens_sum": 215659273, + "pass_at_k": { + "pass@1": 0.7215348857888081, + "pass@2": 0.7638074362212142, + "pass@4": 0.7992237509692353, + "pass@8": 0.8283172993096083, + "pass@16": 0.8521883729736969, + "pass@32": 0.8724319556054612, + "pass@64": 0.8904025689043645, + "pass@128": 0.9065760354094214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9300792772861357, + "pass@2": 0.9506288034515598, + "pass@4": 0.9636359633696227, + "pass@8": 0.9717781371735056, + "pass@16": 0.976361752430648, + "pass@32": 0.9797711596050733, + "pass@64": 0.9841706667237161, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998088685015291, + "pass@2": 0.9999939801102844, + "pass@4": 0.9999999957000787, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9992919921875, + "pass@2": 0.9999169537401574, + "pass@4": 0.9999988030793026, + "pass@8": 0.9999999998346969, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.994171626984127, + "pass@2": 0.9966371391076116, + "pass@4": 0.998588466322662, + "pass@8": 0.9996704275760121, + "pass@16": 0.9999772476512871, + "pass@32": 0.9999999361932187, + "pass@64": 0.999999999999996, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998362649700598, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9945220044378699, + "pass@2": 0.9966806975958625, + "pass@4": 0.997982214963514, + "pass@8": 0.9990981813231818, + "pass@16": 0.9997780458447582, + "pass@32": 0.9999893290784243, + "pass@64": 0.9999999971901181, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9853265224358975, + "pass@2": 0.9928211910710678, + "pass@4": 0.9950341814244371, + "pass@8": 0.9964232589832497, + "pass@16": 0.9976202982400736, + "pass@32": 0.9986469609135363, + "pass@64": 0.999608819891556, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9609154929577465, + "pass@2": 0.9763100255073749, + "pass@4": 0.9853365164741729, + "pass@8": 0.990951142886426, + "pass@16": 0.9936960304062404, + "pass@32": 0.9955088545287316, + "pass@64": 0.9971786389887288, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9119157223796034, + "pass@2": 0.9433006039347777, + "pass@4": 0.9610636509494386, + "pass@8": 0.9707724177448829, + "pass@16": 0.9765900844469294, + "pass@32": 0.9804514883161396, + "pass@64": 0.9834657893820904, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8797007415254238, + "pass@2": 0.9152302565728009, + "pass@4": 0.9368644835709092, + "pass@8": 0.9510211852512445, + "pass@16": 0.9606753690881392, + "pass@32": 0.9682087869225152, + "pass@64": 0.9731223154982348, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.7011902429467085, + "pass@2": 0.7779242928195887, + "pass@4": 0.8325995192019487, + "pass@8": 0.8719230350813825, + "pass@16": 0.9001092072397098, + "pass@32": 0.9216947243084077, + "pass@64": 0.9398832301721639, + "pass@128": 0.9561128526645768 + }, + "14": { + "pass@1": 0.6429857866043613, + "pass@2": 0.7402437173449115, + "pass@4": 0.8083627916837498, + "pass@8": 0.8560466917319087, + "pass@16": 0.8906758368356597, + "pass@32": 0.9150102873575595, + "pass@64": 0.9324748798290006, + "pass@128": 0.9470404984423676 + }, + "15": { + "pass@1": 0.490923049132948, + "pass@2": 0.593261785421692, + "pass@4": 0.6829506191603214, + "pass@8": 0.7569743288240068, + "pass@16": 0.8138482959427785, + "pass@32": 0.8554503362783789, + "pass@64": 0.888217070271757, + "pass@128": 0.9190751445086706 + }, + "16": { + "pass@1": 0.39375, + "pass@2": 0.4888827152654307, + "pass@4": 0.5712073339622871, + "pass@8": 0.6410362751077191, + "pass@16": 0.7001922566374801, + "pass@32": 0.7505681623959332, + "pass@64": 0.7936145904228401, + "pass@128": 0.8290322580645161 + }, + "17": { + "pass@1": 0.2819048049132948, + "pass@2": 0.37064873867825776, + "pass@4": 0.45060537727220507, + "pass@8": 0.5193154487599553, + "pass@16": 0.579726409811185, + "pass@32": 0.6343912297837558, + "pass@64": 0.6846328533391884, + "pass@128": 0.7283236994219653 + }, + "18": { + "pass@1": 0.22379464285714284, + "pass@2": 0.3027994938132731, + "pass@4": 0.3840521158069528, + "pass@8": 0.46058139945706233, + "pass@16": 0.5295414509443225, + "pass@32": 0.592193073420357, + "pass@64": 0.6488709025980459, + "pass@128": 0.7 + }, + "19": { + "pass@1": 0.17630912162162163, + "pass@2": 0.2495322586365894, + "pass@4": 0.3233865428983538, + "pass@8": 0.38856729069713164, + "pass@16": 0.44334964674116345, + "pass@32": 0.49201329032879687, + "pass@64": 0.5396058277950417, + "pass@128": 0.5885885885885885 + }, + "20": { + "pass@1": 0.17933558558558557, + "pass@2": 0.25089742793975073, + "pass@4": 0.32348033522836656, + "pass@8": 0.3905311088984681, + "pass@16": 0.453306389976786, + "pass@32": 0.5135998226509072, + "pass@64": 0.5716527441993955, + "pass@128": 0.6246246246246246 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 598848, + "answer_accuracy": 0.7293063133281372, + "avg_response_len": 276.79440520265, + "resp_tokens_sum": 227281422, + "pass_at_k": { + "pass@1": 0.7293063133281372, + "pass@2": 0.7721860596780367, + "pass@4": 0.8081954867245168, + "pass@8": 0.837972898442128, + "pass@16": 0.8626448010517291, + "pass@32": 0.8835733216214263, + "pass@64": 0.9014402560974997, + "pass@128": 0.9161340607950117 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9367810582822086, + "pass@2": 0.9545342012463163, + "pass@4": 0.9656823695542656, + "pass@8": 0.9742888215981227, + "pass@16": 0.9816129882446868, + "pass@32": 0.986408904087696, + "pass@64": 0.9888729553346032, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9994318181818181, + "pass@2": 0.9999634633738963, + "pass@4": 0.9999997940598334, + "pass@8": 0.9999999999972722, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999992720029819, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9961050724637681, + "pass@2": 0.9988381547415269, + "pass@4": 0.9998410413372242, + "pass@8": 0.9999961121701334, + "pass@16": 0.999999998171191, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9966645773638968, + "pass@2": 0.9984552320465674, + "pass@4": 0.9995937385513057, + "pass@8": 0.9999733183431431, + "pass@16": 0.9999999126220942, + "pass@32": 0.9999999999998295, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9936100072254336, + "pass@2": 0.9963759046015201, + "pass@4": 0.9980907790247318, + "pass@8": 0.9992033281484878, + "pass@16": 0.999815155302214, + "pass@32": 0.9999926083157176, + "pass@64": 0.999999998852127, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9861355633802817, + "pass@2": 0.9926226156149497, + "pass@4": 0.9959982136915982, + "pass@8": 0.9980394885254082, + "pass@16": 0.9991635856551462, + "pass@32": 0.9998083508916638, + "pass@64": 0.999995930319257, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9650542664670658, + "pass@2": 0.9768406101183464, + "pass@4": 0.9827120368374614, + "pass@8": 0.987007989958256, + "pass@16": 0.9909152726094004, + "pass@32": 0.9942995564981604, + "pass@64": 0.9965149398796272, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9141566265060241, + "pass@2": 0.9404475678303766, + "pass@4": 0.9542405067401211, + "pass@8": 0.9626365274799337, + "pass@16": 0.9686866446791836, + "pass@32": 0.9731272196351315, + "pass@64": 0.9758453657740855, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8625342868338558, + "pass@2": 0.8947961148273393, + "pass@4": 0.9139933786685754, + "pass@8": 0.9260792047881582, + "pass@16": 0.9333329464384502, + "pass@32": 0.9390314182250337, + "pass@64": 0.9440685241712219, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.7135635504201681, + "pass@2": 0.7769363158649286, + "pass@4": 0.8207167215967748, + "pass@8": 0.8556100836801505, + "pass@16": 0.8865367253781443, + "pass@32": 0.9131969663245345, + "pass@64": 0.9320169721566308, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6453079312865497, + "pass@2": 0.7307074382971863, + "pass@4": 0.7917011412498877, + "pass@8": 0.8400084203242523, + "pass@16": 0.8792234387626376, + "pass@32": 0.9082357436128263, + "pass@64": 0.9277205938920358, + "pass@128": 0.9385964912280702 + }, + "15": { + "pass@1": 0.5251893939393939, + "pass@2": 0.6354136840849434, + "pass@4": 0.7167874464555566, + "pass@8": 0.7734315453385432, + "pass@16": 0.8150253555478885, + "pass@32": 0.8482512514698151, + "pass@64": 0.8749376454187987, + "pass@128": 0.896969696969697 + }, + "16": { + "pass@1": 0.37506677350427353, + "pass@2": 0.4839536784664738, + "pass@4": 0.5785922203100682, + "pass@8": 0.6535012956218443, + "pass@16": 0.7126001113590912, + "pass@32": 0.7583889583386475, + "pass@64": 0.7941181600631738, + "pass@128": 0.8233618233618234 + }, + "17": { + "pass@1": 0.3039353649068323, + "pass@2": 0.40960891267667615, + "pass@4": 0.512253928565606, + "pass@8": 0.6018165928946795, + "pass@16": 0.6724456620568161, + "pass@32": 0.7258528783479469, + "pass@64": 0.7670180711381164, + "pass@128": 0.7981366459627329 + }, + "18": { + "pass@1": 0.27863051470588235, + "pass@2": 0.37495693897637794, + "pass@4": 0.4687283861576127, + "pass@8": 0.5524870961538081, + "pass@16": 0.6272312718210945, + "pass@32": 0.6982727954187062, + "pass@64": 0.7639304114248799, + "pass@128": 0.8205882352941176 + }, + "19": { + "pass@1": 0.18629092261904762, + "pass@2": 0.2586107400637419, + "pass@4": 0.3343563700742765, + "pass@8": 0.4060499041206816, + "pass@16": 0.4692035735259686, + "pass@32": 0.527745944458945, + "pass@64": 0.5895246970799036, + "pass@128": 0.6488095238095238 + }, + "20": { + "pass@1": 0.1628929640718563, + "pass@2": 0.23410830897260604, + "pass@4": 0.3095850714207131, + "pass@8": 0.37986419480354333, + "pass@16": 0.4437456057573142, + "pass@32": 0.5050701914914746, + "pass@64": 0.5632387354539061, + "pass@128": 0.6137724550898204 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 583032, + "answer_accuracy": 0.7277420514459179, + "avg_response_len": 297.42551600695, + "resp_tokens_sum": 238283047, + "pass_at_k": { + "pass@1": 0.7277420514459179, + "pass@2": 0.7704257247516288, + "pass@4": 0.8066934977711516, + "pass@8": 0.8370893606265589, + "pass@16": 0.8619992501884716, + "pass@32": 0.8820759149769569, + "pass@64": 0.8985866173197917, + "pass@128": 0.9124460776481866 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9455690298507463, + "pass@2": 0.9625102832295216, + "pass@4": 0.9747195342186703, + "pass@8": 0.9845078879905481, + "pass@16": 0.9924228528071358, + "pass@32": 0.9978516109892621, + "pass@64": 0.9998530587131482, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9844209558823529, + "pass@2": 0.9881835485178323, + "pass@4": 0.9913907265268312, + "pass@8": 0.9944537122375137, + "pass@16": 0.9970847979302819, + "pass@32": 0.998871249035941, + "pass@64": 0.9998205754326136, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9980530362776026, + "pass@2": 0.9992722900469461, + "pass@4": 0.9998456231023172, + "pass@8": 0.999993502370847, + "pass@16": 0.9999999931461893, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9922863924050633, + "pass@2": 0.994997368060401, + "pass@4": 0.9962148122505257, + "pass@8": 0.9975230617681402, + "pass@16": 0.9987228504951157, + "pass@32": 0.9996003132546726, + "pass@64": 0.9999792027752626, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9778528528528528, + "pass@2": 0.9860963325530254, + "pass@4": 0.9901573910806193, + "pass@8": 0.9929087339243897, + "pass@16": 0.9957320751066778, + "pass@32": 0.9983784320725285, + "pass@64": 0.9998225387344413, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9698804260450161, + "pass@2": 0.9826984486163508, + "pass@4": 0.9890233779057358, + "pass@8": 0.9935898434710003, + "pass@16": 0.9970979034218608, + "pass@32": 0.9992379626726149, + "pass@64": 0.9999533139874329, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9184523809523809, + "pass@2": 0.9463668603924509, + "pass@4": 0.9613165776896933, + "pass@8": 0.9696770133898424, + "pass@16": 0.9745358856156271, + "pass@32": 0.977831319059118, + "pass@64": 0.9809549621060872, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8733275993883792, + "pass@2": 0.9091482006549643, + "pass@4": 0.9321290414959594, + "pass@8": 0.9474101864198438, + "pass@16": 0.9582113860858212, + "pass@32": 0.9667417948856186, + "pass@64": 0.9751459929525179, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7013647762345679, + "pass@2": 0.7794614865850105, + "pass@4": 0.8347370620917749, + "pass@8": 0.8741468109649858, + "pass@16": 0.9042245400450407, + "pass@32": 0.9270647620892849, + "pass@64": 0.9446597552973784, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.6266459569732937, + "pass@2": 0.7149071824108042, + "pass@4": 0.777194388164387, + "pass@8": 0.8208584828195092, + "pass@16": 0.854085616211117, + "pass@32": 0.8785521084993064, + "pass@64": 0.8946411994274263, + "pass@128": 0.9050445103857567 + }, + "15": { + "pass@1": 0.5228105709876543, + "pass@2": 0.6264497970739771, + "pass@4": 0.7098052864919656, + "pass@8": 0.773887951251961, + "pass@16": 0.8196673549059977, + "pass@32": 0.8508791959747911, + "pass@64": 0.8725903607940296, + "pass@128": 0.8888888888888888 + }, + "16": { + "pass@1": 0.39115505162241887, + "pass@2": 0.49121359138735954, + "pass@4": 0.5756422308384014, + "pass@8": 0.6481636557709216, + "pass@16": 0.7119062076422585, + "pass@32": 0.7692503507300534, + "pass@64": 0.8199119593740152, + "pass@128": 0.8613569321533924 + }, + "17": { + "pass@1": 0.2642601656626506, + "pass@2": 0.3532293544255763, + "pass@4": 0.4450892874083509, + "pass@8": 0.5301218040712592, + "pass@16": 0.6015799792258167, + "pass@32": 0.6594386706097841, + "pass@64": 0.71061255652067, + "pass@128": 0.7560240963855421 + }, + "18": { + "pass@1": 0.26895161290322583, + "pass@2": 0.3575180181610362, + "pass@4": 0.4426478966338885, + "pass@8": 0.5166340111876507, + "pass@16": 0.5783399405568437, + "pass@32": 0.6329287740303212, + "pass@64": 0.6811106058382247, + "pass@128": 0.7193548387096774 + }, + "19": { + "pass@1": 0.19946657854984895, + "pass@2": 0.27503003306610846, + "pass@4": 0.35067128164870626, + "pass@8": 0.4218210302394642, + "pass@16": 0.48489409655088384, + "pass@32": 0.5400624117409185, + "pass@64": 0.5910281024631786, + "pass@128": 0.6435045317220544 + }, + "20": { + "pass@1": 0.17666103603603603, + "pass@2": 0.25825345522688054, + "pass@4": 0.3461097191116876, + "pass@8": 0.43004937374319013, + "pass@16": 0.5014484489893055, + "pass@32": 0.5554789580885844, + "pass@64": 0.5965143125091258, + "pass@128": 0.6306306306306306 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch4-RL-op11-14/composition/graphs/composition_score_report.jsonl b/composition/Qwen3-100M-epoch4-RL-op11-14/composition/graphs/composition_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..55f1ab2447d548d909c299a6e5ed7498d3df7d2c --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op11-14/composition/graphs/composition_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/Qwen3-100M-epoch4-RL-op11-14/composition", "results_dir": "results/composition/Qwen3-100M-epoch4-RL-op11-14/composition/graphs", "per_file": [{"file": "results/composition/Qwen3-100M-epoch4-RL-op11-14/composition/hf_model_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1025626, "value_mismatch": 6894, "dependency_mismatch": 124913, "answer_mismatch": 6117, "extra_nodes": 9296, "missing_nodes": 8061, "perfect_with_extra": 285, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 338310, "dependency_mismatch": 40379, "value_mismatch": 1920, "answer_mismatch": 1595, "extra_nodes": 2936, "perfect_with_extra": 148, "missing_nodes": 2646}, "teachers_in_school": {"total": 390656, "perfect": 347034, "dependency_mismatch": 43071, "value_mismatch": 2554, "answer_mismatch": 2352, "extra_nodes": 3464, "missing_nodes": 2896, "perfect_with_extra": 92}, "movie_festival_awards": {"total": 382336, "perfect": 340282, "dependency_mismatch": 41463, "value_mismatch": 2420, "answer_mismatch": 2170, "extra_nodes": 2896, "missing_nodes": 2519, "perfect_with_extra": 45}}, "results_path": "results/composition/Qwen3-100M-epoch4-RL-op11-14/composition/graphs/hf_model_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/Qwen3-100M-epoch4-RL-op11-14/composition/hf_model_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 478960, "value_mismatch": 503206, "dependency_mismatch": 729045, "answer_mismatch": 393839, "extra_nodes": 52878, "missing_nodes": 255638, "perfect_with_extra": 290, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 168671, "dependency_mismatch": 245690, "answer_mismatch": 131003, "perfect": 160967, "extra_nodes": 19992, "missing_nodes": 85266, "perfect_with_extra": 68}, "movie_festival_awards": {"total": 418816, "perfect": 154516, "value_mismatch": 169445, "dependency_mismatch": 240313, "answer_mismatch": 133168, "extra_nodes": 17869, "missing_nodes": 84865, "perfect_with_extra": 210}, "teachers_in_school": {"total": 430464, "perfect": 163477, "dependency_mismatch": 243042, "value_mismatch": 165090, "answer_mismatch": 129668, "missing_nodes": 85507, "extra_nodes": 15017, "perfect_with_extra": 12}}, "results_path": "results/composition/Qwen3-100M-epoch4-RL-op11-14/composition/graphs/hf_model_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1504586, "value_mismatch": 510100, "dependency_mismatch": 853958, "answer_mismatch": 399956, "extra_nodes": 62174, "missing_nodes": 263699, "perfect_with_extra": 575, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/Qwen3-100M-epoch4-RL-op11-14/composition/graphs/composition_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 499277, "dependency_mismatch": 286069, "value_mismatch": 170591, "answer_mismatch": 132598, "extra_nodes": 22928, "perfect_with_extra": 216, "missing_nodes": 87912}, "teachers_in_school": {"total": 821120, "perfect": 510511, "dependency_mismatch": 286113, "value_mismatch": 167644, "answer_mismatch": 132020, "extra_nodes": 18481, "missing_nodes": 88403, "perfect_with_extra": 104}, "movie_festival_awards": {"total": 801152, "perfect": 494798, "dependency_mismatch": 281776, "value_mismatch": 171865, "answer_mismatch": 135338, "extra_nodes": 20765, "missing_nodes": 87384, "perfect_with_extra": 255}}}} diff --git a/composition/Qwen3-100M-epoch4-RL-op11-14/composition/graphs/results.json b/composition/Qwen3-100M-epoch4-RL-op11-14/composition/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..95b3448a337f9b0d8edfd597540d157b99f1131e --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op11-14/composition/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "hf_model_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.996875, + "pass@4": 0.996875, + "pass@8": 0.996875, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9619047619047619, + "pass@2": 0.9841269841269841, + "pass@4": 0.9968253968253968, + "pass@8": 0.9968253968253968, + "pass@16": 0.9968253968253968, + "pass@32": 0.9968253968253968, + "pass@64": 0.9968253968253968, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9790419161676647, + "pass@2": 0.9820359281437125, + "pass@4": 0.9940119760479041, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.878698224852071, + "pass@2": 0.9112426035502958, + "pass@4": 0.9585798816568047, + "pass@8": 0.9911242603550295, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8205128205128205, + "pass@2": 0.9038461538461539, + "pass@4": 0.9519230769230769, + "pass@8": 0.9839743589743589, + "pass@16": 0.9935897435897436, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7549295774647887, + "pass@2": 0.8112676056338028, + "pass@4": 0.895774647887324, + "pass@8": 0.9577464788732394, + "pass@16": 0.9859154929577465, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6489675516224189, + "pass@2": 0.7315634218289085, + "pass@4": 0.8377581120943953, + "pass@8": 0.9174041297935103, + "pass@16": 0.9380530973451328, + "pass@32": 0.9528023598820059, + "pass@64": 0.9646017699115044, + "pass@128": 0.9705014749262537 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9619047619047619, + "6": 0.9790419161676647, + "7": 0.878698224852071, + "8": 0.8205128205128205, + "9": 0.7549295774647887, + "10": 0.6489675516224189 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9841269841269841, + "6": 0.9820359281437125, + "7": 0.9112426035502958, + "8": 0.9038461538461539, + "9": 0.8112676056338028, + "10": 0.7315634218289085 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9968253968253968, + "6": 0.9940119760479041, + "7": 0.9585798816568047, + "8": 0.9519230769230769, + "9": 0.895774647887324, + "10": 0.8377581120943953 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9911242603550295, + "8": 0.9839743589743589, + "9": 0.9577464788732394, + "10": 0.9174041297935103 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 0.9935897435897436, + "9": 0.9859154929577465, + "10": 0.9380530973451328 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9528023598820059 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9646017699115044 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9705014749262537 + } + }, + "total_pass_at_k": { + "pass@1": 0.8905775075987842, + "pass@2": 0.9223235393448159, + "pass@4": 0.9577845322526174, + "pass@8": 0.9821006416751098, + "pass@16": 0.9902060114826072, + "pass@32": 0.993920972644377, + "pass@64": 0.9956095913542722, + "pass@128": 0.9962850388382303 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 0.9941520467836257, + "pass@4": 0.9970760233918129, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9647058823529412, + "pass@2": 0.9823529411764705, + "pass@4": 0.9941176470588236, + "pass@8": 0.9970588235294118, + "pass@16": 0.9970588235294118, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9716088328075709, + "pass@2": 0.9747634069400631, + "pass@4": 0.9905362776025236, + "pass@8": 0.9905362776025236, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8575949367088608, + "pass@2": 0.9177215189873418, + "pass@4": 0.9525316455696202, + "pass@8": 0.9746835443037974, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8258258258258259, + "pass@2": 0.8798798798798799, + "pass@4": 0.948948948948949, + "pass@8": 0.975975975975976, + "pass@16": 0.990990990990991, + "pass@32": 0.990990990990991, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.77491961414791, + "pass@2": 0.8456591639871383, + "pass@4": 0.8938906752411575, + "pass@8": 0.9517684887459807, + "pass@16": 0.977491961414791, + "pass@32": 0.9903536977491961, + "pass@64": 0.9967845659163987, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.582089552238806, + "pass@2": 0.6507462686567164, + "pass@4": 0.7343283582089553, + "pass@8": 0.8567164179104477, + "pass@16": 0.9134328358208955, + "pass@32": 0.9522388059701492, + "pass@64": 0.9641791044776119, + "pass@128": 0.9701492537313433 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.9647058823529412, + "6": 0.9716088328075709, + "7": 0.8575949367088608, + "8": 0.8258258258258259, + "9": 0.77491961414791, + "10": 0.582089552238806 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9823529411764705, + "6": 0.9747634069400631, + "7": 0.9177215189873418, + "8": 0.8798798798798799, + "9": 0.8456591639871383, + "10": 0.6507462686567164 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9941176470588236, + "6": 0.9905362776025236, + "7": 0.9525316455696202, + "8": 0.948948948948949, + "9": 0.8938906752411575, + "10": 0.7343283582089553 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9905362776025236, + "7": 0.9746835443037974, + "8": 0.975975975975976, + "9": 0.9517684887459807, + "10": 0.8567164179104477 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 0.990990990990991, + "9": 0.977491961414791, + "10": 0.9134328358208955 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.990990990990991, + "9": 0.9903536977491961, + "10": 0.9522388059701492 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 0.9967845659163987, + "10": 0.9641791044776119 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9701492537313433 + } + }, + "total_pass_at_k": { + "pass@1": 0.8868429862738534, + "pass@2": 0.9173083361232005, + "pass@4": 0.9464345497154335, + "pass@8": 0.9722129226648811, + "pass@16": 0.9866086374288584, + "pass@32": 0.9926347505858721, + "pass@64": 0.9953130231001004, + "pass@128": 0.9963173752929361 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9792899408284024, + "pass@2": 0.9881656804733728, + "pass@4": 0.9940828402366864, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9594202898550724, + "pass@2": 0.9826086956521739, + "pass@4": 0.9942028985507246, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9770773638968482, + "pass@2": 0.9799426934097422, + "pass@4": 0.997134670487106, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8554913294797688, + "pass@2": 0.8872832369942196, + "pass@4": 0.9479768786127167, + "pass@8": 0.9797687861271677, + "pass@16": 0.9942196531791907, + "pass@32": 0.9971098265895953, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8112676056338028, + "pass@2": 0.8788732394366198, + "pass@4": 0.9492957746478873, + "pass@8": 0.9802816901408451, + "pass@16": 0.9971830985915493, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7335329341317365, + "pass@2": 0.8143712574850299, + "pass@4": 0.8862275449101796, + "pass@8": 0.9431137724550899, + "pass@16": 0.9790419161676647, + "pass@32": 0.9940119760479041, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6687116564417178, + "pass@2": 0.7300613496932515, + "pass@4": 0.8159509202453987, + "pass@8": 0.8742331288343558, + "pass@16": 0.9325153374233128, + "pass@32": 0.950920245398773, + "pass@64": 0.9601226993865031, + "pass@128": 0.9662576687116564 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9792899408284024, + "5": 0.9594202898550724, + "6": 0.9770773638968482, + "7": 0.8554913294797688, + "8": 0.8112676056338028, + "9": 0.7335329341317365, + "10": 0.6687116564417178 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9881656804733728, + "5": 0.9826086956521739, + "6": 0.9799426934097422, + "7": 0.8872832369942196, + "8": 0.8788732394366198, + "9": 0.8143712574850299, + "10": 0.7300613496932515 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9940828402366864, + "5": 0.9942028985507246, + "6": 0.997134670487106, + "7": 0.9479768786127167, + "8": 0.9492957746478873, + "9": 0.8862275449101796, + "10": 0.8159509202453987 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9797687861271677, + "8": 0.9802816901408451, + "9": 0.9431137724550899, + "10": 0.8742331288343558 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9942196531791907, + "8": 0.9971830985915493, + "9": 0.9790419161676647, + "10": 0.9325153374233128 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 1.0, + "9": 0.9940119760479041, + "10": 0.950920245398773 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9601226993865031 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9662576687116564 + } + }, + "total_pass_at_k": { + "pass@1": 0.8876146788990825, + "pass@2": 0.9184141546526867, + "pass@4": 0.9544560943643512, + "pass@8": 0.9757536041939712, + "pass@16": 0.9895150720838795, + "pass@32": 0.9937745740498034, + "pass@64": 0.995740498034076, + "pass@128": 0.9963958060288336 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.987, + "pass@2": 0.993, + "pass@4": 0.996, + "pass@8": 0.999, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.962, + "pass@2": 0.983, + "pass@4": 0.995, + "pass@8": 0.998, + "pass@16": 0.998, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.976, + "pass@2": 0.979, + "pass@4": 0.994, + "pass@8": 0.997, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.864, + "pass@2": 0.905, + "pass@4": 0.953, + "pass@8": 0.982, + "pass@16": 0.998, + "pass@32": 0.999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.819, + "pass@2": 0.887, + "pass@4": 0.95, + "pass@8": 0.98, + "pass@16": 0.994, + "pass@32": 0.997, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.754, + "pass@2": 0.823, + "pass@4": 0.892, + "pass@8": 0.951, + "pass@16": 0.981, + "pass@32": 0.994, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.633, + "pass@2": 0.704, + "pass@4": 0.796, + "pass@8": 0.883, + "pass@16": 0.928, + "pass@32": 0.952, + "pass@64": 0.963, + "pass@128": 0.969 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.987, + "5": 0.962, + "6": 0.976, + "7": 0.864, + "8": 0.819, + "9": 0.754, + "10": 0.633 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.993, + "5": 0.983, + "6": 0.979, + "7": 0.905, + "8": 0.887, + "9": 0.823, + "10": 0.704 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996, + "5": 0.995, + "6": 0.994, + "7": 0.953, + "8": 0.95, + "9": 0.892, + "10": 0.796 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.998, + "6": 0.997, + "7": 0.982, + "8": 0.98, + "9": 0.951, + "10": 0.883 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 1.0, + "7": 0.998, + "8": 0.994, + "9": 0.981, + "10": 0.928 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.999, + "8": 0.997, + "9": 0.994, + "10": 0.952 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.999, + "10": 0.963 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 1.0, + "10": 0.969 + } + }, + "total_pass_at_k": { + "pass@1": 0.8883333333333333, + "pass@2": 0.9193333333333333, + "pass@4": 0.9528888888888889, + "pass@8": 0.9766666666666667, + "pass@16": 0.9887777777777778, + "pass@32": 0.9934444444444445, + "pass@64": 0.9955555555555555, + "pass@128": 0.9963333333333333 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "hf_model_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6118980169971672, + "pass@2": 0.6883852691218131, + "pass@4": 0.7677053824362606, + "pass@8": 0.8356940509915014, + "pass@16": 0.9036827195467422, + "pass@32": 0.9206798866855525, + "pass@64": 0.9405099150141643, + "pass@128": 0.9461756373937678 + }, + "12": { + "pass@1": 0.6384180790960452, + "pass@2": 0.7033898305084746, + "pass@4": 0.7429378531073446, + "pass@8": 0.8135593220338984, + "pass@16": 0.844632768361582, + "pass@32": 0.8531073446327684, + "pass@64": 0.8757062146892656, + "pass@128": 0.8813559322033898 + }, + "13": { + "pass@1": 0.5297805642633229, + "pass@2": 0.5893416927899686, + "pass@4": 0.6489028213166145, + "pass@8": 0.7053291536050157, + "pass@16": 0.7617554858934169, + "pass@32": 0.7962382445141066, + "pass@64": 0.8181818181818182, + "pass@128": 0.8432601880877743 + }, + "14": { + "pass@1": 0.5077881619937694, + "pass@2": 0.5389408099688473, + "pass@4": 0.5794392523364486, + "pass@8": 0.6261682242990654, + "pass@16": 0.7009345794392523, + "pass@32": 0.7694704049844237, + "pass@64": 0.8161993769470405, + "pass@128": 0.838006230529595 + }, + "15": { + "pass@1": 0.43352601156069365, + "pass@2": 0.4884393063583815, + "pass@4": 0.5289017341040463, + "pass@8": 0.5838150289017341, + "pass@16": 0.653179190751445, + "pass@32": 0.7456647398843931, + "pass@64": 0.8092485549132948, + "pass@128": 0.8815028901734104 + }, + "16": { + "pass@1": 0.33225806451612905, + "pass@2": 0.3870967741935484, + "pass@4": 0.4483870967741935, + "pass@8": 0.5129032258064516, + "pass@16": 0.5709677419354838, + "pass@32": 0.5903225806451613, + "pass@64": 0.6258064516129033, + "pass@128": 0.6774193548387096 + }, + "17": { + "pass@1": 0.3063583815028902, + "pass@2": 0.3670520231213873, + "pass@4": 0.4277456647398844, + "pass@8": 0.47398843930635837, + "pass@16": 0.5, + "pass@32": 0.5549132947976878, + "pass@64": 0.6011560693641619, + "pass@128": 0.6647398843930635 + }, + "18": { + "pass@1": 0.2257142857142857, + "pass@2": 0.26, + "pass@4": 0.29714285714285715, + "pass@8": 0.34, + "pass@16": 0.38285714285714284, + "pass@32": 0.43142857142857144, + "pass@64": 0.49142857142857144, + "pass@128": 0.5285714285714286 + }, + "19": { + "pass@1": 0.11411411411411411, + "pass@2": 0.15915915915915915, + "pass@4": 0.19519519519519518, + "pass@8": 0.21921921921921922, + "pass@16": 0.25225225225225223, + "pass@32": 0.2882882882882883, + "pass@64": 0.32732732732732733, + "pass@128": 0.3483483483483483 + }, + "20": { + "pass@1": 0.08408408408408409, + "pass@2": 0.12312312312312312, + "pass@4": 0.16216216216216217, + "pass@8": 0.1981981981981982, + "pass@16": 0.24324324324324326, + "pass@32": 0.26126126126126126, + "pass@64": 0.30930930930930933, + "pass@128": 0.3483483483483483 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6118980169971672, + "12": 0.6384180790960452, + "13": 0.5297805642633229, + "14": 0.5077881619937694, + "15": 0.43352601156069365, + "16": 0.33225806451612905, + "17": 0.3063583815028902, + "18": 0.2257142857142857, + "19": 0.11411411411411411, + "20": 0.08408408408408409 + }, + "pass@2": { + "11": 0.6883852691218131, + "12": 0.7033898305084746, + "13": 0.5893416927899686, + "14": 0.5389408099688473, + "15": 0.4884393063583815, + "16": 0.3870967741935484, + "17": 0.3670520231213873, + "18": 0.26, + "19": 0.15915915915915915, + "20": 0.12312312312312312 + }, + "pass@4": { + "11": 0.7677053824362606, + "12": 0.7429378531073446, + "13": 0.6489028213166145, + "14": 0.5794392523364486, + "15": 0.5289017341040463, + "16": 0.4483870967741935, + "17": 0.4277456647398844, + "18": 0.29714285714285715, + "19": 0.19519519519519518, + "20": 0.16216216216216217 + }, + "pass@8": { + "11": 0.8356940509915014, + "12": 0.8135593220338984, + "13": 0.7053291536050157, + "14": 0.6261682242990654, + "15": 0.5838150289017341, + "16": 0.5129032258064516, + "17": 0.47398843930635837, + "18": 0.34, + "19": 0.21921921921921922, + "20": 0.1981981981981982 + }, + "pass@16": { + "11": 0.9036827195467422, + "12": 0.844632768361582, + "13": 0.7617554858934169, + "14": 0.7009345794392523, + "15": 0.653179190751445, + "16": 0.5709677419354838, + "17": 0.5, + "18": 0.38285714285714284, + "19": 0.25225225225225223, + "20": 0.24324324324324326 + }, + "pass@32": { + "11": 0.9206798866855525, + "12": 0.8531073446327684, + "13": 0.7962382445141066, + "14": 0.7694704049844237, + "15": 0.7456647398843931, + "16": 0.5903225806451613, + "17": 0.5549132947976878, + "18": 0.43142857142857144, + "19": 0.2882882882882883, + "20": 0.26126126126126126 + }, + "pass@64": { + "11": 0.9405099150141643, + "12": 0.8757062146892656, + "13": 0.8181818181818182, + "14": 0.8161993769470405, + "15": 0.8092485549132948, + "16": 0.6258064516129033, + "17": 0.6011560693641619, + "18": 0.49142857142857144, + "19": 0.32732732732732733, + "20": 0.30930930930930933 + }, + "pass@128": { + "11": 0.9461756373937678, + "12": 0.8813559322033898, + "13": 0.8432601880877743, + "14": 0.838006230529595, + "15": 0.8815028901734104, + "16": 0.6774193548387096, + "17": 0.6647398843930635, + "18": 0.5285714285714286, + "19": 0.3483483483483483, + "20": 0.3483483483483483 + } + }, + "total_pass_at_k": { + "pass@1": 0.37979197622585437, + "pass@2": 0.4320950965824666, + "pass@4": 0.4814264487369985, + "pass@8": 0.5325408618127786, + "pass@16": 0.5827637444279347, + "pass@32": 0.6225854383358098, + "pass@64": 0.663001485884101, + "pass@128": 0.6971768202080237 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5968253968253968, + "pass@2": 0.6793650793650794, + "pass@4": 0.7523809523809524, + "pass@8": 0.8507936507936508, + "pass@16": 0.9333333333333333, + "pass@32": 0.9587301587301588, + "pass@64": 0.9650793650793651, + "pass@128": 0.9777777777777777 + }, + "12": { + "pass@1": 0.6299694189602446, + "pass@2": 0.6850152905198776, + "pass@4": 0.7553516819571865, + "pass@8": 0.8256880733944955, + "pass@16": 0.8654434250764526, + "pass@32": 0.8776758409785933, + "pass@64": 0.8960244648318043, + "pass@128": 0.9021406727828746 + }, + "13": { + "pass@1": 0.5524691358024691, + "pass@2": 0.6018518518518519, + "pass@4": 0.6697530864197531, + "pass@8": 0.7129629629629629, + "pass@16": 0.7376543209876543, + "pass@32": 0.7623456790123457, + "pass@64": 0.7901234567901234, + "pass@128": 0.8055555555555556 + }, + "14": { + "pass@1": 0.4629080118694362, + "pass@2": 0.516320474777448, + "pass@4": 0.5934718100890207, + "pass@8": 0.658753709198813, + "pass@16": 0.712166172106825, + "pass@32": 0.7774480712166172, + "pass@64": 0.8160237388724035, + "pass@128": 0.8486646884272997 + }, + "15": { + "pass@1": 0.41975308641975306, + "pass@2": 0.48148148148148145, + "pass@4": 0.5277777777777778, + "pass@8": 0.5709876543209876, + "pass@16": 0.6265432098765432, + "pass@32": 0.7407407407407407, + "pass@64": 0.7901234567901234, + "pass@128": 0.8425925925925926 + }, + "16": { + "pass@1": 0.3480825958702065, + "pass@2": 0.4247787610619469, + "pass@4": 0.4778761061946903, + "pass@8": 0.5309734513274337, + "pass@16": 0.56047197640118, + "pass@32": 0.6106194690265486, + "pass@64": 0.6519174041297935, + "pass@128": 0.6932153392330384 + }, + "17": { + "pass@1": 0.25301204819277107, + "pass@2": 0.3042168674698795, + "pass@4": 0.3493975903614458, + "pass@8": 0.39457831325301207, + "pass@16": 0.4457831325301205, + "pass@32": 0.5030120481927711, + "pass@64": 0.5512048192771084, + "pass@128": 0.6024096385542169 + }, + "18": { + "pass@1": 0.22903225806451613, + "pass@2": 0.2645161290322581, + "pass@4": 0.3096774193548387, + "pass@8": 0.34838709677419355, + "pass@16": 0.38387096774193546, + "pass@32": 0.43870967741935485, + "pass@64": 0.4967741935483871, + "pass@128": 0.5516129032258065 + }, + "19": { + "pass@1": 0.11178247734138973, + "pass@2": 0.13897280966767372, + "pass@4": 0.1691842900302115, + "pass@8": 0.22658610271903323, + "pass@16": 0.2628398791540785, + "pass@32": 0.283987915407855, + "pass@64": 0.30513595166163143, + "pass@128": 0.33534743202416917 + }, + "20": { + "pass@1": 0.0960960960960961, + "pass@2": 0.11411411411411411, + "pass@4": 0.14714714714714713, + "pass@8": 0.16216216216216217, + "pass@16": 0.18618618618618618, + "pass@32": 0.21921921921921922, + "pass@64": 0.24924924924924924, + "pass@128": 0.27627627627627627 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5968253968253968, + "12": 0.6299694189602446, + "13": 0.5524691358024691, + "14": 0.4629080118694362, + "15": 0.41975308641975306, + "16": 0.3480825958702065, + "17": 0.25301204819277107, + "18": 0.22903225806451613, + "19": 0.11178247734138973, + "20": 0.0960960960960961 + }, + "pass@2": { + "11": 0.6793650793650794, + "12": 0.6850152905198776, + "13": 0.6018518518518519, + "14": 0.516320474777448, + "15": 0.48148148148148145, + "16": 0.4247787610619469, + "17": 0.3042168674698795, + "18": 0.2645161290322581, + "19": 0.13897280966767372, + "20": 0.11411411411411411 + }, + "pass@4": { + "11": 0.7523809523809524, + "12": 0.7553516819571865, + "13": 0.6697530864197531, + "14": 0.5934718100890207, + "15": 0.5277777777777778, + "16": 0.4778761061946903, + "17": 0.3493975903614458, + "18": 0.3096774193548387, + "19": 0.1691842900302115, + "20": 0.14714714714714713 + }, + "pass@8": { + "11": 0.8507936507936508, + "12": 0.8256880733944955, + "13": 0.7129629629629629, + "14": 0.658753709198813, + "15": 0.5709876543209876, + "16": 0.5309734513274337, + "17": 0.39457831325301207, + "18": 0.34838709677419355, + "19": 0.22658610271903323, + "20": 0.16216216216216217 + }, + "pass@16": { + "11": 0.9333333333333333, + "12": 0.8654434250764526, + "13": 0.7376543209876543, + "14": 0.712166172106825, + "15": 0.6265432098765432, + "16": 0.56047197640118, + "17": 0.4457831325301205, + "18": 0.38387096774193546, + "19": 0.2628398791540785, + "20": 0.18618618618618618 + }, + "pass@32": { + "11": 0.9587301587301588, + "12": 0.8776758409785933, + "13": 0.7623456790123457, + "14": 0.7774480712166172, + "15": 0.7407407407407407, + "16": 0.6106194690265486, + "17": 0.5030120481927711, + "18": 0.43870967741935485, + "19": 0.283987915407855, + "20": 0.21921921921921922 + }, + "pass@64": { + "11": 0.9650793650793651, + "12": 0.8960244648318043, + "13": 0.7901234567901234, + "14": 0.8160237388724035, + "15": 0.7901234567901234, + "16": 0.6519174041297935, + "17": 0.5512048192771084, + "18": 0.4967741935483871, + "19": 0.30513595166163143, + "20": 0.24924924924924924 + }, + "pass@128": { + "11": 0.9777777777777777, + "12": 0.9021406727828746, + "13": 0.8055555555555556, + "14": 0.8486646884272997, + "15": 0.8425925925925926, + "16": 0.6932153392330384, + "17": 0.6024096385542169, + "18": 0.5516129032258065, + "19": 0.33534743202416917, + "20": 0.27627627627627627 + } + }, + "total_pass_at_k": { + "pass@1": 0.3688875305623472, + "pass@2": 0.41992665036674814, + "pass@4": 0.47402200488997553, + "pass@8": 0.5268948655256723, + "pass@16": 0.5699877750611247, + "pass@32": 0.6158312958435208, + "pass@64": 0.6497555012224939, + "pass@128": 0.6821515892420538 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6506024096385542, + "pass@2": 0.7349397590361446, + "pass@4": 0.8042168674698795, + "pass@8": 0.858433734939759, + "pass@16": 0.9066265060240963, + "pass@32": 0.9337349397590361, + "pass@64": 0.9608433734939759, + "pass@128": 0.9728915662650602 + }, + "12": { + "pass@1": 0.5924764890282131, + "pass@2": 0.6394984326018809, + "pass@4": 0.7178683385579937, + "pass@8": 0.774294670846395, + "pass@16": 0.8181818181818182, + "pass@32": 0.8463949843260188, + "pass@64": 0.8589341692789969, + "pass@128": 0.877742946708464 + }, + "13": { + "pass@1": 0.5742296918767507, + "pass@2": 0.6190476190476191, + "pass@4": 0.7030812324929971, + "pass@8": 0.773109243697479, + "pass@16": 0.8291316526610645, + "pass@32": 0.8487394957983193, + "pass@64": 0.8599439775910365, + "pass@128": 0.8739495798319328 + }, + "14": { + "pass@1": 0.4853801169590643, + "pass@2": 0.52046783625731, + "pass@4": 0.5906432748538012, + "pass@8": 0.6549707602339181, + "pass@16": 0.7251461988304093, + "pass@32": 0.7865497076023392, + "pass@64": 0.8216374269005848, + "pass@128": 0.8625730994152047 + }, + "15": { + "pass@1": 0.4666666666666667, + "pass@2": 0.5181818181818182, + "pass@4": 0.5787878787878787, + "pass@8": 0.6272727272727273, + "pass@16": 0.6848484848484848, + "pass@32": 0.7424242424242424, + "pass@64": 0.8121212121212121, + "pass@128": 0.8636363636363636 + }, + "16": { + "pass@1": 0.38461538461538464, + "pass@2": 0.4301994301994302, + "pass@4": 0.46438746438746437, + "pass@8": 0.5270655270655271, + "pass@16": 0.5726495726495726, + "pass@32": 0.6381766381766382, + "pass@64": 0.6866096866096866, + "pass@128": 0.717948717948718 + }, + "17": { + "pass@1": 0.27639751552795033, + "pass@2": 0.32919254658385094, + "pass@4": 0.35403726708074534, + "pass@8": 0.39751552795031053, + "pass@16": 0.4440993788819876, + "pass@32": 0.5217391304347826, + "pass@64": 0.577639751552795, + "pass@128": 0.6428571428571429 + }, + "18": { + "pass@1": 0.21470588235294116, + "pass@2": 0.25588235294117645, + "pass@4": 0.3, + "pass@8": 0.32941176470588235, + "pass@16": 0.35294117647058826, + "pass@32": 0.4088235294117647, + "pass@64": 0.4588235294117647, + "pass@128": 0.5058823529411764 + }, + "19": { + "pass@1": 0.13690476190476192, + "pass@2": 0.1636904761904762, + "pass@4": 0.20238095238095238, + "pass@8": 0.23511904761904762, + "pass@16": 0.2857142857142857, + "pass@32": 0.3125, + "pass@64": 0.34523809523809523, + "pass@128": 0.37202380952380953 + }, + "20": { + "pass@1": 0.08982035928143713, + "pass@2": 0.11077844311377245, + "pass@4": 0.1497005988023952, + "pass@8": 0.17664670658682635, + "pass@16": 0.20359281437125748, + "pass@32": 0.23652694610778444, + "pass@64": 0.26646706586826346, + "pass@128": 0.2934131736526946 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6506024096385542, + "12": 0.5924764890282131, + "13": 0.5742296918767507, + "14": 0.4853801169590643, + "15": 0.4666666666666667, + "16": 0.38461538461538464, + "17": 0.27639751552795033, + "18": 0.21470588235294116, + "19": 0.13690476190476192, + "20": 0.08982035928143713 + }, + "pass@2": { + "11": 0.7349397590361446, + "12": 0.6394984326018809, + "13": 0.6190476190476191, + "14": 0.52046783625731, + "15": 0.5181818181818182, + "16": 0.4301994301994302, + "17": 0.32919254658385094, + "18": 0.25588235294117645, + "19": 0.1636904761904762, + "20": 0.11077844311377245 + }, + "pass@4": { + "11": 0.8042168674698795, + "12": 0.7178683385579937, + "13": 0.7030812324929971, + "14": 0.5906432748538012, + "15": 0.5787878787878787, + "16": 0.46438746438746437, + "17": 0.35403726708074534, + "18": 0.3, + "19": 0.20238095238095238, + "20": 0.1497005988023952 + }, + "pass@8": { + "11": 0.858433734939759, + "12": 0.774294670846395, + "13": 0.773109243697479, + "14": 0.6549707602339181, + "15": 0.6272727272727273, + "16": 0.5270655270655271, + "17": 0.39751552795031053, + "18": 0.32941176470588235, + "19": 0.23511904761904762, + "20": 0.17664670658682635 + }, + "pass@16": { + "11": 0.9066265060240963, + "12": 0.8181818181818182, + "13": 0.8291316526610645, + "14": 0.7251461988304093, + "15": 0.6848484848484848, + "16": 0.5726495726495726, + "17": 0.4440993788819876, + "18": 0.35294117647058826, + "19": 0.2857142857142857, + "20": 0.20359281437125748 + }, + "pass@32": { + "11": 0.9337349397590361, + "12": 0.8463949843260188, + "13": 0.8487394957983193, + "14": 0.7865497076023392, + "15": 0.7424242424242424, + "16": 0.6381766381766382, + "17": 0.5217391304347826, + "18": 0.4088235294117647, + "19": 0.3125, + "20": 0.23652694610778444 + }, + "pass@64": { + "11": 0.9608433734939759, + "12": 0.8589341692789969, + "13": 0.8599439775910365, + "14": 0.8216374269005848, + "15": 0.8121212121212121, + "16": 0.6866096866096866, + "17": 0.577639751552795, + "18": 0.4588235294117647, + "19": 0.34523809523809523, + "20": 0.26646706586826346 + }, + "pass@128": { + "11": 0.9728915662650602, + "12": 0.877742946708464, + "13": 0.8739495798319328, + "14": 0.8625730994152047, + "15": 0.8636363636363636, + "16": 0.717948717948718, + "17": 0.6428571428571429, + "18": 0.5058823529411764, + "19": 0.37202380952380953, + "20": 0.2934131736526946 + } + }, + "total_pass_at_k": { + "pass@1": 0.3874516800475766, + "pass@2": 0.43235206660719594, + "pass@4": 0.48676776687481416, + "pass@8": 0.535831103181683, + "pass@16": 0.5828129646149272, + "pass@32": 0.6280107047279215, + "pass@64": 0.6651798988997919, + "pass@128": 0.6984834968777877 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.62, + "pass@2": 0.701, + "pass@4": 0.775, + "pass@8": 0.848, + "pass@16": 0.914, + "pass@32": 0.937, + "pass@64": 0.955, + "pass@128": 0.965 + }, + "12": { + "pass@1": 0.621, + "pass@2": 0.677, + "pass@4": 0.739, + "pass@8": 0.805, + "pass@16": 0.843, + "pass@32": 0.859, + "pass@64": 0.877, + "pass@128": 0.887 + }, + "13": { + "pass@1": 0.553, + "pass@2": 0.604, + "pass@4": 0.675, + "pass@8": 0.732, + "pass@16": 0.778, + "pass@32": 0.804, + "pass@64": 0.824, + "pass@128": 0.842 + }, + "14": { + "pass@1": 0.485, + "pass@2": 0.525, + "pass@4": 0.588, + "pass@8": 0.647, + "pass@16": 0.713, + "pass@32": 0.778, + "pass@64": 0.818, + "pass@128": 0.85 + }, + "15": { + "pass@1": 0.44, + "pass@2": 0.496, + "pass@4": 0.545, + "pass@8": 0.594, + "pass@16": 0.655, + "pass@32": 0.743, + "pass@64": 0.804, + "pass@128": 0.863 + }, + "16": { + "pass@1": 0.356, + "pass@2": 0.415, + "pass@4": 0.464, + "pass@8": 0.524, + "pass@16": 0.568, + "pass@32": 0.614, + "pass@64": 0.656, + "pass@128": 0.697 + }, + "17": { + "pass@1": 0.279, + "pass@2": 0.334, + "pass@4": 0.378, + "pass@8": 0.423, + "pass@16": 0.464, + "pass@32": 0.527, + "pass@64": 0.577, + "pass@128": 0.637 + }, + "18": { + "pass@1": 0.223, + "pass@2": 0.26, + "pass@4": 0.302, + "pass@8": 0.339, + "pass@16": 0.373, + "pass@32": 0.426, + "pass@64": 0.482, + "pass@128": 0.528 + }, + "19": { + "pass@1": 0.121, + "pass@2": 0.154, + "pass@4": 0.189, + "pass@8": 0.227, + "pass@16": 0.267, + "pass@32": 0.295, + "pass@64": 0.326, + "pass@128": 0.352 + }, + "20": { + "pass@1": 0.09, + "pass@2": 0.116, + "pass@4": 0.153, + "pass@8": 0.179, + "pass@16": 0.211, + "pass@32": 0.239, + "pass@64": 0.275, + "pass@128": 0.306 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.62, + "12": 0.621, + "13": 0.553, + "14": 0.485, + "15": 0.44, + "16": 0.356, + "17": 0.279, + "18": 0.223, + "19": 0.121, + "20": 0.09 + }, + "pass@2": { + "11": 0.701, + "12": 0.677, + "13": 0.604, + "14": 0.525, + "15": 0.496, + "16": 0.415, + "17": 0.334, + "18": 0.26, + "19": 0.154, + "20": 0.116 + }, + "pass@4": { + "11": 0.775, + "12": 0.739, + "13": 0.675, + "14": 0.588, + "15": 0.545, + "16": 0.464, + "17": 0.378, + "18": 0.302, + "19": 0.189, + "20": 0.153 + }, + "pass@8": { + "11": 0.848, + "12": 0.805, + "13": 0.732, + "14": 0.647, + "15": 0.594, + "16": 0.524, + "17": 0.423, + "18": 0.339, + "19": 0.227, + "20": 0.179 + }, + "pass@16": { + "11": 0.914, + "12": 0.843, + "13": 0.778, + "14": 0.713, + "15": 0.655, + "16": 0.568, + "17": 0.464, + "18": 0.373, + "19": 0.267, + "20": 0.211 + }, + "pass@32": { + "11": 0.937, + "12": 0.859, + "13": 0.804, + "14": 0.778, + "15": 0.743, + "16": 0.614, + "17": 0.527, + "18": 0.426, + "19": 0.295, + "20": 0.239 + }, + "pass@64": { + "11": 0.955, + "12": 0.877, + "13": 0.824, + "14": 0.818, + "15": 0.804, + "16": 0.656, + "17": 0.577, + "18": 0.482, + "19": 0.326, + "20": 0.275 + }, + "pass@128": { + "11": 0.965, + "12": 0.887, + "13": 0.842, + "14": 0.85, + "15": 0.863, + "16": 0.697, + "17": 0.637, + "18": 0.528, + "19": 0.352, + "20": 0.306 + } + }, + "total_pass_at_k": { + "pass@1": 0.3788, + "pass@2": 0.4282, + "pass@4": 0.4808, + "pass@8": 0.5318, + "pass@16": 0.5786, + "pass@32": 0.6222, + "pass@64": 0.6594, + "pass@128": 0.6927 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/Qwen3-100M-epoch4-RL-op11-14/composition/hf_model_metrics.json b/composition/Qwen3-100M-epoch4-RL-op11-14/composition/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0fe466ce51fdd27a79c96a5b4fa7b6083114ea9e --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op11-14/composition/hf_model_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.9946901041666667, + "avg_response_len": 199.69106684027778, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9946901041666667, + "pass@2": 0.9961465578521432, + "pass@4": 0.9971940083322918, + "pass@8": 0.9979334699813251, + "pass@16": 0.9984288641924057, + "pass@32": 0.9987519232835602, + "pass@64": 0.9989922224197697, + "pass@128": 0.9992222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9630703125, + "pass@2": 0.971617249015748, + "pass@4": 0.9783617097862765, + "pass@8": 0.9833621198515833, + "pass@16": 0.9867538015919473, + "pass@32": 0.9889978617362172, + "pass@64": 0.9909396817980759, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9988515625, + "pass@2": 0.9994415600393702, + "pass@4": 0.9997261106111736, + "pass@8": 0.9999287673940046, + "pass@16": 0.9999959202789485, + "pass@32": 0.9999999941392413, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99859375, + "pass@2": 0.999097563976378, + "pass@4": 0.9993847810273716, + "pass@8": 0.9996419787260388, + "pass@16": 0.9998812336885909, + "pass@32": 0.9999901022858958, + "pass@64": 0.9999999879305072, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995046875, + "pass@2": 0.9971327509842518, + "pass@4": 0.9981651927259092, + "pass@8": 0.9987532396728784, + "pass@16": 0.9992837011149076, + "pass@32": 0.9997811580893737, + "pass@64": 0.999990332213311, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966640625, + "pass@2": 0.9980300196850395, + "pass@4": 0.9991082808398951, + "pass@8": 0.9997151241874243, + "pass@16": 0.9999451210572601, + "pass@32": 0.9999981933013141, + "pass@64": 0.9999999998360333, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9630703125, + "2": 1.0, + "3": 1.0, + "4": 0.999984375, + "5": 0.9988515625, + "6": 1.0, + "7": 0.99859375, + "8": 0.995046875, + "9": 0.9966640625 + }, + "per_op_avg_response_len": { + "10": 280.7720078125, + "2": 115.459953125, + "3": 144.460953125, + "4": 159.6328828125, + "5": 172.4409140625, + "6": 198.4999609375, + "7": 223.721234375, + "8": 243.211546875, + "9": 259.0201484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.7720078125, + "2": 115.459953125, + "3": 144.460953125, + "4": 159.6328828125, + "5": 172.4409140625, + "6": 198.4999609375, + "7": 223.721234375, + "8": 243.211546875, + "9": 259.0201484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 377413, + "answer_accuracy": 0.9957916455589328, + "avg_response_len": 190.16311529044242, + "resp_tokens_sum": 72073342, + "pass_at_k": { + "pass@1": 0.9957916455589328, + "pass@2": 0.9970020369794202, + "pass@4": 0.997782766230969, + "pass@8": 0.9983188993715499, + "pass@16": 0.9986629623472383, + "pass@32": 0.9988731854196147, + "pass@64": 0.9990719110949116, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9696718289085545, + "pass@2": 0.977735291385037, + "pass@4": 0.9829392385133277, + "pass@8": 0.9862529137910082, + "pass@16": 0.9884945156128231, + "pass@32": 0.9901631698480607, + "pass@64": 0.9918935956224192, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 0.9999996155265748, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9976934523809524, + "pass@2": 0.9983291151106111, + "pass@4": 0.9991311398575179, + "pass@8": 0.9997738647527661, + "pass@16": 0.9999870485045981, + "pass@32": 0.9999999813944173, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9983223157051282, + "pass@2": 0.999707011533414, + "pass@4": 0.9999820905800235, + "pass@8": 0.9999999321607189, + "pass@16": 0.9999999999997491, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9974911971830986, + "pass@2": 0.9979971858711323, + "pass@4": 0.9985849229233669, + "pass@8": 0.9993064141767372, + "pass@16": 0.9998463956015636, + "pass@32": 0.9999949107316761, + "pass@64": 0.9999999995381218, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 388304, + "answer_accuracy": 0.9939793577981652, + "avg_response_len": 197.754469405308, + "resp_tokens_sum": 77253970, + "pass_at_k": { + "pass@1": 0.9939793577981652, + "pass@2": 0.995537084885605, + "pass@4": 0.9966102919965935, + "pass@8": 0.9972568589652675, + "pass@16": 0.997632233697408, + "pass@32": 0.9978867368057548, + "pass@64": 0.9981144771018301, + "pass@128": 0.9983617300131061 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9559528374233128, + "pass@2": 0.9649733859475385, + "pass@4": 0.9714398099585708, + "pass@8": 0.975561770359765, + "pass@16": 0.9781992510202098, + "pass@32": 0.9802460689834376, + "pass@64": 0.982347841056068, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996376811594203, + "pass@2": 0.9999868053178136, + "pass@4": 0.9999999807089331, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9963421242774566, + "pass@2": 0.9974252281416413, + "pass@4": 0.9982221816745449, + "pass@8": 0.9989652564397086, + "pass@16": 0.9996567447647136, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9975792253521126, + "pass@2": 0.9988195907729843, + "pass@4": 0.9995380836198291, + "pass@8": 0.9999297766515117, + "pass@16": 0.9999987404291422, + "pass@32": 0.9999999998867772, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947136976047904, + "pass@2": 0.9973422986703758, + "pass@4": 0.9992345665811235, + "pass@8": 0.9999333742074483, + "pass@16": 0.9999995056016906, + "pass@32": 0.9999999999906032, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380166, + "answer_accuracy": 0.9943243639102779, + "avg_response_len": 211.11482308754603, + "resp_tokens_sum": 80716797, + "pass_at_k": { + "pass@1": 0.9943243639102779, + "pass@2": 0.9959212608310557, + "pass@4": 0.9972067937754019, + "pass@8": 0.9982427302413036, + "pass@16": 0.9990107696608616, + "pass@32": 0.9995157304296618, + "pass@64": 0.9998100732879506, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9633162313432836, + "pass@2": 0.9718915266188743, + "pass@4": 0.9804654623395956, + "pass@8": 0.9880275968332796, + "pass@16": 0.9933170297510806, + "pass@32": 0.9963353094600719, + "pass@64": 0.9983353931277541, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9991268382352941, + "pass@2": 0.9999189439555349, + "pass@4": 0.9999994359160987, + "pass@8": 0.9999999999908352, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9996291534810127, + "pass@2": 0.9999645700438552, + "pass@4": 0.9999997030632247, + "pass@8": 0.9999999999933531, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9892783408408409, + "pass@2": 0.9929225485327845, + "pass@4": 0.9949992816213291, + "pass@8": 0.9963309012836256, + "pass@16": 0.9978502950830038, + "pass@32": 0.9993428172059096, + "pass@64": 0.9999709676075407, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9978145096463023, + "pass@2": 0.9988060802086235, + "pass@4": 0.9995700577620724, + "pass@8": 0.999947267425707, + "pass@16": 0.9999994075490037, + "pass@32": 0.9999999999829826, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.6923140625, + "avg_response_len": 370.38072421875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.6923140625, + "pass@2": 0.7566085999015568, + "pass@4": 0.8051764815429241, + "pass@8": 0.841390031933904, + "pass@16": 0.8683821539726206, + "pass@32": 0.8888443014438344, + "pass@64": 0.9050825874826325, + "pass@128": 0.9189 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9720546875, + "pass@2": 0.978343503937008, + "pass@4": 0.9824010070303709, + "pass@8": 0.985353171884976, + "pass@16": 0.9872199861395465, + "pass@32": 0.9886904652675699, + "pass@64": 0.9903527652246648, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.9254609375, + "pass@2": 0.938867864173229, + "pass@4": 0.9472267765279342, + "pass@8": 0.953345147771704, + "pass@16": 0.9582913864816682, + "pass@32": 0.962623772416308, + "pass@64": 0.9667177032690246, + "pass@128": 0.971 + }, + "13": { + "pass@1": 0.8862421875, + "pass@2": 0.9129928641732293, + "pass@4": 0.928584318522685, + "pass@8": 0.9392455741111944, + "pass@16": 0.9471919321150946, + "pass@32": 0.953261156660872, + "pass@64": 0.9587720174315262, + "pass@128": 0.966 + }, + "14": { + "pass@1": 0.8654296875, + "pass@2": 0.8976216781496069, + "pass@4": 0.9171189835020619, + "pass@8": 0.9308024546679728, + "pass@16": 0.9411124973730856, + "pass@32": 0.9494987963075991, + "pass@64": 0.9563614168302956, + "pass@128": 0.962 + }, + "15": { + "pass@1": 0.8511796875, + "pass@2": 0.9029140009842532, + "pass@4": 0.9313769341019875, + "pass@8": 0.9477949526020941, + "pass@16": 0.9576107444633487, + "pass@32": 0.9640797766591122, + "pass@64": 0.9690906438054824, + "pass@128": 0.973 + }, + "16": { + "pass@1": 0.67528125, + "pass@2": 0.7668586368110242, + "pass@4": 0.826207869985001, + "pass@8": 0.8647069385416863, + "pass@16": 0.8913230015087684, + "pass@32": 0.9104233636374655, + "pass@64": 0.9251542736823251, + "pass@128": 0.937 + }, + "17": { + "pass@1": 0.60003125, + "pass@2": 0.697222687007874, + "pass@4": 0.7651455865204351, + "pass@8": 0.8134073039977633, + "pass@16": 0.8494884938027143, + "pass@32": 0.8762317637849516, + "pass@64": 0.8954231772733214, + "pass@128": 0.91 + }, + "18": { + "pass@1": 0.5249453125, + "pass@2": 0.6298601131889769, + "pass@4": 0.708367460629921, + "pass@8": 0.7653297211414652, + "pass@16": 0.8074251452312192, + "pass@32": 0.8394719843589634, + "pass@64": 0.8645644482383638, + "pass@128": 0.885 + }, + "19": { + "pass@1": 0.3302109375, + "pass@2": 0.4413890255905518, + "pass@4": 0.5427782260029989, + "pass@8": 0.6252117600790023, + "pass@16": 0.6887203084201948, + "pass@32": 0.7379025713933277, + "pass@64": 0.7779212141995803, + "pass@128": 0.813 + }, + "20": { + "pass@1": 0.2923046875, + "pass@2": 0.40001562500000043, + "pass@4": 0.5025576526059238, + "pass@8": 0.5887032945412243, + "pass@16": 0.6554380441904466, + "pass@32": 0.7062593639521223, + "pass@64": 0.7464682148716819, + "pass@128": 0.78 + } + }, + "per_op_accuracy": { + "11": 0.9720546875, + "12": 0.9254609375, + "13": 0.8862421875, + "14": 0.8654296875, + "15": 0.8511796875, + "16": 0.67528125, + "17": 0.60003125, + "18": 0.5249453125, + "19": 0.3302109375, + "20": 0.2923046875 + }, + "per_op_avg_response_len": { + "11": 298.95765625, + "12": 309.1229609375, + "13": 329.4066875, + "14": 356.717890625, + "15": 377.9163984375, + "16": 381.429625, + "17": 402.068875, + "18": 414.9804453125, + "19": 408.995625, + "20": 424.211078125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.95765625, + "12": 309.1229609375, + "13": 329.4066875, + "14": 356.717890625, + "15": 377.9163984375, + "16": 381.429625, + "17": 402.068875, + "18": 414.9804453125, + "19": 408.995625, + "20": 424.211078125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 299718, + "answer_accuracy": 0.6958534546805349, + "avg_response_len": 352.9888628343239, + "resp_tokens_sum": 152039363, + "pass_at_k": { + "pass@1": 0.6958534546805349, + "pass@2": 0.7603084671994053, + "pass@4": 0.808447550519786, + "pass@8": 0.8442062610716532, + "pass@16": 0.8709738783920659, + "pass@32": 0.8911416922756314, + "pass@64": 0.9073298350053864, + "pass@128": 0.9215453194650817 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9695024787535411, + "pass@2": 0.9771049887354734, + "pass@4": 0.9824912624236418, + "pass@8": 0.9866358976381672, + "pass@16": 0.9892977990867144, + "pass@32": 0.9912248584617577, + "pass@64": 0.993630773170722, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.9129811087570622, + "pass@2": 0.9284856838604917, + "pass@4": 0.9395466395620038, + "pass@8": 0.9490970855715675, + "pass@16": 0.9571701176219718, + "pass@32": 0.9632504243457556, + "pass@64": 0.9685818639992152, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.8993191614420063, + "pass@2": 0.9310942629526323, + "pass@4": 0.9483289223447383, + "pass@8": 0.9592160242365076, + "pass@16": 0.9663745706975613, + "pass@32": 0.9709750564674896, + "pass@64": 0.9748951075359104, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.8654351635514018, + "pass@2": 0.8911723176834208, + "pass@4": 0.9050400945792989, + "pass@8": 0.9153452032146256, + "pass@16": 0.9248815484346806, + "pass@32": 0.9342724814537383, + "pass@64": 0.943300051612627, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.8456015173410405, + "pass@2": 0.90106880945337, + "pass@4": 0.9342364244751189, + "pass@8": 0.9549646042253275, + "pass@16": 0.9676115180659557, + "pass@32": 0.9752756808271649, + "pass@64": 0.9805027399469873, + "pass@128": 0.9855491329479769 + }, + "16": { + "pass@1": 0.6807459677419355, + "pass@2": 0.7693917481584964, + "pass@4": 0.8267656827170792, + "pass@8": 0.8651684062311218, + "pass@16": 0.8915341014703523, + "pass@32": 0.9118703649147746, + "pass@64": 0.9304013527930276, + "pass@128": 0.9451612903225807 + }, + "17": { + "pass@1": 0.6200776734104047, + "pass@2": 0.7156233643211509, + "pass@4": 0.7799566124501779, + "pass@8": 0.8233111633123623, + "pass@16": 0.8552042906495471, + "pass@32": 0.8796848464391268, + "pass@64": 0.898560801911599, + "pass@128": 0.9132947976878613 + }, + "18": { + "pass@1": 0.49966517857142856, + "pass@2": 0.6061709083239591, + "pass@4": 0.6878156971449993, + "pass@8": 0.7466317628026085, + "pass@16": 0.7907171097865348, + "pass@32": 0.8251357952323664, + "pass@64": 0.8511277872053582, + "pass@128": 0.8714285714285714 + }, + "19": { + "pass@1": 0.3405358483483483, + "pass@2": 0.4503104236125886, + "pass@4": 0.545574125587905, + "pass@8": 0.622756746658054, + "pass@16": 0.6851441723433122, + "pass@32": 0.7350690819484497, + "pass@64": 0.7759458720196761, + "pass@128": 0.8078078078078078 + }, + "20": { + "pass@1": 0.31529185435435436, + "pass@2": 0.4256771535314842, + "pass@4": 0.5289863502422556, + "pass@8": 0.614086400199437, + "pass@16": 0.6775749305899774, + "pass@32": 0.7209746892325677, + "pass@64": 0.7533912624120301, + "pass@128": 0.7837837837837838 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 300796, + "answer_accuracy": 0.6987715581326197, + "avg_response_len": 365.3741102624145, + "resp_tokens_sum": 157280401, + "pass_at_k": { + "pass@1": 0.6987715581326197, + "pass@2": 0.7616315798253867, + "pass@4": 0.8089510211480073, + "pass@8": 0.8442964358439059, + "pass@16": 0.8706082500109146, + "pass@32": 0.8908089151833344, + "pass@64": 0.9073645126165447, + "pass@128": 0.9214986619090099 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9718326430722891, + "pass@2": 0.9759069496489899, + "pass@4": 0.9777419097085754, + "pass@8": 0.9791210531950711, + "pass@16": 0.9804853321345913, + "pass@32": 0.9819854324427932, + "pass@64": 0.9838250638344616, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9206259796238244, + "pass@2": 0.9325324587169551, + "pass@4": 0.9385155593795288, + "pass@8": 0.94239576563847, + "pass@16": 0.9460691225075901, + "pass@32": 0.9503599313563301, + "pass@64": 0.9548659759989492, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.8924851190476191, + "pass@2": 0.9173645343964358, + "pass@4": 0.9331037462228988, + "pass@8": 0.9435011875221508, + "pass@16": 0.9501677446036246, + "pass@32": 0.9551335916692331, + "pass@64": 0.9605950086730393, + "pass@128": 0.969187675070028 + }, + "14": { + "pass@1": 0.8716191520467836, + "pass@2": 0.908149073882212, + "pass@4": 0.9303283430031767, + "pass@8": 0.9451095148112428, + "pass@16": 0.9553346285919043, + "pass@32": 0.9639150092585048, + "pass@64": 0.9715032098154107, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8668797348484848, + "pass@2": 0.9162427672393225, + "pass@4": 0.9402515895740304, + "pass@8": 0.9531202338414846, + "pass@16": 0.9612437030923452, + "pass@32": 0.9664068646740456, + "pass@64": 0.9700282353302756, + "pass@128": 0.9727272727272728 + }, + "16": { + "pass@1": 0.6875222578347578, + "pass@2": 0.7761002030194939, + "pass@4": 0.8332365158415024, + "pass@8": 0.8719852773687603, + "pass@16": 0.9009490366933964, + "pass@32": 0.9217410166049883, + "pass@64": 0.9359481180049737, + "pass@128": 0.9458689458689459 + }, + "17": { + "pass@1": 0.636233501552795, + "pass@2": 0.7325195933388761, + "pass@4": 0.7973493899442688, + "pass@8": 0.8420161183796606, + "pass@16": 0.873911112162783, + "pass@32": 0.8950428209077892, + "pass@64": 0.9078423872775889, + "pass@128": 0.9192546583850931 + }, + "18": { + "pass@1": 0.5431525735294118, + "pass@2": 0.6473685734136173, + "pass@4": 0.7243213946602258, + "pass@8": 0.780417991903133, + "pass@16": 0.8212411992698683, + "pass@32": 0.8530416660502275, + "pass@64": 0.8812526805915043, + "pass@128": 0.9029411764705882 + }, + "19": { + "pass@1": 0.33010137648809523, + "pass@2": 0.440707606861642, + "pass@4": 0.5439537635920508, + "pass@8": 0.6269219877098245, + "pass@16": 0.6883648113498766, + "pass@32": 0.7355810026120768, + "pass@64": 0.7761923987319511, + "pass@128": 0.8154761904761905 + }, + "20": { + "pass@1": 0.26672436377245506, + "pass@2": 0.36819238000377186, + "pass@4": 0.46910746111825846, + "pass@8": 0.5566032696790372, + "pass@16": 0.6264926354927363, + "pass@32": 0.6830126971788019, + "pass@64": 0.729548680785725, + "pass@128": 0.7664670658682635 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 285648, + "answer_accuracy": 0.6820369804400978, + "avg_response_len": 393.41277076329465, + "resp_tokens_sum": 164767563, + "pass_at_k": { + "pass@1": 0.6820369804400978, + "pass@2": 0.7476408936237856, + "pass@4": 0.7979329229246728, + "pass@8": 0.8355065211155187, + "pass@16": 0.8634287573808466, + "pass@32": 0.8844623588536797, + "pass@64": 0.9004260770487954, + "pass@128": 0.9135085574572127 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9751488095238096, + "pass@2": 0.9822994781902263, + "pass@4": 0.9872104043542176, + "pass@8": 0.9904841598664734, + "pass@16": 0.9919896279150862, + "pass@32": 0.9929172274906727, + "pass@64": 0.9935593050868524, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.9436878822629969, + "pass@2": 0.956287699559344, + "pass@4": 0.9640391519294034, + "pass@8": 0.9686254747421928, + "pass@16": 0.9714284855155032, + "pass@32": 0.9739091868355997, + "pass@64": 0.9762613977664752, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.8664882330246914, + "pass@2": 0.8903538750364539, + "pass@4": 0.9041646755961056, + "pass@8": 0.9148942544269757, + "pass@16": 0.9250264174045647, + "pass@32": 0.9337574735241562, + "pass@64": 0.9408890741706648, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.859143175074184, + "pass@2": 0.8930812489777795, + "pass@4": 0.9152190499555503, + "pass@8": 0.9310064996161205, + "pass@16": 0.9421396953920551, + "pass@32": 0.9493720967197038, + "pass@64": 0.9534362092159403, + "pass@128": 0.9554896142433235 + }, + "15": { + "pass@1": 0.8411458333333334, + "pass@2": 0.891308888038301, + "pass@4": 0.9192842798585358, + "pass@8": 0.9347145752235823, + "pass@16": 0.9432306703458466, + "pass@32": 0.9497534745385128, + "pass@64": 0.9559486979161531, + "pass@128": 0.9598765432098766 + }, + "16": { + "pass@1": 0.657609697640118, + "pass@2": 0.7549735210089886, + "pass@4": 0.8184203282664885, + "pass@8": 0.8567489682997159, + "pass@16": 0.8811631804530315, + "pass@32": 0.8973818692788625, + "pass@64": 0.909180132438763, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.5440276731927711, + "pass@2": 0.6438123310169818, + "pass@4": 0.7184761899115022, + "pass@8": 0.7753387089561334, + "pass@16": 0.8198446720528769, + "pass@32": 0.8543886101949005, + "pass@64": 0.8801081057485678, + "pass@128": 0.8975903614457831 + }, + "18": { + "pass@1": 0.5335181451612904, + "pass@2": 0.6374031623063245, + "pass@4": 0.714073201434498, + "pass@8": 0.7698918932693165, + "pass@16": 0.8111359646908977, + "pass@32": 0.8407751276469603, + "pass@64": 0.8614316494044385, + "pass@128": 0.8806451612903226 + }, + "19": { + "pass@1": 0.31993485649546827, + "pass@2": 0.43310543390346606, + "pass@4": 0.5387721378105682, + "pass@8": 0.6259455455328414, + "pass@16": 0.6926789196867486, + "pass@32": 0.7431098194164243, + "pass@64": 0.7816634224866824, + "pass@128": 0.8157099697885196 + }, + "20": { + "pass@1": 0.29497466216216217, + "pass@2": 0.4062729067650326, + "pass@4": 0.5096795974827469, + "pass@8": 0.595516610216258, + "pass@16": 0.6623334896378719, + "pass@32": 0.7148605152551266, + "pass@64": 0.7565155108289564, + "pass@128": 0.7897897897897898 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.8355448190789474, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8355448190789474, + "pass@2": 0.8700739484044647, + "pass@4": 0.8961321521273532, + "pass@8": 0.9155421867984816, + "pass@16": 0.9299832272346062, + "pass@32": 0.9409058065258158, + "pass@64": 0.9495660987686426, + "pass@128": 0.9569473684210527 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9630703125, + "pass@2": 0.971617249015748, + "pass@4": 0.9783617097862765, + "pass@8": 0.9833621198515833, + "pass@16": 0.9867538015919473, + "pass@32": 0.9889978617362172, + "pass@64": 0.9909396817980759, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9988515625, + "pass@2": 0.9994415600393702, + "pass@4": 0.9997261106111736, + "pass@8": 0.9999287673940046, + "pass@16": 0.9999959202789485, + "pass@32": 0.9999999941392413, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99859375, + "pass@2": 0.999097563976378, + "pass@4": 0.9993847810273716, + "pass@8": 0.9996419787260388, + "pass@16": 0.9998812336885909, + "pass@32": 0.9999901022858958, + "pass@64": 0.9999999879305072, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995046875, + "pass@2": 0.9971327509842518, + "pass@4": 0.9981651927259092, + "pass@8": 0.9987532396728784, + "pass@16": 0.9992837011149076, + "pass@32": 0.9997811580893737, + "pass@64": 0.999990332213311, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966640625, + "pass@2": 0.9980300196850395, + "pass@4": 0.9991082808398951, + "pass@8": 0.9997151241874243, + "pass@16": 0.9999451210572601, + "pass@32": 0.9999981933013141, + "pass@64": 0.9999999998360333, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9720546875, + "pass@2": 0.978343503937008, + "pass@4": 0.9824010070303709, + "pass@8": 0.985353171884976, + "pass@16": 0.9872199861395465, + "pass@32": 0.9886904652675699, + "pass@64": 0.9903527652246648, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.9254609375, + "pass@2": 0.938867864173229, + "pass@4": 0.9472267765279342, + "pass@8": 0.953345147771704, + "pass@16": 0.9582913864816682, + "pass@32": 0.962623772416308, + "pass@64": 0.9667177032690246, + "pass@128": 0.971 + }, + "13": { + "pass@1": 0.8862421875, + "pass@2": 0.9129928641732293, + "pass@4": 0.928584318522685, + "pass@8": 0.9392455741111944, + "pass@16": 0.9471919321150946, + "pass@32": 0.953261156660872, + "pass@64": 0.9587720174315262, + "pass@128": 0.966 + }, + "14": { + "pass@1": 0.8654296875, + "pass@2": 0.8976216781496069, + "pass@4": 0.9171189835020619, + "pass@8": 0.9308024546679728, + "pass@16": 0.9411124973730856, + "pass@32": 0.9494987963075991, + "pass@64": 0.9563614168302956, + "pass@128": 0.962 + }, + "15": { + "pass@1": 0.8511796875, + "pass@2": 0.9029140009842532, + "pass@4": 0.9313769341019875, + "pass@8": 0.9477949526020941, + "pass@16": 0.9576107444633487, + "pass@32": 0.9640797766591122, + "pass@64": 0.9690906438054824, + "pass@128": 0.973 + }, + "16": { + "pass@1": 0.67528125, + "pass@2": 0.7668586368110242, + "pass@4": 0.826207869985001, + "pass@8": 0.8647069385416863, + "pass@16": 0.8913230015087684, + "pass@32": 0.9104233636374655, + "pass@64": 0.9251542736823251, + "pass@128": 0.937 + }, + "17": { + "pass@1": 0.60003125, + "pass@2": 0.697222687007874, + "pass@4": 0.7651455865204351, + "pass@8": 0.8134073039977633, + "pass@16": 0.8494884938027143, + "pass@32": 0.8762317637849516, + "pass@64": 0.8954231772733214, + "pass@128": 0.91 + }, + "18": { + "pass@1": 0.5249453125, + "pass@2": 0.6298601131889769, + "pass@4": 0.708367460629921, + "pass@8": 0.7653297211414652, + "pass@16": 0.8074251452312192, + "pass@32": 0.8394719843589634, + "pass@64": 0.8645644482383638, + "pass@128": 0.885 + }, + "19": { + "pass@1": 0.3302109375, + "pass@2": 0.4413890255905518, + "pass@4": 0.5427782260029989, + "pass@8": 0.6252117600790023, + "pass@16": 0.6887203084201948, + "pass@32": 0.7379025713933277, + "pass@64": 0.7779212141995803, + "pass@128": 0.813 + }, + "20": { + "pass@1": 0.2923046875, + "pass@2": 0.40001562500000043, + "pass@4": 0.5025576526059238, + "pass@8": 0.5887032945412243, + "pass@16": 0.6554380441904466, + "pass@32": 0.7062593639521223, + "pass@64": 0.7464682148716819, + "pass@128": 0.78 + } + }, + "per_op_accuracy": { + "10": 0.9630703125, + "2": 1.0, + "3": 1.0, + "4": 0.999984375, + "5": 0.9988515625, + "6": 1.0, + "7": 0.99859375, + "8": 0.995046875, + "9": 0.9966640625, + "11": 0.9720546875, + "12": 0.9254609375, + "13": 0.8862421875, + "14": 0.8654296875, + "15": 0.8511796875, + "16": 0.67528125, + "17": 0.60003125, + "18": 0.5249453125, + "19": 0.3302109375, + "20": 0.2923046875 + }, + "per_op_avg_response_len": { + "10": 280.7720078125, + "2": 115.459953125, + "3": 144.460953125, + "4": 159.6328828125, + "5": 172.4409140625, + "6": 198.4999609375, + "7": 223.721234375, + "8": 243.211546875, + "9": 259.0201484375, + "11": 298.95765625, + "12": 309.1229609375, + "13": 329.4066875, + "14": 356.717890625, + "15": 377.9163984375, + "16": 381.429625, + "17": 402.068875, + "18": 414.9804453125, + "19": 408.995625, + "20": 424.211078125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.7720078125, + "2": 115.459953125, + "3": 144.460953125, + "4": 159.6328828125, + "5": 172.4409140625, + "6": 198.4999609375, + "7": 223.721234375, + "8": 243.211546875, + "9": 259.0201484375, + "11": 298.95765625, + "12": 309.1229609375, + "13": 329.4066875, + "14": 356.717890625, + "15": 377.9163984375, + "16": 381.429625, + "17": 402.068875, + "18": 414.9804453125, + "19": 408.995625, + "20": 424.211078125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 677131, + "answer_accuracy": 0.8362450106702498, + "avg_response_len": 276.775293678865, + "resp_tokens_sum": 224112705, + "pass_at_k": { + "pass@1": 0.8362450106702498, + "pass@2": 0.8710972215652841, + "pass@4": 0.8970693610984718, + "pass@8": 0.9163415000861901, + "pass@16": 0.9307410895193654, + "pass@32": 0.9415673880074282, + "pass@64": 0.95027139164483, + "pass@128": 0.9579513120455264 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9696718289085545, + "pass@2": 0.977735291385037, + "pass@4": 0.9829392385133277, + "pass@8": 0.9862529137910082, + "pass@16": 0.9884945156128231, + "pass@32": 0.9901631698480607, + "pass@64": 0.9918935956224192, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 0.9999996155265748, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9976934523809524, + "pass@2": 0.9983291151106111, + "pass@4": 0.9991311398575179, + "pass@8": 0.9997738647527661, + "pass@16": 0.9999870485045981, + "pass@32": 0.9999999813944173, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9983223157051282, + "pass@2": 0.999707011533414, + "pass@4": 0.9999820905800235, + "pass@8": 0.9999999321607189, + "pass@16": 0.9999999999997491, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9974911971830986, + "pass@2": 0.9979971858711323, + "pass@4": 0.9985849229233669, + "pass@8": 0.9993064141767372, + "pass@16": 0.9998463956015636, + "pass@32": 0.9999949107316761, + "pass@64": 0.9999999995381218, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9695024787535411, + "pass@2": 0.9771049887354734, + "pass@4": 0.9824912624236418, + "pass@8": 0.9866358976381672, + "pass@16": 0.9892977990867144, + "pass@32": 0.9912248584617577, + "pass@64": 0.993630773170722, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.9129811087570622, + "pass@2": 0.9284856838604917, + "pass@4": 0.9395466395620038, + "pass@8": 0.9490970855715675, + "pass@16": 0.9571701176219718, + "pass@32": 0.9632504243457556, + "pass@64": 0.9685818639992152, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.8993191614420063, + "pass@2": 0.9310942629526323, + "pass@4": 0.9483289223447383, + "pass@8": 0.9592160242365076, + "pass@16": 0.9663745706975613, + "pass@32": 0.9709750564674896, + "pass@64": 0.9748951075359104, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.8654351635514018, + "pass@2": 0.8911723176834208, + "pass@4": 0.9050400945792989, + "pass@8": 0.9153452032146256, + "pass@16": 0.9248815484346806, + "pass@32": 0.9342724814537383, + "pass@64": 0.943300051612627, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.8456015173410405, + "pass@2": 0.90106880945337, + "pass@4": 0.9342364244751189, + "pass@8": 0.9549646042253275, + "pass@16": 0.9676115180659557, + "pass@32": 0.9752756808271649, + "pass@64": 0.9805027399469873, + "pass@128": 0.9855491329479769 + }, + "16": { + "pass@1": 0.6807459677419355, + "pass@2": 0.7693917481584964, + "pass@4": 0.8267656827170792, + "pass@8": 0.8651684062311218, + "pass@16": 0.8915341014703523, + "pass@32": 0.9118703649147746, + "pass@64": 0.9304013527930276, + "pass@128": 0.9451612903225807 + }, + "17": { + "pass@1": 0.6200776734104047, + "pass@2": 0.7156233643211509, + "pass@4": 0.7799566124501779, + "pass@8": 0.8233111633123623, + "pass@16": 0.8552042906495471, + "pass@32": 0.8796848464391268, + "pass@64": 0.898560801911599, + "pass@128": 0.9132947976878613 + }, + "18": { + "pass@1": 0.49966517857142856, + "pass@2": 0.6061709083239591, + "pass@4": 0.6878156971449993, + "pass@8": 0.7466317628026085, + "pass@16": 0.7907171097865348, + "pass@32": 0.8251357952323664, + "pass@64": 0.8511277872053582, + "pass@128": 0.8714285714285714 + }, + "19": { + "pass@1": 0.3405358483483483, + "pass@2": 0.4503104236125886, + "pass@4": 0.545574125587905, + "pass@8": 0.622756746658054, + "pass@16": 0.6851441723433122, + "pass@32": 0.7350690819484497, + "pass@64": 0.7759458720196761, + "pass@128": 0.8078078078078078 + }, + "20": { + "pass@1": 0.31529185435435436, + "pass@2": 0.4256771535314842, + "pass@4": 0.5289863502422556, + "pass@8": 0.614086400199437, + "pass@16": 0.6775749305899774, + "pass@32": 0.7209746892325677, + "pass@64": 0.7533912624120301, + "pass@128": 0.7837837837837838 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 689100, + "answer_accuracy": 0.8392196024941543, + "avg_response_len": 285.62740037996883, + "resp_tokens_sum": 234534371, + "pass_at_k": { + "pass@1": 0.8392196024941543, + "pass@2": 0.8729144483279085, + "pass@4": 0.8982317841456428, + "pass@8": 0.917068877210444, + "pass@16": 0.9310411725691664, + "pass@32": 0.9417522529217024, + "pass@64": 0.9505397100614599, + "pass@128": 0.9580670303975058 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9559528374233128, + "pass@2": 0.9649733859475385, + "pass@4": 0.9714398099585708, + "pass@8": 0.975561770359765, + "pass@16": 0.9781992510202098, + "pass@32": 0.9802460689834376, + "pass@64": 0.982347841056068, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996376811594203, + "pass@2": 0.9999868053178136, + "pass@4": 0.9999999807089331, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9963421242774566, + "pass@2": 0.9974252281416413, + "pass@4": 0.9982221816745449, + "pass@8": 0.9989652564397086, + "pass@16": 0.9996567447647136, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9975792253521126, + "pass@2": 0.9988195907729843, + "pass@4": 0.9995380836198291, + "pass@8": 0.9999297766515117, + "pass@16": 0.9999987404291422, + "pass@32": 0.9999999998867772, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947136976047904, + "pass@2": 0.9973422986703758, + "pass@4": 0.9992345665811235, + "pass@8": 0.9999333742074483, + "pass@16": 0.9999995056016906, + "pass@32": 0.9999999999906032, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9718326430722891, + "pass@2": 0.9759069496489899, + "pass@4": 0.9777419097085754, + "pass@8": 0.9791210531950711, + "pass@16": 0.9804853321345913, + "pass@32": 0.9819854324427932, + "pass@64": 0.9838250638344616, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9206259796238244, + "pass@2": 0.9325324587169551, + "pass@4": 0.9385155593795288, + "pass@8": 0.94239576563847, + "pass@16": 0.9460691225075901, + "pass@32": 0.9503599313563301, + "pass@64": 0.9548659759989492, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.8924851190476191, + "pass@2": 0.9173645343964358, + "pass@4": 0.9331037462228988, + "pass@8": 0.9435011875221508, + "pass@16": 0.9501677446036246, + "pass@32": 0.9551335916692331, + "pass@64": 0.9605950086730393, + "pass@128": 0.969187675070028 + }, + "14": { + "pass@1": 0.8716191520467836, + "pass@2": 0.908149073882212, + "pass@4": 0.9303283430031767, + "pass@8": 0.9451095148112428, + "pass@16": 0.9553346285919043, + "pass@32": 0.9639150092585048, + "pass@64": 0.9715032098154107, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8668797348484848, + "pass@2": 0.9162427672393225, + "pass@4": 0.9402515895740304, + "pass@8": 0.9531202338414846, + "pass@16": 0.9612437030923452, + "pass@32": 0.9664068646740456, + "pass@64": 0.9700282353302756, + "pass@128": 0.9727272727272728 + }, + "16": { + "pass@1": 0.6875222578347578, + "pass@2": 0.7761002030194939, + "pass@4": 0.8332365158415024, + "pass@8": 0.8719852773687603, + "pass@16": 0.9009490366933964, + "pass@32": 0.9217410166049883, + "pass@64": 0.9359481180049737, + "pass@128": 0.9458689458689459 + }, + "17": { + "pass@1": 0.636233501552795, + "pass@2": 0.7325195933388761, + "pass@4": 0.7973493899442688, + "pass@8": 0.8420161183796606, + "pass@16": 0.873911112162783, + "pass@32": 0.8950428209077892, + "pass@64": 0.9078423872775889, + "pass@128": 0.9192546583850931 + }, + "18": { + "pass@1": 0.5431525735294118, + "pass@2": 0.6473685734136173, + "pass@4": 0.7243213946602258, + "pass@8": 0.780417991903133, + "pass@16": 0.8212411992698683, + "pass@32": 0.8530416660502275, + "pass@64": 0.8812526805915043, + "pass@128": 0.9029411764705882 + }, + "19": { + "pass@1": 0.33010137648809523, + "pass@2": 0.440707606861642, + "pass@4": 0.5439537635920508, + "pass@8": 0.6269219877098245, + "pass@16": 0.6883648113498766, + "pass@32": 0.7355810026120768, + "pass@64": 0.7761923987319511, + "pass@128": 0.8154761904761905 + }, + "20": { + "pass@1": 0.26672436377245506, + "pass@2": 0.36819238000377186, + "pass@4": 0.46910746111825846, + "pass@8": 0.5566032696790372, + "pass@16": 0.6264926354927363, + "pass@32": 0.6830126971788019, + "pass@64": 0.729548680785725, + "pass@128": 0.7664670658682635 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 665814, + "answer_accuracy": 0.831070758108324, + "avg_response_len": 306.4142135325132, + "resp_tokens_sum": 245484360, + "pass_at_k": { + "pass@1": 0.831070758108324, + "pass@2": 0.8661284246747585, + "pass@4": 0.8930329472466244, + "pass@8": 0.9131695753827629, + "pass@16": 0.9281329386686574, + "pass@32": 0.9393695997703541, + "pass@64": 0.947855378337564, + "pass@128": 0.9547851094424029 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9633162313432836, + "pass@2": 0.9718915266188743, + "pass@4": 0.9804654623395956, + "pass@8": 0.9880275968332796, + "pass@16": 0.9933170297510806, + "pass@32": 0.9963353094600719, + "pass@64": 0.9983353931277541, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9991268382352941, + "pass@2": 0.9999189439555349, + "pass@4": 0.9999994359160987, + "pass@8": 0.9999999999908352, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9996291534810127, + "pass@2": 0.9999645700438552, + "pass@4": 0.9999997030632247, + "pass@8": 0.9999999999933531, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9892783408408409, + "pass@2": 0.9929225485327845, + "pass@4": 0.9949992816213291, + "pass@8": 0.9963309012836256, + "pass@16": 0.9978502950830038, + "pass@32": 0.9993428172059096, + "pass@64": 0.9999709676075407, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9978145096463023, + "pass@2": 0.9988060802086235, + "pass@4": 0.9995700577620724, + "pass@8": 0.999947267425707, + "pass@16": 0.9999994075490037, + "pass@32": 0.9999999999829826, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9751488095238096, + "pass@2": 0.9822994781902263, + "pass@4": 0.9872104043542176, + "pass@8": 0.9904841598664734, + "pass@16": 0.9919896279150862, + "pass@32": 0.9929172274906727, + "pass@64": 0.9935593050868524, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.9436878822629969, + "pass@2": 0.956287699559344, + "pass@4": 0.9640391519294034, + "pass@8": 0.9686254747421928, + "pass@16": 0.9714284855155032, + "pass@32": 0.9739091868355997, + "pass@64": 0.9762613977664752, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.8664882330246914, + "pass@2": 0.8903538750364539, + "pass@4": 0.9041646755961056, + "pass@8": 0.9148942544269757, + "pass@16": 0.9250264174045647, + "pass@32": 0.9337574735241562, + "pass@64": 0.9408890741706648, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.859143175074184, + "pass@2": 0.8930812489777795, + "pass@4": 0.9152190499555503, + "pass@8": 0.9310064996161205, + "pass@16": 0.9421396953920551, + "pass@32": 0.9493720967197038, + "pass@64": 0.9534362092159403, + "pass@128": 0.9554896142433235 + }, + "15": { + "pass@1": 0.8411458333333334, + "pass@2": 0.891308888038301, + "pass@4": 0.9192842798585358, + "pass@8": 0.9347145752235823, + "pass@16": 0.9432306703458466, + "pass@32": 0.9497534745385128, + "pass@64": 0.9559486979161531, + "pass@128": 0.9598765432098766 + }, + "16": { + "pass@1": 0.657609697640118, + "pass@2": 0.7549735210089886, + "pass@4": 0.8184203282664885, + "pass@8": 0.8567489682997159, + "pass@16": 0.8811631804530315, + "pass@32": 0.8973818692788625, + "pass@64": 0.909180132438763, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.5440276731927711, + "pass@2": 0.6438123310169818, + "pass@4": 0.7184761899115022, + "pass@8": 0.7753387089561334, + "pass@16": 0.8198446720528769, + "pass@32": 0.8543886101949005, + "pass@64": 0.8801081057485678, + "pass@128": 0.8975903614457831 + }, + "18": { + "pass@1": 0.5335181451612904, + "pass@2": 0.6374031623063245, + "pass@4": 0.714073201434498, + "pass@8": 0.7698918932693165, + "pass@16": 0.8111359646908977, + "pass@32": 0.8407751276469603, + "pass@64": 0.8614316494044385, + "pass@128": 0.8806451612903226 + }, + "19": { + "pass@1": 0.31993485649546827, + "pass@2": 0.43310543390346606, + "pass@4": 0.5387721378105682, + "pass@8": 0.6259455455328414, + "pass@16": 0.6926789196867486, + "pass@32": 0.7431098194164243, + "pass@64": 0.7816634224866824, + "pass@128": 0.8157099697885196 + }, + "20": { + "pass@1": 0.29497466216216217, + "pass@2": 0.4062729067650326, + "pass@4": 0.5096795974827469, + "pass@8": 0.595516610216258, + "pass@16": 0.6623334896378719, + "pass@32": 0.7148605152551266, + "pass@64": 0.7565155108289564, + "pass@128": 0.7897897897897898 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch4-RL-op11-14/composition/summary.csv b/composition/Qwen3-100M-epoch4-RL-op11-14/composition/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..60f8ebd0865d885b3ad7c763db43990b3ea5a4b6 --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op11-14/composition/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.9946901041666667,0.6923140625,0.8355448190789474,0.0,0.0,0.0,199.69106684027778,370.38072421875,1152000,1280000,2432000 diff --git a/composition/Qwen3-100M-epoch4-RL-op11-14/composition/summary.json b/composition/Qwen3-100M-epoch4-RL-op11-14/composition/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2b19f31b9e09ba742675c3b67ab4264f23cd298f --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op11-14/composition/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.9946901041666667, + "ood_acc": 0.6923140625, + "total_acc": 0.8355448190789474, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.69106684027778, + "ood_avg_resp_len": 370.38072421875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 677131, + "answer_accuracy": 0.8362450106702498, + "avg_response_len": 276.775293678865, + "resp_tokens_sum": 224112705, + "pass_at_k": { + "pass@1": 0.8362450106702498, + "pass@2": 0.8710972215652841, + "pass@4": 0.8970693610984718, + "pass@8": 0.9163415000861901, + "pass@16": 0.9307410895193654, + "pass@32": 0.9415673880074282, + "pass@64": 0.95027139164483, + "pass@128": 0.9579513120455264 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9696718289085545, + "pass@2": 0.977735291385037, + "pass@4": 0.9829392385133277, + "pass@8": 0.9862529137910082, + "pass@16": 0.9884945156128231, + "pass@32": 0.9901631698480607, + "pass@64": 0.9918935956224192, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 0.9999996155265748, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9976934523809524, + "pass@2": 0.9983291151106111, + "pass@4": 0.9991311398575179, + "pass@8": 0.9997738647527661, + "pass@16": 0.9999870485045981, + "pass@32": 0.9999999813944173, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9983223157051282, + "pass@2": 0.999707011533414, + "pass@4": 0.9999820905800235, + "pass@8": 0.9999999321607189, + "pass@16": 0.9999999999997491, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9974911971830986, + "pass@2": 0.9979971858711323, + "pass@4": 0.9985849229233669, + "pass@8": 0.9993064141767372, + "pass@16": 0.9998463956015636, + "pass@32": 0.9999949107316761, + "pass@64": 0.9999999995381218, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9695024787535411, + "pass@2": 0.9771049887354734, + "pass@4": 0.9824912624236418, + "pass@8": 0.9866358976381672, + "pass@16": 0.9892977990867144, + "pass@32": 0.9912248584617577, + "pass@64": 0.993630773170722, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.9129811087570622, + "pass@2": 0.9284856838604917, + "pass@4": 0.9395466395620038, + "pass@8": 0.9490970855715675, + "pass@16": 0.9571701176219718, + "pass@32": 0.9632504243457556, + "pass@64": 0.9685818639992152, + "pass@128": 0.9745762711864406 + }, + "13": { + "pass@1": 0.8993191614420063, + "pass@2": 0.9310942629526323, + "pass@4": 0.9483289223447383, + "pass@8": 0.9592160242365076, + "pass@16": 0.9663745706975613, + "pass@32": 0.9709750564674896, + "pass@64": 0.9748951075359104, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.8654351635514018, + "pass@2": 0.8911723176834208, + "pass@4": 0.9050400945792989, + "pass@8": 0.9153452032146256, + "pass@16": 0.9248815484346806, + "pass@32": 0.9342724814537383, + "pass@64": 0.943300051612627, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.8456015173410405, + "pass@2": 0.90106880945337, + "pass@4": 0.9342364244751189, + "pass@8": 0.9549646042253275, + "pass@16": 0.9676115180659557, + "pass@32": 0.9752756808271649, + "pass@64": 0.9805027399469873, + "pass@128": 0.9855491329479769 + }, + "16": { + "pass@1": 0.6807459677419355, + "pass@2": 0.7693917481584964, + "pass@4": 0.8267656827170792, + "pass@8": 0.8651684062311218, + "pass@16": 0.8915341014703523, + "pass@32": 0.9118703649147746, + "pass@64": 0.9304013527930276, + "pass@128": 0.9451612903225807 + }, + "17": { + "pass@1": 0.6200776734104047, + "pass@2": 0.7156233643211509, + "pass@4": 0.7799566124501779, + "pass@8": 0.8233111633123623, + "pass@16": 0.8552042906495471, + "pass@32": 0.8796848464391268, + "pass@64": 0.898560801911599, + "pass@128": 0.9132947976878613 + }, + "18": { + "pass@1": 0.49966517857142856, + "pass@2": 0.6061709083239591, + "pass@4": 0.6878156971449993, + "pass@8": 0.7466317628026085, + "pass@16": 0.7907171097865348, + "pass@32": 0.8251357952323664, + "pass@64": 0.8511277872053582, + "pass@128": 0.8714285714285714 + }, + "19": { + "pass@1": 0.3405358483483483, + "pass@2": 0.4503104236125886, + "pass@4": 0.545574125587905, + "pass@8": 0.622756746658054, + "pass@16": 0.6851441723433122, + "pass@32": 0.7350690819484497, + "pass@64": 0.7759458720196761, + "pass@128": 0.8078078078078078 + }, + "20": { + "pass@1": 0.31529185435435436, + "pass@2": 0.4256771535314842, + "pass@4": 0.5289863502422556, + "pass@8": 0.614086400199437, + "pass@16": 0.6775749305899774, + "pass@32": 0.7209746892325677, + "pass@64": 0.7533912624120301, + "pass@128": 0.7837837837837838 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 689100, + "answer_accuracy": 0.8392196024941543, + "avg_response_len": 285.62740037996883, + "resp_tokens_sum": 234534371, + "pass_at_k": { + "pass@1": 0.8392196024941543, + "pass@2": 0.8729144483279085, + "pass@4": 0.8982317841456428, + "pass@8": 0.917068877210444, + "pass@16": 0.9310411725691664, + "pass@32": 0.9417522529217024, + "pass@64": 0.9505397100614599, + "pass@128": 0.9580670303975058 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9559528374233128, + "pass@2": 0.9649733859475385, + "pass@4": 0.9714398099585708, + "pass@8": 0.975561770359765, + "pass@16": 0.9781992510202098, + "pass@32": 0.9802460689834376, + "pass@64": 0.982347841056068, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996376811594203, + "pass@2": 0.9999868053178136, + "pass@4": 0.9999999807089331, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9963421242774566, + "pass@2": 0.9974252281416413, + "pass@4": 0.9982221816745449, + "pass@8": 0.9989652564397086, + "pass@16": 0.9996567447647136, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9975792253521126, + "pass@2": 0.9988195907729843, + "pass@4": 0.9995380836198291, + "pass@8": 0.9999297766515117, + "pass@16": 0.9999987404291422, + "pass@32": 0.9999999998867772, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947136976047904, + "pass@2": 0.9973422986703758, + "pass@4": 0.9992345665811235, + "pass@8": 0.9999333742074483, + "pass@16": 0.9999995056016906, + "pass@32": 0.9999999999906032, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9718326430722891, + "pass@2": 0.9759069496489899, + "pass@4": 0.9777419097085754, + "pass@8": 0.9791210531950711, + "pass@16": 0.9804853321345913, + "pass@32": 0.9819854324427932, + "pass@64": 0.9838250638344616, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9206259796238244, + "pass@2": 0.9325324587169551, + "pass@4": 0.9385155593795288, + "pass@8": 0.94239576563847, + "pass@16": 0.9460691225075901, + "pass@32": 0.9503599313563301, + "pass@64": 0.9548659759989492, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.8924851190476191, + "pass@2": 0.9173645343964358, + "pass@4": 0.9331037462228988, + "pass@8": 0.9435011875221508, + "pass@16": 0.9501677446036246, + "pass@32": 0.9551335916692331, + "pass@64": 0.9605950086730393, + "pass@128": 0.969187675070028 + }, + "14": { + "pass@1": 0.8716191520467836, + "pass@2": 0.908149073882212, + "pass@4": 0.9303283430031767, + "pass@8": 0.9451095148112428, + "pass@16": 0.9553346285919043, + "pass@32": 0.9639150092585048, + "pass@64": 0.9715032098154107, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8668797348484848, + "pass@2": 0.9162427672393225, + "pass@4": 0.9402515895740304, + "pass@8": 0.9531202338414846, + "pass@16": 0.9612437030923452, + "pass@32": 0.9664068646740456, + "pass@64": 0.9700282353302756, + "pass@128": 0.9727272727272728 + }, + "16": { + "pass@1": 0.6875222578347578, + "pass@2": 0.7761002030194939, + "pass@4": 0.8332365158415024, + "pass@8": 0.8719852773687603, + "pass@16": 0.9009490366933964, + "pass@32": 0.9217410166049883, + "pass@64": 0.9359481180049737, + "pass@128": 0.9458689458689459 + }, + "17": { + "pass@1": 0.636233501552795, + "pass@2": 0.7325195933388761, + "pass@4": 0.7973493899442688, + "pass@8": 0.8420161183796606, + "pass@16": 0.873911112162783, + "pass@32": 0.8950428209077892, + "pass@64": 0.9078423872775889, + "pass@128": 0.9192546583850931 + }, + "18": { + "pass@1": 0.5431525735294118, + "pass@2": 0.6473685734136173, + "pass@4": 0.7243213946602258, + "pass@8": 0.780417991903133, + "pass@16": 0.8212411992698683, + "pass@32": 0.8530416660502275, + "pass@64": 0.8812526805915043, + "pass@128": 0.9029411764705882 + }, + "19": { + "pass@1": 0.33010137648809523, + "pass@2": 0.440707606861642, + "pass@4": 0.5439537635920508, + "pass@8": 0.6269219877098245, + "pass@16": 0.6883648113498766, + "pass@32": 0.7355810026120768, + "pass@64": 0.7761923987319511, + "pass@128": 0.8154761904761905 + }, + "20": { + "pass@1": 0.26672436377245506, + "pass@2": 0.36819238000377186, + "pass@4": 0.46910746111825846, + "pass@8": 0.5566032696790372, + "pass@16": 0.6264926354927363, + "pass@32": 0.6830126971788019, + "pass@64": 0.729548680785725, + "pass@128": 0.7664670658682635 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 665814, + "answer_accuracy": 0.831070758108324, + "avg_response_len": 306.4142135325132, + "resp_tokens_sum": 245484360, + "pass_at_k": { + "pass@1": 0.831070758108324, + "pass@2": 0.8661284246747585, + "pass@4": 0.8930329472466244, + "pass@8": 0.9131695753827629, + "pass@16": 0.9281329386686574, + "pass@32": 0.9393695997703541, + "pass@64": 0.947855378337564, + "pass@128": 0.9547851094424029 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9633162313432836, + "pass@2": 0.9718915266188743, + "pass@4": 0.9804654623395956, + "pass@8": 0.9880275968332796, + "pass@16": 0.9933170297510806, + "pass@32": 0.9963353094600719, + "pass@64": 0.9983353931277541, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9991268382352941, + "pass@2": 0.9999189439555349, + "pass@4": 0.9999994359160987, + "pass@8": 0.9999999999908352, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9996291534810127, + "pass@2": 0.9999645700438552, + "pass@4": 0.9999997030632247, + "pass@8": 0.9999999999933531, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9892783408408409, + "pass@2": 0.9929225485327845, + "pass@4": 0.9949992816213291, + "pass@8": 0.9963309012836256, + "pass@16": 0.9978502950830038, + "pass@32": 0.9993428172059096, + "pass@64": 0.9999709676075407, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9978145096463023, + "pass@2": 0.9988060802086235, + "pass@4": 0.9995700577620724, + "pass@8": 0.999947267425707, + "pass@16": 0.9999994075490037, + "pass@32": 0.9999999999829826, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9751488095238096, + "pass@2": 0.9822994781902263, + "pass@4": 0.9872104043542176, + "pass@8": 0.9904841598664734, + "pass@16": 0.9919896279150862, + "pass@32": 0.9929172274906727, + "pass@64": 0.9935593050868524, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.9436878822629969, + "pass@2": 0.956287699559344, + "pass@4": 0.9640391519294034, + "pass@8": 0.9686254747421928, + "pass@16": 0.9714284855155032, + "pass@32": 0.9739091868355997, + "pass@64": 0.9762613977664752, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.8664882330246914, + "pass@2": 0.8903538750364539, + "pass@4": 0.9041646755961056, + "pass@8": 0.9148942544269757, + "pass@16": 0.9250264174045647, + "pass@32": 0.9337574735241562, + "pass@64": 0.9408890741706648, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.859143175074184, + "pass@2": 0.8930812489777795, + "pass@4": 0.9152190499555503, + "pass@8": 0.9310064996161205, + "pass@16": 0.9421396953920551, + "pass@32": 0.9493720967197038, + "pass@64": 0.9534362092159403, + "pass@128": 0.9554896142433235 + }, + "15": { + "pass@1": 0.8411458333333334, + "pass@2": 0.891308888038301, + "pass@4": 0.9192842798585358, + "pass@8": 0.9347145752235823, + "pass@16": 0.9432306703458466, + "pass@32": 0.9497534745385128, + "pass@64": 0.9559486979161531, + "pass@128": 0.9598765432098766 + }, + "16": { + "pass@1": 0.657609697640118, + "pass@2": 0.7549735210089886, + "pass@4": 0.8184203282664885, + "pass@8": 0.8567489682997159, + "pass@16": 0.8811631804530315, + "pass@32": 0.8973818692788625, + "pass@64": 0.909180132438763, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.5440276731927711, + "pass@2": 0.6438123310169818, + "pass@4": 0.7184761899115022, + "pass@8": 0.7753387089561334, + "pass@16": 0.8198446720528769, + "pass@32": 0.8543886101949005, + "pass@64": 0.8801081057485678, + "pass@128": 0.8975903614457831 + }, + "18": { + "pass@1": 0.5335181451612904, + "pass@2": 0.6374031623063245, + "pass@4": 0.714073201434498, + "pass@8": 0.7698918932693165, + "pass@16": 0.8111359646908977, + "pass@32": 0.8407751276469603, + "pass@64": 0.8614316494044385, + "pass@128": 0.8806451612903226 + }, + "19": { + "pass@1": 0.31993485649546827, + "pass@2": 0.43310543390346606, + "pass@4": 0.5387721378105682, + "pass@8": 0.6259455455328414, + "pass@16": 0.6926789196867486, + "pass@32": 0.7431098194164243, + "pass@64": 0.7816634224866824, + "pass@128": 0.8157099697885196 + }, + "20": { + "pass@1": 0.29497466216216217, + "pass@2": 0.4062729067650326, + "pass@4": 0.5096795974827469, + "pass@8": 0.595516610216258, + "pass@16": 0.6623334896378719, + "pass@32": 0.7148605152551266, + "pass@64": 0.7565155108289564, + "pass@128": 0.7897897897897898 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch4-RL-op17-20/composition/graphs/composition_score_report.jsonl b/composition/Qwen3-100M-epoch4-RL-op17-20/composition/graphs/composition_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2b10e0627acae07f8a52f727de4644478264277a --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op17-20/composition/graphs/composition_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/Qwen3-100M-epoch4-RL-op17-20/composition", "results_dir": "results/composition/Qwen3-100M-epoch4-RL-op17-20/composition/graphs", "per_file": [{"file": "results/composition/Qwen3-100M-epoch4-RL-op17-20/composition/hf_model_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1025447, "value_mismatch": 7189, "dependency_mismatch": 125023, "answer_mismatch": 6720, "extra_nodes": 9555, "missing_nodes": 8328, "perfect_with_extra": 473, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 338167, "value_mismatch": 2016, "answer_mismatch": 1771, "dependency_mismatch": 40536, "extra_nodes": 2956, "perfect_with_extra": 83, "missing_nodes": 2660}, "teachers_in_school": {"total": 390656, "perfect": 347274, "dependency_mismatch": 42723, "value_mismatch": 2663, "answer_mismatch": 2681, "extra_nodes": 3523, "missing_nodes": 3112, "perfect_with_extra": 189}, "movie_festival_awards": {"total": 382336, "perfect": 340006, "value_mismatch": 2510, "dependency_mismatch": 41764, "answer_mismatch": 2268, "extra_nodes": 3076, "perfect_with_extra": 201, "missing_nodes": 2556}}, "results_path": "results/composition/Qwen3-100M-epoch4-RL-op17-20/composition/graphs/hf_model_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/Qwen3-100M-epoch4-RL-op17-20/composition/hf_model_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 525970, "value_mismatch": 422254, "dependency_mismatch": 662298, "answer_mismatch": 322709, "extra_nodes": 44396, "missing_nodes": 160582, "perfect_with_extra": 888, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 143333, "dependency_mismatch": 224569, "answer_mismatch": 110128, "extra_nodes": 16212, "perfect": 175763, "missing_nodes": 55535, "perfect_with_extra": 105}, "movie_festival_awards": {"total": 418816, "perfect": 171910, "value_mismatch": 141226, "dependency_mismatch": 216095, "answer_mismatch": 106992, "extra_nodes": 14356, "perfect_with_extra": 519, "missing_nodes": 52586}, "teachers_in_school": {"total": 430464, "perfect": 178297, "value_mismatch": 137695, "answer_mismatch": 105589, "dependency_mismatch": 221634, "extra_nodes": 13828, "missing_nodes": 52461, "perfect_with_extra": 264}}, "results_path": "results/composition/Qwen3-100M-epoch4-RL-op17-20/composition/graphs/hf_model_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1551417, "value_mismatch": 429443, "dependency_mismatch": 787321, "answer_mismatch": 329429, "extra_nodes": 53951, "missing_nodes": 168910, "perfect_with_extra": 1361, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/Qwen3-100M-epoch4-RL-op17-20/composition/graphs/composition_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 513930, "value_mismatch": 145349, "answer_mismatch": 111899, "dependency_mismatch": 265105, "extra_nodes": 19168, "perfect_with_extra": 188, "missing_nodes": 58195}, "teachers_in_school": {"total": 821120, "perfect": 525571, "dependency_mismatch": 264357, "value_mismatch": 140358, "answer_mismatch": 108270, "extra_nodes": 17351, "missing_nodes": 55573, "perfect_with_extra": 453}, "movie_festival_awards": {"total": 801152, "perfect": 511916, "value_mismatch": 143736, "dependency_mismatch": 257859, "answer_mismatch": 109260, "extra_nodes": 17432, "perfect_with_extra": 720, "missing_nodes": 55142}}}} diff --git a/composition/Qwen3-100M-epoch4-RL-op17-20/composition/graphs/results.json b/composition/Qwen3-100M-epoch4-RL-op17-20/composition/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..83e85982ec1a1db40408404d0b579ed282a605cd --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op17-20/composition/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "hf_model_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.99375, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9650793650793651, + "pass@2": 0.9809523809523809, + "pass@4": 0.9968253968253968, + "pass@8": 0.9968253968253968, + "pass@16": 0.9968253968253968, + "pass@32": 0.9968253968253968, + "pass@64": 0.9968253968253968, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9700598802395209, + "pass@2": 0.9790419161676647, + "pass@4": 0.9910179640718563, + "pass@8": 0.9940119760479041, + "pass@16": 0.9970059880239521, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8875739644970414, + "pass@2": 0.9349112426035503, + "pass@4": 0.9674556213017751, + "pass@8": 0.985207100591716, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8076923076923077, + "pass@2": 0.8782051282051282, + "pass@4": 0.9294871794871795, + "pass@8": 0.9647435897435898, + "pass@16": 0.9935897435897436, + "pass@32": 0.9967948717948718, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7380281690140845, + "pass@2": 0.8225352112676056, + "pass@4": 0.9042253521126761, + "pass@8": 0.9549295774647887, + "pass@16": 0.9802816901408451, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6666666666666666, + "pass@2": 0.7315634218289085, + "pass@4": 0.7817109144542773, + "pass@8": 0.8702064896755162, + "pass@16": 0.9085545722713865, + "pass@32": 0.9528023598820059, + "pass@64": 0.967551622418879, + "pass@128": 0.9705014749262537 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9650793650793651, + "6": 0.9700598802395209, + "7": 0.8875739644970414, + "8": 0.8076923076923077, + "9": 0.7380281690140845, + "10": 0.6666666666666666 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9809523809523809, + "6": 0.9790419161676647, + "7": 0.9349112426035503, + "8": 0.8782051282051282, + "9": 0.8225352112676056, + "10": 0.7315634218289085 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 0.9910179640718563, + "7": 0.9674556213017751, + "8": 0.9294871794871795, + "9": 0.9042253521126761, + "10": 0.7817109144542773 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 0.9940119760479041, + "7": 0.985207100591716, + "8": 0.9647435897435898, + "9": 0.9549295774647887, + "10": 0.8702064896755162 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 0.9970059880239521, + "7": 1.0, + "8": 0.9935897435897436, + "9": 0.9802816901408451, + "10": 0.9085545722713865 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 0.9971830985915493, + "10": 0.9528023598820059 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.967551622418879 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9705014749262537 + } + }, + "total_pass_at_k": { + "pass@1": 0.889564336372847, + "pass@2": 0.922661263086795, + "pass@4": 0.9510300574130361, + "pass@8": 0.9733198243836542, + "pass@16": 0.9858156028368794, + "pass@32": 0.9935832489023978, + "pass@64": 0.9959473150962512, + "pass@128": 0.9962850388382303 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9912280701754386, + "pass@2": 0.9970760233918129, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9647058823529412, + "pass@2": 0.9882352941176471, + "pass@4": 0.9970588235294118, + "pass@8": 0.9970588235294118, + "pass@16": 0.9970588235294118, + "pass@32": 0.9970588235294118, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9716088328075709, + "pass@2": 0.9810725552050473, + "pass@4": 0.9936908517350158, + "pass@8": 0.9968454258675079, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8575949367088608, + "pass@2": 0.9050632911392406, + "pass@4": 0.9493670886075949, + "pass@8": 0.9873417721518988, + "pass@16": 0.9968354430379747, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7957957957957958, + "pass@2": 0.8588588588588588, + "pass@4": 0.9219219219219219, + "pass@8": 0.960960960960961, + "pass@16": 0.9819819819819819, + "pass@32": 0.996996996996997, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.7652733118971061, + "pass@2": 0.8360128617363344, + "pass@4": 0.9035369774919614, + "pass@8": 0.9485530546623794, + "pass@16": 0.977491961414791, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6, + "pass@2": 0.6656716417910448, + "pass@4": 0.7492537313432835, + "pass@8": 0.844776119402985, + "pass@16": 0.9194029850746268, + "pass@32": 0.9522388059701492, + "pass@64": 0.9671641791044776, + "pass@128": 0.9761194029850746 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9647058823529412, + "6": 0.9716088328075709, + "7": 0.8575949367088608, + "8": 0.7957957957957958, + "9": 0.7652733118971061, + "10": 0.6 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9882352941176471, + "6": 0.9810725552050473, + "7": 0.9050632911392406, + "8": 0.8588588588588588, + "9": 0.8360128617363344, + "10": 0.6656716417910448 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9936908517350158, + "7": 0.9493670886075949, + "8": 0.9219219219219219, + "9": 0.9035369774919614, + "10": 0.7492537313432835 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9968454258675079, + "7": 0.9873417721518988, + "8": 0.960960960960961, + "9": 0.9485530546623794, + "10": 0.844776119402985 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.9819819819819819, + "9": 0.977491961414791, + "10": 0.9194029850746268 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9522388059701492 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9671641791044776 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9761194029850746 + } + }, + "total_pass_at_k": { + "pass@1": 0.8848342818881821, + "pass@2": 0.9159691998660864, + "pass@4": 0.9467693337797121, + "pass@8": 0.970873786407767, + "pass@16": 0.9859390693003013, + "pass@32": 0.9939738868429863, + "pass@64": 0.9959825912286575, + "pass@128": 0.9969869434214932 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9822485207100592, + "pass@2": 0.9911242603550295, + "pass@4": 0.9970414201183432, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9536231884057971, + "pass@2": 0.9768115942028985, + "pass@4": 0.991304347826087, + "pass@8": 0.9942028985507246, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9799426934097422, + "pass@2": 0.9799426934097422, + "pass@4": 0.9885386819484241, + "pass@8": 0.997134670487106, + "pass@16": 0.997134670487106, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8410404624277457, + "pass@2": 0.9017341040462428, + "pass@4": 0.9653179190751445, + "pass@8": 0.9884393063583815, + "pass@16": 0.9942196531791907, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.8112676056338028, + "pass@2": 0.8901408450704226, + "pass@4": 0.9436619718309859, + "pass@8": 0.9802816901408451, + "pass@16": 0.9915492957746479, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7544910179640718, + "pass@2": 0.8383233532934131, + "pass@4": 0.9191616766467066, + "pass@8": 0.9580838323353293, + "pass@16": 0.9880239520958084, + "pass@32": 0.9970059880239521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.656441717791411, + "pass@2": 0.7177914110429447, + "pass@4": 0.7883435582822086, + "pass@8": 0.8558282208588958, + "pass@16": 0.9202453987730062, + "pass@32": 0.9539877300613497, + "pass@64": 0.9631901840490797, + "pass@128": 0.9754601226993865 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9822485207100592, + "5": 0.9536231884057971, + "6": 0.9799426934097422, + "7": 0.8410404624277457, + "8": 0.8112676056338028, + "9": 0.7544910179640718, + "10": 0.656441717791411 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9911242603550295, + "5": 0.9768115942028985, + "6": 0.9799426934097422, + "7": 0.9017341040462428, + "8": 0.8901408450704226, + "9": 0.8383233532934131, + "10": 0.7177914110429447 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.991304347826087, + "6": 0.9885386819484241, + "7": 0.9653179190751445, + "8": 0.9436619718309859, + "9": 0.9191616766467066, + "10": 0.7883435582822086 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9942028985507246, + "6": 0.997134670487106, + "7": 0.9884393063583815, + "8": 0.9802816901408451, + "9": 0.9580838323353293, + "10": 0.8558282208588958 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9942196531791907, + "8": 0.9915492957746479, + "9": 0.9880239520958084, + "10": 0.9202453987730062 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9971830985915493, + "9": 0.9970059880239521, + "10": 0.9539877300613497 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 1.0, + "9": 1.0, + "10": 0.9631901840490797 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 1.0, + "9": 1.0, + "10": 0.9754601226993865 + } + }, + "total_pass_at_k": { + "pass@1": 0.8869593709043251, + "pass@2": 0.922346002621232, + "pass@4": 0.9554390563564875, + "pass@8": 0.9754259501965924, + "pass@16": 0.9882044560943644, + "pass@32": 0.9941022280471822, + "pass@64": 0.995740498034076, + "pass@128": 0.997051114023591 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.989, + "pass@2": 0.994, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.961, + "pass@2": 0.982, + "pass@4": 0.995, + "pass@8": 0.996, + "pass@16": 0.998, + "pass@32": 0.998, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.974, + "pass@2": 0.98, + "pass@4": 0.991, + "pass@8": 0.996, + "pass@16": 0.998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.862, + "pass@2": 0.914, + "pass@4": 0.961, + "pass@8": 0.987, + "pass@16": 0.997, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.805, + "pass@2": 0.876, + "pass@4": 0.932, + "pass@8": 0.969, + "pass@16": 0.989, + "pass@32": 0.997, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.752, + "pass@2": 0.832, + "pass@4": 0.909, + "pass@8": 0.954, + "pass@16": 0.982, + "pass@32": 0.998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.641, + "pass@2": 0.705, + "pass@4": 0.773, + "pass@8": 0.857, + "pass@16": 0.916, + "pass@32": 0.953, + "pass@64": 0.966, + "pass@128": 0.974 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.989, + "5": 0.961, + "6": 0.974, + "7": 0.862, + "8": 0.805, + "9": 0.752, + "10": 0.641 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.994, + "5": 0.982, + "6": 0.98, + "7": 0.914, + "8": 0.876, + "9": 0.832, + "10": 0.705 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.995, + "6": 0.991, + "7": 0.961, + "8": 0.932, + "9": 0.909, + "10": 0.773 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.996, + "6": 0.996, + "7": 0.987, + "8": 0.969, + "9": 0.954, + "10": 0.857 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 0.998, + "7": 0.997, + "8": 0.989, + "9": 0.982, + "10": 0.916 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 1.0, + "7": 0.999, + "8": 0.997, + "9": 0.998, + "10": 0.953 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.999, + "8": 0.999, + "9": 1.0, + "10": 0.966 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.999, + "8": 0.999, + "9": 1.0, + "10": 0.974 + } + }, + "total_pass_at_k": { + "pass@1": 0.8871111111111111, + "pass@2": 0.9203333333333333, + "pass@4": 0.9511111111111111, + "pass@8": 0.9732222222222222, + "pass@16": 0.9866666666666667, + "pass@32": 0.9938888888888889, + "pass@64": 0.9958888888888889, + "pass@128": 0.9967777777777778 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "hf_model_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5524079320113314, + "pass@2": 0.6487252124645893, + "pass@4": 0.7450424929178471, + "pass@8": 0.8328611898016998, + "pass@16": 0.8895184135977338, + "pass@32": 0.9405099150141643, + "pass@64": 0.9546742209631728, + "pass@128": 0.9546742209631728 + }, + "12": { + "pass@1": 0.6299435028248588, + "pass@2": 0.6807909604519774, + "pass@4": 0.7344632768361582, + "pass@8": 0.7966101694915254, + "pass@16": 0.8389830508474576, + "pass@32": 0.8615819209039548, + "pass@64": 0.8700564971751412, + "pass@128": 0.8785310734463276 + }, + "13": { + "pass@1": 0.5109717868338558, + "pass@2": 0.5830721003134797, + "pass@4": 0.6614420062695925, + "pass@8": 0.7398119122257053, + "pass@16": 0.786833855799373, + "pass@32": 0.8119122257053292, + "pass@64": 0.8244514106583072, + "pass@128": 0.8338557993730408 + }, + "14": { + "pass@1": 0.4797507788161994, + "pass@2": 0.5202492211838006, + "pass@4": 0.5638629283489096, + "pass@8": 0.6230529595015576, + "pass@16": 0.7133956386292835, + "pass@32": 0.7601246105919003, + "pass@64": 0.7850467289719626, + "pass@128": 0.8255451713395638 + }, + "15": { + "pass@1": 0.430635838150289, + "pass@2": 0.4797687861271676, + "pass@4": 0.5346820809248555, + "pass@8": 0.5924855491329479, + "pass@16": 0.6473988439306358, + "pass@32": 0.7543352601156069, + "pass@64": 0.8352601156069365, + "pass@128": 0.8757225433526011 + }, + "16": { + "pass@1": 0.36774193548387096, + "pass@2": 0.4290322580645161, + "pass@4": 0.4645161290322581, + "pass@8": 0.5, + "pass@16": 0.5483870967741935, + "pass@32": 0.603225806451613, + "pass@64": 0.6483870967741936, + "pass@128": 0.6774193548387096 + }, + "17": { + "pass@1": 0.35260115606936415, + "pass@2": 0.40173410404624277, + "pass@4": 0.430635838150289, + "pass@8": 0.48265895953757226, + "pass@16": 0.5491329479768786, + "pass@32": 0.5895953757225434, + "pass@64": 0.630057803468208, + "pass@128": 0.6936416184971098 + }, + "18": { + "pass@1": 0.3057142857142857, + "pass@2": 0.3342857142857143, + "pass@4": 0.36857142857142855, + "pass@8": 0.4228571428571429, + "pass@16": 0.4514285714285714, + "pass@32": 0.5057142857142857, + "pass@64": 0.58, + "pass@128": 0.6314285714285715 + }, + "19": { + "pass@1": 0.2042042042042042, + "pass@2": 0.24924924924924924, + "pass@4": 0.2822822822822823, + "pass@8": 0.3033033033033033, + "pass@16": 0.32432432432432434, + "pass@32": 0.3663663663663664, + "pass@64": 0.4114114114114114, + "pass@128": 0.45345345345345345 + }, + "20": { + "pass@1": 0.2222222222222222, + "pass@2": 0.25825825825825827, + "pass@4": 0.2882882882882883, + "pass@8": 0.32732732732732733, + "pass@16": 0.37237237237237236, + "pass@32": 0.4114114114114114, + "pass@64": 0.45045045045045046, + "pass@128": 0.4864864864864865 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5524079320113314, + "12": 0.6299435028248588, + "13": 0.5109717868338558, + "14": 0.4797507788161994, + "15": 0.430635838150289, + "16": 0.36774193548387096, + "17": 0.35260115606936415, + "18": 0.3057142857142857, + "19": 0.2042042042042042, + "20": 0.2222222222222222 + }, + "pass@2": { + "11": 0.6487252124645893, + "12": 0.6807909604519774, + "13": 0.5830721003134797, + "14": 0.5202492211838006, + "15": 0.4797687861271676, + "16": 0.4290322580645161, + "17": 0.40173410404624277, + "18": 0.3342857142857143, + "19": 0.24924924924924924, + "20": 0.25825825825825827 + }, + "pass@4": { + "11": 0.7450424929178471, + "12": 0.7344632768361582, + "13": 0.6614420062695925, + "14": 0.5638629283489096, + "15": 0.5346820809248555, + "16": 0.4645161290322581, + "17": 0.430635838150289, + "18": 0.36857142857142855, + "19": 0.2822822822822823, + "20": 0.2882882882882883 + }, + "pass@8": { + "11": 0.8328611898016998, + "12": 0.7966101694915254, + "13": 0.7398119122257053, + "14": 0.6230529595015576, + "15": 0.5924855491329479, + "16": 0.5, + "17": 0.48265895953757226, + "18": 0.4228571428571429, + "19": 0.3033033033033033, + "20": 0.32732732732732733 + }, + "pass@16": { + "11": 0.8895184135977338, + "12": 0.8389830508474576, + "13": 0.786833855799373, + "14": 0.7133956386292835, + "15": 0.6473988439306358, + "16": 0.5483870967741935, + "17": 0.5491329479768786, + "18": 0.4514285714285714, + "19": 0.32432432432432434, + "20": 0.37237237237237236 + }, + "pass@32": { + "11": 0.9405099150141643, + "12": 0.8615819209039548, + "13": 0.8119122257053292, + "14": 0.7601246105919003, + "15": 0.7543352601156069, + "16": 0.603225806451613, + "17": 0.5895953757225434, + "18": 0.5057142857142857, + "19": 0.3663663663663664, + "20": 0.4114114114114114 + }, + "pass@64": { + "11": 0.9546742209631728, + "12": 0.8700564971751412, + "13": 0.8244514106583072, + "14": 0.7850467289719626, + "15": 0.8352601156069365, + "16": 0.6483870967741936, + "17": 0.630057803468208, + "18": 0.58, + "19": 0.4114114114114114, + "20": 0.45045045045045046 + }, + "pass@128": { + "11": 0.9546742209631728, + "12": 0.8785310734463276, + "13": 0.8338557993730408, + "14": 0.8255451713395638, + "15": 0.8757225433526011, + "16": 0.6774193548387096, + "17": 0.6936416184971098, + "18": 0.6314285714285715, + "19": 0.45345345345345345, + "20": 0.4864864864864865 + } + }, + "total_pass_at_k": { + "pass@1": 0.40683506686478454, + "pass@2": 0.45973254086181276, + "pass@4": 0.5087667161961367, + "pass@8": 0.5637444279346211, + "pass@16": 0.6136701337295691, + "pass@32": 0.662109955423477, + "pass@64": 0.70074294205052, + "pass@128": 0.7328380386329866 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5841269841269842, + "pass@2": 0.6761904761904762, + "pass@4": 0.7714285714285715, + "pass@8": 0.8444444444444444, + "pass@16": 0.9206349206349206, + "pass@32": 0.9523809523809523, + "pass@64": 0.9746031746031746, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.6116207951070336, + "pass@2": 0.6758409785932722, + "pass@4": 0.7522935779816514, + "pass@8": 0.8287461773700305, + "pass@16": 0.8623853211009175, + "pass@32": 0.8929663608562691, + "pass@64": 0.908256880733945, + "pass@128": 0.9113149847094801 + }, + "13": { + "pass@1": 0.5401234567901234, + "pass@2": 0.5740740740740741, + "pass@4": 0.6203703703703703, + "pass@8": 0.6728395061728395, + "pass@16": 0.7129629629629629, + "pass@32": 0.7530864197530864, + "pass@64": 0.7746913580246914, + "pass@128": 0.7839506172839507 + }, + "14": { + "pass@1": 0.4658753709198813, + "pass@2": 0.49851632047477745, + "pass@4": 0.5727002967359051, + "pass@8": 0.6023738872403561, + "pass@16": 0.6735905044510386, + "pass@32": 0.7477744807121661, + "pass@64": 0.8041543026706232, + "pass@128": 0.8367952522255193 + }, + "15": { + "pass@1": 0.4166666666666667, + "pass@2": 0.44135802469135804, + "pass@4": 0.49074074074074076, + "pass@8": 0.558641975308642, + "pass@16": 0.6388888888888888, + "pass@32": 0.7129629629629629, + "pass@64": 0.7839506172839507, + "pass@128": 0.845679012345679 + }, + "16": { + "pass@1": 0.40117994100294985, + "pass@2": 0.43952802359882004, + "pass@4": 0.4896755162241888, + "pass@8": 0.528023598820059, + "pass@16": 0.5722713864306784, + "pass@32": 0.6135693215339233, + "pass@64": 0.6548672566371682, + "pass@128": 0.696165191740413 + }, + "17": { + "pass@1": 0.33433734939759036, + "pass@2": 0.37349397590361444, + "pass@4": 0.39457831325301207, + "pass@8": 0.44879518072289154, + "pass@16": 0.49096385542168675, + "pass@32": 0.5481927710843374, + "pass@64": 0.5993975903614458, + "pass@128": 0.6355421686746988 + }, + "18": { + "pass@1": 0.33225806451612905, + "pass@2": 0.3870967741935484, + "pass@4": 0.4096774193548387, + "pass@8": 0.44193548387096776, + "pass@16": 0.4774193548387097, + "pass@32": 0.5096774193548387, + "pass@64": 0.567741935483871, + "pass@128": 0.6451612903225806 + }, + "19": { + "pass@1": 0.21450151057401812, + "pass@2": 0.26586102719033233, + "pass@4": 0.3081570996978852, + "pass@8": 0.32628398791540786, + "pass@16": 0.3504531722054381, + "pass@32": 0.3776435045317221, + "pass@64": 0.41389728096676737, + "pass@128": 0.459214501510574 + }, + "20": { + "pass@1": 0.1921921921921922, + "pass@2": 0.24024024024024024, + "pass@4": 0.27627627627627627, + "pass@8": 0.3063063063063063, + "pass@16": 0.3333333333333333, + "pass@32": 0.36036036036036034, + "pass@64": 0.4084084084084084, + "pass@128": 0.43843843843843844 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5841269841269842, + "12": 0.6116207951070336, + "13": 0.5401234567901234, + "14": 0.4658753709198813, + "15": 0.4166666666666667, + "16": 0.40117994100294985, + "17": 0.33433734939759036, + "18": 0.33225806451612905, + "19": 0.21450151057401812, + "20": 0.1921921921921922 + }, + "pass@2": { + "11": 0.6761904761904762, + "12": 0.6758409785932722, + "13": 0.5740740740740741, + "14": 0.49851632047477745, + "15": 0.44135802469135804, + "16": 0.43952802359882004, + "17": 0.37349397590361444, + "18": 0.3870967741935484, + "19": 0.26586102719033233, + "20": 0.24024024024024024 + }, + "pass@4": { + "11": 0.7714285714285715, + "12": 0.7522935779816514, + "13": 0.6203703703703703, + "14": 0.5727002967359051, + "15": 0.49074074074074076, + "16": 0.4896755162241888, + "17": 0.39457831325301207, + "18": 0.4096774193548387, + "19": 0.3081570996978852, + "20": 0.27627627627627627 + }, + "pass@8": { + "11": 0.8444444444444444, + "12": 0.8287461773700305, + "13": 0.6728395061728395, + "14": 0.6023738872403561, + "15": 0.558641975308642, + "16": 0.528023598820059, + "17": 0.44879518072289154, + "18": 0.44193548387096776, + "19": 0.32628398791540786, + "20": 0.3063063063063063 + }, + "pass@16": { + "11": 0.9206349206349206, + "12": 0.8623853211009175, + "13": 0.7129629629629629, + "14": 0.6735905044510386, + "15": 0.6388888888888888, + "16": 0.5722713864306784, + "17": 0.49096385542168675, + "18": 0.4774193548387097, + "19": 0.3504531722054381, + "20": 0.3333333333333333 + }, + "pass@32": { + "11": 0.9523809523809523, + "12": 0.8929663608562691, + "13": 0.7530864197530864, + "14": 0.7477744807121661, + "15": 0.7129629629629629, + "16": 0.6135693215339233, + "17": 0.5481927710843374, + "18": 0.5096774193548387, + "19": 0.3776435045317221, + "20": 0.36036036036036034 + }, + "pass@64": { + "11": 0.9746031746031746, + "12": 0.908256880733945, + "13": 0.7746913580246914, + "14": 0.8041543026706232, + "15": 0.7839506172839507, + "16": 0.6548672566371682, + "17": 0.5993975903614458, + "18": 0.567741935483871, + "19": 0.41389728096676737, + "20": 0.4084084084084084 + }, + "pass@128": { + "11": 0.9841269841269841, + "12": 0.9113149847094801, + "13": 0.7839506172839507, + "14": 0.8367952522255193, + "15": 0.845679012345679, + "16": 0.696165191740413, + "17": 0.6355421686746988, + "18": 0.6451612903225806, + "19": 0.459214501510574, + "20": 0.43843843843843844 + } + }, + "total_pass_at_k": { + "pass@1": 0.4083129584352078, + "pass@2": 0.4559902200488998, + "pass@4": 0.5073349633251834, + "pass@8": 0.55440097799511, + "pass@16": 0.6017726161369193, + "pass@32": 0.6454767726161369, + "pass@64": 0.6876528117359413, + "pass@128": 0.7221882640586798 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6536144578313253, + "pass@2": 0.7018072289156626, + "pass@4": 0.7530120481927711, + "pass@8": 0.8433734939759037, + "pass@16": 0.9036144578313253, + "pass@32": 0.9337349397590361, + "pass@64": 0.9608433734939759, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.554858934169279, + "pass@2": 0.6238244514106583, + "pass@4": 0.7084639498432602, + "pass@8": 0.7648902821316614, + "pass@16": 0.8401253918495298, + "pass@32": 0.8557993730407524, + "pass@64": 0.8746081504702194, + "pass@128": 0.8871473354231975 + }, + "13": { + "pass@1": 0.5406162464985994, + "pass@2": 0.6078431372549019, + "pass@4": 0.6778711484593838, + "pass@8": 0.7591036414565826, + "pass@16": 0.7983193277310925, + "pass@32": 0.834733893557423, + "pass@64": 0.8627450980392157, + "pass@128": 0.8711484593837535 + }, + "14": { + "pass@1": 0.49122807017543857, + "pass@2": 0.543859649122807, + "pass@4": 0.5877192982456141, + "pass@8": 0.6608187134502924, + "pass@16": 0.716374269005848, + "pass@32": 0.7777777777777778, + "pass@64": 0.8333333333333334, + "pass@128": 0.8654970760233918 + }, + "15": { + "pass@1": 0.4666666666666667, + "pass@2": 0.5151515151515151, + "pass@4": 0.5848484848484848, + "pass@8": 0.6393939393939394, + "pass@16": 0.703030303030303, + "pass@32": 0.7484848484848485, + "pass@64": 0.8333333333333334, + "pass@128": 0.8757575757575757 + }, + "16": { + "pass@1": 0.39886039886039887, + "pass@2": 0.4472934472934473, + "pass@4": 0.49572649572649574, + "pass@8": 0.5612535612535613, + "pass@16": 0.603988603988604, + "pass@32": 0.6552706552706553, + "pass@64": 0.7094017094017094, + "pass@128": 0.7378917378917379 + }, + "17": { + "pass@1": 0.2919254658385093, + "pass@2": 0.3447204968944099, + "pass@4": 0.37888198757763975, + "pass@8": 0.4440993788819876, + "pass@16": 0.48757763975155277, + "pass@32": 0.5683229813664596, + "pass@64": 0.6273291925465838, + "pass@128": 0.6925465838509317 + }, + "18": { + "pass@1": 0.29705882352941176, + "pass@2": 0.32941176470588235, + "pass@4": 0.3558823529411765, + "pass@8": 0.3911764705882353, + "pass@16": 0.4294117647058823, + "pass@32": 0.49117647058823527, + "pass@64": 0.5588235294117647, + "pass@128": 0.6029411764705882 + }, + "19": { + "pass@1": 0.2261904761904762, + "pass@2": 0.28273809523809523, + "pass@4": 0.32142857142857145, + "pass@8": 0.35714285714285715, + "pass@16": 0.38095238095238093, + "pass@32": 0.4107142857142857, + "pass@64": 0.44345238095238093, + "pass@128": 0.4880952380952381 + }, + "20": { + "pass@1": 0.19760479041916168, + "pass@2": 0.25449101796407186, + "pass@4": 0.2934131736526946, + "pass@8": 0.32335329341317365, + "pass@16": 0.3473053892215569, + "pass@32": 0.39221556886227543, + "pass@64": 0.41916167664670656, + "pass@128": 0.47604790419161674 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6536144578313253, + "12": 0.554858934169279, + "13": 0.5406162464985994, + "14": 0.49122807017543857, + "15": 0.4666666666666667, + "16": 0.39886039886039887, + "17": 0.2919254658385093, + "18": 0.29705882352941176, + "19": 0.2261904761904762, + "20": 0.19760479041916168 + }, + "pass@2": { + "11": 0.7018072289156626, + "12": 0.6238244514106583, + "13": 0.6078431372549019, + "14": 0.543859649122807, + "15": 0.5151515151515151, + "16": 0.4472934472934473, + "17": 0.3447204968944099, + "18": 0.32941176470588235, + "19": 0.28273809523809523, + "20": 0.25449101796407186 + }, + "pass@4": { + "11": 0.7530120481927711, + "12": 0.7084639498432602, + "13": 0.6778711484593838, + "14": 0.5877192982456141, + "15": 0.5848484848484848, + "16": 0.49572649572649574, + "17": 0.37888198757763975, + "18": 0.3558823529411765, + "19": 0.32142857142857145, + "20": 0.2934131736526946 + }, + "pass@8": { + "11": 0.8433734939759037, + "12": 0.7648902821316614, + "13": 0.7591036414565826, + "14": 0.6608187134502924, + "15": 0.6393939393939394, + "16": 0.5612535612535613, + "17": 0.4440993788819876, + "18": 0.3911764705882353, + "19": 0.35714285714285715, + "20": 0.32335329341317365 + }, + "pass@16": { + "11": 0.9036144578313253, + "12": 0.8401253918495298, + "13": 0.7983193277310925, + "14": 0.716374269005848, + "15": 0.703030303030303, + "16": 0.603988603988604, + "17": 0.48757763975155277, + "18": 0.4294117647058823, + "19": 0.38095238095238093, + "20": 0.3473053892215569 + }, + "pass@32": { + "11": 0.9337349397590361, + "12": 0.8557993730407524, + "13": 0.834733893557423, + "14": 0.7777777777777778, + "15": 0.7484848484848485, + "16": 0.6552706552706553, + "17": 0.5683229813664596, + "18": 0.49117647058823527, + "19": 0.4107142857142857, + "20": 0.39221556886227543 + }, + "pass@64": { + "11": 0.9608433734939759, + "12": 0.8746081504702194, + "13": 0.8627450980392157, + "14": 0.8333333333333334, + "15": 0.8333333333333334, + "16": 0.7094017094017094, + "17": 0.6273291925465838, + "18": 0.5588235294117647, + "19": 0.44345238095238093, + "20": 0.41916167664670656 + }, + "pass@128": { + "11": 0.9698795180722891, + "12": 0.8871473354231975, + "13": 0.8711484593837535, + "14": 0.8654970760233918, + "15": 0.8757575757575757, + "16": 0.7378917378917379, + "17": 0.6925465838509317, + "18": 0.6029411764705882, + "19": 0.4880952380952381, + "20": 0.47604790419161674 + } + }, + "total_pass_at_k": { + "pass@1": 0.41213202497769846, + "pass@2": 0.46535831103181685, + "pass@4": 0.5159084151055605, + "pass@8": 0.5747844186738031, + "pass@16": 0.6211715730002974, + "pass@32": 0.6669640202200416, + "pass@64": 0.7124591138864109, + "pass@128": 0.7466547725245317 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.596, + "pass@2": 0.675, + "pass@4": 0.756, + "pass@8": 0.84, + "pass@16": 0.904, + "pass@32": 0.942, + "pass@64": 0.963, + "pass@128": 0.969 + }, + "12": { + "pass@1": 0.6, + "pass@2": 0.661, + "pass@4": 0.732, + "pass@8": 0.797, + "pass@16": 0.847, + "pass@32": 0.87, + "pass@64": 0.884, + "pass@128": 0.892 + }, + "13": { + "pass@1": 0.531, + "pass@2": 0.589, + "pass@4": 0.654, + "pass@8": 0.725, + "pass@16": 0.767, + "pass@32": 0.801, + "pass@64": 0.822, + "pass@128": 0.831 + }, + "14": { + "pass@1": 0.479, + "pass@2": 0.521, + "pass@4": 0.575, + "pass@8": 0.629, + "pass@16": 0.701, + "pass@32": 0.762, + "pass@64": 0.808, + "pass@128": 0.843 + }, + "15": { + "pass@1": 0.438, + "pass@2": 0.479, + "pass@4": 0.537, + "pass@8": 0.597, + "pass@16": 0.663, + "pass@32": 0.739, + "pass@64": 0.818, + "pass@128": 0.866 + }, + "16": { + "pass@1": 0.39, + "pass@2": 0.439, + "pass@4": 0.484, + "pass@8": 0.531, + "pass@16": 0.576, + "pass@32": 0.625, + "pass@64": 0.672, + "pass@128": 0.705 + }, + "17": { + "pass@1": 0.327, + "pass@2": 0.374, + "pass@4": 0.402, + "pass@8": 0.459, + "pass@16": 0.51, + "pass@32": 0.569, + "pass@64": 0.619, + "pass@128": 0.674 + }, + "18": { + "pass@1": 0.311, + "pass@2": 0.349, + "pass@4": 0.377, + "pass@8": 0.418, + "pass@16": 0.452, + "pass@32": 0.502, + "pass@64": 0.569, + "pass@128": 0.626 + }, + "19": { + "pass@1": 0.215, + "pass@2": 0.266, + "pass@4": 0.304, + "pass@8": 0.329, + "pass@16": 0.352, + "pass@32": 0.385, + "pass@64": 0.423, + "pass@128": 0.467 + }, + "20": { + "pass@1": 0.204, + "pass@2": 0.251, + "pass@4": 0.286, + "pass@8": 0.319, + "pass@16": 0.351, + "pass@32": 0.388, + "pass@64": 0.426, + "pass@128": 0.467 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.596, + "12": 0.6, + "13": 0.531, + "14": 0.479, + "15": 0.438, + "16": 0.39, + "17": 0.327, + "18": 0.311, + "19": 0.215, + "20": 0.204 + }, + "pass@2": { + "11": 0.675, + "12": 0.661, + "13": 0.589, + "14": 0.521, + "15": 0.479, + "16": 0.439, + "17": 0.374, + "18": 0.349, + "19": 0.266, + "20": 0.251 + }, + "pass@4": { + "11": 0.756, + "12": 0.732, + "13": 0.654, + "14": 0.575, + "15": 0.537, + "16": 0.484, + "17": 0.402, + "18": 0.377, + "19": 0.304, + "20": 0.286 + }, + "pass@8": { + "11": 0.84, + "12": 0.797, + "13": 0.725, + "14": 0.629, + "15": 0.597, + "16": 0.531, + "17": 0.459, + "18": 0.418, + "19": 0.329, + "20": 0.319 + }, + "pass@16": { + "11": 0.904, + "12": 0.847, + "13": 0.767, + "14": 0.701, + "15": 0.663, + "16": 0.576, + "17": 0.51, + "18": 0.452, + "19": 0.352, + "20": 0.351 + }, + "pass@32": { + "11": 0.942, + "12": 0.87, + "13": 0.801, + "14": 0.762, + "15": 0.739, + "16": 0.625, + "17": 0.569, + "18": 0.502, + "19": 0.385, + "20": 0.388 + }, + "pass@64": { + "11": 0.963, + "12": 0.884, + "13": 0.822, + "14": 0.808, + "15": 0.818, + "16": 0.672, + "17": 0.619, + "18": 0.569, + "19": 0.423, + "20": 0.426 + }, + "pass@128": { + "11": 0.969, + "12": 0.892, + "13": 0.831, + "14": 0.843, + "15": 0.866, + "16": 0.705, + "17": 0.674, + "18": 0.626, + "19": 0.467, + "20": 0.467 + } + }, + "total_pass_at_k": { + "pass@1": 0.4091, + "pass@2": 0.4604, + "pass@4": 0.5107, + "pass@8": 0.5644, + "pass@16": 0.6123, + "pass@32": 0.6583, + "pass@64": 0.7004, + "pass@128": 0.734 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/Qwen3-100M-epoch4-RL-op17-20/composition/hf_model_metrics.json b/composition/Qwen3-100M-epoch4-RL-op17-20/composition/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..fffba95100aa60807928821a50d7fbfecc5888cd --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op17-20/composition/hf_model_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.9941666666666666, + "avg_response_len": 199.51870659722223, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9941666666666666, + "pass@2": 0.9958068952318455, + "pass@4": 0.996966953859934, + "pass@8": 0.9977300520098478, + "pass@16": 0.9982031379013875, + "pass@32": 0.9985849198979351, + "pass@64": 0.9989939773444118, + "pass@128": 0.9993333333333333 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9616484375, + "pass@2": 0.9712715305118106, + "pass@4": 0.9786533301462318, + "pass@8": 0.983622736665287, + "pass@16": 0.9868989010859961, + "pass@32": 0.9895558252700736, + "pass@64": 0.9923754338993602, + "pass@128": 0.995 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99828125, + "pass@2": 0.9989266732283465, + "pass@4": 0.9994362400637421, + "pass@8": 0.9998281526533629, + "pass@16": 0.9999821832367909, + "pass@32": 0.9999998582404139, + "pass@64": 0.9999999999997746, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999609375, + "pass@2": 0.9999995078740158, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9986953125, + "pass@2": 0.9989669045275591, + "pass@4": 0.9989991694788152, + "pass@8": 0.9989999996570519, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.9932734375, + "pass@2": 0.9953978838582678, + "pass@4": 0.9965545253093364, + "pass@8": 0.9973638473817131, + "pass@16": 0.9979768743591723, + "pass@32": 0.9987090283739593, + "pass@64": 0.9995703622047244, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9956484375, + "pass@2": 0.9976995570866142, + "pass@4": 0.9990593197412824, + "pass@8": 0.9997557317312142, + "pass@16": 0.9999702824305289, + "pass@32": 0.9999995671969636, + "pass@64": 0.9999999999958465, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9616484375, + "2": 1.0, + "3": 1.0, + "4": 0.9999921875, + "5": 0.99828125, + "6": 0.9999609375, + "7": 0.9986953125, + "8": 0.9932734375, + "9": 0.9956484375 + }, + "per_op_avg_response_len": { + "10": 280.8438359375, + "2": 115.3596875, + "3": 144.295109375, + "4": 159.4446640625, + "5": 172.1992421875, + "6": 198.2354296875, + "7": 223.3115078125, + "8": 243.026265625, + "9": 258.9526171875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.8438359375, + "2": 115.3596875, + "3": 144.295109375, + "4": 159.4446640625, + "5": 172.1992421875, + "6": 198.2354296875, + "7": 223.3115078125, + "8": 243.026265625, + "9": 258.9526171875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 377237, + "answer_accuracy": 0.9953272754137116, + "avg_response_len": 189.93479821006417, + "resp_tokens_sum": 71986808, + "pass_at_k": { + "pass@1": 0.9953272754137116, + "pass@2": 0.9968100729430098, + "pass@4": 0.9977169102595711, + "pass@8": 0.9982487930861937, + "pass@16": 0.9985337231115786, + "pass@32": 0.9987765305288758, + "pass@64": 0.999157020264595, + "pass@128": 0.999662276258021 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9683582227138643, + "pass@2": 0.9777432757299143, + "pass@4": 0.9832795228583151, + "pass@8": 0.9858626452225466, + "pass@16": 0.9873392250168272, + "pass@32": 0.9893153006929356, + "pass@64": 0.9926369823240244, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.997470238095238, + "pass@2": 0.9980275903012124, + "pass@4": 0.9987849956255469, + "pass@8": 0.9995512543309223, + "pass@16": 0.9999456111182645, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9958433493589743, + "pass@2": 0.9980196661114474, + "pass@4": 0.9991082061256767, + "pass@8": 0.9997716496433244, + "pass@16": 0.9999869239709128, + "pass@32": 0.9999999812155175, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9972931338028169, + "pass@2": 0.9981389320173008, + "pass@4": 0.9987858566094732, + "pass@8": 0.9994925064642773, + "pass@16": 0.9999199128774705, + "pass@32": 0.9999987816218918, + "pass@64": 0.9999999999883001, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387975, + "answer_accuracy": 0.9931371846330275, + "avg_response_len": 197.5918301523591, + "resp_tokens_sum": 77190434, + "pass_at_k": { + "pass@1": 0.9931371846330275, + "pass@2": 0.9949155178352133, + "pass@4": 0.9960758765635129, + "pass@8": 0.9966983575278021, + "pass@16": 0.9970701582932401, + "pass@32": 0.9974654318362896, + "pass@64": 0.9979553421026679, + "pass@128": 0.9983617300131061 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9523820935582822, + "pass@2": 0.9626954162842377, + "pass@4": 0.9704093263265402, + "pass@8": 0.9749882039712388, + "pass@16": 0.9779861413506765, + "pass@32": 0.9810598822693485, + "pass@64": 0.9846863055012435, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994791666666667, + "pass@2": 0.999952570466735, + "pass@4": 0.9999996101030848, + "pass@8": 0.9999999999869519, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999328438395415, + "pass@2": 0.99999894242267, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9965679190751445, + "pass@2": 0.9970198642528788, + "pass@4": 0.9971074275939497, + "pass@8": 0.9971098255984155, + "pass@16": 0.9971098265895953, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9943441901408451, + "pass@2": 0.9960425446379061, + "pass@4": 0.996916455425466, + "pass@8": 0.9974621349733664, + "pass@16": 0.9978447220313824, + "pass@32": 0.9984196516998427, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9939651946107785, + "pass@2": 0.9972940438021596, + "pass@4": 0.999298675988855, + "pass@8": 0.9999346560243496, + "pass@16": 0.999999295537448, + "pass@32": 0.9999999999733643, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380068, + "answer_accuracy": 0.9940680448610646, + "avg_response_len": 210.98800008369602, + "resp_tokens_sum": 80668308, + "pass_at_k": { + "pass@1": 0.9940680448610646, + "pass@2": 0.9957232241946071, + "pass@4": 0.9971339933675852, + "pass@8": 0.9982699714046059, + "pass@16": 0.9990330645691781, + "pass@32": 0.999538826321076, + "pass@64": 0.9998935905587203, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9638759328358208, + "pass@2": 0.9730682218827124, + "pass@4": 0.9819944224322704, + "pass@8": 0.9897586431051344, + "pass@16": 0.9951266320148406, + "pass@32": 0.998066915568241, + "pass@64": 0.9995933173077908, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9978170955882353, + "pass@2": 0.9987186631542381, + "pass@4": 0.9994679439886189, + "pass@8": 0.9999103163047764, + "pass@16": 0.9999979874545221, + "pass@32": 0.9999999996892099, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997527689873418, + "pass@2": 0.9999949385776936, + "pass@4": 0.9999999985167994, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9897240990990991, + "pass@2": 0.9922541882433614, + "pass@4": 0.9937760460122664, + "pass@8": 0.9950031074397921, + "pass@16": 0.9962344677450653, + "pass@32": 0.9978079816254464, + "pass@64": 0.9994546357381001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9955787781350482, + "pass@2": 0.9976335227992, + "pass@4": 0.9991144143557619, + "pass@8": 0.9998640412355725, + "pass@16": 0.99999661945183, + "pass@32": 0.9999999991321172, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.74788671875, + "avg_response_len": 376.800671875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.74788671875, + "pass@2": 0.8067386811023451, + "pass@4": 0.8475433721878449, + "pass@8": 0.875955598974259, + "pass@16": 0.8965947021519254, + "pass@32": 0.912481556268192, + "pass@64": 0.9249759125915514, + "pass@128": 0.9346 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.962078125, + "pass@2": 0.9721006397637795, + "pass@4": 0.9779260251218596, + "pass@8": 0.9815156048528346, + "pass@16": 0.983999923109336, + "pass@32": 0.9859616557843597, + "pass@64": 0.9878721878006305, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.911546875, + "pass@2": 0.9321042076771657, + "pass@4": 0.9444718401762275, + "pass@8": 0.9526936469372225, + "pass@16": 0.958706941498739, + "pass@32": 0.9636245617881141, + "pass@64": 0.9677446891273659, + "pass@128": 0.97 + }, + "13": { + "pass@1": 0.8599375, + "pass@2": 0.8935477362204735, + "pass@4": 0.913129875890513, + "pass@8": 0.9257893913998072, + "pass@16": 0.9359653621214782, + "pass@32": 0.9449610897716896, + "pass@64": 0.9530347465917045, + "pass@128": 0.96 + }, + "14": { + "pass@1": 0.8687578125, + "pass@2": 0.901551181102363, + "pass@4": 0.9207215549306342, + "pass@8": 0.9331041964852135, + "pass@16": 0.9420800015814609, + "pass@32": 0.949228459513912, + "pass@64": 0.954773812574869, + "pass@128": 0.959 + }, + "15": { + "pass@1": 0.856390625, + "pass@2": 0.903252706692913, + "pass@4": 0.9310627197225344, + "pass@8": 0.9475995549360254, + "pass@16": 0.9576153119204723, + "pass@32": 0.9643163370676473, + "pass@64": 0.968880108784568, + "pass@128": 0.972 + }, + "16": { + "pass@1": 0.7253125, + "pass@2": 0.796625861220473, + "pass@4": 0.8427367949943759, + "pass@8": 0.8733032161795067, + "pass@16": 0.8954036628057763, + "pass@32": 0.9127415263332597, + "pass@64": 0.9261882031645849, + "pass@128": 0.935 + }, + "17": { + "pass@1": 0.6796875, + "pass@2": 0.7575906742125995, + "pass@4": 0.8083111899137609, + "pass@8": 0.843515919349626, + "pass@16": 0.8700326994453809, + "pass@32": 0.8908313662744057, + "pass@64": 0.9068133641477848, + "pass@128": 0.917 + }, + "18": { + "pass@1": 0.6506875, + "pass@2": 0.7346328740157485, + "pass@4": 0.7924272819647548, + "pass@8": 0.8312656677167265, + "pass@16": 0.8585926458863701, + "pass@32": 0.8795615768483948, + "pass@64": 0.8978518714797509, + "pass@128": 0.915 + }, + "19": { + "pass@1": 0.4735078125, + "pass@2": 0.580681594488189, + "pass@4": 0.6674846351706033, + "pass@8": 0.7343573825173302, + "pass@16": 0.7844422649335011, + "pass@32": 0.8215604017488102, + "pass@64": 0.8481176641261097, + "pass@128": 0.866 + }, + "20": { + "pass@1": 0.4909609375, + "pass@2": 0.5952993356299213, + "pass@4": 0.6771618039932508, + "pass@8": 0.7364114093683414, + "pass@16": 0.7791082082166665, + "pass@32": 0.8120285875513084, + "pass@64": 0.8384824781180986, + "pass@128": 0.862 + } + }, + "per_op_accuracy": { + "11": 0.962078125, + "12": 0.911546875, + "13": 0.8599375, + "14": 0.8687578125, + "15": 0.856390625, + "16": 0.7253125, + "17": 0.6796875, + "18": 0.6506875, + "19": 0.4735078125, + "20": 0.4909609375 + }, + "per_op_avg_response_len": { + "11": 298.8864765625, + "12": 308.4374375, + "13": 327.587546875, + "14": 355.72346875, + "15": 378.28559375, + "16": 382.699671875, + "17": 407.78859375, + "18": 426.4286875, + "19": 426.6126484375, + "20": 455.55659375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.8864765625, + "12": 308.4374375, + "13": 327.587546875, + "14": 355.72346875, + "15": 378.28559375, + "16": 382.699671875, + "17": 407.78859375, + "18": 426.4286875, + "19": 426.6126484375, + "20": 455.55659375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 320592, + "answer_accuracy": 0.7443164933135216, + "avg_response_len": 358.68860976968796, + "resp_tokens_sum": 154494358, + "pass_at_k": { + "pass@1": 0.7443164933135216, + "pass@2": 0.8033921227667924, + "pass@4": 0.8448562877077224, + "pass@8": 0.8739947943411771, + "pass@16": 0.8953814286611144, + "pass@32": 0.9120494694770329, + "pass@64": 0.9252491970597507, + "pass@128": 0.9355126300148588 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9498937677053825, + "pass@2": 0.9626099267248109, + "pass@4": 0.9712518073165783, + "pass@8": 0.9765331823453457, + "pass@16": 0.9803012693488047, + "pass@32": 0.9835281569287806, + "pass@64": 0.9868876826293069, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8905367231638418, + "pass@2": 0.9109814993994398, + "pass@4": 0.9243411350593884, + "pass@8": 0.9352700378666337, + "pass@16": 0.9447120596531021, + "pass@32": 0.9527390976579254, + "pass@64": 0.9594652093284125, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8733836206896551, + "pass@2": 0.9120185958828032, + "pass@4": 0.9328977089070761, + "pass@8": 0.944504818788995, + "pass@16": 0.9529491966962685, + "pass@32": 0.9601035751249352, + "pass@64": 0.9664615260372531, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.864753699376947, + "pass@2": 0.8983299911693281, + "pass@4": 0.9175780206212538, + "pass@8": 0.929303745916473, + "pass@16": 0.9381800154652723, + "pass@32": 0.945806784489565, + "pass@64": 0.951369716376544, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.8553784320809249, + "pass@2": 0.9072715881161535, + "pass@4": 0.9411106580427439, + "pass@8": 0.96146052262591, + "pass@16": 0.9721804556217477, + "pass@32": 0.9785321263160908, + "pass@64": 0.9828728618930074, + "pass@128": 0.9855491329479769 + }, + "16": { + "pass@1": 0.7206653225806452, + "pass@2": 0.7915060642621284, + "pass@4": 0.8401170204530883, + "pass@8": 0.8740411002173647, + "pass@16": 0.8980309267291599, + "pass@32": 0.9157798892337399, + "pass@64": 0.9281169036284177, + "pass@128": 0.9354838709677419 + }, + "17": { + "pass@1": 0.688651553468208, + "pass@2": 0.7637589037822585, + "pass@4": 0.8106093944723668, + "pass@8": 0.843426436073923, + "pass@16": 0.8699720418885797, + "pass@32": 0.8916278055186606, + "pass@64": 0.9084156806983802, + "pass@128": 0.9190751445086706 + }, + "18": { + "pass@1": 0.615, + "pass@2": 0.7006025028121483, + "pass@4": 0.7613126795757668, + "pass@8": 0.802003657466518, + "pass@16": 0.8316371641456359, + "pass@32": 0.8578101762690888, + "pass@64": 0.8837918179465672, + "pass@128": 0.9085714285714286 + }, + "19": { + "pass@1": 0.4704861111111111, + "pass@2": 0.5738281342365986, + "pass@4": 0.6604913746705078, + "pass@8": 0.730225881843159, + "pass@16": 0.7835294980895021, + "pass@32": 0.8224721616727019, + "pass@64": 0.8484112478170668, + "pass@128": 0.8648648648648649 + }, + "20": { + "pass@1": 0.5054194819819819, + "pass@2": 0.6065729853869616, + "pass@4": 0.6840881815448743, + "pass@8": 0.7395705976213225, + "pass@16": 0.7795039933857825, + "pass@32": 0.8097011199068305, + "pass@64": 0.8342355814027039, + "pass@128": 0.8558558558558559 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 324878, + "answer_accuracy": 0.754715841510556, + "avg_response_len": 372.0350273193577, + "resp_tokens_sum": 160147686, + "pass_at_k": { + "pass@1": 0.754715841510556, + "pass@2": 0.8124908540368733, + "pass@4": 0.8518243681014028, + "pass@8": 0.879027191976133, + "pass@16": 0.8986502712434014, + "pass@32": 0.9137201662428952, + "pass@64": 0.9258139250742512, + "pass@128": 0.935474278917633 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9614316641566265, + "pass@2": 0.9703316217626413, + "pass@4": 0.9746360038014824, + "pass@8": 0.9771279121273282, + "pass@16": 0.97919069694216, + "pass@32": 0.9812769099715625, + "pass@64": 0.9834362683036023, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.915948275862069, + "pass@2": 0.9348037204107326, + "pass@4": 0.9445616263092502, + "pass@8": 0.9491905448375989, + "pass@16": 0.9518151877840872, + "pass@32": 0.953993987145514, + "pass@64": 0.9556394841251691, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.8646489845938375, + "pass@2": 0.895317083526324, + "pass@4": 0.9132425684184432, + "pass@8": 0.9238891808894892, + "pass@16": 0.9317491539466264, + "pass@32": 0.9395458787945532, + "pass@64": 0.9476153266989242, + "pass@128": 0.9551820728291317 + }, + "14": { + "pass@1": 0.8673245614035088, + "pass@2": 0.9032105464612973, + "pass@4": 0.9232121300078712, + "pass@8": 0.9370405377624225, + "pass@16": 0.9477325424992136, + "pass@32": 0.9554437417199615, + "pass@64": 0.96073395334272, + "pass@128": 0.9649122807017544 + }, + "15": { + "pass@1": 0.8745738636363637, + "pass@2": 0.9187630487950369, + "pass@4": 0.9422371891013612, + "pass@8": 0.9557051494925514, + "pass@16": 0.9639843231142565, + "pass@32": 0.9698065376209328, + "pass@64": 0.9744502319823559, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.7344195156695157, + "pass@2": 0.8059656605424319, + "pass@4": 0.8503229879384728, + "pass@8": 0.8794052763758482, + "pass@16": 0.9013382569723788, + "pass@32": 0.9188712822787252, + "pass@64": 0.9315446476345826, + "pass@128": 0.9373219373219374 + }, + "17": { + "pass@1": 0.7180221273291926, + "pass@2": 0.7960634506284539, + "pass@4": 0.8425463527074643, + "pass@8": 0.8697402243673399, + "pass@16": 0.8868366785220958, + "pass@32": 0.9001743577142274, + "pass@64": 0.9121557652242751, + "pass@128": 0.922360248447205 + }, + "18": { + "pass@1": 0.6733455882352941, + "pass@2": 0.7566263316350158, + "pass@4": 0.8151107903791436, + "pass@8": 0.8552001099597542, + "pass@16": 0.8814870107761298, + "pass@32": 0.8981819096791793, + "pass@64": 0.9116619079421308, + "pass@128": 0.9264705882352942 + }, + "19": { + "pass@1": 0.47521391369047616, + "pass@2": 0.578692189843457, + "pass@4": 0.6627576187463173, + "pass@8": 0.7278592995590935, + "pass@16": 0.7759477134403571, + "pass@32": 0.8122013490049438, + "pass@64": 0.8417435190127524, + "pass@128": 0.8630952380952381 + }, + "20": { + "pass@1": 0.4647267964071856, + "pass@2": 0.5671147861756798, + "pass@4": 0.6511949015355114, + "pass@8": 0.7162719427216688, + "pass@16": 0.7671004422783796, + "pass@32": 0.8079510222505945, + "pass@64": 0.8390626780614435, + "pass@128": 0.8652694610778443 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 311825, + "answer_accuracy": 0.7445393681234719, + "avg_response_len": 400.32571821515893, + "resp_tokens_sum": 167662816, + "pass_at_k": { + "pass@1": 0.7445393681234719, + "pass@2": 0.8042682077589922, + "pass@4": 0.8459067737827112, + "pass@8": 0.8748151161885284, + "pass@16": 0.8957297224581334, + "pass@32": 0.9116528663865573, + "pass@64": 0.923833541498983, + "pass@128": 0.9327628361858191 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9764136904761904, + "pass@2": 0.9846007530308712, + "pass@4": 0.9888729646889376, + "pass@8": 0.9917235703449355, + "pass@16": 0.9932135353790832, + "pass@32": 0.993626292945846, + "pass@64": 0.993650777002188, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.9299980886850153, + "pass@2": 0.952337523176575, + "pass@4": 0.9661771240750868, + "pass@8": 0.974973301924281, + "pass@16": 0.9805805121664143, + "pass@32": 0.984803789962659, + "pass@64": 0.9885168488965713, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8415075231481481, + "pass@2": 0.8734123620832115, + "pass@4": 0.8935429315548514, + "pass@8": 0.9094565328042286, + "pass@16": 0.9238892605445178, + "pass@32": 0.9360190449912967, + "pass@64": 0.9457865683774824, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8740263353115727, + "pass@2": 0.9029354511787661, + "pass@4": 0.9211883140905603, + "pass@8": 0.9327294662649187, + "pass@16": 0.9400584186420708, + "pass@32": 0.9461801840490655, + "pass@64": 0.9519677198064921, + "pass@128": 0.9554896142433235 + }, + "15": { + "pass@1": 0.8389515817901234, + "pass@2": 0.8831633675998828, + "pass@4": 0.9089511099885664, + "pass@8": 0.9245417122682672, + "pass@16": 0.9355742211347028, + "pass@32": 0.9435434072449757, + "pass@64": 0.9482639568376837, + "pass@128": 0.9506172839506173 + }, + "16": { + "pass@1": 0.7201327433628318, + "pass@2": 0.7916372697605276, + "pass@4": 0.8372777282817518, + "pass@8": 0.8663103926377614, + "pass@16": 0.8868564817770899, + "pass@32": 0.9036163439261593, + "pass@64": 0.9188784416520243, + "pass@128": 0.9321533923303835 + }, + "17": { + "pass@1": 0.6331654743975904, + "pass@2": 0.7138483807276351, + "pass@4": 0.7727120899232472, + "pass@8": 0.8181747597041118, + "pass@16": 0.8537980797223456, + "pass@32": 0.8809397662077352, + "pass@64": 0.8999619946503863, + "pass@128": 0.9096385542168675 + }, + "18": { + "pass@1": 0.6661290322580645, + "pass@2": 0.7489324041148084, + "pass@4": 0.8026779851107314, + "pass@8": 0.8380527426358955, + "pass@16": 0.863916305714564, + "pass@32": 0.8836973092364269, + "pass@64": 0.8985796338649285, + "pass@128": 0.9096774193548387 + }, + "19": { + "pass@1": 0.4748158987915408, + "pass@2": 0.5895959214501513, + "pass@4": 0.6793185725273767, + "pass@8": 0.7451100882528764, + "pass@16": 0.7939834149659781, + "pass@32": 0.8301435608644672, + "pass@64": 0.8542927378088837, + "pass@128": 0.8700906344410876 + }, + "20": { + "pass@1": 0.5028153153153153, + "pass@2": 0.6122948736137706, + "pass@4": 0.6962803075854251, + "pass@8": 0.7534521666408532, + "pass@16": 0.7907562483430126, + "pass@32": 0.8184458654373412, + "pass@64": 0.842147432547975, + "pass@128": 0.8648648648648649 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.8645456414473685, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8645456414473685, + "pass@2": 0.8962973088478919, + "pass@4": 0.918322963506194, + "pass@8": 0.9336382346227033, + "pass@16": 0.9447250138227116, + "pass@32": 0.9532673600928167, + "pass@64": 0.9600371011586956, + "pass@128": 0.9652631578947368 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9616484375, + "pass@2": 0.9712715305118106, + "pass@4": 0.9786533301462318, + "pass@8": 0.983622736665287, + "pass@16": 0.9868989010859961, + "pass@32": 0.9895558252700736, + "pass@64": 0.9923754338993602, + "pass@128": 0.995 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99828125, + "pass@2": 0.9989266732283465, + "pass@4": 0.9994362400637421, + "pass@8": 0.9998281526533629, + "pass@16": 0.9999821832367909, + "pass@32": 0.9999998582404139, + "pass@64": 0.9999999999997746, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999609375, + "pass@2": 0.9999995078740158, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9986953125, + "pass@2": 0.9989669045275591, + "pass@4": 0.9989991694788152, + "pass@8": 0.9989999996570519, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.9932734375, + "pass@2": 0.9953978838582678, + "pass@4": 0.9965545253093364, + "pass@8": 0.9973638473817131, + "pass@16": 0.9979768743591723, + "pass@32": 0.9987090283739593, + "pass@64": 0.9995703622047244, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9956484375, + "pass@2": 0.9976995570866142, + "pass@4": 0.9990593197412824, + "pass@8": 0.9997557317312142, + "pass@16": 0.9999702824305289, + "pass@32": 0.9999995671969636, + "pass@64": 0.9999999999958465, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.962078125, + "pass@2": 0.9721006397637795, + "pass@4": 0.9779260251218596, + "pass@8": 0.9815156048528346, + "pass@16": 0.983999923109336, + "pass@32": 0.9859616557843597, + "pass@64": 0.9878721878006305, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.911546875, + "pass@2": 0.9321042076771657, + "pass@4": 0.9444718401762275, + "pass@8": 0.9526936469372225, + "pass@16": 0.958706941498739, + "pass@32": 0.9636245617881141, + "pass@64": 0.9677446891273659, + "pass@128": 0.97 + }, + "13": { + "pass@1": 0.8599375, + "pass@2": 0.8935477362204735, + "pass@4": 0.913129875890513, + "pass@8": 0.9257893913998072, + "pass@16": 0.9359653621214782, + "pass@32": 0.9449610897716896, + "pass@64": 0.9530347465917045, + "pass@128": 0.96 + }, + "14": { + "pass@1": 0.8687578125, + "pass@2": 0.901551181102363, + "pass@4": 0.9207215549306342, + "pass@8": 0.9331041964852135, + "pass@16": 0.9420800015814609, + "pass@32": 0.949228459513912, + "pass@64": 0.954773812574869, + "pass@128": 0.959 + }, + "15": { + "pass@1": 0.856390625, + "pass@2": 0.903252706692913, + "pass@4": 0.9310627197225344, + "pass@8": 0.9475995549360254, + "pass@16": 0.9576153119204723, + "pass@32": 0.9643163370676473, + "pass@64": 0.968880108784568, + "pass@128": 0.972 + }, + "16": { + "pass@1": 0.7253125, + "pass@2": 0.796625861220473, + "pass@4": 0.8427367949943759, + "pass@8": 0.8733032161795067, + "pass@16": 0.8954036628057763, + "pass@32": 0.9127415263332597, + "pass@64": 0.9261882031645849, + "pass@128": 0.935 + }, + "17": { + "pass@1": 0.6796875, + "pass@2": 0.7575906742125995, + "pass@4": 0.8083111899137609, + "pass@8": 0.843515919349626, + "pass@16": 0.8700326994453809, + "pass@32": 0.8908313662744057, + "pass@64": 0.9068133641477848, + "pass@128": 0.917 + }, + "18": { + "pass@1": 0.6506875, + "pass@2": 0.7346328740157485, + "pass@4": 0.7924272819647548, + "pass@8": 0.8312656677167265, + "pass@16": 0.8585926458863701, + "pass@32": 0.8795615768483948, + "pass@64": 0.8978518714797509, + "pass@128": 0.915 + }, + "19": { + "pass@1": 0.4735078125, + "pass@2": 0.580681594488189, + "pass@4": 0.6674846351706033, + "pass@8": 0.7343573825173302, + "pass@16": 0.7844422649335011, + "pass@32": 0.8215604017488102, + "pass@64": 0.8481176641261097, + "pass@128": 0.866 + }, + "20": { + "pass@1": 0.4909609375, + "pass@2": 0.5952993356299213, + "pass@4": 0.6771618039932508, + "pass@8": 0.7364114093683414, + "pass@16": 0.7791082082166665, + "pass@32": 0.8120285875513084, + "pass@64": 0.8384824781180986, + "pass@128": 0.862 + } + }, + "per_op_accuracy": { + "10": 0.9616484375, + "2": 1.0, + "3": 1.0, + "4": 0.9999921875, + "5": 0.99828125, + "6": 0.9999609375, + "7": 0.9986953125, + "8": 0.9932734375, + "9": 0.9956484375, + "11": 0.962078125, + "12": 0.911546875, + "13": 0.8599375, + "14": 0.8687578125, + "15": 0.856390625, + "16": 0.7253125, + "17": 0.6796875, + "18": 0.6506875, + "19": 0.4735078125, + "20": 0.4909609375 + }, + "per_op_avg_response_len": { + "10": 280.8438359375, + "2": 115.3596875, + "3": 144.295109375, + "4": 159.4446640625, + "5": 172.1992421875, + "6": 198.2354296875, + "7": 223.3115078125, + "8": 243.026265625, + "9": 258.9526171875, + "11": 298.8864765625, + "12": 308.4374375, + "13": 327.587546875, + "14": 355.72346875, + "15": 378.28559375, + "16": 382.699671875, + "17": 407.78859375, + "18": 426.4286875, + "19": 426.6126484375, + "20": 455.55659375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.8438359375, + "2": 115.3596875, + "3": 144.295109375, + "4": 159.4446640625, + "5": 172.1992421875, + "6": 198.2354296875, + "7": 223.3115078125, + "8": 243.026265625, + "9": 258.9526171875, + "11": 298.8864765625, + "12": 308.4374375, + "13": 327.587546875, + "14": 355.72346875, + "15": 378.28559375, + "16": 382.699671875, + "17": 407.78859375, + "18": 426.4286875, + "19": 426.6126484375, + "20": 455.55659375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 697829, + "answer_accuracy": 0.8618066807619349, + "avg_response_len": 279.70030182975023, + "resp_tokens_sum": 226481166, + "pass_at_k": { + "pass@1": 0.8618066807619349, + "pass@2": 0.8939249318834062, + "pass@4": 0.9164054978525218, + "pass@8": 0.9321541510095228, + "pass@16": 0.9436637466927077, + "pass@32": 0.952643656605473, + "pass@64": 0.9598431054551935, + "pass@128": 0.9655390452102435 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9683582227138643, + "pass@2": 0.9777432757299143, + "pass@4": 0.9832795228583151, + "pass@8": 0.9858626452225466, + "pass@16": 0.9873392250168272, + "pass@32": 0.9893153006929356, + "pass@64": 0.9926369823240244, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.997470238095238, + "pass@2": 0.9980275903012124, + "pass@4": 0.9987849956255469, + "pass@8": 0.9995512543309223, + "pass@16": 0.9999456111182645, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9958433493589743, + "pass@2": 0.9980196661114474, + "pass@4": 0.9991082061256767, + "pass@8": 0.9997716496433244, + "pass@16": 0.9999869239709128, + "pass@32": 0.9999999812155175, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9972931338028169, + "pass@2": 0.9981389320173008, + "pass@4": 0.9987858566094732, + "pass@8": 0.9994925064642773, + "pass@16": 0.9999199128774705, + "pass@32": 0.9999987816218918, + "pass@64": 0.9999999999883001, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9498937677053825, + "pass@2": 0.9626099267248109, + "pass@4": 0.9712518073165783, + "pass@8": 0.9765331823453457, + "pass@16": 0.9803012693488047, + "pass@32": 0.9835281569287806, + "pass@64": 0.9868876826293069, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8905367231638418, + "pass@2": 0.9109814993994398, + "pass@4": 0.9243411350593884, + "pass@8": 0.9352700378666337, + "pass@16": 0.9447120596531021, + "pass@32": 0.9527390976579254, + "pass@64": 0.9594652093284125, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8733836206896551, + "pass@2": 0.9120185958828032, + "pass@4": 0.9328977089070761, + "pass@8": 0.944504818788995, + "pass@16": 0.9529491966962685, + "pass@32": 0.9601035751249352, + "pass@64": 0.9664615260372531, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.864753699376947, + "pass@2": 0.8983299911693281, + "pass@4": 0.9175780206212538, + "pass@8": 0.929303745916473, + "pass@16": 0.9381800154652723, + "pass@32": 0.945806784489565, + "pass@64": 0.951369716376544, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.8553784320809249, + "pass@2": 0.9072715881161535, + "pass@4": 0.9411106580427439, + "pass@8": 0.96146052262591, + "pass@16": 0.9721804556217477, + "pass@32": 0.9785321263160908, + "pass@64": 0.9828728618930074, + "pass@128": 0.9855491329479769 + }, + "16": { + "pass@1": 0.7206653225806452, + "pass@2": 0.7915060642621284, + "pass@4": 0.8401170204530883, + "pass@8": 0.8740411002173647, + "pass@16": 0.8980309267291599, + "pass@32": 0.9157798892337399, + "pass@64": 0.9281169036284177, + "pass@128": 0.9354838709677419 + }, + "17": { + "pass@1": 0.688651553468208, + "pass@2": 0.7637589037822585, + "pass@4": 0.8106093944723668, + "pass@8": 0.843426436073923, + "pass@16": 0.8699720418885797, + "pass@32": 0.8916278055186606, + "pass@64": 0.9084156806983802, + "pass@128": 0.9190751445086706 + }, + "18": { + "pass@1": 0.615, + "pass@2": 0.7006025028121483, + "pass@4": 0.7613126795757668, + "pass@8": 0.802003657466518, + "pass@16": 0.8316371641456359, + "pass@32": 0.8578101762690888, + "pass@64": 0.8837918179465672, + "pass@128": 0.9085714285714286 + }, + "19": { + "pass@1": 0.4704861111111111, + "pass@2": 0.5738281342365986, + "pass@4": 0.6604913746705078, + "pass@8": 0.730225881843159, + "pass@16": 0.7835294980895021, + "pass@32": 0.8224721616727019, + "pass@64": 0.8484112478170668, + "pass@128": 0.8648648648648649 + }, + "20": { + "pass@1": 0.5054194819819819, + "pass@2": 0.6065729853869616, + "pass@4": 0.6840881815448743, + "pass@8": 0.7395705976213225, + "pass@16": 0.7795039933857825, + "pass@32": 0.8097011199068305, + "pass@64": 0.8342355814027039, + "pass@128": 0.8558558558558559 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 712853, + "answer_accuracy": 0.8681471648480125, + "avg_response_len": 289.0419427123928, + "resp_tokens_sum": 237338120, + "pass_at_k": { + "pass@1": 0.8681471648480125, + "pass@2": 0.8992812007106781, + "pass@4": 0.9204534567726954, + "pass@8": 0.9350104183617381, + "pass@16": 0.9454745105693739, + "pass@32": 0.9535628085797692, + "pass@64": 0.960135921141399, + "pass@128": 0.9653936087295402 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9523820935582822, + "pass@2": 0.9626954162842377, + "pass@4": 0.9704093263265402, + "pass@8": 0.9749882039712388, + "pass@16": 0.9779861413506765, + "pass@32": 0.9810598822693485, + "pass@64": 0.9846863055012435, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994791666666667, + "pass@2": 0.999952570466735, + "pass@4": 0.9999996101030848, + "pass@8": 0.9999999999869519, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999328438395415, + "pass@2": 0.99999894242267, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9965679190751445, + "pass@2": 0.9970198642528788, + "pass@4": 0.9971074275939497, + "pass@8": 0.9971098255984155, + "pass@16": 0.9971098265895953, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9943441901408451, + "pass@2": 0.9960425446379061, + "pass@4": 0.996916455425466, + "pass@8": 0.9974621349733664, + "pass@16": 0.9978447220313824, + "pass@32": 0.9984196516998427, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9939651946107785, + "pass@2": 0.9972940438021596, + "pass@4": 0.999298675988855, + "pass@8": 0.9999346560243496, + "pass@16": 0.999999295537448, + "pass@32": 0.9999999999733643, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9614316641566265, + "pass@2": 0.9703316217626413, + "pass@4": 0.9746360038014824, + "pass@8": 0.9771279121273282, + "pass@16": 0.97919069694216, + "pass@32": 0.9812769099715625, + "pass@64": 0.9834362683036023, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.915948275862069, + "pass@2": 0.9348037204107326, + "pass@4": 0.9445616263092502, + "pass@8": 0.9491905448375989, + "pass@16": 0.9518151877840872, + "pass@32": 0.953993987145514, + "pass@64": 0.9556394841251691, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.8646489845938375, + "pass@2": 0.895317083526324, + "pass@4": 0.9132425684184432, + "pass@8": 0.9238891808894892, + "pass@16": 0.9317491539466264, + "pass@32": 0.9395458787945532, + "pass@64": 0.9476153266989242, + "pass@128": 0.9551820728291317 + }, + "14": { + "pass@1": 0.8673245614035088, + "pass@2": 0.9032105464612973, + "pass@4": 0.9232121300078712, + "pass@8": 0.9370405377624225, + "pass@16": 0.9477325424992136, + "pass@32": 0.9554437417199615, + "pass@64": 0.96073395334272, + "pass@128": 0.9649122807017544 + }, + "15": { + "pass@1": 0.8745738636363637, + "pass@2": 0.9187630487950369, + "pass@4": 0.9422371891013612, + "pass@8": 0.9557051494925514, + "pass@16": 0.9639843231142565, + "pass@32": 0.9698065376209328, + "pass@64": 0.9744502319823559, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.7344195156695157, + "pass@2": 0.8059656605424319, + "pass@4": 0.8503229879384728, + "pass@8": 0.8794052763758482, + "pass@16": 0.9013382569723788, + "pass@32": 0.9188712822787252, + "pass@64": 0.9315446476345826, + "pass@128": 0.9373219373219374 + }, + "17": { + "pass@1": 0.7180221273291926, + "pass@2": 0.7960634506284539, + "pass@4": 0.8425463527074643, + "pass@8": 0.8697402243673399, + "pass@16": 0.8868366785220958, + "pass@32": 0.9001743577142274, + "pass@64": 0.9121557652242751, + "pass@128": 0.922360248447205 + }, + "18": { + "pass@1": 0.6733455882352941, + "pass@2": 0.7566263316350158, + "pass@4": 0.8151107903791436, + "pass@8": 0.8552001099597542, + "pass@16": 0.8814870107761298, + "pass@32": 0.8981819096791793, + "pass@64": 0.9116619079421308, + "pass@128": 0.9264705882352942 + }, + "19": { + "pass@1": 0.47521391369047616, + "pass@2": 0.578692189843457, + "pass@4": 0.6627576187463173, + "pass@8": 0.7278592995590935, + "pass@16": 0.7759477134403571, + "pass@32": 0.8122013490049438, + "pass@64": 0.8417435190127524, + "pass@128": 0.8630952380952381 + }, + "20": { + "pass@1": 0.4647267964071856, + "pass@2": 0.5671147861756798, + "pass@4": 0.6511949015355114, + "pass@8": 0.7162719427216688, + "pass@16": 0.7671004422783796, + "pass@32": 0.8079510222505945, + "pass@64": 0.8390626780614435, + "pass@128": 0.8652694610778443 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 691893, + "answer_accuracy": 0.8636226334078926, + "avg_response_len": 309.96755172551525, + "resp_tokens_sum": 248331124, + "pass_at_k": { + "pass@1": 0.8636226334078926, + "pass@2": 0.8956368184145412, + "pass@4": 0.9180773609212313, + "pass@8": 0.9337318205391284, + "pass@16": 0.9450294640918934, + "pass@32": 0.953594927790042, + "pass@64": 0.9601318905230214, + "pass@128": 0.9648506151142355 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9638759328358208, + "pass@2": 0.9730682218827124, + "pass@4": 0.9819944224322704, + "pass@8": 0.9897586431051344, + "pass@16": 0.9951266320148406, + "pass@32": 0.998066915568241, + "pass@64": 0.9995933173077908, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9978170955882353, + "pass@2": 0.9987186631542381, + "pass@4": 0.9994679439886189, + "pass@8": 0.9999103163047764, + "pass@16": 0.9999979874545221, + "pass@32": 0.9999999996892099, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997527689873418, + "pass@2": 0.9999949385776936, + "pass@4": 0.9999999985167994, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9897240990990991, + "pass@2": 0.9922541882433614, + "pass@4": 0.9937760460122664, + "pass@8": 0.9950031074397921, + "pass@16": 0.9962344677450653, + "pass@32": 0.9978079816254464, + "pass@64": 0.9994546357381001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9955787781350482, + "pass@2": 0.9976335227992, + "pass@4": 0.9991144143557619, + "pass@8": 0.9998640412355725, + "pass@16": 0.99999661945183, + "pass@32": 0.9999999991321172, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9764136904761904, + "pass@2": 0.9846007530308712, + "pass@4": 0.9888729646889376, + "pass@8": 0.9917235703449355, + "pass@16": 0.9932135353790832, + "pass@32": 0.993626292945846, + "pass@64": 0.993650777002188, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.9299980886850153, + "pass@2": 0.952337523176575, + "pass@4": 0.9661771240750868, + "pass@8": 0.974973301924281, + "pass@16": 0.9805805121664143, + "pass@32": 0.984803789962659, + "pass@64": 0.9885168488965713, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8415075231481481, + "pass@2": 0.8734123620832115, + "pass@4": 0.8935429315548514, + "pass@8": 0.9094565328042286, + "pass@16": 0.9238892605445178, + "pass@32": 0.9360190449912967, + "pass@64": 0.9457865683774824, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8740263353115727, + "pass@2": 0.9029354511787661, + "pass@4": 0.9211883140905603, + "pass@8": 0.9327294662649187, + "pass@16": 0.9400584186420708, + "pass@32": 0.9461801840490655, + "pass@64": 0.9519677198064921, + "pass@128": 0.9554896142433235 + }, + "15": { + "pass@1": 0.8389515817901234, + "pass@2": 0.8831633675998828, + "pass@4": 0.9089511099885664, + "pass@8": 0.9245417122682672, + "pass@16": 0.9355742211347028, + "pass@32": 0.9435434072449757, + "pass@64": 0.9482639568376837, + "pass@128": 0.9506172839506173 + }, + "16": { + "pass@1": 0.7201327433628318, + "pass@2": 0.7916372697605276, + "pass@4": 0.8372777282817518, + "pass@8": 0.8663103926377614, + "pass@16": 0.8868564817770899, + "pass@32": 0.9036163439261593, + "pass@64": 0.9188784416520243, + "pass@128": 0.9321533923303835 + }, + "17": { + "pass@1": 0.6331654743975904, + "pass@2": 0.7138483807276351, + "pass@4": 0.7727120899232472, + "pass@8": 0.8181747597041118, + "pass@16": 0.8537980797223456, + "pass@32": 0.8809397662077352, + "pass@64": 0.8999619946503863, + "pass@128": 0.9096385542168675 + }, + "18": { + "pass@1": 0.6661290322580645, + "pass@2": 0.7489324041148084, + "pass@4": 0.8026779851107314, + "pass@8": 0.8380527426358955, + "pass@16": 0.863916305714564, + "pass@32": 0.8836973092364269, + "pass@64": 0.8985796338649285, + "pass@128": 0.9096774193548387 + }, + "19": { + "pass@1": 0.4748158987915408, + "pass@2": 0.5895959214501513, + "pass@4": 0.6793185725273767, + "pass@8": 0.7451100882528764, + "pass@16": 0.7939834149659781, + "pass@32": 0.8301435608644672, + "pass@64": 0.8542927378088837, + "pass@128": 0.8700906344410876 + }, + "20": { + "pass@1": 0.5028153153153153, + "pass@2": 0.6122948736137706, + "pass@4": 0.6962803075854251, + "pass@8": 0.7534521666408532, + "pass@16": 0.7907562483430126, + "pass@32": 0.8184458654373412, + "pass@64": 0.842147432547975, + "pass@128": 0.8648648648648649 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch4-RL-op17-20/composition/summary.csv b/composition/Qwen3-100M-epoch4-RL-op17-20/composition/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..9f386778f041e506528ebb7836112e5e4b939fb4 --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op17-20/composition/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.9941666666666666,0.74788671875,0.8645456414473685,0.0,0.0,0.0,199.51870659722223,376.800671875,1152000,1280000,2432000 diff --git a/composition/Qwen3-100M-epoch4-RL-op17-20/composition/summary.json b/composition/Qwen3-100M-epoch4-RL-op17-20/composition/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6f7777dec696c7f94dff08bea87f482d42a1b590 --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op17-20/composition/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.9941666666666666, + "ood_acc": 0.74788671875, + "total_acc": 0.8645456414473685, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.51870659722223, + "ood_avg_resp_len": 376.800671875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 697829, + "answer_accuracy": 0.8618066807619349, + "avg_response_len": 279.70030182975023, + "resp_tokens_sum": 226481166, + "pass_at_k": { + "pass@1": 0.8618066807619349, + "pass@2": 0.8939249318834062, + "pass@4": 0.9164054978525218, + "pass@8": 0.9321541510095228, + "pass@16": 0.9436637466927077, + "pass@32": 0.952643656605473, + "pass@64": 0.9598431054551935, + "pass@128": 0.9655390452102435 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9683582227138643, + "pass@2": 0.9777432757299143, + "pass@4": 0.9832795228583151, + "pass@8": 0.9858626452225466, + "pass@16": 0.9873392250168272, + "pass@32": 0.9893153006929356, + "pass@64": 0.9926369823240244, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.997470238095238, + "pass@2": 0.9980275903012124, + "pass@4": 0.9987849956255469, + "pass@8": 0.9995512543309223, + "pass@16": 0.9999456111182645, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9958433493589743, + "pass@2": 0.9980196661114474, + "pass@4": 0.9991082061256767, + "pass@8": 0.9997716496433244, + "pass@16": 0.9999869239709128, + "pass@32": 0.9999999812155175, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9972931338028169, + "pass@2": 0.9981389320173008, + "pass@4": 0.9987858566094732, + "pass@8": 0.9994925064642773, + "pass@16": 0.9999199128774705, + "pass@32": 0.9999987816218918, + "pass@64": 0.9999999999883001, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9498937677053825, + "pass@2": 0.9626099267248109, + "pass@4": 0.9712518073165783, + "pass@8": 0.9765331823453457, + "pass@16": 0.9803012693488047, + "pass@32": 0.9835281569287806, + "pass@64": 0.9868876826293069, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8905367231638418, + "pass@2": 0.9109814993994398, + "pass@4": 0.9243411350593884, + "pass@8": 0.9352700378666337, + "pass@16": 0.9447120596531021, + "pass@32": 0.9527390976579254, + "pass@64": 0.9594652093284125, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8733836206896551, + "pass@2": 0.9120185958828032, + "pass@4": 0.9328977089070761, + "pass@8": 0.944504818788995, + "pass@16": 0.9529491966962685, + "pass@32": 0.9601035751249352, + "pass@64": 0.9664615260372531, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.864753699376947, + "pass@2": 0.8983299911693281, + "pass@4": 0.9175780206212538, + "pass@8": 0.929303745916473, + "pass@16": 0.9381800154652723, + "pass@32": 0.945806784489565, + "pass@64": 0.951369716376544, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.8553784320809249, + "pass@2": 0.9072715881161535, + "pass@4": 0.9411106580427439, + "pass@8": 0.96146052262591, + "pass@16": 0.9721804556217477, + "pass@32": 0.9785321263160908, + "pass@64": 0.9828728618930074, + "pass@128": 0.9855491329479769 + }, + "16": { + "pass@1": 0.7206653225806452, + "pass@2": 0.7915060642621284, + "pass@4": 0.8401170204530883, + "pass@8": 0.8740411002173647, + "pass@16": 0.8980309267291599, + "pass@32": 0.9157798892337399, + "pass@64": 0.9281169036284177, + "pass@128": 0.9354838709677419 + }, + "17": { + "pass@1": 0.688651553468208, + "pass@2": 0.7637589037822585, + "pass@4": 0.8106093944723668, + "pass@8": 0.843426436073923, + "pass@16": 0.8699720418885797, + "pass@32": 0.8916278055186606, + "pass@64": 0.9084156806983802, + "pass@128": 0.9190751445086706 + }, + "18": { + "pass@1": 0.615, + "pass@2": 0.7006025028121483, + "pass@4": 0.7613126795757668, + "pass@8": 0.802003657466518, + "pass@16": 0.8316371641456359, + "pass@32": 0.8578101762690888, + "pass@64": 0.8837918179465672, + "pass@128": 0.9085714285714286 + }, + "19": { + "pass@1": 0.4704861111111111, + "pass@2": 0.5738281342365986, + "pass@4": 0.6604913746705078, + "pass@8": 0.730225881843159, + "pass@16": 0.7835294980895021, + "pass@32": 0.8224721616727019, + "pass@64": 0.8484112478170668, + "pass@128": 0.8648648648648649 + }, + "20": { + "pass@1": 0.5054194819819819, + "pass@2": 0.6065729853869616, + "pass@4": 0.6840881815448743, + "pass@8": 0.7395705976213225, + "pass@16": 0.7795039933857825, + "pass@32": 0.8097011199068305, + "pass@64": 0.8342355814027039, + "pass@128": 0.8558558558558559 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 712853, + "answer_accuracy": 0.8681471648480125, + "avg_response_len": 289.0419427123928, + "resp_tokens_sum": 237338120, + "pass_at_k": { + "pass@1": 0.8681471648480125, + "pass@2": 0.8992812007106781, + "pass@4": 0.9204534567726954, + "pass@8": 0.9350104183617381, + "pass@16": 0.9454745105693739, + "pass@32": 0.9535628085797692, + "pass@64": 0.960135921141399, + "pass@128": 0.9653936087295402 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9523820935582822, + "pass@2": 0.9626954162842377, + "pass@4": 0.9704093263265402, + "pass@8": 0.9749882039712388, + "pass@16": 0.9779861413506765, + "pass@32": 0.9810598822693485, + "pass@64": 0.9846863055012435, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994791666666667, + "pass@2": 0.999952570466735, + "pass@4": 0.9999996101030848, + "pass@8": 0.9999999999869519, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999328438395415, + "pass@2": 0.99999894242267, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9965679190751445, + "pass@2": 0.9970198642528788, + "pass@4": 0.9971074275939497, + "pass@8": 0.9971098255984155, + "pass@16": 0.9971098265895953, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9943441901408451, + "pass@2": 0.9960425446379061, + "pass@4": 0.996916455425466, + "pass@8": 0.9974621349733664, + "pass@16": 0.9978447220313824, + "pass@32": 0.9984196516998427, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9939651946107785, + "pass@2": 0.9972940438021596, + "pass@4": 0.999298675988855, + "pass@8": 0.9999346560243496, + "pass@16": 0.999999295537448, + "pass@32": 0.9999999999733643, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9614316641566265, + "pass@2": 0.9703316217626413, + "pass@4": 0.9746360038014824, + "pass@8": 0.9771279121273282, + "pass@16": 0.97919069694216, + "pass@32": 0.9812769099715625, + "pass@64": 0.9834362683036023, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.915948275862069, + "pass@2": 0.9348037204107326, + "pass@4": 0.9445616263092502, + "pass@8": 0.9491905448375989, + "pass@16": 0.9518151877840872, + "pass@32": 0.953993987145514, + "pass@64": 0.9556394841251691, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.8646489845938375, + "pass@2": 0.895317083526324, + "pass@4": 0.9132425684184432, + "pass@8": 0.9238891808894892, + "pass@16": 0.9317491539466264, + "pass@32": 0.9395458787945532, + "pass@64": 0.9476153266989242, + "pass@128": 0.9551820728291317 + }, + "14": { + "pass@1": 0.8673245614035088, + "pass@2": 0.9032105464612973, + "pass@4": 0.9232121300078712, + "pass@8": 0.9370405377624225, + "pass@16": 0.9477325424992136, + "pass@32": 0.9554437417199615, + "pass@64": 0.96073395334272, + "pass@128": 0.9649122807017544 + }, + "15": { + "pass@1": 0.8745738636363637, + "pass@2": 0.9187630487950369, + "pass@4": 0.9422371891013612, + "pass@8": 0.9557051494925514, + "pass@16": 0.9639843231142565, + "pass@32": 0.9698065376209328, + "pass@64": 0.9744502319823559, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.7344195156695157, + "pass@2": 0.8059656605424319, + "pass@4": 0.8503229879384728, + "pass@8": 0.8794052763758482, + "pass@16": 0.9013382569723788, + "pass@32": 0.9188712822787252, + "pass@64": 0.9315446476345826, + "pass@128": 0.9373219373219374 + }, + "17": { + "pass@1": 0.7180221273291926, + "pass@2": 0.7960634506284539, + "pass@4": 0.8425463527074643, + "pass@8": 0.8697402243673399, + "pass@16": 0.8868366785220958, + "pass@32": 0.9001743577142274, + "pass@64": 0.9121557652242751, + "pass@128": 0.922360248447205 + }, + "18": { + "pass@1": 0.6733455882352941, + "pass@2": 0.7566263316350158, + "pass@4": 0.8151107903791436, + "pass@8": 0.8552001099597542, + "pass@16": 0.8814870107761298, + "pass@32": 0.8981819096791793, + "pass@64": 0.9116619079421308, + "pass@128": 0.9264705882352942 + }, + "19": { + "pass@1": 0.47521391369047616, + "pass@2": 0.578692189843457, + "pass@4": 0.6627576187463173, + "pass@8": 0.7278592995590935, + "pass@16": 0.7759477134403571, + "pass@32": 0.8122013490049438, + "pass@64": 0.8417435190127524, + "pass@128": 0.8630952380952381 + }, + "20": { + "pass@1": 0.4647267964071856, + "pass@2": 0.5671147861756798, + "pass@4": 0.6511949015355114, + "pass@8": 0.7162719427216688, + "pass@16": 0.7671004422783796, + "pass@32": 0.8079510222505945, + "pass@64": 0.8390626780614435, + "pass@128": 0.8652694610778443 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 691893, + "answer_accuracy": 0.8636226334078926, + "avg_response_len": 309.96755172551525, + "resp_tokens_sum": 248331124, + "pass_at_k": { + "pass@1": 0.8636226334078926, + "pass@2": 0.8956368184145412, + "pass@4": 0.9180773609212313, + "pass@8": 0.9337318205391284, + "pass@16": 0.9450294640918934, + "pass@32": 0.953594927790042, + "pass@64": 0.9601318905230214, + "pass@128": 0.9648506151142355 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9638759328358208, + "pass@2": 0.9730682218827124, + "pass@4": 0.9819944224322704, + "pass@8": 0.9897586431051344, + "pass@16": 0.9951266320148406, + "pass@32": 0.998066915568241, + "pass@64": 0.9995933173077908, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9978170955882353, + "pass@2": 0.9987186631542381, + "pass@4": 0.9994679439886189, + "pass@8": 0.9999103163047764, + "pass@16": 0.9999979874545221, + "pass@32": 0.9999999996892099, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997527689873418, + "pass@2": 0.9999949385776936, + "pass@4": 0.9999999985167994, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9897240990990991, + "pass@2": 0.9922541882433614, + "pass@4": 0.9937760460122664, + "pass@8": 0.9950031074397921, + "pass@16": 0.9962344677450653, + "pass@32": 0.9978079816254464, + "pass@64": 0.9994546357381001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9955787781350482, + "pass@2": 0.9976335227992, + "pass@4": 0.9991144143557619, + "pass@8": 0.9998640412355725, + "pass@16": 0.99999661945183, + "pass@32": 0.9999999991321172, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9764136904761904, + "pass@2": 0.9846007530308712, + "pass@4": 0.9888729646889376, + "pass@8": 0.9917235703449355, + "pass@16": 0.9932135353790832, + "pass@32": 0.993626292945846, + "pass@64": 0.993650777002188, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.9299980886850153, + "pass@2": 0.952337523176575, + "pass@4": 0.9661771240750868, + "pass@8": 0.974973301924281, + "pass@16": 0.9805805121664143, + "pass@32": 0.984803789962659, + "pass@64": 0.9885168488965713, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8415075231481481, + "pass@2": 0.8734123620832115, + "pass@4": 0.8935429315548514, + "pass@8": 0.9094565328042286, + "pass@16": 0.9238892605445178, + "pass@32": 0.9360190449912967, + "pass@64": 0.9457865683774824, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8740263353115727, + "pass@2": 0.9029354511787661, + "pass@4": 0.9211883140905603, + "pass@8": 0.9327294662649187, + "pass@16": 0.9400584186420708, + "pass@32": 0.9461801840490655, + "pass@64": 0.9519677198064921, + "pass@128": 0.9554896142433235 + }, + "15": { + "pass@1": 0.8389515817901234, + "pass@2": 0.8831633675998828, + "pass@4": 0.9089511099885664, + "pass@8": 0.9245417122682672, + "pass@16": 0.9355742211347028, + "pass@32": 0.9435434072449757, + "pass@64": 0.9482639568376837, + "pass@128": 0.9506172839506173 + }, + "16": { + "pass@1": 0.7201327433628318, + "pass@2": 0.7916372697605276, + "pass@4": 0.8372777282817518, + "pass@8": 0.8663103926377614, + "pass@16": 0.8868564817770899, + "pass@32": 0.9036163439261593, + "pass@64": 0.9188784416520243, + "pass@128": 0.9321533923303835 + }, + "17": { + "pass@1": 0.6331654743975904, + "pass@2": 0.7138483807276351, + "pass@4": 0.7727120899232472, + "pass@8": 0.8181747597041118, + "pass@16": 0.8537980797223456, + "pass@32": 0.8809397662077352, + "pass@64": 0.8999619946503863, + "pass@128": 0.9096385542168675 + }, + "18": { + "pass@1": 0.6661290322580645, + "pass@2": 0.7489324041148084, + "pass@4": 0.8026779851107314, + "pass@8": 0.8380527426358955, + "pass@16": 0.863916305714564, + "pass@32": 0.8836973092364269, + "pass@64": 0.8985796338649285, + "pass@128": 0.9096774193548387 + }, + "19": { + "pass@1": 0.4748158987915408, + "pass@2": 0.5895959214501513, + "pass@4": 0.6793185725273767, + "pass@8": 0.7451100882528764, + "pass@16": 0.7939834149659781, + "pass@32": 0.8301435608644672, + "pass@64": 0.8542927378088837, + "pass@128": 0.8700906344410876 + }, + "20": { + "pass@1": 0.5028153153153153, + "pass@2": 0.6122948736137706, + "pass@4": 0.6962803075854251, + "pass@8": 0.7534521666408532, + "pass@16": 0.7907562483430126, + "pass@32": 0.8184458654373412, + "pass@64": 0.842147432547975, + "pass@128": 0.8648648648648649 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch4-RL-op7-10/composition/hf_model_metrics.json b/composition/Qwen3-100M-epoch4-RL-op7-10/composition/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..21d7ac76cdbad6be9d1b56161db68135425a721e --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op7-10/composition/hf_model_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.9963550347222222, + "avg_response_len": 200.36481336805556, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9963550347222222, + "pass@2": 0.9974193870297459, + "pass@4": 0.9981923617985251, + "pass@8": 0.9987978084815454, + "pass@16": 0.9992550423630918, + "pass@32": 0.9995743572021227, + "pass@64": 0.9997688052927273, + "pass@128": 0.9998888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9741484375, + "pass@2": 0.9814842519685039, + "pass@4": 0.9872013810461194, + "pass@8": 0.9916198616262941, + "pass@16": 0.9947940596672118, + "pass@32": 0.9970391170143708, + "pass@64": 0.998421360200285, + "pass@128": 0.999 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994296875, + "pass@2": 0.9997584891732283, + "pass@4": 0.9999441632920885, + "pass@8": 0.9999972911187579, + "pass@16": 0.9999999960747445, + "pass@32": 0.9999999999999993, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999296875, + "pass@2": 0.9999988927165354, + "pass@4": 0.9999999999062616, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9970546875, + "pass@2": 0.9981833169291339, + "pass@4": 0.9987903293025873, + "pass@8": 0.9992771876033886, + "pass@16": 0.999711012970505, + "pass@32": 0.9999493838432147, + "pass@64": 0.9999993322377224, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966484375, + "pass@2": 0.9973496555118111, + "pass@4": 0.99779538263967, + "pass@8": 0.998285935985465, + "pass@16": 0.9987903125553651, + "pass@32": 0.9991807139615209, + "pass@64": 0.9994985551965363, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9741484375, + "2": 1.0, + "3": 1.0, + "4": 0.999984375, + "5": 0.9994296875, + "6": 1.0, + "7": 0.9999296875, + "8": 0.9970546875, + "9": 0.9966484375 + }, + "per_op_avg_response_len": { + "10": 282.1082265625, + "2": 115.728890625, + "3": 144.8953125, + "4": 160.0852109375, + "5": 172.9949375, + "6": 199.2273984375, + "7": 224.3072421875, + "8": 243.96909375, + "9": 259.9670078125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 282.1082265625, + "2": 115.728890625, + "3": 144.8953125, + "4": 160.0852109375, + "5": 172.9949375, + "6": 199.2273984375, + "7": 224.3072421875, + "8": 243.96909375, + "9": 259.9670078125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 377809, + "answer_accuracy": 0.9968364783856805, + "avg_response_len": 190.90999398429585, + "resp_tokens_sum": 72356415, + "pass_at_k": { + "pass@1": 0.9968364783856805, + "pass@2": 0.9977998926995829, + "pass@4": 0.9984679191570153, + "pass@8": 0.9989338089915449, + "pass@16": 0.999261082962728, + "pass@32": 0.9995169191542642, + "pass@64": 0.9997462797081136, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9772077802359882, + "pass@2": 0.984406937379509, + "pass@4": 0.9896404088869423, + "pass@8": 0.9934608180094582, + "pass@16": 0.9961270518521915, + "pass@32": 0.9979929133208758, + "pass@64": 0.9992588029962364, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 0.9999996155265748, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9984375, + "pass@2": 0.9992372047244095, + "pass@4": 0.9998227409073865, + "pass@8": 0.9999914003770088, + "pass@16": 0.9999999875388716, + "pass@32": 0.999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.999824719551282, + "pass@2": 0.9999917190339189, + "pass@4": 0.9999999894844874, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9970730633802817, + "pass@2": 0.9972260729732727, + "pass@4": 0.9972711267605634, + "pass@8": 0.9973591549295775, + "pass@16": 0.9975352112676056, + "pass@32": 0.997887323943662, + "pass@64": 0.9985915492957746, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 389268, + "answer_accuracy": 0.996447001965924, + "avg_response_len": 198.399264314384, + "resp_tokens_sum": 77505863, + "pass_at_k": { + "pass@1": 0.996447001965924, + "pass@2": 0.9975232050623833, + "pass@4": 0.9982371935753278, + "pass@8": 0.998833699091369, + "pass@16": 0.9993589299246104, + "pass@32": 0.9997763275129983, + "pass@64": 0.9999754864341297, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9769699003067485, + "pass@2": 0.9845029376600167, + "pass@4": 0.9894387797000833, + "pass@8": 0.9929804552880818, + "pass@16": 0.9957689493418113, + "pass@32": 0.9982698846328565, + "pass@64": 0.9997769849567611, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999320652173913, + "pass@2": 0.9999996433869679, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999096820809249, + "pass@2": 0.9999989332529243, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9947623239436619, + "pass@2": 0.9956644393922591, + "pass@4": 0.9966619528192779, + "pass@8": 0.9979645098268615, + "pass@16": 0.9991859520597687, + "pass@32": 0.9998574192766612, + "pass@64": 0.9999981189794995, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957428892215568, + "pass@2": 0.9971032344759301, + "pass@4": 0.9977481417929045, + "pass@8": 0.9983575455521196, + "pass@16": 0.999137016955876, + "pass@32": 0.9997963632818704, + "pass@64": 0.9999956744411302, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380724, + "answer_accuracy": 0.9957838131904921, + "avg_response_len": 211.7456556536659, + "resp_tokens_sum": 80957987, + "pass_at_k": { + "pass@1": 0.9957838131904921, + "pass@2": 0.996936116214884, + "pass@4": 0.9978733956380661, + "pass@8": 0.9986263201483373, + "pass@16": 0.9991429060881413, + "pass@32": 0.9994249299074852, + "pass@64": 0.9995799560836476, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9683069029850746, + "pass@2": 0.9755890821483134, + "pass@4": 0.9825559410484136, + "pass@8": 0.9884328832751443, + "pass@16": 0.9924964525489203, + "pass@32": 0.9948762298754696, + "pass@64": 0.9962547131004683, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9998391544117647, + "pass@2": 0.9999967432839278, + "pass@4": 0.9999999997242991, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9969031531531531, + "pass@2": 0.9991742480669644, + "pass@4": 0.9999259439416918, + "pass@8": 0.9999993592037622, + "pass@16": 0.9999999999678294, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9971362540192926, + "pass@2": 0.9977553674962655, + "pass@4": 0.9984445443113824, + "pass@8": 0.9992669317397335, + "pass@16": 0.9998506395244453, + "pass@32": 0.9999959119787014, + "pass@64": 0.9999999997852048, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.45476875, + "avg_response_len": 331.08028671875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.45476875, + "pass@2": 0.5179418676181082, + "pass@4": 0.5729673668072683, + "pass@8": 0.6191010629854551, + "pass@16": 0.6567590157825484, + "pass@32": 0.6875423879772198, + "pass@64": 0.7135269636404079, + "pass@128": 0.7366 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9614609375, + "pass@2": 0.9715167322834649, + "pass@4": 0.9777205376827895, + "pass@8": 0.9819796795575038, + "pass@16": 0.9843140925694238, + "pass@32": 0.9857100860522761, + "pass@64": 0.9870012889094854, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.876828125, + "pass@2": 0.9050915354330714, + "pass@4": 0.9210701640419947, + "pass@8": 0.9323957294688712, + "pass@16": 0.9417445571439508, + "pass@32": 0.9496818880297414, + "pass@64": 0.9564819002934766, + "pass@128": 0.962 + }, + "13": { + "pass@1": 0.7169296875, + "pass@2": 0.7833086860236226, + "pass@4": 0.826883173509561, + "pass@8": 0.8569773128392556, + "pass@16": 0.8789262377032174, + "pass@32": 0.8962766853473002, + "pass@64": 0.9113192583979964, + "pass@128": 0.925 + }, + "14": { + "pass@1": 0.5856640625, + "pass@2": 0.6690511811023618, + "pass@4": 0.730025177540307, + "pass@8": 0.7743548745619078, + "pass@16": 0.8074436240805946, + "pass@32": 0.8332576053171329, + "pass@64": 0.8561163266122096, + "pass@128": 0.879 + }, + "15": { + "pass@1": 0.4001875, + "pass@2": 0.4909625984251965, + "pass@4": 0.5661360222159728, + "pass@8": 0.6283070554789572, + "pass@16": 0.6798224689664408, + "pass@32": 0.7212807512964146, + "pass@64": 0.7544626021658781, + "pass@128": 0.784 + }, + "16": { + "pass@1": 0.251875, + "pass@2": 0.3296031003937008, + "pass@4": 0.4040292357517809, + "pass@8": 0.4689540280820206, + "pass@16": 0.5237924778696498, + "pass@32": 0.5709675615189868, + "pass@64": 0.6123160157191254, + "pass@128": 0.649 + }, + "17": { + "pass@1": 0.2203046875, + "pass@2": 0.2941596948818898, + "pass@4": 0.36483695781777314, + "pass@8": 0.42847699450621324, + "pass@16": 0.484524570191649, + "pass@32": 0.5324919647838158, + "pass@64": 0.5734902117491443, + "pass@128": 0.61 + }, + "18": { + "pass@1": 0.196734375, + "pass@2": 0.2648484251968504, + "pass@4": 0.33271673950131214, + "pass@8": 0.39401642909309553, + "pass@16": 0.445642840840323, + "pass@32": 0.48856944938555213, + "pass@64": 0.5241226688826054, + "pass@128": 0.553 + }, + "19": { + "pass@1": 0.1768671875, + "pass@2": 0.24542593503937, + "pass@4": 0.31300527493438296, + "pass@8": 0.37219563370498276, + "pass@16": 0.4201604402396626, + "pass@32": 0.4573659930356741, + "pass@64": 0.4859184267552251, + "pass@128": 0.508 + }, + "20": { + "pass@1": 0.1608359375, + "pass@2": 0.22545078740157487, + "pass@4": 0.2932503850768652, + "pass@8": 0.3533528925617542, + "pass@16": 0.4012188482204422, + "pass@32": 0.4398218950052932, + "pass@64": 0.4740409369188567, + "pass@128": 0.508 + } + }, + "per_op_accuracy": { + "11": 0.9614609375, + "12": 0.876828125, + "13": 0.7169296875, + "14": 0.5856640625, + "15": 0.4001875, + "16": 0.251875, + "17": 0.2203046875, + "18": 0.196734375, + "19": 0.1768671875, + "20": 0.1608359375 + }, + "per_op_avg_response_len": { + "11": 299.8383984375, + "12": 308.3663671875, + "13": 321.0816015625, + "14": 336.80084375, + "15": 339.737796875, + "16": 333.0804453125, + "17": 343.17490625, + "18": 341.5718828125, + "19": 341.74196875, + "20": 345.40865625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 299.8383984375, + "12": 308.3663671875, + "13": 321.0816015625, + "14": 336.80084375, + "15": 339.737796875, + "16": 333.0804453125, + "17": 343.17490625, + "18": 341.5718828125, + "19": 341.74196875, + "20": 345.40865625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 201512, + "answer_accuracy": 0.46784918276374443, + "avg_response_len": 317.5699665676077, + "resp_tokens_sum": 136783736, + "pass_at_k": { + "pass@1": 0.46784918276374443, + "pass@2": 0.5333591218073984, + "pass@4": 0.5902502452516635, + "pass@8": 0.6371174034815448, + "pass@16": 0.67485727501802, + "pass@32": 0.705756673823739, + "pass@64": 0.7315732185590125, + "pass@128": 0.7536404160475483 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9561791784702549, + "pass@2": 0.9675670852758136, + "pass@4": 0.975681331795282, + "pass@8": 0.9810131072714366, + "pass@16": 0.9835244887974681, + "pass@32": 0.9846345398787658, + "pass@64": 0.9854899479948672, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8586246468926554, + "pass@2": 0.885517107633792, + "pass@4": 0.9009458769984255, + "pass@8": 0.9127409043055847, + "pass@16": 0.9235052258167523, + "pass@32": 0.9340428536048586, + "pass@64": 0.9437995860562227, + "pass@128": 0.9519774011299436 + }, + "13": { + "pass@1": 0.749828565830721, + "pass@2": 0.8144654185076395, + "pass@4": 0.8556652423737, + "pass@8": 0.8817049955442472, + "pass@16": 0.9004751710961635, + "pass@32": 0.9147945484990276, + "pass@64": 0.9252460963875951, + "pass@128": 0.9341692789968652 + }, + "14": { + "pass@1": 0.6094723520249221, + "pass@2": 0.6954702179458873, + "pass@4": 0.759014966003081, + "pass@8": 0.8040812778826939, + "pass@16": 0.8354157537957082, + "pass@32": 0.8577169175303099, + "pass@64": 0.8760674765786626, + "pass@128": 0.8940809968847352 + }, + "15": { + "pass@1": 0.42860368497109824, + "pass@2": 0.5288988894451778, + "pass@4": 0.612769557165181, + "pass@8": 0.6788524746204303, + "pass@16": 0.7287998078593918, + "pass@32": 0.7667274694817797, + "pass@64": 0.7958846324093649, + "pass@128": 0.8208092485549133 + }, + "16": { + "pass@1": 0.28805443548387094, + "pass@2": 0.3804229108458218, + "pass@4": 0.46554370048502003, + "pass@8": 0.5358038258607328, + "pass@16": 0.5923360339603841, + "pass@32": 0.6386729617302079, + "pass@64": 0.6768725367410589, + "pass@128": 0.7096774193548387 + }, + "17": { + "pass@1": 0.22473356213872833, + "pass@2": 0.30177705839515717, + "pass@4": 0.3746679429702789, + "pass@8": 0.4379777145579087, + "pass@16": 0.49274628514686913, + "pass@32": 0.5410815735405551, + "pass@64": 0.5859895708211883, + "pass@128": 0.6271676300578035 + }, + "18": { + "pass@1": 0.19457589285714286, + "pass@2": 0.2576493953880764, + "pass@4": 0.3194338641598372, + "pass@8": 0.37903471601193817, + "pass@16": 0.4346304669993643, + "pass@32": 0.48400334941028167, + "pass@64": 0.5249235902083653, + "pass@128": 0.5571428571428572 + }, + "19": { + "pass@1": 0.1709834834834835, + "pass@2": 0.24323954860372177, + "pass@4": 0.31548648648648664, + "pass@8": 0.3805533785151254, + "pass@16": 0.4344145866031716, + "pass@32": 0.47760243775952355, + "pass@64": 0.5105839400221093, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.17297766516516516, + "pass@2": 0.24068396644676152, + "pass@4": 0.31054034124112867, + "pass@8": 0.36999794241197487, + "pass@16": 0.4153449246437809, + "pass@32": 0.45214451073733947, + "pass@64": 0.48553465858199635, + "pass@128": 0.5195195195195195 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 195261, + "answer_accuracy": 0.4536058764495986, + "avg_response_len": 325.596231043711, + "resp_tokens_sum": 140157456, + "pass_at_k": { + "pass@1": 0.4536058764495986, + "pass@2": 0.5172185194485628, + "pass@4": 0.5730881503326823, + "pass@8": 0.6194874500005778, + "pass@16": 0.6564931349553177, + "pass@32": 0.6864169731395222, + "pass@64": 0.7117922934759667, + "pass@128": 0.7344632768361582 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9630788780120482, + "pass@2": 0.9725510061901149, + "pass@4": 0.978362016118467, + "pass@8": 0.9820410410160375, + "pass@16": 0.9837182925561317, + "pass@32": 0.9845597646340878, + "pass@64": 0.984919964089104, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8823961598746082, + "pass@2": 0.911209056352282, + "pass@4": 0.9269458410175219, + "pass@8": 0.9371800278526862, + "pass@16": 0.9452997534834727, + "pass@32": 0.9515967660438803, + "pass@64": 0.9562321695616715, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7165835084033614, + "pass@2": 0.7848075746046448, + "pass@4": 0.8304916675331546, + "pass@8": 0.8631105161177065, + "pass@16": 0.8860488644391403, + "pass@32": 0.9036331706942465, + "pass@64": 0.9195475785189685, + "pass@128": 0.9327731092436975 + }, + "14": { + "pass@1": 0.5826937134502924, + "pass@2": 0.6628357824515355, + "pass@4": 0.7239523921900111, + "pass@8": 0.7699296975890264, + "pass@16": 0.8033629507371602, + "pass@32": 0.8279615689251119, + "pass@64": 0.8480355965835796, + "pass@128": 0.8654970760233918 + }, + "15": { + "pass@1": 0.3943655303030303, + "pass@2": 0.48420499284180357, + "pass@4": 0.5579077541443676, + "pass@8": 0.6173687878816342, + "pass@16": 0.6663543368828038, + "pass@32": 0.7072083491588049, + "pass@64": 0.7419320321919645, + "pass@128": 0.7757575757575758 + }, + "16": { + "pass@1": 0.252559650997151, + "pass@2": 0.3288831684500974, + "pass@4": 0.4029608232731589, + "pass@8": 0.46807047166728, + "pass@16": 0.5228549084834682, + "pass@32": 0.5696850244103808, + "pass@64": 0.6106096798964789, + "pass@128": 0.6495726495726496 + }, + "17": { + "pass@1": 0.23238548136645962, + "pass@2": 0.31177824069545657, + "pass@4": 0.3867177496640558, + "pass@8": 0.4531720556392637, + "pass@16": 0.5098132628952892, + "pass@32": 0.556123889213355, + "pass@64": 0.5943525415671919, + "pass@128": 0.6273291925465838 + }, + "18": { + "pass@1": 0.19457720588235294, + "pass@2": 0.2666968214451134, + "pass@4": 0.34146044988861685, + "pass@8": 0.4093940578883462, + "pass@16": 0.4663716932969577, + "pass@32": 0.5134669551231071, + "pass@64": 0.5519850631468153, + "pass@128": 0.5823529411764706 + }, + "19": { + "pass@1": 0.18171037946428573, + "pass@2": 0.248879900754593, + "pass@4": 0.3143549832498615, + "pass@8": 0.3685924624601657, + "pass@16": 0.4103747750358778, + "pass@32": 0.4436010678719908, + "pass@64": 0.47334645015893895, + "pass@128": 0.5029761904761905 + }, + "20": { + "pass@1": 0.14467159431137724, + "pass@2": 0.20676142793153857, + "pass@4": 0.2721992303357292, + "pass@8": 0.32905671999847613, + "pass@16": 0.37287595740616736, + "pass@32": 0.4077495726571193, + "pass@64": 0.4375955624374268, + "pass@128": 0.46407185628742514 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 185331, + "answer_accuracy": 0.4425117474022005, + "avg_response_len": 350.6111872516809, + "resp_tokens_sum": 146841575, + "pass_at_k": { + "pass@1": 0.4425117474022005, + "pass@2": 0.5028298748147022, + "pass@4": 0.5550691146797369, + "pass@8": 0.6001755112430429, + "pass@16": 0.6384196254691307, + "pass@32": 0.669967112373797, + "pass@64": 0.6967506946800839, + "pass@128": 0.7212713936430318 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9656746031746032, + "pass@2": 0.9748527527809023, + "pass@4": 0.9793296451038858, + "pass@8": 0.9829981811852763, + "pass@16": 0.9858269044294672, + "pass@32": 0.988127782909697, + "pass@64": 0.9908886006022682, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8911028287461774, + "pass@2": 0.9203143435912254, + "pass@4": 0.9371241599387236, + "pass@8": 0.9490061787758015, + "pass@16": 0.9580216692464635, + "pass@32": 0.9647441880294301, + "pass@64": 0.97045499920306, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.6849199459876543, + "pass@2": 0.7509812141537865, + "pass@4": 0.7945692465525145, + "pass@8": 0.8258733487858642, + "pass@16": 0.849861770119659, + "pass@32": 0.8699388346859993, + "pass@64": 0.88854095098482, + "pass@128": 0.9074074074074074 + }, + "14": { + "pass@1": 0.5660005563798219, + "pass@2": 0.6501940757961634, + "pass@4": 0.7085746448793305, + "pass@8": 0.7505306462496025, + "pass@16": 0.7849407654601296, + "pass@32": 0.815334178686988, + "pass@64": 0.845313034418024, + "pass@128": 0.8783382789317508 + }, + "15": { + "pass@1": 0.3757716049382716, + "pass@2": 0.4573331328375619, + "pass@4": 0.5247166560777126, + "pass@8": 0.5854703680844102, + "pass@16": 0.641237050233849, + "pass@32": 0.6870811470164, + "pass@64": 0.7229905824965706, + "pass@128": 0.7530864197530864 + }, + "16": { + "pass@1": 0.21808167404129794, + "pass@2": 0.2838761236150791, + "pass@4": 0.3488833027508728, + "pass@8": 0.4087377773155697, + "pass@16": 0.462083287505114, + "pass@32": 0.5103820053527391, + "pass@64": 0.5550487660346097, + "pass@128": 0.5929203539823009 + }, + "17": { + "pass@1": 0.20397213855421686, + "pass@2": 0.2691332505217722, + "pass@4": 0.3333696811994885, + "pass@8": 0.39462440769076684, + "pass@16": 0.451429171320933, + "pass@32": 0.5006200241327831, + "pass@64": 0.5402298248806521, + "pass@128": 0.5753012048192772 + }, + "18": { + "pass@1": 0.20153729838709677, + "pass@2": 0.27094908877317764, + "pass@4": 0.33812365833303115, + "pass@8": 0.3940654800221941, + "pass@16": 0.4353412957083234, + "pass@32": 0.4664177817745076, + "pass@64": 0.49265964787019434, + "pass@128": 0.5161290322580645 + }, + "19": { + "pass@1": 0.1778700906344411, + "pass@2": 0.24411939184527925, + "pass@4": 0.30913897450885125, + "pass@8": 0.3674449887396694, + "pass@16": 0.41575365102341894, + "pass@32": 0.4509801282681683, + "pass@64": 0.47386576276271636, + "pass@128": 0.48942598187311176 + }, + "20": { + "pass@1": 0.1649070945945946, + "pass@2": 0.228963092029037, + "pass@4": 0.2970748003346429, + "pass@8": 0.3610769767539805, + "pass@16": 0.41552077639760837, + "pass@32": 0.4596679149017448, + "pass@64": 0.4991020353061597, + "pass@128": 0.5405405405405406 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7113096217105264, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7113096217105264, + "pass@2": 0.7450627978657143, + "pass@4": 0.7743897328557456, + "pass@8": 0.7989574161151894, + "pass@16": 0.8189939757417569, + "pass@32": 0.8353470049785026, + "pass@64": 0.8491152044230846, + "pass@128": 0.8613157894736843 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9741484375, + "pass@2": 0.9814842519685039, + "pass@4": 0.9872013810461194, + "pass@8": 0.9916198616262941, + "pass@16": 0.9947940596672118, + "pass@32": 0.9970391170143708, + "pass@64": 0.998421360200285, + "pass@128": 0.999 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994296875, + "pass@2": 0.9997584891732283, + "pass@4": 0.9999441632920885, + "pass@8": 0.9999972911187579, + "pass@16": 0.9999999960747445, + "pass@32": 0.9999999999999993, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999296875, + "pass@2": 0.9999988927165354, + "pass@4": 0.9999999999062616, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9970546875, + "pass@2": 0.9981833169291339, + "pass@4": 0.9987903293025873, + "pass@8": 0.9992771876033886, + "pass@16": 0.999711012970505, + "pass@32": 0.9999493838432147, + "pass@64": 0.9999993322377224, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966484375, + "pass@2": 0.9973496555118111, + "pass@4": 0.99779538263967, + "pass@8": 0.998285935985465, + "pass@16": 0.9987903125553651, + "pass@32": 0.9991807139615209, + "pass@64": 0.9994985551965363, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9614609375, + "pass@2": 0.9715167322834649, + "pass@4": 0.9777205376827895, + "pass@8": 0.9819796795575038, + "pass@16": 0.9843140925694238, + "pass@32": 0.9857100860522761, + "pass@64": 0.9870012889094854, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.876828125, + "pass@2": 0.9050915354330714, + "pass@4": 0.9210701640419947, + "pass@8": 0.9323957294688712, + "pass@16": 0.9417445571439508, + "pass@32": 0.9496818880297414, + "pass@64": 0.9564819002934766, + "pass@128": 0.962 + }, + "13": { + "pass@1": 0.7169296875, + "pass@2": 0.7833086860236226, + "pass@4": 0.826883173509561, + "pass@8": 0.8569773128392556, + "pass@16": 0.8789262377032174, + "pass@32": 0.8962766853473002, + "pass@64": 0.9113192583979964, + "pass@128": 0.925 + }, + "14": { + "pass@1": 0.5856640625, + "pass@2": 0.6690511811023618, + "pass@4": 0.730025177540307, + "pass@8": 0.7743548745619078, + "pass@16": 0.8074436240805946, + "pass@32": 0.8332576053171329, + "pass@64": 0.8561163266122096, + "pass@128": 0.879 + }, + "15": { + "pass@1": 0.4001875, + "pass@2": 0.4909625984251965, + "pass@4": 0.5661360222159728, + "pass@8": 0.6283070554789572, + "pass@16": 0.6798224689664408, + "pass@32": 0.7212807512964146, + "pass@64": 0.7544626021658781, + "pass@128": 0.784 + }, + "16": { + "pass@1": 0.251875, + "pass@2": 0.3296031003937008, + "pass@4": 0.4040292357517809, + "pass@8": 0.4689540280820206, + "pass@16": 0.5237924778696498, + "pass@32": 0.5709675615189868, + "pass@64": 0.6123160157191254, + "pass@128": 0.649 + }, + "17": { + "pass@1": 0.2203046875, + "pass@2": 0.2941596948818898, + "pass@4": 0.36483695781777314, + "pass@8": 0.42847699450621324, + "pass@16": 0.484524570191649, + "pass@32": 0.5324919647838158, + "pass@64": 0.5734902117491443, + "pass@128": 0.61 + }, + "18": { + "pass@1": 0.196734375, + "pass@2": 0.2648484251968504, + "pass@4": 0.33271673950131214, + "pass@8": 0.39401642909309553, + "pass@16": 0.445642840840323, + "pass@32": 0.48856944938555213, + "pass@64": 0.5241226688826054, + "pass@128": 0.553 + }, + "19": { + "pass@1": 0.1768671875, + "pass@2": 0.24542593503937, + "pass@4": 0.31300527493438296, + "pass@8": 0.37219563370498276, + "pass@16": 0.4201604402396626, + "pass@32": 0.4573659930356741, + "pass@64": 0.4859184267552251, + "pass@128": 0.508 + }, + "20": { + "pass@1": 0.1608359375, + "pass@2": 0.22545078740157487, + "pass@4": 0.2932503850768652, + "pass@8": 0.3533528925617542, + "pass@16": 0.4012188482204422, + "pass@32": 0.4398218950052932, + "pass@64": 0.4740409369188567, + "pass@128": 0.508 + } + }, + "per_op_accuracy": { + "10": 0.9741484375, + "2": 1.0, + "3": 1.0, + "4": 0.999984375, + "5": 0.9994296875, + "6": 1.0, + "7": 0.9999296875, + "8": 0.9970546875, + "9": 0.9966484375, + "11": 0.9614609375, + "12": 0.876828125, + "13": 0.7169296875, + "14": 0.5856640625, + "15": 0.4001875, + "16": 0.251875, + "17": 0.2203046875, + "18": 0.196734375, + "19": 0.1768671875, + "20": 0.1608359375 + }, + "per_op_avg_response_len": { + "10": 282.1082265625, + "2": 115.728890625, + "3": 144.8953125, + "4": 160.0852109375, + "5": 172.9949375, + "6": 199.2273984375, + "7": 224.3072421875, + "8": 243.96909375, + "9": 259.9670078125, + "11": 299.8383984375, + "12": 308.3663671875, + "13": 321.0816015625, + "14": 336.80084375, + "15": 339.737796875, + "16": 333.0804453125, + "17": 343.17490625, + "18": 341.5718828125, + "19": 341.74196875, + "20": 345.40865625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 282.1082265625, + "2": 115.728890625, + "3": 144.8953125, + "4": 160.0852109375, + "5": 172.9949375, + "6": 199.2273984375, + "7": 224.3072421875, + "8": 243.96909375, + "9": 259.9670078125, + "11": 299.8383984375, + "12": 308.3663671875, + "13": 321.0816015625, + "14": 336.80084375, + "15": 339.737796875, + "16": 333.0804453125, + "17": 343.17490625, + "18": 341.5718828125, + "19": 341.74196875, + "20": 345.40865625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 579321, + "answer_accuracy": 0.7154513614448309, + "avg_response_len": 258.28444983994626, + "resp_tokens_sum": 209140151, + "pass_at_k": { + "pass@1": 0.7154513614448309, + "pass@2": 0.7507491190586985, + "pass@4": 0.7813239936604092, + "pass@8": 0.8064721895572778, + "pass@16": 0.8267004105419402, + "pass@32": 0.8432565294076302, + "pass@64": 0.8570965246074687, + "pass@128": 0.8689535251343661 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9772077802359882, + "pass@2": 0.984406937379509, + "pass@4": 0.9896404088869423, + "pass@8": 0.9934608180094582, + "pass@16": 0.9961270518521915, + "pass@32": 0.9979929133208758, + "pass@64": 0.9992588029962364, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 0.9999996155265748, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9984375, + "pass@2": 0.9992372047244095, + "pass@4": 0.9998227409073865, + "pass@8": 0.9999914003770088, + "pass@16": 0.9999999875388716, + "pass@32": 0.999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.999824719551282, + "pass@2": 0.9999917190339189, + "pass@4": 0.9999999894844874, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9970730633802817, + "pass@2": 0.9972260729732727, + "pass@4": 0.9972711267605634, + "pass@8": 0.9973591549295775, + "pass@16": 0.9975352112676056, + "pass@32": 0.997887323943662, + "pass@64": 0.9985915492957746, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9561791784702549, + "pass@2": 0.9675670852758136, + "pass@4": 0.975681331795282, + "pass@8": 0.9810131072714366, + "pass@16": 0.9835244887974681, + "pass@32": 0.9846345398787658, + "pass@64": 0.9854899479948672, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8586246468926554, + "pass@2": 0.885517107633792, + "pass@4": 0.9009458769984255, + "pass@8": 0.9127409043055847, + "pass@16": 0.9235052258167523, + "pass@32": 0.9340428536048586, + "pass@64": 0.9437995860562227, + "pass@128": 0.9519774011299436 + }, + "13": { + "pass@1": 0.749828565830721, + "pass@2": 0.8144654185076395, + "pass@4": 0.8556652423737, + "pass@8": 0.8817049955442472, + "pass@16": 0.9004751710961635, + "pass@32": 0.9147945484990276, + "pass@64": 0.9252460963875951, + "pass@128": 0.9341692789968652 + }, + "14": { + "pass@1": 0.6094723520249221, + "pass@2": 0.6954702179458873, + "pass@4": 0.759014966003081, + "pass@8": 0.8040812778826939, + "pass@16": 0.8354157537957082, + "pass@32": 0.8577169175303099, + "pass@64": 0.8760674765786626, + "pass@128": 0.8940809968847352 + }, + "15": { + "pass@1": 0.42860368497109824, + "pass@2": 0.5288988894451778, + "pass@4": 0.612769557165181, + "pass@8": 0.6788524746204303, + "pass@16": 0.7287998078593918, + "pass@32": 0.7667274694817797, + "pass@64": 0.7958846324093649, + "pass@128": 0.8208092485549133 + }, + "16": { + "pass@1": 0.28805443548387094, + "pass@2": 0.3804229108458218, + "pass@4": 0.46554370048502003, + "pass@8": 0.5358038258607328, + "pass@16": 0.5923360339603841, + "pass@32": 0.6386729617302079, + "pass@64": 0.6768725367410589, + "pass@128": 0.7096774193548387 + }, + "17": { + "pass@1": 0.22473356213872833, + "pass@2": 0.30177705839515717, + "pass@4": 0.3746679429702789, + "pass@8": 0.4379777145579087, + "pass@16": 0.49274628514686913, + "pass@32": 0.5410815735405551, + "pass@64": 0.5859895708211883, + "pass@128": 0.6271676300578035 + }, + "18": { + "pass@1": 0.19457589285714286, + "pass@2": 0.2576493953880764, + "pass@4": 0.3194338641598372, + "pass@8": 0.37903471601193817, + "pass@16": 0.4346304669993643, + "pass@32": 0.48400334941028167, + "pass@64": 0.5249235902083653, + "pass@128": 0.5571428571428572 + }, + "19": { + "pass@1": 0.1709834834834835, + "pass@2": 0.24323954860372177, + "pass@4": 0.31548648648648664, + "pass@8": 0.3805533785151254, + "pass@16": 0.4344145866031716, + "pass@32": 0.47760243775952355, + "pass@64": 0.5105839400221093, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.17297766516516516, + "pass@2": 0.24068396644676152, + "pass@4": 0.31054034124112867, + "pass@8": 0.36999794241197487, + "pass@16": 0.4153449246437809, + "pass@32": 0.45214451073733947, + "pass@64": 0.48553465858199635, + "pass@128": 0.5195195195195195 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 584529, + "answer_accuracy": 0.711867936477007, + "avg_response_len": 265.081010083788, + "resp_tokens_sum": 217663319, + "pass_at_k": { + "pass@1": 0.711867936477007, + "pass@2": 0.7457282467273387, + "pass@4": 0.775357032636113, + "pass@8": 0.7999651978143069, + "pass@16": 0.8196149441909102, + "pass@32": 0.8355008000370855, + "pass@64": 0.8488983113884123, + "pass@128": 0.8607950116913484 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9769699003067485, + "pass@2": 0.9845029376600167, + "pass@4": 0.9894387797000833, + "pass@8": 0.9929804552880818, + "pass@16": 0.9957689493418113, + "pass@32": 0.9982698846328565, + "pass@64": 0.9997769849567611, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999320652173913, + "pass@2": 0.9999996433869679, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999096820809249, + "pass@2": 0.9999989332529243, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9947623239436619, + "pass@2": 0.9956644393922591, + "pass@4": 0.9966619528192779, + "pass@8": 0.9979645098268615, + "pass@16": 0.9991859520597687, + "pass@32": 0.9998574192766612, + "pass@64": 0.9999981189794995, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957428892215568, + "pass@2": 0.9971032344759301, + "pass@4": 0.9977481417929045, + "pass@8": 0.9983575455521196, + "pass@16": 0.999137016955876, + "pass@32": 0.9997963632818704, + "pass@64": 0.9999956744411302, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9630788780120482, + "pass@2": 0.9725510061901149, + "pass@4": 0.978362016118467, + "pass@8": 0.9820410410160375, + "pass@16": 0.9837182925561317, + "pass@32": 0.9845597646340878, + "pass@64": 0.984919964089104, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8823961598746082, + "pass@2": 0.911209056352282, + "pass@4": 0.9269458410175219, + "pass@8": 0.9371800278526862, + "pass@16": 0.9452997534834727, + "pass@32": 0.9515967660438803, + "pass@64": 0.9562321695616715, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7165835084033614, + "pass@2": 0.7848075746046448, + "pass@4": 0.8304916675331546, + "pass@8": 0.8631105161177065, + "pass@16": 0.8860488644391403, + "pass@32": 0.9036331706942465, + "pass@64": 0.9195475785189685, + "pass@128": 0.9327731092436975 + }, + "14": { + "pass@1": 0.5826937134502924, + "pass@2": 0.6628357824515355, + "pass@4": 0.7239523921900111, + "pass@8": 0.7699296975890264, + "pass@16": 0.8033629507371602, + "pass@32": 0.8279615689251119, + "pass@64": 0.8480355965835796, + "pass@128": 0.8654970760233918 + }, + "15": { + "pass@1": 0.3943655303030303, + "pass@2": 0.48420499284180357, + "pass@4": 0.5579077541443676, + "pass@8": 0.6173687878816342, + "pass@16": 0.6663543368828038, + "pass@32": 0.7072083491588049, + "pass@64": 0.7419320321919645, + "pass@128": 0.7757575757575758 + }, + "16": { + "pass@1": 0.252559650997151, + "pass@2": 0.3288831684500974, + "pass@4": 0.4029608232731589, + "pass@8": 0.46807047166728, + "pass@16": 0.5228549084834682, + "pass@32": 0.5696850244103808, + "pass@64": 0.6106096798964789, + "pass@128": 0.6495726495726496 + }, + "17": { + "pass@1": 0.23238548136645962, + "pass@2": 0.31177824069545657, + "pass@4": 0.3867177496640558, + "pass@8": 0.4531720556392637, + "pass@16": 0.5098132628952892, + "pass@32": 0.556123889213355, + "pass@64": 0.5943525415671919, + "pass@128": 0.6273291925465838 + }, + "18": { + "pass@1": 0.19457720588235294, + "pass@2": 0.2666968214451134, + "pass@4": 0.34146044988861685, + "pass@8": 0.4093940578883462, + "pass@16": 0.4663716932969577, + "pass@32": 0.5134669551231071, + "pass@64": 0.5519850631468153, + "pass@128": 0.5823529411764706 + }, + "19": { + "pass@1": 0.18171037946428573, + "pass@2": 0.248879900754593, + "pass@4": 0.3143549832498615, + "pass@8": 0.3685924624601657, + "pass@16": 0.4103747750358778, + "pass@32": 0.4436010678719908, + "pass@64": 0.47334645015893895, + "pass@128": 0.5029761904761905 + }, + "20": { + "pass@1": 0.14467159431137724, + "pass@2": 0.20676142793153857, + "pass@4": 0.2721992303357292, + "pass@8": 0.32905671999847613, + "pass@16": 0.37287595740616736, + "pass@32": 0.4077495726571193, + "pass@64": 0.4375955624374268, + "pass@128": 0.46407185628742514 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 566055, + "answer_accuracy": 0.706551316104809, + "avg_response_len": 284.34000289583, + "resp_tokens_sum": 227799562, + "pass_at_k": { + "pass@1": 0.706551316104809, + "pass@2": 0.7386335723801782, + "pass@4": 0.7663898347983656, + "pass@8": 0.7903293003787032, + "pass@16": 0.810568601217498, + "pass@32": 0.8271951841062035, + "pass@64": 0.8412707464155824, + "pass@128": 0.8541300527240774 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9683069029850746, + "pass@2": 0.9755890821483134, + "pass@4": 0.9825559410484136, + "pass@8": 0.9884328832751443, + "pass@16": 0.9924964525489203, + "pass@32": 0.9948762298754696, + "pass@64": 0.9962547131004683, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9998391544117647, + "pass@2": 0.9999967432839278, + "pass@4": 0.9999999997242991, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9969031531531531, + "pass@2": 0.9991742480669644, + "pass@4": 0.9999259439416918, + "pass@8": 0.9999993592037622, + "pass@16": 0.9999999999678294, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9971362540192926, + "pass@2": 0.9977553674962655, + "pass@4": 0.9984445443113824, + "pass@8": 0.9992669317397335, + "pass@16": 0.9998506395244453, + "pass@32": 0.9999959119787014, + "pass@64": 0.9999999997852048, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9656746031746032, + "pass@2": 0.9748527527809023, + "pass@4": 0.9793296451038858, + "pass@8": 0.9829981811852763, + "pass@16": 0.9858269044294672, + "pass@32": 0.988127782909697, + "pass@64": 0.9908886006022682, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8911028287461774, + "pass@2": 0.9203143435912254, + "pass@4": 0.9371241599387236, + "pass@8": 0.9490061787758015, + "pass@16": 0.9580216692464635, + "pass@32": 0.9647441880294301, + "pass@64": 0.97045499920306, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.6849199459876543, + "pass@2": 0.7509812141537865, + "pass@4": 0.7945692465525145, + "pass@8": 0.8258733487858642, + "pass@16": 0.849861770119659, + "pass@32": 0.8699388346859993, + "pass@64": 0.88854095098482, + "pass@128": 0.9074074074074074 + }, + "14": { + "pass@1": 0.5660005563798219, + "pass@2": 0.6501940757961634, + "pass@4": 0.7085746448793305, + "pass@8": 0.7505306462496025, + "pass@16": 0.7849407654601296, + "pass@32": 0.815334178686988, + "pass@64": 0.845313034418024, + "pass@128": 0.8783382789317508 + }, + "15": { + "pass@1": 0.3757716049382716, + "pass@2": 0.4573331328375619, + "pass@4": 0.5247166560777126, + "pass@8": 0.5854703680844102, + "pass@16": 0.641237050233849, + "pass@32": 0.6870811470164, + "pass@64": 0.7229905824965706, + "pass@128": 0.7530864197530864 + }, + "16": { + "pass@1": 0.21808167404129794, + "pass@2": 0.2838761236150791, + "pass@4": 0.3488833027508728, + "pass@8": 0.4087377773155697, + "pass@16": 0.462083287505114, + "pass@32": 0.5103820053527391, + "pass@64": 0.5550487660346097, + "pass@128": 0.5929203539823009 + }, + "17": { + "pass@1": 0.20397213855421686, + "pass@2": 0.2691332505217722, + "pass@4": 0.3333696811994885, + "pass@8": 0.39462440769076684, + "pass@16": 0.451429171320933, + "pass@32": 0.5006200241327831, + "pass@64": 0.5402298248806521, + "pass@128": 0.5753012048192772 + }, + "18": { + "pass@1": 0.20153729838709677, + "pass@2": 0.27094908877317764, + "pass@4": 0.33812365833303115, + "pass@8": 0.3940654800221941, + "pass@16": 0.4353412957083234, + "pass@32": 0.4664177817745076, + "pass@64": 0.49265964787019434, + "pass@128": 0.5161290322580645 + }, + "19": { + "pass@1": 0.1778700906344411, + "pass@2": 0.24411939184527925, + "pass@4": 0.30913897450885125, + "pass@8": 0.3674449887396694, + "pass@16": 0.41575365102341894, + "pass@32": 0.4509801282681683, + "pass@64": 0.47386576276271636, + "pass@128": 0.48942598187311176 + }, + "20": { + "pass@1": 0.1649070945945946, + "pass@2": 0.228963092029037, + "pass@4": 0.2970748003346429, + "pass@8": 0.3610769767539805, + "pass@16": 0.41552077639760837, + "pass@32": 0.4596679149017448, + "pass@64": 0.4991020353061597, + "pass@128": 0.5405405405405406 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch4-RL-op7-10/composition/summary.csv b/composition/Qwen3-100M-epoch4-RL-op7-10/composition/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..98a5e21931173d4588f959a8a8fd98cab042db02 --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op7-10/composition/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.9963550347222222,0.45476875,0.7113096217105264,0.0,0.0,0.0,200.36481336805556,331.08028671875,1152000,1280000,2432000 diff --git a/composition/Qwen3-100M-epoch4-RL-op7-10/composition/summary.json b/composition/Qwen3-100M-epoch4-RL-op7-10/composition/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..53ba21263b046a4fe103264f4240782e5517ee5f --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op7-10/composition/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.9963550347222222, + "ood_acc": 0.45476875, + "total_acc": 0.7113096217105264, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.36481336805556, + "ood_avg_resp_len": 331.08028671875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 579321, + "answer_accuracy": 0.7154513614448309, + "avg_response_len": 258.28444983994626, + "resp_tokens_sum": 209140151, + "pass_at_k": { + "pass@1": 0.7154513614448309, + "pass@2": 0.7507491190586985, + "pass@4": 0.7813239936604092, + "pass@8": 0.8064721895572778, + "pass@16": 0.8267004105419402, + "pass@32": 0.8432565294076302, + "pass@64": 0.8570965246074687, + "pass@128": 0.8689535251343661 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9772077802359882, + "pass@2": 0.984406937379509, + "pass@4": 0.9896404088869423, + "pass@8": 0.9934608180094582, + "pass@16": 0.9961270518521915, + "pass@32": 0.9979929133208758, + "pass@64": 0.9992588029962364, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 0.9999996155265748, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9984375, + "pass@2": 0.9992372047244095, + "pass@4": 0.9998227409073865, + "pass@8": 0.9999914003770088, + "pass@16": 0.9999999875388716, + "pass@32": 0.999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.999824719551282, + "pass@2": 0.9999917190339189, + "pass@4": 0.9999999894844874, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9970730633802817, + "pass@2": 0.9972260729732727, + "pass@4": 0.9972711267605634, + "pass@8": 0.9973591549295775, + "pass@16": 0.9975352112676056, + "pass@32": 0.997887323943662, + "pass@64": 0.9985915492957746, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9561791784702549, + "pass@2": 0.9675670852758136, + "pass@4": 0.975681331795282, + "pass@8": 0.9810131072714366, + "pass@16": 0.9835244887974681, + "pass@32": 0.9846345398787658, + "pass@64": 0.9854899479948672, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8586246468926554, + "pass@2": 0.885517107633792, + "pass@4": 0.9009458769984255, + "pass@8": 0.9127409043055847, + "pass@16": 0.9235052258167523, + "pass@32": 0.9340428536048586, + "pass@64": 0.9437995860562227, + "pass@128": 0.9519774011299436 + }, + "13": { + "pass@1": 0.749828565830721, + "pass@2": 0.8144654185076395, + "pass@4": 0.8556652423737, + "pass@8": 0.8817049955442472, + "pass@16": 0.9004751710961635, + "pass@32": 0.9147945484990276, + "pass@64": 0.9252460963875951, + "pass@128": 0.9341692789968652 + }, + "14": { + "pass@1": 0.6094723520249221, + "pass@2": 0.6954702179458873, + "pass@4": 0.759014966003081, + "pass@8": 0.8040812778826939, + "pass@16": 0.8354157537957082, + "pass@32": 0.8577169175303099, + "pass@64": 0.8760674765786626, + "pass@128": 0.8940809968847352 + }, + "15": { + "pass@1": 0.42860368497109824, + "pass@2": 0.5288988894451778, + "pass@4": 0.612769557165181, + "pass@8": 0.6788524746204303, + "pass@16": 0.7287998078593918, + "pass@32": 0.7667274694817797, + "pass@64": 0.7958846324093649, + "pass@128": 0.8208092485549133 + }, + "16": { + "pass@1": 0.28805443548387094, + "pass@2": 0.3804229108458218, + "pass@4": 0.46554370048502003, + "pass@8": 0.5358038258607328, + "pass@16": 0.5923360339603841, + "pass@32": 0.6386729617302079, + "pass@64": 0.6768725367410589, + "pass@128": 0.7096774193548387 + }, + "17": { + "pass@1": 0.22473356213872833, + "pass@2": 0.30177705839515717, + "pass@4": 0.3746679429702789, + "pass@8": 0.4379777145579087, + "pass@16": 0.49274628514686913, + "pass@32": 0.5410815735405551, + "pass@64": 0.5859895708211883, + "pass@128": 0.6271676300578035 + }, + "18": { + "pass@1": 0.19457589285714286, + "pass@2": 0.2576493953880764, + "pass@4": 0.3194338641598372, + "pass@8": 0.37903471601193817, + "pass@16": 0.4346304669993643, + "pass@32": 0.48400334941028167, + "pass@64": 0.5249235902083653, + "pass@128": 0.5571428571428572 + }, + "19": { + "pass@1": 0.1709834834834835, + "pass@2": 0.24323954860372177, + "pass@4": 0.31548648648648664, + "pass@8": 0.3805533785151254, + "pass@16": 0.4344145866031716, + "pass@32": 0.47760243775952355, + "pass@64": 0.5105839400221093, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.17297766516516516, + "pass@2": 0.24068396644676152, + "pass@4": 0.31054034124112867, + "pass@8": 0.36999794241197487, + "pass@16": 0.4153449246437809, + "pass@32": 0.45214451073733947, + "pass@64": 0.48553465858199635, + "pass@128": 0.5195195195195195 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 584529, + "answer_accuracy": 0.711867936477007, + "avg_response_len": 265.081010083788, + "resp_tokens_sum": 217663319, + "pass_at_k": { + "pass@1": 0.711867936477007, + "pass@2": 0.7457282467273387, + "pass@4": 0.775357032636113, + "pass@8": 0.7999651978143069, + "pass@16": 0.8196149441909102, + "pass@32": 0.8355008000370855, + "pass@64": 0.8488983113884123, + "pass@128": 0.8607950116913484 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9769699003067485, + "pass@2": 0.9845029376600167, + "pass@4": 0.9894387797000833, + "pass@8": 0.9929804552880818, + "pass@16": 0.9957689493418113, + "pass@32": 0.9982698846328565, + "pass@64": 0.9997769849567611, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999320652173913, + "pass@2": 0.9999996433869679, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999096820809249, + "pass@2": 0.9999989332529243, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9947623239436619, + "pass@2": 0.9956644393922591, + "pass@4": 0.9966619528192779, + "pass@8": 0.9979645098268615, + "pass@16": 0.9991859520597687, + "pass@32": 0.9998574192766612, + "pass@64": 0.9999981189794995, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957428892215568, + "pass@2": 0.9971032344759301, + "pass@4": 0.9977481417929045, + "pass@8": 0.9983575455521196, + "pass@16": 0.999137016955876, + "pass@32": 0.9997963632818704, + "pass@64": 0.9999956744411302, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9630788780120482, + "pass@2": 0.9725510061901149, + "pass@4": 0.978362016118467, + "pass@8": 0.9820410410160375, + "pass@16": 0.9837182925561317, + "pass@32": 0.9845597646340878, + "pass@64": 0.984919964089104, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8823961598746082, + "pass@2": 0.911209056352282, + "pass@4": 0.9269458410175219, + "pass@8": 0.9371800278526862, + "pass@16": 0.9452997534834727, + "pass@32": 0.9515967660438803, + "pass@64": 0.9562321695616715, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7165835084033614, + "pass@2": 0.7848075746046448, + "pass@4": 0.8304916675331546, + "pass@8": 0.8631105161177065, + "pass@16": 0.8860488644391403, + "pass@32": 0.9036331706942465, + "pass@64": 0.9195475785189685, + "pass@128": 0.9327731092436975 + }, + "14": { + "pass@1": 0.5826937134502924, + "pass@2": 0.6628357824515355, + "pass@4": 0.7239523921900111, + "pass@8": 0.7699296975890264, + "pass@16": 0.8033629507371602, + "pass@32": 0.8279615689251119, + "pass@64": 0.8480355965835796, + "pass@128": 0.8654970760233918 + }, + "15": { + "pass@1": 0.3943655303030303, + "pass@2": 0.48420499284180357, + "pass@4": 0.5579077541443676, + "pass@8": 0.6173687878816342, + "pass@16": 0.6663543368828038, + "pass@32": 0.7072083491588049, + "pass@64": 0.7419320321919645, + "pass@128": 0.7757575757575758 + }, + "16": { + "pass@1": 0.252559650997151, + "pass@2": 0.3288831684500974, + "pass@4": 0.4029608232731589, + "pass@8": 0.46807047166728, + "pass@16": 0.5228549084834682, + "pass@32": 0.5696850244103808, + "pass@64": 0.6106096798964789, + "pass@128": 0.6495726495726496 + }, + "17": { + "pass@1": 0.23238548136645962, + "pass@2": 0.31177824069545657, + "pass@4": 0.3867177496640558, + "pass@8": 0.4531720556392637, + "pass@16": 0.5098132628952892, + "pass@32": 0.556123889213355, + "pass@64": 0.5943525415671919, + "pass@128": 0.6273291925465838 + }, + "18": { + "pass@1": 0.19457720588235294, + "pass@2": 0.2666968214451134, + "pass@4": 0.34146044988861685, + "pass@8": 0.4093940578883462, + "pass@16": 0.4663716932969577, + "pass@32": 0.5134669551231071, + "pass@64": 0.5519850631468153, + "pass@128": 0.5823529411764706 + }, + "19": { + "pass@1": 0.18171037946428573, + "pass@2": 0.248879900754593, + "pass@4": 0.3143549832498615, + "pass@8": 0.3685924624601657, + "pass@16": 0.4103747750358778, + "pass@32": 0.4436010678719908, + "pass@64": 0.47334645015893895, + "pass@128": 0.5029761904761905 + }, + "20": { + "pass@1": 0.14467159431137724, + "pass@2": 0.20676142793153857, + "pass@4": 0.2721992303357292, + "pass@8": 0.32905671999847613, + "pass@16": 0.37287595740616736, + "pass@32": 0.4077495726571193, + "pass@64": 0.4375955624374268, + "pass@128": 0.46407185628742514 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 566055, + "answer_accuracy": 0.706551316104809, + "avg_response_len": 284.34000289583, + "resp_tokens_sum": 227799562, + "pass_at_k": { + "pass@1": 0.706551316104809, + "pass@2": 0.7386335723801782, + "pass@4": 0.7663898347983656, + "pass@8": 0.7903293003787032, + "pass@16": 0.810568601217498, + "pass@32": 0.8271951841062035, + "pass@64": 0.8412707464155824, + "pass@128": 0.8541300527240774 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9683069029850746, + "pass@2": 0.9755890821483134, + "pass@4": 0.9825559410484136, + "pass@8": 0.9884328832751443, + "pass@16": 0.9924964525489203, + "pass@32": 0.9948762298754696, + "pass@64": 0.9962547131004683, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9998391544117647, + "pass@2": 0.9999967432839278, + "pass@4": 0.9999999997242991, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9969031531531531, + "pass@2": 0.9991742480669644, + "pass@4": 0.9999259439416918, + "pass@8": 0.9999993592037622, + "pass@16": 0.9999999999678294, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9971362540192926, + "pass@2": 0.9977553674962655, + "pass@4": 0.9984445443113824, + "pass@8": 0.9992669317397335, + "pass@16": 0.9998506395244453, + "pass@32": 0.9999959119787014, + "pass@64": 0.9999999997852048, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9656746031746032, + "pass@2": 0.9748527527809023, + "pass@4": 0.9793296451038858, + "pass@8": 0.9829981811852763, + "pass@16": 0.9858269044294672, + "pass@32": 0.988127782909697, + "pass@64": 0.9908886006022682, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8911028287461774, + "pass@2": 0.9203143435912254, + "pass@4": 0.9371241599387236, + "pass@8": 0.9490061787758015, + "pass@16": 0.9580216692464635, + "pass@32": 0.9647441880294301, + "pass@64": 0.97045499920306, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.6849199459876543, + "pass@2": 0.7509812141537865, + "pass@4": 0.7945692465525145, + "pass@8": 0.8258733487858642, + "pass@16": 0.849861770119659, + "pass@32": 0.8699388346859993, + "pass@64": 0.88854095098482, + "pass@128": 0.9074074074074074 + }, + "14": { + "pass@1": 0.5660005563798219, + "pass@2": 0.6501940757961634, + "pass@4": 0.7085746448793305, + "pass@8": 0.7505306462496025, + "pass@16": 0.7849407654601296, + "pass@32": 0.815334178686988, + "pass@64": 0.845313034418024, + "pass@128": 0.8783382789317508 + }, + "15": { + "pass@1": 0.3757716049382716, + "pass@2": 0.4573331328375619, + "pass@4": 0.5247166560777126, + "pass@8": 0.5854703680844102, + "pass@16": 0.641237050233849, + "pass@32": 0.6870811470164, + "pass@64": 0.7229905824965706, + "pass@128": 0.7530864197530864 + }, + "16": { + "pass@1": 0.21808167404129794, + "pass@2": 0.2838761236150791, + "pass@4": 0.3488833027508728, + "pass@8": 0.4087377773155697, + "pass@16": 0.462083287505114, + "pass@32": 0.5103820053527391, + "pass@64": 0.5550487660346097, + "pass@128": 0.5929203539823009 + }, + "17": { + "pass@1": 0.20397213855421686, + "pass@2": 0.2691332505217722, + "pass@4": 0.3333696811994885, + "pass@8": 0.39462440769076684, + "pass@16": 0.451429171320933, + "pass@32": 0.5006200241327831, + "pass@64": 0.5402298248806521, + "pass@128": 0.5753012048192772 + }, + "18": { + "pass@1": 0.20153729838709677, + "pass@2": 0.27094908877317764, + "pass@4": 0.33812365833303115, + "pass@8": 0.3940654800221941, + "pass@16": 0.4353412957083234, + "pass@32": 0.4664177817745076, + "pass@64": 0.49265964787019434, + "pass@128": 0.5161290322580645 + }, + "19": { + "pass@1": 0.1778700906344411, + "pass@2": 0.24411939184527925, + "pass@4": 0.30913897450885125, + "pass@8": 0.3674449887396694, + "pass@16": 0.41575365102341894, + "pass@32": 0.4509801282681683, + "pass@64": 0.47386576276271636, + "pass@128": 0.48942598187311176 + }, + "20": { + "pass@1": 0.1649070945945946, + "pass@2": 0.228963092029037, + "pass@4": 0.2970748003346429, + "pass@8": 0.3610769767539805, + "pass@16": 0.41552077639760837, + "pass@32": 0.4596679149017448, + "pass@64": 0.4991020353061597, + "pass@128": 0.5405405405405406 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch4-RL-op9-12/composition/hf_model_metrics.json b/composition/Qwen3-100M-epoch4-RL-op9-12/composition/hf_model_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..c9c1704cd36ff88a1061f77460c9192ea4490810 --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op9-12/composition/hf_model_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "hf_model", + "id": { + "answer_accuracy": 0.9964331597222222, + "avg_response_len": 200.05666840277777, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9964331597222222, + "pass@2": 0.9976662975721784, + "pass@4": 0.9986044053659957, + "pass@8": 0.9992345061249156, + "pass@16": 0.9995909235695006, + "pass@32": 0.9997975132668376, + "pass@64": 0.9999273035339215, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.974828125, + "pass@2": 0.982788139763779, + "pass@4": 0.9893376028308961, + "pass@8": 0.9940825912459509, + "pass@16": 0.9968557654879298, + "pass@32": 0.9984090878440416, + "pass@64": 0.9993745507070829, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999453125, + "pass@2": 0.9999974163385827, + "pass@4": 0.9999999967191601, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9991171875, + "pass@2": 0.999505659448819, + "pass@4": 0.9997710318710161, + "pass@8": 0.9999506014374756, + "pass@16": 0.9999981090802741, + "pass@32": 0.999999998952485, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.999796875, + "pass@2": 0.9999715797244095, + "pass@4": 0.9999993143044619, + "pass@8": 0.9999999997763381, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.996578125, + "pass@2": 0.9979938484251969, + "pass@4": 0.9988948359580053, + "pass@8": 0.9992646525162459, + "pass@16": 0.9994928828837889, + "pass@32": 0.9997689651292465, + "pass@64": 0.9999711811023622, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.997640625, + "pass@2": 0.998740034448819, + "pass@4": 0.9994368666104239, + "pass@8": 0.9998127101482287, + "pass@16": 0.9999715546735147, + "pass@32": 0.9999995674757656, + "pass@64": 0.9999999999958465, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.974828125, + "2": 1.0, + "3": 0.9999921875, + "4": 0.9999453125, + "5": 0.9991171875, + "6": 1.0, + "7": 0.999796875, + "8": 0.996578125, + "9": 0.997640625 + }, + "per_op_avg_response_len": { + "10": 281.6217734375, + "2": 115.6109375, + "3": 144.6905625, + "4": 159.9184609375, + "5": 172.7354765625, + "6": 198.87459375, + "7": 223.9909296875, + "8": 243.5658828125, + "9": 259.5013984375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.6217734375, + "2": 115.6109375, + "3": 144.6905625, + "4": 159.9184609375, + "5": 172.7354765625, + "6": 198.87459375, + "7": 223.9909296875, + "8": 243.5658828125, + "9": 259.5013984375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 377828, + "answer_accuracy": 0.9968866092536305, + "avg_response_len": 190.5977894925701, + "resp_tokens_sum": 72238087, + "pass_at_k": { + "pass@1": 0.9968866092536305, + "pass@2": 0.9980258036229515, + "pass@4": 0.9989005383002403, + "pass@8": 0.9994903936696694, + "pass@16": 0.9997709622000206, + "pass@32": 0.9998943315580754, + "pass@64": 0.9999798854866389, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9783370206489675, + "pass@2": 0.9862172061180408, + "pass@4": 0.9923453603144738, + "pass@8": 0.9962260019031219, + "pass@16": 0.9980889112757674, + "pass@32": 0.9990783165640255, + "pass@64": 0.9998243095282938, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998291015625, + "pass@2": 0.999991926058071, + "pass@4": 0.9999999897473753, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9977926587301588, + "pass@2": 0.998470503687039, + "pass@4": 0.9992734164181859, + "pass@8": 0.999843179173257, + "pass@16": 0.9999939970802351, + "pass@32": 0.9999999966745557, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9994683801775148, + "pass@2": 0.9999159163444068, + "pass@4": 0.9999979713149763, + "pass@8": 0.9999999993382783, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9997495993589743, + "pass@2": 0.9999964510145367, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9975792253521126, + "pass@2": 0.9981427442608406, + "pass@4": 0.9987858579297307, + "pass@8": 0.9994925064642773, + "pass@16": 0.9999199128774705, + "pass@32": 0.9999987816218918, + "pass@64": 0.9999999999883001, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 389404, + "answer_accuracy": 0.9967951343381389, + "avg_response_len": 198.0918250327654, + "resp_tokens_sum": 77385760, + "pass_at_k": { + "pass@1": 0.9967951343381389, + "pass@2": 0.9980740652831239, + "pass@4": 0.9988689264534103, + "pass@8": 0.9992935799391331, + "pass@16": 0.9995851478046265, + "pass@32": 0.9998371455803265, + "pass@64": 0.9999784564928254, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9760352760736196, + "pass@2": 0.9843927376696779, + "pass@4": 0.9899295643067624, + "pass@8": 0.9934119367875759, + "pass@16": 0.9961162150859647, + "pass@32": 0.9984753629176755, + "pass@64": 0.9997983104788434, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9997961956521739, + "pass@2": 0.9999960772566472, + "pass@4": 0.999999998641474, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999774205202312, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9971390845070423, + "pass@2": 0.9987000249528667, + "pass@4": 0.9997053884989727, + "pass@8": 0.9999854279093834, + "pass@16": 0.9999999724377853, + "pass@32": 0.9999999999999866, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9973802395209581, + "pass@2": 0.9990205366825403, + "pass@4": 0.9998069255190407, + "pass@8": 0.9999906816576446, + "pass@16": 0.9999999843301874, + "pass@32": 0.9999999999999962, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380659, + "answer_accuracy": 0.9956138056578507, + "avg_response_len": 211.44081383913627, + "resp_tokens_sum": 80841435, + "pass_at_k": { + "pass@1": 0.9956138056578507, + "pass@2": 0.9968932796712262, + "pass@4": 0.9980405727656994, + "pass@8": 0.9989204866066669, + "pass@16": 0.9994183535157435, + "pass@32": 0.999661042968504, + "pass@64": 0.999822913178189, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9701026119402985, + "pass@2": 0.9777566400282055, + "pass@4": 0.9857178737918953, + "pass@8": 0.9925662245016207, + "pass@16": 0.9963275774549865, + "pass@32": 0.9976673737841041, + "pass@64": 0.998507046450413, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996553308823529, + "pass@2": 0.999967070981936, + "pass@4": 0.9999997240234677, + "pass@8": 0.9999999999938223, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999505537974683, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9930086336336337, + "pass@2": 0.9953647052564376, + "pass@4": 0.9969952643869967, + "pass@8": 0.9978072841093536, + "pass@16": 0.9984771551602851, + "pass@32": 0.9993062015893434, + "pass@64": 0.9999134567638505, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9979903536977492, + "pass@2": 0.9991205819429324, + "pass@4": 0.9997825528038898, + "pass@8": 0.9999870825715659, + "pass@16": 0.9999999705328944, + "pass@32": 0.9999999999999847, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.5711796875, + "avg_response_len": 353.29377890625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5711796875, + "pass@2": 0.6364642716535321, + "pass@4": 0.6911855868297601, + "pass@8": 0.7351047646934672, + "pass@16": 0.7696548619807136, + "pass@32": 0.7974647590406176, + "pass@64": 0.8205509479586844, + "pass@128": 0.8402 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9701796875, + "pass@2": 0.9771525590551182, + "pass@4": 0.9816584605361829, + "pass@8": 0.9846877446479687, + "pass@16": 0.9870587339705751, + "pass@32": 0.9890986156384529, + "pass@64": 0.9908720537724355, + "pass@128": 0.993 + }, + "12": { + "pass@1": 0.9171484375, + "pass@2": 0.9324242125984255, + "pass@4": 0.9421808832020993, + "pass@8": 0.9492329604508329, + "pass@16": 0.9544579834486829, + "pass@32": 0.9586347150611692, + "pass@64": 0.9625264601384094, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.8500390625, + "pass@2": 0.8903901328740159, + "pass@4": 0.9159092870266217, + "pass@8": 0.9322993304826687, + "pass@16": 0.9432918888207025, + "pass@32": 0.951584080383266, + "pass@64": 0.9582833961461636, + "pass@128": 0.964 + }, + "14": { + "pass@1": 0.80325, + "pass@2": 0.8586851624015741, + "pass@4": 0.892811513967004, + "pass@8": 0.9152137728434167, + "pass@16": 0.9307627635196196, + "pass@32": 0.9419410563109959, + "pass@64": 0.9505134140911546, + "pass@128": 0.957 + }, + "15": { + "pass@1": 0.695640625, + "pass@2": 0.776031373031497, + "pass@4": 0.82871843991376, + "pass@8": 0.8629510737508274, + "pass@16": 0.8864226575431764, + "pass@32": 0.9039585573242785, + "pass@64": 0.9181303650391566, + "pass@128": 0.931 + }, + "16": { + "pass@1": 0.455140625, + "pass@2": 0.5644215059055124, + "pass@4": 0.6546008473940751, + "pass@8": 0.723307014400029, + "pass@16": 0.7736868909222655, + "pass@32": 0.8129277513357028, + "pass@64": 0.8442980719861337, + "pass@128": 0.868 + }, + "17": { + "pass@1": 0.355390625, + "pass@2": 0.46065944881889753, + "pass@4": 0.5534846547619047, + "pass@8": 0.6275692447117021, + "pass@16": 0.684206072882574, + "pass@32": 0.72916822255401, + "pass@64": 0.7670412575669749, + "pass@128": 0.799 + }, + "18": { + "pass@1": 0.2858828125, + "pass@2": 0.378095349409449, + "pass@4": 0.46274084608174015, + "pass@8": 0.5345160881384796, + "pass@16": 0.5937176533496914, + "pass@32": 0.6421777947174774, + "pass@64": 0.6812458742737494, + "pass@128": 0.712 + }, + "19": { + "pass@1": 0.19996875, + "pass@2": 0.27603789370078713, + "pass@4": 0.3533704972815893, + "pass@8": 0.4253751028295304, + "pass@16": 0.4891721250058378, + "pass@32": 0.5438867640387719, + "pass@64": 0.5898362071219185, + "pass@128": 0.631 + }, + "20": { + "pass@1": 0.17915625, + "pass@2": 0.2507450787401575, + "pass@4": 0.32638043813273326, + "pass@8": 0.3958953146792503, + "pass@16": 0.4537718503438972, + "pass@32": 0.5012700330420176, + "pass@64": 0.5427623794506787, + "pass@128": 0.58 + } + }, + "per_op_accuracy": { + "11": 0.9701796875, + "12": 0.9171484375, + "13": 0.8500390625, + "14": 0.80325, + "15": 0.695640625, + "16": 0.455140625, + "17": 0.355390625, + "18": 0.2858828125, + "19": 0.19996875, + "20": 0.17915625 + }, + "per_op_avg_response_len": { + "11": 299.4068671875, + "12": 309.453609375, + "13": 328.448828125, + "14": 354.0121875, + "15": 367.3119609375, + "16": 363.401328125, + "17": 376.1721875, + "18": 377.4971875, + "19": 374.2378984375, + "20": 382.995734375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 299.4068671875, + "12": 309.453609375, + "13": 328.448828125, + "14": 354.0121875, + "15": 367.3119609375, + "16": 363.401328125, + "17": 376.1721875, + "18": 377.4971875, + "19": 374.2378984375, + "20": 382.995734375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 249521, + "answer_accuracy": 0.5793113855869242, + "avg_response_len": 336.56192421991085, + "resp_tokens_sum": 144963952, + "pass_at_k": { + "pass@1": 0.5793113855869242, + "pass@2": 0.6440018398053173, + "pass@4": 0.6975395708987366, + "pass@8": 0.7403502993594158, + "pass@16": 0.774355982347351, + "pass@32": 0.8018187652267494, + "pass@64": 0.8244726877215482, + "pass@128": 0.8439821693907875 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.968683604815864, + "pass@2": 0.9789929819767574, + "pass@4": 0.9860749473419221, + "pass@8": 0.9897070568664353, + "pass@16": 0.9917076454481355, + "pass@32": 0.9939863300081206, + "pass@64": 0.9968296937386972, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9008430437853108, + "pass@2": 0.9165818652964992, + "pass@4": 0.927459671884235, + "pass@8": 0.9360445467076817, + "pass@16": 0.9432223808868564, + "pass@32": 0.9496503636415805, + "pass@64": 0.9559704286203133, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8701998432601881, + "pass@2": 0.9126545800113549, + "pass@4": 0.9398911460518841, + "pass@8": 0.9570679389519005, + "pass@16": 0.9671074301439438, + "pass@32": 0.9728464279999752, + "pass@64": 0.9763005928045156, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8175379672897196, + "pass@2": 0.8708705877302721, + "pass@4": 0.8993175306941774, + "pass@8": 0.9171035195588915, + "pass@16": 0.9308228683447412, + "pass@32": 0.9421168792214055, + "pass@64": 0.9521133810218851, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.7054281069364162, + "pass@2": 0.7905655466296483, + "pass@4": 0.8474447166286287, + "pass@8": 0.8839813831382466, + "pass@16": 0.9076477470834142, + "pass@32": 0.9237123268955159, + "pass@64": 0.935420710689402, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.4908266129032258, + "pass@2": 0.5978314706629414, + "pass@4": 0.6797848857602473, + "pass@8": 0.7399091549637592, + "pass@16": 0.7856011678337806, + "pass@32": 0.8230687707717842, + "pass@64": 0.8559382046199904, + "pass@128": 0.8838709677419355 + }, + "17": { + "pass@1": 0.37229046242774566, + "pass@2": 0.47453781405033935, + "pass@4": 0.5644480215803944, + "pass@8": 0.6348009526237541, + "pass@16": 0.6883834382986443, + "pass@32": 0.7330710395952293, + "pass@64": 0.7728938279339617, + "pass@128": 0.8063583815028902 + }, + "18": { + "pass@1": 0.26310267857142855, + "pass@2": 0.34792955568053996, + "pass@4": 0.4297920528683913, + "pass@8": 0.5048888152803351, + "pass@16": 0.5699261319707205, + "pass@32": 0.6207125552742997, + "pass@64": 0.6580750858985634, + "pass@128": 0.6885714285714286 + }, + "19": { + "pass@1": 0.19761167417417416, + "pass@2": 0.27271021021021014, + "pass@4": 0.3471924486353619, + "pass@8": 0.41691812297090086, + "pass@16": 0.48171163739621536, + "pass@32": 0.5404352797339113, + "pass@64": 0.5899459963388314, + "pass@128": 0.6336336336336337 + }, + "20": { + "pass@1": 0.19693130630630631, + "pass@2": 0.2723244898441749, + "pass@4": 0.3514077086084959, + "pass@8": 0.42174819195840463, + "pass@16": 0.4768338569181501, + "pass@32": 0.5186081284129838, + "pass@64": 0.5519277042276182, + "pass@128": 0.5795795795795796 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 246996, + "answer_accuracy": 0.5737901427297056, + "avg_response_len": 348.280924769551, + "resp_tokens_sum": 149922400, + "pass_at_k": { + "pass@1": 0.5737901427297056, + "pass@2": 0.6394622554149983, + "pass@4": 0.6945944505877448, + "pass@8": 0.7388007257179604, + "pass@16": 0.7735097498008077, + "pass@32": 0.8016001444066486, + "pass@64": 0.824782798417057, + "pass@128": 0.8432946773713946 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9709855045180723, + "pass@2": 0.9759058379186036, + "pass@4": 0.9788218743929598, + "pass@8": 0.9812271434030994, + "pass@16": 0.9835566388472978, + "pass@32": 0.9853343040816687, + "pass@64": 0.9864394200635662, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.9212137539184952, + "pass@2": 0.9338549508799643, + "pass@4": 0.9409774663746967, + "pass@8": 0.9461605137565879, + "pass@16": 0.9500469032321945, + "pass@32": 0.9528488646686972, + "pass@64": 0.9544922426331286, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.8534445028011205, + "pass@2": 0.8905946729085336, + "pass@4": 0.9134338682979749, + "pass@8": 0.9279350316050726, + "pass@16": 0.937872928293743, + "pass@32": 0.9456857085216124, + "pass@64": 0.9521581923333696, + "pass@128": 0.957983193277311 + }, + "14": { + "pass@1": 0.8009868421052632, + "pass@2": 0.860120383800709, + "pass@4": 0.8988449900451034, + "pass@8": 0.924327747686637, + "pass@16": 0.9411481778605455, + "pass@32": 0.9520906949142818, + "pass@64": 0.9598763018066186, + "pass@128": 0.9649122807017544 + }, + "15": { + "pass@1": 0.7052083333333333, + "pass@2": 0.7861254026485326, + "pass@4": 0.8397210235084246, + "pass@8": 0.8743415993819952, + "pass@16": 0.8982540136113785, + "pass@32": 0.9157051499837666, + "pass@64": 0.9282322085493483, + "pass@128": 0.9393939393939394 + }, + "16": { + "pass@1": 0.4555956196581197, + "pass@2": 0.5646556800592232, + "pass@4": 0.6547937362530536, + "pass@8": 0.7227923664185703, + "pass@16": 0.770888499121302, + "pass@32": 0.8079454921479431, + "pass@64": 0.836781426539472, + "pass@128": 0.8547008547008547 + }, + "17": { + "pass@1": 0.3923718944099379, + "pass@2": 0.5017893058883942, + "pass@4": 0.5944229474227205, + "pass@8": 0.6676706766755468, + "pass@16": 0.72361349680337, + "pass@32": 0.7682336081279583, + "pass@64": 0.8049794395003316, + "pass@128": 0.8322981366459627 + }, + "18": { + "pass@1": 0.2898207720588235, + "pass@2": 0.3896733151922183, + "pass@4": 0.480811416495732, + "pass@8": 0.5562591169523126, + "pass@16": 0.6177651169032992, + "pass@32": 0.6700276603939938, + "pass@64": 0.7120274324237998, + "pass@128": 0.7352941176470589 + }, + "19": { + "pass@1": 0.1942196800595238, + "pass@2": 0.27048657492032246, + "pass@4": 0.3513666932816434, + "pass@8": 0.42792231718413776, + "pass@16": 0.4948232856154525, + "pass@32": 0.5520586849245674, + "pass@64": 0.6022906289407459, + "pass@128": 0.6488095238095238 + }, + "20": { + "pass@1": 0.15578218562874252, + "pass@2": 0.2213878100099014, + "pass@4": 0.29161606848170923, + "pass@8": 0.35769019300391275, + "pass@16": 0.41528207411562973, + "pass@32": 0.4641400644546501, + "pass@64": 0.5086113800320359, + "pass@128": 0.5538922155688623 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 234593, + "answer_accuracy": 0.5601338057762836, + "avg_response_len": 375.6534731242359, + "resp_tokens_sum": 157329685, + "pass_at_k": { + "pass@1": 0.5601338057762836, + "pass@2": 0.6256311004370192, + "pass@4": 0.6811473334036958, + "pass@8": 0.7259113841689498, + "pass@16": 0.7608580228081064, + "pass@32": 0.7887366013993125, + "pass@64": 0.8121681950877634, + "pass@128": 0.8331295843520783 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9710069444444445, + "pass@2": 0.9764041213598301, + "pass@4": 0.9796988629397515, + "pass@8": 0.9827102919501212, + "pass@16": 0.9855400985081924, + "pass@32": 0.9875887371126105, + "pass@64": 0.9888675695922908, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9308342889908257, + "pass@2": 0.9481789081124038, + "pass@4": 0.9592915828182028, + "pass@8": 0.9665076055900975, + "pass@16": 0.9709244601946977, + "pass@32": 0.974005194197968, + "pass@64": 0.9774614770851698, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.8264371141975309, + "pass@2": 0.8682439000680472, + "pass@4": 0.8950250631286828, + "pass@8": 0.9127217644259317, + "pass@16": 0.9258147629750567, + "pass@32": 0.937148987373638, + "pass@64": 0.9472933098102161, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7919371290801187, + "pass@2": 0.8456217581018252, + "pass@4": 0.8804913947143843, + "pass@8": 0.9041645500183436, + "pass@16": 0.9201660117289387, + "pass@32": 0.9314733543627315, + "pass@64": 0.9394876070185926, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6754436728395061, + "pass@2": 0.7502293550111786, + "pass@4": 0.7975142907483781, + "pass@8": 0.8288913191016511, + "pass@16": 0.8517059029646968, + "pass@32": 0.8708993602586046, + "pass@64": 0.8893770688868474, + "pass@128": 0.9074074074074074 + }, + "16": { + "pass@1": 0.42203632005899705, + "pass@2": 0.5336271572248159, + "pass@4": 0.6313714790076016, + "pass@8": 0.7086579815585403, + "pass@16": 0.7656892793575719, + "pass@32": 0.8088128750811807, + "pass@64": 0.8414364243025991, + "pass@128": 0.8672566371681416 + }, + "17": { + "pass@1": 0.30191076807228917, + "pass@2": 0.40630484536571476, + "pass@4": 0.5023537657378669, + "pass@8": 0.5811390277540882, + "pass@16": 0.6416321001824091, + "pass@32": 0.6872121115568036, + "pass@64": 0.7241464264539637, + "pass@128": 0.7590361445783133 + }, + "18": { + "pass@1": 0.3072832661290323, + "pass@2": 0.39945508953517883, + "pass@4": 0.4801217611911414, + "pass@8": 0.5441190420212122, + "pass@16": 0.5942044110090919, + "pass@32": 0.6358677285081136, + "pass@64": 0.6736460231779342, + "pass@128": 0.7129032258064516 + }, + "19": { + "pass@1": 0.2081759818731118, + "pass@2": 0.2850208595998762, + "pass@4": 0.3616199484694322, + "pass@8": 0.4312974903515109, + "pass@16": 0.49094116551693745, + "pass@32": 0.5390637394342128, + "pass@64": 0.5770831994471198, + "pass@128": 0.6102719033232629 + }, + "20": { + "pass@1": 0.18482545045045046, + "pass@2": 0.258611096332553, + "pass@4": 0.3362219348144546, + "pass@8": 0.4083622891705557, + "pass@16": 0.46931520491151213, + "pass@32": 0.5211734076655891, + "pass@64": 0.5678506096461912, + "pass@128": 0.6066066066066066 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7726155427631579, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7726155427631579, + "pass@2": 0.8075599681413029, + "pass@4": 0.8368050271890203, + "pass@8": 0.8602188527399527, + "pass@16": 0.8785719437859261, + "pass@32": 0.8933065899898898, + "pass@64": 0.905518695336424, + "pass@128": 0.9158947368421053 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.974828125, + "pass@2": 0.982788139763779, + "pass@4": 0.9893376028308961, + "pass@8": 0.9940825912459509, + "pass@16": 0.9968557654879298, + "pass@32": 0.9984090878440416, + "pass@64": 0.9993745507070829, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999453125, + "pass@2": 0.9999974163385827, + "pass@4": 0.9999999967191601, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9991171875, + "pass@2": 0.999505659448819, + "pass@4": 0.9997710318710161, + "pass@8": 0.9999506014374756, + "pass@16": 0.9999981090802741, + "pass@32": 0.999999998952485, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.999796875, + "pass@2": 0.9999715797244095, + "pass@4": 0.9999993143044619, + "pass@8": 0.9999999997763381, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.996578125, + "pass@2": 0.9979938484251969, + "pass@4": 0.9988948359580053, + "pass@8": 0.9992646525162459, + "pass@16": 0.9994928828837889, + "pass@32": 0.9997689651292465, + "pass@64": 0.9999711811023622, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.997640625, + "pass@2": 0.998740034448819, + "pass@4": 0.9994368666104239, + "pass@8": 0.9998127101482287, + "pass@16": 0.9999715546735147, + "pass@32": 0.9999995674757656, + "pass@64": 0.9999999999958465, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9701796875, + "pass@2": 0.9771525590551182, + "pass@4": 0.9816584605361829, + "pass@8": 0.9846877446479687, + "pass@16": 0.9870587339705751, + "pass@32": 0.9890986156384529, + "pass@64": 0.9908720537724355, + "pass@128": 0.993 + }, + "12": { + "pass@1": 0.9171484375, + "pass@2": 0.9324242125984255, + "pass@4": 0.9421808832020993, + "pass@8": 0.9492329604508329, + "pass@16": 0.9544579834486829, + "pass@32": 0.9586347150611692, + "pass@64": 0.9625264601384094, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.8500390625, + "pass@2": 0.8903901328740159, + "pass@4": 0.9159092870266217, + "pass@8": 0.9322993304826687, + "pass@16": 0.9432918888207025, + "pass@32": 0.951584080383266, + "pass@64": 0.9582833961461636, + "pass@128": 0.964 + }, + "14": { + "pass@1": 0.80325, + "pass@2": 0.8586851624015741, + "pass@4": 0.892811513967004, + "pass@8": 0.9152137728434167, + "pass@16": 0.9307627635196196, + "pass@32": 0.9419410563109959, + "pass@64": 0.9505134140911546, + "pass@128": 0.957 + }, + "15": { + "pass@1": 0.695640625, + "pass@2": 0.776031373031497, + "pass@4": 0.82871843991376, + "pass@8": 0.8629510737508274, + "pass@16": 0.8864226575431764, + "pass@32": 0.9039585573242785, + "pass@64": 0.9181303650391566, + "pass@128": 0.931 + }, + "16": { + "pass@1": 0.455140625, + "pass@2": 0.5644215059055124, + "pass@4": 0.6546008473940751, + "pass@8": 0.723307014400029, + "pass@16": 0.7736868909222655, + "pass@32": 0.8129277513357028, + "pass@64": 0.8442980719861337, + "pass@128": 0.868 + }, + "17": { + "pass@1": 0.355390625, + "pass@2": 0.46065944881889753, + "pass@4": 0.5534846547619047, + "pass@8": 0.6275692447117021, + "pass@16": 0.684206072882574, + "pass@32": 0.72916822255401, + "pass@64": 0.7670412575669749, + "pass@128": 0.799 + }, + "18": { + "pass@1": 0.2858828125, + "pass@2": 0.378095349409449, + "pass@4": 0.46274084608174015, + "pass@8": 0.5345160881384796, + "pass@16": 0.5937176533496914, + "pass@32": 0.6421777947174774, + "pass@64": 0.6812458742737494, + "pass@128": 0.712 + }, + "19": { + "pass@1": 0.19996875, + "pass@2": 0.27603789370078713, + "pass@4": 0.3533704972815893, + "pass@8": 0.4253751028295304, + "pass@16": 0.4891721250058378, + "pass@32": 0.5438867640387719, + "pass@64": 0.5898362071219185, + "pass@128": 0.631 + }, + "20": { + "pass@1": 0.17915625, + "pass@2": 0.2507450787401575, + "pass@4": 0.32638043813273326, + "pass@8": 0.3958953146792503, + "pass@16": 0.4537718503438972, + "pass@32": 0.5012700330420176, + "pass@64": 0.5427623794506787, + "pass@128": 0.58 + } + }, + "per_op_accuracy": { + "10": 0.974828125, + "2": 1.0, + "3": 0.9999921875, + "4": 0.9999453125, + "5": 0.9991171875, + "6": 1.0, + "7": 0.999796875, + "8": 0.996578125, + "9": 0.997640625, + "11": 0.9701796875, + "12": 0.9171484375, + "13": 0.8500390625, + "14": 0.80325, + "15": 0.695640625, + "16": 0.455140625, + "17": 0.355390625, + "18": 0.2858828125, + "19": 0.19996875, + "20": 0.17915625 + }, + "per_op_avg_response_len": { + "10": 281.6217734375, + "2": 115.6109375, + "3": 144.6905625, + "4": 159.9184609375, + "5": 172.7354765625, + "6": 198.87459375, + "7": 223.9909296875, + "8": 243.5658828125, + "9": 259.5013984375, + "11": 299.4068671875, + "12": 309.453609375, + "13": 328.448828125, + "14": 354.0121875, + "15": 367.3119609375, + "16": 363.401328125, + "17": 376.1721875, + "18": 377.4971875, + "19": 374.2378984375, + "20": 382.995734375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.6217734375, + "2": 115.6109375, + "3": 144.6905625, + "4": 159.9184609375, + "5": 172.7354765625, + "6": 198.87459375, + "7": 223.9909296875, + "8": 243.5658828125, + "9": 259.5013984375, + "11": 299.4068671875, + "12": 309.453609375, + "13": 328.448828125, + "14": 354.0121875, + "15": 367.3119609375, + "16": 363.401328125, + "17": 376.1721875, + "18": 377.4971875, + "19": 374.2378984375, + "20": 382.995734375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 627349, + "answer_accuracy": 0.7747651063073032, + "avg_response_len": 268.240741335362, + "resp_tokens_sum": 217202039, + "pass_at_k": { + "pass@1": 0.7747651063073032, + "pass@2": 0.8097092310263018, + "pass@4": 0.8385970834620956, + "pass@8": 0.8616455600696019, + "pass@16": 0.8798655864168684, + "pass@32": 0.894531656770708, + "pass@64": 0.9066220415916817, + "pass@128": 0.9170091685109074 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9783370206489675, + "pass@2": 0.9862172061180408, + "pass@4": 0.9923453603144738, + "pass@8": 0.9962260019031219, + "pass@16": 0.9980889112757674, + "pass@32": 0.9990783165640255, + "pass@64": 0.9998243095282938, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998291015625, + "pass@2": 0.999991926058071, + "pass@4": 0.9999999897473753, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9977926587301588, + "pass@2": 0.998470503687039, + "pass@4": 0.9992734164181859, + "pass@8": 0.999843179173257, + "pass@16": 0.9999939970802351, + "pass@32": 0.9999999966745557, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9994683801775148, + "pass@2": 0.9999159163444068, + "pass@4": 0.9999979713149763, + "pass@8": 0.9999999993382783, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9997495993589743, + "pass@2": 0.9999964510145367, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9975792253521126, + "pass@2": 0.9981427442608406, + "pass@4": 0.9987858579297307, + "pass@8": 0.9994925064642773, + "pass@16": 0.9999199128774705, + "pass@32": 0.9999987816218918, + "pass@64": 0.9999999999883001, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.968683604815864, + "pass@2": 0.9789929819767574, + "pass@4": 0.9860749473419221, + "pass@8": 0.9897070568664353, + "pass@16": 0.9917076454481355, + "pass@32": 0.9939863300081206, + "pass@64": 0.9968296937386972, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9008430437853108, + "pass@2": 0.9165818652964992, + "pass@4": 0.927459671884235, + "pass@8": 0.9360445467076817, + "pass@16": 0.9432223808868564, + "pass@32": 0.9496503636415805, + "pass@64": 0.9559704286203133, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8701998432601881, + "pass@2": 0.9126545800113549, + "pass@4": 0.9398911460518841, + "pass@8": 0.9570679389519005, + "pass@16": 0.9671074301439438, + "pass@32": 0.9728464279999752, + "pass@64": 0.9763005928045156, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8175379672897196, + "pass@2": 0.8708705877302721, + "pass@4": 0.8993175306941774, + "pass@8": 0.9171035195588915, + "pass@16": 0.9308228683447412, + "pass@32": 0.9421168792214055, + "pass@64": 0.9521133810218851, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.7054281069364162, + "pass@2": 0.7905655466296483, + "pass@4": 0.8474447166286287, + "pass@8": 0.8839813831382466, + "pass@16": 0.9076477470834142, + "pass@32": 0.9237123268955159, + "pass@64": 0.935420710689402, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.4908266129032258, + "pass@2": 0.5978314706629414, + "pass@4": 0.6797848857602473, + "pass@8": 0.7399091549637592, + "pass@16": 0.7856011678337806, + "pass@32": 0.8230687707717842, + "pass@64": 0.8559382046199904, + "pass@128": 0.8838709677419355 + }, + "17": { + "pass@1": 0.37229046242774566, + "pass@2": 0.47453781405033935, + "pass@4": 0.5644480215803944, + "pass@8": 0.6348009526237541, + "pass@16": 0.6883834382986443, + "pass@32": 0.7330710395952293, + "pass@64": 0.7728938279339617, + "pass@128": 0.8063583815028902 + }, + "18": { + "pass@1": 0.26310267857142855, + "pass@2": 0.34792955568053996, + "pass@4": 0.4297920528683913, + "pass@8": 0.5048888152803351, + "pass@16": 0.5699261319707205, + "pass@32": 0.6207125552742997, + "pass@64": 0.6580750858985634, + "pass@128": 0.6885714285714286 + }, + "19": { + "pass@1": 0.19761167417417416, + "pass@2": 0.27271021021021014, + "pass@4": 0.3471924486353619, + "pass@8": 0.41691812297090086, + "pass@16": 0.48171163739621536, + "pass@32": 0.5404352797339113, + "pass@64": 0.5899459963388314, + "pass@128": 0.6336336336336337 + }, + "20": { + "pass@1": 0.19693130630630631, + "pass@2": 0.2723244898441749, + "pass@4": 0.3514077086084959, + "pass@8": 0.42174819195840463, + "pass@16": 0.4768338569181501, + "pass@32": 0.5186081284129838, + "pass@64": 0.5519277042276182, + "pass@128": 0.5795795795795796 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 636400, + "answer_accuracy": 0.7750389711613406, + "avg_response_len": 276.8269680436477, + "resp_tokens_sum": 227308160, + "pass_at_k": { + "pass@1": 0.7750389711613406, + "pass@2": 0.8100753877170177, + "pass@4": 0.8393560562528998, + "pass@8": 0.8627327897994856, + "pass@16": 0.8810673670584344, + "pass@32": 0.8959133677241993, + "pass@64": 0.9081338737790662, + "pass@128": 0.9178487918939985 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9760352760736196, + "pass@2": 0.9843927376696779, + "pass@4": 0.9899295643067624, + "pass@8": 0.9934119367875759, + "pass@16": 0.9961162150859647, + "pass@32": 0.9984753629176755, + "pass@64": 0.9997983104788434, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9997961956521739, + "pass@2": 0.9999960772566472, + "pass@4": 0.999999998641474, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999774205202312, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9971390845070423, + "pass@2": 0.9987000249528667, + "pass@4": 0.9997053884989727, + "pass@8": 0.9999854279093834, + "pass@16": 0.9999999724377853, + "pass@32": 0.9999999999999866, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9973802395209581, + "pass@2": 0.9990205366825403, + "pass@4": 0.9998069255190407, + "pass@8": 0.9999906816576446, + "pass@16": 0.9999999843301874, + "pass@32": 0.9999999999999962, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9709855045180723, + "pass@2": 0.9759058379186036, + "pass@4": 0.9788218743929598, + "pass@8": 0.9812271434030994, + "pass@16": 0.9835566388472978, + "pass@32": 0.9853343040816687, + "pass@64": 0.9864394200635662, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.9212137539184952, + "pass@2": 0.9338549508799643, + "pass@4": 0.9409774663746967, + "pass@8": 0.9461605137565879, + "pass@16": 0.9500469032321945, + "pass@32": 0.9528488646686972, + "pass@64": 0.9544922426331286, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.8534445028011205, + "pass@2": 0.8905946729085336, + "pass@4": 0.9134338682979749, + "pass@8": 0.9279350316050726, + "pass@16": 0.937872928293743, + "pass@32": 0.9456857085216124, + "pass@64": 0.9521581923333696, + "pass@128": 0.957983193277311 + }, + "14": { + "pass@1": 0.8009868421052632, + "pass@2": 0.860120383800709, + "pass@4": 0.8988449900451034, + "pass@8": 0.924327747686637, + "pass@16": 0.9411481778605455, + "pass@32": 0.9520906949142818, + "pass@64": 0.9598763018066186, + "pass@128": 0.9649122807017544 + }, + "15": { + "pass@1": 0.7052083333333333, + "pass@2": 0.7861254026485326, + "pass@4": 0.8397210235084246, + "pass@8": 0.8743415993819952, + "pass@16": 0.8982540136113785, + "pass@32": 0.9157051499837666, + "pass@64": 0.9282322085493483, + "pass@128": 0.9393939393939394 + }, + "16": { + "pass@1": 0.4555956196581197, + "pass@2": 0.5646556800592232, + "pass@4": 0.6547937362530536, + "pass@8": 0.7227923664185703, + "pass@16": 0.770888499121302, + "pass@32": 0.8079454921479431, + "pass@64": 0.836781426539472, + "pass@128": 0.8547008547008547 + }, + "17": { + "pass@1": 0.3923718944099379, + "pass@2": 0.5017893058883942, + "pass@4": 0.5944229474227205, + "pass@8": 0.6676706766755468, + "pass@16": 0.72361349680337, + "pass@32": 0.7682336081279583, + "pass@64": 0.8049794395003316, + "pass@128": 0.8322981366459627 + }, + "18": { + "pass@1": 0.2898207720588235, + "pass@2": 0.3896733151922183, + "pass@4": 0.480811416495732, + "pass@8": 0.5562591169523126, + "pass@16": 0.6177651169032992, + "pass@32": 0.6700276603939938, + "pass@64": 0.7120274324237998, + "pass@128": 0.7352941176470589 + }, + "19": { + "pass@1": 0.1942196800595238, + "pass@2": 0.27048657492032246, + "pass@4": 0.3513666932816434, + "pass@8": 0.42792231718413776, + "pass@16": 0.4948232856154525, + "pass@32": 0.5520586849245674, + "pass@64": 0.6022906289407459, + "pass@128": 0.6488095238095238 + }, + "20": { + "pass@1": 0.15578218562874252, + "pass@2": 0.2213878100099014, + "pass@4": 0.29161606848170923, + "pass@8": 0.35769019300391275, + "pass@16": 0.41528207411562973, + "pass@32": 0.4641400644546501, + "pass@64": 0.5086113800320359, + "pass@128": 0.5538922155688623 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 615252, + "answer_accuracy": 0.7679591388400703, + "avg_response_len": 297.28580843585235, + "resp_tokens_sum": 238171120, + "pass_at_k": { + "pass@1": 0.7679591388400703, + "pass@2": 0.80280958412012, + "pass@4": 0.8323791765055107, + "pass@8": 0.8562002783982897, + "pass@16": 0.8747068337721167, + "pass@32": 0.8893966600296346, + "pass@64": 0.9017231787810277, + "pass@128": 0.9127656175107844 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9701026119402985, + "pass@2": 0.9777566400282055, + "pass@4": 0.9857178737918953, + "pass@8": 0.9925662245016207, + "pass@16": 0.9963275774549865, + "pass@32": 0.9976673737841041, + "pass@64": 0.998507046450413, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996553308823529, + "pass@2": 0.999967070981936, + "pass@4": 0.9999997240234677, + "pass@8": 0.9999999999938223, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999505537974683, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9930086336336337, + "pass@2": 0.9953647052564376, + "pass@4": 0.9969952643869967, + "pass@8": 0.9978072841093536, + "pass@16": 0.9984771551602851, + "pass@32": 0.9993062015893434, + "pass@64": 0.9999134567638505, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9979903536977492, + "pass@2": 0.9991205819429324, + "pass@4": 0.9997825528038898, + "pass@8": 0.9999870825715659, + "pass@16": 0.9999999705328944, + "pass@32": 0.9999999999999847, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9710069444444445, + "pass@2": 0.9764041213598301, + "pass@4": 0.9796988629397515, + "pass@8": 0.9827102919501212, + "pass@16": 0.9855400985081924, + "pass@32": 0.9875887371126105, + "pass@64": 0.9888675695922908, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9308342889908257, + "pass@2": 0.9481789081124038, + "pass@4": 0.9592915828182028, + "pass@8": 0.9665076055900975, + "pass@16": 0.9709244601946977, + "pass@32": 0.974005194197968, + "pass@64": 0.9774614770851698, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.8264371141975309, + "pass@2": 0.8682439000680472, + "pass@4": 0.8950250631286828, + "pass@8": 0.9127217644259317, + "pass@16": 0.9258147629750567, + "pass@32": 0.937148987373638, + "pass@64": 0.9472933098102161, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7919371290801187, + "pass@2": 0.8456217581018252, + "pass@4": 0.8804913947143843, + "pass@8": 0.9041645500183436, + "pass@16": 0.9201660117289387, + "pass@32": 0.9314733543627315, + "pass@64": 0.9394876070185926, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6754436728395061, + "pass@2": 0.7502293550111786, + "pass@4": 0.7975142907483781, + "pass@8": 0.8288913191016511, + "pass@16": 0.8517059029646968, + "pass@32": 0.8708993602586046, + "pass@64": 0.8893770688868474, + "pass@128": 0.9074074074074074 + }, + "16": { + "pass@1": 0.42203632005899705, + "pass@2": 0.5336271572248159, + "pass@4": 0.6313714790076016, + "pass@8": 0.7086579815585403, + "pass@16": 0.7656892793575719, + "pass@32": 0.8088128750811807, + "pass@64": 0.8414364243025991, + "pass@128": 0.8672566371681416 + }, + "17": { + "pass@1": 0.30191076807228917, + "pass@2": 0.40630484536571476, + "pass@4": 0.5023537657378669, + "pass@8": 0.5811390277540882, + "pass@16": 0.6416321001824091, + "pass@32": 0.6872121115568036, + "pass@64": 0.7241464264539637, + "pass@128": 0.7590361445783133 + }, + "18": { + "pass@1": 0.3072832661290323, + "pass@2": 0.39945508953517883, + "pass@4": 0.4801217611911414, + "pass@8": 0.5441190420212122, + "pass@16": 0.5942044110090919, + "pass@32": 0.6358677285081136, + "pass@64": 0.6736460231779342, + "pass@128": 0.7129032258064516 + }, + "19": { + "pass@1": 0.2081759818731118, + "pass@2": 0.2850208595998762, + "pass@4": 0.3616199484694322, + "pass@8": 0.4312974903515109, + "pass@16": 0.49094116551693745, + "pass@32": 0.5390637394342128, + "pass@64": 0.5770831994471198, + "pass@128": 0.6102719033232629 + }, + "20": { + "pass@1": 0.18482545045045046, + "pass@2": 0.258611096332553, + "pass@4": 0.3362219348144546, + "pass@8": 0.4083622891705557, + "pass@16": 0.46931520491151213, + "pass@32": 0.5211734076655891, + "pass@64": 0.5678506096461912, + "pass@128": 0.6066066066066066 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/Qwen3-100M-epoch4-RL-op9-12/composition/summary.csv b/composition/Qwen3-100M-epoch4-RL-op9-12/composition/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..c2c79dfd94cba9e500f1804e8667ee2d93d173a7 --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op9-12/composition/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +hf_model,0.9964331597222222,0.5711796875,0.7726155427631579,0.0,0.0,0.0,200.05666840277777,353.29377890625,1152000,1280000,2432000 diff --git a/composition/Qwen3-100M-epoch4-RL-op9-12/composition/summary.json b/composition/Qwen3-100M-epoch4-RL-op9-12/composition/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..16acf2f9c022c82a5f4d437f58cfb2a6f3686fb8 --- /dev/null +++ b/composition/Qwen3-100M-epoch4-RL-op9-12/composition/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "hf_model", + "id_acc": 0.9964331597222222, + "ood_acc": 0.5711796875, + "total_acc": 0.7726155427631579, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.05666840277777, + "ood_avg_resp_len": 353.29377890625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 627349, + "answer_accuracy": 0.7747651063073032, + "avg_response_len": 268.240741335362, + "resp_tokens_sum": 217202039, + "pass_at_k": { + "pass@1": 0.7747651063073032, + "pass@2": 0.8097092310263018, + "pass@4": 0.8385970834620956, + "pass@8": 0.8616455600696019, + "pass@16": 0.8798655864168684, + "pass@32": 0.894531656770708, + "pass@64": 0.9066220415916817, + "pass@128": 0.9170091685109074 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9783370206489675, + "pass@2": 0.9862172061180408, + "pass@4": 0.9923453603144738, + "pass@8": 0.9962260019031219, + "pass@16": 0.9980889112757674, + "pass@32": 0.9990783165640255, + "pass@64": 0.9998243095282938, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998291015625, + "pass@2": 0.999991926058071, + "pass@4": 0.9999999897473753, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9977926587301588, + "pass@2": 0.998470503687039, + "pass@4": 0.9992734164181859, + "pass@8": 0.999843179173257, + "pass@16": 0.9999939970802351, + "pass@32": 0.9999999966745557, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9994683801775148, + "pass@2": 0.9999159163444068, + "pass@4": 0.9999979713149763, + "pass@8": 0.9999999993382783, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9997495993589743, + "pass@2": 0.9999964510145367, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9975792253521126, + "pass@2": 0.9981427442608406, + "pass@4": 0.9987858579297307, + "pass@8": 0.9994925064642773, + "pass@16": 0.9999199128774705, + "pass@32": 0.9999987816218918, + "pass@64": 0.9999999999883001, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.968683604815864, + "pass@2": 0.9789929819767574, + "pass@4": 0.9860749473419221, + "pass@8": 0.9897070568664353, + "pass@16": 0.9917076454481355, + "pass@32": 0.9939863300081206, + "pass@64": 0.9968296937386972, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.9008430437853108, + "pass@2": 0.9165818652964992, + "pass@4": 0.927459671884235, + "pass@8": 0.9360445467076817, + "pass@16": 0.9432223808868564, + "pass@32": 0.9496503636415805, + "pass@64": 0.9559704286203133, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8701998432601881, + "pass@2": 0.9126545800113549, + "pass@4": 0.9398911460518841, + "pass@8": 0.9570679389519005, + "pass@16": 0.9671074301439438, + "pass@32": 0.9728464279999752, + "pass@64": 0.9763005928045156, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8175379672897196, + "pass@2": 0.8708705877302721, + "pass@4": 0.8993175306941774, + "pass@8": 0.9171035195588915, + "pass@16": 0.9308228683447412, + "pass@32": 0.9421168792214055, + "pass@64": 0.9521133810218851, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.7054281069364162, + "pass@2": 0.7905655466296483, + "pass@4": 0.8474447166286287, + "pass@8": 0.8839813831382466, + "pass@16": 0.9076477470834142, + "pass@32": 0.9237123268955159, + "pass@64": 0.935420710689402, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.4908266129032258, + "pass@2": 0.5978314706629414, + "pass@4": 0.6797848857602473, + "pass@8": 0.7399091549637592, + "pass@16": 0.7856011678337806, + "pass@32": 0.8230687707717842, + "pass@64": 0.8559382046199904, + "pass@128": 0.8838709677419355 + }, + "17": { + "pass@1": 0.37229046242774566, + "pass@2": 0.47453781405033935, + "pass@4": 0.5644480215803944, + "pass@8": 0.6348009526237541, + "pass@16": 0.6883834382986443, + "pass@32": 0.7330710395952293, + "pass@64": 0.7728938279339617, + "pass@128": 0.8063583815028902 + }, + "18": { + "pass@1": 0.26310267857142855, + "pass@2": 0.34792955568053996, + "pass@4": 0.4297920528683913, + "pass@8": 0.5048888152803351, + "pass@16": 0.5699261319707205, + "pass@32": 0.6207125552742997, + "pass@64": 0.6580750858985634, + "pass@128": 0.6885714285714286 + }, + "19": { + "pass@1": 0.19761167417417416, + "pass@2": 0.27271021021021014, + "pass@4": 0.3471924486353619, + "pass@8": 0.41691812297090086, + "pass@16": 0.48171163739621536, + "pass@32": 0.5404352797339113, + "pass@64": 0.5899459963388314, + "pass@128": 0.6336336336336337 + }, + "20": { + "pass@1": 0.19693130630630631, + "pass@2": 0.2723244898441749, + "pass@4": 0.3514077086084959, + "pass@8": 0.42174819195840463, + "pass@16": 0.4768338569181501, + "pass@32": 0.5186081284129838, + "pass@64": 0.5519277042276182, + "pass@128": 0.5795795795795796 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 636400, + "answer_accuracy": 0.7750389711613406, + "avg_response_len": 276.8269680436477, + "resp_tokens_sum": 227308160, + "pass_at_k": { + "pass@1": 0.7750389711613406, + "pass@2": 0.8100753877170177, + "pass@4": 0.8393560562528998, + "pass@8": 0.8627327897994856, + "pass@16": 0.8810673670584344, + "pass@32": 0.8959133677241993, + "pass@64": 0.9081338737790662, + "pass@128": 0.9178487918939985 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9760352760736196, + "pass@2": 0.9843927376696779, + "pass@4": 0.9899295643067624, + "pass@8": 0.9934119367875759, + "pass@16": 0.9961162150859647, + "pass@32": 0.9984753629176755, + "pass@64": 0.9997983104788434, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9997961956521739, + "pass@2": 0.9999960772566472, + "pass@4": 0.999999998641474, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999774205202312, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9971390845070423, + "pass@2": 0.9987000249528667, + "pass@4": 0.9997053884989727, + "pass@8": 0.9999854279093834, + "pass@16": 0.9999999724377853, + "pass@32": 0.9999999999999866, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9973802395209581, + "pass@2": 0.9990205366825403, + "pass@4": 0.9998069255190407, + "pass@8": 0.9999906816576446, + "pass@16": 0.9999999843301874, + "pass@32": 0.9999999999999962, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9709855045180723, + "pass@2": 0.9759058379186036, + "pass@4": 0.9788218743929598, + "pass@8": 0.9812271434030994, + "pass@16": 0.9835566388472978, + "pass@32": 0.9853343040816687, + "pass@64": 0.9864394200635662, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.9212137539184952, + "pass@2": 0.9338549508799643, + "pass@4": 0.9409774663746967, + "pass@8": 0.9461605137565879, + "pass@16": 0.9500469032321945, + "pass@32": 0.9528488646686972, + "pass@64": 0.9544922426331286, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.8534445028011205, + "pass@2": 0.8905946729085336, + "pass@4": 0.9134338682979749, + "pass@8": 0.9279350316050726, + "pass@16": 0.937872928293743, + "pass@32": 0.9456857085216124, + "pass@64": 0.9521581923333696, + "pass@128": 0.957983193277311 + }, + "14": { + "pass@1": 0.8009868421052632, + "pass@2": 0.860120383800709, + "pass@4": 0.8988449900451034, + "pass@8": 0.924327747686637, + "pass@16": 0.9411481778605455, + "pass@32": 0.9520906949142818, + "pass@64": 0.9598763018066186, + "pass@128": 0.9649122807017544 + }, + "15": { + "pass@1": 0.7052083333333333, + "pass@2": 0.7861254026485326, + "pass@4": 0.8397210235084246, + "pass@8": 0.8743415993819952, + "pass@16": 0.8982540136113785, + "pass@32": 0.9157051499837666, + "pass@64": 0.9282322085493483, + "pass@128": 0.9393939393939394 + }, + "16": { + "pass@1": 0.4555956196581197, + "pass@2": 0.5646556800592232, + "pass@4": 0.6547937362530536, + "pass@8": 0.7227923664185703, + "pass@16": 0.770888499121302, + "pass@32": 0.8079454921479431, + "pass@64": 0.836781426539472, + "pass@128": 0.8547008547008547 + }, + "17": { + "pass@1": 0.3923718944099379, + "pass@2": 0.5017893058883942, + "pass@4": 0.5944229474227205, + "pass@8": 0.6676706766755468, + "pass@16": 0.72361349680337, + "pass@32": 0.7682336081279583, + "pass@64": 0.8049794395003316, + "pass@128": 0.8322981366459627 + }, + "18": { + "pass@1": 0.2898207720588235, + "pass@2": 0.3896733151922183, + "pass@4": 0.480811416495732, + "pass@8": 0.5562591169523126, + "pass@16": 0.6177651169032992, + "pass@32": 0.6700276603939938, + "pass@64": 0.7120274324237998, + "pass@128": 0.7352941176470589 + }, + "19": { + "pass@1": 0.1942196800595238, + "pass@2": 0.27048657492032246, + "pass@4": 0.3513666932816434, + "pass@8": 0.42792231718413776, + "pass@16": 0.4948232856154525, + "pass@32": 0.5520586849245674, + "pass@64": 0.6022906289407459, + "pass@128": 0.6488095238095238 + }, + "20": { + "pass@1": 0.15578218562874252, + "pass@2": 0.2213878100099014, + "pass@4": 0.29161606848170923, + "pass@8": 0.35769019300391275, + "pass@16": 0.41528207411562973, + "pass@32": 0.4641400644546501, + "pass@64": 0.5086113800320359, + "pass@128": 0.5538922155688623 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 615252, + "answer_accuracy": 0.7679591388400703, + "avg_response_len": 297.28580843585235, + "resp_tokens_sum": 238171120, + "pass_at_k": { + "pass@1": 0.7679591388400703, + "pass@2": 0.80280958412012, + "pass@4": 0.8323791765055107, + "pass@8": 0.8562002783982897, + "pass@16": 0.8747068337721167, + "pass@32": 0.8893966600296346, + "pass@64": 0.9017231787810277, + "pass@128": 0.9127656175107844 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9701026119402985, + "pass@2": 0.9777566400282055, + "pass@4": 0.9857178737918953, + "pass@8": 0.9925662245016207, + "pass@16": 0.9963275774549865, + "pass@32": 0.9976673737841041, + "pass@64": 0.998507046450413, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996553308823529, + "pass@2": 0.999967070981936, + "pass@4": 0.9999997240234677, + "pass@8": 0.9999999999938223, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999505537974683, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9930086336336337, + "pass@2": 0.9953647052564376, + "pass@4": 0.9969952643869967, + "pass@8": 0.9978072841093536, + "pass@16": 0.9984771551602851, + "pass@32": 0.9993062015893434, + "pass@64": 0.9999134567638505, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9979903536977492, + "pass@2": 0.9991205819429324, + "pass@4": 0.9997825528038898, + "pass@8": 0.9999870825715659, + "pass@16": 0.9999999705328944, + "pass@32": 0.9999999999999847, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9710069444444445, + "pass@2": 0.9764041213598301, + "pass@4": 0.9796988629397515, + "pass@8": 0.9827102919501212, + "pass@16": 0.9855400985081924, + "pass@32": 0.9875887371126105, + "pass@64": 0.9888675695922908, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9308342889908257, + "pass@2": 0.9481789081124038, + "pass@4": 0.9592915828182028, + "pass@8": 0.9665076055900975, + "pass@16": 0.9709244601946977, + "pass@32": 0.974005194197968, + "pass@64": 0.9774614770851698, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.8264371141975309, + "pass@2": 0.8682439000680472, + "pass@4": 0.8950250631286828, + "pass@8": 0.9127217644259317, + "pass@16": 0.9258147629750567, + "pass@32": 0.937148987373638, + "pass@64": 0.9472933098102161, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7919371290801187, + "pass@2": 0.8456217581018252, + "pass@4": 0.8804913947143843, + "pass@8": 0.9041645500183436, + "pass@16": 0.9201660117289387, + "pass@32": 0.9314733543627315, + "pass@64": 0.9394876070185926, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6754436728395061, + "pass@2": 0.7502293550111786, + "pass@4": 0.7975142907483781, + "pass@8": 0.8288913191016511, + "pass@16": 0.8517059029646968, + "pass@32": 0.8708993602586046, + "pass@64": 0.8893770688868474, + "pass@128": 0.9074074074074074 + }, + "16": { + "pass@1": 0.42203632005899705, + "pass@2": 0.5336271572248159, + "pass@4": 0.6313714790076016, + "pass@8": 0.7086579815585403, + "pass@16": 0.7656892793575719, + "pass@32": 0.8088128750811807, + "pass@64": 0.8414364243025991, + "pass@128": 0.8672566371681416 + }, + "17": { + "pass@1": 0.30191076807228917, + "pass@2": 0.40630484536571476, + "pass@4": 0.5023537657378669, + "pass@8": 0.5811390277540882, + "pass@16": 0.6416321001824091, + "pass@32": 0.6872121115568036, + "pass@64": 0.7241464264539637, + "pass@128": 0.7590361445783133 + }, + "18": { + "pass@1": 0.3072832661290323, + "pass@2": 0.39945508953517883, + "pass@4": 0.4801217611911414, + "pass@8": 0.5441190420212122, + "pass@16": 0.5942044110090919, + "pass@32": 0.6358677285081136, + "pass@64": 0.6736460231779342, + "pass@128": 0.7129032258064516 + }, + "19": { + "pass@1": 0.2081759818731118, + "pass@2": 0.2850208595998762, + "pass@4": 0.3616199484694322, + "pass@8": 0.4312974903515109, + "pass@16": 0.49094116551693745, + "pass@32": 0.5390637394342128, + "pass@64": 0.5770831994471198, + "pass@128": 0.6102719033232629 + }, + "20": { + "pass@1": 0.18482545045045046, + "pass@2": 0.258611096332553, + "pass@4": 0.3362219348144546, + "pass@8": 0.4083622891705557, + "pass@16": 0.46931520491151213, + "pass@32": 0.5211734076655891, + "pass@64": 0.5678506096461912, + "pass@128": 0.6066066066066066 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_metrics.json b/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..6e46a92cd7fb342fe18a0df8921d447b74ec81c7 --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch4-op11-14", + "id": { + "answer_accuracy": 0.9905703125, + "avg_response_len": 199.1858359375, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9905703125, + "pass@2": 0.9931240567585284, + "pass@4": 0.9946121005395152, + "pass@8": 0.9956282392841028, + "pass@16": 0.9963701188054171, + "pass@32": 0.996918025082482, + "pass@64": 0.9973719313555771, + "pass@128": 0.9977777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.942828125, + "pass@2": 0.9543673720472442, + "pass@4": 0.9623778480502434, + "pass@8": 0.9682502041026957, + "pass@16": 0.972860979589154, + "pass@32": 0.9767782353474479, + "pass@64": 0.9804701699275815, + "pass@128": 0.984 + }, + "2": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999953125, + "pass@2": 0.9999992618110236, + "pass@4": 0.9999999999062618, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998390625, + "pass@2": 0.9988737696850394, + "pass@4": 0.9993528392388451, + "pass@8": 0.9997678683581743, + "pass@16": 0.9999654222866098, + "pass@32": 0.9999993776062154, + "pass@64": 0.9999999999893633, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9933984375, + "pass@2": 0.9961841781496062, + "pass@4": 0.9975373475815523, + "pass@8": 0.9983680305456933, + "pass@16": 0.9990184921424964, + "pass@32": 0.9995291759985511, + "pass@64": 0.9998772846233093, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99665625, + "pass@2": 0.9977063238188978, + "pass@4": 0.9982420018747656, + "pass@8": 0.9986003427777526, + "pass@16": 0.9988616968167805, + "pass@32": 0.9989862397601972, + "pass@64": 0.998999972481556, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9970390625, + "pass@2": 0.9983011811023623, + "pass@4": 0.9990445734908138, + "pass@8": 0.9995488522357531, + "pass@16": 0.9998741388348003, + "pass@32": 0.9999900821873651, + "pass@64": 0.9999999879305058, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9941171875, + "pass@2": 0.9971940206692916, + "pass@4": 0.9985053151481066, + "pass@8": 0.9989375417785472, + "pass@16": 0.9989989424149476, + "pass@32": 0.9989999998940483, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.992765625, + "pass@2": 0.9954905265748032, + "pass@4": 0.9964489795650546, + "pass@8": 0.9971813137583089, + "pass@16": 0.9977513971639642, + "pass@32": 0.9979791149485119, + "pass@64": 0.9979999672478784, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.942828125, + "2": 0.999984375, + "3": 0.999953125, + "4": 0.998390625, + "5": 0.9933984375, + "6": 0.99665625, + "7": 0.9970390625, + "8": 0.9941171875, + "9": 0.992765625 + }, + "per_op_avg_response_len": { + "10": 279.0269453125, + "2": 115.345625, + "3": 144.2695546875, + "4": 159.5131875, + "5": 172.7417421875, + "6": 198.263265625, + "7": 223.090140625, + "8": 242.3848515625, + "9": 258.0372109375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.0269453125, + "2": 115.345625, + "3": 144.2695546875, + "4": 159.5131875, + "5": 172.7417421875, + "6": 198.263265625, + "7": 223.090140625, + "8": 242.3848515625, + "9": 258.0372109375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 375744, + "answer_accuracy": 0.9913880445795339, + "avg_response_len": 189.61989720533603, + "resp_tokens_sum": 71867458, + "pass_at_k": { + "pass@1": 0.9913880445795339, + "pass@2": 0.9933423381917686, + "pass@4": 0.9944101428494183, + "pass@8": 0.995316754918926, + "pass@16": 0.9962039266427511, + "pass@32": 0.9970168336391735, + "pass@64": 0.9977520459936974, + "pass@128": 0.9983113812901047 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9453585914454278, + "pass@2": 0.9535728491626602, + "pass@4": 0.95873854180838, + "pass@8": 0.9629986555885872, + "pass@16": 0.9679666317812168, + "pass@32": 0.9740341139349422, + "pass@64": 0.9803653794455086, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9971923828125, + "pass@2": 0.9978869340551182, + "pass@4": 0.9985996801181102, + "pass@8": 0.9993905663162155, + "pass@16": 0.9998951809167032, + "pass@32": 0.9999980558628433, + "pass@64": 0.9999999999667597, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9949900793650793, + "pass@2": 0.9973585333083365, + "pass@4": 0.9983299611358104, + "pass@8": 0.9991563945664312, + "pass@16": 0.9997971079638622, + "pass@32": 0.9999918808807343, + "pass@64": 0.9999999987391617, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9971229416167665, + "pass@2": 0.9976215138620398, + "pass@4": 0.9981272295117304, + "pass@8": 0.998847506991621, + "pass@16": 0.9995863571069055, + "pass@32": 0.999958801686582, + "pass@64": 0.9999999176094492, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9964866863905325, + "pass@2": 0.9979623363462704, + "pass@4": 0.9990334570789599, + "pass@8": 0.9997250598303445, + "pass@16": 0.9999790093720811, + "pass@32": 0.9999999405368915, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9981971153846154, + "pass@2": 0.9998284657026045, + "pass@4": 0.999995967751627, + "pass@8": 0.9999999975724242, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9949603873239437, + "pass@2": 0.9973813352556282, + "pass@4": 0.9982075685081618, + "pass@8": 0.9989154500291274, + "pass@16": 0.9996108261180917, + "pass@32": 0.9999612387699112, + "pass@64": 0.9999999224832565, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 386609, + "answer_accuracy": 0.989640502129751, + "avg_response_len": 197.2131773222477, + "resp_tokens_sum": 77042511, + "pass_at_k": { + "pass@1": 0.989640502129751, + "pass@2": 0.9919891258217151, + "pass@4": 0.993503951013332, + "pass@8": 0.9945439716423881, + "pass@16": 0.99522975394596, + "pass@32": 0.9956609074616072, + "pass@64": 0.9959856221204596, + "pass@128": 0.9963958060288336 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9309815950920245, + "pass@2": 0.9449852966040289, + "pass@4": 0.9559561028797782, + "pass@8": 0.9638793675635439, + "pass@16": 0.9687407182381751, + "pass@32": 0.9718304058076516, + "pass@64": 0.9746895660017418, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99375, + "pass@2": 0.9960658450302408, + "pass@4": 0.9975695505453124, + "pass@8": 0.9984974444758158, + "pass@16": 0.9992768151012691, + "pass@32": 0.9998558695467676, + "pass@64": 0.999998064534979, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9964183381088825, + "pass@2": 0.9970275026509938, + "pass@4": 0.9971312728100964, + "pass@8": 0.9971346683194743, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9973582008670521, + "pass@2": 0.9977495192526513, + "pass@4": 0.9982556172530457, + "pass@8": 0.9989655495774096, + "pass@16": 0.9996567447729243, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9936839788732394, + "pass@2": 0.9963724769879118, + "pass@4": 0.9971180363018003, + "pass@8": 0.9971826215324542, + "pass@16": 0.9971830985804359, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9924214071856288, + "pass@2": 0.9938524771323495, + "pass@4": 0.9940081237973997, + "pass@8": 0.9940119737766329, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 378784, + "answer_accuracy": 0.9907097422162705, + "avg_response_len": 210.68409461834617, + "resp_tokens_sum": 80552114, + "pass_at_k": { + "pass@1": 0.9907097422162705, + "pass@2": 0.9940673034593474, + "pass@4": 0.995944564240317, + "pass@8": 0.9970448747202597, + "pass@16": 0.9977000446657175, + "pass@32": 0.9981045503059652, + "pass@64": 0.9984116020425892, + "pass@128": 0.9986608637428859 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9517957089552239, + "pass@2": 0.9643014014572805, + "pass@4": 0.9723098293683435, + "pass@8": 0.9778178687535791, + "pass@16": 0.9818233351333638, + "pass@32": 0.9843700251647998, + "pass@64": 0.9862015157581675, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 0.9999553571428571, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998861151603499, + "pass@2": 0.99999784784555, + "pass@4": 0.9999999997267104, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9979669225146199, + "pass@2": 0.9986840666298292, + "pass@4": 0.9994179578978066, + "pass@8": 0.9998914828566827, + "pass@16": 0.999996971910131, + "pass@32": 0.9999999992108352, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9915670955882353, + "pass@2": 0.9952162459471977, + "pass@4": 0.996770337898939, + "pass@8": 0.9975063173915031, + "pass@16": 0.9980350056586529, + "pass@32": 0.9987689956690733, + "pass@64": 0.9996410375173692, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9964264589905363, + "pass@2": 0.9985430276459921, + "pass@4": 0.9995857823568582, + "pass@8": 0.9999535526784058, + "pass@16": 0.9999995379907698, + "pass@32": 0.9999999999901541, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972804588607594, + "pass@2": 0.9992676511262834, + "pass@4": 0.9999203209764919, + "pass@8": 0.9999990565168133, + "pass@16": 0.9999999999228011, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9907563813813813, + "pass@2": 0.99560153164976, + "pass@4": 0.998587598565945, + "pass@8": 0.9998129486244137, + "pass@16": 0.999996824080759, + "pass@32": 0.9999999996818268, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9906300241157556, + "pass@2": 0.9950913993467858, + "pass@4": 0.9970629562582811, + "pass@8": 0.9986055619182425, + "pass@16": 0.99964486782007, + "pass@32": 0.9999770906276318, + "pass@64": 0.9999999831714547, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.6678171875, + "avg_response_len": 367.13116015625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.6678171875, + "pass@2": 0.7378225639763673, + "pass@4": 0.7893652439351264, + "pass@8": 0.8270737370351559, + "pass@16": 0.8550486463709078, + "pass@32": 0.8767224088959323, + "pass@64": 0.8945746837166629, + "pass@128": 0.9096 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.963046875, + "pass@2": 0.9739500492125989, + "pass@4": 0.9801725765841772, + "pass@8": 0.9841731737862502, + "pass@16": 0.986679806459084, + "pass@32": 0.9887283379415103, + "pass@64": 0.9903890917527084, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.9079453125, + "pass@2": 0.9249128937007879, + "pass@4": 0.9345272270341208, + "pass@8": 0.9416891861472778, + "pass@16": 0.9476419112166174, + "pass@32": 0.9523055926849554, + "pass@64": 0.9562749449531718, + "pass@128": 0.961 + }, + "13": { + "pass@1": 0.8557421875, + "pass@2": 0.8929637057086628, + "pass@4": 0.9139262857142856, + "pass@8": 0.9291272854564019, + "pass@16": 0.9415987007431647, + "pass@32": 0.9518761615883907, + "pass@64": 0.9606307335381308, + "pass@128": 0.968 + }, + "14": { + "pass@1": 0.837, + "pass@2": 0.8863106545275599, + "pass@4": 0.9137613407386579, + "pass@8": 0.9312510179501858, + "pass@16": 0.9439751905099076, + "pass@32": 0.9542164522470538, + "pass@64": 0.9626050622049743, + "pass@128": 0.969 + }, + "15": { + "pass@1": 0.77975, + "pass@2": 0.8448986220472454, + "pass@4": 0.8857677772778397, + "pass@8": 0.9115909825900337, + "pass@16": 0.9277541320088688, + "pass@32": 0.9388842588529385, + "pass@64": 0.9474485358545235, + "pass@128": 0.954 + }, + "16": { + "pass@1": 0.6545625, + "pass@2": 0.7510269438976384, + "pass@4": 0.8116648335208099, + "pass@8": 0.8497649349116291, + "pass@16": 0.8757048434440007, + "pass@32": 0.8956257423100381, + "pass@64": 0.9124470610164344, + "pass@128": 0.925 + }, + "17": { + "pass@1": 0.55796875, + "pass@2": 0.6625004921259854, + "pass@4": 0.7365261226096746, + "pass@8": 0.7870371231754127, + "pass@16": 0.8224065318779613, + "pass@32": 0.8483149142099418, + "pass@64": 0.8693774322654676, + "pass@128": 0.888 + }, + "18": { + "pass@1": 0.480296875, + "pass@2": 0.5879370078740163, + "pass@4": 0.6728501369516315, + "pass@8": 0.7364498415117591, + "pass@16": 0.7835286822486924, + "pass@32": 0.8195105952369934, + "pass@64": 0.8482734816022467, + "pass@128": 0.872 + }, + "19": { + "pass@1": 0.33475, + "pass@2": 0.44351464074803204, + "pass@4": 0.5406705727409072, + "pass@8": 0.6195248806878444, + "pass@16": 0.6802573730953311, + "pass@32": 0.7279540135363097, + "pass@64": 0.7678995342731354, + "pass@128": 0.803 + }, + "20": { + "pass@1": 0.307109375, + "pass@2": 0.4102106299212602, + "pass@4": 0.503785566179227, + "pass@8": 0.5801289441348498, + "pass@16": 0.6409392921053366, + "pass@32": 0.6898080203511475, + "pass@64": 0.7304009597057819, + "pass@128": 0.765 + } + }, + "per_op_accuracy": { + "11": 0.963046875, + "12": 0.9079453125, + "13": 0.8557421875, + "14": 0.837, + "15": 0.77975, + "16": 0.6545625, + "17": 0.55796875, + "18": 0.480296875, + "19": 0.33475, + "20": 0.307109375 + }, + "per_op_avg_response_len": { + "11": 297.081765625, + "12": 307.319046875, + "13": 326.3436640625, + "14": 353.279515625, + "15": 371.5872421875, + "16": 377.5440546875, + "17": 398.5268984375, + "18": 408.8486171875, + "19": 406.905546875, + "20": 423.87525 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.081765625, + "12": 307.319046875, + "13": 326.3436640625, + "14": 353.279515625, + "15": 371.5872421875, + "16": 377.5440546875, + "17": 398.5268984375, + "18": 408.8486171875, + "19": 406.905546875, + "20": 423.87525 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 290947, + "answer_accuracy": 0.6754898774145617, + "avg_response_len": 349.3103222511144, + "resp_tokens_sum": 150454942, + "pass_at_k": { + "pass@1": 0.6754898774145617, + "pass@2": 0.7447737390459964, + "pass@4": 0.7952060179698968, + "pass@8": 0.8316698977934396, + "pass@16": 0.8584897543756141, + "pass@32": 0.8792215528888129, + "pass@64": 0.8964337939825147, + "pass@128": 0.9105497771173848 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9636597025495751, + "pass@2": 0.9756313990319198, + "pass@4": 0.9833490821614721, + "pass@8": 0.9890875592461815, + "pass@16": 0.9928681364142148, + "pass@32": 0.9954215669121044, + "pass@64": 0.9969126772151184, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.9084348516949152, + "pass@2": 0.9271233011922239, + "pass@4": 0.9376102685257564, + "pass@8": 0.9448320171203957, + "pass@16": 0.9500880057183615, + "pass@32": 0.9533178342586844, + "pass@64": 0.9561329927476948, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.8727468652037618, + "pass@2": 0.9105426036087183, + "pass@4": 0.9320865086809287, + "pass@8": 0.9476093057668198, + "pass@16": 0.9599091371962115, + "pass@32": 0.9694213182908559, + "pass@64": 0.9768613994766613, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.8451129283489096, + "pass@2": 0.8927793926460124, + "pass@4": 0.9188406133812713, + "pass@8": 0.9352618975128104, + "pass@16": 0.9480306260712734, + "pass@32": 0.9589256381658317, + "pass@64": 0.9683198089214712, + "pass@128": 0.9750778816199377 + }, + "15": { + "pass@1": 0.7921107297687862, + "pass@2": 0.8578609303172365, + "pass@4": 0.8997546105147255, + "pass@8": 0.9264640318143393, + "pass@16": 0.9428917374738313, + "pass@32": 0.952393511786133, + "pass@64": 0.9574741921523455, + "pass@128": 0.9595375722543352 + }, + "16": { + "pass@1": 0.6568800403225806, + "pass@2": 0.7477683674117346, + "pass@4": 0.8092675075897764, + "pass@8": 0.8516551322611292, + "pass@16": 0.8812944796205525, + "pass@32": 0.902773745388599, + "pass@64": 0.9194705540558217, + "pass@128": 0.932258064516129 + }, + "17": { + "pass@1": 0.5657514450867052, + "pass@2": 0.6691021687679213, + "pass@4": 0.739191088198512, + "pass@8": 0.7856207032319886, + "pass@16": 0.8183619373072404, + "pass@32": 0.8430109049522699, + "pass@64": 0.8637531249758412, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.45412946428571427, + "pass@2": 0.5638899043869511, + "pass@4": 0.6530802185441107, + "pass@8": 0.720726478211569, + "pass@16": 0.7704510468029769, + "pass@32": 0.807289056099719, + "pass@64": 0.8378334239926626, + "pass@128": 0.8657142857142858 + }, + "19": { + "pass@1": 0.36369181681681684, + "pass@2": 0.4703841390603205, + "pass@4": 0.5581976540657639, + "pass@8": 0.627149829400672, + "pass@16": 0.6817550885633032, + "pass@32": 0.7263554189791901, + "pass@64": 0.763762224924364, + "pass@128": 0.7957957957957958 + }, + "20": { + "pass@1": 0.32453547297297297, + "pass@2": 0.4275736119978244, + "pass@4": 0.5169445100330928, + "pass@8": 0.585449717921024, + "pass@16": 0.6372318334285229, + "pass@32": 0.6821752409020172, + "pass@64": 0.7234653885399501, + "pass@128": 0.7537537537537538 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 291147, + "answer_accuracy": 0.6763562109723461, + "avg_response_len": 362.8061301293488, + "resp_tokens_sum": 156174978, + "pass_at_k": { + "pass@1": 0.6763562109723461, + "pass@2": 0.7465801048815182, + "pass@4": 0.7976332155904704, + "pass@8": 0.8348140204603103, + "pass@16": 0.8624812646405657, + "pass@32": 0.883998466128224, + "pass@64": 0.9017876930240121, + "pass@128": 0.9170383586083853 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9630082831325302, + "pass@2": 0.9713395906460487, + "pass@4": 0.9757507891521089, + "pass@8": 0.9784955531653872, + "pass@16": 0.9805383638724173, + "pass@32": 0.9824747306648515, + "pass@64": 0.9841054553733752, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9007885971786834, + "pass@2": 0.915418739046726, + "pass@4": 0.924029935129582, + "pass@8": 0.9303735376408099, + "pass@16": 0.9353211411280159, + "pass@32": 0.939039317249673, + "pass@64": 0.9419950990841026, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.8611913515406162, + "pass@2": 0.8988553590727634, + "pass@4": 0.918313749121695, + "pass@8": 0.9307861443620137, + "pass@16": 0.9400313356135247, + "pass@32": 0.9484692530915426, + "pass@64": 0.9581816741685208, + "pass@128": 0.969187675070028 + }, + "14": { + "pass@1": 0.8469709429824561, + "pass@2": 0.8980640886172121, + "pass@4": 0.9245809564374623, + "pass@8": 0.9405203926008627, + "pass@16": 0.9525469091473052, + "pass@32": 0.963199459193292, + "pass@64": 0.9716362530225532, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.7924715909090909, + "pass@2": 0.861036670842281, + "pass@4": 0.9029049320539471, + "pass@8": 0.9281268028368013, + "pass@16": 0.9426543876503707, + "pass@32": 0.9525476884583312, + "pass@64": 0.9611302965599587, + "pass@128": 0.9696969696969697 + }, + "16": { + "pass@1": 0.6614583333333334, + "pass@2": 0.757938160374184, + "pass@4": 0.8144365875205767, + "pass@8": 0.8475187224277707, + "pass@16": 0.8701097260916296, + "pass@32": 0.8890719227083439, + "pass@64": 0.9063072562149335, + "pass@128": 0.9173789173789174 + }, + "17": { + "pass@1": 0.5830260093167702, + "pass@2": 0.6846121986110436, + "pass@4": 0.7533166365656156, + "pass@8": 0.8007323980276663, + "pass@16": 0.8367999826994603, + "pass@32": 0.8654849657428961, + "pass@64": 0.8891736360856827, + "pass@128": 0.906832298136646 + }, + "18": { + "pass@1": 0.5104779411764706, + "pass@2": 0.615626447429365, + "pass@4": 0.6993681317408854, + "pass@8": 0.7636960252154447, + "pass@16": 0.8126314154686012, + "pass@32": 0.8493046683312366, + "pass@64": 0.8760405951252511, + "pass@128": 0.8970588235294118 + }, + "19": { + "pass@1": 0.33535621279761907, + "pass@2": 0.450107066694788, + "pass@4": 0.5589451405069903, + "pass@8": 0.6479033713249467, + "pass@16": 0.7114656294943115, + "pass@32": 0.7547122095604747, + "pass@64": 0.7871870069795173, + "pass@128": 0.8154761904761905 + }, + "20": { + "pass@1": 0.30761133982035926, + "pass@2": 0.4103107760856239, + "pass@4": 0.5020851449519407, + "pass@8": 0.577641649195555, + "pass@16": 0.640643393523646, + "pass@32": 0.6937003584791263, + "pass@64": 0.7400380698580312, + "pass@128": 0.7844311377245509 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 272712, + "answer_accuracy": 0.6511499083129584, + "avg_response_len": 389.90383605210883, + "resp_tokens_sum": 163297965, + "pass_at_k": { + "pass@1": 0.6511499083129584, + "pass@2": 0.7216727124564459, + "pass@4": 0.7748605393801656, + "pass@8": 0.8143913855344584, + "pass@16": 0.8438703995259332, + "pass@32": 0.8666738147613688, + "pass@64": 0.8852491163739664, + "pass@128": 0.9009779951100244 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9624007936507937, + "pass@2": 0.9748172103487065, + "pass@4": 0.981273297087864, + "pass@8": 0.9846499737188555, + "pass@16": 0.9862178333308695, + "pass@32": 0.9878187753676408, + "pass@64": 0.9897012873390828, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9143969801223242, + "pass@2": 0.9317818572563751, + "pass@4": 0.941430099907236, + "pass@8": 0.9493256684380422, + "pass@16": 0.9570131901299108, + "pass@32": 0.9641514897698327, + "pass@64": 0.9703591098246457, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8329957561728395, + "pass@2": 0.8691643579274818, + "pass@4": 0.8912119784216789, + "pass@8": 0.9091026647516887, + "pass@16": 0.9252978369245227, + "pass@32": 0.9383556719133557, + "pass@64": 0.9473490414410928, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8191533753709199, + "pass@2": 0.8682212201219658, + "pass@4": 0.8979430763847394, + "pass@8": 0.9180236634394026, + "pass@16": 0.9314134024114261, + "pass@32": 0.9406145618745281, + "pass@64": 0.9479964540280973, + "pass@128": 0.9554896142433235 + }, + "15": { + "pass@1": 0.7535927854938271, + "pass@2": 0.8146192554923689, + "pass@4": 0.8533767113022905, + "pass@8": 0.878865995821382, + "pass@16": 0.8964124781429048, + "pass@32": 0.910541254887923, + "pass@64": 0.922806998472301, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.6453032817109144, + "pass@2": 0.7468509017954611, + "pass@4": 0.8109872092868917, + "pass@8": 0.8503621602906531, + "pass@16": 0.8763865513376642, + "pass@32": 0.8958750335367089, + "pass@64": 0.9123815407896373, + "pass@128": 0.9262536873156342 + }, + "17": { + "pass@1": 0.5255553463855421, + "pass@2": 0.6341747402997818, + "pass@4": 0.7174640033700606, + "pass@8": 0.7752305050970983, + "pass@16": 0.8126617684952755, + "pass@32": 0.8371897052025431, + "pass@64": 0.8560389765187822, + "pass@128": 0.8734939759036144 + }, + "18": { + "pass@1": 0.4767389112903226, + "pass@2": 0.5847179006858011, + "pass@4": 0.66608676022352, + "pass@8": 0.7243191147240607, + "pass@16": 0.7663746277687996, + "pass@32": 0.8006317366757167, + "pass@64": 0.829606389878158, + "pass@128": 0.8516129032258064 + }, + "19": { + "pass@1": 0.30501793806646527, + "pass@2": 0.40979077955134763, + "pass@4": 0.5044869689627164, + "pass@8": 0.5830466910339542, + "pass@16": 0.6470709277754211, + "pass@32": 0.7023998688940181, + "pass@64": 0.7524830182120951, + "pass@128": 0.797583081570997 + }, + "20": { + "pass@1": 0.2891798048048048, + "pass@2": 0.3927472009410985, + "pass@4": 0.49233214992270086, + "pass@8": 0.5773029346421416, + "pass@16": 0.6449435379481746, + "pass@32": 0.6935367729692132, + "pass@64": 0.7276704804486359, + "pass@128": 0.7567567567567568 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.8207002467105263, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8207002467105263, + "pass@2": 0.8587548500310641, + "pass@4": 0.8865874391687733, + "pass@8": 0.9069153433636171, + "pass@16": 0.9219903964714486, + "pass@32": 0.9336571744579908, + "pass@64": 0.9432681168087804, + "pass@128": 0.9513684210526315 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.942828125, + "pass@2": 0.9543673720472442, + "pass@4": 0.9623778480502434, + "pass@8": 0.9682502041026957, + "pass@16": 0.972860979589154, + "pass@32": 0.9767782353474479, + "pass@64": 0.9804701699275815, + "pass@128": 0.984 + }, + "2": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999953125, + "pass@2": 0.9999992618110236, + "pass@4": 0.9999999999062618, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998390625, + "pass@2": 0.9988737696850394, + "pass@4": 0.9993528392388451, + "pass@8": 0.9997678683581743, + "pass@16": 0.9999654222866098, + "pass@32": 0.9999993776062154, + "pass@64": 0.9999999999893633, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9933984375, + "pass@2": 0.9961841781496062, + "pass@4": 0.9975373475815523, + "pass@8": 0.9983680305456933, + "pass@16": 0.9990184921424964, + "pass@32": 0.9995291759985511, + "pass@64": 0.9998772846233093, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99665625, + "pass@2": 0.9977063238188978, + "pass@4": 0.9982420018747656, + "pass@8": 0.9986003427777526, + "pass@16": 0.9988616968167805, + "pass@32": 0.9989862397601972, + "pass@64": 0.998999972481556, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9970390625, + "pass@2": 0.9983011811023623, + "pass@4": 0.9990445734908138, + "pass@8": 0.9995488522357531, + "pass@16": 0.9998741388348003, + "pass@32": 0.9999900821873651, + "pass@64": 0.9999999879305058, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9941171875, + "pass@2": 0.9971940206692916, + "pass@4": 0.9985053151481066, + "pass@8": 0.9989375417785472, + "pass@16": 0.9989989424149476, + "pass@32": 0.9989999998940483, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.992765625, + "pass@2": 0.9954905265748032, + "pass@4": 0.9964489795650546, + "pass@8": 0.9971813137583089, + "pass@16": 0.9977513971639642, + "pass@32": 0.9979791149485119, + "pass@64": 0.9979999672478784, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.963046875, + "pass@2": 0.9739500492125989, + "pass@4": 0.9801725765841772, + "pass@8": 0.9841731737862502, + "pass@16": 0.986679806459084, + "pass@32": 0.9887283379415103, + "pass@64": 0.9903890917527084, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.9079453125, + "pass@2": 0.9249128937007879, + "pass@4": 0.9345272270341208, + "pass@8": 0.9416891861472778, + "pass@16": 0.9476419112166174, + "pass@32": 0.9523055926849554, + "pass@64": 0.9562749449531718, + "pass@128": 0.961 + }, + "13": { + "pass@1": 0.8557421875, + "pass@2": 0.8929637057086628, + "pass@4": 0.9139262857142856, + "pass@8": 0.9291272854564019, + "pass@16": 0.9415987007431647, + "pass@32": 0.9518761615883907, + "pass@64": 0.9606307335381308, + "pass@128": 0.968 + }, + "14": { + "pass@1": 0.837, + "pass@2": 0.8863106545275599, + "pass@4": 0.9137613407386579, + "pass@8": 0.9312510179501858, + "pass@16": 0.9439751905099076, + "pass@32": 0.9542164522470538, + "pass@64": 0.9626050622049743, + "pass@128": 0.969 + }, + "15": { + "pass@1": 0.77975, + "pass@2": 0.8448986220472454, + "pass@4": 0.8857677772778397, + "pass@8": 0.9115909825900337, + "pass@16": 0.9277541320088688, + "pass@32": 0.9388842588529385, + "pass@64": 0.9474485358545235, + "pass@128": 0.954 + }, + "16": { + "pass@1": 0.6545625, + "pass@2": 0.7510269438976384, + "pass@4": 0.8116648335208099, + "pass@8": 0.8497649349116291, + "pass@16": 0.8757048434440007, + "pass@32": 0.8956257423100381, + "pass@64": 0.9124470610164344, + "pass@128": 0.925 + }, + "17": { + "pass@1": 0.55796875, + "pass@2": 0.6625004921259854, + "pass@4": 0.7365261226096746, + "pass@8": 0.7870371231754127, + "pass@16": 0.8224065318779613, + "pass@32": 0.8483149142099418, + "pass@64": 0.8693774322654676, + "pass@128": 0.888 + }, + "18": { + "pass@1": 0.480296875, + "pass@2": 0.5879370078740163, + "pass@4": 0.6728501369516315, + "pass@8": 0.7364498415117591, + "pass@16": 0.7835286822486924, + "pass@32": 0.8195105952369934, + "pass@64": 0.8482734816022467, + "pass@128": 0.872 + }, + "19": { + "pass@1": 0.33475, + "pass@2": 0.44351464074803204, + "pass@4": 0.5406705727409072, + "pass@8": 0.6195248806878444, + "pass@16": 0.6802573730953311, + "pass@32": 0.7279540135363097, + "pass@64": 0.7678995342731354, + "pass@128": 0.803 + }, + "20": { + "pass@1": 0.307109375, + "pass@2": 0.4102106299212602, + "pass@4": 0.503785566179227, + "pass@8": 0.5801289441348498, + "pass@16": 0.6409392921053366, + "pass@32": 0.6898080203511475, + "pass@64": 0.7304009597057819, + "pass@128": 0.765 + } + }, + "per_op_accuracy": { + "10": 0.942828125, + "2": 0.999984375, + "3": 0.999953125, + "4": 0.998390625, + "5": 0.9933984375, + "6": 0.99665625, + "7": 0.9970390625, + "8": 0.9941171875, + "9": 0.992765625, + "11": 0.963046875, + "12": 0.9079453125, + "13": 0.8557421875, + "14": 0.837, + "15": 0.77975, + "16": 0.6545625, + "17": 0.55796875, + "18": 0.480296875, + "19": 0.33475, + "20": 0.307109375 + }, + "per_op_avg_response_len": { + "10": 279.0269453125, + "2": 115.345625, + "3": 144.2695546875, + "4": 159.5131875, + "5": 172.7417421875, + "6": 198.263265625, + "7": 223.090140625, + "8": 242.3848515625, + "9": 258.0372109375, + "11": 297.081765625, + "12": 307.319046875, + "13": 326.3436640625, + "14": 353.279515625, + "15": 371.5872421875, + "16": 377.5440546875, + "17": 398.5268984375, + "18": 408.8486171875, + "19": 406.905546875, + "20": 423.87525 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.0269453125, + "2": 115.345625, + "3": 144.2695546875, + "4": 159.5131875, + "5": 172.7417421875, + "6": 198.263265625, + "7": 223.090140625, + "8": 242.3848515625, + "9": 258.0372109375, + "11": 297.081765625, + "12": 307.319046875, + "13": 326.3436640625, + "14": 353.279515625, + "15": 371.5872421875, + "16": 377.5440546875, + "17": 398.5268984375, + "18": 408.8486171875, + "19": 406.905546875, + "20": 423.87525 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 666691, + "answer_accuracy": 0.8233517922067657, + "avg_response_len": 274.56429813468225, + "resp_tokens_sum": 222322400, + "pass_at_k": { + "pass@1": 0.8233517922067657, + "pass@2": 0.8611208180960408, + "pass@4": 0.8884471519832107, + "pass@8": 0.9082678023063252, + "pass@16": 0.9229493914421676, + "pass@32": 0.9343577884724095, + "pass@64": 0.9438576549065015, + "pass@128": 0.9516282010749288 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9453585914454278, + "pass@2": 0.9535728491626602, + "pass@4": 0.95873854180838, + "pass@8": 0.9629986555885872, + "pass@16": 0.9679666317812168, + "pass@32": 0.9740341139349422, + "pass@64": 0.9803653794455086, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9971923828125, + "pass@2": 0.9978869340551182, + "pass@4": 0.9985996801181102, + "pass@8": 0.9993905663162155, + "pass@16": 0.9998951809167032, + "pass@32": 0.9999980558628433, + "pass@64": 0.9999999999667597, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9949900793650793, + "pass@2": 0.9973585333083365, + "pass@4": 0.9983299611358104, + "pass@8": 0.9991563945664312, + "pass@16": 0.9997971079638622, + "pass@32": 0.9999918808807343, + "pass@64": 0.9999999987391617, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9971229416167665, + "pass@2": 0.9976215138620398, + "pass@4": 0.9981272295117304, + "pass@8": 0.998847506991621, + "pass@16": 0.9995863571069055, + "pass@32": 0.999958801686582, + "pass@64": 0.9999999176094492, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9964866863905325, + "pass@2": 0.9979623363462704, + "pass@4": 0.9990334570789599, + "pass@8": 0.9997250598303445, + "pass@16": 0.9999790093720811, + "pass@32": 0.9999999405368915, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9981971153846154, + "pass@2": 0.9998284657026045, + "pass@4": 0.999995967751627, + "pass@8": 0.9999999975724242, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9949603873239437, + "pass@2": 0.9973813352556282, + "pass@4": 0.9982075685081618, + "pass@8": 0.9989154500291274, + "pass@16": 0.9996108261180917, + "pass@32": 0.9999612387699112, + "pass@64": 0.9999999224832565, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9636597025495751, + "pass@2": 0.9756313990319198, + "pass@4": 0.9833490821614721, + "pass@8": 0.9890875592461815, + "pass@16": 0.9928681364142148, + "pass@32": 0.9954215669121044, + "pass@64": 0.9969126772151184, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.9084348516949152, + "pass@2": 0.9271233011922239, + "pass@4": 0.9376102685257564, + "pass@8": 0.9448320171203957, + "pass@16": 0.9500880057183615, + "pass@32": 0.9533178342586844, + "pass@64": 0.9561329927476948, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.8727468652037618, + "pass@2": 0.9105426036087183, + "pass@4": 0.9320865086809287, + "pass@8": 0.9476093057668198, + "pass@16": 0.9599091371962115, + "pass@32": 0.9694213182908559, + "pass@64": 0.9768613994766613, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.8451129283489096, + "pass@2": 0.8927793926460124, + "pass@4": 0.9188406133812713, + "pass@8": 0.9352618975128104, + "pass@16": 0.9480306260712734, + "pass@32": 0.9589256381658317, + "pass@64": 0.9683198089214712, + "pass@128": 0.9750778816199377 + }, + "15": { + "pass@1": 0.7921107297687862, + "pass@2": 0.8578609303172365, + "pass@4": 0.8997546105147255, + "pass@8": 0.9264640318143393, + "pass@16": 0.9428917374738313, + "pass@32": 0.952393511786133, + "pass@64": 0.9574741921523455, + "pass@128": 0.9595375722543352 + }, + "16": { + "pass@1": 0.6568800403225806, + "pass@2": 0.7477683674117346, + "pass@4": 0.8092675075897764, + "pass@8": 0.8516551322611292, + "pass@16": 0.8812944796205525, + "pass@32": 0.902773745388599, + "pass@64": 0.9194705540558217, + "pass@128": 0.932258064516129 + }, + "17": { + "pass@1": 0.5657514450867052, + "pass@2": 0.6691021687679213, + "pass@4": 0.739191088198512, + "pass@8": 0.7856207032319886, + "pass@16": 0.8183619373072404, + "pass@32": 0.8430109049522699, + "pass@64": 0.8637531249758412, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.45412946428571427, + "pass@2": 0.5638899043869511, + "pass@4": 0.6530802185441107, + "pass@8": 0.720726478211569, + "pass@16": 0.7704510468029769, + "pass@32": 0.807289056099719, + "pass@64": 0.8378334239926626, + "pass@128": 0.8657142857142858 + }, + "19": { + "pass@1": 0.36369181681681684, + "pass@2": 0.4703841390603205, + "pass@4": 0.5581976540657639, + "pass@8": 0.627149829400672, + "pass@16": 0.6817550885633032, + "pass@32": 0.7263554189791901, + "pass@64": 0.763762224924364, + "pass@128": 0.7957957957957958 + }, + "20": { + "pass@1": 0.32453547297297297, + "pass@2": 0.4275736119978244, + "pass@4": 0.5169445100330928, + "pass@8": 0.585449717921024, + "pass@16": 0.6372318334285229, + "pass@32": 0.6821752409020172, + "pass@64": 0.7234653885399501, + "pass@128": 0.7537537537537538 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 677756, + "answer_accuracy": 0.8254043257989088, + "avg_response_len": 284.02363722720185, + "resp_tokens_sum": 233217489, + "pass_at_k": { + "pass@1": 0.8254043257989088, + "pass@2": 0.8633358853818162, + "pass@4": 0.8908206644619501, + "pass@8": 0.9108071320749088, + "pass@16": 0.9256376776351245, + "pass@32": 0.9371229822544132, + "pass@64": 0.9466032938973412, + "pass@128": 0.9547934528448948 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9309815950920245, + "pass@2": 0.9449852966040289, + "pass@4": 0.9559561028797782, + "pass@8": 0.9638793675635439, + "pass@16": 0.9687407182381751, + "pass@32": 0.9718304058076516, + "pass@64": 0.9746895660017418, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99375, + "pass@2": 0.9960658450302408, + "pass@4": 0.9975695505453124, + "pass@8": 0.9984974444758158, + "pass@16": 0.9992768151012691, + "pass@32": 0.9998558695467676, + "pass@64": 0.999998064534979, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9964183381088825, + "pass@2": 0.9970275026509938, + "pass@4": 0.9971312728100964, + "pass@8": 0.9971346683194743, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9973582008670521, + "pass@2": 0.9977495192526513, + "pass@4": 0.9982556172530457, + "pass@8": 0.9989655495774096, + "pass@16": 0.9996567447729243, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9936839788732394, + "pass@2": 0.9963724769879118, + "pass@4": 0.9971180363018003, + "pass@8": 0.9971826215324542, + "pass@16": 0.9971830985804359, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9924214071856288, + "pass@2": 0.9938524771323495, + "pass@4": 0.9940081237973997, + "pass@8": 0.9940119737766329, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9630082831325302, + "pass@2": 0.9713395906460487, + "pass@4": 0.9757507891521089, + "pass@8": 0.9784955531653872, + "pass@16": 0.9805383638724173, + "pass@32": 0.9824747306648515, + "pass@64": 0.9841054553733752, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9007885971786834, + "pass@2": 0.915418739046726, + "pass@4": 0.924029935129582, + "pass@8": 0.9303735376408099, + "pass@16": 0.9353211411280159, + "pass@32": 0.939039317249673, + "pass@64": 0.9419950990841026, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.8611913515406162, + "pass@2": 0.8988553590727634, + "pass@4": 0.918313749121695, + "pass@8": 0.9307861443620137, + "pass@16": 0.9400313356135247, + "pass@32": 0.9484692530915426, + "pass@64": 0.9581816741685208, + "pass@128": 0.969187675070028 + }, + "14": { + "pass@1": 0.8469709429824561, + "pass@2": 0.8980640886172121, + "pass@4": 0.9245809564374623, + "pass@8": 0.9405203926008627, + "pass@16": 0.9525469091473052, + "pass@32": 0.963199459193292, + "pass@64": 0.9716362530225532, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.7924715909090909, + "pass@2": 0.861036670842281, + "pass@4": 0.9029049320539471, + "pass@8": 0.9281268028368013, + "pass@16": 0.9426543876503707, + "pass@32": 0.9525476884583312, + "pass@64": 0.9611302965599587, + "pass@128": 0.9696969696969697 + }, + "16": { + "pass@1": 0.6614583333333334, + "pass@2": 0.757938160374184, + "pass@4": 0.8144365875205767, + "pass@8": 0.8475187224277707, + "pass@16": 0.8701097260916296, + "pass@32": 0.8890719227083439, + "pass@64": 0.9063072562149335, + "pass@128": 0.9173789173789174 + }, + "17": { + "pass@1": 0.5830260093167702, + "pass@2": 0.6846121986110436, + "pass@4": 0.7533166365656156, + "pass@8": 0.8007323980276663, + "pass@16": 0.8367999826994603, + "pass@32": 0.8654849657428961, + "pass@64": 0.8891736360856827, + "pass@128": 0.906832298136646 + }, + "18": { + "pass@1": 0.5104779411764706, + "pass@2": 0.615626447429365, + "pass@4": 0.6993681317408854, + "pass@8": 0.7636960252154447, + "pass@16": 0.8126314154686012, + "pass@32": 0.8493046683312366, + "pass@64": 0.8760405951252511, + "pass@128": 0.8970588235294118 + }, + "19": { + "pass@1": 0.33535621279761907, + "pass@2": 0.450107066694788, + "pass@4": 0.5589451405069903, + "pass@8": 0.6479033713249467, + "pass@16": 0.7114656294943115, + "pass@32": 0.7547122095604747, + "pass@64": 0.7871870069795173, + "pass@128": 0.8154761904761905 + }, + "20": { + "pass@1": 0.30761133982035926, + "pass@2": 0.4103107760856239, + "pass@4": 0.5020851449519407, + "pass@8": 0.577641649195555, + "pass@16": 0.640643393523646, + "pass@32": 0.6937003584791263, + "pass@64": 0.7400380698580312, + "pass@128": 0.7844311377245509 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 651496, + "answer_accuracy": 0.8131989934494328, + "avg_response_len": 304.374299758348, + "resp_tokens_sum": 243850079, + "pass_at_k": { + "pass@1": 0.8131989934494328, + "pass@2": 0.8516683416824594, + "pass@4": 0.8803690842367309, + "pass@8": 0.9015596188301824, + "pass@16": 0.9172829494592375, + "pass@32": 0.9293968706923023, + "pass@64": 0.9392539645433563, + "pass@128": 0.9475954625339511 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9517957089552239, + "pass@2": 0.9643014014572805, + "pass@4": 0.9723098293683435, + "pass@8": 0.9778178687535791, + "pass@16": 0.9818233351333638, + "pass@32": 0.9843700251647998, + "pass@64": 0.9862015157581675, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 0.9999553571428571, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998861151603499, + "pass@2": 0.99999784784555, + "pass@4": 0.9999999997267104, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9979669225146199, + "pass@2": 0.9986840666298292, + "pass@4": 0.9994179578978066, + "pass@8": 0.9998914828566827, + "pass@16": 0.999996971910131, + "pass@32": 0.9999999992108352, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9915670955882353, + "pass@2": 0.9952162459471977, + "pass@4": 0.996770337898939, + "pass@8": 0.9975063173915031, + "pass@16": 0.9980350056586529, + "pass@32": 0.9987689956690733, + "pass@64": 0.9996410375173692, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9964264589905363, + "pass@2": 0.9985430276459921, + "pass@4": 0.9995857823568582, + "pass@8": 0.9999535526784058, + "pass@16": 0.9999995379907698, + "pass@32": 0.9999999999901541, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972804588607594, + "pass@2": 0.9992676511262834, + "pass@4": 0.9999203209764919, + "pass@8": 0.9999990565168133, + "pass@16": 0.9999999999228011, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9907563813813813, + "pass@2": 0.99560153164976, + "pass@4": 0.998587598565945, + "pass@8": 0.9998129486244137, + "pass@16": 0.999996824080759, + "pass@32": 0.9999999996818268, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9906300241157556, + "pass@2": 0.9950913993467858, + "pass@4": 0.9970629562582811, + "pass@8": 0.9986055619182425, + "pass@16": 0.99964486782007, + "pass@32": 0.9999770906276318, + "pass@64": 0.9999999831714547, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9624007936507937, + "pass@2": 0.9748172103487065, + "pass@4": 0.981273297087864, + "pass@8": 0.9846499737188555, + "pass@16": 0.9862178333308695, + "pass@32": 0.9878187753676408, + "pass@64": 0.9897012873390828, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9143969801223242, + "pass@2": 0.9317818572563751, + "pass@4": 0.941430099907236, + "pass@8": 0.9493256684380422, + "pass@16": 0.9570131901299108, + "pass@32": 0.9641514897698327, + "pass@64": 0.9703591098246457, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8329957561728395, + "pass@2": 0.8691643579274818, + "pass@4": 0.8912119784216789, + "pass@8": 0.9091026647516887, + "pass@16": 0.9252978369245227, + "pass@32": 0.9383556719133557, + "pass@64": 0.9473490414410928, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8191533753709199, + "pass@2": 0.8682212201219658, + "pass@4": 0.8979430763847394, + "pass@8": 0.9180236634394026, + "pass@16": 0.9314134024114261, + "pass@32": 0.9406145618745281, + "pass@64": 0.9479964540280973, + "pass@128": 0.9554896142433235 + }, + "15": { + "pass@1": 0.7535927854938271, + "pass@2": 0.8146192554923689, + "pass@4": 0.8533767113022905, + "pass@8": 0.878865995821382, + "pass@16": 0.8964124781429048, + "pass@32": 0.910541254887923, + "pass@64": 0.922806998472301, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.6453032817109144, + "pass@2": 0.7468509017954611, + "pass@4": 0.8109872092868917, + "pass@8": 0.8503621602906531, + "pass@16": 0.8763865513376642, + "pass@32": 0.8958750335367089, + "pass@64": 0.9123815407896373, + "pass@128": 0.9262536873156342 + }, + "17": { + "pass@1": 0.5255553463855421, + "pass@2": 0.6341747402997818, + "pass@4": 0.7174640033700606, + "pass@8": 0.7752305050970983, + "pass@16": 0.8126617684952755, + "pass@32": 0.8371897052025431, + "pass@64": 0.8560389765187822, + "pass@128": 0.8734939759036144 + }, + "18": { + "pass@1": 0.4767389112903226, + "pass@2": 0.5847179006858011, + "pass@4": 0.66608676022352, + "pass@8": 0.7243191147240607, + "pass@16": 0.7663746277687996, + "pass@32": 0.8006317366757167, + "pass@64": 0.829606389878158, + "pass@128": 0.8516129032258064 + }, + "19": { + "pass@1": 0.30501793806646527, + "pass@2": 0.40979077955134763, + "pass@4": 0.5044869689627164, + "pass@8": 0.5830466910339542, + "pass@16": 0.6470709277754211, + "pass@32": 0.7023998688940181, + "pass@64": 0.7524830182120951, + "pass@128": 0.797583081570997 + }, + "20": { + "pass@1": 0.2891798048048048, + "pass@2": 0.3927472009410985, + "pass@4": 0.49233214992270086, + "pass@8": 0.5773029346421416, + "pass@16": 0.6449435379481746, + "pass@32": 0.6935367729692132, + "pass@64": 0.7276704804486359, + "pass@128": 0.7567567567567568 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block16-epoch4-op11-14/graphs/SDAR-100M-block16-epoch4-op11-14_score_report.jsonl b/composition/SDAR-100M-block16-epoch4-op11-14/graphs/SDAR-100M-block16-epoch4-op11-14_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e5e29c1a349d0c3b1d3fcc3fe1e7c67d9614be66 --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op11-14/graphs/SDAR-100M-block16-epoch4-op11-14_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block16-epoch4-op11-14", "results_dir": "results/composition/SDAR-100M-block16-epoch4-op11-14/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1016321, "value_mismatch": 12425, "dependency_mismatch": 131362, "answer_mismatch": 10863, "extra_nodes": 18014, "missing_nodes": 15676, "perfect_with_extra": 673, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337147, "dependency_mismatch": 40980, "value_mismatch": 3980, "answer_mismatch": 3264, "extra_nodes": 3845, "perfect_with_extra": 102, "missing_nodes": 3158}, "teachers_in_school": {"total": 390656, "perfect": 345288, "value_mismatch": 4377, "dependency_mismatch": 43888, "answer_mismatch": 4047, "extra_nodes": 4823, "missing_nodes": 4304, "perfect_with_extra": 178}, "movie_festival_awards": {"total": 382336, "perfect": 333886, "extra_nodes": 9346, "missing_nodes": 8214, "dependency_mismatch": 46494, "value_mismatch": 4068, "answer_mismatch": 3552, "perfect_with_extra": 393}}, "results_path": "results/composition/SDAR-100M-block16-epoch4-op11-14/graphs/SDAR-100M-block16-epoch4-op11-14_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 325369, "value_mismatch": 496232, "dependency_mismatch": 699787, "answer_mismatch": 425189, "extra_nodes": 445996, "missing_nodes": 609351, "perfect_with_extra": 148, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "dependency_mismatch": 235194, "extra_nodes": 123152, "perfect": 120774, "value_mismatch": 165863, "answer_mismatch": 139773, "missing_nodes": 183975, "perfect_with_extra": 51}, "movie_festival_awards": {"total": 418816, "perfect": 97173, "dependency_mismatch": 235153, "value_mismatch": 168752, "answer_mismatch": 146102, "extra_nodes": 168666, "missing_nodes": 216407, "perfect_with_extra": 83}, "teachers_in_school": {"total": 430464, "perfect": 107422, "extra_nodes": 154178, "missing_nodes": 208969, "value_mismatch": 161617, "answer_mismatch": 139314, "dependency_mismatch": 229440, "perfect_with_extra": 14}}, "results_path": "results/composition/SDAR-100M-block16-epoch4-op11-14/graphs/SDAR-100M-block16-epoch4-op11-14_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1341690, "value_mismatch": 508657, "dependency_mismatch": 831149, "answer_mismatch": 436052, "extra_nodes": 464010, "missing_nodes": 625027, "perfect_with_extra": 821, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block16-epoch4-op11-14/graphs/SDAR-100M-block16-epoch4-op11-14_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 457921, "dependency_mismatch": 276174, "value_mismatch": 169843, "answer_mismatch": 143037, "extra_nodes": 126997, "perfect_with_extra": 153, "missing_nodes": 187133}, "teachers_in_school": {"total": 821120, "perfect": 452710, "value_mismatch": 165994, "dependency_mismatch": 273328, "answer_mismatch": 143361, "extra_nodes": 159001, "missing_nodes": 213273, "perfect_with_extra": 192}, "movie_festival_awards": {"total": 801152, "perfect": 431059, "extra_nodes": 178012, "missing_nodes": 224621, "dependency_mismatch": 281647, "value_mismatch": 172820, "answer_mismatch": 149654, "perfect_with_extra": 476}}}} diff --git a/composition/SDAR-100M-block16-epoch4-op11-14/graphs/results.json b/composition/SDAR-100M-block16-epoch4-op11-14/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..7af212c7b241d0917573f69dcafbff1a104a8e1f --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op11-14/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block16-epoch4-op11-14_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9875, + "pass@2": 0.99375, + "pass@4": 0.996875, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9492063492063492, + "pass@2": 0.9809523809523809, + "pass@4": 0.9873015873015873, + "pass@8": 0.9968253968253968, + "pass@16": 0.9968253968253968, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9520958083832335, + "pass@2": 0.9730538922155688, + "pass@4": 0.9880239520958084, + "pass@8": 0.9940119760479041, + "pass@16": 0.9970059880239521, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8875739644970414, + "pass@2": 0.9349112426035503, + "pass@4": 0.9585798816568047, + "pass@8": 0.9792899408284024, + "pass@16": 0.9940828402366864, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8365384615384616, + "pass@2": 0.8782051282051282, + "pass@4": 0.9391025641025641, + "pass@8": 0.9903846153846154, + "pass@16": 0.9967948717948718, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7661971830985915, + "pass@2": 0.8507042253521127, + "pass@4": 0.9014084507042254, + "pass@8": 0.9661971830985916, + "pass@16": 0.9887323943661972, + "pass@32": 0.9943661971830986, + "pass@64": 0.9971830985915493, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6342182890855457, + "pass@2": 0.7227138643067846, + "pass@4": 0.8053097345132744, + "pass@8": 0.8790560471976401, + "pass@16": 0.911504424778761, + "pass@32": 0.9203539823008849, + "pass@64": 0.9321533923303835, + "pass@128": 0.9380530973451328 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9875, + "5": 0.9492063492063492, + "6": 0.9520958083832335, + "7": 0.8875739644970414, + "8": 0.8365384615384616, + "9": 0.7661971830985915, + "10": 0.6342182890855457 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9809523809523809, + "6": 0.9730538922155688, + "7": 0.9349112426035503, + "8": 0.8782051282051282, + "9": 0.8507042253521127, + "10": 0.7227138643067846 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9873015873015873, + "6": 0.9880239520958084, + "7": 0.9585798816568047, + "8": 0.9391025641025641, + "9": 0.9014084507042254, + "10": 0.8053097345132744 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 0.9940119760479041, + "7": 0.9792899408284024, + "8": 0.9903846153846154, + "9": 0.9661971830985916, + "10": 0.8790560471976401 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 0.9970059880239521, + "7": 0.9940828402366864, + "8": 0.9967948717948718, + "9": 0.9887323943661972, + "10": 0.911504424778761 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9943661971830986, + "10": 0.9203539823008849 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9321533923303835 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9380530973451328 + } + }, + "total_pass_at_k": { + "pass@1": 0.8878757176629517, + "pass@2": 0.9243498817966903, + "pass@4": 0.9517055048969942, + "pass@8": 0.977710233029382, + "pass@16": 0.9868287740628167, + "pass@32": 0.9902060114826072, + "pass@64": 0.9918946301925026, + "pass@128": 0.9929078014184397 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9942857142857143, + "pass@2": 0.9971428571428571, + "pass@4": 0.9971428571428571, + "pass@8": 0.9971428571428571, + "pass@16": 0.9971428571428571, + "pass@32": 0.9971428571428571, + "pass@64": 0.9971428571428571, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9941690962099126, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9824561403508771, + "pass@2": 0.9941520467836257, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9352941176470588, + "pass@2": 0.9735294117647059, + "pass@4": 0.9882352941176471, + "pass@8": 0.9941176470588236, + "pass@16": 0.9970588235294118, + "pass@32": 0.9970588235294118, + "pass@64": 0.9970588235294118, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9495268138801262, + "pass@2": 0.9810725552050473, + "pass@4": 0.9905362776025236, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8481012658227848, + "pass@2": 0.9050632911392406, + "pass@4": 0.9746835443037974, + "pass@8": 0.9936708860759493, + "pass@16": 0.9968354430379747, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7657657657657657, + "pass@2": 0.8378378378378378, + "pass@4": 0.924924924924925, + "pass@8": 0.9669669669669669, + "pass@16": 0.996996996996997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7620578778135049, + "pass@2": 0.8263665594855305, + "pass@4": 0.9003215434083601, + "pass@8": 0.9485530546623794, + "pass@16": 0.9903536977491961, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.5880597014925373, + "pass@2": 0.6776119402985075, + "pass@4": 0.7701492537313432, + "pass@8": 0.8656716417910447, + "pass@16": 0.9164179104477612, + "pass@32": 0.9373134328358209, + "pass@64": 0.9432835820895522, + "pass@128": 0.9522388059701492 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9942857142857143, + "3": 0.9941690962099126, + "4": 0.9824561403508771, + "5": 0.9352941176470588, + "6": 0.9495268138801262, + "7": 0.8481012658227848, + "8": 0.7657657657657657, + "9": 0.7620578778135049, + "10": 0.5880597014925373 + }, + "pass@2": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9735294117647059, + "6": 0.9810725552050473, + "7": 0.9050632911392406, + "8": 0.8378378378378378, + "9": 0.8263665594855305, + "10": 0.6776119402985075 + }, + "pass@4": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 1.0, + "5": 0.9882352941176471, + "6": 0.9905362776025236, + "7": 0.9746835443037974, + "8": 0.924924924924925, + "9": 0.9003215434083601, + "10": 0.7701492537313432 + }, + "pass@8": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 1.0, + "5": 0.9941176470588236, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.9669669669669669, + "9": 0.9485530546623794, + "10": 0.8656716417910447 + }, + "pass@16": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.996996996996997, + "9": 0.9903536977491961, + "10": 0.9164179104477612 + }, + "pass@32": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9373134328358209 + }, + "pass@64": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9432835820895522 + }, + "pass@128": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9522388059701492 + } + }, + "total_pass_at_k": { + "pass@1": 0.8707733511884834, + "pass@2": 0.9116170070304653, + "pass@4": 0.9501171744224974, + "pass@8": 0.9742216270505524, + "pass@16": 0.9882825577502511, + "pass@32": 0.9922999665215936, + "pass@64": 0.9929695346501507, + "pass@128": 0.9943086709072648 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9733727810650887, + "pass@2": 0.9792899408284024, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9594202898550724, + "pass@2": 0.9768115942028985, + "pass@4": 0.9855072463768116, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9742120343839542, + "pass@2": 0.9856733524355301, + "pass@4": 0.994269340974212, + "pass@8": 0.994269340974212, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.8583815028901735, + "pass@2": 0.9075144508670521, + "pass@4": 0.9508670520231214, + "pass@8": 0.9826589595375722, + "pass@16": 0.9971098265895953, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8056338028169014, + "pass@2": 0.8535211267605634, + "pass@4": 0.9323943661971831, + "pass@8": 0.9746478873239437, + "pass@16": 0.9887323943661972, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.7245508982035929, + "pass@2": 0.7994011976047904, + "pass@4": 0.8712574850299402, + "pass@8": 0.9311377245508982, + "pass@16": 0.9670658682634731, + "pass@32": 0.9850299401197605, + "pass@64": 0.9910179640718563, + "pass@128": 0.9940119760479041 + }, + "10": { + "pass@1": 0.647239263803681, + "pass@2": 0.6932515337423313, + "pass@4": 0.7760736196319018, + "pass@8": 0.843558282208589, + "pass@16": 0.8926380368098159, + "pass@32": 0.911042944785276, + "pass@64": 0.9355828220858896, + "pass@128": 0.950920245398773 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9733727810650887, + "5": 0.9594202898550724, + "6": 0.9742120343839542, + "7": 0.8583815028901735, + "8": 0.8056338028169014, + "9": 0.7245508982035929, + "10": 0.647239263803681 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9792899408284024, + "5": 0.9768115942028985, + "6": 0.9856733524355301, + "7": 0.9075144508670521, + "8": 0.8535211267605634, + "9": 0.7994011976047904, + "10": 0.6932515337423313 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9855072463768116, + "6": 0.994269340974212, + "7": 0.9508670520231214, + "8": 0.9323943661971831, + "9": 0.8712574850299402, + "10": 0.7760736196319018 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.994269340974212, + "7": 0.9826589595375722, + "8": 0.9746478873239437, + "9": 0.9311377245508982, + "10": 0.843558282208589 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9971098265895953, + "8": 0.9887323943661972, + "9": 0.9670658682634731, + "10": 0.8926380368098159 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9850299401197605, + "10": 0.911042944785276 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9910179640718563, + "10": 0.9355828220858896 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9940119760479041, + "10": 0.950920245398773 + } + }, + "total_pass_at_k": { + "pass@1": 0.8830275229357798, + "pass@2": 0.9112057667103539, + "pass@4": 0.9462647444298821, + "pass@8": 0.9701834862385321, + "pass@16": 0.9829619921363041, + "pass@32": 0.9882044560943644, + "pass@64": 0.991480996068152, + "pass@128": 0.9934469200524246 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.998, + "pass@2": 0.999, + "pass@4": 0.999, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.998, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.981, + "pass@2": 0.989, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.948, + "pass@2": 0.977, + "pass@4": 0.987, + "pass@8": 0.997, + "pass@16": 0.998, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.959, + "pass@2": 0.98, + "pass@4": 0.991, + "pass@8": 0.996, + "pass@16": 0.998, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.865, + "pass@2": 0.916, + "pass@4": 0.961, + "pass@8": 0.985, + "pass@16": 0.996, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.802, + "pass@2": 0.856, + "pass@4": 0.932, + "pass@8": 0.977, + "pass@16": 0.994, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.751, + "pass@2": 0.826, + "pass@4": 0.891, + "pass@8": 0.949, + "pass@16": 0.982, + "pass@32": 0.993, + "pass@64": 0.996, + "pass@128": 0.998 + }, + "10": { + "pass@1": 0.623, + "pass@2": 0.698, + "pass@4": 0.784, + "pass@8": 0.863, + "pass@16": 0.907, + "pass@32": 0.923, + "pass@64": 0.937, + "pass@128": 0.947 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.998, + "3": 0.998, + "4": 0.981, + "5": 0.948, + "6": 0.959, + "7": 0.865, + "8": 0.802, + "9": 0.751, + "10": 0.623 + }, + "pass@2": { + "2": 0.999, + "3": 1.0, + "4": 0.989, + "5": 0.977, + "6": 0.98, + "7": 0.916, + "8": 0.856, + "9": 0.826, + "10": 0.698 + }, + "pass@4": { + "2": 0.999, + "3": 1.0, + "4": 0.999, + "5": 0.987, + "6": 0.991, + "7": 0.961, + "8": 0.932, + "9": 0.891, + "10": 0.784 + }, + "pass@8": { + "2": 0.999, + "3": 1.0, + "4": 1.0, + "5": 0.997, + "6": 0.996, + "7": 0.985, + "8": 0.977, + "9": 0.949, + "10": 0.863 + }, + "pass@16": { + "2": 0.999, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 0.998, + "7": 0.996, + "8": 0.994, + "9": 0.982, + "10": 0.907 + }, + "pass@32": { + "2": 0.999, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 0.999, + "7": 1.0, + "8": 0.999, + "9": 0.993, + "10": 0.923 + }, + "pass@64": { + "2": 0.999, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 0.999, + "7": 1.0, + "8": 0.999, + "9": 0.996, + "10": 0.937 + }, + "pass@128": { + "2": 0.999, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.999, + "7": 1.0, + "8": 0.999, + "9": 0.998, + "10": 0.947 + } + }, + "total_pass_at_k": { + "pass@1": 0.8805555555555555, + "pass@2": 0.9156666666666666, + "pass@4": 0.9493333333333334, + "pass@8": 0.974, + "pass@16": 0.986, + "pass@32": 0.9902222222222222, + "pass@64": 0.9921111111111112, + "pass@128": 0.9935555555555555 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block16-epoch4-op11-14_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5807365439093485, + "pass@2": 0.6742209631728046, + "pass@4": 0.7507082152974505, + "pass@8": 0.8413597733711048, + "pass@16": 0.8895184135977338, + "pass@32": 0.9206798866855525, + "pass@64": 0.9291784702549575, + "pass@128": 0.9461756373937678 + }, + "12": { + "pass@1": 0.6045197740112994, + "pass@2": 0.6638418079096046, + "pass@4": 0.7429378531073446, + "pass@8": 0.788135593220339, + "pass@16": 0.8248587570621468, + "pass@32": 0.8587570621468926, + "pass@64": 0.8728813559322034, + "pass@128": 0.8785310734463276 + }, + "13": { + "pass@1": 0.48589341692789967, + "pass@2": 0.542319749216301, + "pass@4": 0.6300940438871473, + "pass@8": 0.7178683385579937, + "pass@16": 0.7617554858934169, + "pass@32": 0.7962382445141066, + "pass@64": 0.8275862068965517, + "pass@128": 0.8338557993730408 + }, + "14": { + "pass@1": 0.40498442367601245, + "pass@2": 0.48598130841121495, + "pass@4": 0.5389408099688473, + "pass@8": 0.6074766355140186, + "pass@16": 0.67601246105919, + "pass@32": 0.7570093457943925, + "pass@64": 0.8099688473520249, + "pass@128": 0.8255451713395638 + }, + "15": { + "pass@1": 0.3352601156069364, + "pass@2": 0.3959537572254335, + "pass@4": 0.45664739884393063, + "pass@8": 0.5375722543352601, + "pass@16": 0.6184971098265896, + "pass@32": 0.6820809248554913, + "pass@64": 0.7514450867052023, + "pass@128": 0.8265895953757225 + }, + "16": { + "pass@1": 0.14516129032258066, + "pass@2": 0.1935483870967742, + "pass@4": 0.23548387096774193, + "pass@8": 0.2967741935483871, + "pass@16": 0.36451612903225805, + "pass@32": 0.43548387096774194, + "pass@64": 0.49032258064516127, + "pass@128": 0.5483870967741935 + }, + "17": { + "pass@1": 0.11271676300578035, + "pass@2": 0.15895953757225434, + "pass@4": 0.21098265895953758, + "pass@8": 0.26878612716763006, + "pass@16": 0.3179190751445087, + "pass@32": 0.3786127167630058, + "pass@64": 0.4421965317919075, + "pass@128": 0.4913294797687861 + }, + "18": { + "pass@1": 0.07428571428571429, + "pass@2": 0.1, + "pass@4": 0.15428571428571428, + "pass@8": 0.20857142857142857, + "pass@16": 0.24857142857142858, + "pass@32": 0.28285714285714286, + "pass@64": 0.32857142857142857, + "pass@128": 0.3942857142857143 + }, + "19": { + "pass@1": 0.024024024024024024, + "pass@2": 0.03003003003003003, + "pass@4": 0.036036036036036036, + "pass@8": 0.057057057057057055, + "pass@16": 0.08108108108108109, + "pass@32": 0.11711711711711711, + "pass@64": 0.15615615615615616, + "pass@128": 0.17717717717717718 + }, + "20": { + "pass@1": 0.015015015015015015, + "pass@2": 0.036036036036036036, + "pass@4": 0.05405405405405406, + "pass@8": 0.07207207207207207, + "pass@16": 0.0990990990990991, + "pass@32": 0.11711711711711711, + "pass@64": 0.14714714714714713, + "pass@128": 0.15915915915915915 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5807365439093485, + "12": 0.6045197740112994, + "13": 0.48589341692789967, + "14": 0.40498442367601245, + "15": 0.3352601156069364, + "16": 0.14516129032258066, + "17": 0.11271676300578035, + "18": 0.07428571428571429, + "19": 0.024024024024024024, + "20": 0.015015015015015015 + }, + "pass@2": { + "11": 0.6742209631728046, + "12": 0.6638418079096046, + "13": 0.542319749216301, + "14": 0.48598130841121495, + "15": 0.3959537572254335, + "16": 0.1935483870967742, + "17": 0.15895953757225434, + "18": 0.1, + "19": 0.03003003003003003, + "20": 0.036036036036036036 + }, + "pass@4": { + "11": 0.7507082152974505, + "12": 0.7429378531073446, + "13": 0.6300940438871473, + "14": 0.5389408099688473, + "15": 0.45664739884393063, + "16": 0.23548387096774193, + "17": 0.21098265895953758, + "18": 0.15428571428571428, + "19": 0.036036036036036036, + "20": 0.05405405405405406 + }, + "pass@8": { + "11": 0.8413597733711048, + "12": 0.788135593220339, + "13": 0.7178683385579937, + "14": 0.6074766355140186, + "15": 0.5375722543352601, + "16": 0.2967741935483871, + "17": 0.26878612716763006, + "18": 0.20857142857142857, + "19": 0.057057057057057055, + "20": 0.07207207207207207 + }, + "pass@16": { + "11": 0.8895184135977338, + "12": 0.8248587570621468, + "13": 0.7617554858934169, + "14": 0.67601246105919, + "15": 0.6184971098265896, + "16": 0.36451612903225805, + "17": 0.3179190751445087, + "18": 0.24857142857142858, + "19": 0.08108108108108109, + "20": 0.0990990990990991 + }, + "pass@32": { + "11": 0.9206798866855525, + "12": 0.8587570621468926, + "13": 0.7962382445141066, + "14": 0.7570093457943925, + "15": 0.6820809248554913, + "16": 0.43548387096774194, + "17": 0.3786127167630058, + "18": 0.28285714285714286, + "19": 0.11711711711711711, + "20": 0.11711711711711711 + }, + "pass@64": { + "11": 0.9291784702549575, + "12": 0.8728813559322034, + "13": 0.8275862068965517, + "14": 0.8099688473520249, + "15": 0.7514450867052023, + "16": 0.49032258064516127, + "17": 0.4421965317919075, + "18": 0.32857142857142857, + "19": 0.15615615615615616, + "20": 0.14714714714714713 + }, + "pass@128": { + "11": 0.9461756373937678, + "12": 0.8785310734463276, + "13": 0.8338557993730408, + "14": 0.8255451713395638, + "15": 0.8265895953757225, + "16": 0.5483870967741935, + "17": 0.4913294797687861, + "18": 0.3942857142857143, + "19": 0.17717717717717718, + "20": 0.15915915915915915 + } + }, + "total_pass_at_k": { + "pass@1": 0.2802377414561664, + "pass@2": 0.3301634472511144, + "pass@4": 0.3833580980683507, + "pass@8": 0.44190193164933134, + "pass@16": 0.4903417533432392, + "pass@32": 0.5364041604754829, + "pass@64": 0.5771173848439821, + "pass@128": 0.6098068350668648 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5587301587301587, + "pass@2": 0.6476190476190476, + "pass@4": 0.7555555555555555, + "pass@8": 0.834920634920635, + "pass@16": 0.9079365079365079, + "pass@32": 0.946031746031746, + "pass@64": 0.9650793650793651, + "pass@128": 0.9714285714285714 + }, + "12": { + "pass@1": 0.5351681957186545, + "pass@2": 0.6574923547400612, + "pass@4": 0.709480122324159, + "pass@8": 0.7981651376146789, + "pass@16": 0.8593272171253823, + "pass@32": 0.8715596330275229, + "pass@64": 0.8868501529051988, + "pass@128": 0.8990825688073395 + }, + "13": { + "pass@1": 0.38580246913580246, + "pass@2": 0.48148148148148145, + "pass@4": 0.5802469135802469, + "pass@8": 0.6512345679012346, + "pass@16": 0.7129629629629629, + "pass@32": 0.7407407407407407, + "pass@64": 0.7716049382716049, + "pass@128": 0.7932098765432098 + }, + "14": { + "pass@1": 0.3353115727002967, + "pass@2": 0.41839762611275966, + "pass@4": 0.5014836795252225, + "pass@8": 0.5905044510385756, + "pass@16": 0.6646884272997032, + "pass@32": 0.7270029673590505, + "pass@64": 0.7685459940652819, + "pass@128": 0.8100890207715133 + }, + "15": { + "pass@1": 0.25617283950617287, + "pass@2": 0.32098765432098764, + "pass@4": 0.39197530864197533, + "pass@8": 0.48148148148148145, + "pass@16": 0.5308641975308642, + "pass@32": 0.6049382716049383, + "pass@64": 0.7067901234567902, + "pass@128": 0.7469135802469136 + }, + "16": { + "pass@1": 0.07964601769911504, + "pass@2": 0.12389380530973451, + "pass@4": 0.1976401179941003, + "pass@8": 0.2536873156342183, + "pass@16": 0.30973451327433627, + "pass@32": 0.3687315634218289, + "pass@64": 0.4336283185840708, + "pass@128": 0.49557522123893805 + }, + "17": { + "pass@1": 0.0572289156626506, + "pass@2": 0.0963855421686747, + "pass@4": 0.13855421686746988, + "pass@8": 0.1897590361445783, + "pass@16": 0.23493975903614459, + "pass@32": 0.30120481927710846, + "pass@64": 0.3433734939759036, + "pass@128": 0.3885542168674699 + }, + "18": { + "pass@1": 0.06451612903225806, + "pass@2": 0.11290322580645161, + "pass@4": 0.13870967741935483, + "pass@8": 0.18387096774193548, + "pass@16": 0.22580645161290322, + "pass@32": 0.267741935483871, + "pass@64": 0.3, + "pass@128": 0.3580645161290323 + }, + "19": { + "pass@1": 0.00906344410876133, + "pass@2": 0.03323262839879154, + "pass@4": 0.04229607250755287, + "pass@8": 0.054380664652567974, + "pass@16": 0.0755287009063444, + "pass@32": 0.09063444108761329, + "pass@64": 0.1027190332326284, + "pass@128": 0.12084592145015106 + }, + "20": { + "pass@1": 0.012012012012012012, + "pass@2": 0.021021021021021023, + "pass@4": 0.042042042042042045, + "pass@8": 0.057057057057057055, + "pass@16": 0.07207207207207207, + "pass@32": 0.08708708708708708, + "pass@64": 0.1111111111111111, + "pass@128": 0.12912912912912913 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5587301587301587, + "12": 0.5351681957186545, + "13": 0.38580246913580246, + "14": 0.3353115727002967, + "15": 0.25617283950617287, + "16": 0.07964601769911504, + "17": 0.0572289156626506, + "18": 0.06451612903225806, + "19": 0.00906344410876133, + "20": 0.012012012012012012 + }, + "pass@2": { + "11": 0.6476190476190476, + "12": 0.6574923547400612, + "13": 0.48148148148148145, + "14": 0.41839762611275966, + "15": 0.32098765432098764, + "16": 0.12389380530973451, + "17": 0.0963855421686747, + "18": 0.11290322580645161, + "19": 0.03323262839879154, + "20": 0.021021021021021023 + }, + "pass@4": { + "11": 0.7555555555555555, + "12": 0.709480122324159, + "13": 0.5802469135802469, + "14": 0.5014836795252225, + "15": 0.39197530864197533, + "16": 0.1976401179941003, + "17": 0.13855421686746988, + "18": 0.13870967741935483, + "19": 0.04229607250755287, + "20": 0.042042042042042045 + }, + "pass@8": { + "11": 0.834920634920635, + "12": 0.7981651376146789, + "13": 0.6512345679012346, + "14": 0.5905044510385756, + "15": 0.48148148148148145, + "16": 0.2536873156342183, + "17": 0.1897590361445783, + "18": 0.18387096774193548, + "19": 0.054380664652567974, + "20": 0.057057057057057055 + }, + "pass@16": { + "11": 0.9079365079365079, + "12": 0.8593272171253823, + "13": 0.7129629629629629, + "14": 0.6646884272997032, + "15": 0.5308641975308642, + "16": 0.30973451327433627, + "17": 0.23493975903614459, + "18": 0.22580645161290322, + "19": 0.0755287009063444, + "20": 0.07207207207207207 + }, + "pass@32": { + "11": 0.946031746031746, + "12": 0.8715596330275229, + "13": 0.7407407407407407, + "14": 0.7270029673590505, + "15": 0.6049382716049383, + "16": 0.3687315634218289, + "17": 0.30120481927710846, + "18": 0.267741935483871, + "19": 0.09063444108761329, + "20": 0.08708708708708708 + }, + "pass@64": { + "11": 0.9650793650793651, + "12": 0.8868501529051988, + "13": 0.7716049382716049, + "14": 0.7685459940652819, + "15": 0.7067901234567902, + "16": 0.4336283185840708, + "17": 0.3433734939759036, + "18": 0.3, + "19": 0.1027190332326284, + "20": 0.1111111111111111 + }, + "pass@128": { + "11": 0.9714285714285714, + "12": 0.8990825688073395, + "13": 0.7932098765432098, + "14": 0.8100890207715133, + "15": 0.7469135802469136, + "16": 0.49557522123893805, + "17": 0.3885542168674699, + "18": 0.3580645161290323, + "19": 0.12084592145015106, + "20": 0.12912912912912913 + } + }, + "total_pass_at_k": { + "pass@1": 0.22768948655256724, + "pass@2": 0.28942542787286063, + "pass@4": 0.347799511002445, + "pass@8": 0.4073960880195599, + "pass@16": 0.4572127139364303, + "pass@32": 0.4984718826405868, + "pass@64": 0.5369804400977995, + "pass@128": 0.5693765281173594 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6445783132530121, + "pass@2": 0.7168674698795181, + "pass@4": 0.8132530120481928, + "pass@8": 0.8795180722891566, + "pass@16": 0.9367469879518072, + "pass@32": 0.9487951807228916, + "pass@64": 0.963855421686747, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.5266457680250783, + "pass@2": 0.5987460815047022, + "pass@4": 0.6677115987460815, + "pass@8": 0.7523510971786834, + "pass@16": 0.8025078369905956, + "pass@32": 0.8275862068965517, + "pass@64": 0.8557993730407524, + "pass@128": 0.8714733542319749 + }, + "13": { + "pass@1": 0.47619047619047616, + "pass@2": 0.5686274509803921, + "pass@4": 0.6386554621848739, + "pass@8": 0.7142857142857143, + "pass@16": 0.7619047619047619, + "pass@32": 0.7955182072829131, + "pass@64": 0.8263305322128851, + "pass@128": 0.8515406162464986 + }, + "14": { + "pass@1": 0.31871345029239767, + "pass@2": 0.4152046783625731, + "pass@4": 0.5058479532163743, + "pass@8": 0.5526315789473685, + "pass@16": 0.6549707602339181, + "pass@32": 0.7222222222222222, + "pass@64": 0.7690058479532164, + "pass@128": 0.8216374269005848 + }, + "15": { + "pass@1": 0.25757575757575757, + "pass@2": 0.36363636363636365, + "pass@4": 0.4636363636363636, + "pass@8": 0.5303030303030303, + "pass@16": 0.5969696969696969, + "pass@32": 0.6606060606060606, + "pass@64": 0.7212121212121212, + "pass@128": 0.7848484848484848 + }, + "16": { + "pass@1": 0.09971509971509972, + "pass@2": 0.13675213675213677, + "pass@4": 0.20512820512820512, + "pass@8": 0.2678062678062678, + "pass@16": 0.32193732193732194, + "pass@32": 0.37037037037037035, + "pass@64": 0.42735042735042733, + "pass@128": 0.5042735042735043 + }, + "17": { + "pass@1": 0.08385093167701864, + "pass@2": 0.12111801242236025, + "pass@4": 0.15527950310559005, + "pass@8": 0.18944099378881987, + "pass@16": 0.2453416149068323, + "pass@32": 0.3167701863354037, + "pass@64": 0.36335403726708076, + "pass@128": 0.43788819875776397 + }, + "18": { + "pass@1": 0.03823529411764706, + "pass@2": 0.07941176470588235, + "pass@4": 0.09705882352941177, + "pass@8": 0.13529411764705881, + "pass@16": 0.19117647058823528, + "pass@32": 0.25588235294117645, + "pass@64": 0.32941176470588235, + "pass@128": 0.37941176470588234 + }, + "19": { + "pass@1": 0.01488095238095238, + "pass@2": 0.03273809523809524, + "pass@4": 0.03869047619047619, + "pass@8": 0.047619047619047616, + "pass@16": 0.0625, + "pass@32": 0.08928571428571429, + "pass@64": 0.11607142857142858, + "pass@128": 0.12797619047619047 + }, + "20": { + "pass@1": 0.0029940119760479044, + "pass@2": 0.005988023952095809, + "pass@4": 0.008982035928143712, + "pass@8": 0.041916167664670656, + "pass@16": 0.05688622754491018, + "pass@32": 0.08982035928143713, + "pass@64": 0.11676646706586827, + "pass@128": 0.1377245508982036 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6445783132530121, + "12": 0.5266457680250783, + "13": 0.47619047619047616, + "14": 0.31871345029239767, + "15": 0.25757575757575757, + "16": 0.09971509971509972, + "17": 0.08385093167701864, + "18": 0.03823529411764706, + "19": 0.01488095238095238, + "20": 0.0029940119760479044 + }, + "pass@2": { + "11": 0.7168674698795181, + "12": 0.5987460815047022, + "13": 0.5686274509803921, + "14": 0.4152046783625731, + "15": 0.36363636363636365, + "16": 0.13675213675213677, + "17": 0.12111801242236025, + "18": 0.07941176470588235, + "19": 0.03273809523809524, + "20": 0.005988023952095809 + }, + "pass@4": { + "11": 0.8132530120481928, + "12": 0.6677115987460815, + "13": 0.6386554621848739, + "14": 0.5058479532163743, + "15": 0.4636363636363636, + "16": 0.20512820512820512, + "17": 0.15527950310559005, + "18": 0.09705882352941177, + "19": 0.03869047619047619, + "20": 0.008982035928143712 + }, + "pass@8": { + "11": 0.8795180722891566, + "12": 0.7523510971786834, + "13": 0.7142857142857143, + "14": 0.5526315789473685, + "15": 0.5303030303030303, + "16": 0.2678062678062678, + "17": 0.18944099378881987, + "18": 0.13529411764705881, + "19": 0.047619047619047616, + "20": 0.041916167664670656 + }, + "pass@16": { + "11": 0.9367469879518072, + "12": 0.8025078369905956, + "13": 0.7619047619047619, + "14": 0.6549707602339181, + "15": 0.5969696969696969, + "16": 0.32193732193732194, + "17": 0.2453416149068323, + "18": 0.19117647058823528, + "19": 0.0625, + "20": 0.05688622754491018 + }, + "pass@32": { + "11": 0.9487951807228916, + "12": 0.8275862068965517, + "13": 0.7955182072829131, + "14": 0.7222222222222222, + "15": 0.6606060606060606, + "16": 0.37037037037037035, + "17": 0.3167701863354037, + "18": 0.25588235294117645, + "19": 0.08928571428571429, + "20": 0.08982035928143713 + }, + "pass@64": { + "11": 0.963855421686747, + "12": 0.8557993730407524, + "13": 0.8263305322128851, + "14": 0.7690058479532164, + "15": 0.7212121212121212, + "16": 0.42735042735042733, + "17": 0.36335403726708076, + "18": 0.32941176470588235, + "19": 0.11607142857142858, + "20": 0.11676646706586827 + }, + "pass@128": { + "11": 0.9698795180722891, + "12": 0.8714733542319749, + "13": 0.8515406162464986, + "14": 0.8216374269005848, + "15": 0.7848484848484848, + "16": 0.5042735042735043, + "17": 0.43788819875776397, + "18": 0.37941176470588234, + "19": 0.12797619047619047, + "20": 0.1377245508982036 + } + }, + "total_pass_at_k": { + "pass@1": 0.24591138864109427, + "pass@2": 0.30359797799583704, + "pass@4": 0.3592030924769551, + "pass@8": 0.41094261076419863, + "pass@16": 0.46297948260481714, + "pass@32": 0.5075825156110616, + "pass@64": 0.5489146595301814, + "pass@128": 0.5887600356824264 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.595, + "pass@2": 0.68, + "pass@4": 0.773, + "pass@8": 0.852, + "pass@16": 0.911, + "pass@32": 0.938, + "pass@64": 0.952, + "pass@128": 0.962 + }, + "12": { + "pass@1": 0.557, + "pass@2": 0.641, + "pass@4": 0.708, + "pass@8": 0.78, + "pass@16": 0.829, + "pass@32": 0.853, + "pass@64": 0.872, + "pass@128": 0.883 + }, + "13": { + "pass@1": 0.45, + "pass@2": 0.532, + "pass@4": 0.617, + "pass@8": 0.695, + "pass@16": 0.746, + "pass@32": 0.778, + "pass@64": 0.809, + "pass@128": 0.827 + }, + "14": { + "pass@1": 0.352, + "pass@2": 0.439, + "pass@4": 0.515, + "pass@8": 0.583, + "pass@16": 0.665, + "pass@32": 0.735, + "pass@64": 0.782, + "pass@128": 0.819 + }, + "15": { + "pass@1": 0.284, + "pass@2": 0.361, + "pass@4": 0.438, + "pass@8": 0.517, + "pass@16": 0.583, + "pass@32": 0.65, + "pass@64": 0.727, + "pass@128": 0.787 + }, + "16": { + "pass@1": 0.107, + "pass@2": 0.15, + "pass@4": 0.212, + "pass@8": 0.272, + "pass@16": 0.331, + "pass@32": 0.39, + "pass@64": 0.449, + "pass@128": 0.515 + }, + "17": { + "pass@1": 0.085, + "pass@2": 0.126, + "pass@4": 0.169, + "pass@8": 0.217, + "pass@16": 0.267, + "pass@32": 0.333, + "pass@64": 0.384, + "pass@128": 0.44 + }, + "18": { + "pass@1": 0.059, + "pass@2": 0.097, + "pass@4": 0.13, + "pass@8": 0.176, + "pass@16": 0.222, + "pass@32": 0.269, + "pass@64": 0.32, + "pass@128": 0.378 + }, + "19": { + "pass@1": 0.016, + "pass@2": 0.032, + "pass@4": 0.039, + "pass@8": 0.053, + "pass@16": 0.073, + "pass@32": 0.099, + "pass@64": 0.125, + "pass@128": 0.142 + }, + "20": { + "pass@1": 0.01, + "pass@2": 0.021, + "pass@4": 0.035, + "pass@8": 0.057, + "pass@16": 0.076, + "pass@32": 0.098, + "pass@64": 0.125, + "pass@128": 0.142 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.595, + "12": 0.557, + "13": 0.45, + "14": 0.352, + "15": 0.284, + "16": 0.107, + "17": 0.085, + "18": 0.059, + "19": 0.016, + "20": 0.01 + }, + "pass@2": { + "11": 0.68, + "12": 0.641, + "13": 0.532, + "14": 0.439, + "15": 0.361, + "16": 0.15, + "17": 0.126, + "18": 0.097, + "19": 0.032, + "20": 0.021 + }, + "pass@4": { + "11": 0.773, + "12": 0.708, + "13": 0.617, + "14": 0.515, + "15": 0.438, + "16": 0.212, + "17": 0.169, + "18": 0.13, + "19": 0.039, + "20": 0.035 + }, + "pass@8": { + "11": 0.852, + "12": 0.78, + "13": 0.695, + "14": 0.583, + "15": 0.517, + "16": 0.272, + "17": 0.217, + "18": 0.176, + "19": 0.053, + "20": 0.057 + }, + "pass@16": { + "11": 0.911, + "12": 0.829, + "13": 0.746, + "14": 0.665, + "15": 0.583, + "16": 0.331, + "17": 0.267, + "18": 0.222, + "19": 0.073, + "20": 0.076 + }, + "pass@32": { + "11": 0.938, + "12": 0.853, + "13": 0.778, + "14": 0.735, + "15": 0.65, + "16": 0.39, + "17": 0.333, + "18": 0.269, + "19": 0.099, + "20": 0.098 + }, + "pass@64": { + "11": 0.952, + "12": 0.872, + "13": 0.809, + "14": 0.782, + "15": 0.727, + "16": 0.449, + "17": 0.384, + "18": 0.32, + "19": 0.125, + "20": 0.125 + }, + "pass@128": { + "11": 0.962, + "12": 0.883, + "13": 0.827, + "14": 0.819, + "15": 0.787, + "16": 0.515, + "17": 0.44, + "18": 0.378, + "19": 0.142, + "20": 0.142 + } + }, + "total_pass_at_k": { + "pass@1": 0.2515, + "pass@2": 0.3079, + "pass@4": 0.3636, + "pass@8": 0.4202, + "pass@16": 0.4703, + "pass@32": 0.5143, + "pass@64": 0.5545, + "pass@128": 0.5895 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block16-epoch4-op11-14/summary.csv b/composition/SDAR-100M-block16-epoch4-op11-14/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..b2297da475338d5ed4d78673919ec0f8b1498c44 --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op11-14/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch4-op11-14,0.9905703125,0.6678171875,0.8207002467105263,0.0,0.0,0.0,199.1858359375,367.13116015625,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block16-epoch4-op11-14/summary.json b/composition/SDAR-100M-block16-epoch4-op11-14/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6682b9543009084976a677597dc3031b491f824d --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op11-14/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch4-op11-14", + "id_acc": 0.9905703125, + "ood_acc": 0.6678171875, + "total_acc": 0.8207002467105263, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.1858359375, + "ood_avg_resp_len": 367.13116015625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 666691, + "answer_accuracy": 0.8233517922067657, + "avg_response_len": 274.56429813468225, + "resp_tokens_sum": 222322400, + "pass_at_k": { + "pass@1": 0.8233517922067657, + "pass@2": 0.8611208180960408, + "pass@4": 0.8884471519832107, + "pass@8": 0.9082678023063252, + "pass@16": 0.9229493914421676, + "pass@32": 0.9343577884724095, + "pass@64": 0.9438576549065015, + "pass@128": 0.9516282010749288 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9453585914454278, + "pass@2": 0.9535728491626602, + "pass@4": 0.95873854180838, + "pass@8": 0.9629986555885872, + "pass@16": 0.9679666317812168, + "pass@32": 0.9740341139349422, + "pass@64": 0.9803653794455086, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9971923828125, + "pass@2": 0.9978869340551182, + "pass@4": 0.9985996801181102, + "pass@8": 0.9993905663162155, + "pass@16": 0.9998951809167032, + "pass@32": 0.9999980558628433, + "pass@64": 0.9999999999667597, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9949900793650793, + "pass@2": 0.9973585333083365, + "pass@4": 0.9983299611358104, + "pass@8": 0.9991563945664312, + "pass@16": 0.9997971079638622, + "pass@32": 0.9999918808807343, + "pass@64": 0.9999999987391617, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9971229416167665, + "pass@2": 0.9976215138620398, + "pass@4": 0.9981272295117304, + "pass@8": 0.998847506991621, + "pass@16": 0.9995863571069055, + "pass@32": 0.999958801686582, + "pass@64": 0.9999999176094492, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9964866863905325, + "pass@2": 0.9979623363462704, + "pass@4": 0.9990334570789599, + "pass@8": 0.9997250598303445, + "pass@16": 0.9999790093720811, + "pass@32": 0.9999999405368915, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9981971153846154, + "pass@2": 0.9998284657026045, + "pass@4": 0.999995967751627, + "pass@8": 0.9999999975724242, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9949603873239437, + "pass@2": 0.9973813352556282, + "pass@4": 0.9982075685081618, + "pass@8": 0.9989154500291274, + "pass@16": 0.9996108261180917, + "pass@32": 0.9999612387699112, + "pass@64": 0.9999999224832565, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9636597025495751, + "pass@2": 0.9756313990319198, + "pass@4": 0.9833490821614721, + "pass@8": 0.9890875592461815, + "pass@16": 0.9928681364142148, + "pass@32": 0.9954215669121044, + "pass@64": 0.9969126772151184, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.9084348516949152, + "pass@2": 0.9271233011922239, + "pass@4": 0.9376102685257564, + "pass@8": 0.9448320171203957, + "pass@16": 0.9500880057183615, + "pass@32": 0.9533178342586844, + "pass@64": 0.9561329927476948, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.8727468652037618, + "pass@2": 0.9105426036087183, + "pass@4": 0.9320865086809287, + "pass@8": 0.9476093057668198, + "pass@16": 0.9599091371962115, + "pass@32": 0.9694213182908559, + "pass@64": 0.9768613994766613, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.8451129283489096, + "pass@2": 0.8927793926460124, + "pass@4": 0.9188406133812713, + "pass@8": 0.9352618975128104, + "pass@16": 0.9480306260712734, + "pass@32": 0.9589256381658317, + "pass@64": 0.9683198089214712, + "pass@128": 0.9750778816199377 + }, + "15": { + "pass@1": 0.7921107297687862, + "pass@2": 0.8578609303172365, + "pass@4": 0.8997546105147255, + "pass@8": 0.9264640318143393, + "pass@16": 0.9428917374738313, + "pass@32": 0.952393511786133, + "pass@64": 0.9574741921523455, + "pass@128": 0.9595375722543352 + }, + "16": { + "pass@1": 0.6568800403225806, + "pass@2": 0.7477683674117346, + "pass@4": 0.8092675075897764, + "pass@8": 0.8516551322611292, + "pass@16": 0.8812944796205525, + "pass@32": 0.902773745388599, + "pass@64": 0.9194705540558217, + "pass@128": 0.932258064516129 + }, + "17": { + "pass@1": 0.5657514450867052, + "pass@2": 0.6691021687679213, + "pass@4": 0.739191088198512, + "pass@8": 0.7856207032319886, + "pass@16": 0.8183619373072404, + "pass@32": 0.8430109049522699, + "pass@64": 0.8637531249758412, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.45412946428571427, + "pass@2": 0.5638899043869511, + "pass@4": 0.6530802185441107, + "pass@8": 0.720726478211569, + "pass@16": 0.7704510468029769, + "pass@32": 0.807289056099719, + "pass@64": 0.8378334239926626, + "pass@128": 0.8657142857142858 + }, + "19": { + "pass@1": 0.36369181681681684, + "pass@2": 0.4703841390603205, + "pass@4": 0.5581976540657639, + "pass@8": 0.627149829400672, + "pass@16": 0.6817550885633032, + "pass@32": 0.7263554189791901, + "pass@64": 0.763762224924364, + "pass@128": 0.7957957957957958 + }, + "20": { + "pass@1": 0.32453547297297297, + "pass@2": 0.4275736119978244, + "pass@4": 0.5169445100330928, + "pass@8": 0.585449717921024, + "pass@16": 0.6372318334285229, + "pass@32": 0.6821752409020172, + "pass@64": 0.7234653885399501, + "pass@128": 0.7537537537537538 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 677756, + "answer_accuracy": 0.8254043257989088, + "avg_response_len": 284.02363722720185, + "resp_tokens_sum": 233217489, + "pass_at_k": { + "pass@1": 0.8254043257989088, + "pass@2": 0.8633358853818162, + "pass@4": 0.8908206644619501, + "pass@8": 0.9108071320749088, + "pass@16": 0.9256376776351245, + "pass@32": 0.9371229822544132, + "pass@64": 0.9466032938973412, + "pass@128": 0.9547934528448948 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9309815950920245, + "pass@2": 0.9449852966040289, + "pass@4": 0.9559561028797782, + "pass@8": 0.9638793675635439, + "pass@16": 0.9687407182381751, + "pass@32": 0.9718304058076516, + "pass@64": 0.9746895660017418, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99375, + "pass@2": 0.9960658450302408, + "pass@4": 0.9975695505453124, + "pass@8": 0.9984974444758158, + "pass@16": 0.9992768151012691, + "pass@32": 0.9998558695467676, + "pass@64": 0.999998064534979, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9964183381088825, + "pass@2": 0.9970275026509938, + "pass@4": 0.9971312728100964, + "pass@8": 0.9971346683194743, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9973582008670521, + "pass@2": 0.9977495192526513, + "pass@4": 0.9982556172530457, + "pass@8": 0.9989655495774096, + "pass@16": 0.9996567447729243, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9936839788732394, + "pass@2": 0.9963724769879118, + "pass@4": 0.9971180363018003, + "pass@8": 0.9971826215324542, + "pass@16": 0.9971830985804359, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9924214071856288, + "pass@2": 0.9938524771323495, + "pass@4": 0.9940081237973997, + "pass@8": 0.9940119737766329, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9630082831325302, + "pass@2": 0.9713395906460487, + "pass@4": 0.9757507891521089, + "pass@8": 0.9784955531653872, + "pass@16": 0.9805383638724173, + "pass@32": 0.9824747306648515, + "pass@64": 0.9841054553733752, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9007885971786834, + "pass@2": 0.915418739046726, + "pass@4": 0.924029935129582, + "pass@8": 0.9303735376408099, + "pass@16": 0.9353211411280159, + "pass@32": 0.939039317249673, + "pass@64": 0.9419950990841026, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.8611913515406162, + "pass@2": 0.8988553590727634, + "pass@4": 0.918313749121695, + "pass@8": 0.9307861443620137, + "pass@16": 0.9400313356135247, + "pass@32": 0.9484692530915426, + "pass@64": 0.9581816741685208, + "pass@128": 0.969187675070028 + }, + "14": { + "pass@1": 0.8469709429824561, + "pass@2": 0.8980640886172121, + "pass@4": 0.9245809564374623, + "pass@8": 0.9405203926008627, + "pass@16": 0.9525469091473052, + "pass@32": 0.963199459193292, + "pass@64": 0.9716362530225532, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.7924715909090909, + "pass@2": 0.861036670842281, + "pass@4": 0.9029049320539471, + "pass@8": 0.9281268028368013, + "pass@16": 0.9426543876503707, + "pass@32": 0.9525476884583312, + "pass@64": 0.9611302965599587, + "pass@128": 0.9696969696969697 + }, + "16": { + "pass@1": 0.6614583333333334, + "pass@2": 0.757938160374184, + "pass@4": 0.8144365875205767, + "pass@8": 0.8475187224277707, + "pass@16": 0.8701097260916296, + "pass@32": 0.8890719227083439, + "pass@64": 0.9063072562149335, + "pass@128": 0.9173789173789174 + }, + "17": { + "pass@1": 0.5830260093167702, + "pass@2": 0.6846121986110436, + "pass@4": 0.7533166365656156, + "pass@8": 0.8007323980276663, + "pass@16": 0.8367999826994603, + "pass@32": 0.8654849657428961, + "pass@64": 0.8891736360856827, + "pass@128": 0.906832298136646 + }, + "18": { + "pass@1": 0.5104779411764706, + "pass@2": 0.615626447429365, + "pass@4": 0.6993681317408854, + "pass@8": 0.7636960252154447, + "pass@16": 0.8126314154686012, + "pass@32": 0.8493046683312366, + "pass@64": 0.8760405951252511, + "pass@128": 0.8970588235294118 + }, + "19": { + "pass@1": 0.33535621279761907, + "pass@2": 0.450107066694788, + "pass@4": 0.5589451405069903, + "pass@8": 0.6479033713249467, + "pass@16": 0.7114656294943115, + "pass@32": 0.7547122095604747, + "pass@64": 0.7871870069795173, + "pass@128": 0.8154761904761905 + }, + "20": { + "pass@1": 0.30761133982035926, + "pass@2": 0.4103107760856239, + "pass@4": 0.5020851449519407, + "pass@8": 0.577641649195555, + "pass@16": 0.640643393523646, + "pass@32": 0.6937003584791263, + "pass@64": 0.7400380698580312, + "pass@128": 0.7844311377245509 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 651496, + "answer_accuracy": 0.8131989934494328, + "avg_response_len": 304.374299758348, + "resp_tokens_sum": 243850079, + "pass_at_k": { + "pass@1": 0.8131989934494328, + "pass@2": 0.8516683416824594, + "pass@4": 0.8803690842367309, + "pass@8": 0.9015596188301824, + "pass@16": 0.9172829494592375, + "pass@32": 0.9293968706923023, + "pass@64": 0.9392539645433563, + "pass@128": 0.9475954625339511 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9517957089552239, + "pass@2": 0.9643014014572805, + "pass@4": 0.9723098293683435, + "pass@8": 0.9778178687535791, + "pass@16": 0.9818233351333638, + "pass@32": 0.9843700251647998, + "pass@64": 0.9862015157581675, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 0.9999553571428571, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998861151603499, + "pass@2": 0.99999784784555, + "pass@4": 0.9999999997267104, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9979669225146199, + "pass@2": 0.9986840666298292, + "pass@4": 0.9994179578978066, + "pass@8": 0.9998914828566827, + "pass@16": 0.999996971910131, + "pass@32": 0.9999999992108352, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9915670955882353, + "pass@2": 0.9952162459471977, + "pass@4": 0.996770337898939, + "pass@8": 0.9975063173915031, + "pass@16": 0.9980350056586529, + "pass@32": 0.9987689956690733, + "pass@64": 0.9996410375173692, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9964264589905363, + "pass@2": 0.9985430276459921, + "pass@4": 0.9995857823568582, + "pass@8": 0.9999535526784058, + "pass@16": 0.9999995379907698, + "pass@32": 0.9999999999901541, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972804588607594, + "pass@2": 0.9992676511262834, + "pass@4": 0.9999203209764919, + "pass@8": 0.9999990565168133, + "pass@16": 0.9999999999228011, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9907563813813813, + "pass@2": 0.99560153164976, + "pass@4": 0.998587598565945, + "pass@8": 0.9998129486244137, + "pass@16": 0.999996824080759, + "pass@32": 0.9999999996818268, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9906300241157556, + "pass@2": 0.9950913993467858, + "pass@4": 0.9970629562582811, + "pass@8": 0.9986055619182425, + "pass@16": 0.99964486782007, + "pass@32": 0.9999770906276318, + "pass@64": 0.9999999831714547, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9624007936507937, + "pass@2": 0.9748172103487065, + "pass@4": 0.981273297087864, + "pass@8": 0.9846499737188555, + "pass@16": 0.9862178333308695, + "pass@32": 0.9878187753676408, + "pass@64": 0.9897012873390828, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9143969801223242, + "pass@2": 0.9317818572563751, + "pass@4": 0.941430099907236, + "pass@8": 0.9493256684380422, + "pass@16": 0.9570131901299108, + "pass@32": 0.9641514897698327, + "pass@64": 0.9703591098246457, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8329957561728395, + "pass@2": 0.8691643579274818, + "pass@4": 0.8912119784216789, + "pass@8": 0.9091026647516887, + "pass@16": 0.9252978369245227, + "pass@32": 0.9383556719133557, + "pass@64": 0.9473490414410928, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8191533753709199, + "pass@2": 0.8682212201219658, + "pass@4": 0.8979430763847394, + "pass@8": 0.9180236634394026, + "pass@16": 0.9314134024114261, + "pass@32": 0.9406145618745281, + "pass@64": 0.9479964540280973, + "pass@128": 0.9554896142433235 + }, + "15": { + "pass@1": 0.7535927854938271, + "pass@2": 0.8146192554923689, + "pass@4": 0.8533767113022905, + "pass@8": 0.878865995821382, + "pass@16": 0.8964124781429048, + "pass@32": 0.910541254887923, + "pass@64": 0.922806998472301, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.6453032817109144, + "pass@2": 0.7468509017954611, + "pass@4": 0.8109872092868917, + "pass@8": 0.8503621602906531, + "pass@16": 0.8763865513376642, + "pass@32": 0.8958750335367089, + "pass@64": 0.9123815407896373, + "pass@128": 0.9262536873156342 + }, + "17": { + "pass@1": 0.5255553463855421, + "pass@2": 0.6341747402997818, + "pass@4": 0.7174640033700606, + "pass@8": 0.7752305050970983, + "pass@16": 0.8126617684952755, + "pass@32": 0.8371897052025431, + "pass@64": 0.8560389765187822, + "pass@128": 0.8734939759036144 + }, + "18": { + "pass@1": 0.4767389112903226, + "pass@2": 0.5847179006858011, + "pass@4": 0.66608676022352, + "pass@8": 0.7243191147240607, + "pass@16": 0.7663746277687996, + "pass@32": 0.8006317366757167, + "pass@64": 0.829606389878158, + "pass@128": 0.8516129032258064 + }, + "19": { + "pass@1": 0.30501793806646527, + "pass@2": 0.40979077955134763, + "pass@4": 0.5044869689627164, + "pass@8": 0.5830466910339542, + "pass@16": 0.6470709277754211, + "pass@32": 0.7023998688940181, + "pass@64": 0.7524830182120951, + "pass@128": 0.797583081570997 + }, + "20": { + "pass@1": 0.2891798048048048, + "pass@2": 0.3927472009410985, + "pass@4": 0.49233214992270086, + "pass@8": 0.5773029346421416, + "pass@16": 0.6449435379481746, + "pass@32": 0.6935367729692132, + "pass@64": 0.7276704804486359, + "pass@128": 0.7567567567567568 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_metrics.json b/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..7be54959205e1d861d6c1aa715862c470b93df03 --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch4-op17-20", + "id": { + "answer_accuracy": 0.9890277777777777, + "avg_response_len": 199.14139930555555, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9890277777777777, + "pass@2": 0.9928172162073456, + "pass@4": 0.9948839220097478, + "pass@8": 0.9963080761191235, + "pass@16": 0.9973152803179801, + "pass@32": 0.9980268388493161, + "pass@64": 0.99847883565242, + "pass@128": 0.9987777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9391875, + "pass@2": 0.9553868110236222, + "pass@4": 0.9664699572553431, + "pass@8": 0.9749212520185588, + "pass@16": 0.9816938886763031, + "pass@32": 0.9868280196969069, + "pass@64": 0.989992463600238, + "pass@128": 0.992 + }, + "2": { + "pass@1": 0.999953125, + "pass@2": 0.9999991387795276, + "pass@4": 0.9999999999062616, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99996875, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99815625, + "pass@2": 0.9986737204724409, + "pass@4": 0.9989233598612672, + "pass@8": 0.9990519583151254, + "pass@16": 0.9991249279427927, + "pass@32": 0.9992499999993945, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.992640625, + "pass@2": 0.9958212352362207, + "pass@4": 0.997663693007124, + "pass@8": 0.9989601583261803, + "pass@16": 0.9996727318798748, + "pass@32": 0.9999476997271504, + "pass@64": 0.9999993318674036, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9955546875, + "pass@2": 0.9968681102362207, + "pass@4": 0.9975768131796027, + "pass@8": 0.9981352577610507, + "pass@16": 0.9986862301248822, + "pass@32": 0.9992675062194463, + "pass@64": 0.9998183933036234, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9961640625, + "pass@2": 0.9984712106299214, + "pass@4": 0.9994211929133856, + "pass@8": 0.9998762765811555, + "pass@16": 0.9999913086119966, + "pass@32": 0.9999999700015125, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9908046875, + "pass@2": 0.995634965551181, + "pass@4": 0.9976798744844393, + "pass@8": 0.998685442665731, + "pass@16": 0.998977556450126, + "pass@32": 0.9989998945869978, + "pass@64": 0.9989999999999175, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.9888203125, + "pass@2": 0.9944998769685044, + "pass@4": 0.9962204074803149, + "pass@8": 0.9971423394043094, + "pass@16": 0.9976908791758443, + "pass@32": 0.997948459412436, + "pass@64": 0.9979993321006008, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.9391875, + "2": 0.999953125, + "3": 0.99996875, + "4": 0.99815625, + "5": 0.992640625, + "6": 0.9955546875, + "7": 0.9961640625, + "8": 0.9908046875, + "9": 0.9888203125 + }, + "per_op_avg_response_len": { + "10": 278.9036640625, + "2": 115.269109375, + "3": 144.2801953125, + "4": 159.3698984375, + "5": 173.026875, + "6": 198.258796875, + "7": 223.091875, + "8": 242.26153125, + "9": 257.8106484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.9036640625, + "2": 115.269109375, + "3": 144.2801953125, + "4": 159.3698984375, + "5": 173.026875, + "6": 198.258796875, + "7": 223.091875, + "8": 242.26153125, + "9": 257.8106484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 375474, + "answer_accuracy": 0.9906756585612969, + "avg_response_len": 189.4462069402229, + "resp_tokens_sum": 71801628, + "pass_at_k": { + "pass@1": 0.9906756585612969, + "pass@2": 0.994166370825988, + "pass@4": 0.9958945190108589, + "pass@8": 0.9970284643001964, + "pass@16": 0.997750665029614, + "pass@32": 0.9982530045544682, + "pass@64": 0.9987353591726881, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9463956489675516, + "pass@2": 0.9599948029173347, + "pass@4": 0.9696439955514411, + "pass@8": 0.9772987780870966, + "pass@16": 0.9829576313879089, + "pass@32": 0.9869532680938505, + "pass@64": 0.9904289041602642, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999513239875389, + "pass@2": 0.9999996167243114, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9962890625, + "pass@2": 0.9968965305118112, + "pass@4": 0.9969725551884139, + "pass@8": 0.9970703124996394, + "pass@16": 0.997265625, + "pass@32": 0.99765625, + "pass@64": 0.9984375, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941220238095239, + "pass@2": 0.9976346706661666, + "pass@4": 0.9991279667422525, + "pass@8": 0.999810090374339, + "pass@16": 0.9999911414698637, + "pass@32": 0.9999999920410901, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9975673652694611, + "pass@2": 0.9990827891461174, + "pass@4": 0.9997700759897528, + "pass@8": 0.9999845197580833, + "pass@16": 0.9999999528630369, + "pass@32": 0.9999999999999486, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960012943786982, + "pass@2": 0.9984763022410661, + "pass@4": 0.9996465231971745, + "pass@8": 0.9999815406455984, + "pass@16": 0.9999999678464966, + "pass@32": 0.9999999999999909, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9963441506410257, + "pass@2": 0.9996159209065214, + "pass@4": 0.9999852656639074, + "pass@8": 0.9999999592877328, + "pass@16": 0.9999999999999242, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9913292253521127, + "pass@2": 0.9970926444493733, + "pass@4": 0.9988132281880259, + "pass@8": 0.9997344814601936, + "pass@16": 0.9999856314389629, + "pass@32": 0.9999999749550461, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 386021, + "answer_accuracy": 0.9881353415792923, + "avg_response_len": 197.20948609518348, + "resp_tokens_sum": 77041069, + "pass_at_k": { + "pass@1": 0.9881353415792923, + "pass@2": 0.991507118089597, + "pass@4": 0.9934450351819555, + "pass@8": 0.9948278390202879, + "pass@16": 0.995978093148729, + "pass@32": 0.9968872544058474, + "pass@64": 0.9974407893814518, + "pass@128": 0.9977064220183486 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.924079754601227, + "pass@2": 0.9425480653108548, + "pass@4": 0.9558939233439372, + "pass@8": 0.9660136894297928, + "pass@16": 0.9745849213182081, + "pass@32": 0.9815053815983932, + "pass@64": 0.9856196459168264, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150811, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9920969202898551, + "pass@2": 0.9947952327969871, + "pass@4": 0.9963463162213417, + "pass@8": 0.9977460600531718, + "pass@16": 0.9990916148451592, + "pass@32": 0.9998484564379228, + "pass@64": 0.999998063383779, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9952542979942693, + "pass@2": 0.9966728617196489, + "pass@4": 0.9972907742728435, + "pass@8": 0.997639465969732, + "pass@16": 0.9980869583411133, + "pass@32": 0.9988007408816018, + "pass@64": 0.9996502944295288, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967937138728323, + "pass@2": 0.9983024498202175, + "pass@4": 0.9990612040546953, + "pass@8": 0.9997078494159238, + "pass@16": 0.999975444312545, + "pass@32": 0.9999999133144741, + "pass@64": 0.9999999999999895, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9918573943661971, + "pass@2": 0.9956231978485083, + "pass@4": 0.9969242494336096, + "pass@8": 0.9971736180162805, + "pass@16": 0.9971830890857465, + "pass@32": 0.9971830985915483, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9910179640718563, + "pass@2": 0.9937379178650573, + "pass@4": 0.9939987547402381, + "pass@8": 0.9940119380249567, + "pass@16": 0.9940119760478334, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 377865, + "answer_accuracy": 0.9883060972547707, + "avg_response_len": 210.72615448192167, + "resp_tokens_sum": 80568195, + "pass_at_k": { + "pass@1": 0.9883060972547707, + "pass@2": 0.9928184122668052, + "pass@4": 0.9953523200272051, + "pass@8": 0.9971064069599305, + "pass@16": 0.9982499710141318, + "pass@32": 0.9989670246774082, + "pass@64": 0.9992851801704917, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9465951492537313, + "pass@2": 0.9632176225173348, + "pass@4": 0.9735499216515845, + "pass@8": 0.9811835925161799, + "pass@16": 0.9873330366748251, + "pass@32": 0.9918809176478053, + "pass@64": 0.993806150809024, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999316690962099, + "pass@2": 0.9999996413075917, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981953581871345, + "pass@2": 0.9990279792328591, + "pass@4": 0.9996846265927285, + "pass@8": 0.9999691763603531, + "pass@16": 0.9999997893064115, + "pass@32": 0.9999999999982297, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9918198529411765, + "pass@2": 0.9951822313571099, + "pass@4": 0.9976438364322108, + "pass@8": 0.9994046739409391, + "pass@16": 0.9999673976331993, + "pass@32": 0.9999999551856581, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.993764787066246, + "pass@2": 0.9947496212027123, + "pass@4": 0.9955808440940151, + "pass@8": 0.9967326641590364, + "pass@16": 0.9979618214750134, + "pass@32": 0.9990096140434843, + "pass@64": 0.9998121152924854, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956487341772152, + "pass@2": 0.9986505469450812, + "pass@4": 0.9995743400943553, + "pass@8": 0.9999481010918784, + "pass@16": 0.9999994169232284, + "pass@32": 0.9999999999832521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9844923048048048, + "pass@2": 0.9919176065829609, + "pass@4": 0.9963254145596664, + "pass@8": 0.9990655284449819, + "pass@16": 0.9999326120862155, + "pass@32": 0.9999996834444386, + "pass@64": 0.999999999999752, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9835962620578779, + "pass@2": 0.9923585968554577, + "pass@4": 0.9956467118660005, + "pass@8": 0.9975453735871543, + "pass@16": 0.9990224437782512, + "pass@32": 0.9998343032263491, + "pass@64": 0.9999978524135079, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.6918046875, + "avg_response_len": 372.07711328125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.6918046875, + "pass@2": 0.7649430733267603, + "pass@4": 0.8173947985658003, + "pass@8": 0.8545082829889343, + "pass@16": 0.8807741388344542, + "pass@32": 0.9002820731595781, + "pass@64": 0.9154065233838675, + "pass@128": 0.9274 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.948625, + "pass@2": 0.9650340797244101, + "pass@4": 0.974049004686914, + "pass@8": 0.9799215853539738, + "pass@16": 0.9840942258978155, + "pass@32": 0.9873766462931601, + "pass@64": 0.9897657015488682, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.8919921875, + "pass@2": 0.9173727854330723, + "pass@4": 0.9306753659542553, + "pass@8": 0.9402165770631085, + "pass@16": 0.9484162200957359, + "pass@32": 0.9556793185220008, + "pass@64": 0.9622626730700722, + "pass@128": 0.969 + }, + "13": { + "pass@1": 0.8227109375, + "pass@2": 0.8682181348425212, + "pass@4": 0.8953016328271474, + "pass@8": 0.9137342715759691, + "pass@16": 0.9279086259612404, + "pass@32": 0.9409428563904914, + "pass@64": 0.9531583365904523, + "pass@128": 0.963 + }, + "14": { + "pass@1": 0.8139765625, + "pass@2": 0.8710821850393708, + "pass@4": 0.9045889305399336, + "pass@8": 0.9250908670684431, + "pass@16": 0.9386299476507689, + "pass@32": 0.9487427582732499, + "pass@64": 0.957440936100577, + "pass@128": 0.965 + }, + "15": { + "pass@1": 0.7839609375, + "pass@2": 0.8494299950787403, + "pass@4": 0.8907248334270719, + "pass@8": 0.9178499015041497, + "pass@16": 0.9354141739723325, + "pass@32": 0.9468501950584634, + "pass@64": 0.9547623055339345, + "pass@128": 0.962 + }, + "16": { + "pass@1": 0.66109375, + "pass@2": 0.7585896899606303, + "pass@4": 0.8241256755718038, + "pass@8": 0.8659143995718789, + "pass@16": 0.8935886268502504, + "pass@32": 0.9145088226574314, + "pass@64": 0.9303470680153623, + "pass@128": 0.941 + }, + "17": { + "pass@1": 0.605640625, + "pass@2": 0.7034212598425199, + "pass@4": 0.7721566392950879, + "pass@8": 0.8196972052982548, + "pass@16": 0.8530516738610301, + "pass@32": 0.876275545986001, + "pass@64": 0.8930550155935169, + "pass@128": 0.906 + }, + "18": { + "pass@1": 0.56478125, + "pass@2": 0.6675775098425195, + "pass@4": 0.7450145286839147, + "pass@8": 0.7996635802319876, + "pass@16": 0.8369695037285299, + "pass@32": 0.8625344989141525, + "pass@64": 0.8809200646871671, + "pass@128": 0.896 + }, + "19": { + "pass@1": 0.413765625, + "pass@2": 0.5293097933070865, + "pass@4": 0.6272639149793771, + "pass@8": 0.7019809692420499, + "pass@16": 0.7557623698652861, + "pass@32": 0.7956271976826763, + "pass@64": 0.8254726377576067, + "pass@128": 0.848 + }, + "20": { + "pass@1": 0.4115, + "pass@2": 0.5193953001968503, + "pass@4": 0.6100474596925382, + "pass@8": 0.6810134729795795, + "pass@16": 0.7339060204614524, + "pass@32": 0.7742828918181295, + "pass@64": 0.8068804949410628, + "pass@128": 0.833 + } + }, + "per_op_accuracy": { + "11": 0.948625, + "12": 0.8919921875, + "13": 0.8227109375, + "14": 0.8139765625, + "15": 0.7839609375, + "16": 0.66109375, + "17": 0.605640625, + "18": 0.56478125, + "19": 0.413765625, + "20": 0.4115 + }, + "per_op_avg_response_len": { + "11": 296.949046875, + "12": 306.7407578125, + "13": 325.3369921875, + "14": 353.14478125, + "15": 373.4759765625, + "16": 378.2594296875, + "17": 402.084828125, + "18": 419.432109375, + "19": 421.6563125, + "20": 443.6908984375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 296.949046875, + "12": 306.7407578125, + "13": 325.3369921875, + "14": 353.14478125, + "15": 373.4759765625, + "16": 378.2594296875, + "17": 402.084828125, + "18": 419.432109375, + "19": 421.6563125, + "20": 443.6908984375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 301159, + "answer_accuracy": 0.6991990156017831, + "avg_response_len": 353.8343819650817, + "resp_tokens_sum": 152403545, + "pass_at_k": { + "pass@1": 0.6991990156017831, + "pass@2": 0.7698041221583997, + "pass@4": 0.8194136314684324, + "pass@8": 0.8540800124080026, + "pass@16": 0.8791774558830416, + "pass@32": 0.8984448726143027, + "pass@64": 0.9136238088950139, + "pass@128": 0.9260029717682021 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9385844546742209, + "pass@2": 0.9559055815172538, + "pass@4": 0.9662483132420059, + "pass@8": 0.9743145176794873, + "pass@16": 0.9814528957415205, + "pass@32": 0.9870929428344121, + "pass@64": 0.9904799574420998, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8917284604519774, + "pass@2": 0.9156980542951199, + "pass@4": 0.9281065293215466, + "pass@8": 0.935889004485147, + "pass@16": 0.9418117371185404, + "pass@32": 0.947600022930526, + "pass@64": 0.9542037056880797, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8402233542319749, + "pass@2": 0.8852274269987415, + "pass@4": 0.9120427055748123, + "pass@8": 0.9300158215455117, + "pass@16": 0.9440746211517979, + "pass@32": 0.9571960077579881, + "pass@64": 0.9683369401551648, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8298773364485982, + "pass@2": 0.8854661092305048, + "pass@4": 0.9165421769708681, + "pass@8": 0.9342494936091128, + "pass@16": 0.9469401578089754, + "pass@32": 0.9577873713575069, + "pass@64": 0.9667747235355996, + "pass@128": 0.9750778816199377 + }, + "15": { + "pass@1": 0.7909365968208093, + "pass@2": 0.8548815057348322, + "pass@4": 0.8965234123552469, + "pass@8": 0.9232631518428708, + "pass@16": 0.93956332213618, + "pass@32": 0.9494509870773286, + "pass@64": 0.9560635736921728, + "pass@128": 0.9624277456647399 + }, + "16": { + "pass@1": 0.6512096774193549, + "pass@2": 0.7479818865887731, + "pass@4": 0.8152397825271837, + "pass@8": 0.8589674413513186, + "pass@16": 0.8885187025219686, + "pass@32": 0.9122664817583825, + "pass@64": 0.9310470742858282, + "pass@128": 0.9419354838709677 + }, + "17": { + "pass@1": 0.6124909682080925, + "pass@2": 0.7055230474261528, + "pass@4": 0.768419161297034, + "pass@8": 0.8132989938840093, + "pass@16": 0.8486630253769624, + "pass@32": 0.8756942867659188, + "pass@64": 0.8953656362346946, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.5464732142857143, + "pass@2": 0.6480627812148483, + "pass@4": 0.7236771966896989, + "pass@8": 0.777945603229985, + "pass@16": 0.8163166798894307, + "pass@32": 0.8431802277020857, + "pass@64": 0.8622731277447071, + "pass@128": 0.8771428571428571 + }, + "19": { + "pass@1": 0.4315643768768769, + "pass@2": 0.5412484334728428, + "pass@4": 0.6303853890448375, + "pass@8": 0.6978895463021549, + "pass@16": 0.7471806522306299, + "pass@32": 0.7842191914351475, + "pass@64": 0.8144645683656517, + "pass@128": 0.8438438438438438 + }, + "20": { + "pass@1": 0.4473066816816817, + "pass@2": 0.550585157007401, + "pass@4": 0.6322523291007539, + "pass@8": 0.6917831399797562, + "pass@16": 0.7350186424328486, + "pass@32": 0.7688185050754536, + "pass@64": 0.7971501932040521, + "pass@128": 0.8228228228228228 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 300394, + "answer_accuracy": 0.6978376821290514, + "avg_response_len": 367.61922715953017, + "resp_tokens_sum": 158246843, + "pass_at_k": { + "pass@1": 0.6978376821290514, + "pass@2": 0.7712399262703721, + "pass@4": 0.8234056467183366, + "pass@8": 0.8598974993623416, + "pass@16": 0.8856925425332236, + "pass@32": 0.9053746000361343, + "pass@64": 0.9216224010043141, + "pass@128": 0.9351769253642581 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9533838478915663, + "pass@2": 0.9686280802343229, + "pass@4": 0.9774122846842942, + "pass@8": 0.9826945678160793, + "pass@16": 0.9851505016991625, + "pass@32": 0.9862401453810479, + "pass@64": 0.9872047186881181, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.885384012539185, + "pass@2": 0.9101669525831221, + "pass@4": 0.9239799511738148, + "pass@8": 0.9333532233276649, + "pass@16": 0.9404962983721141, + "pass@32": 0.947095869663446, + "pass@64": 0.9545776664683873, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.8328956582633054, + "pass@2": 0.8768782119146881, + "pass@4": 0.9001525788793208, + "pass@8": 0.9144132901889223, + "pass@16": 0.9252804791130302, + "pass@32": 0.9358350221164605, + "pass@64": 0.9474943865376289, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.8158351608187134, + "pass@2": 0.8764760182115391, + "pass@4": 0.9120771337245117, + "pass@8": 0.9326000038402523, + "pass@16": 0.9449230559731131, + "pass@32": 0.9539728291687525, + "pass@64": 0.9628130945239121, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.7901515151515152, + "pass@2": 0.8613483506323071, + "pass@4": 0.9039200494824501, + "pass@8": 0.9292158194336342, + "pass@16": 0.9452248133475442, + "pass@32": 0.9574867632372372, + "pass@64": 0.9678628579175225, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.6649528133903134, + "pass@2": 0.7624254800681963, + "pass@4": 0.8277979675617472, + "pass@8": 0.8686706781122179, + "pass@16": 0.8961598925297205, + "pass@32": 0.9179229188572494, + "pass@64": 0.9337860632473033, + "pass@128": 0.9430199430199431 + }, + "17": { + "pass@1": 0.6393390916149069, + "pass@2": 0.7354016634469608, + "pass@4": 0.8004865514675568, + "pass@8": 0.8453401336925654, + "pass@16": 0.8772971858982275, + "pass@32": 0.9004220002674332, + "pass@64": 0.9174945886997983, + "pass@128": 0.9316770186335404 + }, + "18": { + "pass@1": 0.5892693014705882, + "pass@2": 0.6895900880037054, + "pass@4": 0.7655650637420315, + "pass@8": 0.8203016454124229, + "pass@16": 0.859618198955504, + "pass@32": 0.8878937646484227, + "pass@64": 0.9097340634600377, + "pass@128": 0.9294117647058824 + }, + "19": { + "pass@1": 0.4204799107142857, + "pass@2": 0.5389387361736032, + "pass@4": 0.6410350558858714, + "pass@8": 0.7190375968217035, + "pass@16": 0.7736645475875377, + "pass@32": 0.8123192037757934, + "pass@64": 0.8394299819521862, + "pass@128": 0.8571428571428571 + }, + "20": { + "pass@1": 0.3888005239520958, + "pass@2": 0.4932697351360271, + "pass@4": 0.581708334175294, + "pass@8": 0.6533021743606491, + "pass@16": 0.7090402348266315, + "pass@32": 0.754332532451074, + "pass@64": 0.7954232145576876, + "pass@128": 0.8293413173652695 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 283957, + "answer_accuracy": 0.6779994078545232, + "avg_response_len": 395.4202251107885, + "resp_tokens_sum": 165608317, + "pass_at_k": { + "pass@1": 0.6779994078545232, + "pass@2": 0.7534718796324861, + "pass@4": 0.8091405641360095, + "pass@8": 0.8494096264611646, + "pass@16": 0.8773610112343615, + "pass@32": 0.8969373335351778, + "pass@64": 0.9108511559686301, + "pass@128": 0.9208435207823961 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9548611111111112, + "pass@2": 0.9714758311461071, + "pass@4": 0.9792459415787311, + "pass@8": 0.9832824257084963, + "pass@16": 0.9859409115458316, + "pass@32": 0.9888924101781106, + "pass@64": 0.9916644760868305, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8987241972477065, + "pass@2": 0.9262153404849626, + "pass@4": 0.9399879209479548, + "pass@8": 0.9515969150881995, + "pass@16": 0.9632921895262051, + "pass@32": 0.9727991681405372, + "pass@64": 0.978484053984944, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7942467206790124, + "pass@2": 0.8419292098036356, + "pass@4": 0.87347388607674, + "pass@8": 0.896955802177486, + "pass@16": 0.9148879344767462, + "pass@32": 0.9305686019139384, + "pass@64": 0.9444548045895704, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.7969445474777448, + "pass@2": 0.8519073167597371, + "pass@4": 0.8856038931706228, + "pass@8": 0.9087465231648425, + "pass@16": 0.9243278096477272, + "pass@32": 0.9348198946343538, + "pass@64": 0.9430984318055536, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.7702064043209876, + "pass@2": 0.8314692542772435, + "pass@4": 0.8710929519226761, + "pass@8": 0.9004926868932659, + "pass@16": 0.9209909756435962, + "pass@32": 0.9332392643870973, + "pass@64": 0.9400295244557415, + "pass@128": 0.9444444444444444 + }, + "16": { + "pass@1": 0.6661366150442478, + "pass@2": 0.7643184708382698, + "pass@4": 0.8284491338472064, + "pass@8": 0.8694132293085024, + "pass@16": 0.8955625569041571, + "pass@32": 0.9130243917210557, + "pass@64": 0.926146214710774, + "pass@128": 0.9380530973451328 + }, + "17": { + "pass@1": 0.5658179593373494, + "pass@2": 0.6702137042263541, + "pass@4": 0.7485751202221707, + "pass@8": 0.8014946697752476, + "pass@16": 0.8341101603053381, + "pass@32": 0.8534621646954216, + "pass@64": 0.8669435177559548, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.55859375, + "pass@2": 0.6654674403098803, + "pass@4": 0.7465657682910599, + "pass@8": 0.801548579552477, + "pass@16": 0.8354467036205091, + "pass@32": 0.8565727072192222, + "pass@64": 0.8703706077422788, + "pass@128": 0.8806451612903226 + }, + "19": { + "pass@1": 0.3890436178247734, + "pass@2": 0.5075246211670672, + "pass@4": 0.6101444158604039, + "pass@8": 0.6887828332064626, + "pass@16": 0.7462233374111003, + "pass@32": 0.7901599832812849, + "pass@64": 0.8223790409544078, + "pass@128": 0.8429003021148036 + }, + "20": { + "pass@1": 0.39846096096096095, + "pass@2": 0.5144094635974552, + "pass@4": 0.6162668182805975, + "pass@8": 0.6980383217113035, + "pass@16": 0.7577338561538128, + "pass@32": 0.799757548916951, + "pass@64": 0.828102483308844, + "pass@128": 0.8468468468468469 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.832594572368421, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.832594572368421, + "pass@2": 0.8728834567964937, + "pass@4": 0.9014685938813386, + "pass@8": 0.9216766060506145, + "pass@16": 0.9359778374318961, + "pass@32": 0.9465822253284079, + "pass@64": 0.954756566037391, + "pass@128": 0.9612105263157895 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9391875, + "pass@2": 0.9553868110236222, + "pass@4": 0.9664699572553431, + "pass@8": 0.9749212520185588, + "pass@16": 0.9816938886763031, + "pass@32": 0.9868280196969069, + "pass@64": 0.989992463600238, + "pass@128": 0.992 + }, + "2": { + "pass@1": 0.999953125, + "pass@2": 0.9999991387795276, + "pass@4": 0.9999999999062616, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99996875, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99815625, + "pass@2": 0.9986737204724409, + "pass@4": 0.9989233598612672, + "pass@8": 0.9990519583151254, + "pass@16": 0.9991249279427927, + "pass@32": 0.9992499999993945, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.992640625, + "pass@2": 0.9958212352362207, + "pass@4": 0.997663693007124, + "pass@8": 0.9989601583261803, + "pass@16": 0.9996727318798748, + "pass@32": 0.9999476997271504, + "pass@64": 0.9999993318674036, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9955546875, + "pass@2": 0.9968681102362207, + "pass@4": 0.9975768131796027, + "pass@8": 0.9981352577610507, + "pass@16": 0.9986862301248822, + "pass@32": 0.9992675062194463, + "pass@64": 0.9998183933036234, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9961640625, + "pass@2": 0.9984712106299214, + "pass@4": 0.9994211929133856, + "pass@8": 0.9998762765811555, + "pass@16": 0.9999913086119966, + "pass@32": 0.9999999700015125, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9908046875, + "pass@2": 0.995634965551181, + "pass@4": 0.9976798744844393, + "pass@8": 0.998685442665731, + "pass@16": 0.998977556450126, + "pass@32": 0.9989998945869978, + "pass@64": 0.9989999999999175, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.9888203125, + "pass@2": 0.9944998769685044, + "pass@4": 0.9962204074803149, + "pass@8": 0.9971423394043094, + "pass@16": 0.9976908791758443, + "pass@32": 0.997948459412436, + "pass@64": 0.9979993321006008, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.948625, + "pass@2": 0.9650340797244101, + "pass@4": 0.974049004686914, + "pass@8": 0.9799215853539738, + "pass@16": 0.9840942258978155, + "pass@32": 0.9873766462931601, + "pass@64": 0.9897657015488682, + "pass@128": 0.991 + }, + "12": { + "pass@1": 0.8919921875, + "pass@2": 0.9173727854330723, + "pass@4": 0.9306753659542553, + "pass@8": 0.9402165770631085, + "pass@16": 0.9484162200957359, + "pass@32": 0.9556793185220008, + "pass@64": 0.9622626730700722, + "pass@128": 0.969 + }, + "13": { + "pass@1": 0.8227109375, + "pass@2": 0.8682181348425212, + "pass@4": 0.8953016328271474, + "pass@8": 0.9137342715759691, + "pass@16": 0.9279086259612404, + "pass@32": 0.9409428563904914, + "pass@64": 0.9531583365904523, + "pass@128": 0.963 + }, + "14": { + "pass@1": 0.8139765625, + "pass@2": 0.8710821850393708, + "pass@4": 0.9045889305399336, + "pass@8": 0.9250908670684431, + "pass@16": 0.9386299476507689, + "pass@32": 0.9487427582732499, + "pass@64": 0.957440936100577, + "pass@128": 0.965 + }, + "15": { + "pass@1": 0.7839609375, + "pass@2": 0.8494299950787403, + "pass@4": 0.8907248334270719, + "pass@8": 0.9178499015041497, + "pass@16": 0.9354141739723325, + "pass@32": 0.9468501950584634, + "pass@64": 0.9547623055339345, + "pass@128": 0.962 + }, + "16": { + "pass@1": 0.66109375, + "pass@2": 0.7585896899606303, + "pass@4": 0.8241256755718038, + "pass@8": 0.8659143995718789, + "pass@16": 0.8935886268502504, + "pass@32": 0.9145088226574314, + "pass@64": 0.9303470680153623, + "pass@128": 0.941 + }, + "17": { + "pass@1": 0.605640625, + "pass@2": 0.7034212598425199, + "pass@4": 0.7721566392950879, + "pass@8": 0.8196972052982548, + "pass@16": 0.8530516738610301, + "pass@32": 0.876275545986001, + "pass@64": 0.8930550155935169, + "pass@128": 0.906 + }, + "18": { + "pass@1": 0.56478125, + "pass@2": 0.6675775098425195, + "pass@4": 0.7450145286839147, + "pass@8": 0.7996635802319876, + "pass@16": 0.8369695037285299, + "pass@32": 0.8625344989141525, + "pass@64": 0.8809200646871671, + "pass@128": 0.896 + }, + "19": { + "pass@1": 0.413765625, + "pass@2": 0.5293097933070865, + "pass@4": 0.6272639149793771, + "pass@8": 0.7019809692420499, + "pass@16": 0.7557623698652861, + "pass@32": 0.7956271976826763, + "pass@64": 0.8254726377576067, + "pass@128": 0.848 + }, + "20": { + "pass@1": 0.4115, + "pass@2": 0.5193953001968503, + "pass@4": 0.6100474596925382, + "pass@8": 0.6810134729795795, + "pass@16": 0.7339060204614524, + "pass@32": 0.7742828918181295, + "pass@64": 0.8068804949410628, + "pass@128": 0.833 + } + }, + "per_op_accuracy": { + "10": 0.9391875, + "2": 0.999953125, + "3": 0.99996875, + "4": 0.99815625, + "5": 0.992640625, + "6": 0.9955546875, + "7": 0.9961640625, + "8": 0.9908046875, + "9": 0.9888203125, + "11": 0.948625, + "12": 0.8919921875, + "13": 0.8227109375, + "14": 0.8139765625, + "15": 0.7839609375, + "16": 0.66109375, + "17": 0.605640625, + "18": 0.56478125, + "19": 0.413765625, + "20": 0.4115 + }, + "per_op_avg_response_len": { + "10": 278.9036640625, + "2": 115.269109375, + "3": 144.2801953125, + "4": 159.3698984375, + "5": 173.026875, + "6": 198.258796875, + "7": 223.091875, + "8": 242.26153125, + "9": 257.8106484375, + "11": 296.949046875, + "12": 306.7407578125, + "13": 325.3369921875, + "14": 353.14478125, + "15": 373.4759765625, + "16": 378.2594296875, + "17": 402.084828125, + "18": 419.432109375, + "19": 421.6563125, + "20": 443.6908984375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.9036640625, + "2": 115.269109375, + "3": 144.2801953125, + "4": 159.3698984375, + "5": 173.026875, + "6": 198.258796875, + "7": 223.091875, + "8": 242.26153125, + "9": 257.8106484375, + "11": 296.949046875, + "12": 306.7407578125, + "13": 325.3369921875, + "14": 353.14478125, + "15": 373.4759765625, + "16": 378.2594296875, + "17": 402.084828125, + "18": 419.432109375, + "19": 421.6563125, + "20": 443.6908984375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 676633, + "answer_accuracy": 0.8356299893297502, + "avg_response_len": 276.8894900509801, + "resp_tokens_sum": 224205173, + "pass_at_k": { + "pass@1": 0.8356299893297502, + "pass@2": 0.8748209761426936, + "pass@4": 0.9020187386472343, + "pass@8": 0.9209896497859279, + "pass@16": 0.9346778150804824, + "pass@32": 0.9451618942195561, + "pass@64": 0.9534618266585658, + "pass@128": 0.9603224786595005 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9463956489675516, + "pass@2": 0.9599948029173347, + "pass@4": 0.9696439955514411, + "pass@8": 0.9772987780870966, + "pass@16": 0.9829576313879089, + "pass@32": 0.9869532680938505, + "pass@64": 0.9904289041602642, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999513239875389, + "pass@2": 0.9999996167243114, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9962890625, + "pass@2": 0.9968965305118112, + "pass@4": 0.9969725551884139, + "pass@8": 0.9970703124996394, + "pass@16": 0.997265625, + "pass@32": 0.99765625, + "pass@64": 0.9984375, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941220238095239, + "pass@2": 0.9976346706661666, + "pass@4": 0.9991279667422525, + "pass@8": 0.999810090374339, + "pass@16": 0.9999911414698637, + "pass@32": 0.9999999920410901, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9975673652694611, + "pass@2": 0.9990827891461174, + "pass@4": 0.9997700759897528, + "pass@8": 0.9999845197580833, + "pass@16": 0.9999999528630369, + "pass@32": 0.9999999999999486, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960012943786982, + "pass@2": 0.9984763022410661, + "pass@4": 0.9996465231971745, + "pass@8": 0.9999815406455984, + "pass@16": 0.9999999678464966, + "pass@32": 0.9999999999999909, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9963441506410257, + "pass@2": 0.9996159209065214, + "pass@4": 0.9999852656639074, + "pass@8": 0.9999999592877328, + "pass@16": 0.9999999999999242, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9913292253521127, + "pass@2": 0.9970926444493733, + "pass@4": 0.9988132281880259, + "pass@8": 0.9997344814601936, + "pass@16": 0.9999856314389629, + "pass@32": 0.9999999749550461, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9385844546742209, + "pass@2": 0.9559055815172538, + "pass@4": 0.9662483132420059, + "pass@8": 0.9743145176794873, + "pass@16": 0.9814528957415205, + "pass@32": 0.9870929428344121, + "pass@64": 0.9904799574420998, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8917284604519774, + "pass@2": 0.9156980542951199, + "pass@4": 0.9281065293215466, + "pass@8": 0.935889004485147, + "pass@16": 0.9418117371185404, + "pass@32": 0.947600022930526, + "pass@64": 0.9542037056880797, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8402233542319749, + "pass@2": 0.8852274269987415, + "pass@4": 0.9120427055748123, + "pass@8": 0.9300158215455117, + "pass@16": 0.9440746211517979, + "pass@32": 0.9571960077579881, + "pass@64": 0.9683369401551648, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8298773364485982, + "pass@2": 0.8854661092305048, + "pass@4": 0.9165421769708681, + "pass@8": 0.9342494936091128, + "pass@16": 0.9469401578089754, + "pass@32": 0.9577873713575069, + "pass@64": 0.9667747235355996, + "pass@128": 0.9750778816199377 + }, + "15": { + "pass@1": 0.7909365968208093, + "pass@2": 0.8548815057348322, + "pass@4": 0.8965234123552469, + "pass@8": 0.9232631518428708, + "pass@16": 0.93956332213618, + "pass@32": 0.9494509870773286, + "pass@64": 0.9560635736921728, + "pass@128": 0.9624277456647399 + }, + "16": { + "pass@1": 0.6512096774193549, + "pass@2": 0.7479818865887731, + "pass@4": 0.8152397825271837, + "pass@8": 0.8589674413513186, + "pass@16": 0.8885187025219686, + "pass@32": 0.9122664817583825, + "pass@64": 0.9310470742858282, + "pass@128": 0.9419354838709677 + }, + "17": { + "pass@1": 0.6124909682080925, + "pass@2": 0.7055230474261528, + "pass@4": 0.768419161297034, + "pass@8": 0.8132989938840093, + "pass@16": 0.8486630253769624, + "pass@32": 0.8756942867659188, + "pass@64": 0.8953656362346946, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.5464732142857143, + "pass@2": 0.6480627812148483, + "pass@4": 0.7236771966896989, + "pass@8": 0.777945603229985, + "pass@16": 0.8163166798894307, + "pass@32": 0.8431802277020857, + "pass@64": 0.8622731277447071, + "pass@128": 0.8771428571428571 + }, + "19": { + "pass@1": 0.4315643768768769, + "pass@2": 0.5412484334728428, + "pass@4": 0.6303853890448375, + "pass@8": 0.6978895463021549, + "pass@16": 0.7471806522306299, + "pass@32": 0.7842191914351475, + "pass@64": 0.8144645683656517, + "pass@128": 0.8438438438438438 + }, + "20": { + "pass@1": 0.4473066816816817, + "pass@2": 0.550585157007401, + "pass@4": 0.6322523291007539, + "pass@8": 0.6917831399797562, + "pass@16": 0.7350186424328486, + "pass@32": 0.7688185050754536, + "pass@64": 0.7971501932040521, + "pass@128": 0.8228228228228228 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 686415, + "answer_accuracy": 0.8359496784879189, + "avg_response_len": 286.5450993764614, + "resp_tokens_sum": 235287912, + "pass_at_k": { + "pass@1": 0.8359496784879189, + "pass@2": 0.8760342317157656, + "pass@4": 0.9043035755711732, + "pass@8": 0.9240919493445696, + "pass@16": 0.9381619892173285, + "pass@32": 0.9489126547729012, + "pass@64": 0.9576937527310582, + "pass@128": 0.9649259547934529 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.924079754601227, + "pass@2": 0.9425480653108548, + "pass@4": 0.9558939233439372, + "pass@8": 0.9660136894297928, + "pass@16": 0.9745849213182081, + "pass@32": 0.9815053815983932, + "pass@64": 0.9856196459168264, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150811, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9920969202898551, + "pass@2": 0.9947952327969871, + "pass@4": 0.9963463162213417, + "pass@8": 0.9977460600531718, + "pass@16": 0.9990916148451592, + "pass@32": 0.9998484564379228, + "pass@64": 0.999998063383779, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9952542979942693, + "pass@2": 0.9966728617196489, + "pass@4": 0.9972907742728435, + "pass@8": 0.997639465969732, + "pass@16": 0.9980869583411133, + "pass@32": 0.9988007408816018, + "pass@64": 0.9996502944295288, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967937138728323, + "pass@2": 0.9983024498202175, + "pass@4": 0.9990612040546953, + "pass@8": 0.9997078494159238, + "pass@16": 0.999975444312545, + "pass@32": 0.9999999133144741, + "pass@64": 0.9999999999999895, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9918573943661971, + "pass@2": 0.9956231978485083, + "pass@4": 0.9969242494336096, + "pass@8": 0.9971736180162805, + "pass@16": 0.9971830890857465, + "pass@32": 0.9971830985915483, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9910179640718563, + "pass@2": 0.9937379178650573, + "pass@4": 0.9939987547402381, + "pass@8": 0.9940119380249567, + "pass@16": 0.9940119760478334, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9533838478915663, + "pass@2": 0.9686280802343229, + "pass@4": 0.9774122846842942, + "pass@8": 0.9826945678160793, + "pass@16": 0.9851505016991625, + "pass@32": 0.9862401453810479, + "pass@64": 0.9872047186881181, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.885384012539185, + "pass@2": 0.9101669525831221, + "pass@4": 0.9239799511738148, + "pass@8": 0.9333532233276649, + "pass@16": 0.9404962983721141, + "pass@32": 0.947095869663446, + "pass@64": 0.9545776664683873, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.8328956582633054, + "pass@2": 0.8768782119146881, + "pass@4": 0.9001525788793208, + "pass@8": 0.9144132901889223, + "pass@16": 0.9252804791130302, + "pass@32": 0.9358350221164605, + "pass@64": 0.9474943865376289, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.8158351608187134, + "pass@2": 0.8764760182115391, + "pass@4": 0.9120771337245117, + "pass@8": 0.9326000038402523, + "pass@16": 0.9449230559731131, + "pass@32": 0.9539728291687525, + "pass@64": 0.9628130945239121, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.7901515151515152, + "pass@2": 0.8613483506323071, + "pass@4": 0.9039200494824501, + "pass@8": 0.9292158194336342, + "pass@16": 0.9452248133475442, + "pass@32": 0.9574867632372372, + "pass@64": 0.9678628579175225, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.6649528133903134, + "pass@2": 0.7624254800681963, + "pass@4": 0.8277979675617472, + "pass@8": 0.8686706781122179, + "pass@16": 0.8961598925297205, + "pass@32": 0.9179229188572494, + "pass@64": 0.9337860632473033, + "pass@128": 0.9430199430199431 + }, + "17": { + "pass@1": 0.6393390916149069, + "pass@2": 0.7354016634469608, + "pass@4": 0.8004865514675568, + "pass@8": 0.8453401336925654, + "pass@16": 0.8772971858982275, + "pass@32": 0.9004220002674332, + "pass@64": 0.9174945886997983, + "pass@128": 0.9316770186335404 + }, + "18": { + "pass@1": 0.5892693014705882, + "pass@2": 0.6895900880037054, + "pass@4": 0.7655650637420315, + "pass@8": 0.8203016454124229, + "pass@16": 0.859618198955504, + "pass@32": 0.8878937646484227, + "pass@64": 0.9097340634600377, + "pass@128": 0.9294117647058824 + }, + "19": { + "pass@1": 0.4204799107142857, + "pass@2": 0.5389387361736032, + "pass@4": 0.6410350558858714, + "pass@8": 0.7190375968217035, + "pass@16": 0.7736645475875377, + "pass@32": 0.8123192037757934, + "pass@64": 0.8394299819521862, + "pass@128": 0.8571428571428571 + }, + "20": { + "pass@1": 0.3888005239520958, + "pass@2": 0.4932697351360271, + "pass@4": 0.581708334175294, + "pass@8": 0.6533021743606491, + "pass@16": 0.7090402348266315, + "pass@32": 0.754332532451074, + "pass@64": 0.7954232145576876, + "pass@128": 0.8293413173652695 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 661822, + "answer_accuracy": 0.8260879333759387, + "avg_response_len": 307.27815945039146, + "resp_tokens_sum": 246176512, + "pass_at_k": { + "pass@1": 0.8260879333759387, + "pass@2": 0.8676958919313628, + "pass@4": 0.8980069189605794, + "pass@8": 0.9198953723230876, + "pass@16": 0.93505318616042, + "pass@32": 0.94562924717024, + "pass@64": 0.9530547716086694, + "pass@128": 0.9583000479309793 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9465951492537313, + "pass@2": 0.9632176225173348, + "pass@4": 0.9735499216515845, + "pass@8": 0.9811835925161799, + "pass@16": 0.9873330366748251, + "pass@32": 0.9918809176478053, + "pass@64": 0.993806150809024, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999316690962099, + "pass@2": 0.9999996413075917, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981953581871345, + "pass@2": 0.9990279792328591, + "pass@4": 0.9996846265927285, + "pass@8": 0.9999691763603531, + "pass@16": 0.9999997893064115, + "pass@32": 0.9999999999982297, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9918198529411765, + "pass@2": 0.9951822313571099, + "pass@4": 0.9976438364322108, + "pass@8": 0.9994046739409391, + "pass@16": 0.9999673976331993, + "pass@32": 0.9999999551856581, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.993764787066246, + "pass@2": 0.9947496212027123, + "pass@4": 0.9955808440940151, + "pass@8": 0.9967326641590364, + "pass@16": 0.9979618214750134, + "pass@32": 0.9990096140434843, + "pass@64": 0.9998121152924854, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956487341772152, + "pass@2": 0.9986505469450812, + "pass@4": 0.9995743400943553, + "pass@8": 0.9999481010918784, + "pass@16": 0.9999994169232284, + "pass@32": 0.9999999999832521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9844923048048048, + "pass@2": 0.9919176065829609, + "pass@4": 0.9963254145596664, + "pass@8": 0.9990655284449819, + "pass@16": 0.9999326120862155, + "pass@32": 0.9999996834444386, + "pass@64": 0.999999999999752, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9835962620578779, + "pass@2": 0.9923585968554577, + "pass@4": 0.9956467118660005, + "pass@8": 0.9975453735871543, + "pass@16": 0.9990224437782512, + "pass@32": 0.9998343032263491, + "pass@64": 0.9999978524135079, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9548611111111112, + "pass@2": 0.9714758311461071, + "pass@4": 0.9792459415787311, + "pass@8": 0.9832824257084963, + "pass@16": 0.9859409115458316, + "pass@32": 0.9888924101781106, + "pass@64": 0.9916644760868305, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8987241972477065, + "pass@2": 0.9262153404849626, + "pass@4": 0.9399879209479548, + "pass@8": 0.9515969150881995, + "pass@16": 0.9632921895262051, + "pass@32": 0.9727991681405372, + "pass@64": 0.978484053984944, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7942467206790124, + "pass@2": 0.8419292098036356, + "pass@4": 0.87347388607674, + "pass@8": 0.896955802177486, + "pass@16": 0.9148879344767462, + "pass@32": 0.9305686019139384, + "pass@64": 0.9444548045895704, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.7969445474777448, + "pass@2": 0.8519073167597371, + "pass@4": 0.8856038931706228, + "pass@8": 0.9087465231648425, + "pass@16": 0.9243278096477272, + "pass@32": 0.9348198946343538, + "pass@64": 0.9430984318055536, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.7702064043209876, + "pass@2": 0.8314692542772435, + "pass@4": 0.8710929519226761, + "pass@8": 0.9004926868932659, + "pass@16": 0.9209909756435962, + "pass@32": 0.9332392643870973, + "pass@64": 0.9400295244557415, + "pass@128": 0.9444444444444444 + }, + "16": { + "pass@1": 0.6661366150442478, + "pass@2": 0.7643184708382698, + "pass@4": 0.8284491338472064, + "pass@8": 0.8694132293085024, + "pass@16": 0.8955625569041571, + "pass@32": 0.9130243917210557, + "pass@64": 0.926146214710774, + "pass@128": 0.9380530973451328 + }, + "17": { + "pass@1": 0.5658179593373494, + "pass@2": 0.6702137042263541, + "pass@4": 0.7485751202221707, + "pass@8": 0.8014946697752476, + "pass@16": 0.8341101603053381, + "pass@32": 0.8534621646954216, + "pass@64": 0.8669435177559548, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.55859375, + "pass@2": 0.6654674403098803, + "pass@4": 0.7465657682910599, + "pass@8": 0.801548579552477, + "pass@16": 0.8354467036205091, + "pass@32": 0.8565727072192222, + "pass@64": 0.8703706077422788, + "pass@128": 0.8806451612903226 + }, + "19": { + "pass@1": 0.3890436178247734, + "pass@2": 0.5075246211670672, + "pass@4": 0.6101444158604039, + "pass@8": 0.6887828332064626, + "pass@16": 0.7462233374111003, + "pass@32": 0.7901599832812849, + "pass@64": 0.8223790409544078, + "pass@128": 0.8429003021148036 + }, + "20": { + "pass@1": 0.39846096096096095, + "pass@2": 0.5144094635974552, + "pass@4": 0.6162668182805975, + "pass@8": 0.6980383217113035, + "pass@16": 0.7577338561538128, + "pass@32": 0.799757548916951, + "pass@64": 0.828102483308844, + "pass@128": 0.8468468468468469 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block16-epoch4-op17-20/graphs/SDAR-100M-block16-epoch4-op17-20_score_report.jsonl b/composition/SDAR-100M-block16-epoch4-op17-20/graphs/SDAR-100M-block16-epoch4-op17-20_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..45ee4aa520e76406492c46cf926db44308bcc781 --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op17-20/graphs/SDAR-100M-block16-epoch4-op17-20_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block16-epoch4-op17-20", "results_dir": "results/composition/SDAR-100M-block16-epoch4-op17-20/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1011416, "value_mismatch": 13774, "dependency_mismatch": 134064, "answer_mismatch": 12640, "extra_nodes": 22254, "missing_nodes": 19662, "perfect_with_extra": 996, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 336163, "value_mismatch": 4058, "answer_mismatch": 3534, "dependency_mismatch": 41686, "extra_nodes": 4478, "perfect_with_extra": 143, "missing_nodes": 4097}, "teachers_in_school": {"total": 390656, "perfect": 345773, "value_mismatch": 4692, "answer_mismatch": 4635, "dependency_mismatch": 43527, "extra_nodes": 4401, "missing_nodes": 3547, "perfect_with_extra": 453}, "movie_festival_awards": {"total": 382336, "perfect": 329480, "dependency_mismatch": 48851, "extra_nodes": 13375, "missing_nodes": 12018, "value_mismatch": 5024, "answer_mismatch": 4471, "perfect_with_extra": 400}}, "results_path": "results/composition/SDAR-100M-block16-epoch4-op17-20/graphs/SDAR-100M-block16-epoch4-op17-20_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 344892, "value_mismatch": 475883, "dependency_mismatch": 672848, "answer_mismatch": 394492, "extra_nodes": 435568, "missing_nodes": 567564, "perfect_with_extra": 423, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 121137, "value_mismatch": 157727, "dependency_mismatch": 224763, "answer_mismatch": 129562, "extra_nodes": 132251, "missing_nodes": 178988, "perfect_with_extra": 102}, "movie_festival_awards": {"total": 418816, "value_mismatch": 161056, "answer_mismatch": 134860, "perfect": 94905, "dependency_mismatch": 230015, "extra_nodes": 180819, "missing_nodes": 220428, "perfect_with_extra": 202}, "teachers_in_school": {"total": 430464, "perfect": 128850, "value_mismatch": 157100, "dependency_mismatch": 218070, "answer_mismatch": 130070, "extra_nodes": 122498, "missing_nodes": 168148, "perfect_with_extra": 119}}, "results_path": "results/composition/SDAR-100M-block16-epoch4-op17-20/graphs/SDAR-100M-block16-epoch4-op17-20_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1356308, "value_mismatch": 489657, "dependency_mismatch": 806912, "answer_mismatch": 407132, "extra_nodes": 457822, "missing_nodes": 587226, "perfect_with_extra": 1419, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block16-epoch4-op17-20/graphs/SDAR-100M-block16-epoch4-op17-20_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 457300, "value_mismatch": 161785, "answer_mismatch": 133096, "dependency_mismatch": 266449, "extra_nodes": 136729, "perfect_with_extra": 245, "missing_nodes": 183085}, "teachers_in_school": {"total": 821120, "perfect": 474623, "value_mismatch": 161792, "answer_mismatch": 134705, "dependency_mismatch": 261597, "extra_nodes": 126899, "missing_nodes": 171695, "perfect_with_extra": 572}, "movie_festival_awards": {"total": 801152, "perfect": 424385, "dependency_mismatch": 278866, "extra_nodes": 194194, "missing_nodes": 232446, "value_mismatch": 166080, "answer_mismatch": 139331, "perfect_with_extra": 602}}}} diff --git a/composition/SDAR-100M-block16-epoch4-op17-20/graphs/results.json b/composition/SDAR-100M-block16-epoch4-op17-20/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..f1962f56ffb646b0fd1f4b62e75c8826340c9206 --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op17-20/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block16-epoch4-op17-20_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 0.9968847352024922, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9938837920489296, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9875, + "pass@2": 0.996875, + "pass@4": 0.996875, + "pass@8": 0.996875, + "pass@16": 0.996875, + "pass@32": 0.996875, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9587301587301588, + "pass@2": 0.9841269841269841, + "pass@4": 0.9904761904761905, + "pass@8": 0.9936507936507937, + "pass@16": 0.9968253968253968, + "pass@32": 0.9968253968253968, + "pass@64": 0.9968253968253968, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9491017964071856, + "pass@2": 0.9700598802395209, + "pass@4": 0.9880239520958084, + "pass@8": 0.9940119760479041, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8994082840236687, + "pass@2": 0.9349112426035503, + "pass@4": 0.9644970414201184, + "pass@8": 0.9940828402366864, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8269230769230769, + "pass@2": 0.8878205128205128, + "pass@4": 0.9647435897435898, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7323943661971831, + "pass@2": 0.7915492957746478, + "pass@4": 0.8985915492957747, + "pass@8": 0.9633802816901409, + "pass@16": 0.9887323943661972, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6578171091445427, + "pass@2": 0.7197640117994101, + "pass@4": 0.799410029498525, + "pass@8": 0.8702064896755162, + "pass@16": 0.9203539823008849, + "pass@32": 0.9380530973451328, + "pass@64": 0.943952802359882, + "pass@128": 0.9616519174041298 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9968847352024922, + "3": 0.9938837920489296, + "4": 0.9875, + "5": 0.9587301587301588, + "6": 0.9491017964071856, + "7": 0.8994082840236687, + "8": 0.8269230769230769, + "9": 0.7323943661971831, + "10": 0.6578171091445427 + }, + "pass@2": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.996875, + "5": 0.9841269841269841, + "6": 0.9700598802395209, + "7": 0.9349112426035503, + "8": 0.8878205128205128, + "9": 0.7915492957746478, + "10": 0.7197640117994101 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9904761904761905, + "6": 0.9880239520958084, + "7": 0.9644970414201184, + "8": 0.9647435897435898, + "9": 0.8985915492957747, + "10": 0.799410029498525 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 0.9940119760479041, + "7": 0.9940828402366864, + "8": 1.0, + "9": 0.9633802816901409, + "10": 0.8702064896755162 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9887323943661972, + "10": 0.9203539823008849 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9380530973451328 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.943952802359882 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9616519174041298 + } + }, + "total_pass_at_k": { + "pass@1": 0.8865248226950354, + "pass@2": 0.9179331306990881, + "pass@4": 0.9544072948328267, + "pass@8": 0.9783856805133401, + "pass@16": 0.9888551165146909, + "pass@32": 0.9922323539344816, + "pass@64": 0.9932455251604188, + "pass@128": 0.9956095913542722 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9885714285714285, + "pass@2": 0.9914285714285714, + "pass@4": 0.9942857142857143, + "pass@8": 0.9971428571428571, + "pass@16": 0.9971428571428571, + "pass@32": 0.9971428571428571, + "pass@64": 0.9971428571428571, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9766763848396501, + "pass@2": 0.9941690962099126, + "pass@4": 0.9970845481049563, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9678362573099415, + "pass@2": 0.9883040935672515, + "pass@4": 0.9941520467836257, + "pass@8": 0.9941520467836257, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9235294117647059, + "pass@2": 0.9588235294117647, + "pass@4": 0.9941176470588236, + "pass@8": 0.9970588235294118, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9463722397476341, + "pass@2": 0.9716088328075709, + "pass@4": 0.9842271293375394, + "pass@8": 0.9905362776025236, + "pass@16": 0.9905362776025236, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8670886075949367, + "pass@2": 0.9208860759493671, + "pass@4": 0.9588607594936709, + "pass@8": 0.9841772151898734, + "pass@16": 0.9968354430379747, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7657657657657657, + "pass@2": 0.8618618618618619, + "pass@4": 0.9279279279279279, + "pass@8": 0.972972972972973, + "pass@16": 0.993993993993994, + "pass@32": 0.996996996996997, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.752411575562701, + "pass@2": 0.8327974276527331, + "pass@4": 0.909967845659164, + "pass@8": 0.9581993569131833, + "pass@16": 0.9807073954983923, + "pass@32": 0.9967845659163987, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.5552238805970149, + "pass@2": 0.6358208955223881, + "pass@4": 0.7343283582089553, + "pass@8": 0.835820895522388, + "pass@16": 0.8925373134328358, + "pass@32": 0.9343283582089552, + "pass@64": 0.9492537313432836, + "pass@128": 0.9522388059701492 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9885714285714285, + "3": 0.9766763848396501, + "4": 0.9678362573099415, + "5": 0.9235294117647059, + "6": 0.9463722397476341, + "7": 0.8670886075949367, + "8": 0.7657657657657657, + "9": 0.752411575562701, + "10": 0.5552238805970149 + }, + "pass@2": { + "2": 0.9914285714285714, + "3": 0.9941690962099126, + "4": 0.9883040935672515, + "5": 0.9588235294117647, + "6": 0.9716088328075709, + "7": 0.9208860759493671, + "8": 0.8618618618618619, + "9": 0.8327974276527331, + "10": 0.6358208955223881 + }, + "pass@4": { + "2": 0.9942857142857143, + "3": 0.9970845481049563, + "4": 0.9941520467836257, + "5": 0.9941176470588236, + "6": 0.9842271293375394, + "7": 0.9588607594936709, + "8": 0.9279279279279279, + "9": 0.909967845659164, + "10": 0.7343283582089553 + }, + "pass@8": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9970588235294118, + "6": 0.9905362776025236, + "7": 0.9841772151898734, + "8": 0.972972972972973, + "9": 0.9581993569131833, + "10": 0.835820895522388 + }, + "pass@16": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 0.9970760233918129, + "5": 1.0, + "6": 0.9905362776025236, + "7": 0.9968354430379747, + "8": 0.993993993993994, + "9": 0.9807073954983923, + "10": 0.8925373134328358 + }, + "pass@32": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 0.9970760233918129, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 0.9967845659163987, + "10": 0.9343283582089552 + }, + "pass@64": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 0.9970760233918129, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9492537313432836 + }, + "pass@128": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9522388059701492 + } + }, + "total_pass_at_k": { + "pass@1": 0.8620689655172413, + "pass@2": 0.9072648141948443, + "pass@4": 0.9444258453297623, + "pass@8": 0.9702042182792099, + "pass@16": 0.983260796786073, + "pass@32": 0.9912956143287579, + "pass@64": 0.9933043187144291, + "pass@128": 0.9939738868429863 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9969604863221885, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996969696969697, + "pass@2": 0.996969696969697, + "pass@4": 0.996969696969697, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985207100591716, + "pass@2": 0.9911242603550295, + "pass@4": 0.9970414201183432, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9594202898550724, + "pass@2": 0.9768115942028985, + "pass@4": 0.9826086956521739, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9713467048710601, + "pass@2": 0.9828080229226361, + "pass@4": 0.9914040114613181, + "pass@8": 0.994269340974212, + "pass@16": 0.997134670487106, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8554913294797688, + "pass@2": 0.9075144508670521, + "pass@4": 0.9566473988439307, + "pass@8": 0.9826589595375722, + "pass@16": 0.9971098265895953, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8366197183098592, + "pass@2": 0.8985915492957747, + "pass@4": 0.9436619718309859, + "pass@8": 0.9746478873239437, + "pass@16": 0.9887323943661972, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.7305389221556886, + "pass@2": 0.7964071856287425, + "pass@4": 0.9011976047904192, + "pass@8": 0.9550898203592815, + "pass@16": 0.9850299401197605, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "10": { + "pass@1": 0.6503067484662577, + "pass@2": 0.7116564417177914, + "pass@4": 0.8006134969325154, + "pass@8": 0.8650306748466258, + "pass@16": 0.8987730061349694, + "pass@32": 0.9325153374233128, + "pass@64": 0.9539877300613497, + "pass@128": 0.9631901840490797 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9969604863221885, + "3": 0.996969696969697, + "4": 0.985207100591716, + "5": 0.9594202898550724, + "6": 0.9713467048710601, + "7": 0.8554913294797688, + "8": 0.8366197183098592, + "9": 0.7305389221556886, + "10": 0.6503067484662577 + }, + "pass@2": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9911242603550295, + "5": 0.9768115942028985, + "6": 0.9828080229226361, + "7": 0.9075144508670521, + "8": 0.8985915492957747, + "9": 0.7964071856287425, + "10": 0.7116564417177914 + }, + "pass@4": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9970414201183432, + "5": 0.9826086956521739, + "6": 0.9914040114613181, + "7": 0.9566473988439307, + "8": 0.9436619718309859, + "9": 0.9011976047904192, + "10": 0.8006134969325154 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.994269340974212, + "7": 0.9826589595375722, + "8": 0.9746478873239437, + "9": 0.9550898203592815, + "10": 0.8650306748466258 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9971098265895953, + "8": 0.9887323943661972, + "9": 0.9850299401197605, + "10": 0.8987730061349694 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9940119760479041, + "10": 0.9325153374233128 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9940119760479041, + "10": 0.9539877300613497 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9940119760479041, + "10": 0.9631901840490797 + } + }, + "total_pass_at_k": { + "pass@1": 0.8876146788990825, + "pass@2": 0.9187418086500655, + "pass@4": 0.9528178243774574, + "pass@8": 0.9750982961992136, + "pass@16": 0.9855832241153342, + "pass@32": 0.9918086500655308, + "pass@64": 0.9941022280471822, + "pass@128": 0.9950851900393185 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.994, + "pass@2": 0.996, + "pass@4": 0.998, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.989, + "pass@2": 0.997, + "pass@4": 0.998, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.98, + "pass@2": 0.992, + "pass@4": 0.996, + "pass@8": 0.997, + "pass@16": 0.998, + "pass@32": 0.998, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.947, + "pass@2": 0.973, + "pass@4": 0.989, + "pass@8": 0.997, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.956, + "pass@2": 0.975, + "pass@4": 0.988, + "pass@8": 0.993, + "pass@16": 0.996, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.874, + "pass@2": 0.921, + "pass@4": 0.96, + "pass@8": 0.987, + "pass@16": 0.998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.81, + "pass@2": 0.883, + "pass@4": 0.945, + "pass@8": 0.982, + "pass@16": 0.994, + "pass@32": 0.998, + "pass@64": 0.998, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.738, + "pass@2": 0.806, + "pass@4": 0.903, + "pass@8": 0.959, + "pass@16": 0.985, + "pass@32": 0.997, + "pass@64": 0.998, + "pass@128": 0.998 + }, + "10": { + "pass@1": 0.621, + "pass@2": 0.689, + "pass@4": 0.778, + "pass@8": 0.857, + "pass@16": 0.904, + "pass@32": 0.935, + "pass@64": 0.949, + "pass@128": 0.959 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.994, + "3": 0.989, + "4": 0.98, + "5": 0.947, + "6": 0.956, + "7": 0.874, + "8": 0.81, + "9": 0.738, + "10": 0.621 + }, + "pass@2": { + "2": 0.996, + "3": 0.997, + "4": 0.992, + "5": 0.973, + "6": 0.975, + "7": 0.921, + "8": 0.883, + "9": 0.806, + "10": 0.689 + }, + "pass@4": { + "2": 0.998, + "3": 0.998, + "4": 0.996, + "5": 0.989, + "6": 0.988, + "7": 0.96, + "8": 0.945, + "9": 0.903, + "10": 0.778 + }, + "pass@8": { + "2": 0.999, + "3": 1.0, + "4": 0.997, + "5": 0.997, + "6": 0.993, + "7": 0.987, + "8": 0.982, + "9": 0.959, + "10": 0.857 + }, + "pass@16": { + "2": 0.999, + "3": 1.0, + "4": 0.998, + "5": 0.999, + "6": 0.996, + "7": 0.998, + "8": 0.994, + "9": 0.985, + "10": 0.904 + }, + "pass@32": { + "2": 0.999, + "3": 1.0, + "4": 0.998, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.998, + "9": 0.997, + "10": 0.935 + }, + "pass@64": { + "2": 0.999, + "3": 1.0, + "4": 0.999, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.998, + "9": 0.998, + "10": 0.949 + }, + "pass@128": { + "2": 0.999, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.998, + "9": 0.998, + "10": 0.959 + } + }, + "total_pass_at_k": { + "pass@1": 0.8787777777777778, + "pass@2": 0.9146666666666666, + "pass@4": 0.9505555555555556, + "pass@8": 0.9745555555555555, + "pass@16": 0.9858888888888889, + "pass@32": 0.9917777777777778, + "pass@64": 0.9935555555555555, + "pass@128": 0.9948888888888889 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block16-epoch4-op17-20_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.56657223796034, + "pass@2": 0.6260623229461756, + "pass@4": 0.7025495750708215, + "pass@8": 0.8186968838526912, + "pass@16": 0.8668555240793201, + "pass@32": 0.9093484419263456, + "pass@64": 0.9348441926345609, + "pass@128": 0.9461756373937678 + }, + "12": { + "pass@1": 0.576271186440678, + "pass@2": 0.655367231638418, + "pass@4": 0.7401129943502824, + "pass@8": 0.7937853107344632, + "pass@16": 0.827683615819209, + "pass@32": 0.8587570621468926, + "pass@64": 0.864406779661017, + "pass@128": 0.8757062146892656 + }, + "13": { + "pass@1": 0.4043887147335423, + "pass@2": 0.5172413793103449, + "pass@4": 0.5893416927899686, + "pass@8": 0.6896551724137931, + "pass@16": 0.7492163009404389, + "pass@32": 0.780564263322884, + "pass@64": 0.8150470219435737, + "pass@128": 0.8401253918495298 + }, + "14": { + "pass@1": 0.3395638629283489, + "pass@2": 0.42990654205607476, + "pass@4": 0.4953271028037383, + "pass@8": 0.5638629283489096, + "pass@16": 0.6386292834890965, + "pass@32": 0.7165109034267912, + "pass@64": 0.7570093457943925, + "pass@128": 0.7850467289719626 + }, + "15": { + "pass@1": 0.3468208092485549, + "pass@2": 0.3901734104046243, + "pass@4": 0.4653179190751445, + "pass@8": 0.5346820809248555, + "pass@16": 0.6011560693641619, + "pass@32": 0.6820809248554913, + "pass@64": 0.7716763005780347, + "pass@128": 0.8121387283236994 + }, + "16": { + "pass@1": 0.14516129032258066, + "pass@2": 0.1870967741935484, + "pass@4": 0.23225806451612904, + "pass@8": 0.27741935483870966, + "pass@16": 0.33225806451612905, + "pass@32": 0.3967741935483871, + "pass@64": 0.45483870967741935, + "pass@128": 0.49032258064516127 + }, + "17": { + "pass@1": 0.15895953757225434, + "pass@2": 0.2023121387283237, + "pass@4": 0.22832369942196531, + "pass@8": 0.2745664739884393, + "pass@16": 0.34104046242774566, + "pass@32": 0.3901734104046243, + "pass@64": 0.4277456647398844, + "pass@128": 0.5057803468208093 + }, + "18": { + "pass@1": 0.12, + "pass@2": 0.1742857142857143, + "pass@4": 0.20857142857142857, + "pass@8": 0.24, + "pass@16": 0.2914285714285714, + "pass@32": 0.3342857142857143, + "pass@64": 0.4, + "pass@128": 0.4657142857142857 + }, + "19": { + "pass@1": 0.03903903903903904, + "pass@2": 0.06006006006006006, + "pass@4": 0.06906906906906907, + "pass@8": 0.07807807807807808, + "pass@16": 0.1021021021021021, + "pass@32": 0.12312312312312312, + "pass@64": 0.15315315315315314, + "pass@128": 0.1891891891891892 + }, + "20": { + "pass@1": 0.057057057057057055, + "pass@2": 0.07507507507507508, + "pass@4": 0.10510510510510511, + "pass@8": 0.12012012012012012, + "pass@16": 0.15315315315315314, + "pass@32": 0.1921921921921922, + "pass@64": 0.2072072072072072, + "pass@128": 0.24324324324324326 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.56657223796034, + "12": 0.576271186440678, + "13": 0.4043887147335423, + "14": 0.3395638629283489, + "15": 0.3468208092485549, + "16": 0.14516129032258066, + "17": 0.15895953757225434, + "18": 0.12, + "19": 0.03903903903903904, + "20": 0.057057057057057055 + }, + "pass@2": { + "11": 0.6260623229461756, + "12": 0.655367231638418, + "13": 0.5172413793103449, + "14": 0.42990654205607476, + "15": 0.3901734104046243, + "16": 0.1870967741935484, + "17": 0.2023121387283237, + "18": 0.1742857142857143, + "19": 0.06006006006006006, + "20": 0.07507507507507508 + }, + "pass@4": { + "11": 0.7025495750708215, + "12": 0.7401129943502824, + "13": 0.5893416927899686, + "14": 0.4953271028037383, + "15": 0.4653179190751445, + "16": 0.23225806451612904, + "17": 0.22832369942196531, + "18": 0.20857142857142857, + "19": 0.06906906906906907, + "20": 0.10510510510510511 + }, + "pass@8": { + "11": 0.8186968838526912, + "12": 0.7937853107344632, + "13": 0.6896551724137931, + "14": 0.5638629283489096, + "15": 0.5346820809248555, + "16": 0.27741935483870966, + "17": 0.2745664739884393, + "18": 0.24, + "19": 0.07807807807807808, + "20": 0.12012012012012012 + }, + "pass@16": { + "11": 0.8668555240793201, + "12": 0.827683615819209, + "13": 0.7492163009404389, + "14": 0.6386292834890965, + "15": 0.6011560693641619, + "16": 0.33225806451612905, + "17": 0.34104046242774566, + "18": 0.2914285714285714, + "19": 0.1021021021021021, + "20": 0.15315315315315314 + }, + "pass@32": { + "11": 0.9093484419263456, + "12": 0.8587570621468926, + "13": 0.780564263322884, + "14": 0.7165109034267912, + "15": 0.6820809248554913, + "16": 0.3967741935483871, + "17": 0.3901734104046243, + "18": 0.3342857142857143, + "19": 0.12312312312312312, + "20": 0.1921921921921922 + }, + "pass@64": { + "11": 0.9348441926345609, + "12": 0.864406779661017, + "13": 0.8150470219435737, + "14": 0.7570093457943925, + "15": 0.7716763005780347, + "16": 0.45483870967741935, + "17": 0.4277456647398844, + "18": 0.4, + "19": 0.15315315315315314, + "20": 0.2072072072072072 + }, + "pass@128": { + "11": 0.9461756373937678, + "12": 0.8757062146892656, + "13": 0.8401253918495298, + "14": 0.7850467289719626, + "15": 0.8121387283236994, + "16": 0.49032258064516127, + "17": 0.5057803468208093, + "18": 0.4657142857142857, + "19": 0.1891891891891892, + "20": 0.24324324324324326 + } + }, + "total_pass_at_k": { + "pass@1": 0.27815750371471026, + "pass@2": 0.3343239227340267, + "pass@4": 0.3863298662704309, + "pass@8": 0.44190193164933134, + "pass@16": 0.49301634472511147, + "pass@32": 0.5408618127786032, + "pass@64": 0.5809806835066865, + "pass@128": 0.6178306092124815 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5904761904761905, + "pass@2": 0.6634920634920635, + "pass@4": 0.7714285714285715, + "pass@8": 0.8507936507936508, + "pass@16": 0.9079365079365079, + "pass@32": 0.9492063492063492, + "pass@64": 0.9619047619047619, + "pass@128": 0.9714285714285714 + }, + "12": { + "pass@1": 0.4740061162079511, + "pass@2": 0.5779816513761468, + "pass@4": 0.691131498470948, + "pass@8": 0.7767584097859327, + "pass@16": 0.8409785932721713, + "pass@32": 0.8654434250764526, + "pass@64": 0.8837920489296636, + "pass@128": 0.9051987767584098 + }, + "13": { + "pass@1": 0.31790123456790126, + "pass@2": 0.404320987654321, + "pass@4": 0.4691358024691358, + "pass@8": 0.5277777777777778, + "pass@16": 0.6049382716049383, + "pass@32": 0.6790123456790124, + "pass@64": 0.7222222222222222, + "pass@128": 0.7592592592592593 + }, + "14": { + "pass@1": 0.28486646884273, + "pass@2": 0.3649851632047478, + "pass@4": 0.4421364985163205, + "pass@8": 0.49851632047477745, + "pass@16": 0.5608308605341247, + "pass@32": 0.6528189910979229, + "pass@64": 0.744807121661721, + "pass@128": 0.7804154302670623 + }, + "15": { + "pass@1": 0.2623456790123457, + "pass@2": 0.3487654320987654, + "pass@4": 0.4382716049382716, + "pass@8": 0.48148148148148145, + "pass@16": 0.5462962962962963, + "pass@32": 0.6203703703703703, + "pass@64": 0.691358024691358, + "pass@128": 0.7561728395061729 + }, + "16": { + "pass@1": 0.11799410029498525, + "pass@2": 0.15339233038348082, + "pass@4": 0.19469026548672566, + "pass@8": 0.25073746312684364, + "pass@16": 0.3008849557522124, + "pass@32": 0.3333333333333333, + "pass@64": 0.39233038348082594, + "pass@128": 0.4218289085545723 + }, + "17": { + "pass@1": 0.0783132530120482, + "pass@2": 0.12048192771084337, + "pass@4": 0.1536144578313253, + "pass@8": 0.20180722891566266, + "pass@16": 0.23795180722891565, + "pass@32": 0.2740963855421687, + "pass@64": 0.3102409638554217, + "pass@128": 0.3463855421686747 + }, + "18": { + "pass@1": 0.1032258064516129, + "pass@2": 0.15161290322580645, + "pass@4": 0.2, + "pass@8": 0.24838709677419354, + "pass@16": 0.27741935483870966, + "pass@32": 0.3096774193548387, + "pass@64": 0.3387096774193548, + "pass@128": 0.3903225806451613 + }, + "19": { + "pass@1": 0.02416918429003021, + "pass@2": 0.03625377643504532, + "pass@4": 0.04833836858006042, + "pass@8": 0.06948640483383686, + "pass@16": 0.08761329305135952, + "pass@32": 0.10574018126888217, + "pass@64": 0.1268882175226586, + "pass@128": 0.1510574018126888 + }, + "20": { + "pass@1": 0.021021021021021023, + "pass@2": 0.03003003003003003, + "pass@4": 0.04504504504504504, + "pass@8": 0.07207207207207207, + "pass@16": 0.08408408408408409, + "pass@32": 0.1111111111111111, + "pass@64": 0.13813813813813813, + "pass@128": 0.16216216216216217 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5904761904761905, + "12": 0.4740061162079511, + "13": 0.31790123456790126, + "14": 0.28486646884273, + "15": 0.2623456790123457, + "16": 0.11799410029498525, + "17": 0.0783132530120482, + "18": 0.1032258064516129, + "19": 0.02416918429003021, + "20": 0.021021021021021023 + }, + "pass@2": { + "11": 0.6634920634920635, + "12": 0.5779816513761468, + "13": 0.404320987654321, + "14": 0.3649851632047478, + "15": 0.3487654320987654, + "16": 0.15339233038348082, + "17": 0.12048192771084337, + "18": 0.15161290322580645, + "19": 0.03625377643504532, + "20": 0.03003003003003003 + }, + "pass@4": { + "11": 0.7714285714285715, + "12": 0.691131498470948, + "13": 0.4691358024691358, + "14": 0.4421364985163205, + "15": 0.4382716049382716, + "16": 0.19469026548672566, + "17": 0.1536144578313253, + "18": 0.2, + "19": 0.04833836858006042, + "20": 0.04504504504504504 + }, + "pass@8": { + "11": 0.8507936507936508, + "12": 0.7767584097859327, + "13": 0.5277777777777778, + "14": 0.49851632047477745, + "15": 0.48148148148148145, + "16": 0.25073746312684364, + "17": 0.20180722891566266, + "18": 0.24838709677419354, + "19": 0.06948640483383686, + "20": 0.07207207207207207 + }, + "pass@16": { + "11": 0.9079365079365079, + "12": 0.8409785932721713, + "13": 0.6049382716049383, + "14": 0.5608308605341247, + "15": 0.5462962962962963, + "16": 0.3008849557522124, + "17": 0.23795180722891565, + "18": 0.27741935483870966, + "19": 0.08761329305135952, + "20": 0.08408408408408409 + }, + "pass@32": { + "11": 0.9492063492063492, + "12": 0.8654434250764526, + "13": 0.6790123456790124, + "14": 0.6528189910979229, + "15": 0.6203703703703703, + "16": 0.3333333333333333, + "17": 0.2740963855421687, + "18": 0.3096774193548387, + "19": 0.10574018126888217, + "20": 0.1111111111111111 + }, + "pass@64": { + "11": 0.9619047619047619, + "12": 0.8837920489296636, + "13": 0.7222222222222222, + "14": 0.744807121661721, + "15": 0.691358024691358, + "16": 0.39233038348082594, + "17": 0.3102409638554217, + "18": 0.3387096774193548, + "19": 0.1268882175226586, + "20": 0.13813813813813813 + }, + "pass@128": { + "11": 0.9714285714285714, + "12": 0.9051987767584098, + "13": 0.7592592592592593, + "14": 0.7804154302670623, + "15": 0.7561728395061729, + "16": 0.4218289085545723, + "17": 0.3463855421686747, + "18": 0.3903225806451613, + "19": 0.1510574018126888, + "20": 0.16216216216216217 + } + }, + "total_pass_at_k": { + "pass@1": 0.22555012224938875, + "pass@2": 0.2830073349633252, + "pass@4": 0.3429095354523227, + "pass@8": 0.3951711491442543, + "pass@16": 0.44223716381418093, + "pass@32": 0.48746943765281175, + "pass@64": 0.5287286063569682, + "pass@128": 0.562041564792176 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6325301204819277, + "pass@2": 0.6987951807228916, + "pass@4": 0.7831325301204819, + "pass@8": 0.8674698795180723, + "pass@16": 0.9126506024096386, + "pass@32": 0.9518072289156626, + "pass@64": 0.963855421686747, + "pass@128": 0.9759036144578314 + }, + "12": { + "pass@1": 0.5329153605015674, + "pass@2": 0.6018808777429467, + "pass@4": 0.677115987460815, + "pass@8": 0.7492163009404389, + "pass@16": 0.8056426332288401, + "pass@32": 0.8369905956112853, + "pass@64": 0.8714733542319749, + "pass@128": 0.8871473354231975 + }, + "13": { + "pass@1": 0.4565826330532213, + "pass@2": 0.5518207282913166, + "pass@4": 0.6162464985994398, + "pass@8": 0.7058823529411765, + "pass@16": 0.7535014005602241, + "pass@32": 0.8011204481792717, + "pass@64": 0.8291316526610645, + "pass@128": 0.8431372549019608 + }, + "14": { + "pass@1": 0.391812865497076, + "pass@2": 0.4590643274853801, + "pass@4": 0.5263157894736842, + "pass@8": 0.5789473684210527, + "pass@16": 0.6578947368421053, + "pass@32": 0.7222222222222222, + "pass@64": 0.783625730994152, + "pass@128": 0.827485380116959 + }, + "15": { + "pass@1": 0.36363636363636365, + "pass@2": 0.43333333333333335, + "pass@4": 0.5060606060606061, + "pass@8": 0.5606060606060606, + "pass@16": 0.6181818181818182, + "pass@32": 0.6878787878787879, + "pass@64": 0.7363636363636363, + "pass@128": 0.793939393939394 + }, + "16": { + "pass@1": 0.17094017094017094, + "pass@2": 0.2222222222222222, + "pass@4": 0.27350427350427353, + "pass@8": 0.31054131054131057, + "pass@16": 0.36752136752136755, + "pass@32": 0.4472934472934473, + "pass@64": 0.49002849002849, + "pass@128": 0.5441595441595442 + }, + "17": { + "pass@1": 0.14596273291925466, + "pass@2": 0.17701863354037267, + "pass@4": 0.2204968944099379, + "pass@8": 0.3105590062111801, + "pass@16": 0.3695652173913043, + "pass@32": 0.4254658385093168, + "pass@64": 0.48757763975155277, + "pass@128": 0.5217391304347826 + }, + "18": { + "pass@1": 0.15588235294117647, + "pass@2": 0.2, + "pass@4": 0.2323529411764706, + "pass@8": 0.27647058823529413, + "pass@16": 0.32941176470588235, + "pass@32": 0.37058823529411766, + "pass@64": 0.4264705882352941, + "pass@128": 0.4852941176470588 + }, + "19": { + "pass@1": 0.03571428571428571, + "pass@2": 0.06547619047619048, + "pass@4": 0.07738095238095238, + "pass@8": 0.08928571428571429, + "pass@16": 0.11607142857142858, + "pass@32": 0.15476190476190477, + "pass@64": 0.17857142857142858, + "pass@128": 0.19940476190476192 + }, + "20": { + "pass@1": 0.08083832335329341, + "pass@2": 0.10778443113772455, + "pass@4": 0.12574850299401197, + "pass@8": 0.16766467065868262, + "pass@16": 0.18862275449101795, + "pass@32": 0.20958083832335328, + "pass@64": 0.23652694610778444, + "pass@128": 0.25748502994011974 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6325301204819277, + "12": 0.5329153605015674, + "13": 0.4565826330532213, + "14": 0.391812865497076, + "15": 0.36363636363636365, + "16": 0.17094017094017094, + "17": 0.14596273291925466, + "18": 0.15588235294117647, + "19": 0.03571428571428571, + "20": 0.08083832335329341 + }, + "pass@2": { + "11": 0.6987951807228916, + "12": 0.6018808777429467, + "13": 0.5518207282913166, + "14": 0.4590643274853801, + "15": 0.43333333333333335, + "16": 0.2222222222222222, + "17": 0.17701863354037267, + "18": 0.2, + "19": 0.06547619047619048, + "20": 0.10778443113772455 + }, + "pass@4": { + "11": 0.7831325301204819, + "12": 0.677115987460815, + "13": 0.6162464985994398, + "14": 0.5263157894736842, + "15": 0.5060606060606061, + "16": 0.27350427350427353, + "17": 0.2204968944099379, + "18": 0.2323529411764706, + "19": 0.07738095238095238, + "20": 0.12574850299401197 + }, + "pass@8": { + "11": 0.8674698795180723, + "12": 0.7492163009404389, + "13": 0.7058823529411765, + "14": 0.5789473684210527, + "15": 0.5606060606060606, + "16": 0.31054131054131057, + "17": 0.3105590062111801, + "18": 0.27647058823529413, + "19": 0.08928571428571429, + "20": 0.16766467065868262 + }, + "pass@16": { + "11": 0.9126506024096386, + "12": 0.8056426332288401, + "13": 0.7535014005602241, + "14": 0.6578947368421053, + "15": 0.6181818181818182, + "16": 0.36752136752136755, + "17": 0.3695652173913043, + "18": 0.32941176470588235, + "19": 0.11607142857142858, + "20": 0.18862275449101795 + }, + "pass@32": { + "11": 0.9518072289156626, + "12": 0.8369905956112853, + "13": 0.8011204481792717, + "14": 0.7222222222222222, + "15": 0.6878787878787879, + "16": 0.4472934472934473, + "17": 0.4254658385093168, + "18": 0.37058823529411766, + "19": 0.15476190476190477, + "20": 0.20958083832335328 + }, + "pass@64": { + "11": 0.963855421686747, + "12": 0.8714733542319749, + "13": 0.8291316526610645, + "14": 0.783625730994152, + "15": 0.7363636363636363, + "16": 0.49002849002849, + "17": 0.48757763975155277, + "18": 0.4264705882352941, + "19": 0.17857142857142858, + "20": 0.23652694610778444 + }, + "pass@128": { + "11": 0.9759036144578314, + "12": 0.8871473354231975, + "13": 0.8431372549019608, + "14": 0.827485380116959, + "15": 0.793939393939394, + "16": 0.5441595441595442, + "17": 0.5217391304347826, + "18": 0.4852941176470588, + "19": 0.19940476190476192, + "20": 0.25748502994011974 + } + }, + "total_pass_at_k": { + "pass@1": 0.296164139161463, + "pass@2": 0.3514719000892061, + "pass@4": 0.40350877192982454, + "pass@8": 0.46119536128456734, + "pass@16": 0.5114481118049361, + "pass@32": 0.560511448111805, + "pass@64": 0.600059470710675, + "pass@128": 0.6333630686886709 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.596, + "pass@2": 0.662, + "pass@4": 0.751, + "pass@8": 0.845, + "pass@16": 0.895, + "pass@32": 0.936, + "pass@64": 0.953, + "pass@128": 0.964 + }, + "12": { + "pass@1": 0.529, + "pass@2": 0.613, + "pass@4": 0.704, + "pass@8": 0.774, + "pass@16": 0.825, + "pass@32": 0.854, + "pass@64": 0.873, + "pass@128": 0.889 + }, + "13": { + "pass@1": 0.395, + "pass@2": 0.493, + "pass@4": 0.56, + "pass@8": 0.643, + "pass@16": 0.704, + "pass@32": 0.755, + "pass@64": 0.79, + "pass@128": 0.815 + }, + "14": { + "pass@1": 0.339, + "pass@2": 0.418, + "pass@4": 0.488, + "pass@8": 0.547, + "pass@16": 0.619, + "pass@32": 0.697, + "pass@64": 0.762, + "pass@128": 0.798 + }, + "15": { + "pass@1": 0.325, + "pass@2": 0.391, + "pass@4": 0.47, + "pass@8": 0.526, + "pass@16": 0.589, + "pass@32": 0.664, + "pass@64": 0.734, + "pass@128": 0.788 + }, + "16": { + "pass@1": 0.145, + "pass@2": 0.188, + "pass@4": 0.234, + "pass@8": 0.28, + "pass@16": 0.334, + "pass@32": 0.393, + "pass@64": 0.446, + "pass@128": 0.486 + }, + "17": { + "pass@1": 0.128, + "pass@2": 0.167, + "pass@4": 0.201, + "pass@8": 0.262, + "pass@16": 0.316, + "pass@32": 0.363, + "pass@64": 0.408, + "pass@128": 0.458 + }, + "18": { + "pass@1": 0.127, + "pass@2": 0.176, + "pass@4": 0.214, + "pass@8": 0.255, + "pass@16": 0.3, + "pass@32": 0.339, + "pass@64": 0.39, + "pass@128": 0.449 + }, + "19": { + "pass@1": 0.033, + "pass@2": 0.054, + "pass@4": 0.065, + "pass@8": 0.079, + "pass@16": 0.102, + "pass@32": 0.128, + "pass@64": 0.153, + "pass@128": 0.18 + }, + "20": { + "pass@1": 0.053, + "pass@2": 0.071, + "pass@4": 0.092, + "pass@8": 0.12, + "pass@16": 0.142, + "pass@32": 0.171, + "pass@64": 0.194, + "pass@128": 0.221 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.596, + "12": 0.529, + "13": 0.395, + "14": 0.339, + "15": 0.325, + "16": 0.145, + "17": 0.128, + "18": 0.127, + "19": 0.033, + "20": 0.053 + }, + "pass@2": { + "11": 0.662, + "12": 0.613, + "13": 0.493, + "14": 0.418, + "15": 0.391, + "16": 0.188, + "17": 0.167, + "18": 0.176, + "19": 0.054, + "20": 0.071 + }, + "pass@4": { + "11": 0.751, + "12": 0.704, + "13": 0.56, + "14": 0.488, + "15": 0.47, + "16": 0.234, + "17": 0.201, + "18": 0.214, + "19": 0.065, + "20": 0.092 + }, + "pass@8": { + "11": 0.845, + "12": 0.774, + "13": 0.643, + "14": 0.547, + "15": 0.526, + "16": 0.28, + "17": 0.262, + "18": 0.255, + "19": 0.079, + "20": 0.12 + }, + "pass@16": { + "11": 0.895, + "12": 0.825, + "13": 0.704, + "14": 0.619, + "15": 0.589, + "16": 0.334, + "17": 0.316, + "18": 0.3, + "19": 0.102, + "20": 0.142 + }, + "pass@32": { + "11": 0.936, + "12": 0.854, + "13": 0.755, + "14": 0.697, + "15": 0.664, + "16": 0.393, + "17": 0.363, + "18": 0.339, + "19": 0.128, + "20": 0.171 + }, + "pass@64": { + "11": 0.953, + "12": 0.873, + "13": 0.79, + "14": 0.762, + "15": 0.734, + "16": 0.446, + "17": 0.408, + "18": 0.39, + "19": 0.153, + "20": 0.194 + }, + "pass@128": { + "11": 0.964, + "12": 0.889, + "13": 0.815, + "14": 0.798, + "15": 0.788, + "16": 0.486, + "17": 0.458, + "18": 0.449, + "19": 0.18, + "20": 0.221 + } + }, + "total_pass_at_k": { + "pass@1": 0.267, + "pass@2": 0.3233, + "pass@4": 0.3779, + "pass@8": 0.4331, + "pass@16": 0.4826, + "pass@32": 0.53, + "pass@64": 0.5703, + "pass@128": 0.6048 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block16-epoch4-op17-20/summary.csv b/composition/SDAR-100M-block16-epoch4-op17-20/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..59937997aaa30eb24560a4e8709d2d1037291429 --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op17-20/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch4-op17-20,0.9890277777777777,0.6918046875,0.832594572368421,0.0,0.0,0.0,199.14139930555555,372.07711328125,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block16-epoch4-op17-20/summary.json b/composition/SDAR-100M-block16-epoch4-op17-20/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fcf3d5a19166901840773a61aef162e566f46786 --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op17-20/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch4-op17-20", + "id_acc": 0.9890277777777777, + "ood_acc": 0.6918046875, + "total_acc": 0.832594572368421, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.14139930555555, + "ood_avg_resp_len": 372.07711328125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 676633, + "answer_accuracy": 0.8356299893297502, + "avg_response_len": 276.8894900509801, + "resp_tokens_sum": 224205173, + "pass_at_k": { + "pass@1": 0.8356299893297502, + "pass@2": 0.8748209761426936, + "pass@4": 0.9020187386472343, + "pass@8": 0.9209896497859279, + "pass@16": 0.9346778150804824, + "pass@32": 0.9451618942195561, + "pass@64": 0.9534618266585658, + "pass@128": 0.9603224786595005 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9463956489675516, + "pass@2": 0.9599948029173347, + "pass@4": 0.9696439955514411, + "pass@8": 0.9772987780870966, + "pass@16": 0.9829576313879089, + "pass@32": 0.9869532680938505, + "pass@64": 0.9904289041602642, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999513239875389, + "pass@2": 0.9999996167243114, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9962890625, + "pass@2": 0.9968965305118112, + "pass@4": 0.9969725551884139, + "pass@8": 0.9970703124996394, + "pass@16": 0.997265625, + "pass@32": 0.99765625, + "pass@64": 0.9984375, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941220238095239, + "pass@2": 0.9976346706661666, + "pass@4": 0.9991279667422525, + "pass@8": 0.999810090374339, + "pass@16": 0.9999911414698637, + "pass@32": 0.9999999920410901, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9975673652694611, + "pass@2": 0.9990827891461174, + "pass@4": 0.9997700759897528, + "pass@8": 0.9999845197580833, + "pass@16": 0.9999999528630369, + "pass@32": 0.9999999999999486, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960012943786982, + "pass@2": 0.9984763022410661, + "pass@4": 0.9996465231971745, + "pass@8": 0.9999815406455984, + "pass@16": 0.9999999678464966, + "pass@32": 0.9999999999999909, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9963441506410257, + "pass@2": 0.9996159209065214, + "pass@4": 0.9999852656639074, + "pass@8": 0.9999999592877328, + "pass@16": 0.9999999999999242, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9913292253521127, + "pass@2": 0.9970926444493733, + "pass@4": 0.9988132281880259, + "pass@8": 0.9997344814601936, + "pass@16": 0.9999856314389629, + "pass@32": 0.9999999749550461, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9385844546742209, + "pass@2": 0.9559055815172538, + "pass@4": 0.9662483132420059, + "pass@8": 0.9743145176794873, + "pass@16": 0.9814528957415205, + "pass@32": 0.9870929428344121, + "pass@64": 0.9904799574420998, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8917284604519774, + "pass@2": 0.9156980542951199, + "pass@4": 0.9281065293215466, + "pass@8": 0.935889004485147, + "pass@16": 0.9418117371185404, + "pass@32": 0.947600022930526, + "pass@64": 0.9542037056880797, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8402233542319749, + "pass@2": 0.8852274269987415, + "pass@4": 0.9120427055748123, + "pass@8": 0.9300158215455117, + "pass@16": 0.9440746211517979, + "pass@32": 0.9571960077579881, + "pass@64": 0.9683369401551648, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8298773364485982, + "pass@2": 0.8854661092305048, + "pass@4": 0.9165421769708681, + "pass@8": 0.9342494936091128, + "pass@16": 0.9469401578089754, + "pass@32": 0.9577873713575069, + "pass@64": 0.9667747235355996, + "pass@128": 0.9750778816199377 + }, + "15": { + "pass@1": 0.7909365968208093, + "pass@2": 0.8548815057348322, + "pass@4": 0.8965234123552469, + "pass@8": 0.9232631518428708, + "pass@16": 0.93956332213618, + "pass@32": 0.9494509870773286, + "pass@64": 0.9560635736921728, + "pass@128": 0.9624277456647399 + }, + "16": { + "pass@1": 0.6512096774193549, + "pass@2": 0.7479818865887731, + "pass@4": 0.8152397825271837, + "pass@8": 0.8589674413513186, + "pass@16": 0.8885187025219686, + "pass@32": 0.9122664817583825, + "pass@64": 0.9310470742858282, + "pass@128": 0.9419354838709677 + }, + "17": { + "pass@1": 0.6124909682080925, + "pass@2": 0.7055230474261528, + "pass@4": 0.768419161297034, + "pass@8": 0.8132989938840093, + "pass@16": 0.8486630253769624, + "pass@32": 0.8756942867659188, + "pass@64": 0.8953656362346946, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.5464732142857143, + "pass@2": 0.6480627812148483, + "pass@4": 0.7236771966896989, + "pass@8": 0.777945603229985, + "pass@16": 0.8163166798894307, + "pass@32": 0.8431802277020857, + "pass@64": 0.8622731277447071, + "pass@128": 0.8771428571428571 + }, + "19": { + "pass@1": 0.4315643768768769, + "pass@2": 0.5412484334728428, + "pass@4": 0.6303853890448375, + "pass@8": 0.6978895463021549, + "pass@16": 0.7471806522306299, + "pass@32": 0.7842191914351475, + "pass@64": 0.8144645683656517, + "pass@128": 0.8438438438438438 + }, + "20": { + "pass@1": 0.4473066816816817, + "pass@2": 0.550585157007401, + "pass@4": 0.6322523291007539, + "pass@8": 0.6917831399797562, + "pass@16": 0.7350186424328486, + "pass@32": 0.7688185050754536, + "pass@64": 0.7971501932040521, + "pass@128": 0.8228228228228228 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 686415, + "answer_accuracy": 0.8359496784879189, + "avg_response_len": 286.5450993764614, + "resp_tokens_sum": 235287912, + "pass_at_k": { + "pass@1": 0.8359496784879189, + "pass@2": 0.8760342317157656, + "pass@4": 0.9043035755711732, + "pass@8": 0.9240919493445696, + "pass@16": 0.9381619892173285, + "pass@32": 0.9489126547729012, + "pass@64": 0.9576937527310582, + "pass@128": 0.9649259547934529 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.924079754601227, + "pass@2": 0.9425480653108548, + "pass@4": 0.9558939233439372, + "pass@8": 0.9660136894297928, + "pass@16": 0.9745849213182081, + "pass@32": 0.9815053815983932, + "pass@64": 0.9856196459168264, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150811, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9920969202898551, + "pass@2": 0.9947952327969871, + "pass@4": 0.9963463162213417, + "pass@8": 0.9977460600531718, + "pass@16": 0.9990916148451592, + "pass@32": 0.9998484564379228, + "pass@64": 0.999998063383779, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9952542979942693, + "pass@2": 0.9966728617196489, + "pass@4": 0.9972907742728435, + "pass@8": 0.997639465969732, + "pass@16": 0.9980869583411133, + "pass@32": 0.9988007408816018, + "pass@64": 0.9996502944295288, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967937138728323, + "pass@2": 0.9983024498202175, + "pass@4": 0.9990612040546953, + "pass@8": 0.9997078494159238, + "pass@16": 0.999975444312545, + "pass@32": 0.9999999133144741, + "pass@64": 0.9999999999999895, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9918573943661971, + "pass@2": 0.9956231978485083, + "pass@4": 0.9969242494336096, + "pass@8": 0.9971736180162805, + "pass@16": 0.9971830890857465, + "pass@32": 0.9971830985915483, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9910179640718563, + "pass@2": 0.9937379178650573, + "pass@4": 0.9939987547402381, + "pass@8": 0.9940119380249567, + "pass@16": 0.9940119760478334, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9533838478915663, + "pass@2": 0.9686280802343229, + "pass@4": 0.9774122846842942, + "pass@8": 0.9826945678160793, + "pass@16": 0.9851505016991625, + "pass@32": 0.9862401453810479, + "pass@64": 0.9872047186881181, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.885384012539185, + "pass@2": 0.9101669525831221, + "pass@4": 0.9239799511738148, + "pass@8": 0.9333532233276649, + "pass@16": 0.9404962983721141, + "pass@32": 0.947095869663446, + "pass@64": 0.9545776664683873, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.8328956582633054, + "pass@2": 0.8768782119146881, + "pass@4": 0.9001525788793208, + "pass@8": 0.9144132901889223, + "pass@16": 0.9252804791130302, + "pass@32": 0.9358350221164605, + "pass@64": 0.9474943865376289, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.8158351608187134, + "pass@2": 0.8764760182115391, + "pass@4": 0.9120771337245117, + "pass@8": 0.9326000038402523, + "pass@16": 0.9449230559731131, + "pass@32": 0.9539728291687525, + "pass@64": 0.9628130945239121, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.7901515151515152, + "pass@2": 0.8613483506323071, + "pass@4": 0.9039200494824501, + "pass@8": 0.9292158194336342, + "pass@16": 0.9452248133475442, + "pass@32": 0.9574867632372372, + "pass@64": 0.9678628579175225, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.6649528133903134, + "pass@2": 0.7624254800681963, + "pass@4": 0.8277979675617472, + "pass@8": 0.8686706781122179, + "pass@16": 0.8961598925297205, + "pass@32": 0.9179229188572494, + "pass@64": 0.9337860632473033, + "pass@128": 0.9430199430199431 + }, + "17": { + "pass@1": 0.6393390916149069, + "pass@2": 0.7354016634469608, + "pass@4": 0.8004865514675568, + "pass@8": 0.8453401336925654, + "pass@16": 0.8772971858982275, + "pass@32": 0.9004220002674332, + "pass@64": 0.9174945886997983, + "pass@128": 0.9316770186335404 + }, + "18": { + "pass@1": 0.5892693014705882, + "pass@2": 0.6895900880037054, + "pass@4": 0.7655650637420315, + "pass@8": 0.8203016454124229, + "pass@16": 0.859618198955504, + "pass@32": 0.8878937646484227, + "pass@64": 0.9097340634600377, + "pass@128": 0.9294117647058824 + }, + "19": { + "pass@1": 0.4204799107142857, + "pass@2": 0.5389387361736032, + "pass@4": 0.6410350558858714, + "pass@8": 0.7190375968217035, + "pass@16": 0.7736645475875377, + "pass@32": 0.8123192037757934, + "pass@64": 0.8394299819521862, + "pass@128": 0.8571428571428571 + }, + "20": { + "pass@1": 0.3888005239520958, + "pass@2": 0.4932697351360271, + "pass@4": 0.581708334175294, + "pass@8": 0.6533021743606491, + "pass@16": 0.7090402348266315, + "pass@32": 0.754332532451074, + "pass@64": 0.7954232145576876, + "pass@128": 0.8293413173652695 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 661822, + "answer_accuracy": 0.8260879333759387, + "avg_response_len": 307.27815945039146, + "resp_tokens_sum": 246176512, + "pass_at_k": { + "pass@1": 0.8260879333759387, + "pass@2": 0.8676958919313628, + "pass@4": 0.8980069189605794, + "pass@8": 0.9198953723230876, + "pass@16": 0.93505318616042, + "pass@32": 0.94562924717024, + "pass@64": 0.9530547716086694, + "pass@128": 0.9583000479309793 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9465951492537313, + "pass@2": 0.9632176225173348, + "pass@4": 0.9735499216515845, + "pass@8": 0.9811835925161799, + "pass@16": 0.9873330366748251, + "pass@32": 0.9918809176478053, + "pass@64": 0.993806150809024, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999316690962099, + "pass@2": 0.9999996413075917, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9981953581871345, + "pass@2": 0.9990279792328591, + "pass@4": 0.9996846265927285, + "pass@8": 0.9999691763603531, + "pass@16": 0.9999997893064115, + "pass@32": 0.9999999999982297, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9918198529411765, + "pass@2": 0.9951822313571099, + "pass@4": 0.9976438364322108, + "pass@8": 0.9994046739409391, + "pass@16": 0.9999673976331993, + "pass@32": 0.9999999551856581, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.993764787066246, + "pass@2": 0.9947496212027123, + "pass@4": 0.9955808440940151, + "pass@8": 0.9967326641590364, + "pass@16": 0.9979618214750134, + "pass@32": 0.9990096140434843, + "pass@64": 0.9998121152924854, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956487341772152, + "pass@2": 0.9986505469450812, + "pass@4": 0.9995743400943553, + "pass@8": 0.9999481010918784, + "pass@16": 0.9999994169232284, + "pass@32": 0.9999999999832521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9844923048048048, + "pass@2": 0.9919176065829609, + "pass@4": 0.9963254145596664, + "pass@8": 0.9990655284449819, + "pass@16": 0.9999326120862155, + "pass@32": 0.9999996834444386, + "pass@64": 0.999999999999752, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9835962620578779, + "pass@2": 0.9923585968554577, + "pass@4": 0.9956467118660005, + "pass@8": 0.9975453735871543, + "pass@16": 0.9990224437782512, + "pass@32": 0.9998343032263491, + "pass@64": 0.9999978524135079, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9548611111111112, + "pass@2": 0.9714758311461071, + "pass@4": 0.9792459415787311, + "pass@8": 0.9832824257084963, + "pass@16": 0.9859409115458316, + "pass@32": 0.9888924101781106, + "pass@64": 0.9916644760868305, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8987241972477065, + "pass@2": 0.9262153404849626, + "pass@4": 0.9399879209479548, + "pass@8": 0.9515969150881995, + "pass@16": 0.9632921895262051, + "pass@32": 0.9727991681405372, + "pass@64": 0.978484053984944, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7942467206790124, + "pass@2": 0.8419292098036356, + "pass@4": 0.87347388607674, + "pass@8": 0.896955802177486, + "pass@16": 0.9148879344767462, + "pass@32": 0.9305686019139384, + "pass@64": 0.9444548045895704, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.7969445474777448, + "pass@2": 0.8519073167597371, + "pass@4": 0.8856038931706228, + "pass@8": 0.9087465231648425, + "pass@16": 0.9243278096477272, + "pass@32": 0.9348198946343538, + "pass@64": 0.9430984318055536, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.7702064043209876, + "pass@2": 0.8314692542772435, + "pass@4": 0.8710929519226761, + "pass@8": 0.9004926868932659, + "pass@16": 0.9209909756435962, + "pass@32": 0.9332392643870973, + "pass@64": 0.9400295244557415, + "pass@128": 0.9444444444444444 + }, + "16": { + "pass@1": 0.6661366150442478, + "pass@2": 0.7643184708382698, + "pass@4": 0.8284491338472064, + "pass@8": 0.8694132293085024, + "pass@16": 0.8955625569041571, + "pass@32": 0.9130243917210557, + "pass@64": 0.926146214710774, + "pass@128": 0.9380530973451328 + }, + "17": { + "pass@1": 0.5658179593373494, + "pass@2": 0.6702137042263541, + "pass@4": 0.7485751202221707, + "pass@8": 0.8014946697752476, + "pass@16": 0.8341101603053381, + "pass@32": 0.8534621646954216, + "pass@64": 0.8669435177559548, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.55859375, + "pass@2": 0.6654674403098803, + "pass@4": 0.7465657682910599, + "pass@8": 0.801548579552477, + "pass@16": 0.8354467036205091, + "pass@32": 0.8565727072192222, + "pass@64": 0.8703706077422788, + "pass@128": 0.8806451612903226 + }, + "19": { + "pass@1": 0.3890436178247734, + "pass@2": 0.5075246211670672, + "pass@4": 0.6101444158604039, + "pass@8": 0.6887828332064626, + "pass@16": 0.7462233374111003, + "pass@32": 0.7901599832812849, + "pass@64": 0.8223790409544078, + "pass@128": 0.8429003021148036 + }, + "20": { + "pass@1": 0.39846096096096095, + "pass@2": 0.5144094635974552, + "pass@4": 0.6162668182805975, + "pass@8": 0.6980383217113035, + "pass@16": 0.7577338561538128, + "pass@32": 0.799757548916951, + "pass@64": 0.828102483308844, + "pass@128": 0.8468468468468469 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_metrics.json b/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..2a39e8f2e3e260ac02ea29c2cfc53d6999363371 --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch4-op7-10", + "id": { + "answer_accuracy": 0.9928402777777777, + "avg_response_len": 200.1375373263889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9928402777777777, + "pass@2": 0.9949716207349067, + "pass@4": 0.9962314358309377, + "pass@8": 0.9970589677382524, + "pass@16": 0.997637285152463, + "pass@32": 0.9980942875801967, + "pass@64": 0.9984440103790677, + "pass@128": 0.9986666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954203125, + "pass@2": 0.9668100393700787, + "pass@4": 0.9751169003562058, + "pass@8": 0.9805623138162619, + "pass@16": 0.9841740075635769, + "pass@32": 0.9868633714733398, + "pass@64": 0.9886806402008643, + "pass@128": 0.99 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990234375, + "pass@2": 0.999410063976378, + "pass@4": 0.9996751137982751, + "pass@8": 0.9998989158969318, + "pass@16": 0.9999915037321406, + "pass@32": 0.9999999700068081, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9949921875, + "pass@2": 0.9967944143700785, + "pass@4": 0.9977303270528682, + "pass@8": 0.9982286507773027, + "pass@16": 0.9986409291833077, + "pass@32": 0.999127515746347, + "pass@64": 0.9996924094487892, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9986875, + "pass@2": 0.9990652066929134, + "pass@4": 0.9992026385451819, + "pass@8": 0.99937067318584, + "pass@16": 0.9996167406956445, + "pass@32": 0.9998739053984705, + "pass@64": 0.9999934280775878, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9977421875, + "pass@2": 0.9986300442913385, + "pass@4": 0.9992237436257967, + "pass@8": 0.999713898666317, + "pass@16": 0.9999638315713223, + "pass@32": 0.9999996347541911, + "pass@64": 0.9999999999983727, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9968125, + "pass@2": 0.998474655511811, + "pass@4": 0.9989533392388451, + "pass@8": 0.9989996318701388, + "pass@16": 0.9989999999891657, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.994109375, + "pass@2": 0.9955601624015749, + "pass@4": 0.9961808598612675, + "pass@8": 0.9967566254314778, + "pass@16": 0.9973485536370097, + "pass@32": 0.9979841908426146, + "pass@64": 0.9986296156860006, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.954203125, + "2": 1.0, + "3": 0.9999921875, + "4": 0.9990234375, + "5": 0.9949921875, + "6": 0.9986875, + "7": 0.9977421875, + "8": 0.9968125, + "9": 0.994109375 + }, + "per_op_avg_response_len": { + "10": 280.9476015625, + "2": 115.7252734375, + "3": 144.8482265625, + "4": 160.08678125, + "5": 173.194703125, + "6": 199.14746875, + "7": 224.20671875, + "8": 243.602875, + "9": 259.4781875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.9476015625, + "2": 115.7252734375, + "3": 144.8482265625, + "4": 160.08678125, + "5": 173.194703125, + "6": 199.14746875, + "7": 224.20671875, + "8": 243.602875, + "9": 259.4781875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376709, + "answer_accuracy": 0.993934164978048, + "avg_response_len": 190.66135279466397, + "resp_tokens_sum": 72262178, + "pass_at_k": { + "pass@1": 0.993934164978048, + "pass@2": 0.9959355978109121, + "pass@4": 0.9969234930043573, + "pass@8": 0.9975032519300019, + "pass@16": 0.9979521898023057, + "pass@32": 0.998386677156877, + "pass@64": 0.9988549563110108, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9639564896755162, + "pass@2": 0.9751933663159359, + "pass@4": 0.9805886808396739, + "pass@8": 0.9832729426842688, + "pass@16": 0.9853094435850058, + "pass@32": 0.9875159779599025, + "pass@64": 0.9903780082743677, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.997314453125, + "pass@2": 0.9981799028051181, + "pass@4": 0.9989848275801462, + "pass@8": 0.9996841121782726, + "pass@16": 0.9999734491629393, + "pass@32": 0.9999999062712751, + "pass@64": 0.9999999999999887, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9964285714285714, + "pass@2": 0.9987411729783777, + "pass@4": 0.9997791192767571, + "pass@8": 0.9999910052051663, + "pass@16": 0.999999987524699, + "pass@32": 0.999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970761601796407, + "pass@2": 0.9973235124239709, + "pass@4": 0.9976157328200742, + "pass@8": 0.9981157895218378, + "pass@16": 0.998852517052828, + "pass@32": 0.9996224712529058, + "pass@64": 0.9999803235855924, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972725591715976, + "pass@2": 0.9980547919675722, + "pass@4": 0.9987735604794961, + "pass@8": 0.9995079830474547, + "pass@16": 0.999928825622786, + "pass@32": 0.9999991140832827, + "pass@64": 0.9999999999952738, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9991486378205128, + "pass@2": 0.9999641158136482, + "pass@4": 0.999999888836011, + "pass@8": 0.9999999999996302, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9955325704225352, + "pass@2": 0.9969474326272597, + "pass@4": 0.9973978846834287, + "pass@8": 0.9976821039095084, + "pass@16": 0.998119293157022, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387445, + "answer_accuracy": 0.9917804923001311, + "avg_response_len": 198.17119921363042, + "resp_tokens_sum": 77416768, + "pass_at_k": { + "pass@1": 0.9917804923001311, + "pass@2": 0.9938671701659427, + "pass@4": 0.9952572525235622, + "pass@8": 0.996152824229983, + "pass@16": 0.9966572318246322, + "pass@32": 0.9970099127946118, + "pass@64": 0.9972736781091658, + "pass@128": 0.9973787680209698 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9412384969325154, + "pass@2": 0.9570337030819772, + "pass@4": 0.9684386097635029, + "pass@8": 0.9755421990555402, + "pass@16": 0.9790090135040668, + "pass@32": 0.9808182150392867, + "pass@64": 0.9815547734367644, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999075443786982, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9963768115942029, + "pass@2": 0.9971339010612804, + "pass@4": 0.9974446123582379, + "pass@8": 0.9977680501821345, + "pass@16": 0.9983124881128967, + "pass@32": 0.999097406302473, + "pass@64": 0.9998273650576287, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997313753581661, + "pass@2": 0.9999841363400491, + "pass@4": 0.9999999435958757, + "pass@8": 0.9999999999999097, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997674313583815, + "pass@2": 0.998275069978608, + "pass@4": 0.9989801556597332, + "pass@8": 0.9996539139236084, + "pass@16": 0.9999649956979594, + "pass@32": 0.9999998098093686, + "pass@64": 0.9999999999999141, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957306338028169, + "pass@2": 0.9968081124542529, + "pass@4": 0.9971466248937193, + "pass@8": 0.9971827493473799, + "pass@16": 0.9971830985789736, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.993310254491018, + "pass@2": 0.9940558106228489, + "pass@4": 0.994196379457059, + "pass@8": 0.9943759135003243, + "pass@16": 0.9947162761091989, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379598, + "answer_accuracy": 0.9928387596250419, + "avg_response_len": 211.5403650192501, + "resp_tokens_sum": 80879497, + "pass_at_k": { + "pass@1": 0.9928387596250419, + "pass@2": 0.9951445189653858, + "pass@4": 0.9965407850654927, + "pass@8": 0.9975444128321481, + "pass@16": 0.9983265017873332, + "pass@32": 0.9989124149016076, + "pass@64": 0.9992324413075104, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9569496268656716, + "pass@2": 0.9678403014455282, + "pass@4": 0.976078659010907, + "pass@8": 0.9827045652363846, + "pass@16": 0.9880512471221918, + "pass@32": 0.9920857219167919, + "pass@64": 0.9938974306133376, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997487207602339, + "pass@2": 0.9999802142330892, + "pass@4": 0.9999999095507799, + "pass@8": 0.9999999999996626, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9922564338235295, + "pass@2": 0.994646320634553, + "pass@4": 0.9961220976973466, + "pass@8": 0.9970632553671733, + "pass@16": 0.997715072688465, + "pass@32": 0.9983497369764539, + "pass@64": 0.9992704955997272, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992360015772871, + "pass@2": 0.9998886118631859, + "pass@4": 0.9999968563669289, + "pass@8": 0.9999999983772165, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998318829113924, + "pass@2": 0.9996340202332306, + "pass@4": 0.9999719820425929, + "pass@8": 0.9999998296794577, + "pass@16": 0.999999999997869, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995777027027027, + "pass@2": 0.9988557701402191, + "pass@4": 0.9998988651193375, + "pass@8": 0.9999992668226257, + "pass@16": 0.9999999999808709, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9933430466237942, + "pass@2": 0.9955922285236853, + "pass@4": 0.996922894083577, + "pass@8": 0.99825698853535, + "pass@16": 0.9992957181536803, + "pass@32": 0.9998834594758289, + "pass@64": 0.9999990174474539, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.4674421875, + "avg_response_len": 332.59690703125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.4674421875, + "pass@2": 0.5340498277559019, + "pass@4": 0.5905295806805306, + "pass@8": 0.6377652123780675, + "pass@16": 0.6765026264659709, + "pass@32": 0.7087523766471188, + "pass@64": 0.7369760709987055, + "pass@128": 0.7626 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9424921875, + "pass@2": 0.9608925935039377, + "pass@4": 0.9707150841769778, + "pass@8": 0.9776590164210477, + "pass@16": 0.983031112952659, + "pass@32": 0.9868108166417683, + "pass@64": 0.9895059512740915, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.850125, + "pass@2": 0.8868829970472448, + "pass@4": 0.9069662009748772, + "pass@8": 0.9210538138715969, + "pass@16": 0.932209399638482, + "pass@32": 0.941670117173944, + "pass@64": 0.9509294871368753, + "pass@128": 0.961 + }, + "13": { + "pass@1": 0.723671875, + "pass@2": 0.7991273375984257, + "pass@4": 0.848705464191976, + "pass@8": 0.8832906602998529, + "pass@16": 0.907676298226515, + "pass@32": 0.9258390025587503, + "pass@64": 0.9408623838730613, + "pass@128": 0.954 + }, + "14": { + "pass@1": 0.621359375, + "pass@2": 0.7092471702755899, + "pass@4": 0.7710694283839518, + "pass@8": 0.8143357774146883, + "pass@16": 0.8459435216064888, + "pass@32": 0.8706776031668482, + "pass@64": 0.8916396601771912, + "pass@128": 0.911 + }, + "15": { + "pass@1": 0.4664140625, + "pass@2": 0.5631503444881897, + "pass@4": 0.6409323717660281, + "pass@8": 0.700385082200824, + "pass@16": 0.7464661637312656, + "pass@32": 0.7838249784646978, + "pass@64": 0.8150213816108757, + "pass@128": 0.842 + }, + "16": { + "pass@1": 0.300421875, + "pass@2": 0.38325036909448806, + "pass@4": 0.45934014332583345, + "pass@8": 0.5256147490839751, + "pass@16": 0.5812667481374817, + "pass@32": 0.6290118259853492, + "pass@64": 0.6714562961595262, + "pass@128": 0.71 + }, + "17": { + "pass@1": 0.2392421875, + "pass@2": 0.3153343996062992, + "pass@4": 0.390559457255343, + "pass@8": 0.45835220794408055, + "pass@16": 0.515216747490821, + "pass@32": 0.5633337200265438, + "pass@64": 0.6057651819056846, + "pass@128": 0.644 + }, + "18": { + "pass@1": 0.20425, + "pass@2": 0.27352423720472446, + "pass@4": 0.3419236596362953, + "pass@8": 0.4029165957544186, + "pass@16": 0.45499798232417954, + "pass@32": 0.5004840348220753, + "pass@64": 0.5415533976453777, + "pass@128": 0.578 + }, + "19": { + "pass@1": 0.169625, + "pass@2": 0.23175504429133828, + "pass@4": 0.29506946662917083, + "pass@8": 0.353428445257125, + "pass@16": 0.40308684493744273, + "pass@32": 0.4453773587288245, + "pass@64": 0.4839624351859198, + "pass@128": 0.52 + }, + "20": { + "pass@1": 0.1568203125, + "pass@2": 0.217333784448819, + "pass@4": 0.2800145304649416, + "pass@8": 0.3406157755330686, + "pass@16": 0.39513144561425667, + "pass@32": 0.4404943089023678, + "pass@64": 0.47906453501838553, + "pass@128": 0.514 + } + }, + "per_op_accuracy": { + "11": 0.9424921875, + "12": 0.850125, + "13": 0.723671875, + "14": 0.621359375, + "15": 0.4664140625, + "16": 0.300421875, + "17": 0.2392421875, + "18": 0.20425, + "19": 0.169625, + "20": 0.1568203125 + }, + "per_op_avg_response_len": { + "11": 298.1274375, + "12": 307.074171875, + "13": 322.5293828125, + "14": 340.953328125, + "15": 345.885015625, + "16": 332.6650234375, + "17": 347.350203125, + "18": 347.9971171875, + "19": 336.1864375, + "20": 347.200953125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.1274375, + "12": 307.074171875, + "13": 322.5293828125, + "14": 340.953328125, + "15": 345.885015625, + "16": 332.6650234375, + "17": 347.350203125, + "18": 347.9971171875, + "19": 336.1864375, + "20": 347.200953125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 200217, + "answer_accuracy": 0.46484258915304605, + "avg_response_len": 315.6339617384844, + "resp_tokens_sum": 135949860, + "pass_at_k": { + "pass@1": 0.46484258915304605, + "pass@2": 0.5302466260486013, + "pass@4": 0.5859201473515673, + "pass@8": 0.6324395587767444, + "pass@16": 0.6708254517301923, + "pass@32": 0.7030991125764335, + "pass@64": 0.7314564132966396, + "pass@128": 0.7575037147102526 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9345122167138811, + "pass@2": 0.9542368143695208, + "pass@4": 0.9657462063877991, + "pass@8": 0.9749512748470323, + "pass@16": 0.9827357531203197, + "pass@32": 0.9884344587547554, + "pass@64": 0.9925691216469926, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.8454493290960452, + "pass@2": 0.8801169285777846, + "pass@4": 0.8989873583069065, + "pass@8": 0.9130351649779792, + "pass@16": 0.9250958046726186, + "pass@32": 0.9364960935539219, + "pass@64": 0.9474650567648585, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.74047315830721, + "pass@2": 0.8119554217164869, + "pass@4": 0.8599147251969675, + "pass@8": 0.8945134207641785, + "pass@16": 0.9205661933194598, + "pass@32": 0.9404403427759488, + "pass@64": 0.9559823521026871, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.6125632788161994, + "pass@2": 0.7013274370201388, + "pass@4": 0.7658476230190846, + "pass@8": 0.8097702298192307, + "pass@16": 0.8407102562368179, + "pass@32": 0.8644964958281782, + "pass@64": 0.8838021812976606, + "pass@128": 0.9003115264797508 + }, + "15": { + "pass@1": 0.48457821531791906, + "pass@2": 0.5827305027081152, + "pass@4": 0.6590647869702699, + "pass@8": 0.7158310076759379, + "pass@16": 0.7589664128271321, + "pass@32": 0.7930825505983045, + "pass@64": 0.8204516163076292, + "pass@128": 0.8410404624277457 + }, + "16": { + "pass@1": 0.2755040322580645, + "pass@2": 0.3586213328676657, + "pass@4": 0.43517604109970115, + "pass@8": 0.5017510566263869, + "pass@16": 0.5583572473734497, + "pass@32": 0.6078831915021241, + "pass@64": 0.6517453480905581, + "pass@128": 0.6903225806451613 + }, + "17": { + "pass@1": 0.21780166184971098, + "pass@2": 0.29274348857584986, + "pass@4": 0.36758594012453627, + "pass@8": 0.4327260544001389, + "pass@16": 0.4852625887317101, + "pass@32": 0.5308733536032256, + "pass@64": 0.5731955827266514, + "pass@128": 0.615606936416185 + }, + "18": { + "pass@1": 0.18082589285714284, + "pass@2": 0.24475007030371193, + "pass@4": 0.30927755396646855, + "pass@8": 0.3685482161042458, + "pass@16": 0.4204588630253465, + "pass@32": 0.46585847061159913, + "pass@64": 0.5068792519157992, + "pass@128": 0.5457142857142857 + }, + "19": { + "pass@1": 0.1753472222222222, + "pass@2": 0.23986560379276908, + "pass@4": 0.3042127763308865, + "pass@8": 0.3648899684541569, + "pass@16": 0.41793983675736257, + "pass@32": 0.46237397780199607, + "pass@64": 0.5029501083188042, + "pass@128": 0.5405405405405406 + }, + "20": { + "pass@1": 0.15636730480480482, + "pass@2": 0.2165424528859568, + "pass@4": 0.27821547475484476, + "pass@8": 0.3365076147164066, + "pass@16": 0.3888313889531125, + "pass@32": 0.4337362063462613, + "pass@64": 0.4738117632886543, + "pass@128": 0.5135135135135135 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 201848, + "answer_accuracy": 0.46890796907523047, + "avg_response_len": 328.5204500260184, + "resp_tokens_sum": 141416227, + "pass_at_k": { + "pass@1": 0.46890796907523047, + "pass@2": 0.5365449638961293, + "pass@4": 0.5941743382935776, + "pass@8": 0.6423905408438726, + "pass@16": 0.6812675313224216, + "pass@32": 0.7127574655170806, + "pass@64": 0.7399460641942278, + "pass@128": 0.7647933392804044 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9478068524096386, + "pass@2": 0.9611561403092685, + "pass@4": 0.9686419691702693, + "pass@8": 0.9750669382908692, + "pass@16": 0.9806081356024587, + "pass@32": 0.9842428892059527, + "pass@64": 0.9862549466314855, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8410315438871473, + "pass@2": 0.8755083244884359, + "pass@4": 0.8937380000305601, + "pass@8": 0.905965360565369, + "pass@16": 0.9151980908995769, + "pass@32": 0.9235809759059825, + "pass@64": 0.9339121914379174, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.7262123599439776, + "pass@2": 0.8062757091025381, + "pass@4": 0.8566690912585506, + "pass@8": 0.8882637919932588, + "pass@16": 0.9082264797954909, + "pass@32": 0.9222958616323473, + "pass@64": 0.9343518871725501, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6329952485380117, + "pass@2": 0.7221707792282543, + "pass@4": 0.7828848704767167, + "pass@8": 0.8258118272832482, + "pass@16": 0.8579023706600551, + "pass@32": 0.8832258032244631, + "pass@64": 0.9049950902372479, + "pass@128": 0.9269005847953217 + }, + "15": { + "pass@1": 0.4446022727272727, + "pass@2": 0.5437966028394173, + "pass@4": 0.6285262524002679, + "pass@8": 0.6960194300230498, + "pass@16": 0.7475932370874276, + "pass@32": 0.785985388651888, + "pass@64": 0.8147437752003887, + "pass@128": 0.8424242424242424 + }, + "16": { + "pass@1": 0.3059116809116809, + "pass@2": 0.39158751990936996, + "pass@4": 0.4727661555660242, + "pass@8": 0.5448737019577484, + "pass@16": 0.6040431432838912, + "pass@32": 0.6528764342680178, + "pass@64": 0.694521859735016, + "pass@128": 0.7293447293447294 + }, + "17": { + "pass@1": 0.26052989130434784, + "pass@2": 0.34320820291485293, + "pass@4": 0.42293633977041684, + "pass@8": 0.4927356348756277, + "pass@16": 0.5482837994495754, + "pass@32": 0.5921106992309274, + "pass@64": 0.6290502589670679, + "pass@128": 0.6614906832298136 + }, + "18": { + "pass@1": 0.2153262867647059, + "pass@2": 0.28677049559981455, + "pass@4": 0.35674156300315396, + "pass@8": 0.4197868754911729, + "pass@16": 0.47482275196634255, + "pass@32": 0.5236327311772606, + "pass@64": 0.5672130434247589, + "pass@128": 0.6029411764705882 + }, + "19": { + "pass@1": 0.16901506696428573, + "pass@2": 0.22576206294525675, + "pass@4": 0.28495160454273577, + "pass@8": 0.34010383127747634, + "pass@16": 0.38708927582856617, + "pass@32": 0.4276740630065036, + "pass@64": 0.4649332236307543, + "pass@128": 0.5 + }, + "20": { + "pass@1": 0.1504491017964072, + "pass@2": 0.2108255192371163, + "pass@4": 0.2735428047167756, + "pass@8": 0.3338204070406974, + "pass@16": 0.38676384570770866, + "pass@32": 0.4292259673414519, + "pass@64": 0.4663031080186876, + "pass@128": 0.5 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 196261, + "answer_accuracy": 0.4686091266809291, + "avg_response_len": 354.23182017878975, + "resp_tokens_sum": 148357954, + "pass_at_k": { + "pass@1": 0.4686091266809291, + "pass@2": 0.5353965975925558, + "pass@4": 0.5915239032047982, + "pass@8": 0.6384882700608202, + "pass@16": 0.6774437383099655, + "pass@32": 0.7104498460016812, + "pass@64": 0.7396000199751166, + "pass@128": 0.7655867970660146 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9458333333333333, + "pass@2": 0.9680735220597424, + "pass@4": 0.9784683795477946, + "pass@8": 0.9834253742459571, + "pass@16": 0.9859158447021265, + "pass@32": 0.9876978205871854, + "pass@64": 0.9894997112731746, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8640577217125383, + "pass@2": 0.9053041248284324, + "pass@4": 0.9285084223990344, + "pass@8": 0.9444538698747363, + "pass@16": 0.9565054855884125, + "pass@32": 0.9649179472227764, + "pass@64": 0.971280941814739, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.7043306327160493, + "pass@2": 0.7786207713619127, + "pass@4": 0.8288944483791375, + "pass@8": 0.86676143621755, + "pass@16": 0.8943791026870926, + "pass@32": 0.9153670080570203, + "pass@64": 0.9331493513324162, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.6179293397626113, + "pass@2": 0.703675538563985, + "pass@4": 0.7640525688183634, + "pass@8": 0.8070382454357383, + "pass@16": 0.8387920432900056, + "pass@32": 0.8638308703360127, + "pass@64": 0.8855514513931839, + "pass@128": 0.9050445103857567 + }, + "15": { + "pass@1": 0.46923225308641975, + "pass@2": 0.5619528136239913, + "pass@4": 0.6342046055007012, + "pass@8": 0.6883368569053809, + "pass@16": 0.7319691872044063, + "pass@32": 0.7717383879708698, + "pass@64": 0.8095051745441575, + "pass@128": 0.8425925925925926 + }, + "16": { + "pass@1": 0.31752396755162243, + "pass@2": 0.3971402254198313, + "pass@4": 0.46753584065266157, + "pass@8": 0.5274963190047928, + "pass@16": 0.578633799879253, + "pass@32": 0.6236236229841198, + "pass@64": 0.6655990132285015, + "pass@128": 0.7079646017699115 + }, + "17": { + "pass@1": 0.2409403237951807, + "pass@2": 0.3118437084953988, + "pass@4": 0.38310006194632296, + "pass@8": 0.4517112008785566, + "pass@16": 0.5143629770086949, + "pass@32": 0.5692526943598468, + "pass@64": 0.6171244785989948, + "pass@128": 0.6566265060240963 + }, + "18": { + "pass@1": 0.21854838709677418, + "pass@2": 0.2914830454660909, + "pass@4": 0.36253027202244875, + "pass@8": 0.4232167175836581, + "pass@16": 0.4722504664411351, + "pass@32": 0.5141885209927322, + "pass@64": 0.5525587893884821, + "pass@128": 0.5870967741935483 + }, + "19": { + "pass@1": 0.1644873489425982, + "pass@2": 0.22967898815329338, + "pass@4": 0.296141610225459, + "pass@8": 0.35542356027993677, + "pass@16": 0.40438333117475245, + "pass@32": 0.44624875815883597, + "pass@64": 0.4841766947910102, + "pass@128": 0.5196374622356495 + }, + "20": { + "pass@1": 0.16366366366366367, + "pass@2": 0.22465292556335845, + "pass@4": 0.28830474653506144, + "pass@8": 0.3515397113540908, + "pass@16": 0.4098242301095972, + "pass@32": 0.4585545918829372, + "pass@64": 0.49711705635141545, + "pass@128": 0.5285285285285285 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7163149671052632, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7163149671052632, + "pass@2": 0.7523812033775175, + "pass@4": 0.7827041436464997, + "pass@8": 0.8079569912329014, + "pass@16": 0.8286190437385256, + "pass@32": 0.8458090712996409, + "pass@64": 0.8608293054420325, + "pass@128": 0.874421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954203125, + "pass@2": 0.9668100393700787, + "pass@4": 0.9751169003562058, + "pass@8": 0.9805623138162619, + "pass@16": 0.9841740075635769, + "pass@32": 0.9868633714733398, + "pass@64": 0.9886806402008643, + "pass@128": 0.99 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990234375, + "pass@2": 0.999410063976378, + "pass@4": 0.9996751137982751, + "pass@8": 0.9998989158969318, + "pass@16": 0.9999915037321406, + "pass@32": 0.9999999700068081, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9949921875, + "pass@2": 0.9967944143700785, + "pass@4": 0.9977303270528682, + "pass@8": 0.9982286507773027, + "pass@16": 0.9986409291833077, + "pass@32": 0.999127515746347, + "pass@64": 0.9996924094487892, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9986875, + "pass@2": 0.9990652066929134, + "pass@4": 0.9992026385451819, + "pass@8": 0.99937067318584, + "pass@16": 0.9996167406956445, + "pass@32": 0.9998739053984705, + "pass@64": 0.9999934280775878, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9977421875, + "pass@2": 0.9986300442913385, + "pass@4": 0.9992237436257967, + "pass@8": 0.999713898666317, + "pass@16": 0.9999638315713223, + "pass@32": 0.9999996347541911, + "pass@64": 0.9999999999983727, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9968125, + "pass@2": 0.998474655511811, + "pass@4": 0.9989533392388451, + "pass@8": 0.9989996318701388, + "pass@16": 0.9989999999891657, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.994109375, + "pass@2": 0.9955601624015749, + "pass@4": 0.9961808598612675, + "pass@8": 0.9967566254314778, + "pass@16": 0.9973485536370097, + "pass@32": 0.9979841908426146, + "pass@64": 0.9986296156860006, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9424921875, + "pass@2": 0.9608925935039377, + "pass@4": 0.9707150841769778, + "pass@8": 0.9776590164210477, + "pass@16": 0.983031112952659, + "pass@32": 0.9868108166417683, + "pass@64": 0.9895059512740915, + "pass@128": 0.992 + }, + "12": { + "pass@1": 0.850125, + "pass@2": 0.8868829970472448, + "pass@4": 0.9069662009748772, + "pass@8": 0.9210538138715969, + "pass@16": 0.932209399638482, + "pass@32": 0.941670117173944, + "pass@64": 0.9509294871368753, + "pass@128": 0.961 + }, + "13": { + "pass@1": 0.723671875, + "pass@2": 0.7991273375984257, + "pass@4": 0.848705464191976, + "pass@8": 0.8832906602998529, + "pass@16": 0.907676298226515, + "pass@32": 0.9258390025587503, + "pass@64": 0.9408623838730613, + "pass@128": 0.954 + }, + "14": { + "pass@1": 0.621359375, + "pass@2": 0.7092471702755899, + "pass@4": 0.7710694283839518, + "pass@8": 0.8143357774146883, + "pass@16": 0.8459435216064888, + "pass@32": 0.8706776031668482, + "pass@64": 0.8916396601771912, + "pass@128": 0.911 + }, + "15": { + "pass@1": 0.4664140625, + "pass@2": 0.5631503444881897, + "pass@4": 0.6409323717660281, + "pass@8": 0.700385082200824, + "pass@16": 0.7464661637312656, + "pass@32": 0.7838249784646978, + "pass@64": 0.8150213816108757, + "pass@128": 0.842 + }, + "16": { + "pass@1": 0.300421875, + "pass@2": 0.38325036909448806, + "pass@4": 0.45934014332583345, + "pass@8": 0.5256147490839751, + "pass@16": 0.5812667481374817, + "pass@32": 0.6290118259853492, + "pass@64": 0.6714562961595262, + "pass@128": 0.71 + }, + "17": { + "pass@1": 0.2392421875, + "pass@2": 0.3153343996062992, + "pass@4": 0.390559457255343, + "pass@8": 0.45835220794408055, + "pass@16": 0.515216747490821, + "pass@32": 0.5633337200265438, + "pass@64": 0.6057651819056846, + "pass@128": 0.644 + }, + "18": { + "pass@1": 0.20425, + "pass@2": 0.27352423720472446, + "pass@4": 0.3419236596362953, + "pass@8": 0.4029165957544186, + "pass@16": 0.45499798232417954, + "pass@32": 0.5004840348220753, + "pass@64": 0.5415533976453777, + "pass@128": 0.578 + }, + "19": { + "pass@1": 0.169625, + "pass@2": 0.23175504429133828, + "pass@4": 0.29506946662917083, + "pass@8": 0.353428445257125, + "pass@16": 0.40308684493744273, + "pass@32": 0.4453773587288245, + "pass@64": 0.4839624351859198, + "pass@128": 0.52 + }, + "20": { + "pass@1": 0.1568203125, + "pass@2": 0.217333784448819, + "pass@4": 0.2800145304649416, + "pass@8": 0.3406157755330686, + "pass@16": 0.39513144561425667, + "pass@32": 0.4404943089023678, + "pass@64": 0.47906453501838553, + "pass@128": 0.514 + } + }, + "per_op_accuracy": { + "10": 0.954203125, + "2": 1.0, + "3": 0.9999921875, + "4": 0.9990234375, + "5": 0.9949921875, + "6": 0.9986875, + "7": 0.9977421875, + "8": 0.9968125, + "9": 0.994109375, + "11": 0.9424921875, + "12": 0.850125, + "13": 0.723671875, + "14": 0.621359375, + "15": 0.4664140625, + "16": 0.300421875, + "17": 0.2392421875, + "18": 0.20425, + "19": 0.169625, + "20": 0.1568203125 + }, + "per_op_avg_response_len": { + "10": 280.9476015625, + "2": 115.7252734375, + "3": 144.8482265625, + "4": 160.08678125, + "5": 173.194703125, + "6": 199.14746875, + "7": 224.20671875, + "8": 243.602875, + "9": 259.4781875, + "11": 298.1274375, + "12": 307.074171875, + "13": 322.5293828125, + "14": 340.953328125, + "15": 345.885015625, + "16": 332.6650234375, + "17": 347.350203125, + "18": 347.9971171875, + "19": 336.1864375, + "20": 347.200953125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.9476015625, + "2": 115.7252734375, + "3": 144.8482265625, + "4": 160.08678125, + "5": 173.194703125, + "6": 199.14746875, + "7": 224.20671875, + "8": 243.602875, + "9": 259.4781875, + "11": 298.1274375, + "12": 307.074171875, + "13": 322.5293828125, + "14": 340.953328125, + "15": 345.885015625, + "16": 332.6650234375, + "17": 347.350203125, + "18": 347.9971171875, + "19": 336.1864375, + "20": 347.200953125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 576926, + "answer_accuracy": 0.7124935780904205, + "avg_response_len": 257.13824642349033, + "resp_tokens_sum": 208212038, + "pass_at_k": { + "pass@1": 0.7124935780904205, + "pass@2": 0.7482208665462601, + "pass@4": 0.7782977803705171, + "pass@8": 0.8033142972254913, + "pass@16": 0.8239431045015444, + "pass@32": 0.841313857869306, + "pass@64": 0.8566171919665101, + "pass@128": 0.8706923806512804 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9639564896755162, + "pass@2": 0.9751933663159359, + "pass@4": 0.9805886808396739, + "pass@8": 0.9832729426842688, + "pass@16": 0.9853094435850058, + "pass@32": 0.9875159779599025, + "pass@64": 0.9903780082743677, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.997314453125, + "pass@2": 0.9981799028051181, + "pass@4": 0.9989848275801462, + "pass@8": 0.9996841121782726, + "pass@16": 0.9999734491629393, + "pass@32": 0.9999999062712751, + "pass@64": 0.9999999999999887, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9964285714285714, + "pass@2": 0.9987411729783777, + "pass@4": 0.9997791192767571, + "pass@8": 0.9999910052051663, + "pass@16": 0.999999987524699, + "pass@32": 0.999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970761601796407, + "pass@2": 0.9973235124239709, + "pass@4": 0.9976157328200742, + "pass@8": 0.9981157895218378, + "pass@16": 0.998852517052828, + "pass@32": 0.9996224712529058, + "pass@64": 0.9999803235855924, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972725591715976, + "pass@2": 0.9980547919675722, + "pass@4": 0.9987735604794961, + "pass@8": 0.9995079830474547, + "pass@16": 0.999928825622786, + "pass@32": 0.9999991140832827, + "pass@64": 0.9999999999952738, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9991486378205128, + "pass@2": 0.9999641158136482, + "pass@4": 0.999999888836011, + "pass@8": 0.9999999999996302, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9955325704225352, + "pass@2": 0.9969474326272597, + "pass@4": 0.9973978846834287, + "pass@8": 0.9976821039095084, + "pass@16": 0.998119293157022, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9345122167138811, + "pass@2": 0.9542368143695208, + "pass@4": 0.9657462063877991, + "pass@8": 0.9749512748470323, + "pass@16": 0.9827357531203197, + "pass@32": 0.9884344587547554, + "pass@64": 0.9925691216469926, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.8454493290960452, + "pass@2": 0.8801169285777846, + "pass@4": 0.8989873583069065, + "pass@8": 0.9130351649779792, + "pass@16": 0.9250958046726186, + "pass@32": 0.9364960935539219, + "pass@64": 0.9474650567648585, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.74047315830721, + "pass@2": 0.8119554217164869, + "pass@4": 0.8599147251969675, + "pass@8": 0.8945134207641785, + "pass@16": 0.9205661933194598, + "pass@32": 0.9404403427759488, + "pass@64": 0.9559823521026871, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.6125632788161994, + "pass@2": 0.7013274370201388, + "pass@4": 0.7658476230190846, + "pass@8": 0.8097702298192307, + "pass@16": 0.8407102562368179, + "pass@32": 0.8644964958281782, + "pass@64": 0.8838021812976606, + "pass@128": 0.9003115264797508 + }, + "15": { + "pass@1": 0.48457821531791906, + "pass@2": 0.5827305027081152, + "pass@4": 0.6590647869702699, + "pass@8": 0.7158310076759379, + "pass@16": 0.7589664128271321, + "pass@32": 0.7930825505983045, + "pass@64": 0.8204516163076292, + "pass@128": 0.8410404624277457 + }, + "16": { + "pass@1": 0.2755040322580645, + "pass@2": 0.3586213328676657, + "pass@4": 0.43517604109970115, + "pass@8": 0.5017510566263869, + "pass@16": 0.5583572473734497, + "pass@32": 0.6078831915021241, + "pass@64": 0.6517453480905581, + "pass@128": 0.6903225806451613 + }, + "17": { + "pass@1": 0.21780166184971098, + "pass@2": 0.29274348857584986, + "pass@4": 0.36758594012453627, + "pass@8": 0.4327260544001389, + "pass@16": 0.4852625887317101, + "pass@32": 0.5308733536032256, + "pass@64": 0.5731955827266514, + "pass@128": 0.615606936416185 + }, + "18": { + "pass@1": 0.18082589285714284, + "pass@2": 0.24475007030371193, + "pass@4": 0.30927755396646855, + "pass@8": 0.3685482161042458, + "pass@16": 0.4204588630253465, + "pass@32": 0.46585847061159913, + "pass@64": 0.5068792519157992, + "pass@128": 0.5457142857142857 + }, + "19": { + "pass@1": 0.1753472222222222, + "pass@2": 0.23986560379276908, + "pass@4": 0.3042127763308865, + "pass@8": 0.3648899684541569, + "pass@16": 0.41793983675736257, + "pass@32": 0.46237397780199607, + "pass@64": 0.5029501083188042, + "pass@128": 0.5405405405405406 + }, + "20": { + "pass@1": 0.15636730480480482, + "pass@2": 0.2165424528859568, + "pass@4": 0.27821547475484476, + "pass@8": 0.3365076147164066, + "pass@16": 0.3888313889531125, + "pass@32": 0.4337362063462613, + "pass@64": 0.4738117632886543, + "pass@128": 0.5135135135135135 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 589293, + "answer_accuracy": 0.71766976812159, + "avg_response_len": 266.5054985872954, + "resp_tokens_sum": 218832995, + "pass_at_k": { + "pass@1": 0.71766976812159, + "pass@2": 0.7541205482352454, + "pass@4": 0.7849935205585589, + "pass@8": 0.8106964627292033, + "pass@16": 0.831317315567593, + "pass@32": 0.8479933921096046, + "pass@64": 0.8623722337450352, + "pass@128": 0.875448168355417 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9412384969325154, + "pass@2": 0.9570337030819772, + "pass@4": 0.9684386097635029, + "pass@8": 0.9755421990555402, + "pass@16": 0.9790090135040668, + "pass@32": 0.9808182150392867, + "pass@64": 0.9815547734367644, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999075443786982, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9963768115942029, + "pass@2": 0.9971339010612804, + "pass@4": 0.9974446123582379, + "pass@8": 0.9977680501821345, + "pass@16": 0.9983124881128967, + "pass@32": 0.999097406302473, + "pass@64": 0.9998273650576287, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997313753581661, + "pass@2": 0.9999841363400491, + "pass@4": 0.9999999435958757, + "pass@8": 0.9999999999999097, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997674313583815, + "pass@2": 0.998275069978608, + "pass@4": 0.9989801556597332, + "pass@8": 0.9996539139236084, + "pass@16": 0.9999649956979594, + "pass@32": 0.9999998098093686, + "pass@64": 0.9999999999999141, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957306338028169, + "pass@2": 0.9968081124542529, + "pass@4": 0.9971466248937193, + "pass@8": 0.9971827493473799, + "pass@16": 0.9971830985789736, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.993310254491018, + "pass@2": 0.9940558106228489, + "pass@4": 0.994196379457059, + "pass@8": 0.9943759135003243, + "pass@16": 0.9947162761091989, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9478068524096386, + "pass@2": 0.9611561403092685, + "pass@4": 0.9686419691702693, + "pass@8": 0.9750669382908692, + "pass@16": 0.9806081356024587, + "pass@32": 0.9842428892059527, + "pass@64": 0.9862549466314855, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8410315438871473, + "pass@2": 0.8755083244884359, + "pass@4": 0.8937380000305601, + "pass@8": 0.905965360565369, + "pass@16": 0.9151980908995769, + "pass@32": 0.9235809759059825, + "pass@64": 0.9339121914379174, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.7262123599439776, + "pass@2": 0.8062757091025381, + "pass@4": 0.8566690912585506, + "pass@8": 0.8882637919932588, + "pass@16": 0.9082264797954909, + "pass@32": 0.9222958616323473, + "pass@64": 0.9343518871725501, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6329952485380117, + "pass@2": 0.7221707792282543, + "pass@4": 0.7828848704767167, + "pass@8": 0.8258118272832482, + "pass@16": 0.8579023706600551, + "pass@32": 0.8832258032244631, + "pass@64": 0.9049950902372479, + "pass@128": 0.9269005847953217 + }, + "15": { + "pass@1": 0.4446022727272727, + "pass@2": 0.5437966028394173, + "pass@4": 0.6285262524002679, + "pass@8": 0.6960194300230498, + "pass@16": 0.7475932370874276, + "pass@32": 0.785985388651888, + "pass@64": 0.8147437752003887, + "pass@128": 0.8424242424242424 + }, + "16": { + "pass@1": 0.3059116809116809, + "pass@2": 0.39158751990936996, + "pass@4": 0.4727661555660242, + "pass@8": 0.5448737019577484, + "pass@16": 0.6040431432838912, + "pass@32": 0.6528764342680178, + "pass@64": 0.694521859735016, + "pass@128": 0.7293447293447294 + }, + "17": { + "pass@1": 0.26052989130434784, + "pass@2": 0.34320820291485293, + "pass@4": 0.42293633977041684, + "pass@8": 0.4927356348756277, + "pass@16": 0.5482837994495754, + "pass@32": 0.5921106992309274, + "pass@64": 0.6290502589670679, + "pass@128": 0.6614906832298136 + }, + "18": { + "pass@1": 0.2153262867647059, + "pass@2": 0.28677049559981455, + "pass@4": 0.35674156300315396, + "pass@8": 0.4197868754911729, + "pass@16": 0.47482275196634255, + "pass@32": 0.5236327311772606, + "pass@64": 0.5672130434247589, + "pass@128": 0.6029411764705882 + }, + "19": { + "pass@1": 0.16901506696428573, + "pass@2": 0.22576206294525675, + "pass@4": 0.28495160454273577, + "pass@8": 0.34010383127747634, + "pass@16": 0.38708927582856617, + "pass@32": 0.4276740630065036, + "pass@64": 0.4649332236307543, + "pass@128": 0.5 + }, + "20": { + "pass@1": 0.1504491017964072, + "pass@2": 0.2108255192371163, + "pass@4": 0.2735428047167756, + "pass@8": 0.3338204070406974, + "pass@16": 0.38676384570770866, + "pass@32": 0.4292259673414519, + "pass@64": 0.4663031080186876, + "pass@128": 0.5 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 575859, + "answer_accuracy": 0.7187886942802365, + "avg_response_len": 286.13477966727913, + "resp_tokens_sum": 229237451, + "pass_at_k": { + "pass@1": 0.7187886942802365, + "pass@2": 0.7548033784106732, + "pass@4": 0.7848112376220922, + "pass@8": 0.8098416329714997, + "pass@16": 0.8305795131153559, + "pass@32": 0.8481136410654446, + "pass@64": 0.8635051234293231, + "pass@128": 0.8771369228311232 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9569496268656716, + "pass@2": 0.9678403014455282, + "pass@4": 0.976078659010907, + "pass@8": 0.9827045652363846, + "pass@16": 0.9880512471221918, + "pass@32": 0.9920857219167919, + "pass@64": 0.9938974306133376, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997487207602339, + "pass@2": 0.9999802142330892, + "pass@4": 0.9999999095507799, + "pass@8": 0.9999999999996626, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9922564338235295, + "pass@2": 0.994646320634553, + "pass@4": 0.9961220976973466, + "pass@8": 0.9970632553671733, + "pass@16": 0.997715072688465, + "pass@32": 0.9983497369764539, + "pass@64": 0.9992704955997272, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992360015772871, + "pass@2": 0.9998886118631859, + "pass@4": 0.9999968563669289, + "pass@8": 0.9999999983772165, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998318829113924, + "pass@2": 0.9996340202332306, + "pass@4": 0.9999719820425929, + "pass@8": 0.9999998296794577, + "pass@16": 0.999999999997869, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995777027027027, + "pass@2": 0.9988557701402191, + "pass@4": 0.9998988651193375, + "pass@8": 0.9999992668226257, + "pass@16": 0.9999999999808709, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9933430466237942, + "pass@2": 0.9955922285236853, + "pass@4": 0.996922894083577, + "pass@8": 0.99825698853535, + "pass@16": 0.9992957181536803, + "pass@32": 0.9998834594758289, + "pass@64": 0.9999990174474539, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9458333333333333, + "pass@2": 0.9680735220597424, + "pass@4": 0.9784683795477946, + "pass@8": 0.9834253742459571, + "pass@16": 0.9859158447021265, + "pass@32": 0.9876978205871854, + "pass@64": 0.9894997112731746, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8640577217125383, + "pass@2": 0.9053041248284324, + "pass@4": 0.9285084223990344, + "pass@8": 0.9444538698747363, + "pass@16": 0.9565054855884125, + "pass@32": 0.9649179472227764, + "pass@64": 0.971280941814739, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.7043306327160493, + "pass@2": 0.7786207713619127, + "pass@4": 0.8288944483791375, + "pass@8": 0.86676143621755, + "pass@16": 0.8943791026870926, + "pass@32": 0.9153670080570203, + "pass@64": 0.9331493513324162, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.6179293397626113, + "pass@2": 0.703675538563985, + "pass@4": 0.7640525688183634, + "pass@8": 0.8070382454357383, + "pass@16": 0.8387920432900056, + "pass@32": 0.8638308703360127, + "pass@64": 0.8855514513931839, + "pass@128": 0.9050445103857567 + }, + "15": { + "pass@1": 0.46923225308641975, + "pass@2": 0.5619528136239913, + "pass@4": 0.6342046055007012, + "pass@8": 0.6883368569053809, + "pass@16": 0.7319691872044063, + "pass@32": 0.7717383879708698, + "pass@64": 0.8095051745441575, + "pass@128": 0.8425925925925926 + }, + "16": { + "pass@1": 0.31752396755162243, + "pass@2": 0.3971402254198313, + "pass@4": 0.46753584065266157, + "pass@8": 0.5274963190047928, + "pass@16": 0.578633799879253, + "pass@32": 0.6236236229841198, + "pass@64": 0.6655990132285015, + "pass@128": 0.7079646017699115 + }, + "17": { + "pass@1": 0.2409403237951807, + "pass@2": 0.3118437084953988, + "pass@4": 0.38310006194632296, + "pass@8": 0.4517112008785566, + "pass@16": 0.5143629770086949, + "pass@32": 0.5692526943598468, + "pass@64": 0.6171244785989948, + "pass@128": 0.6566265060240963 + }, + "18": { + "pass@1": 0.21854838709677418, + "pass@2": 0.2914830454660909, + "pass@4": 0.36253027202244875, + "pass@8": 0.4232167175836581, + "pass@16": 0.4722504664411351, + "pass@32": 0.5141885209927322, + "pass@64": 0.5525587893884821, + "pass@128": 0.5870967741935483 + }, + "19": { + "pass@1": 0.1644873489425982, + "pass@2": 0.22967898815329338, + "pass@4": 0.296141610225459, + "pass@8": 0.35542356027993677, + "pass@16": 0.40438333117475245, + "pass@32": 0.44624875815883597, + "pass@64": 0.4841766947910102, + "pass@128": 0.5196374622356495 + }, + "20": { + "pass@1": 0.16366366366366367, + "pass@2": 0.22465292556335845, + "pass@4": 0.28830474653506144, + "pass@8": 0.3515397113540908, + "pass@16": 0.4098242301095972, + "pass@32": 0.4585545918829372, + "pass@64": 0.49711705635141545, + "pass@128": 0.5285285285285285 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block16-epoch4-op7-10/graphs/SDAR-100M-block16-epoch4-op7-10_score_report.jsonl b/composition/SDAR-100M-block16-epoch4-op7-10/graphs/SDAR-100M-block16-epoch4-op7-10_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0bd320aa0eee6003250598066c0e074fd5a34433 --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op7-10/graphs/SDAR-100M-block16-epoch4-op7-10_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block16-epoch4-op7-10", "results_dir": "results/composition/SDAR-100M-block16-epoch4-op7-10/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1022314, "value_mismatch": 9380, "dependency_mismatch": 127172, "answer_mismatch": 8248, "extra_nodes": 10711, "missing_nodes": 10228, "perfect_with_extra": 179, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337324, "value_mismatch": 2860, "dependency_mismatch": 41065, "answer_mismatch": 2299, "extra_nodes": 3333, "missing_nodes": 2955, "perfect_with_extra": 29}, "teachers_in_school": {"total": 390656, "perfect": 346496, "value_mismatch": 3337, "dependency_mismatch": 43165, "answer_mismatch": 3211, "extra_nodes": 3333, "missing_nodes": 3370, "perfect_with_extra": 53}, "movie_festival_awards": {"total": 382336, "perfect": 338494, "dependency_mismatch": 42942, "value_mismatch": 3183, "answer_mismatch": 2738, "extra_nodes": 4045, "missing_nodes": 3903, "perfect_with_extra": 97}}, "results_path": "results/composition/SDAR-100M-block16-epoch4-op7-10/graphs/SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 254076, "value_mismatch": 747166, "dependency_mismatch": 894487, "answer_mismatch": 681673, "extra_nodes": 132084, "missing_nodes": 760728, "perfect_with_extra": 142, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 87143, "value_mismatch": 253319, "dependency_mismatch": 303919, "answer_mismatch": 230503, "extra_nodes": 36598, "missing_nodes": 255942, "perfect_with_extra": 118}, "movie_festival_awards": {"total": 418816, "perfect": 82434, "dependency_mismatch": 290293, "value_mismatch": 241288, "answer_mismatch": 222556, "extra_nodes": 50977, "missing_nodes": 248055, "perfect_with_extra": 21}, "teachers_in_school": {"total": 430464, "perfect": 84499, "dependency_mismatch": 300275, "value_mismatch": 252559, "answer_mismatch": 228614, "extra_nodes": 44509, "missing_nodes": 256731, "perfect_with_extra": 3}}, "results_path": "results/composition/SDAR-100M-block16-epoch4-op7-10/graphs/SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1276390, "value_mismatch": 756546, "dependency_mismatch": 1021659, "answer_mismatch": 689921, "extra_nodes": 142795, "missing_nodes": 770956, "perfect_with_extra": 321, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block16-epoch4-op7-10/graphs/SDAR-100M-block16-epoch4-op7-10_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 424467, "value_mismatch": 256179, "dependency_mismatch": 344984, "answer_mismatch": 232802, "extra_nodes": 39931, "missing_nodes": 258897, "perfect_with_extra": 147}, "teachers_in_school": {"total": 821120, "perfect": 430995, "value_mismatch": 255896, "dependency_mismatch": 343440, "answer_mismatch": 231825, "extra_nodes": 47842, "missing_nodes": 260101, "perfect_with_extra": 56}, "movie_festival_awards": {"total": 801152, "perfect": 420928, "dependency_mismatch": 333235, "value_mismatch": 244471, "answer_mismatch": 225294, "extra_nodes": 55022, "missing_nodes": 251958, "perfect_with_extra": 118}}}} diff --git a/composition/SDAR-100M-block16-epoch4-op7-10/graphs/results.json b/composition/SDAR-100M-block16-epoch4-op7-10/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..133df52ce2d4778da8e3453538cfc1e72afbd934 --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op7-10/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990625, + "pass@2": 0.99375, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9619047619047619, + "pass@2": 0.9841269841269841, + "pass@4": 0.9936507936507937, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9670658682634731, + "pass@2": 0.9670658682634731, + "pass@4": 0.9850299401197605, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9023668639053254, + "pass@2": 0.9408284023668639, + "pass@4": 0.9704142011834319, + "pass@8": 0.9822485207100592, + "pass@16": 0.9970414201183432, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8205128205128205, + "pass@2": 0.8589743589743589, + "pass@4": 0.9262820512820513, + "pass@8": 0.9743589743589743, + "pass@16": 0.9903846153846154, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7746478873239436, + "pass@2": 0.819718309859155, + "pass@4": 0.9154929577464789, + "pass@8": 0.9577464788732394, + "pass@16": 0.9830985915492958, + "pass@32": 0.9915492957746479, + "pass@64": 0.9971830985915493, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6430678466076696, + "pass@2": 0.7109144542772862, + "pass@4": 0.7905604719764012, + "pass@8": 0.8702064896755162, + "pass@16": 0.9262536873156342, + "pass@32": 0.9528023598820059, + "pass@64": 0.9616519174041298, + "pass@128": 0.9646017699115044 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9619047619047619, + "6": 0.9670658682634731, + "7": 0.9023668639053254, + "8": 0.8205128205128205, + "9": 0.7746478873239436, + "10": 0.6430678466076696 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9841269841269841, + "6": 0.9670658682634731, + "7": 0.9408284023668639, + "8": 0.8589743589743589, + "9": 0.819718309859155, + "10": 0.7109144542772862 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 0.9850299401197605, + "7": 0.9704142011834319, + "8": 0.9262820512820513, + "9": 0.9154929577464789, + "10": 0.7905604719764012 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9822485207100592, + "8": 0.9743589743589743, + "9": 0.9577464788732394, + "10": 0.8702064896755162 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9903846153846154, + "9": 0.9830985915492958, + "10": 0.9262536873156342 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9915492957746479, + "10": 0.9528023598820059 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9616519174041298 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9646017699115044 + } + }, + "total_pass_at_k": { + "pass@1": 0.8932792975346167, + "pass@2": 0.9175954069571091, + "pass@4": 0.9523809523809523, + "pass@8": 0.9753461668355285, + "pass@16": 0.9881796690307328, + "pass@32": 0.9935832489023978, + "pass@64": 0.9952718676122931, + "pass@128": 0.9959473150962512 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9971428571428571, + "pass@2": 0.9971428571428571, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9912280701754386, + "pass@2": 0.9970760233918129, + "pass@4": 0.9970760233918129, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9382352941176471, + "pass@2": 0.9588235294117647, + "pass@4": 0.9941176470588236, + "pass@8": 0.9970588235294118, + "pass@16": 0.9970588235294118, + "pass@32": 0.9970588235294118, + "pass@64": 0.9970588235294118, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9747634069400631, + "pass@2": 0.9810725552050473, + "pass@4": 0.9968454258675079, + "pass@8": 0.9968454258675079, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8481012658227848, + "pass@2": 0.9018987341772152, + "pass@4": 0.9556962025316456, + "pass@8": 0.9810126582278481, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7807807807807807, + "pass@2": 0.8468468468468469, + "pass@4": 0.924924924924925, + "pass@8": 0.978978978978979, + "pass@16": 0.993993993993994, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7556270096463023, + "pass@2": 0.8456591639871383, + "pass@4": 0.909967845659164, + "pass@8": 0.954983922829582, + "pass@16": 0.9871382636655949, + "pass@32": 0.9967845659163987, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6179104477611941, + "pass@2": 0.6955223880597015, + "pass@4": 0.7850746268656716, + "pass@8": 0.8686567164179104, + "pass@16": 0.9134328358208955, + "pass@32": 0.9343283582089552, + "pass@64": 0.9552238805970149, + "pass@128": 0.9701492537313433 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9382352941176471, + "6": 0.9747634069400631, + "7": 0.8481012658227848, + "8": 0.7807807807807807, + "9": 0.7556270096463023, + "10": 0.6179104477611941 + }, + "pass@2": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9588235294117647, + "6": 0.9810725552050473, + "7": 0.9018987341772152, + "8": 0.8468468468468469, + "9": 0.8456591639871383, + "10": 0.6955223880597015 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9941176470588236, + "6": 0.9968454258675079, + "7": 0.9556962025316456, + "8": 0.924924924924925, + "9": 0.909967845659164, + "10": 0.7850746268656716 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9968454258675079, + "7": 0.9810126582278481, + "8": 0.978978978978979, + "9": 0.954983922829582, + "10": 0.8686567164179104 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 0.993993993993994, + "9": 0.9871382636655949, + "10": 0.9134328358208955 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9967845659163987, + "10": 0.9343283582089552 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9552238805970149 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9701492537313433 + } + }, + "total_pass_at_k": { + "pass@1": 0.8801473049882825, + "pass@2": 0.9149648476732507, + "pass@4": 0.9521258788081687, + "pass@8": 0.9755607633076665, + "pass@16": 0.9879477736859725, + "pass@32": 0.991965182457315, + "pass@64": 0.9946434549715434, + "pass@128": 0.9963173752929361 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9822485207100592, + "pass@2": 0.9940828402366864, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9594202898550724, + "pass@2": 0.9855072463768116, + "pass@4": 0.9971014492753624, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 0.9971014492753624, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9799426934097422, + "pass@2": 0.9799426934097422, + "pass@4": 0.9885386819484241, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8641618497109826, + "pass@2": 0.9075144508670521, + "pass@4": 0.953757225433526, + "pass@8": 0.9826589595375722, + "pass@16": 0.9942196531791907, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8112676056338028, + "pass@2": 0.8704225352112676, + "pass@4": 0.9408450704225352, + "pass@8": 0.9774647887323944, + "pass@16": 0.9887323943661972, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.7155688622754491, + "pass@2": 0.7964071856287425, + "pass@4": 0.8682634730538922, + "pass@8": 0.9251497005988024, + "pass@16": 0.9640718562874252, + "pass@32": 0.9880239520958084, + "pass@64": 0.9970059880239521, + "pass@128": 0.9970059880239521 + }, + "10": { + "pass@1": 0.6319018404907976, + "pass@2": 0.696319018404908, + "pass@4": 0.7730061349693251, + "pass@8": 0.8680981595092024, + "pass@16": 0.9049079754601227, + "pass@32": 0.9325153374233128, + "pass@64": 0.9570552147239264, + "pass@128": 0.9662576687116564 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9822485207100592, + "5": 0.9594202898550724, + "6": 0.9799426934097422, + "7": 0.8641618497109826, + "8": 0.8112676056338028, + "9": 0.7155688622754491, + "10": 0.6319018404907976 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9940828402366864, + "5": 0.9855072463768116, + "6": 0.9799426934097422, + "7": 0.9075144508670521, + "8": 0.8704225352112676, + "9": 0.7964071856287425, + "10": 0.696319018404908 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.9885386819484241, + "7": 0.953757225433526, + "8": 0.9408450704225352, + "9": 0.8682634730538922, + "10": 0.7730061349693251 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 0.9826589595375722, + "8": 0.9774647887323944, + "9": 0.9251497005988024, + "10": 0.8680981595092024 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 0.9942196531791907, + "8": 0.9887323943661972, + "9": 0.9640718562874252, + "10": 0.9049079754601227 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9880239520958084, + "10": 0.9325153374233128 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9970059880239521, + "10": 0.9570552147239264 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9970059880239521, + "10": 0.9662576687116564 + } + }, + "total_pass_at_k": { + "pass@1": 0.8833551769331586, + "pass@2": 0.9151376146788991, + "pass@4": 0.9475753604193972, + "pass@8": 0.9728047182175622, + "pass@16": 0.9836173001310616, + "pass@32": 0.9908256880733946, + "pass@64": 0.9947575360419397, + "pass@128": 0.995740498034076 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.999, + "pass@2": 0.999, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.988, + "pass@2": 0.995, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.953, + "pass@2": 0.976, + "pass@4": 0.995, + "pass@8": 0.998, + "pass@16": 0.998, + "pass@32": 0.998, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.974, + "pass@2": 0.976, + "pass@4": 0.99, + "pass@8": 0.999, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.872, + "pass@2": 0.917, + "pass@4": 0.96, + "pass@8": 0.982, + "pass@16": 0.997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.804, + "pass@2": 0.859, + "pass@4": 0.931, + "pass@8": 0.977, + "pass@16": 0.991, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.749, + "pass@2": 0.82, + "pass@4": 0.898, + "pass@8": 0.946, + "pass@16": 0.978, + "pass@32": 0.992, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "10": { + "pass@1": 0.631, + "pass@2": 0.701, + "pass@4": 0.783, + "pass@8": 0.869, + "pass@16": 0.915, + "pass@32": 0.94, + "pass@64": 0.958, + "pass@128": 0.967 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.999, + "3": 1.0, + "4": 0.988, + "5": 0.953, + "6": 0.974, + "7": 0.872, + "8": 0.804, + "9": 0.749, + "10": 0.631 + }, + "pass@2": { + "2": 0.999, + "3": 1.0, + "4": 0.995, + "5": 0.976, + "6": 0.976, + "7": 0.917, + "8": 0.859, + "9": 0.82, + "10": 0.701 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.995, + "6": 0.99, + "7": 0.96, + "8": 0.931, + "9": 0.898, + "10": 0.783 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 0.999, + "7": 0.982, + "8": 0.977, + "9": 0.946, + "10": 0.869 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 1.0, + "7": 0.997, + "8": 0.991, + "9": 0.978, + "10": 0.915 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.992, + "10": 0.94 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.998, + "10": 0.958 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.999, + "10": 0.967 + } + }, + "total_pass_at_k": { + "pass@1": 0.8855555555555555, + "pass@2": 0.9158888888888889, + "pass@4": 0.9506666666666667, + "pass@8": 0.9745555555555555, + "pass@16": 0.9865555555555555, + "pass@32": 0.9921111111111112, + "pass@64": 0.9948888888888889, + "pass@128": 0.996 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.56657223796034, + "pass@2": 0.6288951841359773, + "pass@4": 0.7138810198300283, + "pass@8": 0.7790368271954674, + "pass@16": 0.8923512747875354, + "pass@32": 0.9121813031161473, + "pass@64": 0.9320113314447592, + "pass@128": 0.943342776203966 + }, + "12": { + "pass@1": 0.5508474576271186, + "pass@2": 0.6129943502824858, + "pass@4": 0.6779661016949152, + "pass@8": 0.7344632768361582, + "pass@16": 0.7909604519774012, + "pass@32": 0.8192090395480226, + "pass@64": 0.8389830508474576, + "pass@128": 0.8587570621468926 + }, + "13": { + "pass@1": 0.3949843260188088, + "pass@2": 0.47648902821316613, + "pass@4": 0.5705329153605015, + "pass@8": 0.6363636363636364, + "pass@16": 0.6896551724137931, + "pass@32": 0.7241379310344828, + "pass@64": 0.7586206896551724, + "pass@128": 0.7962382445141066 + }, + "14": { + "pass@1": 0.27414330218068533, + "pass@2": 0.3302180685358255, + "pass@4": 0.37694704049844235, + "pass@8": 0.43613707165109034, + "pass@16": 0.5077881619937694, + "pass@32": 0.573208722741433, + "pass@64": 0.6355140186915887, + "pass@128": 0.6635514018691588 + }, + "15": { + "pass@1": 0.15895953757225434, + "pass@2": 0.2138728323699422, + "pass@4": 0.2543352601156069, + "pass@8": 0.3063583815028902, + "pass@16": 0.33236994219653176, + "pass@32": 0.4046242774566474, + "pass@64": 0.47398843930635837, + "pass@128": 0.5260115606936416 + }, + "16": { + "pass@1": 0.02258064516129032, + "pass@2": 0.04516129032258064, + "pass@4": 0.05806451612903226, + "pass@8": 0.08387096774193549, + "pass@16": 0.1032258064516129, + "pass@32": 0.12580645161290321, + "pass@64": 0.15806451612903225, + "pass@128": 0.18064516129032257 + }, + "17": { + "pass@1": 0.023121387283236993, + "pass@2": 0.028901734104046242, + "pass@4": 0.04335260115606936, + "pass@8": 0.05202312138728324, + "pass@16": 0.07225433526011561, + "pass@32": 0.08092485549132948, + "pass@64": 0.09826589595375723, + "pass@128": 0.10982658959537572 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.002857142857142857, + "pass@8": 0.002857142857142857, + "pass@16": 0.008571428571428572, + "pass@32": 0.017142857142857144, + "pass@64": 0.02857142857142857, + "pass@128": 0.04 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.003003003003003003, + "pass@8": 0.003003003003003003, + "pass@16": 0.003003003003003003, + "pass@32": 0.003003003003003003, + "pass@64": 0.003003003003003003, + "pass@128": 0.006006006006006006 + }, + "20": { + "pass@1": 0.003003003003003003, + "pass@2": 0.006006006006006006, + "pass@4": 0.006006006006006006, + "pass@8": 0.006006006006006006, + "pass@16": 0.006006006006006006, + "pass@32": 0.006006006006006006, + "pass@64": 0.006006006006006006, + "pass@128": 0.018018018018018018 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.56657223796034, + "12": 0.5508474576271186, + "13": 0.3949843260188088, + "14": 0.27414330218068533, + "15": 0.15895953757225434, + "16": 0.02258064516129032, + "17": 0.023121387283236993, + "18": 0.0, + "19": 0.0, + "20": 0.003003003003003003 + }, + "pass@2": { + "11": 0.6288951841359773, + "12": 0.6129943502824858, + "13": 0.47648902821316613, + "14": 0.3302180685358255, + "15": 0.2138728323699422, + "16": 0.04516129032258064, + "17": 0.028901734104046242, + "18": 0.0, + "19": 0.0, + "20": 0.006006006006006006 + }, + "pass@4": { + "11": 0.7138810198300283, + "12": 0.6779661016949152, + "13": 0.5705329153605015, + "14": 0.37694704049844235, + "15": 0.2543352601156069, + "16": 0.05806451612903226, + "17": 0.04335260115606936, + "18": 0.002857142857142857, + "19": 0.003003003003003003, + "20": 0.006006006006006006 + }, + "pass@8": { + "11": 0.7790368271954674, + "12": 0.7344632768361582, + "13": 0.6363636363636364, + "14": 0.43613707165109034, + "15": 0.3063583815028902, + "16": 0.08387096774193549, + "17": 0.05202312138728324, + "18": 0.002857142857142857, + "19": 0.003003003003003003, + "20": 0.006006006006006006 + }, + "pass@16": { + "11": 0.8923512747875354, + "12": 0.7909604519774012, + "13": 0.6896551724137931, + "14": 0.5077881619937694, + "15": 0.33236994219653176, + "16": 0.1032258064516129, + "17": 0.07225433526011561, + "18": 0.008571428571428572, + "19": 0.003003003003003003, + "20": 0.006006006006006006 + }, + "pass@32": { + "11": 0.9121813031161473, + "12": 0.8192090395480226, + "13": 0.7241379310344828, + "14": 0.573208722741433, + "15": 0.4046242774566474, + "16": 0.12580645161290321, + "17": 0.08092485549132948, + "18": 0.017142857142857144, + "19": 0.003003003003003003, + "20": 0.006006006006006006 + }, + "pass@64": { + "11": 0.9320113314447592, + "12": 0.8389830508474576, + "13": 0.7586206896551724, + "14": 0.6355140186915887, + "15": 0.47398843930635837, + "16": 0.15806451612903225, + "17": 0.09826589595375723, + "18": 0.02857142857142857, + "19": 0.003003003003003003, + "20": 0.006006006006006006 + }, + "pass@128": { + "11": 0.943342776203966, + "12": 0.8587570621468926, + "13": 0.7962382445141066, + "14": 0.6635514018691588, + "15": 0.5260115606936416, + "16": 0.18064516129032257, + "17": 0.10982658959537572, + "18": 0.04, + "19": 0.006006006006006006, + "20": 0.018018018018018018 + } + }, + "total_pass_at_k": { + "pass@1": 0.20208023774145617, + "pass@2": 0.23684992570579494, + "pass@4": 0.27340267459138184, + "pass@8": 0.30668647845468056, + "pass@16": 0.3435364041604755, + "pass@32": 0.36939078751857357, + "pass@64": 0.39583952451708765, + "pass@128": 0.41664190193164935 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5714285714285714, + "pass@2": 0.6444444444444445, + "pass@4": 0.7428571428571429, + "pass@8": 0.8539682539682539, + "pass@16": 0.9238095238095239, + "pass@32": 0.9492063492063492, + "pass@64": 0.9650793650793651, + "pass@128": 0.9682539682539683 + }, + "12": { + "pass@1": 0.5290519877675841, + "pass@2": 0.599388379204893, + "pass@4": 0.6850152905198776, + "pass@8": 0.7584097859327217, + "pass@16": 0.8165137614678899, + "pass@32": 0.8501529051987767, + "pass@64": 0.8746177370030581, + "pass@128": 0.8868501529051988 + }, + "13": { + "pass@1": 0.36419753086419754, + "pass@2": 0.4506172839506173, + "pass@4": 0.5246913580246914, + "pass@8": 0.5925925925925926, + "pass@16": 0.6574074074074074, + "pass@32": 0.7006172839506173, + "pass@64": 0.7314814814814815, + "pass@128": 0.7469135802469136 + }, + "14": { + "pass@1": 0.2403560830860534, + "pass@2": 0.3026706231454006, + "pass@4": 0.35311572700296734, + "pass@8": 0.4065281899109792, + "pass@16": 0.47774480712166173, + "pass@32": 0.5489614243323442, + "pass@64": 0.6261127596439169, + "pass@128": 0.6765578635014837 + }, + "15": { + "pass@1": 0.1697530864197531, + "pass@2": 0.20679012345679013, + "pass@4": 0.2623456790123457, + "pass@8": 0.3117283950617284, + "pass@16": 0.36728395061728397, + "pass@32": 0.41975308641975306, + "pass@64": 0.5092592592592593, + "pass@128": 0.5709876543209876 + }, + "16": { + "pass@1": 0.035398230088495575, + "pass@2": 0.04129793510324484, + "pass@4": 0.06784660766961652, + "pass@8": 0.09144542772861357, + "pass@16": 0.11209439528023599, + "pass@32": 0.13569321533923304, + "pass@64": 0.168141592920354, + "pass@128": 0.1887905604719764 + }, + "17": { + "pass@1": 0.030120481927710843, + "pass@2": 0.05120481927710843, + "pass@4": 0.05421686746987952, + "pass@8": 0.060240963855421686, + "pass@16": 0.06325301204819277, + "pass@32": 0.07530120481927711, + "pass@64": 0.09036144578313253, + "pass@128": 0.12349397590361445 + }, + "18": { + "pass@1": 0.02258064516129032, + "pass@2": 0.025806451612903226, + "pass@4": 0.041935483870967745, + "pass@8": 0.04838709677419355, + "pass@16": 0.06129032258064516, + "pass@32": 0.07419354838709677, + "pass@64": 0.09032258064516129, + "pass@128": 0.0967741935483871 + }, + "19": { + "pass@1": 0.006042296072507553, + "pass@2": 0.006042296072507553, + "pass@4": 0.00906344410876133, + "pass@8": 0.00906344410876133, + "pass@16": 0.012084592145015106, + "pass@32": 0.015105740181268883, + "pass@64": 0.015105740181268883, + "pass@128": 0.01812688821752266 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.003003003003003003, + "pass@16": 0.006006006006006006, + "pass@32": 0.006006006006006006, + "pass@64": 0.006006006006006006, + "pass@128": 0.006006006006006006 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5714285714285714, + "12": 0.5290519877675841, + "13": 0.36419753086419754, + "14": 0.2403560830860534, + "15": 0.1697530864197531, + "16": 0.035398230088495575, + "17": 0.030120481927710843, + "18": 0.02258064516129032, + "19": 0.006042296072507553, + "20": 0.0 + }, + "pass@2": { + "11": 0.6444444444444445, + "12": 0.599388379204893, + "13": 0.4506172839506173, + "14": 0.3026706231454006, + "15": 0.20679012345679013, + "16": 0.04129793510324484, + "17": 0.05120481927710843, + "18": 0.025806451612903226, + "19": 0.006042296072507553, + "20": 0.0 + }, + "pass@4": { + "11": 0.7428571428571429, + "12": 0.6850152905198776, + "13": 0.5246913580246914, + "14": 0.35311572700296734, + "15": 0.2623456790123457, + "16": 0.06784660766961652, + "17": 0.05421686746987952, + "18": 0.041935483870967745, + "19": 0.00906344410876133, + "20": 0.0 + }, + "pass@8": { + "11": 0.8539682539682539, + "12": 0.7584097859327217, + "13": 0.5925925925925926, + "14": 0.4065281899109792, + "15": 0.3117283950617284, + "16": 0.09144542772861357, + "17": 0.060240963855421686, + "18": 0.04838709677419355, + "19": 0.00906344410876133, + "20": 0.003003003003003003 + }, + "pass@16": { + "11": 0.9238095238095239, + "12": 0.8165137614678899, + "13": 0.6574074074074074, + "14": 0.47774480712166173, + "15": 0.36728395061728397, + "16": 0.11209439528023599, + "17": 0.06325301204819277, + "18": 0.06129032258064516, + "19": 0.012084592145015106, + "20": 0.006006006006006006 + }, + "pass@32": { + "11": 0.9492063492063492, + "12": 0.8501529051987767, + "13": 0.7006172839506173, + "14": 0.5489614243323442, + "15": 0.41975308641975306, + "16": 0.13569321533923304, + "17": 0.07530120481927711, + "18": 0.07419354838709677, + "19": 0.015105740181268883, + "20": 0.006006006006006006 + }, + "pass@64": { + "11": 0.9650793650793651, + "12": 0.8746177370030581, + "13": 0.7314814814814815, + "14": 0.6261127596439169, + "15": 0.5092592592592593, + "16": 0.168141592920354, + "17": 0.09036144578313253, + "18": 0.09032258064516129, + "19": 0.015105740181268883, + "20": 0.006006006006006006 + }, + "pass@128": { + "11": 0.9682539682539683, + "12": 0.8868501529051988, + "13": 0.7469135802469136, + "14": 0.6765578635014837, + "15": 0.5709876543209876, + "16": 0.1887905604719764, + "17": 0.12349397590361445, + "18": 0.0967741935483871, + "19": 0.01812688821752266, + "20": 0.006006006006006006 + } + }, + "total_pass_at_k": { + "pass@1": 0.1949877750611247, + "pass@2": 0.23074572127139364, + "pass@4": 0.2716992665036675, + "pass@8": 0.3108190709046455, + "pass@16": 0.3468826405867971, + "pass@32": 0.3746943765281174, + "pass@64": 0.40495110024449876, + "pass@128": 0.42573349633251834 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6234939759036144, + "pass@2": 0.6837349397590361, + "pass@4": 0.7831325301204819, + "pass@8": 0.8493975903614458, + "pass@16": 0.9156626506024096, + "pass@32": 0.9487951807228916, + "pass@64": 0.9608433734939759, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.4702194357366771, + "pass@2": 0.5579937304075235, + "pass@4": 0.6394984326018809, + "pass@8": 0.7210031347962382, + "pass@16": 0.7648902821316614, + "pass@32": 0.786833855799373, + "pass@64": 0.8275862068965517, + "pass@128": 0.8401253918495298 + }, + "13": { + "pass@1": 0.4061624649859944, + "pass@2": 0.49019607843137253, + "pass@4": 0.5546218487394958, + "pass@8": 0.6190476190476191, + "pass@16": 0.6946778711484594, + "pass@32": 0.742296918767507, + "pass@64": 0.773109243697479, + "pass@128": 0.803921568627451 + }, + "14": { + "pass@1": 0.2309941520467836, + "pass@2": 0.30994152046783624, + "pass@4": 0.38011695906432746, + "pass@8": 0.41812865497076024, + "pass@16": 0.48830409356725146, + "pass@32": 0.5467836257309941, + "pass@64": 0.6023391812865497, + "pass@128": 0.672514619883041 + }, + "15": { + "pass@1": 0.14242424242424243, + "pass@2": 0.2, + "pass@4": 0.2545454545454545, + "pass@8": 0.3090909090909091, + "pass@16": 0.3606060606060606, + "pass@32": 0.3939393939393939, + "pass@64": 0.4575757575757576, + "pass@128": 0.5121212121212121 + }, + "16": { + "pass@1": 0.045584045584045586, + "pass@2": 0.05128205128205128, + "pass@4": 0.05698005698005698, + "pass@8": 0.07407407407407407, + "pass@16": 0.1111111111111111, + "pass@32": 0.1339031339031339, + "pass@64": 0.15954415954415954, + "pass@128": 0.21367521367521367 + }, + "17": { + "pass@1": 0.009316770186335404, + "pass@2": 0.024844720496894408, + "pass@4": 0.027950310559006212, + "pass@8": 0.037267080745341616, + "pass@16": 0.055900621118012424, + "pass@32": 0.07763975155279502, + "pass@64": 0.09627329192546584, + "pass@128": 0.11490683229813664 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0058823529411764705, + "pass@4": 0.011764705882352941, + "pass@8": 0.014705882352941176, + "pass@16": 0.01764705882352941, + "pass@32": 0.03235294117647059, + "pass@64": 0.03823529411764706, + "pass@128": 0.052941176470588235 + }, + "19": { + "pass@1": 0.002976190476190476, + "pass@2": 0.002976190476190476, + "pass@4": 0.002976190476190476, + "pass@8": 0.002976190476190476, + "pass@16": 0.005952380952380952, + "pass@32": 0.005952380952380952, + "pass@64": 0.008928571428571428, + "pass@128": 0.008928571428571428 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0029940119760479044, + "pass@8": 0.005988023952095809, + "pass@16": 0.005988023952095809, + "pass@32": 0.011976047904191617, + "pass@64": 0.017964071856287425, + "pass@128": 0.017964071856287425 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6234939759036144, + "12": 0.4702194357366771, + "13": 0.4061624649859944, + "14": 0.2309941520467836, + "15": 0.14242424242424243, + "16": 0.045584045584045586, + "17": 0.009316770186335404, + "18": 0.0, + "19": 0.002976190476190476, + "20": 0.0 + }, + "pass@2": { + "11": 0.6837349397590361, + "12": 0.5579937304075235, + "13": 0.49019607843137253, + "14": 0.30994152046783624, + "15": 0.2, + "16": 0.05128205128205128, + "17": 0.024844720496894408, + "18": 0.0058823529411764705, + "19": 0.002976190476190476, + "20": 0.0 + }, + "pass@4": { + "11": 0.7831325301204819, + "12": 0.6394984326018809, + "13": 0.5546218487394958, + "14": 0.38011695906432746, + "15": 0.2545454545454545, + "16": 0.05698005698005698, + "17": 0.027950310559006212, + "18": 0.011764705882352941, + "19": 0.002976190476190476, + "20": 0.0029940119760479044 + }, + "pass@8": { + "11": 0.8493975903614458, + "12": 0.7210031347962382, + "13": 0.6190476190476191, + "14": 0.41812865497076024, + "15": 0.3090909090909091, + "16": 0.07407407407407407, + "17": 0.037267080745341616, + "18": 0.014705882352941176, + "19": 0.002976190476190476, + "20": 0.005988023952095809 + }, + "pass@16": { + "11": 0.9156626506024096, + "12": 0.7648902821316614, + "13": 0.6946778711484594, + "14": 0.48830409356725146, + "15": 0.3606060606060606, + "16": 0.1111111111111111, + "17": 0.055900621118012424, + "18": 0.01764705882352941, + "19": 0.005952380952380952, + "20": 0.005988023952095809 + }, + "pass@32": { + "11": 0.9487951807228916, + "12": 0.786833855799373, + "13": 0.742296918767507, + "14": 0.5467836257309941, + "15": 0.3939393939393939, + "16": 0.1339031339031339, + "17": 0.07763975155279502, + "18": 0.03235294117647059, + "19": 0.005952380952380952, + "20": 0.011976047904191617 + }, + "pass@64": { + "11": 0.9608433734939759, + "12": 0.8275862068965517, + "13": 0.773109243697479, + "14": 0.6023391812865497, + "15": 0.4575757575757576, + "16": 0.15954415954415954, + "17": 0.09627329192546584, + "18": 0.03823529411764706, + "19": 0.008928571428571428, + "20": 0.017964071856287425 + }, + "pass@128": { + "11": 0.9698795180722891, + "12": 0.8401253918495298, + "13": 0.803921568627451, + "14": 0.672514619883041, + "15": 0.5121212121212121, + "16": 0.21367521367521367, + "17": 0.11490683229813664, + "18": 0.052941176470588235, + "19": 0.008928571428571428, + "20": 0.017964071856287425 + } + }, + "total_pass_at_k": { + "pass@1": 0.1926851025869759, + "pass@2": 0.23223312518584596, + "pass@4": 0.27088908712459114, + "pass@8": 0.3044900386559619, + "pass@16": 0.3416592328278323, + "pass@32": 0.367826345524829, + "pass@64": 0.39399345822182574, + "pass@128": 0.4207552780255724 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.587, + "pass@2": 0.652, + "pass@4": 0.746, + "pass@8": 0.826, + "pass@16": 0.91, + "pass@32": 0.936, + "pass@64": 0.952, + "pass@128": 0.96 + }, + "12": { + "pass@1": 0.518, + "pass@2": 0.591, + "pass@4": 0.668, + "pass@8": 0.738, + "pass@16": 0.791, + "pass@32": 0.819, + "pass@64": 0.847, + "pass@128": 0.862 + }, + "13": { + "pass@1": 0.389, + "pass@2": 0.473, + "pass@4": 0.55, + "pass@8": 0.616, + "pass@16": 0.681, + "pass@32": 0.723, + "pass@64": 0.755, + "pass@128": 0.783 + }, + "14": { + "pass@1": 0.248, + "pass@2": 0.314, + "pass@4": 0.37, + "pass@8": 0.42, + "pass@16": 0.491, + "pass@32": 0.556, + "pass@64": 0.621, + "pass@128": 0.671 + }, + "15": { + "pass@1": 0.157, + "pass@2": 0.207, + "pass@4": 0.257, + "pass@8": 0.309, + "pass@16": 0.353, + "pass@32": 0.406, + "pass@64": 0.48, + "pass@128": 0.536 + }, + "16": { + "pass@1": 0.035, + "pass@2": 0.046, + "pass@4": 0.061, + "pass@8": 0.083, + "pass@16": 0.109, + "pass@32": 0.132, + "pass@64": 0.162, + "pass@128": 0.195 + }, + "17": { + "pass@1": 0.021, + "pass@2": 0.035, + "pass@4": 0.042, + "pass@8": 0.05, + "pass@16": 0.064, + "pass@32": 0.078, + "pass@64": 0.095, + "pass@128": 0.116 + }, + "18": { + "pass@1": 0.007, + "pass@2": 0.01, + "pass@4": 0.018, + "pass@8": 0.021, + "pass@16": 0.028, + "pass@32": 0.04, + "pass@64": 0.051, + "pass@128": 0.062 + }, + "19": { + "pass@1": 0.003, + "pass@2": 0.003, + "pass@4": 0.005, + "pass@8": 0.005, + "pass@16": 0.007, + "pass@32": 0.008, + "pass@64": 0.009, + "pass@128": 0.011 + }, + "20": { + "pass@1": 0.001, + "pass@2": 0.002, + "pass@4": 0.003, + "pass@8": 0.005, + "pass@16": 0.006, + "pass@32": 0.008, + "pass@64": 0.01, + "pass@128": 0.014 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.587, + "12": 0.518, + "13": 0.389, + "14": 0.248, + "15": 0.157, + "16": 0.035, + "17": 0.021, + "18": 0.007, + "19": 0.003, + "20": 0.001 + }, + "pass@2": { + "11": 0.652, + "12": 0.591, + "13": 0.473, + "14": 0.314, + "15": 0.207, + "16": 0.046, + "17": 0.035, + "18": 0.01, + "19": 0.003, + "20": 0.002 + }, + "pass@4": { + "11": 0.746, + "12": 0.668, + "13": 0.55, + "14": 0.37, + "15": 0.257, + "16": 0.061, + "17": 0.042, + "18": 0.018, + "19": 0.005, + "20": 0.003 + }, + "pass@8": { + "11": 0.826, + "12": 0.738, + "13": 0.616, + "14": 0.42, + "15": 0.309, + "16": 0.083, + "17": 0.05, + "18": 0.021, + "19": 0.005, + "20": 0.005 + }, + "pass@16": { + "11": 0.91, + "12": 0.791, + "13": 0.681, + "14": 0.491, + "15": 0.353, + "16": 0.109, + "17": 0.064, + "18": 0.028, + "19": 0.007, + "20": 0.006 + }, + "pass@32": { + "11": 0.936, + "12": 0.819, + "13": 0.723, + "14": 0.556, + "15": 0.406, + "16": 0.132, + "17": 0.078, + "18": 0.04, + "19": 0.008, + "20": 0.008 + }, + "pass@64": { + "11": 0.952, + "12": 0.847, + "13": 0.755, + "14": 0.621, + "15": 0.48, + "16": 0.162, + "17": 0.095, + "18": 0.051, + "19": 0.009, + "20": 0.01 + }, + "pass@128": { + "11": 0.96, + "12": 0.862, + "13": 0.783, + "14": 0.671, + "15": 0.536, + "16": 0.195, + "17": 0.116, + "18": 0.062, + "19": 0.011, + "20": 0.014 + } + }, + "total_pass_at_k": { + "pass@1": 0.1966, + "pass@2": 0.2333, + "pass@4": 0.272, + "pass@8": 0.3073, + "pass@16": 0.344, + "pass@32": 0.3706, + "pass@64": 0.3982, + "pass@128": 0.421 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block16-epoch4-op7-10/summary.csv b/composition/SDAR-100M-block16-epoch4-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..d644cddb939655f29848942311ea1fa7de64eb82 --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch4-op7-10,0.9928402777777777,0.4674421875,0.7163149671052632,0.0,0.0,0.0,200.1375373263889,332.59690703125,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block16-epoch4-op7-10/summary.json b/composition/SDAR-100M-block16-epoch4-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..96c13ac245b4a8519bb17faa1de18c7e7ed17f3e --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch4-op7-10", + "id_acc": 0.9928402777777777, + "ood_acc": 0.4674421875, + "total_acc": 0.7163149671052632, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.1375373263889, + "ood_avg_resp_len": 332.59690703125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 576926, + "answer_accuracy": 0.7124935780904205, + "avg_response_len": 257.13824642349033, + "resp_tokens_sum": 208212038, + "pass_at_k": { + "pass@1": 0.7124935780904205, + "pass@2": 0.7482208665462601, + "pass@4": 0.7782977803705171, + "pass@8": 0.8033142972254913, + "pass@16": 0.8239431045015444, + "pass@32": 0.841313857869306, + "pass@64": 0.8566171919665101, + "pass@128": 0.8706923806512804 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9639564896755162, + "pass@2": 0.9751933663159359, + "pass@4": 0.9805886808396739, + "pass@8": 0.9832729426842688, + "pass@16": 0.9853094435850058, + "pass@32": 0.9875159779599025, + "pass@64": 0.9903780082743677, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.997314453125, + "pass@2": 0.9981799028051181, + "pass@4": 0.9989848275801462, + "pass@8": 0.9996841121782726, + "pass@16": 0.9999734491629393, + "pass@32": 0.9999999062712751, + "pass@64": 0.9999999999999887, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9964285714285714, + "pass@2": 0.9987411729783777, + "pass@4": 0.9997791192767571, + "pass@8": 0.9999910052051663, + "pass@16": 0.999999987524699, + "pass@32": 0.999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970761601796407, + "pass@2": 0.9973235124239709, + "pass@4": 0.9976157328200742, + "pass@8": 0.9981157895218378, + "pass@16": 0.998852517052828, + "pass@32": 0.9996224712529058, + "pass@64": 0.9999803235855924, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972725591715976, + "pass@2": 0.9980547919675722, + "pass@4": 0.9987735604794961, + "pass@8": 0.9995079830474547, + "pass@16": 0.999928825622786, + "pass@32": 0.9999991140832827, + "pass@64": 0.9999999999952738, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9991486378205128, + "pass@2": 0.9999641158136482, + "pass@4": 0.999999888836011, + "pass@8": 0.9999999999996302, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9955325704225352, + "pass@2": 0.9969474326272597, + "pass@4": 0.9973978846834287, + "pass@8": 0.9976821039095084, + "pass@16": 0.998119293157022, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9345122167138811, + "pass@2": 0.9542368143695208, + "pass@4": 0.9657462063877991, + "pass@8": 0.9749512748470323, + "pass@16": 0.9827357531203197, + "pass@32": 0.9884344587547554, + "pass@64": 0.9925691216469926, + "pass@128": 0.9971671388101983 + }, + "12": { + "pass@1": 0.8454493290960452, + "pass@2": 0.8801169285777846, + "pass@4": 0.8989873583069065, + "pass@8": 0.9130351649779792, + "pass@16": 0.9250958046726186, + "pass@32": 0.9364960935539219, + "pass@64": 0.9474650567648585, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.74047315830721, + "pass@2": 0.8119554217164869, + "pass@4": 0.8599147251969675, + "pass@8": 0.8945134207641785, + "pass@16": 0.9205661933194598, + "pass@32": 0.9404403427759488, + "pass@64": 0.9559823521026871, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.6125632788161994, + "pass@2": 0.7013274370201388, + "pass@4": 0.7658476230190846, + "pass@8": 0.8097702298192307, + "pass@16": 0.8407102562368179, + "pass@32": 0.8644964958281782, + "pass@64": 0.8838021812976606, + "pass@128": 0.9003115264797508 + }, + "15": { + "pass@1": 0.48457821531791906, + "pass@2": 0.5827305027081152, + "pass@4": 0.6590647869702699, + "pass@8": 0.7158310076759379, + "pass@16": 0.7589664128271321, + "pass@32": 0.7930825505983045, + "pass@64": 0.8204516163076292, + "pass@128": 0.8410404624277457 + }, + "16": { + "pass@1": 0.2755040322580645, + "pass@2": 0.3586213328676657, + "pass@4": 0.43517604109970115, + "pass@8": 0.5017510566263869, + "pass@16": 0.5583572473734497, + "pass@32": 0.6078831915021241, + "pass@64": 0.6517453480905581, + "pass@128": 0.6903225806451613 + }, + "17": { + "pass@1": 0.21780166184971098, + "pass@2": 0.29274348857584986, + "pass@4": 0.36758594012453627, + "pass@8": 0.4327260544001389, + "pass@16": 0.4852625887317101, + "pass@32": 0.5308733536032256, + "pass@64": 0.5731955827266514, + "pass@128": 0.615606936416185 + }, + "18": { + "pass@1": 0.18082589285714284, + "pass@2": 0.24475007030371193, + "pass@4": 0.30927755396646855, + "pass@8": 0.3685482161042458, + "pass@16": 0.4204588630253465, + "pass@32": 0.46585847061159913, + "pass@64": 0.5068792519157992, + "pass@128": 0.5457142857142857 + }, + "19": { + "pass@1": 0.1753472222222222, + "pass@2": 0.23986560379276908, + "pass@4": 0.3042127763308865, + "pass@8": 0.3648899684541569, + "pass@16": 0.41793983675736257, + "pass@32": 0.46237397780199607, + "pass@64": 0.5029501083188042, + "pass@128": 0.5405405405405406 + }, + "20": { + "pass@1": 0.15636730480480482, + "pass@2": 0.2165424528859568, + "pass@4": 0.27821547475484476, + "pass@8": 0.3365076147164066, + "pass@16": 0.3888313889531125, + "pass@32": 0.4337362063462613, + "pass@64": 0.4738117632886543, + "pass@128": 0.5135135135135135 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 589293, + "answer_accuracy": 0.71766976812159, + "avg_response_len": 266.5054985872954, + "resp_tokens_sum": 218832995, + "pass_at_k": { + "pass@1": 0.71766976812159, + "pass@2": 0.7541205482352454, + "pass@4": 0.7849935205585589, + "pass@8": 0.8106964627292033, + "pass@16": 0.831317315567593, + "pass@32": 0.8479933921096046, + "pass@64": 0.8623722337450352, + "pass@128": 0.875448168355417 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9412384969325154, + "pass@2": 0.9570337030819772, + "pass@4": 0.9684386097635029, + "pass@8": 0.9755421990555402, + "pass@16": 0.9790090135040668, + "pass@32": 0.9808182150392867, + "pass@64": 0.9815547734367644, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999075443786982, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226678, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9963768115942029, + "pass@2": 0.9971339010612804, + "pass@4": 0.9974446123582379, + "pass@8": 0.9977680501821345, + "pass@16": 0.9983124881128967, + "pass@32": 0.999097406302473, + "pass@64": 0.9998273650576287, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997313753581661, + "pass@2": 0.9999841363400491, + "pass@4": 0.9999999435958757, + "pass@8": 0.9999999999999097, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997674313583815, + "pass@2": 0.998275069978608, + "pass@4": 0.9989801556597332, + "pass@8": 0.9996539139236084, + "pass@16": 0.9999649956979594, + "pass@32": 0.9999998098093686, + "pass@64": 0.9999999999999141, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957306338028169, + "pass@2": 0.9968081124542529, + "pass@4": 0.9971466248937193, + "pass@8": 0.9971827493473799, + "pass@16": 0.9971830985789736, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.993310254491018, + "pass@2": 0.9940558106228489, + "pass@4": 0.994196379457059, + "pass@8": 0.9943759135003243, + "pass@16": 0.9947162761091989, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9478068524096386, + "pass@2": 0.9611561403092685, + "pass@4": 0.9686419691702693, + "pass@8": 0.9750669382908692, + "pass@16": 0.9806081356024587, + "pass@32": 0.9842428892059527, + "pass@64": 0.9862549466314855, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8410315438871473, + "pass@2": 0.8755083244884359, + "pass@4": 0.8937380000305601, + "pass@8": 0.905965360565369, + "pass@16": 0.9151980908995769, + "pass@32": 0.9235809759059825, + "pass@64": 0.9339121914379174, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.7262123599439776, + "pass@2": 0.8062757091025381, + "pass@4": 0.8566690912585506, + "pass@8": 0.8882637919932588, + "pass@16": 0.9082264797954909, + "pass@32": 0.9222958616323473, + "pass@64": 0.9343518871725501, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6329952485380117, + "pass@2": 0.7221707792282543, + "pass@4": 0.7828848704767167, + "pass@8": 0.8258118272832482, + "pass@16": 0.8579023706600551, + "pass@32": 0.8832258032244631, + "pass@64": 0.9049950902372479, + "pass@128": 0.9269005847953217 + }, + "15": { + "pass@1": 0.4446022727272727, + "pass@2": 0.5437966028394173, + "pass@4": 0.6285262524002679, + "pass@8": 0.6960194300230498, + "pass@16": 0.7475932370874276, + "pass@32": 0.785985388651888, + "pass@64": 0.8147437752003887, + "pass@128": 0.8424242424242424 + }, + "16": { + "pass@1": 0.3059116809116809, + "pass@2": 0.39158751990936996, + "pass@4": 0.4727661555660242, + "pass@8": 0.5448737019577484, + "pass@16": 0.6040431432838912, + "pass@32": 0.6528764342680178, + "pass@64": 0.694521859735016, + "pass@128": 0.7293447293447294 + }, + "17": { + "pass@1": 0.26052989130434784, + "pass@2": 0.34320820291485293, + "pass@4": 0.42293633977041684, + "pass@8": 0.4927356348756277, + "pass@16": 0.5482837994495754, + "pass@32": 0.5921106992309274, + "pass@64": 0.6290502589670679, + "pass@128": 0.6614906832298136 + }, + "18": { + "pass@1": 0.2153262867647059, + "pass@2": 0.28677049559981455, + "pass@4": 0.35674156300315396, + "pass@8": 0.4197868754911729, + "pass@16": 0.47482275196634255, + "pass@32": 0.5236327311772606, + "pass@64": 0.5672130434247589, + "pass@128": 0.6029411764705882 + }, + "19": { + "pass@1": 0.16901506696428573, + "pass@2": 0.22576206294525675, + "pass@4": 0.28495160454273577, + "pass@8": 0.34010383127747634, + "pass@16": 0.38708927582856617, + "pass@32": 0.4276740630065036, + "pass@64": 0.4649332236307543, + "pass@128": 0.5 + }, + "20": { + "pass@1": 0.1504491017964072, + "pass@2": 0.2108255192371163, + "pass@4": 0.2735428047167756, + "pass@8": 0.3338204070406974, + "pass@16": 0.38676384570770866, + "pass@32": 0.4292259673414519, + "pass@64": 0.4663031080186876, + "pass@128": 0.5 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 575859, + "answer_accuracy": 0.7187886942802365, + "avg_response_len": 286.13477966727913, + "resp_tokens_sum": 229237451, + "pass_at_k": { + "pass@1": 0.7187886942802365, + "pass@2": 0.7548033784106732, + "pass@4": 0.7848112376220922, + "pass@8": 0.8098416329714997, + "pass@16": 0.8305795131153559, + "pass@32": 0.8481136410654446, + "pass@64": 0.8635051234293231, + "pass@128": 0.8771369228311232 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9569496268656716, + "pass@2": 0.9678403014455282, + "pass@4": 0.976078659010907, + "pass@8": 0.9827045652363846, + "pass@16": 0.9880512471221918, + "pass@32": 0.9920857219167919, + "pass@64": 0.9938974306133376, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997487207602339, + "pass@2": 0.9999802142330892, + "pass@4": 0.9999999095507799, + "pass@8": 0.9999999999996626, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9922564338235295, + "pass@2": 0.994646320634553, + "pass@4": 0.9961220976973466, + "pass@8": 0.9970632553671733, + "pass@16": 0.997715072688465, + "pass@32": 0.9983497369764539, + "pass@64": 0.9992704955997272, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992360015772871, + "pass@2": 0.9998886118631859, + "pass@4": 0.9999968563669289, + "pass@8": 0.9999999983772165, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998318829113924, + "pass@2": 0.9996340202332306, + "pass@4": 0.9999719820425929, + "pass@8": 0.9999998296794577, + "pass@16": 0.999999999997869, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995777027027027, + "pass@2": 0.9988557701402191, + "pass@4": 0.9998988651193375, + "pass@8": 0.9999992668226257, + "pass@16": 0.9999999999808709, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9933430466237942, + "pass@2": 0.9955922285236853, + "pass@4": 0.996922894083577, + "pass@8": 0.99825698853535, + "pass@16": 0.9992957181536803, + "pass@32": 0.9998834594758289, + "pass@64": 0.9999990174474539, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9458333333333333, + "pass@2": 0.9680735220597424, + "pass@4": 0.9784683795477946, + "pass@8": 0.9834253742459571, + "pass@16": 0.9859158447021265, + "pass@32": 0.9876978205871854, + "pass@64": 0.9894997112731746, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8640577217125383, + "pass@2": 0.9053041248284324, + "pass@4": 0.9285084223990344, + "pass@8": 0.9444538698747363, + "pass@16": 0.9565054855884125, + "pass@32": 0.9649179472227764, + "pass@64": 0.971280941814739, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.7043306327160493, + "pass@2": 0.7786207713619127, + "pass@4": 0.8288944483791375, + "pass@8": 0.86676143621755, + "pass@16": 0.8943791026870926, + "pass@32": 0.9153670080570203, + "pass@64": 0.9331493513324162, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.6179293397626113, + "pass@2": 0.703675538563985, + "pass@4": 0.7640525688183634, + "pass@8": 0.8070382454357383, + "pass@16": 0.8387920432900056, + "pass@32": 0.8638308703360127, + "pass@64": 0.8855514513931839, + "pass@128": 0.9050445103857567 + }, + "15": { + "pass@1": 0.46923225308641975, + "pass@2": 0.5619528136239913, + "pass@4": 0.6342046055007012, + "pass@8": 0.6883368569053809, + "pass@16": 0.7319691872044063, + "pass@32": 0.7717383879708698, + "pass@64": 0.8095051745441575, + "pass@128": 0.8425925925925926 + }, + "16": { + "pass@1": 0.31752396755162243, + "pass@2": 0.3971402254198313, + "pass@4": 0.46753584065266157, + "pass@8": 0.5274963190047928, + "pass@16": 0.578633799879253, + "pass@32": 0.6236236229841198, + "pass@64": 0.6655990132285015, + "pass@128": 0.7079646017699115 + }, + "17": { + "pass@1": 0.2409403237951807, + "pass@2": 0.3118437084953988, + "pass@4": 0.38310006194632296, + "pass@8": 0.4517112008785566, + "pass@16": 0.5143629770086949, + "pass@32": 0.5692526943598468, + "pass@64": 0.6171244785989948, + "pass@128": 0.6566265060240963 + }, + "18": { + "pass@1": 0.21854838709677418, + "pass@2": 0.2914830454660909, + "pass@4": 0.36253027202244875, + "pass@8": 0.4232167175836581, + "pass@16": 0.4722504664411351, + "pass@32": 0.5141885209927322, + "pass@64": 0.5525587893884821, + "pass@128": 0.5870967741935483 + }, + "19": { + "pass@1": 0.1644873489425982, + "pass@2": 0.22967898815329338, + "pass@4": 0.296141610225459, + "pass@8": 0.35542356027993677, + "pass@16": 0.40438333117475245, + "pass@32": 0.44624875815883597, + "pass@64": 0.4841766947910102, + "pass@128": 0.5196374622356495 + }, + "20": { + "pass@1": 0.16366366366366367, + "pass@2": 0.22465292556335845, + "pass@4": 0.28830474653506144, + "pass@8": 0.3515397113540908, + "pass@16": 0.4098242301095972, + "pass@32": 0.4585545918829372, + "pass@64": 0.49711705635141545, + "pass@128": 0.5285285285285285 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_metrics.json b/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0c97cf28b5bceeeaf3cfe8aab18778f2e6941e36 --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block16-epoch4-op9-12", + "id": { + "answer_accuracy": 0.9921467013888889, + "avg_response_len": 199.46203125, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9921467013888889, + "pass@2": 0.9944876284995613, + "pass@4": 0.9959433585697617, + "pass@8": 0.9968784496129127, + "pass@16": 0.9976004460052793, + "pass@32": 0.9981871074093316, + "pass@64": 0.9986349505024621, + "pass@128": 0.999 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9524140625, + "pass@2": 0.9645937500000002, + "pass@4": 0.9730021943194608, + "pass@8": 0.9786367820253195, + "pass@16": 0.9831046505061921, + "pass@32": 0.9868706993361868, + "pass@64": 0.9899770280239669, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9987109375, + "pass@2": 0.999171998031496, + "pass@4": 0.9994346713535808, + "pass@8": 0.9996911656406087, + "pass@16": 0.9999126144227602, + "pass@32": 0.9999949322702752, + "pass@64": 0.9999999977569969, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.993890625, + "pass@2": 0.996357775590551, + "pass@4": 0.9976947060367454, + "pass@8": 0.9983995073818661, + "pass@16": 0.9989712933683291, + "pass@32": 0.9994882787743641, + "pass@64": 0.9998748568874752, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.997625, + "pass@2": 0.9985620078740157, + "pass@4": 0.9990585259655044, + "pass@8": 0.9993178674757099, + "pass@16": 0.9995588461117559, + "pass@32": 0.9998290717621782, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997484375, + "pass@2": 0.9984462352362204, + "pass@4": 0.9990977215973003, + "pass@8": 0.9995759792654791, + "pass@16": 0.9998778554116803, + "pass@32": 0.9999900984417422, + "pass@64": 0.999999987930507, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995140625, + "pass@2": 0.9973559301181102, + "pass@4": 0.9985708786089239, + "pass@8": 0.9991065163275745, + "pass@16": 0.9993309358545857, + "pass@32": 0.9995814582857677, + "pass@64": 0.9998779527559055, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9940546875, + "pass@2": 0.9959009596456694, + "pass@4": 0.9966315292463443, + "pass@8": 0.99717822839966, + "pass@16": 0.9976478183722098, + "pass@32": 0.9979294278134688, + "pass@64": 0.9979985548661734, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.9524140625, + "2": 1.0, + "3": 1.0, + "4": 0.9987109375, + "5": 0.993890625, + "6": 0.997625, + "7": 0.997484375, + "8": 0.995140625, + "9": 0.9940546875 + }, + "per_op_avg_response_len": { + "10": 279.8089140625, + "2": 115.4520703125, + "3": 144.40334375, + "4": 159.6451328125, + "5": 172.749, + "6": 198.4754375, + "7": 223.35846875, + "8": 242.756265625, + "9": 258.5096484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.8089140625, + "2": 115.4520703125, + "3": 144.40334375, + "4": 159.6451328125, + "5": 172.749, + "6": 198.4754375, + "7": 223.35846875, + "8": 242.756265625, + "9": 258.5096484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376348, + "answer_accuracy": 0.9929816784869976, + "avg_response_len": 189.93894587977036, + "resp_tokens_sum": 71988380, + "pass_at_k": { + "pass@1": 0.9929816784869976, + "pass@2": 0.9952404553154262, + "pass@4": 0.996518110166574, + "pass@8": 0.9972523121377939, + "pass@16": 0.9978434752201003, + "pass@32": 0.9984079378454725, + "pass@64": 0.9988835021935919, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9574575958702065, + "pass@2": 0.9697600196269713, + "pass@4": 0.9770701256369501, + "pass@8": 0.9807352442070735, + "pass@16": 0.9835831385400656, + "pass@32": 0.9868139186902647, + "pass@64": 0.9902929813385202, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996923828125, + "pass@2": 0.9976178026574803, + "pass@4": 0.9982453347628422, + "pass@8": 0.9990349229744625, + "pass@16": 0.9997269200711651, + "pass@32": 0.9999841633446103, + "pass@64": 0.9999999929906153, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9954861111111111, + "pass@2": 0.9981666354205724, + "pass@4": 0.9995852298224628, + "pass@8": 0.9999710725449563, + "pass@16": 0.9999998592866416, + "pass@32": 0.9999999999993872, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9968890344311377, + "pass@2": 0.997264575180348, + "pass@4": 0.997534851556729, + "pass@8": 0.9979835642250304, + "pass@16": 0.9986793120796699, + "pass@32": 0.9994882388094946, + "pass@64": 0.9999586116800393, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9973187869822485, + "pass@2": 0.9984067785258351, + "pass@4": 0.9992237471795316, + "pass@8": 0.9998073811974663, + "pass@16": 0.9999900064095892, + "pass@32": 0.9999999886267649, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9983974358974359, + "pass@2": 0.999743290051484, + "pass@4": 0.9999842372467864, + "pass@8": 0.9999999472370541, + "pass@16": 0.9999999999998607, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9972687007874016, + "pass@4": 0.997876473539399, + "pass@8": 0.9984546377928358, + "pass@16": 0.999188066655105, + "pass@32": 0.9998084093975909, + "pass@64": 0.9999959303192606, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387303, + "answer_accuracy": 0.991417001146789, + "avg_response_len": 197.48486648099606, + "resp_tokens_sum": 77148648, + "pass_at_k": { + "pass@1": 0.991417001146789, + "pass@2": 0.993621147485578, + "pass@4": 0.9950336770751789, + "pass@8": 0.9959590200409408, + "pass@16": 0.9966420865580624, + "pass@32": 0.9971659138581003, + "pass@64": 0.9976167577265961, + "pass@128": 0.9980340760157274 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9421731211656442, + "pass@2": 0.9570635174629243, + "pass@4": 0.9675180118397778, + "pass@8": 0.9744761195433342, + "pass@16": 0.9791587744555608, + "pass@32": 0.9822004591222739, + "pass@64": 0.984571936009979, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956295289855073, + "pass@2": 0.9968182985279016, + "pass@4": 0.9973139007080636, + "pass@8": 0.9976146824957167, + "pass@16": 0.998064779875966, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9972242120343839, + "pass@2": 0.9987002374613632, + "pass@4": 0.9996714137452016, + "pass@8": 0.9999752561101265, + "pass@16": 0.9999998735735035, + "pass@32": 0.999999999999447, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9973807803468208, + "pass@2": 0.9977498748350099, + "pass@4": 0.9982556172530457, + "pass@8": 0.9989655495774096, + "pass@16": 0.9996567447729243, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954665492957746, + "pass@2": 0.9968725740268383, + "pass@4": 0.9974011850631346, + "pass@8": 0.9976822657434412, + "pass@16": 0.9981192931743675, + "pass@32": 0.9988210100498002, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9930295658682635, + "pass@2": 0.9939206233202886, + "pass@4": 0.9940102609741148, + "pass@8": 0.9940119756217468, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379302, + "answer_accuracy": 0.9920645714763977, + "avg_response_len": 210.92241379310346, + "resp_tokens_sum": 80643232, + "pass_at_k": { + "pass@1": 0.9920645714763977, + "pass@2": 0.9946266910549394, + "pass@4": 0.9963030868735149, + "pass@8": 0.9974472785775886, + "pass@16": 0.9983387464832911, + "pass@32": 0.9990116148739929, + "pass@64": 0.999428911933499, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9572761194029851, + "pass@2": 0.9666937213538606, + "pass@4": 0.9742225369216908, + "pass@8": 0.9805621470686411, + "pass@16": 0.9864603166226772, + "pass@32": 0.991472929033808, + "pass@64": 0.9949171827192708, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.9998089774140073, + "pass@4": 0.9999887842966998, + "pass@8": 0.9999999716046218, + "pass@16": 0.9999999999999635, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906479779411764, + "pass@2": 0.9942146248263086, + "pass@4": 0.996329596759964, + "pass@8": 0.9977398649093597, + "pass@16": 0.9989382018700852, + "pass@32": 0.9997259417849353, + "pass@64": 0.9999908945046169, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9988416798107256, + "pass@2": 0.9997768356143968, + "pass@4": 0.9999891550425282, + "pass@8": 0.9999999767890079, + "pass@16": 0.9999999999999797, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997774920886076, + "pass@2": 0.9992509094986542, + "pass@4": 0.9998849730413445, + "pass@8": 0.9999968489460502, + "pass@16": 0.9999999985879974, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9917417417417418, + "pass@2": 0.9956344139415007, + "pass@4": 0.998493621292834, + "pass@8": 0.999787773125201, + "pass@16": 0.9999957560892757, + "pass@32": 0.9999999991534195, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932174437299035, + "pass@2": 0.9964665037850975, + "pass@4": 0.9980255754807175, + "pass@8": 0.9991216467059164, + "pass@16": 0.9997943881339146, + "pass@32": 0.9999917764544182, + "pass@64": 0.9999999987229452, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.5692328125, + "avg_response_len": 350.66635078125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5692328125, + "pass@2": 0.638573991141724, + "pass@4": 0.6956425176509109, + "pass@8": 0.7415618754091828, + "pass@16": 0.7774673995948668, + "pass@32": 0.805792751873983, + "pass@64": 0.8294758529319611, + "pass@128": 0.8504 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9613984375, + "pass@2": 0.9735031988188977, + "pass@4": 0.9804231168916386, + "pass@8": 0.9847109635922205, + "pass@16": 0.9870854668143153, + "pass@32": 0.9886758259692309, + "pass@64": 0.9904900450431707, + "pass@128": 0.993 + }, + "12": { + "pass@1": 0.899875, + "pass@2": 0.9206909448818899, + "pass@4": 0.9333334387889014, + "pass@8": 0.9429585488390639, + "pass@16": 0.9508381683007954, + "pass@32": 0.957124237853498, + "pass@64": 0.9620219505951612, + "pass@128": 0.966 + }, + "13": { + "pass@1": 0.8238984375, + "pass@2": 0.8724411909448823, + "pass@4": 0.9021637046306712, + "pass@8": 0.9231090688142323, + "pass@16": 0.9379672291934675, + "pass@32": 0.9484869565082493, + "pass@64": 0.9568897147822485, + "pass@128": 0.963 + }, + "14": { + "pass@1": 0.7794453125, + "pass@2": 0.8445839074803148, + "pass@4": 0.8855294000749906, + "pass@8": 0.9114116915660494, + "pass@16": 0.9273511399284904, + "pass@32": 0.9386215902003806, + "pass@64": 0.9495939611176091, + "pass@128": 0.962 + }, + "15": { + "pass@1": 0.6651171875, + "pass@2": 0.7486331200787407, + "pass@4": 0.806329887701538, + "pass@8": 0.8473808946123879, + "pass@16": 0.8765889720497365, + "pass@32": 0.8979515241106913, + "pass@64": 0.9145665425705841, + "pass@128": 0.928 + }, + "16": { + "pass@1": 0.4879609375, + "pass@2": 0.5945366633858277, + "pass@4": 0.6776242561867271, + "pass@8": 0.739991947562845, + "pass@16": 0.7879800006920792, + "pass@32": 0.8266620166339039, + "pass@64": 0.8592523951014696, + "pass@128": 0.89 + }, + "17": { + "pass@1": 0.369609375, + "pass@2": 0.47383956692913415, + "pass@4": 0.566708111173604, + "pass@8": 0.6421709788558302, + "pass@16": 0.699630230918739, + "pass@32": 0.7440501332532892, + "pass@64": 0.7798027890890089, + "pass@128": 0.811 + }, + "18": { + "pass@1": 0.3044921875, + "pass@2": 0.402151697834646, + "pass@4": 0.49101950562429714, + "pass@8": 0.5642945200089501, + "pass@16": 0.6220010627856662, + "pass@32": 0.6671294229443815, + "pass@64": 0.7039513297789204, + "pass@128": 0.734 + }, + "19": { + "pass@1": 0.20428125, + "pass@2": 0.28356422244094465, + "pass@4": 0.363903641076115, + "pass@8": 0.4365091889268906, + "pass@16": 0.4967211171125234, + "pass@32": 0.5470671625654399, + "pass@64": 0.5925308099061153, + "pass@128": 0.635 + }, + "20": { + "pass@1": 0.19625, + "pass@2": 0.27179539862204705, + "pass@4": 0.3493901143607049, + "pass@8": 0.4230809513134006, + "pass@16": 0.4885106081527199, + "pass@32": 0.5421586487007338, + "pass@64": 0.5856589913352467, + "pass@128": 0.622 + } + }, + "per_op_accuracy": { + "11": 0.9613984375, + "12": 0.899875, + "13": 0.8238984375, + "14": 0.7794453125, + "15": 0.6651171875, + "16": 0.4879609375, + "17": 0.369609375, + "18": 0.3044921875, + "19": 0.20428125, + "20": 0.19625 + }, + "per_op_avg_response_len": { + "11": 297.492234375, + "12": 307.809671875, + "13": 326.856828125, + "14": 351.034875, + "15": 362.367296875, + "16": 361.9160234375, + "17": 375.5798671875, + "18": 376.86175, + "19": 367.4975859375, + "20": 379.247375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.492234375, + "12": 307.809671875, + "13": 326.856828125, + "14": 351.034875, + "15": 362.367296875, + "16": 361.9160234375, + "17": 375.5798671875, + "18": 376.86175, + "19": 367.4975859375, + "20": 379.247375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 245625, + "answer_accuracy": 0.5702660661218425, + "avg_response_len": 332.6203403603269, + "resp_tokens_sum": 143266233, + "pass_at_k": { + "pass@1": 0.5702660661218425, + "pass@2": 0.6388949541364937, + "pass@4": 0.6949541250137222, + "pass@8": 0.7397971193299135, + "pass@16": 0.7748348215257485, + "pass@32": 0.8027802603018277, + "pass@64": 0.8269381943859584, + "pass@128": 0.8484398216939079 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9572193696883853, + "pass@2": 0.9719146628449066, + "pass@4": 0.981724504514839, + "pass@8": 0.9890359200866053, + "pass@16": 0.9932697779011666, + "pass@32": 0.9957361168730702, + "pass@64": 0.9978415915007174, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.8983271539548022, + "pass@2": 0.920239098158281, + "pass@4": 0.9334048053315367, + "pass@8": 0.9434168617466765, + "pass@16": 0.9517176538857137, + "pass@32": 0.9580437536392892, + "pass@64": 0.961533417356335, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8390722962382445, + "pass@2": 0.8873212765038385, + "pass@4": 0.9195430402704361, + "pass@8": 0.9430086769866958, + "pass@16": 0.9574399410309512, + "pass@32": 0.9647881464197946, + "pass@64": 0.9690691600729979, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.7840975467289719, + "pass@2": 0.8466142192214293, + "pass@4": 0.8872996447873914, + "pass@8": 0.9131255212050312, + "pass@16": 0.9296828208789675, + "pass@32": 0.9427319318749825, + "pass@64": 0.9554954405969336, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.693099710982659, + "pass@2": 0.7778968583587461, + "pass@4": 0.833728548292012, + "pass@8": 0.8711126510602295, + "pass@16": 0.8958576584940476, + "pass@32": 0.9133306283181201, + "pass@64": 0.9284685661529859, + "pass@128": 0.9421965317919075 + }, + "16": { + "pass@1": 0.4913054435483871, + "pass@2": 0.5962447612395223, + "pass@4": 0.6747601975156323, + "pass@8": 0.7348407821560055, + "pass@16": 0.7834148610294204, + "pass@32": 0.8244895329338998, + "pass@64": 0.8601964216978978, + "pass@128": 0.8903225806451613 + }, + "17": { + "pass@1": 0.35248825867052025, + "pass@2": 0.4572849580128351, + "pass@4": 0.5492719623919837, + "pass@8": 0.6229445136286828, + "pass@16": 0.6790291809157839, + "pass@32": 0.7227658365318381, + "pass@64": 0.7602209527875393, + "pass@128": 0.7976878612716763 + }, + "18": { + "pass@1": 0.27520089285714283, + "pass@2": 0.3652481721034869, + "pass@4": 0.4477995007766887, + "pass@8": 0.5185924991538704, + "pass@16": 0.5777052275710203, + "pass@32": 0.6248875096332369, + "pass@64": 0.6638893307765378, + "pass@128": 0.6971428571428572 + }, + "19": { + "pass@1": 0.2045795795795796, + "pass@2": 0.28665747440353734, + "pass@4": 0.368495601338121, + "pass@8": 0.44158292317157055, + "pass@16": 0.5021246498473418, + "pass@32": 0.5528371750040144, + "pass@64": 0.5987933372999518, + "pass@128": 0.6396396396396397 + }, + "20": { + "pass@1": 0.19566441441441443, + "pass@2": 0.2724017078101725, + "pass@4": 0.34911851418347484, + "pass@8": 0.41767593951939913, + "pass@16": 0.47676148728079454, + "pass@32": 0.5280550430504534, + "pass@64": 0.5749912993919669, + "pass@128": 0.6156156156156156 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 248292, + "answer_accuracy": 0.5768008474576272, + "avg_response_len": 346.25764523862625, + "resp_tokens_sum": 149051451, + "pass_at_k": { + "pass@1": 0.5768008474576272, + "pass@2": 0.6462570475133562, + "pass@4": 0.7029366738948009, + "pass@8": 0.7483487965631102, + "pass@16": 0.7839769463433715, + "pass@32": 0.8124999007618324, + "pass@64": 0.8363338575179757, + "pass@128": 0.8581623550401427 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9631024096385542, + "pass@2": 0.972274185323973, + "pass@4": 0.9774970881840069, + "pass@8": 0.9805179334532464, + "pass@16": 0.9822529509986989, + "pass@32": 0.9834096807350399, + "pass@64": 0.9849397431912068, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8928781347962382, + "pass@2": 0.9105734579024021, + "pass@4": 0.920715678859108, + "pass@8": 0.928907449166582, + "pass@16": 0.9358474938634546, + "pass@32": 0.9417129174503823, + "pass@64": 0.9473011819327243, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.8309042366946778, + "pass@2": 0.8788773875692008, + "pass@4": 0.9060005905249233, + "pass@8": 0.9230680510501225, + "pass@16": 0.9352975407775181, + "pass@32": 0.9462738870529341, + "pass@64": 0.956931111957274, + "pass@128": 0.9663865546218487 + }, + "14": { + "pass@1": 0.7888797514619883, + "pass@2": 0.8581284966846247, + "pass@4": 0.9010667306937505, + "pass@8": 0.9274503748185816, + "pass@16": 0.9419725164541798, + "pass@32": 0.9505580533617753, + "pass@64": 0.9588922804756361, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.6662642045454545, + "pass@2": 0.7540548198520639, + "pass@4": 0.8157365976980145, + "pass@8": 0.8565477326282641, + "pass@16": 0.8831288202183204, + "pass@32": 0.9034765538582886, + "pass@64": 0.9217986009661605, + "pass@128": 0.9393939393939394 + }, + "16": { + "pass@1": 0.5013799857549858, + "pass@2": 0.6062000162639922, + "pass@4": 0.688734711846916, + "pass@8": 0.7507240841046224, + "pass@16": 0.7988058883708389, + "pass@32": 0.8382289811645698, + "pass@64": 0.8684671985838386, + "pass@128": 0.8945868945868946 + }, + "17": { + "pass@1": 0.39540469720496896, + "pass@2": 0.5015440070670513, + "pass@4": 0.5931639159429605, + "pass@8": 0.6665612968773015, + "pass@16": 0.7219907477028704, + "pass@32": 0.7646644496183821, + "pass@64": 0.7996400240328937, + "pass@128": 0.8322981366459627 + }, + "18": { + "pass@1": 0.32417279411764705, + "pass@2": 0.42494535954145407, + "pass@4": 0.517787185425351, + "pass@8": 0.5978804895419353, + "pass@16": 0.6618267341101194, + "pass@32": 0.7122069429681127, + "pass@64": 0.7544866903270767, + "pass@128": 0.788235294117647 + }, + "19": { + "pass@1": 0.20305524553571427, + "pass@2": 0.2782470794666291, + "pass@4": 0.35577190797578867, + "pass@8": 0.42750031369822505, + "pass@16": 0.4901520073095929, + "pass@32": 0.5430748566756388, + "pass@64": 0.5864859314590565, + "pass@128": 0.625 + }, + "20": { + "pass@1": 0.20106661676646706, + "pass@2": 0.2747646930548353, + "pass@4": 0.3491225167774687, + "pass@8": 0.42036866918817706, + "pass@16": 0.4844573511671379, + "pass@32": 0.5371766445282311, + "pass@64": 0.5800651608494923, + "pass@128": 0.6197604790419161 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 234701, + "answer_accuracy": 0.5603916755806846, + "avg_response_len": 373.75660194452934, + "resp_tokens_sum": 156535245, + "pass_at_k": { + "pass@1": 0.5603916755806846, + "pass@2": 0.6303471699146198, + "pass@4": 0.6888534570690135, + "pass@8": 0.7364011139074038, + "pass@16": 0.7734842148415902, + "pass@32": 0.8019971811008832, + "pass@64": 0.825036901704703, + "pass@128": 0.844437652811736 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9642857142857143, + "pass@2": 0.9765787089113863, + "pass@4": 0.9820486778438408, + "pass@8": 0.9842835869687955, + "pass@16": 0.9852484301067794, + "pass@32": 0.9863141673301701, + "pass@64": 0.9881014873140843, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9083763379204893, + "pass@2": 0.9310500644128201, + "pass@4": 0.9455652481513205, + "pass@8": 0.956169735463612, + "pass@16": 0.9645099947486562, + "pass@32": 0.9711630837875163, + "pass@64": 0.9769114489739431, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.8012393904320988, + "pass@2": 0.8506990011665209, + "pass@4": 0.8808248887870499, + "pass@8": 0.9035617488598224, + "pass@16": 0.9217366232932731, + "pass@32": 0.9348758028470283, + "pass@64": 0.9448526103401694, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.7654395400593472, + "pass@2": 0.8289044428608145, + "pass@4": 0.8680753477773732, + "pass@8": 0.8935025847812458, + "pass@16": 0.9102918510356471, + "pass@32": 0.9225928362575211, + "pass@64": 0.934536393956455, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6340663580246914, + "pass@2": 0.7118602362204722, + "pass@4": 0.7674899159017153, + "pass@8": 0.8127011283278409, + "pass@16": 0.849350961539353, + "pass@32": 0.8759008146277355, + "pass@64": 0.892354569021042, + "pass@128": 0.9012345679012346 + }, + "16": { + "pass@1": 0.4710084808259587, + "pass@2": 0.580898471070541, + "pass@4": 0.668739560762868, + "pass@8": 0.7335904176217118, + "pass@16": 0.7809455072412861, + "pass@32": 0.8166722390431593, + "pass@64": 0.8488481347262333, + "pass@128": 0.8849557522123894 + }, + "17": { + "pass@1": 0.36243411144578314, + "pass@2": 0.4642222927141631, + "pass@4": 0.5592205760612748, + "pass@8": 0.6385525287524542, + "pass@16": 0.6994129926552873, + "pass@32": 0.7462385573378134, + "pass@64": 0.7809706376082151, + "pass@128": 0.8042168674698795 + }, + "18": { + "pass@1": 0.31597782258064516, + "pass@2": 0.4188174688849377, + "pass@4": 0.5104581848639886, + "pass@8": 0.5790573511639918, + "pass@16": 0.6283327210915123, + "pass@32": 0.6653820450438427, + "pass@64": 0.6937567396642746, + "pass@128": 0.7161290322580646 + }, + "19": { + "pass@1": 0.2052256419939577, + "pass@2": 0.28584974248876, + "pass@4": 0.36753850377841757, + "pass@8": 0.44054975863490586, + "pass@16": 0.49795327570192016, + "pass@32": 0.5453148986286661, + "pass@64": 0.5923666332779112, + "pass@128": 0.6404833836858006 + }, + "20": { + "pass@1": 0.19200450450450451, + "pass@2": 0.26821087820103573, + "pass@4": 0.3499301157175173, + "pass@8": 0.4312063902239929, + "pass@16": 0.5043251579531292, + "pass@32": 0.5612592194970087, + "pass@64": 0.6019373120540266, + "pass@128": 0.6306306306306306 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7695604440789474, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7695604440789474, + "pass@2": 0.8071646614691095, + "pass@4": 0.8378902844019319, + "pass@8": 0.8625013052951703, + "pass@16": 0.8817409478945274, + "pass@32": 0.8969269202854718, + "pass@64": 0.9096038465179878, + "pass@128": 0.9207894736842105 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9524140625, + "pass@2": 0.9645937500000002, + "pass@4": 0.9730021943194608, + "pass@8": 0.9786367820253195, + "pass@16": 0.9831046505061921, + "pass@32": 0.9868706993361868, + "pass@64": 0.9899770280239669, + "pass@128": 0.993 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9987109375, + "pass@2": 0.999171998031496, + "pass@4": 0.9994346713535808, + "pass@8": 0.9996911656406087, + "pass@16": 0.9999126144227602, + "pass@32": 0.9999949322702752, + "pass@64": 0.9999999977569969, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.993890625, + "pass@2": 0.996357775590551, + "pass@4": 0.9976947060367454, + "pass@8": 0.9983995073818661, + "pass@16": 0.9989712933683291, + "pass@32": 0.9994882787743641, + "pass@64": 0.9998748568874752, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.997625, + "pass@2": 0.9985620078740157, + "pass@4": 0.9990585259655044, + "pass@8": 0.9993178674757099, + "pass@16": 0.9995588461117559, + "pass@32": 0.9998290717621782, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997484375, + "pass@2": 0.9984462352362204, + "pass@4": 0.9990977215973003, + "pass@8": 0.9995759792654791, + "pass@16": 0.9998778554116803, + "pass@32": 0.9999900984417422, + "pass@64": 0.999999987930507, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995140625, + "pass@2": 0.9973559301181102, + "pass@4": 0.9985708786089239, + "pass@8": 0.9991065163275745, + "pass@16": 0.9993309358545857, + "pass@32": 0.9995814582857677, + "pass@64": 0.9998779527559055, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9940546875, + "pass@2": 0.9959009596456694, + "pass@4": 0.9966315292463443, + "pass@8": 0.99717822839966, + "pass@16": 0.9976478183722098, + "pass@32": 0.9979294278134688, + "pass@64": 0.9979985548661734, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.9613984375, + "pass@2": 0.9735031988188977, + "pass@4": 0.9804231168916386, + "pass@8": 0.9847109635922205, + "pass@16": 0.9870854668143153, + "pass@32": 0.9886758259692309, + "pass@64": 0.9904900450431707, + "pass@128": 0.993 + }, + "12": { + "pass@1": 0.899875, + "pass@2": 0.9206909448818899, + "pass@4": 0.9333334387889014, + "pass@8": 0.9429585488390639, + "pass@16": 0.9508381683007954, + "pass@32": 0.957124237853498, + "pass@64": 0.9620219505951612, + "pass@128": 0.966 + }, + "13": { + "pass@1": 0.8238984375, + "pass@2": 0.8724411909448823, + "pass@4": 0.9021637046306712, + "pass@8": 0.9231090688142323, + "pass@16": 0.9379672291934675, + "pass@32": 0.9484869565082493, + "pass@64": 0.9568897147822485, + "pass@128": 0.963 + }, + "14": { + "pass@1": 0.7794453125, + "pass@2": 0.8445839074803148, + "pass@4": 0.8855294000749906, + "pass@8": 0.9114116915660494, + "pass@16": 0.9273511399284904, + "pass@32": 0.9386215902003806, + "pass@64": 0.9495939611176091, + "pass@128": 0.962 + }, + "15": { + "pass@1": 0.6651171875, + "pass@2": 0.7486331200787407, + "pass@4": 0.806329887701538, + "pass@8": 0.8473808946123879, + "pass@16": 0.8765889720497365, + "pass@32": 0.8979515241106913, + "pass@64": 0.9145665425705841, + "pass@128": 0.928 + }, + "16": { + "pass@1": 0.4879609375, + "pass@2": 0.5945366633858277, + "pass@4": 0.6776242561867271, + "pass@8": 0.739991947562845, + "pass@16": 0.7879800006920792, + "pass@32": 0.8266620166339039, + "pass@64": 0.8592523951014696, + "pass@128": 0.89 + }, + "17": { + "pass@1": 0.369609375, + "pass@2": 0.47383956692913415, + "pass@4": 0.566708111173604, + "pass@8": 0.6421709788558302, + "pass@16": 0.699630230918739, + "pass@32": 0.7440501332532892, + "pass@64": 0.7798027890890089, + "pass@128": 0.811 + }, + "18": { + "pass@1": 0.3044921875, + "pass@2": 0.402151697834646, + "pass@4": 0.49101950562429714, + "pass@8": 0.5642945200089501, + "pass@16": 0.6220010627856662, + "pass@32": 0.6671294229443815, + "pass@64": 0.7039513297789204, + "pass@128": 0.734 + }, + "19": { + "pass@1": 0.20428125, + "pass@2": 0.28356422244094465, + "pass@4": 0.363903641076115, + "pass@8": 0.4365091889268906, + "pass@16": 0.4967211171125234, + "pass@32": 0.5470671625654399, + "pass@64": 0.5925308099061153, + "pass@128": 0.635 + }, + "20": { + "pass@1": 0.19625, + "pass@2": 0.27179539862204705, + "pass@4": 0.3493901143607049, + "pass@8": 0.4230809513134006, + "pass@16": 0.4885106081527199, + "pass@32": 0.5421586487007338, + "pass@64": 0.5856589913352467, + "pass@128": 0.622 + } + }, + "per_op_accuracy": { + "10": 0.9524140625, + "2": 1.0, + "3": 1.0, + "4": 0.9987109375, + "5": 0.993890625, + "6": 0.997625, + "7": 0.997484375, + "8": 0.995140625, + "9": 0.9940546875, + "11": 0.9613984375, + "12": 0.899875, + "13": 0.8238984375, + "14": 0.7794453125, + "15": 0.6651171875, + "16": 0.4879609375, + "17": 0.369609375, + "18": 0.3044921875, + "19": 0.20428125, + "20": 0.19625 + }, + "per_op_avg_response_len": { + "10": 279.8089140625, + "2": 115.4520703125, + "3": 144.40334375, + "4": 159.6451328125, + "5": 172.749, + "6": 198.4754375, + "7": 223.35846875, + "8": 242.756265625, + "9": 258.5096484375, + "11": 297.492234375, + "12": 307.809671875, + "13": 326.856828125, + "14": 351.034875, + "15": 362.367296875, + "16": 361.9160234375, + "17": 375.5798671875, + "18": 376.86175, + "19": 367.4975859375, + "20": 379.247375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.8089140625, + "2": 115.4520703125, + "3": 144.40334375, + "4": 159.6451328125, + "5": 172.749, + "6": 198.4754375, + "7": 223.35846875, + "8": 242.756265625, + "9": 258.5096484375, + "11": 297.492234375, + "12": 307.809671875, + "13": 326.856828125, + "14": 351.034875, + "15": 362.367296875, + "16": 361.9160234375, + "17": 375.5798671875, + "18": 376.86175, + "19": 367.4975859375, + "20": 379.247375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 621973, + "answer_accuracy": 0.7681258397881758, + "avg_response_len": 265.83570408828643, + "resp_tokens_sum": 215254613, + "pass_at_k": { + "pass@1": 0.7681258397881758, + "pass@2": 0.8056889833794223, + "pass@4": 0.8361066637487128, + "pass@8": 0.8603037310757391, + "pass@16": 0.8792181006261285, + "pass@32": 0.8943473727278064, + "pass@64": 0.9074203405159689, + "pass@128": 0.9190641795763516 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9574575958702065, + "pass@2": 0.9697600196269713, + "pass@4": 0.9770701256369501, + "pass@8": 0.9807352442070735, + "pass@16": 0.9835831385400656, + "pass@32": 0.9868139186902647, + "pass@64": 0.9902929813385202, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996923828125, + "pass@2": 0.9976178026574803, + "pass@4": 0.9982453347628422, + "pass@8": 0.9990349229744625, + "pass@16": 0.9997269200711651, + "pass@32": 0.9999841633446103, + "pass@64": 0.9999999929906153, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9954861111111111, + "pass@2": 0.9981666354205724, + "pass@4": 0.9995852298224628, + "pass@8": 0.9999710725449563, + "pass@16": 0.9999998592866416, + "pass@32": 0.9999999999993872, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9968890344311377, + "pass@2": 0.997264575180348, + "pass@4": 0.997534851556729, + "pass@8": 0.9979835642250304, + "pass@16": 0.9986793120796699, + "pass@32": 0.9994882388094946, + "pass@64": 0.9999586116800393, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9973187869822485, + "pass@2": 0.9984067785258351, + "pass@4": 0.9992237471795316, + "pass@8": 0.9998073811974663, + "pass@16": 0.9999900064095892, + "pass@32": 0.9999999886267649, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9983974358974359, + "pass@2": 0.999743290051484, + "pass@4": 0.9999842372467864, + "pass@8": 0.9999999472370541, + "pass@16": 0.9999999999998607, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9972687007874016, + "pass@4": 0.997876473539399, + "pass@8": 0.9984546377928358, + "pass@16": 0.999188066655105, + "pass@32": 0.9998084093975909, + "pass@64": 0.9999959303192606, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9572193696883853, + "pass@2": 0.9719146628449066, + "pass@4": 0.981724504514839, + "pass@8": 0.9890359200866053, + "pass@16": 0.9932697779011666, + "pass@32": 0.9957361168730702, + "pass@64": 0.9978415915007174, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.8983271539548022, + "pass@2": 0.920239098158281, + "pass@4": 0.9334048053315367, + "pass@8": 0.9434168617466765, + "pass@16": 0.9517176538857137, + "pass@32": 0.9580437536392892, + "pass@64": 0.961533417356335, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8390722962382445, + "pass@2": 0.8873212765038385, + "pass@4": 0.9195430402704361, + "pass@8": 0.9430086769866958, + "pass@16": 0.9574399410309512, + "pass@32": 0.9647881464197946, + "pass@64": 0.9690691600729979, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.7840975467289719, + "pass@2": 0.8466142192214293, + "pass@4": 0.8872996447873914, + "pass@8": 0.9131255212050312, + "pass@16": 0.9296828208789675, + "pass@32": 0.9427319318749825, + "pass@64": 0.9554954405969336, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.693099710982659, + "pass@2": 0.7778968583587461, + "pass@4": 0.833728548292012, + "pass@8": 0.8711126510602295, + "pass@16": 0.8958576584940476, + "pass@32": 0.9133306283181201, + "pass@64": 0.9284685661529859, + "pass@128": 0.9421965317919075 + }, + "16": { + "pass@1": 0.4913054435483871, + "pass@2": 0.5962447612395223, + "pass@4": 0.6747601975156323, + "pass@8": 0.7348407821560055, + "pass@16": 0.7834148610294204, + "pass@32": 0.8244895329338998, + "pass@64": 0.8601964216978978, + "pass@128": 0.8903225806451613 + }, + "17": { + "pass@1": 0.35248825867052025, + "pass@2": 0.4572849580128351, + "pass@4": 0.5492719623919837, + "pass@8": 0.6229445136286828, + "pass@16": 0.6790291809157839, + "pass@32": 0.7227658365318381, + "pass@64": 0.7602209527875393, + "pass@128": 0.7976878612716763 + }, + "18": { + "pass@1": 0.27520089285714283, + "pass@2": 0.3652481721034869, + "pass@4": 0.4477995007766887, + "pass@8": 0.5185924991538704, + "pass@16": 0.5777052275710203, + "pass@32": 0.6248875096332369, + "pass@64": 0.6638893307765378, + "pass@128": 0.6971428571428572 + }, + "19": { + "pass@1": 0.2045795795795796, + "pass@2": 0.28665747440353734, + "pass@4": 0.368495601338121, + "pass@8": 0.44158292317157055, + "pass@16": 0.5021246498473418, + "pass@32": 0.5528371750040144, + "pass@64": 0.5987933372999518, + "pass@128": 0.6396396396396397 + }, + "20": { + "pass@1": 0.19566441441441443, + "pass@2": 0.2724017078101725, + "pass@4": 0.34911851418347484, + "pass@8": 0.41767593951939913, + "pass@16": 0.47676148728079454, + "pass@32": 0.5280550430504534, + "pass@64": 0.5749912993919669, + "pass@128": 0.6156156156156156 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 635595, + "answer_accuracy": 0.774058602883866, + "avg_response_len": 275.4775172934528, + "resp_tokens_sum": 226200099, + "pass_at_k": { + "pass@1": 0.774058602883866, + "pass@2": 0.8115189700566369, + "pass@4": 0.8419047259145173, + "pass@8": 0.8661518210454625, + "pass@16": 0.8851545001914215, + "pass@32": 0.9003565916378781, + "pass@64": 0.9130658000646252, + "pass@128": 0.9247077162899454 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9421731211656442, + "pass@2": 0.9570635174629243, + "pass@4": 0.9675180118397778, + "pass@8": 0.9744761195433342, + "pass@16": 0.9791587744555608, + "pass@32": 0.9822004591222739, + "pass@64": 0.984571936009979, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956295289855073, + "pass@2": 0.9968182985279016, + "pass@4": 0.9973139007080636, + "pass@8": 0.9976146824957167, + "pass@16": 0.998064779875966, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9972242120343839, + "pass@2": 0.9987002374613632, + "pass@4": 0.9996714137452016, + "pass@8": 0.9999752561101265, + "pass@16": 0.9999998735735035, + "pass@32": 0.999999999999447, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9973807803468208, + "pass@2": 0.9977498748350099, + "pass@4": 0.9982556172530457, + "pass@8": 0.9989655495774096, + "pass@16": 0.9996567447729243, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954665492957746, + "pass@2": 0.9968725740268383, + "pass@4": 0.9974011850631346, + "pass@8": 0.9976822657434412, + "pass@16": 0.9981192931743675, + "pass@32": 0.9988210100498002, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9930295658682635, + "pass@2": 0.9939206233202886, + "pass@4": 0.9940102609741148, + "pass@8": 0.9940119756217468, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9631024096385542, + "pass@2": 0.972274185323973, + "pass@4": 0.9774970881840069, + "pass@8": 0.9805179334532464, + "pass@16": 0.9822529509986989, + "pass@32": 0.9834096807350399, + "pass@64": 0.9849397431912068, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8928781347962382, + "pass@2": 0.9105734579024021, + "pass@4": 0.920715678859108, + "pass@8": 0.928907449166582, + "pass@16": 0.9358474938634546, + "pass@32": 0.9417129174503823, + "pass@64": 0.9473011819327243, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.8309042366946778, + "pass@2": 0.8788773875692008, + "pass@4": 0.9060005905249233, + "pass@8": 0.9230680510501225, + "pass@16": 0.9352975407775181, + "pass@32": 0.9462738870529341, + "pass@64": 0.956931111957274, + "pass@128": 0.9663865546218487 + }, + "14": { + "pass@1": 0.7888797514619883, + "pass@2": 0.8581284966846247, + "pass@4": 0.9010667306937505, + "pass@8": 0.9274503748185816, + "pass@16": 0.9419725164541798, + "pass@32": 0.9505580533617753, + "pass@64": 0.9588922804756361, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.6662642045454545, + "pass@2": 0.7540548198520639, + "pass@4": 0.8157365976980145, + "pass@8": 0.8565477326282641, + "pass@16": 0.8831288202183204, + "pass@32": 0.9034765538582886, + "pass@64": 0.9217986009661605, + "pass@128": 0.9393939393939394 + }, + "16": { + "pass@1": 0.5013799857549858, + "pass@2": 0.6062000162639922, + "pass@4": 0.688734711846916, + "pass@8": 0.7507240841046224, + "pass@16": 0.7988058883708389, + "pass@32": 0.8382289811645698, + "pass@64": 0.8684671985838386, + "pass@128": 0.8945868945868946 + }, + "17": { + "pass@1": 0.39540469720496896, + "pass@2": 0.5015440070670513, + "pass@4": 0.5931639159429605, + "pass@8": 0.6665612968773015, + "pass@16": 0.7219907477028704, + "pass@32": 0.7646644496183821, + "pass@64": 0.7996400240328937, + "pass@128": 0.8322981366459627 + }, + "18": { + "pass@1": 0.32417279411764705, + "pass@2": 0.42494535954145407, + "pass@4": 0.517787185425351, + "pass@8": 0.5978804895419353, + "pass@16": 0.6618267341101194, + "pass@32": 0.7122069429681127, + "pass@64": 0.7544866903270767, + "pass@128": 0.788235294117647 + }, + "19": { + "pass@1": 0.20305524553571427, + "pass@2": 0.2782470794666291, + "pass@4": 0.35577190797578867, + "pass@8": 0.42750031369822505, + "pass@16": 0.4901520073095929, + "pass@32": 0.5430748566756388, + "pass@64": 0.5864859314590565, + "pass@128": 0.625 + }, + "20": { + "pass@1": 0.20106661676646706, + "pass@2": 0.2747646930548353, + "pass@4": 0.3491225167774687, + "pass@8": 0.42036866918817706, + "pass@16": 0.4844573511671379, + "pass@32": 0.5371766445282311, + "pass@64": 0.5800651608494923, + "pass@128": 0.6197604790419161 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 614003, + "answer_accuracy": 0.7664001338073174, + "avg_response_len": 296.0467888740214, + "resp_tokens_sum": 237178477, + "pass_at_k": { + "pass@1": 0.7664001338073174, + "pass@2": 0.8041933002303396, + "pass@4": 0.8355785000832348, + "pass@8": 0.8609809020316768, + "pass@16": 0.8807921691495971, + "pass@32": 0.8960189279742334, + "pass@64": 0.9082624863913064, + "pass@128": 0.918517335037546 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9572761194029851, + "pass@2": 0.9666937213538606, + "pass@4": 0.9742225369216908, + "pass@8": 0.9805621470686411, + "pass@16": 0.9864603166226772, + "pass@32": 0.991472929033808, + "pass@64": 0.9949171827192708, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.9998089774140073, + "pass@4": 0.9999887842966998, + "pass@8": 0.9999999716046218, + "pass@16": 0.9999999999999635, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906479779411764, + "pass@2": 0.9942146248263086, + "pass@4": 0.996329596759964, + "pass@8": 0.9977398649093597, + "pass@16": 0.9989382018700852, + "pass@32": 0.9997259417849353, + "pass@64": 0.9999908945046169, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9988416798107256, + "pass@2": 0.9997768356143968, + "pass@4": 0.9999891550425282, + "pass@8": 0.9999999767890079, + "pass@16": 0.9999999999999797, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997774920886076, + "pass@2": 0.9992509094986542, + "pass@4": 0.9998849730413445, + "pass@8": 0.9999968489460502, + "pass@16": 0.9999999985879974, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9917417417417418, + "pass@2": 0.9956344139415007, + "pass@4": 0.998493621292834, + "pass@8": 0.999787773125201, + "pass@16": 0.9999957560892757, + "pass@32": 0.9999999991534195, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932174437299035, + "pass@2": 0.9964665037850975, + "pass@4": 0.9980255754807175, + "pass@8": 0.9991216467059164, + "pass@16": 0.9997943881339146, + "pass@32": 0.9999917764544182, + "pass@64": 0.9999999987229452, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9642857142857143, + "pass@2": 0.9765787089113863, + "pass@4": 0.9820486778438408, + "pass@8": 0.9842835869687955, + "pass@16": 0.9852484301067794, + "pass@32": 0.9863141673301701, + "pass@64": 0.9881014873140843, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9083763379204893, + "pass@2": 0.9310500644128201, + "pass@4": 0.9455652481513205, + "pass@8": 0.956169735463612, + "pass@16": 0.9645099947486562, + "pass@32": 0.9711630837875163, + "pass@64": 0.9769114489739431, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.8012393904320988, + "pass@2": 0.8506990011665209, + "pass@4": 0.8808248887870499, + "pass@8": 0.9035617488598224, + "pass@16": 0.9217366232932731, + "pass@32": 0.9348758028470283, + "pass@64": 0.9448526103401694, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.7654395400593472, + "pass@2": 0.8289044428608145, + "pass@4": 0.8680753477773732, + "pass@8": 0.8935025847812458, + "pass@16": 0.9102918510356471, + "pass@32": 0.9225928362575211, + "pass@64": 0.934536393956455, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6340663580246914, + "pass@2": 0.7118602362204722, + "pass@4": 0.7674899159017153, + "pass@8": 0.8127011283278409, + "pass@16": 0.849350961539353, + "pass@32": 0.8759008146277355, + "pass@64": 0.892354569021042, + "pass@128": 0.9012345679012346 + }, + "16": { + "pass@1": 0.4710084808259587, + "pass@2": 0.580898471070541, + "pass@4": 0.668739560762868, + "pass@8": 0.7335904176217118, + "pass@16": 0.7809455072412861, + "pass@32": 0.8166722390431593, + "pass@64": 0.8488481347262333, + "pass@128": 0.8849557522123894 + }, + "17": { + "pass@1": 0.36243411144578314, + "pass@2": 0.4642222927141631, + "pass@4": 0.5592205760612748, + "pass@8": 0.6385525287524542, + "pass@16": 0.6994129926552873, + "pass@32": 0.7462385573378134, + "pass@64": 0.7809706376082151, + "pass@128": 0.8042168674698795 + }, + "18": { + "pass@1": 0.31597782258064516, + "pass@2": 0.4188174688849377, + "pass@4": 0.5104581848639886, + "pass@8": 0.5790573511639918, + "pass@16": 0.6283327210915123, + "pass@32": 0.6653820450438427, + "pass@64": 0.6937567396642746, + "pass@128": 0.7161290322580646 + }, + "19": { + "pass@1": 0.2052256419939577, + "pass@2": 0.28584974248876, + "pass@4": 0.36753850377841757, + "pass@8": 0.44054975863490586, + "pass@16": 0.49795327570192016, + "pass@32": 0.5453148986286661, + "pass@64": 0.5923666332779112, + "pass@128": 0.6404833836858006 + }, + "20": { + "pass@1": 0.19200450450450451, + "pass@2": 0.26821087820103573, + "pass@4": 0.3499301157175173, + "pass@8": 0.4312063902239929, + "pass@16": 0.5043251579531292, + "pass@32": 0.5612592194970087, + "pass@64": 0.6019373120540266, + "pass@128": 0.6306306306306306 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block16-epoch4-op9-12/graphs/SDAR-100M-block16-epoch4-op9-12_score_report.jsonl b/composition/SDAR-100M-block16-epoch4-op9-12/graphs/SDAR-100M-block16-epoch4-op9-12_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b67b8a161c2fc73da40aebad1c2b5fc14f3520af --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op9-12/graphs/SDAR-100M-block16-epoch4-op9-12_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block16-epoch4-op9-12", "results_dir": "results/composition/SDAR-100M-block16-epoch4-op9-12/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1022319, "value_mismatch": 10333, "dependency_mismatch": 127026, "answer_mismatch": 9047, "extra_nodes": 12152, "missing_nodes": 10664, "perfect_with_extra": 420, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337782, "value_mismatch": 3292, "answer_mismatch": 2660, "dependency_mismatch": 40549, "extra_nodes": 3314, "perfect_with_extra": 31, "missing_nodes": 2872}, "teachers_in_school": {"total": 390656, "perfect": 345675, "value_mismatch": 3586, "dependency_mismatch": 43945, "answer_mismatch": 3353, "extra_nodes": 4681, "missing_nodes": 4394, "perfect_with_extra": 98}, "movie_festival_awards": {"total": 382336, "perfect": 338862, "dependency_mismatch": 42532, "extra_nodes": 4157, "missing_nodes": 3398, "value_mismatch": 3455, "answer_mismatch": 3034, "perfect_with_extra": 291}}, "results_path": "results/composition/SDAR-100M-block16-epoch4-op9-12/graphs/SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 349116, "value_mismatch": 642474, "dependency_mismatch": 835185, "answer_mismatch": 551376, "extra_nodes": 145027, "missing_nodes": 563419, "perfect_with_extra": 183, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 118889, "dependency_mismatch": 282855, "extra_nodes": 40130, "value_mismatch": 216731, "answer_mismatch": 185094, "perfect_with_extra": 111, "missing_nodes": 189403}, "movie_festival_awards": {"total": 418816, "perfect": 110861, "value_mismatch": 213667, "dependency_mismatch": 274205, "answer_mismatch": 184113, "extra_nodes": 53635, "missing_nodes": 182836, "perfect_with_extra": 62}, "teachers_in_school": {"total": 430464, "perfect": 119366, "dependency_mismatch": 278125, "value_mismatch": 212076, "answer_mismatch": 182169, "extra_nodes": 51262, "missing_nodes": 191180, "perfect_with_extra": 10}}, "results_path": "results/composition/SDAR-100M-block16-epoch4-op9-12/graphs/SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1371435, "value_mismatch": 652807, "dependency_mismatch": 962211, "answer_mismatch": 560423, "extra_nodes": 157179, "missing_nodes": 574083, "perfect_with_extra": 603, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block16-epoch4-op9-12/graphs/SDAR-100M-block16-epoch4-op9-12_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 456671, "value_mismatch": 220023, "answer_mismatch": 187754, "dependency_mismatch": 323404, "extra_nodes": 43444, "perfect_with_extra": 142, "missing_nodes": 192275}, "teachers_in_school": {"total": 821120, "perfect": 465041, "value_mismatch": 215662, "dependency_mismatch": 322070, "answer_mismatch": 185522, "extra_nodes": 55943, "missing_nodes": 195574, "perfect_with_extra": 108}, "movie_festival_awards": {"total": 801152, "perfect": 449723, "dependency_mismatch": 316737, "extra_nodes": 57792, "missing_nodes": 186234, "value_mismatch": 217122, "answer_mismatch": 187147, "perfect_with_extra": 353}}}} diff --git a/composition/SDAR-100M-block16-epoch4-op9-12/graphs/results.json b/composition/SDAR-100M-block16-epoch4-op9-12/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..e710e1396b541e92bc588d0468f6d09ca3a7bb8a --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op9-12/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.99375, + "pass@4": 0.996875, + "pass@8": 0.996875, + "pass@16": 0.996875, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9650793650793651, + "pass@2": 0.9873015873015873, + "pass@4": 0.9936507936507937, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9670658682634731, + "pass@2": 0.9730538922155688, + "pass@4": 0.9910179640718563, + "pass@8": 0.9970059880239521, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9023668639053254, + "pass@2": 0.9378698224852071, + "pass@4": 0.9733727810650887, + "pass@8": 0.9940828402366864, + "pass@16": 0.9970414201183432, + "pass@32": 0.9970414201183432, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8333333333333334, + "pass@2": 0.907051282051282, + "pass@4": 0.9583333333333334, + "pass@8": 0.9839743589743589, + "pass@16": 0.9935897435897436, + "pass@32": 0.9967948717948718, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7661971830985915, + "pass@2": 0.828169014084507, + "pass@4": 0.9126760563380282, + "pass@8": 0.9661971830985916, + "pass@16": 0.9943661971830986, + "pass@32": 0.9943661971830986, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6519174041297935, + "pass@2": 0.7138643067846607, + "pass@4": 0.8082595870206489, + "pass@8": 0.8908554572271387, + "pass@16": 0.9262536873156342, + "pass@32": 0.9410029498525073, + "pass@64": 0.9498525073746312, + "pass@128": 0.9587020648967551 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9650793650793651, + "6": 0.9670658682634731, + "7": 0.9023668639053254, + "8": 0.8333333333333334, + "9": 0.7661971830985915, + "10": 0.6519174041297935 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9873015873015873, + "6": 0.9730538922155688, + "7": 0.9378698224852071, + "8": 0.907051282051282, + "9": 0.828169014084507, + "10": 0.7138643067846607 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 0.9910179640718563, + "7": 0.9733727810650887, + "8": 0.9583333333333334, + "9": 0.9126760563380282, + "10": 0.8082595870206489 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 1.0, + "6": 0.9970059880239521, + "7": 0.9940828402366864, + "8": 0.9839743589743589, + "9": 0.9661971830985916, + "10": 0.8908554572271387 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 1.0, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9935897435897436, + "9": 0.9943661971830986, + "10": 0.9262536873156342 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 0.9943661971830986, + "10": 0.9410029498525073 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9498525073746312 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9587020648967551 + } + }, + "total_pass_at_k": { + "pass@1": 0.895305639986491, + "pass@2": 0.9246876055386694, + "pass@4": 0.9581222559945964, + "pass@8": 0.9804120229652145, + "pass@16": 0.9895305639986491, + "pass@32": 0.9918946301925026, + "pass@64": 0.994258696386356, + "pass@128": 0.9952718676122931 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9971428571428571, + "pass@2": 0.9971428571428571, + "pass@4": 0.9971428571428571, + "pass@8": 0.9971428571428571, + "pass@16": 0.9971428571428571, + "pass@32": 0.9971428571428571, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 0.9941520467836257, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.95, + "pass@2": 0.9705882352941176, + "pass@4": 0.9852941176470589, + "pass@8": 0.9970588235294118, + "pass@16": 0.9970588235294118, + "pass@32": 0.9970588235294118, + "pass@64": 0.9970588235294118, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9621451104100947, + "pass@2": 0.9716088328075709, + "pass@4": 0.9873817034700315, + "pass@8": 0.9968454258675079, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8734177215189873, + "pass@2": 0.8924050632911392, + "pass@4": 0.9556962025316456, + "pass@8": 0.9778481012658228, + "pass@16": 0.9936708860759493, + "pass@32": 0.9968354430379747, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8228228228228228, + "pass@2": 0.8948948948948949, + "pass@4": 0.9459459459459459, + "pass@8": 0.975975975975976, + "pass@16": 0.996996996996997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.77491961414791, + "pass@2": 0.8360128617363344, + "pass@4": 0.8971061093247589, + "pass@8": 0.9485530546623794, + "pass@16": 0.977491961414791, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6119402985074627, + "pass@2": 0.6656716417910448, + "pass@4": 0.7582089552238805, + "pass@8": 0.8238805970149253, + "pass@16": 0.8865671641791045, + "pass@32": 0.9223880597014925, + "pass@64": 0.9343283582089552, + "pass@128": 0.9522388059701492 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.95, + "6": 0.9621451104100947, + "7": 0.8734177215189873, + "8": 0.8228228228228228, + "9": 0.77491961414791, + "10": 0.6119402985074627 + }, + "pass@2": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9705882352941176, + "6": 0.9716088328075709, + "7": 0.8924050632911392, + "8": 0.8948948948948949, + "9": 0.8360128617363344, + "10": 0.6656716417910448 + }, + "pass@4": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 1.0, + "5": 0.9852941176470589, + "6": 0.9873817034700315, + "7": 0.9556962025316456, + "8": 0.9459459459459459, + "9": 0.8971061093247589, + "10": 0.7582089552238805 + }, + "pass@8": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9968454258675079, + "7": 0.9778481012658228, + "8": 0.975975975975976, + "9": 0.9485530546623794, + "10": 0.8238805970149253 + }, + "pass@16": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.996996996996997, + "9": 0.977491961414791, + "10": 0.8865671641791045 + }, + "pass@32": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 1.0, + "9": 1.0, + "10": 0.9223880597014925 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9343283582089552 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9522388059701492 + } + }, + "total_pass_at_k": { + "pass@1": 0.8885169065952461, + "pass@2": 0.9149648476732507, + "pass@4": 0.9481084700368263, + "pass@8": 0.9688650820220958, + "pass@16": 0.983260796786073, + "pass@32": 0.9902912621359223, + "pass@64": 0.9922999665215936, + "pass@128": 0.9943086709072648 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9881656804733728, + "pass@2": 0.9881656804733728, + "pass@4": 0.9940828402366864, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9594202898550724, + "pass@2": 0.9710144927536232, + "pass@4": 0.9971014492753624, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 0.9971014492753624, + "pass@64": 0.9971014492753624, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9713467048710601, + "pass@2": 0.9856733524355301, + "pass@4": 0.9885386819484241, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.861271676300578, + "pass@2": 0.9017341040462428, + "pass@4": 0.9508670520231214, + "pass@8": 0.976878612716763, + "pass@16": 0.9942196531791907, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8422535211267606, + "pass@2": 0.895774647887324, + "pass@4": 0.9408450704225352, + "pass@8": 0.971830985915493, + "pass@16": 0.9943661971830986, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.7425149700598802, + "pass@2": 0.8173652694610778, + "pass@4": 0.8832335329341318, + "pass@8": 0.9311377245508982, + "pass@16": 0.9670658682634731, + "pass@32": 0.9880239520958084, + "pass@64": 0.9910179640718563, + "pass@128": 0.9940119760479041 + }, + "10": { + "pass@1": 0.6625766871165644, + "pass@2": 0.7269938650306749, + "pass@4": 0.7791411042944786, + "pass@8": 0.8466257668711656, + "pass@16": 0.8957055214723927, + "pass@32": 0.9202453987730062, + "pass@64": 0.9355828220858896, + "pass@128": 0.9539877300613497 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9881656804733728, + "5": 0.9594202898550724, + "6": 0.9713467048710601, + "7": 0.861271676300578, + "8": 0.8422535211267606, + "9": 0.7425149700598802, + "10": 0.6625766871165644 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9881656804733728, + "5": 0.9710144927536232, + "6": 0.9856733524355301, + "7": 0.9017341040462428, + "8": 0.895774647887324, + "9": 0.8173652694610778, + "10": 0.7269938650306749 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9940828402366864, + "5": 0.9971014492753624, + "6": 0.9885386819484241, + "7": 0.9508670520231214, + "8": 0.9408450704225352, + "9": 0.8832335329341318, + "10": 0.7791411042944786 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 0.976878612716763, + "8": 0.971830985915493, + "9": 0.9311377245508982, + "10": 0.8466257668711656 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 0.9942196531791907, + "8": 0.9943661971830986, + "9": 0.9670658682634731, + "10": 0.8957055214723927 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9971830985915493, + "9": 0.9880239520958084, + "10": 0.9202453987730062 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9971830985915493, + "9": 0.9910179640718563, + "10": 0.9355828220858896 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9940119760479041, + "10": 0.9539877300613497 + } + }, + "total_pass_at_k": { + "pass@1": 0.8925294888597641, + "pass@2": 0.9213630406290957, + "pass@4": 0.9488859764089121, + "pass@8": 0.9698558322411533, + "pass@16": 0.9836173001310616, + "pass@32": 0.9891874180865007, + "pass@64": 0.9911533420707732, + "pass@128": 0.9937745740498034 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.999, + "pass@2": 0.999, + "pass@4": 0.999, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99, + "pass@2": 0.992, + "pass@4": 0.997, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.958, + "pass@2": 0.976, + "pass@4": 0.992, + "pass@8": 0.998, + "pass@16": 0.998, + "pass@32": 0.998, + "pass@64": 0.998, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.967, + "pass@2": 0.977, + "pass@4": 0.989, + "pass@8": 0.998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.879, + "pass@2": 0.911, + "pass@4": 0.96, + "pass@8": 0.983, + "pass@16": 0.995, + "pass@32": 0.997, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.833, + "pass@2": 0.899, + "pass@4": 0.948, + "pass@8": 0.977, + "pass@16": 0.995, + "pass@32": 0.998, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.761, + "pass@2": 0.827, + "pass@4": 0.898, + "pass@8": 0.949, + "pass@16": 0.98, + "pass@32": 0.994, + "pass@64": 0.997, + "pass@128": 0.998 + }, + "10": { + "pass@1": 0.642, + "pass@2": 0.702, + "pass@4": 0.782, + "pass@8": 0.854, + "pass@16": 0.903, + "pass@32": 0.928, + "pass@64": 0.94, + "pass@128": 0.955 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.999, + "3": 1.0, + "4": 0.99, + "5": 0.958, + "6": 0.967, + "7": 0.879, + "8": 0.833, + "9": 0.761, + "10": 0.642 + }, + "pass@2": { + "2": 0.999, + "3": 1.0, + "4": 0.992, + "5": 0.976, + "6": 0.977, + "7": 0.911, + "8": 0.899, + "9": 0.827, + "10": 0.702 + }, + "pass@4": { + "2": 0.999, + "3": 1.0, + "4": 0.997, + "5": 0.992, + "6": 0.989, + "7": 0.96, + "8": 0.948, + "9": 0.898, + "10": 0.782 + }, + "pass@8": { + "2": 0.999, + "3": 1.0, + "4": 0.999, + "5": 0.998, + "6": 0.998, + "7": 0.983, + "8": 0.977, + "9": 0.949, + "10": 0.854 + }, + "pass@16": { + "2": 0.999, + "3": 1.0, + "4": 0.999, + "5": 0.998, + "6": 1.0, + "7": 0.995, + "8": 0.995, + "9": 0.98, + "10": 0.903 + }, + "pass@32": { + "2": 0.999, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 1.0, + "7": 0.997, + "8": 0.998, + "9": 0.994, + "10": 0.928 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 1.0, + "7": 0.999, + "8": 0.999, + "9": 0.997, + "10": 0.94 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.998, + "10": 0.955 + } + }, + "total_pass_at_k": { + "pass@1": 0.8921111111111111, + "pass@2": 0.9203333333333333, + "pass@4": 0.9516666666666667, + "pass@8": 0.973, + "pass@16": 0.9854444444444445, + "pass@32": 0.9904444444444445, + "pass@64": 0.9925555555555555, + "pass@128": 0.9944444444444445 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5609065155807366, + "pass@2": 0.6685552407932012, + "pass@4": 0.7535410764872521, + "pass@8": 0.8271954674220963, + "pass@16": 0.8980169971671388, + "pass@32": 0.9263456090651558, + "pass@64": 0.9405099150141643, + "pass@128": 0.9461756373937678 + }, + "12": { + "pass@1": 0.5932203389830508, + "pass@2": 0.6497175141242938, + "pass@4": 0.7231638418079096, + "pass@8": 0.7937853107344632, + "pass@16": 0.8389830508474576, + "pass@32": 0.8502824858757062, + "pass@64": 0.864406779661017, + "pass@128": 0.8757062146892656 + }, + "13": { + "pass@1": 0.48589341692789967, + "pass@2": 0.5611285266457681, + "pass@4": 0.6300940438871473, + "pass@8": 0.6990595611285266, + "pass@16": 0.7460815047021944, + "pass@32": 0.7899686520376176, + "pass@64": 0.8181818181818182, + "pass@128": 0.8307210031347962 + }, + "14": { + "pass@1": 0.3925233644859813, + "pass@2": 0.4423676012461059, + "pass@4": 0.5264797507788161, + "pass@8": 0.5919003115264797, + "pass@16": 0.6448598130841121, + "pass@32": 0.6915887850467289, + "pass@64": 0.778816199376947, + "pass@128": 0.8130841121495327 + }, + "15": { + "pass@1": 0.29190751445086704, + "pass@2": 0.36416184971098264, + "pass@4": 0.43352601156069365, + "pass@8": 0.5086705202312138, + "pass@16": 0.569364161849711, + "pass@32": 0.6358381502890174, + "pass@64": 0.6965317919075145, + "pass@128": 0.7601156069364162 + }, + "16": { + "pass@1": 0.16451612903225807, + "pass@2": 0.2161290322580645, + "pass@4": 0.2709677419354839, + "pass@8": 0.3258064516129032, + "pass@16": 0.3774193548387097, + "pass@32": 0.42258064516129035, + "pass@64": 0.47096774193548385, + "pass@128": 0.5 + }, + "17": { + "pass@1": 0.09826589595375723, + "pass@2": 0.14450867052023122, + "pass@4": 0.18786127167630057, + "pass@8": 0.22254335260115607, + "pass@16": 0.2543352601156069, + "pass@32": 0.2947976878612717, + "pass@64": 0.3236994219653179, + "pass@128": 0.3815028901734104 + }, + "18": { + "pass@1": 0.04, + "pass@2": 0.06857142857142857, + "pass@4": 0.08285714285714285, + "pass@8": 0.12285714285714286, + "pass@16": 0.14285714285714285, + "pass@32": 0.16857142857142857, + "pass@64": 0.2, + "pass@128": 0.2342857142857143 + }, + "19": { + "pass@1": 0.012012012012012012, + "pass@2": 0.021021021021021023, + "pass@4": 0.03003003003003003, + "pass@8": 0.042042042042042045, + "pass@16": 0.05405405405405406, + "pass@32": 0.06606606606606606, + "pass@64": 0.07507507507507508, + "pass@128": 0.08708708708708708 + }, + "20": { + "pass@1": 0.015015015015015015, + "pass@2": 0.03303303303303303, + "pass@4": 0.036036036036036036, + "pass@8": 0.042042042042042045, + "pass@16": 0.05405405405405406, + "pass@32": 0.06606606606606606, + "pass@64": 0.08708708708708708, + "pass@128": 0.0990990990990991 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5609065155807366, + "12": 0.5932203389830508, + "13": 0.48589341692789967, + "14": 0.3925233644859813, + "15": 0.29190751445086704, + "16": 0.16451612903225807, + "17": 0.09826589595375723, + "18": 0.04, + "19": 0.012012012012012012, + "20": 0.015015015015015015 + }, + "pass@2": { + "11": 0.6685552407932012, + "12": 0.6497175141242938, + "13": 0.5611285266457681, + "14": 0.4423676012461059, + "15": 0.36416184971098264, + "16": 0.2161290322580645, + "17": 0.14450867052023122, + "18": 0.06857142857142857, + "19": 0.021021021021021023, + "20": 0.03303303303303303 + }, + "pass@4": { + "11": 0.7535410764872521, + "12": 0.7231638418079096, + "13": 0.6300940438871473, + "14": 0.5264797507788161, + "15": 0.43352601156069365, + "16": 0.2709677419354839, + "17": 0.18786127167630057, + "18": 0.08285714285714285, + "19": 0.03003003003003003, + "20": 0.036036036036036036 + }, + "pass@8": { + "11": 0.8271954674220963, + "12": 0.7937853107344632, + "13": 0.6990595611285266, + "14": 0.5919003115264797, + "15": 0.5086705202312138, + "16": 0.3258064516129032, + "17": 0.22254335260115607, + "18": 0.12285714285714286, + "19": 0.042042042042042045, + "20": 0.042042042042042045 + }, + "pass@16": { + "11": 0.8980169971671388, + "12": 0.8389830508474576, + "13": 0.7460815047021944, + "14": 0.6448598130841121, + "15": 0.569364161849711, + "16": 0.3774193548387097, + "17": 0.2543352601156069, + "18": 0.14285714285714285, + "19": 0.05405405405405406, + "20": 0.05405405405405406 + }, + "pass@32": { + "11": 0.9263456090651558, + "12": 0.8502824858757062, + "13": 0.7899686520376176, + "14": 0.6915887850467289, + "15": 0.6358381502890174, + "16": 0.42258064516129035, + "17": 0.2947976878612717, + "18": 0.16857142857142857, + "19": 0.06606606606606606, + "20": 0.06606606606606606 + }, + "pass@64": { + "11": 0.9405099150141643, + "12": 0.864406779661017, + "13": 0.8181818181818182, + "14": 0.778816199376947, + "15": 0.6965317919075145, + "16": 0.47096774193548385, + "17": 0.3236994219653179, + "18": 0.2, + "19": 0.07507507507507508, + "20": 0.08708708708708708 + }, + "pass@128": { + "11": 0.9461756373937678, + "12": 0.8757062146892656, + "13": 0.8307210031347962, + "14": 0.8130841121495327, + "15": 0.7601156069364162, + "16": 0.5, + "17": 0.3815028901734104, + "18": 0.2342857142857143, + "19": 0.08708708708708708, + "20": 0.0990990990990991 + } + }, + "total_pass_at_k": { + "pass@1": 0.26686478454680534, + "pass@2": 0.31857355126300146, + "pass@4": 0.36909361069836555, + "pass@8": 0.41931649331352155, + "pass@16": 0.45973254086181276, + "pass@32": 0.49271916790490344, + "pass@64": 0.5265973254086181, + "pass@128": 0.5539375928677563 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5714285714285714, + "pass@2": 0.6571428571428571, + "pass@4": 0.7714285714285715, + "pass@8": 0.8380952380952381, + "pass@16": 0.9047619047619048, + "pass@32": 0.9492063492063492, + "pass@64": 0.9714285714285714, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.5840978593272171, + "pass@2": 0.6483180428134556, + "pass@4": 0.7247706422018348, + "pass@8": 0.7920489296636085, + "pass@16": 0.8440366972477065, + "pass@32": 0.8715596330275229, + "pass@64": 0.8837920489296636, + "pass@128": 0.8960244648318043 + }, + "13": { + "pass@1": 0.4845679012345679, + "pass@2": 0.5802469135802469, + "pass@4": 0.6327160493827161, + "pass@8": 0.6851851851851852, + "pass@16": 0.7253086419753086, + "pass@32": 0.7685185185185185, + "pass@64": 0.7839506172839507, + "pass@128": 0.808641975308642 + }, + "14": { + "pass@1": 0.35311572700296734, + "pass@2": 0.41839762611275966, + "pass@4": 0.47477744807121663, + "pass@8": 0.5311572700296736, + "pass@16": 0.6112759643916914, + "pass@32": 0.6973293768545994, + "pass@64": 0.7566765578635015, + "pass@128": 0.8100890207715133 + }, + "15": { + "pass@1": 0.28703703703703703, + "pass@2": 0.31790123456790126, + "pass@4": 0.3765432098765432, + "pass@8": 0.4444444444444444, + "pass@16": 0.5339506172839507, + "pass@32": 0.6265432098765432, + "pass@64": 0.7067901234567902, + "pass@128": 0.7314814814814815 + }, + "16": { + "pass@1": 0.1415929203539823, + "pass@2": 0.19174041297935104, + "pass@4": 0.25073746312684364, + "pass@8": 0.29793510324483774, + "pass@16": 0.35398230088495575, + "pass@32": 0.41887905604719766, + "pass@64": 0.4778761061946903, + "pass@128": 0.5162241887905604 + }, + "17": { + "pass@1": 0.06325301204819277, + "pass@2": 0.1144578313253012, + "pass@4": 0.14457831325301204, + "pass@8": 0.20481927710843373, + "pass@16": 0.25, + "pass@32": 0.2921686746987952, + "pass@64": 0.34036144578313254, + "pass@128": 0.3885542168674699 + }, + "18": { + "pass@1": 0.08709677419354839, + "pass@2": 0.10967741935483871, + "pass@4": 0.13548387096774195, + "pass@8": 0.16129032258064516, + "pass@16": 0.18064516129032257, + "pass@32": 0.2032258064516129, + "pass@64": 0.23548387096774193, + "pass@128": 0.2645161290322581 + }, + "19": { + "pass@1": 0.00906344410876133, + "pass@2": 0.030211480362537766, + "pass@4": 0.03625377643504532, + "pass@8": 0.054380664652567974, + "pass@16": 0.07250755287009064, + "pass@32": 0.08459214501510574, + "pass@64": 0.1027190332326284, + "pass@128": 0.10876132930513595 + }, + "20": { + "pass@1": 0.009009009009009009, + "pass@2": 0.015015015015015015, + "pass@4": 0.018018018018018018, + "pass@8": 0.02702702702702703, + "pass@16": 0.04504504504504504, + "pass@32": 0.06306306306306306, + "pass@64": 0.07507507507507508, + "pass@128": 0.1021021021021021 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5714285714285714, + "12": 0.5840978593272171, + "13": 0.4845679012345679, + "14": 0.35311572700296734, + "15": 0.28703703703703703, + "16": 0.1415929203539823, + "17": 0.06325301204819277, + "18": 0.08709677419354839, + "19": 0.00906344410876133, + "20": 0.009009009009009009 + }, + "pass@2": { + "11": 0.6571428571428571, + "12": 0.6483180428134556, + "13": 0.5802469135802469, + "14": 0.41839762611275966, + "15": 0.31790123456790126, + "16": 0.19174041297935104, + "17": 0.1144578313253012, + "18": 0.10967741935483871, + "19": 0.030211480362537766, + "20": 0.015015015015015015 + }, + "pass@4": { + "11": 0.7714285714285715, + "12": 0.7247706422018348, + "13": 0.6327160493827161, + "14": 0.47477744807121663, + "15": 0.3765432098765432, + "16": 0.25073746312684364, + "17": 0.14457831325301204, + "18": 0.13548387096774195, + "19": 0.03625377643504532, + "20": 0.018018018018018018 + }, + "pass@8": { + "11": 0.8380952380952381, + "12": 0.7920489296636085, + "13": 0.6851851851851852, + "14": 0.5311572700296736, + "15": 0.4444444444444444, + "16": 0.29793510324483774, + "17": 0.20481927710843373, + "18": 0.16129032258064516, + "19": 0.054380664652567974, + "20": 0.02702702702702703 + }, + "pass@16": { + "11": 0.9047619047619048, + "12": 0.8440366972477065, + "13": 0.7253086419753086, + "14": 0.6112759643916914, + "15": 0.5339506172839507, + "16": 0.35398230088495575, + "17": 0.25, + "18": 0.18064516129032257, + "19": 0.07250755287009064, + "20": 0.04504504504504504 + }, + "pass@32": { + "11": 0.9492063492063492, + "12": 0.8715596330275229, + "13": 0.7685185185185185, + "14": 0.6973293768545994, + "15": 0.6265432098765432, + "16": 0.41887905604719766, + "17": 0.2921686746987952, + "18": 0.2032258064516129, + "19": 0.08459214501510574, + "20": 0.06306306306306306 + }, + "pass@64": { + "11": 0.9714285714285714, + "12": 0.8837920489296636, + "13": 0.7839506172839507, + "14": 0.7566765578635015, + "15": 0.7067901234567902, + "16": 0.4778761061946903, + "17": 0.34036144578313254, + "18": 0.23548387096774193, + "19": 0.1027190332326284, + "20": 0.07507507507507508 + }, + "pass@128": { + "11": 0.9746031746031746, + "12": 0.8960244648318043, + "13": 0.808641975308642, + "14": 0.8100890207715133, + "15": 0.7314814814814815, + "16": 0.5162241887905604, + "17": 0.3885542168674699, + "18": 0.2645161290322581, + "19": 0.10876132930513595, + "20": 0.1021021021021021 + } + }, + "total_pass_at_k": { + "pass@1": 0.25733496332518335, + "pass@2": 0.3065403422982885, + "pass@4": 0.3545232273838631, + "pass@8": 0.4015892420537897, + "pass@16": 0.4501833740831296, + "pass@32": 0.495721271393643, + "pass@64": 0.5317848410757946, + "pass@128": 0.558679706601467 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6234939759036144, + "pass@2": 0.6716867469879518, + "pass@4": 0.7409638554216867, + "pass@8": 0.8253012048192772, + "pass@16": 0.9096385542168675, + "pass@32": 0.9457831325301205, + "pass@64": 0.9668674698795181, + "pass@128": 0.9728915662650602 + }, + "12": { + "pass@1": 0.5454545454545454, + "pass@2": 0.6050156739811913, + "pass@4": 0.6739811912225705, + "pass@8": 0.7711598746081505, + "pass@16": 0.8119122257053292, + "pass@32": 0.8307210031347962, + "pass@64": 0.8589341692789969, + "pass@128": 0.8808777429467085 + }, + "13": { + "pass@1": 0.49859943977591037, + "pass@2": 0.5826330532212886, + "pass@4": 0.6554621848739496, + "pass@8": 0.7310924369747899, + "pass@16": 0.773109243697479, + "pass@32": 0.8123249299719888, + "pass@64": 0.8375350140056023, + "pass@128": 0.8543417366946778 + }, + "14": { + "pass@1": 0.38011695906432746, + "pass@2": 0.4473684210526316, + "pass@4": 0.52046783625731, + "pass@8": 0.5964912280701754, + "pass@16": 0.6637426900584795, + "pass@32": 0.716374269005848, + "pass@64": 0.7894736842105263, + "pass@128": 0.8187134502923976 + }, + "15": { + "pass@1": 0.32727272727272727, + "pass@2": 0.3878787878787879, + "pass@4": 0.41515151515151516, + "pass@8": 0.46060606060606063, + "pass@16": 0.5121212121212121, + "pass@32": 0.5757575757575758, + "pass@64": 0.6727272727272727, + "pass@128": 0.7212121212121212 + }, + "16": { + "pass@1": 0.17094017094017094, + "pass@2": 0.24216524216524216, + "pass@4": 0.2962962962962963, + "pass@8": 0.3247863247863248, + "pass@16": 0.3789173789173789, + "pass@32": 0.45584045584045585, + "pass@64": 0.5213675213675214, + "pass@128": 0.5612535612535613 + }, + "17": { + "pass@1": 0.08695652173913043, + "pass@2": 0.12732919254658384, + "pass@4": 0.15527950310559005, + "pass@8": 0.17701863354037267, + "pass@16": 0.2360248447204969, + "pass@32": 0.2732919254658385, + "pass@64": 0.3136645962732919, + "pass@128": 0.35093167701863354 + }, + "18": { + "pass@1": 0.05588235294117647, + "pass@2": 0.08529411764705883, + "pass@4": 0.10882352941176471, + "pass@8": 0.1323529411764706, + "pass@16": 0.17058823529411765, + "pass@32": 0.20294117647058824, + "pass@64": 0.2088235294117647, + "pass@128": 0.23823529411764705 + }, + "19": { + "pass@1": 0.017857142857142856, + "pass@2": 0.026785714285714284, + "pass@4": 0.03869047619047619, + "pass@8": 0.05357142857142857, + "pass@16": 0.0625, + "pass@32": 0.0625, + "pass@64": 0.0744047619047619, + "pass@128": 0.08928571428571429 + }, + "20": { + "pass@1": 0.02694610778443114, + "pass@2": 0.03293413173652695, + "pass@4": 0.04491017964071856, + "pass@8": 0.05389221556886228, + "pass@16": 0.08083832335329341, + "pass@32": 0.08682634730538923, + "pass@64": 0.09880239520958084, + "pass@128": 0.10778443113772455 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6234939759036144, + "12": 0.5454545454545454, + "13": 0.49859943977591037, + "14": 0.38011695906432746, + "15": 0.32727272727272727, + "16": 0.17094017094017094, + "17": 0.08695652173913043, + "18": 0.05588235294117647, + "19": 0.017857142857142856, + "20": 0.02694610778443114 + }, + "pass@2": { + "11": 0.6716867469879518, + "12": 0.6050156739811913, + "13": 0.5826330532212886, + "14": 0.4473684210526316, + "15": 0.3878787878787879, + "16": 0.24216524216524216, + "17": 0.12732919254658384, + "18": 0.08529411764705883, + "19": 0.026785714285714284, + "20": 0.03293413173652695 + }, + "pass@4": { + "11": 0.7409638554216867, + "12": 0.6739811912225705, + "13": 0.6554621848739496, + "14": 0.52046783625731, + "15": 0.41515151515151516, + "16": 0.2962962962962963, + "17": 0.15527950310559005, + "18": 0.10882352941176471, + "19": 0.03869047619047619, + "20": 0.04491017964071856 + }, + "pass@8": { + "11": 0.8253012048192772, + "12": 0.7711598746081505, + "13": 0.7310924369747899, + "14": 0.5964912280701754, + "15": 0.46060606060606063, + "16": 0.3247863247863248, + "17": 0.17701863354037267, + "18": 0.1323529411764706, + "19": 0.05357142857142857, + "20": 0.05389221556886228 + }, + "pass@16": { + "11": 0.9096385542168675, + "12": 0.8119122257053292, + "13": 0.773109243697479, + "14": 0.6637426900584795, + "15": 0.5121212121212121, + "16": 0.3789173789173789, + "17": 0.2360248447204969, + "18": 0.17058823529411765, + "19": 0.0625, + "20": 0.08083832335329341 + }, + "pass@32": { + "11": 0.9457831325301205, + "12": 0.8307210031347962, + "13": 0.8123249299719888, + "14": 0.716374269005848, + "15": 0.5757575757575758, + "16": 0.45584045584045585, + "17": 0.2732919254658385, + "18": 0.20294117647058824, + "19": 0.0625, + "20": 0.08682634730538923 + }, + "pass@64": { + "11": 0.9668674698795181, + "12": 0.8589341692789969, + "13": 0.8375350140056023, + "14": 0.7894736842105263, + "15": 0.6727272727272727, + "16": 0.5213675213675214, + "17": 0.3136645962732919, + "18": 0.2088235294117647, + "19": 0.0744047619047619, + "20": 0.09880239520958084 + }, + "pass@128": { + "11": 0.9728915662650602, + "12": 0.8808777429467085, + "13": 0.8543417366946778, + "14": 0.8187134502923976, + "15": 0.7212121212121212, + "16": 0.5612535612535613, + "17": 0.35093167701863354, + "18": 0.23823529411764705, + "19": 0.08928571428571429, + "20": 0.10778443113772455 + } + }, + "total_pass_at_k": { + "pass@1": 0.27326791555159086, + "pass@2": 0.32114183764495985, + "pass@4": 0.3654475170978293, + "pass@8": 0.4130240856378234, + "pass@16": 0.46030330062444247, + "pass@32": 0.49687778768956287, + "pass@64": 0.5349390425215581, + "pass@128": 0.56021409455843 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.585, + "pass@2": 0.666, + "pass@4": 0.755, + "pass@8": 0.83, + "pass@16": 0.904, + "pass@32": 0.94, + "pass@64": 0.959, + "pass@128": 0.964 + }, + "12": { + "pass@1": 0.575, + "pass@2": 0.635, + "pass@4": 0.708, + "pass@8": 0.786, + "pass@16": 0.832, + "pass@32": 0.851, + "pass@64": 0.869, + "pass@128": 0.884 + }, + "13": { + "pass@1": 0.49, + "pass@2": 0.575, + "pass@4": 0.64, + "pass@8": 0.706, + "pass@16": 0.749, + "pass@32": 0.791, + "pass@64": 0.814, + "pass@128": 0.832 + }, + "14": { + "pass@1": 0.375, + "pass@2": 0.436, + "pass@4": 0.507, + "pass@8": 0.573, + "pass@16": 0.64, + "pass@32": 0.702, + "pass@64": 0.775, + "pass@128": 0.814 + }, + "15": { + "pass@1": 0.302, + "pass@2": 0.357, + "pass@4": 0.409, + "pass@8": 0.472, + "pass@16": 0.539, + "pass@32": 0.613, + "pass@64": 0.692, + "pass@128": 0.738 + }, + "16": { + "pass@1": 0.159, + "pass@2": 0.217, + "pass@4": 0.273, + "pass@8": 0.316, + "pass@16": 0.37, + "pass@32": 0.433, + "pass@64": 0.491, + "pass@128": 0.527 + }, + "17": { + "pass@1": 0.083, + "pass@2": 0.129, + "pass@4": 0.163, + "pass@8": 0.202, + "pass@16": 0.247, + "pass@32": 0.287, + "pass@64": 0.326, + "pass@128": 0.374 + }, + "18": { + "pass@1": 0.06, + "pass@2": 0.087, + "pass@4": 0.108, + "pass@8": 0.138, + "pass@16": 0.164, + "pass@32": 0.191, + "pass@64": 0.214, + "pass@128": 0.245 + }, + "19": { + "pass@1": 0.013, + "pass@2": 0.026, + "pass@4": 0.035, + "pass@8": 0.05, + "pass@16": 0.063, + "pass@32": 0.071, + "pass@64": 0.084, + "pass@128": 0.095 + }, + "20": { + "pass@1": 0.017, + "pass@2": 0.027, + "pass@4": 0.033, + "pass@8": 0.041, + "pass@16": 0.06, + "pass@32": 0.072, + "pass@64": 0.087, + "pass@128": 0.103 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.585, + "12": 0.575, + "13": 0.49, + "14": 0.375, + "15": 0.302, + "16": 0.159, + "17": 0.083, + "18": 0.06, + "19": 0.013, + "20": 0.017 + }, + "pass@2": { + "11": 0.666, + "12": 0.635, + "13": 0.575, + "14": 0.436, + "15": 0.357, + "16": 0.217, + "17": 0.129, + "18": 0.087, + "19": 0.026, + "20": 0.027 + }, + "pass@4": { + "11": 0.755, + "12": 0.708, + "13": 0.64, + "14": 0.507, + "15": 0.409, + "16": 0.273, + "17": 0.163, + "18": 0.108, + "19": 0.035, + "20": 0.033 + }, + "pass@8": { + "11": 0.83, + "12": 0.786, + "13": 0.706, + "14": 0.573, + "15": 0.472, + "16": 0.316, + "17": 0.202, + "18": 0.138, + "19": 0.05, + "20": 0.041 + }, + "pass@16": { + "11": 0.904, + "12": 0.832, + "13": 0.749, + "14": 0.64, + "15": 0.539, + "16": 0.37, + "17": 0.247, + "18": 0.164, + "19": 0.063, + "20": 0.06 + }, + "pass@32": { + "11": 0.94, + "12": 0.851, + "13": 0.791, + "14": 0.702, + "15": 0.613, + "16": 0.433, + "17": 0.287, + "18": 0.191, + "19": 0.071, + "20": 0.072 + }, + "pass@64": { + "11": 0.959, + "12": 0.869, + "13": 0.814, + "14": 0.775, + "15": 0.692, + "16": 0.491, + "17": 0.326, + "18": 0.214, + "19": 0.084, + "20": 0.087 + }, + "pass@128": { + "11": 0.964, + "12": 0.884, + "13": 0.832, + "14": 0.814, + "15": 0.738, + "16": 0.527, + "17": 0.374, + "18": 0.245, + "19": 0.095, + "20": 0.103 + } + }, + "total_pass_at_k": { + "pass@1": 0.2659, + "pass@2": 0.3155, + "pass@4": 0.3631, + "pass@8": 0.4114, + "pass@16": 0.4568, + "pass@32": 0.4951, + "pass@64": 0.5311, + "pass@128": 0.5576 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block16-epoch4-op9-12/summary.csv b/composition/SDAR-100M-block16-epoch4-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..568999b6c4c496e0fe924b601b7620b1af7538d9 --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block16-epoch4-op9-12,0.9921467013888889,0.5692328125,0.7695604440789474,0.0,0.0,0.0,199.46203125,350.66635078125,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block16-epoch4-op9-12/summary.json b/composition/SDAR-100M-block16-epoch4-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3e8fd65e30ad6b8543b7399dff2dff84174c2ace --- /dev/null +++ b/composition/SDAR-100M-block16-epoch4-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block16-epoch4-op9-12", + "id_acc": 0.9921467013888889, + "ood_acc": 0.5692328125, + "total_acc": 0.7695604440789474, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.46203125, + "ood_avg_resp_len": 350.66635078125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 621973, + "answer_accuracy": 0.7681258397881758, + "avg_response_len": 265.83570408828643, + "resp_tokens_sum": 215254613, + "pass_at_k": { + "pass@1": 0.7681258397881758, + "pass@2": 0.8056889833794223, + "pass@4": 0.8361066637487128, + "pass@8": 0.8603037310757391, + "pass@16": 0.8792181006261285, + "pass@32": 0.8943473727278064, + "pass@64": 0.9074203405159689, + "pass@128": 0.9190641795763516 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9574575958702065, + "pass@2": 0.9697600196269713, + "pass@4": 0.9770701256369501, + "pass@8": 0.9807352442070735, + "pass@16": 0.9835831385400656, + "pass@32": 0.9868139186902647, + "pass@64": 0.9902929813385202, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996923828125, + "pass@2": 0.9976178026574803, + "pass@4": 0.9982453347628422, + "pass@8": 0.9990349229744625, + "pass@16": 0.9997269200711651, + "pass@32": 0.9999841633446103, + "pass@64": 0.9999999929906153, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9954861111111111, + "pass@2": 0.9981666354205724, + "pass@4": 0.9995852298224628, + "pass@8": 0.9999710725449563, + "pass@16": 0.9999998592866416, + "pass@32": 0.9999999999993872, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9968890344311377, + "pass@2": 0.997264575180348, + "pass@4": 0.997534851556729, + "pass@8": 0.9979835642250304, + "pass@16": 0.9986793120796699, + "pass@32": 0.9994882388094946, + "pass@64": 0.9999586116800393, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9973187869822485, + "pass@2": 0.9984067785258351, + "pass@4": 0.9992237471795316, + "pass@8": 0.9998073811974663, + "pass@16": 0.9999900064095892, + "pass@32": 0.9999999886267649, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9983974358974359, + "pass@2": 0.999743290051484, + "pass@4": 0.9999842372467864, + "pass@8": 0.9999999472370541, + "pass@16": 0.9999999999998607, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9972687007874016, + "pass@4": 0.997876473539399, + "pass@8": 0.9984546377928358, + "pass@16": 0.999188066655105, + "pass@32": 0.9998084093975909, + "pass@64": 0.9999959303192606, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9572193696883853, + "pass@2": 0.9719146628449066, + "pass@4": 0.981724504514839, + "pass@8": 0.9890359200866053, + "pass@16": 0.9932697779011666, + "pass@32": 0.9957361168730702, + "pass@64": 0.9978415915007174, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.8983271539548022, + "pass@2": 0.920239098158281, + "pass@4": 0.9334048053315367, + "pass@8": 0.9434168617466765, + "pass@16": 0.9517176538857137, + "pass@32": 0.9580437536392892, + "pass@64": 0.961533417356335, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8390722962382445, + "pass@2": 0.8873212765038385, + "pass@4": 0.9195430402704361, + "pass@8": 0.9430086769866958, + "pass@16": 0.9574399410309512, + "pass@32": 0.9647881464197946, + "pass@64": 0.9690691600729979, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.7840975467289719, + "pass@2": 0.8466142192214293, + "pass@4": 0.8872996447873914, + "pass@8": 0.9131255212050312, + "pass@16": 0.9296828208789675, + "pass@32": 0.9427319318749825, + "pass@64": 0.9554954405969336, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.693099710982659, + "pass@2": 0.7778968583587461, + "pass@4": 0.833728548292012, + "pass@8": 0.8711126510602295, + "pass@16": 0.8958576584940476, + "pass@32": 0.9133306283181201, + "pass@64": 0.9284685661529859, + "pass@128": 0.9421965317919075 + }, + "16": { + "pass@1": 0.4913054435483871, + "pass@2": 0.5962447612395223, + "pass@4": 0.6747601975156323, + "pass@8": 0.7348407821560055, + "pass@16": 0.7834148610294204, + "pass@32": 0.8244895329338998, + "pass@64": 0.8601964216978978, + "pass@128": 0.8903225806451613 + }, + "17": { + "pass@1": 0.35248825867052025, + "pass@2": 0.4572849580128351, + "pass@4": 0.5492719623919837, + "pass@8": 0.6229445136286828, + "pass@16": 0.6790291809157839, + "pass@32": 0.7227658365318381, + "pass@64": 0.7602209527875393, + "pass@128": 0.7976878612716763 + }, + "18": { + "pass@1": 0.27520089285714283, + "pass@2": 0.3652481721034869, + "pass@4": 0.4477995007766887, + "pass@8": 0.5185924991538704, + "pass@16": 0.5777052275710203, + "pass@32": 0.6248875096332369, + "pass@64": 0.6638893307765378, + "pass@128": 0.6971428571428572 + }, + "19": { + "pass@1": 0.2045795795795796, + "pass@2": 0.28665747440353734, + "pass@4": 0.368495601338121, + "pass@8": 0.44158292317157055, + "pass@16": 0.5021246498473418, + "pass@32": 0.5528371750040144, + "pass@64": 0.5987933372999518, + "pass@128": 0.6396396396396397 + }, + "20": { + "pass@1": 0.19566441441441443, + "pass@2": 0.2724017078101725, + "pass@4": 0.34911851418347484, + "pass@8": 0.41767593951939913, + "pass@16": 0.47676148728079454, + "pass@32": 0.5280550430504534, + "pass@64": 0.5749912993919669, + "pass@128": 0.6156156156156156 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 635595, + "answer_accuracy": 0.774058602883866, + "avg_response_len": 275.4775172934528, + "resp_tokens_sum": 226200099, + "pass_at_k": { + "pass@1": 0.774058602883866, + "pass@2": 0.8115189700566369, + "pass@4": 0.8419047259145173, + "pass@8": 0.8661518210454625, + "pass@16": 0.8851545001914215, + "pass@32": 0.9003565916378781, + "pass@64": 0.9130658000646252, + "pass@128": 0.9247077162899454 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9421731211656442, + "pass@2": 0.9570635174629243, + "pass@4": 0.9675180118397778, + "pass@8": 0.9744761195433342, + "pass@16": 0.9791587744555608, + "pass@32": 0.9822004591222739, + "pass@64": 0.984571936009979, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956295289855073, + "pass@2": 0.9968182985279016, + "pass@4": 0.9973139007080636, + "pass@8": 0.9976146824957167, + "pass@16": 0.998064779875966, + "pass@32": 0.9987868364280551, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9972242120343839, + "pass@2": 0.9987002374613632, + "pass@4": 0.9996714137452016, + "pass@8": 0.9999752561101265, + "pass@16": 0.9999998735735035, + "pass@32": 0.999999999999447, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9973807803468208, + "pass@2": 0.9977498748350099, + "pass@4": 0.9982556172530457, + "pass@8": 0.9989655495774096, + "pass@16": 0.9996567447729243, + "pass@32": 0.999971393889872, + "pass@64": 0.9999999651170723, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954665492957746, + "pass@2": 0.9968725740268383, + "pass@4": 0.9974011850631346, + "pass@8": 0.9976822657434412, + "pass@16": 0.9981192931743675, + "pass@32": 0.9988210100498002, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9930295658682635, + "pass@2": 0.9939206233202886, + "pass@4": 0.9940102609741148, + "pass@8": 0.9940119756217468, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9631024096385542, + "pass@2": 0.972274185323973, + "pass@4": 0.9774970881840069, + "pass@8": 0.9805179334532464, + "pass@16": 0.9822529509986989, + "pass@32": 0.9834096807350399, + "pass@64": 0.9849397431912068, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.8928781347962382, + "pass@2": 0.9105734579024021, + "pass@4": 0.920715678859108, + "pass@8": 0.928907449166582, + "pass@16": 0.9358474938634546, + "pass@32": 0.9417129174503823, + "pass@64": 0.9473011819327243, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.8309042366946778, + "pass@2": 0.8788773875692008, + "pass@4": 0.9060005905249233, + "pass@8": 0.9230680510501225, + "pass@16": 0.9352975407775181, + "pass@32": 0.9462738870529341, + "pass@64": 0.956931111957274, + "pass@128": 0.9663865546218487 + }, + "14": { + "pass@1": 0.7888797514619883, + "pass@2": 0.8581284966846247, + "pass@4": 0.9010667306937505, + "pass@8": 0.9274503748185816, + "pass@16": 0.9419725164541798, + "pass@32": 0.9505580533617753, + "pass@64": 0.9588922804756361, + "pass@128": 0.9707602339181286 + }, + "15": { + "pass@1": 0.6662642045454545, + "pass@2": 0.7540548198520639, + "pass@4": 0.8157365976980145, + "pass@8": 0.8565477326282641, + "pass@16": 0.8831288202183204, + "pass@32": 0.9034765538582886, + "pass@64": 0.9217986009661605, + "pass@128": 0.9393939393939394 + }, + "16": { + "pass@1": 0.5013799857549858, + "pass@2": 0.6062000162639922, + "pass@4": 0.688734711846916, + "pass@8": 0.7507240841046224, + "pass@16": 0.7988058883708389, + "pass@32": 0.8382289811645698, + "pass@64": 0.8684671985838386, + "pass@128": 0.8945868945868946 + }, + "17": { + "pass@1": 0.39540469720496896, + "pass@2": 0.5015440070670513, + "pass@4": 0.5931639159429605, + "pass@8": 0.6665612968773015, + "pass@16": 0.7219907477028704, + "pass@32": 0.7646644496183821, + "pass@64": 0.7996400240328937, + "pass@128": 0.8322981366459627 + }, + "18": { + "pass@1": 0.32417279411764705, + "pass@2": 0.42494535954145407, + "pass@4": 0.517787185425351, + "pass@8": 0.5978804895419353, + "pass@16": 0.6618267341101194, + "pass@32": 0.7122069429681127, + "pass@64": 0.7544866903270767, + "pass@128": 0.788235294117647 + }, + "19": { + "pass@1": 0.20305524553571427, + "pass@2": 0.2782470794666291, + "pass@4": 0.35577190797578867, + "pass@8": 0.42750031369822505, + "pass@16": 0.4901520073095929, + "pass@32": 0.5430748566756388, + "pass@64": 0.5864859314590565, + "pass@128": 0.625 + }, + "20": { + "pass@1": 0.20106661676646706, + "pass@2": 0.2747646930548353, + "pass@4": 0.3491225167774687, + "pass@8": 0.42036866918817706, + "pass@16": 0.4844573511671379, + "pass@32": 0.5371766445282311, + "pass@64": 0.5800651608494923, + "pass@128": 0.6197604790419161 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 614003, + "answer_accuracy": 0.7664001338073174, + "avg_response_len": 296.0467888740214, + "resp_tokens_sum": 237178477, + "pass_at_k": { + "pass@1": 0.7664001338073174, + "pass@2": 0.8041933002303396, + "pass@4": 0.8355785000832348, + "pass@8": 0.8609809020316768, + "pass@16": 0.8807921691495971, + "pass@32": 0.8960189279742334, + "pass@64": 0.9082624863913064, + "pass@128": 0.918517335037546 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9572761194029851, + "pass@2": 0.9666937213538606, + "pass@4": 0.9742225369216908, + "pass@8": 0.9805621470686411, + "pass@16": 0.9864603166226772, + "pass@32": 0.991472929033808, + "pass@64": 0.9949171827192708, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.9998089774140073, + "pass@4": 0.9999887842966998, + "pass@8": 0.9999999716046218, + "pass@16": 0.9999999999999635, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906479779411764, + "pass@2": 0.9942146248263086, + "pass@4": 0.996329596759964, + "pass@8": 0.9977398649093597, + "pass@16": 0.9989382018700852, + "pass@32": 0.9997259417849353, + "pass@64": 0.9999908945046169, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9988416798107256, + "pass@2": 0.9997768356143968, + "pass@4": 0.9999891550425282, + "pass@8": 0.9999999767890079, + "pass@16": 0.9999999999999797, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997774920886076, + "pass@2": 0.9992509094986542, + "pass@4": 0.9998849730413445, + "pass@8": 0.9999968489460502, + "pass@16": 0.9999999985879974, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9917417417417418, + "pass@2": 0.9956344139415007, + "pass@4": 0.998493621292834, + "pass@8": 0.999787773125201, + "pass@16": 0.9999957560892757, + "pass@32": 0.9999999991534195, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932174437299035, + "pass@2": 0.9964665037850975, + "pass@4": 0.9980255754807175, + "pass@8": 0.9991216467059164, + "pass@16": 0.9997943881339146, + "pass@32": 0.9999917764544182, + "pass@64": 0.9999999987229452, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9642857142857143, + "pass@2": 0.9765787089113863, + "pass@4": 0.9820486778438408, + "pass@8": 0.9842835869687955, + "pass@16": 0.9852484301067794, + "pass@32": 0.9863141673301701, + "pass@64": 0.9881014873140843, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9083763379204893, + "pass@2": 0.9310500644128201, + "pass@4": 0.9455652481513205, + "pass@8": 0.956169735463612, + "pass@16": 0.9645099947486562, + "pass@32": 0.9711630837875163, + "pass@64": 0.9769114489739431, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.8012393904320988, + "pass@2": 0.8506990011665209, + "pass@4": 0.8808248887870499, + "pass@8": 0.9035617488598224, + "pass@16": 0.9217366232932731, + "pass@32": 0.9348758028470283, + "pass@64": 0.9448526103401694, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.7654395400593472, + "pass@2": 0.8289044428608145, + "pass@4": 0.8680753477773732, + "pass@8": 0.8935025847812458, + "pass@16": 0.9102918510356471, + "pass@32": 0.9225928362575211, + "pass@64": 0.934536393956455, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6340663580246914, + "pass@2": 0.7118602362204722, + "pass@4": 0.7674899159017153, + "pass@8": 0.8127011283278409, + "pass@16": 0.849350961539353, + "pass@32": 0.8759008146277355, + "pass@64": 0.892354569021042, + "pass@128": 0.9012345679012346 + }, + "16": { + "pass@1": 0.4710084808259587, + "pass@2": 0.580898471070541, + "pass@4": 0.668739560762868, + "pass@8": 0.7335904176217118, + "pass@16": 0.7809455072412861, + "pass@32": 0.8166722390431593, + "pass@64": 0.8488481347262333, + "pass@128": 0.8849557522123894 + }, + "17": { + "pass@1": 0.36243411144578314, + "pass@2": 0.4642222927141631, + "pass@4": 0.5592205760612748, + "pass@8": 0.6385525287524542, + "pass@16": 0.6994129926552873, + "pass@32": 0.7462385573378134, + "pass@64": 0.7809706376082151, + "pass@128": 0.8042168674698795 + }, + "18": { + "pass@1": 0.31597782258064516, + "pass@2": 0.4188174688849377, + "pass@4": 0.5104581848639886, + "pass@8": 0.5790573511639918, + "pass@16": 0.6283327210915123, + "pass@32": 0.6653820450438427, + "pass@64": 0.6937567396642746, + "pass@128": 0.7161290322580646 + }, + "19": { + "pass@1": 0.2052256419939577, + "pass@2": 0.28584974248876, + "pass@4": 0.36753850377841757, + "pass@8": 0.44054975863490586, + "pass@16": 0.49795327570192016, + "pass@32": 0.5453148986286661, + "pass@64": 0.5923666332779112, + "pass@128": 0.6404833836858006 + }, + "20": { + "pass@1": 0.19200450450450451, + "pass@2": 0.26821087820103573, + "pass@4": 0.3499301157175173, + "pass@8": 0.4312063902239929, + "pass@16": 0.5043251579531292, + "pass@32": 0.5612592194970087, + "pass@64": 0.6019373120540266, + "pass@128": 0.6306306306306306 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_metrics.json b/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..dee706e673251e2ce0e53bbb20faa106bff1a6e7 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch1-op11-14", + "id": { + "answer_accuracy": 0.9827213541666666, + "avg_response_len": 199.76456336805555, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9827213541666666, + "pass@2": 0.9873874945319294, + "pass@4": 0.9899934680664909, + "pass@8": 0.9919332484253088, + "pass@16": 0.993608891907506, + "pass@32": 0.9950211980869957, + "pass@64": 0.9962416535993714, + "pass@128": 0.9974444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.92821875, + "pass@2": 0.9428015501968511, + "pass@4": 0.95275325712411, + "pass@8": 0.9602836900431464, + "pass@16": 0.9667643456431163, + "pass@32": 0.9724262544552759, + "pass@64": 0.9773916206622802, + "pass@128": 0.983 + }, + "2": { + "pass@1": 0.999953125, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99825, + "pass@2": 0.9988725393700788, + "pass@4": 0.9990494026996625, + "pass@8": 0.9991214535331311, + "pass@16": 0.9992352362176316, + "pass@32": 0.9994389763779528, + "pass@64": 0.999751968503937, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99721875, + "pass@2": 0.9981986958661418, + "pass@4": 0.99854321287964, + "pass@8": 0.9989217378407179, + "pass@16": 0.9993741635872916, + "pass@32": 0.9997590674152546, + "pass@64": 0.9999711690328693, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.980671875, + "pass@2": 0.9855286663385826, + "pass@4": 0.9891177830896137, + "pass@8": 0.9923601897966485, + "pass@16": 0.9951406369282718, + "pass@32": 0.9969241199883416, + "pass@64": 0.9980015759808923, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.9930234375, + "pass@2": 0.994091658464567, + "pass@4": 0.9947228139295088, + "pass@8": 0.9952883871401448, + "pass@16": 0.9959038226355749, + "pass@32": 0.9965991523224128, + "pass@64": 0.9973491337914664, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.9889375, + "pass@2": 0.9931058070866141, + "pass@4": 0.9949000196850394, + "pass@8": 0.9961006060233324, + "pass@16": 0.9972014482074887, + "pass@32": 0.998169276945154, + "pass@64": 0.99912682009508, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9813515625, + "pass@2": 0.9889160925196855, + "pass@4": 0.9925639879077615, + "pass@8": 0.9947117324737054, + "pass@16": 0.9964938152859911, + "pass@32": 0.9980639650979307, + "pass@64": 0.9992073394908063, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9768671875, + "pass@2": 0.9849725639763786, + "pass@4": 0.9882907352830895, + "pass@8": 0.9906114389769584, + "pass@16": 0.9923665586621832, + "pass@32": 0.9938099701806373, + "pass@64": 0.9953752548370095, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.92821875, + "2": 0.999953125, + "3": 0.99825, + "4": 0.99721875, + "5": 0.980671875, + "6": 0.9930234375, + "7": 0.9889375, + "8": 0.9813515625, + "9": 0.9768671875 + }, + "per_op_avg_response_len": { + "10": 279.5917890625, + "2": 115.366984375, + "3": 144.562125, + "4": 159.6835546875, + "5": 174.0367578125, + "6": 198.8411171875, + "7": 223.6661796875, + "8": 243.4614921875, + "9": 258.6710703125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.5917890625, + "2": 115.366984375, + "3": 144.562125, + "4": 159.6835546875, + "5": 174.0367578125, + "6": 198.8411171875, + "7": 223.6661796875, + "8": 243.4614921875, + "9": 258.6710703125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 372242, + "answer_accuracy": 0.9821481340763256, + "avg_response_len": 190.2342826536643, + "resp_tokens_sum": 72100315, + "pass_at_k": { + "pass@1": 0.9821481340763256, + "pass@2": 0.9872708730557621, + "pass@4": 0.9901135191497727, + "pass@8": 0.9920274304013039, + "pass@16": 0.9934861887733804, + "pass@32": 0.9946622220879497, + "pass@64": 0.9956885677389574, + "pass@128": 0.9966227625802094 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9259080014749262, + "pass@2": 0.9433224020393468, + "pass@4": 0.9555353711317055, + "pass@8": 0.9626092113862927, + "pass@16": 0.9672899140408969, + "pass@32": 0.971873515401767, + "pass@64": 0.9768429344531748, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 0.9999513239875389, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999044342507645, + "pass@2": 0.9999977425413568, + "pass@4": 0.9999999997133384, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9938720703125, + "pass@2": 0.9946638933316929, + "pass@4": 0.995454402184102, + "pass@8": 0.9966304348689696, + "pass@16": 0.9980442612102862, + "pass@32": 0.9992470856726705, + "pass@64": 0.9999099032277166, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9814980158730159, + "pass@2": 0.9859626921634798, + "pass@4": 0.9891130867570126, + "pass@8": 0.9915862161820935, + "pass@16": 0.9933890901170934, + "pass@32": 0.9943969337435874, + "pass@64": 0.9952380498015997, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.992374625748503, + "pass@2": 0.9937386545806025, + "pass@4": 0.9939952238380383, + "pass@8": 0.9940119126755125, + "pass@16": 0.9940119760476699, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9904539571005917, + "pass@2": 0.9942641114941992, + "pass@4": 0.995715497378656, + "pass@8": 0.9967252188259338, + "pass@16": 0.9975999557269617, + "pass@32": 0.9983364053352927, + "pass@64": 0.9992661788080938, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9777393830128205, + "pass@2": 0.9862255987532803, + "pass@4": 0.990327881490775, + "pass@8": 0.9937928780958446, + "pass@16": 0.996912117506564, + "pass@32": 0.9990931142481682, + "pass@64": 0.9999455717474616, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9797535211267606, + "pass@2": 0.9886772901186649, + "pass@4": 0.9919861029695232, + "pass@8": 0.9939178049019348, + "pass@16": 0.9951668490880456, + "pass@32": 0.996002421656276, + "pass@64": 0.9968393035287623, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 383958, + "answer_accuracy": 0.9828544806684142, + "avg_response_len": 197.82942793659896, + "resp_tokens_sum": 77283253, + "pass_at_k": { + "pass@1": 0.9828544806684142, + "pass@2": 0.9864802011202161, + "pass@4": 0.9885723154608951, + "pass@8": 0.9902874693103434, + "pass@16": 0.9919324015926441, + "pass@32": 0.9934557354166121, + "pass@64": 0.9949606718992721, + "pass@128": 0.9967234600262124 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9163391487730062, + "pass@2": 0.9258629564996861, + "pass@4": 0.9336692571212339, + "pass@8": 0.9416640558086454, + "pass@16": 0.9500473404479759, + "pass@32": 0.9578994674031669, + "pass@64": 0.9655855093279391, + "pass@128": 0.9754601226993865 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982717803030303, + "pass@2": 0.9995746092817943, + "pass@4": 0.9999632247673588, + "pass@8": 0.9999996922182559, + "pass@16": 0.9999999999913911, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9986131656804734, + "pass@2": 0.999854036597866, + "pass@4": 0.9999976565429319, + "pass@8": 0.9999999993354288, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9863224637681159, + "pass@2": 0.9906003937007875, + "pass@4": 0.9930274229308294, + "pass@8": 0.9950581299215392, + "pass@16": 0.9969160592890997, + "pass@32": 0.998278697645461, + "pass@64": 0.9992808085046374, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967765042979942, + "pass@2": 0.9973253869322924, + "pass@4": 0.9975613690301607, + "pass@8": 0.9979359856831456, + "pass@16": 0.9985470770655008, + "pass@32": 0.9993380089666443, + "pass@64": 0.9999174243620693, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9916230130057804, + "pass@2": 0.9936738342587959, + "pass@4": 0.9944413127911034, + "pass@8": 0.9949076292657711, + "pass@16": 0.9955414667011108, + "pass@32": 0.9966227126233496, + "pass@64": 0.9982021755951027, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9815140845070423, + "pass@2": 0.9889029056227125, + "pass@4": 0.9929117410499744, + "pass@8": 0.9949431586627508, + "pass@16": 0.9962341274090126, + "pass@32": 0.9974052544005088, + "pass@64": 0.9985526092972625, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9739427395209581, + "pass@2": 0.9802412890753925, + "pass@4": 0.9834003469439074, + "pass@8": 0.9861000031633753, + "pass@16": 0.9883617473597814, + "pass@32": 0.9900186779076805, + "pass@64": 0.9917721721639942, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 375895, + "answer_accuracy": 0.9831535612654838, + "avg_response_len": 211.18913468781386, + "resp_tokens_sum": 80745209, + "pass_at_k": { + "pass@1": 0.9831535612654838, + "pass@2": 0.9884301378809491, + "pass@4": 0.9913265402106796, + "pass@8": 0.9935214791008893, + "pass@16": 0.995443499346777, + "pass@32": 0.9969765780679748, + "pass@64": 0.9980987822841344, + "pass@128": 0.9989956478071644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9421175373134328, + "pass@2": 0.9587580062286991, + "pass@4": 0.9685092193699665, + "pass@8": 0.9760498065360441, + "pass@16": 0.9825003934245147, + "pass@32": 0.9871221085392369, + "pass@64": 0.9894357905723764, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9999330357142857, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9966517857142857, + "pass@2": 0.9971243629622828, + "pass@4": 0.9972639609917566, + "pass@8": 0.99743893615483, + "pass@16": 0.9977703679897155, + "pass@32": 0.9983643626179382, + "pass@64": 0.9992768761047727, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9989720394736842, + "pass@2": 0.9998701334208223, + "pass@4": 0.9999958955240242, + "pass@8": 0.9999999968048913, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.974172794117647, + "pass@2": 0.9799802425891616, + "pass@4": 0.9851549995588786, + "pass@8": 0.9903396378716414, + "pass@16": 0.9949618620195534, + "pass@32": 0.9978909857983171, + "pass@64": 0.9992638569390833, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.98957511829653, + "pass@2": 0.9909034315308379, + "pass@4": 0.9923643261075016, + "pass@8": 0.9937184520602704, + "pass@16": 0.9949870433431968, + "pass@32": 0.9963097387793499, + "pass@64": 0.9980377056438621, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.984375, + "pass@2": 0.9912449074304795, + "pass@4": 0.994530023244721, + "pass@8": 0.9967387415639558, + "pass@16": 0.9985927711809854, + "pass@32": 0.9996839030827411, + "pass@64": 0.9999901863988568, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9845626876876877, + "pass@2": 0.9914509736114068, + "pass@4": 0.9942883507804767, + "pass@8": 0.9953259254730494, + "pass@16": 0.9963787369181493, + "pass@32": 0.9978019463673319, + "pass@64": 0.9992136480632735, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9767132234726688, + "pass@2": 0.9858248727751473, + "pass@4": 0.9893246072335813, + "pass@8": 0.9916823703543539, + "pass@16": 0.9934710727259167, + "pass@32": 0.995379009876187, + "pass@64": 0.9975736224486328, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.615015625, + "avg_response_len": 367.26526484375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.615015625, + "pass@2": 0.688048412893687, + "pass@4": 0.7440958128702582, + "pass@8": 0.7877747365946025, + "pass@16": 0.8222755379094328, + "pass@32": 0.8496575803035179, + "pass@64": 0.8714758455012228, + "pass@128": 0.8895 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9379609375, + "pass@2": 0.9578695866141734, + "pass@4": 0.9690707025684286, + "pass@8": 0.9761228584925753, + "pass@16": 0.9809705725972232, + "pass@32": 0.9846269650875722, + "pass@64": 0.9872749236211029, + "pass@128": 0.989 + }, + "12": { + "pass@1": 0.89421875, + "pass@2": 0.9220222687007882, + "pass@4": 0.9362151971316084, + "pass@8": 0.9447333294059417, + "pass@16": 0.9509831248030044, + "pass@32": 0.9562493532460535, + "pass@64": 0.961528048501336, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.8007421875, + "pass@2": 0.850632997047245, + "pass@4": 0.8801703857330323, + "pass@8": 0.900917953033891, + "pass@16": 0.9170591607756168, + "pass@32": 0.9292019660423374, + "pass@64": 0.9391975690385227, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.777703125, + "pass@2": 0.8398607283464579, + "pass@4": 0.8762118897637787, + "pass@8": 0.9005872635175092, + "pass@16": 0.9194266016183247, + "pass@32": 0.9342833957716893, + "pass@64": 0.9452691557195909, + "pass@128": 0.954 + }, + "15": { + "pass@1": 0.69453125, + "pass@2": 0.768718626968505, + "pass@4": 0.8212702295650541, + "pass@8": 0.8608277460887901, + "pass@16": 0.8904122978547405, + "pass@32": 0.9119605369492704, + "pass@64": 0.927476651698037, + "pass@128": 0.938 + }, + "16": { + "pass@1": 0.5662421875, + "pass@2": 0.6677634104330711, + "pass@4": 0.7407699299775025, + "pass@8": 0.7935083278076046, + "pass@16": 0.8317948982183991, + "pass@32": 0.8604213110568941, + "pass@64": 0.8841963329743866, + "pass@128": 0.905 + }, + "17": { + "pass@1": 0.4808125, + "pass@2": 0.585482406496063, + "pass@4": 0.6681984115110617, + "pass@8": 0.7304857882018998, + "pass@16": 0.7780592856710695, + "pass@32": 0.8142376854501494, + "pass@64": 0.8410238580115026, + "pass@128": 0.864 + }, + "18": { + "pass@1": 0.44084375, + "pass@2": 0.5405799704724411, + "pass@4": 0.6218796379827516, + "pass@8": 0.6872290020729789, + "pass@16": 0.7397756567660226, + "pass@32": 0.7814495225157587, + "pass@64": 0.8142298574628336, + "pass@128": 0.84 + }, + "19": { + "pass@1": 0.2833359375, + "pass@2": 0.37453592519685047, + "pass@4": 0.4594816270153731, + "pass@8": 0.5354858260969473, + "pass@16": 0.6025362039793238, + "pass@32": 0.6610735591981041, + "pass@64": 0.710950493166852, + "pass@128": 0.753 + }, + "20": { + "pass@1": 0.273765625, + "pass@2": 0.37301820866141744, + "pass@4": 0.4676901174540681, + "pass@8": 0.5478492712279519, + "pass@16": 0.6117375768104771, + "pass@32": 0.6630715077173114, + "pass@64": 0.7036115648179885, + "pass@128": 0.736 + } + }, + "per_op_accuracy": { + "11": 0.9379609375, + "12": 0.89421875, + "13": 0.8007421875, + "14": 0.777703125, + "15": 0.69453125, + "16": 0.5662421875, + "17": 0.4808125, + "18": 0.44084375, + "19": 0.2833359375, + "20": 0.273765625 + }, + "per_op_avg_response_len": { + "11": 297.83946875, + "12": 308.6558984375, + "13": 327.6826796875, + "14": 353.76734375, + "15": 371.4339140625, + "16": 376.514875, + "17": 397.728140625, + "18": 409.4316171875, + "19": 405.357671875, + "20": 424.2410390625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.83946875, + "12": 308.6558984375, + "13": 327.6826796875, + "14": 353.76734375, + "15": 371.4339140625, + "16": 376.514875, + "17": 397.728140625, + "18": 409.4316171875, + "19": 405.357671875, + "20": 424.2410390625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 268071, + "answer_accuracy": 0.6223788075780089, + "avg_response_len": 348.7750650074294, + "resp_tokens_sum": 150224396, + "pass_at_k": { + "pass@1": 0.6223788075780089, + "pass@2": 0.6935501880754923, + "pass@4": 0.7469774392428321, + "pass@8": 0.7885196429188797, + "pass@16": 0.8221776320825549, + "pass@32": 0.8495236946886684, + "pass@64": 0.8715822180293861, + "pass@128": 0.8900445765230312 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9369688385269122, + "pass@2": 0.9556891436729048, + "pass@4": 0.9681532095138248, + "pass@8": 0.9773977476860249, + "pass@16": 0.9836760616393073, + "pass@32": 0.9879154034727532, + "pass@64": 0.9905512293835983, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.896340925141243, + "pass@2": 0.9266628019484855, + "pass@4": 0.9416773099337153, + "pass@8": 0.9499502893000226, + "pass@16": 0.9556055452848286, + "pass@32": 0.9601108862491841, + "pass@64": 0.9653730933928352, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8189410266457681, + "pass@2": 0.8701832590773324, + "pass@4": 0.9008475028003936, + "pass@8": 0.9223142658352539, + "pass@16": 0.9375126996972292, + "pass@32": 0.9461958750908102, + "pass@64": 0.9519937435789704, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.8048091900311527, + "pass@2": 0.8633829138764197, + "pass@4": 0.8936096031220395, + "pass@8": 0.9133137052944442, + "pass@16": 0.9308204065370922, + "pass@32": 0.9458697424997475, + "pass@64": 0.9578949685863664, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.7142792630057804, + "pass@2": 0.7852296777570432, + "pass@4": 0.8336160666874729, + "pass@8": 0.8703838255639478, + "pass@16": 0.8991492741726431, + "pass@32": 0.9217285219094363, + "pass@64": 0.9390414573776548, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5767389112903226, + "pass@2": 0.671137207899416, + "pass@4": 0.7352816832371761, + "pass@8": 0.7801929643943304, + "pass@16": 0.8153526018049102, + "pass@32": 0.8451755884363006, + "pass@64": 0.8721477928442585, + "pass@128": 0.896774193548387 + }, + "17": { + "pass@1": 0.49742593930635837, + "pass@2": 0.6042979950844295, + "pass@4": 0.6877961173386561, + "pass@8": 0.750236429513258, + "pass@16": 0.797521632533749, + "pass@32": 0.8336931159590578, + "pass@64": 0.861816202176084, + "pass@128": 0.8872832369942196 + }, + "18": { + "pass@1": 0.4068080357142857, + "pass@2": 0.5067449381327332, + "pass@4": 0.5904474240719905, + "pass@8": 0.6602663145937516, + "pass@16": 0.7184114317717843, + "pass@32": 0.7636522860956406, + "pass@64": 0.7975986551347283, + "pass@128": 0.8228571428571428 + }, + "19": { + "pass@1": 0.28638795045045046, + "pass@2": 0.37785669527795496, + "pass@4": 0.45953530245656216, + "pass@8": 0.5290022873626415, + "pass@16": 0.5902007338149695, + "pass@32": 0.6466124054432371, + "pass@64": 0.6966004194500958, + "pass@128": 0.7417417417417418 + }, + "20": { + "pass@1": 0.2728978978978979, + "pass@2": 0.36531228571090785, + "pass@4": 0.4511123266123264, + "pass@8": 0.5250073329043563, + "pass@16": 0.5869595546727128, + "pass@32": 0.638416275402605, + "pass@64": 0.6777993410094819, + "pass@128": 0.7057057057057057 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 265030, + "answer_accuracy": 0.615684470710675, + "avg_response_len": 363.1408782151353, + "resp_tokens_sum": 156319075, + "pass_at_k": { + "pass@1": 0.615684470710675, + "pass@2": 0.6895557930091535, + "pass@4": 0.7462455992343082, + "pass@8": 0.7906967253680187, + "pass@16": 0.8259765555967202, + "pass@32": 0.8537529719929998, + "pass@64": 0.875734930364443, + "pass@128": 0.8941421349985132 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.942371046686747, + "pass@2": 0.9614907711554884, + "pass@4": 0.9709818822534232, + "pass@8": 0.9762108984873451, + "pass@16": 0.9798418328465113, + "pass@32": 0.9832802016718227, + "pass@64": 0.9869836840656171, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8921189263322884, + "pass@2": 0.9154603923431991, + "pass@4": 0.9262470747191082, + "pass@8": 0.932674949530915, + "pass@16": 0.9378135612950852, + "pass@32": 0.942572203878569, + "pass@64": 0.947465209542479, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7801995798319328, + "pass@2": 0.8263980789166057, + "pass@4": 0.8551042666410398, + "pass@8": 0.8787522658307619, + "pass@16": 0.8997253159536589, + "pass@32": 0.9175379858563458, + "pass@64": 0.9326565965286922, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.7647112573099415, + "pass@2": 0.8316522626283556, + "pass@4": 0.8720206605205054, + "pass@8": 0.899213977549253, + "pass@16": 0.9185417582132726, + "pass@32": 0.9327907985067769, + "pass@64": 0.9426059507605182, + "pass@128": 0.9502923976608187 + }, + "15": { + "pass@1": 0.6902225378787878, + "pass@2": 0.7702934860415173, + "pass@4": 0.8277060339048522, + "pass@8": 0.870181513451221, + "pass@16": 0.9009560991102984, + "pass@32": 0.9213126526056608, + "pass@64": 0.9341966745085428, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.5663951210826211, + "pass@2": 0.6729072728088475, + "pass@4": 0.747877350267114, + "pass@8": 0.8014397473398795, + "pass@16": 0.8403655118855234, + "pass@32": 0.8670101699239859, + "pass@64": 0.8852311741120641, + "pass@128": 0.9002849002849003 + }, + "17": { + "pass@1": 0.499708850931677, + "pass@2": 0.6042562020100748, + "pass@4": 0.6857853701905272, + "pass@8": 0.7446535294212936, + "pass@16": 0.787459898111565, + "pass@32": 0.8184923783964188, + "pass@64": 0.840558376080876, + "pass@128": 0.860248447204969 + }, + "18": { + "pass@1": 0.47116268382352944, + "pass@2": 0.5732706837656322, + "pass@4": 0.6578277650036389, + "pass@8": 0.7272127940450731, + "pass@16": 0.7824553129641325, + "pass@32": 0.8258777832780535, + "pass@64": 0.858667717455225, + "pass@128": 0.8823529411764706 + }, + "19": { + "pass@1": 0.2761346726190476, + "pass@2": 0.365666449896888, + "pass@4": 0.45070808615440894, + "pass@8": 0.529863381029973, + "pass@16": 0.6031771625339937, + "pass@32": 0.6680454701301732, + "pass@64": 0.7240389650730391, + "pass@128": 0.7708333333333334 + }, + "20": { + "pass@1": 0.277156624251497, + "pass@2": 0.3758873738742987, + "pass@4": 0.46919662278367874, + "pass@8": 0.547041580122116, + "pass@16": 0.608975135388662, + "pass@32": 0.6595693283084222, + "pass@64": 0.7034790052169011, + "pass@128": 0.7455089820359282 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 254119, + "answer_accuracy": 0.6067557113386308, + "avg_response_len": 390.5201042940098, + "resp_tokens_sum": 163556068, + "pass_at_k": { + "pass@1": 0.6067557113386308, + "pass@2": 0.6808409578769081, + "pass@4": 0.7389227064259096, + "pass@8": 0.7840054034570462, + "pass@16": 0.818572277098011, + "pass@32": 0.8455859797051636, + "pass@64": 0.8669889121416895, + "pass@128": 0.8841687041564792 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9344246031746032, + "pass@2": 0.956496453568304, + "pass@4": 0.9680845546092448, + "pass@8": 0.9746013817828894, + "pass@16": 0.979128362963511, + "pass@32": 0.9823612720845869, + "pass@64": 0.9839103382187553, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8939698012232415, + "pass@2": 0.9233999133135886, + "pass@4": 0.940026338164151, + "pass@8": 0.9508489849950189, + "pass@16": 0.9588264089267363, + "pass@32": 0.9654115182769986, + "pass@64": 0.9710843168080171, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8054591049382716, + "pass@2": 0.8580877260134149, + "pass@4": 0.8874315097186924, + "pass@8": 0.9042751337372295, + "pass@16": 0.9160205919035335, + "pass@32": 0.9253222868446103, + "pass@64": 0.9338060490002721, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7650686201780416, + "pass@2": 0.8257855761816872, + "pass@4": 0.8638935942539817, + "pass@8": 0.8898587056265527, + "pass@16": 0.909471720507359, + "pass@32": 0.9247618852817582, + "pass@64": 0.9359455063598505, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6778308256172839, + "pass@2": 0.7494824323174885, + "pass@4": 0.801531108958602, + "pass@8": 0.8410958117430899, + "pass@16": 0.8703430132240998, + "pass@32": 0.8920039907677088, + "pass@64": 0.9082821137578678, + "pass@128": 0.9197530864197531 + }, + "16": { + "pass@1": 0.5564850663716814, + "pass@2": 0.659352280909577, + "pass@4": 0.7384296702956372, + "pass@8": 0.7974724410886839, + "pass@16": 0.8379566282804083, + "pass@32": 0.867540734508324, + "pass@64": 0.8941426993490582, + "pass@128": 0.9174041297935103 + }, + "17": { + "pass@1": 0.44517131024096385, + "pass@2": 0.5476650697277298, + "pass@4": 0.630717065393181, + "pass@8": 0.696161286495951, + "pass@16": 0.7486587759712936, + "pass@32": 0.7898353056767135, + "pass@64": 0.8198061896401653, + "pass@128": 0.8433734939759037 + }, + "18": { + "pass@1": 0.44601814516129035, + "pass@2": 0.5429264827279653, + "pass@4": 0.6179406434074768, + "pass@8": 0.6738175548059356, + "pass@16": 0.7170866104454663, + "pass@32": 0.7528154066701483, + "pass@64": 0.7842687233254886, + "pass@128": 0.8129032258064516 + }, + "19": { + "pass@1": 0.28757552870090636, + "pass@2": 0.3801985453291147, + "pass@4": 0.4683336989409553, + "pass@8": 0.5477159165532208, + "pass@16": 0.6142955679985431, + "pass@32": 0.6685448647183322, + "pass@64": 0.7121010913154943, + "pass@128": 0.7462235649546828 + }, + "20": { + "pass@1": 0.2712321696696697, + "pass@2": 0.377846350287295, + "pass@4": 0.48275687892617025, + "pass@8": 0.5715013261561981, + "pass@16": 0.6392863359899448, + "pass@32": 0.6912394364961897, + "pass@64": 0.7295567463044594, + "pass@128": 0.7567567567567568 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7891920230263157, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7891920230263157, + "pass@2": 0.8298406094591582, + "pass@4": 0.8605736495421483, + "pass@8": 0.8844814000933748, + "pass@16": 0.9034334424348239, + "pass@32": 0.9185140308325449, + "pass@64": 0.930575438810868, + "pass@128": 0.9406315789473684 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.92821875, + "pass@2": 0.9428015501968511, + "pass@4": 0.95275325712411, + "pass@8": 0.9602836900431464, + "pass@16": 0.9667643456431163, + "pass@32": 0.9724262544552759, + "pass@64": 0.9773916206622802, + "pass@128": 0.983 + }, + "2": { + "pass@1": 0.999953125, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99825, + "pass@2": 0.9988725393700788, + "pass@4": 0.9990494026996625, + "pass@8": 0.9991214535331311, + "pass@16": 0.9992352362176316, + "pass@32": 0.9994389763779528, + "pass@64": 0.999751968503937, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99721875, + "pass@2": 0.9981986958661418, + "pass@4": 0.99854321287964, + "pass@8": 0.9989217378407179, + "pass@16": 0.9993741635872916, + "pass@32": 0.9997590674152546, + "pass@64": 0.9999711690328693, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.980671875, + "pass@2": 0.9855286663385826, + "pass@4": 0.9891177830896137, + "pass@8": 0.9923601897966485, + "pass@16": 0.9951406369282718, + "pass@32": 0.9969241199883416, + "pass@64": 0.9980015759808923, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.9930234375, + "pass@2": 0.994091658464567, + "pass@4": 0.9947228139295088, + "pass@8": 0.9952883871401448, + "pass@16": 0.9959038226355749, + "pass@32": 0.9965991523224128, + "pass@64": 0.9973491337914664, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.9889375, + "pass@2": 0.9931058070866141, + "pass@4": 0.9949000196850394, + "pass@8": 0.9961006060233324, + "pass@16": 0.9972014482074887, + "pass@32": 0.998169276945154, + "pass@64": 0.99912682009508, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9813515625, + "pass@2": 0.9889160925196855, + "pass@4": 0.9925639879077615, + "pass@8": 0.9947117324737054, + "pass@16": 0.9964938152859911, + "pass@32": 0.9980639650979307, + "pass@64": 0.9992073394908063, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9768671875, + "pass@2": 0.9849725639763786, + "pass@4": 0.9882907352830895, + "pass@8": 0.9906114389769584, + "pass@16": 0.9923665586621832, + "pass@32": 0.9938099701806373, + "pass@64": 0.9953752548370095, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.9379609375, + "pass@2": 0.9578695866141734, + "pass@4": 0.9690707025684286, + "pass@8": 0.9761228584925753, + "pass@16": 0.9809705725972232, + "pass@32": 0.9846269650875722, + "pass@64": 0.9872749236211029, + "pass@128": 0.989 + }, + "12": { + "pass@1": 0.89421875, + "pass@2": 0.9220222687007882, + "pass@4": 0.9362151971316084, + "pass@8": 0.9447333294059417, + "pass@16": 0.9509831248030044, + "pass@32": 0.9562493532460535, + "pass@64": 0.961528048501336, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.8007421875, + "pass@2": 0.850632997047245, + "pass@4": 0.8801703857330323, + "pass@8": 0.900917953033891, + "pass@16": 0.9170591607756168, + "pass@32": 0.9292019660423374, + "pass@64": 0.9391975690385227, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.777703125, + "pass@2": 0.8398607283464579, + "pass@4": 0.8762118897637787, + "pass@8": 0.9005872635175092, + "pass@16": 0.9194266016183247, + "pass@32": 0.9342833957716893, + "pass@64": 0.9452691557195909, + "pass@128": 0.954 + }, + "15": { + "pass@1": 0.69453125, + "pass@2": 0.768718626968505, + "pass@4": 0.8212702295650541, + "pass@8": 0.8608277460887901, + "pass@16": 0.8904122978547405, + "pass@32": 0.9119605369492704, + "pass@64": 0.927476651698037, + "pass@128": 0.938 + }, + "16": { + "pass@1": 0.5662421875, + "pass@2": 0.6677634104330711, + "pass@4": 0.7407699299775025, + "pass@8": 0.7935083278076046, + "pass@16": 0.8317948982183991, + "pass@32": 0.8604213110568941, + "pass@64": 0.8841963329743866, + "pass@128": 0.905 + }, + "17": { + "pass@1": 0.4808125, + "pass@2": 0.585482406496063, + "pass@4": 0.6681984115110617, + "pass@8": 0.7304857882018998, + "pass@16": 0.7780592856710695, + "pass@32": 0.8142376854501494, + "pass@64": 0.8410238580115026, + "pass@128": 0.864 + }, + "18": { + "pass@1": 0.44084375, + "pass@2": 0.5405799704724411, + "pass@4": 0.6218796379827516, + "pass@8": 0.6872290020729789, + "pass@16": 0.7397756567660226, + "pass@32": 0.7814495225157587, + "pass@64": 0.8142298574628336, + "pass@128": 0.84 + }, + "19": { + "pass@1": 0.2833359375, + "pass@2": 0.37453592519685047, + "pass@4": 0.4594816270153731, + "pass@8": 0.5354858260969473, + "pass@16": 0.6025362039793238, + "pass@32": 0.6610735591981041, + "pass@64": 0.710950493166852, + "pass@128": 0.753 + }, + "20": { + "pass@1": 0.273765625, + "pass@2": 0.37301820866141744, + "pass@4": 0.4676901174540681, + "pass@8": 0.5478492712279519, + "pass@16": 0.6117375768104771, + "pass@32": 0.6630715077173114, + "pass@64": 0.7036115648179885, + "pass@128": 0.736 + } + }, + "per_op_accuracy": { + "10": 0.92821875, + "2": 0.999953125, + "3": 0.99825, + "4": 0.99721875, + "5": 0.980671875, + "6": 0.9930234375, + "7": 0.9889375, + "8": 0.9813515625, + "9": 0.9768671875, + "11": 0.9379609375, + "12": 0.89421875, + "13": 0.8007421875, + "14": 0.777703125, + "15": 0.69453125, + "16": 0.5662421875, + "17": 0.4808125, + "18": 0.44084375, + "19": 0.2833359375, + "20": 0.273765625 + }, + "per_op_avg_response_len": { + "10": 279.5917890625, + "2": 115.366984375, + "3": 144.562125, + "4": 159.6835546875, + "5": 174.0367578125, + "6": 198.8411171875, + "7": 223.6661796875, + "8": 243.4614921875, + "9": 258.6710703125, + "11": 297.83946875, + "12": 308.6558984375, + "13": 327.6826796875, + "14": 353.76734375, + "15": 371.4339140625, + "16": 376.514875, + "17": 397.728140625, + "18": 409.4316171875, + "19": 405.357671875, + "20": 424.2410390625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.5917890625, + "2": 115.366984375, + "3": 144.562125, + "4": 159.6835546875, + "5": 174.0367578125, + "6": 198.8411171875, + "7": 223.6661796875, + "8": 243.4614921875, + "9": 258.6710703125, + "11": 297.83946875, + "12": 308.6558984375, + "13": 327.6826796875, + "14": 353.76734375, + "15": 371.4339140625, + "16": 376.514875, + "17": 397.728140625, + "18": 409.4316171875, + "19": 405.357671875, + "20": 424.2410390625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 640313, + "answer_accuracy": 0.790775420882074, + "avg_response_len": 274.5671521794973, + "resp_tokens_sum": 222324711, + "pass_at_k": { + "pass@1": 0.790775420882074, + "pass@2": 0.8310315267138889, + "pass@4": 0.8607817283045475, + "pass@8": 0.8837751849257428, + "pass@16": 0.902361735206419, + "pass@32": 0.9174584369632929, + "pass@64": 0.9296724648662624, + "pass@128": 0.9399304457793234 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9259080014749262, + "pass@2": 0.9433224020393468, + "pass@4": 0.9555353711317055, + "pass@8": 0.9626092113862927, + "pass@16": 0.9672899140408969, + "pass@32": 0.971873515401767, + "pass@64": 0.9768429344531748, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 0.9999513239875389, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999044342507645, + "pass@2": 0.9999977425413568, + "pass@4": 0.9999999997133384, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9938720703125, + "pass@2": 0.9946638933316929, + "pass@4": 0.995454402184102, + "pass@8": 0.9966304348689696, + "pass@16": 0.9980442612102862, + "pass@32": 0.9992470856726705, + "pass@64": 0.9999099032277166, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9814980158730159, + "pass@2": 0.9859626921634798, + "pass@4": 0.9891130867570126, + "pass@8": 0.9915862161820935, + "pass@16": 0.9933890901170934, + "pass@32": 0.9943969337435874, + "pass@64": 0.9952380498015997, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.992374625748503, + "pass@2": 0.9937386545806025, + "pass@4": 0.9939952238380383, + "pass@8": 0.9940119126755125, + "pass@16": 0.9940119760476699, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9904539571005917, + "pass@2": 0.9942641114941992, + "pass@4": 0.995715497378656, + "pass@8": 0.9967252188259338, + "pass@16": 0.9975999557269617, + "pass@32": 0.9983364053352927, + "pass@64": 0.9992661788080938, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9777393830128205, + "pass@2": 0.9862255987532803, + "pass@4": 0.990327881490775, + "pass@8": 0.9937928780958446, + "pass@16": 0.996912117506564, + "pass@32": 0.9990931142481682, + "pass@64": 0.9999455717474616, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9797535211267606, + "pass@2": 0.9886772901186649, + "pass@4": 0.9919861029695232, + "pass@8": 0.9939178049019348, + "pass@16": 0.9951668490880456, + "pass@32": 0.996002421656276, + "pass@64": 0.9968393035287623, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9369688385269122, + "pass@2": 0.9556891436729048, + "pass@4": 0.9681532095138248, + "pass@8": 0.9773977476860249, + "pass@16": 0.9836760616393073, + "pass@32": 0.9879154034727532, + "pass@64": 0.9905512293835983, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.896340925141243, + "pass@2": 0.9266628019484855, + "pass@4": 0.9416773099337153, + "pass@8": 0.9499502893000226, + "pass@16": 0.9556055452848286, + "pass@32": 0.9601108862491841, + "pass@64": 0.9653730933928352, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8189410266457681, + "pass@2": 0.8701832590773324, + "pass@4": 0.9008475028003936, + "pass@8": 0.9223142658352539, + "pass@16": 0.9375126996972292, + "pass@32": 0.9461958750908102, + "pass@64": 0.9519937435789704, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.8048091900311527, + "pass@2": 0.8633829138764197, + "pass@4": 0.8936096031220395, + "pass@8": 0.9133137052944442, + "pass@16": 0.9308204065370922, + "pass@32": 0.9458697424997475, + "pass@64": 0.9578949685863664, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.7142792630057804, + "pass@2": 0.7852296777570432, + "pass@4": 0.8336160666874729, + "pass@8": 0.8703838255639478, + "pass@16": 0.8991492741726431, + "pass@32": 0.9217285219094363, + "pass@64": 0.9390414573776548, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5767389112903226, + "pass@2": 0.671137207899416, + "pass@4": 0.7352816832371761, + "pass@8": 0.7801929643943304, + "pass@16": 0.8153526018049102, + "pass@32": 0.8451755884363006, + "pass@64": 0.8721477928442585, + "pass@128": 0.896774193548387 + }, + "17": { + "pass@1": 0.49742593930635837, + "pass@2": 0.6042979950844295, + "pass@4": 0.6877961173386561, + "pass@8": 0.750236429513258, + "pass@16": 0.797521632533749, + "pass@32": 0.8336931159590578, + "pass@64": 0.861816202176084, + "pass@128": 0.8872832369942196 + }, + "18": { + "pass@1": 0.4068080357142857, + "pass@2": 0.5067449381327332, + "pass@4": 0.5904474240719905, + "pass@8": 0.6602663145937516, + "pass@16": 0.7184114317717843, + "pass@32": 0.7636522860956406, + "pass@64": 0.7975986551347283, + "pass@128": 0.8228571428571428 + }, + "19": { + "pass@1": 0.28638795045045046, + "pass@2": 0.37785669527795496, + "pass@4": 0.45953530245656216, + "pass@8": 0.5290022873626415, + "pass@16": 0.5902007338149695, + "pass@32": 0.6466124054432371, + "pass@64": 0.6966004194500958, + "pass@128": 0.7417417417417418 + }, + "20": { + "pass@1": 0.2728978978978979, + "pass@2": 0.36531228571090785, + "pass@4": 0.4511123266123264, + "pass@8": 0.5250073329043563, + "pass@16": 0.5869595546727128, + "pass@32": 0.638416275402605, + "pass@64": 0.6777993410094819, + "pass@128": 0.7057057057057057 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 648988, + "answer_accuracy": 0.7903692517537022, + "avg_response_len": 284.4923129384256, + "resp_tokens_sum": 233602328, + "pass_at_k": { + "pass@1": 0.7903692517537022, + "pass@2": 0.8308205308976789, + "pass@4": 0.8615349426362584, + "pass@8": 0.8856540052607581, + "pass@16": 0.9049316985397561, + "pass@32": 0.9202179500084126, + "pass@64": 0.9324577617228758, + "pass@128": 0.9429462197973499 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9163391487730062, + "pass@2": 0.9258629564996861, + "pass@4": 0.9336692571212339, + "pass@8": 0.9416640558086454, + "pass@16": 0.9500473404479759, + "pass@32": 0.9578994674031669, + "pass@64": 0.9655855093279391, + "pass@128": 0.9754601226993865 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982717803030303, + "pass@2": 0.9995746092817943, + "pass@4": 0.9999632247673588, + "pass@8": 0.9999996922182559, + "pass@16": 0.9999999999913911, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9986131656804734, + "pass@2": 0.999854036597866, + "pass@4": 0.9999976565429319, + "pass@8": 0.9999999993354288, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9863224637681159, + "pass@2": 0.9906003937007875, + "pass@4": 0.9930274229308294, + "pass@8": 0.9950581299215392, + "pass@16": 0.9969160592890997, + "pass@32": 0.998278697645461, + "pass@64": 0.9992808085046374, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967765042979942, + "pass@2": 0.9973253869322924, + "pass@4": 0.9975613690301607, + "pass@8": 0.9979359856831456, + "pass@16": 0.9985470770655008, + "pass@32": 0.9993380089666443, + "pass@64": 0.9999174243620693, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9916230130057804, + "pass@2": 0.9936738342587959, + "pass@4": 0.9944413127911034, + "pass@8": 0.9949076292657711, + "pass@16": 0.9955414667011108, + "pass@32": 0.9966227126233496, + "pass@64": 0.9982021755951027, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9815140845070423, + "pass@2": 0.9889029056227125, + "pass@4": 0.9929117410499744, + "pass@8": 0.9949431586627508, + "pass@16": 0.9962341274090126, + "pass@32": 0.9974052544005088, + "pass@64": 0.9985526092972625, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9739427395209581, + "pass@2": 0.9802412890753925, + "pass@4": 0.9834003469439074, + "pass@8": 0.9861000031633753, + "pass@16": 0.9883617473597814, + "pass@32": 0.9900186779076805, + "pass@64": 0.9917721721639942, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.942371046686747, + "pass@2": 0.9614907711554884, + "pass@4": 0.9709818822534232, + "pass@8": 0.9762108984873451, + "pass@16": 0.9798418328465113, + "pass@32": 0.9832802016718227, + "pass@64": 0.9869836840656171, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8921189263322884, + "pass@2": 0.9154603923431991, + "pass@4": 0.9262470747191082, + "pass@8": 0.932674949530915, + "pass@16": 0.9378135612950852, + "pass@32": 0.942572203878569, + "pass@64": 0.947465209542479, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7801995798319328, + "pass@2": 0.8263980789166057, + "pass@4": 0.8551042666410398, + "pass@8": 0.8787522658307619, + "pass@16": 0.8997253159536589, + "pass@32": 0.9175379858563458, + "pass@64": 0.9326565965286922, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.7647112573099415, + "pass@2": 0.8316522626283556, + "pass@4": 0.8720206605205054, + "pass@8": 0.899213977549253, + "pass@16": 0.9185417582132726, + "pass@32": 0.9327907985067769, + "pass@64": 0.9426059507605182, + "pass@128": 0.9502923976608187 + }, + "15": { + "pass@1": 0.6902225378787878, + "pass@2": 0.7702934860415173, + "pass@4": 0.8277060339048522, + "pass@8": 0.870181513451221, + "pass@16": 0.9009560991102984, + "pass@32": 0.9213126526056608, + "pass@64": 0.9341966745085428, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.5663951210826211, + "pass@2": 0.6729072728088475, + "pass@4": 0.747877350267114, + "pass@8": 0.8014397473398795, + "pass@16": 0.8403655118855234, + "pass@32": 0.8670101699239859, + "pass@64": 0.8852311741120641, + "pass@128": 0.9002849002849003 + }, + "17": { + "pass@1": 0.499708850931677, + "pass@2": 0.6042562020100748, + "pass@4": 0.6857853701905272, + "pass@8": 0.7446535294212936, + "pass@16": 0.787459898111565, + "pass@32": 0.8184923783964188, + "pass@64": 0.840558376080876, + "pass@128": 0.860248447204969 + }, + "18": { + "pass@1": 0.47116268382352944, + "pass@2": 0.5732706837656322, + "pass@4": 0.6578277650036389, + "pass@8": 0.7272127940450731, + "pass@16": 0.7824553129641325, + "pass@32": 0.8258777832780535, + "pass@64": 0.858667717455225, + "pass@128": 0.8823529411764706 + }, + "19": { + "pass@1": 0.2761346726190476, + "pass@2": 0.365666449896888, + "pass@4": 0.45070808615440894, + "pass@8": 0.529863381029973, + "pass@16": 0.6031771625339937, + "pass@32": 0.6680454701301732, + "pass@64": 0.7240389650730391, + "pass@128": 0.7708333333333334 + }, + "20": { + "pass@1": 0.277156624251497, + "pass@2": 0.3758873738742987, + "pass@4": 0.46919662278367874, + "pass@8": 0.547041580122116, + "pass@16": 0.608975135388662, + "pass@32": 0.6595693283084222, + "pass@64": 0.7034790052169011, + "pass@128": 0.7455089820359282 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 630014, + "answer_accuracy": 0.7863851054481547, + "avg_response_len": 304.9374862697715, + "resp_tokens_sum": 244301277, + "pass_at_k": { + "pass@1": 0.7863851054481547, + "pass@2": 0.8276325988214681, + "pass@4": 0.8593780909146573, + "pass@8": 0.8839933436948046, + "pass@16": 0.902981023040986, + "pass@32": 0.917834536552863, + "pass@64": 0.9295588405832169, + "pass@128": 0.9389678862438089 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9421175373134328, + "pass@2": 0.9587580062286991, + "pass@4": 0.9685092193699665, + "pass@8": 0.9760498065360441, + "pass@16": 0.9825003934245147, + "pass@32": 0.9871221085392369, + "pass@64": 0.9894357905723764, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9999330357142857, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9966517857142857, + "pass@2": 0.9971243629622828, + "pass@4": 0.9972639609917566, + "pass@8": 0.99743893615483, + "pass@16": 0.9977703679897155, + "pass@32": 0.9983643626179382, + "pass@64": 0.9992768761047727, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9989720394736842, + "pass@2": 0.9998701334208223, + "pass@4": 0.9999958955240242, + "pass@8": 0.9999999968048913, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.974172794117647, + "pass@2": 0.9799802425891616, + "pass@4": 0.9851549995588786, + "pass@8": 0.9903396378716414, + "pass@16": 0.9949618620195534, + "pass@32": 0.9978909857983171, + "pass@64": 0.9992638569390833, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.98957511829653, + "pass@2": 0.9909034315308379, + "pass@4": 0.9923643261075016, + "pass@8": 0.9937184520602704, + "pass@16": 0.9949870433431968, + "pass@32": 0.9963097387793499, + "pass@64": 0.9980377056438621, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.984375, + "pass@2": 0.9912449074304795, + "pass@4": 0.994530023244721, + "pass@8": 0.9967387415639558, + "pass@16": 0.9985927711809854, + "pass@32": 0.9996839030827411, + "pass@64": 0.9999901863988568, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9845626876876877, + "pass@2": 0.9914509736114068, + "pass@4": 0.9942883507804767, + "pass@8": 0.9953259254730494, + "pass@16": 0.9963787369181493, + "pass@32": 0.9978019463673319, + "pass@64": 0.9992136480632735, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9767132234726688, + "pass@2": 0.9858248727751473, + "pass@4": 0.9893246072335813, + "pass@8": 0.9916823703543539, + "pass@16": 0.9934710727259167, + "pass@32": 0.995379009876187, + "pass@64": 0.9975736224486328, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9344246031746032, + "pass@2": 0.956496453568304, + "pass@4": 0.9680845546092448, + "pass@8": 0.9746013817828894, + "pass@16": 0.979128362963511, + "pass@32": 0.9823612720845869, + "pass@64": 0.9839103382187553, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8939698012232415, + "pass@2": 0.9233999133135886, + "pass@4": 0.940026338164151, + "pass@8": 0.9508489849950189, + "pass@16": 0.9588264089267363, + "pass@32": 0.9654115182769986, + "pass@64": 0.9710843168080171, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8054591049382716, + "pass@2": 0.8580877260134149, + "pass@4": 0.8874315097186924, + "pass@8": 0.9042751337372295, + "pass@16": 0.9160205919035335, + "pass@32": 0.9253222868446103, + "pass@64": 0.9338060490002721, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7650686201780416, + "pass@2": 0.8257855761816872, + "pass@4": 0.8638935942539817, + "pass@8": 0.8898587056265527, + "pass@16": 0.909471720507359, + "pass@32": 0.9247618852817582, + "pass@64": 0.9359455063598505, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6778308256172839, + "pass@2": 0.7494824323174885, + "pass@4": 0.801531108958602, + "pass@8": 0.8410958117430899, + "pass@16": 0.8703430132240998, + "pass@32": 0.8920039907677088, + "pass@64": 0.9082821137578678, + "pass@128": 0.9197530864197531 + }, + "16": { + "pass@1": 0.5564850663716814, + "pass@2": 0.659352280909577, + "pass@4": 0.7384296702956372, + "pass@8": 0.7974724410886839, + "pass@16": 0.8379566282804083, + "pass@32": 0.867540734508324, + "pass@64": 0.8941426993490582, + "pass@128": 0.9174041297935103 + }, + "17": { + "pass@1": 0.44517131024096385, + "pass@2": 0.5476650697277298, + "pass@4": 0.630717065393181, + "pass@8": 0.696161286495951, + "pass@16": 0.7486587759712936, + "pass@32": 0.7898353056767135, + "pass@64": 0.8198061896401653, + "pass@128": 0.8433734939759037 + }, + "18": { + "pass@1": 0.44601814516129035, + "pass@2": 0.5429264827279653, + "pass@4": 0.6179406434074768, + "pass@8": 0.6738175548059356, + "pass@16": 0.7170866104454663, + "pass@32": 0.7528154066701483, + "pass@64": 0.7842687233254886, + "pass@128": 0.8129032258064516 + }, + "19": { + "pass@1": 0.28757552870090636, + "pass@2": 0.3801985453291147, + "pass@4": 0.4683336989409553, + "pass@8": 0.5477159165532208, + "pass@16": 0.6142955679985431, + "pass@32": 0.6685448647183322, + "pass@64": 0.7121010913154943, + "pass@128": 0.7462235649546828 + }, + "20": { + "pass@1": 0.2712321696696697, + "pass@2": 0.377846350287295, + "pass@4": 0.48275687892617025, + "pass@8": 0.5715013261561981, + "pass@16": 0.6392863359899448, + "pass@32": 0.6912394364961897, + "pass@64": 0.7295567463044594, + "pass@128": 0.7567567567567568 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch1-op11-14/graphs/SDAR-100M-block4-epoch1-op11-14_score_report.jsonl b/composition/SDAR-100M-block4-epoch1-op11-14/graphs/SDAR-100M-block4-epoch1-op11-14_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1fcb3b7b7ed21b2c0514d88ac308e7aba80cfa84 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op11-14/graphs/SDAR-100M-block4-epoch1-op11-14_score_report.jsonl @@ -0,0 +1,2 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch1-op11-14", "results_dir": "results/composition/SDAR-100M-block4-epoch1-op11-14/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1008511, "value_mismatch": 24745, "dependency_mismatch": 140026, "answer_mismatch": 19899, "extra_nodes": 20890, "missing_nodes": 11838, "perfect_with_extra": 2156, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 332237, "dependency_mismatch": 45651, "value_mismatch": 8466, "answer_mismatch": 6766, "missing_nodes": 4002, "extra_nodes": 6867, "perfect_with_extra": 682}, "teachers_in_school": {"total": 390656, "perfect": 342242, "dependency_mismatch": 47358, "missing_nodes": 3773, "value_mismatch": 7872, "answer_mismatch": 6698, "extra_nodes": 7254, "perfect_with_extra": 869}, "movie_festival_awards": {"total": 382336, "perfect": 334032, "dependency_mismatch": 47017, "value_mismatch": 8407, "answer_mismatch": 6435, "missing_nodes": 4063, "extra_nodes": 6769, "perfect_with_extra": 605}}, "results_path": "results/composition/SDAR-100M-block4-epoch1-op11-14/graphs/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 378313, "value_mismatch": 626709, "dependency_mismatch": 810905, "answer_mismatch": 492780, "extra_nodes": 57563, "missing_nodes": 286332, "perfect_with_extra": 395, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "dependency_mismatch": 271907, "extra_nodes": 21669, "perfect": 128370, "value_mismatch": 209014, "answer_mismatch": 162655, "missing_nodes": 103841, "perfect_with_extra": 134}, "movie_festival_awards": {"total": 418816, "perfect": 122434, "value_mismatch": 208587, "dependency_mismatch": 268172, "answer_mismatch": 164690, "missing_nodes": 89945, "extra_nodes": 17527, "perfect_with_extra": 134}, "teachers_in_school": {"total": 430464, "perfect": 127509, "value_mismatch": 209108, "dependency_mismatch": 270826, "answer_mismatch": 165435, "missing_nodes": 92546, "extra_nodes": 18367, "perfect_with_extra": 127}}, "results_path": "results/composition/SDAR-100M-block4-epoch1-op11-14/graphs/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1386824, "value_mismatch": 651454, "dependency_mismatch": 950931, "answer_mismatch": 512679, "extra_nodes": 78453, "missing_nodes": 298170, "perfect_with_extra": 2551, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch1-op11-14/graphs/SDAR-100M-block4-epoch1-op11-14_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 460607, "dependency_mismatch": 317558, "value_mismatch": 217480, "answer_mismatch": 169421, "missing_nodes": 107843, "extra_nodes": 28536, "perfect_with_extra": 816}, "teachers_in_school": {"total": 821120, "perfect": 469751, "dependency_mismatch": 318184, "missing_nodes": 96319, "value_mismatch": 216980, "answer_mismatch": 172133, "extra_nodes": 25621, "perfect_with_extra": 996}, "movie_festival_awards": {"total": 801152, "perfect": 456466, "dependency_mismatch": 315189, "value_mismatch": 216994, "answer_mismatch": 171125, "missing_nodes": 94008, "extra_nodes": 24296, "perfect_with_extra": 739}}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch1-op11-14", "results_dir": "results/composition/SDAR-100M-block4-epoch1-op11-14/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch1-op11-14/graphs/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch1-op11-14/graphs/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch1-op11-14/graphs/SDAR-100M-block4-epoch1-op11-14_score_report.jsonl", "template_stats": {}}} diff --git a/composition/SDAR-100M-block4-epoch1-op11-14/graphs/results.json b/composition/SDAR-100M-block4-epoch1-op11-14/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..0a3c7bb13eaa2b55438f7c7839867a3dc511661b --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op11-14/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990625, + "pass@2": 0.99375, + "pass@4": 0.99375, + "pass@8": 0.996875, + "pass@16": 0.996875, + "pass@32": 0.996875, + "pass@64": 0.996875, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9555555555555556, + "pass@2": 0.9777777777777777, + "pass@4": 0.9873015873015873, + "pass@8": 0.9904761904761905, + "pass@16": 0.9936507936507937, + "pass@32": 0.9936507936507937, + "pass@64": 0.9936507936507937, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9640718562874252, + "pass@2": 0.9700598802395209, + "pass@4": 0.9700598802395209, + "pass@8": 0.9880239520958084, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.8846153846153846, + "pass@2": 0.9201183431952663, + "pass@4": 0.9615384615384616, + "pass@8": 0.9881656804733728, + "pass@16": 0.9940828402366864, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.7852564102564102, + "pass@2": 0.8589743589743589, + "pass@4": 0.9294871794871795, + "pass@8": 0.967948717948718, + "pass@16": 0.9903846153846154, + "pass@32": 0.9935897435897436, + "pass@64": 0.9967948717948718, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.723943661971831, + "pass@2": 0.8140845070422535, + "pass@4": 0.9070422535211268, + "pass@8": 0.9605633802816902, + "pass@16": 0.9774647887323944, + "pass@32": 0.9943661971830986, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "10": { + "pass@1": 0.6489675516224189, + "pass@2": 0.7168141592920354, + "pass@4": 0.7964601769911505, + "pass@8": 0.8584070796460177, + "pass@16": 0.8849557522123894, + "pass@32": 0.8908554572271387, + "pass@64": 0.9056047197640118, + "pass@128": 0.9174041297935103 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.990625, + "5": 0.9555555555555556, + "6": 0.9640718562874252, + "7": 0.8846153846153846, + "8": 0.7852564102564102, + "9": 0.723943661971831, + "10": 0.6489675516224189 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9777777777777777, + "6": 0.9700598802395209, + "7": 0.9201183431952663, + "8": 0.8589743589743589, + "9": 0.8140845070422535, + "10": 0.7168141592920354 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9873015873015873, + "6": 0.9700598802395209, + "7": 0.9615384615384616, + "8": 0.9294871794871795, + "9": 0.9070422535211268, + "10": 0.7964601769911505 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9904761904761905, + "6": 0.9880239520958084, + "7": 0.9881656804733728, + "8": 0.967948717948718, + "9": 0.9605633802816902, + "10": 0.8584070796460177 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 0.9940119760479041, + "7": 0.9940828402366864, + "8": 0.9903846153846154, + "9": 0.9774647887323944, + "10": 0.8849557522123894 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 0.9940119760479041, + "7": 0.9970414201183432, + "8": 0.9935897435897436, + "9": 0.9943661971830986, + "10": 0.8908554572271387 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 0.9940119760479041, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 0.9971830985915493, + "10": 0.9056047197640118 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 0.9940119760479041, + "7": 0.9970414201183432, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9174041297935103 + } + }, + "total_pass_at_k": { + "pass@1": 0.8807835190813914, + "pass@2": 0.9148936170212766, + "pass@4": 0.9483282674772037, + "pass@8": 0.9716312056737588, + "pass@16": 0.9807497467071935, + "pass@32": 0.9841269841269841, + "pass@64": 0.9864910503208375, + "pass@128": 0.9881796690307328 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970845481049563, + "pass@2": 0.9970845481049563, + "pass@4": 0.9970845481049563, + "pass@8": 0.9970845481049563, + "pass@16": 0.9970845481049563, + "pass@32": 0.9970845481049563, + "pass@64": 0.9970845481049563, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9824561403508771, + "pass@2": 0.9941520467836257, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9411764705882353, + "pass@2": 0.961764705882353, + "pass@4": 0.9764705882352941, + "pass@8": 0.9823529411764705, + "pass@16": 0.9852941176470589, + "pass@32": 0.9911764705882353, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9621451104100947, + "pass@2": 0.9684542586750788, + "pass@4": 0.9779179810725552, + "pass@8": 0.9810725552050473, + "pass@16": 0.9873817034700315, + "pass@32": 0.9968454258675079, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.8575949367088608, + "pass@2": 0.9177215189873418, + "pass@4": 0.9430379746835443, + "pass@8": 0.9778481012658228, + "pass@16": 0.990506329113924, + "pass@32": 0.990506329113924, + "pass@64": 0.9936708860759493, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.7627627627627628, + "pass@2": 0.8618618618618619, + "pass@4": 0.918918918918919, + "pass@8": 0.9669669669669669, + "pass@16": 0.993993993993994, + "pass@32": 0.996996996996997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7427652733118971, + "pass@2": 0.819935691318328, + "pass@4": 0.8971061093247589, + "pass@8": 0.9356913183279743, + "pass@16": 0.9871382636655949, + "pass@32": 0.9935691318327974, + "pass@64": 0.9967845659163987, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.591044776119403, + "pass@2": 0.6805970149253732, + "pass@4": 0.7432835820895523, + "pass@8": 0.817910447761194, + "pass@16": 0.8626865671641791, + "pass@32": 0.8895522388059701, + "pass@64": 0.9074626865671642, + "pass@128": 0.9194029850746268 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9824561403508771, + "5": 0.9411764705882353, + "6": 0.9621451104100947, + "7": 0.8575949367088608, + "8": 0.7627627627627628, + "9": 0.7427652733118971, + "10": 0.591044776119403 + }, + "pass@2": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9941520467836257, + "5": 0.961764705882353, + "6": 0.9684542586750788, + "7": 0.9177215189873418, + "8": 0.8618618618618619, + "9": 0.819935691318328, + "10": 0.6805970149253732 + }, + "pass@4": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 1.0, + "5": 0.9764705882352941, + "6": 0.9779179810725552, + "7": 0.9430379746835443, + "8": 0.918918918918919, + "9": 0.8971061093247589, + "10": 0.7432835820895523 + }, + "pass@8": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 1.0, + "5": 0.9823529411764705, + "6": 0.9810725552050473, + "7": 0.9778481012658228, + "8": 0.9669669669669669, + "9": 0.9356913183279743, + "10": 0.817910447761194 + }, + "pass@16": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 1.0, + "5": 0.9852941176470589, + "6": 0.9873817034700315, + "7": 0.990506329113924, + "8": 0.993993993993994, + "9": 0.9871382636655949, + "10": 0.8626865671641791 + }, + "pass@32": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 1.0, + "5": 0.9911764705882353, + "6": 0.9968454258675079, + "7": 0.990506329113924, + "8": 0.996996996996997, + "9": 0.9935691318327974, + "10": 0.8895522388059701 + }, + "pass@64": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 1.0, + "5": 1.0, + "6": 0.9968454258675079, + "7": 0.9936708860759493, + "8": 1.0, + "9": 0.9967845659163987, + "10": 0.9074626865671642 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9968454258675079, + "7": 0.9968354430379747, + "8": 1.0, + "9": 1.0, + "10": 0.9194029850746268 + } + }, + "total_pass_at_k": { + "pass@1": 0.8727820555741547, + "pass@2": 0.912621359223301, + "pass@4": 0.9400736524941413, + "pass@8": 0.9625041848008035, + "pass@16": 0.9782390358218949, + "pass@32": 0.98393036491463, + "pass@64": 0.9879477736859725, + "pass@128": 0.9902912621359223 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9939393939393939, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985207100591716, + "pass@2": 0.9940828402366864, + "pass@4": 0.9970414201183432, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9304347826086956, + "pass@2": 0.9565217391304348, + "pass@4": 0.9855072463768116, + "pass@8": 0.991304347826087, + "pass@16": 0.9942028985507246, + "pass@32": 0.9971014492753624, + "pass@64": 0.9971014492753624, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9770773638968482, + "pass@2": 0.9799426934097422, + "pass@4": 0.9914040114613181, + "pass@8": 0.997134670487106, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.8439306358381503, + "pass@2": 0.9046242774566474, + "pass@4": 0.9479768786127167, + "pass@8": 0.9739884393063584, + "pass@16": 0.9913294797687862, + "pass@32": 0.9942196531791907, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7774647887323943, + "pass@2": 0.8647887323943662, + "pass@4": 0.923943661971831, + "pass@8": 0.9774647887323944, + "pass@16": 0.9859154929577465, + "pass@32": 0.9915492957746479, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.718562874251497, + "pass@2": 0.7844311377245509, + "pass@4": 0.8652694610778443, + "pass@8": 0.9251497005988024, + "pass@16": 0.9550898203592815, + "pass@32": 0.9820359281437125, + "pass@64": 0.9880239520958084, + "pass@128": 0.9910179640718563 + }, + "10": { + "pass@1": 0.6441717791411042, + "pass@2": 0.696319018404908, + "pass@4": 0.7515337423312883, + "pass@8": 0.8190184049079755, + "pass@16": 0.8650306748466258, + "pass@32": 0.8865030674846626, + "pass@64": 0.8957055214723927, + "pass@128": 0.911042944785276 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9939393939393939, + "4": 0.985207100591716, + "5": 0.9304347826086956, + "6": 0.9770773638968482, + "7": 0.8439306358381503, + "8": 0.7774647887323943, + "9": 0.718562874251497, + "10": 0.6441717791411042 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9940828402366864, + "5": 0.9565217391304348, + "6": 0.9799426934097422, + "7": 0.9046242774566474, + "8": 0.8647887323943662, + "9": 0.7844311377245509, + "10": 0.696319018404908 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.9855072463768116, + "6": 0.9914040114613181, + "7": 0.9479768786127167, + "8": 0.923943661971831, + "9": 0.8652694610778443, + "10": 0.7515337423312883 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.991304347826087, + "6": 0.997134670487106, + "7": 0.9739884393063584, + "8": 0.9774647887323944, + "9": 0.9251497005988024, + "10": 0.8190184049079755 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9942028985507246, + "6": 0.997134670487106, + "7": 0.9913294797687862, + "8": 0.9859154929577465, + "9": 0.9550898203592815, + "10": 0.8650306748466258 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.997134670487106, + "7": 0.9942196531791907, + "8": 0.9915492957746479, + "9": 0.9820359281437125, + "10": 0.8865030674846626 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.997134670487106, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9880239520958084, + "10": 0.8957055214723927 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9910179640718563, + "10": 0.911042944785276 + } + }, + "total_pass_at_k": { + "pass@1": 0.8748361730013107, + "pass@2": 0.90956749672346, + "pass@4": 0.9410222804718218, + "pass@8": 0.9655963302752294, + "pass@16": 0.9770642201834863, + "pass@32": 0.9836173001310616, + "pass@64": 0.9865661861074705, + "pass@128": 0.9888597640891219 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.999, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.997, + "pass@2": 0.999, + "pass@4": 0.999, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.986, + "pass@2": 0.994, + "pass@4": 0.997, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.942, + "pass@2": 0.965, + "pass@4": 0.983, + "pass@8": 0.988, + "pass@16": 0.991, + "pass@32": 0.994, + "pass@64": 0.997, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.968, + "pass@2": 0.973, + "pass@4": 0.98, + "pass@8": 0.989, + "pass@16": 0.993, + "pass@32": 0.996, + "pass@64": 0.996, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.862, + "pass@2": 0.914, + "pass@4": 0.951, + "pass@8": 0.98, + "pass@16": 0.992, + "pass@32": 0.994, + "pass@64": 0.997, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.775, + "pass@2": 0.862, + "pass@4": 0.924, + "pass@8": 0.971, + "pass@16": 0.99, + "pass@32": 0.994, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.728, + "pass@2": 0.806, + "pass@4": 0.89, + "pass@8": 0.941, + "pass@16": 0.973, + "pass@32": 0.99, + "pass@64": 0.994, + "pass@128": 0.996 + }, + "10": { + "pass@1": 0.628, + "pass@2": 0.698, + "pass@4": 0.764, + "pass@8": 0.832, + "pass@16": 0.871, + "pass@32": 0.889, + "pass@64": 0.903, + "pass@128": 0.916 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.999, + "3": 0.997, + "4": 0.986, + "5": 0.942, + "6": 0.968, + "7": 0.862, + "8": 0.775, + "9": 0.728, + "10": 0.628 + }, + "pass@2": { + "2": 1.0, + "3": 0.999, + "4": 0.994, + "5": 0.965, + "6": 0.973, + "7": 0.914, + "8": 0.862, + "9": 0.806, + "10": 0.698 + }, + "pass@4": { + "2": 1.0, + "3": 0.999, + "4": 0.997, + "5": 0.983, + "6": 0.98, + "7": 0.951, + "8": 0.924, + "9": 0.89, + "10": 0.764 + }, + "pass@8": { + "2": 1.0, + "3": 0.999, + "4": 0.999, + "5": 0.988, + "6": 0.989, + "7": 0.98, + "8": 0.971, + "9": 0.941, + "10": 0.832 + }, + "pass@16": { + "2": 1.0, + "3": 0.999, + "4": 0.999, + "5": 0.991, + "6": 0.993, + "7": 0.992, + "8": 0.99, + "9": 0.973, + "10": 0.871 + }, + "pass@32": { + "2": 1.0, + "3": 0.999, + "4": 0.999, + "5": 0.994, + "6": 0.996, + "7": 0.994, + "8": 0.994, + "9": 0.99, + "10": 0.889 + }, + "pass@64": { + "2": 1.0, + "3": 0.999, + "4": 0.999, + "5": 0.997, + "6": 0.996, + "7": 0.997, + "8": 0.998, + "9": 0.994, + "10": 0.903 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.998, + "6": 0.996, + "7": 0.998, + "8": 0.999, + "9": 0.996, + "10": 0.916 + } + }, + "total_pass_at_k": { + "pass@1": 0.8761111111111111, + "pass@2": 0.9123333333333333, + "pass@4": 0.9431111111111111, + "pass@8": 0.9665555555555555, + "pass@16": 0.9786666666666667, + "pass@32": 0.9838888888888889, + "pass@64": 0.987, + "pass@128": 0.9891111111111112 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5807365439093485, + "pass@2": 0.6572237960339944, + "pass@4": 0.7535410764872521, + "pass@8": 0.8243626062322946, + "pass@16": 0.8753541076487252, + "pass@32": 0.9093484419263456, + "pass@64": 0.9235127478753541, + "pass@128": 0.9348441926345609 + }, + "12": { + "pass@1": 0.5706214689265536, + "pass@2": 0.6271186440677966, + "pass@4": 0.6977401129943502, + "pass@8": 0.7768361581920904, + "pass@16": 0.8389830508474576, + "pass@32": 0.8502824858757062, + "pass@64": 0.8587570621468926, + "pass@128": 0.8615819209039548 + }, + "13": { + "pass@1": 0.45454545454545453, + "pass@2": 0.5235109717868338, + "pass@4": 0.5830721003134797, + "pass@8": 0.6520376175548589, + "pass@16": 0.7178683385579937, + "pass@32": 0.7711598746081505, + "pass@64": 0.7899686520376176, + "pass@128": 0.8056426332288401 + }, + "14": { + "pass@1": 0.4236760124610592, + "pass@2": 0.4735202492211838, + "pass@4": 0.5358255451713395, + "pass@8": 0.6043613707165109, + "pass@16": 0.6542056074766355, + "pass@32": 0.7071651090342679, + "pass@64": 0.7663551401869159, + "pass@128": 0.7975077881619937 + }, + "15": { + "pass@1": 0.315028901734104, + "pass@2": 0.3554913294797688, + "pass@4": 0.3988439306358382, + "pass@8": 0.45375722543352603, + "pass@16": 0.5317919075144508, + "pass@32": 0.6271676300578035, + "pass@64": 0.7052023121387283, + "pass@128": 0.7485549132947977 + }, + "16": { + "pass@1": 0.24193548387096775, + "pass@2": 0.2838709677419355, + "pass@4": 0.32903225806451614, + "pass@8": 0.35161290322580646, + "pass@16": 0.4096774193548387, + "pass@32": 0.4806451612903226, + "pass@64": 0.535483870967742, + "pass@128": 0.5903225806451613 + }, + "17": { + "pass@1": 0.2254335260115607, + "pass@2": 0.2745664739884393, + "pass@4": 0.3236994219653179, + "pass@8": 0.36416184971098264, + "pass@16": 0.40173410404624277, + "pass@32": 0.4508670520231214, + "pass@64": 0.47398843930635837, + "pass@128": 0.5375722543352601 + }, + "18": { + "pass@1": 0.12285714285714286, + "pass@2": 0.16285714285714287, + "pass@4": 0.19714285714285715, + "pass@8": 0.23142857142857143, + "pass@16": 0.26285714285714284, + "pass@32": 0.30857142857142855, + "pass@64": 0.3485714285714286, + "pass@128": 0.3942857142857143 + }, + "19": { + "pass@1": 0.04804804804804805, + "pass@2": 0.07207207207207207, + "pass@4": 0.0960960960960961, + "pass@8": 0.13213213213213212, + "pass@16": 0.15615615615615616, + "pass@32": 0.18018018018018017, + "pass@64": 0.2072072072072072, + "pass@128": 0.22822822822822822 + }, + "20": { + "pass@1": 0.06306306306306306, + "pass@2": 0.08408408408408409, + "pass@4": 0.11711711711711711, + "pass@8": 0.14714714714714713, + "pass@16": 0.16216216216216217, + "pass@32": 0.1921921921921922, + "pass@64": 0.21021021021021022, + "pass@128": 0.23123123123123124 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5807365439093485, + "12": 0.5706214689265536, + "13": 0.45454545454545453, + "14": 0.4236760124610592, + "15": 0.315028901734104, + "16": 0.24193548387096775, + "17": 0.2254335260115607, + "18": 0.12285714285714286, + "19": 0.04804804804804805, + "20": 0.06306306306306306 + }, + "pass@2": { + "11": 0.6572237960339944, + "12": 0.6271186440677966, + "13": 0.5235109717868338, + "14": 0.4735202492211838, + "15": 0.3554913294797688, + "16": 0.2838709677419355, + "17": 0.2745664739884393, + "18": 0.16285714285714287, + "19": 0.07207207207207207, + "20": 0.08408408408408409 + }, + "pass@4": { + "11": 0.7535410764872521, + "12": 0.6977401129943502, + "13": 0.5830721003134797, + "14": 0.5358255451713395, + "15": 0.3988439306358382, + "16": 0.32903225806451614, + "17": 0.3236994219653179, + "18": 0.19714285714285715, + "19": 0.0960960960960961, + "20": 0.11711711711711711 + }, + "pass@8": { + "11": 0.8243626062322946, + "12": 0.7768361581920904, + "13": 0.6520376175548589, + "14": 0.6043613707165109, + "15": 0.45375722543352603, + "16": 0.35161290322580646, + "17": 0.36416184971098264, + "18": 0.23142857142857143, + "19": 0.13213213213213212, + "20": 0.14714714714714713 + }, + "pass@16": { + "11": 0.8753541076487252, + "12": 0.8389830508474576, + "13": 0.7178683385579937, + "14": 0.6542056074766355, + "15": 0.5317919075144508, + "16": 0.4096774193548387, + "17": 0.40173410404624277, + "18": 0.26285714285714284, + "19": 0.15615615615615616, + "20": 0.16216216216216217 + }, + "pass@32": { + "11": 0.9093484419263456, + "12": 0.8502824858757062, + "13": 0.7711598746081505, + "14": 0.7071651090342679, + "15": 0.6271676300578035, + "16": 0.4806451612903226, + "17": 0.4508670520231214, + "18": 0.30857142857142855, + "19": 0.18018018018018017, + "20": 0.1921921921921922 + }, + "pass@64": { + "11": 0.9235127478753541, + "12": 0.8587570621468926, + "13": 0.7899686520376176, + "14": 0.7663551401869159, + "15": 0.7052023121387283, + "16": 0.535483870967742, + "17": 0.47398843930635837, + "18": 0.3485714285714286, + "19": 0.2072072072072072, + "20": 0.21021021021021022 + }, + "pass@128": { + "11": 0.9348441926345609, + "12": 0.8615819209039548, + "13": 0.8056426332288401, + "14": 0.7975077881619937, + "15": 0.7485549132947977, + "16": 0.5903225806451613, + "17": 0.5375722543352601, + "18": 0.3942857142857143, + "19": 0.22822822822822822, + "20": 0.23123123123123124 + } + }, + "total_pass_at_k": { + "pass@1": 0.3060921248142645, + "pass@2": 0.35304606240713227, + "pass@4": 0.4050520059435364, + "pass@8": 0.45586924219910846, + "pass@16": 0.5031203566121842, + "pass@32": 0.549479940564636, + "pass@64": 0.5833580980683507, + "pass@128": 0.6142644873699852 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5206349206349207, + "pass@2": 0.6253968253968254, + "pass@4": 0.7396825396825397, + "pass@8": 0.8380952380952381, + "pass@16": 0.8888888888888888, + "pass@32": 0.9333333333333333, + "pass@64": 0.9523809523809523, + "pass@128": 0.9714285714285714 + }, + "12": { + "pass@1": 0.5626911314984709, + "pass@2": 0.6391437308868502, + "pass@4": 0.7003058103975535, + "pass@8": 0.7889908256880734, + "pass@16": 0.8562691131498471, + "pass@32": 0.8715596330275229, + "pass@64": 0.8807339449541285, + "pass@128": 0.8899082568807339 + }, + "13": { + "pass@1": 0.4537037037037037, + "pass@2": 0.5061728395061729, + "pass@4": 0.6049382716049383, + "pass@8": 0.6481481481481481, + "pass@16": 0.6975308641975309, + "pass@32": 0.7283950617283951, + "pass@64": 0.7438271604938271, + "pass@128": 0.7716049382716049 + }, + "14": { + "pass@1": 0.36795252225519287, + "pass@2": 0.42433234421364985, + "pass@4": 0.47477744807121663, + "pass@8": 0.5489614243323442, + "pass@16": 0.6201780415430267, + "pass@32": 0.7002967359050445, + "pass@64": 0.7566765578635015, + "pass@128": 0.8011869436201781 + }, + "15": { + "pass@1": 0.30246913580246915, + "pass@2": 0.33641975308641975, + "pass@4": 0.3734567901234568, + "pass@8": 0.4382716049382716, + "pass@16": 0.5123456790123457, + "pass@32": 0.6018518518518519, + "pass@64": 0.6759259259259259, + "pass@128": 0.7314814814814815 + }, + "16": { + "pass@1": 0.22123893805309736, + "pass@2": 0.29793510324483774, + "pass@4": 0.36578171091445427, + "pass@8": 0.41887905604719766, + "pass@16": 0.4690265486725664, + "pass@32": 0.504424778761062, + "pass@64": 0.5575221238938053, + "pass@128": 0.5988200589970502 + }, + "17": { + "pass@1": 0.1566265060240964, + "pass@2": 0.18072289156626506, + "pass@4": 0.24096385542168675, + "pass@8": 0.2710843373493976, + "pass@16": 0.3313253012048193, + "pass@32": 0.3674698795180723, + "pass@64": 0.41566265060240964, + "pass@128": 0.46987951807228917 + }, + "18": { + "pass@1": 0.13870967741935483, + "pass@2": 0.1774193548387097, + "pass@4": 0.2129032258064516, + "pass@8": 0.25161290322580643, + "pass@16": 0.2870967741935484, + "pass@32": 0.33225806451612905, + "pass@64": 0.3548387096774194, + "pass@128": 0.4129032258064516 + }, + "19": { + "pass@1": 0.0755287009063444, + "pass@2": 0.09063444108761329, + "pass@4": 0.1148036253776435, + "pass@8": 0.13293051359516617, + "pass@16": 0.16012084592145015, + "pass@32": 0.18126888217522658, + "pass@64": 0.22658610271903323, + "pass@128": 0.2628398791540785 + }, + "20": { + "pass@1": 0.06606606606606606, + "pass@2": 0.1021021021021021, + "pass@4": 0.13513513513513514, + "pass@8": 0.15315315315315314, + "pass@16": 0.16816816816816818, + "pass@32": 0.1891891891891892, + "pass@64": 0.21021021021021022, + "pass@128": 0.23723723723723725 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5206349206349207, + "12": 0.5626911314984709, + "13": 0.4537037037037037, + "14": 0.36795252225519287, + "15": 0.30246913580246915, + "16": 0.22123893805309736, + "17": 0.1566265060240964, + "18": 0.13870967741935483, + "19": 0.0755287009063444, + "20": 0.06606606606606606 + }, + "pass@2": { + "11": 0.6253968253968254, + "12": 0.6391437308868502, + "13": 0.5061728395061729, + "14": 0.42433234421364985, + "15": 0.33641975308641975, + "16": 0.29793510324483774, + "17": 0.18072289156626506, + "18": 0.1774193548387097, + "19": 0.09063444108761329, + "20": 0.1021021021021021 + }, + "pass@4": { + "11": 0.7396825396825397, + "12": 0.7003058103975535, + "13": 0.6049382716049383, + "14": 0.47477744807121663, + "15": 0.3734567901234568, + "16": 0.36578171091445427, + "17": 0.24096385542168675, + "18": 0.2129032258064516, + "19": 0.1148036253776435, + "20": 0.13513513513513514 + }, + "pass@8": { + "11": 0.8380952380952381, + "12": 0.7889908256880734, + "13": 0.6481481481481481, + "14": 0.5489614243323442, + "15": 0.4382716049382716, + "16": 0.41887905604719766, + "17": 0.2710843373493976, + "18": 0.25161290322580643, + "19": 0.13293051359516617, + "20": 0.15315315315315314 + }, + "pass@16": { + "11": 0.8888888888888888, + "12": 0.8562691131498471, + "13": 0.6975308641975309, + "14": 0.6201780415430267, + "15": 0.5123456790123457, + "16": 0.4690265486725664, + "17": 0.3313253012048193, + "18": 0.2870967741935484, + "19": 0.16012084592145015, + "20": 0.16816816816816818 + }, + "pass@32": { + "11": 0.9333333333333333, + "12": 0.8715596330275229, + "13": 0.7283950617283951, + "14": 0.7002967359050445, + "15": 0.6018518518518519, + "16": 0.504424778761062, + "17": 0.3674698795180723, + "18": 0.33225806451612905, + "19": 0.18126888217522658, + "20": 0.1891891891891892 + }, + "pass@64": { + "11": 0.9523809523809523, + "12": 0.8807339449541285, + "13": 0.7438271604938271, + "14": 0.7566765578635015, + "15": 0.6759259259259259, + "16": 0.5575221238938053, + "17": 0.41566265060240964, + "18": 0.3548387096774194, + "19": 0.22658610271903323, + "20": 0.21021021021021022 + }, + "pass@128": { + "11": 0.9714285714285714, + "12": 0.8899082568807339, + "13": 0.7716049382716049, + "14": 0.8011869436201781, + "15": 0.7314814814814815, + "16": 0.5988200589970502, + "17": 0.46987951807228917, + "18": 0.4129032258064516, + "19": 0.2628398791540785, + "20": 0.23723723723723725 + } + }, + "total_pass_at_k": { + "pass@1": 0.28545232273838633, + "pass@2": 0.3367970660146699, + "pass@4": 0.39486552567237165, + "pass@8": 0.4474327628361858, + "pass@16": 0.49755501222493886, + "pass@32": 0.5394254278728606, + "pass@64": 0.5761002444987775, + "pass@128": 0.6133863080684596 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5783132530120482, + "pass@2": 0.6566265060240963, + "pass@4": 0.7379518072289156, + "pass@8": 0.8162650602409639, + "pass@16": 0.891566265060241, + "pass@32": 0.9457831325301205, + "pass@64": 0.9608433734939759, + "pass@128": 0.9668674698795181 + }, + "12": { + "pass@1": 0.5517241379310345, + "pass@2": 0.6050156739811913, + "pass@4": 0.6865203761755486, + "pass@8": 0.7554858934169278, + "pass@16": 0.8087774294670846, + "pass@32": 0.8307210031347962, + "pass@64": 0.8432601880877743, + "pass@128": 0.8652037617554859 + }, + "13": { + "pass@1": 0.453781512605042, + "pass@2": 0.5238095238095238, + "pass@4": 0.6218487394957983, + "pass@8": 0.7170868347338936, + "pass@16": 0.7647058823529411, + "pass@32": 0.7955182072829131, + "pass@64": 0.8179271708683473, + "pass@128": 0.8487394957983193 + }, + "14": { + "pass@1": 0.38596491228070173, + "pass@2": 0.4298245614035088, + "pass@4": 0.5087719298245614, + "pass@8": 0.5789473684210527, + "pass@16": 0.6461988304093568, + "pass@32": 0.7017543859649122, + "pass@64": 0.7339181286549707, + "pass@128": 0.783625730994152 + }, + "15": { + "pass@1": 0.31212121212121213, + "pass@2": 0.38484848484848483, + "pass@4": 0.4393939393939394, + "pass@8": 0.5121212121212121, + "pass@16": 0.6, + "pass@32": 0.6575757575757576, + "pass@64": 0.7151515151515152, + "pass@128": 0.7606060606060606 + }, + "16": { + "pass@1": 0.2535612535612536, + "pass@2": 0.3076923076923077, + "pass@4": 0.37037037037037035, + "pass@8": 0.4415954415954416, + "pass@16": 0.47293447293447294, + "pass@32": 0.5242165242165242, + "pass@64": 0.5868945868945868, + "pass@128": 0.6267806267806267 + }, + "17": { + "pass@1": 0.16459627329192547, + "pass@2": 0.2236024844720497, + "pass@4": 0.2577639751552795, + "pass@8": 0.30745341614906835, + "pass@16": 0.3385093167701863, + "pass@32": 0.38509316770186336, + "pass@64": 0.4409937888198758, + "pass@128": 0.4751552795031056 + }, + "18": { + "pass@1": 0.14411764705882352, + "pass@2": 0.18823529411764706, + "pass@4": 0.2088235294117647, + "pass@8": 0.24705882352941178, + "pass@16": 0.2735294117647059, + "pass@32": 0.3088235294117647, + "pass@64": 0.35294117647058826, + "pass@128": 0.39705882352941174 + }, + "19": { + "pass@1": 0.05654761904761905, + "pass@2": 0.0744047619047619, + "pass@4": 0.10416666666666667, + "pass@8": 0.13392857142857142, + "pass@16": 0.17261904761904762, + "pass@32": 0.21428571428571427, + "pass@64": 0.23511904761904762, + "pass@128": 0.25595238095238093 + }, + "20": { + "pass@1": 0.08083832335329341, + "pass@2": 0.11676646706586827, + "pass@4": 0.1377245508982036, + "pass@8": 0.15868263473053892, + "pass@16": 0.18862275449101795, + "pass@32": 0.2125748502994012, + "pass@64": 0.2245508982035928, + "pass@128": 0.25149700598802394 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5783132530120482, + "12": 0.5517241379310345, + "13": 0.453781512605042, + "14": 0.38596491228070173, + "15": 0.31212121212121213, + "16": 0.2535612535612536, + "17": 0.16459627329192547, + "18": 0.14411764705882352, + "19": 0.05654761904761905, + "20": 0.08083832335329341 + }, + "pass@2": { + "11": 0.6566265060240963, + "12": 0.6050156739811913, + "13": 0.5238095238095238, + "14": 0.4298245614035088, + "15": 0.38484848484848483, + "16": 0.3076923076923077, + "17": 0.2236024844720497, + "18": 0.18823529411764706, + "19": 0.0744047619047619, + "20": 0.11676646706586827 + }, + "pass@4": { + "11": 0.7379518072289156, + "12": 0.6865203761755486, + "13": 0.6218487394957983, + "14": 0.5087719298245614, + "15": 0.4393939393939394, + "16": 0.37037037037037035, + "17": 0.2577639751552795, + "18": 0.2088235294117647, + "19": 0.10416666666666667, + "20": 0.1377245508982036 + }, + "pass@8": { + "11": 0.8162650602409639, + "12": 0.7554858934169278, + "13": 0.7170868347338936, + "14": 0.5789473684210527, + "15": 0.5121212121212121, + "16": 0.4415954415954416, + "17": 0.30745341614906835, + "18": 0.24705882352941178, + "19": 0.13392857142857142, + "20": 0.15868263473053892 + }, + "pass@16": { + "11": 0.891566265060241, + "12": 0.8087774294670846, + "13": 0.7647058823529411, + "14": 0.6461988304093568, + "15": 0.6, + "16": 0.47293447293447294, + "17": 0.3385093167701863, + "18": 0.2735294117647059, + "19": 0.17261904761904762, + "20": 0.18862275449101795 + }, + "pass@32": { + "11": 0.9457831325301205, + "12": 0.8307210031347962, + "13": 0.7955182072829131, + "14": 0.7017543859649122, + "15": 0.6575757575757576, + "16": 0.5242165242165242, + "17": 0.38509316770186336, + "18": 0.3088235294117647, + "19": 0.21428571428571427, + "20": 0.2125748502994012 + }, + "pass@64": { + "11": 0.9608433734939759, + "12": 0.8432601880877743, + "13": 0.8179271708683473, + "14": 0.7339181286549707, + "15": 0.7151515151515152, + "16": 0.5868945868945868, + "17": 0.4409937888198758, + "18": 0.35294117647058826, + "19": 0.23511904761904762, + "20": 0.2245508982035928 + }, + "pass@128": { + "11": 0.9668674698795181, + "12": 0.8652037617554859, + "13": 0.8487394957983193, + "14": 0.783625730994152, + "15": 0.7606060606060606, + "16": 0.6267806267806267, + "17": 0.4751552795031056, + "18": 0.39705882352941174, + "19": 0.25595238095238093, + "20": 0.25149700598802394 + } + }, + "total_pass_at_k": { + "pass@1": 0.29794826048171275, + "pass@2": 0.3508771929824561, + "pass@4": 0.4073743681236991, + "pass@8": 0.4671424323520666, + "pass@16": 0.5159084151055605, + "pass@32": 0.5578352661314303, + "pass@64": 0.5914362176628011, + "pass@128": 0.623550401427297 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.561, + "pass@2": 0.647, + "pass@4": 0.744, + "pass@8": 0.826, + "pass@16": 0.885, + "pass@32": 0.929, + "pass@64": 0.945, + "pass@128": 0.957 + }, + "12": { + "pass@1": 0.562, + "pass@2": 0.624, + "pass@4": 0.695, + "pass@8": 0.774, + "pass@16": 0.835, + "pass@32": 0.851, + "pass@64": 0.861, + "pass@128": 0.872 + }, + "13": { + "pass@1": 0.454, + "pass@2": 0.518, + "pass@4": 0.604, + "pass@8": 0.674, + "pass@16": 0.728, + "pass@32": 0.766, + "pass@64": 0.785, + "pass@128": 0.81 + }, + "14": { + "pass@1": 0.392, + "pass@2": 0.442, + "pass@4": 0.506, + "pass@8": 0.577, + "pass@16": 0.64, + "pass@32": 0.703, + "pass@64": 0.752, + "pass@128": 0.794 + }, + "15": { + "pass@1": 0.31, + "pass@2": 0.359, + "pass@4": 0.404, + "pass@8": 0.468, + "pass@16": 0.548, + "pass@32": 0.629, + "pass@64": 0.699, + "pass@128": 0.747 + }, + "16": { + "pass@1": 0.239, + "pass@2": 0.297, + "pass@4": 0.356, + "pass@8": 0.406, + "pass@16": 0.452, + "pass@32": 0.504, + "pass@64": 0.561, + "pass@128": 0.606 + }, + "17": { + "pass@1": 0.183, + "pass@2": 0.227, + "pass@4": 0.275, + "pass@8": 0.315, + "pass@16": 0.358, + "pass@32": 0.402, + "pass@64": 0.444, + "pass@128": 0.495 + }, + "18": { + "pass@1": 0.135, + "pass@2": 0.176, + "pass@4": 0.206, + "pass@8": 0.243, + "pass@16": 0.274, + "pass@32": 0.316, + "pass@64": 0.352, + "pass@128": 0.401 + }, + "19": { + "pass@1": 0.06, + "pass@2": 0.079, + "pass@4": 0.105, + "pass@8": 0.133, + "pass@16": 0.163, + "pass@32": 0.192, + "pass@64": 0.223, + "pass@128": 0.249 + }, + "20": { + "pass@1": 0.07, + "pass@2": 0.101, + "pass@4": 0.13, + "pass@8": 0.153, + "pass@16": 0.173, + "pass@32": 0.198, + "pass@64": 0.215, + "pass@128": 0.24 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.561, + "12": 0.562, + "13": 0.454, + "14": 0.392, + "15": 0.31, + "16": 0.239, + "17": 0.183, + "18": 0.135, + "19": 0.06, + "20": 0.07 + }, + "pass@2": { + "11": 0.647, + "12": 0.624, + "13": 0.518, + "14": 0.442, + "15": 0.359, + "16": 0.297, + "17": 0.227, + "18": 0.176, + "19": 0.079, + "20": 0.101 + }, + "pass@4": { + "11": 0.744, + "12": 0.695, + "13": 0.604, + "14": 0.506, + "15": 0.404, + "16": 0.356, + "17": 0.275, + "18": 0.206, + "19": 0.105, + "20": 0.13 + }, + "pass@8": { + "11": 0.826, + "12": 0.774, + "13": 0.674, + "14": 0.577, + "15": 0.468, + "16": 0.406, + "17": 0.315, + "18": 0.243, + "19": 0.133, + "20": 0.153 + }, + "pass@16": { + "11": 0.885, + "12": 0.835, + "13": 0.728, + "14": 0.64, + "15": 0.548, + "16": 0.452, + "17": 0.358, + "18": 0.274, + "19": 0.163, + "20": 0.173 + }, + "pass@32": { + "11": 0.929, + "12": 0.851, + "13": 0.766, + "14": 0.703, + "15": 0.629, + "16": 0.504, + "17": 0.402, + "18": 0.316, + "19": 0.192, + "20": 0.198 + }, + "pass@64": { + "11": 0.945, + "12": 0.861, + "13": 0.785, + "14": 0.752, + "15": 0.699, + "16": 0.561, + "17": 0.444, + "18": 0.352, + "19": 0.223, + "20": 0.215 + }, + "pass@128": { + "11": 0.957, + "12": 0.872, + "13": 0.81, + "14": 0.794, + "15": 0.747, + "16": 0.606, + "17": 0.495, + "18": 0.401, + "19": 0.249, + "20": 0.24 + } + }, + "total_pass_at_k": { + "pass@1": 0.2966, + "pass@2": 0.347, + "pass@4": 0.4025, + "pass@8": 0.4569, + "pass@16": 0.5056, + "pass@32": 0.549, + "pass@64": 0.5837, + "pass@128": 0.6171 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block4-epoch1-op11-14/summary.csv b/composition/SDAR-100M-block4-epoch1-op11-14/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..d27204e3c928fd0f1c8f556ef8c91a9cad20b681 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op11-14/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch1-op11-14,0.9827213541666666,0.615015625,0.7891920230263157,0.0,0.0,0.0,199.76456336805555,367.26526484375,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block4-epoch1-op11-14/summary.json b/composition/SDAR-100M-block4-epoch1-op11-14/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6d3754ec8c3afee4ea27abc4016fc2c6b7d9e498 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op11-14/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch1-op11-14", + "id_acc": 0.9827213541666666, + "ood_acc": 0.615015625, + "total_acc": 0.7891920230263157, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.76456336805555, + "ood_avg_resp_len": 367.26526484375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 640313, + "answer_accuracy": 0.790775420882074, + "avg_response_len": 274.5671521794973, + "resp_tokens_sum": 222324711, + "pass_at_k": { + "pass@1": 0.790775420882074, + "pass@2": 0.8310315267138889, + "pass@4": 0.8607817283045475, + "pass@8": 0.8837751849257428, + "pass@16": 0.902361735206419, + "pass@32": 0.9174584369632929, + "pass@64": 0.9296724648662624, + "pass@128": 0.9399304457793234 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9259080014749262, + "pass@2": 0.9433224020393468, + "pass@4": 0.9555353711317055, + "pass@8": 0.9626092113862927, + "pass@16": 0.9672899140408969, + "pass@32": 0.971873515401767, + "pass@64": 0.9768429344531748, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 0.9999513239875389, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999044342507645, + "pass@2": 0.9999977425413568, + "pass@4": 0.9999999997133384, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9938720703125, + "pass@2": 0.9946638933316929, + "pass@4": 0.995454402184102, + "pass@8": 0.9966304348689696, + "pass@16": 0.9980442612102862, + "pass@32": 0.9992470856726705, + "pass@64": 0.9999099032277166, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9814980158730159, + "pass@2": 0.9859626921634798, + "pass@4": 0.9891130867570126, + "pass@8": 0.9915862161820935, + "pass@16": 0.9933890901170934, + "pass@32": 0.9943969337435874, + "pass@64": 0.9952380498015997, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.992374625748503, + "pass@2": 0.9937386545806025, + "pass@4": 0.9939952238380383, + "pass@8": 0.9940119126755125, + "pass@16": 0.9940119760476699, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9904539571005917, + "pass@2": 0.9942641114941992, + "pass@4": 0.995715497378656, + "pass@8": 0.9967252188259338, + "pass@16": 0.9975999557269617, + "pass@32": 0.9983364053352927, + "pass@64": 0.9992661788080938, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9777393830128205, + "pass@2": 0.9862255987532803, + "pass@4": 0.990327881490775, + "pass@8": 0.9937928780958446, + "pass@16": 0.996912117506564, + "pass@32": 0.9990931142481682, + "pass@64": 0.9999455717474616, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9797535211267606, + "pass@2": 0.9886772901186649, + "pass@4": 0.9919861029695232, + "pass@8": 0.9939178049019348, + "pass@16": 0.9951668490880456, + "pass@32": 0.996002421656276, + "pass@64": 0.9968393035287623, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9369688385269122, + "pass@2": 0.9556891436729048, + "pass@4": 0.9681532095138248, + "pass@8": 0.9773977476860249, + "pass@16": 0.9836760616393073, + "pass@32": 0.9879154034727532, + "pass@64": 0.9905512293835983, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.896340925141243, + "pass@2": 0.9266628019484855, + "pass@4": 0.9416773099337153, + "pass@8": 0.9499502893000226, + "pass@16": 0.9556055452848286, + "pass@32": 0.9601108862491841, + "pass@64": 0.9653730933928352, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8189410266457681, + "pass@2": 0.8701832590773324, + "pass@4": 0.9008475028003936, + "pass@8": 0.9223142658352539, + "pass@16": 0.9375126996972292, + "pass@32": 0.9461958750908102, + "pass@64": 0.9519937435789704, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.8048091900311527, + "pass@2": 0.8633829138764197, + "pass@4": 0.8936096031220395, + "pass@8": 0.9133137052944442, + "pass@16": 0.9308204065370922, + "pass@32": 0.9458697424997475, + "pass@64": 0.9578949685863664, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.7142792630057804, + "pass@2": 0.7852296777570432, + "pass@4": 0.8336160666874729, + "pass@8": 0.8703838255639478, + "pass@16": 0.8991492741726431, + "pass@32": 0.9217285219094363, + "pass@64": 0.9390414573776548, + "pass@128": 0.9508670520231214 + }, + "16": { + "pass@1": 0.5767389112903226, + "pass@2": 0.671137207899416, + "pass@4": 0.7352816832371761, + "pass@8": 0.7801929643943304, + "pass@16": 0.8153526018049102, + "pass@32": 0.8451755884363006, + "pass@64": 0.8721477928442585, + "pass@128": 0.896774193548387 + }, + "17": { + "pass@1": 0.49742593930635837, + "pass@2": 0.6042979950844295, + "pass@4": 0.6877961173386561, + "pass@8": 0.750236429513258, + "pass@16": 0.797521632533749, + "pass@32": 0.8336931159590578, + "pass@64": 0.861816202176084, + "pass@128": 0.8872832369942196 + }, + "18": { + "pass@1": 0.4068080357142857, + "pass@2": 0.5067449381327332, + "pass@4": 0.5904474240719905, + "pass@8": 0.6602663145937516, + "pass@16": 0.7184114317717843, + "pass@32": 0.7636522860956406, + "pass@64": 0.7975986551347283, + "pass@128": 0.8228571428571428 + }, + "19": { + "pass@1": 0.28638795045045046, + "pass@2": 0.37785669527795496, + "pass@4": 0.45953530245656216, + "pass@8": 0.5290022873626415, + "pass@16": 0.5902007338149695, + "pass@32": 0.6466124054432371, + "pass@64": 0.6966004194500958, + "pass@128": 0.7417417417417418 + }, + "20": { + "pass@1": 0.2728978978978979, + "pass@2": 0.36531228571090785, + "pass@4": 0.4511123266123264, + "pass@8": 0.5250073329043563, + "pass@16": 0.5869595546727128, + "pass@32": 0.638416275402605, + "pass@64": 0.6777993410094819, + "pass@128": 0.7057057057057057 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 648988, + "answer_accuracy": 0.7903692517537022, + "avg_response_len": 284.4923129384256, + "resp_tokens_sum": 233602328, + "pass_at_k": { + "pass@1": 0.7903692517537022, + "pass@2": 0.8308205308976789, + "pass@4": 0.8615349426362584, + "pass@8": 0.8856540052607581, + "pass@16": 0.9049316985397561, + "pass@32": 0.9202179500084126, + "pass@64": 0.9324577617228758, + "pass@128": 0.9429462197973499 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9163391487730062, + "pass@2": 0.9258629564996861, + "pass@4": 0.9336692571212339, + "pass@8": 0.9416640558086454, + "pass@16": 0.9500473404479759, + "pass@32": 0.9578994674031669, + "pass@64": 0.9655855093279391, + "pass@128": 0.9754601226993865 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982717803030303, + "pass@2": 0.9995746092817943, + "pass@4": 0.9999632247673588, + "pass@8": 0.9999996922182559, + "pass@16": 0.9999999999913911, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9986131656804734, + "pass@2": 0.999854036597866, + "pass@4": 0.9999976565429319, + "pass@8": 0.9999999993354288, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9863224637681159, + "pass@2": 0.9906003937007875, + "pass@4": 0.9930274229308294, + "pass@8": 0.9950581299215392, + "pass@16": 0.9969160592890997, + "pass@32": 0.998278697645461, + "pass@64": 0.9992808085046374, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967765042979942, + "pass@2": 0.9973253869322924, + "pass@4": 0.9975613690301607, + "pass@8": 0.9979359856831456, + "pass@16": 0.9985470770655008, + "pass@32": 0.9993380089666443, + "pass@64": 0.9999174243620693, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9916230130057804, + "pass@2": 0.9936738342587959, + "pass@4": 0.9944413127911034, + "pass@8": 0.9949076292657711, + "pass@16": 0.9955414667011108, + "pass@32": 0.9966227126233496, + "pass@64": 0.9982021755951027, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9815140845070423, + "pass@2": 0.9889029056227125, + "pass@4": 0.9929117410499744, + "pass@8": 0.9949431586627508, + "pass@16": 0.9962341274090126, + "pass@32": 0.9974052544005088, + "pass@64": 0.9985526092972625, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9739427395209581, + "pass@2": 0.9802412890753925, + "pass@4": 0.9834003469439074, + "pass@8": 0.9861000031633753, + "pass@16": 0.9883617473597814, + "pass@32": 0.9900186779076805, + "pass@64": 0.9917721721639942, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.942371046686747, + "pass@2": 0.9614907711554884, + "pass@4": 0.9709818822534232, + "pass@8": 0.9762108984873451, + "pass@16": 0.9798418328465113, + "pass@32": 0.9832802016718227, + "pass@64": 0.9869836840656171, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8921189263322884, + "pass@2": 0.9154603923431991, + "pass@4": 0.9262470747191082, + "pass@8": 0.932674949530915, + "pass@16": 0.9378135612950852, + "pass@32": 0.942572203878569, + "pass@64": 0.947465209542479, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7801995798319328, + "pass@2": 0.8263980789166057, + "pass@4": 0.8551042666410398, + "pass@8": 0.8787522658307619, + "pass@16": 0.8997253159536589, + "pass@32": 0.9175379858563458, + "pass@64": 0.9326565965286922, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.7647112573099415, + "pass@2": 0.8316522626283556, + "pass@4": 0.8720206605205054, + "pass@8": 0.899213977549253, + "pass@16": 0.9185417582132726, + "pass@32": 0.9327907985067769, + "pass@64": 0.9426059507605182, + "pass@128": 0.9502923976608187 + }, + "15": { + "pass@1": 0.6902225378787878, + "pass@2": 0.7702934860415173, + "pass@4": 0.8277060339048522, + "pass@8": 0.870181513451221, + "pass@16": 0.9009560991102984, + "pass@32": 0.9213126526056608, + "pass@64": 0.9341966745085428, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.5663951210826211, + "pass@2": 0.6729072728088475, + "pass@4": 0.747877350267114, + "pass@8": 0.8014397473398795, + "pass@16": 0.8403655118855234, + "pass@32": 0.8670101699239859, + "pass@64": 0.8852311741120641, + "pass@128": 0.9002849002849003 + }, + "17": { + "pass@1": 0.499708850931677, + "pass@2": 0.6042562020100748, + "pass@4": 0.6857853701905272, + "pass@8": 0.7446535294212936, + "pass@16": 0.787459898111565, + "pass@32": 0.8184923783964188, + "pass@64": 0.840558376080876, + "pass@128": 0.860248447204969 + }, + "18": { + "pass@1": 0.47116268382352944, + "pass@2": 0.5732706837656322, + "pass@4": 0.6578277650036389, + "pass@8": 0.7272127940450731, + "pass@16": 0.7824553129641325, + "pass@32": 0.8258777832780535, + "pass@64": 0.858667717455225, + "pass@128": 0.8823529411764706 + }, + "19": { + "pass@1": 0.2761346726190476, + "pass@2": 0.365666449896888, + "pass@4": 0.45070808615440894, + "pass@8": 0.529863381029973, + "pass@16": 0.6031771625339937, + "pass@32": 0.6680454701301732, + "pass@64": 0.7240389650730391, + "pass@128": 0.7708333333333334 + }, + "20": { + "pass@1": 0.277156624251497, + "pass@2": 0.3758873738742987, + "pass@4": 0.46919662278367874, + "pass@8": 0.547041580122116, + "pass@16": 0.608975135388662, + "pass@32": 0.6595693283084222, + "pass@64": 0.7034790052169011, + "pass@128": 0.7455089820359282 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 630014, + "answer_accuracy": 0.7863851054481547, + "avg_response_len": 304.9374862697715, + "resp_tokens_sum": 244301277, + "pass_at_k": { + "pass@1": 0.7863851054481547, + "pass@2": 0.8276325988214681, + "pass@4": 0.8593780909146573, + "pass@8": 0.8839933436948046, + "pass@16": 0.902981023040986, + "pass@32": 0.917834536552863, + "pass@64": 0.9295588405832169, + "pass@128": 0.9389678862438089 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9421175373134328, + "pass@2": 0.9587580062286991, + "pass@4": 0.9685092193699665, + "pass@8": 0.9760498065360441, + "pass@16": 0.9825003934245147, + "pass@32": 0.9871221085392369, + "pass@64": 0.9894357905723764, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9999330357142857, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9966517857142857, + "pass@2": 0.9971243629622828, + "pass@4": 0.9972639609917566, + "pass@8": 0.99743893615483, + "pass@16": 0.9977703679897155, + "pass@32": 0.9983643626179382, + "pass@64": 0.9992768761047727, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9989720394736842, + "pass@2": 0.9998701334208223, + "pass@4": 0.9999958955240242, + "pass@8": 0.9999999968048913, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.974172794117647, + "pass@2": 0.9799802425891616, + "pass@4": 0.9851549995588786, + "pass@8": 0.9903396378716414, + "pass@16": 0.9949618620195534, + "pass@32": 0.9978909857983171, + "pass@64": 0.9992638569390833, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.98957511829653, + "pass@2": 0.9909034315308379, + "pass@4": 0.9923643261075016, + "pass@8": 0.9937184520602704, + "pass@16": 0.9949870433431968, + "pass@32": 0.9963097387793499, + "pass@64": 0.9980377056438621, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.984375, + "pass@2": 0.9912449074304795, + "pass@4": 0.994530023244721, + "pass@8": 0.9967387415639558, + "pass@16": 0.9985927711809854, + "pass@32": 0.9996839030827411, + "pass@64": 0.9999901863988568, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9845626876876877, + "pass@2": 0.9914509736114068, + "pass@4": 0.9942883507804767, + "pass@8": 0.9953259254730494, + "pass@16": 0.9963787369181493, + "pass@32": 0.9978019463673319, + "pass@64": 0.9992136480632735, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9767132234726688, + "pass@2": 0.9858248727751473, + "pass@4": 0.9893246072335813, + "pass@8": 0.9916823703543539, + "pass@16": 0.9934710727259167, + "pass@32": 0.995379009876187, + "pass@64": 0.9975736224486328, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9344246031746032, + "pass@2": 0.956496453568304, + "pass@4": 0.9680845546092448, + "pass@8": 0.9746013817828894, + "pass@16": 0.979128362963511, + "pass@32": 0.9823612720845869, + "pass@64": 0.9839103382187553, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8939698012232415, + "pass@2": 0.9233999133135886, + "pass@4": 0.940026338164151, + "pass@8": 0.9508489849950189, + "pass@16": 0.9588264089267363, + "pass@32": 0.9654115182769986, + "pass@64": 0.9710843168080171, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8054591049382716, + "pass@2": 0.8580877260134149, + "pass@4": 0.8874315097186924, + "pass@8": 0.9042751337372295, + "pass@16": 0.9160205919035335, + "pass@32": 0.9253222868446103, + "pass@64": 0.9338060490002721, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7650686201780416, + "pass@2": 0.8257855761816872, + "pass@4": 0.8638935942539817, + "pass@8": 0.8898587056265527, + "pass@16": 0.909471720507359, + "pass@32": 0.9247618852817582, + "pass@64": 0.9359455063598505, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6778308256172839, + "pass@2": 0.7494824323174885, + "pass@4": 0.801531108958602, + "pass@8": 0.8410958117430899, + "pass@16": 0.8703430132240998, + "pass@32": 0.8920039907677088, + "pass@64": 0.9082821137578678, + "pass@128": 0.9197530864197531 + }, + "16": { + "pass@1": 0.5564850663716814, + "pass@2": 0.659352280909577, + "pass@4": 0.7384296702956372, + "pass@8": 0.7974724410886839, + "pass@16": 0.8379566282804083, + "pass@32": 0.867540734508324, + "pass@64": 0.8941426993490582, + "pass@128": 0.9174041297935103 + }, + "17": { + "pass@1": 0.44517131024096385, + "pass@2": 0.5476650697277298, + "pass@4": 0.630717065393181, + "pass@8": 0.696161286495951, + "pass@16": 0.7486587759712936, + "pass@32": 0.7898353056767135, + "pass@64": 0.8198061896401653, + "pass@128": 0.8433734939759037 + }, + "18": { + "pass@1": 0.44601814516129035, + "pass@2": 0.5429264827279653, + "pass@4": 0.6179406434074768, + "pass@8": 0.6738175548059356, + "pass@16": 0.7170866104454663, + "pass@32": 0.7528154066701483, + "pass@64": 0.7842687233254886, + "pass@128": 0.8129032258064516 + }, + "19": { + "pass@1": 0.28757552870090636, + "pass@2": 0.3801985453291147, + "pass@4": 0.4683336989409553, + "pass@8": 0.5477159165532208, + "pass@16": 0.6142955679985431, + "pass@32": 0.6685448647183322, + "pass@64": 0.7121010913154943, + "pass@128": 0.7462235649546828 + }, + "20": { + "pass@1": 0.2712321696696697, + "pass@2": 0.377846350287295, + "pass@4": 0.48275687892617025, + "pass@8": 0.5715013261561981, + "pass@16": 0.6392863359899448, + "pass@32": 0.6912394364961897, + "pass@64": 0.7295567463044594, + "pass@128": 0.7567567567567568 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_metrics.json b/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..cb719495580e8e4a2cf391c62c1c364b0b583a1b --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch1-op17-20", + "id": { + "answer_accuracy": 0.9801223958333334, + "avg_response_len": 199.76062586805557, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9801223958333334, + "pass@2": 0.9871357447506511, + "pass@4": 0.9909350812919213, + "pass@8": 0.9933636757087748, + "pass@16": 0.9950767615885424, + "pass@32": 0.9964329006344513, + "pass@64": 0.997616406382146, + "pass@128": 0.9984444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9231171875, + "pass@2": 0.9456815944881896, + "pass@4": 0.9603091880389951, + "pass@8": 0.9705846474455383, + "pass@16": 0.9782733645292219, + "pass@32": 0.984408624764542, + "pass@64": 0.9896463722943792, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.9999453125, + "pass@2": 0.9999987696850394, + "pass@4": 0.9999999995313086, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9976875, + "pass@2": 0.9987242864173229, + "pass@4": 0.9990599458192727, + "pass@8": 0.9991769200116146, + "pass@16": 0.9993323490285145, + "pass@32": 0.9995814585676791, + "pass@64": 0.9998779527559055, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9980625, + "pass@2": 0.9988898868110238, + "pass@4": 0.9991574159167605, + "pass@8": 0.9993262463197925, + "pass@16": 0.9995588902104531, + "pass@32": 0.9998290717623712, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.978015625, + "pass@2": 0.9848229576771658, + "pass@4": 0.988770462129734, + "pass@8": 0.991504906288707, + "pass@16": 0.993650861087387, + "pass@32": 0.9955520574897151, + "pass@64": 0.9970482140193007, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.9920078125, + "pass@2": 0.9939662893700788, + "pass@4": 0.9950509417885265, + "pass@8": 0.9958147741453353, + "pass@16": 0.9964508524538155, + "pass@32": 0.9971523647058177, + "pass@64": 0.99794013537104, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9861875, + "pass@2": 0.9924363927165356, + "pass@4": 0.9952590014998124, + "pass@8": 0.9965242680443928, + "pass@16": 0.9971608534876047, + "pass@32": 0.9976626144839646, + "pass@64": 0.9982518300815967, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.97428125, + "pass@2": 0.9852411417322838, + "pass@4": 0.99118949015748, + "pass@8": 0.9948321772055063, + "pass@16": 0.9971259158175276, + "pass@32": 0.998483992181002, + "pass@64": 0.9994439058232791, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.971796875, + "pass@2": 0.9844603838582684, + "pass@4": 0.9896192867454066, + "pass@8": 0.9925091419180848, + "pass@16": 0.9941377676823531, + "pass@32": 0.9952259217549699, + "pass@64": 0.9963530707926768, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.9231171875, + "2": 0.9999453125, + "3": 0.9976875, + "4": 0.9980625, + "5": 0.978015625, + "6": 0.9920078125, + "7": 0.9861875, + "8": 0.97428125, + "9": 0.971796875 + }, + "per_op_avg_response_len": { + "10": 279.0174609375, + "2": 115.26925, + "3": 144.67940625, + "4": 159.4561328125, + "5": 174.2389375, + "6": 198.7633125, + "7": 223.944734375, + "8": 243.9508515625, + "9": 258.525546875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.0174609375, + "2": 115.26925, + "3": 144.67940625, + "4": 159.4561328125, + "5": 174.2389375, + "6": 198.7633125, + "7": 223.944734375, + "8": 243.9508515625, + "9": 258.525546875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 371099, + "answer_accuracy": 0.979132366599122, + "avg_response_len": 190.26763287318474, + "resp_tokens_sum": 72112955, + "pass_at_k": { + "pass@1": 0.979132366599122, + "pass@2": 0.9861054602217292, + "pass@4": 0.9899340251224931, + "pass@8": 0.9924987218896144, + "pass@16": 0.9943021555612757, + "pass@32": 0.9957042334254549, + "pass@64": 0.9968508765699006, + "pass@128": 0.9976359338061466 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9192477876106194, + "pass@2": 0.9396735564304464, + "pass@4": 0.9543033865788902, + "pass@8": 0.96485112476411, + "pass@16": 0.9724661046772268, + "pass@32": 0.9786862175801799, + "pass@64": 0.9841720997110378, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.9998296339563862, + "pass@2": 0.9999961672431132, + "pass@4": 0.9999999985399022, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995460626911316, + "pass@2": 0.999965009391028, + "pass@4": 0.9999997130519236, + "pass@8": 0.9999999999935767, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995849609375, + "pass@2": 0.996728515625, + "pass@4": 0.9973700640935508, + "pass@8": 0.9978945203621713, + "pass@16": 0.9986215319076661, + "pass@32": 0.99946584925741, + "pass@64": 0.999956800941041, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9784474206349206, + "pass@2": 0.9849374296962881, + "pass@4": 0.9890754283690729, + "pass@8": 0.9921342884438079, + "pass@16": 0.9943983611486136, + "pass@32": 0.9960648708043484, + "pass@64": 0.9967769255677263, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9922342814371258, + "pass@2": 0.9939438298599651, + "pass@4": 0.9941038252516338, + "pass@8": 0.9941991013702666, + "pass@16": 0.9943862275449101, + "pass@32": 0.9947604790419161, + "pass@64": 0.9955089820359282, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9887435281065089, + "pass@2": 0.9931025922517821, + "pass@4": 0.995356032973689, + "pass@8": 0.99687459097789, + "pass@16": 0.997698902492233, + "pass@32": 0.9983400676934957, + "pass@64": 0.9992661790057259, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9668469551282052, + "pass@2": 0.9809139505097918, + "pass@4": 0.9884821623859518, + "pass@8": 0.9934550534186496, + "pass@16": 0.9968600221493318, + "pass@32": 0.9988651869448563, + "pass@64": 0.9998085724581219, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9736575704225352, + "pass@2": 0.987104566374626, + "pass@4": 0.9918345429532572, + "pass@8": 0.9940607002019898, + "pass@16": 0.9952133611440159, + "pass@32": 0.9960034370145185, + "pass@64": 0.9968393035368946, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 383005, + "answer_accuracy": 0.980414994266055, + "avg_response_len": 197.6951153956422, + "resp_tokens_sum": 77230783, + "pass_at_k": { + "pass@1": 0.980414994266055, + "pass@2": 0.9862497387797863, + "pass@4": 0.9896156322837595, + "pass@8": 0.9920707717579302, + "pass@16": 0.993936643245879, + "pass@32": 0.9954022931445885, + "pass@64": 0.9968295599606264, + "pass@128": 0.9980340760157274 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9164350076687117, + "pass@2": 0.9361730713492108, + "pass@4": 0.9499516876916461, + "pass@8": 0.9618509716297367, + "pass@16": 0.9718630754382047, + "pass@32": 0.9796931696551145, + "pass@64": 0.986608430448021, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9973958333333334, + "pass@2": 0.9992700131233596, + "pass@4": 0.9999176963674994, + "pass@8": 0.9999987979293221, + "pass@16": 0.9999999998399193, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990292159763313, + "pass@2": 0.9998897084517542, + "pass@4": 0.9999981377142946, + "pass@8": 0.9999999995762411, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.980570652173913, + "pass@2": 0.9865802949903003, + "pass@4": 0.9889661506985535, + "pass@8": 0.9902841199998196, + "pass@16": 0.9917663843037684, + "pass@32": 0.9936594509266741, + "pass@64": 0.9954604899202021, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.995008058739255, + "pass@2": 0.9963252713038379, + "pass@4": 0.9970167348350797, + "pass@8": 0.9973006814242409, + "pass@16": 0.997492802179556, + "pass@32": 0.9978510028653035, + "pass@64": 0.998567335243553, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9898843930635838, + "pass@2": 0.9931856196804879, + "pass@4": 0.9941058232063475, + "pass@8": 0.9943886704848257, + "pass@16": 0.9945808983691612, + "pass@32": 0.9949421965317783, + "pass@64": 0.9956647398843931, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9714788732394366, + "pass@2": 0.980618207275147, + "pass@4": 0.9870130438800786, + "pass@8": 0.9915782021625016, + "pass@16": 0.9946980159453276, + "pass@32": 0.9967270976372445, + "pass@64": 0.9986017780742946, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9722352170658682, + "pass@2": 0.9828459468857563, + "pass@4": 0.9884671896364751, + "pass@8": 0.9924020194851084, + "pass@16": 0.9944314239658919, + "pass@32": 0.9953209435293606, + "pass@64": 0.9962633785477046, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 374997, + "answer_accuracy": 0.9808048418145296, + "avg_response_len": 211.28144616253766, + "resp_tokens_sum": 80780503, + "pass_at_k": { + "pass@1": 0.9808048418145296, + "pass@2": 0.9890623476007588, + "pass@4": 0.993275585373138, + "pass@8": 0.9955421394237094, + "pass@16": 0.9970095535632802, + "pass@32": 0.9982082597455659, + "pass@64": 0.9991792416859733, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.933535447761194, + "pass@2": 0.9610144405923143, + "pass@4": 0.9764659397052976, + "pass@8": 0.9848856698483921, + "pass@16": 0.9903880371665308, + "pass@32": 0.994788130439683, + "pass@64": 0.9981423348247224, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961962463556852, + "pass@2": 0.9970163965473705, + "pass@4": 0.997338775072941, + "pass@8": 0.9976015064053606, + "pass@16": 0.9980534958639686, + "pass@32": 0.9987797625879855, + "pass@64": 0.9996441771309199, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.999924094603306, + "pass@4": 0.9999989030976391, + "pass@8": 0.999999999845404, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9750229779411764, + "pass@2": 0.9829337222093564, + "pass@4": 0.9882893535366903, + "pass@8": 0.9921605412616765, + "pass@16": 0.9948705139434515, + "pass@32": 0.9969973900489489, + "pass@64": 0.998910627773521, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9884660883280757, + "pass@2": 0.9913928407312652, + "pass@4": 0.9938846173408135, + "pass@8": 0.9958811876044368, + "pass@16": 0.9974790678017365, + "pass@32": 0.9989033586934598, + "pass@64": 0.9998111525900316, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9794056566455697, + "pass@2": 0.9909034560948872, + "pass@4": 0.9964178719155359, + "pass@8": 0.9984878997029001, + "pass@16": 0.9994102329414561, + "pass@32": 0.9999166822897716, + "pass@64": 0.9999995619546246, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9842342342342343, + "pass@2": 0.9942238005722256, + "pass@4": 0.9981784532001067, + "pass@8": 0.9995914077213195, + "pass@16": 0.9999633430821163, + "pass@32": 0.9999998023813371, + "pass@64": 0.9999999999999107, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9692021704180064, + "pass@2": 0.983175937412968, + "pass@4": 0.9883279185158123, + "pass@8": 0.9908531152358594, + "pass@16": 0.9925946265968474, + "pass@32": 0.9942363552283263, + "pass@64": 0.9958943717110796, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.62273671875, + "avg_response_len": 372.483859375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.62273671875, + "pass@2": 0.7036788877952637, + "pass@4": 0.7644948062710833, + "pass@8": 0.8099061159706736, + "pass@16": 0.8439089202647532, + "pass@32": 0.8701182617095065, + "pass@64": 0.891005192451688, + "pass@128": 0.9073 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9151171875, + "pass@2": 0.9456545275590561, + "pass@4": 0.9612200796775406, + "pass@8": 0.9704336964892437, + "pass@16": 0.9765719952540661, + "pass@32": 0.9812601474160046, + "pass@64": 0.9858234208718162, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.8489609375, + "pass@2": 0.8991227854330703, + "pass@4": 0.9283502003187095, + "pass@8": 0.9455536075739712, + "pass@16": 0.9560643833010687, + "pass@32": 0.9634465601168278, + "pass@64": 0.9699825720446108, + "pass@128": 0.976 + }, + "13": { + "pass@1": 0.7550703125, + "pass@2": 0.8208409202755905, + "pass@4": 0.8626456775403079, + "pass@8": 0.890856563637159, + "pass@16": 0.9113103719914247, + "pass@32": 0.9282444428549655, + "pass@64": 0.9434633808607322, + "pass@128": 0.956 + }, + "14": { + "pass@1": 0.7476953125, + "pass@2": 0.8176723671259848, + "pass@4": 0.8618266462317206, + "pass@8": 0.8918874643176163, + "pass@16": 0.9141369979757303, + "pass@32": 0.9316738111035253, + "pass@64": 0.9455826385211541, + "pass@128": 0.956 + }, + "15": { + "pass@1": 0.688828125, + "pass@2": 0.7694915108267728, + "pass@4": 0.8278998459880011, + "pass@8": 0.8700518416925825, + "pass@16": 0.8990736716148421, + "pass@32": 0.9185506095706097, + "pass@64": 0.9329091553818495, + "pass@128": 0.944 + }, + "16": { + "pass@1": 0.569984375, + "pass@2": 0.6710280511811033, + "pass@4": 0.7460000776152982, + "pass@8": 0.8009541520148893, + "pass@16": 0.8402356139913965, + "pass@32": 0.8684268111847793, + "pass@64": 0.8891133101235712, + "pass@128": 0.905 + }, + "17": { + "pass@1": 0.5203671875, + "pass@2": 0.6227608267716539, + "pass@4": 0.7001498685789274, + "pass@8": 0.7556166147417005, + "pass@16": 0.7964709061393579, + "pass@32": 0.8294317709926918, + "pass@64": 0.8556712149259554, + "pass@128": 0.873 + }, + "18": { + "pass@1": 0.4865859375, + "pass@2": 0.5871370570866156, + "pass@4": 0.6677094760030015, + "pass@8": 0.7302382482448312, + "pass@16": 0.777847828734549, + "pass@32": 0.8141039437356614, + "pass@64": 0.8426591360477864, + "pass@128": 0.867 + }, + "19": { + "pass@1": 0.3416015625, + "pass@2": 0.4410281742125985, + "pass@4": 0.5310940967379066, + "pass@8": 0.6078161499766409, + "pass@16": 0.672866477645552, + "pass@32": 0.728375162906233, + "pass@64": 0.7747117439388957, + "pass@128": 0.812 + }, + "20": { + "pass@1": 0.35315625, + "pass@2": 0.4620526574803147, + "pass@4": 0.5580520940194974, + "pass@8": 0.6356528210181952, + "pass@16": 0.6945109559994428, + "pass@32": 0.7376693572137135, + "pass@64": 0.7701353518004574, + "pass@128": 0.794 + } + }, + "per_op_accuracy": { + "11": 0.9151171875, + "12": 0.8489609375, + "13": 0.7550703125, + "14": 0.7476953125, + "15": 0.688828125, + "16": 0.569984375, + "17": 0.5203671875, + "18": 0.4865859375, + "19": 0.3416015625, + "20": 0.35315625 + }, + "per_op_avg_response_len": { + "11": 297.70178125, + "12": 307.7838359375, + "13": 325.8640703125, + "14": 352.953453125, + "15": 372.9394609375, + "16": 377.9108046875, + "17": 403.39990625, + "18": 419.2847265625, + "19": 419.6512578125, + "20": 447.349296875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.70178125, + "12": 307.7838359375, + "13": 325.8640703125, + "14": 352.953453125, + "15": 372.9394609375, + "16": 377.9108046875, + "17": 403.39990625, + "18": 419.2847265625, + "19": 419.6512578125, + "20": 447.349296875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 268987, + "answer_accuracy": 0.6245054791976226, + "avg_response_len": 354.4513581909361, + "resp_tokens_sum": 152669289, + "pass_at_k": { + "pass@1": 0.6245054791976226, + "pass@2": 0.7043694279346253, + "pass@4": 0.7637833167306537, + "pass@8": 0.8083928321539314, + "pass@16": 0.842271539248331, + "pass@32": 0.8685320453378127, + "pass@64": 0.889781440443194, + "pass@128": 0.9069836552748886 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9049442280453258, + "pass@2": 0.9343667049586221, + "pass@4": 0.9518926340829211, + "pass@8": 0.963721513224062, + "pass@16": 0.9715522567392245, + "pass@32": 0.9772850487323939, + "pass@64": 0.9837385106596693, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8367319915254238, + "pass@2": 0.8906847780150366, + "pass@4": 0.9210060177859121, + "pass@8": 0.9395740180895393, + "pass@16": 0.9513328654341852, + "pass@32": 0.9589354102848824, + "pass@64": 0.9654301877762941, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.7604574843260188, + "pass@2": 0.82680597894503, + "pass@4": 0.8719512272956471, + "pass@8": 0.9045499773982028, + "pass@16": 0.9266465799635933, + "pass@32": 0.9410437423571842, + "pass@64": 0.9509158308973618, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.7687889408099688, + "pass@2": 0.8359775523094659, + "pass@4": 0.8736571856672118, + "pass@8": 0.8984409718591754, + "pass@16": 0.9181805371311775, + "pass@32": 0.9350351705266242, + "pass@64": 0.9502913100353946, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.6999638728323699, + "pass@2": 0.7756271761640345, + "pass@4": 0.8292882124380401, + "pass@8": 0.869390789970966, + "pass@16": 0.8979501614129438, + "pass@32": 0.9183570437604774, + "pass@64": 0.9343777336335435, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.5773941532258065, + "pass@2": 0.6756885795021594, + "pass@4": 0.7475061202027162, + "pass@8": 0.7991479896927378, + "pass@16": 0.8357307006684896, + "pass@32": 0.8628873839423076, + "pass@64": 0.8835885830672884, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.5140895953757225, + "pass@2": 0.6143314624960177, + "pass@4": 0.6916131304576808, + "pass@8": 0.7469650535201684, + "pass@16": 0.7860427028084981, + "pass@32": 0.8170511372956204, + "pass@64": 0.8421452842507103, + "pass@128": 0.8583815028901735 + }, + "18": { + "pass@1": 0.4582589285714286, + "pass@2": 0.5605532902137236, + "pass@4": 0.6421960956666121, + "pass@8": 0.7057777001868978, + "pass@16": 0.7555140844320166, + "pass@32": 0.7930716900033024, + "pass@64": 0.822862301855235, + "pass@128": 0.8514285714285714 + }, + "19": { + "pass@1": 0.3429288663663664, + "pass@2": 0.44343987195857243, + "pass@4": 0.531017780998096, + "pass@8": 0.602912347397304, + "pass@16": 0.6667933588207599, + "pass@32": 0.725113626356638, + "pass@64": 0.775691031987435, + "pass@128": 0.8168168168168168 + }, + "20": { + "pass@1": 0.36878284534534533, + "pass@2": 0.4766399026979738, + "pass@4": 0.570342090403114, + "pass@8": 0.6477507011049768, + "pass@16": 0.7086113239762887, + "pass@32": 0.7532795279937846, + "pass@64": 0.7862179025168473, + "pass@128": 0.8108108108108109 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 268409, + "answer_accuracy": 0.6235341399048469, + "avg_response_len": 367.8219200676479, + "resp_tokens_sum": 158334095, + "pass_at_k": { + "pass@1": 0.6235341399048469, + "pass@2": 0.7053542370540048, + "pass@4": 0.7665724703123105, + "pass@8": 0.8122417615531752, + "pass@16": 0.8466933293355862, + "pass@32": 0.8734423092418954, + "pass@64": 0.8947083272749418, + "pass@128": 0.9107939339875112 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9257341867469879, + "pass@2": 0.9553495725026092, + "pass@4": 0.9682735500494665, + "pass@8": 0.9742548441493746, + "pass@16": 0.978185160269226, + "pass@32": 0.9822869779847758, + "pass@64": 0.9871868036674503, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8492358934169278, + "pass@2": 0.8949850973761508, + "pass@4": 0.9207448059000464, + "pass@8": 0.9354093985604018, + "pass@16": 0.943759425744798, + "pass@32": 0.9496060392426146, + "pass@64": 0.9561026755224603, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.740983893557423, + "pass@2": 0.8052852538653259, + "pass@4": 0.845815909040781, + "pass@8": 0.8738550513505369, + "pass@16": 0.895515341616072, + "pass@32": 0.9155451542423412, + "pass@64": 0.9353134265154689, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7266081871345029, + "pass@2": 0.7990618668554583, + "pass@4": 0.8478731022109073, + "pass@8": 0.8837960270998939, + "pass@16": 0.9105606572428289, + "pass@32": 0.930775272521883, + "pass@64": 0.9463526228698942, + "pass@128": 0.9590643274853801 + }, + "15": { + "pass@1": 0.6870501893939394, + "pass@2": 0.7726907361011697, + "pass@4": 0.8339367397257161, + "pass@8": 0.8770493599523325, + "pass@16": 0.9072087108288012, + "pass@32": 0.9277764122574453, + "pass@64": 0.9424641653105227, + "pass@128": 0.9545454545454546 + }, + "16": { + "pass@1": 0.5663728632478633, + "pass@2": 0.668794655876349, + "pass@4": 0.74511328258113, + "pass@8": 0.8007343935830623, + "pass@16": 0.8421667071713081, + "pass@32": 0.8733083123762971, + "pass@64": 0.8957480080042212, + "pass@128": 0.9116809116809117 + }, + "17": { + "pass@1": 0.5376067546583851, + "pass@2": 0.6423940553626447, + "pass@4": 0.7202086238637875, + "pass@8": 0.7743041381948508, + "pass@16": 0.814345713501841, + "pass@32": 0.8470088033255796, + "pass@64": 0.8726913345902738, + "pass@128": 0.8881987577639752 + }, + "18": { + "pass@1": 0.5092830882352941, + "pass@2": 0.6142293162343674, + "pass@4": 0.7005960897351063, + "pass@8": 0.7701591745990866, + "pass@16": 0.8221200586922632, + "pass@32": 0.8587938588657354, + "pass@64": 0.8857339251123276, + "pass@128": 0.9058823529411765 + }, + "19": { + "pass@1": 0.34770275297619047, + "pass@2": 0.44574237790588667, + "pass@4": 0.5338633851014157, + "pass@8": 0.611502029947303, + "pass@16": 0.6783513518981501, + "pass@32": 0.7327937015644227, + "pass@64": 0.7742047877222114, + "pass@128": 0.8035714285714286 + }, + "20": { + "pass@1": 0.35043974550898205, + "pass@2": 0.4594935375312366, + "pass@4": 0.5528386059714089, + "pass@8": 0.6238549348742458, + "pass@16": 0.6762174612820911, + "pass@32": 0.7171414832463683, + "pass@64": 0.7511741428622221, + "pass@128": 0.7754491017964071 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 259707, + "answer_accuracy": 0.6200980860330073, + "avg_response_len": 395.8204939639364, + "resp_tokens_sum": 165775956, + "pass_at_k": { + "pass@1": 0.6200980860330073, + "pass@2": 0.7012467768154549, + "pass@4": 0.7630910709816561, + "pass@8": 0.8090618078867852, + "pass@16": 0.8427309922131079, + "pass@32": 0.8683330680173509, + "pass@64": 0.8884576017114336, + "pass@128": 0.9040342298288508 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.915327380952381, + "pass@2": 0.948085786151731, + "pass@4": 0.9642386070788763, + "pass@8": 0.9739282351224063, + "pass@16": 0.9804970648118635, + "pass@32": 0.9846325349921398, + "pass@64": 0.986722881877331, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8619313837920489, + "pass@2": 0.912293969274483, + "pass@4": 0.9437201129675306, + "pass@8": 0.9619229572767756, + "pass@16": 0.9731904957944275, + "pass@32": 0.9818321050690701, + "pass@64": 0.9884511684407877, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7652874228395061, + "pass@2": 0.8321079548216199, + "pass@4": 0.8720276435352986, + "pass@8": 0.896107615787007, + "pass@16": 0.9136146173645132, + "pass@32": 0.929635398081815, + "pass@64": 0.9451059800569461, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7490031528189911, + "pass@2": 0.8191228913058717, + "pass@4": 0.864718364024971, + "pass@8": 0.8938565907497251, + "pass@16": 0.9139148391085333, + "pass@32": 0.9293839114599252, + "pass@64": 0.9403161156625768, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6787471064814815, + "pass@2": 0.7596807560513266, + "pass@4": 0.8202685196757814, + "pass@8": 0.8636306777109823, + "pass@16": 0.8919877814582083, + "pass@32": 0.9093606678534814, + "pass@64": 0.9216089042968235, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.5669478244837758, + "pass@2": 0.6690786646691279, + "pass@4": 0.7455410565382864, + "pass@8": 0.8028333423672143, + "pass@16": 0.8423557007877164, + "pass@32": 0.8684380664265011, + "pass@64": 0.8872958659682277, + "pass@128": 0.9026548672566371 + }, + "17": { + "pass@1": 0.5101891942771084, + "pass@2": 0.6125037798833123, + "pass@4": 0.6895920137241878, + "pass@8": 0.7465083545933107, + "pass@16": 0.7900024434338143, + "pass@32": 0.8252868759565376, + "pass@64": 0.85326005071428, + "pass@128": 0.8734939759036144 + }, + "18": { + "pass@1": 0.4936743951612903, + "pass@2": 0.5874368967487934, + "pass@4": 0.6604457161604794, + "pass@8": 0.7140707542442847, + "pass@16": 0.7545067071870171, + "pass@32": 0.7888352910327632, + "pass@64": 0.8177670834201991, + "pass@128": 0.8419354838709677 + }, + "19": { + "pass@1": 0.3340728851963746, + "pass@2": 0.4338164890453644, + "pass@4": 0.5283597530860003, + "pass@8": 0.6090080248672055, + "pass@16": 0.6734085647143847, + "pass@32": 0.7271711226701715, + "pass@64": 0.7742411529076022, + "pass@128": 0.8157099697885196 + }, + "20": { + "pass@1": 0.34025431681681684, + "pass@2": 0.45003221725662673, + "pass@4": 0.5509912418042338, + "pass@8": 0.6353882561628813, + "pass@16": 0.6987590181595222, + "pass@32": 0.7426487056681589, + "pass@64": 0.7730709505896831, + "pass@128": 0.7957957957957958 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7920246710526315, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7920246710526315, + "pass@2": 0.837947925300431, + "pass@4": 0.871755989175678, + "pass@8": 0.8968070653203173, + "pass@16": 0.9155147398391692, + "pass@32": 0.9299515117265957, + "pass@64": 0.9415052411555864, + "pass@128": 0.9504736842105264 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9231171875, + "pass@2": 0.9456815944881896, + "pass@4": 0.9603091880389951, + "pass@8": 0.9705846474455383, + "pass@16": 0.9782733645292219, + "pass@32": 0.984408624764542, + "pass@64": 0.9896463722943792, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.9999453125, + "pass@2": 0.9999987696850394, + "pass@4": 0.9999999995313086, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9976875, + "pass@2": 0.9987242864173229, + "pass@4": 0.9990599458192727, + "pass@8": 0.9991769200116146, + "pass@16": 0.9993323490285145, + "pass@32": 0.9995814585676791, + "pass@64": 0.9998779527559055, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9980625, + "pass@2": 0.9988898868110238, + "pass@4": 0.9991574159167605, + "pass@8": 0.9993262463197925, + "pass@16": 0.9995588902104531, + "pass@32": 0.9998290717623712, + "pass@64": 0.9999861763011332, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.978015625, + "pass@2": 0.9848229576771658, + "pass@4": 0.988770462129734, + "pass@8": 0.991504906288707, + "pass@16": 0.993650861087387, + "pass@32": 0.9955520574897151, + "pass@64": 0.9970482140193007, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.9920078125, + "pass@2": 0.9939662893700788, + "pass@4": 0.9950509417885265, + "pass@8": 0.9958147741453353, + "pass@16": 0.9964508524538155, + "pass@32": 0.9971523647058177, + "pass@64": 0.99794013537104, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9861875, + "pass@2": 0.9924363927165356, + "pass@4": 0.9952590014998124, + "pass@8": 0.9965242680443928, + "pass@16": 0.9971608534876047, + "pass@32": 0.9976626144839646, + "pass@64": 0.9982518300815967, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.97428125, + "pass@2": 0.9852411417322838, + "pass@4": 0.99118949015748, + "pass@8": 0.9948321772055063, + "pass@16": 0.9971259158175276, + "pass@32": 0.998483992181002, + "pass@64": 0.9994439058232791, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.971796875, + "pass@2": 0.9844603838582684, + "pass@4": 0.9896192867454066, + "pass@8": 0.9925091419180848, + "pass@16": 0.9941377676823531, + "pass@32": 0.9952259217549699, + "pass@64": 0.9963530707926768, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.9151171875, + "pass@2": 0.9456545275590561, + "pass@4": 0.9612200796775406, + "pass@8": 0.9704336964892437, + "pass@16": 0.9765719952540661, + "pass@32": 0.9812601474160046, + "pass@64": 0.9858234208718162, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.8489609375, + "pass@2": 0.8991227854330703, + "pass@4": 0.9283502003187095, + "pass@8": 0.9455536075739712, + "pass@16": 0.9560643833010687, + "pass@32": 0.9634465601168278, + "pass@64": 0.9699825720446108, + "pass@128": 0.976 + }, + "13": { + "pass@1": 0.7550703125, + "pass@2": 0.8208409202755905, + "pass@4": 0.8626456775403079, + "pass@8": 0.890856563637159, + "pass@16": 0.9113103719914247, + "pass@32": 0.9282444428549655, + "pass@64": 0.9434633808607322, + "pass@128": 0.956 + }, + "14": { + "pass@1": 0.7476953125, + "pass@2": 0.8176723671259848, + "pass@4": 0.8618266462317206, + "pass@8": 0.8918874643176163, + "pass@16": 0.9141369979757303, + "pass@32": 0.9316738111035253, + "pass@64": 0.9455826385211541, + "pass@128": 0.956 + }, + "15": { + "pass@1": 0.688828125, + "pass@2": 0.7694915108267728, + "pass@4": 0.8278998459880011, + "pass@8": 0.8700518416925825, + "pass@16": 0.8990736716148421, + "pass@32": 0.9185506095706097, + "pass@64": 0.9329091553818495, + "pass@128": 0.944 + }, + "16": { + "pass@1": 0.569984375, + "pass@2": 0.6710280511811033, + "pass@4": 0.7460000776152982, + "pass@8": 0.8009541520148893, + "pass@16": 0.8402356139913965, + "pass@32": 0.8684268111847793, + "pass@64": 0.8891133101235712, + "pass@128": 0.905 + }, + "17": { + "pass@1": 0.5203671875, + "pass@2": 0.6227608267716539, + "pass@4": 0.7001498685789274, + "pass@8": 0.7556166147417005, + "pass@16": 0.7964709061393579, + "pass@32": 0.8294317709926918, + "pass@64": 0.8556712149259554, + "pass@128": 0.873 + }, + "18": { + "pass@1": 0.4865859375, + "pass@2": 0.5871370570866156, + "pass@4": 0.6677094760030015, + "pass@8": 0.7302382482448312, + "pass@16": 0.777847828734549, + "pass@32": 0.8141039437356614, + "pass@64": 0.8426591360477864, + "pass@128": 0.867 + }, + "19": { + "pass@1": 0.3416015625, + "pass@2": 0.4410281742125985, + "pass@4": 0.5310940967379066, + "pass@8": 0.6078161499766409, + "pass@16": 0.672866477645552, + "pass@32": 0.728375162906233, + "pass@64": 0.7747117439388957, + "pass@128": 0.812 + }, + "20": { + "pass@1": 0.35315625, + "pass@2": 0.4620526574803147, + "pass@4": 0.5580520940194974, + "pass@8": 0.6356528210181952, + "pass@16": 0.6945109559994428, + "pass@32": 0.7376693572137135, + "pass@64": 0.7701353518004574, + "pass@128": 0.794 + } + }, + "per_op_accuracy": { + "10": 0.9231171875, + "2": 0.9999453125, + "3": 0.9976875, + "4": 0.9980625, + "5": 0.978015625, + "6": 0.9920078125, + "7": 0.9861875, + "8": 0.97428125, + "9": 0.971796875, + "11": 0.9151171875, + "12": 0.8489609375, + "13": 0.7550703125, + "14": 0.7476953125, + "15": 0.688828125, + "16": 0.569984375, + "17": 0.5203671875, + "18": 0.4865859375, + "19": 0.3416015625, + "20": 0.35315625 + }, + "per_op_avg_response_len": { + "10": 279.0174609375, + "2": 115.26925, + "3": 144.67940625, + "4": 159.4561328125, + "5": 174.2389375, + "6": 198.7633125, + "7": 223.944734375, + "8": 243.9508515625, + "9": 258.525546875, + "11": 297.70178125, + "12": 307.7838359375, + "13": 325.8640703125, + "14": 352.953453125, + "15": 372.9394609375, + "16": 377.9108046875, + "17": 403.39990625, + "18": 419.2847265625, + "19": 419.6512578125, + "20": 447.349296875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.0174609375, + "2": 115.26925, + "3": 144.67940625, + "4": 159.4561328125, + "5": 174.2389375, + "6": 198.7633125, + "7": 223.944734375, + "8": 243.9508515625, + "9": 258.525546875, + "11": 297.70178125, + "12": 307.7838359375, + "13": 325.8640703125, + "14": 352.953453125, + "15": 372.9394609375, + "16": 377.9108046875, + "17": 403.39990625, + "18": 419.2847265625, + "19": 419.6512578125, + "20": 447.349296875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 640086, + "answer_accuracy": 0.790495079829276, + "avg_response_len": 277.60216270154916, + "resp_tokens_sum": 224782244, + "pass_at_k": { + "pass@1": 0.790495079829276, + "pass@2": 0.8362411306855028, + "pass@4": 0.8696372919991009, + "pass@8": 0.8945669610675093, + "pass@16": 0.913432249792536, + "pass@32": 0.9280573139004922, + "pass@64": 0.9398972482792994, + "pass@128": 0.9494151122352197 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9192477876106194, + "pass@2": 0.9396735564304464, + "pass@4": 0.9543033865788902, + "pass@8": 0.96485112476411, + "pass@16": 0.9724661046772268, + "pass@32": 0.9786862175801799, + "pass@64": 0.9841720997110378, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.9998296339563862, + "pass@2": 0.9999961672431132, + "pass@4": 0.9999999985399022, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995460626911316, + "pass@2": 0.999965009391028, + "pass@4": 0.9999997130519236, + "pass@8": 0.9999999999935767, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995849609375, + "pass@2": 0.996728515625, + "pass@4": 0.9973700640935508, + "pass@8": 0.9978945203621713, + "pass@16": 0.9986215319076661, + "pass@32": 0.99946584925741, + "pass@64": 0.999956800941041, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9784474206349206, + "pass@2": 0.9849374296962881, + "pass@4": 0.9890754283690729, + "pass@8": 0.9921342884438079, + "pass@16": 0.9943983611486136, + "pass@32": 0.9960648708043484, + "pass@64": 0.9967769255677263, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9922342814371258, + "pass@2": 0.9939438298599651, + "pass@4": 0.9941038252516338, + "pass@8": 0.9941991013702666, + "pass@16": 0.9943862275449101, + "pass@32": 0.9947604790419161, + "pass@64": 0.9955089820359282, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9887435281065089, + "pass@2": 0.9931025922517821, + "pass@4": 0.995356032973689, + "pass@8": 0.99687459097789, + "pass@16": 0.997698902492233, + "pass@32": 0.9983400676934957, + "pass@64": 0.9992661790057259, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9668469551282052, + "pass@2": 0.9809139505097918, + "pass@4": 0.9884821623859518, + "pass@8": 0.9934550534186496, + "pass@16": 0.9968600221493318, + "pass@32": 0.9988651869448563, + "pass@64": 0.9998085724581219, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9736575704225352, + "pass@2": 0.987104566374626, + "pass@4": 0.9918345429532572, + "pass@8": 0.9940607002019898, + "pass@16": 0.9952133611440159, + "pass@32": 0.9960034370145185, + "pass@64": 0.9968393035368946, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9049442280453258, + "pass@2": 0.9343667049586221, + "pass@4": 0.9518926340829211, + "pass@8": 0.963721513224062, + "pass@16": 0.9715522567392245, + "pass@32": 0.9772850487323939, + "pass@64": 0.9837385106596693, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8367319915254238, + "pass@2": 0.8906847780150366, + "pass@4": 0.9210060177859121, + "pass@8": 0.9395740180895393, + "pass@16": 0.9513328654341852, + "pass@32": 0.9589354102848824, + "pass@64": 0.9654301877762941, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.7604574843260188, + "pass@2": 0.82680597894503, + "pass@4": 0.8719512272956471, + "pass@8": 0.9045499773982028, + "pass@16": 0.9266465799635933, + "pass@32": 0.9410437423571842, + "pass@64": 0.9509158308973618, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.7687889408099688, + "pass@2": 0.8359775523094659, + "pass@4": 0.8736571856672118, + "pass@8": 0.8984409718591754, + "pass@16": 0.9181805371311775, + "pass@32": 0.9350351705266242, + "pass@64": 0.9502913100353946, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.6999638728323699, + "pass@2": 0.7756271761640345, + "pass@4": 0.8292882124380401, + "pass@8": 0.869390789970966, + "pass@16": 0.8979501614129438, + "pass@32": 0.9183570437604774, + "pass@64": 0.9343777336335435, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.5773941532258065, + "pass@2": 0.6756885795021594, + "pass@4": 0.7475061202027162, + "pass@8": 0.7991479896927378, + "pass@16": 0.8357307006684896, + "pass@32": 0.8628873839423076, + "pass@64": 0.8835885830672884, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.5140895953757225, + "pass@2": 0.6143314624960177, + "pass@4": 0.6916131304576808, + "pass@8": 0.7469650535201684, + "pass@16": 0.7860427028084981, + "pass@32": 0.8170511372956204, + "pass@64": 0.8421452842507103, + "pass@128": 0.8583815028901735 + }, + "18": { + "pass@1": 0.4582589285714286, + "pass@2": 0.5605532902137236, + "pass@4": 0.6421960956666121, + "pass@8": 0.7057777001868978, + "pass@16": 0.7555140844320166, + "pass@32": 0.7930716900033024, + "pass@64": 0.822862301855235, + "pass@128": 0.8514285714285714 + }, + "19": { + "pass@1": 0.3429288663663664, + "pass@2": 0.44343987195857243, + "pass@4": 0.531017780998096, + "pass@8": 0.602912347397304, + "pass@16": 0.6667933588207599, + "pass@32": 0.725113626356638, + "pass@64": 0.775691031987435, + "pass@128": 0.8168168168168168 + }, + "20": { + "pass@1": 0.36878284534534533, + "pass@2": 0.4766399026979738, + "pass@4": 0.570342090403114, + "pass@8": 0.6477507011049768, + "pass@16": 0.7086113239762887, + "pass@32": 0.7532795279937846, + "pass@64": 0.7862179025168473, + "pass@128": 0.8108108108108109 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 651414, + "answer_accuracy": 0.7933237529228371, + "avg_response_len": 286.8824020849571, + "resp_tokens_sum": 235564878, + "pass_at_k": { + "pass@1": 0.7933237529228371, + "pass@2": 0.8389930634401326, + "pass@4": 0.8726874711442432, + "pass@8": 0.8977972002351476, + "pass@16": 0.9167457991803611, + "pass@32": 0.9314659835787653, + "pass@64": 0.9432935185698375, + "pass@128": 0.9522992985190959 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9164350076687117, + "pass@2": 0.9361730713492108, + "pass@4": 0.9499516876916461, + "pass@8": 0.9618509716297367, + "pass@16": 0.9718630754382047, + "pass@32": 0.9796931696551145, + "pass@64": 0.986608430448021, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9973958333333334, + "pass@2": 0.9992700131233596, + "pass@4": 0.9999176963674994, + "pass@8": 0.9999987979293221, + "pass@16": 0.9999999998399193, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990292159763313, + "pass@2": 0.9998897084517542, + "pass@4": 0.9999981377142946, + "pass@8": 0.9999999995762411, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.980570652173913, + "pass@2": 0.9865802949903003, + "pass@4": 0.9889661506985535, + "pass@8": 0.9902841199998196, + "pass@16": 0.9917663843037684, + "pass@32": 0.9936594509266741, + "pass@64": 0.9954604899202021, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.995008058739255, + "pass@2": 0.9963252713038379, + "pass@4": 0.9970167348350797, + "pass@8": 0.9973006814242409, + "pass@16": 0.997492802179556, + "pass@32": 0.9978510028653035, + "pass@64": 0.998567335243553, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9898843930635838, + "pass@2": 0.9931856196804879, + "pass@4": 0.9941058232063475, + "pass@8": 0.9943886704848257, + "pass@16": 0.9945808983691612, + "pass@32": 0.9949421965317783, + "pass@64": 0.9956647398843931, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9714788732394366, + "pass@2": 0.980618207275147, + "pass@4": 0.9870130438800786, + "pass@8": 0.9915782021625016, + "pass@16": 0.9946980159453276, + "pass@32": 0.9967270976372445, + "pass@64": 0.9986017780742946, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9722352170658682, + "pass@2": 0.9828459468857563, + "pass@4": 0.9884671896364751, + "pass@8": 0.9924020194851084, + "pass@16": 0.9944314239658919, + "pass@32": 0.9953209435293606, + "pass@64": 0.9962633785477046, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9257341867469879, + "pass@2": 0.9553495725026092, + "pass@4": 0.9682735500494665, + "pass@8": 0.9742548441493746, + "pass@16": 0.978185160269226, + "pass@32": 0.9822869779847758, + "pass@64": 0.9871868036674503, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8492358934169278, + "pass@2": 0.8949850973761508, + "pass@4": 0.9207448059000464, + "pass@8": 0.9354093985604018, + "pass@16": 0.943759425744798, + "pass@32": 0.9496060392426146, + "pass@64": 0.9561026755224603, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.740983893557423, + "pass@2": 0.8052852538653259, + "pass@4": 0.845815909040781, + "pass@8": 0.8738550513505369, + "pass@16": 0.895515341616072, + "pass@32": 0.9155451542423412, + "pass@64": 0.9353134265154689, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7266081871345029, + "pass@2": 0.7990618668554583, + "pass@4": 0.8478731022109073, + "pass@8": 0.8837960270998939, + "pass@16": 0.9105606572428289, + "pass@32": 0.930775272521883, + "pass@64": 0.9463526228698942, + "pass@128": 0.9590643274853801 + }, + "15": { + "pass@1": 0.6870501893939394, + "pass@2": 0.7726907361011697, + "pass@4": 0.8339367397257161, + "pass@8": 0.8770493599523325, + "pass@16": 0.9072087108288012, + "pass@32": 0.9277764122574453, + "pass@64": 0.9424641653105227, + "pass@128": 0.9545454545454546 + }, + "16": { + "pass@1": 0.5663728632478633, + "pass@2": 0.668794655876349, + "pass@4": 0.74511328258113, + "pass@8": 0.8007343935830623, + "pass@16": 0.8421667071713081, + "pass@32": 0.8733083123762971, + "pass@64": 0.8957480080042212, + "pass@128": 0.9116809116809117 + }, + "17": { + "pass@1": 0.5376067546583851, + "pass@2": 0.6423940553626447, + "pass@4": 0.7202086238637875, + "pass@8": 0.7743041381948508, + "pass@16": 0.814345713501841, + "pass@32": 0.8470088033255796, + "pass@64": 0.8726913345902738, + "pass@128": 0.8881987577639752 + }, + "18": { + "pass@1": 0.5092830882352941, + "pass@2": 0.6142293162343674, + "pass@4": 0.7005960897351063, + "pass@8": 0.7701591745990866, + "pass@16": 0.8221200586922632, + "pass@32": 0.8587938588657354, + "pass@64": 0.8857339251123276, + "pass@128": 0.9058823529411765 + }, + "19": { + "pass@1": 0.34770275297619047, + "pass@2": 0.44574237790588667, + "pass@4": 0.5338633851014157, + "pass@8": 0.611502029947303, + "pass@16": 0.6783513518981501, + "pass@32": 0.7327937015644227, + "pass@64": 0.7742047877222114, + "pass@128": 0.8035714285714286 + }, + "20": { + "pass@1": 0.35043974550898205, + "pass@2": 0.4594935375312366, + "pass@4": 0.5528386059714089, + "pass@8": 0.6238549348742458, + "pass@16": 0.6762174612820911, + "pass@32": 0.7171414832463683, + "pass@64": 0.7511741428622221, + "pass@128": 0.7754491017964071 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 634704, + "answer_accuracy": 0.7922391755871545, + "avg_response_len": 307.75241027919793, + "resp_tokens_sum": 246556459, + "pass_at_k": { + "pass@1": 0.7922391755871545, + "pass@2": 0.8386018031672131, + "pass@4": 0.8729426678002086, + "pass@8": 0.8980563358146915, + "pass@16": 0.9163577796796314, + "pass@32": 0.9303137674409322, + "pass@64": 0.9412975982929933, + "pass@128": 0.9496724716408372 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.933535447761194, + "pass@2": 0.9610144405923143, + "pass@4": 0.9764659397052976, + "pass@8": 0.9848856698483921, + "pass@16": 0.9903880371665308, + "pass@32": 0.994788130439683, + "pass@64": 0.9981423348247224, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961962463556852, + "pass@2": 0.9970163965473705, + "pass@4": 0.997338775072941, + "pass@8": 0.9976015064053606, + "pass@16": 0.9980534958639686, + "pass@32": 0.9987797625879855, + "pass@64": 0.9996441771309199, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.999924094603306, + "pass@4": 0.9999989030976391, + "pass@8": 0.999999999845404, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9750229779411764, + "pass@2": 0.9829337222093564, + "pass@4": 0.9882893535366903, + "pass@8": 0.9921605412616765, + "pass@16": 0.9948705139434515, + "pass@32": 0.9969973900489489, + "pass@64": 0.998910627773521, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9884660883280757, + "pass@2": 0.9913928407312652, + "pass@4": 0.9938846173408135, + "pass@8": 0.9958811876044368, + "pass@16": 0.9974790678017365, + "pass@32": 0.9989033586934598, + "pass@64": 0.9998111525900316, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9794056566455697, + "pass@2": 0.9909034560948872, + "pass@4": 0.9964178719155359, + "pass@8": 0.9984878997029001, + "pass@16": 0.9994102329414561, + "pass@32": 0.9999166822897716, + "pass@64": 0.9999995619546246, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9842342342342343, + "pass@2": 0.9942238005722256, + "pass@4": 0.9981784532001067, + "pass@8": 0.9995914077213195, + "pass@16": 0.9999633430821163, + "pass@32": 0.9999998023813371, + "pass@64": 0.9999999999999107, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9692021704180064, + "pass@2": 0.983175937412968, + "pass@4": 0.9883279185158123, + "pass@8": 0.9908531152358594, + "pass@16": 0.9925946265968474, + "pass@32": 0.9942363552283263, + "pass@64": 0.9958943717110796, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.915327380952381, + "pass@2": 0.948085786151731, + "pass@4": 0.9642386070788763, + "pass@8": 0.9739282351224063, + "pass@16": 0.9804970648118635, + "pass@32": 0.9846325349921398, + "pass@64": 0.986722881877331, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8619313837920489, + "pass@2": 0.912293969274483, + "pass@4": 0.9437201129675306, + "pass@8": 0.9619229572767756, + "pass@16": 0.9731904957944275, + "pass@32": 0.9818321050690701, + "pass@64": 0.9884511684407877, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7652874228395061, + "pass@2": 0.8321079548216199, + "pass@4": 0.8720276435352986, + "pass@8": 0.896107615787007, + "pass@16": 0.9136146173645132, + "pass@32": 0.929635398081815, + "pass@64": 0.9451059800569461, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7490031528189911, + "pass@2": 0.8191228913058717, + "pass@4": 0.864718364024971, + "pass@8": 0.8938565907497251, + "pass@16": 0.9139148391085333, + "pass@32": 0.9293839114599252, + "pass@64": 0.9403161156625768, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6787471064814815, + "pass@2": 0.7596807560513266, + "pass@4": 0.8202685196757814, + "pass@8": 0.8636306777109823, + "pass@16": 0.8919877814582083, + "pass@32": 0.9093606678534814, + "pass@64": 0.9216089042968235, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.5669478244837758, + "pass@2": 0.6690786646691279, + "pass@4": 0.7455410565382864, + "pass@8": 0.8028333423672143, + "pass@16": 0.8423557007877164, + "pass@32": 0.8684380664265011, + "pass@64": 0.8872958659682277, + "pass@128": 0.9026548672566371 + }, + "17": { + "pass@1": 0.5101891942771084, + "pass@2": 0.6125037798833123, + "pass@4": 0.6895920137241878, + "pass@8": 0.7465083545933107, + "pass@16": 0.7900024434338143, + "pass@32": 0.8252868759565376, + "pass@64": 0.85326005071428, + "pass@128": 0.8734939759036144 + }, + "18": { + "pass@1": 0.4936743951612903, + "pass@2": 0.5874368967487934, + "pass@4": 0.6604457161604794, + "pass@8": 0.7140707542442847, + "pass@16": 0.7545067071870171, + "pass@32": 0.7888352910327632, + "pass@64": 0.8177670834201991, + "pass@128": 0.8419354838709677 + }, + "19": { + "pass@1": 0.3340728851963746, + "pass@2": 0.4338164890453644, + "pass@4": 0.5283597530860003, + "pass@8": 0.6090080248672055, + "pass@16": 0.6734085647143847, + "pass@32": 0.7271711226701715, + "pass@64": 0.7742411529076022, + "pass@128": 0.8157099697885196 + }, + "20": { + "pass@1": 0.34025431681681684, + "pass@2": 0.45003221725662673, + "pass@4": 0.5509912418042338, + "pass@8": 0.6353882561628813, + "pass@16": 0.6987590181595222, + "pass@32": 0.7426487056681589, + "pass@64": 0.7730709505896831, + "pass@128": 0.7957957957957958 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch1-op17-20/graphs/SDAR-100M-block4-epoch1-op17-20_score_report.jsonl b/composition/SDAR-100M-block4-epoch1-op17-20/graphs/SDAR-100M-block4-epoch1-op17-20_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2a53dc819a209fec875b1a237987da5a4096e20a --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op17-20/graphs/SDAR-100M-block4-epoch1-op17-20_score_report.jsonl @@ -0,0 +1,2 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch1-op17-20", "results_dir": "results/composition/SDAR-100M-block4-epoch1-op17-20/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1004814, "value_mismatch": 27407, "dependency_mismatch": 142976, "answer_mismatch": 22898, "extra_nodes": 22811, "missing_nodes": 14262, "perfect_with_extra": 2726, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 330757, "dependency_mismatch": 46889, "value_mismatch": 9443, "answer_mismatch": 7909, "extra_nodes": 7861, "perfect_with_extra": 946, "missing_nodes": 4577}, "teachers_in_school": {"total": 390656, "perfect": 341231, "value_mismatch": 8735, "dependency_mismatch": 48189, "answer_mismatch": 7651, "extra_nodes": 7517, "missing_nodes": 4846, "perfect_with_extra": 915}, "movie_festival_awards": {"total": 382336, "perfect": 332826, "dependency_mismatch": 47898, "value_mismatch": 9229, "answer_mismatch": 7338, "missing_nodes": 4839, "extra_nodes": 7433, "perfect_with_extra": 865}}, "results_path": "results/composition/SDAR-100M-block4-epoch1-op17-20/graphs/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 379933, "value_mismatch": 619682, "dependency_mismatch": 795420, "answer_mismatch": 482897, "extra_nodes": 57514, "missing_nodes": 208965, "perfect_with_extra": 534, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "dependency_mismatch": 267943, "extra_nodes": 20378, "value_mismatch": 208668, "answer_mismatch": 161730, "missing_nodes": 73658, "perfect": 127805, "perfect_with_extra": 202}, "movie_festival_awards": {"total": 418816, "perfect": 123601, "value_mismatch": 203662, "dependency_mismatch": 261155, "answer_mismatch": 159112, "extra_nodes": 18478, "perfect_with_extra": 161, "missing_nodes": 66596}, "teachers_in_school": {"total": 430464, "perfect": 128527, "dependency_mismatch": 266322, "value_mismatch": 207352, "answer_mismatch": 162055, "missing_nodes": 68711, "extra_nodes": 18658, "perfect_with_extra": 171}}, "results_path": "results/composition/SDAR-100M-block4-epoch1-op17-20/graphs/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1384747, "value_mismatch": 647089, "dependency_mismatch": 938396, "answer_mismatch": 505795, "extra_nodes": 80325, "missing_nodes": 223227, "perfect_with_extra": 3260, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch1-op17-20/graphs/SDAR-100M-block4-epoch1-op17-20_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 458562, "dependency_mismatch": 314832, "value_mismatch": 218111, "answer_mismatch": 169639, "extra_nodes": 28239, "perfect_with_extra": 1148, "missing_nodes": 78235}, "teachers_in_school": {"total": 821120, "perfect": 469758, "value_mismatch": 216087, "dependency_mismatch": 314511, "answer_mismatch": 169706, "extra_nodes": 26175, "missing_nodes": 73557, "perfect_with_extra": 1086}, "movie_festival_awards": {"total": 801152, "perfect": 456427, "dependency_mismatch": 309053, "value_mismatch": 212891, "answer_mismatch": 166450, "missing_nodes": 71435, "extra_nodes": 25911, "perfect_with_extra": 1026}}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch1-op17-20", "results_dir": "results/composition/SDAR-100M-block4-epoch1-op17-20/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch1-op17-20/graphs/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch1-op17-20/graphs/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch1-op17-20/graphs/SDAR-100M-block4-epoch1-op17-20_score_report.jsonl", "template_stats": {}}} diff --git a/composition/SDAR-100M-block4-epoch1-op17-20/graphs/results.json b/composition/SDAR-100M-block4-epoch1-op17-20/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..e8baa13601bb6a9f70d8491d55eaaa4a46c8a178 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op17-20/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969418960244648, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.984375, + "pass@2": 0.990625, + "pass@4": 0.996875, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9587301587301588, + "pass@2": 0.9682539682539683, + "pass@4": 0.9873015873015873, + "pass@8": 0.9873015873015873, + "pass@16": 0.9873015873015873, + "pass@32": 0.9936507936507937, + "pass@64": 0.9936507936507937, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9700598802395209, + "pass@2": 0.9760479041916168, + "pass@4": 0.9820359281437125, + "pass@8": 0.9850299401197605, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.878698224852071, + "pass@2": 0.9260355029585798, + "pass@4": 0.9644970414201184, + "pass@8": 0.985207100591716, + "pass@16": 0.9970414201183432, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.7788461538461539, + "pass@2": 0.8717948717948718, + "pass@4": 0.9391025641025641, + "pass@8": 0.9775641025641025, + "pass@16": 0.9967948717948718, + "pass@32": 0.9967948717948718, + "pass@64": 0.9967948717948718, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7380281690140845, + "pass@2": 0.8338028169014085, + "pass@4": 0.9183098591549296, + "pass@8": 0.971830985915493, + "pass@16": 0.9887323943661972, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "10": { + "pass@1": 0.5988200589970502, + "pass@2": 0.6666666666666666, + "pass@4": 0.7492625368731564, + "pass@8": 0.8141592920353983, + "pass@16": 0.8672566371681416, + "pass@32": 0.9203539823008849, + "pass@64": 0.9351032448377581, + "pass@128": 0.9498525073746312 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9968847352024922, + "3": 0.9969418960244648, + "4": 0.984375, + "5": 0.9587301587301588, + "6": 0.9700598802395209, + "7": 0.878698224852071, + "8": 0.7788461538461539, + "9": 0.7380281690140845, + "10": 0.5988200589970502 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9682539682539683, + "6": 0.9760479041916168, + "7": 0.9260355029585798, + "8": 0.8717948717948718, + "9": 0.8338028169014085, + "10": 0.6666666666666666 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9873015873015873, + "6": 0.9820359281437125, + "7": 0.9644970414201184, + "8": 0.9391025641025641, + "9": 0.9183098591549296, + "10": 0.7492625368731564 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9873015873015873, + "6": 0.9850299401197605, + "7": 0.985207100591716, + "8": 0.9775641025641025, + "9": 0.971830985915493, + "10": 0.8141592920353983 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9873015873015873, + "6": 0.9940119760479041, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 0.9887323943661972, + "10": 0.8672566371681416 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 0.9940119760479041, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 0.9971830985915493, + "10": 0.9203539823008849 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 0.9940119760479041, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 0.9971830985915493, + "10": 0.9351032448377581 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 0.9970059880239521, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 0.9971830985915493, + "10": 0.9498525073746312 + } + }, + "total_pass_at_k": { + "pass@1": 0.8753799392097265, + "pass@2": 0.9128672745694022, + "pass@4": 0.9473150962512664, + "pass@8": 0.9682539682539683, + "pass@16": 0.9807497467071935, + "pass@32": 0.9885173927727119, + "pass@64": 0.9902060114826072, + "pass@128": 0.9922323539344816 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970845481049563, + "pass@2": 0.9970845481049563, + "pass@4": 0.9970845481049563, + "pass@8": 0.9970845481049563, + "pass@16": 0.9970845481049563, + "pass@32": 0.9970845481049563, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9912280701754386, + "pass@2": 0.9970760233918129, + "pass@4": 0.9970760233918129, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9441176470588235, + "pass@2": 0.9676470588235294, + "pass@4": 0.9823529411764705, + "pass@8": 0.9911764705882353, + "pass@16": 0.9941176470588236, + "pass@32": 0.9970588235294118, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9558359621451105, + "pass@2": 0.9684542586750788, + "pass@4": 0.9779179810725552, + "pass@8": 0.9842271293375394, + "pass@16": 0.9905362776025236, + "pass@32": 0.9905362776025236, + "pass@64": 0.9968454258675079, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8259493670886076, + "pass@2": 0.9050632911392406, + "pass@4": 0.9556962025316456, + "pass@8": 0.990506329113924, + "pass@16": 0.9936708860759493, + "pass@32": 0.9968354430379747, + "pass@64": 0.9968354430379747, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.7687687687687688, + "pass@2": 0.8438438438438438, + "pass@4": 0.9219219219219219, + "pass@8": 0.975975975975976, + "pass@16": 0.993993993993994, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7556270096463023, + "pass@2": 0.8167202572347267, + "pass@4": 0.8842443729903537, + "pass@8": 0.9356913183279743, + "pass@16": 0.9678456591639871, + "pass@32": 0.9871382636655949, + "pass@64": 0.9935691318327974, + "pass@128": 0.9967845659163987 + }, + "10": { + "pass@1": 0.564179104477612, + "pass@2": 0.6477611940298508, + "pass@4": 0.7373134328358208, + "pass@8": 0.8149253731343283, + "pass@16": 0.8776119402985074, + "pass@32": 0.9104477611940298, + "pass@64": 0.9283582089552239, + "pass@128": 0.9522388059701492 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9912280701754386, + "5": 0.9441176470588235, + "6": 0.9558359621451105, + "7": 0.8259493670886076, + "8": 0.7687687687687688, + "9": 0.7556270096463023, + "10": 0.564179104477612 + }, + "pass@2": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9970760233918129, + "5": 0.9676470588235294, + "6": 0.9684542586750788, + "7": 0.9050632911392406, + "8": 0.8438438438438438, + "9": 0.8167202572347267, + "10": 0.6477611940298508 + }, + "pass@4": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9970760233918129, + "5": 0.9823529411764705, + "6": 0.9779179810725552, + "7": 0.9556962025316456, + "8": 0.9219219219219219, + "9": 0.8842443729903537, + "10": 0.7373134328358208 + }, + "pass@8": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 1.0, + "5": 0.9911764705882353, + "6": 0.9842271293375394, + "7": 0.990506329113924, + "8": 0.975975975975976, + "9": 0.9356913183279743, + "10": 0.8149253731343283 + }, + "pass@16": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 1.0, + "5": 0.9941176470588236, + "6": 0.9905362776025236, + "7": 0.9936708860759493, + "8": 0.993993993993994, + "9": 0.9678456591639871, + "10": 0.8776119402985074 + }, + "pass@32": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9905362776025236, + "7": 0.9968354430379747, + "8": 1.0, + "9": 0.9871382636655949, + "10": 0.9104477611940298 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9968454258675079, + "7": 0.9968354430379747, + "8": 1.0, + "9": 0.9935691318327974, + "10": 0.9283582089552239 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9968354430379747, + "8": 1.0, + "9": 0.9967845659163987, + "10": 0.9522388059701492 + } + }, + "total_pass_at_k": { + "pass@1": 0.8690994308670907, + "pass@2": 0.9062604620020087, + "pass@4": 0.9400736524941413, + "pass@8": 0.9658520254435888, + "pass@16": 0.979578172079009, + "pass@32": 0.9866086374288584, + "pass@64": 0.9906260462002009, + "pass@128": 0.9939738868429863 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996969696969697, + "pass@2": 0.996969696969697, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9792899408284024, + "pass@2": 0.985207100591716, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9507246376811594, + "pass@2": 0.9739130434782609, + "pass@4": 0.9884057971014493, + "pass@8": 0.9884057971014493, + "pass@16": 0.9884057971014493, + "pass@32": 0.9884057971014493, + "pass@64": 0.9942028985507246, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9713467048710601, + "pass@2": 0.9770773638968482, + "pass@4": 0.9885386819484241, + "pass@8": 0.997134670487106, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.869942196531792, + "pass@2": 0.9161849710982659, + "pass@4": 0.9421965317919075, + "pass@8": 0.9739884393063584, + "pass@16": 0.9884393063583815, + "pass@32": 0.9913294797687862, + "pass@64": 0.9942196531791907, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.7436619718309859, + "pass@2": 0.847887323943662, + "pass@4": 0.923943661971831, + "pass@8": 0.9577464788732394, + "pass@16": 0.9802816901408451, + "pass@32": 0.9915492957746479, + "pass@64": 0.9943661971830986, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.7155688622754491, + "pass@2": 0.7964071856287425, + "pass@4": 0.8592814371257484, + "pass@8": 0.9251497005988024, + "pass@16": 0.9850299401197605, + "pass@32": 0.9940119760479041, + "pass@64": 0.9970059880239521, + "pass@128": 0.9970059880239521 + }, + "10": { + "pass@1": 0.6165644171779141, + "pass@2": 0.7147239263803681, + "pass@4": 0.7822085889570553, + "pass@8": 0.8404907975460123, + "pass@16": 0.8773006134969326, + "pass@32": 0.9141104294478528, + "pass@64": 0.9355828220858896, + "pass@128": 0.9478527607361963 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9792899408284024, + "5": 0.9507246376811594, + "6": 0.9713467048710601, + "7": 0.869942196531792, + "8": 0.7436619718309859, + "9": 0.7155688622754491, + "10": 0.6165644171779141 + }, + "pass@2": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.985207100591716, + "5": 0.9739130434782609, + "6": 0.9770773638968482, + "7": 0.9161849710982659, + "8": 0.847887323943662, + "9": 0.7964071856287425, + "10": 0.7147239263803681 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9884057971014493, + "6": 0.9885386819484241, + "7": 0.9421965317919075, + "8": 0.923943661971831, + "9": 0.8592814371257484, + "10": 0.7822085889570553 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9884057971014493, + "6": 0.997134670487106, + "7": 0.9739884393063584, + "8": 0.9577464788732394, + "9": 0.9251497005988024, + "10": 0.8404907975460123 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9884057971014493, + "6": 0.997134670487106, + "7": 0.9884393063583815, + "8": 0.9802816901408451, + "9": 0.9850299401197605, + "10": 0.8773006134969326 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9884057971014493, + "6": 0.997134670487106, + "7": 0.9913294797687862, + "8": 0.9915492957746479, + "9": 0.9940119760479041, + "10": 0.9141104294478528 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9942028985507246, + "6": 0.997134670487106, + "7": 0.9942196531791907, + "8": 0.9943661971830986, + "9": 0.9970059880239521, + "10": 0.9355828220858896 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.997134670487106, + "7": 0.9971098265895953, + "8": 0.9971830985915493, + "9": 0.9970059880239521, + "10": 0.9478527607361963 + } + }, + "total_pass_at_k": { + "pass@1": 0.8718872870249017, + "pass@2": 0.9125163826998689, + "pass@4": 0.9433158584534731, + "pass@8": 0.9652686762778506, + "pass@16": 0.9800131061598951, + "pass@32": 0.9865661861074705, + "pass@64": 0.9904980340760158, + "pass@128": 0.9927916120576671 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.999, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.997, + "pass@2": 0.998, + "pass@4": 0.999, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985, + "pass@2": 0.991, + "pass@4": 0.998, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.951, + "pass@2": 0.97, + "pass@4": 0.986, + "pass@8": 0.989, + "pass@16": 0.99, + "pass@32": 0.993, + "pass@64": 0.996, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.966, + "pass@2": 0.974, + "pass@4": 0.983, + "pass@8": 0.989, + "pass@16": 0.994, + "pass@32": 0.994, + "pass@64": 0.996, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.859, + "pass@2": 0.916, + "pass@4": 0.954, + "pass@8": 0.983, + "pass@16": 0.993, + "pass@32": 0.995, + "pass@64": 0.996, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.763, + "pass@2": 0.854, + "pass@4": 0.928, + "pass@8": 0.97, + "pass@16": 0.99, + "pass@32": 0.996, + "pass@64": 0.997, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.736, + "pass@2": 0.816, + "pass@4": 0.888, + "pass@8": 0.945, + "pass@16": 0.981, + "pass@32": 0.993, + "pass@64": 0.996, + "pass@128": 0.997 + }, + "10": { + "pass@1": 0.593, + "pass@2": 0.676, + "pass@4": 0.756, + "pass@8": 0.823, + "pass@16": 0.874, + "pass@32": 0.915, + "pass@64": 0.933, + "pass@128": 0.95 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.999, + "3": 0.997, + "4": 0.985, + "5": 0.951, + "6": 0.966, + "7": 0.859, + "8": 0.763, + "9": 0.736, + "10": 0.593 + }, + "pass@2": { + "2": 1.0, + "3": 0.998, + "4": 0.991, + "5": 0.97, + "6": 0.974, + "7": 0.916, + "8": 0.854, + "9": 0.816, + "10": 0.676 + }, + "pass@4": { + "2": 1.0, + "3": 0.999, + "4": 0.998, + "5": 0.986, + "6": 0.983, + "7": 0.954, + "8": 0.928, + "9": 0.888, + "10": 0.756 + }, + "pass@8": { + "2": 1.0, + "3": 0.999, + "4": 1.0, + "5": 0.989, + "6": 0.989, + "7": 0.983, + "8": 0.97, + "9": 0.945, + "10": 0.823 + }, + "pass@16": { + "2": 1.0, + "3": 0.999, + "4": 1.0, + "5": 0.99, + "6": 0.994, + "7": 0.993, + "8": 0.99, + "9": 0.981, + "10": 0.874 + }, + "pass@32": { + "2": 1.0, + "3": 0.999, + "4": 1.0, + "5": 0.993, + "6": 0.994, + "7": 0.995, + "8": 0.996, + "9": 0.993, + "10": 0.915 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.996, + "6": 0.996, + "7": 0.996, + "8": 0.997, + "9": 0.996, + "10": 0.933 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.997, + "6": 0.998, + "7": 0.997, + "8": 0.998, + "9": 0.997, + "10": 0.95 + } + }, + "total_pass_at_k": { + "pass@1": 0.8721111111111111, + "pass@2": 0.9105555555555556, + "pass@4": 0.9435555555555556, + "pass@8": 0.9664444444444444, + "pass@16": 0.9801111111111112, + "pass@32": 0.9872222222222222, + "pass@64": 0.9904444444444445, + "pass@128": 0.993 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5269121813031161, + "pass@2": 0.6090651558073654, + "pass@4": 0.7053824362606232, + "pass@8": 0.8073654390934845, + "pass@16": 0.8555240793201133, + "pass@32": 0.8951841359773371, + "pass@64": 0.9150141643059491, + "pass@128": 0.9291784702549575 + }, + "12": { + "pass@1": 0.5310734463276836, + "pass@2": 0.615819209039548, + "pass@4": 0.6864406779661016, + "pass@8": 0.7542372881355932, + "pass@16": 0.7994350282485876, + "pass@32": 0.8361581920903954, + "pass@64": 0.8615819209039548, + "pass@128": 0.8728813559322034 + }, + "13": { + "pass@1": 0.42946708463949845, + "pass@2": 0.49843260188087773, + "pass@4": 0.5642633228840125, + "pass@8": 0.64576802507837, + "pass@16": 0.7053291536050157, + "pass@32": 0.7492163009404389, + "pass@64": 0.780564263322884, + "pass@128": 0.8087774294670846 + }, + "14": { + "pass@1": 0.3582554517133956, + "pass@2": 0.42679127725856697, + "pass@4": 0.4984423676012461, + "pass@8": 0.5451713395638629, + "pass@16": 0.5981308411214953, + "pass@32": 0.6697819314641744, + "pass@64": 0.735202492211838, + "pass@128": 0.7912772585669782 + }, + "15": { + "pass@1": 0.2774566473988439, + "pass@2": 0.32947976878612717, + "pass@4": 0.3786127167630058, + "pass@8": 0.45375722543352603, + "pass@16": 0.5317919075144508, + "pass@32": 0.6069364161849711, + "pass@64": 0.684971098265896, + "pass@128": 0.7427745664739884 + }, + "16": { + "pass@1": 0.23548387096774193, + "pass@2": 0.2709677419354839, + "pass@4": 0.3193548387096774, + "pass@8": 0.3580645161290323, + "pass@16": 0.4161290322580645, + "pass@32": 0.4645161290322581, + "pass@64": 0.5193548387096775, + "pass@128": 0.5548387096774193 + }, + "17": { + "pass@1": 0.21965317919075145, + "pass@2": 0.2514450867052023, + "pass@4": 0.30057803468208094, + "pass@8": 0.3439306358381503, + "pass@16": 0.3872832369942196, + "pass@32": 0.4508670520231214, + "pass@64": 0.5144508670520231, + "pass@128": 0.5520231213872833 + }, + "18": { + "pass@1": 0.14285714285714285, + "pass@2": 0.19428571428571428, + "pass@4": 0.24, + "pass@8": 0.26, + "pass@16": 0.3, + "pass@32": 0.34285714285714286, + "pass@64": 0.38857142857142857, + "pass@128": 0.43142857142857144 + }, + "19": { + "pass@1": 0.0960960960960961, + "pass@2": 0.12612612612612611, + "pass@4": 0.15315315315315314, + "pass@8": 0.17417417417417416, + "pass@16": 0.1921921921921922, + "pass@32": 0.23723723723723725, + "pass@64": 0.24924924924924924, + "pass@128": 0.2822822822822823 + }, + "20": { + "pass@1": 0.08408408408408409, + "pass@2": 0.12612612612612611, + "pass@4": 0.17117117117117117, + "pass@8": 0.2012012012012012, + "pass@16": 0.2222222222222222, + "pass@32": 0.24624624624624625, + "pass@64": 0.27927927927927926, + "pass@128": 0.3183183183183183 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5269121813031161, + "12": 0.5310734463276836, + "13": 0.42946708463949845, + "14": 0.3582554517133956, + "15": 0.2774566473988439, + "16": 0.23548387096774193, + "17": 0.21965317919075145, + "18": 0.14285714285714285, + "19": 0.0960960960960961, + "20": 0.08408408408408409 + }, + "pass@2": { + "11": 0.6090651558073654, + "12": 0.615819209039548, + "13": 0.49843260188087773, + "14": 0.42679127725856697, + "15": 0.32947976878612717, + "16": 0.2709677419354839, + "17": 0.2514450867052023, + "18": 0.19428571428571428, + "19": 0.12612612612612611, + "20": 0.12612612612612611 + }, + "pass@4": { + "11": 0.7053824362606232, + "12": 0.6864406779661016, + "13": 0.5642633228840125, + "14": 0.4984423676012461, + "15": 0.3786127167630058, + "16": 0.3193548387096774, + "17": 0.30057803468208094, + "18": 0.24, + "19": 0.15315315315315314, + "20": 0.17117117117117117 + }, + "pass@8": { + "11": 0.8073654390934845, + "12": 0.7542372881355932, + "13": 0.64576802507837, + "14": 0.5451713395638629, + "15": 0.45375722543352603, + "16": 0.3580645161290323, + "17": 0.3439306358381503, + "18": 0.26, + "19": 0.17417417417417416, + "20": 0.2012012012012012 + }, + "pass@16": { + "11": 0.8555240793201133, + "12": 0.7994350282485876, + "13": 0.7053291536050157, + "14": 0.5981308411214953, + "15": 0.5317919075144508, + "16": 0.4161290322580645, + "17": 0.3872832369942196, + "18": 0.3, + "19": 0.1921921921921922, + "20": 0.2222222222222222 + }, + "pass@32": { + "11": 0.8951841359773371, + "12": 0.8361581920903954, + "13": 0.7492163009404389, + "14": 0.6697819314641744, + "15": 0.6069364161849711, + "16": 0.4645161290322581, + "17": 0.4508670520231214, + "18": 0.34285714285714286, + "19": 0.23723723723723725, + "20": 0.24624624624624625 + }, + "pass@64": { + "11": 0.9150141643059491, + "12": 0.8615819209039548, + "13": 0.780564263322884, + "14": 0.735202492211838, + "15": 0.684971098265896, + "16": 0.5193548387096775, + "17": 0.5144508670520231, + "18": 0.38857142857142857, + "19": 0.24924924924924924, + "20": 0.27927927927927926 + }, + "pass@128": { + "11": 0.9291784702549575, + "12": 0.8728813559322034, + "13": 0.8087774294670846, + "14": 0.7912772585669782, + "15": 0.7427745664739884, + "16": 0.5548387096774193, + "17": 0.5520231213872833, + "18": 0.43142857142857144, + "19": 0.2822822822822823, + "20": 0.3183183183183183 + } + }, + "total_pass_at_k": { + "pass@1": 0.29153046062407134, + "pass@2": 0.3465081723625557, + "pass@4": 0.4035661218424963, + "pass@8": 0.4564635958395245, + "pass@16": 0.5028231797919762, + "pass@32": 0.5518573551263002, + "pass@64": 0.5946508172362556, + "pass@128": 0.6300148588410104 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5746031746031746, + "pass@2": 0.638095238095238, + "pass@4": 0.7492063492063492, + "pass@8": 0.8476190476190476, + "pass@16": 0.9015873015873016, + "pass@32": 0.926984126984127, + "pass@64": 0.9492063492063492, + "pass@128": 0.9682539682539683 + }, + "12": { + "pass@1": 0.5290519877675841, + "pass@2": 0.6269113149847095, + "pass@4": 0.709480122324159, + "pass@8": 0.7767584097859327, + "pass@16": 0.8440366972477065, + "pass@32": 0.8776758409785933, + "pass@64": 0.8837920489296636, + "pass@128": 0.8899082568807339 + }, + "13": { + "pass@1": 0.4104938271604938, + "pass@2": 0.5030864197530864, + "pass@4": 0.5679012345679012, + "pass@8": 0.6203703703703703, + "pass@16": 0.6790123456790124, + "pass@32": 0.6975308641975309, + "pass@64": 0.7345679012345679, + "pass@128": 0.7654320987654321 + }, + "14": { + "pass@1": 0.34421364985163205, + "pass@2": 0.41543026706231456, + "pass@4": 0.4629080118694362, + "pass@8": 0.5341246290801187, + "pass@16": 0.6142433234421365, + "pass@32": 0.7002967359050445, + "pass@64": 0.7596439169139466, + "pass@128": 0.8011869436201781 + }, + "15": { + "pass@1": 0.30246913580246915, + "pass@2": 0.3611111111111111, + "pass@4": 0.39814814814814814, + "pass@8": 0.45987654320987653, + "pass@16": 0.5277777777777778, + "pass@32": 0.595679012345679, + "pass@64": 0.6882716049382716, + "pass@128": 0.7345679012345679 + }, + "16": { + "pass@1": 0.2743362831858407, + "pass@2": 0.32448377581120946, + "pass@4": 0.3834808259587021, + "pass@8": 0.43952802359882004, + "pass@16": 0.49262536873156343, + "pass@32": 0.5427728613569321, + "pass@64": 0.584070796460177, + "pass@128": 0.6224188790560472 + }, + "17": { + "pass@1": 0.20180722891566266, + "pass@2": 0.22590361445783133, + "pass@4": 0.2710843373493976, + "pass@8": 0.3072289156626506, + "pass@16": 0.3614457831325301, + "pass@32": 0.4126506024096386, + "pass@64": 0.4759036144578313, + "pass@128": 0.5060240963855421 + }, + "18": { + "pass@1": 0.16774193548387098, + "pass@2": 0.1967741935483871, + "pass@4": 0.25806451612903225, + "pass@8": 0.2870967741935484, + "pass@16": 0.31290322580645163, + "pass@32": 0.3580645161290323, + "pass@64": 0.4032258064516129, + "pass@128": 0.44193548387096776 + }, + "19": { + "pass@1": 0.08157099697885196, + "pass@2": 0.1148036253776435, + "pass@4": 0.1419939577039275, + "pass@8": 0.1903323262839879, + "pass@16": 0.21148036253776434, + "pass@32": 0.24773413897280966, + "pass@64": 0.2719033232628399, + "pass@128": 0.311178247734139 + }, + "20": { + "pass@1": 0.07507507507507508, + "pass@2": 0.12012012012012012, + "pass@4": 0.15015015015015015, + "pass@8": 0.2072072072072072, + "pass@16": 0.24024024024024024, + "pass@32": 0.2552552552552553, + "pass@64": 0.27927927927927926, + "pass@128": 0.3003003003003003 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5746031746031746, + "12": 0.5290519877675841, + "13": 0.4104938271604938, + "14": 0.34421364985163205, + "15": 0.30246913580246915, + "16": 0.2743362831858407, + "17": 0.20180722891566266, + "18": 0.16774193548387098, + "19": 0.08157099697885196, + "20": 0.07507507507507508 + }, + "pass@2": { + "11": 0.638095238095238, + "12": 0.6269113149847095, + "13": 0.5030864197530864, + "14": 0.41543026706231456, + "15": 0.3611111111111111, + "16": 0.32448377581120946, + "17": 0.22590361445783133, + "18": 0.1967741935483871, + "19": 0.1148036253776435, + "20": 0.12012012012012012 + }, + "pass@4": { + "11": 0.7492063492063492, + "12": 0.709480122324159, + "13": 0.5679012345679012, + "14": 0.4629080118694362, + "15": 0.39814814814814814, + "16": 0.3834808259587021, + "17": 0.2710843373493976, + "18": 0.25806451612903225, + "19": 0.1419939577039275, + "20": 0.15015015015015015 + }, + "pass@8": { + "11": 0.8476190476190476, + "12": 0.7767584097859327, + "13": 0.6203703703703703, + "14": 0.5341246290801187, + "15": 0.45987654320987653, + "16": 0.43952802359882004, + "17": 0.3072289156626506, + "18": 0.2870967741935484, + "19": 0.1903323262839879, + "20": 0.2072072072072072 + }, + "pass@16": { + "11": 0.9015873015873016, + "12": 0.8440366972477065, + "13": 0.6790123456790124, + "14": 0.6142433234421365, + "15": 0.5277777777777778, + "16": 0.49262536873156343, + "17": 0.3614457831325301, + "18": 0.31290322580645163, + "19": 0.21148036253776434, + "20": 0.24024024024024024 + }, + "pass@32": { + "11": 0.926984126984127, + "12": 0.8776758409785933, + "13": 0.6975308641975309, + "14": 0.7002967359050445, + "15": 0.595679012345679, + "16": 0.5427728613569321, + "17": 0.4126506024096386, + "18": 0.3580645161290323, + "19": 0.24773413897280966, + "20": 0.2552552552552553 + }, + "pass@64": { + "11": 0.9492063492063492, + "12": 0.8837920489296636, + "13": 0.7345679012345679, + "14": 0.7596439169139466, + "15": 0.6882716049382716, + "16": 0.584070796460177, + "17": 0.4759036144578313, + "18": 0.4032258064516129, + "19": 0.2719033232628399, + "20": 0.27927927927927926 + }, + "pass@128": { + "11": 0.9682539682539683, + "12": 0.8899082568807339, + "13": 0.7654320987654321, + "14": 0.8011869436201781, + "15": 0.7345679012345679, + "16": 0.6224188790560472, + "17": 0.5060240963855421, + "18": 0.44193548387096776, + "19": 0.311178247734139, + "20": 0.3003003003003003 + } + }, + "total_pass_at_k": { + "pass@1": 0.29492665036674814, + "pass@2": 0.35146699266503667, + "pass@4": 0.4077017114914425, + "pass@8": 0.4654645476772616, + "pass@16": 0.5171149144254279, + "pass@32": 0.5602078239608802, + "pass@64": 0.6017726161369193, + "pass@128": 0.6329462102689487 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5963855421686747, + "pass@2": 0.6656626506024096, + "pass@4": 0.7168674698795181, + "pass@8": 0.8132530120481928, + "pass@16": 0.891566265060241, + "pass@32": 0.927710843373494, + "pass@64": 0.9518072289156626, + "pass@128": 0.9668674698795181 + }, + "12": { + "pass@1": 0.48589341692789967, + "pass@2": 0.542319749216301, + "pass@4": 0.6677115987460815, + "pass@8": 0.7554858934169278, + "pass@16": 0.8025078369905956, + "pass@32": 0.8401253918495298, + "pass@64": 0.8557993730407524, + "pass@128": 0.877742946708464 + }, + "13": { + "pass@1": 0.42296918767507, + "pass@2": 0.49299719887955185, + "pass@4": 0.5714285714285714, + "pass@8": 0.6610644257703081, + "pass@16": 0.7338935574229691, + "pass@32": 0.7563025210084033, + "pass@64": 0.8067226890756303, + "pass@128": 0.8319327731092437 + }, + "14": { + "pass@1": 0.3567251461988304, + "pass@2": 0.4239766081871345, + "pass@4": 0.45614035087719296, + "pass@8": 0.5146198830409356, + "pass@16": 0.5818713450292398, + "pass@32": 0.6608187134502924, + "pass@64": 0.7046783625730995, + "pass@128": 0.7719298245614035 + }, + "15": { + "pass@1": 0.3151515151515151, + "pass@2": 0.37575757575757573, + "pass@4": 0.45151515151515154, + "pass@8": 0.5363636363636364, + "pass@16": 0.6090909090909091, + "pass@32": 0.6606060606060606, + "pass@64": 0.7212121212121212, + "pass@128": 0.7666666666666667 + }, + "16": { + "pass@1": 0.21082621082621084, + "pass@2": 0.28205128205128205, + "pass@4": 0.33618233618233617, + "pass@8": 0.3817663817663818, + "pass@16": 0.45584045584045585, + "pass@32": 0.5185185185185185, + "pass@64": 0.5612535612535613, + "pass@128": 0.603988603988604 + }, + "17": { + "pass@1": 0.17391304347826086, + "pass@2": 0.21739130434782608, + "pass@4": 0.2701863354037267, + "pass@8": 0.2981366459627329, + "pass@16": 0.35093167701863354, + "pass@32": 0.41304347826086957, + "pass@64": 0.4503105590062112, + "pass@128": 0.5031055900621118 + }, + "18": { + "pass@1": 0.18235294117647058, + "pass@2": 0.22647058823529412, + "pass@4": 0.25588235294117645, + "pass@8": 0.28823529411764703, + "pass@16": 0.31470588235294117, + "pass@32": 0.3352941176470588, + "pass@64": 0.37941176470588234, + "pass@128": 0.4470588235294118 + }, + "19": { + "pass@1": 0.09523809523809523, + "pass@2": 0.13988095238095238, + "pass@4": 0.16071428571428573, + "pass@8": 0.18154761904761904, + "pass@16": 0.21428571428571427, + "pass@32": 0.23809523809523808, + "pass@64": 0.27380952380952384, + "pass@128": 0.3125 + }, + "20": { + "pass@1": 0.10479041916167664, + "pass@2": 0.1377245508982036, + "pass@4": 0.16167664670658682, + "pass@8": 0.20059880239520958, + "pass@16": 0.2215568862275449, + "pass@32": 0.24850299401197604, + "pass@64": 0.27844311377245506, + "pass@128": 0.30538922155688625 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5963855421686747, + "12": 0.48589341692789967, + "13": 0.42296918767507, + "14": 0.3567251461988304, + "15": 0.3151515151515151, + "16": 0.21082621082621084, + "17": 0.17391304347826086, + "18": 0.18235294117647058, + "19": 0.09523809523809523, + "20": 0.10479041916167664 + }, + "pass@2": { + "11": 0.6656626506024096, + "12": 0.542319749216301, + "13": 0.49299719887955185, + "14": 0.4239766081871345, + "15": 0.37575757575757573, + "16": 0.28205128205128205, + "17": 0.21739130434782608, + "18": 0.22647058823529412, + "19": 0.13988095238095238, + "20": 0.1377245508982036 + }, + "pass@4": { + "11": 0.7168674698795181, + "12": 0.6677115987460815, + "13": 0.5714285714285714, + "14": 0.45614035087719296, + "15": 0.45151515151515154, + "16": 0.33618233618233617, + "17": 0.2701863354037267, + "18": 0.25588235294117645, + "19": 0.16071428571428573, + "20": 0.16167664670658682 + }, + "pass@8": { + "11": 0.8132530120481928, + "12": 0.7554858934169278, + "13": 0.6610644257703081, + "14": 0.5146198830409356, + "15": 0.5363636363636364, + "16": 0.3817663817663818, + "17": 0.2981366459627329, + "18": 0.28823529411764703, + "19": 0.18154761904761904, + "20": 0.20059880239520958 + }, + "pass@16": { + "11": 0.891566265060241, + "12": 0.8025078369905956, + "13": 0.7338935574229691, + "14": 0.5818713450292398, + "15": 0.6090909090909091, + "16": 0.45584045584045585, + "17": 0.35093167701863354, + "18": 0.31470588235294117, + "19": 0.21428571428571427, + "20": 0.2215568862275449 + }, + "pass@32": { + "11": 0.927710843373494, + "12": 0.8401253918495298, + "13": 0.7563025210084033, + "14": 0.6608187134502924, + "15": 0.6606060606060606, + "16": 0.5185185185185185, + "17": 0.41304347826086957, + "18": 0.3352941176470588, + "19": 0.23809523809523808, + "20": 0.24850299401197604 + }, + "pass@64": { + "11": 0.9518072289156626, + "12": 0.8557993730407524, + "13": 0.8067226890756303, + "14": 0.7046783625730995, + "15": 0.7212121212121212, + "16": 0.5612535612535613, + "17": 0.4503105590062112, + "18": 0.37941176470588234, + "19": 0.27380952380952384, + "20": 0.27844311377245506 + }, + "pass@128": { + "11": 0.9668674698795181, + "12": 0.877742946708464, + "13": 0.8319327731092437, + "14": 0.7719298245614035, + "15": 0.7666666666666667, + "16": 0.603988603988604, + "17": 0.5031055900621118, + "18": 0.4470588235294118, + "19": 0.3125, + "20": 0.30538922155688625 + } + }, + "total_pass_at_k": { + "pass@1": 0.29408266428783825, + "pass@2": 0.3502824858757062, + "pass@4": 0.40440083258994947, + "pass@8": 0.4626821290514422, + "pass@16": 0.5173951828724354, + "pass@32": 0.55961938745168, + "pass@64": 0.5982753493904253, + "pass@128": 0.6387154326494202 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.565, + "pass@2": 0.637, + "pass@4": 0.723, + "pass@8": 0.822, + "pass@16": 0.882, + "pass@32": 0.916, + "pass@64": 0.938, + "pass@128": 0.954 + }, + "12": { + "pass@1": 0.516, + "pass@2": 0.596, + "pass@4": 0.688, + "pass@8": 0.762, + "pass@16": 0.815, + "pass@32": 0.851, + "pass@64": 0.867, + "pass@128": 0.88 + }, + "13": { + "pass@1": 0.421, + "pass@2": 0.498, + "pass@4": 0.568, + "pass@8": 0.643, + "pass@16": 0.707, + "pass@32": 0.735, + "pass@64": 0.775, + "pass@128": 0.803 + }, + "14": { + "pass@1": 0.353, + "pass@2": 0.422, + "pass@4": 0.472, + "pass@8": 0.531, + "pass@16": 0.598, + "pass@32": 0.677, + "pass@64": 0.733, + "pass@128": 0.788 + }, + "15": { + "pass@1": 0.298, + "pass@2": 0.355, + "pass@4": 0.409, + "pass@8": 0.483, + "pass@16": 0.556, + "pass@32": 0.621, + "pass@64": 0.698, + "pass@128": 0.748 + }, + "16": { + "pass@1": 0.24, + "pass@2": 0.293, + "pass@4": 0.347, + "pass@8": 0.394, + "pass@16": 0.456, + "pass@32": 0.51, + "pass@64": 0.556, + "pass@128": 0.595 + }, + "17": { + "pass@1": 0.199, + "pass@2": 0.232, + "pass@4": 0.281, + "pass@8": 0.317, + "pass@16": 0.367, + "pass@32": 0.426, + "pass@64": 0.481, + "pass@128": 0.521 + }, + "18": { + "pass@1": 0.164, + "pass@2": 0.206, + "pass@4": 0.251, + "pass@8": 0.278, + "pass@16": 0.309, + "pass@32": 0.345, + "pass@64": 0.39, + "pass@128": 0.44 + }, + "19": { + "pass@1": 0.091, + "pass@2": 0.127, + "pass@4": 0.152, + "pass@8": 0.182, + "pass@16": 0.206, + "pass@32": 0.241, + "pass@64": 0.265, + "pass@128": 0.302 + }, + "20": { + "pass@1": 0.088, + "pass@2": 0.128, + "pass@4": 0.161, + "pass@8": 0.203, + "pass@16": 0.228, + "pass@32": 0.25, + "pass@64": 0.279, + "pass@128": 0.308 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.565, + "12": 0.516, + "13": 0.421, + "14": 0.353, + "15": 0.298, + "16": 0.24, + "17": 0.199, + "18": 0.164, + "19": 0.091, + "20": 0.088 + }, + "pass@2": { + "11": 0.637, + "12": 0.596, + "13": 0.498, + "14": 0.422, + "15": 0.355, + "16": 0.293, + "17": 0.232, + "18": 0.206, + "19": 0.127, + "20": 0.128 + }, + "pass@4": { + "11": 0.723, + "12": 0.688, + "13": 0.568, + "14": 0.472, + "15": 0.409, + "16": 0.347, + "17": 0.281, + "18": 0.251, + "19": 0.152, + "20": 0.161 + }, + "pass@8": { + "11": 0.822, + "12": 0.762, + "13": 0.643, + "14": 0.531, + "15": 0.483, + "16": 0.394, + "17": 0.317, + "18": 0.278, + "19": 0.182, + "20": 0.203 + }, + "pass@16": { + "11": 0.882, + "12": 0.815, + "13": 0.707, + "14": 0.598, + "15": 0.556, + "16": 0.456, + "17": 0.367, + "18": 0.309, + "19": 0.206, + "20": 0.228 + }, + "pass@32": { + "11": 0.916, + "12": 0.851, + "13": 0.735, + "14": 0.677, + "15": 0.621, + "16": 0.51, + "17": 0.426, + "18": 0.345, + "19": 0.241, + "20": 0.25 + }, + "pass@64": { + "11": 0.938, + "12": 0.867, + "13": 0.775, + "14": 0.733, + "15": 0.698, + "16": 0.556, + "17": 0.481, + "18": 0.39, + "19": 0.265, + "20": 0.279 + }, + "pass@128": { + "11": 0.954, + "12": 0.88, + "13": 0.803, + "14": 0.788, + "15": 0.748, + "16": 0.595, + "17": 0.521, + "18": 0.44, + "19": 0.302, + "20": 0.308 + } + }, + "total_pass_at_k": { + "pass@1": 0.2935, + "pass@2": 0.3494, + "pass@4": 0.4052, + "pass@8": 0.4615, + "pass@16": 0.5124, + "pass@32": 0.5572, + "pass@64": 0.5982, + "pass@128": 0.6339 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block4-epoch1-op17-20/summary.csv b/composition/SDAR-100M-block4-epoch1-op17-20/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..ba3e1516e9556df311f37d5351b1bab898ee068a --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op17-20/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch1-op17-20,0.9801223958333334,0.62273671875,0.7920246710526315,0.0,0.0,0.0,199.76062586805557,372.483859375,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block4-epoch1-op17-20/summary.json b/composition/SDAR-100M-block4-epoch1-op17-20/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f0909cf0fafc05ec1cc2020689d627518d5e578a --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op17-20/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch1-op17-20", + "id_acc": 0.9801223958333334, + "ood_acc": 0.62273671875, + "total_acc": 0.7920246710526315, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.76062586805557, + "ood_avg_resp_len": 372.483859375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 640086, + "answer_accuracy": 0.790495079829276, + "avg_response_len": 277.60216270154916, + "resp_tokens_sum": 224782244, + "pass_at_k": { + "pass@1": 0.790495079829276, + "pass@2": 0.8362411306855028, + "pass@4": 0.8696372919991009, + "pass@8": 0.8945669610675093, + "pass@16": 0.913432249792536, + "pass@32": 0.9280573139004922, + "pass@64": 0.9398972482792994, + "pass@128": 0.9494151122352197 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9192477876106194, + "pass@2": 0.9396735564304464, + "pass@4": 0.9543033865788902, + "pass@8": 0.96485112476411, + "pass@16": 0.9724661046772268, + "pass@32": 0.9786862175801799, + "pass@64": 0.9841720997110378, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.9998296339563862, + "pass@2": 0.9999961672431132, + "pass@4": 0.9999999985399022, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995460626911316, + "pass@2": 0.999965009391028, + "pass@4": 0.9999997130519236, + "pass@8": 0.9999999999935767, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995849609375, + "pass@2": 0.996728515625, + "pass@4": 0.9973700640935508, + "pass@8": 0.9978945203621713, + "pass@16": 0.9986215319076661, + "pass@32": 0.99946584925741, + "pass@64": 0.999956800941041, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9784474206349206, + "pass@2": 0.9849374296962881, + "pass@4": 0.9890754283690729, + "pass@8": 0.9921342884438079, + "pass@16": 0.9943983611486136, + "pass@32": 0.9960648708043484, + "pass@64": 0.9967769255677263, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9922342814371258, + "pass@2": 0.9939438298599651, + "pass@4": 0.9941038252516338, + "pass@8": 0.9941991013702666, + "pass@16": 0.9943862275449101, + "pass@32": 0.9947604790419161, + "pass@64": 0.9955089820359282, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9887435281065089, + "pass@2": 0.9931025922517821, + "pass@4": 0.995356032973689, + "pass@8": 0.99687459097789, + "pass@16": 0.997698902492233, + "pass@32": 0.9983400676934957, + "pass@64": 0.9992661790057259, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9668469551282052, + "pass@2": 0.9809139505097918, + "pass@4": 0.9884821623859518, + "pass@8": 0.9934550534186496, + "pass@16": 0.9968600221493318, + "pass@32": 0.9988651869448563, + "pass@64": 0.9998085724581219, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9736575704225352, + "pass@2": 0.987104566374626, + "pass@4": 0.9918345429532572, + "pass@8": 0.9940607002019898, + "pass@16": 0.9952133611440159, + "pass@32": 0.9960034370145185, + "pass@64": 0.9968393035368946, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9049442280453258, + "pass@2": 0.9343667049586221, + "pass@4": 0.9518926340829211, + "pass@8": 0.963721513224062, + "pass@16": 0.9715522567392245, + "pass@32": 0.9772850487323939, + "pass@64": 0.9837385106596693, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8367319915254238, + "pass@2": 0.8906847780150366, + "pass@4": 0.9210060177859121, + "pass@8": 0.9395740180895393, + "pass@16": 0.9513328654341852, + "pass@32": 0.9589354102848824, + "pass@64": 0.9654301877762941, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.7604574843260188, + "pass@2": 0.82680597894503, + "pass@4": 0.8719512272956471, + "pass@8": 0.9045499773982028, + "pass@16": 0.9266465799635933, + "pass@32": 0.9410437423571842, + "pass@64": 0.9509158308973618, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.7687889408099688, + "pass@2": 0.8359775523094659, + "pass@4": 0.8736571856672118, + "pass@8": 0.8984409718591754, + "pass@16": 0.9181805371311775, + "pass@32": 0.9350351705266242, + "pass@64": 0.9502913100353946, + "pass@128": 0.9626168224299065 + }, + "15": { + "pass@1": 0.6999638728323699, + "pass@2": 0.7756271761640345, + "pass@4": 0.8292882124380401, + "pass@8": 0.869390789970966, + "pass@16": 0.8979501614129438, + "pass@32": 0.9183570437604774, + "pass@64": 0.9343777336335435, + "pass@128": 0.9450867052023122 + }, + "16": { + "pass@1": 0.5773941532258065, + "pass@2": 0.6756885795021594, + "pass@4": 0.7475061202027162, + "pass@8": 0.7991479896927378, + "pass@16": 0.8357307006684896, + "pass@32": 0.8628873839423076, + "pass@64": 0.8835885830672884, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.5140895953757225, + "pass@2": 0.6143314624960177, + "pass@4": 0.6916131304576808, + "pass@8": 0.7469650535201684, + "pass@16": 0.7860427028084981, + "pass@32": 0.8170511372956204, + "pass@64": 0.8421452842507103, + "pass@128": 0.8583815028901735 + }, + "18": { + "pass@1": 0.4582589285714286, + "pass@2": 0.5605532902137236, + "pass@4": 0.6421960956666121, + "pass@8": 0.7057777001868978, + "pass@16": 0.7555140844320166, + "pass@32": 0.7930716900033024, + "pass@64": 0.822862301855235, + "pass@128": 0.8514285714285714 + }, + "19": { + "pass@1": 0.3429288663663664, + "pass@2": 0.44343987195857243, + "pass@4": 0.531017780998096, + "pass@8": 0.602912347397304, + "pass@16": 0.6667933588207599, + "pass@32": 0.725113626356638, + "pass@64": 0.775691031987435, + "pass@128": 0.8168168168168168 + }, + "20": { + "pass@1": 0.36878284534534533, + "pass@2": 0.4766399026979738, + "pass@4": 0.570342090403114, + "pass@8": 0.6477507011049768, + "pass@16": 0.7086113239762887, + "pass@32": 0.7532795279937846, + "pass@64": 0.7862179025168473, + "pass@128": 0.8108108108108109 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 651414, + "answer_accuracy": 0.7933237529228371, + "avg_response_len": 286.8824020849571, + "resp_tokens_sum": 235564878, + "pass_at_k": { + "pass@1": 0.7933237529228371, + "pass@2": 0.8389930634401326, + "pass@4": 0.8726874711442432, + "pass@8": 0.8977972002351476, + "pass@16": 0.9167457991803611, + "pass@32": 0.9314659835787653, + "pass@64": 0.9432935185698375, + "pass@128": 0.9522992985190959 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9164350076687117, + "pass@2": 0.9361730713492108, + "pass@4": 0.9499516876916461, + "pass@8": 0.9618509716297367, + "pass@16": 0.9718630754382047, + "pass@32": 0.9796931696551145, + "pass@64": 0.986608430448021, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9973958333333334, + "pass@2": 0.9992700131233596, + "pass@4": 0.9999176963674994, + "pass@8": 0.9999987979293221, + "pass@16": 0.9999999998399193, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9990292159763313, + "pass@2": 0.9998897084517542, + "pass@4": 0.9999981377142946, + "pass@8": 0.9999999995762411, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.980570652173913, + "pass@2": 0.9865802949903003, + "pass@4": 0.9889661506985535, + "pass@8": 0.9902841199998196, + "pass@16": 0.9917663843037684, + "pass@32": 0.9936594509266741, + "pass@64": 0.9954604899202021, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.995008058739255, + "pass@2": 0.9963252713038379, + "pass@4": 0.9970167348350797, + "pass@8": 0.9973006814242409, + "pass@16": 0.997492802179556, + "pass@32": 0.9978510028653035, + "pass@64": 0.998567335243553, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9898843930635838, + "pass@2": 0.9931856196804879, + "pass@4": 0.9941058232063475, + "pass@8": 0.9943886704848257, + "pass@16": 0.9945808983691612, + "pass@32": 0.9949421965317783, + "pass@64": 0.9956647398843931, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9714788732394366, + "pass@2": 0.980618207275147, + "pass@4": 0.9870130438800786, + "pass@8": 0.9915782021625016, + "pass@16": 0.9946980159453276, + "pass@32": 0.9967270976372445, + "pass@64": 0.9986017780742946, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9722352170658682, + "pass@2": 0.9828459468857563, + "pass@4": 0.9884671896364751, + "pass@8": 0.9924020194851084, + "pass@16": 0.9944314239658919, + "pass@32": 0.9953209435293606, + "pass@64": 0.9962633785477046, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9257341867469879, + "pass@2": 0.9553495725026092, + "pass@4": 0.9682735500494665, + "pass@8": 0.9742548441493746, + "pass@16": 0.978185160269226, + "pass@32": 0.9822869779847758, + "pass@64": 0.9871868036674503, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8492358934169278, + "pass@2": 0.8949850973761508, + "pass@4": 0.9207448059000464, + "pass@8": 0.9354093985604018, + "pass@16": 0.943759425744798, + "pass@32": 0.9496060392426146, + "pass@64": 0.9561026755224603, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.740983893557423, + "pass@2": 0.8052852538653259, + "pass@4": 0.845815909040781, + "pass@8": 0.8738550513505369, + "pass@16": 0.895515341616072, + "pass@32": 0.9155451542423412, + "pass@64": 0.9353134265154689, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7266081871345029, + "pass@2": 0.7990618668554583, + "pass@4": 0.8478731022109073, + "pass@8": 0.8837960270998939, + "pass@16": 0.9105606572428289, + "pass@32": 0.930775272521883, + "pass@64": 0.9463526228698942, + "pass@128": 0.9590643274853801 + }, + "15": { + "pass@1": 0.6870501893939394, + "pass@2": 0.7726907361011697, + "pass@4": 0.8339367397257161, + "pass@8": 0.8770493599523325, + "pass@16": 0.9072087108288012, + "pass@32": 0.9277764122574453, + "pass@64": 0.9424641653105227, + "pass@128": 0.9545454545454546 + }, + "16": { + "pass@1": 0.5663728632478633, + "pass@2": 0.668794655876349, + "pass@4": 0.74511328258113, + "pass@8": 0.8007343935830623, + "pass@16": 0.8421667071713081, + "pass@32": 0.8733083123762971, + "pass@64": 0.8957480080042212, + "pass@128": 0.9116809116809117 + }, + "17": { + "pass@1": 0.5376067546583851, + "pass@2": 0.6423940553626447, + "pass@4": 0.7202086238637875, + "pass@8": 0.7743041381948508, + "pass@16": 0.814345713501841, + "pass@32": 0.8470088033255796, + "pass@64": 0.8726913345902738, + "pass@128": 0.8881987577639752 + }, + "18": { + "pass@1": 0.5092830882352941, + "pass@2": 0.6142293162343674, + "pass@4": 0.7005960897351063, + "pass@8": 0.7701591745990866, + "pass@16": 0.8221200586922632, + "pass@32": 0.8587938588657354, + "pass@64": 0.8857339251123276, + "pass@128": 0.9058823529411765 + }, + "19": { + "pass@1": 0.34770275297619047, + "pass@2": 0.44574237790588667, + "pass@4": 0.5338633851014157, + "pass@8": 0.611502029947303, + "pass@16": 0.6783513518981501, + "pass@32": 0.7327937015644227, + "pass@64": 0.7742047877222114, + "pass@128": 0.8035714285714286 + }, + "20": { + "pass@1": 0.35043974550898205, + "pass@2": 0.4594935375312366, + "pass@4": 0.5528386059714089, + "pass@8": 0.6238549348742458, + "pass@16": 0.6762174612820911, + "pass@32": 0.7171414832463683, + "pass@64": 0.7511741428622221, + "pass@128": 0.7754491017964071 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 634704, + "answer_accuracy": 0.7922391755871545, + "avg_response_len": 307.75241027919793, + "resp_tokens_sum": 246556459, + "pass_at_k": { + "pass@1": 0.7922391755871545, + "pass@2": 0.8386018031672131, + "pass@4": 0.8729426678002086, + "pass@8": 0.8980563358146915, + "pass@16": 0.9163577796796314, + "pass@32": 0.9303137674409322, + "pass@64": 0.9412975982929933, + "pass@128": 0.9496724716408372 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.933535447761194, + "pass@2": 0.9610144405923143, + "pass@4": 0.9764659397052976, + "pass@8": 0.9848856698483921, + "pass@16": 0.9903880371665308, + "pass@32": 0.994788130439683, + "pass@64": 0.9981423348247224, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961962463556852, + "pass@2": 0.9970163965473705, + "pass@4": 0.997338775072941, + "pass@8": 0.9976015064053606, + "pass@16": 0.9980534958639686, + "pass@32": 0.9987797625879855, + "pass@64": 0.9996441771309199, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9991776315789473, + "pass@2": 0.999924094603306, + "pass@4": 0.9999989030976391, + "pass@8": 0.999999999845404, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9750229779411764, + "pass@2": 0.9829337222093564, + "pass@4": 0.9882893535366903, + "pass@8": 0.9921605412616765, + "pass@16": 0.9948705139434515, + "pass@32": 0.9969973900489489, + "pass@64": 0.998910627773521, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9884660883280757, + "pass@2": 0.9913928407312652, + "pass@4": 0.9938846173408135, + "pass@8": 0.9958811876044368, + "pass@16": 0.9974790678017365, + "pass@32": 0.9989033586934598, + "pass@64": 0.9998111525900316, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9794056566455697, + "pass@2": 0.9909034560948872, + "pass@4": 0.9964178719155359, + "pass@8": 0.9984878997029001, + "pass@16": 0.9994102329414561, + "pass@32": 0.9999166822897716, + "pass@64": 0.9999995619546246, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9842342342342343, + "pass@2": 0.9942238005722256, + "pass@4": 0.9981784532001067, + "pass@8": 0.9995914077213195, + "pass@16": 0.9999633430821163, + "pass@32": 0.9999998023813371, + "pass@64": 0.9999999999999107, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9692021704180064, + "pass@2": 0.983175937412968, + "pass@4": 0.9883279185158123, + "pass@8": 0.9908531152358594, + "pass@16": 0.9925946265968474, + "pass@32": 0.9942363552283263, + "pass@64": 0.9958943717110796, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.915327380952381, + "pass@2": 0.948085786151731, + "pass@4": 0.9642386070788763, + "pass@8": 0.9739282351224063, + "pass@16": 0.9804970648118635, + "pass@32": 0.9846325349921398, + "pass@64": 0.986722881877331, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8619313837920489, + "pass@2": 0.912293969274483, + "pass@4": 0.9437201129675306, + "pass@8": 0.9619229572767756, + "pass@16": 0.9731904957944275, + "pass@32": 0.9818321050690701, + "pass@64": 0.9884511684407877, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.7652874228395061, + "pass@2": 0.8321079548216199, + "pass@4": 0.8720276435352986, + "pass@8": 0.896107615787007, + "pass@16": 0.9136146173645132, + "pass@32": 0.929635398081815, + "pass@64": 0.9451059800569461, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.7490031528189911, + "pass@2": 0.8191228913058717, + "pass@4": 0.864718364024971, + "pass@8": 0.8938565907497251, + "pass@16": 0.9139148391085333, + "pass@32": 0.9293839114599252, + "pass@64": 0.9403161156625768, + "pass@128": 0.9465875370919882 + }, + "15": { + "pass@1": 0.6787471064814815, + "pass@2": 0.7596807560513266, + "pass@4": 0.8202685196757814, + "pass@8": 0.8636306777109823, + "pass@16": 0.8919877814582083, + "pass@32": 0.9093606678534814, + "pass@64": 0.9216089042968235, + "pass@128": 0.9320987654320988 + }, + "16": { + "pass@1": 0.5669478244837758, + "pass@2": 0.6690786646691279, + "pass@4": 0.7455410565382864, + "pass@8": 0.8028333423672143, + "pass@16": 0.8423557007877164, + "pass@32": 0.8684380664265011, + "pass@64": 0.8872958659682277, + "pass@128": 0.9026548672566371 + }, + "17": { + "pass@1": 0.5101891942771084, + "pass@2": 0.6125037798833123, + "pass@4": 0.6895920137241878, + "pass@8": 0.7465083545933107, + "pass@16": 0.7900024434338143, + "pass@32": 0.8252868759565376, + "pass@64": 0.85326005071428, + "pass@128": 0.8734939759036144 + }, + "18": { + "pass@1": 0.4936743951612903, + "pass@2": 0.5874368967487934, + "pass@4": 0.6604457161604794, + "pass@8": 0.7140707542442847, + "pass@16": 0.7545067071870171, + "pass@32": 0.7888352910327632, + "pass@64": 0.8177670834201991, + "pass@128": 0.8419354838709677 + }, + "19": { + "pass@1": 0.3340728851963746, + "pass@2": 0.4338164890453644, + "pass@4": 0.5283597530860003, + "pass@8": 0.6090080248672055, + "pass@16": 0.6734085647143847, + "pass@32": 0.7271711226701715, + "pass@64": 0.7742411529076022, + "pass@128": 0.8157099697885196 + }, + "20": { + "pass@1": 0.34025431681681684, + "pass@2": 0.45003221725662673, + "pass@4": 0.5509912418042338, + "pass@8": 0.6353882561628813, + "pass@16": 0.6987590181595222, + "pass@32": 0.7426487056681589, + "pass@64": 0.7730709505896831, + "pass@128": 0.7957957957957958 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_metrics.json b/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..380b9b3e4e659b236aa5e7da521a47e22300abb2 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch1-op7-10", + "id": { + "answer_accuracy": 0.9879244791666667, + "avg_response_len": 200.573390625, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9879244791666667, + "pass@2": 0.9915930938320194, + "pass@4": 0.9936487634462353, + "pass@8": 0.9949288086694491, + "pass@16": 0.9958718418730274, + "pass@32": 0.9967797861614692, + "pass@64": 0.9977356694007103, + "pass@128": 0.9984444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9435703125, + "pass@2": 0.9607268700787404, + "pass@4": 0.9712526024559434, + "pass@8": 0.9776772575007111, + "pass@16": 0.9820311480328475, + "pass@32": 0.9859655999840329, + "pass@64": 0.9900584070418624, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9996328125, + "pass@2": 0.9999351624015749, + "pass@4": 0.9999966226096737, + "pass@8": 0.9999999926430151, + "pass@16": 0.9999999999999936, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998328125, + "pass@2": 0.9986950049212598, + "pass@4": 0.9989937330333709, + "pass@8": 0.9992152625181484, + "pass@16": 0.9994176619420255, + "pass@32": 0.9996886051712321, + "pass@64": 0.9999404409448819, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9875625, + "pass@2": 0.9903880413385826, + "pass@4": 0.9921799958755155, + "pass@8": 0.9936264938768348, + "pass@16": 0.9948819632043239, + "pass@32": 0.996308463535151, + "pass@64": 0.997997987322932, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.995953125, + "pass@2": 0.9969644438976379, + "pass@4": 0.9975323900449944, + "pass@8": 0.997917797864405, + "pass@16": 0.9982963251514316, + "pass@32": 0.998686418855384, + "pass@64": 0.9992519681067729, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.992796875, + "pass@2": 0.9954954478346455, + "pass@4": 0.9970782049118859, + "pass@8": 0.9980214999423029, + "pass@16": 0.998521787995179, + "pass@32": 0.9988284479182614, + "pass@64": 0.9989861762904964, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.987875, + "pass@2": 0.9913147145669291, + "pass@4": 0.9929255646794151, + "pass@8": 0.9939211132530895, + "pass@16": 0.9947769123750306, + "pass@32": 0.9956578151347028, + "pass@64": 0.9964109543118082, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.985609375, + "pass@2": 0.9908181594488191, + "pass@4": 0.9928797574053245, + "pass@8": 0.9939798604265343, + "pass@16": 0.9949207781564106, + "pass@32": 0.9958827248544599, + "pass@64": 0.9969750905876388, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.9435703125, + "2": 0.9999921875, + "3": 0.9996328125, + "4": 0.998328125, + "5": 0.9875625, + "6": 0.995953125, + "7": 0.992796875, + "8": 0.987875, + "9": 0.985609375 + }, + "per_op_avg_response_len": { + "10": 281.299859375, + "2": 115.78703125, + "3": 145.00421875, + "4": 160.341109375, + "5": 174.0563359375, + "6": 199.62221875, + "7": 224.5571171875, + "8": 244.237671875, + "9": 260.254953125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.299859375, + "2": 115.78703125, + "3": 145.00421875, + "4": 160.341109375, + "5": 174.0563359375, + "6": 199.62221875, + "7": 224.5571171875, + "8": 244.237671875, + "9": 260.254953125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 374119, + "answer_accuracy": 0.9871005361364404, + "avg_response_len": 191.1615717874029, + "resp_tokens_sum": 72451765, + "pass_at_k": { + "pass@1": 0.9871005361364404, + "pass@2": 0.9910629106866963, + "pass@4": 0.9933770916046847, + "pass@8": 0.9948866569837124, + "pass@16": 0.9960565960798013, + "pass@32": 0.9972200967202242, + "pass@64": 0.9984194158450799, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9426622418879056, + "pass@2": 0.9608814281234755, + "pass@4": 0.9727997515244226, + "pass@8": 0.9800427227199637, + "pass@16": 0.9848555713566579, + "pass@32": 0.9893375479172565, + "pass@64": 0.9948648057621975, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999522171253823, + "pass@2": 0.9999996237568928, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995068359375, + "pass@2": 0.995943036417323, + "pass@4": 0.996855512396888, + "pass@8": 0.9975476953695743, + "pass@16": 0.9981801935688303, + "pass@32": 0.9990268911601001, + "pass@64": 0.9998138779527558, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9858382936507937, + "pass@2": 0.9894134326959131, + "pass@4": 0.9919654816957404, + "pass@8": 0.993967779218521, + "pass@16": 0.9953518900581235, + "pass@32": 0.9962822752680421, + "pass@64": 0.9967815120663723, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9956025449101796, + "pass@2": 0.9967230892545617, + "pass@4": 0.9970756080639619, + "pass@8": 0.9971929848408553, + "pass@16": 0.9973802395197485, + "pass@32": 0.9977544910179641, + "pass@64": 0.9985029940119761, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9944526627218935, + "pass@2": 0.9979470484088896, + "pass@4": 0.9992837274778522, + "pass@8": 0.9998536405673738, + "pass@16": 0.9999944055596715, + "pass@32": 0.9999999969008433, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9830729166666666, + "pass@2": 0.9861033559206539, + "pass@4": 0.987899911849961, + "pass@8": 0.9900541498080317, + "pass@16": 0.9924767953718153, + "pass@32": 0.9948968439236284, + "pass@64": 0.9965094689481032, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9885343309859155, + "pass@2": 0.9931147415992014, + "pass@4": 0.9947651726632762, + "pass@8": 0.9954927131433333, + "pass@16": 0.9964173683303595, + "pass@32": 0.9977688493163707, + "pass@64": 0.9992201397360542, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 385895, + "answer_accuracy": 0.9878128071756226, + "avg_response_len": 198.62987641300785, + "resp_tokens_sum": 77595953, + "pass_at_k": { + "pass@1": 0.9878128071756226, + "pass@2": 0.9907689694249806, + "pass@4": 0.9925758746899758, + "pass@8": 0.9937724075737618, + "pass@16": 0.994610426375753, + "pass@32": 0.9954936548420763, + "pass@64": 0.9965385954078559, + "pass@128": 0.9973787680209698 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.929879217791411, + "pass@2": 0.9455011986135936, + "pass@4": 0.9572367023907287, + "pass@8": 0.9653763793871797, + "pass@16": 0.9709467034438971, + "pass@32": 0.9763425037418643, + "pass@64": 0.9817604399544487, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997395833333333, + "pass@2": 0.999989188141255, + "pass@4": 0.9999999801161218, + "pass@8": 0.9999999999999979, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997457470414202, + "pass@2": 0.9999799800820015, + "pass@4": 0.9999999084803749, + "pass@8": 0.9999999999996586, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941123188405797, + "pass@2": 0.9961107782722813, + "pass@4": 0.9970168766947609, + "pass@8": 0.997577219367406, + "pass@16": 0.9980644436436175, + "pass@32": 0.998786836422768, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970898997134671, + "pass@2": 0.9972238595086073, + "pass@4": 0.9973116384270019, + "pass@8": 0.9974829659544705, + "pass@16": 0.9978086997721273, + "pass@32": 0.9983924824583174, + "pass@64": 0.9992893080342035, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956195809248555, + "pass@2": 0.996662859564881, + "pass@4": 0.9970522314587844, + "pass@8": 0.9971089149255284, + "pass@16": 0.997109826485928, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9900308098591549, + "pass@2": 0.9928277836309195, + "pass@4": 0.9939063929508812, + "pass@8": 0.9944474182396722, + "pass@16": 0.9947159353923489, + "pass@32": 0.9950704220578349, + "pass@64": 0.995774647887324, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9818488023952096, + "pass@2": 0.9868720973407518, + "pass@4": 0.9893204078344998, + "pass@8": 0.9909175880114821, + "pass@16": 0.9920082924256072, + "pass@32": 0.993079471807874, + "pass@64": 0.994766372753843, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 378075, + "answer_accuracy": 0.9888553523602277, + "avg_response_len": 211.88909231670573, + "resp_tokens_sum": 81012828, + "pass_at_k": { + "pass@1": 0.9888553523602277, + "pass@2": 0.9929607202075135, + "pass@4": 0.9950143064013551, + "pass@8": 0.9961521589491618, + "pass@16": 0.9969775609528468, + "pass@32": 0.9976574270125961, + "pass@64": 0.9982809980262249, + "pass@128": 0.9986608637428859 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9578125, + "pass@2": 0.9753870901398526, + "pass@4": 0.9833263334620483, + "pass@8": 0.98725395468186, + "pass@16": 0.9899596537917327, + "pass@32": 0.9919179552842842, + "pass@64": 0.9932696551144083, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999225583090379, + "pass@2": 0.9998217298730516, + "pass@4": 0.9999901725112349, + "pass@8": 0.9999999785510661, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9825137867647059, + "pass@2": 0.9854840927512737, + "pass@4": 0.9874707255343083, + "pass@8": 0.9893014786507219, + "pass@16": 0.991217308173432, + "pass@32": 0.9938179066466555, + "pass@64": 0.997452524312115, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.995070977917981, + "pass@2": 0.9969331391738493, + "pass@4": 0.998256703913588, + "pass@8": 0.9991602076007862, + "pass@16": 0.9997983877960986, + "pass@32": 0.9999919321054616, + "pass@64": 0.9999999987471165, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9879351265822784, + "pass@2": 0.9915949242499751, + "pass@4": 0.9947475725937738, + "pass@8": 0.9970610280579045, + "pass@16": 0.9984926549110091, + "pass@32": 0.9994571169803051, + "pass@64": 0.9999562540838489, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9900760135135135, + "pass@2": 0.9945843973895155, + "pass@4": 0.996588656710704, + "pass@8": 0.9969831382519523, + "pass@16": 0.9969969824466679, + "pass@32": 0.9969969969969951, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9863092845659164, + "pass@2": 0.9924345519912906, + "pass@4": 0.9945501765052679, + "pass@8": 0.9955417133916913, + "pass@16": 0.9963403303182642, + "pass@32": 0.9967403207823743, + "pass@64": 0.9967844774326561, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.43462734375, + "avg_response_len": 332.18273828125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.43462734375, + "pass@2": 0.5005182947834617, + "pass@4": 0.5548893307274047, + "pass@8": 0.6003465607833282, + "pass@16": 0.6390786153650679, + "pass@32": 0.6726684092686726, + "pass@64": 0.7024698115574631, + "pass@128": 0.73 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9058125, + "pass@2": 0.9348855807086626, + "pass@4": 0.9523219686914133, + "pass@8": 0.9639877513162247, + "pass@16": 0.9719139636312137, + "pass@32": 0.9772367936401185, + "pass@64": 0.9810516493173799, + "pass@128": 0.984 + }, + "12": { + "pass@1": 0.83046875, + "pass@2": 0.8840001230314961, + "pass@4": 0.9133514972815886, + "pass@8": 0.9323962777434657, + "pass@16": 0.9465618674560089, + "pass@32": 0.9567864902434278, + "pass@64": 0.9644186160238337, + "pass@128": 0.971 + }, + "13": { + "pass@1": 0.6527734375, + "pass@2": 0.7388490403543305, + "pass@4": 0.7958859667229103, + "pass@8": 0.8354775748363091, + "pass@16": 0.8643046264976756, + "pass@32": 0.8860494112925553, + "pass@64": 0.9030548966211872, + "pass@128": 0.917 + }, + "14": { + "pass@1": 0.5455625, + "pass@2": 0.6375738188976379, + "pass@4": 0.7070053682039741, + "pass@8": 0.7590985628570953, + "pass@16": 0.7975958195184475, + "pass@32": 0.8266908787586564, + "pass@64": 0.8494597774974879, + "pass@128": 0.868 + }, + "15": { + "pass@1": 0.406609375, + "pass@2": 0.4940346948818898, + "pass@4": 0.5684210074990619, + "pass@8": 0.6314350453515915, + "pass@16": 0.6841069770667974, + "pass@32": 0.7270360115703599, + "pass@64": 0.7626391528642439, + "pass@128": 0.794 + }, + "16": { + "pass@1": 0.2661484375, + "pass@2": 0.33698781988188975, + "pass@4": 0.4029694417885262, + "pass@8": 0.46251322880057405, + "pass@16": 0.5149965944758812, + "pass@32": 0.5617009345680936, + "pass@64": 0.6059889851985129, + "pass@128": 0.652 + }, + "17": { + "pass@1": 0.2140859375, + "pass@2": 0.28265403543307094, + "pass@4": 0.3472788755155604, + "pass@8": 0.40412559729262215, + "pass@16": 0.4549240276430866, + "pass@32": 0.501694346448931, + "pass@64": 0.545903991761549, + "pass@128": 0.589 + }, + "18": { + "pass@1": 0.2059765625, + "pass@2": 0.2638793061023622, + "pass@4": 0.31758961445444284, + "pass@8": 0.3670262508999904, + "pass@16": 0.41459444172398247, + "pass@32": 0.4628889802277007, + "pass@64": 0.5113301409316344, + "pass@128": 0.557 + }, + "19": { + "pass@1": 0.1695625, + "pass@2": 0.22803469488188977, + "pass@4": 0.2843005737720284, + "pass@8": 0.33478337444162465, + "pass@16": 0.37877477451174985, + "pass@32": 0.4178315933259912, + "pass@64": 0.45472330372441766, + "pass@128": 0.492 + }, + "20": { + "pass@1": 0.1492734375, + "pass@2": 0.20428383366141709, + "pass@4": 0.2597689933445819, + "pass@8": 0.3126219442937622, + "pass@16": 0.3630130611257066, + "pass@32": 0.40876865261089623, + "pass@64": 0.44612760163432663, + "pass@128": 0.476 + } + }, + "per_op_accuracy": { + "11": 0.9058125, + "12": 0.83046875, + "13": 0.6527734375, + "14": 0.5455625, + "15": 0.406609375, + "16": 0.2661484375, + "17": 0.2140859375, + "18": 0.2059765625, + "19": 0.1695625, + "20": 0.1492734375 + }, + "per_op_avg_response_len": { + "11": 298.0427109375, + "12": 308.0526796875, + "13": 320.6473828125, + "14": 334.90971875, + "15": 342.809359375, + "16": 331.71221875, + "17": 346.1571328125, + "18": 349.244828125, + "19": 339.274734375, + "20": 350.9766171875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.0427109375, + "12": 308.0526796875, + "13": 320.6473828125, + "14": 334.90971875, + "15": 342.809359375, + "16": 331.71221875, + "17": 346.1571328125, + "18": 349.244828125, + "19": 339.274734375, + "20": 350.9766171875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 185959, + "answer_accuracy": 0.43173987741456166, + "avg_response_len": 315.00408618127784, + "resp_tokens_sum": 135678560, + "pass_at_k": { + "pass@1": 0.43173987741456166, + "pass@2": 0.4975053673175705, + "pass@4": 0.5517866410550846, + "pass@8": 0.5971134428338623, + "pass@16": 0.6360000082863407, + "pass@32": 0.670023386734411, + "pass@64": 0.7002047454766951, + "pass@128": 0.7277860326894502 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9032179532577904, + "pass@2": 0.9327289570832682, + "pass@4": 0.9506815622905493, + "pass@8": 0.9626199544828037, + "pass@16": 0.9704751886621362, + "pass@32": 0.9749374452489946, + "pass@64": 0.9778428066504992, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8037826624293786, + "pass@2": 0.8664708589127629, + "pass@4": 0.9026233156236823, + "pass@8": 0.9260914408190967, + "pass@16": 0.9431713204069097, + "pass@32": 0.9549023146846267, + "pass@64": 0.9638589731267034, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.6582092476489029, + "pass@2": 0.743703024337867, + "pass@4": 0.8042070038071254, + "pass@8": 0.8481824192918073, + "pass@16": 0.877888284808596, + "pass@32": 0.8965888649622713, + "pass@64": 0.9091856827840178, + "pass@128": 0.9184952978056427 + }, + "14": { + "pass@1": 0.5494548286604362, + "pass@2": 0.6482951130816593, + "pass@4": 0.7280674673259301, + "pass@8": 0.7889771206999603, + "pass@16": 0.8312085331718211, + "pass@32": 0.8598877871539242, + "pass@64": 0.8815913721972909, + "pass@128": 0.9003115264797508 + }, + "15": { + "pass@1": 0.4004019147398844, + "pass@2": 0.4798765275818123, + "pass@4": 0.5449855960020893, + "pass@8": 0.6041668693159691, + "pass@16": 0.6603312308303183, + "pass@32": 0.711706774366159, + "pass@64": 0.7567645044134567, + "pass@128": 0.7947976878612717 + }, + "16": { + "pass@1": 0.2763860887096774, + "pass@2": 0.35277455867411744, + "pass@4": 0.42198129618878266, + "pass@8": 0.48262699673255544, + "pass@16": 0.5365621998798281, + "pass@32": 0.5864838038611693, + "pass@64": 0.6325214538236035, + "pass@128": 0.6774193548387096 + }, + "17": { + "pass@1": 0.19533507947976878, + "pass@2": 0.25727059248554907, + "pass@4": 0.31503235406629077, + "pass@8": 0.36664014844214665, + "pass@16": 0.415407852939479, + "pass@32": 0.46282528956899566, + "pass@64": 0.5089321507525993, + "pass@128": 0.5549132947976878 + }, + "18": { + "pass@1": 0.18772321428571428, + "pass@2": 0.24447272215972995, + "pass@4": 0.29619767395147045, + "pass@8": 0.34292779478769564, + "pass@16": 0.38793229002272034, + "pass@32": 0.43505393984941876, + "pass@64": 0.48128101171399873, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.1686373873873874, + "pass@2": 0.22988933815705478, + "pass@4": 0.28967174852017374, + "pass@8": 0.3425929783847028, + "pass@16": 0.3887203432902438, + "pass@32": 0.42960762813584363, + "pass@64": 0.46859317448539534, + "pass@128": 0.5105105105105106 + }, + "20": { + "pass@1": 0.14841403903903905, + "pass@2": 0.19893675663852825, + "pass@4": 0.24889135141891053, + "pass@8": 0.29487053016690784, + "pass@16": 0.3395049732401222, + "pass@32": 0.38101573199885735, + "pass@64": 0.4152574877923962, + "pass@128": 0.4444444444444444 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 184995, + "answer_accuracy": 0.42975719223907227, + "avg_response_len": 328.94082431980377, + "resp_tokens_sum": 141597183, + "pass_at_k": { + "pass@1": 0.42975719223907227, + "pass@2": 0.49514535935294074, + "pass@4": 0.5506315189359147, + "pass@8": 0.5981688833153738, + "pass@16": 0.6393251536784574, + "pass@32": 0.675103391235929, + "pass@64": 0.706971814163072, + "pass@128": 0.735950044603033 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.913074171686747, + "pass@2": 0.9404327447585614, + "pass@4": 0.9566515369690232, + "pass@8": 0.9679729746481276, + "pass@16": 0.976421032163366, + "pass@32": 0.9826638897843468, + "pass@64": 0.9870178875177636, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8431622257053292, + "pass@2": 0.8853485301014486, + "pass@4": 0.9075407620481607, + "pass@8": 0.9225588387029565, + "pass@16": 0.9338780820659167, + "pass@32": 0.9427151597704617, + "pass@64": 0.9499720050474115, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6195509453781513, + "pass@2": 0.7090015632237144, + "pass@4": 0.7696727071405988, + "pass@8": 0.8128208903813218, + "pass@16": 0.8456239738232122, + "pass@32": 0.8723095721456559, + "pass@64": 0.8960096112676782, + "pass@128": 0.9187675070028011 + }, + "14": { + "pass@1": 0.5342881944444444, + "pass@2": 0.6228624176912091, + "pass@4": 0.68985191626266, + "pass@8": 0.7400756479989421, + "pass@16": 0.777373163396424, + "pass@32": 0.8060593902488095, + "pass@64": 0.8296373557844222, + "pass@128": 0.847953216374269 + }, + "15": { + "pass@1": 0.40404829545454546, + "pass@2": 0.4952815557146265, + "pass@4": 0.5744040432445942, + "pass@8": 0.6418894193523699, + "pass@16": 0.6989534805926838, + "pass@32": 0.7435311343356013, + "pass@64": 0.7784359269113569, + "pass@128": 0.8090909090909091 + }, + "16": { + "pass@1": 0.2513577279202279, + "pass@2": 0.321735073580546, + "pass@4": 0.3875065542020923, + "pass@8": 0.4469587390581188, + "pass@16": 0.5009118793512886, + "pass@32": 0.5501861383741561, + "pass@64": 0.5973992977953705, + "pass@128": 0.6438746438746439 + }, + "17": { + "pass@1": 0.21858016304347827, + "pass@2": 0.28791013045923586, + "pass@4": 0.3574839279600918, + "pass@8": 0.42173292646814153, + "pass@16": 0.4792274741006837, + "pass@32": 0.5322220984044989, + "pass@64": 0.5842848428673304, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.20774356617647058, + "pass@2": 0.2646114375868455, + "pass@4": 0.3206001789298397, + "pass@8": 0.3763666437779109, + "pass@16": 0.4341574245513161, + "pass@32": 0.49379397470457037, + "pass@64": 0.5518441577798592, + "pass@128": 0.6058823529411764 + }, + "19": { + "pass@1": 0.1776181175595238, + "pass@2": 0.23588433281777288, + "pass@4": 0.291822026431964, + "pass@8": 0.34046070760558633, + "pass@16": 0.3797265269874325, + "pass@32": 0.4125415160967138, + "pass@64": 0.44444983941437916, + "pass@128": 0.4791666666666667 + }, + "20": { + "pass@1": 0.1407185628742515, + "pass@2": 0.19786669279551142, + "pass@4": 0.25815344816553615, + "pass@8": 0.31686300365274095, + "pass@16": 0.3719599344412786, + "pass@32": 0.41902785399935966, + "pass@64": 0.4536853513921381, + "pass@128": 0.47604790419161674 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 185369, + "answer_accuracy": 0.44260247937041564, + "avg_response_len": 353.18173613233495, + "resp_tokens_sum": 147918162, + "pass_at_k": { + "pass@1": 0.44260247937041564, + "pass@2": 0.509139224727106, + "pass@4": 0.562456437023918, + "pass@8": 0.605909816047592, + "pass@16": 0.6419913306682059, + "pass@32": 0.6728859081903985, + "pass@64": 0.700172046459371, + "pass@128": 0.726161369193154 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9010664682539683, + "pass@2": 0.9314558336457947, + "pass@4": 0.9495970378702663, + "pass@8": 0.9613202533352903, + "pass@16": 0.9687759979531508, + "pass@32": 0.9740935366946681, + "pass@64": 0.9783593648058921, + "pass@128": 0.9809523809523809 + }, + "12": { + "pass@1": 0.8469753440366973, + "pass@2": 0.9016613390642684, + "pass@4": 0.9306340686771954, + "pass@8": 0.9488184652821481, + "pass@16": 0.9626058466450654, + "pass@32": 0.97255331767062, + "pass@64": 0.979117645036257, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.6840277777777778, + "pass@2": 0.7669574614805097, + "pass@4": 0.8165764693186498, + "pass@8": 0.8479331025188009, + "pass@16": 0.8715139042865675, + "pass@32": 0.8908118707209612, + "pass@64": 0.9047815820695196, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.5532965504451038, + "pass@2": 0.642291218836889, + "pass@4": 0.7043512042448701, + "pass@8": 0.7499437255097021, + "pass@16": 0.7861016572365503, + "pass@32": 0.8160076783891812, + "pass@64": 0.8389702413765118, + "pass@128": 0.857566765578635 + }, + "15": { + "pass@1": 0.41584683641975306, + "pass@2": 0.5078842683727031, + "pass@4": 0.5873538796655046, + "pass@8": 0.6499067906851369, + "pass@16": 0.6943756870491424, + "pass@32": 0.7266055349040754, + "pass@64": 0.7528233902976535, + "pass@128": 0.7777777777777778 + }, + "16": { + "pass@1": 0.2721008480825959, + "pass@2": 0.33834423559333837, + "pass@4": 0.40159421665212186, + "pass@8": 0.4602251988321013, + "pass@16": 0.5098591234832823, + "pass@32": 0.5509605333386498, + "pass@64": 0.5906200029115647, + "pass@128": 0.6371681415929203 + }, + "17": { + "pass@1": 0.22926863704819278, + "pass@2": 0.3040100856180628, + "pass@4": 0.3709875186911878, + "pass@8": 0.42611476990632063, + "pass@16": 0.47253513212532294, + "pass@32": 0.5125942185897905, + "pass@64": 0.5472100849333396, + "pass@128": 0.5813253012048193 + }, + "18": { + "pass@1": 0.22464717741935483, + "pass@2": 0.28498698247396503, + "pass@4": 0.33843989592026796, + "pass@8": 0.38398988335421697, + "pass@16": 0.4232406963502674, + "pass@32": 0.46041983832532407, + "pass@64": 0.5008218489889754, + "pass@128": 0.5451612903225806 + }, + "19": { + "pass@1": 0.1623158987915408, + "pass@2": 0.2182006327758879, + "pass@4": 0.2712618750261958, + "pass@8": 0.32116348907565456, + "pass@16": 0.36780298226078906, + "pass@32": 0.4113543919886992, + "pass@64": 0.45119827969048176, + "pass@128": 0.486404833836858 + }, + "20": { + "pass@1": 0.15871340090090091, + "pass@2": 0.21606732224350333, + "pass@4": 0.27226703193632334, + "pass@8": 0.32611956314764734, + "pass@16": 0.3775474082082862, + "pass@32": 0.42623156342189406, + "pass@64": 0.46941726977322745, + "pass@128": 0.5075075075075075 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6967154605263158, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6967154605263158, + "pass@2": 0.7331326732801338, + "pass@4": 0.7627227462257813, + "pass@8": 0.7872539413609735, + "pass@16": 0.8080859331846313, + "pass@32": 0.826194850954748, + "pass@64": 0.8423325863253173, + "pass@128": 0.8571578947368421 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9435703125, + "pass@2": 0.9607268700787404, + "pass@4": 0.9712526024559434, + "pass@8": 0.9776772575007111, + "pass@16": 0.9820311480328475, + "pass@32": 0.9859655999840329, + "pass@64": 0.9900584070418624, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9996328125, + "pass@2": 0.9999351624015749, + "pass@4": 0.9999966226096737, + "pass@8": 0.9999999926430151, + "pass@16": 0.9999999999999936, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998328125, + "pass@2": 0.9986950049212598, + "pass@4": 0.9989937330333709, + "pass@8": 0.9992152625181484, + "pass@16": 0.9994176619420255, + "pass@32": 0.9996886051712321, + "pass@64": 0.9999404409448819, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9875625, + "pass@2": 0.9903880413385826, + "pass@4": 0.9921799958755155, + "pass@8": 0.9936264938768348, + "pass@16": 0.9948819632043239, + "pass@32": 0.996308463535151, + "pass@64": 0.997997987322932, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.995953125, + "pass@2": 0.9969644438976379, + "pass@4": 0.9975323900449944, + "pass@8": 0.997917797864405, + "pass@16": 0.9982963251514316, + "pass@32": 0.998686418855384, + "pass@64": 0.9992519681067729, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.992796875, + "pass@2": 0.9954954478346455, + "pass@4": 0.9970782049118859, + "pass@8": 0.9980214999423029, + "pass@16": 0.998521787995179, + "pass@32": 0.9988284479182614, + "pass@64": 0.9989861762904964, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.987875, + "pass@2": 0.9913147145669291, + "pass@4": 0.9929255646794151, + "pass@8": 0.9939211132530895, + "pass@16": 0.9947769123750306, + "pass@32": 0.9956578151347028, + "pass@64": 0.9964109543118082, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.985609375, + "pass@2": 0.9908181594488191, + "pass@4": 0.9928797574053245, + "pass@8": 0.9939798604265343, + "pass@16": 0.9949207781564106, + "pass@32": 0.9958827248544599, + "pass@64": 0.9969750905876388, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.9058125, + "pass@2": 0.9348855807086626, + "pass@4": 0.9523219686914133, + "pass@8": 0.9639877513162247, + "pass@16": 0.9719139636312137, + "pass@32": 0.9772367936401185, + "pass@64": 0.9810516493173799, + "pass@128": 0.984 + }, + "12": { + "pass@1": 0.83046875, + "pass@2": 0.8840001230314961, + "pass@4": 0.9133514972815886, + "pass@8": 0.9323962777434657, + "pass@16": 0.9465618674560089, + "pass@32": 0.9567864902434278, + "pass@64": 0.9644186160238337, + "pass@128": 0.971 + }, + "13": { + "pass@1": 0.6527734375, + "pass@2": 0.7388490403543305, + "pass@4": 0.7958859667229103, + "pass@8": 0.8354775748363091, + "pass@16": 0.8643046264976756, + "pass@32": 0.8860494112925553, + "pass@64": 0.9030548966211872, + "pass@128": 0.917 + }, + "14": { + "pass@1": 0.5455625, + "pass@2": 0.6375738188976379, + "pass@4": 0.7070053682039741, + "pass@8": 0.7590985628570953, + "pass@16": 0.7975958195184475, + "pass@32": 0.8266908787586564, + "pass@64": 0.8494597774974879, + "pass@128": 0.868 + }, + "15": { + "pass@1": 0.406609375, + "pass@2": 0.4940346948818898, + "pass@4": 0.5684210074990619, + "pass@8": 0.6314350453515915, + "pass@16": 0.6841069770667974, + "pass@32": 0.7270360115703599, + "pass@64": 0.7626391528642439, + "pass@128": 0.794 + }, + "16": { + "pass@1": 0.2661484375, + "pass@2": 0.33698781988188975, + "pass@4": 0.4029694417885262, + "pass@8": 0.46251322880057405, + "pass@16": 0.5149965944758812, + "pass@32": 0.5617009345680936, + "pass@64": 0.6059889851985129, + "pass@128": 0.652 + }, + "17": { + "pass@1": 0.2140859375, + "pass@2": 0.28265403543307094, + "pass@4": 0.3472788755155604, + "pass@8": 0.40412559729262215, + "pass@16": 0.4549240276430866, + "pass@32": 0.501694346448931, + "pass@64": 0.545903991761549, + "pass@128": 0.589 + }, + "18": { + "pass@1": 0.2059765625, + "pass@2": 0.2638793061023622, + "pass@4": 0.31758961445444284, + "pass@8": 0.3670262508999904, + "pass@16": 0.41459444172398247, + "pass@32": 0.4628889802277007, + "pass@64": 0.5113301409316344, + "pass@128": 0.557 + }, + "19": { + "pass@1": 0.1695625, + "pass@2": 0.22803469488188977, + "pass@4": 0.2843005737720284, + "pass@8": 0.33478337444162465, + "pass@16": 0.37877477451174985, + "pass@32": 0.4178315933259912, + "pass@64": 0.45472330372441766, + "pass@128": 0.492 + }, + "20": { + "pass@1": 0.1492734375, + "pass@2": 0.20428383366141709, + "pass@4": 0.2597689933445819, + "pass@8": 0.3126219442937622, + "pass@16": 0.3630130611257066, + "pass@32": 0.40876865261089623, + "pass@64": 0.44612760163432663, + "pass@128": 0.476 + } + }, + "per_op_accuracy": { + "10": 0.9435703125, + "2": 0.9999921875, + "3": 0.9996328125, + "4": 0.998328125, + "5": 0.9875625, + "6": 0.995953125, + "7": 0.992796875, + "8": 0.987875, + "9": 0.985609375, + "11": 0.9058125, + "12": 0.83046875, + "13": 0.6527734375, + "14": 0.5455625, + "15": 0.406609375, + "16": 0.2661484375, + "17": 0.2140859375, + "18": 0.2059765625, + "19": 0.1695625, + "20": 0.1492734375 + }, + "per_op_avg_response_len": { + "10": 281.299859375, + "2": 115.78703125, + "3": 145.00421875, + "4": 160.341109375, + "5": 174.0563359375, + "6": 199.62221875, + "7": 224.5571171875, + "8": 244.237671875, + "9": 260.254953125, + "11": 298.0427109375, + "12": 308.0526796875, + "13": 320.6473828125, + "14": 334.90971875, + "15": 342.809359375, + "16": 331.71221875, + "17": 346.1571328125, + "18": 349.244828125, + "19": 339.274734375, + "20": 350.9766171875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.299859375, + "2": 115.78703125, + "3": 145.00421875, + "4": 160.341109375, + "5": 174.0563359375, + "6": 199.62221875, + "7": 224.5571171875, + "8": 244.237671875, + "9": 260.254953125, + "11": 298.0427109375, + "12": 308.0526796875, + "13": 320.6473828125, + "14": 334.90971875, + "15": 342.809359375, + "16": 331.71221875, + "17": 346.1571328125, + "18": 349.244828125, + "19": 339.274734375, + "20": 350.9766171875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 560078, + "answer_accuracy": 0.691686591052798, + "avg_response_len": 257.03733228936136, + "resp_tokens_sum": 208130325, + "pass_at_k": { + "pass@1": 0.691686591052798, + "pass@2": 0.7285240024607798, + "pass@4": 0.7584811279468562, + "pass@8": 0.7832984708290734, + "pass@16": 0.8045310794934998, + "pass@32": 0.823173791139723, + "pass@64": 0.8397895761692062, + "pass@128": 0.8548846032247865 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9426622418879056, + "pass@2": 0.9608814281234755, + "pass@4": 0.9727997515244226, + "pass@8": 0.9800427227199637, + "pass@16": 0.9848555713566579, + "pass@32": 0.9893375479172565, + "pass@64": 0.9948648057621975, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999522171253823, + "pass@2": 0.9999996237568928, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995068359375, + "pass@2": 0.995943036417323, + "pass@4": 0.996855512396888, + "pass@8": 0.9975476953695743, + "pass@16": 0.9981801935688303, + "pass@32": 0.9990268911601001, + "pass@64": 0.9998138779527558, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9858382936507937, + "pass@2": 0.9894134326959131, + "pass@4": 0.9919654816957404, + "pass@8": 0.993967779218521, + "pass@16": 0.9953518900581235, + "pass@32": 0.9962822752680421, + "pass@64": 0.9967815120663723, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9956025449101796, + "pass@2": 0.9967230892545617, + "pass@4": 0.9970756080639619, + "pass@8": 0.9971929848408553, + "pass@16": 0.9973802395197485, + "pass@32": 0.9977544910179641, + "pass@64": 0.9985029940119761, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9944526627218935, + "pass@2": 0.9979470484088896, + "pass@4": 0.9992837274778522, + "pass@8": 0.9998536405673738, + "pass@16": 0.9999944055596715, + "pass@32": 0.9999999969008433, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9830729166666666, + "pass@2": 0.9861033559206539, + "pass@4": 0.987899911849961, + "pass@8": 0.9900541498080317, + "pass@16": 0.9924767953718153, + "pass@32": 0.9948968439236284, + "pass@64": 0.9965094689481032, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9885343309859155, + "pass@2": 0.9931147415992014, + "pass@4": 0.9947651726632762, + "pass@8": 0.9954927131433333, + "pass@16": 0.9964173683303595, + "pass@32": 0.9977688493163707, + "pass@64": 0.9992201397360542, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9032179532577904, + "pass@2": 0.9327289570832682, + "pass@4": 0.9506815622905493, + "pass@8": 0.9626199544828037, + "pass@16": 0.9704751886621362, + "pass@32": 0.9749374452489946, + "pass@64": 0.9778428066504992, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8037826624293786, + "pass@2": 0.8664708589127629, + "pass@4": 0.9026233156236823, + "pass@8": 0.9260914408190967, + "pass@16": 0.9431713204069097, + "pass@32": 0.9549023146846267, + "pass@64": 0.9638589731267034, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.6582092476489029, + "pass@2": 0.743703024337867, + "pass@4": 0.8042070038071254, + "pass@8": 0.8481824192918073, + "pass@16": 0.877888284808596, + "pass@32": 0.8965888649622713, + "pass@64": 0.9091856827840178, + "pass@128": 0.9184952978056427 + }, + "14": { + "pass@1": 0.5494548286604362, + "pass@2": 0.6482951130816593, + "pass@4": 0.7280674673259301, + "pass@8": 0.7889771206999603, + "pass@16": 0.8312085331718211, + "pass@32": 0.8598877871539242, + "pass@64": 0.8815913721972909, + "pass@128": 0.9003115264797508 + }, + "15": { + "pass@1": 0.4004019147398844, + "pass@2": 0.4798765275818123, + "pass@4": 0.5449855960020893, + "pass@8": 0.6041668693159691, + "pass@16": 0.6603312308303183, + "pass@32": 0.711706774366159, + "pass@64": 0.7567645044134567, + "pass@128": 0.7947976878612717 + }, + "16": { + "pass@1": 0.2763860887096774, + "pass@2": 0.35277455867411744, + "pass@4": 0.42198129618878266, + "pass@8": 0.48262699673255544, + "pass@16": 0.5365621998798281, + "pass@32": 0.5864838038611693, + "pass@64": 0.6325214538236035, + "pass@128": 0.6774193548387096 + }, + "17": { + "pass@1": 0.19533507947976878, + "pass@2": 0.25727059248554907, + "pass@4": 0.31503235406629077, + "pass@8": 0.36664014844214665, + "pass@16": 0.415407852939479, + "pass@32": 0.46282528956899566, + "pass@64": 0.5089321507525993, + "pass@128": 0.5549132947976878 + }, + "18": { + "pass@1": 0.18772321428571428, + "pass@2": 0.24447272215972995, + "pass@4": 0.29619767395147045, + "pass@8": 0.34292779478769564, + "pass@16": 0.38793229002272034, + "pass@32": 0.43505393984941876, + "pass@64": 0.48128101171399873, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.1686373873873874, + "pass@2": 0.22988933815705478, + "pass@4": 0.28967174852017374, + "pass@8": 0.3425929783847028, + "pass@16": 0.3887203432902438, + "pass@32": 0.42960762813584363, + "pass@64": 0.46859317448539534, + "pass@128": 0.5105105105105106 + }, + "20": { + "pass@1": 0.14841403903903905, + "pass@2": 0.19893675663852825, + "pass@4": 0.24889135141891053, + "pass@8": 0.29487053016690784, + "pass@16": 0.3395049732401222, + "pass@32": 0.38101573199885735, + "pass@64": 0.4152574877923962, + "pass@128": 0.4444444444444444 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 570890, + "answer_accuracy": 0.6952576968043648, + "avg_response_len": 266.9440958690569, + "resp_tokens_sum": 219193136, + "pass_at_k": { + "pass@1": 0.6952576968043648, + "pass@2": 0.7309432171767659, + "pass@4": 0.7608909380725237, + "pass@8": 0.786381191349136, + "pass@16": 0.8083556528635224, + "pass@32": 0.8275322430716204, + "pass@64": 0.8447360879524896, + "pass@128": 0.8603273577552611 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.929879217791411, + "pass@2": 0.9455011986135936, + "pass@4": 0.9572367023907287, + "pass@8": 0.9653763793871797, + "pass@16": 0.9709467034438971, + "pass@32": 0.9763425037418643, + "pass@64": 0.9817604399544487, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997395833333333, + "pass@2": 0.999989188141255, + "pass@4": 0.9999999801161218, + "pass@8": 0.9999999999999979, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997457470414202, + "pass@2": 0.9999799800820015, + "pass@4": 0.9999999084803749, + "pass@8": 0.9999999999996586, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941123188405797, + "pass@2": 0.9961107782722813, + "pass@4": 0.9970168766947609, + "pass@8": 0.997577219367406, + "pass@16": 0.9980644436436175, + "pass@32": 0.998786836422768, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970898997134671, + "pass@2": 0.9972238595086073, + "pass@4": 0.9973116384270019, + "pass@8": 0.9974829659544705, + "pass@16": 0.9978086997721273, + "pass@32": 0.9983924824583174, + "pass@64": 0.9992893080342035, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956195809248555, + "pass@2": 0.996662859564881, + "pass@4": 0.9970522314587844, + "pass@8": 0.9971089149255284, + "pass@16": 0.997109826485928, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9900308098591549, + "pass@2": 0.9928277836309195, + "pass@4": 0.9939063929508812, + "pass@8": 0.9944474182396722, + "pass@16": 0.9947159353923489, + "pass@32": 0.9950704220578349, + "pass@64": 0.995774647887324, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9818488023952096, + "pass@2": 0.9868720973407518, + "pass@4": 0.9893204078344998, + "pass@8": 0.9909175880114821, + "pass@16": 0.9920082924256072, + "pass@32": 0.993079471807874, + "pass@64": 0.994766372753843, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.913074171686747, + "pass@2": 0.9404327447585614, + "pass@4": 0.9566515369690232, + "pass@8": 0.9679729746481276, + "pass@16": 0.976421032163366, + "pass@32": 0.9826638897843468, + "pass@64": 0.9870178875177636, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8431622257053292, + "pass@2": 0.8853485301014486, + "pass@4": 0.9075407620481607, + "pass@8": 0.9225588387029565, + "pass@16": 0.9338780820659167, + "pass@32": 0.9427151597704617, + "pass@64": 0.9499720050474115, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6195509453781513, + "pass@2": 0.7090015632237144, + "pass@4": 0.7696727071405988, + "pass@8": 0.8128208903813218, + "pass@16": 0.8456239738232122, + "pass@32": 0.8723095721456559, + "pass@64": 0.8960096112676782, + "pass@128": 0.9187675070028011 + }, + "14": { + "pass@1": 0.5342881944444444, + "pass@2": 0.6228624176912091, + "pass@4": 0.68985191626266, + "pass@8": 0.7400756479989421, + "pass@16": 0.777373163396424, + "pass@32": 0.8060593902488095, + "pass@64": 0.8296373557844222, + "pass@128": 0.847953216374269 + }, + "15": { + "pass@1": 0.40404829545454546, + "pass@2": 0.4952815557146265, + "pass@4": 0.5744040432445942, + "pass@8": 0.6418894193523699, + "pass@16": 0.6989534805926838, + "pass@32": 0.7435311343356013, + "pass@64": 0.7784359269113569, + "pass@128": 0.8090909090909091 + }, + "16": { + "pass@1": 0.2513577279202279, + "pass@2": 0.321735073580546, + "pass@4": 0.3875065542020923, + "pass@8": 0.4469587390581188, + "pass@16": 0.5009118793512886, + "pass@32": 0.5501861383741561, + "pass@64": 0.5973992977953705, + "pass@128": 0.6438746438746439 + }, + "17": { + "pass@1": 0.21858016304347827, + "pass@2": 0.28791013045923586, + "pass@4": 0.3574839279600918, + "pass@8": 0.42173292646814153, + "pass@16": 0.4792274741006837, + "pass@32": 0.5322220984044989, + "pass@64": 0.5842848428673304, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.20774356617647058, + "pass@2": 0.2646114375868455, + "pass@4": 0.3206001789298397, + "pass@8": 0.3763666437779109, + "pass@16": 0.4341574245513161, + "pass@32": 0.49379397470457037, + "pass@64": 0.5518441577798592, + "pass@128": 0.6058823529411764 + }, + "19": { + "pass@1": 0.1776181175595238, + "pass@2": 0.23588433281777288, + "pass@4": 0.291822026431964, + "pass@8": 0.34046070760558633, + "pass@16": 0.3797265269874325, + "pass@32": 0.4125415160967138, + "pass@64": 0.44444983941437916, + "pass@128": 0.4791666666666667 + }, + "20": { + "pass@1": 0.1407185628742515, + "pass@2": 0.19786669279551142, + "pass@4": 0.25815344816553615, + "pass@8": 0.31686300365274095, + "pass@16": 0.3719599344412786, + "pass@32": 0.41902785399935966, + "pass@64": 0.4536853513921381, + "pass@128": 0.47604790419161674 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 563444, + "answer_accuracy": 0.7032922591468286, + "avg_response_len": 285.7522542538744, + "resp_tokens_sum": 228930990, + "pass_at_k": { + "pass@1": 0.7032922591468286, + "pass@2": 0.7400347043564306, + "pass@4": 0.768887233609696, + "pass@8": 0.7921462560934432, + "pass@16": 0.8114023979090202, + "pass@32": 0.8278775245383643, + "pass@64": 0.842439411586423, + "pass@128": 0.8562070618309634 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9578125, + "pass@2": 0.9753870901398526, + "pass@4": 0.9833263334620483, + "pass@8": 0.98725395468186, + "pass@16": 0.9899596537917327, + "pass@32": 0.9919179552842842, + "pass@64": 0.9932696551144083, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999225583090379, + "pass@2": 0.9998217298730516, + "pass@4": 0.9999901725112349, + "pass@8": 0.9999999785510661, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9825137867647059, + "pass@2": 0.9854840927512737, + "pass@4": 0.9874707255343083, + "pass@8": 0.9893014786507219, + "pass@16": 0.991217308173432, + "pass@32": 0.9938179066466555, + "pass@64": 0.997452524312115, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.995070977917981, + "pass@2": 0.9969331391738493, + "pass@4": 0.998256703913588, + "pass@8": 0.9991602076007862, + "pass@16": 0.9997983877960986, + "pass@32": 0.9999919321054616, + "pass@64": 0.9999999987471165, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9879351265822784, + "pass@2": 0.9915949242499751, + "pass@4": 0.9947475725937738, + "pass@8": 0.9970610280579045, + "pass@16": 0.9984926549110091, + "pass@32": 0.9994571169803051, + "pass@64": 0.9999562540838489, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9900760135135135, + "pass@2": 0.9945843973895155, + "pass@4": 0.996588656710704, + "pass@8": 0.9969831382519523, + "pass@16": 0.9969969824466679, + "pass@32": 0.9969969969969951, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9863092845659164, + "pass@2": 0.9924345519912906, + "pass@4": 0.9945501765052679, + "pass@8": 0.9955417133916913, + "pass@16": 0.9963403303182642, + "pass@32": 0.9967403207823743, + "pass@64": 0.9967844774326561, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9010664682539683, + "pass@2": 0.9314558336457947, + "pass@4": 0.9495970378702663, + "pass@8": 0.9613202533352903, + "pass@16": 0.9687759979531508, + "pass@32": 0.9740935366946681, + "pass@64": 0.9783593648058921, + "pass@128": 0.9809523809523809 + }, + "12": { + "pass@1": 0.8469753440366973, + "pass@2": 0.9016613390642684, + "pass@4": 0.9306340686771954, + "pass@8": 0.9488184652821481, + "pass@16": 0.9626058466450654, + "pass@32": 0.97255331767062, + "pass@64": 0.979117645036257, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.6840277777777778, + "pass@2": 0.7669574614805097, + "pass@4": 0.8165764693186498, + "pass@8": 0.8479331025188009, + "pass@16": 0.8715139042865675, + "pass@32": 0.8908118707209612, + "pass@64": 0.9047815820695196, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.5532965504451038, + "pass@2": 0.642291218836889, + "pass@4": 0.7043512042448701, + "pass@8": 0.7499437255097021, + "pass@16": 0.7861016572365503, + "pass@32": 0.8160076783891812, + "pass@64": 0.8389702413765118, + "pass@128": 0.857566765578635 + }, + "15": { + "pass@1": 0.41584683641975306, + "pass@2": 0.5078842683727031, + "pass@4": 0.5873538796655046, + "pass@8": 0.6499067906851369, + "pass@16": 0.6943756870491424, + "pass@32": 0.7266055349040754, + "pass@64": 0.7528233902976535, + "pass@128": 0.7777777777777778 + }, + "16": { + "pass@1": 0.2721008480825959, + "pass@2": 0.33834423559333837, + "pass@4": 0.40159421665212186, + "pass@8": 0.4602251988321013, + "pass@16": 0.5098591234832823, + "pass@32": 0.5509605333386498, + "pass@64": 0.5906200029115647, + "pass@128": 0.6371681415929203 + }, + "17": { + "pass@1": 0.22926863704819278, + "pass@2": 0.3040100856180628, + "pass@4": 0.3709875186911878, + "pass@8": 0.42611476990632063, + "pass@16": 0.47253513212532294, + "pass@32": 0.5125942185897905, + "pass@64": 0.5472100849333396, + "pass@128": 0.5813253012048193 + }, + "18": { + "pass@1": 0.22464717741935483, + "pass@2": 0.28498698247396503, + "pass@4": 0.33843989592026796, + "pass@8": 0.38398988335421697, + "pass@16": 0.4232406963502674, + "pass@32": 0.46041983832532407, + "pass@64": 0.5008218489889754, + "pass@128": 0.5451612903225806 + }, + "19": { + "pass@1": 0.1623158987915408, + "pass@2": 0.2182006327758879, + "pass@4": 0.2712618750261958, + "pass@8": 0.32116348907565456, + "pass@16": 0.36780298226078906, + "pass@32": 0.4113543919886992, + "pass@64": 0.45119827969048176, + "pass@128": 0.486404833836858 + }, + "20": { + "pass@1": 0.15871340090090091, + "pass@2": 0.21606732224350333, + "pass@4": 0.27226703193632334, + "pass@8": 0.32611956314764734, + "pass@16": 0.3775474082082862, + "pass@32": 0.42623156342189406, + "pass@64": 0.46941726977322745, + "pass@128": 0.5075075075075075 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch1-op7-10/graphs/SDAR-100M-block4-epoch1-op7-10_score_report.jsonl b/composition/SDAR-100M-block4-epoch1-op7-10/graphs/SDAR-100M-block4-epoch1-op7-10_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2c8e57735337c81119a9c60d07d3c21f83c90d19 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op7-10/graphs/SDAR-100M-block4-epoch1-op7-10_score_report.jsonl @@ -0,0 +1,2 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch1-op7-10", "results_dir": "results/composition/SDAR-100M-block4-epoch1-op7-10/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1012425, "value_mismatch": 17874, "dependency_mismatch": 136782, "answer_mismatch": 13911, "extra_nodes": 13228, "missing_nodes": 12905, "perfect_with_extra": 764, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 333647, "dependency_mismatch": 44361, "value_mismatch": 6231, "answer_mismatch": 4889, "missing_nodes": 4100, "extra_nodes": 4468, "perfect_with_extra": 279}, "teachers_in_school": {"total": 390656, "perfect": 343675, "dependency_mismatch": 45999, "value_mismatch": 5837, "answer_mismatch": 4761, "missing_nodes": 4356, "extra_nodes": 4584, "perfect_with_extra": 380}, "movie_festival_awards": {"total": 382336, "perfect": 335103, "dependency_mismatch": 46422, "value_mismatch": 5806, "answer_mismatch": 4261, "missing_nodes": 4449, "extra_nodes": 4176, "perfect_with_extra": 105}}, "results_path": "results/composition/SDAR-100M-block4-epoch1-op7-10/graphs/SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 238897, "value_mismatch": 851931, "dependency_mismatch": 1005390, "answer_mismatch": 723673, "extra_nodes": 63000, "missing_nodes": 746013, "perfect_with_extra": 18, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 289665, "dependency_mismatch": 341317, "answer_mismatch": 244760, "extra_nodes": 23987, "perfect": 78380, "missing_nodes": 260262, "perfect_with_extra": 3}, "movie_festival_awards": {"total": 418816, "perfect": 80063, "value_mismatch": 276174, "dependency_mismatch": 327680, "answer_mismatch": 233448, "missing_nodes": 239567, "extra_nodes": 18359, "perfect_with_extra": 6}, "teachers_in_school": {"total": 430464, "perfect": 80454, "dependency_mismatch": 336393, "value_mismatch": 286092, "answer_mismatch": 245465, "missing_nodes": 246184, "extra_nodes": 20654, "perfect_with_extra": 9}}, "results_path": "results/composition/SDAR-100M-block4-epoch1-op7-10/graphs/SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1251322, "value_mismatch": 869805, "dependency_mismatch": 1142172, "answer_mismatch": 737584, "extra_nodes": 76228, "missing_nodes": 758918, "perfect_with_extra": 782, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch1-op7-10/graphs/SDAR-100M-block4-epoch1-op7-10_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 412027, "dependency_mismatch": 385678, "value_mismatch": 295896, "answer_mismatch": 249649, "missing_nodes": 264362, "extra_nodes": 28455, "perfect_with_extra": 282}, "teachers_in_school": {"total": 821120, "perfect": 424129, "dependency_mismatch": 382392, "value_mismatch": 291929, "answer_mismatch": 250226, "missing_nodes": 250540, "extra_nodes": 25238, "perfect_with_extra": 389}, "movie_festival_awards": {"total": 801152, "perfect": 415166, "dependency_mismatch": 374102, "value_mismatch": 281980, "answer_mismatch": 237709, "missing_nodes": 244016, "extra_nodes": 22535, "perfect_with_extra": 111}}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch1-op7-10", "results_dir": "results/composition/SDAR-100M-block4-epoch1-op7-10/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch1-op7-10/graphs/SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch1-op7-10/graphs/SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch1-op7-10/graphs/SDAR-100M-block4-epoch1-op7-10_score_report.jsonl", "template_stats": {}}} diff --git a/composition/SDAR-100M-block4-epoch1-op7-10/graphs/results.json b/composition/SDAR-100M-block4-epoch1-op7-10/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..46b377e4b49acb1ba4b5093deedff1c55e614cc8 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op7-10/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969418960244648, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996875, + "pass@2": 0.996875, + "pass@4": 0.996875, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9619047619047619, + "pass@2": 0.9777777777777777, + "pass@4": 0.9873015873015873, + "pass@8": 0.9936507936507937, + "pass@16": 0.9936507936507937, + "pass@32": 0.9936507936507937, + "pass@64": 0.9936507936507937, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9610778443113772, + "pass@2": 0.9730538922155688, + "pass@4": 0.9940119760479041, + "pass@8": 0.9970059880239521, + "pass@16": 0.9970059880239521, + "pass@32": 0.9970059880239521, + "pass@64": 0.9970059880239521, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8757396449704142, + "pass@2": 0.9142011834319527, + "pass@4": 0.9585798816568047, + "pass@8": 0.9881656804733728, + "pass@16": 0.9940828402366864, + "pass@32": 0.9940828402366864, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.7852564102564102, + "pass@2": 0.8557692307692307, + "pass@4": 0.9294871794871795, + "pass@8": 0.9647435897435898, + "pass@16": 0.9871794871794872, + "pass@32": 0.9967948717948718, + "pass@64": 0.9967948717948718, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7549295774647887, + "pass@2": 0.8225352112676056, + "pass@4": 0.8985915492957747, + "pass@8": 0.9436619718309859, + "pass@16": 0.9830985915492958, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6312684365781711, + "pass@2": 0.7050147492625368, + "pass@4": 0.7846607669616519, + "pass@8": 0.8584070796460177, + "pass@16": 0.9056047197640118, + "pass@32": 0.9262536873156342, + "pass@64": 0.9380530973451328, + "pass@128": 0.9498525073746312 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 0.996875, + "5": 0.9619047619047619, + "6": 0.9610778443113772, + "7": 0.8757396449704142, + "8": 0.7852564102564102, + "9": 0.7549295774647887, + "10": 0.6312684365781711 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9777777777777777, + "6": 0.9730538922155688, + "7": 0.9142011834319527, + "8": 0.8557692307692307, + "9": 0.8225352112676056, + "10": 0.7050147492625368 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9873015873015873, + "6": 0.9940119760479041, + "7": 0.9585798816568047, + "8": 0.9294871794871795, + "9": 0.8985915492957747, + "10": 0.7846607669616519 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 0.9970059880239521, + "7": 0.9881656804733728, + "8": 0.9647435897435898, + "9": 0.9436619718309859, + "10": 0.8584070796460177 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 0.9970059880239521, + "7": 0.9940828402366864, + "8": 0.9871794871794872, + "9": 0.9830985915492958, + "10": 0.9056047197640118 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 0.9970059880239521, + "7": 0.9940828402366864, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9262536873156342 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 0.9970059880239521, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9380530973451328 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9498525073746312 + } + }, + "total_pass_at_k": { + "pass@1": 0.8824721377912867, + "pass@2": 0.9142181695373185, + "pass@4": 0.9486659912191827, + "pass@8": 0.9709557581898007, + "pass@16": 0.9841269841269841, + "pass@32": 0.9895305639986491, + "pass@64": 0.9912191827085444, + "pass@128": 0.9932455251604188 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9941520467836257, + "pass@2": 0.9941520467836257, + "pass@4": 0.9941520467836257, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9441176470588235, + "pass@2": 0.9588235294117647, + "pass@4": 0.9794117647058823, + "pass@8": 0.9823529411764705, + "pass@16": 0.9852941176470589, + "pass@32": 0.9882352941176471, + "pass@64": 0.9970588235294118, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9621451104100947, + "pass@2": 0.9716088328075709, + "pass@4": 0.9873817034700315, + "pass@8": 0.9936908517350158, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8291139240506329, + "pass@2": 0.8924050632911392, + "pass@4": 0.9430379746835443, + "pass@8": 0.9746835443037974, + "pass@16": 0.9873417721518988, + "pass@32": 0.9936708860759493, + "pass@64": 0.9968354430379747, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.7957957957957958, + "pass@2": 0.8588588588588588, + "pass@4": 0.9159159159159159, + "pass@8": 0.96996996996997, + "pass@16": 0.993993993993994, + "pass@32": 0.996996996996997, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.7845659163987139, + "pass@2": 0.8327974276527331, + "pass@4": 0.8906752411575563, + "pass@8": 0.954983922829582, + "pass@16": 0.9839228295819936, + "pass@32": 0.9967845659163987, + "pass@64": 0.9967845659163987, + "pass@128": 0.9967845659163987 + }, + "10": { + "pass@1": 0.6, + "pass@2": 0.6776119402985075, + "pass@4": 0.7701492537313432, + "pass@8": 0.835820895522388, + "pass@16": 0.9283582089552239, + "pass@32": 0.9492537313432836, + "pass@64": 0.9522388059701492, + "pass@128": 0.9611940298507463 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9441176470588235, + "6": 0.9621451104100947, + "7": 0.8291139240506329, + "8": 0.7957957957957958, + "9": 0.7845659163987139, + "10": 0.6 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9588235294117647, + "6": 0.9716088328075709, + "7": 0.8924050632911392, + "8": 0.8588588588588588, + "9": 0.8327974276527331, + "10": 0.6776119402985075 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9794117647058823, + "6": 0.9873817034700315, + "7": 0.9430379746835443, + "8": 0.9159159159159159, + "9": 0.8906752411575563, + "10": 0.7701492537313432 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9823529411764705, + "6": 0.9936908517350158, + "7": 0.9746835443037974, + "8": 0.96996996996997, + "9": 0.954983922829582, + "10": 0.835820895522388 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9852941176470589, + "6": 1.0, + "7": 0.9873417721518988, + "8": 0.993993993993994, + "9": 0.9839228295819936, + "10": 0.9283582089552239 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9882352941176471, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.996996996996997, + "9": 0.9967845659163987, + "10": 0.9492537313432836 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.996996996996997, + "9": 0.9967845659163987, + "10": 0.9522388059701492 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.996996996996997, + "9": 0.9967845659163987, + "10": 0.9611940298507463 + } + }, + "total_pass_at_k": { + "pass@1": 0.8808168731168396, + "pass@2": 0.9109474389019083, + "pass@4": 0.9430867090726481, + "pass@8": 0.9681955138935386, + "pass@16": 0.9866086374288584, + "pass@32": 0.9916303983930365, + "pass@64": 0.9933043187144291, + "pass@128": 0.9943086709072648 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9792899408284024, + "pass@2": 0.9881656804733728, + "pass@4": 0.9970414201183432, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9565217391304348, + "pass@2": 0.9710144927536232, + "pass@4": 0.9942028985507246, + "pass@8": 0.9942028985507246, + "pass@16": 0.9971014492753624, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9713467048710601, + "pass@2": 0.9799426934097422, + "pass@4": 0.9885386819484241, + "pass@8": 0.997134670487106, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8497109826589595, + "pass@2": 0.8872832369942196, + "pass@4": 0.9393063583815029, + "pass@8": 0.9797687861271677, + "pass@16": 0.9942196531791907, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.7690140845070422, + "pass@2": 0.8338028169014085, + "pass@4": 0.9126760563380282, + "pass@8": 0.9746478873239437, + "pass@16": 0.9887323943661972, + "pass@32": 0.9943661971830986, + "pass@64": 0.9943661971830986, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.7005988023952096, + "pass@2": 0.781437125748503, + "pass@4": 0.8682634730538922, + "pass@8": 0.9461077844311377, + "pass@16": 0.9820359281437125, + "pass@32": 0.9880239520958084, + "pass@64": 0.9970059880239521, + "pass@128": 0.9970059880239521 + }, + "10": { + "pass@1": 0.6349693251533742, + "pass@2": 0.6901840490797546, + "pass@4": 0.7638036809815951, + "pass@8": 0.8282208588957055, + "pass@16": 0.8680981595092024, + "pass@32": 0.8957055214723927, + "pass@64": 0.9141104294478528, + "pass@128": 0.9294478527607362 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9792899408284024, + "5": 0.9565217391304348, + "6": 0.9713467048710601, + "7": 0.8497109826589595, + "8": 0.7690140845070422, + "9": 0.7005988023952096, + "10": 0.6349693251533742 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9881656804733728, + "5": 0.9710144927536232, + "6": 0.9799426934097422, + "7": 0.8872832369942196, + "8": 0.8338028169014085, + "9": 0.781437125748503, + "10": 0.6901840490797546 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.9942028985507246, + "6": 0.9885386819484241, + "7": 0.9393063583815029, + "8": 0.9126760563380282, + "9": 0.8682634730538922, + "10": 0.7638036809815951 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9942028985507246, + "6": 0.997134670487106, + "7": 0.9797687861271677, + "8": 0.9746478873239437, + "9": 0.9461077844311377, + "10": 0.8282208588957055 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.997134670487106, + "7": 0.9942196531791907, + "8": 0.9887323943661972, + "9": 0.9820359281437125, + "10": 0.8680981595092024 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9971098265895953, + "8": 0.9943661971830986, + "9": 0.9880239520958084, + "10": 0.8957055214723927 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9943661971830986, + "9": 0.9970059880239521, + "10": 0.9141104294478528 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9943661971830986, + "9": 0.9970059880239521, + "10": 0.9294478527607362 + } + }, + "total_pass_at_k": { + "pass@1": 0.8738532110091743, + "pass@2": 0.903997378768021, + "pass@4": 0.9410222804718218, + "pass@8": 0.9695281782437746, + "pass@16": 0.9813237221494102, + "pass@32": 0.9862385321100917, + "pass@64": 0.9895150720838795, + "pass@128": 0.9911533420707732 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99, + "pass@2": 0.993, + "pass@4": 0.996, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.954, + "pass@2": 0.969, + "pass@4": 0.987, + "pass@8": 0.99, + "pass@16": 0.992, + "pass@32": 0.994, + "pass@64": 0.997, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.965, + "pass@2": 0.975, + "pass@4": 0.99, + "pass@8": 0.996, + "pass@16": 0.998, + "pass@32": 0.998, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.852, + "pass@2": 0.898, + "pass@4": 0.947, + "pass@8": 0.981, + "pass@16": 0.992, + "pass@32": 0.995, + "pass@64": 0.997, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.783, + "pass@2": 0.849, + "pass@4": 0.919, + "pass@8": 0.97, + "pass@16": 0.99, + "pass@32": 0.996, + "pass@64": 0.996, + "pass@128": 0.996 + }, + "9": { + "pass@1": 0.746, + "pass@2": 0.812, + "pass@4": 0.886, + "pass@8": 0.948, + "pass@16": 0.983, + "pass@32": 0.995, + "pass@64": 0.998, + "pass@128": 0.998 + }, + "10": { + "pass@1": 0.622, + "pass@2": 0.691, + "pass@4": 0.773, + "pass@8": 0.841, + "pass@16": 0.901, + "pass@32": 0.924, + "pass@64": 0.935, + "pass@128": 0.947 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.999, + "4": 0.99, + "5": 0.954, + "6": 0.965, + "7": 0.852, + "8": 0.783, + "9": 0.746, + "10": 0.622 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.993, + "5": 0.969, + "6": 0.975, + "7": 0.898, + "8": 0.849, + "9": 0.812, + "10": 0.691 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996, + "5": 0.987, + "6": 0.99, + "7": 0.947, + "8": 0.919, + "9": 0.886, + "10": 0.773 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.99, + "6": 0.996, + "7": 0.981, + "8": 0.97, + "9": 0.948, + "10": 0.841 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.992, + "6": 0.998, + "7": 0.992, + "8": 0.99, + "9": 0.983, + "10": 0.901 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.994, + "6": 0.998, + "7": 0.995, + "8": 0.996, + "9": 0.995, + "10": 0.924 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.997, + "6": 0.999, + "7": 0.997, + "8": 0.996, + "9": 0.998, + "10": 0.935 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 1.0, + "7": 0.997, + "8": 0.996, + "9": 0.998, + "10": 0.947 + } + }, + "total_pass_at_k": { + "pass@1": 0.879, + "pass@2": 0.9096666666666666, + "pass@4": 0.9442222222222222, + "pass@8": 0.9695555555555555, + "pass@16": 0.984, + "pass@32": 0.9891111111111112, + "pass@64": 0.9913333333333333, + "pass@128": 0.9928888888888889 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5155807365439093, + "pass@2": 0.6147308781869688, + "pass@4": 0.7195467422096318, + "pass@8": 0.7932011331444759, + "pass@16": 0.8583569405099151, + "pass@32": 0.9093484419263456, + "pass@64": 0.9178470254957507, + "pass@128": 0.9291784702549575 + }, + "12": { + "pass@1": 0.5169491525423728, + "pass@2": 0.6045197740112994, + "pass@4": 0.6779661016949152, + "pass@8": 0.7429378531073446, + "pass@16": 0.7966101694915254, + "pass@32": 0.8135593220338984, + "pass@64": 0.8361581920903954, + "pass@128": 0.8502824858757062 + }, + "13": { + "pass@1": 0.3385579937304075, + "pass@2": 0.41379310344827586, + "pass@4": 0.5047021943573667, + "pass@8": 0.554858934169279, + "pass@16": 0.6269592476489029, + "pass@32": 0.6865203761755486, + "pass@64": 0.7053291536050157, + "pass@128": 0.7429467084639498 + }, + "14": { + "pass@1": 0.21495327102803738, + "pass@2": 0.2803738317757009, + "pass@4": 0.3302180685358255, + "pass@8": 0.411214953271028, + "pass@16": 0.4672897196261682, + "pass@32": 0.5202492211838006, + "pass@64": 0.557632398753894, + "pass@128": 0.616822429906542 + }, + "15": { + "pass@1": 0.1069364161849711, + "pass@2": 0.14739884393063585, + "pass@4": 0.1676300578034682, + "pass@8": 0.21676300578034682, + "pass@16": 0.2658959537572254, + "pass@32": 0.30346820809248554, + "pass@64": 0.34971098265895956, + "pass@128": 0.3930635838150289 + }, + "16": { + "pass@1": 0.041935483870967745, + "pass@2": 0.06129032258064516, + "pass@4": 0.07741935483870968, + "pass@8": 0.0935483870967742, + "pass@16": 0.11612903225806452, + "pass@32": 0.14193548387096774, + "pass@64": 0.16774193548387098, + "pass@128": 0.19032258064516128 + }, + "17": { + "pass@1": 0.017341040462427744, + "pass@2": 0.023121387283236993, + "pass@4": 0.03468208092485549, + "pass@8": 0.046242774566473986, + "pass@16": 0.06358381502890173, + "pass@32": 0.07225433526011561, + "pass@64": 0.08670520231213873, + "pass@128": 0.09826589595375723 + }, + "18": { + "pass@1": 0.011428571428571429, + "pass@2": 0.017142857142857144, + "pass@4": 0.02, + "pass@8": 0.022857142857142857, + "pass@16": 0.03428571428571429, + "pass@32": 0.037142857142857144, + "pass@64": 0.04857142857142857, + "pass@128": 0.06 + }, + "19": { + "pass@1": 0.003003003003003003, + "pass@2": 0.003003003003003003, + "pass@4": 0.006006006006006006, + "pass@8": 0.006006006006006006, + "pass@16": 0.006006006006006006, + "pass@32": 0.009009009009009009, + "pass@64": 0.009009009009009009, + "pass@128": 0.015015015015015015 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5155807365439093, + "12": 0.5169491525423728, + "13": 0.3385579937304075, + "14": 0.21495327102803738, + "15": 0.1069364161849711, + "16": 0.041935483870967745, + "17": 0.017341040462427744, + "18": 0.011428571428571429, + "19": 0.003003003003003003, + "20": 0.0 + }, + "pass@2": { + "11": 0.6147308781869688, + "12": 0.6045197740112994, + "13": 0.41379310344827586, + "14": 0.2803738317757009, + "15": 0.14739884393063585, + "16": 0.06129032258064516, + "17": 0.023121387283236993, + "18": 0.017142857142857144, + "19": 0.003003003003003003, + "20": 0.0 + }, + "pass@4": { + "11": 0.7195467422096318, + "12": 0.6779661016949152, + "13": 0.5047021943573667, + "14": 0.3302180685358255, + "15": 0.1676300578034682, + "16": 0.07741935483870968, + "17": 0.03468208092485549, + "18": 0.02, + "19": 0.006006006006006006, + "20": 0.0 + }, + "pass@8": { + "11": 0.7932011331444759, + "12": 0.7429378531073446, + "13": 0.554858934169279, + "14": 0.411214953271028, + "15": 0.21676300578034682, + "16": 0.0935483870967742, + "17": 0.046242774566473986, + "18": 0.022857142857142857, + "19": 0.006006006006006006, + "20": 0.0 + }, + "pass@16": { + "11": 0.8583569405099151, + "12": 0.7966101694915254, + "13": 0.6269592476489029, + "14": 0.4672897196261682, + "15": 0.2658959537572254, + "16": 0.11612903225806452, + "17": 0.06358381502890173, + "18": 0.03428571428571429, + "19": 0.006006006006006006, + "20": 0.0 + }, + "pass@32": { + "11": 0.9093484419263456, + "12": 0.8135593220338984, + "13": 0.6865203761755486, + "14": 0.5202492211838006, + "15": 0.30346820809248554, + "16": 0.14193548387096774, + "17": 0.07225433526011561, + "18": 0.037142857142857144, + "19": 0.009009009009009009, + "20": 0.0 + }, + "pass@64": { + "11": 0.9178470254957507, + "12": 0.8361581920903954, + "13": 0.7053291536050157, + "14": 0.557632398753894, + "15": 0.34971098265895956, + "16": 0.16774193548387098, + "17": 0.08670520231213873, + "18": 0.04857142857142857, + "19": 0.009009009009009009, + "20": 0.0 + }, + "pass@128": { + "11": 0.9291784702549575, + "12": 0.8502824858757062, + "13": 0.7429467084639498, + "14": 0.616822429906542, + "15": 0.3930635838150289, + "16": 0.19032258064516128, + "17": 0.09826589595375723, + "18": 0.06, + "19": 0.015015015015015015, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.17919762258543834, + "pass@2": 0.21931649331352154, + "pass@4": 0.2567607726597325, + "pass@8": 0.29182763744427936, + "pass@16": 0.32659732540861813, + "pass@32": 0.3521545319465082, + "pass@64": 0.37057949479940566, + "pass@128": 0.39197622585438335 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5333333333333333, + "pass@2": 0.6285714285714286, + "pass@4": 0.7428571428571429, + "pass@8": 0.8412698412698413, + "pass@16": 0.8793650793650793, + "pass@32": 0.9142857142857143, + "pass@64": 0.9365079365079365, + "pass@128": 0.946031746031746 + }, + "12": { + "pass@1": 0.4892966360856269, + "pass@2": 0.6085626911314985, + "pass@4": 0.6819571865443425, + "pass@8": 0.7492354740061162, + "pass@16": 0.8103975535168195, + "pass@32": 0.8440366972477065, + "pass@64": 0.8685015290519877, + "pass@128": 0.8837920489296636 + }, + "13": { + "pass@1": 0.3734567901234568, + "pass@2": 0.41975308641975306, + "pass@4": 0.48148148148148145, + "pass@8": 0.5462962962962963, + "pass@16": 0.6327160493827161, + "pass@32": 0.6635802469135802, + "pass@64": 0.7006172839506173, + "pass@128": 0.7283950617283951 + }, + "14": { + "pass@1": 0.2195845697329377, + "pass@2": 0.3026706231454006, + "pass@4": 0.32344213649851633, + "pass@8": 0.3620178041543027, + "pass@16": 0.41543026706231456, + "pass@32": 0.4688427299703264, + "pass@64": 0.5311572700296736, + "pass@128": 0.5934718100890207 + }, + "15": { + "pass@1": 0.1728395061728395, + "pass@2": 0.22530864197530864, + "pass@4": 0.2623456790123457, + "pass@8": 0.2932098765432099, + "pass@16": 0.33024691358024694, + "pass@32": 0.3734567901234568, + "pass@64": 0.4012345679012346, + "pass@128": 0.4444444444444444 + }, + "16": { + "pass@1": 0.04129793510324484, + "pass@2": 0.06489675516224189, + "pass@4": 0.08259587020648967, + "pass@8": 0.10029498525073746, + "pass@16": 0.12684365781710916, + "pass@32": 0.15339233038348082, + "pass@64": 0.1887905604719764, + "pass@128": 0.2153392330383481 + }, + "17": { + "pass@1": 0.0391566265060241, + "pass@2": 0.0572289156626506, + "pass@4": 0.06325301204819277, + "pass@8": 0.07530120481927711, + "pass@16": 0.08734939759036145, + "pass@32": 0.0963855421686747, + "pass@64": 0.12951807228915663, + "pass@128": 0.14156626506024098 + }, + "18": { + "pass@1": 0.016129032258064516, + "pass@2": 0.03225806451612903, + "pass@4": 0.035483870967741936, + "pass@8": 0.041935483870967745, + "pass@16": 0.05161290322580645, + "pass@32": 0.05806451612903226, + "pass@64": 0.06774193548387097, + "pass@128": 0.08064516129032258 + }, + "19": { + "pass@1": 0.0030211480362537764, + "pass@2": 0.0030211480362537764, + "pass@4": 0.012084592145015106, + "pass@8": 0.015105740181268883, + "pass@16": 0.015105740181268883, + "pass@32": 0.015105740181268883, + "pass@64": 0.027190332326283987, + "pass@128": 0.030211480362537766 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.003003003003003003, + "pass@4": 0.006006006006006006, + "pass@8": 0.006006006006006006, + "pass@16": 0.009009009009009009, + "pass@32": 0.018018018018018018, + "pass@64": 0.021021021021021023, + "pass@128": 0.024024024024024024 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5333333333333333, + "12": 0.4892966360856269, + "13": 0.3734567901234568, + "14": 0.2195845697329377, + "15": 0.1728395061728395, + "16": 0.04129793510324484, + "17": 0.0391566265060241, + "18": 0.016129032258064516, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@2": { + "11": 0.6285714285714286, + "12": 0.6085626911314985, + "13": 0.41975308641975306, + "14": 0.3026706231454006, + "15": 0.22530864197530864, + "16": 0.06489675516224189, + "17": 0.0572289156626506, + "18": 0.03225806451612903, + "19": 0.0030211480362537764, + "20": 0.003003003003003003 + }, + "pass@4": { + "11": 0.7428571428571429, + "12": 0.6819571865443425, + "13": 0.48148148148148145, + "14": 0.32344213649851633, + "15": 0.2623456790123457, + "16": 0.08259587020648967, + "17": 0.06325301204819277, + "18": 0.035483870967741936, + "19": 0.012084592145015106, + "20": 0.006006006006006006 + }, + "pass@8": { + "11": 0.8412698412698413, + "12": 0.7492354740061162, + "13": 0.5462962962962963, + "14": 0.3620178041543027, + "15": 0.2932098765432099, + "16": 0.10029498525073746, + "17": 0.07530120481927711, + "18": 0.041935483870967745, + "19": 0.015105740181268883, + "20": 0.006006006006006006 + }, + "pass@16": { + "11": 0.8793650793650793, + "12": 0.8103975535168195, + "13": 0.6327160493827161, + "14": 0.41543026706231456, + "15": 0.33024691358024694, + "16": 0.12684365781710916, + "17": 0.08734939759036145, + "18": 0.05161290322580645, + "19": 0.015105740181268883, + "20": 0.009009009009009009 + }, + "pass@32": { + "11": 0.9142857142857143, + "12": 0.8440366972477065, + "13": 0.6635802469135802, + "14": 0.4688427299703264, + "15": 0.3734567901234568, + "16": 0.15339233038348082, + "17": 0.0963855421686747, + "18": 0.05806451612903226, + "19": 0.015105740181268883, + "20": 0.018018018018018018 + }, + "pass@64": { + "11": 0.9365079365079365, + "12": 0.8685015290519877, + "13": 0.7006172839506173, + "14": 0.5311572700296736, + "15": 0.4012345679012346, + "16": 0.1887905604719764, + "17": 0.12951807228915663, + "18": 0.06774193548387097, + "19": 0.027190332326283987, + "20": 0.021021021021021023 + }, + "pass@128": { + "11": 0.946031746031746, + "12": 0.8837920489296636, + "13": 0.7283950617283951, + "14": 0.5934718100890207, + "15": 0.4444444444444444, + "16": 0.2153392330383481, + "17": 0.14156626506024098, + "18": 0.08064516129032258, + "19": 0.030211480362537766, + "20": 0.024024024024024024 + } + }, + "total_pass_at_k": { + "pass@1": 0.18704156479217604, + "pass@2": 0.23257946210268948, + "pass@4": 0.2668092909535452, + "pass@8": 0.3004278728606357, + "pass@16": 0.33312958435207823, + "pass@32": 0.3578850855745721, + "pass@64": 0.3847799511002445, + "pass@128": 0.406479217603912 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5873493975903614, + "pass@2": 0.6506024096385542, + "pass@4": 0.7289156626506024, + "pass@8": 0.8313253012048193, + "pass@16": 0.8855421686746988, + "pass@32": 0.9246987951807228, + "pass@64": 0.9578313253012049, + "pass@128": 0.9608433734939759 + }, + "12": { + "pass@1": 0.49216300940438873, + "pass@2": 0.554858934169279, + "pass@4": 0.6081504702194357, + "pass@8": 0.6927899686520376, + "pass@16": 0.7554858934169278, + "pass@32": 0.8056426332288401, + "pass@64": 0.8401253918495298, + "pass@128": 0.8495297805642633 + }, + "13": { + "pass@1": 0.32492997198879553, + "pass@2": 0.42296918767507, + "pass@4": 0.4957983193277311, + "pass@8": 0.5798319327731093, + "pass@16": 0.6638655462184874, + "pass@32": 0.6918767507002801, + "pass@64": 0.7338935574229691, + "pass@128": 0.7591036414565826 + }, + "14": { + "pass@1": 0.20175438596491227, + "pass@2": 0.260233918128655, + "pass@4": 0.30701754385964913, + "pass@8": 0.3713450292397661, + "pass@16": 0.42105263157894735, + "pass@32": 0.47368421052631576, + "pass@64": 0.5233918128654971, + "pass@128": 0.5760233918128655 + }, + "15": { + "pass@1": 0.13636363636363635, + "pass@2": 0.18484848484848485, + "pass@4": 0.22727272727272727, + "pass@8": 0.2696969696969697, + "pass@16": 0.3212121212121212, + "pass@32": 0.3696969696969697, + "pass@64": 0.41818181818181815, + "pass@128": 0.4696969696969697 + }, + "16": { + "pass@1": 0.039886039886039885, + "pass@2": 0.06552706552706553, + "pass@4": 0.07977207977207977, + "pass@8": 0.1111111111111111, + "pass@16": 0.13675213675213677, + "pass@32": 0.14814814814814814, + "pass@64": 0.16809116809116809, + "pass@128": 0.20512820512820512 + }, + "17": { + "pass@1": 0.024844720496894408, + "pass@2": 0.034161490683229816, + "pass@4": 0.049689440993788817, + "pass@8": 0.055900621118012424, + "pass@16": 0.06521739130434782, + "pass@32": 0.07453416149068323, + "pass@64": 0.10248447204968944, + "pass@128": 0.11490683229813664 + }, + "18": { + "pass@1": 0.014705882352941176, + "pass@2": 0.01764705882352941, + "pass@4": 0.020588235294117647, + "pass@8": 0.03235294117647059, + "pass@16": 0.03235294117647059, + "pass@32": 0.03823529411764706, + "pass@64": 0.05588235294117647, + "pass@128": 0.07941176470588235 + }, + "19": { + "pass@1": 0.002976190476190476, + "pass@2": 0.005952380952380952, + "pass@4": 0.005952380952380952, + "pass@8": 0.005952380952380952, + "pass@16": 0.005952380952380952, + "pass@32": 0.005952380952380952, + "pass@64": 0.008928571428571428, + "pass@128": 0.017857142857142856 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0029940119760479044, + "pass@16": 0.0029940119760479044, + "pass@32": 0.005988023952095809, + "pass@64": 0.005988023952095809, + "pass@128": 0.008982035928143712 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5873493975903614, + "12": 0.49216300940438873, + "13": 0.32492997198879553, + "14": 0.20175438596491227, + "15": 0.13636363636363635, + "16": 0.039886039886039885, + "17": 0.024844720496894408, + "18": 0.014705882352941176, + "19": 0.002976190476190476, + "20": 0.0 + }, + "pass@2": { + "11": 0.6506024096385542, + "12": 0.554858934169279, + "13": 0.42296918767507, + "14": 0.260233918128655, + "15": 0.18484848484848485, + "16": 0.06552706552706553, + "17": 0.034161490683229816, + "18": 0.01764705882352941, + "19": 0.005952380952380952, + "20": 0.0 + }, + "pass@4": { + "11": 0.7289156626506024, + "12": 0.6081504702194357, + "13": 0.4957983193277311, + "14": 0.30701754385964913, + "15": 0.22727272727272727, + "16": 0.07977207977207977, + "17": 0.049689440993788817, + "18": 0.020588235294117647, + "19": 0.005952380952380952, + "20": 0.0 + }, + "pass@8": { + "11": 0.8313253012048193, + "12": 0.6927899686520376, + "13": 0.5798319327731093, + "14": 0.3713450292397661, + "15": 0.2696969696969697, + "16": 0.1111111111111111, + "17": 0.055900621118012424, + "18": 0.03235294117647059, + "19": 0.005952380952380952, + "20": 0.0029940119760479044 + }, + "pass@16": { + "11": 0.8855421686746988, + "12": 0.7554858934169278, + "13": 0.6638655462184874, + "14": 0.42105263157894735, + "15": 0.3212121212121212, + "16": 0.13675213675213677, + "17": 0.06521739130434782, + "18": 0.03235294117647059, + "19": 0.005952380952380952, + "20": 0.0029940119760479044 + }, + "pass@32": { + "11": 0.9246987951807228, + "12": 0.8056426332288401, + "13": 0.6918767507002801, + "14": 0.47368421052631576, + "15": 0.3696969696969697, + "16": 0.14814814814814814, + "17": 0.07453416149068323, + "18": 0.03823529411764706, + "19": 0.005952380952380952, + "20": 0.005988023952095809 + }, + "pass@64": { + "11": 0.9578313253012049, + "12": 0.8401253918495298, + "13": 0.7338935574229691, + "14": 0.5233918128654971, + "15": 0.41818181818181815, + "16": 0.16809116809116809, + "17": 0.10248447204968944, + "18": 0.05588235294117647, + "19": 0.008928571428571428, + "20": 0.005988023952095809 + }, + "pass@128": { + "11": 0.9608433734939759, + "12": 0.8495297805642633, + "13": 0.7591036414565826, + "14": 0.5760233918128655, + "15": 0.4696969696969697, + "16": 0.20512820512820512, + "17": 0.11490683229813664, + "18": 0.07941176470588235, + "19": 0.017857142857142856, + "20": 0.008982035928143712 + } + }, + "total_pass_at_k": { + "pass@1": 0.18138566755872731, + "pass@2": 0.21885221528397264, + "pass@4": 0.25156110615521854, + "pass@8": 0.29467737139458816, + "pass@16": 0.32857567647933394, + "pass@32": 0.35325602140945583, + "pass@64": 0.3809099018733274, + "pass@128": 0.40380612548319955 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.545, + "pass@2": 0.631, + "pass@4": 0.73, + "pass@8": 0.821, + "pass@16": 0.874, + "pass@32": 0.916, + "pass@64": 0.937, + "pass@128": 0.945 + }, + "12": { + "pass@1": 0.5, + "pass@2": 0.59, + "pass@4": 0.657, + "pass@8": 0.729, + "pass@16": 0.788, + "pass@32": 0.821, + "pass@64": 0.848, + "pass@128": 0.861 + }, + "13": { + "pass@1": 0.345, + "pass@2": 0.419, + "pass@4": 0.494, + "pass@8": 0.561, + "pass@16": 0.642, + "pass@32": 0.681, + "pass@64": 0.714, + "pass@128": 0.744 + }, + "14": { + "pass@1": 0.212, + "pass@2": 0.281, + "pass@4": 0.32, + "pass@8": 0.381, + "pass@16": 0.434, + "pass@32": 0.487, + "pass@64": 0.537, + "pass@128": 0.595 + }, + "15": { + "pass@1": 0.138, + "pass@2": 0.185, + "pass@4": 0.218, + "pass@8": 0.259, + "pass@16": 0.305, + "pass@32": 0.348, + "pass@64": 0.389, + "pass@128": 0.435 + }, + "16": { + "pass@1": 0.041, + "pass@2": 0.064, + "pass@4": 0.08, + "pass@8": 0.102, + "pass@16": 0.127, + "pass@32": 0.148, + "pass@64": 0.175, + "pass@128": 0.204 + }, + "17": { + "pass@1": 0.027, + "pass@2": 0.038, + "pass@4": 0.049, + "pass@8": 0.059, + "pass@16": 0.072, + "pass@32": 0.081, + "pass@64": 0.106, + "pass@128": 0.118 + }, + "18": { + "pass@1": 0.014, + "pass@2": 0.022, + "pass@4": 0.025, + "pass@8": 0.032, + "pass@16": 0.039, + "pass@32": 0.044, + "pass@64": 0.057, + "pass@128": 0.073 + }, + "19": { + "pass@1": 0.003, + "pass@2": 0.004, + "pass@4": 0.008, + "pass@8": 0.009, + "pass@16": 0.009, + "pass@32": 0.01, + "pass@64": 0.015, + "pass@128": 0.021 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.001, + "pass@4": 0.002, + "pass@8": 0.003, + "pass@16": 0.004, + "pass@32": 0.008, + "pass@64": 0.009, + "pass@128": 0.011 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.545, + "12": 0.5, + "13": 0.345, + "14": 0.212, + "15": 0.138, + "16": 0.041, + "17": 0.027, + "18": 0.014, + "19": 0.003, + "20": 0.0 + }, + "pass@2": { + "11": 0.631, + "12": 0.59, + "13": 0.419, + "14": 0.281, + "15": 0.185, + "16": 0.064, + "17": 0.038, + "18": 0.022, + "19": 0.004, + "20": 0.001 + }, + "pass@4": { + "11": 0.73, + "12": 0.657, + "13": 0.494, + "14": 0.32, + "15": 0.218, + "16": 0.08, + "17": 0.049, + "18": 0.025, + "19": 0.008, + "20": 0.002 + }, + "pass@8": { + "11": 0.821, + "12": 0.729, + "13": 0.561, + "14": 0.381, + "15": 0.259, + "16": 0.102, + "17": 0.059, + "18": 0.032, + "19": 0.009, + "20": 0.003 + }, + "pass@16": { + "11": 0.874, + "12": 0.788, + "13": 0.642, + "14": 0.434, + "15": 0.305, + "16": 0.127, + "17": 0.072, + "18": 0.039, + "19": 0.009, + "20": 0.004 + }, + "pass@32": { + "11": 0.916, + "12": 0.821, + "13": 0.681, + "14": 0.487, + "15": 0.348, + "16": 0.148, + "17": 0.081, + "18": 0.044, + "19": 0.01, + "20": 0.008 + }, + "pass@64": { + "11": 0.937, + "12": 0.848, + "13": 0.714, + "14": 0.537, + "15": 0.389, + "16": 0.175, + "17": 0.106, + "18": 0.057, + "19": 0.015, + "20": 0.009 + }, + "pass@128": { + "11": 0.945, + "12": 0.861, + "13": 0.744, + "14": 0.595, + "15": 0.435, + "16": 0.204, + "17": 0.118, + "18": 0.073, + "19": 0.021, + "20": 0.011 + } + }, + "total_pass_at_k": { + "pass@1": 0.1825, + "pass@2": 0.2235, + "pass@4": 0.2583, + "pass@8": 0.2956, + "pass@16": 0.3294, + "pass@32": 0.3544, + "pass@64": 0.3787, + "pass@128": 0.4007 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block4-epoch1-op7-10/summary.csv b/composition/SDAR-100M-block4-epoch1-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..404bcd97ae08565e5f51dcb6ac56c9986f56de7b --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch1-op7-10,0.9879244791666667,0.43462734375,0.6967154605263158,0.0,0.0,0.0,200.573390625,332.18273828125,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block4-epoch1-op7-10/summary.json b/composition/SDAR-100M-block4-epoch1-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ea1971e59e91ee13035f29707b41113fc19db2cc --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch1-op7-10", + "id_acc": 0.9879244791666667, + "ood_acc": 0.43462734375, + "total_acc": 0.6967154605263158, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.573390625, + "ood_avg_resp_len": 332.18273828125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 560078, + "answer_accuracy": 0.691686591052798, + "avg_response_len": 257.03733228936136, + "resp_tokens_sum": 208130325, + "pass_at_k": { + "pass@1": 0.691686591052798, + "pass@2": 0.7285240024607798, + "pass@4": 0.7584811279468562, + "pass@8": 0.7832984708290734, + "pass@16": 0.8045310794934998, + "pass@32": 0.823173791139723, + "pass@64": 0.8397895761692062, + "pass@128": 0.8548846032247865 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9426622418879056, + "pass@2": 0.9608814281234755, + "pass@4": 0.9727997515244226, + "pass@8": 0.9800427227199637, + "pass@16": 0.9848555713566579, + "pass@32": 0.9893375479172565, + "pass@64": 0.9948648057621975, + "pass@128": 1.0 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999522171253823, + "pass@2": 0.9999996237568928, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995068359375, + "pass@2": 0.995943036417323, + "pass@4": 0.996855512396888, + "pass@8": 0.9975476953695743, + "pass@16": 0.9981801935688303, + "pass@32": 0.9990268911601001, + "pass@64": 0.9998138779527558, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9858382936507937, + "pass@2": 0.9894134326959131, + "pass@4": 0.9919654816957404, + "pass@8": 0.993967779218521, + "pass@16": 0.9953518900581235, + "pass@32": 0.9962822752680421, + "pass@64": 0.9967815120663723, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9956025449101796, + "pass@2": 0.9967230892545617, + "pass@4": 0.9970756080639619, + "pass@8": 0.9971929848408553, + "pass@16": 0.9973802395197485, + "pass@32": 0.9977544910179641, + "pass@64": 0.9985029940119761, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9944526627218935, + "pass@2": 0.9979470484088896, + "pass@4": 0.9992837274778522, + "pass@8": 0.9998536405673738, + "pass@16": 0.9999944055596715, + "pass@32": 0.9999999969008433, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9830729166666666, + "pass@2": 0.9861033559206539, + "pass@4": 0.987899911849961, + "pass@8": 0.9900541498080317, + "pass@16": 0.9924767953718153, + "pass@32": 0.9948968439236284, + "pass@64": 0.9965094689481032, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9885343309859155, + "pass@2": 0.9931147415992014, + "pass@4": 0.9947651726632762, + "pass@8": 0.9954927131433333, + "pass@16": 0.9964173683303595, + "pass@32": 0.9977688493163707, + "pass@64": 0.9992201397360542, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9032179532577904, + "pass@2": 0.9327289570832682, + "pass@4": 0.9506815622905493, + "pass@8": 0.9626199544828037, + "pass@16": 0.9704751886621362, + "pass@32": 0.9749374452489946, + "pass@64": 0.9778428066504992, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8037826624293786, + "pass@2": 0.8664708589127629, + "pass@4": 0.9026233156236823, + "pass@8": 0.9260914408190967, + "pass@16": 0.9431713204069097, + "pass@32": 0.9549023146846267, + "pass@64": 0.9638589731267034, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.6582092476489029, + "pass@2": 0.743703024337867, + "pass@4": 0.8042070038071254, + "pass@8": 0.8481824192918073, + "pass@16": 0.877888284808596, + "pass@32": 0.8965888649622713, + "pass@64": 0.9091856827840178, + "pass@128": 0.9184952978056427 + }, + "14": { + "pass@1": 0.5494548286604362, + "pass@2": 0.6482951130816593, + "pass@4": 0.7280674673259301, + "pass@8": 0.7889771206999603, + "pass@16": 0.8312085331718211, + "pass@32": 0.8598877871539242, + "pass@64": 0.8815913721972909, + "pass@128": 0.9003115264797508 + }, + "15": { + "pass@1": 0.4004019147398844, + "pass@2": 0.4798765275818123, + "pass@4": 0.5449855960020893, + "pass@8": 0.6041668693159691, + "pass@16": 0.6603312308303183, + "pass@32": 0.711706774366159, + "pass@64": 0.7567645044134567, + "pass@128": 0.7947976878612717 + }, + "16": { + "pass@1": 0.2763860887096774, + "pass@2": 0.35277455867411744, + "pass@4": 0.42198129618878266, + "pass@8": 0.48262699673255544, + "pass@16": 0.5365621998798281, + "pass@32": 0.5864838038611693, + "pass@64": 0.6325214538236035, + "pass@128": 0.6774193548387096 + }, + "17": { + "pass@1": 0.19533507947976878, + "pass@2": 0.25727059248554907, + "pass@4": 0.31503235406629077, + "pass@8": 0.36664014844214665, + "pass@16": 0.415407852939479, + "pass@32": 0.46282528956899566, + "pass@64": 0.5089321507525993, + "pass@128": 0.5549132947976878 + }, + "18": { + "pass@1": 0.18772321428571428, + "pass@2": 0.24447272215972995, + "pass@4": 0.29619767395147045, + "pass@8": 0.34292779478769564, + "pass@16": 0.38793229002272034, + "pass@32": 0.43505393984941876, + "pass@64": 0.48128101171399873, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.1686373873873874, + "pass@2": 0.22988933815705478, + "pass@4": 0.28967174852017374, + "pass@8": 0.3425929783847028, + "pass@16": 0.3887203432902438, + "pass@32": 0.42960762813584363, + "pass@64": 0.46859317448539534, + "pass@128": 0.5105105105105106 + }, + "20": { + "pass@1": 0.14841403903903905, + "pass@2": 0.19893675663852825, + "pass@4": 0.24889135141891053, + "pass@8": 0.29487053016690784, + "pass@16": 0.3395049732401222, + "pass@32": 0.38101573199885735, + "pass@64": 0.4152574877923962, + "pass@128": 0.4444444444444444 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 570890, + "answer_accuracy": 0.6952576968043648, + "avg_response_len": 266.9440958690569, + "resp_tokens_sum": 219193136, + "pass_at_k": { + "pass@1": 0.6952576968043648, + "pass@2": 0.7309432171767659, + "pass@4": 0.7608909380725237, + "pass@8": 0.786381191349136, + "pass@16": 0.8083556528635224, + "pass@32": 0.8275322430716204, + "pass@64": 0.8447360879524896, + "pass@128": 0.8603273577552611 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.929879217791411, + "pass@2": 0.9455011986135936, + "pass@4": 0.9572367023907287, + "pass@8": 0.9653763793871797, + "pass@16": 0.9709467034438971, + "pass@32": 0.9763425037418643, + "pass@64": 0.9817604399544487, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997395833333333, + "pass@2": 0.999989188141255, + "pass@4": 0.9999999801161218, + "pass@8": 0.9999999999999979, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997457470414202, + "pass@2": 0.9999799800820015, + "pass@4": 0.9999999084803749, + "pass@8": 0.9999999999996586, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9941123188405797, + "pass@2": 0.9961107782722813, + "pass@4": 0.9970168766947609, + "pass@8": 0.997577219367406, + "pass@16": 0.9980644436436175, + "pass@32": 0.998786836422768, + "pass@64": 0.9996462398721899, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9970898997134671, + "pass@2": 0.9972238595086073, + "pass@4": 0.9973116384270019, + "pass@8": 0.9974829659544705, + "pass@16": 0.9978086997721273, + "pass@32": 0.9983924824583174, + "pass@64": 0.9992893080342035, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9956195809248555, + "pass@2": 0.996662859564881, + "pass@4": 0.9970522314587844, + "pass@8": 0.9971089149255284, + "pass@16": 0.997109826485928, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9900308098591549, + "pass@2": 0.9928277836309195, + "pass@4": 0.9939063929508812, + "pass@8": 0.9944474182396722, + "pass@16": 0.9947159353923489, + "pass@32": 0.9950704220578349, + "pass@64": 0.995774647887324, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9818488023952096, + "pass@2": 0.9868720973407518, + "pass@4": 0.9893204078344998, + "pass@8": 0.9909175880114821, + "pass@16": 0.9920082924256072, + "pass@32": 0.993079471807874, + "pass@64": 0.994766372753843, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.913074171686747, + "pass@2": 0.9404327447585614, + "pass@4": 0.9566515369690232, + "pass@8": 0.9679729746481276, + "pass@16": 0.976421032163366, + "pass@32": 0.9826638897843468, + "pass@64": 0.9870178875177636, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8431622257053292, + "pass@2": 0.8853485301014486, + "pass@4": 0.9075407620481607, + "pass@8": 0.9225588387029565, + "pass@16": 0.9338780820659167, + "pass@32": 0.9427151597704617, + "pass@64": 0.9499720050474115, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.6195509453781513, + "pass@2": 0.7090015632237144, + "pass@4": 0.7696727071405988, + "pass@8": 0.8128208903813218, + "pass@16": 0.8456239738232122, + "pass@32": 0.8723095721456559, + "pass@64": 0.8960096112676782, + "pass@128": 0.9187675070028011 + }, + "14": { + "pass@1": 0.5342881944444444, + "pass@2": 0.6228624176912091, + "pass@4": 0.68985191626266, + "pass@8": 0.7400756479989421, + "pass@16": 0.777373163396424, + "pass@32": 0.8060593902488095, + "pass@64": 0.8296373557844222, + "pass@128": 0.847953216374269 + }, + "15": { + "pass@1": 0.40404829545454546, + "pass@2": 0.4952815557146265, + "pass@4": 0.5744040432445942, + "pass@8": 0.6418894193523699, + "pass@16": 0.6989534805926838, + "pass@32": 0.7435311343356013, + "pass@64": 0.7784359269113569, + "pass@128": 0.8090909090909091 + }, + "16": { + "pass@1": 0.2513577279202279, + "pass@2": 0.321735073580546, + "pass@4": 0.3875065542020923, + "pass@8": 0.4469587390581188, + "pass@16": 0.5009118793512886, + "pass@32": 0.5501861383741561, + "pass@64": 0.5973992977953705, + "pass@128": 0.6438746438746439 + }, + "17": { + "pass@1": 0.21858016304347827, + "pass@2": 0.28791013045923586, + "pass@4": 0.3574839279600918, + "pass@8": 0.42173292646814153, + "pass@16": 0.4792274741006837, + "pass@32": 0.5322220984044989, + "pass@64": 0.5842848428673304, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.20774356617647058, + "pass@2": 0.2646114375868455, + "pass@4": 0.3206001789298397, + "pass@8": 0.3763666437779109, + "pass@16": 0.4341574245513161, + "pass@32": 0.49379397470457037, + "pass@64": 0.5518441577798592, + "pass@128": 0.6058823529411764 + }, + "19": { + "pass@1": 0.1776181175595238, + "pass@2": 0.23588433281777288, + "pass@4": 0.291822026431964, + "pass@8": 0.34046070760558633, + "pass@16": 0.3797265269874325, + "pass@32": 0.4125415160967138, + "pass@64": 0.44444983941437916, + "pass@128": 0.4791666666666667 + }, + "20": { + "pass@1": 0.1407185628742515, + "pass@2": 0.19786669279551142, + "pass@4": 0.25815344816553615, + "pass@8": 0.31686300365274095, + "pass@16": 0.3719599344412786, + "pass@32": 0.41902785399935966, + "pass@64": 0.4536853513921381, + "pass@128": 0.47604790419161674 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 563444, + "answer_accuracy": 0.7032922591468286, + "avg_response_len": 285.7522542538744, + "resp_tokens_sum": 228930990, + "pass_at_k": { + "pass@1": 0.7032922591468286, + "pass@2": 0.7400347043564306, + "pass@4": 0.768887233609696, + "pass@8": 0.7921462560934432, + "pass@16": 0.8114023979090202, + "pass@32": 0.8278775245383643, + "pass@64": 0.842439411586423, + "pass@128": 0.8562070618309634 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9578125, + "pass@2": 0.9753870901398526, + "pass@4": 0.9833263334620483, + "pass@8": 0.98725395468186, + "pass@16": 0.9899596537917327, + "pass@32": 0.9919179552842842, + "pass@64": 0.9932696551144083, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999225583090379, + "pass@2": 0.9998217298730516, + "pass@4": 0.9999901725112349, + "pass@8": 0.9999999785510661, + "pass@16": 0.9999999999999812, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9825137867647059, + "pass@2": 0.9854840927512737, + "pass@4": 0.9874707255343083, + "pass@8": 0.9893014786507219, + "pass@16": 0.991217308173432, + "pass@32": 0.9938179066466555, + "pass@64": 0.997452524312115, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.995070977917981, + "pass@2": 0.9969331391738493, + "pass@4": 0.998256703913588, + "pass@8": 0.9991602076007862, + "pass@16": 0.9997983877960986, + "pass@32": 0.9999919321054616, + "pass@64": 0.9999999987471165, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9879351265822784, + "pass@2": 0.9915949242499751, + "pass@4": 0.9947475725937738, + "pass@8": 0.9970610280579045, + "pass@16": 0.9984926549110091, + "pass@32": 0.9994571169803051, + "pass@64": 0.9999562540838489, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9900760135135135, + "pass@2": 0.9945843973895155, + "pass@4": 0.996588656710704, + "pass@8": 0.9969831382519523, + "pass@16": 0.9969969824466679, + "pass@32": 0.9969969969969951, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9863092845659164, + "pass@2": 0.9924345519912906, + "pass@4": 0.9945501765052679, + "pass@8": 0.9955417133916913, + "pass@16": 0.9963403303182642, + "pass@32": 0.9967403207823743, + "pass@64": 0.9967844774326561, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.9010664682539683, + "pass@2": 0.9314558336457947, + "pass@4": 0.9495970378702663, + "pass@8": 0.9613202533352903, + "pass@16": 0.9687759979531508, + "pass@32": 0.9740935366946681, + "pass@64": 0.9783593648058921, + "pass@128": 0.9809523809523809 + }, + "12": { + "pass@1": 0.8469753440366973, + "pass@2": 0.9016613390642684, + "pass@4": 0.9306340686771954, + "pass@8": 0.9488184652821481, + "pass@16": 0.9626058466450654, + "pass@32": 0.97255331767062, + "pass@64": 0.979117645036257, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.6840277777777778, + "pass@2": 0.7669574614805097, + "pass@4": 0.8165764693186498, + "pass@8": 0.8479331025188009, + "pass@16": 0.8715139042865675, + "pass@32": 0.8908118707209612, + "pass@64": 0.9047815820695196, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.5532965504451038, + "pass@2": 0.642291218836889, + "pass@4": 0.7043512042448701, + "pass@8": 0.7499437255097021, + "pass@16": 0.7861016572365503, + "pass@32": 0.8160076783891812, + "pass@64": 0.8389702413765118, + "pass@128": 0.857566765578635 + }, + "15": { + "pass@1": 0.41584683641975306, + "pass@2": 0.5078842683727031, + "pass@4": 0.5873538796655046, + "pass@8": 0.6499067906851369, + "pass@16": 0.6943756870491424, + "pass@32": 0.7266055349040754, + "pass@64": 0.7528233902976535, + "pass@128": 0.7777777777777778 + }, + "16": { + "pass@1": 0.2721008480825959, + "pass@2": 0.33834423559333837, + "pass@4": 0.40159421665212186, + "pass@8": 0.4602251988321013, + "pass@16": 0.5098591234832823, + "pass@32": 0.5509605333386498, + "pass@64": 0.5906200029115647, + "pass@128": 0.6371681415929203 + }, + "17": { + "pass@1": 0.22926863704819278, + "pass@2": 0.3040100856180628, + "pass@4": 0.3709875186911878, + "pass@8": 0.42611476990632063, + "pass@16": 0.47253513212532294, + "pass@32": 0.5125942185897905, + "pass@64": 0.5472100849333396, + "pass@128": 0.5813253012048193 + }, + "18": { + "pass@1": 0.22464717741935483, + "pass@2": 0.28498698247396503, + "pass@4": 0.33843989592026796, + "pass@8": 0.38398988335421697, + "pass@16": 0.4232406963502674, + "pass@32": 0.46041983832532407, + "pass@64": 0.5008218489889754, + "pass@128": 0.5451612903225806 + }, + "19": { + "pass@1": 0.1623158987915408, + "pass@2": 0.2182006327758879, + "pass@4": 0.2712618750261958, + "pass@8": 0.32116348907565456, + "pass@16": 0.36780298226078906, + "pass@32": 0.4113543919886992, + "pass@64": 0.45119827969048176, + "pass@128": 0.486404833836858 + }, + "20": { + "pass@1": 0.15871340090090091, + "pass@2": 0.21606732224350333, + "pass@4": 0.27226703193632334, + "pass@8": 0.32611956314764734, + "pass@16": 0.3775474082082862, + "pass@32": 0.42623156342189406, + "pass@64": 0.46941726977322745, + "pass@128": 0.5075075075075075 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_metrics.json b/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..dec6e6da6a2ae174bc6925fb7b97e98aebb01c4e --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch1-op9-12", + "id": { + "answer_accuracy": 0.9861276041666667, + "avg_response_len": 200.2339045138889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9861276041666667, + "pass@2": 0.9900604084645647, + "pass@4": 0.9923328891701035, + "pass@8": 0.9939966568330358, + "pass@16": 0.9953889368549511, + "pass@32": 0.9966042772153859, + "pass@64": 0.9975937986453868, + "pass@128": 0.9982222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9413671875, + "pass@2": 0.9560073818897641, + "pass@4": 0.9661297797150363, + "pass@8": 0.973958866476777, + "pass@16": 0.9801564851612712, + "pass@32": 0.9853814436253552, + "pass@64": 0.9896901293154565, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.999984375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999359375, + "pass@2": 0.999675812007874, + "pass@4": 0.9998979723472066, + "pass@8": 0.9999905979568364, + "pass@16": 0.9999999435227294, + "pass@32": 0.9999999999996563, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9974140625, + "pass@2": 0.998044783464567, + "pass@4": 0.998237083239595, + "pass@8": 0.9984480653393176, + "pass@16": 0.9987941264550821, + "pass@32": 0.9992680481403238, + "pass@64": 0.9997381448050702, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9842578125, + "pass@2": 0.9879093257874021, + "pass@4": 0.9901990563367077, + "pass@8": 0.9922341196998229, + "pass@16": 0.9942029297347884, + "pass@32": 0.9957719067641455, + "pass@64": 0.9967520479354316, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.994, + "pass@2": 0.9949153543307088, + "pass@4": 0.9953240142482191, + "pass@8": 0.9956690460023842, + "pass@16": 0.9961843259195156, + "pass@32": 0.9968943403441024, + "pass@64": 0.9976233493374305, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.990328125, + "pass@2": 0.9936660925196853, + "pass@4": 0.9954954812523434, + "pass@8": 0.9966721388643296, + "pass@16": 0.9974260781790963, + "pass@32": 0.9980104268701047, + "pass@64": 0.9986299091903094, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.986390625, + "pass@2": 0.9911269685039372, + "pass@4": 0.993347476752906, + "pass@8": 0.9948517012143449, + "pass@16": 0.9962994679373544, + "pass@32": 0.9976768848650488, + "pass@64": 0.9986739346860647, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.982046875, + "pass@2": 0.9891979576771657, + "pass@4": 0.9923651386389201, + "pass@8": 0.9941453759435105, + "pass@16": 0.9954370747847234, + "pass@32": 0.996435444329734, + "pass@64": 0.9972366725387218, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.9413671875, + "2": 0.999984375, + "3": 0.999359375, + "4": 0.9974140625, + "5": 0.9842578125, + "6": 0.994, + "7": 0.990328125, + "8": 0.986390625, + "9": 0.982046875 + }, + "per_op_avg_response_len": { + "10": 280.5143671875, + "2": 115.610421875, + "3": 144.835296875, + "4": 160.095046875, + "5": 174.0822578125, + "6": 199.3134609375, + "7": 224.143984375, + "8": 243.89225, + "9": 259.6180546875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.5143671875, + "2": 115.610421875, + "3": 144.835296875, + "4": 160.095046875, + "5": 174.0822578125, + "6": 199.3134609375, + "7": 224.143984375, + "8": 243.89225, + "9": 259.6180546875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 373518, + "answer_accuracy": 0.9855148176291794, + "avg_response_len": 190.75554605707532, + "resp_tokens_sum": 72297878, + "pass_at_k": { + "pass@1": 0.9855148176291794, + "pass@2": 0.9896487340678164, + "pass@4": 0.9921482425828996, + "pass@8": 0.9939548453003074, + "pass@16": 0.9953476381048021, + "pass@32": 0.9965963582818315, + "pass@64": 0.9976863434508711, + "pass@128": 0.9983113812901047 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9390440634218289, + "pass@2": 0.9555319170557223, + "pass@4": 0.9674239525369062, + "pass@8": 0.9756565404139677, + "pass@16": 0.9813557770331947, + "pass@32": 0.986617282494939, + "pass@64": 0.9912821110885033, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9938720703125, + "pass@2": 0.9941333200049213, + "pass@4": 0.9944950684875327, + "pass@8": 0.9951502049040684, + "pass@16": 0.9962316451721318, + "pass@32": 0.9977126504385122, + "pass@64": 0.999181702515844, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9853422619047619, + "pass@2": 0.9885260436195474, + "pass@4": 0.9904835181316618, + "pass@8": 0.9919887051387225, + "pass@16": 0.9930441900209205, + "pass@32": 0.9935671832417211, + "pass@64": 0.9936503542147976, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9926553143712575, + "pass@2": 0.9937994336130889, + "pass@4": 0.9941684698656681, + "pass@8": 0.9943757134349077, + "pass@16": 0.9947162761058925, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9921181582840237, + "pass@2": 0.9955388342729348, + "pass@4": 0.9968490602875822, + "pass@8": 0.9975059751526393, + "pass@16": 0.9980238469616003, + "pass@32": 0.9987617116948309, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9820462740384616, + "pass@2": 0.9870359504340805, + "pass@4": 0.990314350730197, + "pass@8": 0.9933464239381959, + "pass@16": 0.9958232946543495, + "pass@32": 0.9976443884867328, + "pass@64": 0.9991607192427966, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9861575704225352, + "pass@2": 0.9930700343795054, + "pass@4": 0.9960230070712991, + "pass@8": 0.9978186757417113, + "pass@16": 0.9990916543037955, + "pass@32": 0.9998066566382701, + "pass@64": 0.9999959302892975, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 385461, + "answer_accuracy": 0.9867018553407602, + "avg_response_len": 198.25831166038662, + "resp_tokens_sum": 77450799, + "pass_at_k": { + "pass@1": 0.9867018553407602, + "pass@2": 0.9899563649368948, + "pass@4": 0.991872276030618, + "pass@8": 0.9934139864923256, + "pass@16": 0.9948125983863382, + "pass@32": 0.9960225731229545, + "pass@64": 0.997029666582489, + "pass@128": 0.9977064220183486 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9318682898773006, + "pass@2": 0.9437727193130767, + "pass@4": 0.9541319070162242, + "pass@8": 0.9643895337030387, + "pass@16": 0.9734856182486636, + "pass@32": 0.9804536494144921, + "pass@64": 0.9859565813477191, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999289772727272, + "pass@2": 0.9999996271772846, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998543823964497, + "pass@2": 0.9997983448259796, + "pass@4": 0.9999962377114103, + "pass@8": 0.9999999993222367, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906929347826087, + "pass@2": 0.9938455722925938, + "pass@4": 0.99476957146661, + "pass@8": 0.9953830646909647, + "pass@16": 0.9963772680388042, + "pass@32": 0.9978842427305282, + "pass@64": 0.9994736049298187, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967541189111748, + "pass@2": 0.9971184543013785, + "pass@4": 0.9971346140829818, + "pass@8": 0.9971346704870158, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9941067557803468, + "pass@2": 0.9960366790314508, + "pass@4": 0.996919753256999, + "pass@8": 0.9971032378641861, + "pass@16": 0.9971098202636923, + "pass@32": 0.9971098265895949, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.988556338028169, + "pass@2": 0.99323742652767, + "pass@4": 0.9950167355841083, + "pass@8": 0.9959384773710429, + "pass@16": 0.997143266424544, + "pass@32": 0.9986983950949794, + "pass@64": 0.9998376400133082, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9778255988023952, + "pass@2": 0.9839705431656373, + "pass@4": 0.9873886173098122, + "pass@8": 0.9896570716478766, + "pass@16": 0.9912446822176693, + "pass@32": 0.9922904474891701, + "pass@64": 0.9932692843858927, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 377040, + "answer_accuracy": 0.9861483093404754, + "avg_response_len": 211.64834334198193, + "resp_tokens_sum": 80920781, + "pass_at_k": { + "pass@1": 0.9861483093404754, + "pass@2": 0.9905748071037493, + "pass@4": 0.9929865650477151, + "pass@8": 0.9946334542311802, + "pass@16": 0.9960187562742351, + "pass@32": 0.9972064897538371, + "pass@64": 0.9980784675060925, + "pass@128": 0.9986608637428859 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9529617537313433, + "pass@2": 0.9683944940651077, + "pass@4": 0.9764956958738361, + "pass@8": 0.9815531680276173, + "pass@16": 0.9854345229789683, + "pass@32": 0.9889262541804372, + "pass@64": 0.9917123824987969, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982233965014577, + "pass@2": 0.9990552041964142, + "pass@4": 0.9997025432863167, + "pass@8": 0.9999725887954414, + "pass@16": 0.9999998353432346, + "pass@32": 0.9999999999989982, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996116593567251, + "pass@2": 0.9999715804438919, + "pass@4": 0.9999998040266896, + "pass@8": 0.9999999999973679, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9767233455882353, + "pass@2": 0.9813144106067622, + "pass@4": 0.9852977822625111, + "pass@8": 0.9892662360668899, + "pass@16": 0.9930701541317976, + "pass@32": 0.9956711185322674, + "pass@64": 0.9968640960499501, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9923846608832808, + "pass@2": 0.9936656244566434, + "pass@4": 0.9945481545682183, + "pass@8": 0.9954181631392943, + "pass@16": 0.9966848255525158, + "pass@32": 0.9982819052559925, + "pass@64": 0.9995942613043955, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9842761075949367, + "pass@2": 0.9890673278181997, + "pass@4": 0.992488178000693, + "pass@8": 0.9953082245624335, + "pass@16": 0.997132974983664, + "pass@32": 0.9981929377128227, + "pass@64": 0.9992150520076071, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9881522147147147, + "pass@2": 0.9927101067603032, + "pass@4": 0.9944097843625403, + "pass@8": 0.995103475912635, + "pass@16": 0.9958460673407935, + "pass@32": 0.9966183345299415, + "pass@64": 0.9969772614942578, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9818880627009646, + "pass@2": 0.9903920709167786, + "pass@4": 0.9935343181579793, + "pass@8": 0.9947726177646687, + "pass@16": 0.9957678895375398, + "pass@32": 0.9970388159542295, + "pass@64": 0.9983478337657014, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.5314875, + "avg_response_len": 351.76516015625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5314875, + "pass@2": 0.6022682824803063, + "pass@4": 0.6591659706880293, + "pass@8": 0.7050961345515291, + "pass@16": 0.7429625058318009, + "pass@32": 0.7753246502977222, + "pass@64": 0.8030656828422016, + "pass@128": 0.8261 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9324765625, + "pass@2": 0.9533142224409449, + "pass@4": 0.9647201501687288, + "pass@8": 0.9716220664213624, + "pass@16": 0.976113902180434, + "pass@32": 0.9794617764117818, + "pass@64": 0.9824683910904393, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.887390625, + "pass@2": 0.9206886072834655, + "pass@4": 0.9376154036370457, + "pass@8": 0.9486329455856432, + "pass@16": 0.9569428807974291, + "pass@32": 0.9633673292075392, + "pass@64": 0.968846758845277, + "pass@128": 0.974 + }, + "13": { + "pass@1": 0.7620703125, + "pass@2": 0.8222394192913397, + "pass@4": 0.8591277995875507, + "pass@8": 0.8854334828073739, + "pass@16": 0.9066821609763444, + "pass@32": 0.925073595662466, + "pass@64": 0.9415350592744647, + "pass@128": 0.955 + }, + "14": { + "pass@1": 0.701, + "pass@2": 0.7807849409448818, + "pass@4": 0.832770978908886, + "pass@8": 0.86770836585606, + "pass@16": 0.8925915209483687, + "pass@32": 0.9113136378973459, + "pass@64": 0.9257772846219111, + "pass@128": 0.937 + }, + "15": { + "pass@1": 0.603734375, + "pass@2": 0.6913080708661427, + "pass@4": 0.7554190513685792, + "pass@8": 0.8031779786366131, + "pass@16": 0.8383422442454967, + "pass@32": 0.8645934027916754, + "pass@64": 0.8849775249687437, + "pass@128": 0.902 + }, + "16": { + "pass@1": 0.421171875, + "pass@2": 0.5247765748031502, + "pass@4": 0.6098393095238099, + "pass@8": 0.6755907587075335, + "pass@16": 0.7269389528502289, + "pass@32": 0.7697787249307066, + "pass@64": 0.8058135993617338, + "pass@128": 0.833 + }, + "17": { + "pass@1": 0.3317109375, + "pass@2": 0.42685334645669337, + "pass@4": 0.5099146227971499, + "pass@8": 0.5782730292294023, + "pass@16": 0.6352742989388074, + "pass@32": 0.686522191097258, + "pass@64": 0.7313242064501279, + "pass@128": 0.765 + }, + "18": { + "pass@1": 0.3008359375, + "pass@2": 0.38705302657480345, + "pass@4": 0.4653752832770899, + "pass@8": 0.5326123127775788, + "pass@16": 0.5900990793779466, + "pass@32": 0.6406784031888708, + "pass@64": 0.6850894214131005, + "pass@128": 0.725 + }, + "19": { + "pass@1": 0.1969375, + "pass@2": 0.2683048720472441, + "pass@4": 0.33789413226471676, + "pass@8": 0.40227891800683, + "pass@16": 0.4607638000638037, + "pass@32": 0.51352503178184, + "pass@64": 0.5616739114507694, + "pass@128": 0.604 + }, + "20": { + "pass@1": 0.177546875, + "pass@2": 0.2473597440944882, + "pass@4": 0.3189829753468313, + "pass@8": 0.38563148748692394, + "pass@16": 0.4458762179390442, + "pass@32": 0.4989324100077299, + "pass@64": 0.5431506709453723, + "pass@128": 0.58 + } + }, + "per_op_accuracy": { + "11": 0.9324765625, + "12": 0.887390625, + "13": 0.7620703125, + "14": 0.701, + "15": 0.603734375, + "16": 0.421171875, + "17": 0.3317109375, + "18": 0.3008359375, + "19": 0.1969375, + "20": 0.177546875 + }, + "per_op_avg_response_len": { + "11": 298.6321328125, + "12": 309.881109375, + "13": 327.547625, + "14": 349.168734375, + "15": 363.7685234375, + "16": 359.0778984375, + "17": 375.777734375, + "18": 380.173578125, + "19": 370.895421875, + "20": 382.72884375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.6321328125, + "12": 309.881109375, + "13": 327.547625, + "14": 349.168734375, + "15": 363.7685234375, + "16": 359.0778984375, + "17": 375.777734375, + "18": 380.173578125, + "19": 370.895421875, + "20": 382.72884375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 230194, + "answer_accuracy": 0.5344400074294205, + "avg_response_len": 333.67903510401186, + "resp_tokens_sum": 143722234, + "pass_at_k": { + "pass@1": 0.5344400074294205, + "pass@2": 0.6035592116039382, + "pass@4": 0.6583647019234046, + "pass@8": 0.7021859925548064, + "pass@16": 0.7387123981798346, + "pass@32": 0.7706893337971354, + "pass@64": 0.7985455630917165, + "pass@128": 0.8222882615156017 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9276071175637394, + "pass@2": 0.948805304922933, + "pass@4": 0.9625242864471968, + "pass@8": 0.9718624532312413, + "pass@16": 0.9782309268492891, + "pass@32": 0.9824962329796952, + "pass@64": 0.9854704070111471, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8833862994350282, + "pass@2": 0.919130772053917, + "pass@4": 0.9369724676894196, + "pass@8": 0.949119166761525, + "pass@16": 0.9589524280731624, + "pass@32": 0.9663844678408191, + "pass@64": 0.9727848261772267, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.77953565830721, + "pass@2": 0.8406290881939132, + "pass@4": 0.8766417460944336, + "pass@8": 0.9007702961787816, + "pass@16": 0.9203564319065582, + "pass@32": 0.9373645301631106, + "pass@64": 0.9519000597968401, + "pass@128": 0.9655172413793104 + }, + "14": { + "pass@1": 0.726927570093458, + "pass@2": 0.8062917003949269, + "pass@4": 0.8548241513619205, + "pass@8": 0.8863406152747126, + "pass@16": 0.9104425624453125, + "pass@32": 0.9293280986882642, + "pass@64": 0.9439192226652879, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.6221324060693642, + "pass@2": 0.7063558213099081, + "pass@4": 0.7662719325907954, + "pass@8": 0.8111031131548001, + "pass@16": 0.8462265090737875, + "pass@32": 0.8747750472498352, + "pass@64": 0.8970795874266666, + "pass@128": 0.9161849710982659 + }, + "16": { + "pass@1": 0.4443296370967742, + "pass@2": 0.547864014478029, + "pass@4": 0.6275922671964388, + "pass@8": 0.6861379453842721, + "pass@16": 0.7316585228509355, + "pass@32": 0.7729760509118466, + "pass@64": 0.812534223224478, + "pass@128": 0.8483870967741935 + }, + "17": { + "pass@1": 0.3257541546242775, + "pass@2": 0.4236906035228256, + "pass@4": 0.5111456820245731, + "pass@8": 0.5823636363388676, + "pass@16": 0.6392486711470267, + "pass@32": 0.6881428453312798, + "pass@64": 0.7286531028102093, + "pass@128": 0.7572254335260116 + }, + "18": { + "pass@1": 0.26602678571428573, + "pass@2": 0.34801216254218215, + "pass@4": 0.4255049338475545, + "pass@8": 0.4947810451410119, + "pass@16": 0.55549192818462, + "pass@32": 0.6101924953252681, + "pass@64": 0.658631555139458, + "pass@128": 0.7 + }, + "19": { + "pass@1": 0.18520082582582584, + "pass@2": 0.2561013277056583, + "pass@4": 0.32643228098739907, + "pass@8": 0.39033346408424197, + "pass@16": 0.4477674798841049, + "pass@32": 0.4997264097323752, + "pass@64": 0.5464207467355886, + "pass@128": 0.5855855855855856 + }, + "20": { + "pass@1": 0.16732357357357358, + "pass@2": 0.2275554343713792, + "pass@4": 0.28745300840773264, + "pass@8": 0.3421696822977434, + "pass@16": 0.39285128747771964, + "pass@32": 0.4406795710163925, + "pass@64": 0.4845239493330482, + "pass@128": 0.5225225225225225 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 228448, + "answer_accuracy": 0.5307017543859649, + "avg_response_len": 348.2110025460898, + "resp_tokens_sum": 149892301, + "pass_at_k": { + "pass@1": 0.5307017543859649, + "pass@2": 0.6027607857977414, + "pass@4": 0.6611699793658727, + "pass@8": 0.7089237533553516, + "pass@16": 0.7483928661324409, + "pass@32": 0.781609045904677, + "pass@64": 0.809569957323907, + "pass@128": 0.8322925958965209 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9410062123493976, + "pass@2": 0.958402384142871, + "pass@4": 0.9662167536851564, + "pass@8": 0.9709692525310544, + "pass@16": 0.9742474397487791, + "pass@32": 0.9771765082319408, + "pass@64": 0.9803255550853777, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8839880485893417, + "pass@2": 0.9137333232542638, + "pass@4": 0.9281012561517581, + "pass@8": 0.9354493476435397, + "pass@16": 0.9396505982108377, + "pass@32": 0.9434898827285836, + "pass@64": 0.9480917933676873, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7396271008403361, + "pass@2": 0.7994693448245437, + "pass@4": 0.8373948400882663, + "pass@8": 0.8673476898330499, + "pass@16": 0.8936367585473864, + "pass@32": 0.9163371027271825, + "pass@64": 0.9352437459860121, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.6859237938596491, + "pass@2": 0.7680046881475341, + "pass@4": 0.8229837292816459, + "pass@8": 0.8614744075315885, + "pass@16": 0.888244901857619, + "pass@32": 0.9069925793938199, + "pass@64": 0.920712589392341, + "pass@128": 0.9298245614035088 + }, + "15": { + "pass@1": 0.592779356060606, + "pass@2": 0.6902823759246002, + "pass@4": 0.7625650600493118, + "pass@8": 0.8162019416966678, + "pass@16": 0.8545200049180915, + "pass@32": 0.8813196398182785, + "pass@64": 0.9009509702228317, + "pass@128": 0.9151515151515152 + }, + "16": { + "pass@1": 0.419315349002849, + "pass@2": 0.5241369568387281, + "pass@4": 0.6076210393572596, + "pass@8": 0.6690281184725626, + "pass@16": 0.7159874640286359, + "pass@32": 0.7552395900040212, + "pass@64": 0.7868517864460535, + "pass@128": 0.8091168091168092 + }, + "17": { + "pass@1": 0.3528484083850932, + "pass@2": 0.4481074546388222, + "pass@4": 0.5321736775333203, + "pass@8": 0.6030474903401737, + "pass@16": 0.6627250939917279, + "pass@32": 0.7157933575690838, + "pass@64": 0.7650930743179017, + "pass@128": 0.8043478260869565 + }, + "18": { + "pass@1": 0.32125459558823527, + "pass@2": 0.411982905859194, + "pass@4": 0.49527306485586303, + "pass@8": 0.5687953540170917, + "pass@16": 0.6336849247595163, + "pass@32": 0.6896902244753337, + "pass@64": 0.7332202055305614, + "pass@128": 0.7647058823529411 + }, + "19": { + "pass@1": 0.20452008928571427, + "pass@2": 0.27103216101893507, + "pass@4": 0.3350326398932277, + "pass@8": 0.3956876057484762, + "pass@16": 0.4526802546976231, + "pass@32": 0.5058468809363461, + "pass@64": 0.557456349702467, + "pass@128": 0.6071428571428571 + }, + "20": { + "pass@1": 0.17171126497005987, + "pass@2": 0.2459550632985996, + "pass@4": 0.32637963022212046, + "pass@8": 0.4023198144270265, + "pass@16": 0.46881532030483886, + "pass@32": 0.5238391423437389, + "pass@64": 0.5672146906934317, + "pass@128": 0.6047904191616766 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 221662, + "answer_accuracy": 0.5292586720660146, + "avg_response_len": 374.0183517344132, + "resp_tokens_sum": 156644870, + "pass_at_k": { + "pass@1": 0.5292586720660146, + "pass@2": 0.6004344606106715, + "pass@4": 0.6579302702630552, + "pass@8": 0.7041549199371585, + "pass@16": 0.7417520261734117, + "pass@32": 0.7736325407617366, + "pass@64": 0.8010291082328783, + "pass@128": 0.8236552567237164 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9289434523809523, + "pass@2": 0.9530043119610053, + "pass@4": 0.9656035391409405, + "pass@8": 0.9720407256838862, + "pass@16": 0.9757086825588593, + "pass@32": 0.9784698585299846, + "pass@64": 0.9813627083401868, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8950449159021406, + "pass@2": 0.9291601952852221, + "pass@4": 0.9475928115063601, + "pass@8": 0.960967641143043, + "pass@16": 0.9716366374014452, + "pass@32": 0.9794922171298824, + "pass@64": 0.9848307287285822, + "pass@128": 0.9877675840978594 + }, + "13": { + "pass@1": 0.7696035879629629, + "pass@2": 0.8292227902935744, + "pass@4": 0.8658306317034441, + "pass@8": 0.89026121313563, + "pass@16": 0.9075930135701733, + "pass@32": 0.9225986569963891, + "pass@64": 0.9382621076613159, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.6916033011869436, + "pass@2": 0.7694590557022359, + "pass@4": 0.8216973023958051, + "pass@8": 0.8562871839082323, + "pass@16": 0.8799991156323995, + "pass@32": 0.8985396322425133, + "pass@64": 0.91363652395897, + "pass@128": 0.9258160237388724 + }, + "15": { + "pass@1": 0.5952449845679012, + "pass@2": 0.6762832488820844, + "pass@4": 0.736550903937933, + "pass@8": 0.7814495701393551, + "pass@16": 0.8134452792685088, + "pass@32": 0.836684429948149, + "pass@64": 0.8557844677332779, + "pass@128": 0.8734567901234568 + }, + "16": { + "pass@1": 0.4019174041297935, + "pass@2": 0.504326426149165, + "pass@4": 0.5959018344498971, + "pass@8": 0.6727407848216501, + "pass@16": 0.7339622740188452, + "pass@32": 0.7819087110814815, + "pass@64": 0.8193009236565792, + "pass@128": 0.8436578171091446 + }, + "17": { + "pass@1": 0.31741810993975905, + "pass@2": 0.40953553386775415, + "pass@4": 0.48704305015336896, + "pass@8": 0.5499816842368038, + "pass@16": 0.604508368845186, + "pass@32": 0.6564436911909359, + "pass@64": 0.7013562136970772, + "pass@128": 0.7349397590361446 + }, + "18": { + "pass@1": 0.317741935483871, + "pass@2": 0.40378897320294627, + "pass@4": 0.47759907864339524, + "pass@8": 0.5356404084271387, + "pass@16": 0.581367839016435, + "pass@32": 0.6213430758174604, + "pass@64": 0.6621726039803202, + "pass@128": 0.7096774193548387 + }, + "19": { + "pass@1": 0.20104796072507553, + "pass@2": 0.277813667126579, + "pass@4": 0.3523299567727752, + "pass@8": 0.42098742880752116, + "pass@16": 0.48204430116010694, + "pass@32": 0.5352011641883591, + "pass@64": 0.5813005114434726, + "pass@128": 0.6193353474320241 + }, + "20": { + "pass@1": 0.1936233108108108, + "pass@2": 0.268572952874134, + "pass@4": 0.34309407523383884, + "pass@8": 0.412354850640087, + "pass@16": 0.4758931598412825, + "pass@32": 0.5322037216710581, + "pass@64": 0.5776411084860688, + "pass@128": 0.6126126126126126 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7468433388157895, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7468433388157895, + "pass@2": 0.7859592895254661, + "pass@4": 0.8169818794426844, + "pass@8": 0.8419437503690984, + "pass@16": 0.8625329205269855, + "pass@32": 0.880141315679784, + "pass@64": 0.8952105798016025, + "pass@128": 0.9076315789473685 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9413671875, + "pass@2": 0.9560073818897641, + "pass@4": 0.9661297797150363, + "pass@8": 0.973958866476777, + "pass@16": 0.9801564851612712, + "pass@32": 0.9853814436253552, + "pass@64": 0.9896901293154565, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.999984375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999359375, + "pass@2": 0.999675812007874, + "pass@4": 0.9998979723472066, + "pass@8": 0.9999905979568364, + "pass@16": 0.9999999435227294, + "pass@32": 0.9999999999996563, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9974140625, + "pass@2": 0.998044783464567, + "pass@4": 0.998237083239595, + "pass@8": 0.9984480653393176, + "pass@16": 0.9987941264550821, + "pass@32": 0.9992680481403238, + "pass@64": 0.9997381448050702, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9842578125, + "pass@2": 0.9879093257874021, + "pass@4": 0.9901990563367077, + "pass@8": 0.9922341196998229, + "pass@16": 0.9942029297347884, + "pass@32": 0.9957719067641455, + "pass@64": 0.9967520479354316, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.994, + "pass@2": 0.9949153543307088, + "pass@4": 0.9953240142482191, + "pass@8": 0.9956690460023842, + "pass@16": 0.9961843259195156, + "pass@32": 0.9968943403441024, + "pass@64": 0.9976233493374305, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.990328125, + "pass@2": 0.9936660925196853, + "pass@4": 0.9954954812523434, + "pass@8": 0.9966721388643296, + "pass@16": 0.9974260781790963, + "pass@32": 0.9980104268701047, + "pass@64": 0.9986299091903094, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.986390625, + "pass@2": 0.9911269685039372, + "pass@4": 0.993347476752906, + "pass@8": 0.9948517012143449, + "pass@16": 0.9962994679373544, + "pass@32": 0.9976768848650488, + "pass@64": 0.9986739346860647, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.982046875, + "pass@2": 0.9891979576771657, + "pass@4": 0.9923651386389201, + "pass@8": 0.9941453759435105, + "pass@16": 0.9954370747847234, + "pass@32": 0.996435444329734, + "pass@64": 0.9972366725387218, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.9324765625, + "pass@2": 0.9533142224409449, + "pass@4": 0.9647201501687288, + "pass@8": 0.9716220664213624, + "pass@16": 0.976113902180434, + "pass@32": 0.9794617764117818, + "pass@64": 0.9824683910904393, + "pass@128": 0.986 + }, + "12": { + "pass@1": 0.887390625, + "pass@2": 0.9206886072834655, + "pass@4": 0.9376154036370457, + "pass@8": 0.9486329455856432, + "pass@16": 0.9569428807974291, + "pass@32": 0.9633673292075392, + "pass@64": 0.968846758845277, + "pass@128": 0.974 + }, + "13": { + "pass@1": 0.7620703125, + "pass@2": 0.8222394192913397, + "pass@4": 0.8591277995875507, + "pass@8": 0.8854334828073739, + "pass@16": 0.9066821609763444, + "pass@32": 0.925073595662466, + "pass@64": 0.9415350592744647, + "pass@128": 0.955 + }, + "14": { + "pass@1": 0.701, + "pass@2": 0.7807849409448818, + "pass@4": 0.832770978908886, + "pass@8": 0.86770836585606, + "pass@16": 0.8925915209483687, + "pass@32": 0.9113136378973459, + "pass@64": 0.9257772846219111, + "pass@128": 0.937 + }, + "15": { + "pass@1": 0.603734375, + "pass@2": 0.6913080708661427, + "pass@4": 0.7554190513685792, + "pass@8": 0.8031779786366131, + "pass@16": 0.8383422442454967, + "pass@32": 0.8645934027916754, + "pass@64": 0.8849775249687437, + "pass@128": 0.902 + }, + "16": { + "pass@1": 0.421171875, + "pass@2": 0.5247765748031502, + "pass@4": 0.6098393095238099, + "pass@8": 0.6755907587075335, + "pass@16": 0.7269389528502289, + "pass@32": 0.7697787249307066, + "pass@64": 0.8058135993617338, + "pass@128": 0.833 + }, + "17": { + "pass@1": 0.3317109375, + "pass@2": 0.42685334645669337, + "pass@4": 0.5099146227971499, + "pass@8": 0.5782730292294023, + "pass@16": 0.6352742989388074, + "pass@32": 0.686522191097258, + "pass@64": 0.7313242064501279, + "pass@128": 0.765 + }, + "18": { + "pass@1": 0.3008359375, + "pass@2": 0.38705302657480345, + "pass@4": 0.4653752832770899, + "pass@8": 0.5326123127775788, + "pass@16": 0.5900990793779466, + "pass@32": 0.6406784031888708, + "pass@64": 0.6850894214131005, + "pass@128": 0.725 + }, + "19": { + "pass@1": 0.1969375, + "pass@2": 0.2683048720472441, + "pass@4": 0.33789413226471676, + "pass@8": 0.40227891800683, + "pass@16": 0.4607638000638037, + "pass@32": 0.51352503178184, + "pass@64": 0.5616739114507694, + "pass@128": 0.604 + }, + "20": { + "pass@1": 0.177546875, + "pass@2": 0.2473597440944882, + "pass@4": 0.3189829753468313, + "pass@8": 0.38563148748692394, + "pass@16": 0.4458762179390442, + "pass@32": 0.4989324100077299, + "pass@64": 0.5431506709453723, + "pass@128": 0.58 + } + }, + "per_op_accuracy": { + "10": 0.9413671875, + "2": 0.999984375, + "3": 0.999359375, + "4": 0.9974140625, + "5": 0.9842578125, + "6": 0.994, + "7": 0.990328125, + "8": 0.986390625, + "9": 0.982046875, + "11": 0.9324765625, + "12": 0.887390625, + "13": 0.7620703125, + "14": 0.701, + "15": 0.603734375, + "16": 0.421171875, + "17": 0.3317109375, + "18": 0.3008359375, + "19": 0.1969375, + "20": 0.177546875 + }, + "per_op_avg_response_len": { + "10": 280.5143671875, + "2": 115.610421875, + "3": 144.835296875, + "4": 160.095046875, + "5": 174.0822578125, + "6": 199.3134609375, + "7": 224.143984375, + "8": 243.89225, + "9": 259.6180546875, + "11": 298.6321328125, + "12": 309.881109375, + "13": 327.547625, + "14": 349.168734375, + "15": 363.7685234375, + "16": 359.0778984375, + "17": 375.777734375, + "18": 380.173578125, + "19": 370.895421875, + "20": 382.72884375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.5143671875, + "2": 115.610421875, + "3": 144.835296875, + "4": 160.095046875, + "5": 174.0822578125, + "6": 199.3134609375, + "7": 224.143984375, + "8": 243.89225, + "9": 259.6180546875, + "11": 298.6321328125, + "12": 309.881109375, + "13": 327.547625, + "14": 349.168734375, + "15": 363.7685234375, + "16": 359.0778984375, + "17": 375.777734375, + "18": 380.173578125, + "19": 370.895421875, + "20": 382.72884375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 603712, + "answer_accuracy": 0.7455738223205818, + "avg_response_len": 266.78108204236486, + "resp_tokens_sum": 216020112, + "pass_at_k": { + "pass@1": 0.7455738223205818, + "pass@2": 0.7842754740154935, + "pass@4": 0.8145981928960141, + "pass@8": 0.8387537404175, + "pass@16": 0.8588352159822145, + "pass@32": 0.8764292483559714, + "pass@64": 0.8917570475437393, + "pass@128": 0.9046791021182422 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9390440634218289, + "pass@2": 0.9555319170557223, + "pass@4": 0.9674239525369062, + "pass@8": 0.9756565404139677, + "pass@16": 0.9813557770331947, + "pass@32": 0.986617282494939, + "pass@64": 0.9912821110885033, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9938720703125, + "pass@2": 0.9941333200049213, + "pass@4": 0.9944950684875327, + "pass@8": 0.9951502049040684, + "pass@16": 0.9962316451721318, + "pass@32": 0.9977126504385122, + "pass@64": 0.999181702515844, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9853422619047619, + "pass@2": 0.9885260436195474, + "pass@4": 0.9904835181316618, + "pass@8": 0.9919887051387225, + "pass@16": 0.9930441900209205, + "pass@32": 0.9935671832417211, + "pass@64": 0.9936503542147976, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9926553143712575, + "pass@2": 0.9937994336130889, + "pass@4": 0.9941684698656681, + "pass@8": 0.9943757134349077, + "pass@16": 0.9947162761058925, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9921181582840237, + "pass@2": 0.9955388342729348, + "pass@4": 0.9968490602875822, + "pass@8": 0.9975059751526393, + "pass@16": 0.9980238469616003, + "pass@32": 0.9987617116948309, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9820462740384616, + "pass@2": 0.9870359504340805, + "pass@4": 0.990314350730197, + "pass@8": 0.9933464239381959, + "pass@16": 0.9958232946543495, + "pass@32": 0.9976443884867328, + "pass@64": 0.9991607192427966, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9861575704225352, + "pass@2": 0.9930700343795054, + "pass@4": 0.9960230070712991, + "pass@8": 0.9978186757417113, + "pass@16": 0.9990916543037955, + "pass@32": 0.9998066566382701, + "pass@64": 0.9999959302892975, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9276071175637394, + "pass@2": 0.948805304922933, + "pass@4": 0.9625242864471968, + "pass@8": 0.9718624532312413, + "pass@16": 0.9782309268492891, + "pass@32": 0.9824962329796952, + "pass@64": 0.9854704070111471, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8833862994350282, + "pass@2": 0.919130772053917, + "pass@4": 0.9369724676894196, + "pass@8": 0.949119166761525, + "pass@16": 0.9589524280731624, + "pass@32": 0.9663844678408191, + "pass@64": 0.9727848261772267, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.77953565830721, + "pass@2": 0.8406290881939132, + "pass@4": 0.8766417460944336, + "pass@8": 0.9007702961787816, + "pass@16": 0.9203564319065582, + "pass@32": 0.9373645301631106, + "pass@64": 0.9519000597968401, + "pass@128": 0.9655172413793104 + }, + "14": { + "pass@1": 0.726927570093458, + "pass@2": 0.8062917003949269, + "pass@4": 0.8548241513619205, + "pass@8": 0.8863406152747126, + "pass@16": 0.9104425624453125, + "pass@32": 0.9293280986882642, + "pass@64": 0.9439192226652879, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.6221324060693642, + "pass@2": 0.7063558213099081, + "pass@4": 0.7662719325907954, + "pass@8": 0.8111031131548001, + "pass@16": 0.8462265090737875, + "pass@32": 0.8747750472498352, + "pass@64": 0.8970795874266666, + "pass@128": 0.9161849710982659 + }, + "16": { + "pass@1": 0.4443296370967742, + "pass@2": 0.547864014478029, + "pass@4": 0.6275922671964388, + "pass@8": 0.6861379453842721, + "pass@16": 0.7316585228509355, + "pass@32": 0.7729760509118466, + "pass@64": 0.812534223224478, + "pass@128": 0.8483870967741935 + }, + "17": { + "pass@1": 0.3257541546242775, + "pass@2": 0.4236906035228256, + "pass@4": 0.5111456820245731, + "pass@8": 0.5823636363388676, + "pass@16": 0.6392486711470267, + "pass@32": 0.6881428453312798, + "pass@64": 0.7286531028102093, + "pass@128": 0.7572254335260116 + }, + "18": { + "pass@1": 0.26602678571428573, + "pass@2": 0.34801216254218215, + "pass@4": 0.4255049338475545, + "pass@8": 0.4947810451410119, + "pass@16": 0.55549192818462, + "pass@32": 0.6101924953252681, + "pass@64": 0.658631555139458, + "pass@128": 0.7 + }, + "19": { + "pass@1": 0.18520082582582584, + "pass@2": 0.2561013277056583, + "pass@4": 0.32643228098739907, + "pass@8": 0.39033346408424197, + "pass@16": 0.4477674798841049, + "pass@32": 0.4997264097323752, + "pass@64": 0.5464207467355886, + "pass@128": 0.5855855855855856 + }, + "20": { + "pass@1": 0.16732357357357358, + "pass@2": 0.2275554343713792, + "pass@4": 0.28745300840773264, + "pass@8": 0.3421696822977434, + "pass@16": 0.39285128747771964, + "pass@32": 0.4406795710163925, + "pass@64": 0.4845239493330482, + "pass@128": 0.5225225225225225 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 613909, + "answer_accuracy": 0.7476483339828527, + "avg_response_len": 276.8695196804365, + "resp_tokens_sum": 227343100, + "pass_at_k": { + "pass@1": 0.7476483339828527, + "pass@2": 0.7869729303858387, + "pass@4": 0.8185048834065208, + "pass@8": 0.8442728089335293, + "pass@16": 0.8656295025843366, + "pass@32": 0.8836184122445357, + "pass@64": 0.8987557769119414, + "pass@128": 0.9109898674980514 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9318682898773006, + "pass@2": 0.9437727193130767, + "pass@4": 0.9541319070162242, + "pass@8": 0.9643895337030387, + "pass@16": 0.9734856182486636, + "pass@32": 0.9804536494144921, + "pass@64": 0.9859565813477191, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999289772727272, + "pass@2": 0.9999996271772846, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998543823964497, + "pass@2": 0.9997983448259796, + "pass@4": 0.9999962377114103, + "pass@8": 0.9999999993222367, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906929347826087, + "pass@2": 0.9938455722925938, + "pass@4": 0.99476957146661, + "pass@8": 0.9953830646909647, + "pass@16": 0.9963772680388042, + "pass@32": 0.9978842427305282, + "pass@64": 0.9994736049298187, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967541189111748, + "pass@2": 0.9971184543013785, + "pass@4": 0.9971346140829818, + "pass@8": 0.9971346704870158, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9941067557803468, + "pass@2": 0.9960366790314508, + "pass@4": 0.996919753256999, + "pass@8": 0.9971032378641861, + "pass@16": 0.9971098202636923, + "pass@32": 0.9971098265895949, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.988556338028169, + "pass@2": 0.99323742652767, + "pass@4": 0.9950167355841083, + "pass@8": 0.9959384773710429, + "pass@16": 0.997143266424544, + "pass@32": 0.9986983950949794, + "pass@64": 0.9998376400133082, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9778255988023952, + "pass@2": 0.9839705431656373, + "pass@4": 0.9873886173098122, + "pass@8": 0.9896570716478766, + "pass@16": 0.9912446822176693, + "pass@32": 0.9922904474891701, + "pass@64": 0.9932692843858927, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9410062123493976, + "pass@2": 0.958402384142871, + "pass@4": 0.9662167536851564, + "pass@8": 0.9709692525310544, + "pass@16": 0.9742474397487791, + "pass@32": 0.9771765082319408, + "pass@64": 0.9803255550853777, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8839880485893417, + "pass@2": 0.9137333232542638, + "pass@4": 0.9281012561517581, + "pass@8": 0.9354493476435397, + "pass@16": 0.9396505982108377, + "pass@32": 0.9434898827285836, + "pass@64": 0.9480917933676873, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7396271008403361, + "pass@2": 0.7994693448245437, + "pass@4": 0.8373948400882663, + "pass@8": 0.8673476898330499, + "pass@16": 0.8936367585473864, + "pass@32": 0.9163371027271825, + "pass@64": 0.9352437459860121, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.6859237938596491, + "pass@2": 0.7680046881475341, + "pass@4": 0.8229837292816459, + "pass@8": 0.8614744075315885, + "pass@16": 0.888244901857619, + "pass@32": 0.9069925793938199, + "pass@64": 0.920712589392341, + "pass@128": 0.9298245614035088 + }, + "15": { + "pass@1": 0.592779356060606, + "pass@2": 0.6902823759246002, + "pass@4": 0.7625650600493118, + "pass@8": 0.8162019416966678, + "pass@16": 0.8545200049180915, + "pass@32": 0.8813196398182785, + "pass@64": 0.9009509702228317, + "pass@128": 0.9151515151515152 + }, + "16": { + "pass@1": 0.419315349002849, + "pass@2": 0.5241369568387281, + "pass@4": 0.6076210393572596, + "pass@8": 0.6690281184725626, + "pass@16": 0.7159874640286359, + "pass@32": 0.7552395900040212, + "pass@64": 0.7868517864460535, + "pass@128": 0.8091168091168092 + }, + "17": { + "pass@1": 0.3528484083850932, + "pass@2": 0.4481074546388222, + "pass@4": 0.5321736775333203, + "pass@8": 0.6030474903401737, + "pass@16": 0.6627250939917279, + "pass@32": 0.7157933575690838, + "pass@64": 0.7650930743179017, + "pass@128": 0.8043478260869565 + }, + "18": { + "pass@1": 0.32125459558823527, + "pass@2": 0.411982905859194, + "pass@4": 0.49527306485586303, + "pass@8": 0.5687953540170917, + "pass@16": 0.6336849247595163, + "pass@32": 0.6896902244753337, + "pass@64": 0.7332202055305614, + "pass@128": 0.7647058823529411 + }, + "19": { + "pass@1": 0.20452008928571427, + "pass@2": 0.27103216101893507, + "pass@4": 0.3350326398932277, + "pass@8": 0.3956876057484762, + "pass@16": 0.4526802546976231, + "pass@32": 0.5058468809363461, + "pass@64": 0.557456349702467, + "pass@128": 0.6071428571428571 + }, + "20": { + "pass@1": 0.17171126497005987, + "pass@2": 0.2459550632985996, + "pass@4": 0.32637963022212046, + "pass@8": 0.4023198144270265, + "pass@16": 0.46881532030483886, + "pass@32": 0.5238391423437389, + "pass@64": 0.5672146906934317, + "pass@128": 0.6047904191616766 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 598702, + "answer_accuracy": 0.747301386004154, + "avg_response_len": 296.53006046293336, + "resp_tokens_sum": 237565651, + "pass_at_k": { + "pass@1": 0.747301386004154, + "pass@2": 0.7866222246264524, + "pass@4": 0.817830118884518, + "pass@8": 0.8427807997799793, + "pass@16": 0.8630964458588545, + "pass@32": 0.8803293590457137, + "pass@64": 0.8950675226999064, + "pass@128": 0.9071736699153219 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9529617537313433, + "pass@2": 0.9683944940651077, + "pass@4": 0.9764956958738361, + "pass@8": 0.9815531680276173, + "pass@16": 0.9854345229789683, + "pass@32": 0.9889262541804372, + "pass@64": 0.9917123824987969, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982233965014577, + "pass@2": 0.9990552041964142, + "pass@4": 0.9997025432863167, + "pass@8": 0.9999725887954414, + "pass@16": 0.9999998353432346, + "pass@32": 0.9999999999989982, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996116593567251, + "pass@2": 0.9999715804438919, + "pass@4": 0.9999998040266896, + "pass@8": 0.9999999999973679, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9767233455882353, + "pass@2": 0.9813144106067622, + "pass@4": 0.9852977822625111, + "pass@8": 0.9892662360668899, + "pass@16": 0.9930701541317976, + "pass@32": 0.9956711185322674, + "pass@64": 0.9968640960499501, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9923846608832808, + "pass@2": 0.9936656244566434, + "pass@4": 0.9945481545682183, + "pass@8": 0.9954181631392943, + "pass@16": 0.9966848255525158, + "pass@32": 0.9982819052559925, + "pass@64": 0.9995942613043955, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9842761075949367, + "pass@2": 0.9890673278181997, + "pass@4": 0.992488178000693, + "pass@8": 0.9953082245624335, + "pass@16": 0.997132974983664, + "pass@32": 0.9981929377128227, + "pass@64": 0.9992150520076071, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9881522147147147, + "pass@2": 0.9927101067603032, + "pass@4": 0.9944097843625403, + "pass@8": 0.995103475912635, + "pass@16": 0.9958460673407935, + "pass@32": 0.9966183345299415, + "pass@64": 0.9969772614942578, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9818880627009646, + "pass@2": 0.9903920709167786, + "pass@4": 0.9935343181579793, + "pass@8": 0.9947726177646687, + "pass@16": 0.9957678895375398, + "pass@32": 0.9970388159542295, + "pass@64": 0.9983478337657014, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9289434523809523, + "pass@2": 0.9530043119610053, + "pass@4": 0.9656035391409405, + "pass@8": 0.9720407256838862, + "pass@16": 0.9757086825588593, + "pass@32": 0.9784698585299846, + "pass@64": 0.9813627083401868, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8950449159021406, + "pass@2": 0.9291601952852221, + "pass@4": 0.9475928115063601, + "pass@8": 0.960967641143043, + "pass@16": 0.9716366374014452, + "pass@32": 0.9794922171298824, + "pass@64": 0.9848307287285822, + "pass@128": 0.9877675840978594 + }, + "13": { + "pass@1": 0.7696035879629629, + "pass@2": 0.8292227902935744, + "pass@4": 0.8658306317034441, + "pass@8": 0.89026121313563, + "pass@16": 0.9075930135701733, + "pass@32": 0.9225986569963891, + "pass@64": 0.9382621076613159, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.6916033011869436, + "pass@2": 0.7694590557022359, + "pass@4": 0.8216973023958051, + "pass@8": 0.8562871839082323, + "pass@16": 0.8799991156323995, + "pass@32": 0.8985396322425133, + "pass@64": 0.91363652395897, + "pass@128": 0.9258160237388724 + }, + "15": { + "pass@1": 0.5952449845679012, + "pass@2": 0.6762832488820844, + "pass@4": 0.736550903937933, + "pass@8": 0.7814495701393551, + "pass@16": 0.8134452792685088, + "pass@32": 0.836684429948149, + "pass@64": 0.8557844677332779, + "pass@128": 0.8734567901234568 + }, + "16": { + "pass@1": 0.4019174041297935, + "pass@2": 0.504326426149165, + "pass@4": 0.5959018344498971, + "pass@8": 0.6727407848216501, + "pass@16": 0.7339622740188452, + "pass@32": 0.7819087110814815, + "pass@64": 0.8193009236565792, + "pass@128": 0.8436578171091446 + }, + "17": { + "pass@1": 0.31741810993975905, + "pass@2": 0.40953553386775415, + "pass@4": 0.48704305015336896, + "pass@8": 0.5499816842368038, + "pass@16": 0.604508368845186, + "pass@32": 0.6564436911909359, + "pass@64": 0.7013562136970772, + "pass@128": 0.7349397590361446 + }, + "18": { + "pass@1": 0.317741935483871, + "pass@2": 0.40378897320294627, + "pass@4": 0.47759907864339524, + "pass@8": 0.5356404084271387, + "pass@16": 0.581367839016435, + "pass@32": 0.6213430758174604, + "pass@64": 0.6621726039803202, + "pass@128": 0.7096774193548387 + }, + "19": { + "pass@1": 0.20104796072507553, + "pass@2": 0.277813667126579, + "pass@4": 0.3523299567727752, + "pass@8": 0.42098742880752116, + "pass@16": 0.48204430116010694, + "pass@32": 0.5352011641883591, + "pass@64": 0.5813005114434726, + "pass@128": 0.6193353474320241 + }, + "20": { + "pass@1": 0.1936233108108108, + "pass@2": 0.268572952874134, + "pass@4": 0.34309407523383884, + "pass@8": 0.412354850640087, + "pass@16": 0.4758931598412825, + "pass@32": 0.5322037216710581, + "pass@64": 0.5776411084860688, + "pass@128": 0.6126126126126126 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch1-op9-12/graphs/SDAR-100M-block4-epoch1-op9-12_score_report.jsonl b/composition/SDAR-100M-block4-epoch1-op9-12/graphs/SDAR-100M-block4-epoch1-op9-12_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fc02e908e507c00835d0429d8435c2ba8d940b85 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op9-12/graphs/SDAR-100M-block4-epoch1-op9-12_score_report.jsonl @@ -0,0 +1,2 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch1-op9-12", "results_dir": "results/composition/SDAR-100M-block4-epoch1-op9-12/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1011475, "value_mismatch": 19873, "dependency_mismatch": 137487, "answer_mismatch": 15981, "extra_nodes": 16321, "missing_nodes": 12528, "perfect_with_extra": 1512, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 333170, "value_mismatch": 7001, "dependency_mismatch": 44800, "answer_mismatch": 5490, "missing_nodes": 3937, "extra_nodes": 5159, "perfect_with_extra": 419}, "teachers_in_school": {"total": 390656, "perfect": 343279, "value_mismatch": 6175, "dependency_mismatch": 46364, "answer_mismatch": 5195, "extra_nodes": 5795, "missing_nodes": 4315, "perfect_with_extra": 697}, "movie_festival_awards": {"total": 382336, "perfect": 335026, "value_mismatch": 6697, "dependency_mismatch": 46323, "answer_mismatch": 5296, "missing_nodes": 4276, "extra_nodes": 5367, "perfect_with_extra": 396}}, "results_path": "results/composition/SDAR-100M-block4-epoch1-op9-12/graphs/SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 314531, "value_mismatch": 728333, "dependency_mismatch": 907071, "answer_mismatch": 599699, "extra_nodes": 62820, "missing_nodes": 523472, "perfect_with_extra": 159, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 105709, "dependency_mismatch": 306755, "extra_nodes": 23846, "perfect_with_extra": 46, "value_mismatch": 244735, "answer_mismatch": 200532, "missing_nodes": 186056}, "movie_festival_awards": {"total": 418816, "perfect": 103107, "value_mismatch": 239625, "dependency_mismatch": 297087, "answer_mismatch": 197152, "missing_nodes": 166318, "extra_nodes": 18818, "perfect_with_extra": 93}, "teachers_in_school": {"total": 430464, "perfect": 105715, "dependency_mismatch": 303229, "value_mismatch": 243973, "answer_mismatch": 202015, "missing_nodes": 171098, "extra_nodes": 20156, "perfect_with_extra": 20}}, "results_path": "results/composition/SDAR-100M-block4-epoch1-op9-12/graphs/SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1326006, "value_mismatch": 748206, "dependency_mismatch": 1044558, "answer_mismatch": 615680, "extra_nodes": 79141, "missing_nodes": 536000, "perfect_with_extra": 1671, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch1-op9-12/graphs/SDAR-100M-block4-epoch1-op9-12_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 438879, "value_mismatch": 251736, "dependency_mismatch": 351555, "answer_mismatch": 206022, "missing_nodes": 189993, "extra_nodes": 29005, "perfect_with_extra": 465}, "teachers_in_school": {"total": 821120, "perfect": 448994, "value_mismatch": 250148, "dependency_mismatch": 349593, "answer_mismatch": 207210, "extra_nodes": 25951, "missing_nodes": 175413, "perfect_with_extra": 717}, "movie_festival_awards": {"total": 801152, "perfect": 438133, "value_mismatch": 246322, "dependency_mismatch": 343410, "answer_mismatch": 202448, "missing_nodes": 170594, "extra_nodes": 24185, "perfect_with_extra": 489}}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch1-op9-12", "results_dir": "results/composition/SDAR-100M-block4-epoch1-op9-12/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch1-op9-12/graphs/SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch1-op9-12/graphs/SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch1-op9-12/graphs/SDAR-100M-block4-epoch1-op9-12_score_report.jsonl", "template_stats": {}}} diff --git a/composition/SDAR-100M-block4-epoch1-op9-12/graphs/results.json b/composition/SDAR-100M-block4-epoch1-op9-12/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..2bbf0f5b86213480aa2e3747611c3a4c4a1e73e2 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op9-12/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 0.9968847352024922, + "pass@4": 0.9968847352024922, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990625, + "pass@2": 0.99375, + "pass@4": 0.99375, + "pass@8": 0.99375, + "pass@16": 0.996875, + "pass@32": 0.996875, + "pass@64": 0.996875, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9523809523809523, + "pass@2": 0.9746031746031746, + "pass@4": 0.9873015873015873, + "pass@8": 0.9936507936507937, + "pass@16": 0.9936507936507937, + "pass@32": 0.9936507936507937, + "pass@64": 0.9936507936507937, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9670658682634731, + "pass@2": 0.9790419161676647, + "pass@4": 0.9850299401197605, + "pass@8": 0.9880239520958084, + "pass@16": 0.9940119760479041, + "pass@32": 0.9940119760479041, + "pass@64": 0.9940119760479041, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.8698224852071006, + "pass@2": 0.9171597633136095, + "pass@4": 0.9585798816568047, + "pass@8": 0.9911242603550295, + "pass@16": 0.9970414201183432, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.7948717948717948, + "pass@2": 0.8557692307692307, + "pass@4": 0.9294871794871795, + "pass@8": 0.9743589743589743, + "pass@16": 0.9935897435897436, + "pass@32": 0.9935897435897436, + "pass@64": 0.9967948717948718, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7323943661971831, + "pass@2": 0.819718309859155, + "pass@4": 0.8901408450704226, + "pass@8": 0.9492957746478873, + "pass@16": 0.9774647887323944, + "pass@32": 0.9915492957746479, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6312684365781711, + "pass@2": 0.7079646017699115, + "pass@4": 0.7964601769911505, + "pass@8": 0.8672566371681416, + "pass@16": 0.8997050147492626, + "pass@32": 0.9174041297935103, + "pass@64": 0.9351032448377581, + "pass@128": 0.9410029498525073 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.990625, + "5": 0.9523809523809523, + "6": 0.9670658682634731, + "7": 0.8698224852071006, + "8": 0.7948717948717948, + "9": 0.7323943661971831, + "10": 0.6312684365781711 + }, + "pass@2": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.99375, + "5": 0.9746031746031746, + "6": 0.9790419161676647, + "7": 0.9171597633136095, + "8": 0.8557692307692307, + "9": 0.819718309859155, + "10": 0.7079646017699115 + }, + "pass@4": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.99375, + "5": 0.9873015873015873, + "6": 0.9850299401197605, + "7": 0.9585798816568047, + "8": 0.9294871794871795, + "9": 0.8901408450704226, + "10": 0.7964601769911505 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9936507936507937, + "6": 0.9880239520958084, + "7": 0.9911242603550295, + "8": 0.9743589743589743, + "9": 0.9492957746478873, + "10": 0.8672566371681416 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 0.9940119760479041, + "7": 0.9970414201183432, + "8": 0.9935897435897436, + "9": 0.9774647887323944, + "10": 0.8997050147492626 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 0.9940119760479041, + "7": 0.9970414201183432, + "8": 0.9935897435897436, + "9": 0.9915492957746479, + "10": 0.9174041297935103 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 0.9940119760479041, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9351032448377581 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 0.9970059880239521, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9410029498525073 + } + }, + "total_pass_at_k": { + "pass@1": 0.8790949003714961, + "pass@2": 0.9142181695373185, + "pass@4": 0.9473150962512664, + "pass@8": 0.972306653157717, + "pass@16": 0.9831138129010469, + "pass@32": 0.9868287740628167, + "pass@64": 0.9902060114826072, + "pass@128": 0.9912191827085444 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970845481049563, + "pass@2": 0.9970845481049563, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9912280701754386, + "pass@2": 0.9941520467836257, + "pass@4": 0.9970760233918129, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9382352941176471, + "pass@2": 0.9382352941176471, + "pass@4": 0.9735294117647059, + "pass@8": 0.9823529411764705, + "pass@16": 0.9882352941176471, + "pass@32": 0.9911764705882353, + "pass@64": 0.9941176470588236, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9621451104100947, + "pass@2": 0.9716088328075709, + "pass@4": 0.9810725552050473, + "pass@8": 0.9936908517350158, + "pass@16": 0.9968454258675079, + "pass@32": 0.9968454258675079, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8544303797468354, + "pass@2": 0.8892405063291139, + "pass@4": 0.9430379746835443, + "pass@8": 0.9746835443037974, + "pass@16": 0.9841772151898734, + "pass@32": 0.990506329113924, + "pass@64": 0.9968354430379747, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.7747747747747747, + "pass@2": 0.8438438438438438, + "pass@4": 0.9159159159159159, + "pass@8": 0.972972972972973, + "pass@16": 0.987987987987988, + "pass@32": 0.993993993993994, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.7620578778135049, + "pass@2": 0.8360128617363344, + "pass@4": 0.9260450160771704, + "pass@8": 0.954983922829582, + "pass@16": 0.9839228295819936, + "pass@32": 0.9935691318327974, + "pass@64": 0.9967845659163987, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.591044776119403, + "pass@2": 0.6567164179104478, + "pass@4": 0.7432835820895523, + "pass@8": 0.835820895522388, + "pass@16": 0.8776119402985074, + "pass@32": 0.9134328358208955, + "pass@64": 0.9402985074626866, + "pass@128": 0.9462686567164179 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9912280701754386, + "5": 0.9382352941176471, + "6": 0.9621451104100947, + "7": 0.8544303797468354, + "8": 0.7747747747747747, + "9": 0.7620578778135049, + "10": 0.591044776119403 + }, + "pass@2": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9941520467836257, + "5": 0.9382352941176471, + "6": 0.9716088328075709, + "7": 0.8892405063291139, + "8": 0.8438438438438438, + "9": 0.8360128617363344, + "10": 0.6567164179104478 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9735294117647059, + "6": 0.9810725552050473, + "7": 0.9430379746835443, + "8": 0.9159159159159159, + "9": 0.9260450160771704, + "10": 0.7432835820895523 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9823529411764705, + "6": 0.9936908517350158, + "7": 0.9746835443037974, + "8": 0.972972972972973, + "9": 0.954983922829582, + "10": 0.835820895522388 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9882352941176471, + "6": 0.9968454258675079, + "7": 0.9841772151898734, + "8": 0.987987987987988, + "9": 0.9839228295819936, + "10": 0.8776119402985074 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9911764705882353, + "6": 0.9968454258675079, + "7": 0.990506329113924, + "8": 0.993993993993994, + "9": 0.9935691318327974, + "10": 0.9134328358208955 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9941176470588236, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.996996996996997, + "9": 0.9967845659163987, + "10": 0.9402985074626866 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9941176470588236, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9462686567164179 + } + }, + "total_pass_at_k": { + "pass@1": 0.8764646802812186, + "pass@2": 0.9042517576163375, + "pass@4": 0.9427519250083696, + "pass@8": 0.9685302979578172, + "pass@16": 0.9799129561432876, + "pass@32": 0.9866086374288584, + "pass@64": 0.9916303983930365, + "pass@128": 0.9926347505858721 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985207100591716, + "pass@2": 0.9940828402366864, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9594202898550724, + "pass@2": 0.9884057971014493, + "pass@4": 0.991304347826087, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9770773638968482, + "pass@2": 0.9828080229226361, + "pass@4": 0.9856733524355301, + "pass@8": 0.997134670487106, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.8497109826589595, + "pass@2": 0.8930635838150289, + "pass@4": 0.9566473988439307, + "pass@8": 0.976878612716763, + "pass@16": 0.9884393063583815, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.8, + "pass@2": 0.8760563380281691, + "pass@4": 0.9352112676056338, + "pass@8": 0.9774647887323944, + "pass@16": 0.9830985915492958, + "pass@32": 0.9943661971830986, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7065868263473054, + "pass@2": 0.8143712574850299, + "pass@4": 0.8562874251497006, + "pass@8": 0.9131736526946108, + "pass@16": 0.9670658682634731, + "pass@32": 0.9820359281437125, + "pass@64": 0.9940119760479041, + "pass@128": 0.9940119760479041 + }, + "10": { + "pass@1": 0.6349693251533742, + "pass@2": 0.7024539877300614, + "pass@4": 0.7760736196319018, + "pass@8": 0.852760736196319, + "pass@16": 0.8865030674846626, + "pass@32": 0.911042944785276, + "pass@64": 0.9325153374233128, + "pass@128": 0.9447852760736196 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.985207100591716, + "5": 0.9594202898550724, + "6": 0.9770773638968482, + "7": 0.8497109826589595, + "8": 0.8, + "9": 0.7065868263473054, + "10": 0.6349693251533742 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9940828402366864, + "5": 0.9884057971014493, + "6": 0.9828080229226361, + "7": 0.8930635838150289, + "8": 0.8760563380281691, + "9": 0.8143712574850299, + "10": 0.7024539877300614 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.991304347826087, + "6": 0.9856733524355301, + "7": 0.9566473988439307, + "8": 0.9352112676056338, + "9": 0.8562874251497006, + "10": 0.7760736196319018 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.997134670487106, + "7": 0.976878612716763, + "8": 0.9774647887323944, + "9": 0.9131736526946108, + "10": 0.852760736196319 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.997134670487106, + "7": 0.9884393063583815, + "8": 0.9830985915492958, + "9": 0.9670658682634731, + "10": 0.8865030674846626 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9971098265895953, + "8": 0.9943661971830986, + "9": 0.9820359281437125, + "10": 0.911042944785276 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9971098265895953, + "8": 1.0, + "9": 0.9940119760479041, + "10": 0.9325153374233128 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9971098265895953, + "8": 1.0, + "9": 0.9940119760479041, + "10": 0.9447852760736196 + } + }, + "total_pass_at_k": { + "pass@1": 0.8797509829619922, + "pass@2": 0.9174311926605505, + "pass@4": 0.9452817824377457, + "pass@8": 0.968872870249017, + "pass@16": 0.9803407601572739, + "pass@32": 0.987221494102228, + "pass@64": 0.991480996068152, + "pass@128": 0.9927916120576671 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.999, + "pass@2": 0.999, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999, + "pass@2": 0.999, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.989, + "pass@2": 0.994, + "pass@4": 0.997, + "pass@8": 0.998, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.95, + "pass@2": 0.967, + "pass@4": 0.984, + "pass@8": 0.991, + "pass@16": 0.993, + "pass@32": 0.995, + "pass@64": 0.996, + "pass@128": 0.996 + }, + "6": { + "pass@1": 0.969, + "pass@2": 0.978, + "pass@4": 0.984, + "pass@8": 0.993, + "pass@16": 0.996, + "pass@32": 0.996, + "pass@64": 0.997, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.858, + "pass@2": 0.9, + "pass@4": 0.953, + "pass@8": 0.981, + "pass@16": 0.99, + "pass@32": 0.995, + "pass@64": 0.997, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.79, + "pass@2": 0.859, + "pass@4": 0.927, + "pass@8": 0.975, + "pass@16": 0.988, + "pass@32": 0.994, + "pass@64": 0.998, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.733, + "pass@2": 0.823, + "pass@4": 0.89, + "pass@8": 0.939, + "pass@16": 0.976, + "pass@32": 0.989, + "pass@64": 0.997, + "pass@128": 0.998 + }, + "10": { + "pass@1": 0.619, + "pass@2": 0.689, + "pass@4": 0.772, + "pass@8": 0.852, + "pass@16": 0.888, + "pass@32": 0.914, + "pass@64": 0.936, + "pass@128": 0.944 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.999, + "3": 0.999, + "4": 0.989, + "5": 0.95, + "6": 0.969, + "7": 0.858, + "8": 0.79, + "9": 0.733, + "10": 0.619 + }, + "pass@2": { + "2": 0.999, + "3": 0.999, + "4": 0.994, + "5": 0.967, + "6": 0.978, + "7": 0.9, + "8": 0.859, + "9": 0.823, + "10": 0.689 + }, + "pass@4": { + "2": 0.999, + "3": 1.0, + "4": 0.997, + "5": 0.984, + "6": 0.984, + "7": 0.953, + "8": 0.927, + "9": 0.89, + "10": 0.772 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.998, + "5": 0.991, + "6": 0.993, + "7": 0.981, + "8": 0.975, + "9": 0.939, + "10": 0.852 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.993, + "6": 0.996, + "7": 0.99, + "8": 0.988, + "9": 0.976, + "10": 0.888 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.995, + "6": 0.996, + "7": 0.995, + "8": 0.994, + "9": 0.989, + "10": 0.914 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.996, + "6": 0.997, + "7": 0.997, + "8": 0.998, + "9": 0.997, + "10": 0.936 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.996, + "6": 0.998, + "7": 0.997, + "8": 0.998, + "9": 0.998, + "10": 0.944 + } + }, + "total_pass_at_k": { + "pass@1": 0.8784444444444445, + "pass@2": 0.912, + "pass@4": 0.9451111111111111, + "pass@8": 0.9698888888888889, + "pass@16": 0.9811111111111112, + "pass@32": 0.9868888888888889, + "pass@64": 0.9911111111111112, + "pass@128": 0.9922222222222222 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5722379603399433, + "pass@2": 0.6203966005665722, + "pass@4": 0.7082152974504249, + "pass@8": 0.8073654390934845, + "pass@16": 0.8640226628895185, + "pass@32": 0.9036827195467422, + "pass@64": 0.9320113314447592, + "pass@128": 0.9461756373937678 + }, + "12": { + "pass@1": 0.5480225988700564, + "pass@2": 0.6101694915254238, + "pass@4": 0.6949152542372882, + "pass@8": 0.7768361581920904, + "pass@16": 0.8220338983050848, + "pass@32": 0.844632768361582, + "pass@64": 0.8587570621468926, + "pass@128": 0.867231638418079 + }, + "13": { + "pass@1": 0.43573667711598746, + "pass@2": 0.5109717868338558, + "pass@4": 0.5799373040752351, + "pass@8": 0.658307210031348, + "pass@16": 0.7178683385579937, + "pass@32": 0.7523510971786834, + "pass@64": 0.7836990595611285, + "pass@128": 0.8056426332288401 + }, + "14": { + "pass@1": 0.35514018691588783, + "pass@2": 0.40809968847352024, + "pass@4": 0.4766355140186916, + "pass@8": 0.5389408099688473, + "pass@16": 0.5794392523364486, + "pass@32": 0.6573208722741433, + "pass@64": 0.7133956386292835, + "pass@128": 0.7476635514018691 + }, + "15": { + "pass@1": 0.25722543352601157, + "pass@2": 0.2832369942196532, + "pass@4": 0.3265895953757225, + "pass@8": 0.3872832369942196, + "pass@16": 0.4595375722543353, + "pass@32": 0.5202312138728323, + "pass@64": 0.5924855491329479, + "pass@128": 0.661849710982659 + }, + "16": { + "pass@1": 0.12903225806451613, + "pass@2": 0.1870967741935484, + "pass@4": 0.23870967741935484, + "pass@8": 0.2838709677419355, + "pass@16": 0.3387096774193548, + "pass@32": 0.38064516129032255, + "pass@64": 0.42258064516129035, + "pass@128": 0.46774193548387094 + }, + "17": { + "pass@1": 0.08959537572254335, + "pass@2": 0.12716763005780346, + "pass@4": 0.17341040462427745, + "pass@8": 0.20520231213872833, + "pass@16": 0.24277456647398843, + "pass@32": 0.2774566473988439, + "pass@64": 0.3092485549132948, + "pass@128": 0.3439306358381503 + }, + "18": { + "pass@1": 0.04285714285714286, + "pass@2": 0.054285714285714284, + "pass@4": 0.07428571428571429, + "pass@8": 0.1, + "pass@16": 0.12571428571428572, + "pass@32": 0.14, + "pass@64": 0.16285714285714287, + "pass@128": 0.1742857142857143 + }, + "19": { + "pass@1": 0.003003003003003003, + "pass@2": 0.009009009009009009, + "pass@4": 0.02702702702702703, + "pass@8": 0.03303303303303303, + "pass@16": 0.03903903903903904, + "pass@32": 0.04804804804804805, + "pass@64": 0.06306306306306306, + "pass@128": 0.06906906906906907 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.003003003003003003, + "pass@8": 0.018018018018018018, + "pass@16": 0.018018018018018018, + "pass@32": 0.03003003003003003, + "pass@64": 0.042042042042042045, + "pass@128": 0.04504504504504504 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5722379603399433, + "12": 0.5480225988700564, + "13": 0.43573667711598746, + "14": 0.35514018691588783, + "15": 0.25722543352601157, + "16": 0.12903225806451613, + "17": 0.08959537572254335, + "18": 0.04285714285714286, + "19": 0.003003003003003003, + "20": 0.0 + }, + "pass@2": { + "11": 0.6203966005665722, + "12": 0.6101694915254238, + "13": 0.5109717868338558, + "14": 0.40809968847352024, + "15": 0.2832369942196532, + "16": 0.1870967741935484, + "17": 0.12716763005780346, + "18": 0.054285714285714284, + "19": 0.009009009009009009, + "20": 0.0 + }, + "pass@4": { + "11": 0.7082152974504249, + "12": 0.6949152542372882, + "13": 0.5799373040752351, + "14": 0.4766355140186916, + "15": 0.3265895953757225, + "16": 0.23870967741935484, + "17": 0.17341040462427745, + "18": 0.07428571428571429, + "19": 0.02702702702702703, + "20": 0.003003003003003003 + }, + "pass@8": { + "11": 0.8073654390934845, + "12": 0.7768361581920904, + "13": 0.658307210031348, + "14": 0.5389408099688473, + "15": 0.3872832369942196, + "16": 0.2838709677419355, + "17": 0.20520231213872833, + "18": 0.1, + "19": 0.03303303303303303, + "20": 0.018018018018018018 + }, + "pass@16": { + "11": 0.8640226628895185, + "12": 0.8220338983050848, + "13": 0.7178683385579937, + "14": 0.5794392523364486, + "15": 0.4595375722543353, + "16": 0.3387096774193548, + "17": 0.24277456647398843, + "18": 0.12571428571428572, + "19": 0.03903903903903904, + "20": 0.018018018018018018 + }, + "pass@32": { + "11": 0.9036827195467422, + "12": 0.844632768361582, + "13": 0.7523510971786834, + "14": 0.6573208722741433, + "15": 0.5202312138728323, + "16": 0.38064516129032255, + "17": 0.2774566473988439, + "18": 0.14, + "19": 0.04804804804804805, + "20": 0.03003003003003003 + }, + "pass@64": { + "11": 0.9320113314447592, + "12": 0.8587570621468926, + "13": 0.7836990595611285, + "14": 0.7133956386292835, + "15": 0.5924855491329479, + "16": 0.42258064516129035, + "17": 0.3092485549132948, + "18": 0.16285714285714287, + "19": 0.06306306306306306, + "20": 0.042042042042042045 + }, + "pass@128": { + "11": 0.9461756373937678, + "12": 0.867231638418079, + "13": 0.8056426332288401, + "14": 0.7476635514018691, + "15": 0.661849710982659, + "16": 0.46774193548387094, + "17": 0.3439306358381503, + "18": 0.1742857142857143, + "19": 0.06906906906906907, + "20": 0.04504504504504504 + } + }, + "total_pass_at_k": { + "pass@1": 0.2451708766716196, + "pass@2": 0.2826151560178306, + "pass@4": 0.33194650817236254, + "pass@8": 0.3827637444279346, + "pass@16": 0.4225854383358098, + "pass@32": 0.45705794947994055, + "pass@64": 0.48945022288261514, + "pass@128": 0.5141158989598811 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5523809523809524, + "pass@2": 0.6476190476190476, + "pass@4": 0.7365079365079366, + "pass@8": 0.819047619047619, + "pass@16": 0.8920634920634921, + "pass@32": 0.926984126984127, + "pass@64": 0.946031746031746, + "pass@128": 0.9587301587301588 + }, + "12": { + "pass@1": 0.581039755351682, + "pass@2": 0.636085626911315, + "pass@4": 0.7186544342507645, + "pass@8": 0.7889908256880734, + "pass@16": 0.8470948012232415, + "pass@32": 0.8837920489296636, + "pass@64": 0.9021406727828746, + "pass@128": 0.9113149847094801 + }, + "13": { + "pass@1": 0.4444444444444444, + "pass@2": 0.5061728395061729, + "pass@4": 0.5864197530864198, + "pass@8": 0.6358024691358025, + "pass@16": 0.6882716049382716, + "pass@32": 0.7283950617283951, + "pass@64": 0.7561728395061729, + "pass@128": 0.7777777777777778 + }, + "14": { + "pass@1": 0.3323442136498516, + "pass@2": 0.3916913946587537, + "pass@4": 0.4362017804154303, + "pass@8": 0.49554896142433236, + "pass@16": 0.5637982195845698, + "pass@32": 0.6261127596439169, + "pass@64": 0.6973293768545994, + "pass@128": 0.7418397626112759 + }, + "15": { + "pass@1": 0.25925925925925924, + "pass@2": 0.30246913580246915, + "pass@4": 0.3333333333333333, + "pass@8": 0.36728395061728397, + "pass@16": 0.4228395061728395, + "pass@32": 0.49691358024691357, + "pass@64": 0.5617283950617284, + "pass@128": 0.6203703703703703 + }, + "16": { + "pass@1": 0.13569321533923304, + "pass@2": 0.17699115044247787, + "pass@4": 0.22418879056047197, + "pass@8": 0.2743362831858407, + "pass@16": 0.3215339233038348, + "pass@32": 0.3805309734513274, + "pass@64": 0.41887905604719766, + "pass@128": 0.4778761061946903 + }, + "17": { + "pass@1": 0.07228915662650602, + "pass@2": 0.11746987951807229, + "pass@4": 0.14759036144578314, + "pass@8": 0.1746987951807229, + "pass@16": 0.19578313253012047, + "pass@32": 0.22289156626506024, + "pass@64": 0.2620481927710843, + "pass@128": 0.3132530120481928 + }, + "18": { + "pass@1": 0.08064516129032258, + "pass@2": 0.1064516129032258, + "pass@4": 0.12580645161290321, + "pass@8": 0.15161290322580645, + "pass@16": 0.17419354838709677, + "pass@32": 0.18387096774193548, + "pass@64": 0.2161290322580645, + "pass@128": 0.23870967741935484 + }, + "19": { + "pass@1": 0.030211480362537766, + "pass@2": 0.03625377643504532, + "pass@4": 0.04833836858006042, + "pass@8": 0.06042296072507553, + "pass@16": 0.0634441087613293, + "pass@32": 0.08761329305135952, + "pass@64": 0.09667673716012085, + "pass@128": 0.11178247734138973 + }, + "20": { + "pass@1": 0.006006006006006006, + "pass@2": 0.018018018018018018, + "pass@4": 0.03303303303303303, + "pass@8": 0.042042042042042045, + "pass@16": 0.05405405405405406, + "pass@32": 0.06906906906906907, + "pass@64": 0.07807807807807808, + "pass@128": 0.09309309309309309 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5523809523809524, + "12": 0.581039755351682, + "13": 0.4444444444444444, + "14": 0.3323442136498516, + "15": 0.25925925925925924, + "16": 0.13569321533923304, + "17": 0.07228915662650602, + "18": 0.08064516129032258, + "19": 0.030211480362537766, + "20": 0.006006006006006006 + }, + "pass@2": { + "11": 0.6476190476190476, + "12": 0.636085626911315, + "13": 0.5061728395061729, + "14": 0.3916913946587537, + "15": 0.30246913580246915, + "16": 0.17699115044247787, + "17": 0.11746987951807229, + "18": 0.1064516129032258, + "19": 0.03625377643504532, + "20": 0.018018018018018018 + }, + "pass@4": { + "11": 0.7365079365079366, + "12": 0.7186544342507645, + "13": 0.5864197530864198, + "14": 0.4362017804154303, + "15": 0.3333333333333333, + "16": 0.22418879056047197, + "17": 0.14759036144578314, + "18": 0.12580645161290321, + "19": 0.04833836858006042, + "20": 0.03303303303303303 + }, + "pass@8": { + "11": 0.819047619047619, + "12": 0.7889908256880734, + "13": 0.6358024691358025, + "14": 0.49554896142433236, + "15": 0.36728395061728397, + "16": 0.2743362831858407, + "17": 0.1746987951807229, + "18": 0.15161290322580645, + "19": 0.06042296072507553, + "20": 0.042042042042042045 + }, + "pass@16": { + "11": 0.8920634920634921, + "12": 0.8470948012232415, + "13": 0.6882716049382716, + "14": 0.5637982195845698, + "15": 0.4228395061728395, + "16": 0.3215339233038348, + "17": 0.19578313253012047, + "18": 0.17419354838709677, + "19": 0.0634441087613293, + "20": 0.05405405405405406 + }, + "pass@32": { + "11": 0.926984126984127, + "12": 0.8837920489296636, + "13": 0.7283950617283951, + "14": 0.6261127596439169, + "15": 0.49691358024691357, + "16": 0.3805309734513274, + "17": 0.22289156626506024, + "18": 0.18387096774193548, + "19": 0.08761329305135952, + "20": 0.06906906906906907 + }, + "pass@64": { + "11": 0.946031746031746, + "12": 0.9021406727828746, + "13": 0.7561728395061729, + "14": 0.6973293768545994, + "15": 0.5617283950617284, + "16": 0.41887905604719766, + "17": 0.2620481927710843, + "18": 0.2161290322580645, + "19": 0.09667673716012085, + "20": 0.07807807807807808 + }, + "pass@128": { + "11": 0.9587301587301588, + "12": 0.9113149847094801, + "13": 0.7777777777777778, + "14": 0.7418397626112759, + "15": 0.6203703703703703, + "16": 0.4778761061946903, + "17": 0.3132530120481928, + "18": 0.23870967741935484, + "19": 0.11178247734138973, + "20": 0.09309309309309309 + } + }, + "total_pass_at_k": { + "pass@1": 0.2478606356968215, + "pass@2": 0.2921760391198044, + "pass@4": 0.3371026894865526, + "pass@8": 0.37897310513447435, + "pass@16": 0.4202322738386308, + "pass@32": 0.45874083129584353, + "pass@64": 0.49174816625916873, + "pass@128": 0.5229217603911981 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5783132530120482, + "pass@2": 0.6596385542168675, + "pass@4": 0.7379518072289156, + "pass@8": 0.822289156626506, + "pass@16": 0.8855421686746988, + "pass@32": 0.9307228915662651, + "pass@64": 0.9578313253012049, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.5078369905956113, + "pass@2": 0.5768025078369906, + "pass@4": 0.6551724137931034, + "pass@8": 0.7366771159874608, + "pass@16": 0.8056426332288401, + "pass@32": 0.8369905956112853, + "pass@64": 0.8557993730407524, + "pass@128": 0.8714733542319749 + }, + "13": { + "pass@1": 0.4173669467787115, + "pass@2": 0.47619047619047616, + "pass@4": 0.5686274509803921, + "pass@8": 0.6526610644257703, + "pass@16": 0.6918767507002801, + "pass@32": 0.7535014005602241, + "pass@64": 0.8095238095238095, + "pass@128": 0.8375350140056023 + }, + "14": { + "pass@1": 0.30701754385964913, + "pass@2": 0.38596491228070173, + "pass@4": 0.4444444444444444, + "pass@8": 0.49707602339181284, + "pass@16": 0.564327485380117, + "pass@32": 0.652046783625731, + "pass@64": 0.7251461988304093, + "pass@128": 0.7485380116959064 + }, + "15": { + "pass@1": 0.2636363636363636, + "pass@2": 0.33636363636363636, + "pass@4": 0.4, + "pass@8": 0.4666666666666667, + "pass@16": 0.5181818181818182, + "pass@32": 0.5666666666666667, + "pass@64": 0.6424242424242425, + "pass@128": 0.6878787878787879 + }, + "16": { + "pass@1": 0.11396011396011396, + "pass@2": 0.18518518518518517, + "pass@4": 0.24786324786324787, + "pass@8": 0.29914529914529914, + "pass@16": 0.33903133903133903, + "pass@32": 0.39316239316239315, + "pass@64": 0.4188034188034188, + "pass@128": 0.45014245014245013 + }, + "17": { + "pass@1": 0.09316770186335403, + "pass@2": 0.13043478260869565, + "pass@4": 0.15838509316770186, + "pass@8": 0.18322981366459629, + "pass@16": 0.21428571428571427, + "pass@32": 0.2639751552795031, + "pass@64": 0.3198757763975155, + "pass@128": 0.36024844720496896 + }, + "18": { + "pass@1": 0.07352941176470588, + "pass@2": 0.10588235294117647, + "pass@4": 0.12058823529411765, + "pass@8": 0.1411764705882353, + "pass@16": 0.1676470588235294, + "pass@32": 0.20294117647058824, + "pass@64": 0.22058823529411764, + "pass@128": 0.24411764705882352 + }, + "19": { + "pass@1": 0.005952380952380952, + "pass@2": 0.01488095238095238, + "pass@4": 0.020833333333333332, + "pass@8": 0.03273809523809524, + "pass@16": 0.05357142857142857, + "pass@32": 0.0625, + "pass@64": 0.08630952380952381, + "pass@128": 0.09523809523809523 + }, + "20": { + "pass@1": 0.011976047904191617, + "pass@2": 0.017964071856287425, + "pass@4": 0.03293413173652695, + "pass@8": 0.04491017964071856, + "pass@16": 0.05389221556886228, + "pass@32": 0.0658682634730539, + "pass@64": 0.07784431137724551, + "pass@128": 0.10179640718562874 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5783132530120482, + "12": 0.5078369905956113, + "13": 0.4173669467787115, + "14": 0.30701754385964913, + "15": 0.2636363636363636, + "16": 0.11396011396011396, + "17": 0.09316770186335403, + "18": 0.07352941176470588, + "19": 0.005952380952380952, + "20": 0.011976047904191617 + }, + "pass@2": { + "11": 0.6596385542168675, + "12": 0.5768025078369906, + "13": 0.47619047619047616, + "14": 0.38596491228070173, + "15": 0.33636363636363636, + "16": 0.18518518518518517, + "17": 0.13043478260869565, + "18": 0.10588235294117647, + "19": 0.01488095238095238, + "20": 0.017964071856287425 + }, + "pass@4": { + "11": 0.7379518072289156, + "12": 0.6551724137931034, + "13": 0.5686274509803921, + "14": 0.4444444444444444, + "15": 0.4, + "16": 0.24786324786324787, + "17": 0.15838509316770186, + "18": 0.12058823529411765, + "19": 0.020833333333333332, + "20": 0.03293413173652695 + }, + "pass@8": { + "11": 0.822289156626506, + "12": 0.7366771159874608, + "13": 0.6526610644257703, + "14": 0.49707602339181284, + "15": 0.4666666666666667, + "16": 0.29914529914529914, + "17": 0.18322981366459629, + "18": 0.1411764705882353, + "19": 0.03273809523809524, + "20": 0.04491017964071856 + }, + "pass@16": { + "11": 0.8855421686746988, + "12": 0.8056426332288401, + "13": 0.6918767507002801, + "14": 0.564327485380117, + "15": 0.5181818181818182, + "16": 0.33903133903133903, + "17": 0.21428571428571427, + "18": 0.1676470588235294, + "19": 0.05357142857142857, + "20": 0.05389221556886228 + }, + "pass@32": { + "11": 0.9307228915662651, + "12": 0.8369905956112853, + "13": 0.7535014005602241, + "14": 0.652046783625731, + "15": 0.5666666666666667, + "16": 0.39316239316239315, + "17": 0.2639751552795031, + "18": 0.20294117647058824, + "19": 0.0625, + "20": 0.0658682634730539 + }, + "pass@64": { + "11": 0.9578313253012049, + "12": 0.8557993730407524, + "13": 0.8095238095238095, + "14": 0.7251461988304093, + "15": 0.6424242424242425, + "16": 0.4188034188034188, + "17": 0.3198757763975155, + "18": 0.22058823529411764, + "19": 0.08630952380952381, + "20": 0.07784431137724551 + }, + "pass@128": { + "11": 0.9698795180722891, + "12": 0.8714733542319749, + "13": 0.8375350140056023, + "14": 0.7485380116959064, + "15": 0.6878787878787879, + "16": 0.45014245014245013, + "17": 0.36024844720496896, + "18": 0.24411764705882352, + "19": 0.09523809523809523, + "20": 0.10179640718562874 + } + }, + "total_pass_at_k": { + "pass@1": 0.2366934284864704, + "pass@2": 0.28843294677371395, + "pass@4": 0.3383883437407077, + "pass@8": 0.3874516800475766, + "pass@16": 0.42908117752007136, + "pass@32": 0.4727921498661909, + "pass@64": 0.5114481118049361, + "pass@128": 0.536723163841808 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.568, + "pass@2": 0.642, + "pass@4": 0.727, + "pass@8": 0.816, + "pass@16": 0.88, + "pass@32": 0.92, + "pass@64": 0.945, + "pass@128": 0.958 + }, + "12": { + "pass@1": 0.546, + "pass@2": 0.608, + "pass@4": 0.69, + "pass@8": 0.768, + "pass@16": 0.825, + "pass@32": 0.855, + "pass@64": 0.872, + "pass@128": 0.883 + }, + "13": { + "pass@1": 0.432, + "pass@2": 0.497, + "pass@4": 0.578, + "pass@8": 0.649, + "pass@16": 0.699, + "pass@32": 0.745, + "pass@64": 0.784, + "pass@128": 0.808 + }, + "14": { + "pass@1": 0.331, + "pass@2": 0.395, + "pass@4": 0.452, + "pass@8": 0.51, + "pass@16": 0.569, + "pass@32": 0.645, + "pass@64": 0.712, + "pass@128": 0.746 + }, + "15": { + "pass@1": 0.26, + "pass@2": 0.307, + "pass@4": 0.353, + "pass@8": 0.407, + "pass@16": 0.467, + "pass@32": 0.528, + "pass@64": 0.599, + "pass@128": 0.657 + }, + "16": { + "pass@1": 0.126, + "pass@2": 0.183, + "pass@4": 0.237, + "pass@8": 0.286, + "pass@16": 0.333, + "pass@32": 0.385, + "pass@64": 0.42, + "pass@128": 0.465 + }, + "17": { + "pass@1": 0.085, + "pass@2": 0.125, + "pass@4": 0.16, + "pass@8": 0.188, + "pass@16": 0.218, + "pass@32": 0.255, + "pass@64": 0.297, + "pass@128": 0.339 + }, + "18": { + "pass@1": 0.065, + "pass@2": 0.088, + "pass@4": 0.106, + "pass@8": 0.13, + "pass@16": 0.155, + "pass@32": 0.175, + "pass@64": 0.199, + "pass@128": 0.218 + }, + "19": { + "pass@1": 0.013, + "pass@2": 0.02, + "pass@4": 0.032, + "pass@8": 0.042, + "pass@16": 0.052, + "pass@32": 0.066, + "pass@64": 0.082, + "pass@128": 0.092 + }, + "20": { + "pass@1": 0.006, + "pass@2": 0.012, + "pass@4": 0.023, + "pass@8": 0.035, + "pass@16": 0.042, + "pass@32": 0.055, + "pass@64": 0.066, + "pass@128": 0.08 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.568, + "12": 0.546, + "13": 0.432, + "14": 0.331, + "15": 0.26, + "16": 0.126, + "17": 0.085, + "18": 0.065, + "19": 0.013, + "20": 0.006 + }, + "pass@2": { + "11": 0.642, + "12": 0.608, + "13": 0.497, + "14": 0.395, + "15": 0.307, + "16": 0.183, + "17": 0.125, + "18": 0.088, + "19": 0.02, + "20": 0.012 + }, + "pass@4": { + "11": 0.727, + "12": 0.69, + "13": 0.578, + "14": 0.452, + "15": 0.353, + "16": 0.237, + "17": 0.16, + "18": 0.106, + "19": 0.032, + "20": 0.023 + }, + "pass@8": { + "11": 0.816, + "12": 0.768, + "13": 0.649, + "14": 0.51, + "15": 0.407, + "16": 0.286, + "17": 0.188, + "18": 0.13, + "19": 0.042, + "20": 0.035 + }, + "pass@16": { + "11": 0.88, + "12": 0.825, + "13": 0.699, + "14": 0.569, + "15": 0.467, + "16": 0.333, + "17": 0.218, + "18": 0.155, + "19": 0.052, + "20": 0.042 + }, + "pass@32": { + "11": 0.92, + "12": 0.855, + "13": 0.745, + "14": 0.645, + "15": 0.528, + "16": 0.385, + "17": 0.255, + "18": 0.175, + "19": 0.066, + "20": 0.055 + }, + "pass@64": { + "11": 0.945, + "12": 0.872, + "13": 0.784, + "14": 0.712, + "15": 0.599, + "16": 0.42, + "17": 0.297, + "18": 0.199, + "19": 0.082, + "20": 0.066 + }, + "pass@128": { + "11": 0.958, + "12": 0.883, + "13": 0.808, + "14": 0.746, + "15": 0.657, + "16": 0.465, + "17": 0.339, + "18": 0.218, + "19": 0.092, + "20": 0.08 + } + }, + "total_pass_at_k": { + "pass@1": 0.2432, + "pass@2": 0.2877, + "pass@4": 0.3358, + "pass@8": 0.3831, + "pass@16": 0.424, + "pass@32": 0.4629, + "pass@64": 0.4976, + "pass@128": 0.5246 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block4-epoch1-op9-12/summary.csv b/composition/SDAR-100M-block4-epoch1-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..9508b7090177ae78b7d07bbfa23457b24d3fe357 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch1-op9-12,0.9861276041666667,0.5314875,0.7468433388157895,0.0,0.0,0.0,200.2339045138889,351.76516015625,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block4-epoch1-op9-12/summary.json b/composition/SDAR-100M-block4-epoch1-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0908d4d6a2abc32b6ecee09efdb08d04ee9db6e2 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch1-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch1-op9-12", + "id_acc": 0.9861276041666667, + "ood_acc": 0.5314875, + "total_acc": 0.7468433388157895, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.2339045138889, + "ood_avg_resp_len": 351.76516015625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 603712, + "answer_accuracy": 0.7455738223205818, + "avg_response_len": 266.78108204236486, + "resp_tokens_sum": 216020112, + "pass_at_k": { + "pass@1": 0.7455738223205818, + "pass@2": 0.7842754740154935, + "pass@4": 0.8145981928960141, + "pass@8": 0.8387537404175, + "pass@16": 0.8588352159822145, + "pass@32": 0.8764292483559714, + "pass@64": 0.8917570475437393, + "pass@128": 0.9046791021182422 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9390440634218289, + "pass@2": 0.9555319170557223, + "pass@4": 0.9674239525369062, + "pass@8": 0.9756565404139677, + "pass@16": 0.9813557770331947, + "pass@32": 0.986617282494939, + "pass@64": 0.9912821110885033, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9938720703125, + "pass@2": 0.9941333200049213, + "pass@4": 0.9944950684875327, + "pass@8": 0.9951502049040684, + "pass@16": 0.9962316451721318, + "pass@32": 0.9977126504385122, + "pass@64": 0.999181702515844, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9853422619047619, + "pass@2": 0.9885260436195474, + "pass@4": 0.9904835181316618, + "pass@8": 0.9919887051387225, + "pass@16": 0.9930441900209205, + "pass@32": 0.9935671832417211, + "pass@64": 0.9936503542147976, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9926553143712575, + "pass@2": 0.9937994336130889, + "pass@4": 0.9941684698656681, + "pass@8": 0.9943757134349077, + "pass@16": 0.9947162761058925, + "pass@32": 0.995326276580697, + "pass@64": 0.9962633787543024, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.9921181582840237, + "pass@2": 0.9955388342729348, + "pass@4": 0.9968490602875822, + "pass@8": 0.9975059751526393, + "pass@16": 0.9980238469616003, + "pass@32": 0.9987617116948309, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9820462740384616, + "pass@2": 0.9870359504340805, + "pass@4": 0.990314350730197, + "pass@8": 0.9933464239381959, + "pass@16": 0.9958232946543495, + "pass@32": 0.9976443884867328, + "pass@64": 0.9991607192427966, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9861575704225352, + "pass@2": 0.9930700343795054, + "pass@4": 0.9960230070712991, + "pass@8": 0.9978186757417113, + "pass@16": 0.9990916543037955, + "pass@32": 0.9998066566382701, + "pass@64": 0.9999959302892975, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9276071175637394, + "pass@2": 0.948805304922933, + "pass@4": 0.9625242864471968, + "pass@8": 0.9718624532312413, + "pass@16": 0.9782309268492891, + "pass@32": 0.9824962329796952, + "pass@64": 0.9854704070111471, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.8833862994350282, + "pass@2": 0.919130772053917, + "pass@4": 0.9369724676894196, + "pass@8": 0.949119166761525, + "pass@16": 0.9589524280731624, + "pass@32": 0.9663844678408191, + "pass@64": 0.9727848261772267, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.77953565830721, + "pass@2": 0.8406290881939132, + "pass@4": 0.8766417460944336, + "pass@8": 0.9007702961787816, + "pass@16": 0.9203564319065582, + "pass@32": 0.9373645301631106, + "pass@64": 0.9519000597968401, + "pass@128": 0.9655172413793104 + }, + "14": { + "pass@1": 0.726927570093458, + "pass@2": 0.8062917003949269, + "pass@4": 0.8548241513619205, + "pass@8": 0.8863406152747126, + "pass@16": 0.9104425624453125, + "pass@32": 0.9293280986882642, + "pass@64": 0.9439192226652879, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.6221324060693642, + "pass@2": 0.7063558213099081, + "pass@4": 0.7662719325907954, + "pass@8": 0.8111031131548001, + "pass@16": 0.8462265090737875, + "pass@32": 0.8747750472498352, + "pass@64": 0.8970795874266666, + "pass@128": 0.9161849710982659 + }, + "16": { + "pass@1": 0.4443296370967742, + "pass@2": 0.547864014478029, + "pass@4": 0.6275922671964388, + "pass@8": 0.6861379453842721, + "pass@16": 0.7316585228509355, + "pass@32": 0.7729760509118466, + "pass@64": 0.812534223224478, + "pass@128": 0.8483870967741935 + }, + "17": { + "pass@1": 0.3257541546242775, + "pass@2": 0.4236906035228256, + "pass@4": 0.5111456820245731, + "pass@8": 0.5823636363388676, + "pass@16": 0.6392486711470267, + "pass@32": 0.6881428453312798, + "pass@64": 0.7286531028102093, + "pass@128": 0.7572254335260116 + }, + "18": { + "pass@1": 0.26602678571428573, + "pass@2": 0.34801216254218215, + "pass@4": 0.4255049338475545, + "pass@8": 0.4947810451410119, + "pass@16": 0.55549192818462, + "pass@32": 0.6101924953252681, + "pass@64": 0.658631555139458, + "pass@128": 0.7 + }, + "19": { + "pass@1": 0.18520082582582584, + "pass@2": 0.2561013277056583, + "pass@4": 0.32643228098739907, + "pass@8": 0.39033346408424197, + "pass@16": 0.4477674798841049, + "pass@32": 0.4997264097323752, + "pass@64": 0.5464207467355886, + "pass@128": 0.5855855855855856 + }, + "20": { + "pass@1": 0.16732357357357358, + "pass@2": 0.2275554343713792, + "pass@4": 0.28745300840773264, + "pass@8": 0.3421696822977434, + "pass@16": 0.39285128747771964, + "pass@32": 0.4406795710163925, + "pass@64": 0.4845239493330482, + "pass@128": 0.5225225225225225 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 613909, + "answer_accuracy": 0.7476483339828527, + "avg_response_len": 276.8695196804365, + "resp_tokens_sum": 227343100, + "pass_at_k": { + "pass@1": 0.7476483339828527, + "pass@2": 0.7869729303858387, + "pass@4": 0.8185048834065208, + "pass@8": 0.8442728089335293, + "pass@16": 0.8656295025843366, + "pass@32": 0.8836184122445357, + "pass@64": 0.8987557769119414, + "pass@128": 0.9109898674980514 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9318682898773006, + "pass@2": 0.9437727193130767, + "pass@4": 0.9541319070162242, + "pass@8": 0.9643895337030387, + "pass@16": 0.9734856182486636, + "pass@32": 0.9804536494144921, + "pass@64": 0.9859565813477191, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999289772727272, + "pass@2": 0.9999996271772846, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998543823964497, + "pass@2": 0.9997983448259796, + "pass@4": 0.9999962377114103, + "pass@8": 0.9999999993222367, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9906929347826087, + "pass@2": 0.9938455722925938, + "pass@4": 0.99476957146661, + "pass@8": 0.9953830646909647, + "pass@16": 0.9963772680388042, + "pass@32": 0.9978842427305282, + "pass@64": 0.9994736049298187, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9967541189111748, + "pass@2": 0.9971184543013785, + "pass@4": 0.9971346140829818, + "pass@8": 0.9971346704870158, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9941067557803468, + "pass@2": 0.9960366790314508, + "pass@4": 0.996919753256999, + "pass@8": 0.9971032378641861, + "pass@16": 0.9971098202636923, + "pass@32": 0.9971098265895949, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.988556338028169, + "pass@2": 0.99323742652767, + "pass@4": 0.9950167355841083, + "pass@8": 0.9959384773710429, + "pass@16": 0.997143266424544, + "pass@32": 0.9986983950949794, + "pass@64": 0.9998376400133082, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9778255988023952, + "pass@2": 0.9839705431656373, + "pass@4": 0.9873886173098122, + "pass@8": 0.9896570716478766, + "pass@16": 0.9912446822176693, + "pass@32": 0.9922904474891701, + "pass@64": 0.9932692843858927, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.9410062123493976, + "pass@2": 0.958402384142871, + "pass@4": 0.9662167536851564, + "pass@8": 0.9709692525310544, + "pass@16": 0.9742474397487791, + "pass@32": 0.9771765082319408, + "pass@64": 0.9803255550853777, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8839880485893417, + "pass@2": 0.9137333232542638, + "pass@4": 0.9281012561517581, + "pass@8": 0.9354493476435397, + "pass@16": 0.9396505982108377, + "pass@32": 0.9434898827285836, + "pass@64": 0.9480917933676873, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7396271008403361, + "pass@2": 0.7994693448245437, + "pass@4": 0.8373948400882663, + "pass@8": 0.8673476898330499, + "pass@16": 0.8936367585473864, + "pass@32": 0.9163371027271825, + "pass@64": 0.9352437459860121, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.6859237938596491, + "pass@2": 0.7680046881475341, + "pass@4": 0.8229837292816459, + "pass@8": 0.8614744075315885, + "pass@16": 0.888244901857619, + "pass@32": 0.9069925793938199, + "pass@64": 0.920712589392341, + "pass@128": 0.9298245614035088 + }, + "15": { + "pass@1": 0.592779356060606, + "pass@2": 0.6902823759246002, + "pass@4": 0.7625650600493118, + "pass@8": 0.8162019416966678, + "pass@16": 0.8545200049180915, + "pass@32": 0.8813196398182785, + "pass@64": 0.9009509702228317, + "pass@128": 0.9151515151515152 + }, + "16": { + "pass@1": 0.419315349002849, + "pass@2": 0.5241369568387281, + "pass@4": 0.6076210393572596, + "pass@8": 0.6690281184725626, + "pass@16": 0.7159874640286359, + "pass@32": 0.7552395900040212, + "pass@64": 0.7868517864460535, + "pass@128": 0.8091168091168092 + }, + "17": { + "pass@1": 0.3528484083850932, + "pass@2": 0.4481074546388222, + "pass@4": 0.5321736775333203, + "pass@8": 0.6030474903401737, + "pass@16": 0.6627250939917279, + "pass@32": 0.7157933575690838, + "pass@64": 0.7650930743179017, + "pass@128": 0.8043478260869565 + }, + "18": { + "pass@1": 0.32125459558823527, + "pass@2": 0.411982905859194, + "pass@4": 0.49527306485586303, + "pass@8": 0.5687953540170917, + "pass@16": 0.6336849247595163, + "pass@32": 0.6896902244753337, + "pass@64": 0.7332202055305614, + "pass@128": 0.7647058823529411 + }, + "19": { + "pass@1": 0.20452008928571427, + "pass@2": 0.27103216101893507, + "pass@4": 0.3350326398932277, + "pass@8": 0.3956876057484762, + "pass@16": 0.4526802546976231, + "pass@32": 0.5058468809363461, + "pass@64": 0.557456349702467, + "pass@128": 0.6071428571428571 + }, + "20": { + "pass@1": 0.17171126497005987, + "pass@2": 0.2459550632985996, + "pass@4": 0.32637963022212046, + "pass@8": 0.4023198144270265, + "pass@16": 0.46881532030483886, + "pass@32": 0.5238391423437389, + "pass@64": 0.5672146906934317, + "pass@128": 0.6047904191616766 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 598702, + "answer_accuracy": 0.747301386004154, + "avg_response_len": 296.53006046293336, + "resp_tokens_sum": 237565651, + "pass_at_k": { + "pass@1": 0.747301386004154, + "pass@2": 0.7866222246264524, + "pass@4": 0.817830118884518, + "pass@8": 0.8427807997799793, + "pass@16": 0.8630964458588545, + "pass@32": 0.8803293590457137, + "pass@64": 0.8950675226999064, + "pass@128": 0.9071736699153219 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9529617537313433, + "pass@2": 0.9683944940651077, + "pass@4": 0.9764956958738361, + "pass@8": 0.9815531680276173, + "pass@16": 0.9854345229789683, + "pass@32": 0.9889262541804372, + "pass@64": 0.9917123824987969, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982233965014577, + "pass@2": 0.9990552041964142, + "pass@4": 0.9997025432863167, + "pass@8": 0.9999725887954414, + "pass@16": 0.9999998353432346, + "pass@32": 0.9999999999989982, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996116593567251, + "pass@2": 0.9999715804438919, + "pass@4": 0.9999998040266896, + "pass@8": 0.9999999999973679, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9767233455882353, + "pass@2": 0.9813144106067622, + "pass@4": 0.9852977822625111, + "pass@8": 0.9892662360668899, + "pass@16": 0.9930701541317976, + "pass@32": 0.9956711185322674, + "pass@64": 0.9968640960499501, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9923846608832808, + "pass@2": 0.9936656244566434, + "pass@4": 0.9945481545682183, + "pass@8": 0.9954181631392943, + "pass@16": 0.9966848255525158, + "pass@32": 0.9982819052559925, + "pass@64": 0.9995942613043955, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9842761075949367, + "pass@2": 0.9890673278181997, + "pass@4": 0.992488178000693, + "pass@8": 0.9953082245624335, + "pass@16": 0.997132974983664, + "pass@32": 0.9981929377128227, + "pass@64": 0.9992150520076071, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9881522147147147, + "pass@2": 0.9927101067603032, + "pass@4": 0.9944097843625403, + "pass@8": 0.995103475912635, + "pass@16": 0.9958460673407935, + "pass@32": 0.9966183345299415, + "pass@64": 0.9969772614942578, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9818880627009646, + "pass@2": 0.9903920709167786, + "pass@4": 0.9935343181579793, + "pass@8": 0.9947726177646687, + "pass@16": 0.9957678895375398, + "pass@32": 0.9970388159542295, + "pass@64": 0.9983478337657014, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9289434523809523, + "pass@2": 0.9530043119610053, + "pass@4": 0.9656035391409405, + "pass@8": 0.9720407256838862, + "pass@16": 0.9757086825588593, + "pass@32": 0.9784698585299846, + "pass@64": 0.9813627083401868, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8950449159021406, + "pass@2": 0.9291601952852221, + "pass@4": 0.9475928115063601, + "pass@8": 0.960967641143043, + "pass@16": 0.9716366374014452, + "pass@32": 0.9794922171298824, + "pass@64": 0.9848307287285822, + "pass@128": 0.9877675840978594 + }, + "13": { + "pass@1": 0.7696035879629629, + "pass@2": 0.8292227902935744, + "pass@4": 0.8658306317034441, + "pass@8": 0.89026121313563, + "pass@16": 0.9075930135701733, + "pass@32": 0.9225986569963891, + "pass@64": 0.9382621076613159, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.6916033011869436, + "pass@2": 0.7694590557022359, + "pass@4": 0.8216973023958051, + "pass@8": 0.8562871839082323, + "pass@16": 0.8799991156323995, + "pass@32": 0.8985396322425133, + "pass@64": 0.91363652395897, + "pass@128": 0.9258160237388724 + }, + "15": { + "pass@1": 0.5952449845679012, + "pass@2": 0.6762832488820844, + "pass@4": 0.736550903937933, + "pass@8": 0.7814495701393551, + "pass@16": 0.8134452792685088, + "pass@32": 0.836684429948149, + "pass@64": 0.8557844677332779, + "pass@128": 0.8734567901234568 + }, + "16": { + "pass@1": 0.4019174041297935, + "pass@2": 0.504326426149165, + "pass@4": 0.5959018344498971, + "pass@8": 0.6727407848216501, + "pass@16": 0.7339622740188452, + "pass@32": 0.7819087110814815, + "pass@64": 0.8193009236565792, + "pass@128": 0.8436578171091446 + }, + "17": { + "pass@1": 0.31741810993975905, + "pass@2": 0.40953553386775415, + "pass@4": 0.48704305015336896, + "pass@8": 0.5499816842368038, + "pass@16": 0.604508368845186, + "pass@32": 0.6564436911909359, + "pass@64": 0.7013562136970772, + "pass@128": 0.7349397590361446 + }, + "18": { + "pass@1": 0.317741935483871, + "pass@2": 0.40378897320294627, + "pass@4": 0.47759907864339524, + "pass@8": 0.5356404084271387, + "pass@16": 0.581367839016435, + "pass@32": 0.6213430758174604, + "pass@64": 0.6621726039803202, + "pass@128": 0.7096774193548387 + }, + "19": { + "pass@1": 0.20104796072507553, + "pass@2": 0.277813667126579, + "pass@4": 0.3523299567727752, + "pass@8": 0.42098742880752116, + "pass@16": 0.48204430116010694, + "pass@32": 0.5352011641883591, + "pass@64": 0.5813005114434726, + "pass@128": 0.6193353474320241 + }, + "20": { + "pass@1": 0.1936233108108108, + "pass@2": 0.268572952874134, + "pass@4": 0.34309407523383884, + "pass@8": 0.412354850640087, + "pass@16": 0.4758931598412825, + "pass@32": 0.5322037216710581, + "pass@64": 0.5776411084860688, + "pass@128": 0.6126126126126126 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_metrics.json b/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..111712b8b8716771d8fd395dbf8508dcd7d34ab5 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch4-op11-14", + "id": { + "answer_accuracy": 0.9928567708333333, + "avg_response_len": 199.79739930555556, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9928567708333333, + "pass@2": 0.9948959700349949, + "pass@4": 0.9962162588843063, + "pass@8": 0.996999282682993, + "pass@16": 0.9974741171145197, + "pass@32": 0.9978465770581726, + "pass@64": 0.9982309070418071, + "pass@128": 0.9986666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9539296875, + "pass@2": 0.9635426919291339, + "pass@4": 0.9707023041807276, + "pass@8": 0.975490160066206, + "pass@16": 0.9789009518866644, + "pass@32": 0.9818573785913061, + "pass@64": 0.9851069875075766, + "pass@128": 0.989 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999453125, + "pass@2": 0.9999987696850394, + "pass@4": 0.9999999995313085, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99925, + "pass@2": 0.9996391486220473, + "pass@4": 0.9998796986314211, + "pass@8": 0.9999868113249217, + "pass@16": 0.9999998839756833, + "pass@32": 0.9999999999981759, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956640625, + "pass@2": 0.9981594488188975, + "pass@4": 0.9994434281964756, + "pass@8": 0.9998959286259774, + "pass@16": 0.9999928848842898, + "pass@32": 0.9999999799014614, + "pass@64": 0.9999999999999988, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999015625, + "pass@2": 0.9996764271653543, + "pass@4": 0.9999498568616423, + "pass@8": 0.9999984146505028, + "pass@16": 0.9999999988243893, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997109375, + "pass@2": 0.9985327263779529, + "pass@4": 0.999450328927634, + "pass@8": 0.999885866296035, + "pass@16": 0.9999927570983452, + "pass@32": 0.9999999799000695, + "pass@64": 0.9999999999999988, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9948828125, + "pass@2": 0.9972484005905513, + "pass@4": 0.9986864627859019, + "pass@8": 0.9995349095660551, + "pass@16": 0.9998925640782925, + "pass@32": 0.9999928988888469, + "pass@64": 0.9999999947663261, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9959140625, + "pass@2": 0.9972661171259843, + "pass@4": 0.9978342508436445, + "pass@8": 0.9982014536172431, + "pass@16": 0.9984880132830111, + "pass@32": 0.9987689562436927, + "pass@64": 0.9989711811023622, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.9539296875, + "2": 1.0, + "3": 0.9999453125, + "4": 0.99925, + "5": 0.9956640625, + "6": 0.999015625, + "7": 0.997109375, + "8": 0.9948828125, + "9": 0.9959140625 + }, + "per_op_avg_response_len": { + "10": 280.2973984375, + "2": 115.507609375, + "3": 144.67925, + "4": 159.7131171875, + "5": 172.8602109375, + "6": 198.759671875, + "7": 223.891453125, + "8": 243.404640625, + "9": 259.0632421875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.2973984375, + "2": 115.507609375, + "3": 144.67925, + "4": 159.7131171875, + "5": 172.8602109375, + "6": 198.759671875, + "7": 223.891453125, + "8": 243.404640625, + "9": 259.0632421875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376539, + "answer_accuracy": 0.9934856256332321, + "avg_response_len": 190.24900002110775, + "resp_tokens_sum": 72105893, + "pass_at_k": { + "pass@1": 0.9934856256332321, + "pass@2": 0.9952370897121903, + "pass@4": 0.9963667890044646, + "pass@8": 0.99714430403211, + "pass@16": 0.9976664856081482, + "pass@32": 0.9979998351197172, + "pass@64": 0.9983126058744111, + "pass@128": 0.9986491050320837 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9555217551622419, + "pass@2": 0.9649799375188721, + "pass@4": 0.9725824357685375, + "pass@8": 0.9783204632700988, + "pass@16": 0.9825890111220292, + "pass@32": 0.9854794451027657, + "pass@64": 0.9882112861183845, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99765625, + "pass@2": 0.9988723394438976, + "pass@4": 0.9996240582231908, + "pass@8": 0.9999587853903806, + "pass@16": 0.99999963742401, + "pass@32": 0.9999999999942999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956597222222222, + "pass@2": 0.997699506311711, + "pass@4": 0.9989479258545063, + "pass@8": 0.999704562830816, + "pass@16": 0.9999774766429096, + "pass@32": 0.9999999361951407, + "pass@64": 0.999999999999996, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9985669378698225, + "pass@2": 0.999648741438755, + "pass@4": 0.999965682636564, + "pass@8": 0.9999996999237872, + "pass@16": 0.999999999991595, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9986227964743589, + "pass@2": 0.9999313862810418, + "pass@4": 0.9999997707618278, + "pass@8": 0.9999999999988657, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9971675030497948, + "pass@4": 0.9971830761471716, + "pass@8": 0.9971830985915474, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387419, + "answer_accuracy": 0.9917139375819135, + "avg_response_len": 197.73270089285714, + "resp_tokens_sum": 77245466, + "pass_at_k": { + "pass@1": 0.9917139375819135, + "pass@2": 0.9937211609658313, + "pass@4": 0.9950361093870473, + "pass@8": 0.9958125968364286, + "pass@16": 0.9963626120368996, + "pass@32": 0.9969249335365784, + "pass@64": 0.9974844614531034, + "pass@128": 0.9980340760157274 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9392973542944786, + "pass@2": 0.9498639105598762, + "pass@4": 0.9579157637887292, + "pass@8": 0.9633439750573325, + "pass@16": 0.967524339629329, + "pass@32": 0.9719200985373748, + "pass@64": 0.9765380222469652, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9988224637681159, + "pass@2": 0.9997874586328882, + "pass@4": 0.9999922303190363, + "pass@8": 0.9999999910428016, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985449498567335, + "pass@2": 0.9996760288112266, + "pass@4": 0.9999785661641866, + "pass@8": 0.9999999015825531, + "pass@16": 0.9999999999993172, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960711705202312, + "pass@2": 0.9974437184242867, + "pass@4": 0.9988051376164685, + "pass@8": 0.9997047707744012, + "pass@16": 0.9999793315061124, + "pass@32": 0.9999999419107679, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9960607394366198, + "pass@2": 0.9983739048464012, + "pass@4": 0.9996422492963027, + "pass@8": 0.9999796729938163, + "pass@16": 0.999999939849747, + "pass@32": 0.99999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9945499625748503, + "pass@2": 0.9964954441510679, + "pass@4": 0.9973660230158355, + "pass@8": 0.9978422530337244, + "pass@16": 0.9984818259578424, + "pass@32": 0.9993082788304156, + "pass@64": 0.999913715875336, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379813, + "answer_accuracy": 0.9934010922330098, + "avg_response_len": 211.37231388098425, + "resp_tokens_sum": 80815245, + "pass_at_k": { + "pass@1": 0.9934010922330098, + "pass@2": 0.9957581936422663, + "pass@4": 0.9972728697245619, + "pass@8": 0.9980680329303928, + "pass@16": 0.9984191155702485, + "pass@32": 0.9986363523871554, + "pass@64": 0.9989126083117708, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9665578358208955, + "pass@2": 0.9753995769185567, + "pass@4": 0.9812427446569179, + "pass@8": 0.984445931787918, + "pass@16": 0.9862398280511495, + "pass@32": 0.9878623718754762, + "pass@64": 0.9903043768983087, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998633381924198, + "pass@2": 0.9999964130759167, + "pass@4": 0.9999999986335528, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9924632352941176, + "pass@2": 0.9969336208893007, + "pass@4": 0.9993456238007014, + "pass@8": 0.9999676306603055, + "pass@16": 0.9999999404169826, + "pass@32": 0.9999999999999771, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9984966482649842, + "pass@2": 0.9993359404108397, + "pass@4": 0.9998654172565968, + "pass@8": 0.9999951072498166, + "pass@16": 0.9999999962922006, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9966871044303798, + "pass@2": 0.9985314088507925, + "pass@4": 0.9996055398138525, + "pass@8": 0.9999623956767472, + "pass@16": 0.999999710126808, + "pass@32": 0.9999999999966581, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9901229354354354, + "pass@2": 0.9935347503014826, + "pass@4": 0.9964370444685404, + "pass@8": 0.9986250019627753, + "pass@16": 0.9996774337286254, + "pass@32": 0.9999786753419787, + "pass@64": 0.9999999842832614, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9965082395498392, + "pass@2": 0.9982063511152746, + "pass@4": 0.9990804087591943, + "pass@8": 0.9997496498520251, + "pass@16": 0.9999841910388799, + "pass@32": 0.999999971428726, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.691875, + "avg_response_len": 372.99815546875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.691875, + "pass@2": 0.7569655265747888, + "pass@4": 0.8055117342988305, + "pass@8": 0.8415277440961516, + "pass@16": 0.8685987369738101, + "pass@32": 0.8897270269295073, + "pass@64": 0.9066655363534859, + "pass@128": 0.9202 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9625703125, + "pass@2": 0.9716028543307089, + "pass@4": 0.977065253562055, + "pass@8": 0.9805981358009112, + "pass@16": 0.9828878559740792, + "pass@32": 0.9848897753040814, + "pass@64": 0.9870662823610318, + "pass@128": 0.989 + }, + "12": { + "pass@1": 0.91546875, + "pass@2": 0.9322356053149612, + "pass@4": 0.9421446911323584, + "pass@8": 0.9489884448415973, + "pass@16": 0.9547641910879968, + "pass@32": 0.9604217627862317, + "pass@64": 0.9663588439701425, + "pass@128": 0.971 + }, + "13": { + "pass@1": 0.85984375, + "pass@2": 0.8906690452755908, + "pass@4": 0.9091363452380952, + "pass@8": 0.9214996283577351, + "pass@16": 0.9320471191065713, + "pass@32": 0.9430140808713036, + "pass@64": 0.953760160490955, + "pass@128": 0.964 + }, + "14": { + "pass@1": 0.866953125, + "pass@2": 0.9020903051181108, + "pass@4": 0.9213187425009368, + "pass@8": 0.9334330127351448, + "pass@16": 0.9427593873254092, + "pass@32": 0.9511044705782022, + "pass@64": 0.9583887065127129, + "pass@128": 0.965 + }, + "15": { + "pass@1": 0.82159375, + "pass@2": 0.87338939468504, + "pass@4": 0.9061740229658797, + "pass@8": 0.9284084065232308, + "pass@16": 0.9437730707115572, + "pass@32": 0.9545539010961099, + "pass@64": 0.9619371784947097, + "pass@128": 0.967 + }, + "16": { + "pass@1": 0.66428125, + "pass@2": 0.7492256397637792, + "pass@4": 0.8064780830521187, + "pass@8": 0.8457528599909511, + "pass@16": 0.872940244032758, + "pass@32": 0.8926041664005118, + "pass@64": 0.9079683868720971, + "pass@128": 0.92 + }, + "17": { + "pass@1": 0.603, + "pass@2": 0.6990440452755908, + "pass@4": 0.7653448331458559, + "pass@8": 0.8106834321313242, + "pass@16": 0.8440055876598439, + "pass@32": 0.8700455778571285, + "pass@64": 0.8908740562835517, + "pass@128": 0.908 + }, + "18": { + "pass@1": 0.53840625, + "pass@2": 0.6397306840551189, + "pass@4": 0.7131535387139104, + "pass@8": 0.7656584292259977, + "pass@16": 0.805760181584611, + "pass@32": 0.838258166150287, + "pass@64": 0.8651842279438802, + "pass@128": 0.887 + }, + "19": { + "pass@1": 0.3564453125, + "pass@2": 0.4690118110236226, + "pass@4": 0.5694118413010871, + "pass@8": 0.6518199874635647, + "pass@16": 0.7169103192164673, + "pass@32": 0.7679376199362757, + "pass@64": 0.8073694336016357, + "pass@128": 0.838 + }, + "20": { + "pass@1": 0.3301875, + "pass@2": 0.4426558809055117, + "pass@4": 0.5448899913760785, + "pass@8": 0.6284351038911395, + "pass@16": 0.6901394130387092, + "pass@32": 0.7344407483148729, + "pass@64": 0.7677480870040734, + "pass@128": 0.793 + } + }, + "per_op_accuracy": { + "11": 0.9625703125, + "12": 0.91546875, + "13": 0.85984375, + "14": 0.866953125, + "15": 0.82159375, + "16": 0.66428125, + "17": 0.603, + "18": 0.53840625, + "19": 0.3564453125, + "20": 0.3301875 + }, + "per_op_avg_response_len": { + "11": 298.7228203125, + "12": 309.24153125, + "13": 328.715984375, + "14": 356.26971875, + "15": 375.9165859375, + "16": 382.2519140625, + "17": 404.6203515625, + "18": 417.548328125, + "19": 420.9606640625, + "20": 435.73365625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.7228203125, + "12": 309.24153125, + "13": 328.715984375, + "14": 356.26971875, + "15": 375.9165859375, + "16": 382.2519140625, + "17": 404.6203515625, + "18": 417.548328125, + "19": 420.9606640625, + "20": 435.73365625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 300147, + "answer_accuracy": 0.6968494613670134, + "avg_response_len": 355.2223834509658, + "resp_tokens_sum": 153001385, + "pass_at_k": { + "pass@1": 0.6968494613670134, + "pass@2": 0.7615124940038193, + "pass@4": 0.8079247967425347, + "pass@8": 0.8413539091928377, + "pass@16": 0.8668610172246366, + "pass@32": 0.8874146419722095, + "pass@64": 0.9039114091381322, + "pass@128": 0.9170876671619613 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9595432011331445, + "pass@2": 0.9712151608262141, + "pass@4": 0.9783531306993142, + "pass@8": 0.983326614921076, + "pass@16": 0.9865718492617972, + "pass@32": 0.9888296194765508, + "pass@64": 0.9906259618168388, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.906007238700565, + "pass@2": 0.9247933487922062, + "pass@4": 0.9357417314361123, + "pass@8": 0.9421184863804423, + "pass@16": 0.9467630723566625, + "pass@32": 0.9518291711617118, + "pass@64": 0.9579773195275365, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8717672413793104, + "pass@2": 0.9035996161725869, + "pass@4": 0.9226742165654058, + "pass@8": 0.9340267016365622, + "pass@16": 0.9431541830177079, + "pass@32": 0.9532836501149419, + "pass@64": 0.9644626458159502, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8635854750778816, + "pass@2": 0.8954251447249002, + "pass@4": 0.9113720040251972, + "pass@8": 0.9206645009701866, + "pass@16": 0.9286928898625472, + "pass@32": 0.9369032185178561, + "pass@64": 0.9440107739042536, + "pass@128": 0.9501557632398754 + }, + "15": { + "pass@1": 0.8376986994219653, + "pass@2": 0.8865166014291564, + "pass@4": 0.9181329861765831, + "pass@8": 0.9412534150029087, + "pass@16": 0.9562649099753139, + "pass@32": 0.9651885106215669, + "pass@64": 0.9706079061795896, + "pass@128": 0.9739884393063584 + }, + "16": { + "pass@1": 0.6686743951612903, + "pass@2": 0.7454768065786131, + "pass@4": 0.7934225168426517, + "pass@8": 0.8257491148636001, + "pass@16": 0.8513642598353691, + "pass@32": 0.8730630130473881, + "pass@64": 0.8910014008528852, + "pass@128": 0.9064516129032258 + }, + "17": { + "pass@1": 0.6197841401734104, + "pass@2": 0.7161787839652269, + "pass@4": 0.7794292514375006, + "pass@8": 0.8202572762642997, + "pass@16": 0.8505096438381178, + "pass@32": 0.8753376105726489, + "pass@64": 0.8944587031637984, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.5136607142857142, + "pass@2": 0.6187155511811023, + "pass@4": 0.6944143609727348, + "pass@8": 0.7489642516122974, + "pass@16": 0.7936353786265803, + "pass@32": 0.8323554499320751, + "pass@64": 0.8633710891209031, + "pass@128": 0.8857142857142857 + }, + "19": { + "pass@1": 0.37058933933933935, + "pass@2": 0.48287460689035483, + "pass@4": 0.5762031899728748, + "pass@8": 0.6492350344930701, + "pass@16": 0.706125200061887, + "pass@32": 0.7499946710222914, + "pass@64": 0.7842006467978939, + "pass@128": 0.8138138138138138 + }, + "20": { + "pass@1": 0.34649493243243246, + "pass@2": 0.46130899600387765, + "pass@4": 0.5613634281075223, + "pass@8": 0.6403788685222684, + "pass@16": 0.6985517724335117, + "pass@32": 0.7410158124161551, + "pass@64": 0.7727139219338052, + "pass@128": 0.7957957957957958 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 302380, + "answer_accuracy": 0.7024513083556349, + "avg_response_len": 368.4694585377639, + "resp_tokens_sum": 158612837, + "pass_at_k": { + "pass@1": 0.7024513083556349, + "pass@2": 0.7656160003722866, + "pass@4": 0.8130046151122302, + "pass@8": 0.8491108703695945, + "pass@16": 0.8761795320620611, + "pass@32": 0.8966491248930549, + "pass@64": 0.9127342481802431, + "pass@128": 0.9259589652096343 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9681852409638554, + "pass@2": 0.9741481921781614, + "pass@4": 0.9772397477988446, + "pass@8": 0.9789669965699488, + "pass@16": 0.9799547315679026, + "pass@32": 0.9809906007078771, + "pass@64": 0.9826866521200436, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9085276253918495, + "pass@2": 0.9235974409448813, + "pass@4": 0.9330263857104065, + "pass@8": 0.9409230257953107, + "pass@16": 0.9477558425747747, + "pass@32": 0.9527120358305108, + "pass@64": 0.95642424625744, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.8668154761904762, + "pass@2": 0.8926307235492627, + "pass@4": 0.9088578126788774, + "pass@8": 0.9216598215625059, + "pass@16": 0.9320716043391158, + "pass@32": 0.941360763388727, + "pass@64": 0.950378253308469, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.881281980994152, + "pass@2": 0.9201054329557498, + "pass@4": 0.9419351528427365, + "pass@8": 0.9547325691053806, + "pass@16": 0.9625703344338206, + "pass@32": 0.9680469863909118, + "pass@64": 0.9725173602632785, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8279829545454546, + "pass@2": 0.8793225065616795, + "pass@4": 0.9134771270068515, + "pass@8": 0.9379655607421947, + "pass@16": 0.9560091862643495, + "pass@32": 0.9682017448322129, + "pass@64": 0.9747783770514524, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.6672008547008547, + "pass@2": 0.7537498317517997, + "pass@4": 0.8141053287142518, + "pass@8": 0.8573749750163748, + "pass@16": 0.886024033601711, + "pass@32": 0.9049191024077768, + "pass@64": 0.9190486363240371, + "pass@128": 0.9316239316239316 + }, + "17": { + "pass@1": 0.6349718555900621, + "pass@2": 0.7272980143786375, + "pass@4": 0.7871718557501735, + "pass@8": 0.8293996571531411, + "pass@16": 0.8622377160167927, + "pass@32": 0.8877773954413081, + "pass@64": 0.9070769693130836, + "pass@128": 0.922360248447205 + }, + "18": { + "pass@1": 0.5722426470588236, + "pass@2": 0.6690550457387674, + "pass@4": 0.7391316479741499, + "pass@8": 0.7903393556142831, + "pass@16": 0.8293274169491589, + "pass@32": 0.8609394488494504, + "pass@64": 0.8900216102839796, + "pass@128": 0.9176470588235294 + }, + "19": { + "pass@1": 0.36755952380952384, + "pass@2": 0.4771494627624668, + "pass@4": 0.5758954558693551, + "pass@8": 0.6572878262769214, + "pass@16": 0.7219972024993643, + "pass@32": 0.7752524575772055, + "pass@64": 0.8196462374053157, + "pass@128": 0.8541666666666666 + }, + "20": { + "pass@1": 0.3302067739520958, + "pass@2": 0.4390511545806021, + "pass@4": 0.5388564448717861, + "pass@8": 0.6219737802822332, + "pass@16": 0.6834840446587457, + "pass@32": 0.7260101372704013, + "pass@64": 0.7543047807447868, + "pass@128": 0.7724550898203593 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 283073, + "answer_accuracy": 0.6758886957518337, + "avg_response_len": 395.9338158045538, + "resp_tokens_sum": 165823417, + "pass_at_k": { + "pass@1": 0.6758886957518337, + "pass@2": 0.7433982622778896, + "pass@4": 0.7953288145865972, + "pass@8": 0.8339124937270136, + "pass@16": 0.8625942177421915, + "pass@32": 0.8849905232405783, + "pass@64": 0.903260450841926, + "pass@128": 0.9174816625916871 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9600446428571429, + "pass@2": 0.9693546119235098, + "pass@4": 0.9754381020824777, + "pass@8": 0.9792596757858676, + "pass@16": 0.9818508644575293, + "pass@32": 0.9845842545836299, + "pass@64": 0.9876931724947077, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9324827981651376, + "pass@2": 0.9487191932143805, + "pass@4": 0.9579715631876291, + "pass@8": 0.964293747505247, + "pass@16": 0.9702628430959813, + "pass@32": 0.9772449442356358, + "pass@64": 0.9851239703402783, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8404224537037037, + "pass@2": 0.8757765413871874, + "pass@4": 0.8961142932943567, + "pass@8": 0.908989364931657, + "pass@16": 0.9210844814038844, + "pass@32": 0.9347247035643859, + "pass@64": 0.9469491976683431, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8556194362017804, + "pass@2": 0.8901566041262641, + "pass@4": 0.9098709404647859, + "pass@8": 0.9239797308298969, + "pass@16": 0.9360531611369882, + "pass@32": 0.9474375907664061, + "pass@64": 0.957745729612481, + "pass@128": 0.9673590504451038 + }, + "15": { + "pass@1": 0.7978877314814815, + "pass@2": 0.8533278500777681, + "pass@4": 0.8859646846806184, + "pass@8": 0.9049570675533932, + "pass@16": 0.9179703098545164, + "pass@32": 0.9292966068716593, + "pass@64": 0.939598699165408, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.6572409660766961, + "pass@2": 0.7479694359278098, + "pass@4": 0.8105195647557326, + "pass@8": 0.8520118529571892, + "pass@16": 0.879123562506178, + "pass@32": 0.8977227947218035, + "pass@64": 0.9120114491385409, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.5544992469879518, + "pass@2": 0.6537838114268096, + "pass@4": 0.7294969114365216, + "pass@8": 0.7825533883751986, + "pass@16": 0.8195442961880983, + "pass@32": 0.8473326604424434, + "pass@64": 0.871423376114652, + "pass@128": 0.8945783132530121 + }, + "18": { + "pass@1": 0.5292338709677419, + "pass@2": 0.6312952438404874, + "pass@4": 0.7058185550394909, + "pass@8": 0.7574372911381855, + "pass@16": 0.7936015396857868, + "pass@32": 0.8200462776298317, + "pass@64": 0.8399903201774559, + "pass@128": 0.8548387096774194 + }, + "19": { + "pass@1": 0.3309337235649547, + "pass@2": 0.4468046750481719, + "pass@4": 0.5559979029245206, + "pass@8": 0.6488701249194181, + "pass@16": 0.7225968808340559, + "pass@32": 0.7785636517822091, + "pass@64": 0.8182159590626885, + "pass@128": 0.8459214501510574 + }, + "20": { + "pass@1": 0.31386073573573575, + "pass@2": 0.4276183171360337, + "pass@4": 0.5344682199071957, + "pass@8": 0.6229720662431183, + "pass@16": 0.6884024081151023, + "pass@32": 0.7363216124083445, + "pass@64": 0.776265928622691, + "pass@128": 0.8108108108108109 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.8344453125, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8344453125, + "pass@2": 0.8696694208454039, + "pass@4": 0.8958454564708882, + "pass@8": 0.9151721571109854, + "pass@16": 0.9296449696720248, + "pass@32": 0.9409415506746711, + "pass@64": 0.9500386066795303, + "pass@128": 0.9573684210526315 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9539296875, + "pass@2": 0.9635426919291339, + "pass@4": 0.9707023041807276, + "pass@8": 0.975490160066206, + "pass@16": 0.9789009518866644, + "pass@32": 0.9818573785913061, + "pass@64": 0.9851069875075766, + "pass@128": 0.989 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999453125, + "pass@2": 0.9999987696850394, + "pass@4": 0.9999999995313085, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99925, + "pass@2": 0.9996391486220473, + "pass@4": 0.9998796986314211, + "pass@8": 0.9999868113249217, + "pass@16": 0.9999998839756833, + "pass@32": 0.9999999999981759, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956640625, + "pass@2": 0.9981594488188975, + "pass@4": 0.9994434281964756, + "pass@8": 0.9998959286259774, + "pass@16": 0.9999928848842898, + "pass@32": 0.9999999799014614, + "pass@64": 0.9999999999999988, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999015625, + "pass@2": 0.9996764271653543, + "pass@4": 0.9999498568616423, + "pass@8": 0.9999984146505028, + "pass@16": 0.9999999988243893, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.997109375, + "pass@2": 0.9985327263779529, + "pass@4": 0.999450328927634, + "pass@8": 0.999885866296035, + "pass@16": 0.9999927570983452, + "pass@32": 0.9999999799000695, + "pass@64": 0.9999999999999988, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9948828125, + "pass@2": 0.9972484005905513, + "pass@4": 0.9986864627859019, + "pass@8": 0.9995349095660551, + "pass@16": 0.9998925640782925, + "pass@32": 0.9999928988888469, + "pass@64": 0.9999999947663261, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9959140625, + "pass@2": 0.9972661171259843, + "pass@4": 0.9978342508436445, + "pass@8": 0.9982014536172431, + "pass@16": 0.9984880132830111, + "pass@32": 0.9987689562436927, + "pass@64": 0.9989711811023622, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9625703125, + "pass@2": 0.9716028543307089, + "pass@4": 0.977065253562055, + "pass@8": 0.9805981358009112, + "pass@16": 0.9828878559740792, + "pass@32": 0.9848897753040814, + "pass@64": 0.9870662823610318, + "pass@128": 0.989 + }, + "12": { + "pass@1": 0.91546875, + "pass@2": 0.9322356053149612, + "pass@4": 0.9421446911323584, + "pass@8": 0.9489884448415973, + "pass@16": 0.9547641910879968, + "pass@32": 0.9604217627862317, + "pass@64": 0.9663588439701425, + "pass@128": 0.971 + }, + "13": { + "pass@1": 0.85984375, + "pass@2": 0.8906690452755908, + "pass@4": 0.9091363452380952, + "pass@8": 0.9214996283577351, + "pass@16": 0.9320471191065713, + "pass@32": 0.9430140808713036, + "pass@64": 0.953760160490955, + "pass@128": 0.964 + }, + "14": { + "pass@1": 0.866953125, + "pass@2": 0.9020903051181108, + "pass@4": 0.9213187425009368, + "pass@8": 0.9334330127351448, + "pass@16": 0.9427593873254092, + "pass@32": 0.9511044705782022, + "pass@64": 0.9583887065127129, + "pass@128": 0.965 + }, + "15": { + "pass@1": 0.82159375, + "pass@2": 0.87338939468504, + "pass@4": 0.9061740229658797, + "pass@8": 0.9284084065232308, + "pass@16": 0.9437730707115572, + "pass@32": 0.9545539010961099, + "pass@64": 0.9619371784947097, + "pass@128": 0.967 + }, + "16": { + "pass@1": 0.66428125, + "pass@2": 0.7492256397637792, + "pass@4": 0.8064780830521187, + "pass@8": 0.8457528599909511, + "pass@16": 0.872940244032758, + "pass@32": 0.8926041664005118, + "pass@64": 0.9079683868720971, + "pass@128": 0.92 + }, + "17": { + "pass@1": 0.603, + "pass@2": 0.6990440452755908, + "pass@4": 0.7653448331458559, + "pass@8": 0.8106834321313242, + "pass@16": 0.8440055876598439, + "pass@32": 0.8700455778571285, + "pass@64": 0.8908740562835517, + "pass@128": 0.908 + }, + "18": { + "pass@1": 0.53840625, + "pass@2": 0.6397306840551189, + "pass@4": 0.7131535387139104, + "pass@8": 0.7656584292259977, + "pass@16": 0.805760181584611, + "pass@32": 0.838258166150287, + "pass@64": 0.8651842279438802, + "pass@128": 0.887 + }, + "19": { + "pass@1": 0.3564453125, + "pass@2": 0.4690118110236226, + "pass@4": 0.5694118413010871, + "pass@8": 0.6518199874635647, + "pass@16": 0.7169103192164673, + "pass@32": 0.7679376199362757, + "pass@64": 0.8073694336016357, + "pass@128": 0.838 + }, + "20": { + "pass@1": 0.3301875, + "pass@2": 0.4426558809055117, + "pass@4": 0.5448899913760785, + "pass@8": 0.6284351038911395, + "pass@16": 0.6901394130387092, + "pass@32": 0.7344407483148729, + "pass@64": 0.7677480870040734, + "pass@128": 0.793 + } + }, + "per_op_accuracy": { + "10": 0.9539296875, + "2": 1.0, + "3": 0.9999453125, + "4": 0.99925, + "5": 0.9956640625, + "6": 0.999015625, + "7": 0.997109375, + "8": 0.9948828125, + "9": 0.9959140625, + "11": 0.9625703125, + "12": 0.91546875, + "13": 0.85984375, + "14": 0.866953125, + "15": 0.82159375, + "16": 0.66428125, + "17": 0.603, + "18": 0.53840625, + "19": 0.3564453125, + "20": 0.3301875 + }, + "per_op_avg_response_len": { + "10": 280.2973984375, + "2": 115.507609375, + "3": 144.67925, + "4": 159.7131171875, + "5": 172.8602109375, + "6": 198.759671875, + "7": 223.891453125, + "8": 243.404640625, + "9": 259.0632421875, + "11": 298.7228203125, + "12": 309.24153125, + "13": 328.715984375, + "14": 356.26971875, + "15": 375.9165859375, + "16": 382.2519140625, + "17": 404.6203515625, + "18": 417.548328125, + "19": 420.9606640625, + "20": 435.73365625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.2973984375, + "2": 115.507609375, + "3": 144.67925, + "4": 159.7131171875, + "5": 172.8602109375, + "6": 198.759671875, + "7": 223.891453125, + "8": 243.404640625, + "9": 259.0632421875, + "11": 298.7228203125, + "12": 309.24153125, + "13": 328.715984375, + "14": 356.26971875, + "15": 375.9165859375, + "16": 382.2519140625, + "17": 404.6203515625, + "18": 417.548328125, + "19": 420.9606640625, + "20": 435.73365625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 676686, + "answer_accuracy": 0.8356954434081568, + "avg_response_len": 278.0035740396775, + "resp_tokens_sum": 225107278, + "pass_at_k": { + "pass@1": 0.8356954434081568, + "pass@2": 0.8709115657541193, + "pass@4": 0.8961285177491005, + "pass@8": 0.9142744528411244, + "pass@16": 0.9280869090810386, + "pass@32": 0.939176064183682, + "pass@64": 0.948097615830537, + "pass@128": 0.9552639898830224 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9555217551622419, + "pass@2": 0.9649799375188721, + "pass@4": 0.9725824357685375, + "pass@8": 0.9783204632700988, + "pass@16": 0.9825890111220292, + "pass@32": 0.9854794451027657, + "pass@64": 0.9882112861183845, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99765625, + "pass@2": 0.9988723394438976, + "pass@4": 0.9996240582231908, + "pass@8": 0.9999587853903806, + "pass@16": 0.99999963742401, + "pass@32": 0.9999999999942999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956597222222222, + "pass@2": 0.997699506311711, + "pass@4": 0.9989479258545063, + "pass@8": 0.999704562830816, + "pass@16": 0.9999774766429096, + "pass@32": 0.9999999361951407, + "pass@64": 0.999999999999996, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9985669378698225, + "pass@2": 0.999648741438755, + "pass@4": 0.999965682636564, + "pass@8": 0.9999996999237872, + "pass@16": 0.999999999991595, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9986227964743589, + "pass@2": 0.9999313862810418, + "pass@4": 0.9999997707618278, + "pass@8": 0.9999999999988657, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9971675030497948, + "pass@4": 0.9971830761471716, + "pass@8": 0.9971830985915474, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9595432011331445, + "pass@2": 0.9712151608262141, + "pass@4": 0.9783531306993142, + "pass@8": 0.983326614921076, + "pass@16": 0.9865718492617972, + "pass@32": 0.9888296194765508, + "pass@64": 0.9906259618168388, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.906007238700565, + "pass@2": 0.9247933487922062, + "pass@4": 0.9357417314361123, + "pass@8": 0.9421184863804423, + "pass@16": 0.9467630723566625, + "pass@32": 0.9518291711617118, + "pass@64": 0.9579773195275365, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8717672413793104, + "pass@2": 0.9035996161725869, + "pass@4": 0.9226742165654058, + "pass@8": 0.9340267016365622, + "pass@16": 0.9431541830177079, + "pass@32": 0.9532836501149419, + "pass@64": 0.9644626458159502, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8635854750778816, + "pass@2": 0.8954251447249002, + "pass@4": 0.9113720040251972, + "pass@8": 0.9206645009701866, + "pass@16": 0.9286928898625472, + "pass@32": 0.9369032185178561, + "pass@64": 0.9440107739042536, + "pass@128": 0.9501557632398754 + }, + "15": { + "pass@1": 0.8376986994219653, + "pass@2": 0.8865166014291564, + "pass@4": 0.9181329861765831, + "pass@8": 0.9412534150029087, + "pass@16": 0.9562649099753139, + "pass@32": 0.9651885106215669, + "pass@64": 0.9706079061795896, + "pass@128": 0.9739884393063584 + }, + "16": { + "pass@1": 0.6686743951612903, + "pass@2": 0.7454768065786131, + "pass@4": 0.7934225168426517, + "pass@8": 0.8257491148636001, + "pass@16": 0.8513642598353691, + "pass@32": 0.8730630130473881, + "pass@64": 0.8910014008528852, + "pass@128": 0.9064516129032258 + }, + "17": { + "pass@1": 0.6197841401734104, + "pass@2": 0.7161787839652269, + "pass@4": 0.7794292514375006, + "pass@8": 0.8202572762642997, + "pass@16": 0.8505096438381178, + "pass@32": 0.8753376105726489, + "pass@64": 0.8944587031637984, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.5136607142857142, + "pass@2": 0.6187155511811023, + "pass@4": 0.6944143609727348, + "pass@8": 0.7489642516122974, + "pass@16": 0.7936353786265803, + "pass@32": 0.8323554499320751, + "pass@64": 0.8633710891209031, + "pass@128": 0.8857142857142857 + }, + "19": { + "pass@1": 0.37058933933933935, + "pass@2": 0.48287460689035483, + "pass@4": 0.5762031899728748, + "pass@8": 0.6492350344930701, + "pass@16": 0.706125200061887, + "pass@32": 0.7499946710222914, + "pass@64": 0.7842006467978939, + "pass@128": 0.8138138138138138 + }, + "20": { + "pass@1": 0.34649493243243246, + "pass@2": 0.46130899600387765, + "pass@4": 0.5613634281075223, + "pass@8": 0.6403788685222684, + "pass@16": 0.6985517724335117, + "pass@32": 0.7410158124161551, + "pass@64": 0.7727139219338052, + "pass@128": 0.7957957957957958 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 689799, + "answer_accuracy": 0.8400708787996882, + "avg_response_len": 287.2397493667186, + "resp_tokens_sum": 235858303, + "pass_at_k": { + "pass@1": 0.8400708787996882, + "pass@2": 0.8741392973530203, + "pass@4": 0.8996079074780458, + "pass@8": 0.9189056746060298, + "pass@16": 0.9333578266970128, + "pass@32": 0.9443563373607163, + "pass@64": 0.9530550043624433, + "pass@128": 0.9602494154325799 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9392973542944786, + "pass@2": 0.9498639105598762, + "pass@4": 0.9579157637887292, + "pass@8": 0.9633439750573325, + "pass@16": 0.967524339629329, + "pass@32": 0.9719200985373748, + "pass@64": 0.9765380222469652, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9988224637681159, + "pass@2": 0.9997874586328882, + "pass@4": 0.9999922303190363, + "pass@8": 0.9999999910428016, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985449498567335, + "pass@2": 0.9996760288112266, + "pass@4": 0.9999785661641866, + "pass@8": 0.9999999015825531, + "pass@16": 0.9999999999993172, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960711705202312, + "pass@2": 0.9974437184242867, + "pass@4": 0.9988051376164685, + "pass@8": 0.9997047707744012, + "pass@16": 0.9999793315061124, + "pass@32": 0.9999999419107679, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9960607394366198, + "pass@2": 0.9983739048464012, + "pass@4": 0.9996422492963027, + "pass@8": 0.9999796729938163, + "pass@16": 0.999999939849747, + "pass@32": 0.99999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9945499625748503, + "pass@2": 0.9964954441510679, + "pass@4": 0.9973660230158355, + "pass@8": 0.9978422530337244, + "pass@16": 0.9984818259578424, + "pass@32": 0.9993082788304156, + "pass@64": 0.999913715875336, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9681852409638554, + "pass@2": 0.9741481921781614, + "pass@4": 0.9772397477988446, + "pass@8": 0.9789669965699488, + "pass@16": 0.9799547315679026, + "pass@32": 0.9809906007078771, + "pass@64": 0.9826866521200436, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9085276253918495, + "pass@2": 0.9235974409448813, + "pass@4": 0.9330263857104065, + "pass@8": 0.9409230257953107, + "pass@16": 0.9477558425747747, + "pass@32": 0.9527120358305108, + "pass@64": 0.95642424625744, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.8668154761904762, + "pass@2": 0.8926307235492627, + "pass@4": 0.9088578126788774, + "pass@8": 0.9216598215625059, + "pass@16": 0.9320716043391158, + "pass@32": 0.941360763388727, + "pass@64": 0.950378253308469, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.881281980994152, + "pass@2": 0.9201054329557498, + "pass@4": 0.9419351528427365, + "pass@8": 0.9547325691053806, + "pass@16": 0.9625703344338206, + "pass@32": 0.9680469863909118, + "pass@64": 0.9725173602632785, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8279829545454546, + "pass@2": 0.8793225065616795, + "pass@4": 0.9134771270068515, + "pass@8": 0.9379655607421947, + "pass@16": 0.9560091862643495, + "pass@32": 0.9682017448322129, + "pass@64": 0.9747783770514524, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.6672008547008547, + "pass@2": 0.7537498317517997, + "pass@4": 0.8141053287142518, + "pass@8": 0.8573749750163748, + "pass@16": 0.886024033601711, + "pass@32": 0.9049191024077768, + "pass@64": 0.9190486363240371, + "pass@128": 0.9316239316239316 + }, + "17": { + "pass@1": 0.6349718555900621, + "pass@2": 0.7272980143786375, + "pass@4": 0.7871718557501735, + "pass@8": 0.8293996571531411, + "pass@16": 0.8622377160167927, + "pass@32": 0.8877773954413081, + "pass@64": 0.9070769693130836, + "pass@128": 0.922360248447205 + }, + "18": { + "pass@1": 0.5722426470588236, + "pass@2": 0.6690550457387674, + "pass@4": 0.7391316479741499, + "pass@8": 0.7903393556142831, + "pass@16": 0.8293274169491589, + "pass@32": 0.8609394488494504, + "pass@64": 0.8900216102839796, + "pass@128": 0.9176470588235294 + }, + "19": { + "pass@1": 0.36755952380952384, + "pass@2": 0.4771494627624668, + "pass@4": 0.5758954558693551, + "pass@8": 0.6572878262769214, + "pass@16": 0.7219972024993643, + "pass@32": 0.7752524575772055, + "pass@64": 0.8196462374053157, + "pass@128": 0.8541666666666666 + }, + "20": { + "pass@1": 0.3302067739520958, + "pass@2": 0.4390511545806021, + "pass@4": 0.5388564448717861, + "pass@8": 0.6219737802822332, + "pass@16": 0.6834840446587457, + "pass@32": 0.7260101372704013, + "pass@64": 0.7543047807447868, + "pass@128": 0.7724550898203593 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 662886, + "answer_accuracy": 0.827416020929861, + "avg_response_len": 307.85501627656174, + "resp_tokens_sum": 246638662, + "pass_at_k": { + "pass@1": 0.827416020929861, + "pass@2": 0.8638326950922894, + "pass@4": 0.8917031383918479, + "pass@8": 0.9122528988397226, + "pass@16": 0.9274143119764798, + "pass@32": 0.9392260387639582, + "pass@64": 0.9489087963224284, + "pass@128": 0.9565425786866911 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9665578358208955, + "pass@2": 0.9753995769185567, + "pass@4": 0.9812427446569179, + "pass@8": 0.984445931787918, + "pass@16": 0.9862398280511495, + "pass@32": 0.9878623718754762, + "pass@64": 0.9903043768983087, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998633381924198, + "pass@2": 0.9999964130759167, + "pass@4": 0.9999999986335528, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9924632352941176, + "pass@2": 0.9969336208893007, + "pass@4": 0.9993456238007014, + "pass@8": 0.9999676306603055, + "pass@16": 0.9999999404169826, + "pass@32": 0.9999999999999771, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9984966482649842, + "pass@2": 0.9993359404108397, + "pass@4": 0.9998654172565968, + "pass@8": 0.9999951072498166, + "pass@16": 0.9999999962922006, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9966871044303798, + "pass@2": 0.9985314088507925, + "pass@4": 0.9996055398138525, + "pass@8": 0.9999623956767472, + "pass@16": 0.999999710126808, + "pass@32": 0.9999999999966581, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9901229354354354, + "pass@2": 0.9935347503014826, + "pass@4": 0.9964370444685404, + "pass@8": 0.9986250019627753, + "pass@16": 0.9996774337286254, + "pass@32": 0.9999786753419787, + "pass@64": 0.9999999842832614, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9965082395498392, + "pass@2": 0.9982063511152746, + "pass@4": 0.9990804087591943, + "pass@8": 0.9997496498520251, + "pass@16": 0.9999841910388799, + "pass@32": 0.999999971428726, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9600446428571429, + "pass@2": 0.9693546119235098, + "pass@4": 0.9754381020824777, + "pass@8": 0.9792596757858676, + "pass@16": 0.9818508644575293, + "pass@32": 0.9845842545836299, + "pass@64": 0.9876931724947077, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9324827981651376, + "pass@2": 0.9487191932143805, + "pass@4": 0.9579715631876291, + "pass@8": 0.964293747505247, + "pass@16": 0.9702628430959813, + "pass@32": 0.9772449442356358, + "pass@64": 0.9851239703402783, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8404224537037037, + "pass@2": 0.8757765413871874, + "pass@4": 0.8961142932943567, + "pass@8": 0.908989364931657, + "pass@16": 0.9210844814038844, + "pass@32": 0.9347247035643859, + "pass@64": 0.9469491976683431, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8556194362017804, + "pass@2": 0.8901566041262641, + "pass@4": 0.9098709404647859, + "pass@8": 0.9239797308298969, + "pass@16": 0.9360531611369882, + "pass@32": 0.9474375907664061, + "pass@64": 0.957745729612481, + "pass@128": 0.9673590504451038 + }, + "15": { + "pass@1": 0.7978877314814815, + "pass@2": 0.8533278500777681, + "pass@4": 0.8859646846806184, + "pass@8": 0.9049570675533932, + "pass@16": 0.9179703098545164, + "pass@32": 0.9292966068716593, + "pass@64": 0.939598699165408, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.6572409660766961, + "pass@2": 0.7479694359278098, + "pass@4": 0.8105195647557326, + "pass@8": 0.8520118529571892, + "pass@16": 0.879123562506178, + "pass@32": 0.8977227947218035, + "pass@64": 0.9120114491385409, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.5544992469879518, + "pass@2": 0.6537838114268096, + "pass@4": 0.7294969114365216, + "pass@8": 0.7825533883751986, + "pass@16": 0.8195442961880983, + "pass@32": 0.8473326604424434, + "pass@64": 0.871423376114652, + "pass@128": 0.8945783132530121 + }, + "18": { + "pass@1": 0.5292338709677419, + "pass@2": 0.6312952438404874, + "pass@4": 0.7058185550394909, + "pass@8": 0.7574372911381855, + "pass@16": 0.7936015396857868, + "pass@32": 0.8200462776298317, + "pass@64": 0.8399903201774559, + "pass@128": 0.8548387096774194 + }, + "19": { + "pass@1": 0.3309337235649547, + "pass@2": 0.4468046750481719, + "pass@4": 0.5559979029245206, + "pass@8": 0.6488701249194181, + "pass@16": 0.7225968808340559, + "pass@32": 0.7785636517822091, + "pass@64": 0.8182159590626885, + "pass@128": 0.8459214501510574 + }, + "20": { + "pass@1": 0.31386073573573575, + "pass@2": 0.4276183171360337, + "pass@4": 0.5344682199071957, + "pass@8": 0.6229720662431183, + "pass@16": 0.6884024081151023, + "pass@32": 0.7363216124083445, + "pass@64": 0.776265928622691, + "pass@128": 0.8108108108108109 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch4-op11-14/graphs/SDAR-100M-block4-epoch4-op11-14_score_report.jsonl b/composition/SDAR-100M-block4-epoch4-op11-14/graphs/SDAR-100M-block4-epoch4-op11-14_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6ed7b23bf1cbcecd2913164b71c337bd7dc75ec2 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op11-14/graphs/SDAR-100M-block4-epoch4-op11-14_score_report.jsonl @@ -0,0 +1,2 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch4-op11-14", "results_dir": "results/composition/SDAR-100M-block4-epoch4-op11-14/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1023675, "value_mismatch": 9563, "dependency_mismatch": 126480, "answer_mismatch": 8229, "extra_nodes": 11425, "missing_nodes": 8794, "perfect_with_extra": 930, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337592, "value_mismatch": 2927, "answer_mismatch": 2469, "dependency_mismatch": 40863, "extra_nodes": 3546, "missing_nodes": 2776, "perfect_with_extra": 261}, "teachers_in_school": {"total": 390656, "perfect": 346577, "value_mismatch": 3607, "dependency_mismatch": 43451, "answer_mismatch": 3237, "extra_nodes": 3927, "perfect_with_extra": 196, "missing_nodes": 3297}, "movie_festival_awards": {"total": 382336, "perfect": 339506, "value_mismatch": 3029, "answer_mismatch": 2523, "dependency_mismatch": 42166, "extra_nodes": 3952, "missing_nodes": 2721, "perfect_with_extra": 473}}, "results_path": "results/composition/SDAR-100M-block4-epoch4-op11-14/graphs/SDAR-100M-block4-epoch4-op11-14_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 476740, "value_mismatch": 497711, "dependency_mismatch": 711575, "answer_mismatch": 394400, "extra_nodes": 53317, "missing_nodes": 213166, "perfect_with_extra": 538, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "dependency_mismatch": 237979, "extra_nodes": 20557, "perfect": 160919, "perfect_with_extra": 209, "value_mismatch": 165410, "answer_mismatch": 130573, "missing_nodes": 75508}, "movie_festival_awards": {"total": 418816, "perfect": 152187, "value_mismatch": 170919, "dependency_mismatch": 236461, "answer_mismatch": 135743, "missing_nodes": 69320, "extra_nodes": 16869, "perfect_with_extra": 189}, "teachers_in_school": {"total": 430464, "perfect": 163634, "value_mismatch": 161382, "dependency_mismatch": 237135, "answer_mismatch": 128084, "extra_nodes": 15891, "missing_nodes": 68338, "perfect_with_extra": 140}}, "results_path": "results/composition/SDAR-100M-block4-epoch4-op11-14/graphs/SDAR-100M-block4-epoch4-op11-14_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1500415, "value_mismatch": 507274, "dependency_mismatch": 838055, "answer_mismatch": 402629, "extra_nodes": 64742, "missing_nodes": 221960, "perfect_with_extra": 1468, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch4-op11-14/graphs/SDAR-100M-block4-epoch4-op11-14_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 498511, "value_mismatch": 168337, "answer_mismatch": 133042, "dependency_mismatch": 278842, "extra_nodes": 24103, "missing_nodes": 78284, "perfect_with_extra": 470}, "teachers_in_school": {"total": 821120, "perfect": 510211, "value_mismatch": 164989, "dependency_mismatch": 280586, "answer_mismatch": 131321, "extra_nodes": 19818, "perfect_with_extra": 336, "missing_nodes": 71635}, "movie_festival_awards": {"total": 801152, "perfect": 491693, "value_mismatch": 173948, "answer_mismatch": 138266, "dependency_mismatch": 278627, "extra_nodes": 20821, "missing_nodes": 72041, "perfect_with_extra": 662}}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch4-op11-14", "results_dir": "results/composition/SDAR-100M-block4-epoch4-op11-14/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch4-op11-14/graphs/SDAR-100M-block4-epoch4-op11-14_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch4-op11-14/graphs/SDAR-100M-block4-epoch4-op11-14_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch4-op11-14/graphs/SDAR-100M-block4-epoch4-op11-14_score_report.jsonl", "template_stats": {}}} diff --git a/composition/SDAR-100M-block4-epoch4-op11-14/graphs/results.json b/composition/SDAR-100M-block4-epoch4-op11-14/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..b9e1352a33639b3754c150629d49aa02dea25ce0 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op11-14/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block4-epoch4-op11-14_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.99375, + "pass@4": 0.996875, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9650793650793651, + "pass@2": 0.9841269841269841, + "pass@4": 0.9936507936507937, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9640718562874252, + "pass@2": 0.9790419161676647, + "pass@4": 0.9970059880239521, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.893491124260355, + "pass@2": 0.9319526627218935, + "pass@4": 0.9733727810650887, + "pass@8": 0.9940828402366864, + "pass@16": 0.9970414201183432, + "pass@32": 0.9970414201183432, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8173076923076923, + "pass@2": 0.8685897435897436, + "pass@4": 0.9326923076923077, + "pass@8": 0.9775641025641025, + "pass@16": 0.9967948717948718, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7380281690140845, + "pass@2": 0.8535211267605634, + "pass@4": 0.9154929577464789, + "pass@8": 0.952112676056338, + "pass@16": 0.9774647887323944, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "10": { + "pass@1": 0.640117994100295, + "pass@2": 0.7079646017699115, + "pass@4": 0.7846607669616519, + "pass@8": 0.8731563421828908, + "pass@16": 0.911504424778761, + "pass@32": 0.9410029498525073, + "pass@64": 0.9498525073746312, + "pass@128": 0.9616519174041298 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9650793650793651, + "6": 0.9640718562874252, + "7": 0.893491124260355, + "8": 0.8173076923076923, + "9": 0.7380281690140845, + "10": 0.640117994100295 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9841269841269841, + "6": 0.9790419161676647, + "7": 0.9319526627218935, + "8": 0.8685897435897436, + "9": 0.8535211267605634, + "10": 0.7079646017699115 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 0.9970059880239521, + "7": 0.9733727810650887, + "8": 0.9326923076923077, + "9": 0.9154929577464789, + "10": 0.7846607669616519 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9940828402366864, + "8": 0.9775641025641025, + "9": 0.952112676056338, + "10": 0.8731563421828908 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 0.9774647887323944, + "10": 0.911504424778761 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9970414201183432, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9410029498525073 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9498525073746312 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9616519174041298 + } + }, + "total_pass_at_k": { + "pass@1": 0.8875379939209727, + "pass@2": 0.922661263086795, + "pass@4": 0.9537318473488686, + "pass@8": 0.9766970618034447, + "pass@16": 0.9864910503208375, + "pass@32": 0.9925700776764607, + "pass@64": 0.993920972644377, + "pass@128": 0.9952718676122931 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 0.9941520467836257, + "pass@4": 0.9970760233918129, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9441176470588235, + "pass@2": 0.9823529411764705, + "pass@4": 0.9970588235294118, + "pass@8": 0.9970588235294118, + "pass@16": 0.9970588235294118, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9652996845425867, + "pass@2": 0.9716088328075709, + "pass@4": 0.9810725552050473, + "pass@8": 0.9936908517350158, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8639240506329114, + "pass@2": 0.9082278481012658, + "pass@4": 0.9620253164556962, + "pass@8": 0.9841772151898734, + "pass@16": 0.9936708860759493, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8018018018018018, + "pass@2": 0.8678678678678678, + "pass@4": 0.9099099099099099, + "pass@8": 0.960960960960961, + "pass@16": 0.993993993993994, + "pass@32": 0.996996996996997, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.752411575562701, + "pass@2": 0.8006430868167203, + "pass@4": 0.8810289389067524, + "pass@8": 0.9517684887459807, + "pass@16": 0.9935691318327974, + "pass@32": 0.9935691318327974, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6238805970149254, + "pass@2": 0.6925373134328359, + "pass@4": 0.7582089552238805, + "pass@8": 0.8507462686567164, + "pass@16": 0.9014925373134328, + "pass@32": 0.9522388059701492, + "pass@64": 0.9611940298507463, + "pass@128": 0.9701492537313433 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.9441176470588235, + "6": 0.9652996845425867, + "7": 0.8639240506329114, + "8": 0.8018018018018018, + "9": 0.752411575562701, + "10": 0.6238805970149254 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9823529411764705, + "6": 0.9716088328075709, + "7": 0.9082278481012658, + "8": 0.8678678678678678, + "9": 0.8006430868167203, + "10": 0.6925373134328359 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9970588235294118, + "6": 0.9810725552050473, + "7": 0.9620253164556962, + "8": 0.9099099099099099, + "9": 0.8810289389067524, + "10": 0.7582089552238805 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9936908517350158, + "7": 0.9841772151898734, + "8": 0.960960960960961, + "9": 0.9517684887459807, + "10": 0.8507462686567164 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.993993993993994, + "9": 0.9935691318327974, + "10": 0.9014925373134328 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 0.9935691318327974, + "10": 0.9522388059701492 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9611940298507463 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9701492537313433 + } + }, + "total_pass_at_k": { + "pass@1": 0.884164713759625, + "pass@2": 0.9146300636089723, + "pass@4": 0.9437562772012053, + "pass@8": 0.9712085704720456, + "pass@16": 0.9866086374288584, + "pass@32": 0.9936391027787077, + "pass@64": 0.9953130231001004, + "pass@128": 0.9963173752929361 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9792899408284024, + "pass@2": 0.985207100591716, + "pass@4": 0.9970414201183432, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9594202898550724, + "pass@2": 0.9826086956521739, + "pass@4": 0.9942028985507246, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9742120343839542, + "pass@2": 0.9856733524355301, + "pass@4": 0.9914040114613181, + "pass@8": 0.997134670487106, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8641618497109826, + "pass@2": 0.9190751445086706, + "pass@4": 0.9682080924855492, + "pass@8": 0.9913294797687862, + "pass@16": 0.9971098265895953, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.819718309859155, + "pass@2": 0.8676056338028169, + "pass@4": 0.9352112676056338, + "pass@8": 0.9774647887323944, + "pass@16": 0.9943661971830986, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7455089820359282, + "pass@2": 0.8023952095808383, + "pass@4": 0.8712574850299402, + "pass@8": 0.937125748502994, + "pass@16": 0.9670658682634731, + "pass@32": 0.9910179640718563, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6687116564417178, + "pass@2": 0.7331288343558282, + "pass@4": 0.7760736196319018, + "pass@8": 0.8343558282208589, + "pass@16": 0.8957055214723927, + "pass@32": 0.9263803680981595, + "pass@64": 0.9447852760736196, + "pass@128": 0.9539877300613497 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9792899408284024, + "5": 0.9594202898550724, + "6": 0.9742120343839542, + "7": 0.8641618497109826, + "8": 0.819718309859155, + "9": 0.7455089820359282, + "10": 0.6687116564417178 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.985207100591716, + "5": 0.9826086956521739, + "6": 0.9856733524355301, + "7": 0.9190751445086706, + "8": 0.8676056338028169, + "9": 0.8023952095808383, + "10": 0.7331288343558282 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.9942028985507246, + "6": 0.9914040114613181, + "7": 0.9682080924855492, + "8": 0.9352112676056338, + "9": 0.8712574850299402, + "10": 0.7760736196319018 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9913294797687862, + "8": 0.9774647887323944, + "9": 0.937125748502994, + "10": 0.8343558282208589 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9943661971830986, + "9": 0.9670658682634731, + "10": 0.8957055214723927 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9910179640718563, + "10": 0.9263803680981595 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9447852760736196 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9539877300613497 + } + }, + "total_pass_at_k": { + "pass@1": 0.8905635648754915, + "pass@2": 0.9200524246395806, + "pass@4": 0.9488859764089121, + "pass@8": 0.9714941022280472, + "pass@16": 0.9842726081258192, + "pass@32": 0.9908256880733946, + "pass@64": 0.9941022280471822, + "pass@128": 0.9950851900393185 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.987, + "pass@2": 0.991, + "pass@4": 0.997, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.956, + "pass@2": 0.983, + "pass@4": 0.995, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.968, + "pass@2": 0.979, + "pass@4": 0.99, + "pass@8": 0.997, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.874, + "pass@2": 0.92, + "pass@4": 0.968, + "pass@8": 0.99, + "pass@16": 0.996, + "pass@32": 0.999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.813, + "pass@2": 0.868, + "pass@4": 0.926, + "pass@8": 0.972, + "pass@16": 0.995, + "pass@32": 0.998, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.745, + "pass@2": 0.82, + "pass@4": 0.89, + "pass@8": 0.947, + "pass@16": 0.979, + "pass@32": 0.994, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "10": { + "pass@1": 0.644, + "pass@2": 0.711, + "pass@4": 0.773, + "pass@8": 0.853, + "pass@16": 0.903, + "pass@32": 0.94, + "pass@64": 0.952, + "pass@128": 0.962 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.987, + "5": 0.956, + "6": 0.968, + "7": 0.874, + "8": 0.813, + "9": 0.745, + "10": 0.644 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.991, + "5": 0.983, + "6": 0.979, + "7": 0.92, + "8": 0.868, + "9": 0.82, + "10": 0.711 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.997, + "5": 0.995, + "6": 0.99, + "7": 0.968, + "8": 0.926, + "9": 0.89, + "10": 0.773 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 0.997, + "7": 0.99, + "8": 0.972, + "9": 0.947, + "10": 0.853 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.996, + "8": 0.995, + "9": 0.979, + "10": 0.903 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.999, + "8": 0.998, + "9": 0.994, + "10": 0.94 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.999, + "10": 0.952 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.999, + "10": 0.962 + } + }, + "total_pass_at_k": { + "pass@1": 0.8874444444444445, + "pass@2": 0.9191111111111111, + "pass@4": 0.9487777777777778, + "pass@8": 0.9731111111111111, + "pass@16": 0.9857777777777778, + "pass@32": 0.9923333333333333, + "pass@64": 0.9944444444444445, + "pass@128": 0.9955555555555555 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block4-epoch4-op11-14_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5609065155807366, + "pass@2": 0.6515580736543909, + "pass@4": 0.7393767705382436, + "pass@8": 0.830028328611898, + "pass@16": 0.886685552407932, + "pass@32": 0.9291784702549575, + "pass@64": 0.943342776203966, + "pass@128": 0.9518413597733711 + }, + "12": { + "pass@1": 0.6129943502824858, + "pass@2": 0.6666666666666666, + "pass@4": 0.7175141242937854, + "pass@8": 0.7994350282485876, + "pass@16": 0.8305084745762712, + "pass@32": 0.844632768361582, + "pass@64": 0.8728813559322034, + "pass@128": 0.884180790960452 + }, + "13": { + "pass@1": 0.5203761755485894, + "pass@2": 0.5799373040752351, + "pass@4": 0.6677115987460815, + "pass@8": 0.7210031347962382, + "pass@16": 0.7774294670846394, + "pass@32": 0.7962382445141066, + "pass@64": 0.8275862068965517, + "pass@128": 0.8463949843260188 + }, + "14": { + "pass@1": 0.4984423676012461, + "pass@2": 0.5264797507788161, + "pass@4": 0.5700934579439252, + "pass@8": 0.6230529595015576, + "pass@16": 0.6666666666666666, + "pass@32": 0.7383177570093458, + "pass@64": 0.7881619937694704, + "pass@128": 0.8161993769470405 + }, + "15": { + "pass@1": 0.4190751445086705, + "pass@2": 0.4682080924855491, + "pass@4": 0.5260115606936416, + "pass@8": 0.5780346820809249, + "pass@16": 0.653179190751445, + "pass@32": 0.7369942196531792, + "pass@64": 0.8034682080924855, + "pass@128": 0.8554913294797688 + }, + "16": { + "pass@1": 0.3225806451612903, + "pass@2": 0.36451612903225805, + "pass@4": 0.4064516129032258, + "pass@8": 0.4645161290322581, + "pass@16": 0.5258064516129032, + "pass@32": 0.5741935483870968, + "pass@64": 0.6096774193548387, + "pass@128": 0.6580645161290323 + }, + "17": { + "pass@1": 0.3092485549132948, + "pass@2": 0.3786127167630058, + "pass@4": 0.3988439306358382, + "pass@8": 0.4393063583815029, + "pass@16": 0.5086705202312138, + "pass@32": 0.5549132947976878, + "pass@64": 0.6242774566473989, + "pass@128": 0.6705202312138728 + }, + "18": { + "pass@1": 0.21428571428571427, + "pass@2": 0.27714285714285714, + "pass@4": 0.32571428571428573, + "pass@8": 0.36, + "pass@16": 0.4057142857142857, + "pass@32": 0.4542857142857143, + "pass@64": 0.5057142857142857, + "pass@128": 0.5457142857142857 + }, + "19": { + "pass@1": 0.1111111111111111, + "pass@2": 0.15915915915915915, + "pass@4": 0.21021021021021022, + "pass@8": 0.23723723723723725, + "pass@16": 0.27627627627627627, + "pass@32": 0.3033033033033033, + "pass@64": 0.3213213213213213, + "pass@128": 0.3663663663663664 + }, + "20": { + "pass@1": 0.14114114114114115, + "pass@2": 0.16816816816816818, + "pass@4": 0.1891891891891892, + "pass@8": 0.22522522522522523, + "pass@16": 0.25825825825825827, + "pass@32": 0.27927927927927926, + "pass@64": 0.29429429429429427, + "pass@128": 0.33933933933933935 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5609065155807366, + "12": 0.6129943502824858, + "13": 0.5203761755485894, + "14": 0.4984423676012461, + "15": 0.4190751445086705, + "16": 0.3225806451612903, + "17": 0.3092485549132948, + "18": 0.21428571428571427, + "19": 0.1111111111111111, + "20": 0.14114114114114115 + }, + "pass@2": { + "11": 0.6515580736543909, + "12": 0.6666666666666666, + "13": 0.5799373040752351, + "14": 0.5264797507788161, + "15": 0.4682080924855491, + "16": 0.36451612903225805, + "17": 0.3786127167630058, + "18": 0.27714285714285714, + "19": 0.15915915915915915, + "20": 0.16816816816816818 + }, + "pass@4": { + "11": 0.7393767705382436, + "12": 0.7175141242937854, + "13": 0.6677115987460815, + "14": 0.5700934579439252, + "15": 0.5260115606936416, + "16": 0.4064516129032258, + "17": 0.3988439306358382, + "18": 0.32571428571428573, + "19": 0.21021021021021022, + "20": 0.1891891891891892 + }, + "pass@8": { + "11": 0.830028328611898, + "12": 0.7994350282485876, + "13": 0.7210031347962382, + "14": 0.6230529595015576, + "15": 0.5780346820809249, + "16": 0.4645161290322581, + "17": 0.4393063583815029, + "18": 0.36, + "19": 0.23723723723723725, + "20": 0.22522522522522523 + }, + "pass@16": { + "11": 0.886685552407932, + "12": 0.8305084745762712, + "13": 0.7774294670846394, + "14": 0.6666666666666666, + "15": 0.653179190751445, + "16": 0.5258064516129032, + "17": 0.5086705202312138, + "18": 0.4057142857142857, + "19": 0.27627627627627627, + "20": 0.25825825825825827 + }, + "pass@32": { + "11": 0.9291784702549575, + "12": 0.844632768361582, + "13": 0.7962382445141066, + "14": 0.7383177570093458, + "15": 0.7369942196531792, + "16": 0.5741935483870968, + "17": 0.5549132947976878, + "18": 0.4542857142857143, + "19": 0.3033033033033033, + "20": 0.27927927927927926 + }, + "pass@64": { + "11": 0.943342776203966, + "12": 0.8728813559322034, + "13": 0.8275862068965517, + "14": 0.7881619937694704, + "15": 0.8034682080924855, + "16": 0.6096774193548387, + "17": 0.6242774566473989, + "18": 0.5057142857142857, + "19": 0.3213213213213213, + "20": 0.29429429429429427 + }, + "pass@128": { + "11": 0.9518413597733711, + "12": 0.884180790960452, + "13": 0.8463949843260188, + "14": 0.8161993769470405, + "15": 0.8554913294797688, + "16": 0.6580645161290323, + "17": 0.6705202312138728, + "18": 0.5457142857142857, + "19": 0.3663663663663664, + "20": 0.33933933933933935 + } + }, + "total_pass_at_k": { + "pass@1": 0.3720653789004458, + "pass@2": 0.42555720653789003, + "pass@4": 0.47667161961367016, + "pass@8": 0.5295690936106984, + "pass@16": 0.5806835066864785, + "pass@32": 0.6228826151560178, + "pass@64": 0.6609212481426449, + "pass@128": 0.6950965824665676 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5841269841269842, + "pass@2": 0.6698412698412698, + "pass@4": 0.7714285714285715, + "pass@8": 0.8539682539682539, + "pass@16": 0.9047619047619048, + "pass@32": 0.9396825396825397, + "pass@64": 0.9650793650793651, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.6055045871559633, + "pass@2": 0.6697247706422018, + "pass@4": 0.7584097859327217, + "pass@8": 0.8103975535168195, + "pass@16": 0.8593272171253823, + "pass@32": 0.8868501529051988, + "pass@64": 0.9021406727828746, + "pass@128": 0.9143730886850153 + }, + "13": { + "pass@1": 0.5308641975308642, + "pass@2": 0.5771604938271605, + "pass@4": 0.6172839506172839, + "pass@8": 0.6975308641975309, + "pass@16": 0.7469135802469136, + "pass@32": 0.7839506172839507, + "pass@64": 0.8055555555555556, + "pass@128": 0.8148148148148148 + }, + "14": { + "pass@1": 0.456973293768546, + "pass@2": 0.49258160237388726, + "pass@4": 0.5341246290801187, + "pass@8": 0.5934718100890207, + "pass@16": 0.6706231454005934, + "pass@32": 0.7537091988130564, + "pass@64": 0.8130563798219584, + "pass@128": 0.8545994065281899 + }, + "15": { + "pass@1": 0.4012345679012346, + "pass@2": 0.44135802469135804, + "pass@4": 0.47530864197530864, + "pass@8": 0.5308641975308642, + "pass@16": 0.6296296296296297, + "pass@32": 0.7037037037037037, + "pass@64": 0.7777777777777778, + "pass@128": 0.8209876543209876 + }, + "16": { + "pass@1": 0.3421828908554572, + "pass@2": 0.3834808259587021, + "pass@4": 0.4365781710914454, + "pass@8": 0.48672566371681414, + "pass@16": 0.56047197640118, + "pass@32": 0.6017699115044248, + "pass@64": 0.640117994100295, + "pass@128": 0.6873156342182891 + }, + "17": { + "pass@1": 0.28012048192771083, + "pass@2": 0.3313253012048193, + "pass@4": 0.3614457831325301, + "pass@8": 0.3855421686746988, + "pass@16": 0.42168674698795183, + "pass@32": 0.47289156626506024, + "pass@64": 0.5210843373493976, + "pass@128": 0.5813253012048193 + }, + "18": { + "pass@1": 0.24193548387096775, + "pass@2": 0.3064516129032258, + "pass@4": 0.3387096774193548, + "pass@8": 0.3741935483870968, + "pass@16": 0.4, + "pass@32": 0.43548387096774194, + "pass@64": 0.4838709677419355, + "pass@128": 0.5290322580645161 + }, + "19": { + "pass@1": 0.12084592145015106, + "pass@2": 0.14803625377643503, + "pass@4": 0.1933534743202417, + "pass@8": 0.22054380664652568, + "pass@16": 0.25075528700906347, + "pass@32": 0.283987915407855, + "pass@64": 0.3202416918429003, + "pass@128": 0.33534743202416917 + }, + "20": { + "pass@1": 0.11411411411411411, + "pass@2": 0.14414414414414414, + "pass@4": 0.17117117117117117, + "pass@8": 0.2042042042042042, + "pass@16": 0.22822822822822822, + "pass@32": 0.26426426426426425, + "pass@64": 0.2912912912912913, + "pass@128": 0.3213213213213213 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5841269841269842, + "12": 0.6055045871559633, + "13": 0.5308641975308642, + "14": 0.456973293768546, + "15": 0.4012345679012346, + "16": 0.3421828908554572, + "17": 0.28012048192771083, + "18": 0.24193548387096775, + "19": 0.12084592145015106, + "20": 0.11411411411411411 + }, + "pass@2": { + "11": 0.6698412698412698, + "12": 0.6697247706422018, + "13": 0.5771604938271605, + "14": 0.49258160237388726, + "15": 0.44135802469135804, + "16": 0.3834808259587021, + "17": 0.3313253012048193, + "18": 0.3064516129032258, + "19": 0.14803625377643503, + "20": 0.14414414414414414 + }, + "pass@4": { + "11": 0.7714285714285715, + "12": 0.7584097859327217, + "13": 0.6172839506172839, + "14": 0.5341246290801187, + "15": 0.47530864197530864, + "16": 0.4365781710914454, + "17": 0.3614457831325301, + "18": 0.3387096774193548, + "19": 0.1933534743202417, + "20": 0.17117117117117117 + }, + "pass@8": { + "11": 0.8539682539682539, + "12": 0.8103975535168195, + "13": 0.6975308641975309, + "14": 0.5934718100890207, + "15": 0.5308641975308642, + "16": 0.48672566371681414, + "17": 0.3855421686746988, + "18": 0.3741935483870968, + "19": 0.22054380664652568, + "20": 0.2042042042042042 + }, + "pass@16": { + "11": 0.9047619047619048, + "12": 0.8593272171253823, + "13": 0.7469135802469136, + "14": 0.6706231454005934, + "15": 0.6296296296296297, + "16": 0.56047197640118, + "17": 0.42168674698795183, + "18": 0.4, + "19": 0.25075528700906347, + "20": 0.22822822822822822 + }, + "pass@32": { + "11": 0.9396825396825397, + "12": 0.8868501529051988, + "13": 0.7839506172839507, + "14": 0.7537091988130564, + "15": 0.7037037037037037, + "16": 0.6017699115044248, + "17": 0.47289156626506024, + "18": 0.43548387096774194, + "19": 0.283987915407855, + "20": 0.26426426426426425 + }, + "pass@64": { + "11": 0.9650793650793651, + "12": 0.9021406727828746, + "13": 0.8055555555555556, + "14": 0.8130563798219584, + "15": 0.7777777777777778, + "16": 0.640117994100295, + "17": 0.5210843373493976, + "18": 0.4838709677419355, + "19": 0.3202416918429003, + "20": 0.2912912912912913 + }, + "pass@128": { + "11": 0.9746031746031746, + "12": 0.9143730886850153, + "13": 0.8148148148148148, + "14": 0.8545994065281899, + "15": 0.8209876543209876, + "16": 0.6873156342182891, + "17": 0.5813253012048193, + "18": 0.5290322580645161, + "19": 0.33534743202416917, + "20": 0.3213213213213213 + } + }, + "total_pass_at_k": { + "pass@1": 0.36674816625916873, + "pass@2": 0.41503667481662593, + "pass@4": 0.46424205378973105, + "pass@8": 0.5140586797066015, + "pass@16": 0.5657090464547677, + "pass@32": 0.6112469437652812, + "pass@64": 0.6506723716381418, + "pass@128": 0.6821515892420538 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6385542168674698, + "pass@2": 0.7259036144578314, + "pass@4": 0.7951807228915663, + "pass@8": 0.8614457831325302, + "pass@16": 0.9126506024096386, + "pass@32": 0.9548192771084337, + "pass@64": 0.9668674698795181, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.5109717868338558, + "pass@2": 0.5924764890282131, + "pass@4": 0.664576802507837, + "pass@8": 0.7492163009404389, + "pass@16": 0.8338557993730408, + "pass@32": 0.8652037617554859, + "pass@64": 0.8840125391849529, + "pass@128": 0.8996865203761756 + }, + "13": { + "pass@1": 0.5266106442577031, + "pass@2": 0.5938375350140056, + "pass@4": 0.6694677871148459, + "pass@8": 0.7535014005602241, + "pass@16": 0.7955182072829131, + "pass@32": 0.834733893557423, + "pass@64": 0.8515406162464986, + "pass@128": 0.8627450980392157 + }, + "14": { + "pass@1": 0.5058479532163743, + "pass@2": 0.5467836257309941, + "pass@4": 0.5847953216374269, + "pass@8": 0.652046783625731, + "pass@16": 0.7134502923976608, + "pass@32": 0.7807017543859649, + "pass@64": 0.8304093567251462, + "pass@128": 0.868421052631579 + }, + "15": { + "pass@1": 0.44545454545454544, + "pass@2": 0.49393939393939396, + "pass@4": 0.5363636363636364, + "pass@8": 0.5878787878787879, + "pass@16": 0.6606060606060606, + "pass@32": 0.7454545454545455, + "pass@64": 0.796969696969697, + "pass@128": 0.8454545454545455 + }, + "16": { + "pass@1": 0.3789173789173789, + "pass@2": 0.4188034188034188, + "pass@4": 0.47293447293447294, + "pass@8": 0.4985754985754986, + "pass@16": 0.5555555555555556, + "pass@32": 0.603988603988604, + "pass@64": 0.6638176638176638, + "pass@128": 0.7264957264957265 + }, + "17": { + "pass@1": 0.2639751552795031, + "pass@2": 0.32919254658385094, + "pass@4": 0.38198757763975155, + "pass@8": 0.40372670807453415, + "pass@16": 0.4472049689440994, + "pass@32": 0.5341614906832298, + "pass@64": 0.5900621118012422, + "pass@128": 0.6583850931677019 + }, + "18": { + "pass@1": 0.2411764705882353, + "pass@2": 0.27647058823529413, + "pass@4": 0.3264705882352941, + "pass@8": 0.34705882352941175, + "pass@16": 0.38235294117647056, + "pass@32": 0.4323529411764706, + "pass@64": 0.48823529411764705, + "pass@128": 0.5529411764705883 + }, + "19": { + "pass@1": 0.13392857142857142, + "pass@2": 0.1875, + "pass@4": 0.22916666666666666, + "pass@8": 0.2619047619047619, + "pass@16": 0.29464285714285715, + "pass@32": 0.3244047619047619, + "pass@64": 0.37202380952380953, + "pass@128": 0.4107142857142857 + }, + "20": { + "pass@1": 0.12275449101796407, + "pass@2": 0.15868263473053892, + "pass@4": 0.19161676646706588, + "pass@8": 0.2215568862275449, + "pass@16": 0.24850299401197604, + "pass@32": 0.281437125748503, + "pass@64": 0.3023952095808383, + "pass@128": 0.32934131736526945 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6385542168674698, + "12": 0.5109717868338558, + "13": 0.5266106442577031, + "14": 0.5058479532163743, + "15": 0.44545454545454544, + "16": 0.3789173789173789, + "17": 0.2639751552795031, + "18": 0.2411764705882353, + "19": 0.13392857142857142, + "20": 0.12275449101796407 + }, + "pass@2": { + "11": 0.7259036144578314, + "12": 0.5924764890282131, + "13": 0.5938375350140056, + "14": 0.5467836257309941, + "15": 0.49393939393939396, + "16": 0.4188034188034188, + "17": 0.32919254658385094, + "18": 0.27647058823529413, + "19": 0.1875, + "20": 0.15868263473053892 + }, + "pass@4": { + "11": 0.7951807228915663, + "12": 0.664576802507837, + "13": 0.6694677871148459, + "14": 0.5847953216374269, + "15": 0.5363636363636364, + "16": 0.47293447293447294, + "17": 0.38198757763975155, + "18": 0.3264705882352941, + "19": 0.22916666666666666, + "20": 0.19161676646706588 + }, + "pass@8": { + "11": 0.8614457831325302, + "12": 0.7492163009404389, + "13": 0.7535014005602241, + "14": 0.652046783625731, + "15": 0.5878787878787879, + "16": 0.4985754985754986, + "17": 0.40372670807453415, + "18": 0.34705882352941175, + "19": 0.2619047619047619, + "20": 0.2215568862275449 + }, + "pass@16": { + "11": 0.9126506024096386, + "12": 0.8338557993730408, + "13": 0.7955182072829131, + "14": 0.7134502923976608, + "15": 0.6606060606060606, + "16": 0.5555555555555556, + "17": 0.4472049689440994, + "18": 0.38235294117647056, + "19": 0.29464285714285715, + "20": 0.24850299401197604 + }, + "pass@32": { + "11": 0.9548192771084337, + "12": 0.8652037617554859, + "13": 0.834733893557423, + "14": 0.7807017543859649, + "15": 0.7454545454545455, + "16": 0.603988603988604, + "17": 0.5341614906832298, + "18": 0.4323529411764706, + "19": 0.3244047619047619, + "20": 0.281437125748503 + }, + "pass@64": { + "11": 0.9668674698795181, + "12": 0.8840125391849529, + "13": 0.8515406162464986, + "14": 0.8304093567251462, + "15": 0.796969696969697, + "16": 0.6638176638176638, + "17": 0.5900621118012422, + "18": 0.48823529411764705, + "19": 0.37202380952380953, + "20": 0.3023952095808383 + }, + "pass@128": { + "11": 0.9819277108433735, + "12": 0.8996865203761756, + "13": 0.8627450980392157, + "14": 0.868421052631579, + "15": 0.8454545454545455, + "16": 0.7264957264957265, + "17": 0.6583850931677019, + "18": 0.5529411764705883, + "19": 0.4107142857142857, + "20": 0.32934131736526945 + } + }, + "total_pass_at_k": { + "pass@1": 0.3773416592328278, + "pass@2": 0.4326494201605709, + "pass@4": 0.4855783526613143, + "pass@8": 0.5340469818614333, + "pass@16": 0.5845970859351769, + "pass@32": 0.6357418971156705, + "pass@64": 0.6746952126077906, + "pass@128": 0.7136485280999108 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.594, + "pass@2": 0.682, + "pass@4": 0.768, + "pass@8": 0.848, + "pass@16": 0.901, + "pass@32": 0.941, + "pass@64": 0.958, + "pass@128": 0.969 + }, + "12": { + "pass@1": 0.578, + "pass@2": 0.644, + "pass@4": 0.714, + "pass@8": 0.787, + "pass@16": 0.841, + "pass@32": 0.865, + "pass@64": 0.886, + "pass@128": 0.899 + }, + "13": { + "pass@1": 0.526, + "pass@2": 0.584, + "pass@4": 0.652, + "pass@8": 0.725, + "pass@16": 0.774, + "pass@32": 0.806, + "pass@64": 0.829, + "pass@128": 0.842 + }, + "14": { + "pass@1": 0.487, + "pass@2": 0.522, + "pass@4": 0.563, + "pass@8": 0.623, + "pass@16": 0.684, + "pass@32": 0.758, + "pass@64": 0.811, + "pass@128": 0.847 + }, + "15": { + "pass@1": 0.422, + "pass@2": 0.468, + "pass@4": 0.513, + "pass@8": 0.566, + "pass@16": 0.648, + "pass@32": 0.729, + "pass@64": 0.793, + "pass@128": 0.841 + }, + "16": { + "pass@1": 0.349, + "pass@2": 0.39, + "pass@4": 0.44, + "pass@8": 0.484, + "pass@16": 0.548, + "pass@32": 0.594, + "pass@64": 0.639, + "pass@128": 0.692 + }, + "17": { + "pass@1": 0.285, + "pass@2": 0.347, + "pass@4": 0.381, + "pass@8": 0.41, + "pass@16": 0.46, + "pass@32": 0.521, + "pass@64": 0.579, + "pass@128": 0.637 + }, + "18": { + "pass@1": 0.232, + "pass@2": 0.286, + "pass@4": 0.33, + "pass@8": 0.36, + "pass@16": 0.396, + "pass@32": 0.441, + "pass@64": 0.493, + "pass@128": 0.543 + }, + "19": { + "pass@1": 0.122, + "pass@2": 0.165, + "pass@4": 0.211, + "pass@8": 0.24, + "pass@16": 0.274, + "pass@32": 0.304, + "pass@64": 0.338, + "pass@128": 0.371 + }, + "20": { + "pass@1": 0.126, + "pass@2": 0.157, + "pass@4": 0.184, + "pass@8": 0.217, + "pass@16": 0.245, + "pass@32": 0.275, + "pass@64": 0.296, + "pass@128": 0.33 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.594, + "12": 0.578, + "13": 0.526, + "14": 0.487, + "15": 0.422, + "16": 0.349, + "17": 0.285, + "18": 0.232, + "19": 0.122, + "20": 0.126 + }, + "pass@2": { + "11": 0.682, + "12": 0.644, + "13": 0.584, + "14": 0.522, + "15": 0.468, + "16": 0.39, + "17": 0.347, + "18": 0.286, + "19": 0.165, + "20": 0.157 + }, + "pass@4": { + "11": 0.768, + "12": 0.714, + "13": 0.652, + "14": 0.563, + "15": 0.513, + "16": 0.44, + "17": 0.381, + "18": 0.33, + "19": 0.211, + "20": 0.184 + }, + "pass@8": { + "11": 0.848, + "12": 0.787, + "13": 0.725, + "14": 0.623, + "15": 0.566, + "16": 0.484, + "17": 0.41, + "18": 0.36, + "19": 0.24, + "20": 0.217 + }, + "pass@16": { + "11": 0.901, + "12": 0.841, + "13": 0.774, + "14": 0.684, + "15": 0.648, + "16": 0.548, + "17": 0.46, + "18": 0.396, + "19": 0.274, + "20": 0.245 + }, + "pass@32": { + "11": 0.941, + "12": 0.865, + "13": 0.806, + "14": 0.758, + "15": 0.729, + "16": 0.594, + "17": 0.521, + "18": 0.441, + "19": 0.304, + "20": 0.275 + }, + "pass@64": { + "11": 0.958, + "12": 0.886, + "13": 0.829, + "14": 0.811, + "15": 0.793, + "16": 0.639, + "17": 0.579, + "18": 0.493, + "19": 0.338, + "20": 0.296 + }, + "pass@128": { + "11": 0.969, + "12": 0.899, + "13": 0.842, + "14": 0.847, + "15": 0.841, + "16": 0.692, + "17": 0.637, + "18": 0.543, + "19": 0.371, + "20": 0.33 + } + }, + "total_pass_at_k": { + "pass@1": 0.3721, + "pass@2": 0.4245, + "pass@4": 0.4756, + "pass@8": 0.526, + "pass@16": 0.5771, + "pass@32": 0.6234, + "pass@64": 0.6622, + "pass@128": 0.6971 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block4-epoch4-op11-14/summary.csv b/composition/SDAR-100M-block4-epoch4-op11-14/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..98e9bac36dbed8b0fdc945c1f4a12b0cefe26f6c --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op11-14/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch4-op11-14,0.9928567708333333,0.691875,0.8344453125,0.0,0.0,0.0,199.79739930555556,372.99815546875,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block4-epoch4-op11-14/summary.json b/composition/SDAR-100M-block4-epoch4-op11-14/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4897fdecf36111e8f967fbd6fa0c6e55d36a3421 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op11-14/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch4-op11-14", + "id_acc": 0.9928567708333333, + "ood_acc": 0.691875, + "total_acc": 0.8344453125, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.79739930555556, + "ood_avg_resp_len": 372.99815546875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 676686, + "answer_accuracy": 0.8356954434081568, + "avg_response_len": 278.0035740396775, + "resp_tokens_sum": 225107278, + "pass_at_k": { + "pass@1": 0.8356954434081568, + "pass@2": 0.8709115657541193, + "pass@4": 0.8961285177491005, + "pass@8": 0.9142744528411244, + "pass@16": 0.9280869090810386, + "pass@32": 0.939176064183682, + "pass@64": 0.948097615830537, + "pass@128": 0.9552639898830224 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9555217551622419, + "pass@2": 0.9649799375188721, + "pass@4": 0.9725824357685375, + "pass@8": 0.9783204632700988, + "pass@16": 0.9825890111220292, + "pass@32": 0.9854794451027657, + "pass@64": 0.9882112861183845, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99765625, + "pass@2": 0.9988723394438976, + "pass@4": 0.9996240582231908, + "pass@8": 0.9999587853903806, + "pass@16": 0.99999963742401, + "pass@32": 0.9999999999942999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9956597222222222, + "pass@2": 0.997699506311711, + "pass@4": 0.9989479258545063, + "pass@8": 0.999704562830816, + "pass@16": 0.9999774766429096, + "pass@32": 0.9999999361951407, + "pass@64": 0.999999999999996, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9985669378698225, + "pass@2": 0.999648741438755, + "pass@4": 0.999965682636564, + "pass@8": 0.9999996999237872, + "pass@16": 0.999999999991595, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9986227964743589, + "pass@2": 0.9999313862810418, + "pass@4": 0.9999997707618278, + "pass@8": 0.9999999999988657, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9971675030497948, + "pass@4": 0.9971830761471716, + "pass@8": 0.9971830985915474, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9595432011331445, + "pass@2": 0.9712151608262141, + "pass@4": 0.9783531306993142, + "pass@8": 0.983326614921076, + "pass@16": 0.9865718492617972, + "pass@32": 0.9888296194765508, + "pass@64": 0.9906259618168388, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.906007238700565, + "pass@2": 0.9247933487922062, + "pass@4": 0.9357417314361123, + "pass@8": 0.9421184863804423, + "pass@16": 0.9467630723566625, + "pass@32": 0.9518291711617118, + "pass@64": 0.9579773195275365, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8717672413793104, + "pass@2": 0.9035996161725869, + "pass@4": 0.9226742165654058, + "pass@8": 0.9340267016365622, + "pass@16": 0.9431541830177079, + "pass@32": 0.9532836501149419, + "pass@64": 0.9644626458159502, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8635854750778816, + "pass@2": 0.8954251447249002, + "pass@4": 0.9113720040251972, + "pass@8": 0.9206645009701866, + "pass@16": 0.9286928898625472, + "pass@32": 0.9369032185178561, + "pass@64": 0.9440107739042536, + "pass@128": 0.9501557632398754 + }, + "15": { + "pass@1": 0.8376986994219653, + "pass@2": 0.8865166014291564, + "pass@4": 0.9181329861765831, + "pass@8": 0.9412534150029087, + "pass@16": 0.9562649099753139, + "pass@32": 0.9651885106215669, + "pass@64": 0.9706079061795896, + "pass@128": 0.9739884393063584 + }, + "16": { + "pass@1": 0.6686743951612903, + "pass@2": 0.7454768065786131, + "pass@4": 0.7934225168426517, + "pass@8": 0.8257491148636001, + "pass@16": 0.8513642598353691, + "pass@32": 0.8730630130473881, + "pass@64": 0.8910014008528852, + "pass@128": 0.9064516129032258 + }, + "17": { + "pass@1": 0.6197841401734104, + "pass@2": 0.7161787839652269, + "pass@4": 0.7794292514375006, + "pass@8": 0.8202572762642997, + "pass@16": 0.8505096438381178, + "pass@32": 0.8753376105726489, + "pass@64": 0.8944587031637984, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.5136607142857142, + "pass@2": 0.6187155511811023, + "pass@4": 0.6944143609727348, + "pass@8": 0.7489642516122974, + "pass@16": 0.7936353786265803, + "pass@32": 0.8323554499320751, + "pass@64": 0.8633710891209031, + "pass@128": 0.8857142857142857 + }, + "19": { + "pass@1": 0.37058933933933935, + "pass@2": 0.48287460689035483, + "pass@4": 0.5762031899728748, + "pass@8": 0.6492350344930701, + "pass@16": 0.706125200061887, + "pass@32": 0.7499946710222914, + "pass@64": 0.7842006467978939, + "pass@128": 0.8138138138138138 + }, + "20": { + "pass@1": 0.34649493243243246, + "pass@2": 0.46130899600387765, + "pass@4": 0.5613634281075223, + "pass@8": 0.6403788685222684, + "pass@16": 0.6985517724335117, + "pass@32": 0.7410158124161551, + "pass@64": 0.7727139219338052, + "pass@128": 0.7957957957957958 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 689799, + "answer_accuracy": 0.8400708787996882, + "avg_response_len": 287.2397493667186, + "resp_tokens_sum": 235858303, + "pass_at_k": { + "pass@1": 0.8400708787996882, + "pass@2": 0.8741392973530203, + "pass@4": 0.8996079074780458, + "pass@8": 0.9189056746060298, + "pass@16": 0.9333578266970128, + "pass@32": 0.9443563373607163, + "pass@64": 0.9530550043624433, + "pass@128": 0.9602494154325799 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9392973542944786, + "pass@2": 0.9498639105598762, + "pass@4": 0.9579157637887292, + "pass@8": 0.9633439750573325, + "pass@16": 0.967524339629329, + "pass@32": 0.9719200985373748, + "pass@64": 0.9765380222469652, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9988224637681159, + "pass@2": 0.9997874586328882, + "pass@4": 0.9999922303190363, + "pass@8": 0.9999999910428016, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985449498567335, + "pass@2": 0.9996760288112266, + "pass@4": 0.9999785661641866, + "pass@8": 0.9999999015825531, + "pass@16": 0.9999999999993172, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9960711705202312, + "pass@2": 0.9974437184242867, + "pass@4": 0.9988051376164685, + "pass@8": 0.9997047707744012, + "pass@16": 0.9999793315061124, + "pass@32": 0.9999999419107679, + "pass@64": 0.9999999999999963, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9960607394366198, + "pass@2": 0.9983739048464012, + "pass@4": 0.9996422492963027, + "pass@8": 0.9999796729938163, + "pass@16": 0.999999939849747, + "pass@32": 0.99999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9945499625748503, + "pass@2": 0.9964954441510679, + "pass@4": 0.9973660230158355, + "pass@8": 0.9978422530337244, + "pass@16": 0.9984818259578424, + "pass@32": 0.9993082788304156, + "pass@64": 0.999913715875336, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9681852409638554, + "pass@2": 0.9741481921781614, + "pass@4": 0.9772397477988446, + "pass@8": 0.9789669965699488, + "pass@16": 0.9799547315679026, + "pass@32": 0.9809906007078771, + "pass@64": 0.9826866521200436, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9085276253918495, + "pass@2": 0.9235974409448813, + "pass@4": 0.9330263857104065, + "pass@8": 0.9409230257953107, + "pass@16": 0.9477558425747747, + "pass@32": 0.9527120358305108, + "pass@64": 0.95642424625744, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.8668154761904762, + "pass@2": 0.8926307235492627, + "pass@4": 0.9088578126788774, + "pass@8": 0.9216598215625059, + "pass@16": 0.9320716043391158, + "pass@32": 0.941360763388727, + "pass@64": 0.950378253308469, + "pass@128": 0.9607843137254902 + }, + "14": { + "pass@1": 0.881281980994152, + "pass@2": 0.9201054329557498, + "pass@4": 0.9419351528427365, + "pass@8": 0.9547325691053806, + "pass@16": 0.9625703344338206, + "pass@32": 0.9680469863909118, + "pass@64": 0.9725173602632785, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8279829545454546, + "pass@2": 0.8793225065616795, + "pass@4": 0.9134771270068515, + "pass@8": 0.9379655607421947, + "pass@16": 0.9560091862643495, + "pass@32": 0.9682017448322129, + "pass@64": 0.9747783770514524, + "pass@128": 0.9787878787878788 + }, + "16": { + "pass@1": 0.6672008547008547, + "pass@2": 0.7537498317517997, + "pass@4": 0.8141053287142518, + "pass@8": 0.8573749750163748, + "pass@16": 0.886024033601711, + "pass@32": 0.9049191024077768, + "pass@64": 0.9190486363240371, + "pass@128": 0.9316239316239316 + }, + "17": { + "pass@1": 0.6349718555900621, + "pass@2": 0.7272980143786375, + "pass@4": 0.7871718557501735, + "pass@8": 0.8293996571531411, + "pass@16": 0.8622377160167927, + "pass@32": 0.8877773954413081, + "pass@64": 0.9070769693130836, + "pass@128": 0.922360248447205 + }, + "18": { + "pass@1": 0.5722426470588236, + "pass@2": 0.6690550457387674, + "pass@4": 0.7391316479741499, + "pass@8": 0.7903393556142831, + "pass@16": 0.8293274169491589, + "pass@32": 0.8609394488494504, + "pass@64": 0.8900216102839796, + "pass@128": 0.9176470588235294 + }, + "19": { + "pass@1": 0.36755952380952384, + "pass@2": 0.4771494627624668, + "pass@4": 0.5758954558693551, + "pass@8": 0.6572878262769214, + "pass@16": 0.7219972024993643, + "pass@32": 0.7752524575772055, + "pass@64": 0.8196462374053157, + "pass@128": 0.8541666666666666 + }, + "20": { + "pass@1": 0.3302067739520958, + "pass@2": 0.4390511545806021, + "pass@4": 0.5388564448717861, + "pass@8": 0.6219737802822332, + "pass@16": 0.6834840446587457, + "pass@32": 0.7260101372704013, + "pass@64": 0.7543047807447868, + "pass@128": 0.7724550898203593 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 662886, + "answer_accuracy": 0.827416020929861, + "avg_response_len": 307.85501627656174, + "resp_tokens_sum": 246638662, + "pass_at_k": { + "pass@1": 0.827416020929861, + "pass@2": 0.8638326950922894, + "pass@4": 0.8917031383918479, + "pass@8": 0.9122528988397226, + "pass@16": 0.9274143119764798, + "pass@32": 0.9392260387639582, + "pass@64": 0.9489087963224284, + "pass@128": 0.9565425786866911 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9665578358208955, + "pass@2": 0.9753995769185567, + "pass@4": 0.9812427446569179, + "pass@8": 0.984445931787918, + "pass@16": 0.9862398280511495, + "pass@32": 0.9878623718754762, + "pass@64": 0.9903043768983087, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998633381924198, + "pass@2": 0.9999964130759167, + "pass@4": 0.9999999986335528, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9924632352941176, + "pass@2": 0.9969336208893007, + "pass@4": 0.9993456238007014, + "pass@8": 0.9999676306603055, + "pass@16": 0.9999999404169826, + "pass@32": 0.9999999999999771, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9984966482649842, + "pass@2": 0.9993359404108397, + "pass@4": 0.9998654172565968, + "pass@8": 0.9999951072498166, + "pass@16": 0.9999999962922006, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9966871044303798, + "pass@2": 0.9985314088507925, + "pass@4": 0.9996055398138525, + "pass@8": 0.9999623956767472, + "pass@16": 0.999999710126808, + "pass@32": 0.9999999999966581, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9901229354354354, + "pass@2": 0.9935347503014826, + "pass@4": 0.9964370444685404, + "pass@8": 0.9986250019627753, + "pass@16": 0.9996774337286254, + "pass@32": 0.9999786753419787, + "pass@64": 0.9999999842832614, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9965082395498392, + "pass@2": 0.9982063511152746, + "pass@4": 0.9990804087591943, + "pass@8": 0.9997496498520251, + "pass@16": 0.9999841910388799, + "pass@32": 0.999999971428726, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9600446428571429, + "pass@2": 0.9693546119235098, + "pass@4": 0.9754381020824777, + "pass@8": 0.9792596757858676, + "pass@16": 0.9818508644575293, + "pass@32": 0.9845842545836299, + "pass@64": 0.9876931724947077, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9324827981651376, + "pass@2": 0.9487191932143805, + "pass@4": 0.9579715631876291, + "pass@8": 0.964293747505247, + "pass@16": 0.9702628430959813, + "pass@32": 0.9772449442356358, + "pass@64": 0.9851239703402783, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8404224537037037, + "pass@2": 0.8757765413871874, + "pass@4": 0.8961142932943567, + "pass@8": 0.908989364931657, + "pass@16": 0.9210844814038844, + "pass@32": 0.9347247035643859, + "pass@64": 0.9469491976683431, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.8556194362017804, + "pass@2": 0.8901566041262641, + "pass@4": 0.9098709404647859, + "pass@8": 0.9239797308298969, + "pass@16": 0.9360531611369882, + "pass@32": 0.9474375907664061, + "pass@64": 0.957745729612481, + "pass@128": 0.9673590504451038 + }, + "15": { + "pass@1": 0.7978877314814815, + "pass@2": 0.8533278500777681, + "pass@4": 0.8859646846806184, + "pass@8": 0.9049570675533932, + "pass@16": 0.9179703098545164, + "pass@32": 0.9292966068716593, + "pass@64": 0.939598699165408, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.6572409660766961, + "pass@2": 0.7479694359278098, + "pass@4": 0.8105195647557326, + "pass@8": 0.8520118529571892, + "pass@16": 0.879123562506178, + "pass@32": 0.8977227947218035, + "pass@64": 0.9120114491385409, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.5544992469879518, + "pass@2": 0.6537838114268096, + "pass@4": 0.7294969114365216, + "pass@8": 0.7825533883751986, + "pass@16": 0.8195442961880983, + "pass@32": 0.8473326604424434, + "pass@64": 0.871423376114652, + "pass@128": 0.8945783132530121 + }, + "18": { + "pass@1": 0.5292338709677419, + "pass@2": 0.6312952438404874, + "pass@4": 0.7058185550394909, + "pass@8": 0.7574372911381855, + "pass@16": 0.7936015396857868, + "pass@32": 0.8200462776298317, + "pass@64": 0.8399903201774559, + "pass@128": 0.8548387096774194 + }, + "19": { + "pass@1": 0.3309337235649547, + "pass@2": 0.4468046750481719, + "pass@4": 0.5559979029245206, + "pass@8": 0.6488701249194181, + "pass@16": 0.7225968808340559, + "pass@32": 0.7785636517822091, + "pass@64": 0.8182159590626885, + "pass@128": 0.8459214501510574 + }, + "20": { + "pass@1": 0.31386073573573575, + "pass@2": 0.4276183171360337, + "pass@4": 0.5344682199071957, + "pass@8": 0.6229720662431183, + "pass@16": 0.6884024081151023, + "pass@32": 0.7363216124083445, + "pass@64": 0.776265928622691, + "pass@128": 0.8108108108108109 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_metrics.json b/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..5b5aca1af3eca50d9f25aafe5eb71164347cd14f --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch4-op17-20", + "id": { + "answer_accuracy": 0.9928307291666667, + "avg_response_len": 199.24223958333334, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9928307291666667, + "pass@2": 0.9950613790463679, + "pass@4": 0.996422512540099, + "pass@8": 0.9973608871376792, + "pass@16": 0.9980819998075972, + "pass@32": 0.9986988415659556, + "pass@64": 0.9991910668259056, + "pass@128": 0.9995555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954875, + "pass@2": 0.9656734744094486, + "pass@4": 0.9738502501874767, + "pass@8": 0.980187661645057, + "pass@16": 0.985422074320855, + "pass@32": 0.9899195932558139, + "pass@64": 0.9932405463259857, + "pass@128": 0.996 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998828125, + "pass@2": 0.9999931102362204, + "pass@4": 0.9999999690663667, + "pass@8": 0.9999999999998846, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997421875, + "pass@2": 0.9999464812992126, + "pass@4": 0.9999974311023622, + "pass@8": 0.9999999959061942, + "pass@16": 0.9999999999999984, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996453125, + "pass@2": 0.9986684301181104, + "pass@4": 0.9995166817585303, + "pass@8": 0.9998539345097475, + "pass@16": 0.9999828554611712, + "pass@32": 0.9999998583460734, + "pass@64": 0.9999999999997746, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985390625, + "pass@2": 0.9994984005905512, + "pass@4": 0.999916720472441, + "pass@8": 0.9999972141757824, + "pass@16": 0.9999999979510268, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9969921875, + "pass@2": 0.9979735482283465, + "pass@4": 0.998548175384327, + "pass@8": 0.9989748614222378, + "pass@16": 0.9992987726757218, + "pass@32": 0.9995808364436816, + "pass@64": 0.9998779527452687, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.993515625, + "pass@2": 0.9965835383858268, + "pass@4": 0.9981302094113236, + "pass@8": 0.9990134621448256, + "pass@16": 0.9994716291486171, + "pass@32": 0.9997688599880662, + "pass@64": 0.9999711811022796, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954765625, + "pass@2": 0.9972154281496063, + "pass@4": 0.9978431754780654, + "pass@8": 0.998220854435389, + "pass@16": 0.9985626687109834, + "pass@32": 0.9990204260599655, + "pass@64": 0.9996299212598424, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.954875, + "2": 1.0, + "3": 0.9998828125, + "4": 0.9997421875, + "5": 0.996453125, + "6": 0.9985390625, + "7": 0.9969921875, + "8": 0.993515625, + "9": 0.9954765625 + }, + "per_op_avg_response_len": { + "10": 279.504140625, + "2": 115.2683515625, + "3": 144.46059375, + "4": 159.2585859375, + "5": 172.4520703125, + "6": 198.26584375, + "7": 223.2936875, + "8": 242.6784921875, + "9": 257.998390625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.504140625, + "2": 115.2683515625, + "3": 144.46059375, + "4": 159.2585859375, + "5": 172.4520703125, + "6": 198.26584375, + "7": 223.2936875, + "8": 242.6784921875, + "9": 257.998390625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376784, + "answer_accuracy": 0.9941320499831138, + "avg_response_len": 189.61737483113814, + "resp_tokens_sum": 71866502, + "pass_at_k": { + "pass@1": 0.9941320499831138, + "pass@2": 0.9960113031083881, + "pass@4": 0.996948029229376, + "pass@8": 0.9975861451314317, + "pass@16": 0.9981962235801742, + "pass@32": 0.9987960607409337, + "pass@64": 0.9992668397522212, + "pass@128": 0.999662276258021 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9613753687315634, + "pass@2": 0.970923193505679, + "pass@4": 0.97719835463045, + "pass@8": 0.9817599651022448, + "pass@16": 0.9862649009949807, + "pass@32": 0.9907192299119268, + "pass@64": 0.9939562234532361, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999522171253823, + "pass@2": 0.9999996237568928, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99921875, + "pass@2": 0.9998327540600395, + "pass@4": 0.9999919721948819, + "pass@8": 0.9999999872068569, + "pass@16": 0.9999999999999952, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9957093253968254, + "pass@2": 0.9977174728158981, + "pass@4": 0.9987727721534809, + "pass@8": 0.9995512405280322, + "pass@16": 0.9999456111182624, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993450598802395, + "pass@2": 0.9998802837238909, + "pass@4": 0.999995804229561, + "pass@8": 0.9999999967283617, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9984051405325444, + "pass@2": 0.999718993151004, + "pass@4": 0.9999875685842522, + "pass@8": 0.9999999772191345, + "pass@16": 0.9999999999999808, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9976712740384616, + "pass@2": 0.9997326430950938, + "pass@4": 0.9999875658571523, + "pass@8": 0.9999999688739251, + "pass@16": 0.9999999999999599, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9972888017078853, + "pass@4": 0.997440846918783, + "pass@8": 0.997682618016843, + "pass@16": 0.9981192931869433, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387347, + "answer_accuracy": 0.991529632208388, + "avg_response_len": 197.23375808895807, + "resp_tokens_sum": 77050551, + "pass_at_k": { + "pass@1": 0.991529632208388, + "pass@2": 0.9937921499778128, + "pass@4": 0.9952555532561216, + "pass@8": 0.9962415224550635, + "pass@16": 0.9970253819958741, + "pass@32": 0.9978481282955417, + "pass@64": 0.9986697318155959, + "pass@128": 0.9993446920052425 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.939704754601227, + "pass@2": 0.951967522704217, + "pass@4": 0.9620432760322137, + "pass@8": 0.9700759835321355, + "pass@16": 0.9765457934086171, + "pass@32": 0.9828590570934914, + "pass@64": 0.9886812890839152, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9984148550724637, + "pass@2": 0.9996868937578454, + "pass@4": 0.9999772702325253, + "pass@8": 0.9999998752549307, + "pass@16": 0.999999999998829, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9977390759312321, + "pass@2": 0.9992202129819733, + "pass@4": 0.9998640429616784, + "pass@8": 0.9999948677008761, + "pass@16": 0.9999999954075054, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9953937861271677, + "pass@2": 0.9963599033953849, + "pass@4": 0.9971111784798576, + "pass@8": 0.9976008320837435, + "pass@16": 0.9980702739375236, + "pass@32": 0.9987903426808435, + "pass@64": 0.999647262300305, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956646126760563, + "pass@2": 0.9985024814239768, + "pass@4": 0.999711590540619, + "pass@8": 0.9999853934625496, + "pass@16": 0.9999999656383994, + "pass@32": 0.9999999999999744, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9948540419161677, + "pass@2": 0.9966571532132587, + "pass@4": 0.9971588291358788, + "pass@8": 0.9973697231456297, + "pass@16": 0.9977102880819406, + "pass@32": 0.9983202885567448, + "pass@64": 0.9992573907303504, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379610, + "answer_accuracy": 0.992870145631068, + "avg_response_len": 210.83551378891866, + "resp_tokens_sum": 80610007, + "pass_at_k": { + "pass@1": 0.992870145631068, + "pass@2": 0.9954165722065962, + "pass@4": 0.9970939235939164, + "pass@8": 0.9982813130137579, + "pass@16": 0.9990483811168637, + "pass@32": 0.9994716942355882, + "pass@64": 0.999648633219158, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9630597014925373, + "pass@2": 0.9736988042073099, + "pass@4": 0.981951940242544, + "pass@8": 0.9884366055042368, + "pass@16": 0.9932069977085, + "pass@32": 0.995981258815499, + "pass@64": 0.9969530935342759, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997266763848397, + "pass@2": 0.9999802719175408, + "pass@4": 0.9999999098144802, + "pass@8": 0.9999999999996636, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9951516544117647, + "pass@2": 0.9985160230430754, + "pass@4": 0.9997385302940074, + "pass@8": 0.9999862846484304, + "pass@16": 0.9999999645862426, + "pass@32": 0.9999999999999732, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985705835962145, + "pass@2": 0.9994023075585583, + "pass@4": 0.9998913907449267, + "pass@8": 0.9999968657441765, + "pass@16": 0.9999999985924529, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972310126582279, + "pass@2": 0.9978734239509618, + "pass@4": 0.9985819919266421, + "pass@8": 0.9993828519658078, + "pass@16": 0.999893854092864, + "pass@32": 0.9999980312535122, + "pass@64": 0.999999999966339, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.987331081081081, + "pass@2": 0.9915873058097466, + "pass@4": 0.9947041267927096, + "pass@8": 0.997053024555423, + "pass@16": 0.9984133373783722, + "pass@32": 0.999305885850076, + "pass@64": 0.9999134567636024, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947749196141479, + "pass@2": 0.9977312359166519, + "pass@4": 0.9990373822846098, + "pass@8": 0.9997493167484548, + "pass@16": 0.9999841910297439, + "pass@32": 0.999999971428726, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.73775, + "avg_response_len": 375.910928125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.73775, + "pass@2": 0.794706053149587, + "pass@4": 0.8341085378327668, + "pass@8": 0.8628744719585935, + "pass@16": 0.8848724928697924, + "pass@32": 0.9026449216928464, + "pass@64": 0.9174809125133008, + "pass@128": 0.93 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9524765625, + "pass@2": 0.9629094488188978, + "pass@4": 0.9701072982752155, + "pass@8": 0.9754230524252512, + "pass@16": 0.9793903666819255, + "pass@32": 0.9827255399046864, + "pass@64": 0.9855884699571937, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.9019453125, + "pass@2": 0.922604576771654, + "pass@4": 0.9358301445444318, + "pass@8": 0.9450701800335944, + "pass@16": 0.9525441316300564, + "pass@32": 0.9599660340270201, + "pass@64": 0.9672012514625326, + "pass@128": 0.973 + }, + "13": { + "pass@1": 0.8347734375, + "pass@2": 0.8710346948818899, + "pass@4": 0.8919833366141726, + "pass@8": 0.9061311839731385, + "pass@16": 0.9180586164072557, + "pass@32": 0.9293484836585746, + "pass@64": 0.9397442533843364, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.861046875, + "pass@2": 0.8973094242125992, + "pass@4": 0.9173674203224592, + "pass@8": 0.9306128578750045, + "pass@16": 0.9412425628429993, + "pass@32": 0.9508925861196847, + "pass@64": 0.9593212166041116, + "pass@128": 0.966 + }, + "15": { + "pass@1": 0.8366875, + "pass@2": 0.8811536663385828, + "pass@4": 0.9086343062429686, + "pass@8": 0.9272380159376128, + "pass@16": 0.9403654342189126, + "pass@32": 0.9496850768606452, + "pass@64": 0.956857592627593, + "pass@128": 0.963 + }, + "16": { + "pass@1": 0.7048203125, + "pass@2": 0.7783625738188973, + "pass@4": 0.8265161660104984, + "pass@8": 0.8593674600542125, + "pass@16": 0.8829819020113596, + "pass@32": 0.9010030449875494, + "pass@64": 0.9160504722934611, + "pass@128": 0.929 + }, + "17": { + "pass@1": 0.6717890625, + "pass@2": 0.7463127460629921, + "pass@4": 0.7963798781402323, + "pass@8": 0.8328647227302957, + "pass@16": 0.8607759566392034, + "pass@32": 0.8828159686152725, + "pass@64": 0.8998461134101179, + "pass@128": 0.912 + }, + "18": { + "pass@1": 0.645828125, + "pass@2": 0.7231929133858277, + "pass@4": 0.7766033110236222, + "pass@8": 0.8153549865324885, + "pass@16": 0.8446173110762112, + "pass@32": 0.8681309150288821, + "pass@64": 0.8872269659541829, + "pass@128": 0.904 + }, + "19": { + "pass@1": 0.473640625, + "pass@2": 0.5737501230314962, + "pass@4": 0.6525852853393315, + "pass@8": 0.7137386336937745, + "pass@16": 0.7614976906757347, + "pass@32": 0.7998671857278985, + "pass@64": 0.8322612553363816, + "pass@128": 0.86 + }, + "20": { + "pass@1": 0.4944921875, + "pass@2": 0.5904303641732278, + "pass@4": 0.6650782318147722, + "pass@8": 0.7229436263306257, + "pass@16": 0.7672509565141948, + "pass@32": 0.8020143819982127, + "pass@64": 0.8307115341030594, + "pass@128": 0.856 + } + }, + "per_op_accuracy": { + "11": 0.9524765625, + "12": 0.9019453125, + "13": 0.8347734375, + "14": 0.861046875, + "15": 0.8366875, + "16": 0.7048203125, + "17": 0.6717890625, + "18": 0.645828125, + "19": 0.473640625, + "20": 0.4944921875 + }, + "per_op_avg_response_len": { + "11": 297.4130390625, + "12": 307.7463046875, + "13": 326.572140625, + "14": 355.126609375, + "15": 374.6200546875, + "16": 382.372546875, + "17": 406.533796875, + "18": 424.842640625, + "19": 429.3982734375, + "20": 454.483875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.4130390625, + "12": 307.7463046875, + "13": 326.572140625, + "14": 355.126609375, + "15": 374.6200546875, + "16": 382.372546875, + "17": 406.533796875, + "18": 424.842640625, + "19": 429.3982734375, + "20": 454.483875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 316722, + "answer_accuracy": 0.7353315378900446, + "avg_response_len": 357.8785661218425, + "resp_tokens_sum": 154145456, + "pass_at_k": { + "pass@1": 0.7353315378900446, + "pass@2": 0.791600330813965, + "pass@4": 0.8304551045996692, + "pass@8": 0.8583848958316062, + "pass@16": 0.8795357228155043, + "pass@32": 0.8969488755607409, + "pass@64": 0.91220347434442, + "pass@128": 0.9254086181277861 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9425017705382436, + "pass@2": 0.9548415159153264, + "pass@4": 0.9637315494911575, + "pass@8": 0.9707074281466466, + "pass@16": 0.9764958731179654, + "pass@32": 0.9818812626501979, + "pass@64": 0.9869854923560497, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.884622175141243, + "pass@2": 0.9070330224876552, + "pass@4": 0.9212047249390439, + "pass@8": 0.9304309349365792, + "pass@16": 0.93734400736929, + "pass@32": 0.9441033505020082, + "pass@64": 0.9506885517975576, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8455867946708464, + "pass@2": 0.8826244816478657, + "pass@4": 0.9036141996748837, + "pass@8": 0.9170823611637505, + "pass@16": 0.9287953901231346, + "pass@32": 0.9401314416183841, + "pass@64": 0.9503814510788167, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.8548481308411215, + "pass@2": 0.890110644025805, + "pass@4": 0.9090667565502909, + "pass@8": 0.9206775494670285, + "pass@16": 0.929940970807209, + "pass@32": 0.9389608922498822, + "pass@64": 0.9479723802614771, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.8491239161849711, + "pass@2": 0.891381323676665, + "pass@4": 0.9182221973878987, + "pass@8": 0.9374987178643029, + "pass@16": 0.9500343576067202, + "pass@32": 0.9582461300623976, + "pass@64": 0.9655452112581264, + "pass@128": 0.9710982658959537 + }, + "16": { + "pass@1": 0.6878780241935484, + "pass@2": 0.7575481807213613, + "pass@4": 0.8034270958065721, + "pass@8": 0.8332422545728394, + "pass@16": 0.8555616965823695, + "pass@32": 0.8752240577779585, + "pass@64": 0.8934588712339225, + "pass@128": 0.9096774193548387 + }, + "17": { + "pass@1": 0.6582821531791907, + "pass@2": 0.7333793693960219, + "pass@4": 0.7873619814864179, + "pass@8": 0.8268060950621734, + "pass@16": 0.8541651488113018, + "pass@32": 0.8744259072106768, + "pass@64": 0.8921529102764344, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.6214508928571428, + "pass@2": 0.702418799212598, + "pass@4": 0.7587747707429429, + "pass@8": 0.7988930690855568, + "pass@16": 0.8295439274930722, + "pass@32": 0.8551884496518796, + "pass@64": 0.8759012824017393, + "pass@128": 0.8942857142857142 + }, + "19": { + "pass@1": 0.4751783033033033, + "pass@2": 0.5727038554302333, + "pass@4": 0.6456607301902577, + "pass@8": 0.7009178523026034, + "pass@16": 0.7443995896195046, + "pass@32": 0.780325551418712, + "pass@64": 0.8128709427686198, + "pass@128": 0.8408408408408409 + }, + "20": { + "pass@1": 0.5220298423423423, + "pass@2": 0.6148334604289327, + "pass@4": 0.6857166376162438, + "pass@8": 0.7404605152943717, + "pass@16": 0.782629162910974, + "pass@32": 0.8153740821980516, + "pass@64": 0.8413122377082638, + "pass@128": 0.8618618618618619 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 321398, + "answer_accuracy": 0.7466315417781743, + "avg_response_len": 371.1641716845079, + "resp_tokens_sum": 159772814, + "pass_at_k": { + "pass@1": 0.7466315417781743, + "pass@2": 0.8018654033238107, + "pass@4": 0.8403893998921449, + "pass@8": 0.8696697999346771, + "pass@16": 0.8927550391117525, + "pass@32": 0.9113634762036628, + "pass@64": 0.9265986541078121, + "pass@128": 0.9396372286648825 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.961902296686747, + "pass@2": 0.9702211898776207, + "pass@4": 0.9745907889149398, + "pass@8": 0.9773632430076152, + "pass@16": 0.9793331174520167, + "pass@32": 0.9809097132926594, + "pass@64": 0.9826862351534986, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.898951802507837, + "pass@2": 0.9214403401377336, + "pass@4": 0.9366077708154816, + "pass@8": 0.9473168294237594, + "pass@16": 0.9557104493167832, + "pass@32": 0.9632188528557657, + "pass@64": 0.9697692688760374, + "pass@128": 0.9749216300940439 + }, + "13": { + "pass@1": 0.8447785364145658, + "pass@2": 0.8756985569818482, + "pass@4": 0.8945737355309575, + "pass@8": 0.9089284772307276, + "pass@16": 0.9205185085929912, + "pass@32": 0.9308260468285129, + "pass@64": 0.9408199495389861, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.8758223684210527, + "pass@2": 0.9139463035870513, + "pass@4": 0.9350603600536769, + "pass@8": 0.9487797027077906, + "pass@16": 0.9591678574023239, + "pass@32": 0.9683737126824168, + "pass@64": 0.975387169545268, + "pass@128": 0.97953216374269 + }, + "15": { + "pass@1": 0.8341619318181818, + "pass@2": 0.8827543396564063, + "pass@4": 0.9144763560804903, + "pass@8": 0.9369795398942257, + "pass@16": 0.9540667236743633, + "pass@32": 0.9664573273663126, + "pass@64": 0.9747860909253457, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.7109375, + "pass@2": 0.7824315930861211, + "pass@4": 0.8306402492957612, + "pass@8": 0.8662793559814501, + "pass@16": 0.8918811624991876, + "pass@32": 0.9103467438874276, + "pass@64": 0.9247234008320129, + "pass@128": 0.9344729344729344 + }, + "17": { + "pass@1": 0.710549301242236, + "pass@2": 0.7828643107790872, + "pass@4": 0.8275231751892806, + "pass@8": 0.8615275014644767, + "pass@16": 0.8904357410433366, + "pass@32": 0.9134983203987874, + "pass@64": 0.9284336977111665, + "pass@128": 0.937888198757764 + }, + "18": { + "pass@1": 0.6790900735294118, + "pass@2": 0.7502435299907362, + "pass@4": 0.800968797834094, + "pass@8": 0.8404703821156387, + "pass@16": 0.8728190470546322, + "pass@32": 0.8996975551996976, + "pass@64": 0.9220785178168743, + "pass@128": 0.9411764705882353 + }, + "19": { + "pass@1": 0.4720982142857143, + "pass@2": 0.565974175103112, + "pass@4": 0.6416543777451924, + "pass@8": 0.7033531553412125, + "pass@16": 0.7537467764321175, + "pass@32": 0.7949539015604953, + "pass@64": 0.831013697436013, + "pass@128": 0.8660714285714286 + }, + "20": { + "pass@1": 0.4807260479041916, + "pass@2": 0.5753041161770944, + "pass@4": 0.6497437565813254, + "pass@8": 0.7074401183175504, + "pass@16": 0.7516001349159216, + "pass@32": 0.787087003435915, + "pass@64": 0.8178073059749411, + "pass@128": 0.8473053892215568 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 306200, + "answer_accuracy": 0.7311086491442543, + "avg_response_len": 399.3345956219438, + "resp_tokens_sum": 167247718, + "pass_at_k": { + "pass@1": 0.7311086491442543, + "pass@2": 0.7905415852472955, + "pass@4": 0.8314102687996849, + "pass@8": 0.8605073373876343, + "pass@16": 0.8822591763114017, + "pass@32": 0.8995418338000026, + "pass@64": 0.9135370294007938, + "pass@128": 0.9248166259168704 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9537202380952381, + "pass@2": 0.9642443132108488, + "pass@4": 0.972526728206593, + "pass@8": 0.9786626463839903, + "pass@16": 0.9826943761181406, + "pass@32": 0.9855854900825515, + "pass@64": 0.9870817812207828, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.9236190749235474, + "pass@2": 0.9405976095017944, + "pass@4": 0.9509045658283538, + "pass@8": 0.9587264846479097, + "pass@16": 0.9659104577653655, + "pass@32": 0.9739652412486856, + "pass@64": 0.9825721937453857, + "pass@128": 0.9877675840978594 + }, + "13": { + "pass@1": 0.8131028163580247, + "pass@2": 0.8544848777583361, + "pass@4": 0.877677726337449, + "pass@8": 0.8922668036744839, + "pass@16": 0.9047770968835759, + "pass@32": 0.9171038736559595, + "pass@64": 0.9280859521752334, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.8519566023738873, + "pass@2": 0.8872827052033923, + "pass@4": 0.9073185410429038, + "pass@8": 0.9216400776558572, + "pass@16": 0.9338163322916623, + "pass@32": 0.9445172996144986, + "pass@64": 0.9538269155955367, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.8259789737654321, + "pass@2": 0.8686012230242051, + "pass@4": 0.8924451495067742, + "pass@8": 0.9063586771341635, + "pass@16": 0.9160849619581726, + "pass@32": 0.9234598698400384, + "pass@64": 0.9293195664411008, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.7139795353982301, + "pass@2": 0.7931833292685756, + "pass@4": 0.8433599964827404, + "pass@8": 0.8761012011420178, + "pass@16": 0.8988421475917746, + "pass@32": 0.9149023008020526, + "pass@64": 0.9277295236545959, + "pass@128": 0.9410029498525073 + }, + "17": { + "pass@1": 0.6482727786144579, + "pass@2": 0.7243408921117538, + "pass@4": 0.7755728016113949, + "pass@8": 0.8113793926723576, + "pass@16": 0.8388991160678905, + "pass@32": 0.8618016432288235, + "pass@64": 0.880137216239385, + "pass@128": 0.891566265060241 + }, + "18": { + "pass@1": 0.6368699596774193, + "pass@2": 0.7169791402082805, + "pass@4": 0.7700088709677413, + "pass@8": 0.8063951046233131, + "pass@16": 0.8307047111453599, + "pass@32": 0.848121899622024, + "pass@64": 0.8617897453414103, + "pass@128": 0.8741935483870967 + }, + "19": { + "pass@1": 0.47365936555891236, + "pass@2": 0.5826961218688298, + "pass@4": 0.6706477077449906, + "pass@8": 0.7371792407322068, + "pass@16": 0.7865671010611726, + "pass@32": 0.8245143993387958, + "pass@64": 0.8530351330994856, + "pass@128": 0.8731117824773413 + }, + "20": { + "pass@1": 0.480762012012012, + "pass@2": 0.5811989400818144, + "pass@4": 0.659820350721925, + "pass@8": 0.7209768024610734, + "pass@16": 0.7675705711799523, + "pass@32": 0.8036268873233177, + "pass@64": 0.8330538100617901, + "pass@128": 0.8588588588588588 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.8585777138157895, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8585777138157895, + "pass@2": 0.8896112075217446, + "pass@4": 0.9109941047993847, + "pass@8": 0.9265785633592267, + "pass@16": 0.9384980487876894, + "pass@32": 0.9481441468959069, + "pass@64": 0.9561857224508497, + "pass@128": 0.9629473684210527 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.954875, + "pass@2": 0.9656734744094486, + "pass@4": 0.9738502501874767, + "pass@8": 0.980187661645057, + "pass@16": 0.985422074320855, + "pass@32": 0.9899195932558139, + "pass@64": 0.9932405463259857, + "pass@128": 0.996 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998828125, + "pass@2": 0.9999931102362204, + "pass@4": 0.9999999690663667, + "pass@8": 0.9999999999998846, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997421875, + "pass@2": 0.9999464812992126, + "pass@4": 0.9999974311023622, + "pass@8": 0.9999999959061942, + "pass@16": 0.9999999999999984, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996453125, + "pass@2": 0.9986684301181104, + "pass@4": 0.9995166817585303, + "pass@8": 0.9998539345097475, + "pass@16": 0.9999828554611712, + "pass@32": 0.9999998583460734, + "pass@64": 0.9999999999997746, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985390625, + "pass@2": 0.9994984005905512, + "pass@4": 0.999916720472441, + "pass@8": 0.9999972141757824, + "pass@16": 0.9999999979510268, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9969921875, + "pass@2": 0.9979735482283465, + "pass@4": 0.998548175384327, + "pass@8": 0.9989748614222378, + "pass@16": 0.9992987726757218, + "pass@32": 0.9995808364436816, + "pass@64": 0.9998779527452687, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.993515625, + "pass@2": 0.9965835383858268, + "pass@4": 0.9981302094113236, + "pass@8": 0.9990134621448256, + "pass@16": 0.9994716291486171, + "pass@32": 0.9997688599880662, + "pass@64": 0.9999711811022796, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954765625, + "pass@2": 0.9972154281496063, + "pass@4": 0.9978431754780654, + "pass@8": 0.998220854435389, + "pass@16": 0.9985626687109834, + "pass@32": 0.9990204260599655, + "pass@64": 0.9996299212598424, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9524765625, + "pass@2": 0.9629094488188978, + "pass@4": 0.9701072982752155, + "pass@8": 0.9754230524252512, + "pass@16": 0.9793903666819255, + "pass@32": 0.9827255399046864, + "pass@64": 0.9855884699571937, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.9019453125, + "pass@2": 0.922604576771654, + "pass@4": 0.9358301445444318, + "pass@8": 0.9450701800335944, + "pass@16": 0.9525441316300564, + "pass@32": 0.9599660340270201, + "pass@64": 0.9672012514625326, + "pass@128": 0.973 + }, + "13": { + "pass@1": 0.8347734375, + "pass@2": 0.8710346948818899, + "pass@4": 0.8919833366141726, + "pass@8": 0.9061311839731385, + "pass@16": 0.9180586164072557, + "pass@32": 0.9293484836585746, + "pass@64": 0.9397442533843364, + "pass@128": 0.949 + }, + "14": { + "pass@1": 0.861046875, + "pass@2": 0.8973094242125992, + "pass@4": 0.9173674203224592, + "pass@8": 0.9306128578750045, + "pass@16": 0.9412425628429993, + "pass@32": 0.9508925861196847, + "pass@64": 0.9593212166041116, + "pass@128": 0.966 + }, + "15": { + "pass@1": 0.8366875, + "pass@2": 0.8811536663385828, + "pass@4": 0.9086343062429686, + "pass@8": 0.9272380159376128, + "pass@16": 0.9403654342189126, + "pass@32": 0.9496850768606452, + "pass@64": 0.956857592627593, + "pass@128": 0.963 + }, + "16": { + "pass@1": 0.7048203125, + "pass@2": 0.7783625738188973, + "pass@4": 0.8265161660104984, + "pass@8": 0.8593674600542125, + "pass@16": 0.8829819020113596, + "pass@32": 0.9010030449875494, + "pass@64": 0.9160504722934611, + "pass@128": 0.929 + }, + "17": { + "pass@1": 0.6717890625, + "pass@2": 0.7463127460629921, + "pass@4": 0.7963798781402323, + "pass@8": 0.8328647227302957, + "pass@16": 0.8607759566392034, + "pass@32": 0.8828159686152725, + "pass@64": 0.8998461134101179, + "pass@128": 0.912 + }, + "18": { + "pass@1": 0.645828125, + "pass@2": 0.7231929133858277, + "pass@4": 0.7766033110236222, + "pass@8": 0.8153549865324885, + "pass@16": 0.8446173110762112, + "pass@32": 0.8681309150288821, + "pass@64": 0.8872269659541829, + "pass@128": 0.904 + }, + "19": { + "pass@1": 0.473640625, + "pass@2": 0.5737501230314962, + "pass@4": 0.6525852853393315, + "pass@8": 0.7137386336937745, + "pass@16": 0.7614976906757347, + "pass@32": 0.7998671857278985, + "pass@64": 0.8322612553363816, + "pass@128": 0.86 + }, + "20": { + "pass@1": 0.4944921875, + "pass@2": 0.5904303641732278, + "pass@4": 0.6650782318147722, + "pass@8": 0.7229436263306257, + "pass@16": 0.7672509565141948, + "pass@32": 0.8020143819982127, + "pass@64": 0.8307115341030594, + "pass@128": 0.856 + } + }, + "per_op_accuracy": { + "10": 0.954875, + "2": 1.0, + "3": 0.9998828125, + "4": 0.9997421875, + "5": 0.996453125, + "6": 0.9985390625, + "7": 0.9969921875, + "8": 0.993515625, + "9": 0.9954765625, + "11": 0.9524765625, + "12": 0.9019453125, + "13": 0.8347734375, + "14": 0.861046875, + "15": 0.8366875, + "16": 0.7048203125, + "17": 0.6717890625, + "18": 0.645828125, + "19": 0.473640625, + "20": 0.4944921875 + }, + "per_op_avg_response_len": { + "10": 279.504140625, + "2": 115.2683515625, + "3": 144.46059375, + "4": 159.2585859375, + "5": 172.4520703125, + "6": 198.26584375, + "7": 223.2936875, + "8": 242.6784921875, + "9": 257.998390625, + "11": 297.4130390625, + "12": 307.7463046875, + "13": 326.572140625, + "14": 355.126609375, + "15": 374.6200546875, + "16": 382.372546875, + "17": 406.533796875, + "18": 424.842640625, + "19": 429.3982734375, + "20": 454.483875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.504140625, + "2": 115.2683515625, + "3": 144.46059375, + "4": 159.2585859375, + "5": 172.4520703125, + "6": 198.26584375, + "7": 223.2936875, + "8": 242.6784921875, + "9": 257.998390625, + "11": 297.4130390625, + "12": 307.7463046875, + "13": 326.572140625, + "14": 355.126609375, + "15": 374.6200546875, + "16": 382.372546875, + "17": 406.533796875, + "18": 424.842640625, + "19": 429.3982734375, + "20": 454.483875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 693506, + "answer_accuracy": 0.8564678509326589, + "avg_response_len": 279.12083810859946, + "resp_tokens_sum": 226011958, + "pass_at_k": { + "pass@1": 0.8564678509326589, + "pass@2": 0.8872786249909652, + "pass@4": 0.9083851630613429, + "pass@8": 0.92354058650134, + "pass@16": 0.9350769404513234, + "pass@32": 0.9446203133284556, + "pass@64": 0.9529550748775423, + "pass@128": 0.9601644008852356 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9613753687315634, + "pass@2": 0.970923193505679, + "pass@4": 0.97719835463045, + "pass@8": 0.9817599651022448, + "pass@16": 0.9862649009949807, + "pass@32": 0.9907192299119268, + "pass@64": 0.9939562234532361, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999522171253823, + "pass@2": 0.9999996237568928, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99921875, + "pass@2": 0.9998327540600395, + "pass@4": 0.9999919721948819, + "pass@8": 0.9999999872068569, + "pass@16": 0.9999999999999952, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9957093253968254, + "pass@2": 0.9977174728158981, + "pass@4": 0.9987727721534809, + "pass@8": 0.9995512405280322, + "pass@16": 0.9999456111182624, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993450598802395, + "pass@2": 0.9998802837238909, + "pass@4": 0.999995804229561, + "pass@8": 0.9999999967283617, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9984051405325444, + "pass@2": 0.999718993151004, + "pass@4": 0.9999875685842522, + "pass@8": 0.9999999772191345, + "pass@16": 0.9999999999999808, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9976712740384616, + "pass@2": 0.9997326430950938, + "pass@4": 0.9999875658571523, + "pass@8": 0.9999999688739251, + "pass@16": 0.9999999999999599, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9972888017078853, + "pass@4": 0.997440846918783, + "pass@8": 0.997682618016843, + "pass@16": 0.9981192931869433, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9425017705382436, + "pass@2": 0.9548415159153264, + "pass@4": 0.9637315494911575, + "pass@8": 0.9707074281466466, + "pass@16": 0.9764958731179654, + "pass@32": 0.9818812626501979, + "pass@64": 0.9869854923560497, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.884622175141243, + "pass@2": 0.9070330224876552, + "pass@4": 0.9212047249390439, + "pass@8": 0.9304309349365792, + "pass@16": 0.93734400736929, + "pass@32": 0.9441033505020082, + "pass@64": 0.9506885517975576, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8455867946708464, + "pass@2": 0.8826244816478657, + "pass@4": 0.9036141996748837, + "pass@8": 0.9170823611637505, + "pass@16": 0.9287953901231346, + "pass@32": 0.9401314416183841, + "pass@64": 0.9503814510788167, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.8548481308411215, + "pass@2": 0.890110644025805, + "pass@4": 0.9090667565502909, + "pass@8": 0.9206775494670285, + "pass@16": 0.929940970807209, + "pass@32": 0.9389608922498822, + "pass@64": 0.9479723802614771, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.8491239161849711, + "pass@2": 0.891381323676665, + "pass@4": 0.9182221973878987, + "pass@8": 0.9374987178643029, + "pass@16": 0.9500343576067202, + "pass@32": 0.9582461300623976, + "pass@64": 0.9655452112581264, + "pass@128": 0.9710982658959537 + }, + "16": { + "pass@1": 0.6878780241935484, + "pass@2": 0.7575481807213613, + "pass@4": 0.8034270958065721, + "pass@8": 0.8332422545728394, + "pass@16": 0.8555616965823695, + "pass@32": 0.8752240577779585, + "pass@64": 0.8934588712339225, + "pass@128": 0.9096774193548387 + }, + "17": { + "pass@1": 0.6582821531791907, + "pass@2": 0.7333793693960219, + "pass@4": 0.7873619814864179, + "pass@8": 0.8268060950621734, + "pass@16": 0.8541651488113018, + "pass@32": 0.8744259072106768, + "pass@64": 0.8921529102764344, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.6214508928571428, + "pass@2": 0.702418799212598, + "pass@4": 0.7587747707429429, + "pass@8": 0.7988930690855568, + "pass@16": 0.8295439274930722, + "pass@32": 0.8551884496518796, + "pass@64": 0.8759012824017393, + "pass@128": 0.8942857142857142 + }, + "19": { + "pass@1": 0.4751783033033033, + "pass@2": 0.5727038554302333, + "pass@4": 0.6456607301902577, + "pass@8": 0.7009178523026034, + "pass@16": 0.7443995896195046, + "pass@32": 0.780325551418712, + "pass@64": 0.8128709427686198, + "pass@128": 0.8408408408408409 + }, + "20": { + "pass@1": 0.5220298423423423, + "pass@2": 0.6148334604289327, + "pass@4": 0.6857166376162438, + "pass@8": 0.7404605152943717, + "pass@16": 0.782629162910974, + "pass@32": 0.8153740821980516, + "pass@64": 0.8413122377082638, + "pass@128": 0.8618618618618619 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 708745, + "answer_accuracy": 0.8631442420109119, + "avg_response_len": 288.41504895752144, + "resp_tokens_sum": 236823365, + "pass_at_k": { + "pass@1": 0.8631442420109119, + "pass@2": 0.8931764603445276, + "pass@4": 0.9140685113600835, + "pass@8": 0.929887554748734, + "pass@16": 0.9423626909406467, + "pass@32": 0.9525094088902438, + "pass@64": 0.960887185544163, + "pass@128": 0.9680436477007015 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.939704754601227, + "pass@2": 0.951967522704217, + "pass@4": 0.9620432760322137, + "pass@8": 0.9700759835321355, + "pass@16": 0.9765457934086171, + "pass@32": 0.9828590570934914, + "pass@64": 0.9886812890839152, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9984148550724637, + "pass@2": 0.9996868937578454, + "pass@4": 0.9999772702325253, + "pass@8": 0.9999998752549307, + "pass@16": 0.999999999998829, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9977390759312321, + "pass@2": 0.9992202129819733, + "pass@4": 0.9998640429616784, + "pass@8": 0.9999948677008761, + "pass@16": 0.9999999954075054, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9953937861271677, + "pass@2": 0.9963599033953849, + "pass@4": 0.9971111784798576, + "pass@8": 0.9976008320837435, + "pass@16": 0.9980702739375236, + "pass@32": 0.9987903426808435, + "pass@64": 0.999647262300305, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956646126760563, + "pass@2": 0.9985024814239768, + "pass@4": 0.999711590540619, + "pass@8": 0.9999853934625496, + "pass@16": 0.9999999656383994, + "pass@32": 0.9999999999999744, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9948540419161677, + "pass@2": 0.9966571532132587, + "pass@4": 0.9971588291358788, + "pass@8": 0.9973697231456297, + "pass@16": 0.9977102880819406, + "pass@32": 0.9983202885567448, + "pass@64": 0.9992573907303504, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.961902296686747, + "pass@2": 0.9702211898776207, + "pass@4": 0.9745907889149398, + "pass@8": 0.9773632430076152, + "pass@16": 0.9793331174520167, + "pass@32": 0.9809097132926594, + "pass@64": 0.9826862351534986, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.898951802507837, + "pass@2": 0.9214403401377336, + "pass@4": 0.9366077708154816, + "pass@8": 0.9473168294237594, + "pass@16": 0.9557104493167832, + "pass@32": 0.9632188528557657, + "pass@64": 0.9697692688760374, + "pass@128": 0.9749216300940439 + }, + "13": { + "pass@1": 0.8447785364145658, + "pass@2": 0.8756985569818482, + "pass@4": 0.8945737355309575, + "pass@8": 0.9089284772307276, + "pass@16": 0.9205185085929912, + "pass@32": 0.9308260468285129, + "pass@64": 0.9408199495389861, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.8758223684210527, + "pass@2": 0.9139463035870513, + "pass@4": 0.9350603600536769, + "pass@8": 0.9487797027077906, + "pass@16": 0.9591678574023239, + "pass@32": 0.9683737126824168, + "pass@64": 0.975387169545268, + "pass@128": 0.97953216374269 + }, + "15": { + "pass@1": 0.8341619318181818, + "pass@2": 0.8827543396564063, + "pass@4": 0.9144763560804903, + "pass@8": 0.9369795398942257, + "pass@16": 0.9540667236743633, + "pass@32": 0.9664573273663126, + "pass@64": 0.9747860909253457, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.7109375, + "pass@2": 0.7824315930861211, + "pass@4": 0.8306402492957612, + "pass@8": 0.8662793559814501, + "pass@16": 0.8918811624991876, + "pass@32": 0.9103467438874276, + "pass@64": 0.9247234008320129, + "pass@128": 0.9344729344729344 + }, + "17": { + "pass@1": 0.710549301242236, + "pass@2": 0.7828643107790872, + "pass@4": 0.8275231751892806, + "pass@8": 0.8615275014644767, + "pass@16": 0.8904357410433366, + "pass@32": 0.9134983203987874, + "pass@64": 0.9284336977111665, + "pass@128": 0.937888198757764 + }, + "18": { + "pass@1": 0.6790900735294118, + "pass@2": 0.7502435299907362, + "pass@4": 0.800968797834094, + "pass@8": 0.8404703821156387, + "pass@16": 0.8728190470546322, + "pass@32": 0.8996975551996976, + "pass@64": 0.9220785178168743, + "pass@128": 0.9411764705882353 + }, + "19": { + "pass@1": 0.4720982142857143, + "pass@2": 0.565974175103112, + "pass@4": 0.6416543777451924, + "pass@8": 0.7033531553412125, + "pass@16": 0.7537467764321175, + "pass@32": 0.7949539015604953, + "pass@64": 0.831013697436013, + "pass@128": 0.8660714285714286 + }, + "20": { + "pass@1": 0.4807260479041916, + "pass@2": 0.5753041161770944, + "pass@4": 0.6497437565813254, + "pass@8": 0.7074401183175504, + "pass@16": 0.7516001349159216, + "pass@32": 0.787087003435915, + "pass@64": 0.8178073059749411, + "pass@128": 0.8473053892215568 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 685810, + "answer_accuracy": 0.8560298170634286, + "avg_response_len": 309.37665386842946, + "resp_tokens_sum": 247857725, + "pass_at_k": { + "pass@1": 0.8560298170634286, + "pass@2": 0.8883146458076645, + "pass@4": 0.910479940771302, + "pass@8": 0.9262575954472617, + "pass@16": 0.9379948137541099, + "pass@32": 0.9472316393793455, + "pass@64": 0.9546323098937601, + "pass@128": 0.9605368269691644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9630597014925373, + "pass@2": 0.9736988042073099, + "pass@4": 0.981951940242544, + "pass@8": 0.9884366055042368, + "pass@16": 0.9932069977085, + "pass@32": 0.995981258815499, + "pass@64": 0.9969530935342759, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997266763848397, + "pass@2": 0.9999802719175408, + "pass@4": 0.9999999098144802, + "pass@8": 0.9999999999996636, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9951516544117647, + "pass@2": 0.9985160230430754, + "pass@4": 0.9997385302940074, + "pass@8": 0.9999862846484304, + "pass@16": 0.9999999645862426, + "pass@32": 0.9999999999999732, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985705835962145, + "pass@2": 0.9994023075585583, + "pass@4": 0.9998913907449267, + "pass@8": 0.9999968657441765, + "pass@16": 0.9999999985924529, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972310126582279, + "pass@2": 0.9978734239509618, + "pass@4": 0.9985819919266421, + "pass@8": 0.9993828519658078, + "pass@16": 0.999893854092864, + "pass@32": 0.9999980312535122, + "pass@64": 0.999999999966339, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.987331081081081, + "pass@2": 0.9915873058097466, + "pass@4": 0.9947041267927096, + "pass@8": 0.997053024555423, + "pass@16": 0.9984133373783722, + "pass@32": 0.999305885850076, + "pass@64": 0.9999134567636024, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947749196141479, + "pass@2": 0.9977312359166519, + "pass@4": 0.9990373822846098, + "pass@8": 0.9997493167484548, + "pass@16": 0.9999841910297439, + "pass@32": 0.999999971428726, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9537202380952381, + "pass@2": 0.9642443132108488, + "pass@4": 0.972526728206593, + "pass@8": 0.9786626463839903, + "pass@16": 0.9826943761181406, + "pass@32": 0.9855854900825515, + "pass@64": 0.9870817812207828, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.9236190749235474, + "pass@2": 0.9405976095017944, + "pass@4": 0.9509045658283538, + "pass@8": 0.9587264846479097, + "pass@16": 0.9659104577653655, + "pass@32": 0.9739652412486856, + "pass@64": 0.9825721937453857, + "pass@128": 0.9877675840978594 + }, + "13": { + "pass@1": 0.8131028163580247, + "pass@2": 0.8544848777583361, + "pass@4": 0.877677726337449, + "pass@8": 0.8922668036744839, + "pass@16": 0.9047770968835759, + "pass@32": 0.9171038736559595, + "pass@64": 0.9280859521752334, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.8519566023738873, + "pass@2": 0.8872827052033923, + "pass@4": 0.9073185410429038, + "pass@8": 0.9216400776558572, + "pass@16": 0.9338163322916623, + "pass@32": 0.9445172996144986, + "pass@64": 0.9538269155955367, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.8259789737654321, + "pass@2": 0.8686012230242051, + "pass@4": 0.8924451495067742, + "pass@8": 0.9063586771341635, + "pass@16": 0.9160849619581726, + "pass@32": 0.9234598698400384, + "pass@64": 0.9293195664411008, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.7139795353982301, + "pass@2": 0.7931833292685756, + "pass@4": 0.8433599964827404, + "pass@8": 0.8761012011420178, + "pass@16": 0.8988421475917746, + "pass@32": 0.9149023008020526, + "pass@64": 0.9277295236545959, + "pass@128": 0.9410029498525073 + }, + "17": { + "pass@1": 0.6482727786144579, + "pass@2": 0.7243408921117538, + "pass@4": 0.7755728016113949, + "pass@8": 0.8113793926723576, + "pass@16": 0.8388991160678905, + "pass@32": 0.8618016432288235, + "pass@64": 0.880137216239385, + "pass@128": 0.891566265060241 + }, + "18": { + "pass@1": 0.6368699596774193, + "pass@2": 0.7169791402082805, + "pass@4": 0.7700088709677413, + "pass@8": 0.8063951046233131, + "pass@16": 0.8307047111453599, + "pass@32": 0.848121899622024, + "pass@64": 0.8617897453414103, + "pass@128": 0.8741935483870967 + }, + "19": { + "pass@1": 0.47365936555891236, + "pass@2": 0.5826961218688298, + "pass@4": 0.6706477077449906, + "pass@8": 0.7371792407322068, + "pass@16": 0.7865671010611726, + "pass@32": 0.8245143993387958, + "pass@64": 0.8530351330994856, + "pass@128": 0.8731117824773413 + }, + "20": { + "pass@1": 0.480762012012012, + "pass@2": 0.5811989400818144, + "pass@4": 0.659820350721925, + "pass@8": 0.7209768024610734, + "pass@16": 0.7675705711799523, + "pass@32": 0.8036268873233177, + "pass@64": 0.8330538100617901, + "pass@128": 0.8588588588588588 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch4-op17-20/graphs/SDAR-100M-block4-epoch4-op17-20_score_report.jsonl b/composition/SDAR-100M-block4-epoch4-op17-20/graphs/SDAR-100M-block4-epoch4-op17-20_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..abe3a06223355638d57c7d188ea9416d47845bd9 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op17-20/graphs/SDAR-100M-block4-epoch4-op17-20_score_report.jsonl @@ -0,0 +1,2 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch4-op17-20", "results_dir": "results/composition/SDAR-100M-block4-epoch4-op17-20/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1024231, "value_mismatch": 9331, "dependency_mismatch": 125912, "answer_mismatch": 8254, "extra_nodes": 11996, "missing_nodes": 9189, "perfect_with_extra": 1155, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337764, "value_mismatch": 2687, "answer_mismatch": 2224, "dependency_mismatch": 40722, "missing_nodes": 2858, "extra_nodes": 3528, "perfect_with_extra": 197}, "teachers_in_school": {"total": 390656, "perfect": 346913, "value_mismatch": 3573, "dependency_mismatch": 43117, "answer_mismatch": 3304, "missing_nodes": 3471, "extra_nodes": 4306, "perfect_with_extra": 339}, "movie_festival_awards": {"total": 382336, "perfect": 339554, "dependency_mismatch": 42073, "value_mismatch": 3071, "answer_mismatch": 2726, "missing_nodes": 2860, "extra_nodes": 4162, "perfect_with_extra": 619}}, "results_path": "results/composition/SDAR-100M-block4-epoch4-op17-20/graphs/SDAR-100M-block4-epoch4-op17-20_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 510379, "value_mismatch": 427959, "dependency_mismatch": 667358, "answer_mismatch": 335680, "extra_nodes": 42751, "missing_nodes": 149816, "perfect_with_extra": 769, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "extra_nodes": 15638, "perfect": 170868, "perfect_with_extra": 101, "value_mismatch": 145132, "dependency_mismatch": 225312, "answer_mismatch": 113998, "missing_nodes": 54100}, "movie_festival_awards": {"total": 418816, "perfect": 165322, "value_mismatch": 144509, "dependency_mismatch": 219347, "answer_mismatch": 112616, "missing_nodes": 47022, "extra_nodes": 14000, "perfect_with_extra": 485}, "teachers_in_school": {"total": 430464, "perfect": 174189, "value_mismatch": 138318, "dependency_mismatch": 222699, "answer_mismatch": 109066, "missing_nodes": 48694, "extra_nodes": 13113, "perfect_with_extra": 183}}, "results_path": "results/composition/SDAR-100M-block4-epoch4-op17-20/graphs/SDAR-100M-block4-epoch4-op17-20_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1534610, "value_mismatch": 437290, "dependency_mismatch": 793270, "answer_mismatch": 343934, "extra_nodes": 54747, "missing_nodes": 159005, "perfect_with_extra": 1924, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch4-op17-20/graphs/SDAR-100M-block4-epoch4-op17-20_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 508632, "value_mismatch": 147819, "answer_mismatch": 116222, "dependency_mismatch": 266034, "missing_nodes": 56958, "extra_nodes": 19166, "perfect_with_extra": 298}, "teachers_in_school": {"total": 821120, "perfect": 521102, "value_mismatch": 141891, "dependency_mismatch": 265816, "answer_mismatch": 112370, "missing_nodes": 52165, "extra_nodes": 17419, "perfect_with_extra": 522}, "movie_festival_awards": {"total": 801152, "perfect": 504876, "dependency_mismatch": 261420, "value_mismatch": 147580, "answer_mismatch": 115342, "missing_nodes": 49882, "extra_nodes": 18162, "perfect_with_extra": 1104}}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch4-op17-20", "results_dir": "results/composition/SDAR-100M-block4-epoch4-op17-20/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch4-op17-20/graphs/SDAR-100M-block4-epoch4-op17-20_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch4-op17-20/graphs/SDAR-100M-block4-epoch4-op17-20_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch4-op17-20/graphs/SDAR-100M-block4-epoch4-op17-20_score_report.jsonl", "template_stats": {}}} diff --git a/composition/SDAR-100M-block4-epoch4-op17-20/graphs/results.json b/composition/SDAR-100M-block4-epoch4-op17-20/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..aa5ed0fdb540b9ec7c1f8cb71644520037580c8d --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op17-20/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block4-epoch4-op17-20_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9650793650793651, + "pass@2": 0.9777777777777777, + "pass@4": 0.9936507936507937, + "pass@8": 0.9968253968253968, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9640718562874252, + "pass@2": 0.9760479041916168, + "pass@4": 0.9970059880239521, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8875739644970414, + "pass@2": 0.9319526627218935, + "pass@4": 0.9733727810650887, + "pass@8": 0.9911242603550295, + "pass@16": 0.9970414201183432, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8237179487179487, + "pass@2": 0.8878205128205128, + "pass@4": 0.9455128205128205, + "pass@8": 0.9839743589743589, + "pass@16": 0.9935897435897436, + "pass@32": 0.9935897435897436, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7690140845070422, + "pass@2": 0.8450704225352113, + "pass@4": 0.9098591549295775, + "pass@8": 0.9633802816901409, + "pass@16": 0.9887323943661972, + "pass@32": 0.9915492957746479, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.640117994100295, + "pass@2": 0.7109144542772862, + "pass@4": 0.799410029498525, + "pass@8": 0.8820058997050148, + "pass@16": 0.9144542772861357, + "pass@32": 0.9321533923303835, + "pass@64": 0.943952802359882, + "pass@128": 0.9646017699115044 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9650793650793651, + "6": 0.9640718562874252, + "7": 0.8875739644970414, + "8": 0.8237179487179487, + "9": 0.7690140845070422, + "10": 0.640117994100295 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9777777777777777, + "6": 0.9760479041916168, + "7": 0.9319526627218935, + "8": 0.8878205128205128, + "9": 0.8450704225352113, + "10": 0.7109144542772862 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 0.9970059880239521, + "7": 0.9733727810650887, + "8": 0.9455128205128205, + "9": 0.9098591549295775, + "10": 0.799410029498525 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9911242603550295, + "8": 0.9839743589743589, + "9": 0.9633802816901409, + "10": 0.8820058997050148 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9935897435897436, + "9": 0.9887323943661972, + "10": 0.9144542772861357 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9935897435897436, + "9": 0.9915492957746479, + "10": 0.9321533923303835 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.943952802359882 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9646017699115044 + } + }, + "total_pass_at_k": { + "pass@1": 0.8912529550827423, + "pass@2": 0.9236744343127322, + "pass@4": 0.9564336372847011, + "pass@8": 0.9790611279972982, + "pass@16": 0.9878419452887538, + "pass@32": 0.9905437352245863, + "pass@64": 0.9935832489023978, + "pass@128": 0.9959473150962512 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9912280701754386, + "pass@2": 0.9970760233918129, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9647058823529412, + "pass@2": 0.9823529411764705, + "pass@4": 0.9882352941176471, + "pass@8": 0.9970588235294118, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9684542586750788, + "pass@2": 0.9747634069400631, + "pass@4": 0.9842271293375394, + "pass@8": 0.9905362776025236, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.870253164556962, + "pass@2": 0.9177215189873418, + "pass@4": 0.9683544303797469, + "pass@8": 0.990506329113924, + "pass@16": 0.9936708860759493, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7927927927927928, + "pass@2": 0.8468468468468469, + "pass@4": 0.918918918918919, + "pass@8": 0.972972972972973, + "pass@16": 0.993993993993994, + "pass@32": 0.996996996996997, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.7845659163987139, + "pass@2": 0.8392282958199357, + "pass@4": 0.9035369774919614, + "pass@8": 0.9485530546623794, + "pass@16": 0.9903536977491961, + "pass@32": 0.9967845659163987, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6059701492537314, + "pass@2": 0.6985074626865672, + "pass@4": 0.7850746268656716, + "pass@8": 0.8835820895522388, + "pass@16": 0.9432835820895522, + "pass@32": 0.9582089552238806, + "pass@64": 0.9701492537313433, + "pass@128": 0.9731343283582089 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9647058823529412, + "6": 0.9684542586750788, + "7": 0.870253164556962, + "8": 0.7927927927927928, + "9": 0.7845659163987139, + "10": 0.6059701492537314 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9823529411764705, + "6": 0.9747634069400631, + "7": 0.9177215189873418, + "8": 0.8468468468468469, + "9": 0.8392282958199357, + "10": 0.6985074626865672 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9882352941176471, + "6": 0.9842271293375394, + "7": 0.9683544303797469, + "8": 0.918918918918919, + "9": 0.9035369774919614, + "10": 0.7850746268656716 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9905362776025236, + "7": 0.990506329113924, + "8": 0.972972972972973, + "9": 0.9485530546623794, + "10": 0.8835820895522388 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.993993993993994, + "9": 0.9903536977491961, + "10": 0.9432835820895522 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 0.9967845659163987, + "10": 0.9582089552238806 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9701492537313433 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9731343283582089 + } + }, + "total_pass_at_k": { + "pass@1": 0.8881821225309675, + "pass@2": 0.9186474723803147, + "pass@4": 0.950451958486776, + "pass@8": 0.9762303314362236, + "pass@16": 0.9912956143287579, + "pass@32": 0.9946434549715434, + "pass@64": 0.9963173752929361, + "pass@128": 0.9966521593572146 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9763313609467456, + "pass@2": 0.9911242603550295, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9536231884057971, + "pass@2": 0.9855072463768116, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9799426934097422, + "pass@2": 0.9828080229226361, + "pass@4": 0.9914040114613181, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8670520231213873, + "pass@2": 0.9075144508670521, + "pass@4": 0.9450867052023122, + "pass@8": 0.9797687861271677, + "pass@16": 0.9971098265895953, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8309859154929577, + "pass@2": 0.8873239436619719, + "pass@4": 0.9436619718309859, + "pass@8": 0.9746478873239437, + "pass@16": 0.9915492957746479, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7335329341317365, + "pass@2": 0.7964071856287425, + "pass@4": 0.8772455089820359, + "pass@8": 0.9491017964071856, + "pass@16": 0.9730538922155688, + "pass@32": 0.9910179640718563, + "pass@64": 0.9970059880239521, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6656441717791411, + "pass@2": 0.7331288343558282, + "pass@4": 0.8159509202453987, + "pass@8": 0.8773006134969326, + "pass@16": 0.9141104294478528, + "pass@32": 0.9355828220858896, + "pass@64": 0.950920245398773, + "pass@128": 0.9662576687116564 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9763313609467456, + "5": 0.9536231884057971, + "6": 0.9799426934097422, + "7": 0.8670520231213873, + "8": 0.8309859154929577, + "9": 0.7335329341317365, + "10": 0.6656441717791411 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9911242603550295, + "5": 0.9855072463768116, + "6": 0.9828080229226361, + "7": 0.9075144508670521, + "8": 0.8873239436619719, + "9": 0.7964071856287425, + "10": 0.7331288343558282 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9914040114613181, + "7": 0.9450867052023122, + "8": 0.9436619718309859, + "9": 0.8772455089820359, + "10": 0.8159509202453987 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9797687861271677, + "8": 0.9746478873239437, + "9": 0.9491017964071856, + "10": 0.8773006134969326 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9915492957746479, + "9": 0.9730538922155688, + "10": 0.9141104294478528 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9910179640718563, + "10": 0.9355828220858896 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.950920245398773 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9662576687116564 + } + }, + "total_pass_at_k": { + "pass@1": 0.8902359108781127, + "pass@2": 0.9210353866317169, + "pass@4": 0.9531454783748362, + "pass@8": 0.9760812581913499, + "pass@16": 0.9865661861074705, + "pass@32": 0.9921363040629095, + "pass@64": 0.994429882044561, + "pass@128": 0.9963958060288336 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.987, + "pass@2": 0.996, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.961, + "pass@2": 0.982, + "pass@4": 0.994, + "pass@8": 0.998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.971, + "pass@2": 0.978, + "pass@4": 0.991, + "pass@8": 0.997, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.875, + "pass@2": 0.919, + "pass@4": 0.962, + "pass@8": 0.987, + "pass@16": 0.996, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.816, + "pass@2": 0.874, + "pass@4": 0.936, + "pass@8": 0.977, + "pass@16": 0.993, + "pass@32": 0.997, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.762, + "pass@2": 0.827, + "pass@4": 0.897, + "pass@8": 0.954, + "pass@16": 0.984, + "pass@32": 0.993, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.637, + "pass@2": 0.714, + "pass@4": 0.8, + "pass@8": 0.881, + "pass@16": 0.924, + "pass@32": 0.942, + "pass@64": 0.955, + "pass@128": 0.968 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.987, + "5": 0.961, + "6": 0.971, + "7": 0.875, + "8": 0.816, + "9": 0.762, + "10": 0.637 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.996, + "5": 0.982, + "6": 0.978, + "7": 0.919, + "8": 0.874, + "9": 0.827, + "10": 0.714 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.994, + "6": 0.991, + "7": 0.962, + "8": 0.936, + "9": 0.897, + "10": 0.8 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 0.997, + "7": 0.987, + "8": 0.977, + "9": 0.954, + "10": 0.881 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.996, + "8": 0.993, + "9": 0.984, + "10": 0.924 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.997, + "9": 0.993, + "10": 0.942 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.999, + "10": 0.955 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 1.0, + "10": 0.968 + } + }, + "total_pass_at_k": { + "pass@1": 0.8898888888888888, + "pass@2": 0.9211111111111111, + "pass@4": 0.9533333333333334, + "pass@8": 0.9771111111111112, + "pass@16": 0.9885555555555555, + "pass@32": 0.9924444444444445, + "pass@64": 0.9947777777777778, + "pass@128": 0.9963333333333333 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block4-epoch4-op17-20_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5694050991501416, + "pass@2": 0.6458923512747875, + "pass@4": 0.7393767705382436, + "pass@8": 0.8328611898016998, + "pass@16": 0.8980169971671388, + "pass@32": 0.9376770538243626, + "pass@64": 0.9461756373937678, + "pass@128": 0.9518413597733711 + }, + "12": { + "pass@1": 0.5790960451977402, + "pass@2": 0.6299435028248588, + "pass@4": 0.7175141242937854, + "pass@8": 0.7796610169491526, + "pass@16": 0.8135593220338984, + "pass@32": 0.844632768361582, + "pass@64": 0.8615819209039548, + "pass@128": 0.8700564971751412 + }, + "13": { + "pass@1": 0.5141065830721003, + "pass@2": 0.5830721003134797, + "pass@4": 0.6332288401253918, + "pass@8": 0.6927899686520376, + "pass@16": 0.7523510971786834, + "pass@32": 0.7836990595611285, + "pass@64": 0.799373040752351, + "pass@128": 0.8213166144200627 + }, + "14": { + "pass@1": 0.4735202492211838, + "pass@2": 0.514018691588785, + "pass@4": 0.573208722741433, + "pass@8": 0.632398753894081, + "pass@16": 0.6884735202492211, + "pass@32": 0.7445482866043613, + "pass@64": 0.7757009345794392, + "pass@128": 0.8286604361370716 + }, + "15": { + "pass@1": 0.43352601156069365, + "pass@2": 0.4797687861271676, + "pass@4": 0.5317919075144508, + "pass@8": 0.5982658959537572, + "pass@16": 0.653179190751445, + "pass@32": 0.7283236994219653, + "pass@64": 0.8034682080924855, + "pass@128": 0.8497109826589595 + }, + "16": { + "pass@1": 0.32903225806451614, + "pass@2": 0.38387096774193546, + "pass@4": 0.4258064516129032, + "pass@8": 0.45806451612903226, + "pass@16": 0.5193548387096775, + "pass@32": 0.5967741935483871, + "pass@64": 0.6258064516129033, + "pass@128": 0.6838709677419355 + }, + "17": { + "pass@1": 0.36127167630057805, + "pass@2": 0.3872832369942196, + "pass@4": 0.430635838150289, + "pass@8": 0.4653179190751445, + "pass@16": 0.5086705202312138, + "pass@32": 0.5635838150289018, + "pass@64": 0.6589595375722543, + "pass@128": 0.6936416184971098 + }, + "18": { + "pass@1": 0.28, + "pass@2": 0.31142857142857144, + "pass@4": 0.34285714285714286, + "pass@8": 0.37714285714285717, + "pass@16": 0.42, + "pass@32": 0.49142857142857144, + "pass@64": 0.5314285714285715, + "pass@128": 0.6 + }, + "19": { + "pass@1": 0.1891891891891892, + "pass@2": 0.24024024024024024, + "pass@4": 0.26426426426426425, + "pass@8": 0.2882882882882883, + "pass@16": 0.30930930930930933, + "pass@32": 0.35435435435435436, + "pass@64": 0.3933933933933934, + "pass@128": 0.44144144144144143 + }, + "20": { + "pass@1": 0.21621621621621623, + "pass@2": 0.26426426426426425, + "pass@4": 0.2882882882882883, + "pass@8": 0.3213213213213213, + "pass@16": 0.34534534534534533, + "pass@32": 0.3933933933933934, + "pass@64": 0.43243243243243246, + "pass@128": 0.4864864864864865 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5694050991501416, + "12": 0.5790960451977402, + "13": 0.5141065830721003, + "14": 0.4735202492211838, + "15": 0.43352601156069365, + "16": 0.32903225806451614, + "17": 0.36127167630057805, + "18": 0.28, + "19": 0.1891891891891892, + "20": 0.21621621621621623 + }, + "pass@2": { + "11": 0.6458923512747875, + "12": 0.6299435028248588, + "13": 0.5830721003134797, + "14": 0.514018691588785, + "15": 0.4797687861271676, + "16": 0.38387096774193546, + "17": 0.3872832369942196, + "18": 0.31142857142857144, + "19": 0.24024024024024024, + "20": 0.26426426426426425 + }, + "pass@4": { + "11": 0.7393767705382436, + "12": 0.7175141242937854, + "13": 0.6332288401253918, + "14": 0.573208722741433, + "15": 0.5317919075144508, + "16": 0.4258064516129032, + "17": 0.430635838150289, + "18": 0.34285714285714286, + "19": 0.26426426426426425, + "20": 0.2882882882882883 + }, + "pass@8": { + "11": 0.8328611898016998, + "12": 0.7796610169491526, + "13": 0.6927899686520376, + "14": 0.632398753894081, + "15": 0.5982658959537572, + "16": 0.45806451612903226, + "17": 0.4653179190751445, + "18": 0.37714285714285717, + "19": 0.2882882882882883, + "20": 0.3213213213213213 + }, + "pass@16": { + "11": 0.8980169971671388, + "12": 0.8135593220338984, + "13": 0.7523510971786834, + "14": 0.6884735202492211, + "15": 0.653179190751445, + "16": 0.5193548387096775, + "17": 0.5086705202312138, + "18": 0.42, + "19": 0.30930930930930933, + "20": 0.34534534534534533 + }, + "pass@32": { + "11": 0.9376770538243626, + "12": 0.844632768361582, + "13": 0.7836990595611285, + "14": 0.7445482866043613, + "15": 0.7283236994219653, + "16": 0.5967741935483871, + "17": 0.5635838150289018, + "18": 0.49142857142857144, + "19": 0.35435435435435436, + "20": 0.3933933933933934 + }, + "pass@64": { + "11": 0.9461756373937678, + "12": 0.8615819209039548, + "13": 0.799373040752351, + "14": 0.7757009345794392, + "15": 0.8034682080924855, + "16": 0.6258064516129033, + "17": 0.6589595375722543, + "18": 0.5314285714285715, + "19": 0.3933933933933934, + "20": 0.43243243243243246 + }, + "pass@128": { + "11": 0.9518413597733711, + "12": 0.8700564971751412, + "13": 0.8213166144200627, + "14": 0.8286604361370716, + "15": 0.8497109826589595, + "16": 0.6838709677419355, + "17": 0.6936416184971098, + "18": 0.6, + "19": 0.44144144144144143, + "20": 0.4864864864864865 + } + }, + "total_pass_at_k": { + "pass@1": 0.39583952451708765, + "pass@2": 0.4451708766716196, + "pass@4": 0.4962852897473997, + "pass@8": 0.5465081723625557, + "pass@16": 0.5925705794947994, + "pass@32": 0.6454680534918277, + "pass@64": 0.6846953937592868, + "pass@128": 0.7242199108469539 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5904761904761905, + "pass@2": 0.6634920634920635, + "pass@4": 0.7365079365079366, + "pass@8": 0.8444444444444444, + "pass@16": 0.9206349206349206, + "pass@32": 0.946031746031746, + "pass@64": 0.9746031746031746, + "pass@128": 0.9777777777777777 + }, + "12": { + "pass@1": 0.5871559633027523, + "pass@2": 0.654434250764526, + "pass@4": 0.7155963302752294, + "pass@8": 0.8042813455657493, + "pass@16": 0.8470948012232415, + "pass@32": 0.8837920489296636, + "pass@64": 0.908256880733945, + "pass@128": 0.9174311926605505 + }, + "13": { + "pass@1": 0.5092592592592593, + "pass@2": 0.5524691358024691, + "pass@4": 0.6141975308641975, + "pass@8": 0.6759259259259259, + "pass@16": 0.7530864197530864, + "pass@32": 0.7746913580246914, + "pass@64": 0.7932098765432098, + "pass@128": 0.8148148148148148 + }, + "14": { + "pass@1": 0.4836795252225519, + "pass@2": 0.516320474777448, + "pass@4": 0.5578635014836796, + "pass@8": 0.6201780415430267, + "pass@16": 0.7002967359050445, + "pass@32": 0.7744807121661721, + "pass@64": 0.8071216617210683, + "pass@128": 0.827893175074184 + }, + "15": { + "pass@1": 0.3734567901234568, + "pass@2": 0.44135802469135804, + "pass@4": 0.47530864197530864, + "pass@8": 0.5370370370370371, + "pass@16": 0.6049382716049383, + "pass@32": 0.7067901234567902, + "pass@64": 0.7777777777777778, + "pass@128": 0.8240740740740741 + }, + "16": { + "pass@1": 0.37168141592920356, + "pass@2": 0.4336283185840708, + "pass@4": 0.4749262536873156, + "pass@8": 0.5339233038348082, + "pass@16": 0.5752212389380531, + "pass@32": 0.640117994100295, + "pass@64": 0.6784660766961652, + "pass@128": 0.7227138643067846 + }, + "17": { + "pass@1": 0.31626506024096385, + "pass@2": 0.3674698795180723, + "pass@4": 0.4036144578313253, + "pass@8": 0.4457831325301205, + "pass@16": 0.49698795180722893, + "pass@32": 0.5331325301204819, + "pass@64": 0.5963855421686747, + "pass@128": 0.6445783132530121 + }, + "18": { + "pass@1": 0.29354838709677417, + "pass@2": 0.33548387096774196, + "pass@4": 0.36774193548387096, + "pass@8": 0.4258064516129032, + "pass@16": 0.47419354838709676, + "pass@32": 0.532258064516129, + "pass@64": 0.5838709677419355, + "pass@128": 0.632258064516129 + }, + "19": { + "pass@1": 0.19939577039274925, + "pass@2": 0.25075528700906347, + "pass@4": 0.29003021148036257, + "pass@8": 0.3081570996978852, + "pass@16": 0.32628398791540786, + "pass@32": 0.37462235649546827, + "pass@64": 0.4169184290030212, + "pass@128": 0.4501510574018127 + }, + "20": { + "pass@1": 0.19519519519519518, + "pass@2": 0.24324324324324326, + "pass@4": 0.2852852852852853, + "pass@8": 0.3033033033033033, + "pass@16": 0.3213213213213213, + "pass@32": 0.35435435435435436, + "pass@64": 0.39039039039039036, + "pass@128": 0.4444444444444444 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5904761904761905, + "12": 0.5871559633027523, + "13": 0.5092592592592593, + "14": 0.4836795252225519, + "15": 0.3734567901234568, + "16": 0.37168141592920356, + "17": 0.31626506024096385, + "18": 0.29354838709677417, + "19": 0.19939577039274925, + "20": 0.19519519519519518 + }, + "pass@2": { + "11": 0.6634920634920635, + "12": 0.654434250764526, + "13": 0.5524691358024691, + "14": 0.516320474777448, + "15": 0.44135802469135804, + "16": 0.4336283185840708, + "17": 0.3674698795180723, + "18": 0.33548387096774196, + "19": 0.25075528700906347, + "20": 0.24324324324324326 + }, + "pass@4": { + "11": 0.7365079365079366, + "12": 0.7155963302752294, + "13": 0.6141975308641975, + "14": 0.5578635014836796, + "15": 0.47530864197530864, + "16": 0.4749262536873156, + "17": 0.4036144578313253, + "18": 0.36774193548387096, + "19": 0.29003021148036257, + "20": 0.2852852852852853 + }, + "pass@8": { + "11": 0.8444444444444444, + "12": 0.8042813455657493, + "13": 0.6759259259259259, + "14": 0.6201780415430267, + "15": 0.5370370370370371, + "16": 0.5339233038348082, + "17": 0.4457831325301205, + "18": 0.4258064516129032, + "19": 0.3081570996978852, + "20": 0.3033033033033033 + }, + "pass@16": { + "11": 0.9206349206349206, + "12": 0.8470948012232415, + "13": 0.7530864197530864, + "14": 0.7002967359050445, + "15": 0.6049382716049383, + "16": 0.5752212389380531, + "17": 0.49698795180722893, + "18": 0.47419354838709676, + "19": 0.32628398791540786, + "20": 0.3213213213213213 + }, + "pass@32": { + "11": 0.946031746031746, + "12": 0.8837920489296636, + "13": 0.7746913580246914, + "14": 0.7744807121661721, + "15": 0.7067901234567902, + "16": 0.640117994100295, + "17": 0.5331325301204819, + "18": 0.532258064516129, + "19": 0.37462235649546827, + "20": 0.35435435435435436 + }, + "pass@64": { + "11": 0.9746031746031746, + "12": 0.908256880733945, + "13": 0.7932098765432098, + "14": 0.8071216617210683, + "15": 0.7777777777777778, + "16": 0.6784660766961652, + "17": 0.5963855421686747, + "18": 0.5838709677419355, + "19": 0.4169184290030212, + "20": 0.39039039039039036 + }, + "pass@128": { + "11": 0.9777777777777777, + "12": 0.9174311926605505, + "13": 0.8148148148148148, + "14": 0.827893175074184, + "15": 0.8240740740740741, + "16": 0.7227138643067846, + "17": 0.6445783132530121, + "18": 0.632258064516129, + "19": 0.4501510574018127, + "20": 0.4444444444444444 + } + }, + "total_pass_at_k": { + "pass@1": 0.39119804400978, + "pass@2": 0.4449877750611247, + "pass@4": 0.49113691931540343, + "pass@8": 0.5485941320293398, + "pass@16": 0.6005501222493888, + "pass@32": 0.6506723716381418, + "pass@64": 0.691320293398533, + "pass@128": 0.7243276283618582 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6385542168674698, + "pass@2": 0.7048192771084337, + "pass@4": 0.7650602409638554, + "pass@8": 0.8524096385542169, + "pass@16": 0.9216867469879518, + "pass@32": 0.9397590361445783, + "pass@64": 0.9578313253012049, + "pass@128": 0.9668674698795181 + }, + "12": { + "pass@1": 0.567398119122257, + "pass@2": 0.6269592476489029, + "pass@4": 0.7053291536050157, + "pass@8": 0.7554858934169278, + "pass@16": 0.8119122257053292, + "pass@32": 0.8526645768025078, + "pass@64": 0.877742946708464, + "pass@128": 0.8871473354231975 + }, + "13": { + "pass@1": 0.5350140056022409, + "pass@2": 0.5994397759103641, + "pass@4": 0.6722689075630253, + "pass@8": 0.7507002801120448, + "pass@16": 0.8011204481792717, + "pass@32": 0.8235294117647058, + "pass@64": 0.8375350140056023, + "pass@128": 0.8543417366946778 + }, + "14": { + "pass@1": 0.49415204678362573, + "pass@2": 0.5321637426900585, + "pass@4": 0.5847953216374269, + "pass@8": 0.6491228070175439, + "pass@16": 0.7339181286549707, + "pass@32": 0.7953216374269005, + "pass@64": 0.8333333333333334, + "pass@128": 0.8713450292397661 + }, + "15": { + "pass@1": 0.44545454545454544, + "pass@2": 0.49393939393939396, + "pass@4": 0.5424242424242425, + "pass@8": 0.6, + "pass@16": 0.6727272727272727, + "pass@32": 0.7363636363636363, + "pass@64": 0.796969696969697, + "pass@128": 0.8515151515151516 + }, + "16": { + "pass@1": 0.36182336182336183, + "pass@2": 0.42450142450142453, + "pass@4": 0.46153846153846156, + "pass@8": 0.5242165242165242, + "pass@16": 0.5868945868945868, + "pass@32": 0.6438746438746439, + "pass@64": 0.6923076923076923, + "pass@128": 0.7378917378917379 + }, + "17": { + "pass@1": 0.2888198757763975, + "pass@2": 0.3385093167701863, + "pass@4": 0.37267080745341613, + "pass@8": 0.43788819875776397, + "pass@16": 0.5124223602484472, + "pass@32": 0.577639751552795, + "pass@64": 0.6242236024844721, + "pass@128": 0.6645962732919255 + }, + "18": { + "pass@1": 0.2676470588235294, + "pass@2": 0.3088235294117647, + "pass@4": 0.34411764705882353, + "pass@8": 0.37941176470588234, + "pass@16": 0.45, + "pass@32": 0.5029411764705882, + "pass@64": 0.5617647058823529, + "pass@128": 0.6 + }, + "19": { + "pass@1": 0.1875, + "pass@2": 0.22916666666666666, + "pass@4": 0.27976190476190477, + "pass@8": 0.2976190476190476, + "pass@16": 0.32142857142857145, + "pass@32": 0.3601190476190476, + "pass@64": 0.4166666666666667, + "pass@128": 0.47619047619047616 + }, + "20": { + "pass@1": 0.20658682634730538, + "pass@2": 0.25149700598802394, + "pass@4": 0.27844311377245506, + "pass@8": 0.2904191616766467, + "pass@16": 0.3263473053892216, + "pass@32": 0.3592814371257485, + "pass@64": 0.4041916167664671, + "pass@128": 0.4431137724550898 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6385542168674698, + "12": 0.567398119122257, + "13": 0.5350140056022409, + "14": 0.49415204678362573, + "15": 0.44545454545454544, + "16": 0.36182336182336183, + "17": 0.2888198757763975, + "18": 0.2676470588235294, + "19": 0.1875, + "20": 0.20658682634730538 + }, + "pass@2": { + "11": 0.7048192771084337, + "12": 0.6269592476489029, + "13": 0.5994397759103641, + "14": 0.5321637426900585, + "15": 0.49393939393939396, + "16": 0.42450142450142453, + "17": 0.3385093167701863, + "18": 0.3088235294117647, + "19": 0.22916666666666666, + "20": 0.25149700598802394 + }, + "pass@4": { + "11": 0.7650602409638554, + "12": 0.7053291536050157, + "13": 0.6722689075630253, + "14": 0.5847953216374269, + "15": 0.5424242424242425, + "16": 0.46153846153846156, + "17": 0.37267080745341613, + "18": 0.34411764705882353, + "19": 0.27976190476190477, + "20": 0.27844311377245506 + }, + "pass@8": { + "11": 0.8524096385542169, + "12": 0.7554858934169278, + "13": 0.7507002801120448, + "14": 0.6491228070175439, + "15": 0.6, + "16": 0.5242165242165242, + "17": 0.43788819875776397, + "18": 0.37941176470588234, + "19": 0.2976190476190476, + "20": 0.2904191616766467 + }, + "pass@16": { + "11": 0.9216867469879518, + "12": 0.8119122257053292, + "13": 0.8011204481792717, + "14": 0.7339181286549707, + "15": 0.6727272727272727, + "16": 0.5868945868945868, + "17": 0.5124223602484472, + "18": 0.45, + "19": 0.32142857142857145, + "20": 0.3263473053892216 + }, + "pass@32": { + "11": 0.9397590361445783, + "12": 0.8526645768025078, + "13": 0.8235294117647058, + "14": 0.7953216374269005, + "15": 0.7363636363636363, + "16": 0.6438746438746439, + "17": 0.577639751552795, + "18": 0.5029411764705882, + "19": 0.3601190476190476, + "20": 0.3592814371257485 + }, + "pass@64": { + "11": 0.9578313253012049, + "12": 0.877742946708464, + "13": 0.8375350140056023, + "14": 0.8333333333333334, + "15": 0.796969696969697, + "16": 0.6923076923076923, + "17": 0.6242236024844721, + "18": 0.5617647058823529, + "19": 0.4166666666666667, + "20": 0.4041916167664671 + }, + "pass@128": { + "11": 0.9668674698795181, + "12": 0.8871473354231975, + "13": 0.8543417366946778, + "14": 0.8713450292397661, + "15": 0.8515151515151516, + "16": 0.7378917378917379, + "17": 0.6645962732919255, + "18": 0.6, + "19": 0.47619047619047616, + "20": 0.4431137724550898 + } + }, + "total_pass_at_k": { + "pass@1": 0.3993458221825751, + "pass@2": 0.45108534046981863, + "pass@4": 0.5007433838834374, + "pass@8": 0.5539696699375558, + "pass@16": 0.6140350877192983, + "pass@32": 0.6592328278322926, + "pass@64": 0.7002676181980375, + "pass@128": 0.7353553374962831 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.599, + "pass@2": 0.671, + "pass@4": 0.747, + "pass@8": 0.843, + "pass@16": 0.913, + "pass@32": 0.941, + "pass@64": 0.959, + "pass@128": 0.965 + }, + "12": { + "pass@1": 0.578, + "pass@2": 0.637, + "pass@4": 0.713, + "pass@8": 0.78, + "pass@16": 0.824, + "pass@32": 0.86, + "pass@64": 0.882, + "pass@128": 0.891 + }, + "13": { + "pass@1": 0.52, + "pass@2": 0.579, + "pass@4": 0.641, + "pass@8": 0.708, + "pass@16": 0.77, + "pass@32": 0.795, + "pass@64": 0.811, + "pass@128": 0.831 + }, + "14": { + "pass@1": 0.484, + "pass@2": 0.521, + "pass@4": 0.572, + "pass@8": 0.634, + "pass@16": 0.708, + "pass@32": 0.772, + "pass@64": 0.806, + "pass@128": 0.843 + }, + "15": { + "pass@1": 0.418, + "pass@2": 0.472, + "pass@4": 0.517, + "pass@8": 0.579, + "pass@16": 0.644, + "pass@32": 0.724, + "pass@64": 0.793, + "pass@128": 0.842 + }, + "16": { + "pass@1": 0.355, + "pass@2": 0.415, + "pass@4": 0.455, + "pass@8": 0.507, + "pass@16": 0.562, + "pass@32": 0.628, + "pass@64": 0.667, + "pass@128": 0.716 + }, + "17": { + "pass@1": 0.323, + "pass@2": 0.365, + "pass@4": 0.403, + "pass@8": 0.45, + "pass@16": 0.506, + "pass@32": 0.558, + "pass@64": 0.627, + "pass@128": 0.668 + }, + "18": { + "pass@1": 0.28, + "pass@2": 0.318, + "pass@4": 0.351, + "pass@8": 0.393, + "pass@16": 0.447, + "pass@32": 0.508, + "pass@64": 0.558, + "pass@128": 0.61 + }, + "19": { + "pass@1": 0.192, + "pass@2": 0.24, + "pass@4": 0.278, + "pass@8": 0.298, + "pass@16": 0.319, + "pass@32": 0.363, + "pass@64": 0.409, + "pass@128": 0.456 + }, + "20": { + "pass@1": 0.206, + "pass@2": 0.253, + "pass@4": 0.284, + "pass@8": 0.305, + "pass@16": 0.331, + "pass@32": 0.369, + "pass@64": 0.409, + "pass@128": 0.458 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.599, + "12": 0.578, + "13": 0.52, + "14": 0.484, + "15": 0.418, + "16": 0.355, + "17": 0.323, + "18": 0.28, + "19": 0.192, + "20": 0.206 + }, + "pass@2": { + "11": 0.671, + "12": 0.637, + "13": 0.579, + "14": 0.521, + "15": 0.472, + "16": 0.415, + "17": 0.365, + "18": 0.318, + "19": 0.24, + "20": 0.253 + }, + "pass@4": { + "11": 0.747, + "12": 0.713, + "13": 0.641, + "14": 0.572, + "15": 0.517, + "16": 0.455, + "17": 0.403, + "18": 0.351, + "19": 0.278, + "20": 0.284 + }, + "pass@8": { + "11": 0.843, + "12": 0.78, + "13": 0.708, + "14": 0.634, + "15": 0.579, + "16": 0.507, + "17": 0.45, + "18": 0.393, + "19": 0.298, + "20": 0.305 + }, + "pass@16": { + "11": 0.913, + "12": 0.824, + "13": 0.77, + "14": 0.708, + "15": 0.644, + "16": 0.562, + "17": 0.506, + "18": 0.447, + "19": 0.319, + "20": 0.331 + }, + "pass@32": { + "11": 0.941, + "12": 0.86, + "13": 0.795, + "14": 0.772, + "15": 0.724, + "16": 0.628, + "17": 0.558, + "18": 0.508, + "19": 0.363, + "20": 0.369 + }, + "pass@64": { + "11": 0.959, + "12": 0.882, + "13": 0.811, + "14": 0.806, + "15": 0.793, + "16": 0.667, + "17": 0.627, + "18": 0.558, + "19": 0.409, + "20": 0.409 + }, + "pass@128": { + "11": 0.965, + "12": 0.891, + "13": 0.831, + "14": 0.843, + "15": 0.842, + "16": 0.716, + "17": 0.668, + "18": 0.61, + "19": 0.456, + "20": 0.458 + } + }, + "total_pass_at_k": { + "pass@1": 0.3955, + "pass@2": 0.4471, + "pass@4": 0.4961, + "pass@8": 0.5497, + "pass@16": 0.6024, + "pass@32": 0.6518, + "pass@64": 0.6921, + "pass@128": 0.728 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block4-epoch4-op17-20/summary.csv b/composition/SDAR-100M-block4-epoch4-op17-20/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..8ace68e34ff36b952189619a68672be8bb4e6337 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op17-20/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch4-op17-20,0.9928307291666667,0.73775,0.8585777138157895,0.0,0.0,0.0,199.24223958333334,375.910928125,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block4-epoch4-op17-20/summary.json b/composition/SDAR-100M-block4-epoch4-op17-20/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ea76301e4e6ec38fed2724c30850d96785e069b9 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op17-20/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch4-op17-20", + "id_acc": 0.9928307291666667, + "ood_acc": 0.73775, + "total_acc": 0.8585777138157895, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.24223958333334, + "ood_avg_resp_len": 375.910928125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 693506, + "answer_accuracy": 0.8564678509326589, + "avg_response_len": 279.12083810859946, + "resp_tokens_sum": 226011958, + "pass_at_k": { + "pass@1": 0.8564678509326589, + "pass@2": 0.8872786249909652, + "pass@4": 0.9083851630613429, + "pass@8": 0.92354058650134, + "pass@16": 0.9350769404513234, + "pass@32": 0.9446203133284556, + "pass@64": 0.9529550748775423, + "pass@128": 0.9601644008852356 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9613753687315634, + "pass@2": 0.970923193505679, + "pass@4": 0.97719835463045, + "pass@8": 0.9817599651022448, + "pass@16": 0.9862649009949807, + "pass@32": 0.9907192299119268, + "pass@64": 0.9939562234532361, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999522171253823, + "pass@2": 0.9999996237568928, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99921875, + "pass@2": 0.9998327540600395, + "pass@4": 0.9999919721948819, + "pass@8": 0.9999999872068569, + "pass@16": 0.9999999999999952, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9957093253968254, + "pass@2": 0.9977174728158981, + "pass@4": 0.9987727721534809, + "pass@8": 0.9995512405280322, + "pass@16": 0.9999456111182624, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993450598802395, + "pass@2": 0.9998802837238909, + "pass@4": 0.999995804229561, + "pass@8": 0.9999999967283617, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9984051405325444, + "pass@2": 0.999718993151004, + "pass@4": 0.9999875685842522, + "pass@8": 0.9999999772191345, + "pass@16": 0.9999999999999808, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9976712740384616, + "pass@2": 0.9997326430950938, + "pass@4": 0.9999875658571523, + "pass@8": 0.9999999688739251, + "pass@16": 0.9999999999999599, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9966769366197183, + "pass@2": 0.9972888017078853, + "pass@4": 0.997440846918783, + "pass@8": 0.997682618016843, + "pass@16": 0.9981192931869433, + "pass@32": 0.9988210100498001, + "pass@64": 0.9996562049462128, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9425017705382436, + "pass@2": 0.9548415159153264, + "pass@4": 0.9637315494911575, + "pass@8": 0.9707074281466466, + "pass@16": 0.9764958731179654, + "pass@32": 0.9818812626501979, + "pass@64": 0.9869854923560497, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.884622175141243, + "pass@2": 0.9070330224876552, + "pass@4": 0.9212047249390439, + "pass@8": 0.9304309349365792, + "pass@16": 0.93734400736929, + "pass@32": 0.9441033505020082, + "pass@64": 0.9506885517975576, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8455867946708464, + "pass@2": 0.8826244816478657, + "pass@4": 0.9036141996748837, + "pass@8": 0.9170823611637505, + "pass@16": 0.9287953901231346, + "pass@32": 0.9401314416183841, + "pass@64": 0.9503814510788167, + "pass@128": 0.9592476489028213 + }, + "14": { + "pass@1": 0.8548481308411215, + "pass@2": 0.890110644025805, + "pass@4": 0.9090667565502909, + "pass@8": 0.9206775494670285, + "pass@16": 0.929940970807209, + "pass@32": 0.9389608922498822, + "pass@64": 0.9479723802614771, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.8491239161849711, + "pass@2": 0.891381323676665, + "pass@4": 0.9182221973878987, + "pass@8": 0.9374987178643029, + "pass@16": 0.9500343576067202, + "pass@32": 0.9582461300623976, + "pass@64": 0.9655452112581264, + "pass@128": 0.9710982658959537 + }, + "16": { + "pass@1": 0.6878780241935484, + "pass@2": 0.7575481807213613, + "pass@4": 0.8034270958065721, + "pass@8": 0.8332422545728394, + "pass@16": 0.8555616965823695, + "pass@32": 0.8752240577779585, + "pass@64": 0.8934588712339225, + "pass@128": 0.9096774193548387 + }, + "17": { + "pass@1": 0.6582821531791907, + "pass@2": 0.7333793693960219, + "pass@4": 0.7873619814864179, + "pass@8": 0.8268060950621734, + "pass@16": 0.8541651488113018, + "pass@32": 0.8744259072106768, + "pass@64": 0.8921529102764344, + "pass@128": 0.9075144508670521 + }, + "18": { + "pass@1": 0.6214508928571428, + "pass@2": 0.702418799212598, + "pass@4": 0.7587747707429429, + "pass@8": 0.7988930690855568, + "pass@16": 0.8295439274930722, + "pass@32": 0.8551884496518796, + "pass@64": 0.8759012824017393, + "pass@128": 0.8942857142857142 + }, + "19": { + "pass@1": 0.4751783033033033, + "pass@2": 0.5727038554302333, + "pass@4": 0.6456607301902577, + "pass@8": 0.7009178523026034, + "pass@16": 0.7443995896195046, + "pass@32": 0.780325551418712, + "pass@64": 0.8128709427686198, + "pass@128": 0.8408408408408409 + }, + "20": { + "pass@1": 0.5220298423423423, + "pass@2": 0.6148334604289327, + "pass@4": 0.6857166376162438, + "pass@8": 0.7404605152943717, + "pass@16": 0.782629162910974, + "pass@32": 0.8153740821980516, + "pass@64": 0.8413122377082638, + "pass@128": 0.8618618618618619 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 708745, + "answer_accuracy": 0.8631442420109119, + "avg_response_len": 288.41504895752144, + "resp_tokens_sum": 236823365, + "pass_at_k": { + "pass@1": 0.8631442420109119, + "pass@2": 0.8931764603445276, + "pass@4": 0.9140685113600835, + "pass@8": 0.929887554748734, + "pass@16": 0.9423626909406467, + "pass@32": 0.9525094088902438, + "pass@64": 0.960887185544163, + "pass@128": 0.9680436477007015 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.939704754601227, + "pass@2": 0.951967522704217, + "pass@4": 0.9620432760322137, + "pass@8": 0.9700759835321355, + "pass@16": 0.9765457934086171, + "pass@32": 0.9828590570934914, + "pass@64": 0.9886812890839152, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9984148550724637, + "pass@2": 0.9996868937578454, + "pass@4": 0.9999772702325253, + "pass@8": 0.9999998752549307, + "pass@16": 0.999999999998829, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9977390759312321, + "pass@2": 0.9992202129819733, + "pass@4": 0.9998640429616784, + "pass@8": 0.9999948677008761, + "pass@16": 0.9999999954075054, + "pass@32": 0.9999999999999999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9953937861271677, + "pass@2": 0.9963599033953849, + "pass@4": 0.9971111784798576, + "pass@8": 0.9976008320837435, + "pass@16": 0.9980702739375236, + "pass@32": 0.9987903426808435, + "pass@64": 0.999647262300305, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956646126760563, + "pass@2": 0.9985024814239768, + "pass@4": 0.999711590540619, + "pass@8": 0.9999853934625496, + "pass@16": 0.9999999656383994, + "pass@32": 0.9999999999999744, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9948540419161677, + "pass@2": 0.9966571532132587, + "pass@4": 0.9971588291358788, + "pass@8": 0.9973697231456297, + "pass@16": 0.9977102880819406, + "pass@32": 0.9983202885567448, + "pass@64": 0.9992573907303504, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.961902296686747, + "pass@2": 0.9702211898776207, + "pass@4": 0.9745907889149398, + "pass@8": 0.9773632430076152, + "pass@16": 0.9793331174520167, + "pass@32": 0.9809097132926594, + "pass@64": 0.9826862351534986, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.898951802507837, + "pass@2": 0.9214403401377336, + "pass@4": 0.9366077708154816, + "pass@8": 0.9473168294237594, + "pass@16": 0.9557104493167832, + "pass@32": 0.9632188528557657, + "pass@64": 0.9697692688760374, + "pass@128": 0.9749216300940439 + }, + "13": { + "pass@1": 0.8447785364145658, + "pass@2": 0.8756985569818482, + "pass@4": 0.8945737355309575, + "pass@8": 0.9089284772307276, + "pass@16": 0.9205185085929912, + "pass@32": 0.9308260468285129, + "pass@64": 0.9408199495389861, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.8758223684210527, + "pass@2": 0.9139463035870513, + "pass@4": 0.9350603600536769, + "pass@8": 0.9487797027077906, + "pass@16": 0.9591678574023239, + "pass@32": 0.9683737126824168, + "pass@64": 0.975387169545268, + "pass@128": 0.97953216374269 + }, + "15": { + "pass@1": 0.8341619318181818, + "pass@2": 0.8827543396564063, + "pass@4": 0.9144763560804903, + "pass@8": 0.9369795398942257, + "pass@16": 0.9540667236743633, + "pass@32": 0.9664573273663126, + "pass@64": 0.9747860909253457, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.7109375, + "pass@2": 0.7824315930861211, + "pass@4": 0.8306402492957612, + "pass@8": 0.8662793559814501, + "pass@16": 0.8918811624991876, + "pass@32": 0.9103467438874276, + "pass@64": 0.9247234008320129, + "pass@128": 0.9344729344729344 + }, + "17": { + "pass@1": 0.710549301242236, + "pass@2": 0.7828643107790872, + "pass@4": 0.8275231751892806, + "pass@8": 0.8615275014644767, + "pass@16": 0.8904357410433366, + "pass@32": 0.9134983203987874, + "pass@64": 0.9284336977111665, + "pass@128": 0.937888198757764 + }, + "18": { + "pass@1": 0.6790900735294118, + "pass@2": 0.7502435299907362, + "pass@4": 0.800968797834094, + "pass@8": 0.8404703821156387, + "pass@16": 0.8728190470546322, + "pass@32": 0.8996975551996976, + "pass@64": 0.9220785178168743, + "pass@128": 0.9411764705882353 + }, + "19": { + "pass@1": 0.4720982142857143, + "pass@2": 0.565974175103112, + "pass@4": 0.6416543777451924, + "pass@8": 0.7033531553412125, + "pass@16": 0.7537467764321175, + "pass@32": 0.7949539015604953, + "pass@64": 0.831013697436013, + "pass@128": 0.8660714285714286 + }, + "20": { + "pass@1": 0.4807260479041916, + "pass@2": 0.5753041161770944, + "pass@4": 0.6497437565813254, + "pass@8": 0.7074401183175504, + "pass@16": 0.7516001349159216, + "pass@32": 0.787087003435915, + "pass@64": 0.8178073059749411, + "pass@128": 0.8473053892215568 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 685810, + "answer_accuracy": 0.8560298170634286, + "avg_response_len": 309.37665386842946, + "resp_tokens_sum": 247857725, + "pass_at_k": { + "pass@1": 0.8560298170634286, + "pass@2": 0.8883146458076645, + "pass@4": 0.910479940771302, + "pass@8": 0.9262575954472617, + "pass@16": 0.9379948137541099, + "pass@32": 0.9472316393793455, + "pass@64": 0.9546323098937601, + "pass@128": 0.9605368269691644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9630597014925373, + "pass@2": 0.9736988042073099, + "pass@4": 0.981951940242544, + "pass@8": 0.9884366055042368, + "pass@16": 0.9932069977085, + "pass@32": 0.995981258815499, + "pass@64": 0.9969530935342759, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997266763848397, + "pass@2": 0.9999802719175408, + "pass@4": 0.9999999098144802, + "pass@8": 0.9999999999996636, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9951516544117647, + "pass@2": 0.9985160230430754, + "pass@4": 0.9997385302940074, + "pass@8": 0.9999862846484304, + "pass@16": 0.9999999645862426, + "pass@32": 0.9999999999999732, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985705835962145, + "pass@2": 0.9994023075585583, + "pass@4": 0.9998913907449267, + "pass@8": 0.9999968657441765, + "pass@16": 0.9999999985924529, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9972310126582279, + "pass@2": 0.9978734239509618, + "pass@4": 0.9985819919266421, + "pass@8": 0.9993828519658078, + "pass@16": 0.999893854092864, + "pass@32": 0.9999980312535122, + "pass@64": 0.999999999966339, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.987331081081081, + "pass@2": 0.9915873058097466, + "pass@4": 0.9947041267927096, + "pass@8": 0.997053024555423, + "pass@16": 0.9984133373783722, + "pass@32": 0.999305885850076, + "pass@64": 0.9999134567636024, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9947749196141479, + "pass@2": 0.9977312359166519, + "pass@4": 0.9990373822846098, + "pass@8": 0.9997493167484548, + "pass@16": 0.9999841910297439, + "pass@32": 0.999999971428726, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9537202380952381, + "pass@2": 0.9642443132108488, + "pass@4": 0.972526728206593, + "pass@8": 0.9786626463839903, + "pass@16": 0.9826943761181406, + "pass@32": 0.9855854900825515, + "pass@64": 0.9870817812207828, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.9236190749235474, + "pass@2": 0.9405976095017944, + "pass@4": 0.9509045658283538, + "pass@8": 0.9587264846479097, + "pass@16": 0.9659104577653655, + "pass@32": 0.9739652412486856, + "pass@64": 0.9825721937453857, + "pass@128": 0.9877675840978594 + }, + "13": { + "pass@1": 0.8131028163580247, + "pass@2": 0.8544848777583361, + "pass@4": 0.877677726337449, + "pass@8": 0.8922668036744839, + "pass@16": 0.9047770968835759, + "pass@32": 0.9171038736559595, + "pass@64": 0.9280859521752334, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.8519566023738873, + "pass@2": 0.8872827052033923, + "pass@4": 0.9073185410429038, + "pass@8": 0.9216400776558572, + "pass@16": 0.9338163322916623, + "pass@32": 0.9445172996144986, + "pass@64": 0.9538269155955367, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.8259789737654321, + "pass@2": 0.8686012230242051, + "pass@4": 0.8924451495067742, + "pass@8": 0.9063586771341635, + "pass@16": 0.9160849619581726, + "pass@32": 0.9234598698400384, + "pass@64": 0.9293195664411008, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.7139795353982301, + "pass@2": 0.7931833292685756, + "pass@4": 0.8433599964827404, + "pass@8": 0.8761012011420178, + "pass@16": 0.8988421475917746, + "pass@32": 0.9149023008020526, + "pass@64": 0.9277295236545959, + "pass@128": 0.9410029498525073 + }, + "17": { + "pass@1": 0.6482727786144579, + "pass@2": 0.7243408921117538, + "pass@4": 0.7755728016113949, + "pass@8": 0.8113793926723576, + "pass@16": 0.8388991160678905, + "pass@32": 0.8618016432288235, + "pass@64": 0.880137216239385, + "pass@128": 0.891566265060241 + }, + "18": { + "pass@1": 0.6368699596774193, + "pass@2": 0.7169791402082805, + "pass@4": 0.7700088709677413, + "pass@8": 0.8063951046233131, + "pass@16": 0.8307047111453599, + "pass@32": 0.848121899622024, + "pass@64": 0.8617897453414103, + "pass@128": 0.8741935483870967 + }, + "19": { + "pass@1": 0.47365936555891236, + "pass@2": 0.5826961218688298, + "pass@4": 0.6706477077449906, + "pass@8": 0.7371792407322068, + "pass@16": 0.7865671010611726, + "pass@32": 0.8245143993387958, + "pass@64": 0.8530351330994856, + "pass@128": 0.8731117824773413 + }, + "20": { + "pass@1": 0.480762012012012, + "pass@2": 0.5811989400818144, + "pass@4": 0.659820350721925, + "pass@8": 0.7209768024610734, + "pass@16": 0.7675705711799523, + "pass@32": 0.8036268873233177, + "pass@64": 0.8330538100617901, + "pass@128": 0.8588588588588588 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_metrics.json b/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..6840f35ba5c3e88d13d1091ad7367a70a3012207 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch4-op7-10", + "id": { + "answer_accuracy": 0.9950512152777777, + "avg_response_len": 200.65094791666667, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9950512152777777, + "pass@2": 0.9968309820647415, + "pass@4": 0.9979443340311631, + "pass@8": 0.9986641931535145, + "pass@16": 0.99913249866909, + "pass@32": 0.9994619346745741, + "pass@64": 0.9996774367283783, + "pass@128": 0.9997777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9667265625, + "pass@2": 0.9772405265748035, + "pass@4": 0.9843605209036371, + "pass@8": 0.9892329969201861, + "pass@16": 0.9926182699254182, + "pass@32": 0.9952521952142179, + "pass@64": 0.9971000265839046, + "pass@128": 0.998 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999765625, + "pass@2": 0.9999996309055118, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99753125, + "pass@2": 0.9989874507874016, + "pass@4": 0.9995837750281215, + "pass@8": 0.9998582373960525, + "pass@16": 0.9999828674494142, + "pass@32": 0.9999998583460825, + "pass@64": 0.9999999999997746, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996484375, + "pass@2": 0.999943282480315, + "pass@4": 0.9999983755155605, + "pass@8": 0.9999999988902694, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99934375, + "pass@2": 0.9998432578740157, + "pass@4": 0.9999854854705662, + "pass@8": 0.9999998491419404, + "pass@16": 0.9999999999930727, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9958046875, + "pass@2": 0.9976202017716537, + "pass@4": 0.9989555868016498, + "pass@8": 0.9997269929148647, + "pass@16": 0.9999784812111003, + "pass@32": 0.9999999124700476, + "pass@64": 0.999999999999969, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9964296875, + "pass@2": 0.9978444881889764, + "pass@4": 0.9986152625609299, + "pass@8": 0.9991596631183184, + "pass@16": 0.9996128694428026, + "pass@32": 0.9999054460408194, + "pass@64": 0.9999969039717564, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9667265625, + "2": 1.0, + "3": 1.0, + "4": 0.9999765625, + "5": 0.99753125, + "6": 0.9996484375, + "7": 0.99934375, + "8": 0.9958046875, + "9": 0.9964296875 + }, + "per_op_avg_response_len": { + "10": 282.240984375, + "2": 115.8608828125, + "3": 145.0865625, + "4": 160.2766953125, + "5": 173.3148671875, + "6": 199.667, + "7": 224.6848046875, + "8": 244.3722890625, + "9": 260.3544453125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 282.240984375, + "2": 115.8608828125, + "3": 145.0865625, + "4": 160.2766953125, + "5": 173.3148671875, + "6": 199.667, + "7": 224.6848046875, + "8": 244.3722890625, + "9": 260.3544453125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 377405, + "answer_accuracy": 0.9957705378250591, + "avg_response_len": 191.19166086203984, + "resp_tokens_sum": 72463169, + "pass_at_k": { + "pass@1": 0.9957705378250591, + "pass@2": 0.9972615208338319, + "pass@4": 0.9982072057038969, + "pass@8": 0.9988734692572885, + "pass@16": 0.9993439092985485, + "pass@32": 0.9996967349215862, + "pass@64": 0.9999100310573764, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9712389380530974, + "pass@2": 0.9810781914152326, + "pass@4": 0.9877872917987017, + "pass@8": 0.9923114067410685, + "pass@16": 0.9953009685125475, + "pass@32": 0.9976251357688136, + "pass@64": 0.9992232974200413, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999267578125, + "pass@2": 0.9999988465797245, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9967261904761905, + "pass@2": 0.9979080739907511, + "pass@4": 0.998781230917564, + "pass@8": 0.9995512519293239, + "pass@16": 0.9999456111182645, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9986362795857988, + "pass@2": 0.9996065176117038, + "pass@4": 0.9999584012797217, + "pass@8": 0.9999995539350579, + "pass@16": 0.9999999999795047, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9996494391025641, + "pass@2": 0.9999948736876642, + "pass@4": 0.9999999996995569, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9967649647887324, + "pass@2": 0.997464164910724, + "pass@4": 0.99782990673701, + "pass@8": 0.9983444563318933, + "pass@16": 0.9990631538364027, + "pass@32": 0.9997387400876238, + "pass@64": 0.9999912792555583, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 388710, + "answer_accuracy": 0.9950186353211009, + "avg_response_len": 198.65210568889253, + "resp_tokens_sum": 77604637, + "pass_at_k": { + "pass@1": 0.9950186353211009, + "pass@2": 0.9969053063822872, + "pass@4": 0.9981152307018307, + "pass@8": 0.9988235206184004, + "pass@16": 0.999204425400664, + "pass@32": 0.999462311011784, + "pass@64": 0.9996276085051239, + "pass@128": 0.9996723460026212 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9639810199386503, + "pass@2": 0.9763013411187863, + "pass@4": 0.9849948809121252, + "pass@8": 0.9900287848894771, + "pass@16": 0.9927408931048474, + "pass@32": 0.9949717791311338, + "pass@64": 0.9965136850356024, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995018115942029, + "pass@2": 0.9999390191715166, + "pass@4": 0.9999989468707715, + "pass@8": 0.9999999998467666, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993955945558739, + "pass@2": 0.9998854291225775, + "pass@4": 0.9999959845635341, + "pass@8": 0.9999999968689766, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995484104046243, + "pass@2": 0.9999324393518729, + "pass@4": 0.9999986873931221, + "pass@8": 0.999999999745349, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954225352112676, + "pass@2": 0.9976322501940778, + "pass@4": 0.9990125788853857, + "pass@8": 0.9997359054080507, + "pass@16": 0.9999800133300545, + "pass@32": 0.9999999433844207, + "pass@64": 0.9999999999999964, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9961171407185628, + "pass@2": 0.9976218822198123, + "pass@4": 0.9984793601772832, + "pass@8": 0.9992627421174618, + "pass@16": 0.999836737839751, + "pass@32": 0.9999945907477397, + "pass@64": 0.9999999995090815, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380184, + "answer_accuracy": 0.994371442919317, + "avg_response_len": 212.07023665048544, + "resp_tokens_sum": 81082086, + "pass_at_k": { + "pass@1": 0.994371442919317, + "pass@2": 0.9963282491847879, + "pass@4": 0.9975091349478536, + "pass@8": 0.9982939440855182, + "pass@16": 0.9988494363126815, + "pass@32": 0.9992287936927978, + "pass@64": 0.9994977795235602, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9648320895522388, + "pass@2": 0.9742709924785523, + "pass@4": 0.9802755158403708, + "pass@8": 0.9853434213761023, + "pass@16": 0.9897842102909985, + "pass@32": 0.9931238035577333, + "pass@64": 0.9955219923489672, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962775735294118, + "pass@2": 0.9990218996062991, + "pass@4": 0.9999060312313901, + "pass@8": 0.9999988032093555, + "pass@16": 0.9999999998445908, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995563880126183, + "pass@2": 0.9999472167714053, + "pass@4": 0.9999992962236186, + "pass@8": 0.999999999946361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998763844936709, + "pass@2": 0.9999988319794678, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9926097972972973, + "pass@2": 0.9953824395261401, + "pass@4": 0.9979162804536821, + "pass@8": 0.9994617011862063, + "pass@16": 0.9999566861229163, + "pass@32": 0.9999997975032378, + "pass@64": 0.9999999999999107, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9963826366559485, + "pass@2": 0.9985176880522572, + "pass@4": 0.9996576828619894, + "pass@8": 0.9999795024220703, + "pass@16": 0.9999999337376294, + "pass@32": 0.9999999999998973, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.5002296875, + "avg_response_len": 344.50245, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5002296875, + "pass@2": 0.5728906865157404, + "pass@4": 0.6352997881983414, + "pass@8": 0.6874998054154, + "pass@16": 0.7300696915573064, + "pass@32": 0.7647618102242172, + "pass@64": 0.7939591669390739, + "pass@128": 0.8201 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9412421875, + "pass@2": 0.953652312992126, + "pass@4": 0.961258159355081, + "pass@8": 0.9671273765338197, + "pass@16": 0.9721198100900965, + "pass@32": 0.976499939558484, + "pass@64": 0.9798367422797393, + "pass@128": 0.982 + }, + "12": { + "pass@1": 0.8730078125, + "pass@2": 0.9054156003937017, + "pass@4": 0.9239353474878146, + "pass@8": 0.936777551487186, + "pass@16": 0.9463443459241828, + "pass@32": 0.9542127970258184, + "pass@64": 0.9614905316338895, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.7339765625, + "pass@2": 0.7970696358267721, + "pass@4": 0.8382950806149229, + "pass@8": 0.8688697091343058, + "pass@16": 0.892709005541433, + "pass@32": 0.9105660402406084, + "pass@64": 0.923394088323559, + "pass@128": 0.935 + }, + "14": { + "pass@1": 0.677140625, + "pass@2": 0.7601444389763784, + "pass@4": 0.8148952628421446, + "pass@8": 0.8502713306521064, + "pass@16": 0.8758396736542128, + "pass@32": 0.8970577055422668, + "pass@64": 0.9145199374295602, + "pass@128": 0.928 + }, + "15": { + "pass@1": 0.5425546875, + "pass@2": 0.6452209645669293, + "pass@4": 0.7236853342707167, + "pass@8": 0.7831448421952832, + "pass@16": 0.8276656391456853, + "pass@32": 0.8600596388464278, + "pass@64": 0.8842948168837554, + "pass@128": 0.904 + }, + "16": { + "pass@1": 0.3742421875, + "pass@2": 0.482216658464567, + "pass@4": 0.5768006977877758, + "pass@8": 0.65205259160322, + "pass@16": 0.7090840741147838, + "pass@32": 0.754135531429707, + "pass@64": 0.7930790544821106, + "pass@128": 0.829 + }, + "17": { + "pass@1": 0.283015625, + "pass@2": 0.38146296751968556, + "pass@4": 0.4753989731908505, + "pass@8": 0.556040102428998, + "pass@16": 0.6219512049905577, + "pass@32": 0.6764085805789557, + "pass@64": 0.7233771012589468, + "pass@128": 0.766 + }, + "18": { + "pass@1": 0.2356953125, + "pass@2": 0.31902374507874, + "pass@4": 0.4022334073865767, + "pass@8": 0.4796290143755297, + "pass@16": 0.5462268840032527, + "pass@32": 0.600682493693144, + "pass@64": 0.6471233077378016, + "pass@128": 0.692 + }, + "19": { + "pass@1": 0.178984375, + "pass@2": 0.2540275590551181, + "pass@4": 0.33239348884514397, + "pass@8": 0.4048698789485252, + "pass@16": 0.46788108259685424, + "pass@32": 0.5236021150346005, + "pass@64": 0.5747571384612385, + "pass@128": 0.624 + }, + "20": { + "pass@1": 0.1624375, + "pass@2": 0.23067298228346464, + "pass@4": 0.30410213020247473, + "pass@8": 0.37621565679502567, + "pass@16": 0.4408751955118752, + "pass@32": 0.4943932602921467, + "pass@64": 0.537718950900034, + "pass@128": 0.573 + } + }, + "per_op_accuracy": { + "11": 0.9412421875, + "12": 0.8730078125, + "13": 0.7339765625, + "14": 0.677140625, + "15": 0.5425546875, + "16": 0.3742421875, + "17": 0.283015625, + "18": 0.2356953125, + "19": 0.178984375, + "20": 0.1624375 + }, + "per_op_avg_response_len": { + "11": 299.0276796875, + "12": 308.9127109375, + "13": 325.2062734375, + "14": 347.435953125, + "15": 355.9632578125, + "16": 358.99171875, + "17": 365.43715625, + "18": 362.6295546875, + "19": 361.2436640625, + "20": 360.17653125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 299.0276796875, + "12": 308.9127109375, + "13": 325.2062734375, + "14": 347.435953125, + "15": 355.9632578125, + "16": 358.99171875, + "17": 365.43715625, + "18": 362.6295546875, + "19": 361.2436640625, + "20": 360.17653125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 215464, + "answer_accuracy": 0.500241456166419, + "avg_response_len": 328.1673616270431, + "resp_tokens_sum": 141348246, + "pass_at_k": { + "pass@1": 0.500241456166419, + "pass@2": 0.5732799355921903, + "pass@4": 0.6357232889908101, + "pass@8": 0.6875347017891723, + "pass@16": 0.729570207801191, + "pass@32": 0.7639155179011134, + "pass@64": 0.7932490218591338, + "pass@128": 0.8199108469539376 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9256595254957507, + "pass@2": 0.9379046446655216, + "pass@4": 0.9464527073103111, + "pass@8": 0.9544125837859465, + "pass@16": 0.9620550938845545, + "pass@32": 0.9689466009864247, + "pass@64": 0.9733538467202176, + "pass@128": 0.9745042492917847 + }, + "12": { + "pass@1": 0.8546963276836158, + "pass@2": 0.8892605320521376, + "pass@4": 0.9096035241357541, + "pass@8": 0.9232561725273154, + "pass@16": 0.9335660780959657, + "pass@32": 0.9430045267513955, + "pass@64": 0.9524606602559231, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.7480407523510971, + "pass@2": 0.8101477457853037, + "pass@4": 0.8495423074074989, + "pass@8": 0.8771021172653639, + "pass@16": 0.896003390136595, + "pass@32": 0.9090301477413725, + "pass@64": 0.9191436054892381, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.682583722741433, + "pass@2": 0.7649933616650716, + "pass@4": 0.818672544436618, + "pass@8": 0.852279360455284, + "pass@16": 0.8766875266221614, + "pass@32": 0.8978875992415474, + "pass@64": 0.9145590851330114, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5563583815028902, + "pass@2": 0.6612466433025347, + "pass@4": 0.7391046363062996, + "pass@8": 0.7995924425216887, + "pass@16": 0.8472066555948774, + "pass@32": 0.8814056698555999, + "pass@64": 0.9055910973040048, + "pass@128": 0.9248554913294798 + }, + "16": { + "pass@1": 0.35776209677419357, + "pass@2": 0.4704236252222501, + "pass@4": 0.5702631176143302, + "pass@8": 0.6486250520290218, + "pass@16": 0.7058972081786804, + "pass@32": 0.7489029331029314, + "pass@64": 0.7864609089054653, + "pass@128": 0.8225806451612904 + }, + "17": { + "pass@1": 0.2822209176300578, + "pass@2": 0.38103423262482355, + "pass@4": 0.4747528679579794, + "pass@8": 0.5555152783820172, + "pass@16": 0.6224005820967119, + "pass@32": 0.6793292843272832, + "pass@64": 0.727092303936004, + "pass@128": 0.7658959537572254 + }, + "18": { + "pass@1": 0.21580357142857143, + "pass@2": 0.29927622328458936, + "pass@4": 0.3840758931919225, + "pass@8": 0.46175944378352485, + "pass@16": 0.5272364483452201, + "pass@32": 0.5804867351029614, + "pass@64": 0.6281698996492903, + "pass@128": 0.68 + }, + "19": { + "pass@1": 0.18041478978978978, + "pass@2": 0.25600452815019736, + "pass@4": 0.3330379324769089, + "pass@8": 0.40319579360003893, + "pass@16": 0.46454362983819264, + "pass@32": 0.5196802274780482, + "pass@64": 0.5703849188444975, + "pass@128": 0.6186186186186187 + }, + "20": { + "pass@1": 0.17896021021021022, + "pass@2": 0.24850367099382842, + "pass@4": 0.32194917217555025, + "pass@8": 0.3922391954982608, + "pass@16": 0.45386218477737117, + "pass@32": 0.5048449014642736, + "pass@64": 0.55023639343001, + "pass@128": 0.5945945945945946 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 217083, + "answer_accuracy": 0.5043000111507583, + "avg_response_len": 340.7973303226286, + "resp_tokens_sum": 146700982, + "pass_at_k": { + "pass@1": 0.5043000111507583, + "pass@2": 0.5766524999356145, + "pass@4": 0.6387109358598265, + "pass@8": 0.6908086380944928, + "pass@16": 0.7329800101268554, + "pass@32": 0.7668444456595922, + "pass@64": 0.7957990950959042, + "pass@128": 0.8230746357418971 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9549839984939759, + "pass@2": 0.9656675422161087, + "pass@4": 0.9699165173065714, + "pass@8": 0.9719722005758548, + "pass@16": 0.9736488700375456, + "pass@32": 0.9754268332917806, + "pass@64": 0.9774205766085304, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8691222570532915, + "pass@2": 0.8991303717325303, + "pass@4": 0.9174640042055867, + "pass@8": 0.9318582991709181, + "pass@16": 0.9426732941451575, + "pass@32": 0.949884708573006, + "pass@64": 0.9548453788380237, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7502626050420168, + "pass@2": 0.8138757471492536, + "pass@4": 0.8540254450861707, + "pass@8": 0.883486764099256, + "pass@16": 0.9069851064183747, + "pass@32": 0.9246218794102262, + "pass@64": 0.9372843805980681, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.6877741228070176, + "pass@2": 0.7709750713726572, + "pass@4": 0.8255886059746912, + "pass@8": 0.8616636529389289, + "pass@16": 0.8884648994522174, + "pass@32": 0.9102203876391841, + "pass@64": 0.9280497250391677, + "pass@128": 0.9415204678362573 + }, + "15": { + "pass@1": 0.5294981060606061, + "pass@2": 0.6371901843235506, + "pass@4": 0.7218647379304856, + "pass@8": 0.784792093362903, + "pass@16": 0.8296342586994554, + "pass@32": 0.8615023805667635, + "pass@64": 0.8850014181350153, + "pass@128": 0.9030303030303031 + }, + "16": { + "pass@1": 0.38826566951566954, + "pass@2": 0.49237134620992856, + "pass@4": 0.5828217968480429, + "pass@8": 0.6537934355879218, + "pass@16": 0.7056221038183779, + "pass@32": 0.7460727057171335, + "pass@64": 0.784026806543782, + "pass@128": 0.8233618233618234 + }, + "17": { + "pass@1": 0.3012179736024845, + "pass@2": 0.40523403494400156, + "pass@4": 0.5060908428760066, + "pass@8": 0.5919590185783913, + "pass@16": 0.6587760745115138, + "pass@32": 0.7099266521318334, + "pass@64": 0.7537711568802992, + "pass@128": 0.7950310559006211 + }, + "18": { + "pass@1": 0.24391084558823528, + "pass@2": 0.32872459761463624, + "pass@4": 0.4134082233287015, + "pass@8": 0.4939817269103554, + "pass@16": 0.5646029865362203, + "pass@32": 0.6231760486863247, + "pass@64": 0.672253518753805, + "pass@128": 0.7176470588235294 + }, + "19": { + "pass@1": 0.17578125, + "pass@2": 0.24372283054461924, + "pass@4": 0.3149261088457693, + "pass@8": 0.382738789933009, + "pass@16": 0.44227697220974477, + "pass@32": 0.49650961347677647, + "pass@64": 0.5534303542394388, + "pass@128": 0.6190476190476191 + }, + "20": { + "pass@1": 0.14549026946107785, + "pass@2": 0.21075847812249524, + "pass@4": 0.28126393697419555, + "pass@8": 0.35182762438965187, + "pass@16": 0.41691692179414386, + "pass@32": 0.47061910952622776, + "pass@64": 0.511055493082766, + "pass@128": 0.5419161676646707 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 207747, + "answer_accuracy": 0.4960340579156479, + "avg_response_len": 365.10999579767724, + "resp_tokens_sum": 152913908, + "pass_at_k": { + "pass@1": 0.4960340579156479, + "pass@2": 0.5686239378381132, + "pass@4": 0.6313582326506261, + "pass@8": 0.684063060122764, + "pass@16": 0.7275921125505347, + "pass@32": 0.7634915995573603, + "pass@64": 0.7927983967075446, + "pass@128": 0.8172371638141809 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9442212301587302, + "pass@2": 0.9586360298712663, + "pass@4": 0.9687240315198693, + "pass@8": 0.9762697265593556, + "pass@16": 0.9817871018932806, + "pass@32": 0.9860954944679531, + "pass@64": 0.9896482633443495, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.896621750764526, + "pass@2": 0.9290360350598379, + "pass@4": 0.9457635553583317, + "pass@8": 0.956214278217107, + "pass@16": 0.9637589402015467, + "pass@32": 0.9705687233059196, + "pass@64": 0.977748568972364, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7021846064814815, + "pass@2": 0.7656755036696803, + "pass@4": 0.8098888909140981, + "pass@8": 0.8446585152568515, + "pass@16": 0.8737353120262339, + "pass@32": 0.8965907782767284, + "pass@64": 0.9122739330215466, + "pass@128": 0.9259259259259259 + }, + "14": { + "pass@1": 0.6611646884272997, + "pass@2": 0.7445344079300921, + "pass@4": 0.8004453199841117, + "pass@8": 0.8367972897354505, + "pass@16": 0.86221953114493, + "pass@32": 0.882909239208098, + "pass@64": 0.9007521220726056, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.5411120756172839, + "pass@2": 0.6362866208564207, + "pass@4": 0.709073353678012, + "pass@8": 0.7639026736821625, + "pass@16": 0.80479268808345, + "pass@32": 0.8357947268190055, + "pass@64": 0.8608328062716483, + "pass@128": 0.8827160493827161 + }, + "16": { + "pass@1": 0.37479258849557523, + "pass@2": 0.4824867024365314, + "pass@4": 0.5765447806302972, + "pass@8": 0.6533844530467929, + "pass@16": 0.7155828352187087, + "pass@32": 0.7672687388232577, + "pass@64": 0.8085037275060418, + "pass@128": 0.8407079646017699 + }, + "17": { + "pass@1": 0.26618975903614456, + "pass@2": 0.3588547101792997, + "pass@4": 0.4463049080461327, + "pass@8": 0.5217500365258368, + "pass@16": 0.5857671915433449, + "pass@32": 0.6408562235399593, + "pass@64": 0.6900266613904555, + "pass@128": 0.7379518072289156 + }, + "18": { + "pass@1": 0.24914314516129032, + "pass@2": 0.33067968948437876, + "pass@4": 0.41047757689562997, + "pass@8": 0.4840626512960474, + "pass@16": 0.5475132634197127, + "pass@32": 0.5988138059798621, + "pass@64": 0.6409601499492135, + "pass@128": 0.6774193548387096 + }, + "19": { + "pass@1": 0.18079682779456194, + "pass@2": 0.2624990335894569, + "pass@4": 0.34947638899744776, + "pass@8": 0.4290194751124504, + "pass@16": 0.49722958065940137, + "pass@32": 0.555049453644149, + "pass@64": 0.6008047174065532, + "pass@128": 0.6344410876132931 + }, + "20": { + "pass@1": 0.1629129129129129, + "pass@2": 0.2328166010498687, + "pass@4": 0.309161864620526, + "pass@8": 0.3846533878317151, + "pass@16": 0.451918426732032, + "pass@32": 0.5077871637320823, + "pass@64": 0.5519450366312206, + "pass@128": 0.5825825825825826 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7346188322368421, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7346188322368421, + "pass@2": 0.773704510723148, + "pass@4": 0.8070787835928236, + "pass@8": 0.8348934627650431, + "pass@16": 0.8575204949260501, + "pass@32": 0.8759355533849245, + "pass@64": 0.8914046631550582, + "pass@128": 0.9052105263157895 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9667265625, + "pass@2": 0.9772405265748035, + "pass@4": 0.9843605209036371, + "pass@8": 0.9892329969201861, + "pass@16": 0.9926182699254182, + "pass@32": 0.9952521952142179, + "pass@64": 0.9971000265839046, + "pass@128": 0.998 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999765625, + "pass@2": 0.9999996309055118, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99753125, + "pass@2": 0.9989874507874016, + "pass@4": 0.9995837750281215, + "pass@8": 0.9998582373960525, + "pass@16": 0.9999828674494142, + "pass@32": 0.9999998583460825, + "pass@64": 0.9999999999997746, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996484375, + "pass@2": 0.999943282480315, + "pass@4": 0.9999983755155605, + "pass@8": 0.9999999988902694, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99934375, + "pass@2": 0.9998432578740157, + "pass@4": 0.9999854854705662, + "pass@8": 0.9999998491419404, + "pass@16": 0.9999999999930727, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9958046875, + "pass@2": 0.9976202017716537, + "pass@4": 0.9989555868016498, + "pass@8": 0.9997269929148647, + "pass@16": 0.9999784812111003, + "pass@32": 0.9999999124700476, + "pass@64": 0.999999999999969, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9964296875, + "pass@2": 0.9978444881889764, + "pass@4": 0.9986152625609299, + "pass@8": 0.9991596631183184, + "pass@16": 0.9996128694428026, + "pass@32": 0.9999054460408194, + "pass@64": 0.9999969039717564, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9412421875, + "pass@2": 0.953652312992126, + "pass@4": 0.961258159355081, + "pass@8": 0.9671273765338197, + "pass@16": 0.9721198100900965, + "pass@32": 0.976499939558484, + "pass@64": 0.9798367422797393, + "pass@128": 0.982 + }, + "12": { + "pass@1": 0.8730078125, + "pass@2": 0.9054156003937017, + "pass@4": 0.9239353474878146, + "pass@8": 0.936777551487186, + "pass@16": 0.9463443459241828, + "pass@32": 0.9542127970258184, + "pass@64": 0.9614905316338895, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.7339765625, + "pass@2": 0.7970696358267721, + "pass@4": 0.8382950806149229, + "pass@8": 0.8688697091343058, + "pass@16": 0.892709005541433, + "pass@32": 0.9105660402406084, + "pass@64": 0.923394088323559, + "pass@128": 0.935 + }, + "14": { + "pass@1": 0.677140625, + "pass@2": 0.7601444389763784, + "pass@4": 0.8148952628421446, + "pass@8": 0.8502713306521064, + "pass@16": 0.8758396736542128, + "pass@32": 0.8970577055422668, + "pass@64": 0.9145199374295602, + "pass@128": 0.928 + }, + "15": { + "pass@1": 0.5425546875, + "pass@2": 0.6452209645669293, + "pass@4": 0.7236853342707167, + "pass@8": 0.7831448421952832, + "pass@16": 0.8276656391456853, + "pass@32": 0.8600596388464278, + "pass@64": 0.8842948168837554, + "pass@128": 0.904 + }, + "16": { + "pass@1": 0.3742421875, + "pass@2": 0.482216658464567, + "pass@4": 0.5768006977877758, + "pass@8": 0.65205259160322, + "pass@16": 0.7090840741147838, + "pass@32": 0.754135531429707, + "pass@64": 0.7930790544821106, + "pass@128": 0.829 + }, + "17": { + "pass@1": 0.283015625, + "pass@2": 0.38146296751968556, + "pass@4": 0.4753989731908505, + "pass@8": 0.556040102428998, + "pass@16": 0.6219512049905577, + "pass@32": 0.6764085805789557, + "pass@64": 0.7233771012589468, + "pass@128": 0.766 + }, + "18": { + "pass@1": 0.2356953125, + "pass@2": 0.31902374507874, + "pass@4": 0.4022334073865767, + "pass@8": 0.4796290143755297, + "pass@16": 0.5462268840032527, + "pass@32": 0.600682493693144, + "pass@64": 0.6471233077378016, + "pass@128": 0.692 + }, + "19": { + "pass@1": 0.178984375, + "pass@2": 0.2540275590551181, + "pass@4": 0.33239348884514397, + "pass@8": 0.4048698789485252, + "pass@16": 0.46788108259685424, + "pass@32": 0.5236021150346005, + "pass@64": 0.5747571384612385, + "pass@128": 0.624 + }, + "20": { + "pass@1": 0.1624375, + "pass@2": 0.23067298228346464, + "pass@4": 0.30410213020247473, + "pass@8": 0.37621565679502567, + "pass@16": 0.4408751955118752, + "pass@32": 0.4943932602921467, + "pass@64": 0.537718950900034, + "pass@128": 0.573 + } + }, + "per_op_accuracy": { + "10": 0.9667265625, + "2": 1.0, + "3": 1.0, + "4": 0.9999765625, + "5": 0.99753125, + "6": 0.9996484375, + "7": 0.99934375, + "8": 0.9958046875, + "9": 0.9964296875, + "11": 0.9412421875, + "12": 0.8730078125, + "13": 0.7339765625, + "14": 0.677140625, + "15": 0.5425546875, + "16": 0.3742421875, + "17": 0.283015625, + "18": 0.2356953125, + "19": 0.178984375, + "20": 0.1624375 + }, + "per_op_avg_response_len": { + "10": 282.240984375, + "2": 115.8608828125, + "3": 145.0865625, + "4": 160.2766953125, + "5": 173.3148671875, + "6": 199.667, + "7": 224.6848046875, + "8": 244.3722890625, + "9": 260.3544453125, + "11": 299.0276796875, + "12": 308.9127109375, + "13": 325.2062734375, + "14": 347.435953125, + "15": 355.9632578125, + "16": 358.99171875, + "17": 365.43715625, + "18": 362.6295546875, + "19": 361.2436640625, + "20": 360.17653125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 282.240984375, + "2": 115.8608828125, + "3": 145.0865625, + "4": 160.2766953125, + "5": 173.3148671875, + "6": 199.667, + "7": 224.6848046875, + "8": 244.3722890625, + "9": 260.3544453125, + "11": 299.0276796875, + "12": 308.9127109375, + "13": 325.2062734375, + "14": 347.435953125, + "15": 355.9632578125, + "16": 358.99171875, + "17": 365.43715625, + "18": 362.6295546875, + "19": 361.2436640625, + "20": 360.17653125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 592869, + "answer_accuracy": 0.7321829058646854, + "avg_response_len": 264.0533796534145, + "resp_tokens_sum": 213811415, + "pass_at_k": { + "pass@1": 0.7321829058646854, + "pass@2": 0.771732271017493, + "pass@4": 0.805390515893657, + "pass@8": 0.8332625061636697, + "pass@16": 0.8558427228397145, + "pass@32": 0.8742772288713363, + "pass@64": 0.8899804869612601, + "pass@128": 0.9042048687954474 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9712389380530974, + "pass@2": 0.9810781914152326, + "pass@4": 0.9877872917987017, + "pass@8": 0.9923114067410685, + "pass@16": 0.9953009685125475, + "pass@32": 0.9976251357688136, + "pass@64": 0.9992232974200413, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999267578125, + "pass@2": 0.9999988465797245, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9967261904761905, + "pass@2": 0.9979080739907511, + "pass@4": 0.998781230917564, + "pass@8": 0.9995512519293239, + "pass@16": 0.9999456111182645, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9986362795857988, + "pass@2": 0.9996065176117038, + "pass@4": 0.9999584012797217, + "pass@8": 0.9999995539350579, + "pass@16": 0.9999999999795047, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9996494391025641, + "pass@2": 0.9999948736876642, + "pass@4": 0.9999999996995569, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9967649647887324, + "pass@2": 0.997464164910724, + "pass@4": 0.99782990673701, + "pass@8": 0.9983444563318933, + "pass@16": 0.9990631538364027, + "pass@32": 0.9997387400876238, + "pass@64": 0.9999912792555583, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9256595254957507, + "pass@2": 0.9379046446655216, + "pass@4": 0.9464527073103111, + "pass@8": 0.9544125837859465, + "pass@16": 0.9620550938845545, + "pass@32": 0.9689466009864247, + "pass@64": 0.9733538467202176, + "pass@128": 0.9745042492917847 + }, + "12": { + "pass@1": 0.8546963276836158, + "pass@2": 0.8892605320521376, + "pass@4": 0.9096035241357541, + "pass@8": 0.9232561725273154, + "pass@16": 0.9335660780959657, + "pass@32": 0.9430045267513955, + "pass@64": 0.9524606602559231, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.7480407523510971, + "pass@2": 0.8101477457853037, + "pass@4": 0.8495423074074989, + "pass@8": 0.8771021172653639, + "pass@16": 0.896003390136595, + "pass@32": 0.9090301477413725, + "pass@64": 0.9191436054892381, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.682583722741433, + "pass@2": 0.7649933616650716, + "pass@4": 0.818672544436618, + "pass@8": 0.852279360455284, + "pass@16": 0.8766875266221614, + "pass@32": 0.8978875992415474, + "pass@64": 0.9145590851330114, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5563583815028902, + "pass@2": 0.6612466433025347, + "pass@4": 0.7391046363062996, + "pass@8": 0.7995924425216887, + "pass@16": 0.8472066555948774, + "pass@32": 0.8814056698555999, + "pass@64": 0.9055910973040048, + "pass@128": 0.9248554913294798 + }, + "16": { + "pass@1": 0.35776209677419357, + "pass@2": 0.4704236252222501, + "pass@4": 0.5702631176143302, + "pass@8": 0.6486250520290218, + "pass@16": 0.7058972081786804, + "pass@32": 0.7489029331029314, + "pass@64": 0.7864609089054653, + "pass@128": 0.8225806451612904 + }, + "17": { + "pass@1": 0.2822209176300578, + "pass@2": 0.38103423262482355, + "pass@4": 0.4747528679579794, + "pass@8": 0.5555152783820172, + "pass@16": 0.6224005820967119, + "pass@32": 0.6793292843272832, + "pass@64": 0.727092303936004, + "pass@128": 0.7658959537572254 + }, + "18": { + "pass@1": 0.21580357142857143, + "pass@2": 0.29927622328458936, + "pass@4": 0.3840758931919225, + "pass@8": 0.46175944378352485, + "pass@16": 0.5272364483452201, + "pass@32": 0.5804867351029614, + "pass@64": 0.6281698996492903, + "pass@128": 0.68 + }, + "19": { + "pass@1": 0.18041478978978978, + "pass@2": 0.25600452815019736, + "pass@4": 0.3330379324769089, + "pass@8": 0.40319579360003893, + "pass@16": 0.46454362983819264, + "pass@32": 0.5196802274780482, + "pass@64": 0.5703849188444975, + "pass@128": 0.6186186186186187 + }, + "20": { + "pass@1": 0.17896021021021022, + "pass@2": 0.24850367099382842, + "pass@4": 0.32194917217555025, + "pass@8": 0.3922391954982608, + "pass@16": 0.45386218477737117, + "pass@32": 0.5048449014642736, + "pass@64": 0.55023639343001, + "pass@128": 0.5945945945945946 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 605793, + "answer_accuracy": 0.737764273187841, + "avg_response_len": 273.1703271141855, + "resp_tokens_sum": 224305619, + "pass_at_k": { + "pass@1": 0.737764273187841, + "pass@2": 0.7765919489262912, + "pass@4": 0.8097011007636077, + "pass@8": 0.8373497793981471, + "pass@16": 0.8596389213374069, + "pass@32": 0.8775147067749307, + "pass@64": 0.8927725359259874, + "pass@128": 0.9070927513639907 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9639810199386503, + "pass@2": 0.9763013411187863, + "pass@4": 0.9849948809121252, + "pass@8": 0.9900287848894771, + "pass@16": 0.9927408931048474, + "pass@32": 0.9949717791311338, + "pass@64": 0.9965136850356024, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995018115942029, + "pass@2": 0.9999390191715166, + "pass@4": 0.9999989468707715, + "pass@8": 0.9999999998467666, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993955945558739, + "pass@2": 0.9998854291225775, + "pass@4": 0.9999959845635341, + "pass@8": 0.9999999968689766, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995484104046243, + "pass@2": 0.9999324393518729, + "pass@4": 0.9999986873931221, + "pass@8": 0.999999999745349, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954225352112676, + "pass@2": 0.9976322501940778, + "pass@4": 0.9990125788853857, + "pass@8": 0.9997359054080507, + "pass@16": 0.9999800133300545, + "pass@32": 0.9999999433844207, + "pass@64": 0.9999999999999964, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9961171407185628, + "pass@2": 0.9976218822198123, + "pass@4": 0.9984793601772832, + "pass@8": 0.9992627421174618, + "pass@16": 0.999836737839751, + "pass@32": 0.9999945907477397, + "pass@64": 0.9999999995090815, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9549839984939759, + "pass@2": 0.9656675422161087, + "pass@4": 0.9699165173065714, + "pass@8": 0.9719722005758548, + "pass@16": 0.9736488700375456, + "pass@32": 0.9754268332917806, + "pass@64": 0.9774205766085304, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8691222570532915, + "pass@2": 0.8991303717325303, + "pass@4": 0.9174640042055867, + "pass@8": 0.9318582991709181, + "pass@16": 0.9426732941451575, + "pass@32": 0.949884708573006, + "pass@64": 0.9548453788380237, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7502626050420168, + "pass@2": 0.8138757471492536, + "pass@4": 0.8540254450861707, + "pass@8": 0.883486764099256, + "pass@16": 0.9069851064183747, + "pass@32": 0.9246218794102262, + "pass@64": 0.9372843805980681, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.6877741228070176, + "pass@2": 0.7709750713726572, + "pass@4": 0.8255886059746912, + "pass@8": 0.8616636529389289, + "pass@16": 0.8884648994522174, + "pass@32": 0.9102203876391841, + "pass@64": 0.9280497250391677, + "pass@128": 0.9415204678362573 + }, + "15": { + "pass@1": 0.5294981060606061, + "pass@2": 0.6371901843235506, + "pass@4": 0.7218647379304856, + "pass@8": 0.784792093362903, + "pass@16": 0.8296342586994554, + "pass@32": 0.8615023805667635, + "pass@64": 0.8850014181350153, + "pass@128": 0.9030303030303031 + }, + "16": { + "pass@1": 0.38826566951566954, + "pass@2": 0.49237134620992856, + "pass@4": 0.5828217968480429, + "pass@8": 0.6537934355879218, + "pass@16": 0.7056221038183779, + "pass@32": 0.7460727057171335, + "pass@64": 0.784026806543782, + "pass@128": 0.8233618233618234 + }, + "17": { + "pass@1": 0.3012179736024845, + "pass@2": 0.40523403494400156, + "pass@4": 0.5060908428760066, + "pass@8": 0.5919590185783913, + "pass@16": 0.6587760745115138, + "pass@32": 0.7099266521318334, + "pass@64": 0.7537711568802992, + "pass@128": 0.7950310559006211 + }, + "18": { + "pass@1": 0.24391084558823528, + "pass@2": 0.32872459761463624, + "pass@4": 0.4134082233287015, + "pass@8": 0.4939817269103554, + "pass@16": 0.5646029865362203, + "pass@32": 0.6231760486863247, + "pass@64": 0.672253518753805, + "pass@128": 0.7176470588235294 + }, + "19": { + "pass@1": 0.17578125, + "pass@2": 0.24372283054461924, + "pass@4": 0.3149261088457693, + "pass@8": 0.382738789933009, + "pass@16": 0.44227697220974477, + "pass@32": 0.49650961347677647, + "pass@64": 0.5534303542394388, + "pass@128": 0.6190476190476191 + }, + "20": { + "pass@1": 0.14549026946107785, + "pass@2": 0.21075847812249524, + "pass@4": 0.28126393697419555, + "pass@8": 0.35182762438965187, + "pass@16": 0.41691692179414386, + "pass@32": 0.47061910952622776, + "pass@64": 0.511055493082766, + "pass@128": 0.5419161676646707 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 587931, + "answer_accuracy": 0.7338569959258667, + "avg_response_len": 292.07440535628695, + "resp_tokens_sum": 233995994, + "pass_at_k": { + "pass@1": 0.7338569959258667, + "pass@2": 0.7727384574087262, + "pass@4": 0.80609744740726, + "pass@8": 0.8340243399432995, + "pass@16": 0.8570450005642073, + "pass@32": 0.8759931171931754, + "pass@64": 0.8914421187831943, + "pass@128": 0.9042978111519412 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9648320895522388, + "pass@2": 0.9742709924785523, + "pass@4": 0.9802755158403708, + "pass@8": 0.9853434213761023, + "pass@16": 0.9897842102909985, + "pass@32": 0.9931238035577333, + "pass@64": 0.9955219923489672, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962775735294118, + "pass@2": 0.9990218996062991, + "pass@4": 0.9999060312313901, + "pass@8": 0.9999988032093555, + "pass@16": 0.9999999998445908, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995563880126183, + "pass@2": 0.9999472167714053, + "pass@4": 0.9999992962236186, + "pass@8": 0.999999999946361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998763844936709, + "pass@2": 0.9999988319794678, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9926097972972973, + "pass@2": 0.9953824395261401, + "pass@4": 0.9979162804536821, + "pass@8": 0.9994617011862063, + "pass@16": 0.9999566861229163, + "pass@32": 0.9999997975032378, + "pass@64": 0.9999999999999107, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9963826366559485, + "pass@2": 0.9985176880522572, + "pass@4": 0.9996576828619894, + "pass@8": 0.9999795024220703, + "pass@16": 0.9999999337376294, + "pass@32": 0.9999999999998973, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9442212301587302, + "pass@2": 0.9586360298712663, + "pass@4": 0.9687240315198693, + "pass@8": 0.9762697265593556, + "pass@16": 0.9817871018932806, + "pass@32": 0.9860954944679531, + "pass@64": 0.9896482633443495, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.896621750764526, + "pass@2": 0.9290360350598379, + "pass@4": 0.9457635553583317, + "pass@8": 0.956214278217107, + "pass@16": 0.9637589402015467, + "pass@32": 0.9705687233059196, + "pass@64": 0.977748568972364, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7021846064814815, + "pass@2": 0.7656755036696803, + "pass@4": 0.8098888909140981, + "pass@8": 0.8446585152568515, + "pass@16": 0.8737353120262339, + "pass@32": 0.8965907782767284, + "pass@64": 0.9122739330215466, + "pass@128": 0.9259259259259259 + }, + "14": { + "pass@1": 0.6611646884272997, + "pass@2": 0.7445344079300921, + "pass@4": 0.8004453199841117, + "pass@8": 0.8367972897354505, + "pass@16": 0.86221953114493, + "pass@32": 0.882909239208098, + "pass@64": 0.9007521220726056, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.5411120756172839, + "pass@2": 0.6362866208564207, + "pass@4": 0.709073353678012, + "pass@8": 0.7639026736821625, + "pass@16": 0.80479268808345, + "pass@32": 0.8357947268190055, + "pass@64": 0.8608328062716483, + "pass@128": 0.8827160493827161 + }, + "16": { + "pass@1": 0.37479258849557523, + "pass@2": 0.4824867024365314, + "pass@4": 0.5765447806302972, + "pass@8": 0.6533844530467929, + "pass@16": 0.7155828352187087, + "pass@32": 0.7672687388232577, + "pass@64": 0.8085037275060418, + "pass@128": 0.8407079646017699 + }, + "17": { + "pass@1": 0.26618975903614456, + "pass@2": 0.3588547101792997, + "pass@4": 0.4463049080461327, + "pass@8": 0.5217500365258368, + "pass@16": 0.5857671915433449, + "pass@32": 0.6408562235399593, + "pass@64": 0.6900266613904555, + "pass@128": 0.7379518072289156 + }, + "18": { + "pass@1": 0.24914314516129032, + "pass@2": 0.33067968948437876, + "pass@4": 0.41047757689562997, + "pass@8": 0.4840626512960474, + "pass@16": 0.5475132634197127, + "pass@32": 0.5988138059798621, + "pass@64": 0.6409601499492135, + "pass@128": 0.6774193548387096 + }, + "19": { + "pass@1": 0.18079682779456194, + "pass@2": 0.2624990335894569, + "pass@4": 0.34947638899744776, + "pass@8": 0.4290194751124504, + "pass@16": 0.49722958065940137, + "pass@32": 0.555049453644149, + "pass@64": 0.6008047174065532, + "pass@128": 0.6344410876132931 + }, + "20": { + "pass@1": 0.1629129129129129, + "pass@2": 0.2328166010498687, + "pass@4": 0.309161864620526, + "pass@8": 0.3846533878317151, + "pass@16": 0.451918426732032, + "pass@32": 0.5077871637320823, + "pass@64": 0.5519450366312206, + "pass@128": 0.5825825825825826 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch4-op7-10/graphs/SDAR-100M-block4-epoch4-op7-10_score_report.jsonl b/composition/SDAR-100M-block4-epoch4-op7-10/graphs/SDAR-100M-block4-epoch4-op7-10_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9aab5de9756be22b407166783037e107838324ab --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op7-10/graphs/SDAR-100M-block4-epoch4-op7-10_score_report.jsonl @@ -0,0 +1,2 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch4-op7-10", "results_dir": "results/composition/SDAR-100M-block4-epoch4-op7-10/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1023995, "value_mismatch": 6520, "dependency_mismatch": 126470, "answer_mismatch": 5701, "extra_nodes": 8816, "missing_nodes": 8712, "perfect_with_extra": 220, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337509, "value_mismatch": 1965, "dependency_mismatch": 41067, "answer_mismatch": 1603, "extra_nodes": 2801, "missing_nodes": 2686, "perfect_with_extra": 25}, "teachers_in_school": {"total": 390656, "perfect": 346854, "value_mismatch": 2069, "dependency_mismatch": 43298, "answer_mismatch": 1946, "extra_nodes": 3264, "perfect_with_extra": 165, "missing_nodes": 3312}, "movie_festival_awards": {"total": 382336, "perfect": 339632, "dependency_mismatch": 42105, "value_mismatch": 2486, "answer_mismatch": 2152, "missing_nodes": 2714, "extra_nodes": 2751, "perfect_with_extra": 30}}, "results_path": "results/composition/SDAR-100M-block4-epoch4-op7-10/graphs/SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 305126, "value_mismatch": 762183, "dependency_mismatch": 933013, "answer_mismatch": 639705, "extra_nodes": 72816, "missing_nodes": 574062, "perfect_with_extra": 64, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 257598, "dependency_mismatch": 314982, "answer_mismatch": 215255, "perfect": 102037, "extra_nodes": 28149, "missing_nodes": 196542, "perfect_with_extra": 35}, "movie_festival_awards": {"total": 418816, "perfect": 99135, "value_mismatch": 250453, "dependency_mismatch": 306840, "answer_mismatch": 211069, "missing_nodes": 188877, "extra_nodes": 23013, "perfect_with_extra": 17}, "teachers_in_school": {"total": 430464, "perfect": 103954, "value_mismatch": 254132, "answer_mismatch": 213381, "dependency_mismatch": 311191, "missing_nodes": 188643, "extra_nodes": 21654, "perfect_with_extra": 12}}, "results_path": "results/composition/SDAR-100M-block4-epoch4-op7-10/graphs/SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1329121, "value_mismatch": 768703, "dependency_mismatch": 1059483, "answer_mismatch": 645406, "extra_nodes": 81632, "missing_nodes": 582774, "perfect_with_extra": 284, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch4-op7-10/graphs/SDAR-100M-block4-epoch4-op7-10_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 439546, "value_mismatch": 259563, "dependency_mismatch": 356049, "answer_mismatch": 216858, "extra_nodes": 30950, "missing_nodes": 199228, "perfect_with_extra": 60}, "teachers_in_school": {"total": 821120, "perfect": 450808, "value_mismatch": 256201, "dependency_mismatch": 354489, "answer_mismatch": 215327, "extra_nodes": 24918, "perfect_with_extra": 177, "missing_nodes": 191955}, "movie_festival_awards": {"total": 801152, "perfect": 438767, "dependency_mismatch": 348945, "value_mismatch": 252939, "answer_mismatch": 213221, "missing_nodes": 191591, "extra_nodes": 25764, "perfect_with_extra": 47}}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch4-op7-10", "results_dir": "results/composition/SDAR-100M-block4-epoch4-op7-10/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch4-op7-10/graphs/SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch4-op7-10/graphs/SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch4-op7-10/graphs/SDAR-100M-block4-epoch4-op7-10_score_report.jsonl", "template_stats": {}}} diff --git a/composition/SDAR-100M-block4-epoch4-op7-10/graphs/results.json b/composition/SDAR-100M-block4-epoch4-op7-10/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..95747f960e16d008f8395070c1343e497c94186b --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op7-10/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.996875, + "pass@4": 0.996875, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9555555555555556, + "pass@2": 0.9777777777777777, + "pass@4": 0.9936507936507937, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9610778443113772, + "pass@2": 0.9820359281437125, + "pass@4": 0.9850299401197605, + "pass@8": 0.9940119760479041, + "pass@16": 0.9970059880239521, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8875739644970414, + "pass@2": 0.9378698224852071, + "pass@4": 0.9556213017751479, + "pass@8": 0.9792899408284024, + "pass@16": 0.9940828402366864, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8269230769230769, + "pass@2": 0.8910256410256411, + "pass@4": 0.9519230769230769, + "pass@8": 0.9807692307692307, + "pass@16": 0.9903846153846154, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.752112676056338, + "pass@2": 0.828169014084507, + "pass@4": 0.9070422535211268, + "pass@8": 0.9464788732394366, + "pass@16": 0.9915492957746479, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6430678466076696, + "pass@2": 0.7256637168141593, + "pass@4": 0.8053097345132744, + "pass@8": 0.8672566371681416, + "pass@16": 0.9380530973451328, + "pass@32": 0.9616519174041298, + "pass@64": 0.9705014749262537, + "pass@128": 0.9793510324483776 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9555555555555556, + "6": 0.9610778443113772, + "7": 0.8875739644970414, + "8": 0.8269230769230769, + "9": 0.752112676056338, + "10": 0.6430678466076696 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9777777777777777, + "6": 0.9820359281437125, + "7": 0.9378698224852071, + "8": 0.8910256410256411, + "9": 0.828169014084507, + "10": 0.7256637168141593 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 0.9850299401197605, + "7": 0.9556213017751479, + "8": 0.9519230769230769, + "9": 0.9070422535211268, + "10": 0.8053097345132744 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9940119760479041, + "7": 0.9792899408284024, + "8": 0.9807692307692307, + "9": 0.9464788732394366, + "10": 0.8672566371681416 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9970059880239521, + "7": 0.9940828402366864, + "8": 0.9903846153846154, + "9": 0.9915492957746479, + "10": 0.9380530973451328 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9616519174041298 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9705014749262537 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9793510324483776 + } + }, + "total_pass_at_k": { + "pass@1": 0.8885511651469098, + "pass@2": 0.9246876055386694, + "pass@4": 0.9537318473488686, + "pass@8": 0.9733198243836542, + "pass@16": 0.9898682877406282, + "pass@32": 0.9952718676122931, + "pass@64": 0.9966227625802094, + "pass@128": 0.9976359338061466 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 0.9912280701754386, + "pass@4": 0.9970760233918129, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.961764705882353, + "pass@2": 0.9794117647058823, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9621451104100947, + "pass@2": 0.9779179810725552, + "pass@4": 0.9842271293375394, + "pass@8": 0.9936908517350158, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8765822784810127, + "pass@2": 0.9208860759493671, + "pass@4": 0.9525316455696202, + "pass@8": 0.9651898734177216, + "pass@16": 0.990506329113924, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8348348348348348, + "pass@2": 0.8798798798798799, + "pass@4": 0.9369369369369369, + "pass@8": 0.954954954954955, + "pass@16": 0.978978978978979, + "pass@32": 0.993993993993994, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.7491961414790996, + "pass@2": 0.8263665594855305, + "pass@4": 0.8906752411575563, + "pass@8": 0.932475884244373, + "pass@16": 0.9646302250803859, + "pass@32": 0.9903536977491961, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6059701492537314, + "pass@2": 0.6656716417910448, + "pass@4": 0.7492537313432835, + "pass@8": 0.8328358208955224, + "pass@16": 0.9164179104477612, + "pass@32": 0.9492537313432836, + "pass@64": 0.9552238805970149, + "pass@128": 0.9671641791044776 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.961764705882353, + "6": 0.9621451104100947, + "7": 0.8765822784810127, + "8": 0.8348348348348348, + "9": 0.7491961414790996, + "10": 0.6059701492537314 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9794117647058823, + "6": 0.9779179810725552, + "7": 0.9208860759493671, + "8": 0.8798798798798799, + "9": 0.8263665594855305, + "10": 0.6656716417910448 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 1.0, + "6": 0.9842271293375394, + "7": 0.9525316455696202, + "8": 0.9369369369369369, + "9": 0.8906752411575563, + "10": 0.7492537313432835 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9936908517350158, + "7": 0.9651898734177216, + "8": 0.954954954954955, + "9": 0.932475884244373, + "10": 0.8328358208955224 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.990506329113924, + "8": 0.978978978978979, + "9": 0.9646302250803859, + "10": 0.9164179104477612 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.993993993993994, + "9": 0.9903536977491961, + "10": 0.9492537313432836 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9552238805970149 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9671641791044776 + } + }, + "total_pass_at_k": { + "pass@1": 0.8885169065952461, + "pass@2": 0.916973552058922, + "pass@4": 0.9464345497154335, + "pass@8": 0.9648476732507533, + "pass@16": 0.9835955808503515, + "pass@32": 0.9926347505858721, + "pass@64": 0.9946434549715434, + "pass@128": 0.9959825912286575 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9792899408284024, + "pass@2": 0.9822485207100592, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9652173913043478, + "pass@2": 0.9826086956521739, + "pass@4": 0.9971014492753624, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9713467048710601, + "pass@2": 0.9828080229226361, + "pass@4": 0.9914040114613181, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.869942196531792, + "pass@2": 0.9161849710982659, + "pass@4": 0.953757225433526, + "pass@8": 0.976878612716763, + "pass@16": 0.9971098265895953, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7859154929577464, + "pass@2": 0.8591549295774648, + "pass@4": 0.9408450704225352, + "pass@8": 0.9774647887323944, + "pass@16": 0.9915492957746479, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7485029940119761, + "pass@2": 0.7844311377245509, + "pass@4": 0.8622754491017964, + "pass@8": 0.9431137724550899, + "pass@16": 0.9880239520958084, + "pass@32": 0.9970059880239521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6595092024539877, + "pass@2": 0.7116564417177914, + "pass@4": 0.7760736196319018, + "pass@8": 0.8650306748466258, + "pass@16": 0.9202453987730062, + "pass@32": 0.9570552147239264, + "pass@64": 0.9723926380368099, + "pass@128": 0.9785276073619632 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9792899408284024, + "5": 0.9652173913043478, + "6": 0.9713467048710601, + "7": 0.869942196531792, + "8": 0.7859154929577464, + "9": 0.7485029940119761, + "10": 0.6595092024539877 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9822485207100592, + "5": 0.9826086956521739, + "6": 0.9828080229226361, + "7": 0.9161849710982659, + "8": 0.8591549295774648, + "9": 0.7844311377245509, + "10": 0.7116564417177914 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.9914040114613181, + "7": 0.953757225433526, + "8": 0.9408450704225352, + "9": 0.8622754491017964, + "10": 0.7760736196319018 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.976878612716763, + "8": 0.9774647887323944, + "9": 0.9431137724550899, + "10": 0.8650306748466258 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9915492957746479, + "9": 0.9880239520958084, + "10": 0.9202453987730062 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.9570552147239264 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 1.0, + "9": 1.0, + "10": 0.9723926380368099 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9785276073619632 + } + }, + "total_pass_at_k": { + "pass@1": 0.8869593709043251, + "pass@2": 0.913826998689384, + "pass@4": 0.9475753604193972, + "pass@8": 0.9741153342070773, + "pass@16": 0.9888597640891219, + "pass@32": 0.9947575360419397, + "pass@64": 0.9967234600262124, + "pass@128": 0.9977064220183486 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.987, + "pass@2": 0.99, + "pass@4": 0.998, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.961, + "pass@2": 0.98, + "pass@4": 0.997, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.965, + "pass@2": 0.981, + "pass@4": 0.987, + "pass@8": 0.996, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.878, + "pass@2": 0.925, + "pass@4": 0.954, + "pass@8": 0.974, + "pass@16": 0.994, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.815, + "pass@2": 0.876, + "pass@4": 0.943, + "pass@8": 0.971, + "pass@16": 0.987, + "pass@32": 0.998, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.75, + "pass@2": 0.813, + "pass@4": 0.887, + "pass@8": 0.941, + "pass@16": 0.982, + "pass@32": 0.995, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.636, + "pass@2": 0.701, + "pass@4": 0.777, + "pass@8": 0.855, + "pass@16": 0.925, + "pass@32": 0.956, + "pass@64": 0.966, + "pass@128": 0.975 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.987, + "5": 0.961, + "6": 0.965, + "7": 0.878, + "8": 0.815, + "9": 0.75, + "10": 0.636 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.99, + "5": 0.98, + "6": 0.981, + "7": 0.925, + "8": 0.876, + "9": 0.813, + "10": 0.701 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.998, + "5": 0.997, + "6": 0.987, + "7": 0.954, + "8": 0.943, + "9": 0.887, + "10": 0.777 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.996, + "7": 0.974, + "8": 0.971, + "9": 0.941, + "10": 0.855 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.999, + "7": 0.994, + "8": 0.987, + "9": 0.982, + "10": 0.925 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.999, + "8": 0.998, + "9": 0.995, + "10": 0.956 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.999, + "8": 0.999, + "9": 1.0, + "10": 0.966 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 1.0, + "10": 0.975 + } + }, + "total_pass_at_k": { + "pass@1": 0.888, + "pass@2": 0.9184444444444444, + "pass@4": 0.9492222222222222, + "pass@8": 0.9707777777777777, + "pass@16": 0.9874444444444445, + "pass@32": 0.9942222222222222, + "pass@64": 0.996, + "pass@128": 0.9971111111111111 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5807365439093485, + "pass@2": 0.6402266288951841, + "pass@4": 0.7053824362606232, + "pass@8": 0.7903682719546742, + "pass@16": 0.8583569405099151, + "pass@32": 0.8951841359773371, + "pass@64": 0.9291784702549575, + "pass@128": 0.943342776203966 + }, + "12": { + "pass@1": 0.5790960451977402, + "pass@2": 0.6271186440677966, + "pass@4": 0.6949152542372882, + "pass@8": 0.768361581920904, + "pass@16": 0.807909604519774, + "pass@32": 0.8418079096045198, + "pass@64": 0.8531073446327684, + "pass@128": 0.864406779661017 + }, + "13": { + "pass@1": 0.42946708463949845, + "pass@2": 0.49843260188087773, + "pass@4": 0.554858934169279, + "pass@8": 0.6112852664576802, + "pass@16": 0.6739811912225705, + "pass@32": 0.7147335423197492, + "pass@64": 0.7398119122257053, + "pass@128": 0.7617554858934169 + }, + "14": { + "pass@1": 0.32710280373831774, + "pass@2": 0.38006230529595014, + "pass@4": 0.43613707165109034, + "pass@8": 0.49221183800623053, + "pass@16": 0.5763239875389408, + "pass@32": 0.6137071651090342, + "pass@64": 0.6666666666666666, + "pass@128": 0.7165109034267912 + }, + "15": { + "pass@1": 0.23121387283236994, + "pass@2": 0.29190751445086704, + "pass@4": 0.3554913294797688, + "pass@8": 0.42196531791907516, + "pass@16": 0.4682080924855491, + "pass@32": 0.5549132947976878, + "pass@64": 0.6069364161849711, + "pass@128": 0.661849710982659 + }, + "16": { + "pass@1": 0.13225806451612904, + "pass@2": 0.14193548387096774, + "pass@4": 0.16774193548387098, + "pass@8": 0.22580645161290322, + "pass@16": 0.2645161290322581, + "pass@32": 0.31290322580645163, + "pass@64": 0.3419354838709677, + "pass@128": 0.38064516129032255 + }, + "17": { + "pass@1": 0.08092485549132948, + "pass@2": 0.11271676300578035, + "pass@4": 0.13872832369942195, + "pass@8": 0.17341040462427745, + "pass@16": 0.21098265895953758, + "pass@32": 0.23410404624277456, + "pass@64": 0.2658959537572254, + "pass@128": 0.3092485549132948 + }, + "18": { + "pass@1": 0.03142857142857143, + "pass@2": 0.04285714285714286, + "pass@4": 0.05714285714285714, + "pass@8": 0.06571428571428571, + "pass@16": 0.06571428571428571, + "pass@32": 0.08571428571428572, + "pass@64": 0.09428571428571429, + "pass@128": 0.13428571428571429 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.003003003003003003, + "pass@8": 0.006006006006006006, + "pass@16": 0.009009009009009009, + "pass@32": 0.021021021021021023, + "pass@64": 0.03003003003003003, + "pass@128": 0.05105105105105105 + }, + "20": { + "pass@1": 0.003003003003003003, + "pass@2": 0.006006006006006006, + "pass@4": 0.009009009009009009, + "pass@8": 0.018018018018018018, + "pass@16": 0.024024024024024024, + "pass@32": 0.02702702702702703, + "pass@64": 0.03003003003003003, + "pass@128": 0.03303303303303303 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5807365439093485, + "12": 0.5790960451977402, + "13": 0.42946708463949845, + "14": 0.32710280373831774, + "15": 0.23121387283236994, + "16": 0.13225806451612904, + "17": 0.08092485549132948, + "18": 0.03142857142857143, + "19": 0.0, + "20": 0.003003003003003003 + }, + "pass@2": { + "11": 0.6402266288951841, + "12": 0.6271186440677966, + "13": 0.49843260188087773, + "14": 0.38006230529595014, + "15": 0.29190751445086704, + "16": 0.14193548387096774, + "17": 0.11271676300578035, + "18": 0.04285714285714286, + "19": 0.0, + "20": 0.006006006006006006 + }, + "pass@4": { + "11": 0.7053824362606232, + "12": 0.6949152542372882, + "13": 0.554858934169279, + "14": 0.43613707165109034, + "15": 0.3554913294797688, + "16": 0.16774193548387098, + "17": 0.13872832369942195, + "18": 0.05714285714285714, + "19": 0.003003003003003003, + "20": 0.009009009009009009 + }, + "pass@8": { + "11": 0.7903682719546742, + "12": 0.768361581920904, + "13": 0.6112852664576802, + "14": 0.49221183800623053, + "15": 0.42196531791907516, + "16": 0.22580645161290322, + "17": 0.17341040462427745, + "18": 0.06571428571428571, + "19": 0.006006006006006006, + "20": 0.018018018018018018 + }, + "pass@16": { + "11": 0.8583569405099151, + "12": 0.807909604519774, + "13": 0.6739811912225705, + "14": 0.5763239875389408, + "15": 0.4682080924855491, + "16": 0.2645161290322581, + "17": 0.21098265895953758, + "18": 0.06571428571428571, + "19": 0.009009009009009009, + "20": 0.024024024024024024 + }, + "pass@32": { + "11": 0.8951841359773371, + "12": 0.8418079096045198, + "13": 0.7147335423197492, + "14": 0.6137071651090342, + "15": 0.5549132947976878, + "16": 0.31290322580645163, + "17": 0.23410404624277456, + "18": 0.08571428571428572, + "19": 0.021021021021021023, + "20": 0.02702702702702703 + }, + "pass@64": { + "11": 0.9291784702549575, + "12": 0.8531073446327684, + "13": 0.7398119122257053, + "14": 0.6666666666666666, + "15": 0.6069364161849711, + "16": 0.3419354838709677, + "17": 0.2658959537572254, + "18": 0.09428571428571429, + "19": 0.03003003003003003, + "20": 0.03003003003003003 + }, + "pass@128": { + "11": 0.943342776203966, + "12": 0.864406779661017, + "13": 0.7617554858934169, + "14": 0.7165109034267912, + "15": 0.661849710982659, + "16": 0.38064516129032255, + "17": 0.3092485549132948, + "18": 0.13428571428571429, + "19": 0.05105105105105105, + "20": 0.03303303303303303 + } + }, + "total_pass_at_k": { + "pass@1": 0.24160475482912333, + "pass@2": 0.2763744427934621, + "pass@4": 0.31471025260029717, + "pass@8": 0.3598811292719168, + "pass@16": 0.39821693907875183, + "pass@32": 0.4323922734026746, + "pass@64": 0.4579494799405646, + "pass@128": 0.487667161961367 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5587301587301587, + "pass@2": 0.6222222222222222, + "pass@4": 0.7333333333333333, + "pass@8": 0.8476190476190476, + "pass@16": 0.9111111111111111, + "pass@32": 0.9428571428571428, + "pass@64": 0.9619047619047619, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.5932721712538226, + "pass@2": 0.6452599388379205, + "pass@4": 0.6941896024464832, + "pass@8": 0.7889908256880734, + "pass@16": 0.8348623853211009, + "pass@32": 0.8623853211009175, + "pass@64": 0.8837920489296636, + "pass@128": 0.8960244648318043 + }, + "13": { + "pass@1": 0.4228395061728395, + "pass@2": 0.4783950617283951, + "pass@4": 0.5370370370370371, + "pass@8": 0.6172839506172839, + "pass@16": 0.6604938271604939, + "pass@32": 0.7037037037037037, + "pass@64": 0.7222222222222222, + "pass@128": 0.7469135802469136 + }, + "14": { + "pass@1": 0.32344213649851633, + "pass@2": 0.3857566765578635, + "pass@4": 0.42433234421364985, + "pass@8": 0.47477744807121663, + "pass@16": 0.5578635014836796, + "pass@32": 0.6083086053412463, + "pass@64": 0.6706231454005934, + "pass@128": 0.712166172106825 + }, + "15": { + "pass@1": 0.2345679012345679, + "pass@2": 0.2808641975308642, + "pass@4": 0.31790123456790126, + "pass@8": 0.35802469135802467, + "pass@16": 0.39814814814814814, + "pass@32": 0.48148148148148145, + "pass@64": 0.5555555555555556, + "pass@128": 0.6203703703703703 + }, + "16": { + "pass@1": 0.13864306784660768, + "pass@2": 0.168141592920354, + "pass@4": 0.21238938053097345, + "pass@8": 0.26253687315634217, + "pass@16": 0.29793510324483774, + "pass@32": 0.3421828908554572, + "pass@64": 0.3746312684365782, + "pass@128": 0.4218289085545723 + }, + "17": { + "pass@1": 0.04819277108433735, + "pass@2": 0.09036144578313253, + "pass@4": 0.12349397590361445, + "pass@8": 0.1355421686746988, + "pass@16": 0.18674698795180722, + "pass@32": 0.20481927710843373, + "pass@64": 0.23795180722891565, + "pass@128": 0.2680722891566265 + }, + "18": { + "pass@1": 0.04516129032258064, + "pass@2": 0.06451612903225806, + "pass@4": 0.08709677419354839, + "pass@8": 0.09032258064516129, + "pass@16": 0.1032258064516129, + "pass@32": 0.12903225806451613, + "pass@64": 0.14838709677419354, + "pass@128": 0.16451612903225807 + }, + "19": { + "pass@1": 0.006042296072507553, + "pass@2": 0.012084592145015106, + "pass@4": 0.015105740181268883, + "pass@8": 0.021148036253776436, + "pass@16": 0.02416918429003021, + "pass@32": 0.030211480362537766, + "pass@64": 0.045317220543806644, + "pass@128": 0.0634441087613293 + }, + "20": { + "pass@1": 0.003003003003003003, + "pass@2": 0.006006006006006006, + "pass@4": 0.006006006006006006, + "pass@8": 0.009009009009009009, + "pass@16": 0.012012012012012012, + "pass@32": 0.015015015015015015, + "pass@64": 0.015015015015015015, + "pass@128": 0.015015015015015015 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5587301587301587, + "12": 0.5932721712538226, + "13": 0.4228395061728395, + "14": 0.32344213649851633, + "15": 0.2345679012345679, + "16": 0.13864306784660768, + "17": 0.04819277108433735, + "18": 0.04516129032258064, + "19": 0.006042296072507553, + "20": 0.003003003003003003 + }, + "pass@2": { + "11": 0.6222222222222222, + "12": 0.6452599388379205, + "13": 0.4783950617283951, + "14": 0.3857566765578635, + "15": 0.2808641975308642, + "16": 0.168141592920354, + "17": 0.09036144578313253, + "18": 0.06451612903225806, + "19": 0.012084592145015106, + "20": 0.006006006006006006 + }, + "pass@4": { + "11": 0.7333333333333333, + "12": 0.6941896024464832, + "13": 0.5370370370370371, + "14": 0.42433234421364985, + "15": 0.31790123456790126, + "16": 0.21238938053097345, + "17": 0.12349397590361445, + "18": 0.08709677419354839, + "19": 0.015105740181268883, + "20": 0.006006006006006006 + }, + "pass@8": { + "11": 0.8476190476190476, + "12": 0.7889908256880734, + "13": 0.6172839506172839, + "14": 0.47477744807121663, + "15": 0.35802469135802467, + "16": 0.26253687315634217, + "17": 0.1355421686746988, + "18": 0.09032258064516129, + "19": 0.021148036253776436, + "20": 0.009009009009009009 + }, + "pass@16": { + "11": 0.9111111111111111, + "12": 0.8348623853211009, + "13": 0.6604938271604939, + "14": 0.5578635014836796, + "15": 0.39814814814814814, + "16": 0.29793510324483774, + "17": 0.18674698795180722, + "18": 0.1032258064516129, + "19": 0.02416918429003021, + "20": 0.012012012012012012 + }, + "pass@32": { + "11": 0.9428571428571428, + "12": 0.8623853211009175, + "13": 0.7037037037037037, + "14": 0.6083086053412463, + "15": 0.48148148148148145, + "16": 0.3421828908554572, + "17": 0.20481927710843373, + "18": 0.12903225806451613, + "19": 0.030211480362537766, + "20": 0.015015015015015015 + }, + "pass@64": { + "11": 0.9619047619047619, + "12": 0.8837920489296636, + "13": 0.7222222222222222, + "14": 0.6706231454005934, + "15": 0.5555555555555556, + "16": 0.3746312684365782, + "17": 0.23795180722891565, + "18": 0.14838709677419354, + "19": 0.045317220543806644, + "20": 0.015015015015015015 + }, + "pass@128": { + "11": 0.9746031746031746, + "12": 0.8960244648318043, + "13": 0.7469135802469136, + "14": 0.712166172106825, + "15": 0.6203703703703703, + "16": 0.4218289085545723, + "17": 0.2680722891566265, + "18": 0.16451612903225807, + "19": 0.0634441087613293, + "20": 0.015015015015015015 + } + }, + "total_pass_at_k": { + "pass@1": 0.23594132029339854, + "pass@2": 0.27383863080684595, + "pass@4": 0.3132640586797066, + "pass@8": 0.3584963325183374, + "pass@16": 0.3966992665036675, + "pass@32": 0.4300122249388753, + "pass@64": 0.45965770171149145, + "pass@128": 0.48655256723716384 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6656626506024096, + "pass@2": 0.7078313253012049, + "pass@4": 0.7801204819277109, + "pass@8": 0.8343373493975904, + "pass@16": 0.8855421686746988, + "pass@32": 0.9337349397590361, + "pass@64": 0.9578313253012049, + "pass@128": 0.9608433734939759 + }, + "12": { + "pass@1": 0.5329153605015674, + "pass@2": 0.5956112852664577, + "pass@4": 0.6551724137931034, + "pass@8": 0.7366771159874608, + "pass@16": 0.7836990595611285, + "pass@32": 0.8338557993730408, + "pass@64": 0.8463949843260188, + "pass@128": 0.8620689655172413 + }, + "13": { + "pass@1": 0.4649859943977591, + "pass@2": 0.5434173669467787, + "pass@4": 0.6246498599439776, + "pass@8": 0.6750700280112045, + "pass@16": 0.7310924369747899, + "pass@32": 0.7787114845938375, + "pass@64": 0.7983193277310925, + "pass@128": 0.8207282913165266 + }, + "14": { + "pass@1": 0.32748538011695905, + "pass@2": 0.3888888888888889, + "pass@4": 0.4473684210526316, + "pass@8": 0.5029239766081871, + "pass@16": 0.5701754385964912, + "pass@32": 0.6578947368421053, + "pass@64": 0.7222222222222222, + "pass@128": 0.7602339181286549 + }, + "15": { + "pass@1": 0.24848484848484848, + "pass@2": 0.3090909090909091, + "pass@4": 0.3393939393939394, + "pass@8": 0.3939393939393939, + "pass@16": 0.45151515151515154, + "pass@32": 0.5060606060606061, + "pass@64": 0.5757575757575758, + "pass@128": 0.6333333333333333 + }, + "16": { + "pass@1": 0.1452991452991453, + "pass@2": 0.18233618233618235, + "pass@4": 0.23931623931623933, + "pass@8": 0.2792022792022792, + "pass@16": 0.3190883190883191, + "pass@32": 0.3561253561253561, + "pass@64": 0.4017094017094017, + "pass@128": 0.452991452991453 + }, + "17": { + "pass@1": 0.062111801242236024, + "pass@2": 0.09937888198757763, + "pass@4": 0.12422360248447205, + "pass@8": 0.14906832298136646, + "pass@16": 0.16459627329192547, + "pass@32": 0.19254658385093168, + "pass@64": 0.2422360248447205, + "pass@128": 0.2857142857142857 + }, + "18": { + "pass@1": 0.03235294117647059, + "pass@2": 0.047058823529411764, + "pass@4": 0.061764705882352944, + "pass@8": 0.07941176470588235, + "pass@16": 0.10588235294117647, + "pass@32": 0.12941176470588237, + "pass@64": 0.15, + "pass@128": 0.16470588235294117 + }, + "19": { + "pass@1": 0.002976190476190476, + "pass@2": 0.008928571428571428, + "pass@4": 0.011904761904761904, + "pass@8": 0.01488095238095238, + "pass@16": 0.03273809523809524, + "pass@32": 0.03273809523809524, + "pass@64": 0.044642857142857144, + "pass@128": 0.07738095238095238 + }, + "20": { + "pass@1": 0.005988023952095809, + "pass@2": 0.005988023952095809, + "pass@4": 0.005988023952095809, + "pass@8": 0.008982035928143712, + "pass@16": 0.008982035928143712, + "pass@32": 0.014970059880239521, + "pass@64": 0.03293413173652695, + "pass@128": 0.03293413173652695 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6656626506024096, + "12": 0.5329153605015674, + "13": 0.4649859943977591, + "14": 0.32748538011695905, + "15": 0.24848484848484848, + "16": 0.1452991452991453, + "17": 0.062111801242236024, + "18": 0.03235294117647059, + "19": 0.002976190476190476, + "20": 0.005988023952095809 + }, + "pass@2": { + "11": 0.7078313253012049, + "12": 0.5956112852664577, + "13": 0.5434173669467787, + "14": 0.3888888888888889, + "15": 0.3090909090909091, + "16": 0.18233618233618235, + "17": 0.09937888198757763, + "18": 0.047058823529411764, + "19": 0.008928571428571428, + "20": 0.005988023952095809 + }, + "pass@4": { + "11": 0.7801204819277109, + "12": 0.6551724137931034, + "13": 0.6246498599439776, + "14": 0.4473684210526316, + "15": 0.3393939393939394, + "16": 0.23931623931623933, + "17": 0.12422360248447205, + "18": 0.061764705882352944, + "19": 0.011904761904761904, + "20": 0.005988023952095809 + }, + "pass@8": { + "11": 0.8343373493975904, + "12": 0.7366771159874608, + "13": 0.6750700280112045, + "14": 0.5029239766081871, + "15": 0.3939393939393939, + "16": 0.2792022792022792, + "17": 0.14906832298136646, + "18": 0.07941176470588235, + "19": 0.01488095238095238, + "20": 0.008982035928143712 + }, + "pass@16": { + "11": 0.8855421686746988, + "12": 0.7836990595611285, + "13": 0.7310924369747899, + "14": 0.5701754385964912, + "15": 0.45151515151515154, + "16": 0.3190883190883191, + "17": 0.16459627329192547, + "18": 0.10588235294117647, + "19": 0.03273809523809524, + "20": 0.008982035928143712 + }, + "pass@32": { + "11": 0.9337349397590361, + "12": 0.8338557993730408, + "13": 0.7787114845938375, + "14": 0.6578947368421053, + "15": 0.5060606060606061, + "16": 0.3561253561253561, + "17": 0.19254658385093168, + "18": 0.12941176470588237, + "19": 0.03273809523809524, + "20": 0.014970059880239521 + }, + "pass@64": { + "11": 0.9578313253012049, + "12": 0.8463949843260188, + "13": 0.7983193277310925, + "14": 0.7222222222222222, + "15": 0.5757575757575758, + "16": 0.4017094017094017, + "17": 0.2422360248447205, + "18": 0.15, + "19": 0.044642857142857144, + "20": 0.03293413173652695 + }, + "pass@128": { + "11": 0.9608433734939759, + "12": 0.8620689655172413, + "13": 0.8207282913165266, + "14": 0.7602339181286549, + "15": 0.6333333333333333, + "16": 0.452991452991453, + "17": 0.2857142857142857, + "18": 0.16470588235294117, + "19": 0.07738095238095238, + "20": 0.03293413173652695 + } + }, + "total_pass_at_k": { + "pass@1": 0.24858757062146894, + "pass@2": 0.2887303003270889, + "pass@4": 0.32917038358608386, + "pass@8": 0.36752899197145406, + "pass@16": 0.4055902468034493, + "pass@32": 0.4439488551888195, + "pass@64": 0.4775498067201903, + "pass@128": 0.5055010407374368 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.602, + "pass@2": 0.657, + "pass@4": 0.739, + "pass@8": 0.823, + "pass@16": 0.884, + "pass@32": 0.923, + "pass@64": 0.949, + "pass@128": 0.959 + }, + "12": { + "pass@1": 0.569, + "pass@2": 0.623, + "pass@4": 0.682, + "pass@8": 0.765, + "pass@16": 0.809, + "pass@32": 0.846, + "pass@64": 0.861, + "pass@128": 0.874 + }, + "13": { + "pass@1": 0.44, + "pass@2": 0.508, + "pass@4": 0.574, + "pass@8": 0.636, + "pass@16": 0.69, + "pass@32": 0.734, + "pass@64": 0.755, + "pass@128": 0.778 + }, + "14": { + "pass@1": 0.326, + "pass@2": 0.385, + "pass@4": 0.436, + "pass@8": 0.49, + "pass@16": 0.568, + "pass@32": 0.627, + "pass@64": 0.687, + "pass@128": 0.73 + }, + "15": { + "pass@1": 0.238, + "pass@2": 0.294, + "pass@4": 0.338, + "pass@8": 0.392, + "pass@16": 0.44, + "pass@32": 0.515, + "pass@64": 0.58, + "pass@128": 0.639 + }, + "16": { + "pass@1": 0.139, + "pass@2": 0.165, + "pass@4": 0.208, + "pass@8": 0.257, + "pass@16": 0.295, + "pass@32": 0.338, + "pass@64": 0.374, + "pass@128": 0.42 + }, + "17": { + "pass@1": 0.064, + "pass@2": 0.101, + "pass@4": 0.129, + "pass@8": 0.153, + "pass@16": 0.188, + "pass@32": 0.211, + "pass@64": 0.249, + "pass@128": 0.288 + }, + "18": { + "pass@1": 0.036, + "pass@2": 0.051, + "pass@4": 0.068, + "pass@8": 0.078, + "pass@16": 0.091, + "pass@32": 0.114, + "pass@64": 0.13, + "pass@128": 0.154 + }, + "19": { + "pass@1": 0.003, + "pass@2": 0.007, + "pass@4": 0.01, + "pass@8": 0.014, + "pass@16": 0.022, + "pass@32": 0.028, + "pass@64": 0.04, + "pass@128": 0.064 + }, + "20": { + "pass@1": 0.004, + "pass@2": 0.006, + "pass@4": 0.007, + "pass@8": 0.012, + "pass@16": 0.015, + "pass@32": 0.019, + "pass@64": 0.026, + "pass@128": 0.027 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.602, + "12": 0.569, + "13": 0.44, + "14": 0.326, + "15": 0.238, + "16": 0.139, + "17": 0.064, + "18": 0.036, + "19": 0.003, + "20": 0.004 + }, + "pass@2": { + "11": 0.657, + "12": 0.623, + "13": 0.508, + "14": 0.385, + "15": 0.294, + "16": 0.165, + "17": 0.101, + "18": 0.051, + "19": 0.007, + "20": 0.006 + }, + "pass@4": { + "11": 0.739, + "12": 0.682, + "13": 0.574, + "14": 0.436, + "15": 0.338, + "16": 0.208, + "17": 0.129, + "18": 0.068, + "19": 0.01, + "20": 0.007 + }, + "pass@8": { + "11": 0.823, + "12": 0.765, + "13": 0.636, + "14": 0.49, + "15": 0.392, + "16": 0.257, + "17": 0.153, + "18": 0.078, + "19": 0.014, + "20": 0.012 + }, + "pass@16": { + "11": 0.884, + "12": 0.809, + "13": 0.69, + "14": 0.568, + "15": 0.44, + "16": 0.295, + "17": 0.188, + "18": 0.091, + "19": 0.022, + "20": 0.015 + }, + "pass@32": { + "11": 0.923, + "12": 0.846, + "13": 0.734, + "14": 0.627, + "15": 0.515, + "16": 0.338, + "17": 0.211, + "18": 0.114, + "19": 0.028, + "20": 0.019 + }, + "pass@64": { + "11": 0.949, + "12": 0.861, + "13": 0.755, + "14": 0.687, + "15": 0.58, + "16": 0.374, + "17": 0.249, + "18": 0.13, + "19": 0.04, + "20": 0.026 + }, + "pass@128": { + "11": 0.959, + "12": 0.874, + "13": 0.778, + "14": 0.73, + "15": 0.639, + "16": 0.42, + "17": 0.288, + "18": 0.154, + "19": 0.064, + "20": 0.027 + } + }, + "total_pass_at_k": { + "pass@1": 0.2421, + "pass@2": 0.2797, + "pass@4": 0.3191, + "pass@8": 0.362, + "pass@16": 0.4002, + "pass@32": 0.4355, + "pass@64": 0.4651, + "pass@128": 0.4933 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block4-epoch4-op7-10/summary.csv b/composition/SDAR-100M-block4-epoch4-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..9824b208d15cb40f8cfdfb72a7ae37cbee158217 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch4-op7-10,0.9950512152777777,0.5002296875,0.7346188322368421,0.0,0.0,0.0,200.65094791666667,344.50245,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block4-epoch4-op7-10/summary.json b/composition/SDAR-100M-block4-epoch4-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..abfc6c905c29ae01839c24ab7b70b51a1833e3c7 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch4-op7-10", + "id_acc": 0.9950512152777777, + "ood_acc": 0.5002296875, + "total_acc": 0.7346188322368421, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.65094791666667, + "ood_avg_resp_len": 344.50245, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 592869, + "answer_accuracy": 0.7321829058646854, + "avg_response_len": 264.0533796534145, + "resp_tokens_sum": 213811415, + "pass_at_k": { + "pass@1": 0.7321829058646854, + "pass@2": 0.771732271017493, + "pass@4": 0.805390515893657, + "pass@8": 0.8332625061636697, + "pass@16": 0.8558427228397145, + "pass@32": 0.8742772288713363, + "pass@64": 0.8899804869612601, + "pass@128": 0.9042048687954474 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9712389380530974, + "pass@2": 0.9810781914152326, + "pass@4": 0.9877872917987017, + "pass@8": 0.9923114067410685, + "pass@16": 0.9953009685125475, + "pass@32": 0.9976251357688136, + "pass@64": 0.9992232974200413, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999267578125, + "pass@2": 0.9999988465797245, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9967261904761905, + "pass@2": 0.9979080739907511, + "pass@4": 0.998781230917564, + "pass@8": 0.9995512519293239, + "pass@16": 0.9999456111182645, + "pass@32": 0.999999550305024, + "pass@64": 0.9999999999992847, + "pass@128": 1.0 + }, + "6": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9986362795857988, + "pass@2": 0.9996065176117038, + "pass@4": 0.9999584012797217, + "pass@8": 0.9999995539350579, + "pass@16": 0.9999999999795047, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9996494391025641, + "pass@2": 0.9999948736876642, + "pass@4": 0.9999999996995569, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9967649647887324, + "pass@2": 0.997464164910724, + "pass@4": 0.99782990673701, + "pass@8": 0.9983444563318933, + "pass@16": 0.9990631538364027, + "pass@32": 0.9997387400876238, + "pass@64": 0.9999912792555583, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9256595254957507, + "pass@2": 0.9379046446655216, + "pass@4": 0.9464527073103111, + "pass@8": 0.9544125837859465, + "pass@16": 0.9620550938845545, + "pass@32": 0.9689466009864247, + "pass@64": 0.9733538467202176, + "pass@128": 0.9745042492917847 + }, + "12": { + "pass@1": 0.8546963276836158, + "pass@2": 0.8892605320521376, + "pass@4": 0.9096035241357541, + "pass@8": 0.9232561725273154, + "pass@16": 0.9335660780959657, + "pass@32": 0.9430045267513955, + "pass@64": 0.9524606602559231, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.7480407523510971, + "pass@2": 0.8101477457853037, + "pass@4": 0.8495423074074989, + "pass@8": 0.8771021172653639, + "pass@16": 0.896003390136595, + "pass@32": 0.9090301477413725, + "pass@64": 0.9191436054892381, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.682583722741433, + "pass@2": 0.7649933616650716, + "pass@4": 0.818672544436618, + "pass@8": 0.852279360455284, + "pass@16": 0.8766875266221614, + "pass@32": 0.8978875992415474, + "pass@64": 0.9145590851330114, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.5563583815028902, + "pass@2": 0.6612466433025347, + "pass@4": 0.7391046363062996, + "pass@8": 0.7995924425216887, + "pass@16": 0.8472066555948774, + "pass@32": 0.8814056698555999, + "pass@64": 0.9055910973040048, + "pass@128": 0.9248554913294798 + }, + "16": { + "pass@1": 0.35776209677419357, + "pass@2": 0.4704236252222501, + "pass@4": 0.5702631176143302, + "pass@8": 0.6486250520290218, + "pass@16": 0.7058972081786804, + "pass@32": 0.7489029331029314, + "pass@64": 0.7864609089054653, + "pass@128": 0.8225806451612904 + }, + "17": { + "pass@1": 0.2822209176300578, + "pass@2": 0.38103423262482355, + "pass@4": 0.4747528679579794, + "pass@8": 0.5555152783820172, + "pass@16": 0.6224005820967119, + "pass@32": 0.6793292843272832, + "pass@64": 0.727092303936004, + "pass@128": 0.7658959537572254 + }, + "18": { + "pass@1": 0.21580357142857143, + "pass@2": 0.29927622328458936, + "pass@4": 0.3840758931919225, + "pass@8": 0.46175944378352485, + "pass@16": 0.5272364483452201, + "pass@32": 0.5804867351029614, + "pass@64": 0.6281698996492903, + "pass@128": 0.68 + }, + "19": { + "pass@1": 0.18041478978978978, + "pass@2": 0.25600452815019736, + "pass@4": 0.3330379324769089, + "pass@8": 0.40319579360003893, + "pass@16": 0.46454362983819264, + "pass@32": 0.5196802274780482, + "pass@64": 0.5703849188444975, + "pass@128": 0.6186186186186187 + }, + "20": { + "pass@1": 0.17896021021021022, + "pass@2": 0.24850367099382842, + "pass@4": 0.32194917217555025, + "pass@8": 0.3922391954982608, + "pass@16": 0.45386218477737117, + "pass@32": 0.5048449014642736, + "pass@64": 0.55023639343001, + "pass@128": 0.5945945945945946 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 605793, + "answer_accuracy": 0.737764273187841, + "avg_response_len": 273.1703271141855, + "resp_tokens_sum": 224305619, + "pass_at_k": { + "pass@1": 0.737764273187841, + "pass@2": 0.7765919489262912, + "pass@4": 0.8097011007636077, + "pass@8": 0.8373497793981471, + "pass@16": 0.8596389213374069, + "pass@32": 0.8775147067749307, + "pass@64": 0.8927725359259874, + "pass@128": 0.9070927513639907 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9639810199386503, + "pass@2": 0.9763013411187863, + "pass@4": 0.9849948809121252, + "pass@8": 0.9900287848894771, + "pass@16": 0.9927408931048474, + "pass@32": 0.9949717791311338, + "pass@64": 0.9965136850356024, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9995018115942029, + "pass@2": 0.9999390191715166, + "pass@4": 0.9999989468707715, + "pass@8": 0.9999999998467666, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993955945558739, + "pass@2": 0.9998854291225775, + "pass@4": 0.9999959845635341, + "pass@8": 0.9999999968689766, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995484104046243, + "pass@2": 0.9999324393518729, + "pass@4": 0.9999986873931221, + "pass@8": 0.999999999745349, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954225352112676, + "pass@2": 0.9976322501940778, + "pass@4": 0.9990125788853857, + "pass@8": 0.9997359054080507, + "pass@16": 0.9999800133300545, + "pass@32": 0.9999999433844207, + "pass@64": 0.9999999999999964, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9961171407185628, + "pass@2": 0.9976218822198123, + "pass@4": 0.9984793601772832, + "pass@8": 0.9992627421174618, + "pass@16": 0.999836737839751, + "pass@32": 0.9999945907477397, + "pass@64": 0.9999999995090815, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9549839984939759, + "pass@2": 0.9656675422161087, + "pass@4": 0.9699165173065714, + "pass@8": 0.9719722005758548, + "pass@16": 0.9736488700375456, + "pass@32": 0.9754268332917806, + "pass@64": 0.9774205766085304, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8691222570532915, + "pass@2": 0.8991303717325303, + "pass@4": 0.9174640042055867, + "pass@8": 0.9318582991709181, + "pass@16": 0.9426732941451575, + "pass@32": 0.949884708573006, + "pass@64": 0.9548453788380237, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7502626050420168, + "pass@2": 0.8138757471492536, + "pass@4": 0.8540254450861707, + "pass@8": 0.883486764099256, + "pass@16": 0.9069851064183747, + "pass@32": 0.9246218794102262, + "pass@64": 0.9372843805980681, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.6877741228070176, + "pass@2": 0.7709750713726572, + "pass@4": 0.8255886059746912, + "pass@8": 0.8616636529389289, + "pass@16": 0.8884648994522174, + "pass@32": 0.9102203876391841, + "pass@64": 0.9280497250391677, + "pass@128": 0.9415204678362573 + }, + "15": { + "pass@1": 0.5294981060606061, + "pass@2": 0.6371901843235506, + "pass@4": 0.7218647379304856, + "pass@8": 0.784792093362903, + "pass@16": 0.8296342586994554, + "pass@32": 0.8615023805667635, + "pass@64": 0.8850014181350153, + "pass@128": 0.9030303030303031 + }, + "16": { + "pass@1": 0.38826566951566954, + "pass@2": 0.49237134620992856, + "pass@4": 0.5828217968480429, + "pass@8": 0.6537934355879218, + "pass@16": 0.7056221038183779, + "pass@32": 0.7460727057171335, + "pass@64": 0.784026806543782, + "pass@128": 0.8233618233618234 + }, + "17": { + "pass@1": 0.3012179736024845, + "pass@2": 0.40523403494400156, + "pass@4": 0.5060908428760066, + "pass@8": 0.5919590185783913, + "pass@16": 0.6587760745115138, + "pass@32": 0.7099266521318334, + "pass@64": 0.7537711568802992, + "pass@128": 0.7950310559006211 + }, + "18": { + "pass@1": 0.24391084558823528, + "pass@2": 0.32872459761463624, + "pass@4": 0.4134082233287015, + "pass@8": 0.4939817269103554, + "pass@16": 0.5646029865362203, + "pass@32": 0.6231760486863247, + "pass@64": 0.672253518753805, + "pass@128": 0.7176470588235294 + }, + "19": { + "pass@1": 0.17578125, + "pass@2": 0.24372283054461924, + "pass@4": 0.3149261088457693, + "pass@8": 0.382738789933009, + "pass@16": 0.44227697220974477, + "pass@32": 0.49650961347677647, + "pass@64": 0.5534303542394388, + "pass@128": 0.6190476190476191 + }, + "20": { + "pass@1": 0.14549026946107785, + "pass@2": 0.21075847812249524, + "pass@4": 0.28126393697419555, + "pass@8": 0.35182762438965187, + "pass@16": 0.41691692179414386, + "pass@32": 0.47061910952622776, + "pass@64": 0.511055493082766, + "pass@128": 0.5419161676646707 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 587931, + "answer_accuracy": 0.7338569959258667, + "avg_response_len": 292.07440535628695, + "resp_tokens_sum": 233995994, + "pass_at_k": { + "pass@1": 0.7338569959258667, + "pass@2": 0.7727384574087262, + "pass@4": 0.80609744740726, + "pass@8": 0.8340243399432995, + "pass@16": 0.8570450005642073, + "pass@32": 0.8759931171931754, + "pass@64": 0.8914421187831943, + "pass@128": 0.9042978111519412 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9648320895522388, + "pass@2": 0.9742709924785523, + "pass@4": 0.9802755158403708, + "pass@8": 0.9853434213761023, + "pass@16": 0.9897842102909985, + "pass@32": 0.9931238035577333, + "pass@64": 0.9955219923489672, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962775735294118, + "pass@2": 0.9990218996062991, + "pass@4": 0.9999060312313901, + "pass@8": 0.9999988032093555, + "pass@16": 0.9999999998445908, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995563880126183, + "pass@2": 0.9999472167714053, + "pass@4": 0.9999992962236186, + "pass@8": 0.999999999946361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998763844936709, + "pass@2": 0.9999988319794678, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9926097972972973, + "pass@2": 0.9953824395261401, + "pass@4": 0.9979162804536821, + "pass@8": 0.9994617011862063, + "pass@16": 0.9999566861229163, + "pass@32": 0.9999997975032378, + "pass@64": 0.9999999999999107, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9963826366559485, + "pass@2": 0.9985176880522572, + "pass@4": 0.9996576828619894, + "pass@8": 0.9999795024220703, + "pass@16": 0.9999999337376294, + "pass@32": 0.9999999999998973, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9442212301587302, + "pass@2": 0.9586360298712663, + "pass@4": 0.9687240315198693, + "pass@8": 0.9762697265593556, + "pass@16": 0.9817871018932806, + "pass@32": 0.9860954944679531, + "pass@64": 0.9896482633443495, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.896621750764526, + "pass@2": 0.9290360350598379, + "pass@4": 0.9457635553583317, + "pass@8": 0.956214278217107, + "pass@16": 0.9637589402015467, + "pass@32": 0.9705687233059196, + "pass@64": 0.977748568972364, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.7021846064814815, + "pass@2": 0.7656755036696803, + "pass@4": 0.8098888909140981, + "pass@8": 0.8446585152568515, + "pass@16": 0.8737353120262339, + "pass@32": 0.8965907782767284, + "pass@64": 0.9122739330215466, + "pass@128": 0.9259259259259259 + }, + "14": { + "pass@1": 0.6611646884272997, + "pass@2": 0.7445344079300921, + "pass@4": 0.8004453199841117, + "pass@8": 0.8367972897354505, + "pass@16": 0.86221953114493, + "pass@32": 0.882909239208098, + "pass@64": 0.9007521220726056, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.5411120756172839, + "pass@2": 0.6362866208564207, + "pass@4": 0.709073353678012, + "pass@8": 0.7639026736821625, + "pass@16": 0.80479268808345, + "pass@32": 0.8357947268190055, + "pass@64": 0.8608328062716483, + "pass@128": 0.8827160493827161 + }, + "16": { + "pass@1": 0.37479258849557523, + "pass@2": 0.4824867024365314, + "pass@4": 0.5765447806302972, + "pass@8": 0.6533844530467929, + "pass@16": 0.7155828352187087, + "pass@32": 0.7672687388232577, + "pass@64": 0.8085037275060418, + "pass@128": 0.8407079646017699 + }, + "17": { + "pass@1": 0.26618975903614456, + "pass@2": 0.3588547101792997, + "pass@4": 0.4463049080461327, + "pass@8": 0.5217500365258368, + "pass@16": 0.5857671915433449, + "pass@32": 0.6408562235399593, + "pass@64": 0.6900266613904555, + "pass@128": 0.7379518072289156 + }, + "18": { + "pass@1": 0.24914314516129032, + "pass@2": 0.33067968948437876, + "pass@4": 0.41047757689562997, + "pass@8": 0.4840626512960474, + "pass@16": 0.5475132634197127, + "pass@32": 0.5988138059798621, + "pass@64": 0.6409601499492135, + "pass@128": 0.6774193548387096 + }, + "19": { + "pass@1": 0.18079682779456194, + "pass@2": 0.2624990335894569, + "pass@4": 0.34947638899744776, + "pass@8": 0.4290194751124504, + "pass@16": 0.49722958065940137, + "pass@32": 0.555049453644149, + "pass@64": 0.6008047174065532, + "pass@128": 0.6344410876132931 + }, + "20": { + "pass@1": 0.1629129129129129, + "pass@2": 0.2328166010498687, + "pass@4": 0.309161864620526, + "pass@8": 0.3846533878317151, + "pass@16": 0.451918426732032, + "pass@32": 0.5077871637320823, + "pass@64": 0.5519450366312206, + "pass@128": 0.5825825825825826 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_metrics.json b/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..60f607596603c58159fa8d65c9e7ac501e9a415c --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block4-epoch4-op9-12", + "id": { + "answer_accuracy": 0.9942543402777778, + "avg_response_len": 200.17993142361112, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9942543402777778, + "pass@2": 0.9962128034776896, + "pass@4": 0.9975820776257135, + "pass@8": 0.9984810237584633, + "pass@16": 0.9990155200886814, + "pass@32": 0.9993061579381566, + "pass@64": 0.9994829050729563, + "pass@128": 0.9996666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.964, + "pass@2": 0.9748387057086616, + "pass@4": 0.9833349187289088, + "pass@8": 0.9891501055647638, + "pass@16": 0.9926545771584487, + "pass@32": 0.99481551787825, + "pass@64": 0.9963468166102103, + "pass@128": 0.998 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996796875, + "pass@2": 0.9999547244094489, + "pass@4": 0.9999985867079115, + "pass@8": 0.9999999989072665, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966875, + "pass@2": 0.9983604822834645, + "pass@4": 0.999337447225347, + "pass@8": 0.9997908682563258, + "pass@16": 0.999966397971859, + "pass@32": 0.9999993778758997, + "pass@64": 0.9999999999893631, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992578125, + "pass@2": 0.9997546751968504, + "pass@4": 0.9999684317585302, + "pass@8": 0.999999446635866, + "pass@16": 0.9999999998888393, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9984375, + "pass@2": 0.9994987696850394, + "pass@4": 0.9999244012935883, + "pass@8": 0.9999974638450262, + "pass@16": 0.9999999970676888, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9941875, + "pass@2": 0.996166092519685, + "pass@4": 0.997643989782527, + "pass@8": 0.9989351580160544, + "pass@16": 0.9997687565508013, + "pass@32": 0.9999902507610292, + "pass@64": 0.9999999967924652, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9960390625, + "pass@2": 0.997341781496063, + "pass@4": 0.9980309231346081, + "pass@8": 0.9984561726008698, + "pass@16": 0.9987499521604937, + "pass@32": 0.998950274928229, + "pass@64": 0.9989993322645678, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.964, + "2": 1.0, + "3": 1.0, + "4": 0.9996796875, + "5": 0.9966875, + "6": 0.9992578125, + "7": 0.9984375, + "8": 0.9941875, + "9": 0.9960390625 + }, + "per_op_avg_response_len": { + "10": 281.4338828125, + "2": 115.660546875, + "3": 144.800203125, + "4": 159.9624765625, + "5": 172.9998046875, + "6": 199.1400390625, + "7": 224.1442890625, + "8": 243.8543203125, + "9": 259.6238203125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.4338828125, + "2": 115.660546875, + "3": 144.800203125, + "4": 159.9624765625, + "5": 172.9998046875, + "6": 199.1400390625, + "7": 224.1442890625, + "8": 243.8543203125, + "9": 259.6238203125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 377067, + "answer_accuracy": 0.9948787360688957, + "avg_response_len": 190.68579818895643, + "resp_tokens_sum": 72271443, + "pass_at_k": { + "pass@1": 0.9948787360688957, + "pass@2": 0.9965863642044744, + "pass@4": 0.9976738723895583, + "pass@8": 0.9983806804106521, + "pass@16": 0.9988297281125164, + "pass@32": 0.9990484158872104, + "pass@64": 0.9991555623541603, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9659845132743363, + "pass@2": 0.9762676236266927, + "pass@4": 0.9840518931261909, + "pass@8": 0.9893819037909549, + "pass@16": 0.992827041455884, + "pass@32": 0.9946400636162834, + "pass@64": 0.9955741007118146, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9989990234375, + "pass@2": 0.9998585137795276, + "pass@4": 0.9999955834622234, + "pass@8": 0.9999999965852078, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962053571428572, + "pass@2": 0.9975288245219347, + "pass@4": 0.998544181679671, + "pass@8": 0.9993806306839305, + "pass@16": 0.9998935171160672, + "pass@32": 0.9999980250035232, + "pass@64": 0.9999999999662323, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9979890902366864, + "pass@2": 0.9993728305688859, + "pass@4": 0.9999376890018925, + "pass@8": 0.9999995021997173, + "pass@16": 0.9999999999883278, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9989983974358975, + "pass@2": 0.9999191619977791, + "pass@4": 0.9999993645626026, + "pass@8": 0.9999999999826888, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.996875, + "pass@2": 0.9971789397804147, + "pass@4": 0.9971830980634462, + "pass@8": 0.9971830985915493, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 388497, + "answer_accuracy": 0.9944733985910879, + "avg_response_len": 198.1560528956422, + "resp_tokens_sum": 77410851, + "pass_at_k": { + "pass@1": 0.9944733985910879, + "pass@2": 0.9964733658579378, + "pass@4": 0.9978873808860393, + "pass@8": 0.9987639465682274, + "pass@16": 0.9992391331902407, + "pass@32": 0.9995444570264099, + "pass@64": 0.9997914355526902, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9602664877300614, + "pass@2": 0.973362926307908, + "pass@4": 0.9839911745119283, + "pass@8": 0.9905991426701357, + "pass@16": 0.9937449653260887, + "pass@32": 0.9958896624995128, + "pass@64": 0.9980494756223309, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999705615942029, + "pass@2": 0.9999803862832364, + "pass@4": 0.9999999103372947, + "pass@8": 0.9999999999996655, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987464183381088, + "pass@2": 0.9995321982943393, + "pass@4": 0.9999272856831292, + "pass@8": 0.9999984918054343, + "pass@16": 0.9999999996818834, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9992322976878613, + "pass@2": 0.9998545668153475, + "pass@4": 0.9999935653816394, + "pass@8": 0.9999999913233404, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9953125, + "pass@2": 0.9974374792059445, + "pass@4": 0.9986580279577728, + "pass@8": 0.999449156942593, + "pass@16": 0.9999055148797769, + "pass@32": 0.9999982475383377, + "pass@64": 0.999999999970037, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9971570147107359, + "pass@4": 0.9978299348996046, + "pass@8": 0.9984680331236191, + "pass@16": 0.9992530482904565, + "pass@32": 0.999851122735485, + "pass@64": 0.9999980007921191, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379817, + "answer_accuracy": 0.9934115542350184, + "avg_response_len": 211.65934413709408, + "resp_tokens_sum": 80924987, + "pass_at_k": { + "pass@1": 0.9934115542350184, + "pass@2": 0.995576261898674, + "pass@4": 0.9971791349252583, + "pass@8": 0.9982914141961841, + "pass@16": 0.998971215721579, + "pass@32": 0.9993181717967106, + "pass@64": 0.9994921540740299, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.965625, + "pass@2": 0.9748288576801035, + "pass@4": 0.9819707584126611, + "pass@8": 0.9875054318482567, + "pass@16": 0.9914189594286545, + "pass@32": 0.9939477801119064, + "pass@64": 0.9954718430329116, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9940716911764705, + "pass@2": 0.9974872626215842, + "pass@4": 0.9994001792055407, + "pass@8": 0.9999587340911859, + "pass@16": 0.9999998237655816, + "pass@32": 0.9999999999993819, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990881309148265, + "pass@2": 0.9997415174246752, + "pass@4": 0.9999804702054199, + "pass@8": 0.9999999148131528, + "pass@16": 0.999999999999565, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998046875, + "pass@2": 0.999243901375461, + "pass@4": 0.999834458192726, + "pass@8": 0.9999925161507783, + "pass@16": 0.9999999907330216, + "pass@32": 0.9999999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9884806681681682, + "pass@2": 0.9912943208956989, + "pass@4": 0.9943561204624196, + "pass@8": 0.9973895114319368, + "pass@16": 0.99940630260805, + "pass@32": 0.999972591246004, + "pass@64": 0.9999999903997061, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954782958199357, + "pass@2": 0.9977260931209967, + "pass@4": 0.9992145500502148, + "pass@8": 0.9998966223073351, + "pass@16": 0.999998180165534, + "pass@32": 0.9999999997896366, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.58941953125, + "avg_response_len": 358.49741640625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.58941953125, + "pass@2": 0.6611463582677087, + "pass@4": 0.7206630594394348, + "pass@8": 0.768370901754655, + "pass@16": 0.8051262293835766, + "pass@32": 0.8335568122338011, + "pass@64": 0.8565503150232208, + "pass@128": 0.8755 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9586171875, + "pass@2": 0.9686478838582683, + "pass@4": 0.9755326624484442, + "pass@8": 0.9800651950127534, + "pass@16": 0.9828574427616348, + "pass@32": 0.9847804539997522, + "pass@64": 0.9862161770242505, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.9145390625, + "pass@2": 0.9336103592519694, + "pass@4": 0.9449217005999251, + "pass@8": 0.9527443869458525, + "pass@16": 0.959081585245157, + "pass@32": 0.9655409489733907, + "pass@64": 0.9719472610563868, + "pass@128": 0.976 + }, + "13": { + "pass@1": 0.8276640625, + "pass@2": 0.8708352608267721, + "pass@4": 0.8974497278777658, + "pass@8": 0.9151390713116231, + "pass@16": 0.9282975025934377, + "pass@32": 0.9392109882089544, + "pass@64": 0.9478419183711728, + "pass@128": 0.954 + }, + "14": { + "pass@1": 0.812375, + "pass@2": 0.8693993602362209, + "pass@4": 0.9015568980127484, + "pass@8": 0.9207861570009728, + "pass@16": 0.9336252783347806, + "pass@32": 0.9440661346296119, + "pass@64": 0.9530938332686565, + "pass@128": 0.96 + }, + "15": { + "pass@1": 0.70603125, + "pass@2": 0.787097317913386, + "pass@4": 0.8406025208098982, + "pass@8": 0.87726077892471, + "pass@16": 0.9027721805519037, + "pass@32": 0.9205600648585205, + "pass@64": 0.9333755818881951, + "pass@128": 0.944 + }, + "16": { + "pass@1": 0.5262265625, + "pass@2": 0.6398211122047237, + "pass@4": 0.7261620204349456, + "pass@8": 0.7884667385639216, + "pass@16": 0.8334924020394414, + "pass@32": 0.8677150847374071, + "pass@64": 0.8928507015950615, + "pass@128": 0.909 + }, + "17": { + "pass@1": 0.4210078125, + "pass@2": 0.5368409202755909, + "pass@4": 0.6346390308398953, + "pass@8": 0.7096397214779312, + "pass@16": 0.762894409747831, + "pass@32": 0.800335830526573, + "pass@64": 0.8295895976104661, + "pass@128": 0.854 + }, + "18": { + "pass@1": 0.324109375, + "pass@2": 0.42581311515748055, + "pass@4": 0.5186248741094855, + "pass@8": 0.5981313316537631, + "pass@16": 0.6631307957628935, + "pass@32": 0.7148777344144958, + "pass@64": 0.7562454779343195, + "pass@128": 0.791 + }, + "19": { + "pass@1": 0.2183984375, + "pass@2": 0.31187549212598403, + "pass@4": 0.41036086520434906, + "pass@8": 0.5003046941579835, + "pass@16": 0.5738219700180764, + "pass@32": 0.6324042341203246, + "pass@64": 0.6825709398695216, + "pass@128": 0.728 + }, + "20": { + "pass@1": 0.1852265625, + "pass@2": 0.2675227608267715, + "pass@4": 0.3567802940569926, + "pass@8": 0.4411709424970763, + "pass@16": 0.5112887267804792, + "pass@32": 0.5660766478689608, + "pass@64": 0.6117716616140736, + "pass@128": 0.652 + } + }, + "per_op_accuracy": { + "11": 0.9586171875, + "12": 0.9145390625, + "13": 0.8276640625, + "14": 0.812375, + "15": 0.70603125, + "16": 0.5262265625, + "17": 0.4210078125, + "18": 0.324109375, + "19": 0.2183984375, + "20": 0.1852265625 + }, + "per_op_avg_response_len": { + "11": 299.1237578125, + "12": 309.6749375, + "13": 328.88065625, + "14": 354.470578125, + "15": 368.56403125, + "16": 373.670859375, + "17": 387.239296875, + "18": 386.5447734375, + "19": 387.993546875, + "20": 388.8117265625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 299.1237578125, + "12": 309.6749375, + "13": 328.88065625, + "14": 354.470578125, + "15": 368.56403125, + "16": 373.670859375, + "17": 387.239296875, + "18": 386.5447734375, + "19": 387.993546875, + "20": 388.8117265625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 254673, + "answer_accuracy": 0.5912727526002972, + "avg_response_len": 340.9688916233284, + "resp_tokens_sum": 146862121, + "pass_at_k": { + "pass@1": 0.5912727526002972, + "pass@2": 0.6622820532695326, + "pass@4": 0.7210452750891307, + "pass@8": 0.7676588819671737, + "pass@16": 0.8029748732284443, + "pass@32": 0.830823943045339, + "pass@64": 0.8545745693511848, + "pass@128": 0.874591381872214 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9477912535410765, + "pass@2": 0.9614141163480627, + "pass@4": 0.971306078861247, + "pass@8": 0.9777222153354097, + "pass@16": 0.9820192850495909, + "pass@32": 0.9859081102502127, + "pass@64": 0.9893626160626768, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.9022775423728814, + "pass@2": 0.9216494422572181, + "pass@4": 0.9329050222006143, + "pass@8": 0.9403578890122049, + "pass@16": 0.9464571379224949, + "pass@32": 0.9536009096454778, + "pass@64": 0.9612807038635403, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.8387539184952978, + "pass@2": 0.883299805000864, + "pass@4": 0.9113433348260935, + "pass@8": 0.9295898515647626, + "pass@16": 0.943287279603375, + "pass@32": 0.9551149405367227, + "pass@64": 0.9642286009171162, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.8147390965732088, + "pass@2": 0.8667948340569573, + "pass@4": 0.8942816747789692, + "pass@8": 0.9112592074701689, + "pass@16": 0.9250196885538422, + "pass@32": 0.9384068954202733, + "pass@64": 0.9499602716149194, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.7185016257225434, + "pass@2": 0.7971889792226117, + "pass@4": 0.847934874108944, + "pass@8": 0.8830414368074523, + "pass@16": 0.9085139235206722, + "pass@32": 0.9277406485745909, + "pass@64": 0.9418027604093345, + "pass@128": 0.953757225433526 + }, + "16": { + "pass@1": 0.5365171370967742, + "pass@2": 0.6455339566929132, + "pass@4": 0.7244478673028772, + "pass@8": 0.779777220522581, + "pass@16": 0.8186884395276981, + "pass@32": 0.8485246873326461, + "pass@64": 0.8727635997646052, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.420497651734104, + "pass@2": 0.5373062787310544, + "pass@4": 0.6400003012629196, + "pass@8": 0.7204827202394027, + "pass@16": 0.7750126815064169, + "pass@32": 0.8111233885916674, + "pass@64": 0.8380741227154171, + "pass@128": 0.8583815028901735 + }, + "18": { + "pass@1": 0.30125, + "pass@2": 0.404013990438695, + "pass@4": 0.4990766556858963, + "pass@8": 0.5797528146962918, + "pass@16": 0.6458841179962546, + "pass@32": 0.7010997597155946, + "pass@64": 0.7502144482870602, + "pass@128": 0.7971428571428572 + }, + "19": { + "pass@1": 0.21410472972972974, + "pass@2": 0.30520899836844734, + "pass@4": 0.40122690744738015, + "pass@8": 0.4897382851155873, + "pass@16": 0.5619964012938833, + "pass@32": 0.6181259558492432, + "pass@64": 0.6659910429721136, + "pass@128": 0.7117117117117117 + }, + "20": { + "pass@1": 0.20845063813813813, + "pass@2": 0.29438998545789885, + "pass@4": 0.38355911817329136, + "pass@8": 0.46110962180572834, + "pass@16": 0.5194414030776506, + "pass@32": 0.5654700745427053, + "pass@64": 0.6091131851339564, + "pass@128": 0.6516516516516516 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 257305, + "answer_accuracy": 0.5977387191495689, + "avg_response_len": 354.56358255278025, + "resp_tokens_sum": 152626858, + "pass_at_k": { + "pass@1": 0.5977387191495689, + "pass@2": 0.6694296255452482, + "pass@4": 0.7285026617714294, + "pass@8": 0.7757530927649476, + "pass@16": 0.8129403284610648, + "pass@32": 0.8421728400089094, + "pass@64": 0.8656627117097457, + "pass@128": 0.8846268212905144 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9688911897590361, + "pass@2": 0.9749675374727257, + "pass@4": 0.9774471477021697, + "pass@8": 0.9784483556651219, + "pass@16": 0.9788544566955297, + "pass@32": 0.978915040933753, + "pass@64": 0.9789156626487805, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.9082827194357367, + "pass@2": 0.9269358755214379, + "pass@4": 0.9375423858655597, + "pass@8": 0.9452183170723136, + "pass@16": 0.9520156669843709, + "pass@32": 0.958861937474049, + "pass@64": 0.9660584025362567, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.8396358543417367, + "pass@2": 0.8762106712763842, + "pass@4": 0.8982317819516257, + "pass@8": 0.9130115564102529, + "pass@16": 0.9240476363783747, + "pass@32": 0.9335026128044098, + "pass@64": 0.9420415640994138, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.8277366593567251, + "pass@2": 0.8849975681493761, + "pass@4": 0.9186248646002576, + "pass@8": 0.939483467623284, + "pass@16": 0.953015240904691, + "pass@32": 0.9622001675120875, + "pass@64": 0.968673485702234, + "pass@128": 0.9736842105263158 + }, + "15": { + "pass@1": 0.7206202651515151, + "pass@2": 0.8023443092340725, + "pass@4": 0.8544347351467427, + "pass@8": 0.8898100023410217, + "pass@16": 0.9154028263055654, + "pass@32": 0.9328111580456316, + "pass@64": 0.9438559391462911, + "pass@128": 0.9515151515151515 + }, + "16": { + "pass@1": 0.5157140313390314, + "pass@2": 0.6320804450725708, + "pass@4": 0.7233764438633206, + "pass@8": 0.7903596603735945, + "pass@16": 0.8410317719344832, + "pass@32": 0.8811910225527382, + "pass@64": 0.9109104486196058, + "pass@128": 0.9316239316239316 + }, + "17": { + "pass@1": 0.45545419254658387, + "pass@2": 0.5785563744070035, + "pass@4": 0.6760754613553737, + "pass@8": 0.7431020871647102, + "pass@16": 0.7875528862860809, + "pass@32": 0.8186175463395166, + "pass@64": 0.8449973543887125, + "pass@128": 0.8695652173913043 + }, + "18": { + "pass@1": 0.3417279411764706, + "pass@2": 0.44516594777674856, + "pass@4": 0.5401025474756829, + "pass@8": 0.6218323060002307, + "pass@16": 0.6893669338501651, + "pass@32": 0.7441713183922153, + "pass@64": 0.7855624371543468, + "pass@128": 0.8147058823529412 + }, + "19": { + "pass@1": 0.23056175595238096, + "pass@2": 0.3243000386670416, + "pass@4": 0.42189163491840287, + "pass@8": 0.5118960174363878, + "pass@16": 0.5878588842096898, + "pass@32": 0.6511562770502672, + "pass@64": 0.7078413277014779, + "pass@128": 0.7559523809523809 + }, + "20": { + "pass@1": 0.17009730538922155, + "pass@2": 0.2490522154509878, + "pass@4": 0.3370424524965817, + "pass@8": 0.42361443906161345, + "pass@16": 0.4989430300289183, + "pass@32": 0.5583541344634607, + "pass@64": 0.6055054141936476, + "pass@128": 0.6467065868263473 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 242479, + "answer_accuracy": 0.5789630768643031, + "avg_response_len": 380.5673947509169, + "resp_tokens_sum": 159387714, + "pass_at_k": { + "pass@1": 0.5789630768643031, + "pass@2": 0.651464744106524, + "pass@4": 0.7122123447989714, + "pass@8": 0.7615156567110603, + "pass@16": 0.7993073107602144, + "pass@32": 0.8275117032519711, + "pass@64": 0.8492163890236676, + "pass@128": 0.8670537897310513 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.959920634920635, + "pass@2": 0.9700935820522439, + "pass@4": 0.9782513129311214, + "pass@8": 0.9843949172016934, + "pass@16": 0.9880157317975031, + "pass@32": 0.9896987221949869, + "pass@64": 0.9903846779515253, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9339162844036697, + "pass@2": 0.953070068506345, + "pass@4": 0.9651293628434054, + "pass@8": 0.9734955690809288, + "pass@16": 0.9796414698855651, + "pass@32": 0.9849824737145875, + "pass@64": 0.9892393317419794, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8035542052469136, + "pass@2": 0.8526401647710703, + "pass@4": 0.8829088207145402, + "pass@8": 0.903255503160504, + "pass@16": 0.9182217722619813, + "pass@32": 0.9298422512548313, + "pass@64": 0.93809924782445, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7945335682492581, + "pass@2": 0.856050594055936, + "pass@4": 0.8911655095412775, + "pass@8": 0.910886069661382, + "pass@16": 0.9221446466456787, + "pass@32": 0.9310536021085168, + "pass@64": 0.9402678159350233, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.6778549382716049, + "pass@2": 0.7607910773306114, + "pass@4": 0.818683925215366, + "pass@8": 0.8583059907925742, + "pass@16": 0.8837760195460365, + "pass@32": 0.9004139453600427, + "pass@64": 0.9137017495934854, + "pass@128": 0.9259259259259259 + }, + "16": { + "pass@1": 0.5277009587020649, + "pass@2": 0.6426116501753651, + "pass@4": 0.7306137161505246, + "pass@8": 0.7944529776129478, + "pass@16": 0.8392236986337849, + "pass@32": 0.8713108635642363, + "pass@64": 0.8925204076771432, + "pass@128": 0.9056047197640118 + }, + "17": { + "pass@1": 0.3881306475903614, + "pass@2": 0.4958969737216582, + "pass@4": 0.5888633374894398, + "pass@8": 0.6658850247230739, + "pass@16": 0.7263493751882383, + "pass@32": 0.7713623137124465, + "pass@64": 0.8058036236077262, + "pass@128": 0.8343373493975904 + }, + "18": { + "pass@1": 0.3305947580645161, + "pass@2": 0.4291993427736854, + "pass@4": 0.517139285411904, + "pass@8": 0.5928866531289754, + "pass@16": 0.6538277321133854, + "pass@32": 0.6983050653570473, + "pass@64": 0.7309006206495801, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.21037103474320243, + "pass@2": 0.30597003829959324, + "pass@4": 0.40784506251295616, + "pass@8": 0.49916852977603016, + "pass@16": 0.5714700401593907, + "pass@32": 0.6277334795578147, + "pass@64": 0.6735989016680082, + "pass@128": 0.716012084592145 + }, + "20": { + "pass@1": 0.17717717717717718, + "pass@2": 0.25918154867465903, + "pass@4": 0.3497985842985841, + "pass@8": 0.43884148885642665, + "pass@16": 0.5155188213992907, + "pass@32": 0.5744289253316636, + "pass@64": 0.6207152030744365, + "pass@128": 0.6576576576576577 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7811833881578948, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7811833881578948, + "pass@2": 0.8198620428408456, + "pass@4": 0.8518352259487112, + "pass@8": 0.877370433230154, + "pass@16": 0.8969685249807203, + "pass@32": 0.912069660199031, + "pass@64": 0.924255226099405, + "pass@128": 0.9343157894736842 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.964, + "pass@2": 0.9748387057086616, + "pass@4": 0.9833349187289088, + "pass@8": 0.9891501055647638, + "pass@16": 0.9926545771584487, + "pass@32": 0.99481551787825, + "pass@64": 0.9963468166102103, + "pass@128": 0.998 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996796875, + "pass@2": 0.9999547244094489, + "pass@4": 0.9999985867079115, + "pass@8": 0.9999999989072665, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966875, + "pass@2": 0.9983604822834645, + "pass@4": 0.999337447225347, + "pass@8": 0.9997908682563258, + "pass@16": 0.999966397971859, + "pass@32": 0.9999993778758997, + "pass@64": 0.9999999999893631, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9992578125, + "pass@2": 0.9997546751968504, + "pass@4": 0.9999684317585302, + "pass@8": 0.999999446635866, + "pass@16": 0.9999999998888393, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9984375, + "pass@2": 0.9994987696850394, + "pass@4": 0.9999244012935883, + "pass@8": 0.9999974638450262, + "pass@16": 0.9999999970676888, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9941875, + "pass@2": 0.996166092519685, + "pass@4": 0.997643989782527, + "pass@8": 0.9989351580160544, + "pass@16": 0.9997687565508013, + "pass@32": 0.9999902507610292, + "pass@64": 0.9999999967924652, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9960390625, + "pass@2": 0.997341781496063, + "pass@4": 0.9980309231346081, + "pass@8": 0.9984561726008698, + "pass@16": 0.9987499521604937, + "pass@32": 0.998950274928229, + "pass@64": 0.9989993322645678, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9586171875, + "pass@2": 0.9686478838582683, + "pass@4": 0.9755326624484442, + "pass@8": 0.9800651950127534, + "pass@16": 0.9828574427616348, + "pass@32": 0.9847804539997522, + "pass@64": 0.9862161770242505, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.9145390625, + "pass@2": 0.9336103592519694, + "pass@4": 0.9449217005999251, + "pass@8": 0.9527443869458525, + "pass@16": 0.959081585245157, + "pass@32": 0.9655409489733907, + "pass@64": 0.9719472610563868, + "pass@128": 0.976 + }, + "13": { + "pass@1": 0.8276640625, + "pass@2": 0.8708352608267721, + "pass@4": 0.8974497278777658, + "pass@8": 0.9151390713116231, + "pass@16": 0.9282975025934377, + "pass@32": 0.9392109882089544, + "pass@64": 0.9478419183711728, + "pass@128": 0.954 + }, + "14": { + "pass@1": 0.812375, + "pass@2": 0.8693993602362209, + "pass@4": 0.9015568980127484, + "pass@8": 0.9207861570009728, + "pass@16": 0.9336252783347806, + "pass@32": 0.9440661346296119, + "pass@64": 0.9530938332686565, + "pass@128": 0.96 + }, + "15": { + "pass@1": 0.70603125, + "pass@2": 0.787097317913386, + "pass@4": 0.8406025208098982, + "pass@8": 0.87726077892471, + "pass@16": 0.9027721805519037, + "pass@32": 0.9205600648585205, + "pass@64": 0.9333755818881951, + "pass@128": 0.944 + }, + "16": { + "pass@1": 0.5262265625, + "pass@2": 0.6398211122047237, + "pass@4": 0.7261620204349456, + "pass@8": 0.7884667385639216, + "pass@16": 0.8334924020394414, + "pass@32": 0.8677150847374071, + "pass@64": 0.8928507015950615, + "pass@128": 0.909 + }, + "17": { + "pass@1": 0.4210078125, + "pass@2": 0.5368409202755909, + "pass@4": 0.6346390308398953, + "pass@8": 0.7096397214779312, + "pass@16": 0.762894409747831, + "pass@32": 0.800335830526573, + "pass@64": 0.8295895976104661, + "pass@128": 0.854 + }, + "18": { + "pass@1": 0.324109375, + "pass@2": 0.42581311515748055, + "pass@4": 0.5186248741094855, + "pass@8": 0.5981313316537631, + "pass@16": 0.6631307957628935, + "pass@32": 0.7148777344144958, + "pass@64": 0.7562454779343195, + "pass@128": 0.791 + }, + "19": { + "pass@1": 0.2183984375, + "pass@2": 0.31187549212598403, + "pass@4": 0.41036086520434906, + "pass@8": 0.5003046941579835, + "pass@16": 0.5738219700180764, + "pass@32": 0.6324042341203246, + "pass@64": 0.6825709398695216, + "pass@128": 0.728 + }, + "20": { + "pass@1": 0.1852265625, + "pass@2": 0.2675227608267715, + "pass@4": 0.3567802940569926, + "pass@8": 0.4411709424970763, + "pass@16": 0.5112887267804792, + "pass@32": 0.5660766478689608, + "pass@64": 0.6117716616140736, + "pass@128": 0.652 + } + }, + "per_op_accuracy": { + "10": 0.964, + "2": 1.0, + "3": 1.0, + "4": 0.9996796875, + "5": 0.9966875, + "6": 0.9992578125, + "7": 0.9984375, + "8": 0.9941875, + "9": 0.9960390625, + "11": 0.9586171875, + "12": 0.9145390625, + "13": 0.8276640625, + "14": 0.812375, + "15": 0.70603125, + "16": 0.5262265625, + "17": 0.4210078125, + "18": 0.324109375, + "19": 0.2183984375, + "20": 0.1852265625 + }, + "per_op_avg_response_len": { + "10": 281.4338828125, + "2": 115.660546875, + "3": 144.800203125, + "4": 159.9624765625, + "5": 172.9998046875, + "6": 199.1400390625, + "7": 224.1442890625, + "8": 243.8543203125, + "9": 259.6238203125, + "11": 299.1237578125, + "12": 309.6749375, + "13": 328.88065625, + "14": 354.470578125, + "15": 368.56403125, + "16": 373.670859375, + "17": 387.239296875, + "18": 386.5447734375, + "19": 387.993546875, + "20": 388.8117265625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.4338828125, + "2": 115.660546875, + "3": 144.800203125, + "4": 159.9624765625, + "5": 172.9998046875, + "6": 199.1400390625, + "7": 224.1442890625, + "8": 243.8543203125, + "9": 259.6238203125, + "11": 299.1237578125, + "12": 309.6749375, + "13": 328.88065625, + "14": 354.470578125, + "15": 368.56403125, + "16": 373.670859375, + "17": 387.239296875, + "18": 386.5447734375, + "19": 387.993546875, + "20": 388.8117265625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 631740, + "answer_accuracy": 0.7801879149541574, + "avg_response_len": 270.626141123933, + "resp_tokens_sum": 219133564, + "pass_at_k": { + "pass@1": 0.7801879149541574, + "pass@2": 0.8187593002942418, + "pass@4": 0.8505263494815603, + "pass@8": 0.8756524395376909, + "pass@16": 0.8946483201635945, + "pass@32": 0.9095644843170411, + "pass@64": 0.9222483474545463, + "pass@128": 0.9329750237116662 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9659845132743363, + "pass@2": 0.9762676236266927, + "pass@4": 0.9840518931261909, + "pass@8": 0.9893819037909549, + "pass@16": 0.992827041455884, + "pass@32": 0.9946400636162834, + "pass@64": 0.9955741007118146, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9989990234375, + "pass@2": 0.9998585137795276, + "pass@4": 0.9999955834622234, + "pass@8": 0.9999999965852078, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962053571428572, + "pass@2": 0.9975288245219347, + "pass@4": 0.998544181679671, + "pass@8": 0.9993806306839305, + "pass@16": 0.9998935171160672, + "pass@32": 0.9999980250035232, + "pass@64": 0.9999999999662323, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9979890902366864, + "pass@2": 0.9993728305688859, + "pass@4": 0.9999376890018925, + "pass@8": 0.9999995021997173, + "pass@16": 0.9999999999883278, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9989983974358975, + "pass@2": 0.9999191619977791, + "pass@4": 0.9999993645626026, + "pass@8": 0.9999999999826888, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.996875, + "pass@2": 0.9971789397804147, + "pass@4": 0.9971830980634462, + "pass@8": 0.9971830985915493, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9477912535410765, + "pass@2": 0.9614141163480627, + "pass@4": 0.971306078861247, + "pass@8": 0.9777222153354097, + "pass@16": 0.9820192850495909, + "pass@32": 0.9859081102502127, + "pass@64": 0.9893626160626768, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.9022775423728814, + "pass@2": 0.9216494422572181, + "pass@4": 0.9329050222006143, + "pass@8": 0.9403578890122049, + "pass@16": 0.9464571379224949, + "pass@32": 0.9536009096454778, + "pass@64": 0.9612807038635403, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.8387539184952978, + "pass@2": 0.883299805000864, + "pass@4": 0.9113433348260935, + "pass@8": 0.9295898515647626, + "pass@16": 0.943287279603375, + "pass@32": 0.9551149405367227, + "pass@64": 0.9642286009171162, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.8147390965732088, + "pass@2": 0.8667948340569573, + "pass@4": 0.8942816747789692, + "pass@8": 0.9112592074701689, + "pass@16": 0.9250196885538422, + "pass@32": 0.9384068954202733, + "pass@64": 0.9499602716149194, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.7185016257225434, + "pass@2": 0.7971889792226117, + "pass@4": 0.847934874108944, + "pass@8": 0.8830414368074523, + "pass@16": 0.9085139235206722, + "pass@32": 0.9277406485745909, + "pass@64": 0.9418027604093345, + "pass@128": 0.953757225433526 + }, + "16": { + "pass@1": 0.5365171370967742, + "pass@2": 0.6455339566929132, + "pass@4": 0.7244478673028772, + "pass@8": 0.779777220522581, + "pass@16": 0.8186884395276981, + "pass@32": 0.8485246873326461, + "pass@64": 0.8727635997646052, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.420497651734104, + "pass@2": 0.5373062787310544, + "pass@4": 0.6400003012629196, + "pass@8": 0.7204827202394027, + "pass@16": 0.7750126815064169, + "pass@32": 0.8111233885916674, + "pass@64": 0.8380741227154171, + "pass@128": 0.8583815028901735 + }, + "18": { + "pass@1": 0.30125, + "pass@2": 0.404013990438695, + "pass@4": 0.4990766556858963, + "pass@8": 0.5797528146962918, + "pass@16": 0.6458841179962546, + "pass@32": 0.7010997597155946, + "pass@64": 0.7502144482870602, + "pass@128": 0.7971428571428572 + }, + "19": { + "pass@1": 0.21410472972972974, + "pass@2": 0.30520899836844734, + "pass@4": 0.40122690744738015, + "pass@8": 0.4897382851155873, + "pass@16": 0.5619964012938833, + "pass@32": 0.6181259558492432, + "pass@64": 0.6659910429721136, + "pass@128": 0.7117117117117117 + }, + "20": { + "pass@1": 0.20845063813813813, + "pass@2": 0.29438998545789885, + "pass@4": 0.38355911817329136, + "pass@8": 0.46110962180572834, + "pass@16": 0.5194414030776506, + "pass@32": 0.5654700745427053, + "pass@64": 0.6091131851339564, + "pass@128": 0.6516516516516516 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 645802, + "answer_accuracy": 0.786489185502728, + "avg_response_len": 280.1511459957132, + "resp_tokens_sum": 230037709, + "pass_at_k": { + "pass@1": 0.786489185502728, + "pass@2": 0.8250239350439653, + "pass@4": 0.8566651189402132, + "pass@8": 0.8818527226648039, + "pass@16": 0.9015738361825739, + "pass@32": 0.9170439507084295, + "pass@64": 0.9294757850018294, + "pass@128": 0.9395167575993765 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9602664877300614, + "pass@2": 0.973362926307908, + "pass@4": 0.9839911745119283, + "pass@8": 0.9905991426701357, + "pass@16": 0.9937449653260887, + "pass@32": 0.9958896624995128, + "pass@64": 0.9980494756223309, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999705615942029, + "pass@2": 0.9999803862832364, + "pass@4": 0.9999999103372947, + "pass@8": 0.9999999999996655, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987464183381088, + "pass@2": 0.9995321982943393, + "pass@4": 0.9999272856831292, + "pass@8": 0.9999984918054343, + "pass@16": 0.9999999996818834, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9992322976878613, + "pass@2": 0.9998545668153475, + "pass@4": 0.9999935653816394, + "pass@8": 0.9999999913233404, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9953125, + "pass@2": 0.9974374792059445, + "pass@4": 0.9986580279577728, + "pass@8": 0.999449156942593, + "pass@16": 0.9999055148797769, + "pass@32": 0.9999982475383377, + "pass@64": 0.999999999970037, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9971570147107359, + "pass@4": 0.9978299348996046, + "pass@8": 0.9984680331236191, + "pass@16": 0.9992530482904565, + "pass@32": 0.999851122735485, + "pass@64": 0.9999980007921191, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9688911897590361, + "pass@2": 0.9749675374727257, + "pass@4": 0.9774471477021697, + "pass@8": 0.9784483556651219, + "pass@16": 0.9788544566955297, + "pass@32": 0.978915040933753, + "pass@64": 0.9789156626487805, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.9082827194357367, + "pass@2": 0.9269358755214379, + "pass@4": 0.9375423858655597, + "pass@8": 0.9452183170723136, + "pass@16": 0.9520156669843709, + "pass@32": 0.958861937474049, + "pass@64": 0.9660584025362567, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.8396358543417367, + "pass@2": 0.8762106712763842, + "pass@4": 0.8982317819516257, + "pass@8": 0.9130115564102529, + "pass@16": 0.9240476363783747, + "pass@32": 0.9335026128044098, + "pass@64": 0.9420415640994138, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.8277366593567251, + "pass@2": 0.8849975681493761, + "pass@4": 0.9186248646002576, + "pass@8": 0.939483467623284, + "pass@16": 0.953015240904691, + "pass@32": 0.9622001675120875, + "pass@64": 0.968673485702234, + "pass@128": 0.9736842105263158 + }, + "15": { + "pass@1": 0.7206202651515151, + "pass@2": 0.8023443092340725, + "pass@4": 0.8544347351467427, + "pass@8": 0.8898100023410217, + "pass@16": 0.9154028263055654, + "pass@32": 0.9328111580456316, + "pass@64": 0.9438559391462911, + "pass@128": 0.9515151515151515 + }, + "16": { + "pass@1": 0.5157140313390314, + "pass@2": 0.6320804450725708, + "pass@4": 0.7233764438633206, + "pass@8": 0.7903596603735945, + "pass@16": 0.8410317719344832, + "pass@32": 0.8811910225527382, + "pass@64": 0.9109104486196058, + "pass@128": 0.9316239316239316 + }, + "17": { + "pass@1": 0.45545419254658387, + "pass@2": 0.5785563744070035, + "pass@4": 0.6760754613553737, + "pass@8": 0.7431020871647102, + "pass@16": 0.7875528862860809, + "pass@32": 0.8186175463395166, + "pass@64": 0.8449973543887125, + "pass@128": 0.8695652173913043 + }, + "18": { + "pass@1": 0.3417279411764706, + "pass@2": 0.44516594777674856, + "pass@4": 0.5401025474756829, + "pass@8": 0.6218323060002307, + "pass@16": 0.6893669338501651, + "pass@32": 0.7441713183922153, + "pass@64": 0.7855624371543468, + "pass@128": 0.8147058823529412 + }, + "19": { + "pass@1": 0.23056175595238096, + "pass@2": 0.3243000386670416, + "pass@4": 0.42189163491840287, + "pass@8": 0.5118960174363878, + "pass@16": 0.5878588842096898, + "pass@32": 0.6511562770502672, + "pass@64": 0.7078413277014779, + "pass@128": 0.7559523809523809 + }, + "20": { + "pass@1": 0.17009730538922155, + "pass@2": 0.2490522154509878, + "pass@4": 0.3370424524965817, + "pass@8": 0.42361443906161345, + "pass@16": 0.4989430300289183, + "pass@32": 0.5583541344634607, + "pass@64": 0.6055054141936476, + "pass@128": 0.6467065868263473 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 622296, + "answer_accuracy": 0.7767514778718645, + "avg_response_len": 299.9589353830484, + "resp_tokens_sum": 240312701, + "pass_at_k": { + "pass@1": 0.7767514778718645, + "pass@2": 0.8156860420207447, + "pass@4": 0.848207839623572, + "pass@8": 0.8745128108264176, + "pass@16": 0.8945934721469555, + "pass@32": 0.9095033826804982, + "pass@64": 0.9209329108650951, + "pass@128": 0.9303403099536667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.965625, + "pass@2": 0.9748288576801035, + "pass@4": 0.9819707584126611, + "pass@8": 0.9875054318482567, + "pass@16": 0.9914189594286545, + "pass@32": 0.9939477801119064, + "pass@64": 0.9954718430329116, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9940716911764705, + "pass@2": 0.9974872626215842, + "pass@4": 0.9994001792055407, + "pass@8": 0.9999587340911859, + "pass@16": 0.9999998237655816, + "pass@32": 0.9999999999993819, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990881309148265, + "pass@2": 0.9997415174246752, + "pass@4": 0.9999804702054199, + "pass@8": 0.9999999148131528, + "pass@16": 0.999999999999565, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998046875, + "pass@2": 0.999243901375461, + "pass@4": 0.999834458192726, + "pass@8": 0.9999925161507783, + "pass@16": 0.9999999907330216, + "pass@32": 0.9999999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9884806681681682, + "pass@2": 0.9912943208956989, + "pass@4": 0.9943561204624196, + "pass@8": 0.9973895114319368, + "pass@16": 0.99940630260805, + "pass@32": 0.999972591246004, + "pass@64": 0.9999999903997061, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954782958199357, + "pass@2": 0.9977260931209967, + "pass@4": 0.9992145500502148, + "pass@8": 0.9998966223073351, + "pass@16": 0.999998180165534, + "pass@32": 0.9999999997896366, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.959920634920635, + "pass@2": 0.9700935820522439, + "pass@4": 0.9782513129311214, + "pass@8": 0.9843949172016934, + "pass@16": 0.9880157317975031, + "pass@32": 0.9896987221949869, + "pass@64": 0.9903846779515253, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9339162844036697, + "pass@2": 0.953070068506345, + "pass@4": 0.9651293628434054, + "pass@8": 0.9734955690809288, + "pass@16": 0.9796414698855651, + "pass@32": 0.9849824737145875, + "pass@64": 0.9892393317419794, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8035542052469136, + "pass@2": 0.8526401647710703, + "pass@4": 0.8829088207145402, + "pass@8": 0.903255503160504, + "pass@16": 0.9182217722619813, + "pass@32": 0.9298422512548313, + "pass@64": 0.93809924782445, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7945335682492581, + "pass@2": 0.856050594055936, + "pass@4": 0.8911655095412775, + "pass@8": 0.910886069661382, + "pass@16": 0.9221446466456787, + "pass@32": 0.9310536021085168, + "pass@64": 0.9402678159350233, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.6778549382716049, + "pass@2": 0.7607910773306114, + "pass@4": 0.818683925215366, + "pass@8": 0.8583059907925742, + "pass@16": 0.8837760195460365, + "pass@32": 0.9004139453600427, + "pass@64": 0.9137017495934854, + "pass@128": 0.9259259259259259 + }, + "16": { + "pass@1": 0.5277009587020649, + "pass@2": 0.6426116501753651, + "pass@4": 0.7306137161505246, + "pass@8": 0.7944529776129478, + "pass@16": 0.8392236986337849, + "pass@32": 0.8713108635642363, + "pass@64": 0.8925204076771432, + "pass@128": 0.9056047197640118 + }, + "17": { + "pass@1": 0.3881306475903614, + "pass@2": 0.4958969737216582, + "pass@4": 0.5888633374894398, + "pass@8": 0.6658850247230739, + "pass@16": 0.7263493751882383, + "pass@32": 0.7713623137124465, + "pass@64": 0.8058036236077262, + "pass@128": 0.8343373493975904 + }, + "18": { + "pass@1": 0.3305947580645161, + "pass@2": 0.4291993427736854, + "pass@4": 0.517139285411904, + "pass@8": 0.5928866531289754, + "pass@16": 0.6538277321133854, + "pass@32": 0.6983050653570473, + "pass@64": 0.7309006206495801, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.21037103474320243, + "pass@2": 0.30597003829959324, + "pass@4": 0.40784506251295616, + "pass@8": 0.49916852977603016, + "pass@16": 0.5714700401593907, + "pass@32": 0.6277334795578147, + "pass@64": 0.6735989016680082, + "pass@128": 0.716012084592145 + }, + "20": { + "pass@1": 0.17717717717717718, + "pass@2": 0.25918154867465903, + "pass@4": 0.3497985842985841, + "pass@8": 0.43884148885642665, + "pass@16": 0.5155188213992907, + "pass@32": 0.5744289253316636, + "pass@64": 0.6207152030744365, + "pass@128": 0.6576576576576577 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block4-epoch4-op9-12/graphs/SDAR-100M-block4-epoch4-op9-12_score_report.jsonl b/composition/SDAR-100M-block4-epoch4-op9-12/graphs/SDAR-100M-block4-epoch4-op9-12_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..205ae646283e775c11b2b80b279ac2402ed881d9 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op9-12/graphs/SDAR-100M-block4-epoch4-op9-12_score_report.jsonl @@ -0,0 +1,2 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch4-op9-12", "results_dir": "results/composition/SDAR-100M-block4-epoch4-op9-12/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1024490, "value_mismatch": 7557, "dependency_mismatch": 125851, "answer_mismatch": 6619, "extra_nodes": 9839, "missing_nodes": 8665, "perfect_with_extra": 328, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337879, "value_mismatch": 2227, "answer_mismatch": 1941, "dependency_mismatch": 40663, "extra_nodes": 3114, "missing_nodes": 2763, "perfect_with_extra": 52}, "teachers_in_school": {"total": 390656, "perfect": 347019, "value_mismatch": 2435, "dependency_mismatch": 43133, "answer_mismatch": 2159, "extra_nodes": 3469, "perfect_with_extra": 168, "missing_nodes": 3173}, "movie_festival_awards": {"total": 382336, "perfect": 339592, "dependency_mismatch": 42055, "value_mismatch": 2895, "answer_mismatch": 2519, "missing_nodes": 2729, "extra_nodes": 3256, "perfect_with_extra": 108}}, "results_path": "results/composition/SDAR-100M-block4-epoch4-op9-12/graphs/SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 383414, "value_mismatch": 642987, "dependency_mismatch": 836303, "answer_mismatch": 525543, "extra_nodes": 71062, "missing_nodes": 431148, "perfect_with_extra": 171, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 128241, "value_mismatch": 216518, "dependency_mismatch": 281851, "answer_mismatch": 176047, "extra_nodes": 28144, "missing_nodes": 151353, "perfect_with_extra": 29}, "movie_festival_awards": {"total": 418816, "perfect": 124218, "value_mismatch": 215552, "dependency_mismatch": 275331, "answer_mismatch": 176337, "missing_nodes": 141019, "extra_nodes": 22540, "perfect_with_extra": 108}, "teachers_in_school": {"total": 430464, "perfect": 130955, "dependency_mismatch": 279121, "value_mismatch": 210917, "answer_mismatch": 173159, "missing_nodes": 138776, "extra_nodes": 20378, "perfect_with_extra": 34}}, "results_path": "results/composition/SDAR-100M-block4-epoch4-op9-12/graphs/SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1407904, "value_mismatch": 650544, "dependency_mismatch": 962154, "answer_mismatch": 532162, "extra_nodes": 80901, "missing_nodes": 439813, "perfect_with_extra": 499, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch4-op9-12/graphs/SDAR-100M-block4-epoch4-op9-12_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 466120, "value_mismatch": 218745, "answer_mismatch": 177988, "dependency_mismatch": 322514, "extra_nodes": 31258, "missing_nodes": 154116, "perfect_with_extra": 81}, "teachers_in_school": {"total": 821120, "perfect": 477974, "value_mismatch": 213352, "dependency_mismatch": 322254, "answer_mismatch": 175318, "extra_nodes": 23847, "perfect_with_extra": 202, "missing_nodes": 141949}, "movie_festival_awards": {"total": 801152, "perfect": 463810, "dependency_mismatch": 317386, "value_mismatch": 218447, "answer_mismatch": 178856, "missing_nodes": 143748, "extra_nodes": 25796, "perfect_with_extra": 216}}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block4-epoch4-op9-12", "results_dir": "results/composition/SDAR-100M-block4-epoch4-op9-12/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch4-op9-12/graphs/SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block4-epoch4-op9-12/graphs/SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block4-epoch4-op9-12/graphs/SDAR-100M-block4-epoch4-op9-12_score_report.jsonl", "template_stats": {}}} diff --git a/composition/SDAR-100M-block4-epoch4-op9-12/graphs/results.json b/composition/SDAR-100M-block4-epoch4-op9-12/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..67eafe483a6039bec5aa78bd541f20f6e543db0e --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op9-12/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9650793650793651, + "pass@2": 0.9873015873015873, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9610778443113772, + "pass@2": 0.9730538922155688, + "pass@4": 0.9910179640718563, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8727810650887574, + "pass@2": 0.9230769230769231, + "pass@4": 0.9674556213017751, + "pass@8": 0.985207100591716, + "pass@16": 0.9970414201183432, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8141025641025641, + "pass@2": 0.8782051282051282, + "pass@4": 0.9358974358974359, + "pass@8": 0.9871794871794872, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7577464788732394, + "pass@2": 0.8338028169014085, + "pass@4": 0.9042253521126761, + "pass@8": 0.9549295774647887, + "pass@16": 0.9830985915492958, + "pass@32": 0.9915492957746479, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "10": { + "pass@1": 0.6519174041297935, + "pass@2": 0.7404129793510325, + "pass@4": 0.8141592920353983, + "pass@8": 0.8790560471976401, + "pass@16": 0.9262536873156342, + "pass@32": 0.9616519174041298, + "pass@64": 0.9705014749262537, + "pass@128": 0.9734513274336283 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9650793650793651, + "6": 0.9610778443113772, + "7": 0.8727810650887574, + "8": 0.8141025641025641, + "9": 0.7577464788732394, + "10": 0.6519174041297935 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9873015873015873, + "6": 0.9730538922155688, + "7": 0.9230769230769231, + "8": 0.8782051282051282, + "9": 0.8338028169014085, + "10": 0.7404129793510325 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9910179640718563, + "7": 0.9674556213017751, + "8": 0.9358974358974359, + "9": 0.9042253521126761, + "10": 0.8141592920353983 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.985207100591716, + "8": 0.9871794871794872, + "9": 0.9549295774647887, + "10": 0.8790560471976401 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9970414201183432, + "8": 1.0, + "9": 0.9830985915492958, + "10": 0.9262536873156342 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9915492957746479, + "10": 0.9616519174041298 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9705014749262537 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9734513274336283 + } + }, + "total_pass_at_k": { + "pass@1": 0.8888888888888888, + "pass@2": 0.9243498817966903, + "pass@4": 0.955758189800743, + "pass@8": 0.977710233029382, + "pass@16": 0.9891928402566701, + "pass@32": 0.994596420128335, + "pass@64": 0.9962850388382303, + "pass@128": 0.9966227625802094 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 0.9941520467836257, + "pass@4": 0.9970760233918129, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.95, + "pass@2": 0.9764705882352941, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9684542586750788, + "pass@2": 0.9747634069400631, + "pass@4": 0.9842271293375394, + "pass@8": 0.9842271293375394, + "pass@16": 0.9905362776025236, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.879746835443038, + "pass@2": 0.8955696202531646, + "pass@4": 0.9430379746835443, + "pass@8": 0.990506329113924, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8138138138138138, + "pass@2": 0.8888888888888888, + "pass@4": 0.9309309309309309, + "pass@8": 0.963963963963964, + "pass@16": 0.990990990990991, + "pass@32": 0.996996996996997, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.7717041800643086, + "pass@2": 0.8263665594855305, + "pass@4": 0.8906752411575563, + "pass@8": 0.9421221864951769, + "pass@16": 0.9935691318327974, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6029850746268657, + "pass@2": 0.6567164179104478, + "pass@4": 0.755223880597015, + "pass@8": 0.8597014925373134, + "pass@16": 0.9164179104477612, + "pass@32": 0.9432835820895522, + "pass@64": 0.9611940298507463, + "pass@128": 0.9761194029850746 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.95, + "6": 0.9684542586750788, + "7": 0.879746835443038, + "8": 0.8138138138138138, + "9": 0.7717041800643086, + "10": 0.6029850746268657 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9764705882352941, + "6": 0.9747634069400631, + "7": 0.8955696202531646, + "8": 0.8888888888888888, + "9": 0.8263665594855305, + "10": 0.6567164179104478 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 1.0, + "6": 0.9842271293375394, + "7": 0.9430379746835443, + "8": 0.9309309309309309, + "9": 0.8906752411575563, + "10": 0.755223880597015 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9842271293375394, + "7": 0.990506329113924, + "8": 0.963963963963964, + "9": 0.9421221864951769, + "10": 0.8597014925373134 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9905362776025236, + "7": 1.0, + "8": 0.990990990990991, + "9": 0.9935691318327974, + "10": 0.9164179104477612 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9432835820895522 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9611940298507463 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9761194029850746 + } + }, + "total_pass_at_k": { + "pass@1": 0.887847338466689, + "pass@2": 0.9139604954804151, + "pass@4": 0.9454301975225979, + "pass@8": 0.9715433545363241, + "pass@16": 0.9879477736859725, + "pass@32": 0.9933043187144291, + "pass@64": 0.9953130231001004, + "pass@128": 0.9969869434214932 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9792899408284024, + "pass@2": 0.9881656804733728, + "pass@4": 0.9970414201183432, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9623188405797102, + "pass@2": 0.9768115942028985, + "pass@4": 0.991304347826087, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9770773638968482, + "pass@2": 0.9799426934097422, + "pass@4": 0.9885386819484241, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8872832369942196, + "pass@2": 0.9132947976878613, + "pass@4": 0.9682080924855492, + "pass@8": 0.9971098265895953, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8112676056338028, + "pass@2": 0.8845070422535212, + "pass@4": 0.9380281690140845, + "pass@8": 0.9746478873239437, + "pass@16": 0.9887323943661972, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7514970059880239, + "pass@2": 0.8233532934131736, + "pass@4": 0.8862275449101796, + "pass@8": 0.9431137724550899, + "pass@16": 0.9610778443113772, + "pass@32": 0.9910179640718563, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6687116564417178, + "pass@2": 0.7208588957055214, + "pass@4": 0.7822085889570553, + "pass@8": 0.8619631901840491, + "pass@16": 0.9141104294478528, + "pass@32": 0.9386503067484663, + "pass@64": 0.9601226993865031, + "pass@128": 0.9693251533742331 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9792899408284024, + "5": 0.9623188405797102, + "6": 0.9770773638968482, + "7": 0.8872832369942196, + "8": 0.8112676056338028, + "9": 0.7514970059880239, + "10": 0.6687116564417178 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9881656804733728, + "5": 0.9768115942028985, + "6": 0.9799426934097422, + "7": 0.9132947976878613, + "8": 0.8845070422535212, + "9": 0.8233532934131736, + "10": 0.7208588957055214 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.991304347826087, + "6": 0.9885386819484241, + "7": 0.9682080924855492, + "8": 0.9380281690140845, + "9": 0.8862275449101796, + "10": 0.7822085889570553 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9746478873239437, + "9": 0.9431137724550899, + "10": 0.8619631901840491 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9887323943661972, + "9": 0.9610778443113772, + "10": 0.9141104294478528 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9910179640718563, + "10": 0.9386503067484663 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9601226993865031 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9693251533742331 + } + }, + "total_pass_at_k": { + "pass@1": 0.8935124508519003, + "pass@2": 0.9213630406290957, + "pass@4": 0.9508519003931848, + "pass@8": 0.9757536041939712, + "pass@16": 0.9852555701179554, + "pass@32": 0.9921363040629095, + "pass@64": 0.995740498034076, + "pass@128": 0.9967234600262124 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.989, + "pass@2": 0.994, + "pass@4": 0.998, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.959, + "pass@2": 0.98, + "pass@4": 0.997, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.969, + "pass@2": 0.976, + "pass@4": 0.988, + "pass@8": 0.995, + "pass@16": 0.997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.88, + "pass@2": 0.911, + "pass@4": 0.96, + "pass@8": 0.991, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.813, + "pass@2": 0.884, + "pass@4": 0.935, + "pass@8": 0.975, + "pass@16": 0.993, + "pass@32": 0.998, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.76, + "pass@2": 0.828, + "pass@4": 0.894, + "pass@8": 0.947, + "pass@16": 0.979, + "pass@32": 0.994, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "10": { + "pass@1": 0.641, + "pass@2": 0.706, + "pass@4": 0.784, + "pass@8": 0.867, + "pass@16": 0.919, + "pass@32": 0.948, + "pass@64": 0.964, + "pass@128": 0.973 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.989, + "5": 0.959, + "6": 0.969, + "7": 0.88, + "8": 0.813, + "9": 0.76, + "10": 0.641 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.994, + "5": 0.98, + "6": 0.976, + "7": 0.911, + "8": 0.884, + "9": 0.828, + "10": 0.706 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.998, + "5": 0.997, + "6": 0.988, + "7": 0.96, + "8": 0.935, + "9": 0.894, + "10": 0.784 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.995, + "7": 0.991, + "8": 0.975, + "9": 0.947, + "10": 0.867 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997, + "7": 0.999, + "8": 0.993, + "9": 0.979, + "10": 0.919 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.998, + "9": 0.994, + "10": 0.948 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.999, + "10": 0.964 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.999, + "10": 0.973 + } + }, + "total_pass_at_k": { + "pass@1": 0.8901111111111111, + "pass@2": 0.9198888888888889, + "pass@4": 0.9506666666666667, + "pass@8": 0.975, + "pass@16": 0.9874444444444445, + "pass@32": 0.9933333333333333, + "pass@64": 0.9957777777777778, + "pass@128": 0.9967777777777778 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5779036827195467, + "pass@2": 0.6260623229461756, + "pass@4": 0.71671388101983, + "pass@8": 0.7960339943342776, + "pass@16": 0.8696883852691218, + "pass@32": 0.9235127478753541, + "pass@64": 0.9405099150141643, + "pass@128": 0.9490084985835694 + }, + "12": { + "pass@1": 0.6016949152542372, + "pass@2": 0.6468926553672316, + "pass@4": 0.7231638418079096, + "pass@8": 0.7853107344632768, + "pass@16": 0.827683615819209, + "pass@32": 0.8531073446327684, + "pass@64": 0.8615819209039548, + "pass@128": 0.8700564971751412 + }, + "13": { + "pass@1": 0.5078369905956113, + "pass@2": 0.5517241379310345, + "pass@4": 0.5987460815047022, + "pass@8": 0.670846394984326, + "pass@16": 0.7429467084639498, + "pass@32": 0.7774294670846394, + "pass@64": 0.8056426332288401, + "pass@128": 0.8244514106583072 + }, + "14": { + "pass@1": 0.45794392523364486, + "pass@2": 0.4984423676012461, + "pass@4": 0.5482866043613707, + "pass@8": 0.6105919003115264, + "pass@16": 0.67601246105919, + "pass@32": 0.719626168224299, + "pass@64": 0.7694704049844237, + "pass@128": 0.8037383177570093 + }, + "15": { + "pass@1": 0.3439306358381503, + "pass@2": 0.40173410404624277, + "pass@4": 0.4421965317919075, + "pass@8": 0.48265895953757226, + "pass@16": 0.569364161849711, + "pass@32": 0.6473988439306358, + "pass@64": 0.7109826589595376, + "pass@128": 0.7601156069364162 + }, + "16": { + "pass@1": 0.22258064516129034, + "pass@2": 0.2903225806451613, + "pass@4": 0.34838709677419355, + "pass@8": 0.3903225806451613, + "pass@16": 0.44516129032258067, + "pass@32": 0.5064516129032258, + "pass@64": 0.5290322580645161, + "pass@128": 0.5741935483870968 + }, + "17": { + "pass@1": 0.13872832369942195, + "pass@2": 0.20520231213872833, + "pass@4": 0.2774566473988439, + "pass@8": 0.33236994219653176, + "pass@16": 0.37283236994219654, + "pass@32": 0.430635838150289, + "pass@64": 0.48265895953757226, + "pass@128": 0.5289017341040463 + }, + "18": { + "pass@1": 0.05714285714285714, + "pass@2": 0.09142857142857143, + "pass@4": 0.13142857142857142, + "pass@8": 0.1742857142857143, + "pass@16": 0.20285714285714285, + "pass@32": 0.24571428571428572, + "pass@64": 0.2914285714285714, + "pass@128": 0.32 + }, + "19": { + "pass@1": 0.012012012012012012, + "pass@2": 0.03303303303303303, + "pass@4": 0.06006006006006006, + "pass@8": 0.08408408408408409, + "pass@16": 0.0990990990990991, + "pass@32": 0.12612612612612611, + "pass@64": 0.13513513513513514, + "pass@128": 0.14714714714714713 + }, + "20": { + "pass@1": 0.03003003003003003, + "pass@2": 0.042042042042042045, + "pass@4": 0.057057057057057055, + "pass@8": 0.07807807807807808, + "pass@16": 0.08408408408408409, + "pass@32": 0.08708708708708708, + "pass@64": 0.1111111111111111, + "pass@128": 0.12012012012012012 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5779036827195467, + "12": 0.6016949152542372, + "13": 0.5078369905956113, + "14": 0.45794392523364486, + "15": 0.3439306358381503, + "16": 0.22258064516129034, + "17": 0.13872832369942195, + "18": 0.05714285714285714, + "19": 0.012012012012012012, + "20": 0.03003003003003003 + }, + "pass@2": { + "11": 0.6260623229461756, + "12": 0.6468926553672316, + "13": 0.5517241379310345, + "14": 0.4984423676012461, + "15": 0.40173410404624277, + "16": 0.2903225806451613, + "17": 0.20520231213872833, + "18": 0.09142857142857143, + "19": 0.03303303303303303, + "20": 0.042042042042042045 + }, + "pass@4": { + "11": 0.71671388101983, + "12": 0.7231638418079096, + "13": 0.5987460815047022, + "14": 0.5482866043613707, + "15": 0.4421965317919075, + "16": 0.34838709677419355, + "17": 0.2774566473988439, + "18": 0.13142857142857142, + "19": 0.06006006006006006, + "20": 0.057057057057057055 + }, + "pass@8": { + "11": 0.7960339943342776, + "12": 0.7853107344632768, + "13": 0.670846394984326, + "14": 0.6105919003115264, + "15": 0.48265895953757226, + "16": 0.3903225806451613, + "17": 0.33236994219653176, + "18": 0.1742857142857143, + "19": 0.08408408408408409, + "20": 0.07807807807807808 + }, + "pass@16": { + "11": 0.8696883852691218, + "12": 0.827683615819209, + "13": 0.7429467084639498, + "14": 0.67601246105919, + "15": 0.569364161849711, + "16": 0.44516129032258067, + "17": 0.37283236994219654, + "18": 0.20285714285714285, + "19": 0.0990990990990991, + "20": 0.08408408408408409 + }, + "pass@32": { + "11": 0.9235127478753541, + "12": 0.8531073446327684, + "13": 0.7774294670846394, + "14": 0.719626168224299, + "15": 0.6473988439306358, + "16": 0.5064516129032258, + "17": 0.430635838150289, + "18": 0.24571428571428572, + "19": 0.12612612612612611, + "20": 0.08708708708708708 + }, + "pass@64": { + "11": 0.9405099150141643, + "12": 0.8615819209039548, + "13": 0.8056426332288401, + "14": 0.7694704049844237, + "15": 0.7109826589595376, + "16": 0.5290322580645161, + "17": 0.48265895953757226, + "18": 0.2914285714285714, + "19": 0.13513513513513514, + "20": 0.1111111111111111 + }, + "pass@128": { + "11": 0.9490084985835694, + "12": 0.8700564971751412, + "13": 0.8244514106583072, + "14": 0.8037383177570093, + "15": 0.7601156069364162, + "16": 0.5741935483870968, + "17": 0.5289017341040463, + "18": 0.32, + "19": 0.14714714714714713, + "20": 0.12012012012012012 + } + }, + "total_pass_at_k": { + "pass@1": 0.2959881129271917, + "pass@2": 0.3396731054977712, + "pass@4": 0.39167904903417533, + "pass@8": 0.44190193164933134, + "pass@16": 0.4903417533432392, + "pass@32": 0.5331352154531946, + "pass@64": 0.5652303120356612, + "pass@128": 0.5910846953937593 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5904761904761905, + "pass@2": 0.653968253968254, + "pass@4": 0.7301587301587301, + "pass@8": 0.8253968253968254, + "pass@16": 0.9301587301587302, + "pass@32": 0.9587301587301588, + "pass@64": 0.9777777777777777, + "pass@128": 0.9809523809523809 + }, + "12": { + "pass@1": 0.581039755351682, + "pass@2": 0.6391437308868502, + "pass@4": 0.7247706422018348, + "pass@8": 0.7798165137614679, + "pass@16": 0.8654434250764526, + "pass@32": 0.8868501529051988, + "pass@64": 0.908256880733945, + "pass@128": 0.9296636085626911 + }, + "13": { + "pass@1": 0.5216049382716049, + "pass@2": 0.5740740740740741, + "pass@4": 0.6172839506172839, + "pass@8": 0.691358024691358, + "pass@16": 0.7407407407407407, + "pass@32": 0.7654320987654321, + "pass@64": 0.7870370370370371, + "pass@128": 0.8024691358024691 + }, + "14": { + "pass@1": 0.4124629080118694, + "pass@2": 0.456973293768546, + "pass@4": 0.5341246290801187, + "pass@8": 0.5608308605341247, + "pass@16": 0.6261127596439169, + "pass@32": 0.7240356083086054, + "pass@64": 0.7685459940652819, + "pass@128": 0.8189910979228486 + }, + "15": { + "pass@1": 0.3117283950617284, + "pass@2": 0.3734567901234568, + "pass@4": 0.42592592592592593, + "pass@8": 0.47530864197530864, + "pass@16": 0.5432098765432098, + "pass@32": 0.6141975308641975, + "pass@64": 0.6882716049382716, + "pass@128": 0.7283950617283951 + }, + "16": { + "pass@1": 0.23008849557522124, + "pass@2": 0.2920353982300885, + "pass@4": 0.3303834808259587, + "pass@8": 0.41002949852507375, + "pass@16": 0.45722713864306785, + "pass@32": 0.5014749262536873, + "pass@64": 0.5221238938053098, + "pass@128": 0.5722713864306784 + }, + "17": { + "pass@1": 0.13855421686746988, + "pass@2": 0.19578313253012047, + "pass@4": 0.22590361445783133, + "pass@8": 0.2620481927710843, + "pass@16": 0.286144578313253, + "pass@32": 0.32831325301204817, + "pass@64": 0.37349397590361444, + "pass@128": 0.42168674698795183 + }, + "18": { + "pass@1": 0.08709677419354839, + "pass@2": 0.12903225806451613, + "pass@4": 0.17419354838709677, + "pass@8": 0.18387096774193548, + "pass@16": 0.2161290322580645, + "pass@32": 0.25161290322580643, + "pass@64": 0.2838709677419355, + "pass@128": 0.3161290322580645 + }, + "19": { + "pass@1": 0.02416918429003021, + "pass@2": 0.04229607250755287, + "pass@4": 0.0513595166163142, + "pass@8": 0.07250755287009064, + "pass@16": 0.08157099697885196, + "pass@32": 0.09969788519637462, + "pass@64": 0.11782477341389729, + "pass@128": 0.13293051359516617 + }, + "20": { + "pass@1": 0.018018018018018018, + "pass@2": 0.03003003003003003, + "pass@4": 0.042042042042042045, + "pass@8": 0.06006006006006006, + "pass@16": 0.07507507507507508, + "pass@32": 0.08108108108108109, + "pass@64": 0.08708708708708708, + "pass@128": 0.09009009009009009 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5904761904761905, + "12": 0.581039755351682, + "13": 0.5216049382716049, + "14": 0.4124629080118694, + "15": 0.3117283950617284, + "16": 0.23008849557522124, + "17": 0.13855421686746988, + "18": 0.08709677419354839, + "19": 0.02416918429003021, + "20": 0.018018018018018018 + }, + "pass@2": { + "11": 0.653968253968254, + "12": 0.6391437308868502, + "13": 0.5740740740740741, + "14": 0.456973293768546, + "15": 0.3734567901234568, + "16": 0.2920353982300885, + "17": 0.19578313253012047, + "18": 0.12903225806451613, + "19": 0.04229607250755287, + "20": 0.03003003003003003 + }, + "pass@4": { + "11": 0.7301587301587301, + "12": 0.7247706422018348, + "13": 0.6172839506172839, + "14": 0.5341246290801187, + "15": 0.42592592592592593, + "16": 0.3303834808259587, + "17": 0.22590361445783133, + "18": 0.17419354838709677, + "19": 0.0513595166163142, + "20": 0.042042042042042045 + }, + "pass@8": { + "11": 0.8253968253968254, + "12": 0.7798165137614679, + "13": 0.691358024691358, + "14": 0.5608308605341247, + "15": 0.47530864197530864, + "16": 0.41002949852507375, + "17": 0.2620481927710843, + "18": 0.18387096774193548, + "19": 0.07250755287009064, + "20": 0.06006006006006006 + }, + "pass@16": { + "11": 0.9301587301587302, + "12": 0.8654434250764526, + "13": 0.7407407407407407, + "14": 0.6261127596439169, + "15": 0.5432098765432098, + "16": 0.45722713864306785, + "17": 0.286144578313253, + "18": 0.2161290322580645, + "19": 0.08157099697885196, + "20": 0.07507507507507508 + }, + "pass@32": { + "11": 0.9587301587301588, + "12": 0.8868501529051988, + "13": 0.7654320987654321, + "14": 0.7240356083086054, + "15": 0.6141975308641975, + "16": 0.5014749262536873, + "17": 0.32831325301204817, + "18": 0.25161290322580643, + "19": 0.09969788519637462, + "20": 0.08108108108108109 + }, + "pass@64": { + "11": 0.9777777777777777, + "12": 0.908256880733945, + "13": 0.7870370370370371, + "14": 0.7685459940652819, + "15": 0.6882716049382716, + "16": 0.5221238938053098, + "17": 0.37349397590361444, + "18": 0.2838709677419355, + "19": 0.11782477341389729, + "20": 0.08708708708708708 + }, + "pass@128": { + "11": 0.9809523809523809, + "12": 0.9296636085626911, + "13": 0.8024691358024691, + "14": 0.8189910979228486, + "15": 0.7283950617283951, + "16": 0.5722713864306784, + "17": 0.42168674698795183, + "18": 0.3161290322580645, + "19": 0.13293051359516617, + "20": 0.09009009009009009 + } + }, + "total_pass_at_k": { + "pass@1": 0.29034229828850855, + "pass@2": 0.3374083129584352, + "pass@4": 0.3841687041564792, + "pass@8": 0.4306234718826406, + "pass@16": 0.480440097799511, + "pass@32": 0.519559902200489, + "pass@64": 0.5498166259168704, + "pass@128": 0.5779339853300733 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6475903614457831, + "pass@2": 0.6957831325301205, + "pass@4": 0.7710843373493976, + "pass@8": 0.8283132530120482, + "pass@16": 0.9096385542168675, + "pass@32": 0.9397590361445783, + "pass@64": 0.9608433734939759, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.5768025078369906, + "pass@2": 0.6363636363636364, + "pass@4": 0.7084639498432602, + "pass@8": 0.7711598746081505, + "pass@16": 0.8338557993730408, + "pass@32": 0.8714733542319749, + "pass@64": 0.8808777429467085, + "pass@128": 0.8871473354231975 + }, + "13": { + "pass@1": 0.5294117647058824, + "pass@2": 0.5882352941176471, + "pass@4": 0.6526610644257703, + "pass@8": 0.7282913165266106, + "pass@16": 0.803921568627451, + "pass@32": 0.8375350140056023, + "pass@64": 0.8571428571428571, + "pass@128": 0.8683473389355743 + }, + "14": { + "pass@1": 0.4327485380116959, + "pass@2": 0.4853801169590643, + "pass@4": 0.564327485380117, + "pass@8": 0.631578947368421, + "pass@16": 0.6929824561403509, + "pass@32": 0.7631578947368421, + "pass@64": 0.8070175438596491, + "pass@128": 0.847953216374269 + }, + "15": { + "pass@1": 0.3515151515151515, + "pass@2": 0.4303030303030303, + "pass@4": 0.48787878787878786, + "pass@8": 0.5393939393939394, + "pass@16": 0.603030303030303, + "pass@32": 0.6575757575757576, + "pass@64": 0.7181818181818181, + "pass@128": 0.7757575757575758 + }, + "16": { + "pass@1": 0.245014245014245, + "pass@2": 0.2905982905982906, + "pass@4": 0.358974358974359, + "pass@8": 0.4188034188034188, + "pass@16": 0.4700854700854701, + "pass@32": 0.5356125356125356, + "pass@64": 0.5811965811965812, + "pass@128": 0.6239316239316239 + }, + "17": { + "pass@1": 0.13664596273291926, + "pass@2": 0.20496894409937888, + "pass@4": 0.2422360248447205, + "pass@8": 0.2732919254658385, + "pass@16": 0.3447204968944099, + "pass@32": 0.38819875776397517, + "pass@64": 0.4440993788819876, + "pass@128": 0.5 + }, + "18": { + "pass@1": 0.08823529411764706, + "pass@2": 0.1264705882352941, + "pass@4": 0.14705882352941177, + "pass@8": 0.18235294117647058, + "pass@16": 0.21764705882352942, + "pass@32": 0.25, + "pass@64": 0.2852941176470588, + "pass@128": 0.3176470588235294 + }, + "19": { + "pass@1": 0.03571428571428571, + "pass@2": 0.05357142857142857, + "pass@4": 0.07142857142857142, + "pass@8": 0.09821428571428571, + "pass@16": 0.12797619047619047, + "pass@32": 0.1636904761904762, + "pass@64": 0.17261904761904762, + "pass@128": 0.2113095238095238 + }, + "20": { + "pass@1": 0.014970059880239521, + "pass@2": 0.03293413173652695, + "pass@4": 0.04491017964071856, + "pass@8": 0.05389221556886228, + "pass@16": 0.0748502994011976, + "pass@32": 0.10179640718562874, + "pass@64": 0.11377245508982035, + "pass@128": 0.11976047904191617 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6475903614457831, + "12": 0.5768025078369906, + "13": 0.5294117647058824, + "14": 0.4327485380116959, + "15": 0.3515151515151515, + "16": 0.245014245014245, + "17": 0.13664596273291926, + "18": 0.08823529411764706, + "19": 0.03571428571428571, + "20": 0.014970059880239521 + }, + "pass@2": { + "11": 0.6957831325301205, + "12": 0.6363636363636364, + "13": 0.5882352941176471, + "14": 0.4853801169590643, + "15": 0.4303030303030303, + "16": 0.2905982905982906, + "17": 0.20496894409937888, + "18": 0.1264705882352941, + "19": 0.05357142857142857, + "20": 0.03293413173652695 + }, + "pass@4": { + "11": 0.7710843373493976, + "12": 0.7084639498432602, + "13": 0.6526610644257703, + "14": 0.564327485380117, + "15": 0.48787878787878786, + "16": 0.358974358974359, + "17": 0.2422360248447205, + "18": 0.14705882352941177, + "19": 0.07142857142857142, + "20": 0.04491017964071856 + }, + "pass@8": { + "11": 0.8283132530120482, + "12": 0.7711598746081505, + "13": 0.7282913165266106, + "14": 0.631578947368421, + "15": 0.5393939393939394, + "16": 0.4188034188034188, + "17": 0.2732919254658385, + "18": 0.18235294117647058, + "19": 0.09821428571428571, + "20": 0.05389221556886228 + }, + "pass@16": { + "11": 0.9096385542168675, + "12": 0.8338557993730408, + "13": 0.803921568627451, + "14": 0.6929824561403509, + "15": 0.603030303030303, + "16": 0.4700854700854701, + "17": 0.3447204968944099, + "18": 0.21764705882352942, + "19": 0.12797619047619047, + "20": 0.0748502994011976 + }, + "pass@32": { + "11": 0.9397590361445783, + "12": 0.8714733542319749, + "13": 0.8375350140056023, + "14": 0.7631578947368421, + "15": 0.6575757575757576, + "16": 0.5356125356125356, + "17": 0.38819875776397517, + "18": 0.25, + "19": 0.1636904761904762, + "20": 0.10179640718562874 + }, + "pass@64": { + "11": 0.9608433734939759, + "12": 0.8808777429467085, + "13": 0.8571428571428571, + "14": 0.8070175438596491, + "15": 0.7181818181818181, + "16": 0.5811965811965812, + "17": 0.4440993788819876, + "18": 0.2852941176470588, + "19": 0.17261904761904762, + "20": 0.11377245508982035 + }, + "pass@128": { + "11": 0.9698795180722891, + "12": 0.8871473354231975, + "13": 0.8683473389355743, + "14": 0.847953216374269, + "15": 0.7757575757575758, + "16": 0.6239316239316239, + "17": 0.5, + "18": 0.3176470588235294, + "19": 0.2113095238095238, + "20": 0.11976047904191617 + } + }, + "total_pass_at_k": { + "pass@1": 0.30597680642283676, + "pass@2": 0.3544454356229557, + "pass@4": 0.4049955396966994, + "pass@8": 0.4528694617900684, + "pass@16": 0.5081772227178115, + "pass@32": 0.5512934879571811, + "pass@64": 0.5825156110615521, + "pass@128": 0.6125483199524234 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.605, + "pass@2": 0.658, + "pass@4": 0.739, + "pass@8": 0.816, + "pass@16": 0.902, + "pass@32": 0.94, + "pass@64": 0.959, + "pass@128": 0.966 + }, + "12": { + "pass@1": 0.587, + "pass@2": 0.641, + "pass@4": 0.719, + "pass@8": 0.779, + "pass@16": 0.842, + "pass@32": 0.87, + "pass@64": 0.883, + "pass@128": 0.895 + }, + "13": { + "pass@1": 0.52, + "pass@2": 0.572, + "pass@4": 0.624, + "pass@8": 0.698, + "pass@16": 0.764, + "pass@32": 0.795, + "pass@64": 0.818, + "pass@128": 0.833 + }, + "14": { + "pass@1": 0.434, + "pass@2": 0.48, + "pass@4": 0.549, + "pass@8": 0.601, + "pass@16": 0.665, + "pass@32": 0.736, + "pass@64": 0.782, + "pass@128": 0.824 + }, + "15": { + "pass@1": 0.336, + "pass@2": 0.402, + "pass@4": 0.452, + "pass@8": 0.499, + "pass@16": 0.572, + "pass@32": 0.64, + "pass@64": 0.706, + "pass@128": 0.755 + }, + "16": { + "pass@1": 0.233, + "pass@2": 0.291, + "pass@4": 0.346, + "pass@8": 0.407, + "pass@16": 0.458, + "pass@32": 0.515, + "pass@64": 0.545, + "pass@128": 0.591 + }, + "17": { + "pass@1": 0.138, + "pass@2": 0.202, + "pass@4": 0.249, + "pass@8": 0.29, + "pass@16": 0.335, + "pass@32": 0.383, + "pass@64": 0.434, + "pass@128": 0.484 + }, + "18": { + "pass@1": 0.077, + "pass@2": 0.115, + "pass@4": 0.15, + "pass@8": 0.18, + "pass@16": 0.212, + "pass@32": 0.249, + "pass@64": 0.287, + "pass@128": 0.318 + }, + "19": { + "pass@1": 0.024, + "pass@2": 0.043, + "pass@4": 0.061, + "pass@8": 0.085, + "pass@16": 0.103, + "pass@32": 0.13, + "pass@64": 0.142, + "pass@128": 0.164 + }, + "20": { + "pass@1": 0.021, + "pass@2": 0.035, + "pass@4": 0.048, + "pass@8": 0.064, + "pass@16": 0.078, + "pass@32": 0.09, + "pass@64": 0.104, + "pass@128": 0.11 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.605, + "12": 0.587, + "13": 0.52, + "14": 0.434, + "15": 0.336, + "16": 0.233, + "17": 0.138, + "18": 0.077, + "19": 0.024, + "20": 0.021 + }, + "pass@2": { + "11": 0.658, + "12": 0.641, + "13": 0.572, + "14": 0.48, + "15": 0.402, + "16": 0.291, + "17": 0.202, + "18": 0.115, + "19": 0.043, + "20": 0.035 + }, + "pass@4": { + "11": 0.739, + "12": 0.719, + "13": 0.624, + "14": 0.549, + "15": 0.452, + "16": 0.346, + "17": 0.249, + "18": 0.15, + "19": 0.061, + "20": 0.048 + }, + "pass@8": { + "11": 0.816, + "12": 0.779, + "13": 0.698, + "14": 0.601, + "15": 0.499, + "16": 0.407, + "17": 0.29, + "18": 0.18, + "19": 0.085, + "20": 0.064 + }, + "pass@16": { + "11": 0.902, + "12": 0.842, + "13": 0.764, + "14": 0.665, + "15": 0.572, + "16": 0.458, + "17": 0.335, + "18": 0.212, + "19": 0.103, + "20": 0.078 + }, + "pass@32": { + "11": 0.94, + "12": 0.87, + "13": 0.795, + "14": 0.736, + "15": 0.64, + "16": 0.515, + "17": 0.383, + "18": 0.249, + "19": 0.13, + "20": 0.09 + }, + "pass@64": { + "11": 0.959, + "12": 0.883, + "13": 0.818, + "14": 0.782, + "15": 0.706, + "16": 0.545, + "17": 0.434, + "18": 0.287, + "19": 0.142, + "20": 0.104 + }, + "pass@128": { + "11": 0.966, + "12": 0.895, + "13": 0.833, + "14": 0.824, + "15": 0.755, + "16": 0.591, + "17": 0.484, + "18": 0.318, + "19": 0.164, + "20": 0.11 + } + }, + "total_pass_at_k": { + "pass@1": 0.2975, + "pass@2": 0.3439, + "pass@4": 0.3937, + "pass@8": 0.4419, + "pass@16": 0.4931, + "pass@32": 0.5348, + "pass@64": 0.566, + "pass@128": 0.594 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block4-epoch4-op9-12/summary.csv b/composition/SDAR-100M-block4-epoch4-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..e054e206bb958c05e4a1224d56dbf479e6b9c888 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block4-epoch4-op9-12,0.9942543402777778,0.58941953125,0.7811833881578948,0.0,0.0,0.0,200.17993142361112,358.49741640625,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block4-epoch4-op9-12/summary.json b/composition/SDAR-100M-block4-epoch4-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7247a9069949aedd1b273edf2ffcc9a5072d8a22 --- /dev/null +++ b/composition/SDAR-100M-block4-epoch4-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block4-epoch4-op9-12", + "id_acc": 0.9942543402777778, + "ood_acc": 0.58941953125, + "total_acc": 0.7811833881578948, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.17993142361112, + "ood_avg_resp_len": 358.49741640625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 631740, + "answer_accuracy": 0.7801879149541574, + "avg_response_len": 270.626141123933, + "resp_tokens_sum": 219133564, + "pass_at_k": { + "pass@1": 0.7801879149541574, + "pass@2": 0.8187593002942418, + "pass@4": 0.8505263494815603, + "pass@8": 0.8756524395376909, + "pass@16": 0.8946483201635945, + "pass@32": 0.9095644843170411, + "pass@64": 0.9222483474545463, + "pass@128": 0.9329750237116662 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9659845132743363, + "pass@2": 0.9762676236266927, + "pass@4": 0.9840518931261909, + "pass@8": 0.9893819037909549, + "pass@16": 0.992827041455884, + "pass@32": 0.9946400636162834, + "pass@64": 0.9955741007118146, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9989990234375, + "pass@2": 0.9998585137795276, + "pass@4": 0.9999955834622234, + "pass@8": 0.9999999965852078, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9962053571428572, + "pass@2": 0.9975288245219347, + "pass@4": 0.998544181679671, + "pass@8": 0.9993806306839305, + "pass@16": 0.9998935171160672, + "pass@32": 0.9999980250035232, + "pass@64": 0.9999999999662323, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 0.9999996316422274, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9979890902366864, + "pass@2": 0.9993728305688859, + "pass@4": 0.9999376890018925, + "pass@8": 0.9999995021997173, + "pass@16": 0.9999999999883278, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9989983974358975, + "pass@2": 0.9999191619977791, + "pass@4": 0.9999993645626026, + "pass@8": 0.9999999999826888, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.996875, + "pass@2": 0.9971789397804147, + "pass@4": 0.9971830980634462, + "pass@8": 0.9971830985915493, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9477912535410765, + "pass@2": 0.9614141163480627, + "pass@4": 0.971306078861247, + "pass@8": 0.9777222153354097, + "pass@16": 0.9820192850495909, + "pass@32": 0.9859081102502127, + "pass@64": 0.9893626160626768, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.9022775423728814, + "pass@2": 0.9216494422572181, + "pass@4": 0.9329050222006143, + "pass@8": 0.9403578890122049, + "pass@16": 0.9464571379224949, + "pass@32": 0.9536009096454778, + "pass@64": 0.9612807038635403, + "pass@128": 0.9661016949152542 + }, + "13": { + "pass@1": 0.8387539184952978, + "pass@2": 0.883299805000864, + "pass@4": 0.9113433348260935, + "pass@8": 0.9295898515647626, + "pass@16": 0.943287279603375, + "pass@32": 0.9551149405367227, + "pass@64": 0.9642286009171162, + "pass@128": 0.9686520376175548 + }, + "14": { + "pass@1": 0.8147390965732088, + "pass@2": 0.8667948340569573, + "pass@4": 0.8942816747789692, + "pass@8": 0.9112592074701689, + "pass@16": 0.9250196885538422, + "pass@32": 0.9384068954202733, + "pass@64": 0.9499602716149194, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.7185016257225434, + "pass@2": 0.7971889792226117, + "pass@4": 0.847934874108944, + "pass@8": 0.8830414368074523, + "pass@16": 0.9085139235206722, + "pass@32": 0.9277406485745909, + "pass@64": 0.9418027604093345, + "pass@128": 0.953757225433526 + }, + "16": { + "pass@1": 0.5365171370967742, + "pass@2": 0.6455339566929132, + "pass@4": 0.7244478673028772, + "pass@8": 0.779777220522581, + "pass@16": 0.8186884395276981, + "pass@32": 0.8485246873326461, + "pass@64": 0.8727635997646052, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.420497651734104, + "pass@2": 0.5373062787310544, + "pass@4": 0.6400003012629196, + "pass@8": 0.7204827202394027, + "pass@16": 0.7750126815064169, + "pass@32": 0.8111233885916674, + "pass@64": 0.8380741227154171, + "pass@128": 0.8583815028901735 + }, + "18": { + "pass@1": 0.30125, + "pass@2": 0.404013990438695, + "pass@4": 0.4990766556858963, + "pass@8": 0.5797528146962918, + "pass@16": 0.6458841179962546, + "pass@32": 0.7010997597155946, + "pass@64": 0.7502144482870602, + "pass@128": 0.7971428571428572 + }, + "19": { + "pass@1": 0.21410472972972974, + "pass@2": 0.30520899836844734, + "pass@4": 0.40122690744738015, + "pass@8": 0.4897382851155873, + "pass@16": 0.5619964012938833, + "pass@32": 0.6181259558492432, + "pass@64": 0.6659910429721136, + "pass@128": 0.7117117117117117 + }, + "20": { + "pass@1": 0.20845063813813813, + "pass@2": 0.29438998545789885, + "pass@4": 0.38355911817329136, + "pass@8": 0.46110962180572834, + "pass@16": 0.5194414030776506, + "pass@32": 0.5654700745427053, + "pass@64": 0.6091131851339564, + "pass@128": 0.6516516516516516 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 645802, + "answer_accuracy": 0.786489185502728, + "avg_response_len": 280.1511459957132, + "resp_tokens_sum": 230037709, + "pass_at_k": { + "pass@1": 0.786489185502728, + "pass@2": 0.8250239350439653, + "pass@4": 0.8566651189402132, + "pass@8": 0.8818527226648039, + "pass@16": 0.9015738361825739, + "pass@32": 0.9170439507084295, + "pass@64": 0.9294757850018294, + "pass@128": 0.9395167575993765 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9602664877300614, + "pass@2": 0.973362926307908, + "pass@4": 0.9839911745119283, + "pass@8": 0.9905991426701357, + "pass@16": 0.9937449653260887, + "pass@32": 0.9958896624995128, + "pass@64": 0.9980494756223309, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999705615942029, + "pass@2": 0.9999803862832364, + "pass@4": 0.9999999103372947, + "pass@8": 0.9999999999996655, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987464183381088, + "pass@2": 0.9995321982943393, + "pass@4": 0.9999272856831292, + "pass@8": 0.9999984918054343, + "pass@16": 0.9999999996818834, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9992322976878613, + "pass@2": 0.9998545668153475, + "pass@4": 0.9999935653816394, + "pass@8": 0.9999999913233404, + "pass@16": 0.9999999999999979, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9953125, + "pass@2": 0.9974374792059445, + "pass@4": 0.9986580279577728, + "pass@8": 0.999449156942593, + "pass@16": 0.9999055148797769, + "pass@32": 0.9999982475383377, + "pass@64": 0.999999999970037, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9971570147107359, + "pass@4": 0.9978299348996046, + "pass@8": 0.9984680331236191, + "pass@16": 0.9992530482904565, + "pass@32": 0.999851122735485, + "pass@64": 0.9999980007921191, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9688911897590361, + "pass@2": 0.9749675374727257, + "pass@4": 0.9774471477021697, + "pass@8": 0.9784483556651219, + "pass@16": 0.9788544566955297, + "pass@32": 0.978915040933753, + "pass@64": 0.9789156626487805, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.9082827194357367, + "pass@2": 0.9269358755214379, + "pass@4": 0.9375423858655597, + "pass@8": 0.9452183170723136, + "pass@16": 0.9520156669843709, + "pass@32": 0.958861937474049, + "pass@64": 0.9660584025362567, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.8396358543417367, + "pass@2": 0.8762106712763842, + "pass@4": 0.8982317819516257, + "pass@8": 0.9130115564102529, + "pass@16": 0.9240476363783747, + "pass@32": 0.9335026128044098, + "pass@64": 0.9420415640994138, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.8277366593567251, + "pass@2": 0.8849975681493761, + "pass@4": 0.9186248646002576, + "pass@8": 0.939483467623284, + "pass@16": 0.953015240904691, + "pass@32": 0.9622001675120875, + "pass@64": 0.968673485702234, + "pass@128": 0.9736842105263158 + }, + "15": { + "pass@1": 0.7206202651515151, + "pass@2": 0.8023443092340725, + "pass@4": 0.8544347351467427, + "pass@8": 0.8898100023410217, + "pass@16": 0.9154028263055654, + "pass@32": 0.9328111580456316, + "pass@64": 0.9438559391462911, + "pass@128": 0.9515151515151515 + }, + "16": { + "pass@1": 0.5157140313390314, + "pass@2": 0.6320804450725708, + "pass@4": 0.7233764438633206, + "pass@8": 0.7903596603735945, + "pass@16": 0.8410317719344832, + "pass@32": 0.8811910225527382, + "pass@64": 0.9109104486196058, + "pass@128": 0.9316239316239316 + }, + "17": { + "pass@1": 0.45545419254658387, + "pass@2": 0.5785563744070035, + "pass@4": 0.6760754613553737, + "pass@8": 0.7431020871647102, + "pass@16": 0.7875528862860809, + "pass@32": 0.8186175463395166, + "pass@64": 0.8449973543887125, + "pass@128": 0.8695652173913043 + }, + "18": { + "pass@1": 0.3417279411764706, + "pass@2": 0.44516594777674856, + "pass@4": 0.5401025474756829, + "pass@8": 0.6218323060002307, + "pass@16": 0.6893669338501651, + "pass@32": 0.7441713183922153, + "pass@64": 0.7855624371543468, + "pass@128": 0.8147058823529412 + }, + "19": { + "pass@1": 0.23056175595238096, + "pass@2": 0.3243000386670416, + "pass@4": 0.42189163491840287, + "pass@8": 0.5118960174363878, + "pass@16": 0.5878588842096898, + "pass@32": 0.6511562770502672, + "pass@64": 0.7078413277014779, + "pass@128": 0.7559523809523809 + }, + "20": { + "pass@1": 0.17009730538922155, + "pass@2": 0.2490522154509878, + "pass@4": 0.3370424524965817, + "pass@8": 0.42361443906161345, + "pass@16": 0.4989430300289183, + "pass@32": 0.5583541344634607, + "pass@64": 0.6055054141936476, + "pass@128": 0.6467065868263473 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 622296, + "answer_accuracy": 0.7767514778718645, + "avg_response_len": 299.9589353830484, + "resp_tokens_sum": 240312701, + "pass_at_k": { + "pass@1": 0.7767514778718645, + "pass@2": 0.8156860420207447, + "pass@4": 0.848207839623572, + "pass@8": 0.8745128108264176, + "pass@16": 0.8945934721469555, + "pass@32": 0.9095033826804982, + "pass@64": 0.9209329108650951, + "pass@128": 0.9303403099536667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.965625, + "pass@2": 0.9748288576801035, + "pass@4": 0.9819707584126611, + "pass@8": 0.9875054318482567, + "pass@16": 0.9914189594286545, + "pass@32": 0.9939477801119064, + "pass@64": 0.9954718430329116, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9940716911764705, + "pass@2": 0.9974872626215842, + "pass@4": 0.9994001792055407, + "pass@8": 0.9999587340911859, + "pass@16": 0.9999998237655816, + "pass@32": 0.9999999999993819, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990881309148265, + "pass@2": 0.9997415174246752, + "pass@4": 0.9999804702054199, + "pass@8": 0.9999999148131528, + "pass@16": 0.999999999999565, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.998046875, + "pass@2": 0.999243901375461, + "pass@4": 0.999834458192726, + "pass@8": 0.9999925161507783, + "pass@16": 0.9999999907330216, + "pass@32": 0.9999999999999991, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9884806681681682, + "pass@2": 0.9912943208956989, + "pass@4": 0.9943561204624196, + "pass@8": 0.9973895114319368, + "pass@16": 0.99940630260805, + "pass@32": 0.999972591246004, + "pass@64": 0.9999999903997061, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954782958199357, + "pass@2": 0.9977260931209967, + "pass@4": 0.9992145500502148, + "pass@8": 0.9998966223073351, + "pass@16": 0.999998180165534, + "pass@32": 0.9999999997896366, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.959920634920635, + "pass@2": 0.9700935820522439, + "pass@4": 0.9782513129311214, + "pass@8": 0.9843949172016934, + "pass@16": 0.9880157317975031, + "pass@32": 0.9896987221949869, + "pass@64": 0.9903846779515253, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9339162844036697, + "pass@2": 0.953070068506345, + "pass@4": 0.9651293628434054, + "pass@8": 0.9734955690809288, + "pass@16": 0.9796414698855651, + "pass@32": 0.9849824737145875, + "pass@64": 0.9892393317419794, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.8035542052469136, + "pass@2": 0.8526401647710703, + "pass@4": 0.8829088207145402, + "pass@8": 0.903255503160504, + "pass@16": 0.9182217722619813, + "pass@32": 0.9298422512548313, + "pass@64": 0.93809924782445, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.7945335682492581, + "pass@2": 0.856050594055936, + "pass@4": 0.8911655095412775, + "pass@8": 0.910886069661382, + "pass@16": 0.9221446466456787, + "pass@32": 0.9310536021085168, + "pass@64": 0.9402678159350233, + "pass@128": 0.9495548961424333 + }, + "15": { + "pass@1": 0.6778549382716049, + "pass@2": 0.7607910773306114, + "pass@4": 0.818683925215366, + "pass@8": 0.8583059907925742, + "pass@16": 0.8837760195460365, + "pass@32": 0.9004139453600427, + "pass@64": 0.9137017495934854, + "pass@128": 0.9259259259259259 + }, + "16": { + "pass@1": 0.5277009587020649, + "pass@2": 0.6426116501753651, + "pass@4": 0.7306137161505246, + "pass@8": 0.7944529776129478, + "pass@16": 0.8392236986337849, + "pass@32": 0.8713108635642363, + "pass@64": 0.8925204076771432, + "pass@128": 0.9056047197640118 + }, + "17": { + "pass@1": 0.3881306475903614, + "pass@2": 0.4958969737216582, + "pass@4": 0.5888633374894398, + "pass@8": 0.6658850247230739, + "pass@16": 0.7263493751882383, + "pass@32": 0.7713623137124465, + "pass@64": 0.8058036236077262, + "pass@128": 0.8343373493975904 + }, + "18": { + "pass@1": 0.3305947580645161, + "pass@2": 0.4291993427736854, + "pass@4": 0.517139285411904, + "pass@8": 0.5928866531289754, + "pass@16": 0.6538277321133854, + "pass@32": 0.6983050653570473, + "pass@64": 0.7309006206495801, + "pass@128": 0.7580645161290323 + }, + "19": { + "pass@1": 0.21037103474320243, + "pass@2": 0.30597003829959324, + "pass@4": 0.40784506251295616, + "pass@8": 0.49916852977603016, + "pass@16": 0.5714700401593907, + "pass@32": 0.6277334795578147, + "pass@64": 0.6735989016680082, + "pass@128": 0.716012084592145 + }, + "20": { + "pass@1": 0.17717717717717718, + "pass@2": 0.25918154867465903, + "pass@4": 0.3497985842985841, + "pass@8": 0.43884148885642665, + "pass@16": 0.5155188213992907, + "pass@32": 0.5744289253316636, + "pass@64": 0.6207152030744365, + "pass@128": 0.6576576576576577 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_metrics.json b/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..7628996582c492ddad431ce123ed46f0f401973a --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch1-op11-14", + "id": { + "answer_accuracy": 0.9792821180555555, + "avg_response_len": 198.730515625, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9792821180555555, + "pass@2": 0.984920699365702, + "pass@4": 0.9881914878660996, + "pass@8": 0.9904670562279972, + "pass@16": 0.9921862755436088, + "pass@32": 0.9935212775960051, + "pass@64": 0.9946401655204734, + "pass@128": 0.9954444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9203125, + "pass@2": 0.9373841043307087, + "pass@4": 0.9486343601424823, + "pass@8": 0.9568461594757247, + "pass@16": 0.9631299032632213, + "pass@32": 0.9681102653736894, + "pass@64": 0.9720156987023217, + "pass@128": 0.975 + }, + "2": { + "pass@1": 0.9981640625, + "pass@2": 0.9987837106299212, + "pass@4": 0.9990383347394075, + "pass@8": 0.999121313250048, + "pass@16": 0.9992352362042897, + "pass@32": 0.9994389763779528, + "pass@64": 0.9997519685039371, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9952421875, + "pass@2": 0.9970708661417323, + "pass@4": 0.9983012541244844, + "pass@8": 0.9991987368058538, + "pass@16": 0.9997891651186116, + "pass@32": 0.9999863538662602, + "pass@64": 0.9999999869805416, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9923984375, + "pass@2": 0.9935083661417322, + "pass@4": 0.994365091488564, + "pass@8": 0.9951879421387454, + "pass@16": 0.995978519344931, + "pass@32": 0.9968444264374606, + "pass@64": 0.9976963412230191, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.97834375, + "pass@2": 0.983570989173229, + "pass@4": 0.9870388284589425, + "pass@8": 0.9898555406065586, + "pass@16": 0.9925189720292993, + "pass@32": 0.9950251327152541, + "pass@64": 0.9973750477309096, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.99003125, + "pass@2": 0.9921589566929134, + "pass@4": 0.9934118787964004, + "pass@8": 0.9940810662118578, + "pass@16": 0.9944445402431988, + "pass@32": 0.9948313926560495, + "pass@64": 0.9953779527558758, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.985546875, + "pass@2": 0.9910484744094491, + "pass@4": 0.994010423415823, + "pass@8": 0.9954839757776164, + "pass@16": 0.9962401880505216, + "pass@32": 0.9966784691049091, + "pass@64": 0.9969404364588756, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.9819921875, + "pass@2": 0.9892123523622052, + "pass@4": 0.9925807832770904, + "pass@8": 0.9947584005374547, + "pass@16": 0.9960946946258159, + "pass@32": 0.9967326365666029, + "pass@64": 0.9969710370484877, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.9715078125, + "pass@2": 0.9815484744094494, + "pass@4": 0.9863424363517062, + "pass@8": 0.9896703712481156, + "pass@16": 0.9922452610125915, + "pass@32": 0.9940438452658681, + "pass@64": 0.9956330202802935, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.9203125, + "2": 0.9981640625, + "3": 0.9952421875, + "4": 0.9923984375, + "5": 0.97834375, + "6": 0.99003125, + "7": 0.985546875, + "8": 0.9819921875, + "9": 0.9715078125 + }, + "per_op_avg_response_len": { + "10": 278.029125, + "2": 115.1471953125, + "3": 144.109484375, + "4": 159.02309375, + "5": 172.6520546875, + "6": 197.8052734375, + "7": 222.695625, + "8": 241.630140625, + "9": 257.4826484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.029125, + "2": 115.1471953125, + "3": 144.109484375, + "4": 159.02309375, + "5": 172.6520546875, + "6": 197.8052734375, + "7": 222.695625, + "8": 241.630140625, + "9": 257.4826484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 370805, + "answer_accuracy": 0.9783566573792638, + "avg_response_len": 189.4634757049983, + "resp_tokens_sum": 71808173, + "pass_at_k": { + "pass@1": 0.9783566573792638, + "pass@2": 0.984426398162996, + "pass@4": 0.9879441927028629, + "pass@8": 0.9903708442945975, + "pass@16": 0.9922021831471315, + "pass@32": 0.99362419389056, + "pass@64": 0.9947730546856574, + "pass@128": 0.9956095913542722 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9231425147492626, + "pass@2": 0.9383285572433974, + "pass@4": 0.9473531416979957, + "pass@8": 0.9537757890092359, + "pass@16": 0.9589878410934526, + "pass@32": 0.9634827340913612, + "pass@64": 0.9671551508567272, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.999123831775701, + "pass@2": 0.999758536316138, + "pass@4": 0.9999827985870926, + "pass@8": 0.9999999340639316, + "pass@16": 0.999999999999756, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9958428899082569, + "pass@2": 0.9977124419080643, + "pass@4": 0.9987991609993705, + "pass@8": 0.9995675174167592, + "pass@16": 0.9999476070371531, + "pass@32": 0.9999995668075918, + "pass@64": 0.9999999999993109, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9838134765625, + "pass@2": 0.9859390378937007, + "pass@4": 0.9883570163104611, + "pass@8": 0.9906581655190253, + "pass@16": 0.9926442820736442, + "pass@32": 0.9945717745930672, + "pass@64": 0.9963074639597295, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9704117063492064, + "pass@2": 0.9771399668791401, + "pass@4": 0.9815450669856742, + "pass@8": 0.9853588892526388, + "pass@16": 0.9894869670242535, + "pass@32": 0.9933388347853764, + "pass@64": 0.9967933357643882, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.992936002994012, + "pass@2": 0.9954846704229336, + "pass@4": 0.9969230995264813, + "pass@8": 0.9975195058071472, + "pass@16": 0.9980010177318054, + "pass@32": 0.9987468819391533, + "pass@64": 0.9996345890895375, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9860623150887574, + "pass@2": 0.9925427625448447, + "pass@4": 0.9958278568544315, + "pass@8": 0.9969092535432367, + "pass@16": 0.9970388200006978, + "pass@32": 0.9970414196166396, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9829477163461539, + "pass@2": 0.9896030499192406, + "pass@4": 0.9923302090243528, + "pass@8": 0.9939964803271766, + "pass@16": 0.9951222806297688, + "pass@32": 0.9960540651646855, + "pass@64": 0.9967025035329475, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9732394366197183, + "pass@2": 0.9847652351114562, + "pass@4": 0.9913186990710667, + "pass@8": 0.9961890503448468, + "pass@16": 0.9991175807085956, + "pass@32": 0.9999597333809449, + "pass@64": 0.9999999878674508, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 382626, + "answer_accuracy": 0.9794448312581914, + "avg_response_len": 196.59161000982962, + "resp_tokens_sum": 76799692, + "pass_at_k": { + "pass@1": 0.9794448312581914, + "pass@2": 0.9841983135493961, + "pass@4": 0.9869167429902689, + "pass@8": 0.9889238346741729, + "pass@16": 0.9904814962330848, + "pass@32": 0.9917710998181902, + "pass@64": 0.9930290745140739, + "pass@128": 0.9941022280471822 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9112586273006135, + "pass@2": 0.9263716879619345, + "pass@4": 0.9368261261245718, + "pass@8": 0.9442747604605302, + "pass@16": 0.9497976670404737, + "pass@32": 0.954829192728206, + "pass@64": 0.9594253227868381, + "pass@128": 0.9631901840490797 + }, + "2": { + "pass@1": 0.9988839285714286, + "pass@2": 0.999595753655793, + "pass@4": 0.9999491804595854, + "pass@8": 0.9999993314708793, + "pass@16": 0.9999999999510504, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.993655303030303, + "pass@2": 0.9962635707468384, + "pass@4": 0.9979543449114315, + "pass@8": 0.9990869648235982, + "pass@16": 0.9997729214698883, + "pass@32": 0.9999890704069151, + "pass@64": 0.9999999971219998, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9963711168639053, + "pass@2": 0.9970891039230303, + "pass@4": 0.9973102368306034, + "pass@8": 0.9975660626410165, + "pass@16": 0.9980247014241563, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9873414855072464, + "pass@2": 0.990341136026475, + "pass@4": 0.9926458491601591, + "pass@8": 0.9944704905370954, + "pass@16": 0.9957758227738857, + "pass@32": 0.9971598125688518, + "pass@64": 0.9989273079949132, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9867478510028653, + "pass@2": 0.9883521958125577, + "pass@4": 0.9886221656927555, + "pass@8": 0.9887177604361171, + "pass@16": 0.9888968481375355, + "pass@32": 0.9892550143266475, + "pass@64": 0.9899713467048711, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.9869942196531792, + "pass@2": 0.9909770976514497, + "pass@4": 0.993617887810989, + "pass@8": 0.9958918476913389, + "pass@16": 0.9978098101358188, + "pass@32": 0.9990707212295816, + "pass@64": 0.9998278510372128, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9852992957746479, + "pass@2": 0.9926118720195188, + "pass@4": 0.9960676550114335, + "pass@8": 0.9984719701911067, + "pass@16": 0.9997337999101152, + "pass@32": 0.9999920888176553, + "pass@64": 0.9999999988794946, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9658495508982036, + "pass@2": 0.9738565438021595, + "pass@4": 0.977178885479884, + "pass@8": 0.9799258010494726, + "pass@16": 0.9827626400173137, + "pass@32": 0.9852917113826465, + "pass@64": 0.9880352427034073, + "pass@128": 0.9910179640718563 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 374702, + "answer_accuracy": 0.9800332691663877, + "avg_response_len": 210.10234191914964, + "resp_tokens_sum": 80329689, + "pass_at_k": { + "pass@1": 0.9800332691663877, + "pass@2": 0.9861488036082874, + "pass@4": 0.9897391150302732, + "pass@8": 0.9921392342383987, + "pass@16": 0.9939123833580354, + "pass@32": 0.9952075204583134, + "pass@64": 0.9961545829739121, + "pass@128": 0.9966521593572146 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9262593283582089, + "pass@2": 0.9471449347749443, + "pass@4": 0.9614218745977648, + "pass@8": 0.972186851019286, + "pass@16": 0.9802954796338725, + "pass@32": 0.9857173184696202, + "pass@64": 0.9891864099503938, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9966071428571428, + "pass@2": 0.9971263357705287, + "pass@4": 0.9973159314907064, + "pass@8": 0.9974901553188764, + "pass@16": 0.9978149606299211, + "pass@32": 0.9983970753655793, + "pass@64": 0.9992913385826772, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961962463556852, + "pass@2": 0.9972359163012786, + "pass@4": 0.9981603342767286, + "pass@8": 0.9989546945154124, + "pass@16": 0.9996537420769658, + "pass@32": 0.9999711436906581, + "pass@64": 0.999999964811974, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9965049342105263, + "pass@2": 0.9970519207303035, + "pass@4": 0.9970759831007965, + "pass@8": 0.9970760233917945, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9765625, + "pass@2": 0.9826594343446041, + "pass@4": 0.9864391599947065, + "pass@8": 0.989338738990234, + "pass@16": 0.9920232898813793, + "pass@32": 0.9944213659459019, + "pass@64": 0.9963388991378888, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9905855678233438, + "pass@2": 0.9928459319655231, + "pass@4": 0.9949855764283407, + "pass@8": 0.9963629428393239, + "pass@16": 0.9968050483305237, + "pass@32": 0.996845217944392, + "pass@64": 0.9968454258674141, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9834107990506329, + "pass@2": 0.9895283065882586, + "pass@4": 0.9924962614443763, + "pass@8": 0.9935128758822745, + "pass@16": 0.993667318301558, + "pass@32": 0.9936708852185112, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9775713213213213, + "pass@2": 0.9852221808422593, + "pass@4": 0.9890983199172173, + "pass@8": 0.9915133611337327, + "pass@16": 0.9931262585622738, + "pass@32": 0.9938936237385995, + "pass@64": 0.9939935625945567, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9756079180064309, + "pass@2": 0.9861373965111273, + "pass@4": 0.9905033775922705, + "pass@8": 0.9926946650327054, + "pass@16": 0.9945845597918244, + "pass@32": 0.996690380430318, + "pass@64": 0.9988078891460136, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.6068109375, + "avg_response_len": 362.87332109375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.6068109375, + "pass@2": 0.6781929010826631, + "pass@4": 0.7337707151106001, + "pass@8": 0.7774104621906996, + "pass@16": 0.812397911152135, + "pass@32": 0.8407001163553003, + "pass@64": 0.8634577937668082, + "pass@128": 0.882 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9254296875, + "pass@2": 0.9455980561023625, + "pass@4": 0.9575916816647916, + "pass@8": 0.9660786754437783, + "pass@16": 0.9726976088499079, + "pass@32": 0.9780525683523252, + "pass@64": 0.9822852408522206, + "pass@128": 0.985 + }, + "12": { + "pass@1": 0.8876640625, + "pass@2": 0.912449803149607, + "pass@4": 0.925959796587926, + "pass@8": 0.9355666820350127, + "pass@16": 0.9435803573597504, + "pass@32": 0.9503318362515238, + "pass@64": 0.9551076648570225, + "pass@128": 0.958 + }, + "13": { + "pass@1": 0.793328125, + "pass@2": 0.8423879183070874, + "pass@4": 0.8746621142669666, + "pass@8": 0.8993848379953863, + "pass@16": 0.9196587290575087, + "pass@32": 0.9363600136699254, + "pass@64": 0.9495616737849027, + "pass@128": 0.96 + }, + "14": { + "pass@1": 0.7552421875, + "pass@2": 0.819026451771654, + "pass@4": 0.8603135664604424, + "pass@8": 0.8881597259013125, + "pass@16": 0.9086765035548701, + "pass@32": 0.9247914425923657, + "pass@64": 0.9377123715286088, + "pass@128": 0.947 + }, + "15": { + "pass@1": 0.68603125, + "pass@2": 0.7612930610236227, + "pass@4": 0.8163969390701169, + "pass@8": 0.8583675259531693, + "pass@16": 0.8893081787096128, + "pass@32": 0.9109596756292865, + "pass@64": 0.9263257608539854, + "pass@128": 0.939 + }, + "16": { + "pass@1": 0.56871875, + "pass@2": 0.6653614665354327, + "pass@4": 0.7337346590738649, + "pass@8": 0.7822931078383818, + "pass@16": 0.8196891703545264, + "pass@32": 0.8515644610904193, + "pass@64": 0.8783661141839073, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.4713515625, + "pass@2": 0.569911663385827, + "pass@4": 0.6470357365016873, + "pass@8": 0.7044032241446979, + "pass@16": 0.7485911337985129, + "pass@32": 0.7843660456692181, + "pass@64": 0.8140700557347367, + "pass@128": 0.84 + }, + "18": { + "pass@1": 0.41446875, + "pass@2": 0.5168107775590551, + "pass@4": 0.6019622500937379, + "pass@8": 0.6684197409074777, + "pass@16": 0.7204470025893406, + "pass@32": 0.762843391142011, + "pass@64": 0.7975139806671178, + "pass@128": 0.825 + }, + "19": { + "pass@1": 0.284828125, + "pass@2": 0.3740754183070868, + "pass@4": 0.45741562532808394, + "pass@8": 0.5323990298189198, + "pass@16": 0.5975583484264403, + "pass@32": 0.6515244824715801, + "pass@64": 0.6956207724619008, + "pass@128": 0.733 + }, + "20": { + "pass@1": 0.281046875, + "pass@2": 0.3750143946850392, + "pass@4": 0.4626347820584924, + "pass@8": 0.5390320718688989, + "pass@16": 0.6037720788207667, + "pass@32": 0.6562072466843096, + "pass@64": 0.6980143027436173, + "pass@128": 0.733 + } + }, + "per_op_accuracy": { + "11": 0.9254296875, + "12": 0.8876640625, + "13": 0.793328125, + "14": 0.7552421875, + "15": 0.68603125, + "16": 0.56871875, + "17": 0.4713515625, + "18": 0.41446875, + "19": 0.284828125, + "20": 0.281046875 + }, + "per_op_avg_response_len": { + "11": 295.6399296875, + "12": 306.38940625, + "13": 325.8842890625, + "14": 349.951796875, + "15": 367.997828125, + "16": 373.1953515625, + "17": 392.071921875, + "18": 401.8119140625, + "19": 400.8838671875, + "20": 414.90690625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 295.6399296875, + "12": 306.38940625, + "13": 325.8842890625, + "14": 349.951796875, + "15": 367.997828125, + "16": 373.1953515625, + "17": 392.071921875, + "18": 401.8119140625, + "19": 400.8838671875, + "20": 414.90690625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 261800, + "answer_accuracy": 0.6078194650817236, + "avg_response_len": 343.85410939821696, + "resp_tokens_sum": 148104842, + "pass_at_k": { + "pass@1": 0.6078194650817236, + "pass@2": 0.6782466786980434, + "pass@4": 0.7322968574693411, + "pass@8": 0.7744592740301475, + "pass@16": 0.8084620539797583, + "pass@32": 0.8365408446463333, + "pass@64": 0.8594061058570227, + "pass@128": 0.8772659732540862 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9234684844192634, + "pass@2": 0.9449662343021572, + "pass@4": 0.956891268637454, + "pass@8": 0.9643693469153156, + "pass@16": 0.9700565621001835, + "pass@32": 0.9749692654639327, + "pass@64": 0.9787854577528349, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8851739053672316, + "pass@2": 0.9140503358690337, + "pass@4": 0.9297601067874989, + "pass@8": 0.9404788197530561, + "pass@16": 0.94967725068733, + "pass@32": 0.9586145399733833, + "pass@64": 0.9651696834736039, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.8151449843260188, + "pass@2": 0.8586179127687409, + "pass@4": 0.8867077769275696, + "pass@8": 0.9085025252861926, + "pass@16": 0.9271233872904527, + "pass@32": 0.9449082791298123, + "pass@64": 0.9610983812381588, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.7620959890965732, + "pass@2": 0.8263109561655264, + "pass@4": 0.8645924224425215, + "pass@8": 0.8889128593760814, + "pass@16": 0.9094888442681421, + "pass@32": 0.928986852397342, + "pass@64": 0.9461391387088692, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.6877257947976878, + "pass@2": 0.7639100262846479, + "pass@4": 0.8192633533705684, + "pass@8": 0.8602656226549914, + "pass@16": 0.8904892628915874, + "pass@32": 0.912211514532011, + "pass@64": 0.9267263236137377, + "pass@128": 0.9364161849710982 + }, + "16": { + "pass@1": 0.5722278225806452, + "pass@2": 0.6708578073406148, + "pass@4": 0.7376889657945979, + "pass@8": 0.7811584454338113, + "pass@16": 0.8123913701188059, + "pass@32": 0.8397369710307722, + "pass@64": 0.8647941957421232, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.46899837427745666, + "pass@2": 0.5686220614673886, + "pass@4": 0.6493129596155967, + "pass@8": 0.7107867389096841, + "pass@16": 0.7572407054462839, + "pass@32": 0.7920715450323713, + "pass@64": 0.8177465613909459, + "pass@128": 0.8352601156069365 + }, + "18": { + "pass@1": 0.3928348214285714, + "pass@2": 0.4923548228346458, + "pass@4": 0.57355253361187, + "pass@8": 0.6381375838952569, + "pass@16": 0.6909666831451384, + "pass@32": 0.7355241528015719, + "pass@64": 0.7725518509793133, + "pass@128": 0.8028571428571428 + }, + "19": { + "pass@1": 0.2802881006006006, + "pass@2": 0.3656185713272326, + "pass@4": 0.44623301085112094, + "pass@8": 0.5207589952897529, + "pass@16": 0.5834385538957976, + "pass@32": 0.6326537813669033, + "pass@64": 0.6755366923342895, + "pass@128": 0.7147147147147147 + }, + "20": { + "pass@1": 0.27888044294294295, + "pass@2": 0.3691022469319712, + "pass@4": 0.45260455928369264, + "pass@8": 0.5256639851382408, + "pass@16": 0.5888334762511734, + "pass@32": 0.6417662504776702, + "pass@64": 0.6828783160667011, + "pass@128": 0.7177177177177178 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 262004, + "answer_accuracy": 0.60865484686292, + "avg_response_len": 358.8022250408861, + "resp_tokens_sum": 154451441, + "pass_at_k": { + "pass@1": 0.60865484686292, + "pass@2": 0.6813515348243206, + "pass@4": 0.7380243010714206, + "pass@8": 0.7818896408204202, + "pass@16": 0.8163833261403455, + "pass@32": 0.8436691369883188, + "pass@64": 0.8652473006723143, + "pass@128": 0.8819506393101397 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9304405120481928, + "pass@2": 0.9519117315719565, + "pass@4": 0.9657082994520263, + "pass@8": 0.9757115076074933, + "pass@16": 0.9825771503102283, + "pass@32": 0.9865760138899352, + "pass@64": 0.9894810140158518, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8922413793103449, + "pass@2": 0.9128235072692715, + "pass@4": 0.9231989963832886, + "pass@8": 0.9287502632219307, + "pass@16": 0.9322745427114282, + "pass@32": 0.935291197697357, + "pass@64": 0.9379938824694157, + "pass@128": 0.9404388714733543 + }, + "13": { + "pass@1": 0.7765449929971989, + "pass@2": 0.8236345227067203, + "pass@4": 0.8546561067996749, + "pass@8": 0.8796959667664824, + "pass@16": 0.9017360202584387, + "pass@32": 0.9197525073028779, + "pass@64": 0.9340349790683912, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.7601653874269005, + "pass@2": 0.8235343423354977, + "pass@4": 0.8647729458708009, + "pass@8": 0.8932804906568209, + "pass@16": 0.9130388497560374, + "pass@32": 0.9267935902050748, + "pass@64": 0.9365736229030256, + "pass@128": 0.9415204678362573 + }, + "15": { + "pass@1": 0.6853219696969697, + "pass@2": 0.7700440676449535, + "pass@4": 0.8293955920282687, + "pass@8": 0.8725181459664744, + "pass@16": 0.9028329731845267, + "pass@32": 0.9229171920966758, + "pass@64": 0.9364439826397638, + "pass@128": 0.9454545454545454 + }, + "16": { + "pass@1": 0.5576923076923077, + "pass@2": 0.6493164215851224, + "pass@4": 0.7176616638197999, + "pass@8": 0.7699788625723991, + "pass@16": 0.8121033872254676, + "pass@32": 0.8467035952508719, + "pass@64": 0.8723326079810598, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.4888392857142857, + "pass@2": 0.5898313322247761, + "pass@4": 0.6642076815320447, + "pass@8": 0.714858050845223, + "pass@16": 0.7517769648613764, + "pass@32": 0.781466465575029, + "pass@64": 0.8059220394919001, + "pass@128": 0.8260869565217391 + }, + "18": { + "pass@1": 0.4178768382352941, + "pass@2": 0.5259965551181103, + "pass@4": 0.6213802793952226, + "pass@8": 0.6972919448114913, + "pass@16": 0.755379792189837, + "pass@32": 0.8004193580795406, + "pass@64": 0.8365682411731703, + "pass@128": 0.8647058823529412 + }, + "19": { + "pass@1": 0.28429594494047616, + "pass@2": 0.3704518624625045, + "pass@4": 0.4515365584882246, + "pass@8": 0.5245154324308915, + "pass@16": 0.5899039910455528, + "pass@32": 0.6469910765958351, + "pass@64": 0.6931867993172561, + "pass@128": 0.7261904761904762 + }, + "20": { + "pass@1": 0.2971790793413174, + "pass@2": 0.3988401150454995, + "pass@4": 0.48962655734649935, + "pass@8": 0.5629928497263833, + "pass@16": 0.6216007899549488, + "pass@32": 0.6680350459107165, + "pass@64": 0.7074075972497136, + "pass@128": 0.7425149700598802 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 252914, + "answer_accuracy": 0.6038785528728606, + "avg_response_len": 386.6174358190709, + "resp_tokens_sum": 161921568, + "pass_at_k": { + "pass@1": 0.6038785528728606, + "pass@2": 0.6748911141178825, + "pass@4": 0.7309145786120432, + "pass@8": 0.7758417795588501, + "pass@16": 0.8123493808280585, + "pass@32": 0.8419260127219851, + "pass@64": 0.865785366533651, + "pass@128": 0.886919315403423 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9223462301587302, + "pass@2": 0.9396516841644794, + "pass@4": 0.9498219314847549, + "pass@8": 0.9578415093872525, + "pass@16": 0.9652445350017391, + "pass@32": 0.9725243842288842, + "pass@64": 0.9786231035308165, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.885894495412844, + "pass@2": 0.9103525548411951, + "pass@4": 0.9245389569996405, + "pass@8": 0.9368986112374154, + "pass@16": 0.9480092706161772, + "pass@32": 0.9560379114235142, + "pass@64": 0.9609099339438009, + "pass@128": 0.963302752293578 + }, + "13": { + "pass@1": 0.7903404706790124, + "pass@2": 0.8470718503937011, + "pass@4": 0.8848459978382324, + "pass@8": 0.9121020749798062, + "pass@16": 0.9320574360481253, + "pass@32": 0.9462426775320625, + "pass@64": 0.9553111192670196, + "pass@128": 0.9629629629629629 + }, + "14": { + "pass@1": 0.7437175445103857, + "pass@2": 0.8075130260052802, + "pass@4": 0.8517123186011245, + "pass@8": 0.8822456090117424, + "pass@16": 0.9034756615081073, + "pass@32": 0.918763368316569, + "pass@64": 0.930841332255865, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6849440586419753, + "pass@2": 0.749585338291047, + "pass@4": 0.8000965229462056, + "pass@8": 0.8419278776098915, + "pass@16": 0.8742716438216978, + "pass@32": 0.897443883362062, + "pass@64": 0.9155924031250283, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.576926622418879, + "pass@2": 0.6769483253199544, + "pass@4": 0.7467605772176702, + "pass@8": 0.7960808524807924, + "pass@16": 0.8342169814205239, + "pass@32": 0.8674130918519873, + "pass@64": 0.8970240946976298, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.45684299698795183, + "pass@2": 0.551935967294374, + "pass@4": 0.6280077681366633, + "pass@8": 0.6876106027403157, + "pass@16": 0.7364869488817384, + "pass@32": 0.7791478589543936, + "pass@64": 0.8181411107743274, + "pass@128": 0.858433734939759 + }, + "18": { + "pass@1": 0.43515625, + "pass@2": 0.5343476155702312, + "pass@4": 0.6127405430168001, + "pass@8": 0.6709429848652598, + "pass@16": 0.7154178520774157, + "pass@32": 0.7524753416594111, + "pass@64": 0.7828633252434822, + "pass@128": 0.8064516129032258 + }, + "19": { + "pass@1": 0.28993580060422963, + "pass@2": 0.3862616563979352, + "pass@4": 0.47463368296863195, + "pass@8": 0.5521120818448708, + "pass@16": 0.6195334410508603, + "pass@32": 0.6751110922664687, + "pass@64": 0.7182969466585594, + "pass@128": 0.7583081570996979 + }, + "20": { + "pass@1": 0.26703265765765766, + "pass@2": 0.3570292733678559, + "pass@4": 0.4455921731630394, + "pass@8": 0.5283674264542136, + "pass@16": 0.600828430583105, + "pass@32": 0.6587849247479455, + "pass@64": 0.7037287868228262, + "pass@128": 0.7387387387387387 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7832446546052632, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7832446546052632, + "pass@2": 0.823485016058828, + "pass@4": 0.8542858179947747, + "pass@8": 0.8783320067346951, + "pass@16": 0.8975608206007191, + "pass@32": 0.9130890874693275, + "pass@64": 0.9255968119659093, + "pass@128": 0.9357368421052632 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9203125, + "pass@2": 0.9373841043307087, + "pass@4": 0.9486343601424823, + "pass@8": 0.9568461594757247, + "pass@16": 0.9631299032632213, + "pass@32": 0.9681102653736894, + "pass@64": 0.9720156987023217, + "pass@128": 0.975 + }, + "2": { + "pass@1": 0.9981640625, + "pass@2": 0.9987837106299212, + "pass@4": 0.9990383347394075, + "pass@8": 0.999121313250048, + "pass@16": 0.9992352362042897, + "pass@32": 0.9994389763779528, + "pass@64": 0.9997519685039371, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9952421875, + "pass@2": 0.9970708661417323, + "pass@4": 0.9983012541244844, + "pass@8": 0.9991987368058538, + "pass@16": 0.9997891651186116, + "pass@32": 0.9999863538662602, + "pass@64": 0.9999999869805416, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9923984375, + "pass@2": 0.9935083661417322, + "pass@4": 0.994365091488564, + "pass@8": 0.9951879421387454, + "pass@16": 0.995978519344931, + "pass@32": 0.9968444264374606, + "pass@64": 0.9976963412230191, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.97834375, + "pass@2": 0.983570989173229, + "pass@4": 0.9870388284589425, + "pass@8": 0.9898555406065586, + "pass@16": 0.9925189720292993, + "pass@32": 0.9950251327152541, + "pass@64": 0.9973750477309096, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.99003125, + "pass@2": 0.9921589566929134, + "pass@4": 0.9934118787964004, + "pass@8": 0.9940810662118578, + "pass@16": 0.9944445402431988, + "pass@32": 0.9948313926560495, + "pass@64": 0.9953779527558758, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.985546875, + "pass@2": 0.9910484744094491, + "pass@4": 0.994010423415823, + "pass@8": 0.9954839757776164, + "pass@16": 0.9962401880505216, + "pass@32": 0.9966784691049091, + "pass@64": 0.9969404364588756, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.9819921875, + "pass@2": 0.9892123523622052, + "pass@4": 0.9925807832770904, + "pass@8": 0.9947584005374547, + "pass@16": 0.9960946946258159, + "pass@32": 0.9967326365666029, + "pass@64": 0.9969710370484877, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.9715078125, + "pass@2": 0.9815484744094494, + "pass@4": 0.9863424363517062, + "pass@8": 0.9896703712481156, + "pass@16": 0.9922452610125915, + "pass@32": 0.9940438452658681, + "pass@64": 0.9956330202802935, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.9254296875, + "pass@2": 0.9455980561023625, + "pass@4": 0.9575916816647916, + "pass@8": 0.9660786754437783, + "pass@16": 0.9726976088499079, + "pass@32": 0.9780525683523252, + "pass@64": 0.9822852408522206, + "pass@128": 0.985 + }, + "12": { + "pass@1": 0.8876640625, + "pass@2": 0.912449803149607, + "pass@4": 0.925959796587926, + "pass@8": 0.9355666820350127, + "pass@16": 0.9435803573597504, + "pass@32": 0.9503318362515238, + "pass@64": 0.9551076648570225, + "pass@128": 0.958 + }, + "13": { + "pass@1": 0.793328125, + "pass@2": 0.8423879183070874, + "pass@4": 0.8746621142669666, + "pass@8": 0.8993848379953863, + "pass@16": 0.9196587290575087, + "pass@32": 0.9363600136699254, + "pass@64": 0.9495616737849027, + "pass@128": 0.96 + }, + "14": { + "pass@1": 0.7552421875, + "pass@2": 0.819026451771654, + "pass@4": 0.8603135664604424, + "pass@8": 0.8881597259013125, + "pass@16": 0.9086765035548701, + "pass@32": 0.9247914425923657, + "pass@64": 0.9377123715286088, + "pass@128": 0.947 + }, + "15": { + "pass@1": 0.68603125, + "pass@2": 0.7612930610236227, + "pass@4": 0.8163969390701169, + "pass@8": 0.8583675259531693, + "pass@16": 0.8893081787096128, + "pass@32": 0.9109596756292865, + "pass@64": 0.9263257608539854, + "pass@128": 0.939 + }, + "16": { + "pass@1": 0.56871875, + "pass@2": 0.6653614665354327, + "pass@4": 0.7337346590738649, + "pass@8": 0.7822931078383818, + "pass@16": 0.8196891703545264, + "pass@32": 0.8515644610904193, + "pass@64": 0.8783661141839073, + "pass@128": 0.9 + }, + "17": { + "pass@1": 0.4713515625, + "pass@2": 0.569911663385827, + "pass@4": 0.6470357365016873, + "pass@8": 0.7044032241446979, + "pass@16": 0.7485911337985129, + "pass@32": 0.7843660456692181, + "pass@64": 0.8140700557347367, + "pass@128": 0.84 + }, + "18": { + "pass@1": 0.41446875, + "pass@2": 0.5168107775590551, + "pass@4": 0.6019622500937379, + "pass@8": 0.6684197409074777, + "pass@16": 0.7204470025893406, + "pass@32": 0.762843391142011, + "pass@64": 0.7975139806671178, + "pass@128": 0.825 + }, + "19": { + "pass@1": 0.284828125, + "pass@2": 0.3740754183070868, + "pass@4": 0.45741562532808394, + "pass@8": 0.5323990298189198, + "pass@16": 0.5975583484264403, + "pass@32": 0.6515244824715801, + "pass@64": 0.6956207724619008, + "pass@128": 0.733 + }, + "20": { + "pass@1": 0.281046875, + "pass@2": 0.3750143946850392, + "pass@4": 0.4626347820584924, + "pass@8": 0.5390320718688989, + "pass@16": 0.6037720788207667, + "pass@32": 0.6562072466843096, + "pass@64": 0.6980143027436173, + "pass@128": 0.733 + } + }, + "per_op_accuracy": { + "10": 0.9203125, + "2": 0.9981640625, + "3": 0.9952421875, + "4": 0.9923984375, + "5": 0.97834375, + "6": 0.99003125, + "7": 0.985546875, + "8": 0.9819921875, + "9": 0.9715078125, + "11": 0.9254296875, + "12": 0.8876640625, + "13": 0.793328125, + "14": 0.7552421875, + "15": 0.68603125, + "16": 0.56871875, + "17": 0.4713515625, + "18": 0.41446875, + "19": 0.284828125, + "20": 0.281046875 + }, + "per_op_avg_response_len": { + "10": 278.029125, + "2": 115.1471953125, + "3": 144.109484375, + "4": 159.02309375, + "5": 172.6520546875, + "6": 197.8052734375, + "7": 222.695625, + "8": 241.630140625, + "9": 257.4826484375, + "11": 295.6399296875, + "12": 306.38940625, + "13": 325.8842890625, + "14": 349.951796875, + "15": 367.997828125, + "16": 373.1953515625, + "17": 392.071921875, + "18": 401.8119140625, + "19": 400.8838671875, + "20": 414.90690625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.029125, + "2": 115.1471953125, + "3": 144.109484375, + "4": 159.02309375, + "5": 172.6520546875, + "6": 197.8052734375, + "7": 222.695625, + "8": 241.630140625, + "9": 257.4826484375, + "11": 295.6399296875, + "12": 306.38940625, + "13": 325.8842890625, + "14": 349.951796875, + "15": 367.997828125, + "16": 373.1953515625, + "17": 392.071921875, + "18": 401.8119140625, + "19": 400.8838671875, + "20": 414.90690625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 632605, + "answer_accuracy": 0.7812561749130572, + "avg_response_len": 271.58874955540625, + "resp_tokens_sum": 219913015, + "pass_at_k": { + "pass@1": 0.7812561749130572, + "pass@2": 0.8215596962977374, + "pass@4": 0.8519572684124993, + "pass@8": 0.8755206334283433, + "pass@16": 0.8944649819697402, + "pass@32": 0.9100665792514822, + "pass@64": 0.9227670820634137, + "pass@128": 0.9326588681631363 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9231425147492626, + "pass@2": 0.9383285572433974, + "pass@4": 0.9473531416979957, + "pass@8": 0.9537757890092359, + "pass@16": 0.9589878410934526, + "pass@32": 0.9634827340913612, + "pass@64": 0.9671551508567272, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.999123831775701, + "pass@2": 0.999758536316138, + "pass@4": 0.9999827985870926, + "pass@8": 0.9999999340639316, + "pass@16": 0.999999999999756, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9958428899082569, + "pass@2": 0.9977124419080643, + "pass@4": 0.9987991609993705, + "pass@8": 0.9995675174167592, + "pass@16": 0.9999476070371531, + "pass@32": 0.9999995668075918, + "pass@64": 0.9999999999993109, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9838134765625, + "pass@2": 0.9859390378937007, + "pass@4": 0.9883570163104611, + "pass@8": 0.9906581655190253, + "pass@16": 0.9926442820736442, + "pass@32": 0.9945717745930672, + "pass@64": 0.9963074639597295, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9704117063492064, + "pass@2": 0.9771399668791401, + "pass@4": 0.9815450669856742, + "pass@8": 0.9853588892526388, + "pass@16": 0.9894869670242535, + "pass@32": 0.9933388347853764, + "pass@64": 0.9967933357643882, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.992936002994012, + "pass@2": 0.9954846704229336, + "pass@4": 0.9969230995264813, + "pass@8": 0.9975195058071472, + "pass@16": 0.9980010177318054, + "pass@32": 0.9987468819391533, + "pass@64": 0.9996345890895375, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9860623150887574, + "pass@2": 0.9925427625448447, + "pass@4": 0.9958278568544315, + "pass@8": 0.9969092535432367, + "pass@16": 0.9970388200006978, + "pass@32": 0.9970414196166396, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9829477163461539, + "pass@2": 0.9896030499192406, + "pass@4": 0.9923302090243528, + "pass@8": 0.9939964803271766, + "pass@16": 0.9951222806297688, + "pass@32": 0.9960540651646855, + "pass@64": 0.9967025035329475, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9732394366197183, + "pass@2": 0.9847652351114562, + "pass@4": 0.9913186990710667, + "pass@8": 0.9961890503448468, + "pass@16": 0.9991175807085956, + "pass@32": 0.9999597333809449, + "pass@64": 0.9999999878674508, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9234684844192634, + "pass@2": 0.9449662343021572, + "pass@4": 0.956891268637454, + "pass@8": 0.9643693469153156, + "pass@16": 0.9700565621001835, + "pass@32": 0.9749692654639327, + "pass@64": 0.9787854577528349, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8851739053672316, + "pass@2": 0.9140503358690337, + "pass@4": 0.9297601067874989, + "pass@8": 0.9404788197530561, + "pass@16": 0.94967725068733, + "pass@32": 0.9586145399733833, + "pass@64": 0.9651696834736039, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.8151449843260188, + "pass@2": 0.8586179127687409, + "pass@4": 0.8867077769275696, + "pass@8": 0.9085025252861926, + "pass@16": 0.9271233872904527, + "pass@32": 0.9449082791298123, + "pass@64": 0.9610983812381588, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.7620959890965732, + "pass@2": 0.8263109561655264, + "pass@4": 0.8645924224425215, + "pass@8": 0.8889128593760814, + "pass@16": 0.9094888442681421, + "pass@32": 0.928986852397342, + "pass@64": 0.9461391387088692, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.6877257947976878, + "pass@2": 0.7639100262846479, + "pass@4": 0.8192633533705684, + "pass@8": 0.8602656226549914, + "pass@16": 0.8904892628915874, + "pass@32": 0.912211514532011, + "pass@64": 0.9267263236137377, + "pass@128": 0.9364161849710982 + }, + "16": { + "pass@1": 0.5722278225806452, + "pass@2": 0.6708578073406148, + "pass@4": 0.7376889657945979, + "pass@8": 0.7811584454338113, + "pass@16": 0.8123913701188059, + "pass@32": 0.8397369710307722, + "pass@64": 0.8647941957421232, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.46899837427745666, + "pass@2": 0.5686220614673886, + "pass@4": 0.6493129596155967, + "pass@8": 0.7107867389096841, + "pass@16": 0.7572407054462839, + "pass@32": 0.7920715450323713, + "pass@64": 0.8177465613909459, + "pass@128": 0.8352601156069365 + }, + "18": { + "pass@1": 0.3928348214285714, + "pass@2": 0.4923548228346458, + "pass@4": 0.57355253361187, + "pass@8": 0.6381375838952569, + "pass@16": 0.6909666831451384, + "pass@32": 0.7355241528015719, + "pass@64": 0.7725518509793133, + "pass@128": 0.8028571428571428 + }, + "19": { + "pass@1": 0.2802881006006006, + "pass@2": 0.3656185713272326, + "pass@4": 0.44623301085112094, + "pass@8": 0.5207589952897529, + "pass@16": 0.5834385538957976, + "pass@32": 0.6326537813669033, + "pass@64": 0.6755366923342895, + "pass@128": 0.7147147147147147 + }, + "20": { + "pass@1": 0.27888044294294295, + "pass@2": 0.3691022469319712, + "pass@4": 0.45260455928369264, + "pass@8": 0.5256639851382408, + "pass@16": 0.5888334762511734, + "pass@32": 0.6417662504776702, + "pass@64": 0.6828783160667011, + "pass@128": 0.7177177177177178 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 644630, + "answer_accuracy": 0.7850618667186282, + "avg_response_len": 281.62891294816836, + "resp_tokens_sum": 231251133, + "pass_at_k": { + "pass@1": 0.7850618667186282, + "pass@2": 0.8254338993868824, + "pass@4": 0.8564373537193234, + "pass@8": 0.880388215978897, + "pass@16": 0.8992122606879791, + "pass@32": 0.9141301175895304, + "pass@64": 0.9260407494275895, + "pass@128": 0.9353078721745908 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9112586273006135, + "pass@2": 0.9263716879619345, + "pass@4": 0.9368261261245718, + "pass@8": 0.9442747604605302, + "pass@16": 0.9497976670404737, + "pass@32": 0.954829192728206, + "pass@64": 0.9594253227868381, + "pass@128": 0.9631901840490797 + }, + "2": { + "pass@1": 0.9988839285714286, + "pass@2": 0.999595753655793, + "pass@4": 0.9999491804595854, + "pass@8": 0.9999993314708793, + "pass@16": 0.9999999999510504, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.993655303030303, + "pass@2": 0.9962635707468384, + "pass@4": 0.9979543449114315, + "pass@8": 0.9990869648235982, + "pass@16": 0.9997729214698883, + "pass@32": 0.9999890704069151, + "pass@64": 0.9999999971219998, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9963711168639053, + "pass@2": 0.9970891039230303, + "pass@4": 0.9973102368306034, + "pass@8": 0.9975660626410165, + "pass@16": 0.9980247014241563, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9873414855072464, + "pass@2": 0.990341136026475, + "pass@4": 0.9926458491601591, + "pass@8": 0.9944704905370954, + "pass@16": 0.9957758227738857, + "pass@32": 0.9971598125688518, + "pass@64": 0.9989273079949132, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9867478510028653, + "pass@2": 0.9883521958125577, + "pass@4": 0.9886221656927555, + "pass@8": 0.9887177604361171, + "pass@16": 0.9888968481375355, + "pass@32": 0.9892550143266475, + "pass@64": 0.9899713467048711, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.9869942196531792, + "pass@2": 0.9909770976514497, + "pass@4": 0.993617887810989, + "pass@8": 0.9958918476913389, + "pass@16": 0.9978098101358188, + "pass@32": 0.9990707212295816, + "pass@64": 0.9998278510372128, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9852992957746479, + "pass@2": 0.9926118720195188, + "pass@4": 0.9960676550114335, + "pass@8": 0.9984719701911067, + "pass@16": 0.9997337999101152, + "pass@32": 0.9999920888176553, + "pass@64": 0.9999999988794946, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9658495508982036, + "pass@2": 0.9738565438021595, + "pass@4": 0.977178885479884, + "pass@8": 0.9799258010494726, + "pass@16": 0.9827626400173137, + "pass@32": 0.9852917113826465, + "pass@64": 0.9880352427034073, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.9304405120481928, + "pass@2": 0.9519117315719565, + "pass@4": 0.9657082994520263, + "pass@8": 0.9757115076074933, + "pass@16": 0.9825771503102283, + "pass@32": 0.9865760138899352, + "pass@64": 0.9894810140158518, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8922413793103449, + "pass@2": 0.9128235072692715, + "pass@4": 0.9231989963832886, + "pass@8": 0.9287502632219307, + "pass@16": 0.9322745427114282, + "pass@32": 0.935291197697357, + "pass@64": 0.9379938824694157, + "pass@128": 0.9404388714733543 + }, + "13": { + "pass@1": 0.7765449929971989, + "pass@2": 0.8236345227067203, + "pass@4": 0.8546561067996749, + "pass@8": 0.8796959667664824, + "pass@16": 0.9017360202584387, + "pass@32": 0.9197525073028779, + "pass@64": 0.9340349790683912, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.7601653874269005, + "pass@2": 0.8235343423354977, + "pass@4": 0.8647729458708009, + "pass@8": 0.8932804906568209, + "pass@16": 0.9130388497560374, + "pass@32": 0.9267935902050748, + "pass@64": 0.9365736229030256, + "pass@128": 0.9415204678362573 + }, + "15": { + "pass@1": 0.6853219696969697, + "pass@2": 0.7700440676449535, + "pass@4": 0.8293955920282687, + "pass@8": 0.8725181459664744, + "pass@16": 0.9028329731845267, + "pass@32": 0.9229171920966758, + "pass@64": 0.9364439826397638, + "pass@128": 0.9454545454545454 + }, + "16": { + "pass@1": 0.5576923076923077, + "pass@2": 0.6493164215851224, + "pass@4": 0.7176616638197999, + "pass@8": 0.7699788625723991, + "pass@16": 0.8121033872254676, + "pass@32": 0.8467035952508719, + "pass@64": 0.8723326079810598, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.4888392857142857, + "pass@2": 0.5898313322247761, + "pass@4": 0.6642076815320447, + "pass@8": 0.714858050845223, + "pass@16": 0.7517769648613764, + "pass@32": 0.781466465575029, + "pass@64": 0.8059220394919001, + "pass@128": 0.8260869565217391 + }, + "18": { + "pass@1": 0.4178768382352941, + "pass@2": 0.5259965551181103, + "pass@4": 0.6213802793952226, + "pass@8": 0.6972919448114913, + "pass@16": 0.755379792189837, + "pass@32": 0.8004193580795406, + "pass@64": 0.8365682411731703, + "pass@128": 0.8647058823529412 + }, + "19": { + "pass@1": 0.28429594494047616, + "pass@2": 0.3704518624625045, + "pass@4": 0.4515365584882246, + "pass@8": 0.5245154324308915, + "pass@16": 0.5899039910455528, + "pass@32": 0.6469910765958351, + "pass@64": 0.6931867993172561, + "pass@128": 0.7261904761904762 + }, + "20": { + "pass@1": 0.2971790793413174, + "pass@2": 0.3988401150454995, + "pass@4": 0.48962655734649935, + "pass@8": 0.5629928497263833, + "pass@16": 0.6216007899549488, + "pass@32": 0.6680350459107165, + "pass@64": 0.7074075972497136, + "pass@128": 0.7425149700598802 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 627616, + "answer_accuracy": 0.7833919156414763, + "avg_response_len": 302.37864599976035, + "resp_tokens_sum": 242251257, + "pass_at_k": { + "pass@1": 0.7833919156414763, + "pass@2": 0.8234334880606494, + "pass@4": 0.8544341328988612, + "pass@8": 0.8790660161985339, + "pass@16": 0.8989971981402587, + "pass@32": 0.9150769735157885, + "pass@64": 0.9280018307463214, + "pass@128": 0.9392874261064068 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9262593283582089, + "pass@2": 0.9471449347749443, + "pass@4": 0.9614218745977648, + "pass@8": 0.972186851019286, + "pass@16": 0.9802954796338725, + "pass@32": 0.9857173184696202, + "pass@64": 0.9891864099503938, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9966071428571428, + "pass@2": 0.9971263357705287, + "pass@4": 0.9973159314907064, + "pass@8": 0.9974901553188764, + "pass@16": 0.9978149606299211, + "pass@32": 0.9983970753655793, + "pass@64": 0.9992913385826772, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961962463556852, + "pass@2": 0.9972359163012786, + "pass@4": 0.9981603342767286, + "pass@8": 0.9989546945154124, + "pass@16": 0.9996537420769658, + "pass@32": 0.9999711436906581, + "pass@64": 0.999999964811974, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9965049342105263, + "pass@2": 0.9970519207303035, + "pass@4": 0.9970759831007965, + "pass@8": 0.9970760233917945, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9765625, + "pass@2": 0.9826594343446041, + "pass@4": 0.9864391599947065, + "pass@8": 0.989338738990234, + "pass@16": 0.9920232898813793, + "pass@32": 0.9944213659459019, + "pass@64": 0.9963388991378888, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9905855678233438, + "pass@2": 0.9928459319655231, + "pass@4": 0.9949855764283407, + "pass@8": 0.9963629428393239, + "pass@16": 0.9968050483305237, + "pass@32": 0.996845217944392, + "pass@64": 0.9968454258674141, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9834107990506329, + "pass@2": 0.9895283065882586, + "pass@4": 0.9924962614443763, + "pass@8": 0.9935128758822745, + "pass@16": 0.993667318301558, + "pass@32": 0.9936708852185112, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9775713213213213, + "pass@2": 0.9852221808422593, + "pass@4": 0.9890983199172173, + "pass@8": 0.9915133611337327, + "pass@16": 0.9931262585622738, + "pass@32": 0.9938936237385995, + "pass@64": 0.9939935625945567, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9756079180064309, + "pass@2": 0.9861373965111273, + "pass@4": 0.9905033775922705, + "pass@8": 0.9926946650327054, + "pass@16": 0.9945845597918244, + "pass@32": 0.996690380430318, + "pass@64": 0.9988078891460136, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9223462301587302, + "pass@2": 0.9396516841644794, + "pass@4": 0.9498219314847549, + "pass@8": 0.9578415093872525, + "pass@16": 0.9652445350017391, + "pass@32": 0.9725243842288842, + "pass@64": 0.9786231035308165, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.885894495412844, + "pass@2": 0.9103525548411951, + "pass@4": 0.9245389569996405, + "pass@8": 0.9368986112374154, + "pass@16": 0.9480092706161772, + "pass@32": 0.9560379114235142, + "pass@64": 0.9609099339438009, + "pass@128": 0.963302752293578 + }, + "13": { + "pass@1": 0.7903404706790124, + "pass@2": 0.8470718503937011, + "pass@4": 0.8848459978382324, + "pass@8": 0.9121020749798062, + "pass@16": 0.9320574360481253, + "pass@32": 0.9462426775320625, + "pass@64": 0.9553111192670196, + "pass@128": 0.9629629629629629 + }, + "14": { + "pass@1": 0.7437175445103857, + "pass@2": 0.8075130260052802, + "pass@4": 0.8517123186011245, + "pass@8": 0.8822456090117424, + "pass@16": 0.9034756615081073, + "pass@32": 0.918763368316569, + "pass@64": 0.930841332255865, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6849440586419753, + "pass@2": 0.749585338291047, + "pass@4": 0.8000965229462056, + "pass@8": 0.8419278776098915, + "pass@16": 0.8742716438216978, + "pass@32": 0.897443883362062, + "pass@64": 0.9155924031250283, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.576926622418879, + "pass@2": 0.6769483253199544, + "pass@4": 0.7467605772176702, + "pass@8": 0.7960808524807924, + "pass@16": 0.8342169814205239, + "pass@32": 0.8674130918519873, + "pass@64": 0.8970240946976298, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.45684299698795183, + "pass@2": 0.551935967294374, + "pass@4": 0.6280077681366633, + "pass@8": 0.6876106027403157, + "pass@16": 0.7364869488817384, + "pass@32": 0.7791478589543936, + "pass@64": 0.8181411107743274, + "pass@128": 0.858433734939759 + }, + "18": { + "pass@1": 0.43515625, + "pass@2": 0.5343476155702312, + "pass@4": 0.6127405430168001, + "pass@8": 0.6709429848652598, + "pass@16": 0.7154178520774157, + "pass@32": 0.7524753416594111, + "pass@64": 0.7828633252434822, + "pass@128": 0.8064516129032258 + }, + "19": { + "pass@1": 0.28993580060422963, + "pass@2": 0.3862616563979352, + "pass@4": 0.47463368296863195, + "pass@8": 0.5521120818448708, + "pass@16": 0.6195334410508603, + "pass@32": 0.6751110922664687, + "pass@64": 0.7182969466585594, + "pass@128": 0.7583081570996979 + }, + "20": { + "pass@1": 0.26703265765765766, + "pass@2": 0.3570292733678559, + "pass@4": 0.4455921731630394, + "pass@8": 0.5283674264542136, + "pass@16": 0.600828430583105, + "pass@32": 0.6587849247479455, + "pass@64": 0.7037287868228262, + "pass@128": 0.7387387387387387 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch1-op11-14/graphs/SDAR-100M-block8-epoch1-op11-14_score_report.jsonl b/composition/SDAR-100M-block8-epoch1-op11-14/graphs/SDAR-100M-block8-epoch1-op11-14_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c85f5283d7fd67021190274d5b9af4828ff75a99 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op11-14/graphs/SDAR-100M-block8-epoch1-op11-14_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch1-op11-14", "results_dir": "results/composition/SDAR-100M-block8-epoch1-op11-14/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1008875, "value_mismatch": 27575, "dependency_mismatch": 139430, "answer_mismatch": 23867, "extra_nodes": 26080, "missing_nodes": 14353, "perfect_with_extra": 2901, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 331976, "dependency_mismatch": 45745, "value_mismatch": 9422, "answer_mismatch": 8203, "extra_nodes": 9321, "missing_nodes": 4948, "perfect_with_extra": 1145}, "teachers_in_school": {"total": 390656, "perfect": 342305, "dependency_mismatch": 47173, "value_mismatch": 8840, "answer_mismatch": 8030, "extra_nodes": 7985, "missing_nodes": 4573, "perfect_with_extra": 527}, "movie_festival_awards": {"total": 382336, "perfect": 334594, "dependency_mismatch": 46512, "value_mismatch": 9313, "answer_mismatch": 7634, "extra_nodes": 8774, "missing_nodes": 4832, "perfect_with_extra": 1229}}, "results_path": "results/composition/SDAR-100M-block8-epoch1-op11-14/graphs/SDAR-100M-block8-epoch1-op11-14_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 371429, "value_mismatch": 632362, "dependency_mismatch": 818956, "answer_mismatch": 503282, "extra_nodes": 72245, "missing_nodes": 310200, "perfect_with_extra": 462, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 212744, "dependency_mismatch": 277122, "answer_mismatch": 168919, "extra_nodes": 26387, "missing_nodes": 115950, "perfect": 124205, "perfect_with_extra": 65}, "movie_festival_awards": {"total": 418816, "perfect": 120397, "value_mismatch": 209361, "dependency_mismatch": 269338, "answer_mismatch": 165903, "extra_nodes": 22810, "missing_nodes": 94736, "perfect_with_extra": 87}, "teachers_in_school": {"total": 430464, "perfect": 126827, "dependency_mismatch": 272496, "value_mismatch": 210257, "extra_nodes": 23048, "missing_nodes": 99514, "perfect_with_extra": 310, "answer_mismatch": 168460}}, "results_path": "results/composition/SDAR-100M-block8-epoch1-op11-14/graphs/SDAR-100M-block8-epoch1-op11-14_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1380304, "value_mismatch": 659937, "dependency_mismatch": 958386, "answer_mismatch": 527149, "extra_nodes": 98325, "missing_nodes": 324553, "perfect_with_extra": 3363, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch1-op11-14/graphs/SDAR-100M-block8-epoch1-op11-14_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 456181, "dependency_mismatch": 322867, "value_mismatch": 222166, "answer_mismatch": 177122, "extra_nodes": 35708, "missing_nodes": 120898, "perfect_with_extra": 1210}, "teachers_in_school": {"total": 821120, "perfect": 469132, "dependency_mismatch": 319669, "value_mismatch": 219097, "answer_mismatch": 176490, "extra_nodes": 31033, "missing_nodes": 104087, "perfect_with_extra": 837}, "movie_festival_awards": {"total": 801152, "perfect": 454991, "dependency_mismatch": 315850, "value_mismatch": 218674, "answer_mismatch": 173537, "extra_nodes": 31584, "missing_nodes": 99568, "perfect_with_extra": 1316}}}} diff --git a/composition/SDAR-100M-block8-epoch1-op11-14/graphs/results.json b/composition/SDAR-100M-block8-epoch1-op11-14/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..26838934a6f3e24962c8816a3b5f4e23c55983e6 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op11-14/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block8-epoch1-op11-14_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969418960244648, + "pass@2": 0.9969418960244648, + "pass@4": 0.9969418960244648, + "pass@8": 0.9969418960244648, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.978125, + "pass@2": 0.984375, + "pass@4": 0.9875, + "pass@8": 0.990625, + "pass@16": 0.990625, + "pass@32": 0.99375, + "pass@64": 0.99375, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9492063492063492, + "pass@2": 0.9619047619047619, + "pass@4": 0.9777777777777777, + "pass@8": 0.9809523809523809, + "pass@16": 0.9841269841269841, + "pass@32": 0.9873015873015873, + "pass@64": 0.9904761904761905, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9550898203592815, + "pass@2": 0.9580838323353293, + "pass@4": 0.9760479041916168, + "pass@8": 0.9850299401197605, + "pass@16": 0.9970059880239521, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8698224852071006, + "pass@2": 0.908284023668639, + "pass@4": 0.9497041420118343, + "pass@8": 0.9792899408284024, + "pass@16": 0.9881656804733728, + "pass@32": 0.9881656804733728, + "pass@64": 0.9940828402366864, + "pass@128": 0.9940828402366864 + }, + "8": { + "pass@1": 0.8173076923076923, + "pass@2": 0.8782051282051282, + "pass@4": 0.9262820512820513, + "pass@8": 0.9711538461538461, + "pass@16": 0.9935897435897436, + "pass@32": 0.9967948717948718, + "pass@64": 0.9967948717948718, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.6957746478873239, + "pass@2": 0.7915492957746478, + "pass@4": 0.8873239436619719, + "pass@8": 0.9408450704225352, + "pass@16": 0.9859154929577465, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6253687315634219, + "pass@2": 0.7020648967551623, + "pass@4": 0.7728613569321534, + "pass@8": 0.8436578171091446, + "pass@16": 0.8731563421828908, + "pass@32": 0.9056047197640118, + "pass@64": 0.9085545722713865, + "pass@128": 0.911504424778761 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9968847352024922, + "3": 0.9969418960244648, + "4": 0.978125, + "5": 0.9492063492063492, + "6": 0.9550898203592815, + "7": 0.8698224852071006, + "8": 0.8173076923076923, + "9": 0.6957746478873239, + "10": 0.6253687315634219 + }, + "pass@2": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 0.984375, + "5": 0.9619047619047619, + "6": 0.9580838323353293, + "7": 0.908284023668639, + "8": 0.8782051282051282, + "9": 0.7915492957746478, + "10": 0.7020648967551623 + }, + "pass@4": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 0.9875, + "5": 0.9777777777777777, + "6": 0.9760479041916168, + "7": 0.9497041420118343, + "8": 0.9262820512820513, + "9": 0.8873239436619719, + "10": 0.7728613569321534 + }, + "pass@8": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 0.990625, + "5": 0.9809523809523809, + "6": 0.9850299401197605, + "7": 0.9792899408284024, + "8": 0.9711538461538461, + "9": 0.9408450704225352, + "10": 0.8436578171091446 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9841269841269841, + "6": 0.9970059880239521, + "7": 0.9881656804733728, + "8": 0.9935897435897436, + "9": 0.9859154929577465, + "10": 0.8731563421828908 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9873015873015873, + "6": 1.0, + "7": 0.9881656804733728, + "8": 0.9967948717948718, + "9": 0.9971830985915493, + "10": 0.9056047197640118 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9904761904761905, + "6": 1.0, + "7": 0.9940828402366864, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9085545722713865 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 1.0, + "7": 0.9940828402366864, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.911504424778761 + } + }, + "total_pass_at_k": { + "pass@1": 0.873015873015873, + "pass@2": 0.9067882472137792, + "pass@4": 0.9402228976697061, + "pass@8": 0.9645390070921985, + "pass@16": 0.9787234042553191, + "pass@32": 0.9851401553529213, + "pass@64": 0.9868287740628167, + "pass@128": 0.9878419452887538 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9971428571428571, + "pass@2": 0.9971428571428571, + "pass@4": 0.9971428571428571, + "pass@8": 0.9971428571428571, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970845481049563, + "pass@2": 0.9970845481049563, + "pass@4": 0.9970845481049563, + "pass@8": 0.9970845481049563, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9941520467836257, + "pass@2": 0.9970760233918129, + "pass@4": 0.9970760233918129, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9529411764705882, + "pass@2": 0.9676470588235294, + "pass@4": 0.9735294117647059, + "pass@8": 0.9852941176470589, + "pass@16": 0.9882352941176471, + "pass@32": 0.9882352941176471, + "pass@64": 0.9882352941176471, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9589905362776026, + "pass@2": 0.9716088328075709, + "pass@4": 0.9779179810725552, + "pass@8": 0.9905362776025236, + "pass@16": 0.9968454258675079, + "pass@32": 0.9968454258675079, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.8670886075949367, + "pass@2": 0.9082278481012658, + "pass@4": 0.9462025316455697, + "pass@8": 0.9715189873417721, + "pass@16": 0.9841772151898734, + "pass@32": 0.9873417721518988, + "pass@64": 0.990506329113924, + "pass@128": 0.990506329113924 + }, + "8": { + "pass@1": 0.7957957957957958, + "pass@2": 0.8588588588588588, + "pass@4": 0.9159159159159159, + "pass@8": 0.960960960960961, + "pass@16": 0.990990990990991, + "pass@32": 0.990990990990991, + "pass@64": 0.990990990990991, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.7363344051446945, + "pass@2": 0.8102893890675241, + "pass@4": 0.9003215434083601, + "pass@8": 0.9453376205787781, + "pass@16": 0.9742765273311897, + "pass@32": 0.9967845659163987, + "pass@64": 0.9967845659163987, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.5880597014925373, + "pass@2": 0.6626865671641791, + "pass@4": 0.7164179104477612, + "pass@8": 0.8029850746268656, + "pass@16": 0.8656716417910447, + "pass@32": 0.9014925373134328, + "pass@64": 0.9194029850746268, + "pass@128": 0.9283582089552239 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9971428571428571, + "3": 0.9970845481049563, + "4": 0.9941520467836257, + "5": 0.9529411764705882, + "6": 0.9589905362776026, + "7": 0.8670886075949367, + "8": 0.7957957957957958, + "9": 0.7363344051446945, + "10": 0.5880597014925373 + }, + "pass@2": { + "2": 0.9971428571428571, + "3": 0.9970845481049563, + "4": 0.9970760233918129, + "5": 0.9676470588235294, + "6": 0.9716088328075709, + "7": 0.9082278481012658, + "8": 0.8588588588588588, + "9": 0.8102893890675241, + "10": 0.6626865671641791 + }, + "pass@4": { + "2": 0.9971428571428571, + "3": 0.9970845481049563, + "4": 0.9970760233918129, + "5": 0.9735294117647059, + "6": 0.9779179810725552, + "7": 0.9462025316455697, + "8": 0.9159159159159159, + "9": 0.9003215434083601, + "10": 0.7164179104477612 + }, + "pass@8": { + "2": 0.9971428571428571, + "3": 0.9970845481049563, + "4": 0.9970760233918129, + "5": 0.9852941176470589, + "6": 0.9905362776025236, + "7": 0.9715189873417721, + "8": 0.960960960960961, + "9": 0.9453376205787781, + "10": 0.8029850746268656 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9882352941176471, + "6": 0.9968454258675079, + "7": 0.9841772151898734, + "8": 0.990990990990991, + "9": 0.9742765273311897, + "10": 0.8656716417910447 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9882352941176471, + "6": 0.9968454258675079, + "7": 0.9873417721518988, + "8": 0.990990990990991, + "9": 0.9967845659163987, + "10": 0.9014925373134328 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9882352941176471, + "6": 0.9968454258675079, + "7": 0.990506329113924, + "8": 0.990990990990991, + "9": 0.9967845659163987, + "10": 0.9194029850746268 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9941176470588236, + "6": 0.9968454258675079, + "7": 0.990506329113924, + "8": 0.993993993993994, + "9": 1.0, + "10": 0.9283582089552239 + } + }, + "total_pass_at_k": { + "pass@1": 0.8784733846668898, + "pass@2": 0.9092735185805155, + "pass@4": 0.9363910277870774, + "pass@8": 0.9611650485436893, + "pass@16": 0.9775694676933377, + "pass@32": 0.9842651489789086, + "pass@64": 0.9866086374288584, + "pass@128": 0.9889521258788082 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9939393939393939, + "pass@2": 0.996969696969697, + "pass@4": 0.996969696969697, + "pass@8": 0.996969696969697, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9704142011834319, + "pass@2": 0.985207100591716, + "pass@4": 0.9911242603550295, + "pass@8": 0.9970414201183432, + "pass@16": 0.9970414201183432, + "pass@32": 0.9970414201183432, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9507246376811594, + "pass@2": 0.9739130434782609, + "pass@4": 0.9855072463768116, + "pass@8": 0.991304347826087, + "pass@16": 0.991304347826087, + "pass@32": 0.9971014492753624, + "pass@64": 0.9971014492753624, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9598853868194842, + "pass@2": 0.9742120343839542, + "pass@4": 0.9856733524355301, + "pass@8": 0.9856733524355301, + "pass@16": 0.9885386819484241, + "pass@32": 0.9885386819484241, + "pass@64": 0.9914040114613181, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.8439306358381503, + "pass@2": 0.9046242774566474, + "pass@4": 0.953757225433526, + "pass@8": 0.976878612716763, + "pass@16": 0.9942196531791907, + "pass@32": 0.9942196531791907, + "pass@64": 0.9971098265895953, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7971830985915493, + "pass@2": 0.847887323943662, + "pass@4": 0.923943661971831, + "pass@8": 0.9690140845070423, + "pass@16": 0.9887323943661972, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.7245508982035929, + "pass@2": 0.811377245508982, + "pass@4": 0.8862275449101796, + "pass@8": 0.9341317365269461, + "pass@16": 0.9610778443113772, + "pass@32": 0.9790419161676647, + "pass@64": 0.9820359281437125, + "pass@128": 0.9880239520958084 + }, + "10": { + "pass@1": 0.6380368098159509, + "pass@2": 0.6932515337423313, + "pass@4": 0.7699386503067485, + "pass@8": 0.8128834355828221, + "pass@16": 0.8558282208588958, + "pass@32": 0.8865030674846626, + "pass@64": 0.911042944785276, + "pass@128": 0.9171779141104295 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9939393939393939, + "4": 0.9704142011834319, + "5": 0.9507246376811594, + "6": 0.9598853868194842, + "7": 0.8439306358381503, + "8": 0.7971830985915493, + "9": 0.7245508982035929, + "10": 0.6380368098159509 + }, + "pass@2": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.985207100591716, + "5": 0.9739130434782609, + "6": 0.9742120343839542, + "7": 0.9046242774566474, + "8": 0.847887323943662, + "9": 0.811377245508982, + "10": 0.6932515337423313 + }, + "pass@4": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9911242603550295, + "5": 0.9855072463768116, + "6": 0.9856733524355301, + "7": 0.953757225433526, + "8": 0.923943661971831, + "9": 0.8862275449101796, + "10": 0.7699386503067485 + }, + "pass@8": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9970414201183432, + "5": 0.991304347826087, + "6": 0.9856733524355301, + "7": 0.976878612716763, + "8": 0.9690140845070423, + "9": 0.9341317365269461, + "10": 0.8128834355828221 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.991304347826087, + "6": 0.9885386819484241, + "7": 0.9942196531791907, + "8": 0.9887323943661972, + "9": 0.9610778443113772, + "10": 0.8558282208588958 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.9971014492753624, + "6": 0.9885386819484241, + "7": 0.9942196531791907, + "8": 0.9971830985915493, + "9": 0.9790419161676647, + "10": 0.8865030674846626 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.9914040114613181, + "7": 0.9971098265895953, + "8": 0.9971830985915493, + "9": 0.9820359281437125, + "10": 0.911042944785276 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9914040114613181, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9880239520958084, + "10": 0.9171779141104295 + } + }, + "total_pass_at_k": { + "pass@1": 0.8758191349934469, + "pass@2": 0.9102228047182176, + "pass@4": 0.9442988204456094, + "pass@8": 0.963302752293578, + "pass@16": 0.9757536041939712, + "pass@32": 0.9826343381389253, + "pass@64": 0.9865661861074705, + "pass@128": 0.9885321100917431 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.998, + "pass@2": 0.999, + "pass@4": 0.999, + "pass@8": 0.999, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996, + "pass@2": 0.997, + "pass@4": 0.997, + "pass@8": 0.997, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.981, + "pass@2": 0.989, + "pass@4": 0.992, + "pass@8": 0.995, + "pass@16": 0.995, + "pass@32": 0.996, + "pass@64": 0.997, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.951, + "pass@2": 0.968, + "pass@4": 0.979, + "pass@8": 0.986, + "pass@16": 0.988, + "pass@32": 0.991, + "pass@64": 0.992, + "pass@128": 0.996 + }, + "6": { + "pass@1": 0.958, + "pass@2": 0.968, + "pass@4": 0.98, + "pass@8": 0.987, + "pass@16": 0.994, + "pass@32": 0.995, + "pass@64": 0.996, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.86, + "pass@2": 0.907, + "pass@4": 0.95, + "pass@8": 0.976, + "pass@16": 0.989, + "pass@32": 0.99, + "pass@64": 0.994, + "pass@128": 0.995 + }, + "8": { + "pass@1": 0.803, + "pass@2": 0.861, + "pass@4": 0.922, + "pass@8": 0.967, + "pass@16": 0.991, + "pass@32": 0.995, + "pass@64": 0.995, + "pass@128": 0.996 + }, + "9": { + "pass@1": 0.718, + "pass@2": 0.804, + "pass@4": 0.891, + "pass@8": 0.94, + "pass@16": 0.974, + "pass@32": 0.991, + "pass@64": 0.993, + "pass@128": 0.996 + }, + "10": { + "pass@1": 0.617, + "pass@2": 0.686, + "pass@4": 0.753, + "pass@8": 0.82, + "pass@16": 0.865, + "pass@32": 0.898, + "pass@64": 0.913, + "pass@128": 0.919 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.998, + "3": 0.996, + "4": 0.981, + "5": 0.951, + "6": 0.958, + "7": 0.86, + "8": 0.803, + "9": 0.718, + "10": 0.617 + }, + "pass@2": { + "2": 0.999, + "3": 0.997, + "4": 0.989, + "5": 0.968, + "6": 0.968, + "7": 0.907, + "8": 0.861, + "9": 0.804, + "10": 0.686 + }, + "pass@4": { + "2": 0.999, + "3": 0.997, + "4": 0.992, + "5": 0.979, + "6": 0.98, + "7": 0.95, + "8": 0.922, + "9": 0.891, + "10": 0.753 + }, + "pass@8": { + "2": 0.999, + "3": 0.997, + "4": 0.995, + "5": 0.986, + "6": 0.987, + "7": 0.976, + "8": 0.967, + "9": 0.94, + "10": 0.82 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.995, + "5": 0.988, + "6": 0.994, + "7": 0.989, + "8": 0.991, + "9": 0.974, + "10": 0.865 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.996, + "5": 0.991, + "6": 0.995, + "7": 0.99, + "8": 0.995, + "9": 0.991, + "10": 0.898 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.997, + "5": 0.992, + "6": 0.996, + "7": 0.994, + "8": 0.995, + "9": 0.993, + "10": 0.913 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.998, + "5": 0.996, + "6": 0.996, + "7": 0.995, + "8": 0.996, + "9": 0.996, + "10": 0.919 + } + }, + "total_pass_at_k": { + "pass@1": 0.8757777777777778, + "pass@2": 0.9087777777777778, + "pass@4": 0.9403333333333334, + "pass@8": 0.963, + "pass@16": 0.9773333333333334, + "pass@32": 0.984, + "pass@64": 0.9866666666666667, + "pass@128": 0.9884444444444445 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block8-epoch1-op11-14_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5580736543909348, + "pass@2": 0.6203966005665722, + "pass@4": 0.7252124645892352, + "pass@8": 0.8130311614730878, + "pass@16": 0.8725212464589235, + "pass@32": 0.9008498583569405, + "pass@64": 0.9235127478753541, + "pass@128": 0.9376770538243626 + }, + "12": { + "pass@1": 0.5649717514124294, + "pass@2": 0.6412429378531074, + "pass@4": 0.7203389830508474, + "pass@8": 0.7824858757062146, + "pass@16": 0.8107344632768362, + "pass@32": 0.8389830508474576, + "pass@64": 0.8502824858757062, + "pass@128": 0.8587570621468926 + }, + "13": { + "pass@1": 0.45141065830721006, + "pass@2": 0.5266457680250783, + "pass@4": 0.5736677115987461, + "pass@8": 0.6300940438871473, + "pass@16": 0.7147335423197492, + "pass@32": 0.7554858934169278, + "pass@64": 0.7931034482758621, + "pass@128": 0.8244514106583072 + }, + "14": { + "pass@1": 0.411214953271028, + "pass@2": 0.43302180685358255, + "pass@4": 0.4766355140186916, + "pass@8": 0.5514018691588785, + "pass@16": 0.616822429906542, + "pass@32": 0.6978193146417445, + "pass@64": 0.7476635514018691, + "pass@128": 0.7819314641744548 + }, + "15": { + "pass@1": 0.27167630057803466, + "pass@2": 0.32947976878612717, + "pass@4": 0.3930635838150289, + "pass@8": 0.47398843930635837, + "pass@16": 0.5346820809248555, + "pass@32": 0.6040462427745664, + "pass@64": 0.6647398843930635, + "pass@128": 0.7398843930635838 + }, + "16": { + "pass@1": 0.23870967741935484, + "pass@2": 0.2870967741935484, + "pass@4": 0.3225806451612903, + "pass@8": 0.36451612903225805, + "pass@16": 0.4129032258064516, + "pass@32": 0.4612903225806452, + "pass@64": 0.5129032258064516, + "pass@128": 0.5483870967741935 + }, + "17": { + "pass@1": 0.18208092485549132, + "pass@2": 0.21965317919075145, + "pass@4": 0.2658959537572254, + "pass@8": 0.2947976878612717, + "pass@16": 0.3179190751445087, + "pass@32": 0.36416184971098264, + "pass@64": 0.42196531791907516, + "pass@128": 0.4653179190751445 + }, + "18": { + "pass@1": 0.10857142857142857, + "pass@2": 0.13428571428571429, + "pass@4": 0.16857142857142857, + "pass@8": 0.19714285714285715, + "pass@16": 0.2342857142857143, + "pass@32": 0.26857142857142857, + "pass@64": 0.31142857142857144, + "pass@128": 0.36 + }, + "19": { + "pass@1": 0.042042042042042045, + "pass@2": 0.06906906906906907, + "pass@4": 0.09309309309309309, + "pass@8": 0.1111111111111111, + "pass@16": 0.13813813813813813, + "pass@32": 0.15915915915915915, + "pass@64": 0.1981981981981982, + "pass@128": 0.21621621621621623 + }, + "20": { + "pass@1": 0.05405405405405406, + "pass@2": 0.07507507507507508, + "pass@4": 0.10510510510510511, + "pass@8": 0.13513513513513514, + "pass@16": 0.15315315315315314, + "pass@32": 0.17417417417417416, + "pass@64": 0.1891891891891892, + "pass@128": 0.21021021021021022 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5580736543909348, + "12": 0.5649717514124294, + "13": 0.45141065830721006, + "14": 0.411214953271028, + "15": 0.27167630057803466, + "16": 0.23870967741935484, + "17": 0.18208092485549132, + "18": 0.10857142857142857, + "19": 0.042042042042042045, + "20": 0.05405405405405406 + }, + "pass@2": { + "11": 0.6203966005665722, + "12": 0.6412429378531074, + "13": 0.5266457680250783, + "14": 0.43302180685358255, + "15": 0.32947976878612717, + "16": 0.2870967741935484, + "17": 0.21965317919075145, + "18": 0.13428571428571429, + "19": 0.06906906906906907, + "20": 0.07507507507507508 + }, + "pass@4": { + "11": 0.7252124645892352, + "12": 0.7203389830508474, + "13": 0.5736677115987461, + "14": 0.4766355140186916, + "15": 0.3930635838150289, + "16": 0.3225806451612903, + "17": 0.2658959537572254, + "18": 0.16857142857142857, + "19": 0.09309309309309309, + "20": 0.10510510510510511 + }, + "pass@8": { + "11": 0.8130311614730878, + "12": 0.7824858757062146, + "13": 0.6300940438871473, + "14": 0.5514018691588785, + "15": 0.47398843930635837, + "16": 0.36451612903225805, + "17": 0.2947976878612717, + "18": 0.19714285714285715, + "19": 0.1111111111111111, + "20": 0.13513513513513514 + }, + "pass@16": { + "11": 0.8725212464589235, + "12": 0.8107344632768362, + "13": 0.7147335423197492, + "14": 0.616822429906542, + "15": 0.5346820809248555, + "16": 0.4129032258064516, + "17": 0.3179190751445087, + "18": 0.2342857142857143, + "19": 0.13813813813813813, + "20": 0.15315315315315314 + }, + "pass@32": { + "11": 0.9008498583569405, + "12": 0.8389830508474576, + "13": 0.7554858934169278, + "14": 0.6978193146417445, + "15": 0.6040462427745664, + "16": 0.4612903225806452, + "17": 0.36416184971098264, + "18": 0.26857142857142857, + "19": 0.15915915915915915, + "20": 0.17417417417417416 + }, + "pass@64": { + "11": 0.9235127478753541, + "12": 0.8502824858757062, + "13": 0.7931034482758621, + "14": 0.7476635514018691, + "15": 0.6647398843930635, + "16": 0.5129032258064516, + "17": 0.42196531791907516, + "18": 0.31142857142857144, + "19": 0.1981981981981982, + "20": 0.1891891891891892 + }, + "pass@128": { + "11": 0.9376770538243626, + "12": 0.8587570621468926, + "13": 0.8244514106583072, + "14": 0.7819314641744548, + "15": 0.7398843930635838, + "16": 0.5483870967741935, + "17": 0.4653179190751445, + "18": 0.36, + "19": 0.21621621621621623, + "20": 0.21021021021021022 + } + }, + "total_pass_at_k": { + "pass@1": 0.2894502228826152, + "pass@2": 0.33491827637444277, + "pass@4": 0.3863298662704309, + "pass@8": 0.437444279346211, + "pass@16": 0.4823179791976226, + "pass@32": 0.5239227340267459, + "pass@64": 0.562555720653789, + "pass@128": 0.5955423476968796 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5619047619047619, + "pass@2": 0.6126984126984127, + "pass@4": 0.7111111111111111, + "pass@8": 0.8380952380952381, + "pass@16": 0.9015873015873016, + "pass@32": 0.9333333333333333, + "pass@64": 0.9428571428571428, + "pass@128": 0.9587301587301588 + }, + "12": { + "pass@1": 0.5749235474006116, + "pass@2": 0.6422018348623854, + "pass@4": 0.6941896024464832, + "pass@8": 0.7614678899082569, + "pass@16": 0.8195718654434251, + "pass@32": 0.8532110091743119, + "pass@64": 0.8776758409785933, + "pass@128": 0.8929663608562691 + }, + "13": { + "pass@1": 0.4382716049382716, + "pass@2": 0.5092592592592593, + "pass@4": 0.5709876543209876, + "pass@8": 0.6481481481481481, + "pass@16": 0.7160493827160493, + "pass@32": 0.7469135802469136, + "pass@64": 0.7808641975308642, + "pass@128": 0.7901234567901234 + }, + "14": { + "pass@1": 0.34124629080118696, + "pass@2": 0.40059347181008903, + "pass@4": 0.4362017804154303, + "pass@8": 0.4836795252225519, + "pass@16": 0.5548961424332344, + "pass@32": 0.6261127596439169, + "pass@64": 0.7032640949554896, + "pass@128": 0.7507418397626113 + }, + "15": { + "pass@1": 0.2962962962962963, + "pass@2": 0.3271604938271605, + "pass@4": 0.37037037037037035, + "pass@8": 0.4166666666666667, + "pass@16": 0.5185185185185185, + "pass@32": 0.6080246913580247, + "pass@64": 0.6820987654320988, + "pass@128": 0.7283950617283951 + }, + "16": { + "pass@1": 0.23893805309734514, + "pass@2": 0.3008849557522124, + "pass@4": 0.3775811209439528, + "pass@8": 0.4247787610619469, + "pass@16": 0.45427728613569324, + "pass@32": 0.48672566371681414, + "pass@64": 0.5309734513274337, + "pass@128": 0.5811209439528023 + }, + "17": { + "pass@1": 0.1566265060240964, + "pass@2": 0.21987951807228914, + "pass@4": 0.25903614457831325, + "pass@8": 0.28313253012048195, + "pass@16": 0.3313253012048193, + "pass@32": 0.3704819277108434, + "pass@64": 0.4126506024096386, + "pass@128": 0.4789156626506024 + }, + "18": { + "pass@1": 0.16774193548387098, + "pass@2": 0.2161290322580645, + "pass@4": 0.24193548387096775, + "pass@8": 0.26129032258064516, + "pass@16": 0.2838709677419355, + "pass@32": 0.31290322580645163, + "pass@64": 0.35161290322580646, + "pass@128": 0.3967741935483871 + }, + "19": { + "pass@1": 0.06646525679758308, + "pass@2": 0.08157099697885196, + "pass@4": 0.09667673716012085, + "pass@8": 0.1299093655589124, + "pass@16": 0.1510574018126888, + "pass@32": 0.18126888217522658, + "pass@64": 0.19939577039274925, + "pass@128": 0.22356495468277945 + }, + "20": { + "pass@1": 0.04504504504504504, + "pass@2": 0.06006006006006006, + "pass@4": 0.08408408408408409, + "pass@8": 0.10810810810810811, + "pass@16": 0.12912912912912913, + "pass@32": 0.14714714714714713, + "pass@64": 0.18018018018018017, + "pass@128": 0.2132132132132132 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5619047619047619, + "12": 0.5749235474006116, + "13": 0.4382716049382716, + "14": 0.34124629080118696, + "15": 0.2962962962962963, + "16": 0.23893805309734514, + "17": 0.1566265060240964, + "18": 0.16774193548387098, + "19": 0.06646525679758308, + "20": 0.04504504504504504 + }, + "pass@2": { + "11": 0.6126984126984127, + "12": 0.6422018348623854, + "13": 0.5092592592592593, + "14": 0.40059347181008903, + "15": 0.3271604938271605, + "16": 0.3008849557522124, + "17": 0.21987951807228914, + "18": 0.2161290322580645, + "19": 0.08157099697885196, + "20": 0.06006006006006006 + }, + "pass@4": { + "11": 0.7111111111111111, + "12": 0.6941896024464832, + "13": 0.5709876543209876, + "14": 0.4362017804154303, + "15": 0.37037037037037035, + "16": 0.3775811209439528, + "17": 0.25903614457831325, + "18": 0.24193548387096775, + "19": 0.09667673716012085, + "20": 0.08408408408408409 + }, + "pass@8": { + "11": 0.8380952380952381, + "12": 0.7614678899082569, + "13": 0.6481481481481481, + "14": 0.4836795252225519, + "15": 0.4166666666666667, + "16": 0.4247787610619469, + "17": 0.28313253012048195, + "18": 0.26129032258064516, + "19": 0.1299093655589124, + "20": 0.10810810810810811 + }, + "pass@16": { + "11": 0.9015873015873016, + "12": 0.8195718654434251, + "13": 0.7160493827160493, + "14": 0.5548961424332344, + "15": 0.5185185185185185, + "16": 0.45427728613569324, + "17": 0.3313253012048193, + "18": 0.2838709677419355, + "19": 0.1510574018126888, + "20": 0.12912912912912913 + }, + "pass@32": { + "11": 0.9333333333333333, + "12": 0.8532110091743119, + "13": 0.7469135802469136, + "14": 0.6261127596439169, + "15": 0.6080246913580247, + "16": 0.48672566371681414, + "17": 0.3704819277108434, + "18": 0.31290322580645163, + "19": 0.18126888217522658, + "20": 0.14714714714714713 + }, + "pass@64": { + "11": 0.9428571428571428, + "12": 0.8776758409785933, + "13": 0.7808641975308642, + "14": 0.7032640949554896, + "15": 0.6820987654320988, + "16": 0.5309734513274337, + "17": 0.4126506024096386, + "18": 0.35161290322580646, + "19": 0.19939577039274925, + "20": 0.18018018018018017 + }, + "pass@128": { + "11": 0.9587301587301588, + "12": 0.8929663608562691, + "13": 0.7901234567901234, + "14": 0.7507418397626113, + "15": 0.7283950617283951, + "16": 0.5811209439528023, + "17": 0.4789156626506024, + "18": 0.3967741935483871, + "19": 0.22356495468277945, + "20": 0.2132132132132132 + } + }, + "total_pass_at_k": { + "pass@1": 0.28728606356968217, + "pass@2": 0.33557457212713937, + "pass@4": 0.38264058679706603, + "pass@8": 0.433679706601467, + "pass@16": 0.4841075794621027, + "pass@32": 0.5247555012224939, + "pass@64": 0.5644865525672371, + "pass@128": 0.5999388753056235 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5903614457831325, + "pass@2": 0.641566265060241, + "pass@4": 0.713855421686747, + "pass@8": 0.8283132530120482, + "pass@16": 0.8795180722891566, + "pass@32": 0.9096385542168675, + "pass@64": 0.9457831325301205, + "pass@128": 0.9608433734939759 + }, + "12": { + "pass@1": 0.5235109717868338, + "pass@2": 0.5987460815047022, + "pass@4": 0.6865203761755486, + "pass@8": 0.768025078369906, + "pass@16": 0.8213166144200627, + "pass@32": 0.8401253918495298, + "pass@64": 0.8557993730407524, + "pass@128": 0.8683385579937304 + }, + "13": { + "pass@1": 0.45938375350140054, + "pass@2": 0.5294117647058824, + "pass@4": 0.6050420168067226, + "pass@8": 0.6974789915966386, + "pass@16": 0.7366946778711485, + "pass@32": 0.7759103641456583, + "pass@64": 0.8095238095238095, + "pass@128": 0.834733893557423 + }, + "14": { + "pass@1": 0.38011695906432746, + "pass@2": 0.4298245614035088, + "pass@4": 0.48830409356725146, + "pass@8": 0.5350877192982456, + "pass@16": 0.6052631578947368, + "pass@32": 0.6695906432748538, + "pass@64": 0.7251461988304093, + "pass@128": 0.7514619883040936 + }, + "15": { + "pass@1": 0.31212121212121213, + "pass@2": 0.35454545454545455, + "pass@4": 0.4303030303030303, + "pass@8": 0.49393939393939396, + "pass@16": 0.5575757575757576, + "pass@32": 0.6121212121212121, + "pass@64": 0.6909090909090909, + "pass@128": 0.7484848484848485 + }, + "16": { + "pass@1": 0.23931623931623933, + "pass@2": 0.2905982905982906, + "pass@4": 0.32763532763532766, + "pass@8": 0.3561253561253561, + "pass@16": 0.42165242165242167, + "pass@32": 0.47863247863247865, + "pass@64": 0.5384615384615384, + "pass@128": 0.5982905982905983 + }, + "17": { + "pass@1": 0.17391304347826086, + "pass@2": 0.2111801242236025, + "pass@4": 0.2422360248447205, + "pass@8": 0.2795031055900621, + "pass@16": 0.30745341614906835, + "pass@32": 0.36645962732919257, + "pass@64": 0.43788819875776397, + "pass@128": 0.4658385093167702 + }, + "18": { + "pass@1": 0.1411764705882353, + "pass@2": 0.17941176470588235, + "pass@4": 0.2088235294117647, + "pass@8": 0.23529411764705882, + "pass@16": 0.2676470588235294, + "pass@32": 0.3176470588235294, + "pass@64": 0.35, + "pass@128": 0.40294117647058825 + }, + "19": { + "pass@1": 0.06845238095238096, + "pass@2": 0.10119047619047619, + "pass@4": 0.13690476190476192, + "pass@8": 0.14285714285714285, + "pass@16": 0.16964285714285715, + "pass@32": 0.18452380952380953, + "pass@64": 0.19642857142857142, + "pass@128": 0.2261904761904762 + }, + "20": { + "pass@1": 0.0748502994011976, + "pass@2": 0.10179640718562874, + "pass@4": 0.11377245508982035, + "pass@8": 0.1407185628742515, + "pass@16": 0.16766467065868262, + "pass@32": 0.19760479041916168, + "pass@64": 0.218562874251497, + "pass@128": 0.23652694610778444 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5903614457831325, + "12": 0.5235109717868338, + "13": 0.45938375350140054, + "14": 0.38011695906432746, + "15": 0.31212121212121213, + "16": 0.23931623931623933, + "17": 0.17391304347826086, + "18": 0.1411764705882353, + "19": 0.06845238095238096, + "20": 0.0748502994011976 + }, + "pass@2": { + "11": 0.641566265060241, + "12": 0.5987460815047022, + "13": 0.5294117647058824, + "14": 0.4298245614035088, + "15": 0.35454545454545455, + "16": 0.2905982905982906, + "17": 0.2111801242236025, + "18": 0.17941176470588235, + "19": 0.10119047619047619, + "20": 0.10179640718562874 + }, + "pass@4": { + "11": 0.713855421686747, + "12": 0.6865203761755486, + "13": 0.6050420168067226, + "14": 0.48830409356725146, + "15": 0.4303030303030303, + "16": 0.32763532763532766, + "17": 0.2422360248447205, + "18": 0.2088235294117647, + "19": 0.13690476190476192, + "20": 0.11377245508982035 + }, + "pass@8": { + "11": 0.8283132530120482, + "12": 0.768025078369906, + "13": 0.6974789915966386, + "14": 0.5350877192982456, + "15": 0.49393939393939396, + "16": 0.3561253561253561, + "17": 0.2795031055900621, + "18": 0.23529411764705882, + "19": 0.14285714285714285, + "20": 0.1407185628742515 + }, + "pass@16": { + "11": 0.8795180722891566, + "12": 0.8213166144200627, + "13": 0.7366946778711485, + "14": 0.6052631578947368, + "15": 0.5575757575757576, + "16": 0.42165242165242167, + "17": 0.30745341614906835, + "18": 0.2676470588235294, + "19": 0.16964285714285715, + "20": 0.16766467065868262 + }, + "pass@32": { + "11": 0.9096385542168675, + "12": 0.8401253918495298, + "13": 0.7759103641456583, + "14": 0.6695906432748538, + "15": 0.6121212121212121, + "16": 0.47863247863247865, + "17": 0.36645962732919257, + "18": 0.3176470588235294, + "19": 0.18452380952380953, + "20": 0.19760479041916168 + }, + "pass@64": { + "11": 0.9457831325301205, + "12": 0.8557993730407524, + "13": 0.8095238095238095, + "14": 0.7251461988304093, + "15": 0.6909090909090909, + "16": 0.5384615384615384, + "17": 0.43788819875776397, + "18": 0.35, + "19": 0.19642857142857142, + "20": 0.218562874251497 + }, + "pass@128": { + "11": 0.9608433734939759, + "12": 0.8683385579937304, + "13": 0.834733893557423, + "14": 0.7514619883040936, + "15": 0.7484848484848485, + "16": 0.5982905982905983, + "17": 0.4658385093167702, + "18": 0.40294117647058825, + "19": 0.2261904761904762, + "20": 0.23652694610778444 + } + }, + "total_pass_at_k": { + "pass@1": 0.296164139161463, + "pass@2": 0.34374070770145704, + "pass@4": 0.3951828724353256, + "pass@8": 0.44751709782931903, + "pass@16": 0.4933095450490633, + "pass@32": 0.5352363960749331, + "pass@64": 0.5768658935474279, + "pass@128": 0.6095747844186739 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.57, + "pass@2": 0.625, + "pass@4": 0.717, + "pass@8": 0.826, + "pass@16": 0.884, + "pass@32": 0.914, + "pass@64": 0.937, + "pass@128": 0.952 + }, + "12": { + "pass@1": 0.555, + "pass@2": 0.628, + "pass@4": 0.701, + "pass@8": 0.771, + "pass@16": 0.817, + "pass@32": 0.844, + "pass@64": 0.861, + "pass@128": 0.873 + }, + "13": { + "pass@1": 0.45, + "pass@2": 0.522, + "pass@4": 0.584, + "pass@8": 0.66, + "pass@16": 0.723, + "pass@32": 0.76, + "pass@64": 0.795, + "pass@128": 0.817 + }, + "14": { + "pass@1": 0.377, + "pass@2": 0.421, + "pass@4": 0.467, + "pass@8": 0.523, + "pass@16": 0.592, + "pass@32": 0.664, + "pass@64": 0.725, + "pass@128": 0.761 + }, + "15": { + "pass@1": 0.293, + "pass@2": 0.337, + "pass@4": 0.398, + "pass@8": 0.462, + "pass@16": 0.537, + "pass@32": 0.608, + "pass@64": 0.679, + "pass@128": 0.739 + }, + "16": { + "pass@1": 0.239, + "pass@2": 0.293, + "pass@4": 0.343, + "pass@8": 0.382, + "pass@16": 0.43, + "pass@32": 0.476, + "pass@64": 0.528, + "pass@128": 0.577 + }, + "17": { + "pass@1": 0.171, + "pass@2": 0.217, + "pass@4": 0.256, + "pass@8": 0.286, + "pass@16": 0.319, + "pass@32": 0.367, + "pass@64": 0.424, + "pass@128": 0.47 + }, + "18": { + "pass@1": 0.138, + "pass@2": 0.175, + "pass@4": 0.205, + "pass@8": 0.23, + "pass@16": 0.261, + "pass@32": 0.299, + "pass@64": 0.337, + "pass@128": 0.386 + }, + "19": { + "pass@1": 0.059, + "pass@2": 0.084, + "pass@4": 0.109, + "pass@8": 0.128, + "pass@16": 0.153, + "pass@32": 0.175, + "pass@64": 0.198, + "pass@128": 0.222 + }, + "20": { + "pass@1": 0.058, + "pass@2": 0.079, + "pass@4": 0.101, + "pass@8": 0.128, + "pass@16": 0.15, + "pass@32": 0.173, + "pass@64": 0.196, + "pass@128": 0.22 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.57, + "12": 0.555, + "13": 0.45, + "14": 0.377, + "15": 0.293, + "16": 0.239, + "17": 0.171, + "18": 0.138, + "19": 0.059, + "20": 0.058 + }, + "pass@2": { + "11": 0.625, + "12": 0.628, + "13": 0.522, + "14": 0.421, + "15": 0.337, + "16": 0.293, + "17": 0.217, + "18": 0.175, + "19": 0.084, + "20": 0.079 + }, + "pass@4": { + "11": 0.717, + "12": 0.701, + "13": 0.584, + "14": 0.467, + "15": 0.398, + "16": 0.343, + "17": 0.256, + "18": 0.205, + "19": 0.109, + "20": 0.101 + }, + "pass@8": { + "11": 0.826, + "12": 0.771, + "13": 0.66, + "14": 0.523, + "15": 0.462, + "16": 0.382, + "17": 0.286, + "18": 0.23, + "19": 0.128, + "20": 0.128 + }, + "pass@16": { + "11": 0.884, + "12": 0.817, + "13": 0.723, + "14": 0.592, + "15": 0.537, + "16": 0.43, + "17": 0.319, + "18": 0.261, + "19": 0.153, + "20": 0.15 + }, + "pass@32": { + "11": 0.914, + "12": 0.844, + "13": 0.76, + "14": 0.664, + "15": 0.608, + "16": 0.476, + "17": 0.367, + "18": 0.299, + "19": 0.175, + "20": 0.173 + }, + "pass@64": { + "11": 0.937, + "12": 0.861, + "13": 0.795, + "14": 0.725, + "15": 0.679, + "16": 0.528, + "17": 0.424, + "18": 0.337, + "19": 0.198, + "20": 0.196 + }, + "pass@128": { + "11": 0.952, + "12": 0.873, + "13": 0.817, + "14": 0.761, + "15": 0.739, + "16": 0.577, + "17": 0.47, + "18": 0.386, + "19": 0.222, + "20": 0.22 + } + }, + "total_pass_at_k": { + "pass@1": 0.291, + "pass@2": 0.3381, + "pass@4": 0.3881, + "pass@8": 0.4396, + "pass@16": 0.4866, + "pass@32": 0.528, + "pass@64": 0.568, + "pass@128": 0.6017 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block8-epoch1-op11-14/summary.csv b/composition/SDAR-100M-block8-epoch1-op11-14/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..3721bdf8918b392ffd9206d85fc7e20a288f7a44 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op11-14/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch1-op11-14,0.9792821180555555,0.6068109375,0.7832446546052632,0.0,0.0,0.0,198.730515625,362.87332109375,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block8-epoch1-op11-14/summary.json b/composition/SDAR-100M-block8-epoch1-op11-14/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a9f646ea9ab8162e9027924576d3c00141881e4c --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op11-14/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch1-op11-14", + "id_acc": 0.9792821180555555, + "ood_acc": 0.6068109375, + "total_acc": 0.7832446546052632, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 198.730515625, + "ood_avg_resp_len": 362.87332109375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 632605, + "answer_accuracy": 0.7812561749130572, + "avg_response_len": 271.58874955540625, + "resp_tokens_sum": 219913015, + "pass_at_k": { + "pass@1": 0.7812561749130572, + "pass@2": 0.8215596962977374, + "pass@4": 0.8519572684124993, + "pass@8": 0.8755206334283433, + "pass@16": 0.8944649819697402, + "pass@32": 0.9100665792514822, + "pass@64": 0.9227670820634137, + "pass@128": 0.9326588681631363 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9231425147492626, + "pass@2": 0.9383285572433974, + "pass@4": 0.9473531416979957, + "pass@8": 0.9537757890092359, + "pass@16": 0.9589878410934526, + "pass@32": 0.9634827340913612, + "pass@64": 0.9671551508567272, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.999123831775701, + "pass@2": 0.999758536316138, + "pass@4": 0.9999827985870926, + "pass@8": 0.9999999340639316, + "pass@16": 0.999999999999756, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9958428899082569, + "pass@2": 0.9977124419080643, + "pass@4": 0.9987991609993705, + "pass@8": 0.9995675174167592, + "pass@16": 0.9999476070371531, + "pass@32": 0.9999995668075918, + "pass@64": 0.9999999999993109, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9838134765625, + "pass@2": 0.9859390378937007, + "pass@4": 0.9883570163104611, + "pass@8": 0.9906581655190253, + "pass@16": 0.9926442820736442, + "pass@32": 0.9945717745930672, + "pass@64": 0.9963074639597295, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9704117063492064, + "pass@2": 0.9771399668791401, + "pass@4": 0.9815450669856742, + "pass@8": 0.9853588892526388, + "pass@16": 0.9894869670242535, + "pass@32": 0.9933388347853764, + "pass@64": 0.9967933357643882, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.992936002994012, + "pass@2": 0.9954846704229336, + "pass@4": 0.9969230995264813, + "pass@8": 0.9975195058071472, + "pass@16": 0.9980010177318054, + "pass@32": 0.9987468819391533, + "pass@64": 0.9996345890895375, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9860623150887574, + "pass@2": 0.9925427625448447, + "pass@4": 0.9958278568544315, + "pass@8": 0.9969092535432367, + "pass@16": 0.9970388200006978, + "pass@32": 0.9970414196166396, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9829477163461539, + "pass@2": 0.9896030499192406, + "pass@4": 0.9923302090243528, + "pass@8": 0.9939964803271766, + "pass@16": 0.9951222806297688, + "pass@32": 0.9960540651646855, + "pass@64": 0.9967025035329475, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9732394366197183, + "pass@2": 0.9847652351114562, + "pass@4": 0.9913186990710667, + "pass@8": 0.9961890503448468, + "pass@16": 0.9991175807085956, + "pass@32": 0.9999597333809449, + "pass@64": 0.9999999878674508, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9234684844192634, + "pass@2": 0.9449662343021572, + "pass@4": 0.956891268637454, + "pass@8": 0.9643693469153156, + "pass@16": 0.9700565621001835, + "pass@32": 0.9749692654639327, + "pass@64": 0.9787854577528349, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.8851739053672316, + "pass@2": 0.9140503358690337, + "pass@4": 0.9297601067874989, + "pass@8": 0.9404788197530561, + "pass@16": 0.94967725068733, + "pass@32": 0.9586145399733833, + "pass@64": 0.9651696834736039, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.8151449843260188, + "pass@2": 0.8586179127687409, + "pass@4": 0.8867077769275696, + "pass@8": 0.9085025252861926, + "pass@16": 0.9271233872904527, + "pass@32": 0.9449082791298123, + "pass@64": 0.9610983812381588, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.7620959890965732, + "pass@2": 0.8263109561655264, + "pass@4": 0.8645924224425215, + "pass@8": 0.8889128593760814, + "pass@16": 0.9094888442681421, + "pass@32": 0.928986852397342, + "pass@64": 0.9461391387088692, + "pass@128": 0.956386292834891 + }, + "15": { + "pass@1": 0.6877257947976878, + "pass@2": 0.7639100262846479, + "pass@4": 0.8192633533705684, + "pass@8": 0.8602656226549914, + "pass@16": 0.8904892628915874, + "pass@32": 0.912211514532011, + "pass@64": 0.9267263236137377, + "pass@128": 0.9364161849710982 + }, + "16": { + "pass@1": 0.5722278225806452, + "pass@2": 0.6708578073406148, + "pass@4": 0.7376889657945979, + "pass@8": 0.7811584454338113, + "pass@16": 0.8123913701188059, + "pass@32": 0.8397369710307722, + "pass@64": 0.8647941957421232, + "pass@128": 0.8870967741935484 + }, + "17": { + "pass@1": 0.46899837427745666, + "pass@2": 0.5686220614673886, + "pass@4": 0.6493129596155967, + "pass@8": 0.7107867389096841, + "pass@16": 0.7572407054462839, + "pass@32": 0.7920715450323713, + "pass@64": 0.8177465613909459, + "pass@128": 0.8352601156069365 + }, + "18": { + "pass@1": 0.3928348214285714, + "pass@2": 0.4923548228346458, + "pass@4": 0.57355253361187, + "pass@8": 0.6381375838952569, + "pass@16": 0.6909666831451384, + "pass@32": 0.7355241528015719, + "pass@64": 0.7725518509793133, + "pass@128": 0.8028571428571428 + }, + "19": { + "pass@1": 0.2802881006006006, + "pass@2": 0.3656185713272326, + "pass@4": 0.44623301085112094, + "pass@8": 0.5207589952897529, + "pass@16": 0.5834385538957976, + "pass@32": 0.6326537813669033, + "pass@64": 0.6755366923342895, + "pass@128": 0.7147147147147147 + }, + "20": { + "pass@1": 0.27888044294294295, + "pass@2": 0.3691022469319712, + "pass@4": 0.45260455928369264, + "pass@8": 0.5256639851382408, + "pass@16": 0.5888334762511734, + "pass@32": 0.6417662504776702, + "pass@64": 0.6828783160667011, + "pass@128": 0.7177177177177178 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 644630, + "answer_accuracy": 0.7850618667186282, + "avg_response_len": 281.62891294816836, + "resp_tokens_sum": 231251133, + "pass_at_k": { + "pass@1": 0.7850618667186282, + "pass@2": 0.8254338993868824, + "pass@4": 0.8564373537193234, + "pass@8": 0.880388215978897, + "pass@16": 0.8992122606879791, + "pass@32": 0.9141301175895304, + "pass@64": 0.9260407494275895, + "pass@128": 0.9353078721745908 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9112586273006135, + "pass@2": 0.9263716879619345, + "pass@4": 0.9368261261245718, + "pass@8": 0.9442747604605302, + "pass@16": 0.9497976670404737, + "pass@32": 0.954829192728206, + "pass@64": 0.9594253227868381, + "pass@128": 0.9631901840490797 + }, + "2": { + "pass@1": 0.9988839285714286, + "pass@2": 0.999595753655793, + "pass@4": 0.9999491804595854, + "pass@8": 0.9999993314708793, + "pass@16": 0.9999999999510504, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.993655303030303, + "pass@2": 0.9962635707468384, + "pass@4": 0.9979543449114315, + "pass@8": 0.9990869648235982, + "pass@16": 0.9997729214698883, + "pass@32": 0.9999890704069151, + "pass@64": 0.9999999971219998, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9963711168639053, + "pass@2": 0.9970891039230303, + "pass@4": 0.9973102368306034, + "pass@8": 0.9975660626410165, + "pass@16": 0.9980247014241563, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9873414855072464, + "pass@2": 0.990341136026475, + "pass@4": 0.9926458491601591, + "pass@8": 0.9944704905370954, + "pass@16": 0.9957758227738857, + "pass@32": 0.9971598125688518, + "pass@64": 0.9989273079949132, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9867478510028653, + "pass@2": 0.9883521958125577, + "pass@4": 0.9886221656927555, + "pass@8": 0.9887177604361171, + "pass@16": 0.9888968481375355, + "pass@32": 0.9892550143266475, + "pass@64": 0.9899713467048711, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.9869942196531792, + "pass@2": 0.9909770976514497, + "pass@4": 0.993617887810989, + "pass@8": 0.9958918476913389, + "pass@16": 0.9978098101358188, + "pass@32": 0.9990707212295816, + "pass@64": 0.9998278510372128, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9852992957746479, + "pass@2": 0.9926118720195188, + "pass@4": 0.9960676550114335, + "pass@8": 0.9984719701911067, + "pass@16": 0.9997337999101152, + "pass@32": 0.9999920888176553, + "pass@64": 0.9999999988794946, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9658495508982036, + "pass@2": 0.9738565438021595, + "pass@4": 0.977178885479884, + "pass@8": 0.9799258010494726, + "pass@16": 0.9827626400173137, + "pass@32": 0.9852917113826465, + "pass@64": 0.9880352427034073, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.9304405120481928, + "pass@2": 0.9519117315719565, + "pass@4": 0.9657082994520263, + "pass@8": 0.9757115076074933, + "pass@16": 0.9825771503102283, + "pass@32": 0.9865760138899352, + "pass@64": 0.9894810140158518, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8922413793103449, + "pass@2": 0.9128235072692715, + "pass@4": 0.9231989963832886, + "pass@8": 0.9287502632219307, + "pass@16": 0.9322745427114282, + "pass@32": 0.935291197697357, + "pass@64": 0.9379938824694157, + "pass@128": 0.9404388714733543 + }, + "13": { + "pass@1": 0.7765449929971989, + "pass@2": 0.8236345227067203, + "pass@4": 0.8546561067996749, + "pass@8": 0.8796959667664824, + "pass@16": 0.9017360202584387, + "pass@32": 0.9197525073028779, + "pass@64": 0.9340349790683912, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.7601653874269005, + "pass@2": 0.8235343423354977, + "pass@4": 0.8647729458708009, + "pass@8": 0.8932804906568209, + "pass@16": 0.9130388497560374, + "pass@32": 0.9267935902050748, + "pass@64": 0.9365736229030256, + "pass@128": 0.9415204678362573 + }, + "15": { + "pass@1": 0.6853219696969697, + "pass@2": 0.7700440676449535, + "pass@4": 0.8293955920282687, + "pass@8": 0.8725181459664744, + "pass@16": 0.9028329731845267, + "pass@32": 0.9229171920966758, + "pass@64": 0.9364439826397638, + "pass@128": 0.9454545454545454 + }, + "16": { + "pass@1": 0.5576923076923077, + "pass@2": 0.6493164215851224, + "pass@4": 0.7176616638197999, + "pass@8": 0.7699788625723991, + "pass@16": 0.8121033872254676, + "pass@32": 0.8467035952508719, + "pass@64": 0.8723326079810598, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.4888392857142857, + "pass@2": 0.5898313322247761, + "pass@4": 0.6642076815320447, + "pass@8": 0.714858050845223, + "pass@16": 0.7517769648613764, + "pass@32": 0.781466465575029, + "pass@64": 0.8059220394919001, + "pass@128": 0.8260869565217391 + }, + "18": { + "pass@1": 0.4178768382352941, + "pass@2": 0.5259965551181103, + "pass@4": 0.6213802793952226, + "pass@8": 0.6972919448114913, + "pass@16": 0.755379792189837, + "pass@32": 0.8004193580795406, + "pass@64": 0.8365682411731703, + "pass@128": 0.8647058823529412 + }, + "19": { + "pass@1": 0.28429594494047616, + "pass@2": 0.3704518624625045, + "pass@4": 0.4515365584882246, + "pass@8": 0.5245154324308915, + "pass@16": 0.5899039910455528, + "pass@32": 0.6469910765958351, + "pass@64": 0.6931867993172561, + "pass@128": 0.7261904761904762 + }, + "20": { + "pass@1": 0.2971790793413174, + "pass@2": 0.3988401150454995, + "pass@4": 0.48962655734649935, + "pass@8": 0.5629928497263833, + "pass@16": 0.6216007899549488, + "pass@32": 0.6680350459107165, + "pass@64": 0.7074075972497136, + "pass@128": 0.7425149700598802 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 627616, + "answer_accuracy": 0.7833919156414763, + "avg_response_len": 302.37864599976035, + "resp_tokens_sum": 242251257, + "pass_at_k": { + "pass@1": 0.7833919156414763, + "pass@2": 0.8234334880606494, + "pass@4": 0.8544341328988612, + "pass@8": 0.8790660161985339, + "pass@16": 0.8989971981402587, + "pass@32": 0.9150769735157885, + "pass@64": 0.9280018307463214, + "pass@128": 0.9392874261064068 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9262593283582089, + "pass@2": 0.9471449347749443, + "pass@4": 0.9614218745977648, + "pass@8": 0.972186851019286, + "pass@16": 0.9802954796338725, + "pass@32": 0.9857173184696202, + "pass@64": 0.9891864099503938, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9966071428571428, + "pass@2": 0.9971263357705287, + "pass@4": 0.9973159314907064, + "pass@8": 0.9974901553188764, + "pass@16": 0.9978149606299211, + "pass@32": 0.9983970753655793, + "pass@64": 0.9992913385826772, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961962463556852, + "pass@2": 0.9972359163012786, + "pass@4": 0.9981603342767286, + "pass@8": 0.9989546945154124, + "pass@16": 0.9996537420769658, + "pass@32": 0.9999711436906581, + "pass@64": 0.999999964811974, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9965049342105263, + "pass@2": 0.9970519207303035, + "pass@4": 0.9970759831007965, + "pass@8": 0.9970760233917945, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9765625, + "pass@2": 0.9826594343446041, + "pass@4": 0.9864391599947065, + "pass@8": 0.989338738990234, + "pass@16": 0.9920232898813793, + "pass@32": 0.9944213659459019, + "pass@64": 0.9963388991378888, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9905855678233438, + "pass@2": 0.9928459319655231, + "pass@4": 0.9949855764283407, + "pass@8": 0.9963629428393239, + "pass@16": 0.9968050483305237, + "pass@32": 0.996845217944392, + "pass@64": 0.9968454258674141, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9834107990506329, + "pass@2": 0.9895283065882586, + "pass@4": 0.9924962614443763, + "pass@8": 0.9935128758822745, + "pass@16": 0.993667318301558, + "pass@32": 0.9936708852185112, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9775713213213213, + "pass@2": 0.9852221808422593, + "pass@4": 0.9890983199172173, + "pass@8": 0.9915133611337327, + "pass@16": 0.9931262585622738, + "pass@32": 0.9938936237385995, + "pass@64": 0.9939935625945567, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9756079180064309, + "pass@2": 0.9861373965111273, + "pass@4": 0.9905033775922705, + "pass@8": 0.9926946650327054, + "pass@16": 0.9945845597918244, + "pass@32": 0.996690380430318, + "pass@64": 0.9988078891460136, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9223462301587302, + "pass@2": 0.9396516841644794, + "pass@4": 0.9498219314847549, + "pass@8": 0.9578415093872525, + "pass@16": 0.9652445350017391, + "pass@32": 0.9725243842288842, + "pass@64": 0.9786231035308165, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.885894495412844, + "pass@2": 0.9103525548411951, + "pass@4": 0.9245389569996405, + "pass@8": 0.9368986112374154, + "pass@16": 0.9480092706161772, + "pass@32": 0.9560379114235142, + "pass@64": 0.9609099339438009, + "pass@128": 0.963302752293578 + }, + "13": { + "pass@1": 0.7903404706790124, + "pass@2": 0.8470718503937011, + "pass@4": 0.8848459978382324, + "pass@8": 0.9121020749798062, + "pass@16": 0.9320574360481253, + "pass@32": 0.9462426775320625, + "pass@64": 0.9553111192670196, + "pass@128": 0.9629629629629629 + }, + "14": { + "pass@1": 0.7437175445103857, + "pass@2": 0.8075130260052802, + "pass@4": 0.8517123186011245, + "pass@8": 0.8822456090117424, + "pass@16": 0.9034756615081073, + "pass@32": 0.918763368316569, + "pass@64": 0.930841332255865, + "pass@128": 0.9436201780415431 + }, + "15": { + "pass@1": 0.6849440586419753, + "pass@2": 0.749585338291047, + "pass@4": 0.8000965229462056, + "pass@8": 0.8419278776098915, + "pass@16": 0.8742716438216978, + "pass@32": 0.897443883362062, + "pass@64": 0.9155924031250283, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.576926622418879, + "pass@2": 0.6769483253199544, + "pass@4": 0.7467605772176702, + "pass@8": 0.7960808524807924, + "pass@16": 0.8342169814205239, + "pass@32": 0.8674130918519873, + "pass@64": 0.8970240946976298, + "pass@128": 0.9203539823008849 + }, + "17": { + "pass@1": 0.45684299698795183, + "pass@2": 0.551935967294374, + "pass@4": 0.6280077681366633, + "pass@8": 0.6876106027403157, + "pass@16": 0.7364869488817384, + "pass@32": 0.7791478589543936, + "pass@64": 0.8181411107743274, + "pass@128": 0.858433734939759 + }, + "18": { + "pass@1": 0.43515625, + "pass@2": 0.5343476155702312, + "pass@4": 0.6127405430168001, + "pass@8": 0.6709429848652598, + "pass@16": 0.7154178520774157, + "pass@32": 0.7524753416594111, + "pass@64": 0.7828633252434822, + "pass@128": 0.8064516129032258 + }, + "19": { + "pass@1": 0.28993580060422963, + "pass@2": 0.3862616563979352, + "pass@4": 0.47463368296863195, + "pass@8": 0.5521120818448708, + "pass@16": 0.6195334410508603, + "pass@32": 0.6751110922664687, + "pass@64": 0.7182969466585594, + "pass@128": 0.7583081570996979 + }, + "20": { + "pass@1": 0.26703265765765766, + "pass@2": 0.3570292733678559, + "pass@4": 0.4455921731630394, + "pass@8": 0.5283674264542136, + "pass@16": 0.600828430583105, + "pass@32": 0.6587849247479455, + "pass@64": 0.7037287868228262, + "pass@128": 0.7387387387387387 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_metrics.json b/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..222f4004f6d8a0c30dfecd72fea702dd7febe5c1 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch1-op17-20", + "id": { + "answer_accuracy": 0.9763394097222222, + "avg_response_len": 199.7977092013889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9763394097222222, + "pass@2": 0.985418990594919, + "pass@4": 0.9903296812690076, + "pass@8": 0.9934610141867211, + "pass@16": 0.995578293766543, + "pass@32": 0.9971099645277666, + "pass@64": 0.9982153127246509, + "pass@128": 0.999 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.91640625, + "pass@2": 0.9476706446850395, + "pass@4": 0.9669583255530562, + "pass@8": 0.9788451880131132, + "pass@16": 0.9862062777304876, + "pass@32": 0.9912431276236333, + "pass@64": 0.9947722680345038, + "pass@128": 0.997 + }, + "2": { + "pass@1": 0.99846875, + "pass@2": 0.9988816437007875, + "pass@4": 0.9989923884514436, + "pass@8": 0.9989999655709787, + "pass@16": 0.9989999999997617, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.9932734375, + "pass@2": 0.9957941683070867, + "pass@4": 0.9978264966254219, + "pass@8": 0.9990544210606774, + "pass@16": 0.9996775186156238, + "pass@32": 0.9999476963247053, + "pass@64": 0.9999993318674025, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.992609375, + "pass@2": 0.9941601870078742, + "pass@4": 0.995101639295088, + "pass@8": 0.9959862454184044, + "pass@16": 0.9967492275545933, + "pass@32": 0.9973881759502105, + "pass@64": 0.9979968420843298, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9737109375, + "pass@2": 0.9815353100393699, + "pass@4": 0.9869551624484438, + "pass@8": 0.9911598043008092, + "pass@16": 0.9944993273291891, + "pass@32": 0.9971205319942156, + "pass@64": 0.9989734239317924, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9908984375, + "pass@2": 0.9944271653543306, + "pass@4": 0.9961795943007125, + "pass@8": 0.9970593753784408, + "pass@16": 0.9976541287314281, + "pass@32": 0.9981434924936788, + "pass@64": 0.9984968766172793, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.983296875, + "pass@2": 0.9901711368110238, + "pass@4": 0.9935972257217848, + "pass@8": 0.9958620196079306, + "pass@16": 0.997439017820933, + "pass@32": 0.9986513545851102, + "pass@64": 0.9995945257898196, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.974046875, + "pass@2": 0.9852920767716542, + "pass@4": 0.9903052625609297, + "pass@8": 0.9934548398646773, + "pass@16": 0.9956272175270217, + "pass@32": 0.9970991849627101, + "pass@64": 0.997866112633597, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.96434375, + "pass@2": 0.9808385826771662, + "pass@4": 0.9870510364641919, + "pass@8": 0.9907272684654614, + "pass@16": 0.993351928589844, + "pass@32": 0.9953961168156369, + "pass@64": 0.9972384335631338, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.91640625, + "2": 0.99846875, + "3": 0.9932734375, + "4": 0.992609375, + "5": 0.9737109375, + "6": 0.9908984375, + "7": 0.983296875, + "8": 0.974046875, + "9": 0.96434375 + }, + "per_op_avg_response_len": { + "10": 279.87028125, + "2": 115.335296875, + "3": 145.0280703125, + "4": 159.4407109375, + "5": 174.086609375, + "6": 198.6135234375, + "7": 223.5107421875, + "8": 243.1836640625, + "9": 259.110484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.87028125, + "2": 115.335296875, + "3": 145.0280703125, + "4": 159.4407109375, + "5": 174.086609375, + "6": 198.6135234375, + "7": 223.5107421875, + "8": 243.1836640625, + "9": 259.110484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 369255, + "answer_accuracy": 0.974267033941236, + "avg_response_len": 190.63018986406618, + "resp_tokens_sum": 72250367, + "pass_at_k": { + "pass@1": 0.974267033941236, + "pass@2": 0.9841380366018093, + "pass@4": 0.9894483518207642, + "pass@8": 0.992895127635665, + "pass@16": 0.995180950765034, + "pass@32": 0.9967866199320052, + "pass@64": 0.9979591953268966, + "pass@128": 0.9989868287740629 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9147999631268436, + "pass@2": 0.9457115357814785, + "pass@4": 0.9644939742045515, + "pass@8": 0.976467977625549, + "pass@16": 0.9845557191374704, + "pass@32": 0.9904786548196389, + "pass@64": 0.9943848945020379, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 0.9968847352024922, + "pass@4": 0.9968847352024922, + "pass@8": 0.9968847352024922, + "pass@16": 0.9968847352024922, + "pass@32": 0.9968847352024922, + "pass@64": 0.9968847352024922, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9949589067278287, + "pass@2": 0.9971454435454742, + "pass@4": 0.998757460363326, + "pass@8": 0.9996935826438439, + "pass@16": 0.9999781984079241, + "pass@32": 0.9999999385362756, + "pass@64": 0.9999999999999962, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985107421875, + "pass@2": 0.9878137303149606, + "pass@4": 0.9902266852385638, + "pass@8": 0.992418635965382, + "pass@16": 0.9940056498208453, + "pass@32": 0.9952528850596998, + "pass@64": 0.9968748060893958, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9661954365079365, + "pass@2": 0.9754136201724785, + "pass@4": 0.9818137741710856, + "pass@8": 0.9867755777021514, + "pass@16": 0.9913209124055069, + "pass@32": 0.9954008763900372, + "pass@64": 0.9983700744266549, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9920471556886228, + "pass@2": 0.9955804434438211, + "pass@4": 0.9972394134677776, + "pass@8": 0.998204156494888, + "pass@16": 0.999004031075878, + "pass@32": 0.9997223135643741, + "pass@64": 0.999990730945279, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9807923446745562, + "pass@2": 0.9899740250663933, + "pass@4": 0.9942786351040437, + "pass@8": 0.9966196608845793, + "pass@16": 0.9978572891506724, + "pass@32": 0.9987563633971823, + "pass@64": 0.9996389129939015, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9767878605769231, + "pass@2": 0.9883080645568344, + "pass@4": 0.9921772134252452, + "pass@8": 0.9944861816978635, + "pass@16": 0.9959239960690798, + "pass@32": 0.9966348318465099, + "pass@64": 0.9967927316152657, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9637764084507042, + "pass@2": 0.9817663857158702, + "pass@4": 0.9897242576192058, + "pass@8": 0.994632208919704, + "pass@16": 0.9970542065127949, + "pass@32": 0.9978697672017515, + "pass@64": 0.9985915429355597, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 382113, + "answer_accuracy": 0.9781316554718218, + "avg_response_len": 197.60126044397117, + "resp_tokens_sum": 77194118, + "pass_at_k": { + "pass@1": 0.9781316554718218, + "pass@2": 0.9867432353639298, + "pass@4": 0.9912419731220529, + "pass@8": 0.9938570392389242, + "pass@16": 0.9955842173182657, + "pass@32": 0.9969163336213072, + "pass@64": 0.9980768353031001, + "pass@128": 0.9990170380078637 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.915763995398773, + "pass@2": 0.9480459881165166, + "pass@4": 0.9681581655245539, + "pass@8": 0.9794301390879024, + "pass@16": 0.9853206794298328, + "pass@32": 0.989459399841, + "pass@64": 0.9934911833242681, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9990976443768997, + "pass@2": 0.9997371089916952, + "pass@4": 0.9999789687193356, + "pass@8": 0.9999998960323445, + "pass@16": 0.9999999999992757, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.990814393939394, + "pass@2": 0.9942462270341206, + "pass@4": 0.9969956352046904, + "pass@8": 0.9983947582818798, + "pass@16": 0.9992437871465215, + "pass@32": 0.9998493181618995, + "pass@64": 0.9999979765592963, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9959781804733728, + "pass@2": 0.9972641872058892, + "pass@4": 0.9977194152506086, + "pass@8": 0.9982611890819908, + "pass@16": 0.9990160343548016, + "pass@32": 0.9997255997961729, + "pass@64": 0.9999908406382343, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9807065217391304, + "pass@2": 0.9866305774278213, + "pass@4": 0.9910699586464737, + "pass@8": 0.9945806050238211, + "pass@16": 0.997016986481387, + "pass@32": 0.9983197945450435, + "pass@64": 0.9992809854172511, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9867030802292264, + "pass@2": 0.9902159995938904, + "pass@4": 0.9921985556783911, + "pass@8": 0.9933309007238308, + "pass@16": 0.9942316250648926, + "pass@32": 0.9949462457401639, + "pass@64": 0.9957019268812495, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9899972904624278, + "pass@2": 0.9941499590369122, + "pass@4": 0.9955985682425533, + "pass@8": 0.9965680962120179, + "pass@16": 0.9975371405733434, + "pass@32": 0.9983638978272487, + "pass@64": 0.9992831394824679, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9800836267605634, + "pass@2": 0.9910738050349339, + "pass@4": 0.9956219548260692, + "pass@8": 0.997544552113714, + "pass@16": 0.9983496167020315, + "pass@32": 0.9991228206283554, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9617795658682635, + "pass@2": 0.9778661181573863, + "pass@4": 0.9829365481635154, + "pass@8": 0.98602176677128, + "pass@16": 0.9891346058079603, + "pass@32": 0.9921804769068225, + "pass@64": 0.9949455674393275, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 373375, + "answer_accuracy": 0.9765625, + "avg_response_len": 211.12967651489788, + "resp_tokens_sum": 80722476, + "pass_at_k": { + "pass@1": 0.9765625, + "pass@2": 0.985335733058477, + "pass@4": 0.990271195082453, + "pass@8": 0.9936173321038113, + "pass@16": 0.9959661256874036, + "pass@32": 0.9976283391091408, + "pass@64": 0.9986106916015589, + "pass@128": 0.9989956478071644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9186567164179105, + "pass@2": 0.9492878863556237, + "pass@4": 0.9682844965274856, + "pass@8": 0.9806815470310305, + "pass@16": 0.988738350593314, + "pass@32": 0.9937525351988347, + "pass@64": 0.9964109344316464, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 0.9993303571428571, + "pass@2": 0.9999089566929135, + "pass@4": 0.9999980221222345, + "pass@8": 0.9999999993609641, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9940324344023324, + "pass@2": 0.9959951992608066, + "pass@4": 0.9977383308136045, + "pass@8": 0.9990797355776098, + "pass@16": 0.9998081573699138, + "pass@32": 0.999992540612001, + "pass@64": 0.9999999988420875, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9962993421052632, + "pass@2": 0.9970306959985263, + "pass@4": 0.9970758411229297, + "pass@8": 0.9970760233911381, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9735753676470589, + "pass@2": 0.9820366778601207, + "pass@4": 0.9875431995632898, + "pass@8": 0.9917505546806278, + "pass@16": 0.994889351721694, + "pass@32": 0.9974968435685705, + "pass@64": 0.9992203250542475, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9943069794952681, + "pass@2": 0.9978483072108101, + "pass@4": 0.9994458494344357, + "pass@8": 0.9999580528597197, + "pass@16": 0.9999998271433355, + "pass@32": 0.999999999999561, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9786392405063291, + "pass@2": 0.9860254130120601, + "pass@4": 0.9906770330528303, + "pass@8": 0.9942785219607105, + "pass@16": 0.9968841881317372, + "pass@32": 0.9988537819893496, + "pass@64": 0.9998879966359081, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9650431681681682, + "pass@2": 0.9763025821096688, + "pass@4": 0.982883417444441, + "pass@8": 0.9881286341578839, + "pass@16": 0.9924468973701125, + "pass@32": 0.9953769252358343, + "pass@64": 0.996775794068325, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9677451768488746, + "pass@2": 0.9829718079854168, + "pass@4": 0.9884183856037287, + "pass@8": 0.9913233575477782, + "pass@16": 0.9936550705400102, + "pass@32": 0.9960259491065468, + "pass@64": 0.998156322496061, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.608040625, + "avg_response_len": 372.15247421875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.608040625, + "pass@2": 0.6918071481299107, + "pass@4": 0.7545935140138644, + "pass@8": 0.8021140311865859, + "pass@16": 0.838465956835818, + "pass@32": 0.8668699739628293, + "pass@64": 0.8899108409221937, + "pass@128": 0.9093 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.89928125, + "pass@2": 0.9340199311023635, + "pass@4": 0.9530581857892761, + "pass@8": 0.9646084167445874, + "pass@16": 0.9721545862699154, + "pass@32": 0.978247602678375, + "pass@64": 0.9833447019063941, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.84053125, + "pass@2": 0.8881071604330706, + "pass@4": 0.9157314617547808, + "pass@8": 0.9342534388562534, + "pass@16": 0.9479341431792865, + "pass@32": 0.9595398724244704, + "pass@64": 0.9696723257993726, + "pass@128": 0.978 + }, + "13": { + "pass@1": 0.7338671875, + "pass@2": 0.8052672244094486, + "pass@4": 0.8534425268091481, + "pass@8": 0.8896280507676896, + "pass@16": 0.9178852960128545, + "pass@32": 0.9396084313296421, + "pass@64": 0.9551960359821134, + "pass@128": 0.966 + }, + "14": { + "pass@1": 0.7240703125, + "pass@2": 0.8020214074803148, + "pass@4": 0.8532807261904767, + "pass@8": 0.8874175863692658, + "pass@16": 0.9118375285344886, + "pass@32": 0.9301418992032527, + "pass@64": 0.9433938844205266, + "pass@128": 0.954 + }, + "15": { + "pass@1": 0.6850625, + "pass@2": 0.7675178395669306, + "pass@4": 0.8241469076677913, + "pass@8": 0.8637729210547304, + "pass@16": 0.8922964442166276, + "pass@32": 0.9133260490070577, + "pass@64": 0.9290881232929966, + "pass@128": 0.942 + }, + "16": { + "pass@1": 0.5612265625, + "pass@2": 0.667350024606298, + "pass@4": 0.743922721503563, + "pass@8": 0.799555250281722, + "pass@16": 0.8414756351517275, + "pass@32": 0.8738830725990342, + "pass@64": 0.9003686093194375, + "pass@128": 0.923 + }, + "17": { + "pass@1": 0.501125, + "pass@2": 0.6063334153543315, + "pass@4": 0.6872567897450311, + "pass@8": 0.7494785918750635, + "pass@16": 0.7974884869567112, + "pass@32": 0.8351106551153658, + "pass@64": 0.8664328729653366, + "pass@128": 0.893 + }, + "18": { + "pass@1": 0.466578125, + "pass@2": 0.571246186023622, + "pass@4": 0.6533641156730406, + "pass@8": 0.7167335427334711, + "pass@16": 0.7651675894122477, + "pass@32": 0.8020617703237342, + "pass@64": 0.831716328608351, + "pass@128": 0.858 + }, + "19": { + "pass@1": 0.3256015625, + "pass@2": 0.42588127460629893, + "pass@4": 0.5165330664604426, + "pass@8": 0.5937313861165137, + "pass@16": 0.6572152568113311, + "pass@32": 0.7092619621608254, + "pass@64": 0.7536724854580563, + "pass@128": 0.793 + }, + "20": { + "pass@1": 0.3430625, + "pass@2": 0.450327017716536, + "pass@4": 0.5451986385451821, + "pass@8": 0.6219611270666211, + "pass@16": 0.6812046018128655, + "pass@32": 0.7275184247865151, + "pass@64": 0.7662230414692883, + "pass@128": 0.799 + } + }, + "per_op_accuracy": { + "11": 0.89928125, + "12": 0.84053125, + "13": 0.7338671875, + "14": 0.7240703125, + "15": 0.6850625, + "16": 0.5612265625, + "17": 0.501125, + "18": 0.466578125, + "19": 0.3256015625, + "20": 0.3430625 + }, + "per_op_avg_response_len": { + "11": 297.9715859375, + "12": 307.6075703125, + "13": 326.8675625, + "14": 351.52475, + "15": 373.1016015625, + "16": 379.728, + "17": 402.9995078125, + "18": 416.9937421875, + "19": 420.4469375, + "20": 444.283484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.9715859375, + "12": 307.6075703125, + "13": 326.8675625, + "14": 351.52475, + "15": 373.1016015625, + "16": 379.728, + "17": 402.9995078125, + "18": 416.9937421875, + "19": 420.4469375, + "20": 444.283484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 262611, + "answer_accuracy": 0.6097023588410104, + "avg_response_len": 353.31013187221396, + "resp_tokens_sum": 152177740, + "pass_at_k": { + "pass@1": 0.6097023588410104, + "pass@2": 0.6933687668916978, + "pass@4": 0.7545651334816439, + "pass@8": 0.8001828025446808, + "pass@16": 0.8352171976360004, + "pass@32": 0.8633249010412943, + "pass@64": 0.8870723629289942, + "pass@128": 0.9078751857355126 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8884782223796034, + "pass@2": 0.9233639250741679, + "pass@4": 0.9423458353116624, + "pass@8": 0.9535150564785112, + "pass@16": 0.9606443331526207, + "pass@32": 0.9671079842769559, + "pass@64": 0.9738184857703525, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.831302966101695, + "pass@2": 0.8819387678499936, + "pass@4": 0.9108717917145948, + "pass@8": 0.9301873731209482, + "pass@16": 0.9445633356962888, + "pass@32": 0.9571175271859526, + "pass@64": 0.9693614586588243, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.7450528996865203, + "pass@2": 0.817905672253351, + "pass@4": 0.8649554258303918, + "pass@8": 0.900549165326192, + "pass@16": 0.9298497520568195, + "pass@32": 0.9535119750141245, + "pass@64": 0.9707018746756205, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.7362733644859814, + "pass@2": 0.8120324036598229, + "pass@4": 0.8572509446833161, + "pass@8": 0.8877305547576959, + "pass@16": 0.9128675754806822, + "pass@32": 0.9344930311271058, + "pass@64": 0.9501011437979755, + "pass@128": 0.9595015576323987 + }, + "15": { + "pass@1": 0.6830066835260116, + "pass@2": 0.7652260365936912, + "pass@4": 0.8216380976221894, + "pass@8": 0.8606056314618283, + "pass@16": 0.8899187889510047, + "pass@32": 0.9132096707385248, + "pass@64": 0.9310756807309161, + "pass@128": 0.9479768786127167 + }, + "16": { + "pass@1": 0.5638104838709678, + "pass@2": 0.6738494570739141, + "pass@4": 0.7523035856405045, + "pass@8": 0.8063191188545652, + "pass@16": 0.843643593454054, + "pass@32": 0.8708784076063162, + "pass@64": 0.8951979223494801, + "pass@128": 0.9225806451612903 + }, + "17": { + "pass@1": 0.49896134393063585, + "pass@2": 0.6055844920577128, + "pass@4": 0.6884978144892292, + "pass@8": 0.7497283063889675, + "pass@16": 0.7943429447842064, + "pass@32": 0.8296028860318351, + "pass@64": 0.8601444837620584, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.4401785714285714, + "pass@2": 0.5414805961754777, + "pass@4": 0.620062789919117, + "pass@8": 0.6825087231452688, + "pass@16": 0.7330450275756425, + "pass@32": 0.7715735323836825, + "pass@64": 0.8007257492775395, + "pass@128": 0.8228571428571428 + }, + "19": { + "pass@1": 0.3295091966966967, + "pass@2": 0.42963078728334647, + "pass@4": 0.5176115941588382, + "pass@8": 0.5916116555282213, + "pass@16": 0.6522660774252571, + "pass@32": 0.7026441292013839, + "pass@64": 0.7490627189135609, + "pass@128": 0.7957957957957958 + }, + "20": { + "pass@1": 0.3669294294294294, + "pass@2": 0.47350980409543403, + "pass@4": 0.5639906050595023, + "pass@8": 0.6350813513267202, + "pass@16": 0.6886366357150525, + "pass@32": 0.7318928034540644, + "pass@64": 0.770224695930882, + "pass@128": 0.8048048048048048 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 262409, + "answer_accuracy": 0.6095956920903954, + "avg_response_len": 367.53131039994054, + "resp_tokens_sum": 158208998, + "pass_at_k": { + "pass@1": 0.6095956920903954, + "pass@2": 0.6929929922898841, + "pass@4": 0.7563079705079249, + "pass@8": 0.8046729227247553, + "pass@16": 0.8414040625147604, + "pass@32": 0.8690179606347247, + "pass@64": 0.8904732775383289, + "pass@128": 0.9093071662206363 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9141566265060241, + "pass@2": 0.9468826338819848, + "pass@4": 0.9644491272175316, + "pass@8": 0.9749877614946728, + "pass@16": 0.9813777882066246, + "pass@32": 0.986807323784196, + "pass@64": 0.9916378206179335, + "pass@128": 0.9939759036144579 + }, + "12": { + "pass@1": 0.8360844435736677, + "pass@2": 0.8800076518648338, + "pass@4": 0.9078132809809426, + "pass@8": 0.9269145444634191, + "pass@16": 0.9396337022070549, + "pass@32": 0.9486619447609619, + "pass@64": 0.9548071381739272, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7287289915966386, + "pass@2": 0.7963580609409118, + "pass@4": 0.841953924351893, + "pass@8": 0.876928300220035, + "pass@16": 0.904554157815763, + "pass@32": 0.9254462571555292, + "pass@64": 0.9401177901808709, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7159173976608187, + "pass@2": 0.7988492597964728, + "pass@4": 0.856491532308461, + "pass@8": 0.8950593406393607, + "pass@16": 0.9211022424117776, + "pass@32": 0.9385845730381236, + "pass@64": 0.9499761266570184, + "pass@128": 0.9619883040935673 + }, + "15": { + "pass@1": 0.6883049242424243, + "pass@2": 0.7739773472918154, + "pass@4": 0.8314353032575473, + "pass@8": 0.8705559369310063, + "pass@16": 0.8982743454273422, + "pass@32": 0.9179786489223545, + "pass@64": 0.9320109430910483, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.5450498575498576, + "pass@2": 0.6449591437288287, + "pass@4": 0.7198456640569499, + "pass@8": 0.7786853841035807, + "pass@16": 0.8259551044670943, + "pass@32": 0.8620595238262503, + "pass@64": 0.88890031356281, + "pass@128": 0.9116809116809117 + }, + "17": { + "pass@1": 0.5208171583850931, + "pass@2": 0.6292667475668802, + "pass@4": 0.7093731415366554, + "pass@8": 0.7695169309117884, + "pass@16": 0.8136355479127603, + "pass@32": 0.8447643060918684, + "pass@64": 0.8701041757502938, + "pass@128": 0.8944099378881988 + }, + "18": { + "pass@1": 0.48566176470588235, + "pass@2": 0.5931659622510422, + "pass@4": 0.6829272411720586, + "pass@8": 0.7536475960707826, + "pass@16": 0.8045954899897602, + "pass@32": 0.8405545218075591, + "pass@64": 0.8695012729380446, + "pass@128": 0.8970588235294118 + }, + "19": { + "pass@1": 0.32163783482142855, + "pass@2": 0.41333932379546307, + "pass@4": 0.49753887851072176, + "pass@8": 0.5725680674156789, + "pass@16": 0.6380391354389646, + "pass@32": 0.6926487886121019, + "pass@64": 0.7373374929774971, + "pass@128": 0.7738095238095238 + }, + "20": { + "pass@1": 0.3458317739520958, + "pass@2": 0.4579825339478522, + "pass@4": 0.5549417604610802, + "pass@8": 0.6303896133806527, + "pass@16": 0.6882139105165509, + "pass@32": 0.733040383229659, + "pass@64": 0.7701302877722762, + "pass@128": 0.8053892215568862 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 253272, + "answer_accuracy": 0.6047333435207825, + "avg_response_len": 396.2800585459963, + "resp_tokens_sum": 165968429, + "pass_at_k": { + "pass@1": 0.6047333435207825, + "pass@2": 0.6889823189602102, + "pass@4": 0.7528605627001419, + "pass@8": 0.8014700923532283, + "pass@16": 0.8387872359644982, + "pass@32": 0.8683080825213123, + "pass@64": 0.8922519179719953, + "pass@128": 0.910757946210269 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8957093253968254, + "pass@2": 0.9324045431821019, + "pass@4": 0.9530571291683771, + "pass@8": 0.9661005237823533, + "pass@16": 0.9753324158172761, + "pass@32": 0.9817093737531953, + "pass@64": 0.9852794285216058, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8548595183486238, + "pass@2": 0.9026862252883529, + "pass@4": 0.928716852767256, + "pass@8": 0.9458145843657703, + "pass@16": 0.9596806462958717, + "pass@32": 0.9727740288131387, + "pass@64": 0.9845103741794053, + "pass@128": 0.9938837920489296 + }, + "13": { + "pass@1": 0.728515625, + "pass@2": 0.8026403926071737, + "pass@4": 0.854766033875395, + "pass@8": 0.8928687155866702, + "pass@16": 0.920794415945995, + "pass@32": 0.94152406634448, + "pass@64": 0.9565434779815419, + "pass@128": 0.9660493827160493 + }, + "14": { + "pass@1": 0.7207205118694362, + "pass@2": 0.7957049230122201, + "pass@4": 0.8462405605271146, + "pass@8": 0.8793643436005479, + "pass@16": 0.9014542133541928, + "pass@32": 0.9174294131525637, + "pass@64": 0.9303251689752986, + "pass@128": 0.9406528189910979 + }, + "15": { + "pass@1": 0.6839554398148148, + "pass@2": 0.7633861305531252, + "pass@4": 0.8194027031343298, + "pass@8": 0.8602466462398325, + "pass@16": 0.8887469421251796, + "pass@32": 0.9087115703307098, + "pass@64": 0.9239886621606587, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5756130162241888, + "pass@2": 0.6845900692170115, + "pass@4": 0.761188147112142, + "pass@8": 0.8149786242373147, + "pass@16": 0.8555630664100949, + "pass@32": 0.8888727828261448, + "pass@64": 0.9169712192641652, + "pass@128": 0.9351032448377581 + }, + "17": { + "pass@1": 0.4842808734939759, + "pass@2": 0.584871350559719, + "pass@4": 0.6645132360149255, + "pass@8": 0.7297835726231425, + "pass@16": 0.785105968775507, + "pass@32": 0.831487801164966, + "pass@64": 0.8694257138315328, + "pass@128": 0.9006024096385542 + }, + "18": { + "pass@1": 0.4754536290322581, + "pass@2": 0.5808114522479044, + "pass@4": 0.6585383135575791, + "pass@8": 0.7148880869953559, + "pass@16": 0.7581914940782433, + "pass@32": 0.7942661179834702, + "pass@64": 0.8252641405234724, + "pass@128": 0.8548387096774194 + }, + "19": { + "pass@1": 0.325693919939577, + "pass@2": 0.4348405125246804, + "pass@4": 0.5347291311962133, + "pass@8": 0.6173469310391171, + "pass@16": 0.6816601314840767, + "pass@32": 0.7327839098583034, + "pass@64": 0.7748918502096696, + "pass@128": 0.8096676737160121 + }, + "20": { + "pass@1": 0.3164179804804805, + "pass@2": 0.4194657255680876, + "pass@4": 0.5166342914905905, + "pass@8": 0.6003871057227763, + "pass@16": 0.6667422102319082, + "pass@32": 0.7176055052180353, + "pass@64": 0.7583024072323226, + "pass@128": 0.7867867867867868 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7824979440789473, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7824979440789473, + "pass@2": 0.8308864419291083, + "pass@4": 0.8662580142926025, + "pass@8": 0.8927520757656101, + "pass@16": 0.9128875901187812, + "pass@32": 0.9285626010725488, + "pass@64": 0.9412129591444078, + "pass@128": 0.9517894736842105 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.91640625, + "pass@2": 0.9476706446850395, + "pass@4": 0.9669583255530562, + "pass@8": 0.9788451880131132, + "pass@16": 0.9862062777304876, + "pass@32": 0.9912431276236333, + "pass@64": 0.9947722680345038, + "pass@128": 0.997 + }, + "2": { + "pass@1": 0.99846875, + "pass@2": 0.9988816437007875, + "pass@4": 0.9989923884514436, + "pass@8": 0.9989999655709787, + "pass@16": 0.9989999999997617, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.9932734375, + "pass@2": 0.9957941683070867, + "pass@4": 0.9978264966254219, + "pass@8": 0.9990544210606774, + "pass@16": 0.9996775186156238, + "pass@32": 0.9999476963247053, + "pass@64": 0.9999993318674025, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.992609375, + "pass@2": 0.9941601870078742, + "pass@4": 0.995101639295088, + "pass@8": 0.9959862454184044, + "pass@16": 0.9967492275545933, + "pass@32": 0.9973881759502105, + "pass@64": 0.9979968420843298, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9737109375, + "pass@2": 0.9815353100393699, + "pass@4": 0.9869551624484438, + "pass@8": 0.9911598043008092, + "pass@16": 0.9944993273291891, + "pass@32": 0.9971205319942156, + "pass@64": 0.9989734239317924, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9908984375, + "pass@2": 0.9944271653543306, + "pass@4": 0.9961795943007125, + "pass@8": 0.9970593753784408, + "pass@16": 0.9976541287314281, + "pass@32": 0.9981434924936788, + "pass@64": 0.9984968766172793, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.983296875, + "pass@2": 0.9901711368110238, + "pass@4": 0.9935972257217848, + "pass@8": 0.9958620196079306, + "pass@16": 0.997439017820933, + "pass@32": 0.9986513545851102, + "pass@64": 0.9995945257898196, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.974046875, + "pass@2": 0.9852920767716542, + "pass@4": 0.9903052625609297, + "pass@8": 0.9934548398646773, + "pass@16": 0.9956272175270217, + "pass@32": 0.9970991849627101, + "pass@64": 0.997866112633597, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.96434375, + "pass@2": 0.9808385826771662, + "pass@4": 0.9870510364641919, + "pass@8": 0.9907272684654614, + "pass@16": 0.993351928589844, + "pass@32": 0.9953961168156369, + "pass@64": 0.9972384335631338, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.89928125, + "pass@2": 0.9340199311023635, + "pass@4": 0.9530581857892761, + "pass@8": 0.9646084167445874, + "pass@16": 0.9721545862699154, + "pass@32": 0.978247602678375, + "pass@64": 0.9833447019063941, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.84053125, + "pass@2": 0.8881071604330706, + "pass@4": 0.9157314617547808, + "pass@8": 0.9342534388562534, + "pass@16": 0.9479341431792865, + "pass@32": 0.9595398724244704, + "pass@64": 0.9696723257993726, + "pass@128": 0.978 + }, + "13": { + "pass@1": 0.7338671875, + "pass@2": 0.8052672244094486, + "pass@4": 0.8534425268091481, + "pass@8": 0.8896280507676896, + "pass@16": 0.9178852960128545, + "pass@32": 0.9396084313296421, + "pass@64": 0.9551960359821134, + "pass@128": 0.966 + }, + "14": { + "pass@1": 0.7240703125, + "pass@2": 0.8020214074803148, + "pass@4": 0.8532807261904767, + "pass@8": 0.8874175863692658, + "pass@16": 0.9118375285344886, + "pass@32": 0.9301418992032527, + "pass@64": 0.9433938844205266, + "pass@128": 0.954 + }, + "15": { + "pass@1": 0.6850625, + "pass@2": 0.7675178395669306, + "pass@4": 0.8241469076677913, + "pass@8": 0.8637729210547304, + "pass@16": 0.8922964442166276, + "pass@32": 0.9133260490070577, + "pass@64": 0.9290881232929966, + "pass@128": 0.942 + }, + "16": { + "pass@1": 0.5612265625, + "pass@2": 0.667350024606298, + "pass@4": 0.743922721503563, + "pass@8": 0.799555250281722, + "pass@16": 0.8414756351517275, + "pass@32": 0.8738830725990342, + "pass@64": 0.9003686093194375, + "pass@128": 0.923 + }, + "17": { + "pass@1": 0.501125, + "pass@2": 0.6063334153543315, + "pass@4": 0.6872567897450311, + "pass@8": 0.7494785918750635, + "pass@16": 0.7974884869567112, + "pass@32": 0.8351106551153658, + "pass@64": 0.8664328729653366, + "pass@128": 0.893 + }, + "18": { + "pass@1": 0.466578125, + "pass@2": 0.571246186023622, + "pass@4": 0.6533641156730406, + "pass@8": 0.7167335427334711, + "pass@16": 0.7651675894122477, + "pass@32": 0.8020617703237342, + "pass@64": 0.831716328608351, + "pass@128": 0.858 + }, + "19": { + "pass@1": 0.3256015625, + "pass@2": 0.42588127460629893, + "pass@4": 0.5165330664604426, + "pass@8": 0.5937313861165137, + "pass@16": 0.6572152568113311, + "pass@32": 0.7092619621608254, + "pass@64": 0.7536724854580563, + "pass@128": 0.793 + }, + "20": { + "pass@1": 0.3430625, + "pass@2": 0.450327017716536, + "pass@4": 0.5451986385451821, + "pass@8": 0.6219611270666211, + "pass@16": 0.6812046018128655, + "pass@32": 0.7275184247865151, + "pass@64": 0.7662230414692883, + "pass@128": 0.799 + } + }, + "per_op_accuracy": { + "10": 0.91640625, + "2": 0.99846875, + "3": 0.9932734375, + "4": 0.992609375, + "5": 0.9737109375, + "6": 0.9908984375, + "7": 0.983296875, + "8": 0.974046875, + "9": 0.96434375, + "11": 0.89928125, + "12": 0.84053125, + "13": 0.7338671875, + "14": 0.7240703125, + "15": 0.6850625, + "16": 0.5612265625, + "17": 0.501125, + "18": 0.466578125, + "19": 0.3256015625, + "20": 0.3430625 + }, + "per_op_avg_response_len": { + "10": 279.87028125, + "2": 115.335296875, + "3": 145.0280703125, + "4": 159.4407109375, + "5": 174.086609375, + "6": 198.6135234375, + "7": 223.5107421875, + "8": 243.1836640625, + "9": 259.110484375, + "11": 297.9715859375, + "12": 307.6075703125, + "13": 326.8675625, + "14": 351.52475, + "15": 373.1016015625, + "16": 379.728, + "17": 402.9995078125, + "18": 416.9937421875, + "19": 420.4469375, + "20": 444.283484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.87028125, + "2": 115.335296875, + "3": 145.0280703125, + "4": 159.4407109375, + "5": 174.086609375, + "6": 198.6135234375, + "7": 223.5107421875, + "8": 243.1836640625, + "9": 259.110484375, + "11": 297.9715859375, + "12": 307.6075703125, + "13": 326.8675625, + "14": 351.52475, + "15": 373.1016015625, + "16": 379.728, + "17": 402.9995078125, + "18": 416.9937421875, + "19": 420.4469375, + "20": 444.283484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 631866, + "answer_accuracy": 0.7803435227631995, + "avg_response_len": 277.16480966447995, + "resp_tokens_sum": 224428107, + "pass_at_k": { + "pass@1": 0.7803435227631995, + "pass@2": 0.8294686416327026, + "pass@4": 0.8645065197450194, + "pass@8": 0.8903853309345585, + "pass@16": 0.9100911579608634, + "pass@32": 0.9257940995293442, + "pass@64": 0.9389749729084804, + "pass@128": 0.9505216566550743 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9147999631268436, + "pass@2": 0.9457115357814785, + "pass@4": 0.9644939742045515, + "pass@8": 0.976467977625549, + "pass@16": 0.9845557191374704, + "pass@32": 0.9904786548196389, + "pass@64": 0.9943848945020379, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 0.9968847352024922, + "pass@4": 0.9968847352024922, + "pass@8": 0.9968847352024922, + "pass@16": 0.9968847352024922, + "pass@32": 0.9968847352024922, + "pass@64": 0.9968847352024922, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9949589067278287, + "pass@2": 0.9971454435454742, + "pass@4": 0.998757460363326, + "pass@8": 0.9996935826438439, + "pass@16": 0.9999781984079241, + "pass@32": 0.9999999385362756, + "pass@64": 0.9999999999999962, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985107421875, + "pass@2": 0.9878137303149606, + "pass@4": 0.9902266852385638, + "pass@8": 0.992418635965382, + "pass@16": 0.9940056498208453, + "pass@32": 0.9952528850596998, + "pass@64": 0.9968748060893958, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9661954365079365, + "pass@2": 0.9754136201724785, + "pass@4": 0.9818137741710856, + "pass@8": 0.9867755777021514, + "pass@16": 0.9913209124055069, + "pass@32": 0.9954008763900372, + "pass@64": 0.9983700744266549, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9920471556886228, + "pass@2": 0.9955804434438211, + "pass@4": 0.9972394134677776, + "pass@8": 0.998204156494888, + "pass@16": 0.999004031075878, + "pass@32": 0.9997223135643741, + "pass@64": 0.999990730945279, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9807923446745562, + "pass@2": 0.9899740250663933, + "pass@4": 0.9942786351040437, + "pass@8": 0.9966196608845793, + "pass@16": 0.9978572891506724, + "pass@32": 0.9987563633971823, + "pass@64": 0.9996389129939015, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9767878605769231, + "pass@2": 0.9883080645568344, + "pass@4": 0.9921772134252452, + "pass@8": 0.9944861816978635, + "pass@16": 0.9959239960690798, + "pass@32": 0.9966348318465099, + "pass@64": 0.9967927316152657, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9637764084507042, + "pass@2": 0.9817663857158702, + "pass@4": 0.9897242576192058, + "pass@8": 0.994632208919704, + "pass@16": 0.9970542065127949, + "pass@32": 0.9978697672017515, + "pass@64": 0.9985915429355597, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8884782223796034, + "pass@2": 0.9233639250741679, + "pass@4": 0.9423458353116624, + "pass@8": 0.9535150564785112, + "pass@16": 0.9606443331526207, + "pass@32": 0.9671079842769559, + "pass@64": 0.9738184857703525, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.831302966101695, + "pass@2": 0.8819387678499936, + "pass@4": 0.9108717917145948, + "pass@8": 0.9301873731209482, + "pass@16": 0.9445633356962888, + "pass@32": 0.9571175271859526, + "pass@64": 0.9693614586588243, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.7450528996865203, + "pass@2": 0.817905672253351, + "pass@4": 0.8649554258303918, + "pass@8": 0.900549165326192, + "pass@16": 0.9298497520568195, + "pass@32": 0.9535119750141245, + "pass@64": 0.9707018746756205, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.7362733644859814, + "pass@2": 0.8120324036598229, + "pass@4": 0.8572509446833161, + "pass@8": 0.8877305547576959, + "pass@16": 0.9128675754806822, + "pass@32": 0.9344930311271058, + "pass@64": 0.9501011437979755, + "pass@128": 0.9595015576323987 + }, + "15": { + "pass@1": 0.6830066835260116, + "pass@2": 0.7652260365936912, + "pass@4": 0.8216380976221894, + "pass@8": 0.8606056314618283, + "pass@16": 0.8899187889510047, + "pass@32": 0.9132096707385248, + "pass@64": 0.9310756807309161, + "pass@128": 0.9479768786127167 + }, + "16": { + "pass@1": 0.5638104838709678, + "pass@2": 0.6738494570739141, + "pass@4": 0.7523035856405045, + "pass@8": 0.8063191188545652, + "pass@16": 0.843643593454054, + "pass@32": 0.8708784076063162, + "pass@64": 0.8951979223494801, + "pass@128": 0.9225806451612903 + }, + "17": { + "pass@1": 0.49896134393063585, + "pass@2": 0.6055844920577128, + "pass@4": 0.6884978144892292, + "pass@8": 0.7497283063889675, + "pass@16": 0.7943429447842064, + "pass@32": 0.8296028860318351, + "pass@64": 0.8601444837620584, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.4401785714285714, + "pass@2": 0.5414805961754777, + "pass@4": 0.620062789919117, + "pass@8": 0.6825087231452688, + "pass@16": 0.7330450275756425, + "pass@32": 0.7715735323836825, + "pass@64": 0.8007257492775395, + "pass@128": 0.8228571428571428 + }, + "19": { + "pass@1": 0.3295091966966967, + "pass@2": 0.42963078728334647, + "pass@4": 0.5176115941588382, + "pass@8": 0.5916116555282213, + "pass@16": 0.6522660774252571, + "pass@32": 0.7026441292013839, + "pass@64": 0.7490627189135609, + "pass@128": 0.7957957957957958 + }, + "20": { + "pass@1": 0.3669294294294294, + "pass@2": 0.47350980409543403, + "pass@4": 0.5639906050595023, + "pass@8": 0.6350813513267202, + "pass@16": 0.6886366357150525, + "pass@32": 0.7318928034540644, + "pass@64": 0.770224695930882, + "pass@128": 0.8048048048048048 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 644522, + "answer_accuracy": 0.7849303390491037, + "avg_response_len": 286.6854004286828, + "resp_tokens_sum": 235403116, + "pass_at_k": { + "pass@1": 0.7849303390491037, + "pass@2": 0.8327475896183206, + "pass@4": 0.8680801569425771, + "pass@8": 0.8946791462011701, + "pass@16": 0.9147568033503521, + "pass@32": 0.9298668825918631, + "pass@64": 0.9416667394710053, + "pass@128": 0.951987529228371 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.915763995398773, + "pass@2": 0.9480459881165166, + "pass@4": 0.9681581655245539, + "pass@8": 0.9794301390879024, + "pass@16": 0.9853206794298328, + "pass@32": 0.989459399841, + "pass@64": 0.9934911833242681, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9990976443768997, + "pass@2": 0.9997371089916952, + "pass@4": 0.9999789687193356, + "pass@8": 0.9999998960323445, + "pass@16": 0.9999999999992757, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.990814393939394, + "pass@2": 0.9942462270341206, + "pass@4": 0.9969956352046904, + "pass@8": 0.9983947582818798, + "pass@16": 0.9992437871465215, + "pass@32": 0.9998493181618995, + "pass@64": 0.9999979765592963, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9959781804733728, + "pass@2": 0.9972641872058892, + "pass@4": 0.9977194152506086, + "pass@8": 0.9982611890819908, + "pass@16": 0.9990160343548016, + "pass@32": 0.9997255997961729, + "pass@64": 0.9999908406382343, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9807065217391304, + "pass@2": 0.9866305774278213, + "pass@4": 0.9910699586464737, + "pass@8": 0.9945806050238211, + "pass@16": 0.997016986481387, + "pass@32": 0.9983197945450435, + "pass@64": 0.9992809854172511, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9867030802292264, + "pass@2": 0.9902159995938904, + "pass@4": 0.9921985556783911, + "pass@8": 0.9933309007238308, + "pass@16": 0.9942316250648926, + "pass@32": 0.9949462457401639, + "pass@64": 0.9957019268812495, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9899972904624278, + "pass@2": 0.9941499590369122, + "pass@4": 0.9955985682425533, + "pass@8": 0.9965680962120179, + "pass@16": 0.9975371405733434, + "pass@32": 0.9983638978272487, + "pass@64": 0.9992831394824679, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9800836267605634, + "pass@2": 0.9910738050349339, + "pass@4": 0.9956219548260692, + "pass@8": 0.997544552113714, + "pass@16": 0.9983496167020315, + "pass@32": 0.9991228206283554, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9617795658682635, + "pass@2": 0.9778661181573863, + "pass@4": 0.9829365481635154, + "pass@8": 0.98602176677128, + "pass@16": 0.9891346058079603, + "pass@32": 0.9921804769068225, + "pass@64": 0.9949455674393275, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9141566265060241, + "pass@2": 0.9468826338819848, + "pass@4": 0.9644491272175316, + "pass@8": 0.9749877614946728, + "pass@16": 0.9813777882066246, + "pass@32": 0.986807323784196, + "pass@64": 0.9916378206179335, + "pass@128": 0.9939759036144579 + }, + "12": { + "pass@1": 0.8360844435736677, + "pass@2": 0.8800076518648338, + "pass@4": 0.9078132809809426, + "pass@8": 0.9269145444634191, + "pass@16": 0.9396337022070549, + "pass@32": 0.9486619447609619, + "pass@64": 0.9548071381739272, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7287289915966386, + "pass@2": 0.7963580609409118, + "pass@4": 0.841953924351893, + "pass@8": 0.876928300220035, + "pass@16": 0.904554157815763, + "pass@32": 0.9254462571555292, + "pass@64": 0.9401177901808709, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7159173976608187, + "pass@2": 0.7988492597964728, + "pass@4": 0.856491532308461, + "pass@8": 0.8950593406393607, + "pass@16": 0.9211022424117776, + "pass@32": 0.9385845730381236, + "pass@64": 0.9499761266570184, + "pass@128": 0.9619883040935673 + }, + "15": { + "pass@1": 0.6883049242424243, + "pass@2": 0.7739773472918154, + "pass@4": 0.8314353032575473, + "pass@8": 0.8705559369310063, + "pass@16": 0.8982743454273422, + "pass@32": 0.9179786489223545, + "pass@64": 0.9320109430910483, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.5450498575498576, + "pass@2": 0.6449591437288287, + "pass@4": 0.7198456640569499, + "pass@8": 0.7786853841035807, + "pass@16": 0.8259551044670943, + "pass@32": 0.8620595238262503, + "pass@64": 0.88890031356281, + "pass@128": 0.9116809116809117 + }, + "17": { + "pass@1": 0.5208171583850931, + "pass@2": 0.6292667475668802, + "pass@4": 0.7093731415366554, + "pass@8": 0.7695169309117884, + "pass@16": 0.8136355479127603, + "pass@32": 0.8447643060918684, + "pass@64": 0.8701041757502938, + "pass@128": 0.8944099378881988 + }, + "18": { + "pass@1": 0.48566176470588235, + "pass@2": 0.5931659622510422, + "pass@4": 0.6829272411720586, + "pass@8": 0.7536475960707826, + "pass@16": 0.8045954899897602, + "pass@32": 0.8405545218075591, + "pass@64": 0.8695012729380446, + "pass@128": 0.8970588235294118 + }, + "19": { + "pass@1": 0.32163783482142855, + "pass@2": 0.41333932379546307, + "pass@4": 0.49753887851072176, + "pass@8": 0.5725680674156789, + "pass@16": 0.6380391354389646, + "pass@32": 0.6926487886121019, + "pass@64": 0.7373374929774971, + "pass@128": 0.7738095238095238 + }, + "20": { + "pass@1": 0.3458317739520958, + "pass@2": 0.4579825339478522, + "pass@4": 0.5549417604610802, + "pass@8": 0.6303896133806527, + "pass@16": 0.6882139105165509, + "pass@32": 0.733040383229659, + "pass@64": 0.7701302877722762, + "pass@128": 0.8053892215568862 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 626647, + "answer_accuracy": 0.7821824073334398, + "avg_response_len": 307.9202261243809, + "resp_tokens_sum": 246690905, + "pass_at_k": { + "pass@1": 0.7821824073334398, + "pass@2": 0.8304118840523114, + "pass@4": 0.8661606999306755, + "pass@8": 0.8931690546690862, + "pass@16": 0.9137981552171506, + "pass@32": 0.9300239487024705, + "pass@64": 0.9430098116980766, + "pass@128": 0.9528678702668157 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9186567164179105, + "pass@2": 0.9492878863556237, + "pass@4": 0.9682844965274856, + "pass@8": 0.9806815470310305, + "pass@16": 0.988738350593314, + "pass@32": 0.9937525351988347, + "pass@64": 0.9964109344316464, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 0.9993303571428571, + "pass@2": 0.9999089566929135, + "pass@4": 0.9999980221222345, + "pass@8": 0.9999999993609641, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9940324344023324, + "pass@2": 0.9959951992608066, + "pass@4": 0.9977383308136045, + "pass@8": 0.9990797355776098, + "pass@16": 0.9998081573699138, + "pass@32": 0.999992540612001, + "pass@64": 0.9999999988420875, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9962993421052632, + "pass@2": 0.9970306959985263, + "pass@4": 0.9970758411229297, + "pass@8": 0.9970760233911381, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9735753676470589, + "pass@2": 0.9820366778601207, + "pass@4": 0.9875431995632898, + "pass@8": 0.9917505546806278, + "pass@16": 0.994889351721694, + "pass@32": 0.9974968435685705, + "pass@64": 0.9992203250542475, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9943069794952681, + "pass@2": 0.9978483072108101, + "pass@4": 0.9994458494344357, + "pass@8": 0.9999580528597197, + "pass@16": 0.9999998271433355, + "pass@32": 0.999999999999561, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9786392405063291, + "pass@2": 0.9860254130120601, + "pass@4": 0.9906770330528303, + "pass@8": 0.9942785219607105, + "pass@16": 0.9968841881317372, + "pass@32": 0.9988537819893496, + "pass@64": 0.9998879966359081, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9650431681681682, + "pass@2": 0.9763025821096688, + "pass@4": 0.982883417444441, + "pass@8": 0.9881286341578839, + "pass@16": 0.9924468973701125, + "pass@32": 0.9953769252358343, + "pass@64": 0.996775794068325, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9677451768488746, + "pass@2": 0.9829718079854168, + "pass@4": 0.9884183856037287, + "pass@8": 0.9913233575477782, + "pass@16": 0.9936550705400102, + "pass@32": 0.9960259491065468, + "pass@64": 0.998156322496061, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8957093253968254, + "pass@2": 0.9324045431821019, + "pass@4": 0.9530571291683771, + "pass@8": 0.9661005237823533, + "pass@16": 0.9753324158172761, + "pass@32": 0.9817093737531953, + "pass@64": 0.9852794285216058, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8548595183486238, + "pass@2": 0.9026862252883529, + "pass@4": 0.928716852767256, + "pass@8": 0.9458145843657703, + "pass@16": 0.9596806462958717, + "pass@32": 0.9727740288131387, + "pass@64": 0.9845103741794053, + "pass@128": 0.9938837920489296 + }, + "13": { + "pass@1": 0.728515625, + "pass@2": 0.8026403926071737, + "pass@4": 0.854766033875395, + "pass@8": 0.8928687155866702, + "pass@16": 0.920794415945995, + "pass@32": 0.94152406634448, + "pass@64": 0.9565434779815419, + "pass@128": 0.9660493827160493 + }, + "14": { + "pass@1": 0.7207205118694362, + "pass@2": 0.7957049230122201, + "pass@4": 0.8462405605271146, + "pass@8": 0.8793643436005479, + "pass@16": 0.9014542133541928, + "pass@32": 0.9174294131525637, + "pass@64": 0.9303251689752986, + "pass@128": 0.9406528189910979 + }, + "15": { + "pass@1": 0.6839554398148148, + "pass@2": 0.7633861305531252, + "pass@4": 0.8194027031343298, + "pass@8": 0.8602466462398325, + "pass@16": 0.8887469421251796, + "pass@32": 0.9087115703307098, + "pass@64": 0.9239886621606587, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5756130162241888, + "pass@2": 0.6845900692170115, + "pass@4": 0.761188147112142, + "pass@8": 0.8149786242373147, + "pass@16": 0.8555630664100949, + "pass@32": 0.8888727828261448, + "pass@64": 0.9169712192641652, + "pass@128": 0.9351032448377581 + }, + "17": { + "pass@1": 0.4842808734939759, + "pass@2": 0.584871350559719, + "pass@4": 0.6645132360149255, + "pass@8": 0.7297835726231425, + "pass@16": 0.785105968775507, + "pass@32": 0.831487801164966, + "pass@64": 0.8694257138315328, + "pass@128": 0.9006024096385542 + }, + "18": { + "pass@1": 0.4754536290322581, + "pass@2": 0.5808114522479044, + "pass@4": 0.6585383135575791, + "pass@8": 0.7148880869953559, + "pass@16": 0.7581914940782433, + "pass@32": 0.7942661179834702, + "pass@64": 0.8252641405234724, + "pass@128": 0.8548387096774194 + }, + "19": { + "pass@1": 0.325693919939577, + "pass@2": 0.4348405125246804, + "pass@4": 0.5347291311962133, + "pass@8": 0.6173469310391171, + "pass@16": 0.6816601314840767, + "pass@32": 0.7327839098583034, + "pass@64": 0.7748918502096696, + "pass@128": 0.8096676737160121 + }, + "20": { + "pass@1": 0.3164179804804805, + "pass@2": 0.4194657255680876, + "pass@4": 0.5166342914905905, + "pass@8": 0.6003871057227763, + "pass@16": 0.6667422102319082, + "pass@32": 0.7176055052180353, + "pass@64": 0.7583024072323226, + "pass@128": 0.7867867867867868 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch1-op17-20/graphs/SDAR-100M-block8-epoch1-op17-20_score_report.jsonl b/composition/SDAR-100M-block8-epoch1-op17-20/graphs/SDAR-100M-block8-epoch1-op17-20_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e3a507bc8c19507215b7578168b32795122331ce --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op17-20/graphs/SDAR-100M-block8-epoch1-op17-20_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch1-op17-20", "results_dir": "results/composition/SDAR-100M-block8-epoch1-op17-20/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1002994, "value_mismatch": 30405, "dependency_mismatch": 144119, "answer_mismatch": 27256, "extra_nodes": 26622, "missing_nodes": 15835, "perfect_with_extra": 2597, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 329667, "value_mismatch": 10815, "dependency_mismatch": 47737, "answer_mismatch": 9752, "extra_nodes": 9719, "missing_nodes": 5231, "perfect_with_extra": 969}, "teachers_in_school": {"total": 390656, "perfect": 340817, "dependency_mismatch": 48382, "value_mismatch": 9185, "answer_mismatch": 8543, "extra_nodes": 8141, "missing_nodes": 5197, "perfect_with_extra": 535}, "movie_festival_awards": {"total": 382336, "perfect": 332510, "dependency_mismatch": 48000, "value_mismatch": 10405, "answer_mismatch": 8961, "extra_nodes": 8762, "missing_nodes": 5407, "perfect_with_extra": 1093}}, "results_path": "results/composition/SDAR-100M-block8-epoch1-op17-20/graphs/SDAR-100M-block8-epoch1-op17-20_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 363525, "value_mismatch": 635784, "dependency_mismatch": 816161, "answer_mismatch": 501713, "extra_nodes": 71399, "missing_nodes": 224208, "perfect_with_extra": 924, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 214614, "dependency_mismatch": 274977, "answer_mismatch": 168115, "extra_nodes": 23269, "perfect": 122486, "perfect_with_extra": 199, "missing_nodes": 79715}, "movie_festival_awards": {"total": 418816, "perfect": 117667, "value_mismatch": 209062, "dependency_mismatch": 268433, "answer_mismatch": 165545, "extra_nodes": 25925, "missing_nodes": 72002, "perfect_with_extra": 256}, "teachers_in_school": {"total": 430464, "perfect": 123372, "dependency_mismatch": 272751, "value_mismatch": 212108, "answer_mismatch": 168053, "extra_nodes": 22205, "perfect_with_extra": 469, "missing_nodes": 72491}}, "results_path": "results/composition/SDAR-100M-block8-epoch1-op17-20/graphs/SDAR-100M-block8-epoch1-op17-20_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1366519, "value_mismatch": 666189, "dependency_mismatch": 960280, "answer_mismatch": 528969, "extra_nodes": 98021, "missing_nodes": 240043, "perfect_with_extra": 3521, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch1-op17-20/graphs/SDAR-100M-block8-epoch1-op17-20_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 452153, "value_mismatch": 225429, "dependency_mismatch": 322714, "answer_mismatch": 177867, "extra_nodes": 32988, "missing_nodes": 84946, "perfect_with_extra": 1168}, "teachers_in_school": {"total": 821120, "perfect": 464189, "dependency_mismatch": 321133, "value_mismatch": 221293, "answer_mismatch": 176596, "extra_nodes": 30346, "missing_nodes": 77688, "perfect_with_extra": 1004}, "movie_festival_awards": {"total": 801152, "perfect": 450177, "dependency_mismatch": 316433, "value_mismatch": 219467, "answer_mismatch": 174506, "extra_nodes": 34687, "missing_nodes": 77409, "perfect_with_extra": 1349}}}} diff --git a/composition/SDAR-100M-block8-epoch1-op17-20/graphs/results.json b/composition/SDAR-100M-block8-epoch1-op17-20/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..8483f0170061a397a5ebf4036c834d6eca7ab6b1 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op17-20/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block8-epoch1-op17-20_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 0.9968847352024922, + "pass@4": 0.9968847352024922, + "pass@8": 0.9968847352024922, + "pass@16": 0.9968847352024922, + "pass@32": 0.9968847352024922, + "pass@64": 0.9968847352024922, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9938837920489296, + "pass@2": 0.9969418960244648, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.984375, + "pass@2": 0.9875, + "pass@4": 0.990625, + "pass@8": 0.99375, + "pass@16": 0.99375, + "pass@32": 0.99375, + "pass@64": 0.99375, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9301587301587302, + "pass@2": 0.9555555555555556, + "pass@4": 0.9714285714285714, + "pass@8": 0.9809523809523809, + "pass@16": 0.9841269841269841, + "pass@32": 0.9968253968253968, + "pass@64": 0.9968253968253968, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9550898203592815, + "pass@2": 0.9610778443113772, + "pass@4": 0.9760479041916168, + "pass@8": 0.9940119760479041, + "pass@16": 0.9970059880239521, + "pass@32": 0.9970059880239521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8757396449704142, + "pass@2": 0.9230769230769231, + "pass@4": 0.9644970414201184, + "pass@8": 0.9733727810650887, + "pass@16": 0.9881656804733728, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.7660256410256411, + "pass@2": 0.842948717948718, + "pass@4": 0.9326923076923077, + "pass@8": 0.9743589743589743, + "pass@16": 0.9903846153846154, + "pass@32": 0.9935897435897436, + "pass@64": 0.9967948717948718, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7211267605633803, + "pass@2": 0.7859154929577464, + "pass@4": 0.8788732394366198, + "pass@8": 0.9436619718309859, + "pass@16": 0.9859154929577465, + "pass@32": 0.9943661971830986, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "10": { + "pass@1": 0.6076696165191741, + "pass@2": 0.6873156342182891, + "pass@4": 0.7876106194690266, + "pass@8": 0.8584070796460177, + "pass@16": 0.9026548672566371, + "pass@32": 0.9233038348082596, + "pass@64": 0.9321533923303835, + "pass@128": 0.9410029498525073 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9968847352024922, + "3": 0.9938837920489296, + "4": 0.984375, + "5": 0.9301587301587302, + "6": 0.9550898203592815, + "7": 0.8757396449704142, + "8": 0.7660256410256411, + "9": 0.7211267605633803, + "10": 0.6076696165191741 + }, + "pass@2": { + "2": 0.9968847352024922, + "3": 0.9969418960244648, + "4": 0.9875, + "5": 0.9555555555555556, + "6": 0.9610778443113772, + "7": 0.9230769230769231, + "8": 0.842948717948718, + "9": 0.7859154929577464, + "10": 0.6873156342182891 + }, + "pass@4": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.990625, + "5": 0.9714285714285714, + "6": 0.9760479041916168, + "7": 0.9644970414201184, + "8": 0.9326923076923077, + "9": 0.8788732394366198, + "10": 0.7876106194690266 + }, + "pass@8": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.99375, + "5": 0.9809523809523809, + "6": 0.9940119760479041, + "7": 0.9733727810650887, + "8": 0.9743589743589743, + "9": 0.9436619718309859, + "10": 0.8584070796460177 + }, + "pass@16": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.99375, + "5": 0.9841269841269841, + "6": 0.9970059880239521, + "7": 0.9881656804733728, + "8": 0.9903846153846154, + "9": 0.9859154929577465, + "10": 0.9026548672566371 + }, + "pass@32": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.99375, + "5": 0.9968253968253968, + "6": 0.9970059880239521, + "7": 0.9970414201183432, + "8": 0.9935897435897436, + "9": 0.9943661971830986, + "10": 0.9233038348082596 + }, + "pass@64": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.99375, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 0.9971830985915493, + "10": 0.9321533923303835 + }, + "pass@128": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 0.9971830985915493, + "10": 0.9410029498525073 + } + }, + "total_pass_at_k": { + "pass@1": 0.8676122931442081, + "pass@2": 0.9020601148260723, + "pass@4": 0.9429246876055387, + "pass@8": 0.9675785207700102, + "pass@16": 0.9817629179331308, + "pass@32": 0.9878419452887538, + "pass@64": 0.9898682877406282, + "pass@128": 0.9915569064505235 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9912536443148688, + "pass@2": 0.9941690962099126, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 0.9941520467836257, + "pass@4": 0.9970760233918129, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9323529411764706, + "pass@2": 0.961764705882353, + "pass@4": 0.9823529411764705, + "pass@8": 0.9882352941176471, + "pass@16": 0.9911764705882353, + "pass@32": 0.9941176470588236, + "pass@64": 0.9970588235294118, + "pass@128": 0.9970588235294118 + }, + "6": { + "pass@1": 0.9621451104100947, + "pass@2": 0.9652996845425867, + "pass@4": 0.9810725552050473, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8322784810126582, + "pass@2": 0.8607594936708861, + "pass@4": 0.930379746835443, + "pass@8": 0.9651898734177216, + "pass@16": 0.9841772151898734, + "pass@32": 0.990506329113924, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.7777777777777778, + "pass@2": 0.8618618618618619, + "pass@4": 0.93993993993994, + "pass@8": 0.963963963963964, + "pass@16": 0.984984984984985, + "pass@32": 0.990990990990991, + "pass@64": 0.993993993993994, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.7202572347266881, + "pass@2": 0.8038585209003215, + "pass@4": 0.9003215434083601, + "pass@8": 0.9389067524115756, + "pass@16": 0.977491961414791, + "pass@32": 0.9935691318327974, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.5701492537313433, + "pass@2": 0.6388059701492538, + "pass@4": 0.7492537313432835, + "pass@8": 0.808955223880597, + "pass@16": 0.8746268656716418, + "pass@32": 0.8985074626865671, + "pass@64": 0.9343283582089552, + "pass@128": 0.9492537313432836 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9912536443148688, + "4": 0.9883040935672515, + "5": 0.9323529411764706, + "6": 0.9621451104100947, + "7": 0.8322784810126582, + "8": 0.7777777777777778, + "9": 0.7202572347266881, + "10": 0.5701492537313433 + }, + "pass@2": { + "2": 1.0, + "3": 0.9941690962099126, + "4": 0.9941520467836257, + "5": 0.961764705882353, + "6": 0.9652996845425867, + "7": 0.8607594936708861, + "8": 0.8618618618618619, + "9": 0.8038585209003215, + "10": 0.6388059701492538 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9823529411764705, + "6": 0.9810725552050473, + "7": 0.930379746835443, + "8": 0.93993993993994, + "9": 0.9003215434083601, + "10": 0.7492537313432835 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9882352941176471, + "6": 1.0, + "7": 0.9651898734177216, + "8": 0.963963963963964, + "9": 0.9389067524115756, + "10": 0.808955223880597 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9911764705882353, + "6": 1.0, + "7": 0.9841772151898734, + "8": 0.984984984984985, + "9": 0.977491961414791, + "10": 0.8746268656716418 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9941176470588236, + "6": 1.0, + "7": 0.990506329113924, + "8": 0.990990990990991, + "9": 0.9935691318327974, + "10": 0.8985074626865671 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.993993993993994, + "9": 1.0, + "10": 0.9343283582089552 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9492537313432836 + } + }, + "total_pass_at_k": { + "pass@1": 0.8660863742885838, + "pass@2": 0.8995647807164379, + "pass@4": 0.9430867090726481, + "pass@8": 0.962838968865082, + "pass@16": 0.978908603950452, + "pass@32": 0.9849347171074657, + "pass@64": 0.9906260462002009, + "pass@128": 0.9926347505858721 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9969604863221885, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9878787878787879, + "pass@2": 0.996969696969697, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9733727810650887, + "pass@2": 0.9881656804733728, + "pass@4": 0.9940828402366864, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9449275362318841, + "pass@2": 0.9652173913043478, + "pass@4": 0.9884057971014493, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 0.9971014492753624, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9484240687679083, + "pass@2": 0.9713467048710601, + "pass@4": 0.9770773638968482, + "pass@8": 0.9828080229226361, + "pass@16": 0.9914040114613181, + "pass@32": 0.994269340974212, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.8670520231213873, + "pass@2": 0.9046242774566474, + "pass@4": 0.9479768786127167, + "pass@8": 0.9739884393063584, + "pass@16": 0.9913294797687862, + "pass@32": 0.9942196531791907, + "pass@64": 0.9971098265895953, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8084507042253521, + "pass@2": 0.895774647887324, + "pass@4": 0.9380281690140845, + "pass@8": 0.9690140845070423, + "pass@16": 0.9887323943661972, + "pass@32": 0.9943661971830986, + "pass@64": 0.9943661971830986, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.6946107784431138, + "pass@2": 0.7934131736526946, + "pass@4": 0.8652694610778443, + "pass@8": 0.9401197604790419, + "pass@16": 0.9760479041916168, + "pass@32": 0.9910179640718563, + "pass@64": 0.9910179640718563, + "pass@128": 0.9940119760479041 + }, + "10": { + "pass@1": 0.6257668711656442, + "pass@2": 0.7055214723926381, + "pass@4": 0.7730061349693251, + "pass@8": 0.8466257668711656, + "pass@16": 0.901840490797546, + "pass@32": 0.9294478527607362, + "pass@64": 0.9539877300613497, + "pass@128": 0.9631901840490797 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9969604863221885, + "3": 0.9878787878787879, + "4": 0.9733727810650887, + "5": 0.9449275362318841, + "6": 0.9484240687679083, + "7": 0.8670520231213873, + "8": 0.8084507042253521, + "9": 0.6946107784431138, + "10": 0.6257668711656442 + }, + "pass@2": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9881656804733728, + "5": 0.9652173913043478, + "6": 0.9713467048710601, + "7": 0.9046242774566474, + "8": 0.895774647887324, + "9": 0.7934131736526946, + "10": 0.7055214723926381 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9940828402366864, + "5": 0.9884057971014493, + "6": 0.9770773638968482, + "7": 0.9479768786127167, + "8": 0.9380281690140845, + "9": 0.8652694610778443, + "10": 0.7730061349693251 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.9828080229226361, + "7": 0.9739884393063584, + "8": 0.9690140845070423, + "9": 0.9401197604790419, + "10": 0.8466257668711656 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.9914040114613181, + "7": 0.9913294797687862, + "8": 0.9887323943661972, + "9": 0.9760479041916168, + "10": 0.901840490797546 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.994269340974212, + "7": 0.9942196531791907, + "8": 0.9943661971830986, + "9": 0.9910179640718563, + "10": 0.9294478527607362 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9971098265895953, + "8": 0.9943661971830986, + "9": 0.9910179640718563, + "10": 0.9539877300613497 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 1.0, + "8": 1.0, + "9": 0.9940119760479041, + "10": 0.9631901840490797 + } + }, + "total_pass_at_k": { + "pass@1": 0.8725425950196593, + "pass@2": 0.9141546526867628, + "pass@4": 0.9433158584534731, + "pass@8": 0.9682175622542595, + "pass@16": 0.9832896461336829, + "pass@32": 0.9891874180865007, + "pass@64": 0.9927916120576671, + "pass@128": 0.9950851900393185 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.998, + "pass@2": 0.999, + "pass@4": 0.999, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.991, + "pass@2": 0.996, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.982, + "pass@2": 0.99, + "pass@4": 0.994, + "pass@8": 0.997, + "pass@16": 0.997, + "pass@32": 0.997, + "pass@64": 0.997, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.936, + "pass@2": 0.961, + "pass@4": 0.981, + "pass@8": 0.989, + "pass@16": 0.991, + "pass@32": 0.996, + "pass@64": 0.998, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.955, + "pass@2": 0.966, + "pass@4": 0.978, + "pass@8": 0.992, + "pass@16": 0.996, + "pass@32": 0.997, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.859, + "pass@2": 0.897, + "pass@4": 0.948, + "pass@8": 0.971, + "pass@16": 0.988, + "pass@32": 0.994, + "pass@64": 0.996, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.785, + "pass@2": 0.868, + "pass@4": 0.937, + "pass@8": 0.969, + "pass@16": 0.988, + "pass@32": 0.993, + "pass@64": 0.995, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.712, + "pass@2": 0.794, + "pass@4": 0.881, + "pass@8": 0.941, + "pass@16": 0.98, + "pass@32": 0.993, + "pass@64": 0.996, + "pass@128": 0.997 + }, + "10": { + "pass@1": 0.601, + "pass@2": 0.677, + "pass@4": 0.77, + "pass@8": 0.838, + "pass@16": 0.893, + "pass@32": 0.917, + "pass@64": 0.94, + "pass@128": 0.951 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.998, + "3": 0.991, + "4": 0.982, + "5": 0.936, + "6": 0.955, + "7": 0.859, + "8": 0.785, + "9": 0.712, + "10": 0.601 + }, + "pass@2": { + "2": 0.999, + "3": 0.996, + "4": 0.99, + "5": 0.961, + "6": 0.966, + "7": 0.897, + "8": 0.868, + "9": 0.794, + "10": 0.677 + }, + "pass@4": { + "2": 0.999, + "3": 1.0, + "4": 0.994, + "5": 0.981, + "6": 0.978, + "7": 0.948, + "8": 0.937, + "9": 0.881, + "10": 0.77 + }, + "pass@8": { + "2": 0.999, + "3": 1.0, + "4": 0.997, + "5": 0.989, + "6": 0.992, + "7": 0.971, + "8": 0.969, + "9": 0.941, + "10": 0.838 + }, + "pass@16": { + "2": 0.999, + "3": 1.0, + "4": 0.997, + "5": 0.991, + "6": 0.996, + "7": 0.988, + "8": 0.988, + "9": 0.98, + "10": 0.893 + }, + "pass@32": { + "2": 0.999, + "3": 1.0, + "4": 0.997, + "5": 0.996, + "6": 0.997, + "7": 0.994, + "8": 0.993, + "9": 0.993, + "10": 0.917 + }, + "pass@64": { + "2": 0.999, + "3": 1.0, + "4": 0.997, + "5": 0.998, + "6": 0.999, + "7": 0.996, + "8": 0.995, + "9": 0.996, + "10": 0.94 + }, + "pass@128": { + "2": 0.999, + "3": 1.0, + "4": 0.999, + "5": 0.998, + "6": 0.999, + "7": 0.997, + "8": 0.998, + "9": 0.997, + "10": 0.951 + } + }, + "total_pass_at_k": { + "pass@1": 0.8687777777777778, + "pass@2": 0.9053333333333333, + "pass@4": 0.9431111111111111, + "pass@8": 0.9662222222222222, + "pass@16": 0.9813333333333333, + "pass@32": 0.9873333333333333, + "pass@64": 0.9911111111111112, + "pass@128": 0.9931111111111111 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block8-epoch1-op17-20_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5325779036827195, + "pass@2": 0.6005665722379604, + "pass@4": 0.6912181303116147, + "pass@8": 0.7818696883852692, + "pass@16": 0.8526912181303116, + "pass@32": 0.8895184135977338, + "pass@64": 0.9121813031161473, + "pass@128": 0.9263456090651558 + }, + "12": { + "pass@1": 0.5451977401129944, + "pass@2": 0.6214689265536724, + "pass@4": 0.692090395480226, + "pass@8": 0.7598870056497176, + "pass@16": 0.7994350282485876, + "pass@32": 0.8389830508474576, + "pass@64": 0.8587570621468926, + "pass@128": 0.8757062146892656 + }, + "13": { + "pass@1": 0.3949843260188088, + "pass@2": 0.48589341692789967, + "pass@4": 0.5579937304075235, + "pass@8": 0.6238244514106583, + "pass@16": 0.6927899686520376, + "pass@32": 0.7460815047021944, + "pass@64": 0.7962382445141066, + "pass@128": 0.8213166144200627 + }, + "14": { + "pass@1": 0.3644859813084112, + "pass@2": 0.4392523364485981, + "pass@4": 0.4766355140186916, + "pass@8": 0.5202492211838006, + "pass@16": 0.5856697819314641, + "pass@32": 0.6666666666666666, + "pass@64": 0.7133956386292835, + "pass@128": 0.7757009345794392 + }, + "15": { + "pass@1": 0.2774566473988439, + "pass@2": 0.3352601156069364, + "pass@4": 0.3901734104046243, + "pass@8": 0.4393063583815029, + "pass@16": 0.5173410404624278, + "pass@32": 0.6184971098265896, + "pass@64": 0.6763005780346821, + "pass@128": 0.7427745664739884 + }, + "16": { + "pass@1": 0.23870967741935484, + "pass@2": 0.2870967741935484, + "pass@4": 0.31290322580645163, + "pass@8": 0.38064516129032255, + "pass@16": 0.4290322580645161, + "pass@32": 0.4806451612903226, + "pass@64": 0.5258064516129032, + "pass@128": 0.5774193548387097 + }, + "17": { + "pass@1": 0.18497109826589594, + "pass@2": 0.24277456647398843, + "pass@4": 0.27167630057803466, + "pass@8": 0.315028901734104, + "pass@16": 0.37283236994219654, + "pass@32": 0.42196531791907516, + "pass@64": 0.47109826589595377, + "pass@128": 0.5086705202312138 + }, + "18": { + "pass@1": 0.17142857142857143, + "pass@2": 0.18857142857142858, + "pass@4": 0.21428571428571427, + "pass@8": 0.24857142857142858, + "pass@16": 0.2714285714285714, + "pass@32": 0.32, + "pass@64": 0.36, + "pass@128": 0.4114285714285714 + }, + "19": { + "pass@1": 0.07807807807807808, + "pass@2": 0.0960960960960961, + "pass@4": 0.12612612612612611, + "pass@8": 0.15915915915915915, + "pass@16": 0.17117117117117117, + "pass@32": 0.2042042042042042, + "pass@64": 0.23723723723723725, + "pass@128": 0.2732732732732733 + }, + "20": { + "pass@1": 0.08108108108108109, + "pass@2": 0.12612612612612611, + "pass@4": 0.14714714714714713, + "pass@8": 0.18618618618618618, + "pass@16": 0.2072072072072072, + "pass@32": 0.22822822822822822, + "pass@64": 0.25225225225225223, + "pass@128": 0.27927927927927926 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5325779036827195, + "12": 0.5451977401129944, + "13": 0.3949843260188088, + "14": 0.3644859813084112, + "15": 0.2774566473988439, + "16": 0.23870967741935484, + "17": 0.18497109826589594, + "18": 0.17142857142857143, + "19": 0.07807807807807808, + "20": 0.08108108108108109 + }, + "pass@2": { + "11": 0.6005665722379604, + "12": 0.6214689265536724, + "13": 0.48589341692789967, + "14": 0.4392523364485981, + "15": 0.3352601156069364, + "16": 0.2870967741935484, + "17": 0.24277456647398843, + "18": 0.18857142857142858, + "19": 0.0960960960960961, + "20": 0.12612612612612611 + }, + "pass@4": { + "11": 0.6912181303116147, + "12": 0.692090395480226, + "13": 0.5579937304075235, + "14": 0.4766355140186916, + "15": 0.3901734104046243, + "16": 0.31290322580645163, + "17": 0.27167630057803466, + "18": 0.21428571428571427, + "19": 0.12612612612612611, + "20": 0.14714714714714713 + }, + "pass@8": { + "11": 0.7818696883852692, + "12": 0.7598870056497176, + "13": 0.6238244514106583, + "14": 0.5202492211838006, + "15": 0.4393063583815029, + "16": 0.38064516129032255, + "17": 0.315028901734104, + "18": 0.24857142857142858, + "19": 0.15915915915915915, + "20": 0.18618618618618618 + }, + "pass@16": { + "11": 0.8526912181303116, + "12": 0.7994350282485876, + "13": 0.6927899686520376, + "14": 0.5856697819314641, + "15": 0.5173410404624278, + "16": 0.4290322580645161, + "17": 0.37283236994219654, + "18": 0.2714285714285714, + "19": 0.17117117117117117, + "20": 0.2072072072072072 + }, + "pass@32": { + "11": 0.8895184135977338, + "12": 0.8389830508474576, + "13": 0.7460815047021944, + "14": 0.6666666666666666, + "15": 0.6184971098265896, + "16": 0.4806451612903226, + "17": 0.42196531791907516, + "18": 0.32, + "19": 0.2042042042042042, + "20": 0.22822822822822822 + }, + "pass@64": { + "11": 0.9121813031161473, + "12": 0.8587570621468926, + "13": 0.7962382445141066, + "14": 0.7133956386292835, + "15": 0.6763005780346821, + "16": 0.5258064516129032, + "17": 0.47109826589595377, + "18": 0.36, + "19": 0.23723723723723725, + "20": 0.25225225225225223 + }, + "pass@128": { + "11": 0.9263456090651558, + "12": 0.8757062146892656, + "13": 0.8213166144200627, + "14": 0.7757009345794392, + "15": 0.7427745664739884, + "16": 0.5774193548387097, + "17": 0.5086705202312138, + "18": 0.4114285714285714, + "19": 0.2732732732732733, + "20": 0.27927927927927926 + } + }, + "total_pass_at_k": { + "pass@1": 0.28855869242199106, + "pass@2": 0.3438335809806835, + "pass@4": 0.3898959881129272, + "pass@8": 0.44338781575037145, + "pass@16": 0.4918276374442793, + "pass@32": 0.5432392273402674, + "pass@64": 0.5818722139673106, + "pass@128": 0.6205052005943537 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5301587301587302, + "pass@2": 0.6031746031746031, + "pass@4": 0.7047619047619048, + "pass@8": 0.8095238095238095, + "pass@16": 0.8761904761904762, + "pass@32": 0.9333333333333333, + "pass@64": 0.9555555555555556, + "pass@128": 0.9619047619047619 + }, + "12": { + "pass@1": 0.5229357798165137, + "pass@2": 0.5963302752293578, + "pass@4": 0.6819571865443425, + "pass@8": 0.7614678899082569, + "pass@16": 0.8073394495412844, + "pass@32": 0.8562691131498471, + "pass@64": 0.8746177370030581, + "pass@128": 0.8960244648318043 + }, + "13": { + "pass@1": 0.41975308641975306, + "pass@2": 0.49074074074074076, + "pass@4": 0.5833333333333334, + "pass@8": 0.6481481481481481, + "pass@16": 0.691358024691358, + "pass@32": 0.7314814814814815, + "pass@64": 0.7530864197530864, + "pass@128": 0.7993827160493827 + }, + "14": { + "pass@1": 0.3115727002967359, + "pass@2": 0.39465875370919884, + "pass@4": 0.4658753709198813, + "pass@8": 0.5311572700296736, + "pass@16": 0.599406528189911, + "pass@32": 0.655786350148368, + "pass@64": 0.685459940652819, + "pass@128": 0.7329376854599406 + }, + "15": { + "pass@1": 0.25925925925925924, + "pass@2": 0.3425925925925926, + "pass@4": 0.3950617283950617, + "pass@8": 0.4537037037037037, + "pass@16": 0.5216049382716049, + "pass@32": 0.6111111111111112, + "pass@64": 0.6944444444444444, + "pass@128": 0.7623456790123457 + }, + "16": { + "pass@1": 0.2713864306784661, + "pass@2": 0.3333333333333333, + "pass@4": 0.3834808259587021, + "pass@8": 0.4336283185840708, + "pass@16": 0.4808259587020649, + "pass@32": 0.5398230088495575, + "pass@64": 0.5663716814159292, + "pass@128": 0.5988200589970502 + }, + "17": { + "pass@1": 0.1686746987951807, + "pass@2": 0.19879518072289157, + "pass@4": 0.24096385542168675, + "pass@8": 0.2740963855421687, + "pass@16": 0.3253012048192771, + "pass@32": 0.3855421686746988, + "pass@64": 0.4457831325301205, + "pass@128": 0.5030120481927711 + }, + "18": { + "pass@1": 0.1870967741935484, + "pass@2": 0.21935483870967742, + "pass@4": 0.24838709677419354, + "pass@8": 0.2709677419354839, + "pass@16": 0.2967741935483871, + "pass@32": 0.3419354838709677, + "pass@64": 0.3967741935483871, + "pass@128": 0.43548387096774194 + }, + "19": { + "pass@1": 0.08459214501510574, + "pass@2": 0.1148036253776435, + "pass@4": 0.13897280966767372, + "pass@8": 0.15709969788519637, + "pass@16": 0.18126888217522658, + "pass@32": 0.19939577039274925, + "pass@64": 0.24169184290030213, + "pass@128": 0.27794561933534745 + }, + "20": { + "pass@1": 0.07807807807807808, + "pass@2": 0.10510510510510511, + "pass@4": 0.14114114114114115, + "pass@8": 0.16216216216216217, + "pass@16": 0.17717717717717718, + "pass@32": 0.1981981981981982, + "pass@64": 0.2132132132132132, + "pass@128": 0.23723723723723725 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5301587301587302, + "12": 0.5229357798165137, + "13": 0.41975308641975306, + "14": 0.3115727002967359, + "15": 0.25925925925925924, + "16": 0.2713864306784661, + "17": 0.1686746987951807, + "18": 0.1870967741935484, + "19": 0.08459214501510574, + "20": 0.07807807807807808 + }, + "pass@2": { + "11": 0.6031746031746031, + "12": 0.5963302752293578, + "13": 0.49074074074074076, + "14": 0.39465875370919884, + "15": 0.3425925925925926, + "16": 0.3333333333333333, + "17": 0.19879518072289157, + "18": 0.21935483870967742, + "19": 0.1148036253776435, + "20": 0.10510510510510511 + }, + "pass@4": { + "11": 0.7047619047619048, + "12": 0.6819571865443425, + "13": 0.5833333333333334, + "14": 0.4658753709198813, + "15": 0.3950617283950617, + "16": 0.3834808259587021, + "17": 0.24096385542168675, + "18": 0.24838709677419354, + "19": 0.13897280966767372, + "20": 0.14114114114114115 + }, + "pass@8": { + "11": 0.8095238095238095, + "12": 0.7614678899082569, + "13": 0.6481481481481481, + "14": 0.5311572700296736, + "15": 0.4537037037037037, + "16": 0.4336283185840708, + "17": 0.2740963855421687, + "18": 0.2709677419354839, + "19": 0.15709969788519637, + "20": 0.16216216216216217 + }, + "pass@16": { + "11": 0.8761904761904762, + "12": 0.8073394495412844, + "13": 0.691358024691358, + "14": 0.599406528189911, + "15": 0.5216049382716049, + "16": 0.4808259587020649, + "17": 0.3253012048192771, + "18": 0.2967741935483871, + "19": 0.18126888217522658, + "20": 0.17717717717717718 + }, + "pass@32": { + "11": 0.9333333333333333, + "12": 0.8562691131498471, + "13": 0.7314814814814815, + "14": 0.655786350148368, + "15": 0.6111111111111112, + "16": 0.5398230088495575, + "17": 0.3855421686746988, + "18": 0.3419354838709677, + "19": 0.19939577039274925, + "20": 0.1981981981981982 + }, + "pass@64": { + "11": 0.9555555555555556, + "12": 0.8746177370030581, + "13": 0.7530864197530864, + "14": 0.685459940652819, + "15": 0.6944444444444444, + "16": 0.5663716814159292, + "17": 0.4457831325301205, + "18": 0.3967741935483871, + "19": 0.24169184290030213, + "20": 0.2132132132132132 + }, + "pass@128": { + "11": 0.9619047619047619, + "12": 0.8960244648318043, + "13": 0.7993827160493827, + "14": 0.7329376854599406, + "15": 0.7623456790123457, + "16": 0.5988200589970502, + "17": 0.5030120481927711, + "18": 0.43548387096774194, + "19": 0.27794561933534745, + "20": 0.23723723723723725 + } + }, + "total_pass_at_k": { + "pass@1": 0.2820904645476773, + "pass@2": 0.33863080684596575, + "pass@4": 0.3970048899755501, + "pass@8": 0.44865525672371637, + "pass@16": 0.4941931540342298, + "pass@32": 0.5437041564792175, + "pass@64": 0.5809902200488998, + "pass@128": 0.6188875305623472 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6325301204819277, + "pass@2": 0.7018072289156626, + "pass@4": 0.7740963855421686, + "pass@8": 0.8373493975903614, + "pass@16": 0.9066265060240963, + "pass@32": 0.927710843373494, + "pass@64": 0.9518072289156626, + "pass@128": 0.9608433734939759 + }, + "12": { + "pass@1": 0.4952978056426332, + "pass@2": 0.5736677115987461, + "pass@4": 0.6489028213166145, + "pass@8": 0.7304075235109718, + "pass@16": 0.786833855799373, + "pass@32": 0.8244514106583072, + "pass@64": 0.8463949843260188, + "pass@128": 0.8652037617554859 + }, + "13": { + "pass@1": 0.42857142857142855, + "pass@2": 0.5042016806722689, + "pass@4": 0.5658263305322129, + "pass@8": 0.6414565826330533, + "pass@16": 0.6918767507002801, + "pass@32": 0.7450980392156863, + "pass@64": 0.7871148459383753, + "pass@128": 0.8179271708683473 + }, + "14": { + "pass@1": 0.3538011695906433, + "pass@2": 0.41812865497076024, + "pass@4": 0.48830409356725146, + "pass@8": 0.5350877192982456, + "pass@16": 0.6198830409356725, + "pass@32": 0.6900584795321637, + "pass@64": 0.7339181286549707, + "pass@128": 0.7865497076023392 + }, + "15": { + "pass@1": 0.30303030303030304, + "pass@2": 0.3575757575757576, + "pass@4": 0.403030303030303, + "pass@8": 0.48484848484848486, + "pass@16": 0.5363636363636364, + "pass@32": 0.5848484848484848, + "pass@64": 0.6545454545454545, + "pass@128": 0.7242424242424242 + }, + "16": { + "pass@1": 0.2222222222222222, + "pass@2": 0.25925925925925924, + "pass@4": 0.3076923076923077, + "pass@8": 0.37037037037037035, + "pass@16": 0.4301994301994302, + "pass@32": 0.4843304843304843, + "pass@64": 0.5584045584045584, + "pass@128": 0.6153846153846154 + }, + "17": { + "pass@1": 0.16459627329192547, + "pass@2": 0.19875776397515527, + "pass@4": 0.2391304347826087, + "pass@8": 0.2857142857142857, + "pass@16": 0.32919254658385094, + "pass@32": 0.37888198757763975, + "pass@64": 0.45652173913043476, + "pass@128": 0.5124223602484472 + }, + "18": { + "pass@1": 0.17941176470588235, + "pass@2": 0.21176470588235294, + "pass@4": 0.24411764705882352, + "pass@8": 0.28823529411764703, + "pass@16": 0.3323529411764706, + "pass@32": 0.34411764705882353, + "pass@64": 0.38823529411764707, + "pass@128": 0.4441176470588235 + }, + "19": { + "pass@1": 0.09226190476190477, + "pass@2": 0.1130952380952381, + "pass@4": 0.14583333333333334, + "pass@8": 0.1636904761904762, + "pass@16": 0.18154761904761904, + "pass@32": 0.22321428571428573, + "pass@64": 0.23809523809523808, + "pass@128": 0.28273809523809523 + }, + "20": { + "pass@1": 0.08982035928143713, + "pass@2": 0.11377245508982035, + "pass@4": 0.1437125748502994, + "pass@8": 0.16467065868263472, + "pass@16": 0.19461077844311378, + "pass@32": 0.2245508982035928, + "pass@64": 0.23952095808383234, + "pass@128": 0.2694610778443114 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6325301204819277, + "12": 0.4952978056426332, + "13": 0.42857142857142855, + "14": 0.3538011695906433, + "15": 0.30303030303030304, + "16": 0.2222222222222222, + "17": 0.16459627329192547, + "18": 0.17941176470588235, + "19": 0.09226190476190477, + "20": 0.08982035928143713 + }, + "pass@2": { + "11": 0.7018072289156626, + "12": 0.5736677115987461, + "13": 0.5042016806722689, + "14": 0.41812865497076024, + "15": 0.3575757575757576, + "16": 0.25925925925925924, + "17": 0.19875776397515527, + "18": 0.21176470588235294, + "19": 0.1130952380952381, + "20": 0.11377245508982035 + }, + "pass@4": { + "11": 0.7740963855421686, + "12": 0.6489028213166145, + "13": 0.5658263305322129, + "14": 0.48830409356725146, + "15": 0.403030303030303, + "16": 0.3076923076923077, + "17": 0.2391304347826087, + "18": 0.24411764705882352, + "19": 0.14583333333333334, + "20": 0.1437125748502994 + }, + "pass@8": { + "11": 0.8373493975903614, + "12": 0.7304075235109718, + "13": 0.6414565826330533, + "14": 0.5350877192982456, + "15": 0.48484848484848486, + "16": 0.37037037037037035, + "17": 0.2857142857142857, + "18": 0.28823529411764703, + "19": 0.1636904761904762, + "20": 0.16467065868263472 + }, + "pass@16": { + "11": 0.9066265060240963, + "12": 0.786833855799373, + "13": 0.6918767507002801, + "14": 0.6198830409356725, + "15": 0.5363636363636364, + "16": 0.4301994301994302, + "17": 0.32919254658385094, + "18": 0.3323529411764706, + "19": 0.18154761904761904, + "20": 0.19461077844311378 + }, + "pass@32": { + "11": 0.927710843373494, + "12": 0.8244514106583072, + "13": 0.7450980392156863, + "14": 0.6900584795321637, + "15": 0.5848484848484848, + "16": 0.4843304843304843, + "17": 0.37888198757763975, + "18": 0.34411764705882353, + "19": 0.22321428571428573, + "20": 0.2245508982035928 + }, + "pass@64": { + "11": 0.9518072289156626, + "12": 0.8463949843260188, + "13": 0.7871148459383753, + "14": 0.7339181286549707, + "15": 0.6545454545454545, + "16": 0.5584045584045584, + "17": 0.45652173913043476, + "18": 0.38823529411764707, + "19": 0.23809523809523808, + "20": 0.23952095808383234 + }, + "pass@128": { + "11": 0.9608433734939759, + "12": 0.8652037617554859, + "13": 0.8179271708683473, + "14": 0.7865497076023392, + "15": 0.7242424242424242, + "16": 0.6153846153846154, + "17": 0.5124223602484472, + "18": 0.4441176470588235, + "19": 0.28273809523809523, + "20": 0.2694610778443114 + } + }, + "total_pass_at_k": { + "pass@1": 0.295866785608088, + "pass@2": 0.34493012191495687, + "pass@4": 0.39577757954207554, + "pass@8": 0.44989592625631875, + "pass@16": 0.5007433838834374, + "pass@32": 0.5426702349093072, + "pass@64": 0.5854891465953018, + "pass@128": 0.6280107047279215 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.565, + "pass@2": 0.635, + "pass@4": 0.723, + "pass@8": 0.809, + "pass@16": 0.878, + "pass@32": 0.916, + "pass@64": 0.939, + "pass@128": 0.949 + }, + "12": { + "pass@1": 0.522, + "pass@2": 0.598, + "pass@4": 0.675, + "pass@8": 0.751, + "pass@16": 0.798, + "pass@32": 0.84, + "pass@64": 0.86, + "pass@128": 0.879 + }, + "13": { + "pass@1": 0.415, + "pass@2": 0.494, + "pass@4": 0.569, + "pass@8": 0.638, + "pass@16": 0.692, + "pass@32": 0.741, + "pass@64": 0.779, + "pass@128": 0.813 + }, + "14": { + "pass@1": 0.343, + "pass@2": 0.417, + "pass@4": 0.477, + "pass@8": 0.529, + "pass@16": 0.602, + "pass@32": 0.671, + "pass@64": 0.711, + "pass@128": 0.765 + }, + "15": { + "pass@1": 0.28, + "pass@2": 0.345, + "pass@4": 0.396, + "pass@8": 0.459, + "pass@16": 0.525, + "pass@32": 0.605, + "pass@64": 0.675, + "pass@128": 0.743 + }, + "16": { + "pass@1": 0.244, + "pass@2": 0.293, + "pass@4": 0.335, + "pass@8": 0.395, + "pass@16": 0.447, + "pass@32": 0.502, + "pass@64": 0.551, + "pass@128": 0.598 + }, + "17": { + "pass@1": 0.173, + "pass@2": 0.214, + "pass@4": 0.251, + "pass@8": 0.292, + "pass@16": 0.343, + "pass@32": 0.396, + "pass@64": 0.458, + "pass@128": 0.508 + }, + "18": { + "pass@1": 0.179, + "pass@2": 0.206, + "pass@4": 0.235, + "pass@8": 0.269, + "pass@16": 0.3, + "pass@32": 0.335, + "pass@64": 0.381, + "pass@128": 0.43 + }, + "19": { + "pass@1": 0.085, + "pass@2": 0.108, + "pass@4": 0.137, + "pass@8": 0.16, + "pass@16": 0.178, + "pass@32": 0.209, + "pass@64": 0.239, + "pass@128": 0.278 + }, + "20": { + "pass@1": 0.083, + "pass@2": 0.115, + "pass@4": 0.144, + "pass@8": 0.171, + "pass@16": 0.193, + "pass@32": 0.217, + "pass@64": 0.235, + "pass@128": 0.262 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.565, + "12": 0.522, + "13": 0.415, + "14": 0.343, + "15": 0.28, + "16": 0.244, + "17": 0.173, + "18": 0.179, + "19": 0.085, + "20": 0.083 + }, + "pass@2": { + "11": 0.635, + "12": 0.598, + "13": 0.494, + "14": 0.417, + "15": 0.345, + "16": 0.293, + "17": 0.214, + "18": 0.206, + "19": 0.108, + "20": 0.115 + }, + "pass@4": { + "11": 0.723, + "12": 0.675, + "13": 0.569, + "14": 0.477, + "15": 0.396, + "16": 0.335, + "17": 0.251, + "18": 0.235, + "19": 0.137, + "20": 0.144 + }, + "pass@8": { + "11": 0.809, + "12": 0.751, + "13": 0.638, + "14": 0.529, + "15": 0.459, + "16": 0.395, + "17": 0.292, + "18": 0.269, + "19": 0.16, + "20": 0.171 + }, + "pass@16": { + "11": 0.878, + "12": 0.798, + "13": 0.692, + "14": 0.602, + "15": 0.525, + "16": 0.447, + "17": 0.343, + "18": 0.3, + "19": 0.178, + "20": 0.193 + }, + "pass@32": { + "11": 0.916, + "12": 0.84, + "13": 0.741, + "14": 0.671, + "15": 0.605, + "16": 0.502, + "17": 0.396, + "18": 0.335, + "19": 0.209, + "20": 0.217 + }, + "pass@64": { + "11": 0.939, + "12": 0.86, + "13": 0.779, + "14": 0.711, + "15": 0.675, + "16": 0.551, + "17": 0.458, + "18": 0.381, + "19": 0.239, + "20": 0.235 + }, + "pass@128": { + "11": 0.949, + "12": 0.879, + "13": 0.813, + "14": 0.765, + "15": 0.743, + "16": 0.598, + "17": 0.508, + "18": 0.43, + "19": 0.278, + "20": 0.262 + } + }, + "total_pass_at_k": { + "pass@1": 0.2889, + "pass@2": 0.3425, + "pass@4": 0.3942, + "pass@8": 0.4473, + "pass@16": 0.4956, + "pass@32": 0.5432, + "pass@64": 0.5828, + "pass@128": 0.6225 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block8-epoch1-op17-20/summary.csv b/composition/SDAR-100M-block8-epoch1-op17-20/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..17e1e7ebd6b2765dd235f1fe1c67639ecd558f2b --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op17-20/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch1-op17-20,0.9763394097222222,0.608040625,0.7824979440789473,0.0,0.0,0.0,199.7977092013889,372.15247421875,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block8-epoch1-op17-20/summary.json b/composition/SDAR-100M-block8-epoch1-op17-20/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e6f6d82323f9b89125541c2f9f7489f9fc82b99f --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op17-20/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch1-op17-20", + "id_acc": 0.9763394097222222, + "ood_acc": 0.608040625, + "total_acc": 0.7824979440789473, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.7977092013889, + "ood_avg_resp_len": 372.15247421875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 631866, + "answer_accuracy": 0.7803435227631995, + "avg_response_len": 277.16480966447995, + "resp_tokens_sum": 224428107, + "pass_at_k": { + "pass@1": 0.7803435227631995, + "pass@2": 0.8294686416327026, + "pass@4": 0.8645065197450194, + "pass@8": 0.8903853309345585, + "pass@16": 0.9100911579608634, + "pass@32": 0.9257940995293442, + "pass@64": 0.9389749729084804, + "pass@128": 0.9505216566550743 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9147999631268436, + "pass@2": 0.9457115357814785, + "pass@4": 0.9644939742045515, + "pass@8": 0.976467977625549, + "pass@16": 0.9845557191374704, + "pass@32": 0.9904786548196389, + "pass@64": 0.9943848945020379, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 0.9968847352024922, + "pass@4": 0.9968847352024922, + "pass@8": 0.9968847352024922, + "pass@16": 0.9968847352024922, + "pass@32": 0.9968847352024922, + "pass@64": 0.9968847352024922, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9949589067278287, + "pass@2": 0.9971454435454742, + "pass@4": 0.998757460363326, + "pass@8": 0.9996935826438439, + "pass@16": 0.9999781984079241, + "pass@32": 0.9999999385362756, + "pass@64": 0.9999999999999962, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985107421875, + "pass@2": 0.9878137303149606, + "pass@4": 0.9902266852385638, + "pass@8": 0.992418635965382, + "pass@16": 0.9940056498208453, + "pass@32": 0.9952528850596998, + "pass@64": 0.9968748060893958, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9661954365079365, + "pass@2": 0.9754136201724785, + "pass@4": 0.9818137741710856, + "pass@8": 0.9867755777021514, + "pass@16": 0.9913209124055069, + "pass@32": 0.9954008763900372, + "pass@64": 0.9983700744266549, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9920471556886228, + "pass@2": 0.9955804434438211, + "pass@4": 0.9972394134677776, + "pass@8": 0.998204156494888, + "pass@16": 0.999004031075878, + "pass@32": 0.9997223135643741, + "pass@64": 0.999990730945279, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9807923446745562, + "pass@2": 0.9899740250663933, + "pass@4": 0.9942786351040437, + "pass@8": 0.9966196608845793, + "pass@16": 0.9978572891506724, + "pass@32": 0.9987563633971823, + "pass@64": 0.9996389129939015, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9767878605769231, + "pass@2": 0.9883080645568344, + "pass@4": 0.9921772134252452, + "pass@8": 0.9944861816978635, + "pass@16": 0.9959239960690798, + "pass@32": 0.9966348318465099, + "pass@64": 0.9967927316152657, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9637764084507042, + "pass@2": 0.9817663857158702, + "pass@4": 0.9897242576192058, + "pass@8": 0.994632208919704, + "pass@16": 0.9970542065127949, + "pass@32": 0.9978697672017515, + "pass@64": 0.9985915429355597, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8884782223796034, + "pass@2": 0.9233639250741679, + "pass@4": 0.9423458353116624, + "pass@8": 0.9535150564785112, + "pass@16": 0.9606443331526207, + "pass@32": 0.9671079842769559, + "pass@64": 0.9738184857703525, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.831302966101695, + "pass@2": 0.8819387678499936, + "pass@4": 0.9108717917145948, + "pass@8": 0.9301873731209482, + "pass@16": 0.9445633356962888, + "pass@32": 0.9571175271859526, + "pass@64": 0.9693614586588243, + "pass@128": 0.980225988700565 + }, + "13": { + "pass@1": 0.7450528996865203, + "pass@2": 0.817905672253351, + "pass@4": 0.8649554258303918, + "pass@8": 0.900549165326192, + "pass@16": 0.9298497520568195, + "pass@32": 0.9535119750141245, + "pass@64": 0.9707018746756205, + "pass@128": 0.9811912225705329 + }, + "14": { + "pass@1": 0.7362733644859814, + "pass@2": 0.8120324036598229, + "pass@4": 0.8572509446833161, + "pass@8": 0.8877305547576959, + "pass@16": 0.9128675754806822, + "pass@32": 0.9344930311271058, + "pass@64": 0.9501011437979755, + "pass@128": 0.9595015576323987 + }, + "15": { + "pass@1": 0.6830066835260116, + "pass@2": 0.7652260365936912, + "pass@4": 0.8216380976221894, + "pass@8": 0.8606056314618283, + "pass@16": 0.8899187889510047, + "pass@32": 0.9132096707385248, + "pass@64": 0.9310756807309161, + "pass@128": 0.9479768786127167 + }, + "16": { + "pass@1": 0.5638104838709678, + "pass@2": 0.6738494570739141, + "pass@4": 0.7523035856405045, + "pass@8": 0.8063191188545652, + "pass@16": 0.843643593454054, + "pass@32": 0.8708784076063162, + "pass@64": 0.8951979223494801, + "pass@128": 0.9225806451612903 + }, + "17": { + "pass@1": 0.49896134393063585, + "pass@2": 0.6055844920577128, + "pass@4": 0.6884978144892292, + "pass@8": 0.7497283063889675, + "pass@16": 0.7943429447842064, + "pass@32": 0.8296028860318351, + "pass@64": 0.8601444837620584, + "pass@128": 0.884393063583815 + }, + "18": { + "pass@1": 0.4401785714285714, + "pass@2": 0.5414805961754777, + "pass@4": 0.620062789919117, + "pass@8": 0.6825087231452688, + "pass@16": 0.7330450275756425, + "pass@32": 0.7715735323836825, + "pass@64": 0.8007257492775395, + "pass@128": 0.8228571428571428 + }, + "19": { + "pass@1": 0.3295091966966967, + "pass@2": 0.42963078728334647, + "pass@4": 0.5176115941588382, + "pass@8": 0.5916116555282213, + "pass@16": 0.6522660774252571, + "pass@32": 0.7026441292013839, + "pass@64": 0.7490627189135609, + "pass@128": 0.7957957957957958 + }, + "20": { + "pass@1": 0.3669294294294294, + "pass@2": 0.47350980409543403, + "pass@4": 0.5639906050595023, + "pass@8": 0.6350813513267202, + "pass@16": 0.6886366357150525, + "pass@32": 0.7318928034540644, + "pass@64": 0.770224695930882, + "pass@128": 0.8048048048048048 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 644522, + "answer_accuracy": 0.7849303390491037, + "avg_response_len": 286.6854004286828, + "resp_tokens_sum": 235403116, + "pass_at_k": { + "pass@1": 0.7849303390491037, + "pass@2": 0.8327475896183206, + "pass@4": 0.8680801569425771, + "pass@8": 0.8946791462011701, + "pass@16": 0.9147568033503521, + "pass@32": 0.9298668825918631, + "pass@64": 0.9416667394710053, + "pass@128": 0.951987529228371 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.915763995398773, + "pass@2": 0.9480459881165166, + "pass@4": 0.9681581655245539, + "pass@8": 0.9794301390879024, + "pass@16": 0.9853206794298328, + "pass@32": 0.989459399841, + "pass@64": 0.9934911833242681, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9990976443768997, + "pass@2": 0.9997371089916952, + "pass@4": 0.9999789687193356, + "pass@8": 0.9999998960323445, + "pass@16": 0.9999999999992757, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.990814393939394, + "pass@2": 0.9942462270341206, + "pass@4": 0.9969956352046904, + "pass@8": 0.9983947582818798, + "pass@16": 0.9992437871465215, + "pass@32": 0.9998493181618995, + "pass@64": 0.9999979765592963, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9959781804733728, + "pass@2": 0.9972641872058892, + "pass@4": 0.9977194152506086, + "pass@8": 0.9982611890819908, + "pass@16": 0.9990160343548016, + "pass@32": 0.9997255997961729, + "pass@64": 0.9999908406382343, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9807065217391304, + "pass@2": 0.9866305774278213, + "pass@4": 0.9910699586464737, + "pass@8": 0.9945806050238211, + "pass@16": 0.997016986481387, + "pass@32": 0.9983197945450435, + "pass@64": 0.9992809854172511, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9867030802292264, + "pass@2": 0.9902159995938904, + "pass@4": 0.9921985556783911, + "pass@8": 0.9933309007238308, + "pass@16": 0.9942316250648926, + "pass@32": 0.9949462457401639, + "pass@64": 0.9957019268812495, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9899972904624278, + "pass@2": 0.9941499590369122, + "pass@4": 0.9955985682425533, + "pass@8": 0.9965680962120179, + "pass@16": 0.9975371405733434, + "pass@32": 0.9983638978272487, + "pass@64": 0.9992831394824679, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9800836267605634, + "pass@2": 0.9910738050349339, + "pass@4": 0.9956219548260692, + "pass@8": 0.997544552113714, + "pass@16": 0.9983496167020315, + "pass@32": 0.9991228206283554, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9617795658682635, + "pass@2": 0.9778661181573863, + "pass@4": 0.9829365481635154, + "pass@8": 0.98602176677128, + "pass@16": 0.9891346058079603, + "pass@32": 0.9921804769068225, + "pass@64": 0.9949455674393275, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9141566265060241, + "pass@2": 0.9468826338819848, + "pass@4": 0.9644491272175316, + "pass@8": 0.9749877614946728, + "pass@16": 0.9813777882066246, + "pass@32": 0.986807323784196, + "pass@64": 0.9916378206179335, + "pass@128": 0.9939759036144579 + }, + "12": { + "pass@1": 0.8360844435736677, + "pass@2": 0.8800076518648338, + "pass@4": 0.9078132809809426, + "pass@8": 0.9269145444634191, + "pass@16": 0.9396337022070549, + "pass@32": 0.9486619447609619, + "pass@64": 0.9548071381739272, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7287289915966386, + "pass@2": 0.7963580609409118, + "pass@4": 0.841953924351893, + "pass@8": 0.876928300220035, + "pass@16": 0.904554157815763, + "pass@32": 0.9254462571555292, + "pass@64": 0.9401177901808709, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.7159173976608187, + "pass@2": 0.7988492597964728, + "pass@4": 0.856491532308461, + "pass@8": 0.8950593406393607, + "pass@16": 0.9211022424117776, + "pass@32": 0.9385845730381236, + "pass@64": 0.9499761266570184, + "pass@128": 0.9619883040935673 + }, + "15": { + "pass@1": 0.6883049242424243, + "pass@2": 0.7739773472918154, + "pass@4": 0.8314353032575473, + "pass@8": 0.8705559369310063, + "pass@16": 0.8982743454273422, + "pass@32": 0.9179786489223545, + "pass@64": 0.9320109430910483, + "pass@128": 0.9424242424242424 + }, + "16": { + "pass@1": 0.5450498575498576, + "pass@2": 0.6449591437288287, + "pass@4": 0.7198456640569499, + "pass@8": 0.7786853841035807, + "pass@16": 0.8259551044670943, + "pass@32": 0.8620595238262503, + "pass@64": 0.88890031356281, + "pass@128": 0.9116809116809117 + }, + "17": { + "pass@1": 0.5208171583850931, + "pass@2": 0.6292667475668802, + "pass@4": 0.7093731415366554, + "pass@8": 0.7695169309117884, + "pass@16": 0.8136355479127603, + "pass@32": 0.8447643060918684, + "pass@64": 0.8701041757502938, + "pass@128": 0.8944099378881988 + }, + "18": { + "pass@1": 0.48566176470588235, + "pass@2": 0.5931659622510422, + "pass@4": 0.6829272411720586, + "pass@8": 0.7536475960707826, + "pass@16": 0.8045954899897602, + "pass@32": 0.8405545218075591, + "pass@64": 0.8695012729380446, + "pass@128": 0.8970588235294118 + }, + "19": { + "pass@1": 0.32163783482142855, + "pass@2": 0.41333932379546307, + "pass@4": 0.49753887851072176, + "pass@8": 0.5725680674156789, + "pass@16": 0.6380391354389646, + "pass@32": 0.6926487886121019, + "pass@64": 0.7373374929774971, + "pass@128": 0.7738095238095238 + }, + "20": { + "pass@1": 0.3458317739520958, + "pass@2": 0.4579825339478522, + "pass@4": 0.5549417604610802, + "pass@8": 0.6303896133806527, + "pass@16": 0.6882139105165509, + "pass@32": 0.733040383229659, + "pass@64": 0.7701302877722762, + "pass@128": 0.8053892215568862 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 626647, + "answer_accuracy": 0.7821824073334398, + "avg_response_len": 307.9202261243809, + "resp_tokens_sum": 246690905, + "pass_at_k": { + "pass@1": 0.7821824073334398, + "pass@2": 0.8304118840523114, + "pass@4": 0.8661606999306755, + "pass@8": 0.8931690546690862, + "pass@16": 0.9137981552171506, + "pass@32": 0.9300239487024705, + "pass@64": 0.9430098116980766, + "pass@128": 0.9528678702668157 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9186567164179105, + "pass@2": 0.9492878863556237, + "pass@4": 0.9682844965274856, + "pass@8": 0.9806815470310305, + "pass@16": 0.988738350593314, + "pass@32": 0.9937525351988347, + "pass@64": 0.9964109344316464, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 0.9993303571428571, + "pass@2": 0.9999089566929135, + "pass@4": 0.9999980221222345, + "pass@8": 0.9999999993609641, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9940324344023324, + "pass@2": 0.9959951992608066, + "pass@4": 0.9977383308136045, + "pass@8": 0.9990797355776098, + "pass@16": 0.9998081573699138, + "pass@32": 0.999992540612001, + "pass@64": 0.9999999988420875, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9962993421052632, + "pass@2": 0.9970306959985263, + "pass@4": 0.9970758411229297, + "pass@8": 0.9970760233911381, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9735753676470589, + "pass@2": 0.9820366778601207, + "pass@4": 0.9875431995632898, + "pass@8": 0.9917505546806278, + "pass@16": 0.994889351721694, + "pass@32": 0.9974968435685705, + "pass@64": 0.9992203250542475, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9943069794952681, + "pass@2": 0.9978483072108101, + "pass@4": 0.9994458494344357, + "pass@8": 0.9999580528597197, + "pass@16": 0.9999998271433355, + "pass@32": 0.999999999999561, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9786392405063291, + "pass@2": 0.9860254130120601, + "pass@4": 0.9906770330528303, + "pass@8": 0.9942785219607105, + "pass@16": 0.9968841881317372, + "pass@32": 0.9988537819893496, + "pass@64": 0.9998879966359081, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9650431681681682, + "pass@2": 0.9763025821096688, + "pass@4": 0.982883417444441, + "pass@8": 0.9881286341578839, + "pass@16": 0.9924468973701125, + "pass@32": 0.9953769252358343, + "pass@64": 0.996775794068325, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9677451768488746, + "pass@2": 0.9829718079854168, + "pass@4": 0.9884183856037287, + "pass@8": 0.9913233575477782, + "pass@16": 0.9936550705400102, + "pass@32": 0.9960259491065468, + "pass@64": 0.998156322496061, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8957093253968254, + "pass@2": 0.9324045431821019, + "pass@4": 0.9530571291683771, + "pass@8": 0.9661005237823533, + "pass@16": 0.9753324158172761, + "pass@32": 0.9817093737531953, + "pass@64": 0.9852794285216058, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8548595183486238, + "pass@2": 0.9026862252883529, + "pass@4": 0.928716852767256, + "pass@8": 0.9458145843657703, + "pass@16": 0.9596806462958717, + "pass@32": 0.9727740288131387, + "pass@64": 0.9845103741794053, + "pass@128": 0.9938837920489296 + }, + "13": { + "pass@1": 0.728515625, + "pass@2": 0.8026403926071737, + "pass@4": 0.854766033875395, + "pass@8": 0.8928687155866702, + "pass@16": 0.920794415945995, + "pass@32": 0.94152406634448, + "pass@64": 0.9565434779815419, + "pass@128": 0.9660493827160493 + }, + "14": { + "pass@1": 0.7207205118694362, + "pass@2": 0.7957049230122201, + "pass@4": 0.8462405605271146, + "pass@8": 0.8793643436005479, + "pass@16": 0.9014542133541928, + "pass@32": 0.9174294131525637, + "pass@64": 0.9303251689752986, + "pass@128": 0.9406528189910979 + }, + "15": { + "pass@1": 0.6839554398148148, + "pass@2": 0.7633861305531252, + "pass@4": 0.8194027031343298, + "pass@8": 0.8602466462398325, + "pass@16": 0.8887469421251796, + "pass@32": 0.9087115703307098, + "pass@64": 0.9239886621606587, + "pass@128": 0.9351851851851852 + }, + "16": { + "pass@1": 0.5756130162241888, + "pass@2": 0.6845900692170115, + "pass@4": 0.761188147112142, + "pass@8": 0.8149786242373147, + "pass@16": 0.8555630664100949, + "pass@32": 0.8888727828261448, + "pass@64": 0.9169712192641652, + "pass@128": 0.9351032448377581 + }, + "17": { + "pass@1": 0.4842808734939759, + "pass@2": 0.584871350559719, + "pass@4": 0.6645132360149255, + "pass@8": 0.7297835726231425, + "pass@16": 0.785105968775507, + "pass@32": 0.831487801164966, + "pass@64": 0.8694257138315328, + "pass@128": 0.9006024096385542 + }, + "18": { + "pass@1": 0.4754536290322581, + "pass@2": 0.5808114522479044, + "pass@4": 0.6585383135575791, + "pass@8": 0.7148880869953559, + "pass@16": 0.7581914940782433, + "pass@32": 0.7942661179834702, + "pass@64": 0.8252641405234724, + "pass@128": 0.8548387096774194 + }, + "19": { + "pass@1": 0.325693919939577, + "pass@2": 0.4348405125246804, + "pass@4": 0.5347291311962133, + "pass@8": 0.6173469310391171, + "pass@16": 0.6816601314840767, + "pass@32": 0.7327839098583034, + "pass@64": 0.7748918502096696, + "pass@128": 0.8096676737160121 + }, + "20": { + "pass@1": 0.3164179804804805, + "pass@2": 0.4194657255680876, + "pass@4": 0.5166342914905905, + "pass@8": 0.6003871057227763, + "pass@16": 0.6667422102319082, + "pass@32": 0.7176055052180353, + "pass@64": 0.7583024072323226, + "pass@128": 0.7867867867867868 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_metrics.json b/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..57e223e468ad0c108ab88295bed94a680a17a959 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch1-op7-10", + "id": { + "answer_accuracy": 0.9847204861111111, + "avg_response_len": 199.6684453125, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9847204861111111, + "pass@2": 0.9886822779964979, + "pass@4": 0.9911352969837098, + "pass@8": 0.9930673279925417, + "pass@16": 0.9946392213312069, + "pass@32": 0.9958122468158805, + "pass@64": 0.9965993654566087, + "pass@128": 0.9972222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9323125, + "pass@2": 0.9499406988188978, + "pass@4": 0.9621583696100489, + "pass@8": 0.9718907539598126, + "pass@16": 0.9791354444504873, + "pass@32": 0.9840437401738955, + "pass@64": 0.9871711591050699, + "pass@128": 0.99 + }, + "2": { + "pass@1": 0.999828125, + "pass@2": 0.9999741633858267, + "pass@4": 0.999999438976378, + "pass@8": 0.9999999998576696, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998734375, + "pass@2": 0.999644438976378, + "pass@4": 0.9999579316647917, + "pass@8": 0.9999992906842913, + "pass@16": 0.9999999998783552, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995140625, + "pass@2": 0.9964927411417323, + "pass@4": 0.9973747537495314, + "pass@8": 0.9978259051966901, + "pass@16": 0.9979795297363963, + "pass@32": 0.9979997935880747, + "pass@64": 0.9979999999993951, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.97978125, + "pass@2": 0.9833897637795275, + "pass@4": 0.9860962986501687, + "pass@8": 0.9884541169280153, + "pass@16": 0.990718014081456, + "pass@32": 0.9926000208673001, + "pass@64": 0.9939421204598907, + "pass@128": 0.995 + }, + "6": { + "pass@1": 0.995984375, + "pass@2": 0.9974320866141734, + "pass@4": 0.9982878877952756, + "pass@8": 0.9988890262019753, + "pass@16": 0.9993300424147991, + "pass@32": 0.9996835374287463, + "pass@64": 0.9999404387018788, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9928359375, + "pass@2": 0.9942635334645668, + "pass@4": 0.9949447849643793, + "pass@8": 0.9955137921675251, + "pass@16": 0.9960397802495263, + "pass@32": 0.9964733721669579, + "pass@64": 0.9969998615760639, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.986671875, + "pass@2": 0.9904295029527562, + "pass@4": 0.9924173699850019, + "pass@8": 0.9938992253106746, + "pass@16": 0.9951051047857475, + "pass@32": 0.9960312897117236, + "pass@64": 0.9966908261256258, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.9811953125, + "pass@2": 0.9865735728346459, + "pass@4": 0.9889808374578175, + "pass@8": 0.9911338416262176, + "pass@16": 0.9934450763840873, + "pass@32": 0.9954784674062254, + "pass@64": 0.9966498831415526, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.9323125, + "2": 0.999828125, + "3": 0.998734375, + "4": 0.995140625, + "5": 0.97978125, + "6": 0.995984375, + "7": 0.9928359375, + "8": 0.986671875, + "9": 0.9811953125 + }, + "per_op_avg_response_len": { + "10": 279.759734375, + "2": 115.4348671875, + "3": 144.4752734375, + "4": 159.7785234375, + "5": 173.330203125, + "6": 198.6917734375, + "7": 223.7417265625, + "8": 242.8668671875, + "9": 258.9370390625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.759734375, + "2": 115.4348671875, + "3": 144.4752734375, + "4": 159.7785234375, + "5": 173.330203125, + "6": 198.6917734375, + "7": 223.7417265625, + "8": 242.8668671875, + "9": 258.9370390625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 373242, + "answer_accuracy": 0.9847866008105369, + "avg_response_len": 190.43884561803443, + "resp_tokens_sum": 72177846, + "pass_at_k": { + "pass@1": 0.9847866008105369, + "pass@2": 0.9891751396766894, + "pass@4": 0.9916788649265851, + "pass@8": 0.9935028080912455, + "pass@16": 0.9950088418387877, + "pass@32": 0.9962314724797251, + "pass@64": 0.9970623347944672, + "pass@128": 0.9976359338061466 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9362555309734514, + "pass@2": 0.9539477504471235, + "pass@4": 0.9651495407653686, + "pass@8": 0.973229583190713, + "pass@16": 0.9792803436377407, + "pass@32": 0.9841718673769317, + "pass@64": 0.9878246376873651, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983753822629969, + "pass@2": 0.9995236762262515, + "pass@4": 0.9999511242287376, + "pass@8": 0.9999994411862023, + "pass@16": 0.9999999999675167, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990673828125, + "pass@2": 0.9933247723917322, + "pass@4": 0.9952177153637045, + "pass@8": 0.9963542387601867, + "pass@16": 0.9968111376105945, + "pass@32": 0.9968743549630685, + "pass@64": 0.9968749999981098, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9707093253968254, + "pass@2": 0.9767396262967132, + "pass@4": 0.981683998131186, + "pass@8": 0.9859164493385404, + "pass@16": 0.9901710891964574, + "pass@32": 0.9936881458577858, + "pass@64": 0.995845992043236, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.997029378742515, + "pass@2": 0.9987512671507378, + "pass@4": 0.9995554992377451, + "pass@8": 0.999939295718634, + "pass@16": 0.9999991353050729, + "pass@32": 0.9999999999556102, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9958394970414202, + "pass@2": 0.9969697124120579, + "pass@4": 0.9970408646219519, + "pass@8": 0.9970414201059147, + "pass@16": 0.9970414201183432, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9894080528846154, + "pass@2": 0.9919966434484152, + "pass@4": 0.9931632229865497, + "pass@8": 0.9938714043202131, + "pass@16": 0.9943410041176739, + "pass@32": 0.9949967186169365, + "pass@64": 0.9959998990510801, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9859595070422535, + "pass@2": 0.9918774952866809, + "pass@4": 0.9937267724456977, + "pass@8": 0.9954155716338281, + "pass@16": 0.9975424861780133, + "pass@32": 0.9993143415342073, + "pass@64": 0.9999593384604206, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 384296, + "answer_accuracy": 0.9837196920052425, + "avg_response_len": 197.6666734927916, + "resp_tokens_sum": 77219672, + "pass_at_k": { + "pass@1": 0.9837196920052425, + "pass@2": 0.9873512979742215, + "pass@4": 0.9898586991214896, + "pass@8": 0.9920136735719006, + "pass@16": 0.9938161565342344, + "pass@32": 0.9951304248579761, + "pass@64": 0.9960841018262011, + "pass@128": 0.997051114023591 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9209403757668712, + "pass@2": 0.9380713794019612, + "pass@4": 0.9512705088436028, + "pass@8": 0.9631955401424825, + "pass@16": 0.9726077613588666, + "pass@32": 0.9782359909847477, + "pass@64": 0.9813867257620852, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982007575757575, + "pass@2": 0.9994191422094966, + "pass@4": 0.9999210212359819, + "pass@8": 0.9999984042922265, + "pass@16": 0.9999999996635676, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9975268121301775, + "pass@2": 0.9989032724921958, + "pass@4": 0.9997191858968812, + "pass@8": 0.9999779550101494, + "pass@16": 0.999999898523686, + "pass@32": 0.9999999999996828, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9875679347826087, + "pass@2": 0.9887766746547987, + "pass@4": 0.9894790216440336, + "pass@8": 0.9903547461693829, + "pass@16": 0.9916253749261505, + "pass@32": 0.9931572503524777, + "pass@64": 0.9949313065753761, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9943364971346705, + "pass@2": 0.9965329089863052, + "pass@4": 0.9980393601408706, + "pass@8": 0.9990811493323517, + "pass@16": 0.9997493490657728, + "pass@32": 0.9999854792814306, + "pass@64": 0.999999993573057, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9934293713872833, + "pass@2": 0.9942072077966412, + "pass@4": 0.9943974088679668, + "pass@8": 0.9945809233687215, + "pass@16": 0.994942196531792, + "pass@32": 0.9956647398843931, + "pass@64": 0.9971098265895953, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9836707746478873, + "pass@2": 0.987972371631363, + "pass@4": 0.9905331931219864, + "pass@8": 0.9926688812257355, + "pass@16": 0.9945746668596219, + "pass@32": 0.9961092494376016, + "pass@64": 0.9970112564626833, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9756034805389222, + "pass@2": 0.9805492361733227, + "pass@4": 0.9840381910532142, + "pass@8": 0.9872988101167206, + "pass@16": 0.9901482935355289, + "pass@32": 0.992455558127094, + "pass@64": 0.9937913521118914, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 376860, + "answer_accuracy": 0.9856775192500837, + "avg_response_len": 210.86303931620355, + "resp_tokens_sum": 80620531, + "pass_at_k": { + "pass@1": 0.9856775192500837, + "pass@2": 0.9895536498053247, + "pass@4": 0.9919008383284197, + "pass@8": 0.9937122214373127, + "pass@16": 0.9951137936236103, + "pass@32": 0.9960933293150033, + "pass@64": 0.9966669022462312, + "pass@128": 0.9969869434214932 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9393889925373134, + "pass@2": 0.9574362439769657, + "pass@4": 0.9697268340524599, + "pass@8": 0.9789975497663034, + "pass@16": 0.9853411276247855, + "pass@32": 0.9895658032002611, + "pass@64": 0.9921389084495922, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.99953125, + "pass@2": 0.9999261811023622, + "pass@4": 0.9999983970753656, + "pass@8": 0.9999999995933421, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995900145772595, + "pass@2": 0.9999763263010492, + "pass@4": 0.9999999330440837, + "pass@8": 0.9999999999999082, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9969618055555556, + "pass@2": 0.9970745844269466, + "pass@4": 0.9970760233918129, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9802849264705882, + "pass@2": 0.984084790412228, + "pass@4": 0.9867516963872165, + "pass@8": 0.9888766116409934, + "pass@16": 0.9903040195148527, + "pass@32": 0.9910264810308619, + "pass@64": 0.9911745064640191, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9966975552050473, + "pass@2": 0.9970321077274646, + "pass@4": 0.9972259128097947, + "pass@8": 0.9975709158200651, + "pass@16": 0.9981634334351739, + "pass@32": 0.9990176819380228, + "pass@64": 0.9998121165453686, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.988973496835443, + "pass@2": 0.9914306226951061, + "pass@4": 0.993302117891909, + "pass@8": 0.9949012426776844, + "pass@16": 0.9961701906630579, + "pass@32": 0.9967511777435377, + "pass@64": 0.996835004987544, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9873076201201201, + "pass@2": 0.9915806554586083, + "pass@4": 0.9937272097744536, + "pass@8": 0.9952369199027994, + "pass@16": 0.996386500798402, + "pass@32": 0.9969175073660387, + "pass@64": 0.9969965813136217, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.981762459807074, + "pass@2": 0.9869891225915893, + "pass@4": 0.9888716316007123, + "pass@8": 0.9903649875151894, + "pass@16": 0.9923085651126249, + "pass@32": 0.9943463657464062, + "pass@64": 0.9959421427097476, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.40368125, + "avg_response_len": 320.694090625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.40368125, + "pass@2": 0.46667384350394003, + "pass@4": 0.5202853888451408, + "pass@8": 0.566301432358076, + "pass@16": 0.60583017925103, + "pass@32": 0.6399910807570778, + "pass@64": 0.6699377993651461, + "pass@128": 0.6963 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.894578125, + "pass@2": 0.9257929379921268, + "pass@4": 0.9442590304649411, + "pass@8": 0.9561462437618224, + "pass@16": 0.9641911885882701, + "pass@32": 0.9700611686252756, + "pass@64": 0.9752575710913312, + "pass@128": 0.98 + }, + "12": { + "pass@1": 0.812484375, + "pass@2": 0.8672572588582684, + "pass@4": 0.8985465786464193, + "pass@8": 0.9196361714553097, + "pass@16": 0.9348869636510788, + "pass@32": 0.94673284148625, + "pass@64": 0.9576580920053203, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.5978046875, + "pass@2": 0.6833599901574804, + "pass@4": 0.7455164355080617, + "pass@8": 0.7936291630299753, + "pass@16": 0.8329134533248633, + "pass@32": 0.8646922500467323, + "pass@64": 0.8900894458435458, + "pass@128": 0.911 + }, + "14": { + "pass@1": 0.45890625, + "pass@2": 0.5430552411417325, + "pass@4": 0.6134239566929134, + "pass@8": 0.6705658372251655, + "pass@16": 0.7163013175856843, + "pass@32": 0.7538964282786471, + "pass@64": 0.7852020700568273, + "pass@128": 0.813 + }, + "15": { + "pass@1": 0.3480078125, + "pass@2": 0.4288660187007878, + "pass@4": 0.49983243916385395, + "pass@8": 0.561707925390508, + "pass@16": 0.6160022823168169, + "pass@32": 0.6632416916320305, + "pass@64": 0.7017246400131141, + "pass@128": 0.73 + }, + "16": { + "pass@1": 0.237484375, + "pass@2": 0.3018716781496062, + "pass@4": 0.36146487692163465, + "pass@8": 0.41654215784915727, + "pass@16": 0.467808763276434, + "pass@32": 0.5155089371433209, + "pass@64": 0.5604432358720062, + "pass@128": 0.602 + }, + "17": { + "pass@1": 0.2055234375, + "pass@2": 0.26673203740157486, + "pass@4": 0.3232418611736032, + "pass@8": 0.37323857044448777, + "pass@16": 0.41837508369464593, + "pass@32": 0.4601245400414008, + "pass@64": 0.4978334061713972, + "pass@128": 0.53 + }, + "18": { + "pass@1": 0.1816796875, + "pass@2": 0.24128063484251946, + "pass@4": 0.30061707564679396, + "pass@8": 0.35639546958098467, + "pass@16": 0.40555332405555866, + "pass@32": 0.44808731643199917, + "pass@64": 0.4868599251210825, + "pass@128": 0.525 + }, + "19": { + "pass@1": 0.150984375, + "pass@2": 0.2036835629921259, + "pass@4": 0.25508346662917125, + "pass@8": 0.30095403253935643, + "pass@16": 0.3403886075532727, + "pass@32": 0.37524624378973637, + "pass@64": 0.40789729833590765, + "pass@128": 0.44 + }, + "20": { + "pass@1": 0.149359375, + "pass@2": 0.20483907480314964, + "pass@4": 0.2608681676040494, + "pass@8": 0.31419875230396843, + "pass@16": 0.3618808084635515, + "pass@32": 0.4023193900953739, + "pass@64": 0.436412309140885, + "pass@128": 0.464 + } + }, + "per_op_accuracy": { + "11": 0.894578125, + "12": 0.812484375, + "13": 0.5978046875, + "14": 0.45890625, + "15": 0.3480078125, + "16": 0.237484375, + "17": 0.2055234375, + "18": 0.1816796875, + "19": 0.150984375, + "20": 0.149359375 + }, + "per_op_avg_response_len": { + "11": 296.2161171875, + "12": 304.9042109375, + "13": 316.20178125, + "14": 326.5206875, + "15": 330.5811015625, + "16": 321.36221875, + "17": 328.7864765625, + "18": 332.2248125, + "19": 322.4753359375, + "20": 327.6681640625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 296.2161171875, + "12": 304.9042109375, + "13": 316.20178125, + "14": 326.5206875, + "15": 330.5811015625, + "16": 321.36221875, + "17": 328.7864765625, + "18": 332.2248125, + "19": 322.4753359375, + "20": 327.6681640625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 174941, + "answer_accuracy": 0.40615945393759284, + "avg_response_len": 304.98383172362554, + "resp_tokens_sum": 131362636, + "pass_at_k": { + "pass@1": 0.40615945393759284, + "pass@2": 0.4685217793169621, + "pass@4": 0.5206486526201332, + "pass@8": 0.564898064058231, + "pass@16": 0.6029430039326625, + "pass@32": 0.6357447987596025, + "pass@64": 0.6650622830418412, + "pass@128": 0.6921248142644874 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8889872521246459, + "pass@2": 0.9209322232383845, + "pass@4": 0.9403707372980644, + "pass@8": 0.9523073338351158, + "pass@16": 0.9599911546995261, + "pass@32": 0.9656664426486973, + "pass@64": 0.97105654419183, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.807468220338983, + "pass@2": 0.8675287907602651, + "pass@4": 0.9025985565363646, + "pass@8": 0.9234312139724606, + "pass@16": 0.9362103618765187, + "pass@32": 0.9450079869218969, + "pass@64": 0.9525377320034726, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.6421923981191222, + "pass@2": 0.7228887178189717, + "pass@4": 0.7779022200986624, + "pass@8": 0.8182676410254136, + "pass@16": 0.8505300132234627, + "pass@32": 0.8773909089095185, + "pass@64": 0.9017227414682624, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.4485007788161994, + "pass@2": 0.5347201780852163, + "pass@4": 0.6061306235669136, + "pass@8": 0.6653760404004355, + "pass@16": 0.7137935405594372, + "pass@32": 0.7534356762985341, + "pass@64": 0.7870531477671298, + "pass@128": 0.8130841121495327 + }, + "15": { + "pass@1": 0.3333408598265896, + "pass@2": 0.41465276671521556, + "pass@4": 0.4854122544653017, + "pass@8": 0.5467994374688726, + "pass@16": 0.6020372401229079, + "pass@32": 0.6496721898260739, + "pass@64": 0.686791777517624, + "pass@128": 0.7138728323699421 + }, + "16": { + "pass@1": 0.2532258064516129, + "pass@2": 0.3180189547879094, + "pass@4": 0.374151785260713, + "pass@8": 0.42593478016771535, + "pass@16": 0.4749139463216263, + "pass@32": 0.5228389769740335, + "pass@64": 0.5721693101663297, + "pass@128": 0.6193548387096774 + }, + "17": { + "pass@1": 0.19495122832369943, + "pass@2": 0.2523230195484954, + "pass@4": 0.3023951223040761, + "pass@8": 0.3470740122741218, + "pass@16": 0.39059013486586935, + "pass@32": 0.4324684907446193, + "pass@64": 0.46828111323798854, + "pass@128": 0.49421965317919075 + }, + "18": { + "pass@1": 0.16796875, + "pass@2": 0.22441437007874018, + "pass@4": 0.28228091943864153, + "pass@8": 0.33871351937233285, + "pass@16": 0.38905522009524307, + "pass@32": 0.4321079296175608, + "pass@64": 0.47367519903850336, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.1451998873873874, + "pass@2": 0.20028788631150843, + "pass@4": 0.2549552702308609, + "pass@8": 0.30319246623896356, + "pass@16": 0.34293555954657895, + "pass@32": 0.3751435020629878, + "pass@64": 0.4044465014859663, + "pass@128": 0.43543543543543545 + }, + "20": { + "pass@1": 0.1495870870870871, + "pass@2": 0.20280615260930232, + "pass@4": 0.25683599347378866, + "pass@8": 0.30743826429217297, + "pass@16": 0.3517171120545327, + "pass@32": 0.38878767049319296, + "pass@64": 0.4209157334351972, + "pass@128": 0.45045045045045046 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 170169, + "answer_accuracy": 0.39531528768956287, + "avg_response_len": 315.663583946625, + "resp_tokens_sum": 135881809, + "pass_at_k": { + "pass@1": 0.39531528768956287, + "pass@2": 0.4581659987918574, + "pass@4": 0.5125038153404343, + "pass@8": 0.5601027407054775, + "pass@16": 0.6017583420304699, + "pass@32": 0.638086461585684, + "pass@64": 0.6691368394613612, + "pass@128": 0.6946179006839132 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8997552710843374, + "pass@2": 0.9309293028412866, + "pass@4": 0.9496467379642303, + "pass@8": 0.9619478037609, + "pass@16": 0.9701481675058946, + "pass@32": 0.9759861111837409, + "pass@64": 0.981159773824385, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8052752742946708, + "pass@2": 0.8523317361093967, + "pass@4": 0.8786195395246442, + "pass@8": 0.8979108712169228, + "pass@16": 0.9138855036221462, + "pass@32": 0.9282479851031312, + "pass@64": 0.9436447705145927, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.5835084033613446, + "pass@2": 0.6718991238227576, + "pass@4": 0.7351696928115078, + "pass@8": 0.7855705437580075, + "pass@16": 0.8282254349495101, + "pass@32": 0.8632778064510783, + "pass@64": 0.8906273038627206, + "pass@128": 0.9103641456582633 + }, + "14": { + "pass@1": 0.4359466374269006, + "pass@2": 0.5169355372519222, + "pass@4": 0.5883138300146694, + "pass@8": 0.6484225858141512, + "pass@16": 0.6976308033859067, + "pass@32": 0.7384376268134236, + "pass@64": 0.7720516569697223, + "pass@128": 0.804093567251462 + }, + "15": { + "pass@1": 0.3393465909090909, + "pass@2": 0.4214902469577665, + "pass@4": 0.49189631068843637, + "pass@8": 0.5528083984205381, + "pass@16": 0.6063339208685132, + "pass@32": 0.6533444169232615, + "pass@64": 0.6909033146987849, + "pass@128": 0.7151515151515152 + }, + "16": { + "pass@1": 0.22008547008547008, + "pass@2": 0.2828445022096597, + "pass@4": 0.34362964138029767, + "pass@8": 0.4025933786277824, + "pass@16": 0.45909006271494923, + "pass@32": 0.509012724160067, + "pass@64": 0.5514726875161375, + "pass@128": 0.5897435897435898 + }, + "17": { + "pass@1": 0.20479910714285715, + "pass@2": 0.26985353902772996, + "pass@4": 0.3339852589272611, + "pass@8": 0.39210840306068734, + "pass@16": 0.44226737100715613, + "pass@32": 0.4864598585876851, + "pass@64": 0.5269995343426999, + "pass@128": 0.562111801242236 + }, + "18": { + "pass@1": 0.18855698529411766, + "pass@2": 0.24998842056507656, + "pass@4": 0.30994159333024557, + "pass@8": 0.36606403192727205, + "pass@16": 0.4167227598892368, + "pass@32": 0.4612764576013937, + "pass@64": 0.4994967680463235, + "pass@128": 0.5294117647058824 + }, + "19": { + "pass@1": 0.1353701636904762, + "pass@2": 0.1830628105080615, + "pass@4": 0.23144565969432382, + "pass@8": 0.2755576222539975, + "pass@16": 0.3158109715954468, + "pass@32": 0.35526680353587153, + "pass@64": 0.39177456043247216, + "pass@128": 0.4226190476190476 + }, + "20": { + "pass@1": 0.15468282185628743, + "pass@2": 0.21383758074402373, + "pass@4": 0.2715573271342579, + "pass@8": 0.32506282832444283, + "pass@16": 0.37242304719707847, + "pass@32": 0.41287302118538943, + "pass@64": 0.4455683819577822, + "pass@128": 0.47005988023952094 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 171602, + "answer_accuracy": 0.4097312423594132, + "avg_response_len": 342.0212957480135, + "resp_tokens_sum": 143243991, + "pass_at_k": { + "pass@1": 0.4097312423594132, + "pass@2": 0.47351784648557116, + "pass@4": 0.5279097926023316, + "pass@8": 0.574115776599107, + "pass@16": 0.612984498786154, + "pass@32": 0.6463156446919366, + "pass@64": 0.6757751283945757, + "pass@128": 0.7023227383863081 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8953869047619047, + "pass@2": 0.9258264591926012, + "pass@4": 0.9429379149034938, + "pass@8": 0.9543336002203412, + "pass@16": 0.9626194265631142, + "pass@32": 0.9687413506580436, + "pass@64": 0.9737446542283149, + "pass@128": 0.9777777777777777 + }, + "12": { + "pass@1": 0.8249474388379205, + "pass@2": 0.8815236792361963, + "pass@4": 0.9135995612360381, + "pass@8": 0.9367215712258752, + "pass@16": 0.9539419568542121, + "pass@32": 0.9666327427155964, + "pass@64": 0.9768717219631041, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.5698543595679012, + "pass@2": 0.6570694505200732, + "pass@4": 0.7250310091446903, + "pass@8": 0.7782503130285764, + "pass@16": 0.8207342556469278, + "pass@32": 0.8537480654369178, + "pass@64": 0.878043036531414, + "pass@128": 0.8950617283950617 + }, + "14": { + "pass@1": 0.4921179525222552, + "pass@2": 0.5775018107899714, + "pass@4": 0.64585369929649, + "pass@8": 0.69798102049907, + "pass@16": 0.7376375558104614, + "pass@32": 0.770023465331237, + "pass@64": 0.7967844004152322, + "pass@128": 0.8219584569732937 + }, + "15": { + "pass@1": 0.3724922839506173, + "pass@2": 0.4515567281520361, + "pass@4": 0.5233148660236916, + "pass@8": 0.5866930510725332, + "pass@16": 0.6407629732953167, + "pass@32": 0.6878131369368291, + "pass@64": 0.7286931825352366, + "pass@128": 0.7623456790123457 + }, + "16": { + "pass@1": 0.24110435103244837, + "pass@2": 0.3068064362529904, + "pass@4": 0.3683298506381394, + "pass@8": 0.42239557551272405, + "pass@16": 0.4703387253798914, + "pass@32": 0.5155321182925883, + "pass@64": 0.5590083669683755, + "pass@128": 0.5988200589970502 + }, + "17": { + "pass@1": 0.21724397590361447, + "pass@2": 0.2787211839483919, + "pass@4": 0.33454787795727337, + "pass@8": 0.38220498919307505, + "pass@16": 0.4241590468878052, + "pass@32": 0.4634047222244833, + "pass@64": 0.5003442497973006, + "pass@128": 0.536144578313253 + }, + "18": { + "pass@1": 0.18961693548387096, + "pass@2": 0.2507727171704342, + "pass@4": 0.3110922971322133, + "pass@8": 0.3657547320819236, + "pass@16": 0.41192986664478526, + "pass@32": 0.45166304993993356, + "pass@64": 0.4878861429737296, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.17265388972809667, + "pass@2": 0.22803200157004544, + "pass@4": 0.2792073111329362, + "pass@8": 0.32448211542126465, + "pass@16": 0.36277516546281496, + "pass@32": 0.39563085079972393, + "pass@64": 0.4277352297153177, + "pass@128": 0.4622356495468278 + }, + "20": { + "pass@1": 0.14379222972972974, + "pass@2": 0.1978464685157598, + "pass@4": 0.25417908262593303, + "pass@8": 0.310062539442377, + "pass@16": 0.3614706077644692, + "pass@32": 0.40526578602168234, + "pass@64": 0.44272531631551015, + "pass@128": 0.47147147147147145 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6789103618421053, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6789103618421053, + "pass@2": 0.7139409966846074, + "pass@4": 0.7433195558581301, + "pass@8": 0.7684536987112555, + "pass@16": 0.790002883394277, + "pass@32": 0.8085379488902074, + "pass@64": 0.8246722254084696, + "pass@128": 0.8388421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9323125, + "pass@2": 0.9499406988188978, + "pass@4": 0.9621583696100489, + "pass@8": 0.9718907539598126, + "pass@16": 0.9791354444504873, + "pass@32": 0.9840437401738955, + "pass@64": 0.9871711591050699, + "pass@128": 0.99 + }, + "2": { + "pass@1": 0.999828125, + "pass@2": 0.9999741633858267, + "pass@4": 0.999999438976378, + "pass@8": 0.9999999998576696, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998734375, + "pass@2": 0.999644438976378, + "pass@4": 0.9999579316647917, + "pass@8": 0.9999992906842913, + "pass@16": 0.9999999998783552, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.995140625, + "pass@2": 0.9964927411417323, + "pass@4": 0.9973747537495314, + "pass@8": 0.9978259051966901, + "pass@16": 0.9979795297363963, + "pass@32": 0.9979997935880747, + "pass@64": 0.9979999999993951, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.97978125, + "pass@2": 0.9833897637795275, + "pass@4": 0.9860962986501687, + "pass@8": 0.9884541169280153, + "pass@16": 0.990718014081456, + "pass@32": 0.9926000208673001, + "pass@64": 0.9939421204598907, + "pass@128": 0.995 + }, + "6": { + "pass@1": 0.995984375, + "pass@2": 0.9974320866141734, + "pass@4": 0.9982878877952756, + "pass@8": 0.9988890262019753, + "pass@16": 0.9993300424147991, + "pass@32": 0.9996835374287463, + "pass@64": 0.9999404387018788, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9928359375, + "pass@2": 0.9942635334645668, + "pass@4": 0.9949447849643793, + "pass@8": 0.9955137921675251, + "pass@16": 0.9960397802495263, + "pass@32": 0.9964733721669579, + "pass@64": 0.9969998615760639, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.986671875, + "pass@2": 0.9904295029527562, + "pass@4": 0.9924173699850019, + "pass@8": 0.9938992253106746, + "pass@16": 0.9951051047857475, + "pass@32": 0.9960312897117236, + "pass@64": 0.9966908261256258, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.9811953125, + "pass@2": 0.9865735728346459, + "pass@4": 0.9889808374578175, + "pass@8": 0.9911338416262176, + "pass@16": 0.9934450763840873, + "pass@32": 0.9954784674062254, + "pass@64": 0.9966498831415526, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.894578125, + "pass@2": 0.9257929379921268, + "pass@4": 0.9442590304649411, + "pass@8": 0.9561462437618224, + "pass@16": 0.9641911885882701, + "pass@32": 0.9700611686252756, + "pass@64": 0.9752575710913312, + "pass@128": 0.98 + }, + "12": { + "pass@1": 0.812484375, + "pass@2": 0.8672572588582684, + "pass@4": 0.8985465786464193, + "pass@8": 0.9196361714553097, + "pass@16": 0.9348869636510788, + "pass@32": 0.94673284148625, + "pass@64": 0.9576580920053203, + "pass@128": 0.968 + }, + "13": { + "pass@1": 0.5978046875, + "pass@2": 0.6833599901574804, + "pass@4": 0.7455164355080617, + "pass@8": 0.7936291630299753, + "pass@16": 0.8329134533248633, + "pass@32": 0.8646922500467323, + "pass@64": 0.8900894458435458, + "pass@128": 0.911 + }, + "14": { + "pass@1": 0.45890625, + "pass@2": 0.5430552411417325, + "pass@4": 0.6134239566929134, + "pass@8": 0.6705658372251655, + "pass@16": 0.7163013175856843, + "pass@32": 0.7538964282786471, + "pass@64": 0.7852020700568273, + "pass@128": 0.813 + }, + "15": { + "pass@1": 0.3480078125, + "pass@2": 0.4288660187007878, + "pass@4": 0.49983243916385395, + "pass@8": 0.561707925390508, + "pass@16": 0.6160022823168169, + "pass@32": 0.6632416916320305, + "pass@64": 0.7017246400131141, + "pass@128": 0.73 + }, + "16": { + "pass@1": 0.237484375, + "pass@2": 0.3018716781496062, + "pass@4": 0.36146487692163465, + "pass@8": 0.41654215784915727, + "pass@16": 0.467808763276434, + "pass@32": 0.5155089371433209, + "pass@64": 0.5604432358720062, + "pass@128": 0.602 + }, + "17": { + "pass@1": 0.2055234375, + "pass@2": 0.26673203740157486, + "pass@4": 0.3232418611736032, + "pass@8": 0.37323857044448777, + "pass@16": 0.41837508369464593, + "pass@32": 0.4601245400414008, + "pass@64": 0.4978334061713972, + "pass@128": 0.53 + }, + "18": { + "pass@1": 0.1816796875, + "pass@2": 0.24128063484251946, + "pass@4": 0.30061707564679396, + "pass@8": 0.35639546958098467, + "pass@16": 0.40555332405555866, + "pass@32": 0.44808731643199917, + "pass@64": 0.4868599251210825, + "pass@128": 0.525 + }, + "19": { + "pass@1": 0.150984375, + "pass@2": 0.2036835629921259, + "pass@4": 0.25508346662917125, + "pass@8": 0.30095403253935643, + "pass@16": 0.3403886075532727, + "pass@32": 0.37524624378973637, + "pass@64": 0.40789729833590765, + "pass@128": 0.44 + }, + "20": { + "pass@1": 0.149359375, + "pass@2": 0.20483907480314964, + "pass@4": 0.2608681676040494, + "pass@8": 0.31419875230396843, + "pass@16": 0.3618808084635515, + "pass@32": 0.4023193900953739, + "pass@64": 0.436412309140885, + "pass@128": 0.464 + } + }, + "per_op_accuracy": { + "10": 0.9323125, + "2": 0.999828125, + "3": 0.998734375, + "4": 0.995140625, + "5": 0.97978125, + "6": 0.995984375, + "7": 0.9928359375, + "8": 0.986671875, + "9": 0.9811953125, + "11": 0.894578125, + "12": 0.812484375, + "13": 0.5978046875, + "14": 0.45890625, + "15": 0.3480078125, + "16": 0.237484375, + "17": 0.2055234375, + "18": 0.1816796875, + "19": 0.150984375, + "20": 0.149359375 + }, + "per_op_avg_response_len": { + "10": 279.759734375, + "2": 115.4348671875, + "3": 144.4752734375, + "4": 159.7785234375, + "5": 173.330203125, + "6": 198.6917734375, + "7": 223.7417265625, + "8": 242.8668671875, + "9": 258.9370390625, + "11": 296.2161171875, + "12": 304.9042109375, + "13": 316.20178125, + "14": 326.5206875, + "15": 330.5811015625, + "16": 321.36221875, + "17": 328.7864765625, + "18": 332.2248125, + "19": 322.4753359375, + "20": 327.6681640625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.759734375, + "2": 115.4348671875, + "3": 144.4752734375, + "4": 159.7785234375, + "5": 173.330203125, + "6": 198.6917734375, + "7": 223.7417265625, + "8": 242.8668671875, + "9": 258.9370390625, + "11": 296.2161171875, + "12": 304.9042109375, + "13": 316.20178125, + "14": 326.5206875, + "15": 330.5811015625, + "16": 321.36221875, + "17": 328.7864765625, + "18": 332.2248125, + "19": 322.4753359375, + "20": 327.6681640625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 548183, + "answer_accuracy": 0.6769964728896617, + "avg_response_len": 251.36895599509958, + "resp_tokens_sum": 203540482, + "pass_at_k": { + "pass@1": 0.6769964728896617, + "pass@2": 0.7122231071742398, + "pass@4": 0.7411229584436191, + "pass@8": 0.7655143535115593, + "pass@16": 0.7864565900913838, + "pass@32": 0.8044771795508289, + "pass@64": 0.8204609794123039, + "pass@128": 0.8351248814416693 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9362555309734514, + "pass@2": 0.9539477504471235, + "pass@4": 0.9651495407653686, + "pass@8": 0.973229583190713, + "pass@16": 0.9792803436377407, + "pass@32": 0.9841718673769317, + "pass@64": 0.9878246376873651, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983753822629969, + "pass@2": 0.9995236762262515, + "pass@4": 0.9999511242287376, + "pass@8": 0.9999994411862023, + "pass@16": 0.9999999999675167, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990673828125, + "pass@2": 0.9933247723917322, + "pass@4": 0.9952177153637045, + "pass@8": 0.9963542387601867, + "pass@16": 0.9968111376105945, + "pass@32": 0.9968743549630685, + "pass@64": 0.9968749999981098, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9707093253968254, + "pass@2": 0.9767396262967132, + "pass@4": 0.981683998131186, + "pass@8": 0.9859164493385404, + "pass@16": 0.9901710891964574, + "pass@32": 0.9936881458577858, + "pass@64": 0.995845992043236, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.997029378742515, + "pass@2": 0.9987512671507378, + "pass@4": 0.9995554992377451, + "pass@8": 0.999939295718634, + "pass@16": 0.9999991353050729, + "pass@32": 0.9999999999556102, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9958394970414202, + "pass@2": 0.9969697124120579, + "pass@4": 0.9970408646219519, + "pass@8": 0.9970414201059147, + "pass@16": 0.9970414201183432, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9894080528846154, + "pass@2": 0.9919966434484152, + "pass@4": 0.9931632229865497, + "pass@8": 0.9938714043202131, + "pass@16": 0.9943410041176739, + "pass@32": 0.9949967186169365, + "pass@64": 0.9959998990510801, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9859595070422535, + "pass@2": 0.9918774952866809, + "pass@4": 0.9937267724456977, + "pass@8": 0.9954155716338281, + "pass@16": 0.9975424861780133, + "pass@32": 0.9993143415342073, + "pass@64": 0.9999593384604206, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8889872521246459, + "pass@2": 0.9209322232383845, + "pass@4": 0.9403707372980644, + "pass@8": 0.9523073338351158, + "pass@16": 0.9599911546995261, + "pass@32": 0.9656664426486973, + "pass@64": 0.97105654419183, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.807468220338983, + "pass@2": 0.8675287907602651, + "pass@4": 0.9025985565363646, + "pass@8": 0.9234312139724606, + "pass@16": 0.9362103618765187, + "pass@32": 0.9450079869218969, + "pass@64": 0.9525377320034726, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.6421923981191222, + "pass@2": 0.7228887178189717, + "pass@4": 0.7779022200986624, + "pass@8": 0.8182676410254136, + "pass@16": 0.8505300132234627, + "pass@32": 0.8773909089095185, + "pass@64": 0.9017227414682624, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.4485007788161994, + "pass@2": 0.5347201780852163, + "pass@4": 0.6061306235669136, + "pass@8": 0.6653760404004355, + "pass@16": 0.7137935405594372, + "pass@32": 0.7534356762985341, + "pass@64": 0.7870531477671298, + "pass@128": 0.8130841121495327 + }, + "15": { + "pass@1": 0.3333408598265896, + "pass@2": 0.41465276671521556, + "pass@4": 0.4854122544653017, + "pass@8": 0.5467994374688726, + "pass@16": 0.6020372401229079, + "pass@32": 0.6496721898260739, + "pass@64": 0.686791777517624, + "pass@128": 0.7138728323699421 + }, + "16": { + "pass@1": 0.2532258064516129, + "pass@2": 0.3180189547879094, + "pass@4": 0.374151785260713, + "pass@8": 0.42593478016771535, + "pass@16": 0.4749139463216263, + "pass@32": 0.5228389769740335, + "pass@64": 0.5721693101663297, + "pass@128": 0.6193548387096774 + }, + "17": { + "pass@1": 0.19495122832369943, + "pass@2": 0.2523230195484954, + "pass@4": 0.3023951223040761, + "pass@8": 0.3470740122741218, + "pass@16": 0.39059013486586935, + "pass@32": 0.4324684907446193, + "pass@64": 0.46828111323798854, + "pass@128": 0.49421965317919075 + }, + "18": { + "pass@1": 0.16796875, + "pass@2": 0.22441437007874018, + "pass@4": 0.28228091943864153, + "pass@8": 0.33871351937233285, + "pass@16": 0.38905522009524307, + "pass@32": 0.4321079296175608, + "pass@64": 0.47367519903850336, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.1451998873873874, + "pass@2": 0.20028788631150843, + "pass@4": 0.2549552702308609, + "pass@8": 0.30319246623896356, + "pass@16": 0.34293555954657895, + "pass@32": 0.3751435020629878, + "pass@64": 0.4044465014859663, + "pass@128": 0.43543543543543545 + }, + "20": { + "pass@1": 0.1495870870870871, + "pass@2": 0.20280615260930232, + "pass@4": 0.25683599347378866, + "pass@8": 0.30743826429217297, + "pass@16": 0.3517171120545327, + "pass@32": 0.38878767049319296, + "pass@64": 0.4209157334351972, + "pass@128": 0.45045045045045046 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 554465, + "answer_accuracy": 0.6752545303975058, + "avg_response_len": 259.52538118667184, + "resp_tokens_sum": 213101481, + "pass_at_k": { + "pass@1": 0.6752545303975058, + "pass@2": 0.7099311637341081, + "pass@4": 0.7396101450831859, + "pass@8": 0.765588659194692, + "pass@16": 0.788283743412472, + "pass@32": 0.8079536752890428, + "pass@64": 0.8246852486176389, + "pass@128": 0.8385035074045206 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9209403757668712, + "pass@2": 0.9380713794019612, + "pass@4": 0.9512705088436028, + "pass@8": 0.9631955401424825, + "pass@16": 0.9726077613588666, + "pass@32": 0.9782359909847477, + "pass@64": 0.9813867257620852, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982007575757575, + "pass@2": 0.9994191422094966, + "pass@4": 0.9999210212359819, + "pass@8": 0.9999984042922265, + "pass@16": 0.9999999996635676, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9975268121301775, + "pass@2": 0.9989032724921958, + "pass@4": 0.9997191858968812, + "pass@8": 0.9999779550101494, + "pass@16": 0.999999898523686, + "pass@32": 0.9999999999996828, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9875679347826087, + "pass@2": 0.9887766746547987, + "pass@4": 0.9894790216440336, + "pass@8": 0.9903547461693829, + "pass@16": 0.9916253749261505, + "pass@32": 0.9931572503524777, + "pass@64": 0.9949313065753761, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9943364971346705, + "pass@2": 0.9965329089863052, + "pass@4": 0.9980393601408706, + "pass@8": 0.9990811493323517, + "pass@16": 0.9997493490657728, + "pass@32": 0.9999854792814306, + "pass@64": 0.999999993573057, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9934293713872833, + "pass@2": 0.9942072077966412, + "pass@4": 0.9943974088679668, + "pass@8": 0.9945809233687215, + "pass@16": 0.994942196531792, + "pass@32": 0.9956647398843931, + "pass@64": 0.9971098265895953, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9836707746478873, + "pass@2": 0.987972371631363, + "pass@4": 0.9905331931219864, + "pass@8": 0.9926688812257355, + "pass@16": 0.9945746668596219, + "pass@32": 0.9961092494376016, + "pass@64": 0.9970112564626833, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9756034805389222, + "pass@2": 0.9805492361733227, + "pass@4": 0.9840381910532142, + "pass@8": 0.9872988101167206, + "pass@16": 0.9901482935355289, + "pass@32": 0.992455558127094, + "pass@64": 0.9937913521118914, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.8997552710843374, + "pass@2": 0.9309293028412866, + "pass@4": 0.9496467379642303, + "pass@8": 0.9619478037609, + "pass@16": 0.9701481675058946, + "pass@32": 0.9759861111837409, + "pass@64": 0.981159773824385, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8052752742946708, + "pass@2": 0.8523317361093967, + "pass@4": 0.8786195395246442, + "pass@8": 0.8979108712169228, + "pass@16": 0.9138855036221462, + "pass@32": 0.9282479851031312, + "pass@64": 0.9436447705145927, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.5835084033613446, + "pass@2": 0.6718991238227576, + "pass@4": 0.7351696928115078, + "pass@8": 0.7855705437580075, + "pass@16": 0.8282254349495101, + "pass@32": 0.8632778064510783, + "pass@64": 0.8906273038627206, + "pass@128": 0.9103641456582633 + }, + "14": { + "pass@1": 0.4359466374269006, + "pass@2": 0.5169355372519222, + "pass@4": 0.5883138300146694, + "pass@8": 0.6484225858141512, + "pass@16": 0.6976308033859067, + "pass@32": 0.7384376268134236, + "pass@64": 0.7720516569697223, + "pass@128": 0.804093567251462 + }, + "15": { + "pass@1": 0.3393465909090909, + "pass@2": 0.4214902469577665, + "pass@4": 0.49189631068843637, + "pass@8": 0.5528083984205381, + "pass@16": 0.6063339208685132, + "pass@32": 0.6533444169232615, + "pass@64": 0.6909033146987849, + "pass@128": 0.7151515151515152 + }, + "16": { + "pass@1": 0.22008547008547008, + "pass@2": 0.2828445022096597, + "pass@4": 0.34362964138029767, + "pass@8": 0.4025933786277824, + "pass@16": 0.45909006271494923, + "pass@32": 0.509012724160067, + "pass@64": 0.5514726875161375, + "pass@128": 0.5897435897435898 + }, + "17": { + "pass@1": 0.20479910714285715, + "pass@2": 0.26985353902772996, + "pass@4": 0.3339852589272611, + "pass@8": 0.39210840306068734, + "pass@16": 0.44226737100715613, + "pass@32": 0.4864598585876851, + "pass@64": 0.5269995343426999, + "pass@128": 0.562111801242236 + }, + "18": { + "pass@1": 0.18855698529411766, + "pass@2": 0.24998842056507656, + "pass@4": 0.30994159333024557, + "pass@8": 0.36606403192727205, + "pass@16": 0.4167227598892368, + "pass@32": 0.4612764576013937, + "pass@64": 0.4994967680463235, + "pass@128": 0.5294117647058824 + }, + "19": { + "pass@1": 0.1353701636904762, + "pass@2": 0.1830628105080615, + "pass@4": 0.23144565969432382, + "pass@8": 0.2755576222539975, + "pass@16": 0.3158109715954468, + "pass@32": 0.35526680353587153, + "pass@64": 0.39177456043247216, + "pass@128": 0.4226190476190476 + }, + "20": { + "pass@1": 0.15468282185628743, + "pass@2": 0.21383758074402373, + "pass@4": 0.2715573271342579, + "pass@8": 0.32506282832444283, + "pass@16": 0.37242304719707847, + "pass@32": 0.41287302118538943, + "pass@64": 0.4455683819577822, + "pass@128": 0.47005988023952094 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 548462, + "answer_accuracy": 0.6845916879693241, + "avg_response_len": 279.4282757828727, + "resp_tokens_sum": 223864522, + "pass_at_k": { + "pass@1": 0.6845916879693241, + "pass@2": 0.7197870499551496, + "pass@4": 0.7493415314717663, + "pass@8": 0.7743609564571847, + "pass@16": 0.7953491263112427, + "pass@32": 0.8132410231819698, + "pass@64": 0.8289152032459771, + "pass@128": 0.8429461575331523 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9393889925373134, + "pass@2": 0.9574362439769657, + "pass@4": 0.9697268340524599, + "pass@8": 0.9789975497663034, + "pass@16": 0.9853411276247855, + "pass@32": 0.9895658032002611, + "pass@64": 0.9921389084495922, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.99953125, + "pass@2": 0.9999261811023622, + "pass@4": 0.9999983970753656, + "pass@8": 0.9999999995933421, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995900145772595, + "pass@2": 0.9999763263010492, + "pass@4": 0.9999999330440837, + "pass@8": 0.9999999999999082, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9969618055555556, + "pass@2": 0.9970745844269466, + "pass@4": 0.9970760233918129, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9802849264705882, + "pass@2": 0.984084790412228, + "pass@4": 0.9867516963872165, + "pass@8": 0.9888766116409934, + "pass@16": 0.9903040195148527, + "pass@32": 0.9910264810308619, + "pass@64": 0.9911745064640191, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9966975552050473, + "pass@2": 0.9970321077274646, + "pass@4": 0.9972259128097947, + "pass@8": 0.9975709158200651, + "pass@16": 0.9981634334351739, + "pass@32": 0.9990176819380228, + "pass@64": 0.9998121165453686, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.988973496835443, + "pass@2": 0.9914306226951061, + "pass@4": 0.993302117891909, + "pass@8": 0.9949012426776844, + "pass@16": 0.9961701906630579, + "pass@32": 0.9967511777435377, + "pass@64": 0.996835004987544, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9873076201201201, + "pass@2": 0.9915806554586083, + "pass@4": 0.9937272097744536, + "pass@8": 0.9952369199027994, + "pass@16": 0.996386500798402, + "pass@32": 0.9969175073660387, + "pass@64": 0.9969965813136217, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.981762459807074, + "pass@2": 0.9869891225915893, + "pass@4": 0.9888716316007123, + "pass@8": 0.9903649875151894, + "pass@16": 0.9923085651126249, + "pass@32": 0.9943463657464062, + "pass@64": 0.9959421427097476, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.8953869047619047, + "pass@2": 0.9258264591926012, + "pass@4": 0.9429379149034938, + "pass@8": 0.9543336002203412, + "pass@16": 0.9626194265631142, + "pass@32": 0.9687413506580436, + "pass@64": 0.9737446542283149, + "pass@128": 0.9777777777777777 + }, + "12": { + "pass@1": 0.8249474388379205, + "pass@2": 0.8815236792361963, + "pass@4": 0.9135995612360381, + "pass@8": 0.9367215712258752, + "pass@16": 0.9539419568542121, + "pass@32": 0.9666327427155964, + "pass@64": 0.9768717219631041, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.5698543595679012, + "pass@2": 0.6570694505200732, + "pass@4": 0.7250310091446903, + "pass@8": 0.7782503130285764, + "pass@16": 0.8207342556469278, + "pass@32": 0.8537480654369178, + "pass@64": 0.878043036531414, + "pass@128": 0.8950617283950617 + }, + "14": { + "pass@1": 0.4921179525222552, + "pass@2": 0.5775018107899714, + "pass@4": 0.64585369929649, + "pass@8": 0.69798102049907, + "pass@16": 0.7376375558104614, + "pass@32": 0.770023465331237, + "pass@64": 0.7967844004152322, + "pass@128": 0.8219584569732937 + }, + "15": { + "pass@1": 0.3724922839506173, + "pass@2": 0.4515567281520361, + "pass@4": 0.5233148660236916, + "pass@8": 0.5866930510725332, + "pass@16": 0.6407629732953167, + "pass@32": 0.6878131369368291, + "pass@64": 0.7286931825352366, + "pass@128": 0.7623456790123457 + }, + "16": { + "pass@1": 0.24110435103244837, + "pass@2": 0.3068064362529904, + "pass@4": 0.3683298506381394, + "pass@8": 0.42239557551272405, + "pass@16": 0.4703387253798914, + "pass@32": 0.5155321182925883, + "pass@64": 0.5590083669683755, + "pass@128": 0.5988200589970502 + }, + "17": { + "pass@1": 0.21724397590361447, + "pass@2": 0.2787211839483919, + "pass@4": 0.33454787795727337, + "pass@8": 0.38220498919307505, + "pass@16": 0.4241590468878052, + "pass@32": 0.4634047222244833, + "pass@64": 0.5003442497973006, + "pass@128": 0.536144578313253 + }, + "18": { + "pass@1": 0.18961693548387096, + "pass@2": 0.2507727171704342, + "pass@4": 0.3110922971322133, + "pass@8": 0.3657547320819236, + "pass@16": 0.41192986664478526, + "pass@32": 0.45166304993993356, + "pass@64": 0.4878861429737296, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.17265388972809667, + "pass@2": 0.22803200157004544, + "pass@4": 0.2792073111329362, + "pass@8": 0.32448211542126465, + "pass@16": 0.36277516546281496, + "pass@32": 0.39563085079972393, + "pass@64": 0.4277352297153177, + "pass@128": 0.4622356495468278 + }, + "20": { + "pass@1": 0.14379222972972974, + "pass@2": 0.1978464685157598, + "pass@4": 0.25417908262593303, + "pass@8": 0.310062539442377, + "pass@16": 0.3614706077644692, + "pass@32": 0.40526578602168234, + "pass@64": 0.44272531631551015, + "pass@128": 0.47147147147147145 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_score_report.jsonl b/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c0fc34149a9b74460899ea8bba1fd28a4d3acb35 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_score_report.jsonl @@ -0,0 +1,4 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch1-op7-10", "results_dir": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1011246, "value_mismatch": 20816, "dependency_mismatch": 137385, "answer_mismatch": 17601, "extra_nodes": 17257, "missing_nodes": 15561, "perfect_with_extra": 1278, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 332940, "dependency_mismatch": 44925, "value_mismatch": 6766, "answer_mismatch": 5766, "missing_nodes": 4873, "extra_nodes": 6225, "perfect_with_extra": 580}, "teachers_in_school": {"total": 390656, "perfect": 342682, "dependency_mismatch": 46781, "value_mismatch": 7253, "answer_mismatch": 6360, "extra_nodes": 5413, "missing_nodes": 4920, "perfect_with_extra": 119}, "movie_festival_awards": {"total": 382336, "perfect": 335624, "dependency_mismatch": 45679, "value_mismatch": 6797, "answer_mismatch": 5475, "extra_nodes": 5619, "missing_nodes": 5768, "perfect_with_extra": 579}}, "results_path": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 205404, "value_mismatch": 823819, "dependency_mismatch": 959246, "answer_mismatch": 763283, "extra_nodes": 87309, "missing_nodes": 818929, "perfect_with_extra": 170, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 278048, "dependency_mismatch": 324609, "answer_mismatch": 255775, "extra_nodes": 33550, "missing_nodes": 281498, "perfect": 69624, "perfect_with_extra": 20}, "movie_festival_awards": {"total": 418816, "perfect": 68953, "value_mismatch": 268268, "dependency_mismatch": 316151, "answer_mismatch": 247216, "missing_nodes": 260726, "extra_nodes": 27835, "perfect_with_extra": 15}, "teachers_in_school": {"total": 430464, "perfect": 66827, "value_mismatch": 277503, "dependency_mismatch": 318486, "answer_mismatch": 260292, "extra_nodes": 25924, "missing_nodes": 276705, "perfect_with_extra": 135}}, "results_path": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1216650, "value_mismatch": 844635, "dependency_mismatch": 1096631, "answer_mismatch": 780884, "extra_nodes": 104566, "missing_nodes": 834490, "perfect_with_extra": 1448, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 402564, "dependency_mismatch": 369534, "value_mismatch": 284814, "answer_mismatch": 261541, "missing_nodes": 286371, "extra_nodes": 39775, "perfect_with_extra": 600}, "teachers_in_school": {"total": 821120, "perfect": 409509, "dependency_mismatch": 365267, "value_mismatch": 284756, "answer_mismatch": 266652, "extra_nodes": 31337, "missing_nodes": 281625, "perfect_with_extra": 254}, "movie_festival_awards": {"total": 801152, "perfect": 404577, "dependency_mismatch": 361830, "value_mismatch": 275065, "answer_mismatch": 252691, "extra_nodes": 33454, "missing_nodes": 266494, "perfect_with_extra": 594}}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch1-op7-10", "results_dir": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_score_report.jsonl", "template_stats": {}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch1-op7-10", "results_dir": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_score_report.jsonl", "template_stats": {}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch1-op7-10", "results_dir": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch1-op7-10/graphs/SDAR-100M-block8-epoch1-op7-10_score_report.jsonl", "template_stats": {}}} diff --git a/composition/SDAR-100M-block8-epoch1-op7-10/graphs/results.json b/composition/SDAR-100M-block8-epoch1-op7-10/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..2e2b644fa5b7e7f7734f0c4b168834da9bdda5bd --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op7-10/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.99375, + "pass@4": 0.99375, + "pass@8": 0.99375, + "pass@16": 0.99375, + "pass@32": 0.99375, + "pass@64": 0.996875, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9301587301587302, + "pass@2": 0.9619047619047619, + "pass@4": 0.9714285714285714, + "pass@8": 0.9746031746031746, + "pass@16": 0.9809523809523809, + "pass@32": 0.9904761904761905, + "pass@64": 0.9936507936507937, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9520958083832335, + "pass@2": 0.9760479041916168, + "pass@4": 0.9910179640718563, + "pass@8": 0.9970059880239521, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8846153846153846, + "pass@2": 0.9230769230769231, + "pass@4": 0.9526627218934911, + "pass@8": 0.9792899408284024, + "pass@16": 0.9881656804733728, + "pass@32": 0.9940828402366864, + "pass@64": 0.9940828402366864, + "pass@128": 0.9940828402366864 + }, + "8": { + "pass@1": 0.8108974358974359, + "pass@2": 0.875, + "pass@4": 0.9262820512820513, + "pass@8": 0.9711538461538461, + "pass@16": 0.9935897435897436, + "pass@32": 0.9967948717948718, + "pass@64": 0.9967948717948718, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7295774647887324, + "pass@2": 0.8140845070422535, + "pass@4": 0.9042253521126761, + "pass@8": 0.9549295774647887, + "pass@16": 0.9887323943661972, + "pass@32": 0.9943661971830986, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6371681415929203, + "pass@2": 0.7227138643067846, + "pass@4": 0.7964601769911505, + "pass@8": 0.8702064896755162, + "pass@16": 0.9056047197640118, + "pass@32": 0.9262536873156342, + "pass@64": 0.9380530973451328, + "pass@128": 0.9498525073746312 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9301587301587302, + "6": 0.9520958083832335, + "7": 0.8846153846153846, + "8": 0.8108974358974359, + "9": 0.7295774647887324, + "10": 0.6371681415929203 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9619047619047619, + "6": 0.9760479041916168, + "7": 0.9230769230769231, + "8": 0.875, + "9": 0.8140845070422535, + "10": 0.7227138643067846 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9714285714285714, + "6": 0.9910179640718563, + "7": 0.9526627218934911, + "8": 0.9262820512820513, + "9": 0.9042253521126761, + "10": 0.7964601769911505 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9746031746031746, + "6": 0.9970059880239521, + "7": 0.9792899408284024, + "8": 0.9711538461538461, + "9": 0.9549295774647887, + "10": 0.8702064896755162 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9809523809523809, + "6": 1.0, + "7": 0.9881656804733728, + "8": 0.9935897435897436, + "9": 0.9887323943661972, + "10": 0.9056047197640118 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9904761904761905, + "6": 1.0, + "7": 0.9940828402366864, + "8": 0.9967948717948718, + "9": 0.9943661971830986, + "10": 0.9262536873156342 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 1.0, + "7": 0.9940828402366864, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9380530973451328 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 1.0, + "7": 0.9940828402366864, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9498525073746312 + } + }, + "total_pass_at_k": { + "pass@1": 0.8794326241134752, + "pass@2": 0.9165822357311719, + "pass@4": 0.9473150962512664, + "pass@8": 0.9706180344478217, + "pass@16": 0.9831138129010469, + "pass@32": 0.9881796690307328, + "pass@64": 0.9908814589665653, + "pass@128": 0.9922323539344816 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970845481049563, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.97953216374269, + "pass@2": 0.9883040935672515, + "pass@4": 0.9912280701754386, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.95, + "pass@2": 0.9705882352941176, + "pass@4": 0.9852941176470589, + "pass@8": 0.9882352941176471, + "pass@16": 0.9911764705882353, + "pass@32": 0.9911764705882353, + "pass@64": 0.9911764705882353, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9621451104100947, + "pass@2": 0.9684542586750788, + "pass@4": 0.9842271293375394, + "pass@8": 0.9936908517350158, + "pass@16": 0.9936908517350158, + "pass@32": 0.9968454258675079, + "pass@64": 0.9968454258675079, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8607594936708861, + "pass@2": 0.9113924050632911, + "pass@4": 0.9588607594936709, + "pass@8": 0.9810126582278481, + "pass@16": 0.9873417721518988, + "pass@32": 0.9936708860759493, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.7867867867867868, + "pass@2": 0.8588588588588588, + "pass@4": 0.933933933933934, + "pass@8": 0.9669669669669669, + "pass@16": 0.984984984984985, + "pass@32": 0.996996996996997, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.7331189710610932, + "pass@2": 0.8070739549839229, + "pass@4": 0.9003215434083601, + "pass@8": 0.9453376205787781, + "pass@16": 0.9742765273311897, + "pass@32": 0.9871382636655949, + "pass@64": 0.9935691318327974, + "pass@128": 0.9967845659163987 + }, + "10": { + "pass@1": 0.5761194029850746, + "pass@2": 0.6537313432835821, + "pass@4": 0.7582089552238805, + "pass@8": 0.826865671641791, + "pass@16": 0.8835820895522388, + "pass@32": 0.9253731343283582, + "pass@64": 0.9373134328358209, + "pass@128": 0.9402985074626866 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.97953216374269, + "5": 0.95, + "6": 0.9621451104100947, + "7": 0.8607594936708861, + "8": 0.7867867867867868, + "9": 0.7331189710610932, + "10": 0.5761194029850746 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.9705882352941176, + "6": 0.9684542586750788, + "7": 0.9113924050632911, + "8": 0.8588588588588588, + "9": 0.8070739549839229, + "10": 0.6537313432835821 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9852941176470589, + "6": 0.9842271293375394, + "7": 0.9588607594936709, + "8": 0.933933933933934, + "9": 0.9003215434083601, + "10": 0.7582089552238805 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9882352941176471, + "6": 0.9936908517350158, + "7": 0.9810126582278481, + "8": 0.9669669669669669, + "9": 0.9453376205787781, + "10": 0.826865671641791 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9911764705882353, + "6": 0.9936908517350158, + "7": 0.9873417721518988, + "8": 0.984984984984985, + "9": 0.9742765273311897, + "10": 0.8835820895522388 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9911764705882353, + "6": 0.9968454258675079, + "7": 0.9936708860759493, + "8": 0.996996996996997, + "9": 0.9871382636655949, + "10": 0.9253731343283582 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9911764705882353, + "6": 0.9968454258675079, + "7": 0.9936708860759493, + "8": 0.996996996996997, + "9": 0.9935691318327974, + "10": 0.9373134328358209 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9911764705882353, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.996996996996997, + "9": 0.9967845659163987, + "10": 0.9402985074626866 + } + }, + "total_pass_at_k": { + "pass@1": 0.8737864077669902, + "pass@2": 0.9079343823234014, + "pass@4": 0.9464345497154335, + "pass@8": 0.9668563776364245, + "pass@16": 0.9792433880147305, + "pass@32": 0.987612989621694, + "pass@64": 0.9896216940073652, + "pass@128": 0.9906260462002009 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996969696969697, + "pass@2": 0.996969696969697, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9911242603550295, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.936231884057971, + "pass@2": 0.9623188405797102, + "pass@4": 0.9826086956521739, + "pass@8": 0.9884057971014493, + "pass@16": 0.9884057971014493, + "pass@32": 0.991304347826087, + "pass@64": 0.991304347826087, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.9684813753581661, + "pass@2": 0.9770773638968482, + "pass@4": 0.9856733524355301, + "pass@8": 0.994269340974212, + "pass@16": 0.997134670487106, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8670520231213873, + "pass@2": 0.9017341040462428, + "pass@4": 0.9421965317919075, + "pass@8": 0.9797687861271677, + "pass@16": 0.9942196531791907, + "pass@32": 0.9942196531791907, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7830985915492957, + "pass@2": 0.8619718309859155, + "pass@4": 0.923943661971831, + "pass@8": 0.9577464788732394, + "pass@16": 0.9830985915492958, + "pass@32": 0.9943661971830986, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.7335329341317365, + "pass@2": 0.7964071856287425, + "pass@4": 0.8383233532934131, + "pass@8": 0.8982035928143712, + "pass@16": 0.9700598802395209, + "pass@32": 0.9880239520958084, + "pass@64": 0.9910179640718563, + "pass@128": 0.9910179640718563 + }, + "10": { + "pass@1": 0.6196319018404908, + "pass@2": 0.6840490797546013, + "pass@4": 0.7423312883435583, + "pass@8": 0.8190184049079755, + "pass@16": 0.8773006134969326, + "pass@32": 0.9049079754601227, + "pass@64": 0.9263803680981595, + "pass@128": 0.9355828220858896 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9911242603550295, + "5": 0.936231884057971, + "6": 0.9684813753581661, + "7": 0.8670520231213873, + "8": 0.7830985915492957, + "9": 0.7335329341317365, + "10": 0.6196319018404908 + }, + "pass@2": { + "2": 1.0, + "3": 0.996969696969697, + "4": 1.0, + "5": 0.9623188405797102, + "6": 0.9770773638968482, + "7": 0.9017341040462428, + "8": 0.8619718309859155, + "9": 0.7964071856287425, + "10": 0.6840490797546013 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9826086956521739, + "6": 0.9856733524355301, + "7": 0.9421965317919075, + "8": 0.923943661971831, + "9": 0.8383233532934131, + "10": 0.7423312883435583 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9884057971014493, + "6": 0.994269340974212, + "7": 0.9797687861271677, + "8": 0.9577464788732394, + "9": 0.8982035928143712, + "10": 0.8190184049079755 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9884057971014493, + "6": 0.997134670487106, + "7": 0.9942196531791907, + "8": 0.9830985915492958, + "9": 0.9700598802395209, + "10": 0.8773006134969326 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.991304347826087, + "6": 1.0, + "7": 0.9942196531791907, + "8": 0.9943661971830986, + "9": 0.9880239520958084, + "10": 0.9049079754601227 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.991304347826087, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9910179640718563, + "10": 0.9263803680981595 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9942028985507246, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9910179640718563, + "10": 0.9355828220858896 + } + }, + "total_pass_at_k": { + "pass@1": 0.8777850589777195, + "pass@2": 0.90956749672346, + "pass@4": 0.9357798165137615, + "pass@8": 0.9603538663171691, + "pass@16": 0.9793577981651376, + "pass@32": 0.9862385321100917, + "pass@64": 0.9898427260812582, + "pass@128": 0.9911533420707732 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998, + "pass@2": 0.999, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.988, + "pass@2": 0.994, + "pass@4": 0.995, + "pass@8": 0.997, + "pass@16": 0.997, + "pass@32": 0.997, + "pass@64": 0.998, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.939, + "pass@2": 0.965, + "pass@4": 0.98, + "pass@8": 0.984, + "pass@16": 0.987, + "pass@32": 0.991, + "pass@64": 0.992, + "pass@128": 0.993 + }, + "6": { + "pass@1": 0.961, + "pass@2": 0.974, + "pass@4": 0.987, + "pass@8": 0.995, + "pass@16": 0.997, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.871, + "pass@2": 0.912, + "pass@4": 0.951, + "pass@8": 0.98, + "pass@16": 0.99, + "pass@32": 0.994, + "pass@64": 0.996, + "pass@128": 0.996 + }, + "8": { + "pass@1": 0.793, + "pass@2": 0.865, + "pass@4": 0.928, + "pass@8": 0.965, + "pass@16": 0.987, + "pass@32": 0.996, + "pass@64": 0.997, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.732, + "pass@2": 0.806, + "pass@4": 0.881, + "pass@8": 0.933, + "pass@16": 0.978, + "pass@32": 0.99, + "pass@64": 0.995, + "pass@128": 0.996 + }, + "10": { + "pass@1": 0.611, + "pass@2": 0.687, + "pass@4": 0.766, + "pass@8": 0.839, + "pass@16": 0.889, + "pass@32": 0.919, + "pass@64": 0.934, + "pass@128": 0.942 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.998, + "4": 0.988, + "5": 0.939, + "6": 0.961, + "7": 0.871, + "8": 0.793, + "9": 0.732, + "10": 0.611 + }, + "pass@2": { + "2": 1.0, + "3": 0.999, + "4": 0.994, + "5": 0.965, + "6": 0.974, + "7": 0.912, + "8": 0.865, + "9": 0.806, + "10": 0.687 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.995, + "5": 0.98, + "6": 0.987, + "7": 0.951, + "8": 0.928, + "9": 0.881, + "10": 0.766 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.997, + "5": 0.984, + "6": 0.995, + "7": 0.98, + "8": 0.965, + "9": 0.933, + "10": 0.839 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.997, + "5": 0.987, + "6": 0.997, + "7": 0.99, + "8": 0.987, + "9": 0.978, + "10": 0.889 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.997, + "5": 0.991, + "6": 0.999, + "7": 0.994, + "8": 0.996, + "9": 0.99, + "10": 0.919 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.998, + "5": 0.992, + "6": 0.999, + "7": 0.996, + "8": 0.997, + "9": 0.995, + "10": 0.934 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.998, + "5": 0.993, + "6": 1.0, + "7": 0.996, + "8": 0.997, + "9": 0.996, + "10": 0.942 + } + }, + "total_pass_at_k": { + "pass@1": 0.877, + "pass@2": 0.9113333333333333, + "pass@4": 0.9431111111111111, + "pass@8": 0.9658888888888889, + "pass@16": 0.9805555555555555, + "pass@32": 0.9873333333333333, + "pass@64": 0.9901111111111112, + "pass@128": 0.9913333333333333 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5297450424929179, + "pass@2": 0.5920679886685553, + "pass@4": 0.7053824362606232, + "pass@8": 0.7818696883852692, + "pass@16": 0.8470254957507082, + "pass@32": 0.8753541076487252, + "pass@64": 0.9008498583569405, + "pass@128": 0.9178470254957507 + }, + "12": { + "pass@1": 0.5141242937853108, + "pass@2": 0.5903954802259888, + "pass@4": 0.6610169491525424, + "pass@8": 0.7288135593220338, + "pass@16": 0.7768361581920904, + "pass@32": 0.807909604519774, + "pass@64": 0.8220338983050848, + "pass@128": 0.8361581920903954 + }, + "13": { + "pass@1": 0.30407523510971785, + "pass@2": 0.3667711598746082, + "pass@4": 0.4482758620689655, + "pass@8": 0.5109717868338558, + "pass@16": 0.567398119122257, + "pass@32": 0.6175548589341693, + "pass@64": 0.677115987460815, + "pass@128": 0.7053291536050157 + }, + "14": { + "pass@1": 0.13395638629283488, + "pass@2": 0.18691588785046728, + "pass@4": 0.22741433021806853, + "pass@8": 0.2772585669781931, + "pass@16": 0.3146417445482866, + "pass@32": 0.367601246105919, + "pass@64": 0.4174454828660436, + "pass@128": 0.470404984423676 + }, + "15": { + "pass@1": 0.08670520231213873, + "pass@2": 0.09826589595375723, + "pass@4": 0.13005780346820808, + "pass@8": 0.13872832369942195, + "pass@16": 0.1907514450867052, + "pass@32": 0.23121387283236994, + "pass@64": 0.2630057803468208, + "pass@128": 0.3063583815028902 + }, + "16": { + "pass@1": 0.01935483870967742, + "pass@2": 0.025806451612903226, + "pass@4": 0.035483870967741936, + "pass@8": 0.03870967741935484, + "pass@16": 0.041935483870967745, + "pass@32": 0.04516129032258064, + "pass@64": 0.054838709677419356, + "pass@128": 0.07419354838709677 + }, + "17": { + "pass@1": 0.008670520231213872, + "pass@2": 0.014450867052023121, + "pass@4": 0.017341040462427744, + "pass@8": 0.023121387283236993, + "pass@16": 0.023121387283236993, + "pass@32": 0.031791907514450865, + "pass@64": 0.05202312138728324, + "pass@128": 0.06069364161849711 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.002857142857142857, + "pass@8": 0.005714285714285714, + "pass@16": 0.011428571428571429, + "pass@32": 0.017142857142857144, + "pass@64": 0.017142857142857144, + "pass@128": 0.017142857142857144 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.003003003003003003, + "pass@128": 0.003003003003003003 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.006006006006006006, + "pass@32": 0.006006006006006006, + "pass@64": 0.006006006006006006, + "pass@128": 0.006006006006006006 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5297450424929179, + "12": 0.5141242937853108, + "13": 0.30407523510971785, + "14": 0.13395638629283488, + "15": 0.08670520231213873, + "16": 0.01935483870967742, + "17": 0.008670520231213872, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5920679886685553, + "12": 0.5903954802259888, + "13": 0.3667711598746082, + "14": 0.18691588785046728, + "15": 0.09826589595375723, + "16": 0.025806451612903226, + "17": 0.014450867052023121, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.7053824362606232, + "12": 0.6610169491525424, + "13": 0.4482758620689655, + "14": 0.22741433021806853, + "15": 0.13005780346820808, + "16": 0.035483870967741936, + "17": 0.017341040462427744, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.7818696883852692, + "12": 0.7288135593220338, + "13": 0.5109717868338558, + "14": 0.2772585669781931, + "15": 0.13872832369942195, + "16": 0.03870967741935484, + "17": 0.023121387283236993, + "18": 0.005714285714285714, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.8470254957507082, + "12": 0.7768361581920904, + "13": 0.567398119122257, + "14": 0.3146417445482866, + "15": 0.1907514450867052, + "16": 0.041935483870967745, + "17": 0.023121387283236993, + "18": 0.011428571428571429, + "19": 0.0, + "20": 0.006006006006006006 + }, + "pass@32": { + "11": 0.8753541076487252, + "12": 0.807909604519774, + "13": 0.6175548589341693, + "14": 0.367601246105919, + "15": 0.23121387283236994, + "16": 0.04516129032258064, + "17": 0.031791907514450865, + "18": 0.017142857142857144, + "19": 0.0, + "20": 0.006006006006006006 + }, + "pass@64": { + "11": 0.9008498583569405, + "12": 0.8220338983050848, + "13": 0.677115987460815, + "14": 0.4174454828660436, + "15": 0.2630057803468208, + "16": 0.054838709677419356, + "17": 0.05202312138728324, + "18": 0.017142857142857144, + "19": 0.003003003003003003, + "20": 0.006006006006006006 + }, + "pass@128": { + "11": 0.9178470254957507, + "12": 0.8361581920903954, + "13": 0.7053291536050157, + "14": 0.470404984423676, + "15": 0.3063583815028902, + "16": 0.07419354838709677, + "17": 0.06069364161849711, + "18": 0.017142857142857144, + "19": 0.003003003003003003, + "20": 0.006006006006006006 + } + }, + "total_pass_at_k": { + "pass@1": 0.16285289747399703, + "pass@2": 0.19078751857355125, + "pass@4": 0.22644873699851412, + "pass@8": 0.25438335809806834, + "pass@16": 0.28202080237741456, + "pass@32": 0.3040118870728083, + "pass@64": 0.325111441307578, + "pass@128": 0.34323922734026746 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5301587301587302, + "pass@2": 0.6222222222222222, + "pass@4": 0.707936507936508, + "pass@8": 0.8, + "pass@16": 0.8507936507936508, + "pass@32": 0.9174603174603174, + "pass@64": 0.9301587301587302, + "pass@128": 0.946031746031746 + }, + "12": { + "pass@1": 0.4740061162079511, + "pass@2": 0.5596330275229358, + "pass@4": 0.6422018348623854, + "pass@8": 0.72782874617737, + "pass@16": 0.7981651376146789, + "pass@32": 0.8318042813455657, + "pass@64": 0.8776758409785933, + "pass@128": 0.8899082568807339 + }, + "13": { + "pass@1": 0.28703703703703703, + "pass@2": 0.3395061728395062, + "pass@4": 0.4228395061728395, + "pass@8": 0.5, + "pass@16": 0.5432098765432098, + "pass@32": 0.6018518518518519, + "pass@64": 0.6358024691358025, + "pass@128": 0.654320987654321 + }, + "14": { + "pass@1": 0.16913946587537093, + "pass@2": 0.2166172106824926, + "pass@4": 0.26112759643916916, + "pass@8": 0.3056379821958457, + "pass@16": 0.3649851632047478, + "pass@32": 0.4094955489614243, + "pass@64": 0.4807121661721068, + "pass@128": 0.5548961424332344 + }, + "15": { + "pass@1": 0.12345679012345678, + "pass@2": 0.15432098765432098, + "pass@4": 0.1728395061728395, + "pass@8": 0.18209876543209877, + "pass@16": 0.21604938271604937, + "pass@32": 0.25, + "pass@64": 0.3117283950617284, + "pass@128": 0.3333333333333333 + }, + "16": { + "pass@1": 0.032448377581120944, + "pass@2": 0.035398230088495575, + "pass@4": 0.04129793510324484, + "pass@8": 0.061946902654867256, + "pass@16": 0.07374631268436578, + "pass@32": 0.0855457227138643, + "pass@64": 0.10324483775811209, + "pass@128": 0.12389380530973451 + }, + "17": { + "pass@1": 0.012048192771084338, + "pass@2": 0.012048192771084338, + "pass@4": 0.02108433734939759, + "pass@8": 0.030120481927710843, + "pass@16": 0.03313253012048193, + "pass@32": 0.05120481927710843, + "pass@64": 0.06325301204819277, + "pass@128": 0.07228915662650602 + }, + "18": { + "pass@1": 0.016129032258064516, + "pass@2": 0.02903225806451613, + "pass@4": 0.035483870967741936, + "pass@8": 0.035483870967741936, + "pass@16": 0.04838709677419355, + "pass@32": 0.05161290322580645, + "pass@64": 0.05806451612903226, + "pass@128": 0.06129032258064516 + }, + "19": { + "pass@1": 0.0030211480362537764, + "pass@2": 0.0030211480362537764, + "pass@4": 0.0030211480362537764, + "pass@8": 0.0030211480362537764, + "pass@16": 0.0030211480362537764, + "pass@32": 0.0030211480362537764, + "pass@64": 0.006042296072507553, + "pass@128": 0.006042296072507553 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5301587301587302, + "12": 0.4740061162079511, + "13": 0.28703703703703703, + "14": 0.16913946587537093, + "15": 0.12345679012345678, + "16": 0.032448377581120944, + "17": 0.012048192771084338, + "18": 0.016129032258064516, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@2": { + "11": 0.6222222222222222, + "12": 0.5596330275229358, + "13": 0.3395061728395062, + "14": 0.2166172106824926, + "15": 0.15432098765432098, + "16": 0.035398230088495575, + "17": 0.012048192771084338, + "18": 0.02903225806451613, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@4": { + "11": 0.707936507936508, + "12": 0.6422018348623854, + "13": 0.4228395061728395, + "14": 0.26112759643916916, + "15": 0.1728395061728395, + "16": 0.04129793510324484, + "17": 0.02108433734939759, + "18": 0.035483870967741936, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@8": { + "11": 0.8, + "12": 0.72782874617737, + "13": 0.5, + "14": 0.3056379821958457, + "15": 0.18209876543209877, + "16": 0.061946902654867256, + "17": 0.030120481927710843, + "18": 0.035483870967741936, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@16": { + "11": 0.8507936507936508, + "12": 0.7981651376146789, + "13": 0.5432098765432098, + "14": 0.3649851632047478, + "15": 0.21604938271604937, + "16": 0.07374631268436578, + "17": 0.03313253012048193, + "18": 0.04838709677419355, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@32": { + "11": 0.9174603174603174, + "12": 0.8318042813455657, + "13": 0.6018518518518519, + "14": 0.4094955489614243, + "15": 0.25, + "16": 0.0855457227138643, + "17": 0.05120481927710843, + "18": 0.05161290322580645, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@64": { + "11": 0.9301587301587302, + "12": 0.8776758409785933, + "13": 0.6358024691358025, + "14": 0.4807121661721068, + "15": 0.3117283950617284, + "16": 0.10324483775811209, + "17": 0.06325301204819277, + "18": 0.05806451612903226, + "19": 0.006042296072507553, + "20": 0.0 + }, + "pass@128": { + "11": 0.946031746031746, + "12": 0.8899082568807339, + "13": 0.654320987654321, + "14": 0.5548961424332344, + "15": 0.3333333333333333, + "16": 0.12389380530973451, + "17": 0.07228915662650602, + "18": 0.06129032258064516, + "19": 0.006042296072507553, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.16289731051344744, + "pass@2": 0.1949877750611247, + "pass@4": 0.2283007334963325, + "pass@8": 0.261919315403423, + "pass@16": 0.29034229828850855, + "pass@32": 0.31723716381418093, + "pass@64": 0.34382640586797064, + "pass@128": 0.36155256723716384 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5963855421686747, + "pass@2": 0.6536144578313253, + "pass@4": 0.7168674698795181, + "pass@8": 0.7951807228915663, + "pass@16": 0.8734939759036144, + "pass@32": 0.9126506024096386, + "pass@64": 0.9457831325301205, + "pass@128": 0.9487951807228916 + }, + "12": { + "pass@1": 0.45141065830721006, + "pass@2": 0.5235109717868338, + "pass@4": 0.5799373040752351, + "pass@8": 0.6426332288401254, + "pass@16": 0.7304075235109718, + "pass@32": 0.7931034482758621, + "pass@64": 0.8244514106583072, + "pass@128": 0.8589341692789969 + }, + "13": { + "pass@1": 0.26330532212885155, + "pass@2": 0.34173669467787116, + "pass@4": 0.4117647058823529, + "pass@8": 0.5014005602240896, + "pass@16": 0.5658263305322129, + "pass@32": 0.6190476190476191, + "pass@64": 0.6666666666666666, + "pass@128": 0.7030812324929971 + }, + "14": { + "pass@1": 0.12573099415204678, + "pass@2": 0.15789473684210525, + "pass@4": 0.195906432748538, + "pass@8": 0.24269005847953215, + "pass@16": 0.29239766081871343, + "pass@32": 0.34502923976608185, + "pass@64": 0.391812865497076, + "pass@128": 0.43859649122807015 + }, + "15": { + "pass@1": 0.09393939393939393, + "pass@2": 0.12727272727272726, + "pass@4": 0.15151515151515152, + "pass@8": 0.18787878787878787, + "pass@16": 0.20303030303030303, + "pass@32": 0.2515151515151515, + "pass@64": 0.2787878787878788, + "pass@128": 0.296969696969697 + }, + "16": { + "pass@1": 0.022792022792022793, + "pass@2": 0.02849002849002849, + "pass@4": 0.039886039886039885, + "pass@8": 0.04843304843304843, + "pass@16": 0.06552706552706553, + "pass@32": 0.07407407407407407, + "pass@64": 0.08262108262108261, + "pass@128": 0.09401709401709402 + }, + "17": { + "pass@1": 0.009316770186335404, + "pass@2": 0.012422360248447204, + "pass@4": 0.018633540372670808, + "pass@8": 0.027950310559006212, + "pass@16": 0.040372670807453416, + "pass@32": 0.046583850931677016, + "pass@64": 0.059006211180124224, + "pass@128": 0.06521739130434782 + }, + "18": { + "pass@1": 0.008823529411764706, + "pass@2": 0.008823529411764706, + "pass@4": 0.008823529411764706, + "pass@8": 0.014705882352941176, + "pass@16": 0.01764705882352941, + "pass@32": 0.020588235294117647, + "pass@64": 0.023529411764705882, + "pass@128": 0.026470588235294117 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.002976190476190476, + "pass@8": 0.002976190476190476, + "pass@16": 0.002976190476190476, + "pass@32": 0.005952380952380952, + "pass@64": 0.005952380952380952, + "pass@128": 0.005952380952380952 + }, + "20": { + "pass@1": 0.0029940119760479044, + "pass@2": 0.0029940119760479044, + "pass@4": 0.0029940119760479044, + "pass@8": 0.0029940119760479044, + "pass@16": 0.0029940119760479044, + "pass@32": 0.0029940119760479044, + "pass@64": 0.005988023952095809, + "pass@128": 0.005988023952095809 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5963855421686747, + "12": 0.45141065830721006, + "13": 0.26330532212885155, + "14": 0.12573099415204678, + "15": 0.09393939393939393, + "16": 0.022792022792022793, + "17": 0.009316770186335404, + "18": 0.008823529411764706, + "19": 0.0, + "20": 0.0029940119760479044 + }, + "pass@2": { + "11": 0.6536144578313253, + "12": 0.5235109717868338, + "13": 0.34173669467787116, + "14": 0.15789473684210525, + "15": 0.12727272727272726, + "16": 0.02849002849002849, + "17": 0.012422360248447204, + "18": 0.008823529411764706, + "19": 0.0, + "20": 0.0029940119760479044 + }, + "pass@4": { + "11": 0.7168674698795181, + "12": 0.5799373040752351, + "13": 0.4117647058823529, + "14": 0.195906432748538, + "15": 0.15151515151515152, + "16": 0.039886039886039885, + "17": 0.018633540372670808, + "18": 0.008823529411764706, + "19": 0.002976190476190476, + "20": 0.0029940119760479044 + }, + "pass@8": { + "11": 0.7951807228915663, + "12": 0.6426332288401254, + "13": 0.5014005602240896, + "14": 0.24269005847953215, + "15": 0.18787878787878787, + "16": 0.04843304843304843, + "17": 0.027950310559006212, + "18": 0.014705882352941176, + "19": 0.002976190476190476, + "20": 0.0029940119760479044 + }, + "pass@16": { + "11": 0.8734939759036144, + "12": 0.7304075235109718, + "13": 0.5658263305322129, + "14": 0.29239766081871343, + "15": 0.20303030303030303, + "16": 0.06552706552706553, + "17": 0.040372670807453416, + "18": 0.01764705882352941, + "19": 0.002976190476190476, + "20": 0.0029940119760479044 + }, + "pass@32": { + "11": 0.9126506024096386, + "12": 0.7931034482758621, + "13": 0.6190476190476191, + "14": 0.34502923976608185, + "15": 0.2515151515151515, + "16": 0.07407407407407407, + "17": 0.046583850931677016, + "18": 0.020588235294117647, + "19": 0.005952380952380952, + "20": 0.0029940119760479044 + }, + "pass@64": { + "11": 0.9457831325301205, + "12": 0.8244514106583072, + "13": 0.6666666666666666, + "14": 0.391812865497076, + "15": 0.2787878787878788, + "16": 0.08262108262108261, + "17": 0.059006211180124224, + "18": 0.023529411764705882, + "19": 0.005952380952380952, + "20": 0.005988023952095809 + }, + "pass@128": { + "11": 0.9487951807228916, + "12": 0.8589341692789969, + "13": 0.7030812324929971, + "14": 0.43859649122807015, + "15": 0.296969696969697, + "16": 0.09401709401709402, + "17": 0.06521739130434782, + "18": 0.026470588235294117, + "19": 0.005952380952380952, + "20": 0.005988023952095809 + } + }, + "total_pass_at_k": { + "pass@1": 0.15611061552185548, + "pass@2": 0.18435920309247694, + "pass@4": 0.21171573000297353, + "pass@8": 0.24561403508771928, + "pass@16": 0.2783229259589652, + "pass@32": 0.30597680642283676, + "pass@64": 0.32738626226583406, + "pass@128": 0.3434433541480821 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.552, + "pass@2": 0.622, + "pass@4": 0.71, + "pass@8": 0.792, + "pass@16": 0.857, + "pass@32": 0.901, + "pass@64": 0.925, + "pass@128": 0.937 + }, + "12": { + "pass@1": 0.481, + "pass@2": 0.559, + "pass@4": 0.629, + "pass@8": 0.701, + "pass@16": 0.769, + "pass@32": 0.811, + "pass@64": 0.841, + "pass@128": 0.861 + }, + "13": { + "pass@1": 0.284, + "pass@2": 0.349, + "pass@4": 0.427, + "pass@8": 0.504, + "pass@16": 0.559, + "pass@32": 0.613, + "pass@64": 0.66, + "pass@128": 0.688 + }, + "14": { + "pass@1": 0.143, + "pass@2": 0.187, + "pass@4": 0.228, + "pass@8": 0.275, + "pass@16": 0.324, + "pass@32": 0.374, + "pass@64": 0.43, + "pass@128": 0.488 + }, + "15": { + "pass@1": 0.101, + "pass@2": 0.126, + "pass@4": 0.151, + "pass@8": 0.169, + "pass@16": 0.203, + "pass@32": 0.244, + "pass@64": 0.284, + "pass@128": 0.312 + }, + "16": { + "pass@1": 0.025, + "pass@2": 0.03, + "pass@4": 0.039, + "pass@8": 0.05, + "pass@16": 0.061, + "pass@32": 0.069, + "pass@64": 0.081, + "pass@128": 0.098 + }, + "17": { + "pass@1": 0.01, + "pass@2": 0.013, + "pass@4": 0.019, + "pass@8": 0.027, + "pass@16": 0.032, + "pass@32": 0.043, + "pass@64": 0.058, + "pass@128": 0.066 + }, + "18": { + "pass@1": 0.008, + "pass@2": 0.012, + "pass@4": 0.015, + "pass@8": 0.018, + "pass@16": 0.025, + "pass@32": 0.029, + "pass@64": 0.032, + "pass@128": 0.034 + }, + "19": { + "pass@1": 0.001, + "pass@2": 0.001, + "pass@4": 0.002, + "pass@8": 0.002, + "pass@16": 0.002, + "pass@32": 0.003, + "pass@64": 0.005, + "pass@128": 0.005 + }, + "20": { + "pass@1": 0.001, + "pass@2": 0.001, + "pass@4": 0.001, + "pass@8": 0.001, + "pass@16": 0.003, + "pass@32": 0.003, + "pass@64": 0.004, + "pass@128": 0.004 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.552, + "12": 0.481, + "13": 0.284, + "14": 0.143, + "15": 0.101, + "16": 0.025, + "17": 0.01, + "18": 0.008, + "19": 0.001, + "20": 0.001 + }, + "pass@2": { + "11": 0.622, + "12": 0.559, + "13": 0.349, + "14": 0.187, + "15": 0.126, + "16": 0.03, + "17": 0.013, + "18": 0.012, + "19": 0.001, + "20": 0.001 + }, + "pass@4": { + "11": 0.71, + "12": 0.629, + "13": 0.427, + "14": 0.228, + "15": 0.151, + "16": 0.039, + "17": 0.019, + "18": 0.015, + "19": 0.002, + "20": 0.001 + }, + "pass@8": { + "11": 0.792, + "12": 0.701, + "13": 0.504, + "14": 0.275, + "15": 0.169, + "16": 0.05, + "17": 0.027, + "18": 0.018, + "19": 0.002, + "20": 0.001 + }, + "pass@16": { + "11": 0.857, + "12": 0.769, + "13": 0.559, + "14": 0.324, + "15": 0.203, + "16": 0.061, + "17": 0.032, + "18": 0.025, + "19": 0.002, + "20": 0.003 + }, + "pass@32": { + "11": 0.901, + "12": 0.811, + "13": 0.613, + "14": 0.374, + "15": 0.244, + "16": 0.069, + "17": 0.043, + "18": 0.029, + "19": 0.003, + "20": 0.003 + }, + "pass@64": { + "11": 0.925, + "12": 0.841, + "13": 0.66, + "14": 0.43, + "15": 0.284, + "16": 0.081, + "17": 0.058, + "18": 0.032, + "19": 0.005, + "20": 0.004 + }, + "pass@128": { + "11": 0.937, + "12": 0.861, + "13": 0.688, + "14": 0.488, + "15": 0.312, + "16": 0.098, + "17": 0.066, + "18": 0.034, + "19": 0.005, + "20": 0.004 + } + }, + "total_pass_at_k": { + "pass@1": 0.1606, + "pass@2": 0.19, + "pass@4": 0.2221, + "pass@8": 0.2539, + "pass@16": 0.2835, + "pass@32": 0.309, + "pass@64": 0.332, + "pass@128": 0.3493 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block8-epoch1-op7-10/summary.csv b/composition/SDAR-100M-block8-epoch1-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..c3271bc35d366d11411117e2e9c0d372ad4f6278 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch1-op7-10,0.9847204861111111,0.40368125,0.6789103618421053,0.0,0.0,0.0,199.6684453125,320.694090625,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block8-epoch1-op7-10/summary.json b/composition/SDAR-100M-block8-epoch1-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..bfe5fa505463dfdcf0183dae87482fbd08ebbbfb --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch1-op7-10", + "id_acc": 0.9847204861111111, + "ood_acc": 0.40368125, + "total_acc": 0.6789103618421053, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.6684453125, + "ood_avg_resp_len": 320.694090625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 548183, + "answer_accuracy": 0.6769964728896617, + "avg_response_len": 251.36895599509958, + "resp_tokens_sum": 203540482, + "pass_at_k": { + "pass@1": 0.6769964728896617, + "pass@2": 0.7122231071742398, + "pass@4": 0.7411229584436191, + "pass@8": 0.7655143535115593, + "pass@16": 0.7864565900913838, + "pass@32": 0.8044771795508289, + "pass@64": 0.8204609794123039, + "pass@128": 0.8351248814416693 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9362555309734514, + "pass@2": 0.9539477504471235, + "pass@4": 0.9651495407653686, + "pass@8": 0.973229583190713, + "pass@16": 0.9792803436377407, + "pass@32": 0.9841718673769317, + "pass@64": 0.9878246376873651, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983753822629969, + "pass@2": 0.9995236762262515, + "pass@4": 0.9999511242287376, + "pass@8": 0.9999994411862023, + "pass@16": 0.9999999999675167, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990673828125, + "pass@2": 0.9933247723917322, + "pass@4": 0.9952177153637045, + "pass@8": 0.9963542387601867, + "pass@16": 0.9968111376105945, + "pass@32": 0.9968743549630685, + "pass@64": 0.9968749999981098, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9707093253968254, + "pass@2": 0.9767396262967132, + "pass@4": 0.981683998131186, + "pass@8": 0.9859164493385404, + "pass@16": 0.9901710891964574, + "pass@32": 0.9936881458577858, + "pass@64": 0.995845992043236, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.997029378742515, + "pass@2": 0.9987512671507378, + "pass@4": 0.9995554992377451, + "pass@8": 0.999939295718634, + "pass@16": 0.9999991353050729, + "pass@32": 0.9999999999556102, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9958394970414202, + "pass@2": 0.9969697124120579, + "pass@4": 0.9970408646219519, + "pass@8": 0.9970414201059147, + "pass@16": 0.9970414201183432, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9894080528846154, + "pass@2": 0.9919966434484152, + "pass@4": 0.9931632229865497, + "pass@8": 0.9938714043202131, + "pass@16": 0.9943410041176739, + "pass@32": 0.9949967186169365, + "pass@64": 0.9959998990510801, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9859595070422535, + "pass@2": 0.9918774952866809, + "pass@4": 0.9937267724456977, + "pass@8": 0.9954155716338281, + "pass@16": 0.9975424861780133, + "pass@32": 0.9993143415342073, + "pass@64": 0.9999593384604206, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8889872521246459, + "pass@2": 0.9209322232383845, + "pass@4": 0.9403707372980644, + "pass@8": 0.9523073338351158, + "pass@16": 0.9599911546995261, + "pass@32": 0.9656664426486973, + "pass@64": 0.97105654419183, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.807468220338983, + "pass@2": 0.8675287907602651, + "pass@4": 0.9025985565363646, + "pass@8": 0.9234312139724606, + "pass@16": 0.9362103618765187, + "pass@32": 0.9450079869218969, + "pass@64": 0.9525377320034726, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.6421923981191222, + "pass@2": 0.7228887178189717, + "pass@4": 0.7779022200986624, + "pass@8": 0.8182676410254136, + "pass@16": 0.8505300132234627, + "pass@32": 0.8773909089095185, + "pass@64": 0.9017227414682624, + "pass@128": 0.9278996865203761 + }, + "14": { + "pass@1": 0.4485007788161994, + "pass@2": 0.5347201780852163, + "pass@4": 0.6061306235669136, + "pass@8": 0.6653760404004355, + "pass@16": 0.7137935405594372, + "pass@32": 0.7534356762985341, + "pass@64": 0.7870531477671298, + "pass@128": 0.8130841121495327 + }, + "15": { + "pass@1": 0.3333408598265896, + "pass@2": 0.41465276671521556, + "pass@4": 0.4854122544653017, + "pass@8": 0.5467994374688726, + "pass@16": 0.6020372401229079, + "pass@32": 0.6496721898260739, + "pass@64": 0.686791777517624, + "pass@128": 0.7138728323699421 + }, + "16": { + "pass@1": 0.2532258064516129, + "pass@2": 0.3180189547879094, + "pass@4": 0.374151785260713, + "pass@8": 0.42593478016771535, + "pass@16": 0.4749139463216263, + "pass@32": 0.5228389769740335, + "pass@64": 0.5721693101663297, + "pass@128": 0.6193548387096774 + }, + "17": { + "pass@1": 0.19495122832369943, + "pass@2": 0.2523230195484954, + "pass@4": 0.3023951223040761, + "pass@8": 0.3470740122741218, + "pass@16": 0.39059013486586935, + "pass@32": 0.4324684907446193, + "pass@64": 0.46828111323798854, + "pass@128": 0.49421965317919075 + }, + "18": { + "pass@1": 0.16796875, + "pass@2": 0.22441437007874018, + "pass@4": 0.28228091943864153, + "pass@8": 0.33871351937233285, + "pass@16": 0.38905522009524307, + "pass@32": 0.4321079296175608, + "pass@64": 0.47367519903850336, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.1451998873873874, + "pass@2": 0.20028788631150843, + "pass@4": 0.2549552702308609, + "pass@8": 0.30319246623896356, + "pass@16": 0.34293555954657895, + "pass@32": 0.3751435020629878, + "pass@64": 0.4044465014859663, + "pass@128": 0.43543543543543545 + }, + "20": { + "pass@1": 0.1495870870870871, + "pass@2": 0.20280615260930232, + "pass@4": 0.25683599347378866, + "pass@8": 0.30743826429217297, + "pass@16": 0.3517171120545327, + "pass@32": 0.38878767049319296, + "pass@64": 0.4209157334351972, + "pass@128": 0.45045045045045046 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 554465, + "answer_accuracy": 0.6752545303975058, + "avg_response_len": 259.52538118667184, + "resp_tokens_sum": 213101481, + "pass_at_k": { + "pass@1": 0.6752545303975058, + "pass@2": 0.7099311637341081, + "pass@4": 0.7396101450831859, + "pass@8": 0.765588659194692, + "pass@16": 0.788283743412472, + "pass@32": 0.8079536752890428, + "pass@64": 0.8246852486176389, + "pass@128": 0.8385035074045206 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9209403757668712, + "pass@2": 0.9380713794019612, + "pass@4": 0.9512705088436028, + "pass@8": 0.9631955401424825, + "pass@16": 0.9726077613588666, + "pass@32": 0.9782359909847477, + "pass@64": 0.9813867257620852, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9982007575757575, + "pass@2": 0.9994191422094966, + "pass@4": 0.9999210212359819, + "pass@8": 0.9999984042922265, + "pass@16": 0.9999999996635676, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9975268121301775, + "pass@2": 0.9989032724921958, + "pass@4": 0.9997191858968812, + "pass@8": 0.9999779550101494, + "pass@16": 0.999999898523686, + "pass@32": 0.9999999999996828, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9875679347826087, + "pass@2": 0.9887766746547987, + "pass@4": 0.9894790216440336, + "pass@8": 0.9903547461693829, + "pass@16": 0.9916253749261505, + "pass@32": 0.9931572503524777, + "pass@64": 0.9949313065753761, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9943364971346705, + "pass@2": 0.9965329089863052, + "pass@4": 0.9980393601408706, + "pass@8": 0.9990811493323517, + "pass@16": 0.9997493490657728, + "pass@32": 0.9999854792814306, + "pass@64": 0.999999993573057, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9934293713872833, + "pass@2": 0.9942072077966412, + "pass@4": 0.9943974088679668, + "pass@8": 0.9945809233687215, + "pass@16": 0.994942196531792, + "pass@32": 0.9956647398843931, + "pass@64": 0.9971098265895953, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9836707746478873, + "pass@2": 0.987972371631363, + "pass@4": 0.9905331931219864, + "pass@8": 0.9926688812257355, + "pass@16": 0.9945746668596219, + "pass@32": 0.9961092494376016, + "pass@64": 0.9970112564626833, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9756034805389222, + "pass@2": 0.9805492361733227, + "pass@4": 0.9840381910532142, + "pass@8": 0.9872988101167206, + "pass@16": 0.9901482935355289, + "pass@32": 0.992455558127094, + "pass@64": 0.9937913521118914, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.8997552710843374, + "pass@2": 0.9309293028412866, + "pass@4": 0.9496467379642303, + "pass@8": 0.9619478037609, + "pass@16": 0.9701481675058946, + "pass@32": 0.9759861111837409, + "pass@64": 0.981159773824385, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8052752742946708, + "pass@2": 0.8523317361093967, + "pass@4": 0.8786195395246442, + "pass@8": 0.8979108712169228, + "pass@16": 0.9138855036221462, + "pass@32": 0.9282479851031312, + "pass@64": 0.9436447705145927, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.5835084033613446, + "pass@2": 0.6718991238227576, + "pass@4": 0.7351696928115078, + "pass@8": 0.7855705437580075, + "pass@16": 0.8282254349495101, + "pass@32": 0.8632778064510783, + "pass@64": 0.8906273038627206, + "pass@128": 0.9103641456582633 + }, + "14": { + "pass@1": 0.4359466374269006, + "pass@2": 0.5169355372519222, + "pass@4": 0.5883138300146694, + "pass@8": 0.6484225858141512, + "pass@16": 0.6976308033859067, + "pass@32": 0.7384376268134236, + "pass@64": 0.7720516569697223, + "pass@128": 0.804093567251462 + }, + "15": { + "pass@1": 0.3393465909090909, + "pass@2": 0.4214902469577665, + "pass@4": 0.49189631068843637, + "pass@8": 0.5528083984205381, + "pass@16": 0.6063339208685132, + "pass@32": 0.6533444169232615, + "pass@64": 0.6909033146987849, + "pass@128": 0.7151515151515152 + }, + "16": { + "pass@1": 0.22008547008547008, + "pass@2": 0.2828445022096597, + "pass@4": 0.34362964138029767, + "pass@8": 0.4025933786277824, + "pass@16": 0.45909006271494923, + "pass@32": 0.509012724160067, + "pass@64": 0.5514726875161375, + "pass@128": 0.5897435897435898 + }, + "17": { + "pass@1": 0.20479910714285715, + "pass@2": 0.26985353902772996, + "pass@4": 0.3339852589272611, + "pass@8": 0.39210840306068734, + "pass@16": 0.44226737100715613, + "pass@32": 0.4864598585876851, + "pass@64": 0.5269995343426999, + "pass@128": 0.562111801242236 + }, + "18": { + "pass@1": 0.18855698529411766, + "pass@2": 0.24998842056507656, + "pass@4": 0.30994159333024557, + "pass@8": 0.36606403192727205, + "pass@16": 0.4167227598892368, + "pass@32": 0.4612764576013937, + "pass@64": 0.4994967680463235, + "pass@128": 0.5294117647058824 + }, + "19": { + "pass@1": 0.1353701636904762, + "pass@2": 0.1830628105080615, + "pass@4": 0.23144565969432382, + "pass@8": 0.2755576222539975, + "pass@16": 0.3158109715954468, + "pass@32": 0.35526680353587153, + "pass@64": 0.39177456043247216, + "pass@128": 0.4226190476190476 + }, + "20": { + "pass@1": 0.15468282185628743, + "pass@2": 0.21383758074402373, + "pass@4": 0.2715573271342579, + "pass@8": 0.32506282832444283, + "pass@16": 0.37242304719707847, + "pass@32": 0.41287302118538943, + "pass@64": 0.4455683819577822, + "pass@128": 0.47005988023952094 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 548462, + "answer_accuracy": 0.6845916879693241, + "avg_response_len": 279.4282757828727, + "resp_tokens_sum": 223864522, + "pass_at_k": { + "pass@1": 0.6845916879693241, + "pass@2": 0.7197870499551496, + "pass@4": 0.7493415314717663, + "pass@8": 0.7743609564571847, + "pass@16": 0.7953491263112427, + "pass@32": 0.8132410231819698, + "pass@64": 0.8289152032459771, + "pass@128": 0.8429461575331523 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9393889925373134, + "pass@2": 0.9574362439769657, + "pass@4": 0.9697268340524599, + "pass@8": 0.9789975497663034, + "pass@16": 0.9853411276247855, + "pass@32": 0.9895658032002611, + "pass@64": 0.9921389084495922, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.99953125, + "pass@2": 0.9999261811023622, + "pass@4": 0.9999983970753656, + "pass@8": 0.9999999995933421, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995900145772595, + "pass@2": 0.9999763263010492, + "pass@4": 0.9999999330440837, + "pass@8": 0.9999999999999082, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9969618055555556, + "pass@2": 0.9970745844269466, + "pass@4": 0.9970760233918129, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9802849264705882, + "pass@2": 0.984084790412228, + "pass@4": 0.9867516963872165, + "pass@8": 0.9888766116409934, + "pass@16": 0.9903040195148527, + "pass@32": 0.9910264810308619, + "pass@64": 0.9911745064640191, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9966975552050473, + "pass@2": 0.9970321077274646, + "pass@4": 0.9972259128097947, + "pass@8": 0.9975709158200651, + "pass@16": 0.9981634334351739, + "pass@32": 0.9990176819380228, + "pass@64": 0.9998121165453686, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.988973496835443, + "pass@2": 0.9914306226951061, + "pass@4": 0.993302117891909, + "pass@8": 0.9949012426776844, + "pass@16": 0.9961701906630579, + "pass@32": 0.9967511777435377, + "pass@64": 0.996835004987544, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9873076201201201, + "pass@2": 0.9915806554586083, + "pass@4": 0.9937272097744536, + "pass@8": 0.9952369199027994, + "pass@16": 0.996386500798402, + "pass@32": 0.9969175073660387, + "pass@64": 0.9969965813136217, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.981762459807074, + "pass@2": 0.9869891225915893, + "pass@4": 0.9888716316007123, + "pass@8": 0.9903649875151894, + "pass@16": 0.9923085651126249, + "pass@32": 0.9943463657464062, + "pass@64": 0.9959421427097476, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.8953869047619047, + "pass@2": 0.9258264591926012, + "pass@4": 0.9429379149034938, + "pass@8": 0.9543336002203412, + "pass@16": 0.9626194265631142, + "pass@32": 0.9687413506580436, + "pass@64": 0.9737446542283149, + "pass@128": 0.9777777777777777 + }, + "12": { + "pass@1": 0.8249474388379205, + "pass@2": 0.8815236792361963, + "pass@4": 0.9135995612360381, + "pass@8": 0.9367215712258752, + "pass@16": 0.9539419568542121, + "pass@32": 0.9666327427155964, + "pass@64": 0.9768717219631041, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.5698543595679012, + "pass@2": 0.6570694505200732, + "pass@4": 0.7250310091446903, + "pass@8": 0.7782503130285764, + "pass@16": 0.8207342556469278, + "pass@32": 0.8537480654369178, + "pass@64": 0.878043036531414, + "pass@128": 0.8950617283950617 + }, + "14": { + "pass@1": 0.4921179525222552, + "pass@2": 0.5775018107899714, + "pass@4": 0.64585369929649, + "pass@8": 0.69798102049907, + "pass@16": 0.7376375558104614, + "pass@32": 0.770023465331237, + "pass@64": 0.7967844004152322, + "pass@128": 0.8219584569732937 + }, + "15": { + "pass@1": 0.3724922839506173, + "pass@2": 0.4515567281520361, + "pass@4": 0.5233148660236916, + "pass@8": 0.5866930510725332, + "pass@16": 0.6407629732953167, + "pass@32": 0.6878131369368291, + "pass@64": 0.7286931825352366, + "pass@128": 0.7623456790123457 + }, + "16": { + "pass@1": 0.24110435103244837, + "pass@2": 0.3068064362529904, + "pass@4": 0.3683298506381394, + "pass@8": 0.42239557551272405, + "pass@16": 0.4703387253798914, + "pass@32": 0.5155321182925883, + "pass@64": 0.5590083669683755, + "pass@128": 0.5988200589970502 + }, + "17": { + "pass@1": 0.21724397590361447, + "pass@2": 0.2787211839483919, + "pass@4": 0.33454787795727337, + "pass@8": 0.38220498919307505, + "pass@16": 0.4241590468878052, + "pass@32": 0.4634047222244833, + "pass@64": 0.5003442497973006, + "pass@128": 0.536144578313253 + }, + "18": { + "pass@1": 0.18961693548387096, + "pass@2": 0.2507727171704342, + "pass@4": 0.3110922971322133, + "pass@8": 0.3657547320819236, + "pass@16": 0.41192986664478526, + "pass@32": 0.45166304993993356, + "pass@64": 0.4878861429737296, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.17265388972809667, + "pass@2": 0.22803200157004544, + "pass@4": 0.2792073111329362, + "pass@8": 0.32448211542126465, + "pass@16": 0.36277516546281496, + "pass@32": 0.39563085079972393, + "pass@64": 0.4277352297153177, + "pass@128": 0.4622356495468278 + }, + "20": { + "pass@1": 0.14379222972972974, + "pass@2": 0.1978464685157598, + "pass@4": 0.25417908262593303, + "pass@8": 0.310062539442377, + "pass@16": 0.3614706077644692, + "pass@32": 0.40526578602168234, + "pass@64": 0.44272531631551015, + "pass@128": 0.47147147147147145 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_metrics.json b/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a8f4296d339151d1b2a55dc3a4343c2c49fa8df0 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch1-op9-12", + "id": { + "answer_accuracy": 0.9833871527777778, + "avg_response_len": 198.9335685763889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9833871527777778, + "pass@2": 0.9877975311679758, + "pass@4": 0.9904131950172896, + "pass@8": 0.9922471730307514, + "pass@16": 0.9938091197969837, + "pass@32": 0.9952691718510881, + "pass@64": 0.9965101656296633, + "pass@128": 0.9973333333333333 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.936140625, + "pass@2": 0.9532770669291335, + "pass@4": 0.9646779643794529, + "pass@8": 0.9722428738775302, + "pass@16": 0.9780534399094882, + "pass@32": 0.9829244527998198, + "pass@64": 0.9868526799974783, + "pass@128": 0.99 + }, + "2": { + "pass@1": 0.999421875, + "pass@2": 0.9997268700787402, + "pass@4": 0.9999281513873266, + "pass@8": 0.9999954379593903, + "pass@16": 0.999999987960665, + "pass@32": 0.9999999999999909, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.997, + "pass@2": 0.9982974901574803, + "pass@4": 0.9992196539182601, + "pass@8": 0.9997844261478821, + "pass@16": 0.999981904078783, + "pass@32": 0.9999999207944156, + "pass@64": 0.9999999999999699, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9924765625, + "pass@2": 0.9934320866141733, + "pass@4": 0.9943405821147356, + "pass@8": 0.9951957912852285, + "pass@16": 0.9958488669003491, + "pass@32": 0.9965198504237649, + "pass@64": 0.9973183937001826, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.9787734375, + "pass@2": 0.9831724901574804, + "pass@4": 0.9860739181664795, + "pass@8": 0.9882095975142469, + "pass@16": 0.9906008995622421, + "pass@32": 0.993477161814902, + "pass@64": 0.9959503664688322, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.9934375, + "pass@2": 0.9951104822834648, + "pass@4": 0.9958734730971127, + "pass@8": 0.9965130401102446, + "pass@16": 0.9972419328539005, + "pass@32": 0.9980359313833913, + "pass@64": 0.9987493335625536, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.99075, + "pass@2": 0.9936044537401577, + "pass@4": 0.994668941319835, + "pass@8": 0.9952967222076122, + "pass@16": 0.9959389380230231, + "pass@32": 0.9967077687453606, + "pass@64": 0.997570362137923, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.983, + "pass@2": 0.9881865157480314, + "pass@4": 0.9911749555680541, + "pass@8": 0.9933137034852952, + "pass@16": 0.9948246809075276, + "pass@32": 0.9958187164772767, + "pass@64": 0.9964386629519557, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.979484375, + "pass@2": 0.9853703248031499, + "pass@4": 0.9877611152043493, + "pass@8": 0.9896729646893262, + "pass@16": 0.9917914279768701, + "pass@32": 0.993938744220871, + "pass@64": 0.9957116918480703, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.936140625, + "2": 0.999421875, + "3": 0.997, + "4": 0.9924765625, + "5": 0.9787734375, + "6": 0.9934375, + "7": 0.99075, + "8": 0.983, + "9": 0.979484375 + }, + "per_op_avg_response_len": { + "10": 278.78703125, + "2": 115.1384140625, + "3": 144.0647578125, + "4": 159.348484375, + "5": 172.7458984375, + "6": 197.9644140625, + "7": 222.690140625, + "8": 242.006671875, + "9": 257.6563046875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.78703125, + "2": 115.1384140625, + "3": 144.0647578125, + "4": 159.348484375, + "5": 172.7458984375, + "6": 197.9644140625, + "7": 222.690140625, + "8": 242.006671875, + "9": 257.6563046875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 372515, + "answer_accuracy": 0.9828684354947653, + "avg_response_len": 189.65915495187437, + "resp_tokens_sum": 71882337, + "pass_at_k": { + "pass@1": 0.9828684354947653, + "pass@2": 0.9877498690323291, + "pass@4": 0.990514759869809, + "pass@8": 0.9923264350093163, + "pass@16": 0.9938964450532994, + "pass@32": 0.9954374275388409, + "pass@64": 0.9967093195612354, + "pass@128": 0.9976359338061466 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9386292404129793, + "pass@2": 0.9558454840545374, + "pass@4": 0.9661186383781674, + "pass@8": 0.9715414375472532, + "pass@16": 0.9756175864300959, + "pass@32": 0.9796003458610456, + "pass@64": 0.9837013134257441, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977780963302753, + "pass@2": 0.9989322220616917, + "pass@4": 0.9996694776685024, + "pass@8": 0.9999677623799305, + "pass@16": 0.999999779641568, + "pass@32": 0.9999999999981484, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9837158203125, + "pass@2": 0.9857337290846455, + "pass@4": 0.9882842340996436, + "pass@8": 0.9906827001964222, + "pass@16": 0.9922391181843271, + "pass@32": 0.9935574745502678, + "pass@64": 0.9951263779508654, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9695188492063492, + "pass@2": 0.976019013248344, + "pass@4": 0.9805692556287607, + "pass@8": 0.9843784633909355, + "pass@16": 0.9889107918312874, + "pass@32": 0.9940484172603424, + "pass@64": 0.9981441113782779, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9948774326347305, + "pass@2": 0.9966545747088502, + "pass@4": 0.9974962398038569, + "pass@8": 0.9981117754532004, + "pass@16": 0.9988525144880754, + "pass@32": 0.9996224712529058, + "pass@64": 0.9999803235855924, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9937361316568047, + "pass@2": 0.9963439989749802, + "pass@4": 0.9969306777666104, + "pass@8": 0.9970380005155915, + "pass@16": 0.9970414182830878, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9876302083333334, + "pass@2": 0.9916015230668282, + "pass@4": 0.9930404943372461, + "pass@8": 0.9940772665714049, + "pass@16": 0.9949131147821092, + "pass@32": 0.9957967829765609, + "pass@64": 0.9966039773874414, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9811399647887324, + "pass@2": 0.9891198569368971, + "pass@4": 0.9927404183631975, + "pass@8": 0.9952636734053902, + "pass@16": 0.9975775655877893, + "pass@32": 0.999379044270334, + "pass@64": 0.9999770279396809, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 384130, + "answer_accuracy": 0.9832947657273918, + "avg_response_len": 196.8890993610747, + "resp_tokens_sum": 76915908, + "pass_at_k": { + "pass@1": 0.9832947657273918, + "pass@2": 0.9873143320889366, + "pass@4": 0.9897567537941279, + "pass@8": 0.9917119681000646, + "pass@16": 0.9935082848685695, + "pass@32": 0.9950803838021421, + "pass@64": 0.9962572852675725, + "pass@128": 0.9967234600262124 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9312691717791411, + "pass@2": 0.9492200708902958, + "pass@4": 0.9614929713540408, + "pass@8": 0.9706849500303524, + "pass@16": 0.9780743583738936, + "pass@32": 0.9833452492500756, + "pass@64": 0.9865077334207035, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150811, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961647727272728, + "pass@2": 0.9978696910045335, + "pass@4": 0.9990399083637271, + "pass@8": 0.9997415688663096, + "pass@16": 0.9999820837105999, + "pass@32": 0.9999999593969077, + "pass@64": 0.9999999999999987, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9963480029585798, + "pass@2": 0.9970392361272888, + "pass@4": 0.9973065910215366, + "pass@8": 0.9975660568709271, + "pass@16": 0.9980247014241554, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9867753623188406, + "pass@2": 0.9882117996120052, + "pass@4": 0.9893897974709683, + "pass@8": 0.9906458350692433, + "pass@16": 0.9923677751658371, + "pass@32": 0.9946202004072046, + "pass@64": 0.996556005154711, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9901504297994269, + "pass@2": 0.9921238690973083, + "pass@4": 0.9931829475828413, + "pass@8": 0.9941730137747032, + "pass@16": 0.9951084353395996, + "pass@32": 0.9959328579290595, + "pass@64": 0.9967849648397139, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9909230491329479, + "pass@2": 0.9937133039005962, + "pass@4": 0.9946745414951744, + "pass@8": 0.9952348105100473, + "pass@16": 0.9961070653038383, + "pass@32": 0.9974785593997282, + "pass@64": 0.9991110099676848, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9840008802816902, + "pass@2": 0.9890027170899413, + "pass@4": 0.9920841439714402, + "pass@8": 0.9944509310383022, + "pass@16": 0.9961687806906195, + "pass@32": 0.9970422143124665, + "pass@64": 0.9971821603034191, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9723755613772455, + "pass@2": 0.977356311000047, + "pass@4": 0.9796722011544962, + "pass@8": 0.9821828386883638, + "pass@16": 0.9851973646020666, + "pass@32": 0.988117457332326, + "pass@64": 0.9901673949748785, + "pass@128": 0.9910179640718563 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 376217, + "answer_accuracy": 0.9839957524271845, + "avg_response_len": 210.2162129645129, + "resp_tokens_sum": 80373226, + "pass_at_k": { + "pass@1": 0.9839957524271845, + "pass@2": 0.9883384923909123, + "pass@4": 0.9909832402415216, + "pass@8": 0.9927154524850256, + "pass@16": 0.9940299360398915, + "pass@32": 0.9952952769846484, + "pass@64": 0.9965711284933091, + "pass@128": 0.9976565115500502 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9383628731343283, + "pass@2": 0.9546259842519684, + "pass@4": 0.9663195143517505, + "pass@8": 0.9744687547436317, + "pass@16": 0.9804980217307362, + "pass@32": 0.9858787590967183, + "pass@64": 0.9903773541820934, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9984375, + "pass@2": 0.9992217379077615, + "pass@4": 0.9997947185173283, + "pass@8": 0.999986965598258, + "pass@16": 0.9999999656019002, + "pass@32": 0.999999999999974, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970617711370262, + "pass@2": 0.9981039519294782, + "pass@4": 0.9989637462409036, + "pass@8": 0.9996508749963928, + "pass@16": 0.9999646895378788, + "pass@32": 0.9999998081458937, + "pass@64": 0.9999999999999133, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9968475877192983, + "pass@2": 0.997070267532348, + "pass@4": 0.9970760217472814, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9792279411764706, + "pass@2": 0.9846865591709126, + "pass@4": 0.9878091838703983, + "pass@8": 0.9892869660212149, + "pass@16": 0.9903738755975673, + "pass@32": 0.9917880624629651, + "pass@64": 0.9933033811949977, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9955392350157729, + "pass@2": 0.9967716845922651, + "pass@4": 0.9971258053508296, + "pass@8": 0.9974048116766691, + "pass@16": 0.997893845682534, + "pass@32": 0.9986796800242242, + "pass@64": 0.9996149929208377, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9873664952531646, + "pass@2": 0.9905549966361007, + "pass@4": 0.9922436103714881, + "pass@8": 0.9935020050533728, + "pass@16": 0.9945756140766189, + "pass@32": 0.995506921497008, + "pass@64": 0.996449217370583, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9775947822822822, + "pass@2": 0.9841167446974534, + "pass@4": 0.9884578084833988, + "pass@8": 0.9913859333225822, + "pass@16": 0.9933089247757945, + "pass@32": 0.9945349372602522, + "pass@64": 0.995491156454535, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9852290996784566, + "pass@2": 0.9896970023039725, + "pass@4": 0.9907644742759243, + "pass@8": 0.9913353456864922, + "pass@16": 0.9922684000518165, + "pass@32": 0.9939805554852268, + "pass@64": 0.9967972251057042, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.51230234375, + "avg_response_len": 344.525071875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.51230234375, + "pass@2": 0.5791796751968412, + "pass@4": 0.6338488173040775, + "pass@8": 0.6792057747266516, + "pass@16": 0.7175458302412239, + "pass@32": 0.7502231938070137, + "pass@64": 0.7779405120031644, + "pass@128": 0.8017 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.922875, + "pass@2": 0.94406471456693, + "pass@4": 0.9561565563367079, + "pass@8": 0.9644606428754197, + "pass@16": 0.9710626466016383, + "pass@32": 0.9765538580830442, + "pass@64": 0.9809558784585826, + "pass@128": 0.984 + }, + "12": { + "pass@1": 0.8785, + "pass@2": 0.9094697342519692, + "pass@4": 0.9260017212223469, + "pass@8": 0.9373513202159709, + "pass@16": 0.9468305077816318, + "pass@32": 0.9552749829016273, + "pass@64": 0.961888979167555, + "pass@128": 0.965 + }, + "13": { + "pass@1": 0.747625, + "pass@2": 0.8090279281496071, + "pass@4": 0.8502927079115105, + "pass@8": 0.8808873518388468, + "pass@16": 0.9045519413705413, + "pass@32": 0.9237040116121091, + "pass@64": 0.9397421870047707, + "pass@128": 0.953 + }, + "14": { + "pass@1": 0.662765625, + "pass@2": 0.7385823080708667, + "pass@4": 0.7916132981814766, + "pass@8": 0.8299020217310532, + "pass@16": 0.858302743335784, + "pass@32": 0.8802205162431695, + "pass@64": 0.8972672936992975, + "pass@128": 0.912 + }, + "15": { + "pass@1": 0.5640078125, + "pass@2": 0.6509618602362214, + "pass@4": 0.7146336797900266, + "pass@8": 0.7634876908837895, + "pass@16": 0.8021745427082866, + "pass@32": 0.8324330314086336, + "pass@64": 0.8563183762197656, + "pass@128": 0.878 + }, + "16": { + "pass@1": 0.4091171875, + "pass@2": 0.503646161417323, + "pass@4": 0.5834662753093354, + "pass@8": 0.6506320323760373, + "pass@16": 0.7081603961156945, + "pass@32": 0.7557823960324996, + "pass@64": 0.7951181687007922, + "pass@128": 0.831 + }, + "17": { + "pass@1": 0.30865625, + "pass@2": 0.3904856053149604, + "pass@4": 0.4633976213910753, + "pass@8": 0.5266492028932148, + "pass@16": 0.5819307639208975, + "pass@32": 0.6299621389073284, + "pass@64": 0.6725238032552022, + "pass@128": 0.709 + }, + "18": { + "pass@1": 0.2607265625, + "pass@2": 0.34324064960629913, + "pass@4": 0.4184819655043119, + "pass@8": 0.4815590749232352, + "pass@16": 0.5333000164446824, + "pass@32": 0.5786741441482469, + "pass@64": 0.6197798116311323, + "pass@128": 0.656 + }, + "19": { + "pass@1": 0.18853125, + "pass@2": 0.25494758858267685, + "pass@4": 0.31918568944506914, + "pass@8": 0.3778321490296506, + "pass@16": 0.4313708711520126, + "pass@32": 0.4797498884371214, + "pass@64": 0.5224194950656782, + "pass@128": 0.562 + }, + "20": { + "pass@1": 0.18021875, + "pass@2": 0.24737020177165342, + "pass@4": 0.3152586579490061, + "pass@8": 0.3792962604993045, + "pass@16": 0.4377738729809448, + "pass@32": 0.48987697029632693, + "pass@64": 0.5333911268287946, + "pass@128": 0.567 + } + }, + "per_op_accuracy": { + "11": 0.922875, + "12": 0.8785, + "13": 0.747625, + "14": 0.662765625, + "15": 0.5640078125, + "16": 0.4091171875, + "17": 0.30865625, + "18": 0.2607265625, + "19": 0.18853125, + "20": 0.18021875 + }, + "per_op_avg_response_len": { + "11": 296.1361171875, + "12": 307.14559375, + "13": 324.8332578125, + "14": 344.401625, + "15": 355.428765625, + "16": 354.597453125, + "17": 365.0842890625, + "18": 370.269, + "19": 359.351078125, + "20": 368.0035390625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 296.1361171875, + "12": 307.14559375, + "13": 324.8332578125, + "14": 344.401625, + "15": 355.428765625, + "16": 354.597453125, + "17": 365.0842890625, + "18": 370.269, + "19": 359.351078125, + "20": 368.0035390625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 221157, + "answer_accuracy": 0.5134588595839524, + "avg_response_len": 327.17834323922733, + "resp_tokens_sum": 140922256, + "pass_at_k": { + "pass@1": 0.5134588595839524, + "pass@2": 0.5795858390565243, + "pass@4": 0.6329100493010436, + "pass@8": 0.6767816699546235, + "pass@16": 0.7142134037015744, + "pass@32": 0.746710897314702, + "pass@64": 0.7742172018805795, + "pass@128": 0.7976225854383359 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9211225212464589, + "pass@2": 0.9423424918025475, + "pass@4": 0.953929790822889, + "pass@8": 0.9614565186188683, + "pass@16": 0.9677885340638542, + "pass@32": 0.9743190927138413, + "pass@64": 0.9805943394845731, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8815986935028248, + "pass@2": 0.9165651830597449, + "pass@4": 0.934161171855637, + "pass@8": 0.9452938352373702, + "pass@16": 0.9539612829997371, + "pass@32": 0.9615647682922526, + "pass@64": 0.9668943055418103, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.7734619905956113, + "pass@2": 0.8301158732999284, + "pass@4": 0.8655036610823328, + "pass@8": 0.8898255448648787, + "pass@16": 0.9097976575302069, + "pass@32": 0.92890026616942, + "pass@64": 0.9464289475637889, + "pass@128": 0.9623824451410659 + }, + "14": { + "pass@1": 0.6513093847352025, + "pass@2": 0.7309117208771797, + "pass@4": 0.7865110567370666, + "pass@8": 0.8270123821269293, + "pass@16": 0.8587307099359938, + "pass@32": 0.8847289386929739, + "pass@64": 0.9048481484220781, + "pass@128": 0.9221183800623053 + }, + "15": { + "pass@1": 0.5636741329479769, + "pass@2": 0.6490562133039006, + "pass@4": 0.7107771526752794, + "pass@8": 0.7595685140977119, + "pass@16": 0.8003622070803227, + "pass@32": 0.8331029501088946, + "pass@64": 0.8575529239777137, + "pass@128": 0.8757225433526011 + }, + "16": { + "pass@1": 0.4171118951612903, + "pass@2": 0.5142795910591821, + "pass@4": 0.5949670703460455, + "pass@8": 0.6603091766742203, + "pass@16": 0.7134515965927327, + "pass@32": 0.75739935199652, + "pass@64": 0.7984302135889665, + "pass@128": 0.8419354838709677 + }, + "17": { + "pass@1": 0.31419346098265893, + "pass@2": 0.3958058349642712, + "pass@4": 0.46747656868035986, + "pass@8": 0.5300846478128985, + "pass@16": 0.585240494400111, + "pass@32": 0.6313648084038579, + "pass@64": 0.669614375035597, + "pass@128": 0.6994219653179191 + }, + "18": { + "pass@1": 0.22939732142857142, + "pass@2": 0.3074957817772778, + "pass@4": 0.38013197760994144, + "pass@8": 0.44242261962567625, + "pass@16": 0.49595147536216044, + "pass@32": 0.5439121161703538, + "pass@64": 0.5859423405040576, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.18125938438438438, + "pass@2": 0.24487775176751556, + "pass@4": 0.3061006349431548, + "pass@8": 0.36073586559610316, + "pass@16": 0.40994189428993016, + "pass@32": 0.45423626681010243, + "pass@64": 0.4932719107305281, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.18332394894894896, + "pass@2": 0.25119817159679353, + "pass@4": 0.3202794138345321, + "pass@8": 0.3847688536744722, + "pass@16": 0.4427514993287269, + "pass@32": 0.4952158151961868, + "pass@64": 0.5381104843576482, + "pass@128": 0.5675675675675675 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 220408, + "answer_accuracy": 0.5120242343146001, + "avg_response_len": 339.50148444469227, + "resp_tokens_sum": 146143167, + "pass_at_k": { + "pass@1": 0.5120242343146001, + "pass@2": 0.5796171090093472, + "pass@4": 0.6347312556603939, + "pass@8": 0.6796315668175029, + "pass@16": 0.7166588115324342, + "pass@32": 0.7480615469732743, + "pass@64": 0.7755778775018244, + "pass@128": 0.7995837050252751 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.928699171686747, + "pass@2": 0.9515493074660848, + "pass@4": 0.9659594759804117, + "pass@8": 0.9766550766070073, + "pass@16": 0.9842963536346357, + "pass@32": 0.9884709338432213, + "pass@64": 0.9904158337215808, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8799960815047022, + "pass@2": 0.9035926739565081, + "pass@4": 0.9153743348695834, + "pass@8": 0.9229959569636945, + "pass@16": 0.9296964586106998, + "pass@32": 0.9365110569119893, + "pass@64": 0.94190277023684, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.7331276260504201, + "pass@2": 0.7954751290279893, + "pass@4": 0.8377238864049549, + "pass@8": 0.8699515828594602, + "pass@16": 0.8957292705806627, + "pass@32": 0.9165216166375357, + "pass@64": 0.9327622224638712, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6675804093567251, + "pass@2": 0.7419245291706962, + "pass@4": 0.796447699794104, + "pass@8": 0.8363445657414356, + "pass@16": 0.8626242889188614, + "pass@32": 0.8793508082194177, + "pass@64": 0.890769595219482, + "pass@128": 0.8976608187134503 + }, + "15": { + "pass@1": 0.5677320075757576, + "pass@2": 0.6639912908613691, + "pass@4": 0.7317696557816629, + "pass@8": 0.7795323520517282, + "pass@16": 0.8155345673543264, + "pass@32": 0.845208030617802, + "pass@64": 0.8714240604997285, + "pass@128": 0.896969696969697 + }, + "16": { + "pass@1": 0.4091880341880342, + "pass@2": 0.501739616281939, + "pass@4": 0.5781482055661846, + "pass@8": 0.6406309064652197, + "pass@16": 0.6934298295552146, + "pass@32": 0.7373148005034644, + "pass@64": 0.7740434816672218, + "pass@128": 0.8091168091168092 + }, + "17": { + "pass@1": 0.3116022903726708, + "pass@2": 0.39830262385680026, + "pass@4": 0.47726986174476643, + "pass@8": 0.5445244698438653, + "pass@16": 0.5995497047481053, + "pass@32": 0.6445440362273804, + "pass@64": 0.6840008853131072, + "pass@128": 0.717391304347826 + }, + "18": { + "pass@1": 0.2716911764705882, + "pass@2": 0.35394171201945335, + "pass@4": 0.4276034548071196, + "pass@8": 0.4888071795579154, + "pass@16": 0.5390270190145257, + "pass@32": 0.5858273452671591, + "pass@64": 0.6344610738799759, + "pass@128": 0.6794117647058824 + }, + "19": { + "pass@1": 0.17631603422619047, + "pass@2": 0.24067853334739409, + "pass@4": 0.3043591830597069, + "pass@8": 0.36333850531952205, + "pass@16": 0.41840949299425007, + "pass@32": 0.4696348856330499, + "pass@64": 0.5155865519395338, + "pass@128": 0.5595238095238095 + }, + "20": { + "pass@1": 0.1794068113772455, + "pass@2": 0.24754415872978455, + "pass@4": 0.31347590112238966, + "pass@8": 0.37303300737954526, + "pass@16": 0.4267697863227694, + "pass@32": 0.4749624791026021, + "pass@64": 0.5176129128175474, + "pass@128": 0.5538922155688623 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 214182, + "answer_accuracy": 0.5113988004278729, + "avg_response_len": 367.52814839929704, + "resp_tokens_sum": 153926669, + "pass_at_k": { + "pass@1": 0.5113988004278729, + "pass@2": 0.5783123673425716, + "pass@4": 0.6339072873951591, + "pass@8": 0.68126114567909, + "pass@16": 0.721884662522243, + "pass@32": 0.7560570862270739, + "pass@64": 0.7841979931738258, + "pass@128": 0.8080684596577017 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9187003968253968, + "pass@2": 0.9381061742282217, + "pass@4": 0.9483199814308924, + "pass@8": 0.9549746233950248, + "pass@16": 0.9607838243504727, + "pass@32": 0.9664979946638379, + "pass@64": 0.9713905391268673, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.8736859709480123, + "pass@2": 0.9075217017024249, + "pass@4": 0.927535943614846, + "pass@8": 0.942757162906802, + "pass@16": 0.9558258204982006, + "pass@32": 0.9667707271291923, + "pass@64": 0.9759676186550527, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7381606867283951, + "pass@2": 0.8031985910615335, + "pass@4": 0.8491654709249306, + "pass@8": 0.884136691191738, + "pass@16": 0.9091084537688574, + "pass@32": 0.9265018813718013, + "pass@64": 0.9408495040503696, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.6687917284866469, + "pass@2": 0.742496904133274, + "pass@4": 0.7915671680691242, + "pass@8": 0.8261163370467007, + "pass@16": 0.8535094321545398, + "pass@32": 0.8768087552275502, + "pass@64": 0.8966404643642354, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.560570987654321, + "pass@2": 0.6397261865704283, + "pass@4": 0.7012987609766366, + "pass@8": 0.7513312000892324, + "pass@16": 0.7905025056529893, + "pass@32": 0.8187060511329679, + "pass@64": 0.8396145819708817, + "pass@128": 0.8611111111111112 + }, + "16": { + "pass@1": 0.4017330383480826, + "pass@2": 0.4958964096578634, + "pass@4": 0.5784556440953726, + "pass@8": 0.6521378744476022, + "pass@16": 0.7185738377524138, + "pass@32": 0.7734250794597738, + "pass@64": 0.8139101487404639, + "pass@128": 0.8436578171091446 + }, + "17": { + "pass@1": 0.3000282379518072, + "pass@2": 0.3773594624561235, + "pass@4": 0.4456922685116164, + "pass@8": 0.5057320345187564, + "pass@16": 0.561393216655332, + "pass@32": 0.6143576371517395, + "pass@64": 0.6644245313917596, + "pass@128": 0.7108433734939759 + }, + "18": { + "pass@1": 0.2840725806451613, + "pass@2": 0.37186110934721867, + "pass@4": 0.4517761248593924, + "pass@8": 0.5177958613050213, + "pass@16": 0.5691864954935101, + "pass@32": 0.6100761487025451, + "pass@64": 0.6418813785016764, + "pass@128": 0.667741935483871 + }, + "19": { + "pass@1": 0.2082467900302115, + "pass@2": 0.27956287020957715, + "pass@4": 0.3474002794590251, + "pass@8": 0.4097443141957358, + "pass@16": 0.4660864976054333, + "pass@32": 0.5156854683282558, + "pass@64": 0.5586793575248601, + "pass@128": 0.595166163141994 + }, + "20": { + "pass@1": 0.17792792792792791, + "pass@2": 0.24336775259511484, + "pass@4": 0.3120260125142014, + "pass@8": 0.38010572901182366, + "pass@16": 0.44383337859661776, + "pass@32": 0.499497404852035, + "pass@64": 0.5444973653352457, + "pass@128": 0.5795795795795796 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7354477796052632, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7354477796052632, + "pass@2": 0.7727355017094696, + "pass@4": 0.8027477330629547, + "pass@8": 0.8274885423443955, + "pass@16": 0.8484073884518488, + "pass@32": 0.8662976044594782, + "pass@64": 0.8814735058262421, + "pass@128": 0.8943684210526316 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.936140625, + "pass@2": 0.9532770669291335, + "pass@4": 0.9646779643794529, + "pass@8": 0.9722428738775302, + "pass@16": 0.9780534399094882, + "pass@32": 0.9829244527998198, + "pass@64": 0.9868526799974783, + "pass@128": 0.99 + }, + "2": { + "pass@1": 0.999421875, + "pass@2": 0.9997268700787402, + "pass@4": 0.9999281513873266, + "pass@8": 0.9999954379593903, + "pass@16": 0.999999987960665, + "pass@32": 0.9999999999999909, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.997, + "pass@2": 0.9982974901574803, + "pass@4": 0.9992196539182601, + "pass@8": 0.9997844261478821, + "pass@16": 0.999981904078783, + "pass@32": 0.9999999207944156, + "pass@64": 0.9999999999999699, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9924765625, + "pass@2": 0.9934320866141733, + "pass@4": 0.9943405821147356, + "pass@8": 0.9951957912852285, + "pass@16": 0.9958488669003491, + "pass@32": 0.9965198504237649, + "pass@64": 0.9973183937001826, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.9787734375, + "pass@2": 0.9831724901574804, + "pass@4": 0.9860739181664795, + "pass@8": 0.9882095975142469, + "pass@16": 0.9906008995622421, + "pass@32": 0.993477161814902, + "pass@64": 0.9959503664688322, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.9934375, + "pass@2": 0.9951104822834648, + "pass@4": 0.9958734730971127, + "pass@8": 0.9965130401102446, + "pass@16": 0.9972419328539005, + "pass@32": 0.9980359313833913, + "pass@64": 0.9987493335625536, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.99075, + "pass@2": 0.9936044537401577, + "pass@4": 0.994668941319835, + "pass@8": 0.9952967222076122, + "pass@16": 0.9959389380230231, + "pass@32": 0.9967077687453606, + "pass@64": 0.997570362137923, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.983, + "pass@2": 0.9881865157480314, + "pass@4": 0.9911749555680541, + "pass@8": 0.9933137034852952, + "pass@16": 0.9948246809075276, + "pass@32": 0.9958187164772767, + "pass@64": 0.9964386629519557, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.979484375, + "pass@2": 0.9853703248031499, + "pass@4": 0.9877611152043493, + "pass@8": 0.9896729646893262, + "pass@16": 0.9917914279768701, + "pass@32": 0.993938744220871, + "pass@64": 0.9957116918480703, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.922875, + "pass@2": 0.94406471456693, + "pass@4": 0.9561565563367079, + "pass@8": 0.9644606428754197, + "pass@16": 0.9710626466016383, + "pass@32": 0.9765538580830442, + "pass@64": 0.9809558784585826, + "pass@128": 0.984 + }, + "12": { + "pass@1": 0.8785, + "pass@2": 0.9094697342519692, + "pass@4": 0.9260017212223469, + "pass@8": 0.9373513202159709, + "pass@16": 0.9468305077816318, + "pass@32": 0.9552749829016273, + "pass@64": 0.961888979167555, + "pass@128": 0.965 + }, + "13": { + "pass@1": 0.747625, + "pass@2": 0.8090279281496071, + "pass@4": 0.8502927079115105, + "pass@8": 0.8808873518388468, + "pass@16": 0.9045519413705413, + "pass@32": 0.9237040116121091, + "pass@64": 0.9397421870047707, + "pass@128": 0.953 + }, + "14": { + "pass@1": 0.662765625, + "pass@2": 0.7385823080708667, + "pass@4": 0.7916132981814766, + "pass@8": 0.8299020217310532, + "pass@16": 0.858302743335784, + "pass@32": 0.8802205162431695, + "pass@64": 0.8972672936992975, + "pass@128": 0.912 + }, + "15": { + "pass@1": 0.5640078125, + "pass@2": 0.6509618602362214, + "pass@4": 0.7146336797900266, + "pass@8": 0.7634876908837895, + "pass@16": 0.8021745427082866, + "pass@32": 0.8324330314086336, + "pass@64": 0.8563183762197656, + "pass@128": 0.878 + }, + "16": { + "pass@1": 0.4091171875, + "pass@2": 0.503646161417323, + "pass@4": 0.5834662753093354, + "pass@8": 0.6506320323760373, + "pass@16": 0.7081603961156945, + "pass@32": 0.7557823960324996, + "pass@64": 0.7951181687007922, + "pass@128": 0.831 + }, + "17": { + "pass@1": 0.30865625, + "pass@2": 0.3904856053149604, + "pass@4": 0.4633976213910753, + "pass@8": 0.5266492028932148, + "pass@16": 0.5819307639208975, + "pass@32": 0.6299621389073284, + "pass@64": 0.6725238032552022, + "pass@128": 0.709 + }, + "18": { + "pass@1": 0.2607265625, + "pass@2": 0.34324064960629913, + "pass@4": 0.4184819655043119, + "pass@8": 0.4815590749232352, + "pass@16": 0.5333000164446824, + "pass@32": 0.5786741441482469, + "pass@64": 0.6197798116311323, + "pass@128": 0.656 + }, + "19": { + "pass@1": 0.18853125, + "pass@2": 0.25494758858267685, + "pass@4": 0.31918568944506914, + "pass@8": 0.3778321490296506, + "pass@16": 0.4313708711520126, + "pass@32": 0.4797498884371214, + "pass@64": 0.5224194950656782, + "pass@128": 0.562 + }, + "20": { + "pass@1": 0.18021875, + "pass@2": 0.24737020177165342, + "pass@4": 0.3152586579490061, + "pass@8": 0.3792962604993045, + "pass@16": 0.4377738729809448, + "pass@32": 0.48987697029632693, + "pass@64": 0.5333911268287946, + "pass@128": 0.567 + } + }, + "per_op_accuracy": { + "10": 0.936140625, + "2": 0.999421875, + "3": 0.997, + "4": 0.9924765625, + "5": 0.9787734375, + "6": 0.9934375, + "7": 0.99075, + "8": 0.983, + "9": 0.979484375, + "11": 0.922875, + "12": 0.8785, + "13": 0.747625, + "14": 0.662765625, + "15": 0.5640078125, + "16": 0.4091171875, + "17": 0.30865625, + "18": 0.2607265625, + "19": 0.18853125, + "20": 0.18021875 + }, + "per_op_avg_response_len": { + "10": 278.78703125, + "2": 115.1384140625, + "3": 144.0647578125, + "4": 159.348484375, + "5": 172.7458984375, + "6": 197.9644140625, + "7": 222.690140625, + "8": 242.006671875, + "9": 257.6563046875, + "11": 296.1361171875, + "12": 307.14559375, + "13": 324.8332578125, + "14": 344.401625, + "15": 355.428765625, + "16": 354.597453125, + "17": 365.0842890625, + "18": 370.269, + "19": 359.351078125, + "20": 368.0035390625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 278.78703125, + "2": 115.1384140625, + "3": 144.0647578125, + "4": 159.348484375, + "5": 172.7458984375, + "6": 197.9644140625, + "7": 222.690140625, + "8": 242.006671875, + "9": 257.6563046875, + "11": 296.1361171875, + "12": 307.14559375, + "13": 324.8332578125, + "14": 344.401625, + "15": 355.428765625, + "16": 354.597453125, + "17": 365.0842890625, + "18": 370.269, + "19": 359.351078125, + "20": 368.0035390625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 593672, + "answer_accuracy": 0.7331745969016756, + "avg_response_len": 262.80997199059436, + "resp_tokens_sum": 212804593, + "pass_at_k": { + "pass@1": 0.7331745969016756, + "pass@2": 0.7706344784429154, + "pass@4": 0.8002934745293199, + "pass@8": 0.8244781684255242, + "pass@16": 0.8451241664967811, + "pass@32": 0.8631318988944838, + "pass@64": 0.8783587068525153, + "pass@128": 0.8912424913057224 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9386292404129793, + "pass@2": 0.9558454840545374, + "pass@4": 0.9661186383781674, + "pass@8": 0.9715414375472532, + "pass@16": 0.9756175864300959, + "pass@32": 0.9796003458610456, + "pass@64": 0.9837013134257441, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977780963302753, + "pass@2": 0.9989322220616917, + "pass@4": 0.9996694776685024, + "pass@8": 0.9999677623799305, + "pass@16": 0.999999779641568, + "pass@32": 0.9999999999981484, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9837158203125, + "pass@2": 0.9857337290846455, + "pass@4": 0.9882842340996436, + "pass@8": 0.9906827001964222, + "pass@16": 0.9922391181843271, + "pass@32": 0.9935574745502678, + "pass@64": 0.9951263779508654, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9695188492063492, + "pass@2": 0.976019013248344, + "pass@4": 0.9805692556287607, + "pass@8": 0.9843784633909355, + "pass@16": 0.9889107918312874, + "pass@32": 0.9940484172603424, + "pass@64": 0.9981441113782779, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9948774326347305, + "pass@2": 0.9966545747088502, + "pass@4": 0.9974962398038569, + "pass@8": 0.9981117754532004, + "pass@16": 0.9988525144880754, + "pass@32": 0.9996224712529058, + "pass@64": 0.9999803235855924, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9937361316568047, + "pass@2": 0.9963439989749802, + "pass@4": 0.9969306777666104, + "pass@8": 0.9970380005155915, + "pass@16": 0.9970414182830878, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9876302083333334, + "pass@2": 0.9916015230668282, + "pass@4": 0.9930404943372461, + "pass@8": 0.9940772665714049, + "pass@16": 0.9949131147821092, + "pass@32": 0.9957967829765609, + "pass@64": 0.9966039773874414, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9811399647887324, + "pass@2": 0.9891198569368971, + "pass@4": 0.9927404183631975, + "pass@8": 0.9952636734053902, + "pass@16": 0.9975775655877893, + "pass@32": 0.999379044270334, + "pass@64": 0.9999770279396809, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9211225212464589, + "pass@2": 0.9423424918025475, + "pass@4": 0.953929790822889, + "pass@8": 0.9614565186188683, + "pass@16": 0.9677885340638542, + "pass@32": 0.9743190927138413, + "pass@64": 0.9805943394845731, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8815986935028248, + "pass@2": 0.9165651830597449, + "pass@4": 0.934161171855637, + "pass@8": 0.9452938352373702, + "pass@16": 0.9539612829997371, + "pass@32": 0.9615647682922526, + "pass@64": 0.9668943055418103, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.7734619905956113, + "pass@2": 0.8301158732999284, + "pass@4": 0.8655036610823328, + "pass@8": 0.8898255448648787, + "pass@16": 0.9097976575302069, + "pass@32": 0.92890026616942, + "pass@64": 0.9464289475637889, + "pass@128": 0.9623824451410659 + }, + "14": { + "pass@1": 0.6513093847352025, + "pass@2": 0.7309117208771797, + "pass@4": 0.7865110567370666, + "pass@8": 0.8270123821269293, + "pass@16": 0.8587307099359938, + "pass@32": 0.8847289386929739, + "pass@64": 0.9048481484220781, + "pass@128": 0.9221183800623053 + }, + "15": { + "pass@1": 0.5636741329479769, + "pass@2": 0.6490562133039006, + "pass@4": 0.7107771526752794, + "pass@8": 0.7595685140977119, + "pass@16": 0.8003622070803227, + "pass@32": 0.8331029501088946, + "pass@64": 0.8575529239777137, + "pass@128": 0.8757225433526011 + }, + "16": { + "pass@1": 0.4171118951612903, + "pass@2": 0.5142795910591821, + "pass@4": 0.5949670703460455, + "pass@8": 0.6603091766742203, + "pass@16": 0.7134515965927327, + "pass@32": 0.75739935199652, + "pass@64": 0.7984302135889665, + "pass@128": 0.8419354838709677 + }, + "17": { + "pass@1": 0.31419346098265893, + "pass@2": 0.3958058349642712, + "pass@4": 0.46747656868035986, + "pass@8": 0.5300846478128985, + "pass@16": 0.585240494400111, + "pass@32": 0.6313648084038579, + "pass@64": 0.669614375035597, + "pass@128": 0.6994219653179191 + }, + "18": { + "pass@1": 0.22939732142857142, + "pass@2": 0.3074957817772778, + "pass@4": 0.38013197760994144, + "pass@8": 0.44242261962567625, + "pass@16": 0.49595147536216044, + "pass@32": 0.5439121161703538, + "pass@64": 0.5859423405040576, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.18125938438438438, + "pass@2": 0.24487775176751556, + "pass@4": 0.3061006349431548, + "pass@8": 0.36073586559610316, + "pass@16": 0.40994189428993016, + "pass@32": 0.45423626681010243, + "pass@64": 0.4932719107305281, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.18332394894894896, + "pass@2": 0.25119817159679353, + "pass@4": 0.3202794138345321, + "pass@8": 0.3847688536744722, + "pass@16": 0.4427514993287269, + "pass@32": 0.4952158151961868, + "pass@64": 0.5381104843576482, + "pass@128": 0.5675675675675675 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 604538, + "answer_accuracy": 0.736235872954014, + "avg_response_len": 271.6522250097428, + "resp_tokens_sum": 223059075, + "pass_at_k": { + "pass@1": 0.736235872954014, + "pass@2": 0.7735831144401895, + "pass@4": 0.8036381645152837, + "pass@8": 0.8281069190722744, + "pass@16": 0.8483726997042081, + "pass@32": 0.8655832133804009, + "pass@64": 0.8805682987802501, + "pass@128": 0.8933749025720966 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9312691717791411, + "pass@2": 0.9492200708902958, + "pass@4": 0.9614929713540408, + "pass@8": 0.9706849500303524, + "pass@16": 0.9780743583738936, + "pass@32": 0.9833452492500756, + "pass@64": 0.9865077334207035, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150811, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961647727272728, + "pass@2": 0.9978696910045335, + "pass@4": 0.9990399083637271, + "pass@8": 0.9997415688663096, + "pass@16": 0.9999820837105999, + "pass@32": 0.9999999593969077, + "pass@64": 0.9999999999999987, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9963480029585798, + "pass@2": 0.9970392361272888, + "pass@4": 0.9973065910215366, + "pass@8": 0.9975660568709271, + "pass@16": 0.9980247014241554, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9867753623188406, + "pass@2": 0.9882117996120052, + "pass@4": 0.9893897974709683, + "pass@8": 0.9906458350692433, + "pass@16": 0.9923677751658371, + "pass@32": 0.9946202004072046, + "pass@64": 0.996556005154711, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9901504297994269, + "pass@2": 0.9921238690973083, + "pass@4": 0.9931829475828413, + "pass@8": 0.9941730137747032, + "pass@16": 0.9951084353395996, + "pass@32": 0.9959328579290595, + "pass@64": 0.9967849648397139, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9909230491329479, + "pass@2": 0.9937133039005962, + "pass@4": 0.9946745414951744, + "pass@8": 0.9952348105100473, + "pass@16": 0.9961070653038383, + "pass@32": 0.9974785593997282, + "pass@64": 0.9991110099676848, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9840008802816902, + "pass@2": 0.9890027170899413, + "pass@4": 0.9920841439714402, + "pass@8": 0.9944509310383022, + "pass@16": 0.9961687806906195, + "pass@32": 0.9970422143124665, + "pass@64": 0.9971821603034191, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9723755613772455, + "pass@2": 0.977356311000047, + "pass@4": 0.9796722011544962, + "pass@8": 0.9821828386883638, + "pass@16": 0.9851973646020666, + "pass@32": 0.988117457332326, + "pass@64": 0.9901673949748785, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.928699171686747, + "pass@2": 0.9515493074660848, + "pass@4": 0.9659594759804117, + "pass@8": 0.9766550766070073, + "pass@16": 0.9842963536346357, + "pass@32": 0.9884709338432213, + "pass@64": 0.9904158337215808, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8799960815047022, + "pass@2": 0.9035926739565081, + "pass@4": 0.9153743348695834, + "pass@8": 0.9229959569636945, + "pass@16": 0.9296964586106998, + "pass@32": 0.9365110569119893, + "pass@64": 0.94190277023684, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.7331276260504201, + "pass@2": 0.7954751290279893, + "pass@4": 0.8377238864049549, + "pass@8": 0.8699515828594602, + "pass@16": 0.8957292705806627, + "pass@32": 0.9165216166375357, + "pass@64": 0.9327622224638712, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6675804093567251, + "pass@2": 0.7419245291706962, + "pass@4": 0.796447699794104, + "pass@8": 0.8363445657414356, + "pass@16": 0.8626242889188614, + "pass@32": 0.8793508082194177, + "pass@64": 0.890769595219482, + "pass@128": 0.8976608187134503 + }, + "15": { + "pass@1": 0.5677320075757576, + "pass@2": 0.6639912908613691, + "pass@4": 0.7317696557816629, + "pass@8": 0.7795323520517282, + "pass@16": 0.8155345673543264, + "pass@32": 0.845208030617802, + "pass@64": 0.8714240604997285, + "pass@128": 0.896969696969697 + }, + "16": { + "pass@1": 0.4091880341880342, + "pass@2": 0.501739616281939, + "pass@4": 0.5781482055661846, + "pass@8": 0.6406309064652197, + "pass@16": 0.6934298295552146, + "pass@32": 0.7373148005034644, + "pass@64": 0.7740434816672218, + "pass@128": 0.8091168091168092 + }, + "17": { + "pass@1": 0.3116022903726708, + "pass@2": 0.39830262385680026, + "pass@4": 0.47726986174476643, + "pass@8": 0.5445244698438653, + "pass@16": 0.5995497047481053, + "pass@32": 0.6445440362273804, + "pass@64": 0.6840008853131072, + "pass@128": 0.717391304347826 + }, + "18": { + "pass@1": 0.2716911764705882, + "pass@2": 0.35394171201945335, + "pass@4": 0.4276034548071196, + "pass@8": 0.4888071795579154, + "pass@16": 0.5390270190145257, + "pass@32": 0.5858273452671591, + "pass@64": 0.6344610738799759, + "pass@128": 0.6794117647058824 + }, + "19": { + "pass@1": 0.17631603422619047, + "pass@2": 0.24067853334739409, + "pass@4": 0.3043591830597069, + "pass@8": 0.36333850531952205, + "pass@16": 0.41840949299425007, + "pass@32": 0.4696348856330499, + "pass@64": 0.5155865519395338, + "pass@128": 0.5595238095238095 + }, + "20": { + "pass@1": 0.1794068113772455, + "pass@2": 0.24754415872978455, + "pass@4": 0.31347590112238966, + "pass@8": 0.37303300737954526, + "pass@16": 0.4267697863227694, + "pass@32": 0.4749624791026021, + "pass@64": 0.5176129128175474, + "pass@128": 0.5538922155688623 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 590399, + "answer_accuracy": 0.7369375599137242, + "avg_response_len": 292.45373537106565, + "resp_tokens_sum": 234299895, + "pass_at_k": { + "pass@1": 0.7369375599137242, + "pass@2": 0.7739902768359941, + "pass@4": 0.8043156387535312, + "pass@8": 0.8298973518508928, + "pass@16": 0.8517613092704859, + "pass@32": 0.8702293942304131, + "pass@64": 0.8855494159569103, + "pass@128": 0.8985460936251798 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9383628731343283, + "pass@2": 0.9546259842519684, + "pass@4": 0.9663195143517505, + "pass@8": 0.9744687547436317, + "pass@16": 0.9804980217307362, + "pass@32": 0.9858787590967183, + "pass@64": 0.9903773541820934, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9984375, + "pass@2": 0.9992217379077615, + "pass@4": 0.9997947185173283, + "pass@8": 0.999986965598258, + "pass@16": 0.9999999656019002, + "pass@32": 0.999999999999974, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970617711370262, + "pass@2": 0.9981039519294782, + "pass@4": 0.9989637462409036, + "pass@8": 0.9996508749963928, + "pass@16": 0.9999646895378788, + "pass@32": 0.9999998081458937, + "pass@64": 0.9999999999999133, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9968475877192983, + "pass@2": 0.997070267532348, + "pass@4": 0.9970760217472814, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9792279411764706, + "pass@2": 0.9846865591709126, + "pass@4": 0.9878091838703983, + "pass@8": 0.9892869660212149, + "pass@16": 0.9903738755975673, + "pass@32": 0.9917880624629651, + "pass@64": 0.9933033811949977, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9955392350157729, + "pass@2": 0.9967716845922651, + "pass@4": 0.9971258053508296, + "pass@8": 0.9974048116766691, + "pass@16": 0.997893845682534, + "pass@32": 0.9986796800242242, + "pass@64": 0.9996149929208377, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9873664952531646, + "pass@2": 0.9905549966361007, + "pass@4": 0.9922436103714881, + "pass@8": 0.9935020050533728, + "pass@16": 0.9945756140766189, + "pass@32": 0.995506921497008, + "pass@64": 0.996449217370583, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9775947822822822, + "pass@2": 0.9841167446974534, + "pass@4": 0.9884578084833988, + "pass@8": 0.9913859333225822, + "pass@16": 0.9933089247757945, + "pass@32": 0.9945349372602522, + "pass@64": 0.995491156454535, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9852290996784566, + "pass@2": 0.9896970023039725, + "pass@4": 0.9907644742759243, + "pass@8": 0.9913353456864922, + "pass@16": 0.9922684000518165, + "pass@32": 0.9939805554852268, + "pass@64": 0.9967972251057042, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9187003968253968, + "pass@2": 0.9381061742282217, + "pass@4": 0.9483199814308924, + "pass@8": 0.9549746233950248, + "pass@16": 0.9607838243504727, + "pass@32": 0.9664979946638379, + "pass@64": 0.9713905391268673, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.8736859709480123, + "pass@2": 0.9075217017024249, + "pass@4": 0.927535943614846, + "pass@8": 0.942757162906802, + "pass@16": 0.9558258204982006, + "pass@32": 0.9667707271291923, + "pass@64": 0.9759676186550527, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7381606867283951, + "pass@2": 0.8031985910615335, + "pass@4": 0.8491654709249306, + "pass@8": 0.884136691191738, + "pass@16": 0.9091084537688574, + "pass@32": 0.9265018813718013, + "pass@64": 0.9408495040503696, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.6687917284866469, + "pass@2": 0.742496904133274, + "pass@4": 0.7915671680691242, + "pass@8": 0.8261163370467007, + "pass@16": 0.8535094321545398, + "pass@32": 0.8768087552275502, + "pass@64": 0.8966404643642354, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.560570987654321, + "pass@2": 0.6397261865704283, + "pass@4": 0.7012987609766366, + "pass@8": 0.7513312000892324, + "pass@16": 0.7905025056529893, + "pass@32": 0.8187060511329679, + "pass@64": 0.8396145819708817, + "pass@128": 0.8611111111111112 + }, + "16": { + "pass@1": 0.4017330383480826, + "pass@2": 0.4958964096578634, + "pass@4": 0.5784556440953726, + "pass@8": 0.6521378744476022, + "pass@16": 0.7185738377524138, + "pass@32": 0.7734250794597738, + "pass@64": 0.8139101487404639, + "pass@128": 0.8436578171091446 + }, + "17": { + "pass@1": 0.3000282379518072, + "pass@2": 0.3773594624561235, + "pass@4": 0.4456922685116164, + "pass@8": 0.5057320345187564, + "pass@16": 0.561393216655332, + "pass@32": 0.6143576371517395, + "pass@64": 0.6644245313917596, + "pass@128": 0.7108433734939759 + }, + "18": { + "pass@1": 0.2840725806451613, + "pass@2": 0.37186110934721867, + "pass@4": 0.4517761248593924, + "pass@8": 0.5177958613050213, + "pass@16": 0.5691864954935101, + "pass@32": 0.6100761487025451, + "pass@64": 0.6418813785016764, + "pass@128": 0.667741935483871 + }, + "19": { + "pass@1": 0.2082467900302115, + "pass@2": 0.27956287020957715, + "pass@4": 0.3474002794590251, + "pass@8": 0.4097443141957358, + "pass@16": 0.4660864976054333, + "pass@32": 0.5156854683282558, + "pass@64": 0.5586793575248601, + "pass@128": 0.595166163141994 + }, + "20": { + "pass@1": 0.17792792792792791, + "pass@2": 0.24336775259511484, + "pass@4": 0.3120260125142014, + "pass@8": 0.38010572901182366, + "pass@16": 0.44383337859661776, + "pass@32": 0.499497404852035, + "pass@64": 0.5444973653352457, + "pass@128": 0.5795795795795796 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch1-op9-12/graphs/SDAR-100M-block8-epoch1-op9-12_score_report.jsonl b/composition/SDAR-100M-block8-epoch1-op9-12/graphs/SDAR-100M-block8-epoch1-op9-12_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4a3d51c45c60f6c192be31449f53e7d551bfc5f0 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op9-12/graphs/SDAR-100M-block8-epoch1-op9-12_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch1-op9-12", "results_dir": "results/composition/SDAR-100M-block8-epoch1-op9-12/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch1-op9-12/graphs/SDAR-100M-block8-epoch1-op9-12_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 298747, "value_mismatch": 729057, "dependency_mismatch": 902947, "answer_mismatch": 624250, "extra_nodes": 79528, "missing_nodes": 569228, "perfect_with_extra": 211, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 245683, "dependency_mismatch": 304952, "answer_mismatch": 209561, "perfect": 100492, "extra_nodes": 30011, "missing_nodes": 200315, "perfect_with_extra": 15}, "movie_festival_awards": {"total": 418816, "perfect": 97625, "value_mismatch": 240565, "answer_mismatch": 204642, "dependency_mismatch": 297945, "missing_nodes": 177548, "extra_nodes": 25906, "perfect_with_extra": 21}, "teachers_in_school": {"total": 430464, "perfect": 100630, "dependency_mismatch": 300050, "value_mismatch": 242809, "answer_mismatch": 210047, "extra_nodes": 23611, "missing_nodes": 191365, "perfect_with_extra": 175}}, "results_path": "results/composition/SDAR-100M-block8-epoch1-op9-12/graphs/SDAR-100M-block8-epoch1-op9-12_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 1, "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 298747, "value_mismatch": 729057, "dependency_mismatch": 902947, "answer_mismatch": 624250, "extra_nodes": 79528, "missing_nodes": 569228, "perfect_with_extra": 211, "skipped": 1, "unused_gold_entries": 9000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch1-op9-12/graphs/SDAR-100M-block8-epoch1-op9-12_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 245683, "dependency_mismatch": 304952, "answer_mismatch": 209561, "perfect": 100492, "extra_nodes": 30011, "missing_nodes": 200315, "perfect_with_extra": 15}, "movie_festival_awards": {"total": 418816, "perfect": 97625, "value_mismatch": 240565, "answer_mismatch": 204642, "dependency_mismatch": 297945, "missing_nodes": 177548, "extra_nodes": 25906, "perfect_with_extra": 21}, "teachers_in_school": {"total": 430464, "perfect": 100630, "dependency_mismatch": 300050, "value_mismatch": 242809, "answer_mismatch": 210047, "extra_nodes": 23611, "missing_nodes": 191365, "perfect_with_extra": 175}}}} diff --git a/composition/SDAR-100M-block8-epoch1-op9-12/graphs/results.json b/composition/SDAR-100M-block8-epoch1-op9-12/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..3df9e2da803ce33d4074be4c80e683891aaf4fb4 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op9-12/graphs/results.json @@ -0,0 +1,1704 @@ +{ + "files": { + "SDAR-100M-block8-epoch1-op9-12_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5382436260623229, + "pass@2": 0.6458923512747875, + "pass@4": 0.7110481586402266, + "pass@8": 0.7847025495750708, + "pass@16": 0.8640226628895185, + "pass@32": 0.9036827195467422, + "pass@64": 0.9206798866855525, + "pass@128": 0.9291784702549575 + }, + "12": { + "pass@1": 0.5677966101694916, + "pass@2": 0.6271186440677966, + "pass@4": 0.7005649717514124, + "pass@8": 0.7655367231638418, + "pass@16": 0.8107344632768362, + "pass@32": 0.8361581920903954, + "pass@64": 0.8559322033898306, + "pass@128": 0.8700564971751412 + }, + "13": { + "pass@1": 0.4106583072100313, + "pass@2": 0.4890282131661442, + "pass@4": 0.5579937304075235, + "pass@8": 0.6300940438871473, + "pass@16": 0.7147335423197492, + "pass@32": 0.7492163009404389, + "pass@64": 0.774294670846395, + "pass@128": 0.7899686520376176 + }, + "14": { + "pass@1": 0.3177570093457944, + "pass@2": 0.35514018691588783, + "pass@4": 0.4143302180685358, + "pass@8": 0.46417445482866043, + "pass@16": 0.5358255451713395, + "pass@32": 0.6105919003115264, + "pass@64": 0.6635514018691588, + "pass@128": 0.719626168224299 + }, + "15": { + "pass@1": 0.2138728323699422, + "pass@2": 0.24855491329479767, + "pass@4": 0.30057803468208094, + "pass@8": 0.34104046242774566, + "pass@16": 0.3988439306358382, + "pass@32": 0.47398843930635837, + "pass@64": 0.5433526011560693, + "pass@128": 0.6127167630057804 + }, + "16": { + "pass@1": 0.12258064516129032, + "pass@2": 0.17096774193548386, + "pass@4": 0.1967741935483871, + "pass@8": 0.23548387096774193, + "pass@16": 0.26129032258064516, + "pass@32": 0.31290322580645163, + "pass@64": 0.36129032258064514, + "pass@128": 0.4 + }, + "17": { + "pass@1": 0.07803468208092486, + "pass@2": 0.10115606936416185, + "pass@4": 0.12427745664739884, + "pass@8": 0.1531791907514451, + "pass@16": 0.1676300578034682, + "pass@32": 0.19653179190751446, + "pass@64": 0.24277456647398843, + "pass@128": 0.2774566473988439 + }, + "18": { + "pass@1": 0.022857142857142857, + "pass@2": 0.037142857142857144, + "pass@4": 0.05142857142857143, + "pass@8": 0.06285714285714286, + "pass@16": 0.08, + "pass@32": 0.09714285714285714, + "pass@64": 0.12571428571428572, + "pass@128": 0.1457142857142857 + }, + "19": { + "pass@1": 0.017045454545454544, + "pass@2": 0.017045454545454544, + "pass@4": 0.022727272727272728, + "pass@8": 0.03409090909090909, + "pass@16": 0.05113636363636364, + "pass@32": 0.08522727272727272, + "pass@64": 0.09659090909090909, + "pass@128": 0.09659090909090909 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5382436260623229, + "12": 0.5677966101694916, + "13": 0.4106583072100313, + "14": 0.3177570093457944, + "15": 0.2138728323699422, + "16": 0.12258064516129032, + "17": 0.07803468208092486, + "18": 0.022857142857142857, + "19": 0.017045454545454544 + }, + "pass@2": { + "11": 0.6458923512747875, + "12": 0.6271186440677966, + "13": 0.4890282131661442, + "14": 0.35514018691588783, + "15": 0.24855491329479767, + "16": 0.17096774193548386, + "17": 0.10115606936416185, + "18": 0.037142857142857144, + "19": 0.017045454545454544 + }, + "pass@4": { + "11": 0.7110481586402266, + "12": 0.7005649717514124, + "13": 0.5579937304075235, + "14": 0.4143302180685358, + "15": 0.30057803468208094, + "16": 0.1967741935483871, + "17": 0.12427745664739884, + "18": 0.05142857142857143, + "19": 0.022727272727272728 + }, + "pass@8": { + "11": 0.7847025495750708, + "12": 0.7655367231638418, + "13": 0.6300940438871473, + "14": 0.46417445482866043, + "15": 0.34104046242774566, + "16": 0.23548387096774193, + "17": 0.1531791907514451, + "18": 0.06285714285714286, + "19": 0.03409090909090909 + }, + "pass@16": { + "11": 0.8640226628895185, + "12": 0.8107344632768362, + "13": 0.7147335423197492, + "14": 0.5358255451713395, + "15": 0.3988439306358382, + "16": 0.26129032258064516, + "17": 0.1676300578034682, + "18": 0.08, + "19": 0.05113636363636364 + }, + "pass@32": { + "11": 0.9036827195467422, + "12": 0.8361581920903954, + "13": 0.7492163009404389, + "14": 0.6105919003115264, + "15": 0.47398843930635837, + "16": 0.31290322580645163, + "17": 0.19653179190751446, + "18": 0.09714285714285714, + "19": 0.08522727272727272 + }, + "pass@64": { + "11": 0.9206798866855525, + "12": 0.8559322033898306, + "13": 0.774294670846395, + "14": 0.6635514018691588, + "15": 0.5433526011560693, + "16": 0.36129032258064514, + "17": 0.24277456647398843, + "18": 0.12571428571428572, + "19": 0.09659090909090909 + }, + "pass@128": { + "11": 0.9291784702549575, + "12": 0.8700564971751412, + "13": 0.7899686520376176, + "14": 0.719626168224299, + "15": 0.6127167630057804, + "16": 0.4, + "17": 0.2774566473988439, + "18": 0.1457142857142857, + "19": 0.09659090909090909 + } + }, + "total_pass_at_k": { + "pass@1": 0.26921739130434785, + "pass@2": 0.3165217391304348, + "pass@4": 0.3617391304347826, + "pass@8": 0.40695652173913044, + "pass@16": 0.4542608695652174, + "pass@32": 0.49669565217391304, + "pass@64": 0.5332173913043479, + "pass@128": 0.5631304347826087 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5238095238095238, + "pass@2": 0.6412698412698413, + "pass@4": 0.7333333333333333, + "pass@8": 0.8126984126984127, + "pass@16": 0.8952380952380953, + "pass@32": 0.9238095238095239, + "pass@64": 0.9396825396825397, + "pass@128": 0.9523809523809523 + }, + "12": { + "pass@1": 0.5565749235474006, + "pass@2": 0.6116207951070336, + "pass@4": 0.7003058103975535, + "pass@8": 0.7889908256880734, + "pass@16": 0.8440366972477065, + "pass@32": 0.8715596330275229, + "pass@64": 0.8868501529051988, + "pass@128": 0.8929663608562691 + }, + "13": { + "pass@1": 0.42592592592592593, + "pass@2": 0.5, + "pass@4": 0.5617283950617284, + "pass@8": 0.6388888888888888, + "pass@16": 0.691358024691358, + "pass@32": 0.7160493827160493, + "pass@64": 0.7438271604938271, + "pass@128": 0.7746913580246914 + }, + "14": { + "pass@1": 0.29673590504451036, + "pass@2": 0.34124629080118696, + "pass@4": 0.3857566765578635, + "pass@8": 0.4332344213649852, + "pass@16": 0.49851632047477745, + "pass@32": 0.5727002967359051, + "pass@64": 0.6468842729970327, + "pass@128": 0.6913946587537092 + }, + "15": { + "pass@1": 0.2345679012345679, + "pass@2": 0.26851851851851855, + "pass@4": 0.32407407407407407, + "pass@8": 0.36419753086419754, + "pass@16": 0.41358024691358025, + "pass@32": 0.4845679012345679, + "pass@64": 0.5401234567901234, + "pass@128": 0.6111111111111112 + }, + "16": { + "pass@1": 0.13569321533923304, + "pass@2": 0.17404129793510326, + "pass@4": 0.22418879056047197, + "pass@8": 0.25958702064896755, + "pass@16": 0.30383480825958703, + "pass@32": 0.35398230088495575, + "pass@64": 0.40117994100294985, + "pass@128": 0.44542772861356933 + }, + "17": { + "pass@1": 0.07228915662650602, + "pass@2": 0.09036144578313253, + "pass@4": 0.1144578313253012, + "pass@8": 0.12349397590361445, + "pass@16": 0.1536144578313253, + "pass@32": 0.18072289156626506, + "pass@64": 0.21987951807228914, + "pass@128": 0.2560240963855422 + }, + "18": { + "pass@1": 0.08387096774193549, + "pass@2": 0.1, + "pass@4": 0.12580645161290321, + "pass@8": 0.13225806451612904, + "pass@16": 0.14838709677419354, + "pass@32": 0.15483870967741936, + "pass@64": 0.17419354838709677, + "pass@128": 0.2032258064516129 + }, + "19": { + "pass@1": 0.016853932584269662, + "pass@2": 0.056179775280898875, + "pass@4": 0.06741573033707865, + "pass@8": 0.08426966292134831, + "pass@16": 0.10112359550561797, + "pass@32": 0.12921348314606743, + "pass@64": 0.12921348314606743, + "pass@128": 0.1348314606741573 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5238095238095238, + "12": 0.5565749235474006, + "13": 0.42592592592592593, + "14": 0.29673590504451036, + "15": 0.2345679012345679, + "16": 0.13569321533923304, + "17": 0.07228915662650602, + "18": 0.08387096774193549, + "19": 0.016853932584269662 + }, + "pass@2": { + "11": 0.6412698412698413, + "12": 0.6116207951070336, + "13": 0.5, + "14": 0.34124629080118696, + "15": 0.26851851851851855, + "16": 0.17404129793510326, + "17": 0.09036144578313253, + "18": 0.1, + "19": 0.056179775280898875 + }, + "pass@4": { + "11": 0.7333333333333333, + "12": 0.7003058103975535, + "13": 0.5617283950617284, + "14": 0.3857566765578635, + "15": 0.32407407407407407, + "16": 0.22418879056047197, + "17": 0.1144578313253012, + "18": 0.12580645161290321, + "19": 0.06741573033707865 + }, + "pass@8": { + "11": 0.8126984126984127, + "12": 0.7889908256880734, + "13": 0.6388888888888888, + "14": 0.4332344213649852, + "15": 0.36419753086419754, + "16": 0.25958702064896755, + "17": 0.12349397590361445, + "18": 0.13225806451612904, + "19": 0.08426966292134831 + }, + "pass@16": { + "11": 0.8952380952380953, + "12": 0.8440366972477065, + "13": 0.691358024691358, + "14": 0.49851632047477745, + "15": 0.41358024691358025, + "16": 0.30383480825958703, + "17": 0.1536144578313253, + "18": 0.14838709677419354, + "19": 0.10112359550561797 + }, + "pass@32": { + "11": 0.9238095238095239, + "12": 0.8715596330275229, + "13": 0.7160493827160493, + "14": 0.5727002967359051, + "15": 0.4845679012345679, + "16": 0.35398230088495575, + "17": 0.18072289156626506, + "18": 0.15483870967741936, + "19": 0.12921348314606743 + }, + "pass@64": { + "11": 0.9396825396825397, + "12": 0.8868501529051988, + "13": 0.7438271604938271, + "14": 0.6468842729970327, + "15": 0.5401234567901234, + "16": 0.40117994100294985, + "17": 0.21987951807228914, + "18": 0.17419354838709677, + "19": 0.12921348314606743 + }, + "pass@128": { + "11": 0.9523809523809523, + "12": 0.8929663608562691, + "13": 0.7746913580246914, + "14": 0.6913946587537092, + "15": 0.6111111111111112, + "16": 0.44542772861356933, + "17": 0.2560240963855422, + "18": 0.2032258064516129, + "19": 0.1348314606741573 + } + }, + "total_pass_at_k": { + "pass@1": 0.2727925340990668, + "pass@2": 0.32160804020100503, + "pass@4": 0.3740129217516152, + "pass@8": 0.41995692749461594, + "pass@16": 0.46733668341708545, + "pass@32": 0.505743000717875, + "pass@64": 0.5405599425699928, + "pass@128": 0.5732232591529074 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6114457831325302, + "pass@2": 0.6837349397590361, + "pass@4": 0.7590361445783133, + "pass@8": 0.8343373493975904, + "pass@16": 0.9126506024096386, + "pass@32": 0.9487951807228916, + "pass@64": 0.9578313253012049, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.5141065830721003, + "pass@2": 0.5799373040752351, + "pass@4": 0.6677115987460815, + "pass@8": 0.7554858934169278, + "pass@16": 0.8119122257053292, + "pass@32": 0.8369905956112853, + "pass@64": 0.8589341692789969, + "pass@128": 0.8714733542319749 + }, + "13": { + "pass@1": 0.4173669467787115, + "pass@2": 0.5014005602240896, + "pass@4": 0.5714285714285714, + "pass@8": 0.6358543417366946, + "pass@16": 0.7030812324929971, + "pass@32": 0.7507002801120448, + "pass@64": 0.7899159663865546, + "pass@128": 0.8179271708683473 + }, + "14": { + "pass@1": 0.30701754385964913, + "pass@2": 0.3391812865497076, + "pass@4": 0.40350877192982454, + "pass@8": 0.4824561403508772, + "pass@16": 0.5555555555555556, + "pass@32": 0.6169590643274854, + "pass@64": 0.652046783625731, + "pass@128": 0.695906432748538 + }, + "15": { + "pass@1": 0.24242424242424243, + "pass@2": 0.2909090909090909, + "pass@4": 0.3303030303030303, + "pass@8": 0.37272727272727274, + "pass@16": 0.41515151515151516, + "pass@32": 0.48787878787878786, + "pass@64": 0.5424242424242425, + "pass@128": 0.6151515151515151 + }, + "16": { + "pass@1": 0.10826210826210826, + "pass@2": 0.1623931623931624, + "pass@4": 0.19658119658119658, + "pass@8": 0.23646723646723647, + "pass@16": 0.28205128205128205, + "pass@32": 0.31054131054131057, + "pass@64": 0.3732193732193732, + "pass@128": 0.41595441595441596 + }, + "17": { + "pass@1": 0.07453416149068323, + "pass@2": 0.08695652173913043, + "pass@4": 0.09627329192546584, + "pass@8": 0.12422360248447205, + "pass@16": 0.15527950310559005, + "pass@32": 0.17701863354037267, + "pass@64": 0.2204968944099379, + "pass@128": 0.2639751552795031 + }, + "18": { + "pass@1": 0.03529411764705882, + "pass@2": 0.05, + "pass@4": 0.07058823529411765, + "pass@8": 0.08235294117647059, + "pass@16": 0.10882352941176471, + "pass@32": 0.11764705882352941, + "pass@64": 0.15294117647058825, + "pass@128": 0.19705882352941176 + }, + "19": { + "pass@1": 0.027777777777777776, + "pass@2": 0.03333333333333333, + "pass@4": 0.044444444444444446, + "pass@8": 0.05555555555555555, + "pass@16": 0.07222222222222222, + "pass@32": 0.08333333333333333, + "pass@64": 0.1, + "pass@128": 0.11666666666666667 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6114457831325302, + "12": 0.5141065830721003, + "13": 0.4173669467787115, + "14": 0.30701754385964913, + "15": 0.24242424242424243, + "16": 0.10826210826210826, + "17": 0.07453416149068323, + "18": 0.03529411764705882, + "19": 0.027777777777777776 + }, + "pass@2": { + "11": 0.6837349397590361, + "12": 0.5799373040752351, + "13": 0.5014005602240896, + "14": 0.3391812865497076, + "15": 0.2909090909090909, + "16": 0.1623931623931624, + "17": 0.08695652173913043, + "18": 0.05, + "19": 0.03333333333333333 + }, + "pass@4": { + "11": 0.7590361445783133, + "12": 0.6677115987460815, + "13": 0.5714285714285714, + "14": 0.40350877192982454, + "15": 0.3303030303030303, + "16": 0.19658119658119658, + "17": 0.09627329192546584, + "18": 0.07058823529411765, + "19": 0.044444444444444446 + }, + "pass@8": { + "11": 0.8343373493975904, + "12": 0.7554858934169278, + "13": 0.6358543417366946, + "14": 0.4824561403508772, + "15": 0.37272727272727274, + "16": 0.23646723646723647, + "17": 0.12422360248447205, + "18": 0.08235294117647059, + "19": 0.05555555555555555 + }, + "pass@16": { + "11": 0.9126506024096386, + "12": 0.8119122257053292, + "13": 0.7030812324929971, + "14": 0.5555555555555556, + "15": 0.41515151515151516, + "16": 0.28205128205128205, + "17": 0.15527950310559005, + "18": 0.10882352941176471, + "19": 0.07222222222222222 + }, + "pass@32": { + "11": 0.9487951807228916, + "12": 0.8369905956112853, + "13": 0.7507002801120448, + "14": 0.6169590643274854, + "15": 0.48787878787878786, + "16": 0.31054131054131057, + "17": 0.17701863354037267, + "18": 0.11764705882352941, + "19": 0.08333333333333333 + }, + "pass@64": { + "11": 0.9578313253012049, + "12": 0.8589341692789969, + "13": 0.7899159663865546, + "14": 0.652046783625731, + "15": 0.5424242424242425, + "16": 0.3732193732193732, + "17": 0.2204968944099379, + "18": 0.15294117647058825, + "19": 0.1 + }, + "pass@128": { + "11": 0.9698795180722891, + "12": 0.8714733542319749, + "13": 0.8179271708683473, + "14": 0.695906432748538, + "15": 0.6151515151515151, + "16": 0.41595441595441596, + "17": 0.2639751552795031, + "18": 0.19705882352941176, + "19": 0.11666666666666667 + } + }, + "total_pass_at_k": { + "pass@1": 0.27149321266968324, + "pass@2": 0.3170901496693352, + "pass@4": 0.3647754959972154, + "pass@8": 0.4155934563174382, + "pass@16": 0.4660633484162896, + "pass@32": 0.502262443438914, + "pass@64": 0.5388096066829099, + "pass@128": 0.5750087017055343 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.558, + "pass@2": 0.657, + "pass@4": 0.734, + "pass@8": 0.81, + "pass@16": 0.89, + "pass@32": 0.925, + "pass@64": 0.939, + "pass@128": 0.95 + }, + "12": { + "pass@1": 0.547, + "pass@2": 0.607, + "pass@4": 0.69, + "pass@8": 0.77, + "pass@16": 0.822, + "pass@32": 0.848, + "pass@64": 0.867, + "pass@128": 0.878 + }, + "13": { + "pass@1": 0.418, + "pass@2": 0.497, + "pass@4": 0.564, + "pass@8": 0.635, + "pass@16": 0.703, + "pass@32": 0.739, + "pass@64": 0.77, + "pass@128": 0.795 + }, + "14": { + "pass@1": 0.307, + "pass@2": 0.345, + "pass@4": 0.401, + "pass@8": 0.46, + "pass@16": 0.53, + "pass@32": 0.6, + "pass@64": 0.654, + "pass@128": 0.702 + }, + "15": { + "pass@1": 0.23, + "pass@2": 0.269, + "pass@4": 0.318, + "pass@8": 0.359, + "pass@16": 0.409, + "pass@32": 0.482, + "pass@64": 0.542, + "pass@128": 0.613 + }, + "16": { + "pass@1": 0.122, + "pass@2": 0.169, + "pass@4": 0.206, + "pass@8": 0.244, + "pass@16": 0.283, + "pass@32": 0.326, + "pass@64": 0.379, + "pass@128": 0.421 + }, + "17": { + "pass@1": 0.075, + "pass@2": 0.093, + "pass@4": 0.112, + "pass@8": 0.134, + "pass@16": 0.159, + "pass@32": 0.185, + "pass@64": 0.228, + "pass@128": 0.266 + }, + "18": { + "pass@1": 0.046, + "pass@2": 0.061, + "pass@4": 0.081, + "pass@8": 0.091, + "pass@16": 0.111, + "pass@32": 0.122, + "pass@64": 0.15, + "pass@128": 0.181 + }, + "19": { + "pass@1": 0.020599250936329586, + "pass@2": 0.035580524344569285, + "pass@4": 0.0449438202247191, + "pass@8": 0.05805243445692884, + "pass@16": 0.0749063670411985, + "pass@32": 0.09925093632958802, + "pass@64": 0.10861423220973783, + "pass@128": 0.11610486891385768 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.558, + "12": 0.547, + "13": 0.418, + "14": 0.307, + "15": 0.23, + "16": 0.122, + "17": 0.075, + "18": 0.046, + "19": 0.020599250936329586 + }, + "pass@2": { + "11": 0.657, + "12": 0.607, + "13": 0.497, + "14": 0.345, + "15": 0.269, + "16": 0.169, + "17": 0.093, + "18": 0.061, + "19": 0.035580524344569285 + }, + "pass@4": { + "11": 0.734, + "12": 0.69, + "13": 0.564, + "14": 0.401, + "15": 0.318, + "16": 0.206, + "17": 0.112, + "18": 0.081, + "19": 0.0449438202247191 + }, + "pass@8": { + "11": 0.81, + "12": 0.77, + "13": 0.635, + "14": 0.46, + "15": 0.359, + "16": 0.244, + "17": 0.134, + "18": 0.091, + "19": 0.05805243445692884 + }, + "pass@16": { + "11": 0.89, + "12": 0.822, + "13": 0.703, + "14": 0.53, + "15": 0.409, + "16": 0.283, + "17": 0.159, + "18": 0.111, + "19": 0.0749063670411985 + }, + "pass@32": { + "11": 0.925, + "12": 0.848, + "13": 0.739, + "14": 0.6, + "15": 0.482, + "16": 0.326, + "17": 0.185, + "18": 0.122, + "19": 0.09925093632958802 + }, + "pass@64": { + "11": 0.939, + "12": 0.867, + "13": 0.77, + "14": 0.654, + "15": 0.542, + "16": 0.379, + "17": 0.228, + "18": 0.15, + "19": 0.10861423220973783 + }, + "pass@128": { + "11": 0.95, + "12": 0.878, + "13": 0.795, + "14": 0.702, + "15": 0.613, + "16": 0.421, + "17": 0.266, + "18": 0.181, + "19": 0.11610486891385768 + } + }, + "total_pass_at_k": { + "pass@1": 0.2711506913522381, + "pass@2": 0.31837356456526833, + "pass@4": 0.3667682212327162, + "pass@8": 0.41410827279118817, + "pass@16": 0.462502929458636, + "pass@32": 0.5015233184907429, + "pass@64": 0.537497070541364, + "pass@128": 0.5704241856104992 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 176 + }, + "total": 2875 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 178 + }, + "total": 2786 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 180 + }, + "total": 2873 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 534 + }, + "total": 8534 + } + } + }, + "SDAR-100M-block8-epoch1-op9-12_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969418960244648, + "pass@2": 0.9969418960244648, + "pass@4": 0.9969418960244648, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.975, + "pass@2": 0.978125, + "pass@4": 0.984375, + "pass@8": 0.984375, + "pass@16": 0.9875, + "pass@32": 0.9875, + "pass@64": 0.99375, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.9206349206349206, + "pass@2": 0.9396825396825397, + "pass@4": 0.9682539682539683, + "pass@8": 0.9714285714285714, + "pass@16": 0.9777777777777777, + "pass@32": 0.9841269841269841, + "pass@64": 0.9904761904761905, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9580838323353293, + "pass@2": 0.9640718562874252, + "pass@4": 0.9730538922155688, + "pass@8": 0.9880239520958084, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8757396449704142, + "pass@2": 0.9230769230769231, + "pass@4": 0.9644970414201184, + "pass@8": 0.985207100591716, + "pass@16": 0.9911242603550295, + "pass@32": 0.9940828402366864, + "pass@64": 0.9940828402366864, + "pass@128": 0.9940828402366864 + }, + "8": { + "pass@1": 0.7980769230769231, + "pass@2": 0.8525641025641025, + "pass@4": 0.9391025641025641, + "pass@8": 0.9839743589743589, + "pass@16": 0.9935897435897436, + "pass@32": 0.9967948717948718, + "pass@64": 0.9967948717948718, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7070422535211267, + "pass@2": 0.8, + "pass@4": 0.8985915492957747, + "pass@8": 0.9436619718309859, + "pass@16": 0.9802816901408451, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6489675516224189, + "pass@2": 0.7315634218289085, + "pass@4": 0.8053097345132744, + "pass@8": 0.8584070796460177, + "pass@16": 0.9026548672566371, + "pass@32": 0.9292035398230089, + "pass@64": 0.9321533923303835, + "pass@128": 0.9410029498525073 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 0.975, + "5": 0.9206349206349206, + "6": 0.9580838323353293, + "7": 0.8757396449704142, + "8": 0.7980769230769231, + "9": 0.7070422535211267, + "10": 0.6489675516224189 + }, + "pass@2": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 0.978125, + "5": 0.9396825396825397, + "6": 0.9640718562874252, + "7": 0.9230769230769231, + "8": 0.8525641025641025, + "9": 0.8, + "10": 0.7315634218289085 + }, + "pass@4": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 0.984375, + "5": 0.9682539682539683, + "6": 0.9730538922155688, + "7": 0.9644970414201184, + "8": 0.9391025641025641, + "9": 0.8985915492957747, + "10": 0.8053097345132744 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.984375, + "5": 0.9714285714285714, + "6": 0.9880239520958084, + "7": 0.985207100591716, + "8": 0.9839743589743589, + "9": 0.9436619718309859, + "10": 0.8584070796460177 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.9875, + "5": 0.9777777777777777, + "6": 1.0, + "7": 0.9911242603550295, + "8": 0.9935897435897436, + "9": 0.9802816901408451, + "10": 0.9026548672566371 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.9875, + "5": 0.9841269841269841, + "6": 1.0, + "7": 0.9940828402366864, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9292035398230089 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9904761904761905, + "6": 1.0, + "7": 0.9940828402366864, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9321533923303835 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9940828402366864, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9410029498525073 + } + }, + "total_pass_at_k": { + "pass@1": 0.873015873015873, + "pass@2": 0.9078014184397163, + "pass@4": 0.9466396487673083, + "pass@8": 0.9675785207700102, + "pass@16": 0.9810874704491725, + "pass@32": 0.9878419452887538, + "pass@64": 0.9895305639986491, + "pass@128": 0.9912191827085444 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970845481049563, + "pass@2": 0.9970845481049563, + "pass@4": 0.9970845481049563, + "pass@8": 0.9970845481049563, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9941520467836257, + "pass@2": 0.9970760233918129, + "pass@4": 0.9970760233918129, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9382352941176471, + "pass@2": 0.9558823529411765, + "pass@4": 0.9764705882352941, + "pass@8": 0.9852941176470589, + "pass@16": 0.9852941176470589, + "pass@32": 0.9882352941176471, + "pass@64": 0.9911764705882353, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9716088328075709, + "pass@2": 0.9905362776025236, + "pass@4": 0.9936908517350158, + "pass@8": 0.9936908517350158, + "pass@16": 0.9968454258675079, + "pass@32": 0.9968454258675079, + "pass@64": 0.9968454258675079, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8544303797468354, + "pass@2": 0.8860759493670886, + "pass@4": 0.930379746835443, + "pass@8": 0.9810126582278481, + "pass@16": 0.9873417721518988, + "pass@32": 0.9936708860759493, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.7867867867867868, + "pass@2": 0.8798798798798799, + "pass@4": 0.9429429429429429, + "pass@8": 0.96996996996997, + "pass@16": 0.984984984984985, + "pass@32": 0.993993993993994, + "pass@64": 0.993993993993994, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.752411575562701, + "pass@2": 0.8167202572347267, + "pass@4": 0.9003215434083601, + "pass@8": 0.9453376205787781, + "pass@16": 0.9807073954983923, + "pass@32": 0.9903536977491961, + "pass@64": 0.9935691318327974, + "pass@128": 0.9967845659163987 + }, + "10": { + "pass@1": 0.6298507462686567, + "pass@2": 0.6865671641791045, + "pass@4": 0.7731343283582089, + "pass@8": 0.8417910447761194, + "pass@16": 0.8776119402985074, + "pass@32": 0.9164179104477612, + "pass@64": 0.9313432835820895, + "pass@128": 0.9462686567164179 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9941520467836257, + "5": 0.9382352941176471, + "6": 0.9716088328075709, + "7": 0.8544303797468354, + "8": 0.7867867867867868, + "9": 0.752411575562701, + "10": 0.6298507462686567 + }, + "pass@2": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9970760233918129, + "5": 0.9558823529411765, + "6": 0.9905362776025236, + "7": 0.8860759493670886, + "8": 0.8798798798798799, + "9": 0.8167202572347267, + "10": 0.6865671641791045 + }, + "pass@4": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9970760233918129, + "5": 0.9764705882352941, + "6": 0.9936908517350158, + "7": 0.930379746835443, + "8": 0.9429429429429429, + "9": 0.9003215434083601, + "10": 0.7731343283582089 + }, + "pass@8": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9970760233918129, + "5": 0.9852941176470589, + "6": 0.9936908517350158, + "7": 0.9810126582278481, + "8": 0.96996996996997, + "9": 0.9453376205787781, + "10": 0.8417910447761194 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9852941176470589, + "6": 0.9968454258675079, + "7": 0.9873417721518988, + "8": 0.984984984984985, + "9": 0.9807073954983923, + "10": 0.8776119402985074 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9882352941176471, + "6": 0.9968454258675079, + "7": 0.9936708860759493, + "8": 0.993993993993994, + "9": 0.9903536977491961, + "10": 0.9164179104477612 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9911764705882353, + "6": 0.9968454258675079, + "7": 0.9936708860759493, + "8": 0.993993993993994, + "9": 0.9935691318327974, + "10": 0.9313432835820895 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9941176470588236, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.993993993993994, + "9": 0.9967845659163987, + "10": 0.9462686567164179 + } + }, + "total_pass_at_k": { + "pass@1": 0.8824907934382323, + "pass@2": 0.9136257114161366, + "pass@4": 0.9464345497154335, + "pass@8": 0.9681955138935386, + "pass@16": 0.978908603950452, + "pass@32": 0.9862738533645798, + "pass@64": 0.9886173418145296, + "pass@128": 0.9912956143287579 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996969696969697, + "pass@2": 0.996969696969697, + "pass@4": 0.996969696969697, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9763313609467456, + "pass@2": 0.9881656804733728, + "pass@4": 0.9970414201183432, + "pass@8": 0.9970414201183432, + "pass@16": 0.9970414201183432, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9565217391304348, + "pass@2": 0.9652173913043478, + "pass@4": 0.9826086956521739, + "pass@8": 0.991304347826087, + "pass@16": 0.9942028985507246, + "pass@32": 0.9942028985507246, + "pass@64": 0.9971014492753624, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9770773638968482, + "pass@2": 0.9828080229226361, + "pass@4": 0.994269340974212, + "pass@8": 0.997134670487106, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.8641618497109826, + "pass@2": 0.9219653179190751, + "pass@4": 0.9595375722543352, + "pass@8": 0.9739884393063584, + "pass@16": 0.9884393063583815, + "pass@32": 0.9942196531791907, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8084507042253521, + "pass@2": 0.8901408450704226, + "pass@4": 0.9492957746478873, + "pass@8": 0.9774647887323944, + "pass@16": 0.9915492957746479, + "pass@32": 0.9943661971830986, + "pass@64": 0.9943661971830986, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.7215568862275449, + "pass@2": 0.781437125748503, + "pass@4": 0.8592814371257484, + "pass@8": 0.9251497005988024, + "pass@16": 0.9730538922155688, + "pass@32": 0.9880239520958084, + "pass@64": 0.9880239520958084, + "pass@128": 0.9880239520958084 + }, + "10": { + "pass@1": 0.647239263803681, + "pass@2": 0.7085889570552147, + "pass@4": 0.7822085889570553, + "pass@8": 0.8374233128834356, + "pass@16": 0.8834355828220859, + "pass@32": 0.901840490797546, + "pass@64": 0.9294478527607362, + "pass@128": 0.941717791411043 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9763313609467456, + "5": 0.9565217391304348, + "6": 0.9770773638968482, + "7": 0.8641618497109826, + "8": 0.8084507042253521, + "9": 0.7215568862275449, + "10": 0.647239263803681 + }, + "pass@2": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9881656804733728, + "5": 0.9652173913043478, + "6": 0.9828080229226361, + "7": 0.9219653179190751, + "8": 0.8901408450704226, + "9": 0.781437125748503, + "10": 0.7085889570552147 + }, + "pass@4": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9970414201183432, + "5": 0.9826086956521739, + "6": 0.994269340974212, + "7": 0.9595375722543352, + "8": 0.9492957746478873, + "9": 0.8592814371257484, + "10": 0.7822085889570553 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.991304347826087, + "6": 0.997134670487106, + "7": 0.9739884393063584, + "8": 0.9774647887323944, + "9": 0.9251497005988024, + "10": 0.8374233128834356 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.9942028985507246, + "6": 0.997134670487106, + "7": 0.9884393063583815, + "8": 0.9915492957746479, + "9": 0.9730538922155688, + "10": 0.8834355828220859 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9942028985507246, + "6": 0.997134670487106, + "7": 0.9942196531791907, + "8": 0.9943661971830986, + "9": 0.9880239520958084, + "10": 0.901840490797546 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.997134670487106, + "7": 1.0, + "8": 0.9943661971830986, + "9": 0.9880239520958084, + "10": 0.9294478527607362 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.997134670487106, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9880239520958084, + "10": 0.941717791411043 + } + }, + "total_pass_at_k": { + "pass@1": 0.8836828309305373, + "pass@2": 0.9157929226736566, + "pass@4": 0.9475753604193972, + "pass@8": 0.9672346002621232, + "pass@16": 0.9809960681520314, + "pass@32": 0.9859108781127129, + "pass@64": 0.9898427260812582, + "pass@128": 0.991480996068152 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.997, + "pass@2": 0.997, + "pass@4": 0.997, + "pass@8": 0.999, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.982, + "pass@2": 0.988, + "pass@4": 0.993, + "pass@8": 0.993, + "pass@16": 0.994, + "pass@32": 0.995, + "pass@64": 0.997, + "pass@128": 0.997 + }, + "5": { + "pass@1": 0.939, + "pass@2": 0.954, + "pass@4": 0.976, + "pass@8": 0.983, + "pass@16": 0.986, + "pass@32": 0.989, + "pass@64": 0.993, + "pass@128": 0.996 + }, + "6": { + "pass@1": 0.969, + "pass@2": 0.979, + "pass@4": 0.987, + "pass@8": 0.993, + "pass@16": 0.998, + "pass@32": 0.998, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.865, + "pass@2": 0.911, + "pass@4": 0.952, + "pass@8": 0.98, + "pass@16": 0.989, + "pass@32": 0.994, + "pass@64": 0.996, + "pass@128": 0.996 + }, + "8": { + "pass@1": 0.798, + "pass@2": 0.875, + "pass@4": 0.944, + "pass@8": 0.977, + "pass@16": 0.99, + "pass@32": 0.995, + "pass@64": 0.995, + "pass@128": 0.996 + }, + "9": { + "pass@1": 0.726, + "pass@2": 0.799, + "pass@4": 0.886, + "pass@8": 0.938, + "pass@16": 0.978, + "pass@32": 0.993, + "pass@64": 0.994, + "pass@128": 0.995 + }, + "10": { + "pass@1": 0.642, + "pass@2": 0.709, + "pass@4": 0.787, + "pass@8": 0.846, + "pass@16": 0.888, + "pass@32": 0.916, + "pass@64": 0.931, + "pass@128": 0.943 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.997, + "4": 0.982, + "5": 0.939, + "6": 0.969, + "7": 0.865, + "8": 0.798, + "9": 0.726, + "10": 0.642 + }, + "pass@2": { + "2": 1.0, + "3": 0.997, + "4": 0.988, + "5": 0.954, + "6": 0.979, + "7": 0.911, + "8": 0.875, + "9": 0.799, + "10": 0.709 + }, + "pass@4": { + "2": 1.0, + "3": 0.997, + "4": 0.993, + "5": 0.976, + "6": 0.987, + "7": 0.952, + "8": 0.944, + "9": 0.886, + "10": 0.787 + }, + "pass@8": { + "2": 1.0, + "3": 0.999, + "4": 0.993, + "5": 0.983, + "6": 0.993, + "7": 0.98, + "8": 0.977, + "9": 0.938, + "10": 0.846 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.994, + "5": 0.986, + "6": 0.998, + "7": 0.989, + "8": 0.99, + "9": 0.978, + "10": 0.888 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.995, + "5": 0.989, + "6": 0.998, + "7": 0.994, + "8": 0.995, + "9": 0.993, + "10": 0.916 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.997, + "5": 0.993, + "6": 0.998, + "7": 0.996, + "8": 0.995, + "9": 0.994, + "10": 0.931 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.997, + "5": 0.996, + "6": 0.999, + "7": 0.996, + "8": 0.996, + "9": 0.995, + "10": 0.943 + } + }, + "total_pass_at_k": { + "pass@1": 0.8797777777777778, + "pass@2": 0.9124444444444444, + "pass@4": 0.9468888888888889, + "pass@8": 0.9676666666666667, + "pass@16": 0.9803333333333333, + "pass@32": 0.9866666666666667, + "pass@64": 0.9893333333333333, + "pass@128": 0.9913333333333333 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block8-epoch1-op9-12/summary.csv b/composition/SDAR-100M-block8-epoch1-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..a8822f850af86a576410ced505225caee7f0bb43 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch1-op9-12,0.9833871527777778,0.51230234375,0.7354477796052632,0.0,0.0,0.0,198.9335685763889,344.525071875,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block8-epoch1-op9-12/summary.json b/composition/SDAR-100M-block8-epoch1-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d97409208341e1b5a5451ef891bbdb237cd8bc01 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch1-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch1-op9-12", + "id_acc": 0.9833871527777778, + "ood_acc": 0.51230234375, + "total_acc": 0.7354477796052632, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 198.9335685763889, + "ood_avg_resp_len": 344.525071875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 593672, + "answer_accuracy": 0.7331745969016756, + "avg_response_len": 262.80997199059436, + "resp_tokens_sum": 212804593, + "pass_at_k": { + "pass@1": 0.7331745969016756, + "pass@2": 0.7706344784429154, + "pass@4": 0.8002934745293199, + "pass@8": 0.8244781684255242, + "pass@16": 0.8451241664967811, + "pass@32": 0.8631318988944838, + "pass@64": 0.8783587068525153, + "pass@128": 0.8912424913057224 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9386292404129793, + "pass@2": 0.9558454840545374, + "pass@4": 0.9661186383781674, + "pass@8": 0.9715414375472532, + "pass@16": 0.9756175864300959, + "pass@32": 0.9796003458610456, + "pass@64": 0.9837013134257441, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977780963302753, + "pass@2": 0.9989322220616917, + "pass@4": 0.9996694776685024, + "pass@8": 0.9999677623799305, + "pass@16": 0.999999779641568, + "pass@32": 0.9999999999981484, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9837158203125, + "pass@2": 0.9857337290846455, + "pass@4": 0.9882842340996436, + "pass@8": 0.9906827001964222, + "pass@16": 0.9922391181843271, + "pass@32": 0.9935574745502678, + "pass@64": 0.9951263779508654, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9695188492063492, + "pass@2": 0.976019013248344, + "pass@4": 0.9805692556287607, + "pass@8": 0.9843784633909355, + "pass@16": 0.9889107918312874, + "pass@32": 0.9940484172603424, + "pass@64": 0.9981441113782779, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9948774326347305, + "pass@2": 0.9966545747088502, + "pass@4": 0.9974962398038569, + "pass@8": 0.9981117754532004, + "pass@16": 0.9988525144880754, + "pass@32": 0.9996224712529058, + "pass@64": 0.9999803235855924, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9937361316568047, + "pass@2": 0.9963439989749802, + "pass@4": 0.9969306777666104, + "pass@8": 0.9970380005155915, + "pass@16": 0.9970414182830878, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9876302083333334, + "pass@2": 0.9916015230668282, + "pass@4": 0.9930404943372461, + "pass@8": 0.9940772665714049, + "pass@16": 0.9949131147821092, + "pass@32": 0.9957967829765609, + "pass@64": 0.9966039773874414, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9811399647887324, + "pass@2": 0.9891198569368971, + "pass@4": 0.9927404183631975, + "pass@8": 0.9952636734053902, + "pass@16": 0.9975775655877893, + "pass@32": 0.999379044270334, + "pass@64": 0.9999770279396809, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9211225212464589, + "pass@2": 0.9423424918025475, + "pass@4": 0.953929790822889, + "pass@8": 0.9614565186188683, + "pass@16": 0.9677885340638542, + "pass@32": 0.9743190927138413, + "pass@64": 0.9805943394845731, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8815986935028248, + "pass@2": 0.9165651830597449, + "pass@4": 0.934161171855637, + "pass@8": 0.9452938352373702, + "pass@16": 0.9539612829997371, + "pass@32": 0.9615647682922526, + "pass@64": 0.9668943055418103, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.7734619905956113, + "pass@2": 0.8301158732999284, + "pass@4": 0.8655036610823328, + "pass@8": 0.8898255448648787, + "pass@16": 0.9097976575302069, + "pass@32": 0.92890026616942, + "pass@64": 0.9464289475637889, + "pass@128": 0.9623824451410659 + }, + "14": { + "pass@1": 0.6513093847352025, + "pass@2": 0.7309117208771797, + "pass@4": 0.7865110567370666, + "pass@8": 0.8270123821269293, + "pass@16": 0.8587307099359938, + "pass@32": 0.8847289386929739, + "pass@64": 0.9048481484220781, + "pass@128": 0.9221183800623053 + }, + "15": { + "pass@1": 0.5636741329479769, + "pass@2": 0.6490562133039006, + "pass@4": 0.7107771526752794, + "pass@8": 0.7595685140977119, + "pass@16": 0.8003622070803227, + "pass@32": 0.8331029501088946, + "pass@64": 0.8575529239777137, + "pass@128": 0.8757225433526011 + }, + "16": { + "pass@1": 0.4171118951612903, + "pass@2": 0.5142795910591821, + "pass@4": 0.5949670703460455, + "pass@8": 0.6603091766742203, + "pass@16": 0.7134515965927327, + "pass@32": 0.75739935199652, + "pass@64": 0.7984302135889665, + "pass@128": 0.8419354838709677 + }, + "17": { + "pass@1": 0.31419346098265893, + "pass@2": 0.3958058349642712, + "pass@4": 0.46747656868035986, + "pass@8": 0.5300846478128985, + "pass@16": 0.585240494400111, + "pass@32": 0.6313648084038579, + "pass@64": 0.669614375035597, + "pass@128": 0.6994219653179191 + }, + "18": { + "pass@1": 0.22939732142857142, + "pass@2": 0.3074957817772778, + "pass@4": 0.38013197760994144, + "pass@8": 0.44242261962567625, + "pass@16": 0.49595147536216044, + "pass@32": 0.5439121161703538, + "pass@64": 0.5859423405040576, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.18125938438438438, + "pass@2": 0.24487775176751556, + "pass@4": 0.3061006349431548, + "pass@8": 0.36073586559610316, + "pass@16": 0.40994189428993016, + "pass@32": 0.45423626681010243, + "pass@64": 0.4932719107305281, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.18332394894894896, + "pass@2": 0.25119817159679353, + "pass@4": 0.3202794138345321, + "pass@8": 0.3847688536744722, + "pass@16": 0.4427514993287269, + "pass@32": 0.4952158151961868, + "pass@64": 0.5381104843576482, + "pass@128": 0.5675675675675675 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 604538, + "answer_accuracy": 0.736235872954014, + "avg_response_len": 271.6522250097428, + "resp_tokens_sum": 223059075, + "pass_at_k": { + "pass@1": 0.736235872954014, + "pass@2": 0.7735831144401895, + "pass@4": 0.8036381645152837, + "pass@8": 0.8281069190722744, + "pass@16": 0.8483726997042081, + "pass@32": 0.8655832133804009, + "pass@64": 0.8805682987802501, + "pass@128": 0.8933749025720966 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9312691717791411, + "pass@2": 0.9492200708902958, + "pass@4": 0.9614929713540408, + "pass@8": 0.9706849500303524, + "pass@16": 0.9780743583738936, + "pass@32": 0.9833452492500756, + "pass@64": 0.9865077334207035, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9999050151975684, + "pass@2": 0.9999977562645096, + "pass@4": 0.9999999997150811, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9961647727272728, + "pass@2": 0.9978696910045335, + "pass@4": 0.9990399083637271, + "pass@8": 0.9997415688663096, + "pass@16": 0.9999820837105999, + "pass@32": 0.9999999593969077, + "pass@64": 0.9999999999999987, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9963480029585798, + "pass@2": 0.9970392361272888, + "pass@4": 0.9973065910215366, + "pass@8": 0.9975660568709271, + "pass@16": 0.9980247014241554, + "pass@32": 0.9987617117386955, + "pass@64": 0.9996389134790105, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9867753623188406, + "pass@2": 0.9882117996120052, + "pass@4": 0.9893897974709683, + "pass@8": 0.9906458350692433, + "pass@16": 0.9923677751658371, + "pass@32": 0.9946202004072046, + "pass@64": 0.996556005154711, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9901504297994269, + "pass@2": 0.9921238690973083, + "pass@4": 0.9931829475828413, + "pass@8": 0.9941730137747032, + "pass@16": 0.9951084353395996, + "pass@32": 0.9959328579290595, + "pass@64": 0.9967849648397139, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9909230491329479, + "pass@2": 0.9937133039005962, + "pass@4": 0.9946745414951744, + "pass@8": 0.9952348105100473, + "pass@16": 0.9961070653038383, + "pass@32": 0.9974785593997282, + "pass@64": 0.9991110099676848, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9840008802816902, + "pass@2": 0.9890027170899413, + "pass@4": 0.9920841439714402, + "pass@8": 0.9944509310383022, + "pass@16": 0.9961687806906195, + "pass@32": 0.9970422143124665, + "pass@64": 0.9971821603034191, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.9723755613772455, + "pass@2": 0.977356311000047, + "pass@4": 0.9796722011544962, + "pass@8": 0.9821828386883638, + "pass@16": 0.9851973646020666, + "pass@32": 0.988117457332326, + "pass@64": 0.9901673949748785, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.928699171686747, + "pass@2": 0.9515493074660848, + "pass@4": 0.9659594759804117, + "pass@8": 0.9766550766070073, + "pass@16": 0.9842963536346357, + "pass@32": 0.9884709338432213, + "pass@64": 0.9904158337215808, + "pass@128": 0.9909638554216867 + }, + "12": { + "pass@1": 0.8799960815047022, + "pass@2": 0.9035926739565081, + "pass@4": 0.9153743348695834, + "pass@8": 0.9229959569636945, + "pass@16": 0.9296964586106998, + "pass@32": 0.9365110569119893, + "pass@64": 0.94190277023684, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.7331276260504201, + "pass@2": 0.7954751290279893, + "pass@4": 0.8377238864049549, + "pass@8": 0.8699515828594602, + "pass@16": 0.8957292705806627, + "pass@32": 0.9165216166375357, + "pass@64": 0.9327622224638712, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6675804093567251, + "pass@2": 0.7419245291706962, + "pass@4": 0.796447699794104, + "pass@8": 0.8363445657414356, + "pass@16": 0.8626242889188614, + "pass@32": 0.8793508082194177, + "pass@64": 0.890769595219482, + "pass@128": 0.8976608187134503 + }, + "15": { + "pass@1": 0.5677320075757576, + "pass@2": 0.6639912908613691, + "pass@4": 0.7317696557816629, + "pass@8": 0.7795323520517282, + "pass@16": 0.8155345673543264, + "pass@32": 0.845208030617802, + "pass@64": 0.8714240604997285, + "pass@128": 0.896969696969697 + }, + "16": { + "pass@1": 0.4091880341880342, + "pass@2": 0.501739616281939, + "pass@4": 0.5781482055661846, + "pass@8": 0.6406309064652197, + "pass@16": 0.6934298295552146, + "pass@32": 0.7373148005034644, + "pass@64": 0.7740434816672218, + "pass@128": 0.8091168091168092 + }, + "17": { + "pass@1": 0.3116022903726708, + "pass@2": 0.39830262385680026, + "pass@4": 0.47726986174476643, + "pass@8": 0.5445244698438653, + "pass@16": 0.5995497047481053, + "pass@32": 0.6445440362273804, + "pass@64": 0.6840008853131072, + "pass@128": 0.717391304347826 + }, + "18": { + "pass@1": 0.2716911764705882, + "pass@2": 0.35394171201945335, + "pass@4": 0.4276034548071196, + "pass@8": 0.4888071795579154, + "pass@16": 0.5390270190145257, + "pass@32": 0.5858273452671591, + "pass@64": 0.6344610738799759, + "pass@128": 0.6794117647058824 + }, + "19": { + "pass@1": 0.17631603422619047, + "pass@2": 0.24067853334739409, + "pass@4": 0.3043591830597069, + "pass@8": 0.36333850531952205, + "pass@16": 0.41840949299425007, + "pass@32": 0.4696348856330499, + "pass@64": 0.5155865519395338, + "pass@128": 0.5595238095238095 + }, + "20": { + "pass@1": 0.1794068113772455, + "pass@2": 0.24754415872978455, + "pass@4": 0.31347590112238966, + "pass@8": 0.37303300737954526, + "pass@16": 0.4267697863227694, + "pass@32": 0.4749624791026021, + "pass@64": 0.5176129128175474, + "pass@128": 0.5538922155688623 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 590399, + "answer_accuracy": 0.7369375599137242, + "avg_response_len": 292.45373537106565, + "resp_tokens_sum": 234299895, + "pass_at_k": { + "pass@1": 0.7369375599137242, + "pass@2": 0.7739902768359941, + "pass@4": 0.8043156387535312, + "pass@8": 0.8298973518508928, + "pass@16": 0.8517613092704859, + "pass@32": 0.8702293942304131, + "pass@64": 0.8855494159569103, + "pass@128": 0.8985460936251798 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9383628731343283, + "pass@2": 0.9546259842519684, + "pass@4": 0.9663195143517505, + "pass@8": 0.9744687547436317, + "pass@16": 0.9804980217307362, + "pass@32": 0.9858787590967183, + "pass@64": 0.9903773541820934, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9984375, + "pass@2": 0.9992217379077615, + "pass@4": 0.9997947185173283, + "pass@8": 0.999986965598258, + "pass@16": 0.9999999656019002, + "pass@32": 0.999999999999974, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970617711370262, + "pass@2": 0.9981039519294782, + "pass@4": 0.9989637462409036, + "pass@8": 0.9996508749963928, + "pass@16": 0.9999646895378788, + "pass@32": 0.9999998081458937, + "pass@64": 0.9999999999999133, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9968475877192983, + "pass@2": 0.997070267532348, + "pass@4": 0.9970760217472814, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9792279411764706, + "pass@2": 0.9846865591709126, + "pass@4": 0.9878091838703983, + "pass@8": 0.9892869660212149, + "pass@16": 0.9903738755975673, + "pass@32": 0.9917880624629651, + "pass@64": 0.9933033811949977, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9955392350157729, + "pass@2": 0.9967716845922651, + "pass@4": 0.9971258053508296, + "pass@8": 0.9974048116766691, + "pass@16": 0.997893845682534, + "pass@32": 0.9986796800242242, + "pass@64": 0.9996149929208377, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9873664952531646, + "pass@2": 0.9905549966361007, + "pass@4": 0.9922436103714881, + "pass@8": 0.9935020050533728, + "pass@16": 0.9945756140766189, + "pass@32": 0.995506921497008, + "pass@64": 0.996449217370583, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9775947822822822, + "pass@2": 0.9841167446974534, + "pass@4": 0.9884578084833988, + "pass@8": 0.9913859333225822, + "pass@16": 0.9933089247757945, + "pass@32": 0.9945349372602522, + "pass@64": 0.995491156454535, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9852290996784566, + "pass@2": 0.9896970023039725, + "pass@4": 0.9907644742759243, + "pass@8": 0.9913353456864922, + "pass@16": 0.9922684000518165, + "pass@32": 0.9939805554852268, + "pass@64": 0.9967972251057042, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9187003968253968, + "pass@2": 0.9381061742282217, + "pass@4": 0.9483199814308924, + "pass@8": 0.9549746233950248, + "pass@16": 0.9607838243504727, + "pass@32": 0.9664979946638379, + "pass@64": 0.9713905391268673, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.8736859709480123, + "pass@2": 0.9075217017024249, + "pass@4": 0.927535943614846, + "pass@8": 0.942757162906802, + "pass@16": 0.9558258204982006, + "pass@32": 0.9667707271291923, + "pass@64": 0.9759676186550527, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.7381606867283951, + "pass@2": 0.8031985910615335, + "pass@4": 0.8491654709249306, + "pass@8": 0.884136691191738, + "pass@16": 0.9091084537688574, + "pass@32": 0.9265018813718013, + "pass@64": 0.9408495040503696, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.6687917284866469, + "pass@2": 0.742496904133274, + "pass@4": 0.7915671680691242, + "pass@8": 0.8261163370467007, + "pass@16": 0.8535094321545398, + "pass@32": 0.8768087552275502, + "pass@64": 0.8966404643642354, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.560570987654321, + "pass@2": 0.6397261865704283, + "pass@4": 0.7012987609766366, + "pass@8": 0.7513312000892324, + "pass@16": 0.7905025056529893, + "pass@32": 0.8187060511329679, + "pass@64": 0.8396145819708817, + "pass@128": 0.8611111111111112 + }, + "16": { + "pass@1": 0.4017330383480826, + "pass@2": 0.4958964096578634, + "pass@4": 0.5784556440953726, + "pass@8": 0.6521378744476022, + "pass@16": 0.7185738377524138, + "pass@32": 0.7734250794597738, + "pass@64": 0.8139101487404639, + "pass@128": 0.8436578171091446 + }, + "17": { + "pass@1": 0.3000282379518072, + "pass@2": 0.3773594624561235, + "pass@4": 0.4456922685116164, + "pass@8": 0.5057320345187564, + "pass@16": 0.561393216655332, + "pass@32": 0.6143576371517395, + "pass@64": 0.6644245313917596, + "pass@128": 0.7108433734939759 + }, + "18": { + "pass@1": 0.2840725806451613, + "pass@2": 0.37186110934721867, + "pass@4": 0.4517761248593924, + "pass@8": 0.5177958613050213, + "pass@16": 0.5691864954935101, + "pass@32": 0.6100761487025451, + "pass@64": 0.6418813785016764, + "pass@128": 0.667741935483871 + }, + "19": { + "pass@1": 0.2082467900302115, + "pass@2": 0.27956287020957715, + "pass@4": 0.3474002794590251, + "pass@8": 0.4097443141957358, + "pass@16": 0.4660864976054333, + "pass@32": 0.5156854683282558, + "pass@64": 0.5586793575248601, + "pass@128": 0.595166163141994 + }, + "20": { + "pass@1": 0.17792792792792791, + "pass@2": 0.24336775259511484, + "pass@4": 0.3120260125142014, + "pass@8": 0.38010572901182366, + "pass@16": 0.44383337859661776, + "pass@32": 0.499497404852035, + "pass@64": 0.5444973653352457, + "pass@128": 0.5795795795795796 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_metrics.json b/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..fba0b03611cb67f039f4a958d83b042161a49888 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch4-op11-14", + "id": { + "answer_accuracy": 0.9930911458333334, + "avg_response_len": 199.22368663194445, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9930911458333334, + "pass@2": 0.9949381424978122, + "pass@4": 0.9961457733616635, + "pass@8": 0.9969982051319196, + "pass@16": 0.9976328783683257, + "pass@32": 0.9981484545451106, + "pass@64": 0.9986057317635817, + "pass@128": 0.999 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9525703125, + "pass@2": 0.9631371801181101, + "pass@4": 0.9709609806899138, + "pass@8": 0.9767945716645906, + "pass@16": 0.9810846355987156, + "pass@32": 0.9848159815033373, + "pass@64": 0.9885178829567154, + "pass@128": 0.992 + }, + "2": { + "pass@1": 0.9999296875, + "pass@2": 0.9999955708661418, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999609375, + "pass@2": 0.9998610974409449, + "pass@4": 0.9999817604049493, + "pass@8": 0.9999997360647032, + "pass@16": 0.9999999999758433, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99896875, + "pass@2": 0.9998102854330709, + "pass@4": 0.9999888729846269, + "pass@8": 0.9999999513947477, + "pass@16": 0.9999999999995928, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99821875, + "pass@2": 0.9987826033464566, + "pass@4": 0.9989794553805774, + "pass@8": 0.9989997726088258, + "pass@16": 0.9989999999838891, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9999140625, + "pass@2": 0.9999996309055118, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9944609375, + "pass@2": 0.997048843503937, + "pass@4": 0.9984326085489313, + "pass@8": 0.9990485918604365, + "pass@16": 0.9993936980677535, + "pass@32": 0.9996883057343079, + "pass@64": 0.9999404409432844, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941484375, + "pass@2": 0.9958080708661419, + "pass@4": 0.9969682940569927, + "pass@8": 0.9981412225939772, + "pass@16": 0.9992175716891378, + "pass@32": 0.9998318036683483, + "pass@64": 0.9999932619722337, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9525703125, + "2": 0.9999296875, + "3": 1.0, + "4": 0.999609375, + "5": 0.99896875, + "6": 0.99821875, + "7": 0.9999140625, + "8": 0.9944609375, + "9": 0.9941484375 + }, + "per_op_avg_response_len": { + "10": 279.6533515625, + "2": 115.369796875, + "3": 144.249234375, + "4": 159.452140625, + "5": 172.2081015625, + "6": 198.299765625, + "7": 222.978671875, + "8": 242.5681171875, + "9": 258.234 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.6533515625, + "2": 115.369796875, + "3": 144.249234375, + "4": 159.452140625, + "5": 172.2081015625, + "6": 198.299765625, + "7": 222.978671875, + "8": 242.5681171875, + "9": 258.234 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376431, + "answer_accuracy": 0.9932006712259371, + "avg_response_len": 189.76862757514354, + "resp_tokens_sum": 71923828, + "pass_at_k": { + "pass@1": 0.9932006712259371, + "pass@2": 0.995337226796119, + "pass@4": 0.9966168958806693, + "pass@8": 0.9974370308941563, + "pass@16": 0.9980251463797202, + "pass@32": 0.9985125017885638, + "pass@64": 0.9989902516880416, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9550377949852508, + "pass@2": 0.9653236272733609, + "pass@4": 0.9726180897741764, + "pass@8": 0.9783762148389291, + "pass@16": 0.9829116146743994, + "pass@32": 0.9870127565970104, + "pass@64": 0.991180340449138, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9997809579439252, + "pass@2": 0.9999862020752079, + "pass@4": 0.9999999632055339, + "pass@8": 0.9999999999999803, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998779296875, + "pass@2": 0.9995659295029528, + "pass@4": 0.9999430012654666, + "pass@8": 0.9999991752021973, + "pass@16": 0.9999999999245107, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9982638888888888, + "pass@2": 0.9995289651293588, + "pass@4": 0.9999661601823582, + "pass@8": 0.9999998458721072, + "pass@16": 0.9999999999987073, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9989006362275449, + "pass@2": 0.9996018052477722, + "pass@4": 0.9999499412311484, + "pass@8": 0.9999993414788002, + "pass@16": 0.9999999999517831, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999075443786982, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9933143028846154, + "pass@2": 0.997447096456693, + "pass@4": 0.999438021268976, + "pass@8": 0.9999627194608325, + "pass@16": 0.9999998418611676, + "pass@32": 0.9999999999993773, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9960607394366198, + "pass@2": 0.9976620550072086, + "pass@4": 0.9985522935513342, + "pass@8": 0.9993061284706786, + "pass@16": 0.9998463955935609, + "pass@32": 0.9999949107316761, + "pass@64": 0.9999999995381218, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387539, + "answer_accuracy": 0.9920211132044561, + "avg_response_len": 197.2330285468545, + "resp_tokens_sum": 77050266, + "pass_at_k": { + "pass@1": 0.9920211132044561, + "pass@2": 0.9936430367720664, + "pass@4": 0.9947641457272788, + "pass@8": 0.9956322126604567, + "pass@16": 0.9963545198515974, + "pass@32": 0.9970399392150072, + "pass@64": 0.9977077449554393, + "pass@128": 0.9983617300131061 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9402319785276073, + "pass@2": 0.952008281604754, + "pass@4": 0.9611208099562711, + "pass@8": 0.967649685144444, + "pass@16": 0.9723239331864337, + "pass@32": 0.976739705975646, + "pass@64": 0.9818104174845861, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999411231884058, + "pass@2": 0.9999315302978432, + "pass@4": 0.9999989373610907, + "pass@8": 0.9999999998467666, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9966869627507163, + "pass@2": 0.9970676905895359, + "pass@4": 0.9971333691633818, + "pass@8": 0.9971346702346441, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9998871026011561, + "pass@2": 0.9999989332529243, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9965889084507042, + "pass@2": 0.9972960796273704, + "pass@4": 0.9975222988851745, + "pass@8": 0.9978309304389954, + "pass@16": 0.9983600236590144, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932400823353293, + "pass@2": 0.99476379426187, + "pass@4": 0.995734111922387, + "pass@8": 0.9969632799932823, + "pass@16": 0.9984388741597237, + "pass@32": 0.999581272939488, + "pass@64": 0.9999802411950416, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380071, + "answer_accuracy": 0.9940758913625711, + "avg_response_len": 210.630421932541, + "resp_tokens_sum": 80531593, + "pass_at_k": { + "pass@1": 0.9940758913625711, + "pass@2": 0.995865820458206, + "pass@4": 0.9970904448586044, + "pass@8": 0.9979589168630615, + "pass@16": 0.9985502016396067, + "pass@32": 0.9989202138017588, + "pass@64": 0.9991420867157488, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.962080223880597, + "pass@2": 0.9717545393113174, + "pass@4": 0.9788598931663394, + "pass@8": 0.984093246200315, + "pass@16": 0.9877611821054247, + "pass@32": 0.9904522772205684, + "pass@64": 0.9923508998342762, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999172794117647, + "pass@2": 0.9999478925428441, + "pass@4": 0.9999997033459054, + "pass@8": 0.9999999999938223, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991867113564669, + "pass@2": 0.9998074964604189, + "pass@4": 0.9999893664711469, + "pass@8": 0.9999999767918476, + "pass@16": 0.9999999999999797, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999505537974683, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9932667042042042, + "pass@2": 0.9964121355607576, + "pass@4": 0.9984610504767983, + "pass@8": 0.999490219467908, + "pass@16": 0.9999277447691266, + "pass@32": 0.999999100781228, + "pass@64": 0.9999999999952028, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.992941117363344, + "pass@2": 0.9948132927817303, + "pass@4": 0.9964856477948294, + "pass@8": 0.9980765642094214, + "pass@16": 0.999336068437543, + "pass@32": 0.9999146790894352, + "pass@64": 0.9999995549101495, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.67023828125, + "avg_response_len": 370.87193515625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.67023828125, + "pass@2": 0.7391869832677044, + "pass@4": 0.7914964344206904, + "pass@8": 0.8305876741468599, + "pass@16": 0.8598251061006209, + "pass@32": 0.8822750532094229, + "pass@64": 0.9001723511612838, + "pass@128": 0.9149 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9613515625, + "pass@2": 0.9716198326771655, + "pass@4": 0.9771861735095614, + "pass@8": 0.9808070057015184, + "pass@16": 0.9837384231215315, + "pass@32": 0.9858625291421717, + "pass@64": 0.9869666845018911, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.9076796875, + "pass@2": 0.9273491633858271, + "pass@4": 0.9397097274090743, + "pass@8": 0.9481943683164651, + "pass@16": 0.9539304509846932, + "pass@32": 0.9579559040128569, + "pass@64": 0.9609342516531564, + "pass@128": 0.963 + }, + "13": { + "pass@1": 0.849375, + "pass@2": 0.8836884842519688, + "pass@4": 0.9050689845331829, + "pass@8": 0.919391571172967, + "pass@16": 0.930207215681193, + "pass@32": 0.939676797003826, + "pass@64": 0.9483149333749666, + "pass@128": 0.956 + }, + "14": { + "pass@1": 0.843671875, + "pass@2": 0.8859703494094502, + "pass@4": 0.9096745502437197, + "pass@8": 0.9264951601373707, + "pass@16": 0.9398242906228735, + "pass@32": 0.9497296044610397, + "pass@64": 0.9577593392189998, + "pass@128": 0.966 + }, + "15": { + "pass@1": 0.8058515625, + "pass@2": 0.8670836614173242, + "pass@4": 0.9038101552305964, + "pass@8": 0.9279624319017417, + "pass@16": 0.9447318076178691, + "pass@32": 0.9566893837117887, + "pass@64": 0.9647829004058682, + "pass@128": 0.971 + }, + "16": { + "pass@1": 0.64621875, + "pass@2": 0.7415820620078745, + "pass@4": 0.8050657452193474, + "pass@8": 0.8463069193831734, + "pass@16": 0.8743385001840214, + "pass@32": 0.8966440530869019, + "pass@64": 0.9157644672574666, + "pass@128": 0.929 + }, + "17": { + "pass@1": 0.56553125, + "pass@2": 0.6692714074803152, + "pass@4": 0.7414758831083603, + "pass@8": 0.7884725208544237, + "pass@16": 0.8217109948566683, + "pass@32": 0.8488122736046942, + "pass@64": 0.8727472203835355, + "pass@128": 0.894 + }, + "18": { + "pass@1": 0.4961796875, + "pass@2": 0.6022688238188982, + "pass@4": 0.6869041670416197, + "pass@8": 0.7508196935626941, + "pass@16": 0.7979812625700984, + "pass@32": 0.8342351494830981, + "pass@64": 0.8626833370727636, + "pass@128": 0.884 + }, + "19": { + "pass@1": 0.3210546875, + "pass@2": 0.4304410679133851, + "pass@4": 0.5329773877952755, + "pass@8": 0.6194907254128081, + "pass@16": 0.686649728649235, + "pass@32": 0.7366339679870643, + "pass@64": 0.7755140138496416, + "pass@128": 0.808 + }, + "20": { + "pass@1": 0.30546875, + "pass@2": 0.4125949803149601, + "pass@4": 0.5130915701162351, + "pass@8": 0.597936345025493, + "pass@16": 0.6651383867179285, + "pass@32": 0.7165108696007512, + "pass@64": 0.7562563638944912, + "pass@128": 0.79 + } + }, + "per_op_accuracy": { + "11": 0.9613515625, + "12": 0.9076796875, + "13": 0.849375, + "14": 0.843671875, + "15": 0.8058515625, + "16": 0.64621875, + "17": 0.56553125, + "18": 0.4961796875, + "19": 0.3210546875, + "20": 0.30546875 + }, + "per_op_avg_response_len": { + "11": 297.927390625, + "12": 307.79465625, + "13": 326.4955625, + "14": 354.709875, + "15": 375.8386796875, + "16": 380.9595703125, + "17": 402.584625, + "18": 417.3535078125, + "19": 415.032421875, + "20": 430.0230625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.927390625, + "12": 307.79465625, + "13": 326.4955625, + "14": 354.709875, + "15": 375.8386796875, + "16": 380.9595703125, + "17": 402.584625, + "18": 417.3535078125, + "19": 415.032421875, + "20": 430.0230625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 292022, + "answer_accuracy": 0.6779856983655275, + "avg_response_len": 353.17834323922733, + "resp_tokens_sum": 152120976, + "pass_at_k": { + "pass@1": 0.6779856983655275, + "pass@2": 0.7444212437551985, + "pass@4": 0.7953746970986009, + "pass@8": 0.8348435843436636, + "pass@16": 0.8648672389343015, + "pass@32": 0.8874072691956361, + "pass@64": 0.9046577103739157, + "pass@128": 0.9188707280832095 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9609375, + "pass@2": 0.9711451060649998, + "pass@4": 0.9770741222857058, + "pass@8": 0.9812611025364644, + "pass@16": 0.9842944002686944, + "pass@32": 0.9861828830088263, + "pass@64": 0.9872429473453521, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.9021671963276836, + "pass@2": 0.9215281484941499, + "pass@4": 0.9347171227537232, + "pass@8": 0.9446575312767541, + "pass@16": 0.9525455705033121, + "pass@32": 0.9580749193035334, + "pass@64": 0.9613374661256407, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8692202194357367, + "pass@2": 0.9024653352010464, + "pass@4": 0.9221733035721631, + "pass@8": 0.9359308821175786, + "pass@16": 0.9484843594916188, + "pass@32": 0.960115260632947, + "pass@64": 0.9696726490165624, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8432389018691588, + "pass@2": 0.8816467669929107, + "pass@4": 0.9054283430692655, + "pass@8": 0.9252522133459742, + "pass@16": 0.9410032100908022, + "pass@32": 0.9510890172203004, + "pass@64": 0.9586071337072141, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.8159320809248555, + "pass@2": 0.8758590869782895, + "pass@4": 0.9136971456963835, + "pass@8": 0.9391793646317208, + "pass@16": 0.9568471423736444, + "pass@32": 0.9685935199216499, + "pass@64": 0.9754437045824256, + "pass@128": 0.9797687861271677 + }, + "16": { + "pass@1": 0.6567792338709677, + "pass@2": 0.740235664846329, + "pass@4": 0.795893839278154, + "pass@8": 0.8355960749083245, + "pass@16": 0.864558190334898, + "pass@32": 0.8889121743620298, + "pass@64": 0.9101876710408604, + "pass@128": 0.9258064516129032 + }, + "17": { + "pass@1": 0.5896405346820809, + "pass@2": 0.689234175162714, + "pass@4": 0.7562385357538394, + "pass@8": 0.7996128076425166, + "pass@16": 0.8318120005184033, + "pass@32": 0.8596979457171261, + "pass@64": 0.8843409721762664, + "pass@128": 0.9046242774566474 + }, + "18": { + "pass@1": 0.47859375, + "pass@2": 0.582545345894263, + "pass@4": 0.665977828753548, + "pass@8": 0.731456928764178, + "pass@16": 0.780729199135965, + "pass@32": 0.8185404669626503, + "pass@64": 0.848842739464603, + "pass@128": 0.8742857142857143 + }, + "19": { + "pass@1": 0.32244744744744747, + "pass@2": 0.42991527452649503, + "pass@4": 0.5297148988133239, + "pass@8": 0.6158184040059627, + "pass@16": 0.6829310379104327, + "pass@32": 0.7314577265217167, + "pass@64": 0.7668547527202739, + "pass@128": 0.7927927927927928 + }, + "20": { + "pass@1": 0.33054147897897895, + "pass@2": 0.441439224657729, + "pass@4": 0.5459787081794953, + "pass@8": 0.6340915487675354, + "pass@16": 0.7009979125456074, + "pass@32": 0.7479033979165912, + "pass@64": 0.781451169312069, + "pass@128": 0.8108108108108109 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 291271, + "answer_accuracy": 0.6766442722271782, + "avg_response_len": 366.3032239629795, + "resp_tokens_sum": 157680351, + "pass_at_k": { + "pass@1": 0.6766442722271782, + "pass@2": 0.7441928255845854, + "pass@4": 0.7948994997112895, + "pass@8": 0.8326424323324724, + "pass@16": 0.8614864198568543, + "pass@32": 0.884597208281839, + "pass@64": 0.903587135802124, + "pass@128": 0.9191198334820101 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9623258659638554, + "pass@2": 0.9707566733469312, + "pass@4": 0.9751945817578527, + "pass@8": 0.978528567411368, + "pass@16": 0.9816759959049333, + "pass@32": 0.9840488998900503, + "pass@64": 0.9848917427571217, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9100215517241379, + "pass@2": 0.9266520160195493, + "pass@4": 0.9344253890285655, + "pass@8": 0.9383279633369872, + "pass@16": 0.9410428618385669, + "pass@32": 0.9436759703758117, + "pass@64": 0.9467110995185979, + "pass@128": 0.9498432601880877 + }, + "13": { + "pass@1": 0.8494616596638656, + "pass@2": 0.8807649043869519, + "pass@4": 0.900884443803768, + "pass@8": 0.9155814360333341, + "pass@16": 0.9267145595077136, + "pass@32": 0.9359428582481637, + "pass@64": 0.9445210713800876, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.8588267543859649, + "pass@2": 0.9018694312059676, + "pass@4": 0.9242850279679948, + "pass@8": 0.9383757151048715, + "pass@16": 0.9484554246399113, + "pass@32": 0.9561951272112444, + "pass@64": 0.9627628804004219, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.8121685606060606, + "pass@2": 0.8770117513719882, + "pass@4": 0.915564349342695, + "pass@8": 0.9404922694298705, + "pass@16": 0.9573654432962957, + "pass@32": 0.9699346349116533, + "pass@64": 0.9789174437718314, + "pass@128": 0.9848484848484849 + }, + "16": { + "pass@1": 0.6486823361823362, + "pass@2": 0.7438652500168247, + "pass@4": 0.805919283326763, + "pass@8": 0.8449027197071314, + "pass@16": 0.8727992336539356, + "pass@32": 0.8965191234491018, + "pass@64": 0.9150418318628365, + "pass@128": 0.9230769230769231 + }, + "17": { + "pass@1": 0.5890916149068323, + "pass@2": 0.6926184159045329, + "pass@4": 0.764558463286033, + "pass@8": 0.8095983074474769, + "pass@16": 0.8386906126837259, + "pass@32": 0.8617211263710541, + "pass@64": 0.8851656158646087, + "pass@128": 0.9099378881987578 + }, + "18": { + "pass@1": 0.5270220588235294, + "pass@2": 0.6297294754515979, + "pass@4": 0.7110232550894364, + "pass@8": 0.7729100372599288, + "pass@16": 0.8200711561466233, + "pass@32": 0.8564476725925791, + "pass@64": 0.8842089254988268, + "pass@128": 0.9029411764705882 + }, + "19": { + "pass@1": 0.31570870535714285, + "pass@2": 0.4234084997187852, + "pass@4": 0.5270488630327456, + "pass@8": 0.6161440847378656, + "pass@16": 0.6883030755974877, + "pass@32": 0.7454121901545792, + "pass@64": 0.791591256811504, + "pass@128": 0.8333333333333334 + }, + "20": { + "pass@1": 0.29381081586826346, + "pass@2": 0.39535287201188124, + "pass@4": 0.48994442414383815, + "pass@8": 0.5711483363826257, + "pass@16": 0.6389980310343075, + "pass@32": 0.6950049674288274, + "pass@64": 0.740922892472733, + "pass@128": 0.781437125748503 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 274612, + "answer_accuracy": 0.6556865067237164, + "avg_response_len": 393.7642067160758, + "resp_tokens_sum": 164914750, + "pass_at_k": { + "pass@1": 0.6556865067237164, + "pass@2": 0.7286588860024493, + "pass@4": 0.7840102295052386, + "pass@8": 0.8240988937097103, + "pass@16": 0.852932143035952, + "pass@32": 0.8746102260999513, + "pass@64": 0.8920497489309196, + "pass@128": 0.906479217603912 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9607886904761904, + "pass@2": 0.9730615704286967, + "pass@4": 0.9794108165050797, + "pass@8": 0.9826995305573731, + "pass@16": 0.9852891085277606, + "pass@32": 0.9874150370684425, + "pass@64": 0.9888440174400547, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9113627675840978, + "pass@2": 0.934330904307833, + "pass@4": 0.950269623407166, + "pass@8": 0.9616482628134414, + "pass@16": 0.9680019758410323, + "pass@32": 0.9717576392340137, + "pass@64": 0.9743729293524362, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8297405478395061, + "pass@2": 0.8684227514095461, + "pass@4": 0.8928393649867837, + "pass@8": 0.9073057009677743, + "pass@16": 0.916060516231829, + "pass@32": 0.9236679890966697, + "pass@64": 0.9314670859752837, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.82870456231454, + "pass@2": 0.87395368466553, + "pass@4": 0.898891906686738, + "pass@8": 0.9156222702891602, + "pass@16": 0.9299421512073494, + "pass@32": 0.9418732831664041, + "pass@64": 0.9518740184036798, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.7886525848765432, + "pass@2": 0.8476004301545641, + "pass@4": 0.8812799615325858, + "pass@8": 0.903221984034905, + "pass@16": 0.9189262347802786, + "pass@32": 0.9304864083273199, + "pass@64": 0.9390019202952004, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.634010877581121, + "pass@2": 0.740449271827747, + "pass@4": 0.8125692819924054, + "pass@8": 0.8575554027858113, + "pass@16": 0.8848759001996248, + "pass@32": 0.9038438545841817, + "pass@64": 0.9216124078198928, + "pass@128": 0.9380530973451328 + }, + "17": { + "pass@1": 0.517554593373494, + "pass@2": 0.6258230510625178, + "pass@4": 0.7037033872271984, + "pass@8": 0.756372995216946, + "pass@16": 0.794715859617898, + "pass@32": 0.8249475352261741, + "pass@64": 0.8486202280185033, + "pass@128": 0.8674698795180723 + }, + "18": { + "pass@1": 0.48220766129032255, + "pass@2": 0.5944191325882653, + "pass@4": 0.6840774846692548, + "pass@8": 0.7484527607317959, + "pass@16": 0.7932317734924506, + "pass@32": 0.8275928302086873, + "pass@64": 0.8547011083566157, + "pass@128": 0.8741935483870967 + }, + "19": { + "pass@1": 0.325080249244713, + "pass@2": 0.43810883864214883, + "pass@4": 0.5422776692188405, + "pass@8": 0.6265824302323246, + "pass@16": 0.6887125668408031, + "pass@32": 0.7329306621250592, + "pass@64": 0.767905495181644, + "pass@128": 0.797583081570997 + }, + "20": { + "pass@1": 0.29208896396396394, + "pass@2": 0.401044622378284, + "pass@4": 0.5034210889742385, + "pass@8": 0.5886495943967197, + "pass@16": 0.6554977161404892, + "pass@32": 0.7066888257456404, + "pass@64": 0.746441076359396, + "pass@128": 0.7777777777777778 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.8231685855263158, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8231685855263158, + "pass@2": 0.8603322692188005, + "pass@4": 0.8884355949716634, + "pass@8": 0.9094137151397931, + "pass@16": 0.9251024719116253, + "pass@32": 0.9371624538421249, + "pass@64": 0.946798689341318, + "pass@128": 0.9547368421052631 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9525703125, + "pass@2": 0.9631371801181101, + "pass@4": 0.9709609806899138, + "pass@8": 0.9767945716645906, + "pass@16": 0.9810846355987156, + "pass@32": 0.9848159815033373, + "pass@64": 0.9885178829567154, + "pass@128": 0.992 + }, + "2": { + "pass@1": 0.9999296875, + "pass@2": 0.9999955708661418, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999609375, + "pass@2": 0.9998610974409449, + "pass@4": 0.9999817604049493, + "pass@8": 0.9999997360647032, + "pass@16": 0.9999999999758433, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99896875, + "pass@2": 0.9998102854330709, + "pass@4": 0.9999888729846269, + "pass@8": 0.9999999513947477, + "pass@16": 0.9999999999995928, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99821875, + "pass@2": 0.9987826033464566, + "pass@4": 0.9989794553805774, + "pass@8": 0.9989997726088258, + "pass@16": 0.9989999999838891, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9999140625, + "pass@2": 0.9999996309055118, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9944609375, + "pass@2": 0.997048843503937, + "pass@4": 0.9984326085489313, + "pass@8": 0.9990485918604365, + "pass@16": 0.9993936980677535, + "pass@32": 0.9996883057343079, + "pass@64": 0.9999404409432844, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941484375, + "pass@2": 0.9958080708661419, + "pass@4": 0.9969682940569927, + "pass@8": 0.9981412225939772, + "pass@16": 0.9992175716891378, + "pass@32": 0.9998318036683483, + "pass@64": 0.9999932619722337, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9613515625, + "pass@2": 0.9716198326771655, + "pass@4": 0.9771861735095614, + "pass@8": 0.9808070057015184, + "pass@16": 0.9837384231215315, + "pass@32": 0.9858625291421717, + "pass@64": 0.9869666845018911, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.9076796875, + "pass@2": 0.9273491633858271, + "pass@4": 0.9397097274090743, + "pass@8": 0.9481943683164651, + "pass@16": 0.9539304509846932, + "pass@32": 0.9579559040128569, + "pass@64": 0.9609342516531564, + "pass@128": 0.963 + }, + "13": { + "pass@1": 0.849375, + "pass@2": 0.8836884842519688, + "pass@4": 0.9050689845331829, + "pass@8": 0.919391571172967, + "pass@16": 0.930207215681193, + "pass@32": 0.939676797003826, + "pass@64": 0.9483149333749666, + "pass@128": 0.956 + }, + "14": { + "pass@1": 0.843671875, + "pass@2": 0.8859703494094502, + "pass@4": 0.9096745502437197, + "pass@8": 0.9264951601373707, + "pass@16": 0.9398242906228735, + "pass@32": 0.9497296044610397, + "pass@64": 0.9577593392189998, + "pass@128": 0.966 + }, + "15": { + "pass@1": 0.8058515625, + "pass@2": 0.8670836614173242, + "pass@4": 0.9038101552305964, + "pass@8": 0.9279624319017417, + "pass@16": 0.9447318076178691, + "pass@32": 0.9566893837117887, + "pass@64": 0.9647829004058682, + "pass@128": 0.971 + }, + "16": { + "pass@1": 0.64621875, + "pass@2": 0.7415820620078745, + "pass@4": 0.8050657452193474, + "pass@8": 0.8463069193831734, + "pass@16": 0.8743385001840214, + "pass@32": 0.8966440530869019, + "pass@64": 0.9157644672574666, + "pass@128": 0.929 + }, + "17": { + "pass@1": 0.56553125, + "pass@2": 0.6692714074803152, + "pass@4": 0.7414758831083603, + "pass@8": 0.7884725208544237, + "pass@16": 0.8217109948566683, + "pass@32": 0.8488122736046942, + "pass@64": 0.8727472203835355, + "pass@128": 0.894 + }, + "18": { + "pass@1": 0.4961796875, + "pass@2": 0.6022688238188982, + "pass@4": 0.6869041670416197, + "pass@8": 0.7508196935626941, + "pass@16": 0.7979812625700984, + "pass@32": 0.8342351494830981, + "pass@64": 0.8626833370727636, + "pass@128": 0.884 + }, + "19": { + "pass@1": 0.3210546875, + "pass@2": 0.4304410679133851, + "pass@4": 0.5329773877952755, + "pass@8": 0.6194907254128081, + "pass@16": 0.686649728649235, + "pass@32": 0.7366339679870643, + "pass@64": 0.7755140138496416, + "pass@128": 0.808 + }, + "20": { + "pass@1": 0.30546875, + "pass@2": 0.4125949803149601, + "pass@4": 0.5130915701162351, + "pass@8": 0.597936345025493, + "pass@16": 0.6651383867179285, + "pass@32": 0.7165108696007512, + "pass@64": 0.7562563638944912, + "pass@128": 0.79 + } + }, + "per_op_accuracy": { + "10": 0.9525703125, + "2": 0.9999296875, + "3": 1.0, + "4": 0.999609375, + "5": 0.99896875, + "6": 0.99821875, + "7": 0.9999140625, + "8": 0.9944609375, + "9": 0.9941484375, + "11": 0.9613515625, + "12": 0.9076796875, + "13": 0.849375, + "14": 0.843671875, + "15": 0.8058515625, + "16": 0.64621875, + "17": 0.56553125, + "18": 0.4961796875, + "19": 0.3210546875, + "20": 0.30546875 + }, + "per_op_avg_response_len": { + "10": 279.6533515625, + "2": 115.369796875, + "3": 144.249234375, + "4": 159.452140625, + "5": 172.2081015625, + "6": 198.299765625, + "7": 222.978671875, + "8": 242.5681171875, + "9": 258.234, + "11": 297.927390625, + "12": 307.79465625, + "13": 326.4955625, + "14": 354.709875, + "15": 375.8386796875, + "16": 380.9595703125, + "17": 402.584625, + "18": 417.3535078125, + "19": 415.032421875, + "20": 430.0230625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.6533515625, + "2": 115.369796875, + "3": 144.249234375, + "4": 159.452140625, + "5": 172.2081015625, + "6": 198.299765625, + "7": 222.978671875, + "8": 242.5681171875, + "9": 258.234, + "11": 297.927390625, + "12": 307.79465625, + "13": 326.4955625, + "14": 354.709875, + "15": 375.8386796875, + "16": 380.9595703125, + "17": 402.584625, + "18": 417.3535078125, + "19": 415.032421875, + "20": 430.0230625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 668453, + "answer_accuracy": 0.8255278315681315, + "avg_response_len": 276.6914371245653, + "resp_tokens_sum": 224044804, + "pass_at_k": { + "pass@1": 0.8255278315681315, + "pass@2": 0.861867058770073, + "pass@4": 0.8895697888775553, + "pass@8": 0.9109484207704667, + "pass@16": 0.9271942329187942, + "pass@32": 0.93941210538085, + "pass@64": 0.9488117816403033, + "pass@128": 0.9565286120771419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9550377949852508, + "pass@2": 0.9653236272733609, + "pass@4": 0.9726180897741764, + "pass@8": 0.9783762148389291, + "pass@16": 0.9829116146743994, + "pass@32": 0.9870127565970104, + "pass@64": 0.991180340449138, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9997809579439252, + "pass@2": 0.9999862020752079, + "pass@4": 0.9999999632055339, + "pass@8": 0.9999999999999803, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998779296875, + "pass@2": 0.9995659295029528, + "pass@4": 0.9999430012654666, + "pass@8": 0.9999991752021973, + "pass@16": 0.9999999999245107, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9982638888888888, + "pass@2": 0.9995289651293588, + "pass@4": 0.9999661601823582, + "pass@8": 0.9999998458721072, + "pass@16": 0.9999999999987073, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9989006362275449, + "pass@2": 0.9996018052477722, + "pass@4": 0.9999499412311484, + "pass@8": 0.9999993414788002, + "pass@16": 0.9999999999517831, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999075443786982, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9933143028846154, + "pass@2": 0.997447096456693, + "pass@4": 0.999438021268976, + "pass@8": 0.9999627194608325, + "pass@16": 0.9999998418611676, + "pass@32": 0.9999999999993773, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9960607394366198, + "pass@2": 0.9976620550072086, + "pass@4": 0.9985522935513342, + "pass@8": 0.9993061284706786, + "pass@16": 0.9998463955935609, + "pass@32": 0.9999949107316761, + "pass@64": 0.9999999995381218, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9609375, + "pass@2": 0.9711451060649998, + "pass@4": 0.9770741222857058, + "pass@8": 0.9812611025364644, + "pass@16": 0.9842944002686944, + "pass@32": 0.9861828830088263, + "pass@64": 0.9872429473453521, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.9021671963276836, + "pass@2": 0.9215281484941499, + "pass@4": 0.9347171227537232, + "pass@8": 0.9446575312767541, + "pass@16": 0.9525455705033121, + "pass@32": 0.9580749193035334, + "pass@64": 0.9613374661256407, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8692202194357367, + "pass@2": 0.9024653352010464, + "pass@4": 0.9221733035721631, + "pass@8": 0.9359308821175786, + "pass@16": 0.9484843594916188, + "pass@32": 0.960115260632947, + "pass@64": 0.9696726490165624, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8432389018691588, + "pass@2": 0.8816467669929107, + "pass@4": 0.9054283430692655, + "pass@8": 0.9252522133459742, + "pass@16": 0.9410032100908022, + "pass@32": 0.9510890172203004, + "pass@64": 0.9586071337072141, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.8159320809248555, + "pass@2": 0.8758590869782895, + "pass@4": 0.9136971456963835, + "pass@8": 0.9391793646317208, + "pass@16": 0.9568471423736444, + "pass@32": 0.9685935199216499, + "pass@64": 0.9754437045824256, + "pass@128": 0.9797687861271677 + }, + "16": { + "pass@1": 0.6567792338709677, + "pass@2": 0.740235664846329, + "pass@4": 0.795893839278154, + "pass@8": 0.8355960749083245, + "pass@16": 0.864558190334898, + "pass@32": 0.8889121743620298, + "pass@64": 0.9101876710408604, + "pass@128": 0.9258064516129032 + }, + "17": { + "pass@1": 0.5896405346820809, + "pass@2": 0.689234175162714, + "pass@4": 0.7562385357538394, + "pass@8": 0.7996128076425166, + "pass@16": 0.8318120005184033, + "pass@32": 0.8596979457171261, + "pass@64": 0.8843409721762664, + "pass@128": 0.9046242774566474 + }, + "18": { + "pass@1": 0.47859375, + "pass@2": 0.582545345894263, + "pass@4": 0.665977828753548, + "pass@8": 0.731456928764178, + "pass@16": 0.780729199135965, + "pass@32": 0.8185404669626503, + "pass@64": 0.848842739464603, + "pass@128": 0.8742857142857143 + }, + "19": { + "pass@1": 0.32244744744744747, + "pass@2": 0.42991527452649503, + "pass@4": 0.5297148988133239, + "pass@8": 0.6158184040059627, + "pass@16": 0.6829310379104327, + "pass@32": 0.7314577265217167, + "pass@64": 0.7668547527202739, + "pass@128": 0.7927927927927928 + }, + "20": { + "pass@1": 0.33054147897897895, + "pass@2": 0.441439224657729, + "pass@4": 0.5459787081794953, + "pass@8": 0.6340915487675354, + "pass@16": 0.7009979125456074, + "pass@32": 0.7479033979165912, + "pass@64": 0.781451169312069, + "pass@128": 0.8108108108108109 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 678810, + "answer_accuracy": 0.8266879384255651, + "avg_response_len": 285.86639833398283, + "resp_tokens_sum": 234730617, + "pass_at_k": { + "pass@1": 0.8266879384255651, + "pass@2": 0.8628712425049453, + "pass@4": 0.8899870912375201, + "pass@8": 0.9101864400582642, + "pass@16": 0.9256512587008077, + "pass@32": 0.9380929549393652, + "pass@64": 0.9483659509441283, + "pass@128": 0.9568199532346064 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9402319785276073, + "pass@2": 0.952008281604754, + "pass@4": 0.9611208099562711, + "pass@8": 0.967649685144444, + "pass@16": 0.9723239331864337, + "pass@32": 0.976739705975646, + "pass@64": 0.9818104174845861, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999411231884058, + "pass@2": 0.9999315302978432, + "pass@4": 0.9999989373610907, + "pass@8": 0.9999999998467666, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9966869627507163, + "pass@2": 0.9970676905895359, + "pass@4": 0.9971333691633818, + "pass@8": 0.9971346702346441, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9998871026011561, + "pass@2": 0.9999989332529243, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9965889084507042, + "pass@2": 0.9972960796273704, + "pass@4": 0.9975222988851745, + "pass@8": 0.9978309304389954, + "pass@16": 0.9983600236590144, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932400823353293, + "pass@2": 0.99476379426187, + "pass@4": 0.995734111922387, + "pass@8": 0.9969632799932823, + "pass@16": 0.9984388741597237, + "pass@32": 0.999581272939488, + "pass@64": 0.9999802411950416, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9623258659638554, + "pass@2": 0.9707566733469312, + "pass@4": 0.9751945817578527, + "pass@8": 0.978528567411368, + "pass@16": 0.9816759959049333, + "pass@32": 0.9840488998900503, + "pass@64": 0.9848917427571217, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9100215517241379, + "pass@2": 0.9266520160195493, + "pass@4": 0.9344253890285655, + "pass@8": 0.9383279633369872, + "pass@16": 0.9410428618385669, + "pass@32": 0.9436759703758117, + "pass@64": 0.9467110995185979, + "pass@128": 0.9498432601880877 + }, + "13": { + "pass@1": 0.8494616596638656, + "pass@2": 0.8807649043869519, + "pass@4": 0.900884443803768, + "pass@8": 0.9155814360333341, + "pass@16": 0.9267145595077136, + "pass@32": 0.9359428582481637, + "pass@64": 0.9445210713800876, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.8588267543859649, + "pass@2": 0.9018694312059676, + "pass@4": 0.9242850279679948, + "pass@8": 0.9383757151048715, + "pass@16": 0.9484554246399113, + "pass@32": 0.9561951272112444, + "pass@64": 0.9627628804004219, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.8121685606060606, + "pass@2": 0.8770117513719882, + "pass@4": 0.915564349342695, + "pass@8": 0.9404922694298705, + "pass@16": 0.9573654432962957, + "pass@32": 0.9699346349116533, + "pass@64": 0.9789174437718314, + "pass@128": 0.9848484848484849 + }, + "16": { + "pass@1": 0.6486823361823362, + "pass@2": 0.7438652500168247, + "pass@4": 0.805919283326763, + "pass@8": 0.8449027197071314, + "pass@16": 0.8727992336539356, + "pass@32": 0.8965191234491018, + "pass@64": 0.9150418318628365, + "pass@128": 0.9230769230769231 + }, + "17": { + "pass@1": 0.5890916149068323, + "pass@2": 0.6926184159045329, + "pass@4": 0.764558463286033, + "pass@8": 0.8095983074474769, + "pass@16": 0.8386906126837259, + "pass@32": 0.8617211263710541, + "pass@64": 0.8851656158646087, + "pass@128": 0.9099378881987578 + }, + "18": { + "pass@1": 0.5270220588235294, + "pass@2": 0.6297294754515979, + "pass@4": 0.7110232550894364, + "pass@8": 0.7729100372599288, + "pass@16": 0.8200711561466233, + "pass@32": 0.8564476725925791, + "pass@64": 0.8842089254988268, + "pass@128": 0.9029411764705882 + }, + "19": { + "pass@1": 0.31570870535714285, + "pass@2": 0.4234084997187852, + "pass@4": 0.5270488630327456, + "pass@8": 0.6161440847378656, + "pass@16": 0.6883030755974877, + "pass@32": 0.7454121901545792, + "pass@64": 0.791591256811504, + "pass@128": 0.8333333333333334 + }, + "20": { + "pass@1": 0.29381081586826346, + "pass@2": 0.39535287201188124, + "pass@4": 0.48994442414383815, + "pass@8": 0.5711483363826257, + "pass@16": 0.6389980310343075, + "pass@32": 0.6950049674288274, + "pass@64": 0.740922892472733, + "pass@128": 0.781437125748503 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 654683, + "answer_accuracy": 0.8171770150982585, + "avg_response_len": 306.3667606147148, + "resp_tokens_sum": 245446343, + "pass_at_k": { + "pass@1": 0.8171770150982585, + "pass@2": 0.8561787954479328, + "pass@4": 0.8856990940619518, + "pass@8": 0.9070705967228174, + "pass@16": 0.9224258546590777, + "pass@32": 0.9339350277080865, + "pass@64": 0.9431577235216402, + "pass@128": 0.9507908611599297 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.962080223880597, + "pass@2": 0.9717545393113174, + "pass@4": 0.9788598931663394, + "pass@8": 0.984093246200315, + "pass@16": 0.9877611821054247, + "pass@32": 0.9904522772205684, + "pass@64": 0.9923508998342762, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999172794117647, + "pass@2": 0.9999478925428441, + "pass@4": 0.9999997033459054, + "pass@8": 0.9999999999938223, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991867113564669, + "pass@2": 0.9998074964604189, + "pass@4": 0.9999893664711469, + "pass@8": 0.9999999767918476, + "pass@16": 0.9999999999999797, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999505537974683, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9932667042042042, + "pass@2": 0.9964121355607576, + "pass@4": 0.9984610504767983, + "pass@8": 0.999490219467908, + "pass@16": 0.9999277447691266, + "pass@32": 0.999999100781228, + "pass@64": 0.9999999999952028, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.992941117363344, + "pass@2": 0.9948132927817303, + "pass@4": 0.9964856477948294, + "pass@8": 0.9980765642094214, + "pass@16": 0.999336068437543, + "pass@32": 0.9999146790894352, + "pass@64": 0.9999995549101495, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9607886904761904, + "pass@2": 0.9730615704286967, + "pass@4": 0.9794108165050797, + "pass@8": 0.9826995305573731, + "pass@16": 0.9852891085277606, + "pass@32": 0.9874150370684425, + "pass@64": 0.9888440174400547, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9113627675840978, + "pass@2": 0.934330904307833, + "pass@4": 0.950269623407166, + "pass@8": 0.9616482628134414, + "pass@16": 0.9680019758410323, + "pass@32": 0.9717576392340137, + "pass@64": 0.9743729293524362, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8297405478395061, + "pass@2": 0.8684227514095461, + "pass@4": 0.8928393649867837, + "pass@8": 0.9073057009677743, + "pass@16": 0.916060516231829, + "pass@32": 0.9236679890966697, + "pass@64": 0.9314670859752837, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.82870456231454, + "pass@2": 0.87395368466553, + "pass@4": 0.898891906686738, + "pass@8": 0.9156222702891602, + "pass@16": 0.9299421512073494, + "pass@32": 0.9418732831664041, + "pass@64": 0.9518740184036798, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.7886525848765432, + "pass@2": 0.8476004301545641, + "pass@4": 0.8812799615325858, + "pass@8": 0.903221984034905, + "pass@16": 0.9189262347802786, + "pass@32": 0.9304864083273199, + "pass@64": 0.9390019202952004, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.634010877581121, + "pass@2": 0.740449271827747, + "pass@4": 0.8125692819924054, + "pass@8": 0.8575554027858113, + "pass@16": 0.8848759001996248, + "pass@32": 0.9038438545841817, + "pass@64": 0.9216124078198928, + "pass@128": 0.9380530973451328 + }, + "17": { + "pass@1": 0.517554593373494, + "pass@2": 0.6258230510625178, + "pass@4": 0.7037033872271984, + "pass@8": 0.756372995216946, + "pass@16": 0.794715859617898, + "pass@32": 0.8249475352261741, + "pass@64": 0.8486202280185033, + "pass@128": 0.8674698795180723 + }, + "18": { + "pass@1": 0.48220766129032255, + "pass@2": 0.5944191325882653, + "pass@4": 0.6840774846692548, + "pass@8": 0.7484527607317959, + "pass@16": 0.7932317734924506, + "pass@32": 0.8275928302086873, + "pass@64": 0.8547011083566157, + "pass@128": 0.8741935483870967 + }, + "19": { + "pass@1": 0.325080249244713, + "pass@2": 0.43810883864214883, + "pass@4": 0.5422776692188405, + "pass@8": 0.6265824302323246, + "pass@16": 0.6887125668408031, + "pass@32": 0.7329306621250592, + "pass@64": 0.767905495181644, + "pass@128": 0.797583081570997 + }, + "20": { + "pass@1": 0.29208896396396394, + "pass@2": 0.401044622378284, + "pass@4": 0.5034210889742385, + "pass@8": 0.5886495943967197, + "pass@16": 0.6554977161404892, + "pass@32": 0.7066888257456404, + "pass@64": 0.746441076359396, + "pass@128": 0.7777777777777778 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch4-op11-14/graphs/SDAR-100M-block8-epoch4-op11-14_score_report.jsonl b/composition/SDAR-100M-block8-epoch4-op11-14/graphs/SDAR-100M-block8-epoch4-op11-14_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c4558c85c9bad949e9b225d5feec8db5c4bbede7 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op11-14/graphs/SDAR-100M-block8-epoch4-op11-14_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch4-op11-14", "results_dir": "results/composition/SDAR-100M-block8-epoch4-op11-14/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1024304, "value_mismatch": 8922, "dependency_mismatch": 125612, "answer_mismatch": 7959, "extra_nodes": 9920, "missing_nodes": 8860, "perfect_with_extra": 581, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337505, "dependency_mismatch": 40863, "value_mismatch": 2977, "answer_mismatch": 2577, "missing_nodes": 3005, "extra_nodes": 3521, "perfect_with_extra": 259}, "teachers_in_school": {"total": 390656, "perfect": 346883, "dependency_mismatch": 43079, "value_mismatch": 3350, "answer_mismatch": 3117, "missing_nodes": 3271, "extra_nodes": 3445, "perfect_with_extra": 70}, "movie_festival_awards": {"total": 382336, "perfect": 339916, "dependency_mismatch": 41670, "extra_nodes": 2954, "missing_nodes": 2584, "perfect_with_extra": 252, "value_mismatch": 2595, "answer_mismatch": 2265}}, "results_path": "results/composition/SDAR-100M-block8-epoch4-op11-14/graphs/SDAR-100M-block8-epoch4-op11-14_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 446804, "value_mismatch": 528629, "dependency_mismatch": 738940, "answer_mismatch": 422096, "extra_nodes": 46044, "missing_nodes": 220737, "perfect_with_extra": 270, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 151460, "value_mismatch": 174921, "dependency_mismatch": 246142, "answer_mismatch": 138700, "missing_nodes": 77829, "extra_nodes": 18037, "perfect_with_extra": 150}, "movie_festival_awards": {"total": 418816, "perfect": 143394, "value_mismatch": 179668, "answer_mismatch": 144204, "dependency_mismatch": 243832, "extra_nodes": 15146, "missing_nodes": 73425, "perfect_with_extra": 19}, "teachers_in_school": {"total": 430464, "perfect": 151950, "value_mismatch": 174040, "dependency_mismatch": 248966, "answer_mismatch": 139192, "missing_nodes": 69483, "extra_nodes": 12861, "perfect_with_extra": 101}}, "results_path": "results/composition/SDAR-100M-block8-epoch4-op11-14/graphs/SDAR-100M-block8-epoch4-op11-14_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1471108, "value_mismatch": 537551, "dependency_mismatch": 864552, "answer_mismatch": 430055, "extra_nodes": 55964, "missing_nodes": 229597, "perfect_with_extra": 851, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch4-op11-14/graphs/SDAR-100M-block8-epoch4-op11-14_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 488965, "dependency_mismatch": 287005, "value_mismatch": 177898, "answer_mismatch": 141277, "missing_nodes": 80834, "extra_nodes": 21558, "perfect_with_extra": 409}, "teachers_in_school": {"total": 821120, "perfect": 498833, "dependency_mismatch": 292045, "value_mismatch": 177390, "answer_mismatch": 142309, "missing_nodes": 72754, "extra_nodes": 16306, "perfect_with_extra": 171}, "movie_festival_awards": {"total": 801152, "perfect": 483310, "dependency_mismatch": 285502, "extra_nodes": 18100, "missing_nodes": 76009, "perfect_with_extra": 271, "value_mismatch": 182263, "answer_mismatch": 146469}}}} diff --git a/composition/SDAR-100M-block8-epoch4-op11-14/graphs/results.json b/composition/SDAR-100M-block8-epoch4-op11-14/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..1943f6c7a4a07b205512dafa9dcf5adaa18377d9 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op11-14/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block8-epoch4-op11-14_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9492063492063492, + "pass@2": 0.9777777777777777, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9700598802395209, + "pass@2": 0.9760479041916168, + "pass@4": 0.9910179640718563, + "pass@8": 0.9940119760479041, + "pass@16": 0.9970059880239521, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8875739644970414, + "pass@2": 0.9289940828402367, + "pass@4": 0.9644970414201184, + "pass@8": 0.9911242603550295, + "pass@16": 0.9940828402366864, + "pass@32": 0.9970414201183432, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8301282051282052, + "pass@2": 0.8974358974358975, + "pass@4": 0.9551282051282052, + "pass@8": 0.9871794871794872, + "pass@16": 0.9967948717948718, + "pass@32": 0.9967948717948718, + "pass@64": 0.9967948717948718, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7830985915492957, + "pass@2": 0.8309859154929577, + "pass@4": 0.9183098591549296, + "pass@8": 0.971830985915493, + "pass@16": 0.9915492957746479, + "pass@32": 0.9943661971830986, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6578171091445427, + "pass@2": 0.7109144542772862, + "pass@4": 0.8082595870206489, + "pass@8": 0.8584070796460177, + "pass@16": 0.9056047197640118, + "pass@32": 0.9380530973451328, + "pass@64": 0.943952802359882, + "pass@128": 0.9587020648967551 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9492063492063492, + "6": 0.9700598802395209, + "7": 0.8875739644970414, + "8": 0.8301282051282052, + "9": 0.7830985915492957, + "10": 0.6578171091445427 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9777777777777777, + "6": 0.9760479041916168, + "7": 0.9289940828402367, + "8": 0.8974358974358975, + "9": 0.8309859154929577, + "10": 0.7109144542772862 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9910179640718563, + "7": 0.9644970414201184, + "8": 0.9551282051282052, + "9": 0.9183098591549296, + "10": 0.8082595870206489 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9940119760479041, + "7": 0.9911242603550295, + "8": 0.9871794871794872, + "9": 0.971830985915493, + "10": 0.8584070796460177 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9970059880239521, + "7": 0.9940828402366864, + "8": 0.9967948717948718, + "9": 0.9915492957746479, + "10": 0.9056047197640118 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 0.9943661971830986, + "10": 0.9380530973451328 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.943952802359882 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9587020648967551 + } + }, + "total_pass_at_k": { + "pass@1": 0.895305639986491, + "pass@2": 0.922661263086795, + "pass@4": 0.9584599797365755, + "pass@8": 0.9773725092874029, + "pass@16": 0.9868287740628167, + "pass@32": 0.9915569064505235, + "pass@64": 0.9932455251604188, + "pass@128": 0.9949341438703141 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9912280701754386, + "pass@2": 0.9970760233918129, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9588235294117647, + "pass@2": 0.9941176470588236, + "pass@4": 0.9970588235294118, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9621451104100947, + "pass@2": 0.9716088328075709, + "pass@4": 0.9842271293375394, + "pass@8": 0.9968454258675079, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.870253164556962, + "pass@2": 0.9113924050632911, + "pass@4": 0.9525316455696202, + "pass@8": 0.9810126582278481, + "pass@16": 0.990506329113924, + "pass@32": 0.9968354430379747, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8048048048048048, + "pass@2": 0.8888888888888888, + "pass@4": 0.9459459459459459, + "pass@8": 0.975975975975976, + "pass@16": 0.990990990990991, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7620578778135049, + "pass@2": 0.8135048231511254, + "pass@4": 0.8520900321543409, + "pass@8": 0.9356913183279743, + "pass@16": 0.977491961414791, + "pass@32": 0.9935691318327974, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6298507462686567, + "pass@2": 0.6776119402985075, + "pass@4": 0.7492537313432835, + "pass@8": 0.826865671641791, + "pass@16": 0.9104477611940298, + "pass@32": 0.9402985074626866, + "pass@64": 0.9611940298507463, + "pass@128": 0.9671641791044776 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9588235294117647, + "6": 0.9621451104100947, + "7": 0.870253164556962, + "8": 0.8048048048048048, + "9": 0.7620578778135049, + "10": 0.6298507462686567 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9941176470588236, + "6": 0.9716088328075709, + "7": 0.9113924050632911, + "8": 0.8888888888888888, + "9": 0.8135048231511254, + "10": 0.6776119402985075 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9842271293375394, + "7": 0.9525316455696202, + "8": 0.9459459459459459, + "9": 0.8520900321543409, + "10": 0.7492537313432835 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9968454258675079, + "7": 0.9810126582278481, + "8": 0.975975975975976, + "9": 0.9356913183279743, + "10": 0.826865671641791 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.990506329113924, + "8": 0.990990990990991, + "9": 0.977491961414791, + "10": 0.9104477611940298 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9968354430379747, + "8": 1.0, + "9": 0.9935691318327974, + "10": 0.9402985074626866 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9611940298507463 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9671641791044776 + } + }, + "total_pass_at_k": { + "pass@1": 0.8885169065952461, + "pass@2": 0.9186474723803147, + "pass@4": 0.9434214931369267, + "pass@8": 0.9688650820220958, + "pass@16": 0.9856042852360227, + "pass@32": 0.9922999665215936, + "pass@64": 0.995647807164379, + "pass@128": 0.9963173752929361 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985207100591716, + "pass@2": 0.9940828402366864, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9478260869565217, + "pass@2": 0.9797101449275363, + "pass@4": 0.9971014492753624, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9770773638968482, + "pass@2": 0.9856733524355301, + "pass@4": 0.9885386819484241, + "pass@8": 0.9914040114613181, + "pass@16": 0.994269340974212, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.8930635838150289, + "pass@2": 0.9335260115606936, + "pass@4": 0.9595375722543352, + "pass@8": 0.9855491329479769, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8056338028169014, + "pass@2": 0.8788732394366198, + "pass@4": 0.9464788732394366, + "pass@8": 0.9887323943661972, + "pass@16": 0.9943661971830986, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7485029940119761, + "pass@2": 0.811377245508982, + "pass@4": 0.8802395209580839, + "pass@8": 0.937125748502994, + "pass@16": 0.9820359281437125, + "pass@32": 0.9970059880239521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6503067484662577, + "pass@2": 0.7116564417177914, + "pass@4": 0.7883435582822086, + "pass@8": 0.8619631901840491, + "pass@16": 0.8987730061349694, + "pass@32": 0.9294478527607362, + "pass@64": 0.9447852760736196, + "pass@128": 0.9539877300613497 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.985207100591716, + "5": 0.9478260869565217, + "6": 0.9770773638968482, + "7": 0.8930635838150289, + "8": 0.8056338028169014, + "9": 0.7485029940119761, + "10": 0.6503067484662577 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9940828402366864, + "5": 0.9797101449275363, + "6": 0.9856733524355301, + "7": 0.9335260115606936, + "8": 0.8788732394366198, + "9": 0.811377245508982, + "10": 0.7116564417177914 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.9885386819484241, + "7": 0.9595375722543352, + "8": 0.9464788732394366, + "9": 0.8802395209580839, + "10": 0.7883435582822086 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9914040114613181, + "7": 0.9855491329479769, + "8": 0.9887323943661972, + "9": 0.937125748502994, + "10": 0.8619631901840491 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.994269340974212, + "7": 1.0, + "8": 0.9943661971830986, + "9": 0.9820359281437125, + "10": 0.8987730061349694 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 1.0, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.9294478527607362 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9447852760736196 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9539877300613497 + } + }, + "total_pass_at_k": { + "pass@1": 0.8902359108781127, + "pass@2": 0.922346002621232, + "pass@4": 0.9518348623853211, + "pass@8": 0.9744429882044561, + "pass@16": 0.9859108781127129, + "pass@32": 0.9918086500655308, + "pass@64": 0.9937745740498034, + "pass@128": 0.9947575360419397 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.992, + "pass@2": 0.997, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.952, + "pass@2": 0.984, + "pass@4": 0.998, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.97, + "pass@2": 0.978, + "pass@4": 0.988, + "pass@8": 0.994, + "pass@16": 0.997, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.884, + "pass@2": 0.925, + "pass@4": 0.959, + "pass@8": 0.986, + "pass@16": 0.995, + "pass@32": 0.998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.813, + "pass@2": 0.888, + "pass@4": 0.949, + "pass@8": 0.984, + "pass@16": 0.994, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.765, + "pass@2": 0.819, + "pass@4": 0.885, + "pass@8": 0.949, + "pass@16": 0.984, + "pass@32": 0.995, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.646, + "pass@2": 0.7, + "pass@4": 0.782, + "pass@8": 0.849, + "pass@16": 0.905, + "pass@32": 0.936, + "pass@64": 0.95, + "pass@128": 0.96 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.992, + "5": 0.952, + "6": 0.97, + "7": 0.884, + "8": 0.813, + "9": 0.765, + "10": 0.646 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.997, + "5": 0.984, + "6": 0.978, + "7": 0.925, + "8": 0.888, + "9": 0.819, + "10": 0.7 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 0.988, + "7": 0.959, + "8": 0.949, + "9": 0.885, + "10": 0.782 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.994, + "7": 0.986, + "8": 0.984, + "9": 0.949, + "10": 0.849 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997, + "7": 0.995, + "8": 0.994, + "9": 0.984, + "10": 0.905 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.999, + "7": 0.998, + "8": 0.999, + "9": 0.995, + "10": 0.936 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.999, + "7": 1.0, + "8": 0.999, + "9": 1.0, + "10": 0.95 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.999, + "7": 1.0, + "8": 0.999, + "9": 1.0, + "10": 0.96 + } + }, + "total_pass_at_k": { + "pass@1": 0.8913333333333333, + "pass@2": 0.9212222222222223, + "pass@4": 0.9512222222222222, + "pass@8": 0.9735555555555555, + "pass@16": 0.9861111111111112, + "pass@32": 0.9918888888888889, + "pass@64": 0.9942222222222222, + "pass@128": 0.9953333333333333 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block8-epoch4-op11-14_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5637393767705382, + "pass@2": 0.6430594900849859, + "pass@4": 0.7308781869688386, + "pass@8": 0.8215297450424929, + "pass@16": 0.8951841359773371, + "pass@32": 0.9235127478753541, + "pass@64": 0.9320113314447592, + "pass@128": 0.9490084985835694 + }, + "12": { + "pass@1": 0.6073446327683616, + "pass@2": 0.655367231638418, + "pass@4": 0.7344632768361582, + "pass@8": 0.7824858757062146, + "pass@16": 0.827683615819209, + "pass@32": 0.8361581920903954, + "pass@64": 0.8559322033898306, + "pass@128": 0.867231638418079 + }, + "13": { + "pass@1": 0.5015673981191222, + "pass@2": 0.5391849529780565, + "pass@4": 0.6238244514106583, + "pass@8": 0.6677115987460815, + "pass@16": 0.7241379310344828, + "pass@32": 0.780564263322884, + "pass@64": 0.7931034482758621, + "pass@128": 0.8181818181818182 + }, + "14": { + "pass@1": 0.46417445482866043, + "pass@2": 0.5109034267912772, + "pass@4": 0.5638629283489096, + "pass@8": 0.632398753894081, + "pass@16": 0.6947040498442367, + "pass@32": 0.7601246105919003, + "pass@64": 0.8193146417445483, + "pass@128": 0.8473520249221184 + }, + "15": { + "pass@1": 0.3872832369942196, + "pass@2": 0.43352601156069365, + "pass@4": 0.5028901734104047, + "pass@8": 0.5549132947976878, + "pass@16": 0.630057803468208, + "pass@32": 0.7052023121387283, + "pass@64": 0.8005780346820809, + "pass@128": 0.8323699421965318 + }, + "16": { + "pass@1": 0.29354838709677417, + "pass@2": 0.3419354838709677, + "pass@4": 0.38387096774193546, + "pass@8": 0.4645161290322581, + "pass@16": 0.5096774193548387, + "pass@32": 0.5387096774193548, + "pass@64": 0.5967741935483871, + "pass@128": 0.6548387096774193 + }, + "17": { + "pass@1": 0.2630057803468208, + "pass@2": 0.3265895953757225, + "pass@4": 0.40173410404624277, + "pass@8": 0.430635838150289, + "pass@16": 0.4884393063583815, + "pass@32": 0.5346820809248555, + "pass@64": 0.5809248554913294, + "pass@128": 0.630057803468208 + }, + "18": { + "pass@1": 0.2057142857142857, + "pass@2": 0.24571428571428572, + "pass@4": 0.28285714285714286, + "pass@8": 0.32285714285714284, + "pass@16": 0.3657142857142857, + "pass@32": 0.4114285714285714, + "pass@64": 0.48857142857142855, + "pass@128": 0.5628571428571428 + }, + "19": { + "pass@1": 0.08708708708708708, + "pass@2": 0.11411411411411411, + "pass@4": 0.14414414414414414, + "pass@8": 0.17417417417417416, + "pass@16": 0.2012012012012012, + "pass@32": 0.24024024024024024, + "pass@64": 0.2672672672672673, + "pass@128": 0.3033033033033033 + }, + "20": { + "pass@1": 0.12312312312312312, + "pass@2": 0.14414414414414414, + "pass@4": 0.18018018018018017, + "pass@8": 0.23123123123123124, + "pass@16": 0.24324324324324326, + "pass@32": 0.2732732732732733, + "pass@64": 0.2972972972972973, + "pass@128": 0.32732732732732733 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5637393767705382, + "12": 0.6073446327683616, + "13": 0.5015673981191222, + "14": 0.46417445482866043, + "15": 0.3872832369942196, + "16": 0.29354838709677417, + "17": 0.2630057803468208, + "18": 0.2057142857142857, + "19": 0.08708708708708708, + "20": 0.12312312312312312 + }, + "pass@2": { + "11": 0.6430594900849859, + "12": 0.655367231638418, + "13": 0.5391849529780565, + "14": 0.5109034267912772, + "15": 0.43352601156069365, + "16": 0.3419354838709677, + "17": 0.3265895953757225, + "18": 0.24571428571428572, + "19": 0.11411411411411411, + "20": 0.14414414414414414 + }, + "pass@4": { + "11": 0.7308781869688386, + "12": 0.7344632768361582, + "13": 0.6238244514106583, + "14": 0.5638629283489096, + "15": 0.5028901734104047, + "16": 0.38387096774193546, + "17": 0.40173410404624277, + "18": 0.28285714285714286, + "19": 0.14414414414414414, + "20": 0.18018018018018017 + }, + "pass@8": { + "11": 0.8215297450424929, + "12": 0.7824858757062146, + "13": 0.6677115987460815, + "14": 0.632398753894081, + "15": 0.5549132947976878, + "16": 0.4645161290322581, + "17": 0.430635838150289, + "18": 0.32285714285714284, + "19": 0.17417417417417416, + "20": 0.23123123123123124 + }, + "pass@16": { + "11": 0.8951841359773371, + "12": 0.827683615819209, + "13": 0.7241379310344828, + "14": 0.6947040498442367, + "15": 0.630057803468208, + "16": 0.5096774193548387, + "17": 0.4884393063583815, + "18": 0.3657142857142857, + "19": 0.2012012012012012, + "20": 0.24324324324324326 + }, + "pass@32": { + "11": 0.9235127478753541, + "12": 0.8361581920903954, + "13": 0.780564263322884, + "14": 0.7601246105919003, + "15": 0.7052023121387283, + "16": 0.5387096774193548, + "17": 0.5346820809248555, + "18": 0.4114285714285714, + "19": 0.24024024024024024, + "20": 0.2732732732732733 + }, + "pass@64": { + "11": 0.9320113314447592, + "12": 0.8559322033898306, + "13": 0.7931034482758621, + "14": 0.8193146417445483, + "15": 0.8005780346820809, + "16": 0.5967741935483871, + "17": 0.5809248554913294, + "18": 0.48857142857142855, + "19": 0.2672672672672673, + "20": 0.2972972972972973 + }, + "pass@128": { + "11": 0.9490084985835694, + "12": 0.867231638418079, + "13": 0.8181818181818182, + "14": 0.8473520249221184, + "15": 0.8323699421965318, + "16": 0.6548387096774193, + "17": 0.630057803468208, + "18": 0.5628571428571428, + "19": 0.3033033033033033, + "20": 0.32732732732732733 + } + }, + "total_pass_at_k": { + "pass@1": 0.35096582466567605, + "pass@2": 0.39702823179791974, + "pass@4": 0.45676077265973253, + "pass@8": 0.5099554234769688, + "pass@16": 0.5598811292719168, + "pass@32": 0.6020802377414561, + "pass@64": 0.6448736998514116, + "pass@128": 0.6808320950965825 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5841269841269842, + "pass@2": 0.6603174603174603, + "pass@4": 0.7555555555555555, + "pass@8": 0.8666666666666667, + "pass@16": 0.926984126984127, + "pass@32": 0.9587301587301588, + "pass@64": 0.9714285714285714, + "pass@128": 0.9777777777777777 + }, + "12": { + "pass@1": 0.599388379204893, + "pass@2": 0.6483180428134556, + "pass@4": 0.72782874617737, + "pass@8": 0.8165137614678899, + "pass@16": 0.8562691131498471, + "pass@32": 0.8776758409785933, + "pass@64": 0.8837920489296636, + "pass@128": 0.8960244648318043 + }, + "13": { + "pass@1": 0.5061728395061729, + "pass@2": 0.5648148148148148, + "pass@4": 0.6172839506172839, + "pass@8": 0.6697530864197531, + "pass@16": 0.7191358024691358, + "pass@32": 0.7438271604938271, + "pass@64": 0.7685185185185185, + "pass@128": 0.7870370370370371 + }, + "14": { + "pass@1": 0.4391691394658754, + "pass@2": 0.4807121661721068, + "pass@4": 0.5548961424332344, + "pass@8": 0.6112759643916914, + "pass@16": 0.6735905044510386, + "pass@32": 0.7359050445103857, + "pass@64": 0.8338278931750742, + "pass@128": 0.8545994065281899 + }, + "15": { + "pass@1": 0.38580246913580246, + "pass@2": 0.42592592592592593, + "pass@4": 0.49382716049382713, + "pass@8": 0.5462962962962963, + "pass@16": 0.6358024691358025, + "pass@32": 0.6944444444444444, + "pass@64": 0.7469135802469136, + "pass@128": 0.8117283950617284 + }, + "16": { + "pass@1": 0.336283185840708, + "pass@2": 0.3775811209439528, + "pass@4": 0.4365781710914454, + "pass@8": 0.4808259587020649, + "pass@16": 0.528023598820059, + "pass@32": 0.6017699115044248, + "pass@64": 0.6342182890855457, + "pass@128": 0.6814159292035398 + }, + "17": { + "pass@1": 0.22289156626506024, + "pass@2": 0.2921686746987952, + "pass@4": 0.3373493975903614, + "pass@8": 0.37349397590361444, + "pass@16": 0.42771084337349397, + "pass@32": 0.46987951807228917, + "pass@64": 0.5240963855421686, + "pass@128": 0.5843373493975904 + }, + "18": { + "pass@1": 0.2129032258064516, + "pass@2": 0.25483870967741934, + "pass@4": 0.2870967741935484, + "pass@8": 0.33548387096774196, + "pass@16": 0.3774193548387097, + "pass@32": 0.41935483870967744, + "pass@64": 0.47096774193548385, + "pass@128": 0.5193548387096775 + }, + "19": { + "pass@1": 0.09063444108761329, + "pass@2": 0.11178247734138973, + "pass@4": 0.14803625377643503, + "pass@8": 0.1933534743202417, + "pass@16": 0.2326283987915408, + "pass@32": 0.2598187311178248, + "pass@64": 0.27492447129909364, + "pass@128": 0.31419939577039274 + }, + "20": { + "pass@1": 0.06606606606606606, + "pass@2": 0.0960960960960961, + "pass@4": 0.11711711711711711, + "pass@8": 0.15315315315315314, + "pass@16": 0.16816816816816818, + "pass@32": 0.18618618618618618, + "pass@64": 0.21921921921921922, + "pass@128": 0.24324324324324326 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5841269841269842, + "12": 0.599388379204893, + "13": 0.5061728395061729, + "14": 0.4391691394658754, + "15": 0.38580246913580246, + "16": 0.336283185840708, + "17": 0.22289156626506024, + "18": 0.2129032258064516, + "19": 0.09063444108761329, + "20": 0.06606606606606606 + }, + "pass@2": { + "11": 0.6603174603174603, + "12": 0.6483180428134556, + "13": 0.5648148148148148, + "14": 0.4807121661721068, + "15": 0.42592592592592593, + "16": 0.3775811209439528, + "17": 0.2921686746987952, + "18": 0.25483870967741934, + "19": 0.11178247734138973, + "20": 0.0960960960960961 + }, + "pass@4": { + "11": 0.7555555555555555, + "12": 0.72782874617737, + "13": 0.6172839506172839, + "14": 0.5548961424332344, + "15": 0.49382716049382713, + "16": 0.4365781710914454, + "17": 0.3373493975903614, + "18": 0.2870967741935484, + "19": 0.14803625377643503, + "20": 0.11711711711711711 + }, + "pass@8": { + "11": 0.8666666666666667, + "12": 0.8165137614678899, + "13": 0.6697530864197531, + "14": 0.6112759643916914, + "15": 0.5462962962962963, + "16": 0.4808259587020649, + "17": 0.37349397590361444, + "18": 0.33548387096774196, + "19": 0.1933534743202417, + "20": 0.15315315315315314 + }, + "pass@16": { + "11": 0.926984126984127, + "12": 0.8562691131498471, + "13": 0.7191358024691358, + "14": 0.6735905044510386, + "15": 0.6358024691358025, + "16": 0.528023598820059, + "17": 0.42771084337349397, + "18": 0.3774193548387097, + "19": 0.2326283987915408, + "20": 0.16816816816816818 + }, + "pass@32": { + "11": 0.9587301587301588, + "12": 0.8776758409785933, + "13": 0.7438271604938271, + "14": 0.7359050445103857, + "15": 0.6944444444444444, + "16": 0.6017699115044248, + "17": 0.46987951807228917, + "18": 0.41935483870967744, + "19": 0.2598187311178248, + "20": 0.18618618618618618 + }, + "pass@64": { + "11": 0.9714285714285714, + "12": 0.8837920489296636, + "13": 0.7685185185185185, + "14": 0.8338278931750742, + "15": 0.7469135802469136, + "16": 0.6342182890855457, + "17": 0.5240963855421686, + "18": 0.47096774193548385, + "19": 0.27492447129909364, + "20": 0.21921921921921922 + }, + "pass@128": { + "11": 0.9777777777777777, + "12": 0.8960244648318043, + "13": 0.7870370370370371, + "14": 0.8545994065281899, + "15": 0.8117283950617284, + "16": 0.6814159292035398, + "17": 0.5843373493975904, + "18": 0.5193548387096775, + "19": 0.31419939577039274, + "20": 0.24324324324324326 + } + }, + "total_pass_at_k": { + "pass@1": 0.3432151589242054, + "pass@2": 0.3899755501222494, + "pass@4": 0.44621026894865523, + "pass@8": 0.5030562347188264, + "pass@16": 0.5528728606356969, + "pass@32": 0.5932151589242054, + "pass@64": 0.6314180929095354, + "pass@128": 0.6656479217603912 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6475903614457831, + "pass@2": 0.6957831325301205, + "pass@4": 0.7620481927710844, + "pass@8": 0.8493975903614458, + "pass@16": 0.9246987951807228, + "pass@32": 0.9427710843373494, + "pass@64": 0.9608433734939759, + "pass@128": 0.9759036144578314 + }, + "12": { + "pass@1": 0.5611285266457681, + "pass@2": 0.6394984326018809, + "pass@4": 0.7178683385579937, + "pass@8": 0.7586206896551724, + "pass@16": 0.8087774294670846, + "pass@32": 0.8495297805642633, + "pass@64": 0.8714733542319749, + "pass@128": 0.8746081504702194 + }, + "13": { + "pass@1": 0.5294117647058824, + "pass@2": 0.6022408963585434, + "pass@4": 0.6554621848739496, + "pass@8": 0.742296918767507, + "pass@16": 0.7899159663865546, + "pass@32": 0.8207282913165266, + "pass@64": 0.8403361344537815, + "pass@128": 0.8515406162464986 + }, + "14": { + "pass@1": 0.4824561403508772, + "pass@2": 0.5467836257309941, + "pass@4": 0.5877192982456141, + "pass@8": 0.6374269005847953, + "pass@16": 0.6900584795321637, + "pass@32": 0.7690058479532164, + "pass@64": 0.804093567251462, + "pass@128": 0.8362573099415205 + }, + "15": { + "pass@1": 0.4212121212121212, + "pass@2": 0.47878787878787876, + "pass@4": 0.5393939393939394, + "pass@8": 0.5848484848484848, + "pass@16": 0.6666666666666666, + "pass@32": 0.7424242424242424, + "pass@64": 0.8121212121212121, + "pass@128": 0.8666666666666667 + }, + "16": { + "pass@1": 0.3076923076923077, + "pass@2": 0.36752136752136755, + "pass@4": 0.42735042735042733, + "pass@8": 0.4672364672364672, + "pass@16": 0.5071225071225072, + "pass@32": 0.584045584045584, + "pass@64": 0.6438746438746439, + "pass@128": 0.6894586894586895 + }, + "17": { + "pass@1": 0.2391304347826087, + "pass@2": 0.27639751552795033, + "pass@4": 0.30745341614906835, + "pass@8": 0.36335403726708076, + "pass@16": 0.4161490683229814, + "pass@32": 0.468944099378882, + "pass@64": 0.5217391304347826, + "pass@128": 0.577639751552795 + }, + "18": { + "pass@1": 0.22941176470588234, + "pass@2": 0.27647058823529413, + "pass@4": 0.3088235294117647, + "pass@8": 0.35294117647058826, + "pass@16": 0.39705882352941174, + "pass@32": 0.4235294117647059, + "pass@64": 0.4588235294117647, + "pass@128": 0.5058823529411764 + }, + "19": { + "pass@1": 0.07738095238095238, + "pass@2": 0.11904761904761904, + "pass@4": 0.16071428571428573, + "pass@8": 0.19940476190476192, + "pass@16": 0.22916666666666666, + "pass@32": 0.25297619047619047, + "pass@64": 0.28273809523809523, + "pass@128": 0.3244047619047619 + }, + "20": { + "pass@1": 0.08982035928143713, + "pass@2": 0.11676646706586827, + "pass@4": 0.1467065868263473, + "pass@8": 0.15868263473053892, + "pass@16": 0.18862275449101795, + "pass@32": 0.20958083832335328, + "pass@64": 0.23053892215568864, + "pass@128": 0.25149700598802394 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6475903614457831, + "12": 0.5611285266457681, + "13": 0.5294117647058824, + "14": 0.4824561403508772, + "15": 0.4212121212121212, + "16": 0.3076923076923077, + "17": 0.2391304347826087, + "18": 0.22941176470588234, + "19": 0.07738095238095238, + "20": 0.08982035928143713 + }, + "pass@2": { + "11": 0.6957831325301205, + "12": 0.6394984326018809, + "13": 0.6022408963585434, + "14": 0.5467836257309941, + "15": 0.47878787878787876, + "16": 0.36752136752136755, + "17": 0.27639751552795033, + "18": 0.27647058823529413, + "19": 0.11904761904761904, + "20": 0.11676646706586827 + }, + "pass@4": { + "11": 0.7620481927710844, + "12": 0.7178683385579937, + "13": 0.6554621848739496, + "14": 0.5877192982456141, + "15": 0.5393939393939394, + "16": 0.42735042735042733, + "17": 0.30745341614906835, + "18": 0.3088235294117647, + "19": 0.16071428571428573, + "20": 0.1467065868263473 + }, + "pass@8": { + "11": 0.8493975903614458, + "12": 0.7586206896551724, + "13": 0.742296918767507, + "14": 0.6374269005847953, + "15": 0.5848484848484848, + "16": 0.4672364672364672, + "17": 0.36335403726708076, + "18": 0.35294117647058826, + "19": 0.19940476190476192, + "20": 0.15868263473053892 + }, + "pass@16": { + "11": 0.9246987951807228, + "12": 0.8087774294670846, + "13": 0.7899159663865546, + "14": 0.6900584795321637, + "15": 0.6666666666666666, + "16": 0.5071225071225072, + "17": 0.4161490683229814, + "18": 0.39705882352941174, + "19": 0.22916666666666666, + "20": 0.18862275449101795 + }, + "pass@32": { + "11": 0.9427710843373494, + "12": 0.8495297805642633, + "13": 0.8207282913165266, + "14": 0.7690058479532164, + "15": 0.7424242424242424, + "16": 0.584045584045584, + "17": 0.468944099378882, + "18": 0.4235294117647059, + "19": 0.25297619047619047, + "20": 0.20958083832335328 + }, + "pass@64": { + "11": 0.9608433734939759, + "12": 0.8714733542319749, + "13": 0.8403361344537815, + "14": 0.804093567251462, + "15": 0.8121212121212121, + "16": 0.6438746438746439, + "17": 0.5217391304347826, + "18": 0.4588235294117647, + "19": 0.28273809523809523, + "20": 0.23053892215568864 + }, + "pass@128": { + "11": 0.9759036144578314, + "12": 0.8746081504702194, + "13": 0.8515406162464986, + "14": 0.8362573099415205, + "15": 0.8666666666666667, + "16": 0.6894586894586895, + "17": 0.577639751552795, + "18": 0.5058823529411764, + "19": 0.3244047619047619, + "20": 0.25149700598802394 + } + }, + "total_pass_at_k": { + "pass@1": 0.3586083853702052, + "pass@2": 0.41213202497769846, + "pass@4": 0.4614927148379423, + "pass@8": 0.511745465358311, + "pass@16": 0.5619982158786797, + "pass@32": 0.6066012488849242, + "pass@64": 0.6428783823966696, + "pass@128": 0.6755872732679156 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.598, + "pass@2": 0.666, + "pass@4": 0.749, + "pass@8": 0.845, + "pass@16": 0.915, + "pass@32": 0.941, + "pass@64": 0.954, + "pass@128": 0.967 + }, + "12": { + "pass@1": 0.59, + "pass@2": 0.648, + "pass@4": 0.727, + "pass@8": 0.786, + "pass@16": 0.831, + "pass@32": 0.854, + "pass@64": 0.87, + "pass@128": 0.879 + }, + "13": { + "pass@1": 0.513, + "pass@2": 0.57, + "pass@4": 0.633, + "pass@8": 0.695, + "pass@16": 0.746, + "pass@32": 0.783, + "pass@64": 0.802, + "pass@128": 0.82 + }, + "14": { + "pass@1": 0.462, + "pass@2": 0.513, + "pass@4": 0.569, + "pass@8": 0.627, + "pass@16": 0.686, + "pass@32": 0.755, + "pass@64": 0.819, + "pass@128": 0.846 + }, + "15": { + "pass@1": 0.398, + "pass@2": 0.446, + "pass@4": 0.512, + "pass@8": 0.562, + "pass@16": 0.644, + "pass@32": 0.714, + "pass@64": 0.787, + "pass@128": 0.837 + }, + "16": { + "pass@1": 0.313, + "pass@2": 0.363, + "pass@4": 0.417, + "pass@8": 0.471, + "pass@16": 0.515, + "pass@32": 0.576, + "pass@64": 0.626, + "pass@128": 0.676 + }, + "17": { + "pass@1": 0.242, + "pass@2": 0.299, + "pass@4": 0.35, + "pass@8": 0.39, + "pass@16": 0.445, + "pass@32": 0.492, + "pass@64": 0.543, + "pass@128": 0.598 + }, + "18": { + "pass@1": 0.216, + "pass@2": 0.259, + "pass@4": 0.293, + "pass@8": 0.337, + "pass@16": 0.38, + "pass@32": 0.418, + "pass@64": 0.473, + "pass@128": 0.53 + }, + "19": { + "pass@1": 0.085, + "pass@2": 0.115, + "pass@4": 0.151, + "pass@8": 0.189, + "pass@16": 0.221, + "pass@32": 0.251, + "pass@64": 0.275, + "pass@128": 0.314 + }, + "20": { + "pass@1": 0.093, + "pass@2": 0.119, + "pass@4": 0.148, + "pass@8": 0.181, + "pass@16": 0.2, + "pass@32": 0.223, + "pass@64": 0.249, + "pass@128": 0.274 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.598, + "12": 0.59, + "13": 0.513, + "14": 0.462, + "15": 0.398, + "16": 0.313, + "17": 0.242, + "18": 0.216, + "19": 0.085, + "20": 0.093 + }, + "pass@2": { + "11": 0.666, + "12": 0.648, + "13": 0.57, + "14": 0.513, + "15": 0.446, + "16": 0.363, + "17": 0.299, + "18": 0.259, + "19": 0.115, + "20": 0.119 + }, + "pass@4": { + "11": 0.749, + "12": 0.727, + "13": 0.633, + "14": 0.569, + "15": 0.512, + "16": 0.417, + "17": 0.35, + "18": 0.293, + "19": 0.151, + "20": 0.148 + }, + "pass@8": { + "11": 0.845, + "12": 0.786, + "13": 0.695, + "14": 0.627, + "15": 0.562, + "16": 0.471, + "17": 0.39, + "18": 0.337, + "19": 0.189, + "20": 0.181 + }, + "pass@16": { + "11": 0.915, + "12": 0.831, + "13": 0.746, + "14": 0.686, + "15": 0.644, + "16": 0.515, + "17": 0.445, + "18": 0.38, + "19": 0.221, + "20": 0.2 + }, + "pass@32": { + "11": 0.941, + "12": 0.854, + "13": 0.783, + "14": 0.755, + "15": 0.714, + "16": 0.576, + "17": 0.492, + "18": 0.418, + "19": 0.251, + "20": 0.223 + }, + "pass@64": { + "11": 0.954, + "12": 0.87, + "13": 0.802, + "14": 0.819, + "15": 0.787, + "16": 0.626, + "17": 0.543, + "18": 0.473, + "19": 0.275, + "20": 0.249 + }, + "pass@128": { + "11": 0.967, + "12": 0.879, + "13": 0.82, + "14": 0.846, + "15": 0.837, + "16": 0.676, + "17": 0.598, + "18": 0.53, + "19": 0.314, + "20": 0.274 + } + }, + "total_pass_at_k": { + "pass@1": 0.351, + "pass@2": 0.3998, + "pass@4": 0.4549, + "pass@8": 0.5083, + "pass@16": 0.5583, + "pass@32": 0.6007, + "pass@64": 0.6398, + "pass@128": 0.6741 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block8-epoch4-op11-14/summary.csv b/composition/SDAR-100M-block8-epoch4-op11-14/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..f62b6183b8845b37d35c8982b8db77975e5a5473 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op11-14/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch4-op11-14,0.9930911458333334,0.67023828125,0.8231685855263158,0.0,0.0,0.0,199.22368663194445,370.87193515625,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block8-epoch4-op11-14/summary.json b/composition/SDAR-100M-block8-epoch4-op11-14/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7589737642c2edd5e189661391ce4d753c86790f --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op11-14/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch4-op11-14", + "id_acc": 0.9930911458333334, + "ood_acc": 0.67023828125, + "total_acc": 0.8231685855263158, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.22368663194445, + "ood_avg_resp_len": 370.87193515625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 668453, + "answer_accuracy": 0.8255278315681315, + "avg_response_len": 276.6914371245653, + "resp_tokens_sum": 224044804, + "pass_at_k": { + "pass@1": 0.8255278315681315, + "pass@2": 0.861867058770073, + "pass@4": 0.8895697888775553, + "pass@8": 0.9109484207704667, + "pass@16": 0.9271942329187942, + "pass@32": 0.93941210538085, + "pass@64": 0.9488117816403033, + "pass@128": 0.9565286120771419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9550377949852508, + "pass@2": 0.9653236272733609, + "pass@4": 0.9726180897741764, + "pass@8": 0.9783762148389291, + "pass@16": 0.9829116146743994, + "pass@32": 0.9870127565970104, + "pass@64": 0.991180340449138, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 0.9997809579439252, + "pass@2": 0.9999862020752079, + "pass@4": 0.9999999632055339, + "pass@8": 0.9999999999999803, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.998779296875, + "pass@2": 0.9995659295029528, + "pass@4": 0.9999430012654666, + "pass@8": 0.9999991752021973, + "pass@16": 0.9999999999245107, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9982638888888888, + "pass@2": 0.9995289651293588, + "pass@4": 0.9999661601823582, + "pass@8": 0.9999998458721072, + "pass@16": 0.9999999999987073, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9989006362275449, + "pass@2": 0.9996018052477722, + "pass@4": 0.9999499412311484, + "pass@8": 0.9999993414788002, + "pass@16": 0.9999999999517831, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999075443786982, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9933143028846154, + "pass@2": 0.997447096456693, + "pass@4": 0.999438021268976, + "pass@8": 0.9999627194608325, + "pass@16": 0.9999998418611676, + "pass@32": 0.9999999999993773, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9960607394366198, + "pass@2": 0.9976620550072086, + "pass@4": 0.9985522935513342, + "pass@8": 0.9993061284706786, + "pass@16": 0.9998463955935609, + "pass@32": 0.9999949107316761, + "pass@64": 0.9999999995381218, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9609375, + "pass@2": 0.9711451060649998, + "pass@4": 0.9770741222857058, + "pass@8": 0.9812611025364644, + "pass@16": 0.9842944002686944, + "pass@32": 0.9861828830088263, + "pass@64": 0.9872429473453521, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.9021671963276836, + "pass@2": 0.9215281484941499, + "pass@4": 0.9347171227537232, + "pass@8": 0.9446575312767541, + "pass@16": 0.9525455705033121, + "pass@32": 0.9580749193035334, + "pass@64": 0.9613374661256407, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.8692202194357367, + "pass@2": 0.9024653352010464, + "pass@4": 0.9221733035721631, + "pass@8": 0.9359308821175786, + "pass@16": 0.9484843594916188, + "pass@32": 0.960115260632947, + "pass@64": 0.9696726490165624, + "pass@128": 0.9780564263322884 + }, + "14": { + "pass@1": 0.8432389018691588, + "pass@2": 0.8816467669929107, + "pass@4": 0.9054283430692655, + "pass@8": 0.9252522133459742, + "pass@16": 0.9410032100908022, + "pass@32": 0.9510890172203004, + "pass@64": 0.9586071337072141, + "pass@128": 0.9688473520249221 + }, + "15": { + "pass@1": 0.8159320809248555, + "pass@2": 0.8758590869782895, + "pass@4": 0.9136971456963835, + "pass@8": 0.9391793646317208, + "pass@16": 0.9568471423736444, + "pass@32": 0.9685935199216499, + "pass@64": 0.9754437045824256, + "pass@128": 0.9797687861271677 + }, + "16": { + "pass@1": 0.6567792338709677, + "pass@2": 0.740235664846329, + "pass@4": 0.795893839278154, + "pass@8": 0.8355960749083245, + "pass@16": 0.864558190334898, + "pass@32": 0.8889121743620298, + "pass@64": 0.9101876710408604, + "pass@128": 0.9258064516129032 + }, + "17": { + "pass@1": 0.5896405346820809, + "pass@2": 0.689234175162714, + "pass@4": 0.7562385357538394, + "pass@8": 0.7996128076425166, + "pass@16": 0.8318120005184033, + "pass@32": 0.8596979457171261, + "pass@64": 0.8843409721762664, + "pass@128": 0.9046242774566474 + }, + "18": { + "pass@1": 0.47859375, + "pass@2": 0.582545345894263, + "pass@4": 0.665977828753548, + "pass@8": 0.731456928764178, + "pass@16": 0.780729199135965, + "pass@32": 0.8185404669626503, + "pass@64": 0.848842739464603, + "pass@128": 0.8742857142857143 + }, + "19": { + "pass@1": 0.32244744744744747, + "pass@2": 0.42991527452649503, + "pass@4": 0.5297148988133239, + "pass@8": 0.6158184040059627, + "pass@16": 0.6829310379104327, + "pass@32": 0.7314577265217167, + "pass@64": 0.7668547527202739, + "pass@128": 0.7927927927927928 + }, + "20": { + "pass@1": 0.33054147897897895, + "pass@2": 0.441439224657729, + "pass@4": 0.5459787081794953, + "pass@8": 0.6340915487675354, + "pass@16": 0.7009979125456074, + "pass@32": 0.7479033979165912, + "pass@64": 0.781451169312069, + "pass@128": 0.8108108108108109 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 678810, + "answer_accuracy": 0.8266879384255651, + "avg_response_len": 285.86639833398283, + "resp_tokens_sum": 234730617, + "pass_at_k": { + "pass@1": 0.8266879384255651, + "pass@2": 0.8628712425049453, + "pass@4": 0.8899870912375201, + "pass@8": 0.9101864400582642, + "pass@16": 0.9256512587008077, + "pass@32": 0.9380929549393652, + "pass@64": 0.9483659509441283, + "pass@128": 0.9568199532346064 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9402319785276073, + "pass@2": 0.952008281604754, + "pass@4": 0.9611208099562711, + "pass@8": 0.967649685144444, + "pass@16": 0.9723239331864337, + "pass@32": 0.976739705975646, + "pass@64": 0.9818104174845861, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999411231884058, + "pass@2": 0.9999315302978432, + "pass@4": 0.9999989373610907, + "pass@8": 0.9999999998467666, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9966869627507163, + "pass@2": 0.9970676905895359, + "pass@4": 0.9971333691633818, + "pass@8": 0.9971346702346441, + "pass@16": 0.997134670487106, + "pass@32": 0.997134670487106, + "pass@64": 0.997134670487106, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.9998871026011561, + "pass@2": 0.9999989332529243, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9965889084507042, + "pass@2": 0.9972960796273704, + "pass@4": 0.9975222988851745, + "pass@8": 0.9978309304389954, + "pass@16": 0.9983600236590144, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9932400823353293, + "pass@2": 0.99476379426187, + "pass@4": 0.995734111922387, + "pass@8": 0.9969632799932823, + "pass@16": 0.9984388741597237, + "pass@32": 0.999581272939488, + "pass@64": 0.9999802411950416, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9623258659638554, + "pass@2": 0.9707566733469312, + "pass@4": 0.9751945817578527, + "pass@8": 0.978528567411368, + "pass@16": 0.9816759959049333, + "pass@32": 0.9840488998900503, + "pass@64": 0.9848917427571217, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.9100215517241379, + "pass@2": 0.9266520160195493, + "pass@4": 0.9344253890285655, + "pass@8": 0.9383279633369872, + "pass@16": 0.9410428618385669, + "pass@32": 0.9436759703758117, + "pass@64": 0.9467110995185979, + "pass@128": 0.9498432601880877 + }, + "13": { + "pass@1": 0.8494616596638656, + "pass@2": 0.8807649043869519, + "pass@4": 0.900884443803768, + "pass@8": 0.9155814360333341, + "pass@16": 0.9267145595077136, + "pass@32": 0.9359428582481637, + "pass@64": 0.9445210713800876, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.8588267543859649, + "pass@2": 0.9018694312059676, + "pass@4": 0.9242850279679948, + "pass@8": 0.9383757151048715, + "pass@16": 0.9484554246399113, + "pass@32": 0.9561951272112444, + "pass@64": 0.9627628804004219, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.8121685606060606, + "pass@2": 0.8770117513719882, + "pass@4": 0.915564349342695, + "pass@8": 0.9404922694298705, + "pass@16": 0.9573654432962957, + "pass@32": 0.9699346349116533, + "pass@64": 0.9789174437718314, + "pass@128": 0.9848484848484849 + }, + "16": { + "pass@1": 0.6486823361823362, + "pass@2": 0.7438652500168247, + "pass@4": 0.805919283326763, + "pass@8": 0.8449027197071314, + "pass@16": 0.8727992336539356, + "pass@32": 0.8965191234491018, + "pass@64": 0.9150418318628365, + "pass@128": 0.9230769230769231 + }, + "17": { + "pass@1": 0.5890916149068323, + "pass@2": 0.6926184159045329, + "pass@4": 0.764558463286033, + "pass@8": 0.8095983074474769, + "pass@16": 0.8386906126837259, + "pass@32": 0.8617211263710541, + "pass@64": 0.8851656158646087, + "pass@128": 0.9099378881987578 + }, + "18": { + "pass@1": 0.5270220588235294, + "pass@2": 0.6297294754515979, + "pass@4": 0.7110232550894364, + "pass@8": 0.7729100372599288, + "pass@16": 0.8200711561466233, + "pass@32": 0.8564476725925791, + "pass@64": 0.8842089254988268, + "pass@128": 0.9029411764705882 + }, + "19": { + "pass@1": 0.31570870535714285, + "pass@2": 0.4234084997187852, + "pass@4": 0.5270488630327456, + "pass@8": 0.6161440847378656, + "pass@16": 0.6883030755974877, + "pass@32": 0.7454121901545792, + "pass@64": 0.791591256811504, + "pass@128": 0.8333333333333334 + }, + "20": { + "pass@1": 0.29381081586826346, + "pass@2": 0.39535287201188124, + "pass@4": 0.48994442414383815, + "pass@8": 0.5711483363826257, + "pass@16": 0.6389980310343075, + "pass@32": 0.6950049674288274, + "pass@64": 0.740922892472733, + "pass@128": 0.781437125748503 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 654683, + "answer_accuracy": 0.8171770150982585, + "avg_response_len": 306.3667606147148, + "resp_tokens_sum": 245446343, + "pass_at_k": { + "pass@1": 0.8171770150982585, + "pass@2": 0.8561787954479328, + "pass@4": 0.8856990940619518, + "pass@8": 0.9070705967228174, + "pass@16": 0.9224258546590777, + "pass@32": 0.9339350277080865, + "pass@64": 0.9431577235216402, + "pass@128": 0.9507908611599297 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.962080223880597, + "pass@2": 0.9717545393113174, + "pass@4": 0.9788598931663394, + "pass@8": 0.984093246200315, + "pass@16": 0.9877611821054247, + "pass@32": 0.9904522772205684, + "pass@64": 0.9923508998342762, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.999172794117647, + "pass@2": 0.9999478925428441, + "pass@4": 0.9999997033459054, + "pass@8": 0.9999999999938223, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9991867113564669, + "pass@2": 0.9998074964604189, + "pass@4": 0.9999893664711469, + "pass@8": 0.9999999767918476, + "pass@16": 0.9999999999999797, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999505537974683, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9932667042042042, + "pass@2": 0.9964121355607576, + "pass@4": 0.9984610504767983, + "pass@8": 0.999490219467908, + "pass@16": 0.9999277447691266, + "pass@32": 0.999999100781228, + "pass@64": 0.9999999999952028, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.992941117363344, + "pass@2": 0.9948132927817303, + "pass@4": 0.9964856477948294, + "pass@8": 0.9980765642094214, + "pass@16": 0.999336068437543, + "pass@32": 0.9999146790894352, + "pass@64": 0.9999995549101495, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9607886904761904, + "pass@2": 0.9730615704286967, + "pass@4": 0.9794108165050797, + "pass@8": 0.9826995305573731, + "pass@16": 0.9852891085277606, + "pass@32": 0.9874150370684425, + "pass@64": 0.9888440174400547, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.9113627675840978, + "pass@2": 0.934330904307833, + "pass@4": 0.950269623407166, + "pass@8": 0.9616482628134414, + "pass@16": 0.9680019758410323, + "pass@32": 0.9717576392340137, + "pass@64": 0.9743729293524362, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.8297405478395061, + "pass@2": 0.8684227514095461, + "pass@4": 0.8928393649867837, + "pass@8": 0.9073057009677743, + "pass@16": 0.916060516231829, + "pass@32": 0.9236679890966697, + "pass@64": 0.9314670859752837, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.82870456231454, + "pass@2": 0.87395368466553, + "pass@4": 0.898891906686738, + "pass@8": 0.9156222702891602, + "pass@16": 0.9299421512073494, + "pass@32": 0.9418732831664041, + "pass@64": 0.9518740184036798, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.7886525848765432, + "pass@2": 0.8476004301545641, + "pass@4": 0.8812799615325858, + "pass@8": 0.903221984034905, + "pass@16": 0.9189262347802786, + "pass@32": 0.9304864083273199, + "pass@64": 0.9390019202952004, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.634010877581121, + "pass@2": 0.740449271827747, + "pass@4": 0.8125692819924054, + "pass@8": 0.8575554027858113, + "pass@16": 0.8848759001996248, + "pass@32": 0.9038438545841817, + "pass@64": 0.9216124078198928, + "pass@128": 0.9380530973451328 + }, + "17": { + "pass@1": 0.517554593373494, + "pass@2": 0.6258230510625178, + "pass@4": 0.7037033872271984, + "pass@8": 0.756372995216946, + "pass@16": 0.794715859617898, + "pass@32": 0.8249475352261741, + "pass@64": 0.8486202280185033, + "pass@128": 0.8674698795180723 + }, + "18": { + "pass@1": 0.48220766129032255, + "pass@2": 0.5944191325882653, + "pass@4": 0.6840774846692548, + "pass@8": 0.7484527607317959, + "pass@16": 0.7932317734924506, + "pass@32": 0.8275928302086873, + "pass@64": 0.8547011083566157, + "pass@128": 0.8741935483870967 + }, + "19": { + "pass@1": 0.325080249244713, + "pass@2": 0.43810883864214883, + "pass@4": 0.5422776692188405, + "pass@8": 0.6265824302323246, + "pass@16": 0.6887125668408031, + "pass@32": 0.7329306621250592, + "pass@64": 0.767905495181644, + "pass@128": 0.797583081570997 + }, + "20": { + "pass@1": 0.29208896396396394, + "pass@2": 0.401044622378284, + "pass@4": 0.5034210889742385, + "pass@8": 0.5886495943967197, + "pass@16": 0.6554977161404892, + "pass@32": 0.7066888257456404, + "pass@64": 0.746441076359396, + "pass@128": 0.7777777777777778 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_metrics.json b/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a20cd7c7f788ecd06b25a4ec8dadba72e8522c0b --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch4-op17-20", + "id": { + "answer_accuracy": 0.9918411458333334, + "avg_response_len": 199.07718576388888, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9918411458333334, + "pass@2": 0.9942585848643914, + "pass@4": 0.9959380943319587, + "pass@8": 0.9971967379646093, + "pass@16": 0.9981141062657112, + "pass@32": 0.9987434866069741, + "pass@64": 0.9990930305192568, + "pass@128": 0.9992222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9465625, + "pass@2": 0.9594614911417327, + "pass@4": 0.9697060062804653, + "pass@8": 0.9780383427703782, + "pass@16": 0.9844741779786856, + "pass@32": 0.9891185169487656, + "pass@64": 0.9918733630572701, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.999921875, + "pass@2": 0.9999955708661418, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999609375, + "pass@2": 0.9999996309055118, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999375, + "pass@2": 0.9999986466535433, + "pass@4": 0.9999999995313086, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9977421875, + "pass@2": 0.9991828248031493, + "pass@4": 0.9998286774465691, + "pass@8": 0.9999892946880925, + "pass@16": 0.9999999537118734, + "pass@32": 0.9999999999998068, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.996171875, + "pass@2": 0.9974132627952755, + "pass@4": 0.998452184008249, + "pass@8": 0.9991251858828614, + "pass@16": 0.9994832503889618, + "pass@32": 0.9997689348631511, + "pass@64": 0.9999711811023585, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9988125, + "pass@2": 0.9995367864173228, + "pass@4": 0.9998419919385075, + "pass@8": 0.9999775006585908, + "pass@16": 0.999999640100668, + "pass@32": 0.9999999999754914, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.993953125, + "pass@2": 0.9971211860236221, + "pass@4": 0.9987266791338584, + "pass@8": 0.9995431057153819, + "pass@16": 0.9999044500192191, + "pass@32": 0.9999949205487579, + "pass@64": 0.9999999977569968, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935078125, + "pass@2": 0.9956178641732286, + "pass@4": 0.9968873224596928, + "pass@8": 0.9980972119661831, + "pass@16": 0.999165484191994, + "pass@32": 0.9998090071267925, + "pass@64": 0.9999927327566851, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9465625, + "2": 0.999921875, + "3": 0.9999609375, + "4": 0.9999375, + "5": 0.9977421875, + "6": 0.996171875, + "7": 0.9988125, + "8": 0.993953125, + "9": 0.9935078125 + }, + "per_op_avg_response_len": { + "10": 279.4100078125, + "2": 115.2804375, + "3": 144.097296875, + "4": 159.2261328125, + "5": 172.3862890625, + "6": 198.4094609375, + "7": 222.739765625, + "8": 242.299765625, + "9": 257.845515625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.4100078125, + "2": 115.2804375, + "3": 144.097296875, + "4": 159.2261328125, + "5": 172.3862890625, + "6": 198.4094609375, + "7": 222.739765625, + "8": 242.299765625, + "9": 257.845515625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 375880, + "answer_accuracy": 0.9917468760553867, + "avg_response_len": 189.6703235815603, + "resp_tokens_sum": 71886570, + "pass_at_k": { + "pass@1": 0.9917468760553867, + "pass@2": 0.9946696739370348, + "pass@4": 0.996500682158904, + "pass@8": 0.9976780569502949, + "pass@16": 0.9983965809007429, + "pass@32": 0.9987782425175514, + "pass@64": 0.9989567445453673, + "pass@128": 0.9989868287740629 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.947686209439528, + "pass@2": 0.9617538992636985, + "pass@4": 0.9725950904367042, + "pass@8": 0.9805598544320808, + "pass@16": 0.9860735851683115, + "pass@32": 0.9893290501813384, + "pass@64": 0.9908876713836634, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9997566199376947, + "pass@2": 0.9999862020752079, + "pass@4": 0.9999999632055339, + "pass@8": 0.9999999999999803, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999283256880734, + "pass@2": 0.9999988712706783, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.9999961552657481, + "pass@4": 0.9999999985353394, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9978422619047619, + "pass@2": 0.9992364235720536, + "pass@4": 0.9998443864159838, + "pass@8": 0.9999934611094885, + "pass@16": 0.999999993102673, + "pass@32": 0.9999999999999994, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9969385785515584, + "pass@4": 0.9984315971543977, + "pass@8": 0.9995862366315358, + "pass@16": 0.9999714614533096, + "pass@32": 0.999999909391957, + "pass@64": 0.9999999999999891, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9987056213017751, + "pass@2": 0.9998150887573964, + "pass@4": 0.9999939333581823, + "pass@8": 0.9999999935548729, + "pass@16": 0.999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927634214743589, + "pass@2": 0.997842611169998, + "pass@4": 0.9996803302231451, + "pass@8": 0.9999892239856973, + "pass@16": 0.9999999910956074, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9951144366197183, + "pass@2": 0.9977098813352555, + "pass@4": 0.9988829544194299, + "pass@8": 0.999601568979854, + "pass@16": 0.999951739437933, + "pass@32": 0.9999996009748804, + "pass@64": 0.9999999999993653, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387187, + "answer_accuracy": 0.9911200647116645, + "avg_response_len": 197.10492351327, + "resp_tokens_sum": 77000221, + "pass_at_k": { + "pass@1": 0.9911200647116645, + "pass@2": 0.9931661889454186, + "pass@4": 0.9947510570742057, + "pass@8": 0.9961194415684201, + "pass@16": 0.9973336875489526, + "pass@32": 0.9984023042996628, + "pass@64": 0.9990767801528179, + "pass@128": 0.9993446920052425 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9353671395705522, + "pass@2": 0.9489883489928022, + "pass@4": 0.9602921086167909, + "pass@8": 0.970010030193041, + "pass@16": 0.9784615849176065, + "pass@32": 0.9861956753511112, + "pass@64": 0.9914655018647149, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9968297101449275, + "pass@2": 0.9986281096656396, + "pass@4": 0.9996703835933552, + "pass@8": 0.9999750963168695, + "pass@16": 0.9999998721310288, + "pass@32": 0.9999999999994407, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9955229226361032, + "pass@2": 0.9963235086749543, + "pass@4": 0.9971980834952937, + "pass@8": 0.9978938143484848, + "pass@16": 0.9985466574181527, + "pass@32": 0.9993380089577011, + "pass@64": 0.9999174243620693, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998193641618497, + "pass@2": 0.9999946662646215, + "pass@4": 0.9999999959362016, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9968089788732394, + "pass@2": 0.9978419235887768, + "pass@4": 0.9984183204212148, + "pass@8": 0.9991300714136, + "pass@16": 0.9997538434444305, + "pass@32": 0.9999857247050008, + "pass@64": 0.9999999936816814, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935675523952096, + "pass@2": 0.9949026651421564, + "pass@4": 0.9957427633484436, + "pass@8": 0.9969632965123278, + "pass@16": 0.9984388741597334, + "pass@32": 0.999581272939488, + "pass@64": 0.9999802411950417, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379534, + "answer_accuracy": 0.9926713675929025, + "avg_response_len": 210.41734756863073, + "resp_tokens_sum": 80450127, + "pass_at_k": { + "pass@1": 0.9926713675929025, + "pass@2": 0.9949672415770174, + "pass@4": 0.9965932718194292, + "pass@8": 0.9978203479694808, + "pass@16": 0.9986314917793431, + "pass@32": 0.9990576399885253, + "pass@64": 0.9992447341975484, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9563199626865672, + "pass@2": 0.9673334851333878, + "pass@4": 0.975943412577159, + "pass@8": 0.9832993500745408, + "pass@16": 0.9887067340999656, + "pass@32": 0.9917497873815814, + "pass@64": 0.9932677279113761, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9985753676470588, + "pass@2": 0.9996960398332562, + "pass@4": 0.9999747452524317, + "pass@8": 0.9999998417920103, + "pass@16": 0.9999999999980195, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9974122634069401, + "pass@2": 0.9991131641372115, + "pass@4": 0.999854572299125, + "pass@8": 0.9999950840388245, + "pass@16": 0.9999999962921804, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9978243670886076, + "pass@2": 0.9987377591448221, + "pass@4": 0.9995064679415073, + "pass@8": 0.999928806446341, + "pass@16": 0.9999988610780643, + "pass@32": 0.9999999999224412, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9920232732732732, + "pass@2": 0.9956769022959969, + "pass@4": 0.9981618989630802, + "pass@8": 0.9995654428829316, + "pass@16": 0.9999754906144646, + "pass@32": 0.9999999648002483, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9916097266881029, + "pass@2": 0.9939979618705219, + "pass@4": 0.9958385552489217, + "pass@8": 0.9975977940296384, + "pass@16": 0.9990483367272565, + "pass@32": 0.999836024498138, + "pass@64": 0.9999978527259372, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.6973953125, + "avg_response_len": 374.408946875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.6973953125, + "pass@2": 0.7655862696850245, + "pass@4": 0.814535287570296, + "pass@8": 0.8507059550506876, + "pass@16": 0.8780092767935522, + "pass@32": 0.8989192866811604, + "pass@64": 0.9154332057276459, + "pass@128": 0.9292 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.949765625, + "pass@2": 0.9636034694881894, + "pass@4": 0.9714430826771656, + "pass@8": 0.976761624983819, + "pass@16": 0.9811986254788247, + "pass@32": 0.9850637024974109, + "pass@64": 0.9880005694807404, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.8930546875, + "pass@2": 0.9174040354330714, + "pass@4": 0.9324833617360326, + "pass@8": 0.9431630390274265, + "pass@16": 0.9506346812330824, + "pass@32": 0.9557801855766974, + "pass@64": 0.9596686566076377, + "pass@128": 0.963 + }, + "13": { + "pass@1": 0.816203125, + "pass@2": 0.8637347440944891, + "pass@4": 0.8925301711661043, + "pass@8": 0.9116243283252289, + "pass@16": 0.926667277462485, + "pass@32": 0.9394301326422876, + "pass@64": 0.9501194331153558, + "pass@128": 0.959 + }, + "14": { + "pass@1": 0.832890625, + "pass@2": 0.8779275344488201, + "pass@4": 0.9027908765466814, + "pass@8": 0.9202558362172919, + "pass@16": 0.9348536657941723, + "pass@32": 0.9468987710530873, + "pass@64": 0.9561237781715934, + "pass@128": 0.963 + }, + "15": { + "pass@1": 0.8093046875, + "pass@2": 0.8665110728346466, + "pass@4": 0.9013680777090353, + "pass@8": 0.9241231624779181, + "pass@16": 0.9395981544845722, + "pass@32": 0.9511624667281456, + "pass@64": 0.9609580955836557, + "pass@128": 0.97 + }, + "16": { + "pass@1": 0.6662734375, + "pass@2": 0.7528844734251962, + "pass@4": 0.8093653296775412, + "pass@8": 0.8483589019735052, + "pass@16": 0.8776700227907909, + "pass@32": 0.9009114846997556, + "pass@64": 0.9198523641648548, + "pass@128": 0.937 + }, + "17": { + "pass@1": 0.61553125, + "pass@2": 0.7092402805118122, + "pass@4": 0.7726679700037498, + "pass@8": 0.8156043454313336, + "pass@16": 0.8469887135917317, + "pass@32": 0.872209664689536, + "pass@64": 0.8923636305747241, + "pass@128": 0.909 + }, + "18": { + "pass@1": 0.5731796875, + "pass@2": 0.6679947096456698, + "pass@4": 0.7395980508998866, + "pass@8": 0.7930719507635707, + "pass@16": 0.8322808994809556, + "pass@32": 0.8617942420278873, + "pass@64": 0.8851850179047893, + "pass@128": 0.904 + }, + "19": { + "pass@1": 0.397671875, + "pass@2": 0.5062367125984255, + "pass@4": 0.6007368568616425, + "pass@8": 0.6788454836198332, + "pass@16": 0.7393707639669304, + "pass@32": 0.7840548102990779, + "pass@64": 0.8199610949104585, + "pass@128": 0.853 + }, + "20": { + "pass@1": 0.420078125, + "pass@2": 0.5303256643700788, + "pass@4": 0.6223690984251962, + "pass@8": 0.6952508776870037, + "pass@16": 0.7508299636518877, + "pass@32": 0.7918874065976513, + "pass@64": 0.8220994167625919, + "pass@128": 0.844 + } + }, + "per_op_accuracy": { + "11": 0.949765625, + "12": 0.8930546875, + "13": 0.816203125, + "14": 0.832890625, + "15": 0.8093046875, + "16": 0.6662734375, + "17": 0.61553125, + "18": 0.5731796875, + "19": 0.397671875, + "20": 0.420078125 + }, + "per_op_avg_response_len": { + "11": 297.4345703125, + "12": 306.6816796875, + "13": 324.94728125, + "14": 354.10671875, + "15": 375.599953125, + "16": 381.122984375, + "17": 405.199671875, + "18": 423.66575, + "19": 425.310953125, + "20": 450.01990625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.4345703125, + "12": 306.6816796875, + "13": 324.94728125, + "14": 354.10671875, + "15": 375.599953125, + "16": 381.122984375, + "17": 405.199671875, + "18": 423.66575, + "19": 425.310953125, + "20": 450.01990625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 302613, + "answer_accuracy": 0.7025747585438336, + "avg_response_len": 356.7563614413076, + "resp_tokens_sum": 153662100, + "pass_at_k": { + "pass@1": 0.7025747585438336, + "pass@2": 0.7702944565993196, + "pass@4": 0.8190666393112498, + "pass@8": 0.8552391822468836, + "pass@16": 0.8823689747494595, + "pass@32": 0.9028818370439949, + "pass@64": 0.9189488249813886, + "pass@128": 0.9325408618127786 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9421033994334278, + "pass@2": 0.9585153827708507, + "pass@4": 0.9683495726277837, + "pass@8": 0.9750743355007809, + "pass@16": 0.9806874497365086, + "pass@32": 0.9855674217757143, + "pass@64": 0.9892681802260217, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8856594279661016, + "pass@2": 0.910248176075448, + "pass@4": 0.9253412335640251, + "pass@8": 0.9357745885481913, + "pass@16": 0.9435988842897304, + "pass@32": 0.9494690435475064, + "pass@64": 0.9538236623937969, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8312842868338558, + "pass@2": 0.8800130513662283, + "pass@4": 0.9112304953847852, + "pass@8": 0.9301201681225754, + "pass@16": 0.9426301334874778, + "pass@32": 0.9531258102406001, + "pass@64": 0.962334874645846, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.8299260124610592, + "pass@2": 0.8766005592758852, + "pass@4": 0.9035752581161002, + "pass@8": 0.9220158894644768, + "pass@16": 0.9356224843327265, + "pass@32": 0.945334584744341, + "pass@64": 0.953042570524806, + "pass@128": 0.9595015576323987 + }, + "15": { + "pass@1": 0.8186867774566474, + "pass@2": 0.877645532747713, + "pass@4": 0.9140065072248914, + "pass@8": 0.937924646916376, + "pass@16": 0.9541987911233919, + "pass@32": 0.965099381921458, + "pass@64": 0.9726851787993989, + "pass@128": 0.9797687861271677 + }, + "16": { + "pass@1": 0.6638860887096775, + "pass@2": 0.7459883794767583, + "pass@4": 0.8018100526748667, + "pass@8": 0.8433015429764784, + "pass@16": 0.8753356035074012, + "pass@32": 0.9009404640966499, + "pass@64": 0.9223832471129497, + "pass@128": 0.9419354838709677 + }, + "17": { + "pass@1": 0.6308932442196532, + "pass@2": 0.7216910216876794, + "pass@4": 0.7815059672815464, + "pass@8": 0.8225626990224884, + "pass@16": 0.8534324527753754, + "pass@32": 0.8779455950894322, + "pass@64": 0.8973389717015748, + "pass@128": 0.9132947976878613 + }, + "18": { + "pass@1": 0.551875, + "pass@2": 0.6496741422947138, + "pass@4": 0.7255719232417374, + "pass@8": 0.7824379510059899, + "pass@16": 0.823303427941537, + "pass@32": 0.8522402739187724, + "pass@64": 0.8735607870746009, + "pass@128": 0.8914285714285715 + }, + "19": { + "pass@1": 0.4078688063063063, + "pass@2": 0.5134215169894304, + "pass@4": 0.6041881234440287, + "pass@8": 0.6818408987273152, + "pass@16": 0.7434984007892304, + "pass@32": 0.7890733463330541, + "pass@64": 0.8258918288508185, + "pass@128": 0.8558558558558559 + }, + "20": { + "pass@1": 0.4509196696696697, + "pass@2": 0.5596455215057575, + "pass@4": 0.6476805069974355, + "pass@8": 0.7155572588224337, + "pass@16": 0.7669233647580539, + "pass@32": 0.8067563935871334, + "pass@64": 0.837089022951731, + "pass@128": 0.8618618618618619 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 302599, + "answer_accuracy": 0.7029600617008623, + "avg_response_len": 369.36367965730005, + "resp_tokens_sum": 158997767, + "pass_at_k": { + "pass@1": 0.7029600617008623, + "pass@2": 0.7702698688366508, + "pass@4": 0.8183143234994847, + "pass@8": 0.8539573341142721, + "pass@16": 0.8813696069826088, + "pass@32": 0.9028034603918524, + "pass@64": 0.9200449007829323, + "pass@128": 0.9345822182575082 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9559723268072289, + "pass@2": 0.966372008704108, + "pass@4": 0.9713946641458974, + "pass@8": 0.9749261474893676, + "pass@16": 0.9779525234791777, + "pass@32": 0.9806901170607644, + "pass@64": 0.9830740245540501, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.895694553291536, + "pass@2": 0.9185176671685638, + "pass@4": 0.9316084925121034, + "pass@8": 0.9394861999416354, + "pass@16": 0.9443789712253488, + "pass@32": 0.9476920343833845, + "pass@64": 0.9504452941632805, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.8206188725490197, + "pass@2": 0.8654555818390349, + "pass@4": 0.8914457194951465, + "pass@8": 0.91024432511722, + "pass@16": 0.9260177446913851, + "pass@32": 0.9387626126789113, + "pass@64": 0.9509378123272939, + "pass@128": 0.9635854341736695 + }, + "14": { + "pass@1": 0.8450520833333334, + "pass@2": 0.892094902610858, + "pass@4": 0.9161785703322166, + "pass@8": 0.9328964288192578, + "pass@16": 0.9477363031164306, + "pass@32": 0.960031109634046, + "pass@64": 0.9693199907235079, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8125236742424242, + "pass@2": 0.8730982313290383, + "pass@4": 0.9092206175932549, + "pass@8": 0.9321373846919218, + "pass@16": 0.947976048382885, + "pass@32": 0.9615416739411536, + "pass@64": 0.9736111224580455, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.6658653846153846, + "pass@2": 0.75107889158086, + "pass@4": 0.807817280051532, + "pass@8": 0.8455472972839031, + "pass@16": 0.8726949869908102, + "pass@32": 0.8942385463597932, + "pass@64": 0.9125603746766824, + "pass@128": 0.9287749287749287 + }, + "17": { + "pass@1": 0.6382472826086957, + "pass@2": 0.7308938199980439, + "pass@4": 0.7938274872085086, + "pass@8": 0.8379544601446831, + "pass@16": 0.8705400266432815, + "pass@32": 0.8951335125041588, + "pass@64": 0.9145615683410627, + "pass@128": 0.9347826086956522 + }, + "18": { + "pass@1": 0.597219669117647, + "pass@2": 0.6891023043075495, + "pass@4": 0.7591783864333136, + "pass@8": 0.8121559273453292, + "pass@16": 0.8524030793941533, + "pass@32": 0.8851876403918829, + "pass@64": 0.9114327068575712, + "pass@128": 0.9294117647058824 + }, + "19": { + "pass@1": 0.39669363839285715, + "pass@2": 0.5050988499484439, + "pass@4": 0.5999355119672539, + "pass@8": 0.6770325751872642, + "pass@16": 0.7367422561935473, + "pass@32": 0.7810862819828637, + "pass@64": 0.8162268287403831, + "pass@128": 0.8511904761904762 + }, + "20": { + "pass@1": 0.4049869011976048, + "pass@2": 0.5131853664717809, + "pass@4": 0.6042743593240965, + "pass@8": 0.6785910123757385, + "pass@16": 0.738234545184363, + "pass@32": 0.7842155855252223, + "pass@64": 0.8183162239457017, + "pass@128": 0.8413173652694611 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 287454, + "answer_accuracy": 0.6863491366136919, + "avg_response_len": 397.7488562996638, + "resp_tokens_sum": 166583585, + "pass_at_k": { + "pass@1": 0.6863491366136919, + "pass@2": 0.7559304038802191, + "pass@4": 0.8059910038178307, + "pass@8": 0.8427020744559618, + "pass@16": 0.8700718764122981, + "pass@32": 0.890851909492874, + "pass@64": 0.9070777077875871, + "pass@128": 0.9202322738386308 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9518105158730159, + "pass@2": 0.9663874046994128, + "pass@4": 0.974960809660697, + "pass@8": 0.9805869955097556, + "pass@16": 0.9851927553230163, + "pass@32": 0.98910883729019, + "pass@64": 0.9917724625682862, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8984852828746177, + "pass@2": 0.9240643586409498, + "pass@4": 0.941068672608584, + "pass@8": 0.9547484308868067, + "pass@16": 0.964354080653309, + "pass@32": 0.9705027070109506, + "pass@64": 0.9749939794561082, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.7964891975308642, + "pass@2": 0.8458115370127349, + "pass@4": 0.8753133372795989, + "pass@8": 0.8949344772446929, + "pass@16": 0.9116664506949259, + "pass@32": 0.9266813162011269, + "pass@64": 0.9371907688348377, + "pass@128": 0.941358024691358 + }, + "14": { + "pass@1": 0.8233725890207715, + "pass@2": 0.864813941330405, + "pass@4": 0.8884574113881165, + "pass@8": 0.9057512078427568, + "pass@16": 0.9210475746514785, + "pass@32": 0.9350615129237696, + "pass@64": 0.9456666949723773, + "pass@128": 0.9525222551928784 + }, + "15": { + "pass@1": 0.7960069444444444, + "pass@2": 0.8479114264362789, + "pass@4": 0.8798735259365724, + "pass@8": 0.9012219064707329, + "pass@16": 0.9154730765108868, + "pass@32": 0.9257078030331487, + "pass@64": 0.9355473867528037, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.6688790560471977, + "pass@2": 0.761060132278819, + "pass@4": 0.8178771328915742, + "pass@8": 0.8558947560594331, + "pass@16": 0.8849558857513954, + "pass@32": 0.9077941329129988, + "pass@64": 0.9250881004375349, + "pass@128": 0.9410029498525073 + }, + "17": { + "pass@1": 0.5774896460843374, + "pass@2": 0.6752631836400717, + "pass@4": 0.7429351037445621, + "pass@8": 0.7866756488041113, + "pass@16": 0.8174313143142055, + "pass@32": 0.8439984872357021, + "pass@64": 0.8656491607233635, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.570866935483871, + "pass@2": 0.6655289560579124, + "pass@4": 0.7339587947675894, + "pass@8": 0.7841472664969731, + "pass@16": 0.8203472667980849, + "pass@32": 0.8469237045905657, + "pass@64": 0.8695213616035634, + "pass@128": 0.8903225806451613 + }, + "19": { + "pass@1": 0.3884063444108761, + "pass@2": 0.5001635463996004, + "pass@4": 0.5980781865068977, + "pass@8": 0.6776722630837357, + "pass@16": 0.7378864000697488, + "pass@32": 0.782019320918216, + "pass@64": 0.81778520074431, + "pass@128": 0.851963746223565 + }, + "20": { + "pass@1": 0.4043731231231231, + "pass@2": 0.5181975774987587, + "pass@4": 0.6152067675099175, + "pass@8": 0.6916543914883966, + "pass@16": 0.7473698050927324, + "pass@32": 0.7847132791823201, + "pass@64": 0.8109043643297321, + "pass@128": 0.8288288288288288 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.8368696546052632, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.8368696546052632, + "pass@2": 0.8739047347699785, + "pass@4": 0.9004629328784418, + "pass@8": 0.920096325904657, + "pass@16": 0.9349010381224566, + "pass@32": 0.9462044340144459, + "pass@64": 0.9550615437868266, + "pass@128": 0.9623684210526315 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9465625, + "pass@2": 0.9594614911417327, + "pass@4": 0.9697060062804653, + "pass@8": 0.9780383427703782, + "pass@16": 0.9844741779786856, + "pass@32": 0.9891185169487656, + "pass@64": 0.9918733630572701, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.999921875, + "pass@2": 0.9999955708661418, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999609375, + "pass@2": 0.9999996309055118, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999375, + "pass@2": 0.9999986466535433, + "pass@4": 0.9999999995313086, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9977421875, + "pass@2": 0.9991828248031493, + "pass@4": 0.9998286774465691, + "pass@8": 0.9999892946880925, + "pass@16": 0.9999999537118734, + "pass@32": 0.9999999999998068, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.996171875, + "pass@2": 0.9974132627952755, + "pass@4": 0.998452184008249, + "pass@8": 0.9991251858828614, + "pass@16": 0.9994832503889618, + "pass@32": 0.9997689348631511, + "pass@64": 0.9999711811023585, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9988125, + "pass@2": 0.9995367864173228, + "pass@4": 0.9998419919385075, + "pass@8": 0.9999775006585908, + "pass@16": 0.999999640100668, + "pass@32": 0.9999999999754914, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.993953125, + "pass@2": 0.9971211860236221, + "pass@4": 0.9987266791338584, + "pass@8": 0.9995431057153819, + "pass@16": 0.9999044500192191, + "pass@32": 0.9999949205487579, + "pass@64": 0.9999999977569968, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935078125, + "pass@2": 0.9956178641732286, + "pass@4": 0.9968873224596928, + "pass@8": 0.9980972119661831, + "pass@16": 0.999165484191994, + "pass@32": 0.9998090071267925, + "pass@64": 0.9999927327566851, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.949765625, + "pass@2": 0.9636034694881894, + "pass@4": 0.9714430826771656, + "pass@8": 0.976761624983819, + "pass@16": 0.9811986254788247, + "pass@32": 0.9850637024974109, + "pass@64": 0.9880005694807404, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.8930546875, + "pass@2": 0.9174040354330714, + "pass@4": 0.9324833617360326, + "pass@8": 0.9431630390274265, + "pass@16": 0.9506346812330824, + "pass@32": 0.9557801855766974, + "pass@64": 0.9596686566076377, + "pass@128": 0.963 + }, + "13": { + "pass@1": 0.816203125, + "pass@2": 0.8637347440944891, + "pass@4": 0.8925301711661043, + "pass@8": 0.9116243283252289, + "pass@16": 0.926667277462485, + "pass@32": 0.9394301326422876, + "pass@64": 0.9501194331153558, + "pass@128": 0.959 + }, + "14": { + "pass@1": 0.832890625, + "pass@2": 0.8779275344488201, + "pass@4": 0.9027908765466814, + "pass@8": 0.9202558362172919, + "pass@16": 0.9348536657941723, + "pass@32": 0.9468987710530873, + "pass@64": 0.9561237781715934, + "pass@128": 0.963 + }, + "15": { + "pass@1": 0.8093046875, + "pass@2": 0.8665110728346466, + "pass@4": 0.9013680777090353, + "pass@8": 0.9241231624779181, + "pass@16": 0.9395981544845722, + "pass@32": 0.9511624667281456, + "pass@64": 0.9609580955836557, + "pass@128": 0.97 + }, + "16": { + "pass@1": 0.6662734375, + "pass@2": 0.7528844734251962, + "pass@4": 0.8093653296775412, + "pass@8": 0.8483589019735052, + "pass@16": 0.8776700227907909, + "pass@32": 0.9009114846997556, + "pass@64": 0.9198523641648548, + "pass@128": 0.937 + }, + "17": { + "pass@1": 0.61553125, + "pass@2": 0.7092402805118122, + "pass@4": 0.7726679700037498, + "pass@8": 0.8156043454313336, + "pass@16": 0.8469887135917317, + "pass@32": 0.872209664689536, + "pass@64": 0.8923636305747241, + "pass@128": 0.909 + }, + "18": { + "pass@1": 0.5731796875, + "pass@2": 0.6679947096456698, + "pass@4": 0.7395980508998866, + "pass@8": 0.7930719507635707, + "pass@16": 0.8322808994809556, + "pass@32": 0.8617942420278873, + "pass@64": 0.8851850179047893, + "pass@128": 0.904 + }, + "19": { + "pass@1": 0.397671875, + "pass@2": 0.5062367125984255, + "pass@4": 0.6007368568616425, + "pass@8": 0.6788454836198332, + "pass@16": 0.7393707639669304, + "pass@32": 0.7840548102990779, + "pass@64": 0.8199610949104585, + "pass@128": 0.853 + }, + "20": { + "pass@1": 0.420078125, + "pass@2": 0.5303256643700788, + "pass@4": 0.6223690984251962, + "pass@8": 0.6952508776870037, + "pass@16": 0.7508299636518877, + "pass@32": 0.7918874065976513, + "pass@64": 0.8220994167625919, + "pass@128": 0.844 + } + }, + "per_op_accuracy": { + "10": 0.9465625, + "2": 0.999921875, + "3": 0.9999609375, + "4": 0.9999375, + "5": 0.9977421875, + "6": 0.996171875, + "7": 0.9988125, + "8": 0.993953125, + "9": 0.9935078125, + "11": 0.949765625, + "12": 0.8930546875, + "13": 0.816203125, + "14": 0.832890625, + "15": 0.8093046875, + "16": 0.6662734375, + "17": 0.61553125, + "18": 0.5731796875, + "19": 0.397671875, + "20": 0.420078125 + }, + "per_op_avg_response_len": { + "10": 279.4100078125, + "2": 115.2804375, + "3": 144.097296875, + "4": 159.2261328125, + "5": 172.3862890625, + "6": 198.4094609375, + "7": 222.739765625, + "8": 242.299765625, + "9": 257.845515625, + "11": 297.4345703125, + "12": 306.6816796875, + "13": 324.94728125, + "14": 354.10671875, + "15": 375.599953125, + "16": 381.122984375, + "17": 405.199671875, + "18": 423.66575, + "19": 425.310953125, + "20": 450.01990625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.4100078125, + "2": 115.2804375, + "3": 144.097296875, + "4": 159.2261328125, + "5": 172.3862890625, + "6": 198.4094609375, + "7": 222.739765625, + "8": 242.299765625, + "9": 257.845515625, + "11": 297.4345703125, + "12": 306.6816796875, + "13": 324.94728125, + "14": 354.10671875, + "15": 375.599953125, + "16": 381.122984375, + "17": 405.199671875, + "18": 423.66575, + "19": 425.310953125, + "20": 450.01990625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 678493, + "answer_accuracy": 0.8379270569870376, + "avg_response_len": 278.5486854845084, + "resp_tokens_sum": 225548670, + "pass_at_k": { + "pass@1": 0.8379270569870376, + "pass@2": 0.8753173808068597, + "pass@4": 0.9021178882634914, + "pass@8": 0.9219103027016352, + "pass@16": 0.936677817906899, + "pass@32": 0.9477679035326472, + "pass@64": 0.9563979950460396, + "pass@128": 0.9636421119190641 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.947686209439528, + "pass@2": 0.9617538992636985, + "pass@4": 0.9725950904367042, + "pass@8": 0.9805598544320808, + "pass@16": 0.9860735851683115, + "pass@32": 0.9893290501813384, + "pass@64": 0.9908876713836634, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9997566199376947, + "pass@2": 0.9999862020752079, + "pass@4": 0.9999999632055339, + "pass@8": 0.9999999999999803, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999283256880734, + "pass@2": 0.9999988712706783, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.9999961552657481, + "pass@4": 0.9999999985353394, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9978422619047619, + "pass@2": 0.9992364235720536, + "pass@4": 0.9998443864159838, + "pass@8": 0.9999934611094885, + "pass@16": 0.999999993102673, + "pass@32": 0.9999999999999994, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9969385785515584, + "pass@4": 0.9984315971543977, + "pass@8": 0.9995862366315358, + "pass@16": 0.9999714614533096, + "pass@32": 0.999999909391957, + "pass@64": 0.9999999999999891, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9987056213017751, + "pass@2": 0.9998150887573964, + "pass@4": 0.9999939333581823, + "pass@8": 0.9999999935548729, + "pass@16": 0.999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927634214743589, + "pass@2": 0.997842611169998, + "pass@4": 0.9996803302231451, + "pass@8": 0.9999892239856973, + "pass@16": 0.9999999910956074, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9951144366197183, + "pass@2": 0.9977098813352555, + "pass@4": 0.9988829544194299, + "pass@8": 0.999601568979854, + "pass@16": 0.999951739437933, + "pass@32": 0.9999996009748804, + "pass@64": 0.9999999999993653, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9421033994334278, + "pass@2": 0.9585153827708507, + "pass@4": 0.9683495726277837, + "pass@8": 0.9750743355007809, + "pass@16": 0.9806874497365086, + "pass@32": 0.9855674217757143, + "pass@64": 0.9892681802260217, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8856594279661016, + "pass@2": 0.910248176075448, + "pass@4": 0.9253412335640251, + "pass@8": 0.9357745885481913, + "pass@16": 0.9435988842897304, + "pass@32": 0.9494690435475064, + "pass@64": 0.9538236623937969, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8312842868338558, + "pass@2": 0.8800130513662283, + "pass@4": 0.9112304953847852, + "pass@8": 0.9301201681225754, + "pass@16": 0.9426301334874778, + "pass@32": 0.9531258102406001, + "pass@64": 0.962334874645846, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.8299260124610592, + "pass@2": 0.8766005592758852, + "pass@4": 0.9035752581161002, + "pass@8": 0.9220158894644768, + "pass@16": 0.9356224843327265, + "pass@32": 0.945334584744341, + "pass@64": 0.953042570524806, + "pass@128": 0.9595015576323987 + }, + "15": { + "pass@1": 0.8186867774566474, + "pass@2": 0.877645532747713, + "pass@4": 0.9140065072248914, + "pass@8": 0.937924646916376, + "pass@16": 0.9541987911233919, + "pass@32": 0.965099381921458, + "pass@64": 0.9726851787993989, + "pass@128": 0.9797687861271677 + }, + "16": { + "pass@1": 0.6638860887096775, + "pass@2": 0.7459883794767583, + "pass@4": 0.8018100526748667, + "pass@8": 0.8433015429764784, + "pass@16": 0.8753356035074012, + "pass@32": 0.9009404640966499, + "pass@64": 0.9223832471129497, + "pass@128": 0.9419354838709677 + }, + "17": { + "pass@1": 0.6308932442196532, + "pass@2": 0.7216910216876794, + "pass@4": 0.7815059672815464, + "pass@8": 0.8225626990224884, + "pass@16": 0.8534324527753754, + "pass@32": 0.8779455950894322, + "pass@64": 0.8973389717015748, + "pass@128": 0.9132947976878613 + }, + "18": { + "pass@1": 0.551875, + "pass@2": 0.6496741422947138, + "pass@4": 0.7255719232417374, + "pass@8": 0.7824379510059899, + "pass@16": 0.823303427941537, + "pass@32": 0.8522402739187724, + "pass@64": 0.8735607870746009, + "pass@128": 0.8914285714285715 + }, + "19": { + "pass@1": 0.4078688063063063, + "pass@2": 0.5134215169894304, + "pass@4": 0.6041881234440287, + "pass@8": 0.6818408987273152, + "pass@16": 0.7434984007892304, + "pass@32": 0.7890733463330541, + "pass@64": 0.8258918288508185, + "pass@128": 0.8558558558558559 + }, + "20": { + "pass@1": 0.4509196696696697, + "pass@2": 0.5596455215057575, + "pass@4": 0.6476805069974355, + "pass@8": 0.7155572588224337, + "pass@16": 0.7669233647580539, + "pass@32": 0.8067563935871334, + "pass@64": 0.837089022951731, + "pass@128": 0.8618618618618619 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 689786, + "answer_accuracy": 0.8400550467653936, + "avg_response_len": 287.4098645752143, + "resp_tokens_sum": 235997988, + "pass_at_k": { + "pass@1": 0.8400550467653936, + "pass@2": 0.8763150081931377, + "pass@4": 0.9022558528634762, + "pass@8": 0.9215923694922954, + "pass@16": 0.9365406707220455, + "pass@32": 0.9482855604084754, + "pass@64": 0.9576451027840119, + "pass@128": 0.9653936087295402 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9353671395705522, + "pass@2": 0.9489883489928022, + "pass@4": 0.9602921086167909, + "pass@8": 0.970010030193041, + "pass@16": 0.9784615849176065, + "pass@32": 0.9861956753511112, + "pass@64": 0.9914655018647149, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9968297101449275, + "pass@2": 0.9986281096656396, + "pass@4": 0.9996703835933552, + "pass@8": 0.9999750963168695, + "pass@16": 0.9999998721310288, + "pass@32": 0.9999999999994407, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9955229226361032, + "pass@2": 0.9963235086749543, + "pass@4": 0.9971980834952937, + "pass@8": 0.9978938143484848, + "pass@16": 0.9985466574181527, + "pass@32": 0.9993380089577011, + "pass@64": 0.9999174243620693, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998193641618497, + "pass@2": 0.9999946662646215, + "pass@4": 0.9999999959362016, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9968089788732394, + "pass@2": 0.9978419235887768, + "pass@4": 0.9984183204212148, + "pass@8": 0.9991300714136, + "pass@16": 0.9997538434444305, + "pass@32": 0.9999857247050008, + "pass@64": 0.9999999936816814, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935675523952096, + "pass@2": 0.9949026651421564, + "pass@4": 0.9957427633484436, + "pass@8": 0.9969632965123278, + "pass@16": 0.9984388741597334, + "pass@32": 0.999581272939488, + "pass@64": 0.9999802411950417, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9559723268072289, + "pass@2": 0.966372008704108, + "pass@4": 0.9713946641458974, + "pass@8": 0.9749261474893676, + "pass@16": 0.9779525234791777, + "pass@32": 0.9806901170607644, + "pass@64": 0.9830740245540501, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.895694553291536, + "pass@2": 0.9185176671685638, + "pass@4": 0.9316084925121034, + "pass@8": 0.9394861999416354, + "pass@16": 0.9443789712253488, + "pass@32": 0.9476920343833845, + "pass@64": 0.9504452941632805, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.8206188725490197, + "pass@2": 0.8654555818390349, + "pass@4": 0.8914457194951465, + "pass@8": 0.91024432511722, + "pass@16": 0.9260177446913851, + "pass@32": 0.9387626126789113, + "pass@64": 0.9509378123272939, + "pass@128": 0.9635854341736695 + }, + "14": { + "pass@1": 0.8450520833333334, + "pass@2": 0.892094902610858, + "pass@4": 0.9161785703322166, + "pass@8": 0.9328964288192578, + "pass@16": 0.9477363031164306, + "pass@32": 0.960031109634046, + "pass@64": 0.9693199907235079, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8125236742424242, + "pass@2": 0.8730982313290383, + "pass@4": 0.9092206175932549, + "pass@8": 0.9321373846919218, + "pass@16": 0.947976048382885, + "pass@32": 0.9615416739411536, + "pass@64": 0.9736111224580455, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.6658653846153846, + "pass@2": 0.75107889158086, + "pass@4": 0.807817280051532, + "pass@8": 0.8455472972839031, + "pass@16": 0.8726949869908102, + "pass@32": 0.8942385463597932, + "pass@64": 0.9125603746766824, + "pass@128": 0.9287749287749287 + }, + "17": { + "pass@1": 0.6382472826086957, + "pass@2": 0.7308938199980439, + "pass@4": 0.7938274872085086, + "pass@8": 0.8379544601446831, + "pass@16": 0.8705400266432815, + "pass@32": 0.8951335125041588, + "pass@64": 0.9145615683410627, + "pass@128": 0.9347826086956522 + }, + "18": { + "pass@1": 0.597219669117647, + "pass@2": 0.6891023043075495, + "pass@4": 0.7591783864333136, + "pass@8": 0.8121559273453292, + "pass@16": 0.8524030793941533, + "pass@32": 0.8851876403918829, + "pass@64": 0.9114327068575712, + "pass@128": 0.9294117647058824 + }, + "19": { + "pass@1": 0.39669363839285715, + "pass@2": 0.5050988499484439, + "pass@4": 0.5999355119672539, + "pass@8": 0.6770325751872642, + "pass@16": 0.7367422561935473, + "pass@32": 0.7810862819828637, + "pass@64": 0.8162268287403831, + "pass@128": 0.8511904761904762 + }, + "20": { + "pass@1": 0.4049869011976048, + "pass@2": 0.5131853664717809, + "pass@4": 0.6042743593240965, + "pass@8": 0.6785910123757385, + "pass@16": 0.738234545184363, + "pass@32": 0.7842155855252223, + "pass@64": 0.8183162239457017, + "pass@128": 0.8413173652694611 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 666988, + "answer_accuracy": 0.8325361479469564, + "avg_response_len": 308.34811870905895, + "resp_tokens_sum": 247033712, + "pass_at_k": { + "pass@1": 0.8325361479469564, + "pass@2": 0.8700066196016242, + "pass@4": 0.8969526549634996, + "pass@8": 0.9167296000966144, + "pass@16": 0.9314247396654343, + "pass@32": 0.9424912315875398, + "pass@64": 0.951062834467024, + "pass@128": 0.9579805080683815 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9563199626865672, + "pass@2": 0.9673334851333878, + "pass@4": 0.975943412577159, + "pass@8": 0.9832993500745408, + "pass@16": 0.9887067340999656, + "pass@32": 0.9917497873815814, + "pass@64": 0.9932677279113761, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9985753676470588, + "pass@2": 0.9996960398332562, + "pass@4": 0.9999747452524317, + "pass@8": 0.9999998417920103, + "pass@16": 0.9999999999980195, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9974122634069401, + "pass@2": 0.9991131641372115, + "pass@4": 0.999854572299125, + "pass@8": 0.9999950840388245, + "pass@16": 0.9999999962921804, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9978243670886076, + "pass@2": 0.9987377591448221, + "pass@4": 0.9995064679415073, + "pass@8": 0.999928806446341, + "pass@16": 0.9999988610780643, + "pass@32": 0.9999999999224412, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9920232732732732, + "pass@2": 0.9956769022959969, + "pass@4": 0.9981618989630802, + "pass@8": 0.9995654428829316, + "pass@16": 0.9999754906144646, + "pass@32": 0.9999999648002483, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9916097266881029, + "pass@2": 0.9939979618705219, + "pass@4": 0.9958385552489217, + "pass@8": 0.9975977940296384, + "pass@16": 0.9990483367272565, + "pass@32": 0.999836024498138, + "pass@64": 0.9999978527259372, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9518105158730159, + "pass@2": 0.9663874046994128, + "pass@4": 0.974960809660697, + "pass@8": 0.9805869955097556, + "pass@16": 0.9851927553230163, + "pass@32": 0.98910883729019, + "pass@64": 0.9917724625682862, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8984852828746177, + "pass@2": 0.9240643586409498, + "pass@4": 0.941068672608584, + "pass@8": 0.9547484308868067, + "pass@16": 0.964354080653309, + "pass@32": 0.9705027070109506, + "pass@64": 0.9749939794561082, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.7964891975308642, + "pass@2": 0.8458115370127349, + "pass@4": 0.8753133372795989, + "pass@8": 0.8949344772446929, + "pass@16": 0.9116664506949259, + "pass@32": 0.9266813162011269, + "pass@64": 0.9371907688348377, + "pass@128": 0.941358024691358 + }, + "14": { + "pass@1": 0.8233725890207715, + "pass@2": 0.864813941330405, + "pass@4": 0.8884574113881165, + "pass@8": 0.9057512078427568, + "pass@16": 0.9210475746514785, + "pass@32": 0.9350615129237696, + "pass@64": 0.9456666949723773, + "pass@128": 0.9525222551928784 + }, + "15": { + "pass@1": 0.7960069444444444, + "pass@2": 0.8479114264362789, + "pass@4": 0.8798735259365724, + "pass@8": 0.9012219064707329, + "pass@16": 0.9154730765108868, + "pass@32": 0.9257078030331487, + "pass@64": 0.9355473867528037, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.6688790560471977, + "pass@2": 0.761060132278819, + "pass@4": 0.8178771328915742, + "pass@8": 0.8558947560594331, + "pass@16": 0.8849558857513954, + "pass@32": 0.9077941329129988, + "pass@64": 0.9250881004375349, + "pass@128": 0.9410029498525073 + }, + "17": { + "pass@1": 0.5774896460843374, + "pass@2": 0.6752631836400717, + "pass@4": 0.7429351037445621, + "pass@8": 0.7866756488041113, + "pass@16": 0.8174313143142055, + "pass@32": 0.8439984872357021, + "pass@64": 0.8656491607233635, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.570866935483871, + "pass@2": 0.6655289560579124, + "pass@4": 0.7339587947675894, + "pass@8": 0.7841472664969731, + "pass@16": 0.8203472667980849, + "pass@32": 0.8469237045905657, + "pass@64": 0.8695213616035634, + "pass@128": 0.8903225806451613 + }, + "19": { + "pass@1": 0.3884063444108761, + "pass@2": 0.5001635463996004, + "pass@4": 0.5980781865068977, + "pass@8": 0.6776722630837357, + "pass@16": 0.7378864000697488, + "pass@32": 0.782019320918216, + "pass@64": 0.81778520074431, + "pass@128": 0.851963746223565 + }, + "20": { + "pass@1": 0.4043731231231231, + "pass@2": 0.5181975774987587, + "pass@4": 0.6152067675099175, + "pass@8": 0.6916543914883966, + "pass@16": 0.7473698050927324, + "pass@32": 0.7847132791823201, + "pass@64": 0.8109043643297321, + "pass@128": 0.8288288288288288 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch4-op17-20/graphs/SDAR-100M-block8-epoch4-op17-20_score_report.jsonl b/composition/SDAR-100M-block8-epoch4-op17-20/graphs/SDAR-100M-block8-epoch4-op17-20_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6635e52e196347f0ba5adf687ce55a5cd41adc49 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op17-20/graphs/SDAR-100M-block8-epoch4-op17-20_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch4-op17-20", "results_dir": "results/composition/SDAR-100M-block8-epoch4-op17-20/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1022666, "value_mismatch": 10019, "dependency_mismatch": 126761, "answer_mismatch": 9399, "extra_nodes": 11754, "missing_nodes": 10060, "perfect_with_extra": 840, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337143, "dependency_mismatch": 40869, "value_mismatch": 3355, "answer_mismatch": 3128, "extra_nodes": 3804, "missing_nodes": 3105, "perfect_with_extra": 234}, "teachers_in_school": {"total": 390656, "perfect": 346470, "dependency_mismatch": 43424, "value_mismatch": 3546, "answer_mismatch": 3469, "extra_nodes": 3997, "missing_nodes": 3569, "perfect_with_extra": 145}, "movie_festival_awards": {"total": 382336, "perfect": 339053, "dependency_mismatch": 42468, "value_mismatch": 3118, "answer_mismatch": 2802, "extra_nodes": 3953, "missing_nodes": 3386, "perfect_with_extra": 461}}, "results_path": "results/composition/SDAR-100M-block8-epoch4-op17-20/graphs/SDAR-100M-block8-epoch4-op17-20_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 462476, "value_mismatch": 492979, "dependency_mismatch": 715666, "answer_mismatch": 387337, "extra_nodes": 43393, "missing_nodes": 161049, "perfect_with_extra": 417, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 156481, "value_mismatch": 164641, "answer_mismatch": 128108, "dependency_mismatch": 239441, "missing_nodes": 54318, "extra_nodes": 15043, "perfect_with_extra": 267}, "movie_festival_awards": {"total": 418816, "perfect": 149550, "value_mismatch": 165810, "dependency_mismatch": 235807, "answer_mismatch": 131364, "missing_nodes": 53389, "extra_nodes": 14144, "perfect_with_extra": 47}, "teachers_in_school": {"total": 430464, "perfect": 156445, "dependency_mismatch": 240418, "value_mismatch": 162528, "answer_mismatch": 127865, "extra_nodes": 14206, "perfect_with_extra": 103, "missing_nodes": 53342}}, "results_path": "results/composition/SDAR-100M-block8-epoch4-op17-20/graphs/SDAR-100M-block8-epoch4-op17-20_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1485142, "value_mismatch": 502998, "dependency_mismatch": 842427, "answer_mismatch": 396736, "extra_nodes": 55147, "missing_nodes": 171109, "perfect_with_extra": 1257, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch4-op17-20/graphs/SDAR-100M-block8-epoch4-op17-20_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 493624, "dependency_mismatch": 280310, "value_mismatch": 167996, "answer_mismatch": 131236, "extra_nodes": 18847, "missing_nodes": 57423, "perfect_with_extra": 501}, "teachers_in_school": {"total": 821120, "perfect": 502915, "dependency_mismatch": 283842, "value_mismatch": 166074, "answer_mismatch": 131334, "extra_nodes": 18203, "missing_nodes": 56911, "perfect_with_extra": 248}, "movie_festival_awards": {"total": 801152, "perfect": 488603, "dependency_mismatch": 278275, "value_mismatch": 168928, "answer_mismatch": 134166, "extra_nodes": 18097, "missing_nodes": 56775, "perfect_with_extra": 508}}}} diff --git a/composition/SDAR-100M-block8-epoch4-op17-20/graphs/results.json b/composition/SDAR-100M-block8-epoch4-op17-20/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..80e64c583e10c04bead33310b249701e5a75e5f8 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op17-20/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block8-epoch4-op17-20_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990625, + "pass@2": 0.990625, + "pass@4": 0.99375, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9746031746031746, + "pass@2": 0.9873015873015873, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9700598802395209, + "pass@2": 0.9760479041916168, + "pass@4": 0.9940119760479041, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8964497041420119, + "pass@2": 0.9260355029585798, + "pass@4": 0.9704142011834319, + "pass@8": 0.9792899408284024, + "pass@16": 0.9940828402366864, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8108974358974359, + "pass@2": 0.8910256410256411, + "pass@4": 0.9487179487179487, + "pass@8": 0.9807692307692307, + "pass@16": 0.9935897435897436, + "pass@32": 0.9967948717948718, + "pass@64": 0.9967948717948718, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7464788732394366, + "pass@2": 0.8225352112676056, + "pass@4": 0.8985915492957747, + "pass@8": 0.952112676056338, + "pass@16": 0.9915492957746479, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6460176991150443, + "pass@2": 0.7050147492625368, + "pass@4": 0.7817109144542773, + "pass@8": 0.8731563421828908, + "pass@16": 0.8967551622418879, + "pass@32": 0.9292035398230089, + "pass@64": 0.9469026548672567, + "pass@128": 0.9587020648967551 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9746031746031746, + "6": 0.9700598802395209, + "7": 0.8964497041420119, + "8": 0.8108974358974359, + "9": 0.7464788732394366, + "10": 0.6460176991150443 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9873015873015873, + "6": 0.9760479041916168, + "7": 0.9260355029585798, + "8": 0.8910256410256411, + "9": 0.8225352112676056, + "10": 0.7050147492625368 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 1.0, + "6": 0.9940119760479041, + "7": 0.9704142011834319, + "8": 0.9487179487179487, + "9": 0.8985915492957747, + "10": 0.7817109144542773 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9792899408284024, + "8": 0.9807692307692307, + "9": 0.952112676056338, + "10": 0.8731563421828908 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9940828402366864, + "8": 0.9935897435897436, + "9": 0.9915492957746479, + "10": 0.8967551622418879 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 0.9971830985915493, + "10": 0.9292035398230089 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 0.9971830985915493, + "10": 0.9469026548672567 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9587020648967551 + } + }, + "total_pass_at_k": { + "pass@1": 0.8902397838568051, + "pass@2": 0.9199594731509625, + "pass@4": 0.9527186761229315, + "pass@8": 0.9753461668355285, + "pass@16": 0.9858156028368794, + "pass@32": 0.9912191827085444, + "pass@64": 0.9932455251604188, + "pass@128": 0.9949341438703141 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9558823529411765, + "pass@2": 0.9852941176470589, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9526813880126183, + "pass@2": 0.9684542586750788, + "pass@4": 0.9810725552050473, + "pass@8": 0.9968454258675079, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8639240506329114, + "pass@2": 0.9177215189873418, + "pass@4": 0.9493670886075949, + "pass@8": 0.9873417721518988, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8198198198198198, + "pass@2": 0.8708708708708709, + "pass@4": 0.9369369369369369, + "pass@8": 0.984984984984985, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7717041800643086, + "pass@2": 0.8520900321543409, + "pass@4": 0.9035369774919614, + "pass@8": 0.954983922829582, + "pass@16": 0.9807073954983923, + "pass@32": 0.9935691318327974, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6268656716417911, + "pass@2": 0.6776119402985075, + "pass@4": 0.764179104477612, + "pass@8": 0.8298507462686567, + "pass@16": 0.8955223880597015, + "pass@32": 0.9402985074626866, + "pass@64": 0.9641791044776119, + "pass@128": 0.9701492537313433 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.9558823529411765, + "6": 0.9526813880126183, + "7": 0.8639240506329114, + "8": 0.8198198198198198, + "9": 0.7717041800643086, + "10": 0.6268656716417911 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9852941176470589, + "6": 0.9684542586750788, + "7": 0.9177215189873418, + "8": 0.8708708708708709, + "9": 0.8520900321543409, + "10": 0.6776119402985075 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9810725552050473, + "7": 0.9493670886075949, + "8": 0.9369369369369369, + "9": 0.9035369774919614, + "10": 0.764179104477612 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9968454258675079, + "7": 0.9873417721518988, + "8": 0.984984984984985, + "9": 0.954983922829582, + "10": 0.8298507462686567 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9807073954983923, + "10": 0.8955223880597015 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9935691318327974, + "10": 0.9402985074626866 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9641791044776119 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9701492537313433 + } + }, + "total_pass_at_k": { + "pass@1": 0.8885169065952461, + "pass@2": 0.9203213927017074, + "pass@4": 0.9491128222296619, + "pass@8": 0.9728824907934382, + "pass@16": 0.9862738533645798, + "pass@32": 0.9926347505858721, + "pass@64": 0.9959825912286575, + "pass@128": 0.9966521593572146 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9792899408284024, + "pass@2": 0.9881656804733728, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9623188405797102, + "pass@2": 0.9826086956521739, + "pass@4": 0.9942028985507246, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9713467048710601, + "pass@2": 0.9770773638968482, + "pass@4": 0.9856733524355301, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8757225433526011, + "pass@2": 0.9277456647398844, + "pass@4": 0.9624277456647399, + "pass@8": 0.9855491329479769, + "pass@16": 0.9971098265895953, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7830985915492957, + "pass@2": 0.8619718309859155, + "pass@4": 0.9267605633802817, + "pass@8": 0.9802816901408451, + "pass@16": 0.9971830985915493, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7245508982035929, + "pass@2": 0.811377245508982, + "pass@4": 0.874251497005988, + "pass@8": 0.9311377245508982, + "pass@16": 0.9700598802395209, + "pass@32": 0.9970059880239521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6411042944785276, + "pass@2": 0.696319018404908, + "pass@4": 0.7668711656441718, + "pass@8": 0.8558282208588958, + "pass@16": 0.9079754601226994, + "pass@32": 0.9325153374233128, + "pass@64": 0.9447852760736196, + "pass@128": 0.9693251533742331 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9792899408284024, + "5": 0.9623188405797102, + "6": 0.9713467048710601, + "7": 0.8757225433526011, + "8": 0.7830985915492957, + "9": 0.7245508982035929, + "10": 0.6411042944785276 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9881656804733728, + "5": 0.9826086956521739, + "6": 0.9770773638968482, + "7": 0.9277456647398844, + "8": 0.8619718309859155, + "9": 0.811377245508982, + "10": 0.696319018404908 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9942028985507246, + "6": 0.9856733524355301, + "7": 0.9624277456647399, + "8": 0.9267605633802817, + "9": 0.874251497005988, + "10": 0.7668711656441718 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9855491329479769, + "8": 0.9802816901408451, + "9": 0.9311377245508982, + "10": 0.8558282208588958 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9971830985915493, + "9": 0.9700598802395209, + "10": 0.9079754601226994 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.9325153374233128 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9447852760736196 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9693251533742331 + } + }, + "total_pass_at_k": { + "pass@1": 0.8823722149410222, + "pass@2": 0.916775884665793, + "pass@4": 0.9462647444298821, + "pass@8": 0.973132372214941, + "pass@16": 0.9862385321100917, + "pass@32": 0.9924639580602883, + "pass@64": 0.9941022280471822, + "pass@128": 0.9967234600262124 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.986, + "pass@2": 0.993, + "pass@4": 0.998, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.964, + "pass@2": 0.985, + "pass@4": 0.998, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.965, + "pass@2": 0.974, + "pass@4": 0.987, + "pass@8": 0.999, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.879, + "pass@2": 0.924, + "pass@4": 0.961, + "pass@8": 0.984, + "pass@16": 0.997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.804, + "pass@2": 0.874, + "pass@4": 0.937, + "pass@8": 0.982, + "pass@16": 0.997, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.747, + "pass@2": 0.828, + "pass@4": 0.892, + "pass@8": 0.946, + "pass@16": 0.981, + "pass@32": 0.996, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.638, + "pass@2": 0.693, + "pass@4": 0.771, + "pass@8": 0.853, + "pass@16": 0.9, + "pass@32": 0.934, + "pass@64": 0.952, + "pass@128": 0.966 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.986, + "5": 0.964, + "6": 0.965, + "7": 0.879, + "8": 0.804, + "9": 0.747, + "10": 0.638 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.993, + "5": 0.985, + "6": 0.974, + "7": 0.924, + "8": 0.874, + "9": 0.828, + "10": 0.693 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.998, + "5": 0.998, + "6": 0.987, + "7": 0.961, + "8": 0.937, + "9": 0.892, + "10": 0.771 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.999, + "7": 0.984, + "8": 0.982, + "9": 0.946, + "10": 0.853 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.997, + "8": 0.997, + "9": 0.981, + "10": 0.9 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.996, + "10": 0.934 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 0.999, + "10": 0.952 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 1.0, + "10": 0.966 + } + }, + "total_pass_at_k": { + "pass@1": 0.887, + "pass@2": 0.919, + "pass@4": 0.9493333333333334, + "pass@8": 0.9737777777777777, + "pass@16": 0.9861111111111112, + "pass@32": 0.9921111111111112, + "pass@64": 0.9944444444444445, + "pass@128": 0.9961111111111111 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block8-epoch4-op17-20_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5779036827195467, + "pass@2": 0.6373937677053825, + "pass@4": 0.7308781869688386, + "pass@8": 0.8158640226628895, + "pass@16": 0.8838526912181303, + "pass@32": 0.9178470254957507, + "pass@64": 0.9376770538243626, + "pass@128": 0.943342776203966 + }, + "12": { + "pass@1": 0.5790960451977402, + "pass@2": 0.632768361581921, + "pass@4": 0.6892655367231638, + "pass@8": 0.7457627118644068, + "pass@16": 0.7937853107344632, + "pass@32": 0.8389830508474576, + "pass@64": 0.8559322033898306, + "pass@128": 0.867231638418079 + }, + "13": { + "pass@1": 0.445141065830721, + "pass@2": 0.5015673981191222, + "pass@4": 0.567398119122257, + "pass@8": 0.664576802507837, + "pass@16": 0.7304075235109718, + "pass@32": 0.7774294670846394, + "pass@64": 0.8056426332288401, + "pass@128": 0.8275862068965517 + }, + "14": { + "pass@1": 0.4517133956386293, + "pass@2": 0.5046728971962616, + "pass@4": 0.5545171339563862, + "pass@8": 0.6043613707165109, + "pass@16": 0.6853582554517134, + "pass@32": 0.735202492211838, + "pass@64": 0.7850467289719626, + "pass@128": 0.8286604361370716 + }, + "15": { + "pass@1": 0.3786127167630058, + "pass@2": 0.4421965317919075, + "pass@4": 0.4913294797687861, + "pass@8": 0.5867052023121387, + "pass@16": 0.6502890173410405, + "pass@32": 0.7283236994219653, + "pass@64": 0.8092485549132948, + "pass@128": 0.8583815028901735 + }, + "16": { + "pass@1": 0.3064516129032258, + "pass@2": 0.3548387096774194, + "pass@4": 0.41935483870967744, + "pass@8": 0.4612903225806452, + "pass@16": 0.5064516129032258, + "pass@32": 0.5548387096774193, + "pass@64": 0.6161290322580645, + "pass@128": 0.6612903225806451 + }, + "17": { + "pass@1": 0.30346820809248554, + "pass@2": 0.33815028901734107, + "pass@4": 0.3988439306358382, + "pass@8": 0.43352601156069365, + "pass@16": 0.48265895953757226, + "pass@32": 0.5404624277456648, + "pass@64": 0.5982658959537572, + "pass@128": 0.6589595375722543 + }, + "18": { + "pass@1": 0.22857142857142856, + "pass@2": 0.28285714285714286, + "pass@4": 0.32, + "pass@8": 0.36, + "pass@16": 0.3942857142857143, + "pass@32": 0.44285714285714284, + "pass@64": 0.5257142857142857, + "pass@128": 0.6171428571428571 + }, + "19": { + "pass@1": 0.12612612612612611, + "pass@2": 0.1831831831831832, + "pass@4": 0.21021021021021022, + "pass@8": 0.2552552552552553, + "pass@16": 0.3003003003003003, + "pass@32": 0.3333333333333333, + "pass@64": 0.3813813813813814, + "pass@128": 0.4174174174174174 + }, + "20": { + "pass@1": 0.17117117117117117, + "pass@2": 0.21921921921921922, + "pass@4": 0.25825825825825827, + "pass@8": 0.2912912912912913, + "pass@16": 0.30930930930930933, + "pass@32": 0.35435435435435436, + "pass@64": 0.3813813813813814, + "pass@128": 0.4264264264264264 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5779036827195467, + "12": 0.5790960451977402, + "13": 0.445141065830721, + "14": 0.4517133956386293, + "15": 0.3786127167630058, + "16": 0.3064516129032258, + "17": 0.30346820809248554, + "18": 0.22857142857142856, + "19": 0.12612612612612611, + "20": 0.17117117117117117 + }, + "pass@2": { + "11": 0.6373937677053825, + "12": 0.632768361581921, + "13": 0.5015673981191222, + "14": 0.5046728971962616, + "15": 0.4421965317919075, + "16": 0.3548387096774194, + "17": 0.33815028901734107, + "18": 0.28285714285714286, + "19": 0.1831831831831832, + "20": 0.21921921921921922 + }, + "pass@4": { + "11": 0.7308781869688386, + "12": 0.6892655367231638, + "13": 0.567398119122257, + "14": 0.5545171339563862, + "15": 0.4913294797687861, + "16": 0.41935483870967744, + "17": 0.3988439306358382, + "18": 0.32, + "19": 0.21021021021021022, + "20": 0.25825825825825827 + }, + "pass@8": { + "11": 0.8158640226628895, + "12": 0.7457627118644068, + "13": 0.664576802507837, + "14": 0.6043613707165109, + "15": 0.5867052023121387, + "16": 0.4612903225806452, + "17": 0.43352601156069365, + "18": 0.36, + "19": 0.2552552552552553, + "20": 0.2912912912912913 + }, + "pass@16": { + "11": 0.8838526912181303, + "12": 0.7937853107344632, + "13": 0.7304075235109718, + "14": 0.6853582554517134, + "15": 0.6502890173410405, + "16": 0.5064516129032258, + "17": 0.48265895953757226, + "18": 0.3942857142857143, + "19": 0.3003003003003003, + "20": 0.30930930930930933 + }, + "pass@32": { + "11": 0.9178470254957507, + "12": 0.8389830508474576, + "13": 0.7774294670846394, + "14": 0.735202492211838, + "15": 0.7283236994219653, + "16": 0.5548387096774193, + "17": 0.5404624277456648, + "18": 0.44285714285714284, + "19": 0.3333333333333333, + "20": 0.35435435435435436 + }, + "pass@64": { + "11": 0.9376770538243626, + "12": 0.8559322033898306, + "13": 0.8056426332288401, + "14": 0.7850467289719626, + "15": 0.8092485549132948, + "16": 0.6161290322580645, + "17": 0.5982658959537572, + "18": 0.5257142857142857, + "19": 0.3813813813813814, + "20": 0.3813813813813814 + }, + "pass@128": { + "11": 0.943342776203966, + "12": 0.867231638418079, + "13": 0.8275862068965517, + "14": 0.8286604361370716, + "15": 0.8583815028901735, + "16": 0.6612903225806451, + "17": 0.6589595375722543, + "18": 0.6171428571428571, + "19": 0.4174174174174174, + "20": 0.4264264264264264 + } + }, + "total_pass_at_k": { + "pass@1": 0.3583952451708767, + "pass@2": 0.4112927191679049, + "pass@4": 0.46567607726597327, + "pass@8": 0.5236255572065379, + "pass@16": 0.575334323922734, + "pass@32": 0.6240713224368499, + "pass@64": 0.6713224368499257, + "pass@128": 0.712332838038633 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5746031746031746, + "pass@2": 0.6666666666666666, + "pass@4": 0.7428571428571429, + "pass@8": 0.8507936507936508, + "pass@16": 0.9238095238095239, + "pass@32": 0.9587301587301588, + "pass@64": 0.9777777777777777, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.5932721712538226, + "pass@2": 0.6452599388379205, + "pass@4": 0.7033639143730887, + "pass@8": 0.7522935779816514, + "pass@16": 0.8195718654434251, + "pass@32": 0.8562691131498471, + "pass@64": 0.8868501529051988, + "pass@128": 0.8929663608562691 + }, + "13": { + "pass@1": 0.48148148148148145, + "pass@2": 0.5462962962962963, + "pass@4": 0.5987654320987654, + "pass@8": 0.6481481481481481, + "pass@16": 0.691358024691358, + "pass@32": 0.7253086419753086, + "pass@64": 0.75, + "pass@128": 0.7746913580246914 + }, + "14": { + "pass@1": 0.4332344213649852, + "pass@2": 0.49851632047477745, + "pass@4": 0.5608308605341247, + "pass@8": 0.6112759643916914, + "pass@16": 0.6765578635014837, + "pass@32": 0.7329376854599406, + "pass@64": 0.7863501483679525, + "pass@128": 0.8308605341246291 + }, + "15": { + "pass@1": 0.3765432098765432, + "pass@2": 0.4166666666666667, + "pass@4": 0.4783950617283951, + "pass@8": 0.5370370370370371, + "pass@16": 0.6141975308641975, + "pass@32": 0.6944444444444444, + "pass@64": 0.7685185185185185, + "pass@128": 0.8271604938271605 + }, + "16": { + "pass@1": 0.3274336283185841, + "pass@2": 0.3834808259587021, + "pass@4": 0.4365781710914454, + "pass@8": 0.4896755162241888, + "pass@16": 0.551622418879056, + "pass@32": 0.5958702064896755, + "pass@64": 0.6283185840707964, + "pass@128": 0.6696165191740413 + }, + "17": { + "pass@1": 0.2469879518072289, + "pass@2": 0.3102409638554217, + "pass@4": 0.3614457831325301, + "pass@8": 0.39759036144578314, + "pass@16": 0.4427710843373494, + "pass@32": 0.4879518072289157, + "pass@64": 0.536144578313253, + "pass@128": 0.6054216867469879 + }, + "18": { + "pass@1": 0.22580645161290322, + "pass@2": 0.27419354838709675, + "pass@4": 0.3064516129032258, + "pass@8": 0.3419354838709677, + "pass@16": 0.3903225806451613, + "pass@32": 0.45161290322580644, + "pass@64": 0.5161290322580645, + "pass@128": 0.5870967741935483 + }, + "19": { + "pass@1": 0.14803625377643503, + "pass@2": 0.1903323262839879, + "pass@4": 0.2326283987915408, + "pass@8": 0.26586102719033233, + "pass@16": 0.29003021148036257, + "pass@32": 0.32326283987915405, + "pass@64": 0.3474320241691843, + "pass@128": 0.3806646525679758 + }, + "20": { + "pass@1": 0.11411411411411411, + "pass@2": 0.15915915915915915, + "pass@4": 0.2042042042042042, + "pass@8": 0.23723723723723725, + "pass@16": 0.26426426426426425, + "pass@32": 0.2882882882882883, + "pass@64": 0.34234234234234234, + "pass@128": 0.3783783783783784 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5746031746031746, + "12": 0.5932721712538226, + "13": 0.48148148148148145, + "14": 0.4332344213649852, + "15": 0.3765432098765432, + "16": 0.3274336283185841, + "17": 0.2469879518072289, + "18": 0.22580645161290322, + "19": 0.14803625377643503, + "20": 0.11411411411411411 + }, + "pass@2": { + "11": 0.6666666666666666, + "12": 0.6452599388379205, + "13": 0.5462962962962963, + "14": 0.49851632047477745, + "15": 0.4166666666666667, + "16": 0.3834808259587021, + "17": 0.3102409638554217, + "18": 0.27419354838709675, + "19": 0.1903323262839879, + "20": 0.15915915915915915 + }, + "pass@4": { + "11": 0.7428571428571429, + "12": 0.7033639143730887, + "13": 0.5987654320987654, + "14": 0.5608308605341247, + "15": 0.4783950617283951, + "16": 0.4365781710914454, + "17": 0.3614457831325301, + "18": 0.3064516129032258, + "19": 0.2326283987915408, + "20": 0.2042042042042042 + }, + "pass@8": { + "11": 0.8507936507936508, + "12": 0.7522935779816514, + "13": 0.6481481481481481, + "14": 0.6112759643916914, + "15": 0.5370370370370371, + "16": 0.4896755162241888, + "17": 0.39759036144578314, + "18": 0.3419354838709677, + "19": 0.26586102719033233, + "20": 0.23723723723723725 + }, + "pass@16": { + "11": 0.9238095238095239, + "12": 0.8195718654434251, + "13": 0.691358024691358, + "14": 0.6765578635014837, + "15": 0.6141975308641975, + "16": 0.551622418879056, + "17": 0.4427710843373494, + "18": 0.3903225806451613, + "19": 0.29003021148036257, + "20": 0.26426426426426425 + }, + "pass@32": { + "11": 0.9587301587301588, + "12": 0.8562691131498471, + "13": 0.7253086419753086, + "14": 0.7329376854599406, + "15": 0.6944444444444444, + "16": 0.5958702064896755, + "17": 0.4879518072289157, + "18": 0.45161290322580644, + "19": 0.32326283987915405, + "20": 0.2882882882882883 + }, + "pass@64": { + "11": 0.9777777777777777, + "12": 0.8868501529051988, + "13": 0.75, + "14": 0.7863501483679525, + "15": 0.7685185185185185, + "16": 0.6283185840707964, + "17": 0.536144578313253, + "18": 0.5161290322580645, + "19": 0.3474320241691843, + "20": 0.34234234234234234 + }, + "pass@128": { + "11": 0.9841269841269841, + "12": 0.8929663608562691, + "13": 0.7746913580246914, + "14": 0.8308605341246291, + "15": 0.8271604938271605, + "16": 0.6696165191740413, + "17": 0.6054216867469879, + "18": 0.5870967741935483, + "19": 0.3806646525679758, + "20": 0.3783783783783784 + } + }, + "total_pass_at_k": { + "pass@1": 0.35116136919315405, + "pass@2": 0.4080073349633252, + "pass@4": 0.4614914425427873, + "pass@8": 0.511919315403423, + "pass@16": 0.5650977995110025, + "pass@32": 0.6100244498777506, + "pass@64": 0.6525061124694377, + "pass@128": 0.6916259168704156 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6174698795180723, + "pass@2": 0.6927710843373494, + "pass@4": 0.7650602409638554, + "pass@8": 0.8433734939759037, + "pass@16": 0.9036144578313253, + "pass@32": 0.9487951807228916, + "pass@64": 0.9518072289156626, + "pass@128": 0.9668674698795181 + }, + "12": { + "pass@1": 0.5705329153605015, + "pass@2": 0.6238244514106583, + "pass@4": 0.6990595611285266, + "pass@8": 0.7711598746081505, + "pass@16": 0.8150470219435737, + "pass@32": 0.8432601880877743, + "pass@64": 0.8589341692789969, + "pass@128": 0.8652037617554859 + }, + "13": { + "pass@1": 0.5042016806722689, + "pass@2": 0.5658263305322129, + "pass@4": 0.6526610644257703, + "pass@8": 0.7198879551820728, + "pass@16": 0.7871148459383753, + "pass@32": 0.8235294117647058, + "pass@64": 0.8487394957983193, + "pass@128": 0.8515406162464986 + }, + "14": { + "pass@1": 0.45321637426900585, + "pass@2": 0.52046783625731, + "pass@4": 0.5730994152046783, + "pass@8": 0.6286549707602339, + "pass@16": 0.6842105263157895, + "pass@32": 0.7543859649122807, + "pass@64": 0.7953216374269005, + "pass@128": 0.8362573099415205 + }, + "15": { + "pass@1": 0.3939393939393939, + "pass@2": 0.45454545454545453, + "pass@4": 0.5181818181818182, + "pass@8": 0.5727272727272728, + "pass@16": 0.6181818181818182, + "pass@32": 0.703030303030303, + "pass@64": 0.7787878787878788, + "pass@128": 0.8545454545454545 + }, + "16": { + "pass@1": 0.29914529914529914, + "pass@2": 0.37606837606837606, + "pass@4": 0.43304843304843305, + "pass@8": 0.4928774928774929, + "pass@16": 0.5413105413105413, + "pass@32": 0.6096866096866097, + "pass@64": 0.6666666666666666, + "pass@128": 0.7150997150997151 + }, + "17": { + "pass@1": 0.2329192546583851, + "pass@2": 0.27639751552795033, + "pass@4": 0.33540372670807456, + "pass@8": 0.39751552795031053, + "pass@16": 0.46273291925465837, + "pass@32": 0.5186335403726708, + "pass@64": 0.577639751552795, + "pass@128": 0.6211180124223602 + }, + "18": { + "pass@1": 0.23823529411764705, + "pass@2": 0.2676470588235294, + "pass@4": 0.3176470588235294, + "pass@8": 0.35, + "pass@16": 0.38529411764705884, + "pass@32": 0.4470588235294118, + "pass@64": 0.5205882352941177, + "pass@128": 0.5764705882352941 + }, + "19": { + "pass@1": 0.1636904761904762, + "pass@2": 0.20833333333333334, + "pass@4": 0.24702380952380953, + "pass@8": 0.2916666666666667, + "pass@16": 0.31547619047619047, + "pass@32": 0.34226190476190477, + "pass@64": 0.375, + "pass@128": 0.4255952380952381 + }, + "20": { + "pass@1": 0.1407185628742515, + "pass@2": 0.17065868263473055, + "pass@4": 0.2245508982035928, + "pass@8": 0.25149700598802394, + "pass@16": 0.2874251497005988, + "pass@32": 0.3083832335329341, + "pass@64": 0.3562874251497006, + "pass@128": 0.40718562874251496 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6174698795180723, + "12": 0.5705329153605015, + "13": 0.5042016806722689, + "14": 0.45321637426900585, + "15": 0.3939393939393939, + "16": 0.29914529914529914, + "17": 0.2329192546583851, + "18": 0.23823529411764705, + "19": 0.1636904761904762, + "20": 0.1407185628742515 + }, + "pass@2": { + "11": 0.6927710843373494, + "12": 0.6238244514106583, + "13": 0.5658263305322129, + "14": 0.52046783625731, + "15": 0.45454545454545453, + "16": 0.37606837606837606, + "17": 0.27639751552795033, + "18": 0.2676470588235294, + "19": 0.20833333333333334, + "20": 0.17065868263473055 + }, + "pass@4": { + "11": 0.7650602409638554, + "12": 0.6990595611285266, + "13": 0.6526610644257703, + "14": 0.5730994152046783, + "15": 0.5181818181818182, + "16": 0.43304843304843305, + "17": 0.33540372670807456, + "18": 0.3176470588235294, + "19": 0.24702380952380953, + "20": 0.2245508982035928 + }, + "pass@8": { + "11": 0.8433734939759037, + "12": 0.7711598746081505, + "13": 0.7198879551820728, + "14": 0.6286549707602339, + "15": 0.5727272727272728, + "16": 0.4928774928774929, + "17": 0.39751552795031053, + "18": 0.35, + "19": 0.2916666666666667, + "20": 0.25149700598802394 + }, + "pass@16": { + "11": 0.9036144578313253, + "12": 0.8150470219435737, + "13": 0.7871148459383753, + "14": 0.6842105263157895, + "15": 0.6181818181818182, + "16": 0.5413105413105413, + "17": 0.46273291925465837, + "18": 0.38529411764705884, + "19": 0.31547619047619047, + "20": 0.2874251497005988 + }, + "pass@32": { + "11": 0.9487951807228916, + "12": 0.8432601880877743, + "13": 0.8235294117647058, + "14": 0.7543859649122807, + "15": 0.703030303030303, + "16": 0.6096866096866097, + "17": 0.5186335403726708, + "18": 0.4470588235294118, + "19": 0.34226190476190477, + "20": 0.3083832335329341 + }, + "pass@64": { + "11": 0.9518072289156626, + "12": 0.8589341692789969, + "13": 0.8487394957983193, + "14": 0.7953216374269005, + "15": 0.7787878787878788, + "16": 0.6666666666666666, + "17": 0.577639751552795, + "18": 0.5205882352941177, + "19": 0.375, + "20": 0.3562874251497006 + }, + "pass@128": { + "11": 0.9668674698795181, + "12": 0.8652037617554859, + "13": 0.8515406162464986, + "14": 0.8362573099415205, + "15": 0.8545454545454545, + "16": 0.7150997150997151, + "17": 0.6211180124223602, + "18": 0.5764705882352941, + "19": 0.4255952380952381, + "20": 0.40718562874251496 + } + }, + "total_pass_at_k": { + "pass@1": 0.36128456735057984, + "pass@2": 0.41570026761819806, + "pass@4": 0.4766577460600654, + "pass@8": 0.5319655069878085, + "pass@16": 0.5801367826345525, + "pass@32": 0.6300921796015463, + "pass@64": 0.6732084448409158, + "pass@128": 0.712161760333036 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.59, + "pass@2": 0.665, + "pass@4": 0.746, + "pass@8": 0.836, + "pass@16": 0.903, + "pass@32": 0.941, + "pass@64": 0.955, + "pass@128": 0.964 + }, + "12": { + "pass@1": 0.581, + "pass@2": 0.634, + "pass@4": 0.697, + "pass@8": 0.756, + "pass@16": 0.809, + "pass@32": 0.846, + "pass@64": 0.867, + "pass@128": 0.875 + }, + "13": { + "pass@1": 0.478, + "pass@2": 0.539, + "pass@4": 0.608, + "pass@8": 0.679, + "pass@16": 0.738, + "pass@32": 0.777, + "pass@64": 0.803, + "pass@128": 0.819 + }, + "14": { + "pass@1": 0.446, + "pass@2": 0.508, + "pass@4": 0.563, + "pass@8": 0.615, + "pass@16": 0.682, + "pass@32": 0.741, + "pass@64": 0.789, + "pass@128": 0.832 + }, + "15": { + "pass@1": 0.383, + "pass@2": 0.438, + "pass@4": 0.496, + "pass@8": 0.566, + "pass@16": 0.628, + "pass@32": 0.709, + "pass@64": 0.786, + "pass@128": 0.847 + }, + "16": { + "pass@1": 0.311, + "pass@2": 0.372, + "pass@4": 0.43, + "pass@8": 0.482, + "pass@16": 0.534, + "pass@32": 0.588, + "pass@64": 0.638, + "pass@128": 0.683 + }, + "17": { + "pass@1": 0.262, + "pass@2": 0.309, + "pass@4": 0.366, + "pass@8": 0.41, + "pass@16": 0.463, + "pass@32": 0.516, + "pass@64": 0.571, + "pass@128": 0.629 + }, + "18": { + "pass@1": 0.231, + "pass@2": 0.275, + "pass@4": 0.315, + "pass@8": 0.351, + "pass@16": 0.39, + "pass@32": 0.447, + "pass@64": 0.521, + "pass@128": 0.594 + }, + "19": { + "pass@1": 0.146, + "pass@2": 0.194, + "pass@4": 0.23, + "pass@8": 0.271, + "pass@16": 0.302, + "pass@32": 0.333, + "pass@64": 0.368, + "pass@128": 0.408 + }, + "20": { + "pass@1": 0.142, + "pass@2": 0.183, + "pass@4": 0.229, + "pass@8": 0.26, + "pass@16": 0.287, + "pass@32": 0.317, + "pass@64": 0.36, + "pass@128": 0.404 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.59, + "12": 0.581, + "13": 0.478, + "14": 0.446, + "15": 0.383, + "16": 0.311, + "17": 0.262, + "18": 0.231, + "19": 0.146, + "20": 0.142 + }, + "pass@2": { + "11": 0.665, + "12": 0.634, + "13": 0.539, + "14": 0.508, + "15": 0.438, + "16": 0.372, + "17": 0.309, + "18": 0.275, + "19": 0.194, + "20": 0.183 + }, + "pass@4": { + "11": 0.746, + "12": 0.697, + "13": 0.608, + "14": 0.563, + "15": 0.496, + "16": 0.43, + "17": 0.366, + "18": 0.315, + "19": 0.23, + "20": 0.229 + }, + "pass@8": { + "11": 0.836, + "12": 0.756, + "13": 0.679, + "14": 0.615, + "15": 0.566, + "16": 0.482, + "17": 0.41, + "18": 0.351, + "19": 0.271, + "20": 0.26 + }, + "pass@16": { + "11": 0.903, + "12": 0.809, + "13": 0.738, + "14": 0.682, + "15": 0.628, + "16": 0.534, + "17": 0.463, + "18": 0.39, + "19": 0.302, + "20": 0.287 + }, + "pass@32": { + "11": 0.941, + "12": 0.846, + "13": 0.777, + "14": 0.741, + "15": 0.709, + "16": 0.588, + "17": 0.516, + "18": 0.447, + "19": 0.333, + "20": 0.317 + }, + "pass@64": { + "11": 0.955, + "12": 0.867, + "13": 0.803, + "14": 0.789, + "15": 0.786, + "16": 0.638, + "17": 0.571, + "18": 0.521, + "19": 0.368, + "20": 0.36 + }, + "pass@128": { + "11": 0.964, + "12": 0.875, + "13": 0.819, + "14": 0.832, + "15": 0.847, + "16": 0.683, + "17": 0.629, + "18": 0.594, + "19": 0.408, + "20": 0.404 + } + }, + "total_pass_at_k": { + "pass@1": 0.357, + "pass@2": 0.4117, + "pass@4": 0.468, + "pass@8": 0.5226, + "pass@16": 0.5736, + "pass@32": 0.6215, + "pass@64": 0.6658, + "pass@128": 0.7055 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block8-epoch4-op17-20/summary.csv b/composition/SDAR-100M-block8-epoch4-op17-20/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..05cd8ab980d704c503aa56197b38cbd28d03e67a --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op17-20/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch4-op17-20,0.9918411458333334,0.6973953125,0.8368696546052632,0.0,0.0,0.0,199.07718576388888,374.408946875,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block8-epoch4-op17-20/summary.json b/composition/SDAR-100M-block8-epoch4-op17-20/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5a2f3d9be4b4efa638e89c33092d8005c4f8be6d --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op17-20/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch4-op17-20", + "id_acc": 0.9918411458333334, + "ood_acc": 0.6973953125, + "total_acc": 0.8368696546052632, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.07718576388888, + "ood_avg_resp_len": 374.408946875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 678493, + "answer_accuracy": 0.8379270569870376, + "avg_response_len": 278.5486854845084, + "resp_tokens_sum": 225548670, + "pass_at_k": { + "pass@1": 0.8379270569870376, + "pass@2": 0.8753173808068597, + "pass@4": 0.9021178882634914, + "pass@8": 0.9219103027016352, + "pass@16": 0.936677817906899, + "pass@32": 0.9477679035326472, + "pass@64": 0.9563979950460396, + "pass@128": 0.9636421119190641 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.947686209439528, + "pass@2": 0.9617538992636985, + "pass@4": 0.9725950904367042, + "pass@8": 0.9805598544320808, + "pass@16": 0.9860735851683115, + "pass@32": 0.9893290501813384, + "pass@64": 0.9908876713836634, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9997566199376947, + "pass@2": 0.9999862020752079, + "pass@4": 0.9999999632055339, + "pass@8": 0.9999999999999803, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999283256880734, + "pass@2": 0.9999988712706783, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.9999961552657481, + "pass@4": 0.9999999985353394, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9978422619047619, + "pass@2": 0.9992364235720536, + "pass@4": 0.9998443864159838, + "pass@8": 0.9999934611094885, + "pass@16": 0.999999993102673, + "pass@32": 0.9999999999999994, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9969385785515584, + "pass@4": 0.9984315971543977, + "pass@8": 0.9995862366315358, + "pass@16": 0.9999714614533096, + "pass@32": 0.999999909391957, + "pass@64": 0.9999999999999891, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9987056213017751, + "pass@2": 0.9998150887573964, + "pass@4": 0.9999939333581823, + "pass@8": 0.9999999935548729, + "pass@16": 0.999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927634214743589, + "pass@2": 0.997842611169998, + "pass@4": 0.9996803302231451, + "pass@8": 0.9999892239856973, + "pass@16": 0.9999999910956074, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9951144366197183, + "pass@2": 0.9977098813352555, + "pass@4": 0.9988829544194299, + "pass@8": 0.999601568979854, + "pass@16": 0.999951739437933, + "pass@32": 0.9999996009748804, + "pass@64": 0.9999999999993653, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9421033994334278, + "pass@2": 0.9585153827708507, + "pass@4": 0.9683495726277837, + "pass@8": 0.9750743355007809, + "pass@16": 0.9806874497365086, + "pass@32": 0.9855674217757143, + "pass@64": 0.9892681802260217, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8856594279661016, + "pass@2": 0.910248176075448, + "pass@4": 0.9253412335640251, + "pass@8": 0.9357745885481913, + "pass@16": 0.9435988842897304, + "pass@32": 0.9494690435475064, + "pass@64": 0.9538236623937969, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.8312842868338558, + "pass@2": 0.8800130513662283, + "pass@4": 0.9112304953847852, + "pass@8": 0.9301201681225754, + "pass@16": 0.9426301334874778, + "pass@32": 0.9531258102406001, + "pass@64": 0.962334874645846, + "pass@128": 0.9717868338557993 + }, + "14": { + "pass@1": 0.8299260124610592, + "pass@2": 0.8766005592758852, + "pass@4": 0.9035752581161002, + "pass@8": 0.9220158894644768, + "pass@16": 0.9356224843327265, + "pass@32": 0.945334584744341, + "pass@64": 0.953042570524806, + "pass@128": 0.9595015576323987 + }, + "15": { + "pass@1": 0.8186867774566474, + "pass@2": 0.877645532747713, + "pass@4": 0.9140065072248914, + "pass@8": 0.937924646916376, + "pass@16": 0.9541987911233919, + "pass@32": 0.965099381921458, + "pass@64": 0.9726851787993989, + "pass@128": 0.9797687861271677 + }, + "16": { + "pass@1": 0.6638860887096775, + "pass@2": 0.7459883794767583, + "pass@4": 0.8018100526748667, + "pass@8": 0.8433015429764784, + "pass@16": 0.8753356035074012, + "pass@32": 0.9009404640966499, + "pass@64": 0.9223832471129497, + "pass@128": 0.9419354838709677 + }, + "17": { + "pass@1": 0.6308932442196532, + "pass@2": 0.7216910216876794, + "pass@4": 0.7815059672815464, + "pass@8": 0.8225626990224884, + "pass@16": 0.8534324527753754, + "pass@32": 0.8779455950894322, + "pass@64": 0.8973389717015748, + "pass@128": 0.9132947976878613 + }, + "18": { + "pass@1": 0.551875, + "pass@2": 0.6496741422947138, + "pass@4": 0.7255719232417374, + "pass@8": 0.7824379510059899, + "pass@16": 0.823303427941537, + "pass@32": 0.8522402739187724, + "pass@64": 0.8735607870746009, + "pass@128": 0.8914285714285715 + }, + "19": { + "pass@1": 0.4078688063063063, + "pass@2": 0.5134215169894304, + "pass@4": 0.6041881234440287, + "pass@8": 0.6818408987273152, + "pass@16": 0.7434984007892304, + "pass@32": 0.7890733463330541, + "pass@64": 0.8258918288508185, + "pass@128": 0.8558558558558559 + }, + "20": { + "pass@1": 0.4509196696696697, + "pass@2": 0.5596455215057575, + "pass@4": 0.6476805069974355, + "pass@8": 0.7155572588224337, + "pass@16": 0.7669233647580539, + "pass@32": 0.8067563935871334, + "pass@64": 0.837089022951731, + "pass@128": 0.8618618618618619 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 689786, + "answer_accuracy": 0.8400550467653936, + "avg_response_len": 287.4098645752143, + "resp_tokens_sum": 235997988, + "pass_at_k": { + "pass@1": 0.8400550467653936, + "pass@2": 0.8763150081931377, + "pass@4": 0.9022558528634762, + "pass@8": 0.9215923694922954, + "pass@16": 0.9365406707220455, + "pass@32": 0.9482855604084754, + "pass@64": 0.9576451027840119, + "pass@128": 0.9653936087295402 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9353671395705522, + "pass@2": 0.9489883489928022, + "pass@4": 0.9602921086167909, + "pass@8": 0.970010030193041, + "pass@16": 0.9784615849176065, + "pass@32": 0.9861956753511112, + "pass@64": 0.9914655018647149, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9968297101449275, + "pass@2": 0.9986281096656396, + "pass@4": 0.9996703835933552, + "pass@8": 0.9999750963168695, + "pass@16": 0.9999998721310288, + "pass@32": 0.9999999999994407, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9955229226361032, + "pass@2": 0.9963235086749543, + "pass@4": 0.9971980834952937, + "pass@8": 0.9978938143484848, + "pass@16": 0.9985466574181527, + "pass@32": 0.9993380089577011, + "pass@64": 0.9999174243620693, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998193641618497, + "pass@2": 0.9999946662646215, + "pass@4": 0.9999999959362016, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9968089788732394, + "pass@2": 0.9978419235887768, + "pass@4": 0.9984183204212148, + "pass@8": 0.9991300714136, + "pass@16": 0.9997538434444305, + "pass@32": 0.9999857247050008, + "pass@64": 0.9999999936816814, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935675523952096, + "pass@2": 0.9949026651421564, + "pass@4": 0.9957427633484436, + "pass@8": 0.9969632965123278, + "pass@16": 0.9984388741597334, + "pass@32": 0.999581272939488, + "pass@64": 0.9999802411950417, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9559723268072289, + "pass@2": 0.966372008704108, + "pass@4": 0.9713946641458974, + "pass@8": 0.9749261474893676, + "pass@16": 0.9779525234791777, + "pass@32": 0.9806901170607644, + "pass@64": 0.9830740245540501, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.895694553291536, + "pass@2": 0.9185176671685638, + "pass@4": 0.9316084925121034, + "pass@8": 0.9394861999416354, + "pass@16": 0.9443789712253488, + "pass@32": 0.9476920343833845, + "pass@64": 0.9504452941632805, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.8206188725490197, + "pass@2": 0.8654555818390349, + "pass@4": 0.8914457194951465, + "pass@8": 0.91024432511722, + "pass@16": 0.9260177446913851, + "pass@32": 0.9387626126789113, + "pass@64": 0.9509378123272939, + "pass@128": 0.9635854341736695 + }, + "14": { + "pass@1": 0.8450520833333334, + "pass@2": 0.892094902610858, + "pass@4": 0.9161785703322166, + "pass@8": 0.9328964288192578, + "pass@16": 0.9477363031164306, + "pass@32": 0.960031109634046, + "pass@64": 0.9693199907235079, + "pass@128": 0.9766081871345029 + }, + "15": { + "pass@1": 0.8125236742424242, + "pass@2": 0.8730982313290383, + "pass@4": 0.9092206175932549, + "pass@8": 0.9321373846919218, + "pass@16": 0.947976048382885, + "pass@32": 0.9615416739411536, + "pass@64": 0.9736111224580455, + "pass@128": 0.9818181818181818 + }, + "16": { + "pass@1": 0.6658653846153846, + "pass@2": 0.75107889158086, + "pass@4": 0.807817280051532, + "pass@8": 0.8455472972839031, + "pass@16": 0.8726949869908102, + "pass@32": 0.8942385463597932, + "pass@64": 0.9125603746766824, + "pass@128": 0.9287749287749287 + }, + "17": { + "pass@1": 0.6382472826086957, + "pass@2": 0.7308938199980439, + "pass@4": 0.7938274872085086, + "pass@8": 0.8379544601446831, + "pass@16": 0.8705400266432815, + "pass@32": 0.8951335125041588, + "pass@64": 0.9145615683410627, + "pass@128": 0.9347826086956522 + }, + "18": { + "pass@1": 0.597219669117647, + "pass@2": 0.6891023043075495, + "pass@4": 0.7591783864333136, + "pass@8": 0.8121559273453292, + "pass@16": 0.8524030793941533, + "pass@32": 0.8851876403918829, + "pass@64": 0.9114327068575712, + "pass@128": 0.9294117647058824 + }, + "19": { + "pass@1": 0.39669363839285715, + "pass@2": 0.5050988499484439, + "pass@4": 0.5999355119672539, + "pass@8": 0.6770325751872642, + "pass@16": 0.7367422561935473, + "pass@32": 0.7810862819828637, + "pass@64": 0.8162268287403831, + "pass@128": 0.8511904761904762 + }, + "20": { + "pass@1": 0.4049869011976048, + "pass@2": 0.5131853664717809, + "pass@4": 0.6042743593240965, + "pass@8": 0.6785910123757385, + "pass@16": 0.738234545184363, + "pass@32": 0.7842155855252223, + "pass@64": 0.8183162239457017, + "pass@128": 0.8413173652694611 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 666988, + "answer_accuracy": 0.8325361479469564, + "avg_response_len": 308.34811870905895, + "resp_tokens_sum": 247033712, + "pass_at_k": { + "pass@1": 0.8325361479469564, + "pass@2": 0.8700066196016242, + "pass@4": 0.8969526549634996, + "pass@8": 0.9167296000966144, + "pass@16": 0.9314247396654343, + "pass@32": 0.9424912315875398, + "pass@64": 0.951062834467024, + "pass@128": 0.9579805080683815 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9563199626865672, + "pass@2": 0.9673334851333878, + "pass@4": 0.975943412577159, + "pass@8": 0.9832993500745408, + "pass@16": 0.9887067340999656, + "pass@32": 0.9917497873815814, + "pass@64": 0.9932677279113761, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9985753676470588, + "pass@2": 0.9996960398332562, + "pass@4": 0.9999747452524317, + "pass@8": 0.9999998417920103, + "pass@16": 0.9999999999980195, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9974122634069401, + "pass@2": 0.9991131641372115, + "pass@4": 0.999854572299125, + "pass@8": 0.9999950840388245, + "pass@16": 0.9999999962921804, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9978243670886076, + "pass@2": 0.9987377591448221, + "pass@4": 0.9995064679415073, + "pass@8": 0.999928806446341, + "pass@16": 0.9999988610780643, + "pass@32": 0.9999999999224412, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9920232732732732, + "pass@2": 0.9956769022959969, + "pass@4": 0.9981618989630802, + "pass@8": 0.9995654428829316, + "pass@16": 0.9999754906144646, + "pass@32": 0.9999999648002483, + "pass@64": 0.9999999999999997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9916097266881029, + "pass@2": 0.9939979618705219, + "pass@4": 0.9958385552489217, + "pass@8": 0.9975977940296384, + "pass@16": 0.9990483367272565, + "pass@32": 0.999836024498138, + "pass@64": 0.9999978527259372, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9518105158730159, + "pass@2": 0.9663874046994128, + "pass@4": 0.974960809660697, + "pass@8": 0.9805869955097556, + "pass@16": 0.9851927553230163, + "pass@32": 0.98910883729019, + "pass@64": 0.9917724625682862, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8984852828746177, + "pass@2": 0.9240643586409498, + "pass@4": 0.941068672608584, + "pass@8": 0.9547484308868067, + "pass@16": 0.964354080653309, + "pass@32": 0.9705027070109506, + "pass@64": 0.9749939794561082, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.7964891975308642, + "pass@2": 0.8458115370127349, + "pass@4": 0.8753133372795989, + "pass@8": 0.8949344772446929, + "pass@16": 0.9116664506949259, + "pass@32": 0.9266813162011269, + "pass@64": 0.9371907688348377, + "pass@128": 0.941358024691358 + }, + "14": { + "pass@1": 0.8233725890207715, + "pass@2": 0.864813941330405, + "pass@4": 0.8884574113881165, + "pass@8": 0.9057512078427568, + "pass@16": 0.9210475746514785, + "pass@32": 0.9350615129237696, + "pass@64": 0.9456666949723773, + "pass@128": 0.9525222551928784 + }, + "15": { + "pass@1": 0.7960069444444444, + "pass@2": 0.8479114264362789, + "pass@4": 0.8798735259365724, + "pass@8": 0.9012219064707329, + "pass@16": 0.9154730765108868, + "pass@32": 0.9257078030331487, + "pass@64": 0.9355473867528037, + "pass@128": 0.9475308641975309 + }, + "16": { + "pass@1": 0.6688790560471977, + "pass@2": 0.761060132278819, + "pass@4": 0.8178771328915742, + "pass@8": 0.8558947560594331, + "pass@16": 0.8849558857513954, + "pass@32": 0.9077941329129988, + "pass@64": 0.9250881004375349, + "pass@128": 0.9410029498525073 + }, + "17": { + "pass@1": 0.5774896460843374, + "pass@2": 0.6752631836400717, + "pass@4": 0.7429351037445621, + "pass@8": 0.7866756488041113, + "pass@16": 0.8174313143142055, + "pass@32": 0.8439984872357021, + "pass@64": 0.8656491607233635, + "pass@128": 0.8795180722891566 + }, + "18": { + "pass@1": 0.570866935483871, + "pass@2": 0.6655289560579124, + "pass@4": 0.7339587947675894, + "pass@8": 0.7841472664969731, + "pass@16": 0.8203472667980849, + "pass@32": 0.8469237045905657, + "pass@64": 0.8695213616035634, + "pass@128": 0.8903225806451613 + }, + "19": { + "pass@1": 0.3884063444108761, + "pass@2": 0.5001635463996004, + "pass@4": 0.5980781865068977, + "pass@8": 0.6776722630837357, + "pass@16": 0.7378864000697488, + "pass@32": 0.782019320918216, + "pass@64": 0.81778520074431, + "pass@128": 0.851963746223565 + }, + "20": { + "pass@1": 0.4043731231231231, + "pass@2": 0.5181975774987587, + "pass@4": 0.6152067675099175, + "pass@8": 0.6916543914883966, + "pass@16": 0.7473698050927324, + "pass@32": 0.7847132791823201, + "pass@64": 0.8109043643297321, + "pass@128": 0.8288288288288288 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_metrics.json b/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..1bdf35d9592bf325daac5dbc43cb2c542d9fe90a --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch4-op7-10", + "id": { + "answer_accuracy": 0.994, + "avg_response_len": 200.33833854166667, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.994, + "pass@2": 0.9956650672572176, + "pass@4": 0.9968476424405281, + "pass@8": 0.9977425902955549, + "pass@16": 0.998360031800675, + "pass@32": 0.99881997872285, + "pass@64": 0.9992241214506254, + "pass@128": 0.9995555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9576875, + "pass@2": 0.9681493602362207, + "pass@4": 0.9764409844394448, + "pass@8": 0.9830904285785195, + "pass@16": 0.987603875673519, + "pass@32": 0.9908814180722857, + "pass@64": 0.9936987618034088, + "pass@128": 0.996 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999296875, + "pass@2": 0.9999955708661418, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99984375, + "pass@2": 0.9999933562992126, + "pass@4": 0.9999999802212223, + "pass@8": 0.9999999999999685, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990546875, + "pass@2": 0.9994500492125985, + "pass@4": 0.9997259424446944, + "pass@8": 0.9999287673442558, + "pass@16": 0.9999959202789485, + "pass@32": 0.9999999941392413, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999765625, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956171875, + "pass@2": 0.9977913385826772, + "pass@4": 0.9987899146044246, + "pass@8": 0.9990465879847814, + "pass@16": 0.9991248829432814, + "pass@32": 0.9992499999981759, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.993890625, + "pass@2": 0.9956060531496063, + "pass@4": 0.9966719720659917, + "pass@8": 0.9976175287524772, + "pass@16": 0.9985156073103248, + "pass@32": 0.9992483962959446, + "pass@64": 0.9998183312522196, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9576875, + "2": 1.0, + "3": 1.0, + "4": 0.9999296875, + "5": 0.99984375, + "6": 0.9990546875, + "7": 0.9999765625, + "8": 0.9956171875, + "9": 0.993890625 + }, + "per_op_avg_response_len": { + "10": 281.5152890625, + "2": 115.786484375, + "3": 144.948609375, + "4": 160.1623515625, + "5": 173.081, + "6": 199.3235390625, + "7": 224.2991015625, + "8": 244.00521875, + "9": 259.923453125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.5152890625, + "2": 115.786484375, + "3": 144.948609375, + "4": 160.1623515625, + "5": 173.081, + "6": 199.3235390625, + "7": 224.2991015625, + "8": 244.00521875, + "9": 259.923453125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376870, + "answer_accuracy": 0.9943589581222559, + "avg_response_len": 190.8690449805809, + "resp_tokens_sum": 72340895, + "pass_at_k": { + "pass@1": 0.9943589581222559, + "pass@2": 0.996177755041258, + "pass@4": 0.9971794507196228, + "pass@8": 0.9976981022527546, + "pass@16": 0.9981050816883029, + "pass@32": 0.9986103513984367, + "pass@64": 0.9991686752593077, + "pass@128": 0.999662276258021 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9625737463126843, + "pass@2": 0.9730252537569976, + "pass@4": 0.9790491030988384, + "pass@8": 0.9823822861395556, + "pass@16": 0.9851782276001185, + "pass@32": 0.9887806819385732, + "pass@64": 0.9929144734452152, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999503968253968, + "pass@2": 0.999999609423822, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9978246631736527, + "pass@2": 0.9984241654486302, + "pass@4": 0.9991805660332878, + "pass@8": 0.9997867287338961, + "pass@16": 0.9999877852663126, + "pass@32": 0.9999999824528186, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995292467948718, + "pass@2": 0.9984723589238845, + "pass@4": 0.9997657519372578, + "pass@8": 0.9999935417446474, + "pass@16": 0.9999999969902831, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9949603873239437, + "pass@2": 0.99670414217589, + "pass@4": 0.9974577013260665, + "pass@8": 0.997830234782101, + "pass@16": 0.9983600236132614, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 388220, + "answer_accuracy": 0.9937643348623854, + "avg_response_len": 198.363544909076, + "resp_tokens_sum": 77491909, + "pass_at_k": { + "pass@1": 0.9937643348623854, + "pass@2": 0.9953304467962144, + "pass@4": 0.9965529115135181, + "pass@8": 0.9976747403183903, + "pass@16": 0.99852526799435, + "pass@32": 0.9991184176716927, + "pass@64": 0.999592243195273, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9544190950920245, + "pass@2": 0.9655979783585334, + "pass@4": 0.9755531901495438, + "pass@8": 0.9849001130563421, + "pass@16": 0.9914179568700926, + "pass@32": 0.9953896244003085, + "pass@64": 0.9980909065259532, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999094202898551, + "pass@2": 0.9999992867739359, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998433022922636, + "pass@2": 0.9999925969586897, + "pass@4": 0.9999999905993125, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957086267605634, + "pass@2": 0.9968146972385495, + "pass@4": 0.9972236687843596, + "pass@8": 0.997358535350474, + "pass@16": 0.9975352112222411, + "pass@32": 0.997887323943662, + "pass@64": 0.9985915492957746, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925383607784432, + "pass@2": 0.994316239568108, + "pass@4": 0.9953136485746168, + "pass@8": 0.9962981752872685, + "pass@16": 0.9975205508838629, + "pass@32": 0.99868979993864, + "pass@64": 0.9996344033069231, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379998, + "answer_accuracy": 0.9938849598259123, + "avg_response_len": 211.74297476565116, + "resp_tokens_sum": 80956962, + "pass_at_k": { + "pass@1": 0.9938849598259123, + "pass@2": 0.9954987442302473, + "pass@4": 0.9968198669048192, + "pass@8": 0.9978560175553608, + "pass@16": 0.9984439308363754, + "pass@32": 0.9987228481020659, + "pass@64": 0.998902952588164, + "pass@128": 0.9989956478071644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9559235074626866, + "pass@2": 0.9656980843812436, + "pass@4": 0.9746656671647387, + "pass@8": 0.9820459603607242, + "pass@16": 0.9863468733654595, + "pass@32": 0.9886201472854002, + "pass@64": 0.9902182679940894, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996783088235294, + "pass@2": 0.9999815452755905, + "pass@4": 0.9999999418271246, + "pass@8": 0.9999999999999074, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999482452681388, + "pass@2": 0.9999336328522814, + "pass@4": 0.9999988538498933, + "pass@8": 0.9999999998332318, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995823948948949, + "pass@2": 0.9981944296658863, + "pass@4": 0.9995453380827396, + "pass@8": 0.9999589276607602, + "pass@16": 0.9999996513465985, + "pass@32": 0.9999999999945224, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941217845659164, + "pass@2": 0.9957378092006989, + "pass@4": 0.997233856820952, + "pass@8": 0.998791655494803, + "pass@16": 0.9997618486572567, + "pass@32": 0.9999916332542949, + "pass@64": 0.9999999987229116, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.48264140625, + "avg_response_len": 336.96297421875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.48264140625, + "pass@2": 0.54950445374015, + "pass@4": 0.6049536407011535, + "pass@8": 0.6506849035948025, + "pass@16": 0.6886568937359366, + "pass@32": 0.7208077039860731, + "pass@64": 0.749042282743256, + "pass@128": 0.7745 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9440390625, + "pass@2": 0.9602047244094495, + "pass@4": 0.9703773471128609, + "pass@8": 0.9773500577446682, + "pass@16": 0.9816367907916462, + "pass@32": 0.9844934582399582, + "pass@64": 0.9866168302420555, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.8583203125, + "pass@2": 0.896563607283465, + "pass@4": 0.917817725534308, + "pass@8": 0.9320315514858477, + "pass@16": 0.9425378082351825, + "pass@32": 0.9506772811040405, + "pass@64": 0.9573200207648929, + "pass@128": 0.963 + }, + "13": { + "pass@1": 0.7090390625, + "pass@2": 0.7841931594488197, + "pass@4": 0.8329807440007498, + "pass@8": 0.8652811245279454, + "pass@16": 0.8880327956833413, + "pass@32": 0.9057864037962902, + "pass@64": 0.9204816066732862, + "pass@128": 0.933 + }, + "14": { + "pass@1": 0.6603203125, + "pass@2": 0.7414966781496067, + "pass@4": 0.7985705101237346, + "pass@8": 0.8371451068534077, + "pass@16": 0.8642135734222964, + "pass@32": 0.8856339761559466, + "pass@64": 0.9033478894497897, + "pass@128": 0.917 + }, + "15": { + "pass@1": 0.5500703125, + "pass@2": 0.6516924212598431, + "pass@4": 0.7275813644544425, + "pass@8": 0.7824487694055557, + "pass@16": 0.8224994011911744, + "pass@32": 0.8523507828527137, + "pass@64": 0.8752890364832124, + "pass@128": 0.895 + }, + "16": { + "pass@1": 0.308140625, + "pass@2": 0.3923243110236223, + "pass@4": 0.4673418410198722, + "pass@8": 0.5332682864207966, + "pass@16": 0.5906135151978855, + "pass@32": 0.6396948092170933, + "pass@64": 0.6836045161679587, + "pass@128": 0.724 + }, + "17": { + "pass@1": 0.256015625, + "pass@2": 0.33540366633858254, + "pass@4": 0.409378450881139, + "pass@8": 0.47532668396617, + "pass@16": 0.5329195441450473, + "pass@32": 0.5831595117198585, + "pass@64": 0.6292019186086034, + "pass@128": 0.673 + }, + "18": { + "pass@1": 0.21353125, + "pass@2": 0.28199815452755933, + "pass@4": 0.34949067107236614, + "pass@8": 0.41318767710351856, + "pass@16": 0.47225910889314054, + "pass@32": 0.5252039399741631, + "pass@64": 0.5728921642215775, + "pass@128": 0.619 + }, + "19": { + "pass@1": 0.1728046875, + "pass@2": 0.23523597440944896, + "pass@4": 0.29625588732658387, + "pass@8": 0.3515368692538589, + "pass@16": 0.40054989229306903, + "pass@32": 0.4450398241793384, + "pass@64": 0.48646895299928694, + "pass@128": 0.524 + }, + "20": { + "pass@1": 0.1541328125, + "pass@2": 0.21593184055118106, + "pass@4": 0.2797418654855639, + "pass@8": 0.33927290918628406, + "pass@16": 0.3913065075064811, + "pass@32": 0.43603705262132386, + "pass@64": 0.4751998918218427, + "pass@128": 0.509 + } + }, + "per_op_accuracy": { + "11": 0.9440390625, + "12": 0.8583203125, + "13": 0.7090390625, + "14": 0.6603203125, + "15": 0.5500703125, + "16": 0.308140625, + "17": 0.256015625, + "18": 0.21353125, + "19": 0.1728046875, + "20": 0.1541328125 + }, + "per_op_avg_response_len": { + "11": 299.2021875, + "12": 307.15928125, + "13": 322.1980546875, + "14": 345.4662265625, + "15": 356.8333359375, + "16": 347.8671484375, + "17": 352.1329296875, + "18": 353.96396875, + "19": 339.32628125, + "20": 345.480328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 299.2021875, + "12": 307.15928125, + "13": 322.1980546875, + "14": 345.4662265625, + "15": 356.8333359375, + "16": 347.8671484375, + "17": 352.1329296875, + "18": 353.96396875, + "19": 339.32628125, + "20": 345.480328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 210156, + "answer_accuracy": 0.4879179049034175, + "avg_response_len": 320.37236255572066, + "resp_tokens_sum": 137990784, + "pass_at_k": { + "pass@1": 0.4879179049034175, + "pass@2": 0.553034325970213, + "pass@4": 0.6074735046585026, + "pass@8": 0.6525358099481483, + "pass@16": 0.6896515612060651, + "pass@32": 0.7205012668036456, + "pass@64": 0.7478684166087778, + "pass@128": 0.7735512630014859 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9385844546742209, + "pass@2": 0.9551942294394504, + "pass@4": 0.967026604677248, + "pass@8": 0.9754982084288127, + "pass@16": 0.9808253815147637, + "pass@32": 0.9850623210750536, + "pass@64": 0.9884474194474718, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8624426200564972, + "pass@2": 0.9017520519151211, + "pass@4": 0.9232137743904894, + "pass@8": 0.9364155828181312, + "pass@16": 0.9446959074267556, + "pass@32": 0.9501805451185891, + "pass@64": 0.9558795802602567, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.734203565830721, + "pass@2": 0.801667443166391, + "pass@4": 0.8452445452547275, + "pass@8": 0.8750841253152706, + "pass@16": 0.8976730768256509, + "pass@32": 0.916616734947578, + "pass@64": 0.932356215696857, + "pass@128": 0.9404388714733543 + }, + "14": { + "pass@1": 0.6615313473520249, + "pass@2": 0.741312673240611, + "pass@4": 0.7987274321434115, + "pass@8": 0.8374862263892824, + "pass@16": 0.8636665344499674, + "pass@32": 0.883744442134681, + "pass@64": 0.8996549245285982, + "pass@128": 0.9127725856697819 + }, + "15": { + "pass@1": 0.5426752167630058, + "pass@2": 0.6423150829502521, + "pass@4": 0.7232531651571007, + "pass@8": 0.7860465523211205, + "pass@16": 0.8328430235526624, + "pass@32": 0.8664772705101282, + "pass@64": 0.8907010774062084, + "pass@128": 0.9104046242774566 + }, + "16": { + "pass@1": 0.3149193548387097, + "pass@2": 0.39902527305054625, + "pass@4": 0.4697059656857409, + "pass@8": 0.5302840910372171, + "pass@16": 0.5838085174066465, + "pass@32": 0.6282172083334884, + "pass@64": 0.6665010235625493, + "pass@128": 0.7032258064516129 + }, + "17": { + "pass@1": 0.25410946531791906, + "pass@2": 0.32993918119339133, + "pass@4": 0.40069865038546476, + "pass@8": 0.46448555649405504, + "pass@16": 0.5182130045168546, + "pass@32": 0.5623529995882736, + "pass@64": 0.6040674100873197, + "pass@128": 0.6473988439306358 + }, + "18": { + "pass@1": 0.20060267857142858, + "pass@2": 0.2679288526434197, + "pass@4": 0.3344746652204189, + "pass@8": 0.39710824121270083, + "pass@16": 0.4566949168331932, + "pass@32": 0.5120978334164232, + "pass@64": 0.5656579621163826, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.18132976726726727, + "pass@2": 0.2433429985103213, + "pass@4": 0.30306926977596277, + "pass@8": 0.3572313687285545, + "pass@16": 0.40418699091341775, + "pass@32": 0.44416817763081634, + "pass@64": 0.47951343317052403, + "pass@128": 0.5135135135135135 + }, + "20": { + "pass@1": 0.16441441441441443, + "pass@2": 0.22810002423683523, + "pass@4": 0.29267682502525005, + "pass@8": 0.35116514670673493, + "pass@16": 0.40103786340810277, + "pass@32": 0.4445894691442985, + "pass@64": 0.48528400560602686, + "pass@128": 0.5195195195195195 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 206093, + "answer_accuracy": 0.4787694209039548, + "avg_response_len": 332.3799481489741, + "resp_tokens_sum": 143077602, + "pass_at_k": { + "pass@1": 0.4787694209039548, + "pass@2": 0.5458204338669327, + "pass@4": 0.6009252818921734, + "pass@8": 0.6463440310362696, + "pass@16": 0.6849403853498071, + "pass@32": 0.7189012958637673, + "pass@64": 0.7489942281363414, + "pass@128": 0.7749033600951531 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9485363328313253, + "pass@2": 0.9614666836637891, + "pass@4": 0.9690401578191278, + "pass@8": 0.9745595660303529, + "pass@16": 0.9776023172016188, + "pass@32": 0.979402643004502, + "pass@64": 0.981192486457403, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8519298589341693, + "pass@2": 0.882412744057463, + "pass@4": 0.8987941407402442, + "pass@8": 0.9122516853738211, + "pass@16": 0.9248490185450918, + "pass@32": 0.9363708212156535, + "pass@64": 0.9462338759817461, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.6947435224089635, + "pass@2": 0.777705521184852, + "pass@4": 0.8304399998844679, + "pass@8": 0.8625365131047817, + "pass@16": 0.8834984921646472, + "pass@32": 0.8995238580003326, + "pass@64": 0.9129394155831055, + "pass@128": 0.927170868347339 + }, + "14": { + "pass@1": 0.6441429093567251, + "pass@2": 0.7267150302758209, + "pass@4": 0.7861907489085788, + "pass@8": 0.8273056582111475, + "pass@16": 0.8566846397040585, + "pass@32": 0.8804814970414178, + "pass@64": 0.9011153985940141, + "pass@128": 0.9181286549707602 + }, + "15": { + "pass@1": 0.5651751893939394, + "pass@2": 0.6739184413027917, + "pass@4": 0.7489947995136972, + "pass@8": 0.7974960489060056, + "pass@16": 0.8302977072796687, + "pass@32": 0.8556934809010615, + "pass@64": 0.876922393623797, + "pass@128": 0.8939393939393939 + }, + "16": { + "pass@1": 0.3212250712250712, + "pass@2": 0.3989094712519908, + "pass@4": 0.46772314010962307, + "pass@8": 0.5297939095149375, + "pass@16": 0.5862086467853515, + "pass@32": 0.6380627166781673, + "pass@64": 0.6854883475998351, + "pass@128": 0.7264957264957265 + }, + "17": { + "pass@1": 0.2698466614906832, + "pass@2": 0.35929246955543587, + "pass@4": 0.4414016571880376, + "pass@8": 0.5100927321509832, + "pass@16": 0.5673176419200605, + "pass@32": 0.6157075990502674, + "pass@64": 0.6567645873486623, + "pass@128": 0.6925465838509317 + }, + "18": { + "pass@1": 0.20048253676470587, + "pass@2": 0.2711053294349235, + "pass@4": 0.34233683252828667, + "pass@8": 0.4105063409952905, + "pass@16": 0.4747878989634802, + "pass@32": 0.5336599706135036, + "pass@64": 0.5865359016158412, + "pass@128": 0.6352941176470588 + }, + "19": { + "pass@1": 0.15308779761904762, + "pass@2": 0.20727072201912244, + "pass@4": 0.2626874246745942, + "pass@8": 0.3177729661617527, + "pass@16": 0.3739255500583787, + "pass@32": 0.42984186349128706, + "pass@64": 0.482475527616294, + "pass@128": 0.5238095238095238 + }, + "20": { + "pass@1": 0.14775916916167664, + "pass@2": 0.20598456138903293, + "pass@4": 0.26647546779332226, + "pass@8": 0.32486153243092897, + "pass@16": 0.37714961566942967, + "pass@32": 0.4223905208474426, + "pass@64": 0.46155466688637004, + "pass@128": 0.49700598802395207 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 201532, + "answer_accuracy": 0.48119460574572126, + "avg_response_len": 358.7356285337714, + "resp_tokens_sum": 150244221, + "pass_at_k": { + "pass@1": 0.48119460574572126, + "pass@2": 0.5496607308732678, + "pass@4": 0.6065025491541572, + "pass@8": 0.6532429887217562, + "pass@16": 0.6914538257852948, + "pass@32": 0.723082279668885, + "pass@64": 0.7502989047437116, + "pass@128": 0.7750611246943765 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9454117063492063, + "pass@2": 0.9644895950506187, + "pass@4": 0.9755416611614021, + "pass@8": 0.9823663944356187, + "pass@16": 0.9867982914476112, + "pass@32": 0.9892215283268877, + "pass@64": 0.9902824942007619, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8600917431192661, + "pass@2": 0.9047514237039184, + "pass@4": 0.930534307351489, + "pass@8": 0.9465814909296013, + "pass@16": 0.9574575537927447, + "pass@32": 0.9651714867408759, + "pass@64": 0.9696943208397095, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.7000144675925926, + "pass@2": 0.7741369568387282, + "pass@4": 0.8237057225485699, + "pass@8": 0.8586535597961961, + "pass@16": 0.8835374150406778, + "pass@32": 0.9020236050675741, + "pass@64": 0.9171006250087037, + "pass@128": 0.9320987654320988 + }, + "14": { + "pass@1": 0.6755841988130564, + "pass@2": 0.7566729070772676, + "pass@4": 0.8109844756052377, + "pass@8": 0.846805617430967, + "pass@16": 0.8723752791841834, + "pass@32": 0.8926627247553403, + "pass@64": 0.9091311345903748, + "pass@128": 0.9198813056379822 + }, + "15": { + "pass@1": 0.5425829475308642, + "pass@2": 0.639068879410907, + "pass@4": 0.7103934736745863, + "pass@8": 0.7632807597637843, + "pass@16": 0.803510714937235, + "pass@32": 0.8338604893791952, + "pass@64": 0.8571668944592924, + "pass@128": 0.8796296296296297 + }, + "16": { + "pass@1": 0.2883941740412979, + "pass@2": 0.379378324390867, + "pass@4": 0.4647851607044694, + "pass@8": 0.5395945603525556, + "pass@16": 0.6013971674931184, + "pass@32": 0.6518804161642361, + "pass@64": 0.6972943624071537, + "pass@128": 0.7404129793510325 + }, + "17": { + "pass@1": 0.24458772590361447, + "pass@2": 0.3179293206289724, + "pass@4": 0.3873656151603038, + "pass@8": 0.4529060893572603, + "pass@16": 0.5148842285661349, + "pass@32": 0.5732756836389475, + "pass@64": 0.6286638481690365, + "pass@128": 0.6807228915662651 + }, + "18": { + "pass@1": 0.24243951612903225, + "pass@2": 0.30982981965963907, + "pass@4": 0.37429037156645767, + "pass@8": 0.4342826991634681, + "pass@16": 0.4870580721094836, + "pass@32": 0.5307268008704006, + "pass@64": 0.5660957126821192, + "pass@128": 0.5967741935483871 + }, + "19": { + "pass@1": 0.1842428247734139, + "pass@2": 0.25546765349572986, + "pass@4": 0.3234768755302869, + "pass@8": 0.3800818937670737, + "pass@16": 0.42391733987699637, + "pass@32": 0.4613442746078676, + "pass@64": 0.49752024916745513, + "pass@128": 0.5347432024169184 + }, + "20": { + "pass@1": 0.150243993993994, + "pass@2": 0.2137408077368706, + "pass@4": 0.2801131426702293, + "pass@8": 0.3418353258288631, + "pass@16": 0.3957745566306108, + "pass@32": 0.44117214844812946, + "pass@64": 0.4788019796245884, + "pass@128": 0.5105105105105106 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7248638980263158, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7248638980263158, + "pass@2": 0.7608436917219054, + "pass@4": 0.7905876415250559, + "pass@8": 0.8150806499267463, + "pass@16": 0.8353583801876622, + "pass@32": 0.8524977288614013, + "pass@64": 0.8675494694993787, + "pass@128": 0.8811052631578947 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9576875, + "pass@2": 0.9681493602362207, + "pass@4": 0.9764409844394448, + "pass@8": 0.9830904285785195, + "pass@16": 0.987603875673519, + "pass@32": 0.9908814180722857, + "pass@64": 0.9936987618034088, + "pass@128": 0.996 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999296875, + "pass@2": 0.9999955708661418, + "pass@4": 0.9999999881889764, + "pass@8": 0.9999999999999937, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99984375, + "pass@2": 0.9999933562992126, + "pass@4": 0.9999999802212223, + "pass@8": 0.9999999999999685, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990546875, + "pass@2": 0.9994500492125985, + "pass@4": 0.9997259424446944, + "pass@8": 0.9999287673442558, + "pass@16": 0.9999959202789485, + "pass@32": 0.9999999941392413, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999765625, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9956171875, + "pass@2": 0.9977913385826772, + "pass@4": 0.9987899146044246, + "pass@8": 0.9990465879847814, + "pass@16": 0.9991248829432814, + "pass@32": 0.9992499999981759, + "pass@64": 0.9995, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.993890625, + "pass@2": 0.9956060531496063, + "pass@4": 0.9966719720659917, + "pass@8": 0.9976175287524772, + "pass@16": 0.9985156073103248, + "pass@32": 0.9992483962959446, + "pass@64": 0.9998183312522196, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9440390625, + "pass@2": 0.9602047244094495, + "pass@4": 0.9703773471128609, + "pass@8": 0.9773500577446682, + "pass@16": 0.9816367907916462, + "pass@32": 0.9844934582399582, + "pass@64": 0.9866168302420555, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.8583203125, + "pass@2": 0.896563607283465, + "pass@4": 0.917817725534308, + "pass@8": 0.9320315514858477, + "pass@16": 0.9425378082351825, + "pass@32": 0.9506772811040405, + "pass@64": 0.9573200207648929, + "pass@128": 0.963 + }, + "13": { + "pass@1": 0.7090390625, + "pass@2": 0.7841931594488197, + "pass@4": 0.8329807440007498, + "pass@8": 0.8652811245279454, + "pass@16": 0.8880327956833413, + "pass@32": 0.9057864037962902, + "pass@64": 0.9204816066732862, + "pass@128": 0.933 + }, + "14": { + "pass@1": 0.6603203125, + "pass@2": 0.7414966781496067, + "pass@4": 0.7985705101237346, + "pass@8": 0.8371451068534077, + "pass@16": 0.8642135734222964, + "pass@32": 0.8856339761559466, + "pass@64": 0.9033478894497897, + "pass@128": 0.917 + }, + "15": { + "pass@1": 0.5500703125, + "pass@2": 0.6516924212598431, + "pass@4": 0.7275813644544425, + "pass@8": 0.7824487694055557, + "pass@16": 0.8224994011911744, + "pass@32": 0.8523507828527137, + "pass@64": 0.8752890364832124, + "pass@128": 0.895 + }, + "16": { + "pass@1": 0.308140625, + "pass@2": 0.3923243110236223, + "pass@4": 0.4673418410198722, + "pass@8": 0.5332682864207966, + "pass@16": 0.5906135151978855, + "pass@32": 0.6396948092170933, + "pass@64": 0.6836045161679587, + "pass@128": 0.724 + }, + "17": { + "pass@1": 0.256015625, + "pass@2": 0.33540366633858254, + "pass@4": 0.409378450881139, + "pass@8": 0.47532668396617, + "pass@16": 0.5329195441450473, + "pass@32": 0.5831595117198585, + "pass@64": 0.6292019186086034, + "pass@128": 0.673 + }, + "18": { + "pass@1": 0.21353125, + "pass@2": 0.28199815452755933, + "pass@4": 0.34949067107236614, + "pass@8": 0.41318767710351856, + "pass@16": 0.47225910889314054, + "pass@32": 0.5252039399741631, + "pass@64": 0.5728921642215775, + "pass@128": 0.619 + }, + "19": { + "pass@1": 0.1728046875, + "pass@2": 0.23523597440944896, + "pass@4": 0.29625588732658387, + "pass@8": 0.3515368692538589, + "pass@16": 0.40054989229306903, + "pass@32": 0.4450398241793384, + "pass@64": 0.48646895299928694, + "pass@128": 0.524 + }, + "20": { + "pass@1": 0.1541328125, + "pass@2": 0.21593184055118106, + "pass@4": 0.2797418654855639, + "pass@8": 0.33927290918628406, + "pass@16": 0.3913065075064811, + "pass@32": 0.43603705262132386, + "pass@64": 0.4751998918218427, + "pass@128": 0.509 + } + }, + "per_op_accuracy": { + "10": 0.9576875, + "2": 1.0, + "3": 1.0, + "4": 0.9999296875, + "5": 0.99984375, + "6": 0.9990546875, + "7": 0.9999765625, + "8": 0.9956171875, + "9": 0.993890625, + "11": 0.9440390625, + "12": 0.8583203125, + "13": 0.7090390625, + "14": 0.6603203125, + "15": 0.5500703125, + "16": 0.308140625, + "17": 0.256015625, + "18": 0.21353125, + "19": 0.1728046875, + "20": 0.1541328125 + }, + "per_op_avg_response_len": { + "10": 281.5152890625, + "2": 115.786484375, + "3": 144.948609375, + "4": 160.1623515625, + "5": 173.081, + "6": 199.3235390625, + "7": 224.2991015625, + "8": 244.00521875, + "9": 259.923453125, + "11": 299.2021875, + "12": 307.15928125, + "13": 322.1980546875, + "14": 345.4662265625, + "15": 356.8333359375, + "16": 347.8671484375, + "17": 352.1329296875, + "18": 353.96396875, + "19": 339.32628125, + "20": 345.480328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 281.5152890625, + "2": 115.786484375, + "3": 144.948609375, + "4": 160.1623515625, + "5": 173.081, + "6": 199.3235390625, + "7": 224.2991015625, + "8": 244.00521875, + "9": 259.923453125, + "11": 299.2021875, + "12": 307.15928125, + "13": 322.1980546875, + "14": 345.4662265625, + "15": 356.8333359375, + "16": 347.8671484375, + "17": 352.1329296875, + "18": 353.96396875, + "19": 339.32628125, + "20": 345.480328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 587026, + "answer_accuracy": 0.7249669024660133, + "avg_response_len": 259.7559662009959, + "resp_tokens_sum": 210331679, + "pass_at_k": { + "pass@1": 0.7249669024660133, + "pass@2": 0.7604557128622972, + "pass@4": 0.789882500277682, + "pass@8": 0.8140953337410525, + "pass@16": 0.8340288729588218, + "pass@32": 0.85067531035173, + "pass@64": 0.8654940988509915, + "pass@128": 0.879386658235852 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9625737463126843, + "pass@2": 0.9730252537569976, + "pass@4": 0.9790491030988384, + "pass@8": 0.9823822861395556, + "pass@16": 0.9851782276001185, + "pass@32": 0.9887806819385732, + "pass@64": 0.9929144734452152, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999503968253968, + "pass@2": 0.999999609423822, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9978246631736527, + "pass@2": 0.9984241654486302, + "pass@4": 0.9991805660332878, + "pass@8": 0.9997867287338961, + "pass@16": 0.9999877852663126, + "pass@32": 0.9999999824528186, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995292467948718, + "pass@2": 0.9984723589238845, + "pass@4": 0.9997657519372578, + "pass@8": 0.9999935417446474, + "pass@16": 0.9999999969902831, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9949603873239437, + "pass@2": 0.99670414217589, + "pass@4": 0.9974577013260665, + "pass@8": 0.997830234782101, + "pass@16": 0.9983600236132614, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9385844546742209, + "pass@2": 0.9551942294394504, + "pass@4": 0.967026604677248, + "pass@8": 0.9754982084288127, + "pass@16": 0.9808253815147637, + "pass@32": 0.9850623210750536, + "pass@64": 0.9884474194474718, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8624426200564972, + "pass@2": 0.9017520519151211, + "pass@4": 0.9232137743904894, + "pass@8": 0.9364155828181312, + "pass@16": 0.9446959074267556, + "pass@32": 0.9501805451185891, + "pass@64": 0.9558795802602567, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.734203565830721, + "pass@2": 0.801667443166391, + "pass@4": 0.8452445452547275, + "pass@8": 0.8750841253152706, + "pass@16": 0.8976730768256509, + "pass@32": 0.916616734947578, + "pass@64": 0.932356215696857, + "pass@128": 0.9404388714733543 + }, + "14": { + "pass@1": 0.6615313473520249, + "pass@2": 0.741312673240611, + "pass@4": 0.7987274321434115, + "pass@8": 0.8374862263892824, + "pass@16": 0.8636665344499674, + "pass@32": 0.883744442134681, + "pass@64": 0.8996549245285982, + "pass@128": 0.9127725856697819 + }, + "15": { + "pass@1": 0.5426752167630058, + "pass@2": 0.6423150829502521, + "pass@4": 0.7232531651571007, + "pass@8": 0.7860465523211205, + "pass@16": 0.8328430235526624, + "pass@32": 0.8664772705101282, + "pass@64": 0.8907010774062084, + "pass@128": 0.9104046242774566 + }, + "16": { + "pass@1": 0.3149193548387097, + "pass@2": 0.39902527305054625, + "pass@4": 0.4697059656857409, + "pass@8": 0.5302840910372171, + "pass@16": 0.5838085174066465, + "pass@32": 0.6282172083334884, + "pass@64": 0.6665010235625493, + "pass@128": 0.7032258064516129 + }, + "17": { + "pass@1": 0.25410946531791906, + "pass@2": 0.32993918119339133, + "pass@4": 0.40069865038546476, + "pass@8": 0.46448555649405504, + "pass@16": 0.5182130045168546, + "pass@32": 0.5623529995882736, + "pass@64": 0.6040674100873197, + "pass@128": 0.6473988439306358 + }, + "18": { + "pass@1": 0.20060267857142858, + "pass@2": 0.2679288526434197, + "pass@4": 0.3344746652204189, + "pass@8": 0.39710824121270083, + "pass@16": 0.4566949168331932, + "pass@32": 0.5120978334164232, + "pass@64": 0.5656579621163826, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.18132976726726727, + "pass@2": 0.2433429985103213, + "pass@4": 0.30306926977596277, + "pass@8": 0.3572313687285545, + "pass@16": 0.40418699091341775, + "pass@32": 0.44416817763081634, + "pass@64": 0.47951343317052403, + "pass@128": 0.5135135135135135 + }, + "20": { + "pass@1": 0.16441441441441443, + "pass@2": 0.22810002423683523, + "pass@4": 0.29267682502525005, + "pass@8": 0.35116514670673493, + "pass@16": 0.40103786340810277, + "pass@32": 0.4445894691442985, + "pass@64": 0.48528400560602686, + "pass@128": 0.5195195195195195 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 594313, + "answer_accuracy": 0.7237833690568979, + "avg_response_len": 268.6203125, + "resp_tokens_sum": 220569511, + "pass_at_k": { + "pass@1": 0.7237833690568979, + "pass@2": 0.7596792895894745, + "pass@4": 0.7891490582919107, + "pass@8": 0.8134931073775019, + "pass@16": 0.8341315095635516, + "pass@32": 0.8522173762624881, + "pass@64": 0.8682187241550315, + "pass@128": 0.8819953234606391 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9544190950920245, + "pass@2": 0.9655979783585334, + "pass@4": 0.9755531901495438, + "pass@8": 0.9849001130563421, + "pass@16": 0.9914179568700926, + "pass@32": 0.9953896244003085, + "pass@64": 0.9980909065259532, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999094202898551, + "pass@2": 0.9999992867739359, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998433022922636, + "pass@2": 0.9999925969586897, + "pass@4": 0.9999999905993125, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957086267605634, + "pass@2": 0.9968146972385495, + "pass@4": 0.9972236687843596, + "pass@8": 0.997358535350474, + "pass@16": 0.9975352112222411, + "pass@32": 0.997887323943662, + "pass@64": 0.9985915492957746, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925383607784432, + "pass@2": 0.994316239568108, + "pass@4": 0.9953136485746168, + "pass@8": 0.9962981752872685, + "pass@16": 0.9975205508838629, + "pass@32": 0.99868979993864, + "pass@64": 0.9996344033069231, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9485363328313253, + "pass@2": 0.9614666836637891, + "pass@4": 0.9690401578191278, + "pass@8": 0.9745595660303529, + "pass@16": 0.9776023172016188, + "pass@32": 0.979402643004502, + "pass@64": 0.981192486457403, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8519298589341693, + "pass@2": 0.882412744057463, + "pass@4": 0.8987941407402442, + "pass@8": 0.9122516853738211, + "pass@16": 0.9248490185450918, + "pass@32": 0.9363708212156535, + "pass@64": 0.9462338759817461, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.6947435224089635, + "pass@2": 0.777705521184852, + "pass@4": 0.8304399998844679, + "pass@8": 0.8625365131047817, + "pass@16": 0.8834984921646472, + "pass@32": 0.8995238580003326, + "pass@64": 0.9129394155831055, + "pass@128": 0.927170868347339 + }, + "14": { + "pass@1": 0.6441429093567251, + "pass@2": 0.7267150302758209, + "pass@4": 0.7861907489085788, + "pass@8": 0.8273056582111475, + "pass@16": 0.8566846397040585, + "pass@32": 0.8804814970414178, + "pass@64": 0.9011153985940141, + "pass@128": 0.9181286549707602 + }, + "15": { + "pass@1": 0.5651751893939394, + "pass@2": 0.6739184413027917, + "pass@4": 0.7489947995136972, + "pass@8": 0.7974960489060056, + "pass@16": 0.8302977072796687, + "pass@32": 0.8556934809010615, + "pass@64": 0.876922393623797, + "pass@128": 0.8939393939393939 + }, + "16": { + "pass@1": 0.3212250712250712, + "pass@2": 0.3989094712519908, + "pass@4": 0.46772314010962307, + "pass@8": 0.5297939095149375, + "pass@16": 0.5862086467853515, + "pass@32": 0.6380627166781673, + "pass@64": 0.6854883475998351, + "pass@128": 0.7264957264957265 + }, + "17": { + "pass@1": 0.2698466614906832, + "pass@2": 0.35929246955543587, + "pass@4": 0.4414016571880376, + "pass@8": 0.5100927321509832, + "pass@16": 0.5673176419200605, + "pass@32": 0.6157075990502674, + "pass@64": 0.6567645873486623, + "pass@128": 0.6925465838509317 + }, + "18": { + "pass@1": 0.20048253676470587, + "pass@2": 0.2711053294349235, + "pass@4": 0.34233683252828667, + "pass@8": 0.4105063409952905, + "pass@16": 0.4747878989634802, + "pass@32": 0.5336599706135036, + "pass@64": 0.5865359016158412, + "pass@128": 0.6352941176470588 + }, + "19": { + "pass@1": 0.15308779761904762, + "pass@2": 0.20727072201912244, + "pass@4": 0.2626874246745942, + "pass@8": 0.3177729661617527, + "pass@16": 0.3739255500583787, + "pass@32": 0.42984186349128706, + "pass@64": 0.482475527616294, + "pass@128": 0.5238095238095238 + }, + "20": { + "pass@1": 0.14775916916167664, + "pass@2": 0.20598456138903293, + "pass@4": 0.26647546779332226, + "pass@8": 0.32486153243092897, + "pass@16": 0.37714961566942967, + "pass@32": 0.4223905208474426, + "pass@64": 0.46155466688637004, + "pass@128": 0.49700598802395207 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 581530, + "answer_accuracy": 0.725867251158332, + "avg_response_len": 288.5859150323534, + "resp_tokens_sum": 231201183, + "pass_at_k": { + "pass@1": 0.725867251158332, + "pass@2": 0.7624292475528114, + "pass@4": 0.792774769656026, + "pass@8": 0.8177036241468999, + "pass@16": 0.8379595685217712, + "pass@32": 0.8546269957433247, + "pass@64": 0.8689409068065663, + "pass@128": 0.8819300207700911 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9559235074626866, + "pass@2": 0.9656980843812436, + "pass@4": 0.9746656671647387, + "pass@8": 0.9820459603607242, + "pass@16": 0.9863468733654595, + "pass@32": 0.9886201472854002, + "pass@64": 0.9902182679940894, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996783088235294, + "pass@2": 0.9999815452755905, + "pass@4": 0.9999999418271246, + "pass@8": 0.9999999999999074, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999482452681388, + "pass@2": 0.9999336328522814, + "pass@4": 0.9999988538498933, + "pass@8": 0.9999999998332318, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995823948948949, + "pass@2": 0.9981944296658863, + "pass@4": 0.9995453380827396, + "pass@8": 0.9999589276607602, + "pass@16": 0.9999996513465985, + "pass@32": 0.9999999999945224, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941217845659164, + "pass@2": 0.9957378092006989, + "pass@4": 0.997233856820952, + "pass@8": 0.998791655494803, + "pass@16": 0.9997618486572567, + "pass@32": 0.9999916332542949, + "pass@64": 0.9999999987229116, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9454117063492063, + "pass@2": 0.9644895950506187, + "pass@4": 0.9755416611614021, + "pass@8": 0.9823663944356187, + "pass@16": 0.9867982914476112, + "pass@32": 0.9892215283268877, + "pass@64": 0.9902824942007619, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8600917431192661, + "pass@2": 0.9047514237039184, + "pass@4": 0.930534307351489, + "pass@8": 0.9465814909296013, + "pass@16": 0.9574575537927447, + "pass@32": 0.9651714867408759, + "pass@64": 0.9696943208397095, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.7000144675925926, + "pass@2": 0.7741369568387282, + "pass@4": 0.8237057225485699, + "pass@8": 0.8586535597961961, + "pass@16": 0.8835374150406778, + "pass@32": 0.9020236050675741, + "pass@64": 0.9171006250087037, + "pass@128": 0.9320987654320988 + }, + "14": { + "pass@1": 0.6755841988130564, + "pass@2": 0.7566729070772676, + "pass@4": 0.8109844756052377, + "pass@8": 0.846805617430967, + "pass@16": 0.8723752791841834, + "pass@32": 0.8926627247553403, + "pass@64": 0.9091311345903748, + "pass@128": 0.9198813056379822 + }, + "15": { + "pass@1": 0.5425829475308642, + "pass@2": 0.639068879410907, + "pass@4": 0.7103934736745863, + "pass@8": 0.7632807597637843, + "pass@16": 0.803510714937235, + "pass@32": 0.8338604893791952, + "pass@64": 0.8571668944592924, + "pass@128": 0.8796296296296297 + }, + "16": { + "pass@1": 0.2883941740412979, + "pass@2": 0.379378324390867, + "pass@4": 0.4647851607044694, + "pass@8": 0.5395945603525556, + "pass@16": 0.6013971674931184, + "pass@32": 0.6518804161642361, + "pass@64": 0.6972943624071537, + "pass@128": 0.7404129793510325 + }, + "17": { + "pass@1": 0.24458772590361447, + "pass@2": 0.3179293206289724, + "pass@4": 0.3873656151603038, + "pass@8": 0.4529060893572603, + "pass@16": 0.5148842285661349, + "pass@32": 0.5732756836389475, + "pass@64": 0.6286638481690365, + "pass@128": 0.6807228915662651 + }, + "18": { + "pass@1": 0.24243951612903225, + "pass@2": 0.30982981965963907, + "pass@4": 0.37429037156645767, + "pass@8": 0.4342826991634681, + "pass@16": 0.4870580721094836, + "pass@32": 0.5307268008704006, + "pass@64": 0.5660957126821192, + "pass@128": 0.5967741935483871 + }, + "19": { + "pass@1": 0.1842428247734139, + "pass@2": 0.25546765349572986, + "pass@4": 0.3234768755302869, + "pass@8": 0.3800818937670737, + "pass@16": 0.42391733987699637, + "pass@32": 0.4613442746078676, + "pass@64": 0.49752024916745513, + "pass@128": 0.5347432024169184 + }, + "20": { + "pass@1": 0.150243993993994, + "pass@2": 0.2137408077368706, + "pass@4": 0.2801131426702293, + "pass@8": 0.3418353258288631, + "pass@16": 0.3957745566306108, + "pass@32": 0.44117214844812946, + "pass@64": 0.4788019796245884, + "pass@128": 0.5105105105105106 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_score_report.jsonl b/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..879046eba6784dba8b9caa4424d9ded593237f3c --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_score_report.jsonl @@ -0,0 +1,4 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch4-op7-10", "results_dir": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1023161, "value_mismatch": 7872, "dependency_mismatch": 126907, "answer_mismatch": 6912, "extra_nodes": 9234, "missing_nodes": 9141, "perfect_with_extra": 416, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 336915, "value_mismatch": 2481, "answer_mismatch": 2138, "dependency_mismatch": 41457, "missing_nodes": 3215, "extra_nodes": 3125, "perfect_with_extra": 132}, "teachers_in_school": {"total": 390656, "perfect": 346652, "value_mismatch": 2672, "dependency_mismatch": 43324, "answer_mismatch": 2436, "missing_nodes": 3294, "extra_nodes": 3337, "perfect_with_extra": 125}, "movie_festival_awards": {"total": 382336, "perfect": 339594, "dependency_mismatch": 42126, "value_mismatch": 2719, "answer_mismatch": 2338, "missing_nodes": 2632, "extra_nodes": 2772, "perfect_with_extra": 159}}, "results_path": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 284078, "value_mismatch": 753537, "dependency_mismatch": 906950, "answer_mismatch": 662224, "extra_nodes": 49755, "missing_nodes": 620645, "perfect_with_extra": 22, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 252394, "dependency_mismatch": 306948, "answer_mismatch": 220562, "perfect": 96108, "missing_nodes": 214292, "extra_nodes": 19166, "perfect_with_extra": 12}, "movie_festival_awards": {"total": 418816, "perfect": 92488, "value_mismatch": 245898, "dependency_mismatch": 295297, "answer_mismatch": 217291, "missing_nodes": 197050, "extra_nodes": 16328, "perfect_with_extra": 5}, "teachers_in_school": {"total": 430464, "perfect": 95482, "dependency_mismatch": 304705, "value_mismatch": 255245, "answer_mismatch": 224371, "missing_nodes": 209303, "extra_nodes": 14261, "perfect_with_extra": 5}}, "results_path": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1307239, "value_mismatch": 761409, "dependency_mismatch": 1033857, "answer_mismatch": 669136, "extra_nodes": 58989, "missing_nodes": 629786, "perfect_with_extra": 438, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 433023, "value_mismatch": 254875, "answer_mismatch": 222700, "dependency_mismatch": 348405, "missing_nodes": 217507, "extra_nodes": 22291, "perfect_with_extra": 144}, "teachers_in_school": {"total": 821120, "perfect": 442134, "value_mismatch": 257917, "dependency_mismatch": 348029, "answer_mismatch": 226807, "missing_nodes": 212597, "extra_nodes": 17598, "perfect_with_extra": 130}, "movie_festival_awards": {"total": 801152, "perfect": 432082, "dependency_mismatch": 337423, "value_mismatch": 248617, "answer_mismatch": 219629, "missing_nodes": 199682, "extra_nodes": 19100, "perfect_with_extra": 164}}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch4-op7-10", "results_dir": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_score_report.jsonl", "template_stats": {}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch4-op7-10", "results_dir": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_score_report.jsonl", "template_stats": {}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch4-op7-10", "results_dir": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch4-op7-10/graphs/SDAR-100M-block8-epoch4-op7-10_score_report.jsonl", "template_stats": {}}} diff --git a/composition/SDAR-100M-block8-epoch4-op7-10/graphs/results.json b/composition/SDAR-100M-block8-epoch4-op7-10/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..91eda640138fa8c9cfea648c91f0166a8b37371e --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op7-10/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.99375, + "pass@4": 0.996875, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9619047619047619, + "pass@2": 0.9841269841269841, + "pass@4": 0.9936507936507937, + "pass@8": 0.9968253968253968, + "pass@16": 0.9968253968253968, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9610778443113772, + "pass@2": 0.9820359281437125, + "pass@4": 0.9910179640718563, + "pass@8": 0.9970059880239521, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.893491124260355, + "pass@2": 0.9201183431952663, + "pass@4": 0.9704142011834319, + "pass@8": 0.9792899408284024, + "pass@16": 0.9970414201183432, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7788461538461539, + "pass@2": 0.8493589743589743, + "pass@4": 0.9230769230769231, + "pass@8": 0.967948717948718, + "pass@16": 0.9935897435897436, + "pass@32": 0.9967948717948718, + "pass@64": 0.9967948717948718, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7605633802816901, + "pass@2": 0.8253521126760563, + "pass@4": 0.8929577464788733, + "pass@8": 0.9408450704225352, + "pass@16": 0.9802816901408451, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6430678466076696, + "pass@2": 0.7138643067846607, + "pass@4": 0.7876106194690266, + "pass@8": 0.8525073746312685, + "pass@16": 0.9174041297935103, + "pass@32": 0.9380530973451328, + "pass@64": 0.9557522123893806, + "pass@128": 0.9646017699115044 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9619047619047619, + "6": 0.9610778443113772, + "7": 0.893491124260355, + "8": 0.7788461538461539, + "9": 0.7605633802816901, + "10": 0.6430678466076696 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9841269841269841, + "6": 0.9820359281437125, + "7": 0.9201183431952663, + "8": 0.8493589743589743, + "9": 0.8253521126760563, + "10": 0.7138643067846607 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 0.9910179640718563, + "7": 0.9704142011834319, + "8": 0.9230769230769231, + "9": 0.8929577464788733, + "10": 0.7876106194690266 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 0.9970059880239521, + "7": 0.9792899408284024, + "8": 0.967948717948718, + "9": 0.9408450704225352, + "10": 0.8525073746312685 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9935897435897436, + "9": 0.9802816901408451, + "10": 0.9174041297935103 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 0.9971830985915493, + "10": 0.9380530973451328 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9557522123893806 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9646017699115044 + } + }, + "total_pass_at_k": { + "pass@1": 0.8858493752110773, + "pass@2": 0.916919959473151, + "pass@4": 0.9493414387031408, + "pass@8": 0.9696048632218845, + "pass@16": 0.9868287740628167, + "pass@32": 0.9922323539344816, + "pass@64": 0.994596420128335, + "pass@128": 0.9956095913542722 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9970760233918129, + "pass@2": 0.9970760233918129, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9735294117647059, + "pass@2": 0.9882352941176471, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9684542586750788, + "pass@2": 0.9779179810725552, + "pass@4": 0.9810725552050473, + "pass@8": 0.9905362776025236, + "pass@16": 0.9968454258675079, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8607594936708861, + "pass@2": 0.8987341772151899, + "pass@4": 0.9367088607594937, + "pass@8": 0.990506329113924, + "pass@16": 0.9968354430379747, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7957957957957958, + "pass@2": 0.8648648648648649, + "pass@4": 0.933933933933934, + "pass@8": 0.975975975975976, + "pass@16": 0.987987987987988, + "pass@32": 0.996996996996997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7459807073954984, + "pass@2": 0.8102893890675241, + "pass@4": 0.909967845659164, + "pass@8": 0.9517684887459807, + "pass@16": 0.9839228295819936, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6, + "pass@2": 0.6567164179104478, + "pass@4": 0.755223880597015, + "pass@8": 0.8537313432835821, + "pass@16": 0.9044776119402985, + "pass@32": 0.9313432835820895, + "pass@64": 0.9492537313432836, + "pass@128": 0.9671641791044776 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9735294117647059, + "6": 0.9684542586750788, + "7": 0.8607594936708861, + "8": 0.7957957957957958, + "9": 0.7459807073954984, + "10": 0.6 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9882352941176471, + "6": 0.9779179810725552, + "7": 0.8987341772151899, + "8": 0.8648648648648649, + "9": 0.8102893890675241, + "10": 0.6567164179104478 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9810725552050473, + "7": 0.9367088607594937, + "8": 0.933933933933934, + "9": 0.909967845659164, + "10": 0.755223880597015 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9905362776025236, + "7": 0.990506329113924, + "8": 0.975975975975976, + "9": 0.9517684887459807, + "10": 0.8537313432835821 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9968454258675079, + "7": 0.9968354430379747, + "8": 0.987987987987988, + "9": 0.9839228295819936, + "10": 0.9044776119402985 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9313432835820895 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9492537313432836 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9671641791044776 + } + }, + "total_pass_at_k": { + "pass@1": 0.8844994978239036, + "pass@2": 0.9119517910947439, + "pass@4": 0.9471041178439906, + "pass@8": 0.9738868429862738, + "pass@16": 0.9856042852360227, + "pass@32": 0.991965182457315, + "pass@64": 0.9943086709072648, + "pass@128": 0.9963173752929361 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9822485207100592, + "pass@2": 0.9911242603550295, + "pass@4": 0.9970414201183432, + "pass@8": 0.9970414201183432, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9536231884057971, + "pass@2": 0.9768115942028985, + "pass@4": 0.9971014492753624, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9684813753581661, + "pass@2": 0.9828080229226361, + "pass@4": 0.994269340974212, + "pass@8": 0.997134670487106, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8583815028901735, + "pass@2": 0.9075144508670521, + "pass@4": 0.9450867052023122, + "pass@8": 0.9855491329479769, + "pass@16": 0.9913294797687862, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8140845070422535, + "pass@2": 0.9014084507042254, + "pass@4": 0.9436619718309859, + "pass@8": 0.9802816901408451, + "pass@16": 0.9887323943661972, + "pass@32": 0.9915492957746479, + "pass@64": 0.9971830985915493, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7514970059880239, + "pass@2": 0.8083832335329342, + "pass@4": 0.8832335329341318, + "pass@8": 0.937125748502994, + "pass@16": 0.9760479041916168, + "pass@32": 0.9970059880239521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6349693251533742, + "pass@2": 0.7085889570552147, + "pass@4": 0.7883435582822086, + "pass@8": 0.8343558282208589, + "pass@16": 0.911042944785276, + "pass@32": 0.9294478527607362, + "pass@64": 0.9478527607361963, + "pass@128": 0.9754601226993865 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9822485207100592, + "5": 0.9536231884057971, + "6": 0.9684813753581661, + "7": 0.8583815028901735, + "8": 0.8140845070422535, + "9": 0.7514970059880239, + "10": 0.6349693251533742 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9911242603550295, + "5": 0.9768115942028985, + "6": 0.9828080229226361, + "7": 0.9075144508670521, + "8": 0.9014084507042254, + "9": 0.8083832335329342, + "10": 0.7085889570552147 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.9971014492753624, + "6": 0.994269340974212, + "7": 0.9450867052023122, + "8": 0.9436619718309859, + "9": 0.8832335329341318, + "10": 0.7883435582822086 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9855491329479769, + "8": 0.9802816901408451, + "9": 0.937125748502994, + "10": 0.8343558282208589 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9913294797687862, + "8": 0.9887323943661972, + "9": 0.9760479041916168, + "10": 0.911042944785276 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9915492957746479, + "9": 0.9970059880239521, + "10": 0.9294478527607362 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 1.0, + "10": 0.9478527607361963 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9754601226993865 + } + }, + "total_pass_at_k": { + "pass@1": 0.8853211009174312, + "pass@2": 0.9203800786369594, + "pass@4": 0.950524246395806, + "pass@8": 0.9708387942332897, + "pass@16": 0.9855832241153342, + "pass@32": 0.9911533420707732, + "pass@64": 0.9941022280471822, + "pass@128": 0.9973787680209698 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.991, + "pass@2": 0.994, + "pass@4": 0.998, + "pass@8": 0.999, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.963, + "pass@2": 0.983, + "pass@4": 0.997, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.966, + "pass@2": 0.981, + "pass@4": 0.989, + "pass@8": 0.995, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.871, + "pass@2": 0.909, + "pass@4": 0.951, + "pass@8": 0.985, + "pass@16": 0.995, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.797, + "pass@2": 0.873, + "pass@4": 0.934, + "pass@8": 0.975, + "pass@16": 0.99, + "pass@32": 0.995, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.753, + "pass@2": 0.815, + "pass@4": 0.895, + "pass@8": 0.943, + "pass@16": 0.98, + "pass@32": 0.998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.626, + "pass@2": 0.693, + "pass@4": 0.777, + "pass@8": 0.847, + "pass@16": 0.911, + "pass@32": 0.933, + "pass@64": 0.951, + "pass@128": 0.969 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.991, + "5": 0.963, + "6": 0.966, + "7": 0.871, + "8": 0.797, + "9": 0.753, + "10": 0.626 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.994, + "5": 0.983, + "6": 0.981, + "7": 0.909, + "8": 0.873, + "9": 0.815, + "10": 0.693 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.998, + "5": 0.997, + "6": 0.989, + "7": 0.951, + "8": 0.934, + "9": 0.895, + "10": 0.777 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.999, + "6": 0.995, + "7": 0.985, + "8": 0.975, + "9": 0.943, + "10": 0.847 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 0.999, + "7": 0.995, + "8": 0.99, + "9": 0.98, + "10": 0.911 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.995, + "9": 0.998, + "10": 0.933 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.998, + "9": 1.0, + "10": 0.951 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 1.0, + "10": 0.969 + } + }, + "total_pass_at_k": { + "pass@1": 0.8852222222222222, + "pass@2": 0.9164444444444444, + "pass@4": 0.949, + "pass@8": 0.9714444444444444, + "pass@16": 0.986, + "pass@32": 0.9917777777777778, + "pass@64": 0.9943333333333333, + "pass@128": 0.9964444444444445 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.56657223796034, + "pass@2": 0.6288951841359773, + "pass@4": 0.7308781869688386, + "pass@8": 0.8186968838526912, + "pass@16": 0.8753541076487252, + "pass@32": 0.9093484419263456, + "pass@64": 0.9348441926345609, + "pass@128": 0.9376770538243626 + }, + "12": { + "pass@1": 0.5621468926553672, + "pass@2": 0.6384180790960452, + "pass@4": 0.7033898305084746, + "pass@8": 0.768361581920904, + "pass@16": 0.8050847457627118, + "pass@32": 0.827683615819209, + "pass@64": 0.8418079096045198, + "pass@128": 0.8502824858757062 + }, + "13": { + "pass@1": 0.36990595611285265, + "pass@2": 0.438871473354232, + "pass@4": 0.5109717868338558, + "pass@8": 0.567398119122257, + "pass@16": 0.6269592476489029, + "pass@32": 0.7053291536050157, + "pass@64": 0.7492163009404389, + "pass@128": 0.7554858934169278 + }, + "14": { + "pass@1": 0.3364485981308411, + "pass@2": 0.3925233644859813, + "pass@4": 0.43613707165109034, + "pass@8": 0.4984423676012461, + "pass@16": 0.5545171339563862, + "pass@32": 0.6137071651090342, + "pass@64": 0.6822429906542056, + "pass@128": 0.7227414330218068 + }, + "15": { + "pass@1": 0.21965317919075145, + "pass@2": 0.2745664739884393, + "pass@4": 0.315028901734104, + "pass@8": 0.3670520231213873, + "pass@16": 0.4479768786127168, + "pass@32": 0.5173410404624278, + "pass@64": 0.5895953757225434, + "pass@128": 0.6329479768786127 + }, + "16": { + "pass@1": 0.07096774193548387, + "pass@2": 0.09032258064516129, + "pass@4": 0.1032258064516129, + "pass@8": 0.12903225806451613, + "pass@16": 0.15483870967741936, + "pass@32": 0.1870967741935484, + "pass@64": 0.20967741935483872, + "pass@128": 0.22903225806451613 + }, + "17": { + "pass@1": 0.05202312138728324, + "pass@2": 0.06069364161849711, + "pass@4": 0.08670520231213873, + "pass@8": 0.1069364161849711, + "pass@16": 0.11560693641618497, + "pass@32": 0.13005780346820808, + "pass@64": 0.17052023121387283, + "pass@128": 0.1907514450867052 + }, + "18": { + "pass@1": 0.011428571428571429, + "pass@2": 0.02, + "pass@4": 0.025714285714285714, + "pass@8": 0.03142857142857143, + "pass@16": 0.037142857142857144, + "pass@32": 0.045714285714285714, + "pass@64": 0.06571428571428571, + "pass@128": 0.08571428571428572 + }, + "19": { + "pass@1": 0.003003003003003003, + "pass@2": 0.003003003003003003, + "pass@4": 0.003003003003003003, + "pass@8": 0.006006006006006006, + "pass@16": 0.006006006006006006, + "pass@32": 0.006006006006006006, + "pass@64": 0.009009009009009009, + "pass@128": 0.009009009009009009 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.003003003003003003, + "pass@4": 0.003003003003003003, + "pass@8": 0.003003003003003003, + "pass@16": 0.003003003003003003, + "pass@32": 0.003003003003003003, + "pass@64": 0.006006006006006006, + "pass@128": 0.018018018018018018 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.56657223796034, + "12": 0.5621468926553672, + "13": 0.36990595611285265, + "14": 0.3364485981308411, + "15": 0.21965317919075145, + "16": 0.07096774193548387, + "17": 0.05202312138728324, + "18": 0.011428571428571429, + "19": 0.003003003003003003, + "20": 0.0 + }, + "pass@2": { + "11": 0.6288951841359773, + "12": 0.6384180790960452, + "13": 0.438871473354232, + "14": 0.3925233644859813, + "15": 0.2745664739884393, + "16": 0.09032258064516129, + "17": 0.06069364161849711, + "18": 0.02, + "19": 0.003003003003003003, + "20": 0.003003003003003003 + }, + "pass@4": { + "11": 0.7308781869688386, + "12": 0.7033898305084746, + "13": 0.5109717868338558, + "14": 0.43613707165109034, + "15": 0.315028901734104, + "16": 0.1032258064516129, + "17": 0.08670520231213873, + "18": 0.025714285714285714, + "19": 0.003003003003003003, + "20": 0.003003003003003003 + }, + "pass@8": { + "11": 0.8186968838526912, + "12": 0.768361581920904, + "13": 0.567398119122257, + "14": 0.4984423676012461, + "15": 0.3670520231213873, + "16": 0.12903225806451613, + "17": 0.1069364161849711, + "18": 0.03142857142857143, + "19": 0.006006006006006006, + "20": 0.003003003003003003 + }, + "pass@16": { + "11": 0.8753541076487252, + "12": 0.8050847457627118, + "13": 0.6269592476489029, + "14": 0.5545171339563862, + "15": 0.4479768786127168, + "16": 0.15483870967741936, + "17": 0.11560693641618497, + "18": 0.037142857142857144, + "19": 0.006006006006006006, + "20": 0.003003003003003003 + }, + "pass@32": { + "11": 0.9093484419263456, + "12": 0.827683615819209, + "13": 0.7053291536050157, + "14": 0.6137071651090342, + "15": 0.5173410404624278, + "16": 0.1870967741935484, + "17": 0.13005780346820808, + "18": 0.045714285714285714, + "19": 0.006006006006006006, + "20": 0.003003003003003003 + }, + "pass@64": { + "11": 0.9348441926345609, + "12": 0.8418079096045198, + "13": 0.7492163009404389, + "14": 0.6822429906542056, + "15": 0.5895953757225434, + "16": 0.20967741935483872, + "17": 0.17052023121387283, + "18": 0.06571428571428571, + "19": 0.009009009009009009, + "20": 0.006006006006006006 + }, + "pass@128": { + "11": 0.9376770538243626, + "12": 0.8502824858757062, + "13": 0.7554858934169278, + "14": 0.7227414330218068, + "15": 0.6329479768786127, + "16": 0.22903225806451613, + "17": 0.1907514450867052, + "18": 0.08571428571428572, + "19": 0.009009009009009009, + "20": 0.018018018018018018 + } + }, + "total_pass_at_k": { + "pass@1": 0.22169390787518573, + "pass@2": 0.2576523031203566, + "pass@4": 0.2947994056463596, + "pass@8": 0.3328380386329866, + "pass@16": 0.36582466567607724, + "pass@32": 0.39732540861812776, + "pass@64": 0.42852897473997026, + "pass@128": 0.4457652303120357 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5714285714285714, + "pass@2": 0.6730158730158731, + "pass@4": 0.7333333333333333, + "pass@8": 0.8412698412698413, + "pass@16": 0.8984126984126984, + "pass@32": 0.9396825396825397, + "pass@64": 0.9682539682539683, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.5351681957186545, + "pass@2": 0.6055045871559633, + "pass@4": 0.6666666666666666, + "pass@8": 0.7584097859327217, + "pass@16": 0.8073394495412844, + "pass@32": 0.8409785932721713, + "pass@64": 0.8532110091743119, + "pass@128": 0.8654434250764526 + }, + "13": { + "pass@1": 0.3950617283950617, + "pass@2": 0.4691358024691358, + "pass@4": 0.5401234567901234, + "pass@8": 0.5833333333333334, + "pass@16": 0.6172839506172839, + "pass@32": 0.6604938271604939, + "pass@64": 0.7006172839506173, + "pass@128": 0.7191358024691358 + }, + "14": { + "pass@1": 0.3056379821958457, + "pass@2": 0.3798219584569733, + "pass@4": 0.4332344213649852, + "pass@8": 0.47774480712166173, + "pass@16": 0.5637982195845698, + "pass@32": 0.6083086053412463, + "pass@64": 0.6884272997032641, + "pass@128": 0.7388724035608308 + }, + "15": { + "pass@1": 0.24074074074074073, + "pass@2": 0.2839506172839506, + "pass@4": 0.32098765432098764, + "pass@8": 0.3734567901234568, + "pass@16": 0.4382716049382716, + "pass@32": 0.4783950617283951, + "pass@64": 0.5524691358024691, + "pass@128": 0.6111111111111112 + }, + "16": { + "pass@1": 0.05604719764011799, + "pass@2": 0.09734513274336283, + "pass@4": 0.13569321533923304, + "pass@8": 0.168141592920354, + "pass@16": 0.1887905604719764, + "pass@32": 0.2153392330383481, + "pass@64": 0.25073746312684364, + "pass@128": 0.2949852507374631 + }, + "17": { + "pass@1": 0.04216867469879518, + "pass@2": 0.05421686746987952, + "pass@4": 0.07228915662650602, + "pass@8": 0.09036144578313253, + "pass@16": 0.10240963855421686, + "pass@32": 0.12048192771084337, + "pass@64": 0.14156626506024098, + "pass@128": 0.1686746987951807 + }, + "18": { + "pass@1": 0.041935483870967745, + "pass@2": 0.04516129032258064, + "pass@4": 0.05806451612903226, + "pass@8": 0.08064516129032258, + "pass@16": 0.08064516129032258, + "pass@32": 0.1, + "pass@64": 0.11290322580645161, + "pass@128": 0.12580645161290321 + }, + "19": { + "pass@1": 0.006042296072507553, + "pass@2": 0.00906344410876133, + "pass@4": 0.00906344410876133, + "pass@8": 0.015105740181268883, + "pass@16": 0.01812688821752266, + "pass@32": 0.021148036253776436, + "pass@64": 0.02416918429003021, + "pass@128": 0.030211480362537766 + }, + "20": { + "pass@1": 0.003003003003003003, + "pass@2": 0.003003003003003003, + "pass@4": 0.012012012012012012, + "pass@8": 0.012012012012012012, + "pass@16": 0.012012012012012012, + "pass@32": 0.012012012012012012, + "pass@64": 0.015015015015015015, + "pass@128": 0.021021021021021023 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5714285714285714, + "12": 0.5351681957186545, + "13": 0.3950617283950617, + "14": 0.3056379821958457, + "15": 0.24074074074074073, + "16": 0.05604719764011799, + "17": 0.04216867469879518, + "18": 0.041935483870967745, + "19": 0.006042296072507553, + "20": 0.003003003003003003 + }, + "pass@2": { + "11": 0.6730158730158731, + "12": 0.6055045871559633, + "13": 0.4691358024691358, + "14": 0.3798219584569733, + "15": 0.2839506172839506, + "16": 0.09734513274336283, + "17": 0.05421686746987952, + "18": 0.04516129032258064, + "19": 0.00906344410876133, + "20": 0.003003003003003003 + }, + "pass@4": { + "11": 0.7333333333333333, + "12": 0.6666666666666666, + "13": 0.5401234567901234, + "14": 0.4332344213649852, + "15": 0.32098765432098764, + "16": 0.13569321533923304, + "17": 0.07228915662650602, + "18": 0.05806451612903226, + "19": 0.00906344410876133, + "20": 0.012012012012012012 + }, + "pass@8": { + "11": 0.8412698412698413, + "12": 0.7584097859327217, + "13": 0.5833333333333334, + "14": 0.47774480712166173, + "15": 0.3734567901234568, + "16": 0.168141592920354, + "17": 0.09036144578313253, + "18": 0.08064516129032258, + "19": 0.015105740181268883, + "20": 0.012012012012012012 + }, + "pass@16": { + "11": 0.8984126984126984, + "12": 0.8073394495412844, + "13": 0.6172839506172839, + "14": 0.5637982195845698, + "15": 0.4382716049382716, + "16": 0.1887905604719764, + "17": 0.10240963855421686, + "18": 0.08064516129032258, + "19": 0.01812688821752266, + "20": 0.012012012012012012 + }, + "pass@32": { + "11": 0.9396825396825397, + "12": 0.8409785932721713, + "13": 0.6604938271604939, + "14": 0.6083086053412463, + "15": 0.4783950617283951, + "16": 0.2153392330383481, + "17": 0.12048192771084337, + "18": 0.1, + "19": 0.021148036253776436, + "20": 0.012012012012012012 + }, + "pass@64": { + "11": 0.9682539682539683, + "12": 0.8532110091743119, + "13": 0.7006172839506173, + "14": 0.6884272997032641, + "15": 0.5524691358024691, + "16": 0.25073746312684364, + "17": 0.14156626506024098, + "18": 0.11290322580645161, + "19": 0.02416918429003021, + "20": 0.015015015015015015 + }, + "pass@128": { + "11": 0.9746031746031746, + "12": 0.8654434250764526, + "13": 0.7191358024691358, + "14": 0.7388724035608308, + "15": 0.6111111111111112, + "16": 0.2949852507374631, + "17": 0.1686746987951807, + "18": 0.12580645161290321, + "19": 0.030211480362537766, + "20": 0.021021021021021023 + } + }, + "total_pass_at_k": { + "pass@1": 0.21790953545232275, + "pass@2": 0.26008557457212717, + "pass@4": 0.29614914425427874, + "pass@8": 0.33771393643031783, + "pass@16": 0.3704156479217604, + "pass@32": 0.3973105134474328, + "pass@64": 0.4284841075794621, + "pass@128": 0.45293398533007334 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6325301204819277, + "pass@2": 0.6897590361445783, + "pass@4": 0.7590361445783133, + "pass@8": 0.8493975903614458, + "pass@16": 0.9006024096385542, + "pass@32": 0.9337349397590361, + "pass@64": 0.9518072289156626, + "pass@128": 0.9608433734939759 + }, + "12": { + "pass@1": 0.48589341692789967, + "pass@2": 0.5360501567398119, + "pass@4": 0.6144200626959248, + "pass@8": 0.6959247648902821, + "pass@16": 0.7648902821316614, + "pass@32": 0.8056426332288401, + "pass@64": 0.8181818181818182, + "pass@128": 0.8369905956112853 + }, + "13": { + "pass@1": 0.39215686274509803, + "pass@2": 0.47058823529411764, + "pass@4": 0.5490196078431373, + "pass@8": 0.6162464985994398, + "pass@16": 0.680672268907563, + "pass@32": 0.7226890756302521, + "pass@64": 0.7563025210084033, + "pass@128": 0.7815126050420168 + }, + "14": { + "pass@1": 0.2982456140350877, + "pass@2": 0.3742690058479532, + "pass@4": 0.41812865497076024, + "pass@8": 0.4766081871345029, + "pass@16": 0.5380116959064327, + "pass@32": 0.6023391812865497, + "pass@64": 0.6374269005847953, + "pass@128": 0.6988304093567251 + }, + "15": { + "pass@1": 0.24848484848484848, + "pass@2": 0.30606060606060603, + "pass@4": 0.3575757575757576, + "pass@8": 0.40606060606060607, + "pass@16": 0.46060606060606063, + "pass@32": 0.509090909090909, + "pass@64": 0.5636363636363636, + "pass@128": 0.6151515151515151 + }, + "16": { + "pass@1": 0.08831908831908832, + "pass@2": 0.10826210826210826, + "pass@4": 0.14245014245014245, + "pass@8": 0.16809116809116809, + "pass@16": 0.1908831908831909, + "pass@32": 0.20797720797720798, + "pass@64": 0.22792022792022792, + "pass@128": 0.2678062678062678 + }, + "17": { + "pass@1": 0.052795031055900624, + "pass@2": 0.07142857142857142, + "pass@4": 0.07763975155279502, + "pass@8": 0.09316770186335403, + "pass@16": 0.11180124223602485, + "pass@32": 0.14596273291925466, + "pass@64": 0.15838509316770186, + "pass@128": 0.17391304347826086 + }, + "18": { + "pass@1": 0.020588235294117647, + "pass@2": 0.026470588235294117, + "pass@4": 0.041176470588235294, + "pass@8": 0.058823529411764705, + "pass@16": 0.061764705882352944, + "pass@32": 0.07941176470588235, + "pass@64": 0.09117647058823529, + "pass@128": 0.10588235294117647 + }, + "19": { + "pass@1": 0.005952380952380952, + "pass@2": 0.005952380952380952, + "pass@4": 0.005952380952380952, + "pass@8": 0.005952380952380952, + "pass@16": 0.008928571428571428, + "pass@32": 0.008928571428571428, + "pass@64": 0.01488095238095238, + "pass@128": 0.017857142857142856 + }, + "20": { + "pass@1": 0.0029940119760479044, + "pass@2": 0.0029940119760479044, + "pass@4": 0.0029940119760479044, + "pass@8": 0.008982035928143712, + "pass@16": 0.008982035928143712, + "pass@32": 0.011976047904191617, + "pass@64": 0.020958083832335328, + "pass@128": 0.020958083832335328 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6325301204819277, + "12": 0.48589341692789967, + "13": 0.39215686274509803, + "14": 0.2982456140350877, + "15": 0.24848484848484848, + "16": 0.08831908831908832, + "17": 0.052795031055900624, + "18": 0.020588235294117647, + "19": 0.005952380952380952, + "20": 0.0029940119760479044 + }, + "pass@2": { + "11": 0.6897590361445783, + "12": 0.5360501567398119, + "13": 0.47058823529411764, + "14": 0.3742690058479532, + "15": 0.30606060606060603, + "16": 0.10826210826210826, + "17": 0.07142857142857142, + "18": 0.026470588235294117, + "19": 0.005952380952380952, + "20": 0.0029940119760479044 + }, + "pass@4": { + "11": 0.7590361445783133, + "12": 0.6144200626959248, + "13": 0.5490196078431373, + "14": 0.41812865497076024, + "15": 0.3575757575757576, + "16": 0.14245014245014245, + "17": 0.07763975155279502, + "18": 0.041176470588235294, + "19": 0.005952380952380952, + "20": 0.0029940119760479044 + }, + "pass@8": { + "11": 0.8493975903614458, + "12": 0.6959247648902821, + "13": 0.6162464985994398, + "14": 0.4766081871345029, + "15": 0.40606060606060607, + "16": 0.16809116809116809, + "17": 0.09316770186335403, + "18": 0.058823529411764705, + "19": 0.005952380952380952, + "20": 0.008982035928143712 + }, + "pass@16": { + "11": 0.9006024096385542, + "12": 0.7648902821316614, + "13": 0.680672268907563, + "14": 0.5380116959064327, + "15": 0.46060606060606063, + "16": 0.1908831908831909, + "17": 0.11180124223602485, + "18": 0.061764705882352944, + "19": 0.008928571428571428, + "20": 0.008982035928143712 + }, + "pass@32": { + "11": 0.9337349397590361, + "12": 0.8056426332288401, + "13": 0.7226890756302521, + "14": 0.6023391812865497, + "15": 0.509090909090909, + "16": 0.20797720797720798, + "17": 0.14596273291925466, + "18": 0.07941176470588235, + "19": 0.008928571428571428, + "20": 0.011976047904191617 + }, + "pass@64": { + "11": 0.9518072289156626, + "12": 0.8181818181818182, + "13": 0.7563025210084033, + "14": 0.6374269005847953, + "15": 0.5636363636363636, + "16": 0.22792022792022792, + "17": 0.15838509316770186, + "18": 0.09117647058823529, + "19": 0.01488095238095238, + "20": 0.020958083832335328 + }, + "pass@128": { + "11": 0.9608433734939759, + "12": 0.8369905956112853, + "13": 0.7815126050420168, + "14": 0.6988304093567251, + "15": 0.6151515151515151, + "16": 0.2678062678062678, + "17": 0.17391304347826086, + "18": 0.10588235294117647, + "19": 0.017857142857142856, + "20": 0.020958083832335328 + } + }, + "total_pass_at_k": { + "pass@1": 0.22212310437109722, + "pass@2": 0.2586975914362177, + "pass@4": 0.29646149271483796, + "pass@8": 0.3374962830805828, + "pass@16": 0.3722866488254535, + "pass@32": 0.4023193577163247, + "pass@64": 0.423728813559322, + "pass@128": 0.44781445138269405 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.59, + "pass@2": 0.663, + "pass@4": 0.741, + "pass@8": 0.836, + "pass@16": 0.891, + "pass@32": 0.927, + "pass@64": 0.951, + "pass@128": 0.957 + }, + "12": { + "pass@1": 0.529, + "pass@2": 0.595, + "pass@4": 0.663, + "pass@8": 0.742, + "pass@16": 0.793, + "pass@32": 0.825, + "pass@64": 0.838, + "pass@128": 0.851 + }, + "13": { + "pass@1": 0.386, + "pass@2": 0.46, + "pass@4": 0.534, + "pass@8": 0.59, + "pass@16": 0.643, + "pass@32": 0.697, + "pass@64": 0.736, + "pass@128": 0.753 + }, + "14": { + "pass@1": 0.313, + "pass@2": 0.382, + "pass@4": 0.429, + "pass@8": 0.484, + "pass@16": 0.552, + "pass@32": 0.608, + "pass@64": 0.669, + "pass@128": 0.72 + }, + "15": { + "pass@1": 0.236, + "pass@2": 0.288, + "pass@4": 0.331, + "pass@8": 0.382, + "pass@16": 0.449, + "pass@32": 0.502, + "pass@64": 0.569, + "pass@128": 0.62 + }, + "16": { + "pass@1": 0.072, + "pass@2": 0.099, + "pass@4": 0.128, + "pass@8": 0.156, + "pass@16": 0.179, + "pass@32": 0.204, + "pass@64": 0.23, + "pass@128": 0.265 + }, + "17": { + "pass@1": 0.049, + "pass@2": 0.062, + "pass@4": 0.079, + "pass@8": 0.097, + "pass@16": 0.11, + "pass@32": 0.132, + "pass@64": 0.157, + "pass@128": 0.178 + }, + "18": { + "pass@1": 0.024, + "pass@2": 0.03, + "pass@4": 0.041, + "pass@8": 0.056, + "pass@16": 0.059, + "pass@32": 0.074, + "pass@64": 0.089, + "pass@128": 0.105 + }, + "19": { + "pass@1": 0.005, + "pass@2": 0.006, + "pass@4": 0.006, + "pass@8": 0.009, + "pass@16": 0.011, + "pass@32": 0.012, + "pass@64": 0.016, + "pass@128": 0.019 + }, + "20": { + "pass@1": 0.002, + "pass@2": 0.003, + "pass@4": 0.006, + "pass@8": 0.008, + "pass@16": 0.008, + "pass@32": 0.009, + "pass@64": 0.014, + "pass@128": 0.02 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.59, + "12": 0.529, + "13": 0.386, + "14": 0.313, + "15": 0.236, + "16": 0.072, + "17": 0.049, + "18": 0.024, + "19": 0.005, + "20": 0.002 + }, + "pass@2": { + "11": 0.663, + "12": 0.595, + "13": 0.46, + "14": 0.382, + "15": 0.288, + "16": 0.099, + "17": 0.062, + "18": 0.03, + "19": 0.006, + "20": 0.003 + }, + "pass@4": { + "11": 0.741, + "12": 0.663, + "13": 0.534, + "14": 0.429, + "15": 0.331, + "16": 0.128, + "17": 0.079, + "18": 0.041, + "19": 0.006, + "20": 0.006 + }, + "pass@8": { + "11": 0.836, + "12": 0.742, + "13": 0.59, + "14": 0.484, + "15": 0.382, + "16": 0.156, + "17": 0.097, + "18": 0.056, + "19": 0.009, + "20": 0.008 + }, + "pass@16": { + "11": 0.891, + "12": 0.793, + "13": 0.643, + "14": 0.552, + "15": 0.449, + "16": 0.179, + "17": 0.11, + "18": 0.059, + "19": 0.011, + "20": 0.008 + }, + "pass@32": { + "11": 0.927, + "12": 0.825, + "13": 0.697, + "14": 0.608, + "15": 0.502, + "16": 0.204, + "17": 0.132, + "18": 0.074, + "19": 0.012, + "20": 0.009 + }, + "pass@64": { + "11": 0.951, + "12": 0.838, + "13": 0.736, + "14": 0.669, + "15": 0.569, + "16": 0.23, + "17": 0.157, + "18": 0.089, + "19": 0.016, + "20": 0.014 + }, + "pass@128": { + "11": 0.957, + "12": 0.851, + "13": 0.753, + "14": 0.72, + "15": 0.62, + "16": 0.265, + "17": 0.178, + "18": 0.105, + "19": 0.019, + "20": 0.02 + } + }, + "total_pass_at_k": { + "pass@1": 0.2206, + "pass@2": 0.2588, + "pass@4": 0.2958, + "pass@8": 0.336, + "pass@16": 0.3695, + "pass@32": 0.399, + "pass@64": 0.4269, + "pass@128": 0.4488 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block8-epoch4-op7-10/summary.csv b/composition/SDAR-100M-block8-epoch4-op7-10/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..a3e23590f8f9519b290537b24a41d4eab72c5202 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op7-10/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch4-op7-10,0.994,0.48264140625,0.7248638980263158,0.0,0.0,0.0,200.33833854166667,336.96297421875,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block8-epoch4-op7-10/summary.json b/composition/SDAR-100M-block8-epoch4-op7-10/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0fa75c4ac98093ad1c4cbaa95786483f5f9195cd --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op7-10/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch4-op7-10", + "id_acc": 0.994, + "ood_acc": 0.48264140625, + "total_acc": 0.7248638980263158, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.33833854166667, + "ood_avg_resp_len": 336.96297421875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 587026, + "answer_accuracy": 0.7249669024660133, + "avg_response_len": 259.7559662009959, + "resp_tokens_sum": 210331679, + "pass_at_k": { + "pass@1": 0.7249669024660133, + "pass@2": 0.7604557128622972, + "pass@4": 0.789882500277682, + "pass@8": 0.8140953337410525, + "pass@16": 0.8340288729588218, + "pass@32": 0.85067531035173, + "pass@64": 0.8654940988509915, + "pass@128": 0.879386658235852 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9625737463126843, + "pass@2": 0.9730252537569976, + "pass@4": 0.9790491030988384, + "pass@8": 0.9823822861395556, + "pass@16": 0.9851782276001185, + "pass@32": 0.9887806819385732, + "pass@64": 0.9929144734452152, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999503968253968, + "pass@2": 0.999999609423822, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9978246631736527, + "pass@2": 0.9984241654486302, + "pass@4": 0.9991805660332878, + "pass@8": 0.9997867287338961, + "pass@16": 0.9999877852663126, + "pass@32": 0.9999999824528186, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999537721893491, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995292467948718, + "pass@2": 0.9984723589238845, + "pass@4": 0.9997657519372578, + "pass@8": 0.9999935417446474, + "pass@16": 0.9999999969902831, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9949603873239437, + "pass@2": 0.99670414217589, + "pass@4": 0.9974577013260665, + "pass@8": 0.997830234782101, + "pass@16": 0.9983600236132614, + "pass@32": 0.9991228314770513, + "pass@64": 0.9998322280137517, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9385844546742209, + "pass@2": 0.9551942294394504, + "pass@4": 0.967026604677248, + "pass@8": 0.9754982084288127, + "pass@16": 0.9808253815147637, + "pass@32": 0.9850623210750536, + "pass@64": 0.9884474194474718, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.8624426200564972, + "pass@2": 0.9017520519151211, + "pass@4": 0.9232137743904894, + "pass@8": 0.9364155828181312, + "pass@16": 0.9446959074267556, + "pass@32": 0.9501805451185891, + "pass@64": 0.9558795802602567, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.734203565830721, + "pass@2": 0.801667443166391, + "pass@4": 0.8452445452547275, + "pass@8": 0.8750841253152706, + "pass@16": 0.8976730768256509, + "pass@32": 0.916616734947578, + "pass@64": 0.932356215696857, + "pass@128": 0.9404388714733543 + }, + "14": { + "pass@1": 0.6615313473520249, + "pass@2": 0.741312673240611, + "pass@4": 0.7987274321434115, + "pass@8": 0.8374862263892824, + "pass@16": 0.8636665344499674, + "pass@32": 0.883744442134681, + "pass@64": 0.8996549245285982, + "pass@128": 0.9127725856697819 + }, + "15": { + "pass@1": 0.5426752167630058, + "pass@2": 0.6423150829502521, + "pass@4": 0.7232531651571007, + "pass@8": 0.7860465523211205, + "pass@16": 0.8328430235526624, + "pass@32": 0.8664772705101282, + "pass@64": 0.8907010774062084, + "pass@128": 0.9104046242774566 + }, + "16": { + "pass@1": 0.3149193548387097, + "pass@2": 0.39902527305054625, + "pass@4": 0.4697059656857409, + "pass@8": 0.5302840910372171, + "pass@16": 0.5838085174066465, + "pass@32": 0.6282172083334884, + "pass@64": 0.6665010235625493, + "pass@128": 0.7032258064516129 + }, + "17": { + "pass@1": 0.25410946531791906, + "pass@2": 0.32993918119339133, + "pass@4": 0.40069865038546476, + "pass@8": 0.46448555649405504, + "pass@16": 0.5182130045168546, + "pass@32": 0.5623529995882736, + "pass@64": 0.6040674100873197, + "pass@128": 0.6473988439306358 + }, + "18": { + "pass@1": 0.20060267857142858, + "pass@2": 0.2679288526434197, + "pass@4": 0.3344746652204189, + "pass@8": 0.39710824121270083, + "pass@16": 0.4566949168331932, + "pass@32": 0.5120978334164232, + "pass@64": 0.5656579621163826, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.18132976726726727, + "pass@2": 0.2433429985103213, + "pass@4": 0.30306926977596277, + "pass@8": 0.3572313687285545, + "pass@16": 0.40418699091341775, + "pass@32": 0.44416817763081634, + "pass@64": 0.47951343317052403, + "pass@128": 0.5135135135135135 + }, + "20": { + "pass@1": 0.16441441441441443, + "pass@2": 0.22810002423683523, + "pass@4": 0.29267682502525005, + "pass@8": 0.35116514670673493, + "pass@16": 0.40103786340810277, + "pass@32": 0.4445894691442985, + "pass@64": 0.48528400560602686, + "pass@128": 0.5195195195195195 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 594313, + "answer_accuracy": 0.7237833690568979, + "avg_response_len": 268.6203125, + "resp_tokens_sum": 220569511, + "pass_at_k": { + "pass@1": 0.7237833690568979, + "pass@2": 0.7596792895894745, + "pass@4": 0.7891490582919107, + "pass@8": 0.8134931073775019, + "pass@16": 0.8341315095635516, + "pass@32": 0.8522173762624881, + "pass@64": 0.8682187241550315, + "pass@128": 0.8819953234606391 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9544190950920245, + "pass@2": 0.9655979783585334, + "pass@4": 0.9755531901495438, + "pass@8": 0.9849001130563421, + "pass@16": 0.9914179568700926, + "pass@32": 0.9953896244003085, + "pass@64": 0.9980909065259532, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999868960536739, + "pass@4": 0.9999999650561431, + "pass@8": 0.9999999999999815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9999094202898551, + "pass@2": 0.9999992867739359, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998433022922636, + "pass@2": 0.9999925969586897, + "pass@4": 0.9999999905993125, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9957086267605634, + "pass@2": 0.9968146972385495, + "pass@4": 0.9972236687843596, + "pass@8": 0.997358535350474, + "pass@16": 0.9975352112222411, + "pass@32": 0.997887323943662, + "pass@64": 0.9985915492957746, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925383607784432, + "pass@2": 0.994316239568108, + "pass@4": 0.9953136485746168, + "pass@8": 0.9962981752872685, + "pass@16": 0.9975205508838629, + "pass@32": 0.99868979993864, + "pass@64": 0.9996344033069231, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9485363328313253, + "pass@2": 0.9614666836637891, + "pass@4": 0.9690401578191278, + "pass@8": 0.9745595660303529, + "pass@16": 0.9776023172016188, + "pass@32": 0.979402643004502, + "pass@64": 0.981192486457403, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8519298589341693, + "pass@2": 0.882412744057463, + "pass@4": 0.8987941407402442, + "pass@8": 0.9122516853738211, + "pass@16": 0.9248490185450918, + "pass@32": 0.9363708212156535, + "pass@64": 0.9462338759817461, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.6947435224089635, + "pass@2": 0.777705521184852, + "pass@4": 0.8304399998844679, + "pass@8": 0.8625365131047817, + "pass@16": 0.8834984921646472, + "pass@32": 0.8995238580003326, + "pass@64": 0.9129394155831055, + "pass@128": 0.927170868347339 + }, + "14": { + "pass@1": 0.6441429093567251, + "pass@2": 0.7267150302758209, + "pass@4": 0.7861907489085788, + "pass@8": 0.8273056582111475, + "pass@16": 0.8566846397040585, + "pass@32": 0.8804814970414178, + "pass@64": 0.9011153985940141, + "pass@128": 0.9181286549707602 + }, + "15": { + "pass@1": 0.5651751893939394, + "pass@2": 0.6739184413027917, + "pass@4": 0.7489947995136972, + "pass@8": 0.7974960489060056, + "pass@16": 0.8302977072796687, + "pass@32": 0.8556934809010615, + "pass@64": 0.876922393623797, + "pass@128": 0.8939393939393939 + }, + "16": { + "pass@1": 0.3212250712250712, + "pass@2": 0.3989094712519908, + "pass@4": 0.46772314010962307, + "pass@8": 0.5297939095149375, + "pass@16": 0.5862086467853515, + "pass@32": 0.6380627166781673, + "pass@64": 0.6854883475998351, + "pass@128": 0.7264957264957265 + }, + "17": { + "pass@1": 0.2698466614906832, + "pass@2": 0.35929246955543587, + "pass@4": 0.4414016571880376, + "pass@8": 0.5100927321509832, + "pass@16": 0.5673176419200605, + "pass@32": 0.6157075990502674, + "pass@64": 0.6567645873486623, + "pass@128": 0.6925465838509317 + }, + "18": { + "pass@1": 0.20048253676470587, + "pass@2": 0.2711053294349235, + "pass@4": 0.34233683252828667, + "pass@8": 0.4105063409952905, + "pass@16": 0.4747878989634802, + "pass@32": 0.5336599706135036, + "pass@64": 0.5865359016158412, + "pass@128": 0.6352941176470588 + }, + "19": { + "pass@1": 0.15308779761904762, + "pass@2": 0.20727072201912244, + "pass@4": 0.2626874246745942, + "pass@8": 0.3177729661617527, + "pass@16": 0.3739255500583787, + "pass@32": 0.42984186349128706, + "pass@64": 0.482475527616294, + "pass@128": 0.5238095238095238 + }, + "20": { + "pass@1": 0.14775916916167664, + "pass@2": 0.20598456138903293, + "pass@4": 0.26647546779332226, + "pass@8": 0.32486153243092897, + "pass@16": 0.37714961566942967, + "pass@32": 0.4223905208474426, + "pass@64": 0.46155466688637004, + "pass@128": 0.49700598802395207 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 581530, + "answer_accuracy": 0.725867251158332, + "avg_response_len": 288.5859150323534, + "resp_tokens_sum": 231201183, + "pass_at_k": { + "pass@1": 0.725867251158332, + "pass@2": 0.7624292475528114, + "pass@4": 0.792774769656026, + "pass@8": 0.8177036241468999, + "pass@16": 0.8379595685217712, + "pass@32": 0.8546269957433247, + "pass@64": 0.8689409068065663, + "pass@128": 0.8819300207700911 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9559235074626866, + "pass@2": 0.9656980843812436, + "pass@4": 0.9746656671647387, + "pass@8": 0.9820459603607242, + "pass@16": 0.9863468733654595, + "pass@32": 0.9886201472854002, + "pass@64": 0.9902182679940894, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996783088235294, + "pass@2": 0.9999815452755905, + "pass@4": 0.9999999418271246, + "pass@8": 0.9999999999999074, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999482452681388, + "pass@2": 0.9999336328522814, + "pass@4": 0.9999988538498933, + "pass@8": 0.9999999998332318, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.995823948948949, + "pass@2": 0.9981944296658863, + "pass@4": 0.9995453380827396, + "pass@8": 0.9999589276607602, + "pass@16": 0.9999996513465985, + "pass@32": 0.9999999999945224, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941217845659164, + "pass@2": 0.9957378092006989, + "pass@4": 0.997233856820952, + "pass@8": 0.998791655494803, + "pass@16": 0.9997618486572567, + "pass@32": 0.9999916332542949, + "pass@64": 0.9999999987229116, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9454117063492063, + "pass@2": 0.9644895950506187, + "pass@4": 0.9755416611614021, + "pass@8": 0.9823663944356187, + "pass@16": 0.9867982914476112, + "pass@32": 0.9892215283268877, + "pass@64": 0.9902824942007619, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8600917431192661, + "pass@2": 0.9047514237039184, + "pass@4": 0.930534307351489, + "pass@8": 0.9465814909296013, + "pass@16": 0.9574575537927447, + "pass@32": 0.9651714867408759, + "pass@64": 0.9696943208397095, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.7000144675925926, + "pass@2": 0.7741369568387282, + "pass@4": 0.8237057225485699, + "pass@8": 0.8586535597961961, + "pass@16": 0.8835374150406778, + "pass@32": 0.9020236050675741, + "pass@64": 0.9171006250087037, + "pass@128": 0.9320987654320988 + }, + "14": { + "pass@1": 0.6755841988130564, + "pass@2": 0.7566729070772676, + "pass@4": 0.8109844756052377, + "pass@8": 0.846805617430967, + "pass@16": 0.8723752791841834, + "pass@32": 0.8926627247553403, + "pass@64": 0.9091311345903748, + "pass@128": 0.9198813056379822 + }, + "15": { + "pass@1": 0.5425829475308642, + "pass@2": 0.639068879410907, + "pass@4": 0.7103934736745863, + "pass@8": 0.7632807597637843, + "pass@16": 0.803510714937235, + "pass@32": 0.8338604893791952, + "pass@64": 0.8571668944592924, + "pass@128": 0.8796296296296297 + }, + "16": { + "pass@1": 0.2883941740412979, + "pass@2": 0.379378324390867, + "pass@4": 0.4647851607044694, + "pass@8": 0.5395945603525556, + "pass@16": 0.6013971674931184, + "pass@32": 0.6518804161642361, + "pass@64": 0.6972943624071537, + "pass@128": 0.7404129793510325 + }, + "17": { + "pass@1": 0.24458772590361447, + "pass@2": 0.3179293206289724, + "pass@4": 0.3873656151603038, + "pass@8": 0.4529060893572603, + "pass@16": 0.5148842285661349, + "pass@32": 0.5732756836389475, + "pass@64": 0.6286638481690365, + "pass@128": 0.6807228915662651 + }, + "18": { + "pass@1": 0.24243951612903225, + "pass@2": 0.30982981965963907, + "pass@4": 0.37429037156645767, + "pass@8": 0.4342826991634681, + "pass@16": 0.4870580721094836, + "pass@32": 0.5307268008704006, + "pass@64": 0.5660957126821192, + "pass@128": 0.5967741935483871 + }, + "19": { + "pass@1": 0.1842428247734139, + "pass@2": 0.25546765349572986, + "pass@4": 0.3234768755302869, + "pass@8": 0.3800818937670737, + "pass@16": 0.42391733987699637, + "pass@32": 0.4613442746078676, + "pass@64": 0.49752024916745513, + "pass@128": 0.5347432024169184 + }, + "20": { + "pass@1": 0.150243993993994, + "pass@2": 0.2137408077368706, + "pass@4": 0.2801131426702293, + "pass@8": 0.3418353258288631, + "pass@16": 0.3957745566306108, + "pass@32": 0.44117214844812946, + "pass@64": 0.4788019796245884, + "pass@128": 0.5105105105105106 + } + } + } + } + } +] \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_metrics.json b/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..bfe9a7672788f66757cd12dbe59a9b96296e4b0e --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "SDAR-100M-block8-epoch4-op9-12", + "id": { + "answer_accuracy": 0.9935295138888889, + "avg_response_len": 199.78453819444445, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9935295138888889, + "pass@2": 0.9953216316710405, + "pass@4": 0.9966934193121689, + "pass@8": 0.9977862240274186, + "pass@16": 0.9985941094399321, + "pass@32": 0.9991509274337316, + "pass@64": 0.9994859817348817, + "pass@128": 0.9996666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9566953125, + "pass@2": 0.9671897145669298, + "pass@4": 0.9763015637420321, + "pass@8": 0.9841177284807338, + "pass@16": 0.9898433434950407, + "pass@32": 0.9935109453775409, + "pass@64": 0.9956717768827965, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999296875, + "pass@2": 0.9999977854330709, + "pass@4": 0.9999999985939257, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9993671875, + "pass@2": 0.9999009596456694, + "pass@4": 0.9999937616235471, + "pass@8": 0.9999999729956526, + "pass@16": 0.9999999999998621, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99846875, + "pass@2": 0.9992100147637796, + "pass@4": 0.9997477704349457, + "pass@8": 0.9999643299327841, + "pass@16": 0.9999991554454004, + "pass@32": 0.9999999998305314, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999140625, + "pass@2": 0.9999977854330709, + "pass@4": 0.9999999985939257, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9936171875, + "pass@2": 0.9961348425196852, + "pass@4": 0.9976905641169854, + "pass@8": 0.9985067082658915, + "pass@16": 0.9989825851297969, + "pass@32": 0.9993892496351895, + "pass@64": 0.9997513007685047, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9937734375, + "pass@2": 0.9954635826771654, + "pass@4": 0.9965071167041619, + "pass@8": 0.9974872765717031, + "pass@16": 0.9985219008892885, + "pass@32": 0.9994581520603246, + "pass@64": 0.9999507579626357, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.9566953125, + "2": 1.0, + "3": 1.0, + "4": 0.9999296875, + "5": 0.9993671875, + "6": 0.99846875, + "7": 0.9999140625, + "8": 0.9936171875, + "9": 0.9937734375 + }, + "per_op_avg_response_len": { + "10": 280.7323515625, + "2": 115.57025, + "3": 144.594375, + "4": 159.7522734375, + "5": 172.6185546875, + "6": 198.772390625, + "7": 223.6105859375, + "8": 243.31546875, + "9": 259.09459375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.7323515625, + "2": 115.57025, + "3": 144.594375, + "4": 159.7522734375, + "5": 172.6185546875, + "6": 198.772390625, + "7": 223.6105859375, + "8": 243.31546875, + "9": 259.09459375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376718, + "answer_accuracy": 0.9939579111786558, + "avg_response_len": 190.33322515619724, + "resp_tokens_sum": 72137815, + "pass_at_k": { + "pass@1": 0.9939579111786558, + "pass@2": 0.9960199456450922, + "pass@4": 0.9973071401472489, + "pass@8": 0.9980447346715772, + "pass@16": 0.9985467053785979, + "pass@32": 0.9989975171639501, + "pass@64": 0.9992788389631969, + "pass@128": 0.9993245525160419 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9618132374631269, + "pass@2": 0.9725411121176223, + "pass@4": 0.9802102696233768, + "pass@8": 0.9849936946412662, + "pass@16": 0.9884440697041347, + "pass@32": 0.991615765648179, + "pass@64": 0.9937203955529144, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.999994232898622, + "pass@4": 0.9999999956060179, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996279761904762, + "pass@2": 0.9999808617672791, + "pass@4": 0.9999999687539058, + "pass@8": 0.9999999999999978, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985965568862275, + "pass@2": 0.9993480067424206, + "pass@4": 0.9998631435105043, + "pass@8": 0.9999946418638593, + "pass@16": 0.9999999952012559, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999934480268369, + "pass@4": 0.999999995840017, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9912359775641025, + "pass@2": 0.9958788392136078, + "pass@4": 0.9986500876092409, + "pass@8": 0.9997715179680895, + "pass@16": 0.9999920381521239, + "pass@32": 0.9999999947732479, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9972881085726961, + "pass@4": 0.9977522703148021, + "pass@8": 0.9982272475790946, + "pass@16": 0.998920396325759, + "pass@32": 0.9996448039393536, + "pass@64": 0.9999814875425009, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387814, + "answer_accuracy": 0.9927250573394495, + "avg_response_len": 197.7959125163827, + "resp_tokens_sum": 77270160, + "pass_at_k": { + "pass@1": 0.9927250573394495, + "pass@2": 0.9944081540051185, + "pass@4": 0.9958760632783779, + "pass@8": 0.9973098896013131, + "pass@16": 0.9985301797085706, + "pass@32": 0.999370767579105, + "pass@64": 0.9998439578635286, + "pass@128": 1.0 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9461752300613497, + "pass@2": 0.9579337954688179, + "pass@4": 0.9694229502945566, + "pass@8": 0.9810037051895499, + "pass@16": 0.9905673067988211, + "pass@32": 0.9965809863505134, + "pass@64": 0.9993884580111478, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9990942028985508, + "pass@2": 0.9997621391076116, + "pass@4": 0.9999820552322264, + "pass@8": 0.9999999217268679, + "pass@16": 0.9999999999996003, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9980524713467048, + "pass@2": 0.9987414829772353, + "pass@4": 0.9994561608344803, + "pass@8": 0.9999035515654319, + "pass@16": 0.9999975847114193, + "pass@32": 0.9999999995144166, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999774205202312, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9965228873239437, + "pass@2": 0.9971952284573583, + "pass@4": 0.9973554949117276, + "pass@8": 0.997525506974028, + "pass@16": 0.9978457358323166, + "pass@32": 0.998419651768881, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9928190494011976, + "pass@2": 0.9945048387477016, + "pass@4": 0.9955588426633797, + "pass@8": 0.996690741627332, + "pass@16": 0.9980681831422826, + "pass@32": 0.9992670805169979, + "pass@64": 0.9999136334847853, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 380014, + "answer_accuracy": 0.9939268078339472, + "avg_response_len": 211.1854834491128, + "resp_tokens_sum": 80743813, + "pass_at_k": { + "pass@1": 0.9939268078339472, + "pass@2": 0.9955627519118282, + "pass@4": 0.9969201830290966, + "pass@8": 0.9980166634820948, + "pass@16": 0.9987064217819897, + "pass@32": 0.9990783782824575, + "pass@64": 0.9993255554216345, + "pass@128": 0.9996652159357214 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9617537313432836, + "pass@2": 0.970781672346927, + "pass@4": 0.9790400016229313, + "pass@8": 0.98626166598672, + "pass@16": 0.9905548114893244, + "pass@32": 0.9924411918583297, + "pass@64": 0.9940296282947009, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994025735294118, + "pass@2": 0.9999677946966188, + "pass@4": 0.9999998894439663, + "pass@8": 0.9999999999996586, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987923895899053, + "pass@2": 0.999580450955066, + "pass@4": 0.999947256691494, + "pass@8": 0.9999993061637832, + "pass@16": 0.9999999999491976, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927505630630631, + "pass@2": 0.9952442600080397, + "pass@4": 0.9971487570247412, + "pass@8": 0.9983676867389416, + "pass@16": 0.9992487477653507, + "pass@32": 0.9998506753392894, + "pass@64": 0.9999979947884918, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925391881028939, + "pass@2": 0.9944105723219485, + "pass@4": 0.9961042035139499, + "pass@8": 0.997498057805773, + "pass@16": 0.9985542991129313, + "pass@32": 0.9994502950777389, + "pass@64": 0.9999555508074901, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.5771015625, + "avg_response_len": 354.4255953125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5771015625, + "pass@2": 0.6480805364173114, + "pass@4": 0.7060837591488446, + "pass@8": 0.7523766467121149, + "pass@16": 0.789200237911657, + "pass@32": 0.8187643350468798, + "pass@64": 0.8428475613537451, + "pass@128": 0.8628 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.957140625, + "pass@2": 0.9682801427165355, + "pass@4": 0.9749531122047241, + "pass@8": 0.9791891767647783, + "pass@16": 0.98211903213778, + "pass@32": 0.9846358344909678, + "pass@64": 0.9867111407997167, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.9026015625, + "pass@2": 0.924557701771654, + "pass@4": 0.9376451166104238, + "pass@8": 0.9468465265856568, + "pass@16": 0.954191808841156, + "pass@32": 0.9603990401824475, + "pass@64": 0.9657758332302137, + "pass@128": 0.97 + }, + "13": { + "pass@1": 0.814109375, + "pass@2": 0.8641605561023634, + "pass@4": 0.895520004874391, + "pass@8": 0.9164012838141359, + "pass@16": 0.9319405625887556, + "pass@32": 0.9444791104314104, + "pass@64": 0.9538452723690258, + "pass@128": 0.96 + }, + "14": { + "pass@1": 0.7910234375, + "pass@2": 0.8541248769685043, + "pass@4": 0.8911382658417697, + "pass@8": 0.9145212717761322, + "pass@16": 0.9314696532384339, + "pass@32": 0.9443763740796397, + "pass@64": 0.9539442507806651, + "pass@128": 0.961 + }, + "15": { + "pass@1": 0.6977265625, + "pass@2": 0.782586245078741, + "pass@4": 0.8406202425946752, + "pass@8": 0.8807529357498628, + "pass@16": 0.9092865874591104, + "pass@32": 0.9296244730051035, + "pass@64": 0.9429202877342062, + "pass@128": 0.951 + }, + "16": { + "pass@1": 0.508703125, + "pass@2": 0.6256612942913391, + "pass@4": 0.7164985766779153, + "pass@8": 0.7826049747381607, + "pass@16": 0.8290852993476789, + "pass@32": 0.8612613381637253, + "pass@64": 0.8844828159208924, + "pass@128": 0.903 + }, + "17": { + "pass@1": 0.3892578125, + "pass@2": 0.4965643454724407, + "pass@4": 0.5878991002062244, + "pass@8": 0.6593557882776578, + "pass@16": 0.7147717915717123, + "pass@32": 0.7588696172178708, + "pass@64": 0.7963482777124878, + "pass@128": 0.831 + }, + "18": { + "pass@1": 0.320015625, + "pass@2": 0.4193448572834649, + "pass@4": 0.5105761571053615, + "pass@8": 0.5890740631550005, + "pass@16": 0.6547608784288393, + "pass@32": 0.7090559625862405, + "pass@64": 0.7541917451827876, + "pass@128": 0.793 + }, + "19": { + "pass@1": 0.208, + "pass@2": 0.28776230314960616, + "pass@4": 0.3691554985939253, + "pass@8": 0.44260925321184885, + "pass@16": 0.5048892048968053, + "pass@32": 0.5590981633198217, + "pass@64": 0.6067321047916664, + "pass@128": 0.645 + }, + "20": { + "pass@1": 0.1824375, + "pass@2": 0.2577630413385826, + "pass@4": 0.33683151677915263, + "pass@8": 0.41241119304797336, + "pass@16": 0.47948756060617204, + "pass@32": 0.5358434369915598, + "pass@64": 0.5835238850156994, + "pass@128": 0.626 + } + }, + "per_op_accuracy": { + "11": 0.957140625, + "12": 0.9026015625, + "13": 0.814109375, + "14": 0.7910234375, + "15": 0.6977265625, + "16": 0.508703125, + "17": 0.3892578125, + "18": 0.320015625, + "19": 0.208, + "20": 0.1824375 + }, + "per_op_avg_response_len": { + "11": 298.8628671875, + "12": 308.6419140625, + "13": 326.812234375, + "14": 353.3949140625, + "15": 369.8311484375, + "16": 369.7020390625, + "17": 379.1345, + "18": 386.7295078125, + "19": 371.21990625, + "20": 379.926921875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 298.8628671875, + "12": 308.6419140625, + "13": 326.812234375, + "14": 353.3949140625, + "15": 369.8311484375, + "16": 369.7020390625, + "17": 379.1345, + "18": 386.7295078125, + "19": 371.21990625, + "20": 379.926921875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 251749, + "answer_accuracy": 0.5844841196136701, + "avg_response_len": 336.48382940193164, + "resp_tokens_sum": 144930315, + "pass_at_k": { + "pass@1": 0.5844841196136701, + "pass@2": 0.6537676156240151, + "pass@4": 0.709439007131913, + "pass@8": 0.7540203073917721, + "pass@16": 0.7902984564424712, + "pass@32": 0.8197780068817672, + "pass@64": 0.8446094992676029, + "pass@128": 0.8665676077265974 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9514651203966006, + "pass@2": 0.964051451562535, + "pass@4": 0.9726533114840814, + "pass@8": 0.9784351301207374, + "pass@16": 0.9821156095662544, + "pass@32": 0.9847244967783608, + "pass@64": 0.9857797640615148, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8961864406779662, + "pass@2": 0.9169464416789005, + "pass@4": 0.930045741104395, + "pass@8": 0.9405154502869694, + "pass@16": 0.9496226622064528, + "pass@32": 0.9572528675630418, + "pass@64": 0.9642242554401338, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8420111677115988, + "pass@2": 0.889376943820502, + "pass@4": 0.9181533575936236, + "pass@8": 0.9363334327363076, + "pass@16": 0.949785723044496, + "pass@32": 0.9610582398326716, + "pass@64": 0.9701180751617702, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8046388239875389, + "pass@2": 0.8602070915201021, + "pass@4": 0.8914461434143157, + "pass@8": 0.9119244313646143, + "pass@16": 0.9272654211635459, + "pass@32": 0.9383072121466731, + "pass@64": 0.9462294718077346, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.7136244580924855, + "pass@2": 0.8005655892995307, + "pass@4": 0.8601507764022269, + "pass@8": 0.9013584538970695, + "pass@16": 0.9310673045046043, + "pass@32": 0.9517436419640584, + "pass@64": 0.9651808484660429, + "pass@128": 0.9739884393063584 + }, + "16": { + "pass@1": 0.523109879032258, + "pass@2": 0.6348286290322582, + "pass@4": 0.7166433194842577, + "pass@8": 0.7779303950852562, + "pass@16": 0.8253214388050136, + "pass@32": 0.8612953308847185, + "pass@64": 0.8875623036766475, + "pass@128": 0.9032258064516129 + }, + "17": { + "pass@1": 0.40713059971098264, + "pass@2": 0.5174622940466982, + "pass@4": 0.6078873918758712, + "pass@8": 0.6741004742749396, + "pass@16": 0.7231938373226637, + "pass@32": 0.7607190313851854, + "pass@64": 0.7937445756323053, + "pass@128": 0.8294797687861272 + }, + "18": { + "pass@1": 0.3004017857142857, + "pass@2": 0.3996291479190101, + "pass@4": 0.4907275606620602, + "pass@8": 0.5703714538130185, + "pass@16": 0.6388093892798813, + "pass@32": 0.6961804908111601, + "pass@64": 0.7459388686836753, + "pass@128": 0.7914285714285715 + }, + "19": { + "pass@1": 0.2051661036036036, + "pass@2": 0.27981980504126164, + "pass@4": 0.35245659495265824, + "pass@8": 0.4186875923466568, + "pass@16": 0.4774648463192164, + "pass@32": 0.530660247056415, + "pass@64": 0.5814726524949139, + "pass@128": 0.6306306306306306 + }, + "20": { + "pass@1": 0.19031531531531531, + "pass@2": 0.26815730592797515, + "pass@4": 0.3495825434545906, + "pass@8": 0.42705696041508395, + "pass@16": 0.4956881270636332, + "pass@32": 0.5538044007945283, + "pass@64": 0.6040435053189296, + "pass@128": 0.6486486486486487 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 250403, + "answer_accuracy": 0.5817048580136782, + "avg_response_len": 349.934119926405, + "resp_tokens_sum": 150634041, + "pass_at_k": { + "pass@1": 0.5817048580136782, + "pass@2": 0.6533466322954088, + "pass@4": 0.7110509270919676, + "pass@8": 0.7567381183491475, + "pass@16": 0.7934898486662506, + "pass@32": 0.8236383292718752, + "pass@64": 0.8481969597645258, + "pass@128": 0.8673803151947665 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9585137424698795, + "pass@2": 0.9671995066881702, + "pass@4": 0.9721630024033138, + "pass@8": 0.9750416872186753, + "pass@16": 0.9774501659354414, + "pass@32": 0.9807857011053507, + "pass@64": 0.9848766720425005, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.9050499608150471, + "pass@2": 0.923858545405179, + "pass@4": 0.9327921901611824, + "pass@8": 0.9387195559136081, + "pass@16": 0.9449860445424791, + "pass@32": 0.952132953833246, + "pass@64": 0.9586975480454887, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.8070290616246498, + "pass@2": 0.8562368352852956, + "pass@4": 0.8857674544883569, + "pass@8": 0.9058580167888862, + "pass@16": 0.9223591986829233, + "pass@32": 0.9360243594008619, + "pass@64": 0.9450110575563182, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.7976973684210527, + "pass@2": 0.867424569461712, + "pass@4": 0.9068931860491114, + "pass@8": 0.9283267548205743, + "pass@16": 0.9418403403371376, + "pass@32": 0.9524016855817911, + "pass@64": 0.9614012630596699, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.6974431818181818, + "pass@2": 0.7863684830589354, + "pass@4": 0.8486680735930725, + "pass@8": 0.891768548521778, + "pass@16": 0.9207707356638722, + "pass@32": 0.9400995306433965, + "pass@64": 0.9521569927151597, + "pass@128": 0.9606060606060606 + }, + "16": { + "pass@1": 0.5107505341880342, + "pass@2": 0.6242663677456984, + "pass@4": 0.7127050812024565, + "pass@8": 0.7776122661212889, + "pass@16": 0.8247779610106144, + "pass@32": 0.8566217836455938, + "pass@64": 0.8764856255133433, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.4086034549689441, + "pass@2": 0.5166432575683475, + "pass@4": 0.6060352741233433, + "pass@8": 0.6743064342428341, + "pass@16": 0.7267822114247295, + "pass@32": 0.7705926178718194, + "pass@64": 0.8082578808157032, + "pass@128": 0.8385093167701864 + }, + "18": { + "pass@1": 0.3369255514705882, + "pass@2": 0.440790368805002, + "pass@4": 0.5347838508237939, + "pass@8": 0.6161963128406492, + "pass@16": 0.6852001345712164, + "pass@32": 0.7416791147734874, + "pass@64": 0.7868330160139833, + "pass@128": 0.8235294117647058 + }, + "19": { + "pass@1": 0.21549479166666666, + "pass@2": 0.29482081341394845, + "pass@4": 0.3771499184811719, + "pass@8": 0.45314763547808923, + "pass@16": 0.5185264418909887, + "pass@32": 0.5747617192946216, + "pass@64": 0.6211208246521323, + "pass@128": 0.6517857142857143 + }, + "20": { + "pass@1": 0.18099738023952097, + "pass@2": 0.25491352432929404, + "pass@4": 0.33130779498371077, + "pass@8": 0.4034410216749828, + "pass@16": 0.4688099342745726, + "pass@32": 0.5279449481851225, + "pass@64": 0.5843637752328384, + "pass@128": 0.6377245508982036 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 236538, + "answer_accuracy": 0.5647778499388753, + "avg_response_len": 377.49371084199265, + "resp_tokens_sum": 158100406, + "pass_at_k": { + "pass@1": 0.5647778499388753, + "pass@2": 0.6368192583095919, + "pass@4": 0.697527831503492, + "pass@8": 0.7462034965586087, + "pass@16": 0.783661892458126, + "pass@32": 0.8127123031694083, + "pass@64": 0.8355373694418495, + "pass@128": 0.8542176039119804 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9620535714285714, + "pass@2": 0.9741579177602799, + "pass@4": 0.9804710363585502, + "pass@8": 0.984405510081136, + "pass@16": 0.9870437043502408, + "pass@32": 0.9885943947975548, + "pass@64": 0.9896883459298144, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.907157874617737, + "pass@2": 0.9334794661561804, + "pass@4": 0.9506061639084101, + "pass@8": 0.9616284980049788, + "pass@16": 0.9681187712875272, + "pass@32": 0.9718688464597104, + "pass@64": 0.9743606390761322, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7944396219135802, + "pass@2": 0.8480640766258388, + "pass@4": 0.883981797992843, + "pass@8": 0.9083938480790773, + "pass@16": 0.9249280339745628, + "pass@32": 0.9374717148724703, + "pass@64": 0.9475575890272085, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.771281528189911, + "pass@2": 0.8348344149396016, + "pass@4": 0.87485633286937, + "pass@8": 0.9029845078915546, + "pass@16": 0.9249497229959522, + "pass@32": 0.9420130044854153, + "pass@64": 0.9537250990028946, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.6810378086419753, + "pass@2": 0.759533801764363, + "pass@4": 0.8115666965240457, + "pass@8": 0.8475286718805226, + "pass@16": 0.8743301152204945, + "pass@32": 0.8953343449790683, + "pass@64": 0.9097404523733096, + "pass@128": 0.9166666666666666 + }, + "16": { + "pass@1": 0.4934089233038348, + "pass@2": 0.6187224903026508, + "pass@4": 0.720293994500687, + "pass@8": 0.7920491057615308, + "pass@16": 0.836986988210621, + "pass@32": 0.866034039911091, + "pass@64": 0.889947041964446, + "pass@128": 0.9144542772861357 + }, + "17": { + "pass@1": 0.3518684111445783, + "pass@2": 0.45531103251589033, + "pass@4": 0.5494780251489645, + "pass@8": 0.6294890131094445, + "pass@16": 0.6943459390943068, + "pass@32": 0.7455723174815397, + "pass@64": 0.7875108943405223, + "pass@128": 0.8253012048192772 + }, + "18": { + "pass@1": 0.3236139112903226, + "pass@2": 0.41808364554229105, + "pass@4": 0.5064354890114543, + "pass@8": 0.5804429288858804, + "pass@16": 0.6393856336344137, + "pass@32": 0.6878125541269339, + "pass@64": 0.7277094054798247, + "pass@128": 0.7612903225806451 + }, + "19": { + "pass@1": 0.20324301359516617, + "pass@2": 0.28858765789661495, + "pass@4": 0.3778401204381177, + "pass@8": 0.4559778835038485, + "pass@16": 0.5186360320155117, + "pass@32": 0.5718077443113075, + "pass@64": 0.6175381100536046, + "pass@128": 0.6525679758308157 + }, + "20": { + "pass@1": 0.17600412912912913, + "pass@2": 0.2502268508666147, + "pass@4": 0.3296207996522957, + "pass@8": 0.40676253450542404, + "pass@16": 0.4739966854242499, + "pass@32": 0.5258046811806326, + "pass@64": 0.5621618523024845, + "pass@128": 0.5915915915915916 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7743569078947369, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7743569078947369, + "pass@2": 0.8125631604848585, + "pass@4": 0.8437409665946201, + "pass@8": 0.868623288598324, + "pass@16": 0.8883868086355705, + "pass@32": 0.9042106156511917, + "pass@64": 0.91704470785007, + "pass@128": 0.9276315789473685 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9566953125, + "pass@2": 0.9671897145669298, + "pass@4": 0.9763015637420321, + "pass@8": 0.9841177284807338, + "pass@16": 0.9898433434950407, + "pass@32": 0.9935109453775409, + "pass@64": 0.9956717768827965, + "pass@128": 0.997 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999296875, + "pass@2": 0.9999977854330709, + "pass@4": 0.9999999985939257, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9993671875, + "pass@2": 0.9999009596456694, + "pass@4": 0.9999937616235471, + "pass@8": 0.9999999729956526, + "pass@16": 0.9999999999998621, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99846875, + "pass@2": 0.9992100147637796, + "pass@4": 0.9997477704349457, + "pass@8": 0.9999643299327841, + "pass@16": 0.9999991554454004, + "pass@32": 0.9999999998305314, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999140625, + "pass@2": 0.9999977854330709, + "pass@4": 0.9999999985939257, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9936171875, + "pass@2": 0.9961348425196852, + "pass@4": 0.9976905641169854, + "pass@8": 0.9985067082658915, + "pass@16": 0.9989825851297969, + "pass@32": 0.9993892496351895, + "pass@64": 0.9997513007685047, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9937734375, + "pass@2": 0.9954635826771654, + "pass@4": 0.9965071167041619, + "pass@8": 0.9974872765717031, + "pass@16": 0.9985219008892885, + "pass@32": 0.9994581520603246, + "pass@64": 0.9999507579626357, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.957140625, + "pass@2": 0.9682801427165355, + "pass@4": 0.9749531122047241, + "pass@8": 0.9791891767647783, + "pass@16": 0.98211903213778, + "pass@32": 0.9846358344909678, + "pass@64": 0.9867111407997167, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.9026015625, + "pass@2": 0.924557701771654, + "pass@4": 0.9376451166104238, + "pass@8": 0.9468465265856568, + "pass@16": 0.954191808841156, + "pass@32": 0.9603990401824475, + "pass@64": 0.9657758332302137, + "pass@128": 0.97 + }, + "13": { + "pass@1": 0.814109375, + "pass@2": 0.8641605561023634, + "pass@4": 0.895520004874391, + "pass@8": 0.9164012838141359, + "pass@16": 0.9319405625887556, + "pass@32": 0.9444791104314104, + "pass@64": 0.9538452723690258, + "pass@128": 0.96 + }, + "14": { + "pass@1": 0.7910234375, + "pass@2": 0.8541248769685043, + "pass@4": 0.8911382658417697, + "pass@8": 0.9145212717761322, + "pass@16": 0.9314696532384339, + "pass@32": 0.9443763740796397, + "pass@64": 0.9539442507806651, + "pass@128": 0.961 + }, + "15": { + "pass@1": 0.6977265625, + "pass@2": 0.782586245078741, + "pass@4": 0.8406202425946752, + "pass@8": 0.8807529357498628, + "pass@16": 0.9092865874591104, + "pass@32": 0.9296244730051035, + "pass@64": 0.9429202877342062, + "pass@128": 0.951 + }, + "16": { + "pass@1": 0.508703125, + "pass@2": 0.6256612942913391, + "pass@4": 0.7164985766779153, + "pass@8": 0.7826049747381607, + "pass@16": 0.8290852993476789, + "pass@32": 0.8612613381637253, + "pass@64": 0.8844828159208924, + "pass@128": 0.903 + }, + "17": { + "pass@1": 0.3892578125, + "pass@2": 0.4965643454724407, + "pass@4": 0.5878991002062244, + "pass@8": 0.6593557882776578, + "pass@16": 0.7147717915717123, + "pass@32": 0.7588696172178708, + "pass@64": 0.7963482777124878, + "pass@128": 0.831 + }, + "18": { + "pass@1": 0.320015625, + "pass@2": 0.4193448572834649, + "pass@4": 0.5105761571053615, + "pass@8": 0.5890740631550005, + "pass@16": 0.6547608784288393, + "pass@32": 0.7090559625862405, + "pass@64": 0.7541917451827876, + "pass@128": 0.793 + }, + "19": { + "pass@1": 0.208, + "pass@2": 0.28776230314960616, + "pass@4": 0.3691554985939253, + "pass@8": 0.44260925321184885, + "pass@16": 0.5048892048968053, + "pass@32": 0.5590981633198217, + "pass@64": 0.6067321047916664, + "pass@128": 0.645 + }, + "20": { + "pass@1": 0.1824375, + "pass@2": 0.2577630413385826, + "pass@4": 0.33683151677915263, + "pass@8": 0.41241119304797336, + "pass@16": 0.47948756060617204, + "pass@32": 0.5358434369915598, + "pass@64": 0.5835238850156994, + "pass@128": 0.626 + } + }, + "per_op_accuracy": { + "10": 0.9566953125, + "2": 1.0, + "3": 1.0, + "4": 0.9999296875, + "5": 0.9993671875, + "6": 0.99846875, + "7": 0.9999140625, + "8": 0.9936171875, + "9": 0.9937734375, + "11": 0.957140625, + "12": 0.9026015625, + "13": 0.814109375, + "14": 0.7910234375, + "15": 0.6977265625, + "16": 0.508703125, + "17": 0.3892578125, + "18": 0.320015625, + "19": 0.208, + "20": 0.1824375 + }, + "per_op_avg_response_len": { + "10": 280.7323515625, + "2": 115.57025, + "3": 144.594375, + "4": 159.7522734375, + "5": 172.6185546875, + "6": 198.772390625, + "7": 223.6105859375, + "8": 243.31546875, + "9": 259.09459375, + "11": 298.8628671875, + "12": 308.6419140625, + "13": 326.812234375, + "14": 353.3949140625, + "15": 369.8311484375, + "16": 369.7020390625, + "17": 379.1345, + "18": 386.7295078125, + "19": 371.21990625, + "20": 379.926921875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.7323515625, + "2": 115.57025, + "3": 144.594375, + "4": 159.7522734375, + "5": 172.6185546875, + "6": 198.772390625, + "7": 223.6105859375, + "8": 243.31546875, + "9": 259.09459375, + "11": 298.8628671875, + "12": 308.6419140625, + "13": 326.812234375, + "14": 353.3949140625, + "15": 369.8311484375, + "16": 369.7020390625, + "17": 379.1345, + "18": 386.7295078125, + "19": 371.21990625, + "20": 379.926921875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 628467, + "answer_accuracy": 0.7761458168668985, + "avg_response_len": 268.07536604884604, + "resp_tokens_sum": 217068130, + "pass_at_k": { + "pass@1": 0.7761458168668985, + "pass@2": 0.8139650783480664, + "pass@4": 0.844180951782302, + "pass@8": 0.868240403688873, + "pass@16": 0.8877728581338872, + "pass@32": 0.9036649765222275, + "pass@64": 0.9170053125522537, + "pass@128": 0.9287069238065128 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9618132374631269, + "pass@2": 0.9725411121176223, + "pass@4": 0.9802102696233768, + "pass@8": 0.9849936946412662, + "pass@16": 0.9884440697041347, + "pass@32": 0.991615765648179, + "pass@64": 0.9937203955529144, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.999994232898622, + "pass@4": 0.9999999956060179, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996279761904762, + "pass@2": 0.9999808617672791, + "pass@4": 0.9999999687539058, + "pass@8": 0.9999999999999978, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985965568862275, + "pass@2": 0.9993480067424206, + "pass@4": 0.9998631435105043, + "pass@8": 0.9999946418638593, + "pass@16": 0.9999999952012559, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999934480268369, + "pass@4": 0.999999995840017, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9912359775641025, + "pass@2": 0.9958788392136078, + "pass@4": 0.9986500876092409, + "pass@8": 0.9997715179680895, + "pass@16": 0.9999920381521239, + "pass@32": 0.9999999947732479, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9972881085726961, + "pass@4": 0.9977522703148021, + "pass@8": 0.9982272475790946, + "pass@16": 0.998920396325759, + "pass@32": 0.9996448039393536, + "pass@64": 0.9999814875425009, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9514651203966006, + "pass@2": 0.964051451562535, + "pass@4": 0.9726533114840814, + "pass@8": 0.9784351301207374, + "pass@16": 0.9821156095662544, + "pass@32": 0.9847244967783608, + "pass@64": 0.9857797640615148, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8961864406779662, + "pass@2": 0.9169464416789005, + "pass@4": 0.930045741104395, + "pass@8": 0.9405154502869694, + "pass@16": 0.9496226622064528, + "pass@32": 0.9572528675630418, + "pass@64": 0.9642242554401338, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8420111677115988, + "pass@2": 0.889376943820502, + "pass@4": 0.9181533575936236, + "pass@8": 0.9363334327363076, + "pass@16": 0.949785723044496, + "pass@32": 0.9610582398326716, + "pass@64": 0.9701180751617702, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8046388239875389, + "pass@2": 0.8602070915201021, + "pass@4": 0.8914461434143157, + "pass@8": 0.9119244313646143, + "pass@16": 0.9272654211635459, + "pass@32": 0.9383072121466731, + "pass@64": 0.9462294718077346, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.7136244580924855, + "pass@2": 0.8005655892995307, + "pass@4": 0.8601507764022269, + "pass@8": 0.9013584538970695, + "pass@16": 0.9310673045046043, + "pass@32": 0.9517436419640584, + "pass@64": 0.9651808484660429, + "pass@128": 0.9739884393063584 + }, + "16": { + "pass@1": 0.523109879032258, + "pass@2": 0.6348286290322582, + "pass@4": 0.7166433194842577, + "pass@8": 0.7779303950852562, + "pass@16": 0.8253214388050136, + "pass@32": 0.8612953308847185, + "pass@64": 0.8875623036766475, + "pass@128": 0.9032258064516129 + }, + "17": { + "pass@1": 0.40713059971098264, + "pass@2": 0.5174622940466982, + "pass@4": 0.6078873918758712, + "pass@8": 0.6741004742749396, + "pass@16": 0.7231938373226637, + "pass@32": 0.7607190313851854, + "pass@64": 0.7937445756323053, + "pass@128": 0.8294797687861272 + }, + "18": { + "pass@1": 0.3004017857142857, + "pass@2": 0.3996291479190101, + "pass@4": 0.4907275606620602, + "pass@8": 0.5703714538130185, + "pass@16": 0.6388093892798813, + "pass@32": 0.6961804908111601, + "pass@64": 0.7459388686836753, + "pass@128": 0.7914285714285715 + }, + "19": { + "pass@1": 0.2051661036036036, + "pass@2": 0.27981980504126164, + "pass@4": 0.35245659495265824, + "pass@8": 0.4186875923466568, + "pass@16": 0.4774648463192164, + "pass@32": 0.530660247056415, + "pass@64": 0.5814726524949139, + "pass@128": 0.6306306306306306 + }, + "20": { + "pass@1": 0.19031531531531531, + "pass@2": 0.26815730592797515, + "pass@4": 0.3495825434545906, + "pass@8": 0.42705696041508395, + "pass@16": 0.4956881270636332, + "pass@32": 0.5538044007945283, + "pass@64": 0.6040435053189296, + "pass@128": 0.6486486486486487 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 638217, + "answer_accuracy": 0.7772518024162121, + "avg_response_len": 277.55285585541696, + "resp_tokens_sum": 227904201, + "pass_at_k": { + "pass@1": 0.7772518024162121, + "pass@2": 0.815610040597508, + "pass@4": 0.8465593161240585, + "pass@8": 0.8711925292394912, + "pass@16": 0.8910398237778925, + "pass@32": 0.9072447831633289, + "pass@64": 0.9203445261243395, + "pass@128": 0.9304754481683554 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9461752300613497, + "pass@2": 0.9579337954688179, + "pass@4": 0.9694229502945566, + "pass@8": 0.9810037051895499, + "pass@16": 0.9905673067988211, + "pass@32": 0.9965809863505134, + "pass@64": 0.9993884580111478, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9990942028985508, + "pass@2": 0.9997621391076116, + "pass@4": 0.9999820552322264, + "pass@8": 0.9999999217268679, + "pass@16": 0.9999999999996003, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9980524713467048, + "pass@2": 0.9987414829772353, + "pass@4": 0.9994561608344803, + "pass@8": 0.9999035515654319, + "pass@16": 0.9999975847114193, + "pass@32": 0.9999999995144166, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999774205202312, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9965228873239437, + "pass@2": 0.9971952284573583, + "pass@4": 0.9973554949117276, + "pass@8": 0.997525506974028, + "pass@16": 0.9978457358323166, + "pass@32": 0.998419651768881, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9928190494011976, + "pass@2": 0.9945048387477016, + "pass@4": 0.9955588426633797, + "pass@8": 0.996690741627332, + "pass@16": 0.9980681831422826, + "pass@32": 0.9992670805169979, + "pass@64": 0.9999136334847853, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9585137424698795, + "pass@2": 0.9671995066881702, + "pass@4": 0.9721630024033138, + "pass@8": 0.9750416872186753, + "pass@16": 0.9774501659354414, + "pass@32": 0.9807857011053507, + "pass@64": 0.9848766720425005, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.9050499608150471, + "pass@2": 0.923858545405179, + "pass@4": 0.9327921901611824, + "pass@8": 0.9387195559136081, + "pass@16": 0.9449860445424791, + "pass@32": 0.952132953833246, + "pass@64": 0.9586975480454887, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.8070290616246498, + "pass@2": 0.8562368352852956, + "pass@4": 0.8857674544883569, + "pass@8": 0.9058580167888862, + "pass@16": 0.9223591986829233, + "pass@32": 0.9360243594008619, + "pass@64": 0.9450110575563182, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.7976973684210527, + "pass@2": 0.867424569461712, + "pass@4": 0.9068931860491114, + "pass@8": 0.9283267548205743, + "pass@16": 0.9418403403371376, + "pass@32": 0.9524016855817911, + "pass@64": 0.9614012630596699, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.6974431818181818, + "pass@2": 0.7863684830589354, + "pass@4": 0.8486680735930725, + "pass@8": 0.891768548521778, + "pass@16": 0.9207707356638722, + "pass@32": 0.9400995306433965, + "pass@64": 0.9521569927151597, + "pass@128": 0.9606060606060606 + }, + "16": { + "pass@1": 0.5107505341880342, + "pass@2": 0.6242663677456984, + "pass@4": 0.7127050812024565, + "pass@8": 0.7776122661212889, + "pass@16": 0.8247779610106144, + "pass@32": 0.8566217836455938, + "pass@64": 0.8764856255133433, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.4086034549689441, + "pass@2": 0.5166432575683475, + "pass@4": 0.6060352741233433, + "pass@8": 0.6743064342428341, + "pass@16": 0.7267822114247295, + "pass@32": 0.7705926178718194, + "pass@64": 0.8082578808157032, + "pass@128": 0.8385093167701864 + }, + "18": { + "pass@1": 0.3369255514705882, + "pass@2": 0.440790368805002, + "pass@4": 0.5347838508237939, + "pass@8": 0.6161963128406492, + "pass@16": 0.6852001345712164, + "pass@32": 0.7416791147734874, + "pass@64": 0.7868330160139833, + "pass@128": 0.8235294117647058 + }, + "19": { + "pass@1": 0.21549479166666666, + "pass@2": 0.29482081341394845, + "pass@4": 0.3771499184811719, + "pass@8": 0.45314763547808923, + "pass@16": 0.5185264418909887, + "pass@32": 0.5747617192946216, + "pass@64": 0.6211208246521323, + "pass@128": 0.6517857142857143 + }, + "20": { + "pass@1": 0.18099738023952097, + "pass@2": 0.25491352432929404, + "pass@4": 0.33130779498371077, + "pass@8": 0.4034410216749828, + "pass@16": 0.4688099342745726, + "pass@32": 0.5279449481851225, + "pass@64": 0.5843637752328384, + "pass@128": 0.6377245508982036 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 616552, + "answer_accuracy": 0.7695818022048251, + "avg_response_len": 298.12597234981627, + "resp_tokens_sum": 238844219, + "pass_at_k": { + "pass@1": 0.7695818022048251, + "pass@2": 0.8080234147866366, + "pass@4": 0.8404076771668446, + "pass@8": 0.8663769954562627, + "pass@16": 0.8862881920411901, + "pass@32": 0.901652304185975, + "pass@64": 0.9137024615526762, + "pass@128": 0.9236299728391116 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9617537313432836, + "pass@2": 0.970781672346927, + "pass@4": 0.9790400016229313, + "pass@8": 0.98626166598672, + "pass@16": 0.9905548114893244, + "pass@32": 0.9924411918583297, + "pass@64": 0.9940296282947009, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994025735294118, + "pass@2": 0.9999677946966188, + "pass@4": 0.9999998894439663, + "pass@8": 0.9999999999996586, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987923895899053, + "pass@2": 0.999580450955066, + "pass@4": 0.999947256691494, + "pass@8": 0.9999993061637832, + "pass@16": 0.9999999999491976, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927505630630631, + "pass@2": 0.9952442600080397, + "pass@4": 0.9971487570247412, + "pass@8": 0.9983676867389416, + "pass@16": 0.9992487477653507, + "pass@32": 0.9998506753392894, + "pass@64": 0.9999979947884918, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925391881028939, + "pass@2": 0.9944105723219485, + "pass@4": 0.9961042035139499, + "pass@8": 0.997498057805773, + "pass@16": 0.9985542991129313, + "pass@32": 0.9994502950777389, + "pass@64": 0.9999555508074901, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9620535714285714, + "pass@2": 0.9741579177602799, + "pass@4": 0.9804710363585502, + "pass@8": 0.984405510081136, + "pass@16": 0.9870437043502408, + "pass@32": 0.9885943947975548, + "pass@64": 0.9896883459298144, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.907157874617737, + "pass@2": 0.9334794661561804, + "pass@4": 0.9506061639084101, + "pass@8": 0.9616284980049788, + "pass@16": 0.9681187712875272, + "pass@32": 0.9718688464597104, + "pass@64": 0.9743606390761322, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7944396219135802, + "pass@2": 0.8480640766258388, + "pass@4": 0.883981797992843, + "pass@8": 0.9083938480790773, + "pass@16": 0.9249280339745628, + "pass@32": 0.9374717148724703, + "pass@64": 0.9475575890272085, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.771281528189911, + "pass@2": 0.8348344149396016, + "pass@4": 0.87485633286937, + "pass@8": 0.9029845078915546, + "pass@16": 0.9249497229959522, + "pass@32": 0.9420130044854153, + "pass@64": 0.9537250990028946, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.6810378086419753, + "pass@2": 0.759533801764363, + "pass@4": 0.8115666965240457, + "pass@8": 0.8475286718805226, + "pass@16": 0.8743301152204945, + "pass@32": 0.8953343449790683, + "pass@64": 0.9097404523733096, + "pass@128": 0.9166666666666666 + }, + "16": { + "pass@1": 0.4934089233038348, + "pass@2": 0.6187224903026508, + "pass@4": 0.720293994500687, + "pass@8": 0.7920491057615308, + "pass@16": 0.836986988210621, + "pass@32": 0.866034039911091, + "pass@64": 0.889947041964446, + "pass@128": 0.9144542772861357 + }, + "17": { + "pass@1": 0.3518684111445783, + "pass@2": 0.45531103251589033, + "pass@4": 0.5494780251489645, + "pass@8": 0.6294890131094445, + "pass@16": 0.6943459390943068, + "pass@32": 0.7455723174815397, + "pass@64": 0.7875108943405223, + "pass@128": 0.8253012048192772 + }, + "18": { + "pass@1": 0.3236139112903226, + "pass@2": 0.41808364554229105, + "pass@4": 0.5064354890114543, + "pass@8": 0.5804429288858804, + "pass@16": 0.6393856336344137, + "pass@32": 0.6878125541269339, + "pass@64": 0.7277094054798247, + "pass@128": 0.7612903225806451 + }, + "19": { + "pass@1": 0.20324301359516617, + "pass@2": 0.28858765789661495, + "pass@4": 0.3778401204381177, + "pass@8": 0.4559778835038485, + "pass@16": 0.5186360320155117, + "pass@32": 0.5718077443113075, + "pass@64": 0.6175381100536046, + "pass@128": 0.6525679758308157 + }, + "20": { + "pass@1": 0.17600412912912913, + "pass@2": 0.2502268508666147, + "pass@4": 0.3296207996522957, + "pass@8": 0.40676253450542404, + "pass@16": 0.4739966854242499, + "pass@32": 0.5258046811806326, + "pass@64": 0.5621618523024845, + "pass@128": 0.5915915915915916 + } + } + } + } + } +} \ No newline at end of file diff --git a/composition/SDAR-100M-block8-epoch4-op9-12/graphs/SDAR-100M-block8-epoch4-op9-12_score_report.jsonl b/composition/SDAR-100M-block8-epoch4-op9-12/graphs/SDAR-100M-block8-epoch4-op9-12_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ac287f7c013cc378f639cf20f8016ebaed982827 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op9-12/graphs/SDAR-100M-block8-epoch4-op9-12_score_report.jsonl @@ -0,0 +1,2 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch4-op9-12", "results_dir": "results/composition/SDAR-100M-block8-epoch4-op9-12/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch4-op9-12/graphs/SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl", "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": true, "results_path": "results/composition/SDAR-100M-block8-epoch4-op9-12/graphs/SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 0, "total": 0, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 0, "extra_nodes": 0, "missing_nodes": 0, "perfect_with_extra": 0, "skipped": 2, "unused_gold_entries": 19000, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch4-op9-12/graphs/SDAR-100M-block8-epoch4-op9-12_score_report.jsonl", "template_stats": {}}} +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/composition/SDAR-100M-block8-epoch4-op9-12", "results_dir": "results/composition/SDAR-100M-block8-epoch4-op9-12/graphs", "per_file": [{"file": "results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1024161, "value_mismatch": 8123, "dependency_mismatch": 125708, "answer_mismatch": 7454, "extra_nodes": 9498, "missing_nodes": 8865, "perfect_with_extra": 446, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337244, "value_mismatch": 2533, "answer_mismatch": 2290, "dependency_mismatch": 41094, "extra_nodes": 3293, "missing_nodes": 3057, "perfect_with_extra": 172}, "teachers_in_school": {"total": 390656, "perfect": 346900, "value_mismatch": 3024, "dependency_mismatch": 43078, "answer_mismatch": 2842, "missing_nodes": 3202, "extra_nodes": 3298, "perfect_with_extra": 91}, "movie_festival_awards": {"total": 382336, "perfect": 340017, "dependency_mismatch": 41536, "value_mismatch": 2566, "answer_mismatch": 2322, "extra_nodes": 2907, "missing_nodes": 2606, "perfect_with_extra": 183}}, "results_path": "results/composition/SDAR-100M-block8-epoch4-op9-12/graphs/SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 366409, "value_mismatch": 636949, "dependency_mismatch": 816640, "answer_mismatch": 541311, "extra_nodes": 53661, "missing_nodes": 452277, "perfect_with_extra": 91, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 123171, "value_mismatch": 212400, "answer_mismatch": 178971, "dependency_mismatch": 274327, "missing_nodes": 161492, "extra_nodes": 21837, "perfect_with_extra": 66}, "movie_festival_awards": {"total": 418816, "perfect": 118321, "dependency_mismatch": 268886, "value_mismatch": 213740, "answer_mismatch": 182280, "extra_nodes": 18331, "missing_nodes": 143246, "perfect_with_extra": 3}, "teachers_in_school": {"total": 430464, "perfect": 124917, "dependency_mismatch": 273427, "value_mismatch": 210809, "answer_mismatch": 180060, "extra_nodes": 13493, "missing_nodes": 147539, "perfect_with_extra": 22}}, "results_path": "results/composition/SDAR-100M-block8-epoch4-op9-12/graphs/SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1390570, "value_mismatch": 645072, "dependency_mismatch": 942348, "answer_mismatch": 548765, "extra_nodes": 63159, "missing_nodes": 461142, "perfect_with_extra": 537, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/composition/SDAR-100M-block8-epoch4-op9-12/graphs/SDAR-100M-block8-epoch4-op9-12_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 460415, "value_mismatch": 214933, "answer_mismatch": 181261, "dependency_mismatch": 315421, "extra_nodes": 25130, "missing_nodes": 164549, "perfect_with_extra": 238}, "teachers_in_school": {"total": 821120, "perfect": 471817, "value_mismatch": 213833, "dependency_mismatch": 316505, "answer_mismatch": 182902, "missing_nodes": 150741, "extra_nodes": 16791, "perfect_with_extra": 113}, "movie_festival_awards": {"total": 801152, "perfect": 458338, "dependency_mismatch": 310422, "value_mismatch": 216306, "answer_mismatch": 184602, "extra_nodes": 21238, "missing_nodes": 145852, "perfect_with_extra": 186}}}} diff --git a/composition/SDAR-100M-block8-epoch4-op9-12/graphs/results.json b/composition/SDAR-100M-block8-epoch4-op9-12/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..e95aa3d6c0e09028fcb27d85ed0bf060f9a42eb6 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op9-12/graphs/results.json @@ -0,0 +1,1476 @@ +{ + "files": { + "SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.56657223796034, + "pass@2": 0.6572237960339944, + "pass@4": 0.7337110481586402, + "pass@8": 0.8215297450424929, + "pass@16": 0.8810198300283286, + "pass@32": 0.9121813031161473, + "pass@64": 0.9348441926345609, + "pass@128": 0.9376770538243626 + }, + "12": { + "pass@1": 0.6045197740112994, + "pass@2": 0.6610169491525424, + "pass@4": 0.7203389830508474, + "pass@8": 0.7740112994350282, + "pass@16": 0.8135593220338984, + "pass@32": 0.8418079096045198, + "pass@64": 0.8502824858757062, + "pass@128": 0.8587570621468926 + }, + "13": { + "pass@1": 0.4670846394984326, + "pass@2": 0.5266457680250783, + "pass@4": 0.5987460815047022, + "pass@8": 0.6896551724137931, + "pass@16": 0.7492163009404389, + "pass@32": 0.7774294670846394, + "pass@64": 0.8056426332288401, + "pass@128": 0.8181818181818182 + }, + "14": { + "pass@1": 0.43302180685358255, + "pass@2": 0.4953271028037383, + "pass@4": 0.5389408099688473, + "pass@8": 0.5919003115264797, + "pass@16": 0.67601246105919, + "pass@32": 0.7476635514018691, + "pass@64": 0.8037383177570093, + "pass@128": 0.8255451713395638 + }, + "15": { + "pass@1": 0.3236994219653179, + "pass@2": 0.3699421965317919, + "pass@4": 0.42485549132947975, + "pass@8": 0.4797687861271676, + "pass@16": 0.5664739884393064, + "pass@32": 0.6647398843930635, + "pass@64": 0.7254335260115607, + "pass@128": 0.7803468208092486 + }, + "16": { + "pass@1": 0.2425249169435216, + "pass@2": 0.2823920265780731, + "pass@4": 0.3089700996677741, + "pass@8": 0.3654485049833887, + "pass@16": 0.4219269102990033, + "pass@32": 0.46511627906976744, + "pass@64": 0.5016611295681063, + "pass@128": 0.5348837209302325 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.56657223796034, + "12": 0.6045197740112994, + "13": 0.4670846394984326, + "14": 0.43302180685358255, + "15": 0.3236994219653179, + "16": 0.2425249169435216 + }, + "pass@2": { + "11": 0.6572237960339944, + "12": 0.6610169491525424, + "13": 0.5266457680250783, + "14": 0.4953271028037383, + "15": 0.3699421965317919, + "16": 0.2823920265780731 + }, + "pass@4": { + "11": 0.7337110481586402, + "12": 0.7203389830508474, + "13": 0.5987460815047022, + "14": 0.5389408099688473, + "15": 0.42485549132947975, + "16": 0.3089700996677741 + }, + "pass@8": { + "11": 0.8215297450424929, + "12": 0.7740112994350282, + "13": 0.6896551724137931, + "14": 0.5919003115264797, + "15": 0.4797687861271676, + "16": 0.3654485049833887 + }, + "pass@16": { + "11": 0.8810198300283286, + "12": 0.8135593220338984, + "13": 0.7492163009404389, + "14": 0.67601246105919, + "15": 0.5664739884393064, + "16": 0.4219269102990033 + }, + "pass@32": { + "11": 0.9121813031161473, + "12": 0.8418079096045198, + "13": 0.7774294670846394, + "14": 0.7476635514018691, + "15": 0.6647398843930635, + "16": 0.46511627906976744 + }, + "pass@64": { + "11": 0.9348441926345609, + "12": 0.8502824858757062, + "13": 0.8056426332288401, + "14": 0.8037383177570093, + "15": 0.7254335260115607, + "16": 0.5016611295681063 + }, + "pass@128": { + "11": 0.9376770538243626, + "12": 0.8587570621468926, + "13": 0.8181818181818182, + "14": 0.8255451713395638, + "15": 0.7803468208092486, + "16": 0.5348837209302325 + } + }, + "total_pass_at_k": { + "pass@1": 0.4448345035105316, + "pass@2": 0.5045135406218656, + "pass@4": 0.5606820461384152, + "pass@8": 0.6268806419257773, + "pass@16": 0.691073219658977, + "pass@32": 0.7412236710130391, + "pass@64": 0.7763289869608826, + "pass@128": 0.79839518555667 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5968253968253968, + "pass@2": 0.6634920634920635, + "pass@4": 0.7333333333333333, + "pass@8": 0.8539682539682539, + "pass@16": 0.9238095238095239, + "pass@32": 0.9555555555555556, + "pass@64": 0.9714285714285714, + "pass@128": 0.9777777777777777 + }, + "12": { + "pass@1": 0.581039755351682, + "pass@2": 0.6513761467889908, + "pass@4": 0.7155963302752294, + "pass@8": 0.8012232415902141, + "pass@16": 0.8501529051987767, + "pass@32": 0.8623853211009175, + "pass@64": 0.8929663608562691, + "pass@128": 0.8990825688073395 + }, + "13": { + "pass@1": 0.47530864197530864, + "pass@2": 0.5370370370370371, + "pass@4": 0.5925925925925926, + "pass@8": 0.6728395061728395, + "pass@16": 0.7222222222222222, + "pass@32": 0.75, + "pass@64": 0.7654320987654321, + "pass@128": 0.7746913580246914 + }, + "14": { + "pass@1": 0.4035608308605341, + "pass@2": 0.47477744807121663, + "pass@4": 0.5133531157270029, + "pass@8": 0.56973293768546, + "pass@16": 0.6379821958456974, + "pass@32": 0.712166172106825, + "pass@64": 0.7833827893175074, + "pass@128": 0.8189910979228486 + }, + "15": { + "pass@1": 0.3117283950617284, + "pass@2": 0.36419753086419754, + "pass@4": 0.4104938271604938, + "pass@8": 0.49691358024691357, + "pass@16": 0.5555555555555556, + "pass@32": 0.6296296296296297, + "pass@64": 0.7006172839506173, + "pass@128": 0.7777777777777778 + }, + "16": { + "pass@1": 0.22289156626506024, + "pass@2": 0.2710843373493976, + "pass@4": 0.3313253012048193, + "pass@8": 0.3885542168674699, + "pass@16": 0.42771084337349397, + "pass@32": 0.4819277108433735, + "pass@64": 0.5331325301204819, + "pass@128": 0.5662650602409639 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5968253968253968, + "12": 0.581039755351682, + "13": 0.47530864197530864, + "14": 0.4035608308605341, + "15": 0.3117283950617284, + "16": 0.22289156626506024 + }, + "pass@2": { + "11": 0.6634920634920635, + "12": 0.6513761467889908, + "13": 0.5370370370370371, + "14": 0.47477744807121663, + "15": 0.36419753086419754, + "16": 0.2710843373493976 + }, + "pass@4": { + "11": 0.7333333333333333, + "12": 0.7155963302752294, + "13": 0.5925925925925926, + "14": 0.5133531157270029, + "15": 0.4104938271604938, + "16": 0.3313253012048193 + }, + "pass@8": { + "11": 0.8539682539682539, + "12": 0.8012232415902141, + "13": 0.6728395061728395, + "14": 0.56973293768546, + "15": 0.49691358024691357, + "16": 0.3885542168674699 + }, + "pass@16": { + "11": 0.9238095238095239, + "12": 0.8501529051987767, + "13": 0.7222222222222222, + "14": 0.6379821958456974, + "15": 0.5555555555555556, + "16": 0.42771084337349397 + }, + "pass@32": { + "11": 0.9555555555555556, + "12": 0.8623853211009175, + "13": 0.75, + "14": 0.712166172106825, + "15": 0.6296296296296297, + "16": 0.4819277108433735 + }, + "pass@64": { + "11": 0.9714285714285714, + "12": 0.8929663608562691, + "13": 0.7654320987654321, + "14": 0.7833827893175074, + "15": 0.7006172839506173, + "16": 0.5331325301204819 + }, + "pass@128": { + "11": 0.9777777777777777, + "12": 0.8990825688073395, + "13": 0.7746913580246914, + "14": 0.8189910979228486, + "15": 0.7777777777777778, + "16": 0.5662650602409639 + } + }, + "total_pass_at_k": { + "pass@1": 0.43032159264931086, + "pass@2": 0.4920877998979071, + "pass@4": 0.5477284328739153, + "pass@8": 0.6283818274629913, + "pass@16": 0.6840224604389995, + "pass@32": 0.7299642674834099, + "pass@64": 0.7728432873915263, + "pass@128": 0.8009188361408882 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6445783132530121, + "pass@2": 0.7078313253012049, + "pass@4": 0.7710843373493976, + "pass@8": 0.8644578313253012, + "pass@16": 0.9156626506024096, + "pass@32": 0.9487951807228916, + "pass@64": 0.963855421686747, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.5517241379310345, + "pass@2": 0.6081504702194357, + "pass@4": 0.6802507836990596, + "pass@8": 0.768025078369906, + "pass@16": 0.8244514106583072, + "pass@32": 0.8432601880877743, + "pass@64": 0.8589341692789969, + "pass@128": 0.8683385579937304 + }, + "13": { + "pass@1": 0.46218487394957986, + "pass@2": 0.5294117647058824, + "pass@4": 0.6162464985994398, + "pass@8": 0.6862745098039216, + "pass@16": 0.7647058823529411, + "pass@32": 0.8095238095238095, + "pass@64": 0.8375350140056023, + "pass@128": 0.8487394957983193 + }, + "14": { + "pass@1": 0.43567251461988304, + "pass@2": 0.48830409356725146, + "pass@4": 0.5292397660818714, + "pass@8": 0.5877192982456141, + "pass@16": 0.6432748538011696, + "pass@32": 0.7134502923976608, + "pass@64": 0.7719298245614035, + "pass@128": 0.8128654970760234 + }, + "15": { + "pass@1": 0.34545454545454546, + "pass@2": 0.41515151515151516, + "pass@4": 0.4818181818181818, + "pass@8": 0.5363636363636364, + "pass@16": 0.5696969696969697, + "pass@32": 0.6454545454545455, + "pass@64": 0.7212121212121212, + "pass@128": 0.7848484848484848 + }, + "16": { + "pass@1": 0.2215568862275449, + "pass@2": 0.2694610778443114, + "pass@4": 0.3143712574850299, + "pass@8": 0.36826347305389223, + "pass@16": 0.4341317365269461, + "pass@32": 0.47904191616766467, + "pass@64": 0.5179640718562875, + "pass@128": 0.5718562874251497 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6445783132530121, + "12": 0.5517241379310345, + "13": 0.46218487394957986, + "14": 0.43567251461988304, + "15": 0.34545454545454546, + "16": 0.2215568862275449 + }, + "pass@2": { + "11": 0.7078313253012049, + "12": 0.6081504702194357, + "13": 0.5294117647058824, + "14": 0.48830409356725146, + "15": 0.41515151515151516, + "16": 0.2694610778443114 + }, + "pass@4": { + "11": 0.7710843373493976, + "12": 0.6802507836990596, + "13": 0.6162464985994398, + "14": 0.5292397660818714, + "15": 0.4818181818181818, + "16": 0.3143712574850299 + }, + "pass@8": { + "11": 0.8644578313253012, + "12": 0.768025078369906, + "13": 0.6862745098039216, + "14": 0.5877192982456141, + "15": 0.5363636363636364, + "16": 0.36826347305389223 + }, + "pass@16": { + "11": 0.9156626506024096, + "12": 0.8244514106583072, + "13": 0.7647058823529411, + "14": 0.6432748538011696, + "15": 0.5696969696969697, + "16": 0.4341317365269461 + }, + "pass@32": { + "11": 0.9487951807228916, + "12": 0.8432601880877743, + "13": 0.8095238095238095, + "14": 0.7134502923976608, + "15": 0.6454545454545455, + "16": 0.47904191616766467 + }, + "pass@64": { + "11": 0.963855421686747, + "12": 0.8589341692789969, + "13": 0.8375350140056023, + "14": 0.7719298245614035, + "15": 0.7212121212121212, + "16": 0.5179640718562875 + }, + "pass@128": { + "11": 0.9698795180722891, + "12": 0.8683385579937304, + "13": 0.8487394957983193, + "14": 0.8128654970760234, + "15": 0.7848484848484848, + "16": 0.5718562874251497 + } + }, + "total_pass_at_k": { + "pass@1": 0.4428997020854022, + "pass@2": 0.5024826216484608, + "pass@4": 0.5650446871896723, + "pass@8": 0.634558093346574, + "pass@16": 0.6916583912611718, + "pass@32": 0.7398212512413108, + "pass@64": 0.7785501489572989, + "pass@128": 0.8093346573982125 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.602, + "pass@2": 0.676, + "pass@4": 0.746, + "pass@8": 0.846, + "pass@16": 0.906, + "pass@32": 0.938, + "pass@64": 0.956, + "pass@128": 0.961 + }, + "12": { + "pass@1": 0.58, + "pass@2": 0.641, + "pass@4": 0.706, + "pass@8": 0.781, + "pass@16": 0.829, + "pass@32": 0.849, + "pass@64": 0.867, + "pass@128": 0.875 + }, + "13": { + "pass@1": 0.468, + "pass@2": 0.531, + "pass@4": 0.603, + "pass@8": 0.683, + "pass@16": 0.746, + "pass@32": 0.78, + "pass@64": 0.804, + "pass@128": 0.815 + }, + "14": { + "pass@1": 0.424, + "pass@2": 0.486, + "pass@4": 0.527, + "pass@8": 0.583, + "pass@16": 0.652, + "pass@32": 0.724, + "pass@64": 0.786, + "pass@128": 0.819 + }, + "15": { + "pass@1": 0.327, + "pass@2": 0.383, + "pass@4": 0.439, + "pass@8": 0.504, + "pass@16": 0.564, + "pass@32": 0.647, + "pass@64": 0.716, + "pass@128": 0.781 + }, + "16": { + "pass@1": 0.22854188210961737, + "pass@2": 0.27404343329886244, + "pass@4": 0.3185108583247156, + "pass@8": 0.37435367114788004, + "pass@16": 0.4281282316442606, + "pass@32": 0.4756980351602896, + "pass@64": 0.5180972078593589, + "pass@128": 0.5584281282316442 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.602, + "12": 0.58, + "13": 0.468, + "14": 0.424, + "15": 0.327, + "16": 0.22854188210961737 + }, + "pass@2": { + "11": 0.676, + "12": 0.641, + "13": 0.531, + "14": 0.486, + "15": 0.383, + "16": 0.27404343329886244 + }, + "pass@4": { + "11": 0.746, + "12": 0.706, + "13": 0.603, + "14": 0.527, + "15": 0.439, + "16": 0.3185108583247156 + }, + "pass@8": { + "11": 0.846, + "12": 0.781, + "13": 0.683, + "14": 0.583, + "15": 0.504, + "16": 0.37435367114788004 + }, + "pass@16": { + "11": 0.906, + "12": 0.829, + "13": 0.746, + "14": 0.652, + "15": 0.564, + "16": 0.4281282316442606 + }, + "pass@32": { + "11": 0.938, + "12": 0.849, + "13": 0.78, + "14": 0.724, + "15": 0.647, + "16": 0.4756980351602896 + }, + "pass@64": { + "11": 0.956, + "12": 0.867, + "13": 0.804, + "14": 0.786, + "15": 0.716, + "16": 0.5180972078593589 + }, + "pass@128": { + "11": 0.961, + "12": 0.875, + "13": 0.815, + "14": 0.819, + "15": 0.781, + "16": 0.5584281282316442 + } + }, + "total_pass_at_k": { + "pass@1": 0.4394167923579688, + "pass@2": 0.4997486173956762, + "pass@4": 0.5579017931959108, + "pass@8": 0.6299648064353947, + "pass@16": 0.6889559242500419, + "pass@32": 0.7370537958773253, + "pass@64": 0.77593430534607, + "pass@128": 0.8029160382101559 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 301 + }, + "total": 1994 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 332 + }, + "total": 1959 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 334 + }, + "total": 2014 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 967 + }, + "total": 5967 + } + } + }, + "SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.99375, + "pass@4": 0.996875, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9682539682539683, + "pass@2": 0.9873015873015873, + "pass@4": 0.9968253968253968, + "pass@8": 0.9968253968253968, + "pass@16": 0.9968253968253968, + "pass@32": 0.9968253968253968, + "pass@64": 0.9968253968253968, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9670658682634731, + "pass@2": 0.9850299401197605, + "pass@4": 0.9940119760479041, + "pass@8": 0.9940119760479041, + "pass@16": 0.9970059880239521, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8816568047337278, + "pass@2": 0.9319526627218935, + "pass@4": 0.9644970414201184, + "pass@8": 0.985207100591716, + "pass@16": 0.9970414201183432, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8365384615384616, + "pass@2": 0.875, + "pass@4": 0.9358974358974359, + "pass@8": 0.9647435897435898, + "pass@16": 0.9871794871794872, + "pass@32": 0.9967948717948718, + "pass@64": 0.9967948717948718, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7492957746478873, + "pass@2": 0.8140845070422535, + "pass@4": 0.9098591549295775, + "pass@8": 0.9661971830985916, + "pass@16": 0.9802816901408451, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6519174041297935, + "pass@2": 0.7050147492625368, + "pass@4": 0.7728613569321534, + "pass@8": 0.8584070796460177, + "pass@16": 0.9174041297935103, + "pass@32": 0.943952802359882, + "pass@64": 0.9587020648967551, + "pass@128": 0.9646017699115044 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9682539682539683, + "6": 0.9670658682634731, + "7": 0.8816568047337278, + "8": 0.8365384615384616, + "9": 0.7492957746478873, + "10": 0.6519174041297935 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9873015873015873, + "6": 0.9850299401197605, + "7": 0.9319526627218935, + "8": 0.875, + "9": 0.8140845070422535, + "10": 0.7050147492625368 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9968253968253968, + "6": 0.9940119760479041, + "7": 0.9644970414201184, + "8": 0.9358974358974359, + "9": 0.9098591549295775, + "10": 0.7728613569321534 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 0.9940119760479041, + "7": 0.985207100591716, + "8": 0.9647435897435898, + "9": 0.9661971830985916, + "10": 0.8584070796460177 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 0.9970059880239521, + "7": 0.9970414201183432, + "8": 0.9871794871794872, + "9": 0.9802816901408451, + "10": 0.9174041297935103 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.943952802359882 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9587020648967551 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9646017699115044 + } + }, + "total_pass_at_k": { + "pass@1": 0.8915906788247214, + "pass@2": 0.9192840256670044, + "pass@4": 0.9510300574130361, + "pass@8": 0.9733198243836542, + "pass@16": 0.9858156028368794, + "pass@32": 0.9929078014184397, + "pass@64": 0.994596420128335, + "pass@128": 0.9956095913542722 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9853801169590644, + "pass@2": 0.9883040935672515, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9647058823529412, + "pass@2": 0.9882352941176471, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9747634069400631, + "pass@2": 0.9810725552050473, + "pass@4": 0.9905362776025236, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8575949367088608, + "pass@2": 0.9050632911392406, + "pass@4": 0.9430379746835443, + "pass@8": 0.9841772151898734, + "pass@16": 0.9936708860759493, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7987987987987988, + "pass@2": 0.8708708708708709, + "pass@4": 0.9279279279279279, + "pass@8": 0.972972972972973, + "pass@16": 0.990990990990991, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.77491961414791, + "pass@2": 0.8392282958199357, + "pass@4": 0.909967845659164, + "pass@8": 0.954983922829582, + "pass@16": 0.9807073954983923, + "pass@32": 0.9967845659163987, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.5880597014925373, + "pass@2": 0.6746268656716418, + "pass@4": 0.7671641791044777, + "pass@8": 0.8507462686567164, + "pass@16": 0.9223880597014925, + "pass@32": 0.9582089552238806, + "pass@64": 0.9641791044776119, + "pass@128": 0.982089552238806 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9853801169590644, + "5": 0.9647058823529412, + "6": 0.9747634069400631, + "7": 0.8575949367088608, + "8": 0.7987987987987988, + "9": 0.77491961414791, + "10": 0.5880597014925373 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.9882352941176471, + "6": 0.9810725552050473, + "7": 0.9050632911392406, + "8": 0.8708708708708709, + "9": 0.8392282958199357, + "10": 0.6746268656716418 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9905362776025236, + "7": 0.9430379746835443, + "8": 0.9279279279279279, + "9": 0.909967845659164, + "10": 0.7671641791044777 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9841772151898734, + "8": 0.972972972972973, + "9": 0.954983922829582, + "10": 0.8507462686567164 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.990990990990991, + "9": 0.9807073954983923, + "10": 0.9223880597014925 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9967845659163987, + "10": 0.9582089552238806 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9641791044776119 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.982089552238806 + } + }, + "total_pass_at_k": { + "pass@1": 0.8844994978239036, + "pass@2": 0.9176431201874791, + "pass@4": 0.9494476062939404, + "pass@8": 0.9738868429862738, + "pass@16": 0.987612989621694, + "pass@32": 0.9949782390358219, + "pass@64": 0.9959825912286575, + "pass@128": 0.9979912956143288 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9733727810650887, + "pass@2": 0.9822485207100592, + "pass@4": 0.9911242603550295, + "pass@8": 0.9970414201183432, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9594202898550724, + "pass@2": 0.9768115942028985, + "pass@4": 0.9884057971014493, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9684813753581661, + "pass@2": 0.9828080229226361, + "pass@4": 0.997134670487106, + "pass@8": 0.997134670487106, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.869942196531792, + "pass@2": 0.9132947976878613, + "pass@4": 0.9710982658959537, + "pass@8": 0.9884393063583815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8084507042253521, + "pass@2": 0.8619718309859155, + "pass@4": 0.9352112676056338, + "pass@8": 0.971830985915493, + "pass@16": 0.9915492957746479, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.718562874251497, + "pass@2": 0.7964071856287425, + "pass@4": 0.8712574850299402, + "pass@8": 0.9221556886227545, + "pass@16": 0.9820359281437125, + "pass@32": 0.9970059880239521, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.647239263803681, + "pass@2": 0.7116564417177914, + "pass@4": 0.7914110429447853, + "pass@8": 0.8312883435582822, + "pass@16": 0.8895705521472392, + "pass@32": 0.9355828220858896, + "pass@64": 0.9539877300613497, + "pass@128": 0.9754601226993865 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9733727810650887, + "5": 0.9594202898550724, + "6": 0.9684813753581661, + "7": 0.869942196531792, + "8": 0.8084507042253521, + "9": 0.718562874251497, + "10": 0.647239263803681 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9822485207100592, + "5": 0.9768115942028985, + "6": 0.9828080229226361, + "7": 0.9132947976878613, + "8": 0.8619718309859155, + "9": 0.7964071856287425, + "10": 0.7116564417177914 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9911242603550295, + "5": 0.9884057971014493, + "6": 0.997134670487106, + "7": 0.9710982658959537, + "8": 0.9352112676056338, + "9": 0.8712574850299402, + "10": 0.7914110429447853 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9884393063583815, + "8": 0.971830985915493, + "9": 0.9221556886227545, + "10": 0.8312883435582822 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9915492957746479, + "9": 0.9820359281437125, + "10": 0.8895705521472392 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9970059880239521, + "10": 0.9355828220858896 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 1.0, + "10": 0.9539877300613497 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9754601226993865 + } + }, + "total_pass_at_k": { + "pass@1": 0.8833551769331586, + "pass@2": 0.9144823066841415, + "pass@4": 0.9501965923984272, + "pass@8": 0.9682175622542595, + "pass@16": 0.9852555701179554, + "pass@32": 0.9924639580602883, + "pass@64": 0.9947575360419397, + "pass@128": 0.9973787680209698 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.984, + "pass@2": 0.988, + "pass@4": 0.996, + "pass@8": 0.999, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.964, + "pass@2": 0.984, + "pass@4": 0.995, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.97, + "pass@2": 0.983, + "pass@4": 0.994, + "pass@8": 0.997, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.87, + "pass@2": 0.917, + "pass@4": 0.96, + "pass@8": 0.986, + "pass@16": 0.997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.814, + "pass@2": 0.869, + "pass@4": 0.933, + "pass@8": 0.97, + "pass@16": 0.99, + "pass@32": 0.998, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.747, + "pass@2": 0.816, + "pass@4": 0.897, + "pass@8": 0.948, + "pass@16": 0.981, + "pass@32": 0.998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.629, + "pass@2": 0.697, + "pass@4": 0.777, + "pass@8": 0.847, + "pass@16": 0.91, + "pass@32": 0.946, + "pass@64": 0.959, + "pass@128": 0.974 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.984, + "5": 0.964, + "6": 0.97, + "7": 0.87, + "8": 0.814, + "9": 0.747, + "10": 0.629 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.988, + "5": 0.984, + "6": 0.983, + "7": 0.917, + "8": 0.869, + "9": 0.816, + "10": 0.697 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996, + "5": 0.995, + "6": 0.994, + "7": 0.96, + "8": 0.933, + "9": 0.897, + "10": 0.777 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.999, + "6": 0.997, + "7": 0.986, + "8": 0.97, + "9": 0.948, + "10": 0.847 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 0.999, + "7": 0.997, + "8": 0.99, + "9": 0.981, + "10": 0.91 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.998, + "9": 0.998, + "10": 0.946 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 0.998, + "9": 1.0, + "10": 0.959 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.999, + "9": 1.0, + "10": 0.974 + } + }, + "total_pass_at_k": { + "pass@1": 0.8864444444444445, + "pass@2": 0.9171111111111111, + "pass@4": 0.9502222222222222, + "pass@8": 0.9717777777777777, + "pass@16": 0.9862222222222222, + "pass@32": 0.9934444444444445, + "pass@64": 0.9951111111111111, + "pass@128": 0.997 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + } + } +} diff --git a/composition/SDAR-100M-block8-epoch4-op9-12/summary.csv b/composition/SDAR-100M-block8-epoch4-op9-12/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..d85ae124934bcf76697f95e77d65568554189806 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op9-12/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +SDAR-100M-block8-epoch4-op9-12,0.9935295138888889,0.5771015625,0.7743569078947369,0.0,0.0,0.0,199.78453819444445,354.4255953125,1152000,1280000,2432000 diff --git a/composition/SDAR-100M-block8-epoch4-op9-12/summary.json b/composition/SDAR-100M-block8-epoch4-op9-12/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b84ff3d99850fc8ac0ff7bc73e788019a37cc795 --- /dev/null +++ b/composition/SDAR-100M-block8-epoch4-op9-12/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "SDAR-100M-block8-epoch4-op9-12", + "id_acc": 0.9935295138888889, + "ood_acc": 0.5771015625, + "total_acc": 0.7743569078947369, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.78453819444445, + "ood_avg_resp_len": 354.4255953125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 628467, + "answer_accuracy": 0.7761458168668985, + "avg_response_len": 268.07536604884604, + "resp_tokens_sum": 217068130, + "pass_at_k": { + "pass@1": 0.7761458168668985, + "pass@2": 0.8139650783480664, + "pass@4": 0.844180951782302, + "pass@8": 0.868240403688873, + "pass@16": 0.8877728581338872, + "pass@32": 0.9036649765222275, + "pass@64": 0.9170053125522537, + "pass@128": 0.9287069238065128 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9618132374631269, + "pass@2": 0.9725411121176223, + "pass@4": 0.9802102696233768, + "pass@8": 0.9849936946412662, + "pass@16": 0.9884440697041347, + "pass@32": 0.991615765648179, + "pass@64": 0.9937203955529144, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999853515625, + "pass@2": 0.999994232898622, + "pass@4": 0.9999999956060179, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9996279761904762, + "pass@2": 0.9999808617672791, + "pass@4": 0.9999999687539058, + "pass@8": 0.9999999999999978, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985965568862275, + "pass@2": 0.9993480067424206, + "pass@4": 0.9998631435105043, + "pass@8": 0.9999946418638593, + "pass@16": 0.9999999952012559, + "pass@32": 0.9999999999999998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.999791974852071, + "pass@2": 0.9999934480268369, + "pass@4": 0.999999995840017, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9912359775641025, + "pass@2": 0.9958788392136078, + "pass@4": 0.9986500876092409, + "pass@8": 0.9997715179680895, + "pass@16": 0.9999920381521239, + "pass@32": 0.9999999947732479, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9957526408450704, + "pass@2": 0.9972881085726961, + "pass@4": 0.9977522703148021, + "pass@8": 0.9982272475790946, + "pass@16": 0.998920396325759, + "pass@32": 0.9996448039393536, + "pass@64": 0.9999814875425009, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9514651203966006, + "pass@2": 0.964051451562535, + "pass@4": 0.9726533114840814, + "pass@8": 0.9784351301207374, + "pass@16": 0.9821156095662544, + "pass@32": 0.9847244967783608, + "pass@64": 0.9857797640615148, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8961864406779662, + "pass@2": 0.9169464416789005, + "pass@4": 0.930045741104395, + "pass@8": 0.9405154502869694, + "pass@16": 0.9496226622064528, + "pass@32": 0.9572528675630418, + "pass@64": 0.9642242554401338, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.8420111677115988, + "pass@2": 0.889376943820502, + "pass@4": 0.9181533575936236, + "pass@8": 0.9363334327363076, + "pass@16": 0.949785723044496, + "pass@32": 0.9610582398326716, + "pass@64": 0.9701180751617702, + "pass@128": 0.9749216300940439 + }, + "14": { + "pass@1": 0.8046388239875389, + "pass@2": 0.8602070915201021, + "pass@4": 0.8914461434143157, + "pass@8": 0.9119244313646143, + "pass@16": 0.9272654211635459, + "pass@32": 0.9383072121466731, + "pass@64": 0.9462294718077346, + "pass@128": 0.9532710280373832 + }, + "15": { + "pass@1": 0.7136244580924855, + "pass@2": 0.8005655892995307, + "pass@4": 0.8601507764022269, + "pass@8": 0.9013584538970695, + "pass@16": 0.9310673045046043, + "pass@32": 0.9517436419640584, + "pass@64": 0.9651808484660429, + "pass@128": 0.9739884393063584 + }, + "16": { + "pass@1": 0.523109879032258, + "pass@2": 0.6348286290322582, + "pass@4": 0.7166433194842577, + "pass@8": 0.7779303950852562, + "pass@16": 0.8253214388050136, + "pass@32": 0.8612953308847185, + "pass@64": 0.8875623036766475, + "pass@128": 0.9032258064516129 + }, + "17": { + "pass@1": 0.40713059971098264, + "pass@2": 0.5174622940466982, + "pass@4": 0.6078873918758712, + "pass@8": 0.6741004742749396, + "pass@16": 0.7231938373226637, + "pass@32": 0.7607190313851854, + "pass@64": 0.7937445756323053, + "pass@128": 0.8294797687861272 + }, + "18": { + "pass@1": 0.3004017857142857, + "pass@2": 0.3996291479190101, + "pass@4": 0.4907275606620602, + "pass@8": 0.5703714538130185, + "pass@16": 0.6388093892798813, + "pass@32": 0.6961804908111601, + "pass@64": 0.7459388686836753, + "pass@128": 0.7914285714285715 + }, + "19": { + "pass@1": 0.2051661036036036, + "pass@2": 0.27981980504126164, + "pass@4": 0.35245659495265824, + "pass@8": 0.4186875923466568, + "pass@16": 0.4774648463192164, + "pass@32": 0.530660247056415, + "pass@64": 0.5814726524949139, + "pass@128": 0.6306306306306306 + }, + "20": { + "pass@1": 0.19031531531531531, + "pass@2": 0.26815730592797515, + "pass@4": 0.3495825434545906, + "pass@8": 0.42705696041508395, + "pass@16": 0.4956881270636332, + "pass@32": 0.5538044007945283, + "pass@64": 0.6040435053189296, + "pass@128": 0.6486486486486487 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 638217, + "answer_accuracy": 0.7772518024162121, + "avg_response_len": 277.55285585541696, + "resp_tokens_sum": 227904201, + "pass_at_k": { + "pass@1": 0.7772518024162121, + "pass@2": 0.815610040597508, + "pass@4": 0.8465593161240585, + "pass@8": 0.8711925292394912, + "pass@16": 0.8910398237778925, + "pass@32": 0.9072447831633289, + "pass@64": 0.9203445261243395, + "pass@128": 0.9304754481683554 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9461752300613497, + "pass@2": 0.9579337954688179, + "pass@4": 0.9694229502945566, + "pass@8": 0.9810037051895499, + "pass@16": 0.9905673067988211, + "pass@32": 0.9965809863505134, + "pass@64": 0.9993884580111478, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.9999989080044729, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9990942028985508, + "pass@2": 0.9997621391076116, + "pass@4": 0.9999820552322264, + "pass@8": 0.9999999217268679, + "pass@16": 0.9999999999996003, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9980524713467048, + "pass@2": 0.9987414829772353, + "pass@4": 0.9994561608344803, + "pass@8": 0.9999035515654319, + "pass@16": 0.9999975847114193, + "pass@32": 0.9999999995144166, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999774205202312, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9965228873239437, + "pass@2": 0.9971952284573583, + "pass@4": 0.9973554949117276, + "pass@8": 0.997525506974028, + "pass@16": 0.9978457358323166, + "pass@32": 0.998419651768881, + "pass@64": 0.9993013197294001, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9928190494011976, + "pass@2": 0.9945048387477016, + "pass@4": 0.9955588426633797, + "pass@8": 0.996690741627332, + "pass@16": 0.9980681831422826, + "pass@32": 0.9992670805169979, + "pass@64": 0.9999136334847853, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9585137424698795, + "pass@2": 0.9671995066881702, + "pass@4": 0.9721630024033138, + "pass@8": 0.9750416872186753, + "pass@16": 0.9774501659354414, + "pass@32": 0.9807857011053507, + "pass@64": 0.9848766720425005, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.9050499608150471, + "pass@2": 0.923858545405179, + "pass@4": 0.9327921901611824, + "pass@8": 0.9387195559136081, + "pass@16": 0.9449860445424791, + "pass@32": 0.952132953833246, + "pass@64": 0.9586975480454887, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.8070290616246498, + "pass@2": 0.8562368352852956, + "pass@4": 0.8857674544883569, + "pass@8": 0.9058580167888862, + "pass@16": 0.9223591986829233, + "pass@32": 0.9360243594008619, + "pass@64": 0.9450110575563182, + "pass@128": 0.9495798319327731 + }, + "14": { + "pass@1": 0.7976973684210527, + "pass@2": 0.867424569461712, + "pass@4": 0.9068931860491114, + "pass@8": 0.9283267548205743, + "pass@16": 0.9418403403371376, + "pass@32": 0.9524016855817911, + "pass@64": 0.9614012630596699, + "pass@128": 0.9678362573099415 + }, + "15": { + "pass@1": 0.6974431818181818, + "pass@2": 0.7863684830589354, + "pass@4": 0.8486680735930725, + "pass@8": 0.891768548521778, + "pass@16": 0.9207707356638722, + "pass@32": 0.9400995306433965, + "pass@64": 0.9521569927151597, + "pass@128": 0.9606060606060606 + }, + "16": { + "pass@1": 0.5107505341880342, + "pass@2": 0.6242663677456984, + "pass@4": 0.7127050812024565, + "pass@8": 0.7776122661212889, + "pass@16": 0.8247779610106144, + "pass@32": 0.8566217836455938, + "pass@64": 0.8764856255133433, + "pass@128": 0.8917378917378918 + }, + "17": { + "pass@1": 0.4086034549689441, + "pass@2": 0.5166432575683475, + "pass@4": 0.6060352741233433, + "pass@8": 0.6743064342428341, + "pass@16": 0.7267822114247295, + "pass@32": 0.7705926178718194, + "pass@64": 0.8082578808157032, + "pass@128": 0.8385093167701864 + }, + "18": { + "pass@1": 0.3369255514705882, + "pass@2": 0.440790368805002, + "pass@4": 0.5347838508237939, + "pass@8": 0.6161963128406492, + "pass@16": 0.6852001345712164, + "pass@32": 0.7416791147734874, + "pass@64": 0.7868330160139833, + "pass@128": 0.8235294117647058 + }, + "19": { + "pass@1": 0.21549479166666666, + "pass@2": 0.29482081341394845, + "pass@4": 0.3771499184811719, + "pass@8": 0.45314763547808923, + "pass@16": 0.5185264418909887, + "pass@32": 0.5747617192946216, + "pass@64": 0.6211208246521323, + "pass@128": 0.6517857142857143 + }, + "20": { + "pass@1": 0.18099738023952097, + "pass@2": 0.25491352432929404, + "pass@4": 0.33130779498371077, + "pass@8": 0.4034410216749828, + "pass@16": 0.4688099342745726, + "pass@32": 0.5279449481851225, + "pass@64": 0.5843637752328384, + "pass@128": 0.6377245508982036 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 616552, + "answer_accuracy": 0.7695818022048251, + "avg_response_len": 298.12597234981627, + "resp_tokens_sum": 238844219, + "pass_at_k": { + "pass@1": 0.7695818022048251, + "pass@2": 0.8080234147866366, + "pass@4": 0.8404076771668446, + "pass@8": 0.8663769954562627, + "pass@16": 0.8862881920411901, + "pass@32": 0.901652304185975, + "pass@64": 0.9137024615526762, + "pass@128": 0.9236299728391116 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9617537313432836, + "pass@2": 0.970781672346927, + "pass@4": 0.9790400016229313, + "pass@8": 0.98626166598672, + "pass@16": 0.9905548114893244, + "pass@32": 0.9924411918583297, + "pass@64": 0.9940296282947009, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9994025735294118, + "pass@2": 0.9999677946966188, + "pass@4": 0.9999998894439663, + "pass@8": 0.9999999999996586, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9987923895899053, + "pass@2": 0.999580450955066, + "pass@4": 0.999947256691494, + "pass@8": 0.9999993061637832, + "pass@16": 0.9999999999491976, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9999752768987342, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927505630630631, + "pass@2": 0.9952442600080397, + "pass@4": 0.9971487570247412, + "pass@8": 0.9983676867389416, + "pass@16": 0.9992487477653507, + "pass@32": 0.9998506753392894, + "pass@64": 0.9999979947884918, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925391881028939, + "pass@2": 0.9944105723219485, + "pass@4": 0.9961042035139499, + "pass@8": 0.997498057805773, + "pass@16": 0.9985542991129313, + "pass@32": 0.9994502950777389, + "pass@64": 0.9999555508074901, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9620535714285714, + "pass@2": 0.9741579177602799, + "pass@4": 0.9804710363585502, + "pass@8": 0.984405510081136, + "pass@16": 0.9870437043502408, + "pass@32": 0.9885943947975548, + "pass@64": 0.9896883459298144, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.907157874617737, + "pass@2": 0.9334794661561804, + "pass@4": 0.9506061639084101, + "pass@8": 0.9616284980049788, + "pass@16": 0.9681187712875272, + "pass@32": 0.9718688464597104, + "pass@64": 0.9743606390761322, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.7944396219135802, + "pass@2": 0.8480640766258388, + "pass@4": 0.883981797992843, + "pass@8": 0.9083938480790773, + "pass@16": 0.9249280339745628, + "pass@32": 0.9374717148724703, + "pass@64": 0.9475575890272085, + "pass@128": 0.9567901234567902 + }, + "14": { + "pass@1": 0.771281528189911, + "pass@2": 0.8348344149396016, + "pass@4": 0.87485633286937, + "pass@8": 0.9029845078915546, + "pass@16": 0.9249497229959522, + "pass@32": 0.9420130044854153, + "pass@64": 0.9537250990028946, + "pass@128": 0.9614243323442137 + }, + "15": { + "pass@1": 0.6810378086419753, + "pass@2": 0.759533801764363, + "pass@4": 0.8115666965240457, + "pass@8": 0.8475286718805226, + "pass@16": 0.8743301152204945, + "pass@32": 0.8953343449790683, + "pass@64": 0.9097404523733096, + "pass@128": 0.9166666666666666 + }, + "16": { + "pass@1": 0.4934089233038348, + "pass@2": 0.6187224903026508, + "pass@4": 0.720293994500687, + "pass@8": 0.7920491057615308, + "pass@16": 0.836986988210621, + "pass@32": 0.866034039911091, + "pass@64": 0.889947041964446, + "pass@128": 0.9144542772861357 + }, + "17": { + "pass@1": 0.3518684111445783, + "pass@2": 0.45531103251589033, + "pass@4": 0.5494780251489645, + "pass@8": 0.6294890131094445, + "pass@16": 0.6943459390943068, + "pass@32": 0.7455723174815397, + "pass@64": 0.7875108943405223, + "pass@128": 0.8253012048192772 + }, + "18": { + "pass@1": 0.3236139112903226, + "pass@2": 0.41808364554229105, + "pass@4": 0.5064354890114543, + "pass@8": 0.5804429288858804, + "pass@16": 0.6393856336344137, + "pass@32": 0.6878125541269339, + "pass@64": 0.7277094054798247, + "pass@128": 0.7612903225806451 + }, + "19": { + "pass@1": 0.20324301359516617, + "pass@2": 0.28858765789661495, + "pass@4": 0.3778401204381177, + "pass@8": 0.4559778835038485, + "pass@16": 0.5186360320155117, + "pass@32": 0.5718077443113075, + "pass@64": 0.6175381100536046, + "pass@128": 0.6525679758308157 + }, + "20": { + "pass@1": 0.17600412912912913, + "pass@2": 0.2502268508666147, + "pass@4": 0.3296207996522957, + "pass@8": 0.40676253450542404, + "pass@16": 0.4739966854242499, + "pass@32": 0.5258046811806326, + "pass@64": 0.5621618523024845, + "pass@128": 0.5915915915915916 + } + } + } + } + } +] \ No newline at end of file