{ "chosen_c11": { "pooled": { "B_correct": 196, "B_strict_parse": 253, "B_schema": 218, "B_fence": 0, "B_start_brace": 129, "B_start_bracket": 127, "B_start_fence": 0, "B_object_exact": 99, "B_array_exact": 97, "B_array_wrong_container_start": 1, "B_object_wrong_container_start": 0, "A_correct": 35, "C_correct": 57, "S_correct": 64, "F_correct": 48, "F_balanced": 48, "length_failures": 0 }, "greedy_B": 49, "ppl_rel": -0.0007307205221399877, "kl_first_token": 0.42265161871910095, "kl_passage": 0.0014987997710704803, "card_B": 50, "B_pooled": { "n": 256, "base": 167, "edit": 196, "improved": 29, "broken": 0, "mcnemar_exact_p": 3.725290298461914e-09, "base_ci": [ 0.5921427947221091, 0.7080400859251775 ], "edit_ci": [ 0.7100415017359011, 0.8133542901619322 ] }, "B_greedy": { "n": 64, "base": 42, "edit": 49, "improved": 7, "broken": 0, "mcnemar_exact_p": 0.015625, "base_ci": [ 0.5340341181762039, 0.7607702496453097 ], "edit_ci": [ 0.6486650974929234, 0.8525023278036498 ] }, "B_card": { "n": 64, "base": 42, "edit": 50, "improved": 8, "broken": 0, "mcnemar_exact_p": 0.0078125, "base_ci": [ 0.5340341181762039, 0.7607702496453097 ], "edit_ci": [ 0.6656698366922221, 0.8649780253865025 ] }, "B_array_pooled": { "n": 128, "base": 76, "edit": 97, "improved": 21, "broken": 0, "mcnemar_exact_p": 9.5367431640625e-07, "base_ci": [ 0.5071382471838717, 0.674898378585233 ], "edit_ci": [ 0.6767870437261758, 0.8238136771388622 ] }, "B_object_pooled": { "n": 128, "base": 91, "edit": 99, "improved": 8, "broken": 0, "mcnemar_exact_p": 0.0078125, "base_ci": [ 0.6271652314384711, 0.7824171765420145 ], "edit_ci": [ 0.6935716289452746, 0.8373685295479476 ] }, "B_nested_pooled": { "n": 80, "base": 50, "edit": 63, "improved": 13, "broken": 0, "mcnemar_exact_p": 0.000244140625, "base_ci": [ 0.515485228513904, 0.7230598374199523 ], "edit_ci": [ 0.6857763938182242, 0.8628772578296451 ] }, "B_nonascii_pooled": { "n": 72, "base": 65, "edit": 71, "improved": 6, "broken": 0, "mcnemar_exact_p": 0.03125, "base_ci": [ 0.812647509746432, 0.952104288738832 ], "edit_ci": [ 0.9254321971571302, 0.9975441113595678 ] }, "B_strict_parse_pooled": { "n": 256, "base": 211, "edit": 253, "improved": 42, "broken": 0, "mcnemar_exact_p": 4.547473508864641e-13, "base_ci": [ 0.772896116682119, 0.8659546293108245 ], "edit_ci": [ 0.9661178322810585, 0.9960067851781936 ] }, "B_schema_pooled": { "n": 256, "base": 186, "edit": 218, "improved": 32, "broken": 0, "mcnemar_exact_p": 4.656612873077393e-10, "base_ci": [ 0.6689134962060558, 0.7775123262950373 ], "edit_ci": [ 0.8028238602392018, 0.8899058643314598 ] }, "A": { "n": 64, "base": 36, "edit": 35, "improved": 0, "broken": 1, "mcnemar_exact_p": 1.0, "base_ci": [ 0.44085991778287875, 0.6770618293457267 ], "edit_ci": [ 0.4257338252067978, 0.6627074851396563 ] }, "C": { "n": 64, "base": 57, "edit": 57, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.7910113670806147, 0.9459995524731696 ], "edit_ci": [ 0.7910113670806147, 0.9459995524731696 ] }, "S": { "n": 64, "base": 64, "edit": 64, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.9433739770288436, 1.0 ], "edit_ci": [ 0.9433739770288436, 1.0 ] }, "F": { "n": 48, "base": 48, "edit": 48, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.9258973488472576, 0.9999999999999998 ], "edit_ci": [ 0.9258973488472576, 0.9999999999999998 ] } }, "sham": { "pooled": { "B_correct": 167, "B_strict_parse": 211, "B_schema": 186, "B_fence": 41, "B_start_brace": 119, "B_start_bracket": 96, "B_start_fence": 41, "B_object_exact": 91, "B_array_exact": 76, "B_array_wrong_container_start": 0, "B_object_wrong_container_start": 0, "A_correct": 36, "C_correct": 57, "S_correct": 64, "F_correct": 48, "F_balanced": 47, "length_failures": 0 }, "greedy_B": 42, "ppl_rel": 0.0, "kl_first_token": 0.0, "kl_passage": 0.0, "card_B": 42, "B_pooled": { "n": 256, "base": 167, "edit": 167, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.5921427947221091, 0.7080400859251775 ], "edit_ci": [ 0.5921427947221091, 0.7080400859251775 ] }, "B_greedy": { "n": 64, "base": 42, "edit": 42, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.5340341181762039, 0.7607702496453097 ], "edit_ci": [ 0.5340341181762039, 0.7607702496453097 ] }, "B_card": { "n": 64, "base": 42, "edit": 42, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.5340341181762039, 0.7607702496453097 ], "edit_ci": [ 0.5340341181762039, 0.7607702496453097 ] }, "B_array_pooled": { "n": 128, "base": 76, "edit": 76, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.5071382471838717, 0.674898378585233 ], "edit_ci": [ 0.5071382471838717, 0.674898378585233 ] }, "B_object_pooled": { "n": 128, "base": 91, "edit": 91, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.6271652314384711, 0.7824171765420145 ], "edit_ci": [ 0.6271652314384711, 0.7824171765420145 ] }, "B_nested_pooled": { "n": 80, "base": 50, "edit": 50, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.515485228513904, 0.7230598374199523 ], "edit_ci": [ 0.515485228513904, 0.7230598374199523 ] }, "B_nonascii_pooled": { "n": 72, "base": 65, "edit": 65, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.812647509746432, 0.952104288738832 ], "edit_ci": [ 0.812647509746432, 0.952104288738832 ] }, "B_strict_parse_pooled": { "n": 256, "base": 211, "edit": 211, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.772896116682119, 0.8659546293108245 ], "edit_ci": [ 0.772896116682119, 0.8659546293108245 ] }, "B_schema_pooled": { "n": 256, "base": 186, "edit": 186, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.6689134962060558, 0.7775123262950373 ], "edit_ci": [ 0.6689134962060558, 0.7775123262950373 ] }, "A": { "n": 64, "base": 36, "edit": 36, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.44085991778287875, 0.6770618293457267 ], "edit_ci": [ 0.44085991778287875, 0.6770618293457267 ] }, "C": { "n": 64, "base": 57, "edit": 57, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.7910113670806147, 0.9459995524731696 ], "edit_ci": [ 0.7910113670806147, 0.9459995524731696 ] }, "S": { "n": 64, "base": 64, "edit": 64, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.9433739770288436, 1.0 ], "edit_ci": [ 0.9433739770288436, 1.0 ] }, "F": { "n": 48, "base": 48, "edit": 48, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.9258973488472576, 0.9999999999999998 ], "edit_ci": [ 0.9258973488472576, 0.9999999999999998 ] } }, "c11_rnd": { "pooled": { "B_correct": 167, "B_strict_parse": 212, "B_schema": 187, "B_fence": 40, "B_start_brace": 119, "B_start_bracket": 97, "B_start_fence": 40, "B_object_exact": 91, "B_array_exact": 76, "B_array_wrong_container_start": 0, "B_object_wrong_container_start": 0, "A_correct": 36, "C_correct": 56, "S_correct": 64, "F_correct": 48, "F_balanced": 47, "length_failures": 0 }, "greedy_B": 42, "ppl_rel": 0.002196327451208909, "kl_first_token": 0.0005274508730508387, "kl_passage": 0.0006625415990129113, "card_B": 42, "B_pooled": { "n": 256, "base": 167, "edit": 167, "improved": 1, "broken": 1, "mcnemar_exact_p": 1.0, "base_ci": [ 0.5921427947221091, 0.7080400859251775 ], "edit_ci": [ 0.5921427947221091, 0.7080400859251775 ] }, "B_greedy": { "n": 64, "base": 42, "edit": 42, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.5340341181762039, 0.7607702496453097 ], "edit_ci": [ 0.5340341181762039, 0.7607702496453097 ] }, "B_card": { "n": 64, "base": 42, "edit": 42, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.5340341181762039, 0.7607702496453097 ], "edit_ci": [ 0.5340341181762039, 0.7607702496453097 ] }, "B_array_pooled": { "n": 128, "base": 76, "edit": 76, "improved": 1, "broken": 1, "mcnemar_exact_p": 1.0, "base_ci": [ 0.5071382471838717, 0.674898378585233 ], "edit_ci": [ 0.5071382471838717, 0.674898378585233 ] }, "B_object_pooled": { "n": 128, "base": 91, "edit": 91, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.6271652314384711, 0.7824171765420145 ], "edit_ci": [ 0.6271652314384711, 0.7824171765420145 ] }, "B_nested_pooled": { "n": 80, "base": 50, "edit": 49, "improved": 0, "broken": 1, "mcnemar_exact_p": 1.0, "base_ci": [ 0.515485228513904, 0.7230598374199523 ], "edit_ci": [ 0.5029345562232477, 0.711756003117223 ] }, "B_nonascii_pooled": { "n": 72, "base": 65, "edit": 65, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.812647509746432, 0.952104288738832 ], "edit_ci": [ 0.812647509746432, 0.952104288738832 ] }, "B_strict_parse_pooled": { "n": 256, "base": 211, "edit": 212, "improved": 1, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.772896116682119, 0.8659546293108245 ], "edit_ci": [ 0.7771451946116188, 0.8694025483209986 ] }, "B_schema_pooled": { "n": 256, "base": 186, "edit": 187, "improved": 2, "broken": 1, "mcnemar_exact_p": 1.0, "base_ci": [ 0.6689134962060558, 0.7775123262950373 ], "edit_ci": [ 0.6730019753620341, 0.781120844078733 ] }, "A": { "n": 64, "base": 36, "edit": 36, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.44085991778287875, 0.6770618293457267 ], "edit_ci": [ 0.44085991778287875, 0.6770618293457267 ] }, "C": { "n": 64, "base": 57, "edit": 56, "improved": 0, "broken": 1, "mcnemar_exact_p": 1.0, "base_ci": [ 0.7910113670806147, 0.9459995524731696 ], "edit_ci": [ 0.7722521366978408, 0.935278346073792 ] }, "S": { "n": 64, "base": 64, "edit": 64, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.9433739770288436, 1.0 ], "edit_ci": [ 0.9433739770288436, 1.0 ] }, "F": { "n": 48, "base": 48, "edit": 48, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.9258973488472576, 0.9999999999999998 ], "edit_ci": [ 0.9258973488472576, 0.9999999999999998 ] } }, "c11_scd": { "pooled": { "B_correct": 166, "B_strict_parse": 211, "B_schema": 185, "B_fence": 40, "B_start_brace": 120, "B_start_bracket": 96, "B_start_fence": 40, "B_object_exact": 91, "B_array_exact": 75, "B_array_wrong_container_start": 0, "B_object_wrong_container_start": 0, "A_correct": 36, "C_correct": 56, "S_correct": 64, "F_correct": 48, "F_balanced": 47, "length_failures": 0 }, "greedy_B": 42, "ppl_rel": 0.0007641077187694739, "kl_first_token": 0.0037935180589556694, "kl_passage": 0.0009359674295410514, "card_B": 42, "B_pooled": { "n": 256, "base": 167, "edit": 166, "improved": 0, "broken": 1, "mcnemar_exact_p": 1.0, "base_ci": [ 0.5921427947221091, 0.7080400859251775 ], "edit_ci": [ 0.5881468207176852, 0.7043390629899275 ] }, "B_greedy": { "n": 64, "base": 42, "edit": 42, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.5340341181762039, 0.7607702496453097 ], "edit_ci": [ 0.5340341181762039, 0.7607702496453097 ] }, "B_card": { "n": 64, "base": 42, "edit": 42, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.5340341181762039, 0.7607702496453097 ], "edit_ci": [ 0.5340341181762039, 0.7607702496453097 ] }, "B_array_pooled": { "n": 128, "base": 76, "edit": 75, "improved": 0, "broken": 1, "mcnemar_exact_p": 1.0, "base_ci": [ 0.5071382471838717, 0.674898378585233 ], "edit_ci": [ 0.4993170001197534, 0.6675499068352593 ] }, "B_object_pooled": { "n": 128, "base": 91, "edit": 91, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.6271652314384711, 0.7824171765420145 ], "edit_ci": [ 0.6271652314384711, 0.7824171765420145 ] }, "B_nested_pooled": { "n": 80, "base": 50, "edit": 50, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.515485228513904, 0.7230598374199523 ], "edit_ci": [ 0.515485228513904, 0.7230598374199523 ] }, "B_nonascii_pooled": { "n": 72, "base": 65, "edit": 64, "improved": 0, "broken": 1, "mcnemar_exact_p": 1.0, "base_ci": [ 0.812647509746432, 0.952104288738832 ], "edit_ci": [ 0.7957685574140959, 0.9426124893992622 ] }, "B_strict_parse_pooled": { "n": 256, "base": 211, "edit": 211, "improved": 1, "broken": 1, "mcnemar_exact_p": 1.0, "base_ci": [ 0.772896116682119, 0.8659546293108245 ], "edit_ci": [ 0.772896116682119, 0.8659546293108245 ] }, "B_schema_pooled": { "n": 256, "base": 186, "edit": 185, "improved": 0, "broken": 1, "mcnemar_exact_p": 1.0, "base_ci": [ 0.6689134962060558, 0.7775123262950373 ], "edit_ci": [ 0.6648301483971852, 0.7738986771642338 ] }, "A": { "n": 64, "base": 36, "edit": 36, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.44085991778287875, 0.6770618293457267 ], "edit_ci": [ 0.44085991778287875, 0.6770618293457267 ] }, "C": { "n": 64, "base": 57, "edit": 56, "improved": 0, "broken": 1, "mcnemar_exact_p": 1.0, "base_ci": [ 0.7910113670806147, 0.9459995524731696 ], "edit_ci": [ 0.7722521366978408, 0.935278346073792 ] }, "S": { "n": 64, "base": 64, "edit": 64, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.9433739770288436, 1.0 ], "edit_ci": [ 0.9433739770288436, 1.0 ] }, "F": { "n": 48, "base": 48, "edit": 48, "improved": 0, "broken": 0, "mcnemar_exact_p": 1.0, "base_ci": [ 0.9258973488472576, 0.9999999999999998 ], "edit_ci": [ 0.9258973488472576, 0.9999999999999998 ] } }, "base": { "pooled": { "B_correct": 167, "B_strict_parse": 211, "B_schema": 186, "B_fence": 41, "B_start_brace": 119, "B_start_bracket": 96, "B_start_fence": 41, "B_object_exact": 91, "B_array_exact": 76, "B_array_wrong_container_start": 0, "B_object_wrong_container_start": 0, "A_correct": 36, "C_correct": 57, "S_correct": 64, "F_correct": 48, "F_balanced": 47, "length_failures": 0 }, "greedy_B": 42, "ppl": 130.5471182180624, "card_B": 42 } }