yasserrmd's picture
WSP 2.0 sparse weight edit: strict-JSON fence repair (32,768 BF16 values, 16 w2 columns)
1a2bb00 verified
Raw History Blame Contribute Delete
16.5 kB
{
"chosen_c11": {
"pooled": {
"B_correct": 196,
"B_strict_parse": 253,
"B_schema": 218,
"B_fence": 0,
"B_start_brace": 129,
"B_start_bracket": 127,
"B_start_fence": 0,
"B_object_exact": 99,
"B_array_exact": 97,
"B_array_wrong_container_start": 1,
"B_object_wrong_container_start": 0,
"A_correct": 35,
"C_correct": 57,
"S_correct": 64,
"F_correct": 48,
"F_balanced": 48,
"length_failures": 0
},
"greedy_B": 49,
"ppl_rel": -0.0007307205221399877,
"kl_first_token": 0.42265161871910095,
"kl_passage": 0.0014987997710704803,
"card_B": 50,
"B_pooled": {
"n": 256,
"base": 167,
"edit": 196,
"improved": 29,
"broken": 0,
"mcnemar_exact_p": 3.725290298461914e-09,
"base_ci": [
0.5921427947221091,
0.7080400859251775
],
"edit_ci": [
0.7100415017359011,
0.8133542901619322
]
},
"B_greedy": {
"n": 64,
"base": 42,
"edit": 49,
"improved": 7,
"broken": 0,
"mcnemar_exact_p": 0.015625,
"base_ci": [
0.5340341181762039,
0.7607702496453097
],
"edit_ci": [
0.6486650974929234,
0.8525023278036498
]
},
"B_card": {
"n": 64,
"base": 42,
"edit": 50,
"improved": 8,
"broken": 0,
"mcnemar_exact_p": 0.0078125,
"base_ci": [
0.5340341181762039,
0.7607702496453097
],
"edit_ci": [
0.6656698366922221,
0.8649780253865025
]
},
"B_array_pooled": {
"n": 128,
"base": 76,
"edit": 97,
"improved": 21,
"broken": 0,
"mcnemar_exact_p": 9.5367431640625e-07,
"base_ci": [
0.5071382471838717,
0.674898378585233
],
"edit_ci": [
0.6767870437261758,
0.8238136771388622
]
},
"B_object_pooled": {
"n": 128,
"base": 91,
"edit": 99,
"improved": 8,
"broken": 0,
"mcnemar_exact_p": 0.0078125,
"base_ci": [
0.6271652314384711,
0.7824171765420145
],
"edit_ci": [
0.6935716289452746,
0.8373685295479476
]
},
"B_nested_pooled": {
"n": 80,
"base": 50,
"edit": 63,
"improved": 13,
"broken": 0,
"mcnemar_exact_p": 0.000244140625,
"base_ci": [
0.515485228513904,
0.7230598374199523
],
"edit_ci": [
0.6857763938182242,
0.8628772578296451
]
},
"B_nonascii_pooled": {
"n": 72,
"base": 65,
"edit": 71,
"improved": 6,
"broken": 0,
"mcnemar_exact_p": 0.03125,
"base_ci": [
0.812647509746432,
0.952104288738832
],
"edit_ci": [
0.9254321971571302,
0.9975441113595678
]
},
"B_strict_parse_pooled": {
"n": 256,
"base": 211,
"edit": 253,
"improved": 42,
"broken": 0,
"mcnemar_exact_p": 4.547473508864641e-13,
"base_ci": [
0.772896116682119,
0.8659546293108245
],
"edit_ci": [
0.9661178322810585,
0.9960067851781936
]
},
"B_schema_pooled": {
"n": 256,
"base": 186,
"edit": 218,
"improved": 32,
"broken": 0,
"mcnemar_exact_p": 4.656612873077393e-10,
"base_ci": [
0.6689134962060558,
0.7775123262950373
],
"edit_ci": [
0.8028238602392018,
0.8899058643314598
]
},
"A": {
"n": 64,
"base": 36,
"edit": 35,
"improved": 0,
"broken": 1,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.44085991778287875,
0.6770618293457267
],
"edit_ci": [
0.4257338252067978,
0.6627074851396563
]
},
"C": {
"n": 64,
"base": 57,
"edit": 57,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.7910113670806147,
0.9459995524731696
],
"edit_ci": [
0.7910113670806147,
0.9459995524731696
]
},
"S": {
"n": 64,
"base": 64,
"edit": 64,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.9433739770288436,
1.0
],
"edit_ci": [
0.9433739770288436,
1.0
]
},
"F": {
"n": 48,
"base": 48,
"edit": 48,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.9258973488472576,
0.9999999999999998
],
"edit_ci": [
0.9258973488472576,
0.9999999999999998
]
}
},
"sham": {
"pooled": {
"B_correct": 167,
"B_strict_parse": 211,
"B_schema": 186,
"B_fence": 41,
"B_start_brace": 119,
"B_start_bracket": 96,
"B_start_fence": 41,
"B_object_exact": 91,
"B_array_exact": 76,
"B_array_wrong_container_start": 0,
"B_object_wrong_container_start": 0,
"A_correct": 36,
"C_correct": 57,
"S_correct": 64,
"F_correct": 48,
"F_balanced": 47,
"length_failures": 0
},
"greedy_B": 42,
"ppl_rel": 0.0,
"kl_first_token": 0.0,
"kl_passage": 0.0,
"card_B": 42,
"B_pooled": {
"n": 256,
"base": 167,
"edit": 167,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.5921427947221091,
0.7080400859251775
],
"edit_ci": [
0.5921427947221091,
0.7080400859251775
]
},
"B_greedy": {
"n": 64,
"base": 42,
"edit": 42,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.5340341181762039,
0.7607702496453097
],
"edit_ci": [
0.5340341181762039,
0.7607702496453097
]
},
"B_card": {
"n": 64,
"base": 42,
"edit": 42,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.5340341181762039,
0.7607702496453097
],
"edit_ci": [
0.5340341181762039,
0.7607702496453097
]
},
"B_array_pooled": {
"n": 128,
"base": 76,
"edit": 76,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.5071382471838717,
0.674898378585233
],
"edit_ci": [
0.5071382471838717,
0.674898378585233
]
},
"B_object_pooled": {
"n": 128,
"base": 91,
"edit": 91,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.6271652314384711,
0.7824171765420145
],
"edit_ci": [
0.6271652314384711,
0.7824171765420145
]
},
"B_nested_pooled": {
"n": 80,
"base": 50,
"edit": 50,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.515485228513904,
0.7230598374199523
],
"edit_ci": [
0.515485228513904,
0.7230598374199523
]
},
"B_nonascii_pooled": {
"n": 72,
"base": 65,
"edit": 65,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.812647509746432,
0.952104288738832
],
"edit_ci": [
0.812647509746432,
0.952104288738832
]
},
"B_strict_parse_pooled": {
"n": 256,
"base": 211,
"edit": 211,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.772896116682119,
0.8659546293108245
],
"edit_ci": [
0.772896116682119,
0.8659546293108245
]
},
"B_schema_pooled": {
"n": 256,
"base": 186,
"edit": 186,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.6689134962060558,
0.7775123262950373
],
"edit_ci": [
0.6689134962060558,
0.7775123262950373
]
},
"A": {
"n": 64,
"base": 36,
"edit": 36,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.44085991778287875,
0.6770618293457267
],
"edit_ci": [
0.44085991778287875,
0.6770618293457267
]
},
"C": {
"n": 64,
"base": 57,
"edit": 57,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.7910113670806147,
0.9459995524731696
],
"edit_ci": [
0.7910113670806147,
0.9459995524731696
]
},
"S": {
"n": 64,
"base": 64,
"edit": 64,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.9433739770288436,
1.0
],
"edit_ci": [
0.9433739770288436,
1.0
]
},
"F": {
"n": 48,
"base": 48,
"edit": 48,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.9258973488472576,
0.9999999999999998
],
"edit_ci": [
0.9258973488472576,
0.9999999999999998
]
}
},
"c11_rnd": {
"pooled": {
"B_correct": 167,
"B_strict_parse": 212,
"B_schema": 187,
"B_fence": 40,
"B_start_brace": 119,
"B_start_bracket": 97,
"B_start_fence": 40,
"B_object_exact": 91,
"B_array_exact": 76,
"B_array_wrong_container_start": 0,
"B_object_wrong_container_start": 0,
"A_correct": 36,
"C_correct": 56,
"S_correct": 64,
"F_correct": 48,
"F_balanced": 47,
"length_failures": 0
},
"greedy_B": 42,
"ppl_rel": 0.002196327451208909,
"kl_first_token": 0.0005274508730508387,
"kl_passage": 0.0006625415990129113,
"card_B": 42,
"B_pooled": {
"n": 256,
"base": 167,
"edit": 167,
"improved": 1,
"broken": 1,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.5921427947221091,
0.7080400859251775
],
"edit_ci": [
0.5921427947221091,
0.7080400859251775
]
},
"B_greedy": {
"n": 64,
"base": 42,
"edit": 42,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.5340341181762039,
0.7607702496453097
],
"edit_ci": [
0.5340341181762039,
0.7607702496453097
]
},
"B_card": {
"n": 64,
"base": 42,
"edit": 42,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.5340341181762039,
0.7607702496453097
],
"edit_ci": [
0.5340341181762039,
0.7607702496453097
]
},
"B_array_pooled": {
"n": 128,
"base": 76,
"edit": 76,
"improved": 1,
"broken": 1,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.5071382471838717,
0.674898378585233
],
"edit_ci": [
0.5071382471838717,
0.674898378585233
]
},
"B_object_pooled": {
"n": 128,
"base": 91,
"edit": 91,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.6271652314384711,
0.7824171765420145
],
"edit_ci": [
0.6271652314384711,
0.7824171765420145
]
},
"B_nested_pooled": {
"n": 80,
"base": 50,
"edit": 49,
"improved": 0,
"broken": 1,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.515485228513904,
0.7230598374199523
],
"edit_ci": [
0.5029345562232477,
0.711756003117223
]
},
"B_nonascii_pooled": {
"n": 72,
"base": 65,
"edit": 65,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.812647509746432,
0.952104288738832
],
"edit_ci": [
0.812647509746432,
0.952104288738832
]
},
"B_strict_parse_pooled": {
"n": 256,
"base": 211,
"edit": 212,
"improved": 1,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.772896116682119,
0.8659546293108245
],
"edit_ci": [
0.7771451946116188,
0.8694025483209986
]
},
"B_schema_pooled": {
"n": 256,
"base": 186,
"edit": 187,
"improved": 2,
"broken": 1,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.6689134962060558,
0.7775123262950373
],
"edit_ci": [
0.6730019753620341,
0.781120844078733
]
},
"A": {
"n": 64,
"base": 36,
"edit": 36,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.44085991778287875,
0.6770618293457267
],
"edit_ci": [
0.44085991778287875,
0.6770618293457267
]
},
"C": {
"n": 64,
"base": 57,
"edit": 56,
"improved": 0,
"broken": 1,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.7910113670806147,
0.9459995524731696
],
"edit_ci": [
0.7722521366978408,
0.935278346073792
]
},
"S": {
"n": 64,
"base": 64,
"edit": 64,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.9433739770288436,
1.0
],
"edit_ci": [
0.9433739770288436,
1.0
]
},
"F": {
"n": 48,
"base": 48,
"edit": 48,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.9258973488472576,
0.9999999999999998
],
"edit_ci": [
0.9258973488472576,
0.9999999999999998
]
}
},
"c11_scd": {
"pooled": {
"B_correct": 166,
"B_strict_parse": 211,
"B_schema": 185,
"B_fence": 40,
"B_start_brace": 120,
"B_start_bracket": 96,
"B_start_fence": 40,
"B_object_exact": 91,
"B_array_exact": 75,
"B_array_wrong_container_start": 0,
"B_object_wrong_container_start": 0,
"A_correct": 36,
"C_correct": 56,
"S_correct": 64,
"F_correct": 48,
"F_balanced": 47,
"length_failures": 0
},
"greedy_B": 42,
"ppl_rel": 0.0007641077187694739,
"kl_first_token": 0.0037935180589556694,
"kl_passage": 0.0009359674295410514,
"card_B": 42,
"B_pooled": {
"n": 256,
"base": 167,
"edit": 166,
"improved": 0,
"broken": 1,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.5921427947221091,
0.7080400859251775
],
"edit_ci": [
0.5881468207176852,
0.7043390629899275
]
},
"B_greedy": {
"n": 64,
"base": 42,
"edit": 42,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.5340341181762039,
0.7607702496453097
],
"edit_ci": [
0.5340341181762039,
0.7607702496453097
]
},
"B_card": {
"n": 64,
"base": 42,
"edit": 42,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.5340341181762039,
0.7607702496453097
],
"edit_ci": [
0.5340341181762039,
0.7607702496453097
]
},
"B_array_pooled": {
"n": 128,
"base": 76,
"edit": 75,
"improved": 0,
"broken": 1,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.5071382471838717,
0.674898378585233
],
"edit_ci": [
0.4993170001197534,
0.6675499068352593
]
},
"B_object_pooled": {
"n": 128,
"base": 91,
"edit": 91,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.6271652314384711,
0.7824171765420145
],
"edit_ci": [
0.6271652314384711,
0.7824171765420145
]
},
"B_nested_pooled": {
"n": 80,
"base": 50,
"edit": 50,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.515485228513904,
0.7230598374199523
],
"edit_ci": [
0.515485228513904,
0.7230598374199523
]
},
"B_nonascii_pooled": {
"n": 72,
"base": 65,
"edit": 64,
"improved": 0,
"broken": 1,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.812647509746432,
0.952104288738832
],
"edit_ci": [
0.7957685574140959,
0.9426124893992622
]
},
"B_strict_parse_pooled": {
"n": 256,
"base": 211,
"edit": 211,
"improved": 1,
"broken": 1,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.772896116682119,
0.8659546293108245
],
"edit_ci": [
0.772896116682119,
0.8659546293108245
]
},
"B_schema_pooled": {
"n": 256,
"base": 186,
"edit": 185,
"improved": 0,
"broken": 1,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.6689134962060558,
0.7775123262950373
],
"edit_ci": [
0.6648301483971852,
0.7738986771642338
]
},
"A": {
"n": 64,
"base": 36,
"edit": 36,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.44085991778287875,
0.6770618293457267
],
"edit_ci": [
0.44085991778287875,
0.6770618293457267
]
},
"C": {
"n": 64,
"base": 57,
"edit": 56,
"improved": 0,
"broken": 1,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.7910113670806147,
0.9459995524731696
],
"edit_ci": [
0.7722521366978408,
0.935278346073792
]
},
"S": {
"n": 64,
"base": 64,
"edit": 64,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.9433739770288436,
1.0
],
"edit_ci": [
0.9433739770288436,
1.0
]
},
"F": {
"n": 48,
"base": 48,
"edit": 48,
"improved": 0,
"broken": 0,
"mcnemar_exact_p": 1.0,
"base_ci": [
0.9258973488472576,
0.9999999999999998
],
"edit_ci": [
0.9258973488472576,
0.9999999999999998
]
}
},
"base": {
"pooled": {
"B_correct": 167,
"B_strict_parse": 211,
"B_schema": 186,
"B_fence": 41,
"B_start_brace": 119,
"B_start_bracket": 96,
"B_start_fence": 41,
"B_object_exact": 91,
"B_array_exact": 76,
"B_array_wrong_container_start": 0,
"B_object_wrong_container_start": 0,
"A_correct": 36,
"C_correct": 57,
"S_correct": 64,
"F_correct": 48,
"F_balanced": 47,
"length_failures": 0
},
"greedy_B": 42,
"ppl": 130.5471182180624,
"card_B": 42
}
}