| { | |
| "parse_bench_version": "1.0.4", | |
| "dataset": "llamaindex/ParseBench (full: 2,078 pages, 1,211 documents, 169,011 rules)", | |
| "hardware": "4-core CPU, no GPU", | |
| "final": { | |
| "Tables": { | |
| "score": 42.09, | |
| "documents": 503, | |
| "failed": 0 | |
| }, | |
| "Charts": { | |
| "score": 0.98, | |
| "documents": 568, | |
| "failed": 0 | |
| }, | |
| "Content_Faithfulness": { | |
| "score": 72.5, | |
| "documents": 506, | |
| "failed": 0 | |
| }, | |
| "Semantic_Formatting": { | |
| "score": 35.48, | |
| "documents": 476, | |
| "failed": 0 | |
| }, | |
| "Visual_Grounding": { | |
| "score": 29.95, | |
| "documents": 500, | |
| "failed": 0 | |
| }, | |
| "Overall": 36.2 | |
| }, | |
| "first_run": { | |
| "Tables": { | |
| "score": 42.34, | |
| "documents": 503, | |
| "failed": 0 | |
| }, | |
| "Charts": { | |
| "score": 1.19, | |
| "documents": 568, | |
| "failed": 0 | |
| }, | |
| "Content_Faithfulness": { | |
| "score": 71.82, | |
| "documents": 506, | |
| "failed": 0 | |
| }, | |
| "Semantic_Formatting": { | |
| "score": 24.04, | |
| "documents": 476, | |
| "failed": 0 | |
| }, | |
| "Visual_Grounding": { | |
| "score": 28.93, | |
| "documents": 500, | |
| "failed": 1 | |
| }, | |
| "Overall": 33.66 | |
| }, | |
| "dimensions": { | |
| "chart": { | |
| "total_examples": 568, | |
| "successful": 568, | |
| "failed": 0, | |
| "aggregate_metrics": { | |
| "avg_rule_pass_rate": 0.009846579476861166, | |
| "avg_rule_chart_data_point_pass_rate": 0.009846579476861166 | |
| }, | |
| "tag_metrics": { | |
| "3d_chart": { | |
| "avg_rule_pass_rate": 0.0014084507042253522, | |
| "avg_rule_chart_data_point_pass_rate": 0.0014084507042253522 | |
| }, | |
| "chart": { | |
| "avg_rule_pass_rate": 0.009846579476861166, | |
| "avg_rule_chart_data_point_pass_rate": 0.009846579476861166 | |
| }, | |
| "need_estimate": { | |
| "avg_rule_pass_rate": 0.0018567639257294427, | |
| "avg_rule_chart_data_point_pass_rate": 0.0018567639257294427 | |
| } | |
| } | |
| }, | |
| "layout": { | |
| "total_examples": 500, | |
| "successful": 500, | |
| "failed": 0, | |
| "aggregate_metrics": { | |
| "avg_mAP@[.50:.95]": 0.2071643111595238, | |
| "avg_AP50": 0.38237206250169886, | |
| "avg_AP75": 0.16831471074679888, | |
| "avg_f1_Page-footer": 0.37965536409116174, | |
| "avg_precision_Page-footer": 0.3867704280155642, | |
| "avg_recall_Page-footer": 0.39169909208819714, | |
| "avg_f1_Page-header": 0.36862698412698414, | |
| "avg_precision_Page-header": 0.3933333333333333, | |
| "avg_recall_Page-header": 0.3651556776556777, | |
| "avg_f1_Picture": 0.37582007377665133, | |
| "avg_precision_Picture": 0.45349541440993646, | |
| "avg_recall_Picture": 0.3658841951915754, | |
| "avg_f1_Text": 0.3157437092096685, | |
| "avg_precision_Text": 0.39198181489891953, | |
| "avg_recall_Text": 0.32124203327004147, | |
| "avg_mean_f1": 0.32707792561321625, | |
| "avg_num_predictions": 24.112385321100916, | |
| "avg_num_ground_truth": 32.60550458715596, | |
| "avg_layout_localization_pass_rate": 0.49324317217432195, | |
| "avg_layout_classification_pass_rate": 0.3754582392771476, | |
| "avg_layout_attribution_pass_rate": 0.5095168575223011, | |
| "avg_layout_reading_order_pass_rate": 0.6443816756841821, | |
| "avg_layout_rule_pass_rate": 0.4567376542932833, | |
| "avg_rule_pass_rate": 0.5047324703337827, | |
| "avg_layout_element_rule_pass_rate": 0.2994773497670555, | |
| "avg_lap": 0.8445318397975906, | |
| "avg_lar": 0.8817410426743731, | |
| "avg_af1": 0.8520308688210423, | |
| "avg_unmatched_gt_elements": 19.05045871559633, | |
| "avg_unmatched_pred_elements": 11.337155963302752, | |
| "avg_f1_Section": 0.2744873389621379, | |
| "avg_precision_Section": 0.3739662959731198, | |
| "avg_recall_Section": 0.29901523775819305, | |
| "avg_f1_Table": 0.25928502501083145, | |
| "avg_precision_Table": 0.22753677773839065, | |
| "avg_recall_Table": 0.37231182795698925, | |
| "avg_rule_order_pass_rate": 0.8316971546096849, | |
| "avg_normalized_order": 0.8316971546096849, | |
| "avg_normalized_text_score": 0.8316971546096849, | |
| "avg_content_faithfulness": 0.8316971546096849 | |
| }, | |
| "tag_metrics": { | |
| "easy": { | |
| "avg_mAP@[.50:.95]": 0.21530222613390979, | |
| "avg_AP50": 0.38002565898558743, | |
| "avg_AP75": 0.17616141653732398, | |
| "avg_f1_Page-footer": 0.3838013838013838, | |
| "avg_precision_Page-footer": 0.39658119658119656, | |
| "avg_recall_Page-footer": 0.3903133903133903, | |
| "avg_f1_Page-header": 0.3583932853717027, | |
| "avg_precision_Page-header": 0.3776978417266187, | |
| "avg_recall_Page-header": 0.3511476533059267, | |
| "avg_f1_Picture": 0.39128406100455787, | |
| "avg_precision_Picture": 0.430189634382181, | |
| "avg_recall_Picture": 0.3949792960662526, | |
| "avg_f1_Text": 0.33763735831294645, | |
| "avg_precision_Text": 0.3747107381271595, | |
| "avg_recall_Text": 0.3645832988135569, | |
| "avg_mean_f1": 0.3485089837735459, | |
| "avg_num_predictions": 20.699074074074073, | |
| "avg_num_ground_truth": 21.185185185185187, | |
| "avg_layout_localization_pass_rate": 0.571032593892515, | |
| "avg_layout_classification_pass_rate": 0.4319576526775129, | |
| "avg_layout_attribution_pass_rate": 0.5157011525218308, | |
| "avg_layout_reading_order_pass_rate": 0.6535026277229299, | |
| "avg_layout_rule_pass_rate": 0.5050446887904327, | |
| "avg_rule_pass_rate": 0.5770645881341429, | |
| "avg_layout_element_rule_pass_rate": 0.31983085544638085, | |
| "avg_lap": 0.8138688831653484, | |
| "avg_lar": 0.8496233807356114, | |
| "avg_af1": 0.8175696244964747, | |
| "avg_unmatched_gt_elements": 10.268518518518519, | |
| "avg_unmatched_pred_elements": 10.476851851851851, | |
| "avg_f1_Section": 0.31968017913720564, | |
| "avg_precision_Section": 0.38158295508838985, | |
| "avg_recall_Section": 0.3548951275174101, | |
| "avg_f1_Table": 0.3121578099838969, | |
| "avg_precision_Table": 0.28840579710144926, | |
| "avg_recall_Table": 0.39251207729468596, | |
| "avg_rule_order_pass_rate": 0.901154135180838, | |
| "avg_normalized_order": 0.901154135180838, | |
| "avg_normalized_text_score": 0.901154135180838, | |
| "avg_content_faithfulness": 0.901154135180838 | |
| }, | |
| "hard": { | |
| "avg_mAP@[.50:.95]": 0.19917435827558116, | |
| "avg_AP50": 0.38467580413569913, | |
| "avg_AP75": 0.16061067233428336, | |
| "avg_f1_Page-footer": 0.3761904761904762, | |
| "avg_precision_Page-footer": 0.37857142857142856, | |
| "avg_recall_Page-footer": 0.39285714285714285, | |
| "avg_f1_Page-header": 0.3774622892635315, | |
| "avg_precision_Page-header": 0.40683229813664595, | |
| "avg_recall_Page-header": 0.37724956203217075, | |
| "avg_f1_Picture": 0.36278498506621704, | |
| "avg_precision_Picture": 0.47314060071605496, | |
| "avg_recall_Picture": 0.3413590054490464, | |
| "avg_f1_Section": 0.23651709881504898, | |
| "avg_precision_Section": 0.36756691114567824, | |
| "avg_recall_Section": 0.2520659239878919, | |
| "avg_f1_Text": 0.29575822218419906, | |
| "avg_precision_Text": 0.4077476143373612, | |
| "avg_recall_Text": 0.2816782174757315, | |
| "avg_mean_f1": 0.3060365230558017, | |
| "avg_num_predictions": 27.463636363636365, | |
| "avg_num_ground_truth": 43.81818181818182, | |
| "avg_layout_localization_pass_rate": 0.41686810357827786, | |
| "avg_layout_classification_pass_rate": 0.31998608793860717, | |
| "avg_layout_attribution_pass_rate": 0.5034450042500358, | |
| "avg_layout_reading_order_pass_rate": 0.6357362235147531, | |
| "avg_layout_rule_pass_rate": 0.4093089295142637, | |
| "avg_rule_pass_rate": 0.42381857584524424, | |
| "avg_layout_element_rule_pass_rate": 0.27949390782735417, | |
| "avg_lap": 0.8746372881274282, | |
| "avg_lar": 0.9132747471233392, | |
| "avg_af1": 0.8858655450669813, | |
| "avg_unmatched_gt_elements": 27.672727272727272, | |
| "avg_unmatched_pred_elements": 12.181818181818182, | |
| "avg_f1_Table": 0.19295371295371294, | |
| "avg_precision_Table": 0.1511738261738262, | |
| "avg_recall_Table": 0.34696969696969693, | |
| "avg_rule_order_pass_rate": 0.6233262128962257, | |
| "avg_normalized_order": 0.6233262128962257, | |
| "avg_normalized_text_score": 0.6233262128962257, | |
| "avg_content_faithfulness": 0.6233262128962257 | |
| }, | |
| "layout": { | |
| "avg_mAP@[.50:.95]": 0.2071643111595238, | |
| "avg_AP50": 0.38237206250169886, | |
| "avg_AP75": 0.16831471074679888, | |
| "avg_f1_Page-footer": 0.37965536409116174, | |
| "avg_precision_Page-footer": 0.3867704280155642, | |
| "avg_recall_Page-footer": 0.39169909208819714, | |
| "avg_f1_Page-header": 0.36862698412698414, | |
| "avg_precision_Page-header": 0.3933333333333333, | |
| "avg_recall_Page-header": 0.3651556776556777, | |
| "avg_f1_Picture": 0.37582007377665133, | |
| "avg_precision_Picture": 0.45349541440993646, | |
| "avg_recall_Picture": 0.3658841951915754, | |
| "avg_f1_Text": 0.3157437092096685, | |
| "avg_precision_Text": 0.39198181489891953, | |
| "avg_recall_Text": 0.32124203327004147, | |
| "avg_mean_f1": 0.32707792561321625, | |
| "avg_num_predictions": 24.112385321100916, | |
| "avg_num_ground_truth": 32.60550458715596, | |
| "avg_layout_localization_pass_rate": 0.49324317217432195, | |
| "avg_layout_classification_pass_rate": 0.3754582392771476, | |
| "avg_layout_attribution_pass_rate": 0.5095168575223011, | |
| "avg_layout_reading_order_pass_rate": 0.6443816756841821, | |
| "avg_layout_rule_pass_rate": 0.4567376542932833, | |
| "avg_rule_pass_rate": 0.5047324703337827, | |
| "avg_layout_element_rule_pass_rate": 0.2994773497670555, | |
| "avg_lap": 0.8445318397975906, | |
| "avg_lar": 0.8817410426743731, | |
| "avg_af1": 0.8520308688210423, | |
| "avg_unmatched_gt_elements": 19.05045871559633, | |
| "avg_unmatched_pred_elements": 11.337155963302752, | |
| "avg_f1_Section": 0.2744873389621379, | |
| "avg_precision_Section": 0.3739662959731198, | |
| "avg_recall_Section": 0.29901523775819305, | |
| "avg_f1_Table": 0.25928502501083145, | |
| "avg_precision_Table": 0.22753677773839065, | |
| "avg_recall_Table": 0.37231182795698925, | |
| "avg_rule_order_pass_rate": 0.8316971546096849, | |
| "avg_normalized_order": 0.8316971546096849, | |
| "avg_normalized_text_score": 0.8316971546096849, | |
| "avg_content_faithfulness": 0.8316971546096849 | |
| } | |
| } | |
| }, | |
| "table": { | |
| "total_examples": 503, | |
| "successful": 503, | |
| "failed": 0, | |
| "aggregate_metrics": { | |
| "avg_grits_con": 0.5547074258792586, | |
| "avg_structural_consistency": 1.0, | |
| "avg_table_record_match": 0.2690282540957644, | |
| "avg_table_record_match_perfect": 0.17374009908801163, | |
| "avg_grits_trm_composite": 0.420874000159437, | |
| "avg_tables_expected": 1.7995594713656389, | |
| "avg_tables_actual": 2.5286343612334803, | |
| "avg_tables_unparseable_pred": 0.0, | |
| "avg_tables_paired": 1.6938325991189427, | |
| "avg_tables_unmatched_expected": 0.10572687224669604, | |
| "avg_tables_unmatched_pred": 0.8524229074889867, | |
| "avg_grits_con_predicted": 0.6145767295534517, | |
| "avg_table_record_match_predicted": 0.29806434319420594, | |
| "avg_table_record_match_perfect_predicted": 0.19249178379134327 | |
| }, | |
| "tag_metrics": { | |
| "easy": { | |
| "avg_grits_con": 0.5695847522958832, | |
| "avg_structural_consistency": 1.0, | |
| "avg_table_record_match": 0.2910168441843165, | |
| "avg_table_record_match_perfect": 0.18534255232147506, | |
| "avg_grits_trm_composite": 0.438344420226983, | |
| "avg_tables_expected": 1.8134715025906736, | |
| "avg_tables_actual": 2.4974093264248705, | |
| "avg_tables_unparseable_pred": 0.0, | |
| "avg_tables_paired": 1.6968911917098446, | |
| "avg_tables_unmatched_expected": 0.11658031088082901, | |
| "avg_tables_unmatched_pred": 0.8186528497409327, | |
| "avg_grits_con_predicted": 0.6300846871252388, | |
| "avg_table_record_match_predicted": 0.3219279597582983, | |
| "avg_table_record_match_perfect_predicted": 0.2050291964799737 | |
| }, | |
| "hard": { | |
| "avg_grits_con": 0.47112034193322283, | |
| "avg_structural_consistency": 1.0, | |
| "avg_table_record_match": 0.1454870966245573, | |
| "avg_table_record_match_perfect": 0.10855263157894737, | |
| "avg_grits_trm_composite": 0.32271782425361933, | |
| "avg_tables_expected": 1.7205882352941178, | |
| "avg_tables_actual": 2.7058823529411766, | |
| "avg_tables_unparseable_pred": 0.0, | |
| "avg_tables_paired": 1.6764705882352942, | |
| "avg_tables_unmatched_expected": 0.04411764705882353, | |
| "avg_tables_unmatched_pred": 1.0441176470588236, | |
| "avg_grits_con_predicted": 0.526546264513602, | |
| "avg_table_record_match_predicted": 0.1626032256392111, | |
| "avg_table_record_match_perfect_predicted": 0.1213235294117647 | |
| }, | |
| "table": { | |
| "avg_grits_con": 0.5547074258792586, | |
| "avg_structural_consistency": 1.0, | |
| "avg_table_record_match": 0.2690282540957644, | |
| "avg_table_record_match_perfect": 0.17374009908801163, | |
| "avg_grits_trm_composite": 0.420874000159437, | |
| "avg_tables_expected": 1.7995594713656389, | |
| "avg_tables_actual": 2.5286343612334803, | |
| "avg_tables_unparseable_pred": 0.0, | |
| "avg_tables_paired": 1.6938325991189427, | |
| "avg_tables_unmatched_expected": 0.10572687224669604, | |
| "avg_tables_unmatched_pred": 0.8524229074889867, | |
| "avg_grits_con_predicted": 0.6145767295534517, | |
| "avg_table_record_match_predicted": 0.29806434319420594, | |
| "avg_table_record_match_perfect_predicted": 0.19249178379134327 | |
| } | |
| } | |
| }, | |
| "text_content": { | |
| "total_examples": 506, | |
| "successful": 506, | |
| "failed": 0, | |
| "aggregate_metrics": { | |
| "avg_rule_pass_rate": 0.7279322595207693, | |
| "avg_rule_missing_sentence_percent_pass_rate": 0.5191129926197232, | |
| "avg_rule_unexpected_sentence_percent_pass_rate": 0.5729155928630568, | |
| "avg_rule_too_many_sentence_occurence_percent_pass_rate": 0.9698544465960441, | |
| "avg_rule_unexpected_word_percent_pass_rate": 0.805245619352306, | |
| "avg_rule_too_many_word_occurence_percent_pass_rate": 0.9886470335856518, | |
| "avg_rule_missing_word_percent_pass_rate": 0.7599520810008141, | |
| "avg_rule_bag_of_digit_percent_pass_rate": 0.8777108039031275, | |
| "avg_rule_missing_specific_word_pass_rate": 0.7831279670701689, | |
| "avg_rule_missing_specific_sentence_pass_rate": 0.5195585531132046, | |
| "avg_rule_order_pass_rate": 0.6021493489712944, | |
| "avg_rule_is_header_pass_rate": 0.09854014598540146, | |
| "avg_rule_is_footer_pass_rate": 0.12211221122112212, | |
| "avg_normalized_text_correctness": 0.7843566193048881, | |
| "avg_normalized_order": 0.6021493489712944, | |
| "avg_normalized_text_score": 0.695325652701065, | |
| "avg_content_faithfulness": 0.725002641569006 | |
| }, | |
| "tag_metrics": { | |
| "dense": { | |
| "avg_rule_pass_rate": 0.7078556797726335, | |
| "avg_rule_missing_sentence_percent_pass_rate": 0.5074963436455969, | |
| "avg_rule_unexpected_sentence_percent_pass_rate": 0.47786649184189134, | |
| "avg_rule_too_many_sentence_occurence_percent_pass_rate": 0.952335955870966, | |
| "avg_rule_unexpected_word_percent_pass_rate": 0.7628451895453778, | |
| "avg_rule_too_many_word_occurence_percent_pass_rate": 0.948118996604223, | |
| "avg_rule_missing_word_percent_pass_rate": 0.7770213470739362, | |
| "avg_rule_bag_of_digit_percent_pass_rate": 0.9247246972668713, | |
| "avg_rule_missing_specific_word_pass_rate": 0.7804796815243854, | |
| "avg_rule_missing_specific_sentence_pass_rate": 0.5051118247813631, | |
| "avg_rule_order_pass_rate": 0.513860237493354, | |
| "avg_rule_is_header_pass_rate": 0.0, | |
| "avg_rule_is_footer_pass_rate": 0.3333333333333333, | |
| "avg_normalized_text_correctness": 0.764344145978409, | |
| "avg_normalized_order": 0.513860237493354, | |
| "avg_normalized_text_score": 0.6391021917358815, | |
| "avg_content_faithfulness": 0.680849509816724 | |
| }, | |
| "easy": { | |
| "avg_rule_pass_rate": 0.787928911371259, | |
| "avg_rule_missing_sentence_percent_pass_rate": 0.5941964915668045, | |
| "avg_rule_unexpected_sentence_percent_pass_rate": 0.643150296166318, | |
| "avg_rule_too_many_sentence_occurence_percent_pass_rate": 0.966232392750566, | |
| "avg_rule_unexpected_word_percent_pass_rate": 0.845458899278826, | |
| "avg_rule_too_many_word_occurence_percent_pass_rate": 0.9909978514182909, | |
| "avg_rule_missing_word_percent_pass_rate": 0.8211946285182539, | |
| "avg_rule_bag_of_digit_percent_pass_rate": 0.9284064474536208, | |
| "avg_rule_missing_specific_word_pass_rate": 0.8353397716116893, | |
| "avg_rule_missing_specific_sentence_pass_rate": 0.5946177902504456, | |
| "avg_rule_order_pass_rate": 0.6913040400102768, | |
| "avg_normalized_text_correctness": 0.8269157490707267, | |
| "avg_normalized_order": 0.6913040400102768, | |
| "avg_normalized_text_score": 0.7607883459829268, | |
| "avg_content_faithfulness": 0.78283081367886, | |
| "avg_rule_is_header_pass_rate": 0.109375, | |
| "avg_rule_is_footer_pass_rate": 0.13537117903930132 | |
| }, | |
| "handwritting": { | |
| "avg_rule_pass_rate": 0.1834117714027111, | |
| "avg_rule_missing_sentence_percent_pass_rate": 0.02298812298812299, | |
| "avg_rule_unexpected_sentence_percent_pass_rate": 0.050548383240690935, | |
| "avg_rule_too_many_sentence_occurence_percent_pass_rate": 1.0, | |
| "avg_rule_unexpected_word_percent_pass_rate": 0.24907470155809777, | |
| "avg_rule_too_many_word_occurence_percent_pass_rate": 1.0, | |
| "avg_rule_missing_word_percent_pass_rate": 0.18871049404907675, | |
| "avg_rule_bag_of_digit_percent_pass_rate": 0.34994100951953283, | |
| "avg_rule_missing_specific_word_pass_rate": 0.2115889161865551, | |
| "avg_rule_missing_specific_sentence_pass_rate": 0.02529521760290991, | |
| "avg_rule_order_pass_rate": 0.0038461538461538464, | |
| "avg_normalized_text_correctness": 0.4085693139718195, | |
| "avg_normalized_order": 0.0038461538461538464, | |
| "avg_normalized_text_score": 0.2062077339089867, | |
| "avg_content_faithfulness": 0.27366159392993095, | |
| "avg_rule_is_header_pass_rate": 0.0, | |
| "avg_rule_is_footer_pass_rate": 0.0 | |
| }, | |
| "hard": { | |
| "avg_rule_pass_rate": 0.5628326577891124, | |
| "avg_rule_missing_sentence_percent_pass_rate": 0.3113935214622094, | |
| "avg_rule_unexpected_sentence_percent_pass_rate": 0.3771205515456409, | |
| "avg_rule_too_many_sentence_occurence_percent_pass_rate": 0.9784618512118646, | |
| "avg_rule_unexpected_word_percent_pass_rate": 0.693131509366181, | |
| "avg_rule_too_many_word_occurence_percent_pass_rate": 0.982638657560303, | |
| "avg_rule_missing_word_percent_pass_rate": 0.5919437391360218, | |
| "avg_rule_bag_of_digit_percent_pass_rate": 0.7441198175270335, | |
| "avg_rule_missing_specific_word_pass_rate": 0.639730011795594, | |
| "avg_rule_missing_specific_sentence_pass_rate": 0.3118362222023347, | |
| "avg_rule_order_pass_rate": 0.3577390683110252, | |
| "avg_rule_is_header_pass_rate": 0.08235294117647059, | |
| "avg_rule_is_footer_pass_rate": 0.07792207792207792, | |
| "avg_normalized_text_correctness": 0.666495356124219, | |
| "avg_normalized_order": 0.3577390683110252, | |
| "avg_normalized_text_score": 0.5148507174177063, | |
| "avg_content_faithfulness": 0.5653989303198772 | |
| }, | |
| "misc": { | |
| "avg_rule_pass_rate": 0.7252518366322029, | |
| "avg_rule_missing_sentence_percent_pass_rate": 0.504238694448032, | |
| "avg_rule_unexpected_sentence_percent_pass_rate": 0.5347404180804285, | |
| "avg_rule_too_many_sentence_occurence_percent_pass_rate": 0.9681096548982955, | |
| "avg_rule_unexpected_word_percent_pass_rate": 0.8026989184490682, | |
| "avg_rule_too_many_word_occurence_percent_pass_rate": 0.9712741931469514, | |
| "avg_rule_missing_word_percent_pass_rate": 0.7932643479201368, | |
| "avg_rule_bag_of_digit_percent_pass_rate": 0.9177888548042893, | |
| "avg_rule_missing_specific_word_pass_rate": 0.8187098426775091, | |
| "avg_rule_missing_specific_sentence_pass_rate": 0.5070010907496173, | |
| "avg_rule_order_pass_rate": 0.595554385884535, | |
| "avg_rule_is_header_pass_rate": 0.23529411764705882, | |
| "avg_rule_is_footer_pass_rate": 0.13043478260869565, | |
| "avg_normalized_text_correctness": 0.7846288204333423, | |
| "avg_normalized_order": 0.595554385884535, | |
| "avg_normalized_text_score": 0.6949355402804869, | |
| "avg_content_faithfulness": 0.7248333003314386 | |
| }, | |
| "multicolumns": { | |
| "avg_rule_pass_rate": 0.8215939996602195, | |
| "avg_rule_missing_sentence_percent_pass_rate": 0.5954462181301912, | |
| "avg_rule_unexpected_sentence_percent_pass_rate": 0.6336960721594214, | |
| "avg_rule_too_many_sentence_occurence_percent_pass_rate": 0.9657525900441687, | |
| "avg_rule_unexpected_word_percent_pass_rate": 0.8796451618134619, | |
| "avg_rule_too_many_word_occurence_percent_pass_rate": 0.9893207553345342, | |
| "avg_rule_missing_word_percent_pass_rate": 0.8747027392876436, | |
| "avg_rule_bag_of_digit_percent_pass_rate": 0.9433341874364176, | |
| "avg_rule_missing_specific_word_pass_rate": 0.886060789557353, | |
| "avg_rule_missing_specific_sentence_pass_rate": 0.5949228926044701, | |
| "avg_rule_order_pass_rate": 0.614791869308767, | |
| "avg_rule_is_footer_pass_rate": 0.1643835616438356, | |
| "avg_normalized_text_correctness": 0.8402711034579771, | |
| "avg_normalized_order": 0.614791869308767, | |
| "avg_normalized_text_score": 0.727531486383372, | |
| "avg_content_faithfulness": 0.7651113587415737, | |
| "avg_rule_is_header_pass_rate": 0.12280701754385964 | |
| }, | |
| "multilang": { | |
| "avg_rule_pass_rate": 0.49946365638443696, | |
| "avg_rule_missing_sentence_percent_pass_rate": 0.3293652393285729, | |
| "avg_rule_unexpected_sentence_percent_pass_rate": 0.3722497692954488, | |
| "avg_rule_too_many_sentence_occurence_percent_pass_rate": 0.9836886815332004, | |
| "avg_rule_unexpected_word_percent_pass_rate": 0.5289884508977691, | |
| "avg_rule_too_many_word_occurence_percent_pass_rate": 0.9728855612773274, | |
| "avg_rule_missing_word_percent_pass_rate": 0.5219647363002523, | |
| "avg_rule_bag_of_digit_percent_pass_rate": 0.9071705285692483, | |
| "avg_rule_missing_specific_word_pass_rate": 0.5309053433514255, | |
| "avg_rule_missing_specific_sentence_pass_rate": 0.33227127072210444, | |
| "avg_rule_order_pass_rate": 0.45134766091907136, | |
| "avg_normalized_text_correctness": 0.6556207972163736, | |
| "avg_normalized_order": 0.45134766091907136, | |
| "avg_normalized_text_score": 0.5537243912121401, | |
| "avg_content_faithfulness": 0.5876898598802179, | |
| "avg_rule_is_header_pass_rate": 0.16, | |
| "avg_rule_is_footer_pass_rate": 0.07407407407407407 | |
| }, | |
| "ocr": { | |
| "avg_rule_pass_rate": 0.5630233732287258, | |
| "avg_rule_missing_sentence_percent_pass_rate": 0.28454004472891226, | |
| "avg_rule_unexpected_sentence_percent_pass_rate": 0.3351590023916691, | |
| "avg_rule_too_many_sentence_occurence_percent_pass_rate": 0.9843664941554378, | |
| "avg_rule_unexpected_word_percent_pass_rate": 0.7360778132582102, | |
| "avg_rule_too_many_word_occurence_percent_pass_rate": 0.9960265741782435, | |
| "avg_rule_missing_word_percent_pass_rate": 0.5656002243667991, | |
| "avg_rule_bag_of_digit_percent_pass_rate": 0.7160536165680694, | |
| "avg_rule_missing_specific_word_pass_rate": 0.6322665596887669, | |
| "avg_rule_missing_specific_sentence_pass_rate": 0.2851576540210632, | |
| "avg_rule_order_pass_rate": 0.3647728747347877, | |
| "avg_normalized_text_correctness": 0.6588403748883668, | |
| "avg_normalized_order": 0.3647728747347877, | |
| "avg_normalized_text_score": 0.5119702875080323, | |
| "avg_content_faithfulness": 0.5609269833014771, | |
| "avg_rule_is_header_pass_rate": 0.0, | |
| "avg_rule_is_footer_pass_rate": 0.10526315789473684 | |
| }, | |
| "simple": { | |
| "avg_rule_pass_rate": 0.8965444208826847, | |
| "avg_rule_missing_sentence_percent_pass_rate": 0.7305254271135413, | |
| "avg_rule_unexpected_sentence_percent_pass_rate": 0.8251003019364899, | |
| "avg_rule_too_many_sentence_occurence_percent_pass_rate": 0.9565146830947648, | |
| "avg_rule_unexpected_word_percent_pass_rate": 0.9390052194652466, | |
| "avg_rule_too_many_word_occurence_percent_pass_rate": 0.992428162901977, | |
| "avg_rule_missing_word_percent_pass_rate": 0.9274701895446175, | |
| "avg_rule_bag_of_digit_percent_pass_rate": 0.9678585428994263, | |
| "avg_rule_missing_specific_word_pass_rate": 0.9338597633277089, | |
| "avg_rule_missing_specific_sentence_pass_rate": 0.7297496993422307, | |
| "avg_rule_order_pass_rate": 0.8595959860320048, | |
| "avg_rule_is_footer_pass_rate": 0.10909090909090909, | |
| "avg_normalized_text_correctness": 0.9051754000640813, | |
| "avg_normalized_order": 0.8595959860320048, | |
| "avg_normalized_text_score": 0.8819022986556263, | |
| "avg_content_faithfulness": 0.8896599991251112, | |
| "avg_rule_is_header_pass_rate": 0.15 | |
| }, | |
| "sparse": { | |
| "avg_rule_pass_rate": 0.7326413370930663, | |
| "avg_rule_unexpected_word_percent_pass_rate": 0.7411715642028549, | |
| "avg_rule_too_many_word_occurence_percent_pass_rate": 1.0, | |
| "avg_rule_missing_word_percent_pass_rate": 0.8208943833943834, | |
| "avg_rule_missing_specific_word_pass_rate": 0.8208943833943834, | |
| "avg_normalized_text_correctness": 0.798244772418378, | |
| "avg_normalized_text_score": 0.7561785124072027, | |
| "avg_content_faithfulness": 0.7702005990775944, | |
| "avg_rule_missing_sentence_percent_pass_rate": 0.5515151515151515, | |
| "avg_rule_unexpected_sentence_percent_pass_rate": 0.40025252525252525, | |
| "avg_rule_too_many_sentence_occurence_percent_pass_rate": 0.9893048128342247, | |
| "avg_rule_bag_of_digit_percent_pass_rate": 0.96875, | |
| "avg_rule_missing_specific_sentence_pass_rate": 0.5614718614718615, | |
| "avg_rule_is_footer_pass_rate": 0.0, | |
| "avg_rule_is_header_pass_rate": 0.0, | |
| "avg_rule_order_pass_rate": 0.3333333333333333, | |
| "avg_normalized_order": 0.3333333333333333 | |
| }, | |
| "text_content": { | |
| "avg_rule_pass_rate": 0.7279322595207693, | |
| "avg_rule_missing_sentence_percent_pass_rate": 0.5191129926197232, | |
| "avg_rule_unexpected_sentence_percent_pass_rate": 0.5729155928630568, | |
| "avg_rule_too_many_sentence_occurence_percent_pass_rate": 0.9698544465960441, | |
| "avg_rule_unexpected_word_percent_pass_rate": 0.805245619352306, | |
| "avg_rule_too_many_word_occurence_percent_pass_rate": 0.9886470335856518, | |
| "avg_rule_missing_word_percent_pass_rate": 0.7599520810008141, | |
| "avg_rule_bag_of_digit_percent_pass_rate": 0.8777108039031275, | |
| "avg_rule_missing_specific_word_pass_rate": 0.7831279670701689, | |
| "avg_rule_missing_specific_sentence_pass_rate": 0.5195585531132046, | |
| "avg_rule_order_pass_rate": 0.6021493489712944, | |
| "avg_rule_is_header_pass_rate": 0.09854014598540146, | |
| "avg_rule_is_footer_pass_rate": 0.12211221122112212, | |
| "avg_normalized_text_correctness": 0.7843566193048881, | |
| "avg_normalized_order": 0.6021493489712944, | |
| "avg_normalized_text_score": 0.695325652701065, | |
| "avg_content_faithfulness": 0.725002641569006 | |
| } | |
| } | |
| }, | |
| "text_formatting": { | |
| "total_examples": 476, | |
| "successful": 476, | |
| "failed": 0, | |
| "aggregate_metrics": { | |
| "avg_rule_pass_rate": 0.27362155896974427, | |
| "avg_rule_is_bold_pass_rate": 0.2909983252176997, | |
| "avg_normalized_text_styling": 0.4406143284241916, | |
| "avg_normalized_text_score": 0.3547786072212522, | |
| "avg_semantic_formatting": 0.3547786072212522, | |
| "avg_rule_is_title_pass_rate": 0.31343185720296096, | |
| "avg_rule_title_hierarchy_percent_pass_rate": 0.25611064219134966, | |
| "avg_normalized_title_accuracy": 0.2847712496971553, | |
| "avg_rule_is_italic_pass_rate": 0.2293101097117718, | |
| "avg_rule_is_underline_pass_rate": 0.0, | |
| "avg_rule_is_sup_pass_rate": 0.5845076818606231, | |
| "avg_rule_is_strikeout_pass_rate": 0.0, | |
| "avg_rule_is_latex_pass_rate": 0.03125, | |
| "avg_normalized_latex": 0.03125, | |
| "avg_rule_is_mark_pass_rate": 0.0, | |
| "avg_rule_is_sub_pass_rate": 0.5666666666666667, | |
| "avg_rule_is_code_block_pass_rate": 0.0, | |
| "avg_normalized_code_block": 0.0 | |
| }, | |
| "tag_metrics": { | |
| "dense": { | |
| "avg_rule_pass_rate": 0.25149239363249604, | |
| "avg_rule_is_bold_pass_rate": 0.262963887963888, | |
| "avg_normalized_text_styling": 0.5107553648629827, | |
| "avg_normalized_text_score": 0.35953769154129495, | |
| "avg_semantic_formatting": 0.35953769154129495, | |
| "avg_rule_is_title_pass_rate": 0.2872222222222222, | |
| "avg_rule_title_hierarchy_percent_pass_rate": 0.1911591478696742, | |
| "avg_normalized_title_accuracy": 0.2391906850459482, | |
| "avg_rule_is_italic_pass_rate": 0.0, | |
| "avg_rule_is_underline_pass_rate": 0.0, | |
| "avg_rule_is_latex_pass_rate": 0.0, | |
| "avg_normalized_latex": 0.0, | |
| "avg_rule_is_sup_pass_rate": 1.0 | |
| }, | |
| "easy": { | |
| "avg_rule_pass_rate": 0.3020783952244902, | |
| "avg_rule_is_title_pass_rate": 0.3425627346974813, | |
| "avg_rule_title_hierarchy_percent_pass_rate": 0.28541534093552945, | |
| "avg_rule_is_bold_pass_rate": 0.30768796436866225, | |
| "avg_normalized_text_styling": 0.47106400818404814, | |
| "avg_normalized_title_accuracy": 0.31398903781650533, | |
| "avg_normalized_text_score": 0.38935751000200197, | |
| "avg_semantic_formatting": 0.38935751000200197, | |
| "avg_rule_is_latex_pass_rate": 0.043478260869565216, | |
| "avg_rule_is_underline_pass_rate": 0.0, | |
| "avg_rule_is_strikeout_pass_rate": 0.0, | |
| "avg_rule_is_mark_pass_rate": 0.0, | |
| "avg_normalized_latex": 0.043478260869565216, | |
| "avg_rule_is_italic_pass_rate": 0.2647539447627904, | |
| "avg_rule_is_sup_pass_rate": 0.6233558529013074, | |
| "avg_rule_is_sub_pass_rate": 0.6, | |
| "avg_rule_is_code_block_pass_rate": 0.0, | |
| "avg_normalized_code_block": 0.0 | |
| }, | |
| "handwritting": { | |
| "avg_rule_pass_rate": 0.0606060606060606, | |
| "avg_rule_is_title_pass_rate": 0.125, | |
| "avg_rule_title_hierarchy_percent_pass_rate": 0.125, | |
| "avg_rule_is_bold_pass_rate": 0.0, | |
| "avg_rule_is_sup_pass_rate": 0.0, | |
| "avg_normalized_text_styling": 0.0, | |
| "avg_normalized_title_accuracy": 0.125, | |
| "avg_normalized_text_score": 0.05, | |
| "avg_semantic_formatting": 0.05, | |
| "avg_rule_is_italic_pass_rate": 0.0, | |
| "avg_rule_is_strikeout_pass_rate": 0.0, | |
| "avg_rule_is_latex_pass_rate": 0.0, | |
| "avg_rule_is_underline_pass_rate": 0.0, | |
| "avg_normalized_latex": 0.0 | |
| }, | |
| "hard": { | |
| "avg_rule_pass_rate": 0.18538579004091432, | |
| "avg_rule_is_bold_pass_rate": 0.22985904995658668, | |
| "avg_normalized_text_styling": 0.34492046587558667, | |
| "avg_normalized_text_score": 0.24661933576163808, | |
| "avg_semantic_formatting": 0.24661933576163808, | |
| "avg_rule_is_title_pass_rate": 0.21533752108689533, | |
| "avg_rule_title_hierarchy_percent_pass_rate": 0.15720390834332604, | |
| "avg_normalized_title_accuracy": 0.18627071471511067, | |
| "avg_rule_is_italic_pass_rate": 0.10998334998334998, | |
| "avg_rule_is_underline_pass_rate": 0.0, | |
| "avg_rule_is_sup_pass_rate": 0.4270833333333333, | |
| "avg_rule_is_strikeout_pass_rate": 0.0, | |
| "avg_rule_is_latex_pass_rate": 0.0, | |
| "avg_normalized_latex": 0.0, | |
| "avg_rule_is_mark_pass_rate": 0.0, | |
| "avg_rule_is_sub_pass_rate": 0.4666666666666666, | |
| "avg_rule_is_code_block_pass_rate": 0.0, | |
| "avg_normalized_code_block": 0.0 | |
| }, | |
| "misc": { | |
| "avg_rule_pass_rate": 0.29013324539431723, | |
| "avg_rule_is_title_pass_rate": 0.2912211052836053, | |
| "avg_rule_title_hierarchy_percent_pass_rate": 0.26003086419753085, | |
| "avg_rule_is_bold_pass_rate": 0.3713795266244729, | |
| "avg_normalized_text_styling": 0.5782096773797316, | |
| "avg_normalized_title_accuracy": 0.27562598474056804, | |
| "avg_normalized_text_score": 0.36282397770936614, | |
| "avg_semantic_formatting": 0.36282397770936614, | |
| "avg_rule_is_sup_pass_rate": 0.71875, | |
| "avg_rule_is_italic_pass_rate": 0.14285714285714285, | |
| "avg_rule_is_mark_pass_rate": 0.0, | |
| "avg_rule_is_latex_pass_rate": 0.0, | |
| "avg_rule_is_underline_pass_rate": 0.0, | |
| "avg_rule_is_strikeout_pass_rate": 0.0, | |
| "avg_normalized_latex": 0.0, | |
| "avg_rule_is_sub_pass_rate": 0.5, | |
| "avg_rule_is_code_block_pass_rate": 0.0, | |
| "avg_normalized_code_block": 0.0 | |
| }, | |
| "multicolumns": { | |
| "avg_rule_pass_rate": 0.4076038292828515, | |
| "avg_rule_is_title_pass_rate": 0.4099294464602716, | |
| "avg_rule_title_hierarchy_percent_pass_rate": 0.3227662306840862, | |
| "avg_rule_is_bold_pass_rate": 0.3677595136423659, | |
| "avg_rule_is_italic_pass_rate": 0.35414325627091586, | |
| "avg_rule_is_sup_pass_rate": 0.8486029122392759, | |
| "avg_normalized_text_styling": 0.6240837534453189, | |
| "avg_normalized_title_accuracy": 0.36634783857217895, | |
| "avg_normalized_text_score": 0.4727118289334861, | |
| "avg_semantic_formatting": 0.4727118289334861, | |
| "avg_rule_is_latex_pass_rate": 0.07692307692307693, | |
| "avg_normalized_latex": 0.07692307692307693, | |
| "avg_rule_is_underline_pass_rate": 0.0, | |
| "avg_rule_is_sub_pass_rate": 0.4666666666666666 | |
| }, | |
| "multilang": { | |
| "avg_rule_pass_rate": 0.1604809680695652, | |
| "avg_rule_is_title_pass_rate": 0.14489316239316238, | |
| "avg_rule_title_hierarchy_percent_pass_rate": 0.10684724252019648, | |
| "avg_normalized_title_accuracy": 0.12587020245667943, | |
| "avg_normalized_text_score": 0.2252435581978776, | |
| "avg_semantic_formatting": 0.2252435581978776, | |
| "avg_rule_is_bold_pass_rate": 0.20447330447330447, | |
| "avg_normalized_text_styling": 0.3297982856806386, | |
| "avg_rule_is_italic_pass_rate": 0.14285714285714285, | |
| "avg_rule_is_underline_pass_rate": 0.0, | |
| "avg_rule_is_mark_pass_rate": 0.0, | |
| "avg_rule_is_sup_pass_rate": 0.3333333333333333, | |
| "avg_rule_is_latex_pass_rate": 0.0, | |
| "avg_normalized_latex": 0.0 | |
| }, | |
| "ocr": { | |
| "avg_rule_pass_rate": 0.10292966564195055, | |
| "avg_rule_is_title_pass_rate": 0.13040639915639915, | |
| "avg_rule_title_hierarchy_percent_pass_rate": 0.0900662468844287, | |
| "avg_rule_is_sup_pass_rate": 0.0, | |
| "avg_normalized_text_styling": 0.12234275002467386, | |
| "avg_normalized_title_accuracy": 0.11023632302041393, | |
| "avg_normalized_text_score": 0.11230580886555556, | |
| "avg_semantic_formatting": 0.11230580886555556, | |
| "avg_rule_is_underline_pass_rate": 0.0, | |
| "avg_rule_is_bold_pass_rate": 0.10175130296060529, | |
| "avg_rule_is_italic_pass_rate": 0.10702341137123747, | |
| "avg_rule_is_code_block_pass_rate": 0.0, | |
| "avg_normalized_code_block": 0.0, | |
| "avg_rule_is_strikeout_pass_rate": 0.0, | |
| "avg_rule_is_latex_pass_rate": 0.0, | |
| "avg_normalized_latex": 0.0 | |
| }, | |
| "simple": { | |
| "avg_rule_pass_rate": 0.35776003411298596, | |
| "avg_rule_is_title_pass_rate": 0.44807599807599807, | |
| "avg_rule_title_hierarchy_percent_pass_rate": 0.38569359272063053, | |
| "avg_rule_is_bold_pass_rate": 0.4007423895801888, | |
| "avg_rule_is_italic_pass_rate": 0.20562685757931937, | |
| "avg_rule_is_underline_pass_rate": 0.0, | |
| "avg_normalized_text_styling": 0.5783337922172309, | |
| "avg_normalized_title_accuracy": 0.4168847953983143, | |
| "avg_normalized_text_score": 0.5059816825077105, | |
| "avg_semantic_formatting": 0.5059816825077105, | |
| "avg_rule_is_sup_pass_rate": 0.7617845117845118, | |
| "avg_rule_is_mark_pass_rate": 0.0, | |
| "avg_rule_is_latex_pass_rate": 0.0, | |
| "avg_normalized_latex": 0.0, | |
| "avg_rule_is_strikeout_pass_rate": 0.0, | |
| "avg_rule_is_code_block_pass_rate": 0.0, | |
| "avg_normalized_code_block": 0.0, | |
| "avg_rule_is_sub_pass_rate": 1.0 | |
| }, | |
| "sparse": { | |
| "avg_rule_pass_rate": 0.18225108225108225, | |
| "avg_rule_is_title_pass_rate": 0.21212121212121213, | |
| "avg_rule_title_hierarchy_percent_pass_rate": 0.18484848484848485, | |
| "avg_normalized_title_accuracy": 0.19848484848484846, | |
| "avg_normalized_text_score": 0.19772727272727272, | |
| "avg_semantic_formatting": 0.19772727272727272, | |
| "avg_rule_is_bold_pass_rate": 0.125, | |
| "avg_normalized_text_styling": 0.20833333333333334, | |
| "avg_rule_is_italic_pass_rate": 0.5, | |
| "avg_rule_is_underline_pass_rate": 0.0 | |
| }, | |
| "text_formatting": { | |
| "avg_rule_pass_rate": 0.27362155896974427, | |
| "avg_rule_is_bold_pass_rate": 0.2909983252176997, | |
| "avg_normalized_text_styling": 0.4406143284241916, | |
| "avg_normalized_text_score": 0.3547786072212522, | |
| "avg_semantic_formatting": 0.3547786072212522, | |
| "avg_rule_is_title_pass_rate": 0.31343185720296096, | |
| "avg_rule_title_hierarchy_percent_pass_rate": 0.25611064219134966, | |
| "avg_normalized_title_accuracy": 0.2847712496971553, | |
| "avg_rule_is_italic_pass_rate": 0.2293101097117718, | |
| "avg_rule_is_underline_pass_rate": 0.0, | |
| "avg_rule_is_sup_pass_rate": 0.5845076818606231, | |
| "avg_rule_is_strikeout_pass_rate": 0.0, | |
| "avg_rule_is_latex_pass_rate": 0.03125, | |
| "avg_normalized_latex": 0.03125, | |
| "avg_rule_is_mark_pass_rate": 0.0, | |
| "avg_rule_is_sub_pass_rate": 0.5666666666666667, | |
| "avg_rule_is_code_block_pass_rate": 0.0, | |
| "avg_normalized_code_block": 0.0 | |
| } | |
| } | |
| } | |
| } | |
| } |