| { |
| "format": "tiny_receipt_vqa_runtime_benchmark_v1", |
| "precision": "int8", |
| "providers": [ |
| "CPUExecutionProvider" |
| ], |
| "threads": 1, |
| "timing_scope": "model inference from encoder input through EOS; session creation and image/token preprocessing excluded", |
| "warmups_per_vqa": 1, |
| "measured_repeats_per_vqa": 3, |
| "vqa_count": 20, |
| "aggregate": { |
| "e2e_median_of_vqa_medians_ms": 173.67218298022635 |
| }, |
| "results": [ |
| { |
| "id": "00000", |
| "image": "00000.jpg", |
| "question": "The location of the store is Hyocheon St. [?]. (Fill the blank)", |
| "requested_family": "auto", |
| "selected_family": "address", |
| "token_count": 39, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1064, |
| 5, |
| 6, |
| 299, |
| 342, |
| 339, |
| 330, |
| 54, |
| 306, |
| 1034, |
| 1023, |
| 326, |
| 54, |
| 300, |
| 1348, |
| 1072, |
| 325, |
| 1348, |
| 332, |
| 331, |
| 330, |
| 54, |
| 1128, |
| 54, |
| 14, |
| 12, |
| 21, |
| 7, |
| 8, |
| 1295, |
| 9, |
| 10, |
| 14, |
| 12, |
| 21, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 38, |
| "selected_family_id": 1, |
| "encoder_ms": { |
| "median": 144.62184603326023, |
| "mean": 149.95407732203603, |
| "min": 144.1063239471987, |
| "max": 161.13406198564917 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 34.44688400486484, |
| "mean": 39.39516797739392, |
| "min": 34.44556094473228, |
| "max": 49.293058982584625 |
| }, |
| "e2e_ms": { |
| "median": 193.9781649853103, |
| "mean": 189.40869099848592, |
| "min": 178.60896297497675, |
| "max": 195.63894503517076 |
| } |
| } |
| }, |
| { |
| "id": "00001", |
| "image": "00001.jpg", |
| "question": "What is the first number of the store's phone number?", |
| "requested_family": "auto", |
| "selected_family": "phone", |
| "token_count": 21, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1038, |
| 5, |
| 6, |
| 14, |
| 17, |
| 20, |
| 18, |
| 14, |
| 15, |
| 20, |
| 7, |
| 8, |
| 1063, |
| 13, |
| 9, |
| 10, |
| 14, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 20, |
| "selected_family_id": 0, |
| "encoder_ms": { |
| "median": 139.19997599441558, |
| "mean": 141.873165324796, |
| "min": 136.84602896682918, |
| "max": 149.5734910131432 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 18.195368989836425, |
| "mean": 18.760808006239433, |
| "min": 17.770698003005236, |
| "max": 20.31635702587664 |
| }, |
| "e2e_ms": { |
| "median": 157.02609898289666, |
| "mean": 160.69593998448303, |
| "min": 155.09657200891525, |
| "max": 169.96514896163717 |
| } |
| } |
| }, |
| { |
| "id": "00002", |
| "image": "00002.jpg", |
| "question": "What is the fourth number of the store's phone number?", |
| "requested_family": "auto", |
| "selected_family": "phone", |
| "token_count": 21, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1038, |
| 5, |
| 6, |
| 14, |
| 20, |
| 17, |
| 13, |
| 12, |
| 19, |
| 15, |
| 7, |
| 8, |
| 1063, |
| 16, |
| 9, |
| 10, |
| 13, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 20, |
| "selected_family_id": 0, |
| "encoder_ms": { |
| "median": 139.76027304306626, |
| "mean": 141.8147953421188, |
| "min": 138.6495690094307, |
| "max": 147.0345439738594 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 18.33128498401493, |
| "mean": 18.28849798766896, |
| "min": 17.886195972096175, |
| "max": 18.64801300689578 |
| }, |
| "e2e_ms": { |
| "median": 157.70362701732665, |
| "mean": 160.1600473319801, |
| "min": 157.0382629870437, |
| "max": 165.73825199157 |
| } |
| } |
| }, |
| { |
| "id": "00003", |
| "image": "00003.jpg", |
| "question": "The location of the store is Seonggokbuk St. [?]. (Fill the blank)", |
| "requested_family": "auto", |
| "selected_family": "address", |
| "token_count": 37, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1064, |
| 5, |
| 6, |
| 305, |
| 1369, |
| 337, |
| 336, |
| 54, |
| 311, |
| 1216, |
| 1452, |
| 54, |
| 1303, |
| 1089, |
| 54, |
| 336, |
| 340, |
| 322, |
| 1131, |
| 54, |
| 1128, |
| 54, |
| 13, |
| 12, |
| 13, |
| 7, |
| 8, |
| 1295, |
| 9, |
| 10, |
| 13, |
| 12, |
| 13, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 36, |
| "selected_family_id": 1, |
| "encoder_ms": { |
| "median": 146.8300599954091, |
| "mean": 146.68357032739246, |
| "min": 141.92631602054462, |
| "max": 151.2943349662237 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 36.745416990015656, |
| "mean": 35.83021465844164, |
| "min": 33.407523995265365, |
| "max": 37.33770299004391 |
| }, |
| "e2e_ms": { |
| "median": 180.29850901803002, |
| "mean": 182.57364101009443, |
| "min": 178.7324670003727, |
| "max": 188.68994701188058 |
| } |
| } |
| }, |
| { |
| "id": "00004", |
| "image": "00004.jpg", |
| "question": "What is the fourth number of the store's phone number?", |
| "requested_family": "auto", |
| "selected_family": "phone", |
| "token_count": 21, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1038, |
| 5, |
| 6, |
| 17, |
| 16, |
| 13, |
| 17, |
| 19, |
| 18, |
| 20, |
| 7, |
| 8, |
| 1063, |
| 16, |
| 9, |
| 10, |
| 17, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 20, |
| "selected_family_id": 0, |
| "encoder_ms": { |
| "median": 139.08961898414418, |
| "mean": 139.35535666920865, |
| "min": 137.99181999638677, |
| "max": 140.98463102709502 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 18.172656011302024, |
| "mean": 18.237755672695737, |
| "min": 18.05322099244222, |
| "max": 18.487390014342964 |
| }, |
| "e2e_ms": { |
| "median": 157.63431700179353, |
| "mean": 157.65107799476633, |
| "min": 156.10132599249482, |
| "max": 159.21759099001065 |
| } |
| } |
| }, |
| { |
| "id": "00005", |
| "image": "00005.jpg", |
| "question": "What is the third number of the store's phone number?", |
| "requested_family": "auto", |
| "selected_family": "phone", |
| "token_count": 21, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1038, |
| 5, |
| 6, |
| 17, |
| 14, |
| 15, |
| 17, |
| 14, |
| 19, |
| 19, |
| 7, |
| 8, |
| 1063, |
| 15, |
| 9, |
| 10, |
| 15, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 20, |
| "selected_family_id": 0, |
| "encoder_ms": { |
| "median": 143.3133709942922, |
| "mean": 145.96375731828934, |
| "min": 139.6672679693438, |
| "max": 154.91063299123198 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 18.525541003327817, |
| "mean": 19.011186338805903, |
| "min": 18.129895033780485, |
| "max": 20.37812297930941 |
| }, |
| "e2e_ms": { |
| "median": 161.89832403324544, |
| "mean": 165.04915066373846, |
| "min": 160.15073098242283, |
| "max": 173.09839697554708 |
| } |
| } |
| }, |
| { |
| "id": "00006", |
| "image": "00006.jpg", |
| "question": "What is the third number of the store's phone number?", |
| "requested_family": "auto", |
| "selected_family": "phone", |
| "token_count": 21, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1038, |
| 5, |
| 6, |
| 17, |
| 16, |
| 19, |
| 19, |
| 20, |
| 17, |
| 14, |
| 7, |
| 8, |
| 1063, |
| 15, |
| 9, |
| 10, |
| 19, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 20, |
| "selected_family_id": 0, |
| "encoder_ms": { |
| "median": 140.52525599254295, |
| "mean": 141.90307965812585, |
| "min": 136.78449398139492, |
| "max": 148.39948900043964 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 17.987416998948902, |
| "mean": 17.97556134018426, |
| "min": 17.683705023955554, |
| "max": 18.25556199764833 |
| }, |
| "e2e_ms": { |
| "median": 158.5697800037451, |
| "mean": 159.93933534870544, |
| "min": 154.5344740152359, |
| "max": 166.71375202713534 |
| } |
| } |
| }, |
| { |
| "id": "00007", |
| "image": "00007.jpg", |
| "question": "The location of the store is Hwadoan St. [?]. (Fill the blank)", |
| "requested_family": "auto", |
| "selected_family": "address", |
| "token_count": 37, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1064, |
| 5, |
| 6, |
| 311, |
| 1035, |
| 1026, |
| 1020, |
| 54, |
| 299, |
| 336, |
| 1463, |
| 54, |
| 300, |
| 344, |
| 1375, |
| 1072, |
| 54, |
| 335, |
| 328, |
| 54, |
| 1128, |
| 54, |
| 19, |
| 19, |
| 17, |
| 7, |
| 8, |
| 1295, |
| 9, |
| 10, |
| 19, |
| 19, |
| 17, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 36, |
| "selected_family_id": 1, |
| "encoder_ms": { |
| "median": 145.99315298255533, |
| "mean": 147.306475652537, |
| "min": 142.11521396646276, |
| "max": 153.8110600085929 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 32.903996005188674, |
| "mean": 33.87299568081895, |
| "min": 32.75954403216019, |
| "max": 35.955447005108 |
| }, |
| "e2e_ms": { |
| "median": 178.81044500973076, |
| "mean": 181.23773401991153, |
| "min": 178.1281090225093, |
| "max": 186.77464802749455 |
| } |
| } |
| }, |
| { |
| "id": "00008", |
| "image": "00008.jpg", |
| "question": "The location of the store is Heowanghu St. [?]. (Fill the blank)", |
| "requested_family": "auto", |
| "selected_family": "address", |
| "token_count": 37, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1064, |
| 5, |
| 6, |
| 299, |
| 344, |
| 1168, |
| 334, |
| 1380, |
| 54, |
| 317, |
| 326, |
| 1317, |
| 337, |
| 1380, |
| 54, |
| 300, |
| 326, |
| 1317, |
| 1484, |
| 54, |
| 1128, |
| 54, |
| 16, |
| 20, |
| 16, |
| 7, |
| 8, |
| 1295, |
| 9, |
| 10, |
| 16, |
| 20, |
| 16, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 36, |
| "selected_family_id": 1, |
| "encoder_ms": { |
| "median": 141.71967800939456, |
| "mean": 142.5617953451971, |
| "min": 141.51385001605377, |
| "max": 144.45185801014304 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 32.95013203751296, |
| "mean": 33.51937864984696, |
| "min": 32.933560956735164, |
| "max": 34.67444295529276 |
| }, |
| "e2e_ms": { |
| "median": 176.47038400173187, |
| "mean": 176.1472590151243, |
| "min": 174.5095690130256, |
| "max": 177.4618240306154 |
| } |
| } |
| }, |
| { |
| "id": "00009", |
| "image": "00009.jpg", |
| "question": "What is the third number of the store's phone number?", |
| "requested_family": "auto", |
| "selected_family": "phone", |
| "token_count": 21, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1038, |
| 5, |
| 6, |
| 12, |
| 12, |
| 18, |
| 16, |
| 16, |
| 17, |
| 17, |
| 7, |
| 8, |
| 1063, |
| 15, |
| 9, |
| 10, |
| 18, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 20, |
| "selected_family_id": 0, |
| "encoder_ms": { |
| "median": 140.45495504979044, |
| "mean": 139.57725366344675, |
| "min": 136.6604719660245, |
| "max": 141.61633397452533 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 17.8618700010702, |
| "mean": 18.15502964503442, |
| "min": 17.797669977881014, |
| "max": 18.80554895615205 |
| }, |
| "e2e_ms": { |
| "median": 159.3415469978936, |
| "mean": 157.7961503256423, |
| "min": 154.57697497913614, |
| "max": 159.4699289998971 |
| } |
| } |
| }, |
| { |
| "id": "00010", |
| "image": "00010.jpg", |
| "question": "What is the third number of the store's phone number?", |
| "requested_family": "auto", |
| "selected_family": "phone", |
| "token_count": 21, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1038, |
| 5, |
| 6, |
| 17, |
| 17, |
| 19, |
| 19, |
| 14, |
| 18, |
| 13, |
| 7, |
| 8, |
| 1063, |
| 15, |
| 9, |
| 10, |
| 19, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 20, |
| "selected_family_id": 0, |
| "encoder_ms": { |
| "median": 137.2203869977966, |
| "mean": 137.77373333383971, |
| "min": 136.17218099534512, |
| "max": 139.92863200837746 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 20.692456979304552, |
| "mean": 19.815429657076795, |
| "min": 18.006792990490794, |
| "max": 20.74703900143504 |
| }, |
| "e2e_ms": { |
| "median": 156.9239590317011, |
| "mean": 157.64796402072534, |
| "min": 155.2857210044749, |
| "max": 160.73421202600002 |
| } |
| } |
| }, |
| { |
| "id": "00011", |
| "image": "00011.jpg", |
| "question": "The location of the store is Seongsanjungang St. [?]. (Fill the blank)", |
| "requested_family": "auto", |
| "selected_family": "address", |
| "token_count": 37, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1064, |
| 5, |
| 6, |
| 302, |
| 1317, |
| 344, |
| 1020, |
| 54, |
| 300, |
| 1125, |
| 1355, |
| 54, |
| 311, |
| 1216, |
| 340, |
| 330, |
| 54, |
| 335, |
| 328, |
| 54, |
| 1128, |
| 54, |
| 17, |
| 15, |
| 13, |
| 7, |
| 8, |
| 1295, |
| 9, |
| 10, |
| 17, |
| 15, |
| 13, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 36, |
| "selected_family_id": 1, |
| "encoder_ms": { |
| "median": 143.11863598413765, |
| "mean": 143.99165865809968, |
| "min": 142.37950299866498, |
| "max": 146.47683699149638 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 33.069156983401626, |
| "mean": 34.028114653968565, |
| "min": 32.75625698734075, |
| "max": 36.25892999116331 |
| }, |
| "e2e_ms": { |
| "median": 178.69595100637525, |
| "mean": 178.0818936531432, |
| "min": 176.24534096103162, |
| "max": 179.30438899202272 |
| } |
| } |
| }, |
| { |
| "id": "00012", |
| "image": "00012.jpg", |
| "question": "What is the second number of the store's phone number?", |
| "requested_family": "auto", |
| "selected_family": "phone", |
| "token_count": 21, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1038, |
| 5, |
| 6, |
| 14, |
| 20, |
| 12, |
| 15, |
| 15, |
| 14, |
| 17, |
| 7, |
| 8, |
| 1063, |
| 14, |
| 9, |
| 10, |
| 20, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 20, |
| "selected_family_id": 0, |
| "encoder_ms": { |
| "median": 137.46910699410364, |
| "mean": 137.48986933690807, |
| "min": 137.29785400209948, |
| "max": 137.70264701452106 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 17.865658039227128, |
| "mean": 17.89760801087444, |
| "min": 17.852763005066663, |
| "max": 17.97440298832953 |
| }, |
| "e2e_ms": { |
| "median": 155.38061095867306, |
| "mean": 155.44563031289726, |
| "min": 155.33117798622698, |
| "max": 155.62510199379176 |
| } |
| } |
| }, |
| { |
| "id": "00013", |
| "image": "00013.jpg", |
| "question": "The location of the store is Hagusan St. [?]. (Fill the blank)", |
| "requested_family": "auto", |
| "selected_family": "address", |
| "token_count": 34, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1064, |
| 5, |
| 6, |
| 295, |
| 1026, |
| 336, |
| 1065, |
| 54, |
| 1471, |
| 1484, |
| 54, |
| 300, |
| 1095, |
| 323, |
| 342, |
| 332, |
| 54, |
| 1128, |
| 54, |
| 16, |
| 16, |
| 21, |
| 7, |
| 8, |
| 1295, |
| 9, |
| 10, |
| 16, |
| 16, |
| 21, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 33, |
| "selected_family_id": 1, |
| "encoder_ms": { |
| "median": 143.74126098118722, |
| "mean": 143.4886979792888, |
| "min": 140.56316000642255, |
| "max": 146.16167295025662 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 29.90899997530505, |
| "mean": 30.725151979519676, |
| "min": 29.845640994608402, |
| "max": 32.42081496864557 |
| }, |
| "e2e_ms": { |
| "median": 173.64382895175368, |
| "mean": 174.27336865027124, |
| "min": 173.042084032204, |
| "max": 176.13419296685606 |
| } |
| } |
| }, |
| { |
| "id": "00014", |
| "image": "00014.jpg", |
| "question": "The location of the store is Yongjeonsaeteo St. [?]. (Fill the blank)", |
| "requested_family": "auto", |
| "selected_family": "address", |
| "token_count": 35, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1064, |
| 5, |
| 6, |
| 302, |
| 326, |
| 1355, |
| 54, |
| 301, |
| 335, |
| 331, |
| 326, |
| 54, |
| 317, |
| 1125, |
| 1283, |
| 323, |
| 336, |
| 54, |
| 1128, |
| 54, |
| 18, |
| 16, |
| 12, |
| 7, |
| 8, |
| 1295, |
| 9, |
| 10, |
| 18, |
| 16, |
| 12, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 34, |
| "selected_family_id": 1, |
| "encoder_ms": { |
| "median": 146.05978096369654, |
| "mean": 148.7660360095712, |
| "min": 145.43723803944886, |
| "max": 154.80108902556822 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 30.837270023766905, |
| "mean": 31.616675997308146, |
| "min": 30.832752003334463, |
| "max": 33.18000596482307 |
| }, |
| "e2e_ms": { |
| "median": 176.95632297545671, |
| "mean": 180.44074430751303, |
| "min": 176.32312898058444, |
| "max": 188.04278096649796 |
| } |
| } |
| }, |
| { |
| "id": "00015", |
| "image": "00015.jpg", |
| "question": "The location of the store is Anpo St. [?]. (Fill the blank)", |
| "requested_family": "auto", |
| "selected_family": "address", |
| "token_count": 38, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1064, |
| 5, |
| 6, |
| 299, |
| 1243, |
| 1023, |
| 326, |
| 54, |
| 295, |
| 1026, |
| 1125, |
| 1484, |
| 54, |
| 293, |
| 335, |
| 342, |
| 1249, |
| 1139, |
| 326, |
| 1317, |
| 54, |
| 1128, |
| 54, |
| 16, |
| 19, |
| 15, |
| 7, |
| 8, |
| 1295, |
| 9, |
| 10, |
| 16, |
| 19, |
| 15, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 37, |
| "selected_family_id": 1, |
| "encoder_ms": { |
| "median": 143.99424800649285, |
| "mean": 143.51443034441522, |
| "min": 141.99849602300674, |
| "max": 144.55054700374603 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 33.75143400626257, |
| "mean": 34.56024868258586, |
| "min": 33.58327702153474, |
| "max": 36.346035019960254 |
| }, |
| "e2e_ms": { |
| "median": 178.20059903897345, |
| "mean": 178.13562702698013, |
| "min": 175.8087810012512, |
| "max": 180.39750104071572 |
| } |
| } |
| }, |
| { |
| "id": "00016", |
| "image": "00016.jpg", |
| "question": "The location of the store is Jaeunhyang St. [?]. (Fill the blank)", |
| "requested_family": "auto", |
| "selected_family": "address", |
| "token_count": 37, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1064, |
| 5, |
| 6, |
| 299, |
| 344, |
| 1168, |
| 1484, |
| 54, |
| 317, |
| 326, |
| 1020, |
| 331, |
| 326, |
| 54, |
| 302, |
| 326, |
| 1317, |
| 337, |
| 1380, |
| 54, |
| 1128, |
| 54, |
| 19, |
| 15, |
| 20, |
| 7, |
| 8, |
| 1295, |
| 9, |
| 10, |
| 19, |
| 15, |
| 20, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 36, |
| "selected_family_id": 1, |
| "encoder_ms": { |
| "median": 143.63564300583676, |
| "mean": 144.31006063629562, |
| "min": 143.0889619514346, |
| "max": 146.2055769516155 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 32.95267798239365, |
| "mean": 33.8629106602942, |
| "min": 32.54993900191039, |
| "max": 36.086114996578544 |
| }, |
| "e2e_ms": { |
| "median": 179.22203498892486, |
| "mean": 178.23103066378584, |
| "min": 175.69628899218515, |
| "max": 179.7747680102475 |
| } |
| } |
| }, |
| { |
| "id": "00017", |
| "image": "00017.jpg", |
| "question": "The location of the store is Namchangjangteo St. [?]. (Fill the blank)", |
| "requested_family": "auto", |
| "selected_family": "address", |
| "token_count": 33, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1064, |
| 5, |
| 6, |
| 306, |
| 1034, |
| 1484, |
| 1291, |
| 54, |
| 302, |
| 1065, |
| 1385, |
| 54, |
| 293, |
| 331, |
| 1317, |
| 54, |
| 1128, |
| 54, |
| 14, |
| 16, |
| 21, |
| 7, |
| 8, |
| 1295, |
| 9, |
| 10, |
| 14, |
| 16, |
| 21, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 32, |
| "selected_family_id": 1, |
| "encoder_ms": { |
| "median": 144.28778202272952, |
| "mean": 144.19136698901033, |
| "min": 143.3563659666106, |
| "max": 144.92995297769085 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 29.382618027739227, |
| "mean": 30.325759667903185, |
| "min": 29.346079973038286, |
| "max": 32.24858100293204 |
| }, |
| "e2e_ms": { |
| "median": 173.70053700869903, |
| "mean": 174.58046200529984, |
| "min": 172.8026139899157, |
| "max": 177.23823501728475 |
| } |
| } |
| }, |
| { |
| "id": "00018", |
| "image": "00018.jpg", |
| "question": "The location of the store is Nulha St. [?]. (Fill the blank)", |
| "requested_family": "auto", |
| "selected_family": "address", |
| "token_count": 33, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1064, |
| 5, |
| 6, |
| 296, |
| 1125, |
| 1095, |
| 54, |
| 311, |
| 1216, |
| 1452, |
| 54, |
| 305, |
| 1349, |
| 1501, |
| 336, |
| 54, |
| 1128, |
| 54, |
| 16, |
| 18, |
| 15, |
| 7, |
| 8, |
| 1295, |
| 9, |
| 10, |
| 16, |
| 18, |
| 15, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 32, |
| "selected_family_id": 1, |
| "encoder_ms": { |
| "median": 144.9581069755368, |
| "mean": 147.93412965567163, |
| "min": 141.64840697776526, |
| "max": 157.1958750137128 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 29.792201006785035, |
| "mean": 30.469847668427974, |
| "min": 29.66881700558588, |
| "max": 31.948524992913008 |
| }, |
| "e2e_ms": { |
| "median": 176.9661140278913, |
| "mean": 178.46580765520534, |
| "min": 171.385814959649, |
| "max": 187.04549397807568 |
| } |
| } |
| }, |
| { |
| "id": "00019", |
| "image": "00019.jpg", |
| "question": "What is the second number of the store's phone number?", |
| "requested_family": "auto", |
| "selected_family": "phone", |
| "token_count": 21, |
| "measurement": { |
| "tokens": [ |
| 1, |
| 4, |
| 1038, |
| 5, |
| 6, |
| 12, |
| 12, |
| 16, |
| 12, |
| 20, |
| 20, |
| 14, |
| 7, |
| 8, |
| 1063, |
| 14, |
| 9, |
| 10, |
| 12, |
| 11, |
| 2 |
| ], |
| "decoder_calls": 20, |
| "selected_family_id": 0, |
| "encoder_ms": { |
| "median": 140.01746400026605, |
| "mean": 139.87463398370892, |
| "min": 137.54436100134626, |
| "max": 142.06207694951445 |
| }, |
| "cross_kv_ms": { |
| "median": 0.0, |
| "mean": 0.0, |
| "min": 0.0, |
| "max": 0.0 |
| }, |
| "decoder_ms": { |
| "median": 18.12375499866903, |
| "mean": 18.404580652713776, |
| "min": 17.714502988383174, |
| "max": 19.375483971089125 |
| }, |
| "e2e_ms": { |
| "median": 158.19780400488526, |
| "mean": 158.33805835184953, |
| "min": 155.31836601439863, |
| "max": 161.49800503626466 |
| } |
| } |
| } |
| ], |
| "artifacts": { |
| "encoder": { |
| "filename": "encoder_model_int8.onnx", |
| "bytes": 18666798, |
| "sha256": "a5be30f7507f8e2988c4cf735a80fb6c09a4f1e93c9759b1fc5e12d04c4da45c" |
| }, |
| "decoder": { |
| "filename": "decoder_model_int8.onnx", |
| "bytes": 10062684, |
| "sha256": "75437beed7cdac93b4a704e9a4ffbefbe0199855344a57cb8f78ae105e915419" |
| } |
| } |
| } |
|
|
