Hugging Face's logo

ivere27
/
tiny-receipt-vqa-structured-qa-21m

Visual Question Answering
ONNX
English
Korean
onnxruntime
document-question-answering
receipt
key-information-extraction
lora
routing
Model card Files
xet
 Community
tiny-receipt-vqa-structured-qa-21m / eval /runtime_benchmark_fp32_cpu.json
ivere27's picture
ivere27
TinyReceiptVQA structured-QA 21M — split KV ONNX, FP32 and static W8A8
f536c6e
Raw History Blame Contribute Delete
29.8 kB
{
"format": "tiny_receipt_vqa_runtime_benchmark_v1",
"precision": "fp32",
"providers": [
"CPUExecutionProvider"
],
"threads": 1,
"timing_scope": "model inference from encoder input through EOS; session creation and image/token preprocessing excluded",
"warmups_per_vqa": 1,
"measured_repeats_per_vqa": 3,
"vqa_count": 20,
"aggregate": {
"e2e_median_of_vqa_medians_ms": 288.50183452595957
},
"results": [
{
"id": "00000",
"image": "00000.jpg",
"question": "The location of the store is Hyocheon St. [?]. (Fill the blank)",
"requested_family": "auto",
"selected_family": "address",
"token_count": 37,
"measurement": {
"tokens": [
1,
4,
1064,
5,
6,
299,
342,
339,
330,
54,
306,
1034,
1023,
326,
54,
300,
336,
1345,
1111,
326,
1349,
54,
1128,
54,
14,
12,
21,
7,
8,
1295,
9,
10,
14,
12,
21,
11,
2
],
"decoder_calls": 36,
"selected_family_id": 1,
"encoder_ms": {
"median": 228.1085120048374,
"mean": 228.73557733449465,
"min": 224.96837296057492,
"max": 233.1298470380716
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 62.75751499924809,
"mean": 63.27293966508781,
"min": 60.33691199263558,
"max": 66.72439200337976
},
"e2e_ms": {
"median": 290.924838045612,
"mean": 292.0872653485276,
"min": 285.4173560044728,
"max": 299.91960199549794
}
}
},
{
"id": "00001",
"image": "00001.jpg",
"question": "What is the first number of the store's phone number?",
"requested_family": "auto",
"selected_family": "phone",
"token_count": 21,
"measurement": {
"tokens": [
1,
4,
1038,
5,
6,
14,
17,
20,
18,
14,
15,
20,
7,
8,
1063,
13,
9,
10,
14,
11,
2
],
"decoder_calls": 20,
"selected_family_id": 0,
"encoder_ms": {
"median": 218.68814597837627,
"mean": 218.8280163294015,
"min": 216.8369370047003,
"max": 220.95896600512788
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 34.41185498377308,
"mean": 34.29271699860692,
"min": 32.77926699956879,
"max": 35.68702901247889
},
"e2e_ms": {
"median": 252.58133298484609,
"mean": 253.1810699729249,
"min": 251.52513198554516,
"max": 255.43674494838342
}
}
},
{
"id": "00002",
"image": "00002.jpg",
"question": "What is the fourth number of the store's phone number?",
"requested_family": "auto",
"selected_family": "phone",
"token_count": 21,
"measurement": {
"tokens": [
1,
4,
1038,
5,
6,
14,
20,
17,
13,
12,
19,
15,
7,
8,
1063,
16,
9,
10,
13,
11,
2
],
"decoder_calls": 20,
"selected_family_id": 0,
"encoder_ms": {
"median": 219.080465962179,
"mean": 220.12510864684978,
"min": 218.29582698410377,
"max": 222.9990329942666
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 36.60359600326046,
"mean": 36.09063268716758,
"min": 34.20854202704504,
"max": 37.45976003119722
},
"e2e_ms": {
"median": 255.7410699664615,
"mean": 256.27760131222504,
"min": 252.57335900096223,
"max": 260.51837496925145
}
}
},
{
"id": "00003",
"image": "00003.jpg",
"question": "The location of the store is Seonggokbuk St. [?]. (Fill the blank)",
"requested_family": "auto",
"selected_family": "address",
"token_count": 37,
"measurement": {
"tokens": [
1,
4,
1064,
5,
6,
305,
1369,
337,
336,
54,
311,
1216,
1452,
54,
1303,
1089,
54,
336,
340,
322,
1131,
54,
1128,
54,
13,
12,
13,
7,
8,
1295,
9,
10,
13,
12,
13,
11,
2
],
"decoder_calls": 36,
"selected_family_id": 1,
"encoder_ms": {
"median": 234.17181300465018,
"mean": 232.15498200928172,
"min": 225.09921196615323,
"max": 237.19392105704173
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 62.14503099909052,
"mean": 60.96253534390902,
"min": 58.41115303337574,
"max": 62.33142199926078
},
"e2e_ms": {
"median": 292.640573985409,
"mean": 293.1756866552557,
"min": 287.4870509840548,
"max": 299.39943499630317
}
}
},
{
"id": "00004",
"image": "00004.jpg",
"question": "What is the fourth number of the store's phone number?",
"requested_family": "auto",
"selected_family": "phone",
"token_count": 21,
"measurement": {
"tokens": [
1,
4,
1038,
5,
6,
17,
16,
13,
17,
19,
18,
20,
7,
8,
1063,
16,
9,
10,
17,
11,
2
],
"decoder_calls": 20,
"selected_family_id": 0,
"encoder_ms": {
"median": 219.73750600591302,
"mean": 219.55031366087496,
"min": 218.67956197820604,
"max": 220.23387299850583
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 33.03238400258124,
"mean": 32.85907100265225,
"min": 32.23047300707549,
"max": 33.314355998300016
},
"e2e_ms": {
"median": 252.05038499552757,
"mean": 252.4662456708029,
"min": 252.02497700229287,
"max": 253.32337501458824
}
}
},
{
"id": "00005",
"image": "00005.jpg",
"question": "What is the third number of the store's phone number?",
"requested_family": "auto",
"selected_family": "phone",
"token_count": 21,
"measurement": {
"tokens": [
1,
4,
1038,
5,
6,
17,
14,
15,
17,
14,
19,
19,
7,
8,
1063,
15,
9,
10,
15,
11,
2
],
"decoder_calls": 20,
"selected_family_id": 0,
"encoder_ms": {
"median": 221.06853598961607,
"mean": 222.28092465472096,
"min": 220.07366101024672,
"max": 225.70057696430013
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 34.189306024927646,
"mean": 34.80713399282346,
"min": 32.518005988094956,
"max": 37.71408996544778
},
"e2e_ms": {
"median": 258.8410570169799,
"mean": 257.14559699796763,
"min": 252.6491439784877,
"max": 259.94658999843523
}
}
},
{
"id": "00006",
"image": "00006.jpg",
"question": "What is the third number of the store's phone number?",
"requested_family": "auto",
"selected_family": "phone",
"token_count": 21,
"measurement": {
"tokens": [
1,
4,
1038,
5,
6,
17,
16,
19,
19,
20,
17,
14,
7,
8,
1063,
15,
9,
10,
19,
11,
2
],
"decoder_calls": 20,
"selected_family_id": 0,
"encoder_ms": {
"median": 222.55956701701507,
"mean": 222.51495001061508,
"min": 220.9344040020369,
"max": 224.0508790127933
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 34.30454304907471,
"mean": 35.38931436681499,
"min": 33.86030503315851,
"max": 38.00309501821175
},
"e2e_ms": {
"median": 257.96999502927065,
"mean": 257.9631620125535,
"min": 255.29778801137581,
"max": 260.6217029970139
}
}
},
{
"id": "00007",
"image": "00007.jpg",
"question": "The location of the store is Hwadoan St. [?]. (Fill the blank)",
"requested_family": "auto",
"selected_family": "address",
"token_count": 37,
"measurement": {
"tokens": [
1,
4,
1064,
5,
6,
311,
1035,
1026,
1020,
54,
299,
336,
1463,
54,
300,
344,
1375,
1072,
54,
335,
328,
54,
1128,
54,
19,
19,
17,
7,
8,
1295,
9,
10,
19,
19,
17,
11,
2
],
"decoder_calls": 36,
"selected_family_id": 1,
"encoder_ms": {
"median": 229.6274010441266,
"mean": 228.94571267534047,
"min": 226.80862696142867,
"max": 230.40111002046615
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 62.39598500542343,
"mean": 62.76772233347098,
"min": 61.7439360357821,
"max": 64.16324595920742
},
"e2e_ms": {
"median": 292.20226302277297,
"mean": 291.7710263476086,
"min": 289.26231001969427,
"max": 293.8485060003586
}
}
},
{
"id": "00008",
"image": "00008.jpg",
"question": "The location of the store is Heowanghu St. [?]. (Fill the blank)",
"requested_family": "auto",
"selected_family": "address",
"token_count": 38,
"measurement": {
"tokens": [
1,
4,
1064,
5,
6,
299,
344,
1168,
334,
1380,
54,
317,
326,
1020,
1355,
54,
300,
326,
1113,
322,
1021,
330,
54,
1128,
54,
16,
20,
16,
7,
8,
1295,
9,
10,
16,
20,
16,
11,
2
],
"decoder_calls": 37,
"selected_family_id": 1,
"encoder_ms": {
"median": 230.19460099749267,
"mean": 230.10672200083113,
"min": 229.20020599849522,
"max": 230.92535900650546
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 60.01490697963163,
"mean": 60.61753230945518,
"min": 59.19383798027411,
"max": 62.643851968459785
},
"e2e_ms": {
"median": 291.00127099081874,
"mean": 290.7840140008678,
"min": 289.4475909997709,
"max": 291.90318001201376
}
}
},
{
"id": "00009",
"image": "00009.jpg",
"question": "What is the third number of the store's phone number?",
"requested_family": "auto",
"selected_family": "phone",
"token_count": 21,
"measurement": {
"tokens": [
1,
4,
1038,
5,
6,
12,
12,
18,
16,
16,
17,
17,
7,
8,
1063,
15,
9,
10,
18,
11,
2
],
"decoder_calls": 20,
"selected_family_id": 0,
"encoder_ms": {
"median": 224.44382100366056,
"mean": 225.2649093279615,
"min": 224.18938198825344,
"max": 227.1615249919705
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 33.209228015039116,
"mean": 33.426869000929095,
"min": 32.81561698531732,
"max": 34.255762002430856
},
"e2e_ms": {
"median": 257.70985695999116,
"mean": 258.76177366202074,
"min": 257.06289801746607,
"max": 261.51256600860506
}
}
},
{
"id": "00010",
"image": "00010.jpg",
"question": "What is the third number of the store's phone number?",
"requested_family": "auto",
"selected_family": "phone",
"token_count": 21,
"measurement": {
"tokens": [
1,
4,
1038,
5,
6,
17,
17,
19,
19,
14,
18,
13,
7,
8,
1063,
15,
9,
10,
19,
11,
2
],
"decoder_calls": 20,
"selected_family_id": 0,
"encoder_ms": {
"median": 224.5594309642911,
"mean": 224.42714798186594,
"min": 222.7793749771081,
"max": 225.94263800419867
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 34.14588398300111,
"mean": 34.801219667618476,
"min": 32.99805102869868,
"max": 37.259723991155624
},
"e2e_ms": {
"median": 258.76506802160293,
"mean": 259.28606300537166,
"min": 255.8331110049039,
"max": 263.2600099896081
}
}
},
{
"id": "00011",
"image": "00011.jpg",
"question": "The location of the store is Seongsanjungang St. [?]. (Fill the blank)",
"requested_family": "auto",
"selected_family": "address",
"token_count": 36,
"measurement": {
"tokens": [
1,
4,
1064,
5,
6,
312,
1348,
323,
1380,
54,
300,
1313,
1355,
54,
311,
1216,
1452,
54,
311,
54,
341,
288,
54,
17,
15,
13,
7,
8,
1295,
9,
10,
17,
15,
13,
11,
2
],
"decoder_calls": 35,
"selected_family_id": 1,
"encoder_ms": {
"median": 232.3369620135054,
"mean": 233.1079383341906,
"min": 232.2882090229541,
"max": 234.6986439661123
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 59.2429920216091,
"mean": 58.67435699716831,
"min": 56.36540800333023,
"max": 60.41467096656561
},
"e2e_ms": {
"median": 292.7600480034016,
"mean": 291.8443593468207,
"min": 288.7621429981664,
"max": 294.010887038894
}
}
},
{
"id": "00012",
"image": "00012.jpg",
"question": "What is the second number of the store's phone number?",
"requested_family": "auto",
"selected_family": "phone",
"token_count": 21,
"measurement": {
"tokens": [
1,
4,
1038,
5,
6,
14,
20,
12,
15,
15,
14,
17,
7,
8,
1063,
14,
9,
10,
20,
11,
2
],
"decoder_calls": 20,
"selected_family_id": 0,
"encoder_ms": {
"median": 226.1477159918286,
"mean": 229.65850731513152,
"min": 224.2999929585494,
"max": 238.52781299501657
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 33.429053029976785,
"mean": 34.00779399089515,
"min": 32.332515984307975,
"max": 36.2618129584007
},
"e2e_ms": {
"median": 262.48074299655855,
"mean": 263.72832199558616,
"min": 257.78591400012374,
"max": 270.9183089900762
}
}
},
{
"id": "00013",
"image": "00013.jpg",
"question": "The location of the store is Hagusan St. [?]. (Fill the blank)",
"requested_family": "auto",
"selected_family": "address",
"token_count": 34,
"measurement": {
"tokens": [
1,
4,
1064,
5,
6,
295,
1026,
336,
1065,
54,
1471,
1484,
54,
300,
1095,
323,
342,
332,
54,
1128,
54,
16,
16,
21,
7,
8,
1295,
9,
10,
16,
16,
21,
11,
2
],
"decoder_calls": 33,
"selected_family_id": 1,
"encoder_ms": {
"median": 236.18908500066027,
"mean": 235.46872369479388,
"min": 230.60361604439095,
"max": 239.61347003933042
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 56.88240297604352,
"mean": 55.94197332781429,
"min": 52.23298800410703,
"max": 58.71052900329232
},
"e2e_ms": {
"median": 291.9109900249168,
"mean": 291.4869276961933,
"min": 289.3731670337729,
"max": 293.17662602989003
}
}
},
{
"id": "00014",
"image": "00014.jpg",
"question": "The location of the store is Yongjeonsaeteo St. [?]. (Fill the blank)",
"requested_family": "auto",
"selected_family": "address",
"token_count": 35,
"measurement": {
"tokens": [
1,
4,
1064,
5,
6,
302,
326,
1355,
54,
301,
335,
331,
326,
54,
317,
1125,
1283,
323,
336,
54,
1128,
54,
18,
16,
12,
7,
8,
1295,
9,
10,
18,
16,
12,
11,
2
],
"decoder_calls": 34,
"selected_family_id": 1,
"encoder_ms": {
"median": 232.1202660095878,
"mean": 233.38799566651383,
"min": 231.80684499675408,
"max": 236.23687599319965
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 55.49834400881082,
"mean": 56.75215433196475,
"min": 54.934261017479,
"max": 59.82385796960443
},
"e2e_ms": {
"median": 291.79100500186905,
"mean": 290.1974580211875,
"min": 286.79970605298877,
"max": 292.0016630087048
}
}
},
{
"id": "00015",
"image": "00015.jpg",
"question": "The location of the store is Anpo St. [?]. (Fill the blank)",
"requested_family": "auto",
"selected_family": "address",
"token_count": 38,
"measurement": {
"tokens": [
1,
4,
1064,
5,
6,
299,
1243,
1023,
326,
54,
295,
1026,
1125,
1484,
54,
293,
335,
342,
1249,
1139,
326,
1317,
54,
1128,
54,
16,
19,
15,
7,
8,
1295,
9,
10,
16,
19,
15,
11,
2
],
"decoder_calls": 37,
"selected_family_id": 1,
"encoder_ms": {
"median": 229.83304603258148,
"mean": 230.01183301676065,
"min": 229.7786429990083,
"max": 230.42381001869217
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 63.79281001864001,
"mean": 64.01001233219479,
"min": 61.98739598039538,
"max": 66.249830997549
},
"e2e_ms": {
"median": 293.68569899816066,
"mean": 294.08051931144047,
"min": 292.4686739570461,
"max": 296.0871849791147
}
}
},
{
"id": "00016",
"image": "00016.jpg",
"question": "The location of the store is Jaeunhyang St. [?]. (Fill the blank)",
"requested_family": "auto",
"selected_family": "address",
"token_count": 36,
"measurement": {
"tokens": [
1,
4,
1064,
5,
6,
299,
344,
1035,
1026,
1020,
54,
317,
326,
1355,
54,
302,
326,
1317,
337,
1380,
54,
1128,
54,
19,
15,
20,
7,
8,
1295,
9,
10,
19,
15,
20,
11,
2
],
"decoder_calls": 35,
"selected_family_id": 1,
"encoder_ms": {
"median": 232.301420008298,
"mean": 233.35739333803454,
"min": 230.61087302630767,
"max": 237.15988697949797
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 60.91165903490037,
"mean": 61.055687353170164,
"min": 57.506710989400744,
"max": 64.74869203520939
},
"e2e_ms": {
"median": 294.7251980076544,
"mean": 294.4723453450327,
"min": 293.27324201585725,
"max": 295.4185960115865
}
}
},
{
"id": "00017",
"image": "00017.jpg",
"question": "The location of the store is Namchangjangteo St. [?]. (Fill the blank)",
"requested_family": "auto",
"selected_family": "address",
"token_count": 33,
"measurement": {
"tokens": [
1,
4,
1064,
5,
6,
306,
1034,
1484,
1291,
54,
302,
1065,
1385,
54,
293,
331,
1317,
54,
1128,
54,
14,
16,
21,
7,
8,
1295,
9,
10,
14,
16,
21,
11,
2
],
"decoder_calls": 32,
"selected_family_id": 1,
"encoder_ms": {
"median": 233.2940620253794,
"mean": 232.74352065830803,
"min": 231.19594698073342,
"max": 233.74055296881124
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 55.82460999721661,
"mean": 56.32064832995335,
"min": 55.750339990481734,
"max": 57.38699500216171
},
"e2e_ms": {
"median": 289.1032020561397,
"mean": 289.12289599732804,
"min": 288.6404799646698,
"max": 289.6250059711747
}
}
},
{
"id": "00018",
"image": "00018.jpg",
"question": "The location of the store is Nulha St. [?]. (Fill the blank)",
"requested_family": "auto",
"selected_family": "address",
"token_count": 33,
"measurement": {
"tokens": [
1,
4,
1064,
5,
6,
296,
1125,
1095,
54,
311,
342,
1380,
54,
315,
1020,
331,
1065,
54,
1128,
54,
16,
18,
15,
7,
8,
1295,
9,
10,
16,
18,
15,
11,
2
],
"decoder_calls": 32,
"selected_family_id": 1,
"encoder_ms": {
"median": 231.79093899670988,
"mean": 232.75611198429638,
"min": 230.4946249932982,
"max": 235.9827719628811
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 52.24784801248461,
"mean": 52.896186690001436,
"min": 50.39089702768251,
"max": 56.04981502983719
},
"e2e_ms": {
"median": 287.9004669957794,
"mean": 285.7119679877845,
"min": 280.9447349864058,
"max": 288.29070198116824
}
}
},
{
"id": "00019",
"image": "00019.jpg",
"question": "What is the second number of the store's phone number?",
"requested_family": "auto",
"selected_family": "phone",
"token_count": 21,
"measurement": {
"tokens": [
1,
4,
1038,
5,
6,
12,
12,
16,
12,
20,
20,
14,
7,
8,
1063,
14,
9,
10,
12,
11,
2
],
"decoder_calls": 20,
"selected_family_id": 0,
"encoder_ms": {
"median": 227.41803695680574,
"mean": 228.0839123025847,
"min": 225.56102694943547,
"max": 231.27267300151289
},
"cross_kv_ms": {
"median": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
},
"decoder_ms": {
"median": 34.86111399251968,
"mean": 34.86383269773796,
"min": 32.26642304798588,
"max": 37.46396105270833
},
"e2e_ms": {
"median": 260.49909502035007,
"mean": 263.01347836852074,
"min": 259.74823004798964,
"max": 268.7931100372225
}
}
}
],
"artifacts": {
"encoder": {
"filename": "encoder_model.onnx",
"bytes": 61811550,
"sha256": "0e2206c54756b15d697d2bbd8d22bb68d0d2475d0323965ca03a14fb186e6eb0"
},
"decoder": {
"filename": "decoder_model.onnx",
"bytes": 28616273,
"sha256": "b918a9de531bad205990acff28a2a7e2244b14229c312ed6b2f75d0540d6a6a2"
}
}
}