tinynext / v2 /metrics /tinynext-v2-first.json
cfahlgren1's picture
cfahlgren1 HF Staff
make the 3b-token run v2 and add the size comparison
a739c7e verified
Raw History Blame Contribute Delete
8.19 kB
{
"model": "tinynext-v2",
"sees_after": true,
"cutoffs": {
"0.3": {
"articles/typing": {
"pauses": 1000,
"shown": 0.188,
"right": 0.617,
"avg_right_chars": 5.91,
"multiword_shown": 0.067,
"multiword_right": 0.5373,
"saved_with_tab": 0.0572,
"saved_word_by_word": 0.0626
},
"articles/editing": {
"pauses": 1000,
"shown": 0.148,
"right": 0.6081,
"avg_right_chars": 5.94,
"multiword_shown": 0.058,
"multiword_right": 0.5862,
"saved_with_tab": 0.0446,
"saved_word_by_word": 0.0491
},
"messages/typing": {
"pauses": 1000,
"shown": 0.293,
"right": 0.5563,
"avg_right_chars": 7.71,
"multiword_shown": 0.163,
"multiword_right": 0.546,
"saved_with_tab": 0.1047,
"saved_word_by_word": 0.1232
},
"messages/editing": {
"pauses": 1000,
"shown": 0.276,
"right": 0.6159,
"avg_right_chars": 7.49,
"multiword_shown": 0.152,
"multiword_right": 0.6118,
"saved_with_tab": 0.1061,
"saved_word_by_word": 0.1183
},
"email/typing": {
"pauses": 1000,
"shown": 0.237,
"right": 0.5443,
"avg_right_chars": 6.09,
"multiword_shown": 0.084,
"multiword_right": 0.3929,
"saved_with_tab": 0.0655,
"saved_word_by_word": 0.0744
},
"email/editing": {
"pauses": 1000,
"shown": 0.22,
"right": 0.5273,
"avg_right_chars": 5.77,
"multiword_shown": 0.093,
"multiword_right": 0.4194,
"saved_with_tab": 0.0558,
"saved_word_by_word": 0.0645
}
},
"0.4": {
"articles/typing": {
"pauses": 1000,
"shown": 0.14,
"right": 0.7214,
"avg_right_chars": 5.94,
"multiword_shown": 0.043,
"multiword_right": 0.6744,
"saved_with_tab": 0.05,
"saved_word_by_word": 0.0522
},
"articles/editing": {
"pauses": 1000,
"shown": 0.102,
"right": 0.6667,
"avg_right_chars": 5.65,
"multiword_shown": 0.034,
"multiword_right": 0.7353,
"saved_with_tab": 0.032,
"saved_word_by_word": 0.0341
},
"messages/typing": {
"pauses": 1000,
"shown": 0.212,
"right": 0.6934,
"avg_right_chars": 7.19,
"multiword_shown": 0.105,
"multiword_right": 0.6571,
"saved_with_tab": 0.0881,
"saved_word_by_word": 0.0958
},
"messages/editing": {
"pauses": 1000,
"shown": 0.194,
"right": 0.7113,
"avg_right_chars": 6.83,
"multiword_shown": 0.097,
"multiword_right": 0.6907,
"saved_with_tab": 0.0785,
"saved_word_by_word": 0.086
},
"email/typing": {
"pauses": 1000,
"shown": 0.172,
"right": 0.686,
"avg_right_chars": 5.86,
"multiword_shown": 0.053,
"multiword_right": 0.434,
"saved_with_tab": 0.0576,
"saved_word_by_word": 0.0618
},
"email/editing": {
"pauses": 1000,
"shown": 0.147,
"right": 0.6395,
"avg_right_chars": 5.48,
"multiword_shown": 0.057,
"multiword_right": 0.5439,
"saved_with_tab": 0.0429,
"saved_word_by_word": 0.0459
}
},
"0.5": {
"articles/typing": {
"pauses": 1000,
"shown": 0.106,
"right": 0.7642,
"avg_right_chars": 5.9,
"multiword_shown": 0.034,
"multiword_right": 0.7353,
"saved_with_tab": 0.0398,
"saved_word_by_word": 0.0411
},
"articles/editing": {
"pauses": 1000,
"shown": 0.078,
"right": 0.7692,
"avg_right_chars": 5.55,
"multiword_shown": 0.022,
"multiword_right": 0.8636,
"saved_with_tab": 0.0278,
"saved_word_by_word": 0.0282
},
"messages/typing": {
"pauses": 1000,
"shown": 0.163,
"right": 0.7791,
"avg_right_chars": 6.86,
"multiword_shown": 0.078,
"multiword_right": 0.7179,
"saved_with_tab": 0.0726,
"saved_word_by_word": 0.078
},
"messages/editing": {
"pauses": 1000,
"shown": 0.146,
"right": 0.7877,
"avg_right_chars": 6.63,
"multiword_shown": 0.071,
"multiword_right": 0.7465,
"saved_with_tab": 0.0635,
"saved_word_by_word": 0.0679
},
"email/typing": {
"pauses": 1000,
"shown": 0.113,
"right": 0.7876,
"avg_right_chars": 5.84,
"multiword_shown": 0.03,
"multiword_right": 0.6,
"saved_with_tab": 0.0433,
"saved_word_by_word": 0.0452
},
"email/editing": {
"pauses": 1000,
"shown": 0.104,
"right": 0.7788,
"avg_right_chars": 5.32,
"multiword_shown": 0.03,
"multiword_right": 0.7,
"saved_with_tab": 0.0359,
"saved_word_by_word": 0.0369
}
},
"0.6": {
"articles/typing": {
"pauses": 1000,
"shown": 0.084,
"right": 0.8571,
"avg_right_chars": 5.86,
"multiword_shown": 0.022,
"multiword_right": 0.8636,
"saved_with_tab": 0.0352,
"saved_word_by_word": 0.0353
},
"articles/editing": {
"pauses": 1000,
"shown": 0.056,
"right": 0.8571,
"avg_right_chars": 5.62,
"multiword_shown": 0.016,
"multiword_right": 0.9375,
"saved_with_tab": 0.0225,
"saved_word_by_word": 0.0227
},
"messages/typing": {
"pauses": 1000,
"shown": 0.123,
"right": 0.8455,
"avg_right_chars": 6.73,
"multiword_shown": 0.06,
"multiword_right": 0.8167,
"saved_with_tab": 0.0583,
"saved_word_by_word": 0.061
},
"messages/editing": {
"pauses": 1000,
"shown": 0.108,
"right": 0.8426,
"avg_right_chars": 6.31,
"multiword_shown": 0.052,
"multiword_right": 0.7308,
"saved_with_tab": 0.0478,
"saved_word_by_word": 0.0512
},
"email/typing": {
"pauses": 1000,
"shown": 0.076,
"right": 0.8816,
"avg_right_chars": 5.79,
"multiword_shown": 0.016,
"multiword_right": 0.8125,
"saved_with_tab": 0.0323,
"saved_word_by_word": 0.0324
},
"email/editing": {
"pauses": 1000,
"shown": 0.071,
"right": 0.8873,
"avg_right_chars": 5.25,
"multiword_shown": 0.014,
"multiword_right": 0.8571,
"saved_with_tab": 0.0276,
"saved_word_by_word": 0.0279
}
},
"0.7": {
"articles/typing": {
"pauses": 1000,
"shown": 0.059,
"right": 0.9153,
"avg_right_chars": 5.76,
"multiword_shown": 0.018,
"multiword_right": 0.9444,
"saved_with_tab": 0.0259,
"saved_word_by_word": 0.0261
},
"articles/editing": {
"pauses": 1000,
"shown": 0.035,
"right": 0.8286,
"avg_right_chars": 5.34,
"multiword_shown": 0.007,
"multiword_right": 0.8571,
"saved_with_tab": 0.0129,
"saved_word_by_word": 0.0132
},
"messages/typing": {
"pauses": 1000,
"shown": 0.099,
"right": 0.9293,
"avg_right_chars": 6.37,
"multiword_shown": 0.042,
"multiword_right": 0.9048,
"saved_with_tab": 0.0488,
"saved_word_by_word": 0.0492
},
"messages/editing": {
"pauses": 1000,
"shown": 0.086,
"right": 0.8837,
"avg_right_chars": 5.97,
"multiword_shown": 0.036,
"multiword_right": 0.7222,
"saved_with_tab": 0.0378,
"saved_word_by_word": 0.0408
},
"email/typing": {
"pauses": 1000,
"shown": 0.05,
"right": 0.84,
"avg_right_chars": 5.45,
"multiword_shown": 0.006,
"multiword_right": 0.5,
"saved_with_tab": 0.0191,
"saved_word_by_word": 0.0192
},
"email/editing": {
"pauses": 1000,
"shown": 0.054,
"right": 0.9074,
"avg_right_chars": 5.16,
"multiword_shown": 0.01,
"multiword_right": 0.9,
"saved_with_tab": 0.0211,
"saved_word_by_word": 0.0211
}
}
},
"examples": [
{
"domain": "articles",
"scenario": "editing",
"before": " the Spaghetti Western film genre) and Hate Boat, were publi",
"suggestion": "shed",
"truth": "shed by Comico C",
"right": true
},
{
"domain": "messages",
"scenario": "editing",
"before": "e optimal level of production and pricing for businesses. L'",
"suggestion": "Hpital's Rule",
"truth": "Hpital's Rule can help an",
"right": true
},
{
"domain": "email",
"scenario": "editing",
"before": "ured products and marketing of electricity for the United St",
"suggestion": "ates",
"truth": "ates ? Actively ",
"right": true
}
],
"latency_ms": {
"p50": 16.88,
"p95": 43.43,
"note": "Measured in this eval process, not in a browser."
}
}