cfahlgren1 HF Staff
v2: training script, eval harness, eval set and baseline metrics
3f86878 verified | { | |
| "model": "tinynext-v1", | |
| "sees_after": false, | |
| "cutoffs": { | |
| "0.3": { | |
| "articles/typing": { | |
| "pauses": 1000, | |
| "shown": 0.11, | |
| "right": 0.4909, | |
| "avg_right_chars": 5.24, | |
| "multiword_shown": 0.019, | |
| "multiword_right": 0.3684, | |
| "saved_with_tab": 0.0236, | |
| "saved_word_by_word": 0.0252 | |
| }, | |
| "articles/editing": { | |
| "pauses": 1000, | |
| "shown": 0.081, | |
| "right": 0.5309, | |
| "avg_right_chars": 5.37, | |
| "multiword_shown": 0.018, | |
| "multiword_right": 0.6111, | |
| "saved_with_tab": 0.0192, | |
| "saved_word_by_word": 0.0199 | |
| }, | |
| "messages/typing": { | |
| "pauses": 1000, | |
| "shown": 0.137, | |
| "right": 0.4891, | |
| "avg_right_chars": 5.25, | |
| "multiword_shown": 0.042, | |
| "multiword_right": 0.5238, | |
| "saved_with_tab": 0.0293, | |
| "saved_word_by_word": 0.0313 | |
| }, | |
| "messages/editing": { | |
| "pauses": 1000, | |
| "shown": 0.141, | |
| "right": 0.4539, | |
| "avg_right_chars": 5.39, | |
| "multiword_shown": 0.043, | |
| "multiword_right": 0.4419, | |
| "saved_with_tab": 0.0288, | |
| "saved_word_by_word": 0.0316 | |
| }, | |
| "email/typing": { | |
| "pauses": 1000, | |
| "shown": 0.182, | |
| "right": 0.5165, | |
| "avg_right_chars": 5.76, | |
| "multiword_shown": 0.042, | |
| "multiword_right": 0.3333, | |
| "saved_with_tab": 0.0451, | |
| "saved_word_by_word": 0.0483 | |
| }, | |
| "email/editing": { | |
| "pauses": 1000, | |
| "shown": 0.188, | |
| "right": 0.5372, | |
| "avg_right_chars": 5.47, | |
| "multiword_shown": 0.041, | |
| "multiword_right": 0.4878, | |
| "saved_with_tab": 0.046, | |
| "saved_word_by_word": 0.0486 | |
| } | |
| }, | |
| "0.4": { | |
| "articles/typing": { | |
| "pauses": 1000, | |
| "shown": 0.074, | |
| "right": 0.6081, | |
| "avg_right_chars": 5.07, | |
| "multiword_shown": 0.007, | |
| "multiword_right": 0.4286, | |
| "saved_with_tab": 0.019, | |
| "saved_word_by_word": 0.0192 | |
| }, | |
| "articles/editing": { | |
| "pauses": 1000, | |
| "shown": 0.057, | |
| "right": 0.5614, | |
| "avg_right_chars": 5.34, | |
| "multiword_shown": 0.007, | |
| "multiword_right": 0.8571, | |
| "saved_with_tab": 0.0143, | |
| "saved_word_by_word": 0.0143 | |
| }, | |
| "messages/typing": { | |
| "pauses": 1000, | |
| "shown": 0.092, | |
| "right": 0.6304, | |
| "avg_right_chars": 5.14, | |
| "multiword_shown": 0.026, | |
| "multiword_right": 0.5769, | |
| "saved_with_tab": 0.0248, | |
| "saved_word_by_word": 0.0261 | |
| }, | |
| "messages/editing": { | |
| "pauses": 1000, | |
| "shown": 0.094, | |
| "right": 0.5638, | |
| "avg_right_chars": 5.26, | |
| "multiword_shown": 0.019, | |
| "multiword_right": 0.5789, | |
| "saved_with_tab": 0.0232, | |
| "saved_word_by_word": 0.0239 | |
| }, | |
| "email/typing": { | |
| "pauses": 1000, | |
| "shown": 0.13, | |
| "right": 0.6, | |
| "avg_right_chars": 5.63, | |
| "multiword_shown": 0.024, | |
| "multiword_right": 0.4583, | |
| "saved_with_tab": 0.0366, | |
| "saved_word_by_word": 0.038 | |
| }, | |
| "email/editing": { | |
| "pauses": 1000, | |
| "shown": 0.143, | |
| "right": 0.6224, | |
| "avg_right_chars": 5.39, | |
| "multiword_shown": 0.023, | |
| "multiword_right": 0.6957, | |
| "saved_with_tab": 0.04, | |
| "saved_word_by_word": 0.0408 | |
| } | |
| }, | |
| "0.5": { | |
| "articles/typing": { | |
| "pauses": 1000, | |
| "shown": 0.051, | |
| "right": 0.7059, | |
| "avg_right_chars": 5.06, | |
| "multiword_shown": 0.002, | |
| "multiword_right": 0.5, | |
| "saved_with_tab": 0.0152, | |
| "saved_word_by_word": 0.0152 | |
| }, | |
| "articles/editing": { | |
| "pauses": 1000, | |
| "shown": 0.037, | |
| "right": 0.6486, | |
| "avg_right_chars": 5.12, | |
| "multiword_shown": 0.005, | |
| "multiword_right": 1.0, | |
| "saved_with_tab": 0.0103, | |
| "saved_word_by_word": 0.0103 | |
| }, | |
| "messages/typing": { | |
| "pauses": 1000, | |
| "shown": 0.062, | |
| "right": 0.7581, | |
| "avg_right_chars": 5.04, | |
| "multiword_shown": 0.01, | |
| "multiword_right": 0.8, | |
| "saved_with_tab": 0.0198, | |
| "saved_word_by_word": 0.0198 | |
| }, | |
| "messages/editing": { | |
| "pauses": 1000, | |
| "shown": 0.071, | |
| "right": 0.6338, | |
| "avg_right_chars": 5.16, | |
| "multiword_shown": 0.009, | |
| "multiword_right": 0.5556, | |
| "saved_with_tab": 0.0193, | |
| "saved_word_by_word": 0.0197 | |
| }, | |
| "email/typing": { | |
| "pauses": 1000, | |
| "shown": 0.093, | |
| "right": 0.6774, | |
| "avg_right_chars": 5.68, | |
| "multiword_shown": 0.016, | |
| "multiword_right": 0.4375, | |
| "saved_with_tab": 0.0298, | |
| "saved_word_by_word": 0.0305 | |
| }, | |
| "email/editing": { | |
| "pauses": 1000, | |
| "shown": 0.098, | |
| "right": 0.6939, | |
| "avg_right_chars": 5.22, | |
| "multiword_shown": 0.009, | |
| "multiword_right": 0.6667, | |
| "saved_with_tab": 0.0296, | |
| "saved_word_by_word": 0.0297 | |
| } | |
| }, | |
| "0.6": { | |
| "articles/typing": { | |
| "pauses": 1000, | |
| "shown": 0.036, | |
| "right": 0.8056, | |
| "avg_right_chars": 5.07, | |
| "multiword_shown": 0.002, | |
| "multiword_right": 0.5, | |
| "saved_with_tab": 0.0123, | |
| "saved_word_by_word": 0.0123 | |
| }, | |
| "articles/editing": { | |
| "pauses": 1000, | |
| "shown": 0.024, | |
| "right": 0.6667, | |
| "avg_right_chars": 5.0, | |
| "multiword_shown": 0.003, | |
| "multiword_right": 1.0, | |
| "saved_with_tab": 0.0067, | |
| "saved_word_by_word": 0.0067 | |
| }, | |
| "messages/typing": { | |
| "pauses": 1000, | |
| "shown": 0.041, | |
| "right": 0.7805, | |
| "avg_right_chars": 4.94, | |
| "multiword_shown": 0.007, | |
| "multiword_right": 0.8571, | |
| "saved_with_tab": 0.0132, | |
| "saved_word_by_word": 0.0132 | |
| }, | |
| "messages/editing": { | |
| "pauses": 1000, | |
| "shown": 0.045, | |
| "right": 0.6667, | |
| "avg_right_chars": 5.0, | |
| "multiword_shown": 0.003, | |
| "multiword_right": 0.3333, | |
| "saved_with_tab": 0.0125, | |
| "saved_word_by_word": 0.0125 | |
| }, | |
| "email/typing": { | |
| "pauses": 1000, | |
| "shown": 0.056, | |
| "right": 0.75, | |
| "avg_right_chars": 4.9, | |
| "multiword_shown": 0.004, | |
| "multiword_right": 0.0, | |
| "saved_with_tab": 0.0172, | |
| "saved_word_by_word": 0.0174 | |
| }, | |
| "email/editing": { | |
| "pauses": 1000, | |
| "shown": 0.067, | |
| "right": 0.7313, | |
| "avg_right_chars": 4.86, | |
| "multiword_shown": 0.004, | |
| "multiword_right": 0.75, | |
| "saved_with_tab": 0.0198, | |
| "saved_word_by_word": 0.0198 | |
| } | |
| }, | |
| "0.7": { | |
| "articles/typing": { | |
| "pauses": 1000, | |
| "shown": 0.025, | |
| "right": 0.84, | |
| "avg_right_chars": 5.1, | |
| "multiword_shown": 0.001, | |
| "multiword_right": 1.0, | |
| "saved_with_tab": 0.0089, | |
| "saved_word_by_word": 0.0089 | |
| }, | |
| "articles/editing": { | |
| "pauses": 1000, | |
| "shown": 0.018, | |
| "right": 0.6667, | |
| "avg_right_chars": 5.0, | |
| "multiword_shown": 0.001, | |
| "multiword_right": 1.0, | |
| "saved_with_tab": 0.005, | |
| "saved_word_by_word": 0.005 | |
| }, | |
| "messages/typing": { | |
| "pauses": 1000, | |
| "shown": 0.025, | |
| "right": 0.84, | |
| "avg_right_chars": 4.9, | |
| "multiword_shown": 0.005, | |
| "multiword_right": 0.8, | |
| "saved_with_tab": 0.0086, | |
| "saved_word_by_word": 0.0086 | |
| }, | |
| "messages/editing": { | |
| "pauses": 1000, | |
| "shown": 0.036, | |
| "right": 0.6944, | |
| "avg_right_chars": 4.96, | |
| "multiword_shown": 0.002, | |
| "multiword_right": 0.5, | |
| "saved_with_tab": 0.0103, | |
| "saved_word_by_word": 0.0103 | |
| }, | |
| "email/typing": { | |
| "pauses": 1000, | |
| "shown": 0.042, | |
| "right": 0.8333, | |
| "avg_right_chars": 4.89, | |
| "multiword_shown": 0.001, | |
| "multiword_right": 0.0, | |
| "saved_with_tab": 0.0143, | |
| "saved_word_by_word": 0.0143 | |
| }, | |
| "email/editing": { | |
| "pauses": 1000, | |
| "shown": 0.047, | |
| "right": 0.8298, | |
| "avg_right_chars": 4.95, | |
| "multiword_shown": 0.002, | |
| "multiword_right": 1.0, | |
| "saved_with_tab": 0.0161, | |
| "saved_word_by_word": 0.0161 | |
| } | |
| } | |
| }, | |
| "examples": [ | |
| { | |
| "domain": "messages", | |
| "scenario": "typing", | |
| "before": "onsidered new. Those decisions make the graphs as neat as po", | |
| "suggestion": "ssible", | |
| "truth": "ssible, and it is ", | |
| "right": true | |
| }, | |
| { | |
| "domain": "email", | |
| "scenario": "editing", | |
| "before": "ured products and marketing of electricity for the United St", | |
| "suggestion": "ates", | |
| "truth": "ates ? Actively ", | |
| "right": true | |
| } | |
| ], | |
| "latency_ms": { | |
| "p50": 4.22, | |
| "p95": 16.52, | |
| "note": "Measured in this eval process, not in a browser." | |
| } | |
| } |