prism-os-functiongemma / results.json
AppleLampsX's picture
Round 3: 701/746 held-out (94.0%), browser build 95/100
97ead18 verified
Raw History Blame Contribute Delete
14.2 kB
{ "decoding_check": "4/4", "base": { "accuracy": 0.1447721179624665, "correct": 108, "total": 746, "by_kind": { "two calls": [ 0, 85 ], "photos previous": [ 0, 11 ], "reply": [ 68, 89 ], "timer cancel": [ 0, 3 ], "files create_document": [ 0, 11 ], "search_web": [ 1, 28 ], "files create_folder": [ 0, 13 ], "note read": [ 0, 9 ], "three calls": [ 0, 23 ], "files save_note_as": [ 0, 11 ], "settings accent": [ 1, 14 ], "photos open": [ 7, 18 ], "app move": [ 0, 23 ], "files move": [ 0, 9 ], "files empty_trash": [ 0, 5 ], "app close": [ 0, 8 ], "timer start": [ 13, 20 ], "files list": [ 0, 9 ], "explain": [ 0, 10 ], "files duplicate": [ 0, 7 ], "settings arcs": [ 0, 12 ], "world_time": [ 0, 12 ], "app restore": [ 0, 14 ], "settings motion": [ 0, 10 ], "note clear": [ 1, 10 ], "note append": [ 3, 12 ], "scope page_down": [ 0, 3 ], "layout reset": [ 0, 5 ], "files import": [ 0, 6 ], "scope top": [ 0, 3 ], "open_website": [ 2, 18 ], "files rename": [ 0, 8 ], "app open": [ 0, 13 ], "photos close": [ 1, 10 ], "files download": [ 0, 14 ], "browser forward": [ 0, 3 ], "calculate": [ 2, 17 ], "app maximize": [ 0, 10 ], "photos next": [ 1, 6 ], "layout show_desktop": [ 0, 8 ], "get_stats": [ 1, 8 ], "files open": [ 0, 12 ], "layout side_by_side": [ 0, 10 ], "timer pause": [ 0, 4 ], "stopwatch reset": [ 0, 4 ], "files delete": [ 0, 11 ], "files restore": [ 0, 5 ], "note replace": [ 0, 9 ], "app minimize": [ 0, 14 ], "photos start_slideshow": [ 4, 5 ], "settings theme": [ 0, 7 ], "files search": [ 1, 9 ], "stopwatch start": [ 0, 1 ], "scope down": [ 0, 3 ], "browser reload": [ 0, 1 ], "scope bottom": [ 0, 2 ], "app resize": [ 0, 11 ], "stopwatch lap": [ 0, 3 ], "files trash": [ 0, 3 ], "scope operations": [ 1, 3 ], "settings wallpaper": [ 0, 6 ], "browser home": [ 1, 2 ], "photos stop_slideshow": [ 0, 6 ], "scope state": [ 0, 2 ], "browser back": [ 0, 1 ], "scope up": [ 0, 1 ], "scope outputs": [ 0, 3 ] } }, "curve": [ { "step": 312, "epochs": 0.5, "accuracy": 0.8 }, { "step": 624, "epochs": 1.0, "accuracy": 0.9066666666666666 }, { "step": 936, "epochs": 1.5, "accuracy": 0.88 }, { "step": 1248, "epochs": 2.0, "accuracy": 0.9133333333333333 }, { "step": 1560, "epochs": 2.5, "accuracy": 0.94 }, { "step": 1872, "epochs": 3.0, "accuracy": 0.94 }, { "step": 2184, "epochs": 3.49, "accuracy": 0.9533333333333334 }, { "step": 2496, "epochs": 3.99, "accuracy": 0.9533333333333334 }, { "step": 2808, "epochs": 4.49, "accuracy": 0.9466666666666667 }, { "step": 3120, "epochs": 4.99, "accuracy": 0.9466666666666667 }, { "step": 3125, "epochs": 5.0, "accuracy": 0.9466666666666667 } ], "best_step": 2184, "train_minutes": 18.1, "fine_tuned": { "accuracy": 0.9396782841823056, "correct": 701, "total": 746, "by_kind": { "two calls": [ 85, 85 ], "photos previous": [ 11, 11 ], "reply": [ 82, 89 ], "timer cancel": [ 2, 3 ], "files create_document": [ 11, 11 ], "search_web": [ 27, 28 ], "files create_folder": [ 13, 13 ], "note read": [ 9, 9 ], "three calls": [ 23, 23 ], "files save_note_as": [ 11, 11 ], "settings accent": [ 14, 14 ], "photos open": [ 18, 18 ], "app move": [ 23, 23 ], "files move": [ 9, 9 ], "files empty_trash": [ 5, 5 ], "app close": [ 8, 8 ], "timer start": [ 20, 20 ], "files list": [ 4, 9 ], "explain": [ 8, 10 ], "files duplicate": [ 7, 7 ], "settings arcs": [ 12, 12 ], "world_time": [ 8, 12 ], "app restore": [ 14, 14 ], "settings motion": [ 9, 10 ], "note clear": [ 7, 10 ], "note append": [ 12, 12 ], "scope page_down": [ 3, 3 ], "layout reset": [ 5, 5 ], "files import": [ 6, 6 ], "scope top": [ 3, 3 ], "open_website": [ 18, 18 ], "files rename": [ 8, 8 ], "app open": [ 10, 13 ], "photos close": [ 7, 10 ], "files download": [ 14, 14 ], "browser forward": [ 3, 3 ], "calculate": [ 17, 17 ], "app maximize": [ 10, 10 ], "photos next": [ 6, 6 ], "layout show_desktop": [ 8, 8 ], "get_stats": [ 8, 8 ], "files open": [ 12, 12 ], "layout side_by_side": [ 9, 10 ], "timer pause": [ 4, 4 ], "stopwatch reset": [ 2, 4 ], "files delete": [ 11, 11 ], "files restore": [ 5, 5 ], "note replace": [ 6, 9 ], "app minimize": [ 11, 14 ], "photos start_slideshow": [ 5, 5 ], "settings theme": [ 5, 7 ], "files search": [ 9, 9 ], "stopwatch start": [ 1, 1 ], "scope down": [ 1, 3 ], "browser reload": [ 1, 1 ], "scope bottom": [ 2, 2 ], "app resize": [ 11, 11 ], "stopwatch lap": [ 3, 3 ], "files trash": [ 3, 3 ], "scope operations": [ 3, 3 ], "settings wallpaper": [ 6, 6 ], "browser home": [ 1, 2 ], "photos stop_slideshow": [ 5, 6 ], "scope state": [ 2, 2 ], "browser back": [ 1, 1 ], "scope up": [ 1, 1 ], "scope outputs": [ 3, 3 ] } }, "onnx_bytes": 426243274, "onnx_q4f16": { "accuracy": 0.95, "correct": 95, "total": 100, "by_kind": { "two calls": [ 9, 9 ], "photos previous": [ 1, 1 ], "reply": [ 8, 8 ], "timer cancel": [ 0, 1 ], "files create_document": [ 2, 2 ], "search_web": [ 4, 4 ], "files create_folder": [ 3, 3 ], "note read": [ 3, 3 ], "three calls": [ 1, 1 ], "files save_note_as": [ 3, 3 ], "settings accent": [ 1, 1 ], "photos open": [ 3, 3 ], "app move": [ 4, 4 ], "files move": [ 2, 2 ], "files empty_trash": [ 0, 1 ], "app close": [ 2, 2 ], "timer start": [ 6, 6 ], "files list": [ 1, 1 ], "explain": [ 1, 2 ], "files duplicate": [ 2, 2 ], "settings arcs": [ 5, 5 ], "world_time": [ 1, 2 ], "app restore": [ 2, 2 ], "settings motion": [ 1, 1 ], "note clear": [ 2, 2 ], "note append": [ 1, 1 ], "scope page_down": [ 1, 1 ], "layout reset": [ 3, 3 ], "files import": [ 1, 1 ], "scope top": [ 1, 1 ], "open_website": [ 1, 1 ], "files rename": [ 1, 1 ], "app open": [ 1, 1 ], "photos close": [ 1, 1 ], "files download": [ 2, 2 ], "browser forward": [ 1, 1 ], "calculate": [ 2, 2 ], "app maximize": [ 1, 1 ], "photos next": [ 1, 1 ], "layout show_desktop": [ 1, 1 ], "get_stats": [ 2, 2 ], "files open": [ 2, 2 ], "layout side_by_side": [ 1, 1 ], "timer pause": [ 1, 1 ], "stopwatch reset": [ 1, 1 ], "files delete": [ 1, 1 ], "files restore": [ 1, 1 ], "note replace": [ 0, 1 ] } }, "torch_same_subset": { "accuracy": 0.96, "correct": 96, "total": 100, "by_kind": { "two calls": [ 9, 9 ], "photos previous": [ 1, 1 ], "reply": [ 8, 8 ], "timer cancel": [ 0, 1 ], "files create_document": [ 2, 2 ], "search_web": [ 4, 4 ], "files create_folder": [ 3, 3 ], "note read": [ 3, 3 ], "three calls": [ 1, 1 ], "files save_note_as": [ 3, 3 ], "settings accent": [ 1, 1 ], "photos open": [ 3, 3 ], "app move": [ 4, 4 ], "files move": [ 2, 2 ], "files empty_trash": [ 1, 1 ], "app close": [ 2, 2 ], "timer start": [ 6, 6 ], "files list": [ 1, 1 ], "explain": [ 1, 2 ], "files duplicate": [ 2, 2 ], "settings arcs": [ 5, 5 ], "world_time": [ 1, 2 ], "app restore": [ 2, 2 ], "settings motion": [ 1, 1 ], "note clear": [ 2, 2 ], "note append": [ 1, 1 ], "scope page_down": [ 1, 1 ], "layout reset": [ 3, 3 ], "files import": [ 1, 1 ], "scope top": [ 1, 1 ], "open_website": [ 1, 1 ], "files rename": [ 1, 1 ], "app open": [ 1, 1 ], "photos close": [ 1, 1 ], "files download": [ 2, 2 ], "browser forward": [ 1, 1 ], "calculate": [ 2, 2 ], "app maximize": [ 1, 1 ], "photos next": [ 1, 1 ], "layout show_desktop": [ 1, 1 ], "get_stats": [ 2, 2 ], "files open": [ 2, 2 ], "layout side_by_side": [ 0, 1 ], "timer pause": [ 1, 1 ], "stopwatch reset": [ 1, 1 ], "files delete": [ 1, 1 ], "files restore": [ 1, 1 ], "note replace": [ 1, 1 ] } }, "onnx_agrees_with_torch": "96/100", "prompt_tokens": 1723, "job_minutes": 26.1 }
{
"decoding_check": "4/4",
"base": {
"accuracy": 0.1447721179624665,
"correct": 108,
"total": 746,
"by_kind": {
"two calls": [
0,
85
],
"photos previous": [
0,
11
],
"reply": [
68,
89
],
"timer cancel": [
0,
3
],
"files create_document": [
0,
11
],
"search_web": [
1,
28
],
"files create_folder": [
0,
13
],
"note read": [
0,
9
],
"three calls": [
0,
23
],
"files save_note_as": [
0,
11
],
"settings accent": [
1,
14
],
"photos open": [
7,
18
],
"app move": [
0,
23
],
"files move": [
0,
9
],
"files empty_trash": [
0,
5
],
"app close": [
0,
8
],
"timer start": [
13,
20
],
"files list": [
0,
9
],
"explain": [
0,
10
],
"files duplicate": [
0,
7
],
"settings arcs": [
0,
12
],
"world_time": [
0,
12
],
"app restore": [
0,
14
],
"settings motion": [