AppleLampsX commited on
Commit
7bc434f
·
verified ·
1 Parent(s): 33674c6

Fine-tuned on Prism OS commands: 649/746 held-out

Browse files
Files changed (3) hide show
  1. README.md +3 -3
  2. onnx/model_q4f16.onnx_data +1 -1
  3. results.json +521 -521
README.md CHANGED
@@ -23,8 +23,8 @@ leave out.
23
 
24
  | Model | Correct |
25
  | --- | --- |
26
- | Base FunctionGemma | 95/746 |
27
- | Fine-tuned (PyTorch, bf16) | 629/746 |
28
- | Fine-tuned, browser build (q4f16), first 100 | 73/100 |
29
 
30
  Use is subject to the [Gemma Terms of Use](https://ai.google.dev/gemma/terms).
 
23
 
24
  | Model | Correct |
25
  | --- | --- |
26
+ | Base FunctionGemma | 107/746 |
27
+ | Fine-tuned (PyTorch, bf16) | 649/746 |
28
+ | Fine-tuned, browser build (q4f16), first 100 | 86/100 |
29
 
30
  Use is subject to the [Gemma Terms of Use](https://ai.google.dev/gemma/terms).
onnx/model_q4f16.onnx_data CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4a918963509427d83332cb5afd73f09f273a44932c185b310fcf104b35313fe0
3
  size 425724416
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:117393b8f77b78f7a920168101fa6f3b81ac66979cd1bb4bab16b4f60b589a5a
3
  size 425724416
results.json CHANGED
@@ -1,281 +1,277 @@
1
  {
2
  "decoding_check": "4/4",
3
  "base": {
4
- "accuracy": 0.12734584450402145,
5
- "correct": 95,
6
  "total": 746,
7
  "by_kind": {
8
- "open_website": [
9
- 1,
10
- 15
11
- ],
12
- "app resize": [
13
  0,
14
- 10
15
  ],
16
- "explain": [
17
  0,
18
- 10
19
  ],
20
  "reply": [
21
- 66,
22
- 84
23
  ],
24
- "layout side_by_side": [
25
  0,
26
- 13
27
  ],
28
- "app restore": [
29
  0,
30
- 19
31
  ],
32
- "files list": [
33
- 0,
34
- 12
35
  ],
36
- "two calls": [
37
  0,
38
- 92
39
- ],
40
- "get_stats": [
41
- 1,
42
- 7
43
  ],
44
- "settings arcs": [
45
  0,
46
- 8
47
  ],
48
- "browser reload": [
49
- 2,
50
- 5
51
  ],
52
- "settings theme": [
53
  0,
54
- 9
55
  ],
56
- "scope operations": [
57
  1,
58
- 6
59
  ],
60
  "photos open": [
61
- 5,
62
- 33
63
  ],
64
- "photos previous": [
65
  0,
66
- 4
67
  ],
68
- "photos next": [
69
  0,
70
- 5
71
  ],
72
- "three calls": [
73
  0,
74
- 30
75
  ],
76
- "scope outputs": [
77
  0,
78
- 7
79
  ],
80
- "files import": [
 
 
 
 
81
  0,
82
- 6
83
  ],
84
- "layout reset": [
85
  0,
86
  10
87
  ],
88
- "note append": [
89
- 3,
90
- 15
91
- ],
92
- "scope page_up": [
93
  0,
94
- 3
95
  ],
96
- "calculate": [
97
- 1,
98
- 22
99
  ],
100
- "search_web": [
101
- 3,
102
- 34
103
  ],
104
- "settings wallpaper": [
105
  0,
106
- 5
107
  ],
108
  "settings motion": [
109
  0,
110
  10
111
  ],
112
- "world_time": [
 
 
 
 
 
 
 
 
113
  0,
114
- 16
115
  ],
116
- "files rename": [
117
  0,
118
- 10
119
  ],
120
- "app open": [
121
  0,
122
- 11
123
  ],
124
- "stopwatch reset": [
125
  0,
126
  3
127
  ],
128
- "browser back": [
129
- 1,
130
- 4
131
- ],
132
- "app move": [
133
- 0,
134
- 22
135
  ],
136
- "files duplicate": [
137
  0,
138
- 12
139
  ],
140
- "files download": [
141
  0,
142
  13
143
  ],
144
- "stopwatch lap": [
145
  0,
146
- 1
147
  ],
148
- "stopwatch start": [
149
  0,
150
- 1
151
  ],
152
- "files create_document": [
153
  0,
154
- 5
155
  ],
156
- "timer pause": [
 
 
 
 
157
  0,
158
- 7
159
  ],
160
- "browser home": [
161
  1,
162
- 3
163
  ],
164
- "note replace": [
165
  0,
166
- 5
167
  ],
168
- "files create_folder": [
169
- 0,
170
  8
171
  ],
172
  "files open": [
173
  0,
174
- 9
175
  ],
176
- "note clear": [
177
  0,
178
- 5
179
  ],
180
- "files empty_trash": [
181
  0,
182
- 7
 
 
 
 
183
  ],
184
  "files delete": [
185
  0,
186
- 13
187
  ],
188
  "files restore": [
189
  0,
190
- 4
191
  ],
192
- "scope top": [
193
  0,
194
- 2
195
- ],
196
- "photos start_slideshow": [
197
- 3,
198
- 4
199
  ],
200
- "files move": [
201
  0,
202
- 13
203
  ],
204
- "photos stop_slideshow": [
205
- 0,
206
- 1
207
  ],
208
- "app maximize": [
209
  0,
210
- 11
211
  ],
212
- "note read": [
213
- 0,
214
- 4
215
  ],
216
- "timer cancel": [
217
  0,
218
- 2
219
  ],
220
- "app close": [
221
  0,
222
- 8
223
  ],
224
- "files trash": [
225
  0,
226
- 7
227
  ],
228
- "scope up": [
229
  0,
230
  2
231
  ],
232
- "scope down": [
233
- 0,
234
- 3
235
- ],
236
- "settings accent": [
237
  0,
238
- 7
239
  ],
240
- "layout show_desktop": [
241
  0,
242
- 7
243
  ],
244
- "scope state": [
245
  0,
246
  3
247
  ],
248
- "timer start": [
249
- 7,
250
- 11
251
  ],
252
- "files save_note_as": [
253
  0,
254
  6
255
  ],
256
- "app minimize": [
257
- 0,
258
- 9
259
  ],
260
- "files search": [
261
  0,
262
- 4
263
  ],
264
- "browser forward": [
265
  0,
266
  2
267
  ],
268
- "photos close": [
269
  0,
270
- 3
271
  ],
272
- "timer resume": [
273
  0,
274
- 3
275
  ],
276
- "scope page_down": [
277
  0,
278
- 1
279
  ]
280
  }
281
  },
@@ -283,712 +279,716 @@
283
  {
284
  "step": 312,
285
  "epochs": 0.5,
286
- "accuracy": 0.72
287
  },
288
  {
289
  "step": 624,
290
  "epochs": 1.0,
291
- "accuracy": 0.7733333333333333
292
  },
293
  {
294
  "step": 936,
295
  "epochs": 1.5,
296
- "accuracy": 0.8666666666666667
297
  },
298
  {
299
  "step": 1248,
300
  "epochs": 2.0,
301
- "accuracy": 0.8533333333333334
302
  },
303
  {
304
  "step": 1560,
305
  "epochs": 2.5,
306
- "accuracy": 0.8533333333333334
307
  },
308
  {
309
  "step": 1872,
310
  "epochs": 3.0,
311
- "accuracy": 0.86
312
  },
313
  {
314
  "step": 1875,
315
  "epochs": 3.0,
316
- "accuracy": 0.86
317
  }
318
  ],
319
- "best_step": 936,
320
- "train_minutes": 9.6,
321
  "fine_tuned": {
322
- "accuracy": 0.8431635388739946,
323
- "correct": 629,
324
  "total": 746,
325
  "by_kind": {
326
- "open_website": [
327
- 15,
328
- 15
329
- ],
330
- "app resize": [
331
- 10,
332
- 10
333
  ],
334
- "explain": [
335
  7,
336
- 10
337
  ],
338
  "reply": [
339
- 65,
340
- 84
341
  ],
342
- "layout side_by_side": [
343
- 12,
344
- 13
345
- ],
346
- "app restore": [
347
- 17,
348
- 19
349
  ],
350
- "files list": [
351
- 12,
352
- 12
353
  ],
354
- "two calls": [
355
- 91,
356
- 92
357
  ],
358
- "get_stats": [
359
- 6,
360
- 7
361
  ],
362
- "settings arcs": [
363
- 4,
364
- 8
365
  ],
366
- "browser reload": [
367
- 2,
368
- 5
369
  ],
370
- "settings theme": [
371
- 6,
372
- 9
373
  ],
374
- "scope operations": [
375
- 6,
376
- 6
377
  ],
378
  "photos open": [
379
  18,
380
- 33
381
  ],
382
- "photos previous": [
383
- 2,
384
- 4
385
  ],
386
- "photos next": [
 
 
 
 
387
  2,
388
  5
389
  ],
390
- "three calls": [
391
- 29,
392
- 30
393
  ],
394
- "scope outputs": [
395
- 7,
396
- 7
397
  ],
398
- "files import": [
399
- 0,
400
- 6
401
  ],
402
- "layout reset": [
403
- 3,
404
  10
405
  ],
406
- "note append": [
407
- 11,
408
- 15
409
- ],
410
- "scope page_up": [
411
- 1,
412
- 3
413
  ],
414
- "calculate": [
415
- 22,
416
- 22
417
  ],
418
- "search_web": [
419
- 31,
420
- 34
421
  ],
422
- "settings wallpaper": [
423
- 5,
424
- 5
425
  ],
426
  "settings motion": [
427
- 7,
428
  10
429
  ],
430
- "world_time": [
431
- 15,
432
- 16
433
  ],
434
- "files rename": [
435
  10,
436
- 10
437
  ],
438
- "app open": [
 
 
 
 
 
 
 
 
439
  6,
440
- 11
441
  ],
442
- "stopwatch reset": [
443
- 1,
444
  3
445
  ],
446
- "browser back": [
447
- 4,
448
- 4
449
  ],
450
- "app move": [
451
- 21,
452
- 22
453
  ],
454
- "files duplicate": [
455
- 12,
456
- 12
 
 
 
 
457
  ],
458
  "files download": [
459
- 13,
460
- 13
461
  ],
462
- "stopwatch lap": [
463
- 1,
464
- 1
465
  ],
466
- "stopwatch start": [
467
- 1,
468
- 1
469
  ],
470
- "files create_document": [
471
  5,
472
- 5
473
- ],
474
- "timer pause": [
475
- 0,
476
- 7
477
  ],
478
- "browser home": [
479
- 1,
480
- 3
481
  ],
482
- "note replace": [
483
- 3,
484
- 5
485
  ],
486
- "files create_folder": [
487
  8,
488
  8
489
  ],
490
  "files open": [
 
 
 
 
491
  9,
492
- 9
493
  ],
494
- "note clear": [
495
- 5,
496
- 5
497
  ],
498
- "files empty_trash": [
499
- 7,
500
- 7
501
  ],
502
  "files delete": [
503
- 13,
504
- 13
505
  ],
506
  "files restore": [
507
- 3,
508
- 4
509
- ],
510
- "scope top": [
511
  2,
512
- 2
 
 
 
 
 
 
 
 
513
  ],
514
  "photos start_slideshow": [
515
  4,
516
- 4
517
  ],
518
- "files move": [
519
- 13,
520
- 13
521
  ],
522
- "photos stop_slideshow": [
523
- 0,
 
 
 
 
524
  1
525
  ],
526
- "app maximize": [
527
- 4,
528
- 11
529
  ],
530
- "note read": [
531
- 4,
532
- 4
533
  ],
534
- "timer cancel": [
535
  2,
536
  2
537
  ],
538
- "app close": [
539
- 8,
540
- 8
541
- ],
542
- "files trash": [
543
- 7,
544
- 7
545
- ],
546
- "scope up": [
547
- 2,
548
- 2
549
  ],
550
- "scope down": [
551
- 1,
552
  3
553
  ],
554
- "settings accent": [
555
- 7,
556
- 7
557
- ],
558
- "layout show_desktop": [
559
- 7,
560
- 7
561
- ],
562
- "scope state": [
563
  3,
564
  3
565
  ],
566
- "timer start": [
567
- 11,
568
- 11
569
  ],
570
- "files save_note_as": [
571
  6,
572
  6
573
  ],
574
- "app minimize": [
575
- 7,
576
- 9
577
  ],
578
- "files search": [
579
- 4,
580
- 4
581
  ],
582
- "browser forward": [
583
  2,
584
  2
585
  ],
586
- "photos close": [
587
- 3,
588
- 3
589
  ],
590
- "timer resume": [
 
 
 
 
591
  3,
592
  3
593
- ],
594
- "scope page_down": [
595
- 0,
596
- 1
597
  ]
598
  }
599
  },
600
  "onnx_bytes": 426243274,
601
  "onnx_q4f16": {
602
- "accuracy": 0.73,
603
- "correct": 73,
604
  "total": 100,
605
  "by_kind": {
606
- "open_website": [
607
- 3,
608
- 3
609
  ],
610
- "app resize": [
611
  1,
612
  1
613
  ],
614
- "explain": [
615
- 1,
616
- 2
617
- ],
618
  "reply": [
619
- 11,
620
- 15
621
- ],
622
- "layout side_by_side": [
623
- 1,
624
- 1
625
  ],
626
- "app restore": [
627
  1,
628
  1
629
  ],
630
- "files list": [
631
  2,
632
- 3
633
  ],
634
- "two calls": [
635
- 11,
636
- 11
637
  ],
638
- "get_stats": [
639
- 0,
640
- 1
641
  ],
642
- "settings arcs": [
643
- 1,
644
  3
645
  ],
646
- "browser reload": [
647
  1,
648
  1
649
  ],
650
- "settings theme": [
651
- 0,
652
- 2
653
  ],
654
- "scope operations": [
655
  1,
656
  1
657
  ],
658
  "photos open": [
659
- 1,
660
  3
661
  ],
662
- "photos previous": [
663
- 1,
664
- 1
665
  ],
666
- "photos next": [
667
- 1,
668
  2
669
  ],
670
- "three calls": [
671
- 3,
672
- 3
673
- ],
674
- "scope outputs": [
675
- 1,
676
  1
677
  ],
678
- "files import": [
679
- 0,
680
  2
681
  ],
682
- "layout reset": [
683
- 0,
684
- 2
685
  ],
686
- "note append": [
687
  1,
 
 
 
 
688
  2
689
  ],
690
- "scope page_up": [
691
- 1,
692
  2
693
  ],
694
- "calculate": [
695
- 1,
696
- 1
697
  ],
698
- "search_web": [
699
  2,
700
- 4
701
- ],
702
- "settings wallpaper": [
703
- 1,
704
- 1
705
  ],
706
- "settings motion": [
707
  0,
708
  2
709
  ],
710
- "world_time": [
711
- 5,
712
- 5
713
  ],
714
- "files rename": [
715
- 3,
716
- 3
717
  ],
718
- "app open": [
719
  1,
720
  1
721
  ],
722
- "stopwatch reset": [
723
  1,
724
  1
725
  ],
726
- "browser back": [
727
- 2,
728
- 2
729
  ],
730
- "app move": [
731
  1,
732
  1
733
  ],
734
- "files duplicate": [
735
  1,
736
  1
737
  ],
738
- "files download": [
739
  1,
740
  1
741
  ],
742
- "stopwatch lap": [
743
  1,
744
  1
745
  ],
746
- "stopwatch start": [
747
  1,
748
  1
749
  ],
750
- "files create_document": [
751
- 1,
752
  1
753
  ],
754
- "timer pause": [
755
  0,
 
 
 
 
756
  1
757
  ],
758
- "browser home": [
 
 
 
 
759
  0,
760
  1
761
  ],
762
- "note replace": [
763
  1,
764
- 2
765
  ],
766
- "files create_folder": [
767
  1,
768
  1
769
  ],
 
 
 
 
770
  "files open": [
 
 
 
 
771
  1,
772
  1
773
  ],
774
- "note clear": [
775
  1,
776
  1
777
  ],
778
- "files empty_trash": [
779
  1,
780
  1
781
  ],
782
  "files delete": [
783
- 1,
784
  1
785
  ],
786
  "files restore": [
787
- 1,
788
  1
789
  ],
790
- "scope top": [
791
  1,
792
  1
793
  ]
794
  }
795
  },
796
  "torch_same_subset": {
797
- "accuracy": 0.79,
798
- "correct": 79,
799
  "total": 100,
800
  "by_kind": {
801
- "open_website": [
802
- 3,
803
- 3
804
  ],
805
- "app resize": [
806
  1,
807
  1
808
  ],
809
- "explain": [
810
- 1,
811
- 2
812
- ],
813
  "reply": [
814
- 13,
815
- 15
816
- ],
817
- "layout side_by_side": [
818
- 1,
819
- 1
820
  ],
821
- "app restore": [
822
- 1,
823
  1
824
  ],
825
- "files list": [
826
- 3,
827
- 3
828
  ],
829
- "two calls": [
830
- 11,
831
- 11
832
  ],
833
- "get_stats": [
834
  0,
835
- 1
836
  ],
837
- "settings arcs": [
838
- 1,
839
  3
840
  ],
841
- "browser reload": [
842
  1,
843
  1
844
  ],
845
- "settings theme": [
846
- 1,
847
- 2
848
  ],
849
- "scope operations": [
850
  1,
851
  1
852
  ],
853
  "photos open": [
854
- 1,
855
  3
856
  ],
857
- "photos previous": [
858
- 1,
 
 
 
 
 
 
 
 
859
  1
860
  ],
861
- "photos next": [
862
- 1,
863
  2
864
  ],
865
- "three calls": [
866
- 3,
867
- 3
868
  ],
869
- "scope outputs": [
870
  1,
871
  1
872
  ],
873
- "files import": [
874
- 0,
875
  2
876
  ],
877
- "layout reset": [
878
- 0,
879
  2
880
  ],
881
- "note append": [
882
- 1,
 
 
 
 
883
  2
884
  ],
885
- "scope page_up": [
886
- 1,
887
  2
888
  ],
889
- "calculate": [
890
  1,
891
  1
892
  ],
893
- "search_web": [
894
- 4,
895
- 4
896
  ],
897
- "settings wallpaper": [
898
  1,
899
  1
900
  ],
901
- "settings motion": [
902
- 0,
903
- 2
904
- ],
905
- "world_time": [
906
- 5,
907
- 5
908
  ],
909
- "files rename": [
910
  3,
911
  3
912
  ],
913
- "app open": [
914
  1,
915
  1
916
  ],
917
- "stopwatch reset": [
918
  1,
919
  1
920
  ],
921
- "browser back": [
922
- 2,
923
- 2
924
- ],
925
- "app move": [
926
  1,
927
  1
928
  ],
929
- "files duplicate": [
930
  1,
931
  1
932
  ],
933
- "files download": [
934
  1,
935
  1
936
  ],
937
- "stopwatch lap": [
938
- 1,
939
  1
940
  ],
941
- "stopwatch start": [
942
- 1,
943
- 1
944
  ],
945
- "files create_document": [
946
  1,
947
  1
948
  ],
949
- "timer pause": [
950
- 0,
951
- 1
952
  ],
953
- "browser home": [
954
  0,
955
  1
956
  ],
957
- "note replace": [
958
  1,
959
- 2
960
  ],
961
- "files create_folder": [
962
- 1,
963
  1
964
  ],
 
 
 
 
965
  "files open": [
966
- 1,
 
 
 
 
967
  1
968
  ],
969
- "note clear": [
970
  1,
971
  1
972
  ],
973
- "files empty_trash": [
974
  1,
975
  1
976
  ],
977
  "files delete": [
978
- 1,
979
  1
980
  ],
981
  "files restore": [
982
- 1,
983
  1
984
  ],
985
- "scope top": [
986
  1,
987
  1
988
  ]
989
  }
990
  },
991
  "onnx_agrees_with_torch": "88/100",
992
- "prompt_tokens": 1724,
993
- "job_minutes": 66.3
994
  }
 
1
  {
2
  "decoding_check": "4/4",
3
  "base": {
4
+ "accuracy": 0.14343163538873996,
5
+ "correct": 107,
6
  "total": 746,
7
  "by_kind": {
8
+ "two calls": [
 
 
 
 
9
  0,
10
+ 85
11
  ],
12
+ "photos previous": [
13
  0,
14
+ 11
15
  ],
16
  "reply": [
17
+ 68,
18
+ 89
19
  ],
20
+ "timer cancel": [
21
  0,
22
+ 3
23
  ],
24
+ "files create_document": [
25
  0,
26
+ 11
27
  ],
28
+ "search_web": [
29
+ 1,
30
+ 28
31
  ],
32
+ "files create_folder": [
33
  0,
34
+ 13
 
 
 
 
35
  ],
36
+ "note read": [
37
  0,
38
+ 9
39
  ],
40
+ "three calls": [
41
+ 0,
42
+ 23
43
  ],
44
+ "files save_note_as": [
45
  0,
46
+ 11
47
  ],
48
+ "settings accent": [
49
  1,
50
+ 14
51
  ],
52
  "photos open": [
53
+ 7,
54
+ 18
55
  ],
56
+ "app move": [
57
  0,
58
+ 23
59
  ],
60
+ "files move": [
61
  0,
62
+ 9
63
  ],
64
+ "files empty_trash": [
65
  0,
66
+ 5
67
  ],
68
+ "app close": [
69
  0,
70
+ 8
71
  ],
72
+ "timer start": [
73
+ 13,
74
+ 20
75
+ ],
76
+ "files list": [
77
  0,
78
+ 9
79
  ],
80
+ "explain": [
81
  0,
82
  10
83
  ],
84
+ "files duplicate": [
 
 
 
 
85
  0,
86
+ 7
87
  ],
88
+ "settings arcs": [
89
+ 0,
90
+ 12
91
  ],
92
+ "world_time": [
93
+ 0,
94
+ 12
95
  ],
96
+ "app restore": [
97
  0,
98
+ 14
99
  ],
100
  "settings motion": [
101
  0,
102
  10
103
  ],
104
+ "note clear": [
105
+ 1,
106
+ 10
107
+ ],
108
+ "note append": [
109
+ 3,
110
+ 12
111
+ ],
112
+ "scope page_down": [
113
  0,
114
+ 3
115
  ],
116
+ "layout reset": [
117
  0,
118
+ 5
119
  ],
120
+ "files import": [
121
  0,
122
+ 6
123
  ],
124
+ "scope top": [
125
  0,
126
  3
127
  ],
128
+ "open_website": [
129
+ 2,
130
+ 18
 
 
 
 
131
  ],
132
+ "files rename": [
133
  0,
134
+ 8
135
  ],
136
+ "app open": [
137
  0,
138
  13
139
  ],
140
+ "photos close": [
141
  0,
142
+ 10
143
  ],
144
+ "files download": [
145
  0,
146
+ 14
147
  ],
148
+ "browser forward": [
149
  0,
150
+ 3
151
  ],
152
+ "calculate": [
153
+ 2,
154
+ 17
155
+ ],
156
+ "app maximize": [
157
  0,
158
+ 10
159
  ],
160
+ "photos next": [
161
  1,
162
+ 6
163
  ],
164
+ "layout show_desktop": [
165
  0,
166
+ 8
167
  ],
168
+ "get_stats": [
169
+ 1,
170
  8
171
  ],
172
  "files open": [
173
  0,
174
+ 12
175
  ],
176
+ "layout side_by_side": [
177
  0,
178
+ 10
179
  ],
180
+ "timer pause": [
181
  0,
182
+ 4
183
+ ],
184
+ "stopwatch reset": [
185
+ 0,
186
+ 4
187
  ],
188
  "files delete": [
189
  0,
190
+ 11
191
  ],
192
  "files restore": [
193
  0,
194
+ 5
195
  ],
196
+ "note replace": [
197
  0,
198
+ 9
 
 
 
 
199
  ],
200
+ "app minimize": [
201
  0,
202
+ 14
203
  ],
204
+ "photos start_slideshow": [
205
+ 4,
206
+ 5
207
  ],
208
+ "settings theme": [
209
  0,
210
+ 7
211
  ],
212
+ "files search": [
213
+ 1,
214
+ 9
215
  ],
216
+ "stopwatch start": [
217
  0,
218
+ 1
219
  ],
220
+ "scope down": [
221
  0,
222
+ 3
223
  ],
224
+ "browser reload": [
225
  0,
226
+ 1
227
  ],
228
+ "scope bottom": [
229
  0,
230
  2
231
  ],
232
+ "app resize": [
 
 
 
 
233
  0,
234
+ 11
235
  ],
236
+ "stopwatch lap": [
237
  0,
238
+ 3
239
  ],
240
+ "files trash": [
241
  0,
242
  3
243
  ],
244
+ "scope operations": [
245
+ 1,
246
+ 3
247
  ],
248
+ "settings wallpaper": [
249
  0,
250
  6
251
  ],
252
+ "browser home": [
253
+ 1,
254
+ 2
255
  ],
256
+ "photos stop_slideshow": [
257
  0,
258
+ 6
259
  ],
260
+ "scope state": [
261
  0,
262
  2
263
  ],
264
+ "browser back": [
265
  0,
266
+ 1
267
  ],
268
+ "scope up": [
269
  0,
270
+ 1
271
  ],
272
+ "scope outputs": [
273
  0,
274
+ 3
275
  ]
276
  }
277
  },
 
279
  {
280
  "step": 312,
281
  "epochs": 0.5,
282
+ "accuracy": 0.82
283
  },
284
  {
285
  "step": 624,
286
  "epochs": 1.0,
287
+ "accuracy": 0.88
288
  },
289
  {
290
  "step": 936,
291
  "epochs": 1.5,
292
+ "accuracy": 0.88
293
  },
294
  {
295
  "step": 1248,
296
  "epochs": 2.0,
297
+ "accuracy": 0.92
298
  },
299
  {
300
  "step": 1560,
301
  "epochs": 2.5,
302
+ "accuracy": 0.9333333333333333
303
  },
304
  {
305
  "step": 1872,
306
  "epochs": 3.0,
307
+ "accuracy": 0.9466666666666667
308
  },
309
  {
310
  "step": 1875,
311
  "epochs": 3.0,
312
+ "accuracy": 0.9466666666666667
313
  }
314
  ],
315
+ "best_step": 1872,
316
+ "train_minutes": 10.4,
317
  "fine_tuned": {
318
+ "accuracy": 0.8699731903485255,
319
+ "correct": 649,
320
  "total": 746,
321
  "by_kind": {
322
+ "two calls": [
323
+ 85,
324
+ 85
 
 
 
 
325
  ],
326
+ "photos previous": [
327
  7,
328
+ 11
329
  ],
330
  "reply": [
331
+ 72,
332
+ 89
333
  ],
334
+ "timer cancel": [
335
+ 2,
336
+ 3
 
 
 
 
337
  ],
338
+ "files create_document": [
339
+ 11,
340
+ 11
341
  ],
342
+ "search_web": [
343
+ 26,
344
+ 28
345
  ],
346
+ "files create_folder": [
347
+ 7,
348
+ 13
349
  ],
350
+ "note read": [
351
+ 9,
352
+ 9
353
  ],
354
+ "three calls": [
355
+ 23,
356
+ 23
357
  ],
358
+ "files save_note_as": [
359
+ 11,
360
+ 11
361
  ],
362
+ "settings accent": [
363
+ 13,
364
+ 14
365
  ],
366
  "photos open": [
367
  18,
368
+ 18
369
  ],
370
+ "app move": [
371
+ 23,
372
+ 23
373
  ],
374
+ "files move": [
375
+ 9,
376
+ 9
377
+ ],
378
+ "files empty_trash": [
379
  2,
380
  5
381
  ],
382
+ "app close": [
383
+ 8,
384
+ 8
385
  ],
386
+ "timer start": [
387
+ 20,
388
+ 20
389
  ],
390
+ "files list": [
391
+ 8,
392
+ 9
393
  ],
394
+ "explain": [
395
+ 9,
396
  10
397
  ],
398
+ "files duplicate": [
399
+ 7,
400
+ 7
 
 
 
 
401
  ],
402
+ "settings arcs": [
403
+ 12,
404
+ 12
405
  ],
406
+ "world_time": [
407
+ 9,
408
+ 12
409
  ],
410
+ "app restore": [
411
+ 9,
412
+ 14
413
  ],
414
  "settings motion": [
415
+ 9,
416
  10
417
  ],
418
+ "note clear": [
419
+ 7,
420
+ 10
421
  ],
422
+ "note append": [
423
  10,
424
+ 12
425
  ],
426
+ "scope page_down": [
427
+ 3,
428
+ 3
429
+ ],
430
+ "layout reset": [
431
+ 5,
432
+ 5
433
+ ],
434
+ "files import": [
435
  6,
436
+ 6
437
  ],
438
+ "scope top": [
439
+ 3,
440
  3
441
  ],
442
+ "open_website": [
443
+ 18,
444
+ 18
445
  ],
446
+ "files rename": [
447
+ 8,
448
+ 8
449
  ],
450
+ "app open": [
451
+ 10,
452
+ 13
453
+ ],
454
+ "photos close": [
455
+ 2,
456
+ 10
457
  ],
458
  "files download": [
459
+ 6,
460
+ 14
461
  ],
462
+ "browser forward": [
463
+ 3,
464
+ 3
465
  ],
466
+ "calculate": [
467
+ 17,
468
+ 17
469
  ],
470
+ "app maximize": [
471
  5,
472
+ 10
 
 
 
 
473
  ],
474
+ "photos next": [
475
+ 6,
476
+ 6
477
  ],
478
+ "layout show_desktop": [
479
+ 7,
480
+ 8
481
  ],
482
+ "get_stats": [
483
  8,
484
  8
485
  ],
486
  "files open": [
487
+ 12,
488
+ 12
489
+ ],
490
+ "layout side_by_side": [
491
  9,
492
+ 10
493
  ],
494
+ "timer pause": [
495
+ 4,
496
+ 4
497
  ],
498
+ "stopwatch reset": [
499
+ 2,
500
+ 4
501
  ],
502
  "files delete": [
503
+ 9,
504
+ 11
505
  ],
506
  "files restore": [
 
 
 
 
507
  2,
508
+ 5
509
+ ],
510
+ "note replace": [
511
+ 7,
512
+ 9
513
+ ],
514
+ "app minimize": [
515
+ 6,
516
+ 14
517
  ],
518
  "photos start_slideshow": [
519
  4,
520
+ 5
521
  ],
522
+ "settings theme": [
523
+ 5,
524
+ 7
525
  ],
526
+ "files search": [
527
+ 9,
528
+ 9
529
+ ],
530
+ "stopwatch start": [
531
+ 1,
532
  1
533
  ],
534
+ "scope down": [
535
+ 3,
536
+ 3
537
  ],
538
+ "browser reload": [
539
+ 1,
540
+ 1
541
  ],
542
+ "scope bottom": [
543
  2,
544
  2
545
  ],
546
+ "app resize": [
547
+ 11,
548
+ 11
 
 
 
 
 
 
 
 
549
  ],
550
+ "stopwatch lap": [
551
+ 3,
552
  3
553
  ],
554
+ "files trash": [
 
 
 
 
 
 
 
 
555
  3,
556
  3
557
  ],
558
+ "scope operations": [
559
+ 3,
560
+ 3
561
  ],
562
+ "settings wallpaper": [
563
  6,
564
  6
565
  ],
566
+ "browser home": [
567
+ 1,
568
+ 2
569
  ],
570
+ "photos stop_slideshow": [
571
+ 6,
572
+ 6
573
  ],
574
+ "scope state": [
575
  2,
576
  2
577
  ],
578
+ "browser back": [
579
+ 1,
580
+ 1
581
  ],
582
+ "scope up": [
583
+ 1,
584
+ 1
585
+ ],
586
+ "scope outputs": [
587
  3,
588
  3
 
 
 
 
589
  ]
590
  }
591
  },
592
  "onnx_bytes": 426243274,
593
  "onnx_q4f16": {
594
+ "accuracy": 0.86,
595
+ "correct": 86,
596
  "total": 100,
597
  "by_kind": {
598
+ "two calls": [
599
+ 9,
600
+ 9
601
  ],
602
+ "photos previous": [
603
  1,
604
  1
605
  ],
 
 
 
 
606
  "reply": [
607
+ 5,
608
+ 8
 
 
 
 
609
  ],
610
+ "timer cancel": [
611
  1,
612
  1
613
  ],
614
+ "files create_document": [
615
  2,
616
+ 2
617
  ],
618
+ "search_web": [
619
+ 3,
620
+ 4
621
  ],
622
+ "files create_folder": [
623
+ 2,
624
+ 3
625
  ],
626
+ "note read": [
627
+ 3,
628
  3
629
  ],
630
+ "three calls": [
631
  1,
632
  1
633
  ],
634
+ "files save_note_as": [
635
+ 3,
636
+ 3
637
  ],
638
+ "settings accent": [
639
  1,
640
  1
641
  ],
642
  "photos open": [
643
+ 3,
644
  3
645
  ],
646
+ "app move": [
647
+ 4,
648
+ 4
649
  ],
650
+ "files move": [
651
+ 2,
652
  2
653
  ],
654
+ "files empty_trash": [
655
+ 0,
 
 
 
 
656
  1
657
  ],
658
+ "app close": [
659
+ 2,
660
  2
661
  ],
662
+ "timer start": [
663
+ 6,
664
+ 6
665
  ],
666
+ "files list": [
667
  1,
668
+ 1
669
+ ],
670
+ "explain": [
671
+ 2,
672
  2
673
  ],
674
+ "files duplicate": [
675
+ 2,
676
  2
677
  ],
678
+ "settings arcs": [
679
+ 5,
680
+ 5
681
  ],
682
+ "world_time": [
683
  2,
684
+ 2
 
 
 
 
685
  ],
686
+ "app restore": [
687
  0,
688
  2
689
  ],
690
+ "settings motion": [
691
+ 1,
692
+ 1
693
  ],
694
+ "note clear": [
695
+ 2,
696
+ 2
697
  ],
698
+ "note append": [
699
  1,
700
  1
701
  ],
702
+ "scope page_down": [
703
  1,
704
  1
705
  ],
706
+ "layout reset": [
707
+ 3,
708
+ 3
709
  ],
710
+ "files import": [
711
  1,
712
  1
713
  ],
714
+ "scope top": [
715
  1,
716
  1
717
  ],
718
+ "open_website": [
719
  1,
720
  1
721
  ],
722
+ "files rename": [
723
  1,
724
  1
725
  ],
726
+ "app open": [
727
  1,
728
  1
729
  ],
730
+ "photos close": [
731
+ 0,
732
  1
733
  ],
734
+ "files download": [
735
  0,
736
+ 2
737
+ ],
738
+ "browser forward": [
739
+ 1,
740
  1
741
  ],
742
+ "calculate": [
743
+ 2,
744
+ 2
745
+ ],
746
+ "app maximize": [
747
  0,
748
  1
749
  ],
750
+ "photos next": [
751
  1,
752
+ 1
753
  ],
754
+ "layout show_desktop": [
755
  1,
756
  1
757
  ],
758
+ "get_stats": [
759
+ 2,
760
+ 2
761
+ ],
762
  "files open": [
763
+ 2,
764
+ 2
765
+ ],
766
+ "layout side_by_side": [
767
  1,
768
  1
769
  ],
770
+ "timer pause": [
771
  1,
772
  1
773
  ],
774
+ "stopwatch reset": [
775
  1,
776
  1
777
  ],
778
  "files delete": [
779
+ 0,
780
  1
781
  ],
782
  "files restore": [
783
+ 0,
784
  1
785
  ],
786
+ "note replace": [
787
  1,
788
  1
789
  ]
790
  }
791
  },
792
  "torch_same_subset": {
793
+ "accuracy": 0.83,
794
+ "correct": 83,
795
  "total": 100,
796
  "by_kind": {
797
+ "two calls": [
798
+ 9,
799
+ 9
800
  ],
801
+ "photos previous": [
802
  1,
803
  1
804
  ],
 
 
 
 
805
  "reply": [
806
+ 6,
807
+ 8
 
 
 
 
808
  ],
809
+ "timer cancel": [
810
+ 0,
811
  1
812
  ],
813
+ "files create_document": [
814
+ 2,
815
+ 2
816
  ],
817
+ "search_web": [
818
+ 4,
819
+ 4
820
  ],
821
+ "files create_folder": [
822
  0,
823
+ 3
824
  ],
825
+ "note read": [
826
+ 3,
827
  3
828
  ],
829
+ "three calls": [
830
  1,
831
  1
832
  ],
833
+ "files save_note_as": [
834
+ 3,
835
+ 3
836
  ],
837
+ "settings accent": [
838
  1,
839
  1
840
  ],
841
  "photos open": [
842
+ 3,
843
  3
844
  ],
845
+ "app move": [
846
+ 4,
847
+ 4
848
+ ],
849
+ "files move": [
850
+ 2,
851
+ 2
852
+ ],
853
+ "files empty_trash": [
854
+ 0,
855
  1
856
  ],
857
+ "app close": [
858
+ 2,
859
  2
860
  ],
861
+ "timer start": [
862
+ 6,
863
+ 6
864
  ],
865
+ "files list": [
866
  1,
867
  1
868
  ],
869
+ "explain": [
870
+ 2,
871
  2
872
  ],
873
+ "files duplicate": [
874
+ 2,
875
  2
876
  ],
877
+ "settings arcs": [
878
+ 5,
879
+ 5
880
+ ],
881
+ "world_time": [
882
+ 2,
883
  2
884
  ],
885
+ "app restore": [
886
+ 0,
887
  2
888
  ],
889
+ "settings motion": [
890
  1,
891
  1
892
  ],
893
+ "note clear": [
894
+ 2,
895
+ 2
896
  ],
897
+ "note append": [
898
  1,
899
  1
900
  ],
901
+ "scope page_down": [
902
+ 1,
903
+ 1
 
 
 
 
904
  ],
905
+ "layout reset": [
906
  3,
907
  3
908
  ],
909
+ "files import": [
910
  1,
911
  1
912
  ],
913
+ "scope top": [
914
  1,
915
  1
916
  ],
917
+ "open_website": [
 
 
 
 
918
  1,
919
  1
920
  ],
921
+ "files rename": [
922
  1,
923
  1
924
  ],
925
+ "app open": [
926
  1,
927
  1
928
  ],
929
+ "photos close": [
930
+ 0,
931
  1
932
  ],
933
+ "files download": [
934
+ 0,
935
+ 2
936
  ],
937
+ "browser forward": [
938
  1,
939
  1
940
  ],
941
+ "calculate": [
942
+ 2,
943
+ 2
944
  ],
945
+ "app maximize": [
946
  0,
947
  1
948
  ],
949
+ "photos next": [
950
  1,
951
+ 1
952
  ],
953
+ "layout show_desktop": [
954
+ 0,
955
  1
956
  ],
957
+ "get_stats": [
958
+ 2,
959
+ 2
960
+ ],
961
  "files open": [
962
+ 2,
963
+ 2
964
+ ],
965
+ "layout side_by_side": [
966
+ 0,
967
  1
968
  ],
969
+ "timer pause": [
970
  1,
971
  1
972
  ],
973
+ "stopwatch reset": [
974
  1,
975
  1
976
  ],
977
  "files delete": [
978
+ 0,
979
  1
980
  ],
981
  "files restore": [
982
+ 0,
983
  1
984
  ],
985
+ "note replace": [
986
  1,
987
  1
988
  ]
989
  }
990
  },
991
  "onnx_agrees_with_torch": "88/100",
992
+ "prompt_tokens": 1723,
993
+ "job_minutes": 19.3
994
  }