Hugging Face's logo

SearchingMan
/
basal-2.0-mini-LiteRT

LiteRT
LiteRT
Polish
English
llama
android
on-device
npu
qualcomm
typed-decisions
classification
calibration
polish
Model card Files
xet
 Community
basal-2.0-mini-LiteRT
File size: 3,541 Bytes
49122de
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
{
 "prompt": 2,
 "orders": 1,
 "method": "temperature per question type and per option-count bucket (choice), by NLL, fitted on the out-of-distribution calibration split only; thresholds per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
 "temperature_per_prim": {
  "choice": 1.2599210498948732,
  "noul": 1.8077142767822016,
  "score": 1.9152065613971467
 },
 "temperature_per_bucket": {
  "choice": {
   "2-10": 1.6339154532410998,
   "11-26": 0.9576032806985735,
   "27-255": 0.8908987181403392
  }
 },
 "thresholds": {
  "0.01": {
   "confidence": null,
   "method": "per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
   "per_type": {
    "choice": {
     "confidence": null,
     "heldout_coverage": 0.0,
     "heldout_error": null
    },
    "noul": {
     "confidence": null,
     "heldout_coverage": 0.0,
     "heldout_error": null
    },
    "score": {
     "confidence": null,
     "heldout_coverage": 0.0,
     "heldout_error": null
    }
   },
   "heldout_coverage": 0.0,
   "heldout_error": null
  },
  "0.05": {
   "confidence": null,
   "method": "per type: the most permissive threshold whose accepted calibration decisions have a one-sided 95% Clopper-Pearson upper bound <= max_error; a type without one defers",
   "per_type": {
    "choice": {
     "confidence": 0.9742985069680388,
     "heldout_coverage": 0.09929257200606367,
     "heldout_error": 0.02714164546225617
    },
    "noul": {
     "confidence": 0.9150695844754617,
     "heldout_coverage": 0.17540208136234625,
     "heldout_error": 0.032362459546925515
    },
    "score": {
     "confidence": null,
     "heldout_coverage": 0.0,
     "heldout_error": null
    }
   },
   "heldout_coverage": 0.10458360232408005,
   "heldout_error": 0.029439696106362767
  }
 },
 "gates": {
  "ece_lofo": {
   "choice": {
    "pooled": 0.02387151080828603,
    "n": 11874,
    "gate": 0.05,
    "pass": true
   },
   "noul": {
    "pooled": 0.021839571101782813,
    "n": 5285,
    "gate": 0.05,
    "pass": true
   },
   "score": {
    "pooled": 0.02874265417372889,
    "n": 2978,
    "gate": 0.05,
    "pass": true
   }
  },
  "ece_worst_fold": 0.09937081230552766,
  "worst_fold": "agent",
  "automated_at": {
   "0.01": 0.0,
   "0.05": 0.10458360232408005
  },
  "automated_error": {
   "0.01": null,
   "0.05": 0.029439696106362767
  },
  "note": "held-out numbers: every evaluation row predicted by the fit without its family"
 },
 "test": {
  "T": {
   "choice": {
    "acc": 0.697321879737241,
    "ece": 0.02472238570054493,
    "n": 11874
   },
   "noul": {
    "acc": 0.7547776726584674,
    "ece": 0.02311121112045783,
    "n": 5285
   },
   "score": {
    "acc": 0.5278710543989255,
    "ece": 0.028084103544725424,
    "n": 2978
   },
   "all": {
    "acc": 0.6873417092913542,
    "ece": 0.01744498690342251,
    "n": 20137
   }
  },
  "note": "evaluation pool (development and test splits), full fit"
 },
 "n": {
  "fit": 3560,
  "many": 1500,
  "eval": 20137
 },
 "engine": "fast",
 "fitted_on": {
  "split": "out-of-distribution calibration split (sealed-seed BH-, JDI- and agent-shaped families, many-option items)",
  "orders": 1,
  "readouts": "basal-serve --mode fast, bf16"
 },
 "certified_on": "the same split",
 "evaluated_on": "development and test splits, leave-one-family-out and leave-one-language-out"
}