Hugging Face's logo

bambamdevs
/
openjev-e4b

Zero-Shot Classification
Safetensors
PEFT
English
openjev
classification
decision-model
listwise
gemma4
research
Model card Files
xet
 Community
openjev-e4b
File size: 2,259 Bytes
03223d7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
{
  "model_line": "openjev-e4b",
  "release": "1.0",
  "benchmark_artifact": "local NF4 frozen base + dense adapted layers + LoRA + decision head",
  "source_summary": "eval/raw_run/summary.json",
  "raw_control_summary": "eval/raw_e4b_nf4_control_summary.json",
  "tasks": {
    "arc_challenge": {
      "status": "measured",
      "display": "ARC-Challenge",
      "n": 1172,
      "accuracy": 0.878839590443686,
      "nll": 0.3434354264088279,
      "brier": 0.18004018282065612,
      "ece": 0.018640702693919233,
      "p50_ms": 173.45179200128769,
      "p95_ms": 241.31601209974178,
      "raw_e4b_nf4_accuracy": 0.8737,
      "delta_vs_raw_pp": 0.5139590443685949
    },
    "winogrande": {
      "status": "measured",
      "display": "WinoGrande",
      "n": 1267,
      "accuracy": 0.664561957379637,
      "nll": 0.8732282598352692,
      "brier": 0.5079363220206212,
      "ece": 0.1986676781891346,
      "p50_ms": 168.07529400102794,
      "p95_ms": 241.4132092009823,
      "raw_e4b_nf4_accuracy": 0.6014,
      "delta_vs_raw_pp": 6.316195737963692
    },
    "arc_easy": {
      "status": "measured",
      "display": "ARC-Easy",
      "n": 2376,
      "accuracy": 0.9562289562289562,
      "nll": 0.15331558248276225,
      "brier": 0.07269179989034444,
      "ece": 0.029571857402955024,
      "p50_ms": 172.40582649901626,
      "p95_ms": 231.58336125015921,
      "raw_e4b_nf4_accuracy": 0.9558,
      "delta_vs_raw_pp": 0.04289562289562232
    },
    "hellaswag": {
      "status": "measured",
      "display": "HellaSwag",
      "n": 10042,
      "accuracy": 0.8977295359490142,
      "nll": 0.2736215284007136,
      "brier": 0.14571887516017584,
      "ece": 0.012820075397336044,
      "p50_ms": 230.12878749796073,
      "p95_ms": 276.0439652007335,
      "raw_e4b_nf4_accuracy": 0.7651,
      "delta_vs_raw_pp": 13.262953594901417
    },
    "mmlu": {
      "status": "measured",
      "display": "MMLU",
      "n": 14042,
      "accuracy": 0.6441390115368181,
      "nll": 0.8705675106095903,
      "brier": 0.45396635565340543,
      "ece": 0.03314363695835977,
      "p50_ms": 196.04511249963252,
      "p95_ms": 303.28189720094076,
      "raw_e4b_nf4_accuracy": 0.6619,
      "delta_vs_raw_pp": -1.776098846318197
    }
  }
}