Hugging Face's logo

vynly
/
CAEL-0.3

Text Generation
English
cael
non-neural
cellular-automata
symbolic-reasoning
retrieval
Model card Files
xet
 Community
CAEL-0.3
File size: 4,718 Bytes
abc20fb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
{
  "tasks": {
    "hellaswag": {
      "n": 10042,
      "seconds": 146.01327709999168,
      "routes": {
        "cached/language+evidence": 2973,
        "cached/language": 7068,
        "cached/exact-arithmetic": 1
      },
      "changed_system_predictions": 0,
      "fresh_sql_parity_checks": 3,
      "maximum_likelihood_error": 0.0,
      "prediction_sha256": "ae5e8d2fb43e21a1810c4e8518e5a54256cf7ea66b77a273f1229b753360c7c9",
      "primary": "char",
      "native_correct": 2809,
      "native_percent": 27.972515435172276,
      "previous_native_correct": 2809,
      "previous_native_percent": 27.972515435172276,
      "native_change_points": 0.0,
      "assisted_correct": 2805,
      "assisted_percent": 27.9326827325234,
      "previous_assisted_percent": 27.93,
      "assisted_change_points": 0.0
    },
    "arc_easy": {
      "n": 2376,
      "seconds": 22.353922400041483,
      "routes": {
        "cached/language+evidence": 2374,
        "cached/exact-arithmetic": 1,
        "cached/language": 1
      },
      "changed_system_predictions": 0,
      "fresh_sql_parity_checks": 3,
      "maximum_likelihood_error": 0.0,
      "prediction_sha256": "f9b4a6ca075ad0110315bb1dd19e8eac4b6df47252730596b25c6933dae521d9",
      "primary": "char",
      "native_correct": 632,
      "native_percent": 26.5993265993266,
      "previous_native_correct": 632,
      "previous_native_percent": 26.5993265993266,
      "native_change_points": 0.0,
      "assisted_correct": 1476,
      "assisted_percent": 62.121212121212125,
      "previous_assisted_percent": 62.12,
      "assisted_change_points": 0.0
    },
    "arc_challenge": {
      "n": 1172,
      "seconds": 11.865461300010793,
      "routes": {
        "cached/language+evidence": 1169,
        "cached/exact-arithmetic": 2,
        "cached/language": 1
      },
      "changed_system_predictions": 0,
      "fresh_sql_parity_checks": 3,
      "maximum_likelihood_error": 0.0,
      "prediction_sha256": "301b66946ce1cda420019aca3e1240bc9c262f2cb4f73e909b3ad19e61f76675",
      "primary": "char",
      "native_correct": 260,
      "native_percent": 22.18430034129693,
      "previous_native_correct": 260,
      "previous_native_percent": 22.18430034129693,
      "native_change_points": 0.0,
      "assisted_correct": 371,
      "assisted_percent": 31.655290102389078,
      "previous_assisted_percent": 31.66,
      "assisted_change_points": -0.0
    },
    "piqa": {
      "n": 1838,
      "seconds": 10.459679999970831,
      "routes": {
        "cached/language": 433,
        "cached/language+evidence": 1405
      },
      "changed_system_predictions": 0,
      "fresh_sql_parity_checks": 3,
      "maximum_likelihood_error": 0.0,
      "prediction_sha256": "cf2a6f1adc18959b67dce5af1fa668e7f4301d7eb9b25d3918fdc221c8369597",
      "primary": "char",
      "native_correct": 899,
      "native_percent": 48.91186071817192,
      "previous_native_correct": 899,
      "previous_native_percent": 48.91186071817192,
      "native_change_points": 0.0,
      "assisted_correct": 1078,
      "assisted_percent": 58.65070729053319,
      "previous_assisted_percent": 58.65,
      "assisted_change_points": 0.0
    },
    "arithmark3": {
      "n": 1000,
      "seconds": 9.040633900032844,
      "routes": {
        "cached/exact-arithmetic": 680,
        "cached/language+evidence": 320
      },
      "changed_system_predictions": 0,
      "fresh_sql_parity_checks": 3,
      "maximum_likelihood_error": 0.0,
      "prediction_sha256": "9cb2c70adfac81b8fde8c1fa2079553530d667b15d3190db54a7b76dce26f83d",
      "primary": "token",
      "native_correct": 378,
      "native_percent": 37.8,
      "previous_native_correct": 378,
      "previous_native_percent": 37.8,
      "native_change_points": 0.0,
      "assisted_correct": 695,
      "assisted_percent": 69.5,
      "previous_assisted_percent": 69.5,
      "assisted_change_points": 0.0
    }
  },
  "total_questions": 16428,
  "seconds": 200.08493640000233,
  "native_mean_percent": 32.693600618793546,
  "assisted_mean_percent": 49.97197844933156,
  "current_index_formula": "Chance-normalized HellaSwag + combined ARC + PIQA + 0.65*ArithMark, divided by 3.65",
  "local_index_native": 3.3067104959787845,
  "local_index_assisted": 24.373331426529237,
  "whole_system_evaluation_cached": true,
  "fresh_likelihoods": true,
  "training": false,
  "official_submission": false,
  "freeze_sha256": "d1f3b91f49fe8036b10a814daca5eb20e9e7c64f7551e86cc475a8c14373b1e6",
  "sql_parity_sha256": "27cc34d0f40d3f369c42d6f17a9ef3b42bb94c0f514f21037af7be08235128c1"
}