Instructions to use EARAI/EARAI0.1 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use EARAI/EARAI0.1 with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download EARAI/EARAI0.1 --local-dir EARAI0.1
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Update metrics.json
Browse files- metrics.json +22 -1
metrics.json
CHANGED
|
@@ -8,7 +8,8 @@
|
|
| 8 |
"human_eval_data_sha256": "b796127e635a67f93fb35c04f4cb03cf06f38c8072ee7cee8833d7bee06979ef",
|
| 9 |
"mbpp_data_sha256": "ca95deaa9a01ef0a6f439f88bcf0dd3db3563d22f22aad6cae04ebb9a8d8c8e9",
|
| 10 |
"reference_checks": "10/10 HumanEval and 10/10 MBPP reference solutions passed in the same executor",
|
| 11 |
-
"limitations": "
|
|
|
|
| 12 |
},
|
| 13 |
"earai": {
|
| 14 |
"humaneval_full": {
|
|
@@ -50,5 +51,25 @@
|
|
| 50 |
"syntax_valid": 0,
|
| 51 |
"prefix_exceeded": 0
|
| 52 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
}
|
| 54 |
}
|
|
|
|
| 8 |
"human_eval_data_sha256": "b796127e635a67f93fb35c04f4cb03cf06f38c8072ee7cee8833d7bee06979ef",
|
| 9 |
"mbpp_data_sha256": "ca95deaa9a01ef0a6f439f88bcf0dd3db3563d22f22aad6cae04ebb9a8d8c8e9",
|
| 10 |
"reference_checks": "10/10 HumanEval and 10/10 MBPP reference solutions passed in the same executor",
|
| 11 |
+
"limitations": "Models use different tokenizers and interfaces; the matched 20-task comparison is too small to establish a general ranking; HumanEval prompt forced as prefix and counted within output limit; MBPP is a random 30-task sample with one visible test; training-set contamination not excluded.",
|
| 12 |
+
"market_eval": "research/bench_humaneval_market.py"
|
| 13 |
},
|
| 14 |
"earai": {
|
| 15 |
"humaneval_full": {
|
|
|
|
| 51 |
"syntax_valid": 0,
|
| 52 |
"prefix_exceeded": 0
|
| 53 |
}
|
| 54 |
+
},
|
| 55 |
+
"qwen25_0_5b": {
|
| 56 |
+
"parameters": 494032768,
|
| 57 |
+
"humaneval_matched_subset": {
|
| 58 |
+
"tasks": 20,
|
| 59 |
+
"passed": 7,
|
| 60 |
+
"syntax_valid": 19,
|
| 61 |
+
"prefix_exceeded": 0
|
| 62 |
+
},
|
| 63 |
+
"protocol": "same forced prefix, 768-token budget, docker executor, top-level truncation"
|
| 64 |
+
},
|
| 65 |
+
"qwen25_coder_0_5b": {
|
| 66 |
+
"parameters": 494032768,
|
| 67 |
+
"humaneval_matched_subset": {
|
| 68 |
+
"tasks": 20,
|
| 69 |
+
"passed": 3,
|
| 70 |
+
"syntax_valid": 19,
|
| 71 |
+
"prefix_exceeded": 0
|
| 72 |
+
},
|
| 73 |
+
"protocol": "same forced prefix, 768-token budget, docker executor, top-level truncation"
|
| 74 |
}
|
| 75 |
}
|