EARAI commited on
Commit
6ac4bfa
·
verified ·
1 Parent(s): 5ba5f25

Update metrics.json

Browse files
Files changed (1) hide show
  1. metrics.json +22 -1
metrics.json CHANGED
@@ -8,7 +8,8 @@
8
  "human_eval_data_sha256": "b796127e635a67f93fb35c04f4cb03cf06f38c8072ee7cee8833d7bee06979ef",
9
  "mbpp_data_sha256": "ca95deaa9a01ef0a6f439f88bcf0dd3db3563d22f22aad6cae04ebb9a8d8c8e9",
10
  "reference_checks": "10/10 HumanEval and 10/10 MBPP reference solutions passed in the same executor",
11
- "limitations": "EAR and Tiny use different tokenizers and interfaces; MBPP is a random 30-task sample with one visible test, not standard full benchmark; training-set contamination not excluded."
 
12
  },
13
  "earai": {
14
  "humaneval_full": {
@@ -50,5 +51,25 @@
50
  "syntax_valid": 0,
51
  "prefix_exceeded": 0
52
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
53
  }
54
  }
 
8
  "human_eval_data_sha256": "b796127e635a67f93fb35c04f4cb03cf06f38c8072ee7cee8833d7bee06979ef",
9
  "mbpp_data_sha256": "ca95deaa9a01ef0a6f439f88bcf0dd3db3563d22f22aad6cae04ebb9a8d8c8e9",
10
  "reference_checks": "10/10 HumanEval and 10/10 MBPP reference solutions passed in the same executor",
11
+ "limitations": "Models use different tokenizers and interfaces; the matched 20-task comparison is too small to establish a general ranking; HumanEval prompt forced as prefix and counted within output limit; MBPP is a random 30-task sample with one visible test; training-set contamination not excluded.",
12
+ "market_eval": "research/bench_humaneval_market.py"
13
  },
14
  "earai": {
15
  "humaneval_full": {
 
51
  "syntax_valid": 0,
52
  "prefix_exceeded": 0
53
  }
54
+ },
55
+ "qwen25_0_5b": {
56
+ "parameters": 494032768,
57
+ "humaneval_matched_subset": {
58
+ "tasks": 20,
59
+ "passed": 7,
60
+ "syntax_valid": 19,
61
+ "prefix_exceeded": 0
62
+ },
63
+ "protocol": "same forced prefix, 768-token budget, docker executor, top-level truncation"
64
+ },
65
+ "qwen25_coder_0_5b": {
66
+ "parameters": 494032768,
67
+ "humaneval_matched_subset": {
68
+ "tasks": 20,
69
+ "passed": 3,
70
+ "syntax_valid": 19,
71
+ "prefix_exceeded": 0
72
+ },
73
+ "protocol": "same forced prefix, 768-token budget, docker executor, top-level truncation"
74
  }
75
  }