File size: 2,362 Bytes
2100fa6
 
 
 
 
 
 
 
 
 
6ac4bfa
 
2100fa6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6ac4bfa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2100fa6
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
{
  "release_step": 87776,
  "methodology": {
    "temperature": 0,
    "human_eval_stop": "same top-level stop rules for both models",
    "token_limit": "768 total tokens using each model tokenizer; EAR task includes forced prefix",
    "executor": "Docker isolated, network disabled",
    "human_eval_data_sha256": "b796127e635a67f93fb35c04f4cb03cf06f38c8072ee7cee8833d7bee06979ef",
    "mbpp_data_sha256": "ca95deaa9a01ef0a6f439f88bcf0dd3db3563d22f22aad6cae04ebb9a8d8c8e9",
    "reference_checks": "10/10 HumanEval and 10/10 MBPP reference solutions passed in the same executor",
    "limitations": "Models use different tokenizers and interfaces; the matched 20-task comparison is too small to establish a general ranking; HumanEval prompt forced as prefix and counted within output limit; MBPP is a random 30-task sample with one visible test; training-set contamination not excluded.",
    "market_eval": "research/bench_humaneval_market.py"
  },
  "earai": {
    "humaneval_full": {
      "tasks": 164,
      "passed": 0,
      "syntax_valid": 123,
      "prefix_exceeded": 17
    },
    "mbpp_sanitized_sample": {
      "tasks": 30,
      "passed": 0,
      "syntax_valid": 0,
      "prefix_exceeded": 0
    },
    "humaneval_matched_subset": {
      "tasks": 20,
      "passed": 0,
      "syntax_valid": 15,
      "prefix_exceeded": 3
    },
    "mbpp_matched_subset": {
      "tasks": 10,
      "passed": 0,
      "syntax_valid": 0,
      "prefix_exceeded": 0
    }
  },
  "tinystarcoder_py": {
    "parameters": 164144128,
    "humaneval_matched_subset": {
      "tasks": 20,
      "passed": 1,
      "syntax_valid": 19,
      "prefix_exceeded": 0
    },
    "mbpp_matched_subset": {
      "tasks": 10,
      "passed": 0,
      "syntax_valid": 0,
      "prefix_exceeded": 0
    }
  },
  "qwen25_0_5b": {
    "parameters": 494032768,
    "humaneval_matched_subset": {
      "tasks": 20,
      "passed": 7,
      "syntax_valid": 19,
      "prefix_exceeded": 0
    },
    "protocol": "same forced prefix, 768-token budget, docker executor, top-level truncation"
  },
  "qwen25_coder_0_5b": {
    "parameters": 494032768,
    "humaneval_matched_subset": {
      "tasks": 20,
      "passed": 3,
      "syntax_valid": 19,
      "prefix_exceeded": 0
    },
    "protocol": "same forced prefix, 768-token budget, docker executor, top-level truncation"
  }
}