File size: 2,316 Bytes
6b8c1c2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
{
  "metric": "pass@1",
  "complete": true,
  "graded_cells": 1000,
  "average_pass_at_1": 0.395,
  "harnesses": {
    "opencode": {
      "graded": 250,
      "correct": 100.0,
      "pass_at_1": 0.4,
      "delta_from_base": 0.29200000000000004,
      "difficulty": {
        "easy": {
          "graded": 33,
          "correct": 23.0,
          "pass_at_1": 0.696969696969697
        },
        "medium": {
          "graded": 118,
          "correct": 52.0,
          "pass_at_1": 0.4406779661016949
        },
        "hard": {
          "graded": 99,
          "correct": 25.0,
          "pass_at_1": 0.25252525252525254
        }
      }
    },
    "claude-code": {
      "graded": 250,
      "correct": 116.0,
      "pass_at_1": 0.464,
      "delta_from_base": 0.29600000000000004,
      "difficulty": {
        "easy": {
          "graded": 33,
          "correct": 26.0,
          "pass_at_1": 0.7878787878787878
        },
        "medium": {
          "graded": 118,
          "correct": 63.0,
          "pass_at_1": 0.5338983050847458
        },
        "hard": {
          "graded": 99,
          "correct": 27.0,
          "pass_at_1": 0.2727272727272727
        }
      }
    },
    "codex": {
      "graded": 250,
      "correct": 93.0,
      "pass_at_1": 0.372,
      "delta_from_base": 0.208,
      "difficulty": {
        "easy": {
          "graded": 33,
          "correct": 23.0,
          "pass_at_1": 0.696969696969697
        },
        "medium": {
          "graded": 118,
          "correct": 50.0,
          "pass_at_1": 0.423728813559322
        },
        "hard": {
          "graded": 99,
          "correct": 20.0,
          "pass_at_1": 0.20202020202020202
        }
      }
    },
    "mini-swe-agent": {
      "graded": 250,
      "correct": 86.0,
      "pass_at_1": 0.344,
      "delta_from_base": 0.19999999999999998,
      "difficulty": {
        "easy": {
          "graded": 33,
          "correct": 22.0,
          "pass_at_1": 0.6666666666666666
        },
        "medium": {
          "graded": 118,
          "correct": 45.0,
          "pass_at_1": 0.3813559322033898
        },
        "hard": {
          "graded": 99,
          "correct": 19.0,
          "pass_at_1": 0.1919191919191919
        }
      }
    }
  },
  "tito_pass": true,
  "comparison_ready": true
}