File size: 1,810 Bytes
99b036c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
{
  "metric": "pass@1",
  "complete": true,
  "graded_cells": 1000,
  "average_pass_at_1": 0.523,
  "harnesses": {
    "opencode": {
      "evaluations": 250,
      "pass_at_1": 0.58,
      "difficulty": {
        "easy": {
          "evaluations": 33,
          "pass_at_1": 0.9090909090909091
        },
        "medium": {
          "evaluations": 118,
          "pass_at_1": 0.6101694915254238
        },
        "hard": {
          "evaluations": 99,
          "pass_at_1": 0.43434343434343436
        }
      }
    },
    "claude-code": {
      "evaluations": 250,
      "pass_at_1": 0.42,
      "difficulty": {
        "easy": {
          "evaluations": 33,
          "pass_at_1": 0.6060606060606061
        },
        "medium": {
          "evaluations": 118,
          "pass_at_1": 0.4576271186440678
        },
        "hard": {
          "evaluations": 99,
          "pass_at_1": 0.31313131313131315
        }
      }
    },
    "codex": {
      "evaluations": 250,
      "pass_at_1": 0.432,
      "difficulty": {
        "easy": {
          "evaluations": 33,
          "pass_at_1": 0.6666666666666666
        },
        "medium": {
          "evaluations": 118,
          "pass_at_1": 0.4915254237288136
        },
        "hard": {
          "evaluations": 99,
          "pass_at_1": 0.2828282828282828
        }
      }
    },
    "mini-swe-agent": {
      "evaluations": 250,
      "pass_at_1": 0.66,
      "difficulty": {
        "easy": {
          "evaluations": 33,
          "pass_at_1": 0.8787878787878788
        },
        "medium": {
          "evaluations": 118,
          "pass_at_1": 0.6864406779661016
        },
        "hard": {
          "evaluations": 99,
          "pass_at_1": 0.5555555555555556
        }
      }
    }
  },
  "tito_pass": true,
  "comparison_ready": true
}