File size: 13,493 Bytes
5fd201e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
{
  "space": "dellweb3/VibeCoding",
  "space_commit": "caf8059cb14fac03c3fa17e2009de2018475d004",
  "complete": false,
  "reason": "calculator still failed; iframe popup lacked partitioned session",
  "jobs": [
    {
      "scenario": "erp",
      "run_id": "run-f35638c64a7b42d2b4c61b543881c899",
      "status": "verified",
      "error": null,
      "repairs": 0,
      "source_digest": "4b781dd92bf5e63717b5943a10cdd5f70d6a0fb4916a4c23b3e1d5fabd65a8af",
      "model_calls": [
        {
          "id": 1,
          "run_id": "run-f35638c64a7b42d2b4c61b543881c899",
          "reservation": 19216,
          "charged": 5215,
          "state": "settled",
          "provider": "bailian-qwen3.8-flash",
          "prompt_digest": "90f3c2cf88ab843202162ec1af9a0bafe1961b3414f17eb3cadedb4b0a807d6f",
          "metadata": {
            "finish_reason": "stop",
            "model": "qwen3.8-flash",
            "policy_version": "bailian-task-routing-v2",
            "request_id": "chatcmpl-e9d3b446-c31b-993b-b343-37f55f44592f",
            "requested_output_tokens": 2048,
            "retry_of": null,
            "route_reason": "technical-design",
            "stage": "design"
          }
        },
        {
          "id": 2,
          "run_id": "run-f35638c64a7b42d2b4c61b543881c899",
          "reservation": 16458,
          "charged": 6080,
          "state": "settled",
          "provider": "bailian-qwen3.8-max",
          "prompt_digest": "c2792b96b5106b588f1afd046150efbc0667493cbebe95fa8bcad4adccd17106",
          "metadata": {
            "finish_reason": "stop",
            "model": "qwen3.8-max",
            "policy_version": "bailian-task-routing-v2",
            "request_id": "chatcmpl-e351056a-5192-9e54-afd2-d56bd75de39d",
            "requested_output_tokens": 8192,
            "retry_of": null,
            "route_reason": "business-code",
            "stage": "code"
          }
        }
      ],
      "diagnostics": null
    },
    {
      "scenario": "calculator",
      "run_id": "run-4cf09e70e862457dbeb06aca1dfd5e58",
      "status": "failed",
      "error": "JAVA_TEST_FAILED",
      "repairs": 2,
      "source_digest": "524cadf952160e062ec774da01e0a13657a41d21924733895b598f082bef7871",
      "model_calls": [
        {
          "id": 3,
          "run_id": "run-4cf09e70e862457dbeb06aca1dfd5e58",
          "reservation": 20000,
          "charged": 5605,
          "state": "settled",
          "provider": "bailian-qwen3.8-flash",
          "prompt_digest": "6601055d26e44c2a07faeb048d1bcb3df71440dcc2c08b750d5df8875300945a",
          "metadata": {
            "finish_reason": "stop",
            "model": "qwen3.8-flash",
            "policy_version": "bailian-task-routing-v2",
            "request_id": "chatcmpl-d8338f17-4999-92bc-b985-759a20000941",
            "requested_output_tokens": 1151,
            "retry_of": null,
            "route_reason": "technical-design",
            "stage": "design"
          }
        },
        {
          "id": 4,
          "run_id": "run-4cf09e70e862457dbeb06aca1dfd5e58",
          "reservation": 16966,
          "charged": 3500,
          "state": "settled",
          "provider": "bailian-qwen3.8-max",
          "prompt_digest": "8e9c0824fc9eb316dc13fb841d0b5d0c0b23ac455bd7943c4ed71b1a4d129366",
          "metadata": {
            "finish_reason": "stop",
            "model": "qwen3.8-max",
            "policy_version": "bailian-task-routing-v2",
            "request_id": "chatcmpl-0a23d3ab-85cf-9103-8116-923de7d8fc1d",
            "requested_output_tokens": 8192,
            "retry_of": null,
            "route_reason": "java-code",
            "stage": "code"
          }
        },
        {
          "id": 5,
          "run_id": "run-4cf09e70e862457dbeb06aca1dfd5e58",
          "reservation": 26358,
          "charged": 5730,
          "state": "settled",
          "provider": "bailian-qwen3.8-max",
          "prompt_digest": "f87a694028417addb113ace4acf880b46888bda62cb3a6a08300d8854e9e1132",
          "metadata": {
            "finish_reason": "stop",
            "model": "qwen3.8-max",
            "policy_version": "bailian-task-routing-v2",
            "request_id": "chatcmpl-fbf28bae-1cef-989e-b7c1-6275160f6f8e",
            "requested_output_tokens": 12288,
            "retry_of": null,
            "route_reason": "diagnostic-repair",
            "stage": "repair"
          }
        },
        {
          "id": 6,
          "run_id": "run-4cf09e70e862457dbeb06aca1dfd5e58",
          "reservation": 23930,
          "charged": 4888,
          "state": "settled",
          "provider": "bailian-qwen3.8-max",
          "prompt_digest": "e88f679e89e45109232c9c6b578c526fac4813182834975916b5202b7ecee4ac",
          "metadata": {
            "finish_reason": "stop",
            "model": "qwen3.8-max",
            "policy_version": "bailian-task-routing-v2",
            "request_id": "chatcmpl-184a947e-da09-9c3c-8b9a-27646c1b284a",
            "requested_output_tokens": 12288,
            "retry_of": null,
            "route_reason": "diagnostic-repair",
            "stage": "repair"
          }
        }
      ],
      "diagnostics": {
        "failures": [
          {
            "actual_body": "INVALID_EXPRESSION",
            "actual_status": 400,
            "case_index": 13,
            "expected_status": 200,
            "path": "/api/calculate?expression64=IDEyICsgKCAzICogMiApIA",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "INVALID_EXPRESSION",
            "actual_status": 400,
            "case_index": 16,
            "expected_status": 400,
            "path": "/api/calculate?expression64=MSs",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "INVALID_EXPRESSION",
            "actual_status": 400,
            "case_index": 17,
            "expected_status": 400,
            "path": "/api/calculate?expression64=KDErMg",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "INVALID_EXPRESSION",
            "actual_status": 400,
            "case_index": 18,
            "expected_status": 400,
            "path": "/api/calculate?expression64=MSAy",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "INVALID_EXPRESSION",
            "actual_status": 400,
            "case_index": 19,
            "expected_status": 400,
            "path": "/api/calculate?expression64=MigzKQ",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "INVALID_EXPRESSION",
            "actual_status": 400,
            "case_index": 20,
            "expected_status": 400,
            "path": "/api/calculate?expression64=MSth",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "INVALID_EXPRESSION",
            "actual_status": 400,
            "case_index": 21,
            "expected_status": 400,
            "path": "/api/calculate?expression64=MWUz",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "INVALID_EXPRESSION",
            "actual_status": 400,
            "case_index": 22,
            "expected_status": 400,
            "path": "/api/calculate?expression64=IA",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "{\"expression\":\"1.\",\"result\":\"1\"}",
            "actual_status": 200,
            "case_index": 23,
            "expected_status": 400,
            "path": "/api/calculate?expression64=MS4",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "DIVISION_BY_ZERO",
            "actual_status": 400,
            "case_index": 24,
            "expected_status": 400,
            "path": "/api/calculate?expression64=MS8oMi0yKQ",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "LIMIT_EXCEEDED",
            "actual_status": 422,
            "case_index": 25,
            "expected_status": 422,
            "path": "/api/calculate?expression64=MSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsxKzErMSsx",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "LIMIT_EXCEEDED",
            "actual_status": 422,
            "case_index": 26,
            "expected_status": 422,
            "path": "/api/calculate?expression64=KCgoKCgoKCgoKCgoKCgoKCgxKSkpKSkpKSkpKSkpKSkpKSk",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "LIMIT_EXCEEDED",
            "actual_status": 422,
            "case_index": 27,
            "expected_status": 422,
            "path": "/api/calculate?expression64=LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0x",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "LIMIT_EXCEEDED",
            "actual_status": 422,
            "case_index": 28,
            "expected_status": 422,
            "path": "/api/calculate?expression64=MTExMTExMTExMTExMTExMTExMTExMTExMQ",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "LIMIT_EXCEEDED",
            "actual_status": 422,
            "case_index": 29,
            "expected_status": 422,
            "path": "/api/calculate?expression64=OTk5OTk5OTk5OTk5OTk5OTk5OTk5OTk5KjI",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "INVALID_INPUT",
            "actual_status": 400,
            "case_index": 30,
            "expected_status": 400,
            "path": "/api/calculate",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "INVALID_INPUT",
            "actual_status": 400,
            "case_index": 31,
            "expected_status": 400,
            "path": "/api/calculate?expression64=MQ&expression64=Mg",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "INVALID_INPUT",
            "actual_status": 400,
            "case_index": 32,
            "expected_status": 400,
            "path": "/api/calculate?expression64=MQ&other=Mg",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "INVALID_INPUT",
            "actual_status": 400,
            "case_index": 33,
            "expected_status": 400,
            "path": "/api/calculate?expression64=A",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "INVALID_INPUT",
            "actual_status": 400,
            "case_index": 34,
            "expected_status": 400,
            "path": "/api/calculate?expression64=MQ==",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "INVALID_INPUT",
            "actual_status": 400,
            "case_index": 35,
            "expected_status": 400,
            "path": "/api/calculate?expression64=_w",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "UNAUTHORIZED",
            "actual_status": 401,
            "case_index": 36,
            "expected_status": 401,
            "path": "/api/calculate?expression64=MSsx",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "METHOD_NOT_ALLOWED",
            "actual_status": 405,
            "case_index": 37,
            "expected_status": 405,
            "path": "/api/calculate?expression64=MSsx",
            "reason": "response_did_not_match_public_contract"
          },
          {
            "actual_body": "<h1>404 Not Found</h1>No context found for request",
            "actual_status": 404,
            "case_index": 38,
            "expected_status": 404,
            "path": "/unknown",
            "reason": "response_did_not_match_public_contract"
          }
        ]
      }
    }
  ],
  "model_calls": 6,
  "charged_tokens": 31018,
  "hub_failure": {
    "scope": "fresh Hub iframe session with no prior direct-app login",
    "passed": false,
    "failure": "TimeoutError: Timeout 330000ms exceeded.\n=========================== logs ===========================\nwaiting for navigation to \"https://dellweb3-vibecoding.hf.space/preview-site/?lease=*\" until 'load'\n============================================================",
    "cleanup": {
      "owned_leases": 1
    }
  },
  "protected_tests_weakened": false
}