ritik12 commited on
Commit
c5255b5
·
verified ·
1 Parent(s): 69e2dfc

Leaderboard page

Browse files
Files changed (3) hide show
  1. README.md +6 -5
  2. data.json +1024 -0
  3. index.html +666 -17
README.md CHANGED
@@ -1,10 +1,11 @@
1
  ---
2
  title: WaterfallBench
3
- emoji: 🐨
4
- colorFrom: red
5
- colorTo: blue
6
  sdk: static
 
7
  pinned: false
 
 
8
  ---
9
-
10
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
  title: WaterfallBench
3
+ emoji: 📊
4
+ colorFrom: green
5
+ colorTo: gray
6
  sdk: static
7
+ app_file: index.html
8
  pinned: false
9
+ license: apache-2.0
10
+ short_description: Can a model execute an ABS waterfall to the cent?
11
  ---
 
 
data.json ADDED
@@ -0,0 +1,1024 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "WaterfallBench",
3
+ "version": "1.0",
4
+ "updated": "2026-11-15",
5
+ "harness": "single call, no tools",
6
+ "models": [
7
+ {
8
+ "key": "openai/gpt-6-astra",
9
+ "name": "GPT-6 Astra",
10
+ "lab": "OpenAI",
11
+ "tasks": 20,
12
+ "fully_resolved": 95.0,
13
+ "se": 4.87,
14
+ "nontrivial": 99.17,
15
+ "line_item": 99.58,
16
+ "tolerance": 100.0,
17
+ "cost": 8.3326,
18
+ "cost_total": 166.65,
19
+ "tokens": 409597,
20
+ "input_tokens": 407039,
21
+ "output_tokens": 2558,
22
+ "seconds": 64.2,
23
+ "results": {
24
+ "fw-deere-2026-b-2026-08": {
25
+ "ok": true,
26
+ "nt": 100.0,
27
+ "li": 100.0
28
+ },
29
+ "fw-carmax-2025-2-2025-04": {
30
+ "ok": true,
31
+ "nt": 100.0,
32
+ "li": 100.0
33
+ },
34
+ "fw-carmax-2025-3-2025-07": {
35
+ "ok": true,
36
+ "nt": 100.0,
37
+ "li": 100.0
38
+ },
39
+ "fw-carmax-2025-4-2025-10": {
40
+ "ok": true,
41
+ "nt": 100.0,
42
+ "li": 100.0
43
+ },
44
+ "fw-carmax-2026-1-2026-01": {
45
+ "ok": true,
46
+ "nt": 100.0,
47
+ "li": 100.0
48
+ },
49
+ "fw-carmax-2026-2-2026-04": {
50
+ "ok": true,
51
+ "nt": 100.0,
52
+ "li": 100.0
53
+ },
54
+ "fw-carmax-2026-3-2026-07": {
55
+ "ok": true,
56
+ "nt": 100.0,
57
+ "li": 100.0
58
+ },
59
+ "fw-deere-2025-b-2025-08": {
60
+ "ok": true,
61
+ "nt": 100.0,
62
+ "li": 100.0
63
+ },
64
+ "fw-deere-2026-2026-04": {
65
+ "ok": true,
66
+ "nt": 100.0,
67
+ "li": 100.0
68
+ },
69
+ "fw-nissanlease-2025-a-2025-01": {
70
+ "ok": true,
71
+ "nt": 100.0,
72
+ "li": 100.0
73
+ },
74
+ "fw-nissanlease-2025-b-2025-07": {
75
+ "ok": true,
76
+ "nt": 100.0,
77
+ "li": 100.0
78
+ },
79
+ "fw-nissanlease-2026-a-2026-02": {
80
+ "ok": false,
81
+ "nt": 83.3,
82
+ "li": 91.7
83
+ },
84
+ "fw-nissanlease-2026-b-2026-07": {
85
+ "ok": true,
86
+ "nt": 100.0,
87
+ "li": 100.0
88
+ },
89
+ "fw-worldomni-2025-a-2025-01": {
90
+ "ok": true,
91
+ "nt": 100.0,
92
+ "li": 100.0
93
+ },
94
+ "fw-worldomni-2025-b-2025-05": {
95
+ "ok": true,
96
+ "nt": 100.0,
97
+ "li": 100.0
98
+ },
99
+ "fw-worldomni-2025-c-2025-08": {
100
+ "ok": true,
101
+ "nt": 100.0,
102
+ "li": 100.0
103
+ },
104
+ "fw-worldomni-2025-d-2025-10": {
105
+ "ok": true,
106
+ "nt": 100.0,
107
+ "li": 100.0
108
+ },
109
+ "fw-worldomni-2026-a-2026-02": {
110
+ "ok": true,
111
+ "nt": 100.0,
112
+ "li": 100.0
113
+ },
114
+ "fw-worldomni-2026-b-2026-05": {
115
+ "ok": true,
116
+ "nt": 100.0,
117
+ "li": 100.0
118
+ },
119
+ "fw-worldomni-2026-c-2026-08": {
120
+ "ok": true,
121
+ "nt": 100.0,
122
+ "li": 100.0
123
+ }
124
+ }
125
+ },
126
+ {
127
+ "key": "anthropic/claude-fable-5-2",
128
+ "name": "Claude Fable 5.2",
129
+ "lab": "Anthropic",
130
+ "tasks": 20,
131
+ "fully_resolved": 95.0,
132
+ "se": 4.87,
133
+ "nontrivial": 99.09,
134
+ "line_item": 99.58,
135
+ "tolerance": 100.0,
136
+ "cost": 5.1818,
137
+ "cost_total": 103.64,
138
+ "tokens": 635508,
139
+ "input_tokens": 632454,
140
+ "output_tokens": 3053,
141
+ "seconds": 35.4,
142
+ "results": {
143
+ "fw-carmax-2025-2-2025-04": {
144
+ "ok": true,
145
+ "nt": 100.0,
146
+ "li": 100.0
147
+ },
148
+ "fw-carmax-2025-3-2025-07": {
149
+ "ok": true,
150
+ "nt": 100.0,
151
+ "li": 100.0
152
+ },
153
+ "fw-carmax-2025-4-2025-10": {
154
+ "ok": true,
155
+ "nt": 100.0,
156
+ "li": 100.0
157
+ },
158
+ "fw-carmax-2026-1-2026-01": {
159
+ "ok": true,
160
+ "nt": 100.0,
161
+ "li": 100.0
162
+ },
163
+ "fw-carmax-2026-2-2026-04": {
164
+ "ok": true,
165
+ "nt": 100.0,
166
+ "li": 100.0
167
+ },
168
+ "fw-carmax-2026-3-2026-07": {
169
+ "ok": false,
170
+ "nt": 81.8,
171
+ "li": 91.7
172
+ },
173
+ "fw-deere-2025-b-2025-08": {
174
+ "ok": true,
175
+ "nt": 100.0,
176
+ "li": 100.0
177
+ },
178
+ "fw-deere-2026-2026-04": {
179
+ "ok": true,
180
+ "nt": 100.0,
181
+ "li": 100.0
182
+ },
183
+ "fw-deere-2026-b-2026-08": {
184
+ "ok": true,
185
+ "nt": 100.0,
186
+ "li": 100.0
187
+ },
188
+ "fw-nissanlease-2025-a-2025-01": {
189
+ "ok": true,
190
+ "nt": 100.0,
191
+ "li": 100.0
192
+ },
193
+ "fw-nissanlease-2025-b-2025-07": {
194
+ "ok": true,
195
+ "nt": 100.0,
196
+ "li": 100.0
197
+ },
198
+ "fw-nissanlease-2026-a-2026-02": {
199
+ "ok": true,
200
+ "nt": 100.0,
201
+ "li": 100.0
202
+ },
203
+ "fw-nissanlease-2026-b-2026-07": {
204
+ "ok": true,
205
+ "nt": 100.0,
206
+ "li": 100.0
207
+ },
208
+ "fw-worldomni-2025-a-2025-01": {
209
+ "ok": true,
210
+ "nt": 100.0,
211
+ "li": 100.0
212
+ },
213
+ "fw-worldomni-2025-b-2025-05": {
214
+ "ok": true,
215
+ "nt": 100.0,
216
+ "li": 100.0
217
+ },
218
+ "fw-worldomni-2025-c-2025-08": {
219
+ "ok": true,
220
+ "nt": 100.0,
221
+ "li": 100.0
222
+ },
223
+ "fw-worldomni-2025-d-2025-10": {
224
+ "ok": true,
225
+ "nt": 100.0,
226
+ "li": 100.0
227
+ },
228
+ "fw-worldomni-2026-a-2026-02": {
229
+ "ok": true,
230
+ "nt": 100.0,
231
+ "li": 100.0
232
+ },
233
+ "fw-worldomni-2026-b-2026-05": {
234
+ "ok": true,
235
+ "nt": 100.0,
236
+ "li": 100.0
237
+ },
238
+ "fw-worldomni-2026-c-2026-08": {
239
+ "ok": true,
240
+ "nt": 100.0,
241
+ "li": 100.0
242
+ }
243
+ }
244
+ },
245
+ {
246
+ "key": "openai/gpt-6-luna",
247
+ "name": "GPT-6 Luna",
248
+ "lab": "OpenAI",
249
+ "tasks": 20,
250
+ "fully_resolved": 90.0,
251
+ "se": 6.71,
252
+ "nontrivial": 99.17,
253
+ "line_item": 98.68,
254
+ "tolerance": 98.68,
255
+ "cost": 0.0873,
256
+ "cost_total": 1.75,
257
+ "tokens": 414850,
258
+ "input_tokens": 407039,
259
+ "output_tokens": 7811,
260
+ "seconds": 86.6,
261
+ "results": {
262
+ "fw-carmax-2025-2-2025-04": {
263
+ "ok": true,
264
+ "nt": 100.0,
265
+ "li": 100.0
266
+ },
267
+ "fw-carmax-2025-3-2025-07": {
268
+ "ok": true,
269
+ "nt": 100.0,
270
+ "li": 100.0
271
+ },
272
+ "fw-carmax-2025-4-2025-10": {
273
+ "ok": true,
274
+ "nt": 100.0,
275
+ "li": 100.0
276
+ },
277
+ "fw-carmax-2026-1-2026-01": {
278
+ "ok": false,
279
+ "nt": 83.3,
280
+ "li": 92.6
281
+ },
282
+ "fw-carmax-2026-2-2026-04": {
283
+ "ok": true,
284
+ "nt": 100.0,
285
+ "li": 100.0
286
+ },
287
+ "fw-carmax-2026-3-2026-07": {
288
+ "ok": true,
289
+ "nt": 100.0,
290
+ "li": 100.0
291
+ },
292
+ "fw-deere-2025-b-2025-08": {
293
+ "ok": true,
294
+ "nt": 100.0,
295
+ "li": 100.0
296
+ },
297
+ "fw-deere-2026-2026-04": {
298
+ "ok": true,
299
+ "nt": 100.0,
300
+ "li": 100.0
301
+ },
302
+ "fw-deere-2026-b-2026-08": {
303
+ "ok": true,
304
+ "nt": 100.0,
305
+ "li": 100.0
306
+ },
307
+ "fw-nissanlease-2025-a-2025-01": {
308
+ "ok": true,
309
+ "nt": 100.0,
310
+ "li": 100.0
311
+ },
312
+ "fw-nissanlease-2025-b-2025-07": {
313
+ "ok": true,
314
+ "nt": 100.0,
315
+ "li": 100.0
316
+ },
317
+ "fw-nissanlease-2026-a-2026-02": {
318
+ "ok": true,
319
+ "nt": 100.0,
320
+ "li": 100.0
321
+ },
322
+ "fw-nissanlease-2026-b-2026-07": {
323
+ "ok": true,
324
+ "nt": 100.0,
325
+ "li": 100.0
326
+ },
327
+ "fw-worldomni-2025-a-2025-01": {
328
+ "ok": true,
329
+ "nt": 100.0,
330
+ "li": 100.0
331
+ },
332
+ "fw-worldomni-2025-b-2025-05": {
333
+ "ok": true,
334
+ "nt": 100.0,
335
+ "li": 100.0
336
+ },
337
+ "fw-worldomni-2025-c-2025-08": {
338
+ "ok": true,
339
+ "nt": 100.0,
340
+ "li": 100.0
341
+ },
342
+ "fw-worldomni-2025-d-2025-10": {
343
+ "ok": false,
344
+ "nt": 100.0,
345
+ "li": 81.0
346
+ },
347
+ "fw-worldomni-2026-a-2026-02": {
348
+ "ok": true,
349
+ "nt": 100.0,
350
+ "li": 100.0
351
+ },
352
+ "fw-worldomni-2026-b-2026-05": {
353
+ "ok": true,
354
+ "nt": 100.0,
355
+ "li": 100.0
356
+ },
357
+ "fw-worldomni-2026-c-2026-08": {
358
+ "ok": true,
359
+ "nt": 100.0,
360
+ "li": 100.0
361
+ }
362
+ }
363
+ },
364
+ {
365
+ "key": "fireworks/minimax-m3p1-flash",
366
+ "name": "MiniMax M3.1 Flash",
367
+ "lab": "MiniMax",
368
+ "tasks": 20,
369
+ "fully_resolved": 50.0,
370
+ "se": 11.18,
371
+ "nontrivial": 89.03,
372
+ "line_item": 89.15,
373
+ "tolerance": 90.14,
374
+ "cost": 0.0706,
375
+ "cost_total": 1.41,
376
+ "tokens": 418289,
377
+ "input_tokens": 400871,
378
+ "output_tokens": 17419,
379
+ "seconds": 105.9,
380
+ "results": {
381
+ "fw-carmax-2025-2-2025-04": {
382
+ "ok": true,
383
+ "nt": 100.0,
384
+ "li": 100.0
385
+ },
386
+ "fw-carmax-2025-3-2025-07": {
387
+ "ok": false,
388
+ "nt": 0.0,
389
+ "li": 0.0
390
+ },
391
+ "fw-carmax-2025-4-2025-10": {
392
+ "ok": false,
393
+ "nt": 83.3,
394
+ "li": 92.6
395
+ },
396
+ "fw-carmax-2026-1-2026-01": {
397
+ "ok": true,
398
+ "nt": 100.0,
399
+ "li": 100.0
400
+ },
401
+ "fw-carmax-2026-2-2026-04": {
402
+ "ok": false,
403
+ "nt": 83.3,
404
+ "li": 92.6
405
+ },
406
+ "fw-carmax-2026-3-2026-07": {
407
+ "ok": false,
408
+ "nt": 72.7,
409
+ "li": 87.5
410
+ },
411
+ "fw-deere-2025-b-2025-08": {
412
+ "ok": true,
413
+ "nt": 100.0,
414
+ "li": 100.0
415
+ },
416
+ "fw-deere-2026-2026-04": {
417
+ "ok": true,
418
+ "nt": 100.0,
419
+ "li": 100.0
420
+ },
421
+ "fw-deere-2026-b-2026-08": {
422
+ "ok": true,
423
+ "nt": 100.0,
424
+ "li": 100.0
425
+ },
426
+ "fw-nissanlease-2025-a-2025-01": {
427
+ "ok": true,
428
+ "nt": 100.0,
429
+ "li": 100.0
430
+ },
431
+ "fw-nissanlease-2025-b-2025-07": {
432
+ "ok": true,
433
+ "nt": 100.0,
434
+ "li": 100.0
435
+ },
436
+ "fw-nissanlease-2026-a-2026-02": {
437
+ "ok": true,
438
+ "nt": 100.0,
439
+ "li": 100.0
440
+ },
441
+ "fw-nissanlease-2026-b-2026-07": {
442
+ "ok": true,
443
+ "nt": 100.0,
444
+ "li": 100.0
445
+ },
446
+ "fw-worldomni-2025-a-2025-01": {
447
+ "ok": false,
448
+ "nt": 85.7,
449
+ "li": 95.2
450
+ },
451
+ "fw-worldomni-2025-b-2025-05": {
452
+ "ok": false,
453
+ "nt": 100.0,
454
+ "li": 95.2
455
+ },
456
+ "fw-worldomni-2025-c-2025-08": {
457
+ "ok": false,
458
+ "nt": 100.0,
459
+ "li": 80.0
460
+ },
461
+ "fw-worldomni-2025-d-2025-10": {
462
+ "ok": false,
463
+ "nt": 100.0,
464
+ "li": 76.2
465
+ },
466
+ "fw-worldomni-2026-a-2026-02": {
467
+ "ok": false,
468
+ "nt": 77.8,
469
+ "li": 72.7
470
+ },
471
+ "fw-worldomni-2026-b-2026-05": {
472
+ "ok": false,
473
+ "nt": 77.8,
474
+ "li": 90.9
475
+ },
476
+ "fw-worldomni-2026-c-2026-08": {
477
+ "ok": true,
478
+ "nt": 100.0,
479
+ "li": 100.0
480
+ }
481
+ }
482
+ },
483
+ {
484
+ "key": "fireworks/kimi-k3p1-pro",
485
+ "name": "Kimi K3.1 Pro",
486
+ "lab": "Moonshot AI",
487
+ "tasks": 20,
488
+ "fully_resolved": 45.0,
489
+ "se": 11.12,
490
+ "nontrivial": 92.32,
491
+ "line_item": 96.1,
492
+ "tolerance": 96.1,
493
+ "cost": 0.8904,
494
+ "cost_total": 17.81,
495
+ "tokens": 418687,
496
+ "input_tokens": 412055,
497
+ "output_tokens": 6632,
498
+ "seconds": 106.6,
499
+ "results": {
500
+ "fw-deere-2026-b-2026-08": {
501
+ "ok": true,
502
+ "nt": 100.0,
503
+ "li": 100.0
504
+ },
505
+ "fw-carmax-2025-2-2025-04": {
506
+ "ok": true,
507
+ "nt": 100.0,
508
+ "li": 100.0
509
+ },
510
+ "fw-carmax-2025-3-2025-07": {
511
+ "ok": true,
512
+ "nt": 100.0,
513
+ "li": 100.0
514
+ },
515
+ "fw-carmax-2025-4-2025-10": {
516
+ "ok": false,
517
+ "nt": 75.0,
518
+ "li": 88.9
519
+ },
520
+ "fw-carmax-2026-1-2026-01": {
521
+ "ok": true,
522
+ "nt": 100.0,
523
+ "li": 100.0
524
+ },
525
+ "fw-carmax-2026-2-2026-04": {
526
+ "ok": false,
527
+ "nt": 83.3,
528
+ "li": 92.6
529
+ },
530
+ "fw-carmax-2026-3-2026-07": {
531
+ "ok": true,
532
+ "nt": 100.0,
533
+ "li": 100.0
534
+ },
535
+ "fw-deere-2025-b-2025-08": {
536
+ "ok": false,
537
+ "nt": 85.7,
538
+ "li": 93.8
539
+ },
540
+ "fw-deere-2026-2026-04": {
541
+ "ok": true,
542
+ "nt": 100.0,
543
+ "li": 100.0
544
+ },
545
+ "fw-nissanlease-2025-a-2025-01": {
546
+ "ok": false,
547
+ "nt": 83.3,
548
+ "li": 91.7
549
+ },
550
+ "fw-nissanlease-2025-b-2025-07": {
551
+ "ok": false,
552
+ "nt": 91.7,
553
+ "li": 95.8
554
+ },
555
+ "fw-nissanlease-2026-a-2026-02": {
556
+ "ok": true,
557
+ "nt": 100.0,
558
+ "li": 100.0
559
+ },
560
+ "fw-nissanlease-2026-b-2026-07": {
561
+ "ok": false,
562
+ "nt": 75.0,
563
+ "li": 87.5
564
+ },
565
+ "fw-worldomni-2025-a-2025-01": {
566
+ "ok": true,
567
+ "nt": 100.0,
568
+ "li": 100.0
569
+ },
570
+ "fw-worldomni-2025-b-2025-05": {
571
+ "ok": false,
572
+ "nt": 100.0,
573
+ "li": 95.2
574
+ },
575
+ "fw-worldomni-2025-c-2025-08": {
576
+ "ok": false,
577
+ "nt": 85.7,
578
+ "li": 95.0
579
+ },
580
+ "fw-worldomni-2025-d-2025-10": {
581
+ "ok": false,
582
+ "nt": 100.0,
583
+ "li": 95.2
584
+ },
585
+ "fw-worldomni-2026-a-2026-02": {
586
+ "ok": false,
587
+ "nt": 88.9,
588
+ "li": 95.5
589
+ },
590
+ "fw-worldomni-2026-b-2026-05": {
591
+ "ok": true,
592
+ "nt": 100.0,
593
+ "li": 100.0
594
+ },
595
+ "fw-worldomni-2026-c-2026-08": {
596
+ "ok": false,
597
+ "nt": 77.8,
598
+ "li": 90.9
599
+ }
600
+ }
601
+ },
602
+ {
603
+ "key": "anthropic/claude-haiku-5-5",
604
+ "name": "Claude Haiku 5.5",
605
+ "lab": "Anthropic",
606
+ "tasks": 20,
607
+ "fully_resolved": 40.0,
608
+ "se": 10.95,
609
+ "nontrivial": 85.78,
610
+ "line_item": 93.83,
611
+ "tolerance": 95.73,
612
+ "cost": 0.343,
613
+ "cost_total": 6.86,
614
+ "tokens": 643169,
615
+ "input_tokens": 632454,
616
+ "output_tokens": 10714,
617
+ "seconds": 58.7,
618
+ "results": {
619
+ "fw-carmax-2025-2-2025-04": {
620
+ "ok": false,
621
+ "nt": 88.9,
622
+ "li": 95.8
623
+ },
624
+ "fw-carmax-2025-3-2025-07": {
625
+ "ok": false,
626
+ "nt": 83.3,
627
+ "li": 92.6
628
+ },
629
+ "fw-carmax-2025-4-2025-10": {
630
+ "ok": false,
631
+ "nt": 83.3,
632
+ "li": 92.6
633
+ },
634
+ "fw-carmax-2026-1-2026-01": {
635
+ "ok": false,
636
+ "nt": 75.0,
637
+ "li": 88.9
638
+ },
639
+ "fw-carmax-2026-2-2026-04": {
640
+ "ok": true,
641
+ "nt": 100.0,
642
+ "li": 100.0
643
+ },
644
+ "fw-carmax-2026-3-2026-07": {
645
+ "ok": false,
646
+ "nt": 72.7,
647
+ "li": 87.5
648
+ },
649
+ "fw-deere-2025-b-2025-08": {
650
+ "ok": true,
651
+ "nt": 100.0,
652
+ "li": 100.0
653
+ },
654
+ "fw-deere-2026-2026-04": {
655
+ "ok": false,
656
+ "nt": 85.7,
657
+ "li": 93.8
658
+ },
659
+ "fw-deere-2026-b-2026-08": {
660
+ "ok": true,
661
+ "nt": 100.0,
662
+ "li": 100.0
663
+ },
664
+ "fw-nissanlease-2025-a-2025-01": {
665
+ "ok": false,
666
+ "nt": 75.0,
667
+ "li": 87.5
668
+ },
669
+ "fw-nissanlease-2025-b-2025-07": {
670
+ "ok": false,
671
+ "nt": 75.0,
672
+ "li": 87.5
673
+ },
674
+ "fw-nissanlease-2026-a-2026-02": {
675
+ "ok": true,
676
+ "nt": 100.0,
677
+ "li": 100.0
678
+ },
679
+ "fw-nissanlease-2026-b-2026-07": {
680
+ "ok": false,
681
+ "nt": 75.0,
682
+ "li": 87.5
683
+ },
684
+ "fw-worldomni-2025-a-2025-01": {
685
+ "ok": false,
686
+ "nt": 57.1,
687
+ "li": 85.7
688
+ },
689
+ "fw-worldomni-2025-b-2025-05": {
690
+ "ok": true,
691
+ "nt": 100.0,
692
+ "li": 100.0
693
+ },
694
+ "fw-worldomni-2025-c-2025-08": {
695
+ "ok": true,
696
+ "nt": 100.0,
697
+ "li": 100.0
698
+ },
699
+ "fw-worldomni-2025-d-2025-10": {
700
+ "ok": true,
701
+ "nt": 100.0,
702
+ "li": 100.0
703
+ },
704
+ "fw-worldomni-2026-a-2026-02": {
705
+ "ok": false,
706
+ "nt": 88.9,
707
+ "li": 95.5
708
+ },
709
+ "fw-worldomni-2026-b-2026-05": {
710
+ "ok": false,
711
+ "nt": 55.6,
712
+ "li": 81.8
713
+ },
714
+ "fw-worldomni-2026-c-2026-08": {
715
+ "ok": true,
716
+ "nt": 100.0,
717
+ "li": 100.0
718
+ }
719
+ }
720
+ }
721
+ ],
722
+ "tasks": [
723
+ {
724
+ "id": "fw-carmax-2025-2-2025-04",
725
+ "family": "carmax",
726
+ "family_name": "CarMax Auto Owner Trust",
727
+ "deal": "CarMax Auto Owner Trust 2025-2",
728
+ "period": "202504",
729
+ "classes": 8,
730
+ "graded": 24,
731
+ "nontrivial": 9,
732
+ "ungraded": 3,
733
+ "ungraded_why": "Servicer report departs from the agreement's overcollateralization target",
734
+ "docs": 7,
735
+ "html_mb": 4.91,
736
+ "prompt_tokens": 629698
737
+ },
738
+ {
739
+ "id": "fw-carmax-2025-3-2025-07",
740
+ "family": "carmax",
741
+ "family_name": "CarMax Auto Owner Trust",
742
+ "deal": "CarMax Auto Owner Trust 2025-3",
743
+ "period": "202507",
744
+ "classes": 8,
745
+ "graded": 27,
746
+ "nontrivial": 12,
747
+ "ungraded": 0,
748
+ "ungraded_why": null,
749
+ "docs": 7,
750
+ "html_mb": 5.01,
751
+ "prompt_tokens": 632551
752
+ },
753
+ {
754
+ "id": "fw-carmax-2025-4-2025-10",
755
+ "family": "carmax",
756
+ "family_name": "CarMax Auto Owner Trust",
757
+ "deal": "CarMax Auto Owner Trust 2025-4",
758
+ "period": "202510",
759
+ "classes": 8,
760
+ "graded": 27,
761
+ "nontrivial": 12,
762
+ "ungraded": 0,
763
+ "ungraded_why": null,
764
+ "docs": 7,
765
+ "html_mb": 4.99,
766
+ "prompt_tokens": 633700
767
+ },
768
+ {
769
+ "id": "fw-carmax-2026-1-2026-01",
770
+ "family": "carmax",
771
+ "family_name": "CarMax Auto Owner Trust",
772
+ "deal": "CarMax Auto Owner Trust 2026-1",
773
+ "period": "202601",
774
+ "classes": 8,
775
+ "graded": 27,
776
+ "nontrivial": 12,
777
+ "ungraded": 0,
778
+ "ungraded_why": null,
779
+ "docs": 8,
780
+ "html_mb": 5.08,
781
+ "prompt_tokens": 647530
782
+ },
783
+ {
784
+ "id": "fw-carmax-2026-2-2026-04",
785
+ "family": "carmax",
786
+ "family_name": "CarMax Auto Owner Trust",
787
+ "deal": "CarMax Auto Owner Trust 2026-2",
788
+ "period": "202604",
789
+ "classes": 8,
790
+ "graded": 27,
791
+ "nontrivial": 12,
792
+ "ungraded": 0,
793
+ "ungraded_why": null,
794
+ "docs": 8,
795
+ "html_mb": 5.12,
796
+ "prompt_tokens": 647909
797
+ },
798
+ {
799
+ "id": "fw-carmax-2026-3-2026-07",
800
+ "family": "carmax",
801
+ "family_name": "CarMax Auto Owner Trust",
802
+ "deal": "CarMax Auto Owner Trust 2026-3",
803
+ "period": "202607",
804
+ "classes": 7,
805
+ "graded": 24,
806
+ "nontrivial": 11,
807
+ "ungraded": 0,
808
+ "ungraded_why": null,
809
+ "docs": 8,
810
+ "html_mb": 4.94,
811
+ "prompt_tokens": 614998
812
+ },
813
+ {
814
+ "id": "fw-deere-2025-b-2025-08",
815
+ "family": "deere",
816
+ "family_name": "John Deere Owner Trust",
817
+ "deal": "John Deere Owner Trust 2025-B",
818
+ "period": "202508",
819
+ "classes": 5,
820
+ "graded": 16,
821
+ "nontrivial": 7,
822
+ "ungraded": 2,
823
+ "ungraded_why": "First servicing fee needs a mid-period pool balance no document provides",
824
+ "docs": 6,
825
+ "html_mb": 7.76,
826
+ "prompt_tokens": 503783
827
+ },
828
+ {
829
+ "id": "fw-deere-2026-2026-04",
830
+ "family": "deere",
831
+ "family_name": "John Deere Owner Trust",
832
+ "deal": "John Deere Owner Trust 2026",
833
+ "period": "202604",
834
+ "classes": 5,
835
+ "graded": 16,
836
+ "nontrivial": 7,
837
+ "ungraded": 2,
838
+ "ungraded_why": "First servicing fee needs a mid-period pool balance no document provides",
839
+ "docs": 6,
840
+ "html_mb": 7.52,
841
+ "prompt_tokens": 501473
842
+ },
843
+ {
844
+ "id": "fw-deere-2026-b-2026-08",
845
+ "family": "deere",
846
+ "family_name": "John Deere Owner Trust",
847
+ "deal": "John Deere Owner Trust 2026-B",
848
+ "period": "202608",
849
+ "classes": 5,
850
+ "graded": 16,
851
+ "nontrivial": 7,
852
+ "ungraded": 2,
853
+ "ungraded_why": "First servicing fee needs a mid-period pool balance no document provides",
854
+ "docs": 7,
855
+ "html_mb": 7.67,
856
+ "prompt_tokens": 522651
857
+ },
858
+ {
859
+ "id": "fw-nissanlease-2025-a-2025-01",
860
+ "family": "nissanlease",
861
+ "family_name": "Nissan Auto Lease Trust",
862
+ "deal": "Nissan Auto Lease Trust 2025-A",
863
+ "period": "202501",
864
+ "classes": 7,
865
+ "graded": 24,
866
+ "nontrivial": 12,
867
+ "ungraded": 0,
868
+ "ungraded_why": null,
869
+ "docs": 10,
870
+ "html_mb": 8.23,
871
+ "prompt_tokens": 725064
872
+ },
873
+ {
874
+ "id": "fw-nissanlease-2025-b-2025-07",
875
+ "family": "nissanlease",
876
+ "family_name": "Nissan Auto Lease Trust",
877
+ "deal": "Nissan Auto Lease Trust 2025-B",
878
+ "period": "202507",
879
+ "classes": 7,
880
+ "graded": 24,
881
+ "nontrivial": 12,
882
+ "ungraded": 0,
883
+ "ungraded_why": null,
884
+ "docs": 10,
885
+ "html_mb": 7.76,
886
+ "prompt_tokens": 712865
887
+ },
888
+ {
889
+ "id": "fw-nissanlease-2026-a-2026-02",
890
+ "family": "nissanlease",
891
+ "family_name": "Nissan Auto Lease Trust",
892
+ "deal": "Nissan Auto Lease Trust 2026-A",
893
+ "period": "202602",
894
+ "classes": 7,
895
+ "graded": 24,
896
+ "nontrivial": 12,
897
+ "ungraded": 0,
898
+ "ungraded_why": null,
899
+ "docs": 10,
900
+ "html_mb": 8.48,
901
+ "prompt_tokens": 739161
902
+ },
903
+ {
904
+ "id": "fw-nissanlease-2026-b-2026-07",
905
+ "family": "nissanlease",
906
+ "family_name": "Nissan Auto Lease Trust",
907
+ "deal": "Nissan Auto Lease Trust 2026-B",
908
+ "period": "202607",
909
+ "classes": 7,
910
+ "graded": 24,
911
+ "nontrivial": 12,
912
+ "ungraded": 0,
913
+ "ungraded_why": null,
914
+ "docs": 10,
915
+ "html_mb": 8.2,
916
+ "prompt_tokens": 732825
917
+ },
918
+ {
919
+ "id": "fw-worldomni-2025-a-2025-01",
920
+ "family": "worldomni",
921
+ "family_name": "World Omni Auto Receivables Trust",
922
+ "deal": "World Omni Auto Receivables Trust 2025-A",
923
+ "period": "202501",
924
+ "classes": 7,
925
+ "graded": 21,
926
+ "nontrivial": 7,
927
+ "ungraded": 3,
928
+ "ungraded_why": "Agreement prorates the first servicing fee without a day count",
929
+ "docs": 7,
930
+ "html_mb": 6.37,
931
+ "prompt_tokens": 614546
932
+ },
933
+ {
934
+ "id": "fw-worldomni-2025-b-2025-05",
935
+ "family": "worldomni",
936
+ "family_name": "World Omni Auto Receivables Trust",
937
+ "deal": "World Omni Auto Receivables Trust 2025-B",
938
+ "period": "202505",
939
+ "classes": 7,
940
+ "graded": 21,
941
+ "nontrivial": 5,
942
+ "ungraded": 3,
943
+ "ungraded_why": "Agreement prorates the first servicing fee without a day count",
944
+ "docs": 7,
945
+ "html_mb": 6.35,
946
+ "prompt_tokens": 624079
947
+ },
948
+ {
949
+ "id": "fw-worldomni-2025-c-2025-08",
950
+ "family": "worldomni",
951
+ "family_name": "World Omni Auto Receivables Trust",
952
+ "deal": "World Omni Auto Receivables Trust 2025-C",
953
+ "period": "202508",
954
+ "classes": 7,
955
+ "graded": 20,
956
+ "nontrivial": 7,
957
+ "ungraded": 4,
958
+ "ungraded_why": "Agreement prorates the first servicing fee without a day count",
959
+ "docs": 7,
960
+ "html_mb": 6.58,
961
+ "prompt_tokens": 629893
962
+ },
963
+ {
964
+ "id": "fw-worldomni-2025-d-2025-10",
965
+ "family": "worldomni",
966
+ "family_name": "World Omni Auto Receivables Trust",
967
+ "deal": "World Omni Auto Receivables Trust 2025-D",
968
+ "period": "202510",
969
+ "classes": 7,
970
+ "graded": 21,
971
+ "nontrivial": 7,
972
+ "ungraded": 3,
973
+ "ungraded_why": "Agreement prorates the first servicing fee without a day count",
974
+ "docs": 7,
975
+ "html_mb": 8.05,
976
+ "prompt_tokens": 638804
977
+ },
978
+ {
979
+ "id": "fw-worldomni-2026-a-2026-02",
980
+ "family": "worldomni",
981
+ "family_name": "World Omni Auto Receivables Trust",
982
+ "deal": "World Omni Auto Receivables Trust 2026-A",
983
+ "period": "202602",
984
+ "classes": 7,
985
+ "graded": 22,
986
+ "nontrivial": 9,
987
+ "ungraded": 2,
988
+ "ungraded_why": "Agreement prorates the first servicing fee without a day count",
989
+ "docs": 7,
990
+ "html_mb": 6.98,
991
+ "prompt_tokens": 627265
992
+ },
993
+ {
994
+ "id": "fw-worldomni-2026-b-2026-05",
995
+ "family": "worldomni",
996
+ "family_name": "World Omni Auto Receivables Trust",
997
+ "deal": "World Omni Auto Receivables Trust 2026-B",
998
+ "period": "202605",
999
+ "classes": 7,
1000
+ "graded": 22,
1001
+ "nontrivial": 9,
1002
+ "ungraded": 2,
1003
+ "ungraded_why": "Agreement prorates the first servicing fee without a day count",
1004
+ "docs": 7,
1005
+ "html_mb": 6.35,
1006
+ "prompt_tokens": 633802
1007
+ },
1008
+ {
1009
+ "id": "fw-worldomni-2026-c-2026-08",
1010
+ "family": "worldomni",
1011
+ "family_name": "World Omni Auto Receivables Trust",
1012
+ "deal": "World Omni Auto Receivables Trust 2026-C",
1013
+ "period": "202608",
1014
+ "classes": 7,
1015
+ "graded": 22,
1016
+ "nontrivial": 9,
1017
+ "ungraded": 2,
1018
+ "ungraded_why": "Agreement prorates the first servicing fee without a day count",
1019
+ "docs": 7,
1020
+ "html_mb": 7.76,
1021
+ "prompt_tokens": 636492
1022
+ }
1023
+ ]
1024
+ }
index.html CHANGED
@@ -1,19 +1,668 @@
1
  <!doctype html>
2
- <html>
3
- <head>
4
- <meta charset="utf-8" />
5
- <meta name="viewport" content="width=device-width" />
6
- <title>My static Space</title>
7
- <link rel="stylesheet" href="style.css" />
8
- </head>
9
- <body>
10
- <div class="card">
11
- <h1>Welcome to your static Space!</h1>
12
- <p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
13
- <p>
14
- Also don't forget to check the
15
- <a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
16
- </p>
17
- </div>
18
- </body>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19
  </html>
 
1
  <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>WaterfallBench</title>
7
+ <meta name="description" content="WaterfallBench: can a model execute an asset-backed securities waterfall to the cent from the deal documents? Leaderboard and methodology.">
8
+ <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'%3E%3Crect width='32' height='32' rx='7' fill='%233f5f3a'/%3E%3Crect x='7' y='7' width='18' height='4' rx='1' fill='%23fff'/%3E%3Crect x='10' y='14' width='12' height='4' rx='1' fill='%23fff' opacity='.8'/%3E%3Crect x='13' y='21' width='6' height='4' rx='1' fill='%23fff' opacity='.6'/%3E%3C/svg%3E">
9
+ <link rel="preconnect" href="https://fonts.googleapis.com">
10
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
11
+ <link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500&family=Inter:wght@400;500;600&family=Newsreader:opsz,wght@6..72,400;6..72,500;6..72,600&display=swap" rel="stylesheet">
12
+ <style>
13
+ :root {
14
+ --bg: #f7f6f1; --panel: #ffffff; --panel-2: #f1efe8; --ink: #1c1c19; --ink-2: #3a3935; --muted: #6d6b63;
15
+ --line: #e3e0d6; --line-2: #d4d0c3; --accent: #3f5f3a; --accent-ink: #2f4a2b; --accent-soft: #e6ece0;
16
+ --good: #3f7a3a; --good-soft: #dfeadb; --warn: #a8741a; --warn-soft: #f4ead6; --bad: #a8473a; --bad-soft: #f3e1dd;
17
+ --eff: rgba(63, 95, 58, .07); --ineff: rgba(168, 71, 58, .07); --rm: #ece4cf;
18
+ --anthropic: #d97757; --openai: #1c1c19; --moonshot: #2f6fde; --minimax: #e2365f;
19
+ --serif: "Newsreader", Georgia, "Times New Roman", serif; --sans: "Inter", system-ui, -apple-system, "Segoe UI", sans-serif;
20
+ --mono: "IBM Plex Mono", ui-monospace, "SFMono-Regular", Menlo, monospace;
21
+ color-scheme: light;
22
+ }
23
+ @media (prefers-color-scheme: dark) {
24
+ :root:not([data-theme="light"]) {
25
+ --bg: #141412; --panel: #1b1b18; --panel-2: #22221e; --ink: #ecebe5; --ink-2: #d2d0c8; --muted: #9a988e;
26
+ --line: #2e2d28; --line-2: #3b3a34; --accent: #9fbf8f; --accent-ink: #b9d4aa; --accent-soft: #24301f;
27
+ --good: #8fc382; --good-soft: #213321; --warn: #d9a650; --warn-soft: #362b17; --bad: #e08b7c; --bad-soft: #3a211d;
28
+ --eff: rgba(143, 195, 130, .07); --ineff: rgba(224, 139, 124, .07); --rm: #3a3220;
29
+ --anthropic: #e08a6a; --openai: #ecebe5; --moonshot: #7aa2ff; --minimax: #ff6b86; color-scheme: dark;
30
+ }
31
+ }
32
+ :root[data-theme="dark"] {
33
+ --bg: #141412; --panel: #1b1b18; --panel-2: #22221e; --ink: #ecebe5; --ink-2: #d2d0c8; --muted: #9a988e;
34
+ --line: #2e2d28; --line-2: #3b3a34; --accent: #9fbf8f; --accent-ink: #b9d4aa; --accent-soft: #24301f;
35
+ --good: #8fc382; --good-soft: #213321; --warn: #d9a650; --warn-soft: #362b17; --bad: #e08b7c; --bad-soft: #3a211d;
36
+ --eff: rgba(143, 195, 130, .07); --ineff: rgba(224, 139, 124, .07); --rm: #3a3220;
37
+ --anthropic: #e08a6a; --openai: #ecebe5; --moonshot: #7aa2ff; --minimax: #ff6b86; color-scheme: dark;
38
+ }
39
+ * { box-sizing: border-box; }
40
+ html { -webkit-text-size-adjust: 100%; scroll-behavior: smooth; }
41
+ body { margin: 0; background: var(--bg); color: var(--ink); font-family: var(--serif); font-size: 18px; line-height: 1.6; }
42
+ a { color: inherit; text-decoration-color: var(--line-2); text-underline-offset: 3px; }
43
+ a:hover { text-decoration-color: var(--accent); }
44
+ code, pre { font-family: var(--mono); font-size: .82em; }
45
+ .label { font-family: var(--mono); font-size: 12px; letter-spacing: .06em; text-transform: uppercase; color: var(--muted); }
46
+ .logo { display: inline-block; flex: none; vertical-align: -3px; }
47
+
48
+ /* top bar */
49
+ .top { position: sticky; top: 0; z-index: 20; background: color-mix(in srgb, var(--bg) 92%, transparent); backdrop-filter: blur(8px); border-bottom: 1px solid var(--line); }
50
+ .top .in { max-width: 1200px; margin: 0 auto; padding: 12px 16px; display: flex; align-items: center; gap: 24px; }
51
+ .brand { display: flex; align-items: center; gap: 10px; font-family: var(--sans); font-weight: 600; font-size: 16px; text-decoration: none; letter-spacing: -.01em; }
52
+ .nav { display: flex; gap: 20px; margin-left: auto; font-family: var(--sans); font-size: 14px; }
53
+ .nav a { text-decoration: none; color: var(--ink-2); }
54
+ .nav a:hover { color: var(--accent); }
55
+ @media (max-width: 720px) { .nav a:not(.gh) { display: none; } }
56
+
57
+ /* hero */
58
+ .hero { max-width: 1200px; margin: 0 auto; padding: 44px 16px 24px; }
59
+ h1 { font-family: var(--serif); font-weight: 500; font-size: clamp(40px, 7vw, 56px); line-height: 1.05; letter-spacing: -.02em; margin: 10px 0 14px; }
60
+ .meta { display: flex; flex-wrap: wrap; align-items: center; gap: 14px; }
61
+ .meta .sep { width: 1px; height: 16px; background: var(--line-2); }
62
+ .meta a.label { text-decoration: none; }
63
+ .meta a.label:hover { color: var(--accent); }
64
+ .btn { display: inline-flex; align-items: center; gap: 8px; font-family: var(--mono); font-size: 12px; letter-spacing: .06em; text-transform: uppercase; padding: 6px 10px; border: 1px solid var(--line-2); background: var(--panel); color: var(--ink); text-decoration: none; cursor: pointer; }
65
+ .btn:hover { border-color: var(--accent); color: var(--accent-ink); }
66
+ .btn[aria-pressed="true"] { background: var(--accent-soft); border-color: var(--accent); color: var(--accent-ink); }
67
+
68
+ /* panels */
69
+ .wide { max-width: 1200px; margin: 0 auto; padding: 0 16px; }
70
+ .panel { background: var(--panel); border: 1px solid var(--line); }
71
+ .panel-head { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 8px 16px; padding: 12px 14px; border-bottom: 1px solid var(--line); }
72
+ .panel-head .t { display: flex; align-items: baseline; gap: 12px; flex-wrap: wrap; }
73
+ .panel-head .t b { font-family: var(--serif); font-weight: 600; font-size: 17px; }
74
+ .panel-head .t span { font-family: var(--sans); font-size: 14px; color: var(--accent); }
75
+ .toolbar { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; padding: 10px 14px; border-bottom: 1px solid var(--line); background: var(--panel-2); }
76
+ .toolbar .grow { flex: 1; }
77
+ .seg { display: inline-flex; border: 1px solid var(--line-2); }
78
+ .seg .btn { border: 0; }
79
+ .seg .btn + .btn { border-left: 1px solid var(--line-2); }
80
+ .legend { display: flex; flex-wrap: wrap; gap: 16px; align-items: center; padding: 8px 14px; border-bottom: 1px solid var(--line); }
81
+ .legend span { display: inline-flex; align-items: center; gap: 6px; }
82
+ .legend .fr-key { margin-left: auto; }
83
+ .legend .fr-key i { width: 18px; height: 2px; background: var(--accent); display: inline-block; }
84
+ .legend .se-key { width: 2px; height: 12px; background: var(--muted); display: inline-block; opacity: .6; }
85
+ #chart { position: relative; }
86
+ #chart svg { display: block; width: 100%; }
87
+ .tip { position: absolute; pointer-events: none; background: var(--panel); border: 1px solid var(--line-2); padding: 10px 12px; font-family: var(--sans); font-size: 13px; line-height: 1.45; box-shadow: 0 6px 24px rgba(0,0,0,.08); min-width: 190px; opacity: 0; transition: opacity .12s; z-index: 5; }
88
+ .tip b { font-weight: 600; }
89
+ .tip .row { display: flex; justify-content: space-between; gap: 16px; color: var(--muted); }
90
+ .tip .row span:last-child { color: var(--ink); font-family: var(--mono); font-size: 12px; }
91
+
92
+ /* tables */
93
+ .tbl-wrap { overflow-x: auto; }
94
+ table { width: 100%; border-collapse: collapse; font-family: var(--sans); font-size: 14.5px; }
95
+ th { text-align: left; font-weight: 500; font-size: 13px; color: var(--ink-2); padding: 10px 12px; border-bottom: 1px solid var(--line-2); background: var(--panel-2); white-space: nowrap; }
96
+ th.num, td.num { text-align: right; }
97
+ th.sort { cursor: pointer; user-select: none; }
98
+ th.sort:hover { color: var(--accent); }
99
+ th .arr { font-size: 10px; margin-left: 4px; color: var(--muted); }
100
+ td { padding: 11px 12px; border-bottom: 1px solid var(--line); white-space: nowrap; }
101
+ tbody tr:hover td { background: var(--panel-2); }
102
+ tr.top1 td { background: color-mix(in srgb, var(--accent-soft) 70%, var(--panel)); }
103
+ tr.top2 td { background: color-mix(in srgb, var(--accent-soft) 40%, var(--panel)); }
104
+ tr.top3 td { background: color-mix(in srgb, var(--accent-soft) 20%, var(--panel)); }
105
+ td.rank { font-family: var(--mono); font-size: 12.5px; color: var(--muted); width: 40px; }
106
+ .model { display: inline-flex; align-items: center; gap: 10px; font-weight: 500; }
107
+ .v { font-family: var(--mono); font-size: 13.5px; }
108
+ .v small { color: var(--muted); font-size: 11.5px; margin-left: 4px; }
109
+ .v .u { color: var(--muted); margin: 0 1px; }
110
+ .minibar { display: inline-block; width: 64px; height: 6px; background: var(--line); vertical-align: middle; margin-right: 10px; position: relative; }
111
+ .minibar b { position: absolute; inset: 0 auto 0 0; background: var(--accent); }
112
+
113
+ /* prose */
114
+ .prose { max-width: 820px; margin: 0 auto; padding: 0 16px; }
115
+ section { padding: 64px 0 0; }
116
+ section > .prose > h2 { font-family: var(--serif); font-weight: 500; font-size: 34px; letter-spacing: -.015em; line-height: 1.15; margin: 0 0 16px; }
117
+ .prose h3 { font-family: var(--serif); font-weight: 600; font-size: 21px; margin: 36px 0 6px; }
118
+ .prose p { margin: 0 0 16px; color: var(--ink-2); }
119
+ .prose strong { color: var(--ink); font-weight: 600; }
120
+ .cap { font-family: var(--sans); font-size: 13px; color: var(--muted); margin: 8px 0 0; }
121
+
122
+ /* task: inputs and output */
123
+ .inputs td { white-space: normal; vertical-align: top; font-size: 14px; }
124
+ .inputs td:first-child { font-weight: 500; white-space: nowrap; }
125
+ .inputs td.form { font-family: var(--mono); font-size: 12.5px; color: var(--muted); white-space: nowrap; }
126
+ .io { display: grid; grid-template-columns: 1.35fr 1fr; gap: 16px; margin-top: 18px; }
127
+ .io-eq { grid-template-columns: 1fr 1fr; }
128
+ .io-eq pre.json { font-size: 11.5px; }
129
+ @media (max-width: 960px) { .io, .io-eq { grid-template-columns: 1fr; } }
130
+ .inputs td.num { white-space: nowrap; }
131
+ .io .panel-head .t b { font-size: 15px; }
132
+ .rep { font-family: var(--mono); font-size: 12px; }
133
+ .rep th, .rep td { padding: 6px 10px; font-size: 12px; }
134
+ .rep th { font-family: var(--mono); font-weight: 500; font-size: 11px; color: var(--muted); }
135
+ .rep td.num { text-align: right; }
136
+ .rep .grp td { background: var(--panel-2); font-weight: 500; color: var(--ink-2); }
137
+ .rep td.rm { background: repeating-linear-gradient(-45deg, var(--rm) 0 6px, transparent 6px 12px); color: var(--warn); text-align: center; font-size: 10.5px; letter-spacing: .05em; }
138
+ pre.json { margin: 0; padding: 12px 14px; font-size: 12px; line-height: 1.55; color: var(--ink-2); overflow-x: auto; }
139
+ pre.json .k { color: var(--accent-ink); }
140
+ pre.json .q { background: var(--rm); color: var(--warn); padding: 0 4px; }
141
+
142
+ .wf { margin: 8px 0 0; font-family: var(--sans); }
143
+ .wf-root { display: inline-flex; flex-direction: column; padding: 10px 14px; background: var(--accent); color: var(--bg); }
144
+ .wf-root b { font-weight: 600; font-size: 15px; }
145
+ .wf-root span { font-size: 13px; opacity: .85; }
146
+ .wf-tree, .wf-tree ul { list-style: none; margin: 0; padding: 0; }
147
+ .wf-tree { margin-left: 22px; }
148
+ .wf-tree ul { margin-left: 30px; }
149
+ .wf-tree li { position: relative; padding: 7px 0 0 30px; }
150
+ .wf-tree li::before { content: ""; position: absolute; left: 0; top: 0; bottom: 0; border-left: 2px solid var(--line-2); }
151
+ .wf-tree li:last-child::before { bottom: auto; height: 27px; }
152
+ .wf-tree li::after { content: ""; position: absolute; left: 0; top: 27px; width: 26px; border-top: 2px solid var(--line-2); }
153
+ .wf-tree > li::before, .wf-tree > li::after { border-color: var(--accent); }
154
+ .wf-node, .wf-leaf { display: inline-flex; align-items: baseline; flex-wrap: wrap; gap: 4px 10px; padding: 8px 12px; background: var(--panel); border: 1px solid var(--line); font-size: 14.5px; }
155
+ .wf-node { border-left: 3px solid var(--accent); }
156
+ .wf-node i { font-style: normal; font-family: var(--mono); font-size: 12px; color: var(--muted); }
157
+ .wf-node b { font-weight: 600; }
158
+ .wf-node span, .wf-leaf small { color: var(--muted); font-size: 13px; }
159
+ .wf-leaf { background: var(--panel-2); font-size: 13.5px; padding: 6px 10px; }
160
+ .wf-tree ul li::after { top: 23px; }
161
+ .wf-tree ul li:last-child::before { height: 23px; }
162
+ .wf-seq { gap: 6px; color: var(--muted); }
163
+ .wf-seq em { font-style: normal; font-family: var(--mono); font-size: 12px; color: var(--ink); background: var(--panel); border: 1px solid var(--line); padding: 1px 6px; }
164
+ @media (max-width: 640px) { .wf-tree ul { margin-left: 14px; } .wf-tree li { padding-left: 22px; } .wf-tree li::after { width: 18px; } }
165
+ .g-ok { color: var(--good); font-weight: 600; }
166
+ .g-x { color: var(--bad); font-weight: 600; background: var(--bad-soft); padding: 0 3px; }
167
+ .g-t { color: var(--muted); font-weight: 400; }
168
+ /* dataset */
169
+ .kv { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); border: 1px solid var(--line); background: var(--panel); margin: 4px 0 20px; }
170
+ .kv > div { padding: 12px 14px; border-right: 1px solid var(--line); border-bottom: 1px solid var(--line); }
171
+ .compact th, .compact td { padding: 9px 10px; font-size: 13.5px; }
172
+ .kv b { display: block; font-family: var(--serif); font-weight: 500; font-size: 26px; line-height: 1.15; color: var(--ink); }
173
+
174
+ /* task grid */
175
+ .grid td, .grid th { padding: 8px 10px; }
176
+ .grid th.mh { text-align: center; font-size: 12px; white-space: normal; min-width: 76px; }
177
+ .grid td.cell { text-align: center; font-family: var(--mono); font-size: 12.5px; }
178
+ .cell .ok, .cell .part, .cell .low { display: inline-block; min-width: 50px; padding: 3px 0; }
179
+ .cell .ok { background: var(--good-soft); color: var(--good); }
180
+ .cell .part { background: var(--warn-soft); color: var(--warn); }
181
+ .cell .low { background: var(--bad-soft); color: var(--bad); }
182
+ .grid td.deal { font-size: 13.5px; }
183
+ .grid td.deal small { display: block; color: var(--muted); font-family: var(--mono); font-size: 11px; }
184
+ .grid .sticky { position: sticky; left: 0; background: var(--panel); z-index: 1; }
185
+ .grid tr:hover td.sticky { background: var(--panel-2); }
186
+
187
+ pre.code { background: var(--panel); border: 1px solid var(--line); padding: 14px 16px; overflow-x: auto; line-height: 1.55; color: var(--ink-2); margin: 0 0 16px; }
188
+ .plain { margin: 0 0 16px; padding-left: 20px; color: var(--ink-2); }
189
+ .plain li { margin-bottom: 6px; }
190
+ footer { margin-top: 72px; border-top: 1px solid var(--line); }
191
+ footer .in { max-width: 1200px; margin: 0 auto; padding: 20px 16px 36px; display: flex; flex-wrap: wrap; gap: 12px 24px; justify-content: space-between; font-family: var(--sans); font-size: 13.5px; color: var(--muted); }
192
+ .theme-btn { display: inline-grid; place-items: center; width: 32px; height: 32px; flex: none; background: var(--panel); border: 1px solid var(--line-2); color: var(--ink-2); cursor: pointer; padding: 0; }
193
+ .theme-btn:hover { border-color: var(--accent); color: var(--accent-ink); }
194
+ .theme-btn .i-sun { display: none; }
195
+ :root[data-theme="dark"] .theme-btn .i-sun { display: block; }
196
+ :root[data-theme="dark"] .theme-btn .i-moon { display: none; }
197
+ @media (prefers-color-scheme: dark) {
198
+ :root:not([data-theme="light"]) .theme-btn .i-sun { display: block; }
199
+ :root:not([data-theme="light"]) .theme-btn .i-moon { display: none; }
200
+ }
201
+ @media (max-width: 640px) {
202
+ body { font-size: 17px; }
203
+ .hero { padding-top: 28px; }
204
+ section { padding-top: 48px; }
205
+ section > .prose > h2 { font-size: 28px; }
206
+ .grid td.deal, .grid th.sticky { white-space: normal; min-width: 132px; max-width: 150px; font-size: 12.5px; }
207
+ .grid td, .grid th { padding: 7px 6px; }
208
+ .cell .ok, .cell .part, .cell .low { min-width: 42px; }
209
+ }
210
+ </style>
211
+ </head>
212
+ <body>
213
+ <header class="top"><div class="in">
214
+ <a class="brand" href="#">
215
+ WaterfallBench
216
+ </a>
217
+ <nav class="nav">
218
+ <a href="#leaderboard">Leaderboard</a><a href="#task">Task</a><a href="#dataset">Dataset</a><a href="#methodology">Methodology</a><a href="#results">Results</a>
219
+ <a class="gh" href="https://github.com/RITIK-12/WaterfallBench">GitHub ↗</a>
220
+ </nav>
221
+ <button class="theme-btn" id="theme" aria-label="Switch light or dark theme" title="Switch light / dark">
222
+ <svg class="i-moon" viewBox="0 0 24 24" width="16" height="16" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M21 12.8A9 9 0 1 1 11.2 3a7 7 0 0 0 9.8 9.8z"/></svg>
223
+ <svg class="i-sun" viewBox="0 0 24 24" width="16" height="16" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" aria-hidden="true"><circle cx="12" cy="12" r="4"/><path d="M12 2v2M12 20v2M4.9 4.9l1.4 1.4M17.7 17.7l1.4 1.4M2 12h2M20 12h2M4.9 19.1l1.4-1.4M17.7 6.3l1.4-1.4"/></svg>
224
+ </button>
225
+ </div></header>
226
+
227
+ <main>
228
+ <div class="hero">
229
+ <div class="label">Benchmarks / Finance</div>
230
+ <h1>WaterfallBench</h1>
231
+ <div class="meta">
232
+ <span class="label" id="updated">Updated 11/15/2026</span><span class="sep"></span><span class="label">Version 1.0</span><span class="sep"></span><a class="label" href="https://github.com/RITIK-12/WaterfallBench">Code &amp; tasks ↗</a>
233
+ </div>
234
+ </div>
235
+
236
+ <div class="wide" id="leaderboard">
237
+ <div class="panel">
238
+ <div class="panel-head">
239
+ <div class="t"><b>WaterfallBench v1.0</b><span>First-distribution waterfalls</span></div>
240
+ <div class="label" id="asof"></div>
241
+ </div>
242
+ <div class="toolbar">
243
+ <div class="seg" role="group" aria-label="Score">
244
+ <button class="btn" data-metric="fully_resolved" aria-pressed="true">Fully resolved</button>
245
+ <button class="btn" data-metric="nontrivial" aria-pressed="false">Field accuracy</button>
246
+ </div>
247
+ <span class="label">vs cost / task</span>
248
+ <span class="grow"></span>
249
+ <button class="btn" id="tFrontier" aria-pressed="true">Frontier</button>
250
+ <button class="btn" id="tLog" aria-pressed="true">Log scale</button>
251
+ </div>
252
+ <div class="legend label" id="legend"></div>
253
+ <div id="chart"><div class="tip" id="tip"></div></div>
254
+ </div>
255
+
256
+ <div class="panel" style="margin-top:20px">
257
+ <div class="panel-head">
258
+ <div class="t"><b>Leaderboard</b><span>20 tasks · one API call each · reasoning effort high</span></div>
259
+ </div>
260
+ <div class="tbl-wrap"><table id="board"></table></div>
261
+ </div>
262
+ </div>
263
+
264
+ <section id="summary"><div class="prose">
265
+ <h2>Can a model execute an ABS waterfall to the cent?</h2>
266
+ <p>WaterfallBench tests whether a model can read a securitization's legal documents and compute its monthly payments to investors exactly as the trustee did, to the cent. The answer key is what the trustee actually paid, as filed with the SEC.</p>
267
+ <p id="lede">Loading results…</p>
268
+ </div></section>
269
+
270
+ <section id="task"><div class="prose">
271
+ <h2>The task</h2>
272
+ <h3>What a waterfall is</h3>
273
+ <p>An asset-backed security turns thousands of car loans or leases into bonds, sold in classes from senior (A) to junior (B, C). Every month the borrowers' payments are collected and paid out in a fixed order of priority, the <strong>waterfall</strong>:</p>
274
+ <div class="wf" role="img" aria-label="Waterfall: collections pay the servicing fee, then interest, then principal, then the reserve account, then the residual">
275
+ <div class="wf-root"><b>Collections this month</b><span>loan payments, payoffs and recoveries</span></div>
276
+ <ul class="wf-tree">
277
+ <li><div class="wf-node"><i>1</i><b>Servicing fee</b><span>to the servicer that collects the loans</span></div></li>
278
+ <li><div class="wf-node"><i>2</i><b>Interest</b><span>senior classes first</span></div>
279
+ <ul>
280
+ <li><div class="wf-leaf">Class A notes<small>A-1, A-2, A-3, A-4 share pro rata</small></div></li>
281
+ <li><div class="wf-leaf">Class B</div></li>
282
+ <li><div class="wf-leaf">Class C</div></li>
283
+ </ul></li>
284
+ <li><div class="wf-node"><i>3</i><b>Principal</b><span>one class at a time, until each is paid off</span></div>
285
+ <ul><li><div class="wf-leaf wf-seq"><em>A-1</em>→<em>A-2</em>→<em>A-3</em>→<em>A-4</em>→<em>B</em>→<em>C</em></div></li></ul></li>
286
+ <li><div class="wf-node"><i>4</i><b>Reserve account</b><span>topped up to its required level</span></div></li>
287
+ <li><div class="wf-node"><i>5</i><b>Residual</b><span>whatever is left, to the equity holder</span></div></li>
288
+ </ul>
289
+ </div>
290
+ <p class="cap" style="margin:8px 0 18px">Each step is paid only from the cash left after the steps above it. Simplified: each deal's agreements set the exact order, amounts and targets.</p>
291
+ <h3>Why it matters</h3>
292
+ <p>Securitization funds a large share of US car loans and leases: the 20 deals in this benchmark alone issued <strong id="issued">$23.5 billion</strong> of notes. The waterfall decides who gets paid and in what order, so an error moves cash between senior and junior investors, and because each month starts from the last month's balances, it carries forward. Today this is done with deal-specific cash-flow models built by hand from the documents. WaterfallBench asks whether a model can read the documents and get it right on its own.</p>
293
+ <h3>What the model gets</h3>
294
+ <p>A deal's <strong>first distribution date</strong>. Opening balances come from the prospectus, so there is no prior report to copy from. All documents are EDGAR HTML as filed:</p>
295
+ <div class="panel tbl-wrap"><table class="inputs">
296
+ <thead><tr><th>Input</th><th>Filed as</th><th>What the waterfall needs from it</th></tr></thead>
297
+ <tbody>
298
+ <tr><td>Final prospectus</td><td class="form">424(b)(5)</td><td>Note classes, coupons, day counts, the priority of payments</td></tr>
299
+ <tr><td>Closing agreements</td><td class="form">8-K exhibits</td><td>Indenture and servicing agreements: the fee, reserve and overcollateralization definitions</td></tr>
300
+ <tr><td>Servicer report</td><td class="form">10-D Ex. 99.1</td><td>That month's collections, pool balances, rates and dates, with every waterfall result removed</td></tr>
301
+ </tbody>
302
+ </table></div>
303
+ <p style="margin-top:18px">It must return the servicing fee, each class's interest, principal and ending balance, the reserve balance and the residual, in integer cents. Here is the report from a real task as the model sees it:</p>
304
+ </div>
305
+ <div class="prose">
306
+ <div class="panel">
307
+ <div class="panel-head"><div class="t"><b>Servicer report</b><span>waterfall results removed</span></div><div class="label">Nissan Auto Lease 2026-B · Aug 2026</div></div>
308
+ <div class="tbl-wrap"><table class="rep">
309
+ <thead><tr><th>Summary</th><th class="num">Coupon</th><th class="num">Initial balance</th><th class="num">Beginning</th><th class="num">Ending</th></tr></thead>
310
+ <tbody>
311
+ <tr><td>Total Portfolio</td><td></td><td class="num">1,447,527,618.00</td><td class="num">1,447,527,618.00</td><td class="num">1,428,611,202.92</td></tr>
312
+ <tr><td>Class A-1 Notes</td><td class="num">3.949000%</td><td class="num">127,200,000.00</td><td class="num">127,200,000.00</td><td class="rm">REMOVED</td></tr>
313
+ <tr><td>Class A-2a Notes</td><td class="num">4.440000%</td><td class="num">414,000,000.00</td><td class="num">414,000,000.00</td><td class="rm">REMOVED</td></tr>
314
+ <tr><td>Class A-2b Notes</td><td class="num">3.978070%</td><td class="num">80,000,000.00</td><td class="num">80,000,000.00</td><td class="rm">REMOVED</td></tr>
315
+ <tr><td>Class A-3 Notes</td><td class="num">4.690000%</td><td class="num">494,000,000.00</td><td class="num">494,000,000.00</td><td class="rm">REMOVED</td></tr>
316
+ <tr><td>Class A-4 Notes</td><td class="num">4.780000%</td><td class="num">84,800,000.00</td><td class="num">84,800,000.00</td><td class="rm">REMOVED</td></tr>
317
+ <tr><td>Class B Notes</td><td class="num">4.870000%</td><td class="num">49,940,000.00</td><td class="num">49,940,000.00</td><td class="rm">REMOVED</td></tr>
318
+ <tr><td>Class C Notes</td><td class="num">5.080000%</td><td class="num">39,800,000.00</td><td class="num">39,800,000.00</td><td class="rm">REMOVED</td></tr>
319
+ <tr class="grp"><td colspan="5">III. Distributions</td></tr>
320
+ <tr><td colspan="4">Total Collections</td><td class="num">33,734,591.74</td></tr>
321
+ <tr><td colspan="4">2. Reimbursement of Payment Advance</td><td class="num">0.00</td></tr>
322
+ <tr><td colspan="4">4. Servicing Fee</td><td class="rm">REMOVED</td></tr>
323
+ <tr><td colspan="4">5. Class A-1 Notes Monthly Interest</td><td class="rm">REMOVED</td></tr>
324
+ <tr><td colspan="4">&nbsp;&nbsp;&nbsp;Class A-1 Interest Carryover Shortfall</td><td class="num">0.00</td></tr>
325
+ </tbody>
326
+ </table></div>
327
+ </div>
328
+ <p class="cap">Excerpt of <code>fw-nissanlease-2026-b-2026-07</code>. The same packet holds the 6.0 MB prospectus and 9 closing agreements.</p>
329
+ </div></section>
330
+
331
+ <section id="dataset"><div class="prose">
332
+ <h2>Dataset</h2>
333
+ <p id="ds-lede"></p>
334
+ <div class="kv" id="kv"></div>
335
+ <div class="panel tbl-wrap"><table id="families" class="compact"></table></div>
336
+ <p class="cap">Tokens: mean prompt per task, Claude tokenizer, as sent by the harness. No task needed truncation.</p>
337
+ <h3>Release checks</h3>
338
+ <p>Every task passed these checks: the report's classes match the prospectus; each class's opening balance minus principal paid equals its ending balance, to the cent; class payments add up to the report's totals; no graded value is left in the packet; and the true answer grades 100% while an all-zeros answer resolves nothing.</p>
339
+ </div></section>
340
+
341
+ <section id="methodology"><div class="prose">
342
+ <h2>Methodology</h2>
343
+ <h3>Harness</h3>
344
+ <p>One API call per task: a fixed system prompt, the documents and the task. No tools, no code execution, no second turn, so the comparison is the model alone. EDGAR HTML is converted to text with table rows kept as <code>cell | cell</code>, then packed (report, prospectus, agreements) under a 1M-token cap; no task needed truncation. Reasoning effort is <code>high</code> for every model.</p>
345
+ <h3>Evaluation metrics</h3>
346
+ <p>Every task uses the same answer schema, given in the prompt. The model's reply is parsed, its classes are matched to the deal's by name, and every field is compared with the filed value, exactly.</p>
347
+ </div>
348
+ <div class="wide" style="max-width:1200px">
349
+ <div class="io io-eq">
350
+ <div class="panel">
351
+ <div class="panel-head"><div class="t"><b>In the prompt</b><span>answer_schema.json</span></div><div class="label">same for all 20 tasks</div></div>
352
+ <pre class="json">{
353
+ "type": "object", "additionalProperties": false,
354
+ "required": ["servicing_fee_paid", "reserve_ending_balance",
355
+ "released_to_residual", "classes"],
356
+ "properties": {
357
+ <span class="k">"servicing_fee_paid"</span>: {"type": "integer", "description": "US cents"},
358
+ <span class="k">"reserve_ending_balance"</span>: {"type": "integer", "description": "US cents"},
359
+ <span class="k">"released_to_residual"</span>: {"type": "integer", "description": "US cents"},
360
+ <span class="k">"classes"</span>: {"type": "array", "minItems": 1,
361
+ "description": "one entry per class of notes the trust issued",
362
+ "items": {"type": "object",
363
+ "required": ["class", "interest_paid",
364
+ "principal_paid", "ending_balance"],
365
+ "properties": {
366
+ <span class="k">"class"</span>: {"type": "string",
367
+ "description": "the class name as the prospectus gives it, e.g. A-1"},
368
+ <span class="k">"interest_paid"</span>: {"type": "integer", "description": "US cents"},
369
+ <span class="k">"principal_paid"</span>: {"type": "integer", "description": "US cents"},
370
+ <span class="k">"ending_balance"</span>: {"type": "integer", "description": "US cents"}}}}}
371
+ }</pre>
372
+ </div>
373
+ <div class="panel">
374
+ <div class="panel-head"><div class="t"><b>Actual output, graded</b><span>Kimi K3.1 Pro</span></div><div class="label">Nissan Auto Lease 2026-B</div></div>
375
+ <pre class="json">{
376
+ <span class="k">"servicing_fee_paid"</span>: <span class="g-ok">✓</span>,
377
+ <span class="k">"reserve_ending_balance"</span>: <span class="g-ok">✓</span>,
378
+ <span class="k">"released_to_residual"</span>: <span class="g-x">✗</span>,
379
+ <span class="k">"classes"</span>: [
380
+ {<span class="k">"class"</span>: "A-1", <span class="k">"interest_paid"</span>: <span class="g-ok">✓</span>,
381
+ <span class="k">"principal_paid"</span>: <span class="g-ok">✓</span>, <span class="k">"ending_balance"</span>: <span class="g-ok">✓</span>},
382
+ {<span class="k">"class"</span>: "A-2a", <span class="k">"interest_paid"</span>: <span class="g-ok">✓</span>,
383
+ <span class="k">"principal_paid"</span>: <span class="g-ok g-t">✓</span>, <span class="k">"ending_balance"</span>: <span class="g-ok g-t">✓</span>},
384
+ {<span class="k">"class"</span>: "A-2b", <span class="k">"interest_paid"</span>: <span class="g-ok">✓</span>,
385
+ <span class="k">"principal_paid"</span>: <span class="g-ok g-t">✓</span>, <span class="k">"ending_balance"</span>: <span class="g-ok g-t">✓</span>},
386
+ {<span class="k">"class"</span>: "A-3", <span class="k">"interest_paid"</span>: <span class="g-x">✗</span>,
387
+ <span class="k">"principal_paid"</span>: <span class="g-ok g-t">✓</span>, <span class="k">"ending_balance"</span>: <span class="g-ok g-t">✓</span>},
388
+ {<span class="k">"class"</span>: "A-4", <span class="k">"interest_paid"</span>: <span class="g-ok">✓</span>,
389
+ <span class="k">"principal_paid"</span>: <span class="g-ok g-t">✓</span>, <span class="k">"ending_balance"</span>: <span class="g-ok g-t">✓</span>},
390
+ {<span class="k">"class"</span>: "B", <span class="k">"interest_paid"</span>: <span class="g-x">✗</span>,
391
+ <span class="k">"principal_paid"</span>: <span class="g-ok g-t">✓</span>, <span class="k">"ending_balance"</span>: <span class="g-ok g-t">✓</span>},
392
+ {<span class="k">"class"</span>: "C", <span class="k">"interest_paid"</span>: <span class="g-ok">✓</span>,
393
+ <span class="k">"principal_paid"</span>: <span class="g-ok g-t">✓</span>, <span class="k">"ending_balance"</span>: <span class="g-ok g-t">✓</span>}
394
+ ]
395
+ }</pre>
396
+ <p class="cap" style="padding:0 14px 12px">Values hidden: answer keys stay private. <span class="g-ok">✓</span> exact to the cent · <span class="g-x">✗</span> wrong · <span class="g-ok g-t">✓</span> trivial field (zero, or an opening balance left untouched).</p>
397
+ </div>
398
+ </div>
399
+ </div>
400
+ <div class="prose">
401
+ <div class="panel tbl-wrap" style="margin-top:18px"><table class="inputs">
402
+ <thead><tr><th>Metric</th><th>Definition</th><th class="num">This output</th></tr></thead>
403
+ <tbody>
404
+ <tr><td>Fully resolved <span class="label">primary</span></td><td>1 if every graded field is exact to the cent, else 0</td><td class="num v">0</td></tr>
405
+ <tr><td>Field accuracy</td><td>Share of non-trivial fields exactly right. Trivial fields are left out so untouched balances don't inflate the score</td><td class="num v">9 / 12 = 75%</td></tr>
406
+ <tr><td>All fields</td><td>Share of all graded fields exactly right (the task-by-task grid)</td><td class="num v">21 / 24 = 88%</td></tr>
407
+ </tbody>
408
+ </table></div>
409
+ <p style="margin-top:16px">Leaderboard scores average these over the 20 tasks; ± is one standard error. Grading is deterministic, with no LLM judge. Cost uses each provider's list price for the tokens it reported.</p>
410
+ <p id="ungraded"><strong>Not graded.</strong> Auditing the runs turned up three places where the documents cannot determine a value. John Deere's first fee needs a pool balance no document prints. CarMax 2025-2's report departs from its own agreement's overcollateralization target. World Omni's agreement prorates the first fee without a day count. Those fields are left out of scoring; the filed values are never changed.</p>
411
+ <pre class="code">valkyrie run start --benchmark waterfallbench --dataset full-html \
412
+ --agent waterfallbench-direct --model anthropic/claude-haiku-5-5 \
413
+ -k effort high -s ANTHROPIC_API_KEY anthropic-api-key --concurrency 5</pre>
414
+ </div></section>
415
+
416
+ <section id="results"><div class="prose">
417
+ <h2>Results</h2>
418
+ <h3>Field accuracy vs task completion</h3>
419
+ <p id="db-lede"></p>
420
+ </div>
421
+ <div class="prose"><div class="panel" id="dumbbell"></div></div>
422
+ <div class="prose">
423
+ <h3>Task by task</h3>
424
+ <p>Green means every graded field is exact to the cent. Otherwise the cell shows the share of graded fields the model got exactly right.</p>
425
+ </div>
426
+ <div class="wide" style="max-width:1100px"><div class="panel tbl-wrap"><table class="grid" id="grid"></table></div></div>
427
+ </section>
428
+ </main>
429
+
430
+ <footer><div class="in">
431
+ <span>WaterfallBench · SEC EDGAR filings · tasks public, answer keys private</span>
432
+ <span><a href="https://github.com/RITIK-12/WaterfallBench">GitHub</a></span>
433
+ </div></footer>
434
+
435
+ <script>
436
+ const LOGO = {
437
+ "Anthropic": ["M13.827 3.52h3.603L24 20h-3.603l-6.57-16.48zm-7.258 0h3.767L16.906 20h-3.674l-1.343-3.461H5.017l-1.344 3.46H0L6.57 3.522zm4.132 9.959L8.453 7.687 6.205 13.48H10.7z"],
438
+ "OpenAI": ["M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"],
439
+ "Moonshot AI": ["M21.846 0a1.923 1.923 0 110 3.846H20.15a.226.226 0 01-.227-.226V1.923C19.923.861 20.784 0 21.846 0z", "M11.065 11.199l7.257-7.2c.137-.136.06-.41-.116-.41H14.3a.164.164 0 00-.117.051l-7.82 7.756c-.122.12-.302.013-.302-.179V3.82c0-.127-.083-.23-.185-.23H3.186c-.103 0-.186.103-.186.23V19.77c0 .128.083.23.186.23h2.69c.103 0 .186-.102.186-.23v-3.25c0-.069.025-.135.069-.178l2.424-2.406a.158.158 0 01.205-.023l6.484 4.772a7.677 7.677 0 003.453 1.283c.108.012.2-.095.2-.23v-3.06c0-.117-.07-.212-.164-.227a5.028 5.028 0 01-2.027-.807l-5.613-4.064c-.117-.078-.132-.279-.028-.381z"],
440
+ "MiniMax": ["M16.278 2c1.156 0 2.093.927 2.093 2.07v12.501a.74.74 0 00.744.709.74.74 0 00.743-.709V9.099a2.06 2.06 0 012.071-2.049A2.06 2.06 0 0124 9.1v6.561a.649.649 0 01-.652.645.649.649 0 01-.653-.645V9.1a.762.762 0 00-.766-.758.762.762 0 00-.766.758v7.472a2.037 2.037 0 01-2.048 2.026 2.037 2.037 0 01-2.048-2.026v-12.5a.785.785 0 00-.788-.753.785.785 0 00-.789.752l-.001 15.904A2.037 2.037 0 0113.441 22a2.037 2.037 0 01-2.048-2.026V18.04c0-.356.292-.645.652-.645.36 0 .652.289.652.645v1.934c0 .263.142.506.372.638.23.131.514.131.744 0a.734.734 0 00.372-.638V4.07c0-1.143.937-2.07 2.093-2.07zm-5.674 0c1.156 0 2.093.927 2.093 2.07v11.523a.648.648 0 01-.652.645.648.648 0 01-.652-.645V4.07a.785.785 0 00-.789-.78.785.785 0 00-.789.78v14.013a2.06 2.06 0 01-2.07 2.048 2.06 2.06 0 01-2.071-2.048V9.1a.762.762 0 00-.766-.758.762.762 0 00-.766.758v3.8a2.06 2.06 0 01-2.071 2.049A2.06 2.06 0 010 12.9v-1.378c0-.357.292-.646.652-.646.36 0 .653.29.653.646V12.9c0 .418.343.757.766.757s.766-.339.766-.757V9.099a2.06 2.06 0 012.07-2.048 2.06 2.06 0 012.071 2.048v8.984c0 .419.343.758.767.758.423 0 .766-.339.766-.758V4.07c0-1.143.937-2.07 2.093-2.07z"]
441
+ } /* LobeHub icons (MIT), github.com/lobehub/lobe-icons */;
442
+ const S = { metric: "fully_resolved", log: true, frontier: true, sort: { key: "fully_resolved", dir: -1 } };
443
+ const LAB = { Anthropic: "--anthropic", OpenAI: "--openai", "Moonshot AI": "--moonshot", MiniMax: "--minimax" };
444
+ const css = v => getComputedStyle(document.documentElement).getPropertyValue(v).trim();
445
+ const fmtUSD = v => "$" + v.toFixed(2);
446
+ const fmtDur = s => s >= 60 ? `${Math.floor(s / 60)}m ${String(Math.round(s % 60)).padStart(2, "0")}s` : `${s.toFixed(0)}s`;
447
+ const esc = s => String(s).replace(/[&<>"]/g, c => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
448
+ const logo = (lab, s = 16) => `<svg class="logo" viewBox="0 0 24 24" width="${s}" height="${s}" fill="var(${LAB[lab]})" fill-rule="evenodd" role="img" aria-label="${esc(lab)}">${LOGO[lab].map(d => `<path d="${d}"/>`).join("")}</svg>`;
449
+ const and = xs => xs.length < 2 ? xs.join("") : xs.slice(0, -1).join(", ") + " and " + xs[xs.length - 1];
450
+ let D;
451
+
452
+ function el(tag, attrs = {}, kids = []) {
453
+ const e = document.createElementNS("http://www.w3.org/2000/svg", tag);
454
+ for (const [k, v] of Object.entries(attrs)) e.setAttribute(k, v);
455
+ for (const k of [].concat(kids)) e.append(k);
456
+ return e;
457
+ }
458
+
459
+ function frontier(pts) { // models that no cheaper model beats
460
+ const out = []; let best = -Infinity;
461
+ for (const p of [...pts].sort((a, b) => a.x - b.x || b.y - a.y)) if (p.y > best) { out.push(p); best = p.y; }
462
+ return out;
463
+ }
464
+
465
+ function chart() {
466
+ const box = document.getElementById("chart");
467
+ box.querySelector("svg")?.remove();
468
+ const W = box.clientWidth;
469
+ if (W < 240) return; // not laid out yet; the ResizeObserver draws it once it has a width
470
+ const H = Math.max(320, Math.min(480, W * 0.5));
471
+ const m = { l: 56, r: W < 560 ? 18 : 28, t: 22, b: 42 };
472
+ const pts = D.models.map(md => ({ md, x: md.cost, y: md[S.metric], se: S.metric === "fully_resolved" ? md.se : 0 }));
473
+ const ys = pts.map(p => p.y - p.se);
474
+ const y0 = Math.max(0, Math.floor((Math.min(...ys) - 6) / 10) * 10), y1 = 100 + (100 - y0) * 0.08;
475
+ const xs = pts.map(p => p.x);
476
+ const lx0 = Math.log10(Math.min(...xs) / 2.2), lx1 = Math.log10(Math.max(...xs) * 2.2);
477
+ const X = S.log ? v => m.l + (Math.log10(v) - lx0) / (lx1 - lx0) * (W - m.l - m.r)
478
+ : v => m.l + v / (Math.max(...xs) * 1.15) * (W - m.l - m.r);
479
+ const Y = v => m.t + (y1 - v) / (y1 - y0) * (H - m.t - m.b);
480
+ const svg = el("svg", { viewBox: `0 0 ${W} ${H}`, height: H, role: "img", "aria-label": "Score versus cost per task" });
481
+ const midX = (m.l + W - m.r) / 2, midY = (m.t + H - m.b) / 2;
482
+ svg.append(el("rect", { x: m.l, y: m.t, width: midX - m.l, height: midY - m.t, fill: css("--eff") }));
483
+ svg.append(el("rect", { x: midX, y: midY, width: W - m.r - midX, height: H - m.b - midY, fill: css("--ineff") }));
484
+ const mono = (x, y, t, a = {}) => el("text", { x, y, "font-family": css("--mono"), "font-size": 10.5, "letter-spacing": ".06em", fill: css("--muted"), ...a }, t);
485
+ svg.append(mono(midX - 8, m.t + 14, "EFFICIENT", { "text-anchor": "end" }));
486
+ svg.append(mono(midX + 8, H - m.b - 8, "INEFFICIENT"));
487
+ if (W >= 640) svg.append(mono(m.l + 30, m.t + 14, "BEST PERFORMANCE"));
488
+ svg.append(mono(m.l + 14, H - m.b - 8, "LOWEST COST", { transform: `rotate(-90 ${m.l + 14} ${H - m.b - 8})` }));
489
+ for (let v = y0; v <= 100; v += 10) {
490
+ svg.append(el("line", { x1: m.l, x2: W - m.r, y1: Y(v), y2: Y(v), stroke: css("--line"), "stroke-dasharray": v % 20 ? "2 4" : "" }));
491
+ if (v % 20 === 0 || 100 - y0 <= 30) svg.append(mono(m.l - 8, Y(v) + 4, v + "%", { "text-anchor": "end" }));
492
+ }
493
+ const ticks = S.log ? [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 25].filter(v => Math.log10(v) >= lx0 && Math.log10(v) <= lx1)
494
+ : Array.from({ length: 6 }, (_, i) => +(Math.max(...xs) * 1.15 * i / 5).toFixed(1));
495
+ let lastTick = -Infinity;
496
+ for (const v of ticks) {
497
+ if (X(v) - lastTick < 48) continue;
498
+ lastTick = X(v);
499
+ svg.append(el("line", { x1: X(v), x2: X(v), y1: m.t, y2: H - m.b, stroke: css("--line"), "stroke-dasharray": "2 4" }));
500
+ svg.append(mono(X(v), H - m.b + 18, "$" + (v < 1 ? v.toFixed(2) : v.toFixed(v % 1 ? 1 : 0)), { "text-anchor": "middle" }));
501
+ }
502
+ svg.append(el("line", { x1: m.l, x2: W - m.r, y1: H - m.b, y2: H - m.b, stroke: css("--line-2") }));
503
+ svg.append(mono((m.l + W - m.r) / 2, H - 6, "COST PER TASK" + (S.log ? " (LOG)" : ""), { "text-anchor": "middle" }));
504
+ if (S.frontier) {
505
+ const f = frontier(pts);
506
+ if (f.length > 1) svg.append(el("polyline", { points: f.map(p => `${X(p.x)},${Y(p.y)}`).join(" "), fill: "none", stroke: css("--accent"), "stroke-width": 1.6, opacity: .8 }));
507
+ }
508
+ // markers are lab logos; labels placed greedily so they never overlap each other or a marker
509
+ const tip = document.getElementById("tip");
510
+ const placed = pts.map(p => ({ x0: X(p.x) - 14, x1: X(p.x) + 14, y0: Y(p.y) - 14, y1: Y(p.y) + 14 }));
511
+ const hit = (a, b) => a.x0 < b.x1 && b.x0 < a.x1 && a.y0 < b.y1 && b.y0 < a.y1;
512
+ const labelAt = (cx, cy, name) => {
513
+ const w = name.length * 7 + 4, h = 15;
514
+ const opts = [[cx + 18, cy - 20, "start"], [cx + 18, cy + 6, "start"], [cx - 18 - w, cy - 20, "end"], [cx - 18 - w, cy + 6, "end"],
515
+ [cx - w / 2, cy + 18, "middle"], [cx - w / 2, cy - 32, "middle"], [cx + 12 - w, cy + 18, "end"], [cx - 12, cy + 18, "start"],
516
+ [cx + 12 - w, cy - 32, "end"], [cx - 12, cy - 32, "start"]];
517
+ for (const [x, y, anchor] of opts) {
518
+ const r = { x0: x, x1: x + w, y0: y, y1: y + h };
519
+ if (r.x0 < m.l + 2 || r.x1 > W - 2 || r.y0 < 2 || r.y1 > H - m.b) continue;
520
+ if (placed.some(q => hit(r, q))) continue;
521
+ placed.push(r);
522
+ return { x: anchor === "start" ? x : anchor === "end" ? x + w : x + w / 2, y: y + 11, anchor };
523
+ }
524
+ return { x: cx + 18, y: cy - 9, anchor: "start" };
525
+ };
526
+ for (const p of [...pts].sort((a, b) => b.y - a.y || a.x - b.x)) {
527
+ const cx = X(p.x), cy = Y(p.y), c = css(LAB[p.md.lab]);
528
+ if (p.se) svg.append(el("line", { x1: cx, x2: cx, y1: Y(Math.min(100, p.y + p.se)), y2: Y(p.y - p.se), stroke: c, "stroke-width": 1.4, opacity: .5 }));
529
+ const g = el("g", { tabindex: 0, style: "cursor:pointer" });
530
+ const R = W < 560 ? 10 : 13, ls = W < 560 ? 12 : 16; // marker radius and logo size
531
+ g.append(el("circle", { cx, cy, r: R, fill: css("--panel"), stroke: c, "stroke-width": 1.5 }));
532
+ const lg = el("g", { transform: `translate(${cx - ls / 2} ${cy - ls / 2}) scale(${ls / 24})`, fill: c, "fill-rule": "evenodd" });
533
+ for (const d of LOGO[p.md.lab]) lg.append(el("path", { d }));
534
+ g.append(lg);
535
+ const L = labelAt(cx, cy, p.md.name);
536
+ g.append(el("text", { x: L.x, y: L.y, "text-anchor": L.anchor, "font-family": css("--sans"), "font-size": 12.5, fill: css("--ink") }, p.md.name));
537
+ const show = () => {
538
+ tip.innerHTML = `<b>${esc(p.md.name)}</b><div class="row"><span>Fully resolved</span><span>${p.md.fully_resolved.toFixed(1)}% ±${p.md.se.toFixed(1)}</span></div><div class="row"><span>Field accuracy</span><span>${p.md.nontrivial.toFixed(1)}%</span></div><div class="row"><span>Cost / task</span><span>${fmtUSD(p.md.cost)}</span></div><div class="row"><span>Duration / task</span><span>${fmtDur(p.md.seconds)}</span></div>`;
539
+ tip.style.opacity = 1;
540
+ const tw = tip.offsetWidth;
541
+ tip.style.left = Math.min(Math.max(8, cx - tw / 2), W - tw - 8) + "px";
542
+ tip.style.top = (cy > 150 ? cy - tip.offsetHeight - 20 : cy + 22) + "px";
543
+ };
544
+ g.addEventListener("mouseenter", show); g.addEventListener("focus", show);
545
+ g.addEventListener("mouseleave", () => tip.style.opacity = 0); g.addEventListener("blur", () => tip.style.opacity = 0);
546
+ svg.append(g);
547
+ }
548
+ box.prepend(svg);
549
+ }
550
+
551
+ function board() {
552
+ const cols = [
553
+ { k: "rank", t: "#" }, { k: "name", t: `Model (${D.models.length})` },
554
+ { k: "fully_resolved", t: "Fully resolved", n: 1 }, { k: "nontrivial", t: "Field accuracy", n: 1 },
555
+ { k: "cost", t: "Cost / task", n: 1 }, { k: "tokens", t: "Tokens / task", n: 1 }, { k: "seconds", t: "Duration / task", n: 1 },
556
+ ];
557
+ const ranked = [...D.models].sort((a, b) => b.fully_resolved - a.fully_resolved || b.nontrivial - a.nontrivial);
558
+ const rank = new Map(); ranked.forEach((m, i) => rank.set(m.key, i > 0 && m.fully_resolved === ranked[i - 1].fully_resolved ? rank.get(ranked[i - 1].key) : i + 1));
559
+ const rows = [...D.models].sort((a, b) => {
560
+ const k = S.sort.key; const av = k === "rank" ? rank.get(a.key) : a[k], bv = k === "rank" ? rank.get(b.key) : b[k];
561
+ return (typeof av === "string" ? av.localeCompare(bv) : av - bv) * S.sort.dir;
562
+ });
563
+ const th = cols.map(c => `<th class="${c.n ? "num " : ""}${c.k !== "rank" ? "sort" : ""}" data-k="${c.k}">${c.t}${S.sort.key === c.k ? `<span class="arr">${S.sort.dir < 0 ? "▼" : "▲"}</span>` : ""}</th>`).join("");
564
+ const tr = rows.map(m => {
565
+ const r = rank.get(m.key);
566
+ return `<tr class="${r <= 3 ? "top" + r : ""}"><td class="rank">${r}</td>
567
+ <td><span class="model">${logo(m.lab)}${esc(m.name)}</span></td>
568
+ <td class="num"><span class="minibar"><b style="width:${m.fully_resolved}%"></b></span><span class="v">${m.fully_resolved.toFixed(1)}<span class="u">%</span><small>±${m.se.toFixed(1)}</small></span></td>
569
+ <td class="num v">${m.nontrivial.toFixed(1)}<span class="u">%</span></td>
570
+ <td class="num v"><span class="u">$</span>${m.cost.toFixed(2)}</td>
571
+ <td class="num v">${(m.tokens / 1000).toFixed(1)}<span class="u">k</span></td>
572
+ <td class="num v">${fmtDur(m.seconds)}</td></tr>`;
573
+ }).join("");
574
+ const t = document.getElementById("board");
575
+ t.innerHTML = `<thead><tr>${th}</tr></thead><tbody>${tr}</tbody>`;
576
+ t.querySelectorAll("th.sort").forEach(h => h.onclick = () => {
577
+ const k = h.dataset.k; S.sort = { key: k, dir: S.sort.key === k ? -S.sort.dir : (["name", "cost", "tokens", "seconds"].includes(k) ? 1 : -1) }; board();
578
+ });
579
+ }
580
+
581
+ function dumbbell() {
582
+ const box = document.getElementById("dumbbell");
583
+ const W = box.clientWidth, rowH = 44, m = { l: W < 520 ? 128 : 170, r: 24, t: 36, b: 30 };
584
+ if (W < 240) return;
585
+ const ms = [...D.models].sort((a, b) => b.fully_resolved - a.fully_resolved || b.nontrivial - a.nontrivial);
586
+ const lo = Math.max(0, Math.floor((Math.min(...ms.map(x => x.fully_resolved)) - 5) / 10) * 10);
587
+ const H = m.t + m.b + rowH * ms.length;
588
+ const X = v => m.l + (v - lo) / (100 - lo) * (W - m.l - m.r);
589
+ const svg = el("svg", { viewBox: `0 0 ${W} ${H}`, height: H, role: "img", "aria-label": "Fully resolved versus field accuracy" });
590
+ const mono = (x, y, t, a = {}) => el("text", { x, y, "font-family": css("--mono"), "font-size": 10.5, fill: css("--muted"), ...a }, t);
591
+ const step = (100 - lo) > 60 ? 20 : 10;
592
+ for (let v = lo; v <= 100; v += step) {
593
+ svg.append(el("line", { x1: X(v), x2: X(v), y1: m.t - 6, y2: H - m.b, stroke: css("--line"), "stroke-dasharray": "2 4" }));
594
+ svg.append(mono(X(v), H - m.b + 18, v + "%", { "text-anchor": "middle" }));
595
+ }
596
+ svg.append(el("circle", { cx: m.l + 5, cy: 14, r: 5, fill: css("--accent") })); svg.append(mono(m.l + 16, 18, "FIELD ACCURACY"));
597
+ svg.append(el("rect", { x: m.l + 136, y: 9, width: 10, height: 10, fill: "none", stroke: css("--ink"), "stroke-width": 1.6 })); svg.append(mono(m.l + 152, 18, "TASKS FULLY RESOLVED"));
598
+ ms.forEach((md, i) => {
599
+ const y = m.t + rowH * i + rowH / 2;
600
+ svg.append(el("text", { x: m.l - 14, y: y + 4, "text-anchor": "end", "font-family": css("--sans"), "font-size": 13, fill: css("--ink") }, md.name));
601
+ svg.append(el("line", { x1: X(md.fully_resolved), x2: X(md.nontrivial), y1: y, y2: y, stroke: css("--line-2"), "stroke-width": 3 }));
602
+ svg.append(el("circle", { cx: X(md.nontrivial), cy: y, r: 6, fill: css("--accent") }));
603
+ svg.append(el("rect", { x: X(md.fully_resolved) - 6, y: y - 6, width: 12, height: 12, fill: css("--panel"), stroke: css("--ink"), "stroke-width": 1.8 }));
604
+ svg.append(mono(X(md.fully_resolved) - 10, y - 10, md.fully_resolved.toFixed(0) + "%", { "text-anchor": "end", fill: css("--ink-2") }));
605
+ });
606
+ box.innerHTML = ""; box.append(svg);
607
+ }
608
+
609
+ const month = p => new Date(+p.slice(0, 4), +p.slice(4, 6) - 1, 1).toLocaleString("en-US", { month: "short", year: "numeric" });
610
+
611
+ function grid() {
612
+ const ms = [...D.models].sort((a, b) => b.fully_resolved - a.fully_resolved || b.nontrivial - a.nontrivial);
613
+ const head = `<thead><tr><th class="sticky">Deal · first distribution</th>${ms.map(m => `<th class="mh">${logo(m.lab, 15)}<div style="margin-top:4px">${esc(m.name)}</div></th>`).join("")}</tr></thead>`;
614
+ const body = D.tasks.map(t => `<tr><td class="deal sticky">${esc(t.deal)}<small>${month(t.period)} · ${t.classes} classes · ${t.graded} fields</small></td>
615
+ ${ms.map(m => { const r = m.results[t.id]; if (!r) return `<td class="cell">–</td>`;
616
+ return `<td class="cell"><span class="${r.ok ? "ok" : r.li >= 85 ? "part" : "low"}" title="${r.ok ? "every graded field exact" : `${r.li.toFixed(0)}% of graded fields exact`}">${r.ok ? "✓" : r.li.toFixed(0) + "%"}</span></td>`; }).join("")}</tr>`).join("");
617
+ document.getElementById("grid").innerHTML = head + `<tbody>${body}</tbody>`;
618
+ }
619
+
620
+ function prose() {
621
+ const ms = [...D.models].sort((a, b) => b.fully_resolved - a.fully_resolved || b.nontrivial - a.nontrivial || a.cost - b.cost);
622
+ const top = ms[0].fully_resolved, leaders = ms.filter(m => m.fully_resolved === top), rest = ms.filter(m => m.fully_resolved < top);
623
+ const d = new Date(D.updated + "T12:00:00");
624
+ const long = d.toLocaleDateString("en-US", { month: "long", day: "numeric", year: "numeric" });
625
+ document.getElementById("asof").textContent = "WaterfallBench v1.0 · " + d.toLocaleDateString("en-US", { month: "short", day: "numeric", year: "numeric" });
626
+ document.getElementById("lede").innerHTML = `As of ${long}, ${and(leaders.map(m => `<strong>${esc(m.name)}</strong>`))} ${leaders.length > 1 ? "tie for first" : "ranks first"} with ${top.toFixed(0)}% of deals fully resolved${rest.length ? `, followed by ${and(rest.slice(0, 2).map(m => `${esc(m.name)} (${m.fully_resolved.toFixed(0)}%)`))}` : ""}. Every model gets most fields right; resolving a whole deal to the cent is what separates them.`;
627
+ const nt = ms.map(m => m.nontrivial), fr = ms.map(m => m.fully_resolved);
628
+ document.getElementById("db-lede").textContent = `Field accuracy ranges from ${Math.min(...nt).toFixed(0)}% to ${Math.max(...nt).toFixed(0)}%, while the share of tasks fully resolved ranges from ${Math.min(...fr).toFixed(0)}% to ${Math.max(...fr).toFixed(0)}%. A deal resolves only if every graded field is exact, and one wrong definition usually carries into every line below it.`;
629
+ // dataset
630
+ const T = D.tasks, nf = T.reduce((a, t) => a + t.graded, 0), fams = [...new Set(T.map(t => t.family))];
631
+ const periods = T.map(t => t.period).sort();
632
+ document.getElementById("ds-lede").textContent = `${T.length} tasks from ${fams.length} auto-loan and lease issuers. Each is a trust's first Form 10-D on SEC EDGAR, with first distributions from ${month(periods[0])} to ${month(periods[periods.length - 1])}. Documents were retrieved with Firecrawl and are used as filed; nothing is generated or imputed.`;
633
+ const docs = T.map(t => t.docs + 1), mb = T.map(t => t.html_mb), tok = T.map(t => t.prompt_tokens).filter(Boolean);
634
+ document.getElementById("kv").innerHTML = [[T.length, "tasks"], [nf, "graded fields"], [`${Math.min(...docs)}–${Math.max(...docs)}`, "documents / task"],
635
+ [`${Math.min(...mb).toFixed(0)}–${Math.max(...mb).toFixed(0)} MB`, "HTML / task"], [`${Math.round(Math.min(...tok) / 1000)}–${Math.round(Math.max(...tok) / 1000)}k`, "prompt tokens / task"]]
636
+ .map(([v, k]) => `<div><b>${v}</b><span class="label">${k}</span></div>`).join("");
637
+ const fr2 = fams.map(f => { const ts = T.filter(t => t.family === f), ps = ts.map(t => t.period).sort(), cl = ts.map(t => t.classes), tk = ts.map(t => t.prompt_tokens).filter(Boolean);
638
+ return `<tr><td>${esc(ts[0].family_name)}</td><td class="num v">${ts.length}</td><td class="num v">${month(ps[0])} – ${month(ps[ps.length - 1])}</td><td class="num v">${Math.min(...cl) === Math.max(...cl) ? cl[0] : Math.min(...cl) + "–" + Math.max(...cl)}</td><td class="num v">${Math.round(tk.reduce((a, b) => a + b, 0) / tk.length / 1000)}k</td><td class="num v">${ts.reduce((a, t) => a + t.graded, 0)}</td></tr>`; }).join("");
639
+ document.getElementById("families").innerHTML = `<thead><tr><th>Issuer</th><th class="num">Deals</th><th class="num">First distributions</th><th class="num">Classes</th><th class="num">Tokens</th><th class="num">Fields</th></tr></thead><tbody>${fr2}</tbody>`;
640
+ const nu = T.reduce((a, t) => a + t.ungraded, 0), tu = T.filter(t => t.ungraded).length;
641
+ document.getElementById("ungraded").insertAdjacentHTML("beforeend", ` In all, ${nu} of ${nf + nu} fields across ${tu} tasks are not graded.`);
642
+ document.getElementById("legend").innerHTML = "Lab: " + [...new Set(ms.map(m => m.lab))].map(l => `<span>${logo(l, 13)}${esc(l)}</span>`).join("")
643
+ + `<span class="fr-key"><i></i>Frontier: best score at each cost</span><span><i class="se-key"></i>±1 SE</span>`;
644
+ }
645
+
646
+ function renderAll() { chart(); board(); dumbbell(); grid(); }
647
+
648
+ document.querySelectorAll("[data-metric]").forEach(b => b.onclick = () => {
649
+ S.metric = b.dataset.metric; document.querySelectorAll("[data-metric]").forEach(x => x.setAttribute("aria-pressed", x === b)); chart();
650
+ });
651
+ document.getElementById("tLog").onclick = e => { S.log = !S.log; e.currentTarget.setAttribute("aria-pressed", S.log); chart(); };
652
+ document.getElementById("tFrontier").onclick = e => { S.frontier = !S.frontier; e.currentTarget.setAttribute("aria-pressed", S.frontier); chart(); };
653
+ document.getElementById("theme").onclick = () => {
654
+ const r = document.documentElement, dark = r.dataset.theme ? r.dataset.theme === "dark" : matchMedia("(prefers-color-scheme: dark)").matches;
655
+ r.dataset.theme = dark ? "light" : "dark"; try { localStorage.setItem("wfb-theme", r.dataset.theme); } catch (e) {}
656
+ renderAll();
657
+ };
658
+ try { const t = localStorage.getItem("wfb-theme"); if (t) document.documentElement.dataset.theme = t; } catch (e) {}
659
+ let rz; const redraw = () => { clearTimeout(rz); rz = setTimeout(() => { if (D) { chart(); dumbbell(); } }, 80); };
660
+ if ("ResizeObserver" in window) { const ro = new ResizeObserver(redraw); ro.observe(document.getElementById("chart")); ro.observe(document.getElementById("dumbbell")); }
661
+ else addEventListener("resize", redraw);
662
+ matchMedia("(prefers-color-scheme: dark)").addEventListener?.("change", renderAll);
663
+
664
+ fetch("data.json").then(r => r.json()).then(d => { D = d; prose(); renderAll(); })
665
+ .catch(() => { document.getElementById("lede").textContent = "Could not load results (data.json)."; });
666
+ </script>
667
+ </body>
668
  </html>