Spaces:
Running
Running
File size: 65,128 Bytes
55b13fe c5255b5 1ec2449 c5255b5 1ec2449 c5255b5 1ec2449 c5255b5 535ebf1 c5255b5 535ebf1 c5255b5 535ebf1 c5255b5 535ebf1 c5255b5 1ec2449 c5255b5 1ec2449 c5255b5 535ebf1 c5255b5 535ebf1 c5255b5 535ebf1 c5255b5 535ebf1 c5255b5 55b13fe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 | <!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>WaterfallBench</title>
<meta name="description" content="WaterfallBench: can a model execute an asset-backed securities waterfall to the cent from the deal documents? Leaderboard and methodology.">
<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'%3E%3Crect width='32' height='32' rx='7' fill='%233f5f3a'/%3E%3Crect x='7' y='7' width='18' height='4' rx='1' fill='%23fff'/%3E%3Crect x='10' y='14' width='12' height='4' rx='1' fill='%23fff' opacity='.8'/%3E%3Crect x='13' y='21' width='6' height='4' rx='1' fill='%23fff' opacity='.6'/%3E%3C/svg%3E">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500&family=Inter:wght@400;500;600&family=Newsreader:opsz,wght@6..72,400;6..72,500;6..72,600&display=swap" rel="stylesheet">
<style>
:root {
--bg: #f7f6f1; --panel: #ffffff; --panel-2: #f1efe8; --ink: #1c1c19; --ink-2: #3a3935; --muted: #6d6b63;
--line: #e3e0d6; --line-2: #d4d0c3; --accent: #3f5f3a; --accent-ink: #2f4a2b; --accent-soft: #e6ece0;
--good: #3f7a3a; --good-soft: #dfeadb; --warn: #a8741a; --warn-soft: #f4ead6; --bad: #a8473a; --bad-soft: #f3e1dd;
--eff: rgba(63, 95, 58, .07); --ineff: rgba(168, 71, 58, .07); --rm: #ece4cf;
--lab: #1c1c19;
--serif: "Newsreader", Georgia, "Times New Roman", serif; --sans: "Inter", system-ui, -apple-system, "Segoe UI", sans-serif;
--mono: "IBM Plex Mono", ui-monospace, "SFMono-Regular", Menlo, monospace;
color-scheme: light;
}
@media (prefers-color-scheme: dark) {
:root:not([data-theme="light"]) {
--bg: #141412; --panel: #1b1b18; --panel-2: #22221e; --ink: #ecebe5; --ink-2: #d2d0c8; --muted: #9a988e;
--line: #2e2d28; --line-2: #3b3a34; --accent: #9fbf8f; --accent-ink: #b9d4aa; --accent-soft: #24301f;
--good: #8fc382; --good-soft: #213321; --warn: #d9a650; --warn-soft: #362b17; --bad: #e08b7c; --bad-soft: #3a211d;
--eff: rgba(143, 195, 130, .07); --ineff: rgba(224, 139, 124, .07); --rm: #3a3220;
--lab: #ecebe5; color-scheme: dark;
}
}
:root[data-theme="dark"] {
--bg: #141412; --panel: #1b1b18; --panel-2: #22221e; --ink: #ecebe5; --ink-2: #d2d0c8; --muted: #9a988e;
--line: #2e2d28; --line-2: #3b3a34; --accent: #9fbf8f; --accent-ink: #b9d4aa; --accent-soft: #24301f;
--good: #8fc382; --good-soft: #213321; --warn: #d9a650; --warn-soft: #362b17; --bad: #e08b7c; --bad-soft: #3a211d;
--eff: rgba(143, 195, 130, .07); --ineff: rgba(224, 139, 124, .07); --rm: #3a3220;
--lab: #ecebe5; color-scheme: dark;
}
* { box-sizing: border-box; }
html { -webkit-text-size-adjust: 100%; scroll-behavior: smooth; }
body { margin: 0; background: var(--bg); color: var(--ink); font-family: var(--serif); font-size: 18px; line-height: 1.6; }
a { color: inherit; text-decoration-color: var(--line-2); text-underline-offset: 3px; }
a:hover { text-decoration-color: var(--accent); }
code, pre { font-family: var(--mono); font-size: .82em; }
.label { font-family: var(--mono); font-size: 12px; letter-spacing: .06em; text-transform: uppercase; color: var(--muted); }
.logo { display: inline-block; flex: none; vertical-align: -3px; }
/* top bar */
.top { position: sticky; top: 0; z-index: 20; background: color-mix(in srgb, var(--bg) 92%, transparent); backdrop-filter: blur(8px); border-bottom: 1px solid var(--line); }
.top .in { max-width: 1200px; margin: 0 auto; padding: 12px 16px; display: flex; align-items: center; gap: 24px; }
.brand { display: flex; align-items: center; gap: 10px; font-family: var(--sans); font-weight: 600; font-size: 16px; text-decoration: none; letter-spacing: -.01em; }
.nav { display: flex; gap: 20px; margin-left: auto; font-family: var(--sans); font-size: 14px; }
.nav a { text-decoration: none; color: var(--ink-2); }
.nav a:hover { color: var(--accent); }
@media (max-width: 720px) { .nav a:not(.gh) { display: none; } }
/* hero */
.hero { max-width: 1200px; margin: 0 auto; padding: 44px 16px 24px; }
h1 { font-family: var(--serif); font-weight: 500; font-size: clamp(40px, 7vw, 56px); line-height: 1.05; letter-spacing: -.02em; margin: 10px 0 14px; }
.meta { display: flex; flex-wrap: wrap; align-items: center; gap: 14px; }
.meta .sep { width: 1px; height: 16px; background: var(--line-2); }
.meta a.label { text-decoration: none; }
.meta a.label:hover { color: var(--accent); }
.btn { display: inline-flex; align-items: center; gap: 8px; font-family: var(--mono); font-size: 12px; letter-spacing: .06em; text-transform: uppercase; padding: 6px 10px; border: 1px solid var(--line-2); background: var(--panel); color: var(--ink); text-decoration: none; cursor: pointer; }
.btn:hover { border-color: var(--accent); color: var(--accent-ink); }
.btn[aria-pressed="true"] { background: var(--accent-soft); border-color: var(--accent); color: var(--accent-ink); }
.cta { margin-top: 16px; }
.hf-badge { display: inline-flex; align-items: center; gap: 9px; padding: 7px 13px 7px 10px; border: 1px solid var(--line-2); background: var(--panel); color: var(--ink); font-family: var(--sans); font-size: 14px; text-decoration: none; }
.hf-badge:hover { border-color: #FFD21E; background: color-mix(in srgb, #FFD21E 10%, var(--panel)); }
.hf-badge b { font-weight: 600; }
/* panels */
.wide { max-width: 1200px; margin: 0 auto; padding: 0 16px; }
.panel { background: var(--panel); border: 1px solid var(--line); }
.panel-head { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 8px 16px; padding: 12px 14px; border-bottom: 1px solid var(--line); }
.panel-head .t { display: flex; align-items: baseline; gap: 12px; flex-wrap: wrap; }
.panel-head .t b { font-family: var(--serif); font-weight: 600; font-size: 17px; }
.panel-head .t span { font-family: var(--sans); font-size: 14px; color: var(--accent); }
.toolbar { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; padding: 10px 14px; border-bottom: 1px solid var(--line); background: var(--panel-2); }
.toolbar .grow { flex: 1; }
.seg { display: inline-flex; border: 1px solid var(--line-2); }
.seg .btn { border: 0; }
.seg .btn + .btn { border-left: 1px solid var(--line-2); }
.legend { display: flex; flex-wrap: wrap; gap: 16px; align-items: center; padding: 8px 14px; border-bottom: 1px solid var(--line); }
.legend span { display: inline-flex; align-items: center; gap: 6px; }
.legend .fr-key { margin-left: auto; }
.legend .fr-key i { width: 18px; height: 2px; background: var(--accent); display: inline-block; }
.legend .se-key { width: 2px; height: 12px; background: var(--muted); display: inline-block; opacity: .6; }
#chart { position: relative; }
#chart svg { display: block; width: 100%; }
.tip { position: absolute; pointer-events: none; background: var(--panel); border: 1px solid var(--line-2); padding: 10px 12px; font-family: var(--sans); font-size: 13px; line-height: 1.45; box-shadow: 0 6px 24px rgba(0,0,0,.08); min-width: 190px; opacity: 0; transition: opacity .12s; z-index: 5; }
.tip b { font-weight: 600; }
.tip .row { display: flex; justify-content: space-between; gap: 16px; color: var(--muted); }
.tip .row span:last-child { color: var(--ink); font-family: var(--mono); font-size: 12px; }
/* tables */
.tbl-wrap { overflow-x: auto; }
table { width: 100%; border-collapse: collapse; font-family: var(--sans); font-size: 14.5px; }
th { text-align: left; font-weight: 500; font-size: 13px; color: var(--ink-2); padding: 10px 12px; border-bottom: 1px solid var(--line-2); background: var(--panel-2); white-space: nowrap; }
th.num, td.num { text-align: right; }
th.sort { cursor: pointer; user-select: none; }
th.sort:hover { color: var(--accent); }
th .arr { font-size: 10px; margin-left: 4px; color: var(--muted); }
td { padding: 11px 12px; border-bottom: 1px solid var(--line); white-space: nowrap; }
tbody tr:hover td { background: var(--panel-2); }
tr.top1 td { background: color-mix(in srgb, var(--accent-soft) 70%, var(--panel)); }
tr.top2 td { background: color-mix(in srgb, var(--accent-soft) 40%, var(--panel)); }
tr.top3 td { background: color-mix(in srgb, var(--accent-soft) 20%, var(--panel)); }
td.rank { font-family: var(--mono); font-size: 12.5px; color: var(--muted); width: 40px; }
.model { display: inline-flex; align-items: center; gap: 10px; font-weight: 500; }
.v { font-family: var(--mono); font-size: 13.5px; }
.v small { color: var(--muted); font-size: 11.5px; margin-left: 4px; }
.v .u { color: var(--muted); margin: 0 1px; }
.minibar { display: inline-block; width: 64px; height: 6px; background: var(--line); vertical-align: middle; margin-right: 10px; position: relative; }
.minibar b { position: absolute; inset: 0 auto 0 0; background: var(--accent); }
/* prose */
.prose { max-width: 820px; margin: 0 auto; padding: 0 16px; }
section { padding: 64px 0 0; }
section > .prose > h2 { font-family: var(--serif); font-weight: 500; font-size: 34px; letter-spacing: -.015em; line-height: 1.15; margin: 0 0 16px; }
.prose h3 { font-family: var(--serif); font-weight: 600; font-size: 21px; margin: 36px 0 6px; }
.prose p { margin: 0 0 16px; color: var(--ink-2); }
.prose strong { color: var(--ink); font-weight: 600; }
.cap { font-family: var(--sans); font-size: 13px; color: var(--muted); margin: 8px 0 0; }
/* task: inputs and output */
.inputs td { white-space: normal; vertical-align: top; font-size: 14px; }
.inputs td:first-child { font-weight: 500; white-space: nowrap; }
.inputs td.form { font-family: var(--mono); font-size: 12.5px; color: var(--muted); white-space: nowrap; }
.io { display: grid; grid-template-columns: 1.35fr 1fr; gap: 16px; margin-top: 18px; }
.io-eq { grid-template-columns: 1fr 1fr; }
.io-eq pre.json { font-size: 11.5px; }
@media (max-width: 960px) { .io, .io-eq { grid-template-columns: 1fr; } }
.inputs td.num { white-space: nowrap; }
.io .panel-head .t b { font-size: 15px; }
.rep { font-family: var(--mono); font-size: 12px; }
.rep th, .rep td { padding: 6px 10px; font-size: 12px; }
.rep th { font-family: var(--mono); font-weight: 500; font-size: 11px; color: var(--muted); }
.rep td.num { text-align: right; }
.rep .grp td { background: var(--panel-2); font-weight: 500; color: var(--ink-2); }
.rep td.rm { background: repeating-linear-gradient(-45deg, var(--rm) 0 6px, transparent 6px 12px); color: var(--warn); text-align: center; font-size: 10.5px; letter-spacing: .05em; }
pre.json { margin: 0; padding: 12px 14px; font-size: 12px; line-height: 1.55; color: var(--ink-2); overflow-x: auto; }
pre.json .k { color: var(--accent-ink); }
pre.json .q { background: var(--rm); color: var(--warn); padding: 0 4px; }
.wf { margin: 8px 0 0; font-family: var(--sans); }
.wf-root { display: inline-flex; flex-direction: column; padding: 10px 14px; background: var(--accent); color: var(--bg); }
.wf-root b { font-weight: 600; font-size: 15px; }
.wf-root span { font-size: 13px; opacity: .85; }
.wf-tree, .wf-tree ul { list-style: none; margin: 0; padding: 0; }
.wf-tree { margin-left: 22px; }
.wf-tree ul { margin-left: 30px; }
.wf-tree li { position: relative; padding: 7px 0 0 30px; }
.wf-tree li::before { content: ""; position: absolute; left: 0; top: 0; bottom: 0; border-left: 2px solid var(--line-2); }
.wf-tree li:last-child::before { bottom: auto; height: 27px; }
.wf-tree li::after { content: ""; position: absolute; left: 0; top: 27px; width: 26px; border-top: 2px solid var(--line-2); }
.wf-tree > li::before, .wf-tree > li::after { border-color: var(--accent); }
.wf-node, .wf-leaf { display: inline-flex; align-items: baseline; flex-wrap: wrap; gap: 4px 10px; padding: 8px 12px; background: var(--panel); border: 1px solid var(--line); font-size: 14.5px; }
.wf-node { border-left: 3px solid var(--accent); }
.wf-node i { font-style: normal; font-family: var(--mono); font-size: 12px; color: var(--muted); }
.wf-node b { font-weight: 600; }
.wf-node span, .wf-leaf small { color: var(--muted); font-size: 13px; }
.wf-leaf { background: var(--panel-2); font-size: 13.5px; padding: 6px 10px; }
.wf-tree ul li::after { top: 23px; }
.wf-tree ul li:last-child::before { height: 23px; }
.wf-seq { gap: 6px; color: var(--muted); }
.wf-seq em { font-style: normal; font-family: var(--mono); font-size: 12px; color: var(--ink); background: var(--panel); border: 1px solid var(--line); padding: 1px 6px; }
@media (max-width: 640px) { .wf-tree ul { margin-left: 14px; } .wf-tree li { padding-left: 22px; } .wf-tree li::after { width: 18px; } }
.g-ok { color: var(--good); font-weight: 600; }
.g-x { color: var(--bad); font-weight: 600; background: var(--bad-soft); padding: 0 3px; }
.g-t { color: var(--muted); font-weight: 400; }
/* dataset */
.kv { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); border: 1px solid var(--line); background: var(--panel); margin: 4px 0 20px; }
.kv > div { padding: 12px 14px; border-right: 1px solid var(--line); border-bottom: 1px solid var(--line); }
.compact th, .compact td { padding: 9px 10px; font-size: 13.5px; }
.kv b { display: block; font-family: var(--serif); font-weight: 500; font-size: 26px; line-height: 1.15; color: var(--ink); }
/* task grid */
.grid td, .grid th { padding: 8px 10px; }
.grid th.mh { text-align: center; font-size: 12px; white-space: normal; min-width: 76px; }
.grid td.cell { text-align: center; font-family: var(--mono); font-size: 12.5px; }
.cell .ok, .cell .part, .cell .low { display: inline-block; min-width: 50px; padding: 3px 0; }
.cell .ok { background: var(--good-soft); color: var(--good); }
.cell .part { background: var(--warn-soft); color: var(--warn); }
.cell .low { background: var(--bad-soft); color: var(--bad); }
.grid td.deal { font-size: 13.5px; }
.grid td.deal small { display: block; color: var(--muted); font-family: var(--mono); font-size: 11px; }
.grid .sticky { position: sticky; left: 0; background: var(--panel); z-index: 1; }
.grid tr:hover td.sticky { background: var(--panel-2); }
pre.code { background: var(--panel); border: 1px solid var(--line); padding: 14px 16px; overflow-x: auto; line-height: 1.55; color: var(--ink-2); margin: 0 0 16px; }
.plain { margin: 0 0 16px; padding-left: 20px; color: var(--ink-2); }
.plain li { margin-bottom: 6px; }
footer { margin-top: 72px; border-top: 1px solid var(--line); }
footer .in { max-width: 1200px; margin: 0 auto; padding: 20px 16px 36px; display: flex; flex-wrap: wrap; gap: 12px 24px; justify-content: space-between; font-family: var(--sans); font-size: 13.5px; color: var(--muted); }
.theme-btn { display: inline-grid; place-items: center; width: 32px; height: 32px; flex: none; background: var(--panel); border: 1px solid var(--line-2); color: var(--ink-2); cursor: pointer; padding: 0; }
.theme-btn:hover { border-color: var(--accent); color: var(--accent-ink); }
.theme-btn .i-sun { display: none; }
:root[data-theme="dark"] .theme-btn .i-sun { display: block; }
:root[data-theme="dark"] .theme-btn .i-moon { display: none; }
@media (prefers-color-scheme: dark) {
:root:not([data-theme="light"]) .theme-btn .i-sun { display: block; }
:root:not([data-theme="light"]) .theme-btn .i-moon { display: none; }
}
@media (max-width: 640px) {
body { font-size: 17px; }
.hero { padding-top: 28px; }
section { padding-top: 48px; }
section > .prose > h2 { font-size: 28px; }
.grid td.deal, .grid th.sticky { white-space: normal; min-width: 132px; max-width: 150px; font-size: 12.5px; }
.grid td, .grid th { padding: 7px 6px; }
.cell .ok, .cell .part, .cell .low { min-width: 42px; }
}
</style>
</head>
<body>
<header class="top"><div class="in">
<a class="brand" href="#">
WaterfallBench
</a>
<nav class="nav">
<a href="#leaderboard">Leaderboard</a><a href="#task">Task</a><a href="#dataset">Dataset</a><a href="#methodology">Methodology</a><a href="#results">Results</a>
<a class="gh" href="https://github.com/RITIK-12/WaterfallBench">GitHub ↗</a>
</nav>
<button class="theme-btn" id="theme" aria-label="Switch light or dark theme" title="Switch light / dark">
<svg class="i-moon" viewBox="0 0 24 24" width="16" height="16" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M21 12.8A9 9 0 1 1 11.2 3a7 7 0 0 0 9.8 9.8z"/></svg>
<svg class="i-sun" viewBox="0 0 24 24" width="16" height="16" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" aria-hidden="true"><circle cx="12" cy="12" r="4"/><path d="M12 2v2M12 20v2M4.9 4.9l1.4 1.4M17.7 17.7l1.4 1.4M2 12h2M20 12h2M4.9 19.1l1.4-1.4M17.7 6.3l1.4-1.4"/></svg>
</button>
</div></header>
<main>
<div class="hero">
<div class="label">Benchmarks / Finance</div>
<h1>WaterfallBench</h1>
<div class="meta">
<span class="label" id="updated">Updated 11/15/2026</span><span class="sep"></span><span class="label">Version 1.0</span><span class="sep"></span><a class="label" href="https://github.com/RITIK-12/WaterfallBench">Code & tasks ↗</a>
</div>
<div class="cta"><a class="hf-badge" href="https://huggingface.co/datasets/ritik12/WaterfallBench"><svg class="hf-logo" viewBox="0 0 24 24" width="20" height="20" aria-hidden="true"><path d="M2.25 11.535c0-3.407 1.847-6.554 4.844-8.258a9.822 9.822 0 019.687 0c2.997 1.704 4.844 4.851 4.844 8.258 0 5.266-4.337 9.535-9.687 9.535S2.25 16.8 2.25 11.535z" fill="#FF9D0B"></path><path d="M11.938 20.086c4.797 0 8.687-3.829 8.687-8.551 0-4.722-3.89-8.55-8.687-8.55-4.798 0-8.688 3.828-8.688 8.55 0 4.722 3.89 8.55 8.688 8.55z" fill="#FFD21E"></path><path d="M11.875 15.113c2.457 0 3.25-2.156 3.25-3.263 0-.576-.393-.394-1.023-.089-.582.283-1.365.675-2.224.675-1.798 0-3.25-1.693-3.25-.586 0 1.107.79 3.263 3.25 3.263h-.003z" fill="#FF323D"></path><path d="M14.76 9.21c.32.108.445.753.767.585.447-.233.707-.708.659-1.204a1.235 1.235 0 00-.879-1.059 1.262 1.262 0 00-1.33.394c-.322.384-.377.92-.14 1.36.153.283.638-.177.925-.079l-.002.003zm-5.887 0c-.32.108-.448.753-.768.585a1.226 1.226 0 01-.658-1.204c.048-.495.395-.913.878-1.059a1.262 1.262 0 011.33.394c.322.384.377.92.14 1.36-.152.283-.64-.177-.925-.079l.003.003zm1.12 5.34a2.166 2.166 0 011.325-1.106c.07-.02.144.06.219.171l.192.306c.069.1.139.175.209.175.074 0 .15-.074.223-.172l.205-.302c.08-.11.157-.188.234-.165.537.168.986.536 1.25 1.026.932-.724 1.275-1.905 1.275-2.633 0-.508-.306-.426-.81-.19l-.616.296c-.52.24-1.148.48-1.824.48-.676 0-1.302-.24-1.823-.48l-.589-.283c-.52-.248-.838-.342-.838.177 0 .703.32 1.831 1.187 2.56l.18.14z" fill="#3A3B45"></path><path d="M17.812 10.366a.806.806 0 00.813-.8c0-.441-.364-.8-.813-.8a.806.806 0 00-.812.8c0 .442.364.8.812.8zm-11.624 0a.806.806 0 00.812-.8c0-.441-.364-.8-.812-.8a.806.806 0 00-.813.8c0 .442.364.8.813.8zM4.515 13.073c-.405 0-.765.162-1.017.46a1.455 1.455 0 00-.333.925 1.801 1.801 0 00-.485-.074c-.387 0-.737.146-.985.409a1.41 1.41 0 00-.2 1.722 1.302 1.302 0 00-.447.694c-.06.222-.12.69.2 1.166a1.267 1.267 0 00-.093 1.236c.238.533.81.958 1.89 1.405l.24.096c.768.3 1.473.492 1.478.494.89.243 1.808.375 2.732.394 1.465 0 2.513-.443 3.115-1.314.93-1.342.842-2.575-.274-3.763l-.151-.154c-.692-.684-1.155-1.69-1.25-1.912-.195-.655-.71-1.383-1.562-1.383-.46.007-.889.233-1.15.605-.25-.31-.495-.553-.715-.694a1.87 1.87 0 00-.993-.312zm14.97 0c.405 0 .767.162 1.017.46.216.262.333.588.333.925.158-.047.322-.071.487-.074.388 0 .738.146.985.409a1.41 1.41 0 01.2 1.722c.22.178.377.422.445.694.06.222.12.69-.2 1.166.244.37.279.836.093 1.236-.238.533-.81.958-1.889 1.405l-.239.096c-.77.3-1.475.492-1.48.494-.89.243-1.808.375-2.732.394-1.465 0-2.513-.443-3.115-1.314-.93-1.342-.842-2.575.274-3.763l.151-.154c.695-.684 1.157-1.69 1.252-1.912.195-.655.708-1.383 1.56-1.383.46.007.889.233 1.15.605.25-.31.495-.553.718-.694.244-.162.523-.265.814-.3l.176-.012z" fill="#FF9D0B"></path><path d="M9.785 20.132c.688-.994.638-1.74-.305-2.667-.945-.928-1.495-2.288-1.495-2.288s-.205-.788-.672-.714c-.468.074-.81 1.25.17 1.971.977.721-.195 1.21-.573.534-.375-.677-1.405-2.416-1.94-2.751-.532-.332-.907-.148-.782.541.125.687 2.357 2.35 2.14 2.707-.218.362-.983-.42-.983-.42S2.953 14.9 2.43 15.46c-.52.558.398 1.026 1.7 1.803 1.308.778 1.41.985 1.225 1.28-.187.295-3.07-2.1-3.34-1.083-.27 1.011 2.943 1.304 2.745 2.006-.2.7-2.265-1.324-2.685-.537-.425.79 2.913 1.718 2.94 1.725 1.075.276 3.813.859 4.77-.522zm4.432 0c-.687-.994-.64-1.74.305-2.667.943-.928 1.493-2.288 1.493-2.288s.205-.788.675-.714c.465.074.807 1.25-.17 1.971-.98.721.195 1.21.57.534.377-.677 1.407-2.416 1.94-2.751.532-.332.91-.148.782.541-.125.687-2.355 2.35-2.137 2.707.215.362.98-.42.98-.42S21.05 14.9 21.57 15.46c.52.558-.395 1.026-1.7 1.803-1.308.778-1.408.985-1.225 1.28.187.295 3.07-2.1 3.34-1.083.27 1.011-2.94 1.304-2.743 2.006.2.7 2.263-1.324 2.685-.537.423.79-2.912 1.718-2.94 1.725-1.077.276-3.815.859-4.77-.522z" fill="#FFD21E"></path></svg><span>Available on <b>Hugging Face</b></span></a></div>
</div>
<div class="wide" id="leaderboard">
<div class="panel">
<div class="panel-head">
<div class="t"><b>WaterfallBench v1.0</b><span>First-distribution waterfalls</span></div>
<div class="label" id="asof"></div>
</div>
<div class="toolbar">
<div class="seg" role="group" aria-label="Score">
<button class="btn" data-metric="fully_resolved" aria-pressed="true">Fully resolved</button>
<button class="btn" data-metric="nontrivial" aria-pressed="false">Field accuracy</button>
</div>
<span class="label">vs cost / task</span>
<span class="grow"></span>
<button class="btn" id="tFrontier" aria-pressed="true">Frontier</button>
<button class="btn" id="tLog" aria-pressed="true">Log scale</button>
</div>
<div class="legend label" id="legend"></div>
<div id="chart"><div class="tip" id="tip"></div></div>
</div>
<div class="panel" style="margin-top:20px">
<div class="panel-head">
<div class="t"><b>Leaderboard</b><span>20 tasks · one API call each · reasoning effort high</span></div>
</div>
<div class="tbl-wrap"><table id="board"></table></div>
</div>
</div>
<section id="summary"><div class="prose">
<h2>Can a model execute an ABS waterfall to the cent?</h2>
<p>WaterfallBench tests whether a model can read a securitization's legal documents and compute its monthly payments to investors exactly as the trustee did, to the cent. The answer key is what the trustee actually paid, as filed with the SEC.</p>
<p id="lede">Loading results…</p>
</div></section>
<section id="task"><div class="prose">
<h2>The task</h2>
<h3>What a waterfall is</h3>
<p>An asset-backed security turns thousands of car loans or leases into bonds, sold in classes from senior (A) to junior (B, C). Every month the borrowers' payments are collected and paid out in a fixed order of priority, the <strong>waterfall</strong>:</p>
<div class="wf" role="img" aria-label="Waterfall: collections pay the servicing fee, then interest, then principal, then the reserve account, then the residual">
<div class="wf-root"><b>Collections this month</b><span>loan payments, payoffs and recoveries</span></div>
<ul class="wf-tree">
<li><div class="wf-node"><i>1</i><b>Servicing fee</b><span>to the servicer that collects the loans</span></div></li>
<li><div class="wf-node"><i>2</i><b>Interest</b><span>senior classes first</span></div>
<ul>
<li><div class="wf-leaf">Class A notes<small>A-1, A-2, A-3, A-4 share pro rata</small></div></li>
<li><div class="wf-leaf">Class B</div></li>
<li><div class="wf-leaf">Class C</div></li>
</ul></li>
<li><div class="wf-node"><i>3</i><b>Principal</b><span>one class at a time, until each is paid off</span></div>
<ul><li><div class="wf-leaf wf-seq"><em>A-1</em>→<em>A-2</em>→<em>A-3</em>→<em>A-4</em>→<em>B</em>→<em>C</em></div></li></ul></li>
<li><div class="wf-node"><i>4</i><b>Reserve account</b><span>topped up to its required level</span></div></li>
<li><div class="wf-node"><i>5</i><b>Residual</b><span>whatever is left, to the equity holder</span></div></li>
</ul>
</div>
<p class="cap" style="margin:8px 0 18px">Each step is paid only from the cash left after the steps above it. Simplified: each deal's agreements set the exact order, amounts and targets.</p>
<h3>Why it matters</h3>
<p>Securitization funds a large share of US car loans and leases: the 20 deals in this benchmark alone issued <strong id="issued">$23.5 billion</strong> of notes. The waterfall decides who gets paid and in what order, so an error moves cash between senior and junior investors, and because each month starts from the last month's balances, it carries forward. Today this is done with deal-specific cash-flow models built by hand from the documents. WaterfallBench asks whether a model can read the documents and get it right on its own.</p>
<h3>What the model gets</h3>
<p>A deal's <strong>first distribution date</strong>. Opening balances come from the prospectus, so there is no prior report to copy from. All documents are EDGAR HTML as filed:</p>
<div class="panel tbl-wrap"><table class="inputs">
<thead><tr><th>Input</th><th>Filed as</th><th>What the waterfall needs from it</th></tr></thead>
<tbody>
<tr><td>Final prospectus</td><td class="form">424(b)(5)</td><td>Note classes, coupons, day counts, the priority of payments</td></tr>
<tr><td>Closing agreements</td><td class="form">8-K exhibits</td><td>Indenture and servicing agreements: the fee, reserve and overcollateralization definitions</td></tr>
<tr><td>Servicer report</td><td class="form">10-D Ex. 99.1</td><td>That month's collections, pool balances, rates and dates, with every waterfall result removed</td></tr>
</tbody>
</table></div>
<p style="margin-top:18px">It must return the servicing fee, each class's interest, principal and ending balance, the reserve balance and the residual, in integer cents. Here is the report from a real task as the model sees it:</p>
</div>
<div class="prose">
<div class="panel">
<div class="panel-head"><div class="t"><b>Servicer report</b><span>waterfall results removed</span></div><div class="label">Nissan Auto Lease 2026-B · Aug 2026</div></div>
<div class="tbl-wrap"><table class="rep">
<thead><tr><th>Summary</th><th class="num">Coupon</th><th class="num">Initial balance</th><th class="num">Beginning</th><th class="num">Ending</th></tr></thead>
<tbody>
<tr><td>Total Portfolio</td><td></td><td class="num">1,447,527,618.00</td><td class="num">1,447,527,618.00</td><td class="num">1,428,611,202.92</td></tr>
<tr><td>Class A-1 Notes</td><td class="num">3.949000%</td><td class="num">127,200,000.00</td><td class="num">127,200,000.00</td><td class="rm">REMOVED</td></tr>
<tr><td>Class A-2a Notes</td><td class="num">4.440000%</td><td class="num">414,000,000.00</td><td class="num">414,000,000.00</td><td class="rm">REMOVED</td></tr>
<tr><td>Class A-2b Notes</td><td class="num">3.978070%</td><td class="num">80,000,000.00</td><td class="num">80,000,000.00</td><td class="rm">REMOVED</td></tr>
<tr><td>Class A-3 Notes</td><td class="num">4.690000%</td><td class="num">494,000,000.00</td><td class="num">494,000,000.00</td><td class="rm">REMOVED</td></tr>
<tr><td>Class A-4 Notes</td><td class="num">4.780000%</td><td class="num">84,800,000.00</td><td class="num">84,800,000.00</td><td class="rm">REMOVED</td></tr>
<tr><td>Class B Notes</td><td class="num">4.870000%</td><td class="num">49,940,000.00</td><td class="num">49,940,000.00</td><td class="rm">REMOVED</td></tr>
<tr><td>Class C Notes</td><td class="num">5.080000%</td><td class="num">39,800,000.00</td><td class="num">39,800,000.00</td><td class="rm">REMOVED</td></tr>
<tr class="grp"><td colspan="5">III. Distributions</td></tr>
<tr><td colspan="4">Total Collections</td><td class="num">33,734,591.74</td></tr>
<tr><td colspan="4">2. Reimbursement of Payment Advance</td><td class="num">0.00</td></tr>
<tr><td colspan="4">4. Servicing Fee</td><td class="rm">REMOVED</td></tr>
<tr><td colspan="4">5. Class A-1 Notes Monthly Interest</td><td class="rm">REMOVED</td></tr>
<tr><td colspan="4"> Class A-1 Interest Carryover Shortfall</td><td class="num">0.00</td></tr>
</tbody>
</table></div>
</div>
<p class="cap">Excerpt of <code>fw-nissanlease-2026-b-2026-07</code>. The same packet holds the 6.0 MB prospectus and 9 closing agreements.</p>
</div></section>
<section id="dataset"><div class="prose">
<h2>Dataset</h2>
<p id="ds-lede"></p>
<p class="cta" style="margin:-4px 0 18px"><a class="hf-badge" href="https://huggingface.co/datasets/ritik12/WaterfallBench"><svg class="hf-logo" viewBox="0 0 24 24" width="20" height="20" aria-hidden="true"><path d="M2.25 11.535c0-3.407 1.847-6.554 4.844-8.258a9.822 9.822 0 019.687 0c2.997 1.704 4.844 4.851 4.844 8.258 0 5.266-4.337 9.535-9.687 9.535S2.25 16.8 2.25 11.535z" fill="#FF9D0B"></path><path d="M11.938 20.086c4.797 0 8.687-3.829 8.687-8.551 0-4.722-3.89-8.55-8.687-8.55-4.798 0-8.688 3.828-8.688 8.55 0 4.722 3.89 8.55 8.688 8.55z" fill="#FFD21E"></path><path d="M11.875 15.113c2.457 0 3.25-2.156 3.25-3.263 0-.576-.393-.394-1.023-.089-.582.283-1.365.675-2.224.675-1.798 0-3.25-1.693-3.25-.586 0 1.107.79 3.263 3.25 3.263h-.003z" fill="#FF323D"></path><path d="M14.76 9.21c.32.108.445.753.767.585.447-.233.707-.708.659-1.204a1.235 1.235 0 00-.879-1.059 1.262 1.262 0 00-1.33.394c-.322.384-.377.92-.14 1.36.153.283.638-.177.925-.079l-.002.003zm-5.887 0c-.32.108-.448.753-.768.585a1.226 1.226 0 01-.658-1.204c.048-.495.395-.913.878-1.059a1.262 1.262 0 011.33.394c.322.384.377.92.14 1.36-.152.283-.64-.177-.925-.079l.003.003zm1.12 5.34a2.166 2.166 0 011.325-1.106c.07-.02.144.06.219.171l.192.306c.069.1.139.175.209.175.074 0 .15-.074.223-.172l.205-.302c.08-.11.157-.188.234-.165.537.168.986.536 1.25 1.026.932-.724 1.275-1.905 1.275-2.633 0-.508-.306-.426-.81-.19l-.616.296c-.52.24-1.148.48-1.824.48-.676 0-1.302-.24-1.823-.48l-.589-.283c-.52-.248-.838-.342-.838.177 0 .703.32 1.831 1.187 2.56l.18.14z" fill="#3A3B45"></path><path d="M17.812 10.366a.806.806 0 00.813-.8c0-.441-.364-.8-.813-.8a.806.806 0 00-.812.8c0 .442.364.8.812.8zm-11.624 0a.806.806 0 00.812-.8c0-.441-.364-.8-.812-.8a.806.806 0 00-.813.8c0 .442.364.8.813.8zM4.515 13.073c-.405 0-.765.162-1.017.46a1.455 1.455 0 00-.333.925 1.801 1.801 0 00-.485-.074c-.387 0-.737.146-.985.409a1.41 1.41 0 00-.2 1.722 1.302 1.302 0 00-.447.694c-.06.222-.12.69.2 1.166a1.267 1.267 0 00-.093 1.236c.238.533.81.958 1.89 1.405l.24.096c.768.3 1.473.492 1.478.494.89.243 1.808.375 2.732.394 1.465 0 2.513-.443 3.115-1.314.93-1.342.842-2.575-.274-3.763l-.151-.154c-.692-.684-1.155-1.69-1.25-1.912-.195-.655-.71-1.383-1.562-1.383-.46.007-.889.233-1.15.605-.25-.31-.495-.553-.715-.694a1.87 1.87 0 00-.993-.312zm14.97 0c.405 0 .767.162 1.017.46.216.262.333.588.333.925.158-.047.322-.071.487-.074.388 0 .738.146.985.409a1.41 1.41 0 01.2 1.722c.22.178.377.422.445.694.06.222.12.69-.2 1.166.244.37.279.836.093 1.236-.238.533-.81.958-1.889 1.405l-.239.096c-.77.3-1.475.492-1.48.494-.89.243-1.808.375-2.732.394-1.465 0-2.513-.443-3.115-1.314-.93-1.342-.842-2.575.274-3.763l.151-.154c.695-.684 1.157-1.69 1.252-1.912.195-.655.708-1.383 1.56-1.383.46.007.889.233 1.15.605.25-.31.495-.553.718-.694.244-.162.523-.265.814-.3l.176-.012z" fill="#FF9D0B"></path><path d="M9.785 20.132c.688-.994.638-1.74-.305-2.667-.945-.928-1.495-2.288-1.495-2.288s-.205-.788-.672-.714c-.468.074-.81 1.25.17 1.971.977.721-.195 1.21-.573.534-.375-.677-1.405-2.416-1.94-2.751-.532-.332-.907-.148-.782.541.125.687 2.357 2.35 2.14 2.707-.218.362-.983-.42-.983-.42S2.953 14.9 2.43 15.46c-.52.558.398 1.026 1.7 1.803 1.308.778 1.41.985 1.225 1.28-.187.295-3.07-2.1-3.34-1.083-.27 1.011 2.943 1.304 2.745 2.006-.2.7-2.265-1.324-2.685-.537-.425.79 2.913 1.718 2.94 1.725 1.075.276 3.813.859 4.77-.522zm4.432 0c-.687-.994-.64-1.74.305-2.667.943-.928 1.493-2.288 1.493-2.288s.205-.788.675-.714c.465.074.807 1.25-.17 1.971-.98.721.195 1.21.57.534.377-.677 1.407-2.416 1.94-2.751.532-.332.91-.148.782.541-.125.687-2.355 2.35-2.137 2.707.215.362.98-.42.98-.42S21.05 14.9 21.57 15.46c.52.558-.395 1.026-1.7 1.803-1.308.778-1.408.985-1.225 1.28.187.295 3.07-2.1 3.34-1.083.27 1.011-2.94 1.304-2.743 2.006.2.7 2.263-1.324 2.685-.537.423.79-2.912 1.718-2.94 1.725-1.077.276-3.815.859-4.77-.522z" fill="#FFD21E"></path></svg><span>Download from <b>Hugging Face</b></span></a></p>
<div class="kv" id="kv"></div>
<div class="panel tbl-wrap"><table id="families" class="compact"></table></div>
<p class="cap">Tokens: mean prompt per task, Claude tokenizer, as sent by the harness. No task needed truncation.</p>
<h3>Release checks</h3>
<p>Every task passed these checks: the report's classes match the prospectus; each class's opening balance minus principal paid equals its ending balance, to the cent; class payments add up to the report's totals; no graded value is left in the packet; and the true answer grades 100% while an all-zeros answer resolves nothing.</p>
</div></section>
<section id="methodology"><div class="prose">
<h2>Methodology</h2>
<h3>Harness</h3>
<p>One API call per task: a fixed system prompt, the documents and the task. No tools, no code execution, no second turn, so the comparison is the model alone. EDGAR HTML is converted to text with table rows kept as <code>cell | cell</code>, then packed (report, prospectus, agreements) under a 1M-token cap; no task needed truncation. Reasoning effort is <code>high</code> for every model.</p>
<h3>Evaluation metrics</h3>
<p>Every task uses the same answer schema, given in the prompt. The model's reply is parsed, its classes are matched to the deal's by name, and every field is compared with the filed value, exactly.</p>
</div>
<div class="wide" style="max-width:1200px">
<div class="io io-eq">
<div class="panel">
<div class="panel-head"><div class="t"><b>In the prompt</b><span>answer_schema.json</span></div><div class="label">same for all 20 tasks</div></div>
<pre class="json">{
"type": "object", "additionalProperties": false,
"required": ["servicing_fee_paid", "reserve_ending_balance",
"released_to_residual", "classes"],
"properties": {
<span class="k">"servicing_fee_paid"</span>: {"type": "integer", "description": "US cents"},
<span class="k">"reserve_ending_balance"</span>: {"type": "integer", "description": "US cents"},
<span class="k">"released_to_residual"</span>: {"type": "integer", "description": "US cents"},
<span class="k">"classes"</span>: {"type": "array", "minItems": 1,
"description": "one entry per class of notes the trust issued",
"items": {"type": "object",
"required": ["class", "interest_paid",
"principal_paid", "ending_balance"],
"properties": {
<span class="k">"class"</span>: {"type": "string",
"description": "the class name as the prospectus gives it, e.g. A-1"},
<span class="k">"interest_paid"</span>: {"type": "integer", "description": "US cents"},
<span class="k">"principal_paid"</span>: {"type": "integer", "description": "US cents"},
<span class="k">"ending_balance"</span>: {"type": "integer", "description": "US cents"}}}}}
}</pre>
</div>
<div class="panel">
<div class="panel-head"><div class="t"><b>Actual output, graded</b><span>Kimi K3.1 Pro</span></div><div class="label">Nissan Auto Lease 2026-B</div></div>
<pre class="json">{
<span class="k">"servicing_fee_paid"</span>: <span class="g-ok">✓</span>,
<span class="k">"reserve_ending_balance"</span>: <span class="g-ok">✓</span>,
<span class="k">"released_to_residual"</span>: <span class="g-x">✗</span>,
<span class="k">"classes"</span>: [
{<span class="k">"class"</span>: "A-1", <span class="k">"interest_paid"</span>: <span class="g-ok">✓</span>,
<span class="k">"principal_paid"</span>: <span class="g-ok">✓</span>, <span class="k">"ending_balance"</span>: <span class="g-ok">✓</span>},
{<span class="k">"class"</span>: "A-2a", <span class="k">"interest_paid"</span>: <span class="g-ok">✓</span>,
<span class="k">"principal_paid"</span>: <span class="g-ok g-t">✓</span>, <span class="k">"ending_balance"</span>: <span class="g-ok g-t">✓</span>},
{<span class="k">"class"</span>: "A-2b", <span class="k">"interest_paid"</span>: <span class="g-ok">✓</span>,
<span class="k">"principal_paid"</span>: <span class="g-ok g-t">✓</span>, <span class="k">"ending_balance"</span>: <span class="g-ok g-t">✓</span>},
{<span class="k">"class"</span>: "A-3", <span class="k">"interest_paid"</span>: <span class="g-x">✗</span>,
<span class="k">"principal_paid"</span>: <span class="g-ok g-t">✓</span>, <span class="k">"ending_balance"</span>: <span class="g-ok g-t">✓</span>},
{<span class="k">"class"</span>: "A-4", <span class="k">"interest_paid"</span>: <span class="g-ok">✓</span>,
<span class="k">"principal_paid"</span>: <span class="g-ok g-t">✓</span>, <span class="k">"ending_balance"</span>: <span class="g-ok g-t">✓</span>},
{<span class="k">"class"</span>: "B", <span class="k">"interest_paid"</span>: <span class="g-x">✗</span>,
<span class="k">"principal_paid"</span>: <span class="g-ok g-t">✓</span>, <span class="k">"ending_balance"</span>: <span class="g-ok g-t">✓</span>},
{<span class="k">"class"</span>: "C", <span class="k">"interest_paid"</span>: <span class="g-ok">✓</span>,
<span class="k">"principal_paid"</span>: <span class="g-ok g-t">✓</span>, <span class="k">"ending_balance"</span>: <span class="g-ok g-t">✓</span>}
]
}</pre>
<p class="cap" style="padding:0 14px 12px">Values hidden: answer keys stay private. <span class="g-ok">✓</span> exact to the cent · <span class="g-x">✗</span> wrong · <span class="g-ok g-t">✓</span> trivial field (zero, or an opening balance left untouched).</p>
</div>
</div>
</div>
<div class="prose">
<div class="panel tbl-wrap" style="margin-top:18px"><table class="inputs">
<thead><tr><th>Metric</th><th>Definition</th><th class="num">This output</th></tr></thead>
<tbody>
<tr><td>Fully resolved <span class="label">primary</span></td><td>1 if every graded field is exact to the cent, else 0</td><td class="num v">0</td></tr>
<tr><td>Field accuracy</td><td>Share of non-trivial fields exactly right. Trivial fields are left out so untouched balances don't inflate the score</td><td class="num v">9 / 12 = 75%</td></tr>
<tr><td>All fields</td><td>Share of all graded fields exactly right (the task-by-task grid)</td><td class="num v">21 / 24 = 88%</td></tr>
</tbody>
</table></div>
<p style="margin-top:16px">Leaderboard scores average these over the 20 tasks; ± is one standard error. Grading is deterministic, with no LLM judge. Cost uses each provider's list price for the tokens it reported.</p>
<p id="ungraded"><strong>Not graded.</strong> Auditing the runs turned up three places where the documents cannot determine a value. John Deere's first fee needs a pool balance no document prints. CarMax 2025-2's report departs from its own agreement's overcollateralization target. World Omni's agreement prorates the first fee without a day count. Those fields are left out of scoring; the filed values are never changed.</p>
<pre class="code">valkyrie run start --benchmark waterfallbench --dataset full-html \
--agent waterfallbench-direct --model anthropic/claude-haiku-5-5 \
-k effort high -s ANTHROPIC_API_KEY anthropic-api-key --concurrency 5</pre>
</div></section>
<section id="results"><div class="prose">
<h2>Results</h2>
<h3>Field accuracy vs task completion</h3>
<p id="db-lede"></p>
</div>
<div class="prose"><div class="panel" id="dumbbell"></div></div>
<div class="prose">
<h3>Task by task</h3>
<p>Green means every graded field is exact to the cent. Otherwise the cell shows the share of graded fields the model got exactly right.</p>
</div>
<div class="wide" style="max-width:1100px"><div class="panel tbl-wrap"><table class="grid" id="grid"></table></div></div>
</section>
</main>
<footer><div class="in">
<span>WaterfallBench · SEC EDGAR filings · tasks public, answer keys private</span>
<span><a href="https://github.com/RITIK-12/WaterfallBench">GitHub</a></span>
</div></footer>
<script>
const LOGO = {
"Anthropic": ["M13.827 3.52h3.603L24 20h-3.603l-6.57-16.48zm-7.258 0h3.767L16.906 20h-3.674l-1.343-3.461H5.017l-1.344 3.46H0L6.57 3.522zm4.132 9.959L8.453 7.687 6.205 13.48H10.7z"],
"OpenAI": ["M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"],
"Moonshot AI": ["M21.846 0a1.923 1.923 0 110 3.846H20.15a.226.226 0 01-.227-.226V1.923C19.923.861 20.784 0 21.846 0z", "M11.065 11.199l7.257-7.2c.137-.136.06-.41-.116-.41H14.3a.164.164 0 00-.117.051l-7.82 7.756c-.122.12-.302.013-.302-.179V3.82c0-.127-.083-.23-.185-.23H3.186c-.103 0-.186.103-.186.23V19.77c0 .128.083.23.186.23h2.69c.103 0 .186-.102.186-.23v-3.25c0-.069.025-.135.069-.178l2.424-2.406a.158.158 0 01.205-.023l6.484 4.772a7.677 7.677 0 003.453 1.283c.108.012.2-.095.2-.23v-3.06c0-.117-.07-.212-.164-.227a5.028 5.028 0 01-2.027-.807l-5.613-4.064c-.117-.078-.132-.279-.028-.381z"],
"MiniMax": ["M16.278 2c1.156 0 2.093.927 2.093 2.07v12.501a.74.74 0 00.744.709.74.74 0 00.743-.709V9.099a2.06 2.06 0 012.071-2.049A2.06 2.06 0 0124 9.1v6.561a.649.649 0 01-.652.645.649.649 0 01-.653-.645V9.1a.762.762 0 00-.766-.758.762.762 0 00-.766.758v7.472a2.037 2.037 0 01-2.048 2.026 2.037 2.037 0 01-2.048-2.026v-12.5a.785.785 0 00-.788-.753.785.785 0 00-.789.752l-.001 15.904A2.037 2.037 0 0113.441 22a2.037 2.037 0 01-2.048-2.026V18.04c0-.356.292-.645.652-.645.36 0 .652.289.652.645v1.934c0 .263.142.506.372.638.23.131.514.131.744 0a.734.734 0 00.372-.638V4.07c0-1.143.937-2.07 2.093-2.07zm-5.674 0c1.156 0 2.093.927 2.093 2.07v11.523a.648.648 0 01-.652.645.648.648 0 01-.652-.645V4.07a.785.785 0 00-.789-.78.785.785 0 00-.789.78v14.013a2.06 2.06 0 01-2.07 2.048 2.06 2.06 0 01-2.071-2.048V9.1a.762.762 0 00-.766-.758.762.762 0 00-.766.758v3.8a2.06 2.06 0 01-2.071 2.049A2.06 2.06 0 010 12.9v-1.378c0-.357.292-.646.652-.646.36 0 .653.29.653.646V12.9c0 .418.343.757.766.757s.766-.339.766-.757V9.099a2.06 2.06 0 012.07-2.048 2.06 2.06 0 012.071 2.048v8.984c0 .419.343.758.767.758.423 0 .766-.339.766-.758V4.07c0-1.143.937-2.07 2.093-2.07z"],
"DeepSeek": ["M23.748 4.482c-.254-.124-.364.113-.512.234-.051.039-.094.09-.137.136-.372.397-.806.657-1.373.626-.829-.046-1.537.214-2.163.848-.133-.782-.575-1.248-1.247-1.548-.352-.156-.708-.311-.955-.65-.172-.241-.219-.51-.305-.774-.055-.16-.11-.323-.293-.35-.2-.031-.278.136-.356.276-.313.572-.434 1.202-.422 1.84.027 1.436.633 2.58 1.838 3.393.137.093.172.187.129.323-.082.28-.18.552-.266.833-.055.179-.137.217-.329.14a5.526 5.526 0 01-1.736-1.18c-.857-.828-1.631-1.742-2.597-2.458a11.365 11.365 0 00-.689-.471c-.985-.957.13-1.743.388-1.836.27-.098.093-.432-.779-.428-.872.004-1.67.295-2.687.684a3.055 3.055 0 01-.465.137 9.597 9.597 0 00-2.883-.102c-1.885.21-3.39 1.102-4.497 2.623C.082 8.606-.231 10.684.152 12.85c.403 2.284 1.569 4.175 3.36 5.653 1.858 1.533 3.997 2.284 6.438 2.14 1.482-.085 3.133-.284 4.994-1.86.47.234.962.327 1.78.397.63.059 1.236-.03 1.705-.128.735-.156.684-.837.419-.961-2.155-1.004-1.682-.595-2.113-.926 1.096-1.296 2.746-2.642 3.392-7.003.05-.347.007-.565 0-.845-.004-.17.035-.237.23-.256a4.173 4.173 0 001.545-.475c1.396-.763 1.96-2.015 2.093-3.517.02-.23-.004-.467-.247-.588zM11.581 18c-2.089-1.642-3.102-2.183-3.52-2.16-.392.024-.321.471-.235.763.09.288.207.486.371.739.114.167.192.416-.113.603-.673.416-1.842-.14-1.897-.167-1.361-.802-2.5-1.86-3.301-3.307-.774-1.393-1.224-2.887-1.298-4.482-.02-.386.093-.522.477-.592a4.696 4.696 0 011.529-.039c2.132.312 3.946 1.265 5.468 2.774.868.86 1.525 1.887 2.202 2.891.72 1.066 1.494 2.082 2.48 2.914.348.292.625.514.891.677-.802.09-2.14.11-3.054-.614zm1-6.44a.306.306 0 01.415-.287.302.302 0 01.2.288.306.306 0 01-.31.307.303.303 0 01-.304-.308zm3.11 1.596c-.2.081-.399.151-.59.16a1.245 1.245 0 01-.798-.254c-.274-.23-.47-.358-.552-.758a1.73 1.73 0 01.016-.588c.07-.327-.008-.537-.239-.727-.187-.156-.426-.199-.688-.199a.559.559 0 01-.254-.078c-.11-.054-.2-.19-.114-.358.028-.054.16-.186.192-.21.356-.202.767-.136 1.146.016.352.144.618.408 1.001.782.391.451.462.576.685.914.176.265.336.537.445.848.067.195-.019.354-.25.452z"],
"Z.ai": ["M12.105 2L9.927 4.953H.653L2.83 2h9.276zM23.254 19.048L21.078 22h-9.242l2.174-2.952h9.244zM24 2L9.264 22H0L14.736 2H24z"]
} /* LobeHub icons (MIT), github.com/lobehub/lobe-icons */;
const S = { metric: "fully_resolved", log: true, frontier: true, sort: { key: "fully_resolved", dir: -1 } };
const LAB = { Anthropic: "--lab", OpenAI: "--lab", "Moonshot AI": "--lab", MiniMax: "--lab", DeepSeek: "--lab", "Z.ai": "--lab" }; // identity = logo + name; one ink color
const css = v => getComputedStyle(document.documentElement).getPropertyValue(v).trim();
const fmtUSD = v => "$" + v.toFixed(2);
const fmtDur = s => s >= 60 ? `${Math.floor(s / 60)}m ${String(Math.round(s % 60)).padStart(2, "0")}s` : `${s.toFixed(0)}s`;
const esc = s => String(s).replace(/[&<>"]/g, c => ({ "&": "&", "<": "<", ">": ">", '"': """ }[c]));
const logo = (lab, s = 16) => `<svg class="logo" viewBox="0 0 24 24" width="${s}" height="${s}" fill="var(${LAB[lab]})" fill-rule="evenodd" role="img" aria-label="${esc(lab)}">${LOGO[lab].map(d => `<path d="${d}"/>`).join("")}</svg>`;
const and = xs => xs.length < 2 ? xs.join("") : xs.slice(0, -1).join(", ") + " and " + xs[xs.length - 1];
let D;
function el(tag, attrs = {}, kids = []) {
const e = document.createElementNS("http://www.w3.org/2000/svg", tag);
for (const [k, v] of Object.entries(attrs)) e.setAttribute(k, v);
for (const k of [].concat(kids)) e.append(k);
return e;
}
function frontier(pts) { // models that no cheaper model beats
const out = []; let best = -Infinity;
for (const p of [...pts].sort((a, b) => a.x - b.x || b.y - a.y)) if (p.y > best) { out.push(p); best = p.y; }
return out;
}
function chart() {
const box = document.getElementById("chart");
box.querySelector("svg")?.remove();
const W = box.clientWidth;
if (W < 240) return; // not laid out yet; the ResizeObserver draws it once it has a width
const H = Math.max(320, Math.min(480, W * 0.5));
const m = { l: 56, r: W < 560 ? 18 : 28, t: 22, b: 42 };
const pts = D.models.map(md => ({ md, x: md.cost, y: md[S.metric], se: S.metric === "fully_resolved" ? md.se : 0 }));
const ys = pts.map(p => p.y - p.se);
const y0 = Math.max(0, Math.floor((Math.min(...ys) - 6) / 10) * 10), y1 = 100 + (100 - y0) * 0.08;
const xs = pts.map(p => p.x);
const lx0 = Math.log10(Math.min(...xs) / 2.2), lx1 = Math.log10(Math.max(...xs) * 2.2);
const X = S.log ? v => m.l + (Math.log10(v) - lx0) / (lx1 - lx0) * (W - m.l - m.r)
: v => m.l + v / (Math.max(...xs) * 1.15) * (W - m.l - m.r);
const Y = v => m.t + (y1 - v) / (y1 - y0) * (H - m.t - m.b);
const svg = el("svg", { viewBox: `0 0 ${W} ${H}`, height: H, role: "img", "aria-label": "Score versus cost per task" });
const midX = (m.l + W - m.r) / 2, midY = (m.t + H - m.b) / 2;
svg.append(el("rect", { x: m.l, y: m.t, width: midX - m.l, height: midY - m.t, fill: css("--eff") }));
svg.append(el("rect", { x: midX, y: midY, width: W - m.r - midX, height: H - m.b - midY, fill: css("--ineff") }));
const mono = (x, y, t, a = {}) => el("text", { x, y, "font-family": css("--mono"), "font-size": 10.5, "letter-spacing": ".06em", fill: css("--muted"), ...a }, t);
svg.append(mono(midX - 8, m.t + 14, "EFFICIENT", { "text-anchor": "end" }));
svg.append(mono(midX + 8, H - m.b - 8, "INEFFICIENT"));
if (W >= 640) svg.append(mono(m.l + 30, m.t + 14, "BEST PERFORMANCE"));
svg.append(mono(m.l + 14, H - m.b - 8, "LOWEST COST", { transform: `rotate(-90 ${m.l + 14} ${H - m.b - 8})` }));
for (let v = y0; v <= 100; v += 10) {
svg.append(el("line", { x1: m.l, x2: W - m.r, y1: Y(v), y2: Y(v), stroke: css("--line"), "stroke-dasharray": v % 20 ? "2 4" : "" }));
if (v % 20 === 0 || 100 - y0 <= 30) svg.append(mono(m.l - 8, Y(v) + 4, v + "%", { "text-anchor": "end" }));
}
const ticks = S.log ? [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 25].filter(v => Math.log10(v) >= lx0 && Math.log10(v) <= lx1)
: Array.from({ length: 6 }, (_, i) => +(Math.max(...xs) * 1.15 * i / 5).toFixed(1));
let lastTick = -Infinity;
for (const v of ticks) {
if (X(v) - lastTick < 48) continue;
lastTick = X(v);
svg.append(el("line", { x1: X(v), x2: X(v), y1: m.t, y2: H - m.b, stroke: css("--line"), "stroke-dasharray": "2 4" }));
svg.append(mono(X(v), H - m.b + 18, "$" + (v < 1 ? v.toFixed(2) : v.toFixed(v % 1 ? 1 : 0)), { "text-anchor": "middle" }));
}
svg.append(el("line", { x1: m.l, x2: W - m.r, y1: H - m.b, y2: H - m.b, stroke: css("--line-2") }));
svg.append(mono((m.l + W - m.r) / 2, H - 6, "COST PER TASK" + (S.log ? " (LOG)" : ""), { "text-anchor": "middle" }));
if (S.frontier) {
const f = frontier(pts);
if (f.length > 1) svg.append(el("polyline", { points: f.map(p => `${X(p.x)},${Y(p.y)}`).join(" "), fill: "none", stroke: css("--accent"), "stroke-width": 1.6, opacity: .8 }));
}
// markers are lab logos; labels placed greedily so they never overlap each other or a marker
const tip = document.getElementById("tip");
const placed = pts.map(p => ({ x0: X(p.x) - 14, x1: X(p.x) + 14, y0: Y(p.y) - 14, y1: Y(p.y) + 14 }));
const hit = (a, b) => a.x0 < b.x1 && b.x0 < a.x1 && a.y0 < b.y1 && b.y0 < a.y1;
const labelAt = (cx, cy, name) => {
const w = name.length * 7 + 4, h = 15;
const opts = [[cx + 18, cy - 20, "start"], [cx + 18, cy + 6, "start"], [cx - 18 - w, cy - 20, "end"], [cx - 18 - w, cy + 6, "end"],
[cx - w / 2, cy + 18, "middle"], [cx - w / 2, cy - 32, "middle"], [cx + 12 - w, cy + 18, "end"], [cx - 12, cy + 18, "start"],
[cx + 12 - w, cy - 32, "end"], [cx - 12, cy - 32, "start"]];
for (const [x, y, anchor] of opts) {
const r = { x0: x, x1: x + w, y0: y, y1: y + h };
if (r.x0 < m.l + 2 || r.x1 > W - 2 || r.y0 < 2 || r.y1 > H - m.b) continue;
if (placed.some(q => hit(r, q))) continue;
placed.push(r);
return { x: anchor === "start" ? x : anchor === "end" ? x + w : x + w / 2, y: y + 11, anchor };
}
return { x: cx + 18, y: cy - 9, anchor: "start" };
};
for (const p of [...pts].sort((a, b) => b.y - a.y || a.x - b.x)) {
const cx = X(p.x), cy = Y(p.y), c = css(LAB[p.md.lab]);
if (p.se) svg.append(el("line", { x1: cx, x2: cx, y1: Y(Math.min(100, p.y + p.se)), y2: Y(p.y - p.se), stroke: c, "stroke-width": 1.4, opacity: .5 }));
const g = el("g", { tabindex: 0, style: "cursor:pointer" });
const R = W < 560 ? 10 : 13, ls = W < 560 ? 12 : 16; // marker radius and logo size
g.append(el("circle", { cx, cy, r: R, fill: css("--panel"), stroke: c, "stroke-width": 1.5 }));
const lg = el("g", { transform: `translate(${cx - ls / 2} ${cy - ls / 2}) scale(${ls / 24})`, fill: c, "fill-rule": "evenodd" });
for (const d of LOGO[p.md.lab]) lg.append(el("path", { d }));
g.append(lg);
const L = labelAt(cx, cy, p.md.name);
g.append(el("text", { x: L.x, y: L.y, "text-anchor": L.anchor, "font-family": css("--sans"), "font-size": 12.5, fill: css("--ink") }, p.md.name));
const show = () => {
tip.innerHTML = `<b>${esc(p.md.name)}</b><div class="row"><span>Fully resolved</span><span>${p.md.fully_resolved.toFixed(1)}% ±${p.md.se.toFixed(1)}</span></div><div class="row"><span>Field accuracy</span><span>${p.md.nontrivial.toFixed(1)}%</span></div><div class="row"><span>Cost / task</span><span>${fmtUSD(p.md.cost)}</span></div><div class="row"><span>Duration / task</span><span>${fmtDur(p.md.seconds)}</span></div>`;
tip.style.opacity = 1;
const tw = tip.offsetWidth;
tip.style.left = Math.min(Math.max(8, cx - tw / 2), W - tw - 8) + "px";
tip.style.top = (cy > 150 ? cy - tip.offsetHeight - 20 : cy + 22) + "px";
};
g.addEventListener("mouseenter", show); g.addEventListener("focus", show);
g.addEventListener("mouseleave", () => tip.style.opacity = 0); g.addEventListener("blur", () => tip.style.opacity = 0);
svg.append(g);
}
box.prepend(svg);
}
function board() {
const cols = [
{ k: "rank", t: "#" }, { k: "name", t: `Model (${D.models.length})` },
{ k: "fully_resolved", t: "Fully resolved", n: 1 }, { k: "nontrivial", t: "Field accuracy", n: 1 },
{ k: "cost", t: "Cost / task", n: 1 }, { k: "tokens", t: "Tokens / task", n: 1 }, { k: "seconds", t: "Duration / task", n: 1 },
];
const ranked = [...D.models].sort((a, b) => b.fully_resolved - a.fully_resolved || b.nontrivial - a.nontrivial);
const rank = new Map(); ranked.forEach((m, i) => rank.set(m.key, i > 0 && m.fully_resolved === ranked[i - 1].fully_resolved ? rank.get(ranked[i - 1].key) : i + 1));
const rows = [...D.models].sort((a, b) => {
const k = S.sort.key; const av = k === "rank" ? rank.get(a.key) : a[k], bv = k === "rank" ? rank.get(b.key) : b[k];
return (typeof av === "string" ? av.localeCompare(bv) : av - bv) * S.sort.dir;
});
const th = cols.map(c => `<th class="${c.n ? "num " : ""}${c.k !== "rank" ? "sort" : ""}" data-k="${c.k}">${c.t}${S.sort.key === c.k ? `<span class="arr">${S.sort.dir < 0 ? "▼" : "▲"}</span>` : ""}</th>`).join("");
const tr = rows.map(m => {
const r = rank.get(m.key);
return `<tr class="${r <= 3 ? "top" + r : ""}"><td class="rank">${r}</td>
<td><span class="model">${logo(m.lab)}${esc(m.name)}</span></td>
<td class="num"><span class="minibar"><b style="width:${m.fully_resolved}%"></b></span><span class="v">${m.fully_resolved.toFixed(1)}<span class="u">%</span><small>±${m.se.toFixed(1)}</small></span></td>
<td class="num v">${m.nontrivial.toFixed(1)}<span class="u">%</span></td>
<td class="num v"><span class="u">$</span>${m.cost.toFixed(2)}</td>
<td class="num v">${(m.tokens / 1000).toFixed(1)}<span class="u">k</span></td>
<td class="num v">${fmtDur(m.seconds)}</td></tr>`;
}).join("");
const t = document.getElementById("board");
t.innerHTML = `<thead><tr>${th}</tr></thead><tbody>${tr}</tbody>`;
t.querySelectorAll("th.sort").forEach(h => h.onclick = () => {
const k = h.dataset.k; S.sort = { key: k, dir: S.sort.key === k ? -S.sort.dir : (["name", "cost", "tokens", "seconds"].includes(k) ? 1 : -1) }; board();
});
}
function dumbbell() {
const box = document.getElementById("dumbbell");
const W = box.clientWidth, rowH = 44, m = { l: W < 520 ? 128 : 170, r: 24, t: 36, b: 30 };
if (W < 240) return;
const ms = [...D.models].sort((a, b) => b.fully_resolved - a.fully_resolved || b.nontrivial - a.nontrivial);
const lo = Math.max(0, Math.floor((Math.min(...ms.map(x => x.fully_resolved)) - 5) / 10) * 10);
const H = m.t + m.b + rowH * ms.length;
const X = v => m.l + (v - lo) / (100 - lo) * (W - m.l - m.r);
const svg = el("svg", { viewBox: `0 0 ${W} ${H}`, height: H, role: "img", "aria-label": "Fully resolved versus field accuracy" });
const mono = (x, y, t, a = {}) => el("text", { x, y, "font-family": css("--mono"), "font-size": 10.5, fill: css("--muted"), ...a }, t);
const step = (100 - lo) > 60 ? 20 : 10;
for (let v = lo; v <= 100; v += step) {
svg.append(el("line", { x1: X(v), x2: X(v), y1: m.t - 6, y2: H - m.b, stroke: css("--line"), "stroke-dasharray": "2 4" }));
svg.append(mono(X(v), H - m.b + 18, v + "%", { "text-anchor": "middle" }));
}
svg.append(el("circle", { cx: m.l + 5, cy: 14, r: 5, fill: css("--accent") })); svg.append(mono(m.l + 16, 18, "FIELD ACCURACY"));
svg.append(el("rect", { x: m.l + 136, y: 9, width: 10, height: 10, fill: "none", stroke: css("--ink"), "stroke-width": 1.6 })); svg.append(mono(m.l + 152, 18, "TASKS FULLY RESOLVED"));
ms.forEach((md, i) => {
const y = m.t + rowH * i + rowH / 2;
svg.append(el("text", { x: m.l - 14, y: y + 4, "text-anchor": "end", "font-family": css("--sans"), "font-size": 13, fill: css("--ink") }, md.name));
svg.append(el("line", { x1: X(md.fully_resolved), x2: X(md.nontrivial), y1: y, y2: y, stroke: css("--line-2"), "stroke-width": 3 }));
svg.append(el("circle", { cx: X(md.nontrivial), cy: y, r: 6, fill: css("--accent") }));
svg.append(el("rect", { x: X(md.fully_resolved) - 6, y: y - 6, width: 12, height: 12, fill: css("--panel"), stroke: css("--ink"), "stroke-width": 1.8 }));
svg.append(mono(X(md.fully_resolved) - 10, y - 10, md.fully_resolved.toFixed(0) + "%", { "text-anchor": "end", fill: css("--ink-2") }));
});
box.innerHTML = ""; box.append(svg);
}
const month = p => { p = p.replace(/-/g, ""); return new Date(+p.slice(0, 4), +p.slice(4, 6) - 1, 1).toLocaleString("en-US", { month: "short", year: "numeric" }); };
function grid() {
const ms = [...D.models].sort((a, b) => b.fully_resolved - a.fully_resolved || b.nontrivial - a.nontrivial);
const head = `<thead><tr><th class="sticky">Deal · first distribution</th>${ms.map(m => `<th class="mh">${logo(m.lab, 15)}<div style="margin-top:4px">${esc(m.name)}</div></th>`).join("")}</tr></thead>`;
const body = D.tasks.map(t => `<tr><td class="deal sticky">${esc(t.deal)}<small>${month(t.distribution_date)} · ${t.classes} classes · ${t.graded} fields</small></td>
${ms.map(m => { const r = m.results[t.id]; if (!r) return `<td class="cell">–</td>`;
return `<td class="cell"><span class="${r.ok ? "ok" : r.li >= 85 ? "part" : "low"}" title="${r.ok ? "every graded field exact" : `${r.li.toFixed(0)}% of graded fields exact`}">${r.ok ? "✓" : r.li.toFixed(0) + "%"}</span></td>`; }).join("")}</tr>`).join("");
document.getElementById("grid").innerHTML = head + `<tbody>${body}</tbody>`;
}
function prose() {
const ms = [...D.models].sort((a, b) => b.fully_resolved - a.fully_resolved || b.nontrivial - a.nontrivial || a.cost - b.cost);
const top = ms[0].fully_resolved, leaders = ms.filter(m => m.fully_resolved === top), rest = ms.filter(m => m.fully_resolved < top);
const d = new Date(D.updated + "T12:00:00");
const long = d.toLocaleDateString("en-US", { month: "long", day: "numeric", year: "numeric" });
document.getElementById("asof").textContent = "WaterfallBench v1.0 · " + d.toLocaleDateString("en-US", { month: "short", day: "numeric", year: "numeric" });
document.getElementById("lede").innerHTML = `As of ${long}, ${and(leaders.map(m => `<strong>${esc(m.name)}</strong>`))} ${leaders.length > 1 ? "tie for first" : "ranks first"} with ${top.toFixed(0)}% of deals fully resolved${rest.length ? `, followed by ${and(rest.slice(0, 2).map(m => `${esc(m.name)} (${m.fully_resolved.toFixed(0)}%)`))}` : ""}. Every model gets most fields right; resolving a whole deal to the cent is what separates them.`;
const nt = ms.map(m => m.nontrivial), fr = ms.map(m => m.fully_resolved);
document.getElementById("db-lede").textContent = `Field accuracy ranges from ${Math.min(...nt).toFixed(0)}% to ${Math.max(...nt).toFixed(0)}%, while the share of tasks fully resolved ranges from ${Math.min(...fr).toFixed(0)}% to ${Math.max(...fr).toFixed(0)}%. A deal resolves only if every graded field is exact, and one wrong definition usually carries into every line below it.`;
// dataset
const T = D.tasks, nf = T.reduce((a, t) => a + t.graded, 0), fams = [...new Set(T.map(t => t.family))];
const periods = T.map(t => t.distribution_date).sort();
document.getElementById("ds-lede").textContent = `${T.length} tasks from ${fams.length} auto-loan and lease issuers. Each is a trust's first Form 10-D on SEC EDGAR, with first distributions from ${month(periods[0])} to ${month(periods[periods.length - 1])}. Documents were retrieved with Firecrawl and are used as filed; nothing is generated or imputed.`;
const docs = T.map(t => t.docs + 1), mb = T.map(t => t.html_mb), tok = T.map(t => t.prompt_tokens).filter(Boolean);
document.getElementById("kv").innerHTML = [[T.length, "tasks"], [nf, "graded fields"], [`${Math.min(...docs)}–${Math.max(...docs)}`, "documents / task"],
[`${Math.min(...mb).toFixed(0)}–${Math.max(...mb).toFixed(0)} MB`, "HTML / task"], [`${Math.round(Math.min(...tok) / 1000)}–${Math.round(Math.max(...tok) / 1000)}k`, "prompt tokens / task"]]
.map(([v, k]) => `<div><b>${v}</b><span class="label">${k}</span></div>`).join("");
const fr2 = fams.map(f => { const ts = T.filter(t => t.family === f), ps = ts.map(t => t.distribution_date).sort(), cl = ts.map(t => t.classes), tk = ts.map(t => t.prompt_tokens).filter(Boolean);
return `<tr><td>${esc(ts[0].family_name)}</td><td class="num v">${ts.length}</td><td class="num v">${month(ps[0])} – ${month(ps[ps.length - 1])}</td><td class="num v">${Math.min(...cl) === Math.max(...cl) ? cl[0] : Math.min(...cl) + "–" + Math.max(...cl)}</td><td class="num v">${Math.round(tk.reduce((a, b) => a + b, 0) / tk.length / 1000)}k</td><td class="num v">${ts.reduce((a, t) => a + t.graded, 0)}</td></tr>`; }).join("");
document.getElementById("families").innerHTML = `<thead><tr><th>Issuer</th><th class="num">Deals</th><th class="num">First distributions</th><th class="num">Classes</th><th class="num">Tokens</th><th class="num">Fields</th></tr></thead><tbody>${fr2}</tbody>`;
const nu = T.reduce((a, t) => a + t.ungraded, 0), tu = T.filter(t => t.ungraded).length;
document.getElementById("ungraded").insertAdjacentHTML("beforeend", ` In all, ${nu} of ${nf + nu} fields across ${tu} tasks are not graded.`);
document.getElementById("legend").innerHTML = "Lab: " + [...new Set(ms.map(m => m.lab))].map(l => `<span>${logo(l, 13)}${esc(l)}</span>`).join("")
+ `<span class="fr-key"><i></i>Frontier: best score at each cost</span><span><i class="se-key"></i>±1 SE</span>`;
}
function renderAll() { chart(); board(); dumbbell(); grid(); }
document.querySelectorAll("[data-metric]").forEach(b => b.onclick = () => {
S.metric = b.dataset.metric; document.querySelectorAll("[data-metric]").forEach(x => x.setAttribute("aria-pressed", x === b)); chart();
});
document.getElementById("tLog").onclick = e => { S.log = !S.log; e.currentTarget.setAttribute("aria-pressed", S.log); chart(); };
document.getElementById("tFrontier").onclick = e => { S.frontier = !S.frontier; e.currentTarget.setAttribute("aria-pressed", S.frontier); chart(); };
document.getElementById("theme").onclick = () => {
const r = document.documentElement, dark = r.dataset.theme ? r.dataset.theme === "dark" : matchMedia("(prefers-color-scheme: dark)").matches;
r.dataset.theme = dark ? "light" : "dark"; try { localStorage.setItem("wfb-theme", r.dataset.theme); } catch (e) {}
renderAll();
};
try { const t = localStorage.getItem("wfb-theme"); if (t) document.documentElement.dataset.theme = t; } catch (e) {}
let rz; const redraw = () => { clearTimeout(rz); rz = setTimeout(() => { if (D) { chart(); dumbbell(); } }, 80); };
if ("ResizeObserver" in window) { const ro = new ResizeObserver(redraw); ro.observe(document.getElementById("chart")); ro.observe(document.getElementById("dumbbell")); }
else addEventListener("resize", redraw);
matchMedia("(prefers-color-scheme: dark)").addEventListener?.("change", renderAll);
fetch("data.json").then(r => r.json()).then(d => { D = d; prose(); renderAll(); })
.catch(() => { document.getElementById("lede").textContent = "Could not load results (data.json)."; });
</script>
</body>
</html>
|