Spaces:
Running
Running
Leaderboard: HF badge, real distribution dates, card chart
Browse files- .gitattributes +1 -0
- data.json +40 -20
- index.html +11 -5
- leaderboard.png +3 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
leaderboard.png filter=lfs diff=lfs merge=lfs -text
|
data.json
CHANGED
|
@@ -733,7 +733,8 @@
|
|
| 733 |
"ungraded_why": "Servicer report departs from the agreement's overcollateralization target",
|
| 734 |
"docs": 7,
|
| 735 |
"html_mb": 4.91,
|
| 736 |
-
"prompt_tokens": 629698
|
|
|
|
| 737 |
},
|
| 738 |
{
|
| 739 |
"id": "fw-carmax-2025-3-2025-07",
|
|
@@ -748,7 +749,8 @@
|
|
| 748 |
"ungraded_why": null,
|
| 749 |
"docs": 7,
|
| 750 |
"html_mb": 5.01,
|
| 751 |
-
"prompt_tokens": 632551
|
|
|
|
| 752 |
},
|
| 753 |
{
|
| 754 |
"id": "fw-carmax-2025-4-2025-10",
|
|
@@ -763,7 +765,8 @@
|
|
| 763 |
"ungraded_why": null,
|
| 764 |
"docs": 7,
|
| 765 |
"html_mb": 4.99,
|
| 766 |
-
"prompt_tokens": 633700
|
|
|
|
| 767 |
},
|
| 768 |
{
|
| 769 |
"id": "fw-carmax-2026-1-2026-01",
|
|
@@ -778,7 +781,8 @@
|
|
| 778 |
"ungraded_why": null,
|
| 779 |
"docs": 8,
|
| 780 |
"html_mb": 5.08,
|
| 781 |
-
"prompt_tokens": 647530
|
|
|
|
| 782 |
},
|
| 783 |
{
|
| 784 |
"id": "fw-carmax-2026-2-2026-04",
|
|
@@ -793,7 +797,8 @@
|
|
| 793 |
"ungraded_why": null,
|
| 794 |
"docs": 8,
|
| 795 |
"html_mb": 5.12,
|
| 796 |
-
"prompt_tokens": 647909
|
|
|
|
| 797 |
},
|
| 798 |
{
|
| 799 |
"id": "fw-carmax-2026-3-2026-07",
|
|
@@ -808,7 +813,8 @@
|
|
| 808 |
"ungraded_why": null,
|
| 809 |
"docs": 8,
|
| 810 |
"html_mb": 4.94,
|
| 811 |
-
"prompt_tokens": 614998
|
|
|
|
| 812 |
},
|
| 813 |
{
|
| 814 |
"id": "fw-deere-2025-b-2025-08",
|
|
@@ -823,7 +829,8 @@
|
|
| 823 |
"ungraded_why": "First servicing fee needs a mid-period pool balance no document provides",
|
| 824 |
"docs": 6,
|
| 825 |
"html_mb": 7.76,
|
| 826 |
-
"prompt_tokens": 503783
|
|
|
|
| 827 |
},
|
| 828 |
{
|
| 829 |
"id": "fw-deere-2026-2026-04",
|
|
@@ -838,7 +845,8 @@
|
|
| 838 |
"ungraded_why": "First servicing fee needs a mid-period pool balance no document provides",
|
| 839 |
"docs": 6,
|
| 840 |
"html_mb": 7.52,
|
| 841 |
-
"prompt_tokens": 501473
|
|
|
|
| 842 |
},
|
| 843 |
{
|
| 844 |
"id": "fw-deere-2026-b-2026-08",
|
|
@@ -853,7 +861,8 @@
|
|
| 853 |
"ungraded_why": "First servicing fee needs a mid-period pool balance no document provides",
|
| 854 |
"docs": 7,
|
| 855 |
"html_mb": 7.67,
|
| 856 |
-
"prompt_tokens": 522651
|
|
|
|
| 857 |
},
|
| 858 |
{
|
| 859 |
"id": "fw-nissanlease-2025-a-2025-01",
|
|
@@ -868,7 +877,8 @@
|
|
| 868 |
"ungraded_why": null,
|
| 869 |
"docs": 10,
|
| 870 |
"html_mb": 8.23,
|
| 871 |
-
"prompt_tokens": 725064
|
|
|
|
| 872 |
},
|
| 873 |
{
|
| 874 |
"id": "fw-nissanlease-2025-b-2025-07",
|
|
@@ -883,7 +893,8 @@
|
|
| 883 |
"ungraded_why": null,
|
| 884 |
"docs": 10,
|
| 885 |
"html_mb": 7.76,
|
| 886 |
-
"prompt_tokens": 712865
|
|
|
|
| 887 |
},
|
| 888 |
{
|
| 889 |
"id": "fw-nissanlease-2026-a-2026-02",
|
|
@@ -898,7 +909,8 @@
|
|
| 898 |
"ungraded_why": null,
|
| 899 |
"docs": 10,
|
| 900 |
"html_mb": 8.48,
|
| 901 |
-
"prompt_tokens": 739161
|
|
|
|
| 902 |
},
|
| 903 |
{
|
| 904 |
"id": "fw-nissanlease-2026-b-2026-07",
|
|
@@ -913,7 +925,8 @@
|
|
| 913 |
"ungraded_why": null,
|
| 914 |
"docs": 10,
|
| 915 |
"html_mb": 8.2,
|
| 916 |
-
"prompt_tokens": 732825
|
|
|
|
| 917 |
},
|
| 918 |
{
|
| 919 |
"id": "fw-worldomni-2025-a-2025-01",
|
|
@@ -928,7 +941,8 @@
|
|
| 928 |
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
|
| 929 |
"docs": 7,
|
| 930 |
"html_mb": 6.37,
|
| 931 |
-
"prompt_tokens": 614546
|
|
|
|
| 932 |
},
|
| 933 |
{
|
| 934 |
"id": "fw-worldomni-2025-b-2025-05",
|
|
@@ -943,7 +957,8 @@
|
|
| 943 |
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
|
| 944 |
"docs": 7,
|
| 945 |
"html_mb": 6.35,
|
| 946 |
-
"prompt_tokens": 624079
|
|
|
|
| 947 |
},
|
| 948 |
{
|
| 949 |
"id": "fw-worldomni-2025-c-2025-08",
|
|
@@ -958,7 +973,8 @@
|
|
| 958 |
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
|
| 959 |
"docs": 7,
|
| 960 |
"html_mb": 6.58,
|
| 961 |
-
"prompt_tokens": 629893
|
|
|
|
| 962 |
},
|
| 963 |
{
|
| 964 |
"id": "fw-worldomni-2025-d-2025-10",
|
|
@@ -973,7 +989,8 @@
|
|
| 973 |
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
|
| 974 |
"docs": 7,
|
| 975 |
"html_mb": 8.05,
|
| 976 |
-
"prompt_tokens": 638804
|
|
|
|
| 977 |
},
|
| 978 |
{
|
| 979 |
"id": "fw-worldomni-2026-a-2026-02",
|
|
@@ -988,7 +1005,8 @@
|
|
| 988 |
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
|
| 989 |
"docs": 7,
|
| 990 |
"html_mb": 6.98,
|
| 991 |
-
"prompt_tokens": 627265
|
|
|
|
| 992 |
},
|
| 993 |
{
|
| 994 |
"id": "fw-worldomni-2026-b-2026-05",
|
|
@@ -1003,7 +1021,8 @@
|
|
| 1003 |
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
|
| 1004 |
"docs": 7,
|
| 1005 |
"html_mb": 6.35,
|
| 1006 |
-
"prompt_tokens": 633802
|
|
|
|
| 1007 |
},
|
| 1008 |
{
|
| 1009 |
"id": "fw-worldomni-2026-c-2026-08",
|
|
@@ -1018,7 +1037,8 @@
|
|
| 1018 |
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
|
| 1019 |
"docs": 7,
|
| 1020 |
"html_mb": 7.76,
|
| 1021 |
-
"prompt_tokens": 636492
|
|
|
|
| 1022 |
}
|
| 1023 |
]
|
| 1024 |
}
|
|
|
|
| 733 |
"ungraded_why": "Servicer report departs from the agreement's overcollateralization target",
|
| 734 |
"docs": 7,
|
| 735 |
"html_mb": 4.91,
|
| 736 |
+
"prompt_tokens": 629698,
|
| 737 |
+
"distribution_date": "2025-05-15"
|
| 738 |
},
|
| 739 |
{
|
| 740 |
"id": "fw-carmax-2025-3-2025-07",
|
|
|
|
| 749 |
"ungraded_why": null,
|
| 750 |
"docs": 7,
|
| 751 |
"html_mb": 5.01,
|
| 752 |
+
"prompt_tokens": 632551,
|
| 753 |
+
"distribution_date": "2025-08-15"
|
| 754 |
},
|
| 755 |
{
|
| 756 |
"id": "fw-carmax-2025-4-2025-10",
|
|
|
|
| 765 |
"ungraded_why": null,
|
| 766 |
"docs": 7,
|
| 767 |
"html_mb": 4.99,
|
| 768 |
+
"prompt_tokens": 633700,
|
| 769 |
+
"distribution_date": "2025-11-17"
|
| 770 |
},
|
| 771 |
{
|
| 772 |
"id": "fw-carmax-2026-1-2026-01",
|
|
|
|
| 781 |
"ungraded_why": null,
|
| 782 |
"docs": 8,
|
| 783 |
"html_mb": 5.08,
|
| 784 |
+
"prompt_tokens": 647530,
|
| 785 |
+
"distribution_date": "2026-02-17"
|
| 786 |
},
|
| 787 |
{
|
| 788 |
"id": "fw-carmax-2026-2-2026-04",
|
|
|
|
| 797 |
"ungraded_why": null,
|
| 798 |
"docs": 8,
|
| 799 |
"html_mb": 5.12,
|
| 800 |
+
"prompt_tokens": 647909,
|
| 801 |
+
"distribution_date": "2026-05-15"
|
| 802 |
},
|
| 803 |
{
|
| 804 |
"id": "fw-carmax-2026-3-2026-07",
|
|
|
|
| 813 |
"ungraded_why": null,
|
| 814 |
"docs": 8,
|
| 815 |
"html_mb": 4.94,
|
| 816 |
+
"prompt_tokens": 614998,
|
| 817 |
+
"distribution_date": "2026-08-17"
|
| 818 |
},
|
| 819 |
{
|
| 820 |
"id": "fw-deere-2025-b-2025-08",
|
|
|
|
| 829 |
"ungraded_why": "First servicing fee needs a mid-period pool balance no document provides",
|
| 830 |
"docs": 6,
|
| 831 |
"html_mb": 7.76,
|
| 832 |
+
"prompt_tokens": 503783,
|
| 833 |
+
"distribution_date": "2025-08-15"
|
| 834 |
},
|
| 835 |
{
|
| 836 |
"id": "fw-deere-2026-2026-04",
|
|
|
|
| 845 |
"ungraded_why": "First servicing fee needs a mid-period pool balance no document provides",
|
| 846 |
"docs": 6,
|
| 847 |
"html_mb": 7.52,
|
| 848 |
+
"prompt_tokens": 501473,
|
| 849 |
+
"distribution_date": "2026-04-15"
|
| 850 |
},
|
| 851 |
{
|
| 852 |
"id": "fw-deere-2026-b-2026-08",
|
|
|
|
| 861 |
"ungraded_why": "First servicing fee needs a mid-period pool balance no document provides",
|
| 862 |
"docs": 7,
|
| 863 |
"html_mb": 7.67,
|
| 864 |
+
"prompt_tokens": 522651,
|
| 865 |
+
"distribution_date": "2026-08-17"
|
| 866 |
},
|
| 867 |
{
|
| 868 |
"id": "fw-nissanlease-2025-a-2025-01",
|
|
|
|
| 877 |
"ungraded_why": null,
|
| 878 |
"docs": 10,
|
| 879 |
"html_mb": 8.23,
|
| 880 |
+
"prompt_tokens": 725064,
|
| 881 |
+
"distribution_date": "2025-02-18"
|
| 882 |
},
|
| 883 |
{
|
| 884 |
"id": "fw-nissanlease-2025-b-2025-07",
|
|
|
|
| 893 |
"ungraded_why": null,
|
| 894 |
"docs": 10,
|
| 895 |
"html_mb": 7.76,
|
| 896 |
+
"prompt_tokens": 712865,
|
| 897 |
+
"distribution_date": "2025-08-15"
|
| 898 |
},
|
| 899 |
{
|
| 900 |
"id": "fw-nissanlease-2026-a-2026-02",
|
|
|
|
| 909 |
"ungraded_why": null,
|
| 910 |
"docs": 10,
|
| 911 |
"html_mb": 8.48,
|
| 912 |
+
"prompt_tokens": 739161,
|
| 913 |
+
"distribution_date": "2026-03-16"
|
| 914 |
},
|
| 915 |
{
|
| 916 |
"id": "fw-nissanlease-2026-b-2026-07",
|
|
|
|
| 925 |
"ungraded_why": null,
|
| 926 |
"docs": 10,
|
| 927 |
"html_mb": 8.2,
|
| 928 |
+
"prompt_tokens": 732825,
|
| 929 |
+
"distribution_date": "2026-08-17"
|
| 930 |
},
|
| 931 |
{
|
| 932 |
"id": "fw-worldomni-2025-a-2025-01",
|
|
|
|
| 941 |
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
|
| 942 |
"docs": 7,
|
| 943 |
"html_mb": 6.37,
|
| 944 |
+
"prompt_tokens": 614546,
|
| 945 |
+
"distribution_date": "2025-02-18"
|
| 946 |
},
|
| 947 |
{
|
| 948 |
"id": "fw-worldomni-2025-b-2025-05",
|
|
|
|
| 957 |
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
|
| 958 |
"docs": 7,
|
| 959 |
"html_mb": 6.35,
|
| 960 |
+
"prompt_tokens": 624079,
|
| 961 |
+
"distribution_date": "2025-06-16"
|
| 962 |
},
|
| 963 |
{
|
| 964 |
"id": "fw-worldomni-2025-c-2025-08",
|
|
|
|
| 973 |
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
|
| 974 |
"docs": 7,
|
| 975 |
"html_mb": 6.58,
|
| 976 |
+
"prompt_tokens": 629893,
|
| 977 |
+
"distribution_date": "2025-09-15"
|
| 978 |
},
|
| 979 |
{
|
| 980 |
"id": "fw-worldomni-2025-d-2025-10",
|
|
|
|
| 989 |
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
|
| 990 |
"docs": 7,
|
| 991 |
"html_mb": 8.05,
|
| 992 |
+
"prompt_tokens": 638804,
|
| 993 |
+
"distribution_date": "2025-11-17"
|
| 994 |
},
|
| 995 |
{
|
| 996 |
"id": "fw-worldomni-2026-a-2026-02",
|
|
|
|
| 1005 |
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
|
| 1006 |
"docs": 7,
|
| 1007 |
"html_mb": 6.98,
|
| 1008 |
+
"prompt_tokens": 627265,
|
| 1009 |
+
"distribution_date": "2026-03-16"
|
| 1010 |
},
|
| 1011 |
{
|
| 1012 |
"id": "fw-worldomni-2026-b-2026-05",
|
|
|
|
| 1021 |
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
|
| 1022 |
"docs": 7,
|
| 1023 |
"html_mb": 6.35,
|
| 1024 |
+
"prompt_tokens": 633802,
|
| 1025 |
+
"distribution_date": "2026-06-15"
|
| 1026 |
},
|
| 1027 |
{
|
| 1028 |
"id": "fw-worldomni-2026-c-2026-08",
|
|
|
|
| 1037 |
"ungraded_why": "Agreement prorates the first servicing fee without a day count",
|
| 1038 |
"docs": 7,
|
| 1039 |
"html_mb": 7.76,
|
| 1040 |
+
"prompt_tokens": 636492,
|
| 1041 |
+
"distribution_date": "2026-09-15"
|
| 1042 |
}
|
| 1043 |
]
|
| 1044 |
}
|
index.html
CHANGED
|
@@ -65,6 +65,10 @@ h1 { font-family: var(--serif); font-weight: 500; font-size: clamp(40px, 7vw, 56
|
|
| 65 |
.btn:hover { border-color: var(--accent); color: var(--accent-ink); }
|
| 66 |
.btn[aria-pressed="true"] { background: var(--accent-soft); border-color: var(--accent); color: var(--accent-ink); }
|
| 67 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
/* panels */
|
| 69 |
.wide { max-width: 1200px; margin: 0 auto; padding: 0 16px; }
|
| 70 |
.panel { background: var(--panel); border: 1px solid var(--line); }
|
|
@@ -229,8 +233,9 @@ footer .in { max-width: 1200px; margin: 0 auto; padding: 20px 16px 36px; display
|
|
| 229 |
<div class="label">Benchmarks / Finance</div>
|
| 230 |
<h1>WaterfallBench</h1>
|
| 231 |
<div class="meta">
|
| 232 |
-
<span class="label" id="updated">Updated 11/15/2026</span><span class="sep"></span><span class="label">Version 1.0</span><span class="sep"></span><a class="label" href="https://github.com/RITIK-12/WaterfallBench">Code & tasks β</a>
|
| 233 |
</div>
|
|
|
|
| 234 |
</div>
|
| 235 |
|
| 236 |
<div class="wide" id="leaderboard">
|
|
@@ -331,6 +336,7 @@ footer .in { max-width: 1200px; margin: 0 auto; padding: 20px 16px 36px; display
|
|
| 331 |
<section id="dataset"><div class="prose">
|
| 332 |
<h2>Dataset</h2>
|
| 333 |
<p id="ds-lede"></p>
|
|
|
|
| 334 |
<div class="kv" id="kv"></div>
|
| 335 |
<div class="panel tbl-wrap"><table id="families" class="compact"></table></div>
|
| 336 |
<p class="cap">Tokens: mean prompt per task, Claude tokenizer, as sent by the harness. No task needed truncation.</p>
|
|
@@ -606,12 +612,12 @@ function dumbbell() {
|
|
| 606 |
box.innerHTML = ""; box.append(svg);
|
| 607 |
}
|
| 608 |
|
| 609 |
-
const month = p => new Date(+p.slice(0, 4), +p.slice(4, 6) - 1, 1).toLocaleString("en-US", { month: "short", year: "numeric" });
|
| 610 |
|
| 611 |
function grid() {
|
| 612 |
const ms = [...D.models].sort((a, b) => b.fully_resolved - a.fully_resolved || b.nontrivial - a.nontrivial);
|
| 613 |
const head = `<thead><tr><th class="sticky">Deal Β· first distribution</th>${ms.map(m => `<th class="mh">${logo(m.lab, 15)}<div style="margin-top:4px">${esc(m.name)}</div></th>`).join("")}</tr></thead>`;
|
| 614 |
-
const body = D.tasks.map(t => `<tr><td class="deal sticky">${esc(t.deal)}<small>${month(t.
|
| 615 |
${ms.map(m => { const r = m.results[t.id]; if (!r) return `<td class="cell">β</td>`;
|
| 616 |
return `<td class="cell"><span class="${r.ok ? "ok" : r.li >= 85 ? "part" : "low"}" title="${r.ok ? "every graded field exact" : `${r.li.toFixed(0)}% of graded fields exact`}">${r.ok ? "β" : r.li.toFixed(0) + "%"}</span></td>`; }).join("")}</tr>`).join("");
|
| 617 |
document.getElementById("grid").innerHTML = head + `<tbody>${body}</tbody>`;
|
|
@@ -628,13 +634,13 @@ function prose() {
|
|
| 628 |
document.getElementById("db-lede").textContent = `Field accuracy ranges from ${Math.min(...nt).toFixed(0)}% to ${Math.max(...nt).toFixed(0)}%, while the share of tasks fully resolved ranges from ${Math.min(...fr).toFixed(0)}% to ${Math.max(...fr).toFixed(0)}%. A deal resolves only if every graded field is exact, and one wrong definition usually carries into every line below it.`;
|
| 629 |
// dataset
|
| 630 |
const T = D.tasks, nf = T.reduce((a, t) => a + t.graded, 0), fams = [...new Set(T.map(t => t.family))];
|
| 631 |
-
const periods = T.map(t => t.
|
| 632 |
document.getElementById("ds-lede").textContent = `${T.length} tasks from ${fams.length} auto-loan and lease issuers. Each is a trust's first Form 10-D on SEC EDGAR, with first distributions from ${month(periods[0])} to ${month(periods[periods.length - 1])}. Documents were retrieved with Firecrawl and are used as filed; nothing is generated or imputed.`;
|
| 633 |
const docs = T.map(t => t.docs + 1), mb = T.map(t => t.html_mb), tok = T.map(t => t.prompt_tokens).filter(Boolean);
|
| 634 |
document.getElementById("kv").innerHTML = [[T.length, "tasks"], [nf, "graded fields"], [`${Math.min(...docs)}β${Math.max(...docs)}`, "documents / task"],
|
| 635 |
[`${Math.min(...mb).toFixed(0)}β${Math.max(...mb).toFixed(0)} MB`, "HTML / task"], [`${Math.round(Math.min(...tok) / 1000)}β${Math.round(Math.max(...tok) / 1000)}k`, "prompt tokens / task"]]
|
| 636 |
.map(([v, k]) => `<div><b>${v}</b><span class="label">${k}</span></div>`).join("");
|
| 637 |
-
const fr2 = fams.map(f => { const ts = T.filter(t => t.family === f), ps = ts.map(t => t.
|
| 638 |
return `<tr><td>${esc(ts[0].family_name)}</td><td class="num v">${ts.length}</td><td class="num v">${month(ps[0])} β ${month(ps[ps.length - 1])}</td><td class="num v">${Math.min(...cl) === Math.max(...cl) ? cl[0] : Math.min(...cl) + "β" + Math.max(...cl)}</td><td class="num v">${Math.round(tk.reduce((a, b) => a + b, 0) / tk.length / 1000)}k</td><td class="num v">${ts.reduce((a, t) => a + t.graded, 0)}</td></tr>`; }).join("");
|
| 639 |
document.getElementById("families").innerHTML = `<thead><tr><th>Issuer</th><th class="num">Deals</th><th class="num">First distributions</th><th class="num">Classes</th><th class="num">Tokens</th><th class="num">Fields</th></tr></thead><tbody>${fr2}</tbody>`;
|
| 640 |
const nu = T.reduce((a, t) => a + t.ungraded, 0), tu = T.filter(t => t.ungraded).length;
|
|
|
|
| 65 |
.btn:hover { border-color: var(--accent); color: var(--accent-ink); }
|
| 66 |
.btn[aria-pressed="true"] { background: var(--accent-soft); border-color: var(--accent); color: var(--accent-ink); }
|
| 67 |
|
| 68 |
+
.cta { margin-top: 16px; }
|
| 69 |
+
.hf-badge { display: inline-flex; align-items: center; gap: 9px; padding: 7px 13px 7px 10px; border: 1px solid var(--line-2); background: var(--panel); color: var(--ink); font-family: var(--sans); font-size: 14px; text-decoration: none; }
|
| 70 |
+
.hf-badge:hover { border-color: #FFD21E; background: color-mix(in srgb, #FFD21E 10%, var(--panel)); }
|
| 71 |
+
.hf-badge b { font-weight: 600; }
|
| 72 |
/* panels */
|
| 73 |
.wide { max-width: 1200px; margin: 0 auto; padding: 0 16px; }
|
| 74 |
.panel { background: var(--panel); border: 1px solid var(--line); }
|
|
|
|
| 233 |
<div class="label">Benchmarks / Finance</div>
|
| 234 |
<h1>WaterfallBench</h1>
|
| 235 |
<div class="meta">
|
| 236 |
+
<span class="label" id="updated">Updated 11/15/2026</span><span class="sep"></span><span class="label">Version 1.0</span><span class="sep"></span><a class="label" href="https://github.com/RITIK-12/WaterfallBench">Code & tasks β</a>
|
| 237 |
</div>
|
| 238 |
+
<div class="cta"><a class="hf-badge" href="https://huggingface.co/datasets/ritik12/WaterfallBench"><svg class="hf-logo" viewBox="0 0 24 24" width="20" height="20" aria-hidden="true"><path d="M2.25 11.535c0-3.407 1.847-6.554 4.844-8.258a9.822 9.822 0 019.687 0c2.997 1.704 4.844 4.851 4.844 8.258 0 5.266-4.337 9.535-9.687 9.535S2.25 16.8 2.25 11.535z" fill="#FF9D0B"></path><path d="M11.938 20.086c4.797 0 8.687-3.829 8.687-8.551 0-4.722-3.89-8.55-8.687-8.55-4.798 0-8.688 3.828-8.688 8.55 0 4.722 3.89 8.55 8.688 8.55z" fill="#FFD21E"></path><path d="M11.875 15.113c2.457 0 3.25-2.156 3.25-3.263 0-.576-.393-.394-1.023-.089-.582.283-1.365.675-2.224.675-1.798 0-3.25-1.693-3.25-.586 0 1.107.79 3.263 3.25 3.263h-.003z" fill="#FF323D"></path><path d="M14.76 9.21c.32.108.445.753.767.585.447-.233.707-.708.659-1.204a1.235 1.235 0 00-.879-1.059 1.262 1.262 0 00-1.33.394c-.322.384-.377.92-.14 1.36.153.283.638-.177.925-.079l-.002.003zm-5.887 0c-.32.108-.448.753-.768.585a1.226 1.226 0 01-.658-1.204c.048-.495.395-.913.878-1.059a1.262 1.262 0 011.33.394c.322.384.377.92.14 1.36-.152.283-.64-.177-.925-.079l.003.003zm1.12 5.34a2.166 2.166 0 011.325-1.106c.07-.02.144.06.219.171l.192.306c.069.1.139.175.209.175.074 0 .15-.074.223-.172l.205-.302c.08-.11.157-.188.234-.165.537.168.986.536 1.25 1.026.932-.724 1.275-1.905 1.275-2.633 0-.508-.306-.426-.81-.19l-.616.296c-.52.24-1.148.48-1.824.48-.676 0-1.302-.24-1.823-.48l-.589-.283c-.52-.248-.838-.342-.838.177 0 .703.32 1.831 1.187 2.56l.18.14z" fill="#3A3B45"></path><path d="M17.812 10.366a.806.806 0 00.813-.8c0-.441-.364-.8-.813-.8a.806.806 0 00-.812.8c0 .442.364.8.812.8zm-11.624 0a.806.806 0 00.812-.8c0-.441-.364-.8-.812-.8a.806.806 0 00-.813.8c0 .442.364.8.813.8zM4.515 13.073c-.405 0-.765.162-1.017.46a1.455 1.455 0 00-.333.925 1.801 1.801 0 00-.485-.074c-.387 0-.737.146-.985.409a1.41 1.41 0 00-.2 1.722 1.302 1.302 0 00-.447.694c-.06.222-.12.69.2 1.166a1.267 1.267 0 00-.093 1.236c.238.533.81.958 1.89 1.405l.24.096c.768.3 1.473.492 1.478.494.89.243 1.808.375 2.732.394 1.465 0 2.513-.443 3.115-1.314.93-1.342.842-2.575-.274-3.763l-.151-.154c-.692-.684-1.155-1.69-1.25-1.912-.195-.655-.71-1.383-1.562-1.383-.46.007-.889.233-1.15.605-.25-.31-.495-.553-.715-.694a1.87 1.87 0 00-.993-.312zm14.97 0c.405 0 .767.162 1.017.46.216.262.333.588.333.925.158-.047.322-.071.487-.074.388 0 .738.146.985.409a1.41 1.41 0 01.2 1.722c.22.178.377.422.445.694.06.222.12.69-.2 1.166.244.37.279.836.093 1.236-.238.533-.81.958-1.889 1.405l-.239.096c-.77.3-1.475.492-1.48.494-.89.243-1.808.375-2.732.394-1.465 0-2.513-.443-3.115-1.314-.93-1.342-.842-2.575.274-3.763l.151-.154c.695-.684 1.157-1.69 1.252-1.912.195-.655.708-1.383 1.56-1.383.46.007.889.233 1.15.605.25-.31.495-.553.718-.694.244-.162.523-.265.814-.3l.176-.012z" fill="#FF9D0B"></path><path d="M9.785 20.132c.688-.994.638-1.74-.305-2.667-.945-.928-1.495-2.288-1.495-2.288s-.205-.788-.672-.714c-.468.074-.81 1.25.17 1.971.977.721-.195 1.21-.573.534-.375-.677-1.405-2.416-1.94-2.751-.532-.332-.907-.148-.782.541.125.687 2.357 2.35 2.14 2.707-.218.362-.983-.42-.983-.42S2.953 14.9 2.43 15.46c-.52.558.398 1.026 1.7 1.803 1.308.778 1.41.985 1.225 1.28-.187.295-3.07-2.1-3.34-1.083-.27 1.011 2.943 1.304 2.745 2.006-.2.7-2.265-1.324-2.685-.537-.425.79 2.913 1.718 2.94 1.725 1.075.276 3.813.859 4.77-.522zm4.432 0c-.687-.994-.64-1.74.305-2.667.943-.928 1.493-2.288 1.493-2.288s.205-.788.675-.714c.465.074.807 1.25-.17 1.971-.98.721.195 1.21.57.534.377-.677 1.407-2.416 1.94-2.751.532-.332.91-.148.782.541-.125.687-2.355 2.35-2.137 2.707.215.362.98-.42.98-.42S21.05 14.9 21.57 15.46c.52.558-.395 1.026-1.7 1.803-1.308.778-1.408.985-1.225 1.28.187.295 3.07-2.1 3.34-1.083.27 1.011-2.94 1.304-2.743 2.006.2.7 2.263-1.324 2.685-.537.423.79-2.912 1.718-2.94 1.725-1.077.276-3.815.859-4.77-.522z" fill="#FFD21E"></path></svg><span>Available on <b>Hugging Face</b></span></a></div>
|
| 239 |
</div>
|
| 240 |
|
| 241 |
<div class="wide" id="leaderboard">
|
|
|
|
| 336 |
<section id="dataset"><div class="prose">
|
| 337 |
<h2>Dataset</h2>
|
| 338 |
<p id="ds-lede"></p>
|
| 339 |
+
<p class="cta" style="margin:-4px 0 18px"><a class="hf-badge" href="https://huggingface.co/datasets/ritik12/WaterfallBench"><svg class="hf-logo" viewBox="0 0 24 24" width="20" height="20" aria-hidden="true"><path d="M2.25 11.535c0-3.407 1.847-6.554 4.844-8.258a9.822 9.822 0 019.687 0c2.997 1.704 4.844 4.851 4.844 8.258 0 5.266-4.337 9.535-9.687 9.535S2.25 16.8 2.25 11.535z" fill="#FF9D0B"></path><path d="M11.938 20.086c4.797 0 8.687-3.829 8.687-8.551 0-4.722-3.89-8.55-8.687-8.55-4.798 0-8.688 3.828-8.688 8.55 0 4.722 3.89 8.55 8.688 8.55z" fill="#FFD21E"></path><path d="M11.875 15.113c2.457 0 3.25-2.156 3.25-3.263 0-.576-.393-.394-1.023-.089-.582.283-1.365.675-2.224.675-1.798 0-3.25-1.693-3.25-.586 0 1.107.79 3.263 3.25 3.263h-.003z" fill="#FF323D"></path><path d="M14.76 9.21c.32.108.445.753.767.585.447-.233.707-.708.659-1.204a1.235 1.235 0 00-.879-1.059 1.262 1.262 0 00-1.33.394c-.322.384-.377.92-.14 1.36.153.283.638-.177.925-.079l-.002.003zm-5.887 0c-.32.108-.448.753-.768.585a1.226 1.226 0 01-.658-1.204c.048-.495.395-.913.878-1.059a1.262 1.262 0 011.33.394c.322.384.377.92.14 1.36-.152.283-.64-.177-.925-.079l.003.003zm1.12 5.34a2.166 2.166 0 011.325-1.106c.07-.02.144.06.219.171l.192.306c.069.1.139.175.209.175.074 0 .15-.074.223-.172l.205-.302c.08-.11.157-.188.234-.165.537.168.986.536 1.25 1.026.932-.724 1.275-1.905 1.275-2.633 0-.508-.306-.426-.81-.19l-.616.296c-.52.24-1.148.48-1.824.48-.676 0-1.302-.24-1.823-.48l-.589-.283c-.52-.248-.838-.342-.838.177 0 .703.32 1.831 1.187 2.56l.18.14z" fill="#3A3B45"></path><path d="M17.812 10.366a.806.806 0 00.813-.8c0-.441-.364-.8-.813-.8a.806.806 0 00-.812.8c0 .442.364.8.812.8zm-11.624 0a.806.806 0 00.812-.8c0-.441-.364-.8-.812-.8a.806.806 0 00-.813.8c0 .442.364.8.813.8zM4.515 13.073c-.405 0-.765.162-1.017.46a1.455 1.455 0 00-.333.925 1.801 1.801 0 00-.485-.074c-.387 0-.737.146-.985.409a1.41 1.41 0 00-.2 1.722 1.302 1.302 0 00-.447.694c-.06.222-.12.69.2 1.166a1.267 1.267 0 00-.093 1.236c.238.533.81.958 1.89 1.405l.24.096c.768.3 1.473.492 1.478.494.89.243 1.808.375 2.732.394 1.465 0 2.513-.443 3.115-1.314.93-1.342.842-2.575-.274-3.763l-.151-.154c-.692-.684-1.155-1.69-1.25-1.912-.195-.655-.71-1.383-1.562-1.383-.46.007-.889.233-1.15.605-.25-.31-.495-.553-.715-.694a1.87 1.87 0 00-.993-.312zm14.97 0c.405 0 .767.162 1.017.46.216.262.333.588.333.925.158-.047.322-.071.487-.074.388 0 .738.146.985.409a1.41 1.41 0 01.2 1.722c.22.178.377.422.445.694.06.222.12.69-.2 1.166.244.37.279.836.093 1.236-.238.533-.81.958-1.889 1.405l-.239.096c-.77.3-1.475.492-1.48.494-.89.243-1.808.375-2.732.394-1.465 0-2.513-.443-3.115-1.314-.93-1.342-.842-2.575.274-3.763l.151-.154c.695-.684 1.157-1.69 1.252-1.912.195-.655.708-1.383 1.56-1.383.46.007.889.233 1.15.605.25-.31.495-.553.718-.694.244-.162.523-.265.814-.3l.176-.012z" fill="#FF9D0B"></path><path d="M9.785 20.132c.688-.994.638-1.74-.305-2.667-.945-.928-1.495-2.288-1.495-2.288s-.205-.788-.672-.714c-.468.074-.81 1.25.17 1.971.977.721-.195 1.21-.573.534-.375-.677-1.405-2.416-1.94-2.751-.532-.332-.907-.148-.782.541.125.687 2.357 2.35 2.14 2.707-.218.362-.983-.42-.983-.42S2.953 14.9 2.43 15.46c-.52.558.398 1.026 1.7 1.803 1.308.778 1.41.985 1.225 1.28-.187.295-3.07-2.1-3.34-1.083-.27 1.011 2.943 1.304 2.745 2.006-.2.7-2.265-1.324-2.685-.537-.425.79 2.913 1.718 2.94 1.725 1.075.276 3.813.859 4.77-.522zm4.432 0c-.687-.994-.64-1.74.305-2.667.943-.928 1.493-2.288 1.493-2.288s.205-.788.675-.714c.465.074.807 1.25-.17 1.971-.98.721.195 1.21.57.534.377-.677 1.407-2.416 1.94-2.751.532-.332.91-.148.782.541-.125.687-2.355 2.35-2.137 2.707.215.362.98-.42.98-.42S21.05 14.9 21.57 15.46c.52.558-.395 1.026-1.7 1.803-1.308.778-1.408.985-1.225 1.28.187.295 3.07-2.1 3.34-1.083.27 1.011-2.94 1.304-2.743 2.006.2.7 2.263-1.324 2.685-.537.423.79-2.912 1.718-2.94 1.725-1.077.276-3.815.859-4.77-.522z" fill="#FFD21E"></path></svg><span>Download from <b>Hugging Face</b></span></a></p>
|
| 340 |
<div class="kv" id="kv"></div>
|
| 341 |
<div class="panel tbl-wrap"><table id="families" class="compact"></table></div>
|
| 342 |
<p class="cap">Tokens: mean prompt per task, Claude tokenizer, as sent by the harness. No task needed truncation.</p>
|
|
|
|
| 612 |
box.innerHTML = ""; box.append(svg);
|
| 613 |
}
|
| 614 |
|
| 615 |
+
const month = p => { p = p.replace(/-/g, ""); return new Date(+p.slice(0, 4), +p.slice(4, 6) - 1, 1).toLocaleString("en-US", { month: "short", year: "numeric" }); };
|
| 616 |
|
| 617 |
function grid() {
|
| 618 |
const ms = [...D.models].sort((a, b) => b.fully_resolved - a.fully_resolved || b.nontrivial - a.nontrivial);
|
| 619 |
const head = `<thead><tr><th class="sticky">Deal Β· first distribution</th>${ms.map(m => `<th class="mh">${logo(m.lab, 15)}<div style="margin-top:4px">${esc(m.name)}</div></th>`).join("")}</tr></thead>`;
|
| 620 |
+
const body = D.tasks.map(t => `<tr><td class="deal sticky">${esc(t.deal)}<small>${month(t.distribution_date)} Β· ${t.classes} classes Β· ${t.graded} fields</small></td>
|
| 621 |
${ms.map(m => { const r = m.results[t.id]; if (!r) return `<td class="cell">β</td>`;
|
| 622 |
return `<td class="cell"><span class="${r.ok ? "ok" : r.li >= 85 ? "part" : "low"}" title="${r.ok ? "every graded field exact" : `${r.li.toFixed(0)}% of graded fields exact`}">${r.ok ? "β" : r.li.toFixed(0) + "%"}</span></td>`; }).join("")}</tr>`).join("");
|
| 623 |
document.getElementById("grid").innerHTML = head + `<tbody>${body}</tbody>`;
|
|
|
|
| 634 |
document.getElementById("db-lede").textContent = `Field accuracy ranges from ${Math.min(...nt).toFixed(0)}% to ${Math.max(...nt).toFixed(0)}%, while the share of tasks fully resolved ranges from ${Math.min(...fr).toFixed(0)}% to ${Math.max(...fr).toFixed(0)}%. A deal resolves only if every graded field is exact, and one wrong definition usually carries into every line below it.`;
|
| 635 |
// dataset
|
| 636 |
const T = D.tasks, nf = T.reduce((a, t) => a + t.graded, 0), fams = [...new Set(T.map(t => t.family))];
|
| 637 |
+
const periods = T.map(t => t.distribution_date).sort();
|
| 638 |
document.getElementById("ds-lede").textContent = `${T.length} tasks from ${fams.length} auto-loan and lease issuers. Each is a trust's first Form 10-D on SEC EDGAR, with first distributions from ${month(periods[0])} to ${month(periods[periods.length - 1])}. Documents were retrieved with Firecrawl and are used as filed; nothing is generated or imputed.`;
|
| 639 |
const docs = T.map(t => t.docs + 1), mb = T.map(t => t.html_mb), tok = T.map(t => t.prompt_tokens).filter(Boolean);
|
| 640 |
document.getElementById("kv").innerHTML = [[T.length, "tasks"], [nf, "graded fields"], [`${Math.min(...docs)}β${Math.max(...docs)}`, "documents / task"],
|
| 641 |
[`${Math.min(...mb).toFixed(0)}β${Math.max(...mb).toFixed(0)} MB`, "HTML / task"], [`${Math.round(Math.min(...tok) / 1000)}β${Math.round(Math.max(...tok) / 1000)}k`, "prompt tokens / task"]]
|
| 642 |
.map(([v, k]) => `<div><b>${v}</b><span class="label">${k}</span></div>`).join("");
|
| 643 |
+
const fr2 = fams.map(f => { const ts = T.filter(t => t.family === f), ps = ts.map(t => t.distribution_date).sort(), cl = ts.map(t => t.classes), tk = ts.map(t => t.prompt_tokens).filter(Boolean);
|
| 644 |
return `<tr><td>${esc(ts[0].family_name)}</td><td class="num v">${ts.length}</td><td class="num v">${month(ps[0])} β ${month(ps[ps.length - 1])}</td><td class="num v">${Math.min(...cl) === Math.max(...cl) ? cl[0] : Math.min(...cl) + "β" + Math.max(...cl)}</td><td class="num v">${Math.round(tk.reduce((a, b) => a + b, 0) / tk.length / 1000)}k</td><td class="num v">${ts.reduce((a, t) => a + t.graded, 0)}</td></tr>`; }).join("");
|
| 645 |
document.getElementById("families").innerHTML = `<thead><tr><th>Issuer</th><th class="num">Deals</th><th class="num">First distributions</th><th class="num">Classes</th><th class="num">Tokens</th><th class="num">Fields</th></tr></thead><tbody>${fr2}</tbody>`;
|
| 646 |
const nu = T.reduce((a, t) => a + t.ungraded, 0), tu = T.filter(t => t.ungraded).length;
|
leaderboard.png
ADDED
|
Git LFS Details
|