jiminHuang commited on
Commit
42d3c8f
·
verified ·
1 Parent(s): d80fba2

Add FinDendro leaderboard

Browse files
Files changed (6) hide show
  1. README.md +12 -4
  2. app.js +192 -0
  3. assets/finai-logo.png +0 -0
  4. data.js +1 -0
  5. index.html +128 -17
  6. style.css +128 -22
README.md CHANGED
@@ -1,10 +1,18 @@
1
  ---
2
  title: FinDendro
3
- emoji: 🔥
4
- colorFrom: green
5
- colorTo: pink
6
  sdk: static
 
7
  pinned: false
 
8
  ---
9
 
10
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
1
  ---
2
  title: FinDendro
3
+ emoji: 🌳
4
+ colorFrom: blue
5
+ colorTo: indigo
6
  sdk: static
7
+ app_file: index.html
8
  pinned: false
9
+ short_description: Leaderboard for temporal self-knowledge of LLMs in finance
10
  ---
11
 
12
+ # FinDendro
13
+
14
+ Interactive leaderboard for FinDendro, which measures what LLMs know about dated U.S. financial facts
15
+ from 1999 to 2026 and what they say they know.
16
+
17
+ - `index.html`, `style.css`, `app.js`: the page
18
+ - `data.js`: generated by `leaderboard/build_data.py` from the paper tables and raw results
app.js ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /* FinDendro leaderboard: every number comes from data.js. */
2
+ const D = window.FD, M = D.models, CH = D.meta.chance;
3
+ const $ = (s, el = document) => el.querySelector(s);
4
+ const f2 = v => v.toFixed(2).replace('-', '−');
5
+ const f3 = v => (v > 0 ? '+' : '') + v.toFixed(3).replace('-', '−');
6
+ const pct = v => Math.round(v * 100) + '%';
7
+ const by = name => M.find(m => m.model === name);
8
+ const quarters = (a, b) => { const q = s => +s.slice(0, 4) * 4 + +s.slice(5); return q(b) - q(a); };
9
+ const NS = 'http://www.w3.org/2000/svg';
10
+ const KNOWS = 'var(--knows)', SAYS = 'var(--says)', CHANCE = 'var(--chance)';
11
+
12
+ /* tooltip: any element with data-tip */
13
+ const tip = $('#tip');
14
+ const showTip = (html, e) => {
15
+ tip.innerHTML = html; tip.classList.add('show');
16
+ const w = tip.offsetWidth, h = tip.offsetHeight;
17
+ tip.style.transform = `translate(${Math.max(8, Math.min(innerWidth - w - 8, e.clientX + 12))}px,${e.clientY - h - 12 < 8 ? e.clientY + 16 : e.clientY - h - 12}px)`;
18
+ };
19
+ const hideTip = () => tip.classList.remove('show');
20
+ document.addEventListener('pointermove', e => {
21
+ const t = e.target.closest && e.target.closest('[data-tip]');
22
+ if (t) showTip(t.dataset.tip, e); else if (!e.target.closest || !e.target.closest('.chart')) hideTip();
23
+ });
24
+
25
+ function svg(w, h, label) {
26
+ const s = document.createElementNS(NS, 'svg');
27
+ s.setAttribute('viewBox', `0 0 ${w} ${h}`); s.setAttribute('width', '100%'); s.setAttribute('role', 'img'); s.setAttribute('aria-label', label);
28
+ s.style.display = 'block';
29
+ return s;
30
+ }
31
+ function add(p, tag, attrs, text) {
32
+ const e = document.createElementNS(NS, tag);
33
+ for (const k in attrs) e.setAttribute(k, attrs[k]);
34
+ if (text != null) e.textContent = text;
35
+ p.appendChild(e); return e;
36
+ }
37
+
38
+ /* hero background: one ring per year, darker where GPT-5 was more accurate */
39
+ (function rings() {
40
+ const s = $('#rings'), years = {};
41
+ D.timeline['GPT-5'].forEach(r => { const y = Math.floor(r.x); (years[y] = years[y] || []).push(r.acc); });
42
+ Object.keys(years).sort().forEach((y, i, all) => {
43
+ const a = years[y].reduce((p, q) => p + q, 0) / years[y].length;
44
+ add(s, 'circle', { cx: 0, cy: 0, r: 40 + i * 10.4, fill: 'none', stroke: '#1f4fb8', 'stroke-width': 1 + a * 3.4, 'stroke-opacity': (0.03 + a * a * 0.2).toFixed(3) });
45
+ });
46
+ })();
47
+
48
+ $('#stats').innerHTML = [
49
+ [D.meta.facts.toLocaleString('en-US'), 'dated facts'],
50
+ [D.meta.prompts.toLocaleString('en-US'), 'prompts per model'],
51
+ [`${D.meta.from}–${D.meta.to}`, 'calendar span'],
52
+ [M.length + D.pit.length, 'models and vintages'],
53
+ ].map(([b, s]) => `<div class="stat"><b>${b}</b><span>${s}</span></div>`).join('');
54
+
55
+ /* horizontal dumbbell on a shared 0..1 axis; values sit in a column at the right so labels never meet */
56
+ function dumbbell(el, rows, opts = {}) {
57
+ const W = opts.W || 520, L = opts.L || 112, R = 92, T = 8, rowH = opts.rowH || 34, H = T + rows.length * rowH + 24, sep = opts.sep || ' / ';
58
+ const s = svg(W, H, opts.label || 'Knows and says it knows by model');
59
+ const x = v => L + v * (W - L - R);
60
+ [0, .25, .5, .75, 1].forEach(v => {
61
+ add(s, 'line', { x1: x(v), x2: x(v), y1: T, y2: H - 22, stroke: 'rgba(24,56,152,.09)' });
62
+ add(s, 'text', { x: x(v), y: H - 6, 'text-anchor': 'middle', 'font-size': 11, fill: 'var(--soft)' }, v.toFixed(2));
63
+ });
64
+ if (!opts.noChance) add(s, 'line', { x1: x(CH), x2: x(CH), y1: T, y2: H - 22, stroke: CHANCE, 'stroke-width': 1.4, 'stroke-dasharray': '3 3' });
65
+ rows.forEach((r, i) => {
66
+ const y = T + rowH * (i + .5);
67
+ add(s, 'text', { x: L - 12, y: y + 4, 'text-anchor': 'end', 'font-size': 12.5, fill: 'var(--ink)' }, r.label);
68
+ add(s, 'line', { x1: x(r.a), x2: x(r.b), y1: y, y2: y, stroke: 'rgba(15,21,40,.22)', 'stroke-width': 2 });
69
+ [[r.a, KNOWS], [r.b, SAYS]].forEach(([v, c]) => add(s, 'circle', { cx: x(v), cy: y, r: 6.5, style: `fill:${c};stroke:#fff;stroke-width:2` }));
70
+ add(s, 'text', { x: W, y: y + 4, 'text-anchor': 'end', 'font-size': 12, fill: 'var(--soft)', style: 'font-variant-numeric:tabular-nums' }, f2(r.a) + sep + f2(r.b));
71
+ const hit = add(s, 'rect', { x: L - 4, y: y - rowH / 2, width: W - L + 4, height: rowH, fill: 'transparent' });
72
+ hit.dataset.tip = `<b>${r.label}</b><br>${r.aName || 'Knows'} ${f2(r.a)}<br>${r.bName || 'Says it knows'} ${f2(r.b)}`;
73
+ });
74
+ el.replaceChildren(s);
75
+ }
76
+ dumbbell($('#dumbbell'), M.map(m => ({ label: m.model, a: m.knows, b: m.says })));
77
+
78
+ /* leaderboard */
79
+ const COLS = [
80
+ ['model', 'Model', 'left'], ['knows', 'Knows'], ['says', 'Says'], ['acc_says', 'Acc | says'], ['acc_denies', 'Acc | denies'],
81
+ ['gap', 'Gap [95% CI]'], ['know_end', 'Knowledge ends'], ['claim_end', 'Claims end'], ['self_report', 'Self-report'], ['vendor_cutoff', 'Vendor'],
82
+ ];
83
+ let sortK = 'knows', sortDir = -1;
84
+ function renderLB() {
85
+ const rows = [...M].sort((a, b) => {
86
+ const x = a[sortK], y = b[sortK];
87
+ if (x == null && y == null) return 0;
88
+ if (x == null) return 1;
89
+ if (y == null) return -1;
90
+ return (x < y ? -1 : x > y ? 1 : 0) * sortDir;
91
+ });
92
+ const head = `<thead><tr><th class="left">#</th>${COLS.map(([k, t, c]) =>
93
+ `<th data-k="${k}" tabindex="0" role="button" class="${c || ''}${k === sortK ? ' on' : ''}"${k === sortK ? ` aria-sort="${sortDir < 0 ? 'descending' : 'ascending'}"` : ''}>${t}${k === sortK ? (sortDir < 0 ? ' ↓' : ' ↑') : ''}</th>`).join('')}</tr></thead>`;
94
+ const na = '<span class="na">n/a</span>';
95
+ const body = rows.map((m, i) => {
96
+ const lag = m.know_end && quarters(m.know_end, m.claim_end) > 1;
97
+ return `<tr><td class="left rank">${i + 1}</td>
98
+ <td class="left"><span class="mname">${m.model}</span><span class="vendor">${m.vendor}</span>${m.small ? '<span class="chip">small</span>' : ''}</td>
99
+ <td><span class="bar"><i><b style="width:${m.knows * 100}%"></b><u style="left:${CH * 100}%"></u></i>${f2(m.knows)}</span></td>
100
+ <td>${f2(m.says)}</td><td>${f2(m.acc_says)}</td><td>${f2(m.acc_denies)}</td>
101
+ <td>${f2(m.gap)}<span class="ci">[${f2(m.gap_lo)}, ${f2(m.gap_hi)}]</span></td>
102
+ <td>${m.know_end || na}</td><td${lag ? ' class="flag"' : ''}>${m.claim_end || na}</td>
103
+ <td>${m.self_report || na}</td><td>${m.vendor_cutoff || na}</td></tr>`;
104
+ }).join('');
105
+ const t = $('#lb');
106
+ t.innerHTML = head + `<tbody>${body}</tbody>`;
107
+ t.querySelectorAll('th[data-k]').forEach(th => {
108
+ const go = () => { const k = th.dataset.k; if (k === sortK) sortDir = -sortDir; else { sortK = k; sortDir = k === 'model' ? 1 : -1; } renderLB(); };
109
+ th.onclick = go;
110
+ th.onkeydown = e => { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); go(); } };
111
+ });
112
+ }
113
+ renderLB();
114
+
115
+ /* findings, written from the data */
116
+ (function findings() {
117
+ const g = by('GPT-5'), l = by('Llama-3.3-70B');
118
+ const items = [
119
+ [`${f2(g.knows)} known, ${f2(g.says)} claimed`, `GPT-5 answers ${pct(g.knows)} of controlled facts correctly and says it knows ${pct(g.says)} of them.`],
120
+ [`${f2(g.acc_denies)} on denied facts`, `When GPT-5 says it does not know a value, it still picks the right one ${pct(g.acc_denies)} of the time. Chance is about ${pct(CH)}.`],
121
+ [`${quarters(l.know_end, l.claim_end)} quarters of empty claims`, `Llama-3.3-70B stops knowing after ${l.know_end} and keeps claiming knowledge until ${l.claim_end}.`],
122
+ ];
123
+ $('#findings').innerHTML = items.map(([n, t]) => `<div class="card"><div class="num">${n}</div><p>${t}</p></div>`).join('');
124
+ })();
125
+
126
+ /* timeline small multiples */
127
+ function ymX(s) { const [y, m] = s.split('-').map(Number); return y + (m - .5) / 12; }
128
+ function qX(s) { return +s.slice(0, 4) + (+s.slice(5) - .5) / 4; }
129
+ function drawPanel(el, m) {
130
+ const W = Math.max(320, el.clientWidth || 520), H = 230, L = 34, R = 12, T = 10, B = 26, x0 = 2017, x1 = 2026.9;
131
+ const pts = D.timeline[m.model].filter(r => r.x >= x0);
132
+ const s = svg(W, H, `${m.model}: accuracy and claim rate from ${x0}. Knowledge ends ${m.know_end}, claims end ${m.claim_end}.`);
133
+ const x = v => L + (v - x0) / (x1 - x0) * (W - L - R), y = v => T + (1 - v) * (H - T - B);
134
+ [0, .25, .5, .75, 1].forEach(v => {
135
+ add(s, 'line', { x1: L, x2: W - R, y1: y(v), y2: y(v), stroke: 'rgba(24,56,152,.09)' });
136
+ add(s, 'text', { x: L - 6, y: y(v) + 4, 'text-anchor': 'end', 'font-size': 10.5, fill: 'var(--soft)' }, v.toFixed(2).replace('0.', '.').replace('1.00', '1'));
137
+ });
138
+ for (let yr = 2018; yr <= 2026; yr += W < 420 ? 2 : 1)
139
+ add(s, 'text', { x: x(yr + .5), y: H - 8, 'text-anchor': 'middle', 'font-size': 10.5, fill: 'var(--soft)' }, W < 420 ? yr : `'${String(yr).slice(2)}`);
140
+ add(s, 'line', { x1: L, x2: W - R, y1: y(CH), y2: y(CH), stroke: CHANCE, 'stroke-width': 1.3, 'stroke-dasharray': '3 3' });
141
+ const vline = (v, style, dash) => add(s, 'line', { x1: x(v), x2: x(v), y1: T, y2: H - B, style, 'stroke-width': 2, ...(dash ? { 'stroke-dasharray': dash, 'stroke-linecap': 'round' } : {}) });
142
+ if (m.vendor_cutoff) vline(ymX(m.vendor_cutoff), 'stroke:var(--vendor)', '0.1 4');
143
+ vline(qX(m.know_end), 'stroke:var(--knows);opacity:.55');
144
+ // equal boundaries would overlap exactly, so claims end is drawn dashed on top
145
+ vline(qX(m.claim_end), 'stroke:var(--says);opacity:.8', m.know_end === m.claim_end ? '5 5' : null);
146
+ const path = key => pts.map((r, i) => `${i ? 'L' : 'M'}${x(r.x).toFixed(1)},${y(r[key]).toFixed(1)}`).join('');
147
+ add(s, 'path', { d: path('claim'), fill: 'none', style: `stroke:${SAYS}`, 'stroke-width': 2, 'stroke-linejoin': 'round' });
148
+ add(s, 'path', { d: path('acc'), fill: 'none', style: `stroke:${KNOWS}`, 'stroke-width': 2, 'stroke-linejoin': 'round' });
149
+ const cross = add(s, 'line', { y1: T, y2: H - B, stroke: 'rgba(15,21,40,.35)', visibility: 'hidden' });
150
+ const dA = add(s, 'circle', { r: 4.5, style: `fill:${KNOWS};stroke:#fff;stroke-width:1.5`, visibility: 'hidden' });
151
+ const dC = add(s, 'circle', { r: 4.5, style: `fill:${SAYS};stroke:#fff;stroke-width:1.5`, visibility: 'hidden' });
152
+ el.replaceChildren(s);
153
+ const hide = () => { [cross, dA, dC].forEach(n => n.setAttribute('visibility', 'hidden')); hideTip(); };
154
+ s.addEventListener('pointermove', e => {
155
+ const b = s.getBoundingClientRect(), mx = (e.clientX - b.left) / b.width * W;
156
+ let r = pts[0]; pts.forEach(p => { if (Math.abs(x(p.x) - mx) < Math.abs(x(r.x) - mx)) r = p; });
157
+ cross.setAttribute('x1', x(r.x)); cross.setAttribute('x2', x(r.x));
158
+ dA.setAttribute('cx', x(r.x)); dA.setAttribute('cy', y(r.acc)); dC.setAttribute('cx', x(r.x)); dC.setAttribute('cy', y(r.claim));
159
+ [cross, dA, dC].forEach(n => n.setAttribute('visibility', 'visible'));
160
+ showTip(`<b>${r.p}</b> · n=${r.n}<br>Accuracy ${f2(r.acc)}<br>Claim rate ${f2(r.claim)}`, e);
161
+ });
162
+ s.addEventListener('pointerleave', hide);
163
+ }
164
+ const panels = M.filter(m => m.know_end);
165
+ $('#multiples').innerHTML = panels.map((m, i) => `<article class="card panel"><h3>${m.model}</h3>
166
+ <div class="ends">Knowledge ends <b>${m.know_end}</b> · Claims end <b>${m.claim_end}</b> · Self-report <b>${m.self_report}</b> · Vendor <b>${m.vendor_cutoff || 'n/a'}</b></div>
167
+ <div class="chart" id="chart${i}"></div></article>`).join('');
168
+ const drawAll = () => panels.forEach((m, i) => drawPanel($('#chart' + i), m));
169
+
170
+ /* fact families */
171
+ const FAMS = Object.keys(D.family['GPT-5']);
172
+ $('#fam').innerHTML = FAMS.map((f, i) => `<div class="card"><h3>${f}</h3><div id="fam${i}"></div></div>`).join('');
173
+ FAMS.forEach((f, i) => dumbbell($('#fam' + i), Object.keys(D.family).map(m => ({ label: m, a: D.family[m][f].know, b: D.family[m][f].says })),
174
+ { W: 400, L: 104, rowH: 36, noChance: true, label: `${f}: knows and says it knows by model` }));
175
+
176
+ /* declared date */
177
+ $('#decl').innerHTML = `<div class="card"><h3>Claims of knowledge</h3><div class="legend"><span><i class="dot dot--knows"></i>No date</span><span><i class="dot dot--says"></i>Date declared</span></div><div id="declC"></div></div>
178
+ <div class="card"><h3>Accuracy on the value</h3><div class="legend"><span><i class="dot dot--knows"></i>No date</span><span><i class="dot dot--says"></i>Date declared</span></div><div id="declA"></div></div>`;
179
+ dumbbell($('#declC'), M.map(m => ({ label: m.model, a: m.claims, b: m.claims_declared, aName: 'No date', bName: 'Date declared' })), { W: 520, sep: ' → ', noChance: true, label: 'Claims of knowledge without and with a declared date' });
180
+ dumbbell($('#declA'), M.map(m => ({ label: m.model, a: m.acc, b: m.acc_declared, aName: 'No date', bName: 'Date declared' })), { W: 520, sep: ' → ', noChance: true, label: 'Accuracy without and with a declared date' });
181
+
182
+ /* point-in-time */
183
+ (function pit() {
184
+ const lo = -.08, hi = .12, x = v => (v - lo) / (hi - lo) * 100;
185
+ $('#pitT').innerHTML = `<thead><tr><th class="left">Model</th><th>Cutoff</th><th>Up to cutoff</th><th>After</th><th>Difference [95% CI]</th><th class="left">Interval around zero</th></tr></thead><tbody>` +
186
+ D.pit.map(r => `<tr><td class="left mname">${r.model}</td><td>${r.cutoff}</td><td>${f2(r.before)}</td><td>${f2(r.after)}</td>
187
+ <td>${f3(r.diff)}<span class="ci">[${f3(r.lo)}, ${f3(r.hi)}]</span></td>
188
+ <td class="left"><span class="forest" data-tip="<b>${r.model} ${r.cutoff}</b><br>Difference ${f3(r.diff)}"><s style="left:${x(0)}%"></s><i style="left:${x(r.lo)}%;width:${x(r.hi) - x(r.lo)}%"></i><b style="left:${x(r.diff)}%"></b></span></td></tr>`).join('') + '</tbody>';
189
+ })();
190
+
191
+ drawAll();
192
+ let rz; addEventListener('resize', () => { clearTimeout(rz); rz = setTimeout(drawAll, 120); });
assets/finai-logo.png ADDED
data.js ADDED
@@ -0,0 +1 @@
 
 
1
+ window.FD = {"meta":{"facts":2226,"prompts":11130,"controlled":1340,"chance":0.27,"from":1999,"to":2026},"models":[{"model":"GPT-5","key":"gpt-5","vendor":"OpenAI","small":false,"knows":0.76,"says":0.38,"acc_says":0.95,"acc_denies":0.61,"gap":0.18,"gap_lo":0.13,"gap_hi":0.22,"claims":0.39,"claims_declared":0.01,"acc":0.76,"acc_declared":0.75,"know_end":"2024Q3","claim_end":"2024Q3","self_report":"2024-10","vendor_cutoff":"2024-09"},{"model":"DeepSeek-V3.2","key":"DeepSeek-V3.2","vendor":"DeepSeek","small":false,"knows":0.54,"says":0.28,"acc_says":0.65,"acc_denies":0.48,"gap":0.08,"gap_lo":0.03,"gap_hi":0.15,"claims":0.37,"claims_declared":0.46,"acc":0.44,"acc_declared":0.43,"know_end":"2024Q3","claim_end":"2024Q4","self_report":"2024-07","vendor_cutoff":null},{"model":"GPT-4o","key":"gpt-4o","vendor":"OpenAI","small":false,"knows":0.5,"says":0.2,"acc_says":0.83,"acc_denies":0.34,"gap":0.39,"gap_lo":0.32,"gap_hi":0.45,"claims":0.33,"claims_declared":0.04,"acc":0.43,"acc_declared":0.44,"know_end":"2023Q1","claim_end":"2023Q2","self_report":"2023-10","vendor_cutoff":"2023-10"},{"model":"Llama-3.3-70B","key":"Llama-3.3-70B-Instruct","vendor":"Meta","small":false,"knows":0.43,"says":0.48,"acc_says":0.58,"acc_denies":0.29,"gap":0.28,"gap_lo":0.2,"gap_hi":0.34,"claims":0.63,"claims_declared":0.0,"acc":0.41,"acc_declared":0.41,"know_end":"2021Q2","claim_end":"2023Q1","self_report":"2023-12","vendor_cutoff":"2023-12"},{"model":"Qwen3.5-9B","key":"Qwen3.5-9B","vendor":"Alibaba","small":true,"knows":0.37,"says":0.69,"acc_says":0.4,"acc_denies":0.3,"gap":0.1,"gap_lo":0.03,"gap_hi":0.19,"claims":0.63,"claims_declared":0.53,"acc":0.31,"acc_declared":0.35},{"model":"Qwen3-8B","key":"Qwen3-8B","vendor":"Alibaba","small":true,"knows":0.36,"says":0.71,"acc_says":0.38,"acc_denies":0.31,"gap":0.06,"gap_lo":-0.01,"gap_hi":0.14,"claims":0.68,"claims_declared":0.45,"acc":0.38,"acc_declared":0.41}],"timeline":{"GPT-5":[{"p":"1999","x":1999.5,"n":48,"acc":0.8,"claim":0.292},{"p":"2000","x":2000.5,"n":48,"acc":0.824,"claim":0.312},{"p":"2001","x":2001.5,"n":48,"acc":0.676,"claim":0.375},{"p":"2002","x":2002.5,"n":48,"acc":0.697,"claim":0.354},{"p":"2003","x":2003.5,"n":48,"acc":0.867,"claim":0.417},{"p":"2004","x":2004.5,"n":48,"acc":0.793,"claim":0.333},{"p":"2005","x":2005.5,"n":48,"acc":1.0,"claim":0.458},{"p":"2006","x":2006.5,"n":48,"acc":0.939,"claim":0.417},{"p":"2007","x":2007.5,"n":48,"acc":0.844,"claim":0.438},{"p":"2008","x":2008.5,"n":48,"acc":0.714,"claim":0.458},{"p":"2009","x":2009.5,"n":48,"acc":0.906,"claim":0.458},{"p":"2010","x":2010.5,"n":48,"acc":0.893,"claim":0.375},{"p":"2011","x":2011.5,"n":48,"acc":0.839,"claim":0.417},{"p":"2012","x":2012.5,"n":48,"acc":0.963,"claim":0.5},{"p":"2013","x":2013.5,"n":48,"acc":0.885,"claim":0.479},{"p":"2014","x":2014.5,"n":48,"acc":0.912,"claim":0.562},{"p":"2015","x":2015.5,"n":48,"acc":0.875,"claim":0.562},{"p":"2016","x":2016.5,"n":48,"acc":0.944,"claim":0.5},{"p":"2017","x":2017.5,"n":48,"acc":0.92,"claim":0.479},{"p":"2018","x":2018.5,"n":48,"acc":0.962,"claim":0.583},{"p":"2019","x":2019.5,"n":48,"acc":0.914,"claim":0.583},{"p":"2020","x":2020.5,"n":48,"acc":0.833,"claim":0.688},{"p":"2021Q1","x":2021.125,"n":51,"acc":0.931,"claim":0.569},{"p":"2021Q2","x":2021.375,"n":51,"acc":0.957,"claim":0.569},{"p":"2021Q3","x":2021.625,"n":51,"acc":0.9,"claim":0.529},{"p":"2021Q4","x":2021.875,"n":51,"acc":0.893,"claim":0.608},{"p":"2022Q1","x":2022.125,"n":51,"acc":0.895,"claim":0.451},{"p":"2022Q2","x":2022.375,"n":51,"acc":0.903,"claim":0.5},{"p":"2022Q3","x":2022.625,"n":51,"acc":0.861,"claim":0.588},{"p":"2022Q4","x":2022.875,"n":51,"acc":0.875,"claim":0.46},{"p":"2023Q1","x":2023.125,"n":51,"acc":0.929,"claim":0.431},{"p":"2023Q2","x":2023.375,"n":51,"acc":0.909,"claim":0.49},{"p":"2023Q3","x":2023.625,"n":51,"acc":0.862,"claim":0.569},{"p":"2023Q4","x":2023.875,"n":51,"acc":0.861,"claim":0.51},{"p":"2024Q1","x":2024.125,"n":51,"acc":0.846,"claim":0.392},{"p":"2024Q2","x":2024.375,"n":51,"acc":0.963,"claim":0.431},{"p":"2024Q3","x":2024.625,"n":51,"acc":0.333,"claim":0.196},{"p":"2024Q4","x":2024.875,"n":51,"acc":0.46,"claim":0.02},{"p":"2025Q1","x":2025.125,"n":51,"acc":0.3,"claim":0.0},{"p":"2025Q2","x":2025.375,"n":51,"acc":0.174,"claim":0.0},{"p":"2025Q3","x":2025.625,"n":51,"acc":0.426,"claim":0.0},{"p":"2025Q4","x":2025.875,"n":48,"acc":0.25,"claim":0.0},{"p":"2026Q1","x":2026.125,"n":51,"acc":0.308,"claim":0.0},{"p":"2026Q2","x":2026.375,"n":51,"acc":0.333,"claim":0.0},{"p":"2026Q3","x":2026.625,"n":51,"acc":0.359,"claim":0.0}],"DeepSeek-V3.2":[{"p":"1999","x":1999.5,"n":48,"acc":0.453,"claim":0.354},{"p":"2000","x":2000.5,"n":48,"acc":0.559,"claim":0.375},{"p":"2001","x":2001.5,"n":48,"acc":0.424,"claim":0.375},{"p":"2002","x":2002.5,"n":48,"acc":0.449,"claim":0.354},{"p":"2003","x":2003.5,"n":48,"acc":0.506,"claim":0.25},{"p":"2004","x":2004.5,"n":48,"acc":0.606,"claim":0.375},{"p":"2005","x":2005.5,"n":48,"acc":0.604,"claim":0.292},{"p":"2006","x":2006.5,"n":48,"acc":0.621,"claim":0.375},{"p":"2007","x":2007.5,"n":48,"acc":0.552,"claim":0.396},{"p":"2008","x":2008.5,"n":48,"acc":0.423,"claim":0.375},{"p":"2009","x":2009.5,"n":48,"acc":0.625,"claim":0.417},{"p":"2010","x":2010.5,"n":48,"acc":0.714,"claim":0.375},{"p":"2011","x":2011.5,"n":48,"acc":0.645,"claim":0.333},{"p":"2012","x":2012.5,"n":48,"acc":0.815,"claim":0.312},{"p":"2013","x":2013.5,"n":48,"acc":0.808,"claim":0.333},{"p":"2014","x":2014.5,"n":48,"acc":0.735,"claim":0.333},{"p":"2015","x":2015.5,"n":48,"acc":0.534,"claim":0.333},{"p":"2016","x":2016.5,"n":48,"acc":0.659,"claim":0.354},{"p":"2017","x":2017.5,"n":48,"acc":0.6,"claim":0.417},{"p":"2018","x":2018.5,"n":48,"acc":0.577,"claim":0.458},{"p":"2019","x":2019.5,"n":48,"acc":0.581,"claim":0.312},{"p":"2020","x":2020.5,"n":48,"acc":0.612,"claim":0.396},{"p":"2021Q1","x":2021.125,"n":51,"acc":0.655,"claim":0.255},{"p":"2021Q2","x":2021.375,"n":51,"acc":0.652,"claim":0.235},{"p":"2021Q3","x":2021.625,"n":51,"acc":0.5,"claim":0.28},{"p":"2021Q4","x":2021.875,"n":51,"acc":0.679,"claim":0.333},{"p":"2022Q1","x":2022.125,"n":51,"acc":0.513,"claim":0.28},{"p":"2022Q2","x":2022.375,"n":51,"acc":0.532,"claim":0.38},{"p":"2022Q3","x":2022.625,"n":51,"acc":0.473,"claim":0.294},{"p":"2022Q4","x":2022.875,"n":51,"acc":0.516,"claim":0.333},{"p":"2023Q1","x":2023.125,"n":51,"acc":0.375,"claim":0.412},{"p":"2023Q2","x":2023.375,"n":51,"acc":0.462,"claim":0.32},{"p":"2023Q3","x":2023.625,"n":51,"acc":0.672,"claim":0.412},{"p":"2023Q4","x":2023.875,"n":51,"acc":0.556,"claim":0.255},{"p":"2024Q1","x":2024.125,"n":51,"acc":0.538,"claim":0.353},{"p":"2024Q2","x":2024.375,"n":51,"acc":0.704,"claim":0.38},{"p":"2024Q3","x":2024.625,"n":51,"acc":0.348,"claim":0.255},{"p":"2024Q4","x":2024.875,"n":51,"acc":0.306,"claim":0.137},{"p":"2025Q1","x":2025.125,"n":51,"acc":0.379,"claim":0.059},{"p":"2025Q2","x":2025.375,"n":51,"acc":0.348,"claim":0.039},{"p":"2025Q3","x":2025.625,"n":51,"acc":0.312,"claim":0.0},{"p":"2025Q4","x":2025.875,"n":48,"acc":0.45,"claim":0.021},{"p":"2026Q1","x":2026.125,"n":51,"acc":0.269,"claim":0.0},{"p":"2026Q2","x":2026.375,"n":51,"acc":0.414,"claim":0.0},{"p":"2026Q3","x":2026.625,"n":51,"acc":0.367,"claim":0.0}],"GPT-4o":[{"p":"1999","x":1999.5,"n":48,"acc":0.533,"claim":0.25},{"p":"2000","x":2000.5,"n":48,"acc":0.314,"claim":0.292},{"p":"2001","x":2001.5,"n":48,"acc":0.353,"claim":0.25},{"p":"2002","x":2002.5,"n":48,"acc":0.479,"claim":0.25},{"p":"2003","x":2003.5,"n":48,"acc":0.467,"claim":0.292},{"p":"2004","x":2004.5,"n":48,"acc":0.596,"claim":0.25},{"p":"2005","x":2005.5,"n":48,"acc":0.333,"claim":0.292},{"p":"2006","x":2006.5,"n":48,"acc":0.682,"claim":0.292},{"p":"2007","x":2007.5,"n":48,"acc":0.333,"claim":0.292},{"p":"2008","x":2008.5,"n":48,"acc":0.412,"claim":0.271},{"p":"2009","x":2009.5,"n":48,"acc":0.531,"claim":0.292},{"p":"2010","x":2010.5,"n":48,"acc":0.714,"claim":0.292},{"p":"2011","x":2011.5,"n":48,"acc":0.548,"claim":0.25},{"p":"2012","x":2012.5,"n":48,"acc":0.519,"claim":0.271},{"p":"2013","x":2013.5,"n":48,"acc":0.538,"claim":0.292},{"p":"2014","x":2014.5,"n":48,"acc":0.588,"claim":0.312},{"p":"2015","x":2015.5,"n":48,"acc":0.75,"claim":0.292},{"p":"2016","x":2016.5,"n":48,"acc":0.583,"claim":0.375},{"p":"2017","x":2017.5,"n":48,"acc":0.64,"claim":0.312},{"p":"2018","x":2018.5,"n":48,"acc":0.635,"claim":0.354},{"p":"2019","x":2019.5,"n":48,"acc":0.753,"claim":0.312},{"p":"2020","x":2020.5,"n":48,"acc":0.615,"claim":0.396},{"p":"2021Q1","x":2021.125,"n":51,"acc":0.724,"claim":0.275},{"p":"2021Q2","x":2021.375,"n":51,"acc":0.696,"claim":0.235},{"p":"2021Q3","x":2021.625,"n":51,"acc":0.5,"claim":0.235},{"p":"2021Q4","x":2021.875,"n":51,"acc":0.5,"claim":0.275},{"p":"2022Q1","x":2022.125,"n":51,"acc":0.421,"claim":0.235},{"p":"2022Q2","x":2022.375,"n":51,"acc":0.371,"claim":0.235},{"p":"2022Q3","x":2022.625,"n":51,"acc":0.446,"claim":0.176},{"p":"2022Q4","x":2022.875,"n":51,"acc":0.578,"claim":0.255},{"p":"2023Q1","x":2023.125,"n":51,"acc":0.446,"claim":0.275},{"p":"2023Q2","x":2023.375,"n":51,"acc":0.371,"claim":0.137},{"p":"2023Q3","x":2023.625,"n":51,"acc":0.388,"claim":0.157},{"p":"2023Q4","x":2023.875,"n":51,"acc":0.314,"claim":0.0},{"p":"2024Q1","x":2024.125,"n":51,"acc":0.455,"claim":0.0},{"p":"2024Q2","x":2024.375,"n":51,"acc":0.368,"claim":0.0},{"p":"2024Q3","x":2024.625,"n":51,"acc":0.271,"claim":0.0},{"p":"2024Q4","x":2024.875,"n":51,"acc":0.325,"claim":0.0},{"p":"2025Q1","x":2025.125,"n":51,"acc":0.316,"claim":0.0},{"p":"2025Q2","x":2025.375,"n":51,"acc":0.179,"claim":0.0},{"p":"2025Q3","x":2025.625,"n":51,"acc":0.462,"claim":0.0},{"p":"2025Q4","x":2025.875,"n":48,"acc":0.267,"claim":0.0},{"p":"2026Q1","x":2026.125,"n":51,"acc":0.538,"claim":0.0},{"p":"2026Q2","x":2026.375,"n":51,"acc":0.375,"claim":0.0},{"p":"2026Q3","x":2026.625,"n":51,"acc":0.188,"claim":0.0}],"Llama-3.3-70B":[{"p":"1999","x":1999.5,"n":48,"acc":0.427,"claim":0.708},{"p":"2000","x":2000.5,"n":48,"acc":0.559,"claim":0.708},{"p":"2001","x":2001.5,"n":48,"acc":0.294,"claim":0.688},{"p":"2002","x":2002.5,"n":48,"acc":0.377,"claim":0.646},{"p":"2003","x":2003.5,"n":48,"acc":0.5,"claim":0.667},{"p":"2004","x":2004.5,"n":48,"acc":0.458,"claim":0.667},{"p":"2005","x":2005.5,"n":48,"acc":0.417,"claim":0.729},{"p":"2006","x":2006.5,"n":48,"acc":0.568,"claim":0.75},{"p":"2007","x":2007.5,"n":48,"acc":0.323,"claim":0.708},{"p":"2008","x":2008.5,"n":48,"acc":0.441,"claim":0.729},{"p":"2009","x":2009.5,"n":48,"acc":0.656,"claim":0.729},{"p":"2010","x":2010.5,"n":48,"acc":0.607,"claim":0.708},{"p":"2011","x":2011.5,"n":48,"acc":0.613,"claim":0.708},{"p":"2012","x":2012.5,"n":48,"acc":0.667,"claim":0.75},{"p":"2013","x":2013.5,"n":48,"acc":0.615,"claim":0.688},{"p":"2014","x":2014.5,"n":48,"acc":0.588,"claim":0.833},{"p":"2015","x":2015.5,"n":48,"acc":0.625,"claim":0.792},{"p":"2016","x":2016.5,"n":48,"acc":0.652,"claim":0.75},{"p":"2017","x":2017.5,"n":48,"acc":0.507,"claim":0.75},{"p":"2018","x":2018.5,"n":48,"acc":0.654,"claim":0.812},{"p":"2019","x":2019.5,"n":48,"acc":0.624,"claim":0.792},{"p":"2020","x":2020.5,"n":48,"acc":0.533,"claim":0.792},{"p":"2021Q1","x":2021.125,"n":51,"acc":0.517,"claim":0.745},{"p":"2021Q2","x":2021.375,"n":51,"acc":0.478,"claim":0.745},{"p":"2021Q3","x":2021.625,"n":51,"acc":0.25,"claim":0.686},{"p":"2021Q4","x":2021.875,"n":51,"acc":0.393,"claim":0.725},{"p":"2022Q1","x":2022.125,"n":51,"acc":0.368,"claim":0.667},{"p":"2022Q2","x":2022.375,"n":51,"acc":0.274,"claim":0.686},{"p":"2022Q3","x":2022.625,"n":51,"acc":0.311,"claim":0.647},{"p":"2022Q4","x":2022.875,"n":51,"acc":0.422,"claim":0.588},{"p":"2023Q1","x":2023.125,"n":51,"acc":0.304,"claim":0.235},{"p":"2023Q2","x":2023.375,"n":51,"acc":0.379,"claim":0.078},{"p":"2023Q3","x":2023.625,"n":51,"acc":0.414,"claim":0.039},{"p":"2023Q4","x":2023.875,"n":51,"acc":0.361,"claim":0.0},{"p":"2024Q1","x":2024.125,"n":51,"acc":0.269,"claim":0.0},{"p":"2024Q2","x":2024.375,"n":51,"acc":0.269,"claim":0.0},{"p":"2024Q3","x":2024.625,"n":51,"acc":0.188,"claim":0.0},{"p":"2024Q4","x":2024.875,"n":51,"acc":0.226,"claim":0.0},{"p":"2025Q1","x":2025.125,"n":51,"acc":0.214,"claim":0.0},{"p":"2025Q2","x":2025.375,"n":51,"acc":0.348,"claim":0.0},{"p":"2025Q3","x":2025.625,"n":51,"acc":0.384,"claim":0.0},{"p":"2025Q4","x":2025.875,"n":48,"acc":0.097,"claim":0.0},{"p":"2026Q1","x":2026.125,"n":51,"acc":0.231,"claim":0.0},{"p":"2026Q2","x":2026.375,"n":51,"acc":0.185,"claim":0.0},{"p":"2026Q3","x":2026.625,"n":51,"acc":0.311,"claim":0.0}],"Qwen3.5-9B":[{"p":"1999","x":1999.5,"n":48,"acc":0.387,"claim":0.771},{"p":"2000","x":2000.5,"n":48,"acc":0.235,"claim":0.896},{"p":"2001","x":2001.5,"n":48,"acc":0.241,"claim":0.854},{"p":"2002","x":2002.5,"n":48,"acc":0.234,"claim":0.708},{"p":"2003","x":2003.5,"n":48,"acc":0.273,"claim":0.812},{"p":"2004","x":2004.5,"n":48,"acc":0.409,"claim":0.792},{"p":"2005","x":2005.5,"n":48,"acc":0.354,"claim":0.833},{"p":"2006","x":2006.5,"n":48,"acc":0.394,"claim":0.792},{"p":"2007","x":2007.5,"n":48,"acc":0.083,"claim":0.833},{"p":"2008","x":2008.5,"n":48,"acc":0.443,"claim":0.896},{"p":"2009","x":2009.5,"n":48,"acc":0.5,"claim":0.875},{"p":"2010","x":2010.5,"n":48,"acc":0.536,"claim":0.75},{"p":"2011","x":2011.5,"n":48,"acc":0.516,"claim":0.854},{"p":"2012","x":2012.5,"n":48,"acc":0.519,"claim":0.917},{"p":"2013","x":2013.5,"n":48,"acc":0.462,"claim":0.833},{"p":"2014","x":2014.5,"n":48,"acc":0.529,"claim":0.854},{"p":"2015","x":2015.5,"n":48,"acc":0.515,"claim":0.917},{"p":"2016","x":2016.5,"n":48,"acc":0.523,"claim":0.854},{"p":"2017","x":2017.5,"n":48,"acc":0.493,"claim":0.896},{"p":"2018","x":2018.5,"n":48,"acc":0.375,"claim":0.917},{"p":"2019","x":2019.5,"n":48,"acc":0.333,"claim":0.812},{"p":"2020","x":2020.5,"n":48,"acc":0.445,"claim":0.896},{"p":"2021Q1","x":2021.125,"n":51,"acc":0.483,"claim":0.941},{"p":"2021Q2","x":2021.375,"n":51,"acc":0.304,"claim":0.882},{"p":"2021Q3","x":2021.625,"n":51,"acc":0.25,"claim":0.882},{"p":"2021Q4","x":2021.875,"n":51,"acc":0.357,"claim":0.922},{"p":"2022Q1","x":2022.125,"n":51,"acc":0.27,"claim":0.922},{"p":"2022Q2","x":2022.375,"n":51,"acc":0.339,"claim":0.902},{"p":"2022Q3","x":2022.625,"n":51,"acc":0.176,"claim":0.902},{"p":"2022Q4","x":2022.875,"n":51,"acc":0.453,"claim":0.882},{"p":"2023Q1","x":2023.125,"n":51,"acc":0.304,"claim":0.843},{"p":"2023Q2","x":2023.375,"n":51,"acc":0.348,"claim":0.765},{"p":"2023Q3","x":2023.625,"n":51,"acc":0.224,"claim":0.784},{"p":"2023Q4","x":2023.875,"n":51,"acc":0.306,"claim":0.647},{"p":"2024Q1","x":2024.125,"n":51,"acc":0.462,"claim":0.706},{"p":"2024Q2","x":2024.375,"n":51,"acc":0.37,"claim":0.686},{"p":"2024Q3","x":2024.625,"n":51,"acc":0.379,"claim":0.569},{"p":"2024Q4","x":2024.875,"n":51,"acc":0.339,"claim":0.373},{"p":"2025Q1","x":2025.125,"n":51,"acc":0.367,"claim":0.059},{"p":"2025Q2","x":2025.375,"n":51,"acc":0.348,"claim":0.0},{"p":"2025Q3","x":2025.625,"n":51,"acc":0.312,"claim":0.02},{"p":"2025Q4","x":2025.875,"n":48,"acc":0.339,"claim":0.0},{"p":"2026Q1","x":2026.125,"n":51,"acc":0.231,"claim":0.02},{"p":"2026Q2","x":2026.375,"n":51,"acc":0.379,"claim":0.0},{"p":"2026Q3","x":2026.625,"n":51,"acc":0.379,"claim":0.0}],"Qwen3-8B":[{"p":"1999","x":1999.5,"n":48,"acc":0.293,"claim":0.833},{"p":"2000","x":2000.5,"n":48,"acc":0.48,"claim":0.812},{"p":"2001","x":2001.5,"n":48,"acc":0.253,"claim":0.896},{"p":"2002","x":2002.5,"n":48,"acc":0.383,"claim":0.792},{"p":"2003","x":2003.5,"n":48,"acc":0.221,"claim":0.854},{"p":"2004","x":2004.5,"n":48,"acc":0.309,"claim":0.771},{"p":"2005","x":2005.5,"n":48,"acc":0.552,"claim":0.833},{"p":"2006","x":2006.5,"n":48,"acc":0.326,"claim":0.896},{"p":"2007","x":2007.5,"n":48,"acc":0.229,"claim":0.812},{"p":"2008","x":2008.5,"n":48,"acc":0.309,"claim":0.896},{"p":"2009","x":2009.5,"n":48,"acc":0.25,"claim":0.812},{"p":"2010","x":2010.5,"n":48,"acc":0.357,"claim":0.854},{"p":"2011","x":2011.5,"n":48,"acc":0.516,"claim":0.75},{"p":"2012","x":2012.5,"n":48,"acc":0.63,"claim":0.812},{"p":"2013","x":2013.5,"n":48,"acc":0.731,"claim":0.833},{"p":"2014","x":2014.5,"n":48,"acc":0.529,"claim":0.854},{"p":"2015","x":2015.5,"n":48,"acc":0.542,"claim":0.875},{"p":"2016","x":2016.5,"n":48,"acc":0.472,"claim":0.792},{"p":"2017","x":2017.5,"n":48,"acc":0.147,"claim":0.75},{"p":"2018","x":2018.5,"n":48,"acc":0.452,"claim":0.896},{"p":"2019","x":2019.5,"n":48,"acc":0.419,"claim":0.75},{"p":"2020","x":2020.5,"n":48,"acc":0.494,"claim":0.958},{"p":"2021Q1","x":2021.125,"n":51,"acc":0.345,"claim":0.882},{"p":"2021Q2","x":2021.375,"n":51,"acc":0.391,"claim":0.882},{"p":"2021Q3","x":2021.625,"n":51,"acc":0.4,"claim":0.863},{"p":"2021Q4","x":2021.875,"n":51,"acc":0.464,"claim":0.941},{"p":"2022Q1","x":2022.125,"n":51,"acc":0.25,"claim":0.902},{"p":"2022Q2","x":2022.375,"n":51,"acc":0.242,"claim":0.863},{"p":"2022Q3","x":2022.625,"n":51,"acc":0.338,"claim":0.784},{"p":"2022Q4","x":2022.875,"n":51,"acc":0.391,"claim":0.843},{"p":"2023Q1","x":2023.125,"n":51,"acc":0.268,"claim":0.804},{"p":"2023Q2","x":2023.375,"n":51,"acc":0.371,"claim":0.784},{"p":"2023Q3","x":2023.625,"n":51,"acc":0.397,"claim":0.784},{"p":"2023Q4","x":2023.875,"n":51,"acc":0.361,"claim":0.667},{"p":"2024Q1","x":2024.125,"n":51,"acc":0.346,"claim":0.608},{"p":"2024Q2","x":2024.375,"n":51,"acc":0.407,"claim":0.588},{"p":"2024Q3","x":2024.625,"n":51,"acc":0.348,"claim":0.412},{"p":"2024Q4","x":2024.875,"n":51,"acc":0.339,"claim":0.216},{"p":"2025Q1","x":2025.125,"n":51,"acc":0.367,"claim":0.314},{"p":"2025Q2","x":2025.375,"n":51,"acc":0.348,"claim":0.314},{"p":"2025Q3","x":2025.625,"n":51,"acc":0.304,"claim":0.216},{"p":"2025Q4","x":2025.875,"n":48,"acc":0.274,"claim":0.125},{"p":"2026Q1","x":2026.125,"n":51,"acc":0.231,"claim":0.333},{"p":"2026Q2","x":2026.375,"n":51,"acc":0.379,"claim":0.314},{"p":"2026Q3","x":2026.625,"n":51,"acc":0.379,"claim":0.235}]},"family":{"GPT-5":{"Macroeconomic":{"know":0.747,"says":0.384,"acc_deny":0.648},"Stock prices":{"know":0.59,"says":0.046,"acc_deny":0.57},"Fed decisions":{"know":0.941,"says":0.931,"acc_deny":0.682}},"DeepSeek-V3.2":{"Macroeconomic":{"know":0.478,"says":0.242,"acc_deny":0.475},"Stock prices":{"know":0.49,"says":0.024,"acc_deny":0.479},"Fed decisions":{"know":0.729,"says":0.886,"acc_deny":0.632}},"GPT-4o":{"Macroeconomic":{"know":0.367,"says":0.104,"acc_deny":0.345},"Stock prices":{"know":0.332,"says":0.0,"acc_deny":0.332},"Fed decisions":{"know":0.882,"says":0.892,"acc_deny":0.8}},"Llama-3.3-70B":{"Macroeconomic":{"know":0.322,"says":0.528,"acc_deny":0.29},"Stock prices":{"know":0.272,"says":0.111,"acc_deny":0.272},"Fed decisions":{"know":0.819,"says":0.889,"acc_deny":0.27}}},"pit":[{"model":"TiMaGPT","cutoff":2014,"before":0.29,"after":0.27,"diff":0.016,"lo":-0.023,"hi":0.055},{"model":"TiMaGPT","cutoff":2016,"before":0.28,"after":0.26,"diff":0.021,"lo":-0.02,"hi":0.064},{"model":"TiMaGPT","cutoff":2018,"before":0.28,"after":0.26,"diff":0.021,"lo":-0.02,"hi":0.061},{"model":"TiMaGPT","cutoff":2020,"before":0.28,"after":0.27,"diff":0.006,"lo":-0.032,"hi":0.048},{"model":"TiMaGPT","cutoff":2022,"before":0.27,"after":0.26,"diff":0.006,"lo":-0.037,"hi":0.052},{"model":"ChronoGPT","cutoff":2014,"before":0.28,"after":0.27,"diff":0.001,"lo":-0.038,"hi":0.045},{"model":"ChronoGPT","cutoff":2016,"before":0.29,"after":0.27,"diff":0.023,"lo":-0.015,"hi":0.068},{"model":"ChronoGPT","cutoff":2018,"before":0.28,"after":0.26,"diff":0.017,"lo":-0.019,"hi":0.057},{"model":"ChronoGPT","cutoff":2020,"before":0.28,"after":0.27,"diff":0.011,"lo":-0.028,"hi":0.051},{"model":"ChronoGPT","cutoff":2022,"before":0.27,"after":0.28,"diff":-0.01,"lo":-0.053,"hi":0.042},{"model":"DatedGPT","cutoff":2014,"before":0.29,"after":0.25,"diff":0.041,"lo":-0.003,"hi":0.088}]};
index.html CHANGED
@@ -1,19 +1,130 @@
1
  <!doctype html>
2
- <html>
3
- <head>
4
- <meta charset="utf-8" />
5
- <meta name="viewport" content="width=device-width" />
6
- <title>My static Space</title>
7
- <link rel="stylesheet" href="style.css" />
8
- </head>
9
- <body>
10
- <div class="card">
11
- <h1>Welcome to your static Space!</h1>
12
- <p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
13
- <p>
14
- Also don't forget to check the
15
- <a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
16
- </p>
17
- </div>
18
- </body>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19
  </html>
 
1
  <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8" />
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
6
+ <title>FinDendro · Temporal Self-Knowledge Leaderboard · The Fin AI</title>
7
+ <meta name="description" content="FinDendro measures what LLMs know about dated financial facts and what they say they know." />
8
+ <link rel="preconnect" href="https://fonts.googleapis.com" />
9
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
10
+ <link href="https://fonts.googleapis.com/css2?family=Newsreader:ital,opsz,wght@0,6..72,400;0,6..72,500;1,6..72,400;1,6..72,500&family=Inter:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
11
+ <link rel="icon" href="./assets/finai-logo.png" />
12
+ <link rel="stylesheet" href="./style.css" />
13
+ </head>
14
+ <body>
15
+ <div class="tip" id="tip" role="tooltip"></div>
16
+
17
+ <header class="header">
18
+ <div class="wrap nav">
19
+ <a href="#top" class="brand"><img src="./assets/finai-logo.png" alt="" />The Fin AI<span class="brand-sep">/</span><span class="brand-fd">FinDendro</span></a>
20
+ <nav class="links">
21
+ <a href="#leaderboard">Leaderboard</a>
22
+ <a href="#timeline">Timeline</a>
23
+ <a href="#families">Facts</a>
24
+ <a href="#declared">Declared date</a>
25
+ <a href="#pit">Point-in-time</a>
26
+ <a href="#method">Method</a>
27
+ </nav>
28
+ </div>
29
+ </header>
30
+
31
+ <main id="top">
32
+ <section class="hero">
33
+ <svg class="rings" id="rings" aria-hidden="true" viewBox="-320 -320 640 640"></svg>
34
+ <div class="wrap hero-grid">
35
+ <div class="hero-copy">
36
+ <p class="eyebrow">Benchmark for financial backtesting</p>
37
+ <h1 class="wordmark">FinDendro</h1>
38
+ <p class="hero-sub">Do LLMs know what they <em>know</em>?</p>
39
+ <p class="lead">Backtests trust a model to say when its knowledge ends. FinDendro checks that claim against dated U.S. macro data, stock prices and Fed decisions, one quarter at a time.</p>
40
+ <div class="stats" id="stats"></div>
41
+ <div class="ctas">
42
+ <a class="btn" href="#leaderboard">See the leaderboard</a>
43
+ <a class="btn btn--ghost" href="https://huggingface.co/datasets/TheFinAI/FinDendro" target="_blank" rel="noreferrer">Dataset ↗</a>
44
+ </div>
45
+ </div>
46
+ <figure class="card hero-viz">
47
+ <figcaption>Knows vs says it knows</figcaption>
48
+ <div class="legend">
49
+ <span><i class="dot dot--knows"></i>Knows</span>
50
+ <span><i class="dot dot--says"></i>Says it knows</span>
51
+ <span><i class="tick"></i>Chance</span>
52
+ </div>
53
+ <div id="dumbbell"></div>
54
+ </figure>
55
+ </div>
56
+ </section>
57
+
58
+ <section class="section" id="leaderboard">
59
+ <div class="wrap">
60
+ <h2>Leaderboard</h2>
61
+ <p class="sec-sub">Sorted by what each model knows. Click a column to re-sort.</p>
62
+ <div class="scroll card card--flush"><table class="lb" id="lb"></table></div>
63
+ <p class="note"><b>Knows</b> is accuracy on 1,340 controlled facts. <b>Says</b> is the share of answers that claim knowledge. <b>Acc | says</b> and <b>Acc | denies</b> split accuracy by that claim. <b>Gap</b> is their within-period difference with a 95% interval. Boundaries are not reported for the two small Qwen models, whose accuracy is too close to chance for the rule to locate an end.</p>
64
+ <div class="findings" id="findings"></div>
65
+ </div>
66
+ </section>
67
+
68
+ <section class="section section--tint" id="timeline">
69
+ <div class="wrap">
70
+ <h2>Where knowledge ends and claims end</h2>
71
+ <p class="sec-sub">Accuracy and claim rate by period since 2017. Yearly up to 2020, quarterly after.</p>
72
+ <div class="legend legend--wide">
73
+ <span><i class="ln ln--knows"></i>Accuracy</span>
74
+ <span><i class="ln ln--says"></i>Claim rate</span>
75
+ <span><i class="vl vl--knows"></i>Knowledge ends</span>
76
+ <span><i class="vl vl--says"></i>Claims end</span>
77
+ <span><i class="vl vl--vendor"></i>Vendor cutoff</span>
78
+ <span><i class="tick"></i>Chance</span>
79
+ </div>
80
+ <div class="multiples" id="multiples"></div>
81
+ </div>
82
+ </section>
83
+
84
+ <section class="section" id="families">
85
+ <div class="wrap">
86
+ <h2>What kind of fact matters</h2>
87
+ <p class="sec-sub">The same model can know prices it will not admit to knowing, and admit to Fed decisions it does know.</p>
88
+ <div class="legend">
89
+ <span><i class="dot dot--knows"></i>Knows</span>
90
+ <span><i class="dot dot--says"></i>Says it knows</span>
91
+ </div>
92
+ <div class="fam" id="fam"></div>
93
+ </div>
94
+ </section>
95
+
96
+ <section class="section section--tint" id="declared">
97
+ <div class="wrap">
98
+ <h2>A declared date changes the claim, not the knowledge</h2>
99
+ <p class="sec-sub">Backtests fix the forecast origin with a line such as “Assume that today is May 31, 2023.” Each pair shows the answer without that line and with it, on the Predict wording.</p>
100
+ <div class="decl" id="decl"></div>
101
+ </div>
102
+ </section>
103
+
104
+ <section class="section" id="pit">
105
+ <div class="wrap">
106
+ <h2>Point-in-time models</h2>
107
+ <p class="sec-sub">Models trained only on data before a cutoff are free of look-ahead by construction. At 1 to 1.5B parameters they retain almost none of these facts, so accuracy stays near chance on both sides of the cutoff.</p>
108
+ <div class="scroll card card--flush"><table class="lb pit" id="pitT"></table></div>
109
+ <p class="note">Scored by the likelihood of each candidate value, since these models do not follow a multiple-choice format. The difference is accuracy up to the cutoff year minus accuracy after it, with a 95% interval.</p>
110
+ </div>
111
+ </section>
112
+
113
+ <section class="section section--tint" id="method">
114
+ <div class="wrap method">
115
+ <h2>Method</h2>
116
+ <div class="m-grid">
117
+ <div><h3>Facts</h3><p>2,226 dated U.S. facts sampled from a pool of 20,972, January 1999 to September 2026. Twelve FRED series, month-end prices of 50 large companies and Fed target rate decisions, all effectively unrevised after publication.</p></div>
118
+ <div><h3>Questions</h3><p>Each fact yields five prompts that share one set of candidates. Task 2 asks for the value. Task 1 offers every candidate twice, once with “I know this” and once with “I do not know this”, so one choice reveals both the answer and the claim.</p></div>
119
+ <div><h3>Controls</h3><p>47% of numerical facts can be answered by repeating last month's value, so they are excluded from knowledge. Fed decisions use balanced accuracy over three outcomes, since “unchanged” is right in 78% of months. Chance is about 0.27.</p></div>
120
+ <div><h3>Boundaries</h3><p>Knowledge ends at the last quarter whose two-quarter rolling accuracy stays above the midpoint between the 2015 to 2020 level and chance. Claims end at the last quarter whose claim rate stays above half of its 2015 to 2020 level.</p></div>
121
+ </div>
122
+ </div>
123
+ </section>
124
+ </main>
125
+
126
+ <footer class="foot"><div class="wrap">FinDendro · The Fin AI</div></footer>
127
+ <script src="./data.js"></script>
128
+ <script src="./app.js"></script>
129
+ </body>
130
  </html>
style.css CHANGED
@@ -1,28 +1,134 @@
1
- body {
2
- padding: 2rem;
3
- font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
 
 
 
 
 
 
 
4
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
 
6
- h1 {
7
- font-size: 16px;
8
- margin-top: 0;
9
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
10
 
11
- p {
12
- color: rgb(107, 114, 128);
13
- font-size: 15px;
14
- margin-bottom: 10px;
15
- margin-top: 5px;
16
- }
 
 
 
 
 
 
 
 
 
 
17
 
18
- .card {
19
- max-width: 620px;
20
- margin: 0 auto;
21
- padding: 16px;
22
- border: 1px solid lightgray;
23
- border-radius: 16px;
24
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
 
26
- .card p:last-child {
27
- margin-bottom: 0;
 
 
 
 
 
 
 
28
  }
 
 
1
+ /* FinDendro leaderboard. Fin AI house tokens, with the house blue as accent.
2
+ Knows is blue and Says is orange everywhere on the page. */
3
+ :root{
4
+ --ink:#0f1528; --soft:#4f5a6e; --muted:#7a8496; --line:rgba(24,56,152,.11);
5
+ --paper:#fcfcfd; --paper-2:#f4f6fa;
6
+ --ring:0 0 0 1px rgba(24,56,152,.08),0 1px 2px rgba(20,30,80,.04),0 10px 28px -18px rgba(24,56,152,.2);
7
+ --g:linear-gradient(90deg,#183898,#0060b0 55%,#68a8d8);
8
+ --knows:#1f4fb8; --says:#e0662a; --chance:#8a93a3; --vendor:#5d6678;
9
+ --serif:'Newsreader',Georgia,serif; --sans:'Inter',system-ui,-apple-system,sans-serif; --mono:'JetBrains Mono',ui-monospace,Menlo,monospace;
10
+ --wrap:1180px;
11
  }
12
+ *{box-sizing:border-box}
13
+ html{scroll-behavior:smooth;scroll-padding-top:72px}
14
+ body{margin:0;font-family:var(--sans);color:var(--ink);background:var(--paper);font-size:16px;line-height:1.6;-webkit-font-smoothing:antialiased}
15
+ h1,h2,h3{font-family:var(--serif);font-weight:500;line-height:1.12;letter-spacing:-.02em;margin:0;text-wrap:balance}
16
+ p{margin:0;color:var(--soft)}
17
+ a{color:inherit;text-decoration:none}
18
+ a:focus-visible,button:focus-visible,th:focus-visible{outline:2px solid var(--knows);outline-offset:2px}
19
+ .wrap{width:100%;max-width:var(--wrap);margin:0 auto;padding-inline:32px}
20
+ .header{position:sticky;top:0;z-index:50;background:rgba(252,252,253,.86);backdrop-filter:blur(14px) saturate(1.4);-webkit-backdrop-filter:blur(14px) saturate(1.4);border-bottom:1px solid var(--line)}
21
+ .nav{display:flex;align-items:center;justify-content:space-between;gap:16px;min-height:60px;flex-wrap:wrap}
22
+ .brand{display:inline-flex;align-items:center;gap:10px;font-family:var(--serif);font-size:1.15rem;font-weight:500;white-space:nowrap}
23
+ .brand img{height:22px;width:auto}
24
+ .brand-sep{color:var(--muted)}
25
+ .brand-fd{background:var(--g);-webkit-background-clip:text;background-clip:text;color:transparent;font-weight:500}
26
+ .links{display:flex;gap:2px;flex-wrap:wrap}
27
+ .links a{font-size:.86rem;color:var(--soft);padding:6px 12px;border-radius:999px}
28
+ .links a:hover{background:var(--paper-2);color:var(--ink)}
29
 
30
+ .hero{position:relative;overflow:hidden;padding-block:64px 56px}
31
+ .rings{position:absolute;right:-180px;top:-200px;width:820px;height:820px;pointer-events:none;opacity:.65}
32
+ .hero-grid{position:relative;display:grid;grid-template-columns:1.05fr 1fr;gap:56px;align-items:center}
33
+ .hero-grid>*{min-width:0}
34
+ .eyebrow{font:500 .76rem var(--mono);letter-spacing:.08em;text-transform:uppercase;color:var(--muted)}
35
+ .wordmark{font-size:clamp(3.2rem,7.4vw,5.4rem);margin-top:10px;background:var(--g);-webkit-background-clip:text;background-clip:text;color:transparent;padding-bottom:.06em}
36
+ .hero-sub{font-family:var(--serif);font-size:clamp(1.4rem,2.5vw,1.8rem);color:var(--ink);margin-top:2px}
37
+ .hero-sub em{color:var(--knows)}
38
+ .lead{margin-top:16px;font-size:1.05rem;max-width:34em}
39
+ .stats{display:grid;grid-template-columns:repeat(4,auto);justify-content:start;gap:12px 34px;margin-top:26px}
40
+ .stat b{display:block;font:500 1.65rem var(--serif);color:var(--ink);font-variant-numeric:tabular-nums;line-height:1.15}
41
+ .stat span{font-size:.8rem;color:var(--muted)}
42
+ .ctas{display:flex;gap:10px;margin-top:28px;flex-wrap:wrap}
43
+ .btn{display:inline-flex;align-items:center;font-size:.9rem;font-weight:500;padding:10px 20px;border-radius:999px;background:var(--ink);color:#fff}
44
+ .btn:hover{background:#183898}
45
+ .btn--ghost{background:transparent;color:var(--ink);box-shadow:inset 0 0 0 1px var(--line)}
46
+ .btn--ghost:hover{background:var(--paper-2);color:var(--ink)}
47
 
48
+ .card{background:#fff;border-radius:14px;box-shadow:var(--ring);padding:22px 24px}
49
+ .card--flush{padding:0}
50
+ figure{margin:0}
51
+ figcaption{font:500 1.15rem var(--serif);color:var(--ink)}
52
+ .legend{display:flex;flex-wrap:wrap;gap:6px 18px;margin-top:8px;font-size:.8rem;color:var(--soft)}
53
+ .legend span{display:inline-flex;align-items:center;gap:7px}
54
+ .legend--wide{margin-block:18px 16px}
55
+ .dot{width:10px;height:10px;border-radius:50%;display:inline-block}
56
+ .dot--knows{background:var(--knows)}.dot--says{background:var(--says)}
57
+ .tick{width:0;height:13px;border-left:1.5px dashed var(--chance);display:inline-block}
58
+ .ln{width:18px;height:2.5px;border-radius:2px;display:inline-block}
59
+ .ln--knows{background:var(--knows)}.ln--says{background:var(--says)}
60
+ .vl{width:0;height:13px;display:inline-block;border-left:2px solid}
61
+ .vl--knows{border-color:var(--knows)}.vl--says{border-color:var(--says)}.vl--vendor{border-left:2px dotted var(--vendor)}
62
+ #dumbbell{margin-top:6px}
63
+ svg text{font-family:var(--sans)}
64
 
65
+ .section{padding-block:60px}
66
+ .section--tint{background:var(--paper-2)}
67
+ h2{font-size:clamp(1.7rem,3vw,2.3rem)}
68
+ .sec-sub{margin-top:8px;max-width:48em}
69
+ #leaderboard .scroll,#pit .scroll{margin-top:22px}
70
+ .scroll{overflow-x:auto}
71
+ table.lb{border-collapse:collapse;width:100%;font-size:.9rem;font-variant-numeric:tabular-nums;min-width:920px}
72
+ .lb th{font:500 .74rem var(--sans);color:var(--muted);text-align:right;padding:13px 9px;border-bottom:1px solid var(--line);white-space:nowrap;user-select:none}
73
+ .lb th[data-k]{cursor:pointer}
74
+ .lb th[data-k]:hover{color:var(--ink)}
75
+ .lb th.on{color:var(--ink)}
76
+ .lb th.left,.lb td.left{text-align:left}
77
+ .lb td{padding:11px 9px;text-align:right;border-bottom:1px solid var(--line);color:var(--ink);white-space:nowrap}
78
+ .lb tr:last-child td{border-bottom:0}
79
+ .lb tbody tr:hover{background:var(--paper-2)}
80
+ .rank{color:var(--muted);font-family:var(--mono);font-size:.8rem}
81
+ .mname{font-weight:500}
82
+ .vendor{color:var(--muted);font-size:.78rem;margin-left:7px}
83
+ .chip{font:500 .66rem var(--mono);color:var(--soft);background:var(--paper-2);border-radius:6px;padding:1px 6px;margin-left:7px}
84
+ .bar{display:inline-flex;align-items:center;gap:9px;justify-content:flex-end}
85
+ .bar i{position:relative;display:block;width:60px;height:7px;background:var(--paper-2);border-radius:4px}
86
+ .bar i b{position:absolute;left:0;top:0;bottom:0;border-radius:4px;background:var(--knows)}
87
+ .bar i u{position:absolute;top:-3px;bottom:-3px;border-left:1.5px dashed var(--chance)}
88
+ .ci{color:var(--muted);font-size:.8rem;margin-left:4px}
89
+ .flag{color:#b43c12;font-weight:500}
90
+ .na{color:var(--muted)}
91
+ .note{margin-top:12px;font-size:.84rem;max-width:62em}
92
+ .note b{color:var(--ink);font-weight:500}
93
+ .findings{display:grid;grid-template-columns:repeat(3,1fr);gap:16px;margin-top:26px}
94
+ .findings>div{min-width:0}
95
+ .findings .num{font:500 1.55rem var(--serif);color:var(--ink);line-height:1.2}
96
+ .findings p{margin-top:6px;font-size:.9rem}
97
+
98
+ .multiples{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:16px}
99
+ .panel h3{font-size:1.2rem}
100
+ .panel .ends{font-size:.8rem;color:var(--soft);margin-top:3px}
101
+ .panel .ends b{font-weight:500;color:var(--ink);font-variant-numeric:tabular-nums}
102
+ .chart{position:relative;margin-top:6px}
103
+
104
+ .fam{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:16px;margin-top:18px}
105
+ .fam h3{font-size:1.15rem;margin-bottom:6px}
106
+
107
+ .decl{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:16px;margin-top:22px}
108
+
109
+ .m-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:26px 44px;margin-top:22px}
110
+ .m-grid h3{font-size:1.15rem;margin-bottom:6px}
111
+ .m-grid p{font-size:.93rem}
112
+ .foot{padding-block:26px;font-size:.82rem;color:var(--muted)}
113
+
114
+ .forest{position:relative;display:block;width:190px;height:16px}
115
+ .forest s{position:absolute;top:0;bottom:0;border-left:1.5px dashed var(--chance)}
116
+ .forest i{position:absolute;top:7px;height:2px;background:var(--knows);border-radius:2px}
117
+ .forest b{position:absolute;top:4px;width:8px;height:8px;margin-left:-4px;border-radius:50%;background:var(--knows)}
118
+ table.pit{min-width:640px}
119
+
120
+ .tip{position:fixed;z-index:70;pointer-events:none;background:#0f1528;color:#fff;font-size:.78rem;line-height:1.45;padding:7px 10px;border-radius:8px;max-width:240px;opacity:0;transition:opacity .12s;left:0;top:0;font-variant-numeric:tabular-nums}
121
+ .tip.show{opacity:1}
122
+ .tip b{font-weight:600}
123
 
124
+ @media (max-width:900px){
125
+ .wrap{padding-left:max(16px,env(safe-area-inset-left,0px));padding-right:max(16px,env(safe-area-inset-right,0px))}
126
+ .header{position:static}
127
+ .hero{padding-block:40px}
128
+ .hero-grid,.multiples,.fam,.decl,.m-grid,.findings{grid-template-columns:minmax(0,1fr)}
129
+ .hero-grid{gap:30px}
130
+ .stats{grid-template-columns:repeat(2,auto)}
131
+ .rings{opacity:.4;right:-320px}
132
+ .section{padding-block:42px}
133
  }
134
+ @media (prefers-reduced-motion:reduce){html{scroll-behavior:auto}}