Zauberman commited on
Commit
2e2f4fb
Β·
verified Β·
1 Parent(s): af09fa5

Upload benchmark_dashboard.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. benchmark_dashboard.py +305 -0
benchmark_dashboard.py ADDED
@@ -0,0 +1,305 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Benchmark Dashboard Generator
4
+ Reads benchmark_results.json and produces a self-contained interactive HTML dashboard.
5
+ Usage: python3 benchmark_dashboard.py [--input results.json] [--output dashboard.html]
6
+ """
7
+
8
+ import json
9
+ import os
10
+ import argparse
11
+ from pathlib import Path
12
+ from datetime import datetime
13
+
14
+ SCRIPT_DIR = Path(__file__).resolve().parent
15
+ DEFAULT_INPUT = SCRIPT_DIR / "benchmark_results" / "benchmark_results.json"
16
+ DEFAULT_OUTPUT = SCRIPT_DIR / "benchmark_results" / "dashboard.html"
17
+
18
+ # ── Subject Categories in Order ────────────────────────────────
19
+
20
+ CATEGORIES = [
21
+ ("OHADA Business Law", 40),
22
+ ("Tax Code", 44),
23
+ ("Labor & Social", 20),
24
+ ("Public Procurement", 15),
25
+ ("Banking & Finance", 15),
26
+ ("Civil & Real Estate", 20),
27
+ ("Digital Economy", 10),
28
+ ("Environment & Energy", 10),
29
+ ("Scenarios", 34),
30
+ ("Refusal Handling", 3),
31
+ ("Tone & Identity", 3),
32
+ ]
33
+
34
+
35
+ def load_data(path):
36
+ with open(path, "r", encoding="utf-8") as f:
37
+ return json.load(f)
38
+
39
+
40
+ def build_html(results: dict) -> str:
41
+ model_names = list(results.keys())
42
+ data_json = json.dumps(results, ensure_ascii=False)
43
+ categories_json = json.dumps([c[0] for c in CATEGORIES], ensure_ascii=False)
44
+ model_names_json = json.dumps(model_names, ensure_ascii=False)
45
+
46
+ return f"""<!DOCTYPE html>
47
+ <html lang="fr">
48
+ <head>
49
+ <meta charset="utf-8">
50
+ <meta name="viewport" content="width=device-width, initial-scale=1">
51
+ <title>Solenoid Benchmark Dashboard</title>
52
+ <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.7/dist/chart.umd.min.js"></script>
53
+ <script src="https://cdn.jsdelivr.net/npm/chartjs-plugin-datalabels@2.2.0/dist/chartjs-plugin-datalabels.min.js"></script>
54
+ <style>
55
+ :root {{ --bg: #fff; --fg: #111; --card: #f9fafb; --border: #e5e7eb; --muted: #6b7280; --green: #16a34a; --blue: #2563eb; --red: #dc2626; }}
56
+ @media (prefers-color-scheme: dark) {{ :root {{ --bg: #111; --fg: #eee; --card: #1a1a2e; --border: #333; --muted: #9ca3af; }} }}
57
+ * {{ box-sizing:border-box; margin:0; padding:0; }}
58
+ body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; background: var(--bg); color: var(--fg); padding: 24px; max-width:1100px; margin:0 auto; }}
59
+ h1 {{ font-size: 1.5rem; margin-bottom:4px; }}
60
+ h2 {{ font-size: 1.1rem; margin: 24px 0 12px; border-bottom:1px solid var(--border); padding-bottom:6px; }}
61
+ .meta {{ color: var(--muted); font-size: .8rem; margin-bottom:16px; }}
62
+ .cards {{ display:grid; grid-template-columns: repeat(auto-fit, minmax(200px,1fr)); gap:12px; margin-bottom:24px; }}
63
+ .card {{ background: var(--card); border:1px solid var(--border); border-radius:12px; padding:16px; text-align:center; }}
64
+ .card .label {{ font-size:.75rem; color:var(--muted); text-transform:uppercase; }}
65
+ .card .value {{ font-size:1.8rem; font-weight:700; }}
66
+ .card .gain {{ font-size:.8rem; margin-top:4px; }}
67
+ .gain-up {{ color:var(--green); }}
68
+ .gain-down {{ color:var(--red); }}
69
+ .table-wrap {{ overflow-x:auto; }}
70
+ table {{ width:100%; border-collapse:collapse; font-size:.85rem; }}
71
+ th, td {{ padding:8px 12px; text-align:left; border-bottom:1px solid var(--border); }}
72
+ th {{ font-weight:600; background:var(--card); position:sticky; top:0; }}
73
+ .chart-box {{ background:var(--card); border:1px solid var(--border); border-radius:12px; padding:16px; margin-bottom:16px; }}
74
+ .chart-box canvas {{ max-height:400px; }}
75
+ .comparison {{ margin-bottom:20px; }}
76
+ .comp-item {{ background:var(--card); border:1px solid var(--border); border-radius:12px; padding:16px; margin-bottom:12px; }}
77
+ .comp-item .q {{ font-weight:600; margin-bottom:8px; font-size:.9rem; }}
78
+ .comp-item .label {{ font-size:.7rem; color:var(--muted); text-transform:uppercase; margin-bottom:2px; }}
79
+ .comp-item .a {{ font-size:.82rem; line-height:1.5; color:var(--muted); border-left:3px solid var(--border); padding-left:12px; margin-bottom:8px; }}
80
+ .comp-item .a-sol {{ border-left-color: var(--green); }}
81
+ .generate-date {{ font-size:.75rem; color:var(--muted); text-align:center; margin-top:32px; }}
82
+ </style>
83
+ </head>
84
+ <body>
85
+
86
+ <h1>Solenoid Benchmark Dashboard</h1>
87
+ <p class="meta">{datetime.now().strftime('%Y-%m-%d %H:%M')} β€” {len(results[list(results.keys())[0]])} questions</p>
88
+
89
+ <div class="cards" id="cards"></div>
90
+
91
+ <h2>Per-Subject Domain References</h2>
92
+ <div class="chart-box"><canvas id="barChart"></canvas></div>
93
+
94
+ <h2>Domain Coverage (Radar)</h2>
95
+ <div class="chart-box" style="max-width:500px; margin:0 auto;"><canvas id="radarChart"></canvas></div>
96
+
97
+ <h2>Subject Score Table</h2>
98
+ <div class="table-wrap"><table id="scoreTable"><thead><tr><th>Subject</th></tr></thead><tbody></tbody></table></div>
99
+
100
+ <h2>Highest Divergence β€” Side-by-Side</h2>
101
+ <div class="comparison" id="comparison"></div>
102
+
103
+ <p class="generate-date">Generated by benchmark_dashboard.py</p>
104
+
105
+ <script>
106
+ const DATA = {data_json};
107
+ const MODELS = {model_names_json};
108
+ const CATS = {categories_json};
109
+
110
+ // ── Compute Stats ─────────────────────────────────────────────
111
+ const subjectCounts = {json.dumps([c[1] for c in CATEGORIES])};
112
+ const totalQ = subjectCounts.reduce((a,b)=>a+b,0);
113
+
114
+ function avg(arr, fn) {{ return arr.reduce((s,r)=>s+fn(r),0) / Math.max(arr.length,1); }}
115
+ function score(r) {{ return r.eval.ohada_hits + r.eval.niger_hits + r.eval.citation_hits; }}
116
+
117
+ const globalStats = MODELS.map(m => {{
118
+ const arr = DATA[m];
119
+ return {{
120
+ model: m,
121
+ ohada: avg(arr, r=>r.eval.ohada_hits),
122
+ niger: avg(arr, r=>r.eval.niger_hits),
123
+ cites: avg(arr, r=>r.eval.citation_hits),
124
+ length: avg(arr, r=>r.eval.length),
125
+ total: avg(arr, score),
126
+ }};
127
+ }});
128
+
129
+ // ── Winning model ──────────────────────────────────────────────
130
+ const best = globalStats.reduce((a,b) => a.total > b.total ? a : b, globalStats[0]);
131
+ const worst = globalStats.reduce((a,b) => a.total < b.total ? a : b, globalStats[0]);
132
+
133
+ // ── Cards ──────────────────────────────────────────────────────
134
+ document.getElementById('cards').innerHTML = MODELS.map((m,i) => {{
135
+ const s = globalStats[i];
136
+ const isWinner = m === best.model;
137
+ const gain = globalStats.length>1 && i>0 ? Math.round((s.total / globalStats[0].total - 1) * 100) : 0;
138
+ const gainLabel = gain > 0 ? '<span class="gain gain-up">+'+gain+'% vs '+MODELS[0]+'</span>' : (gain < 0 ? '<span class="gain gain-down">'+gain+'%</span>' : '');
139
+ const cardColor = isWinner ? 'border-color:var(--green);' : '';
140
+ return `<div class="card" style="${{cardColor}}">
141
+ <div class="label">${{m}}</div>
142
+ <div class="value">${{s.total.toFixed(1)}}</div>
143
+ <div style="font-size:.75rem;color:var(--muted);">OHADA ${{s.ohada.toFixed(1)}} Β· Niger ${{s.niger.toFixed(1)}} Β· Cites ${{s.cites.toFixed(1)}}</div>
144
+ ${{gainLabel}}
145
+ </div>`;
146
+ }}).join('');
147
+
148
+ // ── Score Table ────────────────────────────────────────────────
149
+ const thead = document.querySelector('#scoreTable thead tr');
150
+ MODELS.forEach(m => {{
151
+ const th = document.createElement('th');
152
+ th.textContent = m;
153
+ thead.appendChild(th);
154
+ const th2 = document.createElement('th');
155
+ th2.textContent = m + ' (score)';
156
+ thead.appendChild(th2);
157
+ }});
158
+
159
+ const tbody = document.querySelector('#scoreTable tbody');
160
+ let offset = 0;
161
+ CATS.forEach((cat, ci) => {{
162
+ const count = subjectCounts[ci];
163
+ const tr = document.createElement('tr');
164
+ const td = document.createElement('td');
165
+ td.innerHTML = `<strong>${{cat}}</strong><br><span style="font-size:.7rem;color:var(--muted)">${{count}} questions</span>`;
166
+ tr.appendChild(td);
167
+ MODELS.forEach(m => {{
168
+ const arr = DATA[m].slice(offset, offset+count);
169
+ const avgO = avg(arr, r=>r.eval.ohada_hits);
170
+ const avgN = avg(arr, r=>r.eval.niger_hits);
171
+ const avgC = avg(arr, r=>r.eval.citation_hits);
172
+ const avgS = avg(arr, score);
173
+ const td1 = document.createElement('td');
174
+ td1.innerHTML = `O:${{avgO.toFixed(1)}} N:${{avgN.toFixed(1)}} C:${{avgC.toFixed(1)}}`;
175
+ td1.style.fontSize = '.8rem';
176
+ tr.appendChild(td1);
177
+ const td2 = document.createElement('td');
178
+ td2.innerHTML = avgS.toFixed(1);
179
+ td2.style.fontWeight = '600';
180
+ tr.appendChild(td2);
181
+ }});
182
+ tbody.appendChild(tr);
183
+ offset += count;
184
+ }});
185
+
186
+ // ── Convert labels ─────────────────────────────────────────────
187
+ function getLabel(idx) {{
188
+ let off=0;
189
+ for (let i=0; i<CATS.length; i++) {{
190
+ if (idx < off + subjectCounts[i]) return CATS[i];
191
+ off += subjectCounts[i];
192
+ }}
193
+ return '';
194
+ }}
195
+
196
+ // ── Bar Chart ──────────────────────────────────────────────────
197
+ const barCtx = document.getElementById('barChart').getContext('2d');
198
+ const barDatasets = MODELS.map((m,i) => {{
199
+ const colors = ['#94a3b8', '#16a34a', '#2563eb', '#dc2626'];
200
+ const data = CATS.map((cat, ci) => {{
201
+ const off = subjectCounts.slice(0,ci).reduce((a,b)=>a+b,0);
202
+ const count = subjectCounts[ci];
203
+ const arr = DATA[m].slice(off, off+count);
204
+ return avg(arr, score);
205
+ }});
206
+ return {{
207
+ label: m,
208
+ data: data,
209
+ backgroundColor: colors[i % colors.length] + '99',
210
+ borderColor: colors[i % colors.length],
211
+ borderWidth: 1.5,
212
+ }};
213
+ }});
214
+ new Chart(barCtx, {{
215
+ type: 'bar',
216
+ data: {{ labels: CATS, datasets: barDatasets }},
217
+ options: {{
218
+ responsive: true,
219
+ plugins: {{ legend: {{ position:'bottom' }} }},
220
+ scales: {{ y: {{ beginAtZero:true, title:{{display:true, text:'Domain Reference Score'}} }} }},
221
+ }},
222
+ plugins: [ChartDataLabels],
223
+ }});
224
+
225
+ // ── Radar Chart ─────────────────────────────────────────────────
226
+ const radarCtx = document.getElementById('radarChart').getContext('2d');
227
+ const radarDatasets = MODELS.map((m,i) => {{
228
+ const colors = ['#94a3b8', '#16a34a', '#2563eb', '#dc2626'];
229
+ const data = CATS.map((cat, ci) => {{
230
+ const off = subjectCounts.slice(0,ci).reduce((a,b)=>a+b,0);
231
+ const count = subjectCounts[ci];
232
+ const arr = DATA[m].slice(off, off+count);
233
+ return avg(arr, score);
234
+ }});
235
+ return {{
236
+ label: m,
237
+ data: data,
238
+ borderColor: colors[i % colors.length],
239
+ backgroundColor: colors[i % colors.length] + '33',
240
+ pointBackgroundColor: colors[i % colors.length],
241
+ }};
242
+ }});
243
+ new Chart(radarCtx, {{
244
+ type: 'radar',
245
+ data: {{ labels: CATS.slice(0,6), datasets: radarDatasets.map(d=>({{...d, data: d.data.slice(0,6)}})) }},
246
+ options: {{
247
+ responsive: true,
248
+ plugins: {{ legend: {{ position:'bottom' }} }},
249
+ scales: {{ r: {{ beginAtZero:true, min:0 }} }},
250
+ }},
251
+ }});
252
+
253
+ // ── Side-by-Side Divergence ─────────────────────────────────────
254
+ const diffs = [];
255
+ for (let i=0; i<DATA[MODELS[0]].length; i++) {{
256
+ const scores = MODELS.map(m => score(DATA[m][i]));
257
+ const max = Math.max(...scores);
258
+ const min = Math.min(...scores);
259
+ diffs.push({{ idx:i, gap: max-min, label: DATA[MODELS[0]][i].label, q: DATA[MODELS[0]][i].question }});
260
+ }}
261
+ diffs.sort((a,b) => b.gap - a.gap);
262
+ const topDiffs = diffs.slice(0, 10);
263
+
264
+ document.getElementById('comparison').innerHTML = topDiffs.map(d => {{
265
+ const responses = MODELS.map(m => {{
266
+ const r = DATA[m][d.idx];
267
+ return `<div style="margin-top:6px;">
268
+ <div class="label">${{m}} (${{r.eval.ohada_hits + r.eval.niger_hits}} refs, ${{r.eval.citation_hits}} cites)</div>
269
+ <div class="a ${{m.includes('Solenoid') ? 'a-sol' : ''}}">${{(r.response||'').substring(0,400)}}</div>
270
+ </div>`;
271
+ }}).join('');
272
+ return `<div class="comp-item">
273
+ <div class="q">${{d.idx+1}}. ${{d.label}}: ${{d.q.substring(0,120)}}</div>
274
+ ${{responses}}
275
+ </div>`;
276
+ }}).join('');
277
+ </script>
278
+ </body>
279
+ </html>"""
280
+
281
+
282
+ def main():
283
+ parser = argparse.ArgumentParser(description="Benchmark Dashboard Generator")
284
+ parser.add_argument("--input", default=str(DEFAULT_INPUT), help="Path to benchmark_results.json")
285
+ parser.add_argument("--output", default=str(DEFAULT_OUTPUT), help="Path for output HTML file")
286
+ args = parser.parse_args()
287
+
288
+ if not os.path.exists(args.input):
289
+ print(f"ERROR: {args.input} not found. Run benchmark_solenoid.py first.")
290
+ return 1
291
+
292
+ data = load_data(args.input)
293
+ html = build_html(data)
294
+
295
+ os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
296
+ with open(args.output, "w", encoding="utf-8") as f:
297
+ f.write(html)
298
+
299
+ print(f"Dashboard generated: {args.output}")
300
+ print(f"Open with: xdg-open {args.output} (or any browser)")
301
+ return 0
302
+
303
+
304
+ if __name__ == "__main__":
305
+ exit(main())