maxidl Claude Opus 5.5 commited on
Commit
f72f04c
Β·
1 Parent(s): cd4d848

Metric selector: separate strict-match and flexible-extract rows

Browse files

The *_gsm8k_platinum_cot tasks have exact_match rows for two answer filters.
The chart queried by task and metric only, so each series drew both rows per
step. Metrics with several filters now get one option per filter
(strict-match first, the filter the suite aggregates use), and the queries
filter on metric_filter.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Files changed (1) hide show
  1. index.html +29 -7
index.html CHANGED
@@ -552,6 +552,15 @@
552
  function esc(s) { return s.replace(/'/g, "''"); }
553
  function sqlIn(vals) { return vals.map(v => `'${esc(v)}'`).join(', '); }
554
 
 
 
 
 
 
 
 
 
 
555
  // ── Shared helpers ──────────────────────────────────────────
556
  function populateSelect(el, options, selected) {
557
  el.innerHTML = '';
@@ -1322,10 +1331,23 @@
1322
  if (!task) return;
1323
 
1324
  const rows = await query(`
1325
- SELECT DISTINCT metric FROM scores WHERE task = '${esc(task)}' ORDER BY metric
 
1326
  `);
1327
- const prev = defaults?.metric || this.el.metric.value;
1328
- populateSelect(this.el.metric, rows.map(r => r.metric), prev);
 
 
 
 
 
 
 
 
 
 
 
 
1329
  if (defaults?.chartType) this.el.chartType.value = defaults.chartType;
1330
  await this.renderChart();
1331
  }
@@ -1341,7 +1363,7 @@
1341
  }
1342
  const available = await query(`
1343
  SELECT DISTINCT model_display_name FROM scores
1344
- WHERE task = '${esc(task)}' AND metric = '${esc(metric)}'
1345
  `);
1346
  const availableSet = new Set(available.map(r => r.model_display_name));
1347
  nameEls.forEach(el => {
@@ -1381,7 +1403,7 @@
1381
  is_checkpoint, higher_is_better, step
1382
  FROM scores
1383
  WHERE task = '${esc(task)}'
1384
- AND metric = '${esc(metric)}'
1385
  AND model_display_name IN (${sqlIn(models)})
1386
  ORDER BY model_display_name, tokens_trained
1387
  `);
@@ -1403,7 +1425,7 @@
1403
  try {
1404
  const stRows = await query(`
1405
  SELECT DISTINCT subtask_tree FROM scores
1406
- WHERE task = '${esc(task)}' AND metric = '${esc(metric)}'
1407
  AND subtask_tree IS NOT NULL
1408
  `);
1409
  if (stRows.length > 0) {
@@ -1485,7 +1507,7 @@
1485
 
1486
  formatChartTitle(task, metric, higherIsBetter) {
1487
  const arrow = higherIsBetter === true ? ' \u2191' : higherIsBetter === false ? ' \u2193' : '';
1488
- return `${task} \u2014 ${metric}${arrow}`;
1489
  }
1490
 
1491
  renderSubtaskTree(map, keys, depth = 0) {
 
552
  function esc(s) { return s.replace(/'/g, "''"); }
553
  function sqlIn(vals) { return vals.map(v => `'${esc(v)}'`).join(', '); }
554
 
555
+ // A metric option is "metric", or "metric|filter" when the task has rows for several answer filters
556
+ // (e.g. *_gsm8k_platinum_cot exact_match: strict-match and flexible-extract)
557
+ function splitMetric(v) { const i = v.indexOf('|'); return i < 0 ? [v, null] : [v.slice(0, i), v.slice(i + 1)]; }
558
+ function metricWhere(v) {
559
+ const [m, f] = splitMetric(v);
560
+ return `metric = '${esc(m)}'` + (f ? ` AND COALESCE(metric_filter, 'none') = '${esc(f)}'` : '');
561
+ }
562
+ function metricLabel(v) { const [m, f] = splitMetric(v); return f ? `${m} (${f})` : m; }
563
+
564
  // ── Shared helpers ──────────────────────────────────────────
565
  function populateSelect(el, options, selected) {
566
  el.innerHTML = '';
 
1331
  if (!task) return;
1332
 
1333
  const rows = await query(`
1334
+ SELECT DISTINCT metric, COALESCE(metric_filter, 'none') AS metric_filter
1335
+ FROM scores WHERE task = '${esc(task)}' ORDER BY metric, metric_filter
1336
  `);
1337
+ const filters = new Map();
1338
+ rows.forEach(r => filters.set(r.metric, [...(filters.get(r.metric) || []), r.metric_filter]));
1339
+ const options = [];
1340
+ for (const [m, fs] of filters) {
1341
+ if (fs.length < 2) { options.push(m); continue; }
1342
+ // strict-match first: it is the filter the suite aggregates use
1343
+ fs.sort((a, b) => (b === 'strict-match') - (a === 'strict-match') || a.localeCompare(b));
1344
+ fs.forEach(f => options.push({ value: `${m}|${f}`, label: `${m} (${f})` }));
1345
+ }
1346
+ let prev = defaults?.metric || this.el.metric.value;
1347
+ if (prev && !prev.includes('|') && (filters.get(prev) || []).length > 1) {
1348
+ prev = options.find(o => typeof o === 'object' && o.value.startsWith(prev + '|')).value;
1349
+ }
1350
+ populateSelect(this.el.metric, options, prev);
1351
  if (defaults?.chartType) this.el.chartType.value = defaults.chartType;
1352
  await this.renderChart();
1353
  }
 
1363
  }
1364
  const available = await query(`
1365
  SELECT DISTINCT model_display_name FROM scores
1366
+ WHERE task = '${esc(task)}' AND ${metricWhere(metric)}
1367
  `);
1368
  const availableSet = new Set(available.map(r => r.model_display_name));
1369
  nameEls.forEach(el => {
 
1403
  is_checkpoint, higher_is_better, step
1404
  FROM scores
1405
  WHERE task = '${esc(task)}'
1406
+ AND ${metricWhere(metric)}
1407
  AND model_display_name IN (${sqlIn(models)})
1408
  ORDER BY model_display_name, tokens_trained
1409
  `);
 
1425
  try {
1426
  const stRows = await query(`
1427
  SELECT DISTINCT subtask_tree FROM scores
1428
+ WHERE task = '${esc(task)}' AND ${metricWhere(metric)}
1429
  AND subtask_tree IS NOT NULL
1430
  `);
1431
  if (stRows.length > 0) {
 
1507
 
1508
  formatChartTitle(task, metric, higherIsBetter) {
1509
  const arrow = higherIsBetter === true ? ' \u2191' : higherIsBetter === false ? ' \u2193' : '';
1510
+ return `${task} \u2014 ${metricLabel(metric)}${arrow}`;
1511
  }
1512
 
1513
  renderSubtaskTree(map, keys, depth = 0) {