Merge branch 'fix/tools-visualization-filters' into 'prod'
Browse filesfeat: refine leaderboard metrics, metadata, and visualizations
See merge request ai-security/ht-ml-processes/leaderboard_frontend!11
- README.md +7 -1
- messages/en.json +4 -8
- messages/ru.json +4 -8
- reference/index.html +0 -0
- src/lib/data/details.ts +25 -5
- src/lib/data/ranking.ts +1 -0
- src/lib/server/hf-bucket/adapt-details.ts +3 -1
- src/lib/server/hf-bucket/adapt-ranking.ts +4 -23
- src/lib/server/hf-bucket/ranking-payload.ts +9 -1
- src/lib/server/hf-bucket/tools-snapshot.ts +3 -1
- src/lib/server/hf-bucket/types.ts +4 -2
- src/lib/server/hf-bucket/visualizations-payload.ts +2 -2
- src/lib/types/details.ts +7 -2
- src/lib/types/leaderboard.ts +3 -0
- src/lib/types/ranking.ts +1 -0
- src/lib/types/tools-data.ts +2 -2
- src/routes/+page.svelte +3 -8
- src/routes/details/+page.svelte +30 -6
- src/routes/tools/+page.svelte +29 -82
README.md
CHANGED
|
@@ -54,7 +54,7 @@ Required server-only environment variables:
|
|
| 54 |
|
| 55 |
```env
|
| 56 |
HF_TOKEN=hf_...
|
| 57 |
-
HF_BUCKET_ID=
|
| 58 |
HF_BUCKET_PREFIX=latest
|
| 59 |
```
|
| 60 |
|
|
@@ -84,6 +84,12 @@ Lint:
|
|
| 84 |
pnpm lint
|
| 85 |
```
|
| 86 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
Check SvelteKit, TypeScript, and Svelte components:
|
| 88 |
|
| 89 |
```bash
|
|
|
|
| 54 |
|
| 55 |
```env
|
| 56 |
HF_TOKEN=hf_...
|
| 57 |
+
HF_BUCKET_ID=...
|
| 58 |
HF_BUCKET_PREFIX=latest
|
| 59 |
```
|
| 60 |
|
|
|
|
| 84 |
pnpm lint
|
| 85 |
```
|
| 86 |
|
| 87 |
+
Check Tailwind utility classes:
|
| 88 |
+
|
| 89 |
+
```bash
|
| 90 |
+
pnpm tailwind:check
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
Check SvelteKit, TypeScript, and Svelte components:
|
| 94 |
|
| 95 |
```bash
|
messages/en.json
CHANGED
|
@@ -74,8 +74,6 @@
|
|
| 74 |
"ranking_details_meta_tab": "Meta",
|
| 75 |
"ranking_details_overall_f1": "Overall F1",
|
| 76 |
"ranking_details_p50_p95": "p50 / p95 ms",
|
| 77 |
-
"ranking_details_generative_description": "Generative guard based on {family}; multilingual, intent-aware, and returns a safety verdict.",
|
| 78 |
-
"ranking_details_classifier_description": "Classifier guard based on {family}, optimized for fast safety scoring and filtering.",
|
| 79 |
"ranking_details_full_name": "Full name",
|
| 80 |
"ranking_details_family": "Family",
|
| 81 |
"ranking_details_parameters": "Parameters",
|
|
@@ -203,6 +201,7 @@
|
|
| 203 |
"details_benchmark_sheet_ranking": "Model ranking on this benchmark",
|
| 204 |
"details_benchmark_sheet_rank": "Rank",
|
| 205 |
"details_benchmark_sheet_open": "Open benchmark",
|
|
|
|
| 206 |
"details_model_sheet_open": "Open model",
|
| 207 |
"details_model_sheet_overview": "Overview",
|
| 208 |
"details_model_sheet_groups": "Groups",
|
|
@@ -216,7 +215,7 @@
|
|
| 216 |
"details_model_sheet_score_legend": "group score (higher is better)",
|
| 217 |
"details_model_sheet_groups_with_data": "{count} groups with data",
|
| 218 |
"details_model_sheet_weak_title": "Weakest benchmark groups",
|
| 219 |
-
"details_model_sheet_weak_hint": "5 groups with the lowest score.
|
| 220 |
"details_model_sheet_all_benchmarks": "Go to all benchmarks",
|
| 221 |
"details_model_sheet_full_name": "Full name",
|
| 222 |
"details_model_sheet_family": "Family",
|
|
@@ -243,10 +242,7 @@
|
|
| 243 |
"tools_models_title": "Models",
|
| 244 |
"tools_groups_title": "Groups / benchmarks",
|
| 245 |
"tools_language_label": "Language",
|
| 246 |
-
"tools_metric_label": "Metric (scatter/bar/heatmap)",
|
| 247 |
"tools_language_all": "All languages",
|
| 248 |
-
"tools_metric_fnr": "FNR - missed harm",
|
| 249 |
-
"tools_metric_fpr": "FPR - false alarms",
|
| 250 |
"tools_pin": "Pin",
|
| 251 |
"tools_unpin": "unpin",
|
| 252 |
"tools_pinned": "pinned",
|
|
@@ -272,8 +268,8 @@
|
|
| 272 |
"tools_heatmap_header": "{metric} · group / model",
|
| 273 |
"tools_heatmap_empty": "Select at least one model and one group to show the heatmap.",
|
| 274 |
"tools_grouped_bars_title": "Grouped bars",
|
| 275 |
-
"tools_grouped_bars_hint": "
|
| 276 |
-
"tools_grouped_bars_aria": "Grouped
|
| 277 |
"tools_lower_better": "lower = better",
|
| 278 |
"tools_higher_better": "higher = better",
|
| 279 |
"tools_pareto_title": "Pareto: quality × latency",
|
|
|
|
| 74 |
"ranking_details_meta_tab": "Meta",
|
| 75 |
"ranking_details_overall_f1": "Overall F1",
|
| 76 |
"ranking_details_p50_p95": "p50 / p95 ms",
|
|
|
|
|
|
|
| 77 |
"ranking_details_full_name": "Full name",
|
| 78 |
"ranking_details_family": "Family",
|
| 79 |
"ranking_details_parameters": "Parameters",
|
|
|
|
| 201 |
"details_benchmark_sheet_ranking": "Model ranking on this benchmark",
|
| 202 |
"details_benchmark_sheet_rank": "Rank",
|
| 203 |
"details_benchmark_sheet_open": "Open benchmark",
|
| 204 |
+
"details_benchmark_leader_f1": "LEADER · F1",
|
| 205 |
"details_model_sheet_open": "Open model",
|
| 206 |
"details_model_sheet_overview": "Overview",
|
| 207 |
"details_model_sheet_groups": "Groups",
|
|
|
|
| 215 |
"details_model_sheet_score_legend": "group score (higher is better)",
|
| 216 |
"details_model_sheet_groups_with_data": "{count} groups with data",
|
| 217 |
"details_model_sheet_weak_title": "Weakest benchmark groups",
|
| 218 |
+
"details_model_sheet_weak_hint": "5 groups with the lowest score. Higher is better for Recall/Precision and score/F1.",
|
| 219 |
"details_model_sheet_all_benchmarks": "Go to all benchmarks",
|
| 220 |
"details_model_sheet_full_name": "Full name",
|
| 221 |
"details_model_sheet_family": "Family",
|
|
|
|
| 242 |
"tools_models_title": "Models",
|
| 243 |
"tools_groups_title": "Groups / benchmarks",
|
| 244 |
"tools_language_label": "Language",
|
|
|
|
| 245 |
"tools_language_all": "All languages",
|
|
|
|
|
|
|
| 246 |
"tools_pin": "Pin",
|
| 247 |
"tools_unpin": "unpin",
|
| 248 |
"tools_pinned": "pinned",
|
|
|
|
| 268 |
"tools_heatmap_header": "{metric} · group / model",
|
| 269 |
"tools_heatmap_empty": "Select at least one model and one group to show the heatmap.",
|
| 270 |
"tools_grouped_bars_title": "Grouped bars",
|
| 271 |
+
"tools_grouped_bars_hint": "Recall / Precision / F1 by model · averages across selected groups",
|
| 272 |
+
"tools_grouped_bars_aria": "Grouped Recall Precision F1 bars by model",
|
| 273 |
"tools_lower_better": "lower = better",
|
| 274 |
"tools_higher_better": "higher = better",
|
| 275 |
"tools_pareto_title": "Pareto: quality × latency",
|
messages/ru.json
CHANGED
|
@@ -74,8 +74,6 @@
|
|
| 74 |
"ranking_details_meta_tab": "Meta",
|
| 75 |
"ranking_details_overall_f1": "Overall F1",
|
| 76 |
"ranking_details_p50_p95": "p50 / p95 мс",
|
| 77 |
-
"ranking_details_generative_description": "Generative guard на базе {family}, мультиязычный; распознаёт интент и генерирует safety-вердикт.",
|
| 78 |
-
"ranking_details_classifier_description": "Classifier guard на базе {family}, оптимизированный для быстрого safety-скоринга и фильтрации.",
|
| 79 |
"ranking_details_full_name": "Полное имя",
|
| 80 |
"ranking_details_family": "Семейство",
|
| 81 |
"ranking_details_parameters": "Параметры",
|
|
@@ -203,6 +201,7 @@
|
|
| 203 |
"details_benchmark_sheet_ranking": "Рейтинг моделей на этом бенчмарке",
|
| 204 |
"details_benchmark_sheet_rank": "Место",
|
| 205 |
"details_benchmark_sheet_open": "Открыть бенчмарк",
|
|
|
|
| 206 |
"details_model_sheet_open": "Открыть модель",
|
| 207 |
"details_model_sheet_overview": "Обзор",
|
| 208 |
"details_model_sheet_groups": "По группам",
|
|
@@ -216,7 +215,7 @@
|
|
| 216 |
"details_model_sheet_score_legend": "score группы (выше = лучше)",
|
| 217 |
"details_model_sheet_groups_with_data": "{count} групп с данными",
|
| 218 |
"details_model_sheet_weak_title": "Слабейшие бенчмарк-группы",
|
| 219 |
-
"details_model_sheet_weak_hint": "5 групп с наименьшим score.
|
| 220 |
"details_model_sheet_all_benchmarks": "Перейти ко всем бенчмаркам",
|
| 221 |
"details_model_sheet_full_name": "Полное имя",
|
| 222 |
"details_model_sheet_family": "Семейство",
|
|
@@ -243,10 +242,7 @@
|
|
| 243 |
"tools_models_title": "Модели",
|
| 244 |
"tools_groups_title": "Группы / бенчмарки",
|
| 245 |
"tools_language_label": "Язык",
|
| 246 |
-
"tools_metric_label": "Метрика (scatter/bar/heatmap)",
|
| 247 |
"tools_language_all": "Все языки",
|
| 248 |
-
"tools_metric_fnr": "FNR - пропуски вреда",
|
| 249 |
-
"tools_metric_fpr": "FPR - ложные тревоги",
|
| 250 |
"tools_pin": "Pin",
|
| 251 |
"tools_unpin": "unpin",
|
| 252 |
"tools_pinned": "закреплено",
|
|
@@ -272,8 +268,8 @@
|
|
| 272 |
"tools_heatmap_header": "{metric} · группа / модель",
|
| 273 |
"tools_heatmap_empty": "Выберите хотя бы одну модель и одну группу, чтобы показать heatmap.",
|
| 274 |
"tools_grouped_bars_title": "Сгруппированные бары",
|
| 275 |
-
"tools_grouped_bars_hint": "
|
| 276 |
-
"tools_grouped_bars_aria": "Сгруппированные бары
|
| 277 |
"tools_lower_better": "ниже = лучше",
|
| 278 |
"tools_higher_better": "выше = лучше",
|
| 279 |
"tools_pareto_title": "Парето: качество × задержка",
|
|
|
|
| 74 |
"ranking_details_meta_tab": "Meta",
|
| 75 |
"ranking_details_overall_f1": "Overall F1",
|
| 76 |
"ranking_details_p50_p95": "p50 / p95 мс",
|
|
|
|
|
|
|
| 77 |
"ranking_details_full_name": "Полное имя",
|
| 78 |
"ranking_details_family": "Семейство",
|
| 79 |
"ranking_details_parameters": "Параметры",
|
|
|
|
| 201 |
"details_benchmark_sheet_ranking": "Рейтинг моделей на этом бенчмарке",
|
| 202 |
"details_benchmark_sheet_rank": "Место",
|
| 203 |
"details_benchmark_sheet_open": "Открыть бенчмарк",
|
| 204 |
+
"details_benchmark_leader_f1": "ЛИДЕР · F1",
|
| 205 |
"details_model_sheet_open": "Открыть модель",
|
| 206 |
"details_model_sheet_overview": "Обзор",
|
| 207 |
"details_model_sheet_groups": "По группам",
|
|
|
|
| 215 |
"details_model_sheet_score_legend": "score группы (выше = лучше)",
|
| 216 |
"details_model_sheet_groups_with_data": "{count} групп с данными",
|
| 217 |
"details_model_sheet_weak_title": "Слабейшие бенчмарк-группы",
|
| 218 |
+
"details_model_sheet_weak_hint": "5 групп с наименьшим score. Recall/Precision и score/F1 - чем выше, тем лучше.",
|
| 219 |
"details_model_sheet_all_benchmarks": "Перейти ко всем бенчмаркам",
|
| 220 |
"details_model_sheet_full_name": "Полное имя",
|
| 221 |
"details_model_sheet_family": "Семейство",
|
|
|
|
| 242 |
"tools_models_title": "Модели",
|
| 243 |
"tools_groups_title": "Группы / бенчмарки",
|
| 244 |
"tools_language_label": "Язык",
|
|
|
|
| 245 |
"tools_language_all": "Все языки",
|
|
|
|
|
|
|
| 246 |
"tools_pin": "Pin",
|
| 247 |
"tools_unpin": "unpin",
|
| 248 |
"tools_pinned": "закреплено",
|
|
|
|
| 268 |
"tools_heatmap_header": "{metric} · группа / модель",
|
| 269 |
"tools_heatmap_empty": "Выберите хотя бы одну модель и одну группу, чтобы показать heatmap.",
|
| 270 |
"tools_grouped_bars_title": "Сгруппированные бары",
|
| 271 |
+
"tools_grouped_bars_hint": "Recall / Precision / F1 по моделям · средние по выбранным группам",
|
| 272 |
+
"tools_grouped_bars_aria": "Сгруппированные бары Recall Precision F1 по моделям",
|
| 273 |
"tools_lower_better": "ниже = лучше",
|
| 274 |
"tools_higher_better": "выше = лучше",
|
| 275 |
"tools_pareto_title": "Парето: качество × задержка",
|
reference/index.html
DELETED
|
The diff for this file is too large to render.
See raw diff
|
|
|
src/lib/data/details.ts
CHANGED
|
@@ -197,6 +197,25 @@ export function benchmarkRows(
|
|
| 197 |
});
|
| 198 |
}
|
| 199 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 200 |
export function buildBenchmarkCards(
|
| 201 |
search: string,
|
| 202 |
language: BenchmarkFamilyLanguage,
|
|
@@ -222,7 +241,8 @@ export function buildBenchmarkCards(
|
|
| 222 |
})
|
| 223 |
.map<BenchmarkCard>((group) => ({
|
| 224 |
name: group,
|
| 225 |
-
meta: dataset.benchmarkMeta[group]
|
|
|
|
| 226 |
}));
|
| 227 |
}
|
| 228 |
|
|
@@ -366,8 +386,8 @@ export function modelGroupRows(
|
|
| 366 |
.map<ModelGroupRow>(([name, group]) => ({
|
| 367 |
name: groupName(name, dataset, locale),
|
| 368 |
score: group.score,
|
| 369 |
-
|
| 370 |
-
|
| 371 |
}))
|
| 372 |
.sort((a, b) => b.score - a.score);
|
| 373 |
}
|
|
@@ -386,8 +406,8 @@ export function modelRadarGroups(
|
|
| 386 |
rows.push({
|
| 387 |
name: groupName(name, dataset, locale),
|
| 388 |
score: group.score,
|
| 389 |
-
|
| 390 |
-
|
| 391 |
});
|
| 392 |
}
|
| 393 |
|
|
|
|
| 197 |
});
|
| 198 |
}
|
| 199 |
|
| 200 |
+
function benchmarkLeader(
|
| 201 |
+
groupName: string,
|
| 202 |
+
models: readonly GuardModel[]
|
| 203 |
+
): BenchmarkCard['leader'] {
|
| 204 |
+
const metric = models.some((model) => isNumber(model.groups[groupName]?.f1)) ? 'f1' : 'score';
|
| 205 |
+
|
| 206 |
+
return models.reduce<BenchmarkCard['leader']>((leader, model) => {
|
| 207 |
+
const group = model.groups[groupName];
|
| 208 |
+
const value = metric === 'f1' ? group?.f1 : group?.score;
|
| 209 |
+
if (!isNumber(value) || (leader && leader.value >= value)) return leader;
|
| 210 |
+
|
| 211 |
+
return {
|
| 212 |
+
modelShort: model.short,
|
| 213 |
+
value,
|
| 214 |
+
metric
|
| 215 |
+
};
|
| 216 |
+
}, null);
|
| 217 |
+
}
|
| 218 |
+
|
| 219 |
export function buildBenchmarkCards(
|
| 220 |
search: string,
|
| 221 |
language: BenchmarkFamilyLanguage,
|
|
|
|
| 241 |
})
|
| 242 |
.map<BenchmarkCard>((group) => ({
|
| 243 |
name: group,
|
| 244 |
+
meta: dataset.benchmarkMeta[group],
|
| 245 |
+
leader: benchmarkLeader(group, dataset.models)
|
| 246 |
}));
|
| 247 |
}
|
| 248 |
|
|
|
|
| 386 |
.map<ModelGroupRow>(([name, group]) => ({
|
| 387 |
name: groupName(name, dataset, locale),
|
| 388 |
score: group.score,
|
| 389 |
+
recall: group.recall ?? null,
|
| 390 |
+
precision: group.precision ?? null
|
| 391 |
}))
|
| 392 |
.sort((a, b) => b.score - a.score);
|
| 393 |
}
|
|
|
|
| 406 |
rows.push({
|
| 407 |
name: groupName(name, dataset, locale),
|
| 408 |
score: group.score,
|
| 409 |
+
recall: group.recall ?? null,
|
| 410 |
+
precision: group.precision ?? null
|
| 411 |
});
|
| 412 |
}
|
| 413 |
|
src/lib/data/ranking.ts
CHANGED
|
@@ -124,6 +124,7 @@ export function benchmarkDetails(model: GuardModel, groups: readonly string[]):
|
|
| 124 |
.map((group) => ({
|
| 125 |
name: group,
|
| 126 |
score: model.groups[group]?.score ?? null,
|
|
|
|
| 127 |
fpr: model.groups[group]?.fpr ?? null,
|
| 128 |
fnr: model.groups[group]?.fnr ?? null,
|
| 129 |
f1: model.groups[group]?.f1 ?? null
|
|
|
|
| 124 |
.map((group) => ({
|
| 125 |
name: group,
|
| 126 |
score: model.groups[group]?.score ?? null,
|
| 127 |
+
minDatasetScore: model.groups[group]?.minDatasetScore ?? null,
|
| 128 |
fpr: model.groups[group]?.fpr ?? null,
|
| 129 |
fnr: model.groups[group]?.fnr ?? null,
|
| 130 |
f1: model.groups[group]?.f1 ?? null
|
src/lib/server/hf-bucket/adapt-details.ts
CHANGED
|
@@ -18,7 +18,9 @@ export function adaptBucketDetails(snapshot: BucketDetailsSnapshot): DetailsData
|
|
| 18 |
score: cell.score,
|
| 19 |
f1: cell.f1,
|
| 20 |
fpr: cell.fpr,
|
| 21 |
-
fnr: cell.fnr
|
|
|
|
|
|
|
| 22 |
};
|
| 23 |
groupsByModelId.set(cell.model_id, groups);
|
| 24 |
}
|
|
|
|
| 18 |
score: cell.score,
|
| 19 |
f1: cell.f1,
|
| 20 |
fpr: cell.fpr,
|
| 21 |
+
fnr: cell.fnr,
|
| 22 |
+
recall: cell.recall,
|
| 23 |
+
precision: cell.precision
|
| 24 |
};
|
| 25 |
groupsByModelId.set(cell.model_id, groups);
|
| 26 |
}
|
src/lib/server/hf-bucket/adapt-ranking.ts
CHANGED
|
@@ -86,8 +86,7 @@ function completeRankingRow(row: BucketLeaderboardRow): row is CompleteRankingRo
|
|
| 86 |
function adaptModel(
|
| 87 |
model: BucketModel,
|
| 88 |
row: BucketLeaderboardRow,
|
| 89 |
-
groupLabelById: Map<string, string>
|
| 90 |
-
partialModelIds: ReadonlySet<string>
|
| 91 |
): GuardModel | null {
|
| 92 |
if (!completeRankingRow(row)) {
|
| 93 |
console.warn(`[hf-bucket] Ranking skips incomplete model "${model.model_id}".`);
|
|
@@ -100,6 +99,7 @@ function adaptModel(
|
|
| 100 |
if (!label || metrics.score === null) continue;
|
| 101 |
groups[label] = {
|
| 102 |
score: metrics.score,
|
|
|
|
| 103 |
f1: metrics.f1,
|
| 104 |
fpr: metrics.fpr,
|
| 105 |
fnr: metrics.fnr
|
|
@@ -130,28 +130,12 @@ function adaptModel(
|
|
| 130 |
p99: row.latency_ms.p99,
|
| 131 |
errorRate: row.error_rate,
|
| 132 |
leak: model.eval_leak ?? false,
|
| 133 |
-
partial:
|
| 134 |
hfUrl: model.hf_url,
|
| 135 |
groups
|
| 136 |
};
|
| 137 |
}
|
| 138 |
|
| 139 |
-
function partialModelIds(snapshot: BucketRankingSnapshot | BucketDetailsSnapshot) {
|
| 140 |
-
if (!('detailsMatrix' in snapshot)) return new Set<string>();
|
| 141 |
-
|
| 142 |
-
const ids = new Set<string>();
|
| 143 |
-
for (const cell of snapshot.detailsMatrix.dataset_cells) {
|
| 144 |
-
if (
|
| 145 |
-
cell.metrics_evaluated_samples !== null &&
|
| 146 |
-
cell.sample_count !== null &&
|
| 147 |
-
cell.metrics_evaluated_samples < cell.sample_count
|
| 148 |
-
) {
|
| 149 |
-
ids.add(cell.model_id);
|
| 150 |
-
}
|
| 151 |
-
}
|
| 152 |
-
return ids;
|
| 153 |
-
}
|
| 154 |
-
|
| 155 |
function benchmarkMeta(groups: BucketGroup[], snapshot: BucketRankingSnapshot) {
|
| 156 |
const datasetsById = new Map(
|
| 157 |
snapshot.catalog.datasets.map((dataset) => [dataset.dataset_id, dataset])
|
|
@@ -181,13 +165,10 @@ export function adaptBucketRanking(
|
|
| 181 |
);
|
| 182 |
const catalogById = chooseLatest(snapshot.catalog.models, 'catalog.models');
|
| 183 |
const rowsById = chooseLatest(snapshot.leaderboard.rows, 'leaderboard.rows');
|
| 184 |
-
const partialIds = partialModelIds(snapshot);
|
| 185 |
const models = [...catalogById.values()]
|
| 186 |
.map((model) => {
|
| 187 |
const row = rowsById.get(model.model_id);
|
| 188 |
-
return row
|
| 189 |
-
? { model: adaptModel(model, row, groupLabelById, partialIds), rank: row.rank }
|
| 190 |
-
: null;
|
| 191 |
})
|
| 192 |
.filter(
|
| 193 |
(item): item is { model: GuardModel; rank: number } => item !== null && item.model !== null
|
|
|
|
| 86 |
function adaptModel(
|
| 87 |
model: BucketModel,
|
| 88 |
row: BucketLeaderboardRow,
|
| 89 |
+
groupLabelById: Map<string, string>
|
|
|
|
| 90 |
): GuardModel | null {
|
| 91 |
if (!completeRankingRow(row)) {
|
| 92 |
console.warn(`[hf-bucket] Ranking skips incomplete model "${model.model_id}".`);
|
|
|
|
| 99 |
if (!label || metrics.score === null) continue;
|
| 100 |
groups[label] = {
|
| 101 |
score: metrics.score,
|
| 102 |
+
minDatasetScore: metrics.min_dataset_score,
|
| 103 |
f1: metrics.f1,
|
| 104 |
fpr: metrics.fpr,
|
| 105 |
fnr: metrics.fnr
|
|
|
|
| 130 |
p99: row.latency_ms.p99,
|
| 131 |
errorRate: row.error_rate,
|
| 132 |
leak: model.eval_leak ?? false,
|
| 133 |
+
partial: model.partial,
|
| 134 |
hfUrl: model.hf_url,
|
| 135 |
groups
|
| 136 |
};
|
| 137 |
}
|
| 138 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
function benchmarkMeta(groups: BucketGroup[], snapshot: BucketRankingSnapshot) {
|
| 140 |
const datasetsById = new Map(
|
| 141 |
snapshot.catalog.datasets.map((dataset) => [dataset.dataset_id, dataset])
|
|
|
|
| 165 |
);
|
| 166 |
const catalogById = chooseLatest(snapshot.catalog.models, 'catalog.models');
|
| 167 |
const rowsById = chooseLatest(snapshot.leaderboard.rows, 'leaderboard.rows');
|
|
|
|
| 168 |
const models = [...catalogById.values()]
|
| 169 |
.map((model) => {
|
| 170 |
const row = rowsById.get(model.model_id);
|
| 171 |
+
return row ? { model: adaptModel(model, row, groupLabelById), rank: row.rank } : null;
|
|
|
|
|
|
|
| 172 |
})
|
| 173 |
.filter(
|
| 174 |
(item): item is { model: GuardModel; rank: number } => item !== null && item.model !== null
|
src/lib/server/hf-bucket/ranking-payload.ts
CHANGED
|
@@ -86,6 +86,9 @@ function parseModel(value: unknown, index: number): BucketModel {
|
|
| 86 |
if (item.eval_leak !== null && typeof item.eval_leak !== 'boolean') {
|
| 87 |
throw new Error(`Field "${field}.eval_leak" must be a boolean or null.`);
|
| 88 |
}
|
|
|
|
|
|
|
|
|
|
| 89 |
|
| 90 |
return {
|
| 91 |
model_id: string(item.model_id, `${field}.model_id`),
|
|
@@ -101,7 +104,8 @@ function parseModel(value: unknown, index: number): BucketModel {
|
|
| 101 |
note: nullableLocalized(item.note, `${field}.note`),
|
| 102 |
run_date: date(item.run_date, `${field}.run_date`),
|
| 103 |
hf_url: nullableString(item.hf_url, `${field}.hf_url`),
|
| 104 |
-
eval_leak: item.eval_leak
|
|
|
|
| 105 |
};
|
| 106 |
}
|
| 107 |
|
|
@@ -173,6 +177,10 @@ function parseGroupMetrics(value: unknown, field: string): BucketGroupMetrics {
|
|
| 173 |
const item = record(value, field);
|
| 174 |
return {
|
| 175 |
score: nullableNumber(item.score, `${field}.score`),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
f1: nullableNumber(item.f1, `${field}.f1`),
|
| 177 |
fpr: nullableNumber(item.fpr, `${field}.fpr`),
|
| 178 |
fnr: nullableNumber(item.fnr, `${field}.fnr`)
|
|
|
|
| 86 |
if (item.eval_leak !== null && typeof item.eval_leak !== 'boolean') {
|
| 87 |
throw new Error(`Field "${field}.eval_leak" must be a boolean or null.`);
|
| 88 |
}
|
| 89 |
+
if (item.partial !== undefined && item.partial !== null && typeof item.partial !== 'boolean') {
|
| 90 |
+
throw new Error(`Field "${field}.partial" must be a boolean or null when provided.`);
|
| 91 |
+
}
|
| 92 |
|
| 93 |
return {
|
| 94 |
model_id: string(item.model_id, `${field}.model_id`),
|
|
|
|
| 104 |
note: nullableLocalized(item.note, `${field}.note`),
|
| 105 |
run_date: date(item.run_date, `${field}.run_date`),
|
| 106 |
hf_url: nullableString(item.hf_url, `${field}.hf_url`),
|
| 107 |
+
eval_leak: item.eval_leak,
|
| 108 |
+
partial: item.partial === true
|
| 109 |
};
|
| 110 |
}
|
| 111 |
|
|
|
|
| 177 |
const item = record(value, field);
|
| 178 |
return {
|
| 179 |
score: nullableNumber(item.score, `${field}.score`),
|
| 180 |
+
min_dataset_score:
|
| 181 |
+
item.min_dataset_score === undefined
|
| 182 |
+
? undefined
|
| 183 |
+
: nullableNumber(item.min_dataset_score, `${field}.min_dataset_score`),
|
| 184 |
f1: nullableNumber(item.f1, `${field}.f1`),
|
| 185 |
fpr: nullableNumber(item.fpr, `${field}.fpr`),
|
| 186 |
fnr: nullableNumber(item.fnr, `${field}.fnr`)
|
src/lib/server/hf-bucket/tools-snapshot.ts
CHANGED
|
@@ -75,7 +75,9 @@ function verifyVisualizationSnapshot(snapshot: BucketVisualizationSnapshot) {
|
|
| 75 |
for (const modelId of Object.keys(row)) assertKnownId(modelIds, modelId, 'scatter.values');
|
| 76 |
}
|
| 77 |
|
| 78 |
-
const unknownHeatmapRowIds = snapshot.heatmap.row_ids.filter(
|
|
|
|
|
|
|
| 79 |
if (unknownHeatmapRowIds.length) {
|
| 80 |
console.warn(
|
| 81 |
'[hf-bucket] Ignoring heatmap rows that reference unknown dataset ids.',
|
|
|
|
| 75 |
for (const modelId of Object.keys(row)) assertKnownId(modelIds, modelId, 'scatter.values');
|
| 76 |
}
|
| 77 |
|
| 78 |
+
const unknownHeatmapRowIds = snapshot.heatmap.row_ids.filter(
|
| 79 |
+
(datasetId) => !datasetIds.has(datasetId)
|
| 80 |
+
);
|
| 81 |
if (unknownHeatmapRowIds.length) {
|
| 82 |
console.warn(
|
| 83 |
'[hf-bucket] Ignoring heatmap rows that reference unknown dataset ids.',
|
src/lib/server/hf-bucket/types.ts
CHANGED
|
@@ -78,6 +78,7 @@ export type BucketModel = {
|
|
| 78 |
run_date: string;
|
| 79 |
hf_url: string | null;
|
| 80 |
eval_leak: boolean | null;
|
|
|
|
| 81 |
};
|
| 82 |
|
| 83 |
export type BucketGroup = {
|
|
@@ -113,6 +114,7 @@ export type BucketCatalog = {
|
|
| 113 |
|
| 114 |
export type BucketGroupMetrics = {
|
| 115 |
score: NullableMetricValue;
|
|
|
|
| 116 |
f1: NullableMetricValue;
|
| 117 |
fpr: NullableMetricValue;
|
| 118 |
fnr: NullableMetricValue;
|
|
@@ -236,8 +238,8 @@ export type BucketGroupedBarsModel = {
|
|
| 236 |
groups: Record<
|
| 237 |
string,
|
| 238 |
{
|
| 239 |
-
|
| 240 |
-
|
| 241 |
f1: NullableMetricValue;
|
| 242 |
}
|
| 243 |
>;
|
|
|
|
| 78 |
run_date: string;
|
| 79 |
hf_url: string | null;
|
| 80 |
eval_leak: boolean | null;
|
| 81 |
+
partial: boolean;
|
| 82 |
};
|
| 83 |
|
| 84 |
export type BucketGroup = {
|
|
|
|
| 114 |
|
| 115 |
export type BucketGroupMetrics = {
|
| 116 |
score: NullableMetricValue;
|
| 117 |
+
min_dataset_score?: NullableMetricValue;
|
| 118 |
f1: NullableMetricValue;
|
| 119 |
fpr: NullableMetricValue;
|
| 120 |
fnr: NullableMetricValue;
|
|
|
|
| 238 |
groups: Record<
|
| 239 |
string,
|
| 240 |
{
|
| 241 |
+
recall: NullableMetricValue;
|
| 242 |
+
precision: NullableMetricValue;
|
| 243 |
f1: NullableMetricValue;
|
| 244 |
}
|
| 245 |
>;
|
src/lib/server/hf-bucket/visualizations-payload.ts
CHANGED
|
@@ -184,8 +184,8 @@ function parseGroupedBarsModel(value: unknown, index: number): BucketGroupedBars
|
|
| 184 |
return [
|
| 185 |
groupId,
|
| 186 |
{
|
| 187 |
-
|
| 188 |
-
|
| 189 |
f1: nullableNumber(metrics.f1, `${field}.groups.${groupId}.f1`)
|
| 190 |
}
|
| 191 |
];
|
|
|
|
| 184 |
return [
|
| 185 |
groupId,
|
| 186 |
{
|
| 187 |
+
recall: nullableNumber(metrics.recall, `${field}.groups.${groupId}.recall`),
|
| 188 |
+
precision: nullableNumber(metrics.precision, `${field}.groups.${groupId}.precision`),
|
| 189 |
f1: nullableNumber(metrics.f1, `${field}.groups.${groupId}.f1`)
|
| 190 |
}
|
| 191 |
];
|
src/lib/types/details.ts
CHANGED
|
@@ -53,13 +53,18 @@ export type BenchmarkRankingRow = {
|
|
| 53 |
export type BenchmarkCard = {
|
| 54 |
name: string;
|
| 55 |
meta: RankingBenchmarkMeta;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
};
|
| 57 |
|
| 58 |
export type ModelGroupRow = {
|
| 59 |
name: string;
|
| 60 |
score: number;
|
| 61 |
-
|
| 62 |
-
|
| 63 |
};
|
| 64 |
|
| 65 |
export type RadarPoint = ModelGroupRow & {
|
|
|
|
| 53 |
export type BenchmarkCard = {
|
| 54 |
name: string;
|
| 55 |
meta: RankingBenchmarkMeta;
|
| 56 |
+
leader: {
|
| 57 |
+
modelShort: string;
|
| 58 |
+
value: number;
|
| 59 |
+
metric: 'f1' | 'score';
|
| 60 |
+
} | null;
|
| 61 |
};
|
| 62 |
|
| 63 |
export type ModelGroupRow = {
|
| 64 |
name: string;
|
| 65 |
score: number;
|
| 66 |
+
recall: number | null;
|
| 67 |
+
precision: number | null;
|
| 68 |
};
|
| 69 |
|
| 70 |
export type RadarPoint = ModelGroupRow & {
|
src/lib/types/leaderboard.ts
CHANGED
|
@@ -4,9 +4,12 @@ export type BenchmarkScope = 'all' | 'en' | 'robust';
|
|
| 4 |
|
| 5 |
export type GroupScore = {
|
| 6 |
score: number;
|
|
|
|
| 7 |
f1: number | null;
|
| 8 |
fpr: number | null;
|
| 9 |
fnr: number | null;
|
|
|
|
|
|
|
| 10 |
};
|
| 11 |
|
| 12 |
export type GuardrailType = 'llm-judge' | 'classifier' | 'unknown';
|
|
|
|
| 4 |
|
| 5 |
export type GroupScore = {
|
| 6 |
score: number;
|
| 7 |
+
minDatasetScore?: number | null;
|
| 8 |
f1: number | null;
|
| 9 |
fpr: number | null;
|
| 10 |
fnr: number | null;
|
| 11 |
+
recall?: number | null;
|
| 12 |
+
precision?: number | null;
|
| 13 |
};
|
| 14 |
|
| 15 |
export type GuardrailType = 'llm-judge' | 'classifier' | 'unknown';
|
src/lib/types/ranking.ts
CHANGED
|
@@ -30,6 +30,7 @@ export type LeaderboardRow = {
|
|
| 30 |
export type BenchmarkDetail = {
|
| 31 |
name: string;
|
| 32 |
score: number | null;
|
|
|
|
| 33 |
fpr: number | null;
|
| 34 |
fnr: number | null;
|
| 35 |
f1: number | null;
|
|
|
|
| 30 |
export type BenchmarkDetail = {
|
| 31 |
name: string;
|
| 32 |
score: number | null;
|
| 33 |
+
minDatasetScore: number | null;
|
| 34 |
fpr: number | null;
|
| 35 |
fnr: number | null;
|
| 36 |
f1: number | null;
|
src/lib/types/tools-data.ts
CHANGED
|
@@ -55,8 +55,8 @@ export type ToolsGroupedBarsVisualization = ToolsVisualizationMeta & {
|
|
| 55 |
groups: Record<
|
| 56 |
string,
|
| 57 |
{
|
| 58 |
-
|
| 59 |
-
|
| 60 |
f1: number | null;
|
| 61 |
}
|
| 62 |
>;
|
|
|
|
| 55 |
groups: Record<
|
| 56 |
string,
|
| 57 |
{
|
| 58 |
+
recall: number | null;
|
| 59 |
+
precision: number | null;
|
| 60 |
f1: number | null;
|
| 61 |
}
|
| 62 |
>;
|
src/routes/+page.svelte
CHANGED
|
@@ -423,12 +423,7 @@
|
|
| 423 |
}
|
| 424 |
|
| 425 |
function modelDescription(model: GuardModel) {
|
| 426 |
-
|
| 427 |
-
return m.ranking_details_classifier_description({ family: model.family });
|
| 428 |
-
}
|
| 429 |
-
if (model.type === 'unknown') return model.note;
|
| 430 |
-
|
| 431 |
-
return m.ranking_details_generative_description({ family: model.family });
|
| 432 |
}
|
| 433 |
|
| 434 |
function fmtModelType(type: GuardModel['type']) {
|
|
@@ -834,11 +829,11 @@
|
|
| 834 |
>
|
| 835 |
<div class="flex min-w-0 flex-wrap items-center gap-2">
|
| 836 |
<span class="min-w-0 truncate font-medium">{groupLabel(group)}</span>
|
| 837 |
-
{#if benchmark?.
|
| 838 |
<span
|
| 839 |
class="inline-flex rounded-full border bg-muted/40 px-2 py-0.5 font-mono text-xs text-muted-foreground"
|
| 840 |
>
|
| 841 |
-
min {benchmark.
|
| 842 |
</span>
|
| 843 |
{/if}
|
| 844 |
</div>
|
|
|
|
| 423 |
}
|
| 424 |
|
| 425 |
function modelDescription(model: GuardModel) {
|
| 426 |
+
return localized(model.noteLocalized, model.note);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 427 |
}
|
| 428 |
|
| 429 |
function fmtModelType(type: GuardModel['type']) {
|
|
|
|
| 829 |
>
|
| 830 |
<div class="flex min-w-0 flex-wrap items-center gap-2">
|
| 831 |
<span class="min-w-0 truncate font-medium">{groupLabel(group)}</span>
|
| 832 |
+
{#if benchmark?.minDatasetScore !== null && benchmark?.minDatasetScore !== undefined}
|
| 833 |
<span
|
| 834 |
class="inline-flex rounded-full border bg-muted/40 px-2 py-0.5 font-mono text-xs text-muted-foreground"
|
| 835 |
>
|
| 836 |
+
min {benchmark.minDatasetScore.toFixed(2)}
|
| 837 |
</span>
|
| 838 |
{/if}
|
| 839 |
</div>
|
src/routes/details/+page.svelte
CHANGED
|
@@ -863,6 +863,30 @@
|
|
| 863 |
</Badge>
|
| 864 |
{/each}
|
| 865 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 866 |
</button>
|
| 867 |
{/each}
|
| 868 |
</div>
|
|
@@ -1351,8 +1375,8 @@
|
|
| 1351 |
<tr>
|
| 1352 |
<th class="px-3 py-2 text-left font-semibold">{m.details_groups_title()}</th>
|
| 1353 |
<th class="px-3 py-2 text-right font-semibold">score</th>
|
| 1354 |
-
<th class="px-3 py-2 text-right font-semibold">
|
| 1355 |
-
<th class="px-3 py-2 text-right font-semibold">
|
| 1356 |
<th class="px-3 py-2 text-right font-semibold">F1</th>
|
| 1357 |
</tr>
|
| 1358 |
</thead>
|
|
@@ -1366,13 +1390,13 @@
|
|
| 1366 |
</span>
|
| 1367 |
</td>
|
| 1368 |
<td class="px-3 py-3 text-right">
|
| 1369 |
-
<span class={compactMetricClass(group.
|
| 1370 |
-
{fmtPercent(group.
|
| 1371 |
</span>
|
| 1372 |
</td>
|
| 1373 |
<td class="px-3 py-3 text-right">
|
| 1374 |
-
<span class={compactMetricClass(group.
|
| 1375 |
-
{fmtPercent(group.
|
| 1376 |
</span>
|
| 1377 |
</td>
|
| 1378 |
<td class="px-3 py-3 text-right">
|
|
|
|
| 863 |
</Badge>
|
| 864 |
{/each}
|
| 865 |
</div>
|
| 866 |
+
{#if card.leader}
|
| 867 |
+
<div
|
| 868 |
+
class="mt-auto flex min-w-0 items-center justify-between gap-3 border-t pt-3 font-mono text-sm"
|
| 869 |
+
>
|
| 870 |
+
<div class="flex min-w-0 items-center gap-4">
|
| 871 |
+
{#if card.leader.metric === 'f1'}
|
| 872 |
+
<span
|
| 873 |
+
class="shrink-0 text-xs font-semibold tracking-wider text-muted-foreground uppercase"
|
| 874 |
+
>
|
| 875 |
+
{m.details_benchmark_leader_f1()}
|
| 876 |
+
</span>
|
| 877 |
+
{/if}
|
| 878 |
+
<OverflowTooltip
|
| 879 |
+
text={card.leader.modelShort}
|
| 880 |
+
class="block min-w-0 truncate font-semibold"
|
| 881 |
+
triggerClass="block min-w-0"
|
| 882 |
+
contentClass="max-w-sm break-all"
|
| 883 |
+
/>
|
| 884 |
+
</div>
|
| 885 |
+
<span class="shrink-0 font-semibold tabular-nums">
|
| 886 |
+
{fmtScore(card.leader.value)}
|
| 887 |
+
</span>
|
| 888 |
+
</div>
|
| 889 |
+
{/if}
|
| 890 |
</button>
|
| 891 |
{/each}
|
| 892 |
</div>
|
|
|
|
| 1375 |
<tr>
|
| 1376 |
<th class="px-3 py-2 text-left font-semibold">{m.details_groups_title()}</th>
|
| 1377 |
<th class="px-3 py-2 text-right font-semibold">score</th>
|
| 1378 |
+
<th class="px-3 py-2 text-right font-semibold">Recall</th>
|
| 1379 |
+
<th class="px-3 py-2 text-right font-semibold">Precision</th>
|
| 1380 |
<th class="px-3 py-2 text-right font-semibold">F1</th>
|
| 1381 |
</tr>
|
| 1382 |
</thead>
|
|
|
|
| 1390 |
</span>
|
| 1391 |
</td>
|
| 1392 |
<td class="px-3 py-3 text-right">
|
| 1393 |
+
<span class={compactMetricClass(group.recall)}>
|
| 1394 |
+
{fmtPercent(group.recall)}
|
| 1395 |
</span>
|
| 1396 |
</td>
|
| 1397 |
<td class="px-3 py-3 text-right">
|
| 1398 |
+
<span class={compactMetricClass(group.precision)}>
|
| 1399 |
+
{fmtPercent(group.precision)}
|
| 1400 |
</span>
|
| 1401 |
</td>
|
| 1402 |
<td class="px-3 py-3 text-right">
|
src/routes/tools/+page.svelte
CHANGED
|
@@ -41,9 +41,7 @@
|
|
| 41 |
import type {
|
| 42 |
ToolsLanguageFilter as LangFilter,
|
| 43 |
ToolsLatencySortKey as LatencySortKey,
|
| 44 |
-
|
| 45 |
-
ToolsSortDirection as SortDirection,
|
| 46 |
-
ToolsVizMetric as VizMetric
|
| 47 |
} from '$lib/types/tools';
|
| 48 |
import { m } from '$lib/paraglide/messages';
|
| 49 |
import { getLocale } from '$lib/paraglide/runtime';
|
|
@@ -53,6 +51,7 @@
|
|
| 53 |
import type { PageProps } from './$types';
|
| 54 |
|
| 55 |
type LatencyColumn = TableColumn<LatencySortKey>;
|
|
|
|
| 56 |
type VisualizationModelRow = {
|
| 57 |
modelKey: string;
|
| 58 |
model: GuardModel;
|
|
@@ -113,8 +112,6 @@
|
|
| 113 |
let selectedModelKeys = $state(initialTools.models.slice(0, 5).map((model) => modelKey(model)));
|
| 114 |
let selectedGroups = $state<string[]>(initialGroups);
|
| 115 |
let language = $state<LangFilter>('all');
|
| 116 |
-
let metric = $state<VizMetric>('fnr');
|
| 117 |
-
let radarScale = $state<RadarScale>('score');
|
| 118 |
let pinnedModelKeys = $state<string[]>([]);
|
| 119 |
let latencySortKey = $state<LatencySortKey>('p95');
|
| 120 |
let latencySortDirection = $state<SortDirection>('asc');
|
|
@@ -132,11 +129,6 @@
|
|
| 132 |
{ value: 'be', label: 'BE' },
|
| 133 |
{ value: 'uz', label: 'UZ' }
|
| 134 |
];
|
| 135 |
-
const metricOptions = [
|
| 136 |
-
{ value: 'fnr', label: m.tools_metric_fnr() },
|
| 137 |
-
{ value: 'fpr', label: m.tools_metric_fpr() },
|
| 138 |
-
{ value: 'f1', label: 'F1' }
|
| 139 |
-
];
|
| 140 |
const latencyColumns: LatencyColumn[] = [
|
| 141 |
{
|
| 142 |
key: 'model',
|
|
@@ -173,9 +165,6 @@
|
|
| 173 |
const currentLanguageLabel = $derived(
|
| 174 |
languageOptions.find((option) => option.value === language)?.label ?? languageOptions[0].label
|
| 175 |
);
|
| 176 |
-
const currentMetricLabel = $derived(
|
| 177 |
-
metricOptions.find((option) => option.value === metric)?.label ?? metricOptions[0].label
|
| 178 |
-
);
|
| 179 |
const selectedModels = $derived(toolsSelectedModels(models, selectedModelKeys));
|
| 180 |
const modelByKey = $derived(new Map(models.map((model) => [modelKey(model), model] as const)));
|
| 181 |
const selectedModelKeySet = $derived(new Set(selectedModelKeys));
|
|
@@ -367,20 +356,14 @@
|
|
| 367 |
.sort((a, b) => Number(a.deltaScore ?? 0) - Number(b.deltaScore ?? 0));
|
| 368 |
}
|
| 369 |
|
| 370 |
-
function radarVisualizationValue(modelKeyValue: string, group: string
|
| 371 |
-
|
| 372 |
-
scale === 'f1' ? tools.visualizations.radar.f1Values : tools.visualizations.radar.values;
|
| 373 |
-
return source[group]?.[modelKeyValue] ?? null;
|
| 374 |
}
|
| 375 |
|
| 376 |
-
function radarVisualizationPolygon(
|
| 377 |
-
modelKeyValue: string,
|
| 378 |
-
groups: readonly string[],
|
| 379 |
-
scale: RadarScale
|
| 380 |
-
) {
|
| 381 |
return groups
|
| 382 |
.map((group, index) => {
|
| 383 |
-
const value = radarVisualizationValue(modelKeyValue, group
|
| 384 |
const point = radarPoint(index, groups.length, 150 * clamp01(value ?? 0));
|
| 385 |
return `${point.x.toFixed(1)},${point.y.toFixed(1)}`;
|
| 386 |
})
|
|
@@ -389,7 +372,7 @@
|
|
| 389 |
|
| 390 |
function groupedBarsMean(
|
| 391 |
modelKeyValue: string,
|
| 392 |
-
key:
|
| 393 |
modelsByVisualizationKey: ReadonlyMap<
|
| 394 |
string,
|
| 395 |
ToolsDataset['visualizations']['groupedBars']['models'][number]
|
|
@@ -546,7 +529,7 @@
|
|
| 546 |
class="mt-6 min-w-0 rounded-xl border bg-card p-4 shadow-sm"
|
| 547 |
aria-label={m.tools_controls_aria()}
|
| 548 |
>
|
| 549 |
-
<div class="grid min-w-0 gap-4 lg:grid-cols-
|
| 550 |
<div class="min-w-0">
|
| 551 |
<div class="mb-2 flex items-center justify-between gap-3">
|
| 552 |
<h2 class="text-xs font-semibold tracking-wider text-muted-foreground uppercase">
|
|
@@ -665,58 +648,19 @@
|
|
| 665 |
{/each}
|
| 666 |
</div>
|
| 667 |
</div>
|
| 668 |
-
|
| 669 |
-
<div class="grid content-start gap-3">
|
| 670 |
-
<Select
|
| 671 |
-
id="tools-language"
|
| 672 |
-
type="single"
|
| 673 |
-
bind:value={language}
|
| 674 |
-
options={languageOptions}
|
| 675 |
-
label={m.tools_language_label()}
|
| 676 |
-
triggerLabel={currentLanguageLabel}
|
| 677 |
-
contentClass={selectContentClass}
|
| 678 |
-
/>
|
| 679 |
-
<Select
|
| 680 |
-
id="tools-metric"
|
| 681 |
-
type="single"
|
| 682 |
-
bind:value={metric}
|
| 683 |
-
options={metricOptions}
|
| 684 |
-
label={m.tools_metric_label()}
|
| 685 |
-
triggerLabel={currentMetricLabel}
|
| 686 |
-
contentClass={selectContentClass}
|
| 687 |
-
/>
|
| 688 |
-
</div>
|
| 689 |
</div>
|
| 690 |
|
| 691 |
<div class="mt-4 flex flex-wrap items-end gap-3">
|
| 692 |
-
<
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
<Button
|
| 703 |
-
variant={radarScale === 'score' ? 'default' : 'ghost'}
|
| 704 |
-
size="sm"
|
| 705 |
-
aria-pressed={radarScale === 'score'}
|
| 706 |
-
onclick={() => (radarScale = 'score')}
|
| 707 |
-
>
|
| 708 |
-
{m.tools_radar_group_score()}
|
| 709 |
-
</Button>
|
| 710 |
-
<Button
|
| 711 |
-
variant={radarScale === 'f1' ? 'default' : 'ghost'}
|
| 712 |
-
size="sm"
|
| 713 |
-
aria-pressed={radarScale === 'f1'}
|
| 714 |
-
onclick={() => (radarScale = 'f1')}
|
| 715 |
-
>
|
| 716 |
-
F1
|
| 717 |
-
</Button>
|
| 718 |
-
</div>
|
| 719 |
-
</div>
|
| 720 |
<div class="ml-auto flex flex-wrap items-center justify-end gap-3">
|
| 721 |
<span
|
| 722 |
class="rounded-full border border-[#0dce97]/40 bg-[#0dce97]/10 px-4 py-2 font-mono text-xs font-semibold text-[#0dce97]"
|
|
@@ -822,7 +766,7 @@
|
|
| 822 |
{@const isHovered = hoveredRadarModelKey === key}
|
| 823 |
{@const isDimmed = hoveredRadarModelKey !== null && !isHovered}
|
| 824 |
<polygon
|
| 825 |
-
points={radarVisualizationPolygon(key, activeGroups
|
| 826 |
fill={colorOf(model)}
|
| 827 |
fill-opacity={isHovered
|
| 828 |
? 0.24
|
|
@@ -839,7 +783,7 @@
|
|
| 839 |
pointer-events="none"
|
| 840 |
/>
|
| 841 |
{#each activeGroups as group, index (group)}
|
| 842 |
-
{@const value = radarVisualizationValue(key, group
|
| 843 |
{@const point = radarPoint(
|
| 844 |
index,
|
| 845 |
activeGroups.length,
|
|
@@ -854,11 +798,14 @@
|
|
| 854 |
stroke-opacity={isDimmed ? 0.25 : 1}
|
| 855 |
class="transition-[r,fill-opacity,stroke-opacity]"
|
| 856 |
role="img"
|
| 857 |
-
aria-label={`${modelLabel(model)} ${groupLabel(group)} ${
|
| 858 |
onpointerenter={(event) => {
|
| 859 |
hoveredRadarModelKey = key;
|
| 860 |
tooltip.show(event, modelLabel(model), [
|
| 861 |
-
{
|
|
|
|
|
|
|
|
|
|
| 862 |
{ label: 'value', value: fmt(value, 3) }
|
| 863 |
]);
|
| 864 |
}}
|
|
@@ -1168,13 +1115,13 @@
|
|
| 1168 |
{@const key = modelKey(model)}
|
| 1169 |
{@const band = (barSvgWidth - 56) / selectedModels.length}
|
| 1170 |
{@const x0 = 44 + index * band + Math.max(0, (band - barGroupWidth) / 2)}
|
| 1171 |
-
{#each [{ key: '
|
| 1172 |
{@const barKey = `${key}-${series.key}`}
|
| 1173 |
{@const isHovered = hoveredBarKey === barKey}
|
| 1174 |
{@const isDimmed = hoveredBarKey !== null && !isHovered}
|
| 1175 |
{@const value = groupedBarsMean(
|
| 1176 |
key,
|
| 1177 |
-
series.key as
|
| 1178 |
groupedBarsByModelKey,
|
| 1179 |
activeGroups
|
| 1180 |
)}
|
|
@@ -1235,10 +1182,10 @@
|
|
| 1235 |
</div>
|
| 1236 |
<div class="mt-3 flex flex-wrap gap-4 text-xs text-muted-foreground">
|
| 1237 |
<span class="inline-flex items-center gap-2"
|
| 1238 |
-
><span class="size-2 rounded-full bg-[#ff4d4f]"></span>
|
| 1239 |
>
|
| 1240 |
<span class="inline-flex items-center gap-2"
|
| 1241 |
-
><span class="size-2 rounded-full bg-[#f5a623]"></span>
|
| 1242 |
>
|
| 1243 |
<span class="inline-flex items-center gap-2"
|
| 1244 |
><span class="size-2 rounded-full bg-[#0dce97]"></span>F1 ({m.tools_higher_better()})</span
|
|
|
|
| 41 |
import type {
|
| 42 |
ToolsLanguageFilter as LangFilter,
|
| 43 |
ToolsLatencySortKey as LatencySortKey,
|
| 44 |
+
ToolsSortDirection as SortDirection
|
|
|
|
|
|
|
| 45 |
} from '$lib/types/tools';
|
| 46 |
import { m } from '$lib/paraglide/messages';
|
| 47 |
import { getLocale } from '$lib/paraglide/runtime';
|
|
|
|
| 51 |
import type { PageProps } from './$types';
|
| 52 |
|
| 53 |
type LatencyColumn = TableColumn<LatencySortKey>;
|
| 54 |
+
type GroupedBarsMetric = 'recall' | 'precision' | 'f1';
|
| 55 |
type VisualizationModelRow = {
|
| 56 |
modelKey: string;
|
| 57 |
model: GuardModel;
|
|
|
|
| 112 |
let selectedModelKeys = $state(initialTools.models.slice(0, 5).map((model) => modelKey(model)));
|
| 113 |
let selectedGroups = $state<string[]>(initialGroups);
|
| 114 |
let language = $state<LangFilter>('all');
|
|
|
|
|
|
|
| 115 |
let pinnedModelKeys = $state<string[]>([]);
|
| 116 |
let latencySortKey = $state<LatencySortKey>('p95');
|
| 117 |
let latencySortDirection = $state<SortDirection>('asc');
|
|
|
|
| 129 |
{ value: 'be', label: 'BE' },
|
| 130 |
{ value: 'uz', label: 'UZ' }
|
| 131 |
];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
const latencyColumns: LatencyColumn[] = [
|
| 133 |
{
|
| 134 |
key: 'model',
|
|
|
|
| 165 |
const currentLanguageLabel = $derived(
|
| 166 |
languageOptions.find((option) => option.value === language)?.label ?? languageOptions[0].label
|
| 167 |
);
|
|
|
|
|
|
|
|
|
|
| 168 |
const selectedModels = $derived(toolsSelectedModels(models, selectedModelKeys));
|
| 169 |
const modelByKey = $derived(new Map(models.map((model) => [modelKey(model), model] as const)));
|
| 170 |
const selectedModelKeySet = $derived(new Set(selectedModelKeys));
|
|
|
|
| 356 |
.sort((a, b) => Number(a.deltaScore ?? 0) - Number(b.deltaScore ?? 0));
|
| 357 |
}
|
| 358 |
|
| 359 |
+
function radarVisualizationValue(modelKeyValue: string, group: string) {
|
| 360 |
+
return tools.visualizations.radar.values[group]?.[modelKeyValue] ?? null;
|
|
|
|
|
|
|
| 361 |
}
|
| 362 |
|
| 363 |
+
function radarVisualizationPolygon(modelKeyValue: string, groups: readonly string[]) {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 364 |
return groups
|
| 365 |
.map((group, index) => {
|
| 366 |
+
const value = radarVisualizationValue(modelKeyValue, group);
|
| 367 |
const point = radarPoint(index, groups.length, 150 * clamp01(value ?? 0));
|
| 368 |
return `${point.x.toFixed(1)},${point.y.toFixed(1)}`;
|
| 369 |
})
|
|
|
|
| 372 |
|
| 373 |
function groupedBarsMean(
|
| 374 |
modelKeyValue: string,
|
| 375 |
+
key: GroupedBarsMetric,
|
| 376 |
modelsByVisualizationKey: ReadonlyMap<
|
| 377 |
string,
|
| 378 |
ToolsDataset['visualizations']['groupedBars']['models'][number]
|
|
|
|
| 529 |
class="mt-6 min-w-0 rounded-xl border bg-card p-4 shadow-sm"
|
| 530 |
aria-label={m.tools_controls_aria()}
|
| 531 |
>
|
| 532 |
+
<div class="grid min-w-0 gap-4 lg:grid-cols-2">
|
| 533 |
<div class="min-w-0">
|
| 534 |
<div class="mb-2 flex items-center justify-between gap-3">
|
| 535 |
<h2 class="text-xs font-semibold tracking-wider text-muted-foreground uppercase">
|
|
|
|
| 648 |
{/each}
|
| 649 |
</div>
|
| 650 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 651 |
</div>
|
| 652 |
|
| 653 |
<div class="mt-4 flex flex-wrap items-end gap-3">
|
| 654 |
+
<Select
|
| 655 |
+
id="tools-language"
|
| 656 |
+
type="single"
|
| 657 |
+
bind:value={language}
|
| 658 |
+
options={languageOptions}
|
| 659 |
+
label={m.tools_language_label()}
|
| 660 |
+
triggerLabel={currentLanguageLabel}
|
| 661 |
+
containerClass="w-full min-w-0 space-y-2 sm:w-80"
|
| 662 |
+
contentClass={selectContentClass}
|
| 663 |
+
/>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 664 |
<div class="ml-auto flex flex-wrap items-center justify-end gap-3">
|
| 665 |
<span
|
| 666 |
class="rounded-full border border-[#0dce97]/40 bg-[#0dce97]/10 px-4 py-2 font-mono text-xs font-semibold text-[#0dce97]"
|
|
|
|
| 766 |
{@const isHovered = hoveredRadarModelKey === key}
|
| 767 |
{@const isDimmed = hoveredRadarModelKey !== null && !isHovered}
|
| 768 |
<polygon
|
| 769 |
+
points={radarVisualizationPolygon(key, activeGroups)}
|
| 770 |
fill={colorOf(model)}
|
| 771 |
fill-opacity={isHovered
|
| 772 |
? 0.24
|
|
|
|
| 783 |
pointer-events="none"
|
| 784 |
/>
|
| 785 |
{#each activeGroups as group, index (group)}
|
| 786 |
+
{@const value = radarVisualizationValue(key, group) ?? 0}
|
| 787 |
{@const point = radarPoint(
|
| 788 |
index,
|
| 789 |
activeGroups.length,
|
|
|
|
| 798 |
stroke-opacity={isDimmed ? 0.25 : 1}
|
| 799 |
class="transition-[r,fill-opacity,stroke-opacity]"
|
| 800 |
role="img"
|
| 801 |
+
aria-label={`${modelLabel(model)} ${groupLabel(group)} ${m.tools_radar_group_score()} ${fmt(value, 3)}`}
|
| 802 |
onpointerenter={(event) => {
|
| 803 |
hoveredRadarModelKey = key;
|
| 804 |
tooltip.show(event, modelLabel(model), [
|
| 805 |
+
{
|
| 806 |
+
label: groupLabel(group),
|
| 807 |
+
value: m.tools_radar_group_score()
|
| 808 |
+
},
|
| 809 |
{ label: 'value', value: fmt(value, 3) }
|
| 810 |
]);
|
| 811 |
}}
|
|
|
|
| 1115 |
{@const key = modelKey(model)}
|
| 1116 |
{@const band = (barSvgWidth - 56) / selectedModels.length}
|
| 1117 |
{@const x0 = 44 + index * band + Math.max(0, (band - barGroupWidth) / 2)}
|
| 1118 |
+
{#each [{ key: 'recall', color: '#ff4d4f' }, { key: 'precision', color: '#f5a623' }, { key: 'f1', color: '#0dce97' }] as series, seriesIndex (series.key)}
|
| 1119 |
{@const barKey = `${key}-${series.key}`}
|
| 1120 |
{@const isHovered = hoveredBarKey === barKey}
|
| 1121 |
{@const isDimmed = hoveredBarKey !== null && !isHovered}
|
| 1122 |
{@const value = groupedBarsMean(
|
| 1123 |
key,
|
| 1124 |
+
series.key as GroupedBarsMetric,
|
| 1125 |
groupedBarsByModelKey,
|
| 1126 |
activeGroups
|
| 1127 |
)}
|
|
|
|
| 1182 |
</div>
|
| 1183 |
<div class="mt-3 flex flex-wrap gap-4 text-xs text-muted-foreground">
|
| 1184 |
<span class="inline-flex items-center gap-2"
|
| 1185 |
+
><span class="size-2 rounded-full bg-[#ff4d4f]"></span>Recall ({m.tools_higher_better()})</span
|
| 1186 |
>
|
| 1187 |
<span class="inline-flex items-center gap-2"
|
| 1188 |
+
><span class="size-2 rounded-full bg-[#f5a623]"></span>Precision ({m.tools_higher_better()})</span
|
| 1189 |
>
|
| 1190 |
<span class="inline-flex items-center gap-2"
|
| 1191 |
><span class="size-2 rounded-full bg-[#0dce97]"></span>F1 ({m.tools_higher_better()})</span
|