fix: stabilize benchmark leaders, model benchmark sorting, and HF bucket reads
Browse files- Fix benchmark card leader selection on the Details page
- Keep using F1 when a benchmark group has F1 values
- Fall back to FNR, then FPR, when F1 is unavailable
- Treat FNR/FPR as lower-is-better metrics when selecting the leader
- Show the selected leader metric in the card preview label
- Format FNR/FPR leader values as percentages instead of score decimals
- Sort model benchmark cards on the Ranking sheet
- Sort benchmark details by score descending
- Ensure the Benchmarks tab in the model flyout shows highest score first and lowest score last
- Improve Hugging Face bucket read resilience
- Increase default bucket request timeout from 10s to 60s
- Add configurable HF_BUCKET_REQUEST_RETRIES with a default of 2 retries
- Retry timeout/network-level bucket read failures without retrying HTTP errors like 401/403/404
- Load Tools visualization bucket files sequentially instead of issuing all visualization reads in parallel
- Update .env.example and deployment/cache docs with the new timeout and retry defaults
- .env.example +2 -1
- docs/data-loading-cache_en.md +4 -3
- docs/data-loading-cache_ru.md +4 -3
- docs/huggingface-space-deploy_en.md +2 -1
- docs/huggingface-space-deploy_ru.md +2 -1
- messages/en.json +1 -1
- messages/ru.json +1 -1
- src/lib/data/details.ts +37 -4
- src/lib/data/ranking.ts +2 -1
- src/lib/server/hf-bucket/client.ts +43 -26
- src/lib/server/hf-bucket/tools-snapshot.ts +8 -19
- src/lib/types/details.ts +1 -1
- src/routes/details/+page.svelte +12 -8
|
@@ -6,5 +6,6 @@ HF_BUCKET_PREFIX=latest
|
|
| 6 |
|
| 7 |
# Optional runtime tuning.
|
| 8 |
# HF_BUCKET_CACHE_TTL_MS=300000
|
| 9 |
-
# HF_BUCKET_REQUEST_TIMEOUT_MS=
|
|
|
|
| 10 |
# HF_BUCKET_ENDPOINT=https://huggingface.co
|
|
|
|
| 6 |
|
| 7 |
# Optional runtime tuning.
|
| 8 |
# HF_BUCKET_CACHE_TTL_MS=300000
|
| 9 |
+
# HF_BUCKET_REQUEST_TIMEOUT_MS=60000
|
| 10 |
+
# HF_BUCKET_REQUEST_RETRIES=2
|
| 11 |
# HF_BUCKET_ENDPOINT=https://huggingface.co
|
|
@@ -31,12 +31,13 @@ The bucket is configured with these runtime variables:
|
|
| 31 |
HF_TOKEN=<private read token>
|
| 32 |
HF_BUCKET_ID=hivetrace/leaderboard_frontend_v2
|
| 33 |
HF_BUCKET_PREFIX=latest
|
| 34 |
-
HF_BUCKET_REQUEST_TIMEOUT_MS=
|
|
|
|
| 35 |
HF_BUCKET_ENDPOINT=https://huggingface.co
|
| 36 |
```
|
| 37 |
|
| 38 |
-
`HF_BUCKET_ENDPOINT` and `
|
| 39 |
-
their defaults.
|
| 40 |
|
| 41 |
`HF_BUCKET_CACHE_TTL_MS` still controls the regular in-memory manifest cache and the direct
|
| 42 |
Tools snapshot helper. The main Ranking, Details, and Visualizations page-data flow explicitly
|
|
|
|
| 31 |
HF_TOKEN=<private read token>
|
| 32 |
HF_BUCKET_ID=hivetrace/leaderboard_frontend_v2
|
| 33 |
HF_BUCKET_PREFIX=latest
|
| 34 |
+
HF_BUCKET_REQUEST_TIMEOUT_MS=60000
|
| 35 |
+
HF_BUCKET_REQUEST_RETRIES=2
|
| 36 |
HF_BUCKET_ENDPOINT=https://huggingface.co
|
| 37 |
```
|
| 38 |
|
| 39 |
+
`HF_BUCKET_ENDPOINT`, `HF_BUCKET_REQUEST_TIMEOUT_MS`, and `HF_BUCKET_REQUEST_RETRIES` are optional.
|
| 40 |
+
The values above are their defaults.
|
| 41 |
|
| 42 |
`HF_BUCKET_CACHE_TTL_MS` still controls the regular in-memory manifest cache and the direct
|
| 43 |
Tools snapshot helper. The main Ranking, Details, and Visualizations page-data flow explicitly
|
|
@@ -32,12 +32,13 @@ Bucket настраивается runtime-переменными:
|
|
| 32 |
HF_TOKEN=<private read token>
|
| 33 |
HF_BUCKET_ID=hivetrace/leaderboard_frontend_v2
|
| 34 |
HF_BUCKET_PREFIX=latest
|
| 35 |
-
HF_BUCKET_REQUEST_TIMEOUT_MS=
|
|
|
|
| 36 |
HF_BUCKET_ENDPOINT=https://huggingface.co
|
| 37 |
```
|
| 38 |
|
| 39 |
-
`HF_BUCKET_ENDPOINT` и `
|
| 40 |
-
умолчанию.
|
| 41 |
|
| 42 |
`HF_BUCKET_CACHE_TTL_MS` по-прежнему управляет обычным in-memory кешем manifest и прямым helper
|
| 43 |
Tools snapshot. Основной поток page data для Ranking, Details и Visualizations после промаха
|
|
|
|
| 32 |
HF_TOKEN=<private read token>
|
| 33 |
HF_BUCKET_ID=hivetrace/leaderboard_frontend_v2
|
| 34 |
HF_BUCKET_PREFIX=latest
|
| 35 |
+
HF_BUCKET_REQUEST_TIMEOUT_MS=60000
|
| 36 |
+
HF_BUCKET_REQUEST_RETRIES=2
|
| 37 |
HF_BUCKET_ENDPOINT=https://huggingface.co
|
| 38 |
```
|
| 39 |
|
| 40 |
+
`HF_BUCKET_ENDPOINT`, `HF_BUCKET_REQUEST_TIMEOUT_MS` и `HF_BUCKET_REQUEST_RETRIES` опциональны.
|
| 41 |
+
Выше указаны их значения по умолчанию.
|
| 42 |
|
| 43 |
`HF_BUCKET_CACHE_TTL_MS` по-прежнему управляет обычным in-memory кешем manifest и прямым helper
|
| 44 |
Tools snapshot. Основной поток page data для Ranking, Details и Visualizations после промаха
|
|
@@ -52,7 +52,8 @@ Variables:
|
|
| 52 |
HF_BUCKET_ID=hivetrace/leaderboard_frontend_v2
|
| 53 |
HF_BUCKET_PREFIX=latest
|
| 54 |
HF_BUCKET_CACHE_TTL_MS=300000
|
| 55 |
-
HF_BUCKET_REQUEST_TIMEOUT_MS=
|
|
|
|
| 56 |
```
|
| 57 |
|
| 58 |
Optional:
|
|
|
|
| 52 |
HF_BUCKET_ID=hivetrace/leaderboard_frontend_v2
|
| 53 |
HF_BUCKET_PREFIX=latest
|
| 54 |
HF_BUCKET_CACHE_TTL_MS=300000
|
| 55 |
+
HF_BUCKET_REQUEST_TIMEOUT_MS=60000
|
| 56 |
+
HF_BUCKET_REQUEST_RETRIES=2
|
| 57 |
```
|
| 58 |
|
| 59 |
Optional:
|
|
@@ -52,7 +52,8 @@ Variables:
|
|
| 52 |
HF_BUCKET_ID=hivetrace/leaderboard_frontend_v2
|
| 53 |
HF_BUCKET_PREFIX=latest
|
| 54 |
HF_BUCKET_CACHE_TTL_MS=300000
|
| 55 |
-
HF_BUCKET_REQUEST_TIMEOUT_MS=
|
|
|
|
| 56 |
```
|
| 57 |
|
| 58 |
Опционально:
|
|
|
|
| 52 |
HF_BUCKET_ID=hivetrace/leaderboard_frontend_v2
|
| 53 |
HF_BUCKET_PREFIX=latest
|
| 54 |
HF_BUCKET_CACHE_TTL_MS=300000
|
| 55 |
+
HF_BUCKET_REQUEST_TIMEOUT_MS=60000
|
| 56 |
+
HF_BUCKET_REQUEST_RETRIES=2
|
| 57 |
```
|
| 58 |
|
| 59 |
Опционально:
|
|
@@ -201,7 +201,7 @@
|
|
| 201 |
"details_benchmark_sheet_ranking": "Model ranking on this benchmark",
|
| 202 |
"details_benchmark_sheet_rank": "Rank",
|
| 203 |
"details_benchmark_sheet_open": "Open benchmark",
|
| 204 |
-
"
|
| 205 |
"details_model_sheet_open": "Open model",
|
| 206 |
"details_model_sheet_overview": "Overview",
|
| 207 |
"details_model_sheet_groups": "Groups",
|
|
|
|
| 201 |
"details_benchmark_sheet_ranking": "Model ranking on this benchmark",
|
| 202 |
"details_benchmark_sheet_rank": "Rank",
|
| 203 |
"details_benchmark_sheet_open": "Open benchmark",
|
| 204 |
+
"details_benchmark_leader_metric": "LEADER · {metric}",
|
| 205 |
"details_model_sheet_open": "Open model",
|
| 206 |
"details_model_sheet_overview": "Overview",
|
| 207 |
"details_model_sheet_groups": "Groups",
|
|
@@ -201,7 +201,7 @@
|
|
| 201 |
"details_benchmark_sheet_ranking": "Рейтинг моделей на этом бенчмарке",
|
| 202 |
"details_benchmark_sheet_rank": "Место",
|
| 203 |
"details_benchmark_sheet_open": "Открыть бенчмарк",
|
| 204 |
-
"
|
| 205 |
"details_model_sheet_open": "Открыть модель",
|
| 206 |
"details_model_sheet_overview": "Обзор",
|
| 207 |
"details_model_sheet_groups": "По группам",
|
|
|
|
| 201 |
"details_benchmark_sheet_ranking": "Рейтинг моделей на этом бенчмарке",
|
| 202 |
"details_benchmark_sheet_rank": "Место",
|
| 203 |
"details_benchmark_sheet_open": "Открыть бенчмарк",
|
| 204 |
+
"details_benchmark_leader_metric": "ЛИДЕР · {metric}",
|
| 205 |
"details_model_sheet_open": "Открыть модель",
|
| 206 |
"details_model_sheet_overview": "Обзор",
|
| 207 |
"details_model_sheet_groups": "По группам",
|
|
@@ -20,6 +20,7 @@ import type { LocalizedString } from '$lib/types/i18n';
|
|
| 20 |
import type { GroupScore, GuardModel } from '$lib/types/leaderboard';
|
| 21 |
|
| 22 |
type SupportedLocale = keyof LocalizedString;
|
|
|
|
| 23 |
|
| 24 |
function localized(value: LocalizedString | null, locale: SupportedLocale, fallback: string) {
|
| 25 |
return value?.[locale] ?? value?.en ?? fallback;
|
|
@@ -179,6 +180,7 @@ export function benchmarkRows(
|
|
| 179 |
models: readonly GuardModel[]
|
| 180 |
): BenchmarkRankingRow[] {
|
| 181 |
if (!groupName) return [];
|
|
|
|
| 182 |
|
| 183 |
return models
|
| 184 |
.map((model) => {
|
|
@@ -193,20 +195,51 @@ export function benchmarkRows(
|
|
| 193 |
})
|
| 194 |
.sort((a, b) => {
|
| 195 |
if (a.hasData !== b.hasData) return a.hasData ? -1 : 1;
|
| 196 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 197 |
});
|
| 198 |
}
|
| 199 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 200 |
function benchmarkLeader(
|
| 201 |
groupName: string,
|
| 202 |
models: readonly GuardModel[]
|
| 203 |
): BenchmarkCard['leader'] {
|
| 204 |
-
const metric =
|
|
|
|
| 205 |
|
| 206 |
return models.reduce<BenchmarkCard['leader']>((leader, model) => {
|
| 207 |
const group = model.groups[groupName];
|
| 208 |
-
const value =
|
| 209 |
-
if (!isNumber(value)
|
|
|
|
| 210 |
|
| 211 |
return {
|
| 212 |
modelShort: model.short,
|
|
|
|
| 20 |
import type { GroupScore, GuardModel } from '$lib/types/leaderboard';
|
| 21 |
|
| 22 |
type SupportedLocale = keyof LocalizedString;
|
| 23 |
+
type BenchmarkLeaderMetric = NonNullable<BenchmarkCard['leader']>['metric'];
|
| 24 |
|
| 25 |
function localized(value: LocalizedString | null, locale: SupportedLocale, fallback: string) {
|
| 26 |
return value?.[locale] ?? value?.en ?? fallback;
|
|
|
|
| 180 |
models: readonly GuardModel[]
|
| 181 |
): BenchmarkRankingRow[] {
|
| 182 |
if (!groupName) return [];
|
| 183 |
+
const metric = benchmarkLeaderMetric(groupName, models);
|
| 184 |
|
| 185 |
return models
|
| 186 |
.map((model) => {
|
|
|
|
| 195 |
})
|
| 196 |
.sort((a, b) => {
|
| 197 |
if (a.hasData !== b.hasData) return a.hasData ? -1 : 1;
|
| 198 |
+
const aValue = benchmarkRowMetricValue(a, metric);
|
| 199 |
+
const bValue = benchmarkRowMetricValue(b, metric);
|
| 200 |
+
const aEmpty = !isNumber(aValue);
|
| 201 |
+
const bEmpty = !isNumber(bValue);
|
| 202 |
+
if (aEmpty && bEmpty) return 0;
|
| 203 |
+
if (aEmpty) return 1;
|
| 204 |
+
if (bEmpty) return -1;
|
| 205 |
+
|
| 206 |
+
return metric === 'f1' ? (bValue ?? 0) - (aValue ?? 0) : (aValue ?? 0) - (bValue ?? 0);
|
| 207 |
});
|
| 208 |
}
|
| 209 |
|
| 210 |
+
function benchmarkLeaderMetric(
|
| 211 |
+
groupName: string,
|
| 212 |
+
models: readonly GuardModel[]
|
| 213 |
+
): BenchmarkLeaderMetric | null {
|
| 214 |
+
const metrics: BenchmarkLeaderMetric[] = ['f1', 'fnr', 'fpr'];
|
| 215 |
+
return (
|
| 216 |
+
metrics.find((metric) => models.some((model) => isNumber(model.groups[groupName]?.[metric]))) ??
|
| 217 |
+
null
|
| 218 |
+
);
|
| 219 |
+
}
|
| 220 |
+
|
| 221 |
+
function benchmarkMetricValue(group: GroupScore | undefined, metric: BenchmarkLeaderMetric | null) {
|
| 222 |
+
if (!group || !metric) return null;
|
| 223 |
+
return group[metric];
|
| 224 |
+
}
|
| 225 |
+
|
| 226 |
+
function benchmarkRowMetricValue(row: BenchmarkRankingRow, metric: BenchmarkLeaderMetric | null) {
|
| 227 |
+
if (!metric) return null;
|
| 228 |
+
return metric === 'f1' ? row.score : row[metric];
|
| 229 |
+
}
|
| 230 |
+
|
| 231 |
function benchmarkLeader(
|
| 232 |
groupName: string,
|
| 233 |
models: readonly GuardModel[]
|
| 234 |
): BenchmarkCard['leader'] {
|
| 235 |
+
const metric = benchmarkLeaderMetric(groupName, models);
|
| 236 |
+
if (!metric) return null;
|
| 237 |
|
| 238 |
return models.reduce<BenchmarkCard['leader']>((leader, model) => {
|
| 239 |
const group = model.groups[groupName];
|
| 240 |
+
const value = benchmarkMetricValue(group, metric);
|
| 241 |
+
if (!isNumber(value)) return leader;
|
| 242 |
+
if (leader && (metric === 'f1' ? leader.value >= value : leader.value <= value)) return leader;
|
| 243 |
|
| 244 |
return {
|
| 245 |
modelShort: model.short,
|
|
@@ -129,7 +129,8 @@ export function benchmarkDetails(model: GuardModel, groups: readonly string[]):
|
|
| 129 |
fnr: model.groups[group]?.fnr ?? null,
|
| 130 |
f1: model.groups[group]?.f1 ?? null
|
| 131 |
}))
|
| 132 |
-
.filter((group) => group.score !== null || group.fpr !== null || group.fnr !== null)
|
|
|
|
| 133 |
}
|
| 134 |
|
| 135 |
export function rankingRowsToCsv(items: readonly LeaderboardRow[]) {
|
|
|
|
| 129 |
fnr: model.groups[group]?.fnr ?? null,
|
| 130 |
f1: model.groups[group]?.f1 ?? null
|
| 131 |
}))
|
| 132 |
+
.filter((group) => group.score !== null || group.fpr !== null || group.fnr !== null)
|
| 133 |
+
.sort((a, b) => (b.score ?? -1) - (a.score ?? -1));
|
| 134 |
}
|
| 135 |
|
| 136 |
export function rankingRowsToCsv(items: readonly LeaderboardRow[]) {
|
|
@@ -5,7 +5,8 @@ import type { HfBucketManifest } from './types';
|
|
| 5 |
const DEFAULT_ENDPOINT = 'https://huggingface.co';
|
| 6 |
const DEFAULT_BUCKET_ID = 'hivetrace/leaderboard_frontend_v2';
|
| 7 |
const DEFAULT_PREFIX = 'latest';
|
| 8 |
-
const DEFAULT_REQUEST_TIMEOUT_MS =
|
|
|
|
| 9 |
|
| 10 |
type HfBucketConfig = {
|
| 11 |
token: string;
|
|
@@ -13,6 +14,7 @@ type HfBucketConfig = {
|
|
| 13 |
prefix: string;
|
| 14 |
endpoint: string;
|
| 15 |
requestTimeoutMs: number;
|
|
|
|
| 16 |
};
|
| 17 |
|
| 18 |
export class HfBucketError extends Error {
|
|
@@ -31,6 +33,12 @@ function positiveInteger(value: string | undefined, fallback: number) {
|
|
| 31 |
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
| 32 |
}
|
| 33 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
function normalizePrefix(value: string) {
|
| 35 |
return value
|
| 36 |
.split('/')
|
|
@@ -55,7 +63,8 @@ function getConfig(): HfBucketConfig {
|
|
| 55 |
bucketId,
|
| 56 |
prefix: normalizePrefix(env.HF_BUCKET_PREFIX ?? DEFAULT_PREFIX),
|
| 57 |
endpoint: (env.HF_BUCKET_ENDPOINT ?? DEFAULT_ENDPOINT).replace(/\/+$/, ''),
|
| 58 |
-
requestTimeoutMs: positiveInteger(env.HF_BUCKET_REQUEST_TIMEOUT_MS, DEFAULT_REQUEST_TIMEOUT_MS)
|
|
|
|
| 59 |
};
|
| 60 |
}
|
| 61 |
|
|
@@ -78,35 +87,43 @@ function responseErrorMessage(status: number, path: string) {
|
|
| 78 |
|
| 79 |
export async function fetchHfBucketText(path: string): Promise<string> {
|
| 80 |
const config = getConfig();
|
| 81 |
-
const
|
| 82 |
-
const timeout = setTimeout(() => controller.abort(), config.requestTimeoutMs);
|
| 83 |
|
| 84 |
-
|
| 85 |
-
const
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
status: response.status
|
| 94 |
});
|
| 95 |
-
}
|
| 96 |
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
}
|
| 106 |
-
throw new HfBucketError(`Failed to read bucket file "${path}".`, { cause });
|
| 107 |
-
} finally {
|
| 108 |
-
clearTimeout(timeout);
|
| 109 |
}
|
|
|
|
|
|
|
| 110 |
}
|
| 111 |
|
| 112 |
export async function fetchHfBucketJson(path: string): Promise<unknown> {
|
|
|
|
| 5 |
const DEFAULT_ENDPOINT = 'https://huggingface.co';
|
| 6 |
const DEFAULT_BUCKET_ID = 'hivetrace/leaderboard_frontend_v2';
|
| 7 |
const DEFAULT_PREFIX = 'latest';
|
| 8 |
+
const DEFAULT_REQUEST_TIMEOUT_MS = 60_000;
|
| 9 |
+
const DEFAULT_REQUEST_RETRIES = 2;
|
| 10 |
|
| 11 |
type HfBucketConfig = {
|
| 12 |
token: string;
|
|
|
|
| 14 |
prefix: string;
|
| 15 |
endpoint: string;
|
| 16 |
requestTimeoutMs: number;
|
| 17 |
+
requestRetries: number;
|
| 18 |
};
|
| 19 |
|
| 20 |
export class HfBucketError extends Error {
|
|
|
|
| 33 |
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
| 34 |
}
|
| 35 |
|
| 36 |
+
function nonNegativeInteger(value: string | undefined, fallback: number) {
|
| 37 |
+
if (!value) return fallback;
|
| 38 |
+
const parsed = Number(value);
|
| 39 |
+
return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback;
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
function normalizePrefix(value: string) {
|
| 43 |
return value
|
| 44 |
.split('/')
|
|
|
|
| 63 |
bucketId,
|
| 64 |
prefix: normalizePrefix(env.HF_BUCKET_PREFIX ?? DEFAULT_PREFIX),
|
| 65 |
endpoint: (env.HF_BUCKET_ENDPOINT ?? DEFAULT_ENDPOINT).replace(/\/+$/, ''),
|
| 66 |
+
requestTimeoutMs: positiveInteger(env.HF_BUCKET_REQUEST_TIMEOUT_MS, DEFAULT_REQUEST_TIMEOUT_MS),
|
| 67 |
+
requestRetries: nonNegativeInteger(env.HF_BUCKET_REQUEST_RETRIES, DEFAULT_REQUEST_RETRIES)
|
| 68 |
};
|
| 69 |
}
|
| 70 |
|
|
|
|
| 87 |
|
| 88 |
export async function fetchHfBucketText(path: string): Promise<string> {
|
| 89 |
const config = getConfig();
|
| 90 |
+
const maxAttempts = config.requestRetries + 1;
|
|
|
|
| 91 |
|
| 92 |
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
| 93 |
+
const controller = new AbortController();
|
| 94 |
+
const timeout = setTimeout(() => controller.abort(), config.requestTimeoutMs);
|
| 95 |
+
|
| 96 |
+
try {
|
| 97 |
+
const response = await fetch(bucketFileUrl(config, path), {
|
| 98 |
+
headers: { Authorization: `Bearer ${config.token}` },
|
| 99 |
+
redirect: 'follow',
|
| 100 |
+
signal: controller.signal
|
|
|
|
| 101 |
});
|
|
|
|
| 102 |
|
| 103 |
+
if (!response.ok) {
|
| 104 |
+
throw new HfBucketError(responseErrorMessage(response.status, path), {
|
| 105 |
+
status: response.status
|
| 106 |
+
});
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
return await response.text();
|
| 110 |
+
} catch (cause) {
|
| 111 |
+
if (cause instanceof HfBucketError) throw cause;
|
| 112 |
+
|
| 113 |
+
if (attempt < maxAttempts) continue;
|
| 114 |
+
if (cause instanceof Error && cause.name === 'AbortError') {
|
| 115 |
+
throw new HfBucketError(
|
| 116 |
+
`Timed out after ${config.requestTimeoutMs} ms while reading "${path}" after ${maxAttempts} attempts.`,
|
| 117 |
+
{ cause }
|
| 118 |
+
);
|
| 119 |
+
}
|
| 120 |
+
throw new HfBucketError(`Failed to read bucket file "${path}".`, { cause });
|
| 121 |
+
} finally {
|
| 122 |
+
clearTimeout(timeout);
|
| 123 |
}
|
|
|
|
|
|
|
|
|
|
| 124 |
}
|
| 125 |
+
|
| 126 |
+
throw new HfBucketError(`Failed to read bucket file "${path}".`);
|
| 127 |
}
|
| 128 |
|
| 129 |
export async function fetchHfBucketJson(path: string): Promise<unknown> {
|
|
@@ -139,25 +139,14 @@ export async function fetchBucketVisualizationSnapshot(
|
|
| 139 |
rankingSnapshot?: BucketRankingSnapshot
|
| 140 |
): Promise<BucketVisualizationSnapshot> {
|
| 141 |
rankingSnapshot ??= await fetchBucketRankingSnapshot();
|
| 142 |
-
const
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
robustnessText
|
| 151 |
-
] = await Promise.all([
|
| 152 |
-
fetchVisualizationText(rankingSnapshot, 'drilldown_index'),
|
| 153 |
-
fetchVisualizationText(rankingSnapshot, 'radar'),
|
| 154 |
-
fetchVisualizationText(rankingSnapshot, 'scatter'),
|
| 155 |
-
fetchVisualizationText(rankingSnapshot, 'heatmap'),
|
| 156 |
-
fetchVisualizationText(rankingSnapshot, 'grouped_bars'),
|
| 157 |
-
fetchVisualizationText(rankingSnapshot, 'pareto'),
|
| 158 |
-
fetchVisualizationText(rankingSnapshot, 'performance'),
|
| 159 |
-
fetchVisualizationText(rankingSnapshot, 'robustness')
|
| 160 |
-
]);
|
| 161 |
|
| 162 |
let snapshot: BucketVisualizationSnapshot;
|
| 163 |
try {
|
|
|
|
| 139 |
rankingSnapshot?: BucketRankingSnapshot
|
| 140 |
): Promise<BucketVisualizationSnapshot> {
|
| 141 |
rankingSnapshot ??= await fetchBucketRankingSnapshot();
|
| 142 |
+
const drilldownIndexText = await fetchVisualizationText(rankingSnapshot, 'drilldown_index');
|
| 143 |
+
const radarText = await fetchVisualizationText(rankingSnapshot, 'radar');
|
| 144 |
+
const scatterText = await fetchVisualizationText(rankingSnapshot, 'scatter');
|
| 145 |
+
const heatmapText = await fetchVisualizationText(rankingSnapshot, 'heatmap');
|
| 146 |
+
const groupedBarsText = await fetchVisualizationText(rankingSnapshot, 'grouped_bars');
|
| 147 |
+
const paretoText = await fetchVisualizationText(rankingSnapshot, 'pareto');
|
| 148 |
+
const performanceText = await fetchVisualizationText(rankingSnapshot, 'performance');
|
| 149 |
+
const robustnessText = await fetchVisualizationText(rankingSnapshot, 'robustness');
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
|
| 151 |
let snapshot: BucketVisualizationSnapshot;
|
| 152 |
try {
|
|
@@ -56,7 +56,7 @@ export type BenchmarkCard = {
|
|
| 56 |
leader: {
|
| 57 |
modelShort: string;
|
| 58 |
value: number;
|
| 59 |
-
metric: 'f1' | '
|
| 60 |
} | null;
|
| 61 |
};
|
| 62 |
|
|
|
|
| 56 |
leader: {
|
| 57 |
modelShort: string;
|
| 58 |
value: number;
|
| 59 |
+
metric: 'f1' | 'fpr' | 'fnr';
|
| 60 |
} | null;
|
| 61 |
};
|
| 62 |
|
|
@@ -257,6 +257,10 @@
|
|
| 257 |
return fmtPercent(value);
|
| 258 |
}
|
| 259 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
function roundedDetailMetric(value: number | null, metric: DetailMetric) {
|
| 261 |
if (!isNumber(value)) return null;
|
| 262 |
if (metric === 'score' || metric === 'f1') return Number(value.toFixed(3));
|
|
@@ -868,13 +872,13 @@
|
|
| 868 |
class="mt-auto flex min-w-0 items-center justify-between gap-3 border-t pt-3 font-mono text-sm"
|
| 869 |
>
|
| 870 |
<div class="flex min-w-0 items-center gap-4">
|
| 871 |
-
|
| 872 |
-
|
| 873 |
-
|
| 874 |
-
|
| 875 |
-
|
| 876 |
-
|
| 877 |
-
|
| 878 |
<OverflowTooltip
|
| 879 |
text={card.leader.modelShort}
|
| 880 |
class="block min-w-0 truncate font-semibold"
|
|
@@ -883,7 +887,7 @@
|
|
| 883 |
/>
|
| 884 |
</div>
|
| 885 |
<span class="shrink-0 font-semibold tabular-nums">
|
| 886 |
-
{
|
| 887 |
</span>
|
| 888 |
</div>
|
| 889 |
{/if}
|
|
|
|
| 257 |
return fmtPercent(value);
|
| 258 |
}
|
| 259 |
|
| 260 |
+
function benchmarkLeaderMetricLabel(metric: 'f1' | 'fpr' | 'fnr') {
|
| 261 |
+
return metric.toUpperCase();
|
| 262 |
+
}
|
| 263 |
+
|
| 264 |
function roundedDetailMetric(value: number | null, metric: DetailMetric) {
|
| 265 |
if (!isNumber(value)) return null;
|
| 266 |
if (metric === 'score' || metric === 'f1') return Number(value.toFixed(3));
|
|
|
|
| 872 |
class="mt-auto flex min-w-0 items-center justify-between gap-3 border-t pt-3 font-mono text-sm"
|
| 873 |
>
|
| 874 |
<div class="flex min-w-0 items-center gap-4">
|
| 875 |
+
<span
|
| 876 |
+
class="shrink-0 text-xs font-semibold tracking-wider text-muted-foreground uppercase"
|
| 877 |
+
>
|
| 878 |
+
{m.details_benchmark_leader_metric({
|
| 879 |
+
metric: benchmarkLeaderMetricLabel(card.leader.metric)
|
| 880 |
+
})}
|
| 881 |
+
</span>
|
| 882 |
<OverflowTooltip
|
| 883 |
text={card.leader.modelShort}
|
| 884 |
class="block min-w-0 truncate font-semibold"
|
|
|
|
| 887 |
/>
|
| 888 |
</div>
|
| 889 |
<span class="shrink-0 font-semibold tabular-nums">
|
| 890 |
+
{fmtMetric(card.leader.value, card.leader.metric)}
|
| 891 |
</span>
|
| 892 |
</div>
|
| 893 |
{/if}
|