mishig HF Staff commited on
Commit
679ec2a
·
verified ·
1 Parent(s): d0df85f

Sync from huggingface/lerobot-dataset-visualizer via hub-sync

Browse files
src/app/[org]/[dataset]/[episode]/__tests__/fetch-data.test.ts CHANGED
@@ -193,7 +193,6 @@ describe("extractLanguageAtoms", () => {
193
 
194
  import { buildVersionedUrl } from "@/utils/versionUtils";
195
  import { formatStringWithVars } from "@/utils/parquetUtils";
196
- import { buildV3EpisodesMetadataPath } from "@/utils/stringFormatting";
197
  import { formatPathTemplate } from "@huggingface/lerobot";
198
 
199
  const V3_DATA_TEMPLATE =
@@ -311,14 +310,6 @@ describe("v3.0 path construction (lerobot-data-collection/level12_rac_2_2026-02-
311
  const repoId = "lerobot-data-collection/level12_rac_2_2026-02-07";
312
  const version = "v3.0";
313
 
314
- test("episode metadata path for first file", () => {
315
- const path = buildV3EpisodesMetadataPath(0, 0);
316
- const url = buildVersionedUrl(repoId, version, path);
317
- expect(url).toBe(
318
- `${DATASET_BASE}/${repoId}/resolve/main/meta/episodes/chunk-000/file-000.parquet`,
319
- );
320
- });
321
-
322
  test("data path from episode metadata (chunk 0, file 2)", () => {
323
  const path = formatPathTemplate(V3_DATA_TEMPLATE, {
324
  chunk_index: 0,
 
193
 
194
  import { buildVersionedUrl } from "@/utils/versionUtils";
195
  import { formatStringWithVars } from "@/utils/parquetUtils";
 
196
  import { formatPathTemplate } from "@huggingface/lerobot";
197
 
198
  const V3_DATA_TEMPLATE =
 
310
  const repoId = "lerobot-data-collection/level12_rac_2_2026-02-07";
311
  const version = "v3.0";
312
 
 
 
 
 
 
 
 
 
313
  test("data path from episode metadata (chunk 0, file 2)", () => {
314
  const path = formatPathTemplate(V3_DATA_TEMPLATE, {
315
  chunk_index: 0,
src/app/[org]/[dataset]/[episode]/fetch-data.ts CHANGED
@@ -15,7 +15,6 @@ import {
15
  processChartDataGroups,
16
  groupRowBySuffix,
17
  } from "@/utils/dataProcessing";
18
- import { buildV3EpisodesMetadataPath } from "@/utils/stringFormatting";
19
  import { bigIntToNumber } from "@/utils/typeGuards";
20
  import {
21
  isGrayscaleShape,
@@ -64,6 +63,14 @@ async function loadLeRobotEpisode(
64
  * v3.0 concatenates episodes into shared files, so each camera carries a time range; v2.x gives every
65
  * episode its own file and the range simply spans it.
66
  */
 
 
 
 
 
 
 
 
67
  function toVideosInfo(
68
  episode: LeRobotEpisode,
69
  info: DatasetMetadata,
@@ -1311,158 +1318,6 @@ function processEpisodeDataForCharts(
1311
  return { chartDataGroups, flatChartData: chartData, ignoredColumns };
1312
  }
1313
 
1314
- // Walks v3.0 episode-metadata parquet files across chunks/files. A new chunk
1315
- // begins when the current chunk's files run out (404 or empty); iteration ends
1316
- // when file-000 of the next chunk 404s. `chunks_size` caps files per chunk, so
1317
- // large datasets can spill past chunk-000.
1318
- async function* iterateEpisodeMetadataFilesV3(
1319
- repoId: string,
1320
- version: string,
1321
- ): AsyncGenerator<Record<string, unknown>[], void, unknown> {
1322
- let chunkIndex = 0;
1323
- let fileIndex = 0;
1324
-
1325
- while (true) {
1326
- const path = buildV3EpisodesMetadataPath(chunkIndex, fileIndex);
1327
- const url = buildVersionedUrl(repoId, version, path);
1328
- let rows: Record<string, unknown>[];
1329
- try {
1330
- const buf = await fetchParquetFile(url);
1331
- rows = await readParquetAsObjects(buf, []);
1332
- } catch {
1333
- if (fileIndex === 0) return;
1334
- chunkIndex++;
1335
- fileIndex = 0;
1336
- continue;
1337
- }
1338
-
1339
- if (rows.length === 0) {
1340
- if (fileIndex === 0) return;
1341
- chunkIndex++;
1342
- fileIndex = 0;
1343
- continue;
1344
- }
1345
-
1346
- yield rows;
1347
- fileIndex++;
1348
- }
1349
- }
1350
-
1351
- // Metadata loading for v3.0 episodes
1352
- // Simple parser for episode row - focuses on key fields for episodes
1353
- function parseEpisodeRowSimple(
1354
- row: Record<string, unknown>,
1355
- ): EpisodeMetadataV3 {
1356
- // v3.0 uses named keys in the episode metadata
1357
- if (row && typeof row === "object") {
1358
- // Check if this is v3.0 format with named keys
1359
- if ("episode_index" in row) {
1360
- // v3.0 format - use named keys
1361
- // Convert BigInt values to numbers
1362
- const toBigIntSafe = (value: unknown): number => {
1363
- if (typeof value === "bigint") return Number(value);
1364
- if (typeof value === "number") return value;
1365
- if (typeof value === "string") return parseInt(value) || 0;
1366
- return 0;
1367
- };
1368
-
1369
- const toNumSafe = (value: unknown): number => {
1370
- if (typeof value === "number") return value;
1371
- if (typeof value === "bigint") return Number(value);
1372
- if (typeof value === "string") return parseFloat(value) || 0;
1373
- return 0;
1374
- };
1375
-
1376
- // Handle video metadata - look for video-specific keys
1377
- const videoKeys = Object.keys(row).filter(
1378
- (key) => key.includes("videos/") && key.includes("/chunk_index"),
1379
- );
1380
- let videoChunkIndex = 0,
1381
- videoFileIndex = 0,
1382
- videoFromTs = 0,
1383
- videoToTs = 30;
1384
- if (videoKeys.length > 0) {
1385
- const videoBaseName = videoKeys[0].replace("/chunk_index", "");
1386
- videoChunkIndex = toBigIntSafe(row[`${videoBaseName}/chunk_index`]);
1387
- videoFileIndex = toBigIntSafe(row[`${videoBaseName}/file_index`]);
1388
- videoFromTs = toNumSafe(row[`${videoBaseName}/from_timestamp`]);
1389
- videoToTs = toNumSafe(row[`${videoBaseName}/to_timestamp`]) || 30;
1390
- }
1391
-
1392
- // lerobot writes episode.tasks as list[str] (v3.0 multi-task support).
1393
- const rawTasks = row["tasks"];
1394
- const tasks = Array.isArray(rawTasks)
1395
- ? rawTasks.filter((t): t is string => typeof t === "string")
1396
- : undefined;
1397
-
1398
- const episodeData: EpisodeMetadataV3 = {
1399
- episode_index: toBigIntSafe(row["episode_index"]),
1400
- data_chunk_index: toBigIntSafe(row["data/chunk_index"]),
1401
- data_file_index: toBigIntSafe(row["data/file_index"]),
1402
- dataset_from_index: toBigIntSafe(row["dataset_from_index"]),
1403
- dataset_to_index: toBigIntSafe(row["dataset_to_index"]),
1404
- length: toBigIntSafe(row["length"]),
1405
- video_chunk_index: videoChunkIndex,
1406
- video_file_index: videoFileIndex,
1407
- video_from_timestamp: videoFromTs,
1408
- video_to_timestamp: videoToTs,
1409
- ...(tasks && tasks.length > 0 ? { tasks } : {}),
1410
- };
1411
-
1412
- // Store per-camera metadata for extractVideoInfoV3WithSegmentation
1413
- Object.keys(row).forEach((key) => {
1414
- if (key.startsWith("videos/")) {
1415
- const val = row[key];
1416
- episodeData[key] =
1417
- typeof val === "bigint"
1418
- ? Number(val)
1419
- : typeof val === "number" || typeof val === "string"
1420
- ? val
1421
- : 0;
1422
- }
1423
- });
1424
-
1425
- return episodeData as EpisodeMetadataV3;
1426
- } else {
1427
- // Fallback to numeric keys for compatibility
1428
- const toNum = (v: unknown, fallback = 0): number =>
1429
- typeof v === "number"
1430
- ? v
1431
- : typeof v === "bigint"
1432
- ? Number(v)
1433
- : fallback;
1434
- return {
1435
- episode_index: toNum(row["0"]),
1436
- data_chunk_index: toNum(row["1"]),
1437
- data_file_index: toNum(row["2"]),
1438
- dataset_from_index: toNum(row["3"]),
1439
- dataset_to_index: toNum(row["4"]),
1440
- video_chunk_index: toNum(row["5"]),
1441
- video_file_index: toNum(row["6"]),
1442
- video_from_timestamp: toNum(row["7"]),
1443
- video_to_timestamp: toNum(row["8"], 30),
1444
- length: toNum(row["9"], 30),
1445
- };
1446
- }
1447
- }
1448
-
1449
- // Fallback if parsing fails
1450
- const fallback = {
1451
- episode_index: 0,
1452
- data_chunk_index: 0,
1453
- data_file_index: 0,
1454
- dataset_from_index: 0,
1455
- dataset_to_index: 0,
1456
- video_chunk_index: 0,
1457
- video_file_index: 0,
1458
- video_from_timestamp: 0,
1459
- video_to_timestamp: 30,
1460
- length: 30,
1461
- };
1462
-
1463
- return fallback;
1464
- }
1465
-
1466
  // ─── Stats computation ───────────────────────────────────────────
1467
 
1468
  /**
@@ -1519,29 +1374,11 @@ export async function loadAllEpisodeLengthsV3(
1519
  fps: number,
1520
  ): Promise<EpisodeLengthStats | null> {
1521
  try {
1522
- const allEpisodes: { index: number; length: number }[] = [];
1523
- let fileIndex = 0;
1524
- const chunkIndex = 0;
1525
-
1526
- while (true) {
1527
- const path = `meta/episodes/chunk-${chunkIndex.toString().padStart(3, "0")}/file-${fileIndex.toString().padStart(3, "0")}.parquet`;
1528
- const url = buildVersionedUrl(repoId, version, path);
1529
- try {
1530
- const buf = await fetchParquetFile(url);
1531
- const rows = await readParquetAsObjects(buf, []);
1532
- if (rows.length === 0 && fileIndex > 0) break;
1533
- for (const row of rows) {
1534
- const parsed = parseEpisodeRowSimple(row);
1535
- allEpisodes.push({
1536
- index: parsed.episode_index,
1537
- length: parsed.length,
1538
- });
1539
- }
1540
- fileIndex++;
1541
- } catch {
1542
- break;
1543
- }
1544
- }
1545
 
1546
  if (allEpisodes.length === 0) return null;
1547
 
@@ -1663,35 +1500,18 @@ export async function loadAllEpisodeFrameInfo(
1663
  );
1664
 
1665
  if (version === "v3.0") {
1666
- for await (const rows of iterateEpisodeMetadataFilesV3(repoId, version)) {
1667
- for (const row of rows) {
1668
- const epIdx = Number(row["episode_index"] ?? 0);
1669
- if (sampledEpisodeSet && !sampledEpisodeSet.has(epIdx)) continue;
1670
- for (const cam of cameras) {
1671
- const cIdx = Number(
1672
- row[`videos/${cam}/chunk_index`] ?? row["video_chunk_index"] ?? 0,
1673
- );
1674
- const fIdx = Number(
1675
- row[`videos/${cam}/file_index`] ?? row["video_file_index"] ?? 0,
1676
- );
1677
- const fromTs = Number(
1678
- row[`videos/${cam}/from_timestamp`] ??
1679
- row["video_from_timestamp"] ??
1680
- 0,
1681
- );
1682
- const toTs = Number(
1683
- row[`videos/${cam}/to_timestamp`] ??
1684
- row["video_to_timestamp"] ??
1685
- 30,
1686
- );
1687
- const videoPath = `videos/${cam}/chunk-${cIdx.toString().padStart(3, "0")}/file-${fIdx.toString().padStart(3, "0")}.mp4`;
1688
- framesByCamera[cam].push({
1689
- episodeIndex: epIdx,
1690
- videoUrl: buildVersionedUrl(repoId, version, videoPath),
1691
- firstFrameTime: fromTs,
1692
- lastFrameTime: Math.max(0, toTs - 0.05),
1693
- });
1694
- }
1695
  }
1696
  }
1697
  return { cameras, framesByCamera };
@@ -1825,29 +1645,28 @@ export async function loadCrossEpisodeActionVariance(
1825
  // Collect episode metadata
1826
  type EpMeta = {
1827
  index: number;
1828
- chunkIdx: number;
1829
- fileIdx: number;
1830
  from: number;
1831
  to: number;
1832
  };
1833
  const allEps: EpMeta[] = [];
1834
 
1835
  if (version === "v3.0") {
1836
- for await (const rows of iterateEpisodeMetadataFilesV3(repoId, version)) {
1837
- for (const row of rows) {
1838
- const parsed = parseEpisodeRowSimple(row);
1839
- allEps.push({
1840
- index: parsed.episode_index,
1841
- chunkIdx: parsed.data_chunk_index,
1842
- fileIdx: parsed.data_file_index,
1843
- from: parsed.dataset_from_index,
1844
- to: parsed.dataset_to_index,
1845
- });
1846
- }
1847
  }
1848
  } else {
1849
  for (let i = 0; i < info.total_episodes; i++) {
1850
- allEps.push({ index: i, chunkIdx: 0, fileIdx: 0, from: 0, to: 0 });
1851
  }
1852
  }
1853
 
@@ -1880,7 +1699,7 @@ export async function loadCrossEpisodeActionVariance(
1880
  if (version === "v3.0") {
1881
  const byFile = new Map<string, EpMeta[]>();
1882
  for (const ep of sampled) {
1883
- const key = `${ep.chunkIdx}-${ep.fileIdx}`;
1884
  if (!byFile.has(key)) byFile.set(key, []);
1885
  byFile.get(key)!.push(ep);
1886
  }
@@ -1888,13 +1707,10 @@ export async function loadCrossEpisodeActionVariance(
1888
  const fileResults = await Promise.all(
1889
  [...byFile.values()].map(async (eps) => {
1890
  const ep0 = eps[0];
1891
- const dataPath = `data/chunk-${ep0.chunkIdx.toString().padStart(3, "0")}/file-${ep0.fileIdx.toString().padStart(3, "0")}.parquet`;
1892
  const fileEpActions: { index: number; actions: number[][] }[] = [];
1893
  const fileEpStates: (number[][] | null)[] = [];
1894
  try {
1895
- const buf = await fetchParquetFile(
1896
- buildVersionedUrl(repoId, version, dataPath),
1897
- );
1898
  const rows = await readParquetAsObjects(
1899
  buf,
1900
  stateKey ? ["index", actionKey, stateKey] : ["index", actionKey],
 
15
  processChartDataGroups,
16
  groupRowBySuffix,
17
  } from "@/utils/dataProcessing";
 
18
  import { bigIntToNumber } from "@/utils/typeGuards";
19
  import {
20
  isGrayscaleShape,
 
63
  * v3.0 concatenates episodes into shared files, so each camera carries a time range; v2.x gives every
64
  * episode its own file and the range simply spans it.
65
  */
66
+ /** Every episode, normalized. v3 only: v2 needs no index read to locate an episode. */
67
+ async function loadLeRobotEpisodes(
68
+ repoId: string,
69
+ totalEpisodes: number,
70
+ ): Promise<LeRobotEpisode[]> {
71
+ return leRobotDataset(repoId).episodes({ offset: 0, limit: totalEpisodes });
72
+ }
73
+
74
  function toVideosInfo(
75
  episode: LeRobotEpisode,
76
  info: DatasetMetadata,
 
1318
  return { chartDataGroups, flatChartData: chartData, ignoredColumns };
1319
  }
1320
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1321
  // ─── Stats computation ───────────────────────────────────────────
1322
 
1323
  /**
 
1374
  fps: number,
1375
  ): Promise<EpisodeLengthStats | null> {
1376
  try {
1377
+ const dataset = leRobotDataset(repoId);
1378
+ const { totalEpisodes } = await dataset.info();
1379
+ const allEpisodes = (
1380
+ await dataset.episodes({ offset: 0, limit: totalEpisodes })
1381
+ ).map((episode) => ({ index: episode.index, length: episode.length }));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1382
 
1383
  if (allEpisodes.length === 0) return null;
1384
 
 
1500
  );
1501
 
1502
  if (version === "v3.0") {
1503
+ for (const episode of await loadLeRobotEpisodes(
1504
+ repoId,
1505
+ info.total_episodes,
1506
+ )) {
1507
+ if (sampledEpisodeSet && !sampledEpisodeSet.has(episode.index)) continue;
1508
+ for (const video of episode.videos) {
1509
+ framesByCamera[video.cameraKey]?.push({
1510
+ episodeIndex: episode.index,
1511
+ videoUrl: video.url,
1512
+ firstFrameTime: video.fromSec,
1513
+ lastFrameTime: Math.max(0, video.toSec - 0.05),
1514
+ });
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1515
  }
1516
  }
1517
  return { cameras, framesByCamera };
 
1645
  // Collect episode metadata
1646
  type EpMeta = {
1647
  index: number;
1648
+ /** v3 only; v2 derives its per-episode path from the index. */
1649
+ dataUrl?: string;
1650
  from: number;
1651
  to: number;
1652
  };
1653
  const allEps: EpMeta[] = [];
1654
 
1655
  if (version === "v3.0") {
1656
+ for (const episode of await loadLeRobotEpisodes(
1657
+ repoId,
1658
+ info.total_episodes,
1659
+ )) {
1660
+ allEps.push({
1661
+ index: episode.index,
1662
+ dataUrl: episode.data.url,
1663
+ from: episode.data.fromRow,
1664
+ to: episode.data.toRow,
1665
+ });
 
1666
  }
1667
  } else {
1668
  for (let i = 0; i < info.total_episodes; i++) {
1669
+ allEps.push({ index: i, from: 0, to: 0 });
1670
  }
1671
  }
1672
 
 
1699
  if (version === "v3.0") {
1700
  const byFile = new Map<string, EpMeta[]>();
1701
  for (const ep of sampled) {
1702
+ const key = ep.dataUrl ?? "";
1703
  if (!byFile.has(key)) byFile.set(key, []);
1704
  byFile.get(key)!.push(ep);
1705
  }
 
1707
  const fileResults = await Promise.all(
1708
  [...byFile.values()].map(async (eps) => {
1709
  const ep0 = eps[0];
 
1710
  const fileEpActions: { index: number; actions: number[][] }[] = [];
1711
  const fileEpStates: (number[][] | null)[] = [];
1712
  try {
1713
+ const buf = await fetchParquetFile(ep0.dataUrl ?? "");
 
 
1714
  const rows = await readParquetAsObjects(
1715
  buf,
1716
  stateKey ? ["index", actionKey, stateKey] : ["index", actionKey],
src/utils/__tests__/stringFormatting.test.ts CHANGED
@@ -5,7 +5,6 @@ import {
5
  formatEpisodeIndex,
6
  formatFileIndex,
7
  formatChunkIndex,
8
- buildV3EpisodesMetadataPath,
9
  } from "@/utils/stringFormatting";
10
 
11
  // These utilities are the foundation of v3.0 path construction.
@@ -75,15 +74,3 @@ describe("formatChunkIndex — 3-digit padding (v3.0 chunk_index)", () => {
75
  });
76
 
77
  // v3.0 specific path builders
78
- describe("buildV3EpisodesMetadataPath", () => {
79
- test("chunk 0, file 0 (default for most datasets)", () => {
80
- expect(buildV3EpisodesMetadataPath(0, 0)).toBe(
81
- "meta/episodes/chunk-000/file-000.parquet",
82
- );
83
- });
84
- test("chunk 0, file 2 (multiple metadata files)", () => {
85
- expect(buildV3EpisodesMetadataPath(0, 2)).toBe(
86
- "meta/episodes/chunk-000/file-002.parquet",
87
- );
88
- });
89
- });
 
5
  formatEpisodeIndex,
6
  formatFileIndex,
7
  formatChunkIndex,
 
8
  } from "@/utils/stringFormatting";
9
 
10
  // These utilities are the foundation of v3.0 path construction.
 
74
  });
75
 
76
  // v3.0 specific path builders
 
 
 
 
 
 
 
 
 
 
 
 
src/utils/stringFormatting.ts CHANGED
@@ -55,17 +55,3 @@ export function formatFileIndex(fileIndex: number): string {
55
  export function formatChunkIndex(chunkIndex: number): string {
56
  return padNumber(chunkIndex, PADDING.CHUNK_INDEX);
57
  }
58
-
59
- /**
60
- * Build episodes metadata path for v3 datasets
61
- *
62
- * @param chunkIndex - Episode chunk index
63
- * @param fileIndex - File index within chunk
64
- * @returns Formatted episodes metadata path (e.g., "meta/episodes/chunk-001/file-000.parquet")
65
- */
66
- export function buildV3EpisodesMetadataPath(
67
- chunkIndex: number,
68
- fileIndex: number,
69
- ): string {
70
- return `meta/episodes/chunk-${formatChunkIndex(chunkIndex)}/file-${formatFileIndex(fileIndex)}.parquet`;
71
- }
 
55
  export function formatChunkIndex(chunkIndex: number): string {
56
  return padNumber(chunkIndex, PADDING.CHUNK_INDEX);
57
  }