mishig HF Staff commited on
Commit
d0df85f
·
verified ·
1 Parent(s): 45aceed

Sync from huggingface/lerobot-dataset-visualizer via hub-sync

Browse files
src/app/[org]/[dataset]/[episode]/__tests__/fetch-data.test.ts CHANGED
@@ -193,11 +193,13 @@ describe("extractLanguageAtoms", () => {
193
 
194
  import { buildVersionedUrl } from "@/utils/versionUtils";
195
  import { formatStringWithVars } from "@/utils/parquetUtils";
196
- import {
197
- buildV3DataPath,
198
- buildV3VideoPath,
199
- buildV3EpisodesMetadataPath,
200
- } from "@/utils/stringFormatting";
 
 
201
  import { PADDING } from "@/utils/constants";
202
 
203
  const DATASET_BASE = "https://huggingface.co/datasets";
@@ -318,7 +320,10 @@ describe("v3.0 path construction (lerobot-data-collection/level12_rac_2_2026-02-
318
  });
319
 
320
  test("data path from episode metadata (chunk 0, file 2)", () => {
321
- const path = buildV3DataPath(0, 2);
 
 
 
322
  const url = buildVersionedUrl(repoId, version, path);
323
  expect(url).toBe(
324
  `${DATASET_BASE}/${repoId}/resolve/main/data/chunk-000/file-002.parquet`,
@@ -326,7 +331,11 @@ describe("v3.0 path construction (lerobot-data-collection/level12_rac_2_2026-02-
326
  });
327
 
328
  test("video path for top camera (chunk 0, file 0)", () => {
329
- const path = buildV3VideoPath("observation.images.top", 0, 0);
 
 
 
 
330
  const url = buildVersionedUrl(repoId, version, path);
331
  expect(url).toBe(
332
  `${DATASET_BASE}/${repoId}/resolve/main/videos/observation.images.top/chunk-000/file-000.mp4`,
@@ -335,13 +344,20 @@ describe("v3.0 path construction (lerobot-data-collection/level12_rac_2_2026-02-
335
 
336
  test("video path for wrist camera with non-zero file index (per-camera metadata)", () => {
337
  // v3.0 supports per-camera video segmentation — each camera can have different file indices
338
- const path = buildV3VideoPath("observation.images.wrist", 0, 3);
 
 
 
 
339
  expect(path).toBe("videos/observation.images.wrist/chunk-000/file-003.mp4");
340
  });
341
 
342
  test("data path for large dataset spanning multiple chunks", () => {
343
  // Episode in chunk 1, file 5 based on episode metadata
344
- const path = buildV3DataPath(1, 5);
 
 
 
345
  expect(path).toBe("data/chunk-001/file-005.parquet");
346
  });
347
  });
 
193
 
194
  import { buildVersionedUrl } from "@/utils/versionUtils";
195
  import { formatStringWithVars } from "@/utils/parquetUtils";
196
+ import { buildV3EpisodesMetadataPath } from "@/utils/stringFormatting";
197
+ import { formatPathTemplate } from "@huggingface/lerobot";
198
+
199
+ const V3_DATA_TEMPLATE =
200
+ "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet";
201
+ const V3_VIDEO_TEMPLATE =
202
+ "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4";
203
  import { PADDING } from "@/utils/constants";
204
 
205
  const DATASET_BASE = "https://huggingface.co/datasets";
 
320
  });
321
 
322
  test("data path from episode metadata (chunk 0, file 2)", () => {
323
+ const path = formatPathTemplate(V3_DATA_TEMPLATE, {
324
+ chunk_index: 0,
325
+ file_index: 2,
326
+ });
327
  const url = buildVersionedUrl(repoId, version, path);
328
  expect(url).toBe(
329
  `${DATASET_BASE}/${repoId}/resolve/main/data/chunk-000/file-002.parquet`,
 
331
  });
332
 
333
  test("video path for top camera (chunk 0, file 0)", () => {
334
+ const path = formatPathTemplate(V3_VIDEO_TEMPLATE, {
335
+ video_key: "observation.images.top",
336
+ chunk_index: 0,
337
+ file_index: 0,
338
+ });
339
  const url = buildVersionedUrl(repoId, version, path);
340
  expect(url).toBe(
341
  `${DATASET_BASE}/${repoId}/resolve/main/videos/observation.images.top/chunk-000/file-000.mp4`,
 
344
 
345
  test("video path for wrist camera with non-zero file index (per-camera metadata)", () => {
346
  // v3.0 supports per-camera video segmentation — each camera can have different file indices
347
+ const path = formatPathTemplate(V3_VIDEO_TEMPLATE, {
348
+ video_key: "observation.images.wrist",
349
+ chunk_index: 0,
350
+ file_index: 3,
351
+ });
352
  expect(path).toBe("videos/observation.images.wrist/chunk-000/file-003.mp4");
353
  });
354
 
355
  test("data path for large dataset spanning multiple chunks", () => {
356
  // Episode in chunk 1, file 5 based on episode metadata
357
+ const path = formatPathTemplate(V3_DATA_TEMPLATE, {
358
+ chunk_index: 1,
359
+ file_index: 5,
360
+ });
361
  expect(path).toBe("data/chunk-001/file-005.parquet");
362
  });
363
  });
src/utils/__tests__/stringFormatting.test.ts CHANGED
@@ -5,8 +5,6 @@ import {
5
  formatEpisodeIndex,
6
  formatFileIndex,
7
  formatChunkIndex,
8
- buildV3VideoPath,
9
- buildV3DataPath,
10
  buildV3EpisodesMetadataPath,
11
  } from "@/utils/stringFormatting";
12
 
@@ -77,38 +75,6 @@ describe("formatChunkIndex — 3-digit padding (v3.0 chunk_index)", () => {
77
  });
78
 
79
  // v3.0 specific path builders
80
- describe("buildV3VideoPath", () => {
81
- test("single camera, chunk 0, file 0", () => {
82
- expect(buildV3VideoPath("observation.image", 0, 0)).toBe(
83
- "videos/observation.image/chunk-000/file-000.mp4",
84
- );
85
- });
86
-
87
- test("nested camera key, non-zero chunk and file", () => {
88
- expect(buildV3VideoPath("observation.images.wrist", 2, 5)).toBe(
89
- "videos/observation.images.wrist/chunk-002/file-005.mp4",
90
- );
91
- });
92
-
93
- test("two-camera SO101 dataset style", () => {
94
- expect(buildV3VideoPath("observation.images.top", 0, 1)).toBe(
95
- "videos/observation.images.top/chunk-000/file-001.mp4",
96
- );
97
- });
98
- });
99
-
100
- describe("buildV3DataPath", () => {
101
- test("chunk 0, file 0", () => {
102
- expect(buildV3DataPath(0, 0)).toBe("data/chunk-000/file-000.parquet");
103
- });
104
- test("chunk 1, file 3", () => {
105
- expect(buildV3DataPath(1, 3)).toBe("data/chunk-001/file-003.parquet");
106
- });
107
- test("large indices", () => {
108
- expect(buildV3DataPath(10, 99)).toBe("data/chunk-010/file-099.parquet");
109
- });
110
- });
111
-
112
  describe("buildV3EpisodesMetadataPath", () => {
113
  test("chunk 0, file 0 (default for most datasets)", () => {
114
  expect(buildV3EpisodesMetadataPath(0, 0)).toBe(
 
5
  formatEpisodeIndex,
6
  formatFileIndex,
7
  formatChunkIndex,
 
 
8
  buildV3EpisodesMetadataPath,
9
  } from "@/utils/stringFormatting";
10
 
 
75
  });
76
 
77
  // v3.0 specific path builders
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
78
  describe("buildV3EpisodesMetadataPath", () => {
79
  test("chunk 0, file 0 (default for most datasets)", () => {
80
  expect(buildV3EpisodesMetadataPath(0, 0)).toBe(
src/utils/stringFormatting.ts CHANGED
@@ -56,33 +56,6 @@ export function formatChunkIndex(chunkIndex: number): string {
56
  return padNumber(chunkIndex, PADDING.CHUNK_INDEX);
57
  }
58
 
59
- /**
60
- * Build video path for v3 datasets
61
- *
62
- * @param videoKey - Video key/name (e.g., "observation.image")
63
- * @param chunkIndex - Data chunk index
64
- * @param fileIndex - File index within chunk
65
- * @returns Formatted video path (e.g., "videos/observation.image/chunk-001/file-000.mp4")
66
- */
67
- export function buildV3VideoPath(
68
- videoKey: string,
69
- chunkIndex: number,
70
- fileIndex: number,
71
- ): string {
72
- return `videos/${videoKey}/chunk-${formatChunkIndex(chunkIndex)}/file-${formatFileIndex(fileIndex)}.mp4`;
73
- }
74
-
75
- /**
76
- * Build data path for v3 datasets
77
- *
78
- * @param chunkIndex - Data chunk index
79
- * @param fileIndex - File index within chunk
80
- * @returns Formatted data path (e.g., "data/chunk-001/file-000.parquet")
81
- */
82
- export function buildV3DataPath(chunkIndex: number, fileIndex: number): string {
83
- return `data/chunk-${formatChunkIndex(chunkIndex)}/file-${formatFileIndex(fileIndex)}.parquet`;
84
- }
85
-
86
  /**
87
  * Build episodes metadata path for v3 datasets
88
  *
 
56
  return padNumber(chunkIndex, PADDING.CHUNK_INDEX);
57
  }
58
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
59
  /**
60
  * Build episodes metadata path for v3 datasets
61
  *