Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Sync from huggingface/lerobot-dataset-visualizer via hub-sync
Browse files
src/app/[org]/[dataset]/[episode]/__tests__/fetch-data.test.ts
CHANGED
|
@@ -193,11 +193,13 @@ describe("extractLanguageAtoms", () => {
|
|
| 193 |
|
| 194 |
import { buildVersionedUrl } from "@/utils/versionUtils";
|
| 195 |
import { formatStringWithVars } from "@/utils/parquetUtils";
|
| 196 |
-
import {
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
|
|
|
|
|
|
| 201 |
import { PADDING } from "@/utils/constants";
|
| 202 |
|
| 203 |
const DATASET_BASE = "https://huggingface.co/datasets";
|
|
@@ -318,7 +320,10 @@ describe("v3.0 path construction (lerobot-data-collection/level12_rac_2_2026-02-
|
|
| 318 |
});
|
| 319 |
|
| 320 |
test("data path from episode metadata (chunk 0, file 2)", () => {
|
| 321 |
-
const path =
|
|
|
|
|
|
|
|
|
|
| 322 |
const url = buildVersionedUrl(repoId, version, path);
|
| 323 |
expect(url).toBe(
|
| 324 |
`${DATASET_BASE}/${repoId}/resolve/main/data/chunk-000/file-002.parquet`,
|
|
@@ -326,7 +331,11 @@ describe("v3.0 path construction (lerobot-data-collection/level12_rac_2_2026-02-
|
|
| 326 |
});
|
| 327 |
|
| 328 |
test("video path for top camera (chunk 0, file 0)", () => {
|
| 329 |
-
const path =
|
|
|
|
|
|
|
|
|
|
|
|
|
| 330 |
const url = buildVersionedUrl(repoId, version, path);
|
| 331 |
expect(url).toBe(
|
| 332 |
`${DATASET_BASE}/${repoId}/resolve/main/videos/observation.images.top/chunk-000/file-000.mp4`,
|
|
@@ -335,13 +344,20 @@ describe("v3.0 path construction (lerobot-data-collection/level12_rac_2_2026-02-
|
|
| 335 |
|
| 336 |
test("video path for wrist camera with non-zero file index (per-camera metadata)", () => {
|
| 337 |
// v3.0 supports per-camera video segmentation — each camera can have different file indices
|
| 338 |
-
const path =
|
|
|
|
|
|
|
|
|
|
|
|
|
| 339 |
expect(path).toBe("videos/observation.images.wrist/chunk-000/file-003.mp4");
|
| 340 |
});
|
| 341 |
|
| 342 |
test("data path for large dataset spanning multiple chunks", () => {
|
| 343 |
// Episode in chunk 1, file 5 based on episode metadata
|
| 344 |
-
const path =
|
|
|
|
|
|
|
|
|
|
| 345 |
expect(path).toBe("data/chunk-001/file-005.parquet");
|
| 346 |
});
|
| 347 |
});
|
|
|
|
| 193 |
|
| 194 |
import { buildVersionedUrl } from "@/utils/versionUtils";
|
| 195 |
import { formatStringWithVars } from "@/utils/parquetUtils";
|
| 196 |
+
import { buildV3EpisodesMetadataPath } from "@/utils/stringFormatting";
|
| 197 |
+
import { formatPathTemplate } from "@huggingface/lerobot";
|
| 198 |
+
|
| 199 |
+
const V3_DATA_TEMPLATE =
|
| 200 |
+
"data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet";
|
| 201 |
+
const V3_VIDEO_TEMPLATE =
|
| 202 |
+
"videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4";
|
| 203 |
import { PADDING } from "@/utils/constants";
|
| 204 |
|
| 205 |
const DATASET_BASE = "https://huggingface.co/datasets";
|
|
|
|
| 320 |
});
|
| 321 |
|
| 322 |
test("data path from episode metadata (chunk 0, file 2)", () => {
|
| 323 |
+
const path = formatPathTemplate(V3_DATA_TEMPLATE, {
|
| 324 |
+
chunk_index: 0,
|
| 325 |
+
file_index: 2,
|
| 326 |
+
});
|
| 327 |
const url = buildVersionedUrl(repoId, version, path);
|
| 328 |
expect(url).toBe(
|
| 329 |
`${DATASET_BASE}/${repoId}/resolve/main/data/chunk-000/file-002.parquet`,
|
|
|
|
| 331 |
});
|
| 332 |
|
| 333 |
test("video path for top camera (chunk 0, file 0)", () => {
|
| 334 |
+
const path = formatPathTemplate(V3_VIDEO_TEMPLATE, {
|
| 335 |
+
video_key: "observation.images.top",
|
| 336 |
+
chunk_index: 0,
|
| 337 |
+
file_index: 0,
|
| 338 |
+
});
|
| 339 |
const url = buildVersionedUrl(repoId, version, path);
|
| 340 |
expect(url).toBe(
|
| 341 |
`${DATASET_BASE}/${repoId}/resolve/main/videos/observation.images.top/chunk-000/file-000.mp4`,
|
|
|
|
| 344 |
|
| 345 |
test("video path for wrist camera with non-zero file index (per-camera metadata)", () => {
|
| 346 |
// v3.0 supports per-camera video segmentation — each camera can have different file indices
|
| 347 |
+
const path = formatPathTemplate(V3_VIDEO_TEMPLATE, {
|
| 348 |
+
video_key: "observation.images.wrist",
|
| 349 |
+
chunk_index: 0,
|
| 350 |
+
file_index: 3,
|
| 351 |
+
});
|
| 352 |
expect(path).toBe("videos/observation.images.wrist/chunk-000/file-003.mp4");
|
| 353 |
});
|
| 354 |
|
| 355 |
test("data path for large dataset spanning multiple chunks", () => {
|
| 356 |
// Episode in chunk 1, file 5 based on episode metadata
|
| 357 |
+
const path = formatPathTemplate(V3_DATA_TEMPLATE, {
|
| 358 |
+
chunk_index: 1,
|
| 359 |
+
file_index: 5,
|
| 360 |
+
});
|
| 361 |
expect(path).toBe("data/chunk-001/file-005.parquet");
|
| 362 |
});
|
| 363 |
});
|
src/utils/__tests__/stringFormatting.test.ts
CHANGED
|
@@ -5,8 +5,6 @@ import {
|
|
| 5 |
formatEpisodeIndex,
|
| 6 |
formatFileIndex,
|
| 7 |
formatChunkIndex,
|
| 8 |
-
buildV3VideoPath,
|
| 9 |
-
buildV3DataPath,
|
| 10 |
buildV3EpisodesMetadataPath,
|
| 11 |
} from "@/utils/stringFormatting";
|
| 12 |
|
|
@@ -77,38 +75,6 @@ describe("formatChunkIndex — 3-digit padding (v3.0 chunk_index)", () => {
|
|
| 77 |
});
|
| 78 |
|
| 79 |
// v3.0 specific path builders
|
| 80 |
-
describe("buildV3VideoPath", () => {
|
| 81 |
-
test("single camera, chunk 0, file 0", () => {
|
| 82 |
-
expect(buildV3VideoPath("observation.image", 0, 0)).toBe(
|
| 83 |
-
"videos/observation.image/chunk-000/file-000.mp4",
|
| 84 |
-
);
|
| 85 |
-
});
|
| 86 |
-
|
| 87 |
-
test("nested camera key, non-zero chunk and file", () => {
|
| 88 |
-
expect(buildV3VideoPath("observation.images.wrist", 2, 5)).toBe(
|
| 89 |
-
"videos/observation.images.wrist/chunk-002/file-005.mp4",
|
| 90 |
-
);
|
| 91 |
-
});
|
| 92 |
-
|
| 93 |
-
test("two-camera SO101 dataset style", () => {
|
| 94 |
-
expect(buildV3VideoPath("observation.images.top", 0, 1)).toBe(
|
| 95 |
-
"videos/observation.images.top/chunk-000/file-001.mp4",
|
| 96 |
-
);
|
| 97 |
-
});
|
| 98 |
-
});
|
| 99 |
-
|
| 100 |
-
describe("buildV3DataPath", () => {
|
| 101 |
-
test("chunk 0, file 0", () => {
|
| 102 |
-
expect(buildV3DataPath(0, 0)).toBe("data/chunk-000/file-000.parquet");
|
| 103 |
-
});
|
| 104 |
-
test("chunk 1, file 3", () => {
|
| 105 |
-
expect(buildV3DataPath(1, 3)).toBe("data/chunk-001/file-003.parquet");
|
| 106 |
-
});
|
| 107 |
-
test("large indices", () => {
|
| 108 |
-
expect(buildV3DataPath(10, 99)).toBe("data/chunk-010/file-099.parquet");
|
| 109 |
-
});
|
| 110 |
-
});
|
| 111 |
-
|
| 112 |
describe("buildV3EpisodesMetadataPath", () => {
|
| 113 |
test("chunk 0, file 0 (default for most datasets)", () => {
|
| 114 |
expect(buildV3EpisodesMetadataPath(0, 0)).toBe(
|
|
|
|
| 5 |
formatEpisodeIndex,
|
| 6 |
formatFileIndex,
|
| 7 |
formatChunkIndex,
|
|
|
|
|
|
|
| 8 |
buildV3EpisodesMetadataPath,
|
| 9 |
} from "@/utils/stringFormatting";
|
| 10 |
|
|
|
|
| 75 |
});
|
| 76 |
|
| 77 |
// v3.0 specific path builders
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
describe("buildV3EpisodesMetadataPath", () => {
|
| 79 |
test("chunk 0, file 0 (default for most datasets)", () => {
|
| 80 |
expect(buildV3EpisodesMetadataPath(0, 0)).toBe(
|
src/utils/stringFormatting.ts
CHANGED
|
@@ -56,33 +56,6 @@ export function formatChunkIndex(chunkIndex: number): string {
|
|
| 56 |
return padNumber(chunkIndex, PADDING.CHUNK_INDEX);
|
| 57 |
}
|
| 58 |
|
| 59 |
-
/**
|
| 60 |
-
* Build video path for v3 datasets
|
| 61 |
-
*
|
| 62 |
-
* @param videoKey - Video key/name (e.g., "observation.image")
|
| 63 |
-
* @param chunkIndex - Data chunk index
|
| 64 |
-
* @param fileIndex - File index within chunk
|
| 65 |
-
* @returns Formatted video path (e.g., "videos/observation.image/chunk-001/file-000.mp4")
|
| 66 |
-
*/
|
| 67 |
-
export function buildV3VideoPath(
|
| 68 |
-
videoKey: string,
|
| 69 |
-
chunkIndex: number,
|
| 70 |
-
fileIndex: number,
|
| 71 |
-
): string {
|
| 72 |
-
return `videos/${videoKey}/chunk-${formatChunkIndex(chunkIndex)}/file-${formatFileIndex(fileIndex)}.mp4`;
|
| 73 |
-
}
|
| 74 |
-
|
| 75 |
-
/**
|
| 76 |
-
* Build data path for v3 datasets
|
| 77 |
-
*
|
| 78 |
-
* @param chunkIndex - Data chunk index
|
| 79 |
-
* @param fileIndex - File index within chunk
|
| 80 |
-
* @returns Formatted data path (e.g., "data/chunk-001/file-000.parquet")
|
| 81 |
-
*/
|
| 82 |
-
export function buildV3DataPath(chunkIndex: number, fileIndex: number): string {
|
| 83 |
-
return `data/chunk-${formatChunkIndex(chunkIndex)}/file-${formatFileIndex(fileIndex)}.parquet`;
|
| 84 |
-
}
|
| 85 |
-
|
| 86 |
/**
|
| 87 |
* Build episodes metadata path for v3 datasets
|
| 88 |
*
|
|
|
|
| 56 |
return padNumber(chunkIndex, PADDING.CHUNK_INDEX);
|
| 57 |
}
|
| 58 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
/**
|
| 60 |
* Build episodes metadata path for v3 datasets
|
| 61 |
*
|