Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Sync from huggingface/lerobot-dataset-visualizer via hub-sync
Browse files
src/app/[org]/[dataset]/[episode]/__tests__/fetch-data.test.ts
CHANGED
|
@@ -193,7 +193,6 @@ describe("extractLanguageAtoms", () => {
|
|
| 193 |
|
| 194 |
import { buildVersionedUrl } from "@/utils/versionUtils";
|
| 195 |
import { formatStringWithVars } from "@/utils/parquetUtils";
|
| 196 |
-
import { buildV3EpisodesMetadataPath } from "@/utils/stringFormatting";
|
| 197 |
import { formatPathTemplate } from "@huggingface/lerobot";
|
| 198 |
|
| 199 |
const V3_DATA_TEMPLATE =
|
|
@@ -311,14 +310,6 @@ describe("v3.0 path construction (lerobot-data-collection/level12_rac_2_2026-02-
|
|
| 311 |
const repoId = "lerobot-data-collection/level12_rac_2_2026-02-07";
|
| 312 |
const version = "v3.0";
|
| 313 |
|
| 314 |
-
test("episode metadata path for first file", () => {
|
| 315 |
-
const path = buildV3EpisodesMetadataPath(0, 0);
|
| 316 |
-
const url = buildVersionedUrl(repoId, version, path);
|
| 317 |
-
expect(url).toBe(
|
| 318 |
-
`${DATASET_BASE}/${repoId}/resolve/main/meta/episodes/chunk-000/file-000.parquet`,
|
| 319 |
-
);
|
| 320 |
-
});
|
| 321 |
-
|
| 322 |
test("data path from episode metadata (chunk 0, file 2)", () => {
|
| 323 |
const path = formatPathTemplate(V3_DATA_TEMPLATE, {
|
| 324 |
chunk_index: 0,
|
|
|
|
| 193 |
|
| 194 |
import { buildVersionedUrl } from "@/utils/versionUtils";
|
| 195 |
import { formatStringWithVars } from "@/utils/parquetUtils";
|
|
|
|
| 196 |
import { formatPathTemplate } from "@huggingface/lerobot";
|
| 197 |
|
| 198 |
const V3_DATA_TEMPLATE =
|
|
|
|
| 310 |
const repoId = "lerobot-data-collection/level12_rac_2_2026-02-07";
|
| 311 |
const version = "v3.0";
|
| 312 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 313 |
test("data path from episode metadata (chunk 0, file 2)", () => {
|
| 314 |
const path = formatPathTemplate(V3_DATA_TEMPLATE, {
|
| 315 |
chunk_index: 0,
|
src/app/[org]/[dataset]/[episode]/fetch-data.ts
CHANGED
|
@@ -15,7 +15,6 @@ import {
|
|
| 15 |
processChartDataGroups,
|
| 16 |
groupRowBySuffix,
|
| 17 |
} from "@/utils/dataProcessing";
|
| 18 |
-
import { buildV3EpisodesMetadataPath } from "@/utils/stringFormatting";
|
| 19 |
import { bigIntToNumber } from "@/utils/typeGuards";
|
| 20 |
import {
|
| 21 |
isGrayscaleShape,
|
|
@@ -64,6 +63,14 @@ async function loadLeRobotEpisode(
|
|
| 64 |
* v3.0 concatenates episodes into shared files, so each camera carries a time range; v2.x gives every
|
| 65 |
* episode its own file and the range simply spans it.
|
| 66 |
*/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
function toVideosInfo(
|
| 68 |
episode: LeRobotEpisode,
|
| 69 |
info: DatasetMetadata,
|
|
@@ -1311,158 +1318,6 @@ function processEpisodeDataForCharts(
|
|
| 1311 |
return { chartDataGroups, flatChartData: chartData, ignoredColumns };
|
| 1312 |
}
|
| 1313 |
|
| 1314 |
-
// Walks v3.0 episode-metadata parquet files across chunks/files. A new chunk
|
| 1315 |
-
// begins when the current chunk's files run out (404 or empty); iteration ends
|
| 1316 |
-
// when file-000 of the next chunk 404s. `chunks_size` caps files per chunk, so
|
| 1317 |
-
// large datasets can spill past chunk-000.
|
| 1318 |
-
async function* iterateEpisodeMetadataFilesV3(
|
| 1319 |
-
repoId: string,
|
| 1320 |
-
version: string,
|
| 1321 |
-
): AsyncGenerator<Record<string, unknown>[], void, unknown> {
|
| 1322 |
-
let chunkIndex = 0;
|
| 1323 |
-
let fileIndex = 0;
|
| 1324 |
-
|
| 1325 |
-
while (true) {
|
| 1326 |
-
const path = buildV3EpisodesMetadataPath(chunkIndex, fileIndex);
|
| 1327 |
-
const url = buildVersionedUrl(repoId, version, path);
|
| 1328 |
-
let rows: Record<string, unknown>[];
|
| 1329 |
-
try {
|
| 1330 |
-
const buf = await fetchParquetFile(url);
|
| 1331 |
-
rows = await readParquetAsObjects(buf, []);
|
| 1332 |
-
} catch {
|
| 1333 |
-
if (fileIndex === 0) return;
|
| 1334 |
-
chunkIndex++;
|
| 1335 |
-
fileIndex = 0;
|
| 1336 |
-
continue;
|
| 1337 |
-
}
|
| 1338 |
-
|
| 1339 |
-
if (rows.length === 0) {
|
| 1340 |
-
if (fileIndex === 0) return;
|
| 1341 |
-
chunkIndex++;
|
| 1342 |
-
fileIndex = 0;
|
| 1343 |
-
continue;
|
| 1344 |
-
}
|
| 1345 |
-
|
| 1346 |
-
yield rows;
|
| 1347 |
-
fileIndex++;
|
| 1348 |
-
}
|
| 1349 |
-
}
|
| 1350 |
-
|
| 1351 |
-
// Metadata loading for v3.0 episodes
|
| 1352 |
-
// Simple parser for episode row - focuses on key fields for episodes
|
| 1353 |
-
function parseEpisodeRowSimple(
|
| 1354 |
-
row: Record<string, unknown>,
|
| 1355 |
-
): EpisodeMetadataV3 {
|
| 1356 |
-
// v3.0 uses named keys in the episode metadata
|
| 1357 |
-
if (row && typeof row === "object") {
|
| 1358 |
-
// Check if this is v3.0 format with named keys
|
| 1359 |
-
if ("episode_index" in row) {
|
| 1360 |
-
// v3.0 format - use named keys
|
| 1361 |
-
// Convert BigInt values to numbers
|
| 1362 |
-
const toBigIntSafe = (value: unknown): number => {
|
| 1363 |
-
if (typeof value === "bigint") return Number(value);
|
| 1364 |
-
if (typeof value === "number") return value;
|
| 1365 |
-
if (typeof value === "string") return parseInt(value) || 0;
|
| 1366 |
-
return 0;
|
| 1367 |
-
};
|
| 1368 |
-
|
| 1369 |
-
const toNumSafe = (value: unknown): number => {
|
| 1370 |
-
if (typeof value === "number") return value;
|
| 1371 |
-
if (typeof value === "bigint") return Number(value);
|
| 1372 |
-
if (typeof value === "string") return parseFloat(value) || 0;
|
| 1373 |
-
return 0;
|
| 1374 |
-
};
|
| 1375 |
-
|
| 1376 |
-
// Handle video metadata - look for video-specific keys
|
| 1377 |
-
const videoKeys = Object.keys(row).filter(
|
| 1378 |
-
(key) => key.includes("videos/") && key.includes("/chunk_index"),
|
| 1379 |
-
);
|
| 1380 |
-
let videoChunkIndex = 0,
|
| 1381 |
-
videoFileIndex = 0,
|
| 1382 |
-
videoFromTs = 0,
|
| 1383 |
-
videoToTs = 30;
|
| 1384 |
-
if (videoKeys.length > 0) {
|
| 1385 |
-
const videoBaseName = videoKeys[0].replace("/chunk_index", "");
|
| 1386 |
-
videoChunkIndex = toBigIntSafe(row[`${videoBaseName}/chunk_index`]);
|
| 1387 |
-
videoFileIndex = toBigIntSafe(row[`${videoBaseName}/file_index`]);
|
| 1388 |
-
videoFromTs = toNumSafe(row[`${videoBaseName}/from_timestamp`]);
|
| 1389 |
-
videoToTs = toNumSafe(row[`${videoBaseName}/to_timestamp`]) || 30;
|
| 1390 |
-
}
|
| 1391 |
-
|
| 1392 |
-
// lerobot writes episode.tasks as list[str] (v3.0 multi-task support).
|
| 1393 |
-
const rawTasks = row["tasks"];
|
| 1394 |
-
const tasks = Array.isArray(rawTasks)
|
| 1395 |
-
? rawTasks.filter((t): t is string => typeof t === "string")
|
| 1396 |
-
: undefined;
|
| 1397 |
-
|
| 1398 |
-
const episodeData: EpisodeMetadataV3 = {
|
| 1399 |
-
episode_index: toBigIntSafe(row["episode_index"]),
|
| 1400 |
-
data_chunk_index: toBigIntSafe(row["data/chunk_index"]),
|
| 1401 |
-
data_file_index: toBigIntSafe(row["data/file_index"]),
|
| 1402 |
-
dataset_from_index: toBigIntSafe(row["dataset_from_index"]),
|
| 1403 |
-
dataset_to_index: toBigIntSafe(row["dataset_to_index"]),
|
| 1404 |
-
length: toBigIntSafe(row["length"]),
|
| 1405 |
-
video_chunk_index: videoChunkIndex,
|
| 1406 |
-
video_file_index: videoFileIndex,
|
| 1407 |
-
video_from_timestamp: videoFromTs,
|
| 1408 |
-
video_to_timestamp: videoToTs,
|
| 1409 |
-
...(tasks && tasks.length > 0 ? { tasks } : {}),
|
| 1410 |
-
};
|
| 1411 |
-
|
| 1412 |
-
// Store per-camera metadata for extractVideoInfoV3WithSegmentation
|
| 1413 |
-
Object.keys(row).forEach((key) => {
|
| 1414 |
-
if (key.startsWith("videos/")) {
|
| 1415 |
-
const val = row[key];
|
| 1416 |
-
episodeData[key] =
|
| 1417 |
-
typeof val === "bigint"
|
| 1418 |
-
? Number(val)
|
| 1419 |
-
: typeof val === "number" || typeof val === "string"
|
| 1420 |
-
? val
|
| 1421 |
-
: 0;
|
| 1422 |
-
}
|
| 1423 |
-
});
|
| 1424 |
-
|
| 1425 |
-
return episodeData as EpisodeMetadataV3;
|
| 1426 |
-
} else {
|
| 1427 |
-
// Fallback to numeric keys for compatibility
|
| 1428 |
-
const toNum = (v: unknown, fallback = 0): number =>
|
| 1429 |
-
typeof v === "number"
|
| 1430 |
-
? v
|
| 1431 |
-
: typeof v === "bigint"
|
| 1432 |
-
? Number(v)
|
| 1433 |
-
: fallback;
|
| 1434 |
-
return {
|
| 1435 |
-
episode_index: toNum(row["0"]),
|
| 1436 |
-
data_chunk_index: toNum(row["1"]),
|
| 1437 |
-
data_file_index: toNum(row["2"]),
|
| 1438 |
-
dataset_from_index: toNum(row["3"]),
|
| 1439 |
-
dataset_to_index: toNum(row["4"]),
|
| 1440 |
-
video_chunk_index: toNum(row["5"]),
|
| 1441 |
-
video_file_index: toNum(row["6"]),
|
| 1442 |
-
video_from_timestamp: toNum(row["7"]),
|
| 1443 |
-
video_to_timestamp: toNum(row["8"], 30),
|
| 1444 |
-
length: toNum(row["9"], 30),
|
| 1445 |
-
};
|
| 1446 |
-
}
|
| 1447 |
-
}
|
| 1448 |
-
|
| 1449 |
-
// Fallback if parsing fails
|
| 1450 |
-
const fallback = {
|
| 1451 |
-
episode_index: 0,
|
| 1452 |
-
data_chunk_index: 0,
|
| 1453 |
-
data_file_index: 0,
|
| 1454 |
-
dataset_from_index: 0,
|
| 1455 |
-
dataset_to_index: 0,
|
| 1456 |
-
video_chunk_index: 0,
|
| 1457 |
-
video_file_index: 0,
|
| 1458 |
-
video_from_timestamp: 0,
|
| 1459 |
-
video_to_timestamp: 30,
|
| 1460 |
-
length: 30,
|
| 1461 |
-
};
|
| 1462 |
-
|
| 1463 |
-
return fallback;
|
| 1464 |
-
}
|
| 1465 |
-
|
| 1466 |
// ─── Stats computation ───────────────────────────────────────────
|
| 1467 |
|
| 1468 |
/**
|
|
@@ -1519,29 +1374,11 @@ export async function loadAllEpisodeLengthsV3(
|
|
| 1519 |
fps: number,
|
| 1520 |
): Promise<EpisodeLengthStats | null> {
|
| 1521 |
try {
|
| 1522 |
-
const
|
| 1523 |
-
|
| 1524 |
-
const
|
| 1525 |
-
|
| 1526 |
-
|
| 1527 |
-
const path = `meta/episodes/chunk-${chunkIndex.toString().padStart(3, "0")}/file-${fileIndex.toString().padStart(3, "0")}.parquet`;
|
| 1528 |
-
const url = buildVersionedUrl(repoId, version, path);
|
| 1529 |
-
try {
|
| 1530 |
-
const buf = await fetchParquetFile(url);
|
| 1531 |
-
const rows = await readParquetAsObjects(buf, []);
|
| 1532 |
-
if (rows.length === 0 && fileIndex > 0) break;
|
| 1533 |
-
for (const row of rows) {
|
| 1534 |
-
const parsed = parseEpisodeRowSimple(row);
|
| 1535 |
-
allEpisodes.push({
|
| 1536 |
-
index: parsed.episode_index,
|
| 1537 |
-
length: parsed.length,
|
| 1538 |
-
});
|
| 1539 |
-
}
|
| 1540 |
-
fileIndex++;
|
| 1541 |
-
} catch {
|
| 1542 |
-
break;
|
| 1543 |
-
}
|
| 1544 |
-
}
|
| 1545 |
|
| 1546 |
if (allEpisodes.length === 0) return null;
|
| 1547 |
|
|
@@ -1663,35 +1500,18 @@ export async function loadAllEpisodeFrameInfo(
|
|
| 1663 |
);
|
| 1664 |
|
| 1665 |
if (version === "v3.0") {
|
| 1666 |
-
for
|
| 1667 |
-
|
| 1668 |
-
|
| 1669 |
-
|
| 1670 |
-
|
| 1671 |
-
|
| 1672 |
-
|
| 1673 |
-
|
| 1674 |
-
|
| 1675 |
-
|
| 1676 |
-
)
|
| 1677 |
-
|
| 1678 |
-
row[`videos/${cam}/from_timestamp`] ??
|
| 1679 |
-
row["video_from_timestamp"] ??
|
| 1680 |
-
0,
|
| 1681 |
-
);
|
| 1682 |
-
const toTs = Number(
|
| 1683 |
-
row[`videos/${cam}/to_timestamp`] ??
|
| 1684 |
-
row["video_to_timestamp"] ??
|
| 1685 |
-
30,
|
| 1686 |
-
);
|
| 1687 |
-
const videoPath = `videos/${cam}/chunk-${cIdx.toString().padStart(3, "0")}/file-${fIdx.toString().padStart(3, "0")}.mp4`;
|
| 1688 |
-
framesByCamera[cam].push({
|
| 1689 |
-
episodeIndex: epIdx,
|
| 1690 |
-
videoUrl: buildVersionedUrl(repoId, version, videoPath),
|
| 1691 |
-
firstFrameTime: fromTs,
|
| 1692 |
-
lastFrameTime: Math.max(0, toTs - 0.05),
|
| 1693 |
-
});
|
| 1694 |
-
}
|
| 1695 |
}
|
| 1696 |
}
|
| 1697 |
return { cameras, framesByCamera };
|
|
@@ -1825,29 +1645,28 @@ export async function loadCrossEpisodeActionVariance(
|
|
| 1825 |
// Collect episode metadata
|
| 1826 |
type EpMeta = {
|
| 1827 |
index: number;
|
| 1828 |
-
|
| 1829 |
-
|
| 1830 |
from: number;
|
| 1831 |
to: number;
|
| 1832 |
};
|
| 1833 |
const allEps: EpMeta[] = [];
|
| 1834 |
|
| 1835 |
if (version === "v3.0") {
|
| 1836 |
-
for
|
| 1837 |
-
|
| 1838 |
-
|
| 1839 |
-
|
| 1840 |
-
|
| 1841 |
-
|
| 1842 |
-
|
| 1843 |
-
|
| 1844 |
-
|
| 1845 |
-
|
| 1846 |
-
}
|
| 1847 |
}
|
| 1848 |
} else {
|
| 1849 |
for (let i = 0; i < info.total_episodes; i++) {
|
| 1850 |
-
allEps.push({ index: i,
|
| 1851 |
}
|
| 1852 |
}
|
| 1853 |
|
|
@@ -1880,7 +1699,7 @@ export async function loadCrossEpisodeActionVariance(
|
|
| 1880 |
if (version === "v3.0") {
|
| 1881 |
const byFile = new Map<string, EpMeta[]>();
|
| 1882 |
for (const ep of sampled) {
|
| 1883 |
-
const key =
|
| 1884 |
if (!byFile.has(key)) byFile.set(key, []);
|
| 1885 |
byFile.get(key)!.push(ep);
|
| 1886 |
}
|
|
@@ -1888,13 +1707,10 @@ export async function loadCrossEpisodeActionVariance(
|
|
| 1888 |
const fileResults = await Promise.all(
|
| 1889 |
[...byFile.values()].map(async (eps) => {
|
| 1890 |
const ep0 = eps[0];
|
| 1891 |
-
const dataPath = `data/chunk-${ep0.chunkIdx.toString().padStart(3, "0")}/file-${ep0.fileIdx.toString().padStart(3, "0")}.parquet`;
|
| 1892 |
const fileEpActions: { index: number; actions: number[][] }[] = [];
|
| 1893 |
const fileEpStates: (number[][] | null)[] = [];
|
| 1894 |
try {
|
| 1895 |
-
const buf = await fetchParquetFile(
|
| 1896 |
-
buildVersionedUrl(repoId, version, dataPath),
|
| 1897 |
-
);
|
| 1898 |
const rows = await readParquetAsObjects(
|
| 1899 |
buf,
|
| 1900 |
stateKey ? ["index", actionKey, stateKey] : ["index", actionKey],
|
|
|
|
| 15 |
processChartDataGroups,
|
| 16 |
groupRowBySuffix,
|
| 17 |
} from "@/utils/dataProcessing";
|
|
|
|
| 18 |
import { bigIntToNumber } from "@/utils/typeGuards";
|
| 19 |
import {
|
| 20 |
isGrayscaleShape,
|
|
|
|
| 63 |
* v3.0 concatenates episodes into shared files, so each camera carries a time range; v2.x gives every
|
| 64 |
* episode its own file and the range simply spans it.
|
| 65 |
*/
|
| 66 |
+
/** Every episode, normalized. v3 only: v2 needs no index read to locate an episode. */
|
| 67 |
+
async function loadLeRobotEpisodes(
|
| 68 |
+
repoId: string,
|
| 69 |
+
totalEpisodes: number,
|
| 70 |
+
): Promise<LeRobotEpisode[]> {
|
| 71 |
+
return leRobotDataset(repoId).episodes({ offset: 0, limit: totalEpisodes });
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
function toVideosInfo(
|
| 75 |
episode: LeRobotEpisode,
|
| 76 |
info: DatasetMetadata,
|
|
|
|
| 1318 |
return { chartDataGroups, flatChartData: chartData, ignoredColumns };
|
| 1319 |
}
|
| 1320 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1321 |
// ─── Stats computation ───────────────────────────────────────────
|
| 1322 |
|
| 1323 |
/**
|
|
|
|
| 1374 |
fps: number,
|
| 1375 |
): Promise<EpisodeLengthStats | null> {
|
| 1376 |
try {
|
| 1377 |
+
const dataset = leRobotDataset(repoId);
|
| 1378 |
+
const { totalEpisodes } = await dataset.info();
|
| 1379 |
+
const allEpisodes = (
|
| 1380 |
+
await dataset.episodes({ offset: 0, limit: totalEpisodes })
|
| 1381 |
+
).map((episode) => ({ index: episode.index, length: episode.length }));
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1382 |
|
| 1383 |
if (allEpisodes.length === 0) return null;
|
| 1384 |
|
|
|
|
| 1500 |
);
|
| 1501 |
|
| 1502 |
if (version === "v3.0") {
|
| 1503 |
+
for (const episode of await loadLeRobotEpisodes(
|
| 1504 |
+
repoId,
|
| 1505 |
+
info.total_episodes,
|
| 1506 |
+
)) {
|
| 1507 |
+
if (sampledEpisodeSet && !sampledEpisodeSet.has(episode.index)) continue;
|
| 1508 |
+
for (const video of episode.videos) {
|
| 1509 |
+
framesByCamera[video.cameraKey]?.push({
|
| 1510 |
+
episodeIndex: episode.index,
|
| 1511 |
+
videoUrl: video.url,
|
| 1512 |
+
firstFrameTime: video.fromSec,
|
| 1513 |
+
lastFrameTime: Math.max(0, video.toSec - 0.05),
|
| 1514 |
+
});
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1515 |
}
|
| 1516 |
}
|
| 1517 |
return { cameras, framesByCamera };
|
|
|
|
| 1645 |
// Collect episode metadata
|
| 1646 |
type EpMeta = {
|
| 1647 |
index: number;
|
| 1648 |
+
/** v3 only; v2 derives its per-episode path from the index. */
|
| 1649 |
+
dataUrl?: string;
|
| 1650 |
from: number;
|
| 1651 |
to: number;
|
| 1652 |
};
|
| 1653 |
const allEps: EpMeta[] = [];
|
| 1654 |
|
| 1655 |
if (version === "v3.0") {
|
| 1656 |
+
for (const episode of await loadLeRobotEpisodes(
|
| 1657 |
+
repoId,
|
| 1658 |
+
info.total_episodes,
|
| 1659 |
+
)) {
|
| 1660 |
+
allEps.push({
|
| 1661 |
+
index: episode.index,
|
| 1662 |
+
dataUrl: episode.data.url,
|
| 1663 |
+
from: episode.data.fromRow,
|
| 1664 |
+
to: episode.data.toRow,
|
| 1665 |
+
});
|
|
|
|
| 1666 |
}
|
| 1667 |
} else {
|
| 1668 |
for (let i = 0; i < info.total_episodes; i++) {
|
| 1669 |
+
allEps.push({ index: i, from: 0, to: 0 });
|
| 1670 |
}
|
| 1671 |
}
|
| 1672 |
|
|
|
|
| 1699 |
if (version === "v3.0") {
|
| 1700 |
const byFile = new Map<string, EpMeta[]>();
|
| 1701 |
for (const ep of sampled) {
|
| 1702 |
+
const key = ep.dataUrl ?? "";
|
| 1703 |
if (!byFile.has(key)) byFile.set(key, []);
|
| 1704 |
byFile.get(key)!.push(ep);
|
| 1705 |
}
|
|
|
|
| 1707 |
const fileResults = await Promise.all(
|
| 1708 |
[...byFile.values()].map(async (eps) => {
|
| 1709 |
const ep0 = eps[0];
|
|
|
|
| 1710 |
const fileEpActions: { index: number; actions: number[][] }[] = [];
|
| 1711 |
const fileEpStates: (number[][] | null)[] = [];
|
| 1712 |
try {
|
| 1713 |
+
const buf = await fetchParquetFile(ep0.dataUrl ?? "");
|
|
|
|
|
|
|
| 1714 |
const rows = await readParquetAsObjects(
|
| 1715 |
buf,
|
| 1716 |
stateKey ? ["index", actionKey, stateKey] : ["index", actionKey],
|
src/utils/__tests__/stringFormatting.test.ts
CHANGED
|
@@ -5,7 +5,6 @@ import {
|
|
| 5 |
formatEpisodeIndex,
|
| 6 |
formatFileIndex,
|
| 7 |
formatChunkIndex,
|
| 8 |
-
buildV3EpisodesMetadataPath,
|
| 9 |
} from "@/utils/stringFormatting";
|
| 10 |
|
| 11 |
// These utilities are the foundation of v3.0 path construction.
|
|
@@ -75,15 +74,3 @@ describe("formatChunkIndex — 3-digit padding (v3.0 chunk_index)", () => {
|
|
| 75 |
});
|
| 76 |
|
| 77 |
// v3.0 specific path builders
|
| 78 |
-
describe("buildV3EpisodesMetadataPath", () => {
|
| 79 |
-
test("chunk 0, file 0 (default for most datasets)", () => {
|
| 80 |
-
expect(buildV3EpisodesMetadataPath(0, 0)).toBe(
|
| 81 |
-
"meta/episodes/chunk-000/file-000.parquet",
|
| 82 |
-
);
|
| 83 |
-
});
|
| 84 |
-
test("chunk 0, file 2 (multiple metadata files)", () => {
|
| 85 |
-
expect(buildV3EpisodesMetadataPath(0, 2)).toBe(
|
| 86 |
-
"meta/episodes/chunk-000/file-002.parquet",
|
| 87 |
-
);
|
| 88 |
-
});
|
| 89 |
-
});
|
|
|
|
| 5 |
formatEpisodeIndex,
|
| 6 |
formatFileIndex,
|
| 7 |
formatChunkIndex,
|
|
|
|
| 8 |
} from "@/utils/stringFormatting";
|
| 9 |
|
| 10 |
// These utilities are the foundation of v3.0 path construction.
|
|
|
|
| 74 |
});
|
| 75 |
|
| 76 |
// v3.0 specific path builders
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/utils/stringFormatting.ts
CHANGED
|
@@ -55,17 +55,3 @@ export function formatFileIndex(fileIndex: number): string {
|
|
| 55 |
export function formatChunkIndex(chunkIndex: number): string {
|
| 56 |
return padNumber(chunkIndex, PADDING.CHUNK_INDEX);
|
| 57 |
}
|
| 58 |
-
|
| 59 |
-
/**
|
| 60 |
-
* Build episodes metadata path for v3 datasets
|
| 61 |
-
*
|
| 62 |
-
* @param chunkIndex - Episode chunk index
|
| 63 |
-
* @param fileIndex - File index within chunk
|
| 64 |
-
* @returns Formatted episodes metadata path (e.g., "meta/episodes/chunk-001/file-000.parquet")
|
| 65 |
-
*/
|
| 66 |
-
export function buildV3EpisodesMetadataPath(
|
| 67 |
-
chunkIndex: number,
|
| 68 |
-
fileIndex: number,
|
| 69 |
-
): string {
|
| 70 |
-
return `meta/episodes/chunk-${formatChunkIndex(chunkIndex)}/file-${formatFileIndex(fileIndex)}.parquet`;
|
| 71 |
-
}
|
|
|
|
| 55 |
export function formatChunkIndex(chunkIndex: number): string {
|
| 56 |
return padNumber(chunkIndex, PADDING.CHUNK_INDEX);
|
| 57 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|