Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Download src/utils/__tests__/crossEpisode.test.ts from lerobot/visualize_dataset: direct link, hf CLI and curl.
- Browser
- Download file 13.1 kB
-
https://huggingface.co/spaces/lerobot/visualize_dataset/resolve/main/src/utils/__tests__/crossEpisode.test.ts
- Command line
-
hf download hf://spaces/lerobot/visualize_dataset/src/utils/__tests__/crossEpisode.test.ts
-
curl -L -o crossEpisode.test.ts https://huggingface.co/spaces/lerobot/visualize_dataset/resolve/main/src/utils/__tests__/crossEpisode.test.ts
13.1 kB
| import { describe, expect, test } from "bun:test"; | |
| import { | |
| LAG_WINDOW_FRAMES, | |
| computeCrossEpisodeStats, | |
| computeCrossEpisodeStatsInSteps, | |
| computeVelocityStats, | |
| planEpisodeSample, | |
| type EpisodeSeries, | |
| } from "@/utils/crossEpisode"; | |
| /** A deterministic stand-in for random noise. */ | |
| function noise(seed: number): () => number { | |
| let state = seed; | |
| return () => { | |
| state = (state * 1664525 + 1013904223) % 4294967296; | |
| return state / 4294967296 - 0.5; | |
| }; | |
| } | |
| /** | |
| * Episodes of `dims` action dimensions. Each state follows its action `delay` frames later, as a | |
| * real arm follows its command. | |
| */ | |
| function makeEpisodes(options: { | |
| count: number; | |
| length: number | ((episode: number) => number); | |
| dims: number; | |
| delay?: number; | |
| signal: (t: number, d: number, episode: number) => number; | |
| }): EpisodeSeries[] { | |
| const { count, dims, delay = 0, signal } = options; | |
| return Array.from({ length: count }, (_, e) => { | |
| const length = | |
| typeof options.length === "number" ? options.length : options.length(e); | |
| const actions = new Float32Array(length * dims); | |
| const states = new Float32Array(length * dims); | |
| for (let t = 0; t < length; t++) { | |
| for (let d = 0; d < dims; d++) { | |
| actions[t * dims + d] = signal(t, d, e); | |
| states[t * dims + d] = signal(Math.max(0, t - delay), d, e); | |
| } | |
| } | |
| return { index: e, length, actions, states }; | |
| }); | |
| } | |
| const names = (dims: number) => Array.from({ length: dims }, (_, d) => `j${d}`); | |
| const input = (episodes: EpisodeSeries[], dims: number) => ({ | |
| episodes, | |
| actionDim: dims, | |
| stateDim: dims, | |
| actionNames: names(dims).map((n) => `action | ${n}`), | |
| stateNames: names(dims), | |
| fps: 30, | |
| }); | |
| /** A smooth trajectory with enough texture for a cross-correlation to lock onto. */ | |
| const wobble = (t: number, d: number, e: number) => | |
| Math.sin(0.05 * t * (d + 1) + e) + 0.3 * Math.sin(0.31 * t + d + 2 * e); | |
| describe("planEpisodeSample", () => { | |
| const dataset = (files: number, perFile: number, rowsPerEpisode: number) => | |
| Array.from({ length: files * perFile }, (_, i) => { | |
| const inFile = i % perFile; | |
| return { | |
| index: i, | |
| file: `file-${Math.floor(i / perFile)}`, | |
| fromRow: inFile * rowsPerEpisode, | |
| toRow: (inFile + 1) * rowsPerEpisode, | |
| }; | |
| }); | |
| test("takes every episode of a dataset within both limits", () => { | |
| const episodes = dataset(3, 24, 200); | |
| expect(planEpisodeSample(episodes, 120, 1e9)).toEqual(episodes); | |
| }); | |
| test("spreads the sample over every file when the budget allows", () => { | |
| const sample = planEpisodeSample(dataset(6, 100, 500), 120, 1e9); | |
| expect(sample.length).toBe(120); | |
| expect(new Set(sample.map((e) => e.file)).size).toBe(6); | |
| // Evenly within a file too: first and last episode of the first file are both in. | |
| expect(sample[0].index).toBe(0); | |
| expect(sample.some((e) => e.index === 99)).toBe(true); | |
| }); | |
| test("reads only as many files as the row budget allows", () => { | |
| // 100 files of 50,000 rows; room for 4 of them. | |
| const sample = planEpisodeSample(dataset(100, 100, 500), 120, 200_000); | |
| const files = [...new Set(sample.map((e) => e.file))]; | |
| expect(files.length).toBe(4); | |
| expect(sample.length).toBe(120); | |
| // Still from across the dataset, not its first four files. | |
| expect(files[0]).toBe("file-0"); | |
| expect(files[files.length - 1]).toBe("file-99"); | |
| }); | |
| test("a file's unused share of the episodes goes to the later files", () => { | |
| const episodes = [ | |
| ...dataset(1, 2, 100), | |
| ...dataset(1, 200, 100).map((e) => ({ ...e, file: "big" })), | |
| ]; | |
| const sample = planEpisodeSample(episodes, 120, 1e9); | |
| expect(sample.filter((e) => e.file === "file-0").length).toBe(2); | |
| expect(sample.length).toBe(120); | |
| }); | |
| test("never returns a single episode when the dataset has more", () => { | |
| // One episode per file, and a budget smaller than any file. | |
| const sample = planEpisodeSample(dataset(50, 1, 10_000), 120, 1); | |
| expect(sample.length).toBe(2); | |
| }); | |
| test("an empty dataset gives an empty sample", () => { | |
| expect(planEpisodeSample([], 120, 1e9)).toEqual([]); | |
| }); | |
| }); | |
| describe("computeCrossEpisodeStats", () => { | |
| test("needs at least two episodes", () => { | |
| const [only] = makeEpisodes({ | |
| count: 1, | |
| length: 300, | |
| dims: 2, | |
| signal: wobble, | |
| }); | |
| expect(computeCrossEpisodeStats(input([only], 2))).toBeNull(); | |
| }); | |
| test("finds the delay between command and state, in frames", () => { | |
| for (const delay of [0, 2, 5]) { | |
| const episodes = makeEpisodes({ | |
| count: 6, | |
| length: 400, | |
| dims: 3, | |
| delay, | |
| signal: wobble, | |
| }); | |
| const stats = computeCrossEpisodeStats(input(episodes, 3))!; | |
| expect(stats.aggAlignment!.meanPeakLag).toBe(delay); | |
| expect(stats.aggAlignment!.meanPeakCorr).toBeGreaterThan(0.95); | |
| expect(stats.aggAlignment!.numPairs).toBe(3); | |
| } | |
| }); | |
| test("keeps lags in frames on episodes longer than the lag window", () => { | |
| // Subsampling these episodes used to stretch every lag by the subsampling factor. | |
| const length = LAG_WINDOW_FRAMES * 3; | |
| const episodes = makeEpisodes({ | |
| count: 3, | |
| length, | |
| dims: 2, | |
| delay: 4, | |
| signal: wobble, | |
| }); | |
| const stats = computeCrossEpisodeStats(input(episodes, 2))!; | |
| expect(stats.aggAlignment!.meanPeakLag).toBe(4); | |
| // sin(0.05 t) falls to half its autocorrelation at about a sixth of its 126-frame period. | |
| const acf = stats.aggAutocorrelation!; | |
| const firstBelowHalf = acf.chartData.find((row) => row.j0 < 0.5)!.lag; | |
| expect(firstBelowHalf).toBeGreaterThan(15); | |
| expect(firstBelowHalf).toBeLessThan(26); | |
| }); | |
| test("one aborted episode does not shorten the autocorrelation horizon", () => { | |
| const episodes = makeEpisodes({ | |
| count: 8, | |
| length: (e) => (e === 3 ? 30 : 400), | |
| dims: 2, | |
| signal: wobble, | |
| }); | |
| const acf = computeCrossEpisodeStats( | |
| input(episodes, 2), | |
| )!.aggAutocorrelation!; | |
| // The shortest episode allows 15 lags; the typical one allows the full 100. | |
| expect(acf.chartData.length).toBe(100); | |
| expect(acf.chartData[0].lag).toBe(1); | |
| expect(acf.chartData[0].time).toBeCloseTo(1 / 30, 10); | |
| }); | |
| test("motors at rest are reported inactive and kept out of the chunk suggestion", () => { | |
| const episodes = makeEpisodes({ | |
| count: 6, | |
| length: 400, | |
| dims: 3, | |
| // j0 moves; j1 and j2 hold one value in every episode. | |
| signal: (t, d, e) => (d === 0 ? wobble(t, 0, e) : 0.25 * d), | |
| }); | |
| const stats = computeCrossEpisodeStats(input(episodes, 3))!; | |
| expect(stats.aggVelocity.map((v) => v.inactive)).toEqual([ | |
| false, | |
| true, | |
| true, | |
| ]); | |
| expect(stats.aggAutocorrelation!.shortKeys).toEqual(["j0"]); | |
| // Counted in, each resting motor "decorrelates" at lag 1; two of them against one moving | |
| // motor made the median, and so the suggested chunk length, 1. | |
| expect(stats.aggAutocorrelation!.suggestedChunk).toBeGreaterThan(10); | |
| // And they have no command–state correlation to average in. | |
| expect(stats.aggAlignment!.numPairs).toBe(1); | |
| }); | |
| test("a two-valued motor is reported discrete", () => { | |
| const episodes = makeEpisodes({ | |
| count: 4, | |
| length: 300, | |
| dims: 2, | |
| signal: (t, d, e) => | |
| d === 0 ? wobble(t, 0, e) : Math.floor(t / 60) % 2 ? 1 : 0, | |
| }); | |
| const stats = computeCrossEpisodeStats(input(episodes, 2))!; | |
| expect(stats.aggVelocity[0].discrete).toBe(false); | |
| expect(stats.aggVelocity[1].discrete).toBe(true); | |
| }); | |
| test("ranks episodes by movement and by jerkiness", () => { | |
| const rand = noise(11); | |
| const episodes = makeEpisodes({ | |
| count: 5, | |
| length: 300, | |
| dims: 2, | |
| signal: (t, d, e) => { | |
| if (e === 1) return 0.001 * Math.sin(0.05 * t); // barely moves | |
| if (e === 4) return wobble(t, d, e) + 0.4 * rand(); // shaky | |
| return wobble(t, d, e); | |
| }, | |
| }); | |
| const stats = computeCrossEpisodeStats(input(episodes, 2))!; | |
| expect(stats.lowMovementEpisodes[0].episodeIndex).toBe(1); | |
| expect(stats.jerkyEpisodes[0].episodeIndex).toBe(4); | |
| expect(stats.speedDistribution.length).toBe(5); | |
| expect(stats.numEpisodes).toBe(5); | |
| }); | |
| test("variance is zero where every episode agrees and positive where they differ", () => { | |
| const episodes = makeEpisodes({ | |
| count: 4, | |
| length: 200, | |
| dims: 2, | |
| // j0 is the same in every episode; j1 depends on the episode. | |
| signal: (t, d, e) => (d === 0 ? Math.sin(0.05 * t) : e * 0.5), | |
| }); | |
| const { variance, timeBins } = computeCrossEpisodeStats( | |
| input(episodes, 2), | |
| )!; | |
| expect(variance.length).toBe(timeBins.length); | |
| for (const row of variance) { | |
| expect(row[0]).toBeCloseTo(0, 6); | |
| // Values 0, 0.5, 1, 1.5: population variance 0.3125. | |
| expect(row[1]).toBeCloseTo(0.3125, 6); | |
| } | |
| }); | |
| test("episodes without state still give everything but the alignment", () => { | |
| const episodes = makeEpisodes({ | |
| count: 3, | |
| length: 300, | |
| dims: 2, | |
| signal: wobble, | |
| }).map((episode) => ({ ...episode, states: null })); | |
| const stats = computeCrossEpisodeStats(input(episodes, 2))!; | |
| expect(stats.aggAlignment).toBeNull(); | |
| expect(stats.aggAutocorrelation).not.toBeNull(); | |
| expect(stats.aggVelocity.length).toBe(2); | |
| }); | |
| test("the stepped version gives the same statistics", async () => { | |
| const episodes = makeEpisodes({ | |
| count: 5, | |
| length: 350, | |
| dims: 3, | |
| delay: 3, | |
| signal: wobble, | |
| }); | |
| let pauses = 0; | |
| const stepped = await computeCrossEpisodeStatsInSteps( | |
| input(episodes, 3), | |
| async () => { | |
| pauses++; | |
| }, | |
| ); | |
| expect(stepped).toEqual(computeCrossEpisodeStats(input(episodes, 3))); | |
| expect(pauses).toBe(2); | |
| }); | |
| }); | |
| describe("velocity histograms", () => { | |
| /** One episode whose single action follows `value(t)`. */ | |
| const single = (length: number, value: (t: number) => number) => { | |
| const actions = new Float32Array(length); | |
| for (let t = 0; t < length; t++) actions[t] = value(t); | |
| return [{ index: 0, length, actions, states: null }]; | |
| }; | |
| const middle = (bins: number[]) => (bins.length - 1) / 2; | |
| test("are symmetric about zero, with no change in the middle bin", () => { | |
| // A steady ramp up, then the same ramp down: as many +1 steps as -1 steps. | |
| const [stat] = computeVelocityStats( | |
| single(401, (t) => (t <= 200 ? t : 400 - t)), | |
| 1, | |
| ["action | j0"], | |
| ); | |
| expect(stat.bins.length % 2).toBe(1); | |
| expect(stat.lo).toBeCloseTo(-stat.hi, 12); | |
| expect(stat.bins.reduce((a, b) => a + b, 0)).toBe(400); | |
| const m = middle(stat.bins); | |
| const below = stat.bins.slice(0, m).reduce((a, b) => a + b, 0); | |
| const above = stat.bins.slice(m + 1).reduce((a, b) => a + b, 0); | |
| expect(below).toBe(200); | |
| expect(above).toBe(200); | |
| }); | |
| test("one spike does not flatten the rest into a single bin", () => { | |
| const rand = noise(5); | |
| // Small jitter everywhere, and one jump two hundred times its size. | |
| let level = 0; | |
| const values = Array.from({ length: 2001 }, (_, t) => { | |
| level += rand() * 0.02 + (t === 1000 ? 4 : 0); | |
| return level; | |
| }); | |
| const [stat] = computeVelocityStats( | |
| single(values.length, (t) => values[t]), | |
| 1, | |
| ["action | j0"], | |
| ); | |
| // The reach covers the jitter, not the jump... | |
| expect(stat.hi).toBeLessThan(stat.maxAbs / 20); | |
| // ...so the jitter spreads over many bins instead of piling into the middle one. | |
| const occupied = stat.bins.filter((count) => count > 0).length; | |
| expect(occupied).toBeGreaterThan(15); | |
| // The jump is still counted: in the last bin, with everything else. | |
| expect(stat.bins.reduce((a, b) => a + b, 0)).toBe(2000); | |
| expect(stat.bins[stat.bins.length - 1]).toBeGreaterThan(0); | |
| }); | |
| test("a motor that never changes puts everything in the middle bin", () => { | |
| const [stat] = computeVelocityStats( | |
| single(100, () => 0.5), | |
| 1, | |
| ["action | j0"], | |
| ); | |
| expect(stat.inactive).toBe(true); | |
| expect(stat.std).toBe(0); | |
| expect(stat.bins[middle(stat.bins)]).toBe(99); | |
| expect(stat.bins.reduce((a, b) => a + b, 0)).toBe(99); | |
| }); | |
| test("a single episode is enough", () => { | |
| const stats = computeVelocityStats( | |
| single(300, (t) => Math.sin(0.05 * t)), | |
| 1, | |
| ["action | elbow"], | |
| ); | |
| expect(stats.length).toBe(1); | |
| expect(stats[0].name).toBe("elbow"); | |
| expect(stats[0].std).toBeGreaterThan(0); | |
| expect(computeVelocityStats([], 1, ["action | elbow"])).toEqual([]); | |
| }); | |
| test("the dataset-wide statistics use the same histograms", () => { | |
| const episodes = makeEpisodes({ | |
| count: 3, | |
| length: 300, | |
| dims: 2, | |
| signal: wobble, | |
| }); | |
| const names = ["action | j0", "action | j1"]; | |
| expect(computeCrossEpisodeStats(input(episodes, 2))!.aggVelocity).toEqual( | |
| computeVelocityStats(episodes, 2, names), | |
| ); | |
| }); | |
| }); | |