visualize_dataset / src /lib /__tests__ /hand-tracking.test.ts
mishig's picture
mishig HF Staff
Sync from huggingface/lerobot-dataset-visualizer via hub-sync
254aedd verified
Raw History Blame Contribute Delete
11.6 kB
import { describe, expect, test } from "bun:test";
import {
fluToScene,
cameraPathRuns,
handFrameIndex,
handProjectionMetadata,
hasHandTracking,
parseHandCalibration,
projectHandPoint,
readHandTracking,
wristTrail,
} from "../hand-tracking";
import type { DatasetMetadata } from "@/utils/parquetUtils";
function info(): DatasetMetadata {
const widths = {
action: 156,
"annotation.hand_joints_camera": 126,
"annotation.hand_joints_world": 126,
"annotation.hand_action_valid": 1,
"annotation.hand_observed": 2,
"annotation.hand_reference_frame": 1,
};
return {
codebase_version: "v3.0",
robot_type: "human_egocentric",
fps: 10,
total_episodes: 1,
total_frames: 6,
total_tasks: 1,
total_videos: 1,
total_chunks: 1,
chunks_size: 1000,
splits: {},
data_path: "",
video_path: "",
features: {
...Object.fromEntries(
Object.entries(widths).map(([key, width]) => [
key,
{ dtype: "float32", shape: [width], names: null },
]),
),
"observation.images.ego": {
dtype: "video",
shape: [480, 640, 3],
names: null,
},
},
};
}
const calibration = {
model: "pinhole",
width: 640,
height: 480,
fx: 600,
fy: 610,
cx: 320,
cy: 240,
distortion: [],
};
const metadata = {
cameraKey: "observation.images.ego",
calibration: parseHandCalibration(calibration),
overlayIssue: null,
};
function numericFrames(): {
length: number;
timestamps: number[];
series: Record<string, number[][]>;
} {
const vector = () =>
Array.from({ length: 126 }, (_, component) =>
Array.from({ length: 6 }, () =>
component % 3 === 2 ? 1 : component / 100,
),
);
return {
length: 6,
timestamps: [20, 20.1, 20.2, 20.3, 20.4, 20.5],
series: {
"annotation.hand_joints_camera": vector(),
"annotation.hand_joints_world": vector(),
"annotation.hand_action_valid": [[1, 1, 0, 1, 1, 1]],
"annotation.hand_observed": [
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1],
],
"annotation.hand_reference_frame": [[0, 0, -1, 3, 3, 5]],
},
};
}
describe("native hand schema and time alignment", () => {
test("an arbitrary 156D action is not detected as hand tracking", () => {
const metadata = info();
expect(hasHandTracking(metadata)).toBe(true);
delete metadata.features["annotation.hand_joints_camera"];
expect(hasHandTracking(metadata)).toBe(false);
});
test("all source frames survive, with local timestamps and independently masked hands", () => {
const frames = numericFrames();
frames.series["annotation.hand_observed"][1][1] = 0;
const data = readHandTracking(info(), frames, metadata)!;
expect(data.frames).toHaveLength(6);
expect(data.frames[1].time).toBeCloseTo(0.1);
expect(data.frames[0].camera[0]).toHaveLength(21);
expect(data.frames[1].camera[1]).toBeNull();
expect(data.frames[2].camera).toEqual([null, null]);
expect(data.frames[2].segment).toEqual([null, null]);
});
test("a non-finite point hides the affected hand in both views", () => {
const frames = numericFrames();
frames.series["annotation.hand_joints_camera"][0][0] = NaN;
const data = readHandTracking(info(), frames, metadata)!;
expect(data.frames[0].camera[0]).toBeNull();
expect(data.frames[0].segment[0]).toBeNull();
expect(data.frames[0].segment[1]).not.toBeNull();
});
test("selects the displayed frame without interpolating or looking ahead", () => {
const frames = readHandTracking(info(), numericFrames(), metadata)!.frames;
expect(handFrameIndex(frames, 0.099)).toBe(0);
expect(handFrameIndex(frames, 0.1)).toBe(1);
expect(handFrameIndex(frames, 0.2)).toBe(2);
// A frame at 0.2 s lives at media time 12.7 s in a shared video starting at 12.5 s.
expect(handFrameIndex(frames, 12.7 - 12.5)).toBe(2);
expect(handFrameIndex(frames, -1)).toBe(-1);
});
test("trails stop at gaps and segment-reference resets", () => {
const frames = readHandTracking(info(), numericFrames(), metadata)!.frames;
expect(wristTrail(frames, 1, 0)).toHaveLength(2);
expect(wristTrail(frames, 2, 0)).toHaveLength(0);
expect(wristTrail(frames, 4, 0)).toHaveLength(2);
expect(wristTrail(frames, 5, 0)).toHaveLength(1);
expect(fluToScene([1, 2, 3])).toEqual([-2, 3, -1]);
});
});
describe("recording-wide camera and wrist coordinates", () => {
const rigid = (x = 0) => [1, 0, 0, x, 0, 1, 0, 20, 0, 0, 1, 30, 0, 0, 0, 1];
const withPoses = (poses: number[][]) => {
const frames = numericFrames();
frames.series["annotation.camera_world_from_optical"] = Array.from(
{ length: 16 },
(_, a) => poses.map((pose) => pose[a]),
);
return readHandTracking(info(), frames, metadata)!;
};
test("removes the first camera's translation and rotation from the recording frame", () => {
const pose = [0, -1, 0, 10, 1, 0, 0, 20, 0, 0, 1, 30, 0, 0, 0, 1];
const moved = [...pose];
moved[7] += 1;
const tracking = withPoses([pose, moved, pose, pose, pose, pose]);
expect(tracking.coordinateFrame).toBe("recording");
expect(tracking.frames[0].cameraPose?.position).toEqual([0, -0, -0]);
expect(tracking.frames[1].cameraPose?.position).toEqual([0, -1, -0]);
expect(tracking.frames[0].recording[0]?.[0][0]).toBeCloseTo(1);
expect(tracking.frames[1].recording[0]?.[0][1]).toBeCloseTo(-1);
expect(tracking.frames[1].recording[0]?.[0][2]).toBeCloseTo(-0.01);
});
test("rotates optical joints with each camera pose instead of displaying span-local joints as global", () => {
const rotated = [0, -1, 0, 1, 1, 0, 0, 20, 0, 0, 1, 30, 0, 0, 0, 1];
const tracking = withPoses([
rigid(),
rotated,
rigid(),
rigid(),
rigid(),
rigid(),
]);
expect(tracking.frames[1].recording[0]?.[0][1]).toBeCloseTo(-0.99);
expect(tracking.frames[1].recording[0]?.[0][2]).toBeCloseTo(0);
expect(tracking.frames[1].cameraPose?.axes[0]).toEqual([0, 0, -1]);
// Reference frame changes at frame 5, but the saved global trajectory remains continuous.
expect(wristTrail(tracking.frames, 5, 0, 2, "recording")).toHaveLength(3);
expect(wristTrail(tracking.frames, 5, 0)).toHaveLength(1);
});
test("zero, reflected and non-rigid camera placeholders split the trajectory and hide global hands", () => {
const reflection = rigid();
reflection[0] = -1;
const scaled = rigid();
scaled[0] = 2;
const tracking = withPoses([
rigid(),
rigid(1),
Array(16).fill(0),
reflection,
scaled,
rigid(2),
]);
expect(cameraPathRuns(tracking.frames, 5).map((run) => run.length)).toEqual(
[2, 1],
);
for (const i of [2, 3, 4]) {
expect(tracking.frames[i].cameraPose).toBeNull();
expect(tracking.frames[i].recording).toEqual([null, null]);
}
expect(tracking.frames[5].recording[0]).not.toBeNull();
});
test("older native exports without saved camera transforms retain span-local 3D", () => {
const tracking = readHandTracking(info(), numericFrames(), metadata)!;
expect(tracking.coordinateFrame).toBe("segment");
expect(tracking.frames[0].segment[0]).not.toBeNull();
expect(cameraPathRuns(tracking.frames, 5)).toEqual([]);
});
});
describe("episode-specific source camera binding", () => {
test("an incomplete episode mapping cannot select an unindexed report entry", () => {
const selected = handProjectionMetadata(info(), 0, {
source_episode_map: [{ episode_index: 1, source_episode_index: 7 }],
episodes: [{ hand_actions: { camera_intrinsics: calibration } }],
});
expect(selected.calibration).toBeNull();
expect(selected.overlayIssue).toContain("No camera calibration");
});
test("uses the source episode map, not completion order or output index", () => {
const report = {
manifest: { camera: "observation.images.ego" },
source_episode_map: [{ episode_index: 0, source_episode_index: 7 }],
episodes: [
{
episode_index: 0,
hand_actions: { camera_intrinsics: { ...calibration, fx: 100 } },
},
{
episode_index: 7,
hand_actions: {
camera_intrinsics: { ...calibration, fx: 700, measured: false },
},
},
],
};
const selected = handProjectionMetadata(info(), 0, report);
expect(selected.calibration?.fx).toBe(700);
expect(selected.calibration?.estimated).toBe(true);
});
test("never projects source-camera coordinates onto another camera", () => {
expect(
handProjectionMetadata(info(), 0, {
manifest: { camera: "missing.camera" },
}).cameraKey,
).toBeNull();
const data = info();
data.features["observation.images.other"] =
data.features["observation.images.ego"];
expect(handProjectionMetadata(data, 0, null).cameraKey).toBeNull();
});
test("missing calibration keeps 3D available and explains why overlays are unavailable", () => {
const selected = handProjectionMetadata(info(), 0, null);
expect(selected.calibration).toBeNull();
expect(selected.overlayIssue).toContain("No camera calibration");
expect(
readHandTracking(info(), numericFrames(), selected)?.frames[0].segment[0],
).not.toBeNull();
});
test("accepts measured calibration JSON stored on the video feature", () => {
const data = info();
data.features["observation.images.ego"].info = {
"camera.intrinsics": JSON.stringify(calibration),
};
expect(handProjectionMetadata(data, 0, null).calibration?.fx).toBe(600);
expect(parseHandCalibration({ ...calibration, fx: Infinity })).toBeNull();
});
test("rejects malformed calibration at the unknown-data boundary", () => {
for (const value of [
null,
[],
"invalid JSON",
{ ...calibration, width: 640.5 },
{ ...calibration, fx: "600" },
{ ...calibration, distortion: [0, 0, 0, "0"] },
]) {
expect(parseHandCalibration(value)).toBeNull();
}
});
});
describe("source camera projection", () => {
// Reference outputs generated with OpenCV projectPoints, not the implementation under test.
const d = [
0.1, -0.02, 0.003, -0.004, 0.01, 0.02, -0.01, 0.005, 0.001, -0.002, 0.003,
-0.004, 0.02, -0.03,
];
const expected = [
[0, 360, 321.3333333333333],
[4, 360.04582716049384, 321.57562633744857],
[5, 360.04583155006856, 321.5756352629172],
[8, 360.02821795025153, 321.53982094328916],
[12, 360.0409586909922, 321.5792826716842],
[14, 360.2472660915072, 322.00366115201376],
];
for (const [count, x, y] of expected)
test(`pinhole ${count}-coefficient projection matches OpenCV`, () => {
const c = parseHandCalibration({
...calibration,
distortion: d.slice(0, count),
})!;
const p = projectHandPoint([0.1, 0.2, 1.5], c)!;
expect(p[0]).toBeCloseTo(x, 8);
expect(p[1]).toBeCloseTo(y, 8);
});
test("fisheye projection matches OpenCV", () => {
const c = parseHandCalibration({
...calibration,
model: "opencv_fisheye",
distortion: d.slice(0, 4),
})!;
const p = projectHandPoint([0.1, 0.2, 1.5], c)!;
expect(p[0]).toBeCloseTo(359.7941667826154, 8);
expect(p[1]).toBeCloseTo(320.9148057913179, 8);
});
test("points behind the image plane never appear", () => {
expect(projectHandPoint([0, 0, -1], metadata.calibration!)).toBeNull();
expect(projectHandPoint([0, 0, 0], metadata.calibration!)).toBeNull();
});
});