Add standalone DROID-W SLAM with camera-frame poses and manual lens intrinsics

README.md

@@ -41,6 +41,29 @@ tags:
 
 ![LeRobot Annotation Studio — robot video to language](assets/studio-cover.png)
 
+## Standalone SLAM for wrist and handheld cameras
+
+Choose **SLAM camera poses** to run the frozen DROID-W + MoGe Small camera
+pipeline without hand reconstruction or a hosted language model. It runs on an
+HF Jobs GPU and writes a private LeRobot v3 copy with every original action,
+joint state, timestamp and camera retained. The added
+`annotation.camera_initial_from_optical` transform expresses each camera pose in
+the first optical camera frame of that episode: X right, Y down, Z forward.
+`annotation.camera_pose_sampled` distinguishes tracked frames from interpolation.
+The default tracks every source frame; 10 Hz sampling is an explicit preview.
+
+Camera calibration can come from dataset metadata, manually entered pinhole or
+OpenCV fisheye parameters, or experimental MoGe focal estimation. Manual inputs
+must match the selected video's resolution and are recorded in the run report.
+MoGe focal estimation assumes a centered pinhole camera without distortion.
+MoGe's learned metric scale is never fitted to evaluation trajectories.
+
+The 200 FPS target refers to actual tracked frames divided by camera-worker
+wall time. Decode, intrinsics estimation, interpolation, setup and export are
+reported separately in `annotation_studio/slam.json`; cold setup and input
+copying count toward billed Job time. Throughput and accuracy are experimental;
+the current benchmark receipt is exposed at `/api/slam-pipeline`.
+
 **Hand actions** uses EgoHand with native SOMA/MHR hands and
 **DROID-W + MoGe Small** inside the existing annotation wizard. Choose a human-POV
 dataset camera and an HF Jobs GPU. Measured calibration comes from dataset

flora-worker-image.json

@@ -25,7 +25,7 @@
   },
   "expected_code": {
     "worker.py": "c4f53f78e246e2e42cc8348d5ed23954badc1c027bceb579ff24be0db89a9110",
-    "subset.py": "f82fa98d325780b17d904a2a4f1cd8d6f37cd95f3e8b87f3d26185eb34a73632",
+    "subset.py": "c32c62ca23155c602a4c92e67a76613bc2c4dec4ddf9dd4dc554600bb99034d3",
     "flora_captioner.py": "56a2ce4e5beda1c57948a8293bd53f5871f196ff0cbf91fa18e3858d51eea998",
     "flora_acceleration.py": "782a49a896bfe90d5ca4853ee963287dd861a87f09e28bc52efaecca3cdaeaa6",
     "flora_batcher.py": "38e03e2032af8d5f3ed34980cc4d89927fe70c9742aa2ce13f4d0ccdd6fba727",

frontend/src/SlamSettings.tsx

@@ -0,0 +1,31 @@
+export type SlamOptions = {
+  compute: string;
+  calibration_mode: "dataset" | "estimate-moge" | "manual";
+  sample_mode: "all-frames" | "10hz";
+  intrinsics: {model: string; width: number; height: number; fx: number; fy: number; cx: number; cy: number; distortion: number[]};
+};
+export const defaultSlam: SlamOptions = {
+  compute: "rtx-pro-6000", calibration_mode: "dataset", sample_mode: "all-frames",
+  intrinsics: {model: "pinhole", width: 480, height: 360, fx: 0, fy: 0, cx: 240, cy: 180, distortion: []},
+};
+export default function SlamSettings({value, onChange}: {value: SlamOptions; onChange: (value: SlamOptions) => void}) {
+  return <section className="card step-card" aria-label="SLAM settings">
+    <h3>DROID-W + MoGe camera tracking</h3>
+    <p className="field-help">Camera trajectory in the first camera frame: X right, Y down, Z forward. Original actions, joint states and frames are retained.</p>
+    <label className="field-label">HF Jobs GPU<select value={value.compute} onChange={e=>onChange({...value,compute:e.target.value})}>
+      <option value="rtx-pro-6000">RTX PRO 6000 · 96 GB</option><option value="h200">H200 · 141 GB</option><option value="a100-large">A100 · 80 GB</option>
+    </select></label>
+    <label className="field-label">Camera intrinsics<select value={value.calibration_mode} onChange={e=>onChange({...value,calibration_mode:e.target.value as SlamOptions["calibration_mode"]})}>
+      <option value="dataset">Use dataset calibration</option><option value="manual">Enter measured intrinsics</option><option value="estimate-moge">Estimate with MoGe · experimental</option>
+    </select></label>
+    {value.calibration_mode === "manual" && <div className="viz-grid">
+      {(["width","height","fx","fy","cx","cy"] as const).map(key=><label className="field-label" key={key}>{key} (pixels)<input type="number" step="any" value={value.intrinsics[key]} onChange={e=>onChange({...value,intrinsics:{...value.intrinsics,[key]:Number(e.target.value)}})}/></label>)}
+      <label className="field-label">Lens model<select value={value.intrinsics.model} onChange={e=>onChange({...value,intrinsics:{...value.intrinsics,model:e.target.value,distortion:[]}})}><option value="pinhole">Pinhole / OpenCV distortion</option><option value="opencv_fisheye">OpenCV fisheye</option></select></label>
+      <label className="field-label">Distortion coefficients (comma separated)<input value={value.intrinsics.distortion.join(",")} onChange={e=>onChange({...value,intrinsics:{...value.intrinsics,distortion:e.target.value ? e.target.value.split(",").map(Number) : []}})}/></label>
+    </div>}
+    <label className="field-label">Frames to track<select value={value.sample_mode} onChange={e=>onChange({...value,sample_mode:e.target.value as SlamOptions["sample_mode"]})}>
+      <option value="all-frames">Every source frame</option><option value="10hz">10 Hz preview · interpolate remaining frames</option>
+    </select></label>
+    <p className="field-help">MoGe supplies a learned metric scale. Estimated intrinsics assume a centered pinhole lens with no distortion. The 200 FPS target is measured on tracked frames; interpolation is reported separately.</p>
+  </section>;
+}

frontend/src/main.tsx

@@ -38,6 +38,7 @@ import {
 import "./style.css";
 import "./wizard.css";
 import AnnotationExamples from "./AnnotationExamples";
+import SlamSettings, { defaultSlam, type SlamOptions } from "./SlamSettings";
 import JobsView, { type Job, type Progress } from "./JobsView";
 import HandActionSettings, {
   HandPipelineExplanation,
@@ -54,8 +55,10 @@ type Feature =
   | "Interjections & speech"
   | "Visual questions"
   | "Human demo videos"
-  | "Hand actions";
+  | "Hand actions"
+  | "SLAM camera poses";
 const HANDS: Feature = "Hand actions";
+const SLAM: Feature = "SLAM camera poses";
 const HUMAN_VIDEOS: Feature = "Human demo videos";
 type Options = {
   model: string;
@@ -72,6 +75,7 @@ type Options = {
   human_video_segments: number;
   human_video_parallel: number;
   hand_actions: HandOptions;
+  slam: SlamOptions;
 };
 type PromptTemplate = {
   key: string;
@@ -186,6 +190,7 @@ const defaults: Options = {
   human_video_segments: 0,
   human_video_parallel: 8,
   hand_actions: defaultHands,
+  slam: defaultSlam,
 };
 const definitions: {
   name: Feature;
@@ -259,6 +264,7 @@ function initial() {
     return {
       ...defaults,
       ...saved,
+      slam: {...defaultSlam, ...saved.slam},
       hand_actions: {
         ...defaultHands,
         compute: saved.hand_actions?.compute ?? defaultHands.compute,
@@ -357,7 +363,8 @@ function App() {
   );
   const usesFlora = options.captioner === "flora";
   const usesHands = options.features.includes(HANDS);
-  const handsOnly = usesHands && options.features.length === 1;
+  const usesSlam = options.features.includes(SLAM);
+  const handsOnly = (usesHands || usesSlam) && options.features.length === 1;
   const floraNeedsHelper =
     usesFlora &&
     (options.features.some(
@@ -596,6 +603,12 @@ function App() {
     let features = on
       ? [...options.features, name]
       : options.features.filter((x) => x !== name);
+    if (on && name === SLAM) {
+      setVisibility("private");
+      change({features: [SLAM], captioner: "selected-vlm"});
+      return;
+    }
+    if (on) features = features.filter(x=>x !== SLAM);
     // Videos are generated per subtask, so they need the subtask annotations.
     if (on && name === HUMAN_VIDEOS && !features.includes("Subtasks"))
       features = ["Subtasks", ...features];
@@ -1273,6 +1286,11 @@ function App() {
                   <HandPipelineExplanation />
                 </section>
               )}
+              {step === 1 && <section className={`card step-card ${usesSlam ? "selected" : ""}`} aria-label="SLAM camera poses">
+                <h3>SLAM camera poses</h3><p className="field-help">Track wrist or handheld cameras with DROID-W + MoGe. Export 3D trajectories in the initial camera frame for Grabette and robot datasets.</p>
+                <label className="annotation-choice"><input type="checkbox" checked={usesSlam} onChange={()=>toggle(SLAM)}/><span><strong>DROID-W + MoGe Small</strong><small>Standalone GPU pipeline · preserves source actions and joint states</small></span></label>
+              </section>}
+              {step === 2 && usesSlam && <SlamSettings value={options.slam} onChange={slam=>change({slam})}/>}
               {step === 2 && usesHands && (
                 <HandActionSettings
                   value={options.hand_actions}
@@ -1490,7 +1508,7 @@ function App() {
                     <div>
                       <span>Model</span>
                       <strong>
-                        {handsOnly
+                        {usesSlam ? "DROID-W + MoGe · HF Jobs" : handsOnly
                           ? "EgoHand / SOMA · DROID-W + MoGe · HF Jobs"
                           : usesFlora
                             ? "NVIDIA Flora · TempSeg + Captioner · H200 GPU"
@@ -1533,7 +1551,7 @@ function App() {
                               type="radio"
                               name="visibility"
                               value={value}
-                              disabled={usesHands && value === "public"}
+                              disabled={(usesHands || usesSlam) && value === "public"}
                               checked={visibility === value}
                               onChange={() => {
                                 setVisibility(value);
@@ -2057,7 +2075,7 @@ function App() {
             <div>
               <dt>Annotation model</dt>
               <dd>
-                {handsOnly
+                {usesSlam ? "DROID-W + MoGe · HF Jobs" : handsOnly
                   ? "EgoHand / SOMA · DROID-W + MoGe · HF Jobs"
                   : quote?.captioner === "flora"
                     ? "NVIDIA Flora · TempSeg + Captioner · H200 GPU · noncommercial research"

hand-worker-image.json

@@ -70,7 +70,7 @@
     "hand_actions.py": "a0c448022c2933560e27f93d371466eb4aaddb979330512412eb08ca6ae2de8c",
     "hand_setup.py": "e4e40e83510e724a8ab9b08f77dbad2d64401051cf6849ed03d9b572b18e2bdb",
     "pipeline-version.json": "ee3868635d88f578c805ea6afa13a4587989bc8586fcb81a1127cdeb2d59bafe",
-    "studio.py": "c5328f0e36fd327699427b845f088b0af35b2cdbc19d480905edbdcec6014fd6",
+    "studio.py": "fa222a7ad7bf57cdfa7e3835ee5e1773763ead41fa88b5140c3631a732283cb6",
     "worker.py": "c4f53f78e246e2e42cc8348d5ed23954badc1c027bceb579ff24be0db89a9110"
   }
 }

server.py

@@ -23,6 +23,7 @@ from model_catalog import DEFAULT_MODEL, public_models
 from prompt_config import public_catalog, validate_prompts
 from pydantic import BaseModel, Field
 from reporting import annotation_url, episode_progress, hand_visualizer_url, usage_costs
+from slam_pipeline import catalog as slam_catalog
 from studio import COPY, FEATURES, launch, prepare, read_cameras
 
 app = FastAPI(title="LeRobot Annotation Studio", docs_url=None, redoc_url=None, openapi_url=None)
@@ -47,13 +48,21 @@ class HandActionInput(BaseModel):
     calibration_mode: Literal["dataset", "estimate-moge"] = "dataset"
 
 
+class SlamInput(BaseModel):
+    compute: Literal["h200", "a100-large", "rtx-pro-6000"] = "rtx-pro-6000"
+    calibration_mode: Literal["dataset", "estimate-moge", "manual"] = "dataset"
+    sample_mode: Literal["all-frames", "10hz"] = "all-frames"
+    intrinsics: dict | None = None
+
+
 class EstimateInput(DatasetInput):
     model: str = Field(default=DEFAULT_MODEL, min_length=1, max_length=80)
     captioner: Literal["selected-vlm", "flora"] = "selected-vlm"
     start: int = Field(default=0, ge=0)
     count: int = Field(default=100, ge=0)
-    features: list[str] = Field(min_length=1, max_length=8)
+    features: list[str] = Field(min_length=1, max_length=9)
     hand_actions: HandActionInput | None = None
+    slam: SlamInput | None = None
     camera: str | None = Field(default=None, max_length=200)
     hz: float = 0.1
     hours: int = 4
@@ -282,6 +291,7 @@ def calculate(options: EstimateInput, request: Request):
             human_video_parallel=options.human_video_parallel,
             captioner=options.captioner,
             hand_actions=options.hand_actions.model_dump() if options.hand_actions else None,
+            slam=options.slam.model_dump() if options.slam else None,
         )
     except Exception as exc:
         raise HTTPException(400, safe_error(exc)) from None
@@ -328,6 +338,11 @@ def prompts():
     return public_catalog()
 
 
+@app.get("/api/slam-pipeline")
+def slam_pipeline():
+    return slam_catalog()
+
+
 @app.post("/api/prompts/validate")
 def check_prompts(options: PromptInput, request: Request):
     check_origin(request)

slam-pipeline.json

@@ -0,0 +1,18 @@
+{
+  "version": "droid-w-camera-v1",
+  "coordinate_frame": "initial optical camera; X right, Y down, Z forward",
+  "expected_code": {
+    "slam_pipeline.py": "aafd39fe8272914f387b65b8e3796f99f2cadd9bb5323b36c6857be04467ce71",
+    "slam_job.py": "45d881331651480b87460a6abd868a3cb2ae1dcc858fe7f3a98dada87040cc68",
+    "slam_native.py": "148a8c5aa30ca52cbce495841f4dfccb806448cf820be0611e888f963ca2a927",
+    "server.py": "f6be4561676281920e447045245efc938beca86a7e71e5c71dc949151f3c79f9",
+    "studio.py": "fa222a7ad7bf57cdfa7e3835ee5e1773763ead41fa88b5140c3631a732283cb6",
+    "subset.py": "c32c62ca23155c602a4c92e67a76613bc2c4dec4ddf9dd4dc554600bb99034d3"
+  },
+  "benchmark": {
+    "status": "running",
+    "target_tracked_fps": 200,
+    "target_achieved": false,
+    "job": "https://huggingface.co/jobs/pepijn223/6adf27b01e112550ce7f570f"
+  }
+}

slam_job.py

@@ -0,0 +1,216 @@
+"""Standalone camera trajectory annotation; retain source actions and every frame."""
+
+import hashlib
+import json
+import os
+import time
+from pathlib import Path
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.parquet as pq
+from huggingface_hub import HfApi, hf_hub_download
+from slam_native import close, prepare, track
+from slam_pipeline import SLAM, verify_release
+from studio import read_selection, relative_path
+from subset import EpisodeSubset, trim_video
+
+POSE = "annotation.camera_initial_from_optical"
+SAMPLED = "annotation.camera_pose_sampled"
+FEATURES = {
+    POSE: {
+        "dtype": "float64",
+        "shape": [4, 4],
+        "names": ["row", "column"],
+        "info": {
+            "convention": "initial_optical_from_current_optical",
+            "translation_units": "learned metric metres",
+            "axes": "X right, Y down, Z forward",
+            "world": "first camera frame of each episode",
+            "metric_accuracy_validated": False,
+        },
+    },
+    SAMPLED: {"dtype": "bool", "shape": [1], "names": None},
+}
+
+
+def append_poses(table, archive):
+    with np.load(archive, allow_pickle=False) as result:
+        poses, sampled = result["camera_pose"], result["sampled_indices"]
+        times = result["timestamp_s"]
+    if poses.shape != (len(table), 4, 4) or not np.isfinite(poses).all():
+        raise ValueError("Camera trajectory does not cover the source episode")
+    source_times = np.asarray(table["timestamp"].to_pylist())
+    if not np.allclose(source_times - source_times[0], times, atol=1e-4, rtol=0):
+        raise ValueError("Camera and robot-state timestamps differ")
+    if any(name in table.column_names for name in FEATURES):
+        raise ValueError("Source already contains camera pose annotations")
+    mask = np.zeros(len(table), dtype=bool)
+    mask[sampled] = True
+    return table.append_column(
+        POSE, pa.array(poses.tolist(), type=pa.list_(pa.list_(pa.float64(), 4), 4))
+    ).append_column(SAMPLED, pa.array(mask))
+
+
+def run(manifest, work=Path("/workspace/slam-run")):
+    if manifest["features"] != [SLAM] or not manifest.get("slam"):
+        raise ValueError("Standalone SLAM requires the pinned SLAM selection")
+    config = manifest["slam"]
+    if config["release"] != verify_release():
+        raise ValueError("SLAM worker differs from the quoted release")
+    api = HfApi()
+    token = os.environ["HF_TOKEN"]
+    work.mkdir(parents=True, exist_ok=False)
+    source, output = work / "source", work / "dataset"
+    source.mkdir()
+    selection = read_selection(
+        manifest["dataset"],
+        manifest["start"],
+        manifest["count"],
+        token=token,
+        revision=manifest["dataset_revision"],
+    )
+    rows, info = selection["episodes"], selection["info"]
+    if (
+        hashlib.sha256(json.dumps([r["episode_index"] for r in rows]).encode()).hexdigest()
+        != manifest["episode_ids_sha256"]
+    ):
+        raise ValueError("Source episode selection differs from the quote")
+
+    def download(name):
+        name = relative_path(name)
+        return Path(
+            hf_hub_download(
+                manifest["dataset"],
+                name,
+                repo_type="dataset",
+                revision=manifest["dataset_revision"],
+                local_dir=source,
+                token=token,
+            )
+        )
+
+    for name in ("meta/info.json", "meta/tasks.parquet"):
+        download(name)
+    metadata = [
+        x.path
+        for x in api.list_repo_tree(
+            manifest["dataset"],
+            path_in_repo="meta/episodes",
+            recursive=True,
+            repo_type="dataset",
+            revision=manifest["dataset_revision"],
+        )
+        if x.path.endswith(".parquet")
+    ]
+    for name in metadata:
+        download(name)
+    writer = EpisodeSubset(source, output, [r["episode_index"] for r in rows])
+    api.create_repo(
+        manifest["output"],
+        repo_type="dataset",
+        private=manifest.get("visibility", "private") == "private",
+        exist_ok=False,
+    )
+    setup_start = time.perf_counter()
+    prepare(config["runtime"])
+    results = {
+        "pipeline": "DROID-W + MoGe Small",
+        "manifest": manifest,
+        "status": "running",
+        "expected_episodes": len(rows),
+        "written_episode_ids": [],
+        "usage": {"measured_inference_usd": 0},
+        "setup_seconds": time.perf_counter() - setup_start,
+        "episodes": [],
+    }
+    deadline = (
+        float(os.environ.get("ANNOTATION_DEADLINE_EPOCH", "0"))
+        or time.time() + manifest["costs"]["hours"] * 3600 - 60
+    )
+    try:
+        for row in rows:
+            if time.time() >= deadline:
+                raise TimeoutError("SLAM Job reached its quoted time limit")
+            episode = int(row["episode_index"])
+            directory = work / f"episode-{episode}"
+            directory.mkdir()
+            data = download(
+                info["data_path"].format(
+                    chunk_index=int(row["data/chunk_index"]), file_index=int(row["data/file_index"])
+                )
+            )
+            table = pq.read_table(data, filters=[("episode_index", "=", episode)])
+            for camera in selection["cameras"]:
+                download(
+                    info["video_path"].format(
+                        video_key=camera,
+                        chunk_index=int(row[f"videos/{camera}/chunk_index"]),
+                        file_index=int(row[f"videos/{camera}/file_index"]),
+                    )
+                )
+            camera = manifest["camera"]
+            video = directory / "episode.mp4"
+            source_video = source / info["video_path"].format(
+                video_key=camera,
+                chunk_index=int(row[f"videos/{camera}/chunk_index"]),
+                file_index=int(row[f"videos/{camera}/file_index"]),
+            )
+            trim_video(
+                source_video,
+                video,
+                float(row[f"videos/{camera}/from_timestamp"]),
+                int(row["length"]),
+                float(info["fps"]),
+                lossless=True,
+            )
+            archive, report = track(
+                video, directory / "native", config["calibration"], config["runtime"], config["sample_fps"]
+            )
+            annotated = append_poses(table, archive)
+            new_id, _ = writer.add(episode, annotated, extra_features=FEATURES)
+            results["episodes"].append(
+                {"source_episode_index": episode, "episode_index": new_id, "status": "completed", **report}
+            )
+            results["written_episode_ids"].append(new_id)
+            target = output / "annotation_studio"
+            target.mkdir(exist_ok=True)
+            (target / "slam.json").write_text(json.dumps(results, indent=2))
+            (target / "run.json").write_text(json.dumps(results, indent=2))
+            (target / "progress.json").write_text(
+                json.dumps(
+                    {
+                        "status": "running",
+                        "episodes_completed": len(results["episodes"]),
+                        "episodes_total": len(rows),
+                    }
+                )
+            )
+            api.upload_folder(
+                repo_id=manifest["output"],
+                repo_type="dataset",
+                folder_path=output,
+                commit_message=f"SLAM camera poses: episode {episode}",
+            )
+    finally:
+        close()
+    results["status"] = "completed"
+    (output / "annotation_studio/run.json").write_text(json.dumps(results, indent=2))
+    (output / "annotation_studio/slam.json").write_text(json.dumps(results, indent=2))
+    (output / "annotation_studio/progress.json").write_text(
+        json.dumps({"status": "completed", "episodes_completed": len(rows), "episodes_total": len(rows)})
+    )
+    (output / "README.md").write_text(
+        f"---\nlicense: {manifest.get('source_license') or 'other'}\n---\n\n# DROID-W camera trajectories\n\nSource: {manifest['dataset']} at {manifest['dataset_revision']}.\n\nOriginal joint states and actions are retained. Camera poses use a learned metric prior; camera mounting and metric accuracy require validation. Sampling masks distinguish tracked and interpolated frames. See `annotation_studio/slam.json` for provenance, calibration and timings.\n"
+    )
+    api.upload_folder(
+        repo_id=manifest["output"],
+        repo_type="dataset",
+        folder_path=output,
+        commit_message="Complete SLAM annotation",
+    )
+    api.create_tag(manifest["output"], repo_type="dataset", tag="v3.0", exist_ok=True)
+
+
+if __name__ == "__main__":
+    run(json.loads(os.environ["ANNOTATION_MANIFEST"]))

slam_native.py

@@ -0,0 +1,131 @@
+"""Camera-only access to Studio's frozen DROID-W + MoGe runtime."""
+
+import json
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+from types import SimpleNamespace
+
+import numpy as np
+
+
+def prepare(profile):
+    from hand_setup import configure_library_paths, verify_image
+
+    verify_image(profile)
+    backend = profile["backend"]
+    configure_library_paths(backend)
+    os.environ["LD_LIBRARY_PATH"] = os.environ["DROID_LIBRARY_PATH"]
+    env = dict(os.environ, NVIDIA_TF32_OVERRIDE="0", HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1")
+    env.pop("HF_TOKEN", None)
+    env["PYTHONPATH"] = "/opt/camera-adapter/src:/workspace/source/src:/workspace/app"
+    for operation in ("--check-solver", "--build", "--check-contexts"):
+        subprocess.run(
+            [
+                backend["droid_python"],
+                str(Path(__file__).with_name("hand_setup.py")),
+                operation,
+                json.dumps(backend),
+            ],
+            env=env,
+            check=True,
+        )
+
+
+def track(video, destination, calibration, profile, sample_fps=10.0):
+    from lerobot_camera_runtime.egocentric.droidw_camera import estimate_rectified_camera
+    from lerobot_camera_runtime.egocentric.intrinsics import CameraIntrinsics
+    from lerobot_camera_runtime.egocentric.video import rectify_video
+
+    from lerobot.annotations.hands.config import HandBackendConfig
+    from lerobot.annotations.hands.intrinsics import resolve_calibration
+
+    destination = Path(destination)
+    destination.mkdir(parents=True, exist_ok=False)
+    started = time.perf_counter()
+    backend = HandBackendConfig(**profile["backend"])
+    calibration = resolve_calibration(calibration, Path(video), destination, backend)
+    matrix = np.array(
+        [[calibration["fx"], 0, calibration["cx"]], [0, calibration["fy"], calibration["cy"]], [0, 0, 1.0]]
+    )
+    intrinsics = CameraIntrinsics(
+        calibration["width"],
+        calibration["height"],
+        matrix,
+        np.array(calibration.get("distortion", [])),
+        calibration["model"],
+        Path(video),
+    )
+    decoded = rectify_video(Path(video), intrinsics, destination / "rgb")
+    decode_s = time.perf_counter() - started
+    config = SimpleNamespace(
+        droid_root=Path(backend.droid_root),
+        droid_python=Path(backend.droid_python),
+        droid_moge_checkpoint=Path(backend.droid_moge_checkpoint),
+        droid_model_cache=Path(backend.droid_model_cache),
+        droid_resolution=320,
+        droid_sample_fps=sample_fps,
+    )
+    archive, stats = estimate_rectified_camera(decoded, destination / "camera", config)
+    evidence = json.loads((destination / "camera/worker/output/camera/moge-ablation.json").read_text())
+    if (
+        evidence.get("mode") != "on"
+        or evidence.get("metric_depth_reg") is not True
+        or evidence.get("gt_scale_fit") is not False
+        or evidence.get("counts", {}).get("depth_predictions", 0) < 1
+    ):
+        raise ValueError("Camera must retain the MoGe metric prior without fitting ground-truth scale")
+    elapsed = time.perf_counter() - started
+    with np.load(archive, allow_pickle=False) as result:
+        poses = result["camera_pose"]
+        sampled = result["sampled_indices"]
+        if poses.shape != (decoded.frame_count, 4, 4) or not np.isfinite(poses).all():
+            raise ValueError("SLAM must produce finite camera poses for every source frame")
+        values = dict(result)
+    # A sequence-level coordinate frame: optical axes of the first camera pose.
+    poses = np.linalg.inv(poses[0]) @ poses
+    values["camera_pose"] = poses
+    np.savez_compressed(archive, **values)
+    report = {
+        **stats,
+        "decode_and_intrinsics_s": decode_s,
+        "wall_s": elapsed,
+        "source_fps": decoded.frame_count / elapsed,
+        "tracked_fps": len(sampled) / stats["model_s"],
+        "source_model_fps": decoded.frame_count / stats["model_s"],
+        "calibration": calibration,
+        "depth_prior": evidence,
+        "interpolated_frames": decoded.frame_count - len(sampled),
+        "throughput_target_fps": 200,
+        "throughput_target_met": len(sampled) / stats["model_s"] >= 200,
+        "accuracy_validated": False,
+        "pose_convention": "initial_optical_from_current_optical; X right, Y down, Z forward; learned metric scale",
+    }
+    (destination / "report.json").write_text(json.dumps(report, indent=2))
+    print("SLAM_RESULT " + json.dumps(report), flush=True)
+    return archive, report
+
+
+def close():
+    from lerobot_camera_runtime.infra.droidw_worker import close_workers
+
+    close_workers()
+
+
+if __name__ == "__main__":
+    request = json.loads(Path(sys.argv[1]).read_text())
+    try:
+        if request.get("prepare", True):
+            prepare(request["runtime"])
+        for item in request["clips"]:
+            track(
+                item["video"],
+                item["destination"],
+                item["calibration"],
+                request["runtime"],
+                item.get("sample_fps", 10.0),
+            )
+    finally:
+        close()

slam_pipeline.py

@@ -0,0 +1,73 @@
+"""Standalone SLAM selection and immutable runtime identity for Studio."""
+
+import hashlib
+import json
+from copy import deepcopy
+from pathlib import Path
+
+from hand_actions import COMPUTE, prepare_hands, runtime_configuration
+
+SLAM = "SLAM camera poses"
+FILES = {"slam_pipeline.py", "slam_job.py", "slam_native.py", "server.py", "studio.py", "subset.py"}
+
+
+def verify_release():
+    root = Path(__file__).parent
+    release = json.loads((root / "slam-pipeline.json").read_text())
+    if set(release["expected_code"]) != FILES or any(
+        hashlib.sha256((root / name).read_bytes()).hexdigest() != digest
+        for name, digest in release["expected_code"].items()
+    ):
+        raise ValueError("SLAM runtime changed; validate the release before launching")
+    return release
+
+
+def catalog():
+    try:
+        release = verify_release()
+        runtime_configuration()
+        available, reason = True, None
+    except (ValueError, OSError, KeyError) as exc:
+        release, available, reason = {}, False, str(exc)
+    return {
+        "available": available,
+        "reason": reason,
+        "model": "DROID-W + MoGe Small",
+        "profiles": [{"key": k, "name": v} for k, v in COMPUTE.items()],
+        "benchmark": release.get("benchmark"),
+        "target_fps": 200,
+        "output": "Per-frame poses in the initial optical camera frame, source timestamps and sampling mask",
+        "accuracy": "Learned metric scale; validate against calibrated robot or camera poses",
+    }
+
+
+def prepare_slam(selection, camera, options, token):
+    options = options or {}
+    if options.get("sample_mode", "all-frames") not in {"all-frames", "10hz"}:
+        raise ValueError("Choose all source frames or explicit 10 Hz sampling")
+    release = verify_release()
+    # Manual lens calibration applies only to this run; source metadata is retained.
+    prepared = deepcopy(selection)
+    supplied = options.get("intrinsics")
+    manual = options.get("calibration_mode") == "manual"
+    if manual:
+        if not supplied:
+            raise ValueError("Enter the measured camera intrinsics for manual calibration")
+        prepared["info"]["features"][camera].setdefault("info", {})["camera.intrinsics"] = supplied
+        options = dict(options, calibration_mode="dataset")
+    plan = prepare_hands(prepared, camera, options, token)
+    if manual:
+        plan["calibration"]["source"] = "user-supplied"
+    plan.update(
+        sample_fps=float(selection["info"]["fps"])
+        if options.get("sample_mode", "all-frames") == "all-frames"
+        else 10.0,
+        release=release,
+        timing={
+            "frames": sum(r["length"] for r in selection["episodes"]),
+            "planning_fps": None,
+            "target_fps": 200,
+            "basis": "HF Jobs benchmark; throughput depends on scene and GPU",
+        },
+    )
+    return plan

studio.py

@@ -27,6 +27,7 @@ from model_catalog import (
     model_profile,
 )
 from prompt_config import TEMPLATES, validate_prompts
+from slam_pipeline import SLAM, prepare_slam, verify_release
 
 MODEL = model_profile()["route"]
 PROVIDER = model_profile()["provider"]
@@ -40,6 +41,7 @@ FEATURES = {
     # Not a language style: videos are saved under annotation_studio/human_videos/.
     "Human demo videos": "human_video",
     HANDS: "hands",
+    SLAM: "slam",
 }
 HUMAN_VIDEOS = "Human demo videos"
 COPY = "Annotated dataset copy"
@@ -48,7 +50,7 @@ EXPORT = "Annotation files only"  # Historical reports only; new runs always wri
 
 def needs_provider(features, prompts=None, captioner="selected-vlm"):
     """Native hand-only and Flora workflows never need a hosted helper."""
-    features = set(features) - {HANDS}
+    features = set(features) - {HANDS, SLAM}
     if not features:
         return False
     return (
@@ -206,7 +208,7 @@ def estimate(
     if hours not in (1, 4, 12) or vqa_hz not in (0.1, 0.5, 1.0):
         raise ValueError("Choose a supported timeout and visual-question frequency.")
     n, seconds = len(selection["episodes"]), selection["seconds"]
-    needs_subtasks = bool(set(features) - {"Visual questions", HANDS})
+    needs_subtasks = bool(set(features) - {"Visual questions", HANDS, SLAM})
     if captioner == "flora" and not needs_subtasks:
         raise ValueError("Flora needs an annotation type that uses subtasks.")
     if captioner == "flora" and "plan_subtasks" in prompts:
@@ -366,6 +368,7 @@ def prepare(
     human_video_parallel=None,
     captioner="selected-vlm",
     hand_actions=None,
+    slam=None,
 ):
     profile = model_profile(model_key)
     prompts = validate_prompts(prompts or {}, features)
@@ -380,10 +383,14 @@ def prepare(
         raise ValueError("Choose a camera in this dataset.")
     if captioner not in {"selected-vlm", "flora"}:
         raise ValueError("Choose a supported subtask captioner.")
+    if SLAM in features and (set(features) != {SLAM} or captioner == "flora"):
+        raise ValueError("Run standalone SLAM separately from other annotations")
+    camera_slam = prepare_slam(selection, camera, slam, token) if SLAM in features else None
     hands = prepare_hands(selection, camera, hand_actions, token) if HANDS in features else None
     if hands and captioner == "flora":
         raise ValueError("Run Hand actions and Flora separately so each model has the GPU memory it requires")
-    flavor = hands["compute"] if hands else FLORA["hardware"] if captioner == "flora" else "cpu-upgrade"
+    native = hands or camera_slam
+    flavor = native["compute"] if native else FLORA["hardware"] if captioner == "flora" else "cpu-upgrade"
     hardware = next(h for h in HfApi(token=token or False).list_jobs_hardware() if h.name == flavor)
     if hardware.unit_label != "minute":
         raise ValueError("HF changed its billing units; refresh the Space before launching.")
@@ -399,8 +406,8 @@ def prepare(
         human_video_parallel,
         captioner,
     )
-    if hands:
-        costs.update(compute_flavor=flavor, compute_name=COMPUTE[flavor], hand_timing=hands["timing"])
+    if native:
+        costs.update(compute_flavor=flavor, compute_name=COMPUTE[flavor], hand_timing=native["timing"])
     return {
         "selection": selection,
         "features": list(features),
@@ -413,6 +420,7 @@ def prepare(
         "captioner": captioner,
         "captioner_profile": dict(FLORA) if captioner == "flora" else None,
         "hand_actions": hands,
+        "slam": camera_slam,
         "human_video": {
             # 0 = every subtask.
             "segments_per_episode": human_video_segments,
@@ -509,7 +517,10 @@ def launch(
 ):
     if visibility not in ("private", "public"):
         raise ValueError("Choose private or public output visibility.")
-    hands = plan.get("hand_actions") if plan else None
+    camera_slam = plan.get("slam") if plan else None
+    hands = (plan.get("hand_actions") or camera_slam) if plan else None
+    if camera_slam and camera_slam["release"] != verify_release():
+        raise ValueError("SLAM runtime changed; calculate a fresh estimate")
     if hands and visibility != "private":
         raise ValueError("Hand research outputs must be private")
     if hands and hands["runtime"] != pin_runtime(runtime_configuration()):
@@ -616,12 +627,14 @@ def launch(
             "count": plan["selection"]["count"],
             "budget": budget,
             "inference_budget": budget - plan["costs"]["cpu_max"],
-            "model": "EgoHand/SOMA + DROID-W + MoGe"
-            if set(plan["features"]) == {HANDS}
+            "model": "DROID-W + MoGe Small"
+            if camera_slam
+            else "EgoHand/SOMA + DROID-W + MoGe"
+            if set(plan["features"]) in ({HANDS}, {SLAM})
             else profile["route"],
             "provider_required": uses_provider,
             "subtask_models": []
-            if set(plan["features"]) == {HANDS}
+            if set(plan["features"]) in ({HANDS}, {SLAM})
             else [FLORA["segmenter_id"], FLORA["model_id"]]
             if captioner == "flora"
             else [profile["model_id"]],
@@ -686,6 +699,11 @@ def launch(
             raise ValueError("The hand image is still building; wait for its checks before starting a Job")
         image = hands["runtime"]["image"]
         command = job_command(space, revision, timeout_minutes, captioner, hands=True)
+        if camera_slam:
+            command[-1] = command[-1].replace(
+                "python /workspace/app/worker.py",
+                "export PYTHONPATH=/opt/camera-adapter/src:$PYTHONPATH\npython /workspace/app/slam_job.py",
+            )
     job = api.run_job(
         image=image,
         command=command,

subset.py

@@ -103,7 +103,7 @@ class EpisodeSubset:
                 }
             )
 
-    def add(self, source_id, table, *, hand_features=None):
+    def add(self, source_id, table, *, hand_features=None, extra_features=None):
         """Materialize an annotated episode, retaining all its cameras and existing other styles."""
         if any(m["source_episode_index"] == source_id for m in self.mapping):
             raise ValueError("Episode was already added to the subset.")
@@ -112,6 +112,10 @@ class EpisodeSubset:
         length = int(row["length"])
         if len(table) != length or table["frame_index"].to_pylist() != list(range(length)):
             raise ValueError("Episode data length or frame order does not match its metadata.")
+        for name, feature in (extra_features or {}).items():
+            if name in self.source_info["features"] or name not in table.column_names:
+                raise ValueError("New annotations must use a new feature with a matching column")
+            self.info["features"][name] = deepcopy(feature)
         if hand_features:
             description = {
                 "coordinates": "fixed segment-start FLU (X forward, Y left, Z up)",

test_slam.py

@@ -0,0 +1,66 @@
+"""SLAM preserves source supervision and rejects false temporal alignment."""
+
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+import numpy as np
+import pyarrow as pa
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent))
+from slam_job import POSE, SAMPLED, append_poses
+from slam_pipeline import SLAM, prepare_slam
+from studio import needs_provider
+
+
+def source():
+    return pa.table(
+        {
+            "timestamp": [0.0, 0.02, 0.04],
+            "frame_index": [0, 1, 2],
+            "action": [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]],
+            "observation.state": [[0.5], [0.6], [0.7]],
+        }
+    )
+
+
+def test_source_actions_and_joint_states_survive(tmp_path):
+    table = source()
+    path = tmp_path / "camera.npz"
+    poses = np.tile(np.eye(4), (3, 1, 1))
+    poses[:, 0, 3] = [0, 0.01, 0.02]
+    np.savez(path, camera_pose=poses, sampled_indices=[0, 2], timestamp_s=[0, 0.02, 0.04])
+    result = append_poses(table, path)
+    for name in table.column_names:
+        assert result[name].equals(table[name])
+    assert result[SAMPLED].to_pylist() == [True, False, True]
+    assert np.asarray(result[POSE].to_pylist())[2, 0, 3] == 0.02
+
+
+def test_bad_timestamps_are_rejected(tmp_path):
+    path = tmp_path / "camera.npz"
+    np.savez(
+        path,
+        camera_pose=np.tile(np.eye(4), (3, 1, 1)),
+        sampled_indices=[0, 1, 2],
+        timestamp_s=[0, 0.03, 0.06],
+    )
+    with pytest.raises(ValueError, match="timestamps"):
+        append_poses(source(), path)
+
+
+def test_manual_intrinsics_do_not_modify_source():
+    selection = {"info": {"fps": 50, "features": {"cam": {"info": {}}}}, "episodes": [{"length": 500}]}
+    intrinsics = {"model": "pinhole", "width": 480, "height": 360, "fx": 350, "fy": 350, "cx": 240, "cy": 180}
+    with (
+        patch("slam_pipeline.verify_release", return_value={}),
+        patch("slam_pipeline.prepare_hands", return_value={"calibration": dict(intrinsics)}),
+    ):
+        result = prepare_slam(
+            selection, "cam", {"calibration_mode": "manual", "intrinsics": intrinsics}, False
+        )
+    assert selection["info"]["features"]["cam"]["info"] == {}
+    assert result["sample_fps"] == 50
+    assert result["calibration"]["source"] == "user-supplied"
+    assert not needs_provider([SLAM])

test_studio.py

@@ -25,7 +25,7 @@ from lerobot.annotations.steerable_pipeline.vlm_client import StubVlmClient
 @pytest.mark.parametrize(
     "features",
     [
-        [f for f in studio.FEATURES if f != studio.HANDS],
+        [f for f in studio.FEATURES if f not in {studio.HANDS, studio.SLAM}],
         ["Interjections & speech"],
         ["Memory"],
         ["Visual questions"],