Integrate Hand actions into the annotation wizard with HF Jobs and private MANO assets

Dockerfile

@@ -11,8 +11,9 @@ WORKDIR /app
 COPY requirements.txt ./
 RUN pip install --no-cache-dir -r requirements.txt
 COPY --chown=user:user . .
+RUN python bootstrap.py
 COPY --from=frontend --chown=user:user /build/dist /app/frontend/dist
 USER user
-ENV HOME=/home/user PYTHONUNBUFFERED=1 HF_HUB_DISABLE_TELEMETRY=1
+ENV HOME=/home/user PYTHONUNBUFFERED=1 HF_HUB_DISABLE_TELEMETRY=1 PYTHONPATH=/workspace/source/src
 EXPOSE 7860
 CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "7860", "--no-access-log"]

Dockerfile.flora

@@ -1,6 +1,6 @@
 # Build once on CPU; no credentials or model weights are embedded in this image.
 # Verified upstream PyTorch image digest, CUDA 12.8 / Torch 2.11.0.
-FROM pytorch/pytorch:2.11.0-cuda12.8-cudnn9-devel@sha256:53ab3de62f6101d1e42f9be28623ab7a468a24c070d632f211ed576e30b6abd3
+FROM pytorch/pytorch:2.11.0-cuda12.8-cudnn9-devel@sha256:53ab3de62f6101d1e42f9be28623ab7a468a24c070d632f211ed576e30b6abd3 AS build
 ENV DEBIAN_FRONTEND=noninteractive TORCH_CUDA_ARCH_LIST=9.0 MAX_JOBS=4
 RUN apt-get update && apt-get install -y --no-install-recommends python3-venv \
     && rm -rf /var/lib/apt/lists/* \
@@ -11,11 +11,23 @@ COPY worker-requirements.txt flora-requirements.txt flora-acceleration-requireme
 # Keep the reference worker/native dependency overlay order.
 RUN python -m pip install --no-cache-dir -r worker-requirements.txt
 RUN python -m pip install --no-cache-dir -r flora-requirements.txt
+COPY build_causal_conv1d.py ./
+ENV MAX_JOBS=1
+RUN python build_causal_conv1d.py
 RUN python -m pip install --no-cache-dir --no-build-isolation -r flora-acceleration-requirements.txt
+RUN python -c "import causal_conv1d; from fla.ops.gated_delta_rule import chunk_gated_delta_rule, fused_recurrent_gated_delta_rule"
 COPY *.py *.txt prompt-templates.json pipeline-source.tar.gz pipeline-version.json ./
 RUN python bootstrap.py
+
+# Keep the 10-GB compiler toolchain out of each paid Job's image pull.
+FROM pytorch/pytorch:2.11.0-cuda12.8-cudnn9-runtime@sha256:eee11b3b3872a8c838e35ef48f08b2d5def2080902c7f666831310ca1a0ef2be
+COPY --from=build /opt/flora-venv /opt/flora-venv
+COPY --from=build /workspace /workspace
+WORKDIR /workspace/app
+ENV PATH=/opt/flora-venv/bin:$PATH
 ENV PYTHONPATH=/workspace/source/src:/workspace/app PYTHONUNBUFFERED=1 HF_HUB_DISABLE_TELEMETRY=1
-ENV FLORA_EXECUTION=accelerated FLORA_BATCH_SIZE=4 FLORA_ENABLE_INFERENCE=1
+ENV FLORA_EXECUTION=accelerated FLORA_BATCH_SIZE=8 FLORA_ENABLE_INFERENCE=1
+RUN python -c "import worker, torch, causal_conv1d; from fla.ops.gated_delta_rule import chunk_gated_delta_rule, fused_recurrent_gated_delta_rule; assert torch.__version__.split('+')[0] == '2.11.0'"
 # CPU Space builds expose only health information. A GPU Job overrides this CMD
 # with python /workspace/app/worker.py and supplies its own scoped HF secret.
 EXPOSE 7860

build_causal_conv1d.py

@@ -0,0 +1,54 @@
+"""Build pinned upstream kernels for H200 with one compiler process/thread."""
+
+import hashlib
+import os
+import subprocess
+import sys
+import tarfile
+import tempfile
+from pathlib import Path
+
+import requests
+
+URL = "https://files.pythonhosted.org/packages/f4/d1/eba27735f31bd5527d39a3c7f693e6ded7b844cab275db719a15ad37b6cd/causal_conv1d-1.7.0.tar.gz"
+SHA256 = "3202758494eaa7b597ce1c282dfa188889506bfcb92cad3c407d26736bfcd32b"
+
+
+def main():
+    response = requests.get(URL, timeout=60)
+    response.raise_for_status()
+    if hashlib.sha256(response.content).hexdigest() != SHA256:
+        raise RuntimeError("Causal convolution source checksum mismatch")
+    with tempfile.TemporaryDirectory() as temporary:
+        archive = Path(temporary) / "source.tar.gz"
+        archive.write_bytes(response.content)
+        with tarfile.open(archive) as source:
+            source.extractall(temporary, filter="data")
+        root = Path(temporary) / "causal_conv1d-1.7.0"
+        setup = root / "setup.py"
+        content = setup.read_text()
+        old = 'return nvcc_extra_args + ["--threads", "4"]'
+        if content.count(old) != 1:
+            raise RuntimeError("Upstream compiler thread configuration changed")
+        # Compile only Hopper SM90, the hardware this image is explicitly for.
+        # Kernel sources and numeric flags are unchanged; omit unused GPU targets.
+        start = content.index(
+            '        cc_flag.append("-gencode")\n        cc_flag.append("arch=compute_75,code=sm_75")'
+        )
+        end = content.index("    # HACK:", start)
+        content = (
+            content[:start]
+            + '        cc_flag += ["-gencode", "arch=compute_90,code=sm_90"]\n'
+            + content[end:]
+        )
+        setup.write_text(content.replace(old, 'return nvcc_extra_args + ["--threads", "1"]'))
+        env = {**os.environ, "MAX_JOBS": "1", "CAUSAL_CONV1D_FORCE_BUILD": "TRUE"}
+        subprocess.run(
+            [sys.executable, "-m", "pip", "install", "--no-cache-dir", "--no-build-isolation", str(root)],
+            env=env,
+            check=True,
+        )
+
+
+if __name__ == "__main__":
+    main()

deploy.py

@@ -92,8 +92,8 @@ def main():
             ".cache/**",
             ".git/**",
         ],
-        delete_patterns=["app.py"],
-        commit_message="Use scalable annotation runtime, shared human-video module and optional Flora captions",
+        delete_patterns=["app.py", "frontend/src/HandSetupView.tsx", "frontend/src/hands.css"],
+        commit_message="Integrate Hand actions into the annotation wizard with HF Jobs and private MANO assets",
     )
     print(f"https://huggingface.co/spaces/{repo}\nRevision: {result.oid}")
 

flora_acceleration.py

@@ -14,6 +14,8 @@ def require_fast_kernels():
 
     if not torch.cuda.is_available() or not modeling_qwen3_5.is_fast_path_available:
         raise RuntimeError("Accelerated Flora needs CUDA, flash-linear-attention and causal-conv1d")
+    if torch.cuda.get_device_capability(0) != (9, 0):
+        raise RuntimeError("This accelerated Flora image is compiled for H200/Hopper SM90")
     return {
         name: version(name) for name in ("torch", "transformers", "flash-linear-attention", "causal-conv1d")
     }

flora_image.py

@@ -0,0 +1,52 @@
+"""Only enable a prebuilt worker after a complete, measured acceptance run."""
+
+import json
+import math
+import re
+from pathlib import Path
+
+from flora_captioner import FLORA
+from huggingface_hub.utils import validate_repo_id
+
+IDENTITY_FILES = {
+    "worker.py",
+    "subset.py",
+    "flora_captioner.py",
+    "flora_acceleration.py",
+    "flora_speed_entry.py",
+    "studio_runtime.py",
+    "pipeline-version.json",
+}
+
+
+def validated_worker_image(captioner, features, path=None):
+    if captioner != "flora" or set(features) != {"Subtasks"}:
+        return None
+    path = path or Path(__file__).with_name("flora-worker-image.json")
+    if not path.exists():
+        return None
+    profile = json.loads(path.read_text())
+    if type(profile.get("batch_size", 4)) is not int or profile.get("batch_size", 4) not in (1, 2, 4, 8):
+        raise ValueError("Accelerated Flora batch size is not validated")
+    accepted = profile.get("acceptance", {})
+    seconds = accepted.get("end_to_end_seconds")
+    if (
+        accepted.get("episodes") != 25
+        or accepted.get("independent") is not True
+        or accepted.get("fused_kernels") is not True
+        or not isinstance(seconds, (int, float))
+        or isinstance(seconds, bool)
+        or not math.isfinite(seconds)
+        or not 0 < seconds < 900
+        or profile.get("captioner_profile") != FLORA
+    ):
+        raise ValueError("Accelerated Flora worker has not passed its acceptance run")
+    validate_repo_id(profile["repo"])
+    if not re.fullmatch(r"[a-f0-9]{40}", profile["revision"]):
+        raise ValueError("Accelerated Flora image revision must be pinned")
+    hashes = profile.get("expected_code", {})
+    if set(hashes) != IDENTITY_FILES or any(
+        not re.fullmatch(r"[a-f0-9]{64}", value) for value in hashes.values()
+    ):
+        raise ValueError("Accelerated Flora code checksums are incomplete")
+    return profile

frontend/src/HandActionSettings.tsx

@@ -0,0 +1,169 @@
+import { ArrowUpRight, Hand } from "lucide-react";
+
+export type HandOptions = {
+  compute: string;
+  planning_fps: number | null;
+  calibration: string;
+  mano_repo: string;
+  research_rights: boolean;
+};
+export type HandCatalog = {
+  available: boolean;
+  reason: string | null;
+  profiles: { key: string; name: string; reference_fps: number | null }[];
+};
+export const defaultHands: HandOptions = {
+  compute: "h200",
+  planning_fps: null,
+  calibration: "",
+  mano_repo: "",
+  research_rights: false,
+};
+
+export default function HandActionSettings({
+  value,
+  catalog,
+  onChange,
+}: {
+  value: HandOptions;
+  catalog: HandCatalog | null;
+  onChange: (value: HandOptions) => void;
+}) {
+  const update = (patch: Partial<HandOptions>) =>
+    onChange({ ...value, ...patch });
+  return (
+    <section
+      className="card step-card hand-settings"
+      aria-label="Hand action settings"
+    >
+      <div className="category-heading">
+        <h3>
+          <Hand size={16} /> Hand actions
+        </h3>
+        <p className="field-help">
+          Calibrated HaWoR / MANO hands and motion infilling with DROID-W camera
+          tracking. Runs in your Hugging Face Jobs account.
+        </p>
+      </div>
+      {!catalog?.available && (
+        <p className="message" role="status">
+          {catalog?.reason || "Reading hand runtime availability…"}
+        </p>
+      )}
+      <label className="field-label" htmlFor="hand-compute">
+        HF Jobs GPU
+      </label>
+      <select
+        id="hand-compute"
+        value={value.compute}
+        onChange={(e) =>
+          update({ compute: e.target.value, planning_fps: null })
+        }
+      >
+        {(catalog?.profiles || [{ key: "h200", name: "H200 141 GB" }]).map(
+          (p) => (
+            <option key={p.key} value={p.key}>
+              {p.name}
+            </option>
+          ),
+        )}
+      </select>
+      <label className="field-label" htmlFor="hand-fps">
+        Planning speed · input FPS
+      </label>
+      <input
+        id="hand-fps"
+        type="number"
+        min={0.1}
+        step="any"
+        placeholder="Not measured on this GPU"
+        value={value.planning_fps ?? ""}
+        onChange={(e) =>
+          update({
+            planning_fps: e.target.value ? Number(e.target.value) : null,
+          })
+        }
+      />
+      <p className="field-help">
+        HF GPU throughput has not been measured yet. Enter an assumption to
+        estimate processing time. Historical reference: 53.9 input FPS on RTX
+        5090, across ten short calibrated clips. Setup, downloads and copying
+        add time. Source video FPS stays unchanged.
+      </p>
+      <label className="field-label" htmlFor="hand-calibration">
+        Camera calibration
+      </label>
+      <textarea
+        id="hand-calibration"
+        rows={5}
+        value={value.calibration}
+        onChange={(e) => update({ calibration: e.target.value })}
+        placeholder={
+          '{"model":"pinhole","width":1408,"height":1408,"fx":700,"fy":700,"cx":704,"cy":704,"distortion":[]}'
+        }
+      />
+      <p className="field-help">
+        Leave blank to use this camera’s dataset calibration. Otherwise enter
+        measured calibration matching the video resolution. The example shows
+        the format; it is not a calibration for your camera.
+      </p>
+      <label className="field-label" htmlFor="mano-repo">
+        Your private MANO model repository
+      </label>
+      <input
+        id="mano-repo"
+        placeholder="your-account/licensed-mano"
+        value={value.mano_repo}
+        onChange={(e) => update({ mano_repo: e.target.value })}
+      />
+      <p className="field-help">
+        Contains your licensed MANO_LEFT.safetensors and MANO_RIGHT.safetensors.
+        The Job downloads them with your HF login and checks private ownership
+        and the pinned revision. Model files are never included in the output
+        dataset.
+      </p>
+      <label className="hand-consent">
+        <input
+          type="checkbox"
+          checked={value.research_rights}
+          onChange={(e) => update({ research_rights: e.target.checked })}
+        />
+        <span>
+          I obtained MANO lawfully and have permission to use MANO and the other
+          models for non-commercial research on my HF Jobs compute.
+        </span>
+      </label>
+      <p className="field-help">
+        <a href="https://mano.is.tue.mpg.de/" target="_blank" rel="noreferrer">
+          Official MANO access <ArrowUpRight size={12} />
+        </a>{" "}
+        ·{" "}
+        <a
+          href="https://mano.is.tue.mpg.de/license.html"
+          target="_blank"
+          rel="noreferrer"
+        >
+          MANO terms
+        </a>{" "}
+        ·{" "}
+        <a
+          href="https://huggingface.co/macrodata/egovision-safetensors"
+          target="_blank"
+          rel="noreferrer"
+        >
+          Other model terms
+        </a>
+        . This acknowledgment does not grant a license. Public MANO mirrors are
+        disabled.
+      </p>
+      <p className="field-help">
+        The output keeps all selected episodes and frames. It adds 102D actions,
+        a validity mask and a reference-frame index. Existing actions are
+        preserved as annotation.original_action. Unmeasured frames are masked;
+        use annotation.hand_action_valid to exclude unrecoverable frames, or
+        annotation.hand_action_observed_valid for strict observed-only training.
+        Infilled predictions are retained and flagged per hand.
+      </p>
+    </section>
+  );
+}

frontend/src/HandSetupView.tsx

@@ -1,353 +0,0 @@
-import { useEffect, useState } from "react";
-import { ArrowUpRight, Download, Hand, Upload } from "lucide-react";
-import "./hands.css";
-
-type RunConfig = {
-  input_root: string;
-  output_root: string;
-  compute: string;
-  planning_fps: number | null;
-  backend: Record<string, string | number | boolean>;
-  runtime: {
-    backend: string;
-    workers: number;
-    hf_jobs: Record<string, unknown>;
-    [key: string]: unknown;
-  };
-  [key: string]: unknown;
-};
-type Catalog = {
-  profiles: {
-    key: string;
-    name: string;
-    reference_fps: number | null;
-    hf_flavor: string | null;
-  }[];
-  config_template: RunConfig;
-  reference_url: string;
-  lerobot_revision: string;
-  licenses: { mano: string; models: string };
-};
-
-export default function HandSetupView() {
-  const [catalog, setCatalog] = useState<Catalog | null>(null);
-  const [imported, setImported] = useState<RunConfig | null>(null);
-  const [input, setInput] = useState("raw-human-pov");
-  const [output, setOutput] = useState("annotated-human-pov");
-  const [compute, setCompute] = useState("rtx-5090");
-  const [backend, setBackend] = useState("local");
-  const [workers, setWorkers] = useState(1);
-  const [planning, setPlanning] = useState("");
-  const [accepted, setAccepted] = useState(false);
-  const [error, setError] = useState("");
-  const [saved, setSaved] = useState(false);
-  useEffect(
-    () => setSaved(false),
-    [compute, backend, workers, planning, accepted, imported, input, output],
-  );
-  useEffect(() => {
-    fetch("/api/hand-pipeline")
-      .then((response) => {
-        if (!response.ok)
-          throw new Error(
-            "The hand pipeline catalog is unavailable. Please refresh.",
-          );
-        return response.json();
-      })
-      .then(setCatalog)
-      .catch((e) => setError(e.message));
-  }, []);
-  const selected = catalog?.profiles.find((p) => p.key === compute);
-  const fps = planning ? Number(planning) : selected?.reference_fps;
-  const valid =
-    !!catalog &&
-    !!input.trim() &&
-    !!output.trim() &&
-    input !== output &&
-    Number.isInteger(workers) &&
-    workers >= 1 &&
-    workers <= 8 &&
-    (!planning ||
-      (Number.isFinite(Number(planning)) && Number(planning) > 0)) &&
-    !(backend === "hf_jobs" && !selected?.hf_flavor);
-
-  async function importConfig(file: File | undefined) {
-    if (!file) return;
-    try {
-      if (file.size > 1_000_000)
-        throw new Error("Choose a configuration file smaller than 1 MB.");
-      const value = JSON.parse(await file.text());
-      if (
-        !value ||
-        typeof value !== "object" ||
-        !value.backend ||
-        typeof value.backend !== "object" ||
-        Array.isArray(value.backend) ||
-        !value.runtime ||
-        typeof value.runtime !== "object" ||
-        Array.isArray(value.runtime)
-      )
-        throw new Error(
-          "Choose a hand pipeline JSON with backend and runtime settings.",
-        );
-      setImported(value);
-      setError("");
-      setSaved(false);
-    } catch (e) {
-      setError((e as Error).message);
-    }
-  }
-  function download() {
-    if (!valid || !catalog) return;
-    const base = imported || catalog.config_template;
-    const config = {
-      ...base,
-      input_root: input.trim(),
-      output_root: output.trim(),
-      compute,
-      planning_fps: planning ? Number(planning) : null,
-      backend: { ...base.backend, research_terms_accepted: accepted },
-      runtime: {
-        ...base.runtime,
-        backend,
-        workers,
-        hf_jobs: {
-          ...base.runtime.hf_jobs,
-          code_repository: "https://github.com/pkooij/lerobot.git",
-          code_revision: catalog.lerobot_revision,
-        },
-      },
-    };
-    const url = URL.createObjectURL(
-      new Blob([JSON.stringify(config, null, 2) + "\n"], {
-        type: "application/json",
-      }),
-    );
-    const link = document.createElement("a");
-    link.href = url;
-    link.download = "hands.json";
-    link.click();
-    setTimeout(() => URL.revokeObjectURL(url), 1000);
-    setSaved(true);
-  }
-  return (
-    <div className="wizard hand-setup">
-      <div className="page-heading">
-        <div>
-          <h1>Human POV hands</h1>
-          <p>
-            Turn calibrated human recordings into 3D hands and LeRobot action
-            episodes.
-          </p>
-        </div>
-      </div>
-      <div className="step-card">
-        <h2>
-          <Hand size={20} /> Research pipeline
-        </h2>
-        <p>
-          DROID-W camera tracking · calibrated HaWoR / MANO · 102D wrist and
-          finger actions.
-        </p>
-        <p className="field-help">
-          Only continuous segments with both hands directly observed are
-          exported. Missing hands and short segments are reported as rejected.
-        </p>
-        <div className="hand-fields">
-          <label>
-            Input folder
-            <input
-              value={input}
-              onChange={(e) => {
-                setInput(e.target.value);
-                setSaved(false);
-              }}
-            />
-            <small>
-              MP4s with matching .intrinsics.json calibration files.
-            </small>
-          </label>
-          <label>
-            Output folder
-            <input
-              value={output}
-              onChange={(e) => {
-                setOutput(e.target.value);
-                setSaved(false);
-              }}
-            />
-          </label>
-        </div>
-      </div>
-      <div className="step-card">
-        <h2>Compute</h2>
-        <div className="hand-fields">
-          <label>
-            <span>Run on</span>
-            <select
-              value={backend}
-              onChange={(e) => {
-                setBackend(e.target.value);
-                if (e.target.value === "hf_jobs" && !selected?.hf_flavor) {
-                  setCompute("h200");
-                  setPlanning("");
-                }
-              }}
-            >
-              <option value="local">Your GPU machine</option>
-              <option value="slurm">Your Slurm cluster</option>
-              <option value="hf_jobs">Your Hugging Face Jobs image</option>
-            </select>
-          </label>
-          <label>
-            <span>GPU</span>
-            <select
-              value={compute}
-              onChange={(e) => {
-                setCompute(e.target.value);
-                setPlanning("");
-              }}
-            >
-              {(
-                catalog?.profiles || [
-                  { key: "rtx-5090", name: "RTX 5090", hf_flavor: null },
-                ]
-              ).map((p) => (
-                <option
-                  value={p.key}
-                  key={p.key}
-                  disabled={backend === "hf_jobs" && !p.hf_flavor}
-                >
-                  {p.name}
-                </option>
-              ))}
-            </select>
-          </label>
-          <label>
-            GPU workers
-            <input
-              type="number"
-              min={1}
-              max={8}
-              value={workers}
-              onChange={(e) => setWorkers(Number(e.target.value))}
-            />
-            <small>
-              One GPU per worker. Use enough video groups to keep them busy.
-            </small>
-          </label>
-          <label>
-            Planning FPS per GPU
-            <input
-              type="number"
-              min={0.1}
-              step="any"
-              value={planning}
-              placeholder={
-                selected?.reference_fps
-                  ? String(selected.reference_fps)
-                  : "Enter an assumption"
-              }
-              onChange={(e) => setPlanning(e.target.value)}
-            />
-          </label>
-        </div>
-        <div className="hand-speed" aria-live="polite">
-          <strong>
-            {fps && Number.isFinite(fps) && fps > 0
-              ? `~${fps} input FPS per GPU`
-              : "GPU throughput has not been measured yet"}
-          </strong>
-          <p>
-            {planning
-              ? "Your planning assumption; not a measured benchmark."
-              : selected?.reference_fps
-                ? "Historical full-pipeline result on RTX 5090: 1,500 frames across ten short HOT3D clips, including model startup and export."
-                : "The measured reference is ~53.9 input FPS on RTX 5090. Enter an assumption for this GPU to estimate runtime."}
-          </p>
-          <p>
-            Source video FPS is preserved. Processing speed varies with
-            resolution, clip length, visibility, CPU and storage. First-time
-            setup, transfer and queue time are additional.
-          </p>
-          {catalog && (
-            <a href={catalog.reference_url} target="_blank" rel="noreferrer">
-              Benchmark and recipe <ArrowUpRight size={13} />
-            </a>
-          )}
-        </div>
-        {backend === "hf_jobs" && (
-          <p className="field-help">
-            Requires a private, authorized runtime image pinned by digest and a
-            remote artifact store in your imported configuration. RTX 5090 is a
-            local profile.
-          </p>
-        )}
-      </div>
-      <div className="step-card">
-        <h2>Your research runtime</h2>
-        <p>
-          Run with your licensed model assets and calibrated camera setup. Model
-          files are supplied by your runtime.
-        </p>
-        <label className="hand-import">
-          <Upload size={16} /> Import runtime configuration
-          <input
-            type="file"
-            accept="application/json,.json"
-            onChange={(e) => importConfig(e.target.files?.[0])}
-          />
-        </label>
-        <p className="field-help">
-          {imported
-            ? "Runtime settings imported locally in this browser."
-            : "Without an import, the download is a setup template. Fill in backend paths, asset hashes and scheduler settings before running."}
-        </p>
-        <label className="hand-consent">
-          <input
-            type="checkbox"
-            checked={accepted}
-            onChange={(e) => setAccepted(e.target.checked)}
-          />
-          <span>
-            I have the required MANO and upstream model rights for
-            non-commercial research.
-          </span>
-        </label>
-        {catalog && (
-          <p className="field-help">
-            <a href={catalog.licenses.mano} target="_blank" rel="noreferrer">
-              MANO terms
-            </a>{" "}
-            ·{" "}
-            <a href={catalog.licenses.models} target="_blank" rel="noreferrer">
-              Other model terms
-            </a>
-            . A research acknowledgment does not grant redistribution or
-            commercial rights.
-          </p>
-        )}
-      </div>
-      {error && (
-        <p className="message error" role="alert">
-          {error}
-        </p>
-      )}
-      <div className="hand-finish">
-        <button className="button primary" onClick={download} disabled={!valid}>
-          <Download size={16} /> Download configuration
-        </button>
-        <p>
-          {saved
-            ? "Saved hands.json. "
-            : "Run the configuration on your provisioned compute: "}
-          <code>uv run lerobot-annotate-hands --config hands.json</code>
-        </p>
-        <p className="field-help">
-          Downloading the configuration starts no compute. Preview with{" "}
-          <code>--estimate</code> before running; HF Jobs execution is billable.
-        </p>
-      </div>
-    </div>
-  );
-}

frontend/src/hands.css

@@ -1,87 +0,0 @@
-.hand-setup .step-card {
-  margin-bottom: 20px;
-}
-.hand-setup h2 {
-  display: flex;
-  align-items: center;
-  gap: 9px;
-  font-size: 16px;
-  margin: 0 0 14px;
-}
-.hand-setup p {
-  color: var(--fg-soft);
-  line-height: 1.6;
-  font-size: 14px;
-}
-.hand-fields {
-  display: grid;
-  grid-template-columns: repeat(2, minmax(0, 1fr));
-  gap: 18px;
-  margin-top: 22px;
-}
-.hand-fields label {
-  display: flex;
-  flex-direction: column;
-  gap: 9px;
-  font-size: 13px;
-  color: var(--fg-soft);
-}
-.hand-fields input,
-.hand-fields select {
-  width: 100%;
-  padding: 11px;
-  border: 1px solid var(--line-strong);
-  border-radius: 6px;
-  background: var(--surface-0);
-  color: var(--fg);
-}
-.hand-setup small {
-  color: var(--fg-muted);
-  line-height: 1.5;
-}
-.hand-speed {
-  border-top: 1px solid var(--line);
-  margin-top: 24px;
-  padding-top: 22px;
-}
-.hand-speed strong {
-  font-size: 20px;
-  font-weight: 500;
-}
-.hand-setup a {
-  text-decoration: underline;
-  text-underline-offset: 3px;
-}
-.hand-import {
-  display: flex;
-  align-items: center;
-  gap: 9px;
-  font-size: 13px;
-  flex-wrap: wrap;
-}
-.hand-import input {
-  max-width: 100%;
-}
-.hand-consent {
-  display: flex;
-  gap: 10px;
-  align-items: flex-start;
-  margin-top: 24px;
-  font-size: 13px;
-  line-height: 1.6;
-}
-.hand-consent input {
-  margin-top: 4px;
-}
-.hand-finish {
-  padding-bottom: 30px;
-}
-.hand-finish code {
-  overflow-wrap: anywhere;
-  font-size: 12px;
-}
-@media (max-width: 600px) {
-  .hand-fields {
-    grid-template-columns: 1fr;
-  }
-}

frontend/src/main.tsx

@@ -16,6 +16,7 @@ import {
   FolderOpen,
   Globe,
   HelpCircle,
+  Hand,
   Layers3,
   ListChecks,
   LoaderCircle,
@@ -38,7 +39,11 @@ import "./style.css";
 import "./wizard.css";
 import AnnotationExamples from "./AnnotationExamples";
 import JobsView, { type Job, type Progress } from "./JobsView";
-import HandSetupView from "./HandSetupView";
+import HandActionSettings, {
+  defaultHands,
+  type HandOptions,
+  type HandCatalog,
+} from "./HandActionSettings";
 
 type Feature =
   | "Subtasks"
@@ -47,7 +52,9 @@ type Feature =
   | "Task rephrasings"
   | "Interjections & speech"
   | "Visual questions"
-  | "Human demo videos";
+  | "Human demo videos"
+  | "Hand actions";
+const HANDS: Feature = "Hand actions";
 const HUMAN_VIDEOS: Feature = "Human demo videos";
 type Options = {
   model: string;
@@ -63,6 +70,7 @@ type Options = {
   prompts: Record<string, string>;
   human_video_segments: number;
   human_video_parallel: number;
+  hand_actions: HandOptions;
 };
 type PromptTemplate = {
   key: string;
@@ -122,6 +130,12 @@ type Quote = {
     high: number;
     inference: number;
     helper_required?: boolean;
+    compute_name?: string;
+    hand_timing?: {
+      planning_fps: number | null;
+      seconds: number | null;
+      basis: string;
+    };
     media?: number;
     media_max?: number;
     videos?: number;
@@ -170,6 +184,7 @@ const defaults: Options = {
   prompts: {},
   human_video_segments: 0,
   human_video_parallel: 8,
+  hand_actions: defaultHands,
 };
 const definitions: {
   name: Feature;
@@ -242,6 +257,12 @@ function initial() {
     return {
       ...defaults,
       ...JSON.parse(localStorage.getItem("studio-draft") || "{}"),
+      hand_actions: {
+        ...defaultHands,
+        ...JSON.parse(localStorage.getItem("studio-draft") || "{}")
+          .hand_actions,
+        research_rights: false,
+      },
       mode: COPY,
     };
   } catch {
@@ -280,13 +301,14 @@ function reviewDraft(): {
 
 function App() {
   const [modelCatalog, setModelCatalog] = useState<ModelCatalog | null>(null);
-  const [page, setPage] = useState<"new" | "jobs" | "guide" | "hands">("new");
+  const [page, setPage] = useState<"new" | "jobs" | "guide">("new");
+  const [handCatalog, setHandCatalog] = useState<HandCatalog | null>(null);
   const [options, setOptions] = useState<Options>(initial);
   const [savedReview] = useState(reviewDraft);
   const [step, setStep] = useState(savedReview.step);
   const [furthestStep, setFurthestStep] = useState(savedReview.step);
   const stepHeading = useRef<HTMLHeadingElement>(null);
-  const stepLabels = ["Dataset", "Annotations", "Model", "Review"];
+  const stepLabels = ["Dataset", "Annotations", "Models & compute", "Review"];
   const datasetReady =
     /^[^/\s]+\/[^/\s]+$/.test(options.dataset.trim()) &&
     Number.isInteger(options.start) &&
@@ -332,9 +354,13 @@ function App() {
     (m) => m.key === options.model,
   );
   const usesFlora = options.captioner === "flora";
+  const usesHands = options.features.includes(HANDS);
+  const handsOnly = usesHands && options.features.length === 1;
   const floraNeedsHelper =
     usesFlora &&
-    (options.features.some((f) => f !== "Subtasks" && f !== "Plans") ||
+    (options.features.some(
+      (f) => f !== "Subtasks" && f !== "Plans" && f !== HANDS,
+    ) ||
       !!options.prompts.plan_render);
 
   async function api<T>(path: string, init: RequestInit = {}): Promise<T> {
@@ -357,6 +383,9 @@ function App() {
     return data;
   }
   useEffect(() => {
+    api<HandCatalog>("/api/hand-pipeline")
+      .then(setHandCatalog)
+      .catch(() => setHandCatalog(null));
     api<Session>("/api/session")
       .then((s) => {
         setSession(s);
@@ -570,7 +599,10 @@ function App() {
       features = ["Subtasks", ...features];
     if (!on && name === "Subtasks")
       features = features.filter((x) => x !== HUMAN_VIDEOS);
-    change({ features });
+    if (on && name === HANDS) {
+      setVisibility("private");
+      change({ features, captioner: "selected-vlm" });
+    } else change({ features });
   }
   async function estimate(advance = false) {
     setLoading(true);
@@ -598,7 +630,10 @@ function App() {
   const canContinueDataset =
     datasetReady && !camerasLoading && cameras.includes(options.camera || "");
   const canEstimate =
-    canContinueDataset && options.features.length > 0 && !!selectedModel;
+    canContinueDataset &&
+    options.features.length > 0 &&
+    !!selectedModel &&
+    (!usesHands || !!handCatalog?.available);
   function goToStep(next: number) {
     setStep(next);
     setFurthestStep((old) => Math.max(old, next));
@@ -732,12 +767,6 @@ function App() {
         </nav>
 
         <nav aria-label="Resources">
-          <button
-            className={page === "hands" ? "active" : ""}
-            onClick={() => navigate("hands")}
-          >
-            <Layers3 size={18} /> Human POV hands
-          </button>
           <a
             href="https://lerobot-visualize-dataset.hf.space"
             target="_blank"
@@ -768,9 +797,7 @@ function App() {
                 ? "New annotation"
                 : page === "jobs"
                   ? "My jobs"
-                  : page === "hands"
-                    ? "Human POV hands"
-                    : "How it works"}
+                  : "How it works"}
             </strong>
           </div>
           <div className="top-actions">
@@ -863,7 +890,7 @@ function App() {
                     [
                       "Choose your dataset",
                       "What would you like to annotate?",
-                      "Choose your model",
+                      "Choose models & compute",
                       "Review your run",
                     ][step]
                   }
@@ -1207,7 +1234,50 @@ function App() {
                   )}
                 </section>
               )}
-              {step === 2 && (
+              {step === 1 && (
+                <section
+                  className={`card step-card ${usesHands ? "selected" : ""}`}
+                  aria-label="Hand action annotations"
+                >
+                  <div className="category-heading">
+                    <h3>
+                      <Hand size={16} /> Hand actions
+                    </h3>
+                    <p className="field-help">
+                      Reconstruct wrists and fingers in existing human POV
+                      recordings.
+                    </p>
+                  </div>
+                  <div className="annotation-choice">
+                    <label>
+                      <input
+                        type="checkbox"
+                        checked={usesHands}
+                        onChange={() => toggle(HANDS)}
+                      />
+                      <span>
+                        <strong>Hand actions</strong>
+                        <small>
+                          102D actions in this dataset, with validity masks.
+                          Keeps every frame.
+                        </small>
+                      </span>
+                    </label>
+                  </div>
+                  <p className="field-help">
+                    Uses HF Jobs GPUs, measured camera calibration and your
+                    licensed MANO assets. Research use only.
+                  </p>
+                </section>
+              )}
+              {step === 2 && usesHands && (
+                <HandActionSettings
+                  value={options.hand_actions}
+                  catalog={handCatalog}
+                  onChange={(hand_actions) => change({ hand_actions })}
+                />
+              )}
+              {step === 2 && !handsOnly && (
                 <section
                   className="card step-card model-picker"
                   aria-label="Model settings"
@@ -1417,9 +1487,11 @@ function App() {
                     <div>
                       <span>Model</span>
                       <strong>
-                        {usesFlora
-                          ? "NVIDIA Flora · TempSeg + Captioner · H200 GPU"
-                          : `${selectedModel?.name} · ${selectedModel?.provider_name}`}
+                        {handsOnly
+                          ? "HaWoR / MANO · DROID-W · HF Jobs"
+                          : usesFlora
+                            ? "NVIDIA Flora · TempSeg + Captioner · H200 GPU"
+                            : `${selectedModel?.name} · ${selectedModel?.provider_name}`}
                       </strong>
                       <button
                         className="text-action"
@@ -1458,6 +1530,7 @@ function App() {
                               type="radio"
                               name="visibility"
                               value={value}
+                              disabled={usesHands && value === "public"}
                               checked={visibility === value}
                               onChange={() => {
                                 setVisibility(value);
@@ -1537,6 +1610,14 @@ function App() {
                           <span>–</span>
                           {money(quote.costs.high)}
                         </div>
+                        {quote.costs.hand_timing && (
+                          <p className="field-help">
+                            {quote.costs.hand_timing.seconds
+                              ? `~${duration(quote.costs.hand_timing.seconds)} at your ${quote.costs.hand_timing.planning_fps} input FPS assumption.`
+                              : "HF GPU throughput is unmeasured; no runtime estimate yet."}{" "}
+                            Setup, downloads and dataset copying are additional.
+                          </p>
+                        )}
                         {!!quote.costs.video_minutes &&
                           quote.costs.video_minutes >
                             quote.costs.hours * 60 && (
@@ -1583,9 +1664,10 @@ function App() {
                             )}
                             <div>
                               <span>
-                                {options.captioner === "flora"
-                                  ? "H200 GPU"
-                                  : "CPU"}{" "}
+                                {quote?.costs.compute_name ||
+                                  (options.captioner === "flora"
+                                    ? "H200 GPU"
+                                    : "CPU")}{" "}
                                 job reservation{" "}
                                 <span
                                   className="info-tooltip"
@@ -1758,7 +1840,6 @@ function App() {
               </div>
             </div>
           )}
-          {page === "hands" && <HandSetupView />}
           {page === "jobs" && (
             <JobsView
               jobs={jobs}
@@ -1981,9 +2062,11 @@ function App() {
             <div>
               <dt>Annotation model</dt>
               <dd>
-                {quote?.captioner === "flora"
-                  ? "NVIDIA Flora · TempSeg + Captioner · H200 GPU · noncommercial research"
-                  : `${quote?.model_profile.name} · ${quote?.model_profile.provider_name}`}
+                {handsOnly
+                  ? "HaWoR / MANO · DROID-W · HF Jobs"
+                  : quote?.captioner === "flora"
+                    ? "NVIDIA Flora · TempSeg + Captioner · H200 GPU · noncommercial research"
+                    : `${quote?.model_profile.name} · ${quote?.model_profile.provider_name}`}
               </dd>
             </div>
             {quote?.costs.helper_required && (

frontend/src/wizard.css

@@ -502,3 +502,35 @@
     margin-top: 2px;
   }
 }
+
+.hand-settings select,
+.hand-settings textarea,
+.hand-settings > input {
+  width: 100%;
+  border: 1px solid var(--line-strong);
+  border-radius: 7px;
+  background: var(--surface-0);
+  color: var(--fg);
+  padding: 10px;
+  font: inherit;
+}
+.hand-settings .field-label {
+  margin-top: 20px;
+}
+.hand-settings textarea {
+  font-family: ui-monospace, monospace;
+  font-size: 12px;
+  resize: vertical;
+}
+.hand-consent {
+  display: flex;
+  align-items: flex-start;
+  gap: 10px;
+  margin-top: 20px;
+  font-size: 13px;
+  line-height: 1.5;
+}
+.hand-consent input {
+  flex: none;
+  margin-top: 3px;
+}

hand-pipeline-catalog.json

@@ -1,104 +1,36 @@
 {
   "recipe": "calibrated-droid-hawor-bounded-v1",
   "runtime_revision": "d61017dd3a95c59a51d836aab6f7bac1baee34eb",
-  "reference_url": "https://github.com/pkooij/grounded-data-pipe/blob/d61017dd3a95c59a51d836aab6f7bac1baee34eb/docs/egocentric_bounded_plane.md#fresh-integrated-ten-clip-check",
   "profiles": [
     {
-      "key": "rtx-5090",
-      "name": "RTX 5090",
-      "hf_flavor": null,
-      "reference_fps": 53.9
-    },
-    {
-      "key": "a100-80gb",
+      "key": "a100-large",
       "name": "A100 80 GB",
-      "hf_flavor": "a100-large",
-      "reference_fps": null
-    },
-    {
-      "key": "h100",
-      "name": "H100 (local / Slurm)",
-      "hf_flavor": null,
       "reference_fps": null
     },
     {
       "key": "rtx-pro-6000",
-      "name": "RTX PRO 6000",
-      "hf_flavor": "rtx-pro-6000",
+      "name": "RTX PRO 6000 96 GB",
       "reference_fps": null
     },
     {
       "key": "h200",
-      "name": "H200",
-      "hf_flavor": "h200",
+      "name": "H200 141 GB",
       "reference_fps": null
     }
   ],
-  "config_template": {
-    "input_root": "raw-egocentric",
-    "output_root": "hand-annotations",
-    "repo_id": "local/human-pov-hands",
-    "compute": "rtx-5090",
+  "reference": {
+    "frames": 1500,
     "planning_fps": null,
-    "videos_per_group": 32,
-    "minimum_segment_frames": 30,
-    "backend": {
-      "root": "",
-      "python": "",
-      "macrodata_python": "",
-      "droid_root": "",
-      "droid_python": "",
-      "droid_moge_checkpoint": "",
-      "droid_model_cache": "",
-      "assets_manifest": "",
-      "assets_sha256": "",
-      "research_terms_accepted": false,
-      "timeout_s": 7200
-    },
-    "runtime": {
-      "backend": "local",
-      "mode": "run",
-      "run_uri": null,
-      "workers": 1,
-      "batch_size": 1,
-      "shard_size": 1,
-      "max_retries": 0,
-      "verify_sources": false,
-      "retain_materialized_outputs": true,
-      "max_parallel_stages": 1,
-      "resource_budget": null,
-      "slurm": {
-        "partition": null,
-        "account": null,
-        "time_limit": "02:00:00",
-        "max_concurrent": null,
-        "script_dir": ".lerobot_slurm",
-        "working_directory": null,
-        "python_executable": "python",
-        "sbatch": "sbatch",
-        "squeue": "squeue",
-        "requeue": true,
-        "command_timeout_s": 30.0
-      },
-      "hf_jobs": {
-        "cpu_image": null,
-        "gpu_image": null,
-        "cpu_flavor": "cpu-upgrade",
-        "gpu_flavor": null,
-        "code_revision": null,
-        "code_repository": "https://github.com/huggingface/lerobot.git",
-        "namespace": null,
-        "bootstrap_packages": [],
-        "timeout": "2h",
-        "max_parallel": 4,
-        "detach": false,
-        "secret_env": []
-      }
-    }
+    "seconds": null,
+    "basis": "HF hardware has not been benchmarked",
+    "reference_fps": 53.9,
+    "reference_compute": "RTX 5090",
+    "reference_scope": "Historical ten-clip calibrated HOT3D check, 1500 source frames; includes model startup, camera/hands and export; excludes installation and transfers.",
+    "caveat": "Different HF GPUs have no measured FPS yet. Queue, installation, downloads and dataset copying are additional."
   },
   "licenses": {
     "mano": "https://mano.is.tue.mpg.de/license.html",
     "models": "https://huggingface.co/macrodata/egovision-safetensors"
   },
-  "lerobot_revision": "16188355a0b012d6769cc69402a87951524cef5f"
+  "lerobot_revision": "5746c3575818706840d32d99ac28e784742f1373"
 }

hand_actions.py

@@ -0,0 +1,67 @@
+"""Hand actions use the normal annotation wizard and HF Jobs lifecycle."""
+
+import json
+import os
+import re
+from dataclasses import asdict
+
+from lerobot.annotations.hands.assets import resolve_mano
+from lerobot.annotations.hands.config import COMPUTE, RECIPE, HandBackendConfig, estimate, public_catalog
+from lerobot.annotations.hands.writer import validate_calibration
+
+HANDS = "Hand actions"
+
+
+def runtime_configuration():
+    """Read trusted operator provisioning; end users choose HF hardware only."""
+    image = os.environ.get("HAND_JOB_IMAGE", "")
+    if not re.fullmatch(r"[A-Za-z0-9./:_-]+@sha256:[a-f0-9]{64}", image):
+        raise ValueError(
+            "Hand actions are not ready to launch: the Space operator must configure its pinned HF Jobs research image"
+        )
+    backend = HandBackendConfig(**json.loads(os.environ.get("HAND_JOB_BACKEND", "{}")))
+    if any(not value for value in asdict(backend).values()):
+        raise ValueError("The Space operator must complete the HF Jobs hand runtime configuration")
+    return {"image": image, "backend": asdict(backend), "recipe": RECIPE}
+
+
+def catalog():
+    """Expose readiness and hardware without revealing operator paths."""
+    result = public_catalog()
+    try:
+        runtime_configuration()
+        result.update(available=True, reason=None)
+    except (ValueError, TypeError, json.JSONDecodeError) as exc:
+        result.update(available=False, reason=str(exc))
+    return result
+
+
+def prepare_hands(selection, camera, options, token):
+    """Bind measured calibration and the user's private assets to the quote."""
+    options = options or {}
+    compute = options.get("compute", "h200")
+    if compute not in COMPUTE:
+        raise ValueError("Choose a supported HF Jobs GPU")
+    feature = selection["info"]["features"][camera]
+    names = feature.get("names") or ["height", "width", "channel"]
+    shape = dict(zip(names, feature["shape"], strict=True))
+    calibration = options.get("calibration") or feature.get("info", {}).get("camera.intrinsics")
+    if isinstance(calibration, str):
+        try:
+            calibration = json.loads(calibration)
+        except json.JSONDecodeError:
+            raise ValueError("Camera calibration must be valid JSON") from None
+    calibration = validate_calibration(calibration or {}, shape["width"], shape["height"])
+    mano = resolve_mano(options.get("mano_repo", ""), token, options.get("research_rights", False))
+    if "annotation.original_action" in selection["info"]["features"]:
+        raise ValueError(
+            "Choose a source dataset that has not already had its actions replaced by hand annotations"
+        )
+    timing = estimate(sum(r["length"] for r in selection["episodes"]), options.get("planning_fps"))
+    return {
+        "compute": compute,
+        "calibration": calibration,
+        "mano": mano,
+        "timing": timing,
+        "runtime": runtime_configuration(),
+    }

pipeline-source.tar.gz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd6c90a12ff34d2099c8ae814527d6691fa29a081b09f47825d60cfead9065de
-size 1857507
+oid sha256:93558e8f7fe094e5c1b678f7d36ecc34fdbbd94cd812536c577aa6e5c7c765cd
+size 1854542

pipeline-version.json

@@ -1,6 +1,6 @@
 {
-  "git_commit": "16188355a0b012d6769cc69402a87951524cef5f",
-  "sha256": "cd6c90a12ff34d2099c8ae814527d6691fa29a081b09f47825d60cfead9065de",
+  "git_commit": "5746c3575818706840d32d99ac28e784742f1373",
+  "sha256": "93558e8f7fe094e5c1b678f7d36ecc34fdbbd94cd812536c577aa6e5c7c765cd",
   "preset": "abc_phases_schema_with_original_window_recovery",
   "preset_date": "2026-09-25",
   "engine": "lerobot.annotations.runtime",

requirements.txt

@@ -2,4 +2,5 @@ fastapi==0.141.1
 uvicorn==0.54.0
 huggingface_hub==1.26.0
 pyarrow==24.0.0
+numpy==2.2.6
 requests==2.34.2

server.py

@@ -17,6 +17,7 @@ import requests
 from fastapi import FastAPI, HTTPException, Request
 from fastapi.responses import FileResponse, JSONResponse, RedirectResponse, Response
 from fastapi.staticfiles import StaticFiles
+from hand_actions import catalog as hand_catalog
 from huggingface_hub import HfApi, hf_hub_download
 from model_catalog import DEFAULT_MODEL, public_models
 from prompt_config import public_catalog, validate_prompts
@@ -40,12 +41,21 @@ class DatasetInput(BaseModel):
     dataset: str = Field(min_length=3, max_length=200)
 
 
+class HandActionInput(BaseModel):
+    compute: Literal["h200", "a100-large", "rtx-pro-6000"] = "h200"
+    planning_fps: float | None = Field(default=None, gt=0, le=100000, allow_inf_nan=False)
+    calibration: str = Field(default="", max_length=12000)
+    mano_repo: str = Field(default="", max_length=200)
+    research_rights: bool = False
+
+
 class EstimateInput(DatasetInput):
     model: str = Field(default=DEFAULT_MODEL, min_length=1, max_length=80)
     captioner: Literal["selected-vlm", "flora"] = "selected-vlm"
     start: int = Field(default=0, ge=0)
     count: int = Field(default=100, ge=0)
-    features: list[str] = Field(min_length=1, max_length=7)
+    features: list[str] = Field(min_length=1, max_length=8)
+    hand_actions: HandActionInput | None = None
     camera: str | None = Field(default=None, max_length=200)
     hz: float = 0.1
     hours: int = 4
@@ -273,6 +283,7 @@ def calculate(options: EstimateInput, request: Request):
             human_video_segments=options.human_video_segments,
             human_video_parallel=options.human_video_parallel,
             captioner=options.captioner,
+            hand_actions=options.hand_actions.model_dump() if options.hand_actions else None,
         )
     except Exception as exc:
         raise HTTPException(400, safe_error(exc)) from None
@@ -504,7 +515,7 @@ def health():
 
 @app.get("/api/hand-pipeline")
 def hand_pipeline_catalog():
-    return json.loads((BASE / "hand-pipeline-catalog.json").read_text())
+    return hand_catalog()
 
 
 @app.get("/api/models")

studio.py

@@ -13,6 +13,8 @@ from pathlib import Path, PurePosixPath
 
 import pyarrow.parquet as pq
 from flora_captioner import FLORA
+from flora_image import validated_worker_image
+from hand_actions import HANDS, prepare_hands, runtime_configuration
 from huggingface_hub import HfApi, hf_hub_download
 from huggingface_hub.utils import validate_repo_id
 from model_catalog import (
@@ -26,6 +28,8 @@ from model_catalog import (
 )
 from prompt_config import TEMPLATES, validate_prompts
 
+from lerobot.annotations.hands.config import COMPUTE
+
 MODEL = model_profile()["route"]
 PROVIDER = model_profile()["provider"]
 FEATURES = {
@@ -37,6 +41,7 @@ FEATURES = {
     "Visual questions": "vqa",
     # Not a language style: videos are saved under annotation_studio/human_videos/.
     "Human demo videos": "human_video",
+    HANDS: "hands",
 }
 HUMAN_VIDEOS = "Human demo videos"
 COPY = "Annotated dataset copy"
@@ -44,7 +49,10 @@ EXPORT = "Annotation files only"  # Historical reports only; new runs always wri
 
 
 def needs_provider(features, prompts=None, captioner="selected-vlm"):
-    """Flora supplies its own boundaries and captions; default plans are deterministic."""
+    """Native hand-only and Flora workflows never need a hosted helper."""
+    features = set(features) - {HANDS}
+    if not features:
+        return False
     return (
         captioner != "flora"
         or bool(set(features) - {"Subtasks", "Plans"})
@@ -200,7 +208,7 @@ def estimate(
     if hours not in (1, 4, 12) or vqa_hz not in (0.1, 0.5, 1.0):
         raise ValueError("Choose a supported timeout and visual-question frequency.")
     n, seconds = len(selection["episodes"]), selection["seconds"]
-    needs_subtasks = bool(set(features) - {"Visual questions"})
+    needs_subtasks = bool(set(features) - {"Visual questions", HANDS})
     if captioner == "flora" and not needs_subtasks:
         raise ValueError("Flora needs an annotation type that uses subtasks.")
     if captioner == "flora" and "plan_subtasks" in prompts:
@@ -359,6 +367,7 @@ def prepare(
     human_video_segments=None,
     human_video_parallel=None,
     captioner="selected-vlm",
+    hand_actions=None,
 ):
     profile = model_profile(model_key)
     prompts = validate_prompts(prompts or {}, features)
@@ -373,7 +382,10 @@ def prepare(
         raise ValueError("Choose a camera in this dataset.")
     if captioner not in {"selected-vlm", "flora"}:
         raise ValueError("Choose a supported subtask captioner.")
-    flavor = FLORA["hardware"] if captioner == "flora" else "cpu-upgrade"
+    hands = prepare_hands(selection, camera, hand_actions, token) if HANDS in features else None
+    if hands and captioner == "flora":
+        raise ValueError("Run Hand actions and Flora separately so each model has the GPU memory it requires")
+    flavor = hands["compute"] if hands else FLORA["hardware"] if captioner == "flora" else "cpu-upgrade"
     hardware = next(h for h in HfApi(token=token or False).list_jobs_hardware() if h.name == flavor)
     if hardware.unit_label != "minute":
         raise ValueError("HF changed its billing units; refresh the Space before launching.")
@@ -389,6 +401,8 @@ def prepare(
         human_video_parallel,
         captioner,
     )
+    if hands:
+        costs.update(compute_flavor=flavor, compute_name=COMPUTE[flavor], hand_timing=hands["timing"])
     return {
         "selection": selection,
         "features": list(features),
@@ -400,6 +414,7 @@ def prepare(
         "model_profile": profile,
         "captioner": captioner,
         "captioner_profile": dict(FLORA) if captioner == "flora" else None,
+        "hand_actions": hands,
         "human_video": {
             # 0 = every subtask.
             "segments_per_episode": human_video_segments,
@@ -437,7 +452,7 @@ def provider_live(api, model_id, provider):
     return hosted is not None and hosted.status == "live"
 
 
-def job_command(space, revision, timeout_minutes=None, captioner="selected-vlm"):
+def job_command(space, revision, timeout_minutes=None, captioner="selected-vlm", hands=False):
     # Only trusted deployment values enter shell syntax. User options travel in JSON env.
     validate_repo_id(space)
     if not re.fullmatch(r"[a-f0-9]{40}", revision):
@@ -462,7 +477,7 @@ def job_command(space, revision, timeout_minutes=None, captioner="selected-vlm")
                 "pip install --quiet huggingface_hub==1.26.0",
                 "python -c " + shlex.quote(download),
                 "pip install --quiet torch==2.11.0 torchvision==0.26.0 --index-url https://download.pytorch.org/whl/"
-                + ("cu128" if captioner == "flora" else "cpu"),
+                + ("cu128" if captioner == "flora" or hands else "cpu"),
                 "pip install --quiet -r /workspace/app/worker-requirements.txt",
                 "pip install --quiet -r /workspace/app/flora-requirements.txt"
                 if captioner == "flora"
@@ -488,6 +503,11 @@ def launch(
 ):
     if visibility not in ("private", "public"):
         raise ValueError("Choose private or public output visibility.")
+    hands = plan.get("hand_actions") if plan else None
+    if hands and visibility != "private":
+        raise ValueError("Hand research outputs must be private")
+    if hands and hands["runtime"] != runtime_configuration():
+        raise ValueError("The hand runtime changed; calculate a fresh estimate")
     if timeout_minutes is not None and (
         type(timeout_minutes) is not int or not 1 <= timeout_minutes <= plan["costs"]["hours"] * 60
     ):
@@ -528,7 +548,7 @@ def launch(
         raise ValueError("Choose a supported subtask captioner.")
     if captioner == "flora" and plan.get("captioner_profile") != FLORA:
         raise ValueError("Flora configuration changed. Calculate a fresh estimate.")
-    flavor = FLORA["hardware"] if captioner == "flora" else "cpu-upgrade"
+    flavor = hands["compute"] if hands else FLORA["hardware"] if captioner == "flora" else "cpu-upgrade"
     hardware = next(h for h in api.list_jobs_hardware() if h.name == flavor)
     if hardware.unit_label != "minute" or hardware.unit_cost_usd > plan["costs"]["cpu_rate"]:
         raise ValueError("Compute pricing changed. Calculate a fresh estimate.")
@@ -590,25 +610,50 @@ def launch(
             "count": plan["selection"]["count"],
             "budget": budget,
             "inference_budget": budget - plan["costs"]["cpu_max"],
-            "model": profile["route"],
+            "model": "HaWoR/MANO + DROID-W" if set(plan["features"]) == {HANDS} else profile["route"],
             "provider_required": uses_provider,
-            "subtask_models": [FLORA["segmenter_id"], FLORA["model_id"]]
+            "subtask_models": []
+            if set(plan["features"]) == {HANDS}
+            else [FLORA["segmenter_id"], FLORA["model_id"]]
             if captioner == "flora"
             else [profile["model_id"]],
             "helper_model": profile["route"] if captioner == "flora" and uses_provider else None,
             "model_profile": profile,
             "generation_options": profile["generation_options"],
             "request_concurrency": profile["concurrency"],
-            "episode_parallelism": 1 if captioner == "flora" else profile["concurrency"],
+            "episode_parallelism": 1 if captioner == "flora" or hands else profile["concurrency"],
             "space": space,
             "space_revision": revision,
             "owner": username,
             "runtime_limit_minutes": timeout_minutes,
         }
     )
+    image, command = "python:3.12-slim", job_command(space, revision, timeout_minutes, captioner)
+    environment = {"ANNOTATION_MANIFEST": json.dumps(manifest)}
+    accelerated = validated_worker_image(captioner, plan["features"])
+    if accelerated:
+        image_info = api.space_info(accelerated["repo"])
+        if image_info.sha != accelerated["revision"]:
+            raise ValueError("Accelerated Flora image changed; wait for a validated release")
+        image = f"hf.co/spaces/{accelerated['repo']}"
+        command = ["python", "/workspace/app/flora_speed_entry.py"]
+        manifest["worker_image"] = {key: accelerated[key] for key in ("repo", "revision")}
+        environment.update(
+            ANNOTATION_MANIFEST=json.dumps(manifest),
+            FLORA_EXECUTION="accelerated",
+            FLORA_BATCH_SIZE=str(accelerated.get("batch_size", 4)),
+            FLORA_ENABLE_INFERENCE="1",
+            FLORA_EXPECTED_CODE=json.dumps(accelerated["expected_code"]),
+            FLORA_RUNTIME_SECONDS=str(
+                int(timeout_minutes * 60 if timeout_minutes else plan["costs"]["hours"] * 3600)
+            ),
+        )
+    if hands:
+        image = hands["runtime"]["image"]
+        command = job_command(space, revision, timeout_minutes, captioner, hands=True)
     job = api.run_job(
-        image="python:3.12-slim",
-        command=job_command(space, revision, timeout_minutes, captioner),
+        image=image,
+        command=command,
         flavor=flavor,
         timeout=f"{timeout_minutes}m" if timeout_minutes else f"{plan['costs']['hours']}h",
         namespace=username,
@@ -618,7 +663,7 @@ def launch(
             "app": "lerobot-annotation-studio",
             "annotation_output": output.split("/", 1)[1],
         },
-        env={"ANNOTATION_MANIFEST": json.dumps(manifest)},
+        env=environment,
         secrets={"HF_TOKEN": token},
     )
     return {"id": job.id, "owner": username, "output": output}

studio_runtime.py

@@ -72,11 +72,42 @@ class StudioStage:
             else:
                 artifact = Artifact(**item.payload["upstream"])
                 if not context.store.verify(artifact):
-                    raise ValueError("Language checkpoint checksum mismatch")
+                    raise ValueError("Upstream checkpoint checksum mismatch")
                 bundle = context.store.download(artifact, directory / "language.tar.gz")
                 unpack(bundle, directory)
                 bundle.unlink()
                 result = json.loads((directory / "result.json").read_text())
+            if self.phase == "hands":
+                import pyarrow.parquet as pq
+                from subset import trim_video
+
+                from lerobot.annotations.hands.writer import attach_actions
+
+                if worker.HAND_ANNOTATOR is None:
+                    raise RuntimeError("The HF Jobs hand worker was not initialized")
+                native = directory / "hand-native"
+                native.mkdir(parents=True, exist_ok=True)
+                video = native / "episode.mp4"
+                trim_video(
+                    root / worker.video_path(row, info, manifest["camera"]),
+                    video,
+                    float(row[f"videos/{manifest['camera']}/from_timestamp"]),
+                    int(row["length"]),
+                    float(info["fps"]),
+                    lossless=True,
+                )
+                native_report = worker.HAND_ANNOTATOR.annotate(
+                    video, manifest["hand_actions"]["calibration"], native / "work"
+                )
+                table = pq.read_table(
+                    record.data_path, filters=[("episode_index", "=", record.episode_index)]
+                )
+                table, summary = attach_actions(table, native_report, directory=native, fps=info["fps"])
+                pq.write_table(table, directory / "hand_actions.parquet")
+                result["hand_actions"] = summary
+                # Model assets and RGB/prediction caches never enter dataset uploads or checkpoints.
+                shutil.rmtree(native)
+            elif self.phase == "human_video":
                 staging = EpisodeStaging(staging_dir, record.episode_index)
                 frames = worker.SourceFrames(row, info, root, manifest["camera"])
                 full = worker.SourceFrames(row, info, root, manifest["camera"], max_size=None)
@@ -136,7 +167,7 @@ class EpisodeSource:
         payload = {}
         identity = dict(self.identity)
         if upstream:
-            plan, _ = upstream["language"]
+            plan, _ = next(iter(upstream.values()))
             _, result = next(accepted_items(store, plan))
             artifact = result.artifacts[0]
             payload["upstream"] = asdict(artifact)
@@ -162,6 +193,7 @@ def run_episode(record, row, info, root, manifest, provider, staging_dir, media=
                 "captioner",
                 "captioner_profile",
                 "flora_execution",
+                "hand_actions",
             )
         },
         "code": {
@@ -187,6 +219,15 @@ def run_episode(record, row, info, root, manifest, provider, staging_dir, media=
                 ("language",),
             )
         )
+    if manifest.get("hand_actions"):
+        stages.append(
+            StageConfig(
+                "hands",
+                "studio_runtime:StudioStage",
+                {"handle": handle, "phase": "hands", "scientific": scientific},
+                (stages[-1].id,),
+            )
+        )
     # Existing Studio threads own concurrency; one in-process runtime worker
     # shares the same request budget. Accepted media is never automatically rebilled.
     runtime = RuntimeConfig(
@@ -205,6 +246,10 @@ def run_episode(record, row, info, root, manifest, provider, staging_dir, media=
         bundle = store.download(result.artifacts[0], directory / "result.tar.gz")
         unpack(bundle, directory)
         result = json.loads((directory / "result.json").read_text())
+        if (directory / "hand_actions.parquet").exists():
+            target = staging_dir.parent / "hand_actions" / f"episode_{record.episode_index:06d}.parquet"
+            target.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copyfile(directory / "hand_actions.parquet", target)
         if (directory / "staging").exists():
             shutil.rmtree(staging_dir / f"episode_{record.episode_index:06d}", ignore_errors=True)
             shutil.copytree(directory / "staging", staging_dir, dirs_exist_ok=True)

subset.py

@@ -1,6 +1,7 @@
 """Build a self-contained v3 dataset from completed annotations, one episode at a time."""
 
 import json
+from concurrent.futures import ThreadPoolExecutor
 from copy import deepcopy
 from fractions import Fraction
 from pathlib import Path
@@ -21,16 +22,20 @@ VIDEO_PATH = "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.m
 EPISODES_PATH = "meta/episodes/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet"
 
 
-def trim_video(source, destination, offset, length, fps):
+def trim_video(source, destination, offset, length, fps, *, lossless=False):
     """Seek to an episode and encode exactly its frames, with timestamps starting at zero."""
     destination.parent.mkdir(parents=True, exist_ok=True)
     rate = Fraction(str(fps)).limit_denominator(100000)
     with av.open(str(source)) as src, av.open(str(destination), "w") as dst:
         stream = src.streams.video[0]
         stream.codec_context.thread_count = 2
-        output = dst.add_stream("libx264", rate=rate, options={"crf": "18", "preset": "fast", "g": "30"})
+        output = dst.add_stream(
+            "libx264rgb" if lossless else "libx264",
+            rate=rate,
+            options={"crf": "0" if lossless else "18", "preset": "fast", "g": "30"},
+        )
         output.width, output.height = stream.codec_context.width, stream.codec_context.height
-        output.pix_fmt = "yuv420p"
+        output.pix_fmt = "rgb24" if lossless else "yuv420p"
         output.codec_context.thread_count = 2
         output.time_base = 1 / rate
         src.seek(int(offset / stream.time_base), stream=stream, backward=True)
@@ -43,7 +48,7 @@ def trim_video(source, destination, offset, length, fps):
                 continue
             if abs(timestamp - (offset + count / fps)) > 0.51 / fps:
                 raise ValueError("Source video does not match the episode's frame timestamps.")
-            encoded = frame.reformat(format="yuv420p")
+            encoded = frame.reformat(format=output.pix_fmt)
             encoded.pts, encoded.time_base = count, 1 / rate
             for packet in output.encode(encoded):
                 dst.mux(packet)
@@ -63,7 +68,10 @@ def trim_video(source, destination, offset, length, fps):
 class EpisodeSubset:
     """Only completed episodes enter this writer. Upload each returned file set atomically."""
 
-    def __init__(self, source_root, root, source_ids):
+    def __init__(self, source_root, root, source_ids, camera_workers=1):
+        if camera_workers not in (1, 2, 3):
+            raise ValueError("Export supports one to three camera workers")
+        self.camera_workers = camera_workers
         self.source_root, self.root = Path(source_root), Path(root)
         self.root.mkdir(parents=True, exist_ok=True)
         self.info = json.loads((self.source_root / "meta/info.json").read_text())
@@ -95,7 +103,7 @@ class EpisodeSubset:
                 }
             )
 
-    def add(self, source_id, table):
+    def add(self, source_id, table, *, hand_features=None):
         """Materialize an annotated episode, retaining all its cameras and existing other styles."""
         if any(m["source_episode_index"] == source_id for m in self.mapping):
             raise ValueError("Episode was already added to the subset.")
@@ -104,6 +112,25 @@ class EpisodeSubset:
         length = int(row["length"])
         if len(table) != length or table["frame_index"].to_pylist() != list(range(length)):
             raise ValueError("Episode data length or frame order does not match its metadata.")
+        if hand_features:
+            description = {
+                "coordinates": "fixed segment-start FLU (X forward, Y left, Z up)",
+                "wrist_position_units": "inferred metres; monocular metric scale is model-dependent",
+                "rotation_units": "axis-angle radians",
+                "validity_feature": "annotation.hand_action_valid",
+                "observed_training_feature": "annotation.hand_action_observed_valid",
+                "infilling_feature": "annotation.hand_infilled",
+                "reference_feature": "annotation.hand_reference_frame",
+                "invalid_action": "zero placeholder; must be masked during training",
+                "minimum_segment_frames": 30,
+            }
+            for name, feature in hand_features.items():
+                self.info["features"][name] = deepcopy(feature)
+            self.info["features"]["action"]["info"] = description
+            if "action" in self.source_info["features"]:
+                self.info["features"]["annotation.original_action"] = deepcopy(
+                    self.source_info["features"]["action"]
+                )
         new_id = len(self.mapping)
         chunk, file = divmod(new_id, 1000)
         changes = {
@@ -136,7 +163,8 @@ class EpisodeSubset:
             "meta/episodes/chunk_index": chunk,
             "meta/episodes/file_index": file,
         }
-        for camera in self.cameras:
+
+        def trim_camera(camera):
             source_video = self.source_root / relative_path(
                 self.source_info["video_path"].format(
                     video_key=camera,
@@ -154,6 +182,14 @@ class EpisodeSubset:
                 length,
                 float(self.info["fps"]),
             )
+            return target
+
+        if self.camera_workers > 1:
+            with ThreadPoolExecutor(max_workers=min(self.camera_workers, len(self.cameras))) as pool:
+                targets = list(pool.map(trim_camera, self.cameras))
+        else:
+            targets = [trim_camera(camera) for camera in self.cameras]
+        for camera, target in zip(self.cameras, targets, strict=True):
             paths.append(target)
             episode.update(
                 {
@@ -176,6 +212,11 @@ class EpisodeSubset:
                     episode_stats.setdefault(feature, {})[stat] = np.atleast_1d(np.asarray(value))
         for feature in changes:
             episode_stats[feature] = get_feature_stats(np.asarray(changes[feature]), axis=0, keepdims=True)
+        if hand_features:
+            for feature in [*hand_features, "annotation.original_action"]:
+                if feature in table.column_names:
+                    values = np.asarray(table[feature].to_pylist())
+                    episode_stats[feature] = get_feature_stats(values, axis=0, keepdims=values.ndim == 1)
         for feature, stats in episode_stats.items():
             for stat, value in stats.items():
                 episode[f"stats/{feature}/{stat}"] = value.tolist()

test_flora_acceleration.py

@@ -105,32 +105,61 @@ def test_tiny_hybrid_gdn_model_keeps_requests_independent_in_left_padded_batch()
     assert generate_many(model, processor, inputs, 4, batch_size=4) == single
 
 
-def test_tiny_multimodal_model_keeps_video_features_with_their_own_prompt():
+@pytest.mark.parametrize("batch_size", [2, 8])
+def test_tiny_multimodal_model_keeps_video_features_with_their_own_prompt(batch_size):
     from transformers import Qwen3_5Config, Qwen3_5ForConditionalGeneration
 
     torch.manual_seed(42)
     config = Qwen3_5Config(
-        text_config={"vocab_size": 64, "hidden_size": 32, "intermediate_size": 64,
-                     "num_hidden_layers": 4, "num_attention_heads": 4, "num_key_value_heads": 2,
-                     "head_dim": 8, "linear_num_key_heads": 2, "linear_num_value_heads": 2,
-                     "linear_key_head_dim": 8, "linear_value_head_dim": 8,
-                     "layer_types": ["linear_attention"]*3+["full_attention"],
-                     "pad_token_id":0, "eos_token_id":2,
-                     "rope_parameters":{"rope_type":"default", "rope_theta":10000, "mrope_section":[1,1,2]}},
-        vision_config={"depth":1, "hidden_size":16, "intermediate_size":32, "num_heads":2,
-                       "in_channels":3, "patch_size":2, "temporal_patch_size":2,
-                       "spatial_merge_size":1, "out_hidden_size":32, "num_position_embeddings":64},
-        video_token_id=61, image_token_id=59, vision_start_token_id=60, vision_end_token_id=62,
+        text_config={
+            "vocab_size": 64,
+            "hidden_size": 32,
+            "intermediate_size": 64,
+            "num_hidden_layers": 4,
+            "num_attention_heads": 4,
+            "num_key_value_heads": 2,
+            "head_dim": 8,
+            "linear_num_key_heads": 2,
+            "linear_num_value_heads": 2,
+            "linear_key_head_dim": 8,
+            "linear_value_head_dim": 8,
+            "layer_types": ["linear_attention"] * 3 + ["full_attention"],
+            "pad_token_id": 0,
+            "eos_token_id": 2,
+            "rope_parameters": {"rope_type": "default", "rope_theta": 10000, "mrope_section": [1, 1, 2]},
+        },
+        vision_config={
+            "depth": 1,
+            "hidden_size": 16,
+            "intermediate_size": 32,
+            "num_heads": 2,
+            "in_channels": 3,
+            "patch_size": 2,
+            "temporal_patch_size": 2,
+            "spatial_merge_size": 1,
+            "out_hidden_size": 32,
+            "num_position_embeddings": 64,
+        },
+        video_token_id=61,
+        image_token_id=59,
+        vision_start_token_id=60,
+        vision_end_token_id=62,
     )
     model = Qwen3_5ForConditionalGeneration(config).eval()
     model.generation_config.forced_eos_token_id = 2
     model.generation_config.min_new_tokens = 2
     inputs = []
-    for prefix in ([5,6], [7,8,9]):
-        ids = [*prefix,60,61,61,61,61,62,10]
-        inputs.append({"input_ids":torch.tensor([ids]), "attention_mask":torch.ones(1,len(ids),dtype=torch.long),
-                       "mm_token_type_ids":torch.tensor([[0]*(len(prefix)+1)+[2]*4+[0,0]]),
-                       "pixel_values_videos":torch.randn(4,24), "video_grid_thw":torch.tensor([[1,2,2]])})
+    for prefix in ([5, 6], [7, 8, 9]) * (batch_size // 2):
+        ids = [*prefix, 60, 61, 61, 61, 61, 62, 10]
+        inputs.append(
+            {
+                "input_ids": torch.tensor([ids]),
+                "attention_mask": torch.ones(1, len(ids), dtype=torch.long),
+                "mm_token_type_ids": torch.tensor([[0] * (len(prefix) + 1) + [2] * 4 + [0, 0]]),
+                "pixel_values_videos": torch.randn(4, 24),
+                "video_grid_thw": torch.tensor([[1, 2, 2]]),
+            }
+        )
     processor = SimpleNamespace(tokenizer=Tokenizer())
-    single = [generate_many(model,processor,[item],4,batch_size=1)[0] for item in inputs]
-    assert generate_many(model,processor,inputs,4,batch_size=4) == single
+    single = [generate_many(model, processor, [item], 4, batch_size=1)[0] for item in inputs]
+    assert generate_many(model, processor, inputs, 4, batch_size=batch_size) == single

test_flora_native.py

@@ -159,17 +159,19 @@ def test_resident_mode_rejects_smaller_gpu_before_downloading_weights(monkeypatc
 
 
 def test_acceleration_prefetch_does_not_block_tempseg_and_keeps_native_load(monkeypatch):
-    import huggingface_hub
     from threading import Event
 
     import flora_acceleration
+    import huggingface_hub
 
-    monkeypatch.setattr(huggingface_hub, "snapshot_download",
-                        lambda *a, **k: os.environ["FLORA_NATIVE_TEST_ROOT"])
+    monkeypatch.setattr(
+        huggingface_hub, "snapshot_download", lambda *a, **k: os.environ["FLORA_NATIVE_TEST_ROOT"]
+    )
     monkeypatch.setattr(flora_acceleration, "require_fast_kernels", lambda: {"test": "native"})
     # Import the pinned modules once without a model/weight download.
     reference = FloraCaptioner()
     import batch_utils.models as native
+
     entered, release = Event(), Event()
 
     def snapshot(kind):
@@ -181,6 +183,7 @@ def test_acceleration_prefetch_does_not_block_tempseg_and_keeps_native_load(monk
     engine = FloraCaptioner(execution="accelerated")
     monkeypatch.setattr(engine, "check_resident_capacity", lambda: None)
     loaded = []
+
     class Session:
         def load(self, kind):
             loaded.append(kind)

test_flora_runtime.py

@@ -57,6 +57,69 @@ def test_flora_cannot_be_selected_for_vqa_only():
         studio.estimate(selection(), ["Visual questions"], 0.1, 1, 0.083333, captioner="flora")
 
 
+def test_prebuilt_flora_image_requires_complete_measured_acceptance(tmp_path):
+    from flora_image import IDENTITY_FILES, validated_worker_image
+
+    path = tmp_path / "release.json"
+    assert validated_worker_image("flora", ["Subtasks"], path) is None
+    profile = {
+        "repo": "org/worker",
+        "revision": "a" * 40,
+        "captioner_profile": FLORA,
+        "expected_code": dict.fromkeys(IDENTITY_FILES, "b" * 64),
+        "acceptance": {"episodes": 25, "independent": True, "fused_kernels": True, "end_to_end_seconds": 899},
+    }
+    path.write_text(json.dumps(profile))
+    assert validated_worker_image("flora", ["Subtasks"], path) == profile
+    assert validated_worker_image("flora", ["Subtasks", "Memory"], path) is None
+    for key, value in [
+        ("end_to_end_seconds", 900),
+        ("episodes", 24),
+        ("independent", False),
+        ("fused_kernels", False),
+    ]:
+        altered = json.loads(json.dumps(profile))
+        altered["acceptance"][key] = value
+        path.write_text(json.dumps(altered))
+        with pytest.raises(ValueError, match="acceptance"):
+            validated_worker_image("flora", ["Subtasks"], path)
+
+
+def test_flora_launch_uses_only_validated_image_without_reinstalling(monkeypatch):
+    from flora_image import IDENTITY_FILES
+
+    configured = plan()
+    configured.update(captioner="flora", captioner_profile=dict(FLORA))
+    configured["costs"] = studio.estimate(selection(), ["Subtasks"], 0.1, 1, 0.083333, captioner="flora")
+    api = api_mock()
+    api.list_jobs_hardware.return_value.append(
+        SimpleNamespace(name="h200", unit_label="minute", unit_cost_usd=0.083333)
+    )
+    image = {
+        "repo": "org/worker",
+        "revision": api.space_info.return_value.sha,
+        "expected_code": dict.fromkeys(IDENTITY_FILES, "b" * 64),
+    }
+    monkeypatch.setattr(studio, "validated_worker_image", lambda *a: image)
+    studio.launch(
+        configured,
+        "alice/fast",
+        8,
+        True,
+        "secret",
+        space="finerobotics/annotation-studio",
+        api_factory=lambda **_: api,
+    )
+    arguments = api.run_job.call_args.kwargs
+    assert arguments["image"] == "hf.co/spaces/org/worker"
+    assert arguments["command"] == ["python", "/workspace/app/flora_speed_entry.py"]
+    assert arguments["env"]["FLORA_EXECUTION"] == "accelerated"
+    assert "secret" not in json.dumps(arguments["env"])
+    assert (
+        json.loads(arguments["env"]["ANNOTATION_MANIFEST"])["worker_image"]["revision"] == image["revision"]
+    )
+
+
 def test_runtime_resume_restores_annotations_and_videos_without_provider_calls(
     tmp_path, episode, monkeypatch
 ):

test_hand_actions.py

@@ -0,0 +1,158 @@
+"""Native hand flow contracts on the real episode engine, with no paid compute."""
+
+import json
+import sys
+from pathlib import Path
+from unittest.mock import Mock
+
+import av
+import numpy as np
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent))
+import studio
+import worker
+from hand_actions import HANDS, catalog
+from studio_runtime import run_episode
+
+from lerobot.annotations.steerable_pipeline.reader import EpisodeRecord
+
+
+def test_unconfigured_runtime_cannot_claim_ready(monkeypatch):
+    monkeypatch.delenv("HAND_JOB_IMAGE", raising=False)
+    result = catalog()
+    assert result["available"] is False
+    assert "pinned HF Jobs" in result["reason"]
+    assert "config_template" not in result
+
+
+def test_hand_only_has_no_qwen_cost_or_provider():
+    selection = {"episodes": [{"length": 900}], "seconds": 30, "info": {"fps": 30}}
+    costs = studio.estimate(selection, [HANDS], 0.1, 1, 0.08)
+    assert costs["inference"] == costs["calls"] == 0
+    assert studio.needs_provider([HANDS]) is False
+    assert studio.needs_provider([HANDS, "Subtasks"]) is True
+    assert studio.needs_provider([HANDS, "Subtasks"], captioner="flora") is False
+    assert worker.requested_styles([HANDS]) == set()
+
+
+@pytest.mark.parametrize("observed", [True, False])
+def test_hand_stage_preserves_frames_and_resumes_without_inference(tmp_path, monkeypatch, observed):
+    camera = "observation.images.ego"
+    video = tmp_path / f"videos/{camera}/chunk-000/file-000.mp4"
+    video.parent.mkdir(parents=True)
+    with av.open(str(video), "w") as container:
+        stream = container.add_stream("libx264", rate=10, options={"crf": "0"})
+        stream.width = stream.height = 32
+        stream.pix_fmt = "yuv420p"
+        for i in range(40):
+            frame = av.VideoFrame.from_ndarray(np.full((32, 32, 3), i, dtype=np.uint8), format="rgb24")
+            for packet in stream.encode(frame):
+                container.mux(packet)
+        for packet in stream.encode():
+            container.mux(packet)
+    source = pa.table(
+        {
+            "episode_index": [0] * 40,
+            "frame_index": list(range(40)),
+            "timestamp": pa.array(np.arange(40) / 10, type=pa.float32()),
+            "action": [[float(i), 0.0] for i in range(40)],
+        }
+    )
+    path = tmp_path / "source.parquet"
+    pq.write_table(source, path)
+    record = EpisodeRecord(0, "human task", tuple(np.arange(40) / 10), tuple(range(40)), path, 0, 40)
+    row = {
+        "length": 40,
+        f"videos/{camera}/chunk_index": 0,
+        f"videos/{camera}/file_index": 0,
+        f"videos/{camera}/from_timestamp": 0.0,
+    }
+    info = {"fps": 10, "video_path": "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4"}
+    calls = []
+
+    def native(video, calibration, directory):
+        calls.append(video)
+        with av.open(str(video)) as clip:
+            assert sum(1 for f in clip.decode(video=0)) == 40
+        directory.mkdir(parents=True)
+        segments = []
+        if observed:
+            segment = directory / "segment.npz"
+            np.savez(
+                segment,
+                source_frame_index=np.arange(5, 35),
+                timestamp_s=np.arange(5, 35) / 10,
+                action=np.ones((30, 102)),
+                confidence=np.ones((30, 2)),
+            )
+            segments.append(str(segment))
+        return {
+            "segments": segments,
+            "accepted_frames": 30 if observed else 0,
+            "rectified_frames": 40,
+            "rectified_fps": 10,
+        }
+
+    monkeypatch.setattr(worker, "HAND_ANNOTATOR", type("Native", (), {"annotate": staticmethod(native)})())
+    provider = Mock()
+    provider.generate_json.side_effect = AssertionError("Hand-only must never call a VLM")
+    manifest = {
+        "dataset": "test/source",
+        "dataset_revision": "a" * 40,
+        "features": [HANDS],
+        "camera": camera,
+        "vqa_hz": 0.1,
+        "hand_actions": {"calibration": {"fx": 25}},
+    }
+    staging = tmp_path / "staging"
+    result = run_episode(record, row, info, tmp_path, manifest, provider, staging)
+    assert set(result["runtime"]) == {"language", "hands"}
+    assert result["hand_actions"]["valid_frames"] == (30 if observed else 0)
+    target = tmp_path / "hand_actions/episode_000000.parquet"
+    annotated = pq.read_table(target)
+    assert len(annotated) == len(source) == 40
+    assert annotated["annotation.original_action"].equals(source["action"])
+    assert len(calls) == 1
+    resumed = run_episode(record, row, info, tmp_path, manifest, provider, staging)
+    assert resumed["hand_actions"] == result["hand_actions"] and len(calls) == 1
+    provider.generate_json.assert_not_called()
+    assert not list((staging.parent / "runtime").rglob("MANO*"))
+
+
+def test_launch_uses_only_hf_gpu_image_and_private_dataset(monkeypatch):
+    from test_studio import api_mock, plan
+
+    runtime = {"image": "example/research@sha256:" + "c" * 64, "backend": {}, "recipe": "pinned"}
+    monkeypatch.setattr(studio, "runtime_configuration", lambda: runtime)
+    api = api_mock()
+    api.list_jobs_hardware.return_value = [
+        type("Hardware", (), {"name": "h200", "unit_label": "minute", "unit_cost_usd": 0.08})()
+    ]
+    api.run_job.return_value = type("Job", (), {"id": "only-simulated"})()
+    p = plan()
+    p.update(features=[HANDS], hand_actions={"compute": "h200", "runtime": runtime})
+    p["costs"] = studio.estimate(p["selection"], [HANDS], 0.1, 1, 0.08)
+    studio.launch(
+        p,
+        "alice/hands",
+        10,
+        True,
+        "secret-token",
+        space="finerobotics/annotation-studio",
+        api_factory=lambda **kw: api,
+    )
+    launch = api.run_job.call_args.kwargs
+    assert launch["flavor"] == "h200" and launch["image"] == runtime["image"]
+    assert launch["secrets"] == {"HF_TOKEN": "secret-token"}
+    manifest = json.loads(launch["env"]["ANNOTATION_MANIFEST"])
+    assert manifest["provider_required"] is False and manifest["subtask_models"] == []
+    assert manifest["episode_parallelism"] == 1 and manifest["visibility"] == "private"
+    api.model_info.assert_not_called()
+    with pytest.raises(ValueError, match="private"):
+        studio.launch(
+            p, "alice/public", 10, True, "secret-token", visibility="public", api_factory=lambda **kw: api
+        )
+    assert api.run_job.call_count == 1

test_studio.py

@@ -23,7 +23,13 @@ from lerobot.annotations.steerable_pipeline.vlm_client import StubVlmClient
 
 
 @pytest.mark.parametrize(
-    "features", [list(studio.FEATURES), ["Interjections & speech"], ["Memory"], ["Visual questions"]]
+    "features",
+    [
+        [f for f in studio.FEATURES if f != studio.HANDS],
+        ["Interjections & speech"],
+        ["Memory"],
+        ["Visual questions"],
+    ],
 )
 @pytest.mark.parametrize("custom", [False, True])
 def test_real_video_pipeline_and_checkbox_filtering(tmp_path, features, custom):

test_subset.py

@@ -120,7 +120,8 @@ def shared_source(tmp_path):
     return root, cameras, path
 
 
-def test_successful_episodes_only_all_cameras_and_real_lerobot_load(shared_source, tmp_path):
+@pytest.mark.parametrize("camera_workers", [1, 3])
+def test_successful_episodes_only_all_cameras_and_real_lerobot_load(shared_source, tmp_path, camera_workers):
     root, cameras, data_path = shared_source
     records = [r for r in iter_episodes(root) if r.episode_index != 1]
     staging = tmp_path / "staging"
@@ -141,7 +142,7 @@ def test_successful_episodes_only_all_cameras_and_real_lerobot_load(shared_sourc
     PreservingWriter([0, 2], {"subtask"}).write_all(records, staging, root)
     Executor._ensure_annotation_metadata_in_info(root)
     output = tmp_path / "subset"
-    builder = EpisodeSubset(root, output, [0, 1, 2])
+    builder = EpisodeSubset(root, output, [0, 1, 2], camera_workers=camera_workers)
     table = pq.read_table(data_path)
     builder.add(0, table)
     # Loading a checkpoint works before all completed episodes have been copied.
@@ -422,3 +423,42 @@ def test_dense_frame_sampling_matches_source_pixels_with_one_seek(tmp_path, monk
     assert len(seeks) == 1  # Previously 40 GOP seeks for the same pixels.
     source.frames_at(SimpleNamespace(), [i / 30 for i in reversed(requested)])
     assert len(seeks) == 1
+
+
+def test_hand_actions_load_in_same_dataset_and_keep_unobserved_episode(shared_source, tmp_path):
+    from lerobot.annotations.hands.writer import FEATURES as HAND_FEATURES, attach_actions
+
+    root, cameras, data_path = shared_source
+    builder = EpisodeSubset(root, tmp_path / "annotated", [0, 2])
+    source = pq.read_table(data_path)
+    for episode in (0, 2):
+        table = source.filter(pa.compute.equal(source["episode_index"], episode))
+        # Entirely unobserved episodes remain usable dataset episodes, with explicit masks.
+        table, summary = attach_actions(
+            table,
+            {"rectified_frames": 10, "rectified_fps": 10, "accepted_frames": 0, "segments": []},
+            directory=tmp_path,
+            fps=10,
+        )
+        builder.add(episode, table, hand_features=HAND_FEATURES)
+        assert summary["unmeasured_frames"] == 10
+    result = LeRobotDataset("test/annotated", root=builder.root, video_backend="pyav", token=False)
+    assert len(result) == 20 and result.num_episodes == 2
+    assert tuple(result[0]["action"].shape) == (102,)
+    assert tuple(result[0]["annotation.original_action"].shape) == (2,)
+    assert not result[0]["annotation.hand_action_valid"].any()
+    table = pq.read_table(builder.root / "data")
+    assert (
+        table["annotation.original_action"].to_pylist()
+        == source.filter(pa.compute.is_in(source["episode_index"], value_set=pa.array([0, 2])))[
+            "action"
+        ].to_pylist()
+    )
+    stats = json.loads((builder.root / "meta/stats.json").read_text())
+    assert len(stats["action"]["mean"]) == 102
+    assert len(stats["annotation.original_action"]["mean"]) == 2
+    assert builder.mapping == [
+        {"episode_index": 0, "source_episode_index": 0},
+        {"episode_index": 1, "source_episode_index": 2},
+    ]
+    assert len(list((builder.root / "videos").rglob("*.mp4"))) == 4

worker.py

@@ -17,6 +17,7 @@ from pathlib import Path
 
 import av
 import pyarrow.parquet as pq
+from hand_actions import HANDS
 from huggingface_hub import CommitOperationAdd, HfApi, hf_hub_download, snapshot_download
 from human_video import FalMedia, generate_human_videos
 from prompt_config import validate_prompts
@@ -45,6 +46,7 @@ STUDIO_ACTIVE = ContextVar("studio_active", default=False)
 ORIGINAL_LOAD = plan_module.load_prompt
 ACTION_PROMPT = Path(__file__).with_name("action-phases.txt").read_text()
 FLORA_CAPTIONER = None
+HAND_ANNOTATOR = None
 
 
 class IndependentFloraClient:
@@ -244,7 +246,7 @@ class PreservingWriter(LanguageColumnsWriter):
 
 def requested_styles(features):
     # Human demo videos are media files, not language rows.
-    styles = {FEATURES[f] for f in features if f != HUMAN_VIDEOS}
+    styles = {FEATURES[f] for f in features if f not in {HUMAN_VIDEOS, HANDS}}
     if "Interjections & speech" in features:
         styles.add(None)
     return styles
@@ -289,7 +291,7 @@ def _annotate_episode(record, row, info, root, manifest, provider, staging_dir,
         cfg.derive_task_from_video = "off"
         plan.caption_frames = SourceFrames(row, info, root, manifest["camera"], max_size=None)
         plan.caption_fps = info["fps"]
-    if set(features) - {"Visual questions"}:
+    if set(features) - {"Visual questions", HANDS}:
         plan.run_episode(record, staging)
     if "Interjections & speech" in features:
         module = InterjectionsAndSpeechModule(
@@ -443,7 +445,9 @@ def create_output_repo(api, manifest):
 
 
 def run(manifest, work=Path("/workspace/run")):
-    global FLORA_CAPTIONER
+    global FLORA_CAPTIONER, HAND_ANNOTATOR
+    if (HANDS in manifest["features"]) != bool(manifest.get("hand_actions")):
+        raise ValueError("Hand action selection and pinned runtime configuration differ")
     deadline = float(os.environ.get("ANNOTATION_DEADLINE_EPOCH", "0")) or None
     if manifest.get("captioner") == "flora":
         from flora_captioner import FLORA, FloraCaptioner
@@ -486,7 +490,9 @@ def run(manifest, work=Path("/workspace/run")):
         local_dir=root,
         allow_patterns=["meta/**", "README.md", "LICENSE*", "COPYING*"],
     )
-    subset = EpisodeSubset(root, work / "subset", ids)
+    subset = EpisodeSubset(
+        root, work / "subset", ids, camera_workers=3 if manifest.get("captioner") == "flora" else 1
+    )
     attribution = []
     for file in root.iterdir():
         if file.is_file() and (file.name == "README.md" or file.name.startswith(("LICENSE", "COPYING"))):
@@ -585,6 +591,20 @@ def run(manifest, work=Path("/workspace/run")):
         )
         shards[path].append(row)
     try:
+        if manifest.get("hand_actions"):
+            from lerobot.annotations.hands.assets import download_mano
+            from lerobot.annotations.hands.config import HandBackendConfig
+            from lerobot.annotations.hands.processing import HandAnnotator
+
+            hands = manifest["hand_actions"]
+            receipt = download_mano(hands["mano"], token, work / "private-mano")
+            hands["asset_checksums"] = {
+                side: {key: value for key, value in entry.items() if key != "path"}
+                for side, entry in receipt.items()
+            }
+            HAND_ANNOTATOR = HandAnnotator(
+                HandBackendConfig(**hands["runtime"]["backend"]), receipt, work / "hands-native"
+            )
         for data_path, rows in shards.items():
             files = {data_path} | {video_path(row, info, manifest["camera"]) for row in rows}
             sizes = api.get_paths_info(
@@ -654,7 +674,29 @@ def run(manifest, work=Path("/workspace/run")):
                             local_dir=root,
                         )
                     shard_files.update(extra_files)
-                    new_id, written = subset.add(record.episode_index, annotated_table)
+                    hand_features = None
+                    if manifest.get("hand_actions"):
+                        from lerobot.annotations.hands.writer import FEATURES as HAND_FEATURES
+
+                        hand_features = HAND_FEATURES
+
+                        hand_path = (
+                            staging.parent / "hand_actions" / f"episode_{record.episode_index:06d}.parquet"
+                        )
+                        hand_table = pq.read_table(hand_path)
+                        if not hand_table["frame_index"].equals(annotated_table["frame_index"]):
+                            raise ValueError("Hand actions do not match the exported episode")
+                        for name in [*HAND_FEATURES, "annotation.original_action"]:
+                            if name in hand_table.column_names:
+                                index = annotated_table.schema.get_field_index(name)
+                                annotated_table = (
+                                    annotated_table.set_column(index, name, hand_table[name])
+                                    if index >= 0
+                                    else annotated_table.append_column(name, hand_table[name])
+                                )
+                    new_id, written = subset.add(
+                        record.episode_index, annotated_table, hand_features=hand_features
+                    )
                     extra_operations = []
                     with commit_lock:
                         for entry in human_index:
@@ -787,6 +829,10 @@ def run(manifest, work=Path("/workspace/run")):
         progress.update(status="partial", error_type=type(exc).__name__, worker_finished_at=time.time())
         checkpoint()
         raise RuntimeError("Annotation stopped; check the saved run report for completed episodes.") from None
+    finally:
+        if HAND_ANNOTATOR is not None:
+            HAND_ANNOTATOR.close()
+            HAND_ANNOTATOR = None
     print(f"Result: {progress['status']} — https://huggingface.co/datasets/{output}", flush=True)