Record native hand stage timings and private failure diagnostics

flora-worker-image.json

@@ -30,7 +30,7 @@
     "flora_acceleration.py": "782a49a896bfe90d5ca4853ee963287dd861a87f09e28bc52efaecca3cdaeaa6",
     "flora_batcher.py": "38e03e2032af8d5f3ed34980cc4d89927fe70c9742aa2ce13f4d0ccdd6fba727",
     "flora_speed_entry.py": "192b18d75b9d876c2a65d88de340c8cea7c8da90543d0510b5ee6865d123d524",
-    "studio_runtime.py": "c09750c69802cff6b65f3b2a2a3eaa941dabd7d7d44b35767e2ffef025d6e78d",
+    "studio_runtime.py": "a0a248d2956379478d94f3f22dcf59379f0811b19ff59c8b8caf79b4f6b949cb",
     "pipeline-version.json": "4a33b8084deff000f31d10ab5a5937153625fd87c3982758c874b6075f64e1f2",
     "flora_vision.py": "a5a4231abf7b615164c40986df5f87047f745a23c55d86385d88f2a09e70f1e8"
   },
@@ -81,7 +81,7 @@
       "studio_runtime.py": {
         "non_hand_ast_sha256": "5e54e9154333a1a2949b1bd3fdd2a684633f75ef74878adec55724863eced9a7",
         "accepted_source_sha256": "58ea7ba2e09d102b17c4eb8665dab0ea73442c7e1c63b0ecaf005c9a21e619aa",
-        "integrated_source_sha256": "c09750c69802cff6b65f3b2a2a3eaa941dabd7d7d44b35767e2ffef025d6e78d"
+        "integrated_source_sha256": "a0a248d2956379478d94f3f22dcf59379f0811b19ff59c8b8caf79b4f6b949cb"
       },
       "subset.py": {
         "non_hand_ast_sha256": "7a8c6a1f424438ccca92aebf708850a2ba83d44296af35fea59f5e9178eef1d5",

studio_runtime.py

@@ -78,6 +78,8 @@ class StudioStage:
                 bundle.unlink()
                 result = json.loads((directory / "result.json").read_text())
             if self.phase == "hands":
+                import time
+
                 import pyarrow.parquet as pq
                 from subset import trim_video
 
@@ -88,6 +90,14 @@ class StudioStage:
                 native = directory / "hand-native"
                 native.mkdir(parents=True, exist_ok=True)
                 video = native / "episode.mp4"
+                started = time.perf_counter()
+                print(
+                    "HAND_RUNTIME "
+                    + json.dumps(
+                        {"event": "episode_video_prepare_start", "source_episode": record.episode_index}
+                    ),
+                    flush=True,
+                )
                 trim_video(
                     root / worker.video_path(row, info, manifest["camera"]),
                     video,
@@ -96,14 +106,60 @@ class StudioStage:
                     float(info["fps"]),
                     lossless=True,
                 )
-                native_report = worker.HAND_ANNOTATOR.annotate(
-                    video, manifest["hand_actions"]["calibration"], native / "work",
-                    orientation=manifest["hand_actions"].get("orientation", "sensor")
+                prepared = time.perf_counter()
+                print(
+                    "HAND_RUNTIME "
+                    + json.dumps(
+                        {
+                            "event": "native_annotation_start",
+                            "source_episode": record.episode_index,
+                            "video_prepare_seconds": prepared - started,
+                        }
+                    ),
+                    flush=True,
                 )
+                try:
+                    native_report = worker.HAND_ANNOTATOR.annotate(
+                        video,
+                        manifest["hand_actions"]["calibration"],
+                        native / "work",
+                        orientation=manifest["hand_actions"].get("orientation", "sensor"),
+                    )
+                except Exception:
+                    # Native inference is offline and has no Hub credentials.
+                    import logging
+
+                    logging.exception(
+                        "Native hand annotation failed for source episode %s", record.episode_index
+                    )
+                    raise
+                inferred = time.perf_counter()
                 table = pq.read_table(
                     record.data_path, filters=[("episode_index", "=", record.episode_index)]
                 )
                 table, summary = attach_actions(table, native_report, directory=native, fps=info["fps"])
+                summary["timing_seconds"] = {
+                    "source_video_prepare": prepared - started,
+                    "native_annotation": inferred - prepared,
+                    "native_scope": "intrinsics, rectification, camera, hands and native export; includes coordinator wait and first-use model initialization; excludes source preparation and final dataset video copying",
+                }
+                summary["native_components"] = {
+                    key: native_report[key]
+                    for key in ("camera_statistics", "hand_components")
+                    if key in native_report
+                }
+                print(
+                    "HAND_RUNTIME "
+                    + json.dumps(
+                        {
+                            "event": "native_annotation_finished",
+                            "source_episode": record.episode_index,
+                            "native_seconds": inferred - prepared,
+                            "valid_frames": summary["valid_frames"],
+                        }
+                    ),
+                    flush=True,
+                )
                 pq.write_table(table, directory / "hand_actions.parquet")
                 result["hand_actions"] = summary
                 # Model assets and RGB/prediction caches never enter dataset uploads or checkpoints.