diff --git "a/data.json" "b/data.json" --- "a/data.json" +++ "b/data.json" @@ -1,215 +1,4709 @@ { - "tasks": [ + "title": "Programmable VLA", + "updated_at": "2026-10-28T22:34:51.372621+00:00", + "schema": "programvla/benchmark-presentation/1", + "benchmark_complete": true, + "complete": true, + "variants": [ { - "id": "1", - "native_id": "robotwin/handover-block-aloha-agilex", - "label": "Handover block", - "max_steps": 800, - "seed": 0 + "id": "standalone", + "agent": "VLA", + "label": "Standalone VLA", + "model": "Qwen3-VL-OFT", + "effort": "40k checkpoint", + "provider": "StarVLA" }, { - "id": "2", - "native_id": "robotwin/place-dual-shoes-aloha-agilex", - "label": "Place dual shoes", - "max_steps": 600, - "seed": 0 + "id": "kinex", + "agent": "Kinex", + "label": "Kinex", + "model": "GPT-6 Astra", + "effort": "high", + "provider": "OpenAI Codex" }, { - "id": "3", - "native_id": "robotwin/put-bottles-dustbin-aloha-agilex", - "label": "Put bottles in dustbin", - "max_steps": 1700, - "seed": 0 + "id": "kinex_policy", + "agent": "Kinex", + "label": "Kinex + VLA", + "model": "GPT-6 Astra", + "effort": "high", + "provider": "OpenAI Codex" } ], - "conditions": [ + "families": [ { - "id": "standalone", - "label": "Standalone VLA", - "model": "Qwen3-VL-OFT · 40k" + "id": "task03", + "name": "RoboTwin 2.0", + "tasks": 3 + } + ], + "tasks": [ + { + "key": "task03/01", + "slot": "01", + "family": "task03", + "native_id": "robotwin/handover-block-aloha-agilex", + "native_identity": { + "allow_seed_substitution": false, + "embodiment": "aloha_agilex", + "max_steps": 800, + "task_config": "demo_clean", + "task_name": "handover_block" + }, + "native_instruction": "use the left arm to grasp the red block on the table, handover it to the right arm and place it on the blue pad", + "catalog_instruction": "use the left arm to grasp the red block on the table, hand it over to the right arm, and set it down in the center of the blue pad. Release it with the right gripper open", + "title": "Handover block aloha agilex", + "planned_protocol": { + "seed": 0, + "episodes": 1, + "max_control_steps": 800, + "control_frequency_hz": 25, + "provider": "rosbridge", + "mode": "stepped" + }, + "planned_variants": [ + "standalone", + "kinex", + "kinex_policy" + ], + "planned_seed_by_variant": { + "standalone": 0, + "kinex": 0, + "kinex_policy": 0 + } }, { - "id": "kinex", - "label": "Kinex", - "model": "GPT-6 Astra / high" + "key": "task03/02", + "slot": "02", + "family": "task03", + "native_id": "robotwin/place-dual-shoes-aloha-agilex", + "native_identity": { + "allow_seed_substitution": false, + "embodiment": "aloha_agilex", + "max_steps": 600, + "task_config": "demo_clean", + "task_name": "place_dual_shoes" + }, + "native_instruction": "use both arms to pick up the two shoes on the table and put them in the shoebox, with the shoe tip pointing to the left", + "catalog_instruction": "use both arms to pick up the two shoes on the table and put them flat in the shoebox, with the shoe tips pointing to the left and the shoe initially on the left nearer the robot. Leave both grippers open", + "title": "Place dual shoes aloha agilex", + "planned_protocol": { + "seed": 0, + "episodes": 1, + "max_control_steps": 600, + "control_frequency_hz": 25, + "provider": "rosbridge", + "mode": "stepped" + }, + "planned_variants": [ + "standalone", + "kinex", + "kinex_policy" + ], + "planned_seed_by_variant": { + "standalone": 0, + "kinex": 0, + "kinex_policy": 0 + } }, { - "id": "kinex_policy", - "label": "Kinex + VLA", - "model": "GPT-6 Astra / high · native run_policy" + "key": "task03/03", + "slot": "03", + "family": "task03", + "native_id": "robotwin/put-bottles-dustbin-aloha-agilex", + "native_identity": { + "allow_seed_substitution": false, + "embodiment": "aloha_agilex", + "max_steps": 1700, + "task_config": "demo_clean", + "task_name": "put_bottles_dustbin" + }, + "native_instruction": "use arms to grab the bottles and put them into the dustbin to the left of the table", + "catalog_instruction": "use arms to grab the bottles and put them into the dustbin to the left of the table", + "title": "Put bottles dustbin aloha agilex", + "planned_protocol": { + "seed": 0, + "episodes": 1, + "max_control_steps": 1700, + "control_frequency_hz": 25, + "provider": "rosbridge", + "mode": "stepped" + }, + "planned_variants": [ + "standalone", + "kinex", + "kinex_policy" + ], + "planned_seed_by_variant": { + "standalone": 0, + "kinex": 0, + "kinex_policy": 0 + } } ], "episodes": [ { "id": "standalone-1", - "task": "1", - "condition": "standalone", - "status": "complete", + "variant": "standalone", + "task_key": "task03/01", + "family": "task03", + "source_family": "task03", + "provider": "rosbridge", + "slot": "01", + "seed": 0, + "source_episode_id": "standalone-1", + "status": "completed", "success": false, "valid": true, - "control_steps": 800, - "tool_calls": 0, + "execution": { + "status": "finished", + "reason": null + }, + "verdict": { + "evidence_valid": true, + "success": false, + "termination": "step_limit", + "steps": 800 + }, + "steps": 800, + "simulation_time_s": 32.0, + "wall_time_s": 261.565797, + "model": "Qwen3-VL-OFT", + "effort": "40k checkpoint", + "harness": "standalone", + "instruction": "use the left arm to grasp the red block on the table, hand it over to the right arm, and set it down in the center of the blue pad. Release it with the right gripper open", + "native_instruction": "use the left arm to grasp the red block on the table, handover it to the right arm and place it on the blue pad", + "instruction_policy": "modified", + "usage": { + "schema": "rlebench/token-usage/1", + "source": "model-free-controller", + "response_count": 0, + "response_ids_complete": true, + "audit_complete": true, + "input_tokens": 0, + "output_tokens": 0, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "fresh_input_tokens": 0, + "reasoning_output_tokens": 0, + "usage_source": "no-language-agent" + }, + "call_activity": { + "model_tool_calls": 0, + "source": "Standalone executor; no language agent." + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 321, + "published_frames": 321, + "source_sample_coverage": 1.0, + "source_samples_complete": true, + "source_dropped_samples": 0, + "terminal_frame_retained": true, + "terminal_source_index": 320, + "terminal_hold_s": 0, + "simulation_time_s": 32.0, + "duration_s": 8.025, + "sha256": "f48927e16c9c76e137b201cee9c8fea1962a531a9a9dd2fd1ec95edca06e0872", + "source_sha256": { + "third_person.mp4": "4e36cb8ef0de2679bd13de879fa32d580956419072098d13a8580474a15667ee", + "left_wrist.mp4": "49fd041e7dc962a508affa8e83a73d456eefe91239df83f21230db8350f17f64", + "right_wrist.mp4": "40c1dd5f0dad519f07dc9340c3af83b79cd1695107d030fa6ebda1caad731a2c" + }, + "all_frames_compared": 321, + "minimum_frame_psnr_db": 43.1505876100595, + "maximum_frame_rgb_mae": 1.3631350994110107, + "decode_ok": true, + "pts_verified": true, + "playback_kind": "simulation_time", + "resolution_kind": "independent_720_high", + "agent_observation_video": false, + "camera_views": [ + { + "name": "third_person", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "left_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "right_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + } + ], + "coverage_basis": "Every scheduled 10 Hz control-boundary sample, plus the terminal state.", + "frame_mapping": "Each captured sample exactly once; no appended or held frames.", + "sampling_note": "Samples follow the native simulation clock; model thinking adds no footage.", + "scope": "Live owner-only spectator views; no replay, rerender, upscaling or invented frames.", + "recording": { + "schema": "rosbridge/recording/1", + "run_id": "ab542c17c6c7", + "task": "robotwin/handover-block-aloha-agilex", + "state": "complete", + "views": { + "third_person": { + "received": 322, + "dropped": 0, + "segments": [ + { + "file": "third_person.mp4", + "frames": 321, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 32000000000 + } + ], + "error": null + }, + "left_wrist": { + "received": 322, + "dropped": 0, + "segments": [ + { + "file": "left_wrist.mp4", + "frames": 321, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 32000000000 + } + ], + "error": null + }, + "right_wrist": { + "received": 322, + "dropped": 0, + "segments": [ + { + "file": "right_wrist.mp4", + "frames": 321, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 32000000000 + } + ], + "error": null + } + }, + "type": "mp4", + "spectator_views": [ + "third_person", + "left_wrist", + "right_wrist" + ], + "fps": 10.0, + "topics": [], + "error": null, + "bag_messages": 0 + } + }, + "analysis": { + "observed": "The policy consumed all 800 native control frames in 50 inferences. The red block remained away from the blue pad in the terminal recording. No planning agent or manual recovery intervened." + }, + "links": { + "video": "experiments/robotwin-native/standalone-1/presentation/video.mp4", + "poster": "experiments/robotwin-native/standalone-1/presentation/poster.jpg", + "terminal_frame": "experiments/robotwin-native/standalone-1/presentation/terminal.jpg", + "transcript": "experiments/robotwin-native/standalone-1/transcript.json", + "usage": "experiments/robotwin-native/standalone-1/usage.json", + "analysis": "experiments/robotwin-native/standalone-1/analysis.json", + "protocol": "experiments/robotwin-native/standalone-1/protocol.json", + "native_goal": "experiments/robotwin-native/standalone-1/instruction.json", + "verdict": "experiments/robotwin-native/standalone-1/episode.json", + "provenance": "sources.json", + "media_check": "experiments/robotwin-native/standalone-1/presentation/media-validation.json", + "recording_manifest": "experiments/robotwin-native/standalone-1/presentation/recording.json", + "workspace": "experiments/robotwin-native/standalone-1/workspace.tar.gz", + "workspace_changes": "experiments/robotwin-native/standalone-1/workspace.json", + "evidence_archive": "experiments/robotwin-native/standalone-1/evidence.tar.gz", + "tool_calls": "experiments/robotwin-native/standalone-1/tool-calls.json", + "artifact_index": "experiments/robotwin-native/standalone-1/artifact-index.json", + "vla_io": "experiments/robotwin-native/standalone-1/vla-io.json", + "policy_trace": "experiments/robotwin-native/standalone-1/policy/calls.jsonl" + }, + "resources": [], + "session_counts": { + "visible_events": 0, + "observed_images": 0 + }, + "measured_protocol": { + "max_steps": 800, + "seed": 0, + "episodes": 1, + "control_frequency_hz": 25, + "provider": "rosbridge", + "realtime": false, + "stop_on_success": true + }, "vla_calls": 50, - "input_tokens": 0, - "output_tokens": 0, - "cached_input_tokens": 0, - "api_cost_usd": 0, - "path": "experiments/robotwin-native/standalone-1", - "video": "experiments/robotwin-native/standalone-1/combined.mp4", - "poster": "experiments/robotwin-native/standalone-1/poster.jpg", - "wall_time_s": 261.565797 + "kinex_revision": null }, { "id": "kinex-1", - "task": "1", - "condition": "kinex", - "status": "complete", + "variant": "kinex", + "task_key": "task03/01", + "family": "task03", + "source_family": "task03", + "provider": "rosbridge", + "slot": "01", + "seed": 0, + "source_episode_id": "kinex-1", + "status": "completed", "success": true, "valid": true, - "control_steps": 621, - "tool_calls": 61, + "execution": { + "status": "finished", + "reason": null + }, + "verdict": { + "evidence_valid": true, + "success": true, + "termination": "success", + "steps": 621 + }, + "steps": 621, + "simulation_time_s": 24.84, + "wall_time_s": 635.897416, + "model": "GPT-6 Astra", + "effort": "high", + "harness": "kinex", + "instruction": "use the left arm to grasp the red block on the table, hand it over to the right arm, and set it down in the center of the blue pad. Release it with the right gripper open", + "native_instruction": "use the left arm to grasp the red block on the table, handover it to the right arm and place it on the blue pad", + "instruction_policy": "modified", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "audit_source": "Provider terminal response usage", + "cache_hit_rate": 0.9668311309252369, + "cache_read_input_tokens": 1715200, + "cache_reported_input_tokens": 1774043, + "cache_write_basis": "provider-reported", + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 1774043, + "cached_input_tokens": 1715200, + "cost_usd": null, + "fresh_input_tokens": 58843, + "harbor_mismatches": {}, + "input_breakdown_complete": true, + "input_tokens": 1774043, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 1715200, + "known_fresh_input_tokens": 58843, + "known_input_tokens": 1774043, + "known_output_tokens": 8679, + "known_reasoning_output_tokens": 2627, + "output_tokens": 8679, + "parser": "harbor/0.22.0", + "reasoning_output_tokens": 2627, + "reasoning_reported_output_tokens": 8679, + "rejected_attempts": 0, + "reported_responses": { + "cache_reported_input_tokens": 54, + "cache_write_input_tokens": 54, + "cache_write_reported_input_tokens": 54, + "cached_input_tokens": 54, + "fresh_input_tokens": 54, + "input_tokens": 54, + "output_tokens": 54, + "reasoning_output_tokens": 54, + "reasoning_reported_output_tokens": 54 + }, + "request_attempts": 54, + "response_count": 54, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Harbor session metrics", + "total_tokens": 1782722, + "uncached_input_tokens": 58843, + "unresolved_attempts": [], + "usage_records": [ + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 0, + "input_tokens": 11322, + "output_tokens": 19, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11136, + "input_tokens": 11362, + "output_tokens": 29, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 0, + "input_tokens": 11495, + "output_tokens": 30, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11392, + "input_tokens": 14101, + "output_tokens": 223, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 14208, + "input_tokens": 18793, + "output_tokens": 55, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 18688, + "input_tokens": 20034, + "output_tokens": 78, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 19968, + "input_tokens": 20264, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 20224, + "input_tokens": 20772, + "output_tokens": 42, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 20608, + "input_tokens": 21024, + "output_tokens": 164, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 20992, + "input_tokens": 21584, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 21504, + "input_tokens": 21855, + "output_tokens": 167, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 21888, + "input_tokens": 23252, + "output_tokens": 406, + "reasoning_output_tokens": 26 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 23552, + "input_tokens": 24103, + "output_tokens": 140, + "reasoning_output_tokens": 21 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 24064, + "input_tokens": 26193, + "output_tokens": 361, + "reasoning_output_tokens": 25 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 26368, + "input_tokens": 26857, + "output_tokens": 1241, + "reasoning_output_tokens": 258 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 27904, + "input_tokens": 28121, + "output_tokens": 81, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 28032, + "input_tokens": 28847, + "output_tokens": 87, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 28800, + "input_tokens": 28961, + "output_tokens": 81, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 28928, + "input_tokens": 29553, + "output_tokens": 98, + "reasoning_output_tokens": 14 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 29440, + "input_tokens": 30159, + "output_tokens": 56, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 30080, + "input_tokens": 30720, + "output_tokens": 254, + "reasoning_output_tokens": 229 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 30848, + "input_tokens": 30992, + "output_tokens": 23, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 30976, + "input_tokens": 31184, + "output_tokens": 288, + "reasoning_output_tokens": 174 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 31360, + "input_tokens": 31984, + "output_tokens": 160, + "reasoning_output_tokens": 83 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 32000, + "input_tokens": 32638, + "output_tokens": 56, + "reasoning_output_tokens": 21 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 32512, + "input_tokens": 33487, + "output_tokens": 23, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 33408, + "input_tokens": 33528, + "output_tokens": 23, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 33408, + "input_tokens": 33721, + "output_tokens": 555, + "reasoning_output_tokens": 52 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34176, + "input_tokens": 34296, + "output_tokens": 40, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34176, + "input_tokens": 34670, + "output_tokens": 193, + "reasoning_output_tokens": 117 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34688, + "input_tokens": 35365, + "output_tokens": 47, + "reasoning_output_tokens": 13 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 35200, + "input_tokens": 35474, + "output_tokens": 345, + "reasoning_output_tokens": 300 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 35584, + "input_tokens": 36417, + "output_tokens": 453, + "reasoning_output_tokens": 373 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 36736, + "input_tokens": 37369, + "output_tokens": 315, + "reasoning_output_tokens": 223 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 37632, + "input_tokens": 38183, + "output_tokens": 59, + "reasoning_output_tokens": 28 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 38144, + "input_tokens": 38304, + "output_tokens": 93, + "reasoning_output_tokens": 51 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 38272, + "input_tokens": 38878, + "output_tokens": 201, + "reasoning_output_tokens": 80 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 38912, + "input_tokens": 39581, + "output_tokens": 32, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 39424, + "input_tokens": 40093, + "output_tokens": 128, + "reasoning_output_tokens": 44 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 40064, + "input_tokens": 40714, + "output_tokens": 87, + "reasoning_output_tokens": 32 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 40576, + "input_tokens": 41287, + "output_tokens": 105, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 41216, + "input_tokens": 41902, + "output_tokens": 40, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 41728, + "input_tokens": 42274, + "output_tokens": 111, + "reasoning_output_tokens": 36 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 42240, + "input_tokens": 42883, + "output_tokens": 56, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 42880, + "input_tokens": 43430, + "output_tokens": 54, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 43392, + "input_tokens": 43967, + "output_tokens": 81, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 43904, + "input_tokens": 44545, + "output_tokens": 66, + "reasoning_output_tokens": 32 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 44416, + "input_tokens": 45087, + "output_tokens": 225, + "reasoning_output_tokens": 111 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 45184, + "input_tokens": 45813, + "output_tokens": 101, + "reasoning_output_tokens": 29 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 45696, + "input_tokens": 46415, + "output_tokens": 32, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 46208, + "input_tokens": 46923, + "output_tokens": 112, + "reasoning_output_tokens": 54 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 46976, + "input_tokens": 47169, + "output_tokens": 99, + "reasoning_output_tokens": 47 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 47232, + "input_tokens": 47665, + "output_tokens": 745, + "reasoning_output_tokens": 154 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 48256, + "input_tokens": 48433, + "output_tokens": 41, + "reasoning_output_tokens": 0 + } + ], + "usage_source": "provider-raw" + }, + "call_activity": { + "schema": "rlebench/call-activity/1", + "source": "Kinex native trace", + "model_tool_calls": 61, + "model_tool_calls_by_name": { + "ls": 1, + "read": 15, + "find": 1, + "exec": 32, + "devices": 2, + "write": 3, + "edit": 1, + "device": 4, + "device_stop": 2 + }, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "nested_python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_scope": "Recorded instrumented Python invocations only; other paths and automatic fatal stops are not counted." + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 250, + "published_frames": 250, + "source_sample_coverage": 1.0, + "source_samples_complete": true, + "source_dropped_samples": 0, + "terminal_frame_retained": true, + "terminal_source_index": 249, + "terminal_hold_s": 0, + "simulation_time_s": 24.84, + "duration_s": 6.25, + "sha256": "db268699bf6ca884b5ec8fa8d723ca8b7b22a593df1c6d256e7d26321ff3b459", + "source_sha256": { + "third_person.mp4": "7c1bd7ee7b668162aaa82f5aec1eb8c12e4e929f34f1686107f058ae1d7b50c5", + "left_wrist.mp4": "4c2eacfbff0a7bb63f8d121d4ce58f0dee6a3ec8ad3618ca13863f85cc604a26", + "right_wrist.mp4": "bcc3decdacdbb55bba3bd0b96e8aa6de9f73ec560090f7ee69fcc93bbe801cd9" + }, + "all_frames_compared": 250, + "minimum_frame_psnr_db": 42.35292305290214, + "maximum_frame_rgb_mae": 1.2616853713989258, + "decode_ok": true, + "pts_verified": true, + "playback_kind": "simulation_time", + "resolution_kind": "independent_720_high", + "agent_observation_video": false, + "camera_views": [ + { + "name": "third_person", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "left_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "right_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + } + ], + "coverage_basis": "Every scheduled 10 Hz control-boundary sample, plus the terminal state.", + "frame_mapping": "Each captured sample exactly once; no appended or held frames.", + "sampling_note": "Samples follow the native simulation clock; model thinking adds no footage.", + "scope": "Live owner-only spectator views; no replay, rerender, upscaling or invented frames.", + "recording": { + "schema": "rosbridge/recording/1", + "run_id": "7b75d3e8384a", + "task": "robotwin/handover-block-aloha-agilex", + "state": "complete", + "views": { + "third_person": { + "received": 250, + "dropped": 0, + "segments": [ + { + "file": "third_person.mp4", + "frames": 250, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 24840000000 + } + ], + "error": null + }, + "left_wrist": { + "received": 250, + "dropped": 0, + "segments": [ + { + "file": "left_wrist.mp4", + "frames": 250, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 24840000000 + } + ], + "error": null + }, + "right_wrist": { + "received": 250, + "dropped": 0, + "segments": [ + { + "file": "right_wrist.mp4", + "frames": 250, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 24840000000 + } + ], + "error": null + } + }, + "type": "mp4", + "spectator_views": [ + "third_person", + "left_wrist", + "right_wrist" + ], + "fps": 10.0, + "topics": [], + "error": null, + "bag_messages": 0 + } + }, + "analysis": { + "observed": "Kinex measured RGB-D geometry, wrote Cartesian motion and perception tools, and completed the handover. Native success occurred at frame 621; the world ended during final release." + }, + "links": { + "video": "experiments/robotwin-native/kinex-1/presentation/video.mp4", + "poster": "experiments/robotwin-native/kinex-1/presentation/poster.jpg", + "terminal_frame": "experiments/robotwin-native/kinex-1/presentation/terminal.jpg", + "transcript": "experiments/robotwin-native/kinex-1/transcript.json", + "usage": "experiments/robotwin-native/kinex-1/usage.json", + "analysis": "experiments/robotwin-native/kinex-1/analysis.json", + "protocol": "experiments/robotwin-native/kinex-1/protocol.json", + "native_goal": "experiments/robotwin-native/kinex-1/instruction.json", + "verdict": "experiments/robotwin-native/kinex-1/episode.json", + "provenance": "sources.json", + "media_check": "experiments/robotwin-native/kinex-1/presentation/media-validation.json", + "recording_manifest": "experiments/robotwin-native/kinex-1/presentation/recording.json", + "workspace": "experiments/robotwin-native/kinex-1/workspace.tar.gz", + "workspace_changes": "experiments/robotwin-native/kinex-1/workspace.json", + "evidence_archive": "experiments/robotwin-native/kinex-1/evidence.tar.gz", + "tool_calls": "experiments/robotwin-native/kinex-1/tool-calls.json", + "artifact_index": "experiments/robotwin-native/kinex-1/artifact-index.json", + "native_session": "experiments/robotwin-native/kinex-1/session.jsonl", + "trace": "experiments/robotwin-native/kinex-1/trace.json", + "provider_usage": "experiments/robotwin-native/kinex-1/provider-requests.jsonl", + "vla_io": "experiments/robotwin-native/kinex-1/vla-io.json", + "policy_trace": null + }, + "resources": [ + { + "name": ".kinex/tools/cartesian_move.py", + "file": "experiments/robotwin-native/kinex-1/workspace/.kinex/tools/cartesian_move.py", + "kind": "Created during this episode; no imported earlier task resources." + }, + { + "name": ".kinex/tools/color_geometry.py", + "file": "experiments/robotwin-native/kinex-1/workspace/.kinex/tools/color_geometry.py", + "kind": "Created during this episode; no imported earlier task resources." + }, + { + "name": ".kinex/memos/handover-block.md", + "file": "experiments/robotwin-native/kinex-1/workspace/.kinex/memos/handover-block.md", + "kind": "Created during this episode; no imported earlier task resources." + } + ], + "session_counts": { + "visible_events": 117, + "observed_images": 30 + }, + "measured_protocol": { + "max_steps": 800, + "seed": 0, + "episodes": 1, + "control_frequency_hz": 25, + "provider": "rosbridge", + "realtime": false, + "stop_on_success": true + }, "vla_calls": 0, - "input_tokens": 1774043, - "output_tokens": 8679, - "cached_input_tokens": 1715200, - "api_cost_usd": 2.73758, - "path": "experiments/robotwin-native/kinex-1", - "video": "experiments/robotwin-native/kinex-1/combined.mp4", - "poster": "experiments/robotwin-native/kinex-1/poster.jpg", - "wall_time_s": 635.897416 + "kinex_revision": "f4ab59e9e93aa2f569d88386799a2bed0670d59c", + "kinex_version": "1.1.0" }, { "id": "kinex_policy-1", - "task": "1", - "condition": "kinex_policy", - "status": "complete", + "variant": "kinex_policy", + "task_key": "task03/01", + "family": "task03", + "source_family": "task03", + "provider": "rosbridge", + "slot": "01", + "seed": 0, + "source_episode_id": "kinex_policy-1", + "status": "completed", "success": true, "valid": true, - "control_steps": 523, - "tool_calls": 58, + "execution": { + "status": "finished", + "reason": null + }, + "verdict": { + "evidence_valid": true, + "success": true, + "termination": "success", + "steps": 523 + }, + "steps": 523, + "simulation_time_s": 20.92, + "wall_time_s": 518.141632, + "model": "GPT-6 Astra", + "effort": "high", + "harness": "kinex", + "instruction": "use the left arm to grasp the red block on the table, hand it over to the right arm, and set it down in the center of the blue pad. Release it with the right gripper open", + "native_instruction": "use the left arm to grasp the red block on the table, handover it to the right arm and place it on the blue pad", + "instruction_policy": "modified", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "audit_source": "Provider terminal response usage", + "cache_hit_rate": 0.9718491681630644, + "cache_read_input_tokens": 1536512, + "cache_reported_input_tokens": 1581019, + "cache_write_basis": "provider-reported", + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 1581019, + "cached_input_tokens": 1536512, + "cost_usd": null, + "fresh_input_tokens": 44507, + "harbor_mismatches": {}, + "input_breakdown_complete": true, + "input_tokens": 1581019, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 1536512, + "known_fresh_input_tokens": 44507, + "known_input_tokens": 1581019, + "known_output_tokens": 6998, + "known_reasoning_output_tokens": 1710, + "output_tokens": 6998, + "parser": "harbor/0.22.0", + "reasoning_output_tokens": 1710, + "reasoning_reported_output_tokens": 6998, + "rejected_attempts": 0, + "reported_responses": { + "cache_reported_input_tokens": 48, + "cache_write_input_tokens": 48, + "cache_write_reported_input_tokens": 48, + "cached_input_tokens": 48, + "fresh_input_tokens": 48, + "input_tokens": 48, + "output_tokens": 48, + "reasoning_output_tokens": 48, + "reasoning_reported_output_tokens": 48 + }, + "request_attempts": 48, + "response_count": 48, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Harbor session metrics", + "total_tokens": 1588017, + "uncached_input_tokens": 44507, + "unresolved_attempts": [], + "usage_records": [ + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 0, + "input_tokens": 11317, + "output_tokens": 19, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11136, + "input_tokens": 11357, + "output_tokens": 29, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11264, + "input_tokens": 11490, + "output_tokens": 30, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11392, + "input_tokens": 15932, + "output_tokens": 52, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 15872, + "input_tokens": 16139, + "output_tokens": 252, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 16256, + "input_tokens": 21248, + "output_tokens": 114, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 21248, + "input_tokens": 23413, + "output_tokens": 107, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 23424, + "input_tokens": 23737, + "output_tokens": 65, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 23680, + "input_tokens": 24765, + "output_tokens": 63, + "reasoning_output_tokens": 21 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 24704, + "input_tokens": 27118, + "output_tokens": 23, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 27008, + "input_tokens": 27309, + "output_tokens": 412, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 27520, + "input_tokens": 27743, + "output_tokens": 37, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 27648, + "input_tokens": 28194, + "output_tokens": 37, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 28032, + "input_tokens": 28640, + "output_tokens": 36, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 28544, + "input_tokens": 29084, + "output_tokens": 54, + "reasoning_output_tokens": 16 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 28928, + "input_tokens": 29546, + "output_tokens": 59, + "reasoning_output_tokens": 20 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 29568, + "input_tokens": 29928, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 29824, + "input_tokens": 30376, + "output_tokens": 178, + "reasoning_output_tokens": 51 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 30464, + "input_tokens": 31921, + "output_tokens": 171, + "reasoning_output_tokens": 70 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 32000, + "input_tokens": 32167, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 32000, + "input_tokens": 32438, + "output_tokens": 342, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 32640, + "input_tokens": 33195, + "output_tokens": 954, + "reasoning_output_tokens": 252 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34048, + "input_tokens": 34170, + "output_tokens": 74, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34048, + "input_tokens": 34460, + "output_tokens": 75, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34432, + "input_tokens": 34892, + "output_tokens": 113, + "reasoning_output_tokens": 28 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34816, + "input_tokens": 35366, + "output_tokens": 52, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 35200, + "input_tokens": 35776, + "output_tokens": 115, + "reasoning_output_tokens": 27 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 35712, + "input_tokens": 36251, + "output_tokens": 228, + "reasoning_output_tokens": 143 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 36352, + "input_tokens": 36839, + "output_tokens": 296, + "reasoning_output_tokens": 212 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 36992, + "input_tokens": 37492, + "output_tokens": 51, + "reasoning_output_tokens": 12 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 37376, + "input_tokens": 37866, + "output_tokens": 428, + "reasoning_output_tokens": 25 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 38144, + "input_tokens": 38314, + "output_tokens": 31, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 38144, + "input_tokens": 38699, + "output_tokens": 162, + "reasoning_output_tokens": 79 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 38656, + "input_tokens": 39217, + "output_tokens": 159, + "reasoning_output_tokens": 77 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 39168, + "input_tokens": 39737, + "output_tokens": 77, + "reasoning_output_tokens": 16 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 39680, + "input_tokens": 40178, + "output_tokens": 87, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 40064, + "input_tokens": 40625, + "output_tokens": 213, + "reasoning_output_tokens": 130 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 40704, + "input_tokens": 41196, + "output_tokens": 30, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 41088, + "input_tokens": 41576, + "output_tokens": 339, + "reasoning_output_tokens": 256 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 41856, + "input_tokens": 42274, + "output_tokens": 83, + "reasoning_output_tokens": 43 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 42240, + "input_tokens": 42605, + "output_tokens": 159, + "reasoning_output_tokens": 77 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 42624, + "input_tokens": 43122, + "output_tokens": 73, + "reasoning_output_tokens": 10 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 43008, + "input_tokens": 43328, + "output_tokens": 30, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 43264, + "input_tokens": 43420, + "output_tokens": 51, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 43392, + "input_tokens": 43756, + "output_tokens": 127, + "reasoning_output_tokens": 95 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 43776, + "input_tokens": 43901, + "output_tokens": 229, + "reasoning_output_tokens": 50 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 44032, + "input_tokens": 44156, + "output_tokens": 567, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 44544, + "input_tokens": 44746, + "output_tokens": 37, + "reasoning_output_tokens": 0 + } + ], + "usage_source": "provider-raw" + }, + "call_activity": { + "schema": "rlebench/call-activity/1", + "source": "Kinex native trace", + "model_tool_calls": 58, + "model_tool_calls_by_name": { + "ls": 1, + "read": 17, + "find": 1, + "bash": 1, + "devices": 4, + "exec": 26, + "device": 2, + "write": 4, + "device_stop": 1, + "edit": 1 + }, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "nested_python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_scope": "Recorded instrumented Python invocations only; other paths and automatic fatal stops are not counted." + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 210, + "published_frames": 210, + "source_sample_coverage": 1.0, + "source_samples_complete": true, + "source_dropped_samples": 0, + "terminal_frame_retained": true, + "terminal_source_index": 209, + "terminal_hold_s": 0, + "simulation_time_s": 20.92, + "duration_s": 5.25, + "sha256": "ac15e983e0afbe68b6c357a9563df0beae4c8eef2128af0ed33e76bd302a474d", + "source_sha256": { + "third_person.mp4": "363adc62fca2294fe530d8ca0684b0aa98ddaff80ba91aae290a9c825721f05c", + "left_wrist.mp4": "03b8216df236ca19b6cc9fd4b050fa6d465d536befc8bbcecb41bf566e84695c", + "right_wrist.mp4": "9eab1d7fcf5bdc04e0b6a4d6377427b2059a97fdcfacd6f85bddffcfd3238eb0" + }, + "all_frames_compared": 210, + "minimum_frame_psnr_db": 43.772015440757116, + "maximum_frame_rgb_mae": 1.2994675636291504, + "decode_ok": true, + "pts_verified": true, + "playback_kind": "simulation_time", + "resolution_kind": "independent_720_high", + "agent_observation_video": false, + "camera_views": [ + { + "name": "third_person", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "left_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "right_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + } + ], + "coverage_basis": "Every scheduled 10 Hz control-boundary sample, plus the terminal state.", + "frame_mapping": "Each captured sample exactly once; no appended or held frames.", + "sampling_note": "Samples follow the native simulation clock; model thinking adds no footage.", + "scope": "Live owner-only spectator views; no replay, rerender, upscaling or invented frames.", + "recording": { + "schema": "rosbridge/recording/1", + "run_id": "83632f6147cc", + "task": "robotwin/handover-block-aloha-agilex", + "state": "complete", + "views": { + "third_person": { + "received": 211, + "dropped": 0, + "segments": [ + { + "file": "third_person.mp4", + "frames": 210, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 20920000000 + } + ], + "error": null + }, + "left_wrist": { + "received": 211, + "dropped": 0, + "segments": [ + { + "file": "left_wrist.mp4", + "frames": 210, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 20920000000 + } + ], + "error": null + }, + "right_wrist": { + "received": 211, + "dropped": 0, + "segments": [ + { + "file": "right_wrist.mp4", + "frames": 210, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 20920000000 + } + ], + "error": null + } + }, + "type": "mp4", + "spectator_views": [ + "third_person", + "left_wrist", + "right_wrist" + ], + "fps": 10.0, + "topics": [], + "error": null, + "bag_messages": 0 + } + }, + "analysis": { + "observed": "Kinex wrote its own one-chunk policy tool with fresh feedback. Five 16-action inferences used the complete original instruction, without subtask rewriting. It then switched to agent-written geometric control and succeeded at native frame 523. This episode shows autonomous switching; it does not establish a planning benefit from VLA." + }, + "links": { + "video": "experiments/robotwin-native/kinex_policy-1/presentation/video.mp4", + "poster": "experiments/robotwin-native/kinex_policy-1/presentation/poster.jpg", + "terminal_frame": "experiments/robotwin-native/kinex_policy-1/presentation/terminal.jpg", + "transcript": "experiments/robotwin-native/kinex_policy-1/transcript.json", + "usage": "experiments/robotwin-native/kinex_policy-1/usage.json", + "analysis": "experiments/robotwin-native/kinex_policy-1/analysis.json", + "protocol": "experiments/robotwin-native/kinex_policy-1/protocol.json", + "native_goal": "experiments/robotwin-native/kinex_policy-1/instruction.json", + "verdict": "experiments/robotwin-native/kinex_policy-1/episode.json", + "provenance": "sources.json", + "media_check": "experiments/robotwin-native/kinex_policy-1/presentation/media-validation.json", + "recording_manifest": "experiments/robotwin-native/kinex_policy-1/presentation/recording.json", + "workspace": "experiments/robotwin-native/kinex_policy-1/workspace.tar.gz", + "workspace_changes": "experiments/robotwin-native/kinex_policy-1/workspace.json", + "evidence_archive": "experiments/robotwin-native/kinex_policy-1/evidence.tar.gz", + "tool_calls": "experiments/robotwin-native/kinex_policy-1/tool-calls.json", + "artifact_index": "experiments/robotwin-native/kinex_policy-1/artifact-index.json", + "native_session": "experiments/robotwin-native/kinex_policy-1/session.jsonl", + "trace": "experiments/robotwin-native/kinex_policy-1/trace.json", + "provider_usage": "experiments/robotwin-native/kinex_policy-1/provider-requests.jsonl", + "vla_io": "experiments/robotwin-native/kinex_policy-1/vla-io.json", + "policy_trace": "experiments/robotwin-native/kinex_policy-1/policy/calls.jsonl" + }, + "resources": [ + { + "name": ".kinex/tools/cartesian_move.py", + "file": "experiments/robotwin-native/kinex_policy-1/workspace/.kinex/tools/cartesian_move.py", + "kind": "Created during this episode; no imported earlier task resources." + }, + { + "name": ".kinex/tools/color_geometry.py", + "file": "experiments/robotwin-native/kinex_policy-1/workspace/.kinex/tools/color_geometry.py", + "kind": "Created during this episode; no imported earlier task resources." + }, + { + "name": ".kinex/tools/handover_policy.py", + "file": "experiments/robotwin-native/kinex_policy-1/workspace/.kinex/tools/handover_policy.py", + "kind": "Created during this episode; no imported earlier task resources." + }, + { + "name": ".kinex/memos/handover-block.md", + "file": "experiments/robotwin-native/kinex_policy-1/workspace/.kinex/memos/handover-block.md", + "kind": "Created during this episode; no imported earlier task resources." + } + ], + "session_counts": { + "visible_events": 108, + "observed_images": 24 + }, + "measured_protocol": { + "max_steps": 800, + "seed": 0, + "episodes": 1, + "control_frequency_hz": 25, + "provider": "rosbridge", + "realtime": false, + "stop_on_success": true + }, "vla_calls": 5, - "input_tokens": 1581019, - "output_tokens": 6998, - "cached_input_tokens": 1536512, - "api_cost_usd": 2.331482, - "path": "experiments/robotwin-native/kinex_policy-1", - "video": "experiments/robotwin-native/kinex_policy-1/combined.mp4", - "poster": "experiments/robotwin-native/kinex_policy-1/poster.jpg", - "wall_time_s": 518.141632 + "kinex_revision": "f4ab59e9e93aa2f569d88386799a2bed0670d59c", + "kinex_version": "1.1.0" }, { "id": "standalone-2", - "task": "2", - "condition": "standalone", - "status": "complete", + "variant": "standalone", + "task_key": "task03/02", + "family": "task03", + "source_family": "task03", + "provider": "rosbridge", + "slot": "02", + "seed": 0, + "source_episode_id": "standalone-2", + "status": "completed", "success": false, "valid": true, - "control_steps": 600, - "tool_calls": 0, + "execution": { + "status": "finished", + "reason": null + }, + "verdict": { + "evidence_valid": true, + "success": false, + "termination": "step_limit", + "steps": 600 + }, + "steps": 600, + "simulation_time_s": 24.0, + "wall_time_s": 242.908068, + "model": "Qwen3-VL-OFT", + "effort": "40k checkpoint", + "harness": "standalone", + "instruction": "use both arms to pick up the two shoes on the table and put them flat in the shoebox, with the shoe tips pointing to the left and the shoe initially on the left nearer the robot. Leave both grippers open", + "native_instruction": "use both arms to pick up the two shoes on the table and put them in the shoebox, with the shoe tip pointing to the left", + "instruction_policy": "modified", + "usage": { + "schema": "rlebench/token-usage/1", + "source": "model-free-controller", + "response_count": 0, + "response_ids_complete": true, + "audit_complete": true, + "input_tokens": 0, + "output_tokens": 0, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "fresh_input_tokens": 0, + "reasoning_output_tokens": 0, + "usage_source": "no-language-agent" + }, + "call_activity": { + "model_tool_calls": 0, + "source": "Standalone executor; no language agent." + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 241, + "published_frames": 241, + "source_sample_coverage": 1.0, + "source_samples_complete": true, + "source_dropped_samples": 0, + "terminal_frame_retained": true, + "terminal_source_index": 240, + "terminal_hold_s": 0, + "simulation_time_s": 24.0, + "duration_s": 6.025, + "sha256": "853aaf9c401bc5ec3d4ef9724f361e769fe67cf0fbaa7458225473bc31c89024", + "source_sha256": { + "third_person.mp4": "8c5e081683ef1d1c169c27ed1962ae1f3482988daf51f4ecdd647c1a0993e986", + "left_wrist.mp4": "d39cfb3863bc89f623396c2bebe1078a067922da084683a13acb6e53c7f56c4c", + "right_wrist.mp4": "dbad04f5d27e3846ecb827a6592d14f8f6769d52f6065d1ead9dd90ff0c9f680" + }, + "all_frames_compared": 241, + "minimum_frame_psnr_db": 42.00448573582187, + "maximum_frame_rgb_mae": 1.5530121326446533, + "decode_ok": true, + "pts_verified": true, + "playback_kind": "simulation_time", + "resolution_kind": "independent_720_high", + "agent_observation_video": false, + "camera_views": [ + { + "name": "third_person", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "left_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "right_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + } + ], + "coverage_basis": "Every scheduled 10 Hz control-boundary sample, plus the terminal state.", + "frame_mapping": "Each captured sample exactly once; no appended or held frames.", + "sampling_note": "Samples follow the native simulation clock; model thinking adds no footage.", + "scope": "Live owner-only spectator views; no replay, rerender, upscaling or invented frames.", + "recording": { + "schema": "rosbridge/recording/1", + "run_id": "395cc93f0d43", + "task": "robotwin/place-dual-shoes-aloha-agilex", + "state": "complete", + "views": { + "third_person": { + "received": 242, + "dropped": 0, + "segments": [ + { + "file": "third_person.mp4", + "frames": 241, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 24000000000 + } + ], + "error": null + }, + "left_wrist": { + "received": 242, + "dropped": 0, + "segments": [ + { + "file": "left_wrist.mp4", + "frames": 241, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 24000000000 + } + ], + "error": null + }, + "right_wrist": { + "received": 242, + "dropped": 0, + "segments": [ + { + "file": "right_wrist.mp4", + "frames": 241, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 24000000000 + } + ], + "error": null + } + }, + "type": "mp4", + "spectator_views": [ + "third_person", + "left_wrist", + "right_wrist" + ], + "fps": 10.0, + "topics": [], + "error": null, + "bag_messages": 0 + } + }, + "analysis": { + "observed": "The policy consumed all 600 native control frames in 38 inferences. The native evaluator did not accept the final shoe placement. The last chunk was capped by the remaining native budget." + }, + "links": { + "video": "experiments/robotwin-native/standalone-2/presentation/video.mp4", + "poster": "experiments/robotwin-native/standalone-2/presentation/poster.jpg", + "terminal_frame": "experiments/robotwin-native/standalone-2/presentation/terminal.jpg", + "transcript": "experiments/robotwin-native/standalone-2/transcript.json", + "usage": "experiments/robotwin-native/standalone-2/usage.json", + "analysis": "experiments/robotwin-native/standalone-2/analysis.json", + "protocol": "experiments/robotwin-native/standalone-2/protocol.json", + "native_goal": "experiments/robotwin-native/standalone-2/instruction.json", + "verdict": "experiments/robotwin-native/standalone-2/episode.json", + "provenance": "sources.json", + "media_check": "experiments/robotwin-native/standalone-2/presentation/media-validation.json", + "recording_manifest": "experiments/robotwin-native/standalone-2/presentation/recording.json", + "workspace": "experiments/robotwin-native/standalone-2/workspace.tar.gz", + "workspace_changes": "experiments/robotwin-native/standalone-2/workspace.json", + "evidence_archive": "experiments/robotwin-native/standalone-2/evidence.tar.gz", + "tool_calls": "experiments/robotwin-native/standalone-2/tool-calls.json", + "artifact_index": "experiments/robotwin-native/standalone-2/artifact-index.json", + "vla_io": "experiments/robotwin-native/standalone-2/vla-io.json", + "policy_trace": "experiments/robotwin-native/standalone-2/policy/calls.jsonl" + }, + "resources": [], + "session_counts": { + "visible_events": 0, + "observed_images": 0 + }, + "measured_protocol": { + "max_steps": 600, + "seed": 0, + "episodes": 1, + "control_frequency_hz": 25, + "provider": "rosbridge", + "realtime": false, + "stop_on_success": true + }, "vla_calls": 38, - "input_tokens": 0, - "output_tokens": 0, - "cached_input_tokens": 0, - "api_cost_usd": 0, - "path": "experiments/robotwin-native/standalone-2", - "video": "experiments/robotwin-native/standalone-2/combined.mp4", - "poster": "experiments/robotwin-native/standalone-2/poster.jpg", - "wall_time_s": 242.908068 + "kinex_revision": null }, { "id": "kinex-2", - "task": "2", - "condition": "kinex", - "status": "complete", + "variant": "kinex", + "task_key": "task03/02", + "family": "task03", + "source_family": "task03", + "provider": "rosbridge", + "slot": "02", + "seed": 0, + "source_episode_id": "kinex-2", + "status": "completed", "success": true, "valid": true, - "control_steps": 337, - "tool_calls": 39, + "execution": { + "status": "finished", + "reason": null + }, + "verdict": { + "evidence_valid": true, + "success": true, + "termination": "success", + "steps": 337 + }, + "steps": 337, + "simulation_time_s": 13.48, + "wall_time_s": 410.332351, + "model": "GPT-6 Astra", + "effort": "high", + "harness": "kinex", + "instruction": "use both arms to pick up the two shoes on the table and put them flat in the shoebox, with the shoe tips pointing to the left and the shoe initially on the left nearer the robot. Leave both grippers open", + "native_instruction": "use both arms to pick up the two shoes on the table and put them in the shoebox, with the shoe tip pointing to the left", + "instruction_policy": "modified", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "audit_source": "Provider terminal response usage", + "cache_hit_rate": 0.9580669562982812, + "cache_read_input_tokens": 794112, + "cache_reported_input_tokens": 828869, + "cache_write_basis": "provider-reported", + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 828869, + "cached_input_tokens": 794112, + "cost_usd": null, + "fresh_input_tokens": 34757, + "harbor_mismatches": {}, + "input_breakdown_complete": true, + "input_tokens": 828869, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 794112, + "known_fresh_input_tokens": 34757, + "known_input_tokens": 828869, + "known_output_tokens": 6737, + "known_reasoning_output_tokens": 1832, + "output_tokens": 6737, + "parser": "harbor/0.22.0", + "reasoning_output_tokens": 1832, + "reasoning_reported_output_tokens": 6737, + "rejected_attempts": 0, + "reported_responses": { + "cache_reported_input_tokens": 33, + "cache_write_input_tokens": 33, + "cache_write_reported_input_tokens": 33, + "cached_input_tokens": 33, + "fresh_input_tokens": 33, + "input_tokens": 33, + "output_tokens": 33, + "reasoning_output_tokens": 33, + "reasoning_reported_output_tokens": 33 + }, + "request_attempts": 33, + "response_count": 33, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Harbor session metrics", + "total_tokens": 835606, + "uncached_input_tokens": 34757, + "unresolved_attempts": [], + "usage_records": [ + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 0, + "input_tokens": 11324, + "output_tokens": 27, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11264, + "input_tokens": 11455, + "output_tokens": 20, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11392, + "input_tokens": 11503, + "output_tokens": 49, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11392, + "input_tokens": 11751, + "output_tokens": 32, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11648, + "input_tokens": 12198, + "output_tokens": 195, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 12288, + "input_tokens": 16449, + "output_tokens": 55, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 16384, + "input_tokens": 17690, + "output_tokens": 78, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 17664, + "input_tokens": 17920, + "output_tokens": 108, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 17920, + "input_tokens": 19127, + "output_tokens": 84, + "reasoning_output_tokens": 40 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 19072, + "input_tokens": 19421, + "output_tokens": 253, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 19456, + "input_tokens": 20209, + "output_tokens": 256, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 20352, + "input_tokens": 20861, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 20736, + "input_tokens": 21132, + "output_tokens": 250, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 21248, + "input_tokens": 23102, + "output_tokens": 339, + "reasoning_output_tokens": 98 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 23296, + "input_tokens": 23987, + "output_tokens": 512, + "reasoning_output_tokens": 253 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 24320, + "input_tokens": 24976, + "output_tokens": 236, + "reasoning_output_tokens": 160 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 25088, + "input_tokens": 26391, + "output_tokens": 1054, + "reasoning_output_tokens": 118 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 27264, + "input_tokens": 27467, + "output_tokens": 140, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 27392, + "input_tokens": 27740, + "output_tokens": 136, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 27776, + "input_tokens": 28514, + "output_tokens": 158, + "reasoning_output_tokens": 28 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 28544, + "input_tokens": 29311, + "output_tokens": 58, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 29184, + "input_tokens": 29872, + "output_tokens": 170, + "reasoning_output_tokens": 39 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 29952, + "input_tokens": 30680, + "output_tokens": 437, + "reasoning_output_tokens": 332 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 30976, + "input_tokens": 31494, + "output_tokens": 460, + "reasoning_output_tokens": 345 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 31872, + "input_tokens": 32595, + "output_tokens": 129, + "reasoning_output_tokens": 45 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 32640, + "input_tokens": 33301, + "output_tokens": 75, + "reasoning_output_tokens": 20 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 33280, + "input_tokens": 33875, + "output_tokens": 209, + "reasoning_output_tokens": 93 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 33920, + "input_tokens": 34722, + "output_tokens": 114, + "reasoning_output_tokens": 30 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34688, + "input_tokens": 35410, + "output_tokens": 55, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 35328, + "input_tokens": 35599, + "output_tokens": 64, + "reasoning_output_tokens": 31 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 35456, + "input_tokens": 35725, + "output_tokens": 60, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 35584, + "input_tokens": 36103, + "output_tokens": 839, + "reasoning_output_tokens": 200 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 36736, + "input_tokens": 36965, + "output_tokens": 46, + "reasoning_output_tokens": 0 + } + ], + "usage_source": "provider-raw" + }, + "call_activity": { + "schema": "rlebench/call-activity/1", + "source": "Kinex native trace", + "model_tool_calls": 39, + "model_tool_calls_by_name": { + "read": 13, + "ls": 1, + "bash": 1, + "exec": 18, + "devices": 2, + "device": 1, + "write": 2, + "device_stop": 1 + }, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "nested_python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_scope": "Recorded instrumented Python invocations only; other paths and automatic fatal stops are not counted." + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 136, + "published_frames": 136, + "source_sample_coverage": 1.0, + "source_samples_complete": true, + "source_dropped_samples": 0, + "terminal_frame_retained": true, + "terminal_source_index": 135, + "terminal_hold_s": 0, + "simulation_time_s": 13.48, + "duration_s": 3.4, + "sha256": "3311871fad3acde83e150af9b580ee59942654e52f56636545ca6b6b085e636d", + "source_sha256": { + "third_person.mp4": "04138aec81a13c8c1c97ee85fcf318a33d72dd244bd6f420e1333641a637ea91", + "left_wrist.mp4": "25f80f4540dd139417e01ce80dafcc61a97a333894ff7dae9b7614772be8d915", + "right_wrist.mp4": "8cd105e8b248befefa849041ca7ad5b1d3e165270a59b505dc946087aa5724fd" + }, + "all_frames_compared": 136, + "minimum_frame_psnr_db": 41.682987479494216, + "maximum_frame_rgb_mae": 1.6074283123016357, + "decode_ok": true, + "pts_verified": true, + "playback_kind": "simulation_time", + "resolution_kind": "independent_720_high", + "agent_observation_video": false, + "camera_views": [ + { + "name": "third_person", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "left_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "right_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + } + ], + "coverage_basis": "Every scheduled 10 Hz control-boundary sample, plus the terminal state.", + "frame_mapping": "Each captured sample exactly once; no appended or held frames.", + "sampling_note": "Samples follow the native simulation clock; model thinking adds no footage.", + "scope": "Live owner-only spectator views; no replay, rerender, upscaling or invented frames.", + "recording": { + "schema": "rosbridge/recording/1", + "run_id": "456c61025989", + "task": "robotwin/place-dual-shoes-aloha-agilex", + "state": "complete", + "views": { + "third_person": { + "received": 136, + "dropped": 0, + "segments": [ + { + "file": "third_person.mp4", + "frames": 136, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 13480000000 + } + ], + "error": null + }, + "left_wrist": { + "received": 136, + "dropped": 0, + "segments": [ + { + "file": "left_wrist.mp4", + "frames": 136, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 13480000000 + } + ], + "error": null + }, + "right_wrist": { + "received": 136, + "dropped": 0, + "segments": [ + { + "file": "right_wrist.mp4", + "frames": 136, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 13480000000 + } + ], + "error": null + } + }, + "type": "mp4", + "spectator_views": [ + "third_person", + "left_wrist", + "right_wrist" + ], + "fps": 10.0, + "topics": [], + "error": null, + "bag_messages": 0 + } + }, + "analysis": { + "observed": "Kinex reconstructed the shoes and box from RGB-D, wrote a reusable dual-arm motion tool, and staged the grasps, rotations and placements. Native success occurred at frame 337 during final release." + }, + "links": { + "video": "experiments/robotwin-native/kinex-2/presentation/video.mp4", + "poster": "experiments/robotwin-native/kinex-2/presentation/poster.jpg", + "terminal_frame": "experiments/robotwin-native/kinex-2/presentation/terminal.jpg", + "transcript": "experiments/robotwin-native/kinex-2/transcript.json", + "usage": "experiments/robotwin-native/kinex-2/usage.json", + "analysis": "experiments/robotwin-native/kinex-2/analysis.json", + "protocol": "experiments/robotwin-native/kinex-2/protocol.json", + "native_goal": "experiments/robotwin-native/kinex-2/instruction.json", + "verdict": "experiments/robotwin-native/kinex-2/episode.json", + "provenance": "sources.json", + "media_check": "experiments/robotwin-native/kinex-2/presentation/media-validation.json", + "recording_manifest": "experiments/robotwin-native/kinex-2/presentation/recording.json", + "workspace": "experiments/robotwin-native/kinex-2/workspace.tar.gz", + "workspace_changes": "experiments/robotwin-native/kinex-2/workspace.json", + "evidence_archive": "experiments/robotwin-native/kinex-2/evidence.tar.gz", + "tool_calls": "experiments/robotwin-native/kinex-2/tool-calls.json", + "artifact_index": "experiments/robotwin-native/kinex-2/artifact-index.json", + "native_session": "experiments/robotwin-native/kinex-2/session.jsonl", + "trace": "experiments/robotwin-native/kinex-2/trace.json", + "provider_usage": "experiments/robotwin-native/kinex-2/provider-requests.jsonl", + "vla_io": "experiments/robotwin-native/kinex-2/vla-io.json", + "policy_trace": null + }, + "resources": [ + { + "name": ".kinex/tools/dual_shoes.py", + "file": "experiments/robotwin-native/kinex-2/workspace/.kinex/tools/dual_shoes.py", + "kind": "Created during this episode; no imported earlier task resources." + }, + { + "name": ".kinex/memos/dual_shoes.md", + "file": "experiments/robotwin-native/kinex-2/workspace/.kinex/memos/dual_shoes.md", + "kind": "Created during this episode; no imported earlier task resources." + } + ], + "session_counts": { + "visible_events": 74, + "observed_images": 11 + }, + "measured_protocol": { + "max_steps": 600, + "seed": 0, + "episodes": 1, + "control_frequency_hz": 25, + "provider": "rosbridge", + "realtime": false, + "stop_on_success": true + }, "vla_calls": 0, - "input_tokens": 828869, - "output_tokens": 6737, - "cached_input_tokens": 794112, - "api_cost_usd": 1.478532, - "path": "experiments/robotwin-native/kinex-2", - "video": "experiments/robotwin-native/kinex-2/combined.mp4", - "poster": "experiments/robotwin-native/kinex-2/poster.jpg", - "wall_time_s": 410.332351 + "kinex_revision": "f4ab59e9e93aa2f569d88386799a2bed0670d59c", + "kinex_version": "1.1.0" }, { "id": "kinex_policy-2", - "task": "2", - "condition": "kinex_policy", - "status": "complete", + "variant": "kinex_policy", + "task_key": "task03/02", + "family": "task03", + "source_family": "task03", + "provider": "rosbridge", + "slot": "02", + "seed": 0, + "source_episode_id": "kinex_policy-2", + "status": "completed", "success": false, "valid": true, - "control_steps": 498, - "tool_calls": 72, + "execution": { + "status": "finished", + "reason": null + }, + "verdict": { + "evidence_valid": true, + "success": false, + "termination": "stopped", + "steps": 498 + }, + "steps": 498, + "simulation_time_s": 19.92, + "wall_time_s": 817.625303, + "model": "GPT-6 Astra", + "effort": "high", + "harness": "kinex", + "instruction": "use both arms to pick up the two shoes on the table and put them flat in the shoebox, with the shoe tips pointing to the left and the shoe initially on the left nearer the robot. Leave both grippers open", + "native_instruction": "use both arms to pick up the two shoes on the table and put them in the shoebox, with the shoe tip pointing to the left", + "instruction_policy": "modified", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "audit_source": "Provider terminal response usage", + "cache_hit_rate": 0.9554559175384522, + "cache_read_input_tokens": 2249856, + "cache_reported_input_tokens": 2354746, + "cache_write_basis": "provider-reported", + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 2354746, + "cached_input_tokens": 2249856, + "cost_usd": null, + "fresh_input_tokens": 104890, + "harbor_mismatches": {}, + "input_breakdown_complete": true, + "input_tokens": 2354746, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 2249856, + "known_fresh_input_tokens": 104890, + "known_input_tokens": 2354746, + "known_output_tokens": 9984, + "known_reasoning_output_tokens": 3108, + "output_tokens": 9984, + "parser": "harbor/0.22.0", + "reasoning_output_tokens": 3108, + "reasoning_reported_output_tokens": 9984, + "rejected_attempts": 0, + "reported_responses": { + "cache_reported_input_tokens": 63, + "cache_write_input_tokens": 63, + "cache_write_reported_input_tokens": 63, + "cached_input_tokens": 63, + "fresh_input_tokens": 63, + "input_tokens": 63, + "output_tokens": 63, + "reasoning_output_tokens": 63, + "reasoning_reported_output_tokens": 63 + }, + "request_attempts": 63, + "response_count": 63, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Harbor session metrics", + "total_tokens": 2364730, + "uncached_input_tokens": 104890, + "unresolved_attempts": [], + "usage_records": [ + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 0, + "input_tokens": 11328, + "output_tokens": 19, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11264, + "input_tokens": 11368, + "output_tokens": 29, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11264, + "input_tokens": 11501, + "output_tokens": 30, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11392, + "input_tokens": 14555, + "output_tokens": 47, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 14464, + "input_tokens": 15031, + "output_tokens": 252, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 15104, + "input_tokens": 20140, + "output_tokens": 167, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 20224, + "input_tokens": 23684, + "output_tokens": 107, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 23680, + "input_tokens": 24008, + "output_tokens": 40, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 23936, + "input_tokens": 26338, + "output_tokens": 75, + "reasoning_output_tokens": 19 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 26240, + "input_tokens": 26580, + "output_tokens": 465, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 26880, + "input_tokens": 27068, + "output_tokens": 40, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 27008, + "input_tokens": 27615, + "output_tokens": 37, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 27520, + "input_tokens": 28159, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 28032, + "input_tokens": 28708, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 0, + "input_tokens": 29258, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 29184, + "input_tokens": 29804, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 29696, + "input_tokens": 30352, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 30208, + "input_tokens": 30899, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 30848, + "input_tokens": 31448, + "output_tokens": 42, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 31360, + "input_tokens": 32035, + "output_tokens": 40, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 31872, + "input_tokens": 32577, + "output_tokens": 58, + "reasoning_output_tokens": 14 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 32384, + "input_tokens": 33144, + "output_tokens": 61, + "reasoning_output_tokens": 20 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 33152, + "input_tokens": 33713, + "output_tokens": 91, + "reasoning_output_tokens": 49 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 33664, + "input_tokens": 34314, + "output_tokens": 68, + "reasoning_output_tokens": 29 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34304, + "input_tokens": 34705, + "output_tokens": 102, + "reasoning_output_tokens": 68 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34688, + "input_tokens": 34869, + "output_tokens": 128, + "reasoning_output_tokens": 84 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34816, + "input_tokens": 35207, + "output_tokens": 353, + "reasoning_output_tokens": 21 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 35456, + "input_tokens": 35789, + "output_tokens": 120, + "reasoning_output_tokens": 76 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 35712, + "input_tokens": 36413, + "output_tokens": 159, + "reasoning_output_tokens": 119 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 36352, + "input_tokens": 37077, + "output_tokens": 106, + "reasoning_output_tokens": 72 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 36992, + "input_tokens": 37245, + "output_tokens": 37, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 37120, + "input_tokens": 37606, + "output_tokens": 152, + "reasoning_output_tokens": 22 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 37632, + "input_tokens": 38344, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 38272, + "input_tokens": 38615, + "output_tokens": 269, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 38784, + "input_tokens": 39478, + "output_tokens": 238, + "reasoning_output_tokens": 199 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 39680, + "input_tokens": 40182, + "output_tokens": 672, + "reasoning_output_tokens": 53 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 40704, + "input_tokens": 40875, + "output_tokens": 117, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 40832, + "input_tokens": 41422, + "output_tokens": 151, + "reasoning_output_tokens": 126 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 41472, + "input_tokens": 41743, + "output_tokens": 314, + "reasoning_output_tokens": 81 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 41856, + "input_tokens": 42462, + "output_tokens": 248, + "reasoning_output_tokens": 79 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 42624, + "input_tokens": 42845, + "output_tokens": 316, + "reasoning_output_tokens": 177 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 43008, + "input_tokens": 43844, + "output_tokens": 71, + "reasoning_output_tokens": 20 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 43776, + "input_tokens": 44351, + "output_tokens": 75, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 44288, + "input_tokens": 44453, + "output_tokens": 137, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 44416, + "input_tokens": 45140, + "output_tokens": 271, + "reasoning_output_tokens": 144 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 45184, + "input_tokens": 45842, + "output_tokens": 107, + "reasoning_output_tokens": 49 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 45824, + "input_tokens": 46269, + "output_tokens": 119, + "reasoning_output_tokens": 43 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 46336, + "input_tokens": 46820, + "output_tokens": 363, + "reasoning_output_tokens": 267 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 47104, + "input_tokens": 47506, + "output_tokens": 77, + "reasoning_output_tokens": 27 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 47488, + "input_tokens": 47904, + "output_tokens": 327, + "reasoning_output_tokens": 81 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 48000, + "input_tokens": 48655, + "output_tokens": 444, + "reasoning_output_tokens": 158 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 48896, + "input_tokens": 49238, + "output_tokens": 231, + "reasoning_output_tokens": 115 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 49280, + "input_tokens": 49903, + "output_tokens": 203, + "reasoning_output_tokens": 19 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 50048, + "input_tokens": 50456, + "output_tokens": 220, + "reasoning_output_tokens": 126 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 28544, + "input_tokens": 50996, + "output_tokens": 54, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 50944, + "input_tokens": 51369, + "output_tokens": 508, + "reasoning_output_tokens": 431 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 50560, + "input_tokens": 52308, + "output_tokens": 48, + "reasoning_output_tokens": 15 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 52224, + "input_tokens": 52418, + "output_tokens": 297, + "reasoning_output_tokens": 200 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 52608, + "input_tokens": 53035, + "output_tokens": 76, + "reasoning_output_tokens": 14 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 52992, + "input_tokens": 53429, + "output_tokens": 209, + "reasoning_output_tokens": 56 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 53504, + "input_tokens": 53665, + "output_tokens": 544, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 54016, + "input_tokens": 54232, + "output_tokens": 148, + "reasoning_output_tokens": 35 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 54144, + "input_tokens": 54409, + "output_tokens": 32, + "reasoning_output_tokens": 0 + } + ], + "usage_source": "provider-raw" + }, + "call_activity": { + "schema": "rlebench/call-activity/1", + "source": "Kinex native trace", + "model_tool_calls": 72, + "model_tool_calls_by_name": { + "ls": 1, + "read": 16, + "find": 1, + "bash": 1, + "exec": 38, + "devices": 3, + "device": 2, + "write": 3, + "device_stop": 3, + "grep": 1, + "edit": 3 + }, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "nested_python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_scope": "Recorded instrumented Python invocations only; other paths and automatic fatal stops are not counted." + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 200, + "published_frames": 200, + "source_sample_coverage": 1.0, + "source_samples_complete": true, + "source_dropped_samples": 0, + "terminal_frame_retained": true, + "terminal_source_index": 199, + "terminal_hold_s": 0, + "simulation_time_s": 19.92, + "duration_s": 5.0, + "sha256": "da23802e54cf63701938fc000dd89ce5802fb5cd8cd7a221c288c235ce3f8ba5", + "source_sha256": { + "third_person.mp4": "b1fbd04d7a0097d489360513098d6d5081b2cca3c556c50147ab7dd4e4444574", + "left_wrist.mp4": "8c6aef985461d23c5e36fb3a91d33894ceb34f6871d641459a969c8a601264ff", + "right_wrist.mp4": "611a79fab90c57f18084f6b7b4616283cd3525af4024e1fffa06d80efc285318" + }, + "all_frames_compared": 200, + "minimum_frame_psnr_db": 41.467497047947965, + "maximum_frame_rgb_mae": 1.5497524738311768, + "decode_ok": true, + "pts_verified": true, + "playback_kind": "simulation_time", + "resolution_kind": "independent_720_high", + "agent_observation_video": false, + "camera_views": [ + { + "name": "third_person", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "left_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "right_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + } + ], + "coverage_basis": "Every scheduled 10 Hz control-boundary sample, plus the terminal state.", + "frame_mapping": "Each captured sample exactly once; no appended or held frames.", + "sampling_note": "Samples follow the native simulation clock; model thinking adds no footage.", + "scope": "Live owner-only spectator views; no replay, rerender, upscaling or invented frames.", + "recording": { + "schema": "rosbridge/recording/1", + "run_id": "ebcedf7e2ae7", + "task": "robotwin/place-dual-shoes-aloha-agilex", + "state": "complete", + "views": { + "third_person": { + "received": 201, + "dropped": 0, + "segments": [ + { + "file": "third_person.mp4", + "frames": 200, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 19920000000 + } + ], + "error": null + }, + "left_wrist": { + "received": 201, + "dropped": 0, + "segments": [ + { + "file": "left_wrist.mp4", + "frames": 200, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 19920000000 + } + ], + "error": null + }, + "right_wrist": { + "received": 201, + "dropped": 0, + "segments": [ + { + "file": "right_wrist.mp4", + "frames": 200, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 19920000000 + } + ], + "error": null + } + }, + "type": "mp4", + "spectator_views": [ + "third_person", + "left_wrist", + "right_wrist" + ], + "fps": 10.0, + "topics": [], + "error": null, + "bag_messages": 0 + } + }, + "analysis": { + "observed": "Kinex ran 15 policy inferences (240 control frames), always with the full original instruction. The video shows pickup followed by lost placement. It then tried agent-written geometric recovery and stopped at frame 498 of 600, explicitly reporting failure and opening both grippers. This was an agent decision to stop, not budget exhaustion." + }, + "links": { + "video": "experiments/robotwin-native/kinex_policy-2/presentation/video.mp4", + "poster": "experiments/robotwin-native/kinex_policy-2/presentation/poster.jpg", + "terminal_frame": "experiments/robotwin-native/kinex_policy-2/presentation/terminal.jpg", + "transcript": "experiments/robotwin-native/kinex_policy-2/transcript.json", + "usage": "experiments/robotwin-native/kinex_policy-2/usage.json", + "analysis": "experiments/robotwin-native/kinex_policy-2/analysis.json", + "protocol": "experiments/robotwin-native/kinex_policy-2/protocol.json", + "native_goal": "experiments/robotwin-native/kinex_policy-2/instruction.json", + "verdict": "experiments/robotwin-native/kinex_policy-2/episode.json", + "provenance": "sources.json", + "media_check": "experiments/robotwin-native/kinex_policy-2/presentation/media-validation.json", + "recording_manifest": "experiments/robotwin-native/kinex_policy-2/presentation/recording.json", + "workspace": "experiments/robotwin-native/kinex_policy-2/workspace.tar.gz", + "workspace_changes": "experiments/robotwin-native/kinex_policy-2/workspace.json", + "evidence_archive": "experiments/robotwin-native/kinex_policy-2/evidence.tar.gz", + "tool_calls": "experiments/robotwin-native/kinex_policy-2/tool-calls.json", + "artifact_index": "experiments/robotwin-native/kinex_policy-2/artifact-index.json", + "native_session": "experiments/robotwin-native/kinex_policy-2/session.jsonl", + "trace": "experiments/robotwin-native/kinex_policy-2/trace.json", + "provider_usage": "experiments/robotwin-native/kinex_policy-2/provider-requests.jsonl", + "vla_io": "experiments/robotwin-native/kinex_policy-2/vla-io.json", + "policy_trace": "experiments/robotwin-native/kinex_policy-2/policy/calls.jsonl" + }, + "resources": [ + { + "name": ".kinex/tools/cartesian_segment.py", + "file": "experiments/robotwin-native/kinex_policy-2/workspace/.kinex/tools/cartesian_segment.py", + "kind": "Created during this episode; no imported earlier task resources." + }, + { + "name": ".kinex/tools/dual_shoes_policy.py", + "file": "experiments/robotwin-native/kinex_policy-2/workspace/.kinex/tools/dual_shoes_policy.py", + "kind": "Created during this episode; no imported earlier task resources." + }, + { + "name": ".kinex/memos/dual_shoes.md", + "file": "experiments/robotwin-native/kinex_policy-2/workspace/.kinex/memos/dual_shoes.md", + "kind": "Created during this episode; no imported earlier task resources." + } + ], + "session_counts": { + "visible_events": 137, + "observed_images": 42 + }, + "measured_protocol": { + "max_steps": 600, + "seed": 0, + "episodes": 1, + "control_frequency_hz": 25, + "provider": "rosbridge", + "realtime": false, + "stop_on_success": true + }, "vla_calls": 15, - "input_tokens": 2354746, - "output_tokens": 9984, - "cached_input_tokens": 2249856, - "api_cost_usd": 3.797956, - "path": "experiments/robotwin-native/kinex_policy-2", - "video": "experiments/robotwin-native/kinex_policy-2/combined.mp4", - "poster": "experiments/robotwin-native/kinex_policy-2/poster.jpg", - "wall_time_s": 817.625303 + "kinex_revision": "f4ab59e9e93aa2f569d88386799a2bed0670d59c", + "kinex_version": "1.1.0" }, { "id": "standalone-3", - "task": "3", - "condition": "standalone", - "status": "complete", + "variant": "standalone", + "task_key": "task03/03", + "family": "task03", + "source_family": "task03", + "provider": "rosbridge", + "slot": "03", + "seed": 0, + "source_episode_id": "standalone-3", + "status": "completed", "success": false, "valid": true, - "control_steps": 1700, - "tool_calls": 0, + "execution": { + "status": "finished", + "reason": null + }, + "verdict": { + "evidence_valid": true, + "success": false, + "termination": "step_limit", + "steps": 1700 + }, + "steps": 1700, + "simulation_time_s": 68.0, + "wall_time_s": 464.249502, + "model": "Qwen3-VL-OFT", + "effort": "40k checkpoint", + "harness": "standalone", + "instruction": "use arms to grab the bottles and put them into the dustbin to the left of the table", + "native_instruction": "use arms to grab the bottles and put them into the dustbin to the left of the table", + "instruction_policy": "original_native", + "usage": { + "schema": "rlebench/token-usage/1", + "source": "model-free-controller", + "response_count": 0, + "response_ids_complete": true, + "audit_complete": true, + "input_tokens": 0, + "output_tokens": 0, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "fresh_input_tokens": 0, + "reasoning_output_tokens": 0, + "usage_source": "no-language-agent" + }, + "call_activity": { + "model_tool_calls": 0, + "source": "Standalone executor; no language agent." + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 681, + "published_frames": 681, + "source_sample_coverage": 1.0, + "source_samples_complete": true, + "source_dropped_samples": 0, + "terminal_frame_retained": true, + "terminal_source_index": 680, + "terminal_hold_s": 0, + "simulation_time_s": 68.0, + "duration_s": 17.025, + "sha256": "454eda9b04e783daa445dd5483ef8b9f86b0dbae6af0bb53d25ae6fec4219fed", + "source_sha256": { + "third_person.mp4": "78c38f1e218f4ed86b67cb0c795d5fece738f474fa9677e3fc7881e8ade86908", + "left_wrist.mp4": "ffdfa8db776923cef878f1f215d2fe4b6bd63317764d86823137f3272379dc41", + "right_wrist.mp4": "0410e35b89ba5055f387a60bfbbe46e69c7705a7cf78ec160c00bfefa91cd252" + }, + "all_frames_compared": 681, + "minimum_frame_psnr_db": 41.518423372042804, + "maximum_frame_rgb_mae": 1.4805973768234253, + "decode_ok": true, + "pts_verified": true, + "playback_kind": "simulation_time", + "resolution_kind": "independent_720_high", + "agent_observation_video": false, + "camera_views": [ + { + "name": "third_person", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "left_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "right_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + } + ], + "coverage_basis": "Every scheduled 10 Hz control-boundary sample, plus the terminal state.", + "frame_mapping": "Each captured sample exactly once; no appended or held frames.", + "sampling_note": "Samples follow the native simulation clock; model thinking adds no footage.", + "scope": "Live owner-only spectator views; no replay, rerender, upscaling or invented frames.", + "recording": { + "schema": "rosbridge/recording/1", + "run_id": "e931672bd198", + "task": "robotwin/put-bottles-dustbin-aloha-agilex", + "state": "complete", + "views": { + "third_person": { + "received": 682, + "dropped": 0, + "segments": [ + { + "file": "third_person.mp4", + "frames": 681, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 68000000000 + } + ], + "error": null + }, + "left_wrist": { + "received": 682, + "dropped": 0, + "segments": [ + { + "file": "left_wrist.mp4", + "frames": 681, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 68000000000 + } + ], + "error": null + }, + "right_wrist": { + "received": 682, + "dropped": 0, + "segments": [ + { + "file": "right_wrist.mp4", + "frames": 681, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 68000000000 + } + ], + "error": null + } + }, + "type": "mp4", + "spectator_views": [ + "third_person", + "left_wrist", + "right_wrist" + ], + "fps": 10.0, + "topics": [], + "error": null, + "bag_messages": 0 + } + }, + "analysis": { + "observed": "The policy consumed all 1,700 native control frames in 107 inferences without native success. No planning agent or recovery controller intervened. The shared fixed-rate device protocol differs from upstream variable-duration trajectory execution." + }, + "links": { + "video": "experiments/robotwin-native/standalone-3/presentation/video.mp4", + "poster": "experiments/robotwin-native/standalone-3/presentation/poster.jpg", + "terminal_frame": "experiments/robotwin-native/standalone-3/presentation/terminal.jpg", + "transcript": "experiments/robotwin-native/standalone-3/transcript.json", + "usage": "experiments/robotwin-native/standalone-3/usage.json", + "analysis": "experiments/robotwin-native/standalone-3/analysis.json", + "protocol": "experiments/robotwin-native/standalone-3/protocol.json", + "native_goal": "experiments/robotwin-native/standalone-3/instruction.json", + "verdict": "experiments/robotwin-native/standalone-3/episode.json", + "provenance": "sources.json", + "media_check": "experiments/robotwin-native/standalone-3/presentation/media-validation.json", + "recording_manifest": "experiments/robotwin-native/standalone-3/presentation/recording.json", + "workspace": "experiments/robotwin-native/standalone-3/workspace.tar.gz", + "workspace_changes": "experiments/robotwin-native/standalone-3/workspace.json", + "evidence_archive": "experiments/robotwin-native/standalone-3/evidence.tar.gz", + "tool_calls": "experiments/robotwin-native/standalone-3/tool-calls.json", + "artifact_index": "experiments/robotwin-native/standalone-3/artifact-index.json", + "vla_io": "experiments/robotwin-native/standalone-3/vla-io.json", + "policy_trace": "experiments/robotwin-native/standalone-3/policy/calls.jsonl" + }, + "resources": [], + "session_counts": { + "visible_events": 0, + "observed_images": 0 + }, + "measured_protocol": { + "max_steps": 1700, + "seed": 0, + "episodes": 1, + "control_frequency_hz": 25, + "provider": "rosbridge", + "realtime": false, + "stop_on_success": true + }, "vla_calls": 107, - "input_tokens": 0, - "output_tokens": 0, - "cached_input_tokens": 0, - "api_cost_usd": 0, - "path": "experiments/robotwin-native/standalone-3", - "video": "experiments/robotwin-native/standalone-3/combined.mp4", - "poster": "experiments/robotwin-native/standalone-3/poster.jpg", - "wall_time_s": 464.249502 + "kinex_revision": null }, { "id": "kinex-3", - "task": "3", - "condition": "kinex", - "status": "complete", + "variant": "kinex", + "task_key": "task03/03", + "family": "task03", + "source_family": "task03", + "provider": "rosbridge", + "slot": "03", + "seed": 0, + "source_episode_id": "kinex-3", + "status": "completed", "success": true, "valid": true, - "control_steps": 1354, - "tool_calls": 59, + "execution": { + "status": "finished", + "reason": null + }, + "verdict": { + "evidence_valid": true, + "success": true, + "termination": "success", + "steps": 1354 + }, + "steps": 1354, + "simulation_time_s": 54.160000000000004, + "wall_time_s": 756.194745, + "model": "GPT-6 Astra", + "effort": "high", + "harness": "kinex", + "instruction": "use arms to grab the bottles and put them into the dustbin to the left of the table", + "native_instruction": "use arms to grab the bottles and put them into the dustbin to the left of the table", + "instruction_policy": "original_native", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "audit_source": "Provider terminal response usage", + "cache_hit_rate": 0.9721842236356547, + "cache_read_input_tokens": 1598336, + "cache_reported_input_tokens": 1644067, + "cache_write_basis": "provider-reported", + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 1644067, + "cached_input_tokens": 1598336, + "cost_usd": null, + "fresh_input_tokens": 45731, + "harbor_mismatches": {}, + "input_breakdown_complete": true, + "input_tokens": 1644067, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 1598336, + "known_fresh_input_tokens": 45731, + "known_input_tokens": 1644067, + "known_output_tokens": 10285, + "known_reasoning_output_tokens": 2439, + "output_tokens": 10285, + "parser": "harbor/0.22.0", + "reasoning_output_tokens": 2439, + "reasoning_reported_output_tokens": 10285, + "rejected_attempts": 0, + "reported_responses": { + "cache_reported_input_tokens": 49, + "cache_write_input_tokens": 49, + "cache_write_reported_input_tokens": 49, + "cached_input_tokens": 49, + "fresh_input_tokens": 49, + "input_tokens": 49, + "output_tokens": 49, + "reasoning_output_tokens": 49, + "reasoning_reported_output_tokens": 49 + }, + "request_attempts": 49, + "response_count": 49, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Harbor session metrics", + "total_tokens": 1654352, + "uncached_input_tokens": 45731, + "unresolved_attempts": [], + "usage_records": [ + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 0, + "input_tokens": 11304, + "output_tokens": 19, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11136, + "input_tokens": 11344, + "output_tokens": 29, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11264, + "input_tokens": 11477, + "output_tokens": 30, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11392, + "input_tokens": 14222, + "output_tokens": 164, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 14208, + "input_tokens": 17281, + "output_tokens": 173, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 17280, + "input_tokens": 20636, + "output_tokens": 54, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 20608, + "input_tokens": 21876, + "output_tokens": 98, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 21888, + "input_tokens": 22170, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 22016, + "input_tokens": 22678, + "output_tokens": 23, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 22528, + "input_tokens": 22870, + "output_tokens": 152, + "reasoning_output_tokens": 25 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 22912, + "input_tokens": 24389, + "output_tokens": 362, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 24576, + "input_tokens": 25147, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 25088, + "input_tokens": 25418, + "output_tokens": 354, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 25600, + "input_tokens": 27068, + "output_tokens": 321, + "reasoning_output_tokens": 247 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 27264, + "input_tokens": 28568, + "output_tokens": 1068, + "reasoning_output_tokens": 62 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 29440, + "input_tokens": 29657, + "output_tokens": 74, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 29568, + "input_tokens": 30414, + "output_tokens": 280, + "reasoning_output_tokens": 43 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 30592, + "input_tokens": 30842, + "output_tokens": 252, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 30976, + "input_tokens": 31389, + "output_tokens": 393, + "reasoning_output_tokens": 115 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 31616, + "input_tokens": 32053, + "output_tokens": 189, + "reasoning_output_tokens": 94 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 31872, + "input_tokens": 32718, + "output_tokens": 51, + "reasoning_output_tokens": 11 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 32640, + "input_tokens": 32791, + "output_tokens": 277, + "reasoning_output_tokens": 46 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 32896, + "input_tokens": 33530, + "output_tokens": 328, + "reasoning_output_tokens": 126 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 33664, + "input_tokens": 34081, + "output_tokens": 283, + "reasoning_output_tokens": 142 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34176, + "input_tokens": 34391, + "output_tokens": 99, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34304, + "input_tokens": 35033, + "output_tokens": 134, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 35072, + "input_tokens": 35705, + "output_tokens": 52, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 35584, + "input_tokens": 36128, + "output_tokens": 88, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 36096, + "input_tokens": 36756, + "output_tokens": 445, + "reasoning_output_tokens": 353 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 36992, + "input_tokens": 37742, + "output_tokens": 155, + "reasoning_output_tokens": 64 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 37760, + "input_tokens": 38332, + "output_tokens": 52, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 38272, + "input_tokens": 38748, + "output_tokens": 179, + "reasoning_output_tokens": 78 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 38784, + "input_tokens": 39356, + "output_tokens": 795, + "reasoning_output_tokens": 487 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 40064, + "input_tokens": 40560, + "output_tokens": 412, + "reasoning_output_tokens": 214 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 40832, + "input_tokens": 41227, + "output_tokens": 230, + "reasoning_output_tokens": 30 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 41344, + "input_tokens": 41999, + "output_tokens": 52, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 41856, + "input_tokens": 42422, + "output_tokens": 88, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 42368, + "input_tokens": 43056, + "output_tokens": 130, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 43136, + "input_tokens": 43618, + "output_tokens": 51, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 43520, + "input_tokens": 44040, + "output_tokens": 128, + "reasoning_output_tokens": 30 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 44032, + "input_tokens": 44597, + "output_tokens": 169, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 44672, + "input_tokens": 45306, + "output_tokens": 52, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 45184, + "input_tokens": 45725, + "output_tokens": 87, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 45696, + "input_tokens": 46351, + "output_tokens": 132, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 46464, + "input_tokens": 46912, + "output_tokens": 53, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 46848, + "input_tokens": 47103, + "output_tokens": 93, + "reasoning_output_tokens": 42 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 47104, + "input_tokens": 47579, + "output_tokens": 695, + "reasoning_output_tokens": 230 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 48128, + "input_tokens": 48301, + "output_tokens": 829, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 49024, + "input_tokens": 49157, + "output_tokens": 33, + "reasoning_output_tokens": 0 + } + ], + "usage_source": "provider-raw" + }, + "call_activity": { + "schema": "rlebench/call-activity/1", + "source": "Kinex native trace", + "model_tool_calls": 59, + "model_tool_calls_by_name": { + "ls": 3, + "read": 16, + "find": 1, + "exec": 31, + "devices": 2, + "device": 1, + "write": 2, + "device_stop": 1, + "edit": 2 + }, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "nested_python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_scope": "Recorded instrumented Python invocations only; other paths and automatic fatal stops are not counted." + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 543, + "published_frames": 543, + "source_sample_coverage": 1.0, + "source_samples_complete": true, + "source_dropped_samples": 0, + "terminal_frame_retained": true, + "terminal_source_index": 542, + "terminal_hold_s": 0, + "simulation_time_s": 54.160000000000004, + "duration_s": 13.575, + "sha256": "84e99f6003d005afb922686e9101691a134e397f0280974f074a60eac494d9f1", + "source_sha256": { + "third_person.mp4": "dba49f01b2b253ae4cc13159525fdf17948bfb17506fbc8f7a41d0281fd22a37", + "left_wrist.mp4": "4822fe685288688a8fc2f929b032de30a01239e8db2e07ed52042c9c4a57b96b", + "right_wrist.mp4": "72f18a15b2d2aaf41795d027dd461cae4377e49826e392859fdc6139515dbc5d" + }, + "all_frames_compared": 543, + "minimum_frame_psnr_db": 42.00768830044577, + "maximum_frame_rgb_mae": 1.432692289352417, + "decode_ok": true, + "pts_verified": true, + "playback_kind": "simulation_time", + "resolution_kind": "independent_720_high", + "agent_observation_video": false, + "camera_views": [ + { + "name": "third_person", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "left_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "right_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + } + ], + "coverage_basis": "Every scheduled 10 Hz control-boundary sample, plus the terminal state.", + "frame_mapping": "Each captured sample exactly once; no appended or held frames.", + "sampling_note": "Samples follow the native simulation clock; model thinking adds no footage.", + "scope": "Live owner-only spectator views; no replay, rerender, upscaling or invented frames.", + "recording": { + "schema": "rosbridge/recording/1", + "run_id": "c3ff59536ab0", + "task": "robotwin/put-bottles-dustbin-aloha-agilex", + "state": "complete", + "views": { + "third_person": { + "received": 543, + "dropped": 0, + "segments": [ + { + "file": "third_person.mp4", + "frames": 543, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 54160000000 + } + ], + "error": null + }, + "left_wrist": { + "received": 543, + "dropped": 0, + "segments": [ + { + "file": "left_wrist.mp4", + "frames": 543, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 54160000000 + } + ], + "error": null + }, + "right_wrist": { + "received": 543, + "dropped": 0, + "segments": [ + { + "file": "right_wrist.mp4", + "frames": 543, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 54160000000 + } + ], + "error": null + } + }, + "type": "mp4", + "spectator_views": [ + "third_person", + "left_wrist", + "right_wrist" + ], + "fps": 10.0, + "topics": [], + "error": null, + "bag_messages": 0 + } + }, + "analysis": { + "observed": "Kinex wrote a measured Cartesian manipulation tool, picked and deposited the bottles sequentially, and reached native success at control frame 1,354." + }, + "links": { + "video": "experiments/robotwin-native/kinex-3/presentation/video.mp4", + "poster": "experiments/robotwin-native/kinex-3/presentation/poster.jpg", + "terminal_frame": "experiments/robotwin-native/kinex-3/presentation/terminal.jpg", + "transcript": "experiments/robotwin-native/kinex-3/transcript.json", + "usage": "experiments/robotwin-native/kinex-3/usage.json", + "analysis": "experiments/robotwin-native/kinex-3/analysis.json", + "protocol": "experiments/robotwin-native/kinex-3/protocol.json", + "native_goal": "experiments/robotwin-native/kinex-3/instruction.json", + "verdict": "experiments/robotwin-native/kinex-3/episode.json", + "provenance": "sources.json", + "media_check": "experiments/robotwin-native/kinex-3/presentation/media-validation.json", + "recording_manifest": "experiments/robotwin-native/kinex-3/presentation/recording.json", + "workspace": "experiments/robotwin-native/kinex-3/workspace.tar.gz", + "workspace_changes": "experiments/robotwin-native/kinex-3/workspace.json", + "evidence_archive": "experiments/robotwin-native/kinex-3/evidence.tar.gz", + "tool_calls": "experiments/robotwin-native/kinex-3/tool-calls.json", + "artifact_index": "experiments/robotwin-native/kinex-3/artifact-index.json", + "native_session": "experiments/robotwin-native/kinex-3/session.jsonl", + "trace": "experiments/robotwin-native/kinex-3/trace.json", + "provider_usage": "experiments/robotwin-native/kinex-3/provider-requests.jsonl", + "vla_io": "experiments/robotwin-native/kinex-3/vla-io.json", + "policy_trace": null + }, + "resources": [ + { + "name": ".kinex/tools/manipulate.py", + "file": "experiments/robotwin-native/kinex-3/workspace/.kinex/tools/manipulate.py", + "kind": "Created during this episode; no imported earlier task resources." + }, + { + "name": ".kinex/memos/put-bottles-dustbin.md", + "file": "experiments/robotwin-native/kinex-3/workspace/.kinex/memos/put-bottles-dustbin.md", + "kind": "Created during this episode; no imported earlier task resources." + } + ], + "session_counts": { + "visible_events": 110, + "observed_images": 31 + }, + "measured_protocol": { + "max_steps": 1700, + "seed": 0, + "episodes": 1, + "control_frequency_hz": 25, + "provider": "rosbridge", + "realtime": false, + "stop_on_success": true + }, "vla_calls": 0, - "input_tokens": 1644067, - "output_tokens": 10285, - "cached_input_tokens": 1598336, - "api_cost_usd": 2.569896, - "path": "experiments/robotwin-native/kinex-3", - "video": "experiments/robotwin-native/kinex-3/combined.mp4", - "poster": "experiments/robotwin-native/kinex-3/poster.jpg", - "wall_time_s": 756.194745 + "kinex_revision": "f4ab59e9e93aa2f569d88386799a2bed0670d59c", + "kinex_version": "1.1.0" }, { "id": "kinex_policy-3", - "task": "3", - "condition": "kinex_policy", - "status": "complete", + "variant": "kinex_policy", + "task_key": "task03/03", + "family": "task03", + "source_family": "task03", + "provider": "rosbridge", + "slot": "03", + "seed": 0, + "source_episode_id": "kinex_policy-3", + "status": "completed", "success": true, "valid": true, - "control_steps": 1277, - "tool_calls": 69, + "execution": { + "status": "finished", + "reason": null + }, + "verdict": { + "evidence_valid": true, + "success": true, + "termination": "success", + "steps": 1277 + }, + "steps": 1277, + "simulation_time_s": 51.08, + "wall_time_s": 737.794664, + "model": "GPT-6 Astra", + "effort": "high", + "harness": "kinex", + "instruction": "use arms to grab the bottles and put them into the dustbin to the left of the table", + "native_instruction": "use arms to grab the bottles and put them into the dustbin to the left of the table", + "instruction_policy": "original_native", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "audit_source": "Provider terminal response usage", + "cache_hit_rate": 0.9726861083715472, + "cache_read_input_tokens": 2026752, + "cache_reported_input_tokens": 2083665, + "cache_write_basis": "provider-reported", + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 2083665, + "cached_input_tokens": 2026752, + "cost_usd": null, + "fresh_input_tokens": 56913, + "harbor_mismatches": {}, + "input_breakdown_complete": true, + "input_tokens": 2083665, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 2026752, + "known_fresh_input_tokens": 56913, + "known_input_tokens": 2083665, + "known_output_tokens": 5246, + "known_reasoning_output_tokens": 328, + "output_tokens": 5246, + "parser": "harbor/0.22.0", + "reasoning_output_tokens": 328, + "reasoning_reported_output_tokens": 5246, + "rejected_attempts": 0, + "reported_responses": { + "cache_reported_input_tokens": 60, + "cache_write_input_tokens": 60, + "cache_write_reported_input_tokens": 60, + "cached_input_tokens": 60, + "fresh_input_tokens": 60, + "input_tokens": 60, + "output_tokens": 60, + "reasoning_output_tokens": 60, + "reasoning_reported_output_tokens": 60 + }, + "request_attempts": 60, + "response_count": 60, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Harbor session metrics", + "total_tokens": 2088911, + "uncached_input_tokens": 56913, + "unresolved_attempts": [], + "usage_records": [ + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 0, + "input_tokens": 11303, + "output_tokens": 19, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11136, + "input_tokens": 11343, + "output_tokens": 29, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11264, + "input_tokens": 11476, + "output_tokens": 30, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 11392, + "input_tokens": 15499, + "output_tokens": 180, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 15360, + "input_tokens": 17418, + "output_tokens": 210, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 17280, + "input_tokens": 22354, + "output_tokens": 77, + "reasoning_output_tokens": 10 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 22272, + "input_tokens": 22900, + "output_tokens": 81, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 22784, + "input_tokens": 23208, + "output_tokens": 77, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 23040, + "input_tokens": 24559, + "output_tokens": 107, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 24448, + "input_tokens": 24883, + "output_tokens": 23, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 24704, + "input_tokens": 25073, + "output_tokens": 40, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 24960, + "input_tokens": 27403, + "output_tokens": 385, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 27264, + "input_tokens": 27811, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 27648, + "input_tokens": 28265, + "output_tokens": 61, + "reasoning_output_tokens": 22 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 28032, + "input_tokens": 28738, + "output_tokens": 38, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 28544, + "input_tokens": 29189, + "output_tokens": 38, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 28928, + "input_tokens": 29640, + "output_tokens": 49, + "reasoning_output_tokens": 9 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 29568, + "input_tokens": 30101, + "output_tokens": 37, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 29952, + "input_tokens": 30552, + "output_tokens": 37, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 30464, + "input_tokens": 31003, + "output_tokens": 38, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 30848, + "input_tokens": 31456, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 31360, + "input_tokens": 31913, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 31744, + "input_tokens": 32368, + "output_tokens": 39, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 32128, + "input_tokens": 32821, + "output_tokens": 38, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 32640, + "input_tokens": 33269, + "output_tokens": 414, + "reasoning_output_tokens": 35 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 33152, + "input_tokens": 33712, + "output_tokens": 72, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 33664, + "input_tokens": 34232, + "output_tokens": 44, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34176, + "input_tokens": 34724, + "output_tokens": 43, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 34560, + "input_tokens": 35218, + "output_tokens": 45, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 35072, + "input_tokens": 35712, + "output_tokens": 59, + "reasoning_output_tokens": 12 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 35584, + "input_tokens": 36237, + "output_tokens": 45, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 36096, + "input_tokens": 36747, + "output_tokens": 67, + "reasoning_output_tokens": 20 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 36608, + "input_tokens": 37276, + "output_tokens": 59, + "reasoning_output_tokens": 20 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 37120, + "input_tokens": 37660, + "output_tokens": 65, + "reasoning_output_tokens": 21 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 37504, + "input_tokens": 37935, + "output_tokens": 129, + "reasoning_output_tokens": 15 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 37888, + "input_tokens": 38650, + "output_tokens": 33, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 38528, + "input_tokens": 38788, + "output_tokens": 105, + "reasoning_output_tokens": 29 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 38656, + "input_tokens": 39125, + "output_tokens": 220, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 39040, + "input_tokens": 39697, + "output_tokens": 558, + "reasoning_output_tokens": 60 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 39552, + "input_tokens": 40278, + "output_tokens": 63, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 40192, + "input_tokens": 40564, + "output_tokens": 53, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 40448, + "input_tokens": 40840, + "output_tokens": 61, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 40704, + "input_tokens": 41123, + "output_tokens": 45, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 40960, + "input_tokens": 41615, + "output_tokens": 58, + "reasoning_output_tokens": 12 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 41472, + "input_tokens": 42118, + "output_tokens": 46, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 41984, + "input_tokens": 42611, + "output_tokens": 45, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 42368, + "input_tokens": 43105, + "output_tokens": 46, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 43008, + "input_tokens": 43598, + "output_tokens": 81, + "reasoning_output_tokens": 10 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 43520, + "input_tokens": 44118, + "output_tokens": 43, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 44032, + "input_tokens": 44607, + "output_tokens": 46, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 44416, + "input_tokens": 45102, + "output_tokens": 79, + "reasoning_output_tokens": 14 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 44928, + "input_tokens": 45404, + "output_tokens": 46, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 45184, + "input_tokens": 45896, + "output_tokens": 47, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 45696, + "input_tokens": 46388, + "output_tokens": 45, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 46208, + "input_tokens": 46878, + "output_tokens": 43, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 46720, + "input_tokens": 47354, + "output_tokens": 44, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 47232, + "input_tokens": 47529, + "output_tokens": 28, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 47360, + "input_tokens": 47619, + "output_tokens": 49, + "reasoning_output_tokens": 0 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 47488, + "input_tokens": 48050, + "output_tokens": 531, + "reasoning_output_tokens": 39 + }, + { + "cache_write_input_tokens": 0, + "cached_input_tokens": 47872, + "input_tokens": 48610, + "output_tokens": 39, + "reasoning_output_tokens": 0 + } + ], + "usage_source": "provider-raw" + }, + "call_activity": { + "schema": "rlebench/call-activity/1", + "source": "Kinex native trace", + "model_tool_calls": 69, + "model_tool_calls_by_name": { + "ls": 3, + "read": 15, + "find": 1, + "exec": 40, + "devices": 3, + "device": 1, + "write": 3, + "edit": 1, + "relative_pose": 1, + "device_stop": 1 + }, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "nested_python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_scope": "Recorded instrumented Python invocations only; other paths and automatic fatal stops are not counted." + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 512, + "published_frames": 512, + "source_sample_coverage": 1.0, + "source_samples_complete": true, + "source_dropped_samples": 0, + "terminal_frame_retained": true, + "terminal_source_index": 511, + "terminal_hold_s": 0, + "simulation_time_s": 51.08, + "duration_s": 12.8, + "sha256": "e182700df6a1b1b83c33ea1bef6ee088dec66690418ac77fb59dab6b0adb4d67", + "source_sha256": { + "third_person.mp4": "cc7356c76a92f5fa5a5ac2991295bdc3069c94774f69fef9b09e131fa538112f", + "left_wrist.mp4": "40132ed56a64bbbaac618077114b8f34d5f0befdbc1b18291531ad36e88730b9", + "right_wrist.mp4": "947a864a65109bd4bcbb11329f06bf31cbed24ad1ea3312ce7ea5e5b9547c94a" + }, + "all_frames_compared": 512, + "minimum_frame_psnr_db": 41.518423372042804, + "maximum_frame_rgb_mae": 1.4636712074279785, + "decode_ok": true, + "pts_verified": true, + "playback_kind": "simulation_time", + "resolution_kind": "independent_720_high", + "agent_observation_video": false, + "camera_views": [ + { + "name": "third_person", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "left_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + }, + { + "name": "right_wrist", + "width": 960, + "height": 720, + "channel": "spectator" + } + ], + "coverage_basis": "Every scheduled 10 Hz control-boundary sample, plus the terminal state.", + "frame_mapping": "Each captured sample exactly once; no appended or held frames.", + "sampling_note": "Samples follow the native simulation clock; model thinking adds no footage.", + "scope": "Live owner-only spectator views; no replay, rerender, upscaling or invented frames.", + "recording": { + "schema": "rosbridge/recording/1", + "run_id": "88c15864ae34", + "task": "robotwin/put-bottles-dustbin-aloha-agilex", + "state": "complete", + "views": { + "third_person": { + "received": 512, + "dropped": 0, + "segments": [ + { + "file": "third_person.mp4", + "frames": 512, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 51080000000 + } + ], + "error": null + }, + "left_wrist": { + "received": 512, + "dropped": 0, + "segments": [ + { + "file": "left_wrist.mp4", + "frames": 512, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 51080000000 + } + ], + "error": null + }, + "right_wrist": { + "received": 512, + "dropped": 0, + "segments": [ + { + "file": "right_wrist.mp4", + "frames": 512, + "width": 960, + "height": 720, + "fps": 10.0, + "first_stamp_ns": 0, + "last_stamp_ns": 51080000000 + } + ], + "error": null + } + }, + "type": "mp4", + "spectator_views": [ + "third_person", + "left_wrist", + "right_wrist" + ], + "fps": 10.0, + "topics": [], + "error": null, + "bag_messages": 0 + } + }, + "analysis": { + "observed": "Kinex made 72 policy inferences and wrote its own loop of up to four freshly observed chunks per tool call. When grasping stalled, it measured RGB-D geometry and wrote Cartesian corrections, then resumed the policy. Native success occurred at frame 1,277. Every VLA call retained the full original task instruction; this is observed control recovery, not language subtask rewriting." + }, + "links": { + "video": "experiments/robotwin-native/kinex_policy-3/presentation/video.mp4", + "poster": "experiments/robotwin-native/kinex_policy-3/presentation/poster.jpg", + "terminal_frame": "experiments/robotwin-native/kinex_policy-3/presentation/terminal.jpg", + "transcript": "experiments/robotwin-native/kinex_policy-3/transcript.json", + "usage": "experiments/robotwin-native/kinex_policy-3/usage.json", + "analysis": "experiments/robotwin-native/kinex_policy-3/analysis.json", + "protocol": "experiments/robotwin-native/kinex_policy-3/protocol.json", + "native_goal": "experiments/robotwin-native/kinex_policy-3/instruction.json", + "verdict": "experiments/robotwin-native/kinex_policy-3/episode.json", + "provenance": "sources.json", + "media_check": "experiments/robotwin-native/kinex_policy-3/presentation/media-validation.json", + "recording_manifest": "experiments/robotwin-native/kinex_policy-3/presentation/recording.json", + "workspace": "experiments/robotwin-native/kinex_policy-3/workspace.tar.gz", + "workspace_changes": "experiments/robotwin-native/kinex_policy-3/workspace.json", + "evidence_archive": "experiments/robotwin-native/kinex_policy-3/evidence.tar.gz", + "tool_calls": "experiments/robotwin-native/kinex_policy-3/tool-calls.json", + "artifact_index": "experiments/robotwin-native/kinex_policy-3/artifact-index.json", + "native_session": "experiments/robotwin-native/kinex_policy-3/session.jsonl", + "trace": "experiments/robotwin-native/kinex_policy-3/trace.json", + "provider_usage": "experiments/robotwin-native/kinex_policy-3/provider-requests.jsonl", + "vla_io": "experiments/robotwin-native/kinex_policy-3/vla-io.json", + "policy_trace": "experiments/robotwin-native/kinex_policy-3/policy/calls.jsonl" + }, + "resources": [ + { + "name": ".kinex/tools/bottle_policy.py", + "file": "experiments/robotwin-native/kinex_policy-3/workspace/.kinex/tools/bottle_policy.py", + "kind": "Created during this episode; no imported earlier task resources." + }, + { + "name": ".kinex/tools/cartesian_adjust.py", + "file": "experiments/robotwin-native/kinex_policy-3/workspace/.kinex/tools/cartesian_adjust.py", + "kind": "Created during this episode; no imported earlier task resources." + }, + { + "name": ".kinex/memos/put-bottles-dustbin.md", + "file": "experiments/robotwin-native/kinex_policy-3/workspace/.kinex/memos/put-bottles-dustbin.md", + "kind": "Created during this episode; no imported earlier task resources." + } + ], + "session_counts": { + "visible_events": 131, + "observed_images": 40 + }, + "measured_protocol": { + "max_steps": 1700, + "seed": 0, + "episodes": 1, + "control_frequency_hz": 25, + "provider": "rosbridge", + "realtime": false, + "stop_on_success": true + }, "vla_calls": 72, - "input_tokens": 2083665, - "output_tokens": 5246, - "cached_input_tokens": 2026752, - "api_cost_usd": 2.858182, - "path": "experiments/robotwin-native/kinex_policy-3", - "video": "experiments/robotwin-native/kinex_policy-3/combined.mp4", - "poster": "experiments/robotwin-native/kinex_policy-3/poster.jpg", - "wall_time_s": 737.794664 + "kinex_revision": "f4ab59e9e93aa2f569d88386799a2bed0670d59c", + "kinex_version": "1.1.0" + } + ], + "attempt_history": [], + "task_notes": {}, + "runtime_reviews": [], + "variant_sources": { + "standalone": { + "space": "RLE-Bench/programmable-vla", + "evidence_revision": "dd87d4989bae8af7f54039efc1f9f1cae389f6a6" + }, + "kinex": { + "space": "RLE-Bench/programmable-vla", + "evidence_revision": "dd87d4989bae8af7f54039efc1f9f1cae389f6a6" + }, + "kinex_policy": { + "space": "RLE-Bench/programmable-vla", + "evidence_revision": "dd87d4989bae8af7f54039efc1f9f1cae389f6a6" + } + }, + "budget_defaults": { + "tasks": { + "robotwin/handover-block-aloha-agilex": { + "action_step_limit": 800 + }, + "robotwin/place-dual-shoes-aloha-agilex": { + "action_step_limit": 600 + }, + "robotwin/put-bottles-dustbin-aloha-agilex": { + "action_step_limit": 1700 + } } - ] + }, + "template": { + "repo": "https://huggingface.co/spaces/RLE-Bench/kinex-benchmark-native-budget-with-robobridge", + "revision": "17e1414d138a2ff6a5c09f84dbb37d4e3c44f86d" + } }