| [ |
| { |
| "name": "Temporal Synchronization", |
| "category": "Alignment", |
| "question": "Are sensor streams, actions and events aligned on the correct timeline?", |
| "metrics": [ |
| "timestamp error", |
| "cross-modal lag", |
| "event alignment accuracy" |
| ], |
| "inputs": [ |
| "sensor timestamps", |
| "action logs", |
| "event markers" |
| ], |
| "failure": "Small timing offsets can create false cause-and-effect relationships." |
| }, |
| { |
| "name": "Spatial Calibration", |
| "category": "Alignment", |
| "question": "Do observations agree in a common coordinate frame?", |
| "metrics": [ |
| "reprojection error", |
| "pose error", |
| "extrinsic drift" |
| ], |
| "inputs": [ |
| "intrinsics", |
| "extrinsics", |
| "reference targets" |
| ], |
| "failure": "Calibration drift corrupts geometry and state estimation." |
| }, |
| { |
| "name": "Geometry Fidelity", |
| "category": "Reconstruction", |
| "question": "How accurately is real-world geometry preserved?", |
| "metrics": [ |
| "surface distance", |
| "depth error", |
| "mesh completeness", |
| "scale error" |
| ], |
| "inputs": [ |
| "reference geometry", |
| "reconstruction", |
| "depth" |
| ], |
| "failure": "Visual realism does not guarantee metric accuracy." |
| }, |
| { |
| "name": "Object Persistence", |
| "category": "Reconstruction", |
| "question": "Are objects and agents tracked consistently over time?", |
| "metrics": [ |
| "ID switches", |
| "track fragmentation", |
| "trajectory error" |
| ], |
| "inputs": [ |
| "tracks", |
| "ground-truth IDs", |
| "trajectories" |
| ], |
| "failure": "Identity errors break long-horizon interaction histories." |
| }, |
| { |
| "name": "Semantic Accuracy", |
| "category": "Semantics", |
| "question": "Are objects, relations, tasks and affordances represented correctly?", |
| "metrics": [ |
| "object accuracy", |
| "relation F1", |
| "affordance accuracy" |
| ], |
| "inputs": [ |
| "labels", |
| "scene graph", |
| "task annotations" |
| ], |
| "failure": "Correct geometry can still be unusable when semantics are wrong." |
| }, |
| { |
| "name": "Action Alignment", |
| "category": "Interaction", |
| "question": "Are actions paired with the exact state transitions they produced?", |
| "metrics": [ |
| "action-state lag", |
| "transition pairing accuracy", |
| "causal consistency" |
| ], |
| "inputs": [ |
| "state_t", |
| "action_t", |
| "state_t+1" |
| ], |
| "failure": "Misaligned actions damage control and action-conditioned world models." |
| }, |
| { |
| "name": "State Completeness", |
| "category": "Representation", |
| "question": "Does the world state preserve variables required by downstream tasks?", |
| "metrics": [ |
| "state coverage", |
| "missing-variable rate", |
| "task sufficiency" |
| ], |
| "inputs": [ |
| "compiled state", |
| "task requirements" |
| ], |
| "failure": "Overcompression may remove variables needed for planning." |
| }, |
| { |
| "name": "Representation Efficiency", |
| "category": "Representation", |
| "question": "How compact is the representation relative to preserved information?", |
| "metrics": [ |
| "compression ratio", |
| "bytes per step", |
| "encoding latency" |
| ], |
| "inputs": [ |
| "raw episode", |
| "compiled representation" |
| ], |
| "failure": "An accurate representation may still be too costly to scale." |
| }, |
| { |
| "name": "Uncertainty Calibration", |
| "category": "Reliability", |
| "question": "Does confidence reflect actual compilation quality?", |
| "metrics": [ |
| "calibration error", |
| "coverage", |
| "confidence-error correlation" |
| ], |
| "inputs": [ |
| "confidence scores", |
| "ground-truth error" |
| ], |
| "failure": "Overconfident world states are dangerous for autonomous systems." |
| }, |
| { |
| "name": "Real2Sim Fidelity", |
| "category": "Simulation", |
| "question": "Does simulation reproduce relevant real-world behavior?", |
| "metrics": [ |
| "trajectory divergence", |
| "contact consistency", |
| "sensor similarity", |
| "physics error" |
| ], |
| "inputs": [ |
| "real trajectories", |
| "sim trajectories", |
| "contact events" |
| ], |
| "failure": "Visual similarity alone does not guarantee useful physical behavior." |
| }, |
| { |
| "name": "Downstream World-Model Utility", |
| "category": "Downstream", |
| "question": "Does compiled data improve world-model learning and rollout quality?", |
| "metrics": [ |
| "prediction error", |
| "rollout drift", |
| "sample efficiency" |
| ], |
| "inputs": [ |
| "compiled dataset", |
| "world model", |
| "evaluation tasks" |
| ], |
| "failure": "Clean representations may still fail to improve predictive modeling." |
| }, |
| { |
| "name": "Planning & Control Utility", |
| "category": "Downstream", |
| "question": "Does world compilation improve decisions, planning or control?", |
| "metrics": [ |
| "task success", |
| "planning regret", |
| "control return", |
| "policy transfer" |
| ], |
| "inputs": [ |
| "planner", |
| "controller", |
| "compiled world", |
| "task environment" |
| ], |
| "failure": "The strongest test is whether better compilation produces better actions." |
| } |
| ] |
