File size: 5,211 Bytes
15b1dc7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 | [
{
"name": "Temporal Synchronization",
"category": "Alignment",
"question": "Are sensor streams, actions and events aligned on the correct timeline?",
"metrics": [
"timestamp error",
"cross-modal lag",
"event alignment accuracy"
],
"inputs": [
"sensor timestamps",
"action logs",
"event markers"
],
"failure": "Small timing offsets can create false cause-and-effect relationships."
},
{
"name": "Spatial Calibration",
"category": "Alignment",
"question": "Do observations agree in a common coordinate frame?",
"metrics": [
"reprojection error",
"pose error",
"extrinsic drift"
],
"inputs": [
"intrinsics",
"extrinsics",
"reference targets"
],
"failure": "Calibration drift corrupts geometry and state estimation."
},
{
"name": "Geometry Fidelity",
"category": "Reconstruction",
"question": "How accurately is real-world geometry preserved?",
"metrics": [
"surface distance",
"depth error",
"mesh completeness",
"scale error"
],
"inputs": [
"reference geometry",
"reconstruction",
"depth"
],
"failure": "Visual realism does not guarantee metric accuracy."
},
{
"name": "Object Persistence",
"category": "Reconstruction",
"question": "Are objects and agents tracked consistently over time?",
"metrics": [
"ID switches",
"track fragmentation",
"trajectory error"
],
"inputs": [
"tracks",
"ground-truth IDs",
"trajectories"
],
"failure": "Identity errors break long-horizon interaction histories."
},
{
"name": "Semantic Accuracy",
"category": "Semantics",
"question": "Are objects, relations, tasks and affordances represented correctly?",
"metrics": [
"object accuracy",
"relation F1",
"affordance accuracy"
],
"inputs": [
"labels",
"scene graph",
"task annotations"
],
"failure": "Correct geometry can still be unusable when semantics are wrong."
},
{
"name": "Action Alignment",
"category": "Interaction",
"question": "Are actions paired with the exact state transitions they produced?",
"metrics": [
"action-state lag",
"transition pairing accuracy",
"causal consistency"
],
"inputs": [
"state_t",
"action_t",
"state_t+1"
],
"failure": "Misaligned actions damage control and action-conditioned world models."
},
{
"name": "State Completeness",
"category": "Representation",
"question": "Does the world state preserve variables required by downstream tasks?",
"metrics": [
"state coverage",
"missing-variable rate",
"task sufficiency"
],
"inputs": [
"compiled state",
"task requirements"
],
"failure": "Overcompression may remove variables needed for planning."
},
{
"name": "Representation Efficiency",
"category": "Representation",
"question": "How compact is the representation relative to preserved information?",
"metrics": [
"compression ratio",
"bytes per step",
"encoding latency"
],
"inputs": [
"raw episode",
"compiled representation"
],
"failure": "An accurate representation may still be too costly to scale."
},
{
"name": "Uncertainty Calibration",
"category": "Reliability",
"question": "Does confidence reflect actual compilation quality?",
"metrics": [
"calibration error",
"coverage",
"confidence-error correlation"
],
"inputs": [
"confidence scores",
"ground-truth error"
],
"failure": "Overconfident world states are dangerous for autonomous systems."
},
{
"name": "Real2Sim Fidelity",
"category": "Simulation",
"question": "Does simulation reproduce relevant real-world behavior?",
"metrics": [
"trajectory divergence",
"contact consistency",
"sensor similarity",
"physics error"
],
"inputs": [
"real trajectories",
"sim trajectories",
"contact events"
],
"failure": "Visual similarity alone does not guarantee useful physical behavior."
},
{
"name": "Downstream World-Model Utility",
"category": "Downstream",
"question": "Does compiled data improve world-model learning and rollout quality?",
"metrics": [
"prediction error",
"rollout drift",
"sample efficiency"
],
"inputs": [
"compiled dataset",
"world model",
"evaluation tasks"
],
"failure": "Clean representations may still fail to improve predictive modeling."
},
{
"name": "Planning & Control Utility",
"category": "Downstream",
"question": "Does world compilation improve decisions, planning or control?",
"metrics": [
"task success",
"planning regret",
"control return",
"policy transfer"
],
"inputs": [
"planner",
"controller",
"compiled world",
"task environment"
],
"failure": "The strongest test is whether better compilation produces better actions."
}
] |