benchmarks.json

[
  {
    "family": "Latent Dynamics",
    "capability": "State prediction",
    "metrics": [
      "Latent prediction error",
      "Reconstruction quality",
      "Rollout stability"
    ],
    "best_for": "Compact predictive models and model-based RL",
    "questions": "Does the latent state preserve information needed for future prediction?"
  },
  {
    "family": "Visual Prediction",
    "capability": "Future observation prediction",
    "metrics": [
      "Perceptual similarity",
      "Temporal consistency",
      "Object persistence"
    ],
    "best_for": "Video and image-based world models",
    "questions": "Are generated future observations coherent and useful?"
  },
  {
    "family": "Interactive Environments",
    "capability": "Action-conditioned simulation",
    "metrics": [
      "Action fidelity",
      "Controllability",
      "State consistency"
    ],
    "best_for": "Interactive generative environments",
    "questions": "Do control inputs produce stable and meaningful changes?"
  },
  {
    "family": "Model-Based Reinforcement Learning",
    "capability": "Planning utility",
    "metrics": [
      "Return",
      "Sample efficiency",
      "Planning success",
      "Model error"
    ],
    "best_for": "Agents learning through imagined trajectories",
    "questions": "Does the learned world model improve decisions and reduce environment interaction?"
  },
  {
    "family": "Robotics & Manipulation",
    "capability": "Physical interaction",
    "metrics": [
      "Task success",
      "Trajectory error",
      "Action success",
      "Safety violations"
    ],
    "best_for": "Robot control and manipulation",
    "questions": "Can the model predict and support physically grounded actions?"
  },
  {
    "family": "Embodied Navigation",
    "capability": "Spatial planning",
    "metrics": [
      "Navigation success",
      "Path efficiency",
      "Localization error",
      "Goal completion"
    ],
    "best_for": "Embodied agents and mobile robots",
    "questions": "Does the model preserve useful spatial structure over time?"
  },
  {
    "family": "Autonomous Systems",
    "capability": "Trajectory forecasting",
    "metrics": [
      "Trajectory error",
      "Collision rate",
      "Calibration",
      "Scenario coverage"
    ],
    "best_for": "Driving, drones and autonomous machines",
    "questions": "Can the system model multiple actors and plausible future trajectories?"
  },
  {
    "family": "Physical Reasoning",
    "capability": "Physics consistency",
    "metrics": [
      "Constraint violations",
      "Interaction accuracy",
      "State transition validity"
    ],
    "best_for": "Physical AI and simulation",
    "questions": "Do generated dynamics respect useful physical regularities?"
  },
  {
    "family": "Spatial Intelligence",
    "capability": "3D and relational consistency",
    "metrics": [
      "Pose error",
      "Depth consistency",
      "Object relation accuracy",
      "Map consistency"
    ],
    "best_for": "3D, scene understanding and spatial AI",
    "questions": "Does the model maintain geometry and object relationships?"
  },
  {
    "family": "Agent Planning",
    "capability": "Long-horizon decision support",
    "metrics": [
      "Goal success",
      "Plan validity",
      "Recovery rate",
      "Action efficiency"
    ],
    "best_for": "Digital and embodied agents",
    "questions": "Can the model support useful multi-step plans under uncertainty?"
  },
  {
    "family": "Long-Horizon Simulation",
    "capability": "Rollout stability",
    "metrics": [
      "Drift",
      "Consistency decay",
      "Error accumulation",
      "State validity"
    ],
    "best_for": "Persistent simulations and long rollouts",
    "questions": "How quickly do errors compound as the imagined future gets longer?"
  },
  {
    "family": "Multimodal World Modeling",
    "capability": "Cross-modal state understanding",
    "metrics": [
      "Cross-modal consistency",
      "Alignment",
      "Prediction quality",
      "Task utility"
    ],
    "best_for": "Vision-language-sensor world models",
    "questions": "Can multiple modalities contribute to one coherent world representation?"
  }
]