evaluation.json

{
  "schema": "gdc_yellowcab_public_evaluation_v1",
  "checkpoint": {
    "source_file_sha256": "0cb9c71d5be8f7671029e459c24179863ae2abff3ccfec1b5837c169d7d6c703",
    "published_file": "model.safetensors",
    "published_sha256": "af1a98cd96bb68d642f7a0adeedfacac37b230917afc15f3c87230a99a15b8a2",
    "training_manifest_sha256": "8b92ad563892776cc25148f39fec6bf207e1e442c618b58fb07af7e2b899364d",
    "split_sha256": "bfc5e960c8af32d0ea75a5e293bb95b3125111ace2cf8452eb42feb4bc2a1a5e"
  },
  "causal_input_audit": {
    "policy": "When past-to-current displacement is below 4 m, encode unknown current heading as sin=0 and cos=1. No future bearing is used.",
    "corrected_rows": 5974,
    "total_rows": 74680,
    "future_derived_model_inputs": 0
  },
  "protocol": {
    "label_type": "GPS-derived silver maneuver proxy",
    "forecast_horizon": "approximately 20 seconds",
    "split_policy": "Complete forward-in-time route holdout; no frame from a validation or test route enters training.",
    "dataset_samples": 74680,
    "dataset_routes": 36,
    "test_samples": 8594,
    "test_routes": 5,
    "training_seed": 20260726,
    "route_bootstrap_iterations": 500
  },
  "primary_result": {
    "model": "Temporal efficientnet_b0 + telemetry",
    "samples": 8594,
    "accuracy": 0.5755178031184547,
    "balanced_accuracy": 0.39156334041323,
    "macro_f1": 0.3915542657655546,
    "weighted_f1": 0.5807008272180106,
    "log_loss": 1.0841176199931875,
    "brier_score": 0.5563128377237149,
    "ece_15_bin": 0.011817411647322817,
    "p95_latency_ms": 95.37229499110252,
    "continue_recall": 0.7393577296489918,
    "continue_f1": 0.7492904446546831,
    "slow_recall": 0.35436382754994744,
    "slow_f1": 0.37716843872411865,
    "stop_recall": 0.3422712933753943,
    "stop_f1": 0.3580858085808581,
    "turn_left_recall": 0.3157262905162065,
    "turn_left_f1": 0.27568134171907754,
    "turn_right_recall": 0.20609756097560974,
    "turn_right_f1": 0.19754529514903565,
    "macro_f1_route_ci_low": 0.34762239048615595,
    "macro_f1_route_ci_high": 0.42805096076291543
  },
  "baselines": [
    {
      "model": "Temporal efficientnet_b0 + telemetry",
      "samples": 8594,
      "accuracy": 0.5755178031184547,
      "balanced_accuracy": 0.39156334041323,
      "macro_f1": 0.3915542657655546,
      "weighted_f1": 0.5807008272180106,
      "log_loss": 1.0841176199931877,
      "brier_score": 0.5563128377237149,
      "ece_15_bin": 0.0118174116473228,
      "p95_latency_ms": 95.37229499110252,
      "continue_recall": 0.7393577296489918,
      "continue_f1": 0.7492904446546831,
      "slow_recall": 0.3543638275499474,
      "slow_f1": 0.3771684387241186,
      "stop_recall": 0.3422712933753943,
      "stop_f1": 0.3580858085808581,
      "turn_left_recall": 0.3157262905162065,
      "turn_left_f1": 0.2756813417190775,
      "turn_right_recall": 0.2060975609756097,
      "turn_right_f1": 0.1975452951490356,
      "macro_f1_route_ci_low": 0.3476223904861559,
      "macro_f1_route_ci_high": 0.4280509607629154
    },
    {
      "model": "Telemetry gradient boosting",
      "samples": 8594,
      "accuracy": 0.4147079357691413,
      "balanced_accuracy": 0.4216633099655398,
      "macro_f1": 0.341392924380058,
      "weighted_f1": 0.4530356863806382,
      "log_loss": 1.2490630281069546,
      "brier_score": 0.6566714449096165,
      "ece_15_bin": 0.1216700885218681,
      "p95_latency_ms": NaN,
      "continue_recall": 0.4051530993278566,
      "continue_f1": 0.5513911828230212,
      "slow_recall": 0.6466876971608833,
      "slow_f1": 0.3416666666666667,
      "stop_recall": 0.4889589905362776,
      "stop_f1": 0.2986512524084778,
      "turn_left_recall": 0.2797118847539016,
      "turn_left_f1": 0.2583148558758315,
      "turn_right_recall": 0.2878048780487804,
      "turn_right_f1": 0.2569406641262928,
      "macro_f1_route_ci_low": 0.3149153283503967,
      "macro_f1_route_ci_high": 0.3493997015446531
    },
    {
      "model": "Vision linear (efficientnet_b0)",
      "samples": 8594,
      "accuracy": 0.5751687223644403,
      "balanced_accuracy": 0.2967510353079744,
      "macro_f1": 0.305034605073554,
      "weighted_f1": 0.5344562546214928,
      "log_loss": 10.24602122531317,
      "brier_score": 0.8349054717132016,
      "ece_15_bin": 0.4091336187074714,
      "p95_latency_ms": NaN,
      "continue_recall": 0.8269230769230769,
      "continue_f1": 0.7406354515050168,
      "slow_recall": 0.1556256572029442,
      "slow_f1": 0.2148040638606676,
      "stop_recall": 0.252365930599369,
      "stop_f1": 0.255591054313099,
      "turn_left_recall": 0.1500600240096038,
      "turn_left_f1": 0.1810282404055032,
      "turn_right_recall": 0.098780487804878,
      "turn_right_f1": 0.1331142152834839,
      "macro_f1_route_ci_low": 0.2678056005630319,
      "macro_f1_route_ci_high": 0.3273143437147999
    },
    {
      "model": "Past-motion rule",
      "samples": 8594,
      "accuracy": 0.4549685827321387,
      "balanced_accuracy": 0.223773826591683,
      "macro_f1": 0.2235297785953733,
      "weighted_f1": 0.4532542316647034,
      "log_loss": 2.058490921490782,
      "brier_score": 0.966304980218757,
      "ece_15_bin": 0.4450314172678615,
      "p95_latency_ms": NaN,
      "continue_recall": 0.6562733383121733,
      "continue_f1": 0.6534064504136072,
      "slow_recall": 0.1019978969505783,
      "slow_f1": 0.0962779156327543,
      "stop_recall": 0.0,
      "stop_f1": 0.0,
      "turn_left_recall": 0.1776710684273709,
      "turn_left_f1": 0.1792852816474863,
      "turn_right_recall": 0.1829268292682926,
      "turn_right_f1": 0.1886792452830188,
      "macro_f1_route_ci_low": 0.1873463820585989,
      "macro_f1_route_ci_high": 0.2345195759071031
    },
    {
      "model": "Telemetry logistic",
      "samples": 8594,
      "accuracy": 0.196299744007447,
      "balanced_accuracy": 0.2871892331606019,
      "macro_f1": 0.1886698746182765,
      "weighted_f1": 0.1861407158506876,
      "log_loss": 1.5832758165931655,
      "brier_score": 0.8215841116420504,
      "ece_15_bin": 0.1432688177654283,
      "p95_latency_ms": NaN,
      "continue_recall": 0.109596713965646,
      "continue_f1": 0.1816212871287128,
      "slow_recall": 0.601472134595163,
      "slow_f1": 0.2506024096385542,
      "stop_recall": 0.3706624605678233,
      "stop_f1": 0.1760959160734357,
      "turn_left_recall": 0.1956782713085234,
      "turn_left_f1": 0.1721224920802534,
      "turn_right_recall": 0.1585365853658536,
      "turn_right_f1": 0.162907268170426,
      "macro_f1_route_ci_low": 0.1839562547318565,
      "macro_f1_route_ci_high": 0.2044359347158287
    },
    {
      "model": "Majority class",
      "samples": 8594,
      "accuracy": 0.6232255061670933,
      "balanced_accuracy": 0.2,
      "macro_f1": 0.1535770609318996,
      "weighted_f1": 0.4785657076746885,
      "log_loss": 1.1837257781136636,
      "brier_score": 0.5768401032344809,
      "ece_15_bin": 0.032749657591408,
      "p95_latency_ms": NaN,
      "continue_recall": 1.0,
      "continue_f1": 0.7678853046594982,
      "slow_recall": 0.0,
      "slow_f1": 0.0,
      "stop_recall": 0.0,
      "stop_f1": 0.0,
      "turn_left_recall": 0.0,
      "turn_left_f1": 0.0,
      "turn_right_recall": 0.0,
      "turn_right_f1": 0.0,
      "macro_f1_route_ci_low": 0.1460915032679738,
      "macro_f1_route_ci_high": 0.1554760284937855
    }
  ],
  "per_class": [
    {
      "model": "Temporal efficientnet_b0 + telemetry",
      "class": "continue",
      "precision": 0.759493670886076,
      "recall": 0.7393577296489918,
      "f1": 0.7492904446546831,
      "support": 5356
    },
    {
      "model": "Temporal efficientnet_b0 + telemetry",
      "class": "slow",
      "precision": 0.40311004784689,
      "recall": 0.3543638275499474,
      "f1": 0.3771684387241186,
      "support": 951
    },
    {
      "model": "Temporal efficientnet_b0 + telemetry",
      "class": "stop",
      "precision": 0.3754325259515571,
      "recall": 0.3422712933753943,
      "f1": 0.3580858085808581,
      "support": 634
    },
    {
      "model": "Temporal efficientnet_b0 + telemetry",
      "class": "turn_left",
      "precision": 0.2446511627906976,
      "recall": 0.3157262905162065,
      "f1": 0.2756813417190775,
      "support": 833
    },
    {
      "model": "Temporal efficientnet_b0 + telemetry",
      "class": "turn_right",
      "precision": 0.1896745230078563,
      "recall": 0.2060975609756097,
      "f1": 0.1975452951490356,
      "support": 820
    }
  ],
  "subsets": [
    {
      "model": "Temporal efficientnet_b0 + telemetry",
      "subset": "all_test",
      "samples": 8594,
      "macro_f1": 0.3915542657655546,
      "balanced_accuracy": 0.39156334041323,
      "accuracy": 0.5755178031184547
    },
    {
      "model": "Temporal efficientnet_b0 + telemetry",
      "subset": "unseen_grid_cells",
      "samples": 1846,
      "macro_f1": 0.3342824235159851,
      "balanced_accuracy": 0.3360174291252364,
      "accuracy": 0.5384615384615384
    },
    {
      "model": "Temporal efficientnet_b0 + telemetry",
      "subset": "night_utc",
      "samples": 6930,
      "macro_f1": 0.4102137401798389,
      "balanced_accuracy": 0.4080902962744183,
      "accuracy": 0.5946608946608947
    },
    {
      "model": "Temporal efficientnet_b0 + telemetry",
      "subset": "gps_accuracy_le_10m",
      "samples": 8319,
      "macro_f1": 0.393316436996118,
      "balanced_accuracy": 0.3936744515708604,
      "accuracy": 0.5783147012862123
    }
  ],
  "selective_prediction": [
    {
      "confidence_threshold": 0.0,
      "coverage": 1.0,
      "accepted_examples": 8594,
      "selective_accuracy": 0.5755178031184547,
      "selective_macro_f1": 0.3915542657655546
    },
    {
      "confidence_threshold": 0.25,
      "coverage": 0.9987200372352804,
      "accepted_examples": 8583,
      "selective_accuracy": 0.5760223698007689,
      "selective_macro_f1": 0.3919543899984051
    },
    {
      "confidence_threshold": 0.35,
      "coverage": 0.9100535257156156,
      "accepted_examples": 7821,
      "selective_accuracy": 0.6006904487917146,
      "selective_macro_f1": 0.4060706565050537
    },
    {
      "confidence_threshold": 0.45,
      "coverage": 0.6947870607400511,
      "accepted_examples": 5971,
      "selective_accuracy": 0.658348685312343,
      "selective_macro_f1": 0.4327252057994619
    },
    {
      "confidence_threshold": 0.55,
      "coverage": 0.4717244589248313,
      "accepted_examples": 4054,
      "selective_accuracy": 0.7323630981746423,
      "selective_macro_f1": 0.470227025478709
    },
    {
      "confidence_threshold": 0.65,
      "coverage": 0.3019548522224808,
      "accepted_examples": 2595,
      "selective_accuracy": 0.8092485549132948,
      "selective_macro_f1": 0.5275062675719726
    },
    {
      "confidence_threshold": 0.75,
      "coverage": 0.1748894577612287,
      "accepted_examples": 1503,
      "selective_accuracy": 0.8782435129740519,
      "selective_macro_f1": 0.5566727737271018
    },
    {
      "confidence_threshold": 0.85,
      "coverage": 0.0829648592040958,
      "accepted_examples": 713,
      "selective_accuracy": 0.938288920056101,
      "selective_macro_f1": 0.6097924940442192
    }
  ],
  "calibration": {
    "method": "temperature scaling",
    "temperature": 1.106177568435669,
    "fit_split": "validation after restoring the best checkpoint",
    "independent_calibration_split": false
  },
  "limitations": [
    "Labels are GPS-derived proxies rather than human safety annotations.",
    "The test set contains five held-out routes from one operating domain.",
    "Raw accuracy is distorted by the majority continue class; macro F1 is primary.",
    "The calibration set is the validation split, not an independent fourth split.",
    "This evaluation is not evidence of safe vehicle control."
  ]
}