{
  "completed_jobs": [
    {
      "directory": "run-01",
      "model": "reflex-1",
      "cohort": "MOBILE_ROUTING9 (180)",
      "summary": {
        "complete": true,
        "all_requests": {
          "total": 180,
          "correct": 164,
          "accuracy": 0.9111111111111111,
          "macro_f1": 0.9106480025368284,
          "failures": 0,
          "confusion_matrix": {
            "answer": {
              "answer": 17,
              "get_device_info": 0,
              "get_battery_info": 0,
              "get_storage_info": 0,
              "get_media_info": 0,
              "search_local_files": 0,
              "search_notes": 1,
              "save_note": 2,
              "phone_health_check": 0
            },
            "get_device_info": {
              "answer": 2,
              "get_device_info": 17,
              "get_battery_info": 0,
              "get_storage_info": 0,
              "get_media_info": 1,
              "search_local_files": 0,
              "search_notes": 0,
              "save_note": 0,
              "phone_health_check": 0
            },
            "get_battery_info": {
              "answer": 2,
              "get_device_info": 0,
              "get_battery_info": 15,
              "get_storage_info": 0,
              "get_media_info": 1,
              "search_local_files": 0,
              "search_notes": 0,
              "save_note": 0,
              "phone_health_check": 2
            },
            "get_storage_info": {
              "answer": 0,
              "get_device_info": 1,
              "get_battery_info": 0,
              "get_storage_info": 17,
              "get_media_info": 2,
              "search_local_files": 0,
              "search_notes": 0,
              "save_note": 0,
              "phone_health_check": 0
            },
            "get_media_info": {
              "answer": 1,
              "get_device_info": 0,
              "get_battery_info": 0,
              "get_storage_info": 0,
              "get_media_info": 19,
              "search_local_files": 0,
              "search_notes": 0,
              "save_note": 0,
              "phone_health_check": 0
            },
            "search_local_files": {
              "answer": 0,
              "get_device_info": 1,
              "get_battery_info": 0,
              "get_storage_info": 0,
              "get_media_info": 0,
              "search_local_files": 19,
              "search_notes": 0,
              "save_note": 0,
              "phone_health_check": 0
            },
            "search_notes": {
              "answer": 0,
              "get_device_info": 0,
              "get_battery_info": 0,
              "get_storage_info": 0,
              "get_media_info": 0,
              "search_local_files": 0,
              "search_notes": 20,
              "save_note": 0,
              "phone_health_check": 0
            },
            "save_note": {
              "answer": 0,
              "get_device_info": 0,
              "get_battery_info": 0,
              "get_storage_info": 0,
              "get_media_info": 0,
              "search_local_files": 0,
              "search_notes": 0,
              "save_note": 20,
              "phone_health_check": 0
            },
            "phone_health_check": {
              "answer": 0,
              "get_device_info": 0,
              "get_battery_info": 0,
              "get_storage_info": 0,
              "get_media_info": 0,
              "search_local_files": 0,
              "search_notes": 0,
              "save_note": 0,
              "phone_health_check": 20
            }
          },
          "mean_latency_ms": 31364.1449391611,
          "p50_latency_ms": 32300.51620800009,
          "p95_latency_ms": 33802.05279199981,
          "reference_agreement": 0.9444444444444444,
          "parity_valid_cases": 180
        },
        "first_request_ms": 27731.913458000003,
        "warm_requests": {
          "total": 179,
          "correct": 163,
          "accuracy": 0.9106145251396648,
          "macro_f1": 0.9095898014786273,
          "failures": 0,
          "confusion_matrix": {
            "answer": {
              "answer": 16,
              "get_device_info": 0,
              "get_battery_info": 0,
              "get_storage_info": 0,
              "get_media_info": 0,
              "search_local_files": 0,
              "search_notes": 1,
              "save_note": 2,
              "phone_health_check": 0
            },
            "get_device_info": {
              "answer": 2,
              "get_device_info": 17,
              "get_battery_info": 0,
              "get_storage_info": 0,
              "get_media_info": 1,
              "search_local_files": 0,
              "search_notes": 0,
              "save_note": 0,
              "phone_health_check": 0
            },
            "get_battery_info": {
              "answer": 2,
              "get_device_info": 0,
              "get_battery_info": 15,
              "get_storage_info": 0,
              "get_media_info": 1,
              "search_local_files": 0,
              "search_notes": 0,
              "save_note": 0,
              "phone_health_check": 2
            },
            "get_storage_info": {
              "answer": 0,
              "get_device_info": 1,
              "get_battery_info": 0,
              "get_storage_info": 17,
              "get_media_info": 2,
              "search_local_files": 0,
              "search_notes": 0,
              "save_note": 0,
              "phone_health_check": 0
            },
            "get_media_info": {
              "answer": 1,
              "get_device_info": 0,
              "get_battery_info": 0,
              "get_storage_info": 0,
              "get_media_info": 19,
              "search_local_files": 0,
              "search_notes": 0,
              "save_note": 0,
              "phone_health_check": 0
            },
            "search_local_files": {
              "answer": 0,
              "get_device_info": 1,
              "get_battery_info": 0,
              "get_storage_info": 0,
              "get_media_info": 0,
              "search_local_files": 19,
              "search_notes": 0,
              "save_note": 0,
              "phone_health_check": 0
            },
            "search_notes": {
              "answer": 0,
              "get_device_info": 0,
              "get_battery_info": 0,
              "get_storage_info": 0,
              "get_media_info": 0,
              "search_local_files": 0,
              "search_notes": 20,
              "save_note": 0,
              "phone_health_check": 0
            },
            "save_note": {
              "answer": 0,
              "get_device_info": 0,
              "get_battery_info": 0,
              "get_storage_info": 0,
              "get_media_info": 0,
              "search_local_files": 0,
              "search_notes": 0,
              "save_note": 20,
              "phone_health_check": 0
            },
            "phone_health_check": {
              "answer": 0,
              "get_device_info": 0,
              "get_battery_info": 0,
              "get_storage_info": 0,
              "get_media_info": 0,
              "search_local_files": 0,
              "search_notes": 0,
              "save_note": 0,
              "phone_health_check": 20
            }
          },
          "mean_latency_ms": 31384.43673514524,
          "p50_latency_ms": 32342.07145800019,
          "p95_latency_ms": 33865.3933329997,
          "reference_agreement": 0.9441340782122905,
          "parity_valid_cases": 179
        },
        "end_unix": 1792023255.933782
      },
      "resources": {
        "max_android_thermal_status": 3,
        "battery_c_min": 28.9,
        "battery_c_max": 37.5,
        "ap_c_max": 52.6,
        "skin_c_max": 42.6,
        "sampled_max_pss_kib": 1116821,
        "rss_high_water_kib": 1119236
      },
      "token_usage": {
        "requests_with_input_usage": 180,
        "mean_reported_input_tokens": 456.1722222222222,
        "mean_output_tokens": 0,
        "requested_cache_values": [],
        "requests_with_cache_reporting": 0,
        "requests_reporting_nonzero_cached_tokens": 0,
        "mean_reported_cached_tokens": null,
        "cached_fraction_of_reported_input_tokens": null,
        "mean_server_prompt_evaluated_tokens": null,
        "note": "Measured server usage across all responses, including first request. Missing cache reporting means unknown, not zero. Requested flag alone does not prove reuse."
      },
      "groups": {
        "coverage": {
          "everyday": {
            "rows": 90,
            "decisions": 90,
            "correct_decisions": 89,
            "correct_records": 89,
            "accuracy": 0.9888888888888889,
            "record_accuracy": 0.9888888888888889
          },
          "boundary": {
            "rows": 45,
            "decisions": 45,
            "correct_decisions": 37,
            "correct_records": 37,
            "accuracy": 0.8222222222222222,
            "record_accuracy": 0.8222222222222222
          },
          "indirect": {
            "rows": 45,
            "decisions": 45,
            "correct_decisions": 38,
            "correct_records": 38,
            "accuracy": 0.8444444444444444,
            "record_accuracy": 0.8444444444444444
          }
        },
        "route": {
          "answer": {
            "support": 20,
            "correct": 17,
            "recall": 0.85
          },
          "get_device_info": {
            "support": 20,
            "correct": 17,
            "recall": 0.85
          },
          "get_media_info": {
            "support": 20,
            "correct": 19,
            "recall": 0.95
          },
          "get_storage_info": {
            "support": 20,
            "correct": 17,
            "recall": 0.85
          },
          "save_note": {
            "support": 20,
            "correct": 20,
            "recall": 1.0
          },
          "search_notes": {
            "support": 20,
            "correct": 20,
            "recall": 1.0
          },
          "phone_health_check": {
            "support": 20,
            "correct": 20,
            "recall": 1.0
          },
          "search_local_files": {
            "support": 20,
            "correct": 19,
            "recall": 0.95
          },
          "get_battery_info": {
            "support": 20,
            "correct": 15,
            "recall": 0.75
          }
        }
      },
      "calibration": {
        "valid_decisions": 180,
        "ece_10_equal_width_bins": 0.03097985240746409,
        "mean_multiclass_brier_sum": 0.1338160429011829,
        "hard_label_nll": 0.3177925623799157,
        "bins": [
          {
            "bin": 2,
            "n": 1,
            "mean_selected_probability": 0.23801604912448804,
            "accuracy": 1
          },
          {
            "bin": 4,
            "n": 5,
            "mean_selected_probability": 0.4624825915685358,
            "accuracy": 0
          },
          {
            "bin": 5,
            "n": 7,
            "mean_selected_probability": 0.5544954914123206,
            "accuracy": 0.5714285714285714
          },
          {
            "bin": 6,
            "n": 6,
            "mean_selected_probability": 0.6402181918320586,
            "accuracy": 0.6666666666666666
          },
          {
            "bin": 7,
            "n": 7,
            "mean_selected_probability": 0.7619083210773232,
            "accuracy": 0.8571428571428571
          },
          {
            "bin": 8,
            "n": 11,
            "mean_selected_probability": 0.8576935980211633,
            "accuracy": 0.8181818181818182
          },
          {
            "bin": 9,
            "n": 143,
            "mean_selected_probability": 0.9868775019916952,
            "accuracy": 0.9790209790209791
          }
        ],
        "definitions": "Explicit hard gold labels; Brier=sum over options, not divided by class count; NLL clipped at1e-12; failures excluded from calibration only, never accuracy."
      },
      "predictions_sha256": "9c12b8b1f19b352dd4b251f23d1cca30878e552a3e91b58906274ef8ef3f98bc"
    },
    {
      "directory": "qwen-legacy-fresh180",
      "model": "Qwen app router (cache off)",
      "cohort": "MOBILE_ROUTING9 (180)",
      "summary": {
        "complete": true,
        "rows": 180,
        "decisions": 180,
        "accuracy": 0.5555555555555556,
        "record_accuracy": 0.5555555555555556,
        "failures": 0,
        "mean_latency_ms": 41136.67842498889,
        "p50_latency_ms": 41044.5108125,
        "p95_latency_ms": 44694.47058300011,
        "by_question": {
          "route": {
            "accuracy": 0.5555555555555556,
            "macro_f1": 0.5371311510123669,
            "per_label": {
              "answer": {
                "support": 20,
                "predicted": 9,
                "tp": 4
              },
              "get_battery_info": {
                "support": 20,
                "predicted": 18,
                "tp": 15
              },
              "get_device_info": {
                "support": 20,
                "predicted": 14,
                "tp": 9
              },
              "get_media_info": {
                "support": 20,
                "predicted": 21,
                "tp": 16
              },
              "get_storage_info": {
                "support": 20,
                "predicted": 20,
                "tp": 11
              },
              "phone_health_check": {
                "support": 20,
                "predicted": 3,
                "tp": 3
              },
              "save_note": {
                "support": 20,
                "predicted": 23,
                "tp": 16
              },
              "search_local_files": {
                "support": 20,
                "predicted": 41,
                "tp": 18
              },
              "search_notes": {
                "support": 20,
                "predicted": 31,
                "tp": 8
              }
            }
          }
        },
        "warm": {
          "complete": true,
          "rows": 179,
          "decisions": 179,
          "accuracy": 0.5586592178770949,
          "record_accuracy": 0.5586592178770949,
          "failures": 0,
          "mean_latency_ms": 41174.8651941229,
          "p50_latency_ms": 41047.13558399999,
          "p95_latency_ms": 44920.00554200058,
          "by_question": {
            "route": {
              "accuracy": 0.5586592178770949,
              "macro_f1": 0.5405966336067061,
              "per_label": {
                "answer": {
                  "support": 19,
                  "predicted": 9,
                  "tp": 4
                },
                "get_battery_info": {
                  "support": 20,
                  "predicted": 17,
                  "tp": 15
                },
                "get_device_info": {
                  "support": 20,
                  "predicted": 14,
                  "tp": 9
                },
                "get_media_info": {
                  "support": 20,
                  "predicted": 21,
                  "tp": 16
                },
                "get_storage_info": {
                  "support": 20,
                  "predicted": 20,
                  "tp": 11
                },
                "phone_health_check": {
                  "support": 20,
                  "predicted": 3,
                  "tp": 3
                },
                "save_note": {
                  "support": 20,
                  "predicted": 23,
                  "tp": 16
                },
                "search_local_files": {
                  "support": 20,
                  "predicted": 41,
                  "tp": 18
                },
                "search_notes": {
                  "support": 20,
                  "predicted": 31,
                  "tp": 8
                }
              }
            }
          }
        },
        "first_request_ms": 34301.24675000002
      },
      "resources": {
        "max_android_thermal_status": 3,
        "battery_c_min": 31.6,
        "battery_c_max": 37.6,
        "ap_c_max": 53.1,
        "skin_c_max": 42.7,
        "sampled_max_pss_kib": 815335,
        "rss_high_water_kib": 858260
      },
      "token_usage": {
        "requests_with_input_usage": 180,
        "mean_reported_input_tokens": 657.1166666666667,
        "mean_output_tokens": 15.35,
        "requested_cache_values": [
          false
        ],
        "requests_with_cache_reporting": 180,
        "requests_reporting_nonzero_cached_tokens": 0,
        "mean_reported_cached_tokens": 0,
        "cached_fraction_of_reported_input_tokens": 0.0,
        "mean_server_prompt_evaluated_tokens": 657.1166666666667,
        "note": "Measured server usage across all responses, including first request. Missing cache reporting means unknown, not zero. Requested flag alone does not prove reuse."
      },
      "groups": {
        "coverage": {
          "everyday": {
            "rows": 90,
            "decisions": 90,
            "correct_decisions": 54,
            "correct_records": 54,
            "accuracy": 0.6,
            "record_accuracy": 0.6
          },
          "boundary": {
            "rows": 45,
            "decisions": 45,
            "correct_decisions": 21,
            "correct_records": 21,
            "accuracy": 0.4666666666666667,
            "record_accuracy": 0.4666666666666667
          },
          "indirect": {
            "rows": 45,
            "decisions": 45,
            "correct_decisions": 25,
            "correct_records": 25,
            "accuracy": 0.5555555555555556,
            "record_accuracy": 0.5555555555555556
          }
        },
        "workflow": {
          "phone_routing": {
            "rows": 180,
            "decisions": 180,
            "correct_decisions": 100,
            "correct_records": 100,
            "accuracy": 0.5555555555555556,
            "record_accuracy": 0.5555555555555556
          }
        },
        "route": {
          "answer": {
            "support": 20,
            "correct": 4,
            "recall": 0.2
          },
          "get_device_info": {
            "support": 20,
            "correct": 9,
            "recall": 0.45
          },
          "get_media_info": {
            "support": 20,
            "correct": 16,
            "recall": 0.8
          },
          "get_storage_info": {
            "support": 20,
            "correct": 11,
            "recall": 0.55
          },
          "save_note": {
            "support": 20,
            "correct": 16,
            "recall": 0.8
          },
          "search_notes": {
            "support": 20,
            "correct": 8,
            "recall": 0.4
          },
          "phone_health_check": {
            "support": 20,
            "correct": 3,
            "recall": 0.15
          },
          "search_local_files": {
            "support": 20,
            "correct": 18,
            "recall": 0.9
          },
          "get_battery_info": {
            "support": 20,
            "correct": 15,
            "recall": 0.75
          }
        }
      },
      "calibration": {
        "valid_decisions": 0,
        "note": "No decision probability distribution available"
      },
      "predictions_sha256": "4fed70ec9f846bfc37bc07860dc1082fe1ea728e582f5a7b04bd9bd40c29c1fb"
    },
    {
      "directory": "qwen-legacy-cached-fresh180",
      "model": "Qwen app router (cache on)",
      "cohort": "MOBILE_ROUTING9 (180)",
      "summary": {
        "complete": true,
        "rows": 180,
        "decisions": 180,
        "accuracy": 0.5444444444444444,
        "record_accuracy": 0.5444444444444444,
        "failures": 0,
        "mean_latency_ms": 35395.459384916685,
        "p50_latency_ms": 36321.98522950009,
        "p95_latency_ms": 38778.471208000155,
        "by_question": {
          "route": {
            "accuracy": 0.5444444444444444,
            "macro_f1": 0.5252001986234163,
            "per_label": {
              "answer": {
                "support": 20,
                "predicted": 9,
                "tp": 5
              },
              "get_battery_info": {
                "support": 20,
                "predicted": 18,
                "tp": 14
              },
              "get_device_info": {
                "support": 20,
                "predicted": 14,
                "tp": 8
              },
              "get_media_info": {
                "support": 20,
                "predicted": 22,
                "tp": 17
              },
              "get_storage_info": {
                "support": 20,
                "predicted": 18,
                "tp": 8
              },
              "phone_health_check": {
                "support": 20,
                "predicted": 3,
                "tp": 3
              },
              "save_note": {
                "support": 20,
                "predicted": 22,
                "tp": 16
              },
              "search_local_files": {
                "support": 20,
                "predicted": 45,
                "tp": 18
              },
              "search_notes": {
                "support": 20,
                "predicted": 29,
                "tp": 9
              }
            }
          }
        },
        "warm": {
          "complete": true,
          "rows": 179,
          "decisions": 179,
          "accuracy": 0.547486033519553,
          "record_accuracy": 0.547486033519553,
          "failures": 0,
          "mean_latency_ms": 35378.0120590894,
          "p50_latency_ms": 36314.859042000026,
          "p95_latency_ms": 38826.29079100025,
          "by_question": {
            "route": {
              "accuracy": 0.547486033519553,
              "macro_f1": 0.5287813011155895,
              "per_label": {
                "answer": {
                  "support": 19,
                  "predicted": 9,
                  "tp": 5
                },
                "get_battery_info": {
                  "support": 20,
                  "predicted": 17,
                  "tp": 14
                },
                "get_device_info": {
                  "support": 20,
                  "predicted": 14,
                  "tp": 8
                },
                "get_media_info": {
                  "support": 20,
                  "predicted": 22,
                  "tp": 17
                },
                "get_storage_info": {
                  "support": 20,
                  "predicted": 18,
                  "tp": 8
                },
                "phone_health_check": {
                  "support": 20,
                  "predicted": 3,
                  "tp": 3
                },
                "save_note": {
                  "support": 20,
                  "predicted": 22,
                  "tp": 16
                },
                "search_local_files": {
                  "support": 20,
                  "predicted": 45,
                  "tp": 18
                },
                "search_notes": {
                  "support": 20,
                  "predicted": 29,
                  "tp": 9
                }
              }
            }
          }
        },
        "first_request_ms": 38518.530708
      },
      "resources": {
        "max_android_thermal_status": 3,
        "battery_c_min": 29.3,
        "battery_c_max": 37.5,
        "ap_c_max": 52.9,
        "skin_c_max": 42.6,
        "sampled_max_pss_kib": 875343,
        "rss_high_water_kib": 918320
      },
      "token_usage": {
        "requests_with_input_usage": 180,
        "mean_reported_input_tokens": 657.1166666666667,
        "mean_output_tokens": 15.366666666666667,
        "requested_cache_values": [
          true
        ],
        "requests_with_cache_reporting": 180,
        "requests_reporting_nonzero_cached_tokens": 179,
        "mean_reported_cached_tokens": 148.86111111111111,
        "cached_fraction_of_reported_input_tokens": 0.2265368064186133,
        "mean_server_prompt_evaluated_tokens": 508.25555555555553,
        "note": "Measured server usage across all responses, including first request. Missing cache reporting means unknown, not zero. Requested flag alone does not prove reuse."
      },
      "groups": {
        "coverage": {
          "everyday": {
            "rows": 90,
            "decisions": 90,
            "correct_decisions": 52,
            "correct_records": 52,
            "accuracy": 0.5777777777777777,
            "record_accuracy": 0.5777777777777777
          },
          "boundary": {
            "rows": 45,
            "decisions": 45,
            "correct_decisions": 22,
            "correct_records": 22,
            "accuracy": 0.4888888888888889,
            "record_accuracy": 0.4888888888888889
          },
          "indirect": {
            "rows": 45,
            "decisions": 45,
            "correct_decisions": 24,
            "correct_records": 24,
            "accuracy": 0.5333333333333333,
            "record_accuracy": 0.5333333333333333
          }
        },
        "workflow": {
          "phone_routing": {
            "rows": 180,
            "decisions": 180,
            "correct_decisions": 98,
            "correct_records": 98,
            "accuracy": 0.5444444444444444,
            "record_accuracy": 0.5444444444444444
          }
        },
        "route": {
          "answer": {
            "support": 20,
            "correct": 5,
            "recall": 0.25
          },
          "get_device_info": {
            "support": 20,
            "correct": 8,
            "recall": 0.4
          },
          "get_media_info": {
            "support": 20,
            "correct": 17,
            "recall": 0.85
          },
          "get_storage_info": {
            "support": 20,
            "correct": 8,
            "recall": 0.4
          },
          "save_note": {
            "support": 20,
            "correct": 16,
            "recall": 0.8
          },
          "search_notes": {
            "support": 20,
            "correct": 9,
            "recall": 0.45
          },
          "phone_health_check": {
            "support": 20,
            "correct": 3,
            "recall": 0.15
          },
          "search_local_files": {
            "support": 20,
            "correct": 18,
            "recall": 0.9
          },
          "get_battery_info": {
            "support": 20,
            "correct": 14,
            "recall": 0.7
          }
        }
      },
      "calibration": {
        "valid_decisions": 0,
        "note": "No decision probability distribution available"
      },
      "predictions_sha256": "9954bf40b6fb39f98d34af4e1cc96369de0db5a7f868faa017b90d470b4730b4"
    },
    {
      "directory": "qwen-fresh180",
      "model": "Qwen schema JSON (cache off)",
      "cohort": "MOBILE_ROUTING9 (180)",
      "summary": {
        "complete": true,
        "rows": 180,
        "decisions": 180,
        "accuracy": 0.23333333333333334,
        "record_accuracy": 0.23333333333333334,
        "failures": 0,
        "mean_latency_ms": 24925.08963193332,
        "p50_latency_ms": 25422.64104150013,
        "p95_latency_ms": 26657.780999999886,
        "by_question": {
          "route": {
            "accuracy": 0.23333333333333334,
            "macro_f1": 0.22412726206155972,
            "per_label": {
              "answer": {
                "support": 20,
                "predicted": 72,
                "tp": 11
              },
              "get_battery_info": {
                "support": 20,
                "predicted": 11,
                "tp": 6
              },
              "get_device_info": {
                "support": 20,
                "predicted": 0,
                "tp": 0
              },
              "get_media_info": {
                "support": 20,
                "predicted": 6,
                "tp": 3
              },
              "get_storage_info": {
                "support": 20,
                "predicted": 12,
                "tp": 2
              },
              "phone_health_check": {
                "support": 20,
                "predicted": 23,
                "tp": 3
              },
              "save_note": {
                "support": 20,
                "predicted": 4,
                "tp": 3
              },
              "search_local_files": {
                "support": 20,
                "predicted": 37,
                "tp": 7
              },
              "search_notes": {
                "support": 20,
                "predicted": 15,
                "tp": 7
              }
            }
          }
        },
        "warm": {
          "complete": true,
          "rows": 179,
          "decisions": 179,
          "accuracy": 0.2346368715083799,
          "record_accuracy": 0.2346368715083799,
          "failures": 0,
          "mean_latency_ms": 24943.137572150823,
          "p50_latency_ms": 25439.408916000048,
          "p95_latency_ms": 26702.532874999633,
          "by_question": {
            "route": {
              "accuracy": 0.2346368715083799,
              "macro_f1": 0.2249065700491057,
              "per_label": {
                "answer": {
                  "support": 19,
                  "predicted": 72,
                  "tp": 11
                },
                "get_battery_info": {
                  "support": 20,
                  "predicted": 11,
                  "tp": 6
                },
                "get_device_info": {
                  "support": 20,
                  "predicted": 0,
                  "tp": 0
                },
                "get_media_info": {
                  "support": 20,
                  "predicted": 6,
                  "tp": 3
                },
                "get_storage_info": {
                  "support": 20,
                  "predicted": 12,
                  "tp": 2
                },
                "phone_health_check": {
                  "support": 20,
                  "predicted": 23,
                  "tp": 3
                },
                "save_note": {
                  "support": 20,
                  "predicted": 4,
                  "tp": 3
                },
                "search_local_files": {
                  "support": 20,
                  "predicted": 36,
                  "tp": 7
                },
                "search_notes": {
                  "support": 20,
                  "predicted": 15,
                  "tp": 7
                }
              }
            }
          }
        },
        "first_request_ms": 21694.508333000158
      },
      "resources": {
        "max_android_thermal_status": 3,
        "battery_c_min": 31.8,
        "battery_c_max": 37.4,
        "ap_c_max": 53.0,
        "skin_c_max": 42.6,
        "sampled_max_pss_kib": 782891,
        "rss_high_water_kib": 833124
      },
      "token_usage": {
        "requests_with_input_usage": 180,
        "mean_reported_input_tokens": 357.1722222222222,
        "mean_output_tokens": 10.372222222222222,
        "requested_cache_values": [
          false
        ],
        "requests_with_cache_reporting": 180,
        "requests_reporting_nonzero_cached_tokens": 0,
        "mean_reported_cached_tokens": 0,
        "cached_fraction_of_reported_input_tokens": 0.0,
        "mean_server_prompt_evaluated_tokens": 357.1722222222222,
        "note": "Measured server usage across all responses, including first request. Missing cache reporting means unknown, not zero. Requested flag alone does not prove reuse."
      },
      "groups": {
        "coverage": {
          "everyday": {
            "rows": 90,
            "decisions": 90,
            "correct_decisions": 17,
            "correct_records": 17,
            "accuracy": 0.18888888888888888,
            "record_accuracy": 0.18888888888888888
          },
          "boundary": {
            "rows": 45,
            "decisions": 45,
            "correct_decisions": 12,
            "correct_records": 12,
            "accuracy": 0.26666666666666666,
            "record_accuracy": 0.26666666666666666
          },
          "indirect": {
            "rows": 45,
            "decisions": 45,
            "correct_decisions": 13,
            "correct_records": 13,
            "accuracy": 0.28888888888888886,
            "record_accuracy": 0.28888888888888886
          }
        },
        "workflow": {
          "phone_routing": {
            "rows": 180,
            "decisions": 180,
            "correct_decisions": 42,
            "correct_records": 42,
            "accuracy": 0.23333333333333334,
            "record_accuracy": 0.23333333333333334
          }
        },
        "route": {
          "answer": {
            "support": 20,
            "correct": 11,
            "recall": 0.55
          },
          "get_device_info": {
            "support": 20,
            "correct": 0,
            "recall": 0.0
          },
          "get_media_info": {
            "support": 20,
            "correct": 3,
            "recall": 0.15
          },
          "get_storage_info": {
            "support": 20,
            "correct": 2,
            "recall": 0.1
          },
          "save_note": {
            "support": 20,
            "correct": 3,
            "recall": 0.15
          },
          "search_notes": {
            "support": 20,
            "correct": 7,
            "recall": 0.35
          },
          "phone_health_check": {
            "support": 20,
            "correct": 3,
            "recall": 0.15
          },
          "search_local_files": {
            "support": 20,
            "correct": 7,
            "recall": 0.35
          },
          "get_battery_info": {
            "support": 20,
            "correct": 6,
            "recall": 0.3
          }
        }
      },
      "calibration": {
        "valid_decisions": 0,
        "note": "No decision probability distribution available"
      },
      "predictions_sha256": "7d1a0e8b0942562ae8eed7ad24b3b99d3e23119397062b3200026988eaeed38f"
    }
  ],
  "paired_fresh180": {
    "qwen-fresh180": {
      "n": 180,
      "reflex_only_correct": 125,
      "baseline_only_correct": 3,
      "accuracy_difference": 0.6777777777777778,
      "paired_bootstrap_95ci": [
        0.6,
        0.75
      ],
      "exact_mcnemar_two_sided_p": 2.0549580818052442e-33,
      "bootstrap_seed": 3407,
      "bootstrap_replicates": 10000
    },
    "qwen-legacy-fresh180": {
      "n": 180,
      "reflex_only_correct": 69,
      "baseline_only_correct": 5,
      "accuracy_difference": 0.35555555555555557,
      "paired_bootstrap_95ci": [
        0.2777777777777778,
        0.43333333333333335
      ],
      "exact_mcnemar_two_sided_p": 1.8345664258432237e-15,
      "bootstrap_seed": 3407,
      "bootstrap_replicates": 10000
    },
    "qwen-legacy-cached-fresh180": {
      "n": 180,
      "reflex_only_correct": 70,
      "baseline_only_correct": 4,
      "accuracy_difference": 0.36666666666666664,
      "paired_bootstrap_95ci": [
        0.28888888888888886,
        0.4444444444444444
      ],
      "exact_mcnemar_two_sided_p": 1.2898469490022716e-16,
      "bootstrap_seed": 3407,
      "bootstrap_replicates": 10000
    }
  },
  "complete": true,
  "original_expanded_queue_complete": false,
  "required_jobs": [
    "qwen-fresh180",
    "qwen-legacy-cached-fresh180",
    "qwen-legacy-fresh180",
    "run-01"
  ],
  "deferred_jobs": [
    "beta-fresh180",
    "reflex-historical90",
    "qwen-historical90",
    "qwen-cached-fresh180",
    "reflex-typed_decisions",
    "reflex-banking77",
    "reflex-clinc150",
    "beta-typed_decisions",
    "beta-banking77",
    "beta-clinc150"
  ],
  "privacy": "Aggregate metrics only; no held-out requests, per-case responses, reviewer IDs or device-property dumps.",
  "runtime": "Android CPU build11443 f0c41e0168dfd4b5ef72b21d1a311b24cc7a894a; 4 inference/batch threads; Q4_K_M; phone ctx2048, general ctx4096; USB powered",
  "cohort_caveat": "Fresh180 and original historical90 are distinct; historical PyTorch reference used reviewed88. Raw Qwen prompt/JSON grammar treatment is specifically documented, not all possible Qwen inference configurations."
}