| { |
| "completed_jobs": [ |
| { |
| "directory": "run-01", |
| "model": "reflex-1", |
| "cohort": "MOBILE_ROUTING9 (180)", |
| "summary": { |
| "complete": true, |
| "all_requests": { |
| "total": 180, |
| "correct": 164, |
| "accuracy": 0.9111111111111111, |
| "macro_f1": 0.9106480025368284, |
| "failures": 0, |
| "confusion_matrix": { |
| "answer": { |
| "answer": 17, |
| "get_device_info": 0, |
| "get_battery_info": 0, |
| "get_storage_info": 0, |
| "get_media_info": 0, |
| "search_local_files": 0, |
| "search_notes": 1, |
| "save_note": 2, |
| "phone_health_check": 0 |
| }, |
| "get_device_info": { |
| "answer": 2, |
| "get_device_info": 17, |
| "get_battery_info": 0, |
| "get_storage_info": 0, |
| "get_media_info": 1, |
| "search_local_files": 0, |
| "search_notes": 0, |
| "save_note": 0, |
| "phone_health_check": 0 |
| }, |
| "get_battery_info": { |
| "answer": 2, |
| "get_device_info": 0, |
| "get_battery_info": 15, |
| "get_storage_info": 0, |
| "get_media_info": 1, |
| "search_local_files": 0, |
| "search_notes": 0, |
| "save_note": 0, |
| "phone_health_check": 2 |
| }, |
| "get_storage_info": { |
| "answer": 0, |
| "get_device_info": 1, |
| "get_battery_info": 0, |
| "get_storage_info": 17, |
| "get_media_info": 2, |
| "search_local_files": 0, |
| "search_notes": 0, |
| "save_note": 0, |
| "phone_health_check": 0 |
| }, |
| "get_media_info": { |
| "answer": 1, |
| "get_device_info": 0, |
| "get_battery_info": 0, |
| "get_storage_info": 0, |
| "get_media_info": 19, |
| "search_local_files": 0, |
| "search_notes": 0, |
| "save_note": 0, |
| "phone_health_check": 0 |
| }, |
| "search_local_files": { |
| "answer": 0, |
| "get_device_info": 1, |
| "get_battery_info": 0, |
| "get_storage_info": 0, |
| "get_media_info": 0, |
| "search_local_files": 19, |
| "search_notes": 0, |
| "save_note": 0, |
| "phone_health_check": 0 |
| }, |
| "search_notes": { |
| "answer": 0, |
| "get_device_info": 0, |
| "get_battery_info": 0, |
| "get_storage_info": 0, |
| "get_media_info": 0, |
| "search_local_files": 0, |
| "search_notes": 20, |
| "save_note": 0, |
| "phone_health_check": 0 |
| }, |
| "save_note": { |
| "answer": 0, |
| "get_device_info": 0, |
| "get_battery_info": 0, |
| "get_storage_info": 0, |
| "get_media_info": 0, |
| "search_local_files": 0, |
| "search_notes": 0, |
| "save_note": 20, |
| "phone_health_check": 0 |
| }, |
| "phone_health_check": { |
| "answer": 0, |
| "get_device_info": 0, |
| "get_battery_info": 0, |
| "get_storage_info": 0, |
| "get_media_info": 0, |
| "search_local_files": 0, |
| "search_notes": 0, |
| "save_note": 0, |
| "phone_health_check": 20 |
| } |
| }, |
| "mean_latency_ms": 31364.1449391611, |
| "p50_latency_ms": 32300.51620800009, |
| "p95_latency_ms": 33802.05279199981, |
| "reference_agreement": 0.9444444444444444, |
| "parity_valid_cases": 180 |
| }, |
| "first_request_ms": 27731.913458000003, |
| "warm_requests": { |
| "total": 179, |
| "correct": 163, |
| "accuracy": 0.9106145251396648, |
| "macro_f1": 0.9095898014786273, |
| "failures": 0, |
| "confusion_matrix": { |
| "answer": { |
| "answer": 16, |
| "get_device_info": 0, |
| "get_battery_info": 0, |
| "get_storage_info": 0, |
| "get_media_info": 0, |
| "search_local_files": 0, |
| "search_notes": 1, |
| "save_note": 2, |
| "phone_health_check": 0 |
| }, |
| "get_device_info": { |
| "answer": 2, |
| "get_device_info": 17, |
| "get_battery_info": 0, |
| "get_storage_info": 0, |
| "get_media_info": 1, |
| "search_local_files": 0, |
| "search_notes": 0, |
| "save_note": 0, |
| "phone_health_check": 0 |
| }, |
| "get_battery_info": { |
| "answer": 2, |
| "get_device_info": 0, |
| "get_battery_info": 15, |
| "get_storage_info": 0, |
| "get_media_info": 1, |
| "search_local_files": 0, |
| "search_notes": 0, |
| "save_note": 0, |
| "phone_health_check": 2 |
| }, |
| "get_storage_info": { |
| "answer": 0, |
| "get_device_info": 1, |
| "get_battery_info": 0, |
| "get_storage_info": 17, |
| "get_media_info": 2, |
| "search_local_files": 0, |
| "search_notes": 0, |
| "save_note": 0, |
| "phone_health_check": 0 |
| }, |
| "get_media_info": { |
| "answer": 1, |
| "get_device_info": 0, |
| "get_battery_info": 0, |
| "get_storage_info": 0, |
| "get_media_info": 19, |
| "search_local_files": 0, |
| "search_notes": 0, |
| "save_note": 0, |
| "phone_health_check": 0 |
| }, |
| "search_local_files": { |
| "answer": 0, |
| "get_device_info": 1, |
| "get_battery_info": 0, |
| "get_storage_info": 0, |
| "get_media_info": 0, |
| "search_local_files": 19, |
| "search_notes": 0, |
| "save_note": 0, |
| "phone_health_check": 0 |
| }, |
| "search_notes": { |
| "answer": 0, |
| "get_device_info": 0, |
| "get_battery_info": 0, |
| "get_storage_info": 0, |
| "get_media_info": 0, |
| "search_local_files": 0, |
| "search_notes": 20, |
| "save_note": 0, |
| "phone_health_check": 0 |
| }, |
| "save_note": { |
| "answer": 0, |
| "get_device_info": 0, |
| "get_battery_info": 0, |
| "get_storage_info": 0, |
| "get_media_info": 0, |
| "search_local_files": 0, |
| "search_notes": 0, |
| "save_note": 20, |
| "phone_health_check": 0 |
| }, |
| "phone_health_check": { |
| "answer": 0, |
| "get_device_info": 0, |
| "get_battery_info": 0, |
| "get_storage_info": 0, |
| "get_media_info": 0, |
| "search_local_files": 0, |
| "search_notes": 0, |
| "save_note": 0, |
| "phone_health_check": 20 |
| } |
| }, |
| "mean_latency_ms": 31384.43673514524, |
| "p50_latency_ms": 32342.07145800019, |
| "p95_latency_ms": 33865.3933329997, |
| "reference_agreement": 0.9441340782122905, |
| "parity_valid_cases": 179 |
| }, |
| "end_unix": 1792023255.933782 |
| }, |
| "resources": { |
| "max_android_thermal_status": 3, |
| "battery_c_min": 28.9, |
| "battery_c_max": 37.5, |
| "ap_c_max": 52.6, |
| "skin_c_max": 42.6, |
| "sampled_max_pss_kib": 1116821, |
| "rss_high_water_kib": 1119236 |
| }, |
| "token_usage": { |
| "requests_with_input_usage": 180, |
| "mean_reported_input_tokens": 456.1722222222222, |
| "mean_output_tokens": 0, |
| "requested_cache_values": [], |
| "requests_with_cache_reporting": 0, |
| "requests_reporting_nonzero_cached_tokens": 0, |
| "mean_reported_cached_tokens": null, |
| "cached_fraction_of_reported_input_tokens": null, |
| "mean_server_prompt_evaluated_tokens": null, |
| "note": "Measured server usage across all responses, including first request. Missing cache reporting means unknown, not zero. Requested flag alone does not prove reuse." |
| }, |
| "groups": { |
| "coverage": { |
| "everyday": { |
| "rows": 90, |
| "decisions": 90, |
| "correct_decisions": 89, |
| "correct_records": 89, |
| "accuracy": 0.9888888888888889, |
| "record_accuracy": 0.9888888888888889 |
| }, |
| "boundary": { |
| "rows": 45, |
| "decisions": 45, |
| "correct_decisions": 37, |
| "correct_records": 37, |
| "accuracy": 0.8222222222222222, |
| "record_accuracy": 0.8222222222222222 |
| }, |
| "indirect": { |
| "rows": 45, |
| "decisions": 45, |
| "correct_decisions": 38, |
| "correct_records": 38, |
| "accuracy": 0.8444444444444444, |
| "record_accuracy": 0.8444444444444444 |
| } |
| }, |
| "route": { |
| "answer": { |
| "support": 20, |
| "correct": 17, |
| "recall": 0.85 |
| }, |
| "get_device_info": { |
| "support": 20, |
| "correct": 17, |
| "recall": 0.85 |
| }, |
| "get_media_info": { |
| "support": 20, |
| "correct": 19, |
| "recall": 0.95 |
| }, |
| "get_storage_info": { |
| "support": 20, |
| "correct": 17, |
| "recall": 0.85 |
| }, |
| "save_note": { |
| "support": 20, |
| "correct": 20, |
| "recall": 1.0 |
| }, |
| "search_notes": { |
| "support": 20, |
| "correct": 20, |
| "recall": 1.0 |
| }, |
| "phone_health_check": { |
| "support": 20, |
| "correct": 20, |
| "recall": 1.0 |
| }, |
| "search_local_files": { |
| "support": 20, |
| "correct": 19, |
| "recall": 0.95 |
| }, |
| "get_battery_info": { |
| "support": 20, |
| "correct": 15, |
| "recall": 0.75 |
| } |
| } |
| }, |
| "calibration": { |
| "valid_decisions": 180, |
| "ece_10_equal_width_bins": 0.03097985240746409, |
| "mean_multiclass_brier_sum": 0.1338160429011829, |
| "hard_label_nll": 0.3177925623799157, |
| "bins": [ |
| { |
| "bin": 2, |
| "n": 1, |
| "mean_selected_probability": 0.23801604912448804, |
| "accuracy": 1 |
| }, |
| { |
| "bin": 4, |
| "n": 5, |
| "mean_selected_probability": 0.4624825915685358, |
| "accuracy": 0 |
| }, |
| { |
| "bin": 5, |
| "n": 7, |
| "mean_selected_probability": 0.5544954914123206, |
| "accuracy": 0.5714285714285714 |
| }, |
| { |
| "bin": 6, |
| "n": 6, |
| "mean_selected_probability": 0.6402181918320586, |
| "accuracy": 0.6666666666666666 |
| }, |
| { |
| "bin": 7, |
| "n": 7, |
| "mean_selected_probability": 0.7619083210773232, |
| "accuracy": 0.8571428571428571 |
| }, |
| { |
| "bin": 8, |
| "n": 11, |
| "mean_selected_probability": 0.8576935980211633, |
| "accuracy": 0.8181818181818182 |
| }, |
| { |
| "bin": 9, |
| "n": 143, |
| "mean_selected_probability": 0.9868775019916952, |
| "accuracy": 0.9790209790209791 |
| } |
| ], |
| "definitions": "Explicit hard gold labels; Brier=sum over options, not divided by class count; NLL clipped at1e-12; failures excluded from calibration only, never accuracy." |
| }, |
| "predictions_sha256": "9c12b8b1f19b352dd4b251f23d1cca30878e552a3e91b58906274ef8ef3f98bc" |
| }, |
| { |
| "directory": "qwen-legacy-fresh180", |
| "model": "Qwen app router (cache off)", |
| "cohort": "MOBILE_ROUTING9 (180)", |
| "summary": { |
| "complete": true, |
| "rows": 180, |
| "decisions": 180, |
| "accuracy": 0.5555555555555556, |
| "record_accuracy": 0.5555555555555556, |
| "failures": 0, |
| "mean_latency_ms": 41136.67842498889, |
| "p50_latency_ms": 41044.5108125, |
| "p95_latency_ms": 44694.47058300011, |
| "by_question": { |
| "route": { |
| "accuracy": 0.5555555555555556, |
| "macro_f1": 0.5371311510123669, |
| "per_label": { |
| "answer": { |
| "support": 20, |
| "predicted": 9, |
| "tp": 4 |
| }, |
| "get_battery_info": { |
| "support": 20, |
| "predicted": 18, |
| "tp": 15 |
| }, |
| "get_device_info": { |
| "support": 20, |
| "predicted": 14, |
| "tp": 9 |
| }, |
| "get_media_info": { |
| "support": 20, |
| "predicted": 21, |
| "tp": 16 |
| }, |
| "get_storage_info": { |
| "support": 20, |
| "predicted": 20, |
| "tp": 11 |
| }, |
| "phone_health_check": { |
| "support": 20, |
| "predicted": 3, |
| "tp": 3 |
| }, |
| "save_note": { |
| "support": 20, |
| "predicted": 23, |
| "tp": 16 |
| }, |
| "search_local_files": { |
| "support": 20, |
| "predicted": 41, |
| "tp": 18 |
| }, |
| "search_notes": { |
| "support": 20, |
| "predicted": 31, |
| "tp": 8 |
| } |
| } |
| } |
| }, |
| "warm": { |
| "complete": true, |
| "rows": 179, |
| "decisions": 179, |
| "accuracy": 0.5586592178770949, |
| "record_accuracy": 0.5586592178770949, |
| "failures": 0, |
| "mean_latency_ms": 41174.8651941229, |
| "p50_latency_ms": 41047.13558399999, |
| "p95_latency_ms": 44920.00554200058, |
| "by_question": { |
| "route": { |
| "accuracy": 0.5586592178770949, |
| "macro_f1": 0.5405966336067061, |
| "per_label": { |
| "answer": { |
| "support": 19, |
| "predicted": 9, |
| "tp": 4 |
| }, |
| "get_battery_info": { |
| "support": 20, |
| "predicted": 17, |
| "tp": 15 |
| }, |
| "get_device_info": { |
| "support": 20, |
| "predicted": 14, |
| "tp": 9 |
| }, |
| "get_media_info": { |
| "support": 20, |
| "predicted": 21, |
| "tp": 16 |
| }, |
| "get_storage_info": { |
| "support": 20, |
| "predicted": 20, |
| "tp": 11 |
| }, |
| "phone_health_check": { |
| "support": 20, |
| "predicted": 3, |
| "tp": 3 |
| }, |
| "save_note": { |
| "support": 20, |
| "predicted": 23, |
| "tp": 16 |
| }, |
| "search_local_files": { |
| "support": 20, |
| "predicted": 41, |
| "tp": 18 |
| }, |
| "search_notes": { |
| "support": 20, |
| "predicted": 31, |
| "tp": 8 |
| } |
| } |
| } |
| } |
| }, |
| "first_request_ms": 34301.24675000002 |
| }, |
| "resources": { |
| "max_android_thermal_status": 3, |
| "battery_c_min": 31.6, |
| "battery_c_max": 37.6, |
| "ap_c_max": 53.1, |
| "skin_c_max": 42.7, |
| "sampled_max_pss_kib": 815335, |
| "rss_high_water_kib": 858260 |
| }, |
| "token_usage": { |
| "requests_with_input_usage": 180, |
| "mean_reported_input_tokens": 657.1166666666667, |
| "mean_output_tokens": 15.35, |
| "requested_cache_values": [ |
| false |
| ], |
| "requests_with_cache_reporting": 180, |
| "requests_reporting_nonzero_cached_tokens": 0, |
| "mean_reported_cached_tokens": 0, |
| "cached_fraction_of_reported_input_tokens": 0.0, |
| "mean_server_prompt_evaluated_tokens": 657.1166666666667, |
| "note": "Measured server usage across all responses, including first request. Missing cache reporting means unknown, not zero. Requested flag alone does not prove reuse." |
| }, |
| "groups": { |
| "coverage": { |
| "everyday": { |
| "rows": 90, |
| "decisions": 90, |
| "correct_decisions": 54, |
| "correct_records": 54, |
| "accuracy": 0.6, |
| "record_accuracy": 0.6 |
| }, |
| "boundary": { |
| "rows": 45, |
| "decisions": 45, |
| "correct_decisions": 21, |
| "correct_records": 21, |
| "accuracy": 0.4666666666666667, |
| "record_accuracy": 0.4666666666666667 |
| }, |
| "indirect": { |
| "rows": 45, |
| "decisions": 45, |
| "correct_decisions": 25, |
| "correct_records": 25, |
| "accuracy": 0.5555555555555556, |
| "record_accuracy": 0.5555555555555556 |
| } |
| }, |
| "workflow": { |
| "phone_routing": { |
| "rows": 180, |
| "decisions": 180, |
| "correct_decisions": 100, |
| "correct_records": 100, |
| "accuracy": 0.5555555555555556, |
| "record_accuracy": 0.5555555555555556 |
| } |
| }, |
| "route": { |
| "answer": { |
| "support": 20, |
| "correct": 4, |
| "recall": 0.2 |
| }, |
| "get_device_info": { |
| "support": 20, |
| "correct": 9, |
| "recall": 0.45 |
| }, |
| "get_media_info": { |
| "support": 20, |
| "correct": 16, |
| "recall": 0.8 |
| }, |
| "get_storage_info": { |
| "support": 20, |
| "correct": 11, |
| "recall": 0.55 |
| }, |
| "save_note": { |
| "support": 20, |
| "correct": 16, |
| "recall": 0.8 |
| }, |
| "search_notes": { |
| "support": 20, |
| "correct": 8, |
| "recall": 0.4 |
| }, |
| "phone_health_check": { |
| "support": 20, |
| "correct": 3, |
| "recall": 0.15 |
| }, |
| "search_local_files": { |
| "support": 20, |
| "correct": 18, |
| "recall": 0.9 |
| }, |
| "get_battery_info": { |
| "support": 20, |
| "correct": 15, |
| "recall": 0.75 |
| } |
| } |
| }, |
| "calibration": { |
| "valid_decisions": 0, |
| "note": "No decision probability distribution available" |
| }, |
| "predictions_sha256": "4fed70ec9f846bfc37bc07860dc1082fe1ea728e582f5a7b04bd9bd40c29c1fb" |
| }, |
| { |
| "directory": "qwen-legacy-cached-fresh180", |
| "model": "Qwen app router (cache on)", |
| "cohort": "MOBILE_ROUTING9 (180)", |
| "summary": { |
| "complete": true, |
| "rows": 180, |
| "decisions": 180, |
| "accuracy": 0.5444444444444444, |
| "record_accuracy": 0.5444444444444444, |
| "failures": 0, |
| "mean_latency_ms": 35395.459384916685, |
| "p50_latency_ms": 36321.98522950009, |
| "p95_latency_ms": 38778.471208000155, |
| "by_question": { |
| "route": { |
| "accuracy": 0.5444444444444444, |
| "macro_f1": 0.5252001986234163, |
| "per_label": { |
| "answer": { |
| "support": 20, |
| "predicted": 9, |
| "tp": 5 |
| }, |
| "get_battery_info": { |
| "support": 20, |
| "predicted": 18, |
| "tp": 14 |
| }, |
| "get_device_info": { |
| "support": 20, |
| "predicted": 14, |
| "tp": 8 |
| }, |
| "get_media_info": { |
| "support": 20, |
| "predicted": 22, |
| "tp": 17 |
| }, |
| "get_storage_info": { |
| "support": 20, |
| "predicted": 18, |
| "tp": 8 |
| }, |
| "phone_health_check": { |
| "support": 20, |
| "predicted": 3, |
| "tp": 3 |
| }, |
| "save_note": { |
| "support": 20, |
| "predicted": 22, |
| "tp": 16 |
| }, |
| "search_local_files": { |
| "support": 20, |
| "predicted": 45, |
| "tp": 18 |
| }, |
| "search_notes": { |
| "support": 20, |
| "predicted": 29, |
| "tp": 9 |
| } |
| } |
| } |
| }, |
| "warm": { |
| "complete": true, |
| "rows": 179, |
| "decisions": 179, |
| "accuracy": 0.547486033519553, |
| "record_accuracy": 0.547486033519553, |
| "failures": 0, |
| "mean_latency_ms": 35378.0120590894, |
| "p50_latency_ms": 36314.859042000026, |
| "p95_latency_ms": 38826.29079100025, |
| "by_question": { |
| "route": { |
| "accuracy": 0.547486033519553, |
| "macro_f1": 0.5287813011155895, |
| "per_label": { |
| "answer": { |
| "support": 19, |
| "predicted": 9, |
| "tp": 5 |
| }, |
| "get_battery_info": { |
| "support": 20, |
| "predicted": 17, |
| "tp": 14 |
| }, |
| "get_device_info": { |
| "support": 20, |
| "predicted": 14, |
| "tp": 8 |
| }, |
| "get_media_info": { |
| "support": 20, |
| "predicted": 22, |
| "tp": 17 |
| }, |
| "get_storage_info": { |
| "support": 20, |
| "predicted": 18, |
| "tp": 8 |
| }, |
| "phone_health_check": { |
| "support": 20, |
| "predicted": 3, |
| "tp": 3 |
| }, |
| "save_note": { |
| "support": 20, |
| "predicted": 22, |
| "tp": 16 |
| }, |
| "search_local_files": { |
| "support": 20, |
| "predicted": 45, |
| "tp": 18 |
| }, |
| "search_notes": { |
| "support": 20, |
| "predicted": 29, |
| "tp": 9 |
| } |
| } |
| } |
| } |
| }, |
| "first_request_ms": 38518.530708 |
| }, |
| "resources": { |
| "max_android_thermal_status": 3, |
| "battery_c_min": 29.3, |
| "battery_c_max": 37.5, |
| "ap_c_max": 52.9, |
| "skin_c_max": 42.6, |
| "sampled_max_pss_kib": 875343, |
| "rss_high_water_kib": 918320 |
| }, |
| "token_usage": { |
| "requests_with_input_usage": 180, |
| "mean_reported_input_tokens": 657.1166666666667, |
| "mean_output_tokens": 15.366666666666667, |
| "requested_cache_values": [ |
| true |
| ], |
| "requests_with_cache_reporting": 180, |
| "requests_reporting_nonzero_cached_tokens": 179, |
| "mean_reported_cached_tokens": 148.86111111111111, |
| "cached_fraction_of_reported_input_tokens": 0.2265368064186133, |
| "mean_server_prompt_evaluated_tokens": 508.25555555555553, |
| "note": "Measured server usage across all responses, including first request. Missing cache reporting means unknown, not zero. Requested flag alone does not prove reuse." |
| }, |
| "groups": { |
| "coverage": { |
| "everyday": { |
| "rows": 90, |
| "decisions": 90, |
| "correct_decisions": 52, |
| "correct_records": 52, |
| "accuracy": 0.5777777777777777, |
| "record_accuracy": 0.5777777777777777 |
| }, |
| "boundary": { |
| "rows": 45, |
| "decisions": 45, |
| "correct_decisions": 22, |
| "correct_records": 22, |
| "accuracy": 0.4888888888888889, |
| "record_accuracy": 0.4888888888888889 |
| }, |
| "indirect": { |
| "rows": 45, |
| "decisions": 45, |
| "correct_decisions": 24, |
| "correct_records": 24, |
| "accuracy": 0.5333333333333333, |
| "record_accuracy": 0.5333333333333333 |
| } |
| }, |
| "workflow": { |
| "phone_routing": { |
| "rows": 180, |
| "decisions": 180, |
| "correct_decisions": 98, |
| "correct_records": 98, |
| "accuracy": 0.5444444444444444, |
| "record_accuracy": 0.5444444444444444 |
| } |
| }, |
| "route": { |
| "answer": { |
| "support": 20, |
| "correct": 5, |
| "recall": 0.25 |
| }, |
| "get_device_info": { |
| "support": 20, |
| "correct": 8, |
| "recall": 0.4 |
| }, |
| "get_media_info": { |
| "support": 20, |
| "correct": 17, |
| "recall": 0.85 |
| }, |
| "get_storage_info": { |
| "support": 20, |
| "correct": 8, |
| "recall": 0.4 |
| }, |
| "save_note": { |
| "support": 20, |
| "correct": 16, |
| "recall": 0.8 |
| }, |
| "search_notes": { |
| "support": 20, |
| "correct": 9, |
| "recall": 0.45 |
| }, |
| "phone_health_check": { |
| "support": 20, |
| "correct": 3, |
| "recall": 0.15 |
| }, |
| "search_local_files": { |
| "support": 20, |
| "correct": 18, |
| "recall": 0.9 |
| }, |
| "get_battery_info": { |
| "support": 20, |
| "correct": 14, |
| "recall": 0.7 |
| } |
| } |
| }, |
| "calibration": { |
| "valid_decisions": 0, |
| "note": "No decision probability distribution available" |
| }, |
| "predictions_sha256": "9954bf40b6fb39f98d34af4e1cc96369de0db5a7f868faa017b90d470b4730b4" |
| }, |
| { |
| "directory": "qwen-fresh180", |
| "model": "Qwen schema JSON (cache off)", |
| "cohort": "MOBILE_ROUTING9 (180)", |
| "summary": { |
| "complete": true, |
| "rows": 180, |
| "decisions": 180, |
| "accuracy": 0.23333333333333334, |
| "record_accuracy": 0.23333333333333334, |
| "failures": 0, |
| "mean_latency_ms": 24925.08963193332, |
| "p50_latency_ms": 25422.64104150013, |
| "p95_latency_ms": 26657.780999999886, |
| "by_question": { |
| "route": { |
| "accuracy": 0.23333333333333334, |
| "macro_f1": 0.22412726206155972, |
| "per_label": { |
| "answer": { |
| "support": 20, |
| "predicted": 72, |
| "tp": 11 |
| }, |
| "get_battery_info": { |
| "support": 20, |
| "predicted": 11, |
| "tp": 6 |
| }, |
| "get_device_info": { |
| "support": 20, |
| "predicted": 0, |
| "tp": 0 |
| }, |
| "get_media_info": { |
| "support": 20, |
| "predicted": 6, |
| "tp": 3 |
| }, |
| "get_storage_info": { |
| "support": 20, |
| "predicted": 12, |
| "tp": 2 |
| }, |
| "phone_health_check": { |
| "support": 20, |
| "predicted": 23, |
| "tp": 3 |
| }, |
| "save_note": { |
| "support": 20, |
| "predicted": 4, |
| "tp": 3 |
| }, |
| "search_local_files": { |
| "support": 20, |
| "predicted": 37, |
| "tp": 7 |
| }, |
| "search_notes": { |
| "support": 20, |
| "predicted": 15, |
| "tp": 7 |
| } |
| } |
| } |
| }, |
| "warm": { |
| "complete": true, |
| "rows": 179, |
| "decisions": 179, |
| "accuracy": 0.2346368715083799, |
| "record_accuracy": 0.2346368715083799, |
| "failures": 0, |
| "mean_latency_ms": 24943.137572150823, |
| "p50_latency_ms": 25439.408916000048, |
| "p95_latency_ms": 26702.532874999633, |
| "by_question": { |
| "route": { |
| "accuracy": 0.2346368715083799, |
| "macro_f1": 0.2249065700491057, |
| "per_label": { |
| "answer": { |
| "support": 19, |
| "predicted": 72, |
| "tp": 11 |
| }, |
| "get_battery_info": { |
| "support": 20, |
| "predicted": 11, |
| "tp": 6 |
| }, |
| "get_device_info": { |
| "support": 20, |
| "predicted": 0, |
| "tp": 0 |
| }, |
| "get_media_info": { |
| "support": 20, |
| "predicted": 6, |
| "tp": 3 |
| }, |
| "get_storage_info": { |
| "support": 20, |
| "predicted": 12, |
| "tp": 2 |
| }, |
| "phone_health_check": { |
| "support": 20, |
| "predicted": 23, |
| "tp": 3 |
| }, |
| "save_note": { |
| "support": 20, |
| "predicted": 4, |
| "tp": 3 |
| }, |
| "search_local_files": { |
| "support": 20, |
| "predicted": 36, |
| "tp": 7 |
| }, |
| "search_notes": { |
| "support": 20, |
| "predicted": 15, |
| "tp": 7 |
| } |
| } |
| } |
| } |
| }, |
| "first_request_ms": 21694.508333000158 |
| }, |
| "resources": { |
| "max_android_thermal_status": 3, |
| "battery_c_min": 31.8, |
| "battery_c_max": 37.4, |
| "ap_c_max": 53.0, |
| "skin_c_max": 42.6, |
| "sampled_max_pss_kib": 782891, |
| "rss_high_water_kib": 833124 |
| }, |
| "token_usage": { |
| "requests_with_input_usage": 180, |
| "mean_reported_input_tokens": 357.1722222222222, |
| "mean_output_tokens": 10.372222222222222, |
| "requested_cache_values": [ |
| false |
| ], |
| "requests_with_cache_reporting": 180, |
| "requests_reporting_nonzero_cached_tokens": 0, |
| "mean_reported_cached_tokens": 0, |
| "cached_fraction_of_reported_input_tokens": 0.0, |
| "mean_server_prompt_evaluated_tokens": 357.1722222222222, |
| "note": "Measured server usage across all responses, including first request. Missing cache reporting means unknown, not zero. Requested flag alone does not prove reuse." |
| }, |
| "groups": { |
| "coverage": { |
| "everyday": { |
| "rows": 90, |
| "decisions": 90, |
| "correct_decisions": 17, |
| "correct_records": 17, |
| "accuracy": 0.18888888888888888, |
| "record_accuracy": 0.18888888888888888 |
| }, |
| "boundary": { |
| "rows": 45, |
| "decisions": 45, |
| "correct_decisions": 12, |
| "correct_records": 12, |
| "accuracy": 0.26666666666666666, |
| "record_accuracy": 0.26666666666666666 |
| }, |
| "indirect": { |
| "rows": 45, |
| "decisions": 45, |
| "correct_decisions": 13, |
| "correct_records": 13, |
| "accuracy": 0.28888888888888886, |
| "record_accuracy": 0.28888888888888886 |
| } |
| }, |
| "workflow": { |
| "phone_routing": { |
| "rows": 180, |
| "decisions": 180, |
| "correct_decisions": 42, |
| "correct_records": 42, |
| "accuracy": 0.23333333333333334, |
| "record_accuracy": 0.23333333333333334 |
| } |
| }, |
| "route": { |
| "answer": { |
| "support": 20, |
| "correct": 11, |
| "recall": 0.55 |
| }, |
| "get_device_info": { |
| "support": 20, |
| "correct": 0, |
| "recall": 0.0 |
| }, |
| "get_media_info": { |
| "support": 20, |
| "correct": 3, |
| "recall": 0.15 |
| }, |
| "get_storage_info": { |
| "support": 20, |
| "correct": 2, |
| "recall": 0.1 |
| }, |
| "save_note": { |
| "support": 20, |
| "correct": 3, |
| "recall": 0.15 |
| }, |
| "search_notes": { |
| "support": 20, |
| "correct": 7, |
| "recall": 0.35 |
| }, |
| "phone_health_check": { |
| "support": 20, |
| "correct": 3, |
| "recall": 0.15 |
| }, |
| "search_local_files": { |
| "support": 20, |
| "correct": 7, |
| "recall": 0.35 |
| }, |
| "get_battery_info": { |
| "support": 20, |
| "correct": 6, |
| "recall": 0.3 |
| } |
| } |
| }, |
| "calibration": { |
| "valid_decisions": 0, |
| "note": "No decision probability distribution available" |
| }, |
| "predictions_sha256": "7d1a0e8b0942562ae8eed7ad24b3b99d3e23119397062b3200026988eaeed38f" |
| } |
| ], |
| "paired_fresh180": { |
| "qwen-fresh180": { |
| "n": 180, |
| "reflex_only_correct": 125, |
| "baseline_only_correct": 3, |
| "accuracy_difference": 0.6777777777777778, |
| "paired_bootstrap_95ci": [ |
| 0.6, |
| 0.75 |
| ], |
| "exact_mcnemar_two_sided_p": 2.0549580818052442e-33, |
| "bootstrap_seed": 3407, |
| "bootstrap_replicates": 10000 |
| }, |
| "qwen-legacy-fresh180": { |
| "n": 180, |
| "reflex_only_correct": 69, |
| "baseline_only_correct": 5, |
| "accuracy_difference": 0.35555555555555557, |
| "paired_bootstrap_95ci": [ |
| 0.2777777777777778, |
| 0.43333333333333335 |
| ], |
| "exact_mcnemar_two_sided_p": 1.8345664258432237e-15, |
| "bootstrap_seed": 3407, |
| "bootstrap_replicates": 10000 |
| }, |
| "qwen-legacy-cached-fresh180": { |
| "n": 180, |
| "reflex_only_correct": 70, |
| "baseline_only_correct": 4, |
| "accuracy_difference": 0.36666666666666664, |
| "paired_bootstrap_95ci": [ |
| 0.28888888888888886, |
| 0.4444444444444444 |
| ], |
| "exact_mcnemar_two_sided_p": 1.2898469490022716e-16, |
| "bootstrap_seed": 3407, |
| "bootstrap_replicates": 10000 |
| } |
| }, |
| "complete": true, |
| "original_expanded_queue_complete": false, |
| "required_jobs": [ |
| "qwen-fresh180", |
| "qwen-legacy-cached-fresh180", |
| "qwen-legacy-fresh180", |
| "run-01" |
| ], |
| "deferred_jobs": [ |
| "beta-fresh180", |
| "reflex-historical90", |
| "qwen-historical90", |
| "qwen-cached-fresh180", |
| "reflex-typed_decisions", |
| "reflex-banking77", |
| "reflex-clinc150", |
| "beta-typed_decisions", |
| "beta-banking77", |
| "beta-clinc150" |
| ], |
| "privacy": "Aggregate metrics only; no held-out requests, per-case responses, reviewer IDs or device-property dumps.", |
| "runtime": "Android CPU build11443 f0c41e0168dfd4b5ef72b21d1a311b24cc7a894a; 4 inference/batch threads; Q4_K_M; phone ctx2048, general ctx4096; USB powered", |
| "cohort_caveat": "Fresh180 and original historical90 are distinct; historical PyTorch reference used reviewed88. Raw Qwen prompt/JSON grammar treatment is specifically documented, not all possible Qwen inference configurations." |
| } |
|
|
