Spaces:
Running
Running
| { | |
| "study": "repeated-seed-policy-consolidation", | |
| "protocol": "matched-seeds-bootstrap-and-bounded-offline-oracle", | |
| "repetitions": 12, | |
| "bootstrap_samples": 600, | |
| "config": { | |
| "model": "Qwen2.5-3B", | |
| "accelerator": "L4", | |
| "quantization": "int8", | |
| "seed": 7, | |
| "duration_s": 120.0, | |
| "workflow_rate_rps": 0.18, | |
| "dynamic_prompt_tokens_mean": 320, | |
| "dynamic_prompt_cv": 0.4, | |
| "output_tokens_mean": 64, | |
| "output_tokens_cv": 0.45, | |
| "tool_gap_mean_s": 1.2, | |
| "tool_gap_cv": 0.65, | |
| "max_steps": 8, | |
| "shift_fraction": 0.55, | |
| "prefix_cache_budget_fraction": 0.62, | |
| "host_bandwidth_gbps": 32.0, | |
| "transfer_base_ms": 0.15, | |
| "prefetch_policy": "decayed", | |
| "transition_decay": 0.85, | |
| "confidence_threshold": 0.55, | |
| "prior_strength": 0.35, | |
| "forecast_horizon": 3, | |
| "prefetch_top_k": 2, | |
| "forecast_discount": 0.75, | |
| "forecast_min_score": 0.1, | |
| "utility_threshold_ms": 0.0, | |
| "timeline_points": 240 | |
| }, | |
| "policies": [ | |
| { | |
| "label": "No prefetch", | |
| "mean_ttft_ms": 6802.071434116555, | |
| "median_ttft_ms": 6490.661548131788, | |
| "ttft_ci95_low_ms": 5079.289415881574, | |
| "ttft_ci95_high_ms": 8648.920669433282, | |
| "paired_delta_vs_top1_mean_ms": -123.246831951494, | |
| "paired_delta_ci95_low_ms": -347.35450716706316, | |
| "paired_delta_ci95_high_ms": 47.018145035094136, | |
| "ttft_win_rate": 0.4166666666666667, | |
| "pareto_stability": 0.9166666666666666, | |
| "median_oracle_regret_ms": 34.71482691034703, | |
| "mean_oracle_regret_ms": 71.59505551067814, | |
| "oracle_regret_ci95_low_ms": 20.256193997241034, | |
| "oracle_regret_ci95_high_ms": 140.55634036325446, | |
| "worst_seed_ttft_ms": 14328.174603781865, | |
| "mean_unused_prefetch_gb": 0.0, | |
| "mean_hbm_gb": 0.05799153067644665, | |
| "mean_completion_rate": 1.0, | |
| "robust_rank": 2 | |
| }, | |
| { | |
| "label": "Top-1 decayed", | |
| "mean_ttft_ms": 6925.318266068049, | |
| "median_ttft_ms": 6474.91717792896, | |
| "ttft_ci95_low_ms": 5105.737657279251, | |
| "ttft_ci95_high_ms": 8965.982492493727, | |
| "paired_delta_vs_top1_mean_ms": 0.0, | |
| "paired_delta_ci95_low_ms": 0.0, | |
| "paired_delta_ci95_high_ms": 0.0, | |
| "ttft_win_rate": 0.4166666666666667, | |
| "pareto_stability": 0.75, | |
| "median_oracle_regret_ms": 59.272217524663574, | |
| "mean_oracle_regret_ms": 194.84188746217214, | |
| "oracle_regret_ci95_low_ms": 54.639857512317526, | |
| "oracle_regret_ci95_high_ms": 381.3867898898039, | |
| "worst_seed_ttft_ms": 14688.079385783494, | |
| "mean_unused_prefetch_gb": 0.0933888, | |
| "mean_hbm_gb": 0.05756119811086612, | |
| "mean_completion_rate": 1.0, | |
| "robust_rank": 1 | |
| }, | |
| { | |
| "label": "Multi-step top-k", | |
| "mean_ttft_ms": 7053.099997696324, | |
| "median_ttft_ms": 6640.303485776007, | |
| "ttft_ci95_low_ms": 5341.689146280877, | |
| "ttft_ci95_high_ms": 9082.241581969562, | |
| "paired_delta_vs_top1_mean_ms": 127.78173162827515, | |
| "paired_delta_ci95_low_ms": -78.61311912594543, | |
| "paired_delta_ci95_high_ms": 285.7402664741835, | |
| "ttft_win_rate": 0.08333333333333333, | |
| "pareto_stability": 0.8333333333333334, | |
| "median_oracle_regret_ms": 215.01034621539475, | |
| "mean_oracle_regret_ms": 322.6236190904473, | |
| "oracle_regret_ci95_low_ms": 163.1324022990532, | |
| "oracle_regret_ci95_high_ms": 482.28187467234943, | |
| "worst_seed_ttft_ms": 14999.960971925384, | |
| "mean_unused_prefetch_gb": 2.776301568, | |
| "mean_hbm_gb": 0.05447990052348106, | |
| "mean_completion_rate": 1.0, | |
| "robust_rank": 4 | |
| }, | |
| { | |
| "label": "Utility-aware multi-step", | |
| "mean_ttft_ms": 7008.096711452864, | |
| "median_ttft_ms": 6507.529283460886, | |
| "ttft_ci95_low_ms": 5199.467094878383, | |
| "ttft_ci95_high_ms": 9162.994321123462, | |
| "paired_delta_vs_top1_mean_ms": 82.77844538481479, | |
| "paired_delta_ci95_low_ms": -23.44354791792356, | |
| "paired_delta_ci95_high_ms": 214.12739203275032, | |
| "ttft_win_rate": 0.08333333333333333, | |
| "pareto_stability": 0.5833333333333334, | |
| "median_oracle_regret_ms": 133.08541840114094, | |
| "mean_oracle_regret_ms": 277.6203328469869, | |
| "oracle_regret_ci95_low_ms": 100.84880035727063, | |
| "oracle_regret_ci95_high_ms": 481.4559675209379, | |
| "worst_seed_ttft_ms": 14634.729400052454, | |
| "mean_unused_prefetch_gb": 1.69377792, | |
| "mean_hbm_gb": 0.057408777204922434, | |
| "mean_completion_rate": 1.0, | |
| "robust_rank": 3 | |
| } | |
| ], | |
| "seed_runs": [ | |
| { | |
| "rep": 1, | |
| "seed": 7, | |
| "rows": [ | |
| { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 2508.9158110308517, | |
| "p95_workflow_e2e_ms": 26656.556855177485, | |
| "workflow_throughput_rps": 0.1577017775042944, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.42857142857142855, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.0599599629511816, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 49, | |
| "saved_prefill_tokens": 26176, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 49.90402867636931, | |
| "oracle_regret_pct": 2.0294343050518706 | |
| }, | |
| { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 2459.0117823544824, | |
| "p95_workflow_e2e_ms": 26667.23774828302, | |
| "workflow_throughput_rps": 0.15768970423702405, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.4945054945054945, | |
| "prefetch_utilization": 0.8125, | |
| "future_role_recall_at_k": 0.13526570048309178, | |
| "mean_hbm_gb": 0.05817502692720098, | |
| "unused_prefetch_gb": 0.09909043199999998, | |
| "pressure_evictions": 59, | |
| "saved_prefill_tokens": 32320, | |
| "pareto": true, | |
| "ttft_winner": true, | |
| "oracle_regret_ms": 0.0, | |
| "oracle_regret_pct": 0.0 | |
| }, | |
| { | |
| "label": "Multi-step top-k", | |
| "p95_ttft_ms": 2599.2291767138786, | |
| "p95_workflow_e2e_ms": 27096.713211323044, | |
| "workflow_throughput_rps": 0.1572831336707985, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.37362637362637363, | |
| "prefetch_utilization": 0.1640625, | |
| "future_role_recall_at_k": 0.5700483091787439, | |
| "mean_hbm_gb": 0.054251348984577404, | |
| "unused_prefetch_gb": 2.96091648, | |
| "pressure_evictions": 183, | |
| "saved_prefill_tokens": 24320, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 140.21739435939617, | |
| "oracle_regret_pct": 5.702184729881173 | |
| }, | |
| { | |
| "label": "Utility-aware multi-step", | |
| "p95_ttft_ms": 2593.2876340481835, | |
| "p95_workflow_e2e_ms": 26697.511231196324, | |
| "workflow_throughput_rps": 0.15773330264095717, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.4835164835164835, | |
| "prefetch_utilization": 0.36619718309859156, | |
| "future_role_recall_at_k": 0.5096618357487923, | |
| "mean_hbm_gb": 0.0583545951025611, | |
| "unused_prefetch_gb": 1.2386304, | |
| "pressure_evictions": 115, | |
| "saved_prefill_tokens": 31744, | |
| "pareto": false, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 134.27585169370104, | |
| "oracle_regret_pct": 5.460561541723606 | |
| } | |
| ], | |
| "oracle": { | |
| "label": "Top-1 decayed", | |
| "kind": "deployable", | |
| "config": { | |
| "policy": "decayed", | |
| "horizon": 3, | |
| "top_k": 2 | |
| }, | |
| "metrics": { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 2459.0117823544824, | |
| "p95_workflow_e2e_ms": 26667.23774828302, | |
| "workflow_throughput_rps": 0.15768970423702405, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.4945054945054945, | |
| "prefetch_utilization": 0.8125, | |
| "future_role_recall_at_k": 0.13526570048309178, | |
| "mean_hbm_gb": 0.05817502692720098, | |
| "unused_prefetch_gb": 0.09909043199999998, | |
| "pressure_evictions": 59, | |
| "saved_prefill_tokens": 32320 | |
| }, | |
| "candidate_count": 19, | |
| "definition": "bounded-full-trace-serving-oracle", | |
| "note": "Exhaustive upper bound over the declared candidate family, including clairvoyant future-set plans. It uses the complete trace for policy selection and future-role actions, but is not a proof of global optimality." | |
| } | |
| }, | |
| { | |
| "rep": 2, | |
| "seed": 1016, | |
| "rows": [ | |
| { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 14328.174603781865, | |
| "p95_workflow_e2e_ms": 100660.43087786615, | |
| "workflow_throughput_rps": 0.1552783155805788, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5517241379310345, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.056923401277366226, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 63, | |
| "saved_prefill_tokens": 55296, | |
| "pareto": true, | |
| "ttft_winner": true, | |
| "oracle_regret_ms": 0.0, | |
| "oracle_regret_pct": 0.0 | |
| }, | |
| { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 14688.079385783494, | |
| "p95_workflow_e2e_ms": 100710.97712955547, | |
| "workflow_throughput_rps": 0.15479050575342704, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5379310344827586, | |
| "prefetch_utilization": 0.4166666666666667, | |
| "future_role_recall_at_k": 0.08764367816091954, | |
| "mean_hbm_gb": 0.056442561619372084, | |
| "unused_prefetch_gb": 0.202899456, | |
| "pressure_evictions": 77, | |
| "saved_prefill_tokens": 54272, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 359.9047820016294, | |
| "oracle_regret_pct": 2.5118676450706707 | |
| }, | |
| { | |
| "label": "Multi-step top-k", | |
| "p95_ttft_ms": 14999.960971925384, | |
| "p95_workflow_e2e_ms": 101549.98705723105, | |
| "workflow_throughput_rps": 0.15434055828231827, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.45517241379310347, | |
| "prefetch_utilization": 0.1787709497206704, | |
| "future_role_recall_at_k": 0.5689655172413793, | |
| "mean_hbm_gb": 0.05619447488989339, | |
| "unused_prefetch_gb": 4.05798912, | |
| "pressure_evictions": 256, | |
| "saved_prefill_tokens": 48256, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 671.7863681435192, | |
| "oracle_regret_pct": 4.688569107513555 | |
| }, | |
| { | |
| "label": "Utility-aware multi-step", | |
| "p95_ttft_ms": 14634.729400052454, | |
| "p95_workflow_e2e_ms": 100642.13358865703, | |
| "workflow_throughput_rps": 0.15493195402903986, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5172413793103449, | |
| "prefetch_utilization": 0.24603174603174602, | |
| "future_role_recall_at_k": 0.5545977011494253, | |
| "mean_hbm_gb": 0.05871891330180658, | |
| "unused_prefetch_gb": 2.5952256, | |
| "pressure_evictions": 194, | |
| "saved_prefill_tokens": 54976, | |
| "pareto": false, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 306.5547962705896, | |
| "oracle_regret_pct": 2.139524431742168 | |
| } | |
| ], | |
| "oracle": { | |
| "label": "No prefetch", | |
| "kind": "deployable", | |
| "config": { | |
| "policy": "none", | |
| "horizon": 3, | |
| "top_k": 2 | |
| }, | |
| "metrics": { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 14328.174603781865, | |
| "p95_workflow_e2e_ms": 100660.43087786615, | |
| "workflow_throughput_rps": 0.1552783155805788, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5517241379310345, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.056923401277366226, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 63, | |
| "saved_prefill_tokens": 55296 | |
| }, | |
| "candidate_count": 19, | |
| "definition": "bounded-full-trace-serving-oracle", | |
| "note": "Exhaustive upper bound over the declared candidate family, including clairvoyant future-set plans. It uses the complete trace for policy selection and future-role actions, but is not a proof of global optimality." | |
| } | |
| }, | |
| { | |
| "rep": 3, | |
| "seed": 2025, | |
| "rows": [ | |
| { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 5155.081577515879, | |
| "p95_workflow_e2e_ms": 39206.99976041411, | |
| "workflow_throughput_rps": 0.14003550604961185, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5686274509803921, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.05860227184049213, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 41, | |
| "saved_prefill_tokens": 41664, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 67.74273330644246, | |
| "oracle_regret_pct": 1.3315946781006989 | |
| }, | |
| { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 5087.3388442094365, | |
| "p95_workflow_e2e_ms": 38880.38800539655, | |
| "workflow_throughput_rps": 0.14024477205584812, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5980392156862745, | |
| "prefetch_utilization": 0.7142857142857143, | |
| "future_role_recall_at_k": 0.07028112449799197, | |
| "mean_hbm_gb": 0.05831155256626185, | |
| "unused_prefetch_gb": 0.066060288, | |
| "pressure_evictions": 45, | |
| "saved_prefill_tokens": 44288, | |
| "pareto": true, | |
| "ttft_winner": true, | |
| "oracle_regret_ms": 0.0, | |
| "oracle_regret_pct": 0.0 | |
| }, | |
| { | |
| "label": "Multi-step top-k", | |
| "p95_ttft_ms": 5308.289648386624, | |
| "p95_workflow_e2e_ms": 39362.44388383438, | |
| "workflow_throughput_rps": 0.13921395060797437, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5686274509803921, | |
| "prefetch_utilization": 0.24427480916030533, | |
| "future_role_recall_at_k": 0.5341365461847389, | |
| "mean_hbm_gb": 0.054135117019982595, | |
| "unused_prefetch_gb": 2.753298432, | |
| "pressure_evictions": 172, | |
| "saved_prefill_tokens": 43072, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 220.95080417718782, | |
| "oracle_regret_pct": 4.3431509271036814 | |
| }, | |
| { | |
| "label": "Utility-aware multi-step", | |
| "p95_ttft_ms": 5219.233829318017, | |
| "p95_workflow_e2e_ms": 39436.99299180437, | |
| "workflow_throughput_rps": 0.13981017212606242, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5588235294117647, | |
| "prefetch_utilization": 0.25274725274725274, | |
| "future_role_recall_at_k": 0.5281124497991968, | |
| "mean_hbm_gb": 0.05662322676367652, | |
| "unused_prefetch_gb": 1.882718208, | |
| "pressure_evictions": 133, | |
| "saved_prefill_tokens": 42624, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 131.89498510858084, | |
| "oracle_regret_pct": 2.5926125455297266 | |
| } | |
| ], | |
| "oracle": { | |
| "label": "Top-1 decayed", | |
| "kind": "deployable", | |
| "config": { | |
| "policy": "decayed", | |
| "horizon": 3, | |
| "top_k": 2 | |
| }, | |
| "metrics": { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 5087.3388442094365, | |
| "p95_workflow_e2e_ms": 38880.38800539655, | |
| "workflow_throughput_rps": 0.14024477205584812, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5980392156862745, | |
| "prefetch_utilization": 0.7142857142857143, | |
| "future_role_recall_at_k": 0.07028112449799197, | |
| "mean_hbm_gb": 0.05831155256626185, | |
| "unused_prefetch_gb": 0.066060288, | |
| "pressure_evictions": 45, | |
| "saved_prefill_tokens": 44288 | |
| }, | |
| "candidate_count": 19, | |
| "definition": "bounded-full-trace-serving-oracle", | |
| "note": "Exhaustive upper bound over the declared candidate family, including clairvoyant future-set plans. It uses the complete trace for policy selection and future-role actions, but is not a proof of global optimality." | |
| } | |
| }, | |
| { | |
| "rep": 4, | |
| "seed": 3034, | |
| "rows": [ | |
| { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 8081.134170861211, | |
| "p95_workflow_e2e_ms": 48408.234787678535, | |
| "workflow_throughput_rps": 0.19158305063580688, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.6384615384615384, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.058352465405090974, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 44, | |
| "saved_prefill_tokens": 56064, | |
| "pareto": true, | |
| "ttft_winner": true, | |
| "oracle_regret_ms": 0.0, | |
| "oracle_regret_pct": 0.0 | |
| }, | |
| { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 8345.445612762906, | |
| "p95_workflow_e2e_ms": 52337.020757564795, | |
| "workflow_throughput_rps": 0.19128533155691, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.6153846153846154, | |
| "prefetch_utilization": 1.0, | |
| "future_role_recall_at_k": 0.06397306397306397, | |
| "mean_hbm_gb": 0.058647667236896704, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 51, | |
| "saved_prefill_tokens": 54016, | |
| "pareto": false, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 264.3114419016947, | |
| "oracle_regret_pct": 3.270722107977659 | |
| }, | |
| { | |
| "label": "Multi-step top-k", | |
| "p95_ttft_ms": 8856.407713802682, | |
| "p95_workflow_e2e_ms": 55194.71833526091, | |
| "workflow_throughput_rps": 0.1895469793366219, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.4307692307692308, | |
| "prefetch_utilization": 0.15853658536585366, | |
| "future_role_recall_at_k": 0.5168350168350169, | |
| "mean_hbm_gb": 0.0523639442797909, | |
| "unused_prefetch_gb": 3.803185152, | |
| "pressure_evictions": 236, | |
| "saved_prefill_tokens": 41984, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 775.2735429414715, | |
| "oracle_regret_pct": 9.593622956254546 | |
| }, | |
| { | |
| "label": "Utility-aware multi-step", | |
| "p95_ttft_ms": 9065.113180839384, | |
| "p95_workflow_e2e_ms": 54196.43843042602, | |
| "workflow_throughput_rps": 0.1905534529191346, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5076923076923077, | |
| "prefetch_utilization": 0.26666666666666666, | |
| "future_role_recall_at_k": 0.4764309764309764, | |
| "mean_hbm_gb": 0.05500864112250217, | |
| "unused_prefetch_gb": 2.538602496, | |
| "pressure_evictions": 181, | |
| "saved_prefill_tokens": 48640, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 983.979009978173, | |
| "oracle_regret_pct": 12.17624889246592 | |
| } | |
| ], | |
| "oracle": { | |
| "label": "No prefetch", | |
| "kind": "deployable", | |
| "config": { | |
| "policy": "none", | |
| "horizon": 3, | |
| "top_k": 2 | |
| }, | |
| "metrics": { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 8081.134170861211, | |
| "p95_workflow_e2e_ms": 48408.234787678535, | |
| "workflow_throughput_rps": 0.19158305063580688, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.6384615384615384, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.058352465405090974, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 44, | |
| "saved_prefill_tokens": 56064 | |
| }, | |
| "candidate_count": 19, | |
| "definition": "bounded-full-trace-serving-oracle", | |
| "note": "Exhaustive upper bound over the declared candidate family, including clairvoyant future-set plans. It uses the complete trace for policy selection and future-role actions, but is not a proof of global optimality." | |
| } | |
| }, | |
| { | |
| "rep": 5, | |
| "seed": 4043, | |
| "rows": [ | |
| { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 3974.694612251191, | |
| "p95_workflow_e2e_ms": 18764.681149810734, | |
| "workflow_throughput_rps": 0.14120209678649592, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.48484848484848486, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.053247840133508445, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 32, | |
| "saved_prefill_tokens": 21760, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 85.79909391552974, | |
| "oracle_regret_pct": 2.2062586539287987 | |
| }, | |
| { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 3888.895518335661, | |
| "p95_workflow_e2e_ms": 18671.951222411175, | |
| "workflow_throughput_rps": 0.1412992144994211, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5, | |
| "prefetch_utilization": 0.7142857142857143, | |
| "future_role_recall_at_k": 0.05555555555555555, | |
| "mean_hbm_gb": 0.05256473368734914, | |
| "unused_prefetch_gb": 0.061341696, | |
| "pressure_evictions": 38, | |
| "saved_prefill_tokens": 22720, | |
| "pareto": true, | |
| "ttft_winner": true, | |
| "oracle_regret_ms": 0.0, | |
| "oracle_regret_pct": 0.0 | |
| }, | |
| { | |
| "label": "Multi-step top-k", | |
| "p95_ttft_ms": 3990.111065086865, | |
| "p95_workflow_e2e_ms": 18883.9144056687, | |
| "workflow_throughput_rps": 0.1409739259838125, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.3939393939393939, | |
| "prefetch_utilization": 0.16091954022988506, | |
| "future_role_recall_at_k": 0.5104166666666666, | |
| "mean_hbm_gb": 0.045406284458674596, | |
| "unused_prefetch_gb": 2.01719808, | |
| "pressure_evictions": 125, | |
| "saved_prefill_tokens": 18496, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 101.21554675120387, | |
| "oracle_regret_pct": 2.6026810510589726 | |
| }, | |
| { | |
| "label": "Utility-aware multi-step", | |
| "p95_ttft_ms": 3961.805766823279, | |
| "p95_workflow_e2e_ms": 19016.7853637802, | |
| "workflow_throughput_rps": 0.1409052485639348, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.36363636363636365, | |
| "prefetch_utilization": 0.16923076923076924, | |
| "future_role_recall_at_k": 0.4895833333333333, | |
| "mean_hbm_gb": 0.04901078264991126, | |
| "unused_prefetch_gb": 1.519386624, | |
| "pressure_evictions": 105, | |
| "saved_prefill_tokens": 16960, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 72.91024848761799, | |
| "oracle_regret_pct": 1.8748317650565616 | |
| } | |
| ], | |
| "oracle": { | |
| "label": "Top-1 decayed", | |
| "kind": "deployable", | |
| "config": { | |
| "policy": "decayed", | |
| "horizon": 3, | |
| "top_k": 2 | |
| }, | |
| "metrics": { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 3888.895518335661, | |
| "p95_workflow_e2e_ms": 18671.951222411175, | |
| "workflow_throughput_rps": 0.1412992144994211, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5, | |
| "prefetch_utilization": 0.7142857142857143, | |
| "future_role_recall_at_k": 0.05555555555555555, | |
| "mean_hbm_gb": 0.05256473368734914, | |
| "unused_prefetch_gb": 0.061341696, | |
| "pressure_evictions": 38, | |
| "saved_prefill_tokens": 22720 | |
| }, | |
| "candidate_count": 19, | |
| "definition": "bounded-full-trace-serving-oracle", | |
| "note": "Exhaustive upper bound over the declared candidate family, including clairvoyant future-set plans. It uses the complete trace for policy selection and future-role actions, but is not a proof of global optimality." | |
| } | |
| }, | |
| { | |
| "rep": 6, | |
| "seed": 5052, | |
| "rows": [ | |
| { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 7309.107177665744, | |
| "p95_workflow_e2e_ms": 47619.448084895426, | |
| "workflow_throughput_rps": 0.1484392590003076, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.48484848484848486, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.05471706918379192, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 49, | |
| "saved_prefill_tokens": 33280, | |
| "pareto": true, | |
| "ttft_winner": true, | |
| "oracle_regret_ms": 0.0, | |
| "oracle_regret_pct": 0.0 | |
| }, | |
| { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 7309.87009396774, | |
| "p95_workflow_e2e_ms": 47381.367578903475, | |
| "workflow_throughput_rps": 0.14866123026478553, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5454545454545454, | |
| "prefetch_utilization": 0.75, | |
| "future_role_recall_at_k": 0.11842105263157894, | |
| "mean_hbm_gb": 0.054990227626166545, | |
| "unused_prefetch_gb": 0.089653248, | |
| "pressure_evictions": 54, | |
| "saved_prefill_tokens": 38592, | |
| "pareto": false, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 0.7629163019964835, | |
| "oracle_regret_pct": 0.0104378863717816 | |
| }, | |
| { | |
| "label": "Multi-step top-k", | |
| "p95_ttft_ms": 7517.707112366715, | |
| "p95_workflow_e2e_ms": 47497.83258699973, | |
| "workflow_throughput_rps": 0.14871579121181555, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5454545454545454, | |
| "prefetch_utilization": 0.33620689655172414, | |
| "future_role_recall_at_k": 0.5394736842105263, | |
| "mean_hbm_gb": 0.05581068796513829, | |
| "unused_prefetch_gb": 2.137522176, | |
| "pressure_evictions": 159, | |
| "saved_prefill_tokens": 40064, | |
| "pareto": false, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 208.59993470097106, | |
| "oracle_regret_pct": 2.8539728537349225 | |
| }, | |
| { | |
| "label": "Utility-aware multi-step", | |
| "p95_ttft_ms": 7506.956342852778, | |
| "p95_workflow_e2e_ms": 47305.033350410624, | |
| "workflow_throughput_rps": 0.14888303988588122, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5656565656565656, | |
| "prefetch_utilization": 0.34177215189873417, | |
| "future_role_recall_at_k": 0.5197368421052632, | |
| "mean_hbm_gb": 0.05676471264317459, | |
| "unused_prefetch_gb": 1.42737408, | |
| "pressure_evictions": 119, | |
| "saved_prefill_tokens": 41856, | |
| "pareto": false, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 197.8491651870345, | |
| "oracle_regret_pct": 2.7068855385182644 | |
| } | |
| ], | |
| "oracle": { | |
| "label": "No prefetch", | |
| "kind": "deployable", | |
| "config": { | |
| "policy": "none", | |
| "horizon": 3, | |
| "top_k": 2 | |
| }, | |
| "metrics": { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 7309.107177665744, | |
| "p95_workflow_e2e_ms": 47619.448084895426, | |
| "workflow_throughput_rps": 0.1484392590003076, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.48484848484848486, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.05471706918379192, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 49, | |
| "saved_prefill_tokens": 33280 | |
| }, | |
| "candidate_count": 19, | |
| "definition": "bounded-full-trace-serving-oracle", | |
| "note": "Exhaustive upper bound over the declared candidate family, including clairvoyant future-set plans. It uses the complete trace for policy selection and future-role actions, but is not a proof of global optimality." | |
| } | |
| }, | |
| { | |
| "rep": 7, | |
| "seed": 6061, | |
| "rows": [ | |
| { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 5672.215918597831, | |
| "p95_workflow_e2e_ms": 39980.01576300871, | |
| "workflow_throughput_rps": 0.1526413288015884, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.4574468085106383, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.05576787962082442, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 49, | |
| "saved_prefill_tokens": 30272, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 222.25583063586964, | |
| "oracle_regret_pct": 4.0781185008454495 | |
| }, | |
| { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 5639.96426189018, | |
| "p95_workflow_e2e_ms": 42335.09842778248, | |
| "workflow_throughput_rps": 0.15362763283307423, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.4574468085106383, | |
| "prefetch_utilization": 0.375, | |
| "future_role_recall_at_k": 0.07432432432432433, | |
| "mean_hbm_gb": 0.056659199341786655, | |
| "unused_prefetch_gb": 0.155713536, | |
| "pressure_evictions": 57, | |
| "saved_prefill_tokens": 31424, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 190.00417392821873, | |
| "oracle_regret_pct": 3.4863406495013747 | |
| }, | |
| { | |
| "label": "Multi-step top-k", | |
| "p95_ttft_ms": 5449.960087961961, | |
| "p95_workflow_e2e_ms": 37358.71643947743, | |
| "workflow_throughput_rps": 0.15301958531801996, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5319148936170213, | |
| "prefetch_utilization": 0.30392156862745096, | |
| "future_role_recall_at_k": 0.5112612612612613, | |
| "mean_hbm_gb": 0.054008815280052504, | |
| "unused_prefetch_gb": 1.880358912, | |
| "pressure_evictions": 144, | |
| "saved_prefill_tokens": 37120, | |
| "pareto": true, | |
| "ttft_winner": true, | |
| "oracle_regret_ms": 0.0, | |
| "oracle_regret_pct": 0.0 | |
| }, | |
| { | |
| "label": "Utility-aware multi-step", | |
| "p95_ttft_ms": 5471.268696264992, | |
| "p95_workflow_e2e_ms": 37317.903368275336, | |
| "workflow_throughput_rps": 0.15319363127947774, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5531914893617021, | |
| "prefetch_utilization": 0.36363636363636365, | |
| "future_role_recall_at_k": 0.45495495495495497, | |
| "mean_hbm_gb": 0.05709563959469793, | |
| "unused_prefetch_gb": 1.078198272, | |
| "pressure_evictions": 106, | |
| "saved_prefill_tokens": 38208, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 21.30860830303118, | |
| "oracle_regret_pct": 0.39098650190298245 | |
| } | |
| ], | |
| "oracle": { | |
| "label": "Multi-step top-k", | |
| "kind": "deployable", | |
| "config": { | |
| "policy": "multistep", | |
| "horizon": 3, | |
| "top_k": 2 | |
| }, | |
| "metrics": { | |
| "label": "Multi-step top-k", | |
| "p95_ttft_ms": 5449.960087961961, | |
| "p95_workflow_e2e_ms": 37358.71643947743, | |
| "workflow_throughput_rps": 0.15301958531801996, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5319148936170213, | |
| "prefetch_utilization": 0.30392156862745096, | |
| "future_role_recall_at_k": 0.5112612612612613, | |
| "mean_hbm_gb": 0.054008815280052504, | |
| "unused_prefetch_gb": 1.880358912, | |
| "pressure_evictions": 144, | |
| "saved_prefill_tokens": 37120 | |
| }, | |
| "candidate_count": 19, | |
| "definition": "bounded-full-trace-serving-oracle", | |
| "note": "Exhaustive upper bound over the declared candidate family, including clairvoyant future-set plans. It uses the complete trace for policy selection and future-role actions, but is not a proof of global optimality." | |
| } | |
| }, | |
| { | |
| "rep": 8, | |
| "seed": 7070, | |
| "rows": [ | |
| { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 7955.834288389353, | |
| "p95_workflow_e2e_ms": 63458.63274772634, | |
| "workflow_throughput_rps": 0.14314957761896735, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5392156862745098, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.05757162554782224, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 44, | |
| "saved_prefill_tokens": 36608, | |
| "pareto": false, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 360.1196655210979, | |
| "oracle_regret_pct": 4.741089988253289 | |
| }, | |
| { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 7663.490235151211, | |
| "p95_workflow_e2e_ms": 63010.589234624604, | |
| "workflow_throughput_rps": 0.14356812595083052, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5882352941176471, | |
| "prefetch_utilization": 1.0, | |
| "future_role_recall_at_k": 0.08125, | |
| "mean_hbm_gb": 0.05697446543608208, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 43, | |
| "saved_prefill_tokens": 40448, | |
| "pareto": true, | |
| "ttft_winner": true, | |
| "oracle_regret_ms": 67.77561228295599, | |
| "oracle_regret_pct": 0.8922875022042747 | |
| }, | |
| { | |
| "label": "Multi-step top-k", | |
| "p95_ttft_ms": 7804.784511121857, | |
| "p95_workflow_e2e_ms": 63128.95114673241, | |
| "workflow_throughput_rps": 0.14353084410317019, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5980392156862745, | |
| "prefetch_utilization": 0.33766233766233766, | |
| "future_role_recall_at_k": 0.5333333333333333, | |
| "mean_hbm_gb": 0.05700914860577035, | |
| "unused_prefetch_gb": 1.363673088, | |
| "pressure_evictions": 115, | |
| "saved_prefill_tokens": 42816, | |
| "pareto": false, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 209.06988825360168, | |
| "oracle_regret_pct": 2.7524716058204644 | |
| }, | |
| { | |
| "label": "Utility-aware multi-step", | |
| "p95_ttft_ms": 7821.45396304061, | |
| "p95_workflow_e2e_ms": 63158.31108861863, | |
| "workflow_throughput_rps": 0.14349908334078196, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5980392156862745, | |
| "prefetch_utilization": 0.4230769230769231, | |
| "future_role_recall_at_k": 0.5333333333333333, | |
| "mean_hbm_gb": 0.06056314337054253, | |
| "unused_prefetch_gb": 0.7667712, | |
| "pressure_evictions": 90, | |
| "saved_prefill_tokens": 41472, | |
| "pareto": false, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 225.7393401723548, | |
| "oracle_regret_pct": 2.97193024462407 | |
| } | |
| ], | |
| "oracle": { | |
| "label": "clairvoyant H1/K1", | |
| "kind": "clairvoyant", | |
| "config": { | |
| "policy": "oracle_horizon", | |
| "horizon": 1, | |
| "top_k": 1 | |
| }, | |
| "metrics": { | |
| "label": "clairvoyant H1/K1", | |
| "p95_ttft_ms": 7595.714622868255, | |
| "p95_workflow_e2e_ms": 62644.984909819585, | |
| "workflow_throughput_rps": 0.1439811704946119, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.6568627450980392, | |
| "prefetch_utilization": 0.5111111111111111, | |
| "future_role_recall_at_k": 1.0, | |
| "mean_hbm_gb": 0.0593557108369556, | |
| "unused_prefetch_gb": 0.606339072, | |
| "pressure_evictions": 77, | |
| "saved_prefill_tokens": 46272 | |
| }, | |
| "candidate_count": 19, | |
| "definition": "bounded-full-trace-serving-oracle", | |
| "note": "Exhaustive upper bound over the declared candidate family, including clairvoyant future-set plans. It uses the complete trace for policy selection and future-role actions, but is not a proof of global optimality." | |
| } | |
| }, | |
| { | |
| "rep": 9, | |
| "seed": 8079, | |
| "rows": [ | |
| { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 2321.8881795970965, | |
| "p95_workflow_e2e_ms": 23073.36518580538, | |
| "workflow_throughput_rps": 0.14600754420187276, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.05803852146421546, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 38, | |
| "saved_prefill_tokens": 28544, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 23.945841894531895, | |
| "oracle_regret_pct": 1.0420558210556516 | |
| }, | |
| { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 2297.9423377025646, | |
| "p95_workflow_e2e_ms": 22972.359046059566, | |
| "workflow_throughput_rps": 0.14610416511124308, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.524390243902439, | |
| "prefetch_utilization": 0.625, | |
| "future_role_recall_at_k": 0.044270833333333336, | |
| "mean_hbm_gb": 0.05741847323154553, | |
| "unused_prefetch_gb": 0.084934656, | |
| "pressure_evictions": 44, | |
| "saved_prefill_tokens": 30208, | |
| "pareto": true, | |
| "ttft_winner": true, | |
| "oracle_regret_ms": 0.0, | |
| "oracle_regret_pct": 0.0 | |
| }, | |
| { | |
| "label": "Multi-step top-k", | |
| "p95_ttft_ms": 2358.7664926649986, | |
| "p95_workflow_e2e_ms": 23190.068065034015, | |
| "workflow_throughput_rps": 0.14635545368977013, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.4878048780487805, | |
| "prefetch_utilization": 0.23232323232323232, | |
| "future_role_recall_at_k": 0.5130208333333334, | |
| "mean_hbm_gb": 0.052585587672706685, | |
| "unused_prefetch_gb": 2.102132736, | |
| "pressure_evictions": 138, | |
| "saved_prefill_tokens": 28608, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 60.82415496243402, | |
| "oracle_regret_pct": 2.646896484932897 | |
| }, | |
| { | |
| "label": "Utility-aware multi-step", | |
| "p95_ttft_ms": 2354.8786376844973, | |
| "p95_workflow_e2e_ms": 22931.69235351764, | |
| "workflow_throughput_rps": 0.1463801438395915, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.573170731707317, | |
| "prefetch_utilization": 0.3220338983050847, | |
| "future_role_recall_at_k": 0.5208333333333334, | |
| "mean_hbm_gb": 0.0555423079515505, | |
| "unused_prefetch_gb": 1.14425856, | |
| "pressure_evictions": 91, | |
| "saved_prefill_tokens": 34176, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 56.936299981932734, | |
| "oracle_regret_pct": 2.4777079497502306 | |
| } | |
| ], | |
| "oracle": { | |
| "label": "Top-1 decayed", | |
| "kind": "deployable", | |
| "config": { | |
| "policy": "decayed", | |
| "horizon": 3, | |
| "top_k": 2 | |
| }, | |
| "metrics": { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 2297.9423377025646, | |
| "p95_workflow_e2e_ms": 22972.359046059566, | |
| "workflow_throughput_rps": 0.14610416511124308, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.524390243902439, | |
| "prefetch_utilization": 0.625, | |
| "future_role_recall_at_k": 0.044270833333333336, | |
| "mean_hbm_gb": 0.05741847323154553, | |
| "unused_prefetch_gb": 0.084934656, | |
| "pressure_evictions": 44, | |
| "saved_prefill_tokens": 30208 | |
| }, | |
| "candidate_count": 19, | |
| "definition": "bounded-full-trace-serving-oracle", | |
| "note": "Exhaustive upper bound over the declared candidate family, including clairvoyant future-set plans. It uses the complete trace for policy selection and future-role actions, but is not a proof of global optimality." | |
| } | |
| }, | |
| { | |
| "rep": 10, | |
| "seed": 9088, | |
| "rows": [ | |
| { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 8393.380878206393, | |
| "p95_workflow_e2e_ms": 57992.77157876873, | |
| "workflow_throughput_rps": 0.18631227548846913, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5902777777777778, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.0608883540146992, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 57, | |
| "saved_prefill_tokens": 59328, | |
| "pareto": true, | |
| "ttft_winner": true, | |
| "oracle_regret_ms": 45.48381192616216, | |
| "oracle_regret_pct": 0.5448535309555447 | |
| }, | |
| { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 8603.374943396353, | |
| "p95_workflow_e2e_ms": 57847.66200555299, | |
| "workflow_throughput_rps": 0.18647490382989684, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5902777777777778, | |
| "prefetch_utilization": 0.6111111111111112, | |
| "future_role_recall_at_k": 0.09734513274336283, | |
| "mean_hbm_gb": 0.06097675059220976, | |
| "unused_prefetch_gb": 0.207618048, | |
| "pressure_evictions": 75, | |
| "saved_prefill_tokens": 60544, | |
| "pareto": false, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 255.47787711612182, | |
| "oracle_regret_pct": 3.0603860479794 | |
| }, | |
| { | |
| "label": "Multi-step top-k", | |
| "p95_ttft_ms": 9184.166173240812, | |
| "p95_workflow_e2e_ms": 61245.254849217556, | |
| "workflow_throughput_rps": 0.1857623669079808, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.4861111111111111, | |
| "prefetch_utilization": 0.2154696132596685, | |
| "future_role_recall_at_k": 0.6002949852507374, | |
| "mean_hbm_gb": 0.05400505762073378, | |
| "unused_prefetch_gb": 3.925868544, | |
| "pressure_evictions": 253, | |
| "saved_prefill_tokens": 51264, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 836.2691069605808, | |
| "oracle_regret_pct": 10.017721832466448 | |
| }, | |
| { | |
| "label": "Utility-aware multi-step", | |
| "p95_ttft_ms": 8457.279531340011, | |
| "p95_workflow_e2e_ms": 60583.73120156277, | |
| "workflow_throughput_rps": 0.18650546463814055, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.4722222222222222, | |
| "prefetch_utilization": 0.23809523809523808, | |
| "future_role_recall_at_k": 0.5265486725663717, | |
| "mean_hbm_gb": 0.05733022563941532, | |
| "unused_prefetch_gb": 2.691956736, | |
| "pressure_evictions": 200, | |
| "saved_prefill_tokens": 50432, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 109.38246505977986, | |
| "oracle_regret_pct": 1.3102996382359562 | |
| } | |
| ], | |
| "oracle": { | |
| "label": "clairvoyant H3/K3", | |
| "kind": "clairvoyant", | |
| "config": { | |
| "policy": "oracle_horizon", | |
| "horizon": 3, | |
| "top_k": 3 | |
| }, | |
| "metrics": { | |
| "label": "clairvoyant H3/K3", | |
| "p95_ttft_ms": 8347.897066280231, | |
| "p95_workflow_e2e_ms": 58517.02815386715, | |
| "workflow_throughput_rps": 0.18581440744158234, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5416666666666666, | |
| "prefetch_utilization": 0.24, | |
| "future_role_recall_at_k": 1.0, | |
| "mean_hbm_gb": 0.05724896307500904, | |
| "unused_prefetch_gb": 3.104833536, | |
| "pressure_evictions": 214, | |
| "saved_prefill_tokens": 55872 | |
| }, | |
| "candidate_count": 19, | |
| "definition": "bounded-full-trace-serving-oracle", | |
| "note": "Exhaustive upper bound over the declared candidate family, including clairvoyant future-set plans. It uses the complete trace for policy selection and future-role actions, but is not a proof of global optimality." | |
| } | |
| }, | |
| { | |
| "rep": 11, | |
| "seed": 10097, | |
| "rows": [ | |
| { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 5511.991884321129, | |
| "p95_workflow_e2e_ms": 39379.92626385874, | |
| "workflow_throughput_rps": 0.18375402111103414, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5405405405405406, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.06035724347004058, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 48, | |
| "saved_prefill_tokens": 39744, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 3.8896602521344903, | |
| "oracle_regret_pct": 0.07061706725662556 | |
| }, | |
| { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 5558.871046835366, | |
| "p95_workflow_e2e_ms": 39579.196475851335, | |
| "workflow_throughput_rps": 0.18346078167175595, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5135135135135135, | |
| "prefetch_utilization": 0.5, | |
| "future_role_recall_at_k": 0.047619047619047616, | |
| "mean_hbm_gb": 0.059572909587982954, | |
| "unused_prefetch_gb": 0.101449728, | |
| "pressure_evictions": 59, | |
| "saved_prefill_tokens": 37696, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 50.76882276637116, | |
| "oracle_regret_pct": 0.921711702163486 | |
| }, | |
| { | |
| "label": "Multi-step top-k", | |
| "p95_ttft_ms": 5762.899859185298, | |
| "p95_workflow_e2e_ms": 40501.25080020354, | |
| "workflow_throughput_rps": 0.18338521635563007, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.4864864864864865, | |
| "prefetch_utilization": 0.23529411764705882, | |
| "future_role_recall_at_k": 0.5575396825396826, | |
| "mean_hbm_gb": 0.05918820422391816, | |
| "unused_prefetch_gb": 2.8311552, | |
| "pressure_evictions": 190, | |
| "saved_prefill_tokens": 38016, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 254.79763511630335, | |
| "oracle_regret_pct": 4.625869759695146 | |
| }, | |
| { | |
| "label": "Utility-aware multi-step", | |
| "p95_ttft_ms": 5508.1022240689945, | |
| "p95_workflow_e2e_ms": 39558.94633951845, | |
| "workflow_throughput_rps": 0.18374015675838837, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5135135135135135, | |
| "prefetch_utilization": 0.3375, | |
| "future_role_recall_at_k": 0.5555555555555556, | |
| "mean_hbm_gb": 0.06210299331912212, | |
| "unused_prefetch_gb": 1.382547456, | |
| "pressure_evictions": 131, | |
| "saved_prefill_tokens": 39616, | |
| "pareto": true, | |
| "ttft_winner": true, | |
| "oracle_regret_ms": 0.0, | |
| "oracle_regret_pct": 0.0 | |
| } | |
| ], | |
| "oracle": { | |
| "label": "Utility-aware multi-step", | |
| "kind": "deployable", | |
| "config": { | |
| "policy": "utility", | |
| "horizon": 3, | |
| "top_k": 2 | |
| }, | |
| "metrics": { | |
| "label": "Utility-aware multi-step", | |
| "p95_ttft_ms": 5508.1022240689945, | |
| "p95_workflow_e2e_ms": 39558.94633951845, | |
| "workflow_throughput_rps": 0.18374015675838837, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.5135135135135135, | |
| "prefetch_utilization": 0.3375, | |
| "future_role_recall_at_k": 0.5555555555555556, | |
| "mean_hbm_gb": 0.06210299331912212, | |
| "unused_prefetch_gb": 1.382547456, | |
| "pressure_evictions": 131, | |
| "saved_prefill_tokens": 39616 | |
| }, | |
| "candidate_count": 19, | |
| "definition": "bounded-full-trace-serving-oracle", | |
| "note": "Exhaustive upper bound over the declared candidate family, including clairvoyant future-set plans. It uses the complete trace for policy selection and future-role actions, but is not a proof of global optimality." | |
| } | |
| }, | |
| { | |
| "rep": 12, | |
| "seed": 11106, | |
| "rows": [ | |
| { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 10412.438107180116, | |
| "p95_workflow_e2e_ms": 72277.2430475709, | |
| "workflow_throughput_rps": 0.17028600915598113, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.46715328467153283, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.061471733208326614, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 71, | |
| "saved_prefill_tokens": 44800, | |
| "pareto": true, | |
| "ttft_winner": true, | |
| "oracle_regret_ms": 0.0, | |
| "oracle_regret_pct": 0.0 | |
| }, | |
| { | |
| "label": "Top-1 decayed", | |
| "p95_ttft_ms": 11561.535130427194, | |
| "p95_workflow_e2e_ms": 71765.84830537342, | |
| "workflow_throughput_rps": 0.17047301779333604, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.4744525547445255, | |
| "prefetch_utilization": 0.8571428571428571, | |
| "future_role_recall_at_k": 0.05, | |
| "mean_hbm_gb": 0.06000080947753916, | |
| "unused_prefetch_gb": 0.051904512, | |
| "pressure_evictions": 84, | |
| "saved_prefill_tokens": 46080, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 1149.0970232470772, | |
| "oracle_regret_pct": 11.035811319297958 | |
| }, | |
| { | |
| "label": "Multi-step top-k", | |
| "p95_ttft_ms": 10804.917159898814, | |
| "p95_workflow_e2e_ms": 72858.27325261911, | |
| "workflow_throughput_rps": 0.1698397999622364, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.46715328467153283, | |
| "prefetch_utilization": 0.24096385542168675, | |
| "future_role_recall_at_k": 0.5909090909090909, | |
| "mean_hbm_gb": 0.05880013528053406, | |
| "unused_prefetch_gb": 3.482320896, | |
| "pressure_evictions": 236, | |
| "saved_prefill_tokens": 46592, | |
| "pareto": true, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 392.47905271869786, | |
| "oracle_regret_pct": 3.769329033975776 | |
| }, | |
| { | |
| "label": "Utility-aware multi-step", | |
| "p95_ttft_ms": 11503.051331101164, | |
| "p95_workflow_e2e_ms": 70618.34835590329, | |
| "workflow_throughput_rps": 0.1704137373168054, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.45985401459854014, | |
| "prefetch_utilization": 0.3090909090909091, | |
| "future_role_recall_at_k": 0.5409090909090909, | |
| "mean_hbm_gb": 0.06179014500010861, | |
| "unused_prefetch_gb": 2.059665408, | |
| "pressure_evictions": 181, | |
| "saved_prefill_tokens": 45696, | |
| "pareto": false, | |
| "ttft_winner": false, | |
| "oracle_regret_ms": 1090.6132239210474, | |
| "oracle_regret_pct": 10.474138839480755 | |
| } | |
| ], | |
| "oracle": { | |
| "label": "No prefetch", | |
| "kind": "deployable", | |
| "config": { | |
| "policy": "none", | |
| "horizon": 3, | |
| "top_k": 2 | |
| }, | |
| "metrics": { | |
| "label": "No prefetch", | |
| "p95_ttft_ms": 10412.438107180116, | |
| "p95_workflow_e2e_ms": 72277.2430475709, | |
| "workflow_throughput_rps": 0.17028600915598113, | |
| "completion_rate": 1.0, | |
| "prefix_hit_rate": 0.46715328467153283, | |
| "prefetch_utilization": 0.0, | |
| "future_role_recall_at_k": 0.0, | |
| "mean_hbm_gb": 0.061471733208326614, | |
| "unused_prefetch_gb": 0.0, | |
| "pressure_evictions": 71, | |
| "saved_prefill_tokens": 44800 | |
| }, | |
| "candidate_count": 19, | |
| "definition": "bounded-full-trace-serving-oracle", | |
| "note": "Exhaustive upper bound over the declared candidate family, including clairvoyant future-set plans. It uses the complete trace for policy selection and future-role actions, but is not a proof of global optimality." | |
| } | |
| } | |
| ], | |
| "nominal_winner_first_seed": "Top-1 decayed", | |
| "robust_winner": "Top-1 decayed", | |
| "oracle": { | |
| "definition": "bounded-full-trace-serving-oracle", | |
| "candidate_count_per_seed": 19, | |
| "median_ttft_ms": 6408.604700867369, | |
| "mean_ttft_ms": 6730.476378605877, | |
| "selected_plan_frequency": { | |
| "Top-1 decayed": 4, | |
| "No prefetch": 4, | |
| "Multi-step top-k": 1, | |
| "clairvoyant H1/K1": 1, | |
| "clairvoyant H3/K3": 1, | |
| "Utility-aware multi-step": 1 | |
| }, | |
| "note": "The oracle exhaustively selects among the evaluated deployable policies plus clairvoyant future-set plans over H=1..5 and K=1..3 on each complete trace, under the same cache/bandwidth constraints. It is a bounded information upper bound, not a proof of globally optimal action scheduling." | |
| }, | |
| "pareto_objectives": [ | |
| "p95_step_ttft_ms", | |
| "unused_prefetch_gb", | |
| "mean_hbm_gb" | |
| ], | |
| "note": "Robust rank prioritizes how often a policy wins TTFT across matched seeds, then median TTFT and regret to the bounded offline oracle. Pareto stability is reported separately rather than silently overriding latency. Bootstrap intervals quantify seed uncertainty, not real-hardware error." | |
| } |