| { |
| "status": "LOCAL_FUNCTIONAL_RUNTIME_SMOKE_PASS_NOT_BENCHMARK_OR_SEMANTIC_QUALIFICATION", |
| "hardware": "NVIDIA GeForce RTX 5080", |
| "precision": "FP32 parameters,CUDA BF16 autocast,cacheoff/TF32off,one Torch thread", |
| "Select_v3": { |
| "local_packaged_weight_load": "PASS", |
| "finite_aligned_group_scores": "PASS", |
| "groups": 2, |
| "encoded_tokens": 498, |
| "windows": 1, |
| "offline_guard_violations": 0 |
| }, |
| "controls": { |
| "kompress-native-bf16": { |
| "status": "NATIVE_EAGER_BATCH32_RUNTIME_FUNCTIONAL_HEALTH_PASS_NOT_SEMANTIC_QUALIFICATION", |
| "original_worker_and_guardian_waited": true |
| }, |
| "v1-bf16": { |
| "status": "FUNCTIONAL_FINITE_ALIGNED_SCORE_PASS", |
| "original_worker_and_guardian_waited": true |
| }, |
| "v2-bf16": { |
| "status": "FUNCTIONAL_FINITE_ALIGNED_SCORE_PASS", |
| "original_worker_and_guardian_waited": true |
| } |
| }, |
| "Kompress_native_batch32_shape_probe": "PASS;nativeEAGER not batch8export", |
| "Kompress_budget_probes": [ |
| { |
| "ratio": 0.25, |
| "input_tokens": 336, |
| "target_tokens": 84, |
| "output_tokens": 336, |
| "valid_budget": false, |
| "no_op": true, |
| "refusal_not_win": true |
| }, |
| { |
| "ratio": 0.5, |
| "input_tokens": 336, |
| "target_tokens": 168, |
| "output_tokens": 336, |
| "valid_budget": false, |
| "no_op": true, |
| "refusal_not_win": true |
| } |
| ], |
| "new_training_compilation_capture_or_model_downloads": 0, |
| "comparative_latency_quality_superiority_or90percent_claim": false, |
| "independent_semantic_health_not_certified": true, |
| "fresh_machine_install_not_certified": true, |
| "original_failures_retained_in_local_evidence": true, |
| "vendored_loader_source_byte_identical_to_tested_SDK": true, |
| "historical_DEV_benchmark": { |
| "status": "REUSED_HISTORICAL_DEV_OBSERVATIONS_NOT_QUALIFICATION", |
| "new_benchmark_runs": 0, |
| "model_revision": "fd8231d3cdc1efc46f7a6d7d83ca88b336a953591d63ea431407c8b13b990309", |
| "hardware": "NVIDIA GeForce RTX 5080", |
| "kompress_scope_sha256": "8c13f887177adcf31fdfba03c6b5894598bfd6b33b74f086b757ec2eddb66332", |
| "v3_scope_sha256": "711c176b58492358dc0d4c7de4d79f7b1b20112d468ed8a5501d3f34d35ec3b3", |
| "v3_export_manifest_sha256": "19b10e408935e046946125af5f06994e0d034efc53ba5fb0dfdbe0b40f7053e1", |
| "kompress_export_manifest_sha256": "2774d37b3832201e59990182bc264fe15ca6383f3577264d94f9699e7816ddac", |
| "headroom_source_version": "source-1cf496612e781ef8d67ff87ee4f78037492f0bc7", |
| "parameter_counts": { |
| "v3": 149606401, |
| "kompress": 149999875 |
| }, |
| "precision": { |
| "live_parameter_dtype": "float32", |
| "CUDA_autocast": "bfloat16", |
| "cache_enabled": false, |
| "TF32": false, |
| "torch_threads": 1, |
| "GPU": 0, |
| "v3_attention": "admitted-SDPA", |
| "kompress_attention": "native-EAGER" |
| }, |
| "software": { |
| "huggingface_hub": "1.33.0", |
| "numpy": "2.5.3", |
| "peft": "0.21.2", |
| "safetensors": "0.8.0", |
| "tokenizers": "0.23.2", |
| "torch": "2.11.0+cu128", |
| "transformers": "5.19.0" |
| }, |
| "clock": "WARM only: start before fresh source JSON/span grouping, exact o200k count and .25/.5 target derivation; include request serialization/IPC, actual model tokenization/forward, API budget adaptation OR native allocation/recount, native child/relay/guardian wait, delivery and caller output recount. Fixed process/tokenizer/model startup+one case0 warmup disclosed separately, NOT cold cost. No result/source/score cache.", |
| "data": "One exposed DEV source per workload at two budgets; JSON/logs share source population; no independent complete-support truth or representative qualification set.", |
| "runtime": "Historical serialized research runners, not latency certification for the included raw-checkpoint quickstart. Kompress native-EAGER export was limited to batch1..8 and sequence5..512; native API guards on, CCR off; not CPU/INT8.", |
| "eligible_cells": 4, |
| "six_refusals_counted_as_wins": false, |
| "more_than_ten_percent_lower_recorded_times": 3, |
| "earlier_screens": { |
| "8c13f887": "0/4", |
| "a8948ec8": "0/4", |
| "ca18f5cd": "1/4", |
| "711c176b": "3/4" |
| }, |
| "not_contemporaneous_paired_causal_or_statistical": true, |
| "independent_semantic_health_established": false, |
| "quality_p50_p95_cold_or90percent_claim": false, |
| "qualified_superiority": false, |
| "source_receipts": [ |
| { |
| "path": "tmp/select-v3-native-guardian-cost-audit-v1/result.json", |
| "sha256": "3ee330fa203951ff2e9be59e05230d9c033bde253da11c8c80284d5acf147a7c" |
| }, |
| { |
| "path": "tmp/select-v3-native-guardian-cost-v1/result.json", |
| "sha256": "402dec4660ce4c756c76af1af5b09f65700c8f193957a82c605ea60ab8a55e95" |
| }, |
| { |
| "path": "tmp/select-v3-native-guardian-cost-v1/scope.json", |
| "sha256": "711c176b58492358dc0d4c7de4d79f7b1b20112d468ed8a5501d3f34d35ec3b3" |
| }, |
| { |
| "path": "tmp/select-v3-native-guardian-cost-v1/cases.json", |
| "sha256": "abc70207fdf4774c2a9d0f8ad6d9090c3dad5a51ab7896ca827f6081aa53ba37" |
| }, |
| { |
| "path": "tmp/select-v3-kompress-resident-cost-audit-v1/result.json", |
| "sha256": "dd279335902db0e1cf57af20781b95d71b19f0ca84864cbd71dda6411aec45d4" |
| }, |
| { |
| "path": "tmp/select-v3-kompress-resident-cost-v1/result.json", |
| "sha256": "ffbec64a9fb7afc057e5bf97667ff376845c2c9c1f542e14385937c670d47745" |
| }, |
| { |
| "path": "tmp/select-v3-kompress-resident-cost-v1/scope.json", |
| "sha256": "8c13f887177adcf31fdfba03c6b5894598bfd6b33b74f086b757ec2eddb66332" |
| }, |
| { |
| "path": "tmp/select-v3-kompress-resident-cost-v1/cases.json", |
| "sha256": "abc70207fdf4774c2a9d0f8ad6d9090c3dad5a51ab7896ca827f6081aa53ba37" |
| }, |
| { |
| "path": "tmp/select-v3-kompress-budget-review-v1/result.json", |
| "sha256": "5ff4b373715ed3891f6cfdd7905ebee67f1b9d137ac84b5311dc5c4b93aa17fb" |
| } |
| ], |
| "rows": [ |
| { |
| "case_index": 0, |
| "workload": "code", |
| "retention_budget": "0.25", |
| "input_tokens": 4411, |
| "target_tokens": 1102, |
| "v3_output_tokens": 1079, |
| "v3_tokens_removed": 3332, |
| "v3_request_seconds": "0.1589187", |
| "kompress_recorded_seconds": "0.2036338", |
| "kompress_budget_eligible": false, |
| "kompress_recorded_seconds_are_refusal_cost": true, |
| "kompress_returned_tokens": 4411, |
| "kompress_native_attempt_tokens": [ |
| 2032, |
| 1747, |
| 1642, |
| 1607 |
| ], |
| "v3_vs_kompress_time_change_percent": null, |
| "comparison": "Inconclusive", |
| "semantic_health_or_quality_established": false, |
| "v3_tokens_saved_percent": 75.53842666062117 |
| }, |
| { |
| "case_index": 1, |
| "workload": "code", |
| "retention_budget": "0.5", |
| "input_tokens": 4411, |
| "target_tokens": 2205, |
| "v3_output_tokens": 2205, |
| "v3_tokens_removed": 2206, |
| "v3_request_seconds": "0.1460882", |
| "kompress_recorded_seconds": "0.2003043", |
| "kompress_budget_eligible": true, |
| "kompress_recorded_seconds_are_refusal_cost": false, |
| "kompress_returned_tokens": 2149, |
| "kompress_native_attempt_tokens": [ |
| 2469, |
| 2301, |
| 2215, |
| 2149 |
| ], |
| "v3_vs_kompress_time_change_percent": "-27.06686776070209176737593751", |
| "comparison": "27.1% lower time", |
| "semantic_health_or_quality_established": false, |
| "v3_tokens_saved_percent": 50.01133529811834 |
| }, |
| { |
| "case_index": 2, |
| "workload": "json", |
| "retention_budget": "0.25", |
| "input_tokens": 1303, |
| "target_tokens": 325, |
| "v3_output_tokens": 308, |
| "v3_tokens_removed": 995, |
| "v3_request_seconds": "0.0628606", |
| "kompress_recorded_seconds": "0.0804463", |
| "kompress_budget_eligible": false, |
| "kompress_recorded_seconds_are_refusal_cost": true, |
| "kompress_returned_tokens": 1303, |
| "kompress_native_attempt_tokens": [ |
| 984, |
| 984, |
| 984, |
| 984 |
| ], |
| "v3_vs_kompress_time_change_percent": null, |
| "comparison": "Inconclusive", |
| "semantic_health_or_quality_established": false, |
| "v3_tokens_saved_percent": 76.36224098234842 |
| }, |
| { |
| "case_index": 3, |
| "workload": "json", |
| "retention_budget": "0.5", |
| "input_tokens": 1303, |
| "target_tokens": 651, |
| "v3_output_tokens": 651, |
| "v3_tokens_removed": 652, |
| "v3_request_seconds": "0.0658957", |
| "kompress_recorded_seconds": "0.0787157", |
| "kompress_budget_eligible": false, |
| "kompress_recorded_seconds_are_refusal_cost": true, |
| "kompress_returned_tokens": 1303, |
| "kompress_native_attempt_tokens": [ |
| 1020, |
| 984, |
| 984, |
| 984 |
| ], |
| "v3_vs_kompress_time_change_percent": null, |
| "comparison": "Inconclusive", |
| "semantic_health_or_quality_established": false, |
| "v3_tokens_saved_percent": 50.03837298541826 |
| }, |
| { |
| "case_index": 4, |
| "workload": "logs", |
| "retention_budget": "0.25", |
| "input_tokens": 1103, |
| "target_tokens": 275, |
| "v3_output_tokens": 275, |
| "v3_tokens_removed": 828, |
| "v3_request_seconds": "0.0589192", |
| "kompress_recorded_seconds": "0.07833", |
| "kompress_budget_eligible": false, |
| "kompress_recorded_seconds_are_refusal_cost": true, |
| "kompress_returned_tokens": 1103, |
| "kompress_native_attempt_tokens": [ |
| 895, |
| 895, |
| 895, |
| 895 |
| ], |
| "v3_vs_kompress_time_change_percent": null, |
| "comparison": "Inconclusive", |
| "semantic_health_or_quality_established": false, |
| "v3_tokens_saved_percent": 75.06799637352674 |
| }, |
| { |
| "case_index": 5, |
| "workload": "logs", |
| "retention_budget": "0.5", |
| "input_tokens": 1103, |
| "target_tokens": 551, |
| "v3_output_tokens": 503, |
| "v3_tokens_removed": 600, |
| "v3_request_seconds": "0.0579552", |
| "kompress_recorded_seconds": "0.0785184", |
| "kompress_budget_eligible": false, |
| "kompress_recorded_seconds_are_refusal_cost": true, |
| "kompress_returned_tokens": 1103, |
| "kompress_native_attempt_tokens": [ |
| 913, |
| 895, |
| 895, |
| 895 |
| ], |
| "v3_vs_kompress_time_change_percent": null, |
| "comparison": "Inconclusive", |
| "semantic_health_or_quality_established": false, |
| "v3_tokens_saved_percent": 54.39709882139619 |
| }, |
| { |
| "case_index": 6, |
| "workload": "multi-turn", |
| "retention_budget": "0.25", |
| "input_tokens": 587, |
| "target_tokens": 146, |
| "v3_output_tokens": 39, |
| "v3_tokens_removed": 548, |
| "v3_request_seconds": "0.069653", |
| "kompress_recorded_seconds": "0.0906459", |
| "kompress_budget_eligible": true, |
| "kompress_recorded_seconds_are_refusal_cost": false, |
| "kompress_returned_tokens": 144, |
| "kompress_native_attempt_tokens": [ |
| 255, |
| 176, |
| 153, |
| 144 |
| ], |
| "v3_vs_kompress_time_change_percent": "-23.15923831083369462932134824", |
| "comparison": "23.2% lower time", |
| "semantic_health_or_quality_established": false, |
| "v3_tokens_saved_percent": 93.35604770017036 |
| }, |
| { |
| "case_index": 7, |
| "workload": "multi-turn", |
| "retention_budget": "0.5", |
| "input_tokens": 587, |
| "target_tokens": 293, |
| "v3_output_tokens": 235, |
| "v3_tokens_removed": 352, |
| "v3_request_seconds": "0.0693616", |
| "kompress_recorded_seconds": "0.0644106", |
| "kompress_budget_eligible": true, |
| "kompress_recorded_seconds_are_refusal_cost": false, |
| "kompress_returned_tokens": 284, |
| "kompress_native_attempt_tokens": [ |
| 374, |
| 308, |
| 284 |
| ], |
| "v3_vs_kompress_time_change_percent": "7.686623009256240432475400000", |
| "comparison": "7.7% higher time", |
| "semantic_health_or_quality_established": false, |
| "v3_tokens_saved_percent": 59.96592844974446 |
| }, |
| { |
| "case_index": 8, |
| "workload": "retrieval", |
| "retention_budget": "0.25", |
| "input_tokens": 4498, |
| "target_tokens": 1124, |
| "v3_output_tokens": 1115, |
| "v3_tokens_removed": 3383, |
| "v3_request_seconds": "0.1361903", |
| "kompress_recorded_seconds": "0.2695771", |
| "kompress_budget_eligible": false, |
| "kompress_recorded_seconds_are_refusal_cost": true, |
| "kompress_returned_tokens": 4498, |
| "kompress_native_attempt_tokens": [ |
| 1950, |
| 1578, |
| 1428, |
| 1347 |
| ], |
| "v3_vs_kompress_time_change_percent": null, |
| "comparison": "Inconclusive", |
| "semantic_health_or_quality_established": false, |
| "v3_tokens_saved_percent": 75.21120497999111 |
| }, |
| { |
| "case_index": 9, |
| "workload": "retrieval", |
| "retention_budget": "0.5", |
| "input_tokens": 4498, |
| "target_tokens": 2249, |
| "v3_output_tokens": 2217, |
| "v3_tokens_removed": 2281, |
| "v3_request_seconds": "0.139291", |
| "kompress_recorded_seconds": "0.201564", |
| "kompress_budget_eligible": true, |
| "kompress_recorded_seconds_are_refusal_cost": false, |
| "kompress_returned_tokens": 2219, |
| "kompress_native_attempt_tokens": [ |
| 2651, |
| 2361, |
| 2219 |
| ], |
| "v3_vs_kompress_time_change_percent": "-30.89490186739695580560020639", |
| "comparison": "30.9% lower time", |
| "semantic_health_or_quality_established": false, |
| "v3_tokens_saved_percent": 50.711427301022674 |
| } |
| ] |
| }, |
| "Ollama_DEV_answer_quality": { |
| "status": "INCOMPLETE_TERMINAL_NO_RETRY", |
| "report": "QUALITY-EVAL.json", |
| "report_sha256": "b5a8946e3edc0a7c91ef8d1985d5e97bcab10972eb06ec37186b6bd1be187854", |
| "think": false, |
| "planned_requests": 12, |
| "answered_requests": 6, |
| "failed_requests": 1, |
| "unexecuted_requests": 5, |
| "overall_suite_precision_recall_f1": null, |
| "complete_code_subgroup": { |
| "questions": 3, |
| "source_files": 1, |
| "raw": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "em": 1.0 |
| }, |
| "compressed": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "em": 1.0 |
| } |
| }, |
| "not_a_Headroom_comparable_score": true, |
| "qualification": false |
| }, |
| "Kompress_v2_comparison": { |
| "report": "COMPARISON.json", |
| "report_sha256": "2e224621f836a76f2f31c450dd0233c35322fe774bd718ee171d157c30300902", |
| "status": "HISTORICAL_DESCRIPTIVE_COMPARISON_AND_INCOMPLETE_NEW_PILOT", |
| "published_token_metrics": { |
| "threshold": 0.5, |
| "token_precision": 0.94, |
| "token_recall": 0.901, |
| "token_f1": 0.92, |
| "must_keep_recall": 0.989, |
| "keep_rate": 0.866, |
| "test_rows": 12697, |
| "scored_subwords": 16827490, |
| "dataset_name": "dataset_v2_v2.1-f00", |
| "hardware": "H100", |
| "compute": "PyTorch BF16", |
| "source": "https://huggingface.co/chopratejas/kompress-v2-base/blob/e00cc5e60ff73b01674ef8fa6da6c5acf6d54181/README.md", |
| "third_party_reported_not_reproduced": true |
| }, |
| "shared_pilot_status": "PARTIAL_V3_COMPLETE_COMPARATOR_SETUP_FAILED_NO_QA", |
| "general_superiority_established": false, |
| "shared_pilot_V3_valid_compression_cases": 5, |
| "shared_pilot_requested_literal_substrings_present": "12/12 across5 fixed cases; substring-only,not answer accuracy", |
| "shared_pilot_Kompress_setup_failure": "AttributeError on nonexistent KompressConfig.batch_size; our stale harness; no comparator inference", |
| "shared_pilot_answerer_requests": 0 |
| }, |
| "fresh_multi_turn_repeat": { |
| "status": "FIXED_FIVE_REPEAT_BENCHMARK_COMPLETE_DESCRIPTIVE_ONLY", |
| "scope_sha256": "5da79d5c782971553e6988849dc04c19b129b6a24e3004e0adac7f4c467be45e", |
| "planned_timed_samples": 10, |
| "measured_timed_samples": 10, |
| "sample_median_seconds": { |
| "kompress": 0.0777508, |
| "v3": 0.0665719 |
| }, |
| "v3_time_change_percent": -14.377858491488183, |
| "all_scheduled_samples_included": true, |
| "GPU_busy_gate_waived_by_user": true, |
| "controlled_idle": false, |
| "quality_or_statistical_superiority_established": false, |
| "historical_row_retained": true |
| } |
| } |
|
|
