{ "status": "LOCAL_FUNCTIONAL_RUNTIME_SMOKE_PASS_NOT_BENCHMARK_OR_SEMANTIC_QUALIFICATION", "hardware": "NVIDIA GeForce RTX 5080", "precision": "FP32 parameters,CUDA BF16 autocast,cacheoff/TF32off,one Torch thread", "Select_v3": { "local_packaged_weight_load": "PASS", "finite_aligned_group_scores": "PASS", "groups": 2, "encoded_tokens": 498, "windows": 1, "offline_guard_violations": 0 }, "controls": { "kompress-native-bf16": { "status": "NATIVE_EAGER_BATCH32_RUNTIME_FUNCTIONAL_HEALTH_PASS_NOT_SEMANTIC_QUALIFICATION", "original_worker_and_guardian_waited": true }, "v1-bf16": { "status": "FUNCTIONAL_FINITE_ALIGNED_SCORE_PASS", "original_worker_and_guardian_waited": true }, "v2-bf16": { "status": "FUNCTIONAL_FINITE_ALIGNED_SCORE_PASS", "original_worker_and_guardian_waited": true } }, "Kompress_native_batch32_shape_probe": "PASS;nativeEAGER not batch8export", "Kompress_budget_probes": [ { "ratio": 0.25, "input_tokens": 336, "target_tokens": 84, "output_tokens": 336, "valid_budget": false, "no_op": true, "refusal_not_win": true }, { "ratio": 0.5, "input_tokens": 336, "target_tokens": 168, "output_tokens": 336, "valid_budget": false, "no_op": true, "refusal_not_win": true } ], "new_training_compilation_capture_or_model_downloads": 0, "comparative_latency_quality_superiority_or90percent_claim": false, "independent_semantic_health_not_certified": true, "fresh_machine_install_not_certified": true, "original_failures_retained_in_local_evidence": true, "vendored_loader_source_byte_identical_to_tested_SDK": true, "historical_DEV_benchmark": { "status": "REUSED_HISTORICAL_DEV_OBSERVATIONS_NOT_QUALIFICATION", "new_benchmark_runs": 0, "model_revision": "fd8231d3cdc1efc46f7a6d7d83ca88b336a953591d63ea431407c8b13b990309", "hardware": "NVIDIA GeForce RTX 5080", "kompress_scope_sha256": "8c13f887177adcf31fdfba03c6b5894598bfd6b33b74f086b757ec2eddb66332", "v3_scope_sha256": "711c176b58492358dc0d4c7de4d79f7b1b20112d468ed8a5501d3f34d35ec3b3", "v3_export_manifest_sha256": "19b10e408935e046946125af5f06994e0d034efc53ba5fb0dfdbe0b40f7053e1", "kompress_export_manifest_sha256": "2774d37b3832201e59990182bc264fe15ca6383f3577264d94f9699e7816ddac", "headroom_source_version": "source-1cf496612e781ef8d67ff87ee4f78037492f0bc7", "parameter_counts": { "v3": 149606401, "kompress": 149999875 }, "precision": { "live_parameter_dtype": "float32", "CUDA_autocast": "bfloat16", "cache_enabled": false, "TF32": false, "torch_threads": 1, "GPU": 0, "v3_attention": "admitted-SDPA", "kompress_attention": "native-EAGER" }, "software": { "huggingface_hub": "1.33.0", "numpy": "2.5.3", "peft": "0.21.2", "safetensors": "0.8.0", "tokenizers": "0.23.2", "torch": "2.11.0+cu128", "transformers": "5.19.0" }, "clock": "WARM only: start before fresh source JSON/span grouping, exact o200k count and .25/.5 target derivation; include request serialization/IPC, actual model tokenization/forward, API budget adaptation OR native allocation/recount, native child/relay/guardian wait, delivery and caller output recount. Fixed process/tokenizer/model startup+one case0 warmup disclosed separately, NOT cold cost. No result/source/score cache.", "data": "One exposed DEV source per workload at two budgets; JSON/logs share source population; no independent complete-support truth or representative qualification set.", "runtime": "Historical serialized research runners, not latency certification for the included raw-checkpoint quickstart. Kompress native-EAGER export was limited to batch1..8 and sequence5..512; native API guards on, CCR off; not CPU/INT8.", "eligible_cells": 4, "six_refusals_counted_as_wins": false, "more_than_ten_percent_lower_recorded_times": 3, "earlier_screens": { "8c13f887": "0/4", "a8948ec8": "0/4", "ca18f5cd": "1/4", "711c176b": "3/4" }, "not_contemporaneous_paired_causal_or_statistical": true, "independent_semantic_health_established": false, "quality_p50_p95_cold_or90percent_claim": false, "qualified_superiority": false, "source_receipts": [ { "path": "tmp/select-v3-native-guardian-cost-audit-v1/result.json", "sha256": "3ee330fa203951ff2e9be59e05230d9c033bde253da11c8c80284d5acf147a7c" }, { "path": "tmp/select-v3-native-guardian-cost-v1/result.json", "sha256": "402dec4660ce4c756c76af1af5b09f65700c8f193957a82c605ea60ab8a55e95" }, { "path": "tmp/select-v3-native-guardian-cost-v1/scope.json", "sha256": "711c176b58492358dc0d4c7de4d79f7b1b20112d468ed8a5501d3f34d35ec3b3" }, { "path": "tmp/select-v3-native-guardian-cost-v1/cases.json", "sha256": "abc70207fdf4774c2a9d0f8ad6d9090c3dad5a51ab7896ca827f6081aa53ba37" }, { "path": "tmp/select-v3-kompress-resident-cost-audit-v1/result.json", "sha256": "dd279335902db0e1cf57af20781b95d71b19f0ca84864cbd71dda6411aec45d4" }, { "path": "tmp/select-v3-kompress-resident-cost-v1/result.json", "sha256": "ffbec64a9fb7afc057e5bf97667ff376845c2c9c1f542e14385937c670d47745" }, { "path": "tmp/select-v3-kompress-resident-cost-v1/scope.json", "sha256": "8c13f887177adcf31fdfba03c6b5894598bfd6b33b74f086b757ec2eddb66332" }, { "path": "tmp/select-v3-kompress-resident-cost-v1/cases.json", "sha256": "abc70207fdf4774c2a9d0f8ad6d9090c3dad5a51ab7896ca827f6081aa53ba37" }, { "path": "tmp/select-v3-kompress-budget-review-v1/result.json", "sha256": "5ff4b373715ed3891f6cfdd7905ebee67f1b9d137ac84b5311dc5c4b93aa17fb" } ], "rows": [ { "case_index": 0, "workload": "code", "retention_budget": "0.25", "input_tokens": 4411, "target_tokens": 1102, "v3_output_tokens": 1079, "v3_tokens_removed": 3332, "v3_request_seconds": "0.1589187", "kompress_recorded_seconds": "0.2036338", "kompress_budget_eligible": false, "kompress_recorded_seconds_are_refusal_cost": true, "kompress_returned_tokens": 4411, "kompress_native_attempt_tokens": [ 2032, 1747, 1642, 1607 ], "v3_vs_kompress_time_change_percent": null, "comparison": "Inconclusive", "semantic_health_or_quality_established": false, "v3_tokens_saved_percent": 75.53842666062117 }, { "case_index": 1, "workload": "code", "retention_budget": "0.5", "input_tokens": 4411, "target_tokens": 2205, "v3_output_tokens": 2205, "v3_tokens_removed": 2206, "v3_request_seconds": "0.1460882", "kompress_recorded_seconds": "0.2003043", "kompress_budget_eligible": true, "kompress_recorded_seconds_are_refusal_cost": false, "kompress_returned_tokens": 2149, "kompress_native_attempt_tokens": [ 2469, 2301, 2215, 2149 ], "v3_vs_kompress_time_change_percent": "-27.06686776070209176737593751", "comparison": "27.1% lower time", "semantic_health_or_quality_established": false, "v3_tokens_saved_percent": 50.01133529811834 }, { "case_index": 2, "workload": "json", "retention_budget": "0.25", "input_tokens": 1303, "target_tokens": 325, "v3_output_tokens": 308, "v3_tokens_removed": 995, "v3_request_seconds": "0.0628606", "kompress_recorded_seconds": "0.0804463", "kompress_budget_eligible": false, "kompress_recorded_seconds_are_refusal_cost": true, "kompress_returned_tokens": 1303, "kompress_native_attempt_tokens": [ 984, 984, 984, 984 ], "v3_vs_kompress_time_change_percent": null, "comparison": "Inconclusive", "semantic_health_or_quality_established": false, "v3_tokens_saved_percent": 76.36224098234842 }, { "case_index": 3, "workload": "json", "retention_budget": "0.5", "input_tokens": 1303, "target_tokens": 651, "v3_output_tokens": 651, "v3_tokens_removed": 652, "v3_request_seconds": "0.0658957", "kompress_recorded_seconds": "0.0787157", "kompress_budget_eligible": false, "kompress_recorded_seconds_are_refusal_cost": true, "kompress_returned_tokens": 1303, "kompress_native_attempt_tokens": [ 1020, 984, 984, 984 ], "v3_vs_kompress_time_change_percent": null, "comparison": "Inconclusive", "semantic_health_or_quality_established": false, "v3_tokens_saved_percent": 50.03837298541826 }, { "case_index": 4, "workload": "logs", "retention_budget": "0.25", "input_tokens": 1103, "target_tokens": 275, "v3_output_tokens": 275, "v3_tokens_removed": 828, "v3_request_seconds": "0.0589192", "kompress_recorded_seconds": "0.07833", "kompress_budget_eligible": false, "kompress_recorded_seconds_are_refusal_cost": true, "kompress_returned_tokens": 1103, "kompress_native_attempt_tokens": [ 895, 895, 895, 895 ], "v3_vs_kompress_time_change_percent": null, "comparison": "Inconclusive", "semantic_health_or_quality_established": false, "v3_tokens_saved_percent": 75.06799637352674 }, { "case_index": 5, "workload": "logs", "retention_budget": "0.5", "input_tokens": 1103, "target_tokens": 551, "v3_output_tokens": 503, "v3_tokens_removed": 600, "v3_request_seconds": "0.0579552", "kompress_recorded_seconds": "0.0785184", "kompress_budget_eligible": false, "kompress_recorded_seconds_are_refusal_cost": true, "kompress_returned_tokens": 1103, "kompress_native_attempt_tokens": [ 913, 895, 895, 895 ], "v3_vs_kompress_time_change_percent": null, "comparison": "Inconclusive", "semantic_health_or_quality_established": false, "v3_tokens_saved_percent": 54.39709882139619 }, { "case_index": 6, "workload": "multi-turn", "retention_budget": "0.25", "input_tokens": 587, "target_tokens": 146, "v3_output_tokens": 39, "v3_tokens_removed": 548, "v3_request_seconds": "0.069653", "kompress_recorded_seconds": "0.0906459", "kompress_budget_eligible": true, "kompress_recorded_seconds_are_refusal_cost": false, "kompress_returned_tokens": 144, "kompress_native_attempt_tokens": [ 255, 176, 153, 144 ], "v3_vs_kompress_time_change_percent": "-23.15923831083369462932134824", "comparison": "23.2% lower time", "semantic_health_or_quality_established": false, "v3_tokens_saved_percent": 93.35604770017036 }, { "case_index": 7, "workload": "multi-turn", "retention_budget": "0.5", "input_tokens": 587, "target_tokens": 293, "v3_output_tokens": 235, "v3_tokens_removed": 352, "v3_request_seconds": "0.0693616", "kompress_recorded_seconds": "0.0644106", "kompress_budget_eligible": true, "kompress_recorded_seconds_are_refusal_cost": false, "kompress_returned_tokens": 284, "kompress_native_attempt_tokens": [ 374, 308, 284 ], "v3_vs_kompress_time_change_percent": "7.686623009256240432475400000", "comparison": "7.7% higher time", "semantic_health_or_quality_established": false, "v3_tokens_saved_percent": 59.96592844974446 }, { "case_index": 8, "workload": "retrieval", "retention_budget": "0.25", "input_tokens": 4498, "target_tokens": 1124, "v3_output_tokens": 1115, "v3_tokens_removed": 3383, "v3_request_seconds": "0.1361903", "kompress_recorded_seconds": "0.2695771", "kompress_budget_eligible": false, "kompress_recorded_seconds_are_refusal_cost": true, "kompress_returned_tokens": 4498, "kompress_native_attempt_tokens": [ 1950, 1578, 1428, 1347 ], "v3_vs_kompress_time_change_percent": null, "comparison": "Inconclusive", "semantic_health_or_quality_established": false, "v3_tokens_saved_percent": 75.21120497999111 }, { "case_index": 9, "workload": "retrieval", "retention_budget": "0.5", "input_tokens": 4498, "target_tokens": 2249, "v3_output_tokens": 2217, "v3_tokens_removed": 2281, "v3_request_seconds": "0.139291", "kompress_recorded_seconds": "0.201564", "kompress_budget_eligible": true, "kompress_recorded_seconds_are_refusal_cost": false, "kompress_returned_tokens": 2219, "kompress_native_attempt_tokens": [ 2651, 2361, 2219 ], "v3_vs_kompress_time_change_percent": "-30.89490186739695580560020639", "comparison": "30.9% lower time", "semantic_health_or_quality_established": false, "v3_tokens_saved_percent": 50.711427301022674 } ] }, "Ollama_DEV_answer_quality": { "status": "INCOMPLETE_TERMINAL_NO_RETRY", "report": "QUALITY-EVAL.json", "report_sha256": "b5a8946e3edc0a7c91ef8d1985d5e97bcab10972eb06ec37186b6bd1be187854", "think": false, "planned_requests": 12, "answered_requests": 6, "failed_requests": 1, "unexecuted_requests": 5, "overall_suite_precision_recall_f1": null, "complete_code_subgroup": { "questions": 3, "source_files": 1, "raw": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "em": 1.0 }, "compressed": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "em": 1.0 } }, "not_a_Headroom_comparable_score": true, "qualification": false }, "Kompress_v2_comparison": { "report": "COMPARISON.json", "report_sha256": "2e224621f836a76f2f31c450dd0233c35322fe774bd718ee171d157c30300902", "status": "HISTORICAL_DESCRIPTIVE_COMPARISON_AND_INCOMPLETE_NEW_PILOT", "published_token_metrics": { "threshold": 0.5, "token_precision": 0.94, "token_recall": 0.901, "token_f1": 0.92, "must_keep_recall": 0.989, "keep_rate": 0.866, "test_rows": 12697, "scored_subwords": 16827490, "dataset_name": "dataset_v2_v2.1-f00", "hardware": "H100", "compute": "PyTorch BF16", "source": "https://huggingface.co/chopratejas/kompress-v2-base/blob/e00cc5e60ff73b01674ef8fa6da6c5acf6d54181/README.md", "third_party_reported_not_reproduced": true }, "shared_pilot_status": "PARTIAL_V3_COMPLETE_COMPARATOR_SETUP_FAILED_NO_QA", "general_superiority_established": false, "shared_pilot_V3_valid_compression_cases": 5, "shared_pilot_requested_literal_substrings_present": "12/12 across5 fixed cases; substring-only,not answer accuracy", "shared_pilot_Kompress_setup_failure": "AttributeError on nonexistent KompressConfig.batch_size; our stale harness; no comparator inference", "shared_pilot_answerer_requests": 0 }, "fresh_multi_turn_repeat": { "status": "FIXED_FIVE_REPEAT_BENCHMARK_COMPLETE_DESCRIPTIVE_ONLY", "scope_sha256": "5da79d5c782971553e6988849dc04c19b129b6a24e3004e0adac7f4c467be45e", "planned_timed_samples": 10, "measured_timed_samples": 10, "sample_median_seconds": { "kompress": 0.0777508, "v3": 0.0665719 }, "v3_time_change_percent": -14.377858491488183, "all_scheduled_samples_included": true, "GPU_busy_gate_waived_by_user": true, "controlled_idle": false, "quality_or_statistical_superiority_established": false, "historical_row_retained": true } }