{ "artifact": "elephant-1-nvfp4-mlp-public-benchmark-v1", "status": "experimental_not_qualified", "source": { "repo_id": "baselabs/elephant-1", "revision": "b3f8a93462fb1051a9de031e94a7ff380e254072", "model_safetensors_sha256": "682f02348dfe4617dd21bba2915c12e805de8cf16327a61e895a831ea681d0ba", "upstream_declared_license": "mit", "upstream_author": "Base Labs at Baseten", "upstream_base_model": "Qwen/Qwen3.5-9B" }, "quantization": { "scheme": "NVFP4 W4A4", "format": "nvfp4-pack-quantized", "method": "stock llmcompressor oneshot basic pipeline; round-to-nearest weights; static-minmax activation global-scale calibration", "target_modules": 96, "targets": "model.layers.N.mlp.{gate,up,down}_proj across 32 text layers", "group_size": 16, "block_scale_precision": "FP8", "protected_tensors_byte_identical_to_source": 664, "protected_text_tensors": 331, "protected_unused_vision_tensors": 333, "protected_offset_norms_restored": 81, "source_tensor_file_bytes": 18819722392, "export_tensor_file_bytes": 11873990288, "execution_text_only": true, "composite_configuration_and_unused_vision_tensors_retained": true, "head_dtype_on_disk": "bfloat16", "head_dtype_for_scoring": "float32" }, "calibration": { "dataset_version": "public-benchmark-v1", "records": 256, "questions": 768, "forwarded_tokens": 2477543, "maximum_branch_tokens": 16017, "labels_used_for_scale_selection": false, "truncation": false, "longbench_used": false }, "numerical_evaluation": { "mode": "independent_full_batch1_no_cache", "total_rendered_token_limit": 16384, "temperature": 1.0, "tf32_matmul": false, "weight_reload": "decompressed BF16 tensors containing NVFP4 weight values", "activation_quantization": "enabled QDQ on all 96 targeted MLP projections", "native_fp4_tested": false, "multimodal_tested": false, "w4a16_scored": false }, "tested_runtime": { "python": "3.12.3", "torch": "2.13.0+cu132", "transformers": "5.19.0", "llmcompressor": "0.15.0", "compressed-tensors": "0.20.1", "flash-linear-attention": "0.5.2", "fla-core": "0.5.2", "causal-conv1d": "1.7.0", "accelerate": "1.15.0", "onejev_commit": "81ce62f1597c91e46767d4d02d6ac2e18534fe94", "onejev_author_serve_patch_sha256": "f234c125f1d38d82104b9e1f24e75bde51e4401928acfba2fbf5c59ef2fb46b8", "portable_runtime_qualification": false }, "heldout_benchmark": { "labeled_tasks": 278, "unlabeled_support_questions": 556, "bf16_correct": 221, "nvfp4_correct": 221, "bf16_to_nvfp4_correctness_gains": 4, "bf16_to_nvfp4_correctness_losses": 4, "accuracy_delta": 0.0, "accuracy_delta_ci95": [-0.02158273381294964, 0.017985611510791366], "predeclared_accuracy_delta_lower_bound_min": -0.02, "accuracy_noninferiority_pass": false, "confident_labeled_agree": 229, "confident_labeled_count": 230, "confident_bf16_margin_min": 0.5, "confident_modal_agreement_min": 0.98, "confident_modal_agreement_pass": true, "max_probability_abs_difference": 0.4119426021649549, "benchmark_pass": false, "model_qualified": false }, "development_repeatability": { "records": 128, "questions": 384, "independent_labeled_tasks": 72, "fresh_process_runs_per_model": 3, "bf16_errors_per_run": [14, 14, 14], "nvfp4_errors_per_run": [15, 15, 15], "bf16_cross_process_max_probability_abs_difference": 0.008140687317193085, "nvfp4_cross_process_max_probability_abs_difference": 0.058135252214409905, "nvfp4_vs_historical_max_probability_abs_difference": 0.13381917866515347, "bf16_questions_with_top_answer_instability": 0, "nvfp4_questions_with_top_answer_instability": 1, "labeled_correctness_changes_across_same_model_repeats": 0, "within_process_short_long_sentinel_logits_exact": true, "runtime_variation_cause_identified": false, "repeated_observations_are_new_independent_labels": false } }