elephant-1-NVFP4-MLP / provenance.json
Code4me2's picture
Upload experimental 96-MLP NVFP4 W4A4 checkpoint and evaluation limitations
af6d61b verified
Raw History Blame Contribute Delete
4.08 kB
{
"artifact": "elephant-1-nvfp4-mlp-public-benchmark-v1",
"status": "experimental_not_qualified",
"source": {
"repo_id": "baselabs/elephant-1",
"revision": "b3f8a93462fb1051a9de031e94a7ff380e254072",
"model_safetensors_sha256": "682f02348dfe4617dd21bba2915c12e805de8cf16327a61e895a831ea681d0ba",
"upstream_declared_license": "mit",
"upstream_author": "Base Labs at Baseten",
"upstream_base_model": "Qwen/Qwen3.5-9B"
},
"quantization": {
"scheme": "NVFP4 W4A4",
"format": "nvfp4-pack-quantized",
"method": "stock llmcompressor oneshot basic pipeline; round-to-nearest weights; static-minmax activation global-scale calibration",
"target_modules": 96,
"targets": "model.layers.N.mlp.{gate,up,down}_proj across 32 text layers",
"group_size": 16,
"block_scale_precision": "FP8",
"protected_tensors_byte_identical_to_source": 664,
"protected_text_tensors": 331,
"protected_unused_vision_tensors": 333,
"protected_offset_norms_restored": 81,
"source_tensor_file_bytes": 18819722392,
"export_tensor_file_bytes": 11873990288,
"execution_text_only": true,
"composite_configuration_and_unused_vision_tensors_retained": true,
"head_dtype_on_disk": "bfloat16",
"head_dtype_for_scoring": "float32"
},
"calibration": {
"dataset_version": "public-benchmark-v1",
"records": 256,
"questions": 768,
"forwarded_tokens": 2477543,
"maximum_branch_tokens": 16017,
"labels_used_for_scale_selection": false,
"truncation": false,
"longbench_used": false
},
"numerical_evaluation": {
"mode": "independent_full_batch1_no_cache",
"total_rendered_token_limit": 16384,
"temperature": 1.0,
"tf32_matmul": false,
"weight_reload": "decompressed BF16 tensors containing NVFP4 weight values",
"activation_quantization": "enabled QDQ on all 96 targeted MLP projections",
"native_fp4_tested": false,
"multimodal_tested": false,
"w4a16_scored": false
},
"tested_runtime": {
"python": "3.12.3",
"torch": "2.13.0+cu132",
"transformers": "5.19.0",
"llmcompressor": "0.15.0",
"compressed-tensors": "0.20.1",
"flash-linear-attention": "0.5.2",
"fla-core": "0.5.2",
"causal-conv1d": "1.7.0",
"accelerate": "1.15.0",
"onejev_commit": "81ce62f1597c91e46767d4d02d6ac2e18534fe94",
"onejev_author_serve_patch_sha256": "f234c125f1d38d82104b9e1f24e75bde51e4401928acfba2fbf5c59ef2fb46b8",
"portable_runtime_qualification": false
},
"heldout_benchmark": {
"labeled_tasks": 278,
"unlabeled_support_questions": 556,
"bf16_correct": 221,
"nvfp4_correct": 221,
"bf16_to_nvfp4_correctness_gains": 4,
"bf16_to_nvfp4_correctness_losses": 4,
"accuracy_delta": 0.0,
"accuracy_delta_ci95": [-0.02158273381294964, 0.017985611510791366],
"predeclared_accuracy_delta_lower_bound_min": -0.02,
"accuracy_noninferiority_pass": false,
"confident_labeled_agree": 229,
"confident_labeled_count": 230,
"confident_bf16_margin_min": 0.5,
"confident_modal_agreement_min": 0.98,
"confident_modal_agreement_pass": true,
"max_probability_abs_difference": 0.4119426021649549,
"benchmark_pass": false,
"model_qualified": false
},
"development_repeatability": {
"records": 128,
"questions": 384,
"independent_labeled_tasks": 72,
"fresh_process_runs_per_model": 3,
"bf16_errors_per_run": [14, 14, 14],
"nvfp4_errors_per_run": [15, 15, 15],
"bf16_cross_process_max_probability_abs_difference": 0.008140687317193085,
"nvfp4_cross_process_max_probability_abs_difference": 0.058135252214409905,
"nvfp4_vs_historical_max_probability_abs_difference": 0.13381917866515347,
"bf16_questions_with_top_answer_instability": 0,
"nvfp4_questions_with_top_answer_instability": 1,
"labeled_correctness_changes_across_same_model_repeats": 0,
"within_process_short_long_sentinel_logits_exact": true,
"runtime_variation_cause_identified": false,
"repeated_observations_are_new_independent_labels": false
}
}