Instructions to use bi0/NVIDIA_Nemotron-3.5-Rerank-8B-NVFP4 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- TensorRT
How to use bi0/NVIDIA_Nemotron-3.5-Rerank-8B-NVFP4 with TensorRT:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download quant_config.json from bi0/NVIDIA_Nemotron-3.5-Rerank-8B-NVFP4: direct link, hf CLI and curl.
- Browser
- Download file 10.8 kB
-
https://huggingface.co/bi0/NVIDIA_Nemotron-3.5-Rerank-8B-NVFP4/resolve/main/quant_config.json
- Command line
-
hf download hf://bi0/NVIDIA_Nemotron-3.5-Rerank-8B-NVFP4/quant_config.json
-
curl -L -o quant_config.json https://huggingface.co/bi0/NVIDIA_Nemotron-3.5-Rerank-8B-NVFP4/resolve/main/quant_config.json
10.8 kB
| { | |
| "format": "nvfp4_e2m1_block16", | |
| "block_size": 16, | |
| "target_architecture": "Blackwell (SM 12.0 / SM 12.1)", | |
| "base_model": "nvidia/Nemotron-3.5-Rerank-8B-BF16", | |
| "quantized_layers": [ | |
| { | |
| "name": "language_model.model.embed_tokens.weight", | |
| "original_shape": [ | |
| 131072, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 131072, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 131072, | |
| 256 | |
| ], | |
| "original_bytes": 1073741824, | |
| "quantized_bytes": 335544320, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.30.self_attn.q_proj.weight", | |
| "original_shape": [ | |
| 4096, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 4096, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "original_bytes": 33554432, | |
| "quantized_bytes": 10485760, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.30.self_attn.k_proj.weight", | |
| "original_shape": [ | |
| 1024, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 1024, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 1024, | |
| 256 | |
| ], | |
| "original_bytes": 8388608, | |
| "quantized_bytes": 2621440, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.30.self_attn.v_proj.weight", | |
| "original_shape": [ | |
| 1024, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 1024, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 1024, | |
| 256 | |
| ], | |
| "original_bytes": 8388608, | |
| "quantized_bytes": 2621440, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.30.self_attn.o_proj.weight", | |
| "original_shape": [ | |
| 4096, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 4096, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "original_bytes": 33554432, | |
| "quantized_bytes": 10485760, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.30.mlp.gate_proj.weight", | |
| "original_shape": [ | |
| 14336, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 14336, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 14336, | |
| 256 | |
| ], | |
| "original_bytes": 117440512, | |
| "quantized_bytes": 36700160, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.30.mlp.up_proj.weight", | |
| "original_shape": [ | |
| 14336, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 14336, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 14336, | |
| 256 | |
| ], | |
| "original_bytes": 117440512, | |
| "quantized_bytes": 36700160, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.30.mlp.down_proj.weight", | |
| "original_shape": [ | |
| 4096, | |
| 14336 | |
| ], | |
| "packed_shape": [ | |
| 4096, | |
| 7168 | |
| ], | |
| "scales_shape": [ | |
| 4096, | |
| 896 | |
| ], | |
| "original_bytes": 117440512, | |
| "quantized_bytes": 36700160, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.31.self_attn.q_proj.weight", | |
| "original_shape": [ | |
| 4096, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 4096, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "original_bytes": 33554432, | |
| "quantized_bytes": 10485760, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.31.self_attn.k_proj.weight", | |
| "original_shape": [ | |
| 1024, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 1024, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 1024, | |
| 256 | |
| ], | |
| "original_bytes": 8388608, | |
| "quantized_bytes": 2621440, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.31.self_attn.v_proj.weight", | |
| "original_shape": [ | |
| 1024, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 1024, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 1024, | |
| 256 | |
| ], | |
| "original_bytes": 8388608, | |
| "quantized_bytes": 2621440, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.31.self_attn.o_proj.weight", | |
| "original_shape": [ | |
| 4096, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 4096, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "original_bytes": 33554432, | |
| "quantized_bytes": 10485760, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.31.mlp.gate_proj.weight", | |
| "original_shape": [ | |
| 14336, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 14336, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 14336, | |
| 256 | |
| ], | |
| "original_bytes": 117440512, | |
| "quantized_bytes": 36700160, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.31.mlp.up_proj.weight", | |
| "original_shape": [ | |
| 14336, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 14336, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 14336, | |
| 256 | |
| ], | |
| "original_bytes": 117440512, | |
| "quantized_bytes": 36700160, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.31.mlp.down_proj.weight", | |
| "original_shape": [ | |
| 4096, | |
| 14336 | |
| ], | |
| "packed_shape": [ | |
| 4096, | |
| 7168 | |
| ], | |
| "scales_shape": [ | |
| 4096, | |
| 896 | |
| ], | |
| "original_bytes": 117440512, | |
| "quantized_bytes": 36700160, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.32.self_attn.q_proj.weight", | |
| "original_shape": [ | |
| 4096, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 4096, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "original_bytes": 33554432, | |
| "quantized_bytes": 10485760, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.32.self_attn.k_proj.weight", | |
| "original_shape": [ | |
| 1024, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 1024, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 1024, | |
| 256 | |
| ], | |
| "original_bytes": 8388608, | |
| "quantized_bytes": 2621440, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.32.self_attn.v_proj.weight", | |
| "original_shape": [ | |
| 1024, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 1024, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 1024, | |
| 256 | |
| ], | |
| "original_bytes": 8388608, | |
| "quantized_bytes": 2621440, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.32.self_attn.o_proj.weight", | |
| "original_shape": [ | |
| 4096, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 4096, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "original_bytes": 33554432, | |
| "quantized_bytes": 10485760, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.32.mlp.gate_proj.weight", | |
| "original_shape": [ | |
| 14336, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 14336, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 14336, | |
| 256 | |
| ], | |
| "original_bytes": 117440512, | |
| "quantized_bytes": 36700160, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.32.mlp.up_proj.weight", | |
| "original_shape": [ | |
| 14336, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 14336, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 14336, | |
| 256 | |
| ], | |
| "original_bytes": 117440512, | |
| "quantized_bytes": 36700160, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.32.mlp.down_proj.weight", | |
| "original_shape": [ | |
| 4096, | |
| 14336 | |
| ], | |
| "packed_shape": [ | |
| 4096, | |
| 7168 | |
| ], | |
| "scales_shape": [ | |
| 4096, | |
| 896 | |
| ], | |
| "original_bytes": 117440512, | |
| "quantized_bytes": 36700160, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.33.self_attn.q_proj.weight", | |
| "original_shape": [ | |
| 4096, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 4096, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "original_bytes": 33554432, | |
| "quantized_bytes": 10485760, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.33.self_attn.k_proj.weight", | |
| "original_shape": [ | |
| 1024, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 1024, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 1024, | |
| 256 | |
| ], | |
| "original_bytes": 8388608, | |
| "quantized_bytes": 2621440, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.33.self_attn.v_proj.weight", | |
| "original_shape": [ | |
| 1024, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 1024, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 1024, | |
| 256 | |
| ], | |
| "original_bytes": 8388608, | |
| "quantized_bytes": 2621440, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.33.self_attn.o_proj.weight", | |
| "original_shape": [ | |
| 4096, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 4096, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 4096, | |
| 256 | |
| ], | |
| "original_bytes": 33554432, | |
| "quantized_bytes": 10485760, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.33.mlp.gate_proj.weight", | |
| "original_shape": [ | |
| 14336, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 14336, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 14336, | |
| 256 | |
| ], | |
| "original_bytes": 117440512, | |
| "quantized_bytes": 36700160, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.33.mlp.up_proj.weight", | |
| "original_shape": [ | |
| 14336, | |
| 4096 | |
| ], | |
| "packed_shape": [ | |
| 14336, | |
| 2048 | |
| ], | |
| "scales_shape": [ | |
| 14336, | |
| 256 | |
| ], | |
| "original_bytes": 117440512, | |
| "quantized_bytes": 36700160, | |
| "compression_ratio": 3.2 | |
| }, | |
| { | |
| "name": "language_model.model.layers.33.mlp.down_proj.weight", | |
| "original_shape": [ | |
| 4096, | |
| 14336 | |
| ], | |
| "packed_shape": [ | |
| 4096, | |
| 7168 | |
| ], | |
| "scales_shape": [ | |
| 4096, | |
| 896 | |
| ], | |
| "original_bytes": 117440512, | |
| "quantized_bytes": 36700160, | |
| "compression_ratio": 3.2 | |
| } | |
| ] | |
| } |