File size: 1,943 Bytes
5576acd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | {
"fp32": {
"precision": "fp32",
"embedding_compute_dtype": "float32",
"classifier_compute_dtype": "float32",
"examples": 1000,
"load_seconds": 1.5235057300888002,
"inference_seconds": 2.8799891732633114,
"examples_per_second": 347.2235275339252,
"peak_cuda_memory_mb": 4289.07666015625,
"embedding_norm_mean": 1.0,
"ndcg_at_5": 0.8548336029052734,
"precision_at_5": 0.5113999843597412,
"recall_at_5": 0.82269287109375,
"top1_hit": 0.9129999876022339,
"package_size_bytes": 1277686137
},
"fp16_storage_bf16_compute": {
"precision": "fp16_storage_bf16_compute",
"embedding_compute_dtype": "bfloat16",
"classifier_compute_dtype": "float16",
"examples": 1000,
"load_seconds": 1.3328879470936954,
"inference_seconds": 1.2113684518262744,
"examples_per_second": 825.5126658552048,
"peak_cuda_memory_mb": 2176.63623046875,
"embedding_norm_mean": 1.0007888078689575,
"ndcg_at_5": 0.8558993339538574,
"precision_at_5": 0.5113999843597412,
"recall_at_5": 0.8237928748130798,
"top1_hit": 0.9139999747276306,
"package_size_bytes": 658707940
},
"comparison": {
"embedding_cosine_mean": 0.9999097585678101,
"embedding_max_abs_difference": 0.005195870995521545,
"probability_mae": 7.114602340152487e-05,
"top1_agreement": 0.9900000095367432,
"top5_overlap": 0.9907999999999999,
"ndcg_at_5_delta_fp16_minus_fp32": 0.0010657310485839844,
"speedup_fp16_over_fp32": 2.3774675400547234,
"memory_reduction_fraction": 0.49251636122786024,
"package_size_reduction_fraction": 0.48445246377436435
},
"pure_fp16_compute": {
"supported": false,
"error": "EmbeddingGemma produced non-finite embeddings on H100 when computed entirely in FP16.",
"replacement": "FP16 weight storage with BF16 EmbeddingGemma compute and FP16 classifier compute"
}
} |