Download scripts/06_benchmark_quantization.py from AbhijithNusamNu/scam_dectect_sheid: direct link, hf CLI and curl.
- Browser
- Download file 7.41 kB
-
https://huggingface.co/spaces/AbhijithNusamNu/scam_dectect_sheid/resolve/main/scripts/06_benchmark_quantization.py
- Command line
-
hf download hf://spaces/AbhijithNusamNu/scam_dectect_sheid/scripts/06_benchmark_quantization.py
-
curl -L -o 06_benchmark_quantization.py https://huggingface.co/spaces/AbhijithNusamNu/scam_dectect_sheid/resolve/main/scripts/06_benchmark_quantization.py
7.41 kB
| """Benchmark FP32, FP16, and INT8 audio-branch classifiers.""" | |
| import os | |
| import time | |
| import subprocess | |
| from pathlib import Path | |
| import torch | |
| import pandas as pd | |
| import mlflow | |
| from transformers import AutoModelForSequenceClassification, AutoTokenizer | |
| from dotenv import load_dotenv | |
| REPO_ROOT = Path(__file__).resolve().parents[1] | |
| BRANCH_STAGE_MAP = { | |
| "feature/phase-2-audio-asr": "feature/phase-2-audio-asr", | |
| "feature/phase-3-serving-quantization": "feature/phase-3-serving-quantization", | |
| "feature/phase-1.5-ultimate-dataset": "model-modernbert-universal", | |
| "model-long-context": "model-modernbert-universal", | |
| "model-distilbert": "model-distilbert", | |
| "main": "main", | |
| } | |
| def detect_branch(default="main"): | |
| env_branch = os.getenv("DAGSHUB_BRANCH") or os.getenv("GIT_BRANCH") | |
| if env_branch: | |
| return env_branch.strip() | |
| result = subprocess.run( | |
| ["git", "branch", "--show-current"], | |
| cwd=REPO_ROOT, | |
| capture_output=True, | |
| text=True, | |
| check=False, | |
| ) | |
| branch = result.stdout.strip() | |
| return branch or default | |
| def stage_for_branch(branch): | |
| return BRANCH_STAGE_MAP.get(branch, branch.replace("/", "-") or "main") | |
| def benchmark_latency(model, tokenizer, texts, device, num_runs=50): | |
| """Measures average inference latency in milliseconds.""" | |
| model.eval() | |
| model.to(device) | |
| # Warm up | |
| sample = tokenizer("Warmup text", return_tensors="pt").to(device) | |
| for _ in range(5): | |
| with torch.no_grad(): | |
| _ = model(**sample) | |
| latencies = [] | |
| for text in texts[:num_runs]: | |
| inputs = tokenizer(str(text), return_tensors="pt", max_length=512, truncation=True).to(device) | |
| start = time.time() | |
| with torch.no_grad(): | |
| _ = model(**inputs) | |
| latencies.append((time.time() - start) * 1000) # ms | |
| avg_latency = sum(latencies) / len(latencies) | |
| return avg_latency | |
| def benchmark_accuracy(model, tokenizer, texts, labels, device): | |
| model.eval() | |
| model.to(device) | |
| preds = [] | |
| with torch.no_grad(): | |
| for text in texts: | |
| inputs = tokenizer(str(text), return_tensors="pt", max_length=512, truncation=True).to(device) | |
| logits = model(**inputs).logits | |
| preds.append(int(torch.argmax(logits, dim=-1).item())) | |
| correct = sum(int(pred == label) for pred, label in zip(preds, labels)) | |
| return correct / max(len(labels), 1) | |
| def main(): | |
| print("Initializing PTQ Benchmarking...") | |
| load_dotenv() | |
| branch = detect_branch() | |
| stage = stage_for_branch(branch) | |
| stage_slug = stage.replace("/", "-") | |
| # Check MLflow config | |
| repo_owner = os.getenv("DAGSHUB_REPO_OWNER") | |
| repo_name = os.getenv("DAGSHUB_REPO_NAME") | |
| if repo_owner and repo_name: | |
| import dagshub | |
| dagshub.init(repo_owner=repo_owner, repo_name=repo_name, mlflow=True) | |
| mlflow.set_experiment(f"scam-detection/{stage}/quantization") | |
| # Load dataset | |
| csv_path = "data/phase2_asr/ptq_calibration.csv" | |
| if not os.path.exists(csv_path): | |
| print(f"Data not found locally at {csv_path}. Fetching from DagsHub S3...") | |
| repo_owner = os.getenv("DAGSHUB_REPO_OWNER") | |
| repo_name = os.getenv("DAGSHUB_REPO_NAME") | |
| token = os.getenv("MLFLOW_TRACKING_PASSWORD") | |
| if repo_owner and repo_name and token: | |
| import dagshub | |
| dagshub.auth.add_app_token(token) | |
| s3 = dagshub.get_repo_bucket_client(f"{repo_owner}/{repo_name}") | |
| os.makedirs(os.path.dirname(csv_path), exist_ok=True) | |
| try: | |
| s3.download_file(repo_name, f"data/{stage}/phase2_asr/ptq_calibration.csv", csv_path) | |
| print(" [SUCCESS] Data downloaded.") | |
| except Exception as e: | |
| print(f" [FAILED] S3 Download error: {e}") | |
| return | |
| else: | |
| print(" [FAILED] Missing DagsHub credentials. Skipping benchmark.") | |
| return | |
| df = pd.read_csv(csv_path) | |
| texts = df['text'].tolist() | |
| labels = df['label'].tolist() | |
| model_name = f"models:/ModernBERT-Scam-Classifier-{stage_slug}/latest" | |
| print("Loading tokenizer and Baseline FP32 Model...") | |
| if model_name.startswith("models:/"): | |
| local_model_path = mlflow.artifacts.download_artifacts(artifact_uri=model_name) | |
| components = mlflow.transformers.load_model(local_model_path, return_type="components") | |
| tokenizer = components["tokenizer"] | |
| model_fp32 = components["model"] | |
| else: | |
| tokenizer = AutoTokenizer.from_pretrained(model_name) | |
| model_fp32 = AutoModelForSequenceClassification.from_pretrained(model_name) | |
| # Measure Baseline (CPU) | |
| print("Benchmarking Baseline (FP32) on CPU...") | |
| fp32_latency = benchmark_latency(model_fp32, tokenizer, texts, device="cpu") | |
| print(f" FP32 CPU Latency: {fp32_latency:.2f} ms") | |
| fp32_accuracy = benchmark_accuracy(model_fp32, tokenizer, texts, labels, device="cpu") | |
| print(f" FP32 CPU Accuracy: {fp32_accuracy:.4f}") | |
| fp16_latency = None | |
| fp16_accuracy = None | |
| if torch.cuda.is_available(): | |
| print("Benchmarking Baseline (FP16) on CUDA...") | |
| model_fp16 = model_fp32.half().to("cuda") | |
| fp16_latency = benchmark_latency(model_fp16, tokenizer, texts, device="cuda") | |
| fp16_accuracy = benchmark_accuracy(model_fp16, tokenizer, texts, labels, device="cuda") | |
| print(f" FP16 CUDA Latency: {fp16_latency:.2f} ms") | |
| print(f" FP16 CUDA Accuracy: {fp16_accuracy:.4f}") | |
| # Measure Baseline (GPU/MPS if available) | |
| if torch.cuda.is_available(): | |
| print("Benchmarking Baseline (FP32) on CUDA...") | |
| fp32_gpu_latency = benchmark_latency(model_fp32, tokenizer, texts, device="cuda") | |
| print(f" FP32 CUDA Latency: {fp32_gpu_latency:.2f} ms") | |
| # Measure INT8 (Must be CPU) | |
| int8_path = "models/quantized_classifier/classifier_int8.pt" | |
| if os.path.exists(int8_path): | |
| print("Loading Quantized INT8 Model...") | |
| model_int8 = torch.load(int8_path, map_location="cpu") | |
| print("Benchmarking Quantized (INT8) on CPU...") | |
| int8_latency = benchmark_latency(model_int8, tokenizer, texts, device="cpu") | |
| int8_accuracy = benchmark_accuracy(model_int8, tokenizer, texts, labels, device="cpu") | |
| print(f" INT8 CPU Latency: {int8_latency:.2f} ms") | |
| print(f" INT8 CPU Accuracy: {int8_accuracy:.4f}") | |
| speedup = fp32_latency / int8_latency | |
| print(f" Speedup vs FP32 CPU: {speedup:.2f}x") | |
| with mlflow.start_run(run_name=f"{stage}-int8-benchmark"): | |
| mlflow.log_metric("fp32_cpu_latency_ms", fp32_latency) | |
| mlflow.log_metric("fp32_cpu_accuracy", fp32_accuracy) | |
| if fp16_latency is not None: | |
| mlflow.log_metric("fp16_cuda_latency_ms", fp16_latency) | |
| mlflow.log_metric("fp16_cuda_accuracy", fp16_accuracy) | |
| mlflow.log_metric("int8_cpu_latency_ms", int8_latency) | |
| mlflow.log_metric("int8_cpu_accuracy", int8_accuracy) | |
| mlflow.log_metric("cpu_speedup_factor", speedup) | |
| if torch.cuda.is_available(): | |
| mlflow.log_metric("fp32_cuda_latency_ms", fp32_gpu_latency) | |
| else: | |
| print(f"INT8 model not found at {int8_path}. Run 05_quantize_models.py first.") | |
| if __name__ == "__main__": | |
| main() | |