File size: 7,412 Bytes
ae77244
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
"""Benchmark FP32, FP16, and INT8 audio-branch classifiers."""

import os
import time
import subprocess
from pathlib import Path

import torch
import pandas as pd
import mlflow
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from dotenv import load_dotenv

REPO_ROOT = Path(__file__).resolve().parents[1]
BRANCH_STAGE_MAP = {
    "feature/phase-2-audio-asr": "feature/phase-2-audio-asr",
    "feature/phase-3-serving-quantization": "feature/phase-3-serving-quantization",
    "feature/phase-1.5-ultimate-dataset": "model-modernbert-universal",
    "model-long-context": "model-modernbert-universal",
    "model-distilbert": "model-distilbert",
    "main": "main",
}


def detect_branch(default="main"):
    env_branch = os.getenv("DAGSHUB_BRANCH") or os.getenv("GIT_BRANCH")
    if env_branch:
        return env_branch.strip()
    result = subprocess.run(
        ["git", "branch", "--show-current"],
        cwd=REPO_ROOT,
        capture_output=True,
        text=True,
        check=False,
    )
    branch = result.stdout.strip()
    return branch or default


def stage_for_branch(branch):
    return BRANCH_STAGE_MAP.get(branch, branch.replace("/", "-") or "main")

def benchmark_latency(model, tokenizer, texts, device, num_runs=50):
    """Measures average inference latency in milliseconds."""
    model.eval()
    model.to(device)
    
    # Warm up
    sample = tokenizer("Warmup text", return_tensors="pt").to(device)
    for _ in range(5):
        with torch.no_grad():
            _ = model(**sample)
            
    latencies = []
    for text in texts[:num_runs]:
        inputs = tokenizer(str(text), return_tensors="pt", max_length=512, truncation=True).to(device)
        start = time.time()
        with torch.no_grad():
            _ = model(**inputs)
        latencies.append((time.time() - start) * 1000) # ms
        
    avg_latency = sum(latencies) / len(latencies)
    return avg_latency


def benchmark_accuracy(model, tokenizer, texts, labels, device):
    model.eval()
    model.to(device)
    preds = []
    with torch.no_grad():
        for text in texts:
            inputs = tokenizer(str(text), return_tensors="pt", max_length=512, truncation=True).to(device)
            logits = model(**inputs).logits
            preds.append(int(torch.argmax(logits, dim=-1).item()))
    correct = sum(int(pred == label) for pred, label in zip(preds, labels))
    return correct / max(len(labels), 1)

def main():
    print("Initializing PTQ Benchmarking...")
    load_dotenv()
    branch = detect_branch()
    stage = stage_for_branch(branch)
    stage_slug = stage.replace("/", "-")
    
    # Check MLflow config
    repo_owner = os.getenv("DAGSHUB_REPO_OWNER")
    repo_name = os.getenv("DAGSHUB_REPO_NAME")
    if repo_owner and repo_name:
        import dagshub
        dagshub.init(repo_owner=repo_owner, repo_name=repo_name, mlflow=True)
    
    mlflow.set_experiment(f"scam-detection/{stage}/quantization")
    
    # Load dataset
    csv_path = "data/phase2_asr/ptq_calibration.csv"
    if not os.path.exists(csv_path):
        print(f"Data not found locally at {csv_path}. Fetching from DagsHub S3...")
        repo_owner = os.getenv("DAGSHUB_REPO_OWNER")
        repo_name = os.getenv("DAGSHUB_REPO_NAME")
        token = os.getenv("MLFLOW_TRACKING_PASSWORD")
        if repo_owner and repo_name and token:
            import dagshub
            dagshub.auth.add_app_token(token)
            s3 = dagshub.get_repo_bucket_client(f"{repo_owner}/{repo_name}")
            os.makedirs(os.path.dirname(csv_path), exist_ok=True)
            try:
                s3.download_file(repo_name, f"data/{stage}/phase2_asr/ptq_calibration.csv", csv_path)
                print("  [SUCCESS] Data downloaded.")
            except Exception as e:
                print(f"  [FAILED] S3 Download error: {e}")
                return
        else:
            print("  [FAILED] Missing DagsHub credentials. Skipping benchmark.")
            return
        
    df = pd.read_csv(csv_path)
    texts = df['text'].tolist()
    labels = df['label'].tolist()
    
    model_name = f"models:/ModernBERT-Scam-Classifier-{stage_slug}/latest"
    print("Loading tokenizer and Baseline FP32 Model...")
    if model_name.startswith("models:/"):
        local_model_path = mlflow.artifacts.download_artifacts(artifact_uri=model_name)
        components = mlflow.transformers.load_model(local_model_path, return_type="components")
        tokenizer = components["tokenizer"]
        model_fp32 = components["model"]
    else:
        tokenizer = AutoTokenizer.from_pretrained(model_name)
        model_fp32 = AutoModelForSequenceClassification.from_pretrained(model_name)
    
    # Measure Baseline (CPU)
    print("Benchmarking Baseline (FP32) on CPU...")
    fp32_latency = benchmark_latency(model_fp32, tokenizer, texts, device="cpu")
    print(f"  FP32 CPU Latency: {fp32_latency:.2f} ms")
    fp32_accuracy = benchmark_accuracy(model_fp32, tokenizer, texts, labels, device="cpu")
    print(f"  FP32 CPU Accuracy: {fp32_accuracy:.4f}")

    fp16_latency = None
    fp16_accuracy = None
    if torch.cuda.is_available():
        print("Benchmarking Baseline (FP16) on CUDA...")
        model_fp16 = model_fp32.half().to("cuda")
        fp16_latency = benchmark_latency(model_fp16, tokenizer, texts, device="cuda")
        fp16_accuracy = benchmark_accuracy(model_fp16, tokenizer, texts, labels, device="cuda")
        print(f"  FP16 CUDA Latency: {fp16_latency:.2f} ms")
        print(f"  FP16 CUDA Accuracy: {fp16_accuracy:.4f}")
    
    # Measure Baseline (GPU/MPS if available)
    if torch.cuda.is_available():
        print("Benchmarking Baseline (FP32) on CUDA...")
        fp32_gpu_latency = benchmark_latency(model_fp32, tokenizer, texts, device="cuda")
        print(f"  FP32 CUDA Latency: {fp32_gpu_latency:.2f} ms")
    
    # Measure INT8 (Must be CPU)
    int8_path = "models/quantized_classifier/classifier_int8.pt"
    if os.path.exists(int8_path):
        print("Loading Quantized INT8 Model...")
        model_int8 = torch.load(int8_path, map_location="cpu")
        print("Benchmarking Quantized (INT8) on CPU...")
        int8_latency = benchmark_latency(model_int8, tokenizer, texts, device="cpu")
        int8_accuracy = benchmark_accuracy(model_int8, tokenizer, texts, labels, device="cpu")
        print(f"  INT8 CPU Latency: {int8_latency:.2f} ms")
        print(f"  INT8 CPU Accuracy: {int8_accuracy:.4f}")
        speedup = fp32_latency / int8_latency
        print(f"  Speedup vs FP32 CPU: {speedup:.2f}x")
        
        with mlflow.start_run(run_name=f"{stage}-int8-benchmark"):
            mlflow.log_metric("fp32_cpu_latency_ms", fp32_latency)
            mlflow.log_metric("fp32_cpu_accuracy", fp32_accuracy)
            if fp16_latency is not None:
                mlflow.log_metric("fp16_cuda_latency_ms", fp16_latency)
                mlflow.log_metric("fp16_cuda_accuracy", fp16_accuracy)
            mlflow.log_metric("int8_cpu_latency_ms", int8_latency)
            mlflow.log_metric("int8_cpu_accuracy", int8_accuracy)
            mlflow.log_metric("cpu_speedup_factor", speedup)
            if torch.cuda.is_available():
                mlflow.log_metric("fp32_cuda_latency_ms", fp32_gpu_latency)
    else:
        print(f"INT8 model not found at {int8_path}. Run 05_quantize_models.py first.")

if __name__ == "__main__":
    main()