{ "cells": [ { "cell_type": "markdown", "id": "ac6e10d4", "metadata": {}, "source": [ "# Fusion Model Conversion" ] }, { "cell_type": "code", "execution_count": 4, "id": "0f63f311", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Downloading checkpoint from Hugging Face Hub: 'junaid17/best_fusion_model_fp16/best_fusion_model_fp16.pt'...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Loading weights: 100%|██████████| 342/342 [00:00<00:00, 1952.82it/s]\n", "[transformers] \u001b[1mConvNextModel LOAD REPORT\u001b[0m from: facebook/convnext-small-224\n", "Key | Status | | \n", "------------------+------------+--+-\n", "classifier.weight | UNEXPECTED | | \n", "classifier.bias | UNEXPECTED | | \n", "\n", "Notes:\n", "- UNEXPECTED:\tcan be ignored when loading from different task/architecture; not ok if you expect identical arch.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "✅ Model loaded successfully from Hugging Face.\n", "Exporting Fusion Model to temporary ONNX graph...\n", "✅ Base ONNX export successful.\n", "Compressing ONNX model to FP16 precision...\n", "✅ ONNX graph compressed to FP16 successfully.\n", "Uploading 'fusion_model.onnx' to Hugging Face repo: 'junaid17/best_fusion_model_fp16'...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Processing Files (1 / 1): 100%|██████████| 142MB / 142MB, 680kB/s \n", "New Data Upload: 100%|██████████| 142MB / 142MB, 680kB/s \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "🚀 Successfully uploaded FP16 model to https://huggingface.co/junaid17/best_fusion_model_fp16/blob/main/fusion_model.onnx\n", "Downloading 'fusion_model.onnx' from Hugging Face for evaluation...\n", "📸 Found 60 images in 'D:\\DamageLensAI\\test_images'. Starting benchmark...\n", "\n", "===========================================================================\n", "IMAGE NAME | PYTORCH (ms) | ONNX (ms) | MATCH? \n", "===========================================================================\n", "FB_10.jpg | 775.56 | 225.80 | ✅ YES \n", "FB_11.jpg | 683.06 | 263.21 | ✅ YES \n", "FB_13.jpg | 359.21 | 156.10 | ✅ YES \n", "FB_15.jpg | 357.64 | 152.04 | ✅ YES \n", "FB_16.jpg | 340.61 | 157.13 | ✅ YES \n", "FB_19.jpg | 343.34 | 157.87 | ✅ YES \n", "FB_20.jpg | 562.84 | 188.25 | ✅ YES \n", "FB_4.jpg | 328.80 | 162.34 | ✅ YES \n", "FB_5.jpg | 466.30 | 166.86 | ✅ YES \n", "FB_9.jpg | 657.23 | 188.45 | ✅ YES \n", "FC_1.jpg | 372.87 | 234.55 | ✅ YES \n", "FC_10.jpg | 473.04 | 164.85 | ✅ YES \n", "FC_11.jpg | 332.92 | 154.70 | ✅ YES \n", "FC_12.jpg | 379.71 | 197.05 | ✅ YES \n", "FC_3.jpg | 546.80 | 297.15 | ✅ YES \n", "FC_4.jpg | 810.00 | 387.67 | ✅ YES \n", "FC_6.jpg | 528.11 | 211.10 | ✅ YES \n", "FC_7.jpg | 546.06 | 267.98 | ✅ YES \n", "FC_8.jpg | 665.58 | 230.13 | ✅ YES \n", "FC_9.jpg | 516.99 | 213.58 | ✅ YES \n", "FN_1.jpg | 497.13 | 216.17 | ✅ YES \n", "FN_10.jpg | 484.10 | 215.26 | ✅ YES \n", "FN_11.jpg | 521.18 | 221.95 | ✅ YES \n", "FN_12.jpg | 498.07 | 235.80 | ✅ YES \n", "FN_3.jpg | 473.85 | 219.38 | ✅ YES \n", "FN_4.jpg | 474.16 | 419.76 | ✅ YES \n", "FN_5.jpg | 796.23 | 229.17 | ✅ YES \n", "FN_7.jpg | 506.63 | 211.40 | ✅ YES \n", "FN_8.jpg | 432.50 | 237.60 | ✅ YES \n", "FN_9.jpg | 725.21 | 239.89 | ✅ YES \n", "RB_10.jpg | 480.74 | 220.76 | ✅ YES \n", "RB_11.jpg | 449.81 | 224.74 | ✅ YES \n", "RB_13.jpg | 478.89 | 221.27 | ✅ YES \n", "RB_14.jpg | 500.14 | 222.87 | ✅ YES \n", "RB_2.jpg | 494.09 | 223.37 | ✅ YES \n", "RB_3.jpg | 488.29 | 220.04 | ✅ YES \n", "RB_4.jpg | 498.87 | 233.70 | ✅ YES \n", "RB_6.jpg | 496.31 | 221.88 | ✅ YES \n", "RB_7.jpg | 477.51 | 214.55 | ✅ YES \n", "RB_8.jpg | 503.84 | 217.26 | ✅ YES \n", "RC_10.jpg | 491.95 | 221.90 | ✅ YES \n", "RC_11.jpg | 497.75 | 218.97 | ✅ YES \n", "RC_2.jpg | 493.97 | 222.79 | ✅ YES \n", "RC_3.jpg | 481.42 | 220.67 | ✅ YES \n", "RC_4.jpg | 487.19 | 224.59 | ✅ YES \n", "RC_5.jpg | 484.33 | 219.35 | ✅ YES \n", "RC_6.jpg | 483.71 | 216.13 | ✅ YES \n", "RC_7.jpg | 572.23 | 213.54 | ✅ YES \n", "RC_8.jpg | 484.69 | 218.72 | ✅ YES \n", "RC_9.jpg | 578.70 | 218.05 | ✅ YES \n", "RN_1.jpg | 493.88 | 221.69 | ✅ YES \n", "RN_10.jpg | 469.91 | 213.39 | ✅ YES \n", "RN_11.jpg | 488.03 | 212.66 | ✅ YES \n", "RN_12.jpg | 495.62 | 209.92 | ✅ YES \n", "RN_2.jpg | 484.84 | 214.93 | ✅ YES \n", "RN_3.jpg | 467.09 | 218.67 | ✅ YES \n", "RN_4.jpg | 496.70 | 215.96 | ✅ YES \n", "RN_7.jpg | 480.62 | 214.66 | ✅ YES \n", "RN_8.jpg | 502.00 | 213.91 | ✅ YES \n", "RN_9.jpg | 466.22 | 217.81 | ✅ YES \n", "===========================================================================\n", " BENCHMARK SUMMARY\n", "===========================================================================\n", "Total Images Evaluated : 60\n", "Prediction Match Rate : 100.00% (60/60)\n", "PyTorch Avg Latency : 503.75 ms / image\n", "ONNX Runtime Avg Latency: 219.87 ms / image\n", "Performance Gain : 56.35% faster with ONNX Runtime\n", "===========================================================================\n", "\n" ] } ], "source": [ "import os\n", "import glob\n", "import time\n", "import tempfile\n", "import warnings\n", "import torch\n", "import torchvision.transforms as transforms\n", "import onnxruntime as ort\n", "import numpy as np\n", "import onnx\n", "from onnxconverter_common import float16\n", "from PIL import Image\n", "from dotenv import load_dotenv\n", "from transformers import ConvNextImageProcessor\n", "from huggingface_hub import HfApi, hf_hub_download\n", "\n", "# Import model loader from base_fusion_model.py\n", "from base_fusion_model import load_fusion_model_from_hf\n", "\n", "# Filter out PyTorch tracing and transformers load warnings for clean logs\n", "warnings.filterwarnings(\"ignore\", category=torch.jit.TracerWarning)\n", "warnings.filterwarnings(\"ignore\", category=UserWarning)\n", "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n", "\n", "# ==========================================\n", "# 1. PREPROCESSING PIPELINE SETUP\n", "# ==========================================\n", "IMG_SIZE = 260\n", "\n", "val_transforms = transforms.Compose([\n", " transforms.Resize((IMG_SIZE, IMG_SIZE)),\n", "])\n", "\n", "eff_normalize = transforms.Compose([\n", " transforms.ToTensor(),\n", " transforms.Normalize(\n", " mean=[0.485, 0.456, 0.406],\n", " std=[0.229, 0.224, 0.225]\n", " )\n", "])\n", "\n", "\n", "def preprocess_image(pil_image: Image.Image, processor: ConvNextImageProcessor):\n", " resized_image = val_transforms(pil_image)\n", " pixel_eff = eff_normalize(resized_image).unsqueeze(0)\n", " enc = processor(images=resized_image, return_tensors='pt')\n", " pixel_cnx = enc['pixel_values']\n", "\n", " return pixel_eff, pixel_cnx\n", "\n", "\n", "# ==========================================\n", "# 2. CONVERT & UPLOAD TO HUGGING FACE\n", "# ==========================================\n", "def convert_and_upload_onnx(\n", " model: torch.nn.Module,\n", " processor: ConvNextImageProcessor,\n", " repo_id: str,\n", " onnx_filename_in_repo: str = \"fusion_model.onnx\",\n", " hf_token: str = None,\n", "):\n", " model.eval()\n", "\n", " dummy_pil = Image.new(\"RGB\", (IMG_SIZE, IMG_SIZE), color=\"white\")\n", " dummy_eff, dummy_cnx = preprocess_image(dummy_pil, processor)\n", "\n", " with tempfile.TemporaryDirectory() as temp_dir:\n", " temp_onnx_path = os.path.join(temp_dir, onnx_filename_in_repo)\n", "\n", " print(\"Exporting Fusion Model to temporary ONNX graph...\")\n", " \n", " with warnings.catch_warnings():\n", " warnings.simplefilter(\"ignore\")\n", " torch.onnx.export(\n", " model,\n", " (dummy_eff, dummy_cnx),\n", " temp_onnx_path,\n", " export_params=True,\n", " opset_version=18,\n", " do_constant_folding=True,\n", " input_names=[\"pixel_values_eff\", \"pixel_values_cnx\"],\n", " output_names=[\"logits\"],\n", " dynamic_axes={\n", " \"pixel_values_eff\": {0: \"batch_size\"},\n", " \"pixel_values_cnx\": {0: \"batch_size\"},\n", " \"logits\": {0: \"batch_size\"},\n", " },\n", " dynamo=False,\n", " )\n", " print(\"✅ Base ONNX export successful.\")\n", "\n", " # Convert ONNX graph weights to FP16 to keep file size ~142 MB\n", " print(\"Compressing ONNX model to FP16 precision...\")\n", " onnx_model = onnx.load(temp_onnx_path)\n", " onnx_model_fp16 = float16.convert_float_to_float16(onnx_model, keep_io_types=True)\n", " onnx.save(onnx_model_fp16, temp_onnx_path)\n", " print(\"✅ ONNX graph compressed to FP16 successfully.\")\n", "\n", " # Upload directly to Hugging Face Hub using token from .env\n", " api = HfApi(token=hf_token)\n", " print(f\"Uploading '{onnx_filename_in_repo}' to Hugging Face repo: '{repo_id}'...\")\n", " \n", " api.upload_file(\n", " path_or_fileobj=temp_onnx_path,\n", " path_in_repo=onnx_filename_in_repo,\n", " repo_id=repo_id,\n", " repo_type=\"model\",\n", " )\n", " print(f\"🚀 Successfully uploaded FP16 model to https://huggingface.co/{repo_id}/blob/main/{onnx_filename_in_repo}\")\n", "\n", "\n", "# ==========================================\n", "# 3. BENCHMARK ON REAL TEST IMAGES\n", "# ==========================================\n", "def benchmark_fusion_with_folder(\n", " pytorch_model: torch.nn.Module,\n", " processor: ConvNextImageProcessor,\n", " repo_id: str,\n", " onnx_filename_in_repo: str,\n", " image_folder: str = r\"D:\\DamageLensAI\\test_images\",\n", " hf_token: str = None,\n", "):\n", " print(f\"Downloading '{onnx_filename_in_repo}' from Hugging Face for evaluation...\")\n", " onnx_path = hf_hub_download(\n", " repo_id=repo_id, \n", " filename=onnx_filename_in_repo,\n", " token=hf_token\n", " )\n", " session = ort.InferenceSession(onnx_path, providers=[\"CPUExecutionProvider\"])\n", "\n", " supported_exts = (\"*.jpg\", \"*.jpeg\", \"*.png\", \"*.bmp\", \"*.webp\")\n", " image_paths = []\n", " for ext in supported_exts:\n", " image_paths.extend(glob.glob(os.path.join(image_folder, ext)))\n", "\n", " if not image_paths:\n", " raise FileNotFoundError(\n", " f\"❌ No valid images found in folder '{image_folder}'. \"\n", " f\"Please place your test images inside '{image_folder}'.\"\n", " )\n", "\n", " print(f\"📸 Found {len(image_paths)} images in '{image_folder}'. Starting benchmark...\")\n", "\n", " pytorch_latencies = []\n", " onnx_latencies = []\n", " matches = 0\n", "\n", " # Warmup\n", " first_img = Image.open(sorted(image_paths)[0]).convert(\"RGB\")\n", " warmup_eff, warmup_cnx = preprocess_image(first_img, processor)\n", " warmup_eff_np = warmup_eff.numpy()\n", " warmup_cnx_np = warmup_cnx.numpy()\n", "\n", " pytorch_model.eval()\n", " with torch.no_grad():\n", " _ = pytorch_model(warmup_eff, warmup_cnx)\n", " session.run(None, {\"pixel_values_eff\": warmup_eff_np, \"pixel_values_cnx\": warmup_cnx_np})\n", "\n", " # Benchmark loop\n", " print(\"\\n\" + \"=\" * 75)\n", " print(f\"{'IMAGE NAME':<25} | {'PYTORCH (ms)':<14} | {'ONNX (ms)':<14} | {'MATCH?':<10}\")\n", " print(\"=\" * 75)\n", "\n", " for img_path in sorted(image_paths):\n", " file_name = os.path.basename(img_path)\n", " raw_img = Image.open(img_path).convert(\"RGB\")\n", " \n", " pixel_values_eff, pixel_values_cnx = preprocess_image(raw_img, processor)\n", " np_eff = pixel_values_eff.numpy()\n", " np_cnx = pixel_values_cnx.numpy()\n", "\n", " # PyTorch Inference\n", " start_pt = time.perf_counter()\n", " with torch.no_grad():\n", " pt_logits = pytorch_model(pixel_values_eff, pixel_values_cnx)\n", " pt_time = (time.perf_counter() - start_pt) * 1000\n", " pytorch_latencies.append(pt_time)\n", " pt_pred = int(torch.argmax(pt_logits, dim=1)[0])\n", "\n", " # ONNX Runtime Inference\n", " start_onnx = time.perf_counter()\n", " onnx_outputs = session.run(\n", " None, \n", " {\"pixel_values_eff\": np_eff, \"pixel_values_cnx\": np_cnx}\n", " )\n", " onnx_time = (time.perf_counter() - start_onnx) * 1000\n", " onnx_latencies.append(onnx_time)\n", " onnx_pred = int(np.argmax(onnx_outputs[0], axis=1)[0])\n", "\n", " is_match = pt_pred == onnx_pred\n", " if is_match:\n", " matches += 1\n", " match_str = \"✅ YES\" if is_match else \"❌ NO\"\n", "\n", " print(f\"{file_name:<25} | {pt_time:<14.2f} | {onnx_time:<14.2f} | {match_str:<10}\")\n", "\n", " avg_pt = np.mean(pytorch_latencies)\n", " avg_onnx = np.mean(onnx_latencies)\n", " speedup = ((avg_pt - avg_onnx) / avg_pt) * 100\n", " accuracy_match_pct = (matches / len(image_paths)) * 100\n", "\n", " print(\"=\" * 75)\n", " print(\" BENCHMARK SUMMARY\")\n", " print(\"=\" * 75)\n", " print(f\"Total Images Evaluated : {len(image_paths)}\")\n", " print(f\"Prediction Match Rate : {accuracy_match_pct:.2f}% ({matches}/{len(image_paths)})\")\n", " print(f\"PyTorch Avg Latency : {avg_pt:.2f} ms / image\")\n", " print(f\"ONNX Runtime Avg Latency: {avg_onnx:.2f} ms / image\")\n", " print(f\"Performance Gain : {speedup:.2f}% faster with ONNX Runtime\")\n", " print(\"=\" * 75 + \"\\n\")\n", "\n", "\n", "# ==========================================\n", "# MAIN EXECUTION FLOW\n", "# ==========================================\n", "if __name__ == \"__main__\":\n", " # Load environment variables from .env file\n", " load_dotenv()\n", " HF_TOKEN = os.getenv(\"HF_TOKEN\")\n", "\n", " if not HF_TOKEN:\n", " print(\"⚠️ Warning: HF_TOKEN not found in .env file. Pushing/pulling private repos may fail.\")\n", "\n", " NUM_CLASSES = 6\n", " HF_REPO_ID = \"junaid17/best_fusion_model_fp16\"\n", " ONNX_FILENAME = \"fusion_model.onnx\"\n", " TEST_IMAGE_FOLDER = r\"D:\\DamageLensAI\\test_images\"\n", " CONVNEXT_MODEL_NAME = \"facebook/convnext-small-224\"\n", "\n", " # Step 1: Initialize ConvNeXt Processor\n", " processor = ConvNextImageProcessor.from_pretrained(CONVNEXT_MODEL_NAME)\n", "\n", " # Step 2: Load PyTorch model from Hugging Face\n", " py_model = load_fusion_model_from_hf(\n", " repo_id=HF_REPO_ID,\n", " filename=\"best_fusion_model_fp16.pt\",\n", " num_classes=NUM_CLASSES,\n", " device=\"cpu\",\n", " )\n", "\n", " # Step 3: Convert to FP16 ONNX and push directly to Hugging Face\n", " convert_and_upload_onnx(\n", " model=py_model,\n", " processor=processor,\n", " repo_id=HF_REPO_ID,\n", " onnx_filename_in_repo=ONNX_FILENAME,\n", " hf_token=HF_TOKEN,\n", " )\n", "\n", " # Step 4: Run benchmarking on test image folder\n", " benchmark_fusion_with_folder(\n", " pytorch_model=py_model,\n", " processor=processor,\n", " repo_id=HF_REPO_ID,\n", " onnx_filename_in_repo=ONNX_FILENAME,\n", " image_folder=TEST_IMAGE_FOLDER,\n", " hf_token=HF_TOKEN,\n", " )" ] }, { "cell_type": "markdown", "id": "0e10195c", "metadata": {}, "source": [ "# Resnet Model Conversion" ] }, { "cell_type": "code", "execution_count": 12, "id": "000b4500", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Downloading checkpoint 'car-damage-classifier.pt' from Hugging Face repo 'junaid17/car-damage-classifier'...\n", "✅ ResNet-18 model loaded successfully.\n", "Exporting ResNet-18 Model to temporary FP32 ONNX graph...\n", "✅ Base ONNX export successful.\n", "Compressing ONNX model to FP16 precision...\n", "✅ ONNX graph compressed to FP16 successfully.\n", "Uploading 'car-damage-classifier.onnx' to Hugging Face repo: 'junaid17/car-damage-classifier'...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Processing Files (1 / 1): 100%|██████████| 22.6MB / 22.6MB, 1.05MB/s \n", "New Data Upload: 100%|██████████| 22.6MB / 22.6MB, 1.05MB/s \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "🚀 Successfully uploaded FP16 ONNX model to https://huggingface.co/junaid17/car-damage-classifier/blob/main/car-damage-classifier.onnx\n", "Downloading 'car-damage-classifier.onnx' from Hugging Face for evaluation...\n", "📸 Found 60 images in 'D:\\DamageLensAI\\test_images'. Starting benchmark...\n", "\n", "===========================================================================\n", "IMAGE NAME | PYTORCH (ms) | ONNX (ms) | MATCH? \n", "===========================================================================\n", "FB_10.jpg | 46.78 | 14.66 | ✅ YES \n", "FB_11.jpg | 56.13 | 15.03 | ✅ YES \n", "FB_13.jpg | 45.83 | 15.16 | ✅ YES \n", "FB_15.jpg | 44.06 | 14.06 | ✅ YES \n", "FB_16.jpg | 54.86 | 15.04 | ✅ YES \n", "FB_19.jpg | 61.18 | 15.78 | ✅ YES \n", "FB_20.jpg | 47.12 | 13.44 | ✅ YES \n", "FB_4.jpg | 49.21 | 17.78 | ✅ YES \n", "FB_5.jpg | 74.55 | 15.02 | ✅ YES \n", "FB_9.jpg | 48.68 | 14.77 | ✅ YES \n", "FC_1.jpg | 54.30 | 14.43 | ✅ YES \n", "FC_10.jpg | 50.06 | 15.70 | ✅ YES \n", "FC_11.jpg | 48.00 | 13.72 | ✅ YES \n", "FC_12.jpg | 42.24 | 14.62 | ✅ YES \n", "FC_3.jpg | 51.02 | 14.97 | ✅ YES \n", "FC_4.jpg | 47.95 | 15.39 | ✅ YES \n", "FC_6.jpg | 46.55 | 14.33 | ✅ YES \n", "FC_7.jpg | 59.15 | 15.21 | ✅ YES \n", "FC_8.jpg | 48.33 | 16.83 | ✅ YES \n", "FC_9.jpg | 45.76 | 15.09 | ✅ YES \n", "FN_1.jpg | 58.81 | 15.92 | ✅ YES \n", "FN_10.jpg | 46.00 | 15.53 | ✅ YES \n", "FN_11.jpg | 56.70 | 16.44 | ✅ YES \n", "FN_12.jpg | 52.64 | 14.61 | ✅ YES \n", "FN_3.jpg | 52.38 | 14.43 | ✅ YES \n", "FN_4.jpg | 50.76 | 17.84 | ✅ YES \n", "FN_5.jpg | 51.05 | 14.57 | ✅ YES \n", "FN_7.jpg | 49.76 | 14.34 | ✅ YES \n", "FN_8.jpg | 45.30 | 16.05 | ✅ YES \n", "FN_9.jpg | 58.23 | 16.13 | ✅ YES \n", "RB_10.jpg | 58.82 | 16.95 | ✅ YES \n", "RB_11.jpg | 86.68 | 19.35 | ✅ YES \n", "RB_13.jpg | 88.30 | 22.82 | ✅ YES \n", "RB_14.jpg | 76.32 | 18.26 | ✅ YES \n", "RB_2.jpg | 63.31 | 17.44 | ✅ YES \n", "RB_3.jpg | 58.23 | 14.74 | ✅ YES \n", "RB_4.jpg | 55.97 | 15.32 | ✅ YES \n", "RB_6.jpg | 157.29 | 16.34 | ✅ YES \n", "RB_7.jpg | 124.51 | 19.32 | ✅ YES \n", "RB_8.jpg | 63.44 | 15.15 | ✅ YES \n", "RC_10.jpg | 79.38 | 15.67 | ✅ YES \n", "RC_11.jpg | 54.87 | 13.93 | ✅ YES \n", "RC_2.jpg | 37.57 | 13.65 | ✅ YES \n", "RC_3.jpg | 42.66 | 13.28 | ✅ YES \n", "RC_4.jpg | 40.99 | 14.31 | ✅ YES \n", "RC_5.jpg | 55.75 | 17.52 | ✅ YES \n", "RC_6.jpg | 47.62 | 13.57 | ✅ YES \n", "RC_7.jpg | 43.82 | 14.02 | ✅ YES \n", "RC_8.jpg | 47.47 | 14.76 | ✅ YES \n", "RC_9.jpg | 50.02 | 14.33 | ✅ YES \n", "RN_1.jpg | 45.03 | 14.66 | ✅ YES \n", "RN_10.jpg | 44.92 | 14.12 | ✅ YES \n", "RN_11.jpg | 47.08 | 13.45 | ✅ YES \n", "RN_12.jpg | 52.43 | 15.71 | ✅ YES \n", "RN_2.jpg | 49.53 | 16.44 | ✅ YES \n", "RN_3.jpg | 43.17 | 14.90 | ✅ YES \n", "RN_4.jpg | 43.36 | 14.52 | ✅ YES \n", "RN_7.jpg | 50.25 | 14.68 | ✅ YES \n", "RN_8.jpg | 44.29 | 13.59 | ✅ YES \n", "RN_9.jpg | 44.30 | 15.32 | ✅ YES \n", "===========================================================================\n", " BENCHMARK SUMMARY\n", "===========================================================================\n", "Total Images Evaluated : 60\n", "Prediction Match Rate : 100.00% (60/60)\n", "PyTorch Avg Latency : 55.68 ms / image\n", "ONNX Runtime Avg Latency: 15.42 ms / image\n", "Performance Gain : 72.31% faster with ONNX Runtime\n", "===========================================================================\n", "\n" ] } ], "source": [ "import os\n", "import glob\n", "import time\n", "import tempfile\n", "import warnings\n", "import torch\n", "import torchvision.transforms as transforms\n", "import onnxruntime as ort\n", "import numpy as np\n", "import onnx\n", "from onnxconverter_common import float16\n", "from PIL import Image\n", "from dotenv import load_dotenv\n", "from huggingface_hub import HfApi, hf_hub_download\n", "\n", "# Import ResNet loader\n", "from base_model_resnet import load_resnet_model_from_hf\n", "\n", "# Suppress PyTorch tracing and deprecation warnings\n", "warnings.filterwarnings(\"ignore\", category=torch.jit.TracerWarning)\n", "warnings.filterwarnings(\"ignore\", category=UserWarning)\n", "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n", "\n", "# ==========================================\n", "# 1. PREPROCESSING PIPELINE\n", "# ==========================================\n", "test_transforms = transforms.Compose([\n", " transforms.Resize((224, 224)),\n", " transforms.ToTensor(),\n", " transforms.Normalize(\n", " mean=[0.485, 0.456, 0.406],\n", " std=[0.229, 0.224, 0.225]\n", " )\n", "])\n", "\n", "\n", "def preprocess_image(pil_image: Image.Image) -> torch.Tensor:\n", " \"\"\"\n", " Applies standard ResNet preprocessing and adds a batch dimension.\n", " Shape returned: (1, 3, 224, 224)\n", " \"\"\"\n", " return test_transforms(pil_image).unsqueeze(0)\n", "\n", "\n", "# ==========================================\n", "# 2. CONVERT TO FP16 ONNX & UPLOAD\n", "# ==========================================\n", "def convert_and_upload_onnx_fp16(\n", " model: torch.nn.Module,\n", " repo_id: str,\n", " onnx_filename_in_repo: str = \"car-damage-classifier.onnx\",\n", " hf_token: str = None,\n", "):\n", " \"\"\"\n", " Exports PyTorch ResNet model to ONNX, compresses it to FP16,\n", " and pushes it directly to Hugging Face Hub.\n", " \"\"\"\n", " model.eval()\n", "\n", " # Create dummy tensor for graph tracing (1, 3, 224, 224)\n", " dummy_input = torch.randn(1, 3, 224, 224)\n", "\n", " with tempfile.TemporaryDirectory() as temp_dir:\n", " temp_onnx_path = os.path.join(temp_dir, onnx_filename_in_repo)\n", "\n", " print(\"Exporting ResNet-18 Model to temporary FP32 ONNX graph...\")\n", " \n", " with warnings.catch_warnings():\n", " warnings.simplefilter(\"ignore\")\n", " torch.onnx.export(\n", " model,\n", " dummy_input,\n", " temp_onnx_path,\n", " export_params=True,\n", " opset_version=18,\n", " do_constant_folding=True,\n", " input_names=[\"pixel_values\"],\n", " output_names=[\"logits\"],\n", " dynamic_axes={\n", " \"pixel_values\": {0: \"batch_size\"},\n", " \"logits\": {0: \"batch_size\"},\n", " },\n", " dynamo=False,\n", " )\n", " print(\"✅ Base ONNX export successful.\")\n", "\n", " # Convert ONNX graph weights to FP16 to reduce file size\n", " print(\"Compressing ONNX model to FP16 precision...\")\n", " onnx_model = onnx.load(temp_onnx_path)\n", " onnx_model_fp16 = float16.convert_float_to_float16(onnx_model, keep_io_types=True)\n", " onnx.save(onnx_model_fp16, temp_onnx_path)\n", " print(\"✅ ONNX graph compressed to FP16 successfully.\")\n", "\n", " # Upload directly to Hugging Face Hub\n", " api = HfApi(token=hf_token)\n", " print(f\"Uploading '{onnx_filename_in_repo}' to Hugging Face repo: '{repo_id}'...\")\n", " \n", " api.upload_file(\n", " path_or_fileobj=temp_onnx_path,\n", " path_in_repo=onnx_filename_in_repo,\n", " repo_id=repo_id,\n", " repo_type=\"model\",\n", " )\n", " print(f\"🚀 Successfully uploaded FP16 ONNX model to https://huggingface.co/{repo_id}/blob/main/{onnx_filename_in_repo}\")\n", "\n", "\n", "# ==========================================\n", "# 3. BENCHMARK ON REAL TEST IMAGES\n", "# ==========================================\n", "def benchmark_resnet_with_folder(\n", " pytorch_model: torch.nn.Module,\n", " repo_id: str,\n", " onnx_filename_in_repo: str,\n", " image_folder: str = r\"D:\\DamageLensAI\\test_images\",\n", " hf_token: str = None,\n", "):\n", " \"\"\"\n", " Downloads ONNX model from Hugging Face Hub and evaluates PyTorch vs ONNX Runtime performance.\n", " \"\"\"\n", " print(f\"Downloading '{onnx_filename_in_repo}' from Hugging Face for evaluation...\")\n", " onnx_path = hf_hub_download(\n", " repo_id=repo_id, \n", " filename=onnx_filename_in_repo,\n", " token=hf_token\n", " )\n", " session = ort.InferenceSession(onnx_path, providers=[\"CPUExecutionProvider\"])\n", "\n", " supported_exts = (\"*.jpg\", \"*.jpeg\", \"*.png\", \"*.bmp\", \"*.webp\")\n", " image_paths = []\n", " for ext in supported_exts:\n", " image_paths.extend(glob.glob(os.path.join(image_folder, ext)))\n", "\n", " if not image_paths:\n", " raise FileNotFoundError(\n", " f\"❌ No valid images found in folder '{image_folder}'. \"\n", " f\"Please verify your path.\"\n", " )\n", "\n", " print(f\"📸 Found {len(image_paths)} images in '{image_folder}'. Starting benchmark...\")\n", "\n", " pytorch_latencies = []\n", " onnx_latencies = []\n", " matches = 0\n", "\n", " # Warmup run\n", " first_img = Image.open(sorted(image_paths)[0]).convert(\"RGB\")\n", " warmup_tensor = preprocess_image(first_img)\n", " warmup_np = warmup_tensor.numpy()\n", "\n", " pytorch_model.eval()\n", " with torch.no_grad():\n", " _ = pytorch_model(warmup_tensor)\n", " session.run(None, {\"pixel_values\": warmup_np})\n", "\n", " # Benchmark Loop\n", " print(\"\\n\" + \"=\" * 75)\n", " print(f\"{'IMAGE NAME':<25} | {'PYTORCH (ms)':<14} | {'ONNX (ms)':<14} | {'MATCH?':<10}\")\n", " print(\"=\" * 75)\n", "\n", " for img_path in sorted(image_paths):\n", " file_name = os.path.basename(img_path)\n", " raw_img = Image.open(img_path).convert(\"RGB\")\n", " \n", " img_tensor = preprocess_image(raw_img)\n", " np_input = img_tensor.numpy()\n", "\n", " # --- PyTorch Inference ---\n", " start_pt = time.perf_counter()\n", " with torch.no_grad():\n", " pt_logits = pytorch_model(img_tensor)\n", " pt_time = (time.perf_counter() - start_pt) * 1000\n", " pytorch_latencies.append(pt_time)\n", " pt_pred = int(torch.argmax(pt_logits, dim=1)[0])\n", "\n", " # --- ONNX Runtime Inference ---\n", " start_onnx = time.perf_counter()\n", " onnx_outputs = session.run(None, {\"pixel_values\": np_input})\n", " onnx_time = (time.perf_counter() - start_onnx) * 1000\n", " onnx_latencies.append(onnx_time)\n", " onnx_pred = int(np.argmax(onnx_outputs[0], axis=1)[0])\n", "\n", " is_match = pt_pred == onnx_pred\n", " if is_match:\n", " matches += 1\n", " match_str = \"✅ YES\" if is_match else \"❌ NO\"\n", "\n", " print(f\"{file_name:<25} | {pt_time:<14.2f} | {onnx_time:<14.2f} | {match_str:<10}\")\n", "\n", " avg_pt = np.mean(pytorch_latencies)\n", " avg_onnx = np.mean(onnx_latencies)\n", " speedup = ((avg_pt - avg_onnx) / avg_pt) * 100\n", " accuracy_match_pct = (matches / len(image_paths)) * 100\n", "\n", " print(\"=\" * 75)\n", " print(\" BENCHMARK SUMMARY\")\n", " print(\"=\" * 75)\n", " print(f\"Total Images Evaluated : {len(image_paths)}\")\n", " print(f\"Prediction Match Rate : {accuracy_match_pct:.2f}% ({matches}/{len(image_paths)})\")\n", " print(f\"PyTorch Avg Latency : {avg_pt:.2f} ms / image\")\n", " print(f\"ONNX Runtime Avg Latency: {avg_onnx:.2f} ms / image\")\n", " print(f\"Performance Gain : {speedup:.2f}% faster with ONNX Runtime\")\n", " print(\"=\" * 75 + \"\\n\")\n", "\n", "\n", "# ==========================================\n", "# MAIN EXECUTION FLOW\n", "# ==========================================\n", "if __name__ == \"__main__\":\n", " # Load token from .env\n", " load_dotenv()\n", " HF_TOKEN = os.getenv(\"HF_TOKEN\")\n", "\n", " if not HF_TOKEN:\n", " print(\"⚠️ Warning: HF_TOKEN not found in .env file. Pushing/pulling private repos may fail.\")\n", "\n", " NUM_CLASSES = 6\n", " HF_REPO_ID = \"junaid17/car-damage-classifier\"\n", " MODEL_FILENAME = \"car-damage-classifier.pt\"\n", " ONNX_FILENAME = \"car-damage-classifier.onnx\"\n", " TEST_IMAGE_FOLDER = r\"D:\\DamageLensAI\\test_images\"\n", "\n", " # Step 1: Load PyTorch ResNet Model from Hugging Face\n", " py_model = load_resnet_model_from_hf(\n", " repo_id=HF_REPO_ID,\n", " filename=MODEL_FILENAME,\n", " num_classes=NUM_CLASSES,\n", " device=\"cpu\",\n", " hf_token=HF_TOKEN,\n", " )\n", "\n", " # Step 2: Convert to FP16 ONNX & Push directly to Hugging Face\n", " convert_and_upload_onnx_fp16(\n", " model=py_model,\n", " repo_id=HF_REPO_ID,\n", " onnx_filename_in_repo=ONNX_FILENAME,\n", " hf_token=HF_TOKEN,\n", " )\n", "\n", " # Step 3: Run benchmarking on test image folder\n", " benchmark_resnet_with_folder(\n", " pytorch_model=py_model,\n", " repo_id=HF_REPO_ID,\n", " onnx_filename_in_repo=ONNX_FILENAME,\n", " image_folder=TEST_IMAGE_FOLDER,\n", " hf_token=HF_TOKEN,\n", " )" ] }, { "cell_type": "markdown", "id": "0c387aa2", "metadata": {}, "source": [ "# YOLO Model Conversion" ] }, { "cell_type": "code", "execution_count": 11, "id": "f0e5e525", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Downloading YOLO checkpoint 'damage_detector.pt' from Hugging Face repo 'junaid17/Yolo_Model'...\n", "✅ YOLO model loaded successfully.\n", "Exporting YOLO11 Model directly to FP16 ONNX format (imgsz=640)...\n", "WARNING 'half' is deprecated and will be removed in the future. Use 'quantize' instead.\n", "Ultralytics 8.4.116 Python-3.11.0 torch-2.13.0+cpu CPU (12th Gen Intel Core i3-1215U)\n", "YOLO11l summary (fused): 190 layers, 25,280,083 parameters, 0 gradients, 86.8 GFLOPs\n", "\n", "\u001b[34m\u001b[1mPyTorch:\u001b[0m starting from 'C:\\Users\\junai\\.cache\\huggingface\\hub\\models--junaid17--Yolo_Model\\snapshots\\0cbc6f112ebc6cb7df05fe93f3f0175cc847bec0\\damage_detector.pt' with input shape (1, 3, 640, 640) BCHW and output shape(s) (1, 5, 8400) (48.8 MB)\n", "\n", "\u001b[34m\u001b[1mONNX:\u001b[0m starting export with onnx 1.22.0 opset 17...\n", "\u001b[34m\u001b[1mONNX:\u001b[0m slimming with onnxslim 0.1.95...\n", "\u001b[34m\u001b[1mONNX:\u001b[0m converting to FP16...\n", "\u001b[34m\u001b[1mONNX:\u001b[0m export success 7.1s, saved as 'C:\\Users\\junai\\.cache\\huggingface\\hub\\models--junaid17--Yolo_Model\\snapshots\\0cbc6f112ebc6cb7df05fe93f3f0175cc847bec0\\damage_detector.onnx' (48.5 MB)\n", "\n", "Export complete (11.6s)\n", "Results saved to \u001b[1mC:\\Users\\junai\\.cache\\huggingface\\hub\\models--junaid17--Yolo_Model\\snapshots\\0cbc6f112ebc6cb7df05fe93f3f0175cc847bec0\\damage_detector.onnx\u001b[0m\n", "Predict: yolo predict task=detect model=C:\\Users\\junai\\.cache\\huggingface\\hub\\models--junaid17--Yolo_Model\\snapshots\\0cbc6f112ebc6cb7df05fe93f3f0175cc847bec0\\damage_detector.onnx imgsz=640 quantize=16\n", "Validate: yolo val task=detect model=C:\\Users\\junai\\.cache\\huggingface\\hub\\models--junaid17--Yolo_Model\\snapshots\\0cbc6f112ebc6cb7df05fe93f3f0175cc847bec0\\damage_detector.onnx imgsz=640 data=/content/drive/MyDrive/custom_data/dataset_custom.yaml quantize=16 \n", "Visualize: https://netron.app\n", "✅ FP16 ONNX export successful.\n", "📦 Model File Size: 48.50 MB\n", "Uploading 'damage_detector.onnx' to Hugging Face repo: 'junaid17/Yolo_Model'...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Processing Files (1 / 1): 100%|██████████| 50.9MB / 50.9MB, 2.79MB/s \n", "New Data Upload: 100%|██████████| 8.84MB / 8.84MB, 568kB/s \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "🚀 Successfully uploaded ONNX model to https://huggingface.co/junaid17/Yolo_Model/blob/main/damage_detector.onnx\n", "Downloading 'damage_detector.onnx' from Hugging Face for evaluation...\n", "📸 Found 60 images in 'D:\\DamageLensAI\\test_images'. Starting benchmark...\n", "\n", "===========================================================================\n", "IMAGE NAME | PYTORCH (ms) | ONNX (ms) | MATCH? \n", "===========================================================================\n", "FB_10.jpg | 1961.16 | 933.21 | ✅ YES \n", "FB_11.jpg | 1645.58 | 821.06 | ✅ YES \n", "FB_13.jpg | 1973.07 | 884.90 | ✅ YES \n", "FB_15.jpg | 2296.55 | 1064.76 | ✅ YES \n", "FB_16.jpg | 1159.60 | 679.19 | ✅ YES \n", "FB_19.jpg | 1310.77 | 729.79 | ✅ YES \n", "FB_20.jpg | 1263.96 | 678.22 | ✅ YES \n", "FB_4.jpg | 1385.82 | 658.67 | ✅ YES \n", "FB_5.jpg | 1352.15 | 665.79 | ✅ YES \n", "FB_9.jpg | 1281.14 | 654.91 | ✅ YES \n", "FC_1.jpg | 1430.82 | 639.33 | ✅ YES \n", "FC_10.jpg | 1223.63 | 772.76 | ✅ YES \n", "FC_11.jpg | 1229.15 | 665.59 | ✅ YES \n", "FC_12.jpg | 1442.46 | 654.07 | ✅ YES \n", "FC_3.jpg | 1255.18 | 635.24 | ✅ YES \n", "FC_4.jpg | 1381.86 | 646.25 | ✅ YES \n", "FC_6.jpg | 1380.10 | 644.81 | ✅ YES \n", "FC_7.jpg | 1211.48 | 644.08 | ✅ YES \n", "FC_8.jpg | 1201.65 | 761.60 | ✅ YES \n", "FC_9.jpg | 1429.97 | 644.52 | ✅ YES \n", "FN_1.jpg | 1181.08 | 649.43 | ✅ YES \n", "FN_10.jpg | 1190.17 | 637.96 | ✅ YES \n", "FN_11.jpg | 1590.08 | 764.77 | ✅ YES \n", "FN_12.jpg | 1300.76 | 601.92 | ✅ YES \n", "FN_3.jpg | 1182.89 | 662.93 | ✅ YES \n", "FN_4.jpg | 1638.14 | 650.37 | ✅ YES \n", "FN_5.jpg | 1426.86 | 657.21 | ✅ YES \n", "FN_7.jpg | 1440.60 | 717.18 | ✅ YES \n", "FN_8.jpg | 1385.24 | 621.54 | ✅ YES \n", "FN_9.jpg | 1377.56 | 643.62 | ✅ YES \n", "RB_10.jpg | 1362.92 | 694.52 | ✅ YES \n", "RB_11.jpg | 1352.90 | 630.82 | ✅ YES \n", "RB_13.jpg | 1373.38 | 684.08 | ✅ YES \n", "RB_14.jpg | 1395.17 | 693.95 | ✅ YES \n", "RB_2.jpg | 1323.94 | 619.14 | ✅ YES \n", "RB_3.jpg | 1204.51 | 657.64 | ✅ YES \n", "RB_4.jpg | 1468.20 | 673.22 | ✅ YES \n", "RB_6.jpg | 1317.30 | 635.59 | ✅ YES \n", "RB_7.jpg | 1384.84 | 639.61 | ✅ YES \n", "RB_8.jpg | 1476.26 | 668.84 | ✅ YES \n", "RC_10.jpg | 1292.19 | 650.32 | ✅ YES \n", "RC_11.jpg | 1374.14 | 667.15 | ✅ YES \n", "RC_2.jpg | 1403.93 | 776.57 | ✅ YES \n", "RC_3.jpg | 1314.82 | 644.49 | ✅ YES \n", "RC_4.jpg | 1283.66 | 626.54 | ✅ YES \n", "RC_5.jpg | 1343.69 | 682.90 | ✅ YES \n", "RC_6.jpg | 1348.45 | 603.56 | ✅ YES \n", "RC_7.jpg | 1229.81 | 623.58 | ✅ YES \n", "RC_8.jpg | 1312.53 | 670.69 | ✅ YES \n", "RC_9.jpg | 1196.52 | 713.91 | ✅ YES \n", "RN_1.jpg | 1125.28 | 619.14 | ✅ YES \n", "RN_10.jpg | 1177.20 | 745.49 | ✅ YES \n", "RN_11.jpg | 1377.85 | 632.45 | ✅ YES \n", "RN_12.jpg | 1271.99 | 611.52 | ✅ YES \n", "RN_2.jpg | 1166.40 | 715.83 | ✅ YES \n", "RN_3.jpg | 1209.96 | 602.54 | ✅ YES \n", "RN_4.jpg | 1129.91 | 696.41 | ✅ YES \n", "RN_7.jpg | 1165.09 | 642.48 | ✅ YES \n", "RN_8.jpg | 1327.08 | 600.25 | ✅ YES \n", "RN_9.jpg | 1123.79 | 609.49 | ✅ YES \n", "===========================================================================\n", " BENCHMARK SUMMARY\n", "===========================================================================\n", "Total Images Evaluated : 60\n", "Detection Match Rate : 100.00% (60/60)\n", "PyTorch Avg Latency : 1356.05 ms / image\n", "ONNX Runtime Avg Latency: 680.31 ms / image\n", "Performance Gain : 49.83% faster with ONNX Runtime\n", "===========================================================================\n", "\n" ] } ], "source": [ "import os\n", "import glob\n", "import time\n", "import warnings\n", "import torch\n", "import torchvision.transforms as transforms\n", "import onnxruntime as ort\n", "import numpy as np\n", "from PIL import Image\n", "from dotenv import load_dotenv\n", "from huggingface_hub import HfApi, hf_hub_download\n", "from ultralytics import YOLO\n", "\n", "# Import YOLO loader\n", "from base_model_yolo import load_yolo_model_from_hf\n", "\n", "# Suppress warnings\n", "warnings.filterwarnings(\"ignore\")\n", "\n", "# ==========================================\n", "# 1. PREPROCESSING PIPELINE\n", "# ==========================================\n", "IMG_SIZE = 640\n", "\n", "yolo_transforms = transforms.Compose([\n", " transforms.Resize((IMG_SIZE, IMG_SIZE)),\n", " transforms.ToTensor(),\n", "])\n", "\n", "def preprocess_image(pil_image: Image.Image) -> torch.Tensor:\n", " \"\"\"Preprocesses PIL image for YOLO11 (1, 3, 640, 640).\"\"\"\n", " return yolo_transforms(pil_image).unsqueeze(0)\n", "\n", "\n", "# ==========================================\n", "# 2. CONVERT, OPTIMIZE TO FP16 & UPLOAD\n", "# ==========================================\n", "def convert_and_upload_yolo_onnx(\n", " yolo_model: YOLO,\n", " repo_id: str,\n", " onnx_filename_in_repo: str = \"damage_detector.onnx\",\n", " hf_token: str = None,\n", " imgsz: int = 640,\n", "):\n", " print(f\"Exporting YOLO11 Model directly to FP16 ONNX format (imgsz={imgsz})...\")\n", " \n", " # Native Ultralytics FP16 export - lightning fast and avoids converter bugs\n", " exported_onnx_path = yolo_model.export(\n", " format=\"onnx\",\n", " imgsz=imgsz,\n", " half=True, # Shrinks the model to ~50MB natively\n", " dynamic=False, # Disabling dynamic fixes the CPU type-mismatch errors\n", " simplify=True,\n", " opset=17\n", " )\n", " print(\"✅ FP16 ONNX export successful.\")\n", "\n", " file_size_mb = os.path.getsize(exported_onnx_path) / (1024 * 1024)\n", " print(f\"📦 Model File Size: {file_size_mb:.2f} MB\")\n", "\n", " # Upload to Hugging Face Hub\n", " api = HfApi(token=hf_token)\n", " print(f\"Uploading '{onnx_filename_in_repo}' to Hugging Face repo: '{repo_id}'...\")\n", " \n", " api.upload_file(\n", " path_or_fileobj=exported_onnx_path,\n", " path_in_repo=onnx_filename_in_repo,\n", " repo_id=repo_id,\n", " repo_type=\"model\",\n", " )\n", " print(f\"🚀 Successfully uploaded ONNX model to https://huggingface.co/{repo_id}/blob/main/{onnx_filename_in_repo}\")\n", "\n", "\n", "# ==========================================\n", "# 3. BENCHMARK ON REAL TEST IMAGES\n", "# ==========================================\n", "def benchmark_yolo_with_folder(\n", " yolo_model: YOLO,\n", " repo_id: str,\n", " onnx_filename_in_repo: str,\n", " image_folder: str = r\"D:\\DamageLensAI\\test_images\",\n", " hf_token: str = None,\n", " conf_thresh: float = 0.25,\n", "):\n", " print(f\"Downloading '{onnx_filename_in_repo}' from Hugging Face for evaluation...\")\n", " \n", " # force_download=True ensures we grab the newly uploaded 50MB file\n", " onnx_path = hf_hub_download(\n", " repo_id=repo_id, \n", " filename=onnx_filename_in_repo,\n", " token=hf_token,\n", " force_download=True \n", " )\n", " \n", " session = ort.InferenceSession(onnx_path, providers=[\"CPUExecutionProvider\"])\n", " input_name = session.get_inputs()[0].name\n", "\n", " supported_exts = (\"*.jpg\", \"*.jpeg\", \"*.png\", \"*.bmp\", \"*.webp\")\n", " image_paths = []\n", " for ext in supported_exts:\n", " image_paths.extend(glob.glob(os.path.join(image_folder, ext)))\n", "\n", " if not image_paths:\n", " raise FileNotFoundError(f\"❌ No valid images found in folder '{image_folder}'.\")\n", "\n", " print(f\"📸 Found {len(image_paths)} images in '{image_folder}'. Starting benchmark...\")\n", "\n", " pytorch_latencies = []\n", " onnx_latencies = []\n", " matches = 0\n", "\n", " # Warmup\n", " first_img = Image.open(sorted(image_paths)[0]).convert(\"RGB\")\n", " warmup_tensor = preprocess_image(first_img)\n", " warmup_np = warmup_tensor.numpy()\n", "\n", " _ = yolo_model.model(warmup_tensor)\n", " session.run(None, {input_name: warmup_np})\n", "\n", " print(\"\\n\" + \"=\" * 75)\n", " print(f\"{'IMAGE NAME':<25} | {'PYTORCH (ms)':<14} | {'ONNX (ms)':<14} | {'MATCH?':<10}\")\n", " print(\"=\" * 75)\n", "\n", " for img_path in sorted(image_paths):\n", " file_name = os.path.basename(img_path)\n", " raw_img = Image.open(img_path).convert(\"RGB\")\n", " \n", " img_tensor = preprocess_image(raw_img)\n", " np_input = img_tensor.numpy()\n", "\n", " start_pt = time.perf_counter()\n", " with torch.no_grad():\n", " pt_out = yolo_model.model(img_tensor)\n", " pt_time = (time.perf_counter() - start_pt) * 1000\n", " pytorch_latencies.append(pt_time)\n", "\n", " start_onnx = time.perf_counter()\n", " onnx_out = session.run(None, {input_name: np_input})\n", " onnx_time = (time.perf_counter() - start_onnx) * 1000\n", " onnx_latencies.append(onnx_time)\n", "\n", " pt_preds = pt_out[0] if isinstance(pt_out, (list, tuple)) else pt_out\n", " pt_max_conf = float(torch.max(pt_preds[0, 4, :])) if pt_preds.ndim == 3 else 0.0\n", " \n", " onnx_preds = onnx_out[0]\n", " onnx_max_conf = float(np.max(onnx_preds[0, 4, :])) if onnx_preds.ndim == 3 else 0.0\n", "\n", " pt_detected = pt_max_conf >= conf_thresh\n", " onnx_detected = onnx_max_conf >= conf_thresh\n", " is_match = (pt_detected == onnx_detected)\n", "\n", " if is_match:\n", " matches += 1\n", " match_str = \"✅ YES\" if is_match else \"❌ NO\"\n", "\n", " print(f\"{file_name:<25} | {pt_time:<14.2f} | {onnx_time:<14.2f} | {match_str:<10}\")\n", "\n", " avg_pt = np.mean(pytorch_latencies)\n", " avg_onnx = np.mean(onnx_latencies)\n", " speedup = ((avg_pt - avg_onnx) / avg_pt) * 100\n", " accuracy_match_pct = (matches / len(image_paths)) * 100\n", "\n", " print(\"=\" * 75)\n", " print(\" BENCHMARK SUMMARY\")\n", " print(\"=\" * 75)\n", " print(f\"Total Images Evaluated : {len(image_paths)}\")\n", " print(f\"Detection Match Rate : {accuracy_match_pct:.2f}% ({matches}/{len(image_paths)})\")\n", " print(f\"PyTorch Avg Latency : {avg_pt:.2f} ms / image\")\n", " print(f\"ONNX Runtime Avg Latency: {avg_onnx:.2f} ms / image\")\n", " print(f\"Performance Gain : {speedup:.2f}% faster with ONNX Runtime\")\n", " print(\"=\" * 75 + \"\\n\")\n", "\n", "\n", "# ==========================================\n", "# MAIN EXECUTION FLOW\n", "# ==========================================\n", "if __name__ == \"__main__\":\n", " load_dotenv()\n", " HF_TOKEN = os.getenv(\"HF_TOKEN\")\n", "\n", " if not HF_TOKEN:\n", " print(\"⚠️ Warning: HF_TOKEN not found in .env file.\")\n", "\n", " HF_REPO_ID = \"junaid17/Yolo_Model\"\n", " MODEL_FILENAME = \"damage_detector.pt\"\n", " ONNX_FILENAME = \"damage_detector.onnx\"\n", " TEST_IMAGE_FOLDER = r\"D:\\DamageLensAI\\test_images\"\n", "\n", " yolo_model = load_yolo_model_from_hf(\n", " repo_id=HF_REPO_ID,\n", " filename=MODEL_FILENAME,\n", " hf_token=HF_TOKEN,\n", " )\n", "\n", " convert_and_upload_yolo_onnx(\n", " yolo_model=yolo_model,\n", " repo_id=HF_REPO_ID,\n", " onnx_filename_in_repo=ONNX_FILENAME,\n", " hf_token=HF_TOKEN,\n", " imgsz=IMG_SIZE,\n", " )\n", "\n", " benchmark_yolo_with_folder(\n", " yolo_model=yolo_model,\n", " repo_id=HF_REPO_ID,\n", " onnx_filename_in_repo=ONNX_FILENAME,\n", " image_folder=TEST_IMAGE_FOLDER,\n", " hf_token=HF_TOKEN,\n", " )" ] }, { "cell_type": "code", "execution_count": null, "id": "5b257cbc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0e9c3a35", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8bed307b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "myvenv (3.11.0)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.0" } }, "nbformat": 4, "nbformat_minor": 5 }