import numpy as np import streamlit as st import torch from torchvision import models, transforms from torch import nn from PIL import Image import json import cv2 from huggingface_hub import hf_hub_download device = "cuda" if torch.cuda.is_available() else "cpu" # YOLO main classes (hardcoded) yolo_class_names = [ "plastic_waste", "metal_waste", "glass", "composite_paper_waste", "organic_waste", "e_waste", "medical_waste", ] # Colors for each main class box color_map = { "plastic_waste": (0, 255, 255), "metal_waste": (255, 0, 0), "glass": (0, 255, 0), "composite_paper_waste": (255, 165, 0), "organic_waste": (0, 128, 0), "e_waste": (128, 0, 128), "medical_waste": (255, 0, 255), "other_waste": (128, 128, 128) } # Mapping main classes to subclass indices main_to_subclass_indices = { "plastic_waste": [21, 24, 6, 19, 22, 25, 23, 27], "metal_waste": [0, 1, 2], "glass": [8, 9, 10], "composite_paper_waste": [3, 4, 19, 27], "organic_waste": [7], "e_waste": [2, 12, 13, 16, 17, 18, 20, 26], "medical_waste": [5, 28, 11, 14, 15], } # Reverse mapping: subclass idx to main class (for quick lookup) subclass_to_main = {} for main_cls, sub_indices in main_to_subclass_indices.items(): for idx in sub_indices: subclass_to_main[idx] = main_cls # === HUGGING FACE REPO SETTINGS === HF_REPO_ID = "samraatd/waste-classification-models" # Replace with your HF repo YOLO_WEIGHTS_FILENAME = "best_yolov5_object_model.pt" CONVNEXT_WEIGHTS_FILENAME = "best_model_weights.pth" SUBCLASS_JSON_FILENAME = "class_to_idx.json" def load_subclass_labels(): path = hf_hub_download(repo_id=HF_REPO_ID, filename=SUBCLASS_JSON_FILENAME) with open(path, 'r') as f: class_to_idx = json.load(f) idx_to_class = {int(v): k for k, v in class_to_idx.items()} return idx_to_class @st.cache_resource def load_yolo_model(): from ultralytics import YOLO path = hf_hub_download(repo_id=HF_REPO_ID, filename=YOLO_WEIGHTS_FILENAME) model = YOLO(path) return model @st.cache_resource def load_convnext_model(num_classes): path = hf_hub_download(repo_id=HF_REPO_ID, filename=CONVNEXT_WEIGHTS_FILENAME) weights = models.ConvNeXt_Tiny_Weights.IMAGENET1K_V1 model = models.convnext_tiny(weights=weights) model.classifier[2] = nn.Linear(model.classifier[2].in_features, num_classes) checkpoint = torch.load(path, map_location=device) if isinstance(checkpoint, dict) and "model" in checkpoint: model.load_state_dict(checkpoint["model"]) else: model.load_state_dict(checkpoint) model.to(device) model.eval() return model # Preprocessing preprocess = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) st.title("YOLO + ConvNeXt Hierarchical Classification (HF Spaces)") uploaded_file = st.file_uploader("Upload an image", type=["jpg", "jpeg", "png"]) YOLO_CONF_THRESH = 0.5 if uploaded_file: subclass_labels = load_subclass_labels() num_classes = len(subclass_labels) yolo_model = load_yolo_model() convnext_model = load_convnext_model(num_classes) img = Image.open(uploaded_file).convert('RGB') img_cv = np.array(img)[:, :, ::-1].copy() results = yolo_model(img) boxes = results[0].boxes predictions_summary = [] for i, box in enumerate(boxes.xyxy): x1, y1, x2, y2 = map(int, box.tolist()) yolo_class_id = int(boxes.cls[i]) yolo_conf = boxes.conf[i].item() yolo_class = yolo_class_names[yolo_class_id] crop = img.crop((x1, y1, x2, y2)) img_tensor = preprocess(crop).unsqueeze(0).to(device) with torch.no_grad(): outputs = convnext_model(img_tensor) probs = torch.nn.functional.softmax(outputs, dim=1).squeeze(0) convnext_conf, pred_idx = torch.max(probs, dim=0) convnext_conf = convnext_conf.item() pred_idx_int = pred_idx.item() subclass_name = subclass_labels.get(pred_idx_int, "Unknown") main_class_from_subclass = subclass_to_main.get(pred_idx_int, None) if main_class_from_subclass is None: note = f"Not confidently mapped to main class" final_main_class = "other_waste" else: note = "" final_main_class = main_class_from_subclass predictions_summary.append({ "box_index": i, "yolo_class": yolo_class, "yolo_conf": yolo_conf, "main_class": final_main_class, "subclass_name": subclass_name, "convnext_conf": convnext_conf, "note": note, "box_coords": (x1, y1, x2, y2), "color": color_map.get(final_main_class, (255, 255, 255)) }) label = f"{final_main_class} (YOLO: {yolo_conf:.2f}) | {subclass_name} ({convnext_conf:.2f})" if note: label += f"\n{note}" color = color_map.get(final_main_class, (255, 255, 255)) cv2.rectangle(img_cv, (x1, y1), (x2, y2), color, 2) font = cv2.FONT_HERSHEY_SIMPLEX font_scale = 0.5 thickness = 1 lines = label.split('\n') y_offset = 0 for line in lines: text_size = cv2.getTextSize(line, font, font_scale, thickness)[0] text_x = x1 text_y = y1 - 10 - y_offset if y1 - 10 - y_offset > 10 else y1 + text_size[1] + y_offset cv2.rectangle(img_cv, (text_x, text_y - text_size[1]), (text_x + text_size[0], text_y), color, -1) cv2.putText(img_cv, line, (text_x, text_y - 2), font, font_scale, (0, 0, 0), thickness, cv2.LINE_AA) y_offset += text_size[1] + 5 img_cv_rgb = cv2.cvtColor(img_cv, cv2.COLOR_BGR2RGB) st.image(img_cv_rgb, caption="YOLO + ConvNeXt Predictions", use_container_width=True) st.markdown("### Detected objects and subclass predictions:") for pred in predictions_summary: note_text = f" — *{pred['note']}*" if pred['note'] else "" st.markdown(f"- **Box #{pred['box_index']}**: Main class: **{pred['main_class']}** (YOLO confidence: {pred['yolo_conf']:.2f}) | Subclass: **{pred['subclass_name']}** (ConvNeXt confidence: {pred['convnext_conf']:.2f}){note_text}")