Spaces:
Sleeping
Sleeping
Commit ·
02b6b5d
0
Parent(s):
Initialize AMSI-Net Multi-label Backend with Explainability
Browse files- .gitattributes +1 -0
- Dockerfile +28 -0
- README.md +26 -0
- backend/.gitattributes +7 -0
- backend/Dockerfile +28 -0
- backend/app.py +159 -0
- backend/class_mappings.py +26 -0
- backend/explainers.py +156 -0
- backend/hf_space_readme.md +26 -0
- backend/model_loader.py +50 -0
- backend/models_arch.py +325 -0
- backend/preprocessing.py +32 -0
- backend/requirements.txt +17 -0
- backend/verify_model.py +51 -0
- model/best_model.pth +3 -0
- pyproject.toml +25 -0
- requirements.txt +17 -0
.gitattributes
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
model/*.pth filter=lfs diff=lfs merge=lfs -text
|
Dockerfile
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Use official Python image
|
| 2 |
+
FROM python:3.9-slim
|
| 3 |
+
|
| 4 |
+
# Set working directory
|
| 5 |
+
WORKDIR /app
|
| 6 |
+
|
| 7 |
+
# Install system dependencies for OpenCV and other libraries
|
| 8 |
+
RUN apt-get update && apt-get install -y \
|
| 9 |
+
libgl1-mesa-glx \
|
| 10 |
+
libglib2.0-0 \
|
| 11 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 12 |
+
|
| 13 |
+
# Copy requirements and install
|
| 14 |
+
COPY requirements.txt .
|
| 15 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 16 |
+
|
| 17 |
+
# Copy source code and models
|
| 18 |
+
COPY backend/ ./backend/
|
| 19 |
+
COPY model/ ./model/
|
| 20 |
+
|
| 21 |
+
# Set Python path to include backend directory
|
| 22 |
+
ENV PYTHONPATH="/app/backend:${PYTHONPATH}"
|
| 23 |
+
|
| 24 |
+
# Expose port (HF Spaces use 7860)
|
| 25 |
+
EXPOSE 7860
|
| 26 |
+
|
| 27 |
+
# Run the application
|
| 28 |
+
CMD ["python", "backend/app.py"]
|
README.md
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: AMSI-Net LULC Recognition
|
| 3 |
+
emoji: 🛰️
|
| 4 |
+
colorFrom: indigo
|
| 5 |
+
colorTo: slate
|
| 6 |
+
sdk: docker
|
| 7 |
+
app_port: 8000
|
| 8 |
+
pinned: false
|
| 9 |
+
license: mit
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# AMSI-Net Multi-label LULC Recognition
|
| 13 |
+
|
| 14 |
+
This is a professional backend for Land Use & Land Cover (LULC) recognition using the AMSI-Net architecture.
|
| 15 |
+
|
| 16 |
+
## Features
|
| 17 |
+
- **Dataset**: MLRSNet (60 classes)
|
| 18 |
+
- **Architecture**: AMSI-Net (ResNet50 + Dynamic FPN + GAT)
|
| 19 |
+
- **Explainability**: GradCAM, GradCAM++, Saliency, LIME
|
| 20 |
+
- **API**: FastAPI with automatic Swagger docs
|
| 21 |
+
|
| 22 |
+
## API Documentation
|
| 23 |
+
The API documentation is available at `/docs` (Swagger UI).
|
| 24 |
+
|
| 25 |
+
## Deployment
|
| 26 |
+
This Space is built using Docker. The backend runs on port 8000 (mapped to HF default).
|
backend/.gitattributes
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
backend/Dockerfile
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Use official Python image
|
| 2 |
+
FROM python:3.9-slim
|
| 3 |
+
|
| 4 |
+
# Set working directory
|
| 5 |
+
WORKDIR /app
|
| 6 |
+
|
| 7 |
+
# Install system dependencies for OpenCV and other libraries
|
| 8 |
+
RUN apt-get update && apt-get install -y \
|
| 9 |
+
libgl1-mesa-glx \
|
| 10 |
+
libglib2.0-0 \
|
| 11 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 12 |
+
|
| 13 |
+
# Copy requirements and install
|
| 14 |
+
COPY backend/requirements.txt .
|
| 15 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 16 |
+
|
| 17 |
+
# Copy source code and models
|
| 18 |
+
COPY backend/ ./backend/
|
| 19 |
+
COPY model/ ./model/
|
| 20 |
+
|
| 21 |
+
# Set Python path to include backend directory
|
| 22 |
+
ENV PYTHONPATH="/app/backend:${PYTHONPATH}"
|
| 23 |
+
|
| 24 |
+
# Expose port (HF Spaces use 7860)
|
| 25 |
+
EXPOSE 7860
|
| 26 |
+
|
| 27 |
+
# Run the application
|
| 28 |
+
CMD ["python", "backend/app.py"]
|
backend/app.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FastAPI backend for AMSI-Net LULC Recognition System."""
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
import time
|
| 5 |
+
from fastapi import FastAPI, UploadFile, File, HTTPException
|
| 6 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 7 |
+
from pydantic import BaseModel
|
| 8 |
+
from typing import List, Dict, Any
|
| 9 |
+
import io
|
| 10 |
+
|
| 11 |
+
from model_loader import get_model_loader
|
| 12 |
+
from preprocessing import get_preprocessor
|
| 13 |
+
from class_mappings import get_class_label
|
| 14 |
+
from explainers import generate_all_explanations
|
| 15 |
+
|
| 16 |
+
app = FastAPI(
|
| 17 |
+
title="AMSI-Net LULC Recognition API",
|
| 18 |
+
description="Multi-label Remote Sensing Image Classification with AMSI-Net",
|
| 19 |
+
version="1.0.0"
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
# CORS configuration
|
| 23 |
+
app.add_middleware(
|
| 24 |
+
CORSMiddleware,
|
| 25 |
+
allow_origins=["*"],
|
| 26 |
+
allow_credentials=True,
|
| 27 |
+
allow_methods=["*"],
|
| 28 |
+
allow_headers=["*"],
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
# Response models
|
| 32 |
+
class ClassPrediction(BaseModel):
|
| 33 |
+
class_index: int
|
| 34 |
+
class_label: str
|
| 35 |
+
confidence: float
|
| 36 |
+
|
| 37 |
+
class PredictionResponse(BaseModel):
|
| 38 |
+
predicted_labels: List[str]
|
| 39 |
+
all_predictions: List[ClassPrediction]
|
| 40 |
+
explainability_maps: Dict[str, str]
|
| 41 |
+
uncertainty: float
|
| 42 |
+
inference_time_ms: float
|
| 43 |
+
image_info: Dict[str, Any]
|
| 44 |
+
|
| 45 |
+
# Constants
|
| 46 |
+
VALID_IMAGE_FORMATS = {"image/jpeg", "image/png", "image/jpg"}
|
| 47 |
+
MAX_IMAGE_SIZE = 10 * 1024 * 1024 # 10MB
|
| 48 |
+
CONFIDENCE_THRESHOLD = 0.5
|
| 49 |
+
|
| 50 |
+
def validate_image(file: UploadFile) -> bytes:
|
| 51 |
+
"""Validate and read image file."""
|
| 52 |
+
if file.content_type not in VALID_IMAGE_FORMATS:
|
| 53 |
+
raise HTTPException(
|
| 54 |
+
status_code=400,
|
| 55 |
+
detail=f"Invalid image format. Supported: JPEG, PNG. Got: {file.content_type}"
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
content = file.file.read()
|
| 59 |
+
if len(content) > MAX_IMAGE_SIZE:
|
| 60 |
+
raise HTTPException(
|
| 61 |
+
status_code=413,
|
| 62 |
+
detail=f"Image too large. Max size: {MAX_IMAGE_SIZE / 1024 / 1024}MB"
|
| 63 |
+
)
|
| 64 |
+
return content
|
| 65 |
+
|
| 66 |
+
@app.get("/")
|
| 67 |
+
async def root():
|
| 68 |
+
return {"status": "ok", "message": "AMSI-Net API is running"}
|
| 69 |
+
|
| 70 |
+
@app.get("/health")
|
| 71 |
+
async def health():
|
| 72 |
+
loader = get_model_loader()
|
| 73 |
+
return {
|
| 74 |
+
"status": "healthy",
|
| 75 |
+
"device": str(loader.get_device()),
|
| 76 |
+
"model_loaded": loader.model is not None
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
@app.post("/predict", response_model=PredictionResponse)
|
| 80 |
+
async def predict(file: UploadFile = File(...)):
|
| 81 |
+
"""Predict LULC classes for uploaded image."""
|
| 82 |
+
image_bytes = validate_image(file)
|
| 83 |
+
|
| 84 |
+
try:
|
| 85 |
+
preprocessor = get_preprocessor()
|
| 86 |
+
loader = get_model_loader()
|
| 87 |
+
|
| 88 |
+
# 1. Preprocess
|
| 89 |
+
image_info = preprocessor.get_image_info(image_bytes)
|
| 90 |
+
image_tensor = preprocessor.preprocess(image_bytes)
|
| 91 |
+
image_tensor = image_tensor.to(loader.get_device())
|
| 92 |
+
|
| 93 |
+
# 2. Inference
|
| 94 |
+
start_time = time.time()
|
| 95 |
+
model = loader.load_model()
|
| 96 |
+
|
| 97 |
+
with torch.no_grad():
|
| 98 |
+
outputs = model(image_tensor)
|
| 99 |
+
logits = outputs['logits']
|
| 100 |
+
uncertainty = outputs['uncertainty'].item()
|
| 101 |
+
|
| 102 |
+
# Multi-label probability
|
| 103 |
+
probabilities = torch.sigmoid(logits).squeeze()
|
| 104 |
+
|
| 105 |
+
inference_time = (time.time() - start_time) * 1000
|
| 106 |
+
|
| 107 |
+
# 3. Process results
|
| 108 |
+
all_predictions = []
|
| 109 |
+
predicted_labels = []
|
| 110 |
+
|
| 111 |
+
for i, prob in enumerate(probabilities.tolist()):
|
| 112 |
+
label = get_class_label(i)
|
| 113 |
+
conf = float(prob)
|
| 114 |
+
|
| 115 |
+
pred_obj = ClassPrediction(
|
| 116 |
+
class_index=i,
|
| 117 |
+
class_label=label,
|
| 118 |
+
confidence=conf
|
| 119 |
+
)
|
| 120 |
+
all_predictions.append(pred_obj)
|
| 121 |
+
|
| 122 |
+
if conf >= CONFIDENCE_THRESHOLD:
|
| 123 |
+
predicted_labels.append(label)
|
| 124 |
+
|
| 125 |
+
# If no label exceeds threshold, take the top-1
|
| 126 |
+
if not predicted_labels:
|
| 127 |
+
top_idx = torch.argmax(probabilities).item()
|
| 128 |
+
predicted_labels.append(get_class_label(top_idx))
|
| 129 |
+
|
| 130 |
+
# 4. Generate explanations
|
| 131 |
+
explain_maps = generate_all_explanations(model, preprocessor, image_tensor, image_bytes)
|
| 132 |
+
|
| 133 |
+
return PredictionResponse(
|
| 134 |
+
predicted_labels=predicted_labels,
|
| 135 |
+
all_predictions=sorted(all_predictions, key=lambda x: x.confidence, reverse=True),
|
| 136 |
+
explainability_maps=explain_maps,
|
| 137 |
+
uncertainty=uncertainty,
|
| 138 |
+
inference_time_ms=inference_time,
|
| 139 |
+
image_info=image_info
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
except Exception as e:
|
| 143 |
+
import traceback
|
| 144 |
+
traceback.print_exc()
|
| 145 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 146 |
+
|
| 147 |
+
@app.get("/info")
|
| 148 |
+
async def info():
|
| 149 |
+
return {
|
| 150 |
+
"model": "AMSI-Net",
|
| 151 |
+
"task": "Multi-label LULC Recognition",
|
| 152 |
+
"dataset": "MLRSNet (60 classes)",
|
| 153 |
+
"input_size": "224x224",
|
| 154 |
+
"features": ["GradCAM", "GradCAM++", "LIME", "Saliency", "Uncertainty Estimation"]
|
| 155 |
+
}
|
| 156 |
+
|
| 157 |
+
if __name__ == "__main__":
|
| 158 |
+
import uvicorn
|
| 159 |
+
uvicorn.run(app, host="0.0.0.0", port=8000)
|
backend/class_mappings.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# MLRSNet 60 labels as identified from dataset research
|
| 2 |
+
|
| 3 |
+
MLRS_LABELS = [
|
| 4 |
+
"airplane", "airport", "bare soil", "baseball diamond", "basketball court",
|
| 5 |
+
"beach", "bridge", "buildings", "cars", "chaparral",
|
| 6 |
+
"cloud", "containers", "crosswalk", "dense residential area", "desert",
|
| 7 |
+
"dock", "factory", "field", "football field", "forest",
|
| 8 |
+
"freeway", "golf course", "grass", "greenhouse", "gully",
|
| 9 |
+
"habor", "intersection", "island", "lake", "mobile home",
|
| 10 |
+
"mountain", "overpass", "park", "parking lot", "parkway",
|
| 11 |
+
"pavement", "railway", "railway station", "river", "road",
|
| 12 |
+
"roundabout", "runway", "sand", "sea", "ships",
|
| 13 |
+
"snow", "snowberg", "sparse residential area", "stadium", "swimming pool",
|
| 14 |
+
"tanks", "tennis court", "terrace", "track", "trail",
|
| 15 |
+
"transmission tower", "trees", "water", "wetland", "wind turbine"
|
| 16 |
+
]
|
| 17 |
+
|
| 18 |
+
def get_class_label(index: int) -> str:
|
| 19 |
+
"""Return the label for a given class index."""
|
| 20 |
+
if 0 <= index < len(MLRS_LABELS):
|
| 21 |
+
return MLRS_LABELS[index]
|
| 22 |
+
return f"Unknown ({index})"
|
| 23 |
+
|
| 24 |
+
def get_all_labels():
|
| 25 |
+
"""Return all labels."""
|
| 26 |
+
return MLRS_LABELS
|
backend/explainers.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Explainability module for AMSI-Net LULC model."""
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
import numpy as np
|
| 7 |
+
import cv2
|
| 8 |
+
import base64
|
| 9 |
+
from io import BytesIO
|
| 10 |
+
import PIL.Image
|
| 11 |
+
from pytorch_grad_cam import GradCAM, GradCAMPlusPlus
|
| 12 |
+
from pytorch_grad_cam.utils.image import show_cam_on_image
|
| 13 |
+
from lime import lime_image
|
| 14 |
+
from skimage.segmentation import mark_boundaries
|
| 15 |
+
|
| 16 |
+
class ModelWrapper(nn.Module):
|
| 17 |
+
"""Wrapper to make AMSI-Net return only logits for explainability tools."""
|
| 18 |
+
def __init__(self, model):
|
| 19 |
+
super().__init__()
|
| 20 |
+
self.model = model
|
| 21 |
+
|
| 22 |
+
def forward(self, x):
|
| 23 |
+
outputs = self.model(x)
|
| 24 |
+
return outputs['logits']
|
| 25 |
+
|
| 26 |
+
def array_to_base64(img_array):
|
| 27 |
+
"""Convert numpy RGB array to base64 jpeg string."""
|
| 28 |
+
# Convert RGB to BGR for cv2
|
| 29 |
+
img_bgr = cv2.cvtColor(img_array, cv2.COLOR_RGB2BGR)
|
| 30 |
+
_, buffer = cv2.imencode('.jpg', img_bgr)
|
| 31 |
+
return base64.b64encode(buffer).decode('utf-8')
|
| 32 |
+
|
| 33 |
+
def get_saliency_map(model, image_tensor):
|
| 34 |
+
"""Generate vanilla Saliency map."""
|
| 35 |
+
x = image_tensor.clone()
|
| 36 |
+
x.requires_grad = True
|
| 37 |
+
model.eval()
|
| 38 |
+
|
| 39 |
+
# Forward pass
|
| 40 |
+
outputs = model(x)
|
| 41 |
+
logits = outputs['logits']
|
| 42 |
+
|
| 43 |
+
# Get top class
|
| 44 |
+
score, indices = torch.max(logits, 1)
|
| 45 |
+
|
| 46 |
+
# Backward pass
|
| 47 |
+
model.zero_grad()
|
| 48 |
+
score.backward()
|
| 49 |
+
|
| 50 |
+
# Saliency map = max absolute gradient across channels
|
| 51 |
+
saliency, _ = torch.max(x.grad.data.abs(), dim=1)
|
| 52 |
+
saliency = saliency.squeeze().cpu().numpy()
|
| 53 |
+
|
| 54 |
+
# Normalize
|
| 55 |
+
saliency = (saliency - saliency.min()) / (saliency.max() - saliency.min() + 1e-8)
|
| 56 |
+
|
| 57 |
+
# Apply colormap
|
| 58 |
+
saliency_heatmap = cv2.applyColorMap(np.uint8(255 * saliency), cv2.COLORMAP_JET)
|
| 59 |
+
saliency_heatmap = cv2.cvtColor(saliency_heatmap, cv2.COLOR_BGR2RGB)
|
| 60 |
+
saliency_heatmap = cv2.resize(saliency_heatmap, (224, 224))
|
| 61 |
+
|
| 62 |
+
return array_to_base64(saliency_heatmap)
|
| 63 |
+
|
| 64 |
+
def get_gradcam_maps(model, image_tensor, orig_image_np):
|
| 65 |
+
"""Generate GradCAM and GradCAM++."""
|
| 66 |
+
# Target the last convolutional layer of the ResNet backbone in AMSI-Net
|
| 67 |
+
# In AMSI-Net, backbone.layer4 is the final block of the ResNet parts
|
| 68 |
+
target_layers = [model.layer4[-1]]
|
| 69 |
+
|
| 70 |
+
wrapper = ModelWrapper(model)
|
| 71 |
+
|
| 72 |
+
# 1. GradCAM
|
| 73 |
+
cam_algo = GradCAM(model=wrapper, target_layers=target_layers)
|
| 74 |
+
grayscale_cam = cam_algo(input_tensor=image_tensor, targets=None)[0]
|
| 75 |
+
|
| 76 |
+
orig_img_norm = orig_image_np.astype(np.float32) / 255.0
|
| 77 |
+
cam_image = show_cam_on_image(orig_img_norm, grayscale_cam, use_rgb=True)
|
| 78 |
+
|
| 79 |
+
# 2. GradCAM++
|
| 80 |
+
cam_plus_algo = GradCAMPlusPlus(model=wrapper, target_layers=target_layers)
|
| 81 |
+
grayscale_cam_plus = cam_plus_algo(input_tensor=image_tensor, targets=None)[0]
|
| 82 |
+
cam_plus_image = show_cam_on_image(orig_img_norm, grayscale_cam_plus, use_rgb=True)
|
| 83 |
+
|
| 84 |
+
return array_to_base64(cam_image), array_to_base64(cam_plus_image)
|
| 85 |
+
|
| 86 |
+
def get_lime_map(model, preprocessor, image_bytes, orig_image_np, device):
|
| 87 |
+
"""Generate LIME explanation."""
|
| 88 |
+
explainer = lime_image.LimeImageExplainer()
|
| 89 |
+
|
| 90 |
+
def predict_fn(images):
|
| 91 |
+
tensors = []
|
| 92 |
+
for img_arr in images:
|
| 93 |
+
# Re-apply preprocessing to each LIME perturbation
|
| 94 |
+
img_pil = PIL.Image.fromarray(img_arr)
|
| 95 |
+
tensor = preprocessor.transform(img_pil).to(device)
|
| 96 |
+
tensors.append(tensor)
|
| 97 |
+
|
| 98 |
+
batch_tensor = torch.stack(tensors)
|
| 99 |
+
with torch.no_grad():
|
| 100 |
+
outputs = model(batch_tensor)
|
| 101 |
+
probs = torch.sigmoid(outputs['logits']) # Using sigmoid for multi-label
|
| 102 |
+
return probs.cpu().numpy()
|
| 103 |
+
|
| 104 |
+
explanation = explainer.explain_instance(
|
| 105 |
+
orig_image_np,
|
| 106 |
+
predict_fn,
|
| 107 |
+
top_labels=1,
|
| 108 |
+
hide_color=0,
|
| 109 |
+
num_samples=100
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
temp, mask = explanation.get_image_and_mask(
|
| 113 |
+
explanation.top_labels[0],
|
| 114 |
+
positive_only=True,
|
| 115 |
+
num_features=5,
|
| 116 |
+
hide_rest=False
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
img_boundary = mark_boundaries(temp / 255.0, mask)
|
| 120 |
+
img_boundary_uint8 = np.uint8(img_boundary * 255)
|
| 121 |
+
|
| 122 |
+
return array_to_base64(img_boundary_uint8)
|
| 123 |
+
|
| 124 |
+
def generate_all_explanations(model, preprocessor, image_tensor, image_bytes):
|
| 125 |
+
"""Generate all explainability maps."""
|
| 126 |
+
device = image_tensor.device
|
| 127 |
+
|
| 128 |
+
orig_pil = PIL.Image.open(BytesIO(image_bytes)).convert("RGB")
|
| 129 |
+
orig_pil = orig_pil.resize((224, 224))
|
| 130 |
+
orig_np = np.array(orig_pil)
|
| 131 |
+
|
| 132 |
+
result = {}
|
| 133 |
+
|
| 134 |
+
# Saliency
|
| 135 |
+
try:
|
| 136 |
+
saliency_b64 = get_saliency_map(model, image_tensor)
|
| 137 |
+
result["Saliency"] = f"data:image/jpeg;base64,{saliency_b64}"
|
| 138 |
+
except Exception as e:
|
| 139 |
+
print(f"Error generating Saliency: {e}")
|
| 140 |
+
|
| 141 |
+
# GradCAM
|
| 142 |
+
try:
|
| 143 |
+
gc_b64, gcp_b64 = get_gradcam_maps(model, image_tensor, orig_np)
|
| 144 |
+
result["GradCAM"] = f"data:image/jpeg;base64,{gc_b64}"
|
| 145 |
+
result["GradCAM++"] = f"data:image/jpeg;base64,{gcp_b64}"
|
| 146 |
+
except Exception as e:
|
| 147 |
+
print(f"Error generating GradCAM: {e}")
|
| 148 |
+
|
| 149 |
+
# LIME
|
| 150 |
+
try:
|
| 151 |
+
lime_b64 = get_lime_map(model, preprocessor, image_bytes, orig_np, device)
|
| 152 |
+
result["LIME"] = f"data:image/jpeg;base64,{lime_b64}"
|
| 153 |
+
except Exception as e:
|
| 154 |
+
print(f"Error generating LIME: {e}")
|
| 155 |
+
|
| 156 |
+
return result
|
backend/hf_space_readme.md
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: AMSI-Net LULC Recognition
|
| 3 |
+
emoji: 🛰️
|
| 4 |
+
colorFrom: indigo
|
| 5 |
+
colorTo: slate
|
| 6 |
+
sdk: docker
|
| 7 |
+
app_port: 8000
|
| 8 |
+
pinned: false
|
| 9 |
+
license: mit
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# AMSI-Net Multi-label LULC Recognition
|
| 13 |
+
|
| 14 |
+
This is a professional backend for Land Use & Land Cover (LULC) recognition using the AMSI-Net architecture.
|
| 15 |
+
|
| 16 |
+
## Features
|
| 17 |
+
- **Dataset**: MLRSNet (60 classes)
|
| 18 |
+
- **Architecture**: AMSI-Net (ResNet50 + Dynamic FPN + GAT)
|
| 19 |
+
- **Explainability**: GradCAM, GradCAM++, Saliency, LIME
|
| 20 |
+
- **API**: FastAPI with automatic Swagger docs
|
| 21 |
+
|
| 22 |
+
## API Documentation
|
| 23 |
+
The API documentation is available at `/docs` (Swagger UI).
|
| 24 |
+
|
| 25 |
+
## Deployment
|
| 26 |
+
This Space is built using Docker. The backend runs on port 8000 (mapped to HF default).
|
backend/model_loader.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import os
|
| 3 |
+
from models_arch import AMSINet
|
| 4 |
+
|
| 5 |
+
class ModelLoader:
|
| 6 |
+
"""Utility to manage loading and managing the AMSI-Net model."""
|
| 7 |
+
def __init__(self, model_path: str, num_labels: int):
|
| 8 |
+
self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
| 9 |
+
self.model_path = model_path
|
| 10 |
+
self.num_labels = num_labels
|
| 11 |
+
self.model = None
|
| 12 |
+
|
| 13 |
+
def load_model(self):
|
| 14 |
+
"""Load state_dict into the AMSI-Net architecture."""
|
| 15 |
+
if self.model is not None:
|
| 16 |
+
return self.model
|
| 17 |
+
|
| 18 |
+
print(f"Loading model from {self.model_path} on {self.device}...")
|
| 19 |
+
self.model = AMSINet(num_labels=self.num_labels, pretrained=False)
|
| 20 |
+
|
| 21 |
+
if not os.path.exists(self.model_path):
|
| 22 |
+
raise FileNotFoundError(f"Model file not found at {self.model_path}")
|
| 23 |
+
|
| 24 |
+
# map_location handles CPU/GPU switching
|
| 25 |
+
state_dict = torch.load(self.model_path, map_location=self.device)
|
| 26 |
+
|
| 27 |
+
# In case the state_dict is nested or saved differently
|
| 28 |
+
if 'model_state_dict' in state_dict:
|
| 29 |
+
state_dict = state_dict['model_state_dict']
|
| 30 |
+
|
| 31 |
+
self.model.load_state_dict(state_dict)
|
| 32 |
+
self.model.to(self.device)
|
| 33 |
+
self.model.eval()
|
| 34 |
+
print("Model loaded successfully.")
|
| 35 |
+
return self.model
|
| 36 |
+
|
| 37 |
+
def get_device(self):
|
| 38 |
+
return self.device
|
| 39 |
+
|
| 40 |
+
# Singleton-style accessor
|
| 41 |
+
_loader = None
|
| 42 |
+
|
| 43 |
+
def get_model_loader():
|
| 44 |
+
global _loader
|
| 45 |
+
if _loader is None:
|
| 46 |
+
# Default path based on project structure
|
| 47 |
+
base_dir = os.path.dirname(os.path.abspath(__file__))
|
| 48 |
+
model_path = os.path.join(os.path.dirname(base_dir), "model", "best_model.pth")
|
| 49 |
+
_loader = ModelLoader(model_path, num_labels=60)
|
| 50 |
+
return _loader
|
backend/models_arch.py
ADDED
|
@@ -0,0 +1,325 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
from torchvision.models import resnet50, ResNet50_Weights
|
| 5 |
+
|
| 6 |
+
class ScalePredictor(nn.Module):
|
| 7 |
+
"""Predicts importance weights for different scale levels"""
|
| 8 |
+
def __init__(self, in_channels, num_scales=4):
|
| 9 |
+
super().__init__()
|
| 10 |
+
self.num_scales = num_scales
|
| 11 |
+
self.global_pool = nn.AdaptiveAvgPool2d(1)
|
| 12 |
+
self.fc = nn.Sequential(
|
| 13 |
+
nn.Linear(in_channels, in_channels // 4),
|
| 14 |
+
nn.ReLU(),
|
| 15 |
+
nn.Linear(in_channels // 4, num_scales),
|
| 16 |
+
nn.Softmax(dim=1)
|
| 17 |
+
)
|
| 18 |
+
|
| 19 |
+
def forward(self, x):
|
| 20 |
+
b, c, h, w = x.shape
|
| 21 |
+
pooled = self.global_pool(x).view(b, c)
|
| 22 |
+
weights = self.fc(pooled)
|
| 23 |
+
return weights
|
| 24 |
+
|
| 25 |
+
class DynamicScaleAwareFPN(nn.Module):
|
| 26 |
+
"""Dynamic Scale-Aware Feature Pyramid"""
|
| 27 |
+
def __init__(self, in_channels_list, out_channels=256):
|
| 28 |
+
super().__init__()
|
| 29 |
+
self.num_scales = len(in_channels_list)
|
| 30 |
+
|
| 31 |
+
# Lateral connections
|
| 32 |
+
self.lateral_convs = nn.ModuleList([
|
| 33 |
+
nn.Conv2d(in_ch, out_channels, 1) for in_ch in in_channels_list
|
| 34 |
+
])
|
| 35 |
+
|
| 36 |
+
# Scale predictor
|
| 37 |
+
self.scale_predictor = ScalePredictor(in_channels_list[-1], self.num_scales)
|
| 38 |
+
|
| 39 |
+
def forward(self, features):
|
| 40 |
+
# features: list of feature maps from different scales
|
| 41 |
+
# Get scale weights
|
| 42 |
+
scale_weights = self.scale_predictor(features[-1])
|
| 43 |
+
|
| 44 |
+
# Apply lateral connections
|
| 45 |
+
lateral_features = [conv(feat) for conv, feat in zip(self.lateral_convs, features)]
|
| 46 |
+
|
| 47 |
+
# Weighted combination
|
| 48 |
+
# Upsample all to the largest size
|
| 49 |
+
target_size = lateral_features[0].shape[2:]
|
| 50 |
+
upsampled = []
|
| 51 |
+
for feat in lateral_features:
|
| 52 |
+
if feat.shape[2:] != target_size:
|
| 53 |
+
feat = F.interpolate(feat, size=target_size, mode='bilinear', align_corners=False)
|
| 54 |
+
upsampled.append(feat)
|
| 55 |
+
|
| 56 |
+
# Weight and combine
|
| 57 |
+
weighted_features = []
|
| 58 |
+
for i, feat in enumerate(upsampled):
|
| 59 |
+
weighted_features.append(feat * scale_weights[:, i].view(-1, 1, 1, 1))
|
| 60 |
+
|
| 61 |
+
output = sum(weighted_features)
|
| 62 |
+
return output, scale_weights
|
| 63 |
+
|
| 64 |
+
class GraphAttentionLayer(nn.Module):
|
| 65 |
+
"""Graph Attention for label correlation"""
|
| 66 |
+
def __init__(self, in_features, out_features, num_labels):
|
| 67 |
+
super().__init__()
|
| 68 |
+
self.in_features = in_features
|
| 69 |
+
self.out_features = out_features
|
| 70 |
+
self.num_labels = num_labels
|
| 71 |
+
|
| 72 |
+
self.W = nn.Linear(in_features, out_features, bias=False)
|
| 73 |
+
self.a = nn.Linear(2 * out_features, 1, bias=False)
|
| 74 |
+
self.leakyrelu = nn.LeakyReLU(0.2)
|
| 75 |
+
|
| 76 |
+
def forward(self, x, adj=None):
|
| 77 |
+
# x: [batch, num_labels, in_features]
|
| 78 |
+
batch_size = x.size(0)
|
| 79 |
+
|
| 80 |
+
Wx = self.W(x) # [batch, num_labels, out_features]
|
| 81 |
+
|
| 82 |
+
# Compute attention coefficients
|
| 83 |
+
a_input = self._prepare_attentional_mechanism_input(Wx)
|
| 84 |
+
# a_input: [batch, num_labels, num_labels, 2*out_features]
|
| 85 |
+
|
| 86 |
+
e = self.leakyrelu(self.a(a_input)) # [batch, num_labels, num_labels, 1]
|
| 87 |
+
e = e.squeeze(-1) # [batch, num_labels, num_labels]
|
| 88 |
+
|
| 89 |
+
attention = F.softmax(e, dim=2) # [batch, num_labels, num_labels]
|
| 90 |
+
|
| 91 |
+
# Apply attention
|
| 92 |
+
h_prime = torch.bmm(attention, Wx) # [batch, num_labels, out_features]
|
| 93 |
+
|
| 94 |
+
return h_prime, attention
|
| 95 |
+
|
| 96 |
+
def _prepare_attentional_mechanism_input(self, Wx):
|
| 97 |
+
num_labels = Wx.size(1)
|
| 98 |
+
|
| 99 |
+
# Wx: [batch, num_labels, out_features]
|
| 100 |
+
# We need to create pairs: [batch, num_labels, num_labels, 2*out_features]
|
| 101 |
+
|
| 102 |
+
# Repeat for all pairs
|
| 103 |
+
Wx_i = Wx.unsqueeze(2).repeat(1, 1, num_labels, 1) # [batch, num_labels, num_labels, out_features]
|
| 104 |
+
Wx_j = Wx.unsqueeze(1).repeat(1, num_labels, 1, 1) # [batch, num_labels, num_labels, out_features]
|
| 105 |
+
|
| 106 |
+
# Concatenate pairs
|
| 107 |
+
all_combinations = torch.cat([Wx_i, Wx_j], dim=3) # [batch, num_labels, num_labels, 2*out_features]
|
| 108 |
+
|
| 109 |
+
return all_combinations
|
| 110 |
+
|
| 111 |
+
class CrossLabelSemanticGraph(nn.Module):
|
| 112 |
+
"""Cross-Label Semantic Correlation Graph"""
|
| 113 |
+
def __init__(self, feature_dim, num_labels, num_heads=4):
|
| 114 |
+
super().__init__()
|
| 115 |
+
self.num_labels = num_labels
|
| 116 |
+
self.num_heads = num_heads
|
| 117 |
+
|
| 118 |
+
self.gat_layers = nn.ModuleList([
|
| 119 |
+
GraphAttentionLayer(feature_dim, feature_dim, num_labels)
|
| 120 |
+
for _ in range(num_heads)
|
| 121 |
+
])
|
| 122 |
+
|
| 123 |
+
self.output_projection = nn.Linear(feature_dim * num_heads, feature_dim)
|
| 124 |
+
|
| 125 |
+
def forward(self, label_features):
|
| 126 |
+
# label_features: [batch, num_labels, feature_dim]
|
| 127 |
+
|
| 128 |
+
multi_head_outputs = []
|
| 129 |
+
attentions = []
|
| 130 |
+
|
| 131 |
+
for gat in self.gat_layers:
|
| 132 |
+
out, att = gat(label_features)
|
| 133 |
+
multi_head_outputs.append(out)
|
| 134 |
+
attentions.append(att)
|
| 135 |
+
|
| 136 |
+
# Concatenate multi-head outputs
|
| 137 |
+
concatenated = torch.cat(multi_head_outputs, dim=2)
|
| 138 |
+
output = self.output_projection(concatenated)
|
| 139 |
+
|
| 140 |
+
return output, attentions
|
| 141 |
+
|
| 142 |
+
class DeformableAttention(nn.Module):
|
| 143 |
+
"""Memory-efficient Deformable Attention with spatial downsampling"""
|
| 144 |
+
def __init__(self, dim, num_heads=8, max_tokens=196):
|
| 145 |
+
super().__init__()
|
| 146 |
+
self.num_heads = num_heads
|
| 147 |
+
self.scale = (dim // num_heads) ** -0.5
|
| 148 |
+
self.max_tokens = max_tokens # Reduce to 14x14 = 196 tokens max
|
| 149 |
+
|
| 150 |
+
self.qkv = nn.Linear(dim, dim * 3)
|
| 151 |
+
self.proj = nn.Linear(dim, dim)
|
| 152 |
+
|
| 153 |
+
def forward(self, x):
|
| 154 |
+
B, N, C = x.shape
|
| 155 |
+
|
| 156 |
+
# Spatially downsample if needed to reduce memory
|
| 157 |
+
if N > self.max_tokens:
|
| 158 |
+
# Reshape to 2D, apply adaptive pooling, reshape back
|
| 159 |
+
# Assuming square spatial dimensions
|
| 160 |
+
h = w = int(N ** 0.5)
|
| 161 |
+
target_h = target_w = int(self.max_tokens ** 0.5)
|
| 162 |
+
|
| 163 |
+
x_2d = x.transpose(1, 2).reshape(B, C, h, w)
|
| 164 |
+
x_2d = F.adaptive_avg_pool2d(x_2d, (target_h, target_w))
|
| 165 |
+
x = x_2d.reshape(B, C, -1).transpose(1, 2)
|
| 166 |
+
N = x.shape[1]
|
| 167 |
+
|
| 168 |
+
qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
|
| 169 |
+
q, k, v = qkv[0], qkv[1], qkv[2]
|
| 170 |
+
|
| 171 |
+
attn = (q @ k.transpose(-2, -1)) * self.scale
|
| 172 |
+
attn = attn.softmax(dim=-1)
|
| 173 |
+
|
| 174 |
+
x = (attn @ v).transpose(1, 2).reshape(B, N, C)
|
| 175 |
+
x = self.proj(x)
|
| 176 |
+
return x
|
| 177 |
+
|
| 178 |
+
class SpectralSpatialFusion(nn.Module):
|
| 179 |
+
"""Spectral-Spatial Decoupled Multi-Head Classification"""
|
| 180 |
+
def __init__(self, feature_dim, num_labels):
|
| 181 |
+
super().__init__()
|
| 182 |
+
|
| 183 |
+
# Spectral branch
|
| 184 |
+
self.spectral_branch = nn.Sequential(
|
| 185 |
+
nn.Linear(feature_dim, feature_dim // 2),
|
| 186 |
+
nn.ReLU(),
|
| 187 |
+
nn.Dropout(0.3),
|
| 188 |
+
nn.Linear(feature_dim // 2, num_labels)
|
| 189 |
+
)
|
| 190 |
+
|
| 191 |
+
# Spatial branch
|
| 192 |
+
self.spatial_branch = nn.Sequential(
|
| 193 |
+
nn.Linear(feature_dim, feature_dim // 2),
|
| 194 |
+
nn.ReLU(),
|
| 195 |
+
nn.Dropout(0.3),
|
| 196 |
+
nn.Linear(feature_dim // 2, num_labels)
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
# Fusion gate
|
| 200 |
+
self.fusion_gate = nn.Sequential(
|
| 201 |
+
nn.Linear(feature_dim, num_labels),
|
| 202 |
+
nn.Sigmoid()
|
| 203 |
+
)
|
| 204 |
+
|
| 205 |
+
def forward(self, features):
|
| 206 |
+
spectral_out = self.spectral_branch(features)
|
| 207 |
+
spatial_out = self.spatial_branch(features)
|
| 208 |
+
|
| 209 |
+
gate = self.fusion_gate(features)
|
| 210 |
+
|
| 211 |
+
fused_output = gate * spectral_out + (1 - gate) * spatial_out
|
| 212 |
+
|
| 213 |
+
return fused_output, spectral_out, spatial_out, gate
|
| 214 |
+
|
| 215 |
+
class UncertaintyHead(nn.Module):
|
| 216 |
+
"""Evidential uncertainty estimation"""
|
| 217 |
+
def __init__(self, feature_dim, num_labels):
|
| 218 |
+
super().__init__()
|
| 219 |
+
self.num_labels = num_labels
|
| 220 |
+
self.fc = nn.Linear(feature_dim, num_labels)
|
| 221 |
+
|
| 222 |
+
def forward(self, x):
|
| 223 |
+
evidence = F.softplus(self.fc(x))
|
| 224 |
+
alpha = evidence + 1
|
| 225 |
+
uncertainty = self.num_labels / torch.sum(alpha, dim=1, keepdim=True)
|
| 226 |
+
return evidence, uncertainty
|
| 227 |
+
|
| 228 |
+
class AMSINet(nn.Module):
|
| 229 |
+
def __init__(self, num_labels, pretrained=True):
|
| 230 |
+
super().__init__()
|
| 231 |
+
self.num_labels = num_labels
|
| 232 |
+
|
| 233 |
+
# Backbone: ResNet50 as feature extractor
|
| 234 |
+
if pretrained:
|
| 235 |
+
weights = ResNet50_Weights.DEFAULT
|
| 236 |
+
backbone = resnet50(weights=weights)
|
| 237 |
+
else:
|
| 238 |
+
backbone = resnet50()
|
| 239 |
+
|
| 240 |
+
self.conv1 = backbone.conv1
|
| 241 |
+
self.bn1 = backbone.bn1
|
| 242 |
+
self.relu = backbone.relu
|
| 243 |
+
self.maxpool = backbone.maxpool
|
| 244 |
+
|
| 245 |
+
self.layer1 = backbone.layer1 # 256 channels
|
| 246 |
+
self.layer2 = backbone.layer2 # 512 channels
|
| 247 |
+
self.layer3 = backbone.layer3 # 1024 channels
|
| 248 |
+
self.layer4 = backbone.layer4 # 2048 channels
|
| 249 |
+
|
| 250 |
+
# Dynamic Scale-Aware FPN
|
| 251 |
+
self.dsafp = DynamicScaleAwareFPN([256, 512, 1024, 2048], out_channels=256)
|
| 252 |
+
|
| 253 |
+
# Deformable Attention
|
| 254 |
+
self.deformable_attn = DeformableAttention(dim=256, num_heads=8)
|
| 255 |
+
|
| 256 |
+
# Global pooling
|
| 257 |
+
self.global_pool = nn.AdaptiveAvgPool2d(1)
|
| 258 |
+
|
| 259 |
+
# Cross-Label Semantic Correlation Graph
|
| 260 |
+
self.clscg = CrossLabelSemanticGraph(feature_dim=256, num_labels=num_labels, num_heads=4)
|
| 261 |
+
|
| 262 |
+
# Create label embeddings
|
| 263 |
+
self.label_embeddings = nn.Parameter(torch.randn(num_labels, 256))
|
| 264 |
+
|
| 265 |
+
# Spectral-Spatial Fusion
|
| 266 |
+
self.spectral_spatial_fusion = SpectralSpatialFusion(256, num_labels)
|
| 267 |
+
|
| 268 |
+
# Uncertainty Head
|
| 269 |
+
self.uncertainty_head = UncertaintyHead(256, num_labels)
|
| 270 |
+
|
| 271 |
+
def forward(self, x):
|
| 272 |
+
# Backbone feature extraction
|
| 273 |
+
x = self.conv1(x)
|
| 274 |
+
x = self.bn1(x)
|
| 275 |
+
x = self.relu(x)
|
| 276 |
+
x = self.maxpool(x)
|
| 277 |
+
|
| 278 |
+
# Multi-scale features
|
| 279 |
+
c2 = self.layer1(x)
|
| 280 |
+
c3 = self.layer2(c2)
|
| 281 |
+
c4 = self.layer3(c3)
|
| 282 |
+
c5 = self.layer4(c4)
|
| 283 |
+
|
| 284 |
+
# Dynamic FPN
|
| 285 |
+
fpn_features, scale_weights = self.dsafp([c2, c3, c4, c5])
|
| 286 |
+
|
| 287 |
+
# Flatten for attention
|
| 288 |
+
b, c, h, w = fpn_features.shape
|
| 289 |
+
fpn_flat = fpn_features.view(b, c, -1).permute(0, 2, 1) # [B, H*W, C]
|
| 290 |
+
|
| 291 |
+
# Deformable attention
|
| 292 |
+
attn_features = self.deformable_attn(fpn_flat)
|
| 293 |
+
|
| 294 |
+
# Global pooling
|
| 295 |
+
attn_features = attn_features.mean(dim=1) # [B, C]
|
| 296 |
+
|
| 297 |
+
# Prepare label-specific features
|
| 298 |
+
batch_size = x.size(0)
|
| 299 |
+
label_features = self.label_embeddings.unsqueeze(0).expand(batch_size, -1, -1)
|
| 300 |
+
|
| 301 |
+
# Add global context to each label
|
| 302 |
+
global_context = attn_features.unsqueeze(1).expand(-1, self.num_labels, -1)
|
| 303 |
+
label_features = label_features + global_context
|
| 304 |
+
|
| 305 |
+
# Cross-Label Graph reasoning
|
| 306 |
+
graph_features, graph_attentions = self.clscg(label_features)
|
| 307 |
+
|
| 308 |
+
# Aggregate graph features
|
| 309 |
+
aggregated_features = graph_features.mean(dim=1) # [B, C]
|
| 310 |
+
|
| 311 |
+
# Spectral-Spatial Fusion
|
| 312 |
+
output, spectral_out, spatial_out, fusion_gate = self.spectral_spatial_fusion(aggregated_features)
|
| 313 |
+
|
| 314 |
+
# Uncertainty estimation
|
| 315 |
+
evidence, uncertainty = self.uncertainty_head(aggregated_features)
|
| 316 |
+
|
| 317 |
+
return {
|
| 318 |
+
'logits': output,
|
| 319 |
+
'spectral_logits': spectral_out,
|
| 320 |
+
'spatial_logits': spatial_out,
|
| 321 |
+
'fusion_gate': fusion_gate,
|
| 322 |
+
'uncertainty': uncertainty,
|
| 323 |
+
'scale_weights': scale_weights,
|
| 324 |
+
'graph_attentions': graph_attentions
|
| 325 |
+
}
|
backend/preprocessing.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from torchvision import transforms
|
| 3 |
+
from PIL import Image
|
| 4 |
+
import io
|
| 5 |
+
|
| 6 |
+
class Preprocessor:
|
| 7 |
+
"""Handles image preprocessing for LULC models."""
|
| 8 |
+
def __init__(self):
|
| 9 |
+
self.transform = transforms.Compose([
|
| 10 |
+
transforms.Resize((224, 224)),
|
| 11 |
+
transforms.ToTensor(),
|
| 12 |
+
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
|
| 13 |
+
])
|
| 14 |
+
|
| 15 |
+
def preprocess(self, image_bytes: bytes) -> torch.Tensor:
|
| 16 |
+
"""Convert image bytes to a normalized tensor."""
|
| 17 |
+
image = Image.open(io.BytesIO(image_bytes)).convert('RGB')
|
| 18 |
+
tensor = self.transform(image).unsqueeze(0) # Add batch dimension
|
| 19 |
+
return tensor
|
| 20 |
+
|
| 21 |
+
def get_image_info(self, image_bytes: bytes) -> dict:
|
| 22 |
+
"""Extract metadata from image bytes."""
|
| 23 |
+
image = Image.open(io.BytesIO(image_bytes))
|
| 24 |
+
return {
|
| 25 |
+
"width": image.width,
|
| 26 |
+
"height": image.height,
|
| 27 |
+
"format": image.format,
|
| 28 |
+
"mode": image.mode
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
def get_preprocessor():
|
| 32 |
+
return Preprocessor()
|
backend/requirements.txt
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi>=0.104.1
|
| 2 |
+
uvicorn>=0.24.0
|
| 3 |
+
torch>=2.1.1
|
| 4 |
+
torchvision>=0.16.1
|
| 5 |
+
pillow>=10.1.0
|
| 6 |
+
python-multipart>=0.0.6
|
| 7 |
+
pydantic>=2.5.0
|
| 8 |
+
numpy>=1.26.0
|
| 9 |
+
opencv-python-headless>=4.8.1.78
|
| 10 |
+
matplotlib>=3.8.1
|
| 11 |
+
scikit-image>=0.22.0
|
| 12 |
+
scikit-learn>=1.3.2
|
| 13 |
+
lime>=0.2.0.1
|
| 14 |
+
ttach>=0.0.3
|
| 15 |
+
grad-cam>=1.5.0
|
| 16 |
+
pandas>=2.1.3
|
| 17 |
+
tqdm>=4.66.1
|
backend/verify_model.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Verification script for AMSI-Net Model loading and architecture."""
|
| 2 |
+
|
| 3 |
+
import sys
|
| 4 |
+
import os
|
| 5 |
+
import torch
|
| 6 |
+
|
| 7 |
+
# Add current directory to path
|
| 8 |
+
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
| 9 |
+
|
| 10 |
+
try:
|
| 11 |
+
from models_arch import AMSINet
|
| 12 |
+
from model_loader import get_model_loader
|
| 13 |
+
from class_mappings import get_all_labels
|
| 14 |
+
|
| 15 |
+
print("--- AMSI-Net Backend Verification ---")
|
| 16 |
+
|
| 17 |
+
# 1. Check Labels
|
| 18 |
+
labels = get_all_labels()
|
| 19 |
+
print(f"✅ Class Mappings: Found {len(labels)} labels.")
|
| 20 |
+
|
| 21 |
+
# 2. Check Device
|
| 22 |
+
loader = get_model_loader()
|
| 23 |
+
device = loader.get_device()
|
| 24 |
+
print(f"ℹ️ Device: Using {device}")
|
| 25 |
+
|
| 26 |
+
# 3. Load Model
|
| 27 |
+
print("⏳ Loading model state_dict (this may take a moment)...")
|
| 28 |
+
model = loader.load_model()
|
| 29 |
+
print("✅ Model: Architecture initialized and weights loaded.")
|
| 30 |
+
|
| 31 |
+
# 4. Dummy Inference
|
| 32 |
+
print("🧪 Running dummy inference check...")
|
| 33 |
+
dummy_input = torch.randn(1, 3, 224, 224).to(device)
|
| 34 |
+
with torch.no_grad():
|
| 35 |
+
outputs = model(dummy_input)
|
| 36 |
+
|
| 37 |
+
if 'logits' in outputs and outputs['logits'].shape == (1, 60):
|
| 38 |
+
print("✅ Inference: Model produced correct output shape (1, 60).")
|
| 39 |
+
else:
|
| 40 |
+
print("❌ Inference: Unexpected output shape.")
|
| 41 |
+
|
| 42 |
+
print("\n🚀 Everything looks good! You can now start the API with:")
|
| 43 |
+
print(" python backend/app.py")
|
| 44 |
+
|
| 45 |
+
except ImportError as e:
|
| 46 |
+
print(f"\n❌ Dependency Error: {e}")
|
| 47 |
+
print(" Please run: pip install -r backend/requirements.txt")
|
| 48 |
+
except Exception as e:
|
| 49 |
+
print(f"\n❌ Error during verification: {e}")
|
| 50 |
+
import traceback
|
| 51 |
+
traceback.print_exc()
|
model/best_model.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7f42bcf023fc938944d9831b25c84e97ac0a8d1b1cfa14a0b22269d3be8b61c7
|
| 3 |
+
size 106175286
|
pyproject.toml
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "lulc-backend"
|
| 3 |
+
version = "0.1.0"
|
| 4 |
+
description = "Add your description here"
|
| 5 |
+
readme = "README.md"
|
| 6 |
+
requires-python = ">=3.13"
|
| 7 |
+
dependencies = [
|
| 8 |
+
"fastapi>=0.104.1",
|
| 9 |
+
"grad-cam>=1.5.0",
|
| 10 |
+
"lime>=0.2.0.1",
|
| 11 |
+
"matplotlib>=3.8.1",
|
| 12 |
+
"numpy>=1.26.0",
|
| 13 |
+
"opencv-python-headless>=4.8.1.78",
|
| 14 |
+
"pandas>=2.1.3",
|
| 15 |
+
"pillow>=10.1.0",
|
| 16 |
+
"pydantic>=2.5.0",
|
| 17 |
+
"python-multipart>=0.0.6",
|
| 18 |
+
"scikit-image>=0.22.0",
|
| 19 |
+
"scikit-learn>=1.3.2",
|
| 20 |
+
"torch>=2.1.1",
|
| 21 |
+
"torchvision>=0.16.1",
|
| 22 |
+
"tqdm>=4.66.1",
|
| 23 |
+
"ttach>=0.0.3",
|
| 24 |
+
"uvicorn>=0.24.0",
|
| 25 |
+
]
|
requirements.txt
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi>=0.104.1
|
| 2 |
+
uvicorn>=0.24.0
|
| 3 |
+
torch>=2.1.1
|
| 4 |
+
torchvision>=0.16.1
|
| 5 |
+
pillow>=10.1.0
|
| 6 |
+
python-multipart>=0.0.6
|
| 7 |
+
pydantic>=2.5.0
|
| 8 |
+
numpy>=1.26.0
|
| 9 |
+
opencv-python-headless>=4.8.1.78
|
| 10 |
+
matplotlib>=3.8.1
|
| 11 |
+
scikit-image>=0.22.0
|
| 12 |
+
scikit-learn>=1.3.2
|
| 13 |
+
lime>=0.2.0.1
|
| 14 |
+
ttach>=0.0.3
|
| 15 |
+
grad-cam>=1.5.0
|
| 16 |
+
pandas>=2.1.3
|
| 17 |
+
tqdm>=4.66.1
|