Image-Enhancer / engine /upscaler.py
Ribakoo's picture
Fix module structure: relocate engine package to resolve app namespace
1a616a5
Raw History Blame Contribute Delete
6.44 kB
"""
AI Super-Resolution Engine specialized for 2D illustrations, anime, and character art.
Powered by ONNX Runtime with Waifu2x-CUNet architecture (fast CPU/GPU inference).
Preserves RGBA transparency, removes compression artifacts, and reconstructs crisp line art.
"""
import os
from pathlib import Path
from typing import Optional, Tuple
import numpy as np
from PIL import Image
import onnxruntime as ort
class AI2DUpscaler:
def __init__(self, models_dir: Optional[str | Path] = None):
if models_dir is None:
models_dir = Path(__file__).resolve().parent.parent.parent / "models"
self.models_dir = Path(models_dir)
self._scale2x_path = self.models_dir / "waifu2x_art_scale2x.onnx"
self._noise_scale2x_path = self.models_dir / "waifu2x_art_noise1_scale2x.onnx"
self._sessions = {}
self._providers = self._get_best_providers()
def _get_best_providers(self):
available = ort.get_available_providers()
# Prefer GPU if available (CUDA, DirectML, CoreML), fall back to CPU
providers = []
if "DmlExecutionProvider" in available:
providers.append("DmlExecutionProvider")
if "CUDAExecutionProvider" in available:
providers.append("CUDAExecutionProvider")
providers.append("CPUExecutionProvider")
return providers
def _get_session(self, model_path: Path) -> ort.InferenceSession:
key = str(model_path)
if key not in self._sessions:
if not model_path.exists():
raise FileNotFoundError(f"Model file not found: {model_path}")
self._sessions[key] = ort.InferenceSession(
str(model_path), providers=self._providers
)
return self._sessions[key]
def _upscale_rgb_tile(
self, session: ort.InferenceSession, rgb_float: np.ndarray
) -> np.ndarray:
"""
Runs inference on an RGB float32 array of shape (H, W, 3) in [0, 1].
Applies 18px reflection padding for CUNet boundary handling.
Returns upscaled array (2*H, 2*W, 3) in [0, 1].
"""
h, w, c = rgb_float.shape
# Reorder to (1, 3, H, W)
chw = np.transpose(rgb_float, (2, 0, 1))[np.newaxis, ...]
# Reflection pad 18px on all borders
padded = np.pad(chw, ((0, 0), (0, 0), (18, 18), (18, 18)), mode="reflect")
# Run ONNX inference
out = session.run(None, {"x": padded})[0] # shape (1, 3, 2*H, 2*W)
# Transpose back to (2*H, 2*W, 3) and clip to [0, 1]
out_hwc = np.transpose(out[0], (1, 2, 0))
return np.clip(out_hwc, 0.0, 1.0)
def _upscale_large_rgb(
self, session: ort.InferenceSession, rgb_float: np.ndarray, tile_size: int = 512, tile_pad: int = 16
) -> np.ndarray:
"""
Tiled inference with overlap to handle arbitrary image sizes without running out of RAM.
"""
h, w, _ = rgb_float.shape
if h <= tile_size and w <= tile_size:
return self._upscale_rgb_tile(session, rgb_float)
out_h, out_w = h * 2, w * 2
out_img = np.zeros((out_h, out_w, 3), dtype=np.float32)
for y in range(0, h, tile_size):
for x in range(0, w, tile_size):
# Calculate tile boundaries with padding
y_start = max(0, y - tile_pad)
y_end = min(h, y + tile_size + tile_pad)
x_start = max(0, x - tile_pad)
x_end = min(w, x + tile_size + tile_pad)
tile = rgb_float[y_start:y_end, x_start:x_end]
tile_out = self._upscale_rgb_tile(session, tile)
# Calculate placement in destination
top_cut = (y - y_start) * 2
bottom_cut = top_cut + min(tile_size, h - y) * 2
left_cut = (x - x_start) * 2
right_cut = left_cut + min(tile_size, w - x) * 2
dest_y_start = y * 2
dest_y_end = dest_y_start + (bottom_cut - top_cut)
dest_x_start = x * 2
dest_x_end = dest_x_start + (right_cut - left_cut)
out_img[dest_y_start:dest_y_end, dest_x_start:dest_x_end] = tile_out[
top_cut:bottom_cut, left_cut:right_cut
]
return out_img
def process_pil_image(
self,
img: Image.Image,
scale: int = 4,
denoise: bool = True,
) -> Image.Image:
"""
Upscales PIL image by 2x or 4x with 2D character edge reconstruction.
Preserves alpha channel for transparent PNGs.
"""
has_alpha = img.mode in ("RGBA", "LA") or (
img.mode == "P" and "transparency" in img.info
)
if has_alpha:
img_rgba = img.convert("RGBA")
alpha = img_rgba.split()[-1]
rgb_img = img_rgba.convert("RGB")
else:
alpha = None
rgb_img = img.convert("RGB")
# Pick appropriate model
model_path = self._noise_scale2x_path if denoise else self._scale2x_path
session = self._get_session(model_path)
rgb_arr = np.array(rgb_img, dtype=np.float32) / 255.0
# Pass 1: 2x upscale
scaled_rgb = self._upscale_large_rgb(session, rgb_arr)
# Pass 2: If 4x requested, run second 2x pass
if scale >= 4:
clean_session = self._get_session(self._scale2x_path)
scaled_rgb = self._upscale_large_rgb(clean_session, scaled_rgb)
# Convert back to PIL Image (uint8)
scaled_uint8 = (scaled_rgb * 255.0).round().astype(np.uint8)
out_pil = Image.fromarray(scaled_uint8, mode="RGB")
# Handle alpha channel resizing with clean bicubic/box sampling
if alpha is not None:
target_size = out_pil.size
alpha_upscaled = alpha.resize(target_size, resample=Image.Resampling.LANCZOS)
out_pil.putalpha(alpha_upscaled)
return out_pil
def upscale_file(
self,
input_path: str | Path,
output_path: str | Path,
scale: int = 4,
denoise: bool = True,
) -> str:
with Image.open(input_path) as img:
out_img = self.process_pil_image(img, scale=scale, denoise=denoise)
output_str = str(Path(output_path).resolve())
out_img.save(output_str, format="PNG", optimize=True)
return output_str