Spaces:
Paused
Paused
File size: 6,435 Bytes
a34a6d1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 | """
AI Super-Resolution Engine specialized for 2D illustrations, anime, and character art.
Powered by ONNX Runtime with Waifu2x-CUNet architecture (fast CPU/GPU inference).
Preserves RGBA transparency, removes compression artifacts, and reconstructs crisp line art.
"""
import os
from pathlib import Path
from typing import Optional, Tuple
import numpy as np
from PIL import Image
import onnxruntime as ort
class AI2DUpscaler:
def __init__(self, models_dir: Optional[str | Path] = None):
if models_dir is None:
models_dir = Path(__file__).resolve().parent.parent.parent / "models"
self.models_dir = Path(models_dir)
self._scale2x_path = self.models_dir / "waifu2x_art_scale2x.onnx"
self._noise_scale2x_path = self.models_dir / "waifu2x_art_noise1_scale2x.onnx"
self._sessions = {}
self._providers = self._get_best_providers()
def _get_best_providers(self):
available = ort.get_available_providers()
# Prefer GPU if available (CUDA, DirectML, CoreML), fall back to CPU
providers = []
if "DmlExecutionProvider" in available:
providers.append("DmlExecutionProvider")
if "CUDAExecutionProvider" in available:
providers.append("CUDAExecutionProvider")
providers.append("CPUExecutionProvider")
return providers
def _get_session(self, model_path: Path) -> ort.InferenceSession:
key = str(model_path)
if key not in self._sessions:
if not model_path.exists():
raise FileNotFoundError(f"Model file not found: {model_path}")
self._sessions[key] = ort.InferenceSession(
str(model_path), providers=self._providers
)
return self._sessions[key]
def _upscale_rgb_tile(
self, session: ort.InferenceSession, rgb_float: np.ndarray
) -> np.ndarray:
"""
Runs inference on an RGB float32 array of shape (H, W, 3) in [0, 1].
Applies 18px reflection padding for CUNet boundary handling.
Returns upscaled array (2*H, 2*W, 3) in [0, 1].
"""
h, w, c = rgb_float.shape
# Reorder to (1, 3, H, W)
chw = np.transpose(rgb_float, (2, 0, 1))[np.newaxis, ...]
# Reflection pad 18px on all borders
padded = np.pad(chw, ((0, 0), (0, 0), (18, 18), (18, 18)), mode="reflect")
# Run ONNX inference
out = session.run(None, {"x": padded})[0] # shape (1, 3, 2*H, 2*W)
# Transpose back to (2*H, 2*W, 3) and clip to [0, 1]
out_hwc = np.transpose(out[0], (1, 2, 0))
return np.clip(out_hwc, 0.0, 1.0)
def _upscale_large_rgb(
self, session: ort.InferenceSession, rgb_float: np.ndarray, tile_size: int = 512, tile_pad: int = 16
) -> np.ndarray:
"""
Tiled inference with overlap to handle arbitrary image sizes without running out of RAM.
"""
h, w, _ = rgb_float.shape
if h <= tile_size and w <= tile_size:
return self._upscale_rgb_tile(session, rgb_float)
out_h, out_w = h * 2, w * 2
out_img = np.zeros((out_h, out_w, 3), dtype=np.float32)
for y in range(0, h, tile_size):
for x in range(0, w, tile_size):
# Calculate tile boundaries with padding
y_start = max(0, y - tile_pad)
y_end = min(h, y + tile_size + tile_pad)
x_start = max(0, x - tile_pad)
x_end = min(w, x + tile_size + tile_pad)
tile = rgb_float[y_start:y_end, x_start:x_end]
tile_out = self._upscale_rgb_tile(session, tile)
# Calculate placement in destination
top_cut = (y - y_start) * 2
bottom_cut = top_cut + min(tile_size, h - y) * 2
left_cut = (x - x_start) * 2
right_cut = left_cut + min(tile_size, w - x) * 2
dest_y_start = y * 2
dest_y_end = dest_y_start + (bottom_cut - top_cut)
dest_x_start = x * 2
dest_x_end = dest_x_start + (right_cut - left_cut)
out_img[dest_y_start:dest_y_end, dest_x_start:dest_x_end] = tile_out[
top_cut:bottom_cut, left_cut:right_cut
]
return out_img
def process_pil_image(
self,
img: Image.Image,
scale: int = 4,
denoise: bool = True,
) -> Image.Image:
"""
Upscales PIL image by 2x or 4x with 2D character edge reconstruction.
Preserves alpha channel for transparent PNGs.
"""
has_alpha = img.mode in ("RGBA", "LA") or (
img.mode == "P" and "transparency" in img.info
)
if has_alpha:
img_rgba = img.convert("RGBA")
alpha = img_rgba.split()[-1]
rgb_img = img_rgba.convert("RGB")
else:
alpha = None
rgb_img = img.convert("RGB")
# Pick appropriate model
model_path = self._noise_scale2x_path if denoise else self._scale2x_path
session = self._get_session(model_path)
rgb_arr = np.array(rgb_img, dtype=np.float32) / 255.0
# Pass 1: 2x upscale
scaled_rgb = self._upscale_large_rgb(session, rgb_arr)
# Pass 2: If 4x requested, run second 2x pass
if scale >= 4:
clean_session = self._get_session(self._scale2x_path)
scaled_rgb = self._upscale_large_rgb(clean_session, scaled_rgb)
# Convert back to PIL Image (uint8)
scaled_uint8 = (scaled_rgb * 255.0).round().astype(np.uint8)
out_pil = Image.fromarray(scaled_uint8, mode="RGB")
# Handle alpha channel resizing with clean bicubic/box sampling
if alpha is not None:
target_size = out_pil.size
alpha_upscaled = alpha.resize(target_size, resample=Image.Resampling.LANCZOS)
out_pil.putalpha(alpha_upscaled)
return out_pil
def upscale_file(
self,
input_path: str | Path,
output_path: str | Path,
scale: int = 4,
denoise: bool = True,
) -> str:
with Image.open(input_path) as img:
out_img = self.process_pil_image(img, scale=scale, denoise=denoise)
output_str = str(Path(output_path).resolve())
out_img.save(output_str, format="PNG", optimize=True)
return output_str
|