File size: 6,435 Bytes
a34a6d1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
"""
AI Super-Resolution Engine specialized for 2D illustrations, anime, and character art.
Powered by ONNX Runtime with Waifu2x-CUNet architecture (fast CPU/GPU inference).
Preserves RGBA transparency, removes compression artifacts, and reconstructs crisp line art.
"""

import os
from pathlib import Path
from typing import Optional, Tuple
import numpy as np
from PIL import Image
import onnxruntime as ort


class AI2DUpscaler:
    def __init__(self, models_dir: Optional[str | Path] = None):
        if models_dir is None:
            models_dir = Path(__file__).resolve().parent.parent.parent / "models"
        self.models_dir = Path(models_dir)

        self._scale2x_path = self.models_dir / "waifu2x_art_scale2x.onnx"
        self._noise_scale2x_path = self.models_dir / "waifu2x_art_noise1_scale2x.onnx"

        self._sessions = {}
        self._providers = self._get_best_providers()

    def _get_best_providers(self):
        available = ort.get_available_providers()
        # Prefer GPU if available (CUDA, DirectML, CoreML), fall back to CPU
        providers = []
        if "DmlExecutionProvider" in available:
            providers.append("DmlExecutionProvider")
        if "CUDAExecutionProvider" in available:
            providers.append("CUDAExecutionProvider")
        providers.append("CPUExecutionProvider")
        return providers

    def _get_session(self, model_path: Path) -> ort.InferenceSession:
        key = str(model_path)
        if key not in self._sessions:
            if not model_path.exists():
                raise FileNotFoundError(f"Model file not found: {model_path}")
            self._sessions[key] = ort.InferenceSession(
                str(model_path), providers=self._providers
            )
        return self._sessions[key]

    def _upscale_rgb_tile(
        self, session: ort.InferenceSession, rgb_float: np.ndarray
    ) -> np.ndarray:
        """
        Runs inference on an RGB float32 array of shape (H, W, 3) in [0, 1].
        Applies 18px reflection padding for CUNet boundary handling.
        Returns upscaled array (2*H, 2*W, 3) in [0, 1].
        """
        h, w, c = rgb_float.shape
        # Reorder to (1, 3, H, W)
        chw = np.transpose(rgb_float, (2, 0, 1))[np.newaxis, ...]

        # Reflection pad 18px on all borders
        padded = np.pad(chw, ((0, 0), (0, 0), (18, 18), (18, 18)), mode="reflect")

        # Run ONNX inference
        out = session.run(None, {"x": padded})[0]  # shape (1, 3, 2*H, 2*W)

        # Transpose back to (2*H, 2*W, 3) and clip to [0, 1]
        out_hwc = np.transpose(out[0], (1, 2, 0))
        return np.clip(out_hwc, 0.0, 1.0)

    def _upscale_large_rgb(
        self, session: ort.InferenceSession, rgb_float: np.ndarray, tile_size: int = 512, tile_pad: int = 16
    ) -> np.ndarray:
        """
        Tiled inference with overlap to handle arbitrary image sizes without running out of RAM.
        """
        h, w, _ = rgb_float.shape
        if h <= tile_size and w <= tile_size:
            return self._upscale_rgb_tile(session, rgb_float)

        out_h, out_w = h * 2, w * 2
        out_img = np.zeros((out_h, out_w, 3), dtype=np.float32)

        for y in range(0, h, tile_size):
            for x in range(0, w, tile_size):
                # Calculate tile boundaries with padding
                y_start = max(0, y - tile_pad)
                y_end = min(h, y + tile_size + tile_pad)
                x_start = max(0, x - tile_pad)
                x_end = min(w, x + tile_size + tile_pad)

                tile = rgb_float[y_start:y_end, x_start:x_end]
                tile_out = self._upscale_rgb_tile(session, tile)

                # Calculate placement in destination
                top_cut = (y - y_start) * 2
                bottom_cut = top_cut + min(tile_size, h - y) * 2
                left_cut = (x - x_start) * 2
                right_cut = left_cut + min(tile_size, w - x) * 2

                dest_y_start = y * 2
                dest_y_end = dest_y_start + (bottom_cut - top_cut)
                dest_x_start = x * 2
                dest_x_end = dest_x_start + (right_cut - left_cut)

                out_img[dest_y_start:dest_y_end, dest_x_start:dest_x_end] = tile_out[
                    top_cut:bottom_cut, left_cut:right_cut
                ]

        return out_img

    def process_pil_image(
        self,
        img: Image.Image,
        scale: int = 4,
        denoise: bool = True,
    ) -> Image.Image:
        """
        Upscales PIL image by 2x or 4x with 2D character edge reconstruction.
        Preserves alpha channel for transparent PNGs.
        """
        has_alpha = img.mode in ("RGBA", "LA") or (
            img.mode == "P" and "transparency" in img.info
        )
        if has_alpha:
            img_rgba = img.convert("RGBA")
            alpha = img_rgba.split()[-1]
            rgb_img = img_rgba.convert("RGB")
        else:
            alpha = None
            rgb_img = img.convert("RGB")

        # Pick appropriate model
        model_path = self._noise_scale2x_path if denoise else self._scale2x_path
        session = self._get_session(model_path)

        rgb_arr = np.array(rgb_img, dtype=np.float32) / 255.0

        # Pass 1: 2x upscale
        scaled_rgb = self._upscale_large_rgb(session, rgb_arr)

        # Pass 2: If 4x requested, run second 2x pass
        if scale >= 4:
            clean_session = self._get_session(self._scale2x_path)
            scaled_rgb = self._upscale_large_rgb(clean_session, scaled_rgb)

        # Convert back to PIL Image (uint8)
        scaled_uint8 = (scaled_rgb * 255.0).round().astype(np.uint8)
        out_pil = Image.fromarray(scaled_uint8, mode="RGB")

        # Handle alpha channel resizing with clean bicubic/box sampling
        if alpha is not None:
            target_size = out_pil.size
            alpha_upscaled = alpha.resize(target_size, resample=Image.Resampling.LANCZOS)
            out_pil.putalpha(alpha_upscaled)

        return out_pil

    def upscale_file(
        self,
        input_path: str | Path,
        output_path: str | Path,
        scale: int = 4,
        denoise: bool = True,
    ) -> str:
        with Image.open(input_path) as img:
            out_img = self.process_pil_image(img, scale=scale, denoise=denoise)
            output_str = str(Path(output_path).resolve())
            out_img.save(output_str, format="PNG", optimize=True)
            return output_str