Spaces:
Sleeping
Sleeping
Create app.py
Browse files
app.py
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import streamlit as st
|
| 2 |
+
import torch
|
| 3 |
+
from PIL import Image
|
| 4 |
+
import torchvision.transforms as tr
|
| 5 |
+
import os
|
| 6 |
+
import torch.nn as nn
|
| 7 |
+
from torchvision.transforms import v2
|
| 8 |
+
from sklearn.preprocessing import minmax_scale
|
| 9 |
+
import itertools
|
| 10 |
+
import functools
|
| 11 |
+
device = torch.device('cpu')
|
| 12 |
+
import numpy as np
|
| 13 |
+
from torch.optim.lr_scheduler import LambdaLR
|
| 14 |
+
from torch.optim.lr_scheduler import StepLR
|
| 15 |
+
# Дискриминаторы обучаются быстро.. почему бы и не использовать?
|
| 16 |
+
class NLayerDiscriminator(nn.Module):
|
| 17 |
+
"""Defines a PatchGAN discriminator"""
|
| 18 |
+
|
| 19 |
+
def __init__(self, input_nc, ndf=64, n_layers=3, norm_layer=nn.BatchNorm2d):
|
| 20 |
+
"""Construct a PatchGAN discriminator
|
| 21 |
+
|
| 22 |
+
Parameters:
|
| 23 |
+
input_nc (int) -- the number of channels in input images
|
| 24 |
+
ndf (int) -- the number of filters in the last conv layer
|
| 25 |
+
n_layers (int) -- the number of conv layers in the discriminator
|
| 26 |
+
norm_layer -- normalization layer
|
| 27 |
+
"""
|
| 28 |
+
super(NLayerDiscriminator, self).__init__()
|
| 29 |
+
if type(norm_layer) == functools.partial: # no need to use bias as BatchNorm2d has affine parameters
|
| 30 |
+
use_bias = norm_layer.func == nn.InstanceNorm2d
|
| 31 |
+
else:
|
| 32 |
+
use_bias = norm_layer == nn.InstanceNorm2d
|
| 33 |
+
|
| 34 |
+
kw = 4
|
| 35 |
+
padw = 1
|
| 36 |
+
sequence = [nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]
|
| 37 |
+
nf_mult = 1
|
| 38 |
+
nf_mult_prev = 1
|
| 39 |
+
for n in range(1, n_layers): # gradually increase the number of filters
|
| 40 |
+
nf_mult_prev = nf_mult
|
| 41 |
+
nf_mult = min(2 ** n, 8)
|
| 42 |
+
sequence += [
|
| 43 |
+
nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=2, padding=padw, bias=use_bias),
|
| 44 |
+
norm_layer(ndf * nf_mult),
|
| 45 |
+
nn.LeakyReLU(0.2, True)
|
| 46 |
+
]
|
| 47 |
+
|
| 48 |
+
nf_mult_prev = nf_mult
|
| 49 |
+
nf_mult = min(2 ** n_layers, 8)
|
| 50 |
+
sequence += [
|
| 51 |
+
nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=1, padding=padw, bias=use_bias),
|
| 52 |
+
norm_layer(ndf * nf_mult),
|
| 53 |
+
nn.LeakyReLU(0.2, True)
|
| 54 |
+
]
|
| 55 |
+
|
| 56 |
+
sequence += [nn.Conv2d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)] # output 1 channel prediction map
|
| 57 |
+
self.model = nn.Sequential(*sequence)
|
| 58 |
+
|
| 59 |
+
def forward(self, input):
|
| 60 |
+
"""Standard forward."""
|
| 61 |
+
return self.model(input)
|
| 62 |
+
|
| 63 |
+
class DeepGenerator(nn.Module):
|
| 64 |
+
def __init__(self, input_nc=3, output_nc=3, ngf=64, n_blocks=6, use_dropout=False):
|
| 65 |
+
super(DeepGenerator, self).__init__()
|
| 66 |
+
|
| 67 |
+
# Энкодер
|
| 68 |
+
self.enc1 = nn.Sequential(
|
| 69 |
+
nn.Conv2d(input_nc, ngf, 4, 2, 1),
|
| 70 |
+
nn.LeakyReLU(0.2)
|
| 71 |
+
)
|
| 72 |
+
self.enc2 = nn.Sequential(
|
| 73 |
+
nn.Conv2d(ngf, ngf*2, 4, 2, 1),
|
| 74 |
+
nn.InstanceNorm2d(ngf*2),
|
| 75 |
+
nn.LeakyReLU(0.2)
|
| 76 |
+
)
|
| 77 |
+
self.enc3 = nn.Sequential(
|
| 78 |
+
nn.Conv2d(ngf*2, ngf*4, 4, 2, 1),
|
| 79 |
+
nn.InstanceNorm2d(ngf*4),
|
| 80 |
+
nn.LeakyReLU(0.2)
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
# Промежуточные residual-блоки
|
| 84 |
+
self.res_blocks = nn.Sequential(
|
| 85 |
+
*[ResidualBlock(ngf*4, use_dropout=use_dropout) for _ in range(n_blocks)]
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
# Декодер с skip-connections (U-Net like)
|
| 89 |
+
self.dec1 = nn.Sequential(
|
| 90 |
+
nn.ConvTranspose2d(ngf*4, ngf*2, 4, 2, 1),
|
| 91 |
+
nn.InstanceNorm2d(ngf*2),
|
| 92 |
+
nn.ReLU(True)
|
| 93 |
+
)
|
| 94 |
+
self.dec2 = nn.Sequential(
|
| 95 |
+
nn.ConvTranspose2d(ngf*2*2, ngf, 4, 2, 1), # x2 из-за skip-connection
|
| 96 |
+
nn.InstanceNorm2d(ngf),
|
| 97 |
+
nn.ReLU(True)
|
| 98 |
+
)
|
| 99 |
+
self.dec3 = nn.Sequential(
|
| 100 |
+
nn.ConvTranspose2d(ngf*2, output_nc, 4, 2, 1), # x2 из-за skip-connection
|
| 101 |
+
nn.Tanh()
|
| 102 |
+
)
|
| 103 |
+
|
| 104 |
+
# Дополнительные слои
|
| 105 |
+
self.dropout = nn.Dropout(0.5) if use_dropout else None
|
| 106 |
+
|
| 107 |
+
def forward(self, x):
|
| 108 |
+
# Энкодер
|
| 109 |
+
e1 = self.enc1(x) # [batch, ngf, H/2, W/2]
|
| 110 |
+
e2 = self.enc2(e1) # [batch, ngf*2, H/4, W/4]
|
| 111 |
+
e3 = self.enc3(e2) # [batch, ngf*4, H/8, W/8]
|
| 112 |
+
|
| 113 |
+
# Residual-блоки
|
| 114 |
+
m = self.res_blocks(e3)
|
| 115 |
+
if self.dropout:
|
| 116 |
+
m = self.dropout(m)
|
| 117 |
+
|
| 118 |
+
# Декодер с skip-connections
|
| 119 |
+
d1 = self.dec1(m) # [batch, ngf*2, H/4, W/4]
|
| 120 |
+
d1 = torch.cat([d1, e2], dim=1) # Skip-connection
|
| 121 |
+
|
| 122 |
+
d2 = self.dec2(d1) # [batch, ngf, H/2, W/2]
|
| 123 |
+
d2 = torch.cat([d2, e1], dim=1) # Skip-connection
|
| 124 |
+
|
| 125 |
+
output = self.dec3(d2) # [batch, output_nc, H, W]
|
| 126 |
+
|
| 127 |
+
return output
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
class ResidualBlock(nn.Module):
|
| 131 |
+
def __init__(self, channels, use_dropout=False):
|
| 132 |
+
super(ResidualBlock, self).__init__()
|
| 133 |
+
self.block = nn.Sequential(
|
| 134 |
+
nn.Conv2d(channels, channels, 3, 1, 1),
|
| 135 |
+
nn.InstanceNorm2d(channels),
|
| 136 |
+
nn.ReLU(True),
|
| 137 |
+
nn.Dropout(0.5) if use_dropout else nn.Identity(),
|
| 138 |
+
nn.Conv2d(channels, channels, 3, 1, 1),
|
| 139 |
+
nn.InstanceNorm2d(channels)
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
def forward(self, x):
|
| 143 |
+
return x + self.block(x) # Skip-connection
|
| 144 |
+
|
| 145 |
+
class CycleGAN(nn.Module):
|
| 146 |
+
def __init__(self):
|
| 147 |
+
super(CycleGAN, self).__init__()
|
| 148 |
+
# Волевое решение, что все изображения трёхканальные, культ RGB все дела
|
| 149 |
+
self.D_A = NLayerDiscriminator(input_nc=3, n_layers= 3)
|
| 150 |
+
self.D_B = NLayerDiscriminator(input_nc=3, n_layers= 3)
|
| 151 |
+
# self.G_A = ResnetGenerator(input_nc=3, output_nc=3)
|
| 152 |
+
# self.G_B = ResnetGenerator(input_nc=3, output_nc=3)
|
| 153 |
+
# self.G_A = SimpleGenerator(input_nc=3, output_nc=3)
|
| 154 |
+
# self.G_B = SimpleGenerator(input_nc=3, output_nc=3)
|
| 155 |
+
self.G_A = DeepGenerator(input_nc= 3, output_nc= 3)
|
| 156 |
+
self.G_B = DeepGenerator(input_nc= 3, output_nc= 3)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def create_model_and_optimizer(model_class, model_params, lr_d=1e-4, lr_g=1.e-3, weight_decay=1.e-5, device=device):
|
| 160 |
+
model = model_class(**model_params)
|
| 161 |
+
model = model.to(device)
|
| 162 |
+
optimizer_d = torch.optim.Adam(
|
| 163 |
+
itertools.chain(model.D_A.parameters(), model.D_B.parameters()),
|
| 164 |
+
lr = lr_d,
|
| 165 |
+
weight_decay = weight_decay,
|
| 166 |
+
)
|
| 167 |
+
optimizer_g = torch.optim.Adam(
|
| 168 |
+
itertools.chain(model.G_A.parameters(), model.G_B.parameters()),
|
| 169 |
+
lr = lr_g,
|
| 170 |
+
weight_decay=weight_decay,
|
| 171 |
+
)
|
| 172 |
+
return model, optimizer_d, optimizer_g
|
| 173 |
+
|
| 174 |
+
model_name = "cycle_gan_200e_fallout#1"
|
| 175 |
+
checkpoint = torch.load(os.path.join("./model", f"{model_name}.pt"), weights_only = False)
|
| 176 |
+
|
| 177 |
+
# Создаём те же классы, что и внутри чекпоинта
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
total_epochs = 200
|
| 181 |
+
half_epochs = total_epochs/2
|
| 182 |
+
|
| 183 |
+
def get_linear_scheduler(optimizer, start_epoch, n_epochs):
|
| 184 |
+
def lr_lambda(epoch):
|
| 185 |
+
return 1.0 - max(0, epoch - start_epoch) / (n_epochs - start_epoch)
|
| 186 |
+
return LambdaLR(optimizer, lr_lambda)
|
| 187 |
+
|
| 188 |
+
model_params = dict()
|
| 189 |
+
|
| 190 |
+
model, optimizer_d, optimizer_g = create_model_and_optimizer(
|
| 191 |
+
model_class = CycleGAN,
|
| 192 |
+
model_params = model_params,
|
| 193 |
+
lr_d = 1e-3,
|
| 194 |
+
lr_g= 1e-3,
|
| 195 |
+
device = device,
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
scheduler_d = get_linear_scheduler(optimizer_d, start_epoch=half_epochs, n_epochs=total_epochs)
|
| 200 |
+
scheduler_g = get_linear_scheduler(optimizer_g, start_epoch=half_epochs, n_epochs=total_epochs)
|
| 201 |
+
# Загружаем состояния из чекпоинта
|
| 202 |
+
# @st.cache_resource
|
| 203 |
+
def load_model():
|
| 204 |
+
model.load_state_dict(checkpoint['model_state_dict'] )
|
| 205 |
+
model.eval()
|
| 206 |
+
return model
|
| 207 |
+
load_model()
|
| 208 |
+
|
| 209 |
+
# Функция для получения train и val transform-ов, а так же функции для де-нормализации изображения
|
| 210 |
+
def get_transforms(mean, std, random_crop_size, p):
|
| 211 |
+
train_transform = tr.Compose([
|
| 212 |
+
tr.ToPILImage(),
|
| 213 |
+
tr.RandomApply([
|
| 214 |
+
tr.ColorJitter(0.2, 0.2, 0.2, 0.01) # Изменение яркости, контраста и т.д.
|
| 215 |
+
], p=0.8),
|
| 216 |
+
v2.RandomCrop(size=random_crop_size),
|
| 217 |
+
v2.RandomHorizontalFlip(p=p),
|
| 218 |
+
tr.ToTensor(),
|
| 219 |
+
tr.Normalize(mean=mean, std=std),
|
| 220 |
+
])
|
| 221 |
+
|
| 222 |
+
val_transform = tr.Compose([
|
| 223 |
+
# tr.ToPILImage(),
|
| 224 |
+
tr.Resize((random_crop_size, random_crop_size)),
|
| 225 |
+
tr.ToTensor(),
|
| 226 |
+
tr.Normalize(mean=mean, std=std),
|
| 227 |
+
])
|
| 228 |
+
|
| 229 |
+
def de_normalize(img, normalized = True):
|
| 230 |
+
with torch.no_grad():
|
| 231 |
+
if normalized:
|
| 232 |
+
img = img.cpu().permute(1,2,0)
|
| 233 |
+
return minmax_scale(
|
| 234 |
+
(img.reshape(3, -1) + mean[:, None]) * std[:, None],
|
| 235 |
+
feature_range=(0., 1.),
|
| 236 |
+
axis=1,
|
| 237 |
+
).reshape(*img.shape)
|
| 238 |
+
return img
|
| 239 |
+
# def denormalize(tensor):
|
| 240 |
+
# tensor = tensor.clone().squeeze()
|
| 241 |
+
# for t, m, s in zip(tensor, mean, std):
|
| 242 |
+
# t.mul_(s).add_(m) # Обратное преобразование: (t * std) + mean
|
| 243 |
+
# tensor = tensor.permute(1, 2, 0) * 255
|
| 244 |
+
# return tensor.cpu().numpy().astype(np.uint8)
|
| 245 |
+
# def denormalize(tensor):
|
| 246 |
+
# return tensor.cpu().numpy()
|
| 247 |
+
|
| 248 |
+
return train_transform, val_transform, de_normalize
|
| 249 |
+
|
| 250 |
+
hyperparams = dict(
|
| 251 |
+
random_crop_size = 256,
|
| 252 |
+
p = 0.5
|
| 253 |
+
)
|
| 254 |
+
channel_mean_a =np.array([0.2999074, 0.34460328, 0.27964595])
|
| 255 |
+
channel_std_a = np.array([0.26192185, 0.27246983, 0.239288 ])
|
| 256 |
+
channel_mean_b =np.array( [0.40723574, 0.35794826, 0.27471713])
|
| 257 |
+
channel_std_b = np.array( [0.25011656, 0.23407436, 0.21894803])
|
| 258 |
+
train_transform_a, val_transform_a, de_normalize_a = get_transforms(channel_mean_a, channel_std_a, **hyperparams)
|
| 259 |
+
train_transform_b, val_transform_b, de_normalize_b = get_transforms(channel_mean_b, channel_std_b, **hyperparams)
|
| 260 |
+
|
| 261 |
+
# Обработка изображения
|
| 262 |
+
def process_image(model : CycleGAN, image, transform):
|
| 263 |
+
img_tensor = transform(image).unsqueeze(0)
|
| 264 |
+
with torch.no_grad():
|
| 265 |
+
output = model.G_A(img_tensor)
|
| 266 |
+
return output
|
| 267 |
+
# Интерфейс
|
| 268 |
+
st.title("CycleGAN for Fallout 3 to Fallout: New Vegas Task")
|
| 269 |
+
upload_A = st.file_uploader("Upload Fallout 3 image", type=["jpg", "png"])
|
| 270 |
+
|
| 271 |
+
if upload_A is not None:
|
| 272 |
+
image = Image.open(upload_A)
|
| 273 |
+
|
| 274 |
+
col1, col2 = st.columns(2)
|
| 275 |
+
with col1:
|
| 276 |
+
st.image(image, caption="Original")
|
| 277 |
+
|
| 278 |
+
with col2:
|
| 279 |
+
result = process_image(model, image, val_transform_a)
|
| 280 |
+
st.image(de_normalize_b(result[0]), caption="Fallout: New Vegas style", clamp=True)
|
| 281 |
+
|
| 282 |
+
upload_B = st.file_uploader("Upload Fallout: New Vegas image", type=["jpg", "png"])
|
| 283 |
+
|
| 284 |
+
if upload_B is not None:
|
| 285 |
+
image = Image.open(upload_B)
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
col1, col2 = st.columns(2)
|
| 289 |
+
with col1:
|
| 290 |
+
st.image(image, caption="Original")
|
| 291 |
+
|
| 292 |
+
with col2:
|
| 293 |
+
result = process_image(model, image, val_transform_b)
|
| 294 |
+
st.image(de_normalize_a(result[0]), caption="Fallout 3 style", clamp=True)
|