File size: 20,440 Bytes
b9e72b4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
from __future__ import print_function #if running Python2 makes it work like python3

import math
import os
import sys
import time

import numpy as np
import torch
import torch.nn as nn
import yaml #Imports PyYAML for parsing YAML files.
from PIL import Image #Imports the Python Imaging Library for image processing.
from skimage.metrics import peak_signal_noise_ratio as compare_psnr #Imports the PSNR metric from skimage  to quantify reconstruction quality for images and video subject to lossy compression. Higher is better
from skimage.metrics import structural_similarity #Imports the SSIM metric from skimage.A full-reference image quality evaluation index that measures image similarity from three aspects: brightness, contrast, and structure.


def get_config(config):#apth to configuration file
    with open(config, 'r') as stream: #opens the file specified by the config variable in read mode and assigned to variable stream
        return yaml.load(stream) #call loads and parses the YAML content from the file object stream and returns the parsed content.


# Converts a Tensor into a Numpy array then desired imtype
# |imtype|: the desired type of the converted numpy array
def tensor2im(image_tensor, imtype=np.uint8):           
    image_numpy = image_tensor[0].cpu().float().numpy() #image_tensor[0]: first image in batch in tensor
                                                        #.cpu() moves tensor to CPU if in GPU
                                                        #.float() converts to float
                                                        #.numpy() converts to numpy
    if image_numpy.shape[0] == 1: #.shape[0] access channels ,1 = grayscale, 3=RGB
        image_numpy = np.tile(image_numpy, (3, 1, 1))
    #np.tile(A,reps):Construct an array by repeating A the number of times given by reps.
    #repeating graysvale image here in 3 channels to get RGB image
    image_numpy = (np.transpose(image_numpy, (1, 2, 0)) + 1) / 2.0 * 255.0
    #np.transpose(image_numpy, (1, 2, 0)):The image is transposed from the shape (C, H, W) to (H, W, C).
    # + 1: shift the pixel intensity range from [-1, 1] to [0, 2].
    # / 2.0: This scales the values to the range [0, 1].
    # * 255.0: This scales the values to the range [0, 255].
    image_numpy = image_numpy.astype(imtype) #numpy array recast to desired imtype
    if image_numpy.shape[-1] == 6: #after transpose -1 is no.channels and if this is 6
        image_numpy = np.concatenate([image_numpy[:, :, :3], image_numpy[:, :, 3:]], axis=1)
        '''???how does this help '''
        #recombines along width 2 grps of 0-2 and 3-5 
        
    if image_numpy.shape[-1] == 7:#if transposed image has 7 channels
        edge_map = np.tile(image_numpy[:, :, 6:7], (1, 1, 3))
        '''???Tiles (repeats) the seventh channel to create a 3-channel edge_map along the channel axis.'''
        image_numpy = np.concatenate([image_numpy[:, :, :3], image_numpy[:, :, 3:6], edge_map], axis=1)
        '''???how does this help '''
        #recombines along width 2 grps of 0-2 and 3-6 
    return image_numpy


def tensor2numpy(image_tensor):
    image_numpy = torch.squeeze(image_tensor).cpu().float().numpy() #.squeeze(): removes all dimensions of size 1 from the shape of image_tensor
    #PyTorch tensors can have singleton dimensions, which are dimensions with size 1. These dimensions might not carry meaningful information but are retained due to tensor operations.
    #proabably rmeove number of batchs since 1 batch in 1 img (batch,channels,height,width)
    image_numpy = (np.transpose(image_numpy, (1, 2, 0)) + 1) / 2.0 * 255.0
     #np.transpose(image_numpy, (1, 2, 0)):The image is transposed from the shape (C, H, W) to (H, W, C).
    # + 1: shift the pixel intensity range from [-1, 1] to [0, 2].
    # / 2.0: This scales the values to the range [0, 1].
    # * 255.0: This scales the values to the range [0, 255].
    image_numpy = image_numpy.astype(np.float32)
    #numpy array recast to float
    return image_numpy


# Get model list for resume
#retrieve a specific model checkpoint file based on dirname, epoch,key
def get_model_list(dirname, key, epoch=None): 
    #key: main file_name
    #epoch to indicate at which epoch the model was saved.Choosing epoch allows you to continue training from a specific point or to evaluate the model's performance at different stages of training
    if epoch is None:
        return os.path.join(dirname, key + '_latest.pt') # return latest checkpoint by default
    if os.path.exists(dirname) is False: #if directory doesnt exist  no checkpoint files can be retrieved.
        return None

    print(dirname, key)

    #list of generated checkpoints after specific epohs during traininf
    gen_models = [os.path.join(dirname, f) for f in os.listdir(dirname) if
                  os.path.isfile(os.path.join(dirname, f)) and ".pt" in f and 'latest' not in f]
    #isfile():regular file,not a directory or a special file
    #.pt type file with epoch number specifid, latest means epoch=None
    epoch_index = [int(os.path.basename(model_name).split('_')[-2]) for model_name in gen_models if 'latest' not in model_name]
    #os.path.basename(model_name): extracts the filename from the full path.
    #split('_')[-2]: ['dsrnet', 's', 'epoch14.pt']-> s (second last)
    '''??? why not -1 -> to get 14  chatgpt says int(s) will raise VauleError'''
    print('[i] available epoch list: %s' % epoch_index, gen_models)
    i = epoch_index.index(int(epoch))

    return gen_models[i]

# to preprocess a batch of images for compatibility with models pretrained on the ImageNet dataset
def vgg_preprocess(batch):
    # normalize using imagenet mean and std
    #ImageNet normalization helps preprocess images to have zero mean and unit variance across each channel (RGB).
    mean = batch.new(batch.size()) # Pytorch function creates new tensors (mean and std) with the same shape as the input batch.
    std = batch.new(batch.size())
    mean[:, 0, :, :] = 0.485 #assigned to all elements in the first channel of every image in the batch.
    mean[:, 1, :, :] = 0.456 #2nd channel
    mean[:, 2, :, :] = 0.406 #3rd channel
    std[:, 0, :, :] = 0.229 #1st channel
    std[:, 1, :, :] = 0.224 #2nd channel
    std[:, 2, :, :] = 0.225 #3rd channel
    batch = (batch + 1) / 2 #pixel values shifted [-1,1]->[0,2]->[0,1]
    batch -= mean 
    batch = batch / std
    # each pixel value in batch will have zero mean and unit variance 
    return batch



 #print diagnostic information about the gradients of a neural network 
 #useful for inspecting the average magnitude of gradients during training or optimization of a neural network. 
 # It helps in diagnosing potential issues like vanishing or exploding gradients,
def diagnose_network(net, name='network'): #name: Optional name for the network (default is 'network'), used in printing diagnostic information.
    mean = 0.0 #average mean absolute gradient across all parameters that have gradients.
    count = 0 #no. of paramters with gradients
    for param in net.parameters():
        if param.grad is not None: #trainable Parameters without gradients won't contribute to mean.
            #e.g frozen layers, non trainable paramters,bias,batch normalization parameters 
            mean += torch.mean(torch.abs(param.grad.data)) #mean absolute value of the gradient of the current parameter param 
            count += 1
    if count > 0:
        mean = mean / count
    print(name)
    print(mean)


def save_image(image_numpy, image_path): #create a PIL Image object from the numpy array and then saves it using the save() method of the PIL Image object.
    image_pil = Image.fromarray(image_numpy)
    image_pil.save(image_path)

#print shape and statistics of a numpy array 
def print_numpy(x, val=True, shp=False): #val is statistics shp is shape
    x = x.astype(np.float64)
    if shp:
        print('shape,', x.shape)
    if val:
        x = x.flatten() #converts nD to 1D
        print('mean = %3.3f, min = %3.3f, max = %3.3f, median = %3.3f, std=%3.3f' % (
            np.mean(x), np.min(x), np.max(x), np.median(x), np.std(x)))


def mkdirs(paths):
    if isinstance(paths, list) and not isinstance(paths, str): 
        #if list makes path for each string
        for path in paths:
            mkdir(path)
    else:      #if string makes path
        mkdir(paths)


def mkdir(path):
    if not os.path.exists(path):
        os.makedirs(path)


def set_opt_param(optimizer, key, value):
    for group in optimizer.param_groups:
        group[key] = value
'???'


def vis(x):#display an image in image viewer of system regardless if tensor or numpy array
    if isinstance(x, torch.Tensor):
        Image.fromarray(tensor2im(x)).show()
    elif isinstance(x, np.ndarray):
        Image.fromarray(x.astype(np.uint8)).show()
    else:
        raise NotImplementedError('vis for type [%s] is not implemented', type(x))


"""tensorboard"""
from tensorboardX import SummaryWriter #for data logging
from datetime import datetime


def get_summary_writer(log_dir): #log_dir:directy to store logs
    if not os.path.exists(log_dir):
        os.mkdir(log_dir)
    log_dir = os.path.join(log_dir, datetime.now().strftime('%b%d_%H-%M-%S') + '_' + socket.gethostname()) 
    if not os.path.exists(log_dir):
        os.mkdir(log_dir)
    writer = SummaryWriter(log_dir) #each time you run your experiments, logs are saved in a new directory with current time, date and machine hostname
    return writer



# keeps track of running avergae of metrics
class AverageMeters(object): #all classes inherit from object
    def __init__(self, dic=None, total_num=None):
        #dic: dictionary of metrics
        #total_num:track of the counts of each metric)
        self.dic = dic or {}
        #(mingcv) self.total_num = total_num
        self.total_num = total_num or {}
        

    def update(self, new_dic): # method appends self.dic and updates total_num with new values.
        for key in new_dic:
            if not key in self.dic:
                self.dic[key] = new_dic[key]
                self.total_num[key] = 1
            else:
                self.dic[key] += new_dic[key]
                self.total_num[key] += 1
        # (mingcv)self.total_num += 1

    def __getitem__(self, key): #overriding [], returns average value for a metric
        return self.dic[key] / self.total_num[key]

    def __str__(self):  #overrider str() and print() to show key and values formatted
        keys = sorted(self.keys())
        res = ''
        for key in keys:
            res += (key + ': %.4f' % self[key] + ' | ')
        return res

    def keys(self): #returns all the metric names
        return self.dic.keys()

'''???why _loss'''
def write_loss(writer, prefix, avg_meters, iteration): 
    #writer is the writer object for logging,SummaryWriter instance
    #prefix is keyword added before each metric e.g. "train"  it logs "accuracy": 85.4 as "train/accuracy"
    #avg_meters: A dictionary argument where keys are metric names (like ‘loss’, ‘accuracy’) and values are their corresponding values at the current iteration (e.g., 0.23, 85.4).
    #iteration: An integer argument representing the current iteration or step number in the training or evaluation process.
    for key in avg_meters.keys():
            meter = avg_meters[key] # for each key, we get the corresponding value from the avg_meters dictionary and assign it to the variable meter.
            writer.add_scalar(os.path.join(prefix, key), meter, iteration)
    # writer.add_scalar(): This method call logs the scalar value (metric) to the writer
'''

avg_meters = {

    “loss”: 0.23,

    “accuracy”: 85.4,

    “precision”: 88.1

}

write_loss(writer, “train”, avg_meters, 10)



Logging: train/loss = 0.23 at step 10

Logging: train/accuracy = 85.4 at step 10

Logging: train/precision = 88.1 at step 10



'''
"""(Mingcv)progress bar"""
import socket

# _, term_width = os.popen('stty size', 'r').read().split() 
term_width = 136 #hardcoded terminal width to 136 characters instead of dymeically reading using stty siz
 
TOTAL_BAR_LENGTH = 65. #length of the progress bar.
last_time = time.time() #when progress is finished
begin_time = last_time  # used to calculate the elapsed time for each step and the total process.


def progress_bar(current, total, msg=None):
                                        #current: The current progress count.
                                        #total: The total count that represents 100% progress.
                                        #msg: An optional message to display alongside the progress bar.
    global last_time, begin_time # for persistence in step_time and total_time
    if current == 0:
        begin_time = time.time()  # Reset for new bar.

    cur_len = int(TOTAL_BAR_LENGTH * current / total)
    rest_len = int(TOTAL_BAR_LENGTH - cur_len) - 1

    sys.stdout.write(' [')
    for i in range(cur_len):
        sys.stdout.write('=')
    sys.stdout.write('>')
    for i in range(rest_len):
        sys.stdout.write('.')
    sys.stdout.write(']')

    cur_time = time.time()
    step_time = cur_time - last_time
    last_time = cur_time
    tot_time = cur_time - begin_time

    L = []
    L.append('  Step: %s' % format_time(step_time))
    L.append(' | Tot: %s' % format_time(tot_time))
    if msg:
        L.append(' | ' + msg)

    msg = ''.join(L)
    sys.stdout.write(msg)
    for i in range(term_width - int(TOTAL_BAR_LENGTH) - len(msg) - 3):
        sys.stdout.write(' ')

    #(mingcv) Go back to the center of the bar.
    for i in range(term_width - int(TOTAL_BAR_LENGTH / 2) + 2):
        sys.stdout.write('\b')
    sys.stdout.write(' %d/%d ' % (current + 1, total))

    if current < total - 1:
        sys.stdout.write('\r')
    else:
        sys.stdout.write('\n')
    sys.stdout.flush()


def format_time(seconds): #convert seconds to days,hourse,mintures,seconds,milliseconds
    days = int(seconds / 3600 / 24)
    seconds = seconds - days * 3600 * 24
    hours = int(seconds / 3600)
    seconds = seconds - hours * 3600
    minutes = int(seconds / 60)
    seconds = seconds - minutes * 60
    secondsf = int(seconds)
    seconds = seconds - secondsf
    millis = int(seconds * 1000)

    f = ''
    i = 1
    if days > 0:
        f += str(days) + 'D'
        i += 1
    if hours > 0 and i <= 2:
        f += str(hours) + 'h'
        i += 1
    if minutes > 0 and i <= 2:
        f += str(minutes) + 'm'
        i += 1
    if secondsf > 0 and i <= 2:
        f += str(secondsf) + 's'
        i += 1
    if millis > 0 and i <= 2:
        f += str(millis) + 'ms'
        i += 1
    if f == '':
        f = '0ms'
    return f


def parse_args(args): #converts numeric args seperated by commas into a [] of ints
    str_args = args.split(',') #str1,atr2,str3 becomes [a,b,c]
    parsed_args = []
    for str_arg in str_args:
        arg = int(str_arg)
        if arg >= 0:
            parsed_args.append(arg)
    return parsed_args

#In order to overcome vanishing/explosding gradient, Xavier initialization was introduced. It tries to keep variance of all the layers equal but assumes linear actiivation
# Kaiming He initialization, which takes activation function into account. 

def weights_init_kaiming(m): #layer or module in NN
    classname = m.__class__.__name__ # gets the class name of the layer m using its __class__ attribute and __name__ property. 
    if classname.find('Conv') != -1: #returns -1 if the substring is not found, So here if found
        nn.init.kaiming_normal(m.weight.data, a=0, mode='fan_in')

    #nn.init.kaiming_normal:Fill the input Tensor(m.weight.data) with values using a Kaiming normal distribution.The method is described in Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification - He, K. et al. (2015). The resulting tensor will have values sampled from N(0,std^2) where std=gain/sqrt(fan_mode)
    #a (float) – the negative slope of the rectifier used after this layer only used with Leaky RelU (0-not used)
    #mode: fan_in(default) or fan_out.Choosing 'fan_in' preserves the magnitude of the variance of the weights in the forward pass. 
    #here intialising for relu,preserving variance of weights in forward pass
    elif classname.find('Linear') != -1:
        nn.init.kaiming_normal(m.weight.data, a=0, mode='fan_in')
        # here intialising for relu,preserving variance of weights in forward pass
    elif classname.find('BatchNorm') != -1:
        # nn.init.uniform(m.weight.data, 1.0, 0.02)
        m.weight.data.normal_(mean=0, std=math.sqrt(2. / 9. / 64.)).clamp_(-0.025, 0.025)
        nn.init.constant(m.bias.data, 0.0)
        "??? why std=math.sqrt(2. / 9. / 64.)"
        # .clamp_(-0.025, 0.025): Clamps (limits) the values in the tensor to be between -0.025 and 0.025.
        #bias of the batch normalization layer intiliazed to a constant value of 0.0.


def batch_PSNR(img, imclean, data_range): #calculates avg PSNR for 1 batch of images
    #img batch of images to be evaluated with noise
    #imclean: batch of ground truth images no noise
    Img = img.data.cpu().numpy().astype(np.float32) #converts img to float numpy array in cpu
    #- img.data: Detaches the data from the computation graph.
    #cpu(): Transfers the tensor from GPU to CPU if it’s on GPU.
    #numpy(): Converts the tensor to a NumPy array.
    #astype(np.float32): Converts the array type to float32.
    Iclean = imclean.data.cpu().numpy().astype(np.float32) #same for ground truth
    PSNR = 0
    for i in range(Img.shape[0]): #for all batches add the skmetric.psnr()
        PSNR += compare_psnr(Iclean[i, :, :, :], Img[i, :, :, :], data_range=data_range) 
        #data range: maximum - inimum possible values
    return PSNR / Img.shape[0] #average psnr per batch

#calculates avg SSIM for 1 batch of images
def batch_SSIM(img, imclean):
    Img = img.data.cpu().permute(0, 2, 3, 1).numpy().astype(np.float32)
    #converts img to float numpy array in cpu
    #img.data: Detaches the data from the computation graph.
    #cpu(): Transfers the tensor from GPU to CPU if it’s on GPU.
    #numpy(): Converts the tensor to a NumPy array.
    #astype(np.float32): Converts the array type to float32.
    Iclean = imclean.data.cpu().permute(0, 2, 3, 1).numpy().astype(np.float32)
    #permute(0, 2, 3, 1): Changes the order of dimensions from (batch_size, channels, height, width) to (batch_size, height, width, channels). 
    SSIM = 0

    for i in range(Img.shape[0]): #for all batches add the skmetric.ssim()
        SSIM += structural_similarity(Iclean[i, :, :, :], Img[i, :, :, :], win_size=11,
                                      multichannel=True, data_range=1)
        #win_size:The side-length of the sliding window used in comparison. Must be an odd value
        #multichannel deprecated ...equivalent to channel-axis 
        "??? multichannel=True equal to what number in channel-axis"
    return SSIM / Img.shape[0] #average ssim for1 batch


def data_augmentation(image, mode): #augment given image based on mode passed by manipulating numpy array
    out = np.transpose(image, (1, 2, 0))
    if mode == 0:
        #(mingcv)original
        out = out
    elif mode == 1:
        #(mingcv)flip up and down
        out = np.flipud(out)
    elif mode == 2:
        #(mingcv)rotate counterwise 90 degree
        out = np.rot90(out)
    elif mode == 3:
        #(mingcv)rotate 90 degree and flip up and down
        out = np.rot90(out)
        out = np.flipud(out)
    elif mode == 4:
        #(mingcv)rotate 180 degree
        out = np.rot90(out, k=2)
    elif mode == 5:
        #(mingcv)rotate 180 degree and flip
        out = np.rot90(out, k=2)
        out = np.flipud(out)
    elif mode == 6:
        #(mingcv)rotate 270 degree
        out = np.rot90(out, k=3)
    elif mode == 7:
        #(mingcv)rotate 270 degree and flip
        out = np.rot90(out, k=3)
        out = np.flipud(out)
    return np.transpose(out, (2, 0, 1))