SunlitDeadline commited on
Commit
b9e72b4
·
verified ·
1 Parent(s): 3e7ec89

commited from namirahrasul/DSRNet -main local

Browse files
Files changed (47) hide show
  1. DSRNet/LICENSE +201 -0
  2. DSRNet/README.md +55 -0
  3. DSRNet/data/VOC2012_224_train_png.txt +0 -0
  4. DSRNet/data/image_folder.py +51 -0
  5. DSRNet/data/real_test.txt +20 -0
  6. DSRNet/data/sirs_dataset.py +335 -0
  7. DSRNet/data/torchdata.py +74 -0
  8. DSRNet/data/transforms.py +375 -0
  9. DSRNet/engine.py +151 -0
  10. DSRNet/eval_sirs.py +67 -0
  11. DSRNet/eval_sirs_4000 +75 -0
  12. DSRNet/models/__init__.py +10 -0
  13. DSRNet/models/__pycache__/__init__.cpython-38.pyc +0 -0
  14. DSRNet/models/__pycache__/networks.cpython-38.pyc +0 -0
  15. DSRNet/models/__pycache__/vgg.cpython-38.pyc +0 -0
  16. DSRNet/models/arch/__init__.py +49 -0
  17. DSRNet/models/arch/__pycache__/__init__.cpython-38.pyc +0 -0
  18. DSRNet/models/arch/dsrnet.py +306 -0
  19. DSRNet/models/arch/lrm.py +185 -0
  20. DSRNet/models/arch/~/YTMT_US +1197 -0
  21. DSRNet/models/arch/~/YTMT_US.pdf +0 -0
  22. DSRNet/models/arch/~/tmp.pth +3 -0
  23. DSRNet/models/base_model.py +73 -0
  24. DSRNet/models/dsrnet_model_sirs.py +263 -0
  25. DSRNet/models/losses.py +136 -0
  26. DSRNet/models/networks.py +276 -0
  27. DSRNet/models/vgg.py +90 -0
  28. DSRNet/options/__init__.py +0 -0
  29. DSRNet/options/base_option.py +56 -0
  30. DSRNet/options/net_options/__init__.py +0 -0
  31. DSRNet/options/net_options/base_options.py +97 -0
  32. DSRNet/options/net_options/train_options.py +77 -0
  33. DSRNet/requirements.txt +11 -0
  34. DSRNet/scripts.sh +6 -0
  35. DSRNet/test_sirs.py +33 -0
  36. DSRNet/tools/__init__.py +0 -0
  37. DSRNet/tools/mutils.py +37 -0
  38. DSRNet/train_sirs.py +109 -0
  39. DSRNet/train_sirs_4000.py +118 -0
  40. DSRNet/util/__init__.py +17 -0
  41. DSRNet/util/html.py +66 -0
  42. DSRNet/util/index.py +67 -0
  43. DSRNet/util/net_utils.py +13 -0
  44. DSRNet/util/schedulers.py +74 -0
  45. DSRNet/util/util.py +435 -0
  46. DSRNet/util/visualizer.py +152 -0
  47. DSRNet/weights/dsrnet_s_epoch14.pt +3 -0
DSRNet/LICENSE ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [yyyy] [name of copyright owner]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
DSRNet/README.md ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # DSRNet: Single Image Reflection Separation via Component Synergy (ICCV 2023)
2
+
3
+ > :book: [[ICCV](https://openaccess.thecvf.com/content/ICCV2023/papers/Hu_Single_Image_Reflection_Separation_via_Component_Synergy_ICCV_2023_paper.pdf)] [[Arxiv](https://arxiv.org/abs/2308.10027)] [[Supp.](https://github.com/mingcv/DSRNet/files/12387445/full_arxiv_version_supp.pdf)] <br>
4
+ > [Qiming Hu](https://scholar.google.com.hk/citations?user=4zasPbwAAAAJ), [Xiaojie Guo](https://sites.google.com/view/xjguo/homepage) <br>
5
+ > College of Intelligence and Computing, Tianjin University<br>
6
+
7
+
8
+ ### Network Architecture
9
+ ![fig_arch](https://github.com/mingcv/DSRNet/assets/31566437/2a4bb4be-9d03-40eb-b585-f2d5f8a44f42)
10
+
11
+ ### Environment Preparation (Python 3.9)
12
+ ```pip install -r requirements.txt```
13
+ ### Data Preparation
14
+
15
+
16
+ #### Training dataset
17
+ * 7,643 images from the
18
+ [Pascal VOC dataset](http://host.robots.ox.ac.uk/pascal/VOC/), center-cropped as 224 x 224 slices to synthesize training pairs;
19
+ * 90 real-world training pairs provided by [Zhang *et al.*](https://github.com/ceciliavision/perceptual-reflection-removal);
20
+ * 200 real-world training pairs provided by [IBCLN](https://github.com/JHL-HUST/IBCLN) (In our training setting 2, &dagger; labeled in our paper).
21
+
22
+ #### Testing dataset
23
+ * 45 real-world testing images from [CEILNet dataset](https://github.com/fqnchina/CEILNet);
24
+ * 20 real testing pairs provided by [Zhang *et al.*](https://github.com/ceciliavision/perceptual-reflection-removal);
25
+ * 20 real testing pairs provided by [IBCLN](https://github.com/JHL-HUST/IBCLN);
26
+ * 454 real testing pairs from [SIR^2 dataset](https://sir2data.github.io/), containing three subsets (i.e., Objects (200), Postcard (199), Wild (55)).
27
+
28
+ Download all in one by [Google Drive](https://drive.google.com/file/d/1hFZItZAzAt-LnfNj-2phBRwqplDUasQy/view?usp=sharing) or [百度云](https://pan.baidu.com/s/15zlk5o_-kx3ruKj4KfOvtA?pwd=1231).
29
+ ### Usage
30
+
31
+ #### Training
32
+ Setting I (w/o Nature): ```python train_sirs.py --inet dsrnet_l --model dsrnet_model_sirs --dataset sirs_dataset --loss losses --name dsrnet_l --lambda_vgg 0.01 --lambda_rec 0.2 --if_align --seed 2018 --base_dir "[YOUR DATA DIR]"```
33
+
34
+ Setting II (w/ Nature): ```python train_sirs_4000.py --inet dsrnet_l --model dsrnet_model_sirs --dataset sirs_dataset --loss losses --name dsrnet_l_4000 --lambda_vgg 0.01 --lambda_rec 0.2 --if_align --seed 2018 --base_dir "[YOUR DATA DIR]"```
35
+ #### Evaluation
36
+ Setting I (w/o Nature): ```python eval_sirs.py --inet dsrnet_l --model dsrnet_model_sirs --dataset sirs_dataset --name dsrnet_l_test --if_align --resume --weight_path "./weights/dsrnet_l_epoch18.pt" --base_dir "[YOUR_DATA_DIR]"```
37
+
38
+ Setting II (w/ Nature): ```python eval_sirs_4000.py --inet dsrnet_l --model dsrnet_model_sirs --dataset sirs_dataset --name dsrnet_l_4000_test --if_align --resume --weight_path "./weights/dsrnet_l_4000_epoch33.pt" --base_dir "[YOUR_DATA_DIR]"```
39
+
40
+ More commands can be found in [scripts.sh](https://github.com/mingcv/DSRNet/blob/main/scripts.sh).
41
+
42
+ #### Trained weights
43
+
44
+ Download the trained weights by [Google Drive](https://drive.google.com/drive/folders/1AIS9-EgBN3_q-TCq7W0j5OeWMgLO_de0?usp=sharing) or [百度云](https://pan.baidu.com/s/17jW9oBAfIZ03FKa3jc-qig?pwd=1231) and drop them into the "weights" dir.
45
+
46
+ ![image](https://github.com/mingcv/DSRNet/assets/31566437/1bfbd2c6-ca80-40ac-9c9e-a30ba0d095c7)
47
+
48
+
49
+ #### Visual comparison on real20 and SIR^2
50
+ ![image](https://github.com/mingcv/DSRNet/assets/31566437/0d32ee2b-4c9e-46ad-834b-6b08fc6aadd5)
51
+
52
+
53
+ #### Impressive Restoration Quality of Reflection Layers
54
+ ![image](https://github.com/mingcv/DSRNet/assets/31566437/e75e2abb-c413-4250-acd1-3f10e9d887b1)
55
+
DSRNet/data/VOC2012_224_train_png.txt ADDED
The diff for this file is too large to render. See raw diff
 
DSRNet/data/image_folder.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ###############################################################################
2
+ # Code from
3
+ # https://github.com/pytorch/vision/blob/master/torchvision/datasets/folder.py
4
+ # Modified the original code so that it also loads images from the current
5
+ # directory as well as the subdirectories
6
+ ###############################################################################
7
+
8
+ import torch.utils.data as data
9
+
10
+ from PIL import Image
11
+ import os
12
+ import os.path
13
+
14
+ IMG_EXTENSIONS = [
15
+ '.jpg', '.JPG', '.jpeg', '.JPEG',
16
+ '.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP',
17
+ ]
18
+
19
+
20
+ def read_fns(filename):#returns all lines in a file as a list
21
+ with open(filename) as f:
22
+ fns = f.readlines()
23
+ fns = [fn.strip() for fn in fns] #This strips any leading and trailing whitespace characters (including newlines) from each line in fns and turns result into a list
24
+ return fns
25
+
26
+
27
+ def is_image_file(filename):#checks if file is image based on extensions
28
+ return any(filename.endswith(extension) for extension in IMG_EXTENSIONS)
29
+
30
+
31
+ def make_dataset(dir, fns=None):#makes dataset from directory with files
32
+ images = []
33
+ assert os.path.isdir(dir), '%s is not a valid directory' % dir
34
+
35
+ if fns is None:#if no filename list is given, create list of images
36
+ for root, _, fnames in sorted(os.walk(dir)): #walks through the directory tree starting from dir, returning the root directory path, directories, and file names. The results are sorted.
37
+ for fname in fnames:
38
+ if is_image_file(fname):
39
+ path = os.path.join(root, fname)
40
+ images.append(path)
41
+ else:#create list of images from list
42
+ for fname in fns:
43
+ if is_image_file(fname):
44
+ path = os.path.join(dir, fname)
45
+ images.append(path)
46
+
47
+ return images
48
+
49
+
50
+ def default_loader(path):#convert image in path to rgb and open it
51
+ return Image.open(path).convert('RGB') #if grayscale or has alpha channel (rgba)
DSRNet/data/real_test.txt ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 3.jpg
2
+ 4.jpg
3
+ 9.jpg
4
+ 12.jpg
5
+ 15.jpg
6
+ 22.jpg
7
+ 23.jpg
8
+ 25.jpg
9
+ 29.jpg
10
+ 39.jpg
11
+ 46.jpg
12
+ 47.jpg
13
+ 58.jpg
14
+ 86.jpg
15
+ 87.jpg
16
+ 89.jpg
17
+ 93.jpg
18
+ 103.jpg
19
+ 107.jpg
20
+ 110.jpg
DSRNet/data/sirs_dataset.py ADDED
@@ -0,0 +1,335 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import os.path
3
+ import os.path
4
+ import random
5
+ from os.path import join
6
+
7
+ import cv2
8
+ import numpy as np
9
+ import torch.utils.data
10
+ import torchvision.transforms.functional as TF #more flexible and fine-grained transformations compared to the standard transforms API.
11
+ from PIL import Image
12
+ from scipy.signal import convolve2d
13
+
14
+ from data.image_folder import make_dataset
15
+ from data.torchdata import Dataset as BaseDataset #overrides len(), + and []
16
+ from data.transforms import to_tensor
17
+
18
+
19
+ def __scale_width(img, target_width):
20
+ ow, oh = img.size #img.size method from (PIL) /Pillow, returns a tuple (width,height) in px
21
+ #original width and original height
22
+ if (ow == target_width):
23
+ return img
24
+ w = target_width
25
+ h = int(target_width * oh / ow) #maintain aspect ratio by changing height
26
+ h = math.ceil(h / 2.) * 2 #(mingcv) round up to even
27
+ return img.resize((w, h), Image.BICUBIC) #resizes the image to the new dimensions (w, h) using bicubic interpolation. Bicubic interpolation is a method of resizing images that results in smoother and more visually appealing images compared to simpler methods like nearest neighbor or bilinear interpolation. From PIL/pillow
28
+
29
+
30
+ def __scale_height(img, target_height):
31
+ ow, oh = img.size
32
+ if (oh == target_height):
33
+ return img
34
+ h = target_height
35
+ w = int(target_height * ow / oh)
36
+ w = math.ceil(w / 2.) * 2
37
+ return img.resize((w, h), Image.BICUBIC)
38
+
39
+
40
+ def paired_data_transforms(img_1, img_2, unaligned_transforms=False):#img1, img2:PIL Image objects
41
+ #Unaligned transformations refer to transformations applied to paired images where the transformations are intentionally not synchronized between the two images. This approach introduces a controlled misalignment, adding variability to the data and potentially increasing the robustness of a model trained on such data.
42
+ def get_params(img, output_size):#inner nested function for scope & encapsulation
43
+ #getting dimensions for cropping
44
+ w, h = img.size
45
+ th, tw = output_size
46
+ if w == tw and h == th:
47
+ return 0, 0, h, w# no cropping as u crop form top left to bottom right
48
+
49
+ i = random.randint(0, h - th)
50
+ j = random.randint(0, w - tw)
51
+ return i, j, th, tw #i,j = starting position of cropped region
52
+
53
+ target_size = int(random.randint(224, 448) / 2.) * 2
54
+ '''??? why 224-448'''
55
+ ow, oh = img_1.size
56
+ if ow >= oh:#scale the smaller dimension
57
+ img_1 = __scale_height(img_1, target_size)
58
+ img_2 = __scale_height(img_2, target_size)
59
+ else:
60
+ img_1 = __scale_width(img_1, target_size)
61
+ img_2 = __scale_width(img_2, target_size)
62
+
63
+ if random.random() < 0.5: #flip randomly pairs
64
+ img_1 = TF.hflip(img_1)
65
+ img_2 = TF.hflip(img_2)
66
+
67
+ if random.random() < 0.5: #rotate randomly pairs
68
+ angle = random.choice([90, 180, 270])
69
+ img_1 = TF.rotate(img_1, angle)
70
+ img_2 = TF.rotate(img_2, angle)
71
+ #unalined transoformation by cropping 2 images differently
72
+ i, j, h, w = get_params(img_1, (224, 224))
73
+ img_1 = TF.crop(img_1, i, j, h, w)
74
+
75
+ if unaligned_transforms:
76
+ #(mingcv) print('random shift')
77
+ i_shift = random.randint(-10, 10)
78
+ j_shift = random.randint(-10, 10)
79
+ i += i_shift
80
+ j += j_shift
81
+
82
+ img_2 = TF.crop(img_2, i, j, h, w)
83
+
84
+ return img_1, img_2
85
+
86
+
87
+ class ReflectionSynthesis(object):
88
+ def __init__(self):
89
+ #(mingcv) Kernel Size of the Gaussian Blurry
90
+ self.kernel_sizes = [5, 7, 9, 11]
91
+ self.kernel_probs = [0.1, 0.2, 0.3, 0.4]
92
+
93
+ #(mingcv) Sigma of the Gaussian Blurry
94
+ self.sigma_range = [2, 5]
95
+ self.alpha_range = [0.8, 1.0]
96
+ self.beta_range = [0.4, 1.0]
97
+
98
+ def __call__(self, T_, R_):
99
+ T_ = np.asarray(T_, np.float32) / 255.
100
+ R_ = np.asarray(R_, np.float32) / 255.
101
+
102
+ kernel_size = np.random.choice(self.kernel_sizes, p=self.kernel_probs)
103
+ sigma = np.random.uniform(self.sigma_range[0], self.sigma_range[1])
104
+ kernel = cv2.getGaussianKernel(kernel_size, sigma)
105
+ kernel2d = np.dot(kernel, kernel.T)
106
+ for i in range(3):
107
+ R_[..., i] = convolve2d(R_[..., i], kernel2d, mode='same')
108
+
109
+ a = np.random.uniform(self.alpha_range[0], self.alpha_range[1])
110
+ b = np.random.uniform(self.beta_range[0], self.beta_range[1])
111
+ T, R = a * T_, b * R_
112
+
113
+ if random.random() < 0.7:
114
+ I = T + R - T * R
115
+ else:
116
+ I = T + R
117
+ if np.max(I) > 1:
118
+ m = I[I > 1]
119
+ m = (np.mean(m) - 1) * 1.3
120
+ I = np.clip(T + np.clip(R - m, 0, 1), 0, 1)
121
+
122
+ return T_, R_, I
123
+
124
+
125
+ class DataLoader(torch.utils.data.DataLoader):
126
+ def __init__(self, dataset, batch_size, shuffle, *args, **kwargs):
127
+ super(DataLoader, self).__init__(dataset, batch_size, shuffle, *args, **kwargs)
128
+ self.shuffle = shuffle
129
+
130
+ def reset(self):
131
+ if self.shuffle:
132
+ print('Reset Dataset...')
133
+ self.dataset.reset()
134
+
135
+
136
+ class DSRDataset(BaseDataset):
137
+ def __init__(self, datadir, fns=None, size=None, enable_transforms=True):
138
+ super(DSRDataset, self).__init__()
139
+ self.size = size
140
+ self.datadir = datadir
141
+ self.enable_transforms = enable_transforms
142
+ sortkey = lambda key: os.path.split(key)[-1]
143
+ self.paths = sorted(make_dataset(datadir, fns), key=sortkey)
144
+ if size is not None:
145
+ self.paths = np.random.choice(self.paths, size)
146
+
147
+ self.syn_model = ReflectionSynthesis()
148
+ self.reset(shuffle=False)
149
+
150
+ def reset(self, shuffle=True):
151
+ if shuffle:
152
+ random.shuffle(self.paths)
153
+ num_paths = len(self.paths) // 2
154
+ self.B_paths = self.paths[0:num_paths]
155
+ self.R_paths = self.paths[num_paths:2 * num_paths]
156
+
157
+ def data_synthesis(self, t_img, r_img):
158
+ if self.enable_transforms:
159
+ t_img, r_img = paired_data_transforms(t_img, r_img)
160
+
161
+ t_img, r_img, m_img = self.syn_model(t_img, r_img)
162
+
163
+ B = TF.to_tensor(t_img)
164
+ R = TF.to_tensor(r_img)
165
+ M = TF.to_tensor(m_img)
166
+
167
+ return B, R, M
168
+
169
+ def __getitem__(self, index):
170
+ index_B = index % len(self.B_paths)
171
+ index_R = index % len(self.R_paths)
172
+
173
+ B_path = self.B_paths[index_B]
174
+ R_path = self.R_paths[index_R]
175
+
176
+ t_img = Image.open(B_path).convert('RGB')
177
+ r_img = Image.open(R_path).convert('RGB')
178
+
179
+ B, R, M = self.data_synthesis(t_img, r_img)
180
+ fn = os.path.basename(B_path)
181
+ return {'input': M, 'target_t': B, 'target_r': R, 'fn': fn, 'real': False}
182
+
183
+ def __len__(self):
184
+ if self.size is not None:
185
+ return min(max(len(self.B_paths), len(self.R_paths)), self.size)
186
+ else:
187
+ return max(len(self.B_paths), len(self.R_paths))
188
+
189
+
190
+ class DSRTestDataset(BaseDataset):
191
+ def __init__(self, datadir, fns=None, size=None, enable_transforms=False, unaligned_transforms=False,
192
+ round_factor=1, flag=None, if_align=False):
193
+ super(DSRTestDataset, self).__init__()
194
+ self.size = size
195
+ self.datadir = datadir
196
+ self.fns = fns or os.listdir(join(datadir, 'blended'))
197
+ self.enable_transforms = enable_transforms
198
+ self.unaligned_transforms = unaligned_transforms
199
+ self.round_factor = round_factor
200
+ self.flag = flag
201
+ self.if_align = if_align
202
+
203
+ if size is not None:
204
+ self.fns = self.fns[:size]
205
+
206
+ def align(self, x1, x2):
207
+ h, w = x1.height, x1.width
208
+ h, w = h // 32 * 32, w // 32 * 32
209
+ x1 = x1.resize((w, h))
210
+ x2 = x2.resize((w, h))
211
+ return x1, x2
212
+
213
+ def __getitem__(self, index):
214
+ fn = self.fns[index]
215
+
216
+ t_img = Image.open(join(self.datadir, 'transmission_layer', fn)).convert('RGB')
217
+ m_img = Image.open(join(self.datadir, 'blended', fn)).convert('RGB')
218
+
219
+ if self.if_align:
220
+ t_img, m_img = self.align(t_img, m_img)
221
+
222
+ if self.enable_transforms:
223
+ t_img, m_img = paired_data_transforms(t_img, m_img, self.unaligned_transforms)
224
+
225
+ B = TF.to_tensor(t_img)
226
+ M = TF.to_tensor(m_img)
227
+
228
+ dic = {'input': M, 'target_t': B, 'fn': fn, 'real': True, 'target_r': M - B}
229
+ if self.flag is not None:
230
+ dic.update(self.flag)
231
+ return dic
232
+
233
+ def __len__(self):
234
+ if self.size is not None:
235
+ return min(len(self.fns), self.size)
236
+ else:
237
+ return len(self.fns)
238
+
239
+
240
+ class SIRTestDataset(BaseDataset):
241
+ def __init__(self, datadir, fns=None, size=None, if_align=False):
242
+ super(SIRTestDataset, self).__init__()
243
+ self.size = size
244
+ self.datadir = datadir
245
+ self.fns = fns or os.listdir(join(datadir, 'blended'))
246
+ self.if_align = if_align
247
+
248
+ if size is not None:
249
+ self.fns = self.fns[:size]
250
+
251
+ def align(self, x1, x2, x3):
252
+ h, w = x1.height, x1.width
253
+ h, w = h // 32 * 32, w // 32 * 32
254
+ x1 = x1.resize((w, h))
255
+ x2 = x2.resize((w, h))
256
+ x3 = x3.resize((w, h))
257
+ return x1, x2, x3
258
+
259
+ def __getitem__(self, index):
260
+ fn = self.fns[index]
261
+
262
+ t_img = Image.open(join(self.datadir, 'transmission_layer', fn)).convert('RGB')
263
+ r_img = Image.open(join(self.datadir, 'reflection_layer', fn)).convert('RGB')
264
+ m_img = Image.open(join(self.datadir, 'blended', fn)).convert('RGB')
265
+
266
+ if self.if_align:
267
+ t_img, r_img, m_img = self.align(t_img, r_img, m_img)
268
+
269
+ B = TF.to_tensor(t_img)
270
+ R = TF.to_tensor(r_img)
271
+ M = TF.to_tensor(m_img)
272
+
273
+ dic = {'input': M, 'target_t': B, 'fn': fn, 'real': True, 'target_r': R, 'target_r_hat': M - B}
274
+ return dic
275
+
276
+ def __len__(self):
277
+ if self.size is not None:
278
+ return min(len(self.fns), self.size)
279
+ else:
280
+ return len(self.fns)
281
+
282
+
283
+ class RealDataset(BaseDataset):
284
+ def __init__(self, datadir, fns=None, size=None):
285
+ super(RealDataset, self).__init__()
286
+ self.size = size
287
+ self.datadir = datadir
288
+ self.fns = fns or os.listdir(join(datadir))
289
+
290
+ if size is not None:
291
+ self.fns = self.fns[:size]
292
+
293
+ def align(self, x):
294
+ h, w = x.height, x.width
295
+ h, w = h // 32 * 32, w // 32 * 32
296
+ x = x.resize((w, h))
297
+ return x
298
+
299
+ def __getitem__(self, index):
300
+ fn = self.fns[index]
301
+ B = -1
302
+ m_img = Image.open(join(self.datadir, fn)).convert('RGB')
303
+ M = to_tensor(self.align(m_img))
304
+ data = {'input': M, 'target_t': B, 'fn': fn}
305
+ return data
306
+
307
+ def __len__(self):
308
+ if self.size is not None:
309
+ return min(len(self.fns), self.size)
310
+ else:
311
+ return len(self.fns)
312
+
313
+
314
+ class FusionDataset(BaseDataset):
315
+ def __init__(self, datasets, fusion_ratios=None, size=None):
316
+ self.datasets = datasets
317
+ self.size = size or sum([len(dataset) for dataset in datasets])
318
+ self.fusion_ratios = fusion_ratios or [1. / len(datasets)] * len(datasets)
319
+ print('[i] using a fusion dataset: %d %s imgs fused with ratio %s' % (
320
+ self.size, [len(dataset) for dataset in datasets], self.fusion_ratios))
321
+
322
+ def reset(self):
323
+ for dataset in self.datasets:
324
+ dataset.reset()
325
+
326
+ def __getitem__(self, index):
327
+ residual = 1
328
+ for i, ratio in enumerate(self.fusion_ratios):
329
+ if random.random() < ratio / residual or i == len(self.fusion_ratios) - 1:
330
+ dataset = self.datasets[i]
331
+ return dataset[index % len(dataset)]
332
+ residual -= ratio
333
+
334
+ def __len__(self):
335
+ return self.size
DSRNet/data/torchdata.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import bisect #maintains a list in sorted order without having sort after each insertion using bisection algorithm
2
+ import warnings #issue warning message
3
+
4
+
5
+ class Dataset(object): # all python classes inherit from object
6
+ """An abstract class representing a Dataset.
7
+
8
+ All other datasets should subclass it. All subclasses should override
9
+ ``__len__``, that provides the size of the dataset, and ``__getitem__``,
10
+ supporting integer indexing in range from 0 to len(self) exclusive.
11
+ """
12
+ #traditional overloading is not in python
13
+
14
+ def __getitem__(self, index): #__getitem__ for indexing with [ ] (overrriding)
15
+ raise NotImplementedError
16
+
17
+ def __len__(self): # for overriding len()
18
+ raise NotImplementedError #when you call len(obj), Python internally calls obj.__len__().
19
+
20
+ def __add__(self, other): #to override an binary + operator
21
+ return ConcatDataset([self, other])
22
+
23
+ def reset(self): #currently is a placeholder and does nothing
24
+ return
25
+
26
+
27
+ class ConcatDataset(Dataset): #inheriting from Dataset
28
+ """
29
+ Dataset to concatenate multiple datasets.
30
+ Purpose: useful to assemble different existing datasets, possibly
31
+ large-scale datasets as the concatenation operation is done in an
32
+ on-the-fly manner.
33
+
34
+ Arguments:
35
+ datasets (sequence): List of datasets to be concatenated
36
+ """
37
+
38
+ @staticmethod
39
+ def cumsum(sequence): #sequence: an ordered collection of items, where each item holds a relative position.Must be indexable and can be iterated. All sequences are iterables but reverse is not true e.g. sets not indexable
40
+ #sequence are strings, lists, tuples, byte sequences, byte arrays and range objects
41
+ r, s = [], 0 #r:list to hold cumulative sum of iterated datasets in sequence
42
+ #s: cumulative sum of all datasets in sequence
43
+ for e in sequence: #e: current dataset in sequence
44
+ l = len(e) # length of current dataset in sequence
45
+ r.append(l + s)
46
+ s += l
47
+ return r
48
+
49
+ def __init__(self, datasets):
50
+ super(ConcatDataset, self).__init__()
51
+ assert len(datasets) > 0, 'datasets should not be an empty iterable'
52
+ #halts the program and provides a diagnostic message.
53
+ #asserts are usually stripped out or disabled in production code, as they are meant for development and debugging
54
+ self.datasets = list(datasets) #store list of datasets
55
+ self.cumulative_sizes = self.cumsum(self.datasets) #store result of static method cumsum, list of cumulative length of previous datasets
56
+
57
+ def __len__(self):
58
+ return self.cumulative_sizes[-1] #length of concatenated datasets after cumsum
59
+
60
+ def __getitem__(self, idx): #dynamically selecting the correct dataset and item within that dataset based on the given index in the self.datasets
61
+ dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx) #bisect.bisect_right returns the index of the first element in the list that is greater than x (upper bound)
62
+ # line determines which dataset in self.datasets the item at idx belongs to.
63
+ if dataset_idx == 0: #idx belongs to first dataset
64
+ sample_idx = idx #so idx and cumulative idx same
65
+ else:#finds idx within the dataset
66
+ sample_idx = idx - self.cumulative_sizes[dataset_idx - 1]
67
+ return self.datasets[dataset_idx][sample_idx]
68
+
69
+ @property #property of the class. Can be accessed without ()
70
+ def cummulative_sizes(self): # basically a getter for encapsulation
71
+ warnings.warn("cummulative_sizes attribute is renamed to " #basically if user uses wrong function
72
+ "cumulative_sizes", DeprecationWarning, stacklevel=2) #If the stacklevel parameter is not set, the warning appears to originate from within the cummulative_sizes method itself.
73
+ #By setting stacklevel=2, the warning will appear to originate from the caller of the cummulative_sizes method.
74
+ return self.cumulative_sizes
DSRNet/data/transforms.py ADDED
@@ -0,0 +1,375 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import division #This ensures true division (i.e., division results in floating-point numbers) in Python 2 otherwise we get //
2
+
3
+ import math
4
+ import random
5
+
6
+ import kornia #for computer vision operations in PyTorch.
7
+ import torch
8
+ from PIL import Image
9
+
10
+ try:
11
+ import accimage #third-party Python library designed to provide accelerated image loading and processing capabilities, primarily intended for use with the PyTorch deep learning framework.Alternative of PIL/pillow
12
+ except ImportError:
13
+ accimage = None
14
+ import numpy as np
15
+ import scipy.stats as st
16
+ import cv2
17
+ import collections
18
+ import torchvision.transforms as transforms # for common image transformations.
19
+ import util.util as util
20
+ from scipy.signal import convolve2d
21
+
22
+
23
+ # utility
24
+ #
25
+ # PIL and accimage for loading
26
+ def _is_pil_image(img): #if accimage is not supported in environment
27
+ if accimage is not None:
28
+ return isinstance(img, (Image.Image, accimage.Image))
29
+ else:
30
+ return isinstance(img, Image.Image)
31
+
32
+ #in transformation step for augmentation and feeding into a model.
33
+ def _is_tensor_image(img): #to handle different types of image representations that might be encountered in various stages of image processing
34
+ return torch.is_tensor(img) and img.ndimension() == 3 # is 3D tensor
35
+
36
+ # for certain preprocessing steps.
37
+ def _is_numpy_image(img): #to handle different types of image representations that might be encountered in various stages of image processing
38
+ return isinstance(img, np.ndarray) and (img.ndim in {2, 3}) #is numpy array of 2 or 3 dimension
39
+
40
+
41
+ def arrshow(arr):
42
+ Image.fromarray(arr.astype(np.uint8)).show()#converts a NumPy array to a PIL image and then displays it.
43
+ #arr.astype(np.uint8) converts the NumPy array arr to have a data type of np.uint8
44
+ # PIL images expect pixel values to be in the range [0, 255],
45
+ #Image.fromarray() function is a method provided by the PIL (Pillow) library to convert a pixel values to a PIL image.
46
+ #.show() is a method of the PIL Image class to open image in image viewer of system
47
+
48
+ '''"transformations" refer to a series of operations applied to the images to preprocess or augment them. These transformations can include resizing, cropping, flipping, rotating, and many others. They help in normalizing the data, enhancing certain features, or augmenting the dataset to improve the robustness and performance of the model.'''
49
+
50
+ def get_transform(opt): #based on option given, a list of transformations are generated
51
+ transform_list = []
52
+ osizes = util.parse_args(opt.loadSize)
53
+ fineSize = util.parse_args(opt.fineSize)
54
+ if opt.resize_or_crop == 'resize_and_crop':
55
+ transform_list.append(
56
+ transforms.RandomChoice([
57
+ transforms.Resize([osize, osize], Image.BICUBIC) for osize in osizes
58
+ ]))
59
+ transform_list.append(transforms.RandomCrop(fineSize))
60
+ elif opt.resize_or_crop == 'crop':
61
+ transform_list.append(transforms.RandomCrop(fineSize))
62
+ elif opt.resize_or_crop == 'scale_width':
63
+ transform_list.append(transforms.Lambda(
64
+ lambda img: __scale_width(img, fineSize)))
65
+ elif opt.resize_or_crop == 'scale_width_and_crop':
66
+ transform_list.append(transforms.Lambda(
67
+ lambda img: __scale_width(img, opt.loadSize)))
68
+ transform_list.append(transforms.RandomCrop(opt.fineSize))
69
+
70
+ if opt.isTrain and not opt.no_flip:
71
+ transform_list.append(transforms.RandomHorizontalFlip())
72
+
73
+ return transforms.Compose(transform_list)
74
+
75
+
76
+ to_norm_tensor = transforms.Compose([
77
+ transforms.ToTensor(),
78
+ transforms.Normalize(
79
+ (0.5, 0.5, 0.5),
80
+ (0.5, 0.5, 0.5)
81
+ )
82
+ ])
83
+
84
+ to_tensor = transforms.ToTensor()
85
+
86
+
87
+ def __scale_width(img, target_width):
88
+ ow, oh = img.size
89
+ if (ow == target_width):
90
+ return img
91
+ w = target_width
92
+ h = int(target_width * oh / ow)
93
+ h = math.ceil(h / 2.) * 2 # round up to even
94
+ return img.resize((w, h), Image.BICUBIC)
95
+
96
+
97
+ # functional
98
+ def gaussian_blur(img, kernel_size, sigma):
99
+ if not _is_pil_image(img):
100
+ raise TypeError('img should be PIL Image. Got {}'.format(type(img)))
101
+
102
+ img = np.asarray(img)
103
+ # the 3rd dimension (i.e. inter-band) would be filtered which is unwanted for our purpose
104
+ # new = gaussian_filter(img, sigma=sigma, truncate=truncate)
105
+ if isinstance(kernel_size, int):
106
+ kernel_size = (kernel_size, kernel_size)
107
+ elif isinstance(kernel_size, collections.Sequence):
108
+ assert len(kernel_size) == 2
109
+ new = cv2.GaussianBlur(img, kernel_size, sigma) # apply gaussian filter band by band
110
+ return Image.fromarray(new)
111
+
112
+
113
+ # transforms
114
+ class GaussianBlur(object):
115
+ def __init__(self, kernel_size=11, sigma=3):
116
+ self.kernel_size = kernel_size
117
+ self.sigma = sigma
118
+
119
+ def __call__(self, img):
120
+ return gaussian_blur(img, self.kernel_size, self.sigma)
121
+
122
+
123
+ class ReflectionSythesis_0(object):
124
+ """Reflection image data synthesis for weakly-supervised learning
125
+ of ICCV 2017 paper *"A Generic Deep Architecture for Single Image Reflection Removal and Image Smoothing"*
126
+ """
127
+
128
+ def __init__(self, kernel_sizes=None, low_sigma=2, high_sigma=5, low_gamma=1.3,
129
+ high_gamma=1.3, low_delta=0.4, high_delta=1.8):
130
+ self.kernel_sizes = kernel_sizes or [11]
131
+ self.low_sigma = low_sigma
132
+ self.high_sigma = high_sigma
133
+ self.low_gamma = low_gamma
134
+ self.high_gamma = high_gamma
135
+ self.low_delta = low_delta
136
+ self.high_delta = high_delta
137
+ print('[i] reflection sythesis model: {}'.format({
138
+ 'kernel_sizes': kernel_sizes, 'low_sigma': low_sigma, 'high_sigma': high_sigma,
139
+ 'low_gamma': low_gamma, 'high_gamma': high_gamma}))
140
+
141
+ def __call__(self, B, R):
142
+ if not _is_pil_image(B):
143
+ raise TypeError('B should be PIL Image. Got {}'.format(type(B)))
144
+ if not _is_pil_image(R):
145
+ raise TypeError('R should be PIL Image. Got {}'.format(type(R)))
146
+ B_ = np.asarray(B, np.float32)
147
+ if random.random() < 0.4:
148
+ B_ = np.tile(np.random.uniform(0, 30, (1, 1, 1)), B_.shape) / 255.
149
+ else:
150
+ B_ = np.tile(np.random.normal(50, 50, (1, 1, 3)), (B_.shape[0], B_.shape[1], 1)).clip(0, 255) / 255.
151
+ R_ = np.asarray(R, np.float32) / 255.
152
+
153
+ kernel_size = np.random.choice(self.kernel_sizes)
154
+ sigma = np.random.uniform(self.low_sigma, self.high_sigma)
155
+ gamma = np.random.uniform(self.low_gamma, self.high_gamma)
156
+ delta = np.random.uniform(self.low_delta, self.high_delta)
157
+ R_blur = R_
158
+ kernel = cv2.getGaussianKernel(11, sigma)
159
+ kernel2d = np.dot(kernel, kernel.T)
160
+
161
+ for i in range(3):
162
+ R_blur[..., i] = convolve2d(R_blur[..., i], kernel2d, mode='same')
163
+
164
+ R_blur = np.clip(R_blur - np.mean(R_blur) * gamma, 0, 1)
165
+ R_blur = np.clip(R_blur * delta, 0, 1)
166
+ M_ = np.clip(R_blur + B_, 0, 1)
167
+
168
+ return B_, R_blur, M_
169
+
170
+
171
+ class ReflectionSythesis_1(object):
172
+ """Reflection image data synthesis for weakly-supervised learning
173
+ of ICCV 2017 paper *"A Generic Deep Architecture for Single Image Reflection Removal and Image Smoothing"*
174
+ """
175
+
176
+ def __init__(self, kernel_sizes=None, low_sigma=2, high_sigma=5, low_gamma=1.3, high_gamma=1.3):
177
+ self.kernel_sizes = kernel_sizes or [11]
178
+ self.low_sigma = low_sigma
179
+ self.high_sigma = high_sigma
180
+ self.low_gamma = low_gamma
181
+ self.high_gamma = high_gamma
182
+ print('[i] reflection sythesis model: {}'.format({
183
+ 'kernel_sizes': kernel_sizes, 'low_sigma': low_sigma, 'high_sigma': high_sigma,
184
+ 'low_gamma': low_gamma, 'high_gamma': high_gamma}))
185
+
186
+ def __call__(self, B, R):
187
+ if not _is_pil_image(B):
188
+ raise TypeError('B should be PIL Image. Got {}'.format(type(B)))
189
+ if not _is_pil_image(R):
190
+ raise TypeError('R should be PIL Image. Got {}'.format(type(R)))
191
+
192
+ B_ = np.asarray(B, np.float32) / 255.
193
+ R_ = np.asarray(R, np.float32) / 255.
194
+
195
+ kernel_size = np.random.choice(self.kernel_sizes)
196
+ sigma = np.random.uniform(self.low_sigma, self.high_sigma)
197
+ gamma = np.random.uniform(self.low_gamma, self.high_gamma)
198
+ R_blur = R_
199
+ kernel = cv2.getGaussianKernel(11, sigma)
200
+ kernel2d = np.dot(kernel, kernel.T)
201
+
202
+ for i in range(3):
203
+ R_blur[..., i] = convolve2d(R_blur[..., i], kernel2d, mode='same')
204
+
205
+ M_ = B_ + R_blur
206
+
207
+ if np.max(M_) > 1:
208
+ m = M_[M_ > 1]
209
+ m = (np.mean(m) - 1) * gamma
210
+ R_blur = np.clip(R_blur - m, 0, 1)
211
+ M_ = np.clip(R_blur + B_, 0, 1)
212
+
213
+ return B_, R_blur, M_
214
+
215
+
216
+ class NoiseReflectionSythesis(object):
217
+ """Reflection image data synthesis for weakly-supervised learning
218
+ of ICCV 2017 paper *"A Generic Deep Architecture for Single Image Reflection Removal and Image Smoothing"*
219
+ """
220
+
221
+ def __init__(self, kernel_sizes=None, low_sigma=2, high_sigma=5, low_gamma=1.3, high_gamma=1.3):
222
+ self.kernel_sizes = kernel_sizes or [11]
223
+ self.low_sigma = low_sigma
224
+ self.high_sigma = high_sigma
225
+ self.low_gamma = low_gamma
226
+ self.high_gamma = high_gamma
227
+ print('[i] reflection sythesis model: {}'.format({
228
+ 'kernel_sizes': kernel_sizes, 'low_sigma': low_sigma, 'high_sigma': high_sigma,
229
+ 'low_gamma': low_gamma, 'high_gamma': high_gamma}))
230
+
231
+ def __call__(self, B, R, N):
232
+ if not _is_pil_image(B):
233
+ raise TypeError('B should be PIL Image. Got {}'.format(type(B)))
234
+ if not _is_pil_image(R):
235
+ raise TypeError('R should be PIL Image. Got {}'.format(type(R)))
236
+
237
+ B_ = np.asarray(B, np.float32) / 255. + N
238
+ R_ = np.asarray(R, np.float32) / 255.
239
+
240
+ kernel_size = np.random.choice(self.kernel_sizes)
241
+ sigma = np.random.uniform(self.low_sigma, self.high_sigma)
242
+ gamma = np.random.uniform(self.low_gamma, self.high_gamma)
243
+ R_blur = R_
244
+ kernel = cv2.getGaussianKernel(11, sigma)
245
+ kernel2d = np.dot(kernel, kernel.T)
246
+
247
+ for i in range(3):
248
+ R_blur[..., i] = convolve2d(R_blur[..., i], kernel2d, mode='same')
249
+
250
+ M_ = B_ * 0.5 + R_blur * 0.5
251
+
252
+ return B_.astype(np.float32), R_blur.astype(np.float32), M_.astype(np.float32)
253
+
254
+
255
+ class NoiseReflectionSythesisTorch(object):
256
+ """Reflection image data synthesis for weakly-supervised learning
257
+ of ICCV 2017 paper *"A Generic Deep Architecture for Single Image Reflection Removal and Image Smoothing"*
258
+ """
259
+
260
+ def __init__(self, kernel_sizes=None, low_sigma=2, high_sigma=5, low_gamma=1.3, high_gamma=1.3):
261
+ self.kernel_sizes = kernel_sizes or [11]
262
+ self.low_sigma = low_sigma
263
+ self.high_sigma = high_sigma
264
+ self.low_gamma = low_gamma
265
+ self.high_gamma = high_gamma
266
+ print('[i] reflection sythesis model: {}'.format({
267
+ 'kernel_sizes': kernel_sizes, 'low_sigma': low_sigma, 'high_sigma': high_sigma,
268
+ 'low_gamma': low_gamma, 'high_gamma': high_gamma}))
269
+
270
+ def __call__(self, B, R):
271
+ stdN = np.random.uniform(15, 55)
272
+ noise = torch.zeros(B.size()).normal_(mean=0, std=stdN / 255.)
273
+ B = B + noise
274
+
275
+ sigma = np.random.uniform(self.low_sigma, self.high_sigma)
276
+ R = kornia.gaussian_blur2d(R.unsqueeze(0), (11, 11), (sigma, sigma), border_type='replicate').squeeze(0)
277
+ M = B * 0.5 + R * 0.5
278
+
279
+ return B, R, M
280
+
281
+
282
+ class Sobel(object):
283
+ def __call__(self, img):
284
+ if not _is_pil_image(img):
285
+ raise TypeError('img should be PIL Image. Got {}'.format(type(img)))
286
+
287
+ gray_img = np.array(img.convert('L'))
288
+ x = cv2.Sobel(gray_img, cv2.CV_16S, 1, 0)
289
+ y = cv2.Sobel(gray_img, cv2.CV_16S, 0, 1)
290
+
291
+ absX = cv2.convertScaleAbs(x)
292
+ absY = cv2.convertScaleAbs(y)
293
+
294
+ dst = cv2.addWeighted(absX, 0.5, absY, 0.5, 0)
295
+ return Image.fromarray(dst)
296
+
297
+
298
+ class ReflectionSythesis_2(object):
299
+ """Reflection image data synthesis for weakly-supervised learning
300
+ of CVPR 2018 paper *"Single Image Reflection Separation with Perceptual Losses"*
301
+ """
302
+
303
+ def __init__(self, kernel_sizes=None):
304
+ self.kernel_sizes = kernel_sizes or np.linspace(1, 5, 80)
305
+
306
+ @staticmethod
307
+ def gkern(kernlen=100, nsig=1):
308
+ """Returns a 2D Gaussian kernel array."""
309
+ interval = (2 * nsig + 1.) / (kernlen)
310
+ x = np.linspace(-nsig - interval / 2., nsig + interval / 2., kernlen + 1)
311
+ kern1d = np.diff(st.norm.cdf(x))
312
+ kernel_raw = np.sqrt(np.outer(kern1d, kern1d))
313
+ kernel = kernel_raw / kernel_raw.sum()
314
+ kernel = kernel / kernel.max()
315
+ return kernel
316
+
317
+ def __call__(self, t, r):
318
+ t = np.float32(t) / 255.
319
+ r = np.float32(r) / 255.
320
+ ori_t = t
321
+ # create a vignetting mask
322
+ g_mask = self.gkern(560, 3)
323
+ g_mask = np.dstack((g_mask, g_mask, g_mask))
324
+ sigma = self.kernel_sizes[np.random.randint(0, len(self.kernel_sizes))]
325
+
326
+ t = np.power(t, 2.2)
327
+ r = np.power(r, 2.2)
328
+
329
+ sz = int(2 * np.ceil(2 * sigma) + 1)
330
+
331
+ r_blur = cv2.GaussianBlur(r, (sz, sz), sigma, sigma, 0)
332
+ blend = r_blur + t
333
+
334
+ att = 1.08 + np.random.random() / 10.0
335
+
336
+ for i in range(3):
337
+ maski = blend[:, :, i] > 1
338
+ mean_i = max(1., np.sum(blend[:, :, i] * maski) / (maski.sum() + 1e-6))
339
+ r_blur[:, :, i] = r_blur[:, :, i] - (mean_i - 1) * att
340
+ r_blur[r_blur >= 1] = 1
341
+ r_blur[r_blur <= 0] = 0
342
+
343
+ h, w = r_blur.shape[0:2]
344
+ neww = np.random.randint(0, 560 - w - 10)
345
+ newh = np.random.randint(0, 560 - h - 10)
346
+ alpha1 = g_mask[newh:newh + h, neww:neww + w, :]
347
+ alpha2 = 1 - np.random.random() / 5.0
348
+ r_blur_mask = np.multiply(r_blur, alpha1)
349
+ blend = r_blur_mask + t * alpha2
350
+
351
+ t = np.power(t, 1 / 2.2)
352
+ r_blur_mask = np.power(r_blur_mask, 1 / 2.2)
353
+ blend = np.power(blend, 1 / 2.2)
354
+ blend[blend >= 1] = 1
355
+ blend[blend <= 0] = 0
356
+
357
+ return np.float32(ori_t), np.float32(r_blur_mask), np.float32(blend)
358
+
359
+
360
+ # Examples
361
+ if __name__ == '__main__':
362
+ """cv2 imread"""
363
+ # img = cv2.imread('testdata_reflection_real/19-input.png')
364
+ # img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
365
+ # img2 = cv2.GaussianBlur(img, (11,11), 3)
366
+
367
+ """Sobel Operator"""
368
+ # img = np.array(Image.open('datasets/VOC224/train/B/2007_000250.png').convert('L'))
369
+
370
+ """Reflection Sythesis"""
371
+ b = Image.open('')
372
+ r = Image.open('')
373
+ G = ReflectionSythesis_0()
374
+ m, r = G(b, r)
375
+ r.show()
DSRNet/engine.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import util.util as util
3
+ from models import make_model
4
+ import time
5
+ import os
6
+ import sys
7
+ from os.path import join
8
+ from util.visualizer import Visualizer
9
+
10
+
11
+ class Engine(object):
12
+ def __init__(self, opt):
13
+ self.opt = opt
14
+ self.writer = None
15
+ self.visualizer = None
16
+ self.model = None
17
+ self.best_val_loss = 1e6
18
+
19
+ self.__setup()
20
+
21
+ def __setup(self):#private methods prefixed with __
22
+ self.basedir = join('checkpoints', self.opt.name) #creating directory path for checkpoint files
23
+ os.makedirs(self.basedir, exist_ok=True)
24
+
25
+ opt = self.opt
26
+
27
+ """(mingcv)Model"""
28
+ self.model = make_model(self.opt.model)() #(mingcv)models.__dict__[self.opt.model]()
29
+ self.model.initialize(opt)
30
+ if not opt.no_log: # setting up logging and visualisation
31
+ self.writer = util.get_summary_writer(os.path.join(self.basedir, 'logs'))
32
+ self.visualizer = Visualizer(opt)
33
+
34
+ def train(self, train_loader, **kwargs):
35
+ print('\nEpoch: %d' % self.epoch)
36
+ avg_meters = util.AverageMeters()#generating running averga eof metrics
37
+ opt = self.opt
38
+ model = self.model
39
+ epoch = self.epoch
40
+
41
+ epoch_start_time = time.time()
42
+ for i, data in enumerate(train_loader):
43
+ iter_start_time = time.time()
44
+ iterations = self.iterations
45
+
46
+ model.set_input(data, mode='train')
47
+ model.optimize_parameters(**kwargs)
48
+
49
+ errors = model.get_current_errors()
50
+ avg_meters.update(errors)
51
+ util.progress_bar(i, len(train_loader), str(avg_meters))
52
+
53
+ if not opt.no_log:#update logging and visualisation every iteration of training
54
+ util.write_loss(self.writer, 'train', avg_meters, iterations)
55
+
56
+ if iterations % opt.display_freq == 0 and opt.display_id != 0:
57
+ save_result = iterations % opt.update_html_freq == 0
58
+ self.visualizer.display_current_results(model.get_current_visuals(), epoch, save_result)
59
+
60
+ if iterations % opt.print_freq == 0 and opt.display_id != 0:
61
+ t = (time.time() - iter_start_time)
62
+
63
+ self.iterations += 1
64
+
65
+ self.epoch += 1
66
+
67
+ if not self.opt.no_log:#
68
+ if self.epoch % opt.save_epoch_freq == 0:
69
+ print('saving the model at epoch %d, iters %d' %
70
+ (self.epoch, self.iterations))
71
+ model.save()
72
+
73
+ print('saving the latest model at the end of epoch %d, iters %d' %
74
+ (self.epoch, self.iterations))
75
+ model.save(label='latest')
76
+
77
+ print('Time Taken: %d sec' %
78
+ (time.time() - epoch_start_time))
79
+
80
+ # model.update_learning_rate()
81
+ try:
82
+ train_loader.reset()
83
+ except:
84
+ pass
85
+
86
+ def eval(self, val_loader, dataset_name, savedir='./tmp', loss_key=None, max_save_size=None, **kwargs):
87
+ #(mingcv)print(dataset_name)
88
+ if savedir is not None:# create a directory for saving if not exist
89
+ os.makedirs(savedir, exist_ok=True)
90
+ self.f = open(os.path.join(savedir, 'metrics.txt'), 'w+')
91
+ self.f.write(dataset_name + '\n')
92
+ avg_meters = util.AverageMeters()#get running average metrics
93
+ model = self.model
94
+ opt = self.opt
95
+ with torch.no_grad():#disable gradient calculation to save memory
96
+ for i, data in enumerate(val_loader):
97
+ if opt.selected and data['fn'][0].split('.')[0] not in opt.selected:
98
+ continue
99
+ if max_save_size is not None and i > max_save_size:
100
+ index = model.eval(data, savedir=None, **kwargs)
101
+ else:
102
+ index = model.eval(data, savedir=savedir, **kwargs)
103
+
104
+ #(mingcv) print(data['fn'][0], index)
105
+ if savedir is not None:
106
+ self.f.write(f"{data['fn'][0]} {index['PSNR']} {index['SSIM']}\n")
107
+ avg_meters.update(index)
108
+ util.progress_bar(i, len(val_loader), str(avg_meters))
109
+
110
+ if not opt.no_log:
111
+ util.write_loss(self.writer, join('eval', dataset_name), avg_meters, self.epoch)
112
+
113
+ if loss_key is not None:
114
+ val_loss = avg_meters[loss_key]
115
+ if val_loss < self.best_val_loss:
116
+ self.best_val_loss = val_loss
117
+ print('saving the best model at the end of epoch %d, iters %d' %
118
+ (self.epoch, self.iterations))
119
+ model.save(label='best_{}_{}'.format(loss_key, dataset_name))
120
+
121
+ return avg_meters
122
+
123
+ def test(self, test_loader, savedir=None, **kwargs):
124
+ model = self.model
125
+ opt = self.opt
126
+ with torch.no_grad():
127
+ for i, data in enumerate(test_loader):
128
+ model.test(data, savedir=savedir, **kwargs)
129
+ util.progress_bar(i, len(test_loader))
130
+
131
+ def save_model(self):
132
+ self.model.save()
133
+
134
+ def save_eval(self, label):
135
+ self.model.save_eval(label)
136
+
137
+ @property
138
+ def iterations(self):
139
+ return self.model.iterations
140
+
141
+ @iterations.setter
142
+ def iterations(self, i):
143
+ self.model.iterations = i
144
+
145
+ @property
146
+ def epoch(self):
147
+ return self.model.epoch
148
+
149
+ @epoch.setter
150
+ def epoch(self, e):
151
+ self.model.epoch = e
DSRNet/eval_sirs.py ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from os.path import join
3
+
4
+ import torch.backends.cudnn as cudnn
5
+
6
+ import data.sirs_dataset as datasets
7
+ from data.image_folder import read_fns
8
+ from engine import Engine
9
+ from options.net_options.train_options import TrainOptions
10
+ from tools import mutils
11
+
12
+ opt = TrainOptions().parse() #parses training options from command-line arguments using the TrainOptions class and stores them in opt
13
+ #evaluation settings:
14
+ opt.isTrain = False
15
+ cudnn.benchmark = True
16
+ opt.no_log = True
17
+ opt.display_id = 0
18
+ opt.verbose = False
19
+
20
+ datadir = os.path.join(opt.base_dir, 'test')# test data directory
21
+
22
+ #(mingcv) Define evaluation/test dataset
23
+ eval_dataset_real = datasets.DSRTestDataset(join(datadir, f'real20_420'),
24
+ fns=read_fns('data/real_test.txt'),
25
+ if_align=opt.if_align)
26
+ eval_dataset_postcard = datasets.DSRTestDataset(join(datadir, 'SIR2/PostcardDataset'),
27
+ if_align=opt.if_align)
28
+ eval_dataset_solidobject = datasets.DSRTestDataset(join(datadir, 'SIR2/SolidObjectDataset'),
29
+ if_align=opt.if_align)
30
+ eval_dataset_wild = datasets.DSRTestDataset(join(datadir, 'SIR2/WildSceneDataset'),
31
+ if_align=opt.if_align)
32
+
33
+ eval_dataloader_real = datasets.DataLoader(
34
+ eval_dataset_real, batch_size=1, shuffle=True,
35
+ num_workers=opt.nThreads, pin_memory=True)
36
+
37
+ eval_dataloader_solidobject = datasets.DataLoader(
38
+ eval_dataset_solidobject, batch_size=1, shuffle=False,
39
+ num_workers=opt.nThreads, pin_memory=True)
40
+
41
+ eval_dataloader_postcard = datasets.DataLoader(
42
+ eval_dataset_postcard, batch_size=1, shuffle=False,
43
+ num_workers=opt.nThreads, pin_memory=True)
44
+
45
+ eval_dataloader_wild = datasets.DataLoader(
46
+ eval_dataset_wild, batch_size=1, shuffle=False,
47
+ num_workers=opt.nThreads, pin_memory=True)
48
+
49
+ engine = Engine(opt)
50
+
51
+ """Main Loop"""
52
+ result_dir = os.path.join('./checkpoints', opt.name, mutils.get_formatted_time())
53
+
54
+ #
55
+ res = engine.eval(eval_dataloader_real, dataset_name='testdata_real',
56
+ savedir=join(result_dir, 'real20'), suffix='real20')
57
+ print(res)
58
+
59
+ res = engine.eval(eval_dataloader_solidobject, dataset_name='testdata_solidobject',
60
+ savedir=join(result_dir, 'solidobject'), suffix='solidobject')
61
+ print(res)
62
+ res = engine.eval(eval_dataloader_postcard, dataset_name='testdata_postcard',
63
+ savedir=join(result_dir, 'postcard'), suffix='postcard')
64
+ print(res)
65
+ res = engine.eval(eval_dataloader_wild, dataset_name='testdata_wild',
66
+ savedir=join(result_dir, 'wild'), suffix='wild')
67
+ print(res)
DSRNet/eval_sirs_4000 ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from os.path import join
3
+
4
+ import torch.backends.cudnn as cudnn
5
+
6
+ import data.sirs_dataset as datasets
7
+ from data.image_folder import read_fns
8
+ from engine import Engine
9
+ from options.net_options.train_options import TrainOptions
10
+ from tools import mutils
11
+
12
+ opt = TrainOptions().parse()
13
+
14
+ opt.isTrain = False
15
+ cudnn.benchmark = True
16
+ opt.no_log = True
17
+ opt.display_id = 0
18
+ opt.verbose = False
19
+
20
+ datadir = os.path.join(opt.base_dir, 'test')
21
+
22
+ # Define evaluation/test dataset
23
+ eval_dataset_real = datasets.DSRTestDataset(join(datadir, f'real20_420'),
24
+ fns=read_fns('data/real_test.txt'),
25
+ if_align=opt.if_align)
26
+ eval_dataset_postcard = datasets.DSRTestDataset(join(datadir, 'SIR2/PostcardDataset'),
27
+ if_align=opt.if_align)
28
+ eval_dataset_solidobject = datasets.DSRTestDataset(join(datadir, 'SIR2/SolidObjectDataset'),
29
+ if_align=opt.if_align)
30
+ eval_dataset_wild = datasets.DSRTestDataset(join(datadir, 'SIR2/WildSceneDataset'),
31
+ if_align=opt.if_align)
32
+
33
+ eval_dataset_nature = datasets.DSRTestDataset(join(datadir, 'Nature'), if_align=opt.if_align)
34
+
35
+ eval_dataloader_real = datasets.DataLoader(
36
+ eval_dataset_real, batch_size=1, shuffle=True,
37
+ num_workers=opt.nThreads, pin_memory=True)
38
+
39
+ eval_dataloader_solidobject = datasets.DataLoader(
40
+ eval_dataset_solidobject, batch_size=1, shuffle=False,
41
+ num_workers=opt.nThreads, pin_memory=True)
42
+
43
+ eval_dataloader_postcard = datasets.DataLoader(
44
+ eval_dataset_postcard, batch_size=1, shuffle=False,
45
+ num_workers=opt.nThreads, pin_memory=True)
46
+
47
+ eval_dataloader_wild = datasets.DataLoader(
48
+ eval_dataset_wild, batch_size=1, shuffle=False,
49
+ num_workers=opt.nThreads, pin_memory=True)
50
+
51
+ eval_dataloader_nature = datasets.DataLoader(
52
+ eval_dataset_nature, batch_size=1, shuffle=False,
53
+ num_workers=opt.nThreads, pin_memory=True)
54
+ engine = Engine(opt)
55
+
56
+ """Main Loop"""
57
+ result_dir = os.path.join('./checkpoints', opt.name, mutils.get_formatted_time())
58
+
59
+ #
60
+ res = engine.eval(eval_dataloader_real, dataset_name='testdata_real',
61
+ savedir=join(result_dir, 'real20'), suffix='real20')
62
+ print(res)
63
+
64
+ res = engine.eval(eval_dataloader_solidobject, dataset_name='testdata_solidobject',
65
+ savedir=join(result_dir, 'solidobject'), suffix='solidobject')
66
+ print(res)
67
+ res = engine.eval(eval_dataloader_postcard, dataset_name='testdata_postcard',
68
+ savedir=join(result_dir, 'postcard'), suffix='postcard')
69
+ print(res)
70
+ res = engine.eval(eval_dataloader_wild, dataset_name='testdata_wild',
71
+ savedir=join(result_dir, 'wild'), suffix='wild')
72
+ print(res)
73
+ res = engine.eval(eval_dataloader_wild, dataset_name='testdata_nature',
74
+ savedir=join(result_dir, 'nature'), suffix='nature')
75
+ print(res)
DSRNet/models/__init__.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ import importlib
2
+
3
+ import models.losses as losses
4
+ from models.arch import *
5
+
6
+
7
+ def make_model(name: str):
8
+ base = importlib.import_module('models.' + name)
9
+ model = getattr(base, 'DSRNetModel')
10
+ return model
DSRNet/models/__pycache__/__init__.cpython-38.pyc ADDED
Binary file (1.21 kB). View file
 
DSRNet/models/__pycache__/networks.cpython-38.pyc ADDED
Binary file (7.28 kB). View file
 
DSRNet/models/__pycache__/vgg.cpython-38.pyc ADDED
Binary file (2.15 kB). View file
 
DSRNet/models/arch/__init__.py ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+
3
+ from models.arch.dsrnet import DSRNet
4
+
5
+
6
+ def dsrnet_s(in_channels=3, out_channels=3, width=32):
7
+ enc_blks = [2, 2, 2]
8
+ middle_blk_num = 4
9
+ dec_blks = [2, 2, 2]
10
+
11
+ return DSRNet(in_channels, out_channels, width=width,
12
+ middle_blk_num=middle_blk_num,
13
+ enc_blk_nums=enc_blks,
14
+ dec_blk_nums=dec_blks,
15
+ shared_b=False)
16
+
17
+
18
+ def dsrnet_l(in_channels=3, out_channels=3, width=64):
19
+ enc_blks = [2, 2, 4, 8]
20
+ middle_blk_num = 12
21
+ dec_blks = [2, 2, 2, 2]
22
+
23
+ return DSRNet(in_channels, out_channels, width=width,
24
+ middle_blk_num=middle_blk_num,
25
+ enc_blk_nums=enc_blks,
26
+ dec_blk_nums=dec_blks,
27
+ shared_b=True)
28
+
29
+ if __name__ == '__main__':
30
+ from tools import mutils
31
+
32
+ x = torch.ones(1, 3, 256, 256).cuda()
33
+ feats = [
34
+ torch.ones(1, 64, 128, 128).cuda(),
35
+ torch.ones(1, 192, 64, 64).cuda(),
36
+ torch.ones(1, 384, 32, 32).cuda(),
37
+ torch.ones(1, 768, 16, 16).cuda(),
38
+ torch.ones(1, 2560, 8, 8).cuda(),
39
+ ]
40
+ enc_blks = [2, 2, 4, 8]
41
+ middle_blk_num = 12
42
+ dec_blks = [2, 2, 2, 2]
43
+
44
+ model = dsrnet(3, 3).cuda()
45
+ mutils.count_parameters(model)
46
+ mutils.count_parameters(model.intro)
47
+
48
+ out_l, out_r = model(x, feats)
49
+ print(out_l.shape, out_r.shape)
DSRNet/models/arch/__pycache__/__init__.cpython-38.pyc ADDED
Binary file (17.4 kB). View file
 
DSRNet/models/arch/dsrnet.py ADDED
@@ -0,0 +1,306 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from collections import OrderedDict
2
+
3
+ import torch
4
+ import torch.nn as nn
5
+ from models.arch.lrm import LRM
6
+
7
+
8
+ class LayerNormFunction(torch.autograd.Function):
9
+
10
+ @staticmethod
11
+ def forward(ctx, x, weight, bias, eps):
12
+ ctx.eps = eps
13
+ N, C, H, W = x.size()
14
+ mu = x.mean(1, keepdim=True)
15
+ var = (x - mu).pow(2).mean(1, keepdim=True)
16
+ y = (x - mu) / (var + eps).sqrt()
17
+ ctx.save_for_backward(y, var, weight)
18
+ y = weight.view(1, C, 1, 1) * y + bias.view(1, C, 1, 1)
19
+ return y
20
+
21
+ @staticmethod
22
+ def backward(ctx, grad_output):
23
+ eps = ctx.eps
24
+
25
+ N, C, H, W = grad_output.size()
26
+ y, var, weight = ctx.saved_tensors
27
+ g = grad_output * weight.view(1, C, 1, 1)
28
+ mean_g = g.mean(dim=1, keepdim=True)
29
+
30
+ mean_gy = (g * y).mean(dim=1, keepdim=True)
31
+ gx = 1. / torch.sqrt(var + eps) * (g - y * mean_gy - mean_g)
32
+ return gx, (grad_output * y).sum(dim=3).sum(dim=2).sum(dim=0), grad_output.sum(dim=3).sum(dim=2).sum(
33
+ dim=0), None
34
+
35
+
36
+ class LayerNorm2d(nn.Module):
37
+
38
+ def __init__(self, channels, eps=1e-6):
39
+ super(LayerNorm2d, self).__init__()
40
+ self.register_parameter('weight', nn.Parameter(torch.ones(channels)))
41
+ self.register_parameter('bias', nn.Parameter(torch.zeros(channels)))
42
+ self.eps = eps
43
+
44
+ def forward(self, x):
45
+ return LayerNormFunction.apply(x, self.weight, self.bias, self.eps)
46
+
47
+
48
+ class CABlock(nn.Module):
49
+ def __init__(self, channels):
50
+ super(CABlock, self).__init__()
51
+ self.ca = nn.Sequential(
52
+ nn.AdaptiveAvgPool2d(1),
53
+ nn.Conv2d(channels, channels, 1)
54
+ )
55
+
56
+ def forward(self, x):
57
+ return x * self.ca(x)
58
+
59
+
60
+ class DualStreamGate(nn.Module):
61
+ def forward(self, x, y):
62
+ x1, x2 = x.chunk(2, dim=1)
63
+ y1, y2 = y.chunk(2, dim=1)
64
+ return x1 * y2, y1 * x2
65
+
66
+
67
+ class DualStreamSeq(nn.Sequential):
68
+ def forward(self, x, y=None):
69
+ y = y if y is not None else x
70
+ for module in self:
71
+ x, y = module(x, y)
72
+ return x, y
73
+
74
+
75
+ class DualStreamBlock(nn.Module):
76
+ def __init__(self, *args):
77
+ super(DualStreamBlock, self).__init__()
78
+ self.seq = nn.Sequential()
79
+
80
+ if len(args) == 1 and isinstance(args[0], OrderedDict):
81
+ for key, module in args[0].items():
82
+ self.seq.add_module(key, module)
83
+ else:
84
+ for idx, module in enumerate(args):
85
+ self.seq.add_module(str(idx), module)
86
+
87
+ def forward(self, x, y):
88
+ return self.seq(x), self.seq(y)
89
+
90
+
91
+ class MuGIBlock(nn.Module):
92
+ def __init__(self, c, shared_b=False):
93
+ super().__init__()
94
+ self.block1 = DualStreamSeq(
95
+ DualStreamBlock(
96
+ LayerNorm2d(c),
97
+ nn.Conv2d(c, c * 2, 1),
98
+ nn.Conv2d(c * 2, c * 2, 3, padding=1, groups=c * 2)
99
+ ),
100
+ DualStreamGate(),
101
+ DualStreamBlock(CABlock(c)),
102
+ DualStreamBlock(nn.Conv2d(c, c, 1))
103
+ )
104
+
105
+ self.a_l = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
106
+ self.a_r = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
107
+
108
+ self.block2 = DualStreamSeq(
109
+ DualStreamBlock(
110
+ LayerNorm2d(c),
111
+ nn.Conv2d(c, c * 2, 1)
112
+ ),
113
+ DualStreamGate(),
114
+ DualStreamBlock(
115
+ nn.Conv2d(c, c, 1)
116
+ )
117
+
118
+ )
119
+
120
+ self.shared_b = shared_b
121
+ if shared_b:
122
+ self.b = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
123
+ else:
124
+ self.b_l = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
125
+ self.b_r = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
126
+
127
+ def forward(self, inp_l, inp_r):
128
+ x, y = self.block1(inp_l, inp_r)
129
+ x_skip, y_skip = inp_l + x * self.a_l, inp_r + y * self.a_r
130
+ x, y = self.block2(x_skip, y_skip)
131
+ if self.shared_b:
132
+ out_l, out_r = x_skip + x * self.b, y_skip + y * self.b
133
+ else:
134
+ out_l, out_r = x_skip + x * self.b_l, y_skip + y * self.b_r
135
+ return out_l, out_r
136
+
137
+
138
+ class FeaturePyramidVGG(nn.Module):
139
+ def __init__(self, out_channels=64, shared_b=False):
140
+ super().__init__()
141
+ self.device = 'cuda'
142
+ self.block5 = DualStreamSeq(
143
+ MuGIBlock(512, shared_b),
144
+ DualStreamBlock(nn.UpsamplingBilinear2d(scale_factor=2.0)),
145
+ )
146
+
147
+ self.block4 = DualStreamSeq(
148
+ MuGIBlock(512, shared_b)
149
+ )
150
+
151
+ self.ch_map4 = DualStreamSeq(
152
+ DualStreamBlock(
153
+ nn.Conv2d(in_channels=1024, out_channels=1024, kernel_size=1),
154
+ nn.PixelShuffle(2)),
155
+ MuGIBlock(256, shared_b)
156
+ )
157
+
158
+ self.block3 = DualStreamSeq(
159
+ MuGIBlock(256, shared_b)
160
+ )
161
+
162
+ self.ch_map3 = DualStreamSeq(
163
+ DualStreamBlock(
164
+ nn.Conv2d(in_channels=512, out_channels=512, kernel_size=1),
165
+ nn.PixelShuffle(2)),
166
+ MuGIBlock(128, shared_b)
167
+ )
168
+
169
+ self.block2 = DualStreamSeq(
170
+ MuGIBlock(128, shared_b)
171
+ )
172
+
173
+ self.ch_map2 = DualStreamSeq(
174
+ DualStreamBlock(
175
+ nn.Conv2d(in_channels=256, out_channels=256, kernel_size=1),
176
+ nn.PixelShuffle(2)),
177
+ MuGIBlock(64, shared_b)
178
+ )
179
+
180
+ self.block1 = DualStreamSeq(
181
+ MuGIBlock(64, shared_b),
182
+ )
183
+
184
+ self.ch_map1 = DualStreamSeq(
185
+ DualStreamBlock(nn.Conv2d(in_channels=128, out_channels=128, kernel_size=1)),
186
+ MuGIBlock(128, shared_b),
187
+ DualStreamBlock(nn.Conv2d(in_channels=128, out_channels=32, kernel_size=1)),
188
+ MuGIBlock(32, shared_b),
189
+ )
190
+
191
+ self.block_intro = DualStreamSeq(
192
+ DualStreamBlock(nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding=1)),
193
+ MuGIBlock(32, shared_b)
194
+ )
195
+
196
+ self.ch_map0 = DualStreamBlock(
197
+ nn.Conv2d(in_channels=64, out_channels=out_channels, kernel_size=1)
198
+ )
199
+
200
+ def forward(self, inp, vgg_feats):
201
+ # 64,128,256,512,512
202
+ vf1, vf2, vf3, vf4, vf5 = vgg_feats
203
+ # 512=>512,512=>512
204
+ f5_l, f5_r = self.block5(vf5)
205
+ f4_l, f4_r = self.block4(vf4)
206
+ f4_l, f4_r = self.ch_map4(torch.cat([f5_l, f4_l], dim=1), torch.cat([f5_r, f4_r], dim=1))
207
+ # 256 => 256
208
+ f3_l, f3_r = self.block3(vf3)
209
+ # (256+256,256+256)->(128,128)
210
+ f3_l, f3_r = self.ch_map3(torch.cat([f4_l, f3_l], dim=1), torch.cat([f4_r, f3_r], dim=1))
211
+ # (128+128,128+128)->(64,64)
212
+ f2_l, f2_r = self.block2(vf2)
213
+ f2_l, f2_r = self.ch_map2(torch.cat([f3_l, f2_l], dim=1), torch.cat([f3_r, f2_r], dim=1))
214
+ # (64+64,64+64)->(32,32)
215
+ f1_l, f1_r = self.block1(vf1)
216
+ f1_l, f1_r = self.ch_map1(torch.cat([f2_l, f1_l], dim=1), torch.cat([f2_r, f1_r], dim=1))
217
+ # 64
218
+ f0_l, f0_r = self.block_intro(inp, inp)
219
+ f0_l, f0_r = self.ch_map0(torch.cat([f1_l, f0_l], dim=1), torch.cat([f1_r, f0_r], dim=1))
220
+ return f0_l, f0_r
221
+
222
+
223
+ class DSRNet(nn.Module):
224
+ def __init__(self, in_channels=3, out_channels=3, width=48, middle_blk_num=1,
225
+ enc_blk_nums=[], dec_blk_nums=[], shared_b=False):
226
+ super().__init__()
227
+ self.intro = FeaturePyramidVGG(width, shared_b)
228
+ self.ending = DualStreamBlock(nn.Conv2d(width, out_channels, 3, padding=1))
229
+ self.encoders = nn.ModuleList()
230
+ self.decoders = nn.ModuleList()
231
+ self.middle_blks = nn.ModuleList()
232
+ self.ups = nn.ModuleList()
233
+ self.downs = nn.ModuleList()
234
+ self.lrm = LRM(width)
235
+
236
+ c = width
237
+ for num in enc_blk_nums:
238
+ self.encoders.append(
239
+ DualStreamSeq(
240
+ *[MuGIBlock(c, shared_b) for _ in range(num)]
241
+ )
242
+ )
243
+ self.downs.append(
244
+ DualStreamBlock(
245
+ nn.Conv2d(c, c * 2, 2, 2)
246
+ )
247
+ )
248
+ c *= 2
249
+
250
+ self.middle_blks = DualStreamSeq(
251
+ *[MuGIBlock(c, shared_b) for _ in range(middle_blk_num)]
252
+ )
253
+
254
+ for num in dec_blk_nums:
255
+ self.ups.append(
256
+ DualStreamBlock(
257
+ nn.Conv2d(c, c * 2, 1, bias=False),
258
+ nn.PixelShuffle(2)
259
+ )
260
+ )
261
+ c //= 2
262
+
263
+ self.decoders.append(
264
+ DualStreamSeq(
265
+ *[MuGIBlock(c, shared_b) for _ in range(num)]
266
+ )
267
+ )
268
+
269
+ def forward(self, inp, feats_inp, fn=None):
270
+ *_, H, W = inp.shape
271
+ x, y = self.intro(inp, feats_inp)
272
+ encs = []
273
+ for encoder, down in zip(self.encoders, self.downs):
274
+ x, y = encoder(x, y)
275
+ encs.append((x, y))
276
+ x, y = down(x, y)
277
+
278
+ x, y = self.middle_blks(x, y)
279
+
280
+ for decoder, up, (enc_x_skip, enc_y_skip) in zip(self.decoders, self.ups, encs[::-1]):
281
+ x, y = up(x, y)
282
+ x, y = x + enc_x_skip, y + enc_y_skip
283
+ x, y = decoder(x, y)
284
+
285
+ rr = self.lrm(x, y)
286
+ x, y = self.ending(x, y)
287
+ return x, y, rr
288
+
289
+
290
+ if __name__ == '__main__':
291
+ x = torch.ones(1, 3, 224, 224).cuda()
292
+ feats = [
293
+ torch.ones(1, 64, 224, 224).cuda(),
294
+ torch.ones(1, 128, 112, 112).cuda(),
295
+ torch.ones(1, 256, 56, 56).cuda(),
296
+ torch.ones(1, 512, 28, 28).cuda(),
297
+ torch.ones(1, 512, 14, 14).cuda(),
298
+ ]
299
+
300
+ enc_blks = [2, 2, 2]
301
+ middle_blk_num = 4
302
+ dec_blks = [2, 2, 2]
303
+ model = DSRNet(3, 3, width=32, middle_blk_num=middle_blk_num,
304
+ enc_blk_nums=enc_blks, dec_blk_nums=dec_blks).cuda()
305
+ out_t, out_r, out_rr = model(x, feats)
306
+ print(out_t.shape, out_r.shape, out_rr.shape)
DSRNet/models/arch/lrm.py ADDED
@@ -0,0 +1,185 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ from collections import OrderedDict
4
+
5
+
6
+ class LayerNormFunction(torch.autograd.Function):
7
+
8
+ @staticmethod
9
+ def forward(ctx, x, weight, bias, eps):
10
+ ctx.eps = eps
11
+ N, C, H, W = x.size()
12
+ mu = x.mean(1, keepdim=True)
13
+ var = (x - mu).pow(2).mean(1, keepdim=True)
14
+ y = (x - mu) / (var + eps).sqrt()
15
+ ctx.save_for_backward(y, var, weight)
16
+ y = weight.view(1, C, 1, 1) * y + bias.view(1, C, 1, 1)
17
+ return y
18
+
19
+ @staticmethod
20
+ def backward(ctx, grad_output):
21
+ eps = ctx.eps
22
+
23
+ N, C, H, W = grad_output.size()
24
+ y, var, weight = ctx.saved_tensors
25
+ g = grad_output * weight.view(1, C, 1, 1)
26
+ mean_g = g.mean(dim=1, keepdim=True)
27
+
28
+ mean_gy = (g * y).mean(dim=1, keepdim=True)
29
+ gx = 1. / torch.sqrt(var + eps) * (g - y * mean_gy - mean_g)
30
+ return gx, (grad_output * y).sum(dim=3).sum(dim=2).sum(dim=0), grad_output.sum(dim=3).sum(dim=2).sum(
31
+ dim=0), None
32
+
33
+
34
+ class LayerNorm2d(nn.Module):
35
+
36
+ def __init__(self, channels, eps=1e-6):
37
+ super(LayerNorm2d, self).__init__()
38
+ self.register_parameter('weight', nn.Parameter(torch.ones(channels)))
39
+ self.register_parameter('bias', nn.Parameter(torch.zeros(channels)))
40
+ self.eps = eps
41
+
42
+ def forward(self, x):
43
+ return LayerNormFunction.apply(x, self.weight, self.bias, self.eps)
44
+
45
+
46
+ class SimpleGate(nn.Module):
47
+ def forward(self, x):
48
+ x1, x2 = x.chunk(2, dim=1)
49
+ return x1 * x2
50
+
51
+
52
+ class CABlock(nn.Module):
53
+ def __init__(self, channels):
54
+ super(CABlock, self).__init__()
55
+ self.ca = nn.Sequential(
56
+ nn.AdaptiveAvgPool2d(1),
57
+ nn.Conv2d(channels, channels, 1)
58
+ )
59
+
60
+ def forward(self, x):
61
+ return x * self.ca(x)
62
+
63
+
64
+ class DualStreamGate(nn.Module):
65
+ def forward(self, x, y):
66
+ x1, x2 = x.chunk(2, dim=1)
67
+ y1, y2 = y.chunk(2, dim=1)
68
+ return x1 * y2, y1 * x2
69
+
70
+
71
+ class DualStreamSeq(nn.Sequential):
72
+ def forward(self, x, y=None):
73
+ y = y if y is not None else x
74
+ for module in self:
75
+ x, y = module(x, y)
76
+ return x, y
77
+
78
+
79
+ class DualStreamBlock(nn.Module):
80
+ def __init__(self, *args):
81
+ super(DualStreamBlock, self).__init__()
82
+ self.seq_l = nn.Sequential()
83
+ self.seq_r = nn.Sequential()
84
+
85
+ if len(args) == 1 and isinstance(args[0], OrderedDict):
86
+ for key, module in args[0].items():
87
+ self.seq_l.add_module(key, module)
88
+ self.seq_r.add_module(key, module)
89
+ else:
90
+ for idx, module in enumerate(args):
91
+ self.seq_l.add_module(str(idx), module)
92
+ self.seq_r.add_module(str(idx), module)
93
+
94
+ def forward(self, x, y):
95
+ return self.seq_l(x), self.seq_r(y)
96
+
97
+
98
+ class R2Block(nn.Module):
99
+ def __init__(self, c):
100
+ super().__init__()
101
+ self.block1 = DualStreamSeq(
102
+ DualStreamBlock(
103
+ LayerNorm2d(c),
104
+ nn.Conv2d(c, c * 2, 1),
105
+ nn.Conv2d(c * 2, c * 2, 3, padding=1, groups=c * 2)
106
+ ),
107
+ DualStreamGate(),
108
+ DualStreamBlock(CABlock(c)),
109
+ DualStreamBlock(nn.Conv2d(c, c, 1))
110
+ )
111
+
112
+ self.a_l = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
113
+ self.a_r = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
114
+
115
+ self.block2 = DualStreamSeq(
116
+ DualStreamBlock(
117
+ LayerNorm2d(c),
118
+ nn.Conv2d(c, c * 2, 1)
119
+ ),
120
+ DualStreamGate(),
121
+ DualStreamBlock(
122
+ nn.Conv2d(c, c, 1)
123
+ )
124
+
125
+ )
126
+
127
+ self.b_l = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
128
+ self.b_r = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
129
+
130
+ def forward(self, inp_l, inp_r):
131
+ x, y = self.block1(inp_l, inp_r)
132
+ x_skip, y_skip = inp_l + x * self.a_l, inp_r + y * self.a_r
133
+ x, y = self.block2(x_skip, y_skip)
134
+ out_l, out_r = x_skip + x * self.b_l, y_skip + y * self.b_r
135
+ return out_l, out_r
136
+
137
+
138
+ class SinBlock(nn.Module):
139
+ def __init__(self, c):
140
+ super().__init__()
141
+ self.block1 = nn.Sequential(
142
+ LayerNorm2d(c),
143
+ nn.Conv2d(c, c * 2, 1),
144
+ nn.Conv2d(c * 2, c * 2, 3, padding=1, groups=c * 2),
145
+ SimpleGate(),
146
+ CABlock(c),
147
+ nn.Conv2d(c, c, 1)
148
+ )
149
+
150
+ self.block2 = nn.Sequential(
151
+ LayerNorm2d(c),
152
+ nn.Conv2d(c, c * 2, 1),
153
+ SimpleGate(),
154
+ nn.Conv2d(c, c, 1)
155
+ )
156
+
157
+ self.a = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
158
+ self.b = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
159
+
160
+ def forward(self, inp):
161
+ x = self.block1(inp)
162
+ x_skip = inp + x * self.a
163
+ x = self.block2(x_skip)
164
+ out = x_skip + x * self.b
165
+ return out
166
+
167
+ class LRM(nn.Module):
168
+ def __init__(self, in_channels=48, num_blocks=[2, 4]):
169
+ super().__init__()
170
+ self.device = 'cuda'
171
+ channel = in_channels * 2
172
+ self.intro = DualStreamBlock(nn.Conv2d(in_channels, channel, 1))
173
+ self.blocks_inter = DualStreamSeq(*[R2Block(channel) for _ in range(num_blocks[0])])
174
+ self.blocks_merge = nn.Sequential(*[SinBlock(channel * 2) for _ in range(num_blocks[1])])
175
+ self.tail = nn.Sequential(
176
+ nn.Conv2d(channel * 2, 3, 3, padding=1),
177
+ nn.Tanh()
178
+ )
179
+
180
+ def forward(self, ft, fr):
181
+ ft, fr = self.intro(ft, fr)
182
+ ft, fr = self.blocks_inter(ft, fr)
183
+ fs = self.blocks_merge(torch.cat([ft, fr], dim=1))
184
+ out = self.tail(fs)
185
+ return out
DSRNet/models/arch/~/YTMT_US ADDED
@@ -0,0 +1,1197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ digraph {
2
+ graph [size="153.75,153.75"]
3
+ node [align=left fontsize=12 height=0.2 ranksep=0.1 shape=box style=filled]
4
+ 140291558534896 [label=MeanBackward0 fillcolor=darkolivegreen1]
5
+ 140291558534952 -> 140291558534896
6
+ 140291558534952 [label=AddBackward0]
7
+ 140291558535064 -> 140291558534952
8
+ 140291558535064 [label=SigmoidBackward]
9
+ 140291558199424 -> 140291558535064
10
+ 140291558199424 [label=MkldnnConvolutionBackward]
11
+ 140291558199536 -> 140291558199424
12
+ 140291558199536 [label=AddBackward0]
13
+ 140291558199760 -> 140291558199536
14
+ 140291558199760 [label=ReluBackward0]
15
+ 140291558199928 -> 140291558199760
16
+ 140291558199928 [label=MkldnnConvolutionBackward]
17
+ 140291558200040 -> 140291558199928
18
+ 140291558200040 [label=MulBackward0]
19
+ 140291558200264 -> 140291558200040
20
+ 140291558200264 [label=MulBackward0]
21
+ 140291558200432 -> 140291558200264
22
+ 140291558200432 [label=CatBackward]
23
+ 140291558200600 -> 140291558200432
24
+ 140291558200600 [label=AddBackward0]
25
+ 140291558200768 -> 140291558200600
26
+ 140291558200768 [label=ReluBackward0]
27
+ 140291558200936 -> 140291558200768
28
+ 140291558200936 [label=MkldnnConvolutionBackward]
29
+ 140291558201048 -> 140291558200936
30
+ 140291558201048 [label="
31
+ (64, 1475, 3, 3)" fillcolor=lightblue]
32
+ 140291558201104 -> 140291558200936
33
+ 140291558201104 [label="
34
+ (64)" fillcolor=lightblue]
35
+ 140291558200824 -> 140291558200600
36
+ 140291558200824 [label=SubBackward0]
37
+ 140291558200992 -> 140291558200824
38
+ 140291558200992 [label=MkldnnConvolutionBackward]
39
+ 140291558201272 -> 140291558200992
40
+ 140291558201272 [label="
41
+ (64, 1475, 3, 3)" fillcolor=lightblue]
42
+ 140291558201384 -> 140291558200992
43
+ 140291558201384 [label="
44
+ (64)" fillcolor=lightblue]
45
+ 140291558201160 -> 140291558200824
46
+ 140291558201160 [label=ReluBackward0]
47
+ 140291558200992 -> 140291558201160
48
+ 140291558200656 -> 140291558200432
49
+ 140291558200656 [label=ConstantPadNdBackward]
50
+ 140291558200880 -> 140291558200656
51
+ 140291558200880 [label=UpsampleBilinear2DBackward1]
52
+ 140291558201328 -> 140291558200880
53
+ 140291558201328 [label=AddBackward0]
54
+ 140291558201552 -> 140291558201328
55
+ 140291558201552 [label=ReluBackward0]
56
+ 140291558201720 -> 140291558201552
57
+ 140291558201720 [label=MkldnnConvolutionBackward]
58
+ 140291558201832 -> 140291558201720
59
+ 140291558201832 [label=MulBackward0]
60
+ 140291558202056 -> 140291558201832
61
+ 140291558202056 [label=MulBackward0]
62
+ 140291558202224 -> 140291558202056
63
+ 140291558202224 [label=CatBackward]
64
+ 140291558202392 -> 140291558202224
65
+ 140291558202392 [label=AddBackward0]
66
+ 140291558202560 -> 140291558202392
67
+ 140291558202560 [label=ReluBackward0]
68
+ 140291558202728 -> 140291558202560
69
+ 140291558202728 [label=MkldnnConvolutionBackward]
70
+ 140291558202840 -> 140291558202728
71
+ 140291558202840 [label=MulBackward0]
72
+ 140291558203064 -> 140291558202840
73
+ 140291558203064 [label=MulBackward0]
74
+ 140291558203232 -> 140291558203064
75
+ 140291558203232 [label=MaxPool2DWithIndicesBackward]
76
+ 140291558200600 -> 140291558203232
77
+ 140291558203288 -> 140291558203064
78
+ 140291558203288 [label=SigmoidBackward]
79
+ 140291558211656 -> 140291558203288
80
+ 140291558211656 [label=MkldnnConvolutionBackward]
81
+ 140291558211768 -> 140291558211656
82
+ 140291558211768 [label=ReluBackward1]
83
+ 140291558211992 -> 140291558211768
84
+ 140291558211992 [label=MkldnnConvolutionBackward]
85
+ 140291558203232 -> 140291558211992
86
+ 140291558212104 -> 140291558211992
87
+ 140291558212104 [label="
88
+ (8, 64, 1, 1)" fillcolor=lightblue]
89
+ 140291558212160 -> 140291558211992
90
+ 140291558212160 [label="
91
+ (8)" fillcolor=lightblue]
92
+ 140291558211824 -> 140291558211656
93
+ 140291558211824 [label="
94
+ (1, 8, 1, 1)" fillcolor=lightblue]
95
+ 140291558211880 -> 140291558211656
96
+ 140291558211880 [label="
97
+ (1)" fillcolor=lightblue]
98
+ 140291558203120 -> 140291558202840
99
+ 140291558203120 [label=SigmoidBackward]
100
+ 140291558203344 -> 140291558203120
101
+ 140291558203344 [label=ThnnConv2DBackward]
102
+ 140291558211936 -> 140291558203344
103
+ 140291558211936 [label=ReluBackward1]
104
+ 140291558212328 -> 140291558211936
105
+ 140291558212328 [label=ThnnConv2DBackward]
106
+ 140291558212440 -> 140291558212328
107
+ 140291558212440 [label=MeanBackward1]
108
+ 140291558203064 -> 140291558212440
109
+ 140291558212496 -> 140291558212328
110
+ 140291558212496 [label="
111
+ (8, 64, 1, 1)" fillcolor=lightblue]
112
+ 140291558212552 -> 140291558212328
113
+ 140291558212552 [label="
114
+ (8)" fillcolor=lightblue]
115
+ 140291558212216 -> 140291558203344
116
+ 140291558212216 [label="
117
+ (64, 8, 1, 1)" fillcolor=lightblue]
118
+ 140291558212048 -> 140291558203344
119
+ 140291558212048 [label="
120
+ (64)" fillcolor=lightblue]
121
+ 140291558202896 -> 140291558202728
122
+ 140291558202896 [label="
123
+ (128, 64, 3, 3)" fillcolor=lightblue]
124
+ 140291558202952 -> 140291558202728
125
+ 140291558202952 [label="
126
+ (128)" fillcolor=lightblue]
127
+ 140291558202616 -> 140291558202392
128
+ 140291558202616 [label=SubBackward0]
129
+ 140291558202784 -> 140291558202616
130
+ 140291558202784 [label=MkldnnConvolutionBackward]
131
+ 140291558211712 -> 140291558202784
132
+ 140291558211712 [label=MulBackward0]
133
+ 140291558212664 -> 140291558211712
134
+ 140291558212664 [label=MulBackward0]
135
+ 140291558212832 -> 140291558212664
136
+ 140291558212832 [label=MaxPool2DWithIndicesBackward]
137
+ 140291558213000 -> 140291558212832
138
+ 140291558213000 [label=AddBackward0]
139
+ 140291558213112 -> 140291558213000
140
+ 140291558213112 [label=ReluBackward0]
141
+ 140291558200992 -> 140291558213112
142
+ 140291558213168 -> 140291558213000
143
+ 140291558213168 [label=SubBackward0]
144
+ 140291558200936 -> 140291558213168
145
+ 140291558213280 -> 140291558213168
146
+ 140291558213280 [label=ReluBackward0]
147
+ 140291558200936 -> 140291558213280
148
+ 140291558212888 -> 140291558212664
149
+ 140291558212888 [label=SigmoidBackward]
150
+ 140291558213056 -> 140291558212888
151
+ 140291558213056 [label=MkldnnConvolutionBackward]
152
+ 140291558213336 -> 140291558213056
153
+ 140291558213336 [label=ReluBackward1]
154
+ 140291558213560 -> 140291558213336
155
+ 140291558213560 [label=MkldnnConvolutionBackward]
156
+ 140291558212832 -> 140291558213560
157
+ 140291558213672 -> 140291558213560
158
+ 140291558213672 [label="
159
+ (8, 64, 1, 1)" fillcolor=lightblue]
160
+ 140291558213728 -> 140291558213560
161
+ 140291558213728 [label="
162
+ (8)" fillcolor=lightblue]
163
+ 140291558213392 -> 140291558213056
164
+ 140291558213392 [label="
165
+ (1, 8, 1, 1)" fillcolor=lightblue]
166
+ 140291558213448 -> 140291558213056
167
+ 140291558213448 [label="
168
+ (1)" fillcolor=lightblue]
169
+ 140291558212720 -> 140291558211712
170
+ 140291558212720 [label=SigmoidBackward]
171
+ 140291558212944 -> 140291558212720
172
+ 140291558212944 [label=ThnnConv2DBackward]
173
+ 140291558213504 -> 140291558212944
174
+ 140291558213504 [label=ReluBackward1]
175
+ 140291558213896 -> 140291558213504
176
+ 140291558213896 [label=ThnnConv2DBackward]
177
+ 140291558214008 -> 140291558213896
178
+ 140291558214008 [label=MeanBackward1]
179
+ 140291558212664 -> 140291558214008
180
+ 140291558214064 -> 140291558213896
181
+ 140291558214064 [label="
182
+ (8, 64, 1, 1)" fillcolor=lightblue]
183
+ 140291558214120 -> 140291558213896
184
+ 140291558214120 [label="
185
+ (8)" fillcolor=lightblue]
186
+ 140291558213784 -> 140291558212944
187
+ 140291558213784 [label="
188
+ (64, 8, 1, 1)" fillcolor=lightblue]
189
+ 140291558213616 -> 140291558212944
190
+ 140291558213616 [label="
191
+ (64)" fillcolor=lightblue]
192
+ 140291558212272 -> 140291558202784
193
+ 140291558212272 [label="
194
+ (128, 64, 3, 3)" fillcolor=lightblue]
195
+ 140291558212384 -> 140291558202784
196
+ 140291558212384 [label="
197
+ (128)" fillcolor=lightblue]
198
+ 140291558203008 -> 140291558202616
199
+ 140291558203008 [label=ReluBackward0]
200
+ 140291558202784 -> 140291558203008
201
+ 140291558202448 -> 140291558202224
202
+ 140291558202448 [label=ConstantPadNdBackward]
203
+ 140291558202672 -> 140291558202448
204
+ 140291558202672 [label=UpsampleBilinear2DBackward1]
205
+ 140291558212608 -> 140291558202672
206
+ 140291558212608 [label=AddBackward0]
207
+ 140291558213224 -> 140291558212608
208
+ 140291558213224 [label=ReluBackward0]
209
+ 140291558214176 -> 140291558213224
210
+ 140291558214176 [label=MkldnnConvolutionBackward]
211
+ 140291558214288 -> 140291558214176
212
+ 140291558214288 [label=MulBackward0]
213
+ 140291558214512 -> 140291558214288
214
+ 140291558214512 [label=MulBackward0]
215
+ 140291558214680 -> 140291558214512
216
+ 140291558214680 [label=CatBackward]
217
+ 140291558214848 -> 140291558214680
218
+ 140291558214848 [label=AddBackward0]
219
+ 140291558215016 -> 140291558214848
220
+ 140291558215016 [label=ReluBackward0]
221
+ 140291558215184 -> 140291558215016
222
+ 140291558215184 [label=MkldnnConvolutionBackward]
223
+ 140291558215296 -> 140291558215184
224
+ 140291558215296 [label=MulBackward0]
225
+ 140291558215520 -> 140291558215296
226
+ 140291558215520 [label=MulBackward0]
227
+ 140291558232136 -> 140291558215520
228
+ 140291558232136 [label=MaxPool2DWithIndicesBackward]
229
+ 140291558202392 -> 140291558232136
230
+ 140291558232192 -> 140291558215520
231
+ 140291558232192 [label=SigmoidBackward]
232
+ 140291558232304 -> 140291558232192
233
+ 140291558232304 [label=MkldnnConvolutionBackward]
234
+ 140291558232472 -> 140291558232304
235
+ 140291558232472 [label=ReluBackward1]
236
+ 140291558232696 -> 140291558232472
237
+ 140291558232696 [label=MkldnnConvolutionBackward]
238
+ 140291558232136 -> 140291558232696
239
+ 140291558232864 -> 140291558232696
240
+ 140291558232864 [label="
241
+ (16, 128, 1, 1)" fillcolor=lightblue]
242
+ 140291558232920 -> 140291558232696
243
+ 140291558232920 [label="
244
+ (16)" fillcolor=lightblue]
245
+ 140291558232528 -> 140291558232304
246
+ 140291558232528 [label="
247
+ (1, 16, 1, 1)" fillcolor=lightblue]
248
+ 140291558232584 -> 140291558232304
249
+ 140291558232584 [label="
250
+ (1)" fillcolor=lightblue]
251
+ 140291558215576 -> 140291558215296
252
+ 140291558215576 [label=SigmoidBackward]
253
+ 140291558232248 -> 140291558215576
254
+ 140291558232248 [label=ThnnConv2DBackward]
255
+ 140291558232640 -> 140291558232248
256
+ 140291558232640 [label=ReluBackward1]
257
+ 140291558233088 -> 140291558232640
258
+ 140291558233088 [label=ThnnConv2DBackward]
259
+ 140291558233256 -> 140291558233088
260
+ 140291558233256 [label=MeanBackward1]
261
+ 140291558215520 -> 140291558233256
262
+ 140291558233312 -> 140291558233088
263
+ 140291558233312 [label="
264
+ (16, 128, 1, 1)" fillcolor=lightblue]
265
+ 140291558233368 -> 140291558233088
266
+ 140291558233368 [label="
267
+ (16)" fillcolor=lightblue]
268
+ 140291558232752 -> 140291558232248
269
+ 140291558232752 [label="
270
+ (128, 16, 1, 1)" fillcolor=lightblue]
271
+ 140291558232976 -> 140291558232248
272
+ 140291558232976 [label="
273
+ (128)" fillcolor=lightblue]
274
+ 140291558215352 -> 140291558215184
275
+ 140291558215352 [label="
276
+ (256, 128, 3, 3)" fillcolor=lightblue]
277
+ 140291558215408 -> 140291558215184
278
+ 140291558215408 [label="
279
+ (256)" fillcolor=lightblue]
280
+ 140291558215072 -> 140291558214848
281
+ 140291558215072 [label=SubBackward0]
282
+ 140291558215240 -> 140291558215072
283
+ 140291558215240 [label=MkldnnConvolutionBackward]
284
+ 140291558232416 -> 140291558215240
285
+ 140291558232416 [label=MulBackward0]
286
+ 140291558233200 -> 140291558232416
287
+ 140291558233200 [label=MulBackward0]
288
+ 140291558233648 -> 140291558233200
289
+ 140291558233648 [label=MaxPool2DWithIndicesBackward]
290
+ 140291558233816 -> 140291558233648
291
+ 140291558233816 [label=AddBackward0]
292
+ 140291558233928 -> 140291558233816
293
+ 140291558233928 [label=ReluBackward0]
294
+ 140291558202784 -> 140291558233928
295
+ 140291558233984 -> 140291558233816
296
+ 140291558233984 [label=SubBackward0]
297
+ 140291558202728 -> 140291558233984
298
+ 140291558234096 -> 140291558233984
299
+ 140291558234096 [label=ReluBackward0]
300
+ 140291558202728 -> 140291558234096
301
+ 140291558233704 -> 140291558233200
302
+ 140291558233704 [label=SigmoidBackward]
303
+ 140291558233872 -> 140291558233704
304
+ 140291558233872 [label=MkldnnConvolutionBackward]
305
+ 140291558234152 -> 140291558233872
306
+ 140291558234152 [label=ReluBackward1]
307
+ 140291558234376 -> 140291558234152
308
+ 140291558234376 [label=MkldnnConvolutionBackward]
309
+ 140291558233648 -> 140291558234376
310
+ 140291558234488 -> 140291558234376
311
+ 140291558234488 [label="
312
+ (16, 128, 1, 1)" fillcolor=lightblue]
313
+ 140291558234544 -> 140291558234376
314
+ 140291558234544 [label="
315
+ (16)" fillcolor=lightblue]
316
+ 140291558234208 -> 140291558233872
317
+ 140291558234208 [label="
318
+ (1, 16, 1, 1)" fillcolor=lightblue]
319
+ 140291558234264 -> 140291558233872
320
+ 140291558234264 [label="
321
+ (1)" fillcolor=lightblue]
322
+ 140291558233536 -> 140291558232416
323
+ 140291558233536 [label=SigmoidBackward]
324
+ 140291558233760 -> 140291558233536
325
+ 140291558233760 [label=ThnnConv2DBackward]
326
+ 140291558234320 -> 140291558233760
327
+ 140291558234320 [label=ReluBackward1]
328
+ 140291558234712 -> 140291558234320
329
+ 140291558234712 [label=ThnnConv2DBackward]
330
+ 140291558234824 -> 140291558234712
331
+ 140291558234824 [label=MeanBackward1]
332
+ 140291558233200 -> 140291558234824
333
+ 140291558234880 -> 140291558234712
334
+ 140291558234880 [label="
335
+ (16, 128, 1, 1)" fillcolor=lightblue]
336
+ 140291558234936 -> 140291558234712
337
+ 140291558234936 [label="
338
+ (16)" fillcolor=lightblue]
339
+ 140291558234432 -> 140291558233760
340
+ 140291558234432 [label="
341
+ (128, 16, 1, 1)" fillcolor=lightblue]
342
+ 140291558234600 -> 140291558233760
343
+ 140291558234600 [label="
344
+ (128)" fillcolor=lightblue]
345
+ 140291558233032 -> 140291558215240
346
+ 140291558233032 [label="
347
+ (256, 128, 3, 3)" fillcolor=lightblue]
348
+ 140291558233592 -> 140291558215240
349
+ 140291558233592 [label="
350
+ (256)" fillcolor=lightblue]
351
+ 140291558215464 -> 140291558215072
352
+ 140291558215464 [label=ReluBackward0]
353
+ 140291558215240 -> 140291558215464
354
+ 140291558214904 -> 140291558214680
355
+ 140291558214904 [label=ConstantPadNdBackward]
356
+ 140291558215128 -> 140291558214904
357
+ 140291558215128 [label=UpsampleBilinear2DBackward1]
358
+ 140291558233424 -> 140291558215128
359
+ 140291558233424 [label=AddBackward0]
360
+ 140291558234040 -> 140291558233424
361
+ 140291558234040 [label=ReluBackward0]
362
+ 140291558234992 -> 140291558234040
363
+ 140291558234992 [label=MkldnnConvolutionBackward]
364
+ 140291558235104 -> 140291558234992
365
+ 140291558235104 [label=MulBackward0]
366
+ 140291558235328 -> 140291558235104
367
+ 140291558235328 [label=MulBackward0]
368
+ 140291558235496 -> 140291558235328
369
+ 140291558235496 [label=CatBackward]
370
+ 140291558235664 -> 140291558235496
371
+ 140291558235664 [label=AddBackward0]
372
+ 140291558235832 -> 140291558235664
373
+ 140291558235832 [label=ReluBackward0]
374
+ 140291558236000 -> 140291558235832
375
+ 140291558236000 [label=MkldnnConvolutionBackward]
376
+ 140291558236112 -> 140291558236000
377
+ 140291558236112 [label=MulBackward0]
378
+ 140291558248688 -> 140291558236112
379
+ 140291558248688 [label=MulBackward0]
380
+ 140291558248856 -> 140291558248688
381
+ 140291558248856 [label=MaxPool2DWithIndicesBackward]
382
+ 140291558214848 -> 140291558248856
383
+ 140291558248912 -> 140291558248688
384
+ 140291558248912 [label=SigmoidBackward]
385
+ 140291558249024 -> 140291558248912
386
+ 140291558249024 [label=MkldnnConvolutionBackward]
387
+ 140291558249192 -> 140291558249024
388
+ 140291558249192 [label=ReluBackward1]
389
+ 140291558249416 -> 140291558249192
390
+ 140291558249416 [label=MkldnnConvolutionBackward]
391
+ 140291558248856 -> 140291558249416
392
+ 140291558249584 -> 140291558249416
393
+ 140291558249584 [label="
394
+ (32, 256, 1, 1)" fillcolor=lightblue]
395
+ 140291558249640 -> 140291558249416
396
+ 140291558249640 [label="
397
+ (32)" fillcolor=lightblue]
398
+ 140291558249248 -> 140291558249024
399
+ 140291558249248 [label="
400
+ (1, 32, 1, 1)" fillcolor=lightblue]
401
+ 140291558249304 -> 140291558249024
402
+ 140291558249304 [label="
403
+ (1)" fillcolor=lightblue]
404
+ 140291558248744 -> 140291558236112
405
+ 140291558248744 [label=SigmoidBackward]
406
+ 140291558248968 -> 140291558248744
407
+ 140291558248968 [label=ThnnConv2DBackward]
408
+ 140291558249360 -> 140291558248968
409
+ 140291558249360 [label=ReluBackward1]
410
+ 140291558249808 -> 140291558249360
411
+ 140291558249808 [label=ThnnConv2DBackward]
412
+ 140291558249976 -> 140291558249808
413
+ 140291558249976 [label=MeanBackward1]
414
+ 140291558248688 -> 140291558249976
415
+ 140291558250032 -> 140291558249808
416
+ 140291558250032 [label="
417
+ (32, 256, 1, 1)" fillcolor=lightblue]
418
+ 140291558250088 -> 140291558249808
419
+ 140291558250088 [label="
420
+ (32)" fillcolor=lightblue]
421
+ 140291558249472 -> 140291558248968
422
+ 140291558249472 [label="
423
+ (256, 32, 1, 1)" fillcolor=lightblue]
424
+ 140291558249696 -> 140291558248968
425
+ 140291558249696 [label="
426
+ (256)" fillcolor=lightblue]
427
+ 140291558248520 -> 140291558236000
428
+ 140291558248520 [label="
429
+ (512, 256, 3, 3)" fillcolor=lightblue]
430
+ 140291558248576 -> 140291558236000
431
+ 140291558248576 [label="
432
+ (512)" fillcolor=lightblue]
433
+ 140291558235888 -> 140291558235664
434
+ 140291558235888 [label=SubBackward0]
435
+ 140291558236056 -> 140291558235888
436
+ 140291558236056 [label=MkldnnConvolutionBackward]
437
+ 140291558249136 -> 140291558236056
438
+ 140291558249136 [label=MulBackward0]
439
+ 140291558249920 -> 140291558249136
440
+ 140291558249920 [label=MulBackward0]
441
+ 140291558250368 -> 140291558249920
442
+ 140291558250368 [label=MaxPool2DWithIndicesBackward]
443
+ 140291558250536 -> 140291558250368
444
+ 140291558250536 [label=AddBackward0]
445
+ 140291558250648 -> 140291558250536
446
+ 140291558250648 [label=ReluBackward0]
447
+ 140291558215240 -> 140291558250648
448
+ 140291558250704 -> 140291558250536
449
+ 140291558250704 [label=SubBackward0]
450
+ 140291558215184 -> 140291558250704
451
+ 140291558250816 -> 140291558250704
452
+ 140291558250816 [label=ReluBackward0]
453
+ 140291558215184 -> 140291558250816
454
+ 140291558250424 -> 140291558249920
455
+ 140291558250424 [label=SigmoidBackward]
456
+ 140291558250592 -> 140291558250424
457
+ 140291558250592 [label=MkldnnConvolutionBackward]
458
+ 140291558250872 -> 140291558250592
459
+ 140291558250872 [label=ReluBackward1]
460
+ 140291558251096 -> 140291558250872
461
+ 140291558251096 [label=MkldnnConvolutionBackward]
462
+ 140291558250368 -> 140291558251096
463
+ 140291558251208 -> 140291558251096
464
+ 140291558251208 [label="
465
+ (32, 256, 1, 1)" fillcolor=lightblue]
466
+ 140291558251264 -> 140291558251096
467
+ 140291558251264 [label="
468
+ (32)" fillcolor=lightblue]
469
+ 140291558250928 -> 140291558250592
470
+ 140291558250928 [label="
471
+ (1, 32, 1, 1)" fillcolor=lightblue]
472
+ 140291558250984 -> 140291558250592
473
+ 140291558250984 [label="
474
+ (1)" fillcolor=lightblue]
475
+ 140291558250256 -> 140291558249136
476
+ 140291558250256 [label=SigmoidBackward]
477
+ 140291558250480 -> 140291558250256
478
+ 140291558250480 [label=ThnnConv2DBackward]
479
+ 140291558251040 -> 140291558250480
480
+ 140291558251040 [label=ReluBackward1]
481
+ 140291558251432 -> 140291558251040
482
+ 140291558251432 [label=ThnnConv2DBackward]
483
+ 140291558251544 -> 140291558251432
484
+ 140291558251544 [label=MeanBackward1]
485
+ 140291558249920 -> 140291558251544
486
+ 140291558251600 -> 140291558251432
487
+ 140291558251600 [label="
488
+ (32, 256, 1, 1)" fillcolor=lightblue]
489
+ 140291558251656 -> 140291558251432
490
+ 140291558251656 [label="
491
+ (32)" fillcolor=lightblue]
492
+ 140291558251152 -> 140291558250480
493
+ 140291558251152 [label="
494
+ (256, 32, 1, 1)" fillcolor=lightblue]
495
+ 140291558251320 -> 140291558250480
496
+ 140291558251320 [label="
497
+ (256)" fillcolor=lightblue]
498
+ 140291558248800 -> 140291558236056
499
+ 140291558248800 [label="
500
+ (512, 256, 3, 3)" fillcolor=lightblue]
501
+ 140291558250312 -> 140291558236056
502
+ 140291558250312 [label="
503
+ (512)" fillcolor=lightblue]
504
+ 140291558248632 -> 140291558235888
505
+ 140291558248632 [label=ReluBackward0]
506
+ 140291558236056 -> 140291558248632
507
+ 140291558235720 -> 140291558235496
508
+ 140291558235720 [label=ConstantPadNdBackward]
509
+ 140291558235944 -> 140291558235720
510
+ 140291558235944 [label=UpsampleBilinear2DBackward1]
511
+ 140291558250144 -> 140291558235944
512
+ 140291558250144 [label=AddBackward0]
513
+ 140291558250760 -> 140291558250144
514
+ 140291558250760 [label=AddBackward0]
515
+ 140291558251712 -> 140291558250760
516
+ 140291558251712 [label=ReluBackward0]
517
+ 140291558251768 -> 140291558251712
518
+ 140291558251768 [label=MkldnnConvolutionBackward]
519
+ 140291558251992 -> 140291558251768
520
+ 140291558251992 [label=MulBackward0]
521
+ 140291558252216 -> 140291558251992
522
+ 140291558252216 [label=MulBackward0]
523
+ 140291558250200 -> 140291558252216
524
+ 140291558250200 [label=MaxPool2DWithIndicesBackward]
525
+ 140291558235664 -> 140291558250200
526
+ 140291558252384 -> 140291558252216
527
+ 140291558252384 [label=SigmoidBackward]
528
+ 140291558252496 -> 140291558252384
529
+ 140291558252496 [label=ThnnConv2DBackward]
530
+ 140291558269056 -> 140291558252496
531
+ 140291558269056 [label=ReluBackward1]
532
+ 140291558269280 -> 140291558269056
533
+ 140291558269280 [label=MkldnnConvolutionBackward]
534
+ 140291558250200 -> 140291558269280
535
+ 140291558269448 -> 140291558269280
536
+ 140291558269448 [label="
537
+ (64, 512, 1, 1)" fillcolor=lightblue]
538
+ 140291558269504 -> 140291558269280
539
+ 140291558269504 [label="
540
+ (64)" fillcolor=lightblue]
541
+ 140291558269112 -> 140291558252496
542
+ 140291558269112 [label="
543
+ (1, 64, 1, 1)" fillcolor=lightblue]
544
+ 140291558269168 -> 140291558252496
545
+ 140291558269168 [label="
546
+ (1)" fillcolor=lightblue]
547
+ 140291558252272 -> 140291558251992
548
+ 140291558252272 [label=SigmoidBackward]
549
+ 140291558252440 -> 140291558252272
550
+ 140291558252440 [label=ThnnConv2DBackward]
551
+ 140291558269224 -> 140291558252440
552
+ 140291558269224 [label=ReluBackward1]
553
+ 140291558269672 -> 140291558269224
554
+ 140291558269672 [label=ThnnConv2DBackward]
555
+ 140291558269840 -> 140291558269672
556
+ 140291558269840 [label=MeanBackward1]
557
+ 140291558252216 -> 140291558269840
558
+ 140291558269896 -> 140291558269672
559
+ 140291558269896 [label="
560
+ (64, 512, 1, 1)" fillcolor=lightblue]
561
+ 140291558269952 -> 140291558269672
562
+ 140291558269952 [label="
563
+ (64)" fillcolor=lightblue]
564
+ 140291558269336 -> 140291558252440
565
+ 140291558269336 [label="
566
+ (512, 64, 1, 1)" fillcolor=lightblue]
567
+ 140291558269560 -> 140291558252440
568
+ 140291558269560 [label="
569
+ (512)" fillcolor=lightblue]
570
+ 140291558252048 -> 140291558251768
571
+ 140291558252048 [label="
572
+ (512, 512, 3, 3)" fillcolor=lightblue]
573
+ 140291558252104 -> 140291558251768
574
+ 140291558252104 [label="
575
+ (512)" fillcolor=lightblue]
576
+ 140291558251376 -> 140291558250760
577
+ 140291558251376 [label=SubBackward0]
578
+ 140291558251936 -> 140291558251376
579
+ 140291558251936 [label=MkldnnConvolutionBackward]
580
+ 140291558269000 -> 140291558251936
581
+ 140291558269000 [label=MulBackward0]
582
+ 140291558269784 -> 140291558269000
583
+ 140291558269784 [label=MulBackward0]
584
+ 140291558270232 -> 140291558269784
585
+ 140291558270232 [label=MaxPool2DWithIndicesBackward]
586
+ 140291558270400 -> 140291558270232
587
+ 140291558270400 [label=AddBackward0]
588
+ 140291558270512 -> 140291558270400
589
+ 140291558270512 [label=ReluBackward0]
590
+ 140291558236056 -> 140291558270512
591
+ 140291558270568 -> 140291558270400
592
+ 140291558270568 [label=SubBackward0]
593
+ 140291558236000 -> 140291558270568
594
+ 140291558270680 -> 140291558270568
595
+ 140291558270680 [label=ReluBackward0]
596
+ 140291558236000 -> 140291558270680
597
+ 140291558270288 -> 140291558269784
598
+ 140291558270288 [label=SigmoidBackward]
599
+ 140291558270456 -> 140291558270288
600
+ 140291558270456 [label=ThnnConv2DBackward]
601
+ 140291558270736 -> 140291558270456
602
+ 140291558270736 [label=ReluBackward1]
603
+ 140291558270960 -> 140291558270736
604
+ 140291558270960 [label=MkldnnConvolutionBackward]
605
+ 140291558270232 -> 140291558270960
606
+ 140291558271072 -> 140291558270960
607
+ 140291558271072 [label="
608
+ (64, 512, 1, 1)" fillcolor=lightblue]
609
+ 140291558271128 -> 140291558270960
610
+ 140291558271128 [label="
611
+ (64)" fillcolor=lightblue]
612
+ 140291558270792 -> 140291558270456
613
+ 140291558270792 [label="
614
+ (1, 64, 1, 1)" fillcolor=lightblue]
615
+ 140291558270848 -> 140291558270456
616
+ 140291558270848 [label="
617
+ (1)" fillcolor=lightblue]
618
+ 140291558270120 -> 140291558269000
619
+ 140291558270120 [label=SigmoidBackward]
620
+ 140291558270344 -> 140291558270120
621
+ 140291558270344 [label=ThnnConv2DBackward]
622
+ 140291558270904 -> 140291558270344
623
+ 140291558270904 [label=ReluBackward1]
624
+ 140291558271296 -> 140291558270904
625
+ 140291558271296 [label=ThnnConv2DBackward]
626
+ 140291558271408 -> 140291558271296
627
+ 140291558271408 [label=MeanBackward1]
628
+ 140291558269784 -> 140291558271408
629
+ 140291558271464 -> 140291558271296
630
+ 140291558271464 [label="
631
+ (64, 512, 1, 1)" fillcolor=lightblue]
632
+ 140291558271520 -> 140291558271296
633
+ 140291558271520 [label="
634
+ (64)" fillcolor=lightblue]
635
+ 140291558271016 -> 140291558270344
636
+ 140291558271016 [label="
637
+ (512, 64, 1, 1)" fillcolor=lightblue]
638
+ 140291558271184 -> 140291558270344
639
+ 140291558271184 [label="
640
+ (512)" fillcolor=lightblue]
641
+ 140291558269616 -> 140291558251936
642
+ 140291558269616 [label="
643
+ (512, 512, 3, 3)" fillcolor=lightblue]
644
+ 140291558270176 -> 140291558251936
645
+ 140291558270176 [label="
646
+ (512)" fillcolor=lightblue]
647
+ 140291558252160 -> 140291558251376
648
+ 140291558252160 [label=ReluBackward0]
649
+ 140291558251936 -> 140291558252160
650
+ 140291558250200 -> 140291558250144
651
+ 140291558235552 -> 140291558235328
652
+ 140291558235552 [label=SigmoidBackward]
653
+ 140291558235776 -> 140291558235552
654
+ 140291558235776 [label=MkldnnConvolutionBackward]
655
+ 140291558251488 -> 140291558235776
656
+ 140291558251488 [label=ReluBackward1]
657
+ 140291558270008 -> 140291558251488
658
+ 140291558270008 [label=MkldnnConvolutionBackward]
659
+ 140291558235496 -> 140291558270008
660
+ 140291558270624 -> 140291558270008
661
+ 140291558270624 [label="
662
+ (128, 1024, 1, 1)" fillcolor=lightblue]
663
+ 140291558270064 -> 140291558270008
664
+ 140291558270064 [label="
665
+ (128)" fillcolor=lightblue]
666
+ 140291558251880 -> 140291558235776
667
+ 140291558251880 [label="
668
+ (1, 128, 1, 1)" fillcolor=lightblue]
669
+ 140291558251824 -> 140291558235776
670
+ 140291558251824 [label="
671
+ (1)" fillcolor=lightblue]
672
+ 140291558235384 -> 140291558235104
673
+ 140291558235384 [label=SigmoidBackward]
674
+ 140291558235608 -> 140291558235384
675
+ 140291558235608 [label=ThnnConv2DBackward]
676
+ 140291558252328 -> 140291558235608
677
+ 140291558252328 [label=ReluBackward1]
678
+ 140291558271240 -> 140291558252328
679
+ 140291558271240 [label=ThnnConv2DBackward]
680
+ 140291558271632 -> 140291558271240
681
+ 140291558271632 [label=MeanBackward1]
682
+ 140291558235328 -> 140291558271632
683
+ 140291558271800 -> 140291558271240
684
+ 140291558271800 [label="
685
+ (128, 1024, 1, 1)" fillcolor=lightblue]
686
+ 140291558271856 -> 140291558271240
687
+ 140291558271856 [label="
688
+ (128)" fillcolor=lightblue]
689
+ 140291558271352 -> 140291558235608
690
+ 140291558271352 [label="
691
+ (1024, 128, 1, 1)" fillcolor=lightblue]
692
+ 140291558271744 -> 140291558235608
693
+ 140291558271744 [label="
694
+ (1024)" fillcolor=lightblue]
695
+ 140291558235048 -> 140291558234992
696
+ 140291558235048 [label="
697
+ (256, 1024, 3, 3)" fillcolor=lightblue]
698
+ 140291558235216 -> 140291558234992
699
+ 140291558235216 [label="
700
+ (256)" fillcolor=lightblue]
701
+ 140291558233480 -> 140291558233424
702
+ 140291558233480 [label=SubBackward0]
703
+ 140291558234656 -> 140291558233480
704
+ 140291558234656 [label=MkldnnConvolutionBackward]
705
+ 140291558269728 -> 140291558234656
706
+ 140291558269728 [label=MulBackward0]
707
+ 140291558271576 -> 140291558269728
708
+ 140291558271576 [label=MulBackward0]
709
+ 140291558272136 -> 140291558271576
710
+ 140291558272136 [label=CatBackward]
711
+ 140291558270400 -> 140291558272136
712
+ 140291558272304 -> 140291558272136
713
+ 140291558272304 [label=ConstantPadNdBackward]
714
+ 140291558272416 -> 140291558272304
715
+ 140291558272416 [label=UpsampleBilinear2DBackward1]
716
+ 140291558272528 -> 140291558272416
717
+ 140291558272528 [label=AddBackward0]
718
+ 140291558272640 -> 140291558272528
719
+ 140291558272640 [label=AddBackward0]
720
+ 140291558272752 -> 140291558272640
721
+ 140291558272752 [label=ReluBackward0]
722
+ 140291558251936 -> 140291558272752
723
+ 140291558272808 -> 140291558272640
724
+ 140291558272808 [label=SubBackward0]
725
+ 140291558251768 -> 140291558272808
726
+ 140291558272920 -> 140291558272808
727
+ 140291558272920 [label=ReluBackward0]
728
+ 140291558251768 -> 140291558272920
729
+ 140291558270232 -> 140291558272528
730
+ 140291558272192 -> 140291558271576
731
+ 140291558272192 [label=SigmoidBackward]
732
+ 140291558272360 -> 140291558272192
733
+ 140291558272360 [label=MkldnnConvolutionBackward]
734
+ 140291558272584 -> 140291558272360
735
+ 140291558272584 [label=ReluBackward1]
736
+ 140291558281288 -> 140291558272584
737
+ 140291558281288 [label=MkldnnConvolutionBackward]
738
+ 140291558272136 -> 140291558281288
739
+ 140291558281400 -> 140291558281288
740
+ 140291558281400 [label="
741
+ (128, 1024, 1, 1)" fillcolor=lightblue]
742
+ 140291558281456 -> 140291558281288
743
+ 140291558281456 [label="
744
+ (128)" fillcolor=lightblue]
745
+ 140291558272696 -> 140291558272360
746
+ 140291558272696 [label="
747
+ (1, 128, 1, 1)" fillcolor=lightblue]
748
+ 140291558272864 -> 140291558272360
749
+ 140291558272864 [label="
750
+ (1)" fillcolor=lightblue]
751
+ 140291558272024 -> 140291558269728
752
+ 140291558272024 [label=SigmoidBackward]
753
+ 140291558272248 -> 140291558272024
754
+ 140291558272248 [label=ThnnConv2DBackward]
755
+ 140291558272976 -> 140291558272248
756
+ 140291558272976 [label=ReluBackward1]
757
+ 140291558281624 -> 140291558272976
758
+ 140291558281624 [label=ThnnConv2DBackward]
759
+ 140291558281736 -> 140291558281624
760
+ 140291558281736 [label=MeanBackward1]
761
+ 140291558271576 -> 140291558281736
762
+ 140291558281792 -> 140291558281624
763
+ 140291558281792 [label="
764
+ (128, 1024, 1, 1)" fillcolor=lightblue]
765
+ 140291558281848 -> 140291558281624
766
+ 140291558281848 [label="
767
+ (128)" fillcolor=lightblue]
768
+ 140291558281344 -> 140291558272248
769
+ 140291558281344 [label="
770
+ (1024, 128, 1, 1)" fillcolor=lightblue]
771
+ 140291558281512 -> 140291558272248
772
+ 140291558281512 [label="
773
+ (1024)" fillcolor=lightblue]
774
+ 140291558271688 -> 140291558234656
775
+ 140291558271688 [label="
776
+ (256, 1024, 3, 3)" fillcolor=lightblue]
777
+ 140291558271912 -> 140291558234656
778
+ 140291558271912 [label="
779
+ (256)" fillcolor=lightblue]
780
+ 140291558235272 -> 140291558233480
781
+ 140291558235272 [label=ReluBackward0]
782
+ 140291558234656 -> 140291558235272
783
+ 140291558214736 -> 140291558214512
784
+ 140291558214736 [label=SigmoidBackward]
785
+ 140291558214960 -> 140291558214736
786
+ 140291558214960 [label=MkldnnConvolutionBackward]
787
+ 140291558234768 -> 140291558214960
788
+ 140291558234768 [label=ReluBackward1]
789
+ 140291558271968 -> 140291558234768
790
+ 140291558271968 [label=MkldnnConvolutionBackward]
791
+ 140291558214680 -> 140291558271968
792
+ 140291558281680 -> 140291558271968
793
+ 140291558281680 [label="
794
+ (64, 512, 1, 1)" fillcolor=lightblue]
795
+ 140291558281904 -> 140291558271968
796
+ 140291558281904 [label="
797
+ (64)" fillcolor=lightblue]
798
+ 140291558235160 -> 140291558214960
799
+ 140291558235160 [label="
800
+ (1, 64, 1, 1)" fillcolor=lightblue]
801
+ 140291558249752 -> 140291558214960
802
+ 140291558249752 [label="
803
+ (1)" fillcolor=lightblue]
804
+ 140291558214568 -> 140291558214288
805
+ 140291558214568 [label=SigmoidBackward]
806
+ 140291558214792 -> 140291558214568
807
+ 140291558214792 [label=ThnnConv2DBackward]
808
+ 140291558272080 -> 140291558214792
809
+ 140291558272080 [label=ReluBackward1]
810
+ 140291558282016 -> 140291558272080
811
+ 140291558282016 [label=ThnnConv2DBackward]
812
+ 140291558282128 -> 140291558282016
813
+ 140291558282128 [label=MeanBackward1]
814
+ 140291558214512 -> 140291558282128
815
+ 140291558282184 -> 140291558282016
816
+ 140291558282184 [label="
817
+ (64, 512, 1, 1)" fillcolor=lightblue]
818
+ 140291558282240 -> 140291558282016
819
+ 140291558282240 [label="
820
+ (64)" fillcolor=lightblue]
821
+ 140291558272472 -> 140291558214792
822
+ 140291558272472 [label="
823
+ (512, 64, 1, 1)" fillcolor=lightblue]
824
+ 140291558282072 -> 140291558214792
825
+ 140291558282072 [label="
826
+ (512)" fillcolor=lightblue]
827
+ 140291558214344 -> 140291558214176
828
+ 140291558214344 [label="
829
+ (128, 512, 3, 3)" fillcolor=lightblue]
830
+ 140291558214400 -> 140291558214176
831
+ 140291558214400 [label="
832
+ (128)" fillcolor=lightblue]
833
+ 140291558212776 -> 140291558212608
834
+ 140291558212776 [label=SubBackward0]
835
+ 140291558214232 -> 140291558212776
836
+ 140291558214232 [label=MkldnnConvolutionBackward]
837
+ 140291558215632 -> 140291558214232
838
+ 140291558215632 [label=MulBackward0]
839
+ 140291558281568 -> 140291558215632
840
+ 140291558281568 [label=MulBackward0]
841
+ 140291558282520 -> 140291558281568
842
+ 140291558282520 [label=CatBackward]
843
+ 140291558250536 -> 140291558282520
844
+ 140291558282688 -> 140291558282520
845
+ 140291558282688 [label=ConstantPadNdBackward]
846
+ 140291558282800 -> 140291558282688
847
+ 140291558282800 [label=UpsampleBilinear2DBackward1]
848
+ 140291558282912 -> 140291558282800
849
+ 140291558282912 [label=AddBackward0]
850
+ 140291558283024 -> 140291558282912
851
+ 140291558283024 [label=ReluBackward0]
852
+ 140291558234656 -> 140291558283024
853
+ 140291558283080 -> 140291558282912
854
+ 140291558283080 [label=SubBackward0]
855
+ 140291558234992 -> 140291558283080
856
+ 140291558283192 -> 140291558283080
857
+ 140291558283192 [label=ReluBackward0]
858
+ 140291558234992 -> 140291558283192
859
+ 140291558282576 -> 140291558281568
860
+ 140291558282576 [label=SigmoidBackward]
861
+ 140291558282744 -> 140291558282576
862
+ 140291558282744 [label=MkldnnConvolutionBackward]
863
+ 140291558282968 -> 140291558282744
864
+ 140291558282968 [label=ReluBackward1]
865
+ 140291558283360 -> 140291558282968
866
+ 140291558283360 [label=MkldnnConvolutionBackward]
867
+ 140291558282520 -> 140291558283360
868
+ 140291558283472 -> 140291558283360
869
+ 140291558283472 [label="
870
+ (64, 512, 1, 1)" fillcolor=lightblue]
871
+ 140291558283528 -> 140291558283360
872
+ 140291558283528 [label="
873
+ (64)" fillcolor=lightblue]
874
+ 140291558283136 -> 140291558282744
875
+ 140291558283136 [label="
876
+ (1, 64, 1, 1)" fillcolor=lightblue]
877
+ 140291558283248 -> 140291558282744
878
+ 140291558283248 [label="
879
+ (1)" fillcolor=lightblue]
880
+ 140291558282408 -> 140291558215632
881
+ 140291558282408 [label=SigmoidBackward]
882
+ 140291558282632 -> 140291558282408
883
+ 140291558282632 [label=ThnnConv2DBackward]
884
+ 140291558283304 -> 140291558282632
885
+ 140291558283304 [label=ReluBackward1]
886
+ 140291558283696 -> 140291558283304
887
+ 140291558283696 [label=ThnnConv2DBackward]
888
+ 140291558283808 -> 140291558283696
889
+ 140291558283808 [label=MeanBackward1]
890
+ 140291558281568 -> 140291558283808
891
+ 140291558283864 -> 140291558283696
892
+ 140291558283864 [label="
893
+ (64, 512, 1, 1)" fillcolor=lightblue]
894
+ 140291558283920 -> 140291558283696
895
+ 140291558283920 [label="
896
+ (64)" fillcolor=lightblue]
897
+ 140291558283416 -> 140291558282632
898
+ 140291558283416 [label="
899
+ (512, 64, 1, 1)" fillcolor=lightblue]
900
+ 140291558283584 -> 140291558282632
901
+ 140291558283584 [label="
902
+ (512)" fillcolor=lightblue]
903
+ 140291558281960 -> 140291558214232
904
+ 140291558281960 [label="
905
+ (128, 512, 3, 3)" fillcolor=lightblue]
906
+ 140291558282464 -> 140291558214232
907
+ 140291558282464 [label="
908
+ (128)" fillcolor=lightblue]
909
+ 140291558214456 -> 140291558212776
910
+ 140291558214456 [label=ReluBackward0]
911
+ 140291558214232 -> 140291558214456
912
+ 140291558202280 -> 140291558202056
913
+ 140291558202280 [label=SigmoidBackward]
914
+ 140291558202504 -> 140291558202280
915
+ 140291558202504 [label=MkldnnConvolutionBackward]
916
+ 140291558213840 -> 140291558202504
917
+ 140291558213840 [label=ReluBackward1]
918
+ 140291558283752 -> 140291558213840
919
+ 140291558283752 [label=MkldnnConvolutionBackward]
920
+ 140291558202224 -> 140291558283752
921
+ 140291558282352 -> 140291558283752
922
+ 140291558282352 [label="
923
+ (32, 256, 1, 1)" fillcolor=lightblue]
924
+ 140291558284144 -> 140291558283752
925
+ 140291558284144 [label="
926
+ (32)" fillcolor=lightblue]
927
+ 140291558213952 -> 140291558202504
928
+ 140291558213952 [label="
929
+ (1, 32, 1, 1)" fillcolor=lightblue]
930
+ 140291558214624 -> 140291558202504
931
+ 140291558214624 [label="
932
+ (1)" fillcolor=lightblue]
933
+ 140291558202112 -> 140291558201832
934
+ 140291558202112 [label=SigmoidBackward]
935
+ 140291558202336 -> 140291558202112
936
+ 140291558202336 [label=ThnnConv2DBackward]
937
+ 140291558282296 -> 140291558202336
938
+ 140291558282296 [label=ReluBackward1]
939
+ 140291558284088 -> 140291558282296
940
+ 140291558284088 [label=ThnnConv2DBackward]
941
+ 140291558284200 -> 140291558284088
942
+ 140291558284200 [label=MeanBackward1]
943
+ 140291558202056 -> 140291558284200
944
+ 140291558284256 -> 140291558284088
945
+ 140291558284256 [label="
946
+ (32, 256, 1, 1)" fillcolor=lightblue]
947
+ 140291558284312 -> 140291558284088
948
+ 140291558284312 [label="
949
+ (32)" fillcolor=lightblue]
950
+ 140291558282856 -> 140291558202336
951
+ 140291558282856 [label="
952
+ (256, 32, 1, 1)" fillcolor=lightblue]
953
+ 140291558283976 -> 140291558202336
954
+ 140291558283976 [label="
955
+ (256)" fillcolor=lightblue]
956
+ 140291558201888 -> 140291558201720
957
+ 140291558201888 [label="
958
+ (64, 256, 3, 3)" fillcolor=lightblue]
959
+ 140291558201944 -> 140291558201720
960
+ 140291558201944 [label="
961
+ (64)" fillcolor=lightblue]
962
+ 140291558201496 -> 140291558201328
963
+ 140291558201496 [label=SubBackward0]
964
+ 140291558201776 -> 140291558201496
965
+ 140291558201776 [label=MkldnnConvolutionBackward]
966
+ 140291558203176 -> 140291558201776
967
+ 140291558203176 [label=MulBackward0]
968
+ 140291558283640 -> 140291558203176
969
+ 140291558283640 [label=MulBackward0]
970
+ 140291558284592 -> 140291558283640
971
+ 140291558284592 [label=CatBackward]
972
+ 140291558233816 -> 140291558284592
973
+ 140291558284760 -> 140291558284592
974
+ 140291558284760 [label=ConstantPadNdBackward]
975
+ 140291558284872 -> 140291558284760
976
+ 140291558284872 [label=UpsampleBilinear2DBackward1]
977
+ 140291558284984 -> 140291558284872
978
+ 140291558284984 [label=AddBackward0]
979
+ 140291558285096 -> 140291558284984
980
+ 140291558285096 [label=ReluBackward0]
981
+ 140291558214232 -> 140291558285096
982
+ 140291558285152 -> 140291558284984
983
+ 140291558285152 [label=SubBackward0]
984
+ 140291558214176 -> 140291558285152
985
+ 140291558285264 -> 140291558285152
986
+ 140291558285264 [label=ReluBackward0]
987
+ 140291558214176 -> 140291558285264
988
+ 140291558284648 -> 140291558283640
989
+ 140291558284648 [label=SigmoidBackward]
990
+ 140291558284816 -> 140291558284648
991
+ 140291558284816 [label=MkldnnConvolutionBackward]
992
+ 140291558285040 -> 140291558284816
993
+ 140291558285040 [label=ReluBackward1]
994
+ 140291333939384 -> 140291558285040
995
+ 140291333939384 [label=MkldnnConvolutionBackward]
996
+ 140291558284592 -> 140291333939384
997
+ 140291333939496 -> 140291333939384
998
+ 140291333939496 [label="
999
+ (32, 256, 1, 1)" fillcolor=lightblue]
1000
+ 140291333939552 -> 140291333939384
1001
+ 140291333939552 [label="
1002
+ (32)" fillcolor=lightblue]
1003
+ 140291558285208 -> 140291558284816
1004
+ 140291558285208 [label="
1005
+ (1, 32, 1, 1)" fillcolor=lightblue]
1006
+ 140291333939272 -> 140291558284816
1007
+ 140291333939272 [label="
1008
+ (1)" fillcolor=lightblue]
1009
+ 140291558284480 -> 140291558203176
1010
+ 140291558284480 [label=SigmoidBackward]
1011
+ 140291558284704 -> 140291558284480
1012
+ 140291558284704 [label=ThnnConv2DBackward]
1013
+ 140291333939328 -> 140291558284704
1014
+ 140291333939328 [label=ReluBackward1]
1015
+ 140291333939720 -> 140291333939328
1016
+ 140291333939720 [label=ThnnConv2DBackward]
1017
+ 140291333939832 -> 140291333939720
1018
+ 140291333939832 [label=MeanBackward1]
1019
+ 140291558283640 -> 140291333939832
1020
+ 140291333939888 -> 140291333939720
1021
+ 140291333939888 [label="
1022
+ (32, 256, 1, 1)" fillcolor=lightblue]
1023
+ 140291333939944 -> 140291333939720
1024
+ 140291333939944 [label="
1025
+ (32)" fillcolor=lightblue]
1026
+ 140291333939440 -> 140291558284704
1027
+ 140291333939440 [label="
1028
+ (256, 32, 1, 1)" fillcolor=lightblue]
1029
+ 140291333939608 -> 140291558284704
1030
+ 140291333939608 [label="
1031
+ (256)" fillcolor=lightblue]
1032
+ 140291558284032 -> 140291558201776
1033
+ 140291558284032 [label="
1034
+ (64, 256, 3, 3)" fillcolor=lightblue]
1035
+ 140291558284536 -> 140291558201776
1036
+ 140291558284536 [label="
1037
+ (64)" fillcolor=lightblue]
1038
+ 140291558202000 -> 140291558201496
1039
+ 140291558202000 [label=ReluBackward0]
1040
+ 140291558201776 -> 140291558202000
1041
+ 140291558200488 -> 140291558200264
1042
+ 140291558200488 [label=SigmoidBackward]
1043
+ 140291558200712 -> 140291558200488
1044
+ 140291558200712 [label=MkldnnConvolutionBackward]
1045
+ 140291558201608 -> 140291558200712
1046
+ 140291558201608 [label=ReluBackward1]
1047
+ 140291558284424 -> 140291558201608
1048
+ 140291558284424 [label=MkldnnConvolutionBackward]
1049
+ 140291558200432 -> 140291558284424
1050
+ 140291333939776 -> 140291558284424
1051
+ 140291333939776 [label="
1052
+ (16, 128, 1, 1)" fillcolor=lightblue]
1053
+ 140291333940168 -> 140291558284424
1054
+ 140291333940168 [label="
1055
+ (16)" fillcolor=lightblue]
1056
+ 140291558201664 -> 140291558200712
1057
+ 140291558201664 [label="
1058
+ (1, 16, 1, 1)" fillcolor=lightblue]
1059
+ 140291558202168 -> 140291558200712
1060
+ 140291558202168 [label="
1061
+ (1)" fillcolor=lightblue]
1062
+ 140291558200320 -> 140291558200040
1063
+ 140291558200320 [label=SigmoidBackward]
1064
+ 140291558200544 -> 140291558200320
1065
+ 140291558200544 [label=ThnnConv2DBackward]
1066
+ 140291558284368 -> 140291558200544
1067
+ 140291558284368 [label=ReluBackward1]
1068
+ 140291333940112 -> 140291558284368
1069
+ 140291333940112 [label=ThnnConv2DBackward]
1070
+ 140291333940224 -> 140291333940112
1071
+ 140291333940224 [label=MeanBackward1]
1072
+ 140291558200264 -> 140291333940224
1073
+ 140291333940280 -> 140291333940112
1074
+ 140291333940280 [label="
1075
+ (16, 128, 1, 1)" fillcolor=lightblue]
1076
+ 140291333940336 -> 140291333940112
1077
+ 140291333940336 [label="
1078
+ (16)" fillcolor=lightblue]
1079
+ 140291558284928 -> 140291558200544
1080
+ 140291558284928 [label="
1081
+ (128, 16, 1, 1)" fillcolor=lightblue]
1082
+ 140291333940000 -> 140291558200544
1083
+ 140291333940000 [label="
1084
+ (128)" fillcolor=lightblue]
1085
+ 140291558200096 -> 140291558199928
1086
+ 140291558200096 [label="
1087
+ (64, 128, 3, 3)" fillcolor=lightblue]
1088
+ 140291558200152 -> 140291558199928
1089
+ 140291558200152 [label="
1090
+ (64)" fillcolor=lightblue]
1091
+ 140291558199816 -> 140291558199536
1092
+ 140291558199816 [label=SubBackward0]
1093
+ 140291558199984 -> 140291558199816
1094
+ 140291558199984 [label=MkldnnConvolutionBackward]
1095
+ 140291558201440 -> 140291558199984
1096
+ 140291558201440 [label=MulBackward0]
1097
+ 140291333939664 -> 140291558201440
1098
+ 140291333939664 [label=MulBackward0]
1099
+ 140291333940616 -> 140291333939664
1100
+ 140291333940616 [label=CatBackward]
1101
+ 140291558213000 -> 140291333940616
1102
+ 140291333940784 -> 140291333940616
1103
+ 140291333940784 [label=ConstantPadNdBackward]
1104
+ 140291333940896 -> 140291333940784
1105
+ 140291333940896 [label=UpsampleBilinear2DBackward1]
1106
+ 140291333941008 -> 140291333940896
1107
+ 140291333941008 [label=AddBackward0]
1108
+ 140291333941120 -> 140291333941008
1109
+ 140291333941120 [label=ReluBackward0]
1110
+ 140291558201776 -> 140291333941120
1111
+ 140291333941176 -> 140291333941008
1112
+ 140291333941176 [label=SubBackward0]
1113
+ 140291558201720 -> 140291333941176
1114
+ 140291333941288 -> 140291333941176
1115
+ 140291333941288 [label=ReluBackward0]
1116
+ 140291558201720 -> 140291333941288
1117
+ 140291333940672 -> 140291333939664
1118
+ 140291333940672 [label=SigmoidBackward]
1119
+ 140291333940840 -> 140291333940672
1120
+ 140291333940840 [label=MkldnnConvolutionBackward]
1121
+ 140291333941064 -> 140291333940840
1122
+ 140291333941064 [label=ReluBackward1]
1123
+ 140291333941456 -> 140291333941064
1124
+ 140291333941456 [label=MkldnnConvolutionBackward]
1125
+ 140291333940616 -> 140291333941456
1126
+ 140291333941568 -> 140291333941456
1127
+ 140291333941568 [label="
1128
+ (16, 128, 1, 1)" fillcolor=lightblue]
1129
+ 140291333941624 -> 140291333941456
1130
+ 140291333941624 [label="
1131
+ (16)" fillcolor=lightblue]
1132
+ 140291333941232 -> 140291333940840
1133
+ 140291333941232 [label="
1134
+ (1, 16, 1, 1)" fillcolor=lightblue]
1135
+ 140291333941344 -> 140291333940840
1136
+ 140291333941344 [label="
1137
+ (1)" fillcolor=lightblue]
1138
+ 140291333940504 -> 140291558201440
1139
+ 140291333940504 [label=SigmoidBackward]
1140
+ 140291333940728 -> 140291333940504
1141
+ 140291333940728 [label=ThnnConv2DBackward]
1142
+ 140291333941400 -> 140291333940728
1143
+ 140291333941400 [label=ReluBackward1]
1144
+ 140291333941792 -> 140291333941400
1145
+ 140291333941792 [label=ThnnConv2DBackward]
1146
+ 140291333941904 -> 140291333941792
1147
+ 140291333941904 [label=MeanBackward1]
1148
+ 140291333939664 -> 140291333941904
1149
+ 140291333941960 -> 140291333941792
1150
+ 140291333941960 [label="
1151
+ (16, 128, 1, 1)" fillcolor=lightblue]
1152
+ 140291333942016 -> 140291333941792
1153
+ 140291333942016 [label="
1154
+ (16)" fillcolor=lightblue]
1155
+ 140291333941512 -> 140291333940728
1156
+ 140291333941512 [label="
1157
+ (128, 16, 1, 1)" fillcolor=lightblue]
1158
+ 140291333941680 -> 140291333940728
1159
+ 140291333941680 [label="
1160
+ (128)" fillcolor=lightblue]
1161
+ 140291333940056 -> 140291558199984
1162
+ 140291333940056 [label="
1163
+ (64, 128, 3, 3)" fillcolor=lightblue]
1164
+ 140291333940560 -> 140291558199984
1165
+ 140291333940560 [label="
1166
+ (64)" fillcolor=lightblue]
1167
+ 140291558200208 -> 140291558199816
1168
+ 140291558200208 [label=ReluBackward0]
1169
+ 140291558199984 -> 140291558200208
1170
+ 140291558199592 -> 140291558199424
1171
+ 140291558199592 [label="
1172
+ (3, 64, 3, 3)" fillcolor=lightblue]
1173
+ 140291558199648 -> 140291558199424
1174
+ 140291558199648 [label="
1175
+ (3)" fillcolor=lightblue]
1176
+ 140291558534784 -> 140291558534952
1177
+ 140291558534784 [label=SigmoidBackward]
1178
+ 140291558199480 -> 140291558534784
1179
+ 140291558199480 [label=MkldnnConvolutionBackward]
1180
+ 140291558199872 -> 140291558199480
1181
+ 140291558199872 [label=AddBackward0]
1182
+ 140291333940952 -> 140291558199872
1183
+ 140291333940952 [label=ReluBackward0]
1184
+ 140291558199984 -> 140291333940952
1185
+ 140291333940448 -> 140291558199872
1186
+ 140291333940448 [label=SubBackward0]
1187
+ 140291558199928 -> 140291333940448
1188
+ 140291333942072 -> 140291333940448
1189
+ 140291333942072 [label=ReluBackward0]
1190
+ 140291558199928 -> 140291333942072
1191
+ 140291558200376 -> 140291558199480
1192
+ 140291558200376 [label="
1193
+ (3, 64, 3, 3)" fillcolor=lightblue]
1194
+ 140291333940392 -> 140291558199480
1195
+ 140291333940392 [label="
1196
+ (3)" fillcolor=lightblue]
1197
+ }
DSRNet/models/arch/~/YTMT_US.pdf ADDED
Binary file (64.3 kB). View file
 
DSRNet/models/arch/~/tmp.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:678f82d68747dec3e072253174ea87f3131c2a354c71769c5e8648503933ebb6
3
+ size 962515
DSRNet/models/base_model.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import torch
3
+ import util.util as util
4
+ from tools import mutils
5
+
6
+
7
+ class BaseModel:
8
+ def name(self):
9
+ return self.__class__.__name__.lower()
10
+
11
+ def initialize(self, opt):
12
+ self.opt = opt
13
+ self.gpu_ids = opt.gpu_ids
14
+ self.isTrain = opt.isTrain
15
+ self.Tensor = torch.cuda.FloatTensor if self.gpu_ids else torch.Tensor
16
+ last_split = opt.checkpoints_dir.split('/')[-1]
17
+ if opt.resume and last_split != 'checkpoints' and (last_split != opt.name or opt.supp_eval):
18
+
19
+ self.save_dir = opt.checkpoints_dir
20
+ self.model_save_dir = os.path.join(opt.checkpoints_dir.replace(opt.checkpoints_dir.split('/')[-1], ''),
21
+ opt.name, 'weights', mutils.get_formatted_time())
22
+ else:
23
+ self.save_dir = os.path.join(opt.checkpoints_dir, opt.name)
24
+ self.model_save_dir = os.path.join(opt.checkpoints_dir, opt.name, 'weights', mutils.get_formatted_time())
25
+ self._count = 0
26
+
27
+ def set_input(self, input):
28
+ self.input = input
29
+
30
+ def forward(self, mode='train'):
31
+ pass
32
+
33
+ # used in test time, no backprop
34
+ def test(self):
35
+ pass
36
+
37
+ def get_image_paths(self):
38
+ pass
39
+
40
+ def optimize_parameters(self):
41
+ pass
42
+
43
+ def get_current_visuals(self):
44
+ return self.input
45
+
46
+ def get_current_errors(self):
47
+ return {}
48
+
49
+ def print_optimizer_param(self):
50
+ print(self.optimizers[-1])
51
+
52
+ def save(self, label=None):
53
+ epoch = self.epoch
54
+ iterations = self.iterations
55
+
56
+ os.makedirs(self.model_save_dir, exist_ok=True)
57
+ if label is None:
58
+ model_name = os.path.join(self.model_save_dir, self.opt.name + '_%03d_%08d.pt' % ((epoch), (iterations)))
59
+ else:
60
+ model_name = os.path.join(self.model_save_dir, self.opt.name + '_' + label + '.pt')
61
+
62
+ torch.save(self.state_dict(), model_name)
63
+
64
+ def save_eval(self, label=None):
65
+ model_name = os.path.join(self.model_save_dir, label + '.pt')
66
+
67
+ torch.save(self.state_dict_eval(), model_name)
68
+
69
+ def _init_optimizer(self, optimizers):
70
+ self.optimizers = optimizers
71
+ for optimizer in self.optimizers:
72
+ util.set_opt_param(optimizer, 'initial_lr', self.opt.lr)
73
+ util.set_opt_param(optimizer, 'weight_decay', self.opt.wd)
DSRNet/models/dsrnet_model_sirs.py ADDED
@@ -0,0 +1,263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from collections import OrderedDict
3
+ from os.path import join
4
+
5
+ import numpy as np
6
+ import torch
7
+ from PIL import Image
8
+
9
+ import models
10
+ import models.networks as networks
11
+ import util.index as index
12
+ import util.util as util
13
+ from models import arch
14
+ from .base_model import BaseModel
15
+
16
+
17
+ def tensor2im(image_tensor, imtype=np.uint8):
18
+ image_tensor = image_tensor.detach()
19
+ image_numpy = image_tensor[0].cpu().float().numpy()
20
+ image_numpy = np.clip(image_numpy, 0, 1)
21
+ if image_numpy.shape[0] == 1:
22
+ image_numpy = np.tile(image_numpy, (3, 1, 1))
23
+ image_numpy = (np.transpose(image_numpy, (1, 2, 0))) * 255.0
24
+ # image_numpy = image_numpy.astype(imtype)
25
+ return image_numpy
26
+
27
+
28
+ class DSRNetBase(BaseModel):
29
+ def _init_optimizer(self, optimizers):
30
+ self.optimizers = optimizers
31
+ for optimizer in self.optimizers:
32
+ util.set_opt_param(optimizer, 'initial_lr', self.opt.lr)
33
+ util.set_opt_param(optimizer, 'weight_decay', self.opt.wd)
34
+
35
+ def set_input(self, data, mode='train'):
36
+ target_t = None
37
+ target_r = None
38
+ data_name = None
39
+ identity = False
40
+ mode = mode.lower()
41
+ if mode == 'train':
42
+ input, target_t, target_r = data['input'], data['target_t'], data['target_r']
43
+ elif mode == 'eval':
44
+ input, target_t, target_r, data_name = data['input'], data['target_t'], data['target_r'], data['fn']
45
+ elif mode == 'test':
46
+ input, data_name = data['input'], data['fn']
47
+ else:
48
+ raise NotImplementedError('Mode [%s] is not implemented' % mode)
49
+
50
+ if len(self.gpu_ids) > 0: # transfer data into gpu
51
+ input = input.to(device=self.gpu_ids[0])
52
+ if target_t is not None:
53
+ target_t = target_t.to(device=self.gpu_ids[0])
54
+ if target_r is not None:
55
+ target_r = target_r.to(device=self.gpu_ids[0])
56
+
57
+ self.input = input
58
+ self.identity = identity
59
+ self.target_t = target_t
60
+ self.target_r = target_r
61
+ self.data_name = data_name
62
+
63
+ self.issyn = False if 'real' in data else True
64
+ self.aligned = False if 'unaligned' in data else True
65
+
66
+ def eval(self, data, savedir=None, suffix=None, pieapp=None):
67
+ # only the 1st input of the whole minibatch would be evaluated
68
+ self._eval()
69
+ self.set_input(data, 'eval')
70
+
71
+ with torch.no_grad():
72
+ self.forward()
73
+
74
+ output_t = tensor2im(self.output_t)
75
+ output_r = tensor2im(self.output_r)
76
+ output_rr = tensor2im(torch.clip((self.output_rr + 1) / 2, 0., 1.))
77
+ target = tensor2im(self.target_t)
78
+ target_r = tensor2im(self.target_r)
79
+
80
+ if self.aligned:
81
+ res = index.quality_assess(output_t, target)
82
+ # res = index.quality_assess(output_j, target_r)
83
+ else:
84
+ res = {}
85
+
86
+ if savedir is not None:
87
+ if self.data_name is not None:
88
+ name = os.path.splitext(os.path.basename(self.data_name[0]))[0]
89
+ savedir = join(savedir, suffix, name)
90
+ os.makedirs(savedir, exist_ok=True)
91
+ Image.fromarray(output_t.astype(np.uint8)).save(
92
+ join(savedir, '{}_t.png'.format(self.opt.name)))
93
+ Image.fromarray(output_r.astype(np.uint8)).save(
94
+ join(savedir, '{}_r.png'.format(self.opt.name)))
95
+ Image.fromarray(output_rr.astype(np.uint8)).save(
96
+ join(savedir, '{}_rr.png'.format(self.opt.name)))
97
+
98
+ Image.fromarray(target.astype(np.uint8)).save(join(savedir, 't_label.png'))
99
+ Image.fromarray(target_r.astype(np.uint8)).save(join(savedir, 'r_label.png'))
100
+ Image.fromarray(tensor2im(self.input).astype(np.uint8)).save(join(savedir, 'm_input.png'))
101
+ else:
102
+ if not os.path.exists(join(savedir, 'transmission_layer')):
103
+ os.makedirs(join(savedir, 'transmission_layer'))
104
+ os.makedirs(join(savedir, 'blended'))
105
+ Image.fromarray(target.astype(np.uint8)).save(
106
+ join(savedir, 'transmission_layer', str(self._count) + '.png'))
107
+ Image.fromarray(tensor2im(self.input).astype(np.uint8)).save(
108
+ join(savedir, 'blended', str(self._count) + '.png'))
109
+ self._count += 1
110
+
111
+ return res
112
+
113
+ def test(self, data, savedir=None):
114
+ # only the 1st input of the whole minibatch would be evaluated
115
+ self._eval()
116
+ self.set_input(data, 'test')
117
+
118
+ if self.data_name is not None and savedir is not None:
119
+ name = os.path.splitext(os.path.basename(self.data_name[0]))[0]
120
+ if not os.path.exists(join(savedir, name)):
121
+ os.makedirs(join(savedir, name))
122
+
123
+ if os.path.exists(join(savedir, name, '{}.png'.format(self.opt.name))):
124
+ return
125
+
126
+ with torch.no_grad():
127
+ output_i, output_j, output_rr = self.forward()
128
+ output_i = tensor2im(output_i)
129
+ output_j = tensor2im(output_j)
130
+ if self.data_name is not None and savedir is not None:
131
+ Image.fromarray(output_i.astype(np.uint8)).save(join(savedir, name, '{}_l.png'.format(self.opt.name)))
132
+ Image.fromarray(output_j.astype(np.uint8)).save(join(savedir, name, '{}_r.png'.format(self.opt.name)))
133
+ Image.fromarray(tensor2im(torch.clip((output_rr + 1) / 2, 0., 1.)).astype(np.uint8)).save(
134
+ join(savedir, name, '{}_rr.png'.format(self.opt.name)))
135
+ Image.fromarray(tensor2im(self.input).astype(np.uint8)).save(join(savedir, name, 'm_input.png'))
136
+
137
+
138
+ class DSRNetModel(DSRNetBase):
139
+ def name(self):
140
+ return 'dsrnet'
141
+
142
+ def __init__(self):
143
+ self.epoch = 0
144
+ self.iterations = 0
145
+ self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
146
+
147
+ def print_network(self):
148
+ print('--------------------- Model ---------------------')
149
+ print('##################### NetG #####################')
150
+ networks.print_network(self.network)
151
+
152
+ def _eval(self):
153
+ self.network.eval()
154
+
155
+ def _train(self):
156
+ self.network.train()
157
+
158
+ def initialize(self, opt):
159
+ BaseModel.initialize(self, opt)
160
+ in_channels = 3
161
+ losses = getattr(models, self.opt.loss)
162
+ self.vgg = losses.Vgg19(requires_grad=False).to(self.device)
163
+ self.network = arch.__dict__[self.opt.inet](in_channels, 3).to(self.device)
164
+ networks.init_weights(self.network, init_type=opt.init_type) # using default initialization as EDSR
165
+
166
+ if self.isTrain:
167
+ # define loss functions
168
+ self.loss_dic = losses.init_loss(opt)
169
+ self.loss_dic['vgg'] = losses.VGGLoss(self.vgg)
170
+ # initialize optimizers
171
+ self.optimizer_G = torch.optim.Adam(self.network.parameters(),
172
+ lr=opt.lr, betas=(0.9, 0.999), weight_decay=opt.wd)
173
+
174
+ self._init_optimizer([self.optimizer_G])
175
+
176
+ if opt.resume:
177
+ self.load(self)
178
+
179
+ if opt.no_verbose is False:
180
+ self.print_network()
181
+
182
+ def get_loss(self, out_t, out_r, out_rr):
183
+ loss_t_pixel = self.loss_dic['t_pixel'](out_t, self.target_t)
184
+ loss_r_pixel = self.loss_dic['r_pixel'](out_r, self.target_r)
185
+ loss_t_vgg = self.loss_dic['vgg'](out_t, self.target_t) * self.opt.lambda_vgg
186
+ loss_exclu = self.loss_dic['exclu'](self.output_t, self.output_r)
187
+ loss_recons = self.loss_dic['recons'](
188
+ self.output_t, self.output_r, self.output_rr, self.input) * self.opt.lambda_rec
189
+ return loss_t_pixel, loss_r_pixel, loss_t_vgg, loss_exclu, loss_recons
190
+
191
+ def backward_G(self):
192
+ self.loss_t_pixel, self.loss_r_pixel, self.loss_t_vgg, \
193
+ self.loss_exclu, self.loss_recons = self.get_loss(self.output_t, self.output_r, self.output_rr)
194
+ self.loss_G = self.loss_t_pixel + self.loss_r_pixel + self.loss_t_vgg + self.loss_exclu + self.loss_recons
195
+ self.loss_G.backward()
196
+
197
+ def forward(self):
198
+ # without edge
199
+ input_i = self.input
200
+ output_t, output_r, output_rr = self.network(input_i,
201
+ self.vgg(input_i),
202
+ fn=self.data_name[0] if self.data_name else None)
203
+ self.output_t = output_t
204
+ self.output_r = output_r
205
+ self.output_rr = output_rr
206
+ return output_t, output_r, output_rr
207
+
208
+ def optimize_parameters(self):
209
+ self._train()
210
+ self.forward()
211
+
212
+ self.optimizer_G.zero_grad()
213
+ self.backward_G()
214
+
215
+ self.optimizer_G.step()
216
+
217
+ def get_current_errors(self):
218
+ ret_errors = OrderedDict()
219
+ if self.loss_r_pixel is not None:
220
+ ret_errors['R_P'] = self.loss_r_pixel.item()
221
+ if self.loss_t_pixel is not None:
222
+ ret_errors['I_P'] = self.loss_t_pixel.item()
223
+ if self.loss_t_vgg is not None:
224
+ ret_errors['VGG'] = self.loss_t_vgg.item()
225
+ if self.loss_exclu is not None:
226
+ ret_errors['Ex'] = self.loss_exclu.item()
227
+ if self.loss_recons is not None:
228
+ ret_errors['Re'] = self.loss_recons.item()
229
+
230
+ ret_errors['lr'] = self.optimizer_G.param_groups[0]['lr']
231
+ ret_errors['seed'] = self.opt.seed
232
+ return ret_errors
233
+
234
+ def get_current_visuals(self):
235
+ ret_visuals = OrderedDict()
236
+ ret_visuals['input'] = tensor2im(self.input).astype(np.uint8)
237
+ ret_visuals['output_t'] = tensor2im(self.output_t).astype(np.uint8)
238
+ ret_visuals['output_r'] = tensor2im(self.output_r).astype(np.uint8)
239
+ ret_visuals['output_rr'] = tensor2im(torch.clip((self.output_rr + 1) / 2, 0., 1.)).astype(np.uint8)
240
+ ret_visuals['target'] = tensor2im(self.target_t).astype(np.uint8)
241
+ ret_visuals['reflection'] = tensor2im(self.target_r).astype(np.uint8)
242
+
243
+ return ret_visuals
244
+
245
+ def load(self, model):
246
+ weight_path = model.opt.weight_path
247
+ state_dict = torch.load(weight_path)
248
+ model.network.load_state_dict(state_dict)
249
+ return state_dict
250
+
251
+ def state_dict(self, save_extra_state=True):
252
+ state_dict = {
253
+ 'weight': self.network.state_dict(),
254
+ 'epoch': self.epoch,
255
+ 'iterations': self.iterations
256
+ }
257
+
258
+ if save_extra_state:
259
+ state_dict.update({
260
+ 'opt_g': self.optimizer_G.state_dict()
261
+ })
262
+
263
+ return state_dict
DSRNet/models/losses.py ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ import torch.nn.functional as F
4
+
5
+ from models.vgg import Vgg19
6
+
7
+
8
+ def compute_grad(img):
9
+ gradx = img[..., 1:, :] - img[..., :-1, :]
10
+ grady = img[..., 1:] - img[..., :-1]
11
+ return gradx, grady
12
+
13
+
14
+ class GradientLoss(nn.Module):
15
+ def __init__(self):
16
+ super(GradientLoss, self).__init__()
17
+ self.loss = nn.L1Loss()
18
+
19
+ def forward(self, predict, target):
20
+ predict_gradx, predict_grady = compute_grad(predict)
21
+ target_gradx, target_grady = compute_grad(target)
22
+
23
+ return self.loss(predict_gradx, target_gradx) + self.loss(predict_grady, target_grady)
24
+
25
+
26
+ class MultipleLoss(nn.Module):
27
+ def __init__(self, losses, weight=None):
28
+ super(MultipleLoss, self).__init__()
29
+ self.losses = nn.ModuleList(losses)
30
+ self.weight = weight or [1 / len(self.losses)] * len(self.losses)
31
+
32
+ def forward(self, predict, target):
33
+ total_loss = 0
34
+ for weight, loss in zip(self.weight, self.losses):
35
+ total_loss += loss(predict, target) * weight
36
+ return total_loss
37
+
38
+
39
+ class MeanShift(nn.Conv2d):
40
+ def __init__(self, data_mean, data_std, data_range=1, norm=True):
41
+ """norm (bool): normalize/denormalize the stats"""
42
+ c = len(data_mean)
43
+ super(MeanShift, self).__init__(c, c, kernel_size=1)
44
+ std = torch.Tensor(data_std)
45
+ self.weight.data = torch.eye(c).view(c, c, 1, 1)
46
+ if norm:
47
+ self.weight.data.div_(std.view(c, 1, 1, 1))
48
+ self.bias.data = -1 * data_range * torch.Tensor(data_mean)
49
+ self.bias.data.div_(std)
50
+ else:
51
+ self.weight.data.mul_(std.view(c, 1, 1, 1))
52
+ self.bias.data = data_range * torch.Tensor(data_mean)
53
+ self.requires_grad = False
54
+
55
+
56
+ class VGGLoss(nn.Module):
57
+ def __init__(self, vgg=None, weights=None, indices=None, normalize=True):
58
+ super(VGGLoss, self).__init__()
59
+ if vgg is None:
60
+ self.vgg = Vgg19().cuda()
61
+ else:
62
+ self.vgg = vgg
63
+ self.criterion = nn.L1Loss()
64
+ self.weights = weights or [1.0 / 2.6, 1.0 / 4.8, 1.0 / 3.7, 1.0 / 5.6, 10 / 1.5]
65
+ self.indices = indices or [2, 7, 12, 21, 30]
66
+ if normalize:
67
+ self.normalize = MeanShift([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], norm=True).cuda()
68
+ else:
69
+ self.normalize = None
70
+
71
+ def forward(self, x, y):
72
+ if self.normalize is not None:
73
+ x = self.normalize(x)
74
+ y = self.normalize(y)
75
+ x_vgg, y_vgg = self.vgg(x, self.indices), self.vgg(y, self.indices)
76
+ loss = 0
77
+ for i in range(len(x_vgg)):
78
+ loss += self.weights[i] * self.criterion(x_vgg[i], y_vgg[i].detach())
79
+
80
+ return loss
81
+
82
+
83
+ class ReconsLoss(nn.Module):
84
+ def __init__(self):
85
+ super().__init__()
86
+ self.criterion = nn.L1Loss()
87
+
88
+ def forward(self, out_t, out_r, out_rr, input_i):
89
+ content_diff = self.criterion(out_t + out_r + out_rr, input_i)
90
+ return content_diff
91
+
92
+
93
+ class ExclusionLoss(nn.Module):
94
+ def __init__(self, level=3, eps=1e-6):
95
+ super().__init__()
96
+ self.level = level
97
+ self.eps = eps
98
+
99
+ def forward(self, img_T, img_R):
100
+ grad_x_loss = []
101
+ grad_y_loss = []
102
+
103
+ for l in range(self.level):
104
+ grad_x_T, grad_y_T = compute_grad(img_T)
105
+ grad_x_R, grad_y_R = compute_grad(img_R)
106
+
107
+ alphax = (2.0 * torch.mean(torch.abs(grad_x_T))) / (torch.mean(torch.abs(grad_x_R)) + self.eps)
108
+ alphay = (2.0 * torch.mean(torch.abs(grad_y_T))) / (torch.mean(torch.abs(grad_y_R)) + self.eps)
109
+
110
+ gradx1_s = (torch.sigmoid(grad_x_T) * 2) - 1 # mul 2 minus 1 is to change sigmoid into tanh
111
+ grady1_s = (torch.sigmoid(grad_y_T) * 2) - 1
112
+ gradx2_s = (torch.sigmoid(grad_x_R * alphax) * 2) - 1
113
+ grady2_s = (torch.sigmoid(grad_y_R * alphay) * 2) - 1
114
+
115
+ grad_x_loss.append(((torch.mean(torch.mul(gradx1_s.pow(2), gradx2_s.pow(2)))) + self.eps) ** 0.25)
116
+ grad_y_loss.append(((torch.mean(torch.mul(grady1_s.pow(2), grady2_s.pow(2)))) + self.eps) ** 0.25)
117
+
118
+ img_T = F.interpolate(img_T, scale_factor=0.5, mode='bilinear')
119
+ img_R = F.interpolate(img_R, scale_factor=0.5, mode='bilinear')
120
+ loss_gradxy = torch.sum(sum(grad_x_loss) / 3) + torch.sum(sum(grad_y_loss) / 3)
121
+
122
+ return loss_gradxy / 2
123
+
124
+
125
+ def init_loss(opt):
126
+ loss_dic = {}
127
+ pixel_loss = MultipleLoss([nn.MSELoss(), GradientLoss()], [0.3, 0.6])
128
+ loss_dic['t_pixel'] = pixel_loss
129
+ loss_dic['r_pixel'] = pixel_loss
130
+ loss_dic['recons'] = ReconsLoss()
131
+ loss_dic['exclu'] = ExclusionLoss(level=3)
132
+ return loss_dic
133
+
134
+
135
+ if __name__ == '__main__':
136
+ x = torch.randn(3, 32, 224, 224).cuda()
DSRNet/models/networks.py ADDED
@@ -0,0 +1,276 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import functools
2
+
3
+ import numpy as np
4
+ import torch
5
+ import torch.nn as nn
6
+ from torch.nn import init
7
+
8
+
9
+ ###############################################################################
10
+ # Functions
11
+ ###############################################################################
12
+
13
+
14
+ def weights_init_normal(m):
15
+ classname = m.__class__.__name__
16
+ # print(classname)
17
+ if isinstance(m, nn.Sequential):
18
+ return
19
+ if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)):
20
+ init.normal_(m.weight.data, 0.0, 0.02)
21
+ elif isinstance(m, nn.Linear):
22
+ init.normal_(m.weight.data, 0.0, 0.02)
23
+ elif isinstance(m, nn.BatchNorm2d):
24
+ init.normal_(m.weight.data, 1.0, 0.02)
25
+ init.constant_(m.bias.data, 0.0)
26
+
27
+
28
+ def weights_init_xavier(m):
29
+ classname = m.__class__.__name__
30
+ # print(classname)
31
+ if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)):
32
+ init.xavier_normal_(m.weight.data, gain=0.02)
33
+ elif isinstance(m, nn.Linear):
34
+ init.xavier_normal_(m.weight.data, gain=0.02)
35
+ elif isinstance(m, nn.BatchNorm2d):
36
+ init.normal_(m.weight.data, 1.0, 0.02)
37
+ init.constant_(m.bias.data, 0.0)
38
+
39
+
40
+ def weights_init_kaiming(m):
41
+ classname = m.__class__.__name__
42
+ # print(classname)
43
+ if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)):
44
+ init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
45
+ elif isinstance(m, nn.Linear):
46
+ init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
47
+ elif isinstance(m, nn.BatchNorm2d):
48
+ init.normal_(m.weight.data, 1.0, 0.02)
49
+ init.constant_(m.bias.data, 0.0)
50
+
51
+
52
+ def weights_init_orthogonal(m):
53
+ classname = m.__class__.__name__
54
+ print(classname)
55
+ if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)):
56
+ init.orthogonal(m.weight.data, gain=1)
57
+ elif isinstance(m, nn.Linear):
58
+ init.orthogonal(m.weight.data, gain=1)
59
+ elif isinstance(m, nn.BatchNorm2d):
60
+ init.normal(m.weight.data, 1.0, 0.02)
61
+ init.constant_(m.bias.data, 0.0)
62
+
63
+
64
+ def init_weights(net, init_type='normal'):
65
+ print('[i] initialization method [%s]' % init_type)
66
+ if init_type == 'normal':
67
+ net.apply(weights_init_normal)
68
+ elif init_type == 'xavier':
69
+ net.apply(weights_init_xavier)
70
+ elif init_type == 'kaiming':
71
+ net.apply(weights_init_kaiming)
72
+ elif init_type == 'orthogonal':
73
+ net.apply(weights_init_orthogonal)
74
+ elif init_type == 'edsr':
75
+ pass
76
+ else:
77
+ raise NotImplementedError('initialization method [%s] is not implemented' % init_type)
78
+
79
+
80
+ def get_norm_layer(norm_type='instance'):
81
+ if norm_type == 'batch':
82
+ norm_layer = functools.partial(nn.BatchNorm2d, affine=True)
83
+ elif norm_type == 'instance':
84
+ norm_layer = functools.partial(nn.InstanceNorm2d, affine=False)
85
+ elif norm_type == 'none':
86
+ norm_layer = None
87
+ else:
88
+ raise NotImplementedError('normalization layer [%s] is not found' % norm_type)
89
+ return norm_layer
90
+
91
+
92
+ def define_D(opt, in_channels=3):
93
+ # use_sigmoid = opt.gan_type == 'gan'
94
+ use_sigmoid = False # incorporate sigmoid into BCE_stable loss
95
+
96
+ if opt.which_model_D == 'disc_vgg':
97
+ netD = Discriminator_VGG(in_channels, use_sigmoid=use_sigmoid)
98
+ init_weights(netD, init_type='kaiming')
99
+ elif opt.which_model_D == 'disc_patch':
100
+ netD = NLayerDiscriminator(in_channels, 64, 3, nn.InstanceNorm2d, use_sigmoid, getIntermFeat=False)
101
+ init_weights(netD, init_type='normal')
102
+ else:
103
+ raise NotImplementedError('%s is not implemented' % opt.which_model_D)
104
+
105
+ if len(opt.gpu_ids) > 0:
106
+ assert (torch.cuda.is_available())
107
+ netD.cuda(opt.gpu_ids[0])
108
+
109
+ return netD
110
+
111
+
112
+ def print_network(net):
113
+ num_params = 0
114
+ for param in net.parameters():
115
+ num_params += param.numel()
116
+ # print(net)
117
+ print('Network Architecture: ', type(net).__name__)
118
+ print('Total number of parameters: %d,%.3fMb' % (num_params, num_params / (1024 * 1024)))
119
+ print('The size of receptive field: %d' % receptive_field(net))
120
+
121
+
122
+ def receptive_field(net):
123
+ def _f(output_size, ksize, stride, dilation):
124
+ return (output_size - 1) * stride + ksize * dilation - dilation + 1
125
+
126
+ stats = []
127
+ # print(list(net.modules()))
128
+ for m in net.modules():
129
+ if isinstance(m, nn.Conv2d):
130
+ stats.append((m, m.kernel_size, m.stride, m.dilation))
131
+
132
+ rsize = 1
133
+ for (name, ksize, stride, dilation) in reversed(stats):
134
+ if type(ksize) == tuple: ksize = ksize[0]
135
+ if type(stride) == tuple: stride = stride[0]
136
+ if type(dilation) == tuple: dilation = dilation[0]
137
+ rsize = _f(rsize, ksize, stride, dilation)
138
+ return rsize
139
+
140
+
141
+ def debug_network(net):
142
+ def _hook(m, i, o):
143
+ print(o.size())
144
+
145
+ for m in net.modules():
146
+ m.register_forward_hook(_hook)
147
+
148
+
149
+ ##############################################################################
150
+ # Classes
151
+ ##############################################################################
152
+
153
+ # Defines the PatchGAN discriminator with the specified arguments.
154
+ class NLayerDiscriminator(nn.Module):
155
+ def __init__(self, input_nc, ndf=64, n_layers=3,
156
+ norm_layer=nn.BatchNorm2d, use_sigmoid=False,
157
+ branch=1, bias=True, getIntermFeat=False):
158
+ super(NLayerDiscriminator, self).__init__()
159
+ self.getIntermFeat = getIntermFeat
160
+ self.n_layers = n_layers
161
+ kw = 4
162
+ padw = int(np.ceil((kw - 1.0) / 2))
163
+ sequence = [[nn.Conv2d(input_nc * branch, ndf * branch, kernel_size=kw, stride=2, padding=padw, groups=branch,
164
+ bias=True), nn.LeakyReLU(0.2, True)]]
165
+
166
+ nf = ndf
167
+ for n in range(1, n_layers):
168
+ nf_prev = nf
169
+ nf = min(nf * 2, 512)
170
+ sequence += [[
171
+ nn.Conv2d(nf_prev * branch, nf * branch, groups=branch, kernel_size=kw, stride=2, padding=padw,
172
+ bias=bias),
173
+ norm_layer(nf * branch), nn.LeakyReLU(0.2, True)
174
+ ]]
175
+
176
+ nf_prev = nf
177
+ nf = min(nf * 2, 512)
178
+ sequence += [[
179
+ nn.Conv2d(nf_prev * branch, nf * branch, groups=branch, kernel_size=kw, stride=1, padding=padw, bias=bias),
180
+ norm_layer(nf * branch),
181
+ nn.LeakyReLU(0.2, True)
182
+ ]]
183
+
184
+ sequence += [
185
+ [nn.Conv2d(nf * branch, 1 * branch, groups=branch, kernel_size=kw, stride=1, padding=padw, bias=True)]]
186
+
187
+ if use_sigmoid:
188
+ sequence += [[nn.Sigmoid()]]
189
+
190
+ if getIntermFeat:
191
+ for n in range(len(sequence)):
192
+ setattr(self, 'model' + str(n), nn.Sequential(*sequence[n]))
193
+ else:
194
+ sequence_stream = []
195
+ for n in range(len(sequence)):
196
+ sequence_stream += sequence[n]
197
+ self.model = nn.Sequential(*sequence_stream)
198
+
199
+ def forward(self, input):
200
+ if self.getIntermFeat:
201
+ res = [input]
202
+ for n in range(self.n_layers + 2):
203
+ model = getattr(self, 'model' + str(n))
204
+ res.append(model(res[-1]))
205
+ return res[1:]
206
+ else:
207
+ return self.model(input)
208
+
209
+
210
+ class Discriminator_VGG(nn.Module):
211
+ def __init__(self, in_channels=3, use_sigmoid=True):
212
+ super(Discriminator_VGG, self).__init__()
213
+
214
+ def conv(*args, **kwargs):
215
+ return nn.Conv2d(*args, **kwargs)
216
+
217
+ num_groups = 32
218
+
219
+ body = [
220
+ conv(in_channels, 64, kernel_size=3, padding=1), # 224
221
+ nn.LeakyReLU(0.2),
222
+
223
+ conv(64, 64, kernel_size=3, stride=2, padding=1), # 112
224
+ nn.GroupNorm(num_groups, 64),
225
+ nn.LeakyReLU(0.2),
226
+
227
+ conv(64, 128, kernel_size=3, padding=1),
228
+ nn.GroupNorm(num_groups, 128),
229
+ nn.LeakyReLU(0.2),
230
+
231
+ conv(128, 128, kernel_size=3, stride=2, padding=1), # 56
232
+ nn.GroupNorm(num_groups, 128),
233
+ nn.LeakyReLU(0.2),
234
+
235
+ conv(128, 256, kernel_size=3, padding=1),
236
+ nn.GroupNorm(num_groups, 256),
237
+ nn.LeakyReLU(0.2),
238
+
239
+ conv(256, 256, kernel_size=3, stride=2, padding=1), # 28
240
+ nn.GroupNorm(num_groups, 256),
241
+ nn.LeakyReLU(0.2),
242
+
243
+ conv(256, 512, kernel_size=3, padding=1),
244
+ nn.GroupNorm(num_groups, 512),
245
+ nn.LeakyReLU(0.2),
246
+
247
+ conv(512, 512, kernel_size=3, stride=2, padding=1), # 14
248
+ nn.GroupNorm(num_groups, 512),
249
+ nn.LeakyReLU(0.2),
250
+
251
+ conv(512, 512, kernel_size=3, stride=1, padding=1),
252
+ nn.GroupNorm(num_groups, 512),
253
+ nn.LeakyReLU(0.2),
254
+
255
+ conv(512, 512, kernel_size=3, stride=2, padding=1), # 7
256
+ nn.GroupNorm(num_groups, 512),
257
+ nn.LeakyReLU(0.2),
258
+ ]
259
+
260
+ tail = [
261
+ nn.AdaptiveAvgPool2d(1),
262
+ nn.Conv2d(512, 1024, kernel_size=1),
263
+ nn.LeakyReLU(0.2),
264
+ nn.Conv2d(1024, 1, kernel_size=1)
265
+ ]
266
+
267
+ if use_sigmoid:
268
+ tail.append(nn.Sigmoid())
269
+
270
+ self.body = nn.Sequential(*body)
271
+ self.tail = nn.Sequential(*tail)
272
+
273
+ def forward(self, x):
274
+ x = self.body(x)
275
+ out = self.tail(x)
276
+ return out
DSRNet/models/vgg.py ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from collections import namedtuple
2
+
3
+ import torch
4
+ from torchvision import models
5
+
6
+
7
+ class Vgg16(torch.nn.Module):
8
+ def __init__(self, requires_grad=False):
9
+ super(Vgg16, self).__init__()
10
+ vgg_pretrained_features = models.vgg16(pretrained=True).features
11
+ self.slice1 = torch.nn.Sequential()
12
+ self.slice2 = torch.nn.Sequential()
13
+ self.slice3 = torch.nn.Sequential()
14
+ self.slice4 = torch.nn.Sequential()
15
+ for x in range(4):
16
+ self.slice1.add_module(str(x), vgg_pretrained_features[x])
17
+ for x in range(4, 9):
18
+ self.slice2.add_module(str(x), vgg_pretrained_features[x])
19
+ for x in range(9, 16):
20
+ self.slice3.add_module(str(x), vgg_pretrained_features[x])
21
+ for x in range(16, 23):
22
+ self.slice4.add_module(str(x), vgg_pretrained_features[x])
23
+ if not requires_grad:
24
+ for param in self.parameters():
25
+ param.requires_grad = False
26
+
27
+ def forward(self, X):
28
+ h = self.slice1(X)
29
+ h_relu1_2 = h
30
+ h = self.slice2(h)
31
+ h_relu2_2 = h
32
+ h = self.slice3(h)
33
+ h_relu3_3 = h
34
+ h = self.slice4(h)
35
+ h_relu4_3 = h
36
+ vgg_outputs = namedtuple("VggOutputs", ['relu1_2', 'relu2_2', 'relu3_3', 'relu4_3'])
37
+ out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3)
38
+ return out
39
+
40
+
41
+ class Vgg19(torch.nn.Module):
42
+ def __init__(self, requires_grad=False):
43
+ super(Vgg19, self).__init__()
44
+ # vgg_pretrained_features = models.vgg19(pretrained=True).features
45
+ self.vgg_pretrained_features = models.vgg19(pretrained=True).features
46
+ # self.slice1 = torch.nn.Sequential()
47
+ # self.slice2 = torch.nn.Sequential()
48
+ # self.slice3 = torch.nn.Sequential()
49
+ # self.slice4 = torch.nn.Sequential()
50
+ # self.slice5 = torch.nn.Sequential()
51
+ # for x in range(2):
52
+ # self.slice1.add_module(str(x), vgg_pretrained_features[x])
53
+ # for x in range(2, 7):
54
+ # self.slice2.add_module(str(x), vgg_pretrained_features[x])
55
+ # for x in range(7, 12):
56
+ # self.slice3.add_module(str(x), vgg_pretrained_features[x])
57
+ # for x in range(12, 21):
58
+ # self.slice4.add_module(str(x), vgg_pretrained_features[x])
59
+ # for x in range(21, 30):
60
+ # self.slice5.add_module(str(x), vgg_pretrained_features[x])
61
+ if not requires_grad:
62
+ for param in self.parameters():
63
+ param.requires_grad = False
64
+
65
+ def forward(self, X, indices=None):
66
+ if indices is None:
67
+ indices = [2, 7, 12, 21, 30]
68
+ out = []
69
+ # indices = sorted(indices)
70
+ for i in range(indices[-1]):
71
+ X = self.vgg_pretrained_features[i](X)
72
+ if (i + 1) in indices:
73
+ out.append(X)
74
+
75
+ return out
76
+
77
+ # h_relu1 = self.slice1(X)
78
+ # h_relu2 = self.slice2(h_relu1)
79
+ # h_relu3 = self.slice3(h_relu2)
80
+ # h_relu4 = self.slice4(h_relu3)
81
+ # h_relu5 = self.slice5(h_relu4)
82
+ # out = [h_relu1, h_relu2, h_relu3, h_relu4, h_relu5]
83
+ # return out
84
+
85
+
86
+ if __name__ == '__main__':
87
+ vgg = Vgg19()
88
+ import ipdb
89
+
90
+ ipdb.set_trace()
DSRNet/options/__init__.py ADDED
File without changes
DSRNet/options/base_option.py ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #Contains a base class that defines common options used by both training and testing.
2
+
3
+ import argparse #standard library for parsing command-line arguments.
4
+ import models #creates a module object.
5
+ #models.__dict__ is a dictionary that holds the namespace of the models module. This dictionary contains mappings of all names defined within the module e.g. all the symbols (functions, classes, variables, etc.) defined in that module. Stores key and value
6
+ model_names = sorted(name for name in models.__dict__
7
+ if name.islower() and not name.startswith("__")
8
+ and callable(models.__dict__[name]))
9
+ #convention is camelcase for Classes or Exceptions
10
+ #__starting names are magic/dunder methods:the methods starting and ending with double underscores ‘__’. They are defined by built-in classes in Python and commonly used for operator overloading
11
+ #callalble() means it can be called like a function.
12
+ # so left with functioons and callable classes
13
+
14
+ class BaseOptions():
15
+ def __init__(self):
16
+ self.parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) #formatter_class=how help messages should be formatted # argparse.ArgumentDefaultsHelpFormatter: help output should include default values for arguments.
17
+ self.initialized = False #flag for tracking whether the initialize() method has been called to prevent reinitialization
18
+ def initialize(self): # all the arguments added to parser
19
+ # experiment specifics
20
+ self.parser.add_argument('--name', type=str, default=None,
21
+ help='name of the experiment. It decides where to store samples and models')
22
+ self.parser.add_argument('--gpu_ids', type=str, default='0', help='gpu ids: e.g. 0 0,1,2, 0,2. use -1 for CPU')
23
+ self.parser.add_argument('--model', type=str, default='errnet_model', help='chooses which model to use.')
24
+ self.parser.add_argument('--dataset', '-d', type=str, default='sirs_dataset',
25
+ help='chooses which dataset to use.')
26
+ self.parser.add_argument('--loss', '-l', type=str, default='losses',
27
+ help='chooses which loss to use.')
28
+ self.parser.add_argument('--checkpoints_dir', type=str, default='./checkpoints', help='models are saved here')
29
+ self.parser.add_argument('--resume', '-r', action='store_true', help='resume from checkpoint')
30
+ self.parser.add_argument('--resume_epoch', '-re', type=int, default=None,
31
+ help='checkpoint to use. (default: latest')
32
+ self.parser.add_argument('--seed', type=int, default=0, help='random seed to use. Default=0')
33
+ self.parser.add_argument('--supp_eval', action='store_true', help='supplementary evaluation')
34
+ self.parser.add_argument('--start_now', action='store_true', help='supplementary evaluation')
35
+ self.parser.add_argument('--testr', action='store_true', help='test for reflections')
36
+ self.parser.add_argument('--select', type=str, default=None)
37
+
38
+ # for setting input
39
+ self.parser.add_argument('--serial_batches', action='store_true',
40
+ help='if true, takes images in order to make batches, otherwise takes them randomly')
41
+ self.parser.add_argument('--nThreads', default=8, type=int, help='# threads for loading data')
42
+ self.parser.add_argument('--max_dataset_size', type=int, default=None,
43
+ help='Maximum number of samples allowed per dataset. If the dataset directory contains more than max_dataset_size, only a subset is loaded.')
44
+
45
+ #(mingcv) for display
46
+ self.parser.add_argument('--no-log', action='store_true', help='disable tf logger?')
47
+ self.parser.add_argument('--no-verbose', action='store_true', help='disable verbose info?')
48
+ self.parser.add_argument('--display_winsize', type=int, default=256, help='display window size')
49
+ self.parser.add_argument('--display_port', type=int, default=8097, help='visdom port of the web display')
50
+ self.parser.add_argument('--display_id', type=int, default=0,
51
+ help='window id of the web display (use 0 to disable visdom)')
52
+ #In Visdom, you can create multiple windows to display various types of information, such as images, graphs, or tables, during the training and testing of machine learning models. The display_id specifies which window (by ID) the visualizations should be displayed in.
53
+ self.parser.add_argument('--display_single_pane_ncols', type=int, default=0,
54
+ help='if positive, display all images in a single visdom web panel with certain number of images per row.')
55
+
56
+ self.initialized = True
DSRNet/options/net_options/__init__.py ADDED
File without changes
DSRNet/options/net_options/base_options.py ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #configuration based on neural network
2
+ #action=store_true: presence of this argument in the command line will set the corresponding variable to True. If the argument is not specified, the variable will be set to False.Opposite of store_false
3
+ import sys
4
+
5
+ from options.base_option import BaseOptions as Base
6
+ from util import util
7
+ import os
8
+ import torch
9
+ import numpy as np
10
+ import random
11
+
12
+
13
+ class BaseOptions(Base):
14
+ def initialize(self):
15
+ Base.initialize(self)
16
+ # experiment specifics
17
+ self.parser.add_argument('--inet', type=str, default='net_options',
18
+ help='chooses which architecture to use for inet.')
19
+ self.parser.add_argument('--weight_path', type=str, default=None, help='pretrained checkpoint to use.')
20
+ #Path to a pretrained checkpoint for initialization.
21
+ self.parser.add_argument('--init_type', type=str, default='edsr',
22
+ help='network initialization [normal|xavier|kaiming|orthogonal|uniform]')
23
+ #(mingcv)for network
24
+ self.parser.add_argument('--hyper', action='store_true',
25
+ help='if true, augment input with vgg hypercolumn feature')
26
+ # A hypercolumn refers to a vector formed by concatenating the outputs of all feature maps at a particular spatial location across different layers of the VGG network.
27
+ #Benefits: the network can leverage both low-level details and high-level semantic information simultaneously.provides a richer representation of the input image.
28
+ #Implementation: During the forward pass of the neural network:
29
+ #1.Extract feature maps from various layers of the VGG network.
30
+ #2.Concatenate these feature maps spatially (typically by resizing them to a common size).
31
+ #3.Use this concatenated feature vector as an augmented input alongside the original input to the neural network.
32
+
33
+ self.initialized = True
34
+
35
+ def parse(self):
36
+ if not self.initialized:
37
+ self.initialize()
38
+ self.opt = self.parser.parse_args() #Stores parsed command-line arguments.
39
+ self.opt.isTrain = self.isTrain # (mingcv)train or test
40
+ #to use train_options.py or not
41
+ if self.opt.seed == 0:#for reproducibility across modules.
42
+ seed = random.randrange(2 ** 12 - 1)
43
+ self.opt.seed = seed
44
+
45
+ torch.backends.cudnn.deterministic = True
46
+ #PyTorch uses CUDA libraries (like cuDNN) for GPU-accelerated computations. Setting torch.backends.cudnn.deterministic to True ensures that cuDNN uses deterministic algorithms. Operations that rely on cuDNN (like certain convolution operations) will produce the same results on the same input data and configuration.
47
+ torch.manual_seed(self.opt.seed)#Ensure that operations like initializing weights in neural networks or shuffling data batches produce the same results
48
+ np.random.seed(self.opt.seed) #(mingcv) seed for every module
49
+ #np.random.seed:seed for the random number generator in NumPy
50
+ random.seed(self.opt.seed)#seed for the built-in Python random number generator
51
+ #gpu_ids: for multigpu system for parallelization e.g. 0,1,2
52
+ str_ids = self.opt.gpu_ids.split(',')
53
+ self.opt.gpu_ids = []
54
+ for str_id in str_ids:
55
+ id = int(str_id)
56
+ if id >= 0:
57
+ self.opt.gpu_ids.append(id)
58
+ #converting string of gpus ids into a list of ints
59
+ #(mingcv)set gpu ids
60
+ if len(self.opt.gpu_ids) > 0: #if atleast 1 gpu, set first gpu as cuda device
61
+ torch.cuda.set_device(self.opt.gpu_ids[0])
62
+
63
+ args = vars(self.opt)
64
+ #vars() is a built-in function that returns the __dict__ attribute of an object if it exists.
65
+ # args = vars(self.opt) converts the attributes of self.opt into a dictionary args for iteration and accessibility
66
+ print('------------ Options -------------')
67
+ for k, v in sorted(args.items()): # prints all keys and values of dict args
68
+ print('%s: %s' % (str(k), str(v)))
69
+ print('-------------- End ----------------')
70
+
71
+ #(mingcv) save to the disk
72
+ self.opt.name = self.opt.name or '_'.join([self.opt.model])
73
+ # '_'.join([self.opt.model]) if self.opt.name is None
74
+ expr_dir = os.path.join(self.opt.checkpoints_dir, self.opt.name)
75
+ #experiment directory
76
+ util.mkdirs(expr_dir)
77
+ file_name = os.path.join(expr_dir, 'opt.txt')
78
+ with open(file_name, 'wt') as opt_file:
79
+ opt_file.write('------------ Options -------------\n') # write key and values of options in opt.txt
80
+ for k, v in sorted(args.items()):
81
+ opt_file.write('%s: %s\n' % (str(k), str(v)))
82
+ opt_file.write('-------------- End ----------------\n')
83
+
84
+ if self.opt.debug: #debugging mode with small dataset no multithreading,no flipping
85
+ self.opt.display_freq = 20 #display: visual outputs related to training like plots/images
86
+ self.opt.print_freq = 20 #print: training progress information printed in console
87
+ #after processing 20 batches of images, the training results will be printed/displayed.
88
+ self.opt.nEpochs = 40
89
+ self.opt.max_dataset_size = 100
90
+ self.opt.no_log = False
91
+ # controls whether logging of training progress or other information is enabled (False means logging is enabled).
92
+ self.opt.nThreads = 0 #operations are executed in a single thread
93
+ self.opt.decay_iter = 0 # iteration count after which learning rate decay might occur. 0 means no decay
94
+ self.opt.serial_batches = True #whether batches of data are processed in serial (one after another) or in parallel during training. serial:each sample is seen exactly once per epoch in the specified order.
95
+ self.opt.no_flip = True # disables flipping of images during data augmentation.
96
+
97
+ return self.opt
DSRNet/options/net_options/train_options.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from .base_options import BaseOptions #. means look in the same directory(net_options)
2
+
3
+ #action=store_true: presence of this argument in the command line will set the corresponding variable to True. If the argument is not specified, the variable will be set to False.Opposite of store_false
4
+ class TrainOptions(BaseOptions):
5
+ def initialize(self):
6
+ BaseOptions.initialize(self)
7
+ # (mingcv)for displays
8
+ self.parser.add_argument('--display_freq', type=int, default=100,
9
+ help='frequency of showing training results on screen')
10
+ self.parser.add_argument('--update_html_freq', type=int, default=1000,
11
+ help='frequency of saving training results to html')
12
+ self.parser.add_argument('--print_freq', type=int, default=100,
13
+ help='frequency of showing training results on console')
14
+ self.parser.add_argument('--eval_freq', type=int, default=1, help='frequency of evaluation')
15
+ self.parser.add_argument('--save_freq', type=int, default=1, help='frequency of save eval samples')
16
+ self.parser.add_argument('--no_html', action='store_true',
17
+ help='do not save intermediate training results to [opt.checkpoints_dir]/[opt.name]/web/')
18
+ self.parser.add_argument('--save_epoch_freq', type=int, default=1,
19
+ help='frequency of saving checkpoints at the end of epochs')
20
+ self.parser.add_argument('--debug', action='store_true',
21
+ help='only do one epoch and displays at each iteration')
22
+ self.parser.add_argument('--finetune', action='store_true',
23
+ help='finetune the network using identity inputs and outputs')
24
+ #Fine-tuning is a process where a pre-trained model is further trained on a new dataset or task.
25
+ '''???identity inputs and outputs:'''
26
+ self.parser.add_argument('--if_align', action='store_true', help='if align 4x')
27
+ #adjusting images so that they meet specific criteria such as size, orientation, or position relative to a reference frame.
28
+ '''??? here 4x what?'''
29
+ self.parser.add_argument('--weight_dir', type=str, default='')
30
+ self.parser.add_argument('--base_dir', type=str, default='/mnt/lmj_ssd/reflection-removal')
31
+
32
+
33
+ self.parser.add_argument('--nEpochs', '-n', type=int, default=60, help='# of epochs to run')
34
+ #when defining the Adam optimizer
35
+ self.parser.add_argument('--lr', type=float, default=1e-4, help='initial learning rate for adam')
36
+ self.parser.add_argument('--wd', type=float, default=0, help='weight decay for adam')
37
+ self.parser.add_argument('--num_train', type=int, default=-1) #Number of training samples.
38
+ self.parser.add_argument('--r_pixel_weight', '-rw', type=float, default=1.0, help='weight for r_pixel loss')
39
+
40
+ # (mingcv)data augmentation
41
+ self.parser.add_argument('--real20_size', type=int, default=420, help='scale images to compat size')
42
+ self.parser.add_argument('--batchSize', '-b', type=int, default=1, help='input batch size')
43
+ self.parser.add_argument('--loadSize', type=str, default='224,336,448', help='scale images to multiple size')
44
+ self.parser.add_argument('--fineSize', type=str, default='224,224', help='then crop to this size')
45
+ self.parser.add_argument('--no_flip', action='store_true',
46
+ help='if specified, do not flip the images for data augmentation')
47
+ self.parser.add_argument('--resize_or_crop', type=str, default='resize_and_crop',
48
+ help='scaling and cropping of images at load time [resize_and_crop|crop|scale_width|scale_width_and_crop]')
49
+ self.parser.add_argument('--debug_eval', action='store_true',
50
+ help='if specified, do not flip the images for data augmentation')
51
+ self.parser.add_argument('--graph', action='store_true', help='print graph')
52
+ self.parser.add_argument('--selected', type=str, nargs='+')
53
+ #nargs='+': the argument can take one or more values.
54
+
55
+ # (mingcv)for discriminator
56
+ self.parser.add_argument('--which_model_D', type=str, default='disc_vgg', choices=['disc_vgg', 'disc_patch'])
57
+ self.parser.add_argument('--gan_type', type=str, default='rasgan',
58
+ help='gan/sgan : Vanilla GAN; rasgan : relativistic gan')
59
+
60
+ # (mingcv) loss weight
61
+ self.parser.add_argument('--unaligned_loss', type=str, default='vgg',
62
+ help='learning rate policy: vgg|mse|ctx|ctx_vgg')
63
+ #unaligned_loss: loss function used to measure the difference between predicted and target images that are not perfectly aligned.
64
+ #vgg: Perceptual Loss/vgg loss.Uses features extracted from a pre-trained network (e.g., VGG) to measure differences in high-level features instead of raw pixel values.
65
+ #mse:Mean Squared Error.Typically for aligned data, but might be included for certain aligned parts or just for comparison.
66
+ #ctx: Contextual loss. Focuses on the context and structure rather than pixel-wise differences.When structural similarity more important
67
+ #ctx_vgg: combination of contextual and VGG perceptual loss.
68
+ self.parser.add_argument('--vgg_layer', type=int, default=31, help='vgg layer of unaligned loss')
69
+ #--vgg_layer:a specific layer from the VGG network (a pre-trained deep convolutional neural network) used to extract high-level features from images to compute the perceptual loss. If u use vgg/ctx_vgg in --unaligned_loss
70
+ self.parser.add_argument('--init_lr', type=float, default=1e-2, help='initial learning rate')
71
+ self.parser.add_argument('--fixed_lr', type=float, default=0, help='initial learning rate')
72
+ "??? default=0 becuse of scheduler?"
73
+ self.parser.add_argument('--lambda_gan', type=float, default=0.01, help='weight for gan loss')
74
+ self.parser.add_argument('--lambda_vgg', type=float, default=0.1, help='weight for vgg loss')
75
+ self.parser.add_argument('--lambda_rec', type=float, default=0.2, help='weight for reconstruction loss')
76
+
77
+ self.isTrain = True #indicating that these options are for training.
DSRNet/requirements.txt ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dominate==2.9.1
2
+ einops==0.7.0
3
+ kornia==0.7.1
4
+ opencv-python==4.9.0.80
5
+ PyYAML==6.0.1
6
+ scikit-image==0.22.0
7
+ scipy==1.12.0
8
+ tensorboardX==2.6.2.2
9
+ torch
10
+ torchvision
11
+ visdom==0.2.4
DSRNet/scripts.sh ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ ### To train the model
2
+ python train_sirs.py --inet dsrnet_l --model dsrnet_model_sirs --dataset sirs_dataset --loss losses --name dsrnet_l --lambda_vgg 0.01 --lambda_rec 0.2 --if_align --seed 2018 --base_dir "[YOUR_DATA_DIR]"
3
+
4
+ python eval_sirs.py --inet dsrnet_s --model dsrnet_model_sirs --dataset sirs_dataset --name dsrnet_s_test --hyper --if_align --resume --weight_path "./weights/dsrnet_s_epoch14.pt" --base_dir "[YOUR_DATA_DIR]"
5
+
6
+ python eval_sirs.py --inet dsrnet_l --model dsrnet_model_sirs --dataset sirs_dataset --name dsrnet_l_test --hyper --if_align --resume --weight_path "./weights/dsrnet_l_epoch18.pt" --base_dir "[YOUR_DATA_DIR]"
DSRNet/test_sirs.py ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from os.path import join
3
+
4
+ import torch.backends.cudnn as cudnn
5
+
6
+ import data.sirs_dataset as datasets
7
+ from data.image_folder import read_fns
8
+ from engine import Engine
9
+ from options.net_options.train_options import TrainOptions
10
+ from tools import mutils
11
+
12
+ opt = TrainOptions().parse()
13
+
14
+ opt.isTrain = False
15
+ cudnn.benchmark = True
16
+ opt.no_log = True
17
+ opt.display_id = 0
18
+ opt.verbose = False
19
+
20
+ datadir = os.path.join(opt.base_dir, 'test')
21
+ test_dataset_real = datasets.DSRTestDataset(join(datadir, f'real20_420'),
22
+ fns=read_fns('data/real_test.txt'),
23
+ if_align=opt.if_align)
24
+ #test_dataset_real = datasets.RealDataset('./data')
25
+ test_dataloader_real = datasets.DataLoader(test_dataset_real, batch_size=1, shuffle=True, num_workers=opt.nThreads,
26
+ pin_memory=True)
27
+
28
+ engine = Engine(opt)
29
+
30
+ """Main Loop"""
31
+ result_dir = os.path.join('./checkpoints', opt.name, mutils.get_formatted_time())
32
+
33
+ res = engine.test(test_dataloader_real, savedir=join(result_dir, 'test'))
DSRNet/tools/__init__.py ADDED
File without changes
DSRNet/tools/mutils.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import datetime
2
+ import os
3
+ import shutil
4
+ import time
5
+
6
+ #counts number of trainable parameters
7
+ #same as net_utils but different print
8
+ def count_parameters(model):
9
+ number_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
10
+ print('Total Parameters: {:d} or {:.2f}M'.format(number_params, number_params / (1024 * 1024))) #in millions
11
+ return number_params
12
+
13
+
14
+ def contains(key, lst): #return true if lst contains the key
15
+ flag = False
16
+ for item in lst:
17
+ if key == item:
18
+ flag = True
19
+ return flag
20
+
21
+
22
+ def make_empty_dir(new_dir): #deletes previous directory containing files and makes new empty one
23
+ if os.path.exists(new_dir):
24
+ shutil.rmtree(new_dir)
25
+ os.makedirs(new_dir, exist_ok=True)
26
+
27
+
28
+ def get_timestamp():
29
+ return str(time.time()).replace('.', '') #returns time as secondsmilliseconds vs seconds.milliseconds can be used timestamp identifier
30
+
31
+
32
+ def get_formatted_time(): #returns current time formatted as YYYYMMDD-HHMMSS
33
+ return datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
34
+
35
+
36
+ if __name__ == '__main__': #if the script is being run directly (as opposed to being imported as a module).
37
+ pass
DSRNet/train_sirs.py ADDED
@@ -0,0 +1,109 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from os.path import join
3
+
4
+ import data.sirs_dataset as datasets # Custom dataset module for SIRS dataset
5
+ import util.util as util
6
+ from data.image_folder import read_fns
7
+ from engine import Engine
8
+ from options.net_options.train_options import TrainOptions
9
+ from tools import mutils
10
+
11
+ opt = TrainOptions().parse()
12
+ print(opt)
13
+
14
+ opt.display_freq = 10
15
+
16
+ if opt.debug:
17
+ opt.display_id = 1
18
+ opt.display_freq = 1
19
+ opt.print_freq = 20
20
+ opt.nEpochs = 40
21
+ opt.max_dataset_size = 100 #was 9999 (mingcv)
22
+ opt.no_log = False
23
+ opt.nThreads = 0
24
+ opt.decay_iter = 0
25
+ opt.serial_batches = True
26
+ opt.no_flip = True
27
+
28
+ # modify the following code to
29
+ datadir = os.path.join(opt.base_dir)
30
+ datadir_syn = join(datadir, 'train/VOCdevkit/VOC2012/PNGImages')
31
+ datadir_real = join(datadir, 'train/real')
32
+
33
+ train_dataset = datasets.DSRDataset(
34
+ datadir_syn, read_fns('data/VOC2012_224_train_png.txt'), size=opt.max_dataset_size, enable_transforms=True)
35
+
36
+ train_dataset_real = datasets.DSRTestDataset(datadir_real, enable_transforms=True, if_align=opt.if_align)
37
+
38
+ train_dataset_fusion = datasets.FusionDataset([train_dataset,
39
+ train_dataset_real], [0.7, 0.3])
40
+
41
+ train_dataloader_fusion = datasets.DataLoader(
42
+ train_dataset_fusion, batch_size=opt.batchSize, shuffle=not opt.serial_batches,
43
+ pin_memory=True, prefetch_factor=32, num_workers=32)
44
+
45
+ eval_dataset_real = datasets.DSRTestDataset(join(datadir, f'test/real20_{opt.real20_size}'),
46
+ fns=read_fns('data/real_test.txt'), if_align=opt.if_align)
47
+ eval_dataset_solidobject = datasets.DSRTestDataset(join(datadir, 'test/SIR2/SolidObjectDataset'),
48
+ if_align=opt.if_align)
49
+ eval_dataset_postcard = datasets.DSRTestDataset(join(datadir, 'test/SIR2/PostcardDataset'), if_align=opt.if_align)
50
+ eval_dataset_wild = datasets.DSRTestDataset(join(datadir, 'test/SIR2/WildSceneDataset'), if_align=opt.if_align)
51
+
52
+ eval_dataloader_real = datasets.DataLoader(
53
+ eval_dataset_real, batch_size=1, shuffle=False,
54
+ pin_memory=True, prefetch_factor=32, num_workers=32)
55
+
56
+ eval_dataloader_solidobject = datasets.DataLoader(
57
+ eval_dataset_solidobject, batch_size=1, shuffle=False,
58
+ pin_memory=True, prefetch_factor=32, num_workers=32)
59
+ eval_dataloader_postcard = datasets.DataLoader(
60
+ eval_dataset_postcard, batch_size=1, shuffle=False,
61
+ pin_memory=True, prefetch_factor=32, num_workers=32)
62
+
63
+ eval_dataloader_wild = datasets.DataLoader(
64
+ eval_dataset_wild, batch_size=1, shuffle=False,
65
+ pin_memory=True, prefetch_factor=32, num_workers=32)
66
+
67
+ """Main Loop"""
68
+ engine = Engine(opt)
69
+ result_dir = os.path.join(f'./checkpoints/{opt.name}/results',
70
+ mutils.get_formatted_time())
71
+
72
+
73
+ def set_learning_rate(lr):
74
+ for optimizer in engine.model.optimizers:
75
+ print('[i] set learning rate to {}'.format(lr))
76
+ util.set_opt_param(optimizer, 'lr', lr)
77
+
78
+
79
+
80
+ if opt.resume or opt.debug_eval:
81
+ save_dir = os.path.join(result_dir, '%03d' % engine.epoch)
82
+ os.makedirs(save_dir, exist_ok=True)
83
+ engine.save_model()
84
+
85
+ engine.eval(eval_dataloader_real, dataset_name='testdata_real20',
86
+ savedir=save_dir, suffix='real20', max_save_size=10)
87
+ engine.eval(eval_dataloader_solidobject, dataset_name='testdata_solidobject',
88
+ savedir=save_dir, suffix='solidobject', max_save_size=10)
89
+ engine.eval(eval_dataloader_postcard, dataset_name='testdata_postcard',
90
+ savedir=save_dir, suffix='postcard', max_save_size=10)
91
+ engine.eval(eval_dataloader_wild, dataset_name='testdata_wild',
92
+ savedir=save_dir, suffix='wild', max_save_size=10)
93
+ # define training strategy
94
+ set_learning_rate(opt.lr)
95
+ while engine.epoch < 50:
96
+ print('random_seed: ', opt.seed)
97
+ engine.train(train_dataloader_fusion)
98
+
99
+ if engine.epoch % 1 == 0:
100
+ save_dir = os.path.join(result_dir, '%03d' % engine.epoch)
101
+ os.makedirs(save_dir, exist_ok=True)
102
+ engine.eval(eval_dataloader_real, dataset_name='testdata_real20',
103
+ savedir=save_dir, suffix='real20', max_save_size=10)
104
+ engine.eval(eval_dataloader_solidobject, dataset_name='testdata_solidobject',
105
+ savedir=save_dir, suffix='solidobject', max_save_size=10)
106
+ engine.eval(eval_dataloader_postcard, dataset_name='testdata_postcard',
107
+ savedir=save_dir, suffix='postcard', max_save_size=10)
108
+ engine.eval(eval_dataloader_wild, dataset_name='testdata_wild',
109
+ savedir=save_dir, suffix='wild', max_save_size=10)
DSRNet/train_sirs_4000.py ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from os.path import join
3
+
4
+ import torch.backends.cuda
5
+ import torch.backends.cudnn as cudnn
6
+
7
+ import data.sirs_dataset as datasets
8
+ import util.util as util
9
+ from data.image_folder import read_fns
10
+ from engine import Engine
11
+ from options.net_options.train_options import TrainOptions
12
+ from tools import mutils
13
+
14
+ opt = TrainOptions().parse()
15
+ print(opt)
16
+ cudnn.benchmark = False
17
+ torch.backends.cuda.matmul.allow_tf32 = True
18
+
19
+ opt.display_freq = 10
20
+
21
+ if opt.debug:
22
+ opt.display_id = 1
23
+ opt.display_freq = 1
24
+ opt.print_freq = 20
25
+ opt.nEpochs = 40
26
+ opt.max_dataset_size = 9999
27
+ opt.no_log = False
28
+ opt.nThreads = 0
29
+ opt.decay_iter = 0
30
+ opt.serial_batches = True
31
+ opt.no_flip = True
32
+
33
+ # modify the following code to
34
+ # datadir = os.path.join(os.path.expanduser('~'), 'datasets/reflection-removal')
35
+ datadir = os.path.join(opt.base_dir)
36
+
37
+ datadir_syn = join(datadir, 'train/VOCdevkit/VOC2012/PNGImages')
38
+ datadir_real = join(datadir, 'train/real')
39
+ datadir_nature = join(datadir, 'train/nature')
40
+
41
+ train_dataset = datasets.DSRDataset(
42
+ datadir_syn, read_fns('data/VOC2012_224_train_png.txt'), size=opt.max_dataset_size, enable_transforms=True)
43
+
44
+ train_dataset_real = datasets.DSRTestDataset(datadir_real, enable_transforms=True, if_align=opt.if_align)
45
+ train_dataset_nature = datasets.DSRTestDataset(datadir_nature, enable_transforms=True, if_align=opt.if_align)
46
+
47
+ train_dataset_fusion = datasets.FusionDataset([train_dataset,
48
+ train_dataset_real,
49
+ train_dataset_nature], [0.6, 0.2, 0.2],
50
+ size=opt.num_train if opt.num_train > 0 else 4000)
51
+
52
+ train_dataloader_fusion = datasets.DataLoader(
53
+ train_dataset_fusion, batch_size=opt.batchSize, shuffle=not opt.serial_batches,
54
+ num_workers=opt.nThreads, pin_memory=True)
55
+
56
+ eval_dataset_real = datasets.DSRTestDataset(join(datadir, f'test/real20_{opt.real20_size}'),
57
+ fns=read_fns('data/real_test.txt'), if_align=opt.if_align)
58
+ eval_dataset_solidobject = datasets.DSRTestDataset(join(datadir, 'test/SIR2/SolidObjectDataset'),
59
+ if_align=opt.if_align)
60
+ eval_dataset_postcard = datasets.DSRTestDataset(join(datadir, 'test/SIR2/PostcardDataset'), if_align=opt.if_align)
61
+ eval_dataset_wild = datasets.DSRTestDataset(join(datadir, 'test/SIR2/WildSceneDataset'), if_align=opt.if_align)
62
+
63
+ eval_dataloader_real = datasets.DataLoader(
64
+ eval_dataset_real, batch_size=1, shuffle=False,
65
+ num_workers=opt.nThreads, pin_memory=True)
66
+
67
+ eval_dataloader_solidobject = datasets.DataLoader(
68
+ eval_dataset_solidobject, batch_size=1, shuffle=False,
69
+ num_workers=opt.nThreads, pin_memory=True)
70
+ eval_dataloader_postcard = datasets.DataLoader(
71
+ eval_dataset_postcard, batch_size=1, shuffle=False,
72
+ num_workers=opt.nThreads, pin_memory=True)
73
+
74
+ eval_dataloader_wild = datasets.DataLoader(
75
+ eval_dataset_wild, batch_size=1, shuffle=False,
76
+ num_workers=opt.nThreads, pin_memory=True)
77
+
78
+ """Main Loop"""
79
+ engine = Engine(opt)
80
+ result_dir = os.path.join(f'./checkpoints/{opt.name}/results',
81
+ mutils.get_formatted_time())
82
+
83
+
84
+ def set_learning_rate(lr):
85
+ for optimizer in engine.model.optimizers:
86
+ print('[i] set learning rate to {}'.format(lr))
87
+ util.set_opt_param(optimizer, 'lr', lr)
88
+
89
+
90
+ if opt.resume or opt.debug_eval:
91
+ save_dir = os.path.join(result_dir, '%03d' % engine.epoch)
92
+ os.makedirs(save_dir, exist_ok=True)
93
+ engine.save_model()
94
+
95
+ engine.eval(eval_dataloader_real, dataset_name='testdata_real20', savedir=save_dir, suffix='real20')
96
+ engine.eval(eval_dataloader_solidobject, dataset_name='testdata_solidobject', savedir=save_dir,
97
+ suffix='solidobject')
98
+ engine.eval(eval_dataloader_postcard, dataset_name='testdata_postcard', savedir=save_dir, suffix='postcard')
99
+ engine.eval(eval_dataloader_wild, dataset_name='testdata_wild', savedir=save_dir, suffix='wild')
100
+
101
+ # define training strategy
102
+ engine.model.opt.lambda_gan = 0
103
+ # engine.model.opt.lambda_gan = 0.01
104
+ set_learning_rate(opt.lr)
105
+
106
+ while engine.epoch < 120:
107
+ print('random_seed: ', opt.seed)
108
+ engine.train(train_dataloader_fusion)
109
+
110
+ if engine.epoch % 1 == 0:
111
+ save_dir = os.path.join(result_dir, '%03d' % engine.epoch)
112
+ os.makedirs(save_dir, exist_ok=True)
113
+
114
+
115
+ engine.eval(eval_dataloader_real, dataset_name='testdata_real20', savedir=save_dir, suffix='real20')
116
+ engine.eval(eval_dataloader_solidobject, dataset_name='testdata_solidobject', savedir=save_dir, suffix='solidobject')
117
+ engine.eval(eval_dataloader_postcard, dataset_name='testdata_postcard', savedir=save_dir, suffix='postcard')
118
+ engine.eval(eval_dataloader_wild, dataset_name='testdata_wild', savedir=save_dir, suffix='wild')
DSRNet/util/__init__.py ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from .net_utils import *
2
+ from .util import *
3
+
4
+
5
+ def load_checkpoint(model, ckpt_path):
6
+ checkpoint = torch.load(ckpt_path)
7
+ if 'model' in checkpoint:
8
+ checkpoint = checkpoint['model']
9
+ if 'state_dict' in checkpoint:
10
+ checkpoint = checkpoint['state_dict']
11
+ ckpt = {}
12
+ for k, v in checkpoint.items():
13
+ if k.startswith('module.'):
14
+ ckpt[k[7:]] = v
15
+ else:
16
+ ckpt[k] = v
17
+ model.load_state_dict(ckpt,strict=False) #strict=False by nami
DSRNet/util/html.py ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import dominate #used for creating and manipulating HTML documents in a programmatic way.
2
+ from dominate.tags import * #easier to create HTML tags directly.
3
+ import os
4
+
5
+ #ALL reflesh should be refresh
6
+ class HTML:
7
+ def __init__(self, web_dir, title, reflesh=0): #reflesh: refresh_rate
8
+ self.title = title #title of web page
9
+ self.web_dir = web_dir#parent directory of all web content e.g images
10
+ self.img_dir = os.path.join(self.web_dir, 'images')
11
+ if not os.path.exists(self.web_dir):
12
+ os.makedirs(self.web_dir)
13
+ if not os.path.exists(self.img_dir):
14
+ os.makedirs(self.img_dir)
15
+ #(mingcv) print(self.img_dir)
16
+
17
+ self.doc = dominate.document(title=title)
18
+ if reflesh > 0:
19
+ with self.doc.head:
20
+ meta(http_equiv="refresh", content=str(reflesh)) #was "reflesh" in mingcv
21
+ #The http-equiv attribute provides an HTTP header for the information/value of the content attribute.In <head></head>
22
+ def get_image_dir(self):
23
+ return self.img_dir
24
+
25
+ def add_header(self, str):
26
+ with self.doc:
27
+ h3(str)
28
+
29
+ def add_table(self, border=1):
30
+ self.t = table(border=border, style="table-layout: fixed;")
31
+ self.doc.add(self.t)
32
+
33
+ def add_images(self, ims, txts, links, height=400):# add images to the HTML document within a table. Each cell (td) in the table contains an image on one line and its descriptive text on the next line.
34
+ self.add_table()
35
+ with self.t:#opens the table context for adding rows
36
+ with tr():#open table row context
37
+ for im, txt, link in zip(ims, txts, links):#This iterates over the images, texts, and links
38
+ with td(style="word-wrap: break-word;", halign="center", valign="top"):
39
+ with p():
40
+ with a(href=os.path.join('images', link)):
41
+ img(style="height:%dpx" % height, src=os.path.join('images', im))
42
+ br()
43
+ p(txt)
44
+
45
+ def save(self):# save the generated HTML document to a file.
46
+ html_file = '%s/index.html' % self.web_dir # self.web_dir value will be placed in %s like sprintf(html_file, “%s/index.html”, web_dir);
47
+ f = open(html_file, 'wt')#write text mode
48
+ f.write(self.doc.render())# writes the rendered HTML content to the file. The render method of the doc object (which is an instance of the dominate.document class) converts the HTML document into a string.
49
+ f.close()
50
+
51
+
52
+ if __name__ == '__main__':#if the script is being run as the main module. This block will only execute if the script is run directly (not imported as a module).
53
+
54
+ html = HTML('web/', 'test_html')#'web/'=web_dir, 'test_html'=title
55
+ html.add_header('hello world')
56
+
57
+ ims = []#ims for image filenames, txts for image descriptions, and links for image links.
58
+ txts = []
59
+ links = []
60
+ for n in range(4):#adds images, texts and links to the list so that they can be added to html.
61
+ '''??? why 4'''
62
+ ims.append('image_%d.png' % n)
63
+ txts.append('text_%d' % n)
64
+ links.append('image_%d.png' % n)
65
+ html.add_images(ims, txts, links)
66
+ html.save()
DSRNet/util/index.py ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Metrics/Indexes
2
+ from skimage.metrics import peak_signal_noise_ratio as compare_psnr
3
+ from skimage.metrics import structural_similarity as compare_ssim
4
+ from functools import partial
5
+ import numpy as np
6
+
7
+
8
+ class Bandwise(object):
9
+ def __init__(self, index_fn):
10
+ self.index_fn = index_fn
11
+
12
+ def __call__(self, X, Y):
13
+ C = X.shape[-1]
14
+ bwindex = []
15
+ for ch in range(C):
16
+ x = X[..., ch]
17
+ y = Y[..., ch]
18
+ index = self.index_fn(x, y)
19
+ bwindex.append(index)
20
+ return bwindex
21
+
22
+
23
+ cal_bwpsnr = Bandwise(partial(compare_psnr, data_range=255))
24
+ cal_bwssim = Bandwise(partial(compare_ssim, data_range=255))
25
+
26
+
27
+ def compare_ncc(x, y):
28
+ return np.mean((x - np.mean(x)) * (y - np.mean(y))) / (np.std(x) * np.std(y))
29
+
30
+
31
+ def ssq_error(correct, estimate):
32
+ """Compute the sum-squared-error for an image, where the estimate is
33
+ multiplied by a scalar which minimizes the error. Sums over all pixels
34
+ where mask is True. If the inputs are color, each color channel can be
35
+ rescaled independently."""
36
+ assert correct.ndim == 2
37
+ if np.sum(estimate ** 2) > 1e-5:
38
+ alpha = np.sum(correct * estimate) / np.sum(estimate ** 2)
39
+ else:
40
+ alpha = 0.
41
+ return np.sum((correct - alpha * estimate) ** 2)
42
+
43
+
44
+ def local_error(correct, estimate, window_size, window_shift):
45
+ """Returns the sum of the local sum-squared-errors, where the estimate may
46
+ be rescaled within each local region to minimize the error. The windows are
47
+ window_size x window_size, and they are spaced by window_shift."""
48
+ M, N, C = correct.shape
49
+ ssq = total = 0.
50
+ for c in range(C):
51
+ for i in range(0, M - window_size + 1, window_shift):
52
+ for j in range(0, N - window_size + 1, window_shift):
53
+ correct_curr = correct[i:i + window_size, j:j + window_size, c]
54
+ estimate_curr = estimate[i:i + window_size, j:j + window_size, c]
55
+ ssq += ssq_error(correct_curr, estimate_curr)
56
+ total += np.sum(correct_curr ** 2)
57
+ # assert np.isnan(ssq/total)
58
+ return ssq / total
59
+
60
+
61
+ def quality_assess(X, Y):
62
+ # Y: correct; X: estimate
63
+ psnr = np.mean(cal_bwpsnr(Y, X))
64
+ ssim = np.mean(cal_bwssim(Y, X))
65
+ lmse = local_error(Y, X, 20, 10)
66
+ ncc = compare_ncc(Y, X)
67
+ return {'PSNR': psnr, 'SSIM': ssim, 'LMSE': lmse, 'NCC': ncc}
DSRNet/util/net_utils.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # returns the total number of parameters in the model that require gradients.
2
+ def count_parameters(model):
3
+ return sum(p.numel() for p in model.parameters() if p.requires_grad)
4
+ # model.parameters(): This method returns an iterator over all the parameters (weights and biases) in the model.
5
+ #p.numel(): This method returns the number of elements in the parameter tensor p.
6
+
7
+ def count_conv_layers(model): #counts total no. conv layers in a model
8
+ cnt = 0
9
+ for mo in model.modules(): #model.modules(): This method returns an iterator over all submodules (layers) of the model, including the model itself.
10
+ if type(mo).__name__ == 'Conv2d': #type(mo).__name__: This retrieves the string name of the class to which the submodule mo belongs.
11
+ cnt += 1
12
+ #model, total number of conv layers and total paramters with gradient
13
+ print(type(model).__name__, cnt, count_parameters(model))
DSRNet/util/schedulers.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from torch.optim.lr_scheduler import _LRScheduler
2
+ #Learning rate scheduler
3
+ from torch.optim.lr_scheduler import ReduceLROnPlateau
4
+ #Reduce learning rate when a metric has stopped improving. Models often benefit from reducing the learning rate by a factor of 2-10 once learning stagnates
5
+
6
+ class GradualWarmupScheduler(_LRScheduler):
7
+ #Starts with a small learning rate and gradually increases it over a few initial epochs or iterations. This is particularly useful in preventing the model from diverging in the initial phase of training and is often used in training deep networks from scratch.
8
+
9
+ #Pros: Prevents early divergence; stabilizes training.
10
+ #Cons: Requires tuning of warm-up duration and rate limits.
11
+ #Use Case: Crucial for training deep networks from scratch.
12
+ """(mingcv) Gradually warm-up(increasing) learning rate in optimizer.
13
+ Proposed in 'Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour'.
14
+ Args:
15
+ optimizer (Optimizer): Wrapped optimizer.
16
+ multiplier: target learning rate = base lr * multiplier if multiplier > 1.0. if multiplier = 1.0, lr starts from 0 and ends up with the base_lr.
17
+ total_epoch: target learning rate is reached at total_epoch, gradually
18
+ after_scheduler: after target_epoch, use this scheduler(eg. ReduceLROnPlateau)
19
+ """
20
+ #total_epoch (number of epochs to reach the target learning rate
21
+ #after scheduler: scheduler after warmup
22
+
23
+ def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None):
24
+ self.multiplier = multiplier
25
+ if self.multiplier < 1.:
26
+ raise ValueError('multiplier should be greater thant or equal to 1.')
27
+ self.total_epoch = total_epoch
28
+ self.after_scheduler = after_scheduler
29
+ self.finished = False #initializes a flag to indicate if warm-up is finished.
30
+ super(GradualWarmupScheduler, self).__init__(optimizer)
31
+
32
+ def get_lr(self): #get the current learning rates.
33
+ if self.last_epoch > self.total_epoch: #warm is ovcer , target lr reached
34
+ if self.after_scheduler: #plateuLR used
35
+ if not self.finished: #sets the base learning rates for the after_scheduler to be multiplier times the base learning rates and returns rlearningrate of after scheduler.
36
+ self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs] #self.base_lrs intialized from _LRScheduler
37
+ self.finished = True #warm up done
38
+ return self.after_scheduler.get_lr()
39
+ return [base_lr * self.multiplier for base_lr in self.base_lrs]
40
+
41
+ if self.multiplier == 1.0: #increase lienarly and propertionally to number of epochs run
42
+ return [base_lr * (float(self.last_epoch) / self.total_epoch) for base_lr in self.base_lrs]
43
+ else: #increaase lr non linearly
44
+ return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in
45
+ self.base_lrs]
46
+ "??? why this formula for non linear"
47
+
48
+ def step_ReduceLROnPlateau(self, metrics, epoch=None):
49
+ #metrics: A value that will be monitored (e.g., validation loss) to decide if the learning rate needs adjustment.
50
+ if epoch is None: #not resuming from a specific epoch
51
+ epoch = self.last_epoch + 1
52
+ self.last_epoch = epoch if epoch != 0 else 1 # (mingcv) ReduceLROnPlateau is called at the end of epoch, whereas others are called at beginning
53
+ if self.last_epoch <= self.total_epoch:
54
+ warmup_lr = [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in
55
+ self.base_lrs]
56
+ for param_group, lr in zip(self.optimizer.param_groups, warmup_lr):
57
+ param_group['lr'] = lr
58
+ else:
59
+ if epoch is None:
60
+ self.after_scheduler.step(metrics, None)
61
+ else:
62
+ self.after_scheduler.step(metrics, epoch - self.total_epoch)
63
+
64
+ def step(self, epoch=None, metrics=None):
65
+ if type(self.after_scheduler) != ReduceLROnPlateau:
66
+ if self.finished and self.after_scheduler:
67
+ if epoch is None:
68
+ self.after_scheduler.step(None)
69
+ else:
70
+ self.after_scheduler.step(epoch - self.total_epoch)
71
+ else:
72
+ return super(GradualWarmupScheduler, self).step(epoch)
73
+ else:
74
+ self.step_ReduceLROnPlateau(metrics, epoch)
DSRNet/util/util.py ADDED
@@ -0,0 +1,435 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import print_function #if running Python2 makes it work like python3
2
+
3
+ import math
4
+ import os
5
+ import sys
6
+ import time
7
+
8
+ import numpy as np
9
+ import torch
10
+ import torch.nn as nn
11
+ import yaml #Imports PyYAML for parsing YAML files.
12
+ from PIL import Image #Imports the Python Imaging Library for image processing.
13
+ from skimage.metrics import peak_signal_noise_ratio as compare_psnr #Imports the PSNR metric from skimage to quantify reconstruction quality for images and video subject to lossy compression. Higher is better
14
+ from skimage.metrics import structural_similarity #Imports the SSIM metric from skimage.A full-reference image quality evaluation index that measures image similarity from three aspects: brightness, contrast, and structure.
15
+
16
+
17
+ def get_config(config):#apth to configuration file
18
+ with open(config, 'r') as stream: #opens the file specified by the config variable in read mode and assigned to variable stream
19
+ return yaml.load(stream) #call loads and parses the YAML content from the file object stream and returns the parsed content.
20
+
21
+
22
+ # Converts a Tensor into a Numpy array then desired imtype
23
+ # |imtype|: the desired type of the converted numpy array
24
+ def tensor2im(image_tensor, imtype=np.uint8):
25
+ image_numpy = image_tensor[0].cpu().float().numpy() #image_tensor[0]: first image in batch in tensor
26
+ #.cpu() moves tensor to CPU if in GPU
27
+ #.float() converts to float
28
+ #.numpy() converts to numpy
29
+ if image_numpy.shape[0] == 1: #.shape[0] access channels ,1 = grayscale, 3=RGB
30
+ image_numpy = np.tile(image_numpy, (3, 1, 1))
31
+ #np.tile(A,reps):Construct an array by repeating A the number of times given by reps.
32
+ #repeating graysvale image here in 3 channels to get RGB image
33
+ image_numpy = (np.transpose(image_numpy, (1, 2, 0)) + 1) / 2.0 * 255.0
34
+ #np.transpose(image_numpy, (1, 2, 0)):The image is transposed from the shape (C, H, W) to (H, W, C).
35
+ # + 1: shift the pixel intensity range from [-1, 1] to [0, 2].
36
+ # / 2.0: This scales the values to the range [0, 1].
37
+ # * 255.0: This scales the values to the range [0, 255].
38
+ image_numpy = image_numpy.astype(imtype) #numpy array recast to desired imtype
39
+ if image_numpy.shape[-1] == 6: #after transpose -1 is no.channels and if this is 6
40
+ image_numpy = np.concatenate([image_numpy[:, :, :3], image_numpy[:, :, 3:]], axis=1)
41
+ '''???how does this help '''
42
+ #recombines along width 2 grps of 0-2 and 3-5
43
+
44
+ if image_numpy.shape[-1] == 7:#if transposed image has 7 channels
45
+ edge_map = np.tile(image_numpy[:, :, 6:7], (1, 1, 3))
46
+ '''???Tiles (repeats) the seventh channel to create a 3-channel edge_map along the channel axis.'''
47
+ image_numpy = np.concatenate([image_numpy[:, :, :3], image_numpy[:, :, 3:6], edge_map], axis=1)
48
+ '''???how does this help '''
49
+ #recombines along width 2 grps of 0-2 and 3-6
50
+ return image_numpy
51
+
52
+
53
+ def tensor2numpy(image_tensor):
54
+ image_numpy = torch.squeeze(image_tensor).cpu().float().numpy() #.squeeze(): removes all dimensions of size 1 from the shape of image_tensor
55
+ #PyTorch tensors can have singleton dimensions, which are dimensions with size 1. These dimensions might not carry meaningful information but are retained due to tensor operations.
56
+ #proabably rmeove number of batchs since 1 batch in 1 img (batch,channels,height,width)
57
+ image_numpy = (np.transpose(image_numpy, (1, 2, 0)) + 1) / 2.0 * 255.0
58
+ #np.transpose(image_numpy, (1, 2, 0)):The image is transposed from the shape (C, H, W) to (H, W, C).
59
+ # + 1: shift the pixel intensity range from [-1, 1] to [0, 2].
60
+ # / 2.0: This scales the values to the range [0, 1].
61
+ # * 255.0: This scales the values to the range [0, 255].
62
+ image_numpy = image_numpy.astype(np.float32)
63
+ #numpy array recast to float
64
+ return image_numpy
65
+
66
+
67
+ # Get model list for resume
68
+ #retrieve a specific model checkpoint file based on dirname, epoch,key
69
+ def get_model_list(dirname, key, epoch=None):
70
+ #key: main file_name
71
+ #epoch to indicate at which epoch the model was saved.Choosing epoch allows you to continue training from a specific point or to evaluate the model's performance at different stages of training
72
+ if epoch is None:
73
+ return os.path.join(dirname, key + '_latest.pt') # return latest checkpoint by default
74
+ if os.path.exists(dirname) is False: #if directory doesnt exist no checkpoint files can be retrieved.
75
+ return None
76
+
77
+ print(dirname, key)
78
+
79
+ #list of generated checkpoints after specific epohs during traininf
80
+ gen_models = [os.path.join(dirname, f) for f in os.listdir(dirname) if
81
+ os.path.isfile(os.path.join(dirname, f)) and ".pt" in f and 'latest' not in f]
82
+ #isfile():regular file,not a directory or a special file
83
+ #.pt type file with epoch number specifid, latest means epoch=None
84
+ epoch_index = [int(os.path.basename(model_name).split('_')[-2]) for model_name in gen_models if 'latest' not in model_name]
85
+ #os.path.basename(model_name): extracts the filename from the full path.
86
+ #split('_')[-2]: ['dsrnet', 's', 'epoch14.pt']-> s (second last)
87
+ '''??? why not -1 -> to get 14 chatgpt says int(s) will raise VauleError'''
88
+ print('[i] available epoch list: %s' % epoch_index, gen_models)
89
+ i = epoch_index.index(int(epoch))
90
+
91
+ return gen_models[i]
92
+
93
+ # to preprocess a batch of images for compatibility with models pretrained on the ImageNet dataset
94
+ def vgg_preprocess(batch):
95
+ # normalize using imagenet mean and std
96
+ #ImageNet normalization helps preprocess images to have zero mean and unit variance across each channel (RGB).
97
+ mean = batch.new(batch.size()) # Pytorch function creates new tensors (mean and std) with the same shape as the input batch.
98
+ std = batch.new(batch.size())
99
+ mean[:, 0, :, :] = 0.485 #assigned to all elements in the first channel of every image in the batch.
100
+ mean[:, 1, :, :] = 0.456 #2nd channel
101
+ mean[:, 2, :, :] = 0.406 #3rd channel
102
+ std[:, 0, :, :] = 0.229 #1st channel
103
+ std[:, 1, :, :] = 0.224 #2nd channel
104
+ std[:, 2, :, :] = 0.225 #3rd channel
105
+ batch = (batch + 1) / 2 #pixel values shifted [-1,1]->[0,2]->[0,1]
106
+ batch -= mean
107
+ batch = batch / std
108
+ # each pixel value in batch will have zero mean and unit variance
109
+ return batch
110
+
111
+
112
+
113
+ #print diagnostic information about the gradients of a neural network
114
+ #useful for inspecting the average magnitude of gradients during training or optimization of a neural network.
115
+ # It helps in diagnosing potential issues like vanishing or exploding gradients,
116
+ def diagnose_network(net, name='network'): #name: Optional name for the network (default is 'network'), used in printing diagnostic information.
117
+ mean = 0.0 #average mean absolute gradient across all parameters that have gradients.
118
+ count = 0 #no. of paramters with gradients
119
+ for param in net.parameters():
120
+ if param.grad is not None: #trainable Parameters without gradients won't contribute to mean.
121
+ #e.g frozen layers, non trainable paramters,bias,batch normalization parameters
122
+ mean += torch.mean(torch.abs(param.grad.data)) #mean absolute value of the gradient of the current parameter param
123
+ count += 1
124
+ if count > 0:
125
+ mean = mean / count
126
+ print(name)
127
+ print(mean)
128
+
129
+
130
+ def save_image(image_numpy, image_path): #create a PIL Image object from the numpy array and then saves it using the save() method of the PIL Image object.
131
+ image_pil = Image.fromarray(image_numpy)
132
+ image_pil.save(image_path)
133
+
134
+ #print shape and statistics of a numpy array
135
+ def print_numpy(x, val=True, shp=False): #val is statistics shp is shape
136
+ x = x.astype(np.float64)
137
+ if shp:
138
+ print('shape,', x.shape)
139
+ if val:
140
+ x = x.flatten() #converts nD to 1D
141
+ print('mean = %3.3f, min = %3.3f, max = %3.3f, median = %3.3f, std=%3.3f' % (
142
+ np.mean(x), np.min(x), np.max(x), np.median(x), np.std(x)))
143
+
144
+
145
+ def mkdirs(paths):
146
+ if isinstance(paths, list) and not isinstance(paths, str):
147
+ #if list makes path for each string
148
+ for path in paths:
149
+ mkdir(path)
150
+ else: #if string makes path
151
+ mkdir(paths)
152
+
153
+
154
+ def mkdir(path):
155
+ if not os.path.exists(path):
156
+ os.makedirs(path)
157
+
158
+
159
+ def set_opt_param(optimizer, key, value):
160
+ for group in optimizer.param_groups:
161
+ group[key] = value
162
+ '???'
163
+
164
+
165
+ def vis(x):#display an image in image viewer of system regardless if tensor or numpy array
166
+ if isinstance(x, torch.Tensor):
167
+ Image.fromarray(tensor2im(x)).show()
168
+ elif isinstance(x, np.ndarray):
169
+ Image.fromarray(x.astype(np.uint8)).show()
170
+ else:
171
+ raise NotImplementedError('vis for type [%s] is not implemented', type(x))
172
+
173
+
174
+ """tensorboard"""
175
+ from tensorboardX import SummaryWriter #for data logging
176
+ from datetime import datetime
177
+
178
+
179
+ def get_summary_writer(log_dir): #log_dir:directy to store logs
180
+ if not os.path.exists(log_dir):
181
+ os.mkdir(log_dir)
182
+ log_dir = os.path.join(log_dir, datetime.now().strftime('%b%d_%H-%M-%S') + '_' + socket.gethostname())
183
+ if not os.path.exists(log_dir):
184
+ os.mkdir(log_dir)
185
+ writer = SummaryWriter(log_dir) #each time you run your experiments, logs are saved in a new directory with current time, date and machine hostname
186
+ return writer
187
+
188
+
189
+
190
+ # keeps track of running avergae of metrics
191
+ class AverageMeters(object): #all classes inherit from object
192
+ def __init__(self, dic=None, total_num=None):
193
+ #dic: dictionary of metrics
194
+ #total_num:track of the counts of each metric)
195
+ self.dic = dic or {}
196
+ #(mingcv) self.total_num = total_num
197
+ self.total_num = total_num or {}
198
+
199
+
200
+ def update(self, new_dic): # method appends self.dic and updates total_num with new values.
201
+ for key in new_dic:
202
+ if not key in self.dic:
203
+ self.dic[key] = new_dic[key]
204
+ self.total_num[key] = 1
205
+ else:
206
+ self.dic[key] += new_dic[key]
207
+ self.total_num[key] += 1
208
+ # (mingcv)self.total_num += 1
209
+
210
+ def __getitem__(self, key): #overriding [], returns average value for a metric
211
+ return self.dic[key] / self.total_num[key]
212
+
213
+ def __str__(self): #overrider str() and print() to show key and values formatted
214
+ keys = sorted(self.keys())
215
+ res = ''
216
+ for key in keys:
217
+ res += (key + ': %.4f' % self[key] + ' | ')
218
+ return res
219
+
220
+ def keys(self): #returns all the metric names
221
+ return self.dic.keys()
222
+
223
+ '''???why _loss'''
224
+ def write_loss(writer, prefix, avg_meters, iteration):
225
+ #writer is the writer object for logging,SummaryWriter instance
226
+ #prefix is keyword added before each metric e.g. "train" it logs "accuracy": 85.4 as "train/accuracy"
227
+ #avg_meters: A dictionary argument where keys are metric names (like ‘loss’, ‘accuracy’) and values are their corresponding values at the current iteration (e.g., 0.23, 85.4).
228
+ #iteration: An integer argument representing the current iteration or step number in the training or evaluation process.
229
+ for key in avg_meters.keys():
230
+ meter = avg_meters[key] # for each key, we get the corresponding value from the avg_meters dictionary and assign it to the variable meter.
231
+ writer.add_scalar(os.path.join(prefix, key), meter, iteration)
232
+ # writer.add_scalar(): This method call logs the scalar value (metric) to the writer
233
+ '''
234
+ avg_meters = {
235
+ “loss”: 0.23,
236
+ “accuracy”: 85.4,
237
+ “precision”: 88.1
238
+ }
239
+ write_loss(writer, “train”, avg_meters, 10)
240
+
241
+ Logging: train/loss = 0.23 at step 10
242
+ Logging: train/accuracy = 85.4 at step 10
243
+ Logging: train/precision = 88.1 at step 10
244
+
245
+ '''
246
+ """(Mingcv)progress bar"""
247
+ import socket
248
+
249
+ # _, term_width = os.popen('stty size', 'r').read().split()
250
+ term_width = 136 #hardcoded terminal width to 136 characters instead of dymeically reading using stty siz
251
+
252
+ TOTAL_BAR_LENGTH = 65. #length of the progress bar.
253
+ last_time = time.time() #when progress is finished
254
+ begin_time = last_time # used to calculate the elapsed time for each step and the total process.
255
+
256
+
257
+ def progress_bar(current, total, msg=None):
258
+ #current: The current progress count.
259
+ #total: The total count that represents 100% progress.
260
+ #msg: An optional message to display alongside the progress bar.
261
+ global last_time, begin_time # for persistence in step_time and total_time
262
+ if current == 0:
263
+ begin_time = time.time() # Reset for new bar.
264
+
265
+ cur_len = int(TOTAL_BAR_LENGTH * current / total)
266
+ rest_len = int(TOTAL_BAR_LENGTH - cur_len) - 1
267
+
268
+ sys.stdout.write(' [')
269
+ for i in range(cur_len):
270
+ sys.stdout.write('=')
271
+ sys.stdout.write('>')
272
+ for i in range(rest_len):
273
+ sys.stdout.write('.')
274
+ sys.stdout.write(']')
275
+
276
+ cur_time = time.time()
277
+ step_time = cur_time - last_time
278
+ last_time = cur_time
279
+ tot_time = cur_time - begin_time
280
+
281
+ L = []
282
+ L.append(' Step: %s' % format_time(step_time))
283
+ L.append(' | Tot: %s' % format_time(tot_time))
284
+ if msg:
285
+ L.append(' | ' + msg)
286
+
287
+ msg = ''.join(L)
288
+ sys.stdout.write(msg)
289
+ for i in range(term_width - int(TOTAL_BAR_LENGTH) - len(msg) - 3):
290
+ sys.stdout.write(' ')
291
+
292
+ #(mingcv) Go back to the center of the bar.
293
+ for i in range(term_width - int(TOTAL_BAR_LENGTH / 2) + 2):
294
+ sys.stdout.write('\b')
295
+ sys.stdout.write(' %d/%d ' % (current + 1, total))
296
+
297
+ if current < total - 1:
298
+ sys.stdout.write('\r')
299
+ else:
300
+ sys.stdout.write('\n')
301
+ sys.stdout.flush()
302
+
303
+
304
+ def format_time(seconds): #convert seconds to days,hourse,mintures,seconds,milliseconds
305
+ days = int(seconds / 3600 / 24)
306
+ seconds = seconds - days * 3600 * 24
307
+ hours = int(seconds / 3600)
308
+ seconds = seconds - hours * 3600
309
+ minutes = int(seconds / 60)
310
+ seconds = seconds - minutes * 60
311
+ secondsf = int(seconds)
312
+ seconds = seconds - secondsf
313
+ millis = int(seconds * 1000)
314
+
315
+ f = ''
316
+ i = 1
317
+ if days > 0:
318
+ f += str(days) + 'D'
319
+ i += 1
320
+ if hours > 0 and i <= 2:
321
+ f += str(hours) + 'h'
322
+ i += 1
323
+ if minutes > 0 and i <= 2:
324
+ f += str(minutes) + 'm'
325
+ i += 1
326
+ if secondsf > 0 and i <= 2:
327
+ f += str(secondsf) + 's'
328
+ i += 1
329
+ if millis > 0 and i <= 2:
330
+ f += str(millis) + 'ms'
331
+ i += 1
332
+ if f == '':
333
+ f = '0ms'
334
+ return f
335
+
336
+
337
+ def parse_args(args): #converts numeric args seperated by commas into a [] of ints
338
+ str_args = args.split(',') #str1,atr2,str3 becomes [a,b,c]
339
+ parsed_args = []
340
+ for str_arg in str_args:
341
+ arg = int(str_arg)
342
+ if arg >= 0:
343
+ parsed_args.append(arg)
344
+ return parsed_args
345
+
346
+ #In order to overcome vanishing/explosding gradient, Xavier initialization was introduced. It tries to keep variance of all the layers equal but assumes linear actiivation
347
+ # Kaiming He initialization, which takes activation function into account.
348
+
349
+ def weights_init_kaiming(m): #layer or module in NN
350
+ classname = m.__class__.__name__ # gets the class name of the layer m using its __class__ attribute and __name__ property.
351
+ if classname.find('Conv') != -1: #returns -1 if the substring is not found, So here if found
352
+ nn.init.kaiming_normal(m.weight.data, a=0, mode='fan_in')
353
+
354
+ #nn.init.kaiming_normal:Fill the input Tensor(m.weight.data) with values using a Kaiming normal distribution.The method is described in Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification - He, K. et al. (2015). The resulting tensor will have values sampled from N(0,std^2) where std=gain/sqrt(fan_mode)
355
+ #a (float) – the negative slope of the rectifier used after this layer only used with Leaky RelU (0-not used)
356
+ #mode: fan_in(default) or fan_out.Choosing 'fan_in' preserves the magnitude of the variance of the weights in the forward pass.
357
+ #here intialising for relu,preserving variance of weights in forward pass
358
+ elif classname.find('Linear') != -1:
359
+ nn.init.kaiming_normal(m.weight.data, a=0, mode='fan_in')
360
+ # here intialising for relu,preserving variance of weights in forward pass
361
+ elif classname.find('BatchNorm') != -1:
362
+ # nn.init.uniform(m.weight.data, 1.0, 0.02)
363
+ m.weight.data.normal_(mean=0, std=math.sqrt(2. / 9. / 64.)).clamp_(-0.025, 0.025)
364
+ nn.init.constant(m.bias.data, 0.0)
365
+ "??? why std=math.sqrt(2. / 9. / 64.)"
366
+ # .clamp_(-0.025, 0.025): Clamps (limits) the values in the tensor to be between -0.025 and 0.025.
367
+ #bias of the batch normalization layer intiliazed to a constant value of 0.0.
368
+
369
+
370
+ def batch_PSNR(img, imclean, data_range): #calculates avg PSNR for 1 batch of images
371
+ #img batch of images to be evaluated with noise
372
+ #imclean: batch of ground truth images no noise
373
+ Img = img.data.cpu().numpy().astype(np.float32) #converts img to float numpy array in cpu
374
+ #- img.data: Detaches the data from the computation graph.
375
+ #cpu(): Transfers the tensor from GPU to CPU if it’s on GPU.
376
+ #numpy(): Converts the tensor to a NumPy array.
377
+ #astype(np.float32): Converts the array type to float32.
378
+ Iclean = imclean.data.cpu().numpy().astype(np.float32) #same for ground truth
379
+ PSNR = 0
380
+ for i in range(Img.shape[0]): #for all batches add the skmetric.psnr()
381
+ PSNR += compare_psnr(Iclean[i, :, :, :], Img[i, :, :, :], data_range=data_range)
382
+ #data range: maximum - inimum possible values
383
+ return PSNR / Img.shape[0] #average psnr per batch
384
+
385
+ #calculates avg SSIM for 1 batch of images
386
+ def batch_SSIM(img, imclean):
387
+ Img = img.data.cpu().permute(0, 2, 3, 1).numpy().astype(np.float32)
388
+ #converts img to float numpy array in cpu
389
+ #img.data: Detaches the data from the computation graph.
390
+ #cpu(): Transfers the tensor from GPU to CPU if it’s on GPU.
391
+ #numpy(): Converts the tensor to a NumPy array.
392
+ #astype(np.float32): Converts the array type to float32.
393
+ Iclean = imclean.data.cpu().permute(0, 2, 3, 1).numpy().astype(np.float32)
394
+ #permute(0, 2, 3, 1): Changes the order of dimensions from (batch_size, channels, height, width) to (batch_size, height, width, channels).
395
+ SSIM = 0
396
+
397
+ for i in range(Img.shape[0]): #for all batches add the skmetric.ssim()
398
+ SSIM += structural_similarity(Iclean[i, :, :, :], Img[i, :, :, :], win_size=11,
399
+ multichannel=True, data_range=1)
400
+ #win_size:The side-length of the sliding window used in comparison. Must be an odd value
401
+ #multichannel deprecated ...equivalent to channel-axis
402
+ "??? multichannel=True equal to what number in channel-axis"
403
+ return SSIM / Img.shape[0] #average ssim for1 batch
404
+
405
+
406
+ def data_augmentation(image, mode): #augment given image based on mode passed by manipulating numpy array
407
+ out = np.transpose(image, (1, 2, 0))
408
+ if mode == 0:
409
+ #(mingcv)original
410
+ out = out
411
+ elif mode == 1:
412
+ #(mingcv)flip up and down
413
+ out = np.flipud(out)
414
+ elif mode == 2:
415
+ #(mingcv)rotate counterwise 90 degree
416
+ out = np.rot90(out)
417
+ elif mode == 3:
418
+ #(mingcv)rotate 90 degree and flip up and down
419
+ out = np.rot90(out)
420
+ out = np.flipud(out)
421
+ elif mode == 4:
422
+ #(mingcv)rotate 180 degree
423
+ out = np.rot90(out, k=2)
424
+ elif mode == 5:
425
+ #(mingcv)rotate 180 degree and flip
426
+ out = np.rot90(out, k=2)
427
+ out = np.flipud(out)
428
+ elif mode == 6:
429
+ #(mingcv)rotate 270 degree
430
+ out = np.rot90(out, k=3)
431
+ elif mode == 7:
432
+ #(mingcv)rotate 270 degree and flip
433
+ out = np.rot90(out, k=3)
434
+ out = np.flipud(out)
435
+ return np.transpose(out, (2, 0, 1))
DSRNet/util/visualizer.py ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import os
3
+ import ntpath #for handling Windows paths.
4
+ import time
5
+ from . import util # . = same package
6
+ from . import html
7
+ import visdom # for visualizing data.
8
+
9
+
10
+ class Visualizer():
11
+ def __init__(self, opt):
12
+ self.display_id = opt.display_id #In Visdom, you can create multiple windows to display various types of information, such as images, graphs, or tables, during the training and testing of machine learning models. The display_id specifies which window (by ID) the visualizations should be displayed in.
13
+ self.use_html = opt.isTrain and not opt.no_html
14
+ #opt.isTrain==True means using train_options.py subclass
15
+ #HTML visualization in the context of deep learning involves generating web-based visual representations of the training process, model performance, and data. This can be particularly useful for monitoring, debugging, and sharing results.
16
+ self.win_size = opt.display_winsize
17
+ self.name = opt.name
18
+ self.opt = opt
19
+ self.saved = False
20
+ if self.display_id > 0: #if diplay_id==0, disable Visdom
21
+ self.vis = visdom.Visdom(port=opt.display_port, ipv6=False)
22
+
23
+ if self.use_html: #logging training loss
24
+ self.web_dir = os.path.join(opt.checkpoints_dir, opt.name, 'web')
25
+ self.img_dir = os.path.join(self.web_dir, 'images')
26
+ print('create web directory %s...' % self.web_dir)
27
+ util.mkdirs([self.web_dir, self.img_dir])
28
+ self.log_name = os.path.join(opt.checkpoints_dir, opt.name, 'loss_log.txt')
29
+ with open(self.log_name, "a") as log_file:
30
+ now = time.strftime("%c")
31
+ log_file.write('================ Training Loss (%s) ================\n' % now)
32
+
33
+ def reset(self):
34
+ self.saved = False
35
+
36
+ # (mingcv)|visuals|: dictionary of images to display or save
37
+ def display_current_results(self, visuals, epoch, save_result):
38
+ if self.display_id > 0: #(mingcv)show images in the browser
39
+ ncols = self.opt.display_single_pane_ncols
40
+ if ncols > 0:
41
+ h, w = next(iter(visuals.values())).shape[:2]
42
+ table_css = """<style>
43
+ table {border-collapse: separate; border-spacing:4px; white-space:nowrap; text-align:center}
44
+ table td {width: %dpx; height: %dpx; padding: 4px; outline: 4px solid black}
45
+ </style>""" % (w, h)
46
+ title = self.name
47
+ label_html = ''
48
+ label_html_row = ''
49
+ nrows = int(np.ceil(len(visuals.items()) / ncols))
50
+ images = []
51
+ idx = 0
52
+ for label, image_numpy in visuals.items():
53
+ label_html_row += '<td>%s</td>' % label
54
+ images.append(image_numpy.transpose([2, 0, 1]))
55
+ idx += 1
56
+ if idx % ncols == 0:
57
+ label_html += '<tr>%s</tr>' % label_html_row
58
+ label_html_row = ''
59
+ white_image = np.ones_like(image_numpy.transpose([2, 0, 1])) * 255
60
+ while idx % ncols != 0:
61
+ images.append(white_image)
62
+ label_html_row += '<td></td>'
63
+ idx += 1
64
+ if label_html_row != '':
65
+ label_html += '<tr>%s</tr>' % label_html_row
66
+ # pane col = image row
67
+ self.vis.images(images, nrow=ncols, env=self.opt.name, win=self.display_id + 1,
68
+ padding=2, opts=dict(title=title + ' images'))
69
+ label_html = '<table>%s</table>' % label_html
70
+ self.vis.text(table_css + label_html, env=self.opt.name, win=self.display_id + 2,
71
+ opts=dict(title=title + ' labels'))
72
+ else:
73
+ idx = 1
74
+ for label, image_numpy in visuals.items():
75
+ self.vis.image(image_numpy.transpose([2, 0, 1]), opts=dict(title=label),
76
+ env=self.opt.name,
77
+ win=self.display_id + idx)
78
+ idx += 1
79
+
80
+ if self.use_html and (save_result or not self.saved): # save images to a html file
81
+ self.saved = True
82
+ for label, image_numpy in visuals.items():
83
+ img_path = os.path.join(self.img_dir, 'epoch%.3d_%s.png' % (epoch, label))
84
+ util.save_image(image_numpy, img_path)
85
+ # update website
86
+ webpage = html.HTML(self.web_dir, 'Experiment name = %s' % self.name, reflesh=1)
87
+ for n in range(epoch, 0, -1):
88
+ webpage.add_header('epoch [%d]' % n)
89
+ ims = []
90
+ txts = []
91
+ links = []
92
+
93
+ for label, image_numpy in visuals.items():
94
+ img_path = 'epoch%.3d_%s.png' % (n, label)
95
+ ims.append(img_path)
96
+ txts.append(label)
97
+ links.append(img_path)
98
+ webpage.add_images(ims, txts, links, height=self.win_size)
99
+ webpage.save()
100
+
101
+ #(mingcv) errors: dictionary of error labels and values
102
+ def plot_current_errors(self, epoch, counter_ratio, opt, errors):
103
+ if not hasattr(self, 'plot_data'):
104
+ self.plot_data = {'X': [], 'Y': [], 'legend': list(errors.keys())}
105
+ self.plot_data['X'].append(epoch + counter_ratio)
106
+ self.plot_data['Y'].append([errors[k] for k in self.plot_data['legend']])
107
+ self.vis.line(
108
+ X=np.stack([np.array(self.plot_data['X'])] * len(self.plot_data['legend']), 1),
109
+ Y=np.array(self.plot_data['Y']),
110
+ opts={
111
+ 'title': self.name + ' loss over time',
112
+ 'legend': self.plot_data['legend'],
113
+ 'xlabel': 'epoch',
114
+ 'ylabel': 'loss'},
115
+ win=self.display_id,
116
+ env=self.opt.name)
117
+
118
+ # errors: same format as |errors| of plotCurrentErrors
119
+ def print_current_errors(self, epoch, i, errors, t):
120
+ message = '(epoch: %d, iters: %d, time: %.3f) ' % (epoch, i, t)
121
+ for k, v in errors.items():
122
+ message += '%s: %.3f ' % (k, v)
123
+
124
+ print(message)
125
+ with open(self.log_name, "a") as log_file:
126
+ log_file.write('%s\n' % message)
127
+
128
+ # (mingcv) save image to the disk
129
+ def save_images(self, webpage, visuals, image_path, aspect_ratio=1.0):
130
+ image_dir = webpage.get_image_dir()
131
+ short_path = ntpath.basename(image_path[0])
132
+ name = os.path.splitext(short_path)[0]
133
+
134
+ webpage.add_header(name)
135
+ ims = []
136
+ txts = []
137
+ links = []
138
+
139
+ for label, im in visuals.items():
140
+ image_name = '%s_%s.png' % (name, label)
141
+ save_path = os.path.join(image_dir, image_name)
142
+ h, w, _ = im.shape
143
+ if aspect_ratio > 1.0:
144
+ im = np.array(Image.fromarray(im).resize((h, int(w * aspect_ratio))))
145
+ if aspect_ratio < 1.0:
146
+ im = np.array(Image.fromarray(im).resize((h, int(h / aspect_ratio))))
147
+ util.save_image(im, save_path)
148
+
149
+ ims.append(image_name)
150
+ txts.append(label)
151
+ links.append(image_name)
152
+ webpage.add_images(ims, txts, links, height=self.win_size)
DSRNet/weights/dsrnet_s_epoch14.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d85041c53c025511c654b1b42da35a404d6c76cb1fb790a2adad4028875554c
3
+ size 39764466