Spaces:
Build error
Build error
commited from namirahrasul/DSRNet -main local
Browse files- DSRNet/LICENSE +201 -0
- DSRNet/README.md +55 -0
- DSRNet/data/VOC2012_224_train_png.txt +0 -0
- DSRNet/data/image_folder.py +51 -0
- DSRNet/data/real_test.txt +20 -0
- DSRNet/data/sirs_dataset.py +335 -0
- DSRNet/data/torchdata.py +74 -0
- DSRNet/data/transforms.py +375 -0
- DSRNet/engine.py +151 -0
- DSRNet/eval_sirs.py +67 -0
- DSRNet/eval_sirs_4000 +75 -0
- DSRNet/models/__init__.py +10 -0
- DSRNet/models/__pycache__/__init__.cpython-38.pyc +0 -0
- DSRNet/models/__pycache__/networks.cpython-38.pyc +0 -0
- DSRNet/models/__pycache__/vgg.cpython-38.pyc +0 -0
- DSRNet/models/arch/__init__.py +49 -0
- DSRNet/models/arch/__pycache__/__init__.cpython-38.pyc +0 -0
- DSRNet/models/arch/dsrnet.py +306 -0
- DSRNet/models/arch/lrm.py +185 -0
- DSRNet/models/arch/~/YTMT_US +1197 -0
- DSRNet/models/arch/~/YTMT_US.pdf +0 -0
- DSRNet/models/arch/~/tmp.pth +3 -0
- DSRNet/models/base_model.py +73 -0
- DSRNet/models/dsrnet_model_sirs.py +263 -0
- DSRNet/models/losses.py +136 -0
- DSRNet/models/networks.py +276 -0
- DSRNet/models/vgg.py +90 -0
- DSRNet/options/__init__.py +0 -0
- DSRNet/options/base_option.py +56 -0
- DSRNet/options/net_options/__init__.py +0 -0
- DSRNet/options/net_options/base_options.py +97 -0
- DSRNet/options/net_options/train_options.py +77 -0
- DSRNet/requirements.txt +11 -0
- DSRNet/scripts.sh +6 -0
- DSRNet/test_sirs.py +33 -0
- DSRNet/tools/__init__.py +0 -0
- DSRNet/tools/mutils.py +37 -0
- DSRNet/train_sirs.py +109 -0
- DSRNet/train_sirs_4000.py +118 -0
- DSRNet/util/__init__.py +17 -0
- DSRNet/util/html.py +66 -0
- DSRNet/util/index.py +67 -0
- DSRNet/util/net_utils.py +13 -0
- DSRNet/util/schedulers.py +74 -0
- DSRNet/util/util.py +435 -0
- DSRNet/util/visualizer.py +152 -0
- DSRNet/weights/dsrnet_s_epoch14.pt +3 -0
DSRNet/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License
|
| 2 |
+
Version 2.0, January 2004
|
| 3 |
+
http://www.apache.org/licenses/
|
| 4 |
+
|
| 5 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 6 |
+
|
| 7 |
+
1. Definitions.
|
| 8 |
+
|
| 9 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 10 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 11 |
+
|
| 12 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 13 |
+
the copyright owner that is granting the License.
|
| 14 |
+
|
| 15 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 16 |
+
other entities that control, are controlled by, or are under common
|
| 17 |
+
control with that entity. For the purposes of this definition,
|
| 18 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 19 |
+
direction or management of such entity, whether by contract or
|
| 20 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 21 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 22 |
+
|
| 23 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 24 |
+
exercising permissions granted by this License.
|
| 25 |
+
|
| 26 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 27 |
+
including but not limited to software source code, documentation
|
| 28 |
+
source, and configuration files.
|
| 29 |
+
|
| 30 |
+
"Object" form shall mean any form resulting from mechanical
|
| 31 |
+
transformation or translation of a Source form, including but
|
| 32 |
+
not limited to compiled object code, generated documentation,
|
| 33 |
+
and conversions to other media types.
|
| 34 |
+
|
| 35 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 36 |
+
Object form, made available under the License, as indicated by a
|
| 37 |
+
copyright notice that is included in or attached to the work
|
| 38 |
+
(an example is provided in the Appendix below).
|
| 39 |
+
|
| 40 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 41 |
+
form, that is based on (or derived from) the Work and for which the
|
| 42 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 43 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 44 |
+
of this License, Derivative Works shall not include works that remain
|
| 45 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 46 |
+
the Work and Derivative Works thereof.
|
| 47 |
+
|
| 48 |
+
"Contribution" shall mean any work of authorship, including
|
| 49 |
+
the original version of the Work and any modifications or additions
|
| 50 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 51 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 52 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 53 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 54 |
+
means any form of electronic, verbal, or written communication sent
|
| 55 |
+
to the Licensor or its representatives, including but not limited to
|
| 56 |
+
communication on electronic mailing lists, source code control systems,
|
| 57 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 58 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 59 |
+
excluding communication that is conspicuously marked or otherwise
|
| 60 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 61 |
+
|
| 62 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 63 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 64 |
+
subsequently incorporated within the Work.
|
| 65 |
+
|
| 66 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 67 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 68 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 69 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 70 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 71 |
+
Work and such Derivative Works in Source or Object form.
|
| 72 |
+
|
| 73 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 74 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 75 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 76 |
+
(except as stated in this section) patent license to make, have made,
|
| 77 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 78 |
+
where such license applies only to those patent claims licensable
|
| 79 |
+
by such Contributor that are necessarily infringed by their
|
| 80 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 81 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 82 |
+
institute patent litigation against any entity (including a
|
| 83 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 84 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 85 |
+
or contributory patent infringement, then any patent licenses
|
| 86 |
+
granted to You under this License for that Work shall terminate
|
| 87 |
+
as of the date such litigation is filed.
|
| 88 |
+
|
| 89 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 90 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 91 |
+
modifications, and in Source or Object form, provided that You
|
| 92 |
+
meet the following conditions:
|
| 93 |
+
|
| 94 |
+
(a) You must give any other recipients of the Work or
|
| 95 |
+
Derivative Works a copy of this License; and
|
| 96 |
+
|
| 97 |
+
(b) You must cause any modified files to carry prominent notices
|
| 98 |
+
stating that You changed the files; and
|
| 99 |
+
|
| 100 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 101 |
+
that You distribute, all copyright, patent, trademark, and
|
| 102 |
+
attribution notices from the Source form of the Work,
|
| 103 |
+
excluding those notices that do not pertain to any part of
|
| 104 |
+
the Derivative Works; and
|
| 105 |
+
|
| 106 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 107 |
+
distribution, then any Derivative Works that You distribute must
|
| 108 |
+
include a readable copy of the attribution notices contained
|
| 109 |
+
within such NOTICE file, excluding those notices that do not
|
| 110 |
+
pertain to any part of the Derivative Works, in at least one
|
| 111 |
+
of the following places: within a NOTICE text file distributed
|
| 112 |
+
as part of the Derivative Works; within the Source form or
|
| 113 |
+
documentation, if provided along with the Derivative Works; or,
|
| 114 |
+
within a display generated by the Derivative Works, if and
|
| 115 |
+
wherever such third-party notices normally appear. The contents
|
| 116 |
+
of the NOTICE file are for informational purposes only and
|
| 117 |
+
do not modify the License. You may add Your own attribution
|
| 118 |
+
notices within Derivative Works that You distribute, alongside
|
| 119 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 120 |
+
that such additional attribution notices cannot be construed
|
| 121 |
+
as modifying the License.
|
| 122 |
+
|
| 123 |
+
You may add Your own copyright statement to Your modifications and
|
| 124 |
+
may provide additional or different license terms and conditions
|
| 125 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 126 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 127 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 128 |
+
the conditions stated in this License.
|
| 129 |
+
|
| 130 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 131 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 132 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 133 |
+
this License, without any additional terms or conditions.
|
| 134 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 135 |
+
the terms of any separate license agreement you may have executed
|
| 136 |
+
with Licensor regarding such Contributions.
|
| 137 |
+
|
| 138 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 139 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 140 |
+
except as required for reasonable and customary use in describing the
|
| 141 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 142 |
+
|
| 143 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 144 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 145 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 146 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 147 |
+
implied, including, without limitation, any warranties or conditions
|
| 148 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 149 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 150 |
+
appropriateness of using or redistributing the Work and assume any
|
| 151 |
+
risks associated with Your exercise of permissions under this License.
|
| 152 |
+
|
| 153 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 154 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 155 |
+
unless required by applicable law (such as deliberate and grossly
|
| 156 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 157 |
+
liable to You for damages, including any direct, indirect, special,
|
| 158 |
+
incidental, or consequential damages of any character arising as a
|
| 159 |
+
result of this License or out of the use or inability to use the
|
| 160 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 161 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 162 |
+
other commercial damages or losses), even if such Contributor
|
| 163 |
+
has been advised of the possibility of such damages.
|
| 164 |
+
|
| 165 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 166 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 167 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 168 |
+
or other liability obligations and/or rights consistent with this
|
| 169 |
+
License. However, in accepting such obligations, You may act only
|
| 170 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 171 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 172 |
+
defend, and hold each Contributor harmless for any liability
|
| 173 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 174 |
+
of your accepting any such warranty or additional liability.
|
| 175 |
+
|
| 176 |
+
END OF TERMS AND CONDITIONS
|
| 177 |
+
|
| 178 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 179 |
+
|
| 180 |
+
To apply the Apache License to your work, attach the following
|
| 181 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 182 |
+
replaced with your own identifying information. (Don't include
|
| 183 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 184 |
+
comment syntax for the file format. We also recommend that a
|
| 185 |
+
file or class name and description of purpose be included on the
|
| 186 |
+
same "printed page" as the copyright notice for easier
|
| 187 |
+
identification within third-party archives.
|
| 188 |
+
|
| 189 |
+
Copyright [yyyy] [name of copyright owner]
|
| 190 |
+
|
| 191 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 192 |
+
you may not use this file except in compliance with the License.
|
| 193 |
+
You may obtain a copy of the License at
|
| 194 |
+
|
| 195 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 196 |
+
|
| 197 |
+
Unless required by applicable law or agreed to in writing, software
|
| 198 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 199 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 200 |
+
See the License for the specific language governing permissions and
|
| 201 |
+
limitations under the License.
|
DSRNet/README.md
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# DSRNet: Single Image Reflection Separation via Component Synergy (ICCV 2023)
|
| 2 |
+
|
| 3 |
+
> :book: [[ICCV](https://openaccess.thecvf.com/content/ICCV2023/papers/Hu_Single_Image_Reflection_Separation_via_Component_Synergy_ICCV_2023_paper.pdf)] [[Arxiv](https://arxiv.org/abs/2308.10027)] [[Supp.](https://github.com/mingcv/DSRNet/files/12387445/full_arxiv_version_supp.pdf)] <br>
|
| 4 |
+
> [Qiming Hu](https://scholar.google.com.hk/citations?user=4zasPbwAAAAJ), [Xiaojie Guo](https://sites.google.com/view/xjguo/homepage) <br>
|
| 5 |
+
> College of Intelligence and Computing, Tianjin University<br>
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
### Network Architecture
|
| 9 |
+

|
| 10 |
+
|
| 11 |
+
### Environment Preparation (Python 3.9)
|
| 12 |
+
```pip install -r requirements.txt```
|
| 13 |
+
### Data Preparation
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
#### Training dataset
|
| 17 |
+
* 7,643 images from the
|
| 18 |
+
[Pascal VOC dataset](http://host.robots.ox.ac.uk/pascal/VOC/), center-cropped as 224 x 224 slices to synthesize training pairs;
|
| 19 |
+
* 90 real-world training pairs provided by [Zhang *et al.*](https://github.com/ceciliavision/perceptual-reflection-removal);
|
| 20 |
+
* 200 real-world training pairs provided by [IBCLN](https://github.com/JHL-HUST/IBCLN) (In our training setting 2, † labeled in our paper).
|
| 21 |
+
|
| 22 |
+
#### Testing dataset
|
| 23 |
+
* 45 real-world testing images from [CEILNet dataset](https://github.com/fqnchina/CEILNet);
|
| 24 |
+
* 20 real testing pairs provided by [Zhang *et al.*](https://github.com/ceciliavision/perceptual-reflection-removal);
|
| 25 |
+
* 20 real testing pairs provided by [IBCLN](https://github.com/JHL-HUST/IBCLN);
|
| 26 |
+
* 454 real testing pairs from [SIR^2 dataset](https://sir2data.github.io/), containing three subsets (i.e., Objects (200), Postcard (199), Wild (55)).
|
| 27 |
+
|
| 28 |
+
Download all in one by [Google Drive](https://drive.google.com/file/d/1hFZItZAzAt-LnfNj-2phBRwqplDUasQy/view?usp=sharing) or [百度云](https://pan.baidu.com/s/15zlk5o_-kx3ruKj4KfOvtA?pwd=1231).
|
| 29 |
+
### Usage
|
| 30 |
+
|
| 31 |
+
#### Training
|
| 32 |
+
Setting I (w/o Nature): ```python train_sirs.py --inet dsrnet_l --model dsrnet_model_sirs --dataset sirs_dataset --loss losses --name dsrnet_l --lambda_vgg 0.01 --lambda_rec 0.2 --if_align --seed 2018 --base_dir "[YOUR DATA DIR]"```
|
| 33 |
+
|
| 34 |
+
Setting II (w/ Nature): ```python train_sirs_4000.py --inet dsrnet_l --model dsrnet_model_sirs --dataset sirs_dataset --loss losses --name dsrnet_l_4000 --lambda_vgg 0.01 --lambda_rec 0.2 --if_align --seed 2018 --base_dir "[YOUR DATA DIR]"```
|
| 35 |
+
#### Evaluation
|
| 36 |
+
Setting I (w/o Nature): ```python eval_sirs.py --inet dsrnet_l --model dsrnet_model_sirs --dataset sirs_dataset --name dsrnet_l_test --if_align --resume --weight_path "./weights/dsrnet_l_epoch18.pt" --base_dir "[YOUR_DATA_DIR]"```
|
| 37 |
+
|
| 38 |
+
Setting II (w/ Nature): ```python eval_sirs_4000.py --inet dsrnet_l --model dsrnet_model_sirs --dataset sirs_dataset --name dsrnet_l_4000_test --if_align --resume --weight_path "./weights/dsrnet_l_4000_epoch33.pt" --base_dir "[YOUR_DATA_DIR]"```
|
| 39 |
+
|
| 40 |
+
More commands can be found in [scripts.sh](https://github.com/mingcv/DSRNet/blob/main/scripts.sh).
|
| 41 |
+
|
| 42 |
+
#### Trained weights
|
| 43 |
+
|
| 44 |
+
Download the trained weights by [Google Drive](https://drive.google.com/drive/folders/1AIS9-EgBN3_q-TCq7W0j5OeWMgLO_de0?usp=sharing) or [百度云](https://pan.baidu.com/s/17jW9oBAfIZ03FKa3jc-qig?pwd=1231) and drop them into the "weights" dir.
|
| 45 |
+
|
| 46 |
+

|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
#### Visual comparison on real20 and SIR^2
|
| 50 |
+

|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
#### Impressive Restoration Quality of Reflection Layers
|
| 54 |
+

|
| 55 |
+
|
DSRNet/data/VOC2012_224_train_png.txt
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
DSRNet/data/image_folder.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
###############################################################################
|
| 2 |
+
# Code from
|
| 3 |
+
# https://github.com/pytorch/vision/blob/master/torchvision/datasets/folder.py
|
| 4 |
+
# Modified the original code so that it also loads images from the current
|
| 5 |
+
# directory as well as the subdirectories
|
| 6 |
+
###############################################################################
|
| 7 |
+
|
| 8 |
+
import torch.utils.data as data
|
| 9 |
+
|
| 10 |
+
from PIL import Image
|
| 11 |
+
import os
|
| 12 |
+
import os.path
|
| 13 |
+
|
| 14 |
+
IMG_EXTENSIONS = [
|
| 15 |
+
'.jpg', '.JPG', '.jpeg', '.JPEG',
|
| 16 |
+
'.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP',
|
| 17 |
+
]
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def read_fns(filename):#returns all lines in a file as a list
|
| 21 |
+
with open(filename) as f:
|
| 22 |
+
fns = f.readlines()
|
| 23 |
+
fns = [fn.strip() for fn in fns] #This strips any leading and trailing whitespace characters (including newlines) from each line in fns and turns result into a list
|
| 24 |
+
return fns
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def is_image_file(filename):#checks if file is image based on extensions
|
| 28 |
+
return any(filename.endswith(extension) for extension in IMG_EXTENSIONS)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def make_dataset(dir, fns=None):#makes dataset from directory with files
|
| 32 |
+
images = []
|
| 33 |
+
assert os.path.isdir(dir), '%s is not a valid directory' % dir
|
| 34 |
+
|
| 35 |
+
if fns is None:#if no filename list is given, create list of images
|
| 36 |
+
for root, _, fnames in sorted(os.walk(dir)): #walks through the directory tree starting from dir, returning the root directory path, directories, and file names. The results are sorted.
|
| 37 |
+
for fname in fnames:
|
| 38 |
+
if is_image_file(fname):
|
| 39 |
+
path = os.path.join(root, fname)
|
| 40 |
+
images.append(path)
|
| 41 |
+
else:#create list of images from list
|
| 42 |
+
for fname in fns:
|
| 43 |
+
if is_image_file(fname):
|
| 44 |
+
path = os.path.join(dir, fname)
|
| 45 |
+
images.append(path)
|
| 46 |
+
|
| 47 |
+
return images
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def default_loader(path):#convert image in path to rgb and open it
|
| 51 |
+
return Image.open(path).convert('RGB') #if grayscale or has alpha channel (rgba)
|
DSRNet/data/real_test.txt
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
3.jpg
|
| 2 |
+
4.jpg
|
| 3 |
+
9.jpg
|
| 4 |
+
12.jpg
|
| 5 |
+
15.jpg
|
| 6 |
+
22.jpg
|
| 7 |
+
23.jpg
|
| 8 |
+
25.jpg
|
| 9 |
+
29.jpg
|
| 10 |
+
39.jpg
|
| 11 |
+
46.jpg
|
| 12 |
+
47.jpg
|
| 13 |
+
58.jpg
|
| 14 |
+
86.jpg
|
| 15 |
+
87.jpg
|
| 16 |
+
89.jpg
|
| 17 |
+
93.jpg
|
| 18 |
+
103.jpg
|
| 19 |
+
107.jpg
|
| 20 |
+
110.jpg
|
DSRNet/data/sirs_dataset.py
ADDED
|
@@ -0,0 +1,335 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
import os.path
|
| 3 |
+
import os.path
|
| 4 |
+
import random
|
| 5 |
+
from os.path import join
|
| 6 |
+
|
| 7 |
+
import cv2
|
| 8 |
+
import numpy as np
|
| 9 |
+
import torch.utils.data
|
| 10 |
+
import torchvision.transforms.functional as TF #more flexible and fine-grained transformations compared to the standard transforms API.
|
| 11 |
+
from PIL import Image
|
| 12 |
+
from scipy.signal import convolve2d
|
| 13 |
+
|
| 14 |
+
from data.image_folder import make_dataset
|
| 15 |
+
from data.torchdata import Dataset as BaseDataset #overrides len(), + and []
|
| 16 |
+
from data.transforms import to_tensor
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def __scale_width(img, target_width):
|
| 20 |
+
ow, oh = img.size #img.size method from (PIL) /Pillow, returns a tuple (width,height) in px
|
| 21 |
+
#original width and original height
|
| 22 |
+
if (ow == target_width):
|
| 23 |
+
return img
|
| 24 |
+
w = target_width
|
| 25 |
+
h = int(target_width * oh / ow) #maintain aspect ratio by changing height
|
| 26 |
+
h = math.ceil(h / 2.) * 2 #(mingcv) round up to even
|
| 27 |
+
return img.resize((w, h), Image.BICUBIC) #resizes the image to the new dimensions (w, h) using bicubic interpolation. Bicubic interpolation is a method of resizing images that results in smoother and more visually appealing images compared to simpler methods like nearest neighbor or bilinear interpolation. From PIL/pillow
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def __scale_height(img, target_height):
|
| 31 |
+
ow, oh = img.size
|
| 32 |
+
if (oh == target_height):
|
| 33 |
+
return img
|
| 34 |
+
h = target_height
|
| 35 |
+
w = int(target_height * ow / oh)
|
| 36 |
+
w = math.ceil(w / 2.) * 2
|
| 37 |
+
return img.resize((w, h), Image.BICUBIC)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def paired_data_transforms(img_1, img_2, unaligned_transforms=False):#img1, img2:PIL Image objects
|
| 41 |
+
#Unaligned transformations refer to transformations applied to paired images where the transformations are intentionally not synchronized between the two images. This approach introduces a controlled misalignment, adding variability to the data and potentially increasing the robustness of a model trained on such data.
|
| 42 |
+
def get_params(img, output_size):#inner nested function for scope & encapsulation
|
| 43 |
+
#getting dimensions for cropping
|
| 44 |
+
w, h = img.size
|
| 45 |
+
th, tw = output_size
|
| 46 |
+
if w == tw and h == th:
|
| 47 |
+
return 0, 0, h, w# no cropping as u crop form top left to bottom right
|
| 48 |
+
|
| 49 |
+
i = random.randint(0, h - th)
|
| 50 |
+
j = random.randint(0, w - tw)
|
| 51 |
+
return i, j, th, tw #i,j = starting position of cropped region
|
| 52 |
+
|
| 53 |
+
target_size = int(random.randint(224, 448) / 2.) * 2
|
| 54 |
+
'''??? why 224-448'''
|
| 55 |
+
ow, oh = img_1.size
|
| 56 |
+
if ow >= oh:#scale the smaller dimension
|
| 57 |
+
img_1 = __scale_height(img_1, target_size)
|
| 58 |
+
img_2 = __scale_height(img_2, target_size)
|
| 59 |
+
else:
|
| 60 |
+
img_1 = __scale_width(img_1, target_size)
|
| 61 |
+
img_2 = __scale_width(img_2, target_size)
|
| 62 |
+
|
| 63 |
+
if random.random() < 0.5: #flip randomly pairs
|
| 64 |
+
img_1 = TF.hflip(img_1)
|
| 65 |
+
img_2 = TF.hflip(img_2)
|
| 66 |
+
|
| 67 |
+
if random.random() < 0.5: #rotate randomly pairs
|
| 68 |
+
angle = random.choice([90, 180, 270])
|
| 69 |
+
img_1 = TF.rotate(img_1, angle)
|
| 70 |
+
img_2 = TF.rotate(img_2, angle)
|
| 71 |
+
#unalined transoformation by cropping 2 images differently
|
| 72 |
+
i, j, h, w = get_params(img_1, (224, 224))
|
| 73 |
+
img_1 = TF.crop(img_1, i, j, h, w)
|
| 74 |
+
|
| 75 |
+
if unaligned_transforms:
|
| 76 |
+
#(mingcv) print('random shift')
|
| 77 |
+
i_shift = random.randint(-10, 10)
|
| 78 |
+
j_shift = random.randint(-10, 10)
|
| 79 |
+
i += i_shift
|
| 80 |
+
j += j_shift
|
| 81 |
+
|
| 82 |
+
img_2 = TF.crop(img_2, i, j, h, w)
|
| 83 |
+
|
| 84 |
+
return img_1, img_2
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
class ReflectionSynthesis(object):
|
| 88 |
+
def __init__(self):
|
| 89 |
+
#(mingcv) Kernel Size of the Gaussian Blurry
|
| 90 |
+
self.kernel_sizes = [5, 7, 9, 11]
|
| 91 |
+
self.kernel_probs = [0.1, 0.2, 0.3, 0.4]
|
| 92 |
+
|
| 93 |
+
#(mingcv) Sigma of the Gaussian Blurry
|
| 94 |
+
self.sigma_range = [2, 5]
|
| 95 |
+
self.alpha_range = [0.8, 1.0]
|
| 96 |
+
self.beta_range = [0.4, 1.0]
|
| 97 |
+
|
| 98 |
+
def __call__(self, T_, R_):
|
| 99 |
+
T_ = np.asarray(T_, np.float32) / 255.
|
| 100 |
+
R_ = np.asarray(R_, np.float32) / 255.
|
| 101 |
+
|
| 102 |
+
kernel_size = np.random.choice(self.kernel_sizes, p=self.kernel_probs)
|
| 103 |
+
sigma = np.random.uniform(self.sigma_range[0], self.sigma_range[1])
|
| 104 |
+
kernel = cv2.getGaussianKernel(kernel_size, sigma)
|
| 105 |
+
kernel2d = np.dot(kernel, kernel.T)
|
| 106 |
+
for i in range(3):
|
| 107 |
+
R_[..., i] = convolve2d(R_[..., i], kernel2d, mode='same')
|
| 108 |
+
|
| 109 |
+
a = np.random.uniform(self.alpha_range[0], self.alpha_range[1])
|
| 110 |
+
b = np.random.uniform(self.beta_range[0], self.beta_range[1])
|
| 111 |
+
T, R = a * T_, b * R_
|
| 112 |
+
|
| 113 |
+
if random.random() < 0.7:
|
| 114 |
+
I = T + R - T * R
|
| 115 |
+
else:
|
| 116 |
+
I = T + R
|
| 117 |
+
if np.max(I) > 1:
|
| 118 |
+
m = I[I > 1]
|
| 119 |
+
m = (np.mean(m) - 1) * 1.3
|
| 120 |
+
I = np.clip(T + np.clip(R - m, 0, 1), 0, 1)
|
| 121 |
+
|
| 122 |
+
return T_, R_, I
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
class DataLoader(torch.utils.data.DataLoader):
|
| 126 |
+
def __init__(self, dataset, batch_size, shuffle, *args, **kwargs):
|
| 127 |
+
super(DataLoader, self).__init__(dataset, batch_size, shuffle, *args, **kwargs)
|
| 128 |
+
self.shuffle = shuffle
|
| 129 |
+
|
| 130 |
+
def reset(self):
|
| 131 |
+
if self.shuffle:
|
| 132 |
+
print('Reset Dataset...')
|
| 133 |
+
self.dataset.reset()
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
class DSRDataset(BaseDataset):
|
| 137 |
+
def __init__(self, datadir, fns=None, size=None, enable_transforms=True):
|
| 138 |
+
super(DSRDataset, self).__init__()
|
| 139 |
+
self.size = size
|
| 140 |
+
self.datadir = datadir
|
| 141 |
+
self.enable_transforms = enable_transforms
|
| 142 |
+
sortkey = lambda key: os.path.split(key)[-1]
|
| 143 |
+
self.paths = sorted(make_dataset(datadir, fns), key=sortkey)
|
| 144 |
+
if size is not None:
|
| 145 |
+
self.paths = np.random.choice(self.paths, size)
|
| 146 |
+
|
| 147 |
+
self.syn_model = ReflectionSynthesis()
|
| 148 |
+
self.reset(shuffle=False)
|
| 149 |
+
|
| 150 |
+
def reset(self, shuffle=True):
|
| 151 |
+
if shuffle:
|
| 152 |
+
random.shuffle(self.paths)
|
| 153 |
+
num_paths = len(self.paths) // 2
|
| 154 |
+
self.B_paths = self.paths[0:num_paths]
|
| 155 |
+
self.R_paths = self.paths[num_paths:2 * num_paths]
|
| 156 |
+
|
| 157 |
+
def data_synthesis(self, t_img, r_img):
|
| 158 |
+
if self.enable_transforms:
|
| 159 |
+
t_img, r_img = paired_data_transforms(t_img, r_img)
|
| 160 |
+
|
| 161 |
+
t_img, r_img, m_img = self.syn_model(t_img, r_img)
|
| 162 |
+
|
| 163 |
+
B = TF.to_tensor(t_img)
|
| 164 |
+
R = TF.to_tensor(r_img)
|
| 165 |
+
M = TF.to_tensor(m_img)
|
| 166 |
+
|
| 167 |
+
return B, R, M
|
| 168 |
+
|
| 169 |
+
def __getitem__(self, index):
|
| 170 |
+
index_B = index % len(self.B_paths)
|
| 171 |
+
index_R = index % len(self.R_paths)
|
| 172 |
+
|
| 173 |
+
B_path = self.B_paths[index_B]
|
| 174 |
+
R_path = self.R_paths[index_R]
|
| 175 |
+
|
| 176 |
+
t_img = Image.open(B_path).convert('RGB')
|
| 177 |
+
r_img = Image.open(R_path).convert('RGB')
|
| 178 |
+
|
| 179 |
+
B, R, M = self.data_synthesis(t_img, r_img)
|
| 180 |
+
fn = os.path.basename(B_path)
|
| 181 |
+
return {'input': M, 'target_t': B, 'target_r': R, 'fn': fn, 'real': False}
|
| 182 |
+
|
| 183 |
+
def __len__(self):
|
| 184 |
+
if self.size is not None:
|
| 185 |
+
return min(max(len(self.B_paths), len(self.R_paths)), self.size)
|
| 186 |
+
else:
|
| 187 |
+
return max(len(self.B_paths), len(self.R_paths))
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
class DSRTestDataset(BaseDataset):
|
| 191 |
+
def __init__(self, datadir, fns=None, size=None, enable_transforms=False, unaligned_transforms=False,
|
| 192 |
+
round_factor=1, flag=None, if_align=False):
|
| 193 |
+
super(DSRTestDataset, self).__init__()
|
| 194 |
+
self.size = size
|
| 195 |
+
self.datadir = datadir
|
| 196 |
+
self.fns = fns or os.listdir(join(datadir, 'blended'))
|
| 197 |
+
self.enable_transforms = enable_transforms
|
| 198 |
+
self.unaligned_transforms = unaligned_transforms
|
| 199 |
+
self.round_factor = round_factor
|
| 200 |
+
self.flag = flag
|
| 201 |
+
self.if_align = if_align
|
| 202 |
+
|
| 203 |
+
if size is not None:
|
| 204 |
+
self.fns = self.fns[:size]
|
| 205 |
+
|
| 206 |
+
def align(self, x1, x2):
|
| 207 |
+
h, w = x1.height, x1.width
|
| 208 |
+
h, w = h // 32 * 32, w // 32 * 32
|
| 209 |
+
x1 = x1.resize((w, h))
|
| 210 |
+
x2 = x2.resize((w, h))
|
| 211 |
+
return x1, x2
|
| 212 |
+
|
| 213 |
+
def __getitem__(self, index):
|
| 214 |
+
fn = self.fns[index]
|
| 215 |
+
|
| 216 |
+
t_img = Image.open(join(self.datadir, 'transmission_layer', fn)).convert('RGB')
|
| 217 |
+
m_img = Image.open(join(self.datadir, 'blended', fn)).convert('RGB')
|
| 218 |
+
|
| 219 |
+
if self.if_align:
|
| 220 |
+
t_img, m_img = self.align(t_img, m_img)
|
| 221 |
+
|
| 222 |
+
if self.enable_transforms:
|
| 223 |
+
t_img, m_img = paired_data_transforms(t_img, m_img, self.unaligned_transforms)
|
| 224 |
+
|
| 225 |
+
B = TF.to_tensor(t_img)
|
| 226 |
+
M = TF.to_tensor(m_img)
|
| 227 |
+
|
| 228 |
+
dic = {'input': M, 'target_t': B, 'fn': fn, 'real': True, 'target_r': M - B}
|
| 229 |
+
if self.flag is not None:
|
| 230 |
+
dic.update(self.flag)
|
| 231 |
+
return dic
|
| 232 |
+
|
| 233 |
+
def __len__(self):
|
| 234 |
+
if self.size is not None:
|
| 235 |
+
return min(len(self.fns), self.size)
|
| 236 |
+
else:
|
| 237 |
+
return len(self.fns)
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
class SIRTestDataset(BaseDataset):
|
| 241 |
+
def __init__(self, datadir, fns=None, size=None, if_align=False):
|
| 242 |
+
super(SIRTestDataset, self).__init__()
|
| 243 |
+
self.size = size
|
| 244 |
+
self.datadir = datadir
|
| 245 |
+
self.fns = fns or os.listdir(join(datadir, 'blended'))
|
| 246 |
+
self.if_align = if_align
|
| 247 |
+
|
| 248 |
+
if size is not None:
|
| 249 |
+
self.fns = self.fns[:size]
|
| 250 |
+
|
| 251 |
+
def align(self, x1, x2, x3):
|
| 252 |
+
h, w = x1.height, x1.width
|
| 253 |
+
h, w = h // 32 * 32, w // 32 * 32
|
| 254 |
+
x1 = x1.resize((w, h))
|
| 255 |
+
x2 = x2.resize((w, h))
|
| 256 |
+
x3 = x3.resize((w, h))
|
| 257 |
+
return x1, x2, x3
|
| 258 |
+
|
| 259 |
+
def __getitem__(self, index):
|
| 260 |
+
fn = self.fns[index]
|
| 261 |
+
|
| 262 |
+
t_img = Image.open(join(self.datadir, 'transmission_layer', fn)).convert('RGB')
|
| 263 |
+
r_img = Image.open(join(self.datadir, 'reflection_layer', fn)).convert('RGB')
|
| 264 |
+
m_img = Image.open(join(self.datadir, 'blended', fn)).convert('RGB')
|
| 265 |
+
|
| 266 |
+
if self.if_align:
|
| 267 |
+
t_img, r_img, m_img = self.align(t_img, r_img, m_img)
|
| 268 |
+
|
| 269 |
+
B = TF.to_tensor(t_img)
|
| 270 |
+
R = TF.to_tensor(r_img)
|
| 271 |
+
M = TF.to_tensor(m_img)
|
| 272 |
+
|
| 273 |
+
dic = {'input': M, 'target_t': B, 'fn': fn, 'real': True, 'target_r': R, 'target_r_hat': M - B}
|
| 274 |
+
return dic
|
| 275 |
+
|
| 276 |
+
def __len__(self):
|
| 277 |
+
if self.size is not None:
|
| 278 |
+
return min(len(self.fns), self.size)
|
| 279 |
+
else:
|
| 280 |
+
return len(self.fns)
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
class RealDataset(BaseDataset):
|
| 284 |
+
def __init__(self, datadir, fns=None, size=None):
|
| 285 |
+
super(RealDataset, self).__init__()
|
| 286 |
+
self.size = size
|
| 287 |
+
self.datadir = datadir
|
| 288 |
+
self.fns = fns or os.listdir(join(datadir))
|
| 289 |
+
|
| 290 |
+
if size is not None:
|
| 291 |
+
self.fns = self.fns[:size]
|
| 292 |
+
|
| 293 |
+
def align(self, x):
|
| 294 |
+
h, w = x.height, x.width
|
| 295 |
+
h, w = h // 32 * 32, w // 32 * 32
|
| 296 |
+
x = x.resize((w, h))
|
| 297 |
+
return x
|
| 298 |
+
|
| 299 |
+
def __getitem__(self, index):
|
| 300 |
+
fn = self.fns[index]
|
| 301 |
+
B = -1
|
| 302 |
+
m_img = Image.open(join(self.datadir, fn)).convert('RGB')
|
| 303 |
+
M = to_tensor(self.align(m_img))
|
| 304 |
+
data = {'input': M, 'target_t': B, 'fn': fn}
|
| 305 |
+
return data
|
| 306 |
+
|
| 307 |
+
def __len__(self):
|
| 308 |
+
if self.size is not None:
|
| 309 |
+
return min(len(self.fns), self.size)
|
| 310 |
+
else:
|
| 311 |
+
return len(self.fns)
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
class FusionDataset(BaseDataset):
|
| 315 |
+
def __init__(self, datasets, fusion_ratios=None, size=None):
|
| 316 |
+
self.datasets = datasets
|
| 317 |
+
self.size = size or sum([len(dataset) for dataset in datasets])
|
| 318 |
+
self.fusion_ratios = fusion_ratios or [1. / len(datasets)] * len(datasets)
|
| 319 |
+
print('[i] using a fusion dataset: %d %s imgs fused with ratio %s' % (
|
| 320 |
+
self.size, [len(dataset) for dataset in datasets], self.fusion_ratios))
|
| 321 |
+
|
| 322 |
+
def reset(self):
|
| 323 |
+
for dataset in self.datasets:
|
| 324 |
+
dataset.reset()
|
| 325 |
+
|
| 326 |
+
def __getitem__(self, index):
|
| 327 |
+
residual = 1
|
| 328 |
+
for i, ratio in enumerate(self.fusion_ratios):
|
| 329 |
+
if random.random() < ratio / residual or i == len(self.fusion_ratios) - 1:
|
| 330 |
+
dataset = self.datasets[i]
|
| 331 |
+
return dataset[index % len(dataset)]
|
| 332 |
+
residual -= ratio
|
| 333 |
+
|
| 334 |
+
def __len__(self):
|
| 335 |
+
return self.size
|
DSRNet/data/torchdata.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import bisect #maintains a list in sorted order without having sort after each insertion using bisection algorithm
|
| 2 |
+
import warnings #issue warning message
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class Dataset(object): # all python classes inherit from object
|
| 6 |
+
"""An abstract class representing a Dataset.
|
| 7 |
+
|
| 8 |
+
All other datasets should subclass it. All subclasses should override
|
| 9 |
+
``__len__``, that provides the size of the dataset, and ``__getitem__``,
|
| 10 |
+
supporting integer indexing in range from 0 to len(self) exclusive.
|
| 11 |
+
"""
|
| 12 |
+
#traditional overloading is not in python
|
| 13 |
+
|
| 14 |
+
def __getitem__(self, index): #__getitem__ for indexing with [ ] (overrriding)
|
| 15 |
+
raise NotImplementedError
|
| 16 |
+
|
| 17 |
+
def __len__(self): # for overriding len()
|
| 18 |
+
raise NotImplementedError #when you call len(obj), Python internally calls obj.__len__().
|
| 19 |
+
|
| 20 |
+
def __add__(self, other): #to override an binary + operator
|
| 21 |
+
return ConcatDataset([self, other])
|
| 22 |
+
|
| 23 |
+
def reset(self): #currently is a placeholder and does nothing
|
| 24 |
+
return
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class ConcatDataset(Dataset): #inheriting from Dataset
|
| 28 |
+
"""
|
| 29 |
+
Dataset to concatenate multiple datasets.
|
| 30 |
+
Purpose: useful to assemble different existing datasets, possibly
|
| 31 |
+
large-scale datasets as the concatenation operation is done in an
|
| 32 |
+
on-the-fly manner.
|
| 33 |
+
|
| 34 |
+
Arguments:
|
| 35 |
+
datasets (sequence): List of datasets to be concatenated
|
| 36 |
+
"""
|
| 37 |
+
|
| 38 |
+
@staticmethod
|
| 39 |
+
def cumsum(sequence): #sequence: an ordered collection of items, where each item holds a relative position.Must be indexable and can be iterated. All sequences are iterables but reverse is not true e.g. sets not indexable
|
| 40 |
+
#sequence are strings, lists, tuples, byte sequences, byte arrays and range objects
|
| 41 |
+
r, s = [], 0 #r:list to hold cumulative sum of iterated datasets in sequence
|
| 42 |
+
#s: cumulative sum of all datasets in sequence
|
| 43 |
+
for e in sequence: #e: current dataset in sequence
|
| 44 |
+
l = len(e) # length of current dataset in sequence
|
| 45 |
+
r.append(l + s)
|
| 46 |
+
s += l
|
| 47 |
+
return r
|
| 48 |
+
|
| 49 |
+
def __init__(self, datasets):
|
| 50 |
+
super(ConcatDataset, self).__init__()
|
| 51 |
+
assert len(datasets) > 0, 'datasets should not be an empty iterable'
|
| 52 |
+
#halts the program and provides a diagnostic message.
|
| 53 |
+
#asserts are usually stripped out or disabled in production code, as they are meant for development and debugging
|
| 54 |
+
self.datasets = list(datasets) #store list of datasets
|
| 55 |
+
self.cumulative_sizes = self.cumsum(self.datasets) #store result of static method cumsum, list of cumulative length of previous datasets
|
| 56 |
+
|
| 57 |
+
def __len__(self):
|
| 58 |
+
return self.cumulative_sizes[-1] #length of concatenated datasets after cumsum
|
| 59 |
+
|
| 60 |
+
def __getitem__(self, idx): #dynamically selecting the correct dataset and item within that dataset based on the given index in the self.datasets
|
| 61 |
+
dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx) #bisect.bisect_right returns the index of the first element in the list that is greater than x (upper bound)
|
| 62 |
+
# line determines which dataset in self.datasets the item at idx belongs to.
|
| 63 |
+
if dataset_idx == 0: #idx belongs to first dataset
|
| 64 |
+
sample_idx = idx #so idx and cumulative idx same
|
| 65 |
+
else:#finds idx within the dataset
|
| 66 |
+
sample_idx = idx - self.cumulative_sizes[dataset_idx - 1]
|
| 67 |
+
return self.datasets[dataset_idx][sample_idx]
|
| 68 |
+
|
| 69 |
+
@property #property of the class. Can be accessed without ()
|
| 70 |
+
def cummulative_sizes(self): # basically a getter for encapsulation
|
| 71 |
+
warnings.warn("cummulative_sizes attribute is renamed to " #basically if user uses wrong function
|
| 72 |
+
"cumulative_sizes", DeprecationWarning, stacklevel=2) #If the stacklevel parameter is not set, the warning appears to originate from within the cummulative_sizes method itself.
|
| 73 |
+
#By setting stacklevel=2, the warning will appear to originate from the caller of the cummulative_sizes method.
|
| 74 |
+
return self.cumulative_sizes
|
DSRNet/data/transforms.py
ADDED
|
@@ -0,0 +1,375 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import division #This ensures true division (i.e., division results in floating-point numbers) in Python 2 otherwise we get //
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
import random
|
| 5 |
+
|
| 6 |
+
import kornia #for computer vision operations in PyTorch.
|
| 7 |
+
import torch
|
| 8 |
+
from PIL import Image
|
| 9 |
+
|
| 10 |
+
try:
|
| 11 |
+
import accimage #third-party Python library designed to provide accelerated image loading and processing capabilities, primarily intended for use with the PyTorch deep learning framework.Alternative of PIL/pillow
|
| 12 |
+
except ImportError:
|
| 13 |
+
accimage = None
|
| 14 |
+
import numpy as np
|
| 15 |
+
import scipy.stats as st
|
| 16 |
+
import cv2
|
| 17 |
+
import collections
|
| 18 |
+
import torchvision.transforms as transforms # for common image transformations.
|
| 19 |
+
import util.util as util
|
| 20 |
+
from scipy.signal import convolve2d
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
# utility
|
| 24 |
+
#
|
| 25 |
+
# PIL and accimage for loading
|
| 26 |
+
def _is_pil_image(img): #if accimage is not supported in environment
|
| 27 |
+
if accimage is not None:
|
| 28 |
+
return isinstance(img, (Image.Image, accimage.Image))
|
| 29 |
+
else:
|
| 30 |
+
return isinstance(img, Image.Image)
|
| 31 |
+
|
| 32 |
+
#in transformation step for augmentation and feeding into a model.
|
| 33 |
+
def _is_tensor_image(img): #to handle different types of image representations that might be encountered in various stages of image processing
|
| 34 |
+
return torch.is_tensor(img) and img.ndimension() == 3 # is 3D tensor
|
| 35 |
+
|
| 36 |
+
# for certain preprocessing steps.
|
| 37 |
+
def _is_numpy_image(img): #to handle different types of image representations that might be encountered in various stages of image processing
|
| 38 |
+
return isinstance(img, np.ndarray) and (img.ndim in {2, 3}) #is numpy array of 2 or 3 dimension
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def arrshow(arr):
|
| 42 |
+
Image.fromarray(arr.astype(np.uint8)).show()#converts a NumPy array to a PIL image and then displays it.
|
| 43 |
+
#arr.astype(np.uint8) converts the NumPy array arr to have a data type of np.uint8
|
| 44 |
+
# PIL images expect pixel values to be in the range [0, 255],
|
| 45 |
+
#Image.fromarray() function is a method provided by the PIL (Pillow) library to convert a pixel values to a PIL image.
|
| 46 |
+
#.show() is a method of the PIL Image class to open image in image viewer of system
|
| 47 |
+
|
| 48 |
+
'''"transformations" refer to a series of operations applied to the images to preprocess or augment them. These transformations can include resizing, cropping, flipping, rotating, and many others. They help in normalizing the data, enhancing certain features, or augmenting the dataset to improve the robustness and performance of the model.'''
|
| 49 |
+
|
| 50 |
+
def get_transform(opt): #based on option given, a list of transformations are generated
|
| 51 |
+
transform_list = []
|
| 52 |
+
osizes = util.parse_args(opt.loadSize)
|
| 53 |
+
fineSize = util.parse_args(opt.fineSize)
|
| 54 |
+
if opt.resize_or_crop == 'resize_and_crop':
|
| 55 |
+
transform_list.append(
|
| 56 |
+
transforms.RandomChoice([
|
| 57 |
+
transforms.Resize([osize, osize], Image.BICUBIC) for osize in osizes
|
| 58 |
+
]))
|
| 59 |
+
transform_list.append(transforms.RandomCrop(fineSize))
|
| 60 |
+
elif opt.resize_or_crop == 'crop':
|
| 61 |
+
transform_list.append(transforms.RandomCrop(fineSize))
|
| 62 |
+
elif opt.resize_or_crop == 'scale_width':
|
| 63 |
+
transform_list.append(transforms.Lambda(
|
| 64 |
+
lambda img: __scale_width(img, fineSize)))
|
| 65 |
+
elif opt.resize_or_crop == 'scale_width_and_crop':
|
| 66 |
+
transform_list.append(transforms.Lambda(
|
| 67 |
+
lambda img: __scale_width(img, opt.loadSize)))
|
| 68 |
+
transform_list.append(transforms.RandomCrop(opt.fineSize))
|
| 69 |
+
|
| 70 |
+
if opt.isTrain and not opt.no_flip:
|
| 71 |
+
transform_list.append(transforms.RandomHorizontalFlip())
|
| 72 |
+
|
| 73 |
+
return transforms.Compose(transform_list)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
to_norm_tensor = transforms.Compose([
|
| 77 |
+
transforms.ToTensor(),
|
| 78 |
+
transforms.Normalize(
|
| 79 |
+
(0.5, 0.5, 0.5),
|
| 80 |
+
(0.5, 0.5, 0.5)
|
| 81 |
+
)
|
| 82 |
+
])
|
| 83 |
+
|
| 84 |
+
to_tensor = transforms.ToTensor()
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def __scale_width(img, target_width):
|
| 88 |
+
ow, oh = img.size
|
| 89 |
+
if (ow == target_width):
|
| 90 |
+
return img
|
| 91 |
+
w = target_width
|
| 92 |
+
h = int(target_width * oh / ow)
|
| 93 |
+
h = math.ceil(h / 2.) * 2 # round up to even
|
| 94 |
+
return img.resize((w, h), Image.BICUBIC)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
# functional
|
| 98 |
+
def gaussian_blur(img, kernel_size, sigma):
|
| 99 |
+
if not _is_pil_image(img):
|
| 100 |
+
raise TypeError('img should be PIL Image. Got {}'.format(type(img)))
|
| 101 |
+
|
| 102 |
+
img = np.asarray(img)
|
| 103 |
+
# the 3rd dimension (i.e. inter-band) would be filtered which is unwanted for our purpose
|
| 104 |
+
# new = gaussian_filter(img, sigma=sigma, truncate=truncate)
|
| 105 |
+
if isinstance(kernel_size, int):
|
| 106 |
+
kernel_size = (kernel_size, kernel_size)
|
| 107 |
+
elif isinstance(kernel_size, collections.Sequence):
|
| 108 |
+
assert len(kernel_size) == 2
|
| 109 |
+
new = cv2.GaussianBlur(img, kernel_size, sigma) # apply gaussian filter band by band
|
| 110 |
+
return Image.fromarray(new)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
# transforms
|
| 114 |
+
class GaussianBlur(object):
|
| 115 |
+
def __init__(self, kernel_size=11, sigma=3):
|
| 116 |
+
self.kernel_size = kernel_size
|
| 117 |
+
self.sigma = sigma
|
| 118 |
+
|
| 119 |
+
def __call__(self, img):
|
| 120 |
+
return gaussian_blur(img, self.kernel_size, self.sigma)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
class ReflectionSythesis_0(object):
|
| 124 |
+
"""Reflection image data synthesis for weakly-supervised learning
|
| 125 |
+
of ICCV 2017 paper *"A Generic Deep Architecture for Single Image Reflection Removal and Image Smoothing"*
|
| 126 |
+
"""
|
| 127 |
+
|
| 128 |
+
def __init__(self, kernel_sizes=None, low_sigma=2, high_sigma=5, low_gamma=1.3,
|
| 129 |
+
high_gamma=1.3, low_delta=0.4, high_delta=1.8):
|
| 130 |
+
self.kernel_sizes = kernel_sizes or [11]
|
| 131 |
+
self.low_sigma = low_sigma
|
| 132 |
+
self.high_sigma = high_sigma
|
| 133 |
+
self.low_gamma = low_gamma
|
| 134 |
+
self.high_gamma = high_gamma
|
| 135 |
+
self.low_delta = low_delta
|
| 136 |
+
self.high_delta = high_delta
|
| 137 |
+
print('[i] reflection sythesis model: {}'.format({
|
| 138 |
+
'kernel_sizes': kernel_sizes, 'low_sigma': low_sigma, 'high_sigma': high_sigma,
|
| 139 |
+
'low_gamma': low_gamma, 'high_gamma': high_gamma}))
|
| 140 |
+
|
| 141 |
+
def __call__(self, B, R):
|
| 142 |
+
if not _is_pil_image(B):
|
| 143 |
+
raise TypeError('B should be PIL Image. Got {}'.format(type(B)))
|
| 144 |
+
if not _is_pil_image(R):
|
| 145 |
+
raise TypeError('R should be PIL Image. Got {}'.format(type(R)))
|
| 146 |
+
B_ = np.asarray(B, np.float32)
|
| 147 |
+
if random.random() < 0.4:
|
| 148 |
+
B_ = np.tile(np.random.uniform(0, 30, (1, 1, 1)), B_.shape) / 255.
|
| 149 |
+
else:
|
| 150 |
+
B_ = np.tile(np.random.normal(50, 50, (1, 1, 3)), (B_.shape[0], B_.shape[1], 1)).clip(0, 255) / 255.
|
| 151 |
+
R_ = np.asarray(R, np.float32) / 255.
|
| 152 |
+
|
| 153 |
+
kernel_size = np.random.choice(self.kernel_sizes)
|
| 154 |
+
sigma = np.random.uniform(self.low_sigma, self.high_sigma)
|
| 155 |
+
gamma = np.random.uniform(self.low_gamma, self.high_gamma)
|
| 156 |
+
delta = np.random.uniform(self.low_delta, self.high_delta)
|
| 157 |
+
R_blur = R_
|
| 158 |
+
kernel = cv2.getGaussianKernel(11, sigma)
|
| 159 |
+
kernel2d = np.dot(kernel, kernel.T)
|
| 160 |
+
|
| 161 |
+
for i in range(3):
|
| 162 |
+
R_blur[..., i] = convolve2d(R_blur[..., i], kernel2d, mode='same')
|
| 163 |
+
|
| 164 |
+
R_blur = np.clip(R_blur - np.mean(R_blur) * gamma, 0, 1)
|
| 165 |
+
R_blur = np.clip(R_blur * delta, 0, 1)
|
| 166 |
+
M_ = np.clip(R_blur + B_, 0, 1)
|
| 167 |
+
|
| 168 |
+
return B_, R_blur, M_
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
class ReflectionSythesis_1(object):
|
| 172 |
+
"""Reflection image data synthesis for weakly-supervised learning
|
| 173 |
+
of ICCV 2017 paper *"A Generic Deep Architecture for Single Image Reflection Removal and Image Smoothing"*
|
| 174 |
+
"""
|
| 175 |
+
|
| 176 |
+
def __init__(self, kernel_sizes=None, low_sigma=2, high_sigma=5, low_gamma=1.3, high_gamma=1.3):
|
| 177 |
+
self.kernel_sizes = kernel_sizes or [11]
|
| 178 |
+
self.low_sigma = low_sigma
|
| 179 |
+
self.high_sigma = high_sigma
|
| 180 |
+
self.low_gamma = low_gamma
|
| 181 |
+
self.high_gamma = high_gamma
|
| 182 |
+
print('[i] reflection sythesis model: {}'.format({
|
| 183 |
+
'kernel_sizes': kernel_sizes, 'low_sigma': low_sigma, 'high_sigma': high_sigma,
|
| 184 |
+
'low_gamma': low_gamma, 'high_gamma': high_gamma}))
|
| 185 |
+
|
| 186 |
+
def __call__(self, B, R):
|
| 187 |
+
if not _is_pil_image(B):
|
| 188 |
+
raise TypeError('B should be PIL Image. Got {}'.format(type(B)))
|
| 189 |
+
if not _is_pil_image(R):
|
| 190 |
+
raise TypeError('R should be PIL Image. Got {}'.format(type(R)))
|
| 191 |
+
|
| 192 |
+
B_ = np.asarray(B, np.float32) / 255.
|
| 193 |
+
R_ = np.asarray(R, np.float32) / 255.
|
| 194 |
+
|
| 195 |
+
kernel_size = np.random.choice(self.kernel_sizes)
|
| 196 |
+
sigma = np.random.uniform(self.low_sigma, self.high_sigma)
|
| 197 |
+
gamma = np.random.uniform(self.low_gamma, self.high_gamma)
|
| 198 |
+
R_blur = R_
|
| 199 |
+
kernel = cv2.getGaussianKernel(11, sigma)
|
| 200 |
+
kernel2d = np.dot(kernel, kernel.T)
|
| 201 |
+
|
| 202 |
+
for i in range(3):
|
| 203 |
+
R_blur[..., i] = convolve2d(R_blur[..., i], kernel2d, mode='same')
|
| 204 |
+
|
| 205 |
+
M_ = B_ + R_blur
|
| 206 |
+
|
| 207 |
+
if np.max(M_) > 1:
|
| 208 |
+
m = M_[M_ > 1]
|
| 209 |
+
m = (np.mean(m) - 1) * gamma
|
| 210 |
+
R_blur = np.clip(R_blur - m, 0, 1)
|
| 211 |
+
M_ = np.clip(R_blur + B_, 0, 1)
|
| 212 |
+
|
| 213 |
+
return B_, R_blur, M_
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
class NoiseReflectionSythesis(object):
|
| 217 |
+
"""Reflection image data synthesis for weakly-supervised learning
|
| 218 |
+
of ICCV 2017 paper *"A Generic Deep Architecture for Single Image Reflection Removal and Image Smoothing"*
|
| 219 |
+
"""
|
| 220 |
+
|
| 221 |
+
def __init__(self, kernel_sizes=None, low_sigma=2, high_sigma=5, low_gamma=1.3, high_gamma=1.3):
|
| 222 |
+
self.kernel_sizes = kernel_sizes or [11]
|
| 223 |
+
self.low_sigma = low_sigma
|
| 224 |
+
self.high_sigma = high_sigma
|
| 225 |
+
self.low_gamma = low_gamma
|
| 226 |
+
self.high_gamma = high_gamma
|
| 227 |
+
print('[i] reflection sythesis model: {}'.format({
|
| 228 |
+
'kernel_sizes': kernel_sizes, 'low_sigma': low_sigma, 'high_sigma': high_sigma,
|
| 229 |
+
'low_gamma': low_gamma, 'high_gamma': high_gamma}))
|
| 230 |
+
|
| 231 |
+
def __call__(self, B, R, N):
|
| 232 |
+
if not _is_pil_image(B):
|
| 233 |
+
raise TypeError('B should be PIL Image. Got {}'.format(type(B)))
|
| 234 |
+
if not _is_pil_image(R):
|
| 235 |
+
raise TypeError('R should be PIL Image. Got {}'.format(type(R)))
|
| 236 |
+
|
| 237 |
+
B_ = np.asarray(B, np.float32) / 255. + N
|
| 238 |
+
R_ = np.asarray(R, np.float32) / 255.
|
| 239 |
+
|
| 240 |
+
kernel_size = np.random.choice(self.kernel_sizes)
|
| 241 |
+
sigma = np.random.uniform(self.low_sigma, self.high_sigma)
|
| 242 |
+
gamma = np.random.uniform(self.low_gamma, self.high_gamma)
|
| 243 |
+
R_blur = R_
|
| 244 |
+
kernel = cv2.getGaussianKernel(11, sigma)
|
| 245 |
+
kernel2d = np.dot(kernel, kernel.T)
|
| 246 |
+
|
| 247 |
+
for i in range(3):
|
| 248 |
+
R_blur[..., i] = convolve2d(R_blur[..., i], kernel2d, mode='same')
|
| 249 |
+
|
| 250 |
+
M_ = B_ * 0.5 + R_blur * 0.5
|
| 251 |
+
|
| 252 |
+
return B_.astype(np.float32), R_blur.astype(np.float32), M_.astype(np.float32)
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
class NoiseReflectionSythesisTorch(object):
|
| 256 |
+
"""Reflection image data synthesis for weakly-supervised learning
|
| 257 |
+
of ICCV 2017 paper *"A Generic Deep Architecture for Single Image Reflection Removal and Image Smoothing"*
|
| 258 |
+
"""
|
| 259 |
+
|
| 260 |
+
def __init__(self, kernel_sizes=None, low_sigma=2, high_sigma=5, low_gamma=1.3, high_gamma=1.3):
|
| 261 |
+
self.kernel_sizes = kernel_sizes or [11]
|
| 262 |
+
self.low_sigma = low_sigma
|
| 263 |
+
self.high_sigma = high_sigma
|
| 264 |
+
self.low_gamma = low_gamma
|
| 265 |
+
self.high_gamma = high_gamma
|
| 266 |
+
print('[i] reflection sythesis model: {}'.format({
|
| 267 |
+
'kernel_sizes': kernel_sizes, 'low_sigma': low_sigma, 'high_sigma': high_sigma,
|
| 268 |
+
'low_gamma': low_gamma, 'high_gamma': high_gamma}))
|
| 269 |
+
|
| 270 |
+
def __call__(self, B, R):
|
| 271 |
+
stdN = np.random.uniform(15, 55)
|
| 272 |
+
noise = torch.zeros(B.size()).normal_(mean=0, std=stdN / 255.)
|
| 273 |
+
B = B + noise
|
| 274 |
+
|
| 275 |
+
sigma = np.random.uniform(self.low_sigma, self.high_sigma)
|
| 276 |
+
R = kornia.gaussian_blur2d(R.unsqueeze(0), (11, 11), (sigma, sigma), border_type='replicate').squeeze(0)
|
| 277 |
+
M = B * 0.5 + R * 0.5
|
| 278 |
+
|
| 279 |
+
return B, R, M
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
class Sobel(object):
|
| 283 |
+
def __call__(self, img):
|
| 284 |
+
if not _is_pil_image(img):
|
| 285 |
+
raise TypeError('img should be PIL Image. Got {}'.format(type(img)))
|
| 286 |
+
|
| 287 |
+
gray_img = np.array(img.convert('L'))
|
| 288 |
+
x = cv2.Sobel(gray_img, cv2.CV_16S, 1, 0)
|
| 289 |
+
y = cv2.Sobel(gray_img, cv2.CV_16S, 0, 1)
|
| 290 |
+
|
| 291 |
+
absX = cv2.convertScaleAbs(x)
|
| 292 |
+
absY = cv2.convertScaleAbs(y)
|
| 293 |
+
|
| 294 |
+
dst = cv2.addWeighted(absX, 0.5, absY, 0.5, 0)
|
| 295 |
+
return Image.fromarray(dst)
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
class ReflectionSythesis_2(object):
|
| 299 |
+
"""Reflection image data synthesis for weakly-supervised learning
|
| 300 |
+
of CVPR 2018 paper *"Single Image Reflection Separation with Perceptual Losses"*
|
| 301 |
+
"""
|
| 302 |
+
|
| 303 |
+
def __init__(self, kernel_sizes=None):
|
| 304 |
+
self.kernel_sizes = kernel_sizes or np.linspace(1, 5, 80)
|
| 305 |
+
|
| 306 |
+
@staticmethod
|
| 307 |
+
def gkern(kernlen=100, nsig=1):
|
| 308 |
+
"""Returns a 2D Gaussian kernel array."""
|
| 309 |
+
interval = (2 * nsig + 1.) / (kernlen)
|
| 310 |
+
x = np.linspace(-nsig - interval / 2., nsig + interval / 2., kernlen + 1)
|
| 311 |
+
kern1d = np.diff(st.norm.cdf(x))
|
| 312 |
+
kernel_raw = np.sqrt(np.outer(kern1d, kern1d))
|
| 313 |
+
kernel = kernel_raw / kernel_raw.sum()
|
| 314 |
+
kernel = kernel / kernel.max()
|
| 315 |
+
return kernel
|
| 316 |
+
|
| 317 |
+
def __call__(self, t, r):
|
| 318 |
+
t = np.float32(t) / 255.
|
| 319 |
+
r = np.float32(r) / 255.
|
| 320 |
+
ori_t = t
|
| 321 |
+
# create a vignetting mask
|
| 322 |
+
g_mask = self.gkern(560, 3)
|
| 323 |
+
g_mask = np.dstack((g_mask, g_mask, g_mask))
|
| 324 |
+
sigma = self.kernel_sizes[np.random.randint(0, len(self.kernel_sizes))]
|
| 325 |
+
|
| 326 |
+
t = np.power(t, 2.2)
|
| 327 |
+
r = np.power(r, 2.2)
|
| 328 |
+
|
| 329 |
+
sz = int(2 * np.ceil(2 * sigma) + 1)
|
| 330 |
+
|
| 331 |
+
r_blur = cv2.GaussianBlur(r, (sz, sz), sigma, sigma, 0)
|
| 332 |
+
blend = r_blur + t
|
| 333 |
+
|
| 334 |
+
att = 1.08 + np.random.random() / 10.0
|
| 335 |
+
|
| 336 |
+
for i in range(3):
|
| 337 |
+
maski = blend[:, :, i] > 1
|
| 338 |
+
mean_i = max(1., np.sum(blend[:, :, i] * maski) / (maski.sum() + 1e-6))
|
| 339 |
+
r_blur[:, :, i] = r_blur[:, :, i] - (mean_i - 1) * att
|
| 340 |
+
r_blur[r_blur >= 1] = 1
|
| 341 |
+
r_blur[r_blur <= 0] = 0
|
| 342 |
+
|
| 343 |
+
h, w = r_blur.shape[0:2]
|
| 344 |
+
neww = np.random.randint(0, 560 - w - 10)
|
| 345 |
+
newh = np.random.randint(0, 560 - h - 10)
|
| 346 |
+
alpha1 = g_mask[newh:newh + h, neww:neww + w, :]
|
| 347 |
+
alpha2 = 1 - np.random.random() / 5.0
|
| 348 |
+
r_blur_mask = np.multiply(r_blur, alpha1)
|
| 349 |
+
blend = r_blur_mask + t * alpha2
|
| 350 |
+
|
| 351 |
+
t = np.power(t, 1 / 2.2)
|
| 352 |
+
r_blur_mask = np.power(r_blur_mask, 1 / 2.2)
|
| 353 |
+
blend = np.power(blend, 1 / 2.2)
|
| 354 |
+
blend[blend >= 1] = 1
|
| 355 |
+
blend[blend <= 0] = 0
|
| 356 |
+
|
| 357 |
+
return np.float32(ori_t), np.float32(r_blur_mask), np.float32(blend)
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
# Examples
|
| 361 |
+
if __name__ == '__main__':
|
| 362 |
+
"""cv2 imread"""
|
| 363 |
+
# img = cv2.imread('testdata_reflection_real/19-input.png')
|
| 364 |
+
# img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
| 365 |
+
# img2 = cv2.GaussianBlur(img, (11,11), 3)
|
| 366 |
+
|
| 367 |
+
"""Sobel Operator"""
|
| 368 |
+
# img = np.array(Image.open('datasets/VOC224/train/B/2007_000250.png').convert('L'))
|
| 369 |
+
|
| 370 |
+
"""Reflection Sythesis"""
|
| 371 |
+
b = Image.open('')
|
| 372 |
+
r = Image.open('')
|
| 373 |
+
G = ReflectionSythesis_0()
|
| 374 |
+
m, r = G(b, r)
|
| 375 |
+
r.show()
|
DSRNet/engine.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import util.util as util
|
| 3 |
+
from models import make_model
|
| 4 |
+
import time
|
| 5 |
+
import os
|
| 6 |
+
import sys
|
| 7 |
+
from os.path import join
|
| 8 |
+
from util.visualizer import Visualizer
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class Engine(object):
|
| 12 |
+
def __init__(self, opt):
|
| 13 |
+
self.opt = opt
|
| 14 |
+
self.writer = None
|
| 15 |
+
self.visualizer = None
|
| 16 |
+
self.model = None
|
| 17 |
+
self.best_val_loss = 1e6
|
| 18 |
+
|
| 19 |
+
self.__setup()
|
| 20 |
+
|
| 21 |
+
def __setup(self):#private methods prefixed with __
|
| 22 |
+
self.basedir = join('checkpoints', self.opt.name) #creating directory path for checkpoint files
|
| 23 |
+
os.makedirs(self.basedir, exist_ok=True)
|
| 24 |
+
|
| 25 |
+
opt = self.opt
|
| 26 |
+
|
| 27 |
+
"""(mingcv)Model"""
|
| 28 |
+
self.model = make_model(self.opt.model)() #(mingcv)models.__dict__[self.opt.model]()
|
| 29 |
+
self.model.initialize(opt)
|
| 30 |
+
if not opt.no_log: # setting up logging and visualisation
|
| 31 |
+
self.writer = util.get_summary_writer(os.path.join(self.basedir, 'logs'))
|
| 32 |
+
self.visualizer = Visualizer(opt)
|
| 33 |
+
|
| 34 |
+
def train(self, train_loader, **kwargs):
|
| 35 |
+
print('\nEpoch: %d' % self.epoch)
|
| 36 |
+
avg_meters = util.AverageMeters()#generating running averga eof metrics
|
| 37 |
+
opt = self.opt
|
| 38 |
+
model = self.model
|
| 39 |
+
epoch = self.epoch
|
| 40 |
+
|
| 41 |
+
epoch_start_time = time.time()
|
| 42 |
+
for i, data in enumerate(train_loader):
|
| 43 |
+
iter_start_time = time.time()
|
| 44 |
+
iterations = self.iterations
|
| 45 |
+
|
| 46 |
+
model.set_input(data, mode='train')
|
| 47 |
+
model.optimize_parameters(**kwargs)
|
| 48 |
+
|
| 49 |
+
errors = model.get_current_errors()
|
| 50 |
+
avg_meters.update(errors)
|
| 51 |
+
util.progress_bar(i, len(train_loader), str(avg_meters))
|
| 52 |
+
|
| 53 |
+
if not opt.no_log:#update logging and visualisation every iteration of training
|
| 54 |
+
util.write_loss(self.writer, 'train', avg_meters, iterations)
|
| 55 |
+
|
| 56 |
+
if iterations % opt.display_freq == 0 and opt.display_id != 0:
|
| 57 |
+
save_result = iterations % opt.update_html_freq == 0
|
| 58 |
+
self.visualizer.display_current_results(model.get_current_visuals(), epoch, save_result)
|
| 59 |
+
|
| 60 |
+
if iterations % opt.print_freq == 0 and opt.display_id != 0:
|
| 61 |
+
t = (time.time() - iter_start_time)
|
| 62 |
+
|
| 63 |
+
self.iterations += 1
|
| 64 |
+
|
| 65 |
+
self.epoch += 1
|
| 66 |
+
|
| 67 |
+
if not self.opt.no_log:#
|
| 68 |
+
if self.epoch % opt.save_epoch_freq == 0:
|
| 69 |
+
print('saving the model at epoch %d, iters %d' %
|
| 70 |
+
(self.epoch, self.iterations))
|
| 71 |
+
model.save()
|
| 72 |
+
|
| 73 |
+
print('saving the latest model at the end of epoch %d, iters %d' %
|
| 74 |
+
(self.epoch, self.iterations))
|
| 75 |
+
model.save(label='latest')
|
| 76 |
+
|
| 77 |
+
print('Time Taken: %d sec' %
|
| 78 |
+
(time.time() - epoch_start_time))
|
| 79 |
+
|
| 80 |
+
# model.update_learning_rate()
|
| 81 |
+
try:
|
| 82 |
+
train_loader.reset()
|
| 83 |
+
except:
|
| 84 |
+
pass
|
| 85 |
+
|
| 86 |
+
def eval(self, val_loader, dataset_name, savedir='./tmp', loss_key=None, max_save_size=None, **kwargs):
|
| 87 |
+
#(mingcv)print(dataset_name)
|
| 88 |
+
if savedir is not None:# create a directory for saving if not exist
|
| 89 |
+
os.makedirs(savedir, exist_ok=True)
|
| 90 |
+
self.f = open(os.path.join(savedir, 'metrics.txt'), 'w+')
|
| 91 |
+
self.f.write(dataset_name + '\n')
|
| 92 |
+
avg_meters = util.AverageMeters()#get running average metrics
|
| 93 |
+
model = self.model
|
| 94 |
+
opt = self.opt
|
| 95 |
+
with torch.no_grad():#disable gradient calculation to save memory
|
| 96 |
+
for i, data in enumerate(val_loader):
|
| 97 |
+
if opt.selected and data['fn'][0].split('.')[0] not in opt.selected:
|
| 98 |
+
continue
|
| 99 |
+
if max_save_size is not None and i > max_save_size:
|
| 100 |
+
index = model.eval(data, savedir=None, **kwargs)
|
| 101 |
+
else:
|
| 102 |
+
index = model.eval(data, savedir=savedir, **kwargs)
|
| 103 |
+
|
| 104 |
+
#(mingcv) print(data['fn'][0], index)
|
| 105 |
+
if savedir is not None:
|
| 106 |
+
self.f.write(f"{data['fn'][0]} {index['PSNR']} {index['SSIM']}\n")
|
| 107 |
+
avg_meters.update(index)
|
| 108 |
+
util.progress_bar(i, len(val_loader), str(avg_meters))
|
| 109 |
+
|
| 110 |
+
if not opt.no_log:
|
| 111 |
+
util.write_loss(self.writer, join('eval', dataset_name), avg_meters, self.epoch)
|
| 112 |
+
|
| 113 |
+
if loss_key is not None:
|
| 114 |
+
val_loss = avg_meters[loss_key]
|
| 115 |
+
if val_loss < self.best_val_loss:
|
| 116 |
+
self.best_val_loss = val_loss
|
| 117 |
+
print('saving the best model at the end of epoch %d, iters %d' %
|
| 118 |
+
(self.epoch, self.iterations))
|
| 119 |
+
model.save(label='best_{}_{}'.format(loss_key, dataset_name))
|
| 120 |
+
|
| 121 |
+
return avg_meters
|
| 122 |
+
|
| 123 |
+
def test(self, test_loader, savedir=None, **kwargs):
|
| 124 |
+
model = self.model
|
| 125 |
+
opt = self.opt
|
| 126 |
+
with torch.no_grad():
|
| 127 |
+
for i, data in enumerate(test_loader):
|
| 128 |
+
model.test(data, savedir=savedir, **kwargs)
|
| 129 |
+
util.progress_bar(i, len(test_loader))
|
| 130 |
+
|
| 131 |
+
def save_model(self):
|
| 132 |
+
self.model.save()
|
| 133 |
+
|
| 134 |
+
def save_eval(self, label):
|
| 135 |
+
self.model.save_eval(label)
|
| 136 |
+
|
| 137 |
+
@property
|
| 138 |
+
def iterations(self):
|
| 139 |
+
return self.model.iterations
|
| 140 |
+
|
| 141 |
+
@iterations.setter
|
| 142 |
+
def iterations(self, i):
|
| 143 |
+
self.model.iterations = i
|
| 144 |
+
|
| 145 |
+
@property
|
| 146 |
+
def epoch(self):
|
| 147 |
+
return self.model.epoch
|
| 148 |
+
|
| 149 |
+
@epoch.setter
|
| 150 |
+
def epoch(self, e):
|
| 151 |
+
self.model.epoch = e
|
DSRNet/eval_sirs.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from os.path import join
|
| 3 |
+
|
| 4 |
+
import torch.backends.cudnn as cudnn
|
| 5 |
+
|
| 6 |
+
import data.sirs_dataset as datasets
|
| 7 |
+
from data.image_folder import read_fns
|
| 8 |
+
from engine import Engine
|
| 9 |
+
from options.net_options.train_options import TrainOptions
|
| 10 |
+
from tools import mutils
|
| 11 |
+
|
| 12 |
+
opt = TrainOptions().parse() #parses training options from command-line arguments using the TrainOptions class and stores them in opt
|
| 13 |
+
#evaluation settings:
|
| 14 |
+
opt.isTrain = False
|
| 15 |
+
cudnn.benchmark = True
|
| 16 |
+
opt.no_log = True
|
| 17 |
+
opt.display_id = 0
|
| 18 |
+
opt.verbose = False
|
| 19 |
+
|
| 20 |
+
datadir = os.path.join(opt.base_dir, 'test')# test data directory
|
| 21 |
+
|
| 22 |
+
#(mingcv) Define evaluation/test dataset
|
| 23 |
+
eval_dataset_real = datasets.DSRTestDataset(join(datadir, f'real20_420'),
|
| 24 |
+
fns=read_fns('data/real_test.txt'),
|
| 25 |
+
if_align=opt.if_align)
|
| 26 |
+
eval_dataset_postcard = datasets.DSRTestDataset(join(datadir, 'SIR2/PostcardDataset'),
|
| 27 |
+
if_align=opt.if_align)
|
| 28 |
+
eval_dataset_solidobject = datasets.DSRTestDataset(join(datadir, 'SIR2/SolidObjectDataset'),
|
| 29 |
+
if_align=opt.if_align)
|
| 30 |
+
eval_dataset_wild = datasets.DSRTestDataset(join(datadir, 'SIR2/WildSceneDataset'),
|
| 31 |
+
if_align=opt.if_align)
|
| 32 |
+
|
| 33 |
+
eval_dataloader_real = datasets.DataLoader(
|
| 34 |
+
eval_dataset_real, batch_size=1, shuffle=True,
|
| 35 |
+
num_workers=opt.nThreads, pin_memory=True)
|
| 36 |
+
|
| 37 |
+
eval_dataloader_solidobject = datasets.DataLoader(
|
| 38 |
+
eval_dataset_solidobject, batch_size=1, shuffle=False,
|
| 39 |
+
num_workers=opt.nThreads, pin_memory=True)
|
| 40 |
+
|
| 41 |
+
eval_dataloader_postcard = datasets.DataLoader(
|
| 42 |
+
eval_dataset_postcard, batch_size=1, shuffle=False,
|
| 43 |
+
num_workers=opt.nThreads, pin_memory=True)
|
| 44 |
+
|
| 45 |
+
eval_dataloader_wild = datasets.DataLoader(
|
| 46 |
+
eval_dataset_wild, batch_size=1, shuffle=False,
|
| 47 |
+
num_workers=opt.nThreads, pin_memory=True)
|
| 48 |
+
|
| 49 |
+
engine = Engine(opt)
|
| 50 |
+
|
| 51 |
+
"""Main Loop"""
|
| 52 |
+
result_dir = os.path.join('./checkpoints', opt.name, mutils.get_formatted_time())
|
| 53 |
+
|
| 54 |
+
#
|
| 55 |
+
res = engine.eval(eval_dataloader_real, dataset_name='testdata_real',
|
| 56 |
+
savedir=join(result_dir, 'real20'), suffix='real20')
|
| 57 |
+
print(res)
|
| 58 |
+
|
| 59 |
+
res = engine.eval(eval_dataloader_solidobject, dataset_name='testdata_solidobject',
|
| 60 |
+
savedir=join(result_dir, 'solidobject'), suffix='solidobject')
|
| 61 |
+
print(res)
|
| 62 |
+
res = engine.eval(eval_dataloader_postcard, dataset_name='testdata_postcard',
|
| 63 |
+
savedir=join(result_dir, 'postcard'), suffix='postcard')
|
| 64 |
+
print(res)
|
| 65 |
+
res = engine.eval(eval_dataloader_wild, dataset_name='testdata_wild',
|
| 66 |
+
savedir=join(result_dir, 'wild'), suffix='wild')
|
| 67 |
+
print(res)
|
DSRNet/eval_sirs_4000
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from os.path import join
|
| 3 |
+
|
| 4 |
+
import torch.backends.cudnn as cudnn
|
| 5 |
+
|
| 6 |
+
import data.sirs_dataset as datasets
|
| 7 |
+
from data.image_folder import read_fns
|
| 8 |
+
from engine import Engine
|
| 9 |
+
from options.net_options.train_options import TrainOptions
|
| 10 |
+
from tools import mutils
|
| 11 |
+
|
| 12 |
+
opt = TrainOptions().parse()
|
| 13 |
+
|
| 14 |
+
opt.isTrain = False
|
| 15 |
+
cudnn.benchmark = True
|
| 16 |
+
opt.no_log = True
|
| 17 |
+
opt.display_id = 0
|
| 18 |
+
opt.verbose = False
|
| 19 |
+
|
| 20 |
+
datadir = os.path.join(opt.base_dir, 'test')
|
| 21 |
+
|
| 22 |
+
# Define evaluation/test dataset
|
| 23 |
+
eval_dataset_real = datasets.DSRTestDataset(join(datadir, f'real20_420'),
|
| 24 |
+
fns=read_fns('data/real_test.txt'),
|
| 25 |
+
if_align=opt.if_align)
|
| 26 |
+
eval_dataset_postcard = datasets.DSRTestDataset(join(datadir, 'SIR2/PostcardDataset'),
|
| 27 |
+
if_align=opt.if_align)
|
| 28 |
+
eval_dataset_solidobject = datasets.DSRTestDataset(join(datadir, 'SIR2/SolidObjectDataset'),
|
| 29 |
+
if_align=opt.if_align)
|
| 30 |
+
eval_dataset_wild = datasets.DSRTestDataset(join(datadir, 'SIR2/WildSceneDataset'),
|
| 31 |
+
if_align=opt.if_align)
|
| 32 |
+
|
| 33 |
+
eval_dataset_nature = datasets.DSRTestDataset(join(datadir, 'Nature'), if_align=opt.if_align)
|
| 34 |
+
|
| 35 |
+
eval_dataloader_real = datasets.DataLoader(
|
| 36 |
+
eval_dataset_real, batch_size=1, shuffle=True,
|
| 37 |
+
num_workers=opt.nThreads, pin_memory=True)
|
| 38 |
+
|
| 39 |
+
eval_dataloader_solidobject = datasets.DataLoader(
|
| 40 |
+
eval_dataset_solidobject, batch_size=1, shuffle=False,
|
| 41 |
+
num_workers=opt.nThreads, pin_memory=True)
|
| 42 |
+
|
| 43 |
+
eval_dataloader_postcard = datasets.DataLoader(
|
| 44 |
+
eval_dataset_postcard, batch_size=1, shuffle=False,
|
| 45 |
+
num_workers=opt.nThreads, pin_memory=True)
|
| 46 |
+
|
| 47 |
+
eval_dataloader_wild = datasets.DataLoader(
|
| 48 |
+
eval_dataset_wild, batch_size=1, shuffle=False,
|
| 49 |
+
num_workers=opt.nThreads, pin_memory=True)
|
| 50 |
+
|
| 51 |
+
eval_dataloader_nature = datasets.DataLoader(
|
| 52 |
+
eval_dataset_nature, batch_size=1, shuffle=False,
|
| 53 |
+
num_workers=opt.nThreads, pin_memory=True)
|
| 54 |
+
engine = Engine(opt)
|
| 55 |
+
|
| 56 |
+
"""Main Loop"""
|
| 57 |
+
result_dir = os.path.join('./checkpoints', opt.name, mutils.get_formatted_time())
|
| 58 |
+
|
| 59 |
+
#
|
| 60 |
+
res = engine.eval(eval_dataloader_real, dataset_name='testdata_real',
|
| 61 |
+
savedir=join(result_dir, 'real20'), suffix='real20')
|
| 62 |
+
print(res)
|
| 63 |
+
|
| 64 |
+
res = engine.eval(eval_dataloader_solidobject, dataset_name='testdata_solidobject',
|
| 65 |
+
savedir=join(result_dir, 'solidobject'), suffix='solidobject')
|
| 66 |
+
print(res)
|
| 67 |
+
res = engine.eval(eval_dataloader_postcard, dataset_name='testdata_postcard',
|
| 68 |
+
savedir=join(result_dir, 'postcard'), suffix='postcard')
|
| 69 |
+
print(res)
|
| 70 |
+
res = engine.eval(eval_dataloader_wild, dataset_name='testdata_wild',
|
| 71 |
+
savedir=join(result_dir, 'wild'), suffix='wild')
|
| 72 |
+
print(res)
|
| 73 |
+
res = engine.eval(eval_dataloader_wild, dataset_name='testdata_nature',
|
| 74 |
+
savedir=join(result_dir, 'nature'), suffix='nature')
|
| 75 |
+
print(res)
|
DSRNet/models/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import importlib
|
| 2 |
+
|
| 3 |
+
import models.losses as losses
|
| 4 |
+
from models.arch import *
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def make_model(name: str):
|
| 8 |
+
base = importlib.import_module('models.' + name)
|
| 9 |
+
model = getattr(base, 'DSRNetModel')
|
| 10 |
+
return model
|
DSRNet/models/__pycache__/__init__.cpython-38.pyc
ADDED
|
Binary file (1.21 kB). View file
|
|
|
DSRNet/models/__pycache__/networks.cpython-38.pyc
ADDED
|
Binary file (7.28 kB). View file
|
|
|
DSRNet/models/__pycache__/vgg.cpython-38.pyc
ADDED
|
Binary file (2.15 kB). View file
|
|
|
DSRNet/models/arch/__init__.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
from models.arch.dsrnet import DSRNet
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def dsrnet_s(in_channels=3, out_channels=3, width=32):
|
| 7 |
+
enc_blks = [2, 2, 2]
|
| 8 |
+
middle_blk_num = 4
|
| 9 |
+
dec_blks = [2, 2, 2]
|
| 10 |
+
|
| 11 |
+
return DSRNet(in_channels, out_channels, width=width,
|
| 12 |
+
middle_blk_num=middle_blk_num,
|
| 13 |
+
enc_blk_nums=enc_blks,
|
| 14 |
+
dec_blk_nums=dec_blks,
|
| 15 |
+
shared_b=False)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def dsrnet_l(in_channels=3, out_channels=3, width=64):
|
| 19 |
+
enc_blks = [2, 2, 4, 8]
|
| 20 |
+
middle_blk_num = 12
|
| 21 |
+
dec_blks = [2, 2, 2, 2]
|
| 22 |
+
|
| 23 |
+
return DSRNet(in_channels, out_channels, width=width,
|
| 24 |
+
middle_blk_num=middle_blk_num,
|
| 25 |
+
enc_blk_nums=enc_blks,
|
| 26 |
+
dec_blk_nums=dec_blks,
|
| 27 |
+
shared_b=True)
|
| 28 |
+
|
| 29 |
+
if __name__ == '__main__':
|
| 30 |
+
from tools import mutils
|
| 31 |
+
|
| 32 |
+
x = torch.ones(1, 3, 256, 256).cuda()
|
| 33 |
+
feats = [
|
| 34 |
+
torch.ones(1, 64, 128, 128).cuda(),
|
| 35 |
+
torch.ones(1, 192, 64, 64).cuda(),
|
| 36 |
+
torch.ones(1, 384, 32, 32).cuda(),
|
| 37 |
+
torch.ones(1, 768, 16, 16).cuda(),
|
| 38 |
+
torch.ones(1, 2560, 8, 8).cuda(),
|
| 39 |
+
]
|
| 40 |
+
enc_blks = [2, 2, 4, 8]
|
| 41 |
+
middle_blk_num = 12
|
| 42 |
+
dec_blks = [2, 2, 2, 2]
|
| 43 |
+
|
| 44 |
+
model = dsrnet(3, 3).cuda()
|
| 45 |
+
mutils.count_parameters(model)
|
| 46 |
+
mutils.count_parameters(model.intro)
|
| 47 |
+
|
| 48 |
+
out_l, out_r = model(x, feats)
|
| 49 |
+
print(out_l.shape, out_r.shape)
|
DSRNet/models/arch/__pycache__/__init__.cpython-38.pyc
ADDED
|
Binary file (17.4 kB). View file
|
|
|
DSRNet/models/arch/dsrnet.py
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from collections import OrderedDict
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
from models.arch.lrm import LRM
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class LayerNormFunction(torch.autograd.Function):
|
| 9 |
+
|
| 10 |
+
@staticmethod
|
| 11 |
+
def forward(ctx, x, weight, bias, eps):
|
| 12 |
+
ctx.eps = eps
|
| 13 |
+
N, C, H, W = x.size()
|
| 14 |
+
mu = x.mean(1, keepdim=True)
|
| 15 |
+
var = (x - mu).pow(2).mean(1, keepdim=True)
|
| 16 |
+
y = (x - mu) / (var + eps).sqrt()
|
| 17 |
+
ctx.save_for_backward(y, var, weight)
|
| 18 |
+
y = weight.view(1, C, 1, 1) * y + bias.view(1, C, 1, 1)
|
| 19 |
+
return y
|
| 20 |
+
|
| 21 |
+
@staticmethod
|
| 22 |
+
def backward(ctx, grad_output):
|
| 23 |
+
eps = ctx.eps
|
| 24 |
+
|
| 25 |
+
N, C, H, W = grad_output.size()
|
| 26 |
+
y, var, weight = ctx.saved_tensors
|
| 27 |
+
g = grad_output * weight.view(1, C, 1, 1)
|
| 28 |
+
mean_g = g.mean(dim=1, keepdim=True)
|
| 29 |
+
|
| 30 |
+
mean_gy = (g * y).mean(dim=1, keepdim=True)
|
| 31 |
+
gx = 1. / torch.sqrt(var + eps) * (g - y * mean_gy - mean_g)
|
| 32 |
+
return gx, (grad_output * y).sum(dim=3).sum(dim=2).sum(dim=0), grad_output.sum(dim=3).sum(dim=2).sum(
|
| 33 |
+
dim=0), None
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class LayerNorm2d(nn.Module):
|
| 37 |
+
|
| 38 |
+
def __init__(self, channels, eps=1e-6):
|
| 39 |
+
super(LayerNorm2d, self).__init__()
|
| 40 |
+
self.register_parameter('weight', nn.Parameter(torch.ones(channels)))
|
| 41 |
+
self.register_parameter('bias', nn.Parameter(torch.zeros(channels)))
|
| 42 |
+
self.eps = eps
|
| 43 |
+
|
| 44 |
+
def forward(self, x):
|
| 45 |
+
return LayerNormFunction.apply(x, self.weight, self.bias, self.eps)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
class CABlock(nn.Module):
|
| 49 |
+
def __init__(self, channels):
|
| 50 |
+
super(CABlock, self).__init__()
|
| 51 |
+
self.ca = nn.Sequential(
|
| 52 |
+
nn.AdaptiveAvgPool2d(1),
|
| 53 |
+
nn.Conv2d(channels, channels, 1)
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
def forward(self, x):
|
| 57 |
+
return x * self.ca(x)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class DualStreamGate(nn.Module):
|
| 61 |
+
def forward(self, x, y):
|
| 62 |
+
x1, x2 = x.chunk(2, dim=1)
|
| 63 |
+
y1, y2 = y.chunk(2, dim=1)
|
| 64 |
+
return x1 * y2, y1 * x2
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class DualStreamSeq(nn.Sequential):
|
| 68 |
+
def forward(self, x, y=None):
|
| 69 |
+
y = y if y is not None else x
|
| 70 |
+
for module in self:
|
| 71 |
+
x, y = module(x, y)
|
| 72 |
+
return x, y
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class DualStreamBlock(nn.Module):
|
| 76 |
+
def __init__(self, *args):
|
| 77 |
+
super(DualStreamBlock, self).__init__()
|
| 78 |
+
self.seq = nn.Sequential()
|
| 79 |
+
|
| 80 |
+
if len(args) == 1 and isinstance(args[0], OrderedDict):
|
| 81 |
+
for key, module in args[0].items():
|
| 82 |
+
self.seq.add_module(key, module)
|
| 83 |
+
else:
|
| 84 |
+
for idx, module in enumerate(args):
|
| 85 |
+
self.seq.add_module(str(idx), module)
|
| 86 |
+
|
| 87 |
+
def forward(self, x, y):
|
| 88 |
+
return self.seq(x), self.seq(y)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
class MuGIBlock(nn.Module):
|
| 92 |
+
def __init__(self, c, shared_b=False):
|
| 93 |
+
super().__init__()
|
| 94 |
+
self.block1 = DualStreamSeq(
|
| 95 |
+
DualStreamBlock(
|
| 96 |
+
LayerNorm2d(c),
|
| 97 |
+
nn.Conv2d(c, c * 2, 1),
|
| 98 |
+
nn.Conv2d(c * 2, c * 2, 3, padding=1, groups=c * 2)
|
| 99 |
+
),
|
| 100 |
+
DualStreamGate(),
|
| 101 |
+
DualStreamBlock(CABlock(c)),
|
| 102 |
+
DualStreamBlock(nn.Conv2d(c, c, 1))
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
self.a_l = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
|
| 106 |
+
self.a_r = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
|
| 107 |
+
|
| 108 |
+
self.block2 = DualStreamSeq(
|
| 109 |
+
DualStreamBlock(
|
| 110 |
+
LayerNorm2d(c),
|
| 111 |
+
nn.Conv2d(c, c * 2, 1)
|
| 112 |
+
),
|
| 113 |
+
DualStreamGate(),
|
| 114 |
+
DualStreamBlock(
|
| 115 |
+
nn.Conv2d(c, c, 1)
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
self.shared_b = shared_b
|
| 121 |
+
if shared_b:
|
| 122 |
+
self.b = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
|
| 123 |
+
else:
|
| 124 |
+
self.b_l = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
|
| 125 |
+
self.b_r = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
|
| 126 |
+
|
| 127 |
+
def forward(self, inp_l, inp_r):
|
| 128 |
+
x, y = self.block1(inp_l, inp_r)
|
| 129 |
+
x_skip, y_skip = inp_l + x * self.a_l, inp_r + y * self.a_r
|
| 130 |
+
x, y = self.block2(x_skip, y_skip)
|
| 131 |
+
if self.shared_b:
|
| 132 |
+
out_l, out_r = x_skip + x * self.b, y_skip + y * self.b
|
| 133 |
+
else:
|
| 134 |
+
out_l, out_r = x_skip + x * self.b_l, y_skip + y * self.b_r
|
| 135 |
+
return out_l, out_r
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
class FeaturePyramidVGG(nn.Module):
|
| 139 |
+
def __init__(self, out_channels=64, shared_b=False):
|
| 140 |
+
super().__init__()
|
| 141 |
+
self.device = 'cuda'
|
| 142 |
+
self.block5 = DualStreamSeq(
|
| 143 |
+
MuGIBlock(512, shared_b),
|
| 144 |
+
DualStreamBlock(nn.UpsamplingBilinear2d(scale_factor=2.0)),
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
self.block4 = DualStreamSeq(
|
| 148 |
+
MuGIBlock(512, shared_b)
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
self.ch_map4 = DualStreamSeq(
|
| 152 |
+
DualStreamBlock(
|
| 153 |
+
nn.Conv2d(in_channels=1024, out_channels=1024, kernel_size=1),
|
| 154 |
+
nn.PixelShuffle(2)),
|
| 155 |
+
MuGIBlock(256, shared_b)
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
self.block3 = DualStreamSeq(
|
| 159 |
+
MuGIBlock(256, shared_b)
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
self.ch_map3 = DualStreamSeq(
|
| 163 |
+
DualStreamBlock(
|
| 164 |
+
nn.Conv2d(in_channels=512, out_channels=512, kernel_size=1),
|
| 165 |
+
nn.PixelShuffle(2)),
|
| 166 |
+
MuGIBlock(128, shared_b)
|
| 167 |
+
)
|
| 168 |
+
|
| 169 |
+
self.block2 = DualStreamSeq(
|
| 170 |
+
MuGIBlock(128, shared_b)
|
| 171 |
+
)
|
| 172 |
+
|
| 173 |
+
self.ch_map2 = DualStreamSeq(
|
| 174 |
+
DualStreamBlock(
|
| 175 |
+
nn.Conv2d(in_channels=256, out_channels=256, kernel_size=1),
|
| 176 |
+
nn.PixelShuffle(2)),
|
| 177 |
+
MuGIBlock(64, shared_b)
|
| 178 |
+
)
|
| 179 |
+
|
| 180 |
+
self.block1 = DualStreamSeq(
|
| 181 |
+
MuGIBlock(64, shared_b),
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
self.ch_map1 = DualStreamSeq(
|
| 185 |
+
DualStreamBlock(nn.Conv2d(in_channels=128, out_channels=128, kernel_size=1)),
|
| 186 |
+
MuGIBlock(128, shared_b),
|
| 187 |
+
DualStreamBlock(nn.Conv2d(in_channels=128, out_channels=32, kernel_size=1)),
|
| 188 |
+
MuGIBlock(32, shared_b),
|
| 189 |
+
)
|
| 190 |
+
|
| 191 |
+
self.block_intro = DualStreamSeq(
|
| 192 |
+
DualStreamBlock(nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding=1)),
|
| 193 |
+
MuGIBlock(32, shared_b)
|
| 194 |
+
)
|
| 195 |
+
|
| 196 |
+
self.ch_map0 = DualStreamBlock(
|
| 197 |
+
nn.Conv2d(in_channels=64, out_channels=out_channels, kernel_size=1)
|
| 198 |
+
)
|
| 199 |
+
|
| 200 |
+
def forward(self, inp, vgg_feats):
|
| 201 |
+
# 64,128,256,512,512
|
| 202 |
+
vf1, vf2, vf3, vf4, vf5 = vgg_feats
|
| 203 |
+
# 512=>512,512=>512
|
| 204 |
+
f5_l, f5_r = self.block5(vf5)
|
| 205 |
+
f4_l, f4_r = self.block4(vf4)
|
| 206 |
+
f4_l, f4_r = self.ch_map4(torch.cat([f5_l, f4_l], dim=1), torch.cat([f5_r, f4_r], dim=1))
|
| 207 |
+
# 256 => 256
|
| 208 |
+
f3_l, f3_r = self.block3(vf3)
|
| 209 |
+
# (256+256,256+256)->(128,128)
|
| 210 |
+
f3_l, f3_r = self.ch_map3(torch.cat([f4_l, f3_l], dim=1), torch.cat([f4_r, f3_r], dim=1))
|
| 211 |
+
# (128+128,128+128)->(64,64)
|
| 212 |
+
f2_l, f2_r = self.block2(vf2)
|
| 213 |
+
f2_l, f2_r = self.ch_map2(torch.cat([f3_l, f2_l], dim=1), torch.cat([f3_r, f2_r], dim=1))
|
| 214 |
+
# (64+64,64+64)->(32,32)
|
| 215 |
+
f1_l, f1_r = self.block1(vf1)
|
| 216 |
+
f1_l, f1_r = self.ch_map1(torch.cat([f2_l, f1_l], dim=1), torch.cat([f2_r, f1_r], dim=1))
|
| 217 |
+
# 64
|
| 218 |
+
f0_l, f0_r = self.block_intro(inp, inp)
|
| 219 |
+
f0_l, f0_r = self.ch_map0(torch.cat([f1_l, f0_l], dim=1), torch.cat([f1_r, f0_r], dim=1))
|
| 220 |
+
return f0_l, f0_r
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
class DSRNet(nn.Module):
|
| 224 |
+
def __init__(self, in_channels=3, out_channels=3, width=48, middle_blk_num=1,
|
| 225 |
+
enc_blk_nums=[], dec_blk_nums=[], shared_b=False):
|
| 226 |
+
super().__init__()
|
| 227 |
+
self.intro = FeaturePyramidVGG(width, shared_b)
|
| 228 |
+
self.ending = DualStreamBlock(nn.Conv2d(width, out_channels, 3, padding=1))
|
| 229 |
+
self.encoders = nn.ModuleList()
|
| 230 |
+
self.decoders = nn.ModuleList()
|
| 231 |
+
self.middle_blks = nn.ModuleList()
|
| 232 |
+
self.ups = nn.ModuleList()
|
| 233 |
+
self.downs = nn.ModuleList()
|
| 234 |
+
self.lrm = LRM(width)
|
| 235 |
+
|
| 236 |
+
c = width
|
| 237 |
+
for num in enc_blk_nums:
|
| 238 |
+
self.encoders.append(
|
| 239 |
+
DualStreamSeq(
|
| 240 |
+
*[MuGIBlock(c, shared_b) for _ in range(num)]
|
| 241 |
+
)
|
| 242 |
+
)
|
| 243 |
+
self.downs.append(
|
| 244 |
+
DualStreamBlock(
|
| 245 |
+
nn.Conv2d(c, c * 2, 2, 2)
|
| 246 |
+
)
|
| 247 |
+
)
|
| 248 |
+
c *= 2
|
| 249 |
+
|
| 250 |
+
self.middle_blks = DualStreamSeq(
|
| 251 |
+
*[MuGIBlock(c, shared_b) for _ in range(middle_blk_num)]
|
| 252 |
+
)
|
| 253 |
+
|
| 254 |
+
for num in dec_blk_nums:
|
| 255 |
+
self.ups.append(
|
| 256 |
+
DualStreamBlock(
|
| 257 |
+
nn.Conv2d(c, c * 2, 1, bias=False),
|
| 258 |
+
nn.PixelShuffle(2)
|
| 259 |
+
)
|
| 260 |
+
)
|
| 261 |
+
c //= 2
|
| 262 |
+
|
| 263 |
+
self.decoders.append(
|
| 264 |
+
DualStreamSeq(
|
| 265 |
+
*[MuGIBlock(c, shared_b) for _ in range(num)]
|
| 266 |
+
)
|
| 267 |
+
)
|
| 268 |
+
|
| 269 |
+
def forward(self, inp, feats_inp, fn=None):
|
| 270 |
+
*_, H, W = inp.shape
|
| 271 |
+
x, y = self.intro(inp, feats_inp)
|
| 272 |
+
encs = []
|
| 273 |
+
for encoder, down in zip(self.encoders, self.downs):
|
| 274 |
+
x, y = encoder(x, y)
|
| 275 |
+
encs.append((x, y))
|
| 276 |
+
x, y = down(x, y)
|
| 277 |
+
|
| 278 |
+
x, y = self.middle_blks(x, y)
|
| 279 |
+
|
| 280 |
+
for decoder, up, (enc_x_skip, enc_y_skip) in zip(self.decoders, self.ups, encs[::-1]):
|
| 281 |
+
x, y = up(x, y)
|
| 282 |
+
x, y = x + enc_x_skip, y + enc_y_skip
|
| 283 |
+
x, y = decoder(x, y)
|
| 284 |
+
|
| 285 |
+
rr = self.lrm(x, y)
|
| 286 |
+
x, y = self.ending(x, y)
|
| 287 |
+
return x, y, rr
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
if __name__ == '__main__':
|
| 291 |
+
x = torch.ones(1, 3, 224, 224).cuda()
|
| 292 |
+
feats = [
|
| 293 |
+
torch.ones(1, 64, 224, 224).cuda(),
|
| 294 |
+
torch.ones(1, 128, 112, 112).cuda(),
|
| 295 |
+
torch.ones(1, 256, 56, 56).cuda(),
|
| 296 |
+
torch.ones(1, 512, 28, 28).cuda(),
|
| 297 |
+
torch.ones(1, 512, 14, 14).cuda(),
|
| 298 |
+
]
|
| 299 |
+
|
| 300 |
+
enc_blks = [2, 2, 2]
|
| 301 |
+
middle_blk_num = 4
|
| 302 |
+
dec_blks = [2, 2, 2]
|
| 303 |
+
model = DSRNet(3, 3, width=32, middle_blk_num=middle_blk_num,
|
| 304 |
+
enc_blk_nums=enc_blks, dec_blk_nums=dec_blks).cuda()
|
| 305 |
+
out_t, out_r, out_rr = model(x, feats)
|
| 306 |
+
print(out_t.shape, out_r.shape, out_rr.shape)
|
DSRNet/models/arch/lrm.py
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
from collections import OrderedDict
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class LayerNormFunction(torch.autograd.Function):
|
| 7 |
+
|
| 8 |
+
@staticmethod
|
| 9 |
+
def forward(ctx, x, weight, bias, eps):
|
| 10 |
+
ctx.eps = eps
|
| 11 |
+
N, C, H, W = x.size()
|
| 12 |
+
mu = x.mean(1, keepdim=True)
|
| 13 |
+
var = (x - mu).pow(2).mean(1, keepdim=True)
|
| 14 |
+
y = (x - mu) / (var + eps).sqrt()
|
| 15 |
+
ctx.save_for_backward(y, var, weight)
|
| 16 |
+
y = weight.view(1, C, 1, 1) * y + bias.view(1, C, 1, 1)
|
| 17 |
+
return y
|
| 18 |
+
|
| 19 |
+
@staticmethod
|
| 20 |
+
def backward(ctx, grad_output):
|
| 21 |
+
eps = ctx.eps
|
| 22 |
+
|
| 23 |
+
N, C, H, W = grad_output.size()
|
| 24 |
+
y, var, weight = ctx.saved_tensors
|
| 25 |
+
g = grad_output * weight.view(1, C, 1, 1)
|
| 26 |
+
mean_g = g.mean(dim=1, keepdim=True)
|
| 27 |
+
|
| 28 |
+
mean_gy = (g * y).mean(dim=1, keepdim=True)
|
| 29 |
+
gx = 1. / torch.sqrt(var + eps) * (g - y * mean_gy - mean_g)
|
| 30 |
+
return gx, (grad_output * y).sum(dim=3).sum(dim=2).sum(dim=0), grad_output.sum(dim=3).sum(dim=2).sum(
|
| 31 |
+
dim=0), None
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class LayerNorm2d(nn.Module):
|
| 35 |
+
|
| 36 |
+
def __init__(self, channels, eps=1e-6):
|
| 37 |
+
super(LayerNorm2d, self).__init__()
|
| 38 |
+
self.register_parameter('weight', nn.Parameter(torch.ones(channels)))
|
| 39 |
+
self.register_parameter('bias', nn.Parameter(torch.zeros(channels)))
|
| 40 |
+
self.eps = eps
|
| 41 |
+
|
| 42 |
+
def forward(self, x):
|
| 43 |
+
return LayerNormFunction.apply(x, self.weight, self.bias, self.eps)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class SimpleGate(nn.Module):
|
| 47 |
+
def forward(self, x):
|
| 48 |
+
x1, x2 = x.chunk(2, dim=1)
|
| 49 |
+
return x1 * x2
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class CABlock(nn.Module):
|
| 53 |
+
def __init__(self, channels):
|
| 54 |
+
super(CABlock, self).__init__()
|
| 55 |
+
self.ca = nn.Sequential(
|
| 56 |
+
nn.AdaptiveAvgPool2d(1),
|
| 57 |
+
nn.Conv2d(channels, channels, 1)
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
def forward(self, x):
|
| 61 |
+
return x * self.ca(x)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
class DualStreamGate(nn.Module):
|
| 65 |
+
def forward(self, x, y):
|
| 66 |
+
x1, x2 = x.chunk(2, dim=1)
|
| 67 |
+
y1, y2 = y.chunk(2, dim=1)
|
| 68 |
+
return x1 * y2, y1 * x2
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
class DualStreamSeq(nn.Sequential):
|
| 72 |
+
def forward(self, x, y=None):
|
| 73 |
+
y = y if y is not None else x
|
| 74 |
+
for module in self:
|
| 75 |
+
x, y = module(x, y)
|
| 76 |
+
return x, y
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
class DualStreamBlock(nn.Module):
|
| 80 |
+
def __init__(self, *args):
|
| 81 |
+
super(DualStreamBlock, self).__init__()
|
| 82 |
+
self.seq_l = nn.Sequential()
|
| 83 |
+
self.seq_r = nn.Sequential()
|
| 84 |
+
|
| 85 |
+
if len(args) == 1 and isinstance(args[0], OrderedDict):
|
| 86 |
+
for key, module in args[0].items():
|
| 87 |
+
self.seq_l.add_module(key, module)
|
| 88 |
+
self.seq_r.add_module(key, module)
|
| 89 |
+
else:
|
| 90 |
+
for idx, module in enumerate(args):
|
| 91 |
+
self.seq_l.add_module(str(idx), module)
|
| 92 |
+
self.seq_r.add_module(str(idx), module)
|
| 93 |
+
|
| 94 |
+
def forward(self, x, y):
|
| 95 |
+
return self.seq_l(x), self.seq_r(y)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
class R2Block(nn.Module):
|
| 99 |
+
def __init__(self, c):
|
| 100 |
+
super().__init__()
|
| 101 |
+
self.block1 = DualStreamSeq(
|
| 102 |
+
DualStreamBlock(
|
| 103 |
+
LayerNorm2d(c),
|
| 104 |
+
nn.Conv2d(c, c * 2, 1),
|
| 105 |
+
nn.Conv2d(c * 2, c * 2, 3, padding=1, groups=c * 2)
|
| 106 |
+
),
|
| 107 |
+
DualStreamGate(),
|
| 108 |
+
DualStreamBlock(CABlock(c)),
|
| 109 |
+
DualStreamBlock(nn.Conv2d(c, c, 1))
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
self.a_l = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
|
| 113 |
+
self.a_r = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
|
| 114 |
+
|
| 115 |
+
self.block2 = DualStreamSeq(
|
| 116 |
+
DualStreamBlock(
|
| 117 |
+
LayerNorm2d(c),
|
| 118 |
+
nn.Conv2d(c, c * 2, 1)
|
| 119 |
+
),
|
| 120 |
+
DualStreamGate(),
|
| 121 |
+
DualStreamBlock(
|
| 122 |
+
nn.Conv2d(c, c, 1)
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
)
|
| 126 |
+
|
| 127 |
+
self.b_l = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
|
| 128 |
+
self.b_r = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
|
| 129 |
+
|
| 130 |
+
def forward(self, inp_l, inp_r):
|
| 131 |
+
x, y = self.block1(inp_l, inp_r)
|
| 132 |
+
x_skip, y_skip = inp_l + x * self.a_l, inp_r + y * self.a_r
|
| 133 |
+
x, y = self.block2(x_skip, y_skip)
|
| 134 |
+
out_l, out_r = x_skip + x * self.b_l, y_skip + y * self.b_r
|
| 135 |
+
return out_l, out_r
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
class SinBlock(nn.Module):
|
| 139 |
+
def __init__(self, c):
|
| 140 |
+
super().__init__()
|
| 141 |
+
self.block1 = nn.Sequential(
|
| 142 |
+
LayerNorm2d(c),
|
| 143 |
+
nn.Conv2d(c, c * 2, 1),
|
| 144 |
+
nn.Conv2d(c * 2, c * 2, 3, padding=1, groups=c * 2),
|
| 145 |
+
SimpleGate(),
|
| 146 |
+
CABlock(c),
|
| 147 |
+
nn.Conv2d(c, c, 1)
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
self.block2 = nn.Sequential(
|
| 151 |
+
LayerNorm2d(c),
|
| 152 |
+
nn.Conv2d(c, c * 2, 1),
|
| 153 |
+
SimpleGate(),
|
| 154 |
+
nn.Conv2d(c, c, 1)
|
| 155 |
+
)
|
| 156 |
+
|
| 157 |
+
self.a = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
|
| 158 |
+
self.b = nn.Parameter(torch.zeros((1, c, 1, 1)), requires_grad=True)
|
| 159 |
+
|
| 160 |
+
def forward(self, inp):
|
| 161 |
+
x = self.block1(inp)
|
| 162 |
+
x_skip = inp + x * self.a
|
| 163 |
+
x = self.block2(x_skip)
|
| 164 |
+
out = x_skip + x * self.b
|
| 165 |
+
return out
|
| 166 |
+
|
| 167 |
+
class LRM(nn.Module):
|
| 168 |
+
def __init__(self, in_channels=48, num_blocks=[2, 4]):
|
| 169 |
+
super().__init__()
|
| 170 |
+
self.device = 'cuda'
|
| 171 |
+
channel = in_channels * 2
|
| 172 |
+
self.intro = DualStreamBlock(nn.Conv2d(in_channels, channel, 1))
|
| 173 |
+
self.blocks_inter = DualStreamSeq(*[R2Block(channel) for _ in range(num_blocks[0])])
|
| 174 |
+
self.blocks_merge = nn.Sequential(*[SinBlock(channel * 2) for _ in range(num_blocks[1])])
|
| 175 |
+
self.tail = nn.Sequential(
|
| 176 |
+
nn.Conv2d(channel * 2, 3, 3, padding=1),
|
| 177 |
+
nn.Tanh()
|
| 178 |
+
)
|
| 179 |
+
|
| 180 |
+
def forward(self, ft, fr):
|
| 181 |
+
ft, fr = self.intro(ft, fr)
|
| 182 |
+
ft, fr = self.blocks_inter(ft, fr)
|
| 183 |
+
fs = self.blocks_merge(torch.cat([ft, fr], dim=1))
|
| 184 |
+
out = self.tail(fs)
|
| 185 |
+
return out
|
DSRNet/models/arch/~/YTMT_US
ADDED
|
@@ -0,0 +1,1197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
digraph {
|
| 2 |
+
graph [size="153.75,153.75"]
|
| 3 |
+
node [align=left fontsize=12 height=0.2 ranksep=0.1 shape=box style=filled]
|
| 4 |
+
140291558534896 [label=MeanBackward0 fillcolor=darkolivegreen1]
|
| 5 |
+
140291558534952 -> 140291558534896
|
| 6 |
+
140291558534952 [label=AddBackward0]
|
| 7 |
+
140291558535064 -> 140291558534952
|
| 8 |
+
140291558535064 [label=SigmoidBackward]
|
| 9 |
+
140291558199424 -> 140291558535064
|
| 10 |
+
140291558199424 [label=MkldnnConvolutionBackward]
|
| 11 |
+
140291558199536 -> 140291558199424
|
| 12 |
+
140291558199536 [label=AddBackward0]
|
| 13 |
+
140291558199760 -> 140291558199536
|
| 14 |
+
140291558199760 [label=ReluBackward0]
|
| 15 |
+
140291558199928 -> 140291558199760
|
| 16 |
+
140291558199928 [label=MkldnnConvolutionBackward]
|
| 17 |
+
140291558200040 -> 140291558199928
|
| 18 |
+
140291558200040 [label=MulBackward0]
|
| 19 |
+
140291558200264 -> 140291558200040
|
| 20 |
+
140291558200264 [label=MulBackward0]
|
| 21 |
+
140291558200432 -> 140291558200264
|
| 22 |
+
140291558200432 [label=CatBackward]
|
| 23 |
+
140291558200600 -> 140291558200432
|
| 24 |
+
140291558200600 [label=AddBackward0]
|
| 25 |
+
140291558200768 -> 140291558200600
|
| 26 |
+
140291558200768 [label=ReluBackward0]
|
| 27 |
+
140291558200936 -> 140291558200768
|
| 28 |
+
140291558200936 [label=MkldnnConvolutionBackward]
|
| 29 |
+
140291558201048 -> 140291558200936
|
| 30 |
+
140291558201048 [label="
|
| 31 |
+
(64, 1475, 3, 3)" fillcolor=lightblue]
|
| 32 |
+
140291558201104 -> 140291558200936
|
| 33 |
+
140291558201104 [label="
|
| 34 |
+
(64)" fillcolor=lightblue]
|
| 35 |
+
140291558200824 -> 140291558200600
|
| 36 |
+
140291558200824 [label=SubBackward0]
|
| 37 |
+
140291558200992 -> 140291558200824
|
| 38 |
+
140291558200992 [label=MkldnnConvolutionBackward]
|
| 39 |
+
140291558201272 -> 140291558200992
|
| 40 |
+
140291558201272 [label="
|
| 41 |
+
(64, 1475, 3, 3)" fillcolor=lightblue]
|
| 42 |
+
140291558201384 -> 140291558200992
|
| 43 |
+
140291558201384 [label="
|
| 44 |
+
(64)" fillcolor=lightblue]
|
| 45 |
+
140291558201160 -> 140291558200824
|
| 46 |
+
140291558201160 [label=ReluBackward0]
|
| 47 |
+
140291558200992 -> 140291558201160
|
| 48 |
+
140291558200656 -> 140291558200432
|
| 49 |
+
140291558200656 [label=ConstantPadNdBackward]
|
| 50 |
+
140291558200880 -> 140291558200656
|
| 51 |
+
140291558200880 [label=UpsampleBilinear2DBackward1]
|
| 52 |
+
140291558201328 -> 140291558200880
|
| 53 |
+
140291558201328 [label=AddBackward0]
|
| 54 |
+
140291558201552 -> 140291558201328
|
| 55 |
+
140291558201552 [label=ReluBackward0]
|
| 56 |
+
140291558201720 -> 140291558201552
|
| 57 |
+
140291558201720 [label=MkldnnConvolutionBackward]
|
| 58 |
+
140291558201832 -> 140291558201720
|
| 59 |
+
140291558201832 [label=MulBackward0]
|
| 60 |
+
140291558202056 -> 140291558201832
|
| 61 |
+
140291558202056 [label=MulBackward0]
|
| 62 |
+
140291558202224 -> 140291558202056
|
| 63 |
+
140291558202224 [label=CatBackward]
|
| 64 |
+
140291558202392 -> 140291558202224
|
| 65 |
+
140291558202392 [label=AddBackward0]
|
| 66 |
+
140291558202560 -> 140291558202392
|
| 67 |
+
140291558202560 [label=ReluBackward0]
|
| 68 |
+
140291558202728 -> 140291558202560
|
| 69 |
+
140291558202728 [label=MkldnnConvolutionBackward]
|
| 70 |
+
140291558202840 -> 140291558202728
|
| 71 |
+
140291558202840 [label=MulBackward0]
|
| 72 |
+
140291558203064 -> 140291558202840
|
| 73 |
+
140291558203064 [label=MulBackward0]
|
| 74 |
+
140291558203232 -> 140291558203064
|
| 75 |
+
140291558203232 [label=MaxPool2DWithIndicesBackward]
|
| 76 |
+
140291558200600 -> 140291558203232
|
| 77 |
+
140291558203288 -> 140291558203064
|
| 78 |
+
140291558203288 [label=SigmoidBackward]
|
| 79 |
+
140291558211656 -> 140291558203288
|
| 80 |
+
140291558211656 [label=MkldnnConvolutionBackward]
|
| 81 |
+
140291558211768 -> 140291558211656
|
| 82 |
+
140291558211768 [label=ReluBackward1]
|
| 83 |
+
140291558211992 -> 140291558211768
|
| 84 |
+
140291558211992 [label=MkldnnConvolutionBackward]
|
| 85 |
+
140291558203232 -> 140291558211992
|
| 86 |
+
140291558212104 -> 140291558211992
|
| 87 |
+
140291558212104 [label="
|
| 88 |
+
(8, 64, 1, 1)" fillcolor=lightblue]
|
| 89 |
+
140291558212160 -> 140291558211992
|
| 90 |
+
140291558212160 [label="
|
| 91 |
+
(8)" fillcolor=lightblue]
|
| 92 |
+
140291558211824 -> 140291558211656
|
| 93 |
+
140291558211824 [label="
|
| 94 |
+
(1, 8, 1, 1)" fillcolor=lightblue]
|
| 95 |
+
140291558211880 -> 140291558211656
|
| 96 |
+
140291558211880 [label="
|
| 97 |
+
(1)" fillcolor=lightblue]
|
| 98 |
+
140291558203120 -> 140291558202840
|
| 99 |
+
140291558203120 [label=SigmoidBackward]
|
| 100 |
+
140291558203344 -> 140291558203120
|
| 101 |
+
140291558203344 [label=ThnnConv2DBackward]
|
| 102 |
+
140291558211936 -> 140291558203344
|
| 103 |
+
140291558211936 [label=ReluBackward1]
|
| 104 |
+
140291558212328 -> 140291558211936
|
| 105 |
+
140291558212328 [label=ThnnConv2DBackward]
|
| 106 |
+
140291558212440 -> 140291558212328
|
| 107 |
+
140291558212440 [label=MeanBackward1]
|
| 108 |
+
140291558203064 -> 140291558212440
|
| 109 |
+
140291558212496 -> 140291558212328
|
| 110 |
+
140291558212496 [label="
|
| 111 |
+
(8, 64, 1, 1)" fillcolor=lightblue]
|
| 112 |
+
140291558212552 -> 140291558212328
|
| 113 |
+
140291558212552 [label="
|
| 114 |
+
(8)" fillcolor=lightblue]
|
| 115 |
+
140291558212216 -> 140291558203344
|
| 116 |
+
140291558212216 [label="
|
| 117 |
+
(64, 8, 1, 1)" fillcolor=lightblue]
|
| 118 |
+
140291558212048 -> 140291558203344
|
| 119 |
+
140291558212048 [label="
|
| 120 |
+
(64)" fillcolor=lightblue]
|
| 121 |
+
140291558202896 -> 140291558202728
|
| 122 |
+
140291558202896 [label="
|
| 123 |
+
(128, 64, 3, 3)" fillcolor=lightblue]
|
| 124 |
+
140291558202952 -> 140291558202728
|
| 125 |
+
140291558202952 [label="
|
| 126 |
+
(128)" fillcolor=lightblue]
|
| 127 |
+
140291558202616 -> 140291558202392
|
| 128 |
+
140291558202616 [label=SubBackward0]
|
| 129 |
+
140291558202784 -> 140291558202616
|
| 130 |
+
140291558202784 [label=MkldnnConvolutionBackward]
|
| 131 |
+
140291558211712 -> 140291558202784
|
| 132 |
+
140291558211712 [label=MulBackward0]
|
| 133 |
+
140291558212664 -> 140291558211712
|
| 134 |
+
140291558212664 [label=MulBackward0]
|
| 135 |
+
140291558212832 -> 140291558212664
|
| 136 |
+
140291558212832 [label=MaxPool2DWithIndicesBackward]
|
| 137 |
+
140291558213000 -> 140291558212832
|
| 138 |
+
140291558213000 [label=AddBackward0]
|
| 139 |
+
140291558213112 -> 140291558213000
|
| 140 |
+
140291558213112 [label=ReluBackward0]
|
| 141 |
+
140291558200992 -> 140291558213112
|
| 142 |
+
140291558213168 -> 140291558213000
|
| 143 |
+
140291558213168 [label=SubBackward0]
|
| 144 |
+
140291558200936 -> 140291558213168
|
| 145 |
+
140291558213280 -> 140291558213168
|
| 146 |
+
140291558213280 [label=ReluBackward0]
|
| 147 |
+
140291558200936 -> 140291558213280
|
| 148 |
+
140291558212888 -> 140291558212664
|
| 149 |
+
140291558212888 [label=SigmoidBackward]
|
| 150 |
+
140291558213056 -> 140291558212888
|
| 151 |
+
140291558213056 [label=MkldnnConvolutionBackward]
|
| 152 |
+
140291558213336 -> 140291558213056
|
| 153 |
+
140291558213336 [label=ReluBackward1]
|
| 154 |
+
140291558213560 -> 140291558213336
|
| 155 |
+
140291558213560 [label=MkldnnConvolutionBackward]
|
| 156 |
+
140291558212832 -> 140291558213560
|
| 157 |
+
140291558213672 -> 140291558213560
|
| 158 |
+
140291558213672 [label="
|
| 159 |
+
(8, 64, 1, 1)" fillcolor=lightblue]
|
| 160 |
+
140291558213728 -> 140291558213560
|
| 161 |
+
140291558213728 [label="
|
| 162 |
+
(8)" fillcolor=lightblue]
|
| 163 |
+
140291558213392 -> 140291558213056
|
| 164 |
+
140291558213392 [label="
|
| 165 |
+
(1, 8, 1, 1)" fillcolor=lightblue]
|
| 166 |
+
140291558213448 -> 140291558213056
|
| 167 |
+
140291558213448 [label="
|
| 168 |
+
(1)" fillcolor=lightblue]
|
| 169 |
+
140291558212720 -> 140291558211712
|
| 170 |
+
140291558212720 [label=SigmoidBackward]
|
| 171 |
+
140291558212944 -> 140291558212720
|
| 172 |
+
140291558212944 [label=ThnnConv2DBackward]
|
| 173 |
+
140291558213504 -> 140291558212944
|
| 174 |
+
140291558213504 [label=ReluBackward1]
|
| 175 |
+
140291558213896 -> 140291558213504
|
| 176 |
+
140291558213896 [label=ThnnConv2DBackward]
|
| 177 |
+
140291558214008 -> 140291558213896
|
| 178 |
+
140291558214008 [label=MeanBackward1]
|
| 179 |
+
140291558212664 -> 140291558214008
|
| 180 |
+
140291558214064 -> 140291558213896
|
| 181 |
+
140291558214064 [label="
|
| 182 |
+
(8, 64, 1, 1)" fillcolor=lightblue]
|
| 183 |
+
140291558214120 -> 140291558213896
|
| 184 |
+
140291558214120 [label="
|
| 185 |
+
(8)" fillcolor=lightblue]
|
| 186 |
+
140291558213784 -> 140291558212944
|
| 187 |
+
140291558213784 [label="
|
| 188 |
+
(64, 8, 1, 1)" fillcolor=lightblue]
|
| 189 |
+
140291558213616 -> 140291558212944
|
| 190 |
+
140291558213616 [label="
|
| 191 |
+
(64)" fillcolor=lightblue]
|
| 192 |
+
140291558212272 -> 140291558202784
|
| 193 |
+
140291558212272 [label="
|
| 194 |
+
(128, 64, 3, 3)" fillcolor=lightblue]
|
| 195 |
+
140291558212384 -> 140291558202784
|
| 196 |
+
140291558212384 [label="
|
| 197 |
+
(128)" fillcolor=lightblue]
|
| 198 |
+
140291558203008 -> 140291558202616
|
| 199 |
+
140291558203008 [label=ReluBackward0]
|
| 200 |
+
140291558202784 -> 140291558203008
|
| 201 |
+
140291558202448 -> 140291558202224
|
| 202 |
+
140291558202448 [label=ConstantPadNdBackward]
|
| 203 |
+
140291558202672 -> 140291558202448
|
| 204 |
+
140291558202672 [label=UpsampleBilinear2DBackward1]
|
| 205 |
+
140291558212608 -> 140291558202672
|
| 206 |
+
140291558212608 [label=AddBackward0]
|
| 207 |
+
140291558213224 -> 140291558212608
|
| 208 |
+
140291558213224 [label=ReluBackward0]
|
| 209 |
+
140291558214176 -> 140291558213224
|
| 210 |
+
140291558214176 [label=MkldnnConvolutionBackward]
|
| 211 |
+
140291558214288 -> 140291558214176
|
| 212 |
+
140291558214288 [label=MulBackward0]
|
| 213 |
+
140291558214512 -> 140291558214288
|
| 214 |
+
140291558214512 [label=MulBackward0]
|
| 215 |
+
140291558214680 -> 140291558214512
|
| 216 |
+
140291558214680 [label=CatBackward]
|
| 217 |
+
140291558214848 -> 140291558214680
|
| 218 |
+
140291558214848 [label=AddBackward0]
|
| 219 |
+
140291558215016 -> 140291558214848
|
| 220 |
+
140291558215016 [label=ReluBackward0]
|
| 221 |
+
140291558215184 -> 140291558215016
|
| 222 |
+
140291558215184 [label=MkldnnConvolutionBackward]
|
| 223 |
+
140291558215296 -> 140291558215184
|
| 224 |
+
140291558215296 [label=MulBackward0]
|
| 225 |
+
140291558215520 -> 140291558215296
|
| 226 |
+
140291558215520 [label=MulBackward0]
|
| 227 |
+
140291558232136 -> 140291558215520
|
| 228 |
+
140291558232136 [label=MaxPool2DWithIndicesBackward]
|
| 229 |
+
140291558202392 -> 140291558232136
|
| 230 |
+
140291558232192 -> 140291558215520
|
| 231 |
+
140291558232192 [label=SigmoidBackward]
|
| 232 |
+
140291558232304 -> 140291558232192
|
| 233 |
+
140291558232304 [label=MkldnnConvolutionBackward]
|
| 234 |
+
140291558232472 -> 140291558232304
|
| 235 |
+
140291558232472 [label=ReluBackward1]
|
| 236 |
+
140291558232696 -> 140291558232472
|
| 237 |
+
140291558232696 [label=MkldnnConvolutionBackward]
|
| 238 |
+
140291558232136 -> 140291558232696
|
| 239 |
+
140291558232864 -> 140291558232696
|
| 240 |
+
140291558232864 [label="
|
| 241 |
+
(16, 128, 1, 1)" fillcolor=lightblue]
|
| 242 |
+
140291558232920 -> 140291558232696
|
| 243 |
+
140291558232920 [label="
|
| 244 |
+
(16)" fillcolor=lightblue]
|
| 245 |
+
140291558232528 -> 140291558232304
|
| 246 |
+
140291558232528 [label="
|
| 247 |
+
(1, 16, 1, 1)" fillcolor=lightblue]
|
| 248 |
+
140291558232584 -> 140291558232304
|
| 249 |
+
140291558232584 [label="
|
| 250 |
+
(1)" fillcolor=lightblue]
|
| 251 |
+
140291558215576 -> 140291558215296
|
| 252 |
+
140291558215576 [label=SigmoidBackward]
|
| 253 |
+
140291558232248 -> 140291558215576
|
| 254 |
+
140291558232248 [label=ThnnConv2DBackward]
|
| 255 |
+
140291558232640 -> 140291558232248
|
| 256 |
+
140291558232640 [label=ReluBackward1]
|
| 257 |
+
140291558233088 -> 140291558232640
|
| 258 |
+
140291558233088 [label=ThnnConv2DBackward]
|
| 259 |
+
140291558233256 -> 140291558233088
|
| 260 |
+
140291558233256 [label=MeanBackward1]
|
| 261 |
+
140291558215520 -> 140291558233256
|
| 262 |
+
140291558233312 -> 140291558233088
|
| 263 |
+
140291558233312 [label="
|
| 264 |
+
(16, 128, 1, 1)" fillcolor=lightblue]
|
| 265 |
+
140291558233368 -> 140291558233088
|
| 266 |
+
140291558233368 [label="
|
| 267 |
+
(16)" fillcolor=lightblue]
|
| 268 |
+
140291558232752 -> 140291558232248
|
| 269 |
+
140291558232752 [label="
|
| 270 |
+
(128, 16, 1, 1)" fillcolor=lightblue]
|
| 271 |
+
140291558232976 -> 140291558232248
|
| 272 |
+
140291558232976 [label="
|
| 273 |
+
(128)" fillcolor=lightblue]
|
| 274 |
+
140291558215352 -> 140291558215184
|
| 275 |
+
140291558215352 [label="
|
| 276 |
+
(256, 128, 3, 3)" fillcolor=lightblue]
|
| 277 |
+
140291558215408 -> 140291558215184
|
| 278 |
+
140291558215408 [label="
|
| 279 |
+
(256)" fillcolor=lightblue]
|
| 280 |
+
140291558215072 -> 140291558214848
|
| 281 |
+
140291558215072 [label=SubBackward0]
|
| 282 |
+
140291558215240 -> 140291558215072
|
| 283 |
+
140291558215240 [label=MkldnnConvolutionBackward]
|
| 284 |
+
140291558232416 -> 140291558215240
|
| 285 |
+
140291558232416 [label=MulBackward0]
|
| 286 |
+
140291558233200 -> 140291558232416
|
| 287 |
+
140291558233200 [label=MulBackward0]
|
| 288 |
+
140291558233648 -> 140291558233200
|
| 289 |
+
140291558233648 [label=MaxPool2DWithIndicesBackward]
|
| 290 |
+
140291558233816 -> 140291558233648
|
| 291 |
+
140291558233816 [label=AddBackward0]
|
| 292 |
+
140291558233928 -> 140291558233816
|
| 293 |
+
140291558233928 [label=ReluBackward0]
|
| 294 |
+
140291558202784 -> 140291558233928
|
| 295 |
+
140291558233984 -> 140291558233816
|
| 296 |
+
140291558233984 [label=SubBackward0]
|
| 297 |
+
140291558202728 -> 140291558233984
|
| 298 |
+
140291558234096 -> 140291558233984
|
| 299 |
+
140291558234096 [label=ReluBackward0]
|
| 300 |
+
140291558202728 -> 140291558234096
|
| 301 |
+
140291558233704 -> 140291558233200
|
| 302 |
+
140291558233704 [label=SigmoidBackward]
|
| 303 |
+
140291558233872 -> 140291558233704
|
| 304 |
+
140291558233872 [label=MkldnnConvolutionBackward]
|
| 305 |
+
140291558234152 -> 140291558233872
|
| 306 |
+
140291558234152 [label=ReluBackward1]
|
| 307 |
+
140291558234376 -> 140291558234152
|
| 308 |
+
140291558234376 [label=MkldnnConvolutionBackward]
|
| 309 |
+
140291558233648 -> 140291558234376
|
| 310 |
+
140291558234488 -> 140291558234376
|
| 311 |
+
140291558234488 [label="
|
| 312 |
+
(16, 128, 1, 1)" fillcolor=lightblue]
|
| 313 |
+
140291558234544 -> 140291558234376
|
| 314 |
+
140291558234544 [label="
|
| 315 |
+
(16)" fillcolor=lightblue]
|
| 316 |
+
140291558234208 -> 140291558233872
|
| 317 |
+
140291558234208 [label="
|
| 318 |
+
(1, 16, 1, 1)" fillcolor=lightblue]
|
| 319 |
+
140291558234264 -> 140291558233872
|
| 320 |
+
140291558234264 [label="
|
| 321 |
+
(1)" fillcolor=lightblue]
|
| 322 |
+
140291558233536 -> 140291558232416
|
| 323 |
+
140291558233536 [label=SigmoidBackward]
|
| 324 |
+
140291558233760 -> 140291558233536
|
| 325 |
+
140291558233760 [label=ThnnConv2DBackward]
|
| 326 |
+
140291558234320 -> 140291558233760
|
| 327 |
+
140291558234320 [label=ReluBackward1]
|
| 328 |
+
140291558234712 -> 140291558234320
|
| 329 |
+
140291558234712 [label=ThnnConv2DBackward]
|
| 330 |
+
140291558234824 -> 140291558234712
|
| 331 |
+
140291558234824 [label=MeanBackward1]
|
| 332 |
+
140291558233200 -> 140291558234824
|
| 333 |
+
140291558234880 -> 140291558234712
|
| 334 |
+
140291558234880 [label="
|
| 335 |
+
(16, 128, 1, 1)" fillcolor=lightblue]
|
| 336 |
+
140291558234936 -> 140291558234712
|
| 337 |
+
140291558234936 [label="
|
| 338 |
+
(16)" fillcolor=lightblue]
|
| 339 |
+
140291558234432 -> 140291558233760
|
| 340 |
+
140291558234432 [label="
|
| 341 |
+
(128, 16, 1, 1)" fillcolor=lightblue]
|
| 342 |
+
140291558234600 -> 140291558233760
|
| 343 |
+
140291558234600 [label="
|
| 344 |
+
(128)" fillcolor=lightblue]
|
| 345 |
+
140291558233032 -> 140291558215240
|
| 346 |
+
140291558233032 [label="
|
| 347 |
+
(256, 128, 3, 3)" fillcolor=lightblue]
|
| 348 |
+
140291558233592 -> 140291558215240
|
| 349 |
+
140291558233592 [label="
|
| 350 |
+
(256)" fillcolor=lightblue]
|
| 351 |
+
140291558215464 -> 140291558215072
|
| 352 |
+
140291558215464 [label=ReluBackward0]
|
| 353 |
+
140291558215240 -> 140291558215464
|
| 354 |
+
140291558214904 -> 140291558214680
|
| 355 |
+
140291558214904 [label=ConstantPadNdBackward]
|
| 356 |
+
140291558215128 -> 140291558214904
|
| 357 |
+
140291558215128 [label=UpsampleBilinear2DBackward1]
|
| 358 |
+
140291558233424 -> 140291558215128
|
| 359 |
+
140291558233424 [label=AddBackward0]
|
| 360 |
+
140291558234040 -> 140291558233424
|
| 361 |
+
140291558234040 [label=ReluBackward0]
|
| 362 |
+
140291558234992 -> 140291558234040
|
| 363 |
+
140291558234992 [label=MkldnnConvolutionBackward]
|
| 364 |
+
140291558235104 -> 140291558234992
|
| 365 |
+
140291558235104 [label=MulBackward0]
|
| 366 |
+
140291558235328 -> 140291558235104
|
| 367 |
+
140291558235328 [label=MulBackward0]
|
| 368 |
+
140291558235496 -> 140291558235328
|
| 369 |
+
140291558235496 [label=CatBackward]
|
| 370 |
+
140291558235664 -> 140291558235496
|
| 371 |
+
140291558235664 [label=AddBackward0]
|
| 372 |
+
140291558235832 -> 140291558235664
|
| 373 |
+
140291558235832 [label=ReluBackward0]
|
| 374 |
+
140291558236000 -> 140291558235832
|
| 375 |
+
140291558236000 [label=MkldnnConvolutionBackward]
|
| 376 |
+
140291558236112 -> 140291558236000
|
| 377 |
+
140291558236112 [label=MulBackward0]
|
| 378 |
+
140291558248688 -> 140291558236112
|
| 379 |
+
140291558248688 [label=MulBackward0]
|
| 380 |
+
140291558248856 -> 140291558248688
|
| 381 |
+
140291558248856 [label=MaxPool2DWithIndicesBackward]
|
| 382 |
+
140291558214848 -> 140291558248856
|
| 383 |
+
140291558248912 -> 140291558248688
|
| 384 |
+
140291558248912 [label=SigmoidBackward]
|
| 385 |
+
140291558249024 -> 140291558248912
|
| 386 |
+
140291558249024 [label=MkldnnConvolutionBackward]
|
| 387 |
+
140291558249192 -> 140291558249024
|
| 388 |
+
140291558249192 [label=ReluBackward1]
|
| 389 |
+
140291558249416 -> 140291558249192
|
| 390 |
+
140291558249416 [label=MkldnnConvolutionBackward]
|
| 391 |
+
140291558248856 -> 140291558249416
|
| 392 |
+
140291558249584 -> 140291558249416
|
| 393 |
+
140291558249584 [label="
|
| 394 |
+
(32, 256, 1, 1)" fillcolor=lightblue]
|
| 395 |
+
140291558249640 -> 140291558249416
|
| 396 |
+
140291558249640 [label="
|
| 397 |
+
(32)" fillcolor=lightblue]
|
| 398 |
+
140291558249248 -> 140291558249024
|
| 399 |
+
140291558249248 [label="
|
| 400 |
+
(1, 32, 1, 1)" fillcolor=lightblue]
|
| 401 |
+
140291558249304 -> 140291558249024
|
| 402 |
+
140291558249304 [label="
|
| 403 |
+
(1)" fillcolor=lightblue]
|
| 404 |
+
140291558248744 -> 140291558236112
|
| 405 |
+
140291558248744 [label=SigmoidBackward]
|
| 406 |
+
140291558248968 -> 140291558248744
|
| 407 |
+
140291558248968 [label=ThnnConv2DBackward]
|
| 408 |
+
140291558249360 -> 140291558248968
|
| 409 |
+
140291558249360 [label=ReluBackward1]
|
| 410 |
+
140291558249808 -> 140291558249360
|
| 411 |
+
140291558249808 [label=ThnnConv2DBackward]
|
| 412 |
+
140291558249976 -> 140291558249808
|
| 413 |
+
140291558249976 [label=MeanBackward1]
|
| 414 |
+
140291558248688 -> 140291558249976
|
| 415 |
+
140291558250032 -> 140291558249808
|
| 416 |
+
140291558250032 [label="
|
| 417 |
+
(32, 256, 1, 1)" fillcolor=lightblue]
|
| 418 |
+
140291558250088 -> 140291558249808
|
| 419 |
+
140291558250088 [label="
|
| 420 |
+
(32)" fillcolor=lightblue]
|
| 421 |
+
140291558249472 -> 140291558248968
|
| 422 |
+
140291558249472 [label="
|
| 423 |
+
(256, 32, 1, 1)" fillcolor=lightblue]
|
| 424 |
+
140291558249696 -> 140291558248968
|
| 425 |
+
140291558249696 [label="
|
| 426 |
+
(256)" fillcolor=lightblue]
|
| 427 |
+
140291558248520 -> 140291558236000
|
| 428 |
+
140291558248520 [label="
|
| 429 |
+
(512, 256, 3, 3)" fillcolor=lightblue]
|
| 430 |
+
140291558248576 -> 140291558236000
|
| 431 |
+
140291558248576 [label="
|
| 432 |
+
(512)" fillcolor=lightblue]
|
| 433 |
+
140291558235888 -> 140291558235664
|
| 434 |
+
140291558235888 [label=SubBackward0]
|
| 435 |
+
140291558236056 -> 140291558235888
|
| 436 |
+
140291558236056 [label=MkldnnConvolutionBackward]
|
| 437 |
+
140291558249136 -> 140291558236056
|
| 438 |
+
140291558249136 [label=MulBackward0]
|
| 439 |
+
140291558249920 -> 140291558249136
|
| 440 |
+
140291558249920 [label=MulBackward0]
|
| 441 |
+
140291558250368 -> 140291558249920
|
| 442 |
+
140291558250368 [label=MaxPool2DWithIndicesBackward]
|
| 443 |
+
140291558250536 -> 140291558250368
|
| 444 |
+
140291558250536 [label=AddBackward0]
|
| 445 |
+
140291558250648 -> 140291558250536
|
| 446 |
+
140291558250648 [label=ReluBackward0]
|
| 447 |
+
140291558215240 -> 140291558250648
|
| 448 |
+
140291558250704 -> 140291558250536
|
| 449 |
+
140291558250704 [label=SubBackward0]
|
| 450 |
+
140291558215184 -> 140291558250704
|
| 451 |
+
140291558250816 -> 140291558250704
|
| 452 |
+
140291558250816 [label=ReluBackward0]
|
| 453 |
+
140291558215184 -> 140291558250816
|
| 454 |
+
140291558250424 -> 140291558249920
|
| 455 |
+
140291558250424 [label=SigmoidBackward]
|
| 456 |
+
140291558250592 -> 140291558250424
|
| 457 |
+
140291558250592 [label=MkldnnConvolutionBackward]
|
| 458 |
+
140291558250872 -> 140291558250592
|
| 459 |
+
140291558250872 [label=ReluBackward1]
|
| 460 |
+
140291558251096 -> 140291558250872
|
| 461 |
+
140291558251096 [label=MkldnnConvolutionBackward]
|
| 462 |
+
140291558250368 -> 140291558251096
|
| 463 |
+
140291558251208 -> 140291558251096
|
| 464 |
+
140291558251208 [label="
|
| 465 |
+
(32, 256, 1, 1)" fillcolor=lightblue]
|
| 466 |
+
140291558251264 -> 140291558251096
|
| 467 |
+
140291558251264 [label="
|
| 468 |
+
(32)" fillcolor=lightblue]
|
| 469 |
+
140291558250928 -> 140291558250592
|
| 470 |
+
140291558250928 [label="
|
| 471 |
+
(1, 32, 1, 1)" fillcolor=lightblue]
|
| 472 |
+
140291558250984 -> 140291558250592
|
| 473 |
+
140291558250984 [label="
|
| 474 |
+
(1)" fillcolor=lightblue]
|
| 475 |
+
140291558250256 -> 140291558249136
|
| 476 |
+
140291558250256 [label=SigmoidBackward]
|
| 477 |
+
140291558250480 -> 140291558250256
|
| 478 |
+
140291558250480 [label=ThnnConv2DBackward]
|
| 479 |
+
140291558251040 -> 140291558250480
|
| 480 |
+
140291558251040 [label=ReluBackward1]
|
| 481 |
+
140291558251432 -> 140291558251040
|
| 482 |
+
140291558251432 [label=ThnnConv2DBackward]
|
| 483 |
+
140291558251544 -> 140291558251432
|
| 484 |
+
140291558251544 [label=MeanBackward1]
|
| 485 |
+
140291558249920 -> 140291558251544
|
| 486 |
+
140291558251600 -> 140291558251432
|
| 487 |
+
140291558251600 [label="
|
| 488 |
+
(32, 256, 1, 1)" fillcolor=lightblue]
|
| 489 |
+
140291558251656 -> 140291558251432
|
| 490 |
+
140291558251656 [label="
|
| 491 |
+
(32)" fillcolor=lightblue]
|
| 492 |
+
140291558251152 -> 140291558250480
|
| 493 |
+
140291558251152 [label="
|
| 494 |
+
(256, 32, 1, 1)" fillcolor=lightblue]
|
| 495 |
+
140291558251320 -> 140291558250480
|
| 496 |
+
140291558251320 [label="
|
| 497 |
+
(256)" fillcolor=lightblue]
|
| 498 |
+
140291558248800 -> 140291558236056
|
| 499 |
+
140291558248800 [label="
|
| 500 |
+
(512, 256, 3, 3)" fillcolor=lightblue]
|
| 501 |
+
140291558250312 -> 140291558236056
|
| 502 |
+
140291558250312 [label="
|
| 503 |
+
(512)" fillcolor=lightblue]
|
| 504 |
+
140291558248632 -> 140291558235888
|
| 505 |
+
140291558248632 [label=ReluBackward0]
|
| 506 |
+
140291558236056 -> 140291558248632
|
| 507 |
+
140291558235720 -> 140291558235496
|
| 508 |
+
140291558235720 [label=ConstantPadNdBackward]
|
| 509 |
+
140291558235944 -> 140291558235720
|
| 510 |
+
140291558235944 [label=UpsampleBilinear2DBackward1]
|
| 511 |
+
140291558250144 -> 140291558235944
|
| 512 |
+
140291558250144 [label=AddBackward0]
|
| 513 |
+
140291558250760 -> 140291558250144
|
| 514 |
+
140291558250760 [label=AddBackward0]
|
| 515 |
+
140291558251712 -> 140291558250760
|
| 516 |
+
140291558251712 [label=ReluBackward0]
|
| 517 |
+
140291558251768 -> 140291558251712
|
| 518 |
+
140291558251768 [label=MkldnnConvolutionBackward]
|
| 519 |
+
140291558251992 -> 140291558251768
|
| 520 |
+
140291558251992 [label=MulBackward0]
|
| 521 |
+
140291558252216 -> 140291558251992
|
| 522 |
+
140291558252216 [label=MulBackward0]
|
| 523 |
+
140291558250200 -> 140291558252216
|
| 524 |
+
140291558250200 [label=MaxPool2DWithIndicesBackward]
|
| 525 |
+
140291558235664 -> 140291558250200
|
| 526 |
+
140291558252384 -> 140291558252216
|
| 527 |
+
140291558252384 [label=SigmoidBackward]
|
| 528 |
+
140291558252496 -> 140291558252384
|
| 529 |
+
140291558252496 [label=ThnnConv2DBackward]
|
| 530 |
+
140291558269056 -> 140291558252496
|
| 531 |
+
140291558269056 [label=ReluBackward1]
|
| 532 |
+
140291558269280 -> 140291558269056
|
| 533 |
+
140291558269280 [label=MkldnnConvolutionBackward]
|
| 534 |
+
140291558250200 -> 140291558269280
|
| 535 |
+
140291558269448 -> 140291558269280
|
| 536 |
+
140291558269448 [label="
|
| 537 |
+
(64, 512, 1, 1)" fillcolor=lightblue]
|
| 538 |
+
140291558269504 -> 140291558269280
|
| 539 |
+
140291558269504 [label="
|
| 540 |
+
(64)" fillcolor=lightblue]
|
| 541 |
+
140291558269112 -> 140291558252496
|
| 542 |
+
140291558269112 [label="
|
| 543 |
+
(1, 64, 1, 1)" fillcolor=lightblue]
|
| 544 |
+
140291558269168 -> 140291558252496
|
| 545 |
+
140291558269168 [label="
|
| 546 |
+
(1)" fillcolor=lightblue]
|
| 547 |
+
140291558252272 -> 140291558251992
|
| 548 |
+
140291558252272 [label=SigmoidBackward]
|
| 549 |
+
140291558252440 -> 140291558252272
|
| 550 |
+
140291558252440 [label=ThnnConv2DBackward]
|
| 551 |
+
140291558269224 -> 140291558252440
|
| 552 |
+
140291558269224 [label=ReluBackward1]
|
| 553 |
+
140291558269672 -> 140291558269224
|
| 554 |
+
140291558269672 [label=ThnnConv2DBackward]
|
| 555 |
+
140291558269840 -> 140291558269672
|
| 556 |
+
140291558269840 [label=MeanBackward1]
|
| 557 |
+
140291558252216 -> 140291558269840
|
| 558 |
+
140291558269896 -> 140291558269672
|
| 559 |
+
140291558269896 [label="
|
| 560 |
+
(64, 512, 1, 1)" fillcolor=lightblue]
|
| 561 |
+
140291558269952 -> 140291558269672
|
| 562 |
+
140291558269952 [label="
|
| 563 |
+
(64)" fillcolor=lightblue]
|
| 564 |
+
140291558269336 -> 140291558252440
|
| 565 |
+
140291558269336 [label="
|
| 566 |
+
(512, 64, 1, 1)" fillcolor=lightblue]
|
| 567 |
+
140291558269560 -> 140291558252440
|
| 568 |
+
140291558269560 [label="
|
| 569 |
+
(512)" fillcolor=lightblue]
|
| 570 |
+
140291558252048 -> 140291558251768
|
| 571 |
+
140291558252048 [label="
|
| 572 |
+
(512, 512, 3, 3)" fillcolor=lightblue]
|
| 573 |
+
140291558252104 -> 140291558251768
|
| 574 |
+
140291558252104 [label="
|
| 575 |
+
(512)" fillcolor=lightblue]
|
| 576 |
+
140291558251376 -> 140291558250760
|
| 577 |
+
140291558251376 [label=SubBackward0]
|
| 578 |
+
140291558251936 -> 140291558251376
|
| 579 |
+
140291558251936 [label=MkldnnConvolutionBackward]
|
| 580 |
+
140291558269000 -> 140291558251936
|
| 581 |
+
140291558269000 [label=MulBackward0]
|
| 582 |
+
140291558269784 -> 140291558269000
|
| 583 |
+
140291558269784 [label=MulBackward0]
|
| 584 |
+
140291558270232 -> 140291558269784
|
| 585 |
+
140291558270232 [label=MaxPool2DWithIndicesBackward]
|
| 586 |
+
140291558270400 -> 140291558270232
|
| 587 |
+
140291558270400 [label=AddBackward0]
|
| 588 |
+
140291558270512 -> 140291558270400
|
| 589 |
+
140291558270512 [label=ReluBackward0]
|
| 590 |
+
140291558236056 -> 140291558270512
|
| 591 |
+
140291558270568 -> 140291558270400
|
| 592 |
+
140291558270568 [label=SubBackward0]
|
| 593 |
+
140291558236000 -> 140291558270568
|
| 594 |
+
140291558270680 -> 140291558270568
|
| 595 |
+
140291558270680 [label=ReluBackward0]
|
| 596 |
+
140291558236000 -> 140291558270680
|
| 597 |
+
140291558270288 -> 140291558269784
|
| 598 |
+
140291558270288 [label=SigmoidBackward]
|
| 599 |
+
140291558270456 -> 140291558270288
|
| 600 |
+
140291558270456 [label=ThnnConv2DBackward]
|
| 601 |
+
140291558270736 -> 140291558270456
|
| 602 |
+
140291558270736 [label=ReluBackward1]
|
| 603 |
+
140291558270960 -> 140291558270736
|
| 604 |
+
140291558270960 [label=MkldnnConvolutionBackward]
|
| 605 |
+
140291558270232 -> 140291558270960
|
| 606 |
+
140291558271072 -> 140291558270960
|
| 607 |
+
140291558271072 [label="
|
| 608 |
+
(64, 512, 1, 1)" fillcolor=lightblue]
|
| 609 |
+
140291558271128 -> 140291558270960
|
| 610 |
+
140291558271128 [label="
|
| 611 |
+
(64)" fillcolor=lightblue]
|
| 612 |
+
140291558270792 -> 140291558270456
|
| 613 |
+
140291558270792 [label="
|
| 614 |
+
(1, 64, 1, 1)" fillcolor=lightblue]
|
| 615 |
+
140291558270848 -> 140291558270456
|
| 616 |
+
140291558270848 [label="
|
| 617 |
+
(1)" fillcolor=lightblue]
|
| 618 |
+
140291558270120 -> 140291558269000
|
| 619 |
+
140291558270120 [label=SigmoidBackward]
|
| 620 |
+
140291558270344 -> 140291558270120
|
| 621 |
+
140291558270344 [label=ThnnConv2DBackward]
|
| 622 |
+
140291558270904 -> 140291558270344
|
| 623 |
+
140291558270904 [label=ReluBackward1]
|
| 624 |
+
140291558271296 -> 140291558270904
|
| 625 |
+
140291558271296 [label=ThnnConv2DBackward]
|
| 626 |
+
140291558271408 -> 140291558271296
|
| 627 |
+
140291558271408 [label=MeanBackward1]
|
| 628 |
+
140291558269784 -> 140291558271408
|
| 629 |
+
140291558271464 -> 140291558271296
|
| 630 |
+
140291558271464 [label="
|
| 631 |
+
(64, 512, 1, 1)" fillcolor=lightblue]
|
| 632 |
+
140291558271520 -> 140291558271296
|
| 633 |
+
140291558271520 [label="
|
| 634 |
+
(64)" fillcolor=lightblue]
|
| 635 |
+
140291558271016 -> 140291558270344
|
| 636 |
+
140291558271016 [label="
|
| 637 |
+
(512, 64, 1, 1)" fillcolor=lightblue]
|
| 638 |
+
140291558271184 -> 140291558270344
|
| 639 |
+
140291558271184 [label="
|
| 640 |
+
(512)" fillcolor=lightblue]
|
| 641 |
+
140291558269616 -> 140291558251936
|
| 642 |
+
140291558269616 [label="
|
| 643 |
+
(512, 512, 3, 3)" fillcolor=lightblue]
|
| 644 |
+
140291558270176 -> 140291558251936
|
| 645 |
+
140291558270176 [label="
|
| 646 |
+
(512)" fillcolor=lightblue]
|
| 647 |
+
140291558252160 -> 140291558251376
|
| 648 |
+
140291558252160 [label=ReluBackward0]
|
| 649 |
+
140291558251936 -> 140291558252160
|
| 650 |
+
140291558250200 -> 140291558250144
|
| 651 |
+
140291558235552 -> 140291558235328
|
| 652 |
+
140291558235552 [label=SigmoidBackward]
|
| 653 |
+
140291558235776 -> 140291558235552
|
| 654 |
+
140291558235776 [label=MkldnnConvolutionBackward]
|
| 655 |
+
140291558251488 -> 140291558235776
|
| 656 |
+
140291558251488 [label=ReluBackward1]
|
| 657 |
+
140291558270008 -> 140291558251488
|
| 658 |
+
140291558270008 [label=MkldnnConvolutionBackward]
|
| 659 |
+
140291558235496 -> 140291558270008
|
| 660 |
+
140291558270624 -> 140291558270008
|
| 661 |
+
140291558270624 [label="
|
| 662 |
+
(128, 1024, 1, 1)" fillcolor=lightblue]
|
| 663 |
+
140291558270064 -> 140291558270008
|
| 664 |
+
140291558270064 [label="
|
| 665 |
+
(128)" fillcolor=lightblue]
|
| 666 |
+
140291558251880 -> 140291558235776
|
| 667 |
+
140291558251880 [label="
|
| 668 |
+
(1, 128, 1, 1)" fillcolor=lightblue]
|
| 669 |
+
140291558251824 -> 140291558235776
|
| 670 |
+
140291558251824 [label="
|
| 671 |
+
(1)" fillcolor=lightblue]
|
| 672 |
+
140291558235384 -> 140291558235104
|
| 673 |
+
140291558235384 [label=SigmoidBackward]
|
| 674 |
+
140291558235608 -> 140291558235384
|
| 675 |
+
140291558235608 [label=ThnnConv2DBackward]
|
| 676 |
+
140291558252328 -> 140291558235608
|
| 677 |
+
140291558252328 [label=ReluBackward1]
|
| 678 |
+
140291558271240 -> 140291558252328
|
| 679 |
+
140291558271240 [label=ThnnConv2DBackward]
|
| 680 |
+
140291558271632 -> 140291558271240
|
| 681 |
+
140291558271632 [label=MeanBackward1]
|
| 682 |
+
140291558235328 -> 140291558271632
|
| 683 |
+
140291558271800 -> 140291558271240
|
| 684 |
+
140291558271800 [label="
|
| 685 |
+
(128, 1024, 1, 1)" fillcolor=lightblue]
|
| 686 |
+
140291558271856 -> 140291558271240
|
| 687 |
+
140291558271856 [label="
|
| 688 |
+
(128)" fillcolor=lightblue]
|
| 689 |
+
140291558271352 -> 140291558235608
|
| 690 |
+
140291558271352 [label="
|
| 691 |
+
(1024, 128, 1, 1)" fillcolor=lightblue]
|
| 692 |
+
140291558271744 -> 140291558235608
|
| 693 |
+
140291558271744 [label="
|
| 694 |
+
(1024)" fillcolor=lightblue]
|
| 695 |
+
140291558235048 -> 140291558234992
|
| 696 |
+
140291558235048 [label="
|
| 697 |
+
(256, 1024, 3, 3)" fillcolor=lightblue]
|
| 698 |
+
140291558235216 -> 140291558234992
|
| 699 |
+
140291558235216 [label="
|
| 700 |
+
(256)" fillcolor=lightblue]
|
| 701 |
+
140291558233480 -> 140291558233424
|
| 702 |
+
140291558233480 [label=SubBackward0]
|
| 703 |
+
140291558234656 -> 140291558233480
|
| 704 |
+
140291558234656 [label=MkldnnConvolutionBackward]
|
| 705 |
+
140291558269728 -> 140291558234656
|
| 706 |
+
140291558269728 [label=MulBackward0]
|
| 707 |
+
140291558271576 -> 140291558269728
|
| 708 |
+
140291558271576 [label=MulBackward0]
|
| 709 |
+
140291558272136 -> 140291558271576
|
| 710 |
+
140291558272136 [label=CatBackward]
|
| 711 |
+
140291558270400 -> 140291558272136
|
| 712 |
+
140291558272304 -> 140291558272136
|
| 713 |
+
140291558272304 [label=ConstantPadNdBackward]
|
| 714 |
+
140291558272416 -> 140291558272304
|
| 715 |
+
140291558272416 [label=UpsampleBilinear2DBackward1]
|
| 716 |
+
140291558272528 -> 140291558272416
|
| 717 |
+
140291558272528 [label=AddBackward0]
|
| 718 |
+
140291558272640 -> 140291558272528
|
| 719 |
+
140291558272640 [label=AddBackward0]
|
| 720 |
+
140291558272752 -> 140291558272640
|
| 721 |
+
140291558272752 [label=ReluBackward0]
|
| 722 |
+
140291558251936 -> 140291558272752
|
| 723 |
+
140291558272808 -> 140291558272640
|
| 724 |
+
140291558272808 [label=SubBackward0]
|
| 725 |
+
140291558251768 -> 140291558272808
|
| 726 |
+
140291558272920 -> 140291558272808
|
| 727 |
+
140291558272920 [label=ReluBackward0]
|
| 728 |
+
140291558251768 -> 140291558272920
|
| 729 |
+
140291558270232 -> 140291558272528
|
| 730 |
+
140291558272192 -> 140291558271576
|
| 731 |
+
140291558272192 [label=SigmoidBackward]
|
| 732 |
+
140291558272360 -> 140291558272192
|
| 733 |
+
140291558272360 [label=MkldnnConvolutionBackward]
|
| 734 |
+
140291558272584 -> 140291558272360
|
| 735 |
+
140291558272584 [label=ReluBackward1]
|
| 736 |
+
140291558281288 -> 140291558272584
|
| 737 |
+
140291558281288 [label=MkldnnConvolutionBackward]
|
| 738 |
+
140291558272136 -> 140291558281288
|
| 739 |
+
140291558281400 -> 140291558281288
|
| 740 |
+
140291558281400 [label="
|
| 741 |
+
(128, 1024, 1, 1)" fillcolor=lightblue]
|
| 742 |
+
140291558281456 -> 140291558281288
|
| 743 |
+
140291558281456 [label="
|
| 744 |
+
(128)" fillcolor=lightblue]
|
| 745 |
+
140291558272696 -> 140291558272360
|
| 746 |
+
140291558272696 [label="
|
| 747 |
+
(1, 128, 1, 1)" fillcolor=lightblue]
|
| 748 |
+
140291558272864 -> 140291558272360
|
| 749 |
+
140291558272864 [label="
|
| 750 |
+
(1)" fillcolor=lightblue]
|
| 751 |
+
140291558272024 -> 140291558269728
|
| 752 |
+
140291558272024 [label=SigmoidBackward]
|
| 753 |
+
140291558272248 -> 140291558272024
|
| 754 |
+
140291558272248 [label=ThnnConv2DBackward]
|
| 755 |
+
140291558272976 -> 140291558272248
|
| 756 |
+
140291558272976 [label=ReluBackward1]
|
| 757 |
+
140291558281624 -> 140291558272976
|
| 758 |
+
140291558281624 [label=ThnnConv2DBackward]
|
| 759 |
+
140291558281736 -> 140291558281624
|
| 760 |
+
140291558281736 [label=MeanBackward1]
|
| 761 |
+
140291558271576 -> 140291558281736
|
| 762 |
+
140291558281792 -> 140291558281624
|
| 763 |
+
140291558281792 [label="
|
| 764 |
+
(128, 1024, 1, 1)" fillcolor=lightblue]
|
| 765 |
+
140291558281848 -> 140291558281624
|
| 766 |
+
140291558281848 [label="
|
| 767 |
+
(128)" fillcolor=lightblue]
|
| 768 |
+
140291558281344 -> 140291558272248
|
| 769 |
+
140291558281344 [label="
|
| 770 |
+
(1024, 128, 1, 1)" fillcolor=lightblue]
|
| 771 |
+
140291558281512 -> 140291558272248
|
| 772 |
+
140291558281512 [label="
|
| 773 |
+
(1024)" fillcolor=lightblue]
|
| 774 |
+
140291558271688 -> 140291558234656
|
| 775 |
+
140291558271688 [label="
|
| 776 |
+
(256, 1024, 3, 3)" fillcolor=lightblue]
|
| 777 |
+
140291558271912 -> 140291558234656
|
| 778 |
+
140291558271912 [label="
|
| 779 |
+
(256)" fillcolor=lightblue]
|
| 780 |
+
140291558235272 -> 140291558233480
|
| 781 |
+
140291558235272 [label=ReluBackward0]
|
| 782 |
+
140291558234656 -> 140291558235272
|
| 783 |
+
140291558214736 -> 140291558214512
|
| 784 |
+
140291558214736 [label=SigmoidBackward]
|
| 785 |
+
140291558214960 -> 140291558214736
|
| 786 |
+
140291558214960 [label=MkldnnConvolutionBackward]
|
| 787 |
+
140291558234768 -> 140291558214960
|
| 788 |
+
140291558234768 [label=ReluBackward1]
|
| 789 |
+
140291558271968 -> 140291558234768
|
| 790 |
+
140291558271968 [label=MkldnnConvolutionBackward]
|
| 791 |
+
140291558214680 -> 140291558271968
|
| 792 |
+
140291558281680 -> 140291558271968
|
| 793 |
+
140291558281680 [label="
|
| 794 |
+
(64, 512, 1, 1)" fillcolor=lightblue]
|
| 795 |
+
140291558281904 -> 140291558271968
|
| 796 |
+
140291558281904 [label="
|
| 797 |
+
(64)" fillcolor=lightblue]
|
| 798 |
+
140291558235160 -> 140291558214960
|
| 799 |
+
140291558235160 [label="
|
| 800 |
+
(1, 64, 1, 1)" fillcolor=lightblue]
|
| 801 |
+
140291558249752 -> 140291558214960
|
| 802 |
+
140291558249752 [label="
|
| 803 |
+
(1)" fillcolor=lightblue]
|
| 804 |
+
140291558214568 -> 140291558214288
|
| 805 |
+
140291558214568 [label=SigmoidBackward]
|
| 806 |
+
140291558214792 -> 140291558214568
|
| 807 |
+
140291558214792 [label=ThnnConv2DBackward]
|
| 808 |
+
140291558272080 -> 140291558214792
|
| 809 |
+
140291558272080 [label=ReluBackward1]
|
| 810 |
+
140291558282016 -> 140291558272080
|
| 811 |
+
140291558282016 [label=ThnnConv2DBackward]
|
| 812 |
+
140291558282128 -> 140291558282016
|
| 813 |
+
140291558282128 [label=MeanBackward1]
|
| 814 |
+
140291558214512 -> 140291558282128
|
| 815 |
+
140291558282184 -> 140291558282016
|
| 816 |
+
140291558282184 [label="
|
| 817 |
+
(64, 512, 1, 1)" fillcolor=lightblue]
|
| 818 |
+
140291558282240 -> 140291558282016
|
| 819 |
+
140291558282240 [label="
|
| 820 |
+
(64)" fillcolor=lightblue]
|
| 821 |
+
140291558272472 -> 140291558214792
|
| 822 |
+
140291558272472 [label="
|
| 823 |
+
(512, 64, 1, 1)" fillcolor=lightblue]
|
| 824 |
+
140291558282072 -> 140291558214792
|
| 825 |
+
140291558282072 [label="
|
| 826 |
+
(512)" fillcolor=lightblue]
|
| 827 |
+
140291558214344 -> 140291558214176
|
| 828 |
+
140291558214344 [label="
|
| 829 |
+
(128, 512, 3, 3)" fillcolor=lightblue]
|
| 830 |
+
140291558214400 -> 140291558214176
|
| 831 |
+
140291558214400 [label="
|
| 832 |
+
(128)" fillcolor=lightblue]
|
| 833 |
+
140291558212776 -> 140291558212608
|
| 834 |
+
140291558212776 [label=SubBackward0]
|
| 835 |
+
140291558214232 -> 140291558212776
|
| 836 |
+
140291558214232 [label=MkldnnConvolutionBackward]
|
| 837 |
+
140291558215632 -> 140291558214232
|
| 838 |
+
140291558215632 [label=MulBackward0]
|
| 839 |
+
140291558281568 -> 140291558215632
|
| 840 |
+
140291558281568 [label=MulBackward0]
|
| 841 |
+
140291558282520 -> 140291558281568
|
| 842 |
+
140291558282520 [label=CatBackward]
|
| 843 |
+
140291558250536 -> 140291558282520
|
| 844 |
+
140291558282688 -> 140291558282520
|
| 845 |
+
140291558282688 [label=ConstantPadNdBackward]
|
| 846 |
+
140291558282800 -> 140291558282688
|
| 847 |
+
140291558282800 [label=UpsampleBilinear2DBackward1]
|
| 848 |
+
140291558282912 -> 140291558282800
|
| 849 |
+
140291558282912 [label=AddBackward0]
|
| 850 |
+
140291558283024 -> 140291558282912
|
| 851 |
+
140291558283024 [label=ReluBackward0]
|
| 852 |
+
140291558234656 -> 140291558283024
|
| 853 |
+
140291558283080 -> 140291558282912
|
| 854 |
+
140291558283080 [label=SubBackward0]
|
| 855 |
+
140291558234992 -> 140291558283080
|
| 856 |
+
140291558283192 -> 140291558283080
|
| 857 |
+
140291558283192 [label=ReluBackward0]
|
| 858 |
+
140291558234992 -> 140291558283192
|
| 859 |
+
140291558282576 -> 140291558281568
|
| 860 |
+
140291558282576 [label=SigmoidBackward]
|
| 861 |
+
140291558282744 -> 140291558282576
|
| 862 |
+
140291558282744 [label=MkldnnConvolutionBackward]
|
| 863 |
+
140291558282968 -> 140291558282744
|
| 864 |
+
140291558282968 [label=ReluBackward1]
|
| 865 |
+
140291558283360 -> 140291558282968
|
| 866 |
+
140291558283360 [label=MkldnnConvolutionBackward]
|
| 867 |
+
140291558282520 -> 140291558283360
|
| 868 |
+
140291558283472 -> 140291558283360
|
| 869 |
+
140291558283472 [label="
|
| 870 |
+
(64, 512, 1, 1)" fillcolor=lightblue]
|
| 871 |
+
140291558283528 -> 140291558283360
|
| 872 |
+
140291558283528 [label="
|
| 873 |
+
(64)" fillcolor=lightblue]
|
| 874 |
+
140291558283136 -> 140291558282744
|
| 875 |
+
140291558283136 [label="
|
| 876 |
+
(1, 64, 1, 1)" fillcolor=lightblue]
|
| 877 |
+
140291558283248 -> 140291558282744
|
| 878 |
+
140291558283248 [label="
|
| 879 |
+
(1)" fillcolor=lightblue]
|
| 880 |
+
140291558282408 -> 140291558215632
|
| 881 |
+
140291558282408 [label=SigmoidBackward]
|
| 882 |
+
140291558282632 -> 140291558282408
|
| 883 |
+
140291558282632 [label=ThnnConv2DBackward]
|
| 884 |
+
140291558283304 -> 140291558282632
|
| 885 |
+
140291558283304 [label=ReluBackward1]
|
| 886 |
+
140291558283696 -> 140291558283304
|
| 887 |
+
140291558283696 [label=ThnnConv2DBackward]
|
| 888 |
+
140291558283808 -> 140291558283696
|
| 889 |
+
140291558283808 [label=MeanBackward1]
|
| 890 |
+
140291558281568 -> 140291558283808
|
| 891 |
+
140291558283864 -> 140291558283696
|
| 892 |
+
140291558283864 [label="
|
| 893 |
+
(64, 512, 1, 1)" fillcolor=lightblue]
|
| 894 |
+
140291558283920 -> 140291558283696
|
| 895 |
+
140291558283920 [label="
|
| 896 |
+
(64)" fillcolor=lightblue]
|
| 897 |
+
140291558283416 -> 140291558282632
|
| 898 |
+
140291558283416 [label="
|
| 899 |
+
(512, 64, 1, 1)" fillcolor=lightblue]
|
| 900 |
+
140291558283584 -> 140291558282632
|
| 901 |
+
140291558283584 [label="
|
| 902 |
+
(512)" fillcolor=lightblue]
|
| 903 |
+
140291558281960 -> 140291558214232
|
| 904 |
+
140291558281960 [label="
|
| 905 |
+
(128, 512, 3, 3)" fillcolor=lightblue]
|
| 906 |
+
140291558282464 -> 140291558214232
|
| 907 |
+
140291558282464 [label="
|
| 908 |
+
(128)" fillcolor=lightblue]
|
| 909 |
+
140291558214456 -> 140291558212776
|
| 910 |
+
140291558214456 [label=ReluBackward0]
|
| 911 |
+
140291558214232 -> 140291558214456
|
| 912 |
+
140291558202280 -> 140291558202056
|
| 913 |
+
140291558202280 [label=SigmoidBackward]
|
| 914 |
+
140291558202504 -> 140291558202280
|
| 915 |
+
140291558202504 [label=MkldnnConvolutionBackward]
|
| 916 |
+
140291558213840 -> 140291558202504
|
| 917 |
+
140291558213840 [label=ReluBackward1]
|
| 918 |
+
140291558283752 -> 140291558213840
|
| 919 |
+
140291558283752 [label=MkldnnConvolutionBackward]
|
| 920 |
+
140291558202224 -> 140291558283752
|
| 921 |
+
140291558282352 -> 140291558283752
|
| 922 |
+
140291558282352 [label="
|
| 923 |
+
(32, 256, 1, 1)" fillcolor=lightblue]
|
| 924 |
+
140291558284144 -> 140291558283752
|
| 925 |
+
140291558284144 [label="
|
| 926 |
+
(32)" fillcolor=lightblue]
|
| 927 |
+
140291558213952 -> 140291558202504
|
| 928 |
+
140291558213952 [label="
|
| 929 |
+
(1, 32, 1, 1)" fillcolor=lightblue]
|
| 930 |
+
140291558214624 -> 140291558202504
|
| 931 |
+
140291558214624 [label="
|
| 932 |
+
(1)" fillcolor=lightblue]
|
| 933 |
+
140291558202112 -> 140291558201832
|
| 934 |
+
140291558202112 [label=SigmoidBackward]
|
| 935 |
+
140291558202336 -> 140291558202112
|
| 936 |
+
140291558202336 [label=ThnnConv2DBackward]
|
| 937 |
+
140291558282296 -> 140291558202336
|
| 938 |
+
140291558282296 [label=ReluBackward1]
|
| 939 |
+
140291558284088 -> 140291558282296
|
| 940 |
+
140291558284088 [label=ThnnConv2DBackward]
|
| 941 |
+
140291558284200 -> 140291558284088
|
| 942 |
+
140291558284200 [label=MeanBackward1]
|
| 943 |
+
140291558202056 -> 140291558284200
|
| 944 |
+
140291558284256 -> 140291558284088
|
| 945 |
+
140291558284256 [label="
|
| 946 |
+
(32, 256, 1, 1)" fillcolor=lightblue]
|
| 947 |
+
140291558284312 -> 140291558284088
|
| 948 |
+
140291558284312 [label="
|
| 949 |
+
(32)" fillcolor=lightblue]
|
| 950 |
+
140291558282856 -> 140291558202336
|
| 951 |
+
140291558282856 [label="
|
| 952 |
+
(256, 32, 1, 1)" fillcolor=lightblue]
|
| 953 |
+
140291558283976 -> 140291558202336
|
| 954 |
+
140291558283976 [label="
|
| 955 |
+
(256)" fillcolor=lightblue]
|
| 956 |
+
140291558201888 -> 140291558201720
|
| 957 |
+
140291558201888 [label="
|
| 958 |
+
(64, 256, 3, 3)" fillcolor=lightblue]
|
| 959 |
+
140291558201944 -> 140291558201720
|
| 960 |
+
140291558201944 [label="
|
| 961 |
+
(64)" fillcolor=lightblue]
|
| 962 |
+
140291558201496 -> 140291558201328
|
| 963 |
+
140291558201496 [label=SubBackward0]
|
| 964 |
+
140291558201776 -> 140291558201496
|
| 965 |
+
140291558201776 [label=MkldnnConvolutionBackward]
|
| 966 |
+
140291558203176 -> 140291558201776
|
| 967 |
+
140291558203176 [label=MulBackward0]
|
| 968 |
+
140291558283640 -> 140291558203176
|
| 969 |
+
140291558283640 [label=MulBackward0]
|
| 970 |
+
140291558284592 -> 140291558283640
|
| 971 |
+
140291558284592 [label=CatBackward]
|
| 972 |
+
140291558233816 -> 140291558284592
|
| 973 |
+
140291558284760 -> 140291558284592
|
| 974 |
+
140291558284760 [label=ConstantPadNdBackward]
|
| 975 |
+
140291558284872 -> 140291558284760
|
| 976 |
+
140291558284872 [label=UpsampleBilinear2DBackward1]
|
| 977 |
+
140291558284984 -> 140291558284872
|
| 978 |
+
140291558284984 [label=AddBackward0]
|
| 979 |
+
140291558285096 -> 140291558284984
|
| 980 |
+
140291558285096 [label=ReluBackward0]
|
| 981 |
+
140291558214232 -> 140291558285096
|
| 982 |
+
140291558285152 -> 140291558284984
|
| 983 |
+
140291558285152 [label=SubBackward0]
|
| 984 |
+
140291558214176 -> 140291558285152
|
| 985 |
+
140291558285264 -> 140291558285152
|
| 986 |
+
140291558285264 [label=ReluBackward0]
|
| 987 |
+
140291558214176 -> 140291558285264
|
| 988 |
+
140291558284648 -> 140291558283640
|
| 989 |
+
140291558284648 [label=SigmoidBackward]
|
| 990 |
+
140291558284816 -> 140291558284648
|
| 991 |
+
140291558284816 [label=MkldnnConvolutionBackward]
|
| 992 |
+
140291558285040 -> 140291558284816
|
| 993 |
+
140291558285040 [label=ReluBackward1]
|
| 994 |
+
140291333939384 -> 140291558285040
|
| 995 |
+
140291333939384 [label=MkldnnConvolutionBackward]
|
| 996 |
+
140291558284592 -> 140291333939384
|
| 997 |
+
140291333939496 -> 140291333939384
|
| 998 |
+
140291333939496 [label="
|
| 999 |
+
(32, 256, 1, 1)" fillcolor=lightblue]
|
| 1000 |
+
140291333939552 -> 140291333939384
|
| 1001 |
+
140291333939552 [label="
|
| 1002 |
+
(32)" fillcolor=lightblue]
|
| 1003 |
+
140291558285208 -> 140291558284816
|
| 1004 |
+
140291558285208 [label="
|
| 1005 |
+
(1, 32, 1, 1)" fillcolor=lightblue]
|
| 1006 |
+
140291333939272 -> 140291558284816
|
| 1007 |
+
140291333939272 [label="
|
| 1008 |
+
(1)" fillcolor=lightblue]
|
| 1009 |
+
140291558284480 -> 140291558203176
|
| 1010 |
+
140291558284480 [label=SigmoidBackward]
|
| 1011 |
+
140291558284704 -> 140291558284480
|
| 1012 |
+
140291558284704 [label=ThnnConv2DBackward]
|
| 1013 |
+
140291333939328 -> 140291558284704
|
| 1014 |
+
140291333939328 [label=ReluBackward1]
|
| 1015 |
+
140291333939720 -> 140291333939328
|
| 1016 |
+
140291333939720 [label=ThnnConv2DBackward]
|
| 1017 |
+
140291333939832 -> 140291333939720
|
| 1018 |
+
140291333939832 [label=MeanBackward1]
|
| 1019 |
+
140291558283640 -> 140291333939832
|
| 1020 |
+
140291333939888 -> 140291333939720
|
| 1021 |
+
140291333939888 [label="
|
| 1022 |
+
(32, 256, 1, 1)" fillcolor=lightblue]
|
| 1023 |
+
140291333939944 -> 140291333939720
|
| 1024 |
+
140291333939944 [label="
|
| 1025 |
+
(32)" fillcolor=lightblue]
|
| 1026 |
+
140291333939440 -> 140291558284704
|
| 1027 |
+
140291333939440 [label="
|
| 1028 |
+
(256, 32, 1, 1)" fillcolor=lightblue]
|
| 1029 |
+
140291333939608 -> 140291558284704
|
| 1030 |
+
140291333939608 [label="
|
| 1031 |
+
(256)" fillcolor=lightblue]
|
| 1032 |
+
140291558284032 -> 140291558201776
|
| 1033 |
+
140291558284032 [label="
|
| 1034 |
+
(64, 256, 3, 3)" fillcolor=lightblue]
|
| 1035 |
+
140291558284536 -> 140291558201776
|
| 1036 |
+
140291558284536 [label="
|
| 1037 |
+
(64)" fillcolor=lightblue]
|
| 1038 |
+
140291558202000 -> 140291558201496
|
| 1039 |
+
140291558202000 [label=ReluBackward0]
|
| 1040 |
+
140291558201776 -> 140291558202000
|
| 1041 |
+
140291558200488 -> 140291558200264
|
| 1042 |
+
140291558200488 [label=SigmoidBackward]
|
| 1043 |
+
140291558200712 -> 140291558200488
|
| 1044 |
+
140291558200712 [label=MkldnnConvolutionBackward]
|
| 1045 |
+
140291558201608 -> 140291558200712
|
| 1046 |
+
140291558201608 [label=ReluBackward1]
|
| 1047 |
+
140291558284424 -> 140291558201608
|
| 1048 |
+
140291558284424 [label=MkldnnConvolutionBackward]
|
| 1049 |
+
140291558200432 -> 140291558284424
|
| 1050 |
+
140291333939776 -> 140291558284424
|
| 1051 |
+
140291333939776 [label="
|
| 1052 |
+
(16, 128, 1, 1)" fillcolor=lightblue]
|
| 1053 |
+
140291333940168 -> 140291558284424
|
| 1054 |
+
140291333940168 [label="
|
| 1055 |
+
(16)" fillcolor=lightblue]
|
| 1056 |
+
140291558201664 -> 140291558200712
|
| 1057 |
+
140291558201664 [label="
|
| 1058 |
+
(1, 16, 1, 1)" fillcolor=lightblue]
|
| 1059 |
+
140291558202168 -> 140291558200712
|
| 1060 |
+
140291558202168 [label="
|
| 1061 |
+
(1)" fillcolor=lightblue]
|
| 1062 |
+
140291558200320 -> 140291558200040
|
| 1063 |
+
140291558200320 [label=SigmoidBackward]
|
| 1064 |
+
140291558200544 -> 140291558200320
|
| 1065 |
+
140291558200544 [label=ThnnConv2DBackward]
|
| 1066 |
+
140291558284368 -> 140291558200544
|
| 1067 |
+
140291558284368 [label=ReluBackward1]
|
| 1068 |
+
140291333940112 -> 140291558284368
|
| 1069 |
+
140291333940112 [label=ThnnConv2DBackward]
|
| 1070 |
+
140291333940224 -> 140291333940112
|
| 1071 |
+
140291333940224 [label=MeanBackward1]
|
| 1072 |
+
140291558200264 -> 140291333940224
|
| 1073 |
+
140291333940280 -> 140291333940112
|
| 1074 |
+
140291333940280 [label="
|
| 1075 |
+
(16, 128, 1, 1)" fillcolor=lightblue]
|
| 1076 |
+
140291333940336 -> 140291333940112
|
| 1077 |
+
140291333940336 [label="
|
| 1078 |
+
(16)" fillcolor=lightblue]
|
| 1079 |
+
140291558284928 -> 140291558200544
|
| 1080 |
+
140291558284928 [label="
|
| 1081 |
+
(128, 16, 1, 1)" fillcolor=lightblue]
|
| 1082 |
+
140291333940000 -> 140291558200544
|
| 1083 |
+
140291333940000 [label="
|
| 1084 |
+
(128)" fillcolor=lightblue]
|
| 1085 |
+
140291558200096 -> 140291558199928
|
| 1086 |
+
140291558200096 [label="
|
| 1087 |
+
(64, 128, 3, 3)" fillcolor=lightblue]
|
| 1088 |
+
140291558200152 -> 140291558199928
|
| 1089 |
+
140291558200152 [label="
|
| 1090 |
+
(64)" fillcolor=lightblue]
|
| 1091 |
+
140291558199816 -> 140291558199536
|
| 1092 |
+
140291558199816 [label=SubBackward0]
|
| 1093 |
+
140291558199984 -> 140291558199816
|
| 1094 |
+
140291558199984 [label=MkldnnConvolutionBackward]
|
| 1095 |
+
140291558201440 -> 140291558199984
|
| 1096 |
+
140291558201440 [label=MulBackward0]
|
| 1097 |
+
140291333939664 -> 140291558201440
|
| 1098 |
+
140291333939664 [label=MulBackward0]
|
| 1099 |
+
140291333940616 -> 140291333939664
|
| 1100 |
+
140291333940616 [label=CatBackward]
|
| 1101 |
+
140291558213000 -> 140291333940616
|
| 1102 |
+
140291333940784 -> 140291333940616
|
| 1103 |
+
140291333940784 [label=ConstantPadNdBackward]
|
| 1104 |
+
140291333940896 -> 140291333940784
|
| 1105 |
+
140291333940896 [label=UpsampleBilinear2DBackward1]
|
| 1106 |
+
140291333941008 -> 140291333940896
|
| 1107 |
+
140291333941008 [label=AddBackward0]
|
| 1108 |
+
140291333941120 -> 140291333941008
|
| 1109 |
+
140291333941120 [label=ReluBackward0]
|
| 1110 |
+
140291558201776 -> 140291333941120
|
| 1111 |
+
140291333941176 -> 140291333941008
|
| 1112 |
+
140291333941176 [label=SubBackward0]
|
| 1113 |
+
140291558201720 -> 140291333941176
|
| 1114 |
+
140291333941288 -> 140291333941176
|
| 1115 |
+
140291333941288 [label=ReluBackward0]
|
| 1116 |
+
140291558201720 -> 140291333941288
|
| 1117 |
+
140291333940672 -> 140291333939664
|
| 1118 |
+
140291333940672 [label=SigmoidBackward]
|
| 1119 |
+
140291333940840 -> 140291333940672
|
| 1120 |
+
140291333940840 [label=MkldnnConvolutionBackward]
|
| 1121 |
+
140291333941064 -> 140291333940840
|
| 1122 |
+
140291333941064 [label=ReluBackward1]
|
| 1123 |
+
140291333941456 -> 140291333941064
|
| 1124 |
+
140291333941456 [label=MkldnnConvolutionBackward]
|
| 1125 |
+
140291333940616 -> 140291333941456
|
| 1126 |
+
140291333941568 -> 140291333941456
|
| 1127 |
+
140291333941568 [label="
|
| 1128 |
+
(16, 128, 1, 1)" fillcolor=lightblue]
|
| 1129 |
+
140291333941624 -> 140291333941456
|
| 1130 |
+
140291333941624 [label="
|
| 1131 |
+
(16)" fillcolor=lightblue]
|
| 1132 |
+
140291333941232 -> 140291333940840
|
| 1133 |
+
140291333941232 [label="
|
| 1134 |
+
(1, 16, 1, 1)" fillcolor=lightblue]
|
| 1135 |
+
140291333941344 -> 140291333940840
|
| 1136 |
+
140291333941344 [label="
|
| 1137 |
+
(1)" fillcolor=lightblue]
|
| 1138 |
+
140291333940504 -> 140291558201440
|
| 1139 |
+
140291333940504 [label=SigmoidBackward]
|
| 1140 |
+
140291333940728 -> 140291333940504
|
| 1141 |
+
140291333940728 [label=ThnnConv2DBackward]
|
| 1142 |
+
140291333941400 -> 140291333940728
|
| 1143 |
+
140291333941400 [label=ReluBackward1]
|
| 1144 |
+
140291333941792 -> 140291333941400
|
| 1145 |
+
140291333941792 [label=ThnnConv2DBackward]
|
| 1146 |
+
140291333941904 -> 140291333941792
|
| 1147 |
+
140291333941904 [label=MeanBackward1]
|
| 1148 |
+
140291333939664 -> 140291333941904
|
| 1149 |
+
140291333941960 -> 140291333941792
|
| 1150 |
+
140291333941960 [label="
|
| 1151 |
+
(16, 128, 1, 1)" fillcolor=lightblue]
|
| 1152 |
+
140291333942016 -> 140291333941792
|
| 1153 |
+
140291333942016 [label="
|
| 1154 |
+
(16)" fillcolor=lightblue]
|
| 1155 |
+
140291333941512 -> 140291333940728
|
| 1156 |
+
140291333941512 [label="
|
| 1157 |
+
(128, 16, 1, 1)" fillcolor=lightblue]
|
| 1158 |
+
140291333941680 -> 140291333940728
|
| 1159 |
+
140291333941680 [label="
|
| 1160 |
+
(128)" fillcolor=lightblue]
|
| 1161 |
+
140291333940056 -> 140291558199984
|
| 1162 |
+
140291333940056 [label="
|
| 1163 |
+
(64, 128, 3, 3)" fillcolor=lightblue]
|
| 1164 |
+
140291333940560 -> 140291558199984
|
| 1165 |
+
140291333940560 [label="
|
| 1166 |
+
(64)" fillcolor=lightblue]
|
| 1167 |
+
140291558200208 -> 140291558199816
|
| 1168 |
+
140291558200208 [label=ReluBackward0]
|
| 1169 |
+
140291558199984 -> 140291558200208
|
| 1170 |
+
140291558199592 -> 140291558199424
|
| 1171 |
+
140291558199592 [label="
|
| 1172 |
+
(3, 64, 3, 3)" fillcolor=lightblue]
|
| 1173 |
+
140291558199648 -> 140291558199424
|
| 1174 |
+
140291558199648 [label="
|
| 1175 |
+
(3)" fillcolor=lightblue]
|
| 1176 |
+
140291558534784 -> 140291558534952
|
| 1177 |
+
140291558534784 [label=SigmoidBackward]
|
| 1178 |
+
140291558199480 -> 140291558534784
|
| 1179 |
+
140291558199480 [label=MkldnnConvolutionBackward]
|
| 1180 |
+
140291558199872 -> 140291558199480
|
| 1181 |
+
140291558199872 [label=AddBackward0]
|
| 1182 |
+
140291333940952 -> 140291558199872
|
| 1183 |
+
140291333940952 [label=ReluBackward0]
|
| 1184 |
+
140291558199984 -> 140291333940952
|
| 1185 |
+
140291333940448 -> 140291558199872
|
| 1186 |
+
140291333940448 [label=SubBackward0]
|
| 1187 |
+
140291558199928 -> 140291333940448
|
| 1188 |
+
140291333942072 -> 140291333940448
|
| 1189 |
+
140291333942072 [label=ReluBackward0]
|
| 1190 |
+
140291558199928 -> 140291333942072
|
| 1191 |
+
140291558200376 -> 140291558199480
|
| 1192 |
+
140291558200376 [label="
|
| 1193 |
+
(3, 64, 3, 3)" fillcolor=lightblue]
|
| 1194 |
+
140291333940392 -> 140291558199480
|
| 1195 |
+
140291333940392 [label="
|
| 1196 |
+
(3)" fillcolor=lightblue]
|
| 1197 |
+
}
|
DSRNet/models/arch/~/YTMT_US.pdf
ADDED
|
Binary file (64.3 kB). View file
|
|
|
DSRNet/models/arch/~/tmp.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:678f82d68747dec3e072253174ea87f3131c2a354c71769c5e8648503933ebb6
|
| 3 |
+
size 962515
|
DSRNet/models/base_model.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import torch
|
| 3 |
+
import util.util as util
|
| 4 |
+
from tools import mutils
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class BaseModel:
|
| 8 |
+
def name(self):
|
| 9 |
+
return self.__class__.__name__.lower()
|
| 10 |
+
|
| 11 |
+
def initialize(self, opt):
|
| 12 |
+
self.opt = opt
|
| 13 |
+
self.gpu_ids = opt.gpu_ids
|
| 14 |
+
self.isTrain = opt.isTrain
|
| 15 |
+
self.Tensor = torch.cuda.FloatTensor if self.gpu_ids else torch.Tensor
|
| 16 |
+
last_split = opt.checkpoints_dir.split('/')[-1]
|
| 17 |
+
if opt.resume and last_split != 'checkpoints' and (last_split != opt.name or opt.supp_eval):
|
| 18 |
+
|
| 19 |
+
self.save_dir = opt.checkpoints_dir
|
| 20 |
+
self.model_save_dir = os.path.join(opt.checkpoints_dir.replace(opt.checkpoints_dir.split('/')[-1], ''),
|
| 21 |
+
opt.name, 'weights', mutils.get_formatted_time())
|
| 22 |
+
else:
|
| 23 |
+
self.save_dir = os.path.join(opt.checkpoints_dir, opt.name)
|
| 24 |
+
self.model_save_dir = os.path.join(opt.checkpoints_dir, opt.name, 'weights', mutils.get_formatted_time())
|
| 25 |
+
self._count = 0
|
| 26 |
+
|
| 27 |
+
def set_input(self, input):
|
| 28 |
+
self.input = input
|
| 29 |
+
|
| 30 |
+
def forward(self, mode='train'):
|
| 31 |
+
pass
|
| 32 |
+
|
| 33 |
+
# used in test time, no backprop
|
| 34 |
+
def test(self):
|
| 35 |
+
pass
|
| 36 |
+
|
| 37 |
+
def get_image_paths(self):
|
| 38 |
+
pass
|
| 39 |
+
|
| 40 |
+
def optimize_parameters(self):
|
| 41 |
+
pass
|
| 42 |
+
|
| 43 |
+
def get_current_visuals(self):
|
| 44 |
+
return self.input
|
| 45 |
+
|
| 46 |
+
def get_current_errors(self):
|
| 47 |
+
return {}
|
| 48 |
+
|
| 49 |
+
def print_optimizer_param(self):
|
| 50 |
+
print(self.optimizers[-1])
|
| 51 |
+
|
| 52 |
+
def save(self, label=None):
|
| 53 |
+
epoch = self.epoch
|
| 54 |
+
iterations = self.iterations
|
| 55 |
+
|
| 56 |
+
os.makedirs(self.model_save_dir, exist_ok=True)
|
| 57 |
+
if label is None:
|
| 58 |
+
model_name = os.path.join(self.model_save_dir, self.opt.name + '_%03d_%08d.pt' % ((epoch), (iterations)))
|
| 59 |
+
else:
|
| 60 |
+
model_name = os.path.join(self.model_save_dir, self.opt.name + '_' + label + '.pt')
|
| 61 |
+
|
| 62 |
+
torch.save(self.state_dict(), model_name)
|
| 63 |
+
|
| 64 |
+
def save_eval(self, label=None):
|
| 65 |
+
model_name = os.path.join(self.model_save_dir, label + '.pt')
|
| 66 |
+
|
| 67 |
+
torch.save(self.state_dict_eval(), model_name)
|
| 68 |
+
|
| 69 |
+
def _init_optimizer(self, optimizers):
|
| 70 |
+
self.optimizers = optimizers
|
| 71 |
+
for optimizer in self.optimizers:
|
| 72 |
+
util.set_opt_param(optimizer, 'initial_lr', self.opt.lr)
|
| 73 |
+
util.set_opt_param(optimizer, 'weight_decay', self.opt.wd)
|
DSRNet/models/dsrnet_model_sirs.py
ADDED
|
@@ -0,0 +1,263 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from collections import OrderedDict
|
| 3 |
+
from os.path import join
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
import torch
|
| 7 |
+
from PIL import Image
|
| 8 |
+
|
| 9 |
+
import models
|
| 10 |
+
import models.networks as networks
|
| 11 |
+
import util.index as index
|
| 12 |
+
import util.util as util
|
| 13 |
+
from models import arch
|
| 14 |
+
from .base_model import BaseModel
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def tensor2im(image_tensor, imtype=np.uint8):
|
| 18 |
+
image_tensor = image_tensor.detach()
|
| 19 |
+
image_numpy = image_tensor[0].cpu().float().numpy()
|
| 20 |
+
image_numpy = np.clip(image_numpy, 0, 1)
|
| 21 |
+
if image_numpy.shape[0] == 1:
|
| 22 |
+
image_numpy = np.tile(image_numpy, (3, 1, 1))
|
| 23 |
+
image_numpy = (np.transpose(image_numpy, (1, 2, 0))) * 255.0
|
| 24 |
+
# image_numpy = image_numpy.astype(imtype)
|
| 25 |
+
return image_numpy
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class DSRNetBase(BaseModel):
|
| 29 |
+
def _init_optimizer(self, optimizers):
|
| 30 |
+
self.optimizers = optimizers
|
| 31 |
+
for optimizer in self.optimizers:
|
| 32 |
+
util.set_opt_param(optimizer, 'initial_lr', self.opt.lr)
|
| 33 |
+
util.set_opt_param(optimizer, 'weight_decay', self.opt.wd)
|
| 34 |
+
|
| 35 |
+
def set_input(self, data, mode='train'):
|
| 36 |
+
target_t = None
|
| 37 |
+
target_r = None
|
| 38 |
+
data_name = None
|
| 39 |
+
identity = False
|
| 40 |
+
mode = mode.lower()
|
| 41 |
+
if mode == 'train':
|
| 42 |
+
input, target_t, target_r = data['input'], data['target_t'], data['target_r']
|
| 43 |
+
elif mode == 'eval':
|
| 44 |
+
input, target_t, target_r, data_name = data['input'], data['target_t'], data['target_r'], data['fn']
|
| 45 |
+
elif mode == 'test':
|
| 46 |
+
input, data_name = data['input'], data['fn']
|
| 47 |
+
else:
|
| 48 |
+
raise NotImplementedError('Mode [%s] is not implemented' % mode)
|
| 49 |
+
|
| 50 |
+
if len(self.gpu_ids) > 0: # transfer data into gpu
|
| 51 |
+
input = input.to(device=self.gpu_ids[0])
|
| 52 |
+
if target_t is not None:
|
| 53 |
+
target_t = target_t.to(device=self.gpu_ids[0])
|
| 54 |
+
if target_r is not None:
|
| 55 |
+
target_r = target_r.to(device=self.gpu_ids[0])
|
| 56 |
+
|
| 57 |
+
self.input = input
|
| 58 |
+
self.identity = identity
|
| 59 |
+
self.target_t = target_t
|
| 60 |
+
self.target_r = target_r
|
| 61 |
+
self.data_name = data_name
|
| 62 |
+
|
| 63 |
+
self.issyn = False if 'real' in data else True
|
| 64 |
+
self.aligned = False if 'unaligned' in data else True
|
| 65 |
+
|
| 66 |
+
def eval(self, data, savedir=None, suffix=None, pieapp=None):
|
| 67 |
+
# only the 1st input of the whole minibatch would be evaluated
|
| 68 |
+
self._eval()
|
| 69 |
+
self.set_input(data, 'eval')
|
| 70 |
+
|
| 71 |
+
with torch.no_grad():
|
| 72 |
+
self.forward()
|
| 73 |
+
|
| 74 |
+
output_t = tensor2im(self.output_t)
|
| 75 |
+
output_r = tensor2im(self.output_r)
|
| 76 |
+
output_rr = tensor2im(torch.clip((self.output_rr + 1) / 2, 0., 1.))
|
| 77 |
+
target = tensor2im(self.target_t)
|
| 78 |
+
target_r = tensor2im(self.target_r)
|
| 79 |
+
|
| 80 |
+
if self.aligned:
|
| 81 |
+
res = index.quality_assess(output_t, target)
|
| 82 |
+
# res = index.quality_assess(output_j, target_r)
|
| 83 |
+
else:
|
| 84 |
+
res = {}
|
| 85 |
+
|
| 86 |
+
if savedir is not None:
|
| 87 |
+
if self.data_name is not None:
|
| 88 |
+
name = os.path.splitext(os.path.basename(self.data_name[0]))[0]
|
| 89 |
+
savedir = join(savedir, suffix, name)
|
| 90 |
+
os.makedirs(savedir, exist_ok=True)
|
| 91 |
+
Image.fromarray(output_t.astype(np.uint8)).save(
|
| 92 |
+
join(savedir, '{}_t.png'.format(self.opt.name)))
|
| 93 |
+
Image.fromarray(output_r.astype(np.uint8)).save(
|
| 94 |
+
join(savedir, '{}_r.png'.format(self.opt.name)))
|
| 95 |
+
Image.fromarray(output_rr.astype(np.uint8)).save(
|
| 96 |
+
join(savedir, '{}_rr.png'.format(self.opt.name)))
|
| 97 |
+
|
| 98 |
+
Image.fromarray(target.astype(np.uint8)).save(join(savedir, 't_label.png'))
|
| 99 |
+
Image.fromarray(target_r.astype(np.uint8)).save(join(savedir, 'r_label.png'))
|
| 100 |
+
Image.fromarray(tensor2im(self.input).astype(np.uint8)).save(join(savedir, 'm_input.png'))
|
| 101 |
+
else:
|
| 102 |
+
if not os.path.exists(join(savedir, 'transmission_layer')):
|
| 103 |
+
os.makedirs(join(savedir, 'transmission_layer'))
|
| 104 |
+
os.makedirs(join(savedir, 'blended'))
|
| 105 |
+
Image.fromarray(target.astype(np.uint8)).save(
|
| 106 |
+
join(savedir, 'transmission_layer', str(self._count) + '.png'))
|
| 107 |
+
Image.fromarray(tensor2im(self.input).astype(np.uint8)).save(
|
| 108 |
+
join(savedir, 'blended', str(self._count) + '.png'))
|
| 109 |
+
self._count += 1
|
| 110 |
+
|
| 111 |
+
return res
|
| 112 |
+
|
| 113 |
+
def test(self, data, savedir=None):
|
| 114 |
+
# only the 1st input of the whole minibatch would be evaluated
|
| 115 |
+
self._eval()
|
| 116 |
+
self.set_input(data, 'test')
|
| 117 |
+
|
| 118 |
+
if self.data_name is not None and savedir is not None:
|
| 119 |
+
name = os.path.splitext(os.path.basename(self.data_name[0]))[0]
|
| 120 |
+
if not os.path.exists(join(savedir, name)):
|
| 121 |
+
os.makedirs(join(savedir, name))
|
| 122 |
+
|
| 123 |
+
if os.path.exists(join(savedir, name, '{}.png'.format(self.opt.name))):
|
| 124 |
+
return
|
| 125 |
+
|
| 126 |
+
with torch.no_grad():
|
| 127 |
+
output_i, output_j, output_rr = self.forward()
|
| 128 |
+
output_i = tensor2im(output_i)
|
| 129 |
+
output_j = tensor2im(output_j)
|
| 130 |
+
if self.data_name is not None and savedir is not None:
|
| 131 |
+
Image.fromarray(output_i.astype(np.uint8)).save(join(savedir, name, '{}_l.png'.format(self.opt.name)))
|
| 132 |
+
Image.fromarray(output_j.astype(np.uint8)).save(join(savedir, name, '{}_r.png'.format(self.opt.name)))
|
| 133 |
+
Image.fromarray(tensor2im(torch.clip((output_rr + 1) / 2, 0., 1.)).astype(np.uint8)).save(
|
| 134 |
+
join(savedir, name, '{}_rr.png'.format(self.opt.name)))
|
| 135 |
+
Image.fromarray(tensor2im(self.input).astype(np.uint8)).save(join(savedir, name, 'm_input.png'))
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
class DSRNetModel(DSRNetBase):
|
| 139 |
+
def name(self):
|
| 140 |
+
return 'dsrnet'
|
| 141 |
+
|
| 142 |
+
def __init__(self):
|
| 143 |
+
self.epoch = 0
|
| 144 |
+
self.iterations = 0
|
| 145 |
+
self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
| 146 |
+
|
| 147 |
+
def print_network(self):
|
| 148 |
+
print('--------------------- Model ---------------------')
|
| 149 |
+
print('##################### NetG #####################')
|
| 150 |
+
networks.print_network(self.network)
|
| 151 |
+
|
| 152 |
+
def _eval(self):
|
| 153 |
+
self.network.eval()
|
| 154 |
+
|
| 155 |
+
def _train(self):
|
| 156 |
+
self.network.train()
|
| 157 |
+
|
| 158 |
+
def initialize(self, opt):
|
| 159 |
+
BaseModel.initialize(self, opt)
|
| 160 |
+
in_channels = 3
|
| 161 |
+
losses = getattr(models, self.opt.loss)
|
| 162 |
+
self.vgg = losses.Vgg19(requires_grad=False).to(self.device)
|
| 163 |
+
self.network = arch.__dict__[self.opt.inet](in_channels, 3).to(self.device)
|
| 164 |
+
networks.init_weights(self.network, init_type=opt.init_type) # using default initialization as EDSR
|
| 165 |
+
|
| 166 |
+
if self.isTrain:
|
| 167 |
+
# define loss functions
|
| 168 |
+
self.loss_dic = losses.init_loss(opt)
|
| 169 |
+
self.loss_dic['vgg'] = losses.VGGLoss(self.vgg)
|
| 170 |
+
# initialize optimizers
|
| 171 |
+
self.optimizer_G = torch.optim.Adam(self.network.parameters(),
|
| 172 |
+
lr=opt.lr, betas=(0.9, 0.999), weight_decay=opt.wd)
|
| 173 |
+
|
| 174 |
+
self._init_optimizer([self.optimizer_G])
|
| 175 |
+
|
| 176 |
+
if opt.resume:
|
| 177 |
+
self.load(self)
|
| 178 |
+
|
| 179 |
+
if opt.no_verbose is False:
|
| 180 |
+
self.print_network()
|
| 181 |
+
|
| 182 |
+
def get_loss(self, out_t, out_r, out_rr):
|
| 183 |
+
loss_t_pixel = self.loss_dic['t_pixel'](out_t, self.target_t)
|
| 184 |
+
loss_r_pixel = self.loss_dic['r_pixel'](out_r, self.target_r)
|
| 185 |
+
loss_t_vgg = self.loss_dic['vgg'](out_t, self.target_t) * self.opt.lambda_vgg
|
| 186 |
+
loss_exclu = self.loss_dic['exclu'](self.output_t, self.output_r)
|
| 187 |
+
loss_recons = self.loss_dic['recons'](
|
| 188 |
+
self.output_t, self.output_r, self.output_rr, self.input) * self.opt.lambda_rec
|
| 189 |
+
return loss_t_pixel, loss_r_pixel, loss_t_vgg, loss_exclu, loss_recons
|
| 190 |
+
|
| 191 |
+
def backward_G(self):
|
| 192 |
+
self.loss_t_pixel, self.loss_r_pixel, self.loss_t_vgg, \
|
| 193 |
+
self.loss_exclu, self.loss_recons = self.get_loss(self.output_t, self.output_r, self.output_rr)
|
| 194 |
+
self.loss_G = self.loss_t_pixel + self.loss_r_pixel + self.loss_t_vgg + self.loss_exclu + self.loss_recons
|
| 195 |
+
self.loss_G.backward()
|
| 196 |
+
|
| 197 |
+
def forward(self):
|
| 198 |
+
# without edge
|
| 199 |
+
input_i = self.input
|
| 200 |
+
output_t, output_r, output_rr = self.network(input_i,
|
| 201 |
+
self.vgg(input_i),
|
| 202 |
+
fn=self.data_name[0] if self.data_name else None)
|
| 203 |
+
self.output_t = output_t
|
| 204 |
+
self.output_r = output_r
|
| 205 |
+
self.output_rr = output_rr
|
| 206 |
+
return output_t, output_r, output_rr
|
| 207 |
+
|
| 208 |
+
def optimize_parameters(self):
|
| 209 |
+
self._train()
|
| 210 |
+
self.forward()
|
| 211 |
+
|
| 212 |
+
self.optimizer_G.zero_grad()
|
| 213 |
+
self.backward_G()
|
| 214 |
+
|
| 215 |
+
self.optimizer_G.step()
|
| 216 |
+
|
| 217 |
+
def get_current_errors(self):
|
| 218 |
+
ret_errors = OrderedDict()
|
| 219 |
+
if self.loss_r_pixel is not None:
|
| 220 |
+
ret_errors['R_P'] = self.loss_r_pixel.item()
|
| 221 |
+
if self.loss_t_pixel is not None:
|
| 222 |
+
ret_errors['I_P'] = self.loss_t_pixel.item()
|
| 223 |
+
if self.loss_t_vgg is not None:
|
| 224 |
+
ret_errors['VGG'] = self.loss_t_vgg.item()
|
| 225 |
+
if self.loss_exclu is not None:
|
| 226 |
+
ret_errors['Ex'] = self.loss_exclu.item()
|
| 227 |
+
if self.loss_recons is not None:
|
| 228 |
+
ret_errors['Re'] = self.loss_recons.item()
|
| 229 |
+
|
| 230 |
+
ret_errors['lr'] = self.optimizer_G.param_groups[0]['lr']
|
| 231 |
+
ret_errors['seed'] = self.opt.seed
|
| 232 |
+
return ret_errors
|
| 233 |
+
|
| 234 |
+
def get_current_visuals(self):
|
| 235 |
+
ret_visuals = OrderedDict()
|
| 236 |
+
ret_visuals['input'] = tensor2im(self.input).astype(np.uint8)
|
| 237 |
+
ret_visuals['output_t'] = tensor2im(self.output_t).astype(np.uint8)
|
| 238 |
+
ret_visuals['output_r'] = tensor2im(self.output_r).astype(np.uint8)
|
| 239 |
+
ret_visuals['output_rr'] = tensor2im(torch.clip((self.output_rr + 1) / 2, 0., 1.)).astype(np.uint8)
|
| 240 |
+
ret_visuals['target'] = tensor2im(self.target_t).astype(np.uint8)
|
| 241 |
+
ret_visuals['reflection'] = tensor2im(self.target_r).astype(np.uint8)
|
| 242 |
+
|
| 243 |
+
return ret_visuals
|
| 244 |
+
|
| 245 |
+
def load(self, model):
|
| 246 |
+
weight_path = model.opt.weight_path
|
| 247 |
+
state_dict = torch.load(weight_path)
|
| 248 |
+
model.network.load_state_dict(state_dict)
|
| 249 |
+
return state_dict
|
| 250 |
+
|
| 251 |
+
def state_dict(self, save_extra_state=True):
|
| 252 |
+
state_dict = {
|
| 253 |
+
'weight': self.network.state_dict(),
|
| 254 |
+
'epoch': self.epoch,
|
| 255 |
+
'iterations': self.iterations
|
| 256 |
+
}
|
| 257 |
+
|
| 258 |
+
if save_extra_state:
|
| 259 |
+
state_dict.update({
|
| 260 |
+
'opt_g': self.optimizer_G.state_dict()
|
| 261 |
+
})
|
| 262 |
+
|
| 263 |
+
return state_dict
|
DSRNet/models/losses.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
|
| 5 |
+
from models.vgg import Vgg19
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def compute_grad(img):
|
| 9 |
+
gradx = img[..., 1:, :] - img[..., :-1, :]
|
| 10 |
+
grady = img[..., 1:] - img[..., :-1]
|
| 11 |
+
return gradx, grady
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class GradientLoss(nn.Module):
|
| 15 |
+
def __init__(self):
|
| 16 |
+
super(GradientLoss, self).__init__()
|
| 17 |
+
self.loss = nn.L1Loss()
|
| 18 |
+
|
| 19 |
+
def forward(self, predict, target):
|
| 20 |
+
predict_gradx, predict_grady = compute_grad(predict)
|
| 21 |
+
target_gradx, target_grady = compute_grad(target)
|
| 22 |
+
|
| 23 |
+
return self.loss(predict_gradx, target_gradx) + self.loss(predict_grady, target_grady)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class MultipleLoss(nn.Module):
|
| 27 |
+
def __init__(self, losses, weight=None):
|
| 28 |
+
super(MultipleLoss, self).__init__()
|
| 29 |
+
self.losses = nn.ModuleList(losses)
|
| 30 |
+
self.weight = weight or [1 / len(self.losses)] * len(self.losses)
|
| 31 |
+
|
| 32 |
+
def forward(self, predict, target):
|
| 33 |
+
total_loss = 0
|
| 34 |
+
for weight, loss in zip(self.weight, self.losses):
|
| 35 |
+
total_loss += loss(predict, target) * weight
|
| 36 |
+
return total_loss
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class MeanShift(nn.Conv2d):
|
| 40 |
+
def __init__(self, data_mean, data_std, data_range=1, norm=True):
|
| 41 |
+
"""norm (bool): normalize/denormalize the stats"""
|
| 42 |
+
c = len(data_mean)
|
| 43 |
+
super(MeanShift, self).__init__(c, c, kernel_size=1)
|
| 44 |
+
std = torch.Tensor(data_std)
|
| 45 |
+
self.weight.data = torch.eye(c).view(c, c, 1, 1)
|
| 46 |
+
if norm:
|
| 47 |
+
self.weight.data.div_(std.view(c, 1, 1, 1))
|
| 48 |
+
self.bias.data = -1 * data_range * torch.Tensor(data_mean)
|
| 49 |
+
self.bias.data.div_(std)
|
| 50 |
+
else:
|
| 51 |
+
self.weight.data.mul_(std.view(c, 1, 1, 1))
|
| 52 |
+
self.bias.data = data_range * torch.Tensor(data_mean)
|
| 53 |
+
self.requires_grad = False
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class VGGLoss(nn.Module):
|
| 57 |
+
def __init__(self, vgg=None, weights=None, indices=None, normalize=True):
|
| 58 |
+
super(VGGLoss, self).__init__()
|
| 59 |
+
if vgg is None:
|
| 60 |
+
self.vgg = Vgg19().cuda()
|
| 61 |
+
else:
|
| 62 |
+
self.vgg = vgg
|
| 63 |
+
self.criterion = nn.L1Loss()
|
| 64 |
+
self.weights = weights or [1.0 / 2.6, 1.0 / 4.8, 1.0 / 3.7, 1.0 / 5.6, 10 / 1.5]
|
| 65 |
+
self.indices = indices or [2, 7, 12, 21, 30]
|
| 66 |
+
if normalize:
|
| 67 |
+
self.normalize = MeanShift([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], norm=True).cuda()
|
| 68 |
+
else:
|
| 69 |
+
self.normalize = None
|
| 70 |
+
|
| 71 |
+
def forward(self, x, y):
|
| 72 |
+
if self.normalize is not None:
|
| 73 |
+
x = self.normalize(x)
|
| 74 |
+
y = self.normalize(y)
|
| 75 |
+
x_vgg, y_vgg = self.vgg(x, self.indices), self.vgg(y, self.indices)
|
| 76 |
+
loss = 0
|
| 77 |
+
for i in range(len(x_vgg)):
|
| 78 |
+
loss += self.weights[i] * self.criterion(x_vgg[i], y_vgg[i].detach())
|
| 79 |
+
|
| 80 |
+
return loss
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class ReconsLoss(nn.Module):
|
| 84 |
+
def __init__(self):
|
| 85 |
+
super().__init__()
|
| 86 |
+
self.criterion = nn.L1Loss()
|
| 87 |
+
|
| 88 |
+
def forward(self, out_t, out_r, out_rr, input_i):
|
| 89 |
+
content_diff = self.criterion(out_t + out_r + out_rr, input_i)
|
| 90 |
+
return content_diff
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
class ExclusionLoss(nn.Module):
|
| 94 |
+
def __init__(self, level=3, eps=1e-6):
|
| 95 |
+
super().__init__()
|
| 96 |
+
self.level = level
|
| 97 |
+
self.eps = eps
|
| 98 |
+
|
| 99 |
+
def forward(self, img_T, img_R):
|
| 100 |
+
grad_x_loss = []
|
| 101 |
+
grad_y_loss = []
|
| 102 |
+
|
| 103 |
+
for l in range(self.level):
|
| 104 |
+
grad_x_T, grad_y_T = compute_grad(img_T)
|
| 105 |
+
grad_x_R, grad_y_R = compute_grad(img_R)
|
| 106 |
+
|
| 107 |
+
alphax = (2.0 * torch.mean(torch.abs(grad_x_T))) / (torch.mean(torch.abs(grad_x_R)) + self.eps)
|
| 108 |
+
alphay = (2.0 * torch.mean(torch.abs(grad_y_T))) / (torch.mean(torch.abs(grad_y_R)) + self.eps)
|
| 109 |
+
|
| 110 |
+
gradx1_s = (torch.sigmoid(grad_x_T) * 2) - 1 # mul 2 minus 1 is to change sigmoid into tanh
|
| 111 |
+
grady1_s = (torch.sigmoid(grad_y_T) * 2) - 1
|
| 112 |
+
gradx2_s = (torch.sigmoid(grad_x_R * alphax) * 2) - 1
|
| 113 |
+
grady2_s = (torch.sigmoid(grad_y_R * alphay) * 2) - 1
|
| 114 |
+
|
| 115 |
+
grad_x_loss.append(((torch.mean(torch.mul(gradx1_s.pow(2), gradx2_s.pow(2)))) + self.eps) ** 0.25)
|
| 116 |
+
grad_y_loss.append(((torch.mean(torch.mul(grady1_s.pow(2), grady2_s.pow(2)))) + self.eps) ** 0.25)
|
| 117 |
+
|
| 118 |
+
img_T = F.interpolate(img_T, scale_factor=0.5, mode='bilinear')
|
| 119 |
+
img_R = F.interpolate(img_R, scale_factor=0.5, mode='bilinear')
|
| 120 |
+
loss_gradxy = torch.sum(sum(grad_x_loss) / 3) + torch.sum(sum(grad_y_loss) / 3)
|
| 121 |
+
|
| 122 |
+
return loss_gradxy / 2
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def init_loss(opt):
|
| 126 |
+
loss_dic = {}
|
| 127 |
+
pixel_loss = MultipleLoss([nn.MSELoss(), GradientLoss()], [0.3, 0.6])
|
| 128 |
+
loss_dic['t_pixel'] = pixel_loss
|
| 129 |
+
loss_dic['r_pixel'] = pixel_loss
|
| 130 |
+
loss_dic['recons'] = ReconsLoss()
|
| 131 |
+
loss_dic['exclu'] = ExclusionLoss(level=3)
|
| 132 |
+
return loss_dic
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
if __name__ == '__main__':
|
| 136 |
+
x = torch.randn(3, 32, 224, 224).cuda()
|
DSRNet/models/networks.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import functools
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
from torch.nn import init
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
###############################################################################
|
| 10 |
+
# Functions
|
| 11 |
+
###############################################################################
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def weights_init_normal(m):
|
| 15 |
+
classname = m.__class__.__name__
|
| 16 |
+
# print(classname)
|
| 17 |
+
if isinstance(m, nn.Sequential):
|
| 18 |
+
return
|
| 19 |
+
if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)):
|
| 20 |
+
init.normal_(m.weight.data, 0.0, 0.02)
|
| 21 |
+
elif isinstance(m, nn.Linear):
|
| 22 |
+
init.normal_(m.weight.data, 0.0, 0.02)
|
| 23 |
+
elif isinstance(m, nn.BatchNorm2d):
|
| 24 |
+
init.normal_(m.weight.data, 1.0, 0.02)
|
| 25 |
+
init.constant_(m.bias.data, 0.0)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def weights_init_xavier(m):
|
| 29 |
+
classname = m.__class__.__name__
|
| 30 |
+
# print(classname)
|
| 31 |
+
if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)):
|
| 32 |
+
init.xavier_normal_(m.weight.data, gain=0.02)
|
| 33 |
+
elif isinstance(m, nn.Linear):
|
| 34 |
+
init.xavier_normal_(m.weight.data, gain=0.02)
|
| 35 |
+
elif isinstance(m, nn.BatchNorm2d):
|
| 36 |
+
init.normal_(m.weight.data, 1.0, 0.02)
|
| 37 |
+
init.constant_(m.bias.data, 0.0)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def weights_init_kaiming(m):
|
| 41 |
+
classname = m.__class__.__name__
|
| 42 |
+
# print(classname)
|
| 43 |
+
if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)):
|
| 44 |
+
init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
|
| 45 |
+
elif isinstance(m, nn.Linear):
|
| 46 |
+
init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
|
| 47 |
+
elif isinstance(m, nn.BatchNorm2d):
|
| 48 |
+
init.normal_(m.weight.data, 1.0, 0.02)
|
| 49 |
+
init.constant_(m.bias.data, 0.0)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def weights_init_orthogonal(m):
|
| 53 |
+
classname = m.__class__.__name__
|
| 54 |
+
print(classname)
|
| 55 |
+
if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)):
|
| 56 |
+
init.orthogonal(m.weight.data, gain=1)
|
| 57 |
+
elif isinstance(m, nn.Linear):
|
| 58 |
+
init.orthogonal(m.weight.data, gain=1)
|
| 59 |
+
elif isinstance(m, nn.BatchNorm2d):
|
| 60 |
+
init.normal(m.weight.data, 1.0, 0.02)
|
| 61 |
+
init.constant_(m.bias.data, 0.0)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def init_weights(net, init_type='normal'):
|
| 65 |
+
print('[i] initialization method [%s]' % init_type)
|
| 66 |
+
if init_type == 'normal':
|
| 67 |
+
net.apply(weights_init_normal)
|
| 68 |
+
elif init_type == 'xavier':
|
| 69 |
+
net.apply(weights_init_xavier)
|
| 70 |
+
elif init_type == 'kaiming':
|
| 71 |
+
net.apply(weights_init_kaiming)
|
| 72 |
+
elif init_type == 'orthogonal':
|
| 73 |
+
net.apply(weights_init_orthogonal)
|
| 74 |
+
elif init_type == 'edsr':
|
| 75 |
+
pass
|
| 76 |
+
else:
|
| 77 |
+
raise NotImplementedError('initialization method [%s] is not implemented' % init_type)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def get_norm_layer(norm_type='instance'):
|
| 81 |
+
if norm_type == 'batch':
|
| 82 |
+
norm_layer = functools.partial(nn.BatchNorm2d, affine=True)
|
| 83 |
+
elif norm_type == 'instance':
|
| 84 |
+
norm_layer = functools.partial(nn.InstanceNorm2d, affine=False)
|
| 85 |
+
elif norm_type == 'none':
|
| 86 |
+
norm_layer = None
|
| 87 |
+
else:
|
| 88 |
+
raise NotImplementedError('normalization layer [%s] is not found' % norm_type)
|
| 89 |
+
return norm_layer
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def define_D(opt, in_channels=3):
|
| 93 |
+
# use_sigmoid = opt.gan_type == 'gan'
|
| 94 |
+
use_sigmoid = False # incorporate sigmoid into BCE_stable loss
|
| 95 |
+
|
| 96 |
+
if opt.which_model_D == 'disc_vgg':
|
| 97 |
+
netD = Discriminator_VGG(in_channels, use_sigmoid=use_sigmoid)
|
| 98 |
+
init_weights(netD, init_type='kaiming')
|
| 99 |
+
elif opt.which_model_D == 'disc_patch':
|
| 100 |
+
netD = NLayerDiscriminator(in_channels, 64, 3, nn.InstanceNorm2d, use_sigmoid, getIntermFeat=False)
|
| 101 |
+
init_weights(netD, init_type='normal')
|
| 102 |
+
else:
|
| 103 |
+
raise NotImplementedError('%s is not implemented' % opt.which_model_D)
|
| 104 |
+
|
| 105 |
+
if len(opt.gpu_ids) > 0:
|
| 106 |
+
assert (torch.cuda.is_available())
|
| 107 |
+
netD.cuda(opt.gpu_ids[0])
|
| 108 |
+
|
| 109 |
+
return netD
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def print_network(net):
|
| 113 |
+
num_params = 0
|
| 114 |
+
for param in net.parameters():
|
| 115 |
+
num_params += param.numel()
|
| 116 |
+
# print(net)
|
| 117 |
+
print('Network Architecture: ', type(net).__name__)
|
| 118 |
+
print('Total number of parameters: %d,%.3fMb' % (num_params, num_params / (1024 * 1024)))
|
| 119 |
+
print('The size of receptive field: %d' % receptive_field(net))
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def receptive_field(net):
|
| 123 |
+
def _f(output_size, ksize, stride, dilation):
|
| 124 |
+
return (output_size - 1) * stride + ksize * dilation - dilation + 1
|
| 125 |
+
|
| 126 |
+
stats = []
|
| 127 |
+
# print(list(net.modules()))
|
| 128 |
+
for m in net.modules():
|
| 129 |
+
if isinstance(m, nn.Conv2d):
|
| 130 |
+
stats.append((m, m.kernel_size, m.stride, m.dilation))
|
| 131 |
+
|
| 132 |
+
rsize = 1
|
| 133 |
+
for (name, ksize, stride, dilation) in reversed(stats):
|
| 134 |
+
if type(ksize) == tuple: ksize = ksize[0]
|
| 135 |
+
if type(stride) == tuple: stride = stride[0]
|
| 136 |
+
if type(dilation) == tuple: dilation = dilation[0]
|
| 137 |
+
rsize = _f(rsize, ksize, stride, dilation)
|
| 138 |
+
return rsize
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def debug_network(net):
|
| 142 |
+
def _hook(m, i, o):
|
| 143 |
+
print(o.size())
|
| 144 |
+
|
| 145 |
+
for m in net.modules():
|
| 146 |
+
m.register_forward_hook(_hook)
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
##############################################################################
|
| 150 |
+
# Classes
|
| 151 |
+
##############################################################################
|
| 152 |
+
|
| 153 |
+
# Defines the PatchGAN discriminator with the specified arguments.
|
| 154 |
+
class NLayerDiscriminator(nn.Module):
|
| 155 |
+
def __init__(self, input_nc, ndf=64, n_layers=3,
|
| 156 |
+
norm_layer=nn.BatchNorm2d, use_sigmoid=False,
|
| 157 |
+
branch=1, bias=True, getIntermFeat=False):
|
| 158 |
+
super(NLayerDiscriminator, self).__init__()
|
| 159 |
+
self.getIntermFeat = getIntermFeat
|
| 160 |
+
self.n_layers = n_layers
|
| 161 |
+
kw = 4
|
| 162 |
+
padw = int(np.ceil((kw - 1.0) / 2))
|
| 163 |
+
sequence = [[nn.Conv2d(input_nc * branch, ndf * branch, kernel_size=kw, stride=2, padding=padw, groups=branch,
|
| 164 |
+
bias=True), nn.LeakyReLU(0.2, True)]]
|
| 165 |
+
|
| 166 |
+
nf = ndf
|
| 167 |
+
for n in range(1, n_layers):
|
| 168 |
+
nf_prev = nf
|
| 169 |
+
nf = min(nf * 2, 512)
|
| 170 |
+
sequence += [[
|
| 171 |
+
nn.Conv2d(nf_prev * branch, nf * branch, groups=branch, kernel_size=kw, stride=2, padding=padw,
|
| 172 |
+
bias=bias),
|
| 173 |
+
norm_layer(nf * branch), nn.LeakyReLU(0.2, True)
|
| 174 |
+
]]
|
| 175 |
+
|
| 176 |
+
nf_prev = nf
|
| 177 |
+
nf = min(nf * 2, 512)
|
| 178 |
+
sequence += [[
|
| 179 |
+
nn.Conv2d(nf_prev * branch, nf * branch, groups=branch, kernel_size=kw, stride=1, padding=padw, bias=bias),
|
| 180 |
+
norm_layer(nf * branch),
|
| 181 |
+
nn.LeakyReLU(0.2, True)
|
| 182 |
+
]]
|
| 183 |
+
|
| 184 |
+
sequence += [
|
| 185 |
+
[nn.Conv2d(nf * branch, 1 * branch, groups=branch, kernel_size=kw, stride=1, padding=padw, bias=True)]]
|
| 186 |
+
|
| 187 |
+
if use_sigmoid:
|
| 188 |
+
sequence += [[nn.Sigmoid()]]
|
| 189 |
+
|
| 190 |
+
if getIntermFeat:
|
| 191 |
+
for n in range(len(sequence)):
|
| 192 |
+
setattr(self, 'model' + str(n), nn.Sequential(*sequence[n]))
|
| 193 |
+
else:
|
| 194 |
+
sequence_stream = []
|
| 195 |
+
for n in range(len(sequence)):
|
| 196 |
+
sequence_stream += sequence[n]
|
| 197 |
+
self.model = nn.Sequential(*sequence_stream)
|
| 198 |
+
|
| 199 |
+
def forward(self, input):
|
| 200 |
+
if self.getIntermFeat:
|
| 201 |
+
res = [input]
|
| 202 |
+
for n in range(self.n_layers + 2):
|
| 203 |
+
model = getattr(self, 'model' + str(n))
|
| 204 |
+
res.append(model(res[-1]))
|
| 205 |
+
return res[1:]
|
| 206 |
+
else:
|
| 207 |
+
return self.model(input)
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
class Discriminator_VGG(nn.Module):
|
| 211 |
+
def __init__(self, in_channels=3, use_sigmoid=True):
|
| 212 |
+
super(Discriminator_VGG, self).__init__()
|
| 213 |
+
|
| 214 |
+
def conv(*args, **kwargs):
|
| 215 |
+
return nn.Conv2d(*args, **kwargs)
|
| 216 |
+
|
| 217 |
+
num_groups = 32
|
| 218 |
+
|
| 219 |
+
body = [
|
| 220 |
+
conv(in_channels, 64, kernel_size=3, padding=1), # 224
|
| 221 |
+
nn.LeakyReLU(0.2),
|
| 222 |
+
|
| 223 |
+
conv(64, 64, kernel_size=3, stride=2, padding=1), # 112
|
| 224 |
+
nn.GroupNorm(num_groups, 64),
|
| 225 |
+
nn.LeakyReLU(0.2),
|
| 226 |
+
|
| 227 |
+
conv(64, 128, kernel_size=3, padding=1),
|
| 228 |
+
nn.GroupNorm(num_groups, 128),
|
| 229 |
+
nn.LeakyReLU(0.2),
|
| 230 |
+
|
| 231 |
+
conv(128, 128, kernel_size=3, stride=2, padding=1), # 56
|
| 232 |
+
nn.GroupNorm(num_groups, 128),
|
| 233 |
+
nn.LeakyReLU(0.2),
|
| 234 |
+
|
| 235 |
+
conv(128, 256, kernel_size=3, padding=1),
|
| 236 |
+
nn.GroupNorm(num_groups, 256),
|
| 237 |
+
nn.LeakyReLU(0.2),
|
| 238 |
+
|
| 239 |
+
conv(256, 256, kernel_size=3, stride=2, padding=1), # 28
|
| 240 |
+
nn.GroupNorm(num_groups, 256),
|
| 241 |
+
nn.LeakyReLU(0.2),
|
| 242 |
+
|
| 243 |
+
conv(256, 512, kernel_size=3, padding=1),
|
| 244 |
+
nn.GroupNorm(num_groups, 512),
|
| 245 |
+
nn.LeakyReLU(0.2),
|
| 246 |
+
|
| 247 |
+
conv(512, 512, kernel_size=3, stride=2, padding=1), # 14
|
| 248 |
+
nn.GroupNorm(num_groups, 512),
|
| 249 |
+
nn.LeakyReLU(0.2),
|
| 250 |
+
|
| 251 |
+
conv(512, 512, kernel_size=3, stride=1, padding=1),
|
| 252 |
+
nn.GroupNorm(num_groups, 512),
|
| 253 |
+
nn.LeakyReLU(0.2),
|
| 254 |
+
|
| 255 |
+
conv(512, 512, kernel_size=3, stride=2, padding=1), # 7
|
| 256 |
+
nn.GroupNorm(num_groups, 512),
|
| 257 |
+
nn.LeakyReLU(0.2),
|
| 258 |
+
]
|
| 259 |
+
|
| 260 |
+
tail = [
|
| 261 |
+
nn.AdaptiveAvgPool2d(1),
|
| 262 |
+
nn.Conv2d(512, 1024, kernel_size=1),
|
| 263 |
+
nn.LeakyReLU(0.2),
|
| 264 |
+
nn.Conv2d(1024, 1, kernel_size=1)
|
| 265 |
+
]
|
| 266 |
+
|
| 267 |
+
if use_sigmoid:
|
| 268 |
+
tail.append(nn.Sigmoid())
|
| 269 |
+
|
| 270 |
+
self.body = nn.Sequential(*body)
|
| 271 |
+
self.tail = nn.Sequential(*tail)
|
| 272 |
+
|
| 273 |
+
def forward(self, x):
|
| 274 |
+
x = self.body(x)
|
| 275 |
+
out = self.tail(x)
|
| 276 |
+
return out
|
DSRNet/models/vgg.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from collections import namedtuple
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from torchvision import models
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class Vgg16(torch.nn.Module):
|
| 8 |
+
def __init__(self, requires_grad=False):
|
| 9 |
+
super(Vgg16, self).__init__()
|
| 10 |
+
vgg_pretrained_features = models.vgg16(pretrained=True).features
|
| 11 |
+
self.slice1 = torch.nn.Sequential()
|
| 12 |
+
self.slice2 = torch.nn.Sequential()
|
| 13 |
+
self.slice3 = torch.nn.Sequential()
|
| 14 |
+
self.slice4 = torch.nn.Sequential()
|
| 15 |
+
for x in range(4):
|
| 16 |
+
self.slice1.add_module(str(x), vgg_pretrained_features[x])
|
| 17 |
+
for x in range(4, 9):
|
| 18 |
+
self.slice2.add_module(str(x), vgg_pretrained_features[x])
|
| 19 |
+
for x in range(9, 16):
|
| 20 |
+
self.slice3.add_module(str(x), vgg_pretrained_features[x])
|
| 21 |
+
for x in range(16, 23):
|
| 22 |
+
self.slice4.add_module(str(x), vgg_pretrained_features[x])
|
| 23 |
+
if not requires_grad:
|
| 24 |
+
for param in self.parameters():
|
| 25 |
+
param.requires_grad = False
|
| 26 |
+
|
| 27 |
+
def forward(self, X):
|
| 28 |
+
h = self.slice1(X)
|
| 29 |
+
h_relu1_2 = h
|
| 30 |
+
h = self.slice2(h)
|
| 31 |
+
h_relu2_2 = h
|
| 32 |
+
h = self.slice3(h)
|
| 33 |
+
h_relu3_3 = h
|
| 34 |
+
h = self.slice4(h)
|
| 35 |
+
h_relu4_3 = h
|
| 36 |
+
vgg_outputs = namedtuple("VggOutputs", ['relu1_2', 'relu2_2', 'relu3_3', 'relu4_3'])
|
| 37 |
+
out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3)
|
| 38 |
+
return out
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class Vgg19(torch.nn.Module):
|
| 42 |
+
def __init__(self, requires_grad=False):
|
| 43 |
+
super(Vgg19, self).__init__()
|
| 44 |
+
# vgg_pretrained_features = models.vgg19(pretrained=True).features
|
| 45 |
+
self.vgg_pretrained_features = models.vgg19(pretrained=True).features
|
| 46 |
+
# self.slice1 = torch.nn.Sequential()
|
| 47 |
+
# self.slice2 = torch.nn.Sequential()
|
| 48 |
+
# self.slice3 = torch.nn.Sequential()
|
| 49 |
+
# self.slice4 = torch.nn.Sequential()
|
| 50 |
+
# self.slice5 = torch.nn.Sequential()
|
| 51 |
+
# for x in range(2):
|
| 52 |
+
# self.slice1.add_module(str(x), vgg_pretrained_features[x])
|
| 53 |
+
# for x in range(2, 7):
|
| 54 |
+
# self.slice2.add_module(str(x), vgg_pretrained_features[x])
|
| 55 |
+
# for x in range(7, 12):
|
| 56 |
+
# self.slice3.add_module(str(x), vgg_pretrained_features[x])
|
| 57 |
+
# for x in range(12, 21):
|
| 58 |
+
# self.slice4.add_module(str(x), vgg_pretrained_features[x])
|
| 59 |
+
# for x in range(21, 30):
|
| 60 |
+
# self.slice5.add_module(str(x), vgg_pretrained_features[x])
|
| 61 |
+
if not requires_grad:
|
| 62 |
+
for param in self.parameters():
|
| 63 |
+
param.requires_grad = False
|
| 64 |
+
|
| 65 |
+
def forward(self, X, indices=None):
|
| 66 |
+
if indices is None:
|
| 67 |
+
indices = [2, 7, 12, 21, 30]
|
| 68 |
+
out = []
|
| 69 |
+
# indices = sorted(indices)
|
| 70 |
+
for i in range(indices[-1]):
|
| 71 |
+
X = self.vgg_pretrained_features[i](X)
|
| 72 |
+
if (i + 1) in indices:
|
| 73 |
+
out.append(X)
|
| 74 |
+
|
| 75 |
+
return out
|
| 76 |
+
|
| 77 |
+
# h_relu1 = self.slice1(X)
|
| 78 |
+
# h_relu2 = self.slice2(h_relu1)
|
| 79 |
+
# h_relu3 = self.slice3(h_relu2)
|
| 80 |
+
# h_relu4 = self.slice4(h_relu3)
|
| 81 |
+
# h_relu5 = self.slice5(h_relu4)
|
| 82 |
+
# out = [h_relu1, h_relu2, h_relu3, h_relu4, h_relu5]
|
| 83 |
+
# return out
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
if __name__ == '__main__':
|
| 87 |
+
vgg = Vgg19()
|
| 88 |
+
import ipdb
|
| 89 |
+
|
| 90 |
+
ipdb.set_trace()
|
DSRNet/options/__init__.py
ADDED
|
File without changes
|
DSRNet/options/base_option.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#Contains a base class that defines common options used by both training and testing.
|
| 2 |
+
|
| 3 |
+
import argparse #standard library for parsing command-line arguments.
|
| 4 |
+
import models #creates a module object.
|
| 5 |
+
#models.__dict__ is a dictionary that holds the namespace of the models module. This dictionary contains mappings of all names defined within the module e.g. all the symbols (functions, classes, variables, etc.) defined in that module. Stores key and value
|
| 6 |
+
model_names = sorted(name for name in models.__dict__
|
| 7 |
+
if name.islower() and not name.startswith("__")
|
| 8 |
+
and callable(models.__dict__[name]))
|
| 9 |
+
#convention is camelcase for Classes or Exceptions
|
| 10 |
+
#__starting names are magic/dunder methods:the methods starting and ending with double underscores ‘__’. They are defined by built-in classes in Python and commonly used for operator overloading
|
| 11 |
+
#callalble() means it can be called like a function.
|
| 12 |
+
# so left with functioons and callable classes
|
| 13 |
+
|
| 14 |
+
class BaseOptions():
|
| 15 |
+
def __init__(self):
|
| 16 |
+
self.parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) #formatter_class=how help messages should be formatted # argparse.ArgumentDefaultsHelpFormatter: help output should include default values for arguments.
|
| 17 |
+
self.initialized = False #flag for tracking whether the initialize() method has been called to prevent reinitialization
|
| 18 |
+
def initialize(self): # all the arguments added to parser
|
| 19 |
+
# experiment specifics
|
| 20 |
+
self.parser.add_argument('--name', type=str, default=None,
|
| 21 |
+
help='name of the experiment. It decides where to store samples and models')
|
| 22 |
+
self.parser.add_argument('--gpu_ids', type=str, default='0', help='gpu ids: e.g. 0 0,1,2, 0,2. use -1 for CPU')
|
| 23 |
+
self.parser.add_argument('--model', type=str, default='errnet_model', help='chooses which model to use.')
|
| 24 |
+
self.parser.add_argument('--dataset', '-d', type=str, default='sirs_dataset',
|
| 25 |
+
help='chooses which dataset to use.')
|
| 26 |
+
self.parser.add_argument('--loss', '-l', type=str, default='losses',
|
| 27 |
+
help='chooses which loss to use.')
|
| 28 |
+
self.parser.add_argument('--checkpoints_dir', type=str, default='./checkpoints', help='models are saved here')
|
| 29 |
+
self.parser.add_argument('--resume', '-r', action='store_true', help='resume from checkpoint')
|
| 30 |
+
self.parser.add_argument('--resume_epoch', '-re', type=int, default=None,
|
| 31 |
+
help='checkpoint to use. (default: latest')
|
| 32 |
+
self.parser.add_argument('--seed', type=int, default=0, help='random seed to use. Default=0')
|
| 33 |
+
self.parser.add_argument('--supp_eval', action='store_true', help='supplementary evaluation')
|
| 34 |
+
self.parser.add_argument('--start_now', action='store_true', help='supplementary evaluation')
|
| 35 |
+
self.parser.add_argument('--testr', action='store_true', help='test for reflections')
|
| 36 |
+
self.parser.add_argument('--select', type=str, default=None)
|
| 37 |
+
|
| 38 |
+
# for setting input
|
| 39 |
+
self.parser.add_argument('--serial_batches', action='store_true',
|
| 40 |
+
help='if true, takes images in order to make batches, otherwise takes them randomly')
|
| 41 |
+
self.parser.add_argument('--nThreads', default=8, type=int, help='# threads for loading data')
|
| 42 |
+
self.parser.add_argument('--max_dataset_size', type=int, default=None,
|
| 43 |
+
help='Maximum number of samples allowed per dataset. If the dataset directory contains more than max_dataset_size, only a subset is loaded.')
|
| 44 |
+
|
| 45 |
+
#(mingcv) for display
|
| 46 |
+
self.parser.add_argument('--no-log', action='store_true', help='disable tf logger?')
|
| 47 |
+
self.parser.add_argument('--no-verbose', action='store_true', help='disable verbose info?')
|
| 48 |
+
self.parser.add_argument('--display_winsize', type=int, default=256, help='display window size')
|
| 49 |
+
self.parser.add_argument('--display_port', type=int, default=8097, help='visdom port of the web display')
|
| 50 |
+
self.parser.add_argument('--display_id', type=int, default=0,
|
| 51 |
+
help='window id of the web display (use 0 to disable visdom)')
|
| 52 |
+
#In Visdom, you can create multiple windows to display various types of information, such as images, graphs, or tables, during the training and testing of machine learning models. The display_id specifies which window (by ID) the visualizations should be displayed in.
|
| 53 |
+
self.parser.add_argument('--display_single_pane_ncols', type=int, default=0,
|
| 54 |
+
help='if positive, display all images in a single visdom web panel with certain number of images per row.')
|
| 55 |
+
|
| 56 |
+
self.initialized = True
|
DSRNet/options/net_options/__init__.py
ADDED
|
File without changes
|
DSRNet/options/net_options/base_options.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#configuration based on neural network
|
| 2 |
+
#action=store_true: presence of this argument in the command line will set the corresponding variable to True. If the argument is not specified, the variable will be set to False.Opposite of store_false
|
| 3 |
+
import sys
|
| 4 |
+
|
| 5 |
+
from options.base_option import BaseOptions as Base
|
| 6 |
+
from util import util
|
| 7 |
+
import os
|
| 8 |
+
import torch
|
| 9 |
+
import numpy as np
|
| 10 |
+
import random
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class BaseOptions(Base):
|
| 14 |
+
def initialize(self):
|
| 15 |
+
Base.initialize(self)
|
| 16 |
+
# experiment specifics
|
| 17 |
+
self.parser.add_argument('--inet', type=str, default='net_options',
|
| 18 |
+
help='chooses which architecture to use for inet.')
|
| 19 |
+
self.parser.add_argument('--weight_path', type=str, default=None, help='pretrained checkpoint to use.')
|
| 20 |
+
#Path to a pretrained checkpoint for initialization.
|
| 21 |
+
self.parser.add_argument('--init_type', type=str, default='edsr',
|
| 22 |
+
help='network initialization [normal|xavier|kaiming|orthogonal|uniform]')
|
| 23 |
+
#(mingcv)for network
|
| 24 |
+
self.parser.add_argument('--hyper', action='store_true',
|
| 25 |
+
help='if true, augment input with vgg hypercolumn feature')
|
| 26 |
+
# A hypercolumn refers to a vector formed by concatenating the outputs of all feature maps at a particular spatial location across different layers of the VGG network.
|
| 27 |
+
#Benefits: the network can leverage both low-level details and high-level semantic information simultaneously.provides a richer representation of the input image.
|
| 28 |
+
#Implementation: During the forward pass of the neural network:
|
| 29 |
+
#1.Extract feature maps from various layers of the VGG network.
|
| 30 |
+
#2.Concatenate these feature maps spatially (typically by resizing them to a common size).
|
| 31 |
+
#3.Use this concatenated feature vector as an augmented input alongside the original input to the neural network.
|
| 32 |
+
|
| 33 |
+
self.initialized = True
|
| 34 |
+
|
| 35 |
+
def parse(self):
|
| 36 |
+
if not self.initialized:
|
| 37 |
+
self.initialize()
|
| 38 |
+
self.opt = self.parser.parse_args() #Stores parsed command-line arguments.
|
| 39 |
+
self.opt.isTrain = self.isTrain # (mingcv)train or test
|
| 40 |
+
#to use train_options.py or not
|
| 41 |
+
if self.opt.seed == 0:#for reproducibility across modules.
|
| 42 |
+
seed = random.randrange(2 ** 12 - 1)
|
| 43 |
+
self.opt.seed = seed
|
| 44 |
+
|
| 45 |
+
torch.backends.cudnn.deterministic = True
|
| 46 |
+
#PyTorch uses CUDA libraries (like cuDNN) for GPU-accelerated computations. Setting torch.backends.cudnn.deterministic to True ensures that cuDNN uses deterministic algorithms. Operations that rely on cuDNN (like certain convolution operations) will produce the same results on the same input data and configuration.
|
| 47 |
+
torch.manual_seed(self.opt.seed)#Ensure that operations like initializing weights in neural networks or shuffling data batches produce the same results
|
| 48 |
+
np.random.seed(self.opt.seed) #(mingcv) seed for every module
|
| 49 |
+
#np.random.seed:seed for the random number generator in NumPy
|
| 50 |
+
random.seed(self.opt.seed)#seed for the built-in Python random number generator
|
| 51 |
+
#gpu_ids: for multigpu system for parallelization e.g. 0,1,2
|
| 52 |
+
str_ids = self.opt.gpu_ids.split(',')
|
| 53 |
+
self.opt.gpu_ids = []
|
| 54 |
+
for str_id in str_ids:
|
| 55 |
+
id = int(str_id)
|
| 56 |
+
if id >= 0:
|
| 57 |
+
self.opt.gpu_ids.append(id)
|
| 58 |
+
#converting string of gpus ids into a list of ints
|
| 59 |
+
#(mingcv)set gpu ids
|
| 60 |
+
if len(self.opt.gpu_ids) > 0: #if atleast 1 gpu, set first gpu as cuda device
|
| 61 |
+
torch.cuda.set_device(self.opt.gpu_ids[0])
|
| 62 |
+
|
| 63 |
+
args = vars(self.opt)
|
| 64 |
+
#vars() is a built-in function that returns the __dict__ attribute of an object if it exists.
|
| 65 |
+
# args = vars(self.opt) converts the attributes of self.opt into a dictionary args for iteration and accessibility
|
| 66 |
+
print('------------ Options -------------')
|
| 67 |
+
for k, v in sorted(args.items()): # prints all keys and values of dict args
|
| 68 |
+
print('%s: %s' % (str(k), str(v)))
|
| 69 |
+
print('-------------- End ----------------')
|
| 70 |
+
|
| 71 |
+
#(mingcv) save to the disk
|
| 72 |
+
self.opt.name = self.opt.name or '_'.join([self.opt.model])
|
| 73 |
+
# '_'.join([self.opt.model]) if self.opt.name is None
|
| 74 |
+
expr_dir = os.path.join(self.opt.checkpoints_dir, self.opt.name)
|
| 75 |
+
#experiment directory
|
| 76 |
+
util.mkdirs(expr_dir)
|
| 77 |
+
file_name = os.path.join(expr_dir, 'opt.txt')
|
| 78 |
+
with open(file_name, 'wt') as opt_file:
|
| 79 |
+
opt_file.write('------------ Options -------------\n') # write key and values of options in opt.txt
|
| 80 |
+
for k, v in sorted(args.items()):
|
| 81 |
+
opt_file.write('%s: %s\n' % (str(k), str(v)))
|
| 82 |
+
opt_file.write('-------------- End ----------------\n')
|
| 83 |
+
|
| 84 |
+
if self.opt.debug: #debugging mode with small dataset no multithreading,no flipping
|
| 85 |
+
self.opt.display_freq = 20 #display: visual outputs related to training like plots/images
|
| 86 |
+
self.opt.print_freq = 20 #print: training progress information printed in console
|
| 87 |
+
#after processing 20 batches of images, the training results will be printed/displayed.
|
| 88 |
+
self.opt.nEpochs = 40
|
| 89 |
+
self.opt.max_dataset_size = 100
|
| 90 |
+
self.opt.no_log = False
|
| 91 |
+
# controls whether logging of training progress or other information is enabled (False means logging is enabled).
|
| 92 |
+
self.opt.nThreads = 0 #operations are executed in a single thread
|
| 93 |
+
self.opt.decay_iter = 0 # iteration count after which learning rate decay might occur. 0 means no decay
|
| 94 |
+
self.opt.serial_batches = True #whether batches of data are processed in serial (one after another) or in parallel during training. serial:each sample is seen exactly once per epoch in the specified order.
|
| 95 |
+
self.opt.no_flip = True # disables flipping of images during data augmentation.
|
| 96 |
+
|
| 97 |
+
return self.opt
|
DSRNet/options/net_options/train_options.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .base_options import BaseOptions #. means look in the same directory(net_options)
|
| 2 |
+
|
| 3 |
+
#action=store_true: presence of this argument in the command line will set the corresponding variable to True. If the argument is not specified, the variable will be set to False.Opposite of store_false
|
| 4 |
+
class TrainOptions(BaseOptions):
|
| 5 |
+
def initialize(self):
|
| 6 |
+
BaseOptions.initialize(self)
|
| 7 |
+
# (mingcv)for displays
|
| 8 |
+
self.parser.add_argument('--display_freq', type=int, default=100,
|
| 9 |
+
help='frequency of showing training results on screen')
|
| 10 |
+
self.parser.add_argument('--update_html_freq', type=int, default=1000,
|
| 11 |
+
help='frequency of saving training results to html')
|
| 12 |
+
self.parser.add_argument('--print_freq', type=int, default=100,
|
| 13 |
+
help='frequency of showing training results on console')
|
| 14 |
+
self.parser.add_argument('--eval_freq', type=int, default=1, help='frequency of evaluation')
|
| 15 |
+
self.parser.add_argument('--save_freq', type=int, default=1, help='frequency of save eval samples')
|
| 16 |
+
self.parser.add_argument('--no_html', action='store_true',
|
| 17 |
+
help='do not save intermediate training results to [opt.checkpoints_dir]/[opt.name]/web/')
|
| 18 |
+
self.parser.add_argument('--save_epoch_freq', type=int, default=1,
|
| 19 |
+
help='frequency of saving checkpoints at the end of epochs')
|
| 20 |
+
self.parser.add_argument('--debug', action='store_true',
|
| 21 |
+
help='only do one epoch and displays at each iteration')
|
| 22 |
+
self.parser.add_argument('--finetune', action='store_true',
|
| 23 |
+
help='finetune the network using identity inputs and outputs')
|
| 24 |
+
#Fine-tuning is a process where a pre-trained model is further trained on a new dataset or task.
|
| 25 |
+
'''???identity inputs and outputs:'''
|
| 26 |
+
self.parser.add_argument('--if_align', action='store_true', help='if align 4x')
|
| 27 |
+
#adjusting images so that they meet specific criteria such as size, orientation, or position relative to a reference frame.
|
| 28 |
+
'''??? here 4x what?'''
|
| 29 |
+
self.parser.add_argument('--weight_dir', type=str, default='')
|
| 30 |
+
self.parser.add_argument('--base_dir', type=str, default='/mnt/lmj_ssd/reflection-removal')
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
self.parser.add_argument('--nEpochs', '-n', type=int, default=60, help='# of epochs to run')
|
| 34 |
+
#when defining the Adam optimizer
|
| 35 |
+
self.parser.add_argument('--lr', type=float, default=1e-4, help='initial learning rate for adam')
|
| 36 |
+
self.parser.add_argument('--wd', type=float, default=0, help='weight decay for adam')
|
| 37 |
+
self.parser.add_argument('--num_train', type=int, default=-1) #Number of training samples.
|
| 38 |
+
self.parser.add_argument('--r_pixel_weight', '-rw', type=float, default=1.0, help='weight for r_pixel loss')
|
| 39 |
+
|
| 40 |
+
# (mingcv)data augmentation
|
| 41 |
+
self.parser.add_argument('--real20_size', type=int, default=420, help='scale images to compat size')
|
| 42 |
+
self.parser.add_argument('--batchSize', '-b', type=int, default=1, help='input batch size')
|
| 43 |
+
self.parser.add_argument('--loadSize', type=str, default='224,336,448', help='scale images to multiple size')
|
| 44 |
+
self.parser.add_argument('--fineSize', type=str, default='224,224', help='then crop to this size')
|
| 45 |
+
self.parser.add_argument('--no_flip', action='store_true',
|
| 46 |
+
help='if specified, do not flip the images for data augmentation')
|
| 47 |
+
self.parser.add_argument('--resize_or_crop', type=str, default='resize_and_crop',
|
| 48 |
+
help='scaling and cropping of images at load time [resize_and_crop|crop|scale_width|scale_width_and_crop]')
|
| 49 |
+
self.parser.add_argument('--debug_eval', action='store_true',
|
| 50 |
+
help='if specified, do not flip the images for data augmentation')
|
| 51 |
+
self.parser.add_argument('--graph', action='store_true', help='print graph')
|
| 52 |
+
self.parser.add_argument('--selected', type=str, nargs='+')
|
| 53 |
+
#nargs='+': the argument can take one or more values.
|
| 54 |
+
|
| 55 |
+
# (mingcv)for discriminator
|
| 56 |
+
self.parser.add_argument('--which_model_D', type=str, default='disc_vgg', choices=['disc_vgg', 'disc_patch'])
|
| 57 |
+
self.parser.add_argument('--gan_type', type=str, default='rasgan',
|
| 58 |
+
help='gan/sgan : Vanilla GAN; rasgan : relativistic gan')
|
| 59 |
+
|
| 60 |
+
# (mingcv) loss weight
|
| 61 |
+
self.parser.add_argument('--unaligned_loss', type=str, default='vgg',
|
| 62 |
+
help='learning rate policy: vgg|mse|ctx|ctx_vgg')
|
| 63 |
+
#unaligned_loss: loss function used to measure the difference between predicted and target images that are not perfectly aligned.
|
| 64 |
+
#vgg: Perceptual Loss/vgg loss.Uses features extracted from a pre-trained network (e.g., VGG) to measure differences in high-level features instead of raw pixel values.
|
| 65 |
+
#mse:Mean Squared Error.Typically for aligned data, but might be included for certain aligned parts or just for comparison.
|
| 66 |
+
#ctx: Contextual loss. Focuses on the context and structure rather than pixel-wise differences.When structural similarity more important
|
| 67 |
+
#ctx_vgg: combination of contextual and VGG perceptual loss.
|
| 68 |
+
self.parser.add_argument('--vgg_layer', type=int, default=31, help='vgg layer of unaligned loss')
|
| 69 |
+
#--vgg_layer:a specific layer from the VGG network (a pre-trained deep convolutional neural network) used to extract high-level features from images to compute the perceptual loss. If u use vgg/ctx_vgg in --unaligned_loss
|
| 70 |
+
self.parser.add_argument('--init_lr', type=float, default=1e-2, help='initial learning rate')
|
| 71 |
+
self.parser.add_argument('--fixed_lr', type=float, default=0, help='initial learning rate')
|
| 72 |
+
"??? default=0 becuse of scheduler?"
|
| 73 |
+
self.parser.add_argument('--lambda_gan', type=float, default=0.01, help='weight for gan loss')
|
| 74 |
+
self.parser.add_argument('--lambda_vgg', type=float, default=0.1, help='weight for vgg loss')
|
| 75 |
+
self.parser.add_argument('--lambda_rec', type=float, default=0.2, help='weight for reconstruction loss')
|
| 76 |
+
|
| 77 |
+
self.isTrain = True #indicating that these options are for training.
|
DSRNet/requirements.txt
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dominate==2.9.1
|
| 2 |
+
einops==0.7.0
|
| 3 |
+
kornia==0.7.1
|
| 4 |
+
opencv-python==4.9.0.80
|
| 5 |
+
PyYAML==6.0.1
|
| 6 |
+
scikit-image==0.22.0
|
| 7 |
+
scipy==1.12.0
|
| 8 |
+
tensorboardX==2.6.2.2
|
| 9 |
+
torch
|
| 10 |
+
torchvision
|
| 11 |
+
visdom==0.2.4
|
DSRNet/scripts.sh
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
### To train the model
|
| 2 |
+
python train_sirs.py --inet dsrnet_l --model dsrnet_model_sirs --dataset sirs_dataset --loss losses --name dsrnet_l --lambda_vgg 0.01 --lambda_rec 0.2 --if_align --seed 2018 --base_dir "[YOUR_DATA_DIR]"
|
| 3 |
+
|
| 4 |
+
python eval_sirs.py --inet dsrnet_s --model dsrnet_model_sirs --dataset sirs_dataset --name dsrnet_s_test --hyper --if_align --resume --weight_path "./weights/dsrnet_s_epoch14.pt" --base_dir "[YOUR_DATA_DIR]"
|
| 5 |
+
|
| 6 |
+
python eval_sirs.py --inet dsrnet_l --model dsrnet_model_sirs --dataset sirs_dataset --name dsrnet_l_test --hyper --if_align --resume --weight_path "./weights/dsrnet_l_epoch18.pt" --base_dir "[YOUR_DATA_DIR]"
|
DSRNet/test_sirs.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from os.path import join
|
| 3 |
+
|
| 4 |
+
import torch.backends.cudnn as cudnn
|
| 5 |
+
|
| 6 |
+
import data.sirs_dataset as datasets
|
| 7 |
+
from data.image_folder import read_fns
|
| 8 |
+
from engine import Engine
|
| 9 |
+
from options.net_options.train_options import TrainOptions
|
| 10 |
+
from tools import mutils
|
| 11 |
+
|
| 12 |
+
opt = TrainOptions().parse()
|
| 13 |
+
|
| 14 |
+
opt.isTrain = False
|
| 15 |
+
cudnn.benchmark = True
|
| 16 |
+
opt.no_log = True
|
| 17 |
+
opt.display_id = 0
|
| 18 |
+
opt.verbose = False
|
| 19 |
+
|
| 20 |
+
datadir = os.path.join(opt.base_dir, 'test')
|
| 21 |
+
test_dataset_real = datasets.DSRTestDataset(join(datadir, f'real20_420'),
|
| 22 |
+
fns=read_fns('data/real_test.txt'),
|
| 23 |
+
if_align=opt.if_align)
|
| 24 |
+
#test_dataset_real = datasets.RealDataset('./data')
|
| 25 |
+
test_dataloader_real = datasets.DataLoader(test_dataset_real, batch_size=1, shuffle=True, num_workers=opt.nThreads,
|
| 26 |
+
pin_memory=True)
|
| 27 |
+
|
| 28 |
+
engine = Engine(opt)
|
| 29 |
+
|
| 30 |
+
"""Main Loop"""
|
| 31 |
+
result_dir = os.path.join('./checkpoints', opt.name, mutils.get_formatted_time())
|
| 32 |
+
|
| 33 |
+
res = engine.test(test_dataloader_real, savedir=join(result_dir, 'test'))
|
DSRNet/tools/__init__.py
ADDED
|
File without changes
|
DSRNet/tools/mutils.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import datetime
|
| 2 |
+
import os
|
| 3 |
+
import shutil
|
| 4 |
+
import time
|
| 5 |
+
|
| 6 |
+
#counts number of trainable parameters
|
| 7 |
+
#same as net_utils but different print
|
| 8 |
+
def count_parameters(model):
|
| 9 |
+
number_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
|
| 10 |
+
print('Total Parameters: {:d} or {:.2f}M'.format(number_params, number_params / (1024 * 1024))) #in millions
|
| 11 |
+
return number_params
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def contains(key, lst): #return true if lst contains the key
|
| 15 |
+
flag = False
|
| 16 |
+
for item in lst:
|
| 17 |
+
if key == item:
|
| 18 |
+
flag = True
|
| 19 |
+
return flag
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def make_empty_dir(new_dir): #deletes previous directory containing files and makes new empty one
|
| 23 |
+
if os.path.exists(new_dir):
|
| 24 |
+
shutil.rmtree(new_dir)
|
| 25 |
+
os.makedirs(new_dir, exist_ok=True)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def get_timestamp():
|
| 29 |
+
return str(time.time()).replace('.', '') #returns time as secondsmilliseconds vs seconds.milliseconds can be used timestamp identifier
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def get_formatted_time(): #returns current time formatted as YYYYMMDD-HHMMSS
|
| 33 |
+
return datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
if __name__ == '__main__': #if the script is being run directly (as opposed to being imported as a module).
|
| 37 |
+
pass
|
DSRNet/train_sirs.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from os.path import join
|
| 3 |
+
|
| 4 |
+
import data.sirs_dataset as datasets # Custom dataset module for SIRS dataset
|
| 5 |
+
import util.util as util
|
| 6 |
+
from data.image_folder import read_fns
|
| 7 |
+
from engine import Engine
|
| 8 |
+
from options.net_options.train_options import TrainOptions
|
| 9 |
+
from tools import mutils
|
| 10 |
+
|
| 11 |
+
opt = TrainOptions().parse()
|
| 12 |
+
print(opt)
|
| 13 |
+
|
| 14 |
+
opt.display_freq = 10
|
| 15 |
+
|
| 16 |
+
if opt.debug:
|
| 17 |
+
opt.display_id = 1
|
| 18 |
+
opt.display_freq = 1
|
| 19 |
+
opt.print_freq = 20
|
| 20 |
+
opt.nEpochs = 40
|
| 21 |
+
opt.max_dataset_size = 100 #was 9999 (mingcv)
|
| 22 |
+
opt.no_log = False
|
| 23 |
+
opt.nThreads = 0
|
| 24 |
+
opt.decay_iter = 0
|
| 25 |
+
opt.serial_batches = True
|
| 26 |
+
opt.no_flip = True
|
| 27 |
+
|
| 28 |
+
# modify the following code to
|
| 29 |
+
datadir = os.path.join(opt.base_dir)
|
| 30 |
+
datadir_syn = join(datadir, 'train/VOCdevkit/VOC2012/PNGImages')
|
| 31 |
+
datadir_real = join(datadir, 'train/real')
|
| 32 |
+
|
| 33 |
+
train_dataset = datasets.DSRDataset(
|
| 34 |
+
datadir_syn, read_fns('data/VOC2012_224_train_png.txt'), size=opt.max_dataset_size, enable_transforms=True)
|
| 35 |
+
|
| 36 |
+
train_dataset_real = datasets.DSRTestDataset(datadir_real, enable_transforms=True, if_align=opt.if_align)
|
| 37 |
+
|
| 38 |
+
train_dataset_fusion = datasets.FusionDataset([train_dataset,
|
| 39 |
+
train_dataset_real], [0.7, 0.3])
|
| 40 |
+
|
| 41 |
+
train_dataloader_fusion = datasets.DataLoader(
|
| 42 |
+
train_dataset_fusion, batch_size=opt.batchSize, shuffle=not opt.serial_batches,
|
| 43 |
+
pin_memory=True, prefetch_factor=32, num_workers=32)
|
| 44 |
+
|
| 45 |
+
eval_dataset_real = datasets.DSRTestDataset(join(datadir, f'test/real20_{opt.real20_size}'),
|
| 46 |
+
fns=read_fns('data/real_test.txt'), if_align=opt.if_align)
|
| 47 |
+
eval_dataset_solidobject = datasets.DSRTestDataset(join(datadir, 'test/SIR2/SolidObjectDataset'),
|
| 48 |
+
if_align=opt.if_align)
|
| 49 |
+
eval_dataset_postcard = datasets.DSRTestDataset(join(datadir, 'test/SIR2/PostcardDataset'), if_align=opt.if_align)
|
| 50 |
+
eval_dataset_wild = datasets.DSRTestDataset(join(datadir, 'test/SIR2/WildSceneDataset'), if_align=opt.if_align)
|
| 51 |
+
|
| 52 |
+
eval_dataloader_real = datasets.DataLoader(
|
| 53 |
+
eval_dataset_real, batch_size=1, shuffle=False,
|
| 54 |
+
pin_memory=True, prefetch_factor=32, num_workers=32)
|
| 55 |
+
|
| 56 |
+
eval_dataloader_solidobject = datasets.DataLoader(
|
| 57 |
+
eval_dataset_solidobject, batch_size=1, shuffle=False,
|
| 58 |
+
pin_memory=True, prefetch_factor=32, num_workers=32)
|
| 59 |
+
eval_dataloader_postcard = datasets.DataLoader(
|
| 60 |
+
eval_dataset_postcard, batch_size=1, shuffle=False,
|
| 61 |
+
pin_memory=True, prefetch_factor=32, num_workers=32)
|
| 62 |
+
|
| 63 |
+
eval_dataloader_wild = datasets.DataLoader(
|
| 64 |
+
eval_dataset_wild, batch_size=1, shuffle=False,
|
| 65 |
+
pin_memory=True, prefetch_factor=32, num_workers=32)
|
| 66 |
+
|
| 67 |
+
"""Main Loop"""
|
| 68 |
+
engine = Engine(opt)
|
| 69 |
+
result_dir = os.path.join(f'./checkpoints/{opt.name}/results',
|
| 70 |
+
mutils.get_formatted_time())
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def set_learning_rate(lr):
|
| 74 |
+
for optimizer in engine.model.optimizers:
|
| 75 |
+
print('[i] set learning rate to {}'.format(lr))
|
| 76 |
+
util.set_opt_param(optimizer, 'lr', lr)
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
if opt.resume or opt.debug_eval:
|
| 81 |
+
save_dir = os.path.join(result_dir, '%03d' % engine.epoch)
|
| 82 |
+
os.makedirs(save_dir, exist_ok=True)
|
| 83 |
+
engine.save_model()
|
| 84 |
+
|
| 85 |
+
engine.eval(eval_dataloader_real, dataset_name='testdata_real20',
|
| 86 |
+
savedir=save_dir, suffix='real20', max_save_size=10)
|
| 87 |
+
engine.eval(eval_dataloader_solidobject, dataset_name='testdata_solidobject',
|
| 88 |
+
savedir=save_dir, suffix='solidobject', max_save_size=10)
|
| 89 |
+
engine.eval(eval_dataloader_postcard, dataset_name='testdata_postcard',
|
| 90 |
+
savedir=save_dir, suffix='postcard', max_save_size=10)
|
| 91 |
+
engine.eval(eval_dataloader_wild, dataset_name='testdata_wild',
|
| 92 |
+
savedir=save_dir, suffix='wild', max_save_size=10)
|
| 93 |
+
# define training strategy
|
| 94 |
+
set_learning_rate(opt.lr)
|
| 95 |
+
while engine.epoch < 50:
|
| 96 |
+
print('random_seed: ', opt.seed)
|
| 97 |
+
engine.train(train_dataloader_fusion)
|
| 98 |
+
|
| 99 |
+
if engine.epoch % 1 == 0:
|
| 100 |
+
save_dir = os.path.join(result_dir, '%03d' % engine.epoch)
|
| 101 |
+
os.makedirs(save_dir, exist_ok=True)
|
| 102 |
+
engine.eval(eval_dataloader_real, dataset_name='testdata_real20',
|
| 103 |
+
savedir=save_dir, suffix='real20', max_save_size=10)
|
| 104 |
+
engine.eval(eval_dataloader_solidobject, dataset_name='testdata_solidobject',
|
| 105 |
+
savedir=save_dir, suffix='solidobject', max_save_size=10)
|
| 106 |
+
engine.eval(eval_dataloader_postcard, dataset_name='testdata_postcard',
|
| 107 |
+
savedir=save_dir, suffix='postcard', max_save_size=10)
|
| 108 |
+
engine.eval(eval_dataloader_wild, dataset_name='testdata_wild',
|
| 109 |
+
savedir=save_dir, suffix='wild', max_save_size=10)
|
DSRNet/train_sirs_4000.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from os.path import join
|
| 3 |
+
|
| 4 |
+
import torch.backends.cuda
|
| 5 |
+
import torch.backends.cudnn as cudnn
|
| 6 |
+
|
| 7 |
+
import data.sirs_dataset as datasets
|
| 8 |
+
import util.util as util
|
| 9 |
+
from data.image_folder import read_fns
|
| 10 |
+
from engine import Engine
|
| 11 |
+
from options.net_options.train_options import TrainOptions
|
| 12 |
+
from tools import mutils
|
| 13 |
+
|
| 14 |
+
opt = TrainOptions().parse()
|
| 15 |
+
print(opt)
|
| 16 |
+
cudnn.benchmark = False
|
| 17 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 18 |
+
|
| 19 |
+
opt.display_freq = 10
|
| 20 |
+
|
| 21 |
+
if opt.debug:
|
| 22 |
+
opt.display_id = 1
|
| 23 |
+
opt.display_freq = 1
|
| 24 |
+
opt.print_freq = 20
|
| 25 |
+
opt.nEpochs = 40
|
| 26 |
+
opt.max_dataset_size = 9999
|
| 27 |
+
opt.no_log = False
|
| 28 |
+
opt.nThreads = 0
|
| 29 |
+
opt.decay_iter = 0
|
| 30 |
+
opt.serial_batches = True
|
| 31 |
+
opt.no_flip = True
|
| 32 |
+
|
| 33 |
+
# modify the following code to
|
| 34 |
+
# datadir = os.path.join(os.path.expanduser('~'), 'datasets/reflection-removal')
|
| 35 |
+
datadir = os.path.join(opt.base_dir)
|
| 36 |
+
|
| 37 |
+
datadir_syn = join(datadir, 'train/VOCdevkit/VOC2012/PNGImages')
|
| 38 |
+
datadir_real = join(datadir, 'train/real')
|
| 39 |
+
datadir_nature = join(datadir, 'train/nature')
|
| 40 |
+
|
| 41 |
+
train_dataset = datasets.DSRDataset(
|
| 42 |
+
datadir_syn, read_fns('data/VOC2012_224_train_png.txt'), size=opt.max_dataset_size, enable_transforms=True)
|
| 43 |
+
|
| 44 |
+
train_dataset_real = datasets.DSRTestDataset(datadir_real, enable_transforms=True, if_align=opt.if_align)
|
| 45 |
+
train_dataset_nature = datasets.DSRTestDataset(datadir_nature, enable_transforms=True, if_align=opt.if_align)
|
| 46 |
+
|
| 47 |
+
train_dataset_fusion = datasets.FusionDataset([train_dataset,
|
| 48 |
+
train_dataset_real,
|
| 49 |
+
train_dataset_nature], [0.6, 0.2, 0.2],
|
| 50 |
+
size=opt.num_train if opt.num_train > 0 else 4000)
|
| 51 |
+
|
| 52 |
+
train_dataloader_fusion = datasets.DataLoader(
|
| 53 |
+
train_dataset_fusion, batch_size=opt.batchSize, shuffle=not opt.serial_batches,
|
| 54 |
+
num_workers=opt.nThreads, pin_memory=True)
|
| 55 |
+
|
| 56 |
+
eval_dataset_real = datasets.DSRTestDataset(join(datadir, f'test/real20_{opt.real20_size}'),
|
| 57 |
+
fns=read_fns('data/real_test.txt'), if_align=opt.if_align)
|
| 58 |
+
eval_dataset_solidobject = datasets.DSRTestDataset(join(datadir, 'test/SIR2/SolidObjectDataset'),
|
| 59 |
+
if_align=opt.if_align)
|
| 60 |
+
eval_dataset_postcard = datasets.DSRTestDataset(join(datadir, 'test/SIR2/PostcardDataset'), if_align=opt.if_align)
|
| 61 |
+
eval_dataset_wild = datasets.DSRTestDataset(join(datadir, 'test/SIR2/WildSceneDataset'), if_align=opt.if_align)
|
| 62 |
+
|
| 63 |
+
eval_dataloader_real = datasets.DataLoader(
|
| 64 |
+
eval_dataset_real, batch_size=1, shuffle=False,
|
| 65 |
+
num_workers=opt.nThreads, pin_memory=True)
|
| 66 |
+
|
| 67 |
+
eval_dataloader_solidobject = datasets.DataLoader(
|
| 68 |
+
eval_dataset_solidobject, batch_size=1, shuffle=False,
|
| 69 |
+
num_workers=opt.nThreads, pin_memory=True)
|
| 70 |
+
eval_dataloader_postcard = datasets.DataLoader(
|
| 71 |
+
eval_dataset_postcard, batch_size=1, shuffle=False,
|
| 72 |
+
num_workers=opt.nThreads, pin_memory=True)
|
| 73 |
+
|
| 74 |
+
eval_dataloader_wild = datasets.DataLoader(
|
| 75 |
+
eval_dataset_wild, batch_size=1, shuffle=False,
|
| 76 |
+
num_workers=opt.nThreads, pin_memory=True)
|
| 77 |
+
|
| 78 |
+
"""Main Loop"""
|
| 79 |
+
engine = Engine(opt)
|
| 80 |
+
result_dir = os.path.join(f'./checkpoints/{opt.name}/results',
|
| 81 |
+
mutils.get_formatted_time())
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def set_learning_rate(lr):
|
| 85 |
+
for optimizer in engine.model.optimizers:
|
| 86 |
+
print('[i] set learning rate to {}'.format(lr))
|
| 87 |
+
util.set_opt_param(optimizer, 'lr', lr)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
if opt.resume or opt.debug_eval:
|
| 91 |
+
save_dir = os.path.join(result_dir, '%03d' % engine.epoch)
|
| 92 |
+
os.makedirs(save_dir, exist_ok=True)
|
| 93 |
+
engine.save_model()
|
| 94 |
+
|
| 95 |
+
engine.eval(eval_dataloader_real, dataset_name='testdata_real20', savedir=save_dir, suffix='real20')
|
| 96 |
+
engine.eval(eval_dataloader_solidobject, dataset_name='testdata_solidobject', savedir=save_dir,
|
| 97 |
+
suffix='solidobject')
|
| 98 |
+
engine.eval(eval_dataloader_postcard, dataset_name='testdata_postcard', savedir=save_dir, suffix='postcard')
|
| 99 |
+
engine.eval(eval_dataloader_wild, dataset_name='testdata_wild', savedir=save_dir, suffix='wild')
|
| 100 |
+
|
| 101 |
+
# define training strategy
|
| 102 |
+
engine.model.opt.lambda_gan = 0
|
| 103 |
+
# engine.model.opt.lambda_gan = 0.01
|
| 104 |
+
set_learning_rate(opt.lr)
|
| 105 |
+
|
| 106 |
+
while engine.epoch < 120:
|
| 107 |
+
print('random_seed: ', opt.seed)
|
| 108 |
+
engine.train(train_dataloader_fusion)
|
| 109 |
+
|
| 110 |
+
if engine.epoch % 1 == 0:
|
| 111 |
+
save_dir = os.path.join(result_dir, '%03d' % engine.epoch)
|
| 112 |
+
os.makedirs(save_dir, exist_ok=True)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
engine.eval(eval_dataloader_real, dataset_name='testdata_real20', savedir=save_dir, suffix='real20')
|
| 116 |
+
engine.eval(eval_dataloader_solidobject, dataset_name='testdata_solidobject', savedir=save_dir, suffix='solidobject')
|
| 117 |
+
engine.eval(eval_dataloader_postcard, dataset_name='testdata_postcard', savedir=save_dir, suffix='postcard')
|
| 118 |
+
engine.eval(eval_dataloader_wild, dataset_name='testdata_wild', savedir=save_dir, suffix='wild')
|
DSRNet/util/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .net_utils import *
|
| 2 |
+
from .util import *
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def load_checkpoint(model, ckpt_path):
|
| 6 |
+
checkpoint = torch.load(ckpt_path)
|
| 7 |
+
if 'model' in checkpoint:
|
| 8 |
+
checkpoint = checkpoint['model']
|
| 9 |
+
if 'state_dict' in checkpoint:
|
| 10 |
+
checkpoint = checkpoint['state_dict']
|
| 11 |
+
ckpt = {}
|
| 12 |
+
for k, v in checkpoint.items():
|
| 13 |
+
if k.startswith('module.'):
|
| 14 |
+
ckpt[k[7:]] = v
|
| 15 |
+
else:
|
| 16 |
+
ckpt[k] = v
|
| 17 |
+
model.load_state_dict(ckpt,strict=False) #strict=False by nami
|
DSRNet/util/html.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import dominate #used for creating and manipulating HTML documents in a programmatic way.
|
| 2 |
+
from dominate.tags import * #easier to create HTML tags directly.
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
#ALL reflesh should be refresh
|
| 6 |
+
class HTML:
|
| 7 |
+
def __init__(self, web_dir, title, reflesh=0): #reflesh: refresh_rate
|
| 8 |
+
self.title = title #title of web page
|
| 9 |
+
self.web_dir = web_dir#parent directory of all web content e.g images
|
| 10 |
+
self.img_dir = os.path.join(self.web_dir, 'images')
|
| 11 |
+
if not os.path.exists(self.web_dir):
|
| 12 |
+
os.makedirs(self.web_dir)
|
| 13 |
+
if not os.path.exists(self.img_dir):
|
| 14 |
+
os.makedirs(self.img_dir)
|
| 15 |
+
#(mingcv) print(self.img_dir)
|
| 16 |
+
|
| 17 |
+
self.doc = dominate.document(title=title)
|
| 18 |
+
if reflesh > 0:
|
| 19 |
+
with self.doc.head:
|
| 20 |
+
meta(http_equiv="refresh", content=str(reflesh)) #was "reflesh" in mingcv
|
| 21 |
+
#The http-equiv attribute provides an HTTP header for the information/value of the content attribute.In <head></head>
|
| 22 |
+
def get_image_dir(self):
|
| 23 |
+
return self.img_dir
|
| 24 |
+
|
| 25 |
+
def add_header(self, str):
|
| 26 |
+
with self.doc:
|
| 27 |
+
h3(str)
|
| 28 |
+
|
| 29 |
+
def add_table(self, border=1):
|
| 30 |
+
self.t = table(border=border, style="table-layout: fixed;")
|
| 31 |
+
self.doc.add(self.t)
|
| 32 |
+
|
| 33 |
+
def add_images(self, ims, txts, links, height=400):# add images to the HTML document within a table. Each cell (td) in the table contains an image on one line and its descriptive text on the next line.
|
| 34 |
+
self.add_table()
|
| 35 |
+
with self.t:#opens the table context for adding rows
|
| 36 |
+
with tr():#open table row context
|
| 37 |
+
for im, txt, link in zip(ims, txts, links):#This iterates over the images, texts, and links
|
| 38 |
+
with td(style="word-wrap: break-word;", halign="center", valign="top"):
|
| 39 |
+
with p():
|
| 40 |
+
with a(href=os.path.join('images', link)):
|
| 41 |
+
img(style="height:%dpx" % height, src=os.path.join('images', im))
|
| 42 |
+
br()
|
| 43 |
+
p(txt)
|
| 44 |
+
|
| 45 |
+
def save(self):# save the generated HTML document to a file.
|
| 46 |
+
html_file = '%s/index.html' % self.web_dir # self.web_dir value will be placed in %s like sprintf(html_file, “%s/index.html”, web_dir);
|
| 47 |
+
f = open(html_file, 'wt')#write text mode
|
| 48 |
+
f.write(self.doc.render())# writes the rendered HTML content to the file. The render method of the doc object (which is an instance of the dominate.document class) converts the HTML document into a string.
|
| 49 |
+
f.close()
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
if __name__ == '__main__':#if the script is being run as the main module. This block will only execute if the script is run directly (not imported as a module).
|
| 53 |
+
|
| 54 |
+
html = HTML('web/', 'test_html')#'web/'=web_dir, 'test_html'=title
|
| 55 |
+
html.add_header('hello world')
|
| 56 |
+
|
| 57 |
+
ims = []#ims for image filenames, txts for image descriptions, and links for image links.
|
| 58 |
+
txts = []
|
| 59 |
+
links = []
|
| 60 |
+
for n in range(4):#adds images, texts and links to the list so that they can be added to html.
|
| 61 |
+
'''??? why 4'''
|
| 62 |
+
ims.append('image_%d.png' % n)
|
| 63 |
+
txts.append('text_%d' % n)
|
| 64 |
+
links.append('image_%d.png' % n)
|
| 65 |
+
html.add_images(ims, txts, links)
|
| 66 |
+
html.save()
|
DSRNet/util/index.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Metrics/Indexes
|
| 2 |
+
from skimage.metrics import peak_signal_noise_ratio as compare_psnr
|
| 3 |
+
from skimage.metrics import structural_similarity as compare_ssim
|
| 4 |
+
from functools import partial
|
| 5 |
+
import numpy as np
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class Bandwise(object):
|
| 9 |
+
def __init__(self, index_fn):
|
| 10 |
+
self.index_fn = index_fn
|
| 11 |
+
|
| 12 |
+
def __call__(self, X, Y):
|
| 13 |
+
C = X.shape[-1]
|
| 14 |
+
bwindex = []
|
| 15 |
+
for ch in range(C):
|
| 16 |
+
x = X[..., ch]
|
| 17 |
+
y = Y[..., ch]
|
| 18 |
+
index = self.index_fn(x, y)
|
| 19 |
+
bwindex.append(index)
|
| 20 |
+
return bwindex
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
cal_bwpsnr = Bandwise(partial(compare_psnr, data_range=255))
|
| 24 |
+
cal_bwssim = Bandwise(partial(compare_ssim, data_range=255))
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def compare_ncc(x, y):
|
| 28 |
+
return np.mean((x - np.mean(x)) * (y - np.mean(y))) / (np.std(x) * np.std(y))
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def ssq_error(correct, estimate):
|
| 32 |
+
"""Compute the sum-squared-error for an image, where the estimate is
|
| 33 |
+
multiplied by a scalar which minimizes the error. Sums over all pixels
|
| 34 |
+
where mask is True. If the inputs are color, each color channel can be
|
| 35 |
+
rescaled independently."""
|
| 36 |
+
assert correct.ndim == 2
|
| 37 |
+
if np.sum(estimate ** 2) > 1e-5:
|
| 38 |
+
alpha = np.sum(correct * estimate) / np.sum(estimate ** 2)
|
| 39 |
+
else:
|
| 40 |
+
alpha = 0.
|
| 41 |
+
return np.sum((correct - alpha * estimate) ** 2)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def local_error(correct, estimate, window_size, window_shift):
|
| 45 |
+
"""Returns the sum of the local sum-squared-errors, where the estimate may
|
| 46 |
+
be rescaled within each local region to minimize the error. The windows are
|
| 47 |
+
window_size x window_size, and they are spaced by window_shift."""
|
| 48 |
+
M, N, C = correct.shape
|
| 49 |
+
ssq = total = 0.
|
| 50 |
+
for c in range(C):
|
| 51 |
+
for i in range(0, M - window_size + 1, window_shift):
|
| 52 |
+
for j in range(0, N - window_size + 1, window_shift):
|
| 53 |
+
correct_curr = correct[i:i + window_size, j:j + window_size, c]
|
| 54 |
+
estimate_curr = estimate[i:i + window_size, j:j + window_size, c]
|
| 55 |
+
ssq += ssq_error(correct_curr, estimate_curr)
|
| 56 |
+
total += np.sum(correct_curr ** 2)
|
| 57 |
+
# assert np.isnan(ssq/total)
|
| 58 |
+
return ssq / total
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def quality_assess(X, Y):
|
| 62 |
+
# Y: correct; X: estimate
|
| 63 |
+
psnr = np.mean(cal_bwpsnr(Y, X))
|
| 64 |
+
ssim = np.mean(cal_bwssim(Y, X))
|
| 65 |
+
lmse = local_error(Y, X, 20, 10)
|
| 66 |
+
ncc = compare_ncc(Y, X)
|
| 67 |
+
return {'PSNR': psnr, 'SSIM': ssim, 'LMSE': lmse, 'NCC': ncc}
|
DSRNet/util/net_utils.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# returns the total number of parameters in the model that require gradients.
|
| 2 |
+
def count_parameters(model):
|
| 3 |
+
return sum(p.numel() for p in model.parameters() if p.requires_grad)
|
| 4 |
+
# model.parameters(): This method returns an iterator over all the parameters (weights and biases) in the model.
|
| 5 |
+
#p.numel(): This method returns the number of elements in the parameter tensor p.
|
| 6 |
+
|
| 7 |
+
def count_conv_layers(model): #counts total no. conv layers in a model
|
| 8 |
+
cnt = 0
|
| 9 |
+
for mo in model.modules(): #model.modules(): This method returns an iterator over all submodules (layers) of the model, including the model itself.
|
| 10 |
+
if type(mo).__name__ == 'Conv2d': #type(mo).__name__: This retrieves the string name of the class to which the submodule mo belongs.
|
| 11 |
+
cnt += 1
|
| 12 |
+
#model, total number of conv layers and total paramters with gradient
|
| 13 |
+
print(type(model).__name__, cnt, count_parameters(model))
|
DSRNet/util/schedulers.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from torch.optim.lr_scheduler import _LRScheduler
|
| 2 |
+
#Learning rate scheduler
|
| 3 |
+
from torch.optim.lr_scheduler import ReduceLROnPlateau
|
| 4 |
+
#Reduce learning rate when a metric has stopped improving. Models often benefit from reducing the learning rate by a factor of 2-10 once learning stagnates
|
| 5 |
+
|
| 6 |
+
class GradualWarmupScheduler(_LRScheduler):
|
| 7 |
+
#Starts with a small learning rate and gradually increases it over a few initial epochs or iterations. This is particularly useful in preventing the model from diverging in the initial phase of training and is often used in training deep networks from scratch.
|
| 8 |
+
|
| 9 |
+
#Pros: Prevents early divergence; stabilizes training.
|
| 10 |
+
#Cons: Requires tuning of warm-up duration and rate limits.
|
| 11 |
+
#Use Case: Crucial for training deep networks from scratch.
|
| 12 |
+
"""(mingcv) Gradually warm-up(increasing) learning rate in optimizer.
|
| 13 |
+
Proposed in 'Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour'.
|
| 14 |
+
Args:
|
| 15 |
+
optimizer (Optimizer): Wrapped optimizer.
|
| 16 |
+
multiplier: target learning rate = base lr * multiplier if multiplier > 1.0. if multiplier = 1.0, lr starts from 0 and ends up with the base_lr.
|
| 17 |
+
total_epoch: target learning rate is reached at total_epoch, gradually
|
| 18 |
+
after_scheduler: after target_epoch, use this scheduler(eg. ReduceLROnPlateau)
|
| 19 |
+
"""
|
| 20 |
+
#total_epoch (number of epochs to reach the target learning rate
|
| 21 |
+
#after scheduler: scheduler after warmup
|
| 22 |
+
|
| 23 |
+
def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None):
|
| 24 |
+
self.multiplier = multiplier
|
| 25 |
+
if self.multiplier < 1.:
|
| 26 |
+
raise ValueError('multiplier should be greater thant or equal to 1.')
|
| 27 |
+
self.total_epoch = total_epoch
|
| 28 |
+
self.after_scheduler = after_scheduler
|
| 29 |
+
self.finished = False #initializes a flag to indicate if warm-up is finished.
|
| 30 |
+
super(GradualWarmupScheduler, self).__init__(optimizer)
|
| 31 |
+
|
| 32 |
+
def get_lr(self): #get the current learning rates.
|
| 33 |
+
if self.last_epoch > self.total_epoch: #warm is ovcer , target lr reached
|
| 34 |
+
if self.after_scheduler: #plateuLR used
|
| 35 |
+
if not self.finished: #sets the base learning rates for the after_scheduler to be multiplier times the base learning rates and returns rlearningrate of after scheduler.
|
| 36 |
+
self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs] #self.base_lrs intialized from _LRScheduler
|
| 37 |
+
self.finished = True #warm up done
|
| 38 |
+
return self.after_scheduler.get_lr()
|
| 39 |
+
return [base_lr * self.multiplier for base_lr in self.base_lrs]
|
| 40 |
+
|
| 41 |
+
if self.multiplier == 1.0: #increase lienarly and propertionally to number of epochs run
|
| 42 |
+
return [base_lr * (float(self.last_epoch) / self.total_epoch) for base_lr in self.base_lrs]
|
| 43 |
+
else: #increaase lr non linearly
|
| 44 |
+
return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in
|
| 45 |
+
self.base_lrs]
|
| 46 |
+
"??? why this formula for non linear"
|
| 47 |
+
|
| 48 |
+
def step_ReduceLROnPlateau(self, metrics, epoch=None):
|
| 49 |
+
#metrics: A value that will be monitored (e.g., validation loss) to decide if the learning rate needs adjustment.
|
| 50 |
+
if epoch is None: #not resuming from a specific epoch
|
| 51 |
+
epoch = self.last_epoch + 1
|
| 52 |
+
self.last_epoch = epoch if epoch != 0 else 1 # (mingcv) ReduceLROnPlateau is called at the end of epoch, whereas others are called at beginning
|
| 53 |
+
if self.last_epoch <= self.total_epoch:
|
| 54 |
+
warmup_lr = [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in
|
| 55 |
+
self.base_lrs]
|
| 56 |
+
for param_group, lr in zip(self.optimizer.param_groups, warmup_lr):
|
| 57 |
+
param_group['lr'] = lr
|
| 58 |
+
else:
|
| 59 |
+
if epoch is None:
|
| 60 |
+
self.after_scheduler.step(metrics, None)
|
| 61 |
+
else:
|
| 62 |
+
self.after_scheduler.step(metrics, epoch - self.total_epoch)
|
| 63 |
+
|
| 64 |
+
def step(self, epoch=None, metrics=None):
|
| 65 |
+
if type(self.after_scheduler) != ReduceLROnPlateau:
|
| 66 |
+
if self.finished and self.after_scheduler:
|
| 67 |
+
if epoch is None:
|
| 68 |
+
self.after_scheduler.step(None)
|
| 69 |
+
else:
|
| 70 |
+
self.after_scheduler.step(epoch - self.total_epoch)
|
| 71 |
+
else:
|
| 72 |
+
return super(GradualWarmupScheduler, self).step(epoch)
|
| 73 |
+
else:
|
| 74 |
+
self.step_ReduceLROnPlateau(metrics, epoch)
|
DSRNet/util/util.py
ADDED
|
@@ -0,0 +1,435 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import print_function #if running Python2 makes it work like python3
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
import os
|
| 5 |
+
import sys
|
| 6 |
+
import time
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import torch
|
| 10 |
+
import torch.nn as nn
|
| 11 |
+
import yaml #Imports PyYAML for parsing YAML files.
|
| 12 |
+
from PIL import Image #Imports the Python Imaging Library for image processing.
|
| 13 |
+
from skimage.metrics import peak_signal_noise_ratio as compare_psnr #Imports the PSNR metric from skimage to quantify reconstruction quality for images and video subject to lossy compression. Higher is better
|
| 14 |
+
from skimage.metrics import structural_similarity #Imports the SSIM metric from skimage.A full-reference image quality evaluation index that measures image similarity from three aspects: brightness, contrast, and structure.
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def get_config(config):#apth to configuration file
|
| 18 |
+
with open(config, 'r') as stream: #opens the file specified by the config variable in read mode and assigned to variable stream
|
| 19 |
+
return yaml.load(stream) #call loads and parses the YAML content from the file object stream and returns the parsed content.
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
# Converts a Tensor into a Numpy array then desired imtype
|
| 23 |
+
# |imtype|: the desired type of the converted numpy array
|
| 24 |
+
def tensor2im(image_tensor, imtype=np.uint8):
|
| 25 |
+
image_numpy = image_tensor[0].cpu().float().numpy() #image_tensor[0]: first image in batch in tensor
|
| 26 |
+
#.cpu() moves tensor to CPU if in GPU
|
| 27 |
+
#.float() converts to float
|
| 28 |
+
#.numpy() converts to numpy
|
| 29 |
+
if image_numpy.shape[0] == 1: #.shape[0] access channels ,1 = grayscale, 3=RGB
|
| 30 |
+
image_numpy = np.tile(image_numpy, (3, 1, 1))
|
| 31 |
+
#np.tile(A,reps):Construct an array by repeating A the number of times given by reps.
|
| 32 |
+
#repeating graysvale image here in 3 channels to get RGB image
|
| 33 |
+
image_numpy = (np.transpose(image_numpy, (1, 2, 0)) + 1) / 2.0 * 255.0
|
| 34 |
+
#np.transpose(image_numpy, (1, 2, 0)):The image is transposed from the shape (C, H, W) to (H, W, C).
|
| 35 |
+
# + 1: shift the pixel intensity range from [-1, 1] to [0, 2].
|
| 36 |
+
# / 2.0: This scales the values to the range [0, 1].
|
| 37 |
+
# * 255.0: This scales the values to the range [0, 255].
|
| 38 |
+
image_numpy = image_numpy.astype(imtype) #numpy array recast to desired imtype
|
| 39 |
+
if image_numpy.shape[-1] == 6: #after transpose -1 is no.channels and if this is 6
|
| 40 |
+
image_numpy = np.concatenate([image_numpy[:, :, :3], image_numpy[:, :, 3:]], axis=1)
|
| 41 |
+
'''???how does this help '''
|
| 42 |
+
#recombines along width 2 grps of 0-2 and 3-5
|
| 43 |
+
|
| 44 |
+
if image_numpy.shape[-1] == 7:#if transposed image has 7 channels
|
| 45 |
+
edge_map = np.tile(image_numpy[:, :, 6:7], (1, 1, 3))
|
| 46 |
+
'''???Tiles (repeats) the seventh channel to create a 3-channel edge_map along the channel axis.'''
|
| 47 |
+
image_numpy = np.concatenate([image_numpy[:, :, :3], image_numpy[:, :, 3:6], edge_map], axis=1)
|
| 48 |
+
'''???how does this help '''
|
| 49 |
+
#recombines along width 2 grps of 0-2 and 3-6
|
| 50 |
+
return image_numpy
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def tensor2numpy(image_tensor):
|
| 54 |
+
image_numpy = torch.squeeze(image_tensor).cpu().float().numpy() #.squeeze(): removes all dimensions of size 1 from the shape of image_tensor
|
| 55 |
+
#PyTorch tensors can have singleton dimensions, which are dimensions with size 1. These dimensions might not carry meaningful information but are retained due to tensor operations.
|
| 56 |
+
#proabably rmeove number of batchs since 1 batch in 1 img (batch,channels,height,width)
|
| 57 |
+
image_numpy = (np.transpose(image_numpy, (1, 2, 0)) + 1) / 2.0 * 255.0
|
| 58 |
+
#np.transpose(image_numpy, (1, 2, 0)):The image is transposed from the shape (C, H, W) to (H, W, C).
|
| 59 |
+
# + 1: shift the pixel intensity range from [-1, 1] to [0, 2].
|
| 60 |
+
# / 2.0: This scales the values to the range [0, 1].
|
| 61 |
+
# * 255.0: This scales the values to the range [0, 255].
|
| 62 |
+
image_numpy = image_numpy.astype(np.float32)
|
| 63 |
+
#numpy array recast to float
|
| 64 |
+
return image_numpy
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
# Get model list for resume
|
| 68 |
+
#retrieve a specific model checkpoint file based on dirname, epoch,key
|
| 69 |
+
def get_model_list(dirname, key, epoch=None):
|
| 70 |
+
#key: main file_name
|
| 71 |
+
#epoch to indicate at which epoch the model was saved.Choosing epoch allows you to continue training from a specific point or to evaluate the model's performance at different stages of training
|
| 72 |
+
if epoch is None:
|
| 73 |
+
return os.path.join(dirname, key + '_latest.pt') # return latest checkpoint by default
|
| 74 |
+
if os.path.exists(dirname) is False: #if directory doesnt exist no checkpoint files can be retrieved.
|
| 75 |
+
return None
|
| 76 |
+
|
| 77 |
+
print(dirname, key)
|
| 78 |
+
|
| 79 |
+
#list of generated checkpoints after specific epohs during traininf
|
| 80 |
+
gen_models = [os.path.join(dirname, f) for f in os.listdir(dirname) if
|
| 81 |
+
os.path.isfile(os.path.join(dirname, f)) and ".pt" in f and 'latest' not in f]
|
| 82 |
+
#isfile():regular file,not a directory or a special file
|
| 83 |
+
#.pt type file with epoch number specifid, latest means epoch=None
|
| 84 |
+
epoch_index = [int(os.path.basename(model_name).split('_')[-2]) for model_name in gen_models if 'latest' not in model_name]
|
| 85 |
+
#os.path.basename(model_name): extracts the filename from the full path.
|
| 86 |
+
#split('_')[-2]: ['dsrnet', 's', 'epoch14.pt']-> s (second last)
|
| 87 |
+
'''??? why not -1 -> to get 14 chatgpt says int(s) will raise VauleError'''
|
| 88 |
+
print('[i] available epoch list: %s' % epoch_index, gen_models)
|
| 89 |
+
i = epoch_index.index(int(epoch))
|
| 90 |
+
|
| 91 |
+
return gen_models[i]
|
| 92 |
+
|
| 93 |
+
# to preprocess a batch of images for compatibility with models pretrained on the ImageNet dataset
|
| 94 |
+
def vgg_preprocess(batch):
|
| 95 |
+
# normalize using imagenet mean and std
|
| 96 |
+
#ImageNet normalization helps preprocess images to have zero mean and unit variance across each channel (RGB).
|
| 97 |
+
mean = batch.new(batch.size()) # Pytorch function creates new tensors (mean and std) with the same shape as the input batch.
|
| 98 |
+
std = batch.new(batch.size())
|
| 99 |
+
mean[:, 0, :, :] = 0.485 #assigned to all elements in the first channel of every image in the batch.
|
| 100 |
+
mean[:, 1, :, :] = 0.456 #2nd channel
|
| 101 |
+
mean[:, 2, :, :] = 0.406 #3rd channel
|
| 102 |
+
std[:, 0, :, :] = 0.229 #1st channel
|
| 103 |
+
std[:, 1, :, :] = 0.224 #2nd channel
|
| 104 |
+
std[:, 2, :, :] = 0.225 #3rd channel
|
| 105 |
+
batch = (batch + 1) / 2 #pixel values shifted [-1,1]->[0,2]->[0,1]
|
| 106 |
+
batch -= mean
|
| 107 |
+
batch = batch / std
|
| 108 |
+
# each pixel value in batch will have zero mean and unit variance
|
| 109 |
+
return batch
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
#print diagnostic information about the gradients of a neural network
|
| 114 |
+
#useful for inspecting the average magnitude of gradients during training or optimization of a neural network.
|
| 115 |
+
# It helps in diagnosing potential issues like vanishing or exploding gradients,
|
| 116 |
+
def diagnose_network(net, name='network'): #name: Optional name for the network (default is 'network'), used in printing diagnostic information.
|
| 117 |
+
mean = 0.0 #average mean absolute gradient across all parameters that have gradients.
|
| 118 |
+
count = 0 #no. of paramters with gradients
|
| 119 |
+
for param in net.parameters():
|
| 120 |
+
if param.grad is not None: #trainable Parameters without gradients won't contribute to mean.
|
| 121 |
+
#e.g frozen layers, non trainable paramters,bias,batch normalization parameters
|
| 122 |
+
mean += torch.mean(torch.abs(param.grad.data)) #mean absolute value of the gradient of the current parameter param
|
| 123 |
+
count += 1
|
| 124 |
+
if count > 0:
|
| 125 |
+
mean = mean / count
|
| 126 |
+
print(name)
|
| 127 |
+
print(mean)
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def save_image(image_numpy, image_path): #create a PIL Image object from the numpy array and then saves it using the save() method of the PIL Image object.
|
| 131 |
+
image_pil = Image.fromarray(image_numpy)
|
| 132 |
+
image_pil.save(image_path)
|
| 133 |
+
|
| 134 |
+
#print shape and statistics of a numpy array
|
| 135 |
+
def print_numpy(x, val=True, shp=False): #val is statistics shp is shape
|
| 136 |
+
x = x.astype(np.float64)
|
| 137 |
+
if shp:
|
| 138 |
+
print('shape,', x.shape)
|
| 139 |
+
if val:
|
| 140 |
+
x = x.flatten() #converts nD to 1D
|
| 141 |
+
print('mean = %3.3f, min = %3.3f, max = %3.3f, median = %3.3f, std=%3.3f' % (
|
| 142 |
+
np.mean(x), np.min(x), np.max(x), np.median(x), np.std(x)))
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def mkdirs(paths):
|
| 146 |
+
if isinstance(paths, list) and not isinstance(paths, str):
|
| 147 |
+
#if list makes path for each string
|
| 148 |
+
for path in paths:
|
| 149 |
+
mkdir(path)
|
| 150 |
+
else: #if string makes path
|
| 151 |
+
mkdir(paths)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def mkdir(path):
|
| 155 |
+
if not os.path.exists(path):
|
| 156 |
+
os.makedirs(path)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def set_opt_param(optimizer, key, value):
|
| 160 |
+
for group in optimizer.param_groups:
|
| 161 |
+
group[key] = value
|
| 162 |
+
'???'
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def vis(x):#display an image in image viewer of system regardless if tensor or numpy array
|
| 166 |
+
if isinstance(x, torch.Tensor):
|
| 167 |
+
Image.fromarray(tensor2im(x)).show()
|
| 168 |
+
elif isinstance(x, np.ndarray):
|
| 169 |
+
Image.fromarray(x.astype(np.uint8)).show()
|
| 170 |
+
else:
|
| 171 |
+
raise NotImplementedError('vis for type [%s] is not implemented', type(x))
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
"""tensorboard"""
|
| 175 |
+
from tensorboardX import SummaryWriter #for data logging
|
| 176 |
+
from datetime import datetime
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def get_summary_writer(log_dir): #log_dir:directy to store logs
|
| 180 |
+
if not os.path.exists(log_dir):
|
| 181 |
+
os.mkdir(log_dir)
|
| 182 |
+
log_dir = os.path.join(log_dir, datetime.now().strftime('%b%d_%H-%M-%S') + '_' + socket.gethostname())
|
| 183 |
+
if not os.path.exists(log_dir):
|
| 184 |
+
os.mkdir(log_dir)
|
| 185 |
+
writer = SummaryWriter(log_dir) #each time you run your experiments, logs are saved in a new directory with current time, date and machine hostname
|
| 186 |
+
return writer
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
# keeps track of running avergae of metrics
|
| 191 |
+
class AverageMeters(object): #all classes inherit from object
|
| 192 |
+
def __init__(self, dic=None, total_num=None):
|
| 193 |
+
#dic: dictionary of metrics
|
| 194 |
+
#total_num:track of the counts of each metric)
|
| 195 |
+
self.dic = dic or {}
|
| 196 |
+
#(mingcv) self.total_num = total_num
|
| 197 |
+
self.total_num = total_num or {}
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def update(self, new_dic): # method appends self.dic and updates total_num with new values.
|
| 201 |
+
for key in new_dic:
|
| 202 |
+
if not key in self.dic:
|
| 203 |
+
self.dic[key] = new_dic[key]
|
| 204 |
+
self.total_num[key] = 1
|
| 205 |
+
else:
|
| 206 |
+
self.dic[key] += new_dic[key]
|
| 207 |
+
self.total_num[key] += 1
|
| 208 |
+
# (mingcv)self.total_num += 1
|
| 209 |
+
|
| 210 |
+
def __getitem__(self, key): #overriding [], returns average value for a metric
|
| 211 |
+
return self.dic[key] / self.total_num[key]
|
| 212 |
+
|
| 213 |
+
def __str__(self): #overrider str() and print() to show key and values formatted
|
| 214 |
+
keys = sorted(self.keys())
|
| 215 |
+
res = ''
|
| 216 |
+
for key in keys:
|
| 217 |
+
res += (key + ': %.4f' % self[key] + ' | ')
|
| 218 |
+
return res
|
| 219 |
+
|
| 220 |
+
def keys(self): #returns all the metric names
|
| 221 |
+
return self.dic.keys()
|
| 222 |
+
|
| 223 |
+
'''???why _loss'''
|
| 224 |
+
def write_loss(writer, prefix, avg_meters, iteration):
|
| 225 |
+
#writer is the writer object for logging,SummaryWriter instance
|
| 226 |
+
#prefix is keyword added before each metric e.g. "train" it logs "accuracy": 85.4 as "train/accuracy"
|
| 227 |
+
#avg_meters: A dictionary argument where keys are metric names (like ‘loss’, ‘accuracy’) and values are their corresponding values at the current iteration (e.g., 0.23, 85.4).
|
| 228 |
+
#iteration: An integer argument representing the current iteration or step number in the training or evaluation process.
|
| 229 |
+
for key in avg_meters.keys():
|
| 230 |
+
meter = avg_meters[key] # for each key, we get the corresponding value from the avg_meters dictionary and assign it to the variable meter.
|
| 231 |
+
writer.add_scalar(os.path.join(prefix, key), meter, iteration)
|
| 232 |
+
# writer.add_scalar(): This method call logs the scalar value (metric) to the writer
|
| 233 |
+
'''
|
| 234 |
+
avg_meters = {
|
| 235 |
+
“loss”: 0.23,
|
| 236 |
+
“accuracy”: 85.4,
|
| 237 |
+
“precision”: 88.1
|
| 238 |
+
}
|
| 239 |
+
write_loss(writer, “train”, avg_meters, 10)
|
| 240 |
+
|
| 241 |
+
Logging: train/loss = 0.23 at step 10
|
| 242 |
+
Logging: train/accuracy = 85.4 at step 10
|
| 243 |
+
Logging: train/precision = 88.1 at step 10
|
| 244 |
+
|
| 245 |
+
'''
|
| 246 |
+
"""(Mingcv)progress bar"""
|
| 247 |
+
import socket
|
| 248 |
+
|
| 249 |
+
# _, term_width = os.popen('stty size', 'r').read().split()
|
| 250 |
+
term_width = 136 #hardcoded terminal width to 136 characters instead of dymeically reading using stty siz
|
| 251 |
+
|
| 252 |
+
TOTAL_BAR_LENGTH = 65. #length of the progress bar.
|
| 253 |
+
last_time = time.time() #when progress is finished
|
| 254 |
+
begin_time = last_time # used to calculate the elapsed time for each step and the total process.
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def progress_bar(current, total, msg=None):
|
| 258 |
+
#current: The current progress count.
|
| 259 |
+
#total: The total count that represents 100% progress.
|
| 260 |
+
#msg: An optional message to display alongside the progress bar.
|
| 261 |
+
global last_time, begin_time # for persistence in step_time and total_time
|
| 262 |
+
if current == 0:
|
| 263 |
+
begin_time = time.time() # Reset for new bar.
|
| 264 |
+
|
| 265 |
+
cur_len = int(TOTAL_BAR_LENGTH * current / total)
|
| 266 |
+
rest_len = int(TOTAL_BAR_LENGTH - cur_len) - 1
|
| 267 |
+
|
| 268 |
+
sys.stdout.write(' [')
|
| 269 |
+
for i in range(cur_len):
|
| 270 |
+
sys.stdout.write('=')
|
| 271 |
+
sys.stdout.write('>')
|
| 272 |
+
for i in range(rest_len):
|
| 273 |
+
sys.stdout.write('.')
|
| 274 |
+
sys.stdout.write(']')
|
| 275 |
+
|
| 276 |
+
cur_time = time.time()
|
| 277 |
+
step_time = cur_time - last_time
|
| 278 |
+
last_time = cur_time
|
| 279 |
+
tot_time = cur_time - begin_time
|
| 280 |
+
|
| 281 |
+
L = []
|
| 282 |
+
L.append(' Step: %s' % format_time(step_time))
|
| 283 |
+
L.append(' | Tot: %s' % format_time(tot_time))
|
| 284 |
+
if msg:
|
| 285 |
+
L.append(' | ' + msg)
|
| 286 |
+
|
| 287 |
+
msg = ''.join(L)
|
| 288 |
+
sys.stdout.write(msg)
|
| 289 |
+
for i in range(term_width - int(TOTAL_BAR_LENGTH) - len(msg) - 3):
|
| 290 |
+
sys.stdout.write(' ')
|
| 291 |
+
|
| 292 |
+
#(mingcv) Go back to the center of the bar.
|
| 293 |
+
for i in range(term_width - int(TOTAL_BAR_LENGTH / 2) + 2):
|
| 294 |
+
sys.stdout.write('\b')
|
| 295 |
+
sys.stdout.write(' %d/%d ' % (current + 1, total))
|
| 296 |
+
|
| 297 |
+
if current < total - 1:
|
| 298 |
+
sys.stdout.write('\r')
|
| 299 |
+
else:
|
| 300 |
+
sys.stdout.write('\n')
|
| 301 |
+
sys.stdout.flush()
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def format_time(seconds): #convert seconds to days,hourse,mintures,seconds,milliseconds
|
| 305 |
+
days = int(seconds / 3600 / 24)
|
| 306 |
+
seconds = seconds - days * 3600 * 24
|
| 307 |
+
hours = int(seconds / 3600)
|
| 308 |
+
seconds = seconds - hours * 3600
|
| 309 |
+
minutes = int(seconds / 60)
|
| 310 |
+
seconds = seconds - minutes * 60
|
| 311 |
+
secondsf = int(seconds)
|
| 312 |
+
seconds = seconds - secondsf
|
| 313 |
+
millis = int(seconds * 1000)
|
| 314 |
+
|
| 315 |
+
f = ''
|
| 316 |
+
i = 1
|
| 317 |
+
if days > 0:
|
| 318 |
+
f += str(days) + 'D'
|
| 319 |
+
i += 1
|
| 320 |
+
if hours > 0 and i <= 2:
|
| 321 |
+
f += str(hours) + 'h'
|
| 322 |
+
i += 1
|
| 323 |
+
if minutes > 0 and i <= 2:
|
| 324 |
+
f += str(minutes) + 'm'
|
| 325 |
+
i += 1
|
| 326 |
+
if secondsf > 0 and i <= 2:
|
| 327 |
+
f += str(secondsf) + 's'
|
| 328 |
+
i += 1
|
| 329 |
+
if millis > 0 and i <= 2:
|
| 330 |
+
f += str(millis) + 'ms'
|
| 331 |
+
i += 1
|
| 332 |
+
if f == '':
|
| 333 |
+
f = '0ms'
|
| 334 |
+
return f
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
def parse_args(args): #converts numeric args seperated by commas into a [] of ints
|
| 338 |
+
str_args = args.split(',') #str1,atr2,str3 becomes [a,b,c]
|
| 339 |
+
parsed_args = []
|
| 340 |
+
for str_arg in str_args:
|
| 341 |
+
arg = int(str_arg)
|
| 342 |
+
if arg >= 0:
|
| 343 |
+
parsed_args.append(arg)
|
| 344 |
+
return parsed_args
|
| 345 |
+
|
| 346 |
+
#In order to overcome vanishing/explosding gradient, Xavier initialization was introduced. It tries to keep variance of all the layers equal but assumes linear actiivation
|
| 347 |
+
# Kaiming He initialization, which takes activation function into account.
|
| 348 |
+
|
| 349 |
+
def weights_init_kaiming(m): #layer or module in NN
|
| 350 |
+
classname = m.__class__.__name__ # gets the class name of the layer m using its __class__ attribute and __name__ property.
|
| 351 |
+
if classname.find('Conv') != -1: #returns -1 if the substring is not found, So here if found
|
| 352 |
+
nn.init.kaiming_normal(m.weight.data, a=0, mode='fan_in')
|
| 353 |
+
|
| 354 |
+
#nn.init.kaiming_normal:Fill the input Tensor(m.weight.data) with values using a Kaiming normal distribution.The method is described in Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification - He, K. et al. (2015). The resulting tensor will have values sampled from N(0,std^2) where std=gain/sqrt(fan_mode)
|
| 355 |
+
#a (float) – the negative slope of the rectifier used after this layer only used with Leaky RelU (0-not used)
|
| 356 |
+
#mode: fan_in(default) or fan_out.Choosing 'fan_in' preserves the magnitude of the variance of the weights in the forward pass.
|
| 357 |
+
#here intialising for relu,preserving variance of weights in forward pass
|
| 358 |
+
elif classname.find('Linear') != -1:
|
| 359 |
+
nn.init.kaiming_normal(m.weight.data, a=0, mode='fan_in')
|
| 360 |
+
# here intialising for relu,preserving variance of weights in forward pass
|
| 361 |
+
elif classname.find('BatchNorm') != -1:
|
| 362 |
+
# nn.init.uniform(m.weight.data, 1.0, 0.02)
|
| 363 |
+
m.weight.data.normal_(mean=0, std=math.sqrt(2. / 9. / 64.)).clamp_(-0.025, 0.025)
|
| 364 |
+
nn.init.constant(m.bias.data, 0.0)
|
| 365 |
+
"??? why std=math.sqrt(2. / 9. / 64.)"
|
| 366 |
+
# .clamp_(-0.025, 0.025): Clamps (limits) the values in the tensor to be between -0.025 and 0.025.
|
| 367 |
+
#bias of the batch normalization layer intiliazed to a constant value of 0.0.
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
def batch_PSNR(img, imclean, data_range): #calculates avg PSNR for 1 batch of images
|
| 371 |
+
#img batch of images to be evaluated with noise
|
| 372 |
+
#imclean: batch of ground truth images no noise
|
| 373 |
+
Img = img.data.cpu().numpy().astype(np.float32) #converts img to float numpy array in cpu
|
| 374 |
+
#- img.data: Detaches the data from the computation graph.
|
| 375 |
+
#cpu(): Transfers the tensor from GPU to CPU if it’s on GPU.
|
| 376 |
+
#numpy(): Converts the tensor to a NumPy array.
|
| 377 |
+
#astype(np.float32): Converts the array type to float32.
|
| 378 |
+
Iclean = imclean.data.cpu().numpy().astype(np.float32) #same for ground truth
|
| 379 |
+
PSNR = 0
|
| 380 |
+
for i in range(Img.shape[0]): #for all batches add the skmetric.psnr()
|
| 381 |
+
PSNR += compare_psnr(Iclean[i, :, :, :], Img[i, :, :, :], data_range=data_range)
|
| 382 |
+
#data range: maximum - inimum possible values
|
| 383 |
+
return PSNR / Img.shape[0] #average psnr per batch
|
| 384 |
+
|
| 385 |
+
#calculates avg SSIM for 1 batch of images
|
| 386 |
+
def batch_SSIM(img, imclean):
|
| 387 |
+
Img = img.data.cpu().permute(0, 2, 3, 1).numpy().astype(np.float32)
|
| 388 |
+
#converts img to float numpy array in cpu
|
| 389 |
+
#img.data: Detaches the data from the computation graph.
|
| 390 |
+
#cpu(): Transfers the tensor from GPU to CPU if it’s on GPU.
|
| 391 |
+
#numpy(): Converts the tensor to a NumPy array.
|
| 392 |
+
#astype(np.float32): Converts the array type to float32.
|
| 393 |
+
Iclean = imclean.data.cpu().permute(0, 2, 3, 1).numpy().astype(np.float32)
|
| 394 |
+
#permute(0, 2, 3, 1): Changes the order of dimensions from (batch_size, channels, height, width) to (batch_size, height, width, channels).
|
| 395 |
+
SSIM = 0
|
| 396 |
+
|
| 397 |
+
for i in range(Img.shape[0]): #for all batches add the skmetric.ssim()
|
| 398 |
+
SSIM += structural_similarity(Iclean[i, :, :, :], Img[i, :, :, :], win_size=11,
|
| 399 |
+
multichannel=True, data_range=1)
|
| 400 |
+
#win_size:The side-length of the sliding window used in comparison. Must be an odd value
|
| 401 |
+
#multichannel deprecated ...equivalent to channel-axis
|
| 402 |
+
"??? multichannel=True equal to what number in channel-axis"
|
| 403 |
+
return SSIM / Img.shape[0] #average ssim for1 batch
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
def data_augmentation(image, mode): #augment given image based on mode passed by manipulating numpy array
|
| 407 |
+
out = np.transpose(image, (1, 2, 0))
|
| 408 |
+
if mode == 0:
|
| 409 |
+
#(mingcv)original
|
| 410 |
+
out = out
|
| 411 |
+
elif mode == 1:
|
| 412 |
+
#(mingcv)flip up and down
|
| 413 |
+
out = np.flipud(out)
|
| 414 |
+
elif mode == 2:
|
| 415 |
+
#(mingcv)rotate counterwise 90 degree
|
| 416 |
+
out = np.rot90(out)
|
| 417 |
+
elif mode == 3:
|
| 418 |
+
#(mingcv)rotate 90 degree and flip up and down
|
| 419 |
+
out = np.rot90(out)
|
| 420 |
+
out = np.flipud(out)
|
| 421 |
+
elif mode == 4:
|
| 422 |
+
#(mingcv)rotate 180 degree
|
| 423 |
+
out = np.rot90(out, k=2)
|
| 424 |
+
elif mode == 5:
|
| 425 |
+
#(mingcv)rotate 180 degree and flip
|
| 426 |
+
out = np.rot90(out, k=2)
|
| 427 |
+
out = np.flipud(out)
|
| 428 |
+
elif mode == 6:
|
| 429 |
+
#(mingcv)rotate 270 degree
|
| 430 |
+
out = np.rot90(out, k=3)
|
| 431 |
+
elif mode == 7:
|
| 432 |
+
#(mingcv)rotate 270 degree and flip
|
| 433 |
+
out = np.rot90(out, k=3)
|
| 434 |
+
out = np.flipud(out)
|
| 435 |
+
return np.transpose(out, (2, 0, 1))
|
DSRNet/util/visualizer.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
import os
|
| 3 |
+
import ntpath #for handling Windows paths.
|
| 4 |
+
import time
|
| 5 |
+
from . import util # . = same package
|
| 6 |
+
from . import html
|
| 7 |
+
import visdom # for visualizing data.
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class Visualizer():
|
| 11 |
+
def __init__(self, opt):
|
| 12 |
+
self.display_id = opt.display_id #In Visdom, you can create multiple windows to display various types of information, such as images, graphs, or tables, during the training and testing of machine learning models. The display_id specifies which window (by ID) the visualizations should be displayed in.
|
| 13 |
+
self.use_html = opt.isTrain and not opt.no_html
|
| 14 |
+
#opt.isTrain==True means using train_options.py subclass
|
| 15 |
+
#HTML visualization in the context of deep learning involves generating web-based visual representations of the training process, model performance, and data. This can be particularly useful for monitoring, debugging, and sharing results.
|
| 16 |
+
self.win_size = opt.display_winsize
|
| 17 |
+
self.name = opt.name
|
| 18 |
+
self.opt = opt
|
| 19 |
+
self.saved = False
|
| 20 |
+
if self.display_id > 0: #if diplay_id==0, disable Visdom
|
| 21 |
+
self.vis = visdom.Visdom(port=opt.display_port, ipv6=False)
|
| 22 |
+
|
| 23 |
+
if self.use_html: #logging training loss
|
| 24 |
+
self.web_dir = os.path.join(opt.checkpoints_dir, opt.name, 'web')
|
| 25 |
+
self.img_dir = os.path.join(self.web_dir, 'images')
|
| 26 |
+
print('create web directory %s...' % self.web_dir)
|
| 27 |
+
util.mkdirs([self.web_dir, self.img_dir])
|
| 28 |
+
self.log_name = os.path.join(opt.checkpoints_dir, opt.name, 'loss_log.txt')
|
| 29 |
+
with open(self.log_name, "a") as log_file:
|
| 30 |
+
now = time.strftime("%c")
|
| 31 |
+
log_file.write('================ Training Loss (%s) ================\n' % now)
|
| 32 |
+
|
| 33 |
+
def reset(self):
|
| 34 |
+
self.saved = False
|
| 35 |
+
|
| 36 |
+
# (mingcv)|visuals|: dictionary of images to display or save
|
| 37 |
+
def display_current_results(self, visuals, epoch, save_result):
|
| 38 |
+
if self.display_id > 0: #(mingcv)show images in the browser
|
| 39 |
+
ncols = self.opt.display_single_pane_ncols
|
| 40 |
+
if ncols > 0:
|
| 41 |
+
h, w = next(iter(visuals.values())).shape[:2]
|
| 42 |
+
table_css = """<style>
|
| 43 |
+
table {border-collapse: separate; border-spacing:4px; white-space:nowrap; text-align:center}
|
| 44 |
+
table td {width: %dpx; height: %dpx; padding: 4px; outline: 4px solid black}
|
| 45 |
+
</style>""" % (w, h)
|
| 46 |
+
title = self.name
|
| 47 |
+
label_html = ''
|
| 48 |
+
label_html_row = ''
|
| 49 |
+
nrows = int(np.ceil(len(visuals.items()) / ncols))
|
| 50 |
+
images = []
|
| 51 |
+
idx = 0
|
| 52 |
+
for label, image_numpy in visuals.items():
|
| 53 |
+
label_html_row += '<td>%s</td>' % label
|
| 54 |
+
images.append(image_numpy.transpose([2, 0, 1]))
|
| 55 |
+
idx += 1
|
| 56 |
+
if idx % ncols == 0:
|
| 57 |
+
label_html += '<tr>%s</tr>' % label_html_row
|
| 58 |
+
label_html_row = ''
|
| 59 |
+
white_image = np.ones_like(image_numpy.transpose([2, 0, 1])) * 255
|
| 60 |
+
while idx % ncols != 0:
|
| 61 |
+
images.append(white_image)
|
| 62 |
+
label_html_row += '<td></td>'
|
| 63 |
+
idx += 1
|
| 64 |
+
if label_html_row != '':
|
| 65 |
+
label_html += '<tr>%s</tr>' % label_html_row
|
| 66 |
+
# pane col = image row
|
| 67 |
+
self.vis.images(images, nrow=ncols, env=self.opt.name, win=self.display_id + 1,
|
| 68 |
+
padding=2, opts=dict(title=title + ' images'))
|
| 69 |
+
label_html = '<table>%s</table>' % label_html
|
| 70 |
+
self.vis.text(table_css + label_html, env=self.opt.name, win=self.display_id + 2,
|
| 71 |
+
opts=dict(title=title + ' labels'))
|
| 72 |
+
else:
|
| 73 |
+
idx = 1
|
| 74 |
+
for label, image_numpy in visuals.items():
|
| 75 |
+
self.vis.image(image_numpy.transpose([2, 0, 1]), opts=dict(title=label),
|
| 76 |
+
env=self.opt.name,
|
| 77 |
+
win=self.display_id + idx)
|
| 78 |
+
idx += 1
|
| 79 |
+
|
| 80 |
+
if self.use_html and (save_result or not self.saved): # save images to a html file
|
| 81 |
+
self.saved = True
|
| 82 |
+
for label, image_numpy in visuals.items():
|
| 83 |
+
img_path = os.path.join(self.img_dir, 'epoch%.3d_%s.png' % (epoch, label))
|
| 84 |
+
util.save_image(image_numpy, img_path)
|
| 85 |
+
# update website
|
| 86 |
+
webpage = html.HTML(self.web_dir, 'Experiment name = %s' % self.name, reflesh=1)
|
| 87 |
+
for n in range(epoch, 0, -1):
|
| 88 |
+
webpage.add_header('epoch [%d]' % n)
|
| 89 |
+
ims = []
|
| 90 |
+
txts = []
|
| 91 |
+
links = []
|
| 92 |
+
|
| 93 |
+
for label, image_numpy in visuals.items():
|
| 94 |
+
img_path = 'epoch%.3d_%s.png' % (n, label)
|
| 95 |
+
ims.append(img_path)
|
| 96 |
+
txts.append(label)
|
| 97 |
+
links.append(img_path)
|
| 98 |
+
webpage.add_images(ims, txts, links, height=self.win_size)
|
| 99 |
+
webpage.save()
|
| 100 |
+
|
| 101 |
+
#(mingcv) errors: dictionary of error labels and values
|
| 102 |
+
def plot_current_errors(self, epoch, counter_ratio, opt, errors):
|
| 103 |
+
if not hasattr(self, 'plot_data'):
|
| 104 |
+
self.plot_data = {'X': [], 'Y': [], 'legend': list(errors.keys())}
|
| 105 |
+
self.plot_data['X'].append(epoch + counter_ratio)
|
| 106 |
+
self.plot_data['Y'].append([errors[k] for k in self.plot_data['legend']])
|
| 107 |
+
self.vis.line(
|
| 108 |
+
X=np.stack([np.array(self.plot_data['X'])] * len(self.plot_data['legend']), 1),
|
| 109 |
+
Y=np.array(self.plot_data['Y']),
|
| 110 |
+
opts={
|
| 111 |
+
'title': self.name + ' loss over time',
|
| 112 |
+
'legend': self.plot_data['legend'],
|
| 113 |
+
'xlabel': 'epoch',
|
| 114 |
+
'ylabel': 'loss'},
|
| 115 |
+
win=self.display_id,
|
| 116 |
+
env=self.opt.name)
|
| 117 |
+
|
| 118 |
+
# errors: same format as |errors| of plotCurrentErrors
|
| 119 |
+
def print_current_errors(self, epoch, i, errors, t):
|
| 120 |
+
message = '(epoch: %d, iters: %d, time: %.3f) ' % (epoch, i, t)
|
| 121 |
+
for k, v in errors.items():
|
| 122 |
+
message += '%s: %.3f ' % (k, v)
|
| 123 |
+
|
| 124 |
+
print(message)
|
| 125 |
+
with open(self.log_name, "a") as log_file:
|
| 126 |
+
log_file.write('%s\n' % message)
|
| 127 |
+
|
| 128 |
+
# (mingcv) save image to the disk
|
| 129 |
+
def save_images(self, webpage, visuals, image_path, aspect_ratio=1.0):
|
| 130 |
+
image_dir = webpage.get_image_dir()
|
| 131 |
+
short_path = ntpath.basename(image_path[0])
|
| 132 |
+
name = os.path.splitext(short_path)[0]
|
| 133 |
+
|
| 134 |
+
webpage.add_header(name)
|
| 135 |
+
ims = []
|
| 136 |
+
txts = []
|
| 137 |
+
links = []
|
| 138 |
+
|
| 139 |
+
for label, im in visuals.items():
|
| 140 |
+
image_name = '%s_%s.png' % (name, label)
|
| 141 |
+
save_path = os.path.join(image_dir, image_name)
|
| 142 |
+
h, w, _ = im.shape
|
| 143 |
+
if aspect_ratio > 1.0:
|
| 144 |
+
im = np.array(Image.fromarray(im).resize((h, int(w * aspect_ratio))))
|
| 145 |
+
if aspect_ratio < 1.0:
|
| 146 |
+
im = np.array(Image.fromarray(im).resize((h, int(h / aspect_ratio))))
|
| 147 |
+
util.save_image(im, save_path)
|
| 148 |
+
|
| 149 |
+
ims.append(image_name)
|
| 150 |
+
txts.append(label)
|
| 151 |
+
links.append(image_name)
|
| 152 |
+
webpage.add_images(ims, txts, links, height=self.win_size)
|
DSRNet/weights/dsrnet_s_epoch14.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3d85041c53c025511c654b1b42da35a404d6c76cb1fb790a2adad4028875554c
|
| 3 |
+
size 39764466
|