supli6669 commited on
Commit
21dc5ba
·
1 Parent(s): c0c6431

fix: train_custom.py never writes to original yml — use temp file to prevent total_iter corruption

Browse files
models/CodeFormer/options/CodeFormer_stage3_custom.yml CHANGED
@@ -125,7 +125,7 @@ train:
125
  restart_weights:
126
  - 1
127
  eta_min: 5.0e-06
128
- total_iter: 2002
129
  warmup_iter: -1
130
  ema_decay: 0.997
131
  pixel_opt:
 
125
  restart_weights:
126
  - 1
127
  eta_min: 5.0e-06
128
+ total_iter: 20000
129
  warmup_iter: -1
130
  ema_decay: 0.997
131
  pixel_opt:
train_custom.py CHANGED
@@ -48,6 +48,10 @@ def main():
48
  print(f"Reading configuration from {config_path}...")
49
  with open(config_path, "r", encoding="utf-8") as f:
50
  config = yaml.safe_load(f)
 
 
 
 
51
 
52
  # Dynamically set GPU count
53
  config["num_gpu"] = num_gpus
@@ -113,49 +117,63 @@ def main():
113
  else:
114
  print(f" Training from scratch to {config.get('train', {}).get('total_iter', 20000)}...")
115
 
116
- # Write back the updated configuration
117
- with open(config_path, "w", encoding="utf-8") as f:
118
- yaml.dump(config, f, default_flow_style=False, sort_keys=False)
119
- print(f"Updated configuration file: device={device.upper()}, num_gpu={num_gpus}, total_iter={config['train']['total_iter']}")
120
-
121
- # 5. Run the training process
122
- train_script = os.path.join("basicsr", "train.py")
123
- cmd = [
124
- sys.executable,
125
- train_script,
126
- "-opt",
127
- os.path.join("options", "CodeFormer_stage3_custom.yml"),
128
- "--launcher",
129
- "none"
130
- ]
131
-
132
- # Add models/CodeFormer to PYTHONPATH
133
- env = os.environ.copy()
134
- # Force torch / BLAS backends to use limited CPU threads for stability
135
- for _k in ["OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS",
136
- "NUMEXPR_NUM_THREADS", "VECLIB_MAXIMUM_THREADS"]:
137
- env[_k] = "8" # Match torch.set_num_threads — Ryzen 7735HS 8C/16T
138
- # Disable OpenCV threading and OpenCL runtime (prevents segfaults on Windows)
139
- env["OPENCV_OPENCL_RUNTIME"] = "disabled"
140
- env["OPENCV_THREAD_LIMIT"] = "1"
141
- cv2.setNumThreads(0)
142
- # Let torch auto-detect CPU capabilities to avoid SIGILL/segfaults.
143
- env["PYTHONPATH"] = os.path.pathsep.join([codeformer_dir, env.get("PYTHONPATH", "")])
144
-
145
- print("\nStarting training process. Command:")
146
- print(" ".join(cmd))
147
- print(f"Working directory: {codeformer_dir}")
148
- print("------------------------------------------")
149
-
150
  try:
151
- # Run subprocess under models/CodeFormer working directory
152
- result = subprocess.run(cmd, cwd=codeformer_dir, env=env, check=True)
153
- print("------------------------------------------")
154
- print("Training execution completed successfully!")
155
- except subprocess.CalledProcessError as e:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
156
  print("------------------------------------------")
157
- print(f"Training failed with exit code: {e.returncode}")
158
- sys.exit(e.returncode)
 
 
 
 
 
 
 
 
 
 
 
159
 
160
  if __name__ == "__main__":
161
  main()
 
48
  print(f"Reading configuration from {config_path}...")
49
  with open(config_path, "r", encoding="utf-8") as f:
50
  config = yaml.safe_load(f)
51
+
52
+ # Save original total_iter BEFORE any runtime overrides — we must restore
53
+ # this after writing so --verify mode never permanently corrupts the file.
54
+ original_total_iter = config.get("train", {}).get("total_iter", 20000)
55
 
56
  # Dynamically set GPU count
57
  config["num_gpu"] = num_gpus
 
117
  else:
118
  print(f" Training from scratch to {config.get('train', {}).get('total_iter', 20000)}...")
119
 
120
+ # 5. Write runtime config to a TEMP yml file — NEVER modify the original.
121
+ #
122
+ # Strategy: write all runtime overrides (num_gpu, prefetch_mode, resume_state,
123
+ # total_iter) to a separate temp file. basicsr/train.py reads from that temp
124
+ # file. The canonical yml with all comments is never touched again.
125
+ #
126
+ # This fixes two bugs permanently:
127
+ # (a) --verify mode used to persist total_iter=checkpoint+2 to disc, causing
128
+ # the next full run to stop after only 2 iterations.
129
+ # (b) yaml.dump was stripping all comments from the original yml every run.
130
+ import tempfile, shutil
131
+ tmp_fd, tmp_config_path = tempfile.mkstemp(suffix=".yml", prefix="cf_runtime_",
132
+ dir=os.path.join(codeformer_dir, "options"))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
133
  try:
134
+ with os.fdopen(tmp_fd, "w", encoding="utf-8") as f:
135
+ yaml.dump(config, f, default_flow_style=False, sort_keys=False)
136
+ print(f"Runtime config written to temp file: {os.path.basename(tmp_config_path)}")
137
+ print(f" device={device.upper()}, num_gpu={num_gpus}, total_iter={config['train']['total_iter']}")
138
+
139
+ # 6. Run the training process pointing at the temp config
140
+ train_script = os.path.join("basicsr", "train.py")
141
+ cmd = [
142
+ sys.executable,
143
+ train_script,
144
+ "-opt",
145
+ os.path.relpath(tmp_config_path, codeformer_dir),
146
+ "--launcher",
147
+ "none"
148
+ ]
149
+
150
+ # Environment setup — CPU stability settings (Task 8 findings)
151
+ env = os.environ.copy()
152
+ for _k in ["OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS",
153
+ "NUMEXPR_NUM_THREADS", "VECLIB_MAXIMUM_THREADS"]:
154
+ env[_k] = "8" # Match torch.set_num_threads — Ryzen 7735HS 8C/16T
155
+ env["OPENCV_OPENCL_RUNTIME"] = "disabled"
156
+ env["OPENCV_THREAD_LIMIT"] = "1"
157
+ cv2.setNumThreads(0)
158
+ env["PYTHONPATH"] = os.path.pathsep.join([codeformer_dir, env.get("PYTHONPATH", "")])
159
+
160
+ print("\nStarting training process. Command:")
161
+ print(" ".join(cmd))
162
+ print(f"Working directory: {codeformer_dir}")
163
  print("------------------------------------------")
164
+
165
+ try:
166
+ result = subprocess.run(cmd, cwd=codeformer_dir, env=env, check=True)
167
+ print("------------------------------------------")
168
+ print("Training execution completed successfully!")
169
+ except subprocess.CalledProcessError as e:
170
+ print("------------------------------------------")
171
+ print(f"Training failed with exit code: {e.returncode}")
172
+ sys.exit(e.returncode)
173
+ finally:
174
+ # Always clean up the temp file after the run (success or failure)
175
+ if os.path.exists(tmp_config_path):
176
+ os.remove(tmp_config_path)
177
 
178
  if __name__ == "__main__":
179
  main()