supli6669 commited on
Commit ·
21dc5ba
1
Parent(s): c0c6431
fix: train_custom.py never writes to original yml — use temp file to prevent total_iter corruption
Browse files
models/CodeFormer/options/CodeFormer_stage3_custom.yml
CHANGED
|
@@ -125,7 +125,7 @@ train:
|
|
| 125 |
restart_weights:
|
| 126 |
- 1
|
| 127 |
eta_min: 5.0e-06
|
| 128 |
-
total_iter:
|
| 129 |
warmup_iter: -1
|
| 130 |
ema_decay: 0.997
|
| 131 |
pixel_opt:
|
|
|
|
| 125 |
restart_weights:
|
| 126 |
- 1
|
| 127 |
eta_min: 5.0e-06
|
| 128 |
+
total_iter: 20000
|
| 129 |
warmup_iter: -1
|
| 130 |
ema_decay: 0.997
|
| 131 |
pixel_opt:
|
train_custom.py
CHANGED
|
@@ -48,6 +48,10 @@ def main():
|
|
| 48 |
print(f"Reading configuration from {config_path}...")
|
| 49 |
with open(config_path, "r", encoding="utf-8") as f:
|
| 50 |
config = yaml.safe_load(f)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
|
| 52 |
# Dynamically set GPU count
|
| 53 |
config["num_gpu"] = num_gpus
|
|
@@ -113,49 +117,63 @@ def main():
|
|
| 113 |
else:
|
| 114 |
print(f" Training from scratch to {config.get('train', {}).get('total_iter', 20000)}...")
|
| 115 |
|
| 116 |
-
# Write
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
#
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
"none"
|
| 130 |
-
]
|
| 131 |
-
|
| 132 |
-
# Add models/CodeFormer to PYTHONPATH
|
| 133 |
-
env = os.environ.copy()
|
| 134 |
-
# Force torch / BLAS backends to use limited CPU threads for stability
|
| 135 |
-
for _k in ["OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS",
|
| 136 |
-
"NUMEXPR_NUM_THREADS", "VECLIB_MAXIMUM_THREADS"]:
|
| 137 |
-
env[_k] = "8" # Match torch.set_num_threads — Ryzen 7735HS 8C/16T
|
| 138 |
-
# Disable OpenCV threading and OpenCL runtime (prevents segfaults on Windows)
|
| 139 |
-
env["OPENCV_OPENCL_RUNTIME"] = "disabled"
|
| 140 |
-
env["OPENCV_THREAD_LIMIT"] = "1"
|
| 141 |
-
cv2.setNumThreads(0)
|
| 142 |
-
# Let torch auto-detect CPU capabilities to avoid SIGILL/segfaults.
|
| 143 |
-
env["PYTHONPATH"] = os.path.pathsep.join([codeformer_dir, env.get("PYTHONPATH", "")])
|
| 144 |
-
|
| 145 |
-
print("\nStarting training process. Command:")
|
| 146 |
-
print(" ".join(cmd))
|
| 147 |
-
print(f"Working directory: {codeformer_dir}")
|
| 148 |
-
print("------------------------------------------")
|
| 149 |
-
|
| 150 |
try:
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
print("
|
| 154 |
-
print("
|
| 155 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
print("------------------------------------------")
|
| 157 |
-
|
| 158 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
|
| 160 |
if __name__ == "__main__":
|
| 161 |
main()
|
|
|
|
| 48 |
print(f"Reading configuration from {config_path}...")
|
| 49 |
with open(config_path, "r", encoding="utf-8") as f:
|
| 50 |
config = yaml.safe_load(f)
|
| 51 |
+
|
| 52 |
+
# Save original total_iter BEFORE any runtime overrides — we must restore
|
| 53 |
+
# this after writing so --verify mode never permanently corrupts the file.
|
| 54 |
+
original_total_iter = config.get("train", {}).get("total_iter", 20000)
|
| 55 |
|
| 56 |
# Dynamically set GPU count
|
| 57 |
config["num_gpu"] = num_gpus
|
|
|
|
| 117 |
else:
|
| 118 |
print(f" Training from scratch to {config.get('train', {}).get('total_iter', 20000)}...")
|
| 119 |
|
| 120 |
+
# 5. Write runtime config to a TEMP yml file — NEVER modify the original.
|
| 121 |
+
#
|
| 122 |
+
# Strategy: write all runtime overrides (num_gpu, prefetch_mode, resume_state,
|
| 123 |
+
# total_iter) to a separate temp file. basicsr/train.py reads from that temp
|
| 124 |
+
# file. The canonical yml with all comments is never touched again.
|
| 125 |
+
#
|
| 126 |
+
# This fixes two bugs permanently:
|
| 127 |
+
# (a) --verify mode used to persist total_iter=checkpoint+2 to disc, causing
|
| 128 |
+
# the next full run to stop after only 2 iterations.
|
| 129 |
+
# (b) yaml.dump was stripping all comments from the original yml every run.
|
| 130 |
+
import tempfile, shutil
|
| 131 |
+
tmp_fd, tmp_config_path = tempfile.mkstemp(suffix=".yml", prefix="cf_runtime_",
|
| 132 |
+
dir=os.path.join(codeformer_dir, "options"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
try:
|
| 134 |
+
with os.fdopen(tmp_fd, "w", encoding="utf-8") as f:
|
| 135 |
+
yaml.dump(config, f, default_flow_style=False, sort_keys=False)
|
| 136 |
+
print(f"Runtime config written to temp file: {os.path.basename(tmp_config_path)}")
|
| 137 |
+
print(f" device={device.upper()}, num_gpu={num_gpus}, total_iter={config['train']['total_iter']}")
|
| 138 |
+
|
| 139 |
+
# 6. Run the training process pointing at the temp config
|
| 140 |
+
train_script = os.path.join("basicsr", "train.py")
|
| 141 |
+
cmd = [
|
| 142 |
+
sys.executable,
|
| 143 |
+
train_script,
|
| 144 |
+
"-opt",
|
| 145 |
+
os.path.relpath(tmp_config_path, codeformer_dir),
|
| 146 |
+
"--launcher",
|
| 147 |
+
"none"
|
| 148 |
+
]
|
| 149 |
+
|
| 150 |
+
# Environment setup — CPU stability settings (Task 8 findings)
|
| 151 |
+
env = os.environ.copy()
|
| 152 |
+
for _k in ["OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS",
|
| 153 |
+
"NUMEXPR_NUM_THREADS", "VECLIB_MAXIMUM_THREADS"]:
|
| 154 |
+
env[_k] = "8" # Match torch.set_num_threads — Ryzen 7735HS 8C/16T
|
| 155 |
+
env["OPENCV_OPENCL_RUNTIME"] = "disabled"
|
| 156 |
+
env["OPENCV_THREAD_LIMIT"] = "1"
|
| 157 |
+
cv2.setNumThreads(0)
|
| 158 |
+
env["PYTHONPATH"] = os.path.pathsep.join([codeformer_dir, env.get("PYTHONPATH", "")])
|
| 159 |
+
|
| 160 |
+
print("\nStarting training process. Command:")
|
| 161 |
+
print(" ".join(cmd))
|
| 162 |
+
print(f"Working directory: {codeformer_dir}")
|
| 163 |
print("------------------------------------------")
|
| 164 |
+
|
| 165 |
+
try:
|
| 166 |
+
result = subprocess.run(cmd, cwd=codeformer_dir, env=env, check=True)
|
| 167 |
+
print("------------------------------------------")
|
| 168 |
+
print("Training execution completed successfully!")
|
| 169 |
+
except subprocess.CalledProcessError as e:
|
| 170 |
+
print("------------------------------------------")
|
| 171 |
+
print(f"Training failed with exit code: {e.returncode}")
|
| 172 |
+
sys.exit(e.returncode)
|
| 173 |
+
finally:
|
| 174 |
+
# Always clean up the temp file after the run (success or failure)
|
| 175 |
+
if os.path.exists(tmp_config_path):
|
| 176 |
+
os.remove(tmp_config_path)
|
| 177 |
|
| 178 |
if __name__ == "__main__":
|
| 179 |
main()
|