| import subprocess | |
| import sys | |
| import time | |
| import os | |
| # Toàn bộ danh sách participants từ p00 đến p14 | |
| ALL_PARTICIPANTS = [f"p{i:02d}" for i in range(15)] | |
| # Danh sách đã hoàn thành (Cập nhật sau khi dừng) | |
| COMPLETED = ["p00", "p01", "p02", "p03", "p11"] | |
| TODO = [p for p in ALL_PARTICIPANTS if p not in COMPLETED] | |
| print(f"{'='*60}") | |
| print(f" RESUMING 8x8 ABLATION BATCH TRAINING ") | |
| print(f" Participants remaining: {TODO} ") | |
| print(f"{'='*60}") | |
| start_total = time.time() | |
| for participant in TODO: | |
| print(f"\n{'#'*40}") | |
| print(f" TRAINING 8x8 FOR: {participant} ") | |
| print(f"{'#'*40}") | |
| cmd = [ | |
| sys.executable, "src/train_8x8_ablation.py", | |
| "--participant", participant, | |
| "--epochs", "50", | |
| "--workers", "4" | |
| ] | |
| start_fold = time.time() | |
| result = subprocess.run(cmd) | |
| end_fold = time.time() | |
| if result.returncode == 0: | |
| print(f"SUCCESS: {participant} 8x8 training completed in {(end_fold - start_fold)/60:.2f} mins.") | |
| else: | |
| print(f"FAILED: {participant} 8x8 training with exit code {result.returncode}") | |
| end_total = time.time() | |
| print(f"\n{'='*60}") | |
| print(f" 8x8 BATCH COMPLETED in {(end_total - start_total)/3600:.2f} hours ") | |
| print(f"{'='*60}") | |