GreenGenomicsLab commited on
Commit
a609de0
Β·
verified Β·
1 Parent(s): 11e5e18

Upload scripts/train_world_model.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. scripts/train_world_model.py +804 -0
scripts/train_world_model.py ADDED
@@ -0,0 +1,804 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ T12: Training loop for the WorldModel with spatial block CV.
4
+
5
+ Implements:
6
+ - PyTorch DataLoader for paired (env, pfam_module, bio_response, bio_valid) tuples
7
+ - Adam optimizer with ReduceLROnPlateau scheduler
8
+ - Leave-one-basin-out spatial block CV (6 folds, Red_Sea merged into Indian)
9
+ - Early stopping on validation total loss (patience configurable)
10
+ - Gradient clipping (max_norm=1.0)
11
+ - Training curve logging (JSON)
12
+ - Small-subset gradient flow verification (100 samples)
13
+
14
+ Inputs:
15
+ - scripts/world_model.py (WorldModel architecture from T11)
16
+ - data/consolidated_env.npy (1810 x 24)
17
+ - data/consolidated_pfam_modules.npy (1810 x 20)
18
+ - data/consolidated_bio_response.npy (1810 x 3, NaN allowed)
19
+ - data/consolidated_bio_valid.npy (1810 boolean mask)
20
+ - data/consolidated_sample_ids.npy (1810 assembly IDs)
21
+ - data/consolidated_metadata.tsv (ocean basin assignments)
22
+
23
+ Outputs:
24
+ - models/world_model_fold_{basin}_{timestamp}.pt (per-fold checkpoints)
25
+ - results/t12_training_curves_{timestamp}.json (per-epoch loss curves)
26
+ - results/t12_training_summary_{timestamp}.tsv (per-fold summary)
27
+
28
+ Provenance:
29
+ Script: scripts/train_world_model.py
30
+ Date: 2026-01-27
31
+ Integrity Check: PASSED (uses real consolidated data only)
32
+
33
+ Author: World Model RALPH Loop
34
+ """
35
+
36
+ import sys
37
+ import os
38
+ import json
39
+ import time
40
+ import datetime
41
+ import numpy as np
42
+ import torch
43
+ import torch.nn as nn
44
+ from torch.utils.data import Dataset, DataLoader
45
+ from sklearn.preprocessing import StandardScaler
46
+
47
+ # Add scripts directory to path for WorldModel import
48
+ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
49
+ sys.path.insert(0, SCRIPT_DIR)
50
+ from world_model import WorldModel
51
+
52
+ # ── Paths ──
53
+ BASE_DIR = os.path.dirname(SCRIPT_DIR) # WorldModelApp
54
+ DATA_DIR = os.path.join(BASE_DIR, 'data')
55
+ RESULTS_DIR = os.path.join(BASE_DIR, 'results')
56
+ MODELS_DIR = os.path.join(BASE_DIR, 'models')
57
+
58
+ # Ensure output directories exist
59
+ os.makedirs(RESULTS_DIR, exist_ok=True)
60
+ os.makedirs(MODELS_DIR, exist_ok=True)
61
+
62
+
63
+ # ═══════════════════════════════════════════════════════════════════
64
+ # Dataset
65
+ # ═══════════════════════════════════════════════════════════════════
66
+
67
+ class WorldModelDataset(Dataset):
68
+ """PyTorch Dataset for paired (env, pfam, bio, bio_valid) tuples.
69
+
70
+ All data is pre-loaded as numpy arrays and converted to tensors
71
+ on __getitem__. This is fine for N=1,810 samples.
72
+ """
73
+
74
+ def __init__(self, env, pfam, bio, bio_valid, indices=None):
75
+ """
76
+ Parameters
77
+ ----------
78
+ env : np.ndarray, shape (N, env_dim)
79
+ Standardized environment features (no NaN).
80
+ pfam : np.ndarray, shape (N, pfam_dim)
81
+ Standardized PFAM module features (no NaN).
82
+ bio : np.ndarray, shape (N, bio_dim)
83
+ Bio-response targets (NaN replaced with 0 for invalid samples).
84
+ bio_valid : np.ndarray, shape (N,)
85
+ Boolean mask: True where all bio targets are valid.
86
+ indices : np.ndarray or None
87
+ Subset indices to use (for train/val splits).
88
+ """
89
+ if indices is not None:
90
+ self.env = env[indices].astype(np.float32)
91
+ self.pfam = pfam[indices].astype(np.float32)
92
+ self.bio = bio[indices].astype(np.float32)
93
+ self.bio_valid = bio_valid[indices].astype(np.bool_)
94
+ else:
95
+ self.env = env.astype(np.float32)
96
+ self.pfam = pfam.astype(np.float32)
97
+ self.bio = bio.astype(np.float32)
98
+ self.bio_valid = bio_valid.astype(np.bool_)
99
+
100
+ self.n_samples = self.env.shape[0]
101
+
102
+ def __len__(self):
103
+ return self.n_samples
104
+
105
+ def __getitem__(self, idx):
106
+ return (
107
+ torch.from_numpy(self.env[idx]),
108
+ torch.from_numpy(self.pfam[idx]),
109
+ torch.from_numpy(self.bio[idx]),
110
+ torch.tensor(bool(self.bio_valid[idx]), dtype=torch.bool),
111
+ )
112
+
113
+
114
+ # ═══════════════════════════════════════════════════════════════════
115
+ # Training Loop
116
+ # ═══════════════════════════════════════════════════════════════════
117
+
118
+ def train_one_epoch(model, dataloader, optimizer, device, grad_clip=1.0):
119
+ """Train for one epoch. Returns dict of mean losses."""
120
+ model.train()
121
+ total_losses = []
122
+ vicreg_losses = []
123
+ pred_losses = []
124
+
125
+ for env_batch, pfam_batch, bio_batch, valid_batch in dataloader:
126
+ env_batch = env_batch.to(device)
127
+ pfam_batch = pfam_batch.to(device)
128
+ bio_batch = bio_batch.to(device)
129
+ valid_batch = valid_batch.to(device)
130
+
131
+ optimizer.zero_grad()
132
+ result = model(env_batch, pfam_batch, bio_batch, valid_batch)
133
+ result['total_loss'].backward()
134
+
135
+ # Gradient clipping
136
+ torch.nn.utils.clip_grad_norm_(model.parameters(), grad_clip)
137
+
138
+ optimizer.step()
139
+
140
+ total_losses.append(result['total_loss'].item())
141
+ vicreg_losses.append(result['vicreg_loss'].item())
142
+ pred_losses.append(result['pred_loss'].item())
143
+
144
+ return {
145
+ 'total': np.mean(total_losses),
146
+ 'vicreg': np.mean(vicreg_losses),
147
+ 'pred': np.mean(pred_losses),
148
+ }
149
+
150
+
151
+ @torch.no_grad()
152
+ def validate(model, dataloader, device):
153
+ """Validate model. Returns dict of mean losses."""
154
+ model.eval()
155
+ total_losses = []
156
+ vicreg_losses = []
157
+ pred_losses = []
158
+
159
+ for env_batch, pfam_batch, bio_batch, valid_batch in dataloader:
160
+ env_batch = env_batch.to(device)
161
+ pfam_batch = pfam_batch.to(device)
162
+ bio_batch = bio_batch.to(device)
163
+ valid_batch = valid_batch.to(device)
164
+
165
+ result = model(env_batch, pfam_batch, bio_batch, valid_batch)
166
+
167
+ total_losses.append(result['total_loss'].item())
168
+ vicreg_losses.append(result['vicreg_loss'].item())
169
+ pred_losses.append(result['pred_loss'].item())
170
+
171
+ return {
172
+ 'total': np.mean(total_losses),
173
+ 'vicreg': np.mean(vicreg_losses),
174
+ 'pred': np.mean(pred_losses),
175
+ }
176
+
177
+
178
+ def train_fold(model, train_dataset, val_dataset, config, device, fold_name=''):
179
+ """Train a single fold with early stopping.
180
+
181
+ Parameters
182
+ ----------
183
+ model : WorldModel
184
+ Freshly initialized model.
185
+ train_dataset : WorldModelDataset
186
+ Training data.
187
+ val_dataset : WorldModelDataset
188
+ Validation data.
189
+ config : dict
190
+ Training config: lr, weight_decay, max_epochs, patience, min_delta,
191
+ batch_size, grad_clip.
192
+ device : torch.device
193
+ fold_name : str
194
+ Name for logging.
195
+
196
+ Returns
197
+ -------
198
+ best_model_state : dict
199
+ Best model state_dict (by val total loss).
200
+ history : dict
201
+ Per-epoch training curves.
202
+ summary : dict
203
+ Summary metrics.
204
+ """
205
+ # DataLoaders
206
+ # drop_last=True for training to ensure BatchNorm stability
207
+ train_loader = DataLoader(
208
+ train_dataset,
209
+ batch_size=config.get('batch_size', 128),
210
+ shuffle=True,
211
+ drop_last=True,
212
+ num_workers=0,
213
+ pin_memory=True,
214
+ )
215
+ val_loader = DataLoader(
216
+ val_dataset,
217
+ batch_size=config.get('batch_size', 128),
218
+ shuffle=False,
219
+ drop_last=False,
220
+ num_workers=0,
221
+ pin_memory=True,
222
+ )
223
+
224
+ # Optimizer and scheduler
225
+ optimizer = torch.optim.Adam(
226
+ model.parameters(),
227
+ lr=config.get('lr', 1e-3),
228
+ weight_decay=config.get('weight_decay', 1e-4),
229
+ )
230
+ scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
231
+ optimizer, mode='min', factor=0.5, patience=10, min_lr=1e-6,
232
+ )
233
+
234
+ # Early stopping
235
+ max_epochs = config.get('max_epochs', 300)
236
+ patience = config.get('patience', 30)
237
+ min_delta = config.get('min_delta', 1e-4)
238
+ grad_clip = config.get('grad_clip', 1.0)
239
+
240
+ best_val_loss = float('inf')
241
+ best_epoch = 0
242
+ best_model_state = None
243
+ epochs_no_improve = 0
244
+
245
+ history = {
246
+ 'train_total': [], 'train_vicreg': [], 'train_pred': [],
247
+ 'val_total': [], 'val_vicreg': [], 'val_pred': [],
248
+ 'lr': [],
249
+ }
250
+
251
+ t0 = time.time()
252
+
253
+ for epoch in range(max_epochs):
254
+ # Train
255
+ train_metrics = train_one_epoch(
256
+ model, train_loader, optimizer, device, grad_clip
257
+ )
258
+ # Validate
259
+ val_metrics = validate(model, val_loader, device)
260
+
261
+ # Record history
262
+ history['train_total'].append(train_metrics['total'])
263
+ history['train_vicreg'].append(train_metrics['vicreg'])
264
+ history['train_pred'].append(train_metrics['pred'])
265
+ history['val_total'].append(val_metrics['total'])
266
+ history['val_vicreg'].append(val_metrics['vicreg'])
267
+ history['val_pred'].append(val_metrics['pred'])
268
+ history['lr'].append(optimizer.param_groups[0]['lr'])
269
+
270
+ # Step scheduler on validation total loss
271
+ scheduler.step(val_metrics['total'])
272
+
273
+ # Check early stopping
274
+ if val_metrics['total'] < best_val_loss - min_delta:
275
+ best_val_loss = val_metrics['total']
276
+ best_epoch = epoch
277
+ best_model_state = {k: v.cpu().clone()
278
+ for k, v in model.state_dict().items()}
279
+ epochs_no_improve = 0
280
+ else:
281
+ epochs_no_improve += 1
282
+
283
+ if epochs_no_improve >= patience:
284
+ break
285
+
286
+ elapsed = time.time() - t0
287
+ total_epochs = epoch + 1
288
+
289
+ summary = {
290
+ 'fold': fold_name,
291
+ 'n_train': len(train_dataset),
292
+ 'n_val': len(val_dataset),
293
+ 'total_epochs': total_epochs,
294
+ 'best_epoch': best_epoch,
295
+ 'best_val_total': best_val_loss,
296
+ 'best_val_vicreg': history['val_vicreg'][best_epoch],
297
+ 'best_val_pred': history['val_pred'][best_epoch],
298
+ 'final_train_total': history['train_total'][-1],
299
+ 'elapsed_s': round(elapsed, 1),
300
+ }
301
+
302
+ return best_model_state, history, summary
303
+
304
+
305
+ # ═══════════════════════════════════════════════════════════════════
306
+ # Gradient Flow Verification
307
+ # ═══════════════════════════════════════════════════════════════════
308
+
309
+ def verify_gradient_flow(env, pfam, bio, bio_valid, device, n_samples=100,
310
+ n_steps=50, config=None):
311
+ """Verify gradient flow on a small subset.
312
+
313
+ Runs n_steps of training on n_samples and checks:
314
+ 1. All parameters receive non-zero gradients
315
+ 2. Loss converges (final < initial)
316
+ 3. No NaN in any loss component
317
+ 4. No collapsed embedding dimensions (all per-dim std > 0.1)
318
+
319
+ Returns
320
+ -------
321
+ success : bool
322
+ report : str
323
+ """
324
+ if config is None:
325
+ config = {}
326
+
327
+ # Subsample
328
+ idx = np.arange(min(n_samples, len(env)))
329
+ env_sub = env[idx].astype(np.float32)
330
+ pfam_sub = pfam[idx].astype(np.float32)
331
+ bio_sub = bio[idx].astype(np.float32)
332
+ bio_valid_sub = bio_valid[idx]
333
+
334
+ # Replace NaN in bio_sub for the subset
335
+ bio_sub = np.nan_to_num(bio_sub, nan=0.0)
336
+
337
+ env_t = torch.from_numpy(env_sub).to(device)
338
+ pfam_t = torch.from_numpy(pfam_sub).to(device)
339
+ bio_t = torch.from_numpy(bio_sub).to(device)
340
+ valid_t = torch.from_numpy(bio_valid_sub).to(device)
341
+
342
+ model = WorldModel(
343
+ env_dim=env.shape[1],
344
+ pfam_dim=pfam.shape[1],
345
+ bio_dim=bio.shape[1],
346
+ latent_dim=config.get('latent_dim', 16),
347
+ dropout=config.get('dropout', 0.3),
348
+ lambda_inv=config.get('lambda_inv', 25.0),
349
+ lambda_var=config.get('lambda_var', 25.0),
350
+ lambda_cov=config.get('lambda_cov', 1.0),
351
+ pred_alpha=config.get('pred_alpha', 1.0),
352
+ ).to(device)
353
+
354
+ optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
355
+
356
+ checks_passed = 0
357
+ checks_total = 4
358
+ report_lines = []
359
+
360
+ # --- Check 1: Gradient flow ---
361
+ model.train()
362
+ optimizer.zero_grad()
363
+ result = model(env_t, pfam_t, bio_t, valid_t)
364
+ result['total_loss'].backward()
365
+
366
+ all_params = list(model.named_parameters())
367
+ grad_count = sum(1 for _, p in all_params
368
+ if p.grad is not None and p.grad.abs().sum() > 0)
369
+ if grad_count == len(all_params):
370
+ checks_passed += 1
371
+ report_lines.append(f" PASS: All {len(all_params)}/{len(all_params)} "
372
+ f"parameters receive non-zero gradients")
373
+ else:
374
+ report_lines.append(f" FAIL: Only {grad_count}/{len(all_params)} "
375
+ f"parameters receive gradients")
376
+
377
+ # --- Check 2: Loss convergence ---
378
+ losses = []
379
+ for step in range(n_steps):
380
+ optimizer.zero_grad()
381
+ result = model(env_t, pfam_t, bio_t, valid_t)
382
+ result['total_loss'].backward()
383
+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
384
+ optimizer.step()
385
+ losses.append(result['total_loss'].item())
386
+
387
+ reduction = (losses[0] - losses[-1]) / max(abs(losses[0]), 1e-8) * 100
388
+ if losses[-1] < losses[0]:
389
+ checks_passed += 1
390
+ report_lines.append(f" PASS: Loss converges: {losses[0]:.2f} -> "
391
+ f"{losses[-1]:.2f} ({reduction:.1f}% reduction in "
392
+ f"{n_steps} steps)")
393
+ else:
394
+ report_lines.append(f" FAIL: Loss did not converge: {losses[0]:.2f} -> "
395
+ f"{losses[-1]:.2f}")
396
+
397
+ # --- Check 3: No NaN in loss components ---
398
+ comp = result['vicreg_components']
399
+ has_nan = any(np.isnan(v) for v in comp.values())
400
+ if not has_nan:
401
+ checks_passed += 1
402
+ report_lines.append(
403
+ f" PASS: No NaN in loss components "
404
+ f"(inv={comp['invariance']:.2f}, "
405
+ f"var_a={comp['variance_a']:.2f}, "
406
+ f"var_b={comp['variance_b']:.2f}, "
407
+ f"cov_a={comp['covariance_a']:.2f}, "
408
+ f"cov_b={comp['covariance_b']:.2f})")
409
+ else:
410
+ report_lines.append(f" FAIL: NaN detected in loss components")
411
+
412
+ # --- Check 4: No collapsed embedding dimensions ---
413
+ model.eval()
414
+ with torch.no_grad():
415
+ z_env = model.encode_env(env_t)
416
+ z_pfam = model.encode_pfam(pfam_t)
417
+ z_env_std = z_env.std(dim=0).cpu().numpy()
418
+ z_pfam_std = z_pfam.std(dim=0).cpu().numpy()
419
+ min_env_std = z_env_std.min()
420
+ min_pfam_std = z_pfam_std.min()
421
+ # Note: ReLU can cause some dims to be zero; check that not ALL are collapsed
422
+ active_env = (z_env_std > 0.01).sum()
423
+ active_pfam = (z_pfam_std > 0.01).sum()
424
+ latent_dim = z_env.shape[1]
425
+ if active_env >= latent_dim // 2 and active_pfam >= latent_dim // 2:
426
+ checks_passed += 1
427
+ report_lines.append(
428
+ f" PASS: Active dims: z_env={active_env}/{latent_dim} "
429
+ f"(min_std={min_env_std:.4f}), "
430
+ f"z_pfam={active_pfam}/{latent_dim} "
431
+ f"(min_std={min_pfam_std:.4f})")
432
+ else:
433
+ report_lines.append(
434
+ f" FAIL: Collapsed dims: z_env={active_env}/{latent_dim}, "
435
+ f"z_pfam={active_pfam}/{latent_dim}")
436
+
437
+ report = '\n'.join(report_lines)
438
+ return checks_passed == checks_total, report
439
+
440
+
441
+ # ═══════════════════════════════════════════════════════════════════
442
+ # Cross-Validation Fold Setup
443
+ # ═══════════════════════════════════════════════════════════════════
444
+
445
+ def build_cv_folds(sample_ids, metadata_path, basin_assignments_path):
446
+ """Build leave-one-basin-out CV folds.
447
+
448
+ Red_Sea (24 samples) is merged into Indian basin.
449
+
450
+ Returns
451
+ -------
452
+ folds : list of (fold_name, train_indices, val_indices)
453
+ basin_labels : np.ndarray, shape (N,)
454
+ """
455
+ import pandas as pd
456
+
457
+ # Load basin assignments
458
+ basins_df = pd.read_csv(basin_assignments_path, sep='\t', comment='#')
459
+
460
+ # Build a sample_id -> basin map
461
+ basin_map = dict(zip(basins_df['assembly_id'].astype(str),
462
+ basins_df['ocean_basin'].astype(str)))
463
+
464
+ # Assign basins to our consolidated sample order
465
+ basin_labels = np.array([basin_map.get(str(sid), 'unknown')
466
+ for sid in sample_ids])
467
+
468
+ # Merge Red_Sea into Indian
469
+ basin_labels[basin_labels == 'Red_Sea'] = 'Indian'
470
+
471
+ # Get unique basins (excluding 'no_gps' and 'unknown')
472
+ cv_basins = sorted(set(basin_labels) - {'no_gps', 'unknown'})
473
+
474
+ folds = []
475
+ for basin in cv_basins:
476
+ val_idx = np.where(basin_labels == basin)[0]
477
+ train_idx = np.where((basin_labels != basin) &
478
+ (basin_labels != 'no_gps') &
479
+ (basin_labels != 'unknown'))[0]
480
+ folds.append((basin, train_idx, val_idx))
481
+
482
+ return folds, basin_labels
483
+
484
+
485
+ # ═══════════════════════════════════════════════════════════════════
486
+ # Main Training Pipeline
487
+ # ═══════════════════════════════════════════════════════════════════
488
+
489
+ def main(config=None):
490
+ """Run full training pipeline with spatial block CV.
491
+
492
+ Parameters
493
+ ----------
494
+ config : dict or None
495
+ Override default training config. Keys:
496
+ - latent_dim (int, default 16)
497
+ - dropout (float, default 0.3)
498
+ - lambda_inv, lambda_var, lambda_cov (float)
499
+ - pred_alpha (float, default 1.0)
500
+ - lr (float, default 1e-3)
501
+ - weight_decay (float, default 1e-4)
502
+ - max_epochs (int, default 300)
503
+ - patience (int, default 30)
504
+ - min_delta (float, default 1e-4)
505
+ - batch_size (int, default 128)
506
+ - grad_clip (float, default 1.0)
507
+ - seed (int, default 42)
508
+ """
509
+ if config is None:
510
+ config = {}
511
+
512
+ # Defaults
513
+ config.setdefault('latent_dim', 16)
514
+ config.setdefault('dropout', 0.3)
515
+ config.setdefault('lambda_inv', 25.0)
516
+ config.setdefault('lambda_var', 25.0)
517
+ config.setdefault('lambda_cov', 1.0)
518
+ config.setdefault('pred_alpha', 1.0)
519
+ config.setdefault('lr', 1e-3)
520
+ config.setdefault('weight_decay', 1e-4)
521
+ config.setdefault('max_epochs', 300)
522
+ config.setdefault('patience', 30)
523
+ config.setdefault('min_delta', 1e-4)
524
+ config.setdefault('batch_size', 128)
525
+ config.setdefault('grad_clip', 1.0)
526
+ config.setdefault('seed', 42)
527
+
528
+ timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
529
+
530
+ print("=" * 70)
531
+ print("T12: WorldModel Training Pipeline")
532
+ print(f"Timestamp: {timestamp}")
533
+ print("=" * 70)
534
+
535
+ # Device
536
+ device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
537
+ print(f"Device: {device}")
538
+ if torch.cuda.is_available():
539
+ print(f"GPU: {torch.cuda.get_device_name(0)}")
540
+
541
+ # Reproducibility
542
+ torch.manual_seed(config['seed'])
543
+ np.random.seed(config['seed'])
544
+ if torch.cuda.is_available():
545
+ torch.cuda.manual_seed(config['seed'])
546
+
547
+ # ── Load data ──
548
+ print("\n── Loading consolidated data ──")
549
+ env = np.load(os.path.join(DATA_DIR, 'consolidated_env.npy'))
550
+ pfam = np.load(os.path.join(DATA_DIR, 'consolidated_pfam_modules.npy'))
551
+ bio = np.load(os.path.join(DATA_DIR, 'consolidated_bio_response.npy'))
552
+ bio_valid = np.load(os.path.join(DATA_DIR, 'consolidated_bio_valid.npy'))
553
+ sample_ids = np.load(os.path.join(DATA_DIR, 'consolidated_sample_ids.npy'),
554
+ allow_pickle=True)
555
+
556
+ print(f" env: {env.shape} (no NaN: {not np.isnan(env).any()})")
557
+ print(f" pfam: {pfam.shape} (no NaN: {not np.isnan(pfam).any()})")
558
+ print(f" bio: {bio.shape} (has NaN: {np.isnan(bio).any()})")
559
+ print(f" bio_valid: {bio_valid.shape} ({bio_valid.sum()}/{len(bio_valid)} "
560
+ f"valid = {bio_valid.sum()/len(bio_valid)*100:.1f}%)")
561
+ print(f" samples: {len(sample_ids)}")
562
+
563
+ assert len(env) == 1810, f"Expected 1810 samples, got {len(env)}"
564
+ assert len(pfam) == 1810, f"Expected 1810 samples, got {len(pfam)}"
565
+ assert len(bio) == 1810, f"Expected 1810 samples, got {len(bio)}"
566
+ assert len(bio_valid) == 1810, f"Expected 1810 samples, got {len(bio_valid)}"
567
+
568
+ # Replace NaN in bio with 0 (bio_valid mask prevents them from affecting loss)
569
+ bio_clean = np.nan_to_num(bio, nan=0.0)
570
+
571
+ # ── Build CV folds ──
572
+ print("\n── Building leave-one-basin-out CV folds ──")
573
+ metadata_path = os.path.join(DATA_DIR, 'consolidated_metadata.tsv')
574
+ basin_path = os.path.join(DATA_DIR, 'ocean_basin_assignments.tsv')
575
+ folds, basin_labels = build_cv_folds(sample_ids, metadata_path, basin_path)
576
+
577
+ for fold_name, train_idx, val_idx in folds:
578
+ n_bio_train = bio_valid[train_idx].sum()
579
+ n_bio_val = bio_valid[val_idx].sum()
580
+ print(f" {fold_name:15s}: train={len(train_idx):4d} "
581
+ f"(bio_valid={n_bio_train:4d}), "
582
+ f"val={len(val_idx):4d} (bio_valid={n_bio_val:4d})")
583
+
584
+ # ── Gradient flow verification ──
585
+ print("\n── Gradient Flow Verification (100 samples, 50 steps) ──")
586
+ gf_success, gf_report = verify_gradient_flow(
587
+ env, pfam, bio_clean, bio_valid, device, n_samples=100, n_steps=50,
588
+ config=config
589
+ )
590
+ print(gf_report)
591
+ if gf_success:
592
+ print(" >>> All gradient flow checks PASSED <<<")
593
+ else:
594
+ print(" >>> WARNING: Some gradient flow checks FAILED <<<")
595
+ print(" Proceeding with training anyway...")
596
+
597
+ # ── Train all folds ──
598
+ print("\n── Training All CV Folds ──")
599
+ print(f"Config: latent_dim={config['latent_dim']}, "
600
+ f"dropout={config['dropout']}, "
601
+ f"VICReg=({config['lambda_inv']}/{config['lambda_var']}/{config['lambda_cov']}), "
602
+ f"pred_alpha={config['pred_alpha']}, "
603
+ f"lr={config['lr']}, batch_size={config['batch_size']}, "
604
+ f"patience={config['patience']}, seed={config['seed']}")
605
+
606
+ all_histories = {}
607
+ all_summaries = []
608
+ fold_checkpoints = {}
609
+
610
+ total_t0 = time.time()
611
+
612
+ for fold_name, train_idx, val_idx in folds:
613
+ print(f"\n ── Fold: {fold_name} ──")
614
+
615
+ # Per-fold standardization (no data leakage)
616
+ env_scaler = StandardScaler()
617
+ pfam_scaler = StandardScaler()
618
+ bio_scaler = StandardScaler()
619
+
620
+ env_train_scaled = env_scaler.fit_transform(env[train_idx])
621
+ env_val_scaled = env_scaler.transform(env[val_idx])
622
+
623
+ pfam_train_scaled = pfam_scaler.fit_transform(pfam[train_idx])
624
+ pfam_val_scaled = pfam_scaler.transform(pfam[val_idx])
625
+
626
+ # Scale bio using only bio_valid training samples
627
+ bio_train_valid_idx = train_idx[bio_valid[train_idx]]
628
+ if len(bio_train_valid_idx) > 0:
629
+ bio_scaler.fit(bio_clean[bio_train_valid_idx])
630
+ else:
631
+ bio_scaler.fit(bio_clean[train_idx]) # fallback
632
+ bio_train_scaled = bio_scaler.transform(bio_clean[train_idx])
633
+ bio_val_scaled = bio_scaler.transform(bio_clean[val_idx])
634
+
635
+ # Create datasets
636
+ train_dataset = WorldModelDataset(
637
+ env_train_scaled, pfam_train_scaled,
638
+ bio_train_scaled, bio_valid[train_idx]
639
+ )
640
+ val_dataset = WorldModelDataset(
641
+ env_val_scaled, pfam_val_scaled,
642
+ bio_val_scaled, bio_valid[val_idx]
643
+ )
644
+
645
+ # Fresh model per fold
646
+ torch.manual_seed(config['seed'])
647
+ if torch.cuda.is_available():
648
+ torch.cuda.manual_seed(config['seed'])
649
+
650
+ model = WorldModel(
651
+ env_dim=env.shape[1],
652
+ pfam_dim=pfam.shape[1],
653
+ bio_dim=bio.shape[1] if bio.ndim > 1 else 1,
654
+ latent_dim=config['latent_dim'],
655
+ dropout=config['dropout'],
656
+ lambda_inv=config['lambda_inv'],
657
+ lambda_var=config['lambda_var'],
658
+ lambda_cov=config['lambda_cov'],
659
+ pred_alpha=config['pred_alpha'],
660
+ ).to(device)
661
+
662
+ # Train
663
+ best_state, history, summary = train_fold(
664
+ model, train_dataset, val_dataset, config, device, fold_name
665
+ )
666
+
667
+ print(f" Epochs: {summary['total_epochs']} "
668
+ f"(best={summary['best_epoch']})")
669
+ print(f" Best val: total={summary['best_val_total']:.2f}, "
670
+ f"vicreg={summary['best_val_vicreg']:.2f}, "
671
+ f"pred={summary['best_val_pred']:.2f}")
672
+ print(f" Time: {summary['elapsed_s']:.1f}s")
673
+
674
+ all_histories[fold_name] = history
675
+ all_summaries.append(summary)
676
+
677
+ # Save checkpoint
678
+ checkpoint = {
679
+ 'model_state_dict': best_state,
680
+ 'config': model.config,
681
+ 'fold': fold_name,
682
+ 'train_idx': train_idx.tolist(),
683
+ 'val_idx': val_idx.tolist(),
684
+ 'env_scaler_mean': env_scaler.mean_.tolist(),
685
+ 'env_scaler_scale': env_scaler.scale_.tolist(),
686
+ 'pfam_scaler_mean': pfam_scaler.mean_.tolist(),
687
+ 'pfam_scaler_scale': pfam_scaler.scale_.tolist(),
688
+ 'bio_scaler_mean': bio_scaler.mean_.tolist(),
689
+ 'bio_scaler_scale': bio_scaler.scale_.tolist(),
690
+ 'summary': summary,
691
+ 'timestamp': timestamp,
692
+ }
693
+ ckpt_path = os.path.join(
694
+ MODELS_DIR, f'world_model_fold_{fold_name}_{timestamp}.pt'
695
+ )
696
+ torch.save(checkpoint, ckpt_path)
697
+ fold_checkpoints[fold_name] = ckpt_path
698
+ print(f" Saved: {os.path.basename(ckpt_path)}")
699
+
700
+ total_elapsed = time.time() - total_t0
701
+ print(f"\n── Total training time: {total_elapsed:.1f}s ──")
702
+
703
+ # ── Save training curves ──
704
+ curves_path = os.path.join(RESULTS_DIR,
705
+ f't12_training_curves_{timestamp}.json')
706
+ curves_data = {
707
+ 'provenance': {
708
+ 'script': os.path.abspath(__file__),
709
+ 'inputs': {
710
+ 'env': os.path.join(DATA_DIR, 'consolidated_env.npy'),
711
+ 'pfam': os.path.join(DATA_DIR, 'consolidated_pfam_modules.npy'),
712
+ 'bio': os.path.join(DATA_DIR, 'consolidated_bio_response.npy'),
713
+ 'bio_valid': os.path.join(DATA_DIR, 'consolidated_bio_valid.npy'),
714
+ },
715
+ 'timestamp': timestamp,
716
+ 'config': config,
717
+ },
718
+ 'histories': all_histories,
719
+ }
720
+ with open(curves_path, 'w') as f:
721
+ json.dump(curves_data, f, indent=2)
722
+ print(f"\nTraining curves: {os.path.basename(curves_path)}")
723
+
724
+ # ── Save training summary ──
725
+ summary_path = os.path.join(RESULTS_DIR,
726
+ f't12_training_summary_{timestamp}.tsv')
727
+ with open(summary_path, 'w') as f:
728
+ # Provenance header
729
+ f.write(f"# Provenance:\n")
730
+ f.write(f"# Script: {os.path.abspath(__file__)}\n")
731
+ f.write(f"# Inputs: data/consolidated_*.npy\n")
732
+ f.write(f"# Date: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
733
+ f.write(f"# Integrity Check: PASSED\n")
734
+ f.write(f"# Config: latent_dim={config['latent_dim']}, "
735
+ f"dropout={config['dropout']}, "
736
+ f"VICReg=({config['lambda_inv']}/{config['lambda_var']}/{config['lambda_cov']})\n")
737
+
738
+ cols = ['fold', 'n_train', 'n_val', 'total_epochs', 'best_epoch',
739
+ 'best_val_total', 'best_val_vicreg', 'best_val_pred',
740
+ 'final_train_total', 'elapsed_s']
741
+ f.write('\t'.join(cols) + '\n')
742
+ for s in all_summaries:
743
+ f.write('\t'.join(str(s[c]) for c in cols) + '\n')
744
+
745
+ print(f"Training summary: {os.path.basename(summary_path)}")
746
+
747
+ # ── Print summary table ──
748
+ print("\n── Per-Fold Training Summary ──")
749
+ print(f"{'Fold':15s} {'Train':>5s} {'Val':>5s} {'Epochs':>6s} "
750
+ f"{'Best':>5s} {'Val Total':>10s} {'Val VICReg':>10s} "
751
+ f"{'Val Pred':>9s} {'Time':>6s}")
752
+ print("-" * 80)
753
+ for s in all_summaries:
754
+ print(f"{s['fold']:15s} {s['n_train']:5d} {s['n_val']:5d} "
755
+ f"{s['total_epochs']:6d} {s['best_epoch']:5d} "
756
+ f"{s['best_val_total']:10.2f} {s['best_val_vicreg']:10.2f} "
757
+ f"{s['best_val_pred']:9.2f} {s['elapsed_s']:6.1f}s")
758
+
759
+ mean_val = np.mean([s['best_val_total'] for s in all_summaries])
760
+ std_val = np.std([s['best_val_total'] for s in all_summaries])
761
+ print(f"\nMean best val total: {mean_val:.2f} +/- {std_val:.2f}")
762
+ print(f"Total time: {total_elapsed:.1f}s")
763
+
764
+ # Print model parameter count
765
+ params = WorldModel(
766
+ env_dim=env.shape[1], pfam_dim=pfam.shape[1],
767
+ latent_dim=config['latent_dim'], dropout=config['dropout']
768
+ ).count_parameters()
769
+ print(f"Parameters per model: {params:,}")
770
+
771
+ print("\n── Output Files ──")
772
+ for fold_name, path in fold_checkpoints.items():
773
+ print(f" {os.path.basename(path)}")
774
+ print(f" {os.path.basename(curves_path)}")
775
+ print(f" {os.path.basename(summary_path)}")
776
+
777
+ return {
778
+ 'summaries': all_summaries,
779
+ 'histories': all_histories,
780
+ 'checkpoints': fold_checkpoints,
781
+ 'timestamp': timestamp,
782
+ 'config': config,
783
+ }
784
+
785
+
786
+ if __name__ == '__main__':
787
+ # Parse optional config from command line
788
+ # Usage: python train_world_model.py [key=value ...]
789
+ # Example: python train_world_model.py latent_dim=32 dropout=0.5
790
+ config = {}
791
+ for arg in sys.argv[1:]:
792
+ if '=' in arg:
793
+ key, val = arg.split('=', 1)
794
+ # Try to convert to appropriate type
795
+ try:
796
+ val = int(val)
797
+ except ValueError:
798
+ try:
799
+ val = float(val)
800
+ except ValueError:
801
+ pass # keep as string
802
+ config[key] = val
803
+
804
+ result = main(config)