File size: 5,006 Bytes
cd9b2d8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | """Identical score MLPs; nested actor-held-out CREMA-D/RAVDESS evaluation."""
import json
import os
import sys
from pathlib import Path
import numpy as np
import torch
from torch import nn
from study_paths import CODE, ROOT, SEED, read, write
sys.path.insert(0, str(CODE))
from calibrate_acted import actor_splits, metrics
from score_public_benchmarks import ACTED_CLASSES
def fit(x, y, train, test, n_classes, config, seed):
# Scale only within this training fold, including every inner fit.
mean = x[train].mean(0)
scale = np.maximum(x[train].std(0), 1e-5)
device = torch.device('cuda', int(os.environ.get('LOCAL_RANK', 0)))
torch.manual_seed(seed)
model = nn.Sequential(nn.Linear(192, 64), nn.GELU(), nn.Linear(64, n_classes)).to(device)
features = torch.as_tensor((x - mean) / scale, dtype=torch.float32, device=device)
labels = torch.as_tensor(y, dtype=torch.long, device=device)
tr = torch.as_tensor(train, device=device)
te = torch.as_tensor(test, device=device)
optimizer = torch.optim.AdamW(model.parameters(), lr=config['lr'], weight_decay=.01)
for _ in range(config['epochs']):
optimizer.zero_grad(set_to_none=True)
loss = nn.functional.cross_entropy(model(features[tr]), labels[tr])
loss.backward()
optimizer.step()
with torch.no_grad():
logits = model(features[te]).cpu().numpy()
artifact = {'model': {k: v.cpu() for k, v in model.state_dict().items()},
'mean': mean, 'scale': scale, 'config': config, 'seed': seed}
return logits, artifact
def run(kind, scores, rows, output):
path = output / (kind + '_matched_adapter.json')
if path.exists():
return read(path)
cfg = read(ROOT / 'study.json')['acted_adapter']
classes = list(ACTED_CLASSES[kind])
x = np.asarray(scores, np.float32)
if x.shape != (len(rows), 192) or not np.isfinite(x).all():
raise ValueError('Adapter requires finite aligned 192-score predictions')
groups = np.asarray([Path(r['audio_name']).stem.split('-')[-1] if kind == 'ravdess'
else Path(r['audio_name']).name.split('_')[0] for r in rows])
if len(set(r['audio_name'] for r in rows)) != len(rows):
raise ValueError('Repeated benchmark clip')
y = np.asarray([classes.index(r['emotion'].lower()) for r in rows])
parameters = (192 + 1) * 64 + (64 + 1) * len(classes)
assert parameters == cfg['parameters'][kind]
candidates = [{'lr': lr, 'epochs': epochs} for lr in cfg['lr'] for epochs in cfg['epochs']]
prediction = np.full(len(y), -1, np.int64)
fold_ids = np.full(len(y), -1, np.int64)
folds = []
for fold, (train, test) in enumerate(actor_splits(y, groups, 5, SEED)):
inner = actor_splits(y[train], groups[train], 3, SEED + fold + 1)
trials = []
for candidate in candidates:
correct = count = 0
for j, (inside, held) in enumerate(inner):
logits, _ = fit(x, y, train[inside], train[held], len(classes), candidate, SEED + fold * 10 + j)
correct += int((logits.argmax(1) == y[train[held]]).sum())
count += len(held)
trials.append({**candidate, 'inner_accuracy': correct / count})
chosen = max(trials, key=lambda v: (v['inner_accuracy'], -v['epochs'], -v['lr']))
logits, artifact = fit(x, y, train, test, len(classes), chosen, SEED + 100 + fold)
prediction[test], fold_ids[test] = logits.argmax(1), fold
torch.save({**artifact, 'classes': classes, 'train_actors': sorted(set(groups[train])),
'held_actors': sorted(set(groups[test]))}, output / (kind + '_adapter_fold' + str(fold) + '.pt'))
folds.append({'fold': fold, 'train_clips': len(train), 'held_clips': len(test),
'train_actors': sorted(set(groups[train])), 'held_actors': sorted(set(groups[test])),
'trials': trials, 'selected': chosen,
'outer_accuracy': float((prediction[test] == y[test]).mean())})
assert np.all(prediction >= 0)
result = {'dataset': kind, 'parameters': parameters, 'classes': classes, 'seed': SEED,
'protocol': '192 raw predictions -> training-fold StandardScaler -> 64 GELU -> class logits; '
'five actor-disjoint outer folds; three inner folds select LR/epochs; full-batch AdamW',
'comparison': 'Supervised benchmark adaptation; distinct from paper zero-shot',
'metrics': metrics(y, prediction, groups, classes), 'folds': folds}
with (output / (kind + '_matched_oof.jsonl')).open('w') as stream:
for i, row in enumerate(rows):
stream.write(json.dumps({'audio_name': row['audio_name'], 'actor': str(groups[i]),
'outer_fold': int(fold_ids[i]), 'truth': classes[y[i]],
'prediction': classes[prediction[i]]}) + '\n')
write(path, result)
return result
|