training/embedding_probe_study/matched_acted.py

"""Identical score MLPs; nested actor-held-out CREMA-D/RAVDESS evaluation."""
import json
import os
import sys
from pathlib import Path
import numpy as np
import torch
from torch import nn
from study_paths import CODE, ROOT, SEED, read, write

sys.path.insert(0, str(CODE))
from calibrate_acted import actor_splits, metrics
from score_public_benchmarks import ACTED_CLASSES


def fit(x, y, train, test, n_classes, config, seed):
    # Scale only within this training fold, including every inner fit.
    mean = x[train].mean(0)
    scale = np.maximum(x[train].std(0), 1e-5)
    device = torch.device('cuda', int(os.environ.get('LOCAL_RANK', 0)))
    torch.manual_seed(seed)
    model = nn.Sequential(nn.Linear(192, 64), nn.GELU(), nn.Linear(64, n_classes)).to(device)
    features = torch.as_tensor((x - mean) / scale, dtype=torch.float32, device=device)
    labels = torch.as_tensor(y, dtype=torch.long, device=device)
    tr = torch.as_tensor(train, device=device)
    te = torch.as_tensor(test, device=device)
    optimizer = torch.optim.AdamW(model.parameters(), lr=config['lr'], weight_decay=.01)
    for _ in range(config['epochs']):
        optimizer.zero_grad(set_to_none=True)
        loss = nn.functional.cross_entropy(model(features[tr]), labels[tr])
        loss.backward()
        optimizer.step()
    with torch.no_grad():
        logits = model(features[te]).cpu().numpy()
    artifact = {'model': {k: v.cpu() for k, v in model.state_dict().items()},
                'mean': mean, 'scale': scale, 'config': config, 'seed': seed}
    return logits, artifact


def run(kind, scores, rows, output):
    path = output / (kind + '_matched_adapter.json')
    if path.exists():
        return read(path)
    cfg = read(ROOT / 'study.json')['acted_adapter']
    classes = list(ACTED_CLASSES[kind])
    x = np.asarray(scores, np.float32)
    if x.shape != (len(rows), 192) or not np.isfinite(x).all():
        raise ValueError('Adapter requires finite aligned 192-score predictions')
    groups = np.asarray([Path(r['audio_name']).stem.split('-')[-1] if kind == 'ravdess'
                         else Path(r['audio_name']).name.split('_')[0] for r in rows])
    if len(set(r['audio_name'] for r in rows)) != len(rows):
        raise ValueError('Repeated benchmark clip')
    y = np.asarray([classes.index(r['emotion'].lower()) for r in rows])
    parameters = (192 + 1) * 64 + (64 + 1) * len(classes)
    assert parameters == cfg['parameters'][kind]
    candidates = [{'lr': lr, 'epochs': epochs} for lr in cfg['lr'] for epochs in cfg['epochs']]
    prediction = np.full(len(y), -1, np.int64)
    fold_ids = np.full(len(y), -1, np.int64)
    folds = []
    for fold, (train, test) in enumerate(actor_splits(y, groups, 5, SEED)):
        inner = actor_splits(y[train], groups[train], 3, SEED + fold + 1)
        trials = []
        for candidate in candidates:
            correct = count = 0
            for j, (inside, held) in enumerate(inner):
                logits, _ = fit(x, y, train[inside], train[held], len(classes), candidate, SEED + fold * 10 + j)
                correct += int((logits.argmax(1) == y[train[held]]).sum())
                count += len(held)
            trials.append({**candidate, 'inner_accuracy': correct / count})
        chosen = max(trials, key=lambda v: (v['inner_accuracy'], -v['epochs'], -v['lr']))
        logits, artifact = fit(x, y, train, test, len(classes), chosen, SEED + 100 + fold)
        prediction[test], fold_ids[test] = logits.argmax(1), fold
        torch.save({**artifact, 'classes': classes, 'train_actors': sorted(set(groups[train])),
                    'held_actors': sorted(set(groups[test]))}, output / (kind + '_adapter_fold' + str(fold) + '.pt'))
        folds.append({'fold': fold, 'train_clips': len(train), 'held_clips': len(test),
                      'train_actors': sorted(set(groups[train])), 'held_actors': sorted(set(groups[test])),
                      'trials': trials, 'selected': chosen,
                      'outer_accuracy': float((prediction[test] == y[test]).mean())})
    assert np.all(prediction >= 0)
    result = {'dataset': kind, 'parameters': parameters, 'classes': classes, 'seed': SEED,
              'protocol': '192 raw predictions -> training-fold StandardScaler -> 64 GELU -> class logits; '
                          'five actor-disjoint outer folds; three inner folds select LR/epochs; full-batch AdamW',
              'comparison': 'Supervised benchmark adaptation; distinct from paper zero-shot',
              'metrics': metrics(y, prediction, groups, classes), 'folds': folds}
    with (output / (kind + '_matched_oof.jsonl')).open('w') as stream:
        for i, row in enumerate(rows):
            stream.write(json.dumps({'audio_name': row['audio_name'], 'actor': str(groups[i]),
                                     'outer_fold': int(fold_ids[i]), 'truth': classes[y[i]],
                                     'prediction': classes[prediction[i]]}) + '\n')
    write(path, result)
    return result