Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
| """Evaluate the DEPLOYED ReStraV (restrav.pth, the serving path) on restrav_data val vs test, | |
| to check whether the 'awful test' performance matches the AUROC numbers we've quoted. | |
| val = in-distribution (VidProM: VSC reals + VidProM generators) -> expect high. | |
| test = cross-dataset (GenVideo: msrvtt reals + GenVideo generators) -> expect ~chance. | |
| """ | |
| import glob, random, sys | |
| from pathlib import Path | |
| import numpy as np, torch | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from backend.engine import get_engine # noqa: E402 | |
| from backend.registry import get_spec # noqa: E402 | |
| from backend import media # noqa: E402 | |
| ROOT = "/hddstore/ssalehi/SynthScope/restrav_data" | |
| N = 200 # per class per split | |
| def auroc(y, p): | |
| y = np.asarray(y); p = np.asarray(p) | |
| order = np.argsort(-p, kind="mergesort"); y = y[order] | |
| tp = np.cumsum(y == 1); fp = np.cumsum(y == 0) | |
| npos, nneg = (y == 1).sum(), (y == 0).sum() | |
| if npos == 0 or nneg == 0: | |
| return float("nan") | |
| tpr = np.concatenate([[0.0], tp / npos]); fpr = np.concatenate([[0.0], fp / nneg]) | |
| return float(np.sum((fpr[1:] - fpr[:-1]) * (tpr[1:] + tpr[:-1]) / 2.0)) | |
| def main(): | |
| eng = get_engine(); spec = get_spec("restrav"); model = eng._get_model(spec) | |
| mean = torch.tensor(spec.video_norm[0]).view(1, 3, 1, 1) | |
| std = torch.tensor(spec.video_norm[1]).view(1, 3, 1, 1) | |
| for split in ["val", "test"]: | |
| reals = sorted(glob.glob(f"{ROOT}/{split}/genvideo/0_real/*.mp4")) | |
| fakes = sorted(glob.glob(f"{ROOT}/{split}/genvideo/1_fake/*.mp4")) | |
| random.Random(0).shuffle(reals); random.Random(1).shuffle(fakes) | |
| items = [(p, 0) for p in reals[:N]] + [(p, 1) for p in fakes[:N]] # SynthScope: real=0, fake=1 | |
| probs, labels = [], [] | |
| for p, y in items: | |
| try: | |
| clip = media.video_clip_tensor(p, spec.num_frames, spec.fps, spec.frame_size, | |
| window_center=spec.window_center, interp=spec.resize_interp) | |
| clip = (clip - mean) / std | |
| probs.append(float(eng._video_fake_prob(model, spec, clip).item())); labels.append(y) | |
| except Exception: | |
| pass | |
| probs = np.array(probs); labels = np.array(labels) | |
| pred = (probs > 0.5).astype(int) | |
| racc = (pred[labels == 0] == 0).mean() if (labels == 0).any() else float("nan") | |
| facc = (pred[labels == 1] == 1).mean() if (labels == 1).any() else float("nan") | |
| # AUROC with fake as positive (label 1 = fake, score = P(fake)) | |
| a = auroc((labels == 1).astype(int), probs) | |
| acc = (pred == labels).mean() * 100 | |
| # best-threshold balanced accuracy (what "Auto-calibrate" finds, fit IN-SAMPLE) | |
| best_bacc, best_t = 0.0, 0.5 | |
| for t in np.linspace(0.01, 0.99, 99): | |
| p2 = (probs > t).astype(int) | |
| r = (p2[labels == 0] == 0).mean() if (labels == 0).any() else 0 | |
| f = (p2[labels == 1] == 1).mean() if (labels == 1).any() else 0 | |
| if (r + f) / 2 > best_bacc: | |
| best_bacc, best_t = (r + f) / 2, t | |
| print(f"{split:5s} (n={len(labels)}): acc@0.5={acc:.1f} real_acc={racc*100:.1f} fake_acc={facc*100:.1f} " | |
| f"AUROC={a*100:.1f} | best-threshold(t={best_t:.2f}) balanced_acc={best_bacc*100:.1f}", flush=True) | |
| if __name__ == "__main__": | |
| main() | |