Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
File size: 3,437 Bytes
cd24b4a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | """Evaluate the DEPLOYED ReStraV (restrav.pth, the serving path) on restrav_data val vs test,
to check whether the 'awful test' performance matches the AUROC numbers we've quoted.
val = in-distribution (VidProM: VSC reals + VidProM generators) -> expect high.
test = cross-dataset (GenVideo: msrvtt reals + GenVideo generators) -> expect ~chance.
"""
import glob, random, sys
from pathlib import Path
import numpy as np, torch
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from backend.engine import get_engine # noqa: E402
from backend.registry import get_spec # noqa: E402
from backend import media # noqa: E402
ROOT = "/hddstore/ssalehi/SynthScope/restrav_data"
N = 200 # per class per split
def auroc(y, p):
y = np.asarray(y); p = np.asarray(p)
order = np.argsort(-p, kind="mergesort"); y = y[order]
tp = np.cumsum(y == 1); fp = np.cumsum(y == 0)
npos, nneg = (y == 1).sum(), (y == 0).sum()
if npos == 0 or nneg == 0:
return float("nan")
tpr = np.concatenate([[0.0], tp / npos]); fpr = np.concatenate([[0.0], fp / nneg])
return float(np.sum((fpr[1:] - fpr[:-1]) * (tpr[1:] + tpr[:-1]) / 2.0))
@torch.no_grad()
def main():
eng = get_engine(); spec = get_spec("restrav"); model = eng._get_model(spec)
mean = torch.tensor(spec.video_norm[0]).view(1, 3, 1, 1)
std = torch.tensor(spec.video_norm[1]).view(1, 3, 1, 1)
for split in ["val", "test"]:
reals = sorted(glob.glob(f"{ROOT}/{split}/genvideo/0_real/*.mp4"))
fakes = sorted(glob.glob(f"{ROOT}/{split}/genvideo/1_fake/*.mp4"))
random.Random(0).shuffle(reals); random.Random(1).shuffle(fakes)
items = [(p, 0) for p in reals[:N]] + [(p, 1) for p in fakes[:N]] # SynthScope: real=0, fake=1
probs, labels = [], []
for p, y in items:
try:
clip = media.video_clip_tensor(p, spec.num_frames, spec.fps, spec.frame_size,
window_center=spec.window_center, interp=spec.resize_interp)
clip = (clip - mean) / std
probs.append(float(eng._video_fake_prob(model, spec, clip).item())); labels.append(y)
except Exception:
pass
probs = np.array(probs); labels = np.array(labels)
pred = (probs > 0.5).astype(int)
racc = (pred[labels == 0] == 0).mean() if (labels == 0).any() else float("nan")
facc = (pred[labels == 1] == 1).mean() if (labels == 1).any() else float("nan")
# AUROC with fake as positive (label 1 = fake, score = P(fake))
a = auroc((labels == 1).astype(int), probs)
acc = (pred == labels).mean() * 100
# best-threshold balanced accuracy (what "Auto-calibrate" finds, fit IN-SAMPLE)
best_bacc, best_t = 0.0, 0.5
for t in np.linspace(0.01, 0.99, 99):
p2 = (probs > t).astype(int)
r = (p2[labels == 0] == 0).mean() if (labels == 0).any() else 0
f = (p2[labels == 1] == 1).mean() if (labels == 1).any() else 0
if (r + f) / 2 > best_bacc:
best_bacc, best_t = (r + f) / 2, t
print(f"{split:5s} (n={len(labels)}): acc@0.5={acc:.1f} real_acc={racc*100:.1f} fake_acc={facc*100:.1f} "
f"AUROC={a*100:.1f} | best-threshold(t={best_t:.2f}) balanced_acc={best_bacc*100:.1f}", flush=True)
if __name__ == "__main__":
main()
|