Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
| """Decisive ReStraV cross-generator reproduction. | |
| Question: our README reports a weak/unstable ReStraV "cross-generator" number, and the deployed | |
| restrav.pth collapses to AUROC ~0.50. Does the OFFICIAL ReStraV recipe (github.com/ChristianInterno/ | |
| ReStraV) reproduce the paper's cross-generator generalization (Table 3: unseen-generator mAP 97.3, | |
| future/Sora mAP 92.9), or does it also collapse? | |
| This script extracts the official 21-d trajectory-geometry features (verbatim official recipe: | |
| center 2s / 24 frames / BICUBIC-224 / DINOv2 ViT-S/14 / [CLS;patches]) and runs the official MLP | |
| training recipe (21->64->32->1, BCE, Adam 1e-3, 10 epochs, standardized by train mean/std) under | |
| THREE evaluation protocols, to separate *method/code* faithfulness from *protocol* mismatch: | |
| (A) IN-DISTRIBUTION official train.py protocol: GenVideo-Val pool, balanced, stratified 50/50. | |
| (B) UNSEEN-GENERATOR paper Table-3 protocol: SAME real source, leave-one-fake-generator-out | |
| (train on the other generators, test on the held-out one). | |
| (C) CROSS-DATASET our README "cross-generator": train on `extracted` (VidProM/DeMamba pool, | |
| VSC reals), test on GenVideo-Val (DIFFERENT real source + different fakes). | |
| To settle the normalization question, every video's features are computed TWICE in one decode pass: | |
| * raw = official preprocessing (frames/255, NO ImageNet normalization) <- official recipe | |
| * norm = ImageNet-normalized (normalize_dinov2) <- what restrav.sh trains with | |
| Label convention (official): REAL=1, FAKE=0. AUROC is positive-class-invariant; AP is fake-positive. | |
| """ | |
| import os, sys, glob, re, random, time | |
| os.environ.setdefault("HF_HOME", "/hddstore/ssalehi/SynthScope/.cache/huggingface") | |
| os.environ.setdefault("TORCH_HOME", "/hddstore/ssalehi/SynthScope/.cache/torch") | |
| import numpy as np, torch, torch.nn as nn, torch.nn.functional as F | |
| import torchvision.transforms.functional as TF | |
| from torchvision.transforms import InterpolationMode | |
| import decord | |
| from sklearn.model_selection import train_test_split | |
| from sklearn.metrics import roc_auc_score, average_precision_score | |
| GV = "/hddstore/ssalehi/SynthScope/d3_repro/genvideo_val/GenVideo-Val" | |
| EXTRACTED = "/hddstore/ssalehi/SynthScope/d3_repro/extracted" | |
| CACHE = "/hddstore/ssalehi/SynthScope/restrav_xgen_cache" | |
| os.makedirs(CACHE, exist_ok=True) | |
| DEV = "cuda" | |
| T, WIN = 24, 2.0 | |
| K_FAKE_PER_GEN = int(os.environ.get("K_FAKE_PER_GEN", "200")) | |
| N_REAL_GV = int(os.environ.get("N_REAL_GV", "2000")) | |
| N_EX = int(os.environ.get("N_EX", "1500")) # per class for the extracted (cross-dataset) train pool | |
| IMNET_MEAN = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1).to(DEV) | |
| IMNET_STD = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1).to(DEV) | |
| print("loading DINOv2 ViT-S/14...", flush=True) | |
| dinov2 = torch.hub.load("facebookresearch/dinov2", "dinov2_vits14").eval().to(DEV) | |
| def decode_clip(path): | |
| vr = decord.VideoReader(path, ctx=decord.cpu(0)) | |
| n = len(vr); fps = float(vr.get_avg_fps()) or 30.0; dur = n / fps | |
| center = dur / 2.0; start = max(0.0, center - WIN / 2) | |
| delta = WIN / max(T - 1, 1) | |
| idxs = [min(int(round((start + i * delta) * fps)), n - 1) for i in range(T)] | |
| frames = vr.get_batch(idxs).asnumpy() | |
| return torch.from_numpy(frames).permute(0, 3, 1, 2) | |
| def preprocess(frames): | |
| frames = frames.float() / 255.0 | |
| return torch.stack([TF.resize(x, [224, 224], InterpolationMode.BICUBIC, antialias=True) for x in frames]) | |
| def geom21(Z): | |
| dz = Z[:, 1:] - Z[:, :-1]; d = dz.norm(dim=-1) | |
| cos = F.cosine_similarity(dz[:, :-1], dz[:, 1:], dim=-1) | |
| th = torch.rad2deg(torch.acos(cos.clamp(-1, 1))) | |
| def m4(x): return [x.mean(-1), x.amin(-1), x.amax(-1), x.var(-1, unbiased=False)] | |
| stats = torch.stack(m4(d) + m4(th), dim=1) | |
| return torch.cat([d[:, :7], th[:, :6], stats], dim=1)[0].cpu().numpy() | |
| def feats_both(path): | |
| clip = preprocess(decode_clip(path)).to(DEV) | |
| def run(x): | |
| f = dinov2.forward_features(x) | |
| tokens = torch.cat([f["x_norm_clstoken"].unsqueeze(1), f["x_norm_patchtokens"]], dim=1) | |
| return geom21(tokens.flatten(1).unsqueeze(0)) | |
| raw = run(clip) | |
| norm = run((clip - IMNET_MEAN) / IMNET_STD) | |
| return raw, norm | |
| def build(items, tag): | |
| """items: list of (path, label, gen). Returns Xraw, Xnorm, y, gens (np arrays), caching to npz.""" | |
| cache = os.path.join(CACHE, f"feat_{tag}.npz") | |
| if os.path.exists(cache): | |
| z = np.load(cache, allow_pickle=True) | |
| print(f"[{tag}] loaded {len(z['y'])} cached features", flush=True) | |
| return z["Xraw"], z["Xnorm"], z["y"], z["gens"] | |
| Xr, Xn, y, gens = [], [], [], [] | |
| t0 = time.time() | |
| for i, (p, lab, g) in enumerate(items): | |
| try: | |
| r, n = feats_both(p) | |
| if np.isfinite(r).all() and np.isfinite(n).all(): | |
| Xr.append(r); Xn.append(n); y.append(lab); gens.append(g) | |
| except Exception: | |
| pass | |
| if (i + 1) % 200 == 0: | |
| print(f" [{tag}] {i+1}/{len(items)} ({(time.time()-t0)/(i+1):.2f}s/vid)", flush=True) | |
| Xr = np.array(Xr, np.float32); Xn = np.array(Xn, np.float32) | |
| y = np.array(y, np.int64); gens = np.array(gens) | |
| np.savez(cache, Xraw=Xr, Xnorm=Xn, y=y, gens=gens) | |
| print(f"[{tag}] extracted+cached {len(y)} features", flush=True) | |
| return Xr, Xn, y, gens | |
| def train_eval(Xtr, ytr, Xte, yte, seed=0): | |
| """Official recipe. Label real=1. Returns (AUROC, AP_fake). Averaged over a few seeds outside.""" | |
| torch.manual_seed(seed); np.random.seed(seed) | |
| mu = Xtr.mean(0, keepdims=True); sd = Xtr.std(0, keepdims=True) + 1e-8 | |
| Xtr = (Xtr - mu) / sd; Xte = (Xte - mu) / sd | |
| net = nn.Sequential(nn.Linear(21, 64), nn.ReLU(), nn.Linear(64, 32), nn.ReLU(), nn.Linear(32, 1)).to(DEV) | |
| opt = torch.optim.Adam(net.parameters(), lr=1e-3); crit = nn.BCEWithLogitsLoss() | |
| Xt = torch.tensor(Xtr, device=DEV); yt = torch.tensor(ytr, dtype=torch.float32, device=DEV).unsqueeze(1) | |
| for ep in range(10): | |
| perm = torch.randperm(len(Xt), device=DEV) | |
| for i in range(0, len(Xt), 32): | |
| b = perm[i:i+32]; opt.zero_grad() | |
| loss = crit(net(Xt[b]), yt[b]); loss.backward(); opt.step() | |
| net.eval() | |
| with torch.no_grad(): | |
| p_real = torch.sigmoid(net(torch.tensor(Xte, device=DEV))).cpu().numpy().ravel() | |
| auc = roc_auc_score(yte, p_real) | |
| ap_fake = average_precision_score(1 - yte, 1 - p_real) | |
| return auc, ap_fake | |
| def mean_eval(Xtr, ytr, Xte, yte, seeds=(0, 1, 2)): | |
| rs = [train_eval(Xtr, ytr, Xte, yte, s) for s in seeds] | |
| auc = np.mean([r[0] for r in rs]); ap = np.mean([r[1] for r in rs]) | |
| return auc, ap | |
| def balance(X, y, rng): | |
| """Subsample to equal priors (official train.py).""" | |
| ir = np.where(y == 1)[0]; ifk = np.where(y == 0)[0] | |
| k = min(len(ir), len(ifk)) | |
| sel = np.concatenate([rng.choice(ir, k, replace=False), rng.choice(ifk, k, replace=False)]) | |
| rng.shuffle(sel) | |
| return X[sel], y[sel] | |
| def rperm_split(perm): | |
| h = len(perm) // 2 | |
| return perm[:h], perm[h:] | |
| def main(): | |
| random.seed(0) | |
| # ---- assemble GenVideo-Val item list (real=1, fake=0), tagged by generator ---- | |
| gvr = sorted(glob.glob(GV + "/Real/*.mp4")); random.Random(0).shuffle(gvr) | |
| gv_items = [(p, 1, "real") for p in gvr[:N_REAL_GV]] | |
| for gdir in sorted(glob.glob(GV + "/Fake/*/")): | |
| g = os.path.basename(gdir.rstrip("/")) | |
| fk = sorted(glob.glob(gdir + "*.mp4")); random.Random(hash(g) % 100).shuffle(fk) | |
| gv_items += [(p, 0, g) for p in fk[:K_FAKE_PER_GEN]] | |
| print(f"GenVideo-Val items: {len(gv_items)}", flush=True) | |
| Xr_gv, Xn_gv, y_gv, g_gv = build(gv_items, "genvideo") | |
| # ---- extracted (VidProM/DeMamba) pool for cross-dataset ---- | |
| ex = sorted(glob.glob(EXTRACTED + "/*.mp4")) | |
| exr = [p for p in ex if re.search(r"_real_", os.path.basename(p))] | |
| exf = [p for p in ex if not re.search(r"_real_", os.path.basename(p))] | |
| random.Random(2).shuffle(exr); random.Random(3).shuffle(exf) | |
| ex_items = [(p, 1, "real") for p in exr[:N_EX]] + [(p, 0, "vidprom") for p in exf[:N_EX]] | |
| print(f"extracted items: {len(ex_items)}", flush=True) | |
| Xr_ex, Xn_ex, y_ex, g_ex = build(ex_items, "extracted") | |
| for variant, Xgv, Xex in [("raw[0,1] (OFFICIAL)", Xr_gv, Xr_ex), ("imagenet-norm (restrav.sh)", Xn_gv, Xn_ex)]: | |
| print(f"\n{'='*78}\n### FEATURE VARIANT: {variant}\n{'='*78}", flush=True) | |
| # (A) IN-DISTRIBUTION | |
| rng = np.random.default_rng(42) | |
| Xb, yb = balance(Xgv, y_gv, rng) | |
| Xtr, Xte, ytr, yte = train_test_split(Xb, yb, test_size=0.5, stratify=yb, random_state=42) | |
| auc, ap = mean_eval(Xtr, ytr, Xte, yte) | |
| print(f"(A) IN-DISTRIBUTION (official 50/50, shared gens): AUROC={auc:.3f} AP(fake)={ap:.3f} n_test={len(yte)}", flush=True) | |
| # (B) UNSEEN-GENERATOR (leave-one-generator-out, real source fixed) | |
| print("(B) UNSEEN-GENERATOR (paper Table-3 style, leave-one-gen-out, SAME real source):", flush=True) | |
| real_idx = np.where(y_gv == 1)[0] | |
| rng2 = np.random.default_rng(7); rperm = rng2.permutation(real_idx) | |
| tr_real, te_real = rperm_split(rperm) | |
| gens = sorted(set(g_gv[y_gv == 0])) | |
| per = [] | |
| for held in gens: | |
| tr_f = np.where((y_gv == 0) & (g_gv != held))[0] | |
| te_f = np.where((y_gv == 0) & (g_gv == held))[0] | |
| Xtr = np.concatenate([Xgv[tr_real], Xgv[tr_f]]); ytr = np.concatenate([np.ones(len(tr_real)), np.zeros(len(tr_f))]).astype(np.int64) | |
| Xte = np.concatenate([Xgv[te_real], Xgv[te_f]]); yte = np.concatenate([np.ones(len(te_real)), np.zeros(len(te_f))]).astype(np.int64) | |
| rngb = np.random.default_rng(1); Xtr, ytr = balance(Xtr, ytr, rngb) | |
| auc, ap = mean_eval(Xtr, ytr, Xte, yte) | |
| per.append((held, auc, ap, len(te_f))) | |
| print(f" held-out={held:<12} AUROC={auc:.3f} AP(fake)={ap:.3f} (n_fake_test={len(te_f)})", flush=True) | |
| print(f" >>> MEAN over unseen generators: AUROC={np.mean([p[1] for p in per]):.3f} AP(fake)={np.mean([p[2] for p in per]):.3f}", flush=True) | |
| # (C) CROSS-DATASET (our README "cross-generator") | |
| rngc = np.random.default_rng(5); Xtr, ytr = balance(Xex, y_ex, rngc) | |
| auc, ap = mean_eval(Xtr, ytr, Xgv, y_gv) | |
| print(f"(C) CROSS-DATASET (train=extracted/VidProM -> test=GenVideo-Val, real source SHIFTS): AUROC={auc:.3f} AP(fake)={ap:.3f} n_test={len(y_gv)}", flush=True) | |
| if __name__ == "__main__": | |
| main() | |