import os, json, glob, pickle, subprocess, warnings from dataclasses import dataclass from typing import Dict, List import numpy as np, cv2, torch from scipy import signal from scipy.ndimage import uniform_filter1d warnings.filterwarnings("ignore") DEVICE = "cuda" if torch.cuda.is_available() else "cpu" VIDEO_EXT = (".mp4", ".avi", ".mov", ".mkv", ".webm") @dataclass class Cfg: max_seconds: float = 20.0 face_size: int = 224 dino_name: str = "facebook/dinov2-base" wavlm_name: str = "microsoft/wavlm-base-plus" wavlm_layer: int = 6 max_lag_s: float = 0.4 win_s: float = 3.0 hop_s: float = 1.5 n_null: int = 6 dino_frames: int = 64 calib_path: str = "/kaggle/working/calib.json" cache_path: str = "/kaggle/working/feat_cache.pkl" seed: int = 0 CFG = Cfg() def list_videos(root): return sorted(p for p in glob.glob(os.path.join(root, "**", "*"), recursive=True) if p.lower().endswith(VIDEO_EXT)) def zs(x, axis=-1): return (x - x.mean(axis, keepdims=True)) / (x.std(axis, keepdims=True) + 1e-8) def interp_nan(x): x = np.asarray(x, float) m = np.isfinite(x) if m.all(): return x if m.sum() < 2: return np.full_like(x, np.nan) return np.interp(np.arange(len(x)), np.flatnonzero(m), x[m]) def longest_run(valid): best, s = (0, 0), None for i, v in enumerate(list(valid) + [False]): if v and s is None: s = i if (not v) and s is not None: if i - s > best[1] - best[0]: best = (s, i) s = None return best # ---------------------------------------------------------------------------- # Face tracking + frame pass # ---------------------------------------------------------------------------- class FaceTracker: def __init__(self): self.fm, self.haar = None, None try: import mediapipe as mp self.fm = mp.solutions.face_mesh.FaceMesh( static_image_mode=False, max_num_faces=1, refine_landmarks=False, min_detection_confidence=0.5, min_tracking_confidence=0.5) except Exception: self.haar = cv2.CascadeClassifier( cv2.data.haarcascades + "haarcascade_frontalface_default.xml") def detect(self, bgr): h, w = bgr.shape[:2] if self.fm is not None: r = self.fm.process(cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)) if not r.multi_face_landmarks: return None p = np.array([[l.x * w, l.y * h] for l in r.multi_face_landmarks[0].landmark], np.float32) fh = np.linalg.norm(p[10] - p[152]) + 1e-6 return dict(box=(*p.min(0), *p.max(0)), aperture=float(np.linalg.norm(p[13] - p[14]) / fh), width=float(np.linalg.norm(p[61] - p[291]) / fh)) g = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY) f = self.haar.detectMultiScale(g, 1.1, 5, minSize=(64, 64)) if len(f) == 0: return None x, y, fw, fh = max(f, key=lambda b: b[2] * b[3]) return dict(box=(x, y, x + fw, y + fh), aperture=None, width=None) def crop_square(img, cx, cy, side, out): s = out / side M = np.array([[s, 0, out / 2 - s * cx], [0, s, out / 2 - s * cy]], np.float32) return cv2.warpAffine(img, M, (out, out), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE) def track(path, cfg, tracker): cap = cv2.VideoCapture(path) fps = cap.get(cv2.CAP_PROP_FPS) if not fps or not (1 < fps <= 120): fps = 25.0 max_frames = int(cfg.max_seconds * fps) crops, ap, wd, mo = [], [], [], [] state, prev_low = None, None while len(crops) < max_frames: ok, fr = cap.read() if not ok: break H, W = fr.shape[:2] s = min(1.0, 640.0 / max(H, W)) small = cv2.resize(fr, None, fx=s, fy=s, interpolation=cv2.INTER_AREA) if s < 1 else fr d = tracker.detect(small) if d is None: crops.append(None); ap.append(np.nan); wd.append(np.nan); mo.append(np.nan) prev_low = None continue b = np.array(d["box"], np.float32) / s c = np.array([(b[0] + b[2]) / 2, (b[1] + b[3]) / 2, max(b[2] - b[0], b[3] - b[1]) * 1.3], np.float32) if state is None or np.linalg.norm(c[:2] - state[:2]) > state[2]: state = c # first face or shot cut else: state = 0.8 * state + 0.2 * c # heavy smoothing so crop jitter doesn't pollute temporal cues crop = cv2.cvtColor(crop_square(fr, *state, cfg.face_size), cv2.COLOR_BGR2RGB) low = cv2.GaussianBlur(cv2.cvtColor(crop, cv2.COLOR_RGB2GRAY)[140:, 40:184], (5, 5), 0).astype(np.float32) mo.append(np.nan if prev_low is None else float(np.abs(low - prev_low).mean())) prev_low = low crops.append(crop) ap.append(np.nan if d["aperture"] is None else d["aperture"]) wd.append(np.nan if d["width"] is None else d["width"]) cap.release() return dict(fps=fps, crops=crops, aperture=np.array(ap, float), width=np.array(wd, float), motion=np.array(mo, float), has_lm=tracker.fm is not None) # ---------------------------------------------------------------------------- # Foundation models # ---------------------------------------------------------------------------- class Dino: def __init__(self, name): from transformers import AutoModel self.m = AutoModel.from_pretrained(name).to(DEVICE).eval() self.mean = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1) self.std = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1) @torch.no_grad() def __call__(self, rgb_list): x = torch.from_numpy(np.stack(rgb_list)).permute(0, 3, 1, 2).float() / 255 x = (x - self.mean) / self.std cls, pat = [], [] for i in range(0, len(x), 16): h = self.m(pixel_values=x[i:i + 16].to(DEVICE)).last_hidden_state cls.append(h[:, 0].float().cpu()); pat.append(h[:, 1:].float().cpu()) return torch.cat(cls).numpy(), torch.cat(pat).numpy() class WavLM: def __init__(self, name, layer): from transformers import AutoModel, AutoFeatureExtractor self.fe = AutoFeatureExtractor.from_pretrained(name) self.m = AutoModel.from_pretrained(name).to(DEVICE).eval() self.layer = layer @torch.no_grad() def __call__(self, wav): x = self.fe(wav, sampling_rate=16000, return_tensors="pt").input_values.to(DEVICE) return self.m(x, output_hidden_states=True).hidden_states[self.layer][0].float().cpu().numpy() # ---------------------------------------------------------------------------- # Audio I/O + signals # ---------------------------------------------------------------------------- def load_audio(path, sr=16000, max_s=20.0): cmd = ["ffmpeg", "-v", "error", "-i", path, "-t", str(max_s), "-vn", "-ac", "1", "-ar", str(sr), "-f", "f32le", "pipe:1"] try: out = subprocess.run(cmd, capture_output=True, check=True).stdout return np.frombuffer(out, np.float32).copy() except Exception: return np.zeros(0, np.float32) def audio_signals(wav, fps, n, wavlm): import librosa S = librosa.feature.melspectrogram(y=wav, sr=16000, n_fft=400, hop_length=160, n_mels=40, fmin=80, fmax=4000) env = np.log(S.sum(0) + 1e-8) flux = np.r_[0, np.maximum(0, np.diff(np.log(S + 1e-8), axis=1)).sum(0)] t100, tv = np.arange(len(env)) / 100.0, np.arange(n) / fps to_fr = lambda x: np.interp(tv, t100, uniform_filter1d(x, 5)) rows = [to_fr(env), to_fr(flux)] lo, hi = np.percentile(env, 10), np.percentile(env, 95) speech = to_fr(env) > lo + 0.35 * (hi - lo) if wavlm is not None: H = wavlm(wav) U, Sv, _ = np.linalg.svd(H - H.mean(0), full_matrices=False) # per-video PCA, no training pcs = (U[:, :3] * Sv[:3]).T t50 = np.arange(pcs.shape[1]) * 0.02 + 0.0125 rows += [np.interp(tv, t50, uniform_filter1d(p, 2)) for p in pcs] return np.stack(rows), speech # ---------------------------------------------------------------------------- # Cue extractors # ---------------------------------------------------------------------------- def wincorr(M, A, sm, sa, W, lags): m = zs(M[:, sm:sm + W]) out = np.empty((M.shape[0], A.shape[0], len(lags))) for k, l in enumerate(lags): out[:, :, k] = m @ zs(A[:, sa + l: sa + l + W]).T / W return out def sync_features(M, A, speech, fps, cfg, rng): """Lip<->audio consistency, contrasted against a mismatched-audio null from the SAME video.""" n = M.shape[1] W, hop, L = int(cfg.win_s * fps), int(cfg.hop_s * fps), int(round(cfg.max_lag_s * fps)) lags = np.arange(-L, L + 1) all_starts = list(range(L, n - W - L, hop)) starts = [s for s in all_starts if speech[s:s + W].mean() > 0.5 and (M[:, s:s + W].std(1) > 1e-6).any()] if len(starts) < 2: return {} margins, best_lags = [], [] for s in starts: c = np.abs(wincorr(M, A, s, s, W, lags)).max(axis=(0, 1)) best_lags.append(lags[c.argmax()] / fps) others = [o for o in all_starts if abs(o - s) >= W] if not others: continue pick = rng.choice(others, size=min(cfg.n_null, len(others)), replace=False) null = np.mean([np.abs(wincorr(M, A, s, int(o), W, lags)).max() for o in pick]) margins.append(c.max() - null) if not margins: return {} return dict(sync_margin=float(np.mean(margins)), sync_lag_abs=float(np.median(np.abs(best_lags))), sync_lag_std=float(np.std(best_lags))) def visual_temporal(cls, pat): E = cls / np.linalg.norm(cls, axis=1, keepdims=True) d1 = np.linalg.norm(np.diff(E, axis=0), axis=1) d2 = np.linalg.norm(E[2:] - 2 * E[1:-1] + E[:-2], axis=1) med = np.median(d1) mad = 1.4826 * np.median(np.abs(d1 - med)) + 1e-8 P = pat / np.linalg.norm(pat, axis=2, keepdims=True) per_pos = np.linalg.norm(np.diff(P, axis=0), axis=2).mean(0) # temporal change per patch location return dict(vis_jitter=float(np.median(d2) / (med + 1e-8)), vis_spike=float(np.mean(d1 > med + 5 * mad)), vis_flicker_cv=float(per_pos.std() / (per_pos.mean() + 1e-8))) # blend-boundary flicker def freq_features(crops): S = crops[0].shape[0] win = np.outer(np.hanning(S), np.hanning(S)) yy, xx = np.indices((S, S)) r = np.hypot(yy - S // 2, xx - S // 2).astype(int) cnt = np.maximum(np.bincount(r.ravel()), 1) mask = (r > 0.55 * S // 2) & (r < S // 2) profs, peaks = [], [] for c in crops: g = cv2.cvtColor(c, cv2.COLOR_RGB2GRAY).astype(np.float32) / 255 P = np.log(np.abs(np.fft.fftshift(np.fft.fft2((g - g.mean()) * win))) ** 2 + 1e-12) radial = np.bincount(r.ravel(), P.ravel()) / cnt profs.append(radial[:S // 2]) peaks.append(np.percentile((P - radial[r])[mask], 99.9)) # periodic upsampling peaks prof = np.median(profs, 0) k = S // 2 return dict(freq_hf_drop=float(prof[int(.6 * k):int(.98 * k)].mean() - prof[int(.08 * k):int(.35 * k)].mean()), freq_peak=float(np.median(peaks))) def rppg_features(crops, fps): if len(crops) / fps < 6 or fps < 8: return {} rois = [(20, 60, 60, 164), (120, 170, 30, 80), (120, 170, 144, 194)] # forehead, L cheek, R cheek sig = np.array([[c[y0:y1, x0:x1, 1].mean() for (y0, y1, x0, x1) in rois] for c in crops]) b, a = signal.butter(3, [0.7, 3.5], btype="band", fs=fps) f = zs(signal.filtfilt(b, a, signal.detrend(sig, axis=0), axis=0), axis=0) C = np.corrcoef(f.T) fr, P = signal.welch(f, fs=fps, nperseg=min(len(f), int(fps * 6)), axis=0) band = (fr >= 0.7) & (fr <= 3.5) fb, snr, pk = fr[band], [], [] for k in range(3): p = P[band, k] i = p.argmax() near = np.abs(fb - fb[i]) <= 0.15 snr.append(p[near].sum() / (p.sum() - p[near].sum() + 1e-8)); pk.append(fb[i]) return dict(ppg_coherence=float(np.mean([C[0, 1], C[0, 2], C[1, 2]])), ppg_snr=float(np.mean(snr)), ppg_freq_spread=float(np.ptp(pk))) def audio_features(wav, sr=16000): import librosa if len(wav) < sr * 2: return {} S = np.abs(librosa.stft(wav, n_fft=512, hop_length=160)) ** 2 e = S.sum(0) quiet, loud = e < np.percentile(e, 20), e > np.percentile(e, 60) hf = 10 * np.log10(S[128:].sum(0) + 1e-10) - 10 * np.log10(S[8:128].sum(0) + 1e-10) flat = librosa.feature.spectral_flatness(S=np.sqrt(S))[0] out = dict(aud_hf_ratio=float(np.median(hf[loud])), aud_floor_db=float(np.median(10 * np.log10(e[quiet] + 1e-10))), aud_quiet_flat=float(np.median(flat[quiet])), aud_dig_silence=float(np.mean(e < 1e-9))) try: f0 = librosa.yin(wav, fmin=70, fmax=400, sr=sr, frame_length=1024, hop_length=160) m = min(len(f0), len(loud)) v = f0[:m][loud[:m]] if len(v) > 50: st = 12 * np.log2(v / np.median(v)) out["aud_f0_jitter"] = float(np.median(np.abs(np.diff(st)))) except Exception: pass return out # ---------------------------------------------------------------------------- # Label-free fusion # ---------------------------------------------------------------------------- # name: (direction [+1 higher=fake, -1 lower=fake, 0 two-sided], fixed prior (center, scale) or None, weight) # Cues without a prior only activate after calibrate() on REAL videos. Priors are rough starting points. SPEC = { "sync_margin": (-1, (0.12, 0.08), 2.0), "sync_lag_abs": (+1, (0.08, 0.08), 1.0), "sync_lag_std": (+1, (0.10, 0.10), 0.7), "vis_jitter": (+1, None, 1.0), "vis_spike": (+1, None, 0.7), "vis_flicker_cv": (+1, None, 0.7), "ppg_coherence": (-1, None, 0.6), "ppg_snr": (-1, None, 0.4), "ppg_freq_spread": (+1, None, 0.4), "freq_hf_drop": (0, None, 0.8), "freq_peak": (0, None, 0.8), "aud_hf_ratio": (0, None, 0.6), "aud_floor_db": (0, None, 0.5), "aud_quiet_flat": (0, None, 0.5), "aud_dig_silence": (+1, None, 0.5), "aud_f0_jitter": (0, None, 0.6), } class Fuser: def __init__(self, calib=None): self.calib = calib or {} self.thr = self.calib.get("thr", 2.0) def zscores(self, feats): out = {} for k, (d, prior, _) in SPEC.items(): x = feats.get(k, np.nan) if x is None or not np.isfinite(x): continue st = self.calib.get("stats", {}).get(k) if st: c, s = st["center"], st["scale"] elif prior: c, s = prior else: continue z = (x - c) / s z = abs(z) - 0.8 if d == 0 else d * z # 0.8 = mean of half-normal out[k] = float(np.clip(z, -3, 6)) return out def score(self, feats): z = self.zscores(feats) if not z: return dict(score=np.nan, p_fake=np.nan, z=z) w = np.array([SPEC[k][2] for k in z]); zz = np.array(list(z.values())) s = float((w * zz).sum() / np.sqrt((w ** 2).sum())) # weighted Stouffer return dict(score=s, p_fake=float(1 / (1 + np.exp(-(s - self.thr)))), z=z) # ---------------------------------------------------------------------------- # Detector # ---------------------------------------------------------------------------- class ZeroShotAVDetector: def __init__(self, cfg=CFG, use_dino=True, use_wavlm=True): self.cfg = cfg self.tracker = FaceTracker() self.dino = Dino(cfg.dino_name) if use_dino else None self.wavlm = WavLM(cfg.wavlm_name, cfg.wavlm_layer) if use_wavlm else None calib = json.load(open(cfg.calib_path)) if os.path.exists(cfg.calib_path) else None self.fuser = Fuser(calib) self.cache = pickle.load(open(cfg.cache_path, "rb")) if os.path.exists(cfg.cache_path) else {} print(f"device={DEVICE} | landmarks={'mediapipe' if self.tracker.fm else 'haar (fallback)'} | " f"calibrated={calib is not None}") def extract(self, path): cfg, f = self.cfg, {} rng = np.random.default_rng(cfg.seed) tr = track(path, cfg, self.tracker) fps, n = tr["fps"], len(tr["crops"]) wav = load_audio(path, 16000, cfg.max_seconds) valid = np.array([c is not None for c in tr["crops"]]) if n == 0: return f try: f.update(audio_features(wav)) except Exception as e: print("audio cue failed:", e) s0, s1 = longest_run(valid) if valid.mean() >= 0.4 and s1 - s0 >= 16: seg = tr["crops"][s0:s1] for name, fn in [("freq", lambda: freq_features(seg[::max(1, len(seg) // 48)])), ("rppg", lambda: rppg_features(seg, fps))]: try: f.update(fn()) except Exception as e: print(f"{name} cue failed:", e) if self.dino is not None: try: k = min(cfg.dino_frames, len(seg)); st = (len(seg) - k) // 2 cls, pat = self.dino(seg[st:st + k]) f.update(visual_temporal(cls, pat)) except Exception as e: print("dino cue failed:", e) if len(wav) > 16000 * 3 and valid.mean() >= 0.5: try: rows = [] if tr["has_lm"]: rows += [interp_nan(tr["aperture"]), interp_nan(tr["width"])] rows.append(interp_nan(tr["motion"])) M = np.stack(rows) if np.isfinite(M).all(): A, speech = audio_signals(wav, fps, n, self.wavlm) k = int(fps) + 1 M = M - uniform_filter1d(M, k, axis=1) A = A - uniform_filter1d(A, k, axis=1) f.update(sync_features(M, A, speech, fps, cfg, rng)) except Exception as e: print("sync cue failed:", e) return f def extract_cached(self, path): if path not in self.cache: self.cache[path] = self.extract(path) return self.cache[path] def save_cache(self): pickle.dump(self.cache, open(self.cfg.cache_path, "wb")) def predict(self, path): feats = self.extract(path) r = self.fuser.score(feats) r.update(features=feats, is_fake=bool(r["score"] > self.fuser.thr) if np.isfinite(r["score"]) else None, n_cues=len(r["z"]), threshold=self.fuser.thr) return r def calibrate(self, real_paths: List[str], fpr=0.05): """REAL-ONLY calibration: per-cue robust stats + threshold at target FPR. No fake data touched.""" feats = [self.extract_cached(p) for p in real_paths]; self.save_cache() stats = {} for k in SPEC: v = np.array([x.get(k, np.nan) for x in feats], float); v = v[np.isfinite(v)] if len(v) >= 8: med = np.median(v) scale = max(1.4826 * np.median(np.abs(v - med)), 0.5 * v.std(), 1e-6) stats[k] = dict(center=float(med), scale=float(scale), n=int(len(v))) tmp = Fuser(dict(stats=stats)) sc = np.array([tmp.score(x)["score"] for x in feats]); sc = sc[np.isfinite(sc)] calib = dict(stats=stats, thr=float(np.quantile(sc, 1 - fpr)) if len(sc) else 2.0) json.dump(calib, open(self.cfg.calib_path, "w"), indent=1) self.fuser = Fuser(calib) print(f"calibrated {len(stats)} cues on {len(feats)} real videos | thr={calib['thr']:.3f}") def evaluate(self, real_paths: List[str], fake_groups: Dict[str, List[str]]): """Report-only cross-technique evaluation. Do NOT tune on these fakes or you lose zero-shot validity.""" from sklearn.metrics import roc_auc_score Fr = [self.extract_cached(p) for p in real_paths] Zr = [self.fuser.score(f) for f in Fr] results = {} for tech, paths in fake_groups.items(): Ff = [self.extract_cached(p) for p in paths] Zf = [self.fuser.score(f) for f in Ff] y = np.array([0] * len(Zr) + [1] * len(Zf)) s = np.array([z["score"] for z in Zr + Zf]) ok = np.isfinite(s) res = {"FUSED": roc_auc_score(y[ok], s[ok]) if len(set(y[ok])) == 2 else np.nan, "coverage": float(ok.mean())} for k in SPEC: v = np.array([z["z"].get(k, np.nan) for z in Zr + Zf]) m = np.isfinite(v) if m.sum() > 10 and len(set(y[m])) == 2: res[k] = roc_auc_score(y[m], v[m]) results[tech] = res print(f"\n[{tech}] fused AUC={res['FUSED']:.3f} coverage={res['coverage']:.2f}") for k, v in sorted(((k, v) for k, v in res.items() if k in SPEC), key=lambda t: -t[1]): print(f" {k:16s} AUC={v:.3f}") self.save_cache() return results