zero-shot-av-detector / zero_shot_detector.py
advy's picture
Update zero_shot_detector source code
89c76c9 verified
Raw History Blame Contribute Delete
21.3 kB
import os, json, glob, pickle, subprocess, warnings
from dataclasses import dataclass
from typing import Dict, List
import numpy as np, cv2, torch
from scipy import signal
from scipy.ndimage import uniform_filter1d
warnings.filterwarnings("ignore")
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
VIDEO_EXT = (".mp4", ".avi", ".mov", ".mkv", ".webm")
@dataclass
class Cfg:
max_seconds: float = 20.0
face_size: int = 224
dino_name: str = "facebook/dinov2-base"
wavlm_name: str = "microsoft/wavlm-base-plus"
wavlm_layer: int = 6
max_lag_s: float = 0.4
win_s: float = 3.0
hop_s: float = 1.5
n_null: int = 6
dino_frames: int = 64
calib_path: str = "/kaggle/working/calib.json"
cache_path: str = "/kaggle/working/feat_cache.pkl"
seed: int = 0
CFG = Cfg()
def list_videos(root):
return sorted(p for p in glob.glob(os.path.join(root, "**", "*"), recursive=True)
if p.lower().endswith(VIDEO_EXT))
def zs(x, axis=-1):
return (x - x.mean(axis, keepdims=True)) / (x.std(axis, keepdims=True) + 1e-8)
def interp_nan(x):
x = np.asarray(x, float)
m = np.isfinite(x)
if m.all():
return x
if m.sum() < 2:
return np.full_like(x, np.nan)
return np.interp(np.arange(len(x)), np.flatnonzero(m), x[m])
def longest_run(valid):
best, s = (0, 0), None
for i, v in enumerate(list(valid) + [False]):
if v and s is None:
s = i
if (not v) and s is not None:
if i - s > best[1] - best[0]:
best = (s, i)
s = None
return best
# ----------------------------------------------------------------------------
# Face tracking + frame pass
# ----------------------------------------------------------------------------
class FaceTracker:
def __init__(self):
self.fm, self.haar = None, None
try:
import mediapipe as mp
self.fm = mp.solutions.face_mesh.FaceMesh(
static_image_mode=False, max_num_faces=1, refine_landmarks=False,
min_detection_confidence=0.5, min_tracking_confidence=0.5)
except Exception:
self.haar = cv2.CascadeClassifier(
cv2.data.haarcascades + "haarcascade_frontalface_default.xml")
def detect(self, bgr):
h, w = bgr.shape[:2]
if self.fm is not None:
r = self.fm.process(cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB))
if not r.multi_face_landmarks:
return None
p = np.array([[l.x * w, l.y * h] for l in r.multi_face_landmarks[0].landmark], np.float32)
fh = np.linalg.norm(p[10] - p[152]) + 1e-6
return dict(box=(*p.min(0), *p.max(0)),
aperture=float(np.linalg.norm(p[13] - p[14]) / fh),
width=float(np.linalg.norm(p[61] - p[291]) / fh))
g = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY)
f = self.haar.detectMultiScale(g, 1.1, 5, minSize=(64, 64))
if len(f) == 0:
return None
x, y, fw, fh = max(f, key=lambda b: b[2] * b[3])
return dict(box=(x, y, x + fw, y + fh), aperture=None, width=None)
def crop_square(img, cx, cy, side, out):
s = out / side
M = np.array([[s, 0, out / 2 - s * cx], [0, s, out / 2 - s * cy]], np.float32)
return cv2.warpAffine(img, M, (out, out), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE)
def track(path, cfg, tracker):
cap = cv2.VideoCapture(path)
fps = cap.get(cv2.CAP_PROP_FPS)
if not fps or not (1 < fps <= 120):
fps = 25.0
max_frames = int(cfg.max_seconds * fps)
crops, ap, wd, mo = [], [], [], []
state, prev_low = None, None
while len(crops) < max_frames:
ok, fr = cap.read()
if not ok:
break
H, W = fr.shape[:2]
s = min(1.0, 640.0 / max(H, W))
small = cv2.resize(fr, None, fx=s, fy=s, interpolation=cv2.INTER_AREA) if s < 1 else fr
d = tracker.detect(small)
if d is None:
crops.append(None); ap.append(np.nan); wd.append(np.nan); mo.append(np.nan)
prev_low = None
continue
b = np.array(d["box"], np.float32) / s
c = np.array([(b[0] + b[2]) / 2, (b[1] + b[3]) / 2, max(b[2] - b[0], b[3] - b[1]) * 1.3], np.float32)
if state is None or np.linalg.norm(c[:2] - state[:2]) > state[2]:
state = c # first face or shot cut
else:
state = 0.8 * state + 0.2 * c # heavy smoothing so crop jitter doesn't pollute temporal cues
crop = cv2.cvtColor(crop_square(fr, *state, cfg.face_size), cv2.COLOR_BGR2RGB)
low = cv2.GaussianBlur(cv2.cvtColor(crop, cv2.COLOR_RGB2GRAY)[140:, 40:184], (5, 5), 0).astype(np.float32)
mo.append(np.nan if prev_low is None else float(np.abs(low - prev_low).mean()))
prev_low = low
crops.append(crop)
ap.append(np.nan if d["aperture"] is None else d["aperture"])
wd.append(np.nan if d["width"] is None else d["width"])
cap.release()
return dict(fps=fps, crops=crops, aperture=np.array(ap, float), width=np.array(wd, float),
motion=np.array(mo, float), has_lm=tracker.fm is not None)
# ----------------------------------------------------------------------------
# Foundation models
# ----------------------------------------------------------------------------
class Dino:
def __init__(self, name):
from transformers import AutoModel
self.m = AutoModel.from_pretrained(name).to(DEVICE).eval()
self.mean = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)
self.std = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)
@torch.no_grad()
def __call__(self, rgb_list):
x = torch.from_numpy(np.stack(rgb_list)).permute(0, 3, 1, 2).float() / 255
x = (x - self.mean) / self.std
cls, pat = [], []
for i in range(0, len(x), 16):
h = self.m(pixel_values=x[i:i + 16].to(DEVICE)).last_hidden_state
cls.append(h[:, 0].float().cpu()); pat.append(h[:, 1:].float().cpu())
return torch.cat(cls).numpy(), torch.cat(pat).numpy()
class WavLM:
def __init__(self, name, layer):
from transformers import AutoModel, AutoFeatureExtractor
self.fe = AutoFeatureExtractor.from_pretrained(name)
self.m = AutoModel.from_pretrained(name).to(DEVICE).eval()
self.layer = layer
@torch.no_grad()
def __call__(self, wav):
x = self.fe(wav, sampling_rate=16000, return_tensors="pt").input_values.to(DEVICE)
return self.m(x, output_hidden_states=True).hidden_states[self.layer][0].float().cpu().numpy()
# ----------------------------------------------------------------------------
# Audio I/O + signals
# ----------------------------------------------------------------------------
def load_audio(path, sr=16000, max_s=20.0):
cmd = ["ffmpeg", "-v", "error", "-i", path, "-t", str(max_s), "-vn", "-ac", "1",
"-ar", str(sr), "-f", "f32le", "pipe:1"]
try:
out = subprocess.run(cmd, capture_output=True, check=True).stdout
return np.frombuffer(out, np.float32).copy()
except Exception:
return np.zeros(0, np.float32)
def audio_signals(wav, fps, n, wavlm):
import librosa
S = librosa.feature.melspectrogram(y=wav, sr=16000, n_fft=400, hop_length=160,
n_mels=40, fmin=80, fmax=4000)
env = np.log(S.sum(0) + 1e-8)
flux = np.r_[0, np.maximum(0, np.diff(np.log(S + 1e-8), axis=1)).sum(0)]
t100, tv = np.arange(len(env)) / 100.0, np.arange(n) / fps
to_fr = lambda x: np.interp(tv, t100, uniform_filter1d(x, 5))
rows = [to_fr(env), to_fr(flux)]
lo, hi = np.percentile(env, 10), np.percentile(env, 95)
speech = to_fr(env) > lo + 0.35 * (hi - lo)
if wavlm is not None:
H = wavlm(wav)
U, Sv, _ = np.linalg.svd(H - H.mean(0), full_matrices=False) # per-video PCA, no training
pcs = (U[:, :3] * Sv[:3]).T
t50 = np.arange(pcs.shape[1]) * 0.02 + 0.0125
rows += [np.interp(tv, t50, uniform_filter1d(p, 2)) for p in pcs]
return np.stack(rows), speech
# ----------------------------------------------------------------------------
# Cue extractors
# ----------------------------------------------------------------------------
def wincorr(M, A, sm, sa, W, lags):
m = zs(M[:, sm:sm + W])
out = np.empty((M.shape[0], A.shape[0], len(lags)))
for k, l in enumerate(lags):
out[:, :, k] = m @ zs(A[:, sa + l: sa + l + W]).T / W
return out
def sync_features(M, A, speech, fps, cfg, rng):
"""Lip<->audio consistency, contrasted against a mismatched-audio null from the SAME video."""
n = M.shape[1]
W, hop, L = int(cfg.win_s * fps), int(cfg.hop_s * fps), int(round(cfg.max_lag_s * fps))
lags = np.arange(-L, L + 1)
all_starts = list(range(L, n - W - L, hop))
starts = [s for s in all_starts if speech[s:s + W].mean() > 0.5 and (M[:, s:s + W].std(1) > 1e-6).any()]
if len(starts) < 2:
return {}
margins, best_lags = [], []
for s in starts:
c = np.abs(wincorr(M, A, s, s, W, lags)).max(axis=(0, 1))
best_lags.append(lags[c.argmax()] / fps)
others = [o for o in all_starts if abs(o - s) >= W]
if not others:
continue
pick = rng.choice(others, size=min(cfg.n_null, len(others)), replace=False)
null = np.mean([np.abs(wincorr(M, A, s, int(o), W, lags)).max() for o in pick])
margins.append(c.max() - null)
if not margins:
return {}
return dict(sync_margin=float(np.mean(margins)),
sync_lag_abs=float(np.median(np.abs(best_lags))),
sync_lag_std=float(np.std(best_lags)))
def visual_temporal(cls, pat):
E = cls / np.linalg.norm(cls, axis=1, keepdims=True)
d1 = np.linalg.norm(np.diff(E, axis=0), axis=1)
d2 = np.linalg.norm(E[2:] - 2 * E[1:-1] + E[:-2], axis=1)
med = np.median(d1)
mad = 1.4826 * np.median(np.abs(d1 - med)) + 1e-8
P = pat / np.linalg.norm(pat, axis=2, keepdims=True)
per_pos = np.linalg.norm(np.diff(P, axis=0), axis=2).mean(0) # temporal change per patch location
return dict(vis_jitter=float(np.median(d2) / (med + 1e-8)),
vis_spike=float(np.mean(d1 > med + 5 * mad)),
vis_flicker_cv=float(per_pos.std() / (per_pos.mean() + 1e-8))) # blend-boundary flicker
def freq_features(crops):
S = crops[0].shape[0]
win = np.outer(np.hanning(S), np.hanning(S))
yy, xx = np.indices((S, S))
r = np.hypot(yy - S // 2, xx - S // 2).astype(int)
cnt = np.maximum(np.bincount(r.ravel()), 1)
mask = (r > 0.55 * S // 2) & (r < S // 2)
profs, peaks = [], []
for c in crops:
g = cv2.cvtColor(c, cv2.COLOR_RGB2GRAY).astype(np.float32) / 255
P = np.log(np.abs(np.fft.fftshift(np.fft.fft2((g - g.mean()) * win))) ** 2 + 1e-12)
radial = np.bincount(r.ravel(), P.ravel()) / cnt
profs.append(radial[:S // 2])
peaks.append(np.percentile((P - radial[r])[mask], 99.9)) # periodic upsampling peaks
prof = np.median(profs, 0)
k = S // 2
return dict(freq_hf_drop=float(prof[int(.6 * k):int(.98 * k)].mean() - prof[int(.08 * k):int(.35 * k)].mean()),
freq_peak=float(np.median(peaks)))
def rppg_features(crops, fps):
if len(crops) / fps < 6 or fps < 8:
return {}
rois = [(20, 60, 60, 164), (120, 170, 30, 80), (120, 170, 144, 194)] # forehead, L cheek, R cheek
sig = np.array([[c[y0:y1, x0:x1, 1].mean() for (y0, y1, x0, x1) in rois] for c in crops])
b, a = signal.butter(3, [0.7, 3.5], btype="band", fs=fps)
f = zs(signal.filtfilt(b, a, signal.detrend(sig, axis=0), axis=0), axis=0)
C = np.corrcoef(f.T)
fr, P = signal.welch(f, fs=fps, nperseg=min(len(f), int(fps * 6)), axis=0)
band = (fr >= 0.7) & (fr <= 3.5)
fb, snr, pk = fr[band], [], []
for k in range(3):
p = P[band, k]
i = p.argmax()
near = np.abs(fb - fb[i]) <= 0.15
snr.append(p[near].sum() / (p.sum() - p[near].sum() + 1e-8)); pk.append(fb[i])
return dict(ppg_coherence=float(np.mean([C[0, 1], C[0, 2], C[1, 2]])),
ppg_snr=float(np.mean(snr)), ppg_freq_spread=float(np.ptp(pk)))
def audio_features(wav, sr=16000):
import librosa
if len(wav) < sr * 2:
return {}
S = np.abs(librosa.stft(wav, n_fft=512, hop_length=160)) ** 2
e = S.sum(0)
quiet, loud = e < np.percentile(e, 20), e > np.percentile(e, 60)
hf = 10 * np.log10(S[128:].sum(0) + 1e-10) - 10 * np.log10(S[8:128].sum(0) + 1e-10)
flat = librosa.feature.spectral_flatness(S=np.sqrt(S))[0]
out = dict(aud_hf_ratio=float(np.median(hf[loud])),
aud_floor_db=float(np.median(10 * np.log10(e[quiet] + 1e-10))),
aud_quiet_flat=float(np.median(flat[quiet])),
aud_dig_silence=float(np.mean(e < 1e-9)))
try:
f0 = librosa.yin(wav, fmin=70, fmax=400, sr=sr, frame_length=1024, hop_length=160)
m = min(len(f0), len(loud))
v = f0[:m][loud[:m]]
if len(v) > 50:
st = 12 * np.log2(v / np.median(v))
out["aud_f0_jitter"] = float(np.median(np.abs(np.diff(st))))
except Exception:
pass
return out
# ----------------------------------------------------------------------------
# Label-free fusion
# ----------------------------------------------------------------------------
# name: (direction [+1 higher=fake, -1 lower=fake, 0 two-sided], fixed prior (center, scale) or None, weight)
# Cues without a prior only activate after calibrate() on REAL videos. Priors are rough starting points.
SPEC = {
"sync_margin": (-1, (0.12, 0.08), 2.0),
"sync_lag_abs": (+1, (0.08, 0.08), 1.0),
"sync_lag_std": (+1, (0.10, 0.10), 0.7),
"vis_jitter": (+1, None, 1.0),
"vis_spike": (+1, None, 0.7),
"vis_flicker_cv": (+1, None, 0.7),
"ppg_coherence": (-1, None, 0.6),
"ppg_snr": (-1, None, 0.4),
"ppg_freq_spread": (+1, None, 0.4),
"freq_hf_drop": (0, None, 0.8),
"freq_peak": (0, None, 0.8),
"aud_hf_ratio": (0, None, 0.6),
"aud_floor_db": (0, None, 0.5),
"aud_quiet_flat": (0, None, 0.5),
"aud_dig_silence": (+1, None, 0.5),
"aud_f0_jitter": (0, None, 0.6),
}
class Fuser:
def __init__(self, calib=None):
self.calib = calib or {}
self.thr = self.calib.get("thr", 2.0)
def zscores(self, feats):
out = {}
for k, (d, prior, _) in SPEC.items():
x = feats.get(k, np.nan)
if x is None or not np.isfinite(x):
continue
st = self.calib.get("stats", {}).get(k)
if st:
c, s = st["center"], st["scale"]
elif prior:
c, s = prior
else:
continue
z = (x - c) / s
z = abs(z) - 0.8 if d == 0 else d * z # 0.8 = mean of half-normal
out[k] = float(np.clip(z, -3, 6))
return out
def score(self, feats):
z = self.zscores(feats)
if not z:
return dict(score=np.nan, p_fake=np.nan, z=z)
w = np.array([SPEC[k][2] for k in z]); zz = np.array(list(z.values()))
s = float((w * zz).sum() / np.sqrt((w ** 2).sum())) # weighted Stouffer
return dict(score=s, p_fake=float(1 / (1 + np.exp(-(s - self.thr)))), z=z)
# ----------------------------------------------------------------------------
# Detector
# ----------------------------------------------------------------------------
class ZeroShotAVDetector:
def __init__(self, cfg=CFG, use_dino=True, use_wavlm=True):
self.cfg = cfg
self.tracker = FaceTracker()
self.dino = Dino(cfg.dino_name) if use_dino else None
self.wavlm = WavLM(cfg.wavlm_name, cfg.wavlm_layer) if use_wavlm else None
calib = json.load(open(cfg.calib_path)) if os.path.exists(cfg.calib_path) else None
self.fuser = Fuser(calib)
self.cache = pickle.load(open(cfg.cache_path, "rb")) if os.path.exists(cfg.cache_path) else {}
print(f"device={DEVICE} | landmarks={'mediapipe' if self.tracker.fm else 'haar (fallback)'} | "
f"calibrated={calib is not None}")
def extract(self, path):
cfg, f = self.cfg, {}
rng = np.random.default_rng(cfg.seed)
tr = track(path, cfg, self.tracker)
fps, n = tr["fps"], len(tr["crops"])
wav = load_audio(path, 16000, cfg.max_seconds)
valid = np.array([c is not None for c in tr["crops"]])
if n == 0:
return f
try:
f.update(audio_features(wav))
except Exception as e:
print("audio cue failed:", e)
s0, s1 = longest_run(valid)
if valid.mean() >= 0.4 and s1 - s0 >= 16:
seg = tr["crops"][s0:s1]
for name, fn in [("freq", lambda: freq_features(seg[::max(1, len(seg) // 48)])),
("rppg", lambda: rppg_features(seg, fps))]:
try:
f.update(fn())
except Exception as e:
print(f"{name} cue failed:", e)
if self.dino is not None:
try:
k = min(cfg.dino_frames, len(seg)); st = (len(seg) - k) // 2
cls, pat = self.dino(seg[st:st + k])
f.update(visual_temporal(cls, pat))
except Exception as e:
print("dino cue failed:", e)
if len(wav) > 16000 * 3 and valid.mean() >= 0.5:
try:
rows = []
if tr["has_lm"]:
rows += [interp_nan(tr["aperture"]), interp_nan(tr["width"])]
rows.append(interp_nan(tr["motion"]))
M = np.stack(rows)
if np.isfinite(M).all():
A, speech = audio_signals(wav, fps, n, self.wavlm)
k = int(fps) + 1
M = M - uniform_filter1d(M, k, axis=1)
A = A - uniform_filter1d(A, k, axis=1)
f.update(sync_features(M, A, speech, fps, cfg, rng))
except Exception as e:
print("sync cue failed:", e)
return f
def extract_cached(self, path):
if path not in self.cache:
self.cache[path] = self.extract(path)
return self.cache[path]
def save_cache(self):
pickle.dump(self.cache, open(self.cfg.cache_path, "wb"))
def predict(self, path):
feats = self.extract(path)
r = self.fuser.score(feats)
r.update(features=feats, is_fake=bool(r["score"] > self.fuser.thr) if np.isfinite(r["score"]) else None,
n_cues=len(r["z"]), threshold=self.fuser.thr)
return r
def calibrate(self, real_paths: List[str], fpr=0.05):
"""REAL-ONLY calibration: per-cue robust stats + threshold at target FPR. No fake data touched."""
feats = [self.extract_cached(p) for p in real_paths]; self.save_cache()
stats = {}
for k in SPEC:
v = np.array([x.get(k, np.nan) for x in feats], float); v = v[np.isfinite(v)]
if len(v) >= 8:
med = np.median(v)
scale = max(1.4826 * np.median(np.abs(v - med)), 0.5 * v.std(), 1e-6)
stats[k] = dict(center=float(med), scale=float(scale), n=int(len(v)))
tmp = Fuser(dict(stats=stats))
sc = np.array([tmp.score(x)["score"] for x in feats]); sc = sc[np.isfinite(sc)]
calib = dict(stats=stats, thr=float(np.quantile(sc, 1 - fpr)) if len(sc) else 2.0)
json.dump(calib, open(self.cfg.calib_path, "w"), indent=1)
self.fuser = Fuser(calib)
print(f"calibrated {len(stats)} cues on {len(feats)} real videos | thr={calib['thr']:.3f}")
def evaluate(self, real_paths: List[str], fake_groups: Dict[str, List[str]]):
"""Report-only cross-technique evaluation. Do NOT tune on these fakes or you lose zero-shot validity."""
from sklearn.metrics import roc_auc_score
Fr = [self.extract_cached(p) for p in real_paths]
Zr = [self.fuser.score(f) for f in Fr]
results = {}
for tech, paths in fake_groups.items():
Ff = [self.extract_cached(p) for p in paths]
Zf = [self.fuser.score(f) for f in Ff]
y = np.array([0] * len(Zr) + [1] * len(Zf))
s = np.array([z["score"] for z in Zr + Zf])
ok = np.isfinite(s)
res = {"FUSED": roc_auc_score(y[ok], s[ok]) if len(set(y[ok])) == 2 else np.nan,
"coverage": float(ok.mean())}
for k in SPEC:
v = np.array([z["z"].get(k, np.nan) for z in Zr + Zf])
m = np.isfinite(v)
if m.sum() > 10 and len(set(y[m])) == 2:
res[k] = roc_auc_score(y[m], v[m])
results[tech] = res
print(f"\n[{tech}] fused AUC={res['FUSED']:.3f} coverage={res['coverage']:.2f}")
for k, v in sorted(((k, v) for k, v in res.items() if k in SPEC), key=lambda t: -t[1]):
print(f" {k:16s} AUC={v:.3f}")
self.save_cache()
return results