""" Local digital forensics engine - runs 100% offline. Supports: image analysis (ELA + EXIF + AI-pattern detection) and video analysis (frame consistency + face-region checks). Each "reason" is returned as a structured dict: {"key": "", "params": {...}, "en": ""} so the frontend can render it in any supported language while still having a safe English fallback if a key isn't recognized. """ import io import os import json import time import logging from PIL import Image, ExifTags import numpy as np import cv2 import piexif import ai_detector import deepfake_face_detector import face_detector import efficientnetb3_detector import sdxl_face_detector import noise_residual_detector import clone_detection_module import generation_metadata_detector import pixel_forensics_detector import community_forensics_detector import c2pa_detector import steganographia_detector import nonescape_bridge import insightface_detector import video_metadata_detector import blink_detector import audio_extractor import voice_deepfake_detector import lipsync_detector import lighting_balance_detector # ============================================================ # External settings (config.json, project root — next to run.bat / # the packaged .exe). Lets weights be tuned by editing a plain text # file and restarting the app — no code change or rebuild needed. # Falls back to these hardcoded defaults (the current tuned values) if # config.json is missing, unreadable, or missing a specific key, so a # broken/incomplete config file can never crash the app. # ============================================================ _DEFAULT_IMAGE_WEIGHTS = { "exif": 0.10, "model": 0.28, "face_df": 0.10, "effnb3": 0.40, "noise_residual": 0.40, "clone_detection": 0.40, "nonescape": 0.50, } _DEFAULT_VIDEO_WEIGHTS = { "heuristic": 0.25, "model": 0.25, "face_df": 0.20, "rppg": 0.30, "meta": 0.15, "blink": 0.20, "nonescape": 0.40, "noise": 0.20, "pixel": 0.15, "voice": 0.25, "lipsync": 0.20, } _CONFIG_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "config.json") def _load_weights(config_key, defaults): try: with open(_CONFIG_PATH, "r", encoding="utf-8") as f: cfg = json.load(f) loaded = cfg.get(config_key, {}) merged = dict(defaults) for k, v in loaded.items(): if k in merged and isinstance(v, (int, float)): merged[k] = float(v) return merged except Exception as e: print(f"[analysis_engine] Could not load {config_key} from config.json ({e}) — using built-in defaults.") return dict(defaults) IMAGE_WEIGHTS = _load_weights("image_weights", _DEFAULT_IMAGE_WEIGHTS) VIDEO_WEIGHTS = _load_weights("video_weights", _DEFAULT_VIDEO_WEIGHTS) def r(key, en, **params): """Helper to build a structured reason entry.""" return {"key": key, "params": params, "en": en} def _safe_remove(path, retries=5, delay=0.3): """ Deletes a temp file, retrying briefly on Windows file-lock errors (OneDrive syncing the file, antivirus scanning it, etc. can hold a transient lock right after we finish writing/reading it — a real, observed cause of PermissionError here, not a bug in our own code). Silently gives up after the retries; a leftover temp file isn't fatal, just noise in the temp folder. """ for attempt in range(retries): try: if os.path.exists(path): os.remove(path) return except PermissionError: if attempt == retries - 1: print(f"[analysis_engine] Could not remove temp file after {retries} tries (locked by another process): {path}") return time.sleep(delay) # ============================================================ # Image analysis # ============================================================ def check_exif(pil_image): reasons = [] suspicion = 0 matched_software = None exif_raw = pil_image._getexif() if hasattr(pil_image, "_getexif") else None if not exif_raw: suspicion += 25 reasons.append(r("exif_none", "No EXIF data at all — common in AI-generated images, screenshots, or images exported from editing tools that strip metadata")) return suspicion, reasons, matched_software exif = {ExifTags.TAGS.get(k, k): v for k, v in exif_raw.items()} has_make = "Make" in exif or "Model" in exif has_datetime = "DateTime" in exif or "DateTimeOriginal" in exif has_software = "Software" in exif if not has_make: suspicion += 15 reasons.append(r("exif_no_make", "No camera/device info (Make/Model) — unusual for an original photo")) if has_software: software_val = str(exif.get("Software", "")) editing_tools = ["photoshop", "gimp", "lightroom", "midjourney", "dall-e", "stable diffusion", "canva"] matched = next((t for t in editing_tools if t in software_val.lower()), None) if matched: matched_software = software_val suspicion += 30 reasons.append(r("exif_software_match", f"Software field explicitly references an editing/generation tool: '{software_val}'", value=software_val)) else: reasons.append(r("exif_software_present", f"Software field present: '{software_val}'", value=software_val)) if has_make and has_datetime: reasons.append(r("exif_complete", "EXIF data is relatively complete (device + timestamp) — consistent with an original camera photo")) gps_present = False try: exif_dict = piexif.load(pil_image.info.get("exif", b"")) gps_data = exif_dict.get("GPS", {}) gps_present = len(gps_data) > 0 except Exception: pass if gps_present: suspicion = max(suspicion - 15, 0) reasons.append(r("exif_gps_present", "GPS location data found in EXIF — a strong signal of an original, unedited camera/phone photo (GPS tags are almost always stripped by editing or AI-generation tools)")) else: reasons.append(r("exif_gps_absent", "No GPS data found — inconclusive on its own (many legitimate photos have location services disabled)")) return min(suspicion, 100), reasons, matched_software def check_ai_model(pil_image): """ Runs the trained ensemble AI-image detector (aidetector, with automatic fallback to a simpler backend if the full "ultra" ensemble can't fully download). Returns (score, reasons, available) — available=False means every backend in the fallback chain failed to load, so callers should skip this category entirely rather than showing a misleading 0%. """ ai_prob = ai_detector.classify_image(pil_image) if ai_prob is None: return 0, [], False backend = ai_detector.get_active_backend() or "unknown" reasons = [r("ai_model_score", f"Trained model ({backend} backend) estimates a {ai_prob:.0f}% probability this image is AI-generated", value=f"{ai_prob:.0f}", backend=backend)] return ai_prob, reasons, True def check_efficientnetb3(pil_image): """ Fourth trained model: EfficientNet-B3, fine-tuned specifically for deepfake detection on a 399k-image dataset. Ships with real, validated test-set metrics embedded in its checkpoint (93.7% accuracy, 0.988 ROC-AUC) — currently our strongest single model. Returns (score, reasons, available). """ prob = efficientnetb3_detector.classify_image(pil_image) if prob is None: return 0, [], False acc = efficientnetb3_detector.get_reported_accuracy() acc_note = f", model's own validated test accuracy: {acc*100:.1f}%" if acc else "" reasons = [r("effnb3_score", f"EfficientNet-B3 deepfake detector estimates a {prob:.0f}% probability this image is fake{acc_note}", value=f"{prob:.0f}", acc=f"{acc*100:.1f}" if acc else "")] return prob, reasons, True def check_noise_residual(pil_image): """ PRNU-inspired noise-pattern heuristic (2026-08). Tested standalone at 63.0% balanced accuracy on 200 labeled samples — the strongest single signal in this project so far. No model, no download, always available. See noise_residual_detector.py for the method. """ suspicion = noise_residual_detector.classify_image(pil_image) if suspicion is None: return 0, [], False reasons = [r("noise_residual_score", f"Noise-pattern analysis estimates a {suspicion:.0f}% likelihood of an unnatural (AI-smoothed or synthetic) noise signature", value=f"{suspicion:.0f}")] return suspicion, reasons, True def check_clone_detection(pil_image): """ INFORMATIONAL ONLY (2026-08) — not standalone-tested first (unlike noise_residual, which was validated via test_noise_residual.py before integration), so not in the weighted blend yet. Detects copy-pasted regions within the same image (classic splicing signature) via ORB keypoint self-matching + displacement clustering. Test via bulk_calibrate.py + recalibrate.py before adding a weight. """ suspicion = clone_detection_module.classify_image(pil_image) if suspicion is None: return 0, [], False reasons = [r("clone_detection_score", f"Copy-move analysis estimates a {suspicion:.0f}% likelihood of a duplicated/pasted region within the image", value=f"{suspicion:.0f}")] return suspicion, reasons, True def check_insightface(pil_image): """ AUXILIARY/INFORMATIONAL ONLY (2026-08) — per the original plan's explicit note, InsightFace is a supporting feature, not a fake/real decider. Landmark symmetry + detection confidence heuristic, not a validated classifier. Not standalone-tested — test via bulk_calibrate.py + recalibrate.py before considering any weight. """ result = insightface_detector.analyze(pil_image) if result is None or not result.get("face_found"): return None, [], False suspicion = result["naturalness_suspicion"] reasons = [r("insightface_naturalness", f"Face landmark/detection analysis (InsightFace) estimates a {suspicion:.0f}% 'unnatural face' suspicion " f"(detection confidence: {result['det_score']:.2f}) — a weak, indirect heuristic, not a validated classifier", value=f"{suspicion:.0f}")] return suspicion, reasons, True def check_nonescape(pil_image): """ ACTIVE BLEND SIGNAL (2026-08) — DINOv2-large + EfficientNetV2-L cross-attention model (aediliclabs/nonescape, Apache 2.0). Runs via subprocess in an isolated venv (see nonescape_bridge.py) since it requires transformers==4.53.2, conflicting with this project's 4.46.3. Promoted to the active blend at a MODERATE weight based on consistent AUC=0.74-0.78 across two independent tests (n=299 standalone, n=203 in a merged 3-signal test) — stronger evidence than clone_detection ever had, but the sample is still smaller than effnb3/community_forensics' full history, so weighted cautiously rather than aggressively until more data accumulates. """ suspicion = nonescape_bridge.classify_image(pil_image) if suspicion is None: return 0, [], False reasons = [r("nonescape_score", f"Nonescape (DINOv2+EfficientNet, Apache 2.0) estimates a {suspicion:.0f}% probability this image is AI-generated", value=f"{suspicion:.0f}")] return suspicion, reasons, True def check_steganographia(pil_image): """ INFORMATIONAL/SECONDARY ONLY (2026-08) — ViT-Base trained on the Defactify dataset (SD2.1/XL/3, DALL-E 3, Midjourney v6). Deliberately kept secondary per its own model card's disclosed limitations: no guaranteed generalization to Flux/future generators, ~86% recall on AI images (~14% miss rate), 224x224 downscaling loses detail. Not standalone-tested on this project's data yet — test via bulk_calibrate.py + recalibrate.py before considering any weight. """ suspicion = steganographia_detector.classify_image(pil_image) if suspicion is None: return 0, [], False reasons = [r("steganographia_score", f"SteganographIA (Defactify-trained ViT) estimates a {suspicion:.0f}% probability this image is AI-generated " f"— secondary signal only, not guaranteed to generalize beyond its training generators", value=f"{suspicion:.0f}")] return suspicion, reasons, True def check_c2pa(image_bytes): """ DIRECT EVIDENCE, not a model guess. Reads a C2PA (Content Credentials) manifest if present. Presence proves provenance claims (who/what made it, recorded edit actions); absence proves NOTHING — most images don't have this data. See c2pa_detector.py's docstring. """ result = c2pa_detector.check(image_bytes, mime_type="image/jpeg") if not result.get("found"): return None, [], False reasons = [] if result["valid"] is False: reasons.append(r("c2pa_invalid", f"C2PA manifest found but signature validation FAILED — the file may have been altered after signing ({result['raw_summary']})", summary=result["raw_summary"])) else: reasons.append(r("c2pa_found", f"C2PA content credentials found: {result['raw_summary']}", summary=result["raw_summary"])) return result, reasons, True def check_community_forensics(pil_image): """ INFORMATIONAL ONLY (2026-08) — CVPR 2025 model trained on 2.7M images from 4803 different generators, designed for broad generalization across many AI tools rather than one specific generator family. Not standalone-tested yet — test via bulk_calibrate.py + recalibrate.py before adding a weight, same process as every other signal in this project. """ suspicion = community_forensics_detector.classify_image(pil_image) if suspicion is None: return 0, [], False reasons = [r("community_forensics_score", f"Community Forensics (trained on 4803 different AI generators) estimates a {suspicion:.0f}% probability this image is AI-generated", value=f"{suspicion:.0f}")] return suspicion, reasons, True def check_pixel_forensics(pil_image): """ INFORMATIONAL ONLY (2026-08) — classic OpenCV pixel forensics (currently just resize/resampling trace detection; double-JPEG and block-boundary sub-checks are disabled, see pixel_forensics_detector.py for why). Not standalone-tested at scale — same lesson as clone_detection_module.py (looked good at n=94, dropped to near- chance at n=2154). Test via bulk_calibrate.py + recalibrate.py before trusting or adding a weight. """ result = pixel_forensics_detector.classify_image(pil_image) if result is None: return 0, [], False score = result["combined_score"] reasons = [r("pixel_forensics_resize", f"Resize/resampling trace analysis estimates a {score:.0f}% likelihood of interpolation artifacts (image was resized/rescaled)", value=f"{score:.0f}")] return score, reasons, True def check_sdxl_face(pil_image): """ INFORMATIONAL ONLY (2026-08) — not yet in the weighted blend. A ViT trained specifically on SDXL/diffusion-generated faces (unlike the removed stylegan_hybrid_detector, which was StyleGAN2-specific and anti-correlated on this app's mostly-diffusion data). Test standalone accuracy via recalibrate.py before adding a weight for this. Returns (score, reasons, available). """ arr_rgb = np.array(pil_image.convert("RGB")) arr_bgr = cv2.cvtColor(arr_rgb, cv2.COLOR_RGB2BGR) faces = face_detector.detect_faces(arr_bgr) crop = pil_image face_found = False if len(faces) > 0: x, y, w, h = faces[0] pad = int(0.15 * max(w, h)) x0, y0 = max(0, x - pad), max(0, y - pad) x1, y1 = min(arr_rgb.shape[1], x + w + pad), min(arr_rgb.shape[0], y + h + pad) crop = pil_image.crop((x0, y0, x1, y1)) face_found = True suspicion = sdxl_face_detector.classify_image(crop) if suspicion is None: return 0, [], False note = "" if face_found else " (no face detected — ran on the whole image, less reliable)" reasons = [r("sdxl_face_score", f"SDXL/diffusion-focused face model estimates a {suspicion:.0f}% probability this face is AI-generated{note}", value=f"{suspicion:.0f}")] return suspicion, reasons, True def check_face_deepfake(pil_image): """ Second, complementary trained model specialized for FACE deepfakes (face-swap/reenactment), as opposed to check_ai_model()'s whole-image AI-generation focus. Crops the largest detected face (falls back to the full image if no face is found) and runs it through a dedicated ViT deepfake-face classifier. Returns (score, reasons, available). """ arr_rgb = np.array(pil_image.convert("RGB")) arr_bgr = cv2.cvtColor(arr_rgb, cv2.COLOR_RGB2BGR) faces = face_detector.detect_faces(arr_bgr) crop = pil_image face_found = False if len(faces) > 0: # Use the largest detected face (already sorted), with a small margin x, y, w, h = faces[0] pad = int(0.15 * max(w, h)) x0, y0 = max(0, x - pad), max(0, y - pad) x1, y1 = min(arr_rgb.shape[1], x + w + pad), min(arr_rgb.shape[0], y + h + pad) crop = pil_image.crop((x0, y0, x1, y1)) face_found = True prob = deepfake_face_detector.classify_face(crop) if prob is None: return 0, [], False if face_found: reasons = [r("face_deepfake_score", f"Face-specialized model estimates a {prob:.0f}% probability this face is a deepfake/face-swap", value=f"{prob:.0f}")] else: reasons = [r("face_deepfake_no_face", f"No clear face detected — ran the face-deepfake model on the whole image instead (estimate: {prob:.0f}%, less reliable without a cropped face)", value=f"{prob:.0f}")] return prob, reasons, True def analyze_image_bytes(image_bytes): pil_image = Image.open(io.BytesIO(image_bytes)) exif_score, exif_reasons, exif_matched_software = check_exif(pil_image) # DIRECT EVIDENCE check — some AI tools embed generation parameters # straight into the file (see generation_metadata_detector.py's # docstring). When present, this is proof, not a guess. gen_meta = generation_metadata_detector.check(pil_image, raw_bytes=image_bytes) model_score, model_reasons, model_available = check_ai_model(pil_image) face_df_score, face_df_reasons, face_df_available = check_face_deepfake(pil_image) effnb3_score, effnb3_reasons, effnb3_available = check_efficientnetb3(pil_image) # INFORMATIONAL ONLY — not in base_weights below yet. Test standalone # accuracy tomorrow via recalibrate.py before adding a weight. sdxl_score, sdxl_reasons, sdxl_available = check_sdxl_face(pil_image) noise_score, noise_reasons, noise_available = check_noise_residual(pil_image) # INFORMATIONAL ONLY — not in base_weights below yet. Test via # bulk_calibrate.py + recalibrate.py before adding a weight. clone_score, clone_reasons, clone_available = check_clone_detection(pil_image) pixel_score, pixel_reasons, pixel_available = check_pixel_forensics(pil_image) comm_score, comm_reasons, comm_available = check_community_forensics(pil_image) c2pa_result, c2pa_reasons, c2pa_available = check_c2pa(image_bytes) steg_score, steg_reasons, steg_available = check_steganographia(pil_image) nonescape_score, nonescape_reasons, nonescape_available = check_nonescape(pil_image) insightface_score, insightface_reasons, insightface_available = check_insightface(pil_image) # INFORMATIONAL ONLY — see lighting_balance_detector.py's docstring: # a similar region-consistency approach was tried and removed before # (AUC=0.501). Descriptive only until tested via recalibrate.py. light_score, light_reason_raw, light_available = lighting_balance_detector.analyze_lighting_balance(pil_image) light_reasons = [r(light_reason_raw["key"], light_reason_raw["text"], **light_reason_raw["params"])] if light_reason_raw else [] # Weights recalibrated 2026-08 from a 10472-image labeled test (real # AI-generated images + CASIA-style splicing + subtle "edited" # AI-restoration images). ELA, Dual-Stream, Copy-Move DCT, and the # Noise/FFT heuristic ("ai") were all REMOVED entirely from the # pipeline after measuring at-or-below-random accuracy specifically on # the "edited" (subtle AI enhancement) category across multiple # independent test sets — the Noise/FFT check in particular measured # 0% on both "fake" and "edited" across two separate datasets (its # max possible suspicion score was capped at 45/100 by design, so it # could never even flag an obvious fake with confidence). Two # pixel-level segmentation models (Unet++ and DINOv2) were also tried # and REMOVED after measuring ~5% accuracy on "edited"/splicing across # three independent tests — they failed even on the exact task they # were built for. EfficientNet-B3 and the general AI Detector remain # the strongest validated signals (~63-64% each, and ~80-99% on # fake/edited specifically, though with a real-image false-positive # tendency); EXIF metadata (often stripped/regenerated by AI tools) # is a genuinely useful signal too. # Weights recalibrated 2026-08 from an 11572-image labeled test (the # largest so far) via recalibrate.py's weight search. This is a # meaningful jump: the previous hand-picked weights scored only 55.2% # on this log; this combination scored 69.7%. Interesting finding: # EfficientNet-B3 is individually the strongest signal (63.3% # standalone) and got the highest weight as expected, but EXIF (only # 55.2% standalone, weaker than "model"'s 61.2%) still got a bigger # weight than "model" — the search found EXIF+EfficientNet-B3 combine # better together than "model" does, even though "model" alone beats # EXIF alone. Re-run recalibrate.py periodically as more calibration # data accumulates to keep this current. # Weights recalibrated 2026-08 from 2347 samples AFTER cleaning out # the contaminated real_data (wikiart/ghibli-art/processed_ images # mislabeled as "real") and replacing with verified new-data. First # trustworthy recalibration in this project's history — everything # before this used contaminated "real" samples. Balanced accuracy # 55.6% (still modest — face_df/effnb3 remain the strongest signals, # exif still contributes ~nothing, model is weak on its own too). # "noise_residual" added 2026-08 — strongest standalone signal tested # (63.0% balanced accuracy vs. effnb3's 57.4%), but this in-app weight # is a provisional guess (0.20, taken proportionally from the others). # Re-run recalibrate.py after collecting more data with this category # present and let the search find its real weight. # Weights updated 2026-08 from 3143-sample recalibration. # clone_detection PROMOTED to the active blend (2026-08): validated # standalone accuracy grew 60.6% (n=35) -> 62.1% (n=94) as more data # came in — confirms it's a real signal, not sampling noise, so it's # no longer informational-only. # nonescape added 2026-08 at weight 0.40 (raised from 0.35) — see # check_nonescape()'s docstring for the evidence behind promoting # it to the active blend; AUC 0.74-0.78 is the strongest tested # standalone evidence of any signal in this project. base_weights = IMAGE_WEIGHTS available_scores = {"exif": exif_score} if model_available: available_scores["model"] = model_score if face_df_available: available_scores["face_df"] = face_df_score if effnb3_available: available_scores["effnb3"] = effnb3_score if noise_available: available_scores["noise_residual"] = noise_score if clone_available: available_scores["clone_detection"] = clone_score if nonescape_available: available_scores["nonescape"] = nonescape_score weight_sum = sum(base_weights[k] for k in available_scores) final_score = sum(available_scores[k] * base_weights[k] for k in available_scores) / weight_sum # Confidence floor: if any SINGLE trained model is highly confident this # is fake, don't let the weighted average fully dilute that down just # because other signals disagree — a strong individual red flag from a # validated model shouldn't be washed out to "likely real". This does # NOT override the blend when models agree it's real; it only raises # the floor when at least one strong "fake" signal exists. model_keys = ["model", "face_df", "effnb3", "noise_residual", "nonescape"] trained_model_scores = [available_scores[k] for k in model_keys if k in available_scores] if trained_model_scores: strongest_fake_signal = max(trained_model_scores) if strongest_fake_signal >= 65: final_score = max(final_score, strongest_fake_signal * 0.6) authenticity = 100 - final_score # Thresholds shifted down 2026-08 alongside the weight recalibration # above. recalibrate.py's search found threshold=20 as the accuracy- # optimal SINGLE binary cutoff (real vs. fake-or-edited) for the new # weights — that's closest to what "medium" (any suspicion) means # here, so medium moved 40->20. "high" wasn't independently searched # (recalibrate.py only tests one binary boundary, not this app's # three-tier split) — kept the old medium/high gap (20 points) as a # reasonable estimate: 60->40. Re-run recalibrate.py after more # labeled data accumulates and sanity-check this gap specifically if # you get a chance — it's an extrapolation, not a directly tested value. # Thresholds updated 2026-08 alongside the weight recalibration above # (first trustworthy one — see base_weights comment). recalibrate.py # found threshold=50 as the balanced-accuracy-optimal binary cutoff on # cleaned data — moved medium up from 20->50. "high" kept the same # 20-point gap as before (medium+20): 40->70. Not independently # searched — same caveat as last time, re-verify once more real data # accumulates. if final_score >= 70: verdict = "This image is likely edited or AI-generated" verdict_level = "high" verdict_key = "image_high" elif final_score >= 50: verdict = "Moderate signals of possible editing — inconclusive" verdict_level = "medium" verdict_key = "image_medium" else: verdict = "This image is likely authentic / unedited" verdict_level = "low" verdict_key = "image_low" # Sharpen the verdict wording using the face/background consistency # check (2026-08): a big mismatch between the face and the rest of # the image means the manipulation looks LOCALIZED (e.g. a swapped/ # retouched face on an otherwise-real photo) rather than the whole # frame being AI-generated — say "edited" specifically instead of # the generic "edited or AI-generated". Conversely, a suspicious # score with a CONSISTENT face/background (low mismatch) points # toward full-image generation instead. # # Priority order (2026-08 revision): # 0. C2PA invalid signature — a FAILED cryptographic signature check # is the strongest possible tamper evidence (the file changed # after signing), so it outranks even gen_meta. A VALID C2PA # signature naming an AI generator also confirms generation. # 1. Embedded generation metadata (gen_meta) — DEFINITIVE, overrides # everything else when present, since it's direct evidence from # the file rather than any model's estimate. # # 2026-08 NOTE: the "localized edit vs fully generated" wording tier # that used to sit here (based on region_consistency_detector's # face/background noise mismatch) was REMOVED after # recalibrate_v7.py showed region_consistency scores AUC=0.501 on # 698 samples — statistically indistinguishable from a coin flip. # That verdict-sharpening logic was built on noise, not signal — # region_consistency_detector.py has been deleted from the project. if c2pa_available and c2pa_result["valid"] is False: final_score = max(final_score, 90) verdict = f"This image has a C2PA signature that FAILED validation — the file was altered after being signed ({c2pa_result['raw_summary']})" verdict_level = "high" verdict_key = "image_c2pa_invalid" elif c2pa_available and c2pa_result["valid"] and any( term in (c2pa_result["generator"] or "").lower() for term in ("dall", "chatgpt", "midjourney", "stable diffusion", "firefly", "gemini", "imagen") ): final_score = max(final_score, 95) verdict = f"This image has a valid C2PA signature naming {c2pa_result['generator']} as the generator — confirmed AI-generated" verdict_level = "high" verdict_key = "image_c2pa_confirmed_generated" elif gen_meta["found"]: final_score = max(final_score, 95) verdict = f"This image contains embedded generation metadata identifying it as made with {gen_meta['tool']} — confirmed AI-generated, not a guess" verdict_level = "high" verdict_key = "image_confirmed_generated" categories = [ {"key": "cat_exif", "name": "EXIF Metadata Inspection", "score": round(exif_score), "reasons": exif_reasons}, ] if model_available: backend = ai_detector.get_active_backend() or "model" categories.append({"key": "cat_ai_model", "name": f"AI Detector — {backend} (CLIP/HF ensemble)", "score": round(model_score), "reasons": model_reasons}) if face_df_available: categories.append({"key": "cat_face_deepfake", "name": "Face Deepfake Detector (ViT)", "score": round(face_df_score), "reasons": face_df_reasons}) if effnb3_available: categories.append({"key": "cat_effnb3", "name": "EfficientNet-B3 Deepfake Detector", "score": round(effnb3_score), "reasons": effnb3_reasons}) if sdxl_available: categories.append({"key": "cat_sdxl_face", "name": "SDXL/Diffusion Face Detector (ViT)", "score": round(sdxl_score), "reasons": sdxl_reasons}) if noise_available: categories.append({"key": "cat_noise_residual", "name": "Noise Residual Analysis", "score": round(noise_score), "reasons": noise_reasons}) if clone_available: categories.append({"key": "cat_clone_detection", "name": "Clone/Copy-Move Detection", "score": round(clone_score), "reasons": clone_reasons}) if pixel_available: categories.append({"key": "cat_pixel_forensics", "name": "Pixel Forensics (Resize Trace)", "score": round(pixel_score), "reasons": pixel_reasons}) if comm_available: categories.append({"key": "cat_community_forensics", "name": "Community Forensics (4803 generators)", "score": round(comm_score), "reasons": comm_reasons}) if c2pa_available: c2pa_display_score = 100 if c2pa_result["valid"] is False else 0 categories.append({"key": "cat_c2pa", "name": "C2PA Content Credentials", "score": c2pa_display_score, "reasons": c2pa_reasons}) if steg_available: categories.append({"key": "cat_steganographia", "name": "SteganographIA (Defactify ViT)", "score": round(steg_score), "reasons": steg_reasons}) if nonescape_available: categories.append({"key": "cat_nonescape", "name": "Nonescape (DINOv2+EfficientNet)", "score": round(nonescape_score), "reasons": nonescape_reasons}) if insightface_available: categories.append({"key": "cat_insightface", "name": "Face Naturalness (InsightFace)", "score": round(insightface_score), "reasons": insightface_reasons}) if light_available: categories.append({"key": "cat_lighting_balance", "name": "Lighting Balance (informational)", "score": round(light_score), "reasons": light_reasons}) if gen_meta["found"]: categories.append({ "key": "cat_generation_metadata", "name": "Embedded Generation Metadata", "score": 100, "reasons": [r("gen_metadata_found", f"Found embedded generation parameters identifying this as made with: {gen_meta['tool']} — " f"this is direct evidence from the file itself, not a model guess", tool=gen_meta["tool"])], }) return { "type": "image", "suspicion_score": round(final_score), "authenticity_score": round(authenticity), "verdict": verdict, "verdict_key": verdict_key, "verdict_level": verdict_level, "categories": categories, } # ============================================================ # Video analysis # ============================================================ def check_rppg(tmp_path): """ Runs the vendored BioVerify rPPG engine: looks for a physiologically plausible heartbeat signal in facial skin color micro-variations. Synthetic/deepfake video generally lacks a real, coherent pulse. Returns (score 0-100 = suspicion, reasons, available). """ try: from bioverify_engine import analyze_video as _rppg_analyze, Config as _RppgConfig, Verdict as _RppgVerdict except Exception: return 0, [], False # bioverify_engine's own loggers reset themselves to INFO on import; # silence them here (after import) so our server console stays clean. for _name in ("bioverify_engine.ingest", "bioverify_engine.face", "bioverify_engine.roi", "bioverify_engine.stabilization", "bioverify_engine.rppg", "bioverify_engine.quality", "bioverify_engine.features", "bioverify_engine.scoring"): logging.getLogger(_name).setLevel(logging.ERROR) try: result = _rppg_analyze(tmp_path, _RppgConfig()) except Exception: return 0, [], False if result.error is not None: return 0, [], False reasons = [] if result.verdict == _RppgVerdict.SYNTHETIC: suspicion = 70 + min(result.confidence * 30, 30) reasons.append(r("rppg_synthetic", f"No physiologically plausible heartbeat signal detected in facial skin tone (rPPG analysis, confidence={result.confidence:.2f}) — real video of a person almost always shows a faint pulse-driven color signal; synthetic/composited faces typically don't", value=f"{result.confidence:.2f}")) elif result.verdict == _RppgVerdict.HUMAN: suspicion = max(30 - result.confidence * 30, 0) reasons.append(r("rppg_human", f"A physiologically plausible heartbeat signal was detected in facial skin tone (rPPG analysis, confidence={result.confidence:.2f}) — a strong signal of a real, live-recorded face", value=f"{result.confidence:.2f}")) else: suspicion = 20 reasons.append(r("rppg_inconclusive", "Pulse-signal analysis was inconclusive (video too short, face too small/unstable, or poor lighting) — not usable as a signal for this video")) return min(suspicion, 100), reasons, True def analyze_video_bytes(video_bytes, tmp_path): # Stage 1 (roadmap): container metadata scan — cheap, doesn't need the # file on disk or decoding, so run it first on the raw bytes. meta_score, meta_reasons_raw, meta_available = video_metadata_detector.scan_video_metadata(video_bytes) meta_reasons = [r(item["key"], item["text"], **item["params"]) for item in meta_reasons_raw] with open(tmp_path, "wb") as f: f.write(video_bytes) wav_path = tmp_path + ".wav" has_audio = audio_extractor.extract_audio(tmp_path, wav_path) if not has_audio: wav_path = None cap = cv2.VideoCapture(tmp_path) reasons_motion = [] reasons_face = [] suspicion = 0 frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) fps = cap.get(cv2.CAP_PROP_FPS) if not fps or fps <= 0: fps = 25.0 if frame_count <= 1: cap.release() _safe_remove(tmp_path) return { "type": "video", "suspicion_score": 0, "authenticity_score": 0, "verdict": "Could not read the video, or it is empty", "verdict_key": "video_unreadable", "verdict_level": "medium", "categories": [], } sample_every = max(1, int(frame_count / 40)) # Stage 3c (roadmap): blink detection needs denser, evenly-timed # sampling (~3 frames/sec) independent of the sparser motion/face # cadence above, so it's tracked separately within the same pass. blink_sample_every = max(1, int(round(fps / 3))) max_blink_frames = 150 blink_frames = [] prev_gray = None frame_diffs = [] # values only frame_diffs_idx = [] # matching frame index, for temporal localization face_edge_scores = [] face_crops_bgr = [] # keep a few real face crops to run the trained model on idx = 0 while True: ret, frame = cap.read() if not ret: break if idx % blink_sample_every == 0 and len(blink_frames) < max_blink_frames: blink_frames.append(frame.copy()) if idx % sample_every == 0: gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) if prev_gray is not None and prev_gray.shape == gray.shape: diff = cv2.absdiff(gray, prev_gray) frame_diffs.append(float(np.mean(diff))) frame_diffs_idx.append(idx) prev_gray = gray faces = face_detector.detect_faces(frame) for (x, y, w, h) in faces: face_roi = gray[y:y + h, x:x + w] if face_roi.size > 0: edges = cv2.Laplacian(face_roi, cv2.CV_64F).var() face_edge_scores.append(edges) if len(face_crops_bgr) < 8 and w > 40 and h > 40: face_crops_bgr.append(frame[y:y + h, x:x + w].copy()) idx += 1 cap.release() # Run the rPPG (pulse-signal) check while the file still exists on disk rppg_score, rppg_reasons, rppg_available = check_rppg(tmp_path) _safe_remove(tmp_path) # Stage 3c (roadmap): blink-rate check on the densely-sampled frames # collected during the main pass above. samples_per_second = fps / blink_sample_every if blink_sample_every else 0 blink_score, blink_reasons_raw, blink_available = blink_detector.analyze_blinks(blink_frames, samples_per_second) blink_reasons = [r(item["key"], item["text"], **item["params"]) for item in blink_reasons_raw] # Audio-based checks (added 2026-08): both need the extracted WAV, # which is None if the video had no audio track — in that case both # simply report unavailable rather than erroring. voice_score, voice_reason_raw, voice_available = voice_deepfake_detector.classify_audio(wav_path) if wav_path else (0, None, False) voice_reasons = [r(voice_reason_raw["key"], voice_reason_raw["text"], **voice_reason_raw["params"])] if voice_reason_raw else [] lipsync_score, lipsync_reason_raw, lipsync_available = lipsync_detector.analyze_lipsync(blink_frames, samples_per_second, wav_path) lipsync_reasons = [r(lipsync_reason_raw["key"], lipsync_reason_raw["text"], **lipsync_reason_raw["params"])] if lipsync_reason_raw else [] if wav_path and os.path.exists(wav_path): _safe_remove(wav_path) # 1) Inter-frame motion consistency if len(frame_diffs) >= 3: diffs = np.array(frame_diffs) motion_variance = float(np.var(diffs)) motion_mean = float(np.mean(diffs)) spike_ratio = float(np.sum(diffs > (motion_mean + 2 * np.std(diffs))) / len(diffs)) if np.std(diffs) > 0 else 0 if spike_ratio > 0.15: suspicion += 30 reasons_motion.append(r("vid_motion_spikes", f"Frequent sudden jumps in motion between frames (spike ratio={spike_ratio:.2f}) — may indicate cut/spliced clips or frame swapping", value=f"{spike_ratio:.2f}")) # Temporal localization: point at roughly *when* the single # biggest jump happened, not just that jumps exist somewhere. peak_i = int(np.argmax(diffs)) peak_frame_idx = frame_diffs_idx[peak_i] peak_seconds = peak_frame_idx / fps reasons_motion.append(r("vid_motion_peak_location", f"The single largest motion jump is at approximately {peak_seconds:.1f}s into the video — worth reviewing that moment specifically", seconds=f"{peak_seconds:.1f}")) else: reasons_motion.append(r("vid_motion_smooth", f"Motion between frames is smooth and natural (spike ratio={spike_ratio:.2f})", value=f"{spike_ratio:.2f}")) reasons_motion.append(r("vid_motion_stats", f"Average frame-to-frame change: {motion_mean:.1f}, variance: {motion_variance:.1f}", mean=f"{motion_mean:.1f}", variance=f"{motion_variance:.1f}")) else: reasons_motion.append(r("vid_motion_insufficient", "Too few analyzable frames for a reliable motion assessment")) # 2) Face-region sharpness consistency if face_edge_scores: face_scores = np.array(face_edge_scores) face_variance = float(np.var(face_scores)) face_mean = float(np.mean(face_scores)) is_inconsistent = face_mean > 0 and face_variance > (face_mean ** 1.5) if is_inconsistent: suspicion += 25 reasons_face.append(r("face_inconsistent", f"Large variance in facial detail sharpness across frames (variance={face_variance:.1f}) — a possible sign of a swapped/composited face (deepfake)", value="35")) else: reasons_face.append(r("face_consistent", f"Facial detail sharpness is consistent across frames (variance={face_variance:.1f}) — consistent with an uncomposited video", value="10")) reasons_face.append(r("face_detected_count", f"Faces detected in {len(face_edge_scores)} sampled frames", count=str(len(face_edge_scores)))) else: reasons_face.append(r("face_none", "No clear faces detected in the sampled frames — face-based analysis is unavailable for this video")) # 3) Trained AI-detector model applied to sampled face crops model_reasons = [] model_score = 0 model_available = False if face_crops_bgr: probs = [] for crop_bgr in face_crops_bgr: crop_rgb = cv2.cvtColor(crop_bgr, cv2.COLOR_BGR2RGB) pil_crop = Image.fromarray(crop_rgb) p = ai_detector.classify_image(pil_crop) if p is not None: probs.append(p) if probs: model_available = True model_score = float(np.mean(probs)) model_reasons.append(r("ai_model_video_score", f"Trained model estimates an average {model_score:.0f}% probability of AI-generated/synthetic faces across {len(probs)} sampled face crops", value=f"{model_score:.0f}", count=str(len(probs)))) # 3b) Second, face-specialized deepfake model applied to the same sampled face crops face_df_reasons = [] face_df_score = 0 face_df_available = False if face_crops_bgr: probs2 = [] for crop_bgr in face_crops_bgr: crop_rgb = cv2.cvtColor(crop_bgr, cv2.COLOR_BGR2RGB) pil_crop = Image.fromarray(crop_rgb) p = deepfake_face_detector.classify_face(pil_crop) if p is not None: probs2.append(p) if probs2: face_df_available = True face_df_score = float(np.mean(probs2)) face_df_reasons.append(r("face_deepfake_video_score", f"Face-specialized deepfake model estimates an average {face_df_score:.0f}% probability across {len(probs2)} sampled face crops", value=f"{face_df_score:.0f}", count=str(len(probs2)))) # 3c) Nonescape, noise-residual fingerprint, and resize-trace forensics # — these are per-frame image detectors (added 2026-08), reused here on # the same sampled face crops rather than duplicated. Picked # specifically because they're the ones with real, documented # standalone evidence (see nonescape_bridge.py / noise_residual_detector.py # docstrings) — EfficientNet-B3 and clone-detection were deliberately # left out of the video pipeline: EfficientNet-B3 only hit 57.4% # balanced accuracy in this project's own testing (near coin-flip), # and clone-detection was never standalone-tested at all. nonescape_reasons, nonescape_score, nonescape_available = [], 0, False noise_reasons, noise_score, noise_available = [], 0, False pixel_reasons, pixel_score, pixel_available = [], 0, False if face_crops_bgr: ne_probs, noise_probs, pixel_probs = [], [], [] for crop_bgr in face_crops_bgr: crop_rgb = cv2.cvtColor(crop_bgr, cv2.COLOR_BGR2RGB) pil_crop = Image.fromarray(crop_rgb) p_ne = nonescape_bridge.classify_image(pil_crop) if p_ne is not None: ne_probs.append(p_ne) p_noise = noise_residual_detector.classify_image(pil_crop) if p_noise is not None: noise_probs.append(p_noise) p_pixel = pixel_forensics_detector.classify_image(pil_crop) if p_pixel is not None: pixel_probs.append(p_pixel["combined_score"]) if ne_probs: nonescape_available = True nonescape_score = float(np.mean(ne_probs)) nonescape_reasons.append(r("nonescape_video_score", f"Nonescape model estimates an average {nonescape_score:.0f}% probability across {len(ne_probs)} sampled face crops", value=f"{nonescape_score:.0f}", count=str(len(ne_probs)))) if noise_probs: noise_available = True noise_score = float(np.mean(noise_probs)) noise_reasons.append(r("noise_residual_video_score", f"Noise fingerprint analysis estimates an average {noise_score:.0f}% probability of an unnatural noise pattern across {len(noise_probs)} sampled face crops", value=f"{noise_score:.0f}", count=str(len(noise_probs)))) if pixel_probs: pixel_available = True pixel_score = float(np.mean(pixel_probs)) pixel_reasons.append(r("pixel_forensics_video_score", f"Resize-trace analysis estimates an average {pixel_score:.0f}% probability of interpolation/upscaling artifacts across {len(pixel_probs)} sampled face crops", value=f"{pixel_score:.0f}", count=str(len(pixel_probs)))) # Blend all available signals, weighted by how much each is trusted: # rPPG (biology-based) and the two trained models carry more weight # than the hand-tuned motion/face heuristics. Weights are normalized # by whichever signals actually loaded, so a missing model doesn't # just get silently dropped — the remaining signals fill in its share. # meta (container metadata) and blink (behavioral biometrics) added # 2026-08 per the video-pipeline roadmap — both at a moderate weight # since each is real evidence but narrower/noisier than the trained # models or rPPG (meta only fires strongly when a named app is found; # blink rate is affected by clip length and camera angle too). base_weights = VIDEO_WEIGHTS available_scores = {"heuristic": min(suspicion, 100)} if model_available: available_scores["model"] = model_score if face_df_available: available_scores["face_df"] = face_df_score if rppg_available: available_scores["rppg"] = rppg_score if meta_available: available_scores["meta"] = meta_score if blink_available: available_scores["blink"] = blink_score if nonescape_available: available_scores["nonescape"] = nonescape_score if noise_available: available_scores["noise"] = noise_score if pixel_available: available_scores["pixel"] = pixel_score if voice_available: available_scores["voice"] = voice_score if lipsync_available: available_scores["lipsync"] = lipsync_score weight_sum = sum(base_weights[k] for k in available_scores) final_score = sum(available_scores[k] * base_weights[k] for k in available_scores) / weight_sum final_score = min(final_score, 100) authenticity = 100 - final_score if final_score >= 55: verdict = "This video shows strong signals of being composited or manipulated (possible deepfake)" verdict_level = "high" verdict_key = "video_high" elif final_score >= 25: verdict = "Moderate signals — inconclusive, further review recommended" verdict_level = "medium" verdict_key = "video_medium" else: verdict = "This video is likely authentic / uncomposited" verdict_level = "low" verdict_key = "video_low" categories = [ {"key": "cat_motion", "name": "Optical Flow — Frame Motion Analysis", "score": min(round(suspicion if reasons_motion else 0), 100), "reasons": reasons_motion}, {"key": "cat_face", "name": "Laplacian Edge — Face Sharpness Analysis", "score": 0, "reasons": reasons_face}, ] if model_available: backend = ai_detector.get_active_backend() or "model" categories.append({"key": "cat_ai_model", "name": f"AI Detector — {backend} (CLIP/HF ensemble)", "score": round(model_score), "reasons": model_reasons}) if face_df_available: categories.append({"key": "cat_face_deepfake", "name": "Face Deepfake Detector (ViT)", "score": round(face_df_score), "reasons": face_df_reasons}) if rppg_available: categories.append({"key": "cat_rppg", "name": "BioVerify rPPG — Pulse Signal Analysis", "score": round(rppg_score), "reasons": rppg_reasons}) if meta_available: categories.append({"key": "cat_vid_meta", "name": "Container Metadata — Editing App Detection", "score": round(meta_score), "reasons": meta_reasons}) if blink_available: categories.append({"key": "cat_blink", "name": "Blink Rate — Behavioral Biometrics (MediaPipe)", "score": round(blink_score), "reasons": blink_reasons}) if nonescape_available: categories.append({"key": "cat_nonescape", "name": "Nonescape (DINOv2+EfficientNet)", "score": round(nonescape_score), "reasons": nonescape_reasons}) if noise_available: categories.append({"key": "cat_noise_residual", "name": "Noise Residual Analysis", "score": round(noise_score), "reasons": noise_reasons}) if pixel_available: categories.append({"key": "cat_pixel_forensics", "name": "Pixel Forensics (Resize Trace)", "score": round(pixel_score), "reasons": pixel_reasons}) if voice_available: categories.append({"key": "cat_voice_deepfake", "name": "Voice Deepfake Detector", "score": round(voice_score), "reasons": voice_reasons}) if lipsync_available: categories.append({"key": "cat_lipsync", "name": "Lip-Sync Consistency", "score": round(lipsync_score), "reasons": lipsync_reasons}) return { "type": "video", "suspicion_score": round(final_score), "authenticity_score": round(authenticity), "verdict": verdict, "verdict_key": verdict_key, "verdict_level": verdict_level, "categories": categories, }