Spaces:
Running on Zero
Running on Zero
File size: 6,144 Bytes
7591a5b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | """
Test Suite for UnivFD Benchmark Framework
Validates metadata loading, resilience to missing files, corrupted file rejection,
unsupported format handling, DetectorResult contract adherence, and probability bounds.
"""
import os
import sys
import json
import tempfile
from PIL import Image
import torch
# Ensure project root is in sys.path
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if BASE_DIR not in sys.path:
sys.path.insert(0, BASE_DIR)
from src.univfd_adapter import UnivFDAdapter
from src.detector_schema import (
DetectorResult,
PREDICTION_REAL,
PREDICTION_FAKE,
PREDICTION_ERROR,
VALID_PREDICTIONS,
)
METADATA_PATH = os.path.join(BASE_DIR, "benchmark", "metadata.json")
def test_metadata_loading():
print("[TEST 1] Verifying benchmark/metadata.json structure...")
assert os.path.exists(METADATA_PATH), f"Metadata file missing at {METADATA_PATH}"
with open(METADATA_PATH, "r", encoding="utf-8") as f:
data = json.load(f)
assert "entries" in data, "Missing 'entries' key in metadata.json"
entries = data["entries"]
assert len(entries) > 0, "Metadata contains zero entries"
for e in entries:
assert "id" in e, f"Entry missing 'id': {e}"
assert "filename" in e, f"Entry missing 'filename': {e}"
assert "category" in e, f"Entry missing 'category': {e}"
assert "ground_truth" in e, f"Entry missing 'ground_truth': {e}"
print(f" PASS: Successfully loaded {len(entries)} valid metadata entries.\n")
def test_missing_files_handling():
print("[TEST 2] Verifying resilience to missing files...")
adapter = UnivFDAdapter(lazy_load=True)
non_existent_path = os.path.join(BASE_DIR, "benchmark", "images", "non_existent_image.png")
result = adapter.predict(non_existent_path)
assert isinstance(result, DetectorResult), "Return value must be DetectorResult"
assert result.prediction == PREDICTION_ERROR, f"Expected ERROR for missing file, got: {result.prediction}"
assert result.error is not None, "Error message must be present for missing file"
assert "does not exist" in result.error or "Invalid" in result.error
print(" PASS: Missing file correctly reported as ERROR without crashing.\n")
def test_valid_image_inference():
print("[TEST 3] Verifying valid image inference & DetectorResult schema...")
adapter = UnivFDAdapter(lazy_load=True)
# Create a small valid test PIL image
test_img = Image.new("RGB", (256, 256), color=(120, 150, 180))
result = adapter.predict(test_img)
assert isinstance(result, DetectorResult), "Return value must be DetectorResult"
assert result.prediction in {PREDICTION_REAL, PREDICTION_FAKE}, f"Unexpected prediction: {result.prediction}"
assert result.error is None, f"Valid image produced unexpected error: {result.error}"
# Verify probability bounds [0.0, 1.0]
assert result.fake_probability is not None, "fake_probability must not be None"
assert 0.0 <= result.fake_probability <= 1.0, f"fake_probability out of bounds: {result.fake_probability}"
assert result.real_probability is not None, "real_probability must not be None"
assert 0.0 <= result.real_probability <= 1.0, f"real_probability out of bounds: {result.real_probability}"
prob_sum = result.fake_probability + result.real_probability
assert abs(prob_sum - 1.0) < 1e-4, f"Real and Fake probabilities must sum to 1.0 (got {prob_sum})"
assert result.confidence is not None, "confidence must not be None"
assert 0.0 <= result.confidence <= 1.0, f"confidence out of bounds: {result.confidence}"
assert result.threshold == 0.50, f"threshold must be 0.50, got {result.threshold}"
assert result.raw_logit is not None, "raw_logit must not be None"
assert result.inference_time_ms is not None and result.inference_time_ms >= 0, "Invalid latency"
print(f" PASS: Valid image successfully inferred (Prediction: {result.prediction}, Latency: {result.inference_time_ms:.2f} ms).\n")
def test_corrupted_file_rejection():
print("[TEST 4] Verifying corrupted file rejection...")
adapter = UnivFDAdapter(lazy_load=True)
with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp:
tmp.write(b"CORRUPTED_TRUNCATED_HEADER_TEST")
tmp_path = tmp.name
try:
result = adapter.predict(tmp_path)
assert isinstance(result, DetectorResult), "Return value must be DetectorResult"
assert result.prediction == PREDICTION_ERROR, f"Corrupted file must return ERROR, got: {result.prediction}"
assert result.error is not None, "Error message must be present for corrupted file"
print(f" PASS: Corrupted file cleanly rejected with ERROR ('{result.error}').\n")
finally:
if os.path.exists(tmp_path):
os.remove(tmp_path)
def test_unsupported_file_rejection():
print("[TEST 5] Verifying unsupported format rejection...")
adapter = UnivFDAdapter(lazy_load=True)
with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:
tmp.write(b"This is a plain text document, not an image format.")
tmp_path = tmp.name
try:
result = adapter.predict(tmp_path)
assert isinstance(result, DetectorResult), "Return value must be DetectorResult"
assert result.prediction == PREDICTION_ERROR, f"Unsupported file must return ERROR, got: {result.prediction}"
assert result.error is not None, "Error message must be present for unsupported file"
print(f" PASS: Unsupported text format cleanly rejected with ERROR.\n")
finally:
if os.path.exists(tmp_path):
os.remove(tmp_path)
def run_all_tests():
print("=" * 75)
print("RUNNING UNIVFD BENCHMARK TEST SUITE")
print("=" * 75 + "\n")
test_metadata_loading()
test_missing_files_handling()
test_valid_image_inference()
test_corrupted_file_rejection()
test_unsupported_file_rejection()
print("=" * 75)
print("ALL 5 BENCHMARK CONTRACT TESTS PASSED SUCCESSFULLY.")
print("=" * 75)
if __name__ == "__main__":
run_all_tests()
|