Spaces:
Running on Zero
Running on Zero
File size: 7,003 Bytes
61e55d1 151f388 61e55d1 151f388 61e55d1 151f388 61e55d1 151f388 61e55d1 151f388 61e55d1 151f388 61e55d1 151f388 61e55d1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 | """
Detector Contract Verification Suite
Tests that both UnivFDAdapter and ViTAdapter adhere to the identical DetectorResult contract:
- Common interface: .predict(image_input) -> DetectorResult
- All required fields present and of correct types
- Probabilities bounded in [0.0, 1.0] and sum to 1.0
- Confidence matches the predicted class
- Proper error propagation and rejection for corrupted/invalid inputs
- Detector independence (one detector's failure does not impact the other)
"""
import os
import sys
import json
import math
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if BASE_DIR not in sys.path:
sys.path.insert(0, BASE_DIR)
from src.detector_schema import (
DetectorResult,
PREDICTION_REAL,
PREDICTION_FAKE,
PREDICTION_ERROR,
)
from src.univfd_adapter import UnivFDAdapter
from src.vit_adapter import ViTAdapter
from src.convnext_adapter import ConvNeXtDetectorAdapter
# Resolve asset paths within the project repository
ASSET_DIR = os.path.join(BASE_DIR, "benchmark", "test_images")
FALLBACK_DIR = os.path.join(BASE_DIR, "benchmark", "images")
def assert_contract(result: DetectorResult, test_name: str, expect_error: bool = False):
"""Rigorous contract checks against DetectorResult."""
assert isinstance(result, DetectorResult), f"{test_name}: Result is not instance of DetectorResult"
res_dict = result.to_dict()
# Required fields presence
required_keys = [
"detector_name", "model_name", "prediction", "fake_probability",
"real_probability", "raw_logit", "confidence", "threshold",
"device", "preprocessing", "inference_time_ms", "error", "warning"
]
for key in required_keys:
assert key in res_dict, f"{test_name}: Missing required field '{key}'"
if expect_error:
assert result.prediction == PREDICTION_ERROR, f"{test_name}: Expected prediction ERROR, got {result.prediction}"
assert result.error is not None and len(result.error) > 0, f"{test_name}: Error message must be populated"
assert result.fake_probability is None, f"{test_name}: fake_probability must be None on error"
assert result.real_probability is None, f"{test_name}: real_probability must be None on error"
else:
assert result.prediction in (PREDICTION_REAL, PREDICTION_FAKE), f"{test_name}: Invalid prediction '{result.prediction}'"
assert result.error is None, f"{test_name}: Error should be None on success, got: {result.error}"
assert result.fake_probability is not None, f"{test_name}: fake_probability is None"
assert result.real_probability is not None, f"{test_name}: real_probability is None"
assert 0.0 <= result.fake_probability <= 1.0, f"{test_name}: fake_probability out of bounds: {result.fake_probability}"
assert 0.0 <= result.real_probability <= 1.0, f"{test_name}: real_probability out of bounds: {result.real_probability}"
prob_sum = result.fake_probability + result.real_probability
assert math.isclose(prob_sum, 1.0, abs_tol=1e-3), f"{test_name}: Probabilities do not sum to 1.0: {prob_sum}"
assert result.confidence is not None, f"{test_name}: confidence is None"
assert 0.0 <= result.confidence <= 1.0, f"{test_name}: confidence out of bounds: {result.confidence}"
if result.prediction == PREDICTION_FAKE:
assert math.isclose(result.confidence, result.fake_probability, abs_tol=1e-5), f"{test_name}: Confidence does not match fake_prob"
else:
assert math.isclose(result.confidence, result.real_probability, abs_tol=1e-5), f"{test_name}: Confidence does not match real_prob"
assert result.threshold is not None and 0.0 <= result.threshold <= 1.0
assert result.inference_time_ms is not None and result.inference_time_ms >= 0.0
assert result.raw_logit is not None
def main():
print("=" * 75)
print("DETECTOR CONTRACT VERIFICATION SUITE")
print("Testing UnivFDAdapter and ViTAdapter adherence to DetectorResult contract.")
print("=" * 75 + "\n")
univfd = UnivFDAdapter(lazy_load=True)
vit = ViTAdapter(lazy_load=True)
convnext = ConvNeXtDetectorAdapter(lazy_load=True)
adapters = [
("UnivFDAdapter", univfd),
("ViTAdapter", vit),
("ConvNeXtDetectorAdapter", convnext),
]
# Locate test fixtures
real_img = os.path.join(ASSET_DIR, "real.jpg")
if not os.path.isfile(real_img):
real_img = os.path.join(FALLBACK_DIR, "real", "real_phone_photo.png")
corrupt_img = os.path.join(ASSET_DIR, "corrupted_image.png")
if not os.path.isfile(corrupt_img):
corrupt_img = os.path.join(FALLBACK_DIR, "corrupted", "corrupted_image.png")
nonexistent = os.path.join(ASSET_DIR, "nonexistent_image_123.jpg")
contract_results = []
for name, adapter in adapters:
print(f"Testing {name}...")
# 1. Valid Real Image
res_real = adapter.predict(real_img)
assert_contract(res_real, f"{name} - Valid Image", expect_error=False)
print(f" [PASS] Valid image inference: pred={res_real.prediction}, conf={res_real.confidence*100:.1f}%, logit={res_real.raw_logit:+.3f}")
# 2. Corrupted Image Rejection
res_corrupt = adapter.predict(corrupt_img)
assert_contract(res_corrupt, f"{name} - Corrupted Image", expect_error=True)
print(f" [PASS] Corrupted file rejection: pred={res_corrupt.prediction}, err={res_corrupt.error[:40]}...")
# 3. Missing File Handling
res_missing = adapter.predict(nonexistent)
assert_contract(res_missing, f"{name} - Missing File", expect_error=True)
print(f" [PASS] Missing file rejection: pred={res_missing.prediction}")
# 4. None Input Handling
res_none = adapter.predict(None)
assert_contract(res_none, f"{name} - None Input", expect_error=True)
print(f" [PASS] None input rejection: pred={res_none.prediction}")
# 5. Unload Model Verification
if hasattr(adapter, "unload_model"):
adapter.unload_model()
assert adapter.model is None, f"{name}: unload_model() did not set self.model to None"
print(f" [PASS] unload_model() verified: weights successfully released")
contract_results.append({
"adapter": name,
"contract_status": "COMPLIANT",
"schema_verified": True,
"error_handling_verified": True,
"unload_model_verified": hasattr(adapter, "unload_model"),
})
print()
print("=" * 75)
print("CONTRACT VERIFICATION SUMMARY: ALL ADAPTERS FULLY COMPLIANT")
print("=" * 75)
contract_summary_file = os.path.join(BASE_DIR, "scripts", "contract_test_summary.json")
with open(contract_summary_file, "w", encoding="utf-8") as f:
json.dump(contract_results, f, indent=2)
print(f"Saved contract summary to {contract_summary_file}")
if __name__ == "__main__":
main()
|