File size: 6,144 Bytes
7591a5b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
"""
Test Suite for UnivFD Benchmark Framework
Validates metadata loading, resilience to missing files, corrupted file rejection,
unsupported format handling, DetectorResult contract adherence, and probability bounds.
"""

import os
import sys
import json
import tempfile
from PIL import Image
import torch

# Ensure project root is in sys.path
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if BASE_DIR not in sys.path:
    sys.path.insert(0, BASE_DIR)

from src.univfd_adapter import UnivFDAdapter
from src.detector_schema import (
    DetectorResult,
    PREDICTION_REAL,
    PREDICTION_FAKE,
    PREDICTION_ERROR,
    VALID_PREDICTIONS,
)

METADATA_PATH = os.path.join(BASE_DIR, "benchmark", "metadata.json")


def test_metadata_loading():
    print("[TEST 1] Verifying benchmark/metadata.json structure...")
    assert os.path.exists(METADATA_PATH), f"Metadata file missing at {METADATA_PATH}"
    with open(METADATA_PATH, "r", encoding="utf-8") as f:
        data = json.load(f)

    assert "entries" in data, "Missing 'entries' key in metadata.json"
    entries = data["entries"]
    assert len(entries) > 0, "Metadata contains zero entries"

    for e in entries:
        assert "id" in e, f"Entry missing 'id': {e}"
        assert "filename" in e, f"Entry missing 'filename': {e}"
        assert "category" in e, f"Entry missing 'category': {e}"
        assert "ground_truth" in e, f"Entry missing 'ground_truth': {e}"
    print(f"  PASS: Successfully loaded {len(entries)} valid metadata entries.\n")


def test_missing_files_handling():
    print("[TEST 2] Verifying resilience to missing files...")
    adapter = UnivFDAdapter(lazy_load=True)
    non_existent_path = os.path.join(BASE_DIR, "benchmark", "images", "non_existent_image.png")
    
    result = adapter.predict(non_existent_path)
    assert isinstance(result, DetectorResult), "Return value must be DetectorResult"
    assert result.prediction == PREDICTION_ERROR, f"Expected ERROR for missing file, got: {result.prediction}"
    assert result.error is not None, "Error message must be present for missing file"
    assert "does not exist" in result.error or "Invalid" in result.error
    print("  PASS: Missing file correctly reported as ERROR without crashing.\n")


def test_valid_image_inference():
    print("[TEST 3] Verifying valid image inference & DetectorResult schema...")
    adapter = UnivFDAdapter(lazy_load=True)
    
    # Create a small valid test PIL image
    test_img = Image.new("RGB", (256, 256), color=(120, 150, 180))
    result = adapter.predict(test_img)

    assert isinstance(result, DetectorResult), "Return value must be DetectorResult"
    assert result.prediction in {PREDICTION_REAL, PREDICTION_FAKE}, f"Unexpected prediction: {result.prediction}"
    assert result.error is None, f"Valid image produced unexpected error: {result.error}"

    # Verify probability bounds [0.0, 1.0]
    assert result.fake_probability is not None, "fake_probability must not be None"
    assert 0.0 <= result.fake_probability <= 1.0, f"fake_probability out of bounds: {result.fake_probability}"

    assert result.real_probability is not None, "real_probability must not be None"
    assert 0.0 <= result.real_probability <= 1.0, f"real_probability out of bounds: {result.real_probability}"

    prob_sum = result.fake_probability + result.real_probability
    assert abs(prob_sum - 1.0) < 1e-4, f"Real and Fake probabilities must sum to 1.0 (got {prob_sum})"

    assert result.confidence is not None, "confidence must not be None"
    assert 0.0 <= result.confidence <= 1.0, f"confidence out of bounds: {result.confidence}"
    assert result.threshold == 0.50, f"threshold must be 0.50, got {result.threshold}"
    assert result.raw_logit is not None, "raw_logit must not be None"
    assert result.inference_time_ms is not None and result.inference_time_ms >= 0, "Invalid latency"

    print(f"  PASS: Valid image successfully inferred (Prediction: {result.prediction}, Latency: {result.inference_time_ms:.2f} ms).\n")


def test_corrupted_file_rejection():
    print("[TEST 4] Verifying corrupted file rejection...")
    adapter = UnivFDAdapter(lazy_load=True)

    with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp:
        tmp.write(b"CORRUPTED_TRUNCATED_HEADER_TEST")
        tmp_path = tmp.name

    try:
        result = adapter.predict(tmp_path)
        assert isinstance(result, DetectorResult), "Return value must be DetectorResult"
        assert result.prediction == PREDICTION_ERROR, f"Corrupted file must return ERROR, got: {result.prediction}"
        assert result.error is not None, "Error message must be present for corrupted file"
        print(f"  PASS: Corrupted file cleanly rejected with ERROR ('{result.error}').\n")
    finally:
        if os.path.exists(tmp_path):
            os.remove(tmp_path)


def test_unsupported_file_rejection():
    print("[TEST 5] Verifying unsupported format rejection...")
    adapter = UnivFDAdapter(lazy_load=True)

    with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:
        tmp.write(b"This is a plain text document, not an image format.")
        tmp_path = tmp.name

    try:
        result = adapter.predict(tmp_path)
        assert isinstance(result, DetectorResult), "Return value must be DetectorResult"
        assert result.prediction == PREDICTION_ERROR, f"Unsupported file must return ERROR, got: {result.prediction}"
        assert result.error is not None, "Error message must be present for unsupported file"
        print(f"  PASS: Unsupported text format cleanly rejected with ERROR.\n")
    finally:
        if os.path.exists(tmp_path):
            os.remove(tmp_path)


def run_all_tests():
    print("=" * 75)
    print("RUNNING UNIVFD BENCHMARK TEST SUITE")
    print("=" * 75 + "\n")

    test_metadata_loading()
    test_missing_files_handling()
    test_valid_image_inference()
    test_corrupted_file_rejection()
    test_unsupported_file_rejection()

    print("=" * 75)
    print("ALL 5 BENCHMARK CONTRACT TESTS PASSED SUCCESSFULLY.")
    print("=" * 75)


if __name__ == "__main__":
    run_all_tests()