Spaces:
Running on Zero
Running on Zero
Download scripts/test_univfd_benchmark.py from Thiru1122/Multimodel_setup: direct link, hf CLI and curl.
- Browser
- Download file 6.14 kB
-
https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/test_univfd_benchmark.py
- Command line
-
hf download hf://spaces/Thiru1122/Multimodel_setup/scripts/test_univfd_benchmark.py
-
curl -L -o test_univfd_benchmark.py https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/test_univfd_benchmark.py
6.14 kB
| """ | |
| Test Suite for UnivFD Benchmark Framework | |
| Validates metadata loading, resilience to missing files, corrupted file rejection, | |
| unsupported format handling, DetectorResult contract adherence, and probability bounds. | |
| """ | |
| import os | |
| import sys | |
| import json | |
| import tempfile | |
| from PIL import Image | |
| import torch | |
| # Ensure project root is in sys.path | |
| BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| if BASE_DIR not in sys.path: | |
| sys.path.insert(0, BASE_DIR) | |
| from src.univfd_adapter import UnivFDAdapter | |
| from src.detector_schema import ( | |
| DetectorResult, | |
| PREDICTION_REAL, | |
| PREDICTION_FAKE, | |
| PREDICTION_ERROR, | |
| VALID_PREDICTIONS, | |
| ) | |
| METADATA_PATH = os.path.join(BASE_DIR, "benchmark", "metadata.json") | |
| def test_metadata_loading(): | |
| print("[TEST 1] Verifying benchmark/metadata.json structure...") | |
| assert os.path.exists(METADATA_PATH), f"Metadata file missing at {METADATA_PATH}" | |
| with open(METADATA_PATH, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| assert "entries" in data, "Missing 'entries' key in metadata.json" | |
| entries = data["entries"] | |
| assert len(entries) > 0, "Metadata contains zero entries" | |
| for e in entries: | |
| assert "id" in e, f"Entry missing 'id': {e}" | |
| assert "filename" in e, f"Entry missing 'filename': {e}" | |
| assert "category" in e, f"Entry missing 'category': {e}" | |
| assert "ground_truth" in e, f"Entry missing 'ground_truth': {e}" | |
| print(f" PASS: Successfully loaded {len(entries)} valid metadata entries.\n") | |
| def test_missing_files_handling(): | |
| print("[TEST 2] Verifying resilience to missing files...") | |
| adapter = UnivFDAdapter(lazy_load=True) | |
| non_existent_path = os.path.join(BASE_DIR, "benchmark", "images", "non_existent_image.png") | |
| result = adapter.predict(non_existent_path) | |
| assert isinstance(result, DetectorResult), "Return value must be DetectorResult" | |
| assert result.prediction == PREDICTION_ERROR, f"Expected ERROR for missing file, got: {result.prediction}" | |
| assert result.error is not None, "Error message must be present for missing file" | |
| assert "does not exist" in result.error or "Invalid" in result.error | |
| print(" PASS: Missing file correctly reported as ERROR without crashing.\n") | |
| def test_valid_image_inference(): | |
| print("[TEST 3] Verifying valid image inference & DetectorResult schema...") | |
| adapter = UnivFDAdapter(lazy_load=True) | |
| # Create a small valid test PIL image | |
| test_img = Image.new("RGB", (256, 256), color=(120, 150, 180)) | |
| result = adapter.predict(test_img) | |
| assert isinstance(result, DetectorResult), "Return value must be DetectorResult" | |
| assert result.prediction in {PREDICTION_REAL, PREDICTION_FAKE}, f"Unexpected prediction: {result.prediction}" | |
| assert result.error is None, f"Valid image produced unexpected error: {result.error}" | |
| # Verify probability bounds [0.0, 1.0] | |
| assert result.fake_probability is not None, "fake_probability must not be None" | |
| assert 0.0 <= result.fake_probability <= 1.0, f"fake_probability out of bounds: {result.fake_probability}" | |
| assert result.real_probability is not None, "real_probability must not be None" | |
| assert 0.0 <= result.real_probability <= 1.0, f"real_probability out of bounds: {result.real_probability}" | |
| prob_sum = result.fake_probability + result.real_probability | |
| assert abs(prob_sum - 1.0) < 1e-4, f"Real and Fake probabilities must sum to 1.0 (got {prob_sum})" | |
| assert result.confidence is not None, "confidence must not be None" | |
| assert 0.0 <= result.confidence <= 1.0, f"confidence out of bounds: {result.confidence}" | |
| assert result.threshold == 0.50, f"threshold must be 0.50, got {result.threshold}" | |
| assert result.raw_logit is not None, "raw_logit must not be None" | |
| assert result.inference_time_ms is not None and result.inference_time_ms >= 0, "Invalid latency" | |
| print(f" PASS: Valid image successfully inferred (Prediction: {result.prediction}, Latency: {result.inference_time_ms:.2f} ms).\n") | |
| def test_corrupted_file_rejection(): | |
| print("[TEST 4] Verifying corrupted file rejection...") | |
| adapter = UnivFDAdapter(lazy_load=True) | |
| with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp: | |
| tmp.write(b"CORRUPTED_TRUNCATED_HEADER_TEST") | |
| tmp_path = tmp.name | |
| try: | |
| result = adapter.predict(tmp_path) | |
| assert isinstance(result, DetectorResult), "Return value must be DetectorResult" | |
| assert result.prediction == PREDICTION_ERROR, f"Corrupted file must return ERROR, got: {result.prediction}" | |
| assert result.error is not None, "Error message must be present for corrupted file" | |
| print(f" PASS: Corrupted file cleanly rejected with ERROR ('{result.error}').\n") | |
| finally: | |
| if os.path.exists(tmp_path): | |
| os.remove(tmp_path) | |
| def test_unsupported_file_rejection(): | |
| print("[TEST 5] Verifying unsupported format rejection...") | |
| adapter = UnivFDAdapter(lazy_load=True) | |
| with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp: | |
| tmp.write(b"This is a plain text document, not an image format.") | |
| tmp_path = tmp.name | |
| try: | |
| result = adapter.predict(tmp_path) | |
| assert isinstance(result, DetectorResult), "Return value must be DetectorResult" | |
| assert result.prediction == PREDICTION_ERROR, f"Unsupported file must return ERROR, got: {result.prediction}" | |
| assert result.error is not None, "Error message must be present for unsupported file" | |
| print(f" PASS: Unsupported text format cleanly rejected with ERROR.\n") | |
| finally: | |
| if os.path.exists(tmp_path): | |
| os.remove(tmp_path) | |
| def run_all_tests(): | |
| print("=" * 75) | |
| print("RUNNING UNIVFD BENCHMARK TEST SUITE") | |
| print("=" * 75 + "\n") | |
| test_metadata_loading() | |
| test_missing_files_handling() | |
| test_valid_image_inference() | |
| test_corrupted_file_rejection() | |
| test_unsupported_file_rejection() | |
| print("=" * 75) | |
| print("ALL 5 BENCHMARK CONTRACT TESTS PASSED SUCCESSFULLY.") | |
| print("=" * 75) | |
| if __name__ == "__main__": | |
| run_all_tests() | |