Spaces:
Running on Zero
Running on Zero
Download scripts/test_convnext_adapter.py from Thiru1122/Multimodel_setup: direct link, hf CLI and curl.
- Browser
- Download file 14.3 kB
-
https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/test_convnext_adapter.py
- Command line
-
hf download hf://spaces/Thiru1122/Multimodel_setup/scripts/test_convnext_adapter.py
-
curl -L -o test_convnext_adapter.py https://huggingface.co/spaces/Thiru1122/Multimodel_setup/resolve/main/scripts/test_convnext_adapter.py
14.3 kB
| """ | |
| Diagnostic Test Suite for ConvNeXtDetectorAdapter. | |
| Tests adapter integration, lazy loading, detector isolation, valid-image | |
| inference, invalid-input handling, and probability consistency. | |
| This is an operational test suite, not an accuracy evaluation. | |
| """ | |
| import os | |
| import sys | |
| import json | |
| import traceback | |
| BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| if BASE_DIR not in sys.path: | |
| sys.path.insert(0, BASE_DIR) | |
| from src.convnext_adapter import ConvNeXtDetectorAdapter | |
| from src.detector_schema import ( | |
| PREDICTION_REAL, | |
| PREDICTION_FAKE, | |
| PREDICTION_ERROR, | |
| ) | |
| ASSET_DIR = os.path.join(BASE_DIR, "benchmark", "test_images") | |
| RESULTS_FILE = os.path.join( | |
| BASE_DIR, "scripts", "convnext_adapter_test_results.json" | |
| ) | |
| def safe_prediction(adapter, image_path): | |
| """Run prediction and return a structured error if an exception escapes.""" | |
| try: | |
| result = adapter.predict(image_path) | |
| return result, None | |
| except Exception as exc: | |
| return None, f"{type(exc).__name__}: {exc}" | |
| def record_check(results, name, passed, detail=""): | |
| """Record a diagnostic check without hiding failures.""" | |
| status = "PASS" if passed else "FAIL" | |
| results[name] = { | |
| "status": status, | |
| "detail": detail, | |
| } | |
| print(f"[{status}] {name}" + (f": {detail}" if detail else "")) | |
| def result_to_dict(result, exception=None): | |
| if exception: | |
| return { | |
| "prediction": PREDICTION_ERROR, | |
| "error": exception, | |
| } | |
| return result.to_dict() | |
| def main(): | |
| print("=" * 80) | |
| print("CHEN2024 CONVNEXT DETECTOR ADAPTER DIAGNOSTIC TEST") | |
| print("Operational checks only; predictions do not establish accuracy.") | |
| print(f"Project directory: {BASE_DIR}") | |
| print(f"Test images: {ASSET_DIR}") | |
| print("=" * 80) | |
| os.makedirs(ASSET_DIR, exist_ok=True) | |
| os.makedirs(os.path.dirname(RESULTS_FILE), exist_ok=True) | |
| required_images = [ | |
| "real.jpg", | |
| "ai_generated.jpg", | |
| "manipulated.jpg", | |
| ] | |
| missing = [ | |
| name for name in required_images | |
| if not os.path.isfile(os.path.join(ASSET_DIR, name)) | |
| ] | |
| if missing: | |
| print("\nSETUP FAILED. Missing test images:") | |
| for name in missing: | |
| print(f" - {os.path.join(ASSET_DIR, name)}") | |
| print("\nCopy the three images into benchmark\\test_images and rerun.") | |
| sys.exit(1) | |
| corrupt_path = os.path.join(ASSET_DIR, "corrupted_image.png") | |
| unsupported_path = os.path.join(ASSET_DIR, "_temp_convnext_unsupported.txt") | |
| missing_path = os.path.join( | |
| BASE_DIR, "models", "non_existent_image_12345.png" | |
| ) | |
| # Create invalid-input fixtures. | |
| with open(corrupt_path, "wb") as file: | |
| file.write(bytes([0, 1, 2, 3, 4, 5, 6, 7])) | |
| with open(unsupported_path, "w", encoding="utf-8") as file: | |
| file.write("Plain text file, not an image.") | |
| image_paths = { | |
| "real": os.path.join(ASSET_DIR, "real.jpg"), | |
| "ai_generated": os.path.join(ASSET_DIR, "ai_generated.jpg"), | |
| "manipulated": os.path.join(ASSET_DIR, "manipulated.jpg"), | |
| } | |
| results = { | |
| "metadata": { | |
| "test_type": "operational_diagnostics", | |
| "model": "chen2024_convnext", | |
| "accuracy_evaluation": False, | |
| "test_images": image_paths, | |
| }, | |
| "checks": {}, | |
| "inferences": {}, | |
| } | |
| adapter = None | |
| try: | |
| # Test 11: lazy loading. | |
| adapter = ConvNeXtDetectorAdapter(lazy_load=True) | |
| lazy_ok = getattr(adapter, "model", None) is None | |
| record_check( | |
| results["checks"], | |
| "test_11_lazy_loading", | |
| lazy_ok, | |
| "Model is not initialized before first prediction." | |
| if lazy_ok else | |
| "Model was already initialized in the constructor.", | |
| ) | |
| # Test 12: check imports/instances without assuming absent variables | |
| # are proof of isolation. | |
| univfd_module = sys.modules.get("src.univfd_adapter") | |
| vit_module = sys.modules.get("src.vit_adapter") | |
| univfd_instances = ( | |
| getattr(univfd_module, "_INSTANCES", None) | |
| if univfd_module is not None else None | |
| ) | |
| vit_instances = ( | |
| getattr(vit_module, "_INSTANCES", None) | |
| if vit_module is not None else None | |
| ) | |
| isolation_ok = ( | |
| not univfd_instances and not vit_instances | |
| ) | |
| record_check( | |
| results["checks"], | |
| "test_12_detector_isolation", | |
| isolation_ok, | |
| "No UnivFD/ViT instances found at this point." | |
| if isolation_ok else | |
| "Other detector instances appear to exist; inspect module state.", | |
| ) | |
| # Tests 1-3: valid image inference. | |
| valid_results = {} | |
| valid_cases = [ | |
| ("test_01_valid_real_image", "real"), | |
| ("test_02_valid_ai_image", "ai_generated"), | |
| ("test_03_valid_manipulated_image", "manipulated"), | |
| ] | |
| for check_name, label in valid_cases: | |
| path = image_paths[label] | |
| print(f"\n{check_name}: {path}") | |
| result, exception = safe_prediction(adapter, path) | |
| results["inferences"][label] = result_to_dict(result, exception) | |
| valid = ( | |
| result is not None | |
| and result.prediction in (PREDICTION_REAL, PREDICTION_FAKE) | |
| and result.error is None | |
| ) | |
| record_check( | |
| results["checks"], | |
| check_name, | |
| valid, | |
| ( | |
| f"Prediction={result.prediction}, " | |
| f"fake_probability={result.fake_probability}, " | |
| f"real_probability={result.real_probability}, " | |
| f"device={result.device}, " | |
| f"latency_ms={result.inference_time_ms}" | |
| ) if result is not None else exception, | |
| ) | |
| if result is not None: | |
| valid_results[label] = result | |
| # Verify on-demand loading after first prediction. | |
| model_loaded = getattr(adapter, "model", None) is not None | |
| record_check( | |
| results["checks"], | |
| "test_model_loaded_after_prediction", | |
| model_loaded, | |
| "Model is initialized." if model_loaded | |
| else "Model attribute is still None after predictions.", | |
| ) | |
| # Test 4: corrupted file rejection. | |
| corrupt_result, corrupt_exception = safe_prediction( | |
| adapter, corrupt_path | |
| ) | |
| results["inferences"]["corrupted"] = result_to_dict( | |
| corrupt_result, corrupt_exception | |
| ) | |
| corrupt_ok = ( | |
| corrupt_result is not None | |
| and corrupt_result.prediction == PREDICTION_ERROR | |
| and corrupt_result.error is not None | |
| ) | |
| record_check( | |
| results["checks"], | |
| "test_04_corrupted_image", | |
| corrupt_ok, | |
| ( | |
| f"Prediction={corrupt_result.prediction}, " | |
| f"error={corrupt_result.error}" | |
| ) if corrupt_result is not None else corrupt_exception, | |
| ) | |
| # Test 5: unsupported file rejection. | |
| unsupported_result, unsupported_exception = safe_prediction( | |
| adapter, unsupported_path | |
| ) | |
| results["inferences"]["unsupported_text"] = result_to_dict( | |
| unsupported_result, unsupported_exception | |
| ) | |
| unsupported_ok = ( | |
| unsupported_result is not None | |
| and unsupported_result.prediction == PREDICTION_ERROR | |
| and unsupported_result.error is not None | |
| ) | |
| record_check( | |
| results["checks"], | |
| "test_05_unsupported_file", | |
| unsupported_ok, | |
| ( | |
| f"Prediction={unsupported_result.prediction}, " | |
| f"error={unsupported_result.error}" | |
| ) if unsupported_result is not None else unsupported_exception, | |
| ) | |
| # Test 6: missing path rejection. | |
| missing_result, missing_exception = safe_prediction( | |
| adapter, missing_path | |
| ) | |
| results["inferences"]["missing_path"] = result_to_dict( | |
| missing_result, missing_exception | |
| ) | |
| missing_ok = ( | |
| missing_result is not None | |
| and missing_result.prediction == PREDICTION_ERROR | |
| and missing_result.error is not None | |
| and "does not exist" in missing_result.error.lower() | |
| ) | |
| record_check( | |
| results["checks"], | |
| "test_06_missing_path", | |
| missing_ok, | |
| ( | |
| f"Prediction={missing_result.prediction}, " | |
| f"error={missing_result.error}" | |
| ) if missing_result is not None else missing_exception, | |
| ) | |
| # Test 7: None input rejection. | |
| none_result, none_exception = safe_prediction(adapter, None) | |
| results["inferences"]["none_input"] = result_to_dict( | |
| none_result, none_exception | |
| ) | |
| none_ok = ( | |
| none_result is not None | |
| and none_result.prediction == PREDICTION_ERROR | |
| and none_result.error is not None | |
| ) | |
| record_check( | |
| results["checks"], | |
| "test_07_none_input", | |
| none_ok, | |
| ( | |
| f"Prediction={none_result.prediction}, " | |
| f"error={none_result.error}" | |
| ) if none_result is not None else none_exception, | |
| ) | |
| # Test 8: probability bounds. | |
| bounds_ok = True | |
| bounds_details = [] | |
| for label, result in valid_results.items(): | |
| values = { | |
| "fake_probability": result.fake_probability, | |
| "real_probability": result.real_probability, | |
| "confidence": result.confidence, | |
| } | |
| for field, value in values.items(): | |
| if ( | |
| value is None | |
| or not isinstance(value, (int, float)) | |
| or not 0.0 <= value <= 1.0 | |
| ): | |
| bounds_ok = False | |
| bounds_details.append(f"{label}.{field}={value}") | |
| if len(valid_results) != 3: | |
| bounds_ok = False | |
| bounds_details.append("Not all three images produced results.") | |
| record_check( | |
| results["checks"], | |
| "test_08_probability_bounds", | |
| bounds_ok, | |
| "; ".join(bounds_details) if bounds_details | |
| else "All valid-result probabilities are within [0, 1].", | |
| ) | |
| # Test 9: probability sum close to 1. | |
| sum_ok = len(valid_results) == 3 | |
| sum_details = [] | |
| for label, result in valid_results.items(): | |
| if ( | |
| result.fake_probability is None | |
| or result.real_probability is None | |
| ): | |
| sum_ok = False | |
| sum_details.append(f"{label}: missing probability") | |
| continue | |
| probability_sum = ( | |
| result.fake_probability + result.real_probability | |
| ) | |
| if abs(probability_sum - 1.0) >= 1e-4: | |
| sum_ok = False | |
| sum_details.append(f"{label}: sum={probability_sum}") | |
| record_check( | |
| results["checks"], | |
| "test_09_probability_sum", | |
| sum_ok, | |
| "; ".join(sum_details) if sum_details | |
| else "Each valid result's probabilities sum to approximately 1.", | |
| ) | |
| # Test 10: confidence equals the probability of the predicted class. | |
| confidence_ok = len(valid_results) == 3 | |
| confidence_details = [] | |
| for label, result in valid_results.items(): | |
| if result.confidence is None: | |
| confidence_ok = False | |
| confidence_details.append(f"{label}: confidence missing") | |
| continue | |
| if result.prediction == PREDICTION_FAKE: | |
| expected_confidence = result.fake_probability | |
| elif result.prediction == PREDICTION_REAL: | |
| expected_confidence = result.real_probability | |
| else: | |
| confidence_ok = False | |
| confidence_details.append( | |
| f"{label}: unexpected prediction {result.prediction}" | |
| ) | |
| continue | |
| if ( | |
| expected_confidence is None | |
| or abs(result.confidence - expected_confidence) >= 1e-6 | |
| or result.confidence < 0.5 | |
| ): | |
| confidence_ok = False | |
| confidence_details.append( | |
| f"{label}: confidence={result.confidence}, " | |
| f"expected={expected_confidence}" | |
| ) | |
| record_check( | |
| results["checks"], | |
| "test_10_confidence_alignment", | |
| confidence_ok, | |
| "; ".join(confidence_details) if confidence_details | |
| else "Confidence matches the predicted class probability.", | |
| ) | |
| except Exception as exc: | |
| results["fatal_error"] = { | |
| "error": f"{type(exc).__name__}: {exc}", | |
| "traceback": traceback.format_exc(), | |
| } | |
| print(f"\nUNEXPECTED TEST ERROR: {type(exc).__name__}: {exc}") | |
| traceback.print_exc() | |
| finally: | |
| for path in (unsupported_path,): | |
| try: | |
| if os.path.exists(path): | |
| os.remove(path) | |
| except OSError as exc: | |
| print(f"Warning: could not remove {path}: {exc}") | |
| with open(RESULTS_FILE, "w", encoding="utf-8") as file: | |
| json.dump(results, file, indent=2) | |
| checks = results["checks"] | |
| passed = sum( | |
| 1 for item in checks.values() if item.get("status") == "PASS" | |
| ) | |
| failed = sum( | |
| 1 for item in checks.values() if item.get("status") == "FAIL" | |
| ) | |
| print("\n" + "=" * 80) | |
| print(f"DIAGNOSTIC SUMMARY: {passed} passed, {failed} failed") | |
| print(f"Results saved to: {RESULTS_FILE}") | |
| print("Passing contract checks does not establish detector accuracy.") | |
| print("=" * 80) | |
| if failed or "fatal_error" in results: | |
| sys.exit(1) | |
| if __name__ == "__main__": | |
| main() | |