Spaces:
Running on Zero
Running on Zero
| import pytest | |
| from pathlib import Path | |
| import numpy as np | |
| import nibabel as nib | |
| from cad_benchmark_space.validator import validate_submission | |
| def mock_gt(): | |
| return { | |
| "canonical_orientation": "RAS+", | |
| "cases": { | |
| "case001": { | |
| "labels": {"2c": 1, "2d": 0}, | |
| "shape": (32, 64, 64), | |
| "spacing": (1.5, 0.75, 0.75), | |
| "orientation": ("R", "A", "S"), | |
| "findings": { | |
| "case001_f0": {"class": "2c", "mask_path": "masks/case001_f0.nii.gz"} | |
| }, | |
| "gt_masks": {"2c": np.zeros((32, 64, 64), dtype=bool), "2d": np.zeros((32, 64, 64), dtype=bool)}, | |
| }, | |
| "case002": { | |
| "labels": {"2c": 0, "2d": 1}, | |
| "shape": (32, 64, 64), | |
| "spacing": (1.5, 0.75, 0.75), | |
| "orientation": ("R", "A", "S"), | |
| "findings": { | |
| "case002_f0": {"class": "2d", "mask_path": "masks/case002_f0.nii.gz"} | |
| }, | |
| "gt_masks": {"2c": np.zeros((32, 64, 64), dtype=bool), "2d": np.zeros((32, 64, 64), dtype=bool)}, | |
| }, | |
| }, | |
| } | |
| def test_reject_invalid_model_type(mock_gt): | |
| manifest = {"model_name": "TestModel", "model_type": "invalid_type"} | |
| res = validate_submission(manifest, mock_gt) | |
| assert not res.is_valid | |
| assert any("model_type" in err for err in res.errors) | |
| def test_classification_reject_floats(mock_gt): | |
| manifest = { | |
| "model_name": "TestModel", | |
| "model_type": "classification", | |
| "predictions": [ | |
| {"scan_id": "case001", "2c": 0.85, "2d": 0} | |
| ], | |
| } | |
| res = validate_submission(manifest, mock_gt) | |
| assert not res.is_valid | |
| assert any("must be int 0 or 1" in err for err in res.errors) | |
| def test_classification_reject_unknown_scan_id(mock_gt): | |
| manifest = { | |
| "model_name": "TestModel", | |
| "model_type": "classification", | |
| "predictions": [ | |
| {"scan_id": "case999", "2c": 1, "2d": 0} | |
| ], | |
| } | |
| res = validate_submission(manifest, mock_gt) | |
| assert not res.is_valid | |
| assert any("case999" in err for err in res.errors) | |
| def test_classification_reject_duplicates(mock_gt): | |
| manifest = { | |
| "model_name": "TestModel", | |
| "model_type": "classification", | |
| "predictions": [ | |
| {"scan_id": "case001", "2c": 1, "2d": 0}, | |
| {"scan_id": "case001", "2c": 0, "2d": 1}, | |
| ], | |
| } | |
| res = validate_submission(manifest, mock_gt) | |
| assert not res.is_valid | |
| assert any("duplicate scan_id" in err for err in res.errors) | |
| def test_classification_valid(mock_gt): | |
| manifest = { | |
| "model_name": "TestModel", | |
| "model_type": "classification", | |
| "predictions": [ | |
| {"scan_id": "case001", "2c": 1, "2d": 0}, | |
| {"scan_id": "case002", "2c": 0, "2d": 1}, | |
| ], | |
| } | |
| res = validate_submission(manifest, mock_gt) | |
| assert res.is_valid | |
| assert len(res.classification_samples) == 2 | |
| def test_segmentation_validation(mock_gt, tmp_path): | |
| shape = (32, 64, 64) | |
| spacing = (1.5, 0.75, 0.75) | |
| affine = np.diag([spacing[0], spacing[1], spacing[2], 1.0]) | |
| mask_data = np.zeros(shape, dtype=np.uint8) | |
| mask_data[10:15, 20:30, 20:30] = 1 | |
| mask_file = tmp_path / "case001_f0.nii.gz" | |
| nii = nib.Nifti1Image(mask_data, affine) | |
| nib.save(nii, str(mask_file)) | |
| manifest = { | |
| "model_name": "SegModel", | |
| "model_type": "segmentation", | |
| "predictions": [ | |
| { | |
| "scan_id": "case001", | |
| "finding_id": "case001_f0", | |
| "class": "2c", | |
| "mask_path": "case001_f0.nii.gz", | |
| "is_soft_mask": False, | |
| } | |
| ], | |
| } | |
| res = validate_submission(manifest, mock_gt, files_dir=tmp_path) | |
| assert res.is_valid | |
| assert len(res.localization_samples) == 1 | |
| def test_segmentation_reject_shape_mismatch(mock_gt, tmp_path): | |
| wrong_shape = (20, 64, 64) | |
| spacing = (1.5, 0.75, 0.75) | |
| affine = np.diag([spacing[0], spacing[1], spacing[2], 1.0]) | |
| mask_data = np.zeros(wrong_shape, dtype=np.uint8) | |
| mask_file = tmp_path / "case001_wrong.nii.gz" | |
| nii = nib.Nifti1Image(mask_data, affine) | |
| nib.save(nii, str(mask_file)) | |
| manifest = { | |
| "model_name": "SegModel", | |
| "model_type": "segmentation", | |
| "predictions": [ | |
| { | |
| "scan_id": "case001", | |
| "finding_id": "case001_f0", | |
| "class": "2c", | |
| "mask_path": "case001_wrong.nii.gz", | |
| } | |
| ], | |
| } | |
| res = validate_submission(manifest, mock_gt, files_dir=tmp_path) | |
| assert not res.is_valid | |
| assert any("mask shape" in err for err in res.errors) | |