Buckets:
| import importlib.util | |
| import unittest | |
| from collections import Counter | |
| from pathlib import Path | |
| class pipeline_contract(unittest.TestCase): | |
| def test_preparation_module_exists(self): | |
| self.assertTrue(Path("prepare.py").is_file(), "dataset preparation is not implemented") | |
| def load(self): | |
| if not Path("prepare.py").is_file(): | |
| self.skipTest("implementation absent") | |
| spec = importlib.util.spec_from_file_location("prepare", "prepare.py") | |
| module = importlib.util.module_from_spec(spec) | |
| spec.loader.exec_module(module) | |
| return module | |
| def test_windows_cover_native_edges_without_tiny_extra_tiles(self): | |
| module = self.load() | |
| self.assertEqual(module.starts(1024), [0]) | |
| self.assertEqual(module.starts(800), [0]) | |
| self.assertEqual(module.starts(2048), [0, 819, 1024]) | |
| with self.assertRaises(ValueError): | |
| module.starts(0) | |
| def test_boundary_geometry_retains_orientation_and_marks_fragments(self): | |
| module = self.load() | |
| full = module.clip_object([1000, 10, 1040, 10, 1040, 30, 1000, 30], (0, 0, 1024, 1024)) | |
| self.assertEqual(full["bbox"], [1000.0, 10.0, 24.0, 20.0]) | |
| self.assertAlmostEqual(full["visibility"], 0.6) | |
| self.assertTrue(full["ignore"]) | |
| self.assertEqual(len(full["original_polygon"]), 8) | |
| with self.assertRaises(ValueError): | |
| module.clip_object([0, 0, 1, 1, 0, 1, 1, 0], (0, 0, 1024, 1024)) | |
| def test_mask_mapping_preserves_ignore_and_rejects_unknown_ids(self): | |
| module = self.load() | |
| import numpy as np | |
| mask = np.array([[0, 1, 2, 3, 4, 5, 6, 7, 8]], dtype=np.uint8) | |
| mapped = module.map_oem_mask(mask) | |
| self.assertEqual(mapped.tolist(), [[255, 15, 13, 16, 2, 13, 12, 14, 1]]) | |
| with self.assertRaises(ValueError): | |
| module.map_oem_mask(np.array([[9]], dtype=np.uint8)) | |
| def test_group_splits_are_deterministic_and_disjoint(self): | |
| module = self.load() | |
| groups = [f"region-{i}" for i in range(20)] | |
| first = module.split_groups(groups) | |
| self.assertEqual(first, module.split_groups(list(reversed(groups)))) | |
| self.assertEqual(list(first.values()).count("train"), 16) | |
| self.assertEqual(list(first.values()).count("val"), 2) | |
| self.assertEqual(list(first.values()).count("test"), 2) | |
| with self.assertRaises(ValueError): | |
| module.split_groups(["only-one"]) | |
| def test_unmapped_dota_labels_are_not_silently_accepted(self): | |
| module = self.load() | |
| labels, meta = module.parse_dota("imagesource:GoogleEarth\ngsd:0.3\n0 0 10 0 10 5 0 5 small-vehicle 0\n") | |
| self.assertEqual(labels[0]["canonical_class"], "vehicle") | |
| self.assertEqual(labels[0]["source_line"], 3) | |
| self.assertEqual(meta["gsd"], "0.3") | |
| with self.assertRaises(ValueError): | |
| module.parse_dota("0 0 1 0 1 1 0 1 typo-class 0\n") | |
| def test_segmentation_build_has_real_masks_provenance_and_padding(self): | |
| import tempfile | |
| import json | |
| import numpy as np | |
| from PIL import Image | |
| module = self.load() | |
| self.assertTrue(hasattr(module, "corpus"), "corpus writer is not implemented") | |
| with tempfile.TemporaryDirectory() as temp: | |
| writer = module.corpus(Path(temp), maximum_bytes=10000000) | |
| image = Image.fromarray(np.full((32, 40, 3), 120, dtype=np.uint8)) | |
| mask = np.full((32, 40), 8, dtype=np.uint8) | |
| writer.segmentation(image, mask, {"id": "region-a_1", "source_dataset": "openearthmap", "source_image_id": "a.tif", "group": "region-a", "split": "train", "license": "CC-BY-4.0", "license_group": "commercial_compatible"}) | |
| writer.finish() | |
| samples = [json.loads(line) for line in (Path(temp) / "metadata" / "samples.jsonl").read_text().splitlines()] | |
| self.assertEqual(len(samples), 1) | |
| sample = samples[0] | |
| self.assertEqual(sample["original_dimensions"], [40, 32]) | |
| output = np.array(Image.open(Path(temp) / sample["mask"])) | |
| self.assertEqual(output.shape, (1024, 1024)) | |
| self.assertTrue(np.all(output[:32, :40] == 1)) | |
| self.assertEqual(int(output[32, 0]), 255) | |
| self.assertEqual(sample["source_image_id"], "a.tif") | |
| def test_detection_preserves_obb_and_never_fills_it_as_mask(self): | |
| import tempfile | |
| import json | |
| import numpy as np | |
| from PIL import Image | |
| module = self.load() | |
| self.assertTrue(hasattr(module, "corpus"), "corpus writer is not implemented") | |
| with tempfile.TemporaryDirectory() as temp: | |
| writer = module.corpus(Path(temp), maximum_bytes=10000000) | |
| image = Image.fromarray(np.full((64, 64, 3), 100, dtype=np.uint8)) | |
| labels, _ = module.parse_dota("10 10 20 10 20 15 10 15 small-vehicle 0\n") | |
| writer.detection(image, labels, {"id": "dota-p1", "source_dataset": "dota", "source_image_id": "P1", "group": "P1", "split": "train", "license": "academic only", "license_group": "academic_only"}) | |
| writer.finish() | |
| document = json.loads((Path(temp) / "detection" / "train" / "annotations.json").read_text()) | |
| self.assertEqual(document["annotations"][0]["bbox"], [10, 10, 10, 5]) | |
| self.assertEqual(document["annotations"][0]["category_id"], 4) | |
| self.assertEqual(document["annotations"][0]["original_class"], "small-vehicle") | |
| self.assertEqual(document["annotations"][0]["obb_polygon"], [10, 10, 20, 10, 20, 15, 10, 15]) | |
| self.assertNotIn("segmentation", document["annotations"][0]) | |
| def test_unmapped_dota_objects_are_preserved_as_loss_ignore_regions(self): | |
| import tempfile | |
| import json | |
| import numpy as np | |
| from PIL import Image | |
| module = self.load() | |
| with tempfile.TemporaryDirectory() as temp: | |
| writer = module.corpus(Path(temp), maximum_bytes=10000000) | |
| image = Image.fromarray(np.full((64, 64, 3), 100, dtype=np.uint8)) | |
| labels, _ = module.parse_dota("10 10 30 10 30 20 10 20 harbor 0\n") | |
| writer.detection(image, labels, {"id": "dota-ignore", "source_dataset": "dota", "source_image_id": "P2", "group": "P2", "split": "train", "license": "academic", "license_group": "academic_only"}) | |
| writer.finish() | |
| document = json.loads((Path(temp) / "detection/train/annotations.json").read_text()) | |
| self.assertEqual(len(document["annotations"]), 0) | |
| self.assertEqual(len(document["ignored_annotations"]), 1) | |
| self.assertEqual(document["ignored_annotations"][0]["original_class"], "harbor") | |
| self.assertTrue(document["ignored_annotations"][0]["ignore"]) | |
| def test_exact_duplicate_is_rejected_across_splits(self): | |
| import tempfile | |
| import numpy as np | |
| from PIL import Image | |
| module = self.load() | |
| self.assertTrue(hasattr(module, "corpus"), "corpus writer is not implemented") | |
| with tempfile.TemporaryDirectory() as temp: | |
| writer = module.corpus(Path(temp), maximum_bytes=10000000) | |
| image = Image.fromarray(np.full((16, 16, 3), 100, dtype=np.uint8)) | |
| mask = np.ones((16, 16), dtype=np.uint8) | |
| meta = {"id": "one", "source_dataset": "openearthmap", "source_image_id": "one", "group": "a", "split": "train", "license": "test", "license_group": "academic_only"} | |
| writer.segmentation(image, mask, meta) | |
| writer.segmentation(image, mask, dict(meta, id="two", group="b", split="test")) | |
| self.assertEqual(len(writer.samples), 1) | |
| self.assertEqual(writer.rejected[-1]["reason"], "duplicate_source_pixels") | |
| def test_acquisition_selection_rejects_unpaired_and_unknown_license(self): | |
| self.assertTrue(Path("acquire.py").exists(), "source acquisition is not implemented") | |
| spec = importlib.util.spec_from_file_location("acquire", "acquire.py") | |
| module = importlib.util.module_from_spec(spec) | |
| spec.loader.exec_module(module) | |
| names = ["root/a/images/a_1.tif", "root/a/labels/a_1.tif", "root/a/images/a_2.tif", "root/b/images/b_1.tif", "root/b/labels/b_1.tif"] | |
| selected = module.select_oem(names, {"a": {"license_group": "commercial_compatible"}}, 20) | |
| self.assertEqual(len(selected), 1) | |
| self.assertEqual(selected[0]["image_member"], "root/a/images/a_1.tif") | |
| def test_loader_masks_ignored_regions_and_padding(self): | |
| import tempfile | |
| import json | |
| import numpy as np | |
| from PIL import Image | |
| self.assertTrue(Path("load_dataset.py").exists(), "training loader is not implemented") | |
| spec = importlib.util.spec_from_file_location("load_dataset", "load_dataset.py") | |
| module = importlib.util.module_from_spec(spec) | |
| spec.loader.exec_module(module) | |
| with tempfile.TemporaryDirectory() as temp: | |
| root = Path(temp) | |
| Image.new("RGB", (1024, 1024), (100, 100, 100)).save(root / "image.png") | |
| (root / "labels.json").write_text(json.dumps({"annotations": [], "ignored_annotations": [{"clipped_polygon": [10, 10, 20, 10, 20, 20, 10, 20]}]})) | |
| sample = {"image": "image.png", "labels": "labels.json", "task": "detection", "valid_width": 64, "valid_height": 32, "annotated_classes": [4]} | |
| data = module.load_sample(root, sample) | |
| self.assertEqual(data["image"].shape, (1024, 1024, 3)) | |
| self.assertTrue(data["valid_mask"][0, 0]) | |
| self.assertFalse(data["valid_mask"][15, 15]) | |
| self.assertFalse(data["valid_mask"][32, 0]) | |
| self.assertEqual(data["boxes"].shape, (0, 4)) | |
| def test_scene_selection_retains_rare_class(self): | |
| module = self.load() | |
| self.assertTrue(hasattr(module, "select_dota"), "scene selection is not implemented") | |
| labels = {f"P{i}": [{"canonical_class": "vehicle"}] for i in range(30)} | |
| labels["rare"] = [{"canonical_class": "aircraft"}] | |
| selected = module.select_dota(labels, 10) | |
| self.assertIn("rare", selected) | |
| self.assertEqual(len(selected), 10) | |
| def test_detection_scene_splits_preserve_rare_class_coverage(self): | |
| module = self.load() | |
| self.assertTrue(hasattr(module, "stratified_scene_splits"), "scene-level class stratification is not implemented") | |
| labels = {} | |
| classes = ["vehicle", "large_vehicle", "aircraft", "vessel", "storage_tank", "bridge"] | |
| for index in range(60): | |
| label = classes[index % len(classes)] | |
| labels[f"scene-{index:03}"] = [{"canonical_class": label, "difficult": False}] | |
| splits = module.stratified_scene_splits(labels) | |
| self.assertEqual(splits, module.stratified_scene_splits(dict(reversed(list(labels.items()))))) | |
| for split in ("train", "val", "test"): | |
| covered = {row["canonical_class"] for scene, rows in labels.items() if splits[scene] == split for row in rows} | |
| self.assertEqual(covered, set(classes)) | |
| self.assertEqual(Counter(splits.values()), Counter(train=48, val=6, test=6)) | |
| def test_detection_tile_cap_keeps_positive_rare_class_tiles(self): | |
| module = self.load() | |
| self.assertTrue(hasattr(module, "select_detection_tiles"), "label-aware tile selection is not implemented") | |
| labels = [ | |
| {"canonical_class": "vehicle", "difficult": False, "vertices": [10, 10, 40, 10, 40, 30, 10, 30]}, | |
| {"canonical_class": "storage_tank", "difficult": False, "vertices": [2820, 10, 2860, 10, 2860, 40, 2820, 40]}, | |
| ] | |
| selected = module.select_detection_tiles((3000, 1024), labels, 2) | |
| self.assertEqual(len(selected), 2) | |
| self.assertTrue(any(bounds[0] == 1976 for bounds in selected)) | |
| self.assertTrue(any(bounds[0] == 0 for bounds in selected)) | |
| def test_nodata_tiles_are_not_retained_as_labeled_negatives(self): | |
| import tempfile | |
| import numpy as np | |
| from PIL import Image | |
| module = self.load() | |
| with tempfile.TemporaryDirectory() as temp: | |
| writer = module.corpus(Path(temp), maximum_bytes=10000000) | |
| pixels = np.full((1024, 2048, 3), 100, dtype=np.uint8) | |
| pixels[:, :1024] = 0 | |
| meta = {"id": "nodata", "source_dataset": "openearthmap", "source_image_id": "one", "group": "a", "split": "train", "license": "test", "license_group": "academic_only"} | |
| writer.segmentation(Image.fromarray(pixels), np.ones((1024, 2048), dtype=np.uint8), meta) | |
| self.assertEqual(len(writer.samples), 2) | |
| self.assertTrue(any(row["reason"] == "mostly_black_tile" for row in writer.rejected)) | |
| overlap = next(row for row in writer.samples if row["tile_coordinates"][0] == 819) | |
| mask = np.array(Image.open(Path(temp) / overlap["mask"])) | |
| self.assertTrue(np.all(mask[:, :205] == 255)) | |
| def test_storage_budget_counts_existing_files(self): | |
| import tempfile | |
| module = self.load() | |
| self.assertTrue(hasattr(module, "storage_guard"), "workspace storage guard is not implemented") | |
| with tempfile.TemporaryDirectory() as temp: | |
| root = Path(temp) | |
| (root / "old-corpus.bin").write_bytes(b"12345678") | |
| budget = module.storage_guard(root, 10) | |
| with self.assertRaises(OSError): | |
| budget.reserve(3) | |
| budget.reserve(2) | |
| def test_loader_rejects_changed_approved_file(self): | |
| import tempfile | |
| from PIL import Image | |
| spec = importlib.util.spec_from_file_location("load_dataset", "load_dataset.py") | |
| module = importlib.util.module_from_spec(spec) | |
| spec.loader.exec_module(module) | |
| with tempfile.TemporaryDirectory() as temp: | |
| root = Path(temp) | |
| Image.new("RGB", (16, 16), "red").save(root / "image.png") | |
| sample = {"image": "image.png", "task": "segmentation"} | |
| with self.assertRaises(ValueError): | |
| module.load_sample(root, sample, expected_hashes={"image.png": "not-the-actual-hash"}) | |
| def test_rgb_container_does_not_relicense_panchromatic_bands_as_rgb(self): | |
| import tempfile | |
| from PIL import Image | |
| module = self.load() | |
| with tempfile.TemporaryDirectory() as temp: | |
| writer = module.corpus(Path(temp), maximum_bytes=10000000) | |
| meta = {"id": "gray", "source_dataset": "dota", "source_image_id": "gray", "source_metadata": {"imagesource": "GF"}, "split": "train", "group": "gray"} | |
| self.assertFalse(writer.accept(Image.new("RGB", (10, 10), (100, 100, 100)), meta, "detection")) | |
| self.assertEqual(writer.rejected[-1]["reason"], "panchromatic_source_not_rgb") | |
| if __name__ == "__main__": | |
| unittest.main() | |
Xet Storage Details
- Size:
- 15.1 kB
- Xet hash:
- 5b0a5a600b55d4d8cc5f8cbc62085fba6f8a5fff74aae8469325544c5f877309
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.