Spaces:
Sleeping
Sleeping
Download plant_classifier/data.py from ZZW45/image-classifier: direct link, hf CLI and curl.
- Browser
- Download file 12.5 kB
-
https://huggingface.co/spaces/ZZW45/image-classifier/resolve/main/plant_classifier/data.py
- Command line
-
hf download hf://spaces/ZZW45/image-classifier/plant_classifier/data.py
-
curl -L -o data.py https://huggingface.co/spaces/ZZW45/image-classifier/resolve/main/plant_classifier/data.py
12.5 kB
| """Reproducible PlantVillage splits and shared training/inference transforms.""" | |
| from __future__ import annotations | |
| import csv | |
| import hashlib | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import tempfile | |
| from typing import Callable, Sequence | |
| from datasets import Dataset as HFDataset, load_dataset | |
| import numpy as np | |
| from PIL import Image, ImageOps | |
| from sklearn.model_selection import train_test_split | |
| from torch.utils.data import Dataset | |
| from torchvision import transforms | |
| from torchvision.transforms import InterpolationMode | |
| DATASET_ID = "DScomp380/plant_village" | |
| DATASET_REVISION = "5ce680f815ea9fab7b6f8346ae4c71e7099696a5" | |
| DEFAULT_MEAN = (0.5, 0.5, 0.5) | |
| DEFAULT_STD = (0.5, 0.5, 0.5) | |
| SPLIT_RATIOS = {"train": 0.70, "validation": 0.15, "test": 0.15} | |
| def prepare_image(image: Image.Image) -> Image.Image: | |
| """Apply camera orientation before consistently converting all inputs to RGB.""" | |
| return ImageOps.exif_transpose(image).convert("RGB") | |
| def make_transform( | |
| image_size: int = 128, | |
| augment: bool = False, | |
| *, | |
| mean: Sequence[float] = DEFAULT_MEAN, | |
| std: Sequence[float] = DEFAULT_STD, | |
| ) -> Callable: | |
| """Build the shared preprocessing pipeline; augment only training images.""" | |
| if image_size < 1: | |
| raise ValueError("image_size must be positive") | |
| if len(mean) != 3 or len(std) != 3 or any(value <= 0 for value in std): | |
| raise ValueError("mean/std must contain three channels with positive std") | |
| operations = [ | |
| prepare_image, | |
| transforms.Resize( | |
| (image_size, image_size), | |
| interpolation=InterpolationMode.BILINEAR, | |
| antialias=True, | |
| ), | |
| ] | |
| if augment: | |
| operations.extend([ | |
| transforms.RandomHorizontalFlip(), | |
| transforms.RandomVerticalFlip(), | |
| transforms.RandomRotation(15, interpolation=InterpolationMode.BILINEAR), | |
| transforms.ColorJitter(brightness=0.15, contrast=0.15, saturation=0.10, hue=0.02), | |
| ]) | |
| operations.extend([transforms.ToTensor(), transforms.Normalize(mean=mean, std=std)]) | |
| return transforms.Compose(operations) | |
| class ImageDataset(Dataset): | |
| """Decode only requested HF rows, keeping the full dataset out of RAM.""" | |
| def __init__(self, raw: HFDataset, indices: Sequence[int], transform: Callable): | |
| self.raw = raw | |
| self.indices = list(indices) | |
| self.transform = transform | |
| def __len__(self) -> int: | |
| return len(self.indices) | |
| def __getitem__(self, position: int): | |
| row = self.raw[int(self.indices[position])] | |
| return self.transform(row["image"]), int(row["label"]) | |
| def stratified_subset( | |
| indices: Sequence[int], labels: Sequence[int], fraction: float, seed: int | |
| ) -> list[int]: | |
| """Select a reproducible subset, retaining every represented class. | |
| Counts follow the original proportions to within rounding constraints. Very | |
| small fractions that cannot retain all classes are rejected explicitly. | |
| """ | |
| if not math.isfinite(fraction) or not 0 < fraction <= 1: | |
| raise ValueError("fraction must be in (0, 1]") | |
| indices_array = np.asarray(indices) | |
| labels_array = np.asarray(labels) | |
| if indices_array.ndim != 1 or len(indices_array) == 0: | |
| raise ValueError("indices must be a nonempty one-dimensional sequence") | |
| if not np.issubdtype(indices_array.dtype, np.integer): | |
| raise ValueError("indices must contain integers") | |
| if ( | |
| len(np.unique(indices_array)) != len(indices_array) | |
| or np.any(indices_array < 0) | |
| or np.any(indices_array >= len(labels_array)) | |
| ): | |
| raise ValueError("indices must be unique valid rows of the full label array") | |
| if fraction == 1: | |
| return indices_array.tolist() | |
| classes, counts = np.unique(labels_array[indices_array], return_counts=True) | |
| target = math.floor(len(indices_array) * fraction) | |
| if target < len(classes): | |
| raise ValueError("fraction is too small to include every class") | |
| ideal = counts * fraction | |
| allocation = np.maximum(1, np.floor(ideal).astype(int)) | |
| while allocation.sum() < target: | |
| priority = np.where(allocation < counts, ideal - allocation, -np.inf) | |
| allocation[np.argmax(priority)] += 1 | |
| while allocation.sum() > target: | |
| priority = np.where(allocation > 1, allocation - ideal, -np.inf) | |
| allocation[np.argmax(priority)] -= 1 | |
| rng = np.random.default_rng(seed) | |
| selected = [] | |
| for label, count in zip(classes, allocation): | |
| candidates = indices_array[labels_array[indices_array] == label] | |
| selected.extend(rng.permutation(candidates)[:count].tolist()) | |
| return rng.permutation(selected).tolist() | |
| def _load_source(cache_dir: str | Path) -> HFDataset: | |
| return load_dataset( | |
| DATASET_ID, | |
| "default", | |
| split="train", | |
| revision=DATASET_REVISION, | |
| cache_dir=str(cache_dir), | |
| ) | |
| def _source_metadata(raw: HFDataset) -> tuple[list[str], np.ndarray]: | |
| if "image" not in raw.column_names or "label" not in raw.column_names: | |
| raise ValueError("PlantVillage must contain image and label columns") | |
| class_names = getattr(raw.features["label"], "names", None) | |
| if class_names is None or len(class_names) != 39: | |
| raise ValueError("Expected the specified PlantVillage dataset with 39 named classes") | |
| labels = np.asarray(raw["label"], dtype=np.int64) | |
| if set(np.unique(labels)) != set(range(len(class_names))): | |
| raise ValueError("Dataset labels must represent all 39 classes") | |
| return list(class_names), labels | |
| def _split_hash(splits: dict) -> str: | |
| encoded = json.dumps(splits, sort_keys=True, separators=(",", ":")).encode("utf-8") | |
| return hashlib.sha256(encoded).hexdigest() | |
| def _validate_manifest(raw: HFDataset, manifest: dict, expected_seed: int | None = None) -> None: | |
| class_names, labels = _source_metadata(raw) | |
| expected = { | |
| "schema_version": 1, | |
| "dataset_id": DATASET_ID, | |
| "revision": DATASET_REVISION, | |
| "class_names": class_names, | |
| "sample_count": len(raw), | |
| "split_ratios": SPLIT_RATIOS, | |
| } | |
| for field, value in expected.items(): | |
| if manifest.get(field) != value: | |
| raise ValueError(f"Existing data manifest has incompatible {field}; use a new data directory") | |
| if not isinstance(manifest.get("seed"), int): | |
| raise ValueError("Data manifest must contain an integer seed") | |
| if expected_seed is not None and manifest["seed"] != expected_seed: | |
| raise ValueError("Existing data manifest uses a different seed; use a new data directory") | |
| splits = manifest.get("splits", {}) | |
| if set(splits) != set(SPLIT_RATIOS): | |
| raise ValueError("Data manifest must contain train, validation, and test splits") | |
| all_indices = [] | |
| for name, ratio in SPLIT_RATIOS.items(): | |
| indices = splits[name] | |
| if not isinstance(indices, list) or any(type(index) is not int for index in indices): | |
| raise ValueError(f"{name} indices must be an integer list") | |
| if abs(len(indices) - len(raw) * ratio) > 1: | |
| raise ValueError(f"{name} split size is inconsistent with the fixed split ratios") | |
| if any(index < 0 or index >= len(raw) for index in indices): | |
| raise ValueError(f"{name} split contains invalid row indices") | |
| if set(labels[indices]) != set(range(len(class_names))): | |
| raise ValueError(f"{name} split does not contain every class") | |
| if manifest.get("split_sizes", {}).get(name) != len(indices): | |
| raise ValueError(f"{name} split size does not match the manifest") | |
| all_indices.extend(indices) | |
| if len(all_indices) != len(raw) or len(set(all_indices)) != len(raw): | |
| raise ValueError("Data splits must be disjoint and cover the complete dataset") | |
| if manifest.get("split_hash") != _split_hash(splits): | |
| raise ValueError("Data split hash does not match the saved indices") | |
| def _write_distribution_reports(data_dir: Path, manifest: dict, labels: np.ndarray) -> None: | |
| """Write counts and proportions without requiring a graphical backend.""" | |
| from matplotlib.backends.backend_agg import FigureCanvasAgg | |
| from matplotlib.figure import Figure | |
| class_names = manifest["class_names"] | |
| groups = {"original": np.arange(len(labels)), **manifest["splits"]} | |
| counts = {name: np.bincount(labels[indices], minlength=len(class_names)) for name, indices in groups.items()} | |
| fields = ["class_id", "class_name"] | |
| for name in groups: | |
| fields.extend([f"{name}_count", f"{name}_fraction"]) | |
| with (data_dir / "class_distribution.csv").open("w", newline="", encoding="utf-8") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=fields) | |
| writer.writeheader() | |
| for index, class_name in enumerate(class_names): | |
| row = {"class_id": index, "class_name": class_name} | |
| for name, indices in groups.items(): | |
| row[f"{name}_count"] = int(counts[name][index]) | |
| row[f"{name}_fraction"] = float(counts[name][index] / len(indices)) | |
| writer.writerow(row) | |
| figure = Figure(figsize=(17, 16)) | |
| FigureCanvasAgg(figure) | |
| axes = figure.subplots(1, 2, sharey=True) | |
| positions = np.arange(len(class_names)) | |
| axes[0].barh(positions, counts["original"], color="#2d6a4f") | |
| axes[0].set_yticks(positions, class_names, fontsize=8) | |
| axes[0].invert_yaxis() | |
| axes[0].set_title("Original class counts") | |
| axes[0].set_xlabel("Images") | |
| for offset, (name, indices) in enumerate(groups.items()): | |
| axes[1].barh( | |
| positions + (offset - 1.5) * 0.2, | |
| counts[name] / len(indices) * 100, | |
| height=0.2, | |
| label=name, | |
| ) | |
| axes[1].set_title("Class proportions across splits") | |
| axes[1].set_xlabel("Percentage of split") | |
| axes[1].legend() | |
| figure.tight_layout() | |
| figure.savefig(data_dir / "class_distribution.png", dpi=120) | |
| def prepare_data(cache_dir: str | Path, data_dir: str | Path, seed: int = 42) -> dict: | |
| """Download the pinned dataset and create, or verify, immutable split indices.""" | |
| data_dir = Path(data_dir) | |
| manifest_path = data_dir / "manifest.json" | |
| raw = _load_source(cache_dir) | |
| class_names, labels = _source_metadata(raw) | |
| if manifest_path.exists(): | |
| manifest = json.loads(manifest_path.read_text(encoding="utf-8")) | |
| _validate_manifest(raw, manifest, expected_seed=seed) | |
| if not all((data_dir / name).exists() for name in ("class_distribution.csv", "class_distribution.png")): | |
| _write_distribution_reports(data_dir, manifest, labels) | |
| return manifest | |
| indices = np.arange(len(raw)) | |
| train_indices, held_out = train_test_split(indices, test_size=0.30, stratify=labels, random_state=seed) | |
| validation_indices, test_indices = train_test_split( | |
| held_out, test_size=0.50, stratify=labels[held_out], random_state=seed | |
| ) | |
| splits = {"train": train_indices.tolist(), "validation": validation_indices.tolist(), "test": test_indices.tolist()} | |
| manifest = { | |
| "schema_version": 1, | |
| "dataset_id": DATASET_ID, | |
| "revision": DATASET_REVISION, | |
| "seed": seed, | |
| "sample_count": len(raw), | |
| "class_names": class_names, | |
| "split_ratios": SPLIT_RATIOS.copy(), | |
| "split_sizes": {name: len(values) for name, values in splits.items()}, | |
| "split_hash": _split_hash(splits), | |
| "splits": splits, | |
| } | |
| _validate_manifest(raw, manifest, expected_seed=seed) | |
| data_dir.mkdir(parents=True, exist_ok=True) | |
| _write_distribution_reports(data_dir, manifest, labels) | |
| with tempfile.NamedTemporaryFile(mode="w", dir=data_dir, suffix=".json", delete=False, encoding="utf-8") as handle: | |
| json.dump(manifest, handle, indent=2) | |
| temporary_path = Path(handle.name) | |
| try: | |
| os.replace(temporary_path, manifest_path) | |
| finally: | |
| temporary_path.unlink(missing_ok=True) | |
| return manifest | |
| def load_prepared_data(cache_dir: str | Path, data_dir: str | Path) -> tuple[HFDataset, dict]: | |
| """Load the exact persisted partition; never create a replacement implicitly.""" | |
| manifest_path = Path(data_dir) / "manifest.json" | |
| if not manifest_path.is_file(): | |
| raise FileNotFoundError(f"Missing {manifest_path}; run the prepare command first") | |
| manifest = json.loads(manifest_path.read_text(encoding="utf-8")) | |
| raw = _load_source(cache_dir) | |
| _validate_manifest(raw, manifest) | |
| return raw, manifest | |