"""Official EuroSAT RGB archive. No dataset is redistributed by this project.""" import hashlib import io import json import urllib.request import zipfile from pathlib import Path import numpy as np from PIL import Image from sklearn.model_selection import train_test_split from src.common import CLASSES, SOURCE_CLASSES, preprocess URL = "https://zenodo.org/api/records/7711810/files/EuroSAT_RGB.zip/content" def sha256(path): h = hashlib.sha256() with open(path, "rb") as f: for block in iter(lambda: f.read(1024 * 1024), b""): h.update(block) return h.hexdigest() def load_data(data_dir="data", seed=42): root = Path(data_dir) root.mkdir(parents=True, exist_ok=True) archive = root / "EuroSAT_RGB.zip" record = root / "zenodo-record.json" if not record.exists(): urllib.request.urlretrieve("https://zenodo.org/api/records/7711810", record) metadata = json.loads(record.read_text()) upstream = next(f for f in metadata["files"] if f["key"] == archive.name) if not archive.exists(): urllib.request.urlretrieve(URL, archive) algorithm, expected = upstream["checksum"].split(":") h = hashlib.new(algorithm) with archive.open("rb") as f: for block in iter(lambda: f.read(1024 * 1024), b""): h.update(block) if h.hexdigest() != expected: raise ValueError("EuroSAT upstream checksum mismatch") pixels, labels, names, hashes = [], [], [], [] seen = {} duplicates = 0 with zipfile.ZipFile(archive) as z: for name in sorted(z.namelist()): parts = Path(name).parts if ( len(parts) < 2 or parts[-2] not in SOURCE_CLASSES or not name.lower().endswith(".jpg") ): continue with Image.open(io.BytesIO(z.read(name))) as im: if im.size != (64, 64) or im.mode != "RGB": raise ValueError(f"Unexpected source schema: {name}") x = preprocess(im).astype(np.uint8) digest = hashlib.sha256(x.tobytes()).hexdigest() if digest in seen: if seen[digest] != SOURCE_CLASSES.index(parts[-2]): raise ValueError( "Identical RGB pixels carry conflicting source labels" ) duplicates += 1 continue seen[digest] = SOURCE_CLASSES.index(parts[-2]) pixels.append(x) labels.append(SOURCE_CLASSES.index(parts[-2])) names.append(name) hashes.append(digest) if set(labels) != set(range(len(CLASSES))): raise ValueError("Dataset is missing one or more required source classes") x, y = np.stack(pixels), np.array(labels, dtype=np.int64) indices = np.arange(len(y)) train, held = train_test_split( indices, test_size=0.4, random_state=seed, stratify=y ) val, test = train_test_split( held, test_size=0.5, random_state=seed, stratify=y[held] ) splits = {"train": train, "validation": val, "test": test} provenance = { "source": URL, "record": "7711810", "source_license": "MIT", "archive_sha256": sha256(archive), "upstream_checksum": upstream["checksum"], "seed": seed, "split_method": "stratified 60/20/20 after global pixel deduplication", "provided_splits": False, "duplicate_images_removed": duplicates, "spatial_independence": "Not established; geographic grouping unavailable in RGB archive.", "classes": dict(zip(CLASSES, SOURCE_CLASSES)), "counts": { s: {c: int(np.sum(y[idx] == i)) for i, c in enumerate(CLASSES)} for s, idx in splits.items() }, "samples": { s: [ {"filename": names[i], "pixel_sha256": hashes[i], "label": int(y[i])} for i in idx ] for s, idx in splits.items() }, } return {s: (x[idx], y[idx]) for s, idx in splits.items()}, provenance def pipeline(x, y, batch_size, training=False, seed=42): import tensorflow as tf ds = tf.data.Dataset.from_tensor_slices((x, y)) if training: ds = ds.shuffle(len(y), seed=seed, reshuffle_each_iteration=True) options = tf.data.Options() options.threading.private_threadpool_size = 4 return ds.batch(batch_size).with_options(options).prefetch(1)