Celyrix-1.0-Vision / src /dataset.py
mikexsav's picture
Release CELYRIX v1.0.0
17124c2
Raw History Blame Contribute Delete
4.49 kB
"""Official EuroSAT RGB archive. No dataset is redistributed by this project."""
import hashlib
import io
import json
import urllib.request
import zipfile
from pathlib import Path
import numpy as np
from PIL import Image
from sklearn.model_selection import train_test_split
from src.common import CLASSES, SOURCE_CLASSES, preprocess
URL = "https://zenodo.org/api/records/7711810/files/EuroSAT_RGB.zip/content"
def sha256(path):
h = hashlib.sha256()
with open(path, "rb") as f:
for block in iter(lambda: f.read(1024 * 1024), b""):
h.update(block)
return h.hexdigest()
def load_data(data_dir="data", seed=42):
root = Path(data_dir)
root.mkdir(parents=True, exist_ok=True)
archive = root / "EuroSAT_RGB.zip"
record = root / "zenodo-record.json"
if not record.exists():
urllib.request.urlretrieve("https://zenodo.org/api/records/7711810", record)
metadata = json.loads(record.read_text())
upstream = next(f for f in metadata["files"] if f["key"] == archive.name)
if not archive.exists():
urllib.request.urlretrieve(URL, archive)
algorithm, expected = upstream["checksum"].split(":")
h = hashlib.new(algorithm)
with archive.open("rb") as f:
for block in iter(lambda: f.read(1024 * 1024), b""):
h.update(block)
if h.hexdigest() != expected:
raise ValueError("EuroSAT upstream checksum mismatch")
pixels, labels, names, hashes = [], [], [], []
seen = {}
duplicates = 0
with zipfile.ZipFile(archive) as z:
for name in sorted(z.namelist()):
parts = Path(name).parts
if (
len(parts) < 2
or parts[-2] not in SOURCE_CLASSES
or not name.lower().endswith(".jpg")
):
continue
with Image.open(io.BytesIO(z.read(name))) as im:
if im.size != (64, 64) or im.mode != "RGB":
raise ValueError(f"Unexpected source schema: {name}")
x = preprocess(im).astype(np.uint8)
digest = hashlib.sha256(x.tobytes()).hexdigest()
if digest in seen:
if seen[digest] != SOURCE_CLASSES.index(parts[-2]):
raise ValueError(
"Identical RGB pixels carry conflicting source labels"
)
duplicates += 1
continue
seen[digest] = SOURCE_CLASSES.index(parts[-2])
pixels.append(x)
labels.append(SOURCE_CLASSES.index(parts[-2]))
names.append(name)
hashes.append(digest)
if set(labels) != set(range(len(CLASSES))):
raise ValueError("Dataset is missing one or more required source classes")
x, y = np.stack(pixels), np.array(labels, dtype=np.int64)
indices = np.arange(len(y))
train, held = train_test_split(
indices, test_size=0.4, random_state=seed, stratify=y
)
val, test = train_test_split(
held, test_size=0.5, random_state=seed, stratify=y[held]
)
splits = {"train": train, "validation": val, "test": test}
provenance = {
"source": URL,
"record": "7711810",
"source_license": "MIT",
"archive_sha256": sha256(archive),
"upstream_checksum": upstream["checksum"],
"seed": seed,
"split_method": "stratified 60/20/20 after global pixel deduplication",
"provided_splits": False,
"duplicate_images_removed": duplicates,
"spatial_independence": "Not established; geographic grouping unavailable in RGB archive.",
"classes": dict(zip(CLASSES, SOURCE_CLASSES)),
"counts": {
s: {c: int(np.sum(y[idx] == i)) for i, c in enumerate(CLASSES)}
for s, idx in splits.items()
},
"samples": {
s: [
{"filename": names[i], "pixel_sha256": hashes[i], "label": int(y[i])}
for i in idx
]
for s, idx in splits.items()
},
}
return {s: (x[idx], y[idx]) for s, idx in splits.items()}, provenance
def pipeline(x, y, batch_size, training=False, seed=42):
import tensorflow as tf
ds = tf.data.Dataset.from_tensor_slices((x, y))
if training:
ds = ds.shuffle(len(y), seed=seed, reshuffle_each_iteration=True)
options = tf.data.Options()
options.threading.private_threadpool_size = 4
return ds.batch(batch_size).with_options(options).prefetch(1)