File size: 4,492 Bytes
17124c2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
"""Official EuroSAT RGB archive. No dataset is redistributed by this project."""

import hashlib
import io
import json
import urllib.request
import zipfile
from pathlib import Path

import numpy as np
from PIL import Image
from sklearn.model_selection import train_test_split

from src.common import CLASSES, SOURCE_CLASSES, preprocess

URL = "https://zenodo.org/api/records/7711810/files/EuroSAT_RGB.zip/content"


def sha256(path):
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for block in iter(lambda: f.read(1024 * 1024), b""):
            h.update(block)
    return h.hexdigest()


def load_data(data_dir="data", seed=42):
    root = Path(data_dir)
    root.mkdir(parents=True, exist_ok=True)
    archive = root / "EuroSAT_RGB.zip"
    record = root / "zenodo-record.json"
    if not record.exists():
        urllib.request.urlretrieve("https://zenodo.org/api/records/7711810", record)
    metadata = json.loads(record.read_text())
    upstream = next(f for f in metadata["files"] if f["key"] == archive.name)
    if not archive.exists():
        urllib.request.urlretrieve(URL, archive)
    algorithm, expected = upstream["checksum"].split(":")
    h = hashlib.new(algorithm)
    with archive.open("rb") as f:
        for block in iter(lambda: f.read(1024 * 1024), b""):
            h.update(block)
    if h.hexdigest() != expected:
        raise ValueError("EuroSAT upstream checksum mismatch")
    pixels, labels, names, hashes = [], [], [], []
    seen = {}
    duplicates = 0
    with zipfile.ZipFile(archive) as z:
        for name in sorted(z.namelist()):
            parts = Path(name).parts
            if (
                len(parts) < 2
                or parts[-2] not in SOURCE_CLASSES
                or not name.lower().endswith(".jpg")
            ):
                continue
            with Image.open(io.BytesIO(z.read(name))) as im:
                if im.size != (64, 64) or im.mode != "RGB":
                    raise ValueError(f"Unexpected source schema: {name}")
                x = preprocess(im).astype(np.uint8)
            digest = hashlib.sha256(x.tobytes()).hexdigest()
            if digest in seen:
                if seen[digest] != SOURCE_CLASSES.index(parts[-2]):
                    raise ValueError(
                        "Identical RGB pixels carry conflicting source labels"
                    )
                duplicates += 1
                continue
            seen[digest] = SOURCE_CLASSES.index(parts[-2])
            pixels.append(x)
            labels.append(SOURCE_CLASSES.index(parts[-2]))
            names.append(name)
            hashes.append(digest)
    if set(labels) != set(range(len(CLASSES))):
        raise ValueError("Dataset is missing one or more required source classes")
    x, y = np.stack(pixels), np.array(labels, dtype=np.int64)
    indices = np.arange(len(y))
    train, held = train_test_split(
        indices, test_size=0.4, random_state=seed, stratify=y
    )
    val, test = train_test_split(
        held, test_size=0.5, random_state=seed, stratify=y[held]
    )
    splits = {"train": train, "validation": val, "test": test}
    provenance = {
        "source": URL,
        "record": "7711810",
        "source_license": "MIT",
        "archive_sha256": sha256(archive),
        "upstream_checksum": upstream["checksum"],
        "seed": seed,
        "split_method": "stratified 60/20/20 after global pixel deduplication",
        "provided_splits": False,
        "duplicate_images_removed": duplicates,
        "spatial_independence": "Not established; geographic grouping unavailable in RGB archive.",
        "classes": dict(zip(CLASSES, SOURCE_CLASSES)),
        "counts": {
            s: {c: int(np.sum(y[idx] == i)) for i, c in enumerate(CLASSES)}
            for s, idx in splits.items()
        },
        "samples": {
            s: [
                {"filename": names[i], "pixel_sha256": hashes[i], "label": int(y[i])}
                for i in idx
            ]
            for s, idx in splits.items()
        },
    }
    return {s: (x[idx], y[idx]) for s, idx in splits.items()}, provenance


def pipeline(x, y, batch_size, training=False, seed=42):
    import tensorflow as tf

    ds = tf.data.Dataset.from_tensor_slices((x, y))
    if training:
        ds = ds.shuffle(len(y), seed=seed, reshuffle_each_iteration=True)
    options = tf.data.Options()
    options.threading.private_threadpool_size = 4
    return ds.batch(batch_size).with_options(options).prefetch(1)