code/more_data.sh

#!/bin/bash
# Download 6 more FLAIR departements and pack them as prep/flair2_{x,y}.npy (low CPU priority).
cd /workspace
export HF_HUB_ENABLE_HF_TRANSFER=1
nice -n 19 python - <<'EOF'
import glob, os, zipfile
import numpy as np
from concurrent.futures import ProcessPoolExecutor
from huggingface_hub import hf_hub_download
import prep
out = "/workspace/data/flair2/unz"
for z in ["D013_2020", "D030_2021", "D046_2019", "D063_2019", "D081_2020", "D021_2020"]:
    p = hf_hub_download("IGNF/FLAIR-1-2", f"data/train-val/{z}.zip", repo_type="dataset", local_dir="/workspace/data/flair2")
    with zipfile.ZipFile(p) as zf:
        zf.extractall(out, members=[n for n in zf.namelist() if not n.startswith("sentinel")])
    os.remove(p)
    print("got", z, flush=True)
imgs = sorted(glob.glob(f"{out}/aerial/**/IMG_*.tif", recursive=True))
with ProcessPoolExecutor(8) as ex:
    res = [r for r in ex.map(prep.flair_one, imgs, chunksize=32) if r is not None]
fx = np.stack([r[0] for r in res]); fy = np.stack([r[1] for r in res])
np.save("/workspace/prep/flair2_x.npy", fx); np.save("/workspace/prep/flair2_y.npy", fy)
print("MORE_DONE", fx.shape, (fy == 1).mean(), (fy == 2).mean(), flush=True)
EOF