Download ai/make_notebook.py from m4az19/saef-api: direct link, hf CLI and curl.
- Browser
- Download file 12.9 kB
-
https://huggingface.co/spaces/m4az19/saef-api/resolve/main/ai/make_notebook.py
- Command line
-
hf download hf://spaces/m4az19/saef-api/ai/make_notebook.py
-
curl -L -o make_notebook.py https://huggingface.co/spaces/m4az19/saef-api/resolve/main/ai/make_notebook.py
12.9 kB
| """Generate the deliverable notebook; this script does not train a model.""" | |
| import json | |
| from pathlib import Path | |
| cells=[] | |
| def md(text):cells.append({'cell_type':'markdown','metadata':{},'source':text.splitlines(True)}) | |
| def code(text):cells.append({'cell_type':'code','execution_count':None,'metadata':{},'outputs':[],'source':text.splitlines(True)}) | |
| md('''# Satforge · Kaggle-only RGB water segmentation | |
| This notebook trains a lightweight U-Net for **ground-computer RGB inference**. No local training is required to start Satforge. The installed Sentinel-2 UNet++ is a separate pretrained six-band model; do not use its weights with this RGB network. | |
| Enable a Kaggle GPU and attach a licensed dataset containing RGB images, binary masks (0 land / 255 water), and `manifest.csv`. Required columns: `image`, `mask`, `scene_id`, `spatial_group`. Paths may be absolute Kaggle input paths or relative to the manifest. Every overlapping/neighboring scene, including observations at different dates of the same area, **must share a spatial_group**. Groups are split before tiles are created. Never use randomly generated images to report model performance. Dataset licensing and grouping are the user's responsibility. | |
| Water-like confounders (shadows, dark roads, wet soil, vegetation) should be included as labeled negative examples. RGB camera photographs and satellite images are different domains: maintain separate held-out groups and report domain-specific metrics. An exported RGB network is NOT automatically suitable for ESP32 memory or camera accuracy. | |
| ''') | |
| code('''%pip install -q segmentation-models-pytorch==0.5.0 timm==1.0.22 onnx==1.17.0 onnxruntime==1.23.2 | |
| ''') | |
| code('''import os, json, random, hashlib, copy, time | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| from PIL import Image | |
| import torch | |
| from torch import nn | |
| from torch.utils.data import Dataset, DataLoader | |
| import segmentation_models_pytorch as smp | |
| CFG = dict(seed=42, manifest='/kaggle/input/YOUR-DATASET/manifest.csv', epochs=40, | |
| batch_size=8, tile_size=256, learning_rate=1e-4, patience=7, | |
| encoder='mobilenet_v2', encoder_weights='imagenet', val_fraction=0.15, | |
| test_fraction=0.15, output='/kaggle/working/satforge-water') | |
| OUT=Path(CFG['output']); OUT.mkdir(exist_ok=True, parents=True) | |
| random.seed(CFG['seed']); np.random.seed(CFG['seed']); torch.manual_seed(CFG['seed']) | |
| torch.cuda.manual_seed_all(CFG['seed']) | |
| torch.backends.cudnn.deterministic=True; torch.backends.cudnn.benchmark=False | |
| device=torch.device('cuda' if torch.cuda.is_available() else 'cpu') | |
| assert device.type=='cuda', 'Enable Kaggle GPU; this notebook intentionally refuses accidental CPU training.' | |
| (OUT/'config.json').write_text(json.dumps(CFG,indent=2)) | |
| print(device, torch.cuda.get_device_name(0)) | |
| ''') | |
| md('''## Grouped splits and audit | |
| The manifest must be curated before training. The automatic checks prevent identical files from crossing splits and keep spatial groups together; they cannot discover incorrectly labeled neighboring scenes without geographic metadata. Keep the generated split manifest with the model. Test data is not used for early stopping or threshold selection. | |
| ''') | |
| code('''manifest=Path(CFG['manifest']); df=pd.read_csv(manifest) | |
| required={'image','mask','scene_id','spatial_group'} | |
| assert required.issubset(df.columns), f'Missing columns: {required-set(df.columns)}' | |
| assert not df[list(required)].isna().any().any() | |
| for col in ['image','mask']: | |
| df[col]=df[col].map(lambda p: str((manifest.parent/p).resolve()) if not Path(p).is_absolute() else p) | |
| assert df[col].map(lambda p: Path(p).is_file()).all(), f'Missing {col} files' | |
| assert df.groupby('scene_id').spatial_group.nunique().max()==1, 'A scene cannot span groups' | |
| groups=sorted(df.spatial_group.unique()); random.Random(CFG['seed']).shuffle(groups) | |
| assert len(groups)>=7, 'Provide at least seven independent spatial groups for useful splits' | |
| n_test=max(1,round(len(groups)*CFG['test_fraction'])); n_val=max(1,round(len(groups)*CFG['val_fraction'])) | |
| test_groups=set(groups[:n_test]); val_groups=set(groups[n_test:n_test+n_val]) | |
| df['split']=df.spatial_group.map(lambda g:'test' if g in test_groups else 'val' if g in val_groups else 'train') | |
| df['sha256']=df.image.map(lambda p:hashlib.sha256(Path(p).read_bytes()).hexdigest()) | |
| assert df.groupby('sha256').split.nunique().max()==1, 'Identical image leakage across splits' | |
| df.to_csv(OUT/'split_manifest.csv',index=False) | |
| print(df.groupby('split').agg(images=('image','count'), scenes=('scene_id','nunique'), groups=('spatial_group','nunique'))) | |
| ''') | |
| code('''class Tiles(Dataset): | |
| def __init__(self, rows, augment=False): | |
| self.items=[]; self.augment=augment; size=CFG['tile_size'] | |
| for row in rows.to_dict('records'): | |
| with Image.open(row['image']) as image, Image.open(row['mask']) as mask: | |
| assert image.size==mask.size, f"Unaligned image and mask: {row['scene_id']}" | |
| assert image.mode in ('RGB','RGBA'), 'Input must be display RGB; use separate preprocessing for satellite spectral data' | |
| w,h=image.size | |
| for y in range(0,h,size): | |
| for x in range(0,w,size): self.items.append((row,x,y)) | |
| def __len__(self): return len(self.items) | |
| def __getitem__(self,i): | |
| row,x,y=self.items[i]; s=CFG['tile_size'] | |
| with Image.open(row['image']) as im: a=np.asarray(im.convert('RGB').crop((x,y,x+s,y+s))).copy() | |
| with Image.open(row['mask']) as im: | |
| width,height=im.size; b=np.asarray(im.convert('L').crop((x,y,x+s,y+s))).copy()>127 | |
| valid=np.zeros((s,s),np.float32); valid[:min(s,height-y),:min(s,width-x)]=1 | |
| if self.augment: | |
| k=random.randrange(4); a=np.rot90(a,k);b=np.rot90(b,k);valid=np.rot90(valid,k) | |
| if random.random()<.5: a=np.fliplr(a);b=np.fliplr(b);valid=np.fliplr(valid) | |
| # Modest RGB-only brightness augmentation, never applied to masks. | |
| a=np.clip(a.astype(np.float32)*random.uniform(.85,1.15),0,255) | |
| return torch.from_numpy(np.ascontiguousarray(a.transpose(2,0,1))).float()/255, torch.from_numpy(np.ascontiguousarray(b[None])).float(), torch.from_numpy(np.ascontiguousarray(valid[None])).float(),row['scene_id'] | |
| datasets={split:Tiles(df[df.split==split],augment=split=='train') for split in ['train','val','test']} | |
| loaders={split:DataLoader(ds,batch_size=CFG['batch_size'],shuffle=split=='train',num_workers=0) for split,ds in datasets.items()} | |
| model=smp.Unet(encoder_name=CFG['encoder'],encoder_weights=CFG['encoder_weights'],in_channels=3,classes=1).to(device) | |
| optimizer=torch.optim.AdamW(model.parameters(),lr=CFG['learning_rate'],weight_decay=1e-4) | |
| def loss_fn(logits,target,valid): | |
| bce=(nn.functional.binary_cross_entropy_with_logits(logits,target,reduction='none')*valid).sum()/valid.sum().clamp_min(1) | |
| p=logits.sigmoid()*valid; t=target*valid | |
| dice=1-(2*(p*t).sum()+1)/(p.sum()+t.sum()+1) | |
| return bce+dice | |
| ''') | |
| md('''## Training on Kaggle | |
| Best checkpoint and early stopping use validation loss only. The test loader is untouched until the final evaluation cell. No metrics are filled in until these cells execute on your data. | |
| ''') | |
| code('''best=float('inf'); stale=0; history=[] | |
| for epoch in range(CFG['epochs']): | |
| row={'epoch':epoch+1} | |
| for split in ['train','val']: | |
| model.train(split=='train'); total=0.; count=0 | |
| for x,y,valid,_ in loaders[split]: | |
| x,y,valid=x.to(device),y.to(device),valid.to(device) | |
| optimizer.zero_grad(set_to_none=True) | |
| with torch.set_grad_enabled(split=='train'): | |
| logits=model(x); loss=loss_fn(logits,y,valid) | |
| if split=='train': loss.backward();optimizer.step() | |
| total+=loss.item()*len(x);count+=len(x) | |
| row[split+'_loss']=total/max(count,1) | |
| history.append(row);pd.DataFrame(history).to_csv(OUT/'history.csv',index=False) | |
| print(row) | |
| if row['val_loss']<best: | |
| best=row['val_loss'];stale=0 | |
| torch.save({'state_dict':model.state_dict(),'config':CFG,'epoch':epoch+1},OUT/'best.pt') | |
| else: stale+=1 | |
| if stale>=CFG['patience']: break | |
| model.load_state_dict(torch.load(OUT/'best.pt',map_location=device,weights_only=True)['state_dict']);model.eval() | |
| ''') | |
| code('''def counts(pred,truth,valid): | |
| return np.array([np.sum(pred & truth & valid),np.sum(pred & ~truth & valid),np.sum(~pred & truth & valid),np.sum(~pred & ~truth & valid)],dtype=np.int64) | |
| def metric(c): | |
| tp,fp,fn,tn=map(int,c); div=lambda x,y:x/y if y else None | |
| return dict(iou=div(tp,tp+fp+fn),dice=div(2*tp,2*tp+fp+fn),precision=div(tp,tp+fp),recall=div(tp,tp+fn),f1=div(2*tp,2*tp+fp+fn),false_positive_rate=div(fp,fp+tn),false_negative_rate=div(fn,fn+tp),tp=tp,fp=fp,fn=fn,tn=tn) | |
| thresholds=np.arange(.2,.81,.05); totals={float(t):np.zeros(4,dtype=np.int64) for t in thresholds} | |
| with torch.inference_mode(): | |
| for x,y,v,_ in loaders['val']: | |
| probs=model(x.to(device)).sigmoid().cpu().numpy(); truth=y.numpy()>0.5; valid=v.numpy()>0.5 | |
| for t in totals: totals[t]+=counts(probs>=t,truth,valid) | |
| threshold=max(totals,key=lambda t:metric(totals[t])['iou'] if metric(totals[t])['iou'] is not None else -1) | |
| (OUT/'threshold.json').write_text(json.dumps({'value':threshold,'selected_on':'validation groups only'},indent=2)) | |
| print('Frozen validation threshold:',threshold) | |
| ''') | |
| md('''## Final held-out evaluation — run once after freezing the threshold | |
| Do not repeatedly adjust architecture or threshold to improve this test result. If you do, reserve new independent test scenes. This evaluation uses non-overlapping tiles; deployment uses overlap-weighted blending, so full-scene deployment validation remains necessary. | |
| ''') | |
| code('''per_scene={} | |
| with torch.inference_mode(): | |
| for x,y,v,scenes in loaders['test']: | |
| probs=model(x.to(device)).sigmoid().cpu().numpy() | |
| for i,scene in enumerate(scenes): | |
| per_scene.setdefault(scene,np.zeros(4,dtype=np.int64)) | |
| per_scene[scene]+=counts(probs[i]>=threshold,y[i].numpy()>.5,v[i].numpy()>.5) | |
| results={scene:metric(c) for scene,c in per_scene.items()} | |
| results['aggregate']=metric(sum(per_scene.values(),np.zeros(4,dtype=np.int64))) | |
| (OUT/'heldout_metrics.json').write_text(json.dumps(results,indent=2)) | |
| pd.DataFrame(results).T.to_csv(OUT/'per_scene_metrics.csv') | |
| print(json.dumps(results['aggregate'],indent=2)) | |
| ''') | |
| code('''class ProbabilityModel(nn.Module): | |
| def __init__(self,network):super().__init__();self.network=network | |
| def forward(self,x):return self.network(x).sigmoid() | |
| wrapped=ProbabilityModel(model.cpu()).eval() | |
| sample=torch.zeros(1,3,256,256) | |
| torch.onnx.export(wrapped,sample,str(OUT/'kaggle-water.onnx'),input_names=['rgb'],output_names=['water_probability'],opset_version=17,dynamo=False) | |
| import onnxruntime as ort | |
| session=ort.InferenceSession(str(OUT/'kaggle-water.onnx'),providers=['CPUExecutionProvider']) | |
| with torch.inference_mode():expected=wrapped(sample).numpy() | |
| actual=session.run(None,{'rgb':sample.numpy()})[0] | |
| assert np.allclose(expected,actual,atol=1e-4), 'ONNX parity failed' | |
| manifest_out={'architecture':'U-Net / MobileNetV2','input':'float32 NCHW RGB 0..1, 256×256','output':'sigmoid water probability', | |
| 'threshold':threshold,'sha256':hashlib.sha256((OUT/'kaggle-water.onnx').read_bytes()).hexdigest(), | |
| 'training_location':'Kaggle GPU','dataset_manifest_sha256':hashlib.sha256(manifest.read_bytes()).hexdigest(), | |
| 'limitations':['Uncalibrated probabilities','No UAE or ESP32-camera accuracy implied','Not an ESP32 deployment artifact']} | |
| (OUT/'model_manifest.json').write_text(json.dumps(manifest_out,indent=2)) | |
| import shutil | |
| shutil.make_archive('/kaggle/working/satforge-water-export','zip',OUT) | |
| print('Download satforge-water-export.zip. Copy kaggle-water.onnx to the local models/ folder. Select Kaggle RGB U-Net in Water intelligence and set the frozen threshold.') | |
| ''') | |
| md('''## ESP32 deployment boundary | |
| The current firmware captures JPEGs and sends sensor telemetry to the ground station. Neither the six-band EfficientNet-B4 model nor this RGB U-Net is represented as fitting ESP32. A future board-specific tiny int8 model requires exact board/PSRAM details, a representative calibration dataset, TensorFlow Lite Micro operator checks, measured tensor-arena usage, latency/power measurements, and independent ESP32-camera evaluation. Do not flash these checkpoints to a microcontroller. | |
| ''') | |
| notebook={'cells':cells,'metadata':{'kernelspec':{'display_name':'Python 3','language':'python','name':'python3'},'language_info':{'name':'python','version':'3.10'},'kaggle':{'accelerator':'gpu','isInternetEnabled':True}},'nbformat':4,'nbformat_minor':5} | |
| Path(__file__).with_name('kaggle_water_training.ipynb').write_text(json.dumps(notebook,indent=1),encoding='utf-8') | |
| print('Generated notebook. No training executed.') | |