Update SatlasPretrain model package
Browse files- conf/config.yaml +34 -13
- config.json +28 -32
- configuration.json +2 -1
- model/satlaspretrain.py +125 -81
- scripts/fake_data.py +54 -44
- scripts/inference.py +34 -39
- scripts/result.py +145 -103
- scripts/train.py +72 -88
conf/config.yaml
CHANGED
|
@@ -1,21 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
model:
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
embed_dim:
|
| 6 |
window_size: 4
|
| 7 |
segmentation_classes: 4
|
| 8 |
property_classes: 3
|
| 9 |
classification_classes: 3
|
| 10 |
-
data:
|
| 11 |
-
path: ./data/fake_satlaspretrain.npz
|
| 12 |
training:
|
| 13 |
-
epochs:
|
| 14 |
-
batch_size:
|
| 15 |
-
learning_rate: 0.
|
| 16 |
-
|
| 17 |
-
|
| 18 |
runtime:
|
| 19 |
device: auto
|
| 20 |
-
|
| 21 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
seed: 42
|
| 2 |
+
data:
|
| 3 |
+
root: data
|
| 4 |
+
protocol: satlas_naip4_sentinel2_8_512_npz_v2
|
| 5 |
+
highres_time_steps: 4
|
| 6 |
+
highres_channels: 3
|
| 7 |
+
highres_size: 512
|
| 8 |
+
lowres_time_steps: 8
|
| 9 |
+
lowres_channels: 9
|
| 10 |
+
lowres_size: 512
|
| 11 |
+
train_samples: 1
|
| 12 |
+
test_samples: 1
|
| 13 |
model:
|
| 14 |
+
highres_channels: 3
|
| 15 |
+
lowres_channels: 9
|
| 16 |
+
output_size: 512
|
| 17 |
+
embed_dim: 8
|
| 18 |
window_size: 4
|
| 19 |
segmentation_classes: 4
|
| 20 |
property_classes: 3
|
| 21 |
classification_classes: 3
|
|
|
|
|
|
|
| 22 |
training:
|
| 23 |
+
epochs: 1
|
| 24 |
+
batch_size: 1
|
| 25 |
+
learning_rate: 0.0003
|
| 26 |
+
weight_decay: 0.01
|
| 27 |
+
num_workers: 0
|
| 28 |
runtime:
|
| 29 |
device: auto
|
| 30 |
+
amp: true
|
| 31 |
+
evaluation:
|
| 32 |
+
mask_threshold: 0.5
|
| 33 |
+
point_peak_threshold: 0.5
|
| 34 |
+
point_nms_radius: 3
|
| 35 |
+
point_distance_tolerance: 6
|
| 36 |
+
polygon_iou_threshold: 0.5
|
| 37 |
+
polyline_distance_tolerance: 2
|
| 38 |
+
paths:
|
| 39 |
+
checkpoint: result/checkpoints/satlaspretrain.pt
|
| 40 |
+
training_metrics: result/training/metrics.json
|
| 41 |
+
inference_dir: result/output
|
| 42 |
+
evaluation_dir: result/evaluation
|
config.json
CHANGED
|
@@ -5,22 +5,31 @@
|
|
| 5 |
"SatlasPretrain"
|
| 6 |
],
|
| 7 |
"framework": "PyTorch",
|
| 8 |
-
"domain": "earth-
|
| 9 |
"task": "remote-sensing-image-understanding",
|
| 10 |
"implementation": {
|
| 11 |
"entry_point": "model/satlaspretrain.py",
|
| 12 |
-
"scope": "
|
| 13 |
},
|
| 14 |
"architecture": {
|
| 15 |
-
"family": "
|
| 16 |
-
"
|
| 17 |
-
"
|
| 18 |
-
"
|
| 19 |
-
|
| 20 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
"window_size": 4,
|
| 22 |
-
"
|
| 23 |
-
"
|
|
|
|
| 24 |
"task_heads": [
|
| 25 |
"segmentation",
|
| 26 |
"regression",
|
|
@@ -32,31 +41,18 @@
|
|
| 32 |
]
|
| 33 |
},
|
| 34 |
"data": {
|
| 35 |
-
"
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
"Sentinel-2"
|
| 39 |
-
"NAIP"
|
| 40 |
],
|
| 41 |
-
"protocol": "
|
| 42 |
-
"
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
3,
|
| 46 |
-
64,
|
| 47 |
-
64
|
| 48 |
-
]
|
| 49 |
},
|
| 50 |
-
"metrics": [
|
| 51 |
-
"segmentation_f1",
|
| 52 |
-
"regression_mae",
|
| 53 |
-
"point_map",
|
| 54 |
-
"polygon_map",
|
| 55 |
-
"polyline_geo",
|
| 56 |
-
"property_accuracy",
|
| 57 |
-
"classification_accuracy"
|
| 58 |
-
],
|
| 59 |
"configuration_sources": [
|
|
|
|
| 60 |
"conf/config.yaml",
|
| 61 |
"model/satlaspretrain.py"
|
| 62 |
]
|
|
|
|
| 5 |
"SatlasPretrain"
|
| 6 |
],
|
| 7 |
"framework": "PyTorch",
|
| 8 |
+
"domain": "earth-observation",
|
| 9 |
"task": "remote-sensing-image-understanding",
|
| 10 |
"implementation": {
|
| 11 |
"entry_point": "model/satlaspretrain.py",
|
| 12 |
+
"scope": "dual-resolution temporal hierarchy with feature pyramid fusion and seven remote-sensing task heads"
|
| 13 |
},
|
| 14 |
"architecture": {
|
| 15 |
+
"family": "dual-sensor hierarchical encoder with temporal pooling and feature pyramid network",
|
| 16 |
+
"highres_time_steps": 4,
|
| 17 |
+
"highres_channels": 3,
|
| 18 |
+
"highres_image_shape": [
|
| 19 |
+
512,
|
| 20 |
+
512
|
| 21 |
+
],
|
| 22 |
+
"lowres_time_steps": 8,
|
| 23 |
+
"lowres_channels": 9,
|
| 24 |
+
"lowres_image_shape": [
|
| 25 |
+
512,
|
| 26 |
+
512
|
| 27 |
+
],
|
| 28 |
+
"embed_dim": 8,
|
| 29 |
"window_size": 4,
|
| 30 |
+
"segmentation_classes": 4,
|
| 31 |
+
"property_classes": 3,
|
| 32 |
+
"classification_classes": 3,
|
| 33 |
"task_heads": [
|
| 34 |
"segmentation",
|
| 35 |
"regression",
|
|
|
|
| 41 |
]
|
| 42 |
},
|
| 43 |
"data": {
|
| 44 |
+
"datasets": [
|
| 45 |
+
"SatlasPretrain",
|
| 46 |
+
"NAIP",
|
| 47 |
+
"Sentinel-2"
|
|
|
|
| 48 |
],
|
| 49 |
+
"protocol": "satlas_naip4_sentinel2_8_512_npz_v2",
|
| 50 |
+
"input_format": "BTCHW",
|
| 51 |
+
"train_samples": 1,
|
| 52 |
+
"test_samples": 1
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
"configuration_sources": [
|
| 55 |
+
"configuration.json",
|
| 56 |
"conf/config.yaml",
|
| 57 |
"model/satlaspretrain.py"
|
| 58 |
]
|
configuration.json
CHANGED
|
@@ -2,8 +2,9 @@
|
|
| 2 |
"framework": "PyTorch",
|
| 3 |
"task": "remote_sensing_image_understanding",
|
| 4 |
"model": "SatlasPretrain",
|
|
|
|
| 5 |
"input_format": "BTCHW",
|
| 6 |
-
"protocol": "
|
| 7 |
"default_config": "conf/config.yaml",
|
| 8 |
"train": "scripts/train.py",
|
| 9 |
"inference": "scripts/inference.py",
|
|
|
|
| 2 |
"framework": "PyTorch",
|
| 3 |
"task": "remote_sensing_image_understanding",
|
| 4 |
"model": "SatlasPretrain",
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
"input_format": "BTCHW",
|
| 7 |
+
"protocol": "satlas_naip4_sentinel2_8_512_npz_v2",
|
| 8 |
"default_config": "conf/config.yaml",
|
| 9 |
"train": "scripts/train.py",
|
| 10 |
"inference": "scripts/inference.py",
|
model/satlaspretrain.py
CHANGED
|
@@ -1,111 +1,106 @@
|
|
| 1 |
-
"""Compact SatlasNet
|
| 2 |
|
| 3 |
import torch
|
|
|
|
| 4 |
from torch import nn
|
| 5 |
from torch.nn import functional as F
|
| 6 |
|
| 7 |
|
| 8 |
class WindowBlock(nn.Module):
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
super().__init__()
|
| 13 |
-
self.window_size = window_size
|
| 14 |
self.norm1 = nn.LayerNorm(channels)
|
| 15 |
self.attention = nn.MultiheadAttention(channels, heads, batch_first=True)
|
| 16 |
self.norm2 = nn.LayerNorm(channels)
|
| 17 |
-
self.mlp = nn.Sequential(
|
| 18 |
-
nn.Linear(channels, channels * 4), nn.GELU(), nn.Linear(channels * 4, channels)
|
| 19 |
-
)
|
| 20 |
|
| 21 |
def forward(self, x):
|
| 22 |
-
batch, channels, height, width = x.shape
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
x = F.pad(x, (0, pad_w, 0, pad_h))
|
| 27 |
-
padded_h, padded_w = x.shape[-2:]
|
| 28 |
-
windows = x.permute(0, 2, 3, 1).reshape(
|
| 29 |
-
batch, padded_h // size, size, padded_w // size, size, channels
|
| 30 |
-
)
|
| 31 |
windows = windows.permute(0, 1, 3, 2, 4, 5).reshape(-1, size * size, channels)
|
| 32 |
normalized = self.norm1(windows)
|
| 33 |
-
|
| 34 |
-
windows = windows + attended
|
| 35 |
windows = windows + self.mlp(self.norm2(windows))
|
| 36 |
-
x = windows.reshape(batch,
|
| 37 |
-
x = x.permute(0, 1, 3, 2, 4, 5).reshape(batch,
|
| 38 |
return x[:, :height, :width].permute(0, 3, 1, 2).contiguous()
|
| 39 |
|
| 40 |
|
| 41 |
-
class
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
def __init__(
|
| 47 |
-
self,
|
| 48 |
-
input_channels=3,
|
| 49 |
-
embed_dim=24,
|
| 50 |
-
window_size=4,
|
| 51 |
-
segmentation_classes=4,
|
| 52 |
-
property_classes=3,
|
| 53 |
-
classification_classes=3,
|
| 54 |
-
):
|
| 55 |
-
super().__init__()
|
| 56 |
-
dimensions = [embed_dim * (2**index) for index in range(4)]
|
| 57 |
-
self.patch_embed = nn.Conv2d(input_channels, dimensions[0], 4, stride=4)
|
| 58 |
self.stages = nn.ModuleList()
|
| 59 |
-
for index,
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
self.stages.append(nn.Sequential(downsample, WindowBlock(channels, window_size, heads)))
|
| 63 |
-
self.lateral = nn.ModuleList(nn.Conv2d(channels, embed_dim, 1) for channels in dimensions)
|
| 64 |
-
dense_channels = {
|
| 65 |
-
"segmentation": segmentation_classes,
|
| 66 |
-
"regression": 1,
|
| 67 |
-
"point": 1,
|
| 68 |
-
"polygon": 1,
|
| 69 |
-
"polyline": 1,
|
| 70 |
-
}
|
| 71 |
-
self.dense_heads = nn.ModuleDict(
|
| 72 |
-
{name: nn.Conv2d(embed_dim, channels, 1) for name, channels in dense_channels.items()}
|
| 73 |
-
)
|
| 74 |
-
self.property_head = nn.Linear(dimensions[-1], property_classes)
|
| 75 |
-
self.classification_head = nn.Linear(dimensions[-1], classification_classes)
|
| 76 |
|
| 77 |
-
def
|
| 78 |
-
features = []
|
| 79 |
-
x =
|
| 80 |
-
for stage in self.stages:
|
| 81 |
-
x = stage(x)
|
| 82 |
-
features.append(x)
|
| 83 |
return features
|
| 84 |
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
batch, time, channels, height, width = images.shape
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
|
| 104 |
|
| 105 |
def multitask_loss(outputs, targets):
|
| 106 |
losses = {
|
| 107 |
"segmentation": F.cross_entropy(outputs["segmentation"], targets["segmentation"].long()),
|
| 108 |
-
"regression": F.
|
| 109 |
"point": F.binary_cross_entropy_with_logits(outputs["point"], targets["point"]),
|
| 110 |
"polygon": F.binary_cross_entropy_with_logits(outputs["polygon"], targets["polygon"]),
|
| 111 |
"polyline": F.binary_cross_entropy_with_logits(outputs["polyline"], targets["polyline"]),
|
|
@@ -115,4 +110,53 @@ def multitask_loss(outputs, targets):
|
|
| 115 |
return sum(losses.values()), losses
|
| 116 |
|
| 117 |
|
| 118 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Compact SatlasNet: dual-resolution temporal hierarchy, FPN, and seven tasks."""
|
| 2 |
|
| 3 |
import torch
|
| 4 |
+
import numpy as np
|
| 5 |
from torch import nn
|
| 6 |
from torch.nn import functional as F
|
| 7 |
|
| 8 |
|
| 9 |
class WindowBlock(nn.Module):
|
| 10 |
+
def __init__(self, channels, window_size=4):
|
| 11 |
+
super().__init__(); self.window_size = window_size
|
| 12 |
+
heads = max(1, channels // 16)
|
|
|
|
|
|
|
| 13 |
self.norm1 = nn.LayerNorm(channels)
|
| 14 |
self.attention = nn.MultiheadAttention(channels, heads, batch_first=True)
|
| 15 |
self.norm2 = nn.LayerNorm(channels)
|
| 16 |
+
self.mlp = nn.Sequential(nn.Linear(channels, channels * 2), nn.GELU(), nn.Linear(channels * 2, channels))
|
|
|
|
|
|
|
| 17 |
|
| 18 |
def forward(self, x):
|
| 19 |
+
batch, channels, height, width = x.shape; size = self.window_size
|
| 20 |
+
pad_h, pad_w = (-height) % size, (-width) % size
|
| 21 |
+
x = F.pad(x, (0, pad_w, 0, pad_h)); ph, pw = x.shape[-2:]
|
| 22 |
+
windows = x.permute(0, 2, 3, 1).reshape(batch, ph // size, size, pw // size, size, channels)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
windows = windows.permute(0, 1, 3, 2, 4, 5).reshape(-1, size * size, channels)
|
| 24 |
normalized = self.norm1(windows)
|
| 25 |
+
windows = windows + self.attention(normalized, normalized, normalized, need_weights=False)[0]
|
|
|
|
| 26 |
windows = windows + self.mlp(self.norm2(windows))
|
| 27 |
+
x = windows.reshape(batch, ph // size, pw // size, size, size, channels)
|
| 28 |
+
x = x.permute(0, 1, 3, 2, 4, 5).reshape(batch, ph, pw, channels)
|
| 29 |
return x[:, :height, :width].permute(0, 3, 1, 2).contiguous()
|
| 30 |
|
| 31 |
|
| 32 |
+
class HierarchicalEncoder(nn.Module):
|
| 33 |
+
def __init__(self, channels, embed_dim, patch_size, window_size):
|
| 34 |
+
super().__init__(); dimensions = [embed_dim * 2 ** index for index in range(4)]
|
| 35 |
+
self.patch = nn.Conv2d(channels, dimensions[0], patch_size, patch_size)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
self.stages = nn.ModuleList()
|
| 37 |
+
for index, dimension in enumerate(dimensions):
|
| 38 |
+
down = nn.Identity() if index == 0 else nn.Conv2d(dimensions[index - 1], dimension, 2, 2)
|
| 39 |
+
self.stages.append(nn.Sequential(down, WindowBlock(dimension, window_size)))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
|
| 41 |
+
def forward(self, x):
|
| 42 |
+
features = []; x = self.patch(x)
|
| 43 |
+
for stage in self.stages: x = stage(x); features.append(x)
|
|
|
|
|
|
|
|
|
|
| 44 |
return features
|
| 45 |
|
| 46 |
+
|
| 47 |
+
class SatlasPretrain(nn.Module):
|
| 48 |
+
"""Satlas paper task families represented by dense raster and global heads."""
|
| 49 |
+
|
| 50 |
+
def __init__(self, highres_channels=3, lowres_channels=9, output_size=512, embed_dim=8,
|
| 51 |
+
window_size=4, segmentation_classes=4, property_classes=3, classification_classes=3):
|
| 52 |
+
super().__init__(); self.output_size = output_size
|
| 53 |
+
# The official inputs and outputs remain 512px; stride 8 only limits smoke-test activations.
|
| 54 |
+
self.highres = HierarchicalEncoder(highres_channels, embed_dim, 8, window_size)
|
| 55 |
+
self.lowres = HierarchicalEncoder(lowres_channels, embed_dim, 8, window_size)
|
| 56 |
+
dimensions = [embed_dim * 2 ** index for index in range(4)]
|
| 57 |
+
self.lateral = nn.ModuleList(nn.Conv2d(dimension, embed_dim, 1) for dimension in dimensions)
|
| 58 |
+
self.sensor_fusion = nn.ModuleList(nn.Conv2d(dimension * 2, dimension, 1) for dimension in dimensions)
|
| 59 |
+
self.refine = nn.Sequential(nn.Conv2d(embed_dim, embed_dim, 3, padding=1), nn.GELU())
|
| 60 |
+
dense_channels = {"segmentation": segmentation_classes, "regression": 1,
|
| 61 |
+
"point": 1, "polygon": 1, "polyline": 1}
|
| 62 |
+
self.dense_heads = nn.ModuleDict({name: nn.Conv2d(embed_dim, count, 1) for name, count in dense_channels.items()})
|
| 63 |
+
self.property_head = nn.Linear(dimensions[-1], property_classes)
|
| 64 |
+
self.classification_head = nn.Linear(dimensions[-1], classification_classes)
|
| 65 |
+
|
| 66 |
+
@staticmethod
|
| 67 |
+
def temporal_encode(encoder, images, valid_times):
|
| 68 |
+
if images.ndim != 5: raise ValueError("sensor inputs must have shape [B,T,C,H,W]")
|
| 69 |
batch, time, channels, height, width = images.shape
|
| 70 |
+
if valid_times.shape != (batch, time) or not torch.all(valid_times.any(dim=1)):
|
| 71 |
+
raise ValueError("valid_times must be bool [B,T] with at least one valid frame per sample")
|
| 72 |
+
features = encoder(images.reshape(batch * time, channels, height, width))
|
| 73 |
+
pooled = []
|
| 74 |
+
for feature in features:
|
| 75 |
+
feature = feature.reshape(batch, time, *feature.shape[1:])
|
| 76 |
+
mask = valid_times[:, :, None, None, None]
|
| 77 |
+
pooled.append(feature.masked_fill(~mask, torch.finfo(feature.dtype).min).amax(1))
|
| 78 |
+
return pooled
|
| 79 |
+
|
| 80 |
+
def forward(self, highres_images, lowres_images, valid_highres_times, valid_lowres_times):
|
| 81 |
+
high = self.temporal_encode(self.highres, highres_images, valid_highres_times)
|
| 82 |
+
low = self.temporal_encode(self.lowres, lowres_images, valid_lowres_times)
|
| 83 |
+
fused = []
|
| 84 |
+
for high_feature, low_feature, fusion in zip(high, low, self.sensor_fusion):
|
| 85 |
+
low_feature = F.interpolate(low_feature, size=high_feature.shape[-2:], mode="bilinear", align_corners=False)
|
| 86 |
+
fused.append(fusion(torch.cat((high_feature, low_feature), dim=1)))
|
| 87 |
+
pyramid = self.lateral[-1](fused[-1])
|
| 88 |
+
for index in range(len(fused) - 2, -1, -1):
|
| 89 |
+
pyramid = F.interpolate(pyramid, size=fused[index].shape[-2:], mode="nearest") + self.lateral[index](fused[index])
|
| 90 |
+
pyramid = self.refine(pyramid)
|
| 91 |
+
outputs = {name: F.interpolate(head(pyramid), size=(self.output_size, self.output_size),
|
| 92 |
+
mode="bilinear", align_corners=False)
|
| 93 |
+
for name, head in self.dense_heads.items()}
|
| 94 |
+
global_feature = fused[-1].mean((-2, -1))
|
| 95 |
+
outputs["property"] = self.property_head(global_feature)
|
| 96 |
+
outputs["classification"] = self.classification_head(global_feature)
|
| 97 |
+
return outputs
|
| 98 |
|
| 99 |
|
| 100 |
def multitask_loss(outputs, targets):
|
| 101 |
losses = {
|
| 102 |
"segmentation": F.cross_entropy(outputs["segmentation"], targets["segmentation"].long()),
|
| 103 |
+
"regression": F.smooth_l1_loss(outputs["regression"], targets["regression"]),
|
| 104 |
"point": F.binary_cross_entropy_with_logits(outputs["point"], targets["point"]),
|
| 105 |
"polygon": F.binary_cross_entropy_with_logits(outputs["polygon"], targets["polygon"]),
|
| 106 |
"polyline": F.binary_cross_entropy_with_logits(outputs["polyline"], targets["polyline"]),
|
|
|
|
| 110 |
return sum(losses.values()), losses
|
| 111 |
|
| 112 |
|
| 113 |
+
def validate_npz(archive, config):
|
| 114 |
+
"""Strict Satlas NPZ contract shared by training and inference."""
|
| 115 |
+
required = {"highres_images", "lowres_images", "valid_highres_times", "valid_lowres_times",
|
| 116 |
+
"sample_ids", "segmentation",
|
| 117 |
+
"regression", "point", "polygon", "polyline", "property", "classification",
|
| 118 |
+
"protocol", "source"}
|
| 119 |
+
missing = required.difference(archive.files)
|
| 120 |
+
if missing: raise ValueError(f"NPZ missing fields: {sorted(missing)}")
|
| 121 |
+
d, model = config["data"], config["model"]
|
| 122 |
+
high, low = archive["highres_images"], archive["lowres_images"]
|
| 123 |
+
valid_high, valid_low = archive["valid_highres_times"], archive["valid_lowres_times"]
|
| 124 |
+
high_shape = (d["highres_time_steps"], d["highres_channels"], d["highres_size"], d["highres_size"])
|
| 125 |
+
low_shape = (d["lowres_time_steps"], d["lowres_channels"], d["lowres_size"], d["lowres_size"])
|
| 126 |
+
if high.ndim != 5 or high.shape[1:] != high_shape:
|
| 127 |
+
raise ValueError(f"highres_images must be [N,{high_shape[0]},{high_shape[1]},{high_shape[2]},{high_shape[3]}]")
|
| 128 |
+
if low.ndim != 5 or low.shape[1:] != low_shape:
|
| 129 |
+
raise ValueError(f"lowres_images must be [N,{low_shape[0]},{low_shape[1]},{low_shape[2]},{low_shape[3]}]")
|
| 130 |
+
if high.dtype != np.float32 or low.dtype != np.float32:
|
| 131 |
+
raise ValueError("sensor arrays must be float32")
|
| 132 |
+
samples = high.shape[0]
|
| 133 |
+
if samples == 0 or low.shape[0] != samples: raise ValueError("sensor arrays must share a non-zero sample count")
|
| 134 |
+
expected_masks = (("valid_highres_times", valid_high, d["highres_time_steps"]),
|
| 135 |
+
("valid_lowres_times", valid_low, d["lowres_time_steps"]))
|
| 136 |
+
for name, valid, time_steps in expected_masks:
|
| 137 |
+
if valid.shape != (samples, time_steps) or valid.dtype != np.bool_ or not valid.any(axis=1).all():
|
| 138 |
+
raise ValueError(f"{name} must be bool [N,{time_steps}] with at least one valid frame per sample")
|
| 139 |
+
if np.any(high[~valid_high]) or np.any(low[~valid_low]):
|
| 140 |
+
raise ValueError("invalid temporal slots must be zero-filled")
|
| 141 |
+
size = model["output_size"]
|
| 142 |
+
shapes = {"segmentation": (samples, size, size), "regression": (samples, 1, size, size),
|
| 143 |
+
"point": (samples, 1, size, size), "polygon": (samples, 1, size, size),
|
| 144 |
+
"polyline": (samples, 1, size, size), "property": (samples,), "classification": (samples,)}
|
| 145 |
+
for key, shape in shapes.items():
|
| 146 |
+
if archive[key].shape != shape: raise ValueError(f"{key} must have shape {shape}")
|
| 147 |
+
if archive["segmentation"].dtype != np.int64 or archive["property"].dtype != np.int64 or archive["classification"].dtype != np.int64:
|
| 148 |
+
raise ValueError("segmentation/property/classification targets must be int64")
|
| 149 |
+
for key in ("regression", "point", "polygon", "polyline"):
|
| 150 |
+
if archive[key].dtype != np.float32: raise ValueError(f"{key} must be float32")
|
| 151 |
+
if archive["segmentation"].min() < 0 or archive["segmentation"].max() >= model["segmentation_classes"]:
|
| 152 |
+
raise ValueError("segmentation labels are outside configured class range")
|
| 153 |
+
for key, classes in (("property", model["property_classes"]), ("classification", model["classification_classes"])):
|
| 154 |
+
if archive[key].min() < 0 or archive[key].max() >= classes: raise ValueError(f"{key} labels are outside configured class range")
|
| 155 |
+
sample_ids = archive["sample_ids"]
|
| 156 |
+
if sample_ids.shape != (samples,) or len(np.unique(sample_ids)) != samples: raise ValueError("sample_ids must be unique [N]")
|
| 157 |
+
if str(archive["protocol"]) != d["protocol"]: raise ValueError("NPZ protocol does not match configuration")
|
| 158 |
+
source = np.asarray(archive["source"])
|
| 159 |
+
if source.ndim != 0 or not str(source).strip(): raise ValueError("source must be a non-empty scalar string")
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
__all__ = ["SatlasPretrain", "multitask_loss", "validate_npz"]
|
scripts/fake_data.py
CHANGED
|
@@ -1,49 +1,59 @@
|
|
| 1 |
-
|
| 2 |
-
from pathlib import Path
|
| 3 |
|
|
|
|
|
|
|
| 4 |
import numpy as np
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
|
| 6 |
|
| 7 |
def main():
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
polygon[index, 0, args.image_size // 3:2 * args.image_size // 3, args.image_size // 3:2 * args.image_size // 3] = 1
|
| 27 |
-
polyline[index, 0, np.arange(args.image_size), (np.arange(args.image_size) + index) % args.image_size] = 1
|
| 28 |
-
regression = images[:, -1, :1] * 0.75 + images[:, 0, 1:2] * 0.25
|
| 29 |
-
properties = np.arange(args.samples, dtype=np.int64) % 3
|
| 30 |
-
classification = (images.mean(axis=(1, 2, 3, 4)) * 3).astype(np.int64).clip(0, 2)
|
| 31 |
-
output = Path(args.output)
|
| 32 |
-
output.parent.mkdir(parents=True, exist_ok=True)
|
| 33 |
-
np.savez_compressed(
|
| 34 |
-
output,
|
| 35 |
-
images=images,
|
| 36 |
-
segmentation=segmentation,
|
| 37 |
-
regression=regression.astype(np.float32),
|
| 38 |
-
point=point,
|
| 39 |
-
polygon=polygon,
|
| 40 |
-
polyline=polyline,
|
| 41 |
-
property=properties,
|
| 42 |
-
classification=classification,
|
| 43 |
-
)
|
| 44 |
-
print(f"images: {images.shape}")
|
| 45 |
-
print(f"saved: {output}")
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
if __name__ == "__main__":
|
| 49 |
-
main()
|
|
|
|
| 1 |
+
"""Generate tiny Satlas-compatible independent train/test NPZ archives."""
|
|
|
|
| 2 |
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
import numpy as np
|
| 6 |
+
import yaml
|
| 7 |
+
|
| 8 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def make_split(samples, config, seed):
|
| 12 |
+
rng = np.random.default_rng(seed); d = config["data"]; size = d["highres_size"]
|
| 13 |
+
highres = rng.random((samples, d["highres_time_steps"], d["highres_channels"], size, size), dtype=np.float32)
|
| 14 |
+
lowres = rng.random((samples, d["lowres_time_steps"], d["lowres_channels"],
|
| 15 |
+
d["lowres_size"], d["lowres_size"]), dtype=np.float32)
|
| 16 |
+
valid_highres_times = np.ones((samples, d["highres_time_steps"]), dtype=np.bool_)
|
| 17 |
+
valid_lowres_times = np.ones((samples, d["lowres_time_steps"]), dtype=np.bool_)
|
| 18 |
+
# Exercise independent missing-observation masks even with a single smoke sample.
|
| 19 |
+
valid_highres_times[:, -1] = False
|
| 20 |
+
valid_lowres_times[:, -2:] = False
|
| 21 |
+
highres[~valid_highres_times] = 0
|
| 22 |
+
lowres[~valid_lowres_times] = 0
|
| 23 |
+
yy, xx = np.mgrid[:size, :size]; segmentation = np.empty((samples, size, size), dtype=np.int64)
|
| 24 |
+
point = np.zeros((samples, 1, size, size), np.float32); polygon = np.zeros_like(point); polyline = np.zeros_like(point)
|
| 25 |
+
for index in range(samples):
|
| 26 |
+
segmentation[index] = (xx >= size // 2) + 2 * (yy >= size // 2)
|
| 27 |
+
center = size // 4 + index * 8; point[index, 0, center-2:center+3, center-2:center+3] = 1
|
| 28 |
+
polygon[index, 0, size//3:2*size//3, size//3:2*size//3] = 1
|
| 29 |
+
polyline[index, 0, np.arange(size), (np.arange(size) + index * 8) % size] = 1
|
| 30 |
+
last_valid = valid_highres_times.sum(1) - 1
|
| 31 |
+
regression = np.stack([0.7 * highres[index, last_valid[index], :1] + 0.3 * highres[index, 0, 1:2]
|
| 32 |
+
for index in range(samples)]).astype(np.float32)
|
| 33 |
+
return dict(highres_images=highres, lowres_images=lowres,
|
| 34 |
+
valid_highres_times=valid_highres_times, valid_lowres_times=valid_lowres_times,
|
| 35 |
+
sample_ids=np.asarray([f"sample-{seed}-{index}" for index in range(samples)]), segmentation=segmentation,
|
| 36 |
+
regression=regression, point=point, polygon=polygon, polyline=polyline,
|
| 37 |
+
property=np.arange(samples, dtype=np.int64) % 3,
|
| 38 |
+
classification=(np.arange(samples, dtype=np.int64) + 1) % 3)
|
| 39 |
|
| 40 |
|
| 41 |
def main():
|
| 42 |
+
config = yaml.safe_load((ROOT / "conf/config.yaml").read_text()); d = config["data"]
|
| 43 |
+
output = ROOT / d["root"]; output.mkdir(parents=True, exist_ok=True)
|
| 44 |
+
for split, count, seed in (("train", d["train_samples"], config["seed"]), ("test", d["test_samples"], config["seed"] + 1)):
|
| 45 |
+
np.savez_compressed(output / f"{split}.npz", **make_split(count, config, seed),
|
| 46 |
+
source=np.asarray("synthetic"), protocol=np.asarray(d["protocol"]))
|
| 47 |
+
(output / "format.json").write_text(json.dumps({
|
| 48 |
+
"protocol": d["protocol"], "source": "synthetic",
|
| 49 |
+
"highres_images": "float32 [N,4,3,512,512] NAIP RGB",
|
| 50 |
+
"lowres_images": "float32 [N,8,9,512,512] Sentinel-2 multispectral",
|
| 51 |
+
"valid_highres_times": "bool [N,4], false slots are zero-filled",
|
| 52 |
+
"valid_lowres_times": "bool [N,8], false slots are zero-filled",
|
| 53 |
+
"dense_targets": "segmentation [N,512,512]; regression/point/polygon/polyline [N,1,512,512]",
|
| 54 |
+
"global_targets": "property/classification int64 [N]"
|
| 55 |
+
}, indent=2) + "\n")
|
| 56 |
+
print(f"created {output / 'train.npz'} and {output / 'test.npz'}")
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
if __name__ == "__main__": main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
scripts/inference.py
CHANGED
|
@@ -1,52 +1,47 @@
|
|
|
|
|
|
|
|
| 1 |
import argparse
|
|
|
|
| 2 |
from pathlib import Path
|
| 3 |
-
import sys
|
| 4 |
-
|
| 5 |
import numpy as np
|
| 6 |
import torch
|
| 7 |
import yaml
|
| 8 |
|
| 9 |
-
|
| 10 |
-
from model.satlaspretrain import SatlasPretrain
|
| 11 |
|
| 12 |
|
| 13 |
def main():
|
| 14 |
-
parser = argparse.ArgumentParser(description=
|
| 15 |
-
parser.add_argument("--config", default="conf/config.yaml")
|
| 16 |
-
parser.add_argument("--
|
| 17 |
-
parser.add_argument("--
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
model = SatlasPretrain(**
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
)
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
|
|
|
| 33 |
predictions = {}
|
| 34 |
for name, value in outputs.items():
|
| 35 |
-
if name
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
output_dir.mkdir(parents=True, exist_ok=True)
|
| 45 |
-
for task, prediction in predictions.items():
|
| 46 |
-
output = output_dir / f"{task}.npy"
|
| 47 |
-
np.save(output, prediction)
|
| 48 |
-
print(f"{task}: {prediction.shape} -> {output}")
|
| 49 |
|
| 50 |
|
| 51 |
-
if __name__ == "__main__":
|
| 52 |
-
main()
|
|
|
|
| 1 |
+
"""Run checkpoint-backed SatlasNet inference on a test NPZ."""
|
| 2 |
+
|
| 3 |
import argparse
|
| 4 |
+
import importlib.util
|
| 5 |
from pathlib import Path
|
|
|
|
|
|
|
| 6 |
import numpy as np
|
| 7 |
import torch
|
| 8 |
import yaml
|
| 9 |
|
| 10 |
+
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
| 11 |
|
| 12 |
|
| 13 |
def main():
|
| 14 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 15 |
+
parser.add_argument("--config", type=Path, default=ROOT / "conf/config.yaml"); parser.add_argument("--data", type=Path)
|
| 16 |
+
parser.add_argument("--checkpoint", type=Path); parser.add_argument("--output-dir", type=Path)
|
| 17 |
+
parser.add_argument("--device", choices=("auto", "cpu", "cuda"), default="auto"); args = parser.parse_args()
|
| 18 |
+
config = yaml.safe_load(args.config.read_text()); checkpoint_path = args.checkpoint or ROOT / config["paths"]["checkpoint"]
|
| 19 |
+
if not checkpoint_path.is_file(): raise FileNotFoundError(f"checkpoint not found: {checkpoint_path}")
|
| 20 |
+
spec = importlib.util.spec_from_file_location("satlaspretrain", ROOT / "model/satlaspretrain.py")
|
| 21 |
+
module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module)
|
| 22 |
+
checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False)
|
| 23 |
+
model = module.SatlasPretrain(**config["model"]); model.load_state_dict(checkpoint["model"])
|
| 24 |
+
use_cuda = torch.cuda.is_available() and args.device != "cpu"
|
| 25 |
+
if args.device == "cuda" and not use_cuda: raise RuntimeError("CUDA requested but unavailable")
|
| 26 |
+
device = torch.device("cuda" if use_cuda else "cpu"); model.to(device).eval()
|
| 27 |
+
data_path = args.data or ROOT / config["data"]["root"] / "test.npz"; archive = np.load(data_path)
|
| 28 |
+
module.validate_npz(archive, config)
|
| 29 |
+
with torch.inference_mode():
|
| 30 |
+
outputs = model(torch.from_numpy(archive["highres_images"]).to(device),
|
| 31 |
+
torch.from_numpy(archive["lowres_images"]).to(device),
|
| 32 |
+
torch.from_numpy(archive["valid_highres_times"]).to(device),
|
| 33 |
+
torch.from_numpy(archive["valid_lowres_times"]).to(device))
|
| 34 |
predictions = {}
|
| 35 |
for name, value in outputs.items():
|
| 36 |
+
if name in ("segmentation", "property", "classification"): value = value.softmax(1)
|
| 37 |
+
elif name != "regression": value = value.sigmoid()
|
| 38 |
+
predictions[name] = value.cpu().numpy().astype(np.float32)
|
| 39 |
+
output = args.output_dir or ROOT / config["paths"]["inference_dir"]; output.mkdir(parents=True, exist_ok=True)
|
| 40 |
+
np.savez_compressed(output / "predictions.npz", **predictions, checkpoint=np.asarray(str(checkpoint_path)),
|
| 41 |
+
sample_ids=archive["sample_ids"],
|
| 42 |
+
source=archive["source"],
|
| 43 |
+
protocol=archive["protocol"] if "protocol" in archive else np.asarray("provided_npz"))
|
| 44 |
+
print(f"inference={output / 'predictions.npz'} checkpoint={checkpoint_path}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
|
| 46 |
|
| 47 |
+
if __name__ == "__main__": main()
|
|
|
scripts/result.py
CHANGED
|
@@ -1,118 +1,160 @@
|
|
|
|
|
|
|
|
| 1 |
import argparse
|
| 2 |
import json
|
|
|
|
| 3 |
from pathlib import Path
|
| 4 |
|
|
|
|
| 5 |
import numpy as np
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
return np.logical_or.reduce(neighborhoods)
|
| 37 |
|
| 38 |
|
| 39 |
-
def
|
| 40 |
-
prediction =
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
return float(2 * precision * recall / (precision + recall
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
def colorize_segmentation(segmentation):
|
| 48 |
-
palette = np.asarray([[0, 0, 0], [32, 180, 96], [40, 120, 220], [230, 180, 40]], dtype=np.uint8)
|
| 49 |
-
return palette[segmentation % len(palette)]
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
def colorize_scalar(values):
|
| 53 |
-
values = np.clip(values, 0, 1)
|
| 54 |
-
return np.stack((values * 255, (1 - np.abs(values - 0.5) * 2) * 255, (1 - values) * 255), axis=-1).astype(np.uint8)
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
def overlay(image, mask, color):
|
| 58 |
-
output = image.copy()
|
| 59 |
-
selected = mask >= 0.5
|
| 60 |
-
output[selected] = (0.35 * output[selected] + 0.65 * np.asarray(color)).astype(np.uint8)
|
| 61 |
-
return output
|
| 62 |
|
| 63 |
|
| 64 |
-
def
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
colorize_scalar(target["regression"][0, 0]),
|
| 71 |
-
colorize_scalar(prediction["regression"][0, 0]),
|
| 72 |
-
overlay(image, target["point"][0, 0], [255, 230, 40]),
|
| 73 |
-
overlay(image, prediction["point"][0, 0], [255, 230, 40]),
|
| 74 |
-
overlay(image, target["polygon"][0, 0], [255, 70, 190]),
|
| 75 |
-
overlay(image, prediction["polygon"][0, 0], [255, 70, 190]),
|
| 76 |
-
overlay(image, target["polyline"][0, 0], [40, 240, 255]),
|
| 77 |
-
overlay(image, prediction["polyline"][0, 0], [40, 240, 255]),
|
| 78 |
-
]
|
| 79 |
-
separator = np.full((image.shape[0], 2, 3), 255, dtype=np.uint8)
|
| 80 |
-
comparison = panels[0]
|
| 81 |
-
for panel in panels[1:]:
|
| 82 |
-
comparison = np.concatenate((comparison, separator, panel), axis=1)
|
| 83 |
-
with open(path, "wb") as handle:
|
| 84 |
-
handle.write(f"P6\n{comparison.shape[1]} {comparison.shape[0]}\n255\n".encode("ascii"))
|
| 85 |
-
handle.write(comparison.tobytes())
|
| 86 |
|
| 87 |
|
| 88 |
def main():
|
| 89 |
-
parser = argparse.ArgumentParser(description=
|
| 90 |
-
parser.add_argument("--
|
| 91 |
-
parser.add_argument("--target",
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
target = np.load(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
metrics = {
|
| 98 |
-
"
|
| 99 |
-
"
|
| 100 |
-
"
|
| 101 |
-
"
|
| 102 |
-
"
|
| 103 |
-
"
|
| 104 |
-
"
|
|
|
|
|
|
|
|
|
|
| 105 |
}
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Compute Satlas semantic, instance, geometric, and global metrics."""
|
| 2 |
+
|
| 3 |
import argparse
|
| 4 |
import json
|
| 5 |
+
from collections import deque
|
| 6 |
from pathlib import Path
|
| 7 |
|
| 8 |
+
import matplotlib.pyplot as plt
|
| 9 |
import numpy as np
|
| 10 |
+
import torch
|
| 11 |
+
from torch.nn import functional as F
|
| 12 |
+
import yaml
|
| 13 |
+
|
| 14 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def precision_recall_f1(tp, fp, fn):
|
| 18 |
+
precision = tp / max(tp + fp, 1); recall = tp / max(tp + fn, 1)
|
| 19 |
+
return float(precision), float(recall), float(2 * precision * recall / max(precision + recall, 1e-12))
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def connected_components(mask):
|
| 23 |
+
"""Return 4-connected component coordinate arrays without SciPy."""
|
| 24 |
+
mask = np.asarray(mask, dtype=bool); visited = np.zeros_like(mask); components = []
|
| 25 |
+
height, width = mask.shape
|
| 26 |
+
for row, col in np.argwhere(mask):
|
| 27 |
+
if visited[row, col]: continue
|
| 28 |
+
queue = deque([(int(row), int(col))]); visited[row, col] = True; points = []
|
| 29 |
+
while queue:
|
| 30 |
+
y, x = queue.popleft(); points.append((y, x))
|
| 31 |
+
for ny, nx in ((y - 1, x), (y + 1, x), (y, x - 1), (y, x + 1)):
|
| 32 |
+
if 0 <= ny < height and 0 <= nx < width and mask[ny, nx] and not visited[ny, nx]:
|
| 33 |
+
visited[ny, nx] = True; queue.append((ny, nx))
|
| 34 |
+
components.append(np.asarray(points, dtype=np.int32))
|
| 35 |
+
return components
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def local_peaks(scores, threshold, radius):
|
| 39 |
+
tensor = torch.from_numpy(scores)[None, None]
|
| 40 |
+
pooled = F.max_pool2d(tensor, kernel_size=2 * radius + 1, stride=1, padding=radius)
|
| 41 |
+
candidates = np.argwhere(np.logical_and(scores >= threshold, scores == pooled[0, 0].numpy()))
|
| 42 |
+
if not len(candidates): return np.empty((0, 2), dtype=np.float32)
|
| 43 |
+
order = sorted(candidates, key=lambda point: scores[tuple(point)], reverse=True); selected = []
|
| 44 |
+
for point in order:
|
| 45 |
+
if all(np.sum((point - previous) ** 2) > radius ** 2 for previous in selected): selected.append(point)
|
| 46 |
+
return np.asarray(selected, dtype=np.float32).reshape(-1, 2)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def point_counts(predictions, targets, threshold, nms_radius, tolerance):
|
| 50 |
+
tp = fp = fn = 0
|
| 51 |
+
for scores, target in zip(predictions[:, 0], targets[:, 0]):
|
| 52 |
+
predicted = local_peaks(scores, threshold, nms_radius)
|
| 53 |
+
truth_components = connected_components(target >= 0.5)
|
| 54 |
+
truth = np.asarray([component.mean(0) for component in truth_components], dtype=np.float32).reshape(-1, 2)
|
| 55 |
+
candidates = sorted((float(np.linalg.norm(p - t)), pi, ti) for pi, p in enumerate(predicted)
|
| 56 |
+
for ti, t in enumerate(truth) if np.linalg.norm(p - t) <= tolerance)
|
| 57 |
+
matched_pred, matched_truth = set(), set()
|
| 58 |
+
for _, pi, ti in candidates:
|
| 59 |
+
if pi not in matched_pred and ti not in matched_truth: matched_pred.add(pi); matched_truth.add(ti)
|
| 60 |
+
tp += len(matched_pred); fp += len(predicted) - len(matched_pred); fn += len(truth) - len(matched_truth)
|
| 61 |
+
return tp, fp, fn
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def component_mask(component, shape):
|
| 65 |
+
mask = np.zeros(shape, dtype=bool); mask[component[:, 0], component[:, 1]] = True; return mask
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def polygon_counts(predictions, targets, threshold, iou_threshold):
|
| 69 |
+
tp = fp = fn = 0; matched_ious = []
|
| 70 |
+
for scores, target in zip(predictions[:, 0], targets[:, 0]):
|
| 71 |
+
predicted = connected_components(scores >= threshold); truth = connected_components(target >= 0.5)
|
| 72 |
+
pred_masks = [component_mask(component, scores.shape) for component in predicted]
|
| 73 |
+
truth_masks = [component_mask(component, scores.shape) for component in truth]
|
| 74 |
+
candidates = []
|
| 75 |
+
for pi, pred in enumerate(pred_masks):
|
| 76 |
+
for ti, actual in enumerate(truth_masks):
|
| 77 |
+
union = np.logical_or(pred, actual).sum(); iou = np.logical_and(pred, actual).sum() / max(union, 1)
|
| 78 |
+
if iou >= iou_threshold: candidates.append((float(iou), pi, ti))
|
| 79 |
+
matched_pred, matched_truth = set(), set()
|
| 80 |
+
for iou, pi, ti in sorted(candidates, reverse=True):
|
| 81 |
+
if pi not in matched_pred and ti not in matched_truth:
|
| 82 |
+
matched_pred.add(pi); matched_truth.add(ti); matched_ious.append(iou)
|
| 83 |
+
tp += len(matched_pred); fp += len(predicted) - len(matched_pred); fn += len(truth) - len(matched_truth)
|
| 84 |
+
return tp, fp, fn, float(np.mean(matched_ious)) if matched_ious else 0.0
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def dilate(mask, radius):
|
| 88 |
+
height, width = mask.shape[-2:]; padded = np.pad(mask, ((0, 0), (radius, radius), (radius, radius)))
|
| 89 |
+
neighborhoods = [padded[:, dy:dy + height, dx:dx + width]
|
| 90 |
+
for dy in range(2 * radius + 1) for dx in range(2 * radius + 1)
|
| 91 |
+
if (dy - radius) ** 2 + (dx - radius) ** 2 <= radius ** 2]
|
| 92 |
return np.logical_or.reduce(neighborhoods)
|
| 93 |
|
| 94 |
|
| 95 |
+
def tolerant_line_metrics(prediction, target, threshold, tolerance):
|
| 96 |
+
predicted, truth = prediction[:, 0] >= threshold, target[:, 0] >= 0.5
|
| 97 |
+
matched_pred = np.logical_and(predicted, dilate(truth, tolerance)).sum()
|
| 98 |
+
matched_truth = np.logical_and(truth, dilate(predicted, tolerance)).sum()
|
| 99 |
+
precision = matched_pred / max(predicted.sum(), 1); recall = matched_truth / max(truth.sum(), 1)
|
| 100 |
+
return float(precision), float(recall), float(2 * precision * recall / max(precision + recall, 1e-12))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
|
| 102 |
|
| 103 |
+
def mean_iou(prediction, target, classes):
|
| 104 |
+
values = []
|
| 105 |
+
for label in range(classes):
|
| 106 |
+
pred, truth = prediction == label, target == label; union = np.logical_or(pred, truth).sum()
|
| 107 |
+
if union: values.append(np.logical_and(pred, truth).sum() / union)
|
| 108 |
+
return float(np.mean(values))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
|
| 110 |
|
| 111 |
def main():
|
| 112 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 113 |
+
parser.add_argument("--config", type=Path, default=ROOT / "conf/config.yaml"); parser.add_argument("--prediction", type=Path)
|
| 114 |
+
parser.add_argument("--target", type=Path); parser.add_argument("--output-dir", type=Path); args = parser.parse_args()
|
| 115 |
+
config = yaml.safe_load(args.config.read_text()); evaluation = config["evaluation"]
|
| 116 |
+
pred_path = args.prediction or ROOT / config["paths"]["inference_dir"] / "predictions.npz"
|
| 117 |
+
target_path = args.target or ROOT / config["data"]["root"] / "test.npz"
|
| 118 |
+
if not pred_path.is_file(): raise FileNotFoundError("Run inference before evaluation")
|
| 119 |
+
prediction, target = np.load(pred_path), np.load(target_path)
|
| 120 |
+
if str(prediction["protocol"]) != str(target["protocol"]) or str(target["protocol"]) != config["data"]["protocol"]:
|
| 121 |
+
raise ValueError("prediction, target, and configuration protocols do not match")
|
| 122 |
+
if str(prediction["source"]) != str(target["source"]): raise ValueError("prediction and target sources do not match")
|
| 123 |
+
if prediction["sample_ids"].shape != target["sample_ids"].shape or not np.array_equal(prediction["sample_ids"], target["sample_ids"]):
|
| 124 |
+
raise ValueError("prediction sample_ids do not exactly match target identity/order")
|
| 125 |
+
segmentation = prediction["segmentation"].argmax(1)
|
| 126 |
+
point = precision_recall_f1(*point_counts(prediction["point"], target["point"], evaluation["point_peak_threshold"],
|
| 127 |
+
evaluation["point_nms_radius"], evaluation["point_distance_tolerance"]))
|
| 128 |
+
polygon_counts_result = polygon_counts(prediction["polygon"], target["polygon"], evaluation["mask_threshold"],
|
| 129 |
+
evaluation["polygon_iou_threshold"])
|
| 130 |
+
polygon = precision_recall_f1(*polygon_counts_result[:3]); line = tolerant_line_metrics(
|
| 131 |
+
prediction["polyline"], target["polyline"], evaluation["mask_threshold"], evaluation["polyline_distance_tolerance"])
|
| 132 |
metrics = {
|
| 133 |
+
"segmentation_mIoU": mean_iou(segmentation, target["segmentation"], config["model"]["segmentation_classes"]),
|
| 134 |
+
"regression_MAE": float(np.abs(prediction["regression"] - target["regression"]).mean()),
|
| 135 |
+
"point_precision": point[0], "point_recall": point[1], "point_F1": point[2],
|
| 136 |
+
"polygon_precision": polygon[0], "polygon_recall": polygon[1], "polygon_F1": polygon[2],
|
| 137 |
+
"polygon_matched_IoU": polygon_counts_result[3],
|
| 138 |
+
"polyline_precision": line[0], "polyline_recall": line[1], "polyline_F1": line[2],
|
| 139 |
+
"property_accuracy": float((prediction["property"].argmax(1) == target["property"]).mean()),
|
| 140 |
+
"classification_accuracy": float((prediction["classification"].argmax(1) == target["classification"]).mean()),
|
| 141 |
+
"samples": int(len(segmentation)), "protocol": str(prediction["protocol"]), "checkpoint": str(prediction["checkpoint"]),
|
| 142 |
+
"source": str(prediction["source"]), "identity_verified": True,
|
| 143 |
}
|
| 144 |
+
output = args.output_dir or ROOT / config["paths"]["evaluation_dir"]; output.mkdir(parents=True, exist_ok=True)
|
| 145 |
+
(output / "metrics.json").write_text(json.dumps(metrics, indent=2) + "\n")
|
| 146 |
+
last_highres = np.flatnonzero(target["valid_highres_times"][0])[-1]
|
| 147 |
+
last_lowres = np.flatnonzero(target["valid_lowres_times"][0])[-1]
|
| 148 |
+
image = np.clip(target["highres_images"][0, last_highres].transpose(1, 2, 0), 0, 1)
|
| 149 |
+
figure, axes = plt.subplots(2, 4, figsize=(12, 6)); panels = [
|
| 150 |
+
("NAIP RGB", image, None), ("Segmentation GT", target["segmentation"][0], "tab20"),
|
| 151 |
+
("Segmentation pred", segmentation[0], "tab20"), ("Regression", prediction["regression"][0, 0], "viridis"),
|
| 152 |
+
("Point", prediction["point"][0, 0], "magma"), ("Polygon", prediction["polygon"][0, 0], "magma"),
|
| 153 |
+
("Polyline", prediction["polyline"][0, 0], "magma"),
|
| 154 |
+
("Sentinel band", target["lowres_images"][0, last_lowres, 0], "viridis")]
|
| 155 |
+
for axis, (title, panel, cmap) in zip(axes.flat, panels): axis.imshow(panel, cmap=cmap); axis.set_title(title); axis.axis("off")
|
| 156 |
+
figure.tight_layout(); figure.savefig(output / "multitask_predictions.png", dpi=150); plt.close(figure)
|
| 157 |
+
print(json.dumps(metrics, indent=2)); print(f"evaluation={output}")
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
if __name__ == "__main__": main()
|
scripts/train.py
CHANGED
|
@@ -1,107 +1,91 @@
|
|
|
|
|
|
|
|
| 1 |
import argparse
|
|
|
|
|
|
|
| 2 |
import os
|
|
|
|
|
|
|
| 3 |
from pathlib import Path
|
| 4 |
-
import sys
|
| 5 |
-
|
| 6 |
import numpy as np
|
| 7 |
import torch
|
| 8 |
import yaml
|
|
|
|
| 9 |
from torch.nn.parallel import DistributedDataParallel
|
| 10 |
from torch.utils.data import DataLoader, Dataset, DistributedSampler
|
| 11 |
|
| 12 |
-
|
| 13 |
-
from model.satlaspretrain import SatlasPretrain, multitask_loss
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
class NpzDataset(Dataset):
|
| 17 |
-
def __init__(self, path):
|
| 18 |
-
data = np.load(path)
|
| 19 |
-
self.data = {key: data[key] for key in data.files}
|
| 20 |
-
|
| 21 |
-
def __len__(self):
|
| 22 |
-
return len(self.data["images"])
|
| 23 |
|
| 24 |
-
def __getitem__(self, index):
|
| 25 |
-
return {key: torch.as_tensor(value[index]) for key, value in self.data.items()}
|
| 26 |
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
-
def load_config(path):
|
| 29 |
-
with open(path, encoding="utf-8") as handle:
|
| 30 |
-
return yaml.safe_load(handle)
|
| 31 |
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
|
| 39 |
|
| 40 |
def main():
|
| 41 |
-
parser = argparse.ArgumentParser(description=
|
| 42 |
-
parser.add_argument("--config", default="conf/config.yaml")
|
| 43 |
-
parser.add_argument("--
|
| 44 |
-
parser.
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
model =
|
| 58 |
-
if
|
| 59 |
-
model = DistributedDataParallel(model, device_ids=[local_rank] if
|
| 60 |
-
optimizer = torch.optim.
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
epochs = args.epochs or config["training"]["epochs"]
|
| 66 |
-
for epoch in range(epochs):
|
| 67 |
-
if sampler is not None:
|
| 68 |
-
sampler.set_epoch(epoch)
|
| 69 |
-
model.train()
|
| 70 |
-
epoch_losses = []
|
| 71 |
for batch in loader:
|
| 72 |
-
batch = {key: value.to(device) for key, value in batch.items()}
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
best_validation = val_history[-1]
|
| 98 |
-
torch.save(checkpoint, save_dir / "best.pth")
|
| 99 |
-
if local_rank == 0:
|
| 100 |
-
print(f"saved: {save_dir / 'last.pth'}")
|
| 101 |
-
print(f"best: {save_dir / 'best.pth'}")
|
| 102 |
-
if distributed:
|
| 103 |
-
torch.distributed.destroy_process_group()
|
| 104 |
|
| 105 |
|
| 106 |
-
if __name__ == "__main__":
|
| 107 |
-
main()
|
|
|
|
| 1 |
+
"""Train compact SatlasNet on seven task families; supports AMP and torchrun."""
|
| 2 |
+
|
| 3 |
import argparse
|
| 4 |
+
import importlib.util
|
| 5 |
+
import json
|
| 6 |
import os
|
| 7 |
+
from contextlib import nullcontext
|
| 8 |
+
from functools import partial
|
| 9 |
from pathlib import Path
|
|
|
|
|
|
|
| 10 |
import numpy as np
|
| 11 |
import torch
|
| 12 |
import yaml
|
| 13 |
+
from torch import distributed as dist
|
| 14 |
from torch.nn.parallel import DistributedDataParallel
|
| 15 |
from torch.utils.data import DataLoader, Dataset, DistributedSampler
|
| 16 |
|
| 17 |
+
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
|
|
|
|
|
|
| 19 |
|
| 20 |
+
def load_module():
|
| 21 |
+
spec = importlib.util.spec_from_file_location("satlaspretrain", ROOT / "model/satlaspretrain.py")
|
| 22 |
+
module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); return module
|
| 23 |
|
|
|
|
|
|
|
|
|
|
| 24 |
|
| 25 |
+
class NpzDataset(Dataset):
|
| 26 |
+
keys = ("highres_images", "lowres_images", "valid_highres_times", "valid_lowres_times",
|
| 27 |
+
"segmentation", "regression", "point", "polygon", "polyline", "property", "classification")
|
| 28 |
+
def __init__(self, path, config, module):
|
| 29 |
+
archive = np.load(path)
|
| 30 |
+
module.validate_npz(archive, config)
|
| 31 |
+
self.data = {key: archive[key] for key in self.keys}
|
| 32 |
+
self.source = str(archive["source"])
|
| 33 |
+
self.protocol = str(archive["protocol"]) if "protocol" in archive else "provided_npz"
|
| 34 |
+
def __len__(self): return len(self.data["highres_images"])
|
| 35 |
+
def __getitem__(self, index): return {key: torch.as_tensor(value[index]) for key, value in self.data.items()}
|
| 36 |
|
| 37 |
|
| 38 |
def main():
|
| 39 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 40 |
+
parser.add_argument("--config", type=Path, default=ROOT / "conf/config.yaml"); parser.add_argument("--data", type=Path)
|
| 41 |
+
parser.add_argument("--checkpoint", type=Path); parser.add_argument("--device", choices=("auto", "cpu", "cuda"))
|
| 42 |
+
args = parser.parse_args(); config = yaml.safe_load(args.config.read_text()); train_cfg = config["training"]
|
| 43 |
+
world_size, rank, local_rank = int(os.environ.get("WORLD_SIZE", 1)), int(os.environ.get("RANK", 0)), int(os.environ.get("LOCAL_RANK", 0))
|
| 44 |
+
requested = args.device or config["runtime"]["device"]; use_cuda = torch.cuda.is_available() and requested != "cpu"
|
| 45 |
+
if requested == "cuda" and not use_cuda: raise RuntimeError("CUDA requested but unavailable")
|
| 46 |
+
if world_size > 1: dist.init_process_group("nccl" if use_cuda else "gloo")
|
| 47 |
+
device = torch.device(f"cuda:{local_rank}" if use_cuda else "cpu")
|
| 48 |
+
if use_cuda: torch.cuda.set_device(local_rank)
|
| 49 |
+
torch.manual_seed(config["seed"] + rank)
|
| 50 |
+
module = load_module()
|
| 51 |
+
data_path = args.data or ROOT / config["data"]["root"] / "train.npz"; dataset = NpzDataset(data_path, config, module)
|
| 52 |
+
sampler = DistributedSampler(dataset, shuffle=True) if world_size > 1 else None
|
| 53 |
+
loader = DataLoader(dataset, batch_size=train_cfg["batch_size"], sampler=sampler, shuffle=sampler is None,
|
| 54 |
+
num_workers=train_cfg["num_workers"], pin_memory=use_cuda)
|
| 55 |
+
model = module.SatlasPretrain(**config["model"]).to(device); raw_model = model
|
| 56 |
+
if world_size > 1:
|
| 57 |
+
model = DistributedDataParallel(model, device_ids=[local_rank] if use_cuda else None); raw_model = model.module
|
| 58 |
+
optimizer = torch.optim.AdamW(model.parameters(), lr=train_cfg["learning_rate"], weight_decay=train_cfg["weight_decay"])
|
| 59 |
+
amp = bool(config["runtime"]["amp"] and use_cuda); scaler = torch.amp.GradScaler("cuda", enabled=amp); history = []
|
| 60 |
+
for epoch in range(train_cfg["epochs"]):
|
| 61 |
+
if sampler is not None: sampler.set_epoch(epoch)
|
| 62 |
+
model.train(); totals = {"total": 0.0}; steps = 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
for batch in loader:
|
| 64 |
+
batch = {key: value.to(device, non_blocking=use_cuda) for key, value in batch.items()}
|
| 65 |
+
autocast = partial(torch.amp.autocast, "cuda") if amp else nullcontext
|
| 66 |
+
with autocast():
|
| 67 |
+
outputs = model(batch["highres_images"], batch["lowres_images"],
|
| 68 |
+
batch["valid_highres_times"], batch["valid_lowres_times"])
|
| 69 |
+
loss, parts = module.multitask_loss(outputs, batch)
|
| 70 |
+
if not torch.isfinite(loss): raise ValueError("non-finite multitask loss")
|
| 71 |
+
optimizer.zero_grad(set_to_none=True); scaler.scale(loss).backward(); scaler.step(optimizer); scaler.update()
|
| 72 |
+
totals["total"] += loss.detach().item(); steps += 1
|
| 73 |
+
for name, value in parts.items(): totals[name] = totals.get(name, 0.0) + value.detach().item()
|
| 74 |
+
names = list(totals); statistics = torch.tensor([totals[name] for name in names] + [steps], dtype=torch.float64, device=device)
|
| 75 |
+
if world_size > 1: dist.all_reduce(statistics, op=dist.ReduceOp.SUM)
|
| 76 |
+
global_steps = max(statistics[-1].item(), 1)
|
| 77 |
+
record = {"epoch": epoch + 1, **{f"{name}_loss": statistics[index].item() / global_steps for index, name in enumerate(names)}}; history.append(record)
|
| 78 |
+
if rank == 0: print(json.dumps(record))
|
| 79 |
+
if rank == 0:
|
| 80 |
+
checkpoint_path = args.checkpoint or ROOT / config["paths"]["checkpoint"]; checkpoint_path.parent.mkdir(parents=True, exist_ok=True)
|
| 81 |
+
torch.save({"model": raw_model.state_dict(), "optimizer": optimizer.state_dict(), "scaler": scaler.state_dict() if amp else None,
|
| 82 |
+
"config": config, "epoch": train_cfg["epochs"], "history": history}, checkpoint_path)
|
| 83 |
+
metrics = ROOT / config["paths"]["training_metrics"]; metrics.parent.mkdir(parents=True, exist_ok=True)
|
| 84 |
+
metrics.write_text(json.dumps({"history": history, "protocol": dataset.protocol,
|
| 85 |
+
"source": dataset.source,
|
| 86 |
+
"world_size": world_size, "amp": amp}, indent=2) + "\n")
|
| 87 |
+
print(f"checkpoint={checkpoint_path}")
|
| 88 |
+
if world_size > 1: dist.destroy_process_group()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
|
| 90 |
|
| 91 |
+
if __name__ == "__main__": main()
|
|
|