CircleStar commited on
Commit
cbf9e0a
ยท
verified ยท
1 Parent(s): ea0a3ef

Fix OOM crash when training KNN with large k by chunking full-image prediction; make the labeling-accuracy button actually jump to the Resultats tab

Browse files
Files changed (1) hide show
  1. app.py +22 -3
app.py CHANGED
@@ -56,6 +56,14 @@ CLASS_REMAP_LUT = np.array([0, 1, 4, 7, 5, 6, 2, 3], dtype=np.uint8)
56
 
57
  BASE_DIR = os.path.dirname(os.path.abspath(__file__))
58
 
 
 
 
 
 
 
 
 
59
  # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
60
  # Data loading (once at startup)
61
  # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
@@ -92,6 +100,15 @@ DATA = load_data()
92
  # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
93
  # Model training (from the student's own labels + real spectral bands)
94
  # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
 
 
 
 
 
 
 
 
 
95
  def train_and_predict(student_labels, k):
96
  """Train a KNN classifier on the student's 23 labeled polygons (using the
97
  real spectral bands), then evaluate it only at the Ground Truth pixel
@@ -122,7 +139,7 @@ def train_and_predict(student_labels, k):
122
  # with a known Ground Truth. We can't evaluate accuracy outside GT locations
123
  # (no known answer there) but we can still show what the model predicted.
124
  H, W, n_bands = bands.shape
125
- pred_full = clf.predict(scaler.transform(bands.reshape(-1, n_bands))).reshape(H, W)
126
 
127
  return model_oa, model_matrix, pred_gt, y_gt, pred_full
128
 
@@ -153,7 +170,7 @@ def train_and_predict_logreg(student_labels):
153
  model_oa = model_matrix.diagonal().sum() / model_matrix.sum()
154
 
155
  H, W, n_bands = bands.shape
156
- pred_full = clf.predict(scaler.transform(bands.reshape(-1, n_bands))).reshape(H, W)
157
 
158
  return model_oa, model_matrix, pred_gt, y_gt, pred_full
159
 
@@ -183,7 +200,7 @@ def train_and_predict_rf(student_labels):
183
  model_oa = model_matrix.diagonal().sum() / model_matrix.sum()
184
 
185
  H, W, n_bands = bands.shape
186
- pred_full = clf.predict(bands.reshape(-1, n_bands)).reshape(H, W)
187
 
188
  return model_oa, model_matrix, pred_gt, y_gt, pred_full
189
 
@@ -687,6 +704,7 @@ sur les pixels de vรฉritรฉ terrain (indรฉpendants des polygones d'entraรฎnement)
687
  gr.update(value=table, visible=True),
688
  gr.update(visible=True),
689
  gr.update(value=student_fig),
 
690
  )
691
 
692
  def on_train(current_labels, k):
@@ -879,6 +897,7 @@ sur les pixels de vรฉritรฉ terrain (indรฉpendants des polygones d'entraรฎnement)
879
  results_tbl,
880
  row_plots_student,
881
  student_matrix_plot,
 
882
  ],
883
  )
884
 
 
56
 
57
  BASE_DIR = os.path.dirname(os.path.abspath(__file__))
58
 
59
+ # Chunk size for predicting the full ~4.6M-pixel image. KNN's predict() builds
60
+ # a (n_queries, k) neighbor distance/index array โ€” at k=100 over the full
61
+ # image that's ~4.6M x 100 x 8 bytes x 2 โ‰ˆ 7.4GB in one call, which (combined
62
+ # with n_jobs=-1 worker overhead) exceeded the Space's 16GB memory limit and
63
+ # crashed it. Predicting in bounded chunks keeps peak memory constant
64
+ # regardless of k or image size.
65
+ FULL_PREDICT_CHUNK = 200_000
66
+
67
  # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
68
  # Data loading (once at startup)
69
  # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
 
100
  # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
101
  # Model training (from the student's own labels + real spectral bands)
102
  # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
103
+ def predict_full_image(clf, X_full, H, W):
104
+ """Predict on the whole image in bounded chunks (see FULL_PREDICT_CHUNK)
105
+ instead of one call over all ~4.6M pixels at once."""
106
+ preds = np.empty(X_full.shape[0], dtype=np.uint8)
107
+ for start in range(0, X_full.shape[0], FULL_PREDICT_CHUNK):
108
+ end = start + FULL_PREDICT_CHUNK
109
+ preds[start:end] = clf.predict(X_full[start:end])
110
+ return preds.reshape(H, W)
111
+
112
  def train_and_predict(student_labels, k):
113
  """Train a KNN classifier on the student's 23 labeled polygons (using the
114
  real spectral bands), then evaluate it only at the Ground Truth pixel
 
139
  # with a known Ground Truth. We can't evaluate accuracy outside GT locations
140
  # (no known answer there) but we can still show what the model predicted.
141
  H, W, n_bands = bands.shape
142
+ pred_full = predict_full_image(clf, scaler.transform(bands.reshape(-1, n_bands)), H, W)
143
 
144
  return model_oa, model_matrix, pred_gt, y_gt, pred_full
145
 
 
170
  model_oa = model_matrix.diagonal().sum() / model_matrix.sum()
171
 
172
  H, W, n_bands = bands.shape
173
+ pred_full = predict_full_image(clf, scaler.transform(bands.reshape(-1, n_bands)), H, W)
174
 
175
  return model_oa, model_matrix, pred_gt, y_gt, pred_full
176
 
 
200
  model_oa = model_matrix.diagonal().sum() / model_matrix.sum()
201
 
202
  H, W, n_bands = bands.shape
203
+ pred_full = predict_full_image(clf, bands.reshape(-1, n_bands), H, W)
204
 
205
  return model_oa, model_matrix, pred_gt, y_gt, pred_full
206
 
 
704
  gr.update(value=table, visible=True),
705
  gr.update(visible=True),
706
  gr.update(value=student_fig),
707
+ gr.update(selected="tab_results"),
708
  )
709
 
710
  def on_train(current_labels, k):
 
897
  results_tbl,
898
  row_plots_student,
899
  student_matrix_plot,
900
+ tabs,
901
  ],
902
  )
903