Deagin Claude Opus 4.7 (1M context) commited on
Commit
fe58fef
·
1 Parent(s): fca6cb3

Support dedicated T4/older GPUs: fp16 autocast + load-to-cuda at startup

Browse files

T4 (Turing) lacks native bfloat16, so autocast falls back to slow
emulation. Detect GPU capability and pick fp16 for pre-Ampere cards,
bfloat16 for Ampere+ (A10G/A100/RTX 30xx+).

Also load model directly to cuda at startup when a GPU is present
(dedicated Space). ZeroGPU still works — cuda.is_available() is
False at ZeroGPU startup so we fall through to cpu.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Files changed (2) hide show
  1. app.py +4 -2
  2. radio_backbone.py +8 -1
app.py CHANGED
@@ -25,11 +25,13 @@ import gradio as gr
25
  import numpy as np
26
  import torch
27
 
28
- # Pre-load model weights to CPU at startup (moved to GPU per-request by ZeroGPU)
 
 
29
  from radio_backbone import load_model
30
  print("Initializing RoofSight...")
31
  device = "cuda" if torch.cuda.is_available() else "cpu"
32
- load_model(device="cpu")
33
 
34
  GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", "")
35
 
 
25
  import numpy as np
26
  import torch
27
 
28
+ # On ZeroGPU: torch.cuda.is_available() is False at startup, so we load to CPU;
29
+ # the @spaces.GPU decorator handles per-request GPU movement.
30
+ # On dedicated GPU (t4-small, a10g, etc): load straight to cuda for lower latency.
31
  from radio_backbone import load_model
32
  print("Initializing RoofSight...")
33
  device = "cuda" if torch.cuda.is_available() else "cpu"
34
+ load_model(device=device)
35
 
36
  GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", "")
37
 
radio_backbone.py CHANGED
@@ -221,7 +221,14 @@ def zero_shot_segment(
221
 
222
  all_labels = roof_prompts + non_roof_prompts
223
 
224
- with torch.no_grad(), torch.autocast(device, dtype=torch.bfloat16):
 
 
 
 
 
 
 
225
  vis_output = model(pixel_values)
226
 
227
  # Get SigLIP2-aligned spatial features
 
221
 
222
  all_labels = roof_prompts + non_roof_prompts
223
 
224
+ # T4/older GPUs have no native bfloat16 — pick fp16 unless the GPU is Ampere+
225
+ if device == "cuda" and torch.cuda.is_available():
226
+ major, _ = torch.cuda.get_device_capability()
227
+ autocast_dtype = torch.bfloat16 if major >= 8 else torch.float16
228
+ else:
229
+ autocast_dtype = torch.bfloat16
230
+
231
+ with torch.no_grad(), torch.autocast(device, dtype=autocast_dtype):
232
  vis_output = model(pixel_values)
233
 
234
  # Get SigLIP2-aligned spatial features