michaelmccamy commited on
Commit
4a5c719
·
verified ·
1 Parent(s): e6f374e

Deploy backend

Browse files
Files changed (1) hide show
  1. pipeline.py +45 -10
pipeline.py CHANGED
@@ -3,6 +3,7 @@
3
  Both functions take a PIL image and a mode ("indoor" or "outdoor") and return a dict.
4
  main.py adds audio_b64 to the describe() result, so don't add it here.
5
  """
 
6
  import threading
7
  from concurrent.futures import ThreadPoolExecutor
8
 
@@ -74,10 +75,28 @@ DETECT_DEPTH_MARGIN_M = 0.5
74
  # GPU when the Space has one (all three models together drop from about 1.1 s to about 0.1 s a frame),
75
  # CPU otherwise, so laptops and the CPU Space keep working unchanged.
76
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
77
 
78
  # Load once at import so each request doesn't reload the model. Outdoor loads on first use.
79
- _depth_pipes = {"indoor": hf_pipeline("depth-estimation", model=DEPTH_MODELS["indoor"], device=DEVICE)}
80
- _detector = hf_pipeline("object-detection", model=DETECT_MODEL, device=DEVICE)
81
  # find() runs depth and detection side by side instead of one after the other. On a GPU that
82
  # overlaps them; on CPU it's about even, since each model already uses every core.
83
  _parallel = ThreadPoolExecutor(max_workers=4)
@@ -86,9 +105,9 @@ _parallel = ThreadPoolExecutor(max_workers=4)
86
  def depth_map(image: Image.Image, mode: str) -> np.ndarray:
87
  """Depth in meters as a 2D array. Uses predicted_depth (meters), not depth (a picture for viewing)."""
88
  if mode not in _depth_pipes:
89
- _depth_pipes[mode] = hf_pipeline("depth-estimation", model=DEPTH_MODELS[mode], device=DEVICE)
90
  out = _depth_pipes[mode](image)
91
- return out["predicted_depth"].squeeze().cpu().numpy() * DEPTH_SCALE[mode]
92
 
93
 
94
  def _rows(depth: np.ndarray, span: tuple[float, float]) -> slice:
@@ -296,7 +315,7 @@ def _open_detector():
296
  """Loaded on first use, so the server starts as fast as before."""
297
  with _open_lock:
298
  if "pipe" not in _open:
299
- _open["pipe"] = hf_pipeline("zero-shot-object-detection", model=OPEN_VOCAB_MODEL, device=DEVICE)
300
  return _open["pipe"]
301
 
302
 
@@ -313,7 +332,17 @@ def _match_model():
313
  with _match_lock:
314
  if not _match:
315
  _match["processor"] = AutoProcessor.from_pretrained(MATCH_MODEL)
316
- _match["model"] = AutoModel.from_pretrained(MATCH_MODEL).eval().to(DEVICE)
 
 
 
 
 
 
 
 
 
 
317
  return _match["processor"], _match["model"]
318
 
319
 
@@ -364,10 +393,16 @@ def parse_request(text: str) -> tuple[str | None, str | None]:
364
 
365
  def _match_scores(crops: list[Image.Image], description: str) -> list[float]:
366
  processor, model = _match_model()
367
- inputs = processor(text=[f"a photo of a {description}"], images=crops, padding="max_length", return_tensors="pt").to(DEVICE)
368
  with torch.inference_mode():
369
  logits = model(**inputs).logits_per_image[:, 0]
370
- return torch.sigmoid(logits).tolist()
 
 
 
 
 
 
371
 
372
 
373
  def feet_words(meters: float) -> str:
@@ -426,9 +461,9 @@ SCENE_PROMPTS = ("a photo taken indoors", "a photo taken outdoors")
426
 
427
  def guess_scene(image: Image.Image) -> str:
428
  processor, model = _match_model()
429
- inputs = processor(text=list(SCENE_PROMPTS), images=[image], padding="max_length", return_tensors="pt").to(DEVICE)
430
  with torch.inference_mode():
431
- logits = model(**inputs).logits_per_image[0]
432
  return "outdoor" if float(logits[1] - logits[0]) > 0 else "indoor"
433
 
434
 
 
3
  Both functions take a PIL image and a mode ("indoor" or "outdoor") and return a dict.
4
  main.py adds audio_b64 to the describe() result, so don't add it here.
5
  """
6
+ import os
7
  import threading
8
  from concurrent.futures import ThreadPoolExecutor
9
 
 
75
  # GPU when the Space has one (all three models together drop from about 1.1 s to about 0.1 s a frame),
76
  # CPU otherwise, so laptops and the CPU Space keep working unchanged.
77
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
78
+ # Half precision on the GPU: faster on a T4, same answers to the precision we report. Each model is
79
+ # tried on a tiny image at load; if fp16 fails there, that model stays at full precision. FP16=0 in
80
+ # the Space's settings turns it off.
81
+ HALF = DEVICE == "cuda" and os.getenv("FP16", "1") != "0"
82
+
83
+
84
+ def _pipe(task: str, model: str, **probe):
85
+ """A pipeline on DEVICE, in fp16 when HALF and it works, else fp32."""
86
+ if HALF:
87
+ try:
88
+ p = hf_pipeline(task, model=model, device=DEVICE, dtype=torch.float16)
89
+ p(Image.new("RGB", (64, 64)), **probe)
90
+ print(f"{model}: fp16", flush=True)
91
+ return p
92
+ except Exception as e: # an op without half support: keep this model at fp32
93
+ print(f"{model}: fp16 failed ({type(e).__name__}: {e}), using fp32", flush=True)
94
+ return hf_pipeline(task, model=model, device=DEVICE)
95
+
96
 
97
  # Load once at import so each request doesn't reload the model. Outdoor loads on first use.
98
+ _depth_pipes = {"indoor": _pipe("depth-estimation", DEPTH_MODELS["indoor"])}
99
+ _detector = _pipe("object-detection", DETECT_MODEL)
100
  # find() runs depth and detection side by side instead of one after the other. On a GPU that
101
  # overlaps them; on CPU it's about even, since each model already uses every core.
102
  _parallel = ThreadPoolExecutor(max_workers=4)
 
105
  def depth_map(image: Image.Image, mode: str) -> np.ndarray:
106
  """Depth in meters as a 2D array. Uses predicted_depth (meters), not depth (a picture for viewing)."""
107
  if mode not in _depth_pipes:
108
+ _depth_pipes[mode] = _pipe("depth-estimation", DEPTH_MODELS[mode])
109
  out = _depth_pipes[mode](image)
110
+ return out["predicted_depth"].squeeze().float().cpu().numpy() * DEPTH_SCALE[mode]
111
 
112
 
113
  def _rows(depth: np.ndarray, span: tuple[float, float]) -> slice:
 
315
  """Loaded on first use, so the server starts as fast as before."""
316
  with _open_lock:
317
  if "pipe" not in _open:
318
+ _open["pipe"] = _pipe("zero-shot-object-detection", OPEN_VOCAB_MODEL, candidate_labels=["door"])
319
  return _open["pipe"]
320
 
321
 
 
332
  with _match_lock:
333
  if not _match:
334
  _match["processor"] = AutoProcessor.from_pretrained(MATCH_MODEL)
335
+ model = AutoModel.from_pretrained(MATCH_MODEL).eval().to(DEVICE)
336
+ if HALF:
337
+ try:
338
+ model = model.half()
339
+ with torch.inference_mode():
340
+ model(**_match_inputs(_match["processor"], model, ["a photo"], [Image.new("RGB", (64, 64))]))
341
+ print(f"{MATCH_MODEL}: fp16", flush=True)
342
+ except Exception as e:
343
+ print(f"{MATCH_MODEL}: fp16 failed ({type(e).__name__}: {e}), using fp32", flush=True)
344
+ model = model.float()
345
+ _match["model"] = model
346
  return _match["processor"], _match["model"]
347
 
348
 
 
393
 
394
  def _match_scores(crops: list[Image.Image], description: str) -> list[float]:
395
  processor, model = _match_model()
396
+ inputs = _match_inputs(processor, model, [f"a photo of a {description}"], crops)
397
  with torch.inference_mode():
398
  logits = model(**inputs).logits_per_image[:, 0]
399
+ return torch.sigmoid(logits.float()).tolist()
400
+
401
+
402
+ def _match_inputs(processor, model, texts: list[str], images: list[Image.Image]):
403
+ inputs = processor(text=texts, images=images, padding="max_length", return_tensors="pt").to(DEVICE)
404
+ inputs["pixel_values"] = inputs["pixel_values"].to(model.dtype) # fp16 model on the GPU
405
+ return inputs
406
 
407
 
408
  def feet_words(meters: float) -> str:
 
461
 
462
  def guess_scene(image: Image.Image) -> str:
463
  processor, model = _match_model()
464
+ inputs = _match_inputs(processor, model, list(SCENE_PROMPTS), [image])
465
  with torch.inference_mode():
466
+ logits = model(**inputs).logits_per_image[0].float()
467
  return "outdoor" if float(logits[1] - logits[0]) > 0 else "indoor"
468
 
469