import base64 import io from typing import Any, Dict, Optional import numpy as np import torch from PIL import Image from diffusers import MarigoldIntrinsicsPipeline, utils as diffusers_utils # We will import MarigoldNormalsPipeline lazily to avoid loading it unless asked NormalsPipe = None def _clean_b64(s: str) -> bytes: s = s.strip() if s.startswith("data:image"): s = s.split(",", 1)[1] s = "".join(s.split()) # strip whitespace/newlines pad = (-len(s)) % 4 if pad: s += "=" * pad try: return base64.b64decode(s, validate=False) except Exception: return base64.urlsafe_b64decode(s) def _image_from_inputs(obj: Any) -> Image.Image: if isinstance(obj, dict): if "image_url" in obj: return diffusers_utils.load_image(obj["image_url"]).convert("RGB") if "image_b64" in obj: raw = _clean_b64(obj["image_b64"]) return Image.open(io.BytesIO(raw)).convert("RGB") raise ValueError("inputs dict must contain 'image_url' or 'image_b64'.") if isinstance(obj, (str, bytes, bytearray)): if isinstance(obj, (bytes, bytearray)): obj = obj.decode("utf-8", errors="ignore") s = obj.strip() if s.startswith("http://") or s.startswith("https://"): return diffusers_utils.load_image(s).convert("RGB") raw = _clean_b64(s) return Image.open(io.BytesIO(raw)).convert("RGB") raise ValueError("Unsupported 'inputs' type. Provide URL string, base64 string, or {'image_url'|'image_b64'}.") def _to_uint8_png_b64(img_like: Any) -> Optional[str]: """ Best-effort: accept PIL.Image or np.ndarray / torch.Tensor (H,W,3) or (3,H,W), map to [0,255] and return PNG b64 string. """ try: if isinstance(img_like, Image.Image): buf = io.BytesIO() img_like.save(buf, format="PNG") return base64.b64encode(buf.getvalue()).decode("utf-8") if isinstance(img_like, torch.Tensor): arr = img_like.detach().cpu().float().numpy() elif isinstance(img_like, np.ndarray): arr = img_like else: return None # Move channels-last if needed if arr.ndim == 3 and arr.shape[0] in (1, 3) and arr.shape[-1] not in (1, 3): arr = np.transpose(arr, (1, 2, 0)) if arr.ndim != 3 or arr.shape[-1] not in (1, 3): # If it's something else (e.g., batch), try take first item if arr.ndim == 4: # (B,H,W,C) arr = arr[0] else: return None # Heuristic normalization arr = arr.astype(np.float32) a_min, a_max = float(np.nanmin(arr)), float(np.nanmax(arr)) if a_max <= 1.0 and a_min >= 0.0: # already [0,1] pass elif a_max <= 1.0 and a_min >= -1.0: # [-1,1] normals style arr = (arr + 1.0) * 0.5 else: # generic min-max rng = (a_max - a_min) if (a_max - a_min) > 1e-6 else 1.0 arr = (arr - a_min) / rng arr = np.clip(arr * 255.0, 0, 255).astype(np.uint8) img = Image.fromarray(arr) buf = io.BytesIO() img.save(buf, format="PNG") return base64.b64encode(buf.getvalue()).decode("utf-8") except Exception: return None class EndpointHandler: """HF Inference Toolkit-compatible class handler with Intrinsics (as-is) + optional Normals.""" def __init__(self, path: Optional[str] = None): device = "cuda" if torch.cuda.is_available() else "cpu" dtype = torch.float16 if device == "cuda" else torch.float32 # ---- Intrinsics pipeline (UNCHANGED from your working code) ---- self.pipe = MarigoldIntrinsicsPipeline.from_pretrained( "prs-eth/marigold-iid-lighting-v1-1", torch_dtype=dtype, ).to(device) try: self.pipe.set_progress_bar_config(disable=True) except Exception: pass # Expected targets for lighting self.targets = ["albedo", "shading", "residual"] # ---- Normals (lazy) ---- self._normals_pipe = None self._device = device self._dtype = dtype # Defaults self.default_intr_steps = 4 # as before self.fixed_processing_res = 768 # per your latest preference # ---------------- Normals helpers ---------------- def _ensure_normals_loaded(self): global NormalsPipe if self._normals_pipe is not None: return if NormalsPipe is None: # Lazy import to avoid loading unless requested from diffusers.pipelines.marigold.pipeline_marigold_normals import MarigoldNormalsPipeline as _NP NormalsPipe = _NP self._normals_pipe = NormalsPipe.from_pretrained( "prs-eth/marigold-normals-v1-1", torch_dtype=self._dtype, ).to(self._device) try: self._normals_pipe.set_progress_bar_config(disable=True) except Exception: pass def _run_normals(self, pil_img: Image.Image) -> Dict[str, Any]: try: self._ensure_normals_loaded() # Do NOT pass processing_res to avoid the earlier error. # Most versions work with just the image (and optionally steps if supported). out = self._normals_pipe(pil_img) # Try common fields in order pred = None for key in ("prediction", "pred_normals"): if hasattr(out, key): pred = getattr(out, key) break if pred is None and hasattr(out, "images") and out.images: # Some pipelines return PIL in images[0] pred = out.images[0] # Convert to info + small preview b64 if isinstance(pred, torch.Tensor): arr = pred.detach().cpu().numpy() shape = list(arr.shape) dtype_str = str(arr.dtype) png_b64 = _to_uint8_png_b64(arr) elif isinstance(pred, np.ndarray): shape = list(pred.shape) dtype_str = str(pred.dtype) png_b64 = _to_uint8_png_b64(pred) elif isinstance(pred, Image.Image): shape = [pred.height, pred.width, 3] dtype_str = "uint8" png_b64 = _to_uint8_png_b64(pred) else: return {"error": f"Unexpected prediction type: {type(pred)}"} if png_b64 is None: return {"error": "Too many dimensions: 4 > 3."} return { "tensors_info": {"shape": shape, "dtype": dtype_str}, "preview_png_b64": png_b64, } except torch.cuda.OutOfMemoryError: return {"error": "GPU out of memory. Try a smaller image."} except Exception as e: return {"error": f"Normals prediction failed: {str(e)}"} # ---------------- Intrinsics (unchanged behavior) ---------------- def _run_intrinsics(self, pil_img: Image.Image, num_steps: int) -> Dict[str, Any]: # Inference (as before) with torch.inference_mode(): out = self.pipe(pil_img, num_inference_steps=num_steps) pred = out.prediction # Support torch.Tensor OR numpy.ndarray if isinstance(pred, torch.Tensor): pred_np = pred.detach().cpu().numpy() elif isinstance(pred, np.ndarray): pred_np = pred else: return {"error": f"Unexpected prediction type: {type(pred)}"} # pred_np expected shape: (T, H, W, 3) or (H, W, 3) depending on pipeline shape = list(pred_np.shape) dtype_str = str(pred_np.dtype) # Visualizations (best-effort) — unchanged vis = {} vis_err = None try: # visualize_intrinsics expects numpy prediction and the pipeline's target_properties vis_list = self.pipe.image_processor.visualize_intrinsics(pred_np, self.pipe.target_properties) # vis_list is a list over batch; we run single image, so index 0 if vis_list and isinstance(vis_list[0], dict): for name, im in vis_list[0].items(): if isinstance(im, Image.Image): buf = io.BytesIO() im.save(buf, format="PNG") vis[name] = base64.b64encode(buf.getvalue()).decode("utf-8") except Exception as e: vis_err = f"Failed to visualize: {e}" return { "targets": self.targets, "tensors_info": {"shape": shape, "dtype": dtype_str}, "uncertainty_available": False, "visualizations_png_b64": vis or None, "visualizations_error": vis_err, } # ---------------- Entry point ---------------- def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]: try: if "inputs" not in data: return {"error": "Body must include an 'inputs' key."} # Parameters can be at top-level or under 'parameters' params = {} params.update({k: v for k, v in data.items() if k != "inputs"}) if isinstance(data.get("parameters"), dict): params.update(data["parameters"]) # Which outputs to return? ret = params.get("return", ["intrinsics"]) if isinstance(ret, str): ret = [ret] ret = [r.lower() for r in ret] # Steps (intrinsics) try: num_steps = int(params.get("num_inference_steps", self.default_intr_steps)) except Exception: num_steps = self.default_intr_steps # Decode image try: pil_img = _image_from_inputs(data["inputs"]) except Exception as e: return {"error": f"Failed to decode 'inputs': {e}"} result: Dict[str, Any] = {"outputs": {}} if "normals" in ret: result["outputs"]["normals"] = self._run_normals(pil_img) if "intrinsics" in ret or not ret: intr_res = self._run_intrinsics(pil_img, num_steps=num_steps) # sanity check that vis contains expected keys when available if isinstance(intr_res, dict) and "targets" in intr_res: # Keep your original “intrinsics-only” structure nested result["outputs"]["intrinsics"] = intr_res else: result["outputs"]["intrinsics"] = { "error": "Intrinsics pipeline did not return expected outputs (albedo, diffuse_shading, residual)." } # If only one branch requested, return only that branch in 'outputs' return result except Exception as e: return {"error": f"{e}"} finally: if torch.cuda.is_available(): torch.cuda.empty_cache()