"""Hugging Face ZeroGPU Space: SatQuery EarthMind-4B Official Multimodal Specialist Service. Official Model: sy1998/EarthMind-4B Official Repo: https://github.com/shuyansy/EarthMind Capabilities: 1. Single-image Remote Sensing VQA 2. Scene description and captioning 3. Text-guided spatial / pixel-level grounding & segmentation (real prediction_masks) 4. Optical and SAR image understanding 5. Joint optical + SAR multisensor reasoning (predict_forward_multi) NO mock fallbacks. NO heuristic bounding boxes. Real model inference only. """ import os import sys import io import time import base64 import logging from typing import Optional, Dict, Any, List, Tuple import numpy as np from PIL import Image import torch import gradio as gr from fastapi import FastAPI, Request, Response, status from pydantic import BaseModel, Field # Setup logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger("satquery.earthmind") try: import spaces gpu_decorator = spaces.GPU(duration=20) has_spaces = True except Exception: def gpu_decorator(fn): return fn has_spaces = False MODEL_ID = os.getenv("MODEL_ID", "sy1998/EarthMind-4B") DEVICE = "cuda" if torch.cuda.is_available() else "cpu" _model = None _tokenizer = None _load_error = None def load_model_and_tokenizer(target_device: Optional[str] = None): """Load the official EarthMind-4B custom-code model and tokenizer.""" global _model, _tokenizer, _load_error resolved_dev = target_device or DEVICE if _model is not None and _tokenizer is not None: if resolved_dev == "cpu": try: _model = _model.to("cpu") except Exception: pass return _model, _tokenizer logger.info(f"[EarthMind-4B] Loading official model from {MODEL_ID} on {resolved_dev}...") try: from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig from transformers.dynamic_module_utils import get_class_from_dynamic_module # Pre-import and patch Sa2VAChatConfig to ensure modern transformers compatibility try: config_class = get_class_from_dynamic_module( "configuration_earthmind_chat.Sa2VAChatConfig", pretrained_model_name_or_path=MODEL_ID, token=os.getenv("HF_TOKEN", None) ) orig_init = config_class.__init__ def safe_init(self, *args, **kwargs): llm_cfg = kwargs.get("llm_config") if llm_cfg is not None and not isinstance(llm_cfg, dict): cfg_dict = llm_cfg.to_dict() if hasattr(llm_cfg, "to_dict") else dict(llm_cfg) if "architectures" not in cfg_dict: cfg_dict["architectures"] = ["Qwen2ForCausalLM"] kwargs["llm_config"] = cfg_dict elif isinstance(llm_cfg, dict) and "architectures" not in llm_cfg: llm_cfg["architectures"] = ["Qwen2ForCausalLM"] elif llm_cfg is None: kwargs["llm_config"] = {"architectures": ["Qwen2ForCausalLM"]} orig_init(self, *args, **kwargs) config_class.__init__ = safe_init logger.info("[EarthMind-4B] Applied dynamic compatibility patch to Sa2VAChatConfig") except Exception as patch_err: logger.warning(f"[EarthMind-4B] Dynamic patch warning: {patch_err}") # Allow EarthMind's internal try/except fallback to handle optional flash_attn try: import transformers.dynamic_module_utils transformers.dynamic_module_utils.check_imports = lambda filename: [] except Exception: pass from huggingface_hub import snapshot_download model_path = snapshot_download( MODEL_ID, token=os.getenv("HF_TOKEN", None) ) logger.info(f"[EarthMind-4B] Model snapshot located at: {model_path}") if model_path not in sys.path: sys.path.insert(0, model_path) # Robustly patch configuration_earthmind_chat.py on disk and across HF cache to fix KeyError: 'architectures' import glob for search_dir in [model_path, os.path.expanduser("~/.cache/huggingface")]: if os.path.exists(search_dir): for p in glob.glob(os.path.join(search_dir, "**", "configuration_earthmind_chat.py"), recursive=True): try: with open(p, "r", encoding="utf-8") as f: content = f.read() new_content = content.replace( "if llm_config is None:\n llm_config = {}", 'if llm_config is None:\n llm_config = {"architectures": ["Qwen2ForCausalLM"]}' ).replace( "if llm_config['architectures'][0]", "if llm_config.get('architectures', ['Qwen2ForCausalLM'])[0]" ) if new_content != content: with open(p, "w", encoding="utf-8") as f: f.write(new_content) logger.info(f"[EarthMind-4B] Patched {p}") except Exception as ex: logger.warning(f"[EarthMind-4B] Patch error on {p}: {ex}") for p in glob.glob(os.path.join(search_dir, "**", "modeling_earthmind_chat.py"), recursive=True): try: with open(p, "r", encoding="utf-8") as f: content = f.read() if "all_tied_weights_keys = {}" not in content: content = content.replace( "class Sa2VAChatModel(PreTrainedModel):", "class Sa2VAChatModel(PreTrainedModel):\n all_tied_weights_keys = {}" ) with open(p, "w", encoding="utf-8") as f: f.write(content) logger.info(f"[EarthMind-4B] Patched modeling file: {p}") except Exception as ex: logger.warning(f"[EarthMind-4B] Modeling patch error on {p}: {ex}") # Also mark has_no_defaults_at_init on any imported Sa2VAChatConfig for mod_name, mod in list(sys.modules.items()): if "configuration_earthmind_chat" in mod_name and hasattr(mod, "Sa2VAChatConfig"): try: mod.Sa2VAChatConfig.has_no_defaults_at_init = True except Exception: pass if "modeling_earthmind_chat" in mod_name and hasattr(mod, "Sa2VAChatModel"): try: mod.Sa2VAChatModel.all_tied_weights_keys = {} except Exception: pass tokenizer = AutoTokenizer.from_pretrained( model_path, trust_remote_code=True, token=os.getenv("HF_TOKEN", None) ) if resolved_dev == "cpu": torch_dtype = torch.float32 dev_map = None else: torch_dtype = torch.bfloat16 if (torch.cuda.is_available() and torch.cuda.is_bf16_supported()) else ( torch.float16 if torch.cuda.is_available() else torch.float32 ) dev_map = "auto" if torch.cuda.is_available() else None model = AutoModelForCausalLM.from_pretrained( model_path, torch_dtype=torch_dtype, device_map=dev_map, low_cpu_mem_usage=True, trust_remote_code=True, _attn_implementation="eager", token=os.getenv("HF_TOKEN", None) ) if resolved_dev != "cpu" and torch.cuda.is_available() and not hasattr(model, "hf_device_map"): model = model.to("cuda") elif resolved_dev == "cpu": model = model.to("cpu") model.eval() _model = model _tokenizer = tokenizer _load_error = None logger.info("[EarthMind-4B] Successfully loaded official checkpoint!") return _model, _tokenizer except Exception as exc: _load_error = str(exc) logger.error(f"[EarthMind-4B] Failed to load official model: {exc}", exc_info=True) raise exc def decode_image(img_input: Any) -> Optional[Image.Image]: """Decode PIL image, file path, or base64 string safely.""" if img_input is None: return None if isinstance(img_input, Image.Image): return img_input.convert("RGB") if isinstance(img_input, str): if img_input.startswith("data:image"): _, b64 = img_input.split(",", 1) raw = base64.b64decode(b64) return Image.open(io.BytesIO(raw)).convert("RGB") elif os.path.exists(img_input): return Image.open(img_input).convert("RGB") else: try: raw = base64.b64decode(img_input) return Image.open(io.BytesIO(raw)).convert("RGB") except Exception: return None return None def mask_to_bbox(mask_array: np.ndarray) -> Optional[Dict[str, Any]]: """Derive true normalized bounding box from real predicted binary mask.""" if mask_array is None or mask_array.size == 0: return None # Squeeze extra dimensions if present (e.g., [1, H, W] -> [H, W]) m = np.squeeze(mask_array) if m.ndim != 2: return None y_indices, x_indices = np.where(m > 0) if len(y_indices) == 0 or len(x_indices) == 0: return None h, w = m.shape ymin = round(float(np.min(y_indices)) / h, 4) xmin = round(float(np.min(x_indices)) / w, 4) ymax = round(float(np.max(y_indices)) / h, 4) xmax = round(float(np.max(x_indices)) / w, 4) return { "format": "ymin_xmin_ymax_xmax", "normalized_coords": { "ymin": ymin, "xmin": xmin, "ymax": ymax, "xmax": xmax }, "raw_coords_1000": [ int(ymin * 1000), int(xmin * 1000), int(ymax * 1000), int(xmax * 1000) ] } def mask_to_base64_png(mask_array: np.ndarray) -> Optional[str]: """Encode binary mask array as base64 PNG.""" if mask_array is None or mask_array.size == 0: return None m = np.squeeze(mask_array) if m.ndim != 2: return None # 255 for positive mask, 0 for background uint8_mask = (m > 0).astype(np.uint8) * 255 pil_mask = Image.fromarray(uint8_mask, mode="L") buf = io.BytesIO() pil_mask.save(buf, format="PNG") return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode("ascii") def _run_earthmind_inference_core( image: Optional[Image.Image], query: str, image_sar: Optional[Image.Image] = None, task_type: str = "vqa", target_device: Optional[str] = None ) -> Dict[str, Any]: """Core official EarthMind-4B forward execution.""" t0 = time.time() model, tokenizer = load_model_and_tokenizer(target_device=target_device) if target_device and str(next(model.parameters()).device) != target_device: try: model = model.to(target_device) except Exception as dev_err: logger.warning(f"[EarthMind-4B] Could not move model to {target_device}: {dev_err}") if not query or not query.strip(): query = "Describe this satellite image and identify all major geographical features." # Format official input text containing prompt_text = f"\n{query.strip()}" is_multisensor = (image_sar is not None and image is not None) or task_type == "multisensor" with torch.inference_mode(): if is_multisensor: logger.info("[EarthMind-4B] Executing official multisensor cross-modal path (predict_forward_multi)...") sar_input = image_sar if image_sar is not None else image optical_input = image if image is not None else image_sar if hasattr(model, "predict_forward_multi"): result = model.predict_forward_multi( image=sar_input, rgb_image=optical_input, text=prompt_text, tokenizer=tokenizer ) else: # Fallback to predict_forward with primary image if multi method is unavailable result = model.predict_forward( image=optical_input, text=prompt_text, tokenizer=tokenizer ) else: logger.info("[EarthMind-4B] Executing official single-image path (predict_forward)...") target_image = image if image is not None else image_sar if target_image is None: raise ValueError("No input image provided for EarthMind-4B inference.") result = model.predict_forward( image=target_image, text=prompt_text, tokenizer=tokenizer ) prediction = result.get("prediction", "") raw_masks = result.get("prediction_masks", []) # Process true prediction masks derived_bboxes = [] mask_b64 = None if raw_masks and len(raw_masks) > 0: for idx, mask_tensor in enumerate(raw_masks): if isinstance(mask_tensor, torch.Tensor): mask_np = mask_tensor.cpu().numpy() else: mask_np = np.array(mask_tensor) bbox = mask_to_bbox(mask_np) if bbox: bbox["box_id"] = f"mask_bbox_{idx + 1}" bbox["label"] = f"Segmented Region {idx + 1}" derived_bboxes.append(bbox) if mask_b64 is None: mask_b64 = mask_to_base64_png(mask_np) elapsed_sec = round(time.time() - t0, 3) return { "status": "success", "model": "EarthMind-4B", "task": task_type, "answer": prediction, "confidence": None, # No fabricated confidence score "mask": mask_b64, "bounding_boxes": derived_bboxes, "metadata": { "model_id": MODEL_ID, "checkpoint": "sy1998/EarthMind-4B", "task": task_type, "is_multisensor": is_multisensor, "has_prediction_masks": bool(raw_masks), "derived_bboxes_count": len(derived_bboxes), "inference_time_sec": elapsed_sec, "device": str(next(model.parameters()).device) } } @gpu_decorator def _run_earthmind_inference_gpu(image, query, image_sar, task_type): return _run_earthmind_inference_core(image, query, image_sar, task_type, target_device="cuda") def _run_earthmind_inference_cpu(image, query, image_sar, task_type): return _run_earthmind_inference_core(image, query, image_sar, task_type, target_device="cpu") # ============================================================================== # Gradio Web Interface # ============================================================================== def gradio_fn(optical_img, sar_img, query, task): try: res = _run_earthmind_inference_gpu(optical_img, query, sar_img, task) return res["answer"], res.get("bounding_boxes", []), res.get("mask") except Exception as err: err_msg = str(err) if any(kw in err_msg for kw in ["ZeroGPU quota", "exceeded your free ZeroGPU", "rate limit", "busy", "CUDA out of memory", "Invalid file descriptor"]): logger.warning("[EarthMind-4B] ZeroGPU lease unavailable. Seamlessly executing on CPU with official model...") try: res = _run_earthmind_inference_cpu(optical_img, query, sar_img, task) return res["answer"], res.get("bounding_boxes", []), res.get("mask") except Exception as cpu_err: return f"Error: {str(cpu_err)}", [], None return f"Error: {str(err)}", [], None demo = gr.Interface( fn=gradio_fn, inputs=[ gr.Image(type="pil", label="Primary Optical / Multi-Spectral Scene"), gr.Image(type="pil", label="Secondary SAR Scene (Optional - for Sentinel-1/2 Multisensor Fusion)"), gr.Textbox(lines=2, placeholder="e.g. Describe major land-cover types or segment the water body...", label="Operator Query / Instruction"), gr.Radio(["vqa", "captioning", "grounding", "multisensor"], value="vqa", label="Task Modality") ], outputs=[ gr.Textbox(label="EarthMind-4B Official Prediction"), gr.JSON(label="Real Derived Grounding Bounding Boxes (from prediction_masks)"), gr.Image(type="pil", label="Real Predicted Segmentation Mask") ], title="🛰️ SatQuery: Official EarthMind-4B Specialist Service", description="Official sy1998/EarthMind-4B Remote Sensing Specialist. Single-Image VQA, Captioning, Grounding, and Cross-Modal Optical+SAR Reasoning.", api_name="predict" ) if __name__ == "__main__": demo.launch(server_name="0.0.0.0", server_port=7860)