Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /MarkinHaus_ToolBoxV2 /llm-gateway /model_manager.py
| """ | |
| Model Manager - Ollama Backend | |
| Manages models via Ollama API (native or Docker). | |
| Supports: text, vision, omni, embedding, tts, audio | |
| """ | |
| import asyncio | |
| import json | |
| import re | |
| from pathlib import Path | |
| from typing import Dict, List, Optional, Any | |
| import httpx | |
| try: | |
| from huggingface_hub import HfApi, hf_hub_download, list_repo_files | |
| except ImportError: | |
| HfApi = None | |
| # Capability matrix per model type | |
| MODEL_CAPABILITIES = { | |
| "text": {"text": True, "vision": False, "audio": False, "embedding": False, "tts": False}, | |
| "vision": {"text": True, "vision": True, "audio": False, "embedding": False, "tts": False}, | |
| "omni": {"text": True, "vision": True, "audio": True, "embedding": False, "tts": False}, | |
| "embedding": {"text": True, "vision": False, "audio": False, "embedding": True, "tts": False}, | |
| "vision-embedding": {"text": True, "vision": True, "audio": False, "embedding": True, "tts": False}, | |
| "audio": {"text": False, "vision": False, "audio": True, "embedding": False, "tts": False}, | |
| "tts": {"text": False, "vision": False, "audio": False, "embedding": False, "tts": True}, | |
| } | |
| def detect_model_type(name: str) -> str: | |
| """Auto-detect model type from model name/tag.""" | |
| n = name.lower() | |
| tts_patterns = [ | |
| "tts", "speech", "vocoder", "outetts", "parler", "kokoro", | |
| "f5-tts", "f5tts", "coqui", "xtts", "bark", "vits", | |
| "piper", "silero", "speecht5", "tortoise", | |
| ] | |
| if any(p in n for p in tts_patterns): | |
| return "tts" | |
| if "omni" in n: | |
| return "omni" | |
| if "whisper" in n: | |
| return "audio" | |
| is_vision = any(x in n for x in ["-vl", "_vl", "vision", "llava", "minicpm-v", "bakllava"]) | |
| is_embed = "embed" in n | |
| if is_vision and is_embed: | |
| return "vision-embedding" | |
| if is_embed: | |
| return "embedding" | |
| if is_vision: | |
| return "vision" | |
| return "text" | |
| class ModelManager: | |
| """Manages models via Ollama HTTP API.""" | |
| def __init__(self, config: Dict): | |
| self.config = config | |
| self.ollama_url = config.get("ollama_url", "http://127.0.0.1:11434") | |
| # Loaded model slots: model_name -> slot info | |
| self.loaded_models: Dict[str, Dict] = {} | |
| self.models_dir = Path(config.get("models_dir", "data/models")) | |
| self.models_dir.mkdir(parents=True, exist_ok=True) | |
| # ------------------------------------------------------------------ | |
| # Ollama API helpers | |
| # ------------------------------------------------------------------ | |
| def _client(self, timeout: float = 30.0) -> httpx.AsyncClient: | |
| return httpx.AsyncClient(base_url=self.ollama_url, timeout=timeout) | |
| async def _ollama_health(self) -> bool: | |
| try: | |
| async with self._client(timeout=5.0) as c: | |
| r = await c.get("/") | |
| return r.status_code == 200 | |
| except Exception: | |
| return False | |
| # ------------------------------------------------------------------ | |
| # Model lifecycle | |
| # ------------------------------------------------------------------ | |
| async def pull_model(self, model_name: str) -> Dict: | |
| """Pull model from Ollama registry (ollama pull).""" | |
| async with self._client(timeout=600.0) as c: | |
| r = await c.post("/api/pull", json={"name": model_name, "stream": False}) | |
| r.raise_for_status() | |
| return r.json() | |
| async def load_model( | |
| self, | |
| model_name: str, | |
| model_type: str = "auto", | |
| keep_alive: str = "-1", | |
| ) -> Dict: | |
| """Load / warm-up a model in Ollama so it stays in VRAM/RAM.""" | |
| if model_type == "auto": | |
| model_type = detect_model_type(model_name) | |
| if model_type not in MODEL_CAPABILITIES: | |
| raise ValueError(f"Invalid model_type '{model_type}'. Must be: {list(MODEL_CAPABILITIES.keys())}") | |
| # Check Ollama is reachable | |
| if not await self._ollama_health(): | |
| raise RuntimeError(f"Ollama not reachable at {self.ollama_url}") | |
| # Ensure model exists locally (pull if needed) | |
| local_models = await self.list_ollama_models() | |
| local_names = [m["name"] for m in local_models] | |
| # Normalise: ollama tags may include :latest | |
| norm = model_name if ":" in model_name else f"{model_name}:latest" | |
| if norm not in local_names and model_name not in local_names: | |
| await self.pull_model(model_name) | |
| # Warm-up: send a tiny generate request with keep_alive to load into memory | |
| if model_type == "embedding": | |
| async with self._client(timeout=300.0) as c: | |
| r = await c.post("/api/embed", json={ | |
| "model": model_name, | |
| "input": "warmup", | |
| "keep_alive": keep_alive, | |
| }) | |
| r.raise_for_status() | |
| else: | |
| async with self._client(timeout=300.0) as c: | |
| r = await c.post("/api/generate", json={ | |
| "model": model_name, | |
| "prompt": "hi", | |
| "options": {"num_predict": 1}, | |
| "keep_alive": keep_alive, | |
| "stream": False, | |
| }) | |
| r.raise_for_status() | |
| info = { | |
| "name": model_name, | |
| "model_type": model_type, | |
| "capabilities": MODEL_CAPABILITIES[model_type], | |
| "keep_alive": keep_alive, | |
| "status": "running", | |
| } | |
| self.loaded_models[model_name] = info | |
| # Persist to config | |
| self._save_loaded_config() | |
| return {"status": "loaded", "model": model_name, "model_type": model_type} | |
| async def unload_model(self, model_name: str) -> Dict: | |
| """Unload model from Ollama memory.""" | |
| try: | |
| async with self._client(timeout=30.0) as c: | |
| await c.post("/api/generate", json={ | |
| "model": model_name, | |
| "prompt": "", | |
| "keep_alive": "0", | |
| "stream": False, | |
| }) | |
| except Exception: | |
| pass | |
| self.loaded_models.pop(model_name, None) | |
| self._save_loaded_config() | |
| return {"status": "unloaded", "model": model_name} | |
| async def delete_model(self, model_name: str) -> Dict: | |
| """Delete model from Ollama.""" | |
| # Unload first | |
| await self.unload_model(model_name) | |
| async with self._client(timeout=60.0) as c: | |
| r = await c.request("DELETE", "/api/delete", json={"name": model_name}) | |
| r.raise_for_status() | |
| return {"status": "deleted", "model": model_name} | |
| async def shutdown_all(self): | |
| """Unload all loaded models.""" | |
| for name in list(self.loaded_models.keys()): | |
| try: | |
| await self.unload_model(name) | |
| except Exception as e: | |
| print(f"Error unloading {name}: {e}") | |
| async def restore_models(self): | |
| """Restore models from config on startup.""" | |
| models = self.config.get("loaded_models", {}) | |
| for name, info in models.items(): | |
| try: | |
| await self.load_model( | |
| model_name=name, | |
| model_type=info.get("model_type", "auto"), | |
| keep_alive=info.get("keep_alive", "-1"), | |
| ) | |
| except Exception as e: | |
| print(f"Failed to restore model {name}: {e}") | |
| # ------------------------------------------------------------------ | |
| # Ollama model listing | |
| # ------------------------------------------------------------------ | |
| async def list_ollama_models(self) -> List[Dict]: | |
| """List models available in Ollama.""" | |
| async with self._client() as c: | |
| r = await c.get("/api/tags") | |
| r.raise_for_status() | |
| data = r.json() | |
| models = [] | |
| for m in data.get("models", []): | |
| models.append({ | |
| "name": m.get("name", ""), | |
| "size_gb": round(m.get("size", 0) / (1024 ** 3), 2), | |
| "modified_at": m.get("modified_at", ""), | |
| "family": m.get("details", {}).get("family", ""), | |
| "parameter_size": m.get("details", {}).get("parameter_size", ""), | |
| "quantization": m.get("details", {}).get("quantization_level", ""), | |
| "format": m.get("details", {}).get("format", ""), | |
| }) | |
| return models | |
| async def list_running_models(self) -> List[Dict]: | |
| """List models currently loaded in Ollama memory.""" | |
| try: | |
| async with self._client() as c: | |
| r = await c.get("/api/ps") | |
| r.raise_for_status() | |
| data = r.json() | |
| return data.get("models", []) | |
| except Exception: | |
| return [] | |
| async def search_ollama_library(self, query: str) -> List[Dict]: | |
| """Search Ollama library via the website (scrape).""" | |
| try: | |
| async with httpx.AsyncClient(timeout=15.0) as c: | |
| r = await c.get(f"https://ollama.com/search?q={query}") | |
| if r.status_code != 200: | |
| return [] | |
| # Parse basic info from the page - simplified | |
| # We return a minimal hint; the admin UI can link to ollama.com | |
| return [{"query": query, "hint": "Use 'ollama pull <model>' or enter model name to load"}] | |
| except Exception: | |
| return [] | |
| # ------------------------------------------------------------------ | |
| # Model import from GGUF / HuggingFace | |
| # ------------------------------------------------------------------ | |
| async def import_gguf(self, gguf_path: str, model_name: str) -> Dict: | |
| """Import a local GGUF file into Ollama via Modelfile.""" | |
| p = Path(gguf_path) | |
| if not p.exists(): | |
| # Try relative to models_dir | |
| p = self.models_dir / gguf_path | |
| if not p.exists(): | |
| raise FileNotFoundError(f"GGUF file not found: {gguf_path}") | |
| # Create a Modelfile | |
| modelfile_content = f'FROM "{p.resolve()}"\n' | |
| async with self._client(timeout=600.0) as c: | |
| r = await c.post("/api/create", json={ | |
| "name": model_name, | |
| "modelfile": modelfile_content, | |
| "stream": False, | |
| }) | |
| r.raise_for_status() | |
| return {"status": "imported", "model": model_name, "source": str(p)} | |
| async def download_hf_model(self, repo_id: str, filename: str) -> Dict: | |
| """Download GGUF from HuggingFace to local models dir.""" | |
| if HfApi is None: | |
| return {"error": "huggingface_hub not installed"} | |
| hf_token = self.config.get("hf_token") | |
| try: | |
| local_path = hf_hub_download( | |
| repo_id=repo_id, | |
| filename=filename, | |
| local_dir=str(self.models_dir), | |
| token=hf_token, | |
| ) | |
| return { | |
| "status": "downloaded", | |
| "path": str(Path(local_path).relative_to(self.models_dir)), | |
| "repo": repo_id, | |
| "filename": filename, | |
| } | |
| except Exception as e: | |
| return {"error": str(e)} | |
| async def search_hf_models(self, query: str) -> Any: | |
| """Search HuggingFace for GGUF models.""" | |
| if HfApi is None: | |
| return {"error": "huggingface_hub not installed"} | |
| api = HfApi() | |
| try: | |
| models = api.list_models(search=query, sort="downloads", direction=-1, limit=20) | |
| results = [] | |
| for model in models: | |
| try: | |
| files = list_repo_files(model.id) | |
| model_files = [f for f in files if f.endswith(".gguf")] | |
| if model_files: | |
| results.append({ | |
| "repo_id": model.id, | |
| "downloads": model.downloads, | |
| "likes": model.likes, | |
| "files": model_files[:15], | |
| }) | |
| except Exception: | |
| pass | |
| if len(results) >= 10: | |
| break | |
| return results | |
| except Exception as e: | |
| return {"error": str(e)} | |
| def list_local_gguf(self) -> List[Dict]: | |
| """List GGUF files in local models directory.""" | |
| models = [] | |
| for f in self.models_dir.glob("**/*.gguf"): | |
| size_mb = f.stat().st_size / (1024 ** 2) | |
| models.append({ | |
| "name": f.name, | |
| "path": str(f.relative_to(self.models_dir)), | |
| "size_mb": round(size_mb, 1), | |
| "size_gb": round(size_mb / 1024, 2), | |
| "detected_type": detect_model_type(f.name), | |
| }) | |
| return sorted(models, key=lambda x: x["name"]) | |
| # ------------------------------------------------------------------ | |
| # Smart model finding | |
| # ------------------------------------------------------------------ | |
| def find_model(self, model_name: str) -> Optional[Dict]: | |
| """Find loaded model by name (exact or partial match).""" | |
| if model_name in self.loaded_models: | |
| return self.loaded_models[model_name] | |
| # Partial match | |
| for name, info in self.loaded_models.items(): | |
| if model_name.lower() in name.lower() or name.lower() in model_name.lower(): | |
| return info | |
| return None | |
| def find_model_for_request( | |
| self, | |
| model_name: Optional[str] = None, | |
| needs_audio: bool = False, | |
| needs_vision: bool = False, | |
| needs_embedding: bool = False, | |
| ) -> Optional[Dict]: | |
| """Smart model finding based on request requirements.""" | |
| # 1. Exact match | |
| if model_name: | |
| m = self.find_model(model_name) | |
| if m: | |
| caps = m.get("capabilities", {}) | |
| if needs_audio and not caps.get("audio"): | |
| pass | |
| elif needs_vision and not caps.get("vision"): | |
| pass | |
| elif needs_embedding and not caps.get("embedding"): | |
| pass | |
| else: | |
| return m | |
| # 2. By capability | |
| for info in self.loaded_models.values(): | |
| caps = info.get("capabilities", {}) | |
| if needs_embedding and caps.get("embedding"): | |
| return info | |
| if needs_audio and caps.get("audio"): | |
| return info | |
| if needs_audio and not caps.get("audio"): | |
| continue | |
| if needs_vision and not caps.get("vision"): | |
| continue | |
| if caps.get("text"): | |
| return info | |
| return None | |
| def find_text_model(self) -> Optional[Dict]: | |
| for info in self.loaded_models.values(): | |
| if info.get("capabilities", {}).get("text"): | |
| return info | |
| return None | |
| def find_vision_model(self) -> Optional[Dict]: | |
| for info in self.loaded_models.values(): | |
| if info.get("capabilities", {}).get("vision"): | |
| return info | |
| return None | |
| def find_audio_model(self) -> Optional[Dict]: | |
| for info in self.loaded_models.values(): | |
| if info.get("capabilities", {}).get("audio"): | |
| return info | |
| return None | |
| def find_embedding_model(self) -> Optional[Dict]: | |
| for info in self.loaded_models.values(): | |
| if info.get("capabilities", {}).get("embedding"): | |
| return info | |
| return None | |
| def find_tts_model(self) -> Optional[Dict]: | |
| for info in self.loaded_models.values(): | |
| if info.get("capabilities", {}).get("tts"): | |
| return info | |
| return None | |
| # ------------------------------------------------------------------ | |
| # Status | |
| # ------------------------------------------------------------------ | |
| def get_active_models(self) -> List[Dict]: | |
| return [ | |
| { | |
| "name": info["name"], | |
| "type": info["model_type"], | |
| "capabilities": info.get("capabilities", {}), | |
| } | |
| for info in self.loaded_models.values() | |
| ] | |
| def get_models_status(self) -> List[Dict]: | |
| result = [] | |
| for name, info in self.loaded_models.items(): | |
| result.append({ | |
| "model_name": info["name"], | |
| "model_type": info["model_type"], | |
| "capabilities": info.get("capabilities", {}), | |
| "status": info.get("status", "unknown"), | |
| "keep_alive": info.get("keep_alive", "-1"), | |
| }) | |
| return result | |
| # ------------------------------------------------------------------ | |
| # Persistence | |
| # ------------------------------------------------------------------ | |
| def _save_loaded_config(self): | |
| self.config["loaded_models"] = { | |
| name: { | |
| "model_type": info["model_type"], | |
| "keep_alive": info.get("keep_alive", "-1"), | |
| } | |
| for name, info in self.loaded_models.items() | |
| } | |
| config_path = Path(self.config.get("_config_path", "data/config.json")) | |
| config_path.write_text(json.dumps(self.config, indent=2)) | |
Xet Storage Details
- Size:
- 17.4 kB
- Xet hash:
- e39f06dd1ac3ae69f4d1746434420d702882a4c5603cd38c95f0dfd73d0809a7
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.