| from fastapi import APIRouter, HTTPException |
| from pydantic import BaseModel |
| import logging |
|
|
| from app.llm.manager import LLMManager, HAS_LLAMA_CPP, GGUF_MODEL_MAP, AVAILABLE_CLOUD_MODELS |
|
|
| logger = logging.getLogger(__name__) |
|
|
| router = APIRouter( |
| prefix="/api/models", |
| tags=["Models"] |
| ) |
|
|
| class ModelRequest(BaseModel): |
| model_name: str |
|
|
| class EngineRequest(BaseModel): |
| engine: str |
|
|
| @router.get("") |
| def get_models(): |
| manager = LLMManager.get_instance() |
| settings = manager.settings |
| |
| models = [] |
| for m in settings.get("recommended_models", []): |
| models.append({ |
| "name": m, |
| "is_llm": True, |
| "capabilities": ["text-generation"], |
| "has_gguf": m in GGUF_MODEL_MAP |
| }) |
| |
| return { |
| "models": models, |
| "catalog": AVAILABLE_CLOUD_MODELS, |
| "hardware": settings, |
| "active_model": manager.active_model_name, |
| "active_engine": manager.get_active_engine(), |
| "preferred_engine": manager.preferred_engine, |
| "has_llama_cpp": HAS_LLAMA_CPP, |
| "available_engines": [ |
| { |
| "id": "llama.cpp", |
| "label": "llama.cpp (C++ GGUF Q4)", |
| "description": "Ultra-fast 4-bit native C++ execution (~2-4s response)", |
| "available": HAS_LLAMA_CPP |
| }, |
| { |
| "id": "transformers", |
| "label": "Transformers (PyTorch FP32)", |
| "description": "Standard unquantized PyTorch Hugging Face pipeline", |
| "available": True |
| } |
| ] |
| } |
|
|
| @router.post("/set-engine") |
| def set_inference_engine(req: EngineRequest): |
| manager = LLMManager.get_instance() |
| try: |
| active_engine = manager.set_engine(req.engine) |
| return { |
| "status": "success", |
| "message": f"Inference engine switched to {active_engine}", |
| "active_engine": active_engine, |
| "preferred_engine": manager.preferred_engine |
| } |
| except Exception as e: |
| logger.error(f"Error setting inference engine: {e}") |
| raise HTTPException(status_code=400, detail=str(e)) |
|
|
| @router.post("/set-default") |
| def set_default_model(req: ModelRequest): |
| manager = LLMManager.get_instance() |
| manager.active_model_name = req.model_name |
| try: |
| manager.load_model(req.model_name) |
| except Exception as e: |
| logger.warning(f"Error preloading model {req.model_name}: {e}") |
| return { |
| "message": f"Default model set to {req.model_name}", |
| "active_model": manager.active_model_name, |
| "active_engine": manager.get_active_engine() |
| } |
|
|
| @router.post("/pull") |
| def pull_model(req: ModelRequest): |
| manager = LLMManager.get_instance() |
| manager.load_model(req.model_name) |
| return { |
| "message": f"Model {req.model_name} pulled and loaded.", |
| "active_engine": manager.get_active_engine() |
| } |
|
|
