| """ |
| System API Router - System monitoring and resource management |
| """ |
|
|
| from fastapi import APIRouter, HTTPException |
| from pydantic import BaseModel, Field |
| from typing import Dict, Any, List, Optional |
| from datetime import datetime |
| import logging |
| import psutil |
| import torch |
| import platform |
| import os |
| import shutil |
|
|
| from app.config import settings |
|
|
| logger = logging.getLogger(__name__) |
| router = APIRouter(prefix="/api/system", tags=["System"]) |
|
|
|
|
| class SystemInfo(BaseModel): |
| """System information.""" |
| platform: str |
| python_version: str |
| torch_version: str |
| transformers_version: str |
| cuda_available: bool |
| cuda_version: Optional[str] |
| gpu_count: int |
| gpu_names: List[str] |
| cpu_count: int |
| total_memory_gb: float |
| |
|
|
| class GPUInfo(BaseModel): |
| """GPU information.""" |
| available: bool |
| count: int = 0 |
| names: List[str] = [] |
| memory_used_gb: Optional[float] = None |
| memory_total_gb: Optional[float] = None |
| utilization: Optional[float] = None |
|
|
|
|
| class ResourceUsage(BaseModel): |
| """Current resource usage.""" |
| cpu: Dict[str, float] |
| memory: Dict[str, float] |
| disk: Dict[str, float] |
| gpu: GPUInfo |
| cache: Dict[str, Any] |
|
|
|
|
| class StorageInfo(BaseModel): |
| """Storage information.""" |
| path: str |
| exists: bool |
| size_gb: float |
| file_count: int |
| |
|
|
| class ConfigResponse(BaseModel): |
| """Configuration response.""" |
| max_concurrent_jobs: int |
| max_upload_size_mb: int |
| supported_tasks: List[str] |
| supported_dataset_sources: List[str] |
| default_hardware: str |
| available_hardware: List[str] |
|
|
|
|
| @router.get("/info", response_model=SystemInfo) |
| async def get_system_info(): |
| """Get system information.""" |
| import transformers |
| |
| cuda_version = None |
| if torch.cuda.is_available(): |
| cuda_version = torch.version.cuda |
| |
| gpu_names = [] |
| if torch.cuda.is_available(): |
| for i in range(torch.cuda.device_count()): |
| gpu_names.append(torch.cuda.get_device_name(i)) |
| |
| return SystemInfo( |
| platform=f"{platform.system()} {platform.release()}", |
| python_version=platform.python_version(), |
| torch_version=torch.__version__, |
| transformers_version=transformers.__version__, |
| cuda_available=torch.cuda.is_available(), |
| cuda_version=cuda_version, |
| gpu_count=torch.cuda.device_count(), |
| gpu_names=gpu_names, |
| cpu_count=os.cpu_count() or 1, |
| total_memory_gb=round(psutil.virtual_memory().total / (1024**3), 2) |
| ) |
|
|
|
|
| @router.get("/resources", response_model=ResourceUsage) |
| async def get_resource_usage(): |
| """Get current resource usage.""" |
| |
| cpu_percent = psutil.cpu_percent(interval=1) |
| memory = psutil.virtual_memory() |
| |
| |
| disk = shutil.disk_usage('/') |
| |
| |
|
|
| gpu_available = torch.cuda.is_available() |
| gpu_info = GPUInfo(available=gpu_available, count=0, names=[]) |
| |
| if torch.cuda.is_available(): |
| try: |
| gpu_names = [] |
| for i in range(torch.cuda.device_count()): |
| gpu_names.append(torch.cuda.get_device_name(i)) |
| |
| gpu_memory_used = round(torch.cuda.memory_allocated() / (1024**3), 2) |
| gpu_memory_total = round(torch.cuda.get_device_properties(0).total_memory / (1024**3), 2) |
| |
| gpu_info = GPUInfo( |
| available=True, |
| count=torch.cuda.device_count(), |
| names=gpu_names, |
| memory_used_gb=gpu_memory_used, |
| memory_total_gb=gpu_memory_total, |
| utilization=None |
| ) |
| except Exception as e: |
| logger.error(f"Error getting GPU info: {e}") |
| |
| |
| cache_total_bytes = 0 |
| cache_dirs = [settings.CACHE_DIR, settings.HF_CACHE_DIR] |
| for cache_path in cache_dirs: |
| if os.path.exists(cache_path): |
| for root, dirs, files in os.walk(cache_path): |
| for f in files: |
| try: |
| cache_total_bytes += os.path.getsize(os.path.join(root, f)) |
| except: |
| pass |
| |
| return ResourceUsage( |
| cpu={ |
| "percent": round(cpu_percent, 1) |
| }, |
| memory={ |
| "percent": round(memory.percent, 1), |
| "used_gb": round(memory.used / (1024**3), 2), |
| "total_gb": round(memory.total / (1024**3), 2) |
| }, |
| disk={ |
| "percent": round((disk.used / disk.total) * 100, 1), |
| "used_gb": round(disk.used / (1024**3), 2), |
| "total_gb": round(disk.total / (1024**3), 2) |
| }, |
| gpu=gpu_info, |
| cache={ |
| "total_bytes": cache_total_bytes |
| } |
| ) |
|
|
|
|
| @router.get("/storage", response_model=List[StorageInfo]) |
| async def get_storage_info(): |
| """Get storage information for important directories.""" |
| paths = [ |
| ("Models", settings.MODELS_DIR), |
| ("Cache", settings.CACHE_DIR), |
| ("Logs", settings.LOGS_DIR), |
| ("Uploads", settings.UPLOAD_DIR), |
| ("Outputs", settings.OUTPUT_DIR) |
| ] |
| |
| result = [] |
| for name, path in paths: |
| exists = os.path.exists(path) |
| size = 0 |
| file_count = 0 |
| |
| if exists: |
| for root, dirs, files in os.walk(path): |
| file_count += len(files) |
| for f in files: |
| try: |
| size += os.path.getsize(os.path.join(root, f)) |
| except: |
| pass |
| |
| result.append(StorageInfo( |
| path=f"{name} ({path})", |
| exists=exists, |
| size_gb=round(size / (1024**3), 4), |
| file_count=file_count |
| )) |
| |
| return result |
|
|
|
|
| @router.get("/config", response_model=ConfigResponse) |
| async def get_config(): |
| """Get current configuration.""" |
| return ConfigResponse( |
| max_concurrent_jobs=settings.MAX_CONCURRENT_JOBS, |
| max_upload_size_mb=settings.MAX_UPLOAD_SIZE_MB, |
| supported_tasks=settings.SUPPORTED_TASKS, |
| supported_dataset_sources=settings.DATASET_SOURCES, |
| default_hardware=settings.DEFAULT_HARDWARE, |
| available_hardware=settings.AVAILABLE_HARDWARE |
| ) |
|
|
|
|
| @router.post("/clear-cache") |
| async def clear_cache(): |
| """Clear the model and dataset cache.""" |
| cache_paths = [settings.CACHE_DIR, settings.HF_CACHE_DIR] |
| cleared = [] |
| |
| for path in cache_paths: |
| if os.path.exists(path): |
| try: |
| shutil.rmtree(path) |
| os.makedirs(path, exist_ok=True) |
| cleared.append(path) |
| except Exception as e: |
| logger.error(f"Failed to clear {path}: {e}") |
| |
| |
| if torch.cuda.is_available(): |
| torch.cuda.empty_cache() |
| |
| return { |
| "message": "Cache cleared", |
| "paths_cleared": cleared, |
| "timestamp": datetime.utcnow().isoformat() |
| } |
|
|
|
|
| @router.get("/environment") |
| async def get_environment(): |
| """Get relevant environment variables (safe to expose).""" |
| safe_vars = [ |
| "HF_HOME", |
| "HF_HUB_CACHE", |
| "TRANSFORMERS_CACHE", |
| "WANDB_MODE", |
| "WANDB_PROJECT" |
| ] |
| |
| env_vars = {} |
| for var in safe_vars: |
| env_vars[var] = os.getenv(var, "not set") |
| |
| |
| tokens_status = { |
| "HF_TOKEN": bool(os.getenv("HF_TOKEN")), |
| "WANDB_API_KEY": bool(os.getenv("WANDB_API_KEY")) |
| } |
| |
| return { |
| "environment_variables": env_vars, |
| "tokens_configured": tokens_status |
| } |
|
|
|
|
| @router.get("/processes") |
| async def get_processes(): |
| """Get running training processes.""" |
| processes = [] |
| |
| for proc in psutil.process_iter(['pid', 'name', 'cmdline', 'cpu_percent', 'memory_percent']): |
| try: |
| |
| cmdline = ' '.join(proc.info['cmdline'] or []) |
| if any(x in cmdline.lower() for x in ['train', 'torch', 'python']): |
| processes.append({ |
| "pid": proc.info['pid'], |
| "name": proc.info['name'], |
| "cpu_percent": proc.info['cpu_percent'], |
| "memory_percent": round(proc.info['memory_percent'], 2) |
| }) |
| except (psutil.NoSuchProcess, psutil.AccessDenied): |
| continue |
| |
| return { |
| "processes": processes[:20], |
| "total": len(processes) |
| } |
|
|
|
|
| @router.get("/health") |
| async def health_check(): |
| """Detailed health check.""" |
| health_status = { |
| "status": "healthy", |
| "timestamp": datetime.utcnow().isoformat(), |
| "components": {} |
| } |
| |
| |
| disk = shutil.disk_usage('/') |
| disk_percent = (disk.used / disk.total) * 100 |
| health_status["components"]["disk"] = { |
| "status": "ok" if disk_percent < 90 else "warning", |
| "used_percent": round(disk_percent, 1) |
| } |
| |
| |
| memory = psutil.virtual_memory() |
| health_status["components"]["memory"] = { |
| "status": "ok" if memory.percent < 90 else "warning", |
| "used_percent": round(memory.percent, 1) |
| } |
| |
| |
| if torch.cuda.is_available(): |
| try: |
| gpu_mem = torch.cuda.memory_allocated() / torch.cuda.get_device_properties(0).total_memory |
| health_status["components"]["gpu"] = { |
| "status": "ok" if gpu_mem < 0.9 else "warning", |
| "memory_used_percent": round(gpu_mem * 100, 1) |
| } |
| except: |
| health_status["components"]["gpu"] = {"status": "unavailable"} |
| |
| |
| if any(c.get("status") == "warning" for c in health_status["components"].values()): |
| health_status["status"] = "warning" |
| |
| return health_status |
|
|
|
|
| @router.get("/logs") |
| async def get_system_logs( |
| lines: int = 100, |
| level: str = "INFO" |
| ): |
| """Get recent system logs.""" |
| log_file = os.path.join(settings.LOGS_DIR, "app.log") |
| |
| if not os.path.exists(log_file): |
| return {"message": "No log file found", "logs": []} |
| |
| try: |
| with open(log_file, 'r') as f: |
| all_lines = f.readlines() |
| |
| |
| if level.upper() != "ALL": |
| filtered = [l for l in all_lines if level.upper() in l] |
| else: |
| filtered = all_lines |
| |
| return { |
| "total_lines": len(filtered), |
| "logs": filtered[-lines:] |
| } |
| except Exception as e: |
| return {"error": str(e), "logs": []} |
