Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /ctrl-gaurav_effGen /effgen /gpu /allocator.py
| """GPU allocation and resource management for multi-GPU deployments. | |
| Supports multiple allocation strategies and tensor/pipeline parallelism, and | |
| prevents resource overallocation. | |
| Author: effGen Team | |
| License: Apache-2.0 | |
| """ | |
| from __future__ import annotations | |
| import logging | |
| import warnings | |
| from dataclasses import dataclass, field | |
| from enum import Enum | |
| from threading import RLock | |
| try: | |
| import torch | |
| TORCH_AVAILABLE = True | |
| except ImportError: | |
| TORCH_AVAILABLE = False | |
| warnings.warn("PyTorch not available. GPU allocation will use CPU fallback.") | |
| logger = logging.getLogger(__name__) | |
| def _device_free_memory(device_id: int, total_memory: int) -> int: | |
| """Return real free memory on a device in bytes. | |
| Uses ``torch.cuda.mem_get_info`` (the CUDA driver's view), which reflects | |
| memory consumed by *all* processes sharing the GPU — not just this one — so | |
| the allocator does not over-commit GPUs that other tenants are already using. | |
| The call is device-scoped and does not mutate the process-wide current device. | |
| Falls back to this-process reservations only if the driver query is | |
| unavailable (e.g. older torch without a live CUDA context). | |
| """ | |
| try: | |
| free, _total = torch.cuda.mem_get_info(device_id) | |
| return int(free) | |
| except Exception as e: | |
| logger.debug( | |
| f"mem_get_info unavailable for GPU {device_id}, " | |
| f"falling back to per-process reservations: {e}" | |
| ) | |
| return total_memory - torch.cuda.memory_reserved(device_id) | |
| class AllocationStrategy(Enum): | |
| """GPU allocation strategies.""" | |
| GREEDY = "greedy" # Fill GPUs sequentially | |
| BALANCED = "balanced" # Distribute evenly across GPUs | |
| OPTIMIZE = "optimize" # Minimize inter-GPU communication | |
| PRIORITY = "priority" # Allocate based on task priority | |
| class ParallelismType(Enum): | |
| """Types of model parallelism.""" | |
| TENSOR = "tensor" # Tensor parallelism (split layers across GPUs) | |
| PIPELINE = "pipeline" # Pipeline parallelism (different layers on different GPUs) | |
| DATA = "data" # Data parallelism (replicate model, split batches) | |
| NONE = "none" # No parallelism | |
| class GPUInfo: | |
| """Information about a single GPU.""" | |
| device_id: int | |
| name: str | |
| total_memory: int # in bytes | |
| available_memory: int # in bytes | |
| utilization: float # 0.0 to 1.0 | |
| temperature: float | None = None # in Celsius | |
| power_usage: float | None = None # in Watts | |
| def used_memory(self) -> int: | |
| """Calculate used memory in bytes.""" | |
| return self.total_memory - self.available_memory | |
| def memory_utilization(self) -> float: | |
| """Calculate memory utilization (0.0 to 1.0).""" | |
| if self.total_memory == 0: | |
| return 0.0 | |
| return self.used_memory / self.total_memory | |
| class AllocationRequest: | |
| """Request for GPU allocation.""" | |
| requester_id: str | |
| memory_required: int # in bytes | |
| num_gpus: int = 1 | |
| strategy: AllocationStrategy = AllocationStrategy.BALANCED | |
| parallelism_type: ParallelismType = ParallelismType.NONE | |
| preferred_devices: list[int] | None = None | |
| priority: int = 0 # Higher priority = more important | |
| allow_shared: bool = True # Allow sharing GPU with other tasks | |
| class Allocation: | |
| """Represents an active GPU allocation.""" | |
| requester_id: str | |
| device_ids: list[int] | |
| memory_allocated: dict[int, int] # device_id -> memory in bytes | |
| parallelism_type: ParallelismType | |
| timestamp: float = field(default_factory=lambda: __import__('time').time()) | |
| class GPUAllocator: | |
| """ | |
| Intelligent GPU allocation and resource management. | |
| This class handles: | |
| - Device discovery and monitoring | |
| - Smart allocation based on various strategies | |
| - Tensor and pipeline parallelism support | |
| - Resource tracking and deallocation | |
| - Prevention of overallocation | |
| Example: | |
| >>> allocator = GPUAllocator() | |
| >>> request = AllocationRequest( | |
| ... requester_id="agent_1", | |
| ... memory_required=8 * 1024**3, # 8GB | |
| ... num_gpus=2, | |
| ... strategy=AllocationStrategy.BALANCED | |
| ... ) | |
| >>> allocation = allocator.allocate(request) | |
| >>> if allocation: | |
| ... print(f"Allocated GPUs: {allocation.device_ids}") | |
| >>> allocator.deallocate("agent_1") | |
| """ | |
| def __init__(self, enable_monitoring: bool = True) -> None: | |
| """ | |
| Initialize GPU allocator. | |
| Args: | |
| enable_monitoring: Enable real-time GPU monitoring | |
| """ | |
| # Reentrant so that lock-holding helpers (e.g. reset -> deallocate) do | |
| # not deadlock by re-acquiring the same lock. | |
| self._lock = RLock() | |
| self._allocations: dict[str, Allocation] = {} | |
| self._devices: dict[int, GPUInfo] = {} | |
| self._enable_monitoring = enable_monitoring | |
| # Initialize devices | |
| self._discover_devices() | |
| logger.info( | |
| f"GPUAllocator initialized with {len(self._devices)} GPU(s). " | |
| f"CUDA available: {self.is_cuda_available()}" | |
| ) | |
| def is_cuda_available(self) -> bool: | |
| """Check if CUDA is available.""" | |
| if not TORCH_AVAILABLE: | |
| return False | |
| return torch.cuda.is_available() | |
| def _discover_devices(self) -> None: | |
| """Discover available GPU devices""" | |
| if not self.is_cuda_available(): | |
| logger.warning("CUDA not available. Running in CPU mode.") | |
| return | |
| try: | |
| num_devices = torch.cuda.device_count() | |
| for device_id in range(num_devices): | |
| props = torch.cuda.get_device_properties(device_id) | |
| # Real free memory from the driver (respects other processes); | |
| # device-scoped, so the global current device is left untouched. | |
| total_memory = props.total_memory | |
| available = _device_free_memory(device_id, total_memory) | |
| gpu_info = GPUInfo( | |
| device_id=device_id, | |
| name=props.name, | |
| total_memory=total_memory, | |
| available_memory=available, | |
| utilization=0.0 # Will be updated by monitor | |
| ) | |
| self._devices[device_id] = gpu_info | |
| logger.info( | |
| f"Discovered GPU {device_id}: {gpu_info.name} " | |
| f"with {gpu_info.total_memory / (1024**3):.2f} GB VRAM" | |
| ) | |
| except Exception as e: | |
| logger.error(f"Error discovering GPU devices: {e}") | |
| def get_device_info(self, device_id: int | None = None) -> dict[int, GPUInfo]: | |
| """ | |
| Get information about GPU devices. | |
| Args: | |
| device_id: Specific device ID, or None for all devices | |
| Returns: | |
| Dictionary mapping device IDs to GPUInfo objects | |
| """ | |
| self._refresh_device_info() | |
| if device_id is not None: | |
| if device_id in self._devices: | |
| return {device_id: self._devices[device_id]} | |
| else: | |
| raise ValueError(f"Device {device_id} not found") | |
| return self._devices.copy() | |
| def _refresh_device_info(self) -> None: | |
| """Refresh device memory information""" | |
| if not self.is_cuda_available(): | |
| return | |
| try: | |
| for device_id in self._devices: | |
| total = self._devices[device_id].total_memory | |
| # Driver-reported free memory (includes other processes); | |
| # device-scoped query, no global current-device mutation. | |
| self._devices[device_id].available_memory = _device_free_memory( | |
| device_id, total | |
| ) | |
| except Exception as e: | |
| logger.error(f"Error refreshing device info: {e}") | |
| def allocate(self, request: AllocationRequest) -> Allocation | None: | |
| """ | |
| Allocate GPUs based on the request. | |
| Args: | |
| request: Allocation request specifying requirements | |
| Returns: | |
| Allocation object if successful, None if allocation failed | |
| """ | |
| with self._lock: | |
| # Check if requester already has an allocation | |
| if request.requester_id in self._allocations: | |
| logger.warning( | |
| f"Requester {request.requester_id} already has an allocation. " | |
| f"Deallocate first before requesting new allocation." | |
| ) | |
| return None | |
| # Refresh device info | |
| self._refresh_device_info() | |
| # Select devices based on strategy | |
| selected_devices = self._select_devices(request) | |
| if not selected_devices: | |
| logger.warning( | |
| f"Could not allocate {request.num_gpus} GPU(s) for " | |
| f"{request.requester_id} with {request.memory_required / (1024**3):.2f} GB" | |
| ) | |
| return None | |
| # Create allocation | |
| memory_per_device = {} | |
| for device_id in selected_devices: | |
| if request.parallelism_type == ParallelismType.TENSOR: | |
| # Tensor parallelism: split memory across devices | |
| memory_per_device[device_id] = request.memory_required // len(selected_devices) | |
| else: | |
| # Other types: full memory on each device | |
| memory_per_device[device_id] = request.memory_required | |
| allocation = Allocation( | |
| requester_id=request.requester_id, | |
| device_ids=selected_devices, | |
| memory_allocated=memory_per_device, | |
| parallelism_type=request.parallelism_type | |
| ) | |
| # Update allocations | |
| self._allocations[request.requester_id] = allocation | |
| logger.info( | |
| f"Allocated GPU(s) {selected_devices} to {request.requester_id} " | |
| f"with parallelism: {request.parallelism_type.value}" | |
| ) | |
| return allocation | |
| def _select_devices(self, request: AllocationRequest) -> list[int]: | |
| """ | |
| Select GPU devices based on allocation strategy. | |
| Args: | |
| request: Allocation request | |
| Returns: | |
| List of selected device IDs | |
| """ | |
| if not self._devices: | |
| logger.warning("No GPU devices available") | |
| return [] | |
| # Filter available devices | |
| available_devices = [] | |
| for device_id, info in self._devices.items(): | |
| # Check if device is in preferred list (if specified) | |
| if request.preferred_devices and device_id not in request.preferred_devices: | |
| continue | |
| # Check if device has enough memory | |
| required_memory = request.memory_required | |
| if request.parallelism_type == ParallelismType.TENSOR: | |
| # Tensor parallelism splits memory across devices | |
| required_memory = request.memory_required // request.num_gpus | |
| # Start from the driver's real free memory (which already reflects | |
| # other processes on shared boxes), then subtract effGen's own | |
| # outstanding logical reservations as an overlay. | |
| reserved_by_effgen = sum( | |
| alloc.memory_allocated.get(device_id, 0) | |
| for alloc in self._allocations.values() | |
| ) | |
| free_memory = max(0, info.available_memory - reserved_by_effgen) | |
| if free_memory >= required_memory: | |
| available_devices.append((device_id, free_memory, info.utilization)) | |
| if len(available_devices) < request.num_gpus: | |
| logger.warning( | |
| f"Only {len(available_devices)} GPU(s) available, " | |
| f"but {request.num_gpus} requested" | |
| ) | |
| return [] | |
| # Apply allocation strategy | |
| if request.strategy == AllocationStrategy.GREEDY: | |
| return self._greedy_selection(available_devices, request.num_gpus) | |
| elif request.strategy == AllocationStrategy.BALANCED: | |
| return self._balanced_selection(available_devices, request.num_gpus) | |
| elif request.strategy == AllocationStrategy.OPTIMIZE: | |
| return self._optimized_selection(available_devices, request.num_gpus) | |
| elif request.strategy == AllocationStrategy.PRIORITY: | |
| return self._priority_selection(available_devices, request) | |
| else: | |
| logger.warning(f"Unknown strategy {request.strategy}, using BALANCED") | |
| return self._balanced_selection(available_devices, request.num_gpus) | |
| def _greedy_selection( | |
| self, | |
| available_devices: list[tuple[int, int, float]], | |
| num_gpus: int | |
| ) -> list[int]: | |
| """ | |
| Greedy selection: Fill GPUs sequentially. | |
| Args: | |
| available_devices: List of (device_id, free_memory, utilization) | |
| num_gpus: Number of GPUs to select | |
| Returns: | |
| List of selected device IDs | |
| """ | |
| # Sort by device ID | |
| sorted_devices = sorted(available_devices, key=lambda x: x[0]) | |
| return [device_id for device_id, _, _ in sorted_devices[:num_gpus]] | |
| def _balanced_selection( | |
| self, | |
| available_devices: list[tuple[int, int, float]], | |
| num_gpus: int | |
| ) -> list[int]: | |
| """ | |
| Balanced selection: Select GPUs with lowest utilization. | |
| Args: | |
| available_devices: List of (device_id, free_memory, utilization) | |
| num_gpus: Number of GPUs to select | |
| Returns: | |
| List of selected device IDs | |
| """ | |
| # Sort by utilization (lower is better), then by free memory (higher is better) | |
| sorted_devices = sorted( | |
| available_devices, | |
| key=lambda x: (x[2], -x[1]) # utilization asc, free_memory desc | |
| ) | |
| return [device_id for device_id, _, _ in sorted_devices[:num_gpus]] | |
| def _optimized_selection( | |
| self, | |
| available_devices: list[tuple[int, int, float]], | |
| num_gpus: int | |
| ) -> list[int]: | |
| """ | |
| Optimized selection: Minimize inter-GPU communication by selecting adjacent GPUs. | |
| Args: | |
| available_devices: List of (device_id, free_memory, utilization) | |
| num_gpus: Number of GPUs to select | |
| Returns: | |
| List of selected device IDs | |
| """ | |
| if num_gpus == 1: | |
| return self._balanced_selection(available_devices, num_gpus) | |
| # Sort by device ID | |
| sorted_devices = sorted(available_devices, key=lambda x: x[0]) | |
| device_ids = [d[0] for d in sorted_devices] | |
| # Find contiguous sequences of GPUs | |
| best_sequence = [] | |
| best_score = float('inf') | |
| for i in range(len(device_ids) - num_gpus + 1): | |
| sequence = device_ids[i:i + num_gpus] | |
| # Check if sequence is contiguous | |
| is_contiguous = all( | |
| sequence[j+1] - sequence[j] == 1 | |
| for j in range(len(sequence) - 1) | |
| ) | |
| if is_contiguous: | |
| # Score based on average utilization | |
| devices_info = [d for d in sorted_devices if d[0] in sequence] | |
| avg_utilization = sum(d[2] for d in devices_info) / len(devices_info) | |
| if avg_utilization < best_score: | |
| best_score = avg_utilization | |
| best_sequence = sequence | |
| # If no contiguous sequence found, fall back to balanced | |
| if not best_sequence: | |
| return self._balanced_selection(available_devices, num_gpus) | |
| return best_sequence | |
| def _priority_selection( | |
| self, | |
| available_devices: list[tuple[int, int, float]], | |
| request: AllocationRequest | |
| ) -> list[int]: | |
| """ | |
| Priority-based selection: Allocate best GPUs to high priority tasks. | |
| Args: | |
| available_devices: List of (device_id, free_memory, utilization) | |
| request: Allocation request with priority | |
| Returns: | |
| List of selected device IDs | |
| """ | |
| # High priority tasks get GPUs with most free memory and lowest utilization | |
| sorted_devices = sorted( | |
| available_devices, | |
| key=lambda x: (-x[1], x[2]) # free_memory desc, utilization asc | |
| ) | |
| return [device_id for device_id, _, _ in sorted_devices[:request.num_gpus]] | |
| def deallocate(self, requester_id: str) -> bool: | |
| """ | |
| Deallocate GPU resources for a requester. | |
| Args: | |
| requester_id: ID of the requester to deallocate | |
| Returns: | |
| True if deallocation successful, False otherwise | |
| """ | |
| with self._lock: | |
| return self._deallocate_locked(requester_id) | |
| def _deallocate_locked(self, requester_id: str) -> bool: | |
| """Deallocate without acquiring ``self._lock``. | |
| Caller MUST already hold ``self._lock``. This is the shared core used by | |
| both :meth:`deallocate` (which takes the lock) and :meth:`reset` (which | |
| already holds it), so reset never re-acquires the lock and deadlocks. | |
| """ | |
| if requester_id not in self._allocations: | |
| logger.warning(f"No allocation found for {requester_id}") | |
| return False | |
| allocation = self._allocations.pop(requester_id) | |
| logger.info( | |
| f"Deallocated GPU(s) {allocation.device_ids} from {requester_id}" | |
| ) | |
| # Clear GPU cache if using PyTorch. Use a device-scoped context so the | |
| # process-wide current CUDA device is not mutated. | |
| if self.is_cuda_available(): | |
| try: | |
| for device_id in allocation.device_ids: | |
| with torch.cuda.device(device_id): | |
| torch.cuda.empty_cache() | |
| except Exception as e: | |
| logger.error(f"Error clearing GPU cache: {e}") | |
| return True | |
| def get_allocation(self, requester_id: str) -> Allocation | None: | |
| """ | |
| Get allocation for a specific requester. | |
| Args: | |
| requester_id: ID of the requester | |
| Returns: | |
| Allocation object if found, None otherwise | |
| """ | |
| return self._allocations.get(requester_id) | |
| def list_allocations(self) -> dict[str, Allocation]: | |
| """ | |
| List all active allocations. | |
| Returns: | |
| Dictionary mapping requester IDs to Allocation objects | |
| """ | |
| return self._allocations.copy() | |
| def get_available_memory(self, device_id: int) -> int: | |
| """ | |
| Get available memory on a specific GPU. | |
| Args: | |
| device_id: GPU device ID | |
| Returns: | |
| Available memory in bytes | |
| """ | |
| self._refresh_device_info() | |
| if device_id not in self._devices: | |
| raise ValueError(f"Device {device_id} not found") | |
| # Driver-reported free memory (respects other processes) minus effGen's | |
| # own outstanding logical reservations. | |
| reserved_by_effgen = sum( | |
| alloc.memory_allocated.get(device_id, 0) | |
| for alloc in self._allocations.values() | |
| ) | |
| return max(0, self._devices[device_id].available_memory - reserved_by_effgen) | |
| def can_allocate(self, request: AllocationRequest) -> bool: | |
| """ | |
| Check if allocation request can be fulfilled. | |
| Args: | |
| request: Allocation request | |
| Returns: | |
| True if request can be fulfilled, False otherwise | |
| """ | |
| with self._lock: | |
| self._refresh_device_info() | |
| selected_devices = self._select_devices(request) | |
| return len(selected_devices) >= request.num_gpus | |
| def get_total_memory(self, device_id: int | None = None) -> int: | |
| """ | |
| Get total memory across GPUs. | |
| Args: | |
| device_id: Specific device ID, or None for all devices | |
| Returns: | |
| Total memory in bytes | |
| """ | |
| if device_id is not None: | |
| if device_id not in self._devices: | |
| raise ValueError(f"Device {device_id} not found") | |
| return self._devices[device_id].total_memory | |
| return sum(info.total_memory for info in self._devices.values()) | |
| def reset(self) -> None: | |
| """Clear all allocations and reset state.""" | |
| with self._lock: | |
| requester_ids = list(self._allocations.keys()) | |
| for requester_id in requester_ids: | |
| self._deallocate_locked(requester_id) | |
| logger.info("GPU allocator reset complete") | |
| def __repr__(self) -> str: | |
| """String representation""" | |
| num_devices = len(self._devices) | |
| num_allocations = len(self._allocations) | |
| total_memory = self.get_total_memory() / (1024**3) | |
| return ( | |
| f"GPUAllocator(devices={num_devices}, " | |
| f"allocations={num_allocations}, " | |
| f"total_memory={total_memory:.2f}GB)" | |
| ) | |
Xet Storage Details
- Size:
- 21.2 kB
- Xet hash:
- 30fe6cf7e8de8a651302eade1e45cc7fcda1aae4113511c98e50fa3883028b79
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.