Spaces:
Paused
Paused
| """vLLM compatibility validation chain. | |
| Chain order (each link depends on the previous): | |
| GPU -> driver -> torch -> vLLM -> model -> quantization | |
| Every link produces a CompatCheck with PASS / FAIL / SKIPPED (blocked by an | |
| earlier failure) plus a human-readable reason and remediation. On this | |
| project's development machine (macOS, no CUDA/ROCm) the chain fails honestly | |
| at the GPU link and reports vLLM as unsupported. | |
| """ | |
| from __future__ import annotations | |
| from typing import List, Optional | |
| from pydantic import BaseModel, Field | |
| from sensorflow.retro.inference.env_detect import EnvironmentReport, detect_environment | |
| PASS = "PASS" | |
| FAIL = "FAIL" | |
| SKIPPED = "SKIPPED" | |
| # Quantization methods vLLM supports on CUDA; ROCm support is narrower. | |
| # See docs/retro/compatibility-matrix.md for the substantiated matrix. | |
| CUDA_QUANTIZATIONS = {"awq", "gptq", "fp8", "int8", "none"} | |
| ROCM_QUANTIZATIONS = {"fp8", "gptq", "none"} # verify against current vLLM release | |
| # Very rough VRAM floor (GB) per model family for a usable context window. | |
| MODEL_VRAM_FLOOR_GB = { | |
| "7b": 16, "8b": 18, "13b": 28, "14b": 30, "32b": 68, "70b": 140, | |
| } | |
| class CompatCheck(BaseModel): | |
| link: str # gpu | driver | torch | vllm | model | quantization | |
| status: str # PASS | FAIL | SKIPPED | |
| reason: str | |
| remediation: Optional[str] = None | |
| class CompatReport(BaseModel): | |
| vllm_supported: bool | |
| platform_summary: str | |
| checks: List[CompatCheck] | |
| failed_link: Optional[str] = None | |
| notes: List[str] = Field(default_factory=list) | |
| def _estimate_vram_floor(model_name: str) -> Optional[int]: | |
| lname = model_name.lower() | |
| for size, gb in MODEL_VRAM_FLOOR_GB.items(): | |
| if size in lname: | |
| return gb | |
| return None | |
| def check_vllm_compatibility(env: Optional[EnvironmentReport] = None, | |
| model_name: str = "meta-llama/Llama-3.1-8B-Instruct", | |
| quantization: str = "none") -> CompatReport: | |
| """Run the full GPU->driver->torch->vLLM->model->quantization chain.""" | |
| env = env or detect_environment(probe_ollama=False) | |
| checks: List[CompatCheck] = [] | |
| failed: Optional[str] = None | |
| def blocked(link: str, by: str) -> None: | |
| checks.append(CompatCheck( | |
| link=link, status=SKIPPED, | |
| reason=f"not evaluated: blocked by failed '{by}' check")) | |
| # --- 1. GPU ------------------------------------------------------------- | |
| cuda_gpus = [g for g in env.gpus if g.vendor == "nvidia"] | |
| rocm_gpus = [g for g in env.gpus if g.vendor == "amd"] | |
| gpu_vendor = "nvidia" if cuda_gpus else ("amd" if rocm_gpus else None) | |
| if gpu_vendor: | |
| gpu = (cuda_gpus or rocm_gpus)[0] | |
| checks.append(CompatCheck( | |
| link="gpu", status=PASS, | |
| reason=f"{gpu.vendor.upper()} GPU detected: {gpu.model}" | |
| + (f" ({gpu.memory_mb} MB VRAM)" if gpu.memory_mb else ""))) | |
| else: | |
| failed = "gpu" | |
| apple = next((g for g in env.gpus if g.vendor == "apple"), None) | |
| if env.is_macos: | |
| reason = (f"No CUDA or ROCm capable GPU. This is macOS " | |
| f"({env.machine_arch})" | |
| + (f" with an Apple GPU ({apple.model}), which exposes " | |
| f"Metal/MPS only" if apple else "") | |
| + ". vLLM requires NVIDIA CUDA or AMD ROCm and does not " | |
| "run on macOS.") | |
| remediation = ("Run the vLLM server on a Linux host with an NVIDIA " | |
| "or AMD data-center/consumer GPU; use the 'ollama' " | |
| "backend for local inference on this machine.") | |
| else: | |
| reason = "No NVIDIA or AMD GPU detected (nvidia-smi / rocm-smi absent or empty)." | |
| remediation = "Install GPU drivers or move to a GPU host." | |
| checks.append(CompatCheck(link="gpu", status=FAIL, reason=reason, | |
| remediation=remediation)) | |
| # --- 2. Driver ---------------------------------------------------------- | |
| if failed: | |
| blocked("driver", failed) | |
| elif gpu_vendor == "nvidia": | |
| drv = cuda_gpus[0].driver_version | |
| if drv: | |
| checks.append(CompatCheck(link="driver", status=PASS, | |
| reason=f"NVIDIA driver {drv} reported by nvidia-smi")) | |
| else: | |
| failed = "driver" | |
| checks.append(CompatCheck( | |
| link="driver", status=FAIL, | |
| reason="nvidia-smi present but did not report a driver version", | |
| remediation="Reinstall the NVIDIA driver (>=535 recommended for " | |
| "current vLLM/CUDA 12 wheels).")) | |
| else: # amd | |
| if env.rocm_smi_present: | |
| checks.append(CompatCheck(link="driver", status=PASS, | |
| reason="ROCm stack present (rocm-smi responds)")) | |
| else: | |
| failed = "driver" | |
| checks.append(CompatCheck(link="driver", status=FAIL, | |
| reason="AMD GPU without a working ROCm install", | |
| remediation="Install ROCm >= 6.x for the target GPU.")) | |
| # --- 3. torch ----------------------------------------------------------- | |
| if failed: | |
| blocked("torch", failed) | |
| elif not env.torch.installed: | |
| failed = "torch" | |
| checks.append(CompatCheck( | |
| link="torch", status=FAIL, | |
| reason=f"PyTorch is not installed in this environment ({env.torch.detail})", | |
| remediation="pip install a torch build matching the GPU stack " | |
| "(cu12x wheels for CUDA; rocm wheels for AMD).")) | |
| elif gpu_vendor == "nvidia" and not env.torch.cuda_available: | |
| failed = "torch" | |
| checks.append(CompatCheck( | |
| link="torch", status=FAIL, | |
| reason=f"torch {env.torch.version} installed but torch.cuda.is_available() " | |
| "is False (CPU-only build or driver mismatch)", | |
| remediation="Install a CUDA-enabled torch wheel matching the driver.")) | |
| elif gpu_vendor == "amd" and not env.torch.rocm_version: | |
| failed = "torch" | |
| checks.append(CompatCheck( | |
| link="torch", status=FAIL, | |
| reason=f"torch {env.torch.version} is not a ROCm build", | |
| remediation="Install the ROCm torch wheel (torch.version.hip must be set).")) | |
| else: | |
| checks.append(CompatCheck( | |
| link="torch", status=PASS, | |
| reason=f"torch {env.torch.version} with a matching accelerator build")) | |
| # --- 4. vLLM ------------------------------------------------------------ | |
| if failed: | |
| blocked("vllm", failed) | |
| elif not env.vllm_installed: | |
| failed = "vllm" | |
| checks.append(CompatCheck( | |
| link="vllm", status=FAIL, | |
| reason="vllm package is not installed", | |
| remediation="pip install vllm (CUDA) or build the ROCm image; see " | |
| "docs/retro/compatibility-matrix.md.")) | |
| else: | |
| checks.append(CompatCheck(link="vllm", status=PASS, | |
| reason=f"vllm {env.vllm_version} importable")) | |
| # --- 5. Model fits ------------------------------------------------------ | |
| if failed: | |
| blocked("model", failed) | |
| else: | |
| floor = _estimate_vram_floor(model_name) | |
| vram_mb = (cuda_gpus or rocm_gpus)[0].memory_mb | |
| if floor is None or vram_mb is None: | |
| checks.append(CompatCheck( | |
| link="model", status=PASS, | |
| reason=f"cannot pre-validate VRAM for '{model_name}' " | |
| "(unknown size class or unreported VRAM); vLLM will " | |
| "validate at load time")) | |
| elif vram_mb / 1024 < floor: | |
| failed = "model" | |
| checks.append(CompatCheck( | |
| link="model", status=FAIL, | |
| reason=f"'{model_name}' needs roughly >={floor} GB VRAM unquantized; " | |
| f"GPU reports {vram_mb / 1024:.0f} GB", | |
| remediation="Use quantization (AWQ/GPTQ/FP8), a smaller model, or " | |
| "tensor parallelism across more GPUs.")) | |
| else: | |
| checks.append(CompatCheck( | |
| link="model", status=PASS, | |
| reason=f"'{model_name}' plausibly fits in {vram_mb / 1024:.0f} GB VRAM")) | |
| # --- 6. Quantization ---------------------------------------------------- | |
| q = (quantization or "none").lower() | |
| if failed: | |
| blocked("quantization", failed) | |
| else: | |
| supported = CUDA_QUANTIZATIONS if gpu_vendor == "nvidia" else ROCM_QUANTIZATIONS | |
| if q in supported: | |
| checks.append(CompatCheck( | |
| link="quantization", status=PASS, | |
| reason=f"quantization '{q}' is supported on {gpu_vendor} " | |
| "(verify against the installed vLLM release)")) | |
| else: | |
| failed = "quantization" | |
| checks.append(CompatCheck( | |
| link="quantization", status=FAIL, | |
| reason=f"quantization '{q}' is not supported on {gpu_vendor} " | |
| f"(supported: {sorted(supported)})", | |
| remediation="Pick a supported method or run unquantized.")) | |
| if env.is_macos: | |
| summary = (f"macOS {env.os_version} ({env.machine_arch}): vLLM UNSUPPORTED " | |
| "on this machine. Use the ollama or mock backend locally; run " | |
| "vLLM on a CUDA/ROCm Linux host.") | |
| elif failed: | |
| summary = f"vLLM unsupported here: chain failed at '{failed}'." | |
| else: | |
| summary = "All compatibility checks passed; vLLM should run on this host." | |
| return CompatReport( | |
| vllm_supported=failed is None, | |
| platform_summary=summary, | |
| checks=checks, | |
| failed_link=failed, | |
| notes=list(env.notes), | |
| ) | |
| def format_report(report: CompatReport) -> str: | |
| lines = [report.platform_summary, ""] | |
| for c in report.checks: | |
| mark = {"PASS": "[PASS]", "FAIL": "[FAIL]", "SKIPPED": "[SKIP]"}[c.status] | |
| lines.append(f"{mark} {c.link:<13} {c.reason}") | |
| if c.remediation: | |
| lines.append(f" remediation: {c.remediation}") | |
| return "\n".join(lines) | |
| if __name__ == "__main__": | |
| print(format_report(check_vllm_compatibility())) | |