Spaces:
Paused
Paused
File size: 10,381 Bytes
6b3874e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 | """vLLM compatibility validation chain.
Chain order (each link depends on the previous):
GPU -> driver -> torch -> vLLM -> model -> quantization
Every link produces a CompatCheck with PASS / FAIL / SKIPPED (blocked by an
earlier failure) plus a human-readable reason and remediation. On this
project's development machine (macOS, no CUDA/ROCm) the chain fails honestly
at the GPU link and reports vLLM as unsupported.
"""
from __future__ import annotations
from typing import List, Optional
from pydantic import BaseModel, Field
from sensorflow.retro.inference.env_detect import EnvironmentReport, detect_environment
PASS = "PASS"
FAIL = "FAIL"
SKIPPED = "SKIPPED"
# Quantization methods vLLM supports on CUDA; ROCm support is narrower.
# See docs/retro/compatibility-matrix.md for the substantiated matrix.
CUDA_QUANTIZATIONS = {"awq", "gptq", "fp8", "int8", "none"}
ROCM_QUANTIZATIONS = {"fp8", "gptq", "none"} # verify against current vLLM release
# Very rough VRAM floor (GB) per model family for a usable context window.
MODEL_VRAM_FLOOR_GB = {
"7b": 16, "8b": 18, "13b": 28, "14b": 30, "32b": 68, "70b": 140,
}
class CompatCheck(BaseModel):
link: str # gpu | driver | torch | vllm | model | quantization
status: str # PASS | FAIL | SKIPPED
reason: str
remediation: Optional[str] = None
class CompatReport(BaseModel):
vllm_supported: bool
platform_summary: str
checks: List[CompatCheck]
failed_link: Optional[str] = None
notes: List[str] = Field(default_factory=list)
def _estimate_vram_floor(model_name: str) -> Optional[int]:
lname = model_name.lower()
for size, gb in MODEL_VRAM_FLOOR_GB.items():
if size in lname:
return gb
return None
def check_vllm_compatibility(env: Optional[EnvironmentReport] = None,
model_name: str = "meta-llama/Llama-3.1-8B-Instruct",
quantization: str = "none") -> CompatReport:
"""Run the full GPU->driver->torch->vLLM->model->quantization chain."""
env = env or detect_environment(probe_ollama=False)
checks: List[CompatCheck] = []
failed: Optional[str] = None
def blocked(link: str, by: str) -> None:
checks.append(CompatCheck(
link=link, status=SKIPPED,
reason=f"not evaluated: blocked by failed '{by}' check"))
# --- 1. GPU -------------------------------------------------------------
cuda_gpus = [g for g in env.gpus if g.vendor == "nvidia"]
rocm_gpus = [g for g in env.gpus if g.vendor == "amd"]
gpu_vendor = "nvidia" if cuda_gpus else ("amd" if rocm_gpus else None)
if gpu_vendor:
gpu = (cuda_gpus or rocm_gpus)[0]
checks.append(CompatCheck(
link="gpu", status=PASS,
reason=f"{gpu.vendor.upper()} GPU detected: {gpu.model}"
+ (f" ({gpu.memory_mb} MB VRAM)" if gpu.memory_mb else "")))
else:
failed = "gpu"
apple = next((g for g in env.gpus if g.vendor == "apple"), None)
if env.is_macos:
reason = (f"No CUDA or ROCm capable GPU. This is macOS "
f"({env.machine_arch})"
+ (f" with an Apple GPU ({apple.model}), which exposes "
f"Metal/MPS only" if apple else "")
+ ". vLLM requires NVIDIA CUDA or AMD ROCm and does not "
"run on macOS.")
remediation = ("Run the vLLM server on a Linux host with an NVIDIA "
"or AMD data-center/consumer GPU; use the 'ollama' "
"backend for local inference on this machine.")
else:
reason = "No NVIDIA or AMD GPU detected (nvidia-smi / rocm-smi absent or empty)."
remediation = "Install GPU drivers or move to a GPU host."
checks.append(CompatCheck(link="gpu", status=FAIL, reason=reason,
remediation=remediation))
# --- 2. Driver ----------------------------------------------------------
if failed:
blocked("driver", failed)
elif gpu_vendor == "nvidia":
drv = cuda_gpus[0].driver_version
if drv:
checks.append(CompatCheck(link="driver", status=PASS,
reason=f"NVIDIA driver {drv} reported by nvidia-smi"))
else:
failed = "driver"
checks.append(CompatCheck(
link="driver", status=FAIL,
reason="nvidia-smi present but did not report a driver version",
remediation="Reinstall the NVIDIA driver (>=535 recommended for "
"current vLLM/CUDA 12 wheels)."))
else: # amd
if env.rocm_smi_present:
checks.append(CompatCheck(link="driver", status=PASS,
reason="ROCm stack present (rocm-smi responds)"))
else:
failed = "driver"
checks.append(CompatCheck(link="driver", status=FAIL,
reason="AMD GPU without a working ROCm install",
remediation="Install ROCm >= 6.x for the target GPU."))
# --- 3. torch -----------------------------------------------------------
if failed:
blocked("torch", failed)
elif not env.torch.installed:
failed = "torch"
checks.append(CompatCheck(
link="torch", status=FAIL,
reason=f"PyTorch is not installed in this environment ({env.torch.detail})",
remediation="pip install a torch build matching the GPU stack "
"(cu12x wheels for CUDA; rocm wheels for AMD)."))
elif gpu_vendor == "nvidia" and not env.torch.cuda_available:
failed = "torch"
checks.append(CompatCheck(
link="torch", status=FAIL,
reason=f"torch {env.torch.version} installed but torch.cuda.is_available() "
"is False (CPU-only build or driver mismatch)",
remediation="Install a CUDA-enabled torch wheel matching the driver."))
elif gpu_vendor == "amd" and not env.torch.rocm_version:
failed = "torch"
checks.append(CompatCheck(
link="torch", status=FAIL,
reason=f"torch {env.torch.version} is not a ROCm build",
remediation="Install the ROCm torch wheel (torch.version.hip must be set)."))
else:
checks.append(CompatCheck(
link="torch", status=PASS,
reason=f"torch {env.torch.version} with a matching accelerator build"))
# --- 4. vLLM ------------------------------------------------------------
if failed:
blocked("vllm", failed)
elif not env.vllm_installed:
failed = "vllm"
checks.append(CompatCheck(
link="vllm", status=FAIL,
reason="vllm package is not installed",
remediation="pip install vllm (CUDA) or build the ROCm image; see "
"docs/retro/compatibility-matrix.md."))
else:
checks.append(CompatCheck(link="vllm", status=PASS,
reason=f"vllm {env.vllm_version} importable"))
# --- 5. Model fits ------------------------------------------------------
if failed:
blocked("model", failed)
else:
floor = _estimate_vram_floor(model_name)
vram_mb = (cuda_gpus or rocm_gpus)[0].memory_mb
if floor is None or vram_mb is None:
checks.append(CompatCheck(
link="model", status=PASS,
reason=f"cannot pre-validate VRAM for '{model_name}' "
"(unknown size class or unreported VRAM); vLLM will "
"validate at load time"))
elif vram_mb / 1024 < floor:
failed = "model"
checks.append(CompatCheck(
link="model", status=FAIL,
reason=f"'{model_name}' needs roughly >={floor} GB VRAM unquantized; "
f"GPU reports {vram_mb / 1024:.0f} GB",
remediation="Use quantization (AWQ/GPTQ/FP8), a smaller model, or "
"tensor parallelism across more GPUs."))
else:
checks.append(CompatCheck(
link="model", status=PASS,
reason=f"'{model_name}' plausibly fits in {vram_mb / 1024:.0f} GB VRAM"))
# --- 6. Quantization ----------------------------------------------------
q = (quantization or "none").lower()
if failed:
blocked("quantization", failed)
else:
supported = CUDA_QUANTIZATIONS if gpu_vendor == "nvidia" else ROCM_QUANTIZATIONS
if q in supported:
checks.append(CompatCheck(
link="quantization", status=PASS,
reason=f"quantization '{q}' is supported on {gpu_vendor} "
"(verify against the installed vLLM release)"))
else:
failed = "quantization"
checks.append(CompatCheck(
link="quantization", status=FAIL,
reason=f"quantization '{q}' is not supported on {gpu_vendor} "
f"(supported: {sorted(supported)})",
remediation="Pick a supported method or run unquantized."))
if env.is_macos:
summary = (f"macOS {env.os_version} ({env.machine_arch}): vLLM UNSUPPORTED "
"on this machine. Use the ollama or mock backend locally; run "
"vLLM on a CUDA/ROCm Linux host.")
elif failed:
summary = f"vLLM unsupported here: chain failed at '{failed}'."
else:
summary = "All compatibility checks passed; vLLM should run on this host."
return CompatReport(
vllm_supported=failed is None,
platform_summary=summary,
checks=checks,
failed_link=failed,
notes=list(env.notes),
)
def format_report(report: CompatReport) -> str:
lines = [report.platform_summary, ""]
for c in report.checks:
mark = {"PASS": "[PASS]", "FAIL": "[FAIL]", "SKIPPED": "[SKIP]"}[c.status]
lines.append(f"{mark} {c.link:<13} {c.reason}")
if c.remediation:
lines.append(f" remediation: {c.remediation}")
return "\n".join(lines)
if __name__ == "__main__":
print(format_report(check_vllm_compatibility()))
|