Commit ·
cd380a1
1
Parent(s): 9e548e3
Initialize CUDA before llama.cpp
Browse filesLoad Torch CUDA dependencies before llama-cpp-python and retain a bounded failure diagnostic for deployment verification.
Co-Authored-By: Codex <codex@openai.com>
- app/model_endpoint.py +16 -1
app/model_endpoint.py
CHANGED
|
@@ -81,6 +81,10 @@ def _build_model(config: ModelConfig) -> Any:
|
|
| 81 |
for library in root.glob("nvidia/cuda_runtime/lib/libcudart.so*"):
|
| 82 |
ctypes.CDLL(str(library), mode=ctypes.RTLD_GLOBAL)
|
| 83 |
break
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
from llama_cpp import Llama
|
| 85 |
except ImportError as exc:
|
| 86 |
raise ModelRuntimeError("llama-cpp-python is not installed.") from exc
|
|
@@ -225,4 +229,15 @@ def call_model(
|
|
| 225 |
close()
|
| 226 |
del ephemeral_model
|
| 227 |
gc.collect()
|
| 228 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
for library in root.glob("nvidia/cuda_runtime/lib/libcudart.so*"):
|
| 82 |
ctypes.CDLL(str(library), mode=ctypes.RTLD_GLOBAL)
|
| 83 |
break
|
| 84 |
+
import torch
|
| 85 |
+
|
| 86 |
+
if torch.cuda.is_available():
|
| 87 |
+
torch.cuda.init()
|
| 88 |
from llama_cpp import Llama
|
| 89 |
except ImportError as exc:
|
| 90 |
raise ModelRuntimeError("llama-cpp-python is not installed.") from exc
|
|
|
|
| 229 |
close()
|
| 230 |
del ephemeral_model
|
| 231 |
gc.collect()
|
| 232 |
+
detail = (
|
| 233 |
+
f"{type(last_error).__name__}: {last_error}"
|
| 234 |
+
if last_error is not None
|
| 235 |
+
else "unknown generation error"
|
| 236 |
+
)
|
| 237 |
+
return {
|
| 238 |
+
"risk_label": "Inappropriate",
|
| 239 |
+
"simple_explanation": f"Temporary diagnostic: {detail[:500]}",
|
| 240 |
+
"red_flags": ["Temporary model diagnostic"],
|
| 241 |
+
"safe_next_steps": ["Do not use this response as an assessment."],
|
| 242 |
+
"reply_draft": "",
|
| 243 |
+
}
|