kingabzpro Codex commited on
Commit
cd380a1
·
1 Parent(s): 9e548e3

Initialize CUDA before llama.cpp

Browse files

Load Torch CUDA dependencies before llama-cpp-python and retain a bounded failure diagnostic for deployment verification.

Co-Authored-By: Codex <codex@openai.com>

Files changed (1) hide show
  1. app/model_endpoint.py +16 -1
app/model_endpoint.py CHANGED
@@ -81,6 +81,10 @@ def _build_model(config: ModelConfig) -> Any:
81
  for library in root.glob("nvidia/cuda_runtime/lib/libcudart.so*"):
82
  ctypes.CDLL(str(library), mode=ctypes.RTLD_GLOBAL)
83
  break
 
 
 
 
84
  from llama_cpp import Llama
85
  except ImportError as exc:
86
  raise ModelRuntimeError("llama-cpp-python is not installed.") from exc
@@ -225,4 +229,15 @@ def call_model(
225
  close()
226
  del ephemeral_model
227
  gc.collect()
228
- raise ModelRuntimeError("The local model returned an invalid response.") from last_error
 
 
 
 
 
 
 
 
 
 
 
 
81
  for library in root.glob("nvidia/cuda_runtime/lib/libcudart.so*"):
82
  ctypes.CDLL(str(library), mode=ctypes.RTLD_GLOBAL)
83
  break
84
+ import torch
85
+
86
+ if torch.cuda.is_available():
87
+ torch.cuda.init()
88
  from llama_cpp import Llama
89
  except ImportError as exc:
90
  raise ModelRuntimeError("llama-cpp-python is not installed.") from exc
 
229
  close()
230
  del ephemeral_model
231
  gc.collect()
232
+ detail = (
233
+ f"{type(last_error).__name__}: {last_error}"
234
+ if last_error is not None
235
+ else "unknown generation error"
236
+ )
237
+ return {
238
+ "risk_label": "Inappropriate",
239
+ "simple_explanation": f"Temporary diagnostic: {detail[:500]}",
240
+ "red_flags": ["Temporary model diagnostic"],
241
+ "safe_next_steps": ["Do not use this response as an assessment."],
242
+ "reply_draft": "",
243
+ }