Deploy CUDA-enforced runtime changes for OCR

app.py

@@ -257,6 +257,27 @@ def _assert_model_on_cuda(model, model_id: str, context: str) -> None:
         )
 
 
+def _move_pipeline_to_cuda(pipeline, model_id: str, context: str):
+    if torch is None or not torch.cuda.is_available():
+        return pipeline
+    candidate_model = getattr(pipeline, "model", None)
+    try:
+        if hasattr(pipeline, "to"):
+            pipeline = pipeline.to("cuda")
+        elif candidate_model is not None:
+            candidate_model = candidate_model.to("cuda")
+            if hasattr(pipeline, "model"):
+                pipeline.model = candidate_model
+        else:
+            raise RuntimeError("pipeline has no move target.")
+        return pipeline
+    except Exception as exc:
+        raise RuntimeError(
+            f"{context}: {model_id} failed to move pipeline to CUDA ({type(exc).__name__}: {exc}). "
+            "If you intentionally allow CPU execution, set ALLOW_CPU_FALLBACK=1."
+        )
+
+
 def _inference_torch_kwargs_for_model(strict_cuda: bool = False) -> Tuple[dict, list]:
     kwargs_priority = []
     dtype = _inference_dtype()
@@ -754,6 +775,11 @@ def _build_local_pipeline(model_id: str, hf_token: str, task: str, trust_remote_
             try:
                 candidate_pipeline = hf_pipeline(candidate_task, **cleaned_kwargs)
                 if strict_cuda:
+                    candidate_pipeline = _move_pipeline_to_cuda(
+                        candidate_pipeline,
+                        model_id,
+                        f"pipeline load for {candidate_task}",
+                    )
                     candidate_model = getattr(candidate_pipeline, "model", None)
                     if candidate_model is not None and torch is not None and torch.cuda.is_available():
                         try:
@@ -1083,6 +1109,12 @@ def run_local_model(entry: dict, image_bytes: bytes, prompt: str, hf_token: str)
 
     if pipeline is not None:
         try:
+            if torch is not None and torch.cuda.is_available():
+                pipeline = _move_pipeline_to_cuda(
+                    pipeline,
+                    model_id,
+                    "pipeline execution",
+                )
             if _strict_cuda_required():
                 _assert_model_on_cuda(getattr(pipeline, "model", None), model_id, "pipeline execution")
                 pipeline_model = getattr(pipeline, "model", None)
@@ -1238,7 +1270,9 @@ def run_single_model(entry: dict, image_bytes: bytes, data_uri: str, hf_token: s
     prompt = entry.get("prompt", DEFAULT_OCR_PROMPT)
     params = sanitize_generation_params(entry.get("parameters", {}))
     notes = entry.get("notes", "")
-    notes = f"{notes} [runtime: {_inference_device_label()}]" if notes else f"runtime: {_inference_device_label()}"
+    runtime = _get_runtime_profile()
+    runtime_tag = f"runtime: {_inference_device_label()} | provider: {runtime.get('provider')}"
+    notes = f"{notes} [{runtime_tag}]" if notes else runtime_tag
     start = time.perf_counter()
     output = ""
 
@@ -1298,6 +1332,8 @@ def _get_runtime_profile() -> Dict[str, object]:
 
         if torch.cuda.is_available():
             profile["has_cuda"] = True
+            if not profile["provider"] or profile["provider"] == "cpu":
+                profile["provider"] = "cuda"
             props = torch.cuda.get_device_properties(0)
             profile["gpu_name"] = props.name
             profile["gpu_vram_gb"] = round(props.total_memory / (1024 ** 3), 1)
@@ -1310,7 +1346,9 @@ def _get_runtime_profile() -> Dict[str, object]:
                 profile["gpu_free_vram_gb"] = round(free_approx / (1024 ** 3), 1)
             profile["gpu_reserved_vram_gb"] = round(torch.cuda.memory_reserved(0) / (1024 ** 3), 1)
             profile["gpu_allocated_vram_gb"] = round(torch.cuda.memory_allocated(0) / (1024 ** 3), 1)
-            if "a10g" in (props.name or "").lower() or "l4" in (props.name or "").lower():
+            provider = str(profile["provider"] or "").lower()
+            normalized_name = str(props.name or "").lower()
+            if "zero" in provider or "a10g" in normalized_name or "l4" in normalized_name:
                 profile["provider"] = "zero_gpu"
     except Exception:
         pass
@@ -1343,6 +1381,10 @@ def _startup_cuda_audit() -> None:
         f"cuda_built={runtime.get('torch_cuda_built')}"
     )
     if runtime.get("has_cuda"):
+        provider = runtime.get("provider")
+        if str(provider).lower() == "cpu":
+            provider = "cuda"
+            runtime["provider"] = provider
         print(
             "[startup] Compute backend: "
             f"{runtime.get('provider')} · {runtime.get('gpu_name')} · total {runtime.get('gpu_vram_gb')}GB · "