| """Create an InferenceSession that survives a provider this machine cannot run. |
| |
| Shipped inside the onnxruntime package by scripts/runtime/stage_python_bundle.ps1 |
| so that it is importable wherever `import onnxruntime` works: |
| |
| from onnxruntime.uort import create_session |
| |
| sess = create_session("model.onnx") # best available provider |
| sess = create_session("model.onnx", prefer=["DmlExecutionProvider"]) |
| |
| Two ONNX Runtime behaviours make the obvious call unreliable. |
| |
| 1. Provider fallback is narrower than it looks. The python binding retries with |
| the remaining providers only for exceptions derived from ValueError or |
| RuntimeError (onnxruntime/capi/onnxruntime_inference_collection.py, "Falling |
| back to ... and retrying"). TensorRT fails with |
| `onnxruntime.capi.onnxruntime_pybind11_state.Fail`, whose base class is |
| Exception, so an engine that cannot be built for this GPU aborts session |
| creation even when CUDAExecutionProvider is in the list. |
| |
| 2. DirectML refuses to share a session: "DML EP can only be used with CPU EPs". |
| Passing ['CUDAExecutionProvider', 'DmlExecutionProvider', ...] fails for |
| DirectML *and* makes the failure look like a CUDA/TensorRT error, because the |
| provider that is initialised first reports it. |
| |
| So the helper tries whole provider lists, best first, and keeps DirectML in a |
| list of its own. This is what the C++ UORT runtime does by probing providers |
| before creating a session. |
| """ |
|
|
| from __future__ import annotations |
|
|
| |
| |
| DEFAULT_PREFERENCE = [ |
| "TensorrtExecutionProvider", |
| "CUDAExecutionProvider", |
| "DmlExecutionProvider", |
| "OpenVINOExecutionProvider", |
| "CPUExecutionProvider", |
| ] |
|
|
|
|
| def _candidate_lists(prefer, available): |
| names = [name for name in prefer if name in available] |
| if "CPUExecutionProvider" not in names: |
| names.append("CPUExecutionProvider") |
|
|
| lists = [] |
| has_trt_cuda = "TensorrtExecutionProvider" in names and "CUDAExecutionProvider" in names |
| if has_trt_cuda: |
| lists.append(["TensorrtExecutionProvider", "CUDAExecutionProvider", "CPUExecutionProvider"]) |
|
|
| for name in names: |
| if name == "CPUExecutionProvider": |
| continue |
| |
| |
| |
| if has_trt_cuda and name == "TensorrtExecutionProvider": |
| continue |
| lists.append([name, "CPUExecutionProvider"]) |
|
|
| lists.append(["CPUExecutionProvider"]) |
| return lists |
|
|
|
|
| def create_session(model, prefer=None, provider_options=None, quiet=False, allow_fallback=True, **kwargs): |
| """Return an InferenceSession that runs on the best provider this host has. |
| |
| model path to the .onnx file |
| prefer provider names, best first (default: DEFAULT_PREFERENCE) |
| provider_options {provider_name: {option: value}}, applied to that provider |
| quiet do not report which provider was dropped and why |
| allow_fallback False = only try the first candidate (explicit --ep) |
| kwargs forwarded to ort.InferenceSession (sess_options, ...) |
| |
| Raises the last error when no provider can run the model. |
| """ |
| import onnxruntime as ort |
|
|
| options = provider_options or {} |
| available = ort.get_available_providers() |
| candidates = _candidate_lists(prefer or DEFAULT_PREFERENCE, available) |
| if not allow_fallback: |
| candidates = candidates[:1] |
|
|
| last_error = None |
| for names in candidates: |
| providers = [(name, options[name]) if name in options else name for name in names] |
| try: |
| session = ort.InferenceSession(model, providers=providers, **kwargs) |
| if last_error is not None and not quiet: |
| print("using %s" % session.get_providers()[0]) |
| return session |
| except Exception as exc: |
| last_error = exc |
| if not quiet: |
| print("%s cannot run this model: %s" % ("+".join(names), exc)) |
|
|
| raise last_error |
|
|
