# # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import numpy as np import tensorrt as trt import pycuda.driver as cuda import pycuda.autoinit class WrapperBase: def close(self): pass class SP_ONNXClassifierWrapper(WrapperBase): def __init__( self, file, target_dtype=np.float32, batch_size=1, max_keypoints=512, descriptor_dim=256, ): self.target_dtype = target_dtype self.load(file) self.stream = None self.max_keypoints = max_keypoints self.batch_size = batch_size self.descriptor_dim = descriptor_dim def load(self, file): f = open(file, "rb") self.runtime = trt.Runtime(trt.Logger(trt.Logger.WARNING)) self.engine = self.runtime.deserialize_cuda_engine(f.read()) self.context = self.engine.create_execution_context() def allocate_memory(self, batch): self.keypoints = np.empty( (self.batch_size, self.max_keypoints, 2), dtype=np.float32 ) self.scores = np.empty((self.batch_size, self.max_keypoints), dtype=np.float32) self.descriptors = np.empty( (self.batch_size, self.max_keypoints, self.descriptor_dim), dtype=np.float32 ) # Allocate device memory self.d_input = cuda.mem_alloc(1 * batch.nbytes) self.d_keypoints = cuda.mem_alloc(1 * self.keypoints.nbytes) self.d_scores = cuda.mem_alloc(1 * self.scores.nbytes) self.d_descriptors = cuda.mem_alloc(1 * self.descriptors.nbytes) tensor_names = [ self.engine.get_tensor_name(i) for i in range(self.engine.num_io_tensors) ] self.context.set_tensor_address(tensor_names[0], int(self.d_input)) self.context.set_tensor_address(tensor_names[1], int(self.d_keypoints)) self.context.set_tensor_address(tensor_names[2], int(self.d_scores)) self.context.set_tensor_address(tensor_names[3], int(self.d_descriptors)) self.stream = cuda.Stream() def predict(self, batch): if self.stream is None: self.allocate_memory(batch) # Transfer input data to device cuda.memcpy_htod_async(self.d_input, batch, self.stream) # Execute model self.context.execute_async_v3(self.stream.handle) # Transfer predictions back cuda.memcpy_dtoh_async(self.keypoints, self.d_keypoints, self.stream) cuda.memcpy_dtoh_async(self.scores, self.d_scores, self.stream) cuda.memcpy_dtoh_async(self.descriptors, self.d_descriptors, self.stream) # Syncronize threads self.stream.synchronize() return self.keypoints, self.scores, self.descriptors