1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_ENGINE_UTILS_H_ 17 #define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_ENGINE_UTILS_H_ 18 19 #include <string> 20 #include <vector> 21 22 #include "tensorflow/compiler/tf2tensorrt/common/datavec.h" 23 #include "tensorflow/compiler/tf2tensorrt/common/utils.h" 24 #include "tensorflow/compiler/tf2tensorrt/utils/trt_shape_optimization_profiles.h" 25 #include "tensorflow/core/framework/op_kernel.h" 26 #include "tensorflow/core/framework/tensor.h" 27 #include "tensorflow/core/framework/tensor_shape.h" 28 #include "tensorflow/core/lib/core/status.h" 29 #include "tensorflow/stream_executor/lib/statusor.h" 30 31 #if GOOGLE_CUDA && GOOGLE_TENSORRT 32 #include "third_party/tensorrt/NvInfer.h" 33 34 namespace tensorflow { 35 namespace tensorrt { 36 using ::stream_executor::port::StatusOr; 37 38 // Creates a TensorRT execution context. 39 ExecutionContext CreateExecutionContext(nvinfer1::ICudaEngine* cuda_engine); 40 41 // Sets input buffers for TRT from a list of input tensors. The input tensors 42 // are either defined by ctx or by input_vec. 43 Status SetTrtEngineInputs(nvinfer1::ICudaEngine* cuda_engine, 44 nvinfer1::IExecutionContext* execution_context, 45 const int trt_profile_idx, 46 std::vector<void*>& buffers, bool use_implicit_batch, 47 int num_batch, 48 const TrtShapeOptimizationProfile& profiles, 49 OpKernelContext* ctx = nullptr, 50 const DataVec* input_vec = nullptr); 51 52 // Returns the shape of a binding from TensorRT. 53 // 54 // The binding is identified by its binding_index. The batch_size argument is 55 // ignored if use_implicit_batch==false. The shape is returned in the last 56 // argument. 57 Status GetTrtBindingShape(const nvinfer1::ICudaEngine* cuda_engine, 58 const nvinfer1::IExecutionContext* execution_context, 59 int binding_index, bool use_implicit_batch, 60 int batch_size, TensorShape& shape); 61 62 // Defines output buffers for TRT. The buffers are allocated by ctx, if ctx is 63 // not null. Otherwise it is expected that the outputs DataVec is not null, and 64 // the Tensors in outputs are already allocated. 65 Status SetTrtEngineOutputs(nvinfer1::ICudaEngine* cuda_engine, 66 nvinfer1::IExecutionContext* execution_context, 67 int trt_profile_idx, std::vector<void*>& buffers, 68 bool use_implicit_batch, int batch_size = 0, 69 OpKernelContext* ctx = nullptr, 70 DataVec* outputs = nullptr); 71 72 // Enqueues TensorRT inference job. The batch_size argument is only relevant in 73 // implicit batch mode. 74 Status TrtEnqueue(nvinfer1::IExecutionContext* execution_context, 75 std::vector<void*>& buffers, cudaStream_t stream, 76 bool use_implicit_batch, int batch_size = 1); 77 78 } // namespace tensorrt 79 } // namespace tensorflow 80 81 #endif // GOOGLE_CUDA && GOOGLE_TENSORRT 82 83 #endif // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_ENGINE_UTILS_H_ 84