1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 #include "tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.h"
16 
17 #include <stdint.h>
18 
19 #include <limits>
20 
21 #include "tensorflow/lite/c/builtin_op_data.h"
22 #include "tensorflow/lite/delegates/hexagon/hexagon_nn/hexagon_nn.h"
23 #include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
24 #include "tensorflow/lite/kernels/kernel_util.h"
25 #include "tensorflow/lite/kernels/padding.h"
26 
27 namespace tflite {
28 namespace delegates {
29 namespace hexagon {
30 namespace {
31 
32 constexpr uint8_t k8BitSignFlipConstant = 0x80;
33 // 1/1024 ~ 0.0009766 is a restriction set by Hexagon's kernels.
34 // TODO(b/151103818): Figure out a way to retrieve this constant reliably.
35 constexpr float kHexagonMinRelativeScale = 0.0009766f;
36 
37 }  // namespace
38 
PopulateSubGraph(const TfLiteIntArray * inputs,const TfLiteIntArray * outputs,TfLiteContext * context)39 TfLiteStatus TransposeConv2dOpBuilder::PopulateSubGraph(
40     const TfLiteIntArray* inputs, const TfLiteIntArray* outputs,
41     TfLiteContext* context) {
42   // DATA TENSOR.
43   int tensor_id = inputs->data[2];
44   const auto& data_tensor = context->tensors[tensor_id];
45   AddInput(graph_builder_->GetHexagonTensorId(tensor_id));
46 
47   // WEIGHTS.
48   tensor_id = inputs->data[1];
49   const auto& weights_tensor = context->tensors[tensor_id];
50   if (weights_tensor.allocation_type != kTfLiteMmapRo) {
51     TF_LITE_KERNEL_LOG(
52         context, "Weights tensor doesn't have correct allocation type: %s",
53         weights_tensor.name);
54     return kTfLiteError;
55   }
56   int filter_batch_size, filter_height_size, filter_width_size,
57       filter_depth_size;
58   GetDims(&filter_batch_size, &filter_height_size, &filter_width_size,
59           &filter_depth_size, weights_tensor.dims);
60   // Weights tensor could be int8 even for per-tensor quantization.
61   // Therefore, we look at the number of scale values to check if it is
62   // per-channel quantized.
63   TfLiteAffineQuantization* weights_quant_params =
64       reinterpret_cast<TfLiteAffineQuantization*>(
65           weights_tensor.quantization.params);
66   const bool is_per_channel_quant = weights_quant_params->scale->size > 1;
67   AddInput(graph_builder_->GetHexagonTensorId(tensor_id));
68 
69   // Handle weights quantization.
70   float weights_min = 0;
71   float weights_max = 0;
72   if (is_per_channel_quant) {
73     ProcessPerChannelQuantizedWeights(weights_tensor, context, &weights_min,
74                                       &weights_max, graph_builder_,
75                                       &per_channel_quant_);
76   } else {
77     TF_LITE_ENSURE_STATUS(ComputeMinAndMaxQuantValues(
78         weights_tensor, &weights_min, &weights_max));
79   }
80   auto* weights_min_const = graph_builder_->AddConstNodeWithData(
81       kScalarShape, reinterpret_cast<char*>(&weights_min), sizeof(weights_min));
82   auto* weights_max_const = graph_builder_->AddConstNodeWithData(
83       kScalarShape, reinterpret_cast<char*>(&weights_max), sizeof(weights_max));
84 
85   // Min/max inputs for data & weights tensors.
86   TF_LITE_ENSURE_STATUS(ComputeAndAddMinAndMax(context, data_tensor));
87   AddInput(TensorID(weights_min_const->GetID(), 0));
88   AddInput(TensorID(weights_max_const->GetID(), 0));
89 
90   // Output dims are required to compute padding.
91   int output_batch_size, output_height_size, output_width_size,
92       output_depth_size;
93   GetDims(&output_batch_size, &output_height_size, &output_width_size,
94           &output_depth_size, context->tensors[outputs->data[0]].dims);
95 
96   // PADDING & STRIDE.
97   // Hexagon TransposeConv requires an explicit padding tensor. So we compute
98   // the same using stride, input & output info.
99   const TfLiteTransposeConvParams* params =
100       reinterpret_cast<const TfLiteTransposeConvParams*>(builtin_data_);
101   int unused_output_height, unused_output_width;
102   TfLitePaddingValues padding = ComputePaddingHeightWidth(
103       params->stride_height, params->stride_width, 1, 1, output_height_size,
104       output_width_size, filter_height_size, filter_width_size, params->padding,
105       &unused_output_height, &unused_output_width);
106   std::vector<int> padding_tensor = {padding.height, padding.height,
107                                      padding.width, padding.width};
108   std::vector<int> padding_tensor_shape = {1, 1, 2, 2};
109   auto* padding_const = graph_builder_->AddConstNodeWithData(
110       padding_tensor_shape.data(),
111       reinterpret_cast<char*>(padding_tensor.data()), (sizeof(int) * 4));
112   AddInput(TensorID(padding_const->GetID(), 0));
113 
114   // Stride shape.
115   int stride_height = params->stride_height;
116   int stride_width = params->stride_width;
117   static int dummy = 0;
118   stride_shape_ = {1, stride_height, stride_width, 1};
119   auto* stride_node = graph_builder_->AddConstNodeWithData(
120       stride_shape_.data(), reinterpret_cast<char*>(&dummy), sizeof(dummy));
121   AddInput(TensorID(stride_node->GetID(), 0));
122 
123   // BIAS.
124   const bool has_bias = inputs->size == 4;
125   OpBuilder* bias_const = nullptr;
126   OpBuilder* bias_min_const = nullptr;
127   OpBuilder* bias_max_const = nullptr;
128   if (!has_bias) {
129     // If the TFLite node does not have a bias, we simply feed in 0s.
130     std::vector<int> bias_data(output_depth_size, 0);
131     bias_shape_ = {1, 1, 1, output_depth_size};
132     bias_const = graph_builder_->AddConstNodeWithData(
133         bias_shape_.data(), reinterpret_cast<char*>(bias_data.data()),
134         sizeof(bias_data[0]) * bias_data.size());
135     float zero_bound = 0;
136     bias_min_const = graph_builder_->AddConstNodeWithData(
137         kScalarShape, reinterpret_cast<char*>(&zero_bound), sizeof(zero_bound));
138     bias_max_const = graph_builder_->AddConstNodeWithData(
139         kScalarShape, reinterpret_cast<char*>(&zero_bound), sizeof(zero_bound));
140   } else {
141     const auto& bias_tensor = context->tensors[inputs->data[3]];
142     if (bias_tensor.allocation_type != kTfLiteMmapRo) {
143       TF_LITE_KERNEL_LOG(context,
144                          "Bias tensor doesn't have correct allocation type: %s",
145                          bias_tensor.name);
146       return kTfLiteError;
147     }
148     float bias_min = 0;
149     float bias_max = 0;
150     if (per_channel_quant_.channel_scales_node != nullptr) {
151       std::vector<int> preprocessed_bias_data;
152       ProcessPerChannelQuantizedBias(data_tensor, bias_tensor, inputs->data[3],
153                                      context, &bias_min, &bias_max,
154                                      graph_builder_, &per_channel_quant_,
155                                      &preprocessed_bias_data, &bias_const);
156     } else {
157       bias_const =
158           graph_builder_->AddConstNodeWithData(inputs->data[3], bias_tensor);
159       TF_LITE_ENSURE_STATUS(
160           ComputeMinAndMaxQuantValues(bias_tensor, &bias_min, &bias_max));
161     }
162 
163     bias_min_const = graph_builder_->AddConstNodeWithData(
164         kScalarShape, reinterpret_cast<char*>(&bias_min), sizeof(bias_min));
165     bias_max_const = graph_builder_->AddConstNodeWithData(
166         kScalarShape, reinterpret_cast<char*>(&bias_max), sizeof(bias_max));
167   }
168   AddInput(TensorID(bias_const->GetID(), 0));
169   AddInput(TensorID(bias_min_const->GetID(), 0));
170   AddInput(TensorID(bias_max_const->GetID(), 0));
171 
172   // Output quantization.
173   TF_LITE_ENSURE_STATUS(
174       ComputeAndAddMinAndMax(context, context->tensors[outputs->data[0]]));
175 
176   // Channel scales, if this op is per-channel quantized.
177   if (per_channel_quant_.channel_scales_node != nullptr) {
178     AddInput(TensorID(per_channel_quant_.channel_scales_node->GetID(), 0));
179   }
180 
181   // Hexagon outputs for this node.
182   node_output_ = AddOutput(sizeof(uint8_t), 4,
183                            {output_batch_size, output_height_size,
184                             output_width_size, output_depth_size});
185   AddOutput(sizeof(float), 4, kScalarShape);
186   AddOutput(sizeof(float), 4, kScalarShape);
187 
188   return kTfLiteOk;
189 }
190 
RegisterOutputs(const TfLiteIntArray * outputs,TfLiteContext * context)191 TfLiteStatus TransposeConv2dOpBuilder::RegisterOutputs(
192     const TfLiteIntArray* outputs, TfLiteContext* context) {
193   // Should be only 1 output.
194   graph_builder_->AddTensorWithID(outputs->data[0], node_output_.first,
195                                   node_output_.second);
196   return kTfLiteOk;
197 }
198 
~TransposeConv2dOpBuilder()199 TransposeConv2dOpBuilder::~TransposeConv2dOpBuilder() {}
200 
CreateTransposeConv2DBuilder(GraphBuilder * graph_builder,int op_type)201 OpBuilder* CreateTransposeConv2DBuilder(GraphBuilder* graph_builder,
202                                         int op_type) {
203   return new TransposeConv2dOpBuilder(graph_builder, op_type);
204 }
205 
206 }  // namespace hexagon
207 }  // namespace delegates
208 }  // namespace tflite
209