1 //
2 // Copyright © 2023 Arm Ltd and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5
6 #pragma once
7
8 #include <armnn_delegate.hpp>
9 #include <DelegateUtils.hpp>
10
11 #include <armnn/ArmNN.hpp>
12 #include <armnn/BackendHelper.hpp>
13 #include <armnn/utility/Assert.hpp>
14 #include <armnn/utility/NumericCast.hpp>
15
16 #include <armnnUtils/Permute.hpp>
17 #include <armnnUtils/TensorUtils.hpp>
18
19 #include <tensorflow/lite/builtin_ops.h>
20 #include <tensorflow/lite/c/builtin_op_data.h>
21 #include <tensorflow/lite/c/common.h>
22 #include <tensorflow/lite/minimal_logging.h>
23 #include <tensorflow/lite/kernels/kernel_util.h>
24
25 namespace
26 {
27
28 // Macro to call an Is<layer_name>Supported function and log caller name together with reason for lack of support
29 #define FORWARD_LAYER_SUPPORT_FUNC(opName, tfLiteContext, func, backends, supported, setBackend, ...) \
30 try \
31 { \
32 for (auto&& backendId : backends) \
33 { \
34 auto layerSupportObject = armnn::GetILayerSupportByBackendId(backendId); \
35 if (layerSupportObject.IsBackendRegistered()) \
36 { \
37 std::string reasonIfUnsupported; \
38 supported = \
39 layerSupportObject.func(__VA_ARGS__, armnn::Optional<std::string&>(reasonIfUnsupported)); \
40 if (supported) \
41 { \
42 setBackend = backendId; \
43 break; \
44 } \
45 else \
46 { \
47 if (reasonIfUnsupported.size() > 0) \
48 { \
49 TFLITE_LOG_PROD(tflite::TFLITE_LOG_WARNING, \
50 "%s: not supported by armnn: %s", opName, reasonIfUnsupported.c_str()); \
51 } \
52 else \
53 { \
54 TFLITE_LOG_PROD(tflite::TFLITE_LOG_WARNING, \
55 "%s: not supported by armnn", opName); \
56 } \
57 } \
58 } \
59 else \
60 { \
61 TF_LITE_KERNEL_LOG(tfLiteContext, "%s: backend not registered: %s", opName, backendId.Get().c_str()); \
62 } \
63 } \
64 if (!supported) \
65 { \
66 TF_LITE_KERNEL_LOG(tfLiteContext, "%s: not supported by any specified backend", opName); \
67 } \
68 } \
69 catch (const armnn::InvalidArgumentException &e) \
70 { \
71 throw armnn::InvalidArgumentException(e, "Failed to check layer support", CHECK_LOCATION()); \
72 }
73
ValidateNumInputs(TfLiteContext * tfLiteContext,TfLiteNode * tfLiteNode,const unsigned int expectedSize,int nodeIndex)74 TfLiteStatus ValidateNumInputs(TfLiteContext* tfLiteContext,
75 TfLiteNode* tfLiteNode,
76 const unsigned int expectedSize,
77 int nodeIndex)
78 {
79 auto numInputs = tfLiteNode->inputs->size;
80 if (static_cast<unsigned int >(numInputs) != expectedSize)
81 {
82 TF_LITE_MAYBE_KERNEL_LOG(
83 tfLiteContext, "TfLiteArmnnDelegate: Unexpected number of inputs (%d != %d) in node #%d",
84 numInputs, expectedSize, nodeIndex);
85 return kTfLiteError;
86 }
87 return kTfLiteOk;
88 }
89
ValidateNumOutputs(TfLiteContext * tfLiteContext,TfLiteNode * tfLiteNode,const unsigned int expectedSize,int nodeIndex)90 TfLiteStatus ValidateNumOutputs(TfLiteContext* tfLiteContext,
91 TfLiteNode* tfLiteNode,
92 const unsigned int expectedSize,
93 int nodeIndex)
94 {
95 auto numOutputs = tfLiteNode->outputs->size;
96 if (static_cast<unsigned int >(numOutputs) != expectedSize)
97 {
98 TF_LITE_MAYBE_KERNEL_LOG(
99 tfLiteContext, "TfLiteArmnnDelegate: Unexpected number of outputs (%d != %d) in node #%d",
100 numOutputs, expectedSize, nodeIndex);
101 return kTfLiteError;
102 }
103 return kTfLiteOk;
104 }
105
IsDynamicTensor(const TfLiteTensor & tfLiteTensor)106 bool IsDynamicTensor(const TfLiteTensor& tfLiteTensor)
107 {
108 auto tensorAllocationType = tfLiteTensor.allocation_type;
109 if (tensorAllocationType == kTfLiteDynamic)
110 {
111 return true;
112 }
113 return false;
114 }
115
IsValid(const TfLiteTensor * tfLiteTensor)116 bool IsValid(const TfLiteTensor* tfLiteTensor)
117 {
118 return tfLiteTensor == nullptr ? false : true;
119 }
120
IsValid(TfLiteContext * tfLiteContext,const TfLiteTensor & tfLiteTensor,int32_t operatorCode,int32_t nodeIndex)121 bool IsValid(TfLiteContext* tfLiteContext, const TfLiteTensor& tfLiteTensor, int32_t operatorCode, int32_t nodeIndex)
122 {
123 if(!IsValid(&tfLiteTensor))
124 {
125 std::cout << "..Is Not Valid" << std::endl;
126 TF_LITE_MAYBE_KERNEL_LOG(
127 tfLiteContext,
128 "TfLiteArmnnDelegate: Invalid TfLite tensor in operator #%d node #%d: ",
129 operatorCode, nodeIndex);
130 return false;
131 }
132 if (IsDynamicTensor(tfLiteTensor))
133 {
134 std::cout << "..IsDynamicTensor" << std::endl;
135 TF_LITE_MAYBE_KERNEL_LOG(
136 tfLiteContext,
137 "TfLiteArmnnDelegate: Dynamic tensors are not supported in operator #%d node #%d: ",
138 operatorCode, nodeIndex);
139 return false;
140 }
141 return true;
142 }
143
IsAffineQuantization(const TfLiteTensor & tfLiteTensor)144 bool IsAffineQuantization(const TfLiteTensor& tfLiteTensor)
145 {
146 auto quantizationInfo = tfLiteTensor.quantization;
147 if (quantizationInfo.type == kTfLiteAffineQuantization)
148 {
149 return true;
150 }
151 return false;
152 }
153
Connect(armnn::IConnectableLayer * layer,TfLiteNode * tfLiteNode,armnnDelegate::DelegateData & data)154 TfLiteStatus Connect(armnn::IConnectableLayer* layer,
155 TfLiteNode* tfLiteNode,
156 armnnDelegate::DelegateData& data)
157 {
158 ARMNN_ASSERT(static_cast<unsigned int>(tfLiteNode->outputs->size) == layer->GetNumOutputSlots());
159
160 // Connect the input slots
161 for (unsigned int inputIndex = 0; inputIndex < layer->GetNumInputSlots(); ++inputIndex)
162 {
163 if (data.m_OutputSlotForNode[tfLiteNode->inputs->data[inputIndex]] != nullptr)
164 {
165 data.m_OutputSlotForNode[tfLiteNode->inputs->data[inputIndex]]->Connect(layer->GetInputSlot(inputIndex));
166 }
167 }
168
169 // Prepare output slots
170 for (unsigned int outputIndex = 0; outputIndex < layer->GetNumOutputSlots(); ++outputIndex)
171 {
172 armnn::IOutputSlot& outputSlot = layer->GetOutputSlot(outputIndex);
173 data.m_OutputSlotForNode[static_cast<unsigned long>(tfLiteNode->outputs->data[outputIndex])] = &outputSlot;
174 }
175
176 return kTfLiteOk;
177 }
178
FusedActivation(TfLiteContext * tfLiteContext,TfLiteNode * tfLiteNode,TfLiteFusedActivation activationType,armnn::IConnectableLayer * prevLayer,unsigned int outputSlotIndex,armnnDelegate::DelegateData & data)179 TfLiteStatus FusedActivation(TfLiteContext* tfLiteContext,
180 TfLiteNode* tfLiteNode,
181 TfLiteFusedActivation activationType,
182 armnn::IConnectableLayer* prevLayer,
183 unsigned int outputSlotIndex,
184 armnnDelegate::DelegateData& data)
185 {
186
187 const armnn::TensorInfo& activationOutputInfo = prevLayer->GetOutputSlot(outputSlotIndex).GetTensorInfo();
188
189 armnn::ActivationDescriptor activationDesc;
190
191 switch (activationType)
192 {
193 case kTfLiteActNone:
194 {
195 // No Activation
196 return kTfLiteOk;
197 }
198 case kTfLiteActRelu:
199 {
200 activationDesc.m_Function = armnn::ActivationFunction::ReLu;
201 break;
202 }
203 // The name of kTfLiteActRelu1 changed after TF Lite v2.3
204 #if defined(ARMNN_POST_TFLITE_2_3)
205 case kTfLiteActReluN1To1:
206 #else
207 case kTfLiteActRelu1:
208 #endif
209 {
210 activationDesc.m_Function = armnn::ActivationFunction::BoundedReLu;
211 activationDesc.m_A = 1.0f;
212 activationDesc.m_B = -1.0f;
213 break;
214 }
215 case kTfLiteActRelu6:
216 {
217 activationDesc.m_Function = armnn::ActivationFunction::BoundedReLu;
218 activationDesc.m_A = 6.0f;
219 activationDesc.m_B = 0.0f;
220 break;
221 }
222 case kTfLiteActSigmoid:
223 {
224 activationDesc.m_Function = armnn::ActivationFunction::Sigmoid;
225 break;
226 }
227 case kTfLiteActTanh:
228 {
229 activationDesc.m_Function = armnn::ActivationFunction::TanH;
230 activationDesc.m_A = 1.0f;
231 activationDesc.m_B = 1.0f;
232 break;
233 }
234 default:
235 return kTfLiteError;
236 }
237
238 bool isSupported = false;
239 armnn::BackendId setBackend;
240 FORWARD_LAYER_SUPPORT_FUNC("ACTIVATION",
241 tfLiteContext,
242 IsActivationSupported,
243 data.m_Backends,
244 isSupported,
245 setBackend,
246 activationOutputInfo,
247 activationOutputInfo,
248 activationDesc);
249 if (!isSupported)
250 {
251 return kTfLiteError;
252 }
253 armnn::IConnectableLayer* activationLayer = data.m_Network->AddActivationLayer(activationDesc);
254 activationLayer->SetBackendId(setBackend);
255
256 ARMNN_ASSERT(activationLayer != nullptr);
257 activationLayer->GetOutputSlot(0).SetTensorInfo(activationOutputInfo);
258
259 // Connect and prepare output slots
260 for (unsigned int outputIndex = 0; outputIndex < activationLayer->GetNumOutputSlots(); ++outputIndex)
261 {
262 data.m_OutputSlotForNode[static_cast<unsigned long>(
263 tfLiteNode->outputs->data[outputIndex])]->Connect(activationLayer->GetInputSlot(0));
264 armnn::IOutputSlot& outputSlot = activationLayer->GetOutputSlot(outputIndex);
265 data.m_OutputSlotForNode[static_cast<unsigned long>(
266 tfLiteNode->outputs->data[outputIndex])] = &outputSlot;
267 }
268 return kTfLiteOk;
269 }
270
AddReshapeLayer(TfLiteContext * tfLiteContext,TfLiteNode * tfLiteNode,armnn::IConnectableLayer * prevLayer,armnn::TensorInfo reshapedOutputTensorInfo,armnn::TensorInfo outputTensorInfo,armnnDelegate::DelegateData & data)271 armnn::IConnectableLayer* AddReshapeLayer(TfLiteContext* tfLiteContext,
272 TfLiteNode* tfLiteNode,
273 armnn::IConnectableLayer* prevLayer,
274 armnn::TensorInfo reshapedOutputTensorInfo,
275 armnn::TensorInfo outputTensorInfo,
276 armnnDelegate::DelegateData& data)
277 {
278 armnn::ReshapeDescriptor desc;
279 desc.m_TargetShape = outputTensorInfo.GetShape();
280
281 bool isSupported = false;
282 armnn::BackendId setBackend;
283 FORWARD_LAYER_SUPPORT_FUNC("RESHAPE",
284 tfLiteContext,
285 IsReshapeSupported,
286 data.m_Backends,
287 isSupported,
288 setBackend,
289 reshapedOutputTensorInfo,
290 outputTensorInfo,
291 desc);
292
293 if (!isSupported)
294 {
295 return nullptr;
296 }
297
298 armnn::IConnectableLayer* reshapeLayer = data.m_Network->AddReshapeLayer(desc);
299 reshapeLayer->SetBackendId(setBackend);
300 ARMNN_ASSERT(reshapeLayer != nullptr);
301
302 prevLayer->GetOutputSlot(0).SetTensorInfo(reshapedOutputTensorInfo);
303 reshapeLayer->GetOutputSlot(0).SetTensorInfo(outputTensorInfo);
304
305 // Connect and prepare output slots
306 for (unsigned int outputIndex = 0; outputIndex < reshapeLayer->GetNumOutputSlots(); ++outputIndex)
307 {
308 data.m_OutputSlotForNode[static_cast<unsigned long>(
309 tfLiteNode->outputs->data[outputIndex])]->Connect(reshapeLayer->GetInputSlot(0));
310 armnn::IOutputSlot& outputSlot = reshapeLayer->GetOutputSlot(outputIndex);
311 data.m_OutputSlotForNode[static_cast<unsigned long>(
312 tfLiteNode->outputs->data[outputIndex])] = &outputSlot;
313 }
314 return reshapeLayer;
315 }
316
GetDataType(const TfLiteTensor & tfLiteTensor)317 armnn::DataType GetDataType(const TfLiteTensor& tfLiteTensor)
318 {
319 switch (tfLiteTensor.type)
320 {
321 case kTfLiteBool:
322 return armnn::DataType::Boolean;
323 case kTfLiteFloat32:
324 return armnn::DataType::Float32;
325 case kTfLiteFloat16:
326 return armnn::DataType::Float16;
327 case kTfLiteUInt8:
328 return armnn::DataType::QAsymmU8;
329 case kTfLiteInt8:
330 {
331 auto quantizationInfo = tfLiteTensor.quantization;
332 if (quantizationInfo.type == kTfLiteAffineQuantization)
333 {
334 auto* quantization =
335 reinterpret_cast<TfLiteAffineQuantization*>(tfLiteTensor.quantization.params);
336 if (quantization->zero_point != nullptr && quantization->zero_point->size == 1)
337 {
338 return armnn::DataType::QAsymmS8;
339 }
340 else
341 {
342 return armnn::DataType::QSymmS8;
343 }
344 }
345 else
346 {
347 return armnn::DataType::QAsymmS8;
348 }
349 }
350 case kTfLiteInt16:
351 return armnn::DataType::QSymmS16;
352 case kTfLiteInt32:
353 return armnn::DataType::Signed32;
354 case kTfLiteInt64:
355 return armnn::DataType::Signed64;
356 default:
357 throw armnn::Exception(&"TfLiteArmnnDelegate: Unsupported data type: " [ tfLiteTensor.type]);
358 }
359 }
360
GetTensorInfoForTfLiteTensor(const TfLiteTensor & tfLiteTensor,bool isOutput=false)361 armnn::TensorInfo GetTensorInfoForTfLiteTensor(const TfLiteTensor& tfLiteTensor, bool isOutput = false)
362 {
363 armnn::DataType type = GetDataType(tfLiteTensor);
364 armnn::TensorInfo ret;
365 auto tensorDimensionSize = tfLiteTensor.dims->size;
366 if (tensorDimensionSize == 0)
367 {
368 // If input tensor does not have a shape
369 // assuming that it has 1D tensor
370 if (!isOutput)
371 {
372 std::vector<unsigned int> safeShape = { 1 };
373 bool dimensionsSpecificity[1] = { true };
374 armnn::TensorShape tensorShape(armnn::numeric_cast<unsigned int>(safeShape.size()),
375 safeShape.data(),
376 dimensionsSpecificity);
377 ret = armnn::TensorInfo(tensorShape, type);
378 if(tflite::IsConstantTensor(&tfLiteTensor))
379 {
380 ret.SetConstant(true);
381 }
382 }
383 else
384 {
385 armnn::TensorShape tensorShape(armnn::Dimensionality::NotSpecified);
386 ret = armnn::TensorInfo(tensorShape, type);
387 }
388 }
389 else
390 {
391 std::vector<unsigned int> tensorDims(static_cast<unsigned int>(tensorDimensionSize));
392 bool dimensionsSpecificity[5] = { true, true, true, true, true };
393 for (unsigned int i = 0; i < static_cast<unsigned int>(tensorDimensionSize); ++i) {
394 auto dim = tfLiteTensor.dims->data[i];
395 if (dim == 0)
396 {
397 dimensionsSpecificity[i] = false;
398 }
399 tensorDims[i] = static_cast<unsigned int>(dim);
400 }
401 armnn::TensorShape tensorShape(static_cast<unsigned int>(tensorDimensionSize),
402 tensorDims.data(),
403 dimensionsSpecificity);
404
405 if(tflite::IsConstantTensor(&tfLiteTensor))
406 {
407 ret = armnn::TensorInfo(tensorShape, type);
408 ret.SetConstant(true);
409 }
410 else
411 {
412 ret = armnn::TensorInfo(tensorShape, type);
413 }
414 }
415
416 auto quantizationInfo = tfLiteTensor.quantization;
417 if (quantizationInfo.type == kTfLiteAffineQuantization)
418 {
419 // get per-channel quantization parameters
420 const auto* affineQuantization =
421 reinterpret_cast<TfLiteAffineQuantization*>(tfLiteTensor.quantization.params);
422 if (affineQuantization->scale->size > 1)
423 {
424 std::vector<float> quantizationScales;
425 for (unsigned int i = 0; i < static_cast<unsigned int>(affineQuantization->scale->size); ++i)
426 {
427 quantizationScales.push_back(affineQuantization->scale->data[i]);
428 }
429 ret.SetQuantizationScales(quantizationScales);
430 ret.SetQuantizationDim(armnn::numeric_cast<unsigned int>(affineQuantization->quantized_dimension));
431 }
432 else
433 {
434 ret.SetQuantizationScale(affineQuantization->scale->data[0]);
435 ret.SetQuantizationOffset(affineQuantization->zero_point->data[0]);
436 }
437 }
438 else
439 {
440 auto quantizationParameters = tfLiteTensor.params;
441 ret.SetQuantizationScale(quantizationParameters.scale);
442 ret.SetQuantizationOffset(quantizationParameters.zero_point);
443 }
444
445 return ret;
446 }
447
CreateConstTensor(const TfLiteTensor * tfLiteTensor,const armnn::TensorInfo & tensorInfo)448 armnn::ConstTensor CreateConstTensor(const TfLiteTensor* tfLiteTensor,
449 const armnn::TensorInfo& tensorInfo)
450 {
451 if (tfLiteTensor->allocation_type != kTfLiteMmapRo)
452 {
453 throw armnn::Exception(
454 "TfLiteArmnnDelegate: Not constant allocation type: " + std::to_string(tfLiteTensor->allocation_type));
455 }
456
457 return armnn::ConstTensor(tensorInfo, tfLiteTensor->data.data);
458 }
459
GetConstTensorForTfLiteTensor(const TfLiteTensor * tfLiteTensors,TfLiteNode * tfLiteNode,int index)460 armnn::ConstTensor* GetConstTensorForTfLiteTensor(const TfLiteTensor* tfLiteTensors, TfLiteNode* tfLiteNode, int index)
461 {
462 const TfLiteTensor &tfLiteTensor = tfLiteTensors[tfLiteNode->inputs->data[index]];
463 armnn::TensorInfo tensorInfo = GetTensorInfoForTfLiteTensor(tfLiteTensor);
464 return new armnn::ConstTensor(tensorInfo, tfLiteTensor.data.data);
465 }
466
IsOptionalOperandPresent(TfLiteNode * tfLiteNode,const int operandIndex)467 bool IsOptionalOperandPresent(TfLiteNode* tfLiteNode, const int operandIndex)
468 {
469 // If the inputs array has fewer than operandIndex entries or if the entry at operandIndex has a value of -1 or
470 // less then the input is not present.
471 if (tfLiteNode->inputs->size > operandIndex && tfLiteNode->inputs->data[operandIndex] >= 0)
472 {
473 return true;
474 }
475 return false;
476 }
477
ProcessInputs(armnn::IConnectableLayer * layer,armnnDelegate::DelegateData & delegateData,TfLiteContext * tfLiteContext,TfLiteNode * tfLiteNode)478 TfLiteStatus ProcessInputs(armnn::IConnectableLayer* layer,
479 armnnDelegate::DelegateData& delegateData,
480 TfLiteContext* tfLiteContext,
481 TfLiteNode* tfLiteNode)
482 {
483 const TfLiteTensor* tfLiteTensors = tfLiteContext->tensors;
484 // Process input tensors
485 // If input tensor is a Constant tensor create a constant layer and connect it to the network
486 for (unsigned int inputIndex = 0; inputIndex < layer->GetNumInputSlots(); ++inputIndex)
487 {
488 const TfLiteTensor& tfLiteInputTensor = tfLiteTensors[tfLiteNode->inputs->data[inputIndex]];
489 if (tflite::IsConstantTensor(&tfLiteInputTensor))
490 {
491 armnn::TensorInfo inputTensorInfo = GetTensorInfoForTfLiteTensor(tfLiteInputTensor);
492 bool isSupported = false;
493 armnn::BackendId setBackend;
494 FORWARD_LAYER_SUPPORT_FUNC("CONSTANT",
495 tfLiteContext,
496 IsConstantSupported,
497 delegateData.m_Backends,
498 isSupported,
499 setBackend,
500 inputTensorInfo);
501 if (!isSupported)
502 {
503 return kTfLiteError;
504 }
505 auto constantInput = CreateConstTensor(&tfLiteInputTensor,
506 inputTensorInfo);
507 armnn::IConnectableLayer* constantLayer = delegateData.m_Network->AddConstantLayer(constantInput);
508 constantLayer->SetBackendId(setBackend);
509 armnn::IOutputSlot& outputSlot = constantLayer->GetOutputSlot(0);
510 outputSlot.SetTensorInfo(inputTensorInfo);
511
512 delegateData.m_OutputSlotForNode[tfLiteNode->inputs->data[inputIndex]] = &outputSlot;
513 }
514 }
515 return kTfLiteOk;
516 }
517
518 } // namespace anonymous
519