1 //
2 // Copyright © 2020-2023 Arm Ltd and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5
6 #include <armnn_delegate.hpp>
7
8 #include "Version.hpp"
9
10 #include "Activation.hpp"
11 #include "ArgMinMax.hpp"
12 #include "BatchMatMul.hpp"
13 #include "BatchSpace.hpp"
14 #include "Comparison.hpp"
15 #include "Convolution.hpp"
16 #include "Control.hpp"
17 #include "ElementwiseBinary.hpp"
18 #include "ElementwiseUnary.hpp"
19 #include "Fill.hpp"
20 #include "FullyConnected.hpp"
21 #include "Gather.hpp"
22 #include "GatherNd.hpp"
23 #include "LogicalBinary.hpp"
24 #include "Lstm.hpp"
25 #include "Normalization.hpp"
26 #include "Pack.hpp"
27 #include "Pad.hpp"
28 #include "Pooling.hpp"
29 #include "Prelu.hpp"
30 #include "Quantization.hpp"
31 #include "Redefine.hpp"
32 #include "Reduce.hpp"
33 #include "Resize.hpp"
34 #include "Round.hpp"
35 #include "Shape.hpp"
36 #include "Slice.hpp"
37 #include "StridedSlice.hpp"
38 #include "Softmax.hpp"
39 #include "SpaceDepth.hpp"
40 #include "Split.hpp"
41 #include "Transpose.hpp"
42 #include "UnidirectionalSequenceLstm.hpp"
43 #include "Unpack.hpp"
44
45 #include <armnnUtils/Filesystem.hpp>
46 #include <armnn/utility/Timer.hpp>
47 #include <flatbuffers/flatbuffers.h>
48 #include <tensorflow/lite/context_util.h>
49 #include <tensorflow/lite/schema/schema_generated.h>
50
51 #include <algorithm>
52 #include <iostream>
53 #include <sstream>
54
55 namespace armnnDelegate
56 {
57
TfLiteArmnnDelegateOptionsDefault()58 DelegateOptions TfLiteArmnnDelegateOptionsDefault()
59 {
60 DelegateOptions options(armnn::Compute::CpuRef);
61 return options;
62 }
63
TfLiteArmnnDelegateCreate(armnnDelegate::DelegateOptions options)64 TfLiteDelegate* TfLiteArmnnDelegateCreate(armnnDelegate::DelegateOptions options)
65 {
66 auto* armnnDelegate = new ::armnnDelegate::Delegate(options);
67 return armnnDelegate->GetDelegate();
68 }
69
TfLiteArmnnDelegateDelete(TfLiteDelegate * tfLiteDelegate)70 void TfLiteArmnnDelegateDelete(TfLiteDelegate* tfLiteDelegate)
71 {
72 if (tfLiteDelegate != nullptr)
73 {
74 delete static_cast<::armnnDelegate::Delegate*>(tfLiteDelegate->data_);
75 }
76 }
77
DoPrepare(TfLiteContext * tfLiteContext,TfLiteDelegate * tfLiteDelegate)78 TfLiteStatus DoPrepare(TfLiteContext* tfLiteContext, TfLiteDelegate* tfLiteDelegate)
79 {
80 TfLiteIntArray* supportedOperators =
81 static_cast<::armnnDelegate::Delegate*>(tfLiteDelegate->data_)->IdentifyOperatorsToDelegate(tfLiteContext);
82
83 // ArmNN Delegate Registration
84 static const TfLiteRegistration kArmnnSubgraphRegistration = {
85 // ArmnnSubgraph Init
86 .init = [](TfLiteContext* tfLiteContext, const char* buffer, size_t length) -> void* {
87 armnn::IgnoreUnused(length);
88 const TfLiteDelegateParams* parameters = reinterpret_cast<const TfLiteDelegateParams*>(buffer);
89
90 return static_cast<void*>(ArmnnSubgraph::Create(
91 tfLiteContext, parameters, static_cast<::armnnDelegate::Delegate*>(parameters->delegate->data_)));
92 },
93 // ArmnnSubgraph Free
94 .free = [](TfLiteContext* tfLiteContext, void* buffer) -> void {
95 armnn::IgnoreUnused(tfLiteContext);
96 if (buffer != nullptr)
97 {
98 delete static_cast<ArmnnSubgraph*>(buffer);
99 }
100 },
101 // ArmnnSubgraph Prepare
102 .prepare = [](TfLiteContext* tfLiteContext, TfLiteNode* tfLiteNode) -> TfLiteStatus {
103 if (tfLiteNode->user_data == nullptr)
104 {
105 return kTfLiteError;
106 }
107 return static_cast<ArmnnSubgraph*>(tfLiteNode->user_data)->Prepare(tfLiteContext);
108 },
109 // ArmnnSubgraph Invoke
110 .invoke = [](TfLiteContext* tfLiteContext, TfLiteNode* tfLiteNode) -> TfLiteStatus {
111 if (tfLiteNode->user_data == nullptr)
112 {
113 return kTfLiteError;
114 }
115
116 return static_cast<ArmnnSubgraph*>(tfLiteNode->user_data)->Invoke(tfLiteContext, tfLiteNode);
117 },
118
119 .profiling_string = nullptr,
120 .builtin_code = kTfLiteBuiltinDelegate,
121 .custom_name = "TfLiteArmNnDelegate",
122 .version = 1,
123 .registration_external = nullptr,
124 };
125
126 const TfLiteStatus status =
127 tfLiteContext->ReplaceNodeSubsetsWithDelegateKernels(
128 tfLiteContext, kArmnnSubgraphRegistration, supportedOperators, tfLiteDelegate);
129
130 TfLiteIntArrayFree(supportedOperators);
131 return status;
132
133 }
134
Delegate(armnnDelegate::DelegateOptions options)135 Delegate::Delegate(armnnDelegate::DelegateOptions options)
136 : m_Options(std::move(options))
137 {
138 // Configures logging for ARMNN
139 if (m_Options.IsLoggingEnabled())
140 {
141 armnn::ConfigureLogging(true, true, m_Options.GetLoggingSeverity());
142 }
143 // Create/Get the static ArmNN Runtime. Note that the m_Runtime will be shared by all armnn_delegate
144 // instances so the RuntimeOptions cannot be altered for different armnn_delegate instances.
145 m_Runtime = GetRuntime(m_Options.GetRuntimeOptions());
146 std::vector<armnn::BackendId> backends;
147 if (m_Runtime)
148 {
149 const armnn::BackendIdSet supportedDevices = m_Runtime->GetDeviceSpec().GetSupportedBackends();
150 for (auto& backend : m_Options.GetBackends())
151 {
152 if (std::find(supportedDevices.cbegin(), supportedDevices.cend(), backend) == supportedDevices.cend())
153 {
154 TFLITE_LOG_PROD(tflite::TFLITE_LOG_INFO,
155 "TfLiteArmnnDelegate: Requested unknown backend %s", backend.Get().c_str());
156 }
157 else
158 {
159 backends.push_back(backend);
160 }
161 }
162 }
163
164 if (backends.empty())
165 {
166 // No known backend specified
167 throw armnn::InvalidArgumentException("TfLiteArmnnDelegate: No known backend specified.");
168 }
169 m_Options.SetBackends(backends);
170
171 TFLITE_LOG_PROD_ONCE(tflite::TFLITE_LOG_INFO, "TfLiteArmnnDelegate: Created TfLite ArmNN delegate.");
172 }
173
IdentifyOperatorsToDelegate(TfLiteContext * tfLiteContext)174 TfLiteIntArray* Delegate::IdentifyOperatorsToDelegate(TfLiteContext* tfLiteContext)
175 {
176 TfLiteIntArray* executionPlan = nullptr;
177 if (tfLiteContext->GetExecutionPlan(tfLiteContext, &executionPlan) != kTfLiteOk)
178 {
179 TF_LITE_KERNEL_LOG(tfLiteContext, "TfLiteArmnnDelegate: Unable to get graph execution plan.");
180 return nullptr;
181 }
182
183 // Delegate data with null network
184 DelegateData delegateData(m_Options.GetBackends());
185
186 TfLiteIntArray* nodesToDelegate = TfLiteIntArrayCreate(executionPlan->size);
187 nodesToDelegate->size = 0;
188
189 std::set<int32_t> unsupportedOperators;
190
191 for (int i = 0; i < executionPlan->size; ++i)
192 {
193 const int nodeIndex = executionPlan->data[i];
194
195 // If TfLite nodes can be delegated to ArmNN
196 TfLiteNode* tfLiteNode = nullptr;
197 TfLiteRegistration* tfLiteRegistration = nullptr;
198 if (tfLiteContext->GetNodeAndRegistration(
199 tfLiteContext, nodeIndex, &tfLiteNode, &tfLiteRegistration) != kTfLiteOk)
200 {
201 TF_LITE_KERNEL_LOG(tfLiteContext,
202 "TfLiteArmnnDelegate: Unable to get node and registration for node %d.",
203 nodeIndex);
204 continue;
205 }
206
207 TfLiteStatus visitStatus;
208
209 try
210 {
211 visitStatus = ArmnnSubgraph::VisitNode(
212 delegateData, tfLiteContext, tfLiteRegistration, tfLiteNode, nodeIndex);
213 }
214 catch(std::exception& ex)
215 {
216 ARMNN_LOG(error) << "ArmNN Failed to visit node with error: " << ex.what();
217 visitStatus = kTfLiteError;
218 }
219
220 if ( visitStatus != kTfLiteOk)
221 {
222 // node is not supported by ArmNN
223 unsupportedOperators.insert(tfLiteRegistration->builtin_code);
224 continue;
225 }
226
227 nodesToDelegate->data[nodesToDelegate->size++] = nodeIndex;
228 }
229
230 for (std::set<int32_t>::iterator it=unsupportedOperators.begin(); it!=unsupportedOperators.end(); ++it)
231 {
232 TF_LITE_KERNEL_LOG(tfLiteContext,
233 "Operator %s [%d] is not supported by armnn_delegate.",
234 tflite::EnumNameBuiltinOperator(tflite::BuiltinOperator(*it)),
235 *it);
236 }
237
238 if (!unsupportedOperators.empty() && m_Options.TfLiteRuntimeFallbackDisabled())
239 {
240 std::stringstream exMessage;
241 exMessage << "TfLiteArmnnDelegate: There are unsupported operators in the model. ";
242 exMessage << "Not falling back to TfLite Runtime as fallback is disabled. ";
243 exMessage << "This should only be disabled under test conditions.";
244 throw armnn::Exception(exMessage.str());
245 }
246 if (nodesToDelegate->size == 0)
247 {
248 ARMNN_LOG(info) << "No operators in this model are supported by the Arm NN TfLite delegate." <<
249 " The model will be executed entirely by TfLite runtime.";
250 }
251
252 std::sort(&nodesToDelegate->data[0], &nodesToDelegate->data[nodesToDelegate->size]);
253 return nodesToDelegate;
254 }
255
GetDelegate()256 TfLiteDelegate* Delegate::GetDelegate()
257 {
258 return &m_Delegate;
259 }
260
GetVersion()261 const std::string Delegate::GetVersion()
262 {
263 return DELEGATE_VERSION;
264 }
265
AddInputLayer(DelegateData & delegateData,TfLiteContext * tfLiteContext,const TfLiteIntArray * inputs,std::vector<armnn::BindingPointInfo> & inputBindings)266 TfLiteStatus ArmnnSubgraph::AddInputLayer(DelegateData& delegateData,
267 TfLiteContext* tfLiteContext,
268 const TfLiteIntArray* inputs,
269 std::vector<armnn::BindingPointInfo>& inputBindings)
270 {
271 const size_t numInputs = static_cast<size_t>(inputs->size);
272 for (unsigned int i = 0; i < numInputs; ++i)
273 {
274 const int32_t tensorId = inputs->data[i];
275 const TfLiteTensor tensor = tfLiteContext->tensors[tensorId];
276 // Do not create bindings for constant inputs
277 if (tensor.allocation_type == kTfLiteMmapRo)
278 {
279 continue;
280 }
281
282 auto bindingId = static_cast<armnn::LayerBindingId>((tensorId));
283 armnn::IConnectableLayer* layer = delegateData.m_Network->AddInputLayer(bindingId);
284
285 auto tensorInfo = GetTensorInfoForTfLiteTensor(tensor);
286 armnn::IOutputSlot& outputSlot = layer->GetOutputSlot(0);
287 outputSlot.SetTensorInfo(tensorInfo);
288
289 // Store for creating connections
290 delegateData.m_OutputSlotForNode[static_cast<unsigned long>(tensorId)] = &outputSlot;
291
292 inputBindings.push_back(std::make_pair(bindingId, tensorInfo));
293 }
294
295 return kTfLiteOk;
296 }
297
AddOutputLayer(DelegateData & delegateData,TfLiteContext * tfLiteContext,const TfLiteIntArray * outputs,std::vector<armnn::BindingPointInfo> & outputBindings)298 TfLiteStatus ArmnnSubgraph::AddOutputLayer(DelegateData& delegateData,
299 TfLiteContext* tfLiteContext,
300 const TfLiteIntArray* outputs,
301 std::vector<armnn::BindingPointInfo>& outputBindings)
302 {
303 const size_t numOutputs = static_cast<size_t>(outputs->size);
304 for (unsigned int i = 0; i < numOutputs; ++i)
305 {
306 const int32_t tensorId = outputs->data[i];
307 const TfLiteTensor tensor = tfLiteContext->tensors[tensorId];
308
309 auto bindingId = static_cast<armnn::LayerBindingId>((tensorId));
310 armnn::IConnectableLayer* layer = delegateData.m_Network->AddOutputLayer(bindingId);
311
312 auto tensorInfo = GetTensorInfoForTfLiteTensor(tensor);
313 ARMNN_ASSERT(delegateData.m_OutputSlotForNode[static_cast<unsigned long>(tensorId)] != nullptr);
314 delegateData.m_OutputSlotForNode[static_cast<unsigned long>(tensorId)]->Connect(layer->GetInputSlot(0));
315 outputBindings.push_back(std::make_pair(bindingId, tensorInfo));
316 }
317
318 return kTfLiteOk;
319 }
320
Create(TfLiteContext * tfLiteContext,const TfLiteDelegateParams * parameters,const Delegate * delegate)321 ArmnnSubgraph* ArmnnSubgraph::Create(TfLiteContext* tfLiteContext,
322 const TfLiteDelegateParams* parameters,
323 const Delegate* delegate)
324 {
325 const auto startTime = armnn::GetTimeNow();
326 ARMNN_LOG(info) << "ArmnnSubgraph creation";
327
328 TfLiteIntArray* executionPlan;
329 if (tfLiteContext->GetExecutionPlan(tfLiteContext, &executionPlan) != kTfLiteOk)
330 {
331 return nullptr;
332 }
333
334 // Initialize DelegateData holds network and output slots information
335 DelegateData delegateData(delegate->m_Options.GetBackends());
336
337 // Build ArmNN Network
338 armnn::NetworkOptions networkOptions = delegate->m_Options.GetOptimizerOptions().GetModelOptions();
339 armnn::NetworkId networkId;
340 delegateData.m_Network = armnn::INetwork::Create(networkOptions);
341
342 delegateData.m_OutputSlotForNode = std::vector<armnn::IOutputSlot*>(tfLiteContext->tensors_size, nullptr);
343
344 std::vector<armnn::BindingPointInfo> inputBindings;
345 std::vector<armnn::BindingPointInfo> outputBindings;
346
347 // Add input layer
348 auto status = AddInputLayer(delegateData, tfLiteContext, parameters->input_tensors, inputBindings);
349 if (status != kTfLiteOk)
350 {
351 throw armnn::Exception("TfLiteArmnnDelegate: Unable to add Inputs to the network!");
352 }
353
354 // Parse TfLite delegate nodes to ArmNN
355 const auto parseStartTime = armnn::GetTimeNow();
356 for (int i = 0; i < parameters->nodes_to_replace->size; ++i)
357 {
358 const int nodeIndex = parameters->nodes_to_replace->data[i];
359
360 TfLiteNode* tfLiteNode = nullptr;
361 TfLiteRegistration* tfLiteRegistration = nullptr;
362 if (tfLiteContext->GetNodeAndRegistration(
363 tfLiteContext, nodeIndex, &tfLiteNode, &tfLiteRegistration) != kTfLiteOk)
364 {
365 throw armnn::Exception(&"TfLiteArmnnDelegate: Unable to get node registration: " [ nodeIndex]);
366 }
367
368 if (VisitNode(delegateData, tfLiteContext, tfLiteRegistration, tfLiteNode, nodeIndex) != kTfLiteOk)
369 {
370 throw armnn::Exception(&"TfLiteArmnnDelegate: Unable to parse node: " [ nodeIndex]);
371 }
372 }
373 ARMNN_LOG(info) << "Parse nodes to ArmNN time: " << std::setprecision(2)
374 << std::fixed << armnn::GetTimeDuration(parseStartTime).count() << " ms";
375
376 // Add Output layer
377 status = AddOutputLayer(delegateData, tfLiteContext, parameters->output_tensors, outputBindings);
378 if (status != kTfLiteOk)
379 {
380 throw armnn::Exception("TfLiteArmnnDelegate: Unable to add Outputs to the network!");
381 }
382
383 // Optimize ArmNN network
384 armnn::IOptimizedNetworkPtr optNet(nullptr, nullptr);
385 try
386 {
387 const auto optimizeStartTime = armnn::GetTimeNow();
388 optNet = armnn::Optimize(*(delegateData.m_Network.get()),
389 delegate->m_Options.GetBackends(),
390 delegate->m_Runtime->GetDeviceSpec(),
391 delegate->m_Options.GetOptimizerOptions());
392 ARMNN_LOG(info) << "Optimize ArmnnSubgraph time: " << std::setprecision(2)
393 << std::fixed << armnn::GetTimeDuration(optimizeStartTime).count() << " ms";
394 }
395 catch (std::exception& ex)
396 {
397 std::stringstream exMessage;
398 exMessage << "TfLiteArmnnDelegate: Exception (" << ex.what() << ") caught from optimize.";
399 throw armnn::Exception(exMessage.str());
400 }
401 if (!optNet)
402 {
403 // Optimize failed
404 throw armnn::Exception("TfLiteArmnnDelegate: Unable to optimize the network!");
405 }
406
407 // If set, we will serialize the optimized model into a dot file.
408 const std::string serializeToDotFile = delegate->m_Options.GetSerializeToDot();
409 if (!serializeToDotFile.empty())
410 {
411 ARMNN_LOG(info) << "Writing graph to dot file: " << serializeToDotFile;
412 fs::path filename = serializeToDotFile;
413 std::fstream file(filename.c_str(), std::ios_base::out);
414 optNet->SerializeToDot(file);
415 }
416
417 try
418 {
419 const auto loadStartTime = armnn::GetTimeNow();
420
421 // Load graph into runtime
422 std::string errorMessage;
423 armnn::Status loadingStatus;
424 armnn::MemorySource inputSource = armnn::MemorySource::Undefined;
425 armnn::MemorySource outputSource = armnn::MemorySource::Undefined;
426 // There's a bit of an assumption here that the delegate will only support Malloc memory source.
427 if (delegate->m_Options.GetOptimizerOptions().GetImportEnabled())
428 {
429 inputSource = armnn::MemorySource::Malloc;
430 }
431 if (delegate->m_Options.GetOptimizerOptions().GetExportEnabled())
432 {
433 outputSource = armnn::MemorySource::Malloc;
434 }
435 armnn::INetworkProperties networkProperties(false,
436 inputSource,
437 outputSource,
438 delegate->m_Options.GetInternalProfilingState(),
439 delegate->m_Options.GetInternalProfilingDetail());
440 loadingStatus = delegate->m_Runtime->LoadNetwork(networkId,
441 std::move(optNet),
442 errorMessage,
443 networkProperties);
444 if (loadingStatus != armnn::Status::Success)
445 {
446 // Network load failed.
447 throw armnn::Exception("TfLiteArmnnDelegate: Network could not be loaded: " + errorMessage);
448 }
449
450 ARMNN_LOG(info) << "Load ArmnnSubgraph time: " << std::setprecision(2)
451 << std::fixed << armnn::GetTimeDuration(loadStartTime).count() << " ms";
452 }
453 catch (std::exception& ex)
454 {
455 std::stringstream exMessage;
456 exMessage << "TfLiteArmnnDelegate: Exception (" << ex.what() << ") caught from LoadNetwork.";
457 throw armnn::Exception(exMessage.str());
458 }
459
460 // Register debug callback function
461 if (delegate->m_Options.GetDebugCallbackFunction().has_value())
462 {
463 delegate->m_Runtime->RegisterDebugCallback(networkId, delegate->m_Options.GetDebugCallbackFunction().value());
464 }
465
466 ARMNN_LOG(info) << "Overall ArmnnSubgraph creation time: " << std::setprecision(2)
467 << std::fixed << armnn::GetTimeDuration(startTime).count() << " ms\n";
468
469 // Create a new SubGraph with networkId and runtime
470 return new ArmnnSubgraph(networkId, delegate->m_Runtime, inputBindings, outputBindings);
471 }
472
Prepare(TfLiteContext * tfLiteContext)473 TfLiteStatus ArmnnSubgraph::Prepare(TfLiteContext* tfLiteContext)
474 {
475 armnn::IgnoreUnused(tfLiteContext);
476 return kTfLiteOk;
477 }
478
Invoke(TfLiteContext * tfLiteContext,TfLiteNode * tfLiteNode)479 TfLiteStatus ArmnnSubgraph::Invoke(TfLiteContext* tfLiteContext, TfLiteNode* tfLiteNode)
480 {
481 // Prepare inputs
482 armnn::InputTensors inputTensors;
483 size_t inputIndex = 0;
484 for (auto inputIdx : tflite::TfLiteIntArrayView(tfLiteNode->inputs))
485 {
486 TfLiteTensor* tensor = &tfLiteContext->tensors[inputIdx];
487 if (tensor->allocation_type != kTfLiteMmapRo)
488 {
489 const armnn::BindingPointInfo& inputBinding = m_InputBindings[inputIndex];
490 armnn::TensorInfo inputTensorInfo = inputBinding.second;
491 inputTensorInfo.SetConstant(true);
492 const armnn::ConstTensor inputTensor(inputTensorInfo, tensor->data.data);
493 inputTensors.emplace_back(inputIdx, inputTensor);
494
495 ++inputIndex;
496 }
497 }
498
499 // Prepare outputs
500 armnn::OutputTensors outputTensors;
501 size_t outputIndex = 0;
502 for (auto outputIdx : tflite::TfLiteIntArrayView(tfLiteNode->outputs))
503 {
504 const armnn::BindingPointInfo& outputBinding = m_OutputBindings[outputIndex];
505 TfLiteTensor* tensor = &tfLiteContext->tensors[outputIdx];
506 const armnn::Tensor outputTensor(outputBinding.second, tensor->data.data);
507 outputTensors.emplace_back(outputIdx, outputTensor);
508
509 ++outputIndex;
510 }
511
512 // Run graph
513 auto status = m_Runtime->EnqueueWorkload(m_NetworkId, inputTensors, outputTensors);
514 // The delegate holds its own Arm NN runtime so this is our last chance to print internal profiling data.
515 std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkId);
516 if (profiler && profiler->IsProfilingEnabled())
517 {
518 profiler->Print(std::cout);
519 }
520 return (status == armnn::Status::Success) ? kTfLiteOk : kTfLiteError;
521 }
522
VisitNode(DelegateData & delegateData,TfLiteContext * tfLiteContext,TfLiteRegistration * tfLiteRegistration,TfLiteNode * tfLiteNode,int nodeIndex)523 TfLiteStatus ArmnnSubgraph::VisitNode(DelegateData& delegateData,
524 TfLiteContext* tfLiteContext,
525 TfLiteRegistration* tfLiteRegistration,
526 TfLiteNode* tfLiteNode,
527 int nodeIndex)
528 {
529 switch (tfLiteRegistration->builtin_code)
530 {
531 case kTfLiteBuiltinCustom:
532 {
533 #if defined(ARMNN_POST_TFLITE_2_5)
534 // Custom operators are defined by the name rather than the builtin code.
535 // Parse the custom_name param in the registration to point to the correct visitor function.
536 std::string customOperatorName = tfLiteRegistration->custom_name;
537 if ( customOperatorName == "AveragePool3D" )
538 {
539 return VisitPooling3dOperator(delegateData,
540 tfLiteContext,
541 tfLiteNode,
542 nodeIndex,
543 customOperatorName);
544 }
545 else if (customOperatorName == "MaxPool3D")
546 {
547 return VisitPooling3dOperator(delegateData,
548 tfLiteContext,
549 tfLiteNode,
550 nodeIndex,
551 customOperatorName);
552 }
553 #endif
554 // Invalid or unsupported custom operator
555 return kTfLiteError;
556 }
557 case kTfLiteBuiltinAbs:
558 return VisitElementwiseUnaryOperator(delegateData,
559 tfLiteContext,
560 tfLiteNode,
561 nodeIndex,
562 armnn::UnaryOperation::Abs);
563 case kTfLiteBuiltinAdd:
564 return VisitElementwiseBinaryOperator(delegateData,
565 tfLiteContext,
566 tfLiteNode,
567 nodeIndex,
568 kTfLiteBuiltinAdd);
569 case kTfLiteBuiltinArgMax:
570 return VisitArgMinMaxOperator(delegateData,
571 tfLiteContext,
572 tfLiteNode,
573 nodeIndex,
574 kTfLiteBuiltinArgMax);
575 case kTfLiteBuiltinArgMin:
576 return VisitArgMinMaxOperator(delegateData,
577 tfLiteContext,
578 tfLiteNode,
579 nodeIndex,
580 kTfLiteBuiltinArgMin);
581 case kTfLiteBuiltinAveragePool2d:
582 return VisitPooling2dOperator(delegateData,
583 tfLiteContext,
584 tfLiteNode,
585 nodeIndex,
586 kTfLiteBuiltinAveragePool2d);
587 case kTfLiteBuiltinBatchMatmul:
588 return VisitBatchMatMulOperator(delegateData,
589 tfLiteContext,
590 tfLiteNode,
591 nodeIndex,
592 kTfLiteBuiltinBatchMatmul);
593 case kTfLiteBuiltinBatchToSpaceNd:
594 return VisitBatchToSpaceNdOperator(delegateData,
595 tfLiteContext,
596 tfLiteNode,
597 nodeIndex,
598 kTfLiteBuiltinBatchToSpaceNd);
599 case kTfLiteBuiltinCast:
600 return VisitCastOperator(delegateData,
601 tfLiteContext,
602 tfLiteNode,
603 nodeIndex,
604 kTfLiteBuiltinCast);
605 case kTfLiteBuiltinCeil:
606 return VisitElementwiseUnaryOperator(delegateData,
607 tfLiteContext,
608 tfLiteNode,
609 nodeIndex,
610 armnn::UnaryOperation::Ceil);
611 case kTfLiteBuiltinConcatenation:
612 return VisitControlOperator(delegateData,
613 tfLiteContext,
614 tfLiteNode,
615 nodeIndex,
616 kTfLiteBuiltinConcatenation);
617 case kTfLiteBuiltinConv2d:
618 return VisitConvolutionOperator(delegateData,
619 tfLiteContext,
620 tfLiteNode,
621 nodeIndex,
622 kTfLiteBuiltinConv2d);
623 // Conv3d is only correctly supported for external delegates from TF Lite v2.6, as there was a breaking bug in v2.5.
624 #if defined(ARMNN_POST_TFLITE_2_5)
625 case kTfLiteBuiltinConv3d:
626 return VisitConvolutionOperator(delegateData,
627 tfLiteContext,
628 tfLiteNode,
629 nodeIndex,
630 kTfLiteBuiltinConv3d);
631 #endif
632 case kTfLiteBuiltinDepthToSpace:
633 return VisitDepthToSpaceOperator(delegateData,
634 tfLiteContext,
635 tfLiteNode,
636 nodeIndex,
637 kTfLiteBuiltinDepthToSpace);
638 case kTfLiteBuiltinDepthwiseConv2d:
639 return VisitConvolutionOperator(delegateData,
640 tfLiteContext,
641 tfLiteNode,
642 nodeIndex,
643 kTfLiteBuiltinDepthwiseConv2d);
644 case kTfLiteBuiltinDequantize:
645 return VisitDequantizeOperator(delegateData,
646 tfLiteContext,
647 tfLiteNode,
648 nodeIndex,
649 kTfLiteBuiltinDequantize);
650 case kTfLiteBuiltinDiv:
651 return VisitElementwiseBinaryOperator(delegateData,
652 tfLiteContext,
653 tfLiteNode,
654 nodeIndex,
655 kTfLiteBuiltinDiv);
656 case kTfLiteBuiltinElu:
657 return VisitActivationOperator(delegateData,
658 tfLiteContext,
659 tfLiteNode,
660 nodeIndex,
661 kTfLiteBuiltinElu);
662 case kTfLiteBuiltinEqual:
663 return VisitComparisonOperator(delegateData,
664 tfLiteContext,
665 tfLiteNode,
666 nodeIndex,
667 kTfLiteBuiltinEqual);
668 case kTfLiteBuiltinExp:
669 return VisitElementwiseUnaryOperator(delegateData,
670 tfLiteContext,
671 tfLiteNode,
672 nodeIndex,
673 armnn::UnaryOperation::Exp);
674 case kTfLiteBuiltinExpandDims:
675 return VisitExpandDimsOperator(delegateData,
676 tfLiteContext,
677 tfLiteNode,
678 nodeIndex,
679 kTfLiteBuiltinExpandDims);
680 case kTfLiteBuiltinFill:
681 return VisitFillOperator(delegateData,
682 tfLiteContext,
683 tfLiteNode,
684 nodeIndex,
685 kTfLiteBuiltinFill);
686 case kTfLiteBuiltinFloor:
687 return VisitFloorOperator(delegateData,
688 tfLiteContext,
689 tfLiteNode,
690 nodeIndex,
691 kTfLiteBuiltinFloor);
692 case kTfLiteBuiltinFloorDiv:
693 return VisitElementwiseBinaryOperator(delegateData,
694 tfLiteContext,
695 tfLiteNode,
696 nodeIndex,
697 kTfLiteBuiltinFloorDiv);
698 case kTfLiteBuiltinFullyConnected:
699 return VisitFullyConnectedOperator(delegateData,
700 tfLiteContext,
701 tfLiteNode,
702 nodeIndex,
703 kTfLiteBuiltinFullyConnected);
704 case kTfLiteBuiltinGather:
705 return VisitGatherOperator(delegateData,
706 tfLiteContext,
707 tfLiteNode,
708 nodeIndex,
709 kTfLiteBuiltinGather);
710 case kTfLiteBuiltinGatherNd:
711 return VisitGatherNdOperator(delegateData,
712 tfLiteContext,
713 tfLiteNode,
714 nodeIndex,
715 kTfLiteBuiltinGatherNd);
716 case kTfLiteBuiltinGreater:
717 return VisitComparisonOperator(delegateData,
718 tfLiteContext,
719 tfLiteNode,
720 nodeIndex,
721 kTfLiteBuiltinGreater);
722 case kTfLiteBuiltinGreaterEqual:
723 return VisitComparisonOperator(delegateData,
724 tfLiteContext,
725 tfLiteNode,
726 nodeIndex,
727 kTfLiteBuiltinGreaterEqual);
728 case kTfLiteBuiltinHardSwish:
729 return VisitActivationOperator(delegateData,
730 tfLiteContext,
731 tfLiteNode,
732 nodeIndex,
733 kTfLiteBuiltinHardSwish);
734 case kTfLiteBuiltinL2Normalization:
735 return VisitL2NormalizationOperator(delegateData,
736 tfLiteContext,
737 tfLiteNode,
738 nodeIndex,
739 kTfLiteBuiltinL2Normalization);
740 case kTfLiteBuiltinL2Pool2d:
741 return VisitPooling2dOperator(delegateData,
742 tfLiteContext,
743 tfLiteNode,
744 nodeIndex,
745 kTfLiteBuiltinL2Pool2d);
746 case kTfLiteBuiltinLess:
747 return VisitComparisonOperator(delegateData,
748 tfLiteContext,
749 tfLiteNode,
750 nodeIndex,
751 kTfLiteBuiltinLess);
752 case kTfLiteBuiltinLessEqual:
753 return VisitComparisonOperator(delegateData,
754 tfLiteContext,
755 tfLiteNode,
756 nodeIndex,
757 kTfLiteBuiltinLessEqual);
758 case kTfLiteBuiltinLocalResponseNormalization:
759 return VisitLocalResponseNormalizationOperator(delegateData,
760 tfLiteContext,
761 tfLiteNode,
762 nodeIndex,
763 kTfLiteBuiltinLocalResponseNormalization);
764 case kTfLiteBuiltinLog:
765 return VisitElementwiseUnaryOperator(delegateData,
766 tfLiteContext,
767 tfLiteNode,
768 nodeIndex,
769 armnn::UnaryOperation::Log);
770 case kTfLiteBuiltinLogicalAnd:
771 return VisitLogicalBinaryOperator(delegateData,
772 tfLiteContext,
773 tfLiteNode,
774 nodeIndex,
775 kTfLiteBuiltinLogicalAnd,
776 armnn::LogicalBinaryOperation::LogicalAnd);
777 case kTfLiteBuiltinLogicalNot:
778 return VisitElementwiseUnaryOperator(delegateData,
779 tfLiteContext,
780 tfLiteNode,
781 nodeIndex,
782 armnn::UnaryOperation::LogicalNot);
783 case kTfLiteBuiltinLogicalOr:
784 return VisitLogicalBinaryOperator(delegateData,
785 tfLiteContext,
786 tfLiteNode,
787 nodeIndex,
788 kTfLiteBuiltinLogicalOr,
789 armnn::LogicalBinaryOperation::LogicalOr);
790 case kTfLiteBuiltinLogistic:
791 return VisitActivationOperator(delegateData,
792 tfLiteContext,
793 tfLiteNode,
794 nodeIndex,
795 kTfLiteBuiltinLogistic);
796 case kTfLiteBuiltinLogSoftmax:
797 return VisitSoftmaxOperator(delegateData,
798 tfLiteContext,
799 tfLiteNode,
800 nodeIndex,
801 kTfLiteBuiltinLogSoftmax);
802 case kTfLiteBuiltinLstm:
803 return VisitLstmOperator(delegateData,
804 tfLiteContext,
805 tfLiteNode,
806 nodeIndex,
807 kTfLiteBuiltinLstm);
808 case kTfLiteBuiltinMaxPool2d:
809 return VisitPooling2dOperator(delegateData,
810 tfLiteContext,
811 tfLiteNode,
812 nodeIndex,
813 kTfLiteBuiltinMaxPool2d);
814 case kTfLiteBuiltinMaximum:
815 return VisitElementwiseBinaryOperator(delegateData,
816 tfLiteContext,
817 tfLiteNode,
818 nodeIndex,
819 kTfLiteBuiltinMaximum);
820 case kTfLiteBuiltinMean:
821 return VisitControlOperator(delegateData,
822 tfLiteContext,
823 tfLiteNode,
824 nodeIndex,
825 kTfLiteBuiltinMean);
826 case kTfLiteBuiltinMinimum:
827 return VisitElementwiseBinaryOperator(delegateData,
828 tfLiteContext,
829 tfLiteNode,
830 nodeIndex,
831 kTfLiteBuiltinMinimum);
832 case kTfLiteBuiltinMirrorPad:
833 return VisitPadOperator(delegateData,
834 tfLiteContext,
835 tfLiteNode,
836 nodeIndex,
837 kTfLiteBuiltinMirrorPad);
838 case kTfLiteBuiltinMul:
839 return VisitElementwiseBinaryOperator(delegateData,
840 tfLiteContext,
841 tfLiteNode,
842 nodeIndex,
843 kTfLiteBuiltinMul);
844 case kTfLiteBuiltinNeg:
845 return VisitElementwiseUnaryOperator(delegateData,
846 tfLiteContext,
847 tfLiteNode,
848 nodeIndex,
849 armnn::UnaryOperation::Neg);
850 case kTfLiteBuiltinNotEqual:
851 return VisitComparisonOperator(delegateData,
852 tfLiteContext,
853 tfLiteNode,
854 nodeIndex,
855 kTfLiteBuiltinNotEqual);
856 case kTfLiteBuiltinPack:
857 return VisitPackOperator(delegateData,
858 tfLiteContext,
859 tfLiteNode,
860 nodeIndex,
861 kTfLiteBuiltinPack);
862 case kTfLiteBuiltinPad:
863 return VisitPadOperator(delegateData,
864 tfLiteContext,
865 tfLiteNode,
866 nodeIndex,
867 kTfLiteBuiltinPad);
868 case kTfLiteBuiltinPadv2:
869 return VisitPadOperator(delegateData,
870 tfLiteContext,
871 tfLiteNode,
872 nodeIndex,
873 kTfLiteBuiltinPadv2);
874 case kTfLiteBuiltinPrelu:
875 return VisitPreluOperator(delegateData,
876 tfLiteContext,
877 tfLiteNode,
878 nodeIndex,
879 kTfLiteBuiltinPrelu);
880 case kTfLiteBuiltinQuantize:
881 return VisitQuantizeOperator(delegateData,
882 tfLiteContext,
883 tfLiteNode,
884 nodeIndex,
885 kTfLiteBuiltinQuantize);
886 case kTfLiteBuiltinRank:
887 return VisitControlOperator(delegateData,
888 tfLiteContext,
889 tfLiteNode,
890 nodeIndex,
891 kTfLiteBuiltinRank);
892 case kTfLiteBuiltinReduceMax:
893 return VisitReduceOperator(delegateData,
894 tfLiteContext,
895 tfLiteNode,
896 nodeIndex,
897 kTfLiteBuiltinReduceMax);
898 case kTfLiteBuiltinReduceMin:
899 return VisitReduceOperator(delegateData,
900 tfLiteContext,
901 tfLiteNode,
902 nodeIndex,
903 kTfLiteBuiltinReduceMin);
904 case kTfLiteBuiltinReduceProd:
905 return VisitReduceOperator(delegateData,
906 tfLiteContext,
907 tfLiteNode,
908 nodeIndex,
909 kTfLiteBuiltinReduceProd);
910 case kTfLiteBuiltinRelu:
911 return VisitActivationOperator(delegateData,
912 tfLiteContext,
913 tfLiteNode,
914 nodeIndex,
915 kTfLiteBuiltinRelu);
916 case kTfLiteBuiltinReluN1To1:
917 return VisitActivationOperator(delegateData,
918 tfLiteContext,
919 tfLiteNode,
920 nodeIndex,
921 kTfLiteBuiltinReluN1To1);
922 case kTfLiteBuiltinRelu6:
923 return VisitActivationOperator(delegateData,
924 tfLiteContext,
925 tfLiteNode,
926 nodeIndex,
927 kTfLiteBuiltinRelu6);
928 case kTfLiteBuiltinReshape:
929 return VisitReshapeOperator(delegateData,
930 tfLiteContext,
931 tfLiteNode,
932 nodeIndex,
933 kTfLiteBuiltinReshape);
934 case kTfLiteBuiltinResizeBilinear:
935 return VisitResizeOperator(delegateData,
936 tfLiteContext,
937 tfLiteNode,
938 nodeIndex,
939 kTfLiteBuiltinResizeBilinear);
940 case kTfLiteBuiltinResizeNearestNeighbor:
941 return VisitResizeOperator(delegateData,
942 tfLiteContext,
943 tfLiteNode,
944 nodeIndex,
945 kTfLiteBuiltinResizeNearestNeighbor);
946 case kTfLiteBuiltinRsqrt:
947 return VisitElementwiseUnaryOperator(delegateData,
948 tfLiteContext,
949 tfLiteNode,
950 nodeIndex,
951 armnn::UnaryOperation::Rsqrt);
952 case kTfLiteBuiltinShape:
953 return VisitShapeOperator(delegateData,
954 tfLiteContext,
955 tfLiteNode,
956 nodeIndex,
957 kTfLiteBuiltinShape);
958 case kTfLiteBuiltinSin:
959 return VisitElementwiseUnaryOperator(delegateData,
960 tfLiteContext,
961 tfLiteNode,
962 nodeIndex,
963 armnn::UnaryOperation::Sin);
964 case kTfLiteBuiltinSplit:
965 return VisitSplitOperator(delegateData,
966 tfLiteContext,
967 tfLiteNode,
968 nodeIndex,
969 kTfLiteBuiltinSplit);
970 case kTfLiteBuiltinSplitV:
971 return VisitSplitVOperator(delegateData,
972 tfLiteContext,
973 tfLiteNode,
974 nodeIndex,
975 kTfLiteBuiltinSplitV);
976 case kTfLiteBuiltinSqrt:
977 return VisitElementwiseUnaryOperator(delegateData,
978 tfLiteContext,
979 tfLiteNode,
980 nodeIndex,
981 armnn::UnaryOperation::Sqrt);
982 case kTfLiteBuiltinSqueeze:
983 return VisitSqueezeOperator(delegateData,
984 tfLiteContext,
985 tfLiteNode,
986 nodeIndex,
987 kTfLiteBuiltinSqueeze);
988 case kTfLiteBuiltinSlice:
989 return VisitSliceOperator(delegateData,
990 tfLiteContext,
991 tfLiteNode,
992 nodeIndex,
993 kTfLiteBuiltinSlice);
994 case kTfLiteBuiltinStridedSlice:
995 return VisitStridedSliceOperator(delegateData,
996 tfLiteContext,
997 tfLiteNode,
998 nodeIndex,
999 kTfLiteBuiltinStridedSlice);
1000 case kTfLiteBuiltinSum:
1001 return VisitReduceOperator(delegateData,
1002 tfLiteContext,
1003 tfLiteNode,
1004 nodeIndex,
1005 kTfLiteBuiltinSum);
1006 case kTfLiteBuiltinTranspose:
1007 return VisitTransposeOperator(delegateData,
1008 tfLiteContext,
1009 tfLiteNode,
1010 nodeIndex,
1011 kTfLiteBuiltinTranspose);
1012 case kTfLiteBuiltinTransposeConv:
1013 return VisitConvolutionOperator(delegateData,
1014 tfLiteContext,
1015 tfLiteNode,
1016 nodeIndex,
1017 kTfLiteBuiltinTransposeConv);
1018 case kTfLiteBuiltinSoftmax:
1019 return VisitSoftmaxOperator(delegateData,
1020 tfLiteContext,
1021 tfLiteNode,
1022 nodeIndex,
1023 kTfLiteBuiltinSoftmax);
1024 case kTfLiteBuiltinSpaceToBatchNd:
1025 return VisitSpaceToBatchNdOperator(delegateData,
1026 tfLiteContext,
1027 tfLiteNode,
1028 nodeIndex,
1029 kTfLiteBuiltinSpaceToBatchNd);
1030 case kTfLiteBuiltinSpaceToDepth:
1031 return VisitSpaceToDepthOperator(delegateData,
1032 tfLiteContext,
1033 tfLiteNode,
1034 nodeIndex,
1035 kTfLiteBuiltinSpaceToDepth);
1036 case kTfLiteBuiltinSub:
1037 return VisitElementwiseBinaryOperator(delegateData,
1038 tfLiteContext,
1039 tfLiteNode,
1040 nodeIndex,
1041 kTfLiteBuiltinSub);
1042 case kTfLiteBuiltinTanh:
1043 return VisitActivationOperator(delegateData,
1044 tfLiteContext,
1045 tfLiteNode,
1046 nodeIndex,
1047 kTfLiteBuiltinTanh);
1048 case kTfLiteBuiltinUnidirectionalSequenceLstm:
1049 return VisitUnidirectionalSequenceLstmOperator(delegateData,
1050 tfLiteContext,
1051 tfLiteNode,
1052 nodeIndex,
1053 kTfLiteBuiltinUnidirectionalSequenceLstm);
1054 case kTfLiteBuiltinUnpack:
1055 return VisitUnpackOperator(delegateData,
1056 tfLiteContext,
1057 tfLiteNode,
1058 nodeIndex,
1059 kTfLiteBuiltinUnpack);
1060 default:
1061 return kTfLiteError;
1062 }
1063 }
1064
1065 } // armnnDelegate namespace