1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 #include <string>
16 #include <utility>
17
18 #include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
19 #include "tensorflow/lite/nnapi/nnapi_implementation.h"
20 #include "tensorflow/lite/nnapi/nnapi_util.h"
21 #include "tensorflow/lite/nnapi/sl/include/SupportLibrary.h"
22 #include "tensorflow/lite/tools/delegates/delegate_provider.h"
23
24 namespace tflite {
25 namespace tools {
26
27 namespace {
28
29 using nnapi::NnApiSupportLibrary;
30
31 // StatefulNnApiDelegate that holds onto an NnApiSupportLibrary instance
32 // passed to the constructor for later destruction.
33 // Note that the support library must outlive the delegate.
34 class NnApiSupportLibraryDelegate : public StatefulNnApiDelegate {
35 public:
NnApiSupportLibraryDelegate(const NnApiSupportLibrary * nnapi_sl,Options options)36 NnApiSupportLibraryDelegate(const NnApiSupportLibrary* nnapi_sl,
37 Options options)
38 : StatefulNnApiDelegate(nnapi_sl->getFL5(), options),
39 nnapi_sl_(nnapi_sl) {}
get_nnapi_sl() const40 const NnApiSupportLibrary* get_nnapi_sl() const { return nnapi_sl_; }
41
42 private:
43 const NnApiSupportLibrary* const nnapi_sl_;
44 };
45
46 } // namespace
47
48 class NnapiDelegateProvider : public DelegateProvider {
49 public:
NnapiDelegateProvider()50 NnapiDelegateProvider() {
51 default_params_.AddParam("use_nnapi", ToolParam::Create<bool>(false));
52 default_params_.AddParam("nnapi_execution_preference",
53 ToolParam::Create<std::string>(""));
54 default_params_.AddParam("nnapi_execution_priority",
55 ToolParam::Create<std::string>(""));
56 default_params_.AddParam("nnapi_accelerator_name",
57 ToolParam::Create<std::string>(""));
58 default_params_.AddParam("disable_nnapi_cpu",
59 ToolParam::Create<bool>(true));
60 default_params_.AddParam("nnapi_allow_fp16",
61 ToolParam::Create<bool>(false));
62 default_params_.AddParam("nnapi_allow_dynamic_dimensions",
63 ToolParam::Create<bool>(false));
64 default_params_.AddParam("nnapi_use_burst_mode",
65 ToolParam::Create<bool>(false));
66 default_params_.AddParam("nnapi_support_library_path",
67 ToolParam::Create<std::string>(""));
68 }
69
70 std::vector<Flag> CreateFlags(ToolParams* params) const final;
71
72 void LogParams(const ToolParams& params, bool verbose) const final;
73
74 TfLiteDelegatePtr CreateTfLiteDelegate(const ToolParams& params) const final;
75 std::pair<TfLiteDelegatePtr, int> CreateRankedTfLiteDelegate(
76 const ToolParams& params) const final;
77
GetName() const78 std::string GetName() const final { return "NNAPI"; }
79 };
80 REGISTER_DELEGATE_PROVIDER(NnapiDelegateProvider);
81
CreateFlags(ToolParams * params) const82 std::vector<Flag> NnapiDelegateProvider::CreateFlags(ToolParams* params) const {
83 std::vector<Flag> flags = {
84 CreateFlag<bool>("use_nnapi", params, "use nnapi delegate api"),
85 CreateFlag<std::string>("nnapi_execution_preference", params,
86 "execution preference for nnapi delegate. Should "
87 "be one of the following: fast_single_answer, "
88 "sustained_speed, low_power, undefined"),
89 CreateFlag<std::string>("nnapi_execution_priority", params,
90 "The model execution priority in nnapi, and it "
91 "should be one of the following: default, low, "
92 "medium and high. This requires Android 11+."),
93 CreateFlag<std::string>(
94 "nnapi_accelerator_name", params,
95 "the name of the nnapi accelerator to use (requires Android Q+)"),
96 CreateFlag<bool>("disable_nnapi_cpu", params,
97 "Disable the NNAPI CPU device"),
98 CreateFlag<bool>("nnapi_allow_fp16", params,
99 "Allow fp32 computation to be run in fp16"),
100 CreateFlag<bool>(
101 "nnapi_allow_dynamic_dimensions", params,
102 "Whether to allow dynamic dimension sizes without re-compilation. "
103 "This requires Android 9+."),
104 CreateFlag<bool>(
105 "nnapi_use_burst_mode", params,
106 "use NNAPI Burst mode if supported. Burst mode allows accelerators "
107 "to efficiently manage resources, which would significantly reduce "
108 "overhead especially if the same delegate instance is to be used for "
109 "multiple inferences."),
110 CreateFlag<std::string>(
111 "nnapi_support_library_path", params,
112 "Path from which NNAPI support library will be loaded to construct "
113 "the delegate. In order to use NNAPI delegate with support library, "
114 "--nnapi_accelerator_name must be specified and must be equal to one "
115 "of the devices provided by the support library."),
116 };
117
118 return flags;
119 }
120
LogParams(const ToolParams & params,bool verbose) const121 void NnapiDelegateProvider::LogParams(const ToolParams& params,
122 bool verbose) const {
123 LOG_TOOL_PARAM(params, bool, "use_nnapi", "Use NNAPI", verbose);
124 if (!params.Get<bool>("use_nnapi")) return;
125
126 LOG_TOOL_PARAM(params, std::string, "nnapi_execution_preference",
127 "NNAPI execution preference", verbose);
128 LOG_TOOL_PARAM(params, std::string, "nnapi_execution_priority",
129 "Model execution priority in nnapi", verbose);
130 LOG_TOOL_PARAM(params, std::string, "nnapi_accelerator_name",
131 "NNAPI accelerator name", verbose);
132
133 std::string string_device_names_list =
134 nnapi::GetStringDeviceNamesList(NnApiImplementation());
135 // Print available devices when possible as it's informative.
136 if (!string_device_names_list.empty()) {
137 TFLITE_LOG(INFO) << "NNAPI accelerators available: ["
138 << string_device_names_list << "]";
139 }
140
141 LOG_TOOL_PARAM(params, bool, "disable_nnapi_cpu", "Disable NNAPI cpu",
142 verbose);
143 LOG_TOOL_PARAM(params, bool, "nnapi_allow_fp16", "Allow fp16 in NNAPI",
144 verbose);
145 LOG_TOOL_PARAM(params, bool, "nnapi_allow_dynamic_dimensions",
146 "Allow dynamic dimensions in NNAPI", verbose);
147 LOG_TOOL_PARAM(params, bool, "nnapi_use_burst_mode",
148 "Use burst mode in NNAPI", verbose);
149 }
150
CreateTfLiteDelegate(const ToolParams & params) const151 TfLiteDelegatePtr NnapiDelegateProvider::CreateTfLiteDelegate(
152 const ToolParams& params) const {
153 TfLiteDelegatePtr null_delegate = CreateNullDelegate();
154 if (params.Get<bool>("use_nnapi")) {
155 StatefulNnApiDelegate::Options options;
156 std::string accelerator_name =
157 params.Get<std::string>("nnapi_accelerator_name");
158 if (!accelerator_name.empty()) {
159 options.accelerator_name = accelerator_name.c_str();
160 } else {
161 options.disallow_nnapi_cpu = params.Get<bool>("disable_nnapi_cpu");
162 }
163
164 if (params.Get<bool>("nnapi_allow_fp16")) {
165 options.allow_fp16 = true;
166 }
167
168 if (params.Get<bool>("nnapi_allow_dynamic_dimensions")) {
169 options.allow_dynamic_dimensions = true;
170 }
171
172 if (params.Get<bool>("nnapi_use_burst_mode")) {
173 options.use_burst_computation = true;
174 }
175
176 std::string string_execution_preference =
177 params.Get<std::string>("nnapi_execution_preference");
178 // Only set execution preference if user explicitly passes one. Otherwise,
179 // leave it as whatever NNAPI has as the default.
180 if (!string_execution_preference.empty()) {
181 tflite::StatefulNnApiDelegate::Options::ExecutionPreference
182 execution_preference =
183 tflite::StatefulNnApiDelegate::Options::kUndefined;
184 if (string_execution_preference == "low_power") {
185 execution_preference =
186 tflite::StatefulNnApiDelegate::Options::kLowPower;
187 } else if (string_execution_preference == "sustained_speed") {
188 execution_preference =
189 tflite::StatefulNnApiDelegate::Options::kSustainedSpeed;
190 } else if (string_execution_preference == "fast_single_answer") {
191 execution_preference =
192 tflite::StatefulNnApiDelegate::Options::kFastSingleAnswer;
193 } else if (string_execution_preference == "undefined") {
194 execution_preference =
195 tflite::StatefulNnApiDelegate::Options::kUndefined;
196 } else {
197 TFLITE_LOG(WARN) << "The provided value ("
198 << string_execution_preference
199 << ") is not a valid nnapi execution preference.";
200 }
201 options.execution_preference = execution_preference;
202 }
203
204 std::string string_execution_priority =
205 params.Get<std::string>("nnapi_execution_priority");
206 // Only set execution priority if user explicitly passes one. Otherwise,
207 // leave it as whatever NNAPI has as the default.
208 if (!string_execution_priority.empty()) {
209 int execution_priority = 0;
210 if (string_execution_priority == "default") {
211 execution_priority = ANEURALNETWORKS_PRIORITY_DEFAULT;
212 } else if (string_execution_priority == "low") {
213 execution_priority = ANEURALNETWORKS_PRIORITY_LOW;
214 } else if (string_execution_priority == "medium") {
215 execution_priority = ANEURALNETWORKS_PRIORITY_MEDIUM;
216 } else if (string_execution_priority == "high") {
217 execution_priority = ANEURALNETWORKS_PRIORITY_HIGH;
218 } else {
219 TFLITE_LOG(WARN) << "The provided value (" << string_execution_priority
220 << ") is not a valid nnapi execution priority.";
221 }
222 options.execution_priority = execution_priority;
223 }
224
225 int max_delegated_partitions = params.Get<int>("max_delegated_partitions");
226 if (max_delegated_partitions >= 0) {
227 options.max_number_delegated_partitions = max_delegated_partitions;
228 }
229
230 // Serialization.
231 std::string serialize_dir =
232 params.Get<std::string>("delegate_serialize_dir");
233 std::string serialize_token =
234 params.Get<std::string>("delegate_serialize_token");
235 if (!serialize_dir.empty() && !serialize_token.empty()) {
236 options.cache_dir = serialize_dir.c_str();
237 options.model_token = serialize_token.c_str();
238 }
239
240 if (params.Get<std::string>("nnapi_support_library_path").empty()) {
241 const auto* nnapi_impl = NnApiImplementation();
242 if (!nnapi_impl->nnapi_exists) {
243 TFLITE_LOG(WARN)
244 << "NNAPI acceleration is unsupported on this platform.";
245 return null_delegate;
246 }
247 return TfLiteDelegatePtr(
248 new StatefulNnApiDelegate(nnapi_impl, options),
249 [](TfLiteDelegate* delegate) {
250 delete reinterpret_cast<StatefulNnApiDelegate*>(delegate);
251 });
252 } else {
253 std::string sl_path =
254 params.Get<std::string>("nnapi_support_library_path");
255 auto nnapi_impl = nnapi::loadNnApiSupportLibrary(sl_path);
256 if (!nnapi_impl) {
257 TFLITE_LOG(WARN) << "Couldn't load NNAPI support library from path: "
258 << sl_path;
259 return null_delegate;
260 }
261 return TfLiteDelegatePtr(
262 new NnApiSupportLibraryDelegate(nnapi_impl.release(), options),
263 [](TfLiteDelegate* delegate) {
264 NnApiSupportLibraryDelegate* sl_delegate =
265 reinterpret_cast<NnApiSupportLibraryDelegate*>(delegate);
266 const NnApiSupportLibrary* sl = sl_delegate->get_nnapi_sl();
267 delete sl_delegate;
268 delete sl;
269 });
270 }
271 } else if (!params.Get<std::string>("nnapi_accelerator_name").empty()) {
272 TFLITE_LOG(WARN)
273 << "`--use_nnapi=true` must be set for the provided NNAPI accelerator ("
274 << params.Get<std::string>("nnapi_accelerator_name") << ") to be used.";
275 } else if (!params.Get<std::string>("nnapi_execution_preference").empty()) {
276 TFLITE_LOG(WARN) << "`--use_nnapi=true` must be set for the provided NNAPI "
277 "execution preference ("
278 << params.Get<std::string>("nnapi_execution_preference")
279 << ") to be used.";
280 }
281 return null_delegate;
282 }
283
284 std::pair<TfLiteDelegatePtr, int>
CreateRankedTfLiteDelegate(const ToolParams & params) const285 NnapiDelegateProvider::CreateRankedTfLiteDelegate(
286 const ToolParams& params) const {
287 auto ptr = CreateTfLiteDelegate(params);
288 return std::make_pair(std::move(ptr), params.GetPosition<bool>("use_nnapi"));
289 }
290
291 } // namespace tools
292 } // namespace tflite
293