1 //
2 // Copyright © 2020 STMicroelectronics and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5
6 #include <algorithm>
7 #include <getopt.h>
8 #include <numeric>
9 #include <signal.h>
10 #include <string>
11 #include <sys/time.h>
12 #include <vector>
13
14 #include <armnn/BackendId.hpp>
15 #include <armnn/BackendRegistry.hpp>
16 #include <armnn/IRuntime.hpp>
17 #include <armnn/utility/NumericCast.hpp>
18 #include <armnnTfLiteParser/ITfLiteParser.hpp>
19
20 // Application parameters
21 std::vector<armnn::BackendId> default_preferred_backends_order = {armnn::Compute::CpuAcc, armnn::Compute::CpuRef};
22 std::vector<armnn::BackendId> preferred_backends_order;
23 std::string model_file_str;
24 std::string preferred_backend_str;
25 int nb_loops = 1;
26
get_us(struct timeval t)27 double get_us(struct timeval t)
28 {
29 return (armnn::numeric_cast<double>(t.tv_sec) *
30 armnn::numeric_cast<double>(1000000) +
31 armnn::numeric_cast<double>(t.tv_usec));
32 }
33
get_ms(struct timeval t)34 double get_ms(struct timeval t)
35 {
36 return (armnn::numeric_cast<double>(t.tv_sec) *
37 armnn::numeric_cast<double>(1000) +
38 armnn::numeric_cast<double>(t.tv_usec) / 1000);
39 }
40
print_help(char ** argv)41 static void print_help(char** argv)
42 {
43 std::cout <<
44 "Usage: " << argv[0] << " -m <model .tflite>\n"
45 "\n"
46 "-m --model_file <.tflite file path>: .tflite model to be executed\n"
47 "-b --backend <device>: preferred backend device to run layers on by default. Possible choices: "
48 << armnn::BackendRegistryInstance().GetBackendIdsAsString() << "\n"
49 " (by default CpuAcc, CpuRef)\n"
50 "-l --loops <int>: provide the number of times the inference will be executed\n"
51 " (by default nb_loops=1)\n"
52 "--help: show this help\n";
53 exit(1);
54 }
55
process_args(int argc,char ** argv)56 void process_args(int argc, char** argv)
57 {
58 const char* const short_opts = "m:b:l:h";
59 const option long_opts[] = {
60 {"model_file", required_argument, nullptr, 'm'},
61 {"backend", required_argument, nullptr, 'b'},
62 {"loops", required_argument, nullptr, 'l'},
63 {"help", no_argument, nullptr, 'h'},
64 {nullptr, no_argument, nullptr, 0}
65 };
66
67 while (true)
68 {
69 const auto opt = getopt_long(argc, argv, short_opts, long_opts, nullptr);
70
71 if (-1 == opt)
72 {
73 break;
74 }
75
76 switch (opt)
77 {
78 case 'm':
79 model_file_str = std::string(optarg);
80 std::cout << "model file set to: " << model_file_str << std::endl;
81 break;
82 case 'b':
83 preferred_backend_str = std::string(optarg);
84 // Overwrite the backend
85 preferred_backends_order.push_back(preferred_backend_str);
86
87 std::cout << "backend device set to:" << preferred_backend_str << std::endl;;
88 break;
89 case 'l':
90 nb_loops = std::stoi(optarg);
91 std::cout << "benchmark will execute " << nb_loops << " inference(s)" << std::endl;
92 break;
93 case 'h': // -h or --help
94 case '?': // Unrecognized option
95 default:
96 print_help(argv);
97 break;
98 }
99 }
100
101 if (model_file_str.empty())
102 {
103 print_help(argv);
104 }
105 }
106
main(int argc,char * argv[])107 int main(int argc, char* argv[])
108 {
109 std::vector<double> inferenceTimes;
110
111 // Get options
112 process_args(argc, argv);
113
114 // Create the runtime
115 armnn::IRuntime::CreationOptions options;
116 armnn::IRuntimePtr runtime(armnn::IRuntime::Create(options));
117
118 // Create Parser
119 armnnTfLiteParser::ITfLiteParserPtr armnnparser(armnnTfLiteParser::ITfLiteParser::Create());
120
121 // Create a network
122 armnn::INetworkPtr network = armnnparser->CreateNetworkFromBinaryFile(model_file_str.c_str());
123 if (!network)
124 {
125 throw armnn::Exception("Failed to create an ArmNN network");
126 }
127
128 // Optimize the network
129 if (preferred_backends_order.size() == 0)
130 {
131 preferred_backends_order = default_preferred_backends_order;
132 }
133 armnn::IOptimizedNetworkPtr optimizedNet = armnn::Optimize(*network,
134 preferred_backends_order,
135 runtime->GetDeviceSpec());
136 armnn::NetworkId networkId;
137
138 // Load the network in to the runtime
139 runtime->LoadNetwork(networkId, std::move(optimizedNet));
140
141 // Check the number of subgraph
142 if (armnnparser->GetSubgraphCount() != 1)
143 {
144 std::cout << "Model with more than 1 subgraph is not supported by this benchmark application.\n";
145 exit(0);
146 }
147 size_t subgraphId = 0;
148
149 // Set up the input network
150 std::cout << "\nModel information:" << std::endl;
151 std::vector<armnnTfLiteParser::BindingPointInfo> inputBindings;
152 std::vector<armnn::TensorInfo> inputTensorInfos;
153 std::vector<std::string> inputTensorNames = armnnparser->GetSubgraphInputTensorNames(subgraphId);
154 for (unsigned int i = 0; i < inputTensorNames.size() ; i++)
155 {
156 std::cout << "inputTensorNames[" << i << "] = " << inputTensorNames[i] << std::endl;
157 armnnTfLiteParser::BindingPointInfo inputBinding = armnnparser->GetNetworkInputBindingInfo(
158 subgraphId,
159 inputTensorNames[i]);
160 armnn::TensorInfo inputTensorInfo = runtime->GetInputTensorInfo(networkId, inputBinding.first);
161 inputBindings.push_back(inputBinding);
162 inputTensorInfos.push_back(inputTensorInfo);
163 }
164
165 // Set up the output network
166 std::vector<armnnTfLiteParser::BindingPointInfo> outputBindings;
167 std::vector<armnn::TensorInfo> outputTensorInfos;
168 std::vector<std::string> outputTensorNames = armnnparser->GetSubgraphOutputTensorNames(subgraphId);
169 for (unsigned int i = 0; i < outputTensorNames.size() ; i++)
170 {
171 std::cout << "outputTensorNames[" << i << "] = " << outputTensorNames[i] << std::endl;
172 armnnTfLiteParser::BindingPointInfo outputBinding = armnnparser->GetNetworkOutputBindingInfo(
173 subgraphId,
174 outputTensorNames[i]);
175 armnn::TensorInfo outputTensorInfo = runtime->GetOutputTensorInfo(networkId, outputBinding.first);
176 outputBindings.push_back(outputBinding);
177 outputTensorInfos.push_back(outputTensorInfo);
178 }
179
180 // Allocate input tensors
181 unsigned int nb_inputs = armnn::numeric_cast<unsigned int>(inputTensorInfos.size());
182 armnn::InputTensors inputTensors;
183 std::vector<std::vector<float>> in;
184 for (unsigned int i = 0 ; i < nb_inputs ; i++)
185 {
186 std::vector<float> in_data(inputTensorInfos.at(i).GetNumElements());
187 in.push_back(in_data);
188 inputTensors.push_back({ inputBindings[i].first, armnn::ConstTensor(inputBindings[i].second, in[i].data()) });
189 }
190
191 // Allocate output tensors
192 unsigned int nb_ouputs = armnn::numeric_cast<unsigned int>(outputTensorInfos.size());
193 armnn::OutputTensors outputTensors;
194 std::vector<std::vector<float>> out;
195 for (unsigned int i = 0; i < nb_ouputs ; i++)
196 {
197 std::vector<float> out_data(outputTensorInfos.at(i).GetNumElements());
198 out.push_back(out_data);
199 outputTensors.push_back({ outputBindings[i].first, armnn::Tensor(outputBindings[i].second, out[i].data()) });
200 }
201
202 // Run the inferences
203 std::cout << "\ninferences are running: " << std::flush;
204 for (int i = 0 ; i < nb_loops ; i++)
205 {
206 struct timeval start_time, stop_time;
207 gettimeofday(&start_time, nullptr);
208
209 runtime->EnqueueWorkload(networkId, inputTensors, outputTensors);
210
211 gettimeofday(&stop_time, nullptr);
212 inferenceTimes.push_back((get_us(stop_time) - get_us(start_time)));
213 std::cout << "# " << std::flush;
214 }
215
216 auto maxInfTime = *std::max_element(inferenceTimes.begin(), inferenceTimes.end());
217 auto minInfTime = *std::min_element(inferenceTimes.begin(), inferenceTimes.end());
218 auto avgInfTime = accumulate(inferenceTimes.begin(), inferenceTimes.end(), 0.0) /
219 armnn::numeric_cast<double>(inferenceTimes.size());
220 std::cout << "\n\ninference time: ";
221 std::cout << "min=" << minInfTime << "us ";
222 std::cout << "max=" << maxInfTime << "us ";
223 std::cout << "avg=" << avgInfTime << "us" << std::endl;
224
225 return 0;
226 }
227