1 // 2 // Copyright © 2017 Arm Ltd. All rights reserved. 3 // SPDX-License-Identifier: MIT 4 // 5 #pragma once 6 7 #include "InferenceTest.hpp" 8 #include "YoloDatabase.hpp" 9 10 #include <armnn/utility/Assert.hpp> 11 #include <armnn/utility/IgnoreUnused.hpp> 12 #include <armnnUtils/FloatingPointComparison.hpp> 13 14 #include <algorithm> 15 #include <array> 16 #include <utility> 17 18 constexpr size_t YoloOutputSize = 1470; 19 20 template <typename Model> 21 class YoloTestCase : public InferenceModelTestCase<Model> 22 { 23 public: YoloTestCase(Model & model,unsigned int testCaseId,YoloTestCaseData & testCaseData)24 YoloTestCase(Model& model, 25 unsigned int testCaseId, 26 YoloTestCaseData& testCaseData) 27 : InferenceModelTestCase<Model>(model, testCaseId, { std::move(testCaseData.m_InputImage) }, { YoloOutputSize }) 28 , m_TopObjectDetections(std::move(testCaseData.m_TopObjectDetections)) 29 { 30 } 31 ProcessResult(const InferenceTestOptions & options)32 virtual TestCaseResult ProcessResult(const InferenceTestOptions& options) override 33 { 34 armnn::IgnoreUnused(options); 35 36 const std::vector<float>& output = mapbox::util::get<std::vector<float>>(this->GetOutputs()[0]); 37 ARMNN_ASSERT(output.size() == YoloOutputSize); 38 39 constexpr unsigned int gridSize = 7; 40 constexpr unsigned int numClasses = 20; 41 constexpr unsigned int numScales = 2; 42 43 const float* outputPtr = output.data(); 44 45 // Range 0-980. Class probabilities. 7x7x20 46 vector<vector<vector<float>>> classProbabilities(gridSize, vector<vector<float>>(gridSize, 47 vector<float>(numClasses))); 48 for (unsigned int y = 0; y < gridSize; ++y) 49 { 50 for (unsigned int x = 0; x < gridSize; ++x) 51 { 52 for (unsigned int c = 0; c < numClasses; ++c) 53 { 54 classProbabilities[y][x][c] = *outputPtr++; 55 } 56 } 57 } 58 59 // Range 980-1078. Scales. 7x7x2 60 vector<vector<vector<float>>> scales(gridSize, vector<vector<float>>(gridSize, vector<float>(numScales))); 61 for (unsigned int y = 0; y < gridSize; ++y) 62 { 63 for (unsigned int x = 0; x < gridSize; ++x) 64 { 65 for (unsigned int s = 0; s < numScales; ++s) 66 { 67 scales[y][x][s] = *outputPtr++; 68 } 69 } 70 } 71 72 // Range 1078-1469. Bounding boxes. 7x7x2x4 73 constexpr float imageWidthAsFloat = static_cast<float>(YoloImageWidth); 74 constexpr float imageHeightAsFloat = static_cast<float>(YoloImageHeight); 75 76 vector<vector<vector<vector<float>>>> boxes(gridSize, vector<vector<vector<float>>> 77 (gridSize, vector<vector<float>>(numScales, vector<float>(4)))); 78 for (unsigned int y = 0; y < gridSize; ++y) 79 { 80 for (unsigned int x = 0; x < gridSize; ++x) 81 { 82 for (unsigned int s = 0; s < numScales; ++s) 83 { 84 float bx = *outputPtr++; 85 float by = *outputPtr++; 86 float bw = *outputPtr++; 87 float bh = *outputPtr++; 88 89 boxes[y][x][s][0] = ((bx + static_cast<float>(x)) / 7.0f) * imageWidthAsFloat; 90 boxes[y][x][s][1] = ((by + static_cast<float>(y)) / 7.0f) * imageHeightAsFloat; 91 boxes[y][x][s][2] = bw * bw * static_cast<float>(imageWidthAsFloat); 92 boxes[y][x][s][3] = bh * bh * static_cast<float>(imageHeightAsFloat); 93 } 94 } 95 } 96 ARMNN_ASSERT(output.data() + YoloOutputSize == outputPtr); 97 98 std::vector<YoloDetectedObject> detectedObjects; 99 detectedObjects.reserve(gridSize * gridSize * numScales * numClasses); 100 101 for (unsigned int y = 0; y < gridSize; ++y) 102 { 103 for (unsigned int x = 0; x < gridSize; ++x) 104 { 105 for (unsigned int s = 0; s < numScales; ++s) 106 { 107 for (unsigned int c = 0; c < numClasses; ++c) 108 { 109 // Resolved confidence: class probabilities * scales. 110 const float confidence = classProbabilities[y][x][c] * scales[y][x][s]; 111 112 // Resolves bounding box and stores. 113 YoloBoundingBox box; 114 box.m_X = boxes[y][x][s][0]; 115 box.m_Y = boxes[y][x][s][1]; 116 box.m_W = boxes[y][x][s][2]; 117 box.m_H = boxes[y][x][s][3]; 118 119 detectedObjects.emplace_back(c, box, confidence); 120 } 121 } 122 } 123 } 124 125 // Sorts detected objects by confidence. 126 std::sort(detectedObjects.begin(), detectedObjects.end(), 127 [](const YoloDetectedObject& a, const YoloDetectedObject& b) 128 { 129 // Sorts by largest confidence first, then by class. 130 return a.m_Confidence > b.m_Confidence 131 || (a.m_Confidence == b.m_Confidence && a.m_Class > b.m_Class); 132 }); 133 134 // Checks the top N detections. 135 auto outputIt = detectedObjects.begin(); 136 auto outputEnd = detectedObjects.end(); 137 138 for (const YoloDetectedObject& expectedDetection : m_TopObjectDetections) 139 { 140 if (outputIt == outputEnd) 141 { 142 // Somehow expected more things to check than detections found by the model. 143 return TestCaseResult::Abort; 144 } 145 146 const YoloDetectedObject& detectedObject = *outputIt; 147 if (detectedObject.m_Class != expectedDetection.m_Class) 148 { 149 ARMNN_LOG(error) << "Prediction for test case " << this->GetTestCaseId() << 150 " is incorrect: Expected (" << expectedDetection.m_Class << ")" << 151 " but predicted (" << detectedObject.m_Class << ")"; 152 return TestCaseResult::Failed; 153 } 154 155 if (!armnnUtils::within_percentage_tolerance(detectedObject.m_Box.m_X, expectedDetection.m_Box.m_X) || 156 !armnnUtils::within_percentage_tolerance(detectedObject.m_Box.m_Y, expectedDetection.m_Box.m_Y) || 157 !armnnUtils::within_percentage_tolerance(detectedObject.m_Box.m_W, expectedDetection.m_Box.m_W) || 158 !armnnUtils::within_percentage_tolerance(detectedObject.m_Box.m_H, expectedDetection.m_Box.m_H) || 159 !armnnUtils::within_percentage_tolerance(detectedObject.m_Confidence, expectedDetection.m_Confidence)) 160 { 161 ARMNN_LOG(error) << "Detected bounding box for test case " << this->GetTestCaseId() << 162 " is incorrect"; 163 return TestCaseResult::Failed; 164 } 165 166 ++outputIt; 167 } 168 169 return TestCaseResult::Ok; 170 } 171 172 private: 173 std::vector<YoloDetectedObject> m_TopObjectDetections; 174 }; 175 176 template <typename Model> 177 class YoloTestCaseProvider : public IInferenceTestCaseProvider 178 { 179 public: 180 template <typename TConstructModelCallable> YoloTestCaseProvider(TConstructModelCallable constructModel)181 explicit YoloTestCaseProvider(TConstructModelCallable constructModel) 182 : m_ConstructModel(constructModel) 183 { 184 } 185 AddCommandLineOptions(cxxopts::Options & options,std::vector<std::string> & required)186 virtual void AddCommandLineOptions(cxxopts::Options& options, std::vector<std::string>& required) override 187 { 188 options 189 .allow_unrecognised_options() 190 .add_options() 191 ("d,data-dir", "Path to directory containing test data", cxxopts::value<std::string>(m_DataDir)); 192 193 Model::AddCommandLineOptions(options, m_ModelCommandLineOptions, required); 194 } 195 ProcessCommandLineOptions(const InferenceTestOptions & commonOptions)196 virtual bool ProcessCommandLineOptions(const InferenceTestOptions& commonOptions) override 197 { 198 if (!ValidateDirectory(m_DataDir)) 199 { 200 return false; 201 } 202 203 m_Model = m_ConstructModel(commonOptions, m_ModelCommandLineOptions); 204 if (!m_Model) 205 { 206 return false; 207 } 208 209 m_Database = std::make_unique<YoloDatabase>(m_DataDir.c_str()); 210 if (!m_Database) 211 { 212 return false; 213 } 214 215 return true; 216 } 217 GetTestCase(unsigned int testCaseId)218 virtual std::unique_ptr<IInferenceTestCase> GetTestCase(unsigned int testCaseId) override 219 { 220 std::unique_ptr<YoloTestCaseData> testCaseData = m_Database->GetTestCaseData(testCaseId); 221 if (!testCaseData) 222 { 223 return nullptr; 224 } 225 226 return std::make_unique<YoloTestCase<Model>>(*m_Model, testCaseId, *testCaseData); 227 } 228 229 private: 230 typename Model::CommandLineOptions m_ModelCommandLineOptions; 231 std::function<std::unique_ptr<Model>(const InferenceTestOptions&, 232 typename Model::CommandLineOptions)> m_ConstructModel; 233 std::unique_ptr<Model> m_Model; 234 235 std::string m_DataDir; 236 std::unique_ptr<YoloDatabase> m_Database; 237 }; 238