1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 #include "tensorflow/lite/delegates/telemetry.h"
16
17 #include <cstdint>
18 #include <string>
19
20 #include <gmock/gmock.h>
21 #include <gtest/gtest.h>
22 #include "flatbuffers/flatbuffers.h" // from @flatbuffers
23 #include "tensorflow/lite/c/common.h"
24 #include "tensorflow/lite/core/api/profiler.h"
25 #include "tensorflow/lite/experimental/acceleration/configuration/configuration_generated.h"
26 #include "tensorflow/lite/profiling/profile_buffer.h"
27
28 namespace tflite {
29 namespace delegates {
30 namespace {
31
32 constexpr int32_t kDummyCode = 2;
33 constexpr bool kDummyGpuPrecisionLossAllowed = true;
34 constexpr tflite::Delegate kDummyDelegate = tflite::Delegate_GPU;
35 constexpr DelegateStatusSource kDummySource =
36 DelegateStatusSource::TFLITE_NNAPI;
37
TEST(TelemetryTest,StatusConversion)38 TEST(TelemetryTest, StatusConversion) {
39 DelegateStatus status(kDummySource, kDummyCode);
40 int64_t serialized_int = status.full_status();
41 DelegateStatus deserialized_status(serialized_int);
42
43 EXPECT_EQ(kDummyCode, deserialized_status.code());
44 EXPECT_EQ(kDummySource, deserialized_status.source());
45 EXPECT_EQ(serialized_int, deserialized_status.full_status());
46 }
47
48 // Dummy profiler to test delegate reporting.
49 class DelegateProfiler : public Profiler {
50 public:
DelegateProfiler()51 DelegateProfiler() {}
52 ~DelegateProfiler() override = default;
53
BeginEvent(const char * tag,EventType event_type,int64_t event_metadata1,int64_t event_metadata2)54 uint32_t BeginEvent(const char* tag, EventType event_type,
55 int64_t event_metadata1,
56 int64_t event_metadata2) override {
57 int event_handle = -1;
58 if (event_type ==
59 Profiler::EventType::GENERAL_RUNTIME_INSTRUMENTATION_EVENT &&
60 std::string(tag) == kDelegateSettingsTag) {
61 event_buffer_.emplace_back();
62 event_handle = event_buffer_.size();
63
64 // event_metadata1 is a pointer to a TfLiteDelegate.
65 EXPECT_NE(event_metadata1, 0);
66 auto* delegate = reinterpret_cast<TfLiteDelegate*>(event_metadata1);
67 EXPECT_EQ(delegate->flags, kTfLiteDelegateFlagsNone);
68 // event_metadata2 is a pointer to TFLiteSettings.
69 EXPECT_NE(event_metadata2, 0);
70 auto* settings = reinterpret_cast<TFLiteSettings*>(event_metadata2);
71 EXPECT_EQ(settings->delegate(), kDummyDelegate);
72 EXPECT_EQ(settings->gpu_settings()->is_precision_loss_allowed(),
73 kDummyGpuPrecisionLossAllowed);
74 } else if (event_type ==
75 Profiler::EventType::GENERAL_RUNTIME_INSTRUMENTATION_EVENT &&
76 std::string(tag) == kDelegateStatusTag) {
77 event_buffer_.emplace_back();
78 event_handle = event_buffer_.size();
79
80 EXPECT_EQ(event_metadata2, static_cast<int64_t>(kTfLiteOk));
81 DelegateStatus reported_status(event_metadata1);
82 EXPECT_EQ(reported_status.source(), kDummySource);
83 EXPECT_EQ(reported_status.code(), kDummyCode);
84 }
85
86 EXPECT_NE(-1, event_handle);
87 return event_handle;
88 }
89
EndEvent(uint32_t event_handle)90 void EndEvent(uint32_t event_handle) override {
91 EXPECT_EQ(event_handle, event_buffer_.size());
92 }
93
NumRecordedEvents()94 int NumRecordedEvents() { return event_buffer_.size(); }
95
96 private:
97 std::vector<profiling::ProfileEvent> event_buffer_;
98 };
99
TEST(TelemetryTest,DelegateStatusReport)100 TEST(TelemetryTest, DelegateStatusReport) {
101 DelegateProfiler profiler;
102 TfLiteDelegate delegate = TfLiteDelegateCreate();
103 TfLiteContext context;
104 context.profiler = &profiler;
105 DelegateStatus status(kDummySource, kDummyCode);
106
107 EXPECT_EQ(ReportDelegateStatus(&context, &delegate, status), kTfLiteOk);
108 EXPECT_EQ(ReportDelegateStatus(&context, &delegate, status), kTfLiteOk);
109 EXPECT_EQ(profiler.NumRecordedEvents(), 2);
110 }
111
TEST(TelemetryTest,DelegateSettingsReport)112 TEST(TelemetryTest, DelegateSettingsReport) {
113 DelegateProfiler profiler;
114 TfLiteDelegate delegate = TfLiteDelegateCreate();
115 TfLiteContext context;
116 context.profiler = &profiler;
117
118 flatbuffers::FlatBufferBuilder flatbuffer_builder;
119 flatbuffers::Offset<tflite::GPUSettings> gpu_settings =
120 tflite::CreateGPUSettings(
121 flatbuffer_builder,
122 /**is_precision_loss_allowed**/ kDummyGpuPrecisionLossAllowed);
123 auto* tflite_settings_ptr = flatbuffers::GetTemporaryPointer(
124 flatbuffer_builder,
125 CreateTFLiteSettings(flatbuffer_builder, kDummyDelegate,
126 /*nnapi_settings=*/0,
127 /*gpu_settings=*/gpu_settings));
128
129 EXPECT_EQ(ReportDelegateSettings(&context, &delegate, *tflite_settings_ptr),
130 kTfLiteOk);
131 EXPECT_EQ(profiler.NumRecordedEvents(), 1);
132
133 // Also report status to simulate typical use-case.
134 DelegateStatus status(kDummySource, kDummyCode);
135 EXPECT_EQ(ReportDelegateStatus(&context, &delegate, status), kTfLiteOk);
136 EXPECT_EQ(ReportDelegateStatus(&context, &delegate, status), kTfLiteOk);
137 EXPECT_EQ(profiler.NumRecordedEvents(), 3);
138 }
139
140 } // namespace
141 } // namespace delegates
142 } // namespace tflite
143