1 // Copyright 2019 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "testing/perf/luci_test_result.h"
6
7 #include <optional>
8
9 #include "base/files/file_path.h"
10 #include "base/files/file_util.h"
11 #include "base/files/scoped_temp_dir.h"
12 #include "base/json/json_reader.h"
13 #include "base/strings/stringprintf.h"
14 #include "base/time/time.h"
15 #include "testing/gtest/include/gtest/gtest.h"
16
17 namespace perf_test {
18
19 class LuciTestResultTest : public testing::Test {
20 public:
21 LuciTestResultTest() = default;
22
23 LuciTestResultTest(const LuciTestResultTest&) = delete;
24 LuciTestResultTest& operator=(const LuciTestResultTest&) = delete;
25
26 ~LuciTestResultTest() override = default;
27
28 // testing::Test:
SetUp()29 void SetUp() override {
30 testing::Test::SetUp();
31 ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
32 }
33
GetResultFilePath() const34 base::FilePath GetResultFilePath() const {
35 return temp_dir_.GetPath().AppendASCII("luci_test_results.json");
36 }
37
38 // Validates that |result| is written to file that contains an equivalent JSON
39 // as |expected_json|.
ValidateResult(const LuciTestResult & result,const std::string & expected_json)40 void ValidateResult(const LuciTestResult& result,
41 const std::string& expected_json) {
42 const base::FilePath result_file = GetResultFilePath();
43 result.WriteToFile(result_file);
44
45 std::string json;
46 ASSERT_TRUE(ReadFileToString(GetResultFilePath(), &json));
47 std::optional<base::Value> value = base::JSONReader::Read(json);
48 ASSERT_TRUE(value.has_value());
49
50 std::optional<base::Value> expected_value =
51 base::JSONReader::Read(expected_json);
52 ASSERT_TRUE(expected_value.has_value());
53
54 EXPECT_EQ(expected_value, value) << "Expected:\n====\n"
55 << expected_json << "\nActual:\n====\n"
56 << json;
57 }
58
59 private:
60 base::ScopedTempDir temp_dir_;
61 };
62
TEST_F(LuciTestResultTest,Basic)63 TEST_F(LuciTestResultTest, Basic) {
64 LuciTestResult result;
65 result.set_test_path("FakeTestSuite.FakeTest");
66 result.set_status(LuciTestResult::Status::kPass);
67 result.set_is_expected(true);
68
69 result.AddVariant("variantKey", "variantValue");
70 result.AddVariant("param/instantiation", "FooType");
71 result.AddVariant("param/index", "0");
72
73 // 2019/9/11 12:30 UTC
74 base::Time start_time;
75 ASSERT_TRUE(
76 base::Time::FromUTCExploded({2019, 9, 3, 11, 12, 30, 0}, &start_time));
77 result.set_start_time(start_time);
78
79 result.set_duration(base::Milliseconds(1500));
80
81 result.AddOutputArtifactContents("plain", "plain data", "text/plain");
82 result.AddOutputArtifactContents("new_line", "first\nsecond", "text/plain");
83 result.AddOutputArtifactFile(
84 "file.json", base::FilePath(FILE_PATH_LITERAL("/tmp/file.json")),
85 "application/json");
86 result.AddTag("tbmv2", "umaMetric");
87
88 const std::string expected_json =
89 R"({
90 "testResult":{
91 "outputArtifacts":{
92 "file.json":{
93 "contentType":"application/json",
94 "filePath":"/tmp/file.json"
95 },
96 "new_line":{
97 "contentType":"text/plain",
98 "contents":"first\nsecond"
99 },
100 "plain":{
101 "contentType":"text/plain",
102 "contents":"plain data"
103 }
104 },
105 "expected":true,
106 "runDuration":"1.50s",
107 "startTime":"2019-09-11T12:30:00.000Z",
108 "status":"PASS",
109 "tags":[
110 {"key":"tbmv2","value":"umaMetric"}
111 ],
112 "variant":{
113 "variantKey": "variantValue",
114 "param/instantiation": "FooType",
115 "param/index": "0"
116 },
117 "testPath":"FakeTestSuite.FakeTest"
118 }
119 })";
120 ValidateResult(result, expected_json);
121 }
122
TEST_F(LuciTestResultTest,Status)123 TEST_F(LuciTestResultTest, Status) {
124 using Status = LuciTestResult::Status;
125
126 LuciTestResult result;
127 result.set_test_path("FakeTestSuite.Status");
128
129 static constexpr char kJsonTemplate[] =
130 R"({
131 "testResult":{
132 "expected":false,
133 "status":"%s",
134 "testPath":"FakeTestSuite.Status"
135 }
136 })";
137
138 const struct {
139 Status status;
140 const char* status_text;
141 } kTestCases[] = {
142 {Status::kUnspecified, "UNSPECIFIED"},
143 {Status::kPass, "PASS"},
144 {Status::kFail, "FAIL"},
145 {Status::kCrash, "CRASH"},
146 {Status::kAbort, "ABORT"},
147 {Status::kSkip, "SKIP"},
148 };
149
150 for (const auto& test_case : kTestCases) {
151 result.set_status(test_case.status);
152 const std::string expected_json =
153 base::StringPrintf(kJsonTemplate, test_case.status_text);
154 ValidateResult(result, expected_json);
155 }
156 }
157
158 ///////////////////////////////////////////////////////////////////////////////
159
160 class LuciTestResultParameterizedTest
161 : public LuciTestResultTest,
162 public testing::WithParamInterface<int> {
163 public:
164 LuciTestResultParameterizedTest() = default;
165 ~LuciTestResultParameterizedTest() override = default;
166 };
167
TEST_P(LuciTestResultParameterizedTest,Variant)168 TEST_P(LuciTestResultParameterizedTest, Variant) {
169 LuciTestResult result = LuciTestResult::CreateForGTest();
170
171 // 2019/9/11 12:30 UTC
172 base::Time start_time;
173 ASSERT_TRUE(
174 base::Time::FromUTCExploded({2019, 9, 3, 11, 12, 30, 0}, &start_time));
175 result.set_start_time(start_time);
176
177 result.set_duration(base::Milliseconds(1500));
178
179 static constexpr char kJsonTemplate[] =
180 R"({
181 "testResult":{
182 "expected":true,
183 "runDuration":"1.50s",
184 "startTime":"2019-09-11T12:30:00.000Z",
185 "status":"PASS",
186 "testPath":
187 "ZeroToFiveSequence/LuciTestResultParameterizedTest.Variant",
188 "variant":{"param/index":"%d"}
189 }
190 })";
191 const std::string expected_json =
192 base::StringPrintf(kJsonTemplate, GetParam());
193 ValidateResult(result, expected_json);
194 }
195 INSTANTIATE_TEST_SUITE_P(ZeroToFiveSequence,
196 LuciTestResultParameterizedTest,
197 testing::Range(0, 5));
198
199 ///////////////////////////////////////////////////////////////////////////////
200
201 template <typename T>
202 class LuciTestResultTypedTest : public LuciTestResultTest {
203 public:
204 LuciTestResultTypedTest() = default;
205 ~LuciTestResultTypedTest() override = default;
206 };
207
208 TYPED_TEST_SUITE_P(LuciTestResultTypedTest);
209
TYPED_TEST_P(LuciTestResultTypedTest,Variant)210 TYPED_TEST_P(LuciTestResultTypedTest, Variant) {
211 LuciTestResult result = LuciTestResult::CreateForGTest();
212
213 // 2019/9/11 12:30 UTC
214 base::Time start_time;
215 ASSERT_TRUE(
216 base::Time::FromUTCExploded({2019, 9, 3, 11, 12, 30, 0}, &start_time));
217 result.set_start_time(start_time);
218
219 result.set_duration(base::Milliseconds(1500));
220
221 std::string test_suite_name =
222 testing::UnitTest::GetInstance()->current_test_info()->test_suite_name();
223 auto pos = test_suite_name.rfind('/');
224 ASSERT_NE(pos, std::string::npos);
225 std::string type_param_name = test_suite_name.substr(pos + 1);
226
227 static constexpr char kJsonTemplate[] =
228 R"({
229 "testResult":{
230 "expected":true,
231 "runDuration":"1.50s",
232 "startTime":"2019-09-11T12:30:00.000Z",
233 "status":"PASS",
234 "testPath":"SomeTypes/LuciTestResultTypedTest/%s.Variant",
235 "variant":{"param/instantiation":"%s"}
236 }
237 })";
238 // Note that chromium has RTTI disabled. As a result, type_param() and
239 // GetTypeName<> always returns a generic "<type>".
240 const std::string expected_json =
241 base::StringPrintf(kJsonTemplate, type_param_name.c_str(),
242 testing::internal::GetTypeName<TypeParam>().c_str());
243 this->ValidateResult(result, expected_json);
244 }
245
246 REGISTER_TYPED_TEST_SUITE_P(LuciTestResultTypedTest, Variant);
247
248 using SomeTypes = testing::Types<int, double>;
249 INSTANTIATE_TYPED_TEST_SUITE_P(SomeTypes, LuciTestResultTypedTest, SomeTypes);
250
251 } // namespace perf_test
252