1 // Copyright 2013 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "base/test/launcher/unit_test_launcher.h"
6
7 #include <map>
8 #include <memory>
9 #include <string_view>
10 #include <utility>
11
12 #include "base/base_paths.h"
13 #include "base/base_switches.h"
14 #include "base/command_line.h"
15 #include "base/compiler_specific.h"
16 #include "base/debug/debugger.h"
17 #include "base/files/file_util.h"
18 #include "base/format_macros.h"
19 #include "base/functional/bind.h"
20 #include "base/functional/callback_helpers.h"
21 #include "base/i18n/icu_util.h"
22 #include "base/location.h"
23 #include "base/logging.h"
24 #include "base/message_loop/message_pump_type.h"
25 #include "base/sequence_checker.h"
26 #include "base/strings/strcat.h"
27 #include "base/strings/string_number_conversions.h"
28 #include "base/strings/string_util.h"
29 #include "base/system/sys_info.h"
30 #include "base/task/single_thread_task_executor.h"
31 #include "base/task/single_thread_task_runner.h"
32 #include "base/test/allow_check_is_test_for_testing.h"
33 #include "base/test/launcher/test_launcher.h"
34 #include "base/test/scoped_block_tests_writing_to_special_dirs.h"
35 #include "base/test/test_switches.h"
36 #include "base/test/test_timeouts.h"
37 #include "base/threading/thread_checker.h"
38 #include "build/build_config.h"
39 #include "testing/gtest/include/gtest/gtest.h"
40
41 #if BUILDFLAG(IS_POSIX)
42 #include "base/files/file_descriptor_watcher_posix.h"
43 #endif
44
45 #if BUILDFLAG(IS_IOS)
46 #include "base/test/test_support_ios.h"
47 #endif
48
49 namespace base {
50
51 namespace {
52
53 // This constant controls how many tests are run in a single batch by default.
54 const size_t kDefaultTestBatchLimit =
55 #if BUILDFLAG(IS_IOS)
56 100;
57 #else
58 10;
59 #endif
60
61 #if !BUILDFLAG(IS_ANDROID)
PrintUsage()62 void PrintUsage() {
63 fprintf(
64 stdout,
65 "Runs tests using the gtest framework, each batch of tests being\n"
66 "run in their own process. Supported command-line flags:\n"
67 "\n"
68 " Common flags:\n"
69 " --gtest_filter=...\n"
70 " Runs a subset of tests (see --gtest_help for more info).\n"
71 "\n"
72 " --help\n"
73 " Shows this message.\n"
74 "\n"
75 " --gtest_help\n"
76 " Shows the gtest help message.\n"
77 "\n"
78 " --test-launcher-jobs=N\n"
79 " Sets the number of parallel test jobs to N.\n"
80 "\n"
81 " --single-process-tests\n"
82 " Runs the tests and the launcher in the same process. Useful\n"
83 " for debugging a specific test in a debugger.\n"
84 "\n"
85 " Other flags:\n"
86 " --test-launcher-filter-file=PATH\n"
87 " Like --gtest_filter, but read the test filter from PATH.\n"
88 " Supports multiple filter paths separated by ';'.\n"
89 " One pattern per line; lines starting with '-' are exclusions.\n"
90 " See also //testing/buildbot/filters/README.md file.\n"
91 "\n"
92 " --test-launcher-batch-limit=N\n"
93 " Sets the limit of test batch to run in a single process to N.\n"
94 "\n"
95 " --test-launcher-debug-launcher\n"
96 " Disables autodetection of debuggers and similar tools,\n"
97 " making it possible to use them to debug launcher itself.\n"
98 "\n"
99 " --test-launcher-retry-limit=N\n"
100 " Sets the limit of test retries on failures to N.\n"
101 " --gtest_repeat=N\n"
102 " Forces the launcher to run every test N times. -1 is a special"
103 " value, causing the infinite amount of iterations."
104 " Repeated tests are run in parallel, unless the number of"
105 " iterations is infinite or --gtest_break_on_failure is specified"
106 " (see below)."
107 " Consider using --test_launcher-jobs flag to speed up the"
108 " parallel execution."
109 "\n"
110 " --gtest_break_on_failure\n"
111 " Stop running repeated tests as soon as one repeat of the test fails."
112 " This flag forces sequential repeats and prevents parallelised"
113 " execution."
114 "\n"
115 " --test-launcher-summary-output=PATH\n"
116 " Saves a JSON machine-readable summary of the run.\n"
117 "\n"
118 " --test-launcher-print-test-stdio=auto|always|never\n"
119 " Controls when full test output is printed.\n"
120 " auto means to print it when the test failed.\n"
121 "\n"
122 " --test-launcher-test-part-results-limit=N\n"
123 " Sets the limit of failed EXPECT/ASSERT entries in the xml and\n"
124 " JSON outputs per test to N (default N=10). Negative value \n"
125 " will disable this limit.\n"
126 "\n"
127 " --test-launcher-total-shards=N\n"
128 " Sets the total number of shards to N.\n"
129 "\n"
130 " --test-launcher-shard-index=N\n"
131 " Sets the shard index to run to N (from 0 to TOTAL - 1).\n"
132 "\n"
133 " --test-launcher-print-temp-leaks\n"
134 " Prints information about leaked files and/or directories in\n"
135 " child process's temporary directories (Windows and macOS).\n");
136 fflush(stdout);
137 }
138
GetSwitchValueAsInt(const std::string & switch_name,int * result)139 bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
140 if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
141 return true;
142
143 std::string switch_value =
144 CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
145 if (!StringToInt(switch_value, result) || *result < 0) {
146 LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
147 return false;
148 }
149
150 return true;
151 }
152
RunTestSuite(RunTestSuiteCallback run_test_suite,size_t parallel_jobs,int default_batch_limit,size_t retry_limit,bool use_job_objects,RepeatingClosure timeout_callback,OnceClosure gtest_init)153 int RunTestSuite(RunTestSuiteCallback run_test_suite,
154 size_t parallel_jobs,
155 int default_batch_limit,
156 size_t retry_limit,
157 bool use_job_objects,
158 RepeatingClosure timeout_callback,
159 OnceClosure gtest_init) {
160 bool force_single_process = false;
161 if (CommandLine::ForCurrentProcess()->HasSwitch(
162 switches::kTestLauncherDebugLauncher)) {
163 fprintf(stdout, "Forcing test launcher debugging mode.\n");
164 fflush(stdout);
165 } else {
166 if (base::debug::BeingDebugged()) {
167 fprintf(stdout,
168 "Debugger detected, switching to single process mode.\n"
169 "Pass --test-launcher-debug-launcher to debug the launcher "
170 "itself.\n");
171 fflush(stdout);
172 force_single_process = true;
173 }
174 }
175
176 if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) ||
177 CommandLine::ForCurrentProcess()->HasSwitch(kGTestListTestsFlag) ||
178 CommandLine::ForCurrentProcess()->HasSwitch(
179 switches::kSingleProcessTests) ||
180 CommandLine::ForCurrentProcess()->HasSwitch(
181 switches::kTestChildProcess) ||
182 force_single_process) {
183 return std::move(run_test_suite).Run();
184 }
185
186 // ICU must be initialized before any attempts to format times, e.g. for logs.
187 CHECK(base::i18n::InitializeICU());
188
189 if (CommandLine::ForCurrentProcess()->HasSwitch(switches::kHelpFlag)) {
190 PrintUsage();
191 return 0;
192 }
193
194 TimeTicks start_time(TimeTicks::Now());
195
196 std::move(gtest_init).Run();
197 TestTimeouts::Initialize();
198
199 int batch_limit = default_batch_limit;
200 if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit))
201 return 1;
202
203 fprintf(stdout,
204 "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n"
205 "own process. For debugging a test inside a debugger, use the\n"
206 "--gtest_filter=<your_test_name> flag along with\n"
207 "--single-process-tests.\n");
208 fflush(stdout);
209
210 base::SingleThreadTaskExecutor executor(base::MessagePumpType::IO);
211 #if BUILDFLAG(IS_POSIX)
212 FileDescriptorWatcher file_descriptor_watcher(executor.task_runner());
213 #endif
214 DefaultUnitTestPlatformDelegate platform_delegate;
215 UnitTestLauncherDelegate delegate(&platform_delegate, batch_limit,
216 use_job_objects, timeout_callback);
217 TestLauncher launcher(&delegate, parallel_jobs, retry_limit);
218 bool success = launcher.Run();
219
220 fprintf(stdout, "Tests took %" PRId64 " seconds.\n",
221 (TimeTicks::Now() - start_time).InSeconds());
222 fflush(stdout);
223
224 return (success ? 0 : 1);
225 }
226 #endif
227
LaunchUnitTestsInternal(RunTestSuiteCallback run_test_suite,size_t parallel_jobs,int default_batch_limit,size_t retry_limit,bool use_job_objects,RepeatingClosure timeout_callback,OnceClosure gtest_init)228 int LaunchUnitTestsInternal(RunTestSuiteCallback run_test_suite,
229 size_t parallel_jobs,
230 int default_batch_limit,
231 size_t retry_limit,
232 bool use_job_objects,
233 RepeatingClosure timeout_callback,
234 OnceClosure gtest_init) {
235 base::test::AllowCheckIsTestForTesting();
236
237 #if BUILDFLAG(IS_ANDROID)
238 // We can't easily fork on Android, just run the test suite directly.
239 return std::move(run_test_suite).Run();
240 #elif BUILDFLAG(IS_IOS)
241 InitIOSRunHook(base::BindOnce(&RunTestSuite, std::move(run_test_suite),
242 parallel_jobs, default_batch_limit, retry_limit,
243 use_job_objects, timeout_callback,
244 std::move(gtest_init)));
245 return RunTestsFromIOSApp();
246 #else
247 ScopedBlockTestsWritingToSpecialDirs scoped_blocker(
248 {
249 // Please keep these in alphabetic order within each platform type.
250 base::DIR_SRC_TEST_DATA_ROOT, base::DIR_USER_DESKTOP,
251 #if BUILDFLAG(IS_WIN)
252 base::DIR_COMMON_DESKTOP, base::DIR_START_MENU,
253 base::DIR_USER_STARTUP,
254
255 #endif // BUILDFLAG(IS_WIN)
256 },
257 ([](const base::FilePath& path) {
258 ADD_FAILURE()
259 << "Attempting to write file in dir " << path
260 << " Use ScopedPathOverride or other mechanism to not write to this"
261 " directory.";
262 }));
263 return RunTestSuite(std::move(run_test_suite), parallel_jobs,
264 default_batch_limit, retry_limit, use_job_objects,
265 timeout_callback, std::move(gtest_init));
266 #endif
267 }
268
InitGoogleTestChar(int * argc,char ** argv)269 void InitGoogleTestChar(int* argc, char** argv) {
270 testing::InitGoogleTest(argc, argv);
271 }
272
273 #if BUILDFLAG(IS_WIN)
InitGoogleTestWChar(int * argc,wchar_t ** argv)274 void InitGoogleTestWChar(int* argc, wchar_t** argv) {
275 testing::InitGoogleTest(argc, argv);
276 }
277 #endif // BUILDFLAG(IS_WIN)
278
279 } // namespace
280
281 MergeTestFilterSwitchHandler::~MergeTestFilterSwitchHandler() = default;
ResolveDuplicate(std::string_view key,CommandLine::StringPieceType new_value,CommandLine::StringType & out_value)282 void MergeTestFilterSwitchHandler::ResolveDuplicate(
283 std::string_view key,
284 CommandLine::StringPieceType new_value,
285 CommandLine::StringType& out_value) {
286 if (key != switches::kTestLauncherFilterFile) {
287 out_value = CommandLine::StringType(new_value);
288 return;
289 }
290 if (!out_value.empty()) {
291 #if BUILDFLAG(IS_WIN)
292 StrAppend(&out_value, {L";"});
293 #else
294 StrAppend(&out_value, {";"});
295 #endif
296 }
297 StrAppend(&out_value, {new_value});
298 }
299
LaunchUnitTests(int argc,char ** argv,RunTestSuiteCallback run_test_suite,size_t retry_limit)300 int LaunchUnitTests(int argc,
301 char** argv,
302 RunTestSuiteCallback run_test_suite,
303 size_t retry_limit) {
304 CommandLine::SetDuplicateSwitchHandler(
305 std::make_unique<MergeTestFilterSwitchHandler>());
306 CommandLine::Init(argc, argv);
307 size_t parallel_jobs = NumParallelJobs(/*cores_per_job=*/1);
308 if (parallel_jobs == 0U) {
309 return 1;
310 }
311 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
312 kDefaultTestBatchLimit, retry_limit, true,
313 DoNothing(),
314 BindOnce(&InitGoogleTestChar, &argc, argv));
315 }
316
LaunchUnitTestsSerially(int argc,char ** argv,RunTestSuiteCallback run_test_suite)317 int LaunchUnitTestsSerially(int argc,
318 char** argv,
319 RunTestSuiteCallback run_test_suite) {
320 CommandLine::Init(argc, argv);
321 return LaunchUnitTestsInternal(std::move(run_test_suite), 1U,
322 kDefaultTestBatchLimit, 1U, true, DoNothing(),
323 BindOnce(&InitGoogleTestChar, &argc, argv));
324 }
325
LaunchUnitTestsWithOptions(int argc,char ** argv,size_t parallel_jobs,int default_batch_limit,bool use_job_objects,RepeatingClosure timeout_callback,RunTestSuiteCallback run_test_suite)326 int LaunchUnitTestsWithOptions(int argc,
327 char** argv,
328 size_t parallel_jobs,
329 int default_batch_limit,
330 bool use_job_objects,
331 RepeatingClosure timeout_callback,
332 RunTestSuiteCallback run_test_suite) {
333 CommandLine::Init(argc, argv);
334 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
335 default_batch_limit, 1U, use_job_objects,
336 timeout_callback,
337 BindOnce(&InitGoogleTestChar, &argc, argv));
338 }
339
340 #if BUILDFLAG(IS_WIN)
LaunchUnitTests(int argc,wchar_t ** argv,bool use_job_objects,RunTestSuiteCallback run_test_suite)341 int LaunchUnitTests(int argc,
342 wchar_t** argv,
343 bool use_job_objects,
344 RunTestSuiteCallback run_test_suite) {
345 // Windows CommandLine::Init ignores argv anyway.
346 CommandLine::Init(argc, NULL);
347 size_t parallel_jobs = NumParallelJobs(/*cores_per_job=*/1);
348 if (parallel_jobs == 0U) {
349 return 1;
350 }
351 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
352 kDefaultTestBatchLimit, 1U, use_job_objects,
353 DoNothing(),
354 BindOnce(&InitGoogleTestWChar, &argc, argv));
355 }
356 #endif // BUILDFLAG(IS_WIN)
357
358 DefaultUnitTestPlatformDelegate::DefaultUnitTestPlatformDelegate() = default;
359
GetTests(std::vector<TestIdentifier> * output)360 bool DefaultUnitTestPlatformDelegate::GetTests(
361 std::vector<TestIdentifier>* output) {
362 *output = GetCompiledInTests();
363 return true;
364 }
365
CreateResultsFile(const base::FilePath & temp_dir,base::FilePath * path)366 bool DefaultUnitTestPlatformDelegate::CreateResultsFile(
367 const base::FilePath& temp_dir,
368 base::FilePath* path) {
369 if (!CreateTemporaryDirInDir(temp_dir, FilePath::StringType(), path))
370 return false;
371 *path = path->AppendASCII("test_results.xml");
372 return true;
373 }
374
CreateTemporaryFile(const base::FilePath & temp_dir,base::FilePath * path)375 bool DefaultUnitTestPlatformDelegate::CreateTemporaryFile(
376 const base::FilePath& temp_dir,
377 base::FilePath* path) {
378 if (temp_dir.empty())
379 return false;
380 return CreateTemporaryFileInDir(temp_dir, path);
381 }
382
GetCommandLineForChildGTestProcess(const std::vector<std::string> & test_names,const base::FilePath & output_file,const base::FilePath & flag_file)383 CommandLine DefaultUnitTestPlatformDelegate::GetCommandLineForChildGTestProcess(
384 const std::vector<std::string>& test_names,
385 const base::FilePath& output_file,
386 const base::FilePath& flag_file) {
387 CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
388
389 CHECK(base::PathExists(flag_file));
390
391 // Any `--gtest_filter` flag specified on the original command line is
392 // no longer needed; the test launcher has already determined the list
393 // of actual tests to run in each child process. Since the test launcher
394 // internally uses `--gtest_filter` via a flagfile to pass this info to
395 // the child process, remove any original `--gtest_filter` flags on the
396 // command line, as GoogleTest provides no guarantee about whether the
397 // command line or the flagfile takes precedence.
398 new_cmd_line.RemoveSwitch(kGTestFilterFlag);
399
400 std::string long_flags(
401 StrCat({"--", kGTestFilterFlag, "=", JoinString(test_names, ":")}));
402 CHECK(WriteFile(flag_file, long_flags));
403
404 new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file);
405 new_cmd_line.AppendSwitchPath(kGTestFlagfileFlag, flag_file);
406 new_cmd_line.AppendSwitch(switches::kSingleProcessTests);
407
408 return new_cmd_line;
409 }
410
GetWrapperForChildGTestProcess()411 std::string DefaultUnitTestPlatformDelegate::GetWrapperForChildGTestProcess() {
412 return std::string();
413 }
414
UnitTestLauncherDelegate(UnitTestPlatformDelegate * platform_delegate,size_t batch_limit,bool use_job_objects,RepeatingClosure timeout_callback)415 UnitTestLauncherDelegate::UnitTestLauncherDelegate(
416 UnitTestPlatformDelegate* platform_delegate,
417 size_t batch_limit,
418 bool use_job_objects,
419 RepeatingClosure timeout_callback)
420 : platform_delegate_(platform_delegate),
421 batch_limit_(batch_limit),
422 use_job_objects_(use_job_objects),
423 timeout_callback_(timeout_callback) {}
424
~UnitTestLauncherDelegate()425 UnitTestLauncherDelegate::~UnitTestLauncherDelegate() {
426 DCHECK(thread_checker_.CalledOnValidThread());
427 }
428
GetTests(std::vector<TestIdentifier> * output)429 bool UnitTestLauncherDelegate::GetTests(std::vector<TestIdentifier>* output) {
430 DCHECK(thread_checker_.CalledOnValidThread());
431 return platform_delegate_->GetTests(output);
432 }
433
GetCommandLine(const std::vector<std::string> & test_names,const FilePath & temp_dir,FilePath * output_file)434 CommandLine UnitTestLauncherDelegate::GetCommandLine(
435 const std::vector<std::string>& test_names,
436 const FilePath& temp_dir,
437 FilePath* output_file) {
438 CHECK(!test_names.empty());
439
440 // Create a dedicated temporary directory to store the xml result data
441 // per run to ensure clean state and make it possible to launch multiple
442 // processes in parallel.
443 CHECK(platform_delegate_->CreateResultsFile(temp_dir, output_file));
444 FilePath flag_file;
445 platform_delegate_->CreateTemporaryFile(temp_dir, &flag_file);
446
447 return CommandLine(platform_delegate_->GetCommandLineForChildGTestProcess(
448 test_names, *output_file, flag_file));
449 }
450
GetWrapper()451 std::string UnitTestLauncherDelegate::GetWrapper() {
452 return platform_delegate_->GetWrapperForChildGTestProcess();
453 }
454
GetLaunchOptions()455 int UnitTestLauncherDelegate::GetLaunchOptions() {
456 return use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0;
457 }
458
GetTimeout()459 TimeDelta UnitTestLauncherDelegate::GetTimeout() {
460 return TestTimeouts::test_launcher_timeout();
461 }
462
GetBatchSize()463 size_t UnitTestLauncherDelegate::GetBatchSize() {
464 return batch_limit_;
465 }
466
OnTestTimedOut(const CommandLine & cmd_line)467 void UnitTestLauncherDelegate::OnTestTimedOut(const CommandLine& cmd_line) {
468 timeout_callback_.Run();
469 }
470
471 } // namespace base
472