1 /*
<lambda>null2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License
15  */
16 
17 package com.android.testutils.host
18 
19 import android.cts.install.lib.host.InstallUtilsHost
20 import com.android.tradefed.config.Option
21 import com.android.tradefed.testtype.DeviceJUnit4ClassRunner
22 import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test
23 import com.android.tradefed.testtype.junit4.DeviceTestRunOptions
24 import com.android.tradefed.util.AaptParser
25 import org.junit.Before
26 import org.junit.Test
27 import org.junit.runner.RunWith
28 import kotlin.test.fail
29 
30 private data class TestFailure(val description: String, val stacktrace: String)
31 
32 /**
33  * Base class for host-driven tests to deflake a test package.
34  *
35  * <p>Classes implementing this base class must define a test APK to be run, and default run
36  * count, timeout and test classes. In manual runs, the run count, timeout and test classes can be
37  * overridden via command-line parameters, such as:
38  *
39  * <pre>
40  * atest TestName -- \
41  *   --test-arg com.android.tradefed.testtype.HostTest:set-option:deflake_run_count:10 \
42  *   --test-arg com.android.tradefed.testtype.HostTest:set-option:deflake_single_run_timeout:10s \
43  *   --test-arg \
44  *      com.android.tradefed.testtype.HostTest:set-option:deflake_test:one.test.Class \
45  *   --test-arg \
46  *      com.android.tradefed.testtype.HostTest:set-option:deflake_test:another.test.Class
47  * </pre>
48  */
49 @RunWith(DeviceJUnit4ClassRunner::class)
50 abstract class DeflakeHostTestBase : BaseHostJUnit4Test() {
51 
52     /**
53      * Number of times the device test will be run.
54      */
55     protected abstract val runCount: Int
56 
57     @Option(name = "deflake_run_count",
58             description = "How many times to run each test case.",
59             importance = Option.Importance.ALWAYS)
60     private var mRunCountOption: Int? = null
61 
62     /**
63      * Filename of the APK to run as part of the test.
64      *
65      * <p>Typically the java_test_host build rule will have a 'data: [":DeviceTest"]' dependency
66      * on the build rule for the device tests. In that case the filename will be "DeviceTest.apk".
67      */
68     protected abstract val testApkFilename: String
69 
70     /**
71      * Timeout for each run of the test, in milliseconds. The host-driven test will fail if any run
72      * takes more than the specified timeout.
73      */
74     protected open val singleRunTimeoutMs = 5 * 60_000L
75 
76     @Option(name = "deflake_single_run_timeout",
77             description = "Timeout for each single run.",
78             importance = Option.Importance.ALWAYS,
79             isTimeVal = true)
80     private var mSingleRunTimeoutMsOption: Long? = null
81 
82     /**
83      * List of classes to run in the test package. If empty, all classes in the package will be run.
84      */
85     protected open val testClasses: List<String> = emptyList()
86 
87     // TODO: also support single methods, not just whole classes
88     @Option(name = "deflake_test",
89             description = "Test class to deflake. Can be repeated. " +
90                     "Default classes configured for the test are run if omitted.",
91             importance = Option.Importance.ALWAYS)
92     private var mTestClassesOption: ArrayList<String?> = ArrayList()
93 
94     @Before
95     fun setUp() {
96         // APK will be auto-cleaned
97         installPackage(testApkFilename)
98     }
99 
100     @Test
101     fun testDeflake() {
102         val apkFile = InstallUtilsHost(this).getTestFile(testApkFilename)
103         val pkgName = AaptParser.parse(apkFile)?.packageName
104                 ?: fail("Could not parse test package name")
105 
106         val classes = mTestClassesOption.filterNotNull().ifEmpty { testClasses }
107                 .ifEmpty { listOf(null) } // null class name runs all classes in the package
108         val runOptions = DeviceTestRunOptions(pkgName)
109                 .setDevice(device)
110                 .setTestTimeoutMs(mSingleRunTimeoutMsOption ?: singleRunTimeoutMs)
111                 .setCheckResults(false)
112         // Pair is (test identifier, last stacktrace)
113         val failures = ArrayList<TestFailure>()
114         val count = mRunCountOption ?: runCount
115         repeat(count) {
116             classes.forEach { testClass ->
117                 runDeviceTests(runOptions.setTestClassName(testClass))
118                 failures += getLastRunFailures()
119             }
120         }
121         if (failures.isEmpty()) return
122         val failuresByTest = failures.groupBy(TestFailure::description)
123         val failMessage = failuresByTest.toList().fold("") { msg, (testDescription, failures) ->
124             val stacktraces = formatStacktraces(failures)
125             msg + "\n$testDescription: ${failures.count()}/$count failures. " +
126                     "Stacktraces:\n$stacktraces"
127         }
128         fail("Some tests failed:$failMessage")
129     }
130 
131     private fun getLastRunFailures(): List<TestFailure> {
132         with(lastDeviceRunResults) {
133             if (isRunFailure) {
134                 return listOf(TestFailure("All tests in run", runFailureMessage))
135             }
136 
137             return failedTests.map {
138                 val stackTrace = testResults[it]?.stackTrace
139                         ?: fail("Missing stacktrace for failed test $it")
140                 TestFailure(it.toString(), stackTrace)
141             }
142         }
143     }
144 
145     private fun formatStacktraces(failures: List<TestFailure>): String {
146         // Calculate list of (stacktrace, frequency) pairs ordered from most to least frequent
147         val frequencies = failures.groupingBy(TestFailure::stacktrace).eachCount().toList()
148                 .sortedByDescending { it.second }
149         // Print each stacktrace with its frequency
150         return frequencies.fold("") { msg, (stacktrace, numFailures) ->
151             "$msg\n$numFailures failures:\n$stacktrace"
152         }
153     }
154 }
155