1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.testutils
18 
19 import android.content.Context
20 import androidx.test.core.app.ApplicationProvider
21 import androidx.test.ext.junit.runners.AndroidJUnit4
22 import com.android.net.module.util.LinkPropertiesUtils.CompareOrUpdateResult
23 import com.android.testutils.DevSdkIgnoreRule.IgnoreAfter
24 import com.android.testutils.DevSdkIgnoreRule.IgnoreUpTo
25 import java.lang.reflect.Modifier
26 import org.junit.runner.Description
27 import org.junit.runner.Runner
28 import org.junit.runner.manipulation.Filter
29 import org.junit.runner.manipulation.Filterable
30 import org.junit.runner.manipulation.NoTestsRemainException
31 import org.junit.runner.manipulation.Sortable
32 import org.junit.runner.manipulation.Sorter
33 import org.junit.runner.notification.Failure
34 import org.junit.runner.notification.RunNotifier
35 import org.junit.runners.Parameterized
36 import org.mockito.Mockito
37 
38 /**
39  * A runner that can skip tests based on the development SDK as defined in [DevSdkIgnoreRule].
40  *
41  * Generally [DevSdkIgnoreRule] should be used for that purpose (using rules is preferable over
42  * replacing the test runner), however JUnit runners inspect all methods in the test class before
43  * processing test rules. This may cause issues if the test methods are referencing classes that do
44  * not exist on the SDK of the device the test is run on.
45  *
46  * This runner inspects [IgnoreAfter] and [IgnoreUpTo] annotations on the test class, and will skip
47  * the whole class if they do not match the development SDK as defined in [DevSdkIgnoreRule].
48  * Otherwise, it will delegate to [AndroidJUnit4] to run the test as usual.
49  *
50  * This class automatically uses the Parameterized runner as its base runner when needed, so the
51  * @Parameterized.Parameters annotation and its friends can be used in tests using this runner.
52  *
53  * Example usage:
54  *
55  *     @RunWith(DevSdkIgnoreRunner::class)
56  *     @IgnoreUpTo(Build.VERSION_CODES.Q)
57  *     class MyTestClass { ... }
58  */
59 class DevSdkIgnoreRunner(private val klass: Class<*>) : Runner(), Filterable, Sortable {
60     private val leakMonitorDesc = Description.createTestDescription(klass, "ThreadLeakMonitor")
61     private val shouldThreadLeakFailTest = klass.isAnnotationPresent(MonitorThreadLeak::class.java)
62     private val restoreDefaultNetworkDesc =
63             Description.createTestDescription(klass, "RestoreDefaultNetwork")
64     val ctx = ApplicationProvider.getApplicationContext<Context>()
65     private val restoreDefaultNetwork =
66             klass.isAnnotationPresent(RestoreDefaultNetwork::class.java) &&
67             !ctx.applicationInfo.isInstantApp()
68 
69     // Inference correctly infers Runner & Filterable & Sortable for |baseRunner|, but the
70     // Java bytecode doesn't have a way to express this. Give this type a name by wrapping it.
71     private class RunnerWrapper<T>(private val wrapped: T) :
72             Runner(), Filterable by wrapped, Sortable by wrapped
73             where T : Runner, T : Filterable, T : Sortable {
getDescriptionnull74         override fun getDescription(): Description = wrapped.description
75         override fun run(notifier: RunNotifier?) = wrapped.run(notifier)
76     }
77 
78     // Annotation for test classes to indicate the test runner should monitor thread leak.
79     // TODO(b/307693729): Remove this annotation and monitor thread leak by default.
80     annotation class MonitorThreadLeak
81 
82     // Annotation for test classes to indicate the test runner should verify the default network is
83     // restored after each test.
84     annotation class RestoreDefaultNetwork
85 
86     private val baseRunner: RunnerWrapper<*>? = klass.let {
87         val ignoreAfter = it.getAnnotation(IgnoreAfter::class.java)
88         val ignoreUpTo = it.getAnnotation(IgnoreUpTo::class.java)
89 
90         if (!isDevSdkInRange(ignoreUpTo, ignoreAfter)) {
91             null
92         } else if (it.hasParameterizedMethod()) {
93             // Parameterized throws if there is no static method annotated with @Parameters, which
94             // isn't too useful. Use it if there are, otherwise use its base AndroidJUnit4 runner.
95             RunnerWrapper(Parameterized(klass))
96         } else {
97             RunnerWrapper(AndroidJUnit4(klass))
98         }
99     }
100 
<lambda>null101     private fun <T> Class<T>.hasParameterizedMethod(): Boolean = methods.any {
102         Modifier.isStatic(it.modifiers) &&
103                 it.isAnnotationPresent(Parameterized.Parameters::class.java) }
104 
checkThreadLeaknull105     private fun checkThreadLeak(
106             notifier: RunNotifier,
107             threadCountsBeforeTest: Map<String, Int>
108     ) {
109         notifier.fireTestStarted(leakMonitorDesc)
110         val threadCountsAfterTest = getAllThreadNameCounts()
111         // TODO : move CompareOrUpdateResult to its own util instead of LinkProperties.
112         val threadsDiff = CompareOrUpdateResult(
113                 threadCountsBeforeTest.entries,
114                 threadCountsAfterTest.entries
115         ) { it.key }
116         // Ignore removed threads, which typically are generated by previous tests.
117         // Because this is in the threadsDiff.updated member, for sure there is a
118         // corresponding key in threadCountsBeforeTest.
119         val increasedThreads = threadsDiff.updated
120                 .filter { threadCountsBeforeTest[it.key]!! < it.value }
121         if (threadsDiff.added.isNotEmpty() || increasedThreads.isNotEmpty()) {
122             notifier.fireTestFailure(Failure(
123                     leakMonitorDesc,
124                     IllegalStateException("Unexpected thread changes: $threadsDiff")
125             ))
126         }
127         notifier.fireTestFinished(leakMonitorDesc)
128     }
129 
runnull130     override fun run(notifier: RunNotifier) {
131         if (baseRunner == null) {
132             // Report a single, skipped placeholder test for this class, as the class is expected to
133             // report results when run. In practice runners that apply the Filterable implementation
134             // would see a NoTestsRemainException and not call the run method.
135             notifier.fireTestIgnored(
136                     Description.createTestDescription(klass, "skippedClassForDevSdkMismatch")
137             )
138             return
139         }
140 
141         val networkRestoreMonitor = if (restoreDefaultNetwork) {
142             DefaultNetworkRestoreMonitor(ctx, notifier).apply{
143                 init(ConnectUtil(ctx))
144             }
145         } else {
146             null
147         }
148         val threadCountsBeforeTest = if (shouldThreadLeakFailTest) {
149             // Dump threads as a baseline to monitor thread leaks.
150             getAllThreadNameCounts()
151         } else {
152             null
153         }
154 
155         baseRunner.run(notifier)
156 
157         if (threadCountsBeforeTest != null) {
158             checkThreadLeak(notifier, threadCountsBeforeTest)
159         }
160         networkRestoreMonitor?.reportResultAndCleanUp(restoreDefaultNetworkDesc)
161         // Clears up internal state of all inline mocks.
162         // TODO: Call clearInlineMocks() at the end of each test.
163         Mockito.framework().clearInlineMocks()
164     }
165 
getAllThreadNameCountsnull166     private fun getAllThreadNameCounts(): Map<String, Int> {
167         // Get the counts of threads in the group per name.
168         // Filter system thread groups.
169         // Also ignore threads with 1 count, this effectively filtered out threads created by the
170         // test runner or other system components. e.g. hwuiTask*, queued-work-looper,
171         // SurfaceSyncGroupTimer, RenderThread, Time-limited test, etc.
172         return Thread.getAllStackTraces().keys
173                 .filter { it.threadGroup?.name != "system" }
174                 .groupingBy { it.name }.eachCount()
175                 .filter { it.value != 1 }
176     }
177 
getDescriptionnull178     override fun getDescription(): Description {
179         if (baseRunner == null) {
180             return Description.createSuiteDescription(klass)
181         }
182 
183         return baseRunner.description.also {
184             if (shouldThreadLeakFailTest) {
185                 it.addChild(leakMonitorDesc)
186             }
187             if (restoreDefaultNetwork) {
188                 it.addChild(restoreDefaultNetworkDesc)
189             }
190         }
191     }
192 
193     /**
194      * Get the test count before applying the [Filterable] implementation.
195      */
testCountnull196     override fun testCount(): Int {
197         // When ignoring the tests, a skipped placeholder test is reported, so test count is 1.
198         if (baseRunner == null) return 1
199 
200         var testCount = baseRunner.testCount()
201         if (shouldThreadLeakFailTest) {
202             testCount += 1
203         }
204         if (restoreDefaultNetwork) {
205             testCount += 1
206         }
207         return testCount
208     }
209 
210     @Throws(NoTestsRemainException::class)
filternull211     override fun filter(filter: Filter?) {
212         baseRunner?.filter(filter) ?: throw NoTestsRemainException()
213     }
214 
sortnull215     override fun sort(sorter: Sorter?) {
216         baseRunner?.sort(sorter)
217     }
218 }
219