1 /* 2 * Copyright (C) 2024 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.tools.flicker.rules 18 19 import android.platform.test.rule.TestWatcher 20 import android.tools.FLICKER_TAG 21 import android.tools.flicker.FlickerConfig 22 import android.tools.flicker.FlickerService 23 import android.tools.flicker.FlickerServiceResultsCollector 24 import android.tools.flicker.FlickerServiceTracesCollector 25 import android.tools.flicker.IFlickerServiceResultsCollector 26 import android.tools.flicker.annotation.FlickerTest 27 import android.tools.flicker.assertions.AssertionResult 28 import android.tools.flicker.config.FlickerConfig 29 import android.tools.flicker.config.FlickerServiceConfig 30 import android.tools.flicker.config.ScenarioId 31 import android.tools.traces.getDefaultFlickerOutputDir 32 import android.util.Log 33 import androidx.test.platform.app.InstrumentationRegistry 34 import com.google.common.truth.Truth 35 import org.junit.AssumptionViolatedException 36 import org.junit.runner.Description 37 import org.junit.runner.notification.Failure 38 39 /** 40 * A test rule that runs Flicker as a Service on the tests this rule is applied to. 41 * 42 * Note there are performance implications to using this test rule in tests. Tracing will be enabled 43 * during the test which will slow down everything. So if the test is performance critical then an 44 * alternative should be used. 45 * 46 * @see TODO for examples on how to use this test rule in your own tests 47 */ 48 open class FlickerServiceRule 49 @JvmOverloads 50 constructor( 51 enabled: Boolean = true, 52 failTestOnFlicker: Boolean = enabled, 53 failTestOnServiceError: Boolean = false, 54 config: FlickerConfig = FlickerConfig().use(FlickerServiceConfig.DEFAULT), 55 private val metricsCollector: IFlickerServiceResultsCollector = 56 FlickerServiceResultsCollector( 57 flickerService = FlickerService(config), 58 tracesCollector = FlickerServiceTracesCollector(getDefaultFlickerOutputDir()), 59 instrumentation = InstrumentationRegistry.getInstrumentation(), 60 ), 61 ) : TestWatcher() { 62 private val enabled: Boolean = <lambda>null63 InstrumentationRegistry.getArguments().getString("faas:enabled")?.let { it.toBoolean() } 64 ?: enabled 65 66 private val failTestOnFlicker: Boolean = <lambda>null67 InstrumentationRegistry.getArguments().getString("faas:failTestOnFlicker")?.let { 68 it.toBoolean() 69 } ?: failTestOnFlicker 70 71 private val failTestOnServiceError: Boolean = <lambda>null72 InstrumentationRegistry.getArguments().getString("faas:failTestOnServiceError")?.let { 73 it.toBoolean() 74 } ?: failTestOnServiceError 75 76 private var testFailed = false 77 private var testSkipped = false 78 79 /** Invoked when a test is about to start */ startingnull80 public override fun starting(description: Description) { 81 if (shouldRun(description)) { 82 handleStarting(description) 83 } 84 } 85 86 /** Invoked when a test succeeds */ succeedednull87 public override fun succeeded(description: Description) { 88 if (shouldRun(description)) { 89 handleSucceeded(description) 90 } 91 } 92 93 /** Invoked when a test fails */ failednull94 public override fun failed(e: Throwable?, description: Description) { 95 if (shouldRun(description)) { 96 handleFailed(e, description) 97 } 98 } 99 100 /** Invoked when a test is skipped due to a failed assumption. */ skippednull101 public override fun skipped(e: AssumptionViolatedException, description: Description) { 102 if (shouldRun(description)) { 103 handleSkipped(e, description) 104 } 105 } 106 107 /** Invoked when a test method finishes (whether passing or failing) */ finishednull108 public override fun finished(description: Description) { 109 if (shouldRun(description)) { 110 handleFinished(description) 111 } 112 } 113 handleStartingnull114 private fun handleStarting(description: Description) { 115 Log.i(LOG_TAG, "Test starting $description") 116 metricsCollector.testStarted(description) 117 testFailed = false 118 testSkipped = false 119 } 120 handleSucceedednull121 private fun handleSucceeded(description: Description) { 122 Log.i(LOG_TAG, "Test succeeded $description") 123 } 124 handleFailednull125 private fun handleFailed(e: Throwable?, description: Description) { 126 Log.e(LOG_TAG, "$description test failed with", e) 127 metricsCollector.testFailure(Failure(description, e)) 128 testFailed = true 129 } 130 handleSkippednull131 private fun handleSkipped(e: AssumptionViolatedException, description: Description) { 132 Log.i(LOG_TAG, "Test skipped $description with", e) 133 metricsCollector.testSkipped(description) 134 testSkipped = true 135 } 136 shouldRunnull137 private fun shouldRun(description: Description): Boolean { 138 // Only run FaaS if test rule is enabled and on tests with FlickerTest annotation if it's 139 // used within the class, otherwise run on all tests 140 if (!enabled) { 141 return false 142 } 143 144 if (description.annotations.none { it is FlickerTest }) { 145 // FlickerTest annotation is not used within the test class, so run on all tests 146 return true 147 } 148 149 return testClassHasFlickerTestAnnotations(description.testClass) 150 } 151 testClassHasFlickerTestAnnotationsnull152 private fun testClassHasFlickerTestAnnotations(testClass: Class<*>): Boolean { 153 return testClass.methods.flatMap { it.annotations.asList() }.any { it is FlickerTest } 154 } 155 handleFinishednull156 private fun handleFinished(description: Description) { 157 Log.i(LOG_TAG, "Test finished $description") 158 metricsCollector.testFinished(description) 159 for (executionError in metricsCollector.executionErrors) { 160 Log.e(LOG_TAG, "FaaS reported execution errors", executionError) 161 } 162 163 if (failTestOnServiceError && testContainsServiceError()) { 164 throw metricsCollector.executionErrors.first() 165 } 166 167 if (testSkipped || testFailed || metricsCollector.executionErrors.isNotEmpty()) { 168 // If we had an execution error or the underlying test failed or was skipped, then we 169 // have no guarantees about the correctness of the flicker assertions and detect 170 // scenarios, so we should not check those and instead return immediately. 171 return 172 } 173 174 val failedMetrics = 175 metricsCollector.resultsForTest(description).filter { 176 it.status == AssertionResult.Status.FAIL 177 } 178 val assertionErrors = failedMetrics.flatMap { it.assertionErrors } 179 assertionErrors.forEach { 180 Log.e(LOG_TAG, "FaaS reported an assertion failure:") 181 Log.e(LOG_TAG, it.message) 182 Log.e(LOG_TAG, it.stackTraceToString()) 183 } 184 185 if (failTestOnFlicker && testContainsFlicker(description)) { 186 throw assertionErrors.firstOrNull() ?: error("Unexpectedly missing assertion error") 187 } 188 189 val flickerTestAnnotation: FlickerTest? = 190 description.annotations.filterIsInstance<FlickerTest>().firstOrNull() 191 if (failTestOnFlicker && flickerTestAnnotation != null) { 192 val detectedScenarios = metricsCollector.detectedScenariosForTest(description) 193 Truth.assertThat(detectedScenarios) 194 .containsAtLeastElementsIn(flickerTestAnnotation.expected.map { ScenarioId(it) }) 195 } 196 } 197 testContainsFlickernull198 private fun testContainsFlicker(description: Description): Boolean { 199 val resultsForTest = metricsCollector.resultsForTest(description) 200 return resultsForTest.any { it.status == AssertionResult.Status.FAIL } 201 } 202 testContainsServiceErrornull203 private fun testContainsServiceError(): Boolean { 204 return metricsCollector.executionErrors.isNotEmpty() 205 } 206 207 companion object { 208 const val LOG_TAG = "$FLICKER_TAG-ServiceRule" 209 } 210 } 211