1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tests.apex; 18 19 import static com.google.common.truth.Truth.assertThat; 20 import static com.google.common.truth.Truth.assertWithMessage; 21 22 import static org.junit.Assume.assumeFalse; 23 import static org.junit.Assume.assumeTrue; 24 25 import android.cts.install.lib.host.InstallUtilsHost; 26 27 import com.android.tests.rollback.host.AbandonSessionsRule; 28 import com.android.tradefed.device.DeviceNotAvailableException; 29 import com.android.tradefed.device.ITestDevice; 30 import com.android.tradefed.device.ITestDevice.ApexInfo; 31 import com.android.tradefed.testtype.DeviceJUnit4ClassRunner; 32 import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test; 33 34 import org.junit.After; 35 import org.junit.Before; 36 import org.junit.Rule; 37 import org.junit.Test; 38 import org.junit.runner.RunWith; 39 40 import java.io.File; 41 import java.time.Duration; 42 import java.util.Set; 43 44 /** 45 * Test for automatic recovery of apex update that causes boot loop. 46 */ 47 @RunWith(DeviceJUnit4ClassRunner.class) 48 public class ApexRollbackTests extends BaseHostJUnit4Test { 49 private final InstallUtilsHost mHostUtils = new InstallUtilsHost(this); 50 @Rule 51 public AbandonSessionsRule mHostTestRule = new AbandonSessionsRule(this); 52 53 private boolean mWasAdbRoot = false; 54 55 private static boolean sCheckedIfCrashingProcessExists = false; 56 private static boolean sAlreadyCrashingProcessExists = false; 57 private static String sCrashingProcess; 58 59 @Before setUp()60 public void setUp() throws Exception { 61 mHostUtils.uninstallShimApexIfNecessary(); 62 resetProperties(); 63 mWasAdbRoot = getDevice().isAdbRoot(); 64 if (!mWasAdbRoot) { 65 assumeTrue("Requires root", getDevice().enableAdbRoot()); 66 } 67 if (!sCheckedIfCrashingProcessExists) { 68 sAlreadyCrashingProcessExists = 69 getDevice().getBooleanProperty("sys.init.updatable_crashing", false); 70 sCrashingProcess = getDevice().getProperty("sys.init.updatable_crashing_process_name"); 71 sCheckedIfCrashingProcessExists = true; 72 } 73 } 74 75 /** 76 * Uninstalls any version greater than 1 of shim apex and reboots the device if necessary 77 * to complete the uninstall. 78 */ 79 @After tearDown()80 public void tearDown() throws Exception { 81 mHostUtils.uninstallShimApexIfNecessary(); 82 resetProperties(); 83 if (!mWasAdbRoot) { 84 getDevice().disableAdbRoot(); 85 } 86 } 87 resetProperties()88 private void resetProperties() throws Exception { 89 resetProperty("persist.debug.trigger_watchdog.apex"); 90 resetProperty("persist.debug.trigger_updatable_crashing_for_testing"); 91 resetProperty("persist.debug.trigger_reboot_after_activation"); 92 resetProperty("persist.debug.trigger_reboot_twice_after_activation"); 93 } 94 resetProperty(String propertyName)95 private void resetProperty(String propertyName) throws Exception { 96 assertWithMessage("Failed to reset value of property %s", propertyName).that( 97 getDevice().setProperty(propertyName, "")).isTrue(); 98 } 99 100 /** 101 * Test for automatic recovery of apex update that causes boot loop. 102 */ 103 @Test testAutomaticBootLoopRecovery()104 public void testAutomaticBootLoopRecovery() throws Exception { 105 assumeTrue("Device does not support updating APEX", mHostUtils.isApexUpdateSupported()); 106 ITestDevice device = getDevice(); 107 // Skip this test if there is already crashing process on device 108 assumeFalse( 109 "Device already has a crashing process: " + sCrashingProcess, 110 sAlreadyCrashingProcessExists); 111 File apexFile = mHostUtils.getTestFile("com.android.apex.cts.shim.v2.apex"); 112 113 // To simulate an apex update that causes a boot loop, we install a 114 // trigger_watchdog.rc file that arranges for a trigger_watchdog.sh 115 // script to be run at boot. The trigger_watchdog.sh script checks if 116 // the apex version specified in the property 117 // persist.debug.trigger_watchdog.apex is installed. If so, 118 // trigger_watchdog.sh repeatedly kills the system server causing a 119 // boot loop. 120 assertThat(device.setProperty("persist.debug.trigger_watchdog.apex", 121 "com.android.apex.cts.shim@2")).isTrue(); 122 String error = mHostUtils.installStagedPackage(apexFile); 123 assertThat(error).isNull(); 124 125 String sessionIdToCheck = device.executeShellCommand("pm get-stagedsessions --only-ready " 126 + "--only-parent --only-sessionid").trim(); 127 assertThat(sessionIdToCheck).isNotEmpty(); 128 129 // After we reboot the device, we expect the device to go into boot 130 // loop from trigger_watchdog.sh. Native watchdog should detect and 131 // report the boot loop, causing apexd to roll back to the previous 132 // version of the apex and force reboot. When the device comes up 133 // after the forced reboot, trigger_watchdog.sh will see the different 134 // version of the apex and refrain from forcing a boot loop, so the 135 // device will be recovered. 136 device.reboot(); 137 138 ApexInfo ctsShimV1 = new ApexInfo("com.android.apex.cts.shim", 1L); 139 ApexInfo ctsShimV2 = new ApexInfo("com.android.apex.cts.shim", 2L); 140 Set<ApexInfo> activatedApexes = device.getActiveApexes(); 141 assertThat(activatedApexes).contains(ctsShimV1); 142 assertThat(activatedApexes).doesNotContain(ctsShimV2); 143 144 // Assert that a session has failed with the expected reason 145 String sessionInfo = device.executeShellCommand("cmd -w apexservice getStagedSessionInfo " 146 + sessionIdToCheck); 147 assertThat(sessionInfo).contains("revertReason: zygote"); 148 } 149 150 /** 151 * Test to verify that a device that does not support checkpointing will not revert a session 152 * if it reboots during boot. 153 */ 154 @Test testSessionNotRevertedWithCheckpointingDisabled()155 public void testSessionNotRevertedWithCheckpointingDisabled() throws Exception { 156 assumeTrue("Device does not support updating APEX", mHostUtils.isApexUpdateSupported()); 157 assumeFalse("Fs checkpointing is enabled", mHostUtils.isCheckpointSupported()); 158 159 File apexFile = mHostUtils.getTestFile("com.android.apex.cts.shim.v2.apex"); 160 161 ITestDevice device = getDevice(); 162 assertThat(device.setProperty("persist.debug.trigger_reboot_after_activation", 163 "[email protected]")).isTrue(); 164 assertThat(device.setProperty("debug.trigger_reboot_once_after_activation", 165 "1")).isTrue(); 166 167 String error = mHostUtils.installStagedPackage(apexFile); 168 assertThat(error).isNull(); 169 170 String sessionIdToCheck = device.executeShellCommand("pm get-stagedsessions --only-ready " 171 + "--only-parent --only-sessionid").trim(); 172 assertThat(sessionIdToCheck).isNotEmpty(); 173 174 // After we reboot the device, the apexd session should be activated as normal. After this, 175 // trigger_reboot.sh will reboot the device before the system server boots. 176 device.reboot(); 177 178 ApexInfo ctsShimV1 = new ApexInfo("com.android.apex.cts.shim", 1L); 179 ApexInfo ctsShimV2 = new ApexInfo("com.android.apex.cts.shim", 2L); 180 String stagedSessionInfo = getStagedSession(sessionIdToCheck); 181 assertThat(stagedSessionInfo).contains("isApplied = true"); 182 183 Set<ApexInfo> activatedApexes = device.getActiveApexes(); 184 assertThat(activatedApexes).contains(ctsShimV2); 185 assertThat(activatedApexes).doesNotContain(ctsShimV1); 186 } 187 188 /** 189 * Test to verify that rebooting twice when a session is activated will cause the session to 190 * be reverted due to filesystem checkpointing. 191 */ 192 @Test testCheckpointingRevertsSession()193 public void testCheckpointingRevertsSession() throws Exception { 194 assumeTrue("Device does not support updating APEX", mHostUtils.isApexUpdateSupported()); 195 assumeTrue("Device doesn't support fs checkpointing", mHostUtils.isCheckpointSupported()); 196 197 File apexFile = mHostUtils.getTestFile("com.android.apex.cts.shim.v2.apex"); 198 199 ITestDevice device = getDevice(); 200 assertThat(device.setProperty("persist.debug.trigger_reboot_after_activation", 201 "[email protected]")).isTrue(); 202 assertThat(device.setProperty("persist.debug.trigger_reboot_twice_after_activation", 203 "1")).isTrue(); 204 String error = mHostUtils.installStagedPackage(apexFile); 205 assertThat(error).isNull(); 206 207 String sessionIdToCheck = device.executeShellCommand("pm get-stagedsessions --only-ready " 208 + "--only-parent --only-sessionid").trim(); 209 assertThat(sessionIdToCheck).isNotEmpty(); 210 211 // After we reboot the device, the apexd session should be activated as normal. After this, 212 // trigger_reboot.sh will reboot the device before the system server boots. Checkpointing 213 // will kick in, and at the next boot any non-finalized sessions will be reverted. 214 device.reboot(); 215 216 ApexInfo ctsShimV1 = new ApexInfo("com.android.apex.cts.shim", 1L); 217 ApexInfo ctsShimV2 = new ApexInfo("com.android.apex.cts.shim", 2L); 218 String stagedSessionInfo = getStagedSession(sessionIdToCheck); 219 assertThat(stagedSessionInfo).contains("isFailed = true"); 220 221 Set<ApexInfo> activatedApexes = device.getActiveApexes(); 222 assertThat(activatedApexes).contains(ctsShimV1); 223 assertThat(activatedApexes).doesNotContain(ctsShimV2); 224 } 225 226 /** 227 * Test to verify that rebooting once upon apex activation does not cause checkpointing to kick 228 * in and revert a session, since the checkpointing retry count should be 2. 229 */ 230 @Test testRebootingOnceDoesNotRevertSession()231 public void testRebootingOnceDoesNotRevertSession() throws Exception { 232 assumeTrue("Device does not support updating APEX", mHostUtils.isApexUpdateSupported()); 233 assumeTrue("Device doesn't support fs checkpointing", mHostUtils.isCheckpointSupported()); 234 235 File apexFile = mHostUtils.getTestFile("com.android.apex.cts.shim.v2.apex"); 236 237 ITestDevice device = getDevice(); 238 assertThat(device.setProperty("persist.debug.trigger_reboot_after_activation", 239 "[email protected]")).isTrue(); 240 assertThat(device.setProperty("debug.trigger_reboot_once_after_activation", 241 "1")).isTrue(); 242 String error = mHostUtils.installStagedPackage(apexFile); 243 assertThat(error).isNull(); 244 245 String sessionIdToCheck = device.executeShellCommand("pm get-stagedsessions --only-ready " 246 + "--only-parent --only-sessionid").trim(); 247 assertThat(sessionIdToCheck).isNotEmpty(); 248 249 // After we reboot the device, the apexd session should be activated as normal. After this, 250 // trigger_reboot.sh will reboot the device before the system server boots. Checkpointing 251 // will kick in, and at the next boot any non-finalized sessions will be reverted. 252 device.reboot(); 253 254 ApexInfo ctsShimV1 = new ApexInfo("com.android.apex.cts.shim", 1L); 255 ApexInfo ctsShimV2 = new ApexInfo("com.android.apex.cts.shim", 2L); 256 String stagedSessionInfo = getStagedSession(sessionIdToCheck); 257 assertThat(stagedSessionInfo).contains("isApplied = true"); 258 259 Set<ApexInfo> activatedApexes = device.getActiveApexes(); 260 assertThat(activatedApexes).contains(ctsShimV2); 261 assertThat(activatedApexes).doesNotContain(ctsShimV1); 262 } 263 264 /** 265 * Test to verify that apexd won't boot loop a device in case {@code sys.init 266 * .updatable_crashing} is {@code true} and there is no apex session to revert. 267 */ 268 @Test testApexdDoesNotBootLoopDeviceIfThereIsNothingToRevert()269 public void testApexdDoesNotBootLoopDeviceIfThereIsNothingToRevert() throws Exception { 270 assumeTrue("Device does not support updating APEX", mHostUtils.isApexUpdateSupported()); 271 // On next boot trigger setprop sys.init.updatable_crashing 1, which will trigger a 272 // revert mechanism in apexd. Since there is nothing to revert, this should be a no-op 273 // and device will boot successfully. 274 assertThat(getDevice().setProperty("persist.debug.trigger_updatable_crashing_for_testing", 275 "1")).isTrue(); 276 getDevice().reboot(); 277 assertWithMessage("Device didn't boot in 1 minute").that( 278 getDevice().waitForBootComplete(Duration.ofMinutes(1).toMillis())).isTrue(); 279 // Verify that property was set to true. 280 assertThat(getDevice().getBooleanProperty("sys.init.updatable_crashing", false)).isTrue(); 281 } 282 283 /** 284 * Test to verify that boot cleanup logic in apexd is triggered when there is a crash looping 285 * process, but there is nothing to revert. 286 */ 287 @Test testBootCompletedCleanupHappensEvenWhenThereIsCrashingProcess()288 public void testBootCompletedCleanupHappensEvenWhenThereIsCrashingProcess() throws Exception { 289 assumeTrue("Device does not support updating APEX", mHostUtils.isApexUpdateSupported()); 290 assumeTrue("Device requires root", getDevice().isAdbRoot()); 291 try { 292 // On next boot trigger setprop sys.init.updatable_crashing 1, which will trigger a 293 // revert mechanism in apexd. Since there is nothing to revert, this should be a no-op 294 // and device will boot successfully. 295 getDevice().setProperty("persist.debug.trigger_updatable_crashing_for_testing", "1"); 296 assertThat(getDevice().pushFile(mHostUtils.getTestFile("apex.apexd_test_v2.apex"), 297 "/data/apex/active/apexd_test_v2.apex")).isTrue(); 298 getDevice().reboot(); 299 assertWithMessage("Timed out waiting for device to boot").that( 300 getDevice().waitForBootComplete(Duration.ofMinutes(2).toMillis())).isTrue(); 301 // Verify that property was set to true. 302 assertThat( 303 getDevice().getBooleanProperty("sys.init.updatable_crashing", false)).isTrue(); 304 final Set<ITestDevice.ApexInfo> activeApexes = getDevice().getActiveApexes(); 305 ITestDevice.ApexInfo testApex = new ITestDevice.ApexInfo( 306 "com.android.apex.cts.shim", 2L); 307 assertThat(activeApexes).doesNotContain(testApex); 308 mHostUtils.waitForFileDeleted("/data/apex/active/apexd_test_v2.apex", 309 Duration.ofMinutes(3)); 310 } finally { 311 getDevice().executeShellV2Command("rm /data/apex/active/apexd_test_v2.apex"); 312 } 313 } 314 315 /** 316 * Test reason for revert is properly logged during boot loops 317 */ 318 @Test testReasonForRevertIsLoggedDuringBootloop()319 public void testReasonForRevertIsLoggedDuringBootloop() throws Exception { 320 assumeTrue("Device does not support updating APEX", mHostUtils.isApexUpdateSupported()); 321 assumeTrue("Fs checkpointing is enabled", mHostUtils.isCheckpointSupported()); 322 323 ITestDevice device = getDevice(); 324 assumeFalse( 325 "Device already has a crashing process: " + sCrashingProcess, 326 sAlreadyCrashingProcessExists); 327 final File apexFile = mHostUtils.getTestFile("com.android.apex.cts.shim.v2.apex"); 328 329 // To simulate an apex update that causes a boot loop, we install a 330 // trigger_watchdog.rc file that arranges for a trigger_watchdog.sh 331 // script to be run at boot. The trigger_watchdog.sh script checks if 332 // the apex version specified in the property 333 // persist.debug.trigger_watchdog.apex is installed. If so, 334 // trigger_watchdog.sh repeatedly kills the system server causing a 335 // boot loop. 336 assertThat(device.setProperty("persist.debug.trigger_watchdog.apex", 337 "com.android.apex.cts.shim@2")).isTrue(); 338 final String error = mHostUtils.installStagedPackage(apexFile); 339 assertThat(error).isNull(); 340 341 final String sessionIdToCheck = device.executeShellCommand("pm get-stagedsessions " 342 + "--only-ready --only-parent --only-sessionid").trim(); 343 assertThat(sessionIdToCheck).isNotEmpty(); 344 345 // After we reboot the device, we expect the device to go into boot 346 // loop from trigger_watchdog.sh. Native watchdog should detect and 347 // report the boot loop, causing apexd to roll back to the previous 348 // version of the apex and force reboot. When the device comes up 349 // after the forced reboot, trigger_watchdog.sh will see the different 350 // version of the apex and refrain from forcing a boot loop, so the 351 // device will be recovered. 352 device.reboot(); 353 354 final ApexInfo ctsShimV1 = new ApexInfo("com.android.apex.cts.shim", 1L); 355 final ApexInfo ctsShimV2 = new ApexInfo("com.android.apex.cts.shim", 2L); 356 final Set<ApexInfo> activatedApexes = device.getActiveApexes(); 357 assertThat(activatedApexes).contains(ctsShimV1); 358 assertThat(activatedApexes).doesNotContain(ctsShimV2); 359 360 // Assert that a session has failed with the expected reason 361 final String stagedSessionString = getStagedSession(sessionIdToCheck); 362 assertThat(stagedSessionString).contains("Session reverted due to crashing native process"); 363 } 364 getStagedSession(String sessionId)365 String getStagedSession(String sessionId) throws DeviceNotAvailableException { 366 final String[] lines = getDevice().executeShellCommand( 367 "pm get-stagedsessions").split("\n"); 368 for (int i = 0; i < lines.length; i++) { 369 if (lines[i].startsWith("sessionId = " + sessionId + ";")) { 370 // Join all lines realted to this session 371 final StringBuilder result = new StringBuilder(lines[i]); 372 for (int j = i + 1; j < lines.length; j++) { 373 if (lines[j].startsWith("sessionId = ")) { 374 // A new session block has started 375 break; 376 } 377 result.append(lines[j]); 378 } 379 return result.toString(); 380 } 381 } 382 return ""; 383 } 384 } 385