1# Copyright 2024, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Tests for build_test_suites.py""" 16 17import argparse 18import functools 19from importlib import resources 20import json 21import multiprocessing 22import os 23import pathlib 24import shutil 25import signal 26import stat 27import subprocess 28import sys 29import tempfile 30import textwrap 31import time 32from typing import Callable 33import unittest 34from unittest import mock 35from build_context import BuildContext 36import build_test_suites 37import ci_test_lib 38import optimized_targets 39from pyfakefs import fake_filesystem_unittest 40import metrics_agent 41import test_discovery_agent 42 43 44class BuildTestSuitesTest(fake_filesystem_unittest.TestCase): 45 46 def setUp(self): 47 self.setUpPyfakefs() 48 49 os_environ_patcher = mock.patch.dict('os.environ', {}) 50 self.addCleanup(os_environ_patcher.stop) 51 self.mock_os_environ = os_environ_patcher.start() 52 53 subprocess_run_patcher = mock.patch('subprocess.run') 54 self.addCleanup(subprocess_run_patcher.stop) 55 self.mock_subprocess_run = subprocess_run_patcher.start() 56 57 metrics_agent_finalize_patcher = mock.patch('metrics_agent.MetricsAgent.end_reporting') 58 self.addCleanup(metrics_agent_finalize_patcher.stop) 59 self.mock_metrics_agent_end = metrics_agent_finalize_patcher.start() 60 61 self._setup_working_build_env() 62 63 def test_missing_target_release_env_var_raises(self): 64 del os.environ['TARGET_RELEASE'] 65 66 with self.assert_raises_word(build_test_suites.Error, 'TARGET_RELEASE'): 67 build_test_suites.main([]) 68 69 def test_missing_target_product_env_var_raises(self): 70 del os.environ['TARGET_PRODUCT'] 71 72 with self.assert_raises_word(build_test_suites.Error, 'TARGET_PRODUCT'): 73 build_test_suites.main([]) 74 75 def test_missing_top_env_var_raises(self): 76 del os.environ['TOP'] 77 78 with self.assert_raises_word(build_test_suites.Error, 'TOP'): 79 build_test_suites.main([]) 80 81 def test_missing_dist_dir_env_var_raises(self): 82 del os.environ['DIST_DIR'] 83 84 with self.assert_raises_word(build_test_suites.Error, 'DIST_DIR'): 85 build_test_suites.main([]) 86 87 def test_invalid_arg_raises(self): 88 invalid_args = ['--invalid_arg'] 89 90 with self.assertRaisesRegex(SystemExit, '2'): 91 build_test_suites.main(invalid_args) 92 93 def test_build_failure_returns(self): 94 self.mock_subprocess_run.side_effect = subprocess.CalledProcessError( 95 42, None 96 ) 97 98 with self.assertRaisesRegex(SystemExit, '42'): 99 build_test_suites.main([]) 100 101 def test_incorrectly_formatted_build_context_raises(self): 102 build_context = self.fake_top.joinpath('build_context') 103 build_context.touch() 104 os.environ['BUILD_CONTEXT'] = str(build_context) 105 106 with self.assert_raises_word(build_test_suites.Error, 'JSON'): 107 build_test_suites.main([]) 108 109 def test_build_success_returns(self): 110 with self.assertRaisesRegex(SystemExit, '0'): 111 build_test_suites.main([]) 112 113 def assert_raises_word(self, cls, word): 114 return self.assertRaisesRegex(cls, rf'\b{word}\b') 115 116 def _setup_working_build_env(self): 117 self.fake_top = pathlib.Path('/fake/top') 118 self.fake_top.mkdir(parents=True) 119 120 self.soong_ui_dir = self.fake_top.joinpath('build/soong') 121 self.soong_ui_dir.mkdir(parents=True, exist_ok=True) 122 123 self.logs_dir = self.fake_top.joinpath('dist/logs') 124 self.logs_dir.mkdir(parents=True, exist_ok=True) 125 126 self.soong_ui = self.soong_ui_dir.joinpath('soong_ui.bash') 127 self.soong_ui.touch() 128 129 self.mock_os_environ.update({ 130 'TARGET_RELEASE': 'release', 131 'TARGET_PRODUCT': 'product', 132 'TOP': str(self.fake_top), 133 'DIST_DIR': str(self.fake_top.joinpath('dist')), 134 }) 135 136 self.mock_subprocess_run.return_value = 0 137 138 139class RunCommandIntegrationTest(ci_test_lib.TestCase): 140 141 def setUp(self): 142 self.temp_dir = ci_test_lib.TestTemporaryDirectory.create(self) 143 144 # Copy the Python executable from 'non-code' resources and make it 145 # executable for use by tests that launch a subprocess. Note that we don't 146 # use Python's native `sys.executable` property since that is not set when 147 # running via the embedded launcher. 148 base_name = 'py3-cmd' 149 dest_file = self.temp_dir.joinpath(base_name) 150 with resources.as_file( 151 resources.files('testdata').joinpath(base_name) 152 ) as p: 153 shutil.copy(p, dest_file) 154 dest_file.chmod(dest_file.stat().st_mode | stat.S_IEXEC) 155 self.python_executable = dest_file 156 157 self._managed_processes = [] 158 159 def tearDown(self): 160 self._terminate_managed_processes() 161 162 def test_raises_on_nonzero_exit(self): 163 with self.assertRaises(Exception): 164 build_test_suites.run_command([ 165 self.python_executable, 166 '-c', 167 textwrap.dedent(f"""\ 168 import sys 169 sys.exit(1) 170 """), 171 ]) 172 173 def test_streams_stdout(self): 174 175 def run_slow_command(stdout_file, marker): 176 with open(stdout_file, 'w') as f: 177 build_test_suites.run_command( 178 [ 179 self.python_executable, 180 '-c', 181 textwrap.dedent(f"""\ 182 import time 183 184 print('{marker}', end='', flush=True) 185 186 # Keep process alive until we check stdout. 187 time.sleep(10) 188 """), 189 ], 190 stdout=f, 191 ) 192 193 marker = 'Spinach' 194 stdout_file = self.temp_dir.joinpath('stdout.txt') 195 196 p = self.start_process(target=run_slow_command, args=[stdout_file, marker]) 197 198 self.assert_file_eventually_contains(stdout_file, marker) 199 200 def test_propagates_interruptions(self): 201 202 def run(pid_file): 203 build_test_suites.run_command([ 204 self.python_executable, 205 '-c', 206 textwrap.dedent(f"""\ 207 import os 208 import pathlib 209 import time 210 211 pathlib.Path('{pid_file}').write_text(str(os.getpid())) 212 213 # Keep the process alive for us to explicitly interrupt it. 214 time.sleep(10) 215 """), 216 ]) 217 218 pid_file = self.temp_dir.joinpath('pid.txt') 219 p = self.start_process(target=run, args=[pid_file]) 220 subprocess_pid = int(read_eventual_file_contents(pid_file)) 221 222 os.kill(p.pid, signal.SIGINT) 223 p.join() 224 225 self.assert_process_eventually_dies(p.pid) 226 self.assert_process_eventually_dies(subprocess_pid) 227 228 def start_process(self, *args, **kwargs) -> multiprocessing.Process: 229 p = multiprocessing.Process(*args, **kwargs) 230 self._managed_processes.append(p) 231 p.start() 232 return p 233 234 def assert_process_eventually_dies(self, pid: int): 235 try: 236 wait_until(lambda: not ci_test_lib.process_alive(pid)) 237 except TimeoutError as e: 238 self.fail(f'Process {pid} did not die after a while: {e}') 239 240 def assert_file_eventually_contains(self, file: pathlib.Path, substring: str): 241 wait_until(lambda: file.is_file() and file.stat().st_size > 0) 242 self.assertIn(substring, read_file_contents(file)) 243 244 def _terminate_managed_processes(self): 245 for p in self._managed_processes: 246 if not p.is_alive(): 247 continue 248 249 # We terminate the process with `SIGINT` since using `terminate` or 250 # `SIGKILL` doesn't kill any grandchild processes and we don't have 251 # `psutil` available to easily query all children. 252 os.kill(p.pid, signal.SIGINT) 253 254 255class BuildPlannerTest(unittest.TestCase): 256 257 class TestOptimizedBuildTarget(optimized_targets.OptimizedBuildTarget): 258 259 def __init__( 260 self, target, build_context, args, output_targets, packaging_commands 261 ): 262 super().__init__(target, build_context, args) 263 self.output_targets = output_targets 264 self.packaging_commands = packaging_commands 265 266 def get_build_targets_impl(self): 267 return self.output_targets 268 269 def get_package_outputs_commands_impl(self): 270 return self.packaging_commands 271 272 def get_enabled_flag(self): 273 return f'{self.target}_enabled' 274 275 def setUp(self): 276 test_discovery_agent_patcher = mock.patch('test_discovery_agent.TestDiscoveryAgent.discover_test_zip_regexes') 277 self.addCleanup(test_discovery_agent_patcher.stop) 278 self.mock_test_discovery_agent_end = test_discovery_agent_patcher.start() 279 280 281 def test_build_optimization_off_builds_everything(self): 282 build_targets = {'target_1', 'target_2'} 283 build_planner = self.create_build_planner( 284 build_context=self.create_build_context(optimized_build_enabled=False), 285 build_targets=build_targets, 286 ) 287 288 build_plan = build_planner.create_build_plan() 289 290 self.assertSetEqual(build_targets, build_plan.build_targets) 291 292 def test_build_optimization_off_doesnt_package(self): 293 build_targets = {'target_1', 'target_2'} 294 build_planner = self.create_build_planner( 295 build_context=self.create_build_context(optimized_build_enabled=False), 296 build_targets=build_targets, 297 ) 298 299 build_plan = build_planner.create_build_plan() 300 301 for packaging_command in self.run_packaging_commands(build_plan): 302 self.assertEqual(len(packaging_command), 0) 303 304 def test_build_optimization_on_optimizes_target(self): 305 build_targets = {'target_1', 'target_2'} 306 build_planner = self.create_build_planner( 307 build_targets=build_targets, 308 build_context=self.create_build_context( 309 enabled_build_features=[{'name': self.get_target_flag('target_1')}] 310 ), 311 ) 312 313 build_plan = build_planner.create_build_plan() 314 315 expected_targets = {self.get_optimized_target_name('target_1'), 'target_2'} 316 self.assertSetEqual(expected_targets, build_plan.build_targets) 317 318 def test_build_optimization_on_packages_target(self): 319 build_targets = {'target_1', 'target_2'} 320 optimized_target_name = self.get_optimized_target_name('target_1') 321 packaging_commands = [[f'packaging {optimized_target_name}']] 322 build_planner = self.create_build_planner( 323 build_targets=build_targets, 324 build_context=self.create_build_context( 325 enabled_build_features=[{'name': self.get_target_flag('target_1')}] 326 ), 327 packaging_commands=packaging_commands, 328 ) 329 330 build_plan = build_planner.create_build_plan() 331 332 self.assertIn(packaging_commands, self.run_packaging_commands(build_plan)) 333 334 def test_individual_build_optimization_off_doesnt_optimize(self): 335 build_targets = {'target_1', 'target_2'} 336 build_planner = self.create_build_planner( 337 build_targets=build_targets, 338 ) 339 340 build_plan = build_planner.create_build_plan() 341 342 self.assertSetEqual(build_targets, build_plan.build_targets) 343 344 def test_individual_build_optimization_off_doesnt_package(self): 345 build_targets = {'target_1', 'target_2'} 346 packaging_commands = [['packaging command']] 347 build_planner = self.create_build_planner( 348 build_targets=build_targets, 349 packaging_commands=packaging_commands, 350 ) 351 352 build_plan = build_planner.create_build_plan() 353 354 for packaging_command in self.run_packaging_commands(build_plan): 355 self.assertEqual(len(packaging_command), 0) 356 357 def test_target_output_used_target_built(self): 358 build_target = 'test_target' 359 build_planner = self.create_build_planner( 360 build_targets={build_target}, 361 build_context=self.create_build_context( 362 test_context=self.get_test_context(build_target), 363 enabled_build_features=[{'name': 'test_target_unused_exclusion'}], 364 ), 365 ) 366 367 build_plan = build_planner.create_build_plan() 368 369 self.assertSetEqual(build_plan.build_targets, {build_target}) 370 371 def test_target_regex_used_target_built(self): 372 build_target = 'test_target' 373 test_context = self.get_test_context(build_target) 374 test_context['testInfos'][0]['extraOptions'] = [{ 375 'key': 'additional-files-filter', 376 'values': [f'.*{build_target}.*\.zip'], 377 }] 378 build_planner = self.create_build_planner( 379 build_targets={build_target}, 380 build_context=self.create_build_context( 381 test_context=test_context, 382 enabled_build_features=[{'name': 'test_target_unused_exclusion'}], 383 ), 384 ) 385 386 build_plan = build_planner.create_build_plan() 387 388 self.assertSetEqual(build_plan.build_targets, {build_target}) 389 390 def test_target_output_not_used_target_not_built(self): 391 build_target = 'test_target' 392 test_context = self.get_test_context(build_target) 393 test_context['testInfos'][0]['extraOptions'] = [] 394 build_planner = self.create_build_planner( 395 build_targets={build_target}, 396 build_context=self.create_build_context( 397 test_context=test_context, 398 enabled_build_features=[{'name': 'test_target_unused_exclusion'}], 399 ), 400 ) 401 402 build_plan = build_planner.create_build_plan() 403 404 self.assertSetEqual(build_plan.build_targets, set()) 405 406 def test_target_regex_matching_not_too_broad(self): 407 build_target = 'test_target' 408 test_context = self.get_test_context(build_target) 409 test_context['testInfos'][0]['extraOptions'] = [{ 410 'key': 'additional-files-filter', 411 'values': [f'.*a{build_target}.*\.zip'], 412 }] 413 build_planner = self.create_build_planner( 414 build_targets={build_target}, 415 build_context=self.create_build_context( 416 test_context=test_context, 417 enabled_build_features=[{'name': 'test_target_unused_exclusion'}], 418 ), 419 ) 420 421 build_plan = build_planner.create_build_plan() 422 423 self.assertSetEqual(build_plan.build_targets, set()) 424 425 def create_build_planner( 426 self, 427 build_targets: set[str], 428 build_context: BuildContext = None, 429 args: argparse.Namespace = None, 430 target_optimizations: dict[ 431 str, optimized_targets.OptimizedBuildTarget 432 ] = None, 433 packaging_commands: list[list[str]] = [], 434 ) -> build_test_suites.BuildPlanner: 435 if not build_context: 436 build_context = self.create_build_context() 437 if not args: 438 args = self.create_args(extra_build_targets=build_targets) 439 if not target_optimizations: 440 target_optimizations = self.create_target_optimizations( 441 build_context, 442 build_targets, 443 packaging_commands, 444 ) 445 return build_test_suites.BuildPlanner( 446 build_context, args, target_optimizations 447 ) 448 449 def create_build_context( 450 self, 451 optimized_build_enabled: bool = True, 452 enabled_build_features: list[dict[str, str]] = [], 453 test_context: dict[str, any] = {}, 454 ) -> BuildContext: 455 build_context_dict = {} 456 build_context_dict['enabledBuildFeatures'] = enabled_build_features 457 if optimized_build_enabled: 458 build_context_dict['enabledBuildFeatures'].append( 459 {'name': 'optimized_build'} 460 ) 461 build_context_dict['testContext'] = test_context 462 return BuildContext(build_context_dict) 463 464 def create_args( 465 self, extra_build_targets: set[str] = set() 466 ) -> argparse.Namespace: 467 parser = argparse.ArgumentParser() 468 parser.add_argument('extra_targets', nargs='*') 469 return parser.parse_args(extra_build_targets) 470 471 def create_target_optimizations( 472 self, 473 build_context: BuildContext, 474 build_targets: set[str], 475 packaging_commands: list[list[str]] = [], 476 ): 477 target_optimizations = dict() 478 for target in build_targets: 479 target_optimizations[target] = functools.partial( 480 self.TestOptimizedBuildTarget, 481 output_targets={self.get_optimized_target_name(target)}, 482 packaging_commands=packaging_commands, 483 ) 484 485 return target_optimizations 486 487 def get_target_flag(self, target: str): 488 return f'{target}_enabled' 489 490 def get_optimized_target_name(self, target: str): 491 return f'{target}_optimized' 492 493 def get_test_context(self, target: str): 494 return { 495 'testInfos': [ 496 { 497 'name': 'atp_test', 498 'target': 'test_target', 499 'branch': 'branch', 500 'extraOptions': [{ 501 'key': 'additional-files-filter', 502 'values': [f'{target}.zip'], 503 }], 504 'command': '/tf/command', 505 'extraBuildTargets': [ 506 'extra_build_target', 507 ], 508 }, 509 ], 510 } 511 512 def run_packaging_commands(self, build_plan: build_test_suites.BuildPlan): 513 return [ 514 packaging_command_getter() 515 for packaging_command_getter in build_plan.packaging_commands_getters 516 ] 517 518 519def wait_until( 520 condition_function: Callable[[], bool], 521 timeout_secs: float = 3.0, 522 polling_interval_secs: float = 0.1, 523): 524 """Waits until a condition function returns True.""" 525 526 start_time_secs = time.time() 527 528 while not condition_function(): 529 if time.time() - start_time_secs > timeout_secs: 530 raise TimeoutError( 531 f'Condition not met within timeout: {timeout_secs} seconds' 532 ) 533 534 time.sleep(polling_interval_secs) 535 536 537def read_file_contents(file: pathlib.Path) -> str: 538 with open(file, 'r') as f: 539 return f.read() 540 541 542def read_eventual_file_contents(file: pathlib.Path) -> str: 543 wait_until(lambda: file.is_file() and file.stat().st_size > 0) 544 return read_file_contents(file) 545 546 547if __name__ == '__main__': 548 ci_test_lib.main() 549