1#!/usr/bin/env python3 2# Lint as: python2, python3 3# Copyright 2019 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7from __future__ import print_function 8 9import argparse 10import contextlib 11import copy 12from enum import Enum 13import logging 14import os 15import re 16import shutil 17import stat 18import subprocess 19import tempfile 20import textwrap 21import zipfile 22# Use 'sudo pip install jinja2' to install. 23from jinja2 import Template 24 25# Type of source storage from where the generated control files should 26# retrieve the xTS bundle zip file. 27# 'MOBLAB' means the bucket for moblab used by 3PL. 28# 'LATEST' means the latest official xTS release. 29# 'DEV' means the preview version build from development branch. 30SourceType = Enum('SourceType', ['MOBLAB', 'LATEST', 'DEV']) 31 32 33# TODO(ihf): Assign better TIME to control files. Scheduling uses this to run 34# LENGTHY first, then LONG, MEDIUM etc. But we need LENGTHY for the collect 35# job, downgrade all others. Make sure this still works in CQ/smoke suite. 36_CONTROLFILE_TEMPLATE = Template( 37 textwrap.dedent("""\ 38 # Copyright {{year}} The Chromium OS Authors. All rights reserved. 39 # Use of this source code is governed by a BSD-style license that can be 40 # found in the LICENSE file. 41 42 # This file has been automatically generated. Do not edit! 43 {%- if servo_support_needed %} 44 from autotest_lib.server import utils as server_utils 45 {%- endif %} 46 {%- if wifi_info_needed %} 47 from autotest_lib.client.common_lib import utils, global_config 48 import pipes 49 {%- endif %} 50 51 AUTHOR = 'ARC++ Team' 52 NAME = '{{name}}' 53 ATTRIBUTES = '{{attributes}}' 54 DEPENDENCIES = '{{dependencies}}' 55 JOB_RETRIES = {{job_retries}} 56 TEST_TYPE = 'server' 57 TIME = '{{test_length}}' 58 MAX_RESULT_SIZE_KB = {{max_result_size_kb}} 59 PY_VERSION = 3 60 {%- if sync_count and sync_count > 1 %} 61 SYNC_COUNT = {{sync_count}} 62 {%- endif %} 63 {%- if priority %} 64 PRIORITY = {{priority}} 65 {%- endif %} 66 DOC = '{{DOC}}' 67 {%- if servo_support_needed %} 68 69 # For local debugging, if your test setup doesn't have servo, REMOVE these 70 # two lines. 71 args_dict = server_utils.args_to_dict(args) 72 servo_args = hosts.CrosHost.get_servo_arguments(args_dict) 73 74 {%- endif %} 75 {% if sync_count and sync_count > 1 %} 76 from autotest_lib.server import utils as server_utils 77 def {{test_func_name}}(ntuples): 78 host_list = [hosts.create_host(machine) for machine in ntuples] 79 {% else %} 80 def {{test_func_name}}(machine): 81 {%- if servo_support_needed %} 82 # REMOVE 'servo_args=servo_args' arg for local debugging if your test 83 # setup doesn't have servo. 84 try: 85 host_list = [hosts.create_host(machine, servo_args=servo_args)] 86 except: 87 # Just ignore any servo setup flakiness. 88 host_list = [hosts.create_host(machine)] 89 {%- else %} 90 host_list = [hosts.create_host(machine)] 91 {%- endif %} 92 {%- if wifi_info_needed %} 93 ssid = utils.get_wireless_ssid(machine['hostname']) 94 wifipass = global_config.global_config.get_config_value('CLIENT', 95 'wireless_password', default=None) 96 {%- endif %} 97 {%- endif %} 98 job.run_test( 99 '{{base_name}}', 100 {%- if camera_facing %} 101 camera_facing='{{camera_facing}}', 102 cmdline_args=args, 103 {%- endif %} 104 hosts=host_list, 105 iterations=1, 106 {%- if max_retries != None %} 107 max_retry={{max_retries}}, 108 {%- endif %} 109 {%- if enable_default_apps %} 110 enable_default_apps=True, 111 {%- endif %} 112 {%- if needs_push_media %} 113 needs_push_media={{needs_push_media}}, 114 {%- endif %} 115 {%- if needs_cts_helpers %} 116 use_helpers={{needs_cts_helpers}}, 117 {%- endif %} 118 tag='{{tag}}', 119 test_name='{{name}}', 120 {%- if authkey %} 121 authkey='{{authkey}}', 122 {%- endif %} 123 run_template={{run_template}}, 124 retry_template={{retry_template}}, 125 target_module={% if target_module %}'{{target_module}}'{% else %}None{%endif%}, 126 target_plan={% if target_plan %}'{{target_plan}}'{% else %}None{% endif %}, 127 {%- if abi %} 128 bundle='{{abi}}', 129 {%- endif %} 130 {%- if extra_artifacts %} 131 extra_artifacts={{extra_artifacts}}, 132 {%- endif %} 133 {%- if extra_artifacts_host %} 134 extra_artifacts_host={{extra_artifacts_host}}, 135 {%- endif %} 136 {%- if uri %} 137 uri='{{uri}}', 138 {%- endif %} 139 {%- for arg in extra_args %} 140 {{arg}}, 141 {%- endfor %} 142 {%- if servo_support_needed %} 143 hard_reboot_on_failure=True, 144 {%- endif %} 145 {%- if camera_facing %} 146 retry_manual_tests=True, 147 {%- endif %} 148 timeout={{timeout}}) 149 150 {% if sync_count and sync_count > 1 -%} 151 ntuples, failures = server_utils.form_ntuples_from_machines(machines, 152 SYNC_COUNT) 153 # Use log=False in parallel_simple to avoid an exception in setting up 154 # the incremental parser when SYNC_COUNT > 1. 155 parallel_simple({{test_func_name}}, ntuples, log=False) 156 {% else -%} 157 parallel_simple({{test_func_name}}, machines) 158 {% endif %} 159""")) 160 161CONFIG = None 162 163_COLLECT = 'tradefed-run-collect-tests-only-internal' 164_PUBLIC_COLLECT = 'tradefed-run-collect-tests-only' 165_CTSHARDWARE_COLLECT = 'tradefed-run-collect-tests-only-hardware-internal' 166_PUBLIC_CTSHARDWARE_COLLECT = 'tradefed-run-collect-tests-only-hardware' 167 168 169_TEST_LENGTH = {1: 'FAST', 2: 'SHORT', 3: 'MEDIUM', 4: 'LONG', 5: 'LENGTHY'} 170 171_ALL = 'all' 172 173 174def get_tradefed_build(line): 175 """Gets the build of Android CTS from tradefed. 176 177 @param line Tradefed identification output on startup. Example: 178 Android Compatibility Test Suite 7.0 (3423912) 179 @return Tradefed CTS build. Example: 2813453. 180 """ 181 # Sample string: 182 # - Android Compatibility Test Suite 7.0 (3423912) 183 # - Android Compatibility Test Suite for Instant Apps 1.0 (4898911) 184 # - Android Google Mobile Services (GMS) Test Suite 6.0_r1 (4756896) 185 m = re.search(r' \((.*)\)', line) 186 if m: 187 return m.group(1) 188 logging.warning('Could not identify build in line "%s".', line) 189 return '<unknown>' 190 191 192def get_tradefed_revision(line): 193 """Gets the revision of Android CTS from tradefed. 194 195 @param line Tradefed identification output on startup. 196 Example: 197 Android Compatibility Test Suite 6.0_r6 (2813453) 198 Android Compatibility Test Suite for Instant Apps 1.0 (4898911) 199 @return Tradefed CTS revision. Example: 6.0_r6. 200 """ 201 tradefed_identifier_list = [ 202 r'Android Google Mobile Services \(GMS\) Test Suite (.*) \(', 203 r'Android Compatibility Test Suite(?: for Instant Apps)? (.*) \(', 204 r'Android Vendor Test Suite (.*) \(', 205 r'Android Security Test Suite (.*) \(' 206 ] 207 208 for identifier in tradefed_identifier_list: 209 m = re.search(identifier, line) 210 if m: 211 return m.group(1) 212 213 logging.warning('Could not identify revision in line "%s".', line) 214 return None 215 216 217def get_bundle_abi(filename): 218 """Makes an educated guess about the ABI. 219 220 In this case we chose to guess by filename, but we could also parse the 221 xml files in the module. (Maybe this needs to be done in the future.) 222 """ 223 if CONFIG.get('SINGLE_CONTROL_FILE'): 224 return None 225 if filename.endswith('arm.zip'): 226 return 'arm' 227 if filename.endswith('arm64.zip'): 228 return 'arm64' 229 if filename.endswith('x86.zip'): 230 return 'x86' 231 if filename.endswith('x86_64.zip'): 232 return 'x86_64' 233 234 assert(CONFIG['TRADEFED_CTS_COMMAND'] =='gts'), 'Only GTS has empty ABI' 235 return '' 236 237 238def get_extension(module, 239 abi, 240 revision, 241 is_public=False, 242 led_provision=None, 243 camera_facing=None, 244 hardware_suite=False, 245 abi_bits=None): 246 """Defines a unique string. 247 248 Notice we chose module revision first, then abi, as the module revision 249 changes regularly. This ordering makes it simpler to add/remove modules. 250 @param module: CTS module which will be tested in the control file. If 'all' 251 is specified, the control file will runs all the tests. 252 @param is_public: boolean variable to specify whether or not the bundle is 253 from public source or not. 254 @param led_provision: string or None indicate whether the camerabox has led 255 light or not. 256 @param camera_facing: string or None indicate whether it's camerabox tests 257 for specific camera facing or not. 258 @param abi_bits: 32 or 64 or None indicate the bitwidth for the specific 259 abi to run. 260 @return string: unique string for specific tests. If public=True then the 261 string is "<abi>.<module>", otherwise, the unique string is 262 "internal.<abi>.<module>" for internal. Note that if abi is empty, 263 the abi part is omitted. 264 """ 265 ext_parts = [] 266 if not CONFIG.get('SINGLE_CONTROL_FILE') and not is_public: 267 if module == _COLLECT: 268 ext_parts = [revision] 269 else: 270 ext_parts = ['internal'] 271 if not CONFIG.get('SINGLE_CONTROL_FILE') and abi: 272 ext_parts += [abi] 273 ext_parts += [module] 274 if led_provision: 275 ext_parts += [led_provision] 276 if camera_facing: 277 ext_parts += ['camerabox', camera_facing] 278 if hardware_suite and module not in get_collect_modules( 279 is_public, hardware_suite): 280 ext_parts += ['ctshardware'] 281 if not CONFIG.get('SINGLE_CONTROL_FILE') and abi and abi_bits: 282 ext_parts += [str(abi_bits)] 283 return '.'.join(ext_parts) 284 285 286def get_doc(modules, abi, is_public): 287 """Defines the control file DOC string.""" 288 if modules.intersection(get_collect_modules(is_public)) or CONFIG.get( 289 'SINGLE_CONTROL_FILE'): 290 module_text = 'all' 291 else: 292 # Generate per-module DOC 293 module_text = 'module ' + ', '.join(sorted(list(modules))) 294 295 abi_text = (' using %s ABI' % abi) if abi else '' 296 297 doc = ('Run %s of the %s%s in the ARC++ container.' 298 % (module_text, CONFIG['DOC_TITLE'], abi_text)) 299 return doc 300 301 302def servo_support_needed(modules, is_public=True): 303 """Determines if servo support is needed for a module.""" 304 return not is_public and any(module in CONFIG['NEEDS_POWER_CYCLE'] 305 for module in modules) 306 307 308def wifi_info_needed(modules, is_public): 309 """Determines if Wifi AP info needs to be retrieved.""" 310 return not is_public and any(module in CONFIG.get('WIFI_MODULES', []) 311 for module in modules) 312 313 314def get_controlfile_name(module, 315 abi, 316 revision, 317 is_public=False, 318 led_provision=None, 319 camera_facing=None, 320 abi_bits=None, 321 hardware_suite=False): 322 """Defines the control file name. 323 324 @param module: CTS module which will be tested in the control file. If 'all' 325 is specified, the control file will runs all the tests. 326 @param public: boolean variable to specify whether or not the bundle is from 327 public source or not. 328 @param camera_facing: string or None indicate whether it's camerabox tests 329 for specific camera facing or not. 330 @param led_provision: string or None indicate whether the camerabox has led 331 light or not. 332 @param abi_bits: 32 or 64 or None indicate the bitwidth for the specific 333 abi to run. 334 @return string: control file for specific tests. If public=True or 335 module=all, then the name will be "control.<abi>.<module>", 336 otherwise, the name will be 337 "control.<revision>.<abi>.<module>". 338 """ 339 return 'control.%s' % get_extension(module, abi, revision, is_public, 340 led_provision, camera_facing, hardware_suite, 341 abi_bits) 342 343 344def get_sync_count(_modules, _abi, _is_public): 345 return 1 346 347 348def get_suites(modules, abi, is_public, camera_facing=None, 349 hardware_suite=False): 350 """Defines the suites associated with a module. 351 352 @param module: CTS module which will be tested in the control file. If 'all' 353 is specified, the control file will runs all the tests. 354 # TODO(ihf): Make this work with the "all" and "collect" generation, 355 # which currently bypass this function. 356 """ 357 cts_hardware_modules = set(CONFIG.get('HARDWARE_MODULES', [])) 358 359 if is_public: 360 suites = set([CONFIG['MOBLAB_SUITE_NAME']]) 361 if hardware_suite: 362 suites = set([CONFIG['MOBLAB_HARDWARE_SUITE_NAME']]) 363 return sorted(list(suites)) 364 365 suites = set(CONFIG['INTERNAL_SUITE_NAMES']) 366 367 for module in modules: 368 if module in get_collect_modules(is_public, hardware_suite): 369 # We collect all tests both in arc-gts and arc-gts-qual as both have 370 # a chance to be complete (and used for submission). 371 suites |= set(CONFIG['QUAL_SUITE_NAMES']) 372 if module in CONFIG['EXTRA_ATTRIBUTES']: 373 # Special cases come with their own suite definitions. 374 suites |= set(CONFIG['EXTRA_ATTRIBUTES'][module]) 375 if module in CONFIG['SMOKE'] and (abi == 'arm' or abi == ''): 376 # Handle VMTest by adding a few jobs to suite:smoke. 377 suites.add('suite:smoke') 378 if module in CONFIG['HARDWARE_DEPENDENT_MODULES']: 379 # CTS modules to be run on all unibuild models. 380 suites.add('suite:arc-cts-unibuild-hw') 381 if abi == 'x86': 382 # Handle a special builder for running all of CTS in a betty VM. 383 # TODO(ihf): figure out if this builder is still alive/needed. 384 vm_suite = None 385 for suite in CONFIG['VMTEST_INFO_SUITES']: 386 if not vm_suite: 387 vm_suite = suite 388 if module in CONFIG['VMTEST_INFO_SUITES'][suite]: 389 vm_suite = suite 390 if vm_suite is not None: 391 suites.add('suite:%s' % vm_suite) 392 # One or two modules hould be in suite:bvt-arc to cover CQ/PFQ. A few 393 # spare/fast modules can run in suite:bvt-perbuild in case we need a 394 # replacement for the module in suite:bvt-arc (integration test for 395 # cheets_CTS only, not a correctness test for CTS content). 396 if module in CONFIG['BVT_ARC'] and (abi == 'arm' or abi == ''): 397 suites.add('suite:bvt-arc') 398 elif module in CONFIG['BVT_PERBUILD'] and (abi == 'arm' or abi == ''): 399 suites.add('suite:bvt-perbuild') 400 401 if hardware_suite: 402 suites = set([CONFIG['HARDWARE_SUITE_NAME']]) 403 404 if camera_facing != None: 405 suites.add('suite:arc-cts-camera') 406 407 return sorted(list(suites)) 408 409 410def get_dependencies(modules, abi, is_public, led_provision, camera_facing): 411 """Defines lab dependencies needed to schedule a module. 412 413 @param module: CTS module which will be tested in the control file. If 'all' 414 is specified, the control file will runs all the tests. 415 @param abi: string that specifies the application binary interface of the 416 current test. 417 @param is_public: boolean variable to specify whether or not the bundle is 418 from public source or not. 419 @param led_provision: specify if led is provisioned in the camerabox setup. 'noled' when 420 there is no led light in the box and 'led' otherwise. 421 @param camera_facing: specify requirement of camerabox setup with target 422 test camera facing. Set to None if it's not camerabox 423 related test. 424 """ 425 dependencies = ['arc'] 426 if abi in CONFIG['LAB_DEPENDENCY']: 427 dependencies += CONFIG['LAB_DEPENDENCY'][abi] 428 429 if led_provision is not None: 430 dependencies.append('camerabox_light:'+led_provision) 431 432 if camera_facing is not None: 433 dependencies.append('camerabox_facing:'+camera_facing) 434 435 for module in modules: 436 if is_public and module in CONFIG['PUBLIC_DEPENDENCIES']: 437 dependencies.extend(CONFIG['PUBLIC_DEPENDENCIES'][module]) 438 439 return ', '.join(dependencies) 440 441 442def get_job_retries(modules, is_public, suites): 443 """Define the number of job retries associated with a module. 444 445 @param module: CTS module which will be tested in the control file. If a 446 special module is specified, the control file will runs all 447 the tests without retry. 448 @param is_public: true if the control file is for moblab (public) use. 449 @param suites: the list of suites that the control file belongs to. 450 """ 451 # TODO(haddowk): remove this when cts p has stabalized. 452 if is_public: 453 return CONFIG['CTS_JOB_RETRIES_IN_PUBLIC'] 454 # Presubmit check forces to set 2 or more retries for CQ tests. 455 if 'suite:bvt-arc' in suites: 456 return 2 457 retries = 1 # 0 is NO job retries, 1 is one retry etc. 458 for module in modules: 459 # We don't want job retries for module collection or special cases. 460 if (module in get_collect_modules(is_public) or module == _ALL or 461 ('CtsDeqpTestCases' in CONFIG['EXTRA_MODULES'] and 462 module in CONFIG['EXTRA_MODULES']['CtsDeqpTestCases'] 463 )): 464 retries = 0 465 return retries 466 467 468def get_max_retries(modules, abi, suites, is_public): 469 """Partners experiance issues where some modules are flaky and require more 470 471 retries. Calculate the retry number per module on moblab. 472 @param module: CTS module which will be tested in the control file. 473 """ 474 retry = -1 475 if is_public: 476 if _ALL in CONFIG['PUBLIC_MODULE_RETRY_COUNT']: 477 retry = CONFIG['PUBLIC_MODULE_RETRY_COUNT'][_ALL] 478 479 # In moblab at partners we may need many more retries than in lab. 480 for module in modules: 481 if module in CONFIG['PUBLIC_MODULE_RETRY_COUNT']: 482 retry = max(retry, CONFIG['PUBLIC_MODULE_RETRY_COUNT'][module]) 483 else: 484 # See if we have any special values for the module, chose the largest. 485 for module in modules: 486 if module in CONFIG['CTS_MAX_RETRIES']: 487 retry = max(retry, CONFIG['CTS_MAX_RETRIES'][module]) 488 489 # Ugly overrides. 490 # In bvt we don't want to hold the CQ/PFQ too long. 491 if 'suite:bvt-arc' in suites: 492 retry = 3 493 # Not strict as CQ for bvt-perbuild. Let per-module config take priority. 494 if retry == -1 and 'suite:bvt-perbuild' in suites: 495 retry = 3 496 # During qualification we want at least 9 retries, possibly more. 497 # TODO(kinaba&yoshiki): do not abuse suite names 498 if CONFIG.get('QUAL_SUITE_NAMES') and \ 499 set(CONFIG['QUAL_SUITE_NAMES']) & set(suites): 500 retry = max(retry, CONFIG['CTS_QUAL_RETRIES']) 501 # Collection should never have a retry. This needs to be last. 502 if modules.intersection(get_collect_modules(is_public)): 503 retry = 0 504 505 if retry >= 0: 506 return retry 507 # Default case omits the retries in the control file, so tradefed_test.py 508 # can chose its own value. 509 return None 510 511 512def get_max_result_size_kb(modules, is_public): 513 """Returns the maximum expected result size in kB for autotest. 514 515 @param modules: List of CTS modules to be tested by the control file. 516 """ 517 for module in modules: 518 if (module in get_collect_modules(is_public) or 519 module == 'CtsDeqpTestCases'): 520 # CTS tests and dump logs for android-cts. 521 return CONFIG['LARGE_MAX_RESULT_SIZE'] 522 # Individual module normal produces less results than all modules. 523 return CONFIG['NORMAL_MAX_RESULT_SIZE'] 524 525 526def get_extra_args(modules, is_public): 527 """Generate a list of extra arguments to pass to the test. 528 529 Some params are specific to a particular module, particular mode or 530 combination of both, generate a list of arguments to pass into the template. 531 532 @param modules: List of CTS modules to be tested by the control file. 533 """ 534 extra_args = set() 535 preconditions = [] 536 login_preconditions = [] 537 prerequisites = [] 538 for module in sorted(modules): 539 # Remove this once JDK9 is the base JDK for lab. 540 if CONFIG.get('USE_JDK9', False): 541 extra_args.add('use_jdk9=True') 542 if module in CONFIG.get('USE_OLD_ADB', []): 543 extra_args.add('use_old_adb=True') 544 if is_public: 545 extra_args.add('warn_on_test_retry=False') 546 extra_args.add('retry_manual_tests=True') 547 preconditions.extend(CONFIG['PUBLIC_PRECONDITION'].get(module, [])) 548 else: 549 preconditions.extend(CONFIG['PRECONDITION'].get(module, [])) 550 login_preconditions.extend( 551 CONFIG['LOGIN_PRECONDITION'].get(module, [])) 552 prerequisites.extend(CONFIG['PREREQUISITES'].get(module,[])) 553 554 # Notice: we are just squishing the preconditions for all modules together 555 # with duplicated command removed. This may not always be correct. 556 # In such a case one should split the bookmarks in a way that the modules 557 # with conflicting preconditions end up in separate control files. 558 def deduped(lst): 559 """Keep only the first occurrence of each element.""" 560 return [e for i, e in enumerate(lst) if e not in lst[0:i]] 561 562 if preconditions: 563 # To properly escape the public preconditions we need to format the list 564 # manually using join. 565 extra_args.add('precondition_commands=[%s]' % ', '.join( 566 deduped(preconditions))) 567 if login_preconditions: 568 extra_args.add('login_precondition_commands=[%s]' % ', '.join( 569 deduped(login_preconditions))) 570 if prerequisites: 571 extra_args.add("prerequisites=['%s']" % "', '".join( 572 deduped(prerequisites))) 573 return sorted(list(extra_args)) 574 575 576def get_test_length(modules): 577 """ Calculate the test length based on the module name. 578 579 To better optimize DUT's connected to moblab, it is better to run the 580 longest tests and tests that require limited resources. For these modules 581 override from the default test length. 582 583 @param module: CTS module which will be tested in the control file. If 'all' 584 is specified, the control file will runs all the tests. 585 586 @return string: one of the specified test lengths: 587 ['FAST', 'SHORT', 'MEDIUM', 'LONG', 'LENGTHY'] 588 """ 589 length = 3 # 'MEDIUM' 590 for module in modules: 591 if module in CONFIG['OVERRIDE_TEST_LENGTH']: 592 length = max(length, CONFIG['OVERRIDE_TEST_LENGTH'][module]) 593 return _TEST_LENGTH[length] 594 595 596def get_test_priority(modules, is_public): 597 """ Calculate the test priority based on the module name. 598 599 On moblab run all long running tests and tests that have some unique 600 characteristic at a higher priority (50). 601 602 This optimizes the total run time of the suite assuring the shortest 603 time between suite kick off and 100% complete. 604 605 @param module: CTS module which will be tested in the control file. 606 607 @return int: 0 if priority not to be overridden, or priority number otherwise. 608 """ 609 if not is_public: 610 return 0 611 612 priority = 0 613 overide_test_priority_dict = CONFIG.get('PUBLIC_OVERRIDE_TEST_PRIORITY', {}) 614 for module in modules: 615 if module in overide_test_priority_dict: 616 priority = max(priority, overide_test_priority_dict[module]) 617 elif (module in CONFIG['OVERRIDE_TEST_LENGTH'] or 618 module in CONFIG['PUBLIC_DEPENDENCIES'] or 619 module in CONFIG['PUBLIC_PRECONDITION'] or 620 module.split('.')[0] in CONFIG['OVERRIDE_TEST_LENGTH']): 621 priority = max(priority, 50) 622 return priority 623 624 625def get_authkey(is_public): 626 if is_public or not CONFIG['AUTHKEY']: 627 return None 628 return CONFIG['AUTHKEY'] 629 630 631def _format_collect_cmd(is_public, abi_to_run, retry, is_hardware=False): 632 """Returns a list specifying tokens for tradefed to list all tests.""" 633 if retry: 634 return None 635 cmd = ['run', 'commandAndExit', 'collect-tests-only'] 636 if CONFIG['TRADEFED_DISABLE_REBOOT_ON_COLLECTION']: 637 cmd += ['--disable-reboot'] 638 if is_hardware: 639 cmd.append('--subplan') 640 cmd.append('cts-hardware') 641 for m in CONFIG['MEDIA_MODULES']: 642 cmd.append('--module-arg') 643 cmd.append('%s:skip-media-download:true' % m) 644 if (not is_public and 645 not CONFIG.get('NEEDS_DYNAMIC_CONFIG_ON_COLLECTION', True)): 646 cmd.append('--dynamic-config-url=') 647 if abi_to_run: 648 cmd += ['--abi', abi_to_run] 649 return cmd 650 651 652def _get_special_command_line(modules, _is_public): 653 """This function allows us to split a module like Deqp into segments.""" 654 cmd = [] 655 for module in sorted(modules): 656 cmd += CONFIG['EXTRA_COMMANDLINE'].get(module, []) 657 return cmd 658 659 660def _format_modules_cmd(is_public, 661 abi_to_run, 662 modules=None, 663 retry=False, 664 whole_module_set=None, 665 is_hardware=False): 666 """Returns list of command tokens for tradefed.""" 667 if retry: 668 assert(CONFIG['TRADEFED_RETRY_COMMAND'] == 'cts' or 669 CONFIG['TRADEFED_RETRY_COMMAND'] == 'retry') 670 671 cmd = ['run', 'commandAndExit', CONFIG['TRADEFED_RETRY_COMMAND'], 672 '--retry', '{session_id}'] 673 else: 674 # For runs create a logcat file for each individual failure. 675 cmd = ['run', 'commandAndExit', CONFIG['TRADEFED_CTS_COMMAND']] 676 677 special_cmd = _get_special_command_line(modules, is_public) 678 if special_cmd: 679 if is_hardware: 680 # For hardware suite we want to exclude [instant] modules. 681 filtered = [] 682 i = 0 683 while i < len(special_cmd): 684 if (special_cmd[i] == '--include-filter' 685 and '[instant]' in special_cmd[i + 1]): 686 i += 2 687 elif (special_cmd[i] == '--module' 688 and i + 3 < len(special_cmd) 689 and special_cmd[i + 2] == '--test'): 690 # [--module, x, --test, y] ==> [--include-filter, "x y"] 691 # because --module implicitly include [instant] modules 692 filtered.append('--include-filter') 693 filtered.append( 694 '%s %s' % 695 (special_cmd[i + 1], special_cmd[i + 3])) 696 i += 4 697 elif special_cmd[i] == '--module': 698 # [--module, x] ==> [--include-filter, x] 699 filtered.append('--include-filter') 700 filtered.append(special_cmd[i + 1]) 701 i += 2 702 else: 703 filtered.append(special_cmd[i]) 704 i += 1 705 special_cmd = filtered 706 cmd.extend(special_cmd) 707 elif _ALL in modules: 708 pass 709 elif len(modules) == 1 and not is_hardware: 710 cmd += ['--module', list(modules)[0]] 711 else: 712 if whole_module_set is None: 713 assert (CONFIG['TRADEFED_CTS_COMMAND'] != 'cts-instant'), \ 714 'cts-instant cannot include multiple modules' 715 # We run each module with its own --include-filter option. 716 # https://source.android.com/compatibility/cts/run 717 for module in sorted(modules): 718 # b/196756614 32-bit jobs should skip [parameter] modules. 719 if is_parameterized_module(module) and abi_to_run in [ 720 'x86', 'armeabi-v7a' 721 ]: 722 continue 723 cmd += ['--include-filter', module] 724 else: 725 # CTS-Instant does not support --include-filter due to 726 # its implementation detail. Instead, exclude the complement. 727 for module in sorted(whole_module_set - set(modules)): 728 cmd += ['--exclude-filter', module] 729 730 # For runs create a logcat file for each individual failure. 731 # Not needed on moblab, nobody is going to look at them. 732 if (not modules.intersection(CONFIG['DISABLE_LOGCAT_ON_FAILURE']) and 733 not is_public and 734 CONFIG['TRADEFED_CTS_COMMAND'] != 'gts'): 735 cmd.append('--logcat-on-failure') 736 737 if CONFIG['TRADEFED_IGNORE_BUSINESS_LOGIC_FAILURE']: 738 cmd.append('--ignore-business-logic-failure') 739 740 if CONFIG['TRADEFED_DISABLE_REBOOT']: 741 cmd.append('--disable-reboot') 742 if (CONFIG['TRADEFED_MAY_SKIP_DEVICE_INFO'] and 743 not (modules.intersection(CONFIG['BVT_ARC'] + CONFIG['SMOKE'] + 744 CONFIG['NEEDS_DEVICE_INFO']))): 745 cmd.append('--skip-device-info') 746 if abi_to_run: 747 cmd += ['--abi', abi_to_run] 748 # If NEEDS_DYNAMIC_CONFIG is set, disable the feature except on the modules 749 # that explicitly set as needed. 750 if (not is_public and CONFIG.get('NEEDS_DYNAMIC_CONFIG') and 751 not modules.intersection(CONFIG['NEEDS_DYNAMIC_CONFIG'])): 752 cmd.append('--dynamic-config-url=') 753 754 return cmd 755 756 757def get_run_template(modules, 758 is_public, 759 retry=False, 760 abi_to_run=None, 761 whole_module_set=None, 762 is_hardware=False): 763 """Command to run the modules specified by a control file.""" 764 no_intersection = not modules.intersection(get_collect_modules(is_public, 765 is_hardware)) 766 collect_present = (_COLLECT in modules or _PUBLIC_COLLECT in modules or 767 _CTSHARDWARE_COLLECT in modules or 768 _PUBLIC_CTSHARDWARE_COLLECT in modules) 769 all_present = _ALL in modules 770 if no_intersection or (all_present and not collect_present): 771 return _format_modules_cmd(is_public, 772 abi_to_run, 773 modules, 774 retry=retry, 775 whole_module_set=whole_module_set, 776 is_hardware=is_hardware) 777 elif collect_present: 778 return _format_collect_cmd(is_public, abi_to_run, retry=retry, 779 is_hardware=is_hardware) 780 return None 781 782def get_retry_template(modules, is_public): 783 """Command to retry the failed modules as specified by a control file.""" 784 return get_run_template(modules, is_public, retry=True) 785 786 787def get_extra_modules_dict(source_type, abi): 788 if source_type != SourceType.MOBLAB: 789 return CONFIG['EXTRA_MODULES'] 790 791 extra_modules = copy.deepcopy(CONFIG['PUBLIC_EXTRA_MODULES']) 792 if abi in CONFIG['EXTRA_SUBMODULE_OVERRIDE']: 793 for _, config in extra_modules.items(): 794 for old, news in CONFIG['EXTRA_SUBMODULE_OVERRIDE'][abi].items(): 795 if old in config.keys(): 796 suite = config[old] 797 config.pop(old) 798 for module in news: 799 config[module] = suite 800 801 return extra_modules 802 803def get_extra_hardware_modules_dict(is_public, abi): 804 return CONFIG.get('HARDWAREONLY_EXTRA_MODULES', {}) 805 806 807def get_extra_artifacts(modules): 808 artifacts = [] 809 for module in modules: 810 if module in CONFIG['EXTRA_ARTIFACTS']: 811 artifacts += CONFIG['EXTRA_ARTIFACTS'][module] 812 return artifacts 813 814 815def get_extra_artifacts_host(modules): 816 if not 'EXTRA_ARTIFACTS_HOST' in CONFIG: 817 return 818 819 artifacts = [] 820 for module in modules: 821 if module in CONFIG['EXTRA_ARTIFACTS_HOST']: 822 artifacts += CONFIG['EXTRA_ARTIFACTS_HOST'][module] 823 return artifacts 824 825 826def calculate_timeout(modules, suites): 827 """Calculation for timeout of tradefed run. 828 829 Timeout is at least one hour, except if part of BVT_ARC. 830 Notice these do get adjusted dynamically by number of ABIs on the DUT. 831 """ 832 if 'suite:bvt-arc' in suites: 833 return int(3600 * CONFIG['BVT_TIMEOUT']) 834 if CONFIG.get('QUAL_SUITE_NAMES') and \ 835 CONFIG.get('QUAL_TIMEOUT') and \ 836 ((set(CONFIG['QUAL_SUITE_NAMES']) & set(suites)) and \ 837 not (_COLLECT in modules or _PUBLIC_COLLECT in modules)): 838 return int(3600 * CONFIG['QUAL_TIMEOUT']) 839 840 timeout = 0 841 # First module gets 1h (standard), all other half hour extra (heuristic). 842 default_timeout = int(3600 * CONFIG['CTS_TIMEOUT_DEFAULT']) 843 delta = default_timeout 844 for module in modules: 845 if module in CONFIG['CTS_TIMEOUT']: 846 # Modules that run very long are encoded here. 847 timeout += int(3600 * CONFIG['CTS_TIMEOUT'][module]) 848 elif module.startswith('CtsDeqpTestCases.dEQP-VK.'): 849 # TODO: Optimize this temporary hack by reducing this value or 850 # setting appropriate values for each test if possible. 851 timeout = max(timeout, int(3600 * 12)) 852 elif 'Jvmti' in module: 853 # We have too many of these modules and they run fast. 854 timeout += 300 855 else: 856 timeout += delta 857 delta = default_timeout // 2 858 return timeout 859 860 861def needs_push_media(modules): 862 """Oracle to determine if to push several GB of media files to DUT.""" 863 if modules.intersection(set(CONFIG['NEEDS_PUSH_MEDIA'])): 864 return True 865 return False 866 867 868def needs_cts_helpers(modules): 869 """Oracle to determine if CTS helpers should be downloaded from DUT.""" 870 if 'NEEDS_CTS_HELPERS' not in CONFIG: 871 return False 872 if modules.intersection(set(CONFIG['NEEDS_CTS_HELPERS'])): 873 return True 874 return False 875 876 877def enable_default_apps(modules): 878 """Oracle to determine if to enable default apps (eg. Files.app).""" 879 if modules.intersection(set(CONFIG['ENABLE_DEFAULT_APPS'])): 880 return True 881 return False 882 883 884def is_parameterized_module(module): 885 """Determines if the given module is a parameterized module.""" 886 return '[' in module 887 888 889def get_controlfile_content(combined, 890 modules, 891 abi, 892 revision, 893 build, 894 uri, 895 suites=None, 896 source_type=None, 897 abi_bits=None, 898 led_provision=None, 899 camera_facing=None, 900 hardware_suite=False, 901 whole_module_set=None): 902 """Returns the text inside of a control file. 903 904 @param combined: name to use for this combination of modules. 905 @param modules: set of CTS modules which will be tested in the control 906 file. If 'all' is specified, the control file will run 907 all the tests. 908 """ 909 is_public = (source_type == SourceType.MOBLAB) 910 # We tag results with full revision now to get result directories containing 911 # the revision. This fits stainless/ better. 912 tag = '%s' % get_extension(combined, abi, revision, is_public, 913 led_provision, camera_facing, hardware_suite, abi_bits) 914 # For test_that the NAME should be the same as for the control file name. 915 # We could try some trickery here to get shorter extensions for a default 916 # suite/ARM. But with the monthly uprevs this will quickly get confusing. 917 name = '%s.%s' % (CONFIG['TEST_NAME'], tag) 918 if not suites: 919 suites = get_suites(modules, abi, is_public, camera_facing, hardware_suite) 920 attributes = ', '.join(suites) 921 uri = { 922 SourceType.MOBLAB: None, 923 SourceType.LATEST: 'LATEST', 924 SourceType.DEV: 'DEV' 925 }.get(source_type) 926 target_module = None 927 if (combined not in get_collect_modules(is_public) and combined != _ALL): 928 target_module = combined 929 for target, config in get_extra_modules_dict(source_type, abi).items(): 930 if combined in config.keys(): 931 target_module = target 932 abi_to_run = { 933 ("arm", 32): 'armeabi-v7a', 934 ("arm", 64): 'arm64-v8a', 935 ("x86", 32): 'x86', 936 ("x86", 64): 'x86_64' 937 }.get((abi, abi_bits), None) 938 subplan = None 939 if _CTSHARDWARE_COLLECT in modules or _PUBLIC_CTSHARDWARE_COLLECT in modules: 940 subplan = 'cts-hardware' 941 return _CONTROLFILE_TEMPLATE.render( 942 year=CONFIG['COPYRIGHT_YEAR'], 943 name=name, 944 base_name=CONFIG['TEST_NAME'], 945 test_func_name=CONFIG['CONTROLFILE_TEST_FUNCTION_NAME'], 946 attributes=attributes, 947 dependencies=get_dependencies(modules, abi, is_public, 948 led_provision, camera_facing), 949 extra_artifacts=get_extra_artifacts(modules), 950 extra_artifacts_host=get_extra_artifacts_host(modules), 951 job_retries=get_job_retries(modules, is_public, suites), 952 max_result_size_kb=get_max_result_size_kb(modules, is_public), 953 revision=revision, 954 build=build, 955 abi=abi, 956 needs_push_media=needs_push_media(modules), 957 needs_cts_helpers=needs_cts_helpers(modules), 958 enable_default_apps=enable_default_apps(modules), 959 tag=tag, 960 uri=uri, 961 DOC=get_doc(modules, abi, is_public), 962 servo_support_needed=servo_support_needed(modules, is_public), 963 wifi_info_needed=wifi_info_needed(modules, is_public), 964 max_retries=get_max_retries(modules, abi, suites, is_public), 965 timeout=calculate_timeout(modules, suites), 966 run_template=get_run_template(modules, 967 is_public, 968 abi_to_run=CONFIG.get( 969 'REPRESENTATIVE_ABI', 970 {}).get(abi, abi_to_run), 971 whole_module_set=whole_module_set, 972 is_hardware=hardware_suite), 973 retry_template=get_retry_template(modules, is_public), 974 target_module=target_module, 975 target_plan=subplan, 976 test_length=get_test_length(modules), 977 priority=get_test_priority(modules, is_public), 978 extra_args=get_extra_args(modules, is_public), 979 authkey=get_authkey(is_public), 980 sync_count=get_sync_count(modules, abi, is_public), 981 camera_facing=camera_facing) 982 983 984def get_tradefed_data(path, is_public, abi): 985 """Queries tradefed to provide us with a list of modules. 986 987 Notice that the parsing gets broken at times with major new CTS drops. 988 """ 989 tradefed = os.path.join(path, CONFIG['TRADEFED_EXECUTABLE_PATH']) 990 # Python's zipfle module does not set the executable bit. 991 # tradefed and java command need chmod +x. 992 os.chmod(tradefed, os.stat(tradefed).st_mode | stat.S_IEXEC) 993 java = CONFIG.get('JAVA_EXECUTABLE_PATH', None) 994 if java: 995 java = os.path.join(path, java) 996 os.chmod(java, os.stat(java).st_mode | stat.S_IEXEC) 997 cmd_list = [tradefed, 'list', 'modules'] 998 logging.info('Calling tradefed for list of modules.') 999 with open(os.devnull, 'w') as devnull: 1000 # tradefed terminates itself if stdin is not a tty. 1001 tradefed_output = subprocess.check_output(cmd_list, 1002 stdin=devnull).decode() 1003 1004 _ABI_PREFIXES = ('arm', 'x86') 1005 _MODULE_PREFIXES = ('Cts', 'cts-', 'signed-Cts', 'vm-tests-tf', 'Sts') 1006 1007 # Some CTS/GTS versions insert extra linebreaks due to a bug b/196912758. 1008 # Below is a heurestical workaround for the situation. 1009 lines = [] 1010 prev_line_abi_prefixed = False 1011 for line in tradefed_output.splitlines(): 1012 abi_prefixed = line.startswith(_ABI_PREFIXES) 1013 end_of_modules = (len(line) == 0 or 'Saved log to' in line) 1014 if prev_line_abi_prefixed and not end_of_modules and not abi_prefixed: 1015 # Merge a line immediately following 'abi XtsModuleName' 1016 lines[-1] += line 1017 else: 1018 lines.append(line) 1019 prev_line_abi_prefixed = abi_prefixed 1020 1021 modules = set() 1022 build = '<unknown>' 1023 revision = None 1024 for line in lines: 1025 # Android Compatibility Test Suite 7.0 (3423912) 1026 if (line.startswith('Android Compatibility Test Suite ') 1027 or line.startswith('Android Google ') 1028 or line.startswith('Android Vendor Test Suite') 1029 or line.startswith('Android Security Test Suite')): 1030 logging.info('Unpacking: %s.', line) 1031 build = get_tradefed_build(line) 1032 revision = get_tradefed_revision(line) 1033 elif line.startswith(_ABI_PREFIXES): 1034 # Newer CTS shows ABI-module pairs like "arm64-v8a CtsNetTestCases" 1035 line = line.split()[1] 1036 if line not in CONFIG.get('EXCLUDE_MODULES', []): 1037 modules.add(line) 1038 elif line.startswith(_MODULE_PREFIXES): 1039 # Old CTS plainly lists up the module name 1040 modules.add(line) 1041 elif line.isspace() or line.startswith('Use "help"'): 1042 pass 1043 else: 1044 logging.warning('Ignoring "%s"', line) 1045 1046 if not modules: 1047 raise Exception("no modules found.") 1048 return list(modules), build, revision 1049 1050 1051def download(uri, destination): 1052 """Download |uri| to local |destination|. 1053 1054 |destination| must be a file path (not a directory path).""" 1055 if uri.startswith('http://') or uri.startswith('https://'): 1056 subprocess.check_call(['wget', uri, '-O', destination]) 1057 elif uri.startswith('gs://'): 1058 subprocess.check_call(['gsutil', 'cp', uri, destination]) 1059 else: 1060 raise Exception 1061 1062 1063@contextlib.contextmanager 1064def pushd(d): 1065 """Defines pushd.""" 1066 current = os.getcwd() 1067 os.chdir(d) 1068 try: 1069 yield 1070 finally: 1071 os.chdir(current) 1072 1073 1074def unzip(filename, destination): 1075 """Unzips a zip file to the destination directory.""" 1076 with pushd(destination): 1077 # We are trusting Android to have a valid zip file for us. 1078 with zipfile.ZipFile(filename) as zf: 1079 zf.extractall() 1080 1081 1082def get_collect_modules(is_public, is_hardware=False): 1083 if is_public: 1084 if is_hardware: 1085 return set([_PUBLIC_CTSHARDWARE_COLLECT]) 1086 return set([_PUBLIC_COLLECT]) 1087 else: 1088 if is_hardware: 1089 return set([_CTSHARDWARE_COLLECT]) 1090 return set([_COLLECT]) 1091 1092 1093@contextlib.contextmanager 1094def TemporaryDirectory(prefix): 1095 """Poor man's python 3.2 import.""" 1096 tmp = tempfile.mkdtemp(prefix=prefix) 1097 try: 1098 yield tmp 1099 finally: 1100 shutil.rmtree(tmp) 1101 1102 1103def get_word_pattern(m, l=1): 1104 """Return the first few words of the CamelCase module name. 1105 1106 Break after l+1 CamelCase word. 1107 Example: CtsDebugTestCases -> CtsDebug. 1108 """ 1109 s = re.findall('^[a-z-_]+|[A-Z]*[^A-Z0-9]*', m)[0:l + 1] 1110 # Ignore Test or TestCases at the end as they don't add anything. 1111 if len(s) > l: 1112 if s[l].startswith('Test') or s[l].startswith('['): 1113 return ''.join(s[0:l]) 1114 if s[l - 1] == 'Test' and s[l].startswith('Cases'): 1115 return ''.join(s[0:l - 1]) 1116 return ''.join(s[0:l + 1]) 1117 1118 1119def combine_modules_by_common_word(modules): 1120 """Returns a dictionary of (combined name, set of module) pairs. 1121 1122 This gives a mild compaction of control files (from about 320 to 135). 1123 Example: 1124 'CtsVoice' -> ['CtsVoiceInteractionTestCases', 'CtsVoiceSettingsTestCases'] 1125 """ 1126 d = dict() 1127 # On first pass group modules with common first word together. 1128 for module in modules: 1129 pattern = get_word_pattern(module) 1130 v = d.get(pattern, []) 1131 v.append(module) 1132 v.sort() 1133 d[pattern] = v 1134 # Second pass extend names to maximum common prefix. This keeps control file 1135 # names identical if they contain only one module and less ambiguous if they 1136 # contain multiple modules. 1137 combined = dict() 1138 for key in sorted(d): 1139 # Instead if a one syllable prefix use longest common prefix of modules. 1140 prefix = os.path.commonprefix(d[key]) 1141 # Beautification: strip Tests/TestCases from end of prefix, but only if 1142 # there is more than one module in the control file. This avoids 1143 # slightly strange combination of having CtsDpiTestCases1/2 inside of 1144 # CtsDpiTestCases (now just CtsDpi to make it clearer there are several 1145 # modules in this control file). 1146 if len(d[key]) > 1: 1147 prefix = re.sub('TestCases$', '', prefix) 1148 prefix = re.sub('Tests$', '', prefix) 1149 # Beautification: CtsMedia files run very long and are unstable. Give 1150 # each module its own control file, even though this heuristic would 1151 # lump them together. 1152 if prefix.startswith('CtsMedia'): 1153 # Separate each CtsMedia* modules, but group extra modules with 1154 # optional parametrization (ex: secondary_user, instant) together. 1155 prev = ' ' 1156 for media in sorted(d[key]): 1157 if media.startswith(prev): 1158 combined[prev].add(media) 1159 else: 1160 prev = media 1161 combined[media] = set([media]) 1162 1163 else: 1164 combined[prefix] = set(d[key]) 1165 print('Reduced number of control files from %d to %d.' % (len(modules), 1166 len(combined))) 1167 return combined 1168 1169 1170def combine_modules_by_bookmark(modules): 1171 """Return a manually curated list of name, module pairs. 1172 1173 Ideally we split "all" into a dictionary of maybe 10-20 equal runtime parts. 1174 (Say 2-5 hours each.) But it is ok to run problematic modules alone. 1175 """ 1176 d = dict() 1177 # Figure out sets of modules between bookmarks. Not optimum time complexity. 1178 for bookmark in CONFIG['QUAL_BOOKMARKS']: 1179 if modules: 1180 for module in sorted(modules): 1181 if module < bookmark: 1182 v = d.get(bookmark, set()) 1183 v.add(module) 1184 d[bookmark] = v 1185 # Remove processed modules. 1186 if bookmark in d: 1187 modules = modules - d[bookmark] 1188 # Clean up names. 1189 combined = dict() 1190 for key in sorted(d): 1191 v = sorted(d[key]) 1192 # New name is first element '_-_' last element. 1193 # Notice there is a bug in $ADB_VENDOR_KEYS path name preventing 1194 # arbitrary characters. 1195 prefix = re.sub(r'\[[^]]*\]', '', v[0] + '_-_' + v[-1]) 1196 combined[prefix] = set(v) 1197 return combined 1198 1199 1200def write_controlfile(name, 1201 modules, 1202 abi, 1203 revision, 1204 build, 1205 uri, 1206 suites, 1207 source_type, 1208 whole_module_set=None, 1209 hardware_suite=False, 1210 abi_bits=None): 1211 """Write control files per each ABI or combined.""" 1212 is_public = (source_type == SourceType.MOBLAB) 1213 abi_bits_list = [] 1214 config_key = 'PUBLIC_SPLIT_BY_BITS_MODULES' if is_public else 'SPLIT_BY_BITS_MODULES' 1215 if modules & set(CONFIG.get(config_key, [])): 1216 # If |abi| is predefined (like CTS), splits the modules by 1217 # 32/64-bits. If not (like GTS) generate both arm and x86 jobs. 1218 for abi_arch in [abi] if abi else ['arm', 'x86']: 1219 for abi_bits in [32, 64]: 1220 abi_bits_list.append((abi_arch, abi_bits)) 1221 else: 1222 abi_bits_list.append((abi, None)) 1223 1224 for abi, abi_bits in abi_bits_list: 1225 filename = get_controlfile_name(name, 1226 abi, 1227 revision, 1228 is_public, 1229 abi_bits=abi_bits) 1230 content = get_controlfile_content(name, 1231 modules, 1232 abi, 1233 revision, 1234 build, 1235 uri, 1236 suites, 1237 source_type, 1238 hardware_suite=hardware_suite, 1239 whole_module_set=whole_module_set, 1240 abi_bits=abi_bits) 1241 with open(filename, 'w') as f: 1242 f.write(content) 1243 1244 1245def write_moblab_controlfiles(modules, abi, revision, build, uri): 1246 """Write all control files for moblab. 1247 1248 Nothing gets combined. 1249 1250 Moblab uses one module per job. In some cases like Deqp which can run super 1251 long it even creates several jobs per module. Moblab can do this as it has 1252 less relative overhead spinning up jobs than the lab. 1253 """ 1254 for module in modules: 1255 # No need to generate control files with extra suffix, since --module 1256 # option will cover variants with optional parameters. 1257 if is_parameterized_module(module): 1258 continue 1259 write_controlfile(module, 1260 set([module]), 1261 abi, 1262 revision, 1263 build, 1264 uri, 1265 None, 1266 source_type=SourceType.MOBLAB) 1267 1268 1269def write_regression_controlfiles(modules, abi, revision, build, uri, 1270 source_type): 1271 """Write all control files for stainless/ToT regression lab coverage. 1272 1273 Regression coverage on tot currently relies heavily on watching stainless 1274 dashboard and sponge. So instead of running everything in a single run 1275 we split CTS into many jobs. It used to be one job per module, but that 1276 became too much in P (more than 300 per ABI). Instead we combine modules 1277 with similar names and run these in the same job (alphabetically). 1278 """ 1279 1280 if CONFIG.get('SINGLE_CONTROL_FILE'): 1281 module_set = set(modules) 1282 write_controlfile('all', 1283 module_set, 1284 abi, 1285 revision, 1286 build, 1287 uri, 1288 None, 1289 source_type, 1290 whole_module_set=module_set) 1291 else: 1292 combined = combine_modules_by_common_word(set(modules)) 1293 for key in combined: 1294 write_controlfile(key, combined[key], abi, revision, build, uri, 1295 None, source_type) 1296 1297 1298def write_qualification_controlfiles(modules, abi, revision, build, uri, 1299 source_type): 1300 """Write all control files to run "all" tests for qualification. 1301 1302 Qualification was performed on N by running all tests using tradefed 1303 sharding (specifying SYNC_COUNT=2) in the control files. In skylab 1304 this is currently not implemented, so we fall back to autotest sharding 1305 all CTS tests into 10-20 hand chosen shards. 1306 """ 1307 combined = combine_modules_by_bookmark(set(modules)) 1308 for key in combined: 1309 if combined[key] & set(CONFIG.get('SPLIT_BY_BITS_MODULES', [])): 1310 # If |abi| is predefined (like CTS), splits the modules by 1311 # 32/64-bits. If not (like GTS) generate both arm and x86 jobs. 1312 for abi_arch in [abi] if abi else ['arm', 'x86']: 1313 for abi_bits in [32, 64]: 1314 write_controlfile('all.' + key, 1315 combined[key], 1316 abi_arch, 1317 revision, 1318 build, 1319 uri, 1320 CONFIG.get('QUAL_SUITE_NAMES'), 1321 source_type, 1322 abi_bits=abi_bits) 1323 else: 1324 write_controlfile('all.' + key, combined[key], abi, 1325 revision, build, uri, 1326 CONFIG.get('QUAL_SUITE_NAMES'), source_type) 1327 1328 1329def write_qualification_and_regression_controlfile(modules, abi, revision, 1330 build, uri, source_type): 1331 """Write a control file to run "all" tests for qualification and regression. 1332 """ 1333 # For cts-instant, qualication control files are expected to cover 1334 # regressions as well. Hence the 'suite:arc-cts' is added. 1335 suites = ['suite:arc-cts', 'suite:arc-cts-qual'] 1336 module_set = set(modules) 1337 combined = combine_modules_by_bookmark(module_set) 1338 for key in combined: 1339 write_controlfile('all.' + key, 1340 combined[key], 1341 abi, 1342 revision, 1343 build, 1344 uri, 1345 suites, 1346 source_type, 1347 whole_module_set=module_set) 1348 1349 1350def write_collect_controlfiles(_modules, 1351 abi, 1352 revision, 1353 build, 1354 uri, 1355 source_type, 1356 is_hardware=False): 1357 """Write all control files for test collection used as reference to 1358 1359 compute completeness (missing tests) on the CTS dashboard. 1360 """ 1361 is_public = (source_type == SourceType.MOBLAB) 1362 if is_public: 1363 if is_hardware: 1364 suites = [CONFIG['MOBLAB_HARDWARE_SUITE_NAME']] 1365 else: 1366 suites = [CONFIG['MOBLAB_SUITE_NAME']] 1367 else: 1368 if is_hardware: 1369 suites = [CONFIG['HARDWARE_SUITE_NAME']] 1370 else: 1371 suites = CONFIG['INTERNAL_SUITE_NAMES'] \ 1372 + CONFIG.get('QUAL_SUITE_NAMES', []) 1373 for module in get_collect_modules(is_public, is_hardware=is_hardware): 1374 write_controlfile(module, 1375 set([module]), 1376 abi, 1377 revision, 1378 build, 1379 uri, 1380 suites, 1381 source_type, 1382 hardware_suite=is_hardware) 1383 1384 1385def write_extra_controlfiles(_modules, abi, revision, build, uri, source_type): 1386 """Write all extra control files as specified in config. 1387 1388 This is used by moblab to load balance large modules like Deqp, as well as 1389 making custom modules such as WM presubmit. A similar approach was also used 1390 during bringup of grunt to split media tests. 1391 """ 1392 for module, config in get_extra_modules_dict(source_type, abi).items(): 1393 for submodule, suites in config.items(): 1394 write_controlfile(submodule, set([submodule]), abi, revision, 1395 build, uri, suites, source_type) 1396 1397 1398def write_hardwaresuite_controlfiles(abi, revision, build, uri, source_type): 1399 """Control files for Build variant hardware only tests.""" 1400 is_public = (source_type == SourceType.MOBLAB) 1401 cts_hardware_modules = set(CONFIG.get('HARDWARE_MODULES', [])) 1402 for module in cts_hardware_modules: 1403 name = get_controlfile_name(module, abi, revision, is_public, 1404 hardware_suite=True) 1405 1406 content = get_controlfile_content(module, 1407 set([module]), 1408 abi, 1409 revision, 1410 build, 1411 uri, 1412 None, 1413 source_type, 1414 hardware_suite=True) 1415 1416 with open(name, 'w') as f: 1417 f.write(content) 1418 1419 for module, config in get_extra_hardware_modules_dict(is_public, abi).items(): 1420 for submodule, suites in config.items(): 1421 name = get_controlfile_name(submodule, abi, revision, is_public, 1422 hardware_suite=True) 1423 content = get_controlfile_content(submodule, 1424 set([submodule]), 1425 abi, 1426 revision, 1427 build, 1428 uri, 1429 None, 1430 source_type, 1431 hardware_suite=True) 1432 with open(name, 'w') as f: 1433 f.write(content) 1434 1435 1436def write_extra_camera_controlfiles(abi, revision, build, uri, source_type): 1437 """Control files for CtsCameraTestCases.camerabox.*""" 1438 module = 'CtsCameraTestCases' 1439 is_public = (source_type == SourceType.MOBLAB) 1440 for facing in ['back', 'front']: 1441 led_provision = 'noled' 1442 name = get_controlfile_name(module, abi, revision, is_public, 1443 led_provision, facing) 1444 content = get_controlfile_content(module, 1445 set([module]), 1446 abi, 1447 revision, 1448 build, 1449 uri, 1450 None, 1451 source_type, 1452 led_provision=led_provision, 1453 camera_facing=facing) 1454 with open(name, 'w') as f: 1455 f.write(content) 1456 1457 1458def run(uris, source_type, cache_dir): 1459 """Downloads each bundle in |uris| and generates control files for each 1460 1461 module as reported to us by tradefed. 1462 """ 1463 for uri in uris: 1464 abi = get_bundle_abi(uri) 1465 is_public = (source_type == SourceType.MOBLAB) 1466 # Get tradefed data by downloading & unzipping the files 1467 with TemporaryDirectory(prefix='cts-android_') as tmp: 1468 if cache_dir is not None: 1469 assert(os.path.isdir(cache_dir)) 1470 bundle = os.path.join(cache_dir, os.path.basename(uri)) 1471 if not os.path.exists(bundle): 1472 logging.info('Downloading to %s.', cache_dir) 1473 download(uri, bundle) 1474 else: 1475 bundle = os.path.join(tmp, os.path.basename(uri)) 1476 logging.info('Downloading to %s.', tmp) 1477 download(uri, bundle) 1478 logging.info('Extracting %s.', bundle) 1479 unzip(bundle, tmp) 1480 modules, build, revision = get_tradefed_data(tmp, is_public, abi) 1481 if not revision: 1482 raise Exception('Could not determine revision.') 1483 1484 logging.info('Writing all control files.') 1485 if source_type == SourceType.MOBLAB: 1486 write_moblab_controlfiles(modules, abi, revision, build, uri) 1487 1488 if CONFIG['CONTROLFILE_WRITE_SIMPLE_QUAL_AND_REGRESS']: 1489 # Might be worth generating DEV control files, but since this 1490 # is used for only ARC-P CTS_Instant modules whose regression 1491 # is 99.99% coverved by CTS DEV runs, having only LATEST is 1492 # sufficient. 1493 if source_type == SourceType.LATEST: 1494 write_qualification_and_regression_controlfile( 1495 modules, abi, revision, build, uri, source_type) 1496 else: 1497 if source_type == SourceType.DEV: 1498 write_regression_controlfiles(modules, abi, revision, 1499 build, uri, source_type) 1500 if source_type == SourceType.LATEST: 1501 write_qualification_controlfiles(modules, abi, revision, 1502 build, uri, source_type) 1503 1504 if CONFIG['CONTROLFILE_WRITE_CAMERA']: 1505 # For now camerabox is not stable for qualification purpose. 1506 # Hence, the usage is limited to DEV. In the future we need 1507 # to reconsider. 1508 if source_type == SourceType.DEV: 1509 write_extra_camera_controlfiles(abi, revision, build, uri, 1510 source_type) 1511 1512 if CONFIG.get('CONTROLFILE_WRITE_COLLECT', True): 1513 # Collect-test control files are used for checking the test 1514 # completeness before qualification. Not needed for DEV. 1515 if source_type == SourceType.LATEST or source_type == SourceType.MOBLAB: 1516 for_hardware_suite = [False] 1517 if 'HARDWARE_MODULES' in CONFIG: 1518 for_hardware_suite.append(True) 1519 for is_hardware in for_hardware_suite: 1520 write_collect_controlfiles(modules, 1521 abi, 1522 revision, 1523 build, 1524 uri, 1525 source_type, 1526 is_hardware=is_hardware) 1527 1528 if CONFIG['CONTROLFILE_WRITE_EXTRA']: 1529 # "EXTRA" control files are for workaround test instability 1530 # by running only sub-tests. For now let's attribute them to 1531 # qualification suites, since it is sometimes critical to 1532 # have the stability for qualification. If needed we need to 1533 # implement some way to add them to DEV suites as well. 1534 if source_type == SourceType.LATEST or source_type == SourceType.MOBLAB: 1535 write_extra_controlfiles(None, abi, revision, build, uri, 1536 source_type) 1537 1538 # "Hardware only" jobs are for reducing tests on qualification. 1539 if source_type == SourceType.LATEST or source_type == SourceType.MOBLAB: 1540 write_hardwaresuite_controlfiles(abi, revision, build, uri, 1541 source_type) 1542 1543 1544def main(config): 1545 """ Entry method of generator """ 1546 1547 global CONFIG 1548 CONFIG = config 1549 1550 logging.basicConfig(level=logging.INFO) 1551 parser = argparse.ArgumentParser( 1552 description='Create control files for a CTS bundle on GS.', 1553 formatter_class=argparse.RawTextHelpFormatter) 1554 parser.add_argument( 1555 'uris', 1556 nargs='+', 1557 help='List of Google Storage URIs to CTS bundles. Example:\n' 1558 'gs://chromeos-arc-images/cts/bundle/P/' 1559 'android-cts-9.0_r9-linux_x86-x86.zip') 1560 parser.add_argument( 1561 '--is_public', 1562 dest='is_public', 1563 default=False, 1564 action='store_true', 1565 help='Generate the public control files for CTS, default generate' 1566 ' the internal control files') 1567 parser.add_argument( 1568 '--is_latest', 1569 dest='is_latest', 1570 default=False, 1571 action='store_true', 1572 help='Generate the control files for CTS from the latest CTS bundle' 1573 ' stored in the internal storage') 1574 parser.add_argument( 1575 '--cache_dir', 1576 dest='cache_dir', 1577 default=None, 1578 action='store', 1579 help='Cache directory for downloaded bundle file. Uses the cached ' 1580 'bundle file if exists, or caches a downloaded file to this ' 1581 'directory if not.') 1582 args = parser.parse_args() 1583 if args.is_public: 1584 source_type = SourceType.MOBLAB 1585 elif args.is_latest: 1586 source_type = SourceType.LATEST 1587 else: 1588 source_type = SourceType.DEV 1589 run(args.uris, source_type, args.cache_dir) 1590