1*9c5db199SXin Li# Lint as: python2, python3 2*9c5db199SXin Li# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 3*9c5db199SXin Li# Use of this source code is governed by a BSD-style license that can be 4*9c5db199SXin Li# found in the LICENSE file. 5*9c5db199SXin Li 6*9c5db199SXin Lifrom __future__ import absolute_import 7*9c5db199SXin Lifrom __future__ import division 8*9c5db199SXin Lifrom __future__ import print_function 9*9c5db199SXin Li 10*9c5db199SXin Liimport abc 11*9c5db199SXin Liimport datetime 12*9c5db199SXin Liimport difflib 13*9c5db199SXin Liimport functools 14*9c5db199SXin Liimport hashlib 15*9c5db199SXin Liimport logging 16*9c5db199SXin Liimport operator 17*9c5db199SXin Liimport os 18*9c5db199SXin Liimport re 19*9c5db199SXin Liimport six 20*9c5db199SXin Liimport sys 21*9c5db199SXin Liimport warnings 22*9c5db199SXin Li 23*9c5db199SXin Liimport common 24*9c5db199SXin Li 25*9c5db199SXin Lifrom autotest_lib.frontend.afe.json_rpc import proxy 26*9c5db199SXin Lifrom autotest_lib.client.common_lib import autotest_enum 27*9c5db199SXin Lifrom autotest_lib.client.common_lib import error 28*9c5db199SXin Lifrom autotest_lib.client.common_lib import global_config 29*9c5db199SXin Lifrom autotest_lib.client.common_lib import priorities 30*9c5db199SXin Lifrom autotest_lib.client.common_lib import time_utils 31*9c5db199SXin Lifrom autotest_lib.client.common_lib import utils 32*9c5db199SXin Lifrom autotest_lib.frontend.afe import model_attributes 33*9c5db199SXin Lifrom autotest_lib.frontend.afe.json_rpc import proxy 34*9c5db199SXin Lifrom autotest_lib.server.cros import provision 35*9c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import constants 36*9c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import control_file_getter 37*9c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import frontend_wrappers 38*9c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import job_status 39*9c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import suite_common 40*9c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import tools 41*9c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite.job_status import Status 42*9c5db199SXin Li 43*9c5db199SXin Litry: 44*9c5db199SXin Li from autotest_lib.server.cros.dynamic_suite import boolparse_lib 45*9c5db199SXin Liexcept ImportError as e: 46*9c5db199SXin Li print('Unable to import boolparse_lib: %s' % (e,)) 47*9c5db199SXin Li print('This script must be either:') 48*9c5db199SXin Li print(' - Be run in the chroot.') 49*9c5db199SXin Li print(' - (not yet supported) be run after running ') 50*9c5db199SXin Li print(' ../utils/build_externals.py') 51*9c5db199SXin Li 52*9c5db199SXin Li_FILE_BUG_SUITES = [ 53*9c5db199SXin Li 'au', 'bvt', 'bvt-cq', 'bvt-inline', 'calibration', 'paygen_au_beta', 54*9c5db199SXin Li 'paygen_au_canary', 'paygen_au_dev', 'paygen_au_stable', 'sanity', 55*9c5db199SXin Li 'push_to_prod' 56*9c5db199SXin Li] 57*9c5db199SXin Li_AUTOTEST_DIR = global_config.global_config.get_config_value( 58*9c5db199SXin Li 'SCHEDULER', 'drone_installation_directory') 59*9c5db199SXin Li 60*9c5db199SXin Li 61*9c5db199SXin Liclass RetryHandler(object): 62*9c5db199SXin Li """Maintain retry information. 63*9c5db199SXin Li 64*9c5db199SXin Li @var _retry_map: A dictionary that stores retry history. 65*9c5db199SXin Li The key is afe job id. The value is a dictionary. 66*9c5db199SXin Li {job_id: {'state':RetryHandler.States, 'retry_max':int}} 67*9c5db199SXin Li - state: 68*9c5db199SXin Li The retry state of a job. 69*9c5db199SXin Li NOT_ATTEMPTED: 70*9c5db199SXin Li We haven't done anything about the job. 71*9c5db199SXin Li ATTEMPTED: 72*9c5db199SXin Li We've made an attempt to schedule a retry job. The 73*9c5db199SXin Li scheduling may or may not be successful, e.g. 74*9c5db199SXin Li it might encounter an rpc error. Note failure 75*9c5db199SXin Li in scheduling a retry is different from a retry job failure. 76*9c5db199SXin Li For each job, we only attempt to schedule a retry once. 77*9c5db199SXin Li For example, assume we have a test with JOB_RETRIES=5 and 78*9c5db199SXin Li its second retry job failed. When we attempt to create 79*9c5db199SXin Li a third retry job to retry the second, we hit an rpc 80*9c5db199SXin Li error. In such case, we will give up on all following 81*9c5db199SXin Li retries. 82*9c5db199SXin Li RETRIED: 83*9c5db199SXin Li A retry job has already been successfully 84*9c5db199SXin Li scheduled. 85*9c5db199SXin Li - retry_max: 86*9c5db199SXin Li The maximum of times the job can still 87*9c5db199SXin Li be retried, taking into account retries 88*9c5db199SXin Li that have occurred. 89*9c5db199SXin Li @var _retry_level: A retry might be triggered only if the result 90*9c5db199SXin Li is worse than the level. 91*9c5db199SXin Li @var _max_retries: Maximum retry limit at suite level. 92*9c5db199SXin Li Regardless how many times each individual test 93*9c5db199SXin Li has been retried, the total number of retries happening in 94*9c5db199SXin Li the suite can't exceed _max_retries. 95*9c5db199SXin Li """ 96*9c5db199SXin Li 97*9c5db199SXin Li States = autotest_enum.AutotestEnum('NOT_ATTEMPTED', 'ATTEMPTED', 'RETRIED', 98*9c5db199SXin Li start_value=1, step=1) 99*9c5db199SXin Li 100*9c5db199SXin Li def __init__(self, initial_jobs_to_tests, retry_level='WARN', 101*9c5db199SXin Li max_retries=None): 102*9c5db199SXin Li """Initialize RetryHandler. 103*9c5db199SXin Li 104*9c5db199SXin Li @param initial_jobs_to_tests: A dictionary that maps a job id to 105*9c5db199SXin Li a ControlData object. This dictionary should contain 106*9c5db199SXin Li jobs that are originally scheduled by the suite. 107*9c5db199SXin Li @param retry_level: A retry might be triggered only if the result is 108*9c5db199SXin Li worse than the level. 109*9c5db199SXin Li @param max_retries: Integer, maxmium total retries allowed 110*9c5db199SXin Li for the suite. Default to None, no max. 111*9c5db199SXin Li """ 112*9c5db199SXin Li self._retry_map = {} 113*9c5db199SXin Li self._retry_level = retry_level 114*9c5db199SXin Li self._max_retries = (max_retries 115*9c5db199SXin Li if max_retries is not None else sys.maxsize) 116*9c5db199SXin Li for job_id, test in initial_jobs_to_tests.items(): 117*9c5db199SXin Li if test.job_retries > 0: 118*9c5db199SXin Li self._add_job(new_job_id=job_id, 119*9c5db199SXin Li retry_max=test.job_retries) 120*9c5db199SXin Li else: 121*9c5db199SXin Li logging.debug("Test %s has no retries", test.name) 122*9c5db199SXin Li 123*9c5db199SXin Li 124*9c5db199SXin Li def _add_job(self, new_job_id, retry_max): 125*9c5db199SXin Li """Add a newly-created job to the retry map. 126*9c5db199SXin Li 127*9c5db199SXin Li @param new_job_id: The afe_job_id of a newly created job. 128*9c5db199SXin Li @param retry_max: The maximum of times that we could retry 129*9c5db199SXin Li the test if the job fails. 130*9c5db199SXin Li 131*9c5db199SXin Li @raises ValueError if new_job_id is already in retry map. 132*9c5db199SXin Li 133*9c5db199SXin Li """ 134*9c5db199SXin Li if new_job_id in self._retry_map: 135*9c5db199SXin Li raise ValueError('add_job called when job is already in retry map.') 136*9c5db199SXin Li 137*9c5db199SXin Li self._retry_map[new_job_id] = { 138*9c5db199SXin Li 'state': self.States.NOT_ATTEMPTED, 139*9c5db199SXin Li 'retry_max': retry_max} 140*9c5db199SXin Li 141*9c5db199SXin Li 142*9c5db199SXin Li def _suite_max_reached(self): 143*9c5db199SXin Li """Return whether maximum retry limit for a suite has been reached.""" 144*9c5db199SXin Li return self._max_retries <= 0 145*9c5db199SXin Li 146*9c5db199SXin Li 147*9c5db199SXin Li def add_retry(self, old_job_id, new_job_id): 148*9c5db199SXin Li """Record a retry. 149*9c5db199SXin Li 150*9c5db199SXin Li Update retry map with the retry information. 151*9c5db199SXin Li 152*9c5db199SXin Li @param old_job_id: The afe_job_id of the job that is retried. 153*9c5db199SXin Li @param new_job_id: The afe_job_id of the retry job. 154*9c5db199SXin Li 155*9c5db199SXin Li @raises KeyError if old_job_id isn't in the retry map. 156*9c5db199SXin Li @raises ValueError if we have already retried or made an attempt 157*9c5db199SXin Li to retry the old job. 158*9c5db199SXin Li 159*9c5db199SXin Li """ 160*9c5db199SXin Li old_record = self._retry_map[old_job_id] 161*9c5db199SXin Li if old_record['state'] != self.States.NOT_ATTEMPTED: 162*9c5db199SXin Li raise ValueError( 163*9c5db199SXin Li 'We have already retried or attempted to retry job %d' % 164*9c5db199SXin Li old_job_id) 165*9c5db199SXin Li old_record['state'] = self.States.RETRIED 166*9c5db199SXin Li self._add_job(new_job_id=new_job_id, 167*9c5db199SXin Li retry_max=old_record['retry_max'] - 1) 168*9c5db199SXin Li self._max_retries -= 1 169*9c5db199SXin Li 170*9c5db199SXin Li 171*9c5db199SXin Li def set_attempted(self, job_id): 172*9c5db199SXin Li """Set the state of the job to ATTEMPTED. 173*9c5db199SXin Li 174*9c5db199SXin Li @param job_id: afe_job_id of a job. 175*9c5db199SXin Li 176*9c5db199SXin Li @raises KeyError if job_id isn't in the retry map. 177*9c5db199SXin Li @raises ValueError if the current state is not NOT_ATTEMPTED. 178*9c5db199SXin Li 179*9c5db199SXin Li """ 180*9c5db199SXin Li current_state = self._retry_map[job_id]['state'] 181*9c5db199SXin Li if current_state != self.States.NOT_ATTEMPTED: 182*9c5db199SXin Li # We are supposed to retry or attempt to retry each job 183*9c5db199SXin Li # only once. Raise an error if this is not the case. 184*9c5db199SXin Li raise ValueError('Unexpected state transition: %s -> %s' % 185*9c5db199SXin Li (self.States.get_string(current_state), 186*9c5db199SXin Li self.States.get_string(self.States.ATTEMPTED))) 187*9c5db199SXin Li else: 188*9c5db199SXin Li self._retry_map[job_id]['state'] = self.States.ATTEMPTED 189*9c5db199SXin Li 190*9c5db199SXin Li 191*9c5db199SXin Li def has_following_retry(self, result): 192*9c5db199SXin Li """Check whether there will be a following retry. 193*9c5db199SXin Li 194*9c5db199SXin Li We have the following cases for a given job id (result.id), 195*9c5db199SXin Li - no retry map entry -> retry not required, no following retry 196*9c5db199SXin Li - has retry map entry: 197*9c5db199SXin Li - already retried -> has following retry 198*9c5db199SXin Li - has not retried 199*9c5db199SXin Li (this branch can be handled by checking should_retry(result)) 200*9c5db199SXin Li - retry_max == 0 --> the last retry job, no more retry 201*9c5db199SXin Li - retry_max > 0 202*9c5db199SXin Li - attempted, but has failed in scheduling a 203*9c5db199SXin Li following retry due to rpc error --> no more retry 204*9c5db199SXin Li - has not attempped --> has following retry if test failed. 205*9c5db199SXin Li 206*9c5db199SXin Li @param result: A result, encapsulating the status of the job. 207*9c5db199SXin Li 208*9c5db199SXin Li @returns: True, if there will be a following retry. 209*9c5db199SXin Li False otherwise. 210*9c5db199SXin Li 211*9c5db199SXin Li """ 212*9c5db199SXin Li return (result.test_executed 213*9c5db199SXin Li and result.id in self._retry_map 214*9c5db199SXin Li and (self._retry_map[result.id]['state'] == self.States.RETRIED 215*9c5db199SXin Li or self._should_retry(result))) 216*9c5db199SXin Li 217*9c5db199SXin Li 218*9c5db199SXin Li def _should_retry(self, result): 219*9c5db199SXin Li """Check whether we should retry a job based on its result. 220*9c5db199SXin Li 221*9c5db199SXin Li We will retry the job that corresponds to the result 222*9c5db199SXin Li when all of the following are true. 223*9c5db199SXin Li a) The test was actually executed, meaning that if 224*9c5db199SXin Li a job was aborted before it could ever reach the state 225*9c5db199SXin Li of 'Running', the job will not be retried. 226*9c5db199SXin Li b) The result is worse than |self._retry_level| which 227*9c5db199SXin Li defaults to 'WARN'. 228*9c5db199SXin Li c) The test requires retry, i.e. the job has an entry in the retry map. 229*9c5db199SXin Li d) We haven't made any retry attempt yet, i.e. state == NOT_ATTEMPTED 230*9c5db199SXin Li Note that if a test has JOB_RETRIES=5, and the second time 231*9c5db199SXin Li it was retried it hit an rpc error, we will give up on 232*9c5db199SXin Li all following retries. 233*9c5db199SXin Li e) The job has not reached its retry max, i.e. retry_max > 0 234*9c5db199SXin Li 235*9c5db199SXin Li @param result: A result, encapsulating the status of the job. 236*9c5db199SXin Li 237*9c5db199SXin Li @returns: True if we should retry the job. 238*9c5db199SXin Li 239*9c5db199SXin Li """ 240*9c5db199SXin Li return ( 241*9c5db199SXin Li result.test_executed 242*9c5db199SXin Li and result.id in self._retry_map 243*9c5db199SXin Li and not self._suite_max_reached() 244*9c5db199SXin Li and result.is_worse_than( 245*9c5db199SXin Li job_status.Status(self._retry_level, '', 'reason')) 246*9c5db199SXin Li and self._retry_map[result.id]['state'] == self.States.NOT_ATTEMPTED 247*9c5db199SXin Li and self._retry_map[result.id]['retry_max'] > 0 248*9c5db199SXin Li ) 249*9c5db199SXin Li 250*9c5db199SXin Li def _should_retry_local_job(self, job_id): 251*9c5db199SXin Li """Check whether we should retry a job based on information available 252*9c5db199SXin Li for a local job without a Result object. 253*9c5db199SXin Li 254*9c5db199SXin Li We will retry the job that corresponds to the result 255*9c5db199SXin Li when all of the following are true. 256*9c5db199SXin Li a) The test requires retry, i.e. the job has an entry in the retry map. 257*9c5db199SXin Li b) We haven't made any retry attempt yet for this job, i.e. 258*9c5db199SXin Li state == NOT_ATTEMPTED 259*9c5db199SXin Li If the job is aborted, we will give up on all following retries, 260*9c5db199SXin Li regardless of max_retries. 261*9c5db199SXin Li c) The job has not reached its retry max, i.e. retry_max > 0 262*9c5db199SXin Li 263*9c5db199SXin Li @param job_id: the id for the job, to look up relevant information. 264*9c5db199SXin Li 265*9c5db199SXin Li @returns: True if we should retry the job. 266*9c5db199SXin Li 267*9c5db199SXin Li """ 268*9c5db199SXin Li if self._suite_max_reached(): 269*9c5db199SXin Li logging.debug('suite max_retries reached, not retrying.') 270*9c5db199SXin Li return False 271*9c5db199SXin Li if job_id not in self._retry_map: 272*9c5db199SXin Li logging.debug('job_id not in retry map, not retrying.') 273*9c5db199SXin Li return False 274*9c5db199SXin Li if self._retry_map[job_id]['state'] != self.States.NOT_ATTEMPTED: 275*9c5db199SXin Li logging.debug("job state was %s not 'Not Attempted', not retrying", 276*9c5db199SXin Li self._retry_map[job_id]['state']) 277*9c5db199SXin Li return False 278*9c5db199SXin Li if self._retry_map[job_id]['retry_max'] <= 0: 279*9c5db199SXin Li logging.debug('test-level retries exhausted, not retrying') 280*9c5db199SXin Li return False 281*9c5db199SXin Li return True 282*9c5db199SXin Li 283*9c5db199SXin Li 284*9c5db199SXin Li def job_present(self, job_id): 285*9c5db199SXin Li """Check whether a job id present in the retry map. 286*9c5db199SXin Li 287*9c5db199SXin Li @param job_id: afe_job_id of a job. 288*9c5db199SXin Li 289*9c5db199SXin Li @returns: A True if the job is present, False if not. 290*9c5db199SXin Li """ 291*9c5db199SXin Li return bool(self._retry_map.get(job_id)) 292*9c5db199SXin Li 293*9c5db199SXin Li 294*9c5db199SXin Li 295*9c5db199SXin Li def get_retry_max(self, job_id): 296*9c5db199SXin Li """Get the maximum times the job can still be retried. 297*9c5db199SXin Li 298*9c5db199SXin Li @param job_id: afe_job_id of a job. 299*9c5db199SXin Li 300*9c5db199SXin Li @returns: An int, representing the maximum times the job can still be 301*9c5db199SXin Li retried. 302*9c5db199SXin Li @raises KeyError if job_id isn't in the retry map. 303*9c5db199SXin Li 304*9c5db199SXin Li """ 305*9c5db199SXin Li return self._retry_map[job_id]['retry_max'] 306*9c5db199SXin Li 307*9c5db199SXin Li 308*9c5db199SXin Liclass _SuiteChildJobCreator(object): 309*9c5db199SXin Li """Create test jobs for a suite.""" 310*9c5db199SXin Li 311*9c5db199SXin Li def __init__( 312*9c5db199SXin Li self, 313*9c5db199SXin Li tag, 314*9c5db199SXin Li builds, 315*9c5db199SXin Li board, 316*9c5db199SXin Li afe=None, 317*9c5db199SXin Li max_runtime_mins=24*60, 318*9c5db199SXin Li timeout_mins=24*60, 319*9c5db199SXin Li suite_job_id=None, 320*9c5db199SXin Li ignore_deps=False, 321*9c5db199SXin Li extra_deps=(), 322*9c5db199SXin Li priority=priorities.Priority.DEFAULT, 323*9c5db199SXin Li offload_failures_only=False, 324*9c5db199SXin Li test_source_build=None, 325*9c5db199SXin Li job_keyvals=None, 326*9c5db199SXin Li ): 327*9c5db199SXin Li """ 328*9c5db199SXin Li Constructor 329*9c5db199SXin Li 330*9c5db199SXin Li @param tag: a string with which to tag jobs run in this suite. 331*9c5db199SXin Li @param builds: the builds on which we're running this suite. 332*9c5db199SXin Li @param board: the board on which we're running this suite. 333*9c5db199SXin Li @param afe: an instance of AFE as defined in server/frontend.py. 334*9c5db199SXin Li @param max_runtime_mins: Maximum suite runtime, in minutes. 335*9c5db199SXin Li @param timeout_mins: Maximum job lifetime, in minutes. 336*9c5db199SXin Li @param suite_job_id: Job id that will act as parent id to all sub jobs. 337*9c5db199SXin Li Default: None 338*9c5db199SXin Li @param ignore_deps: True if jobs should ignore the DEPENDENCIES 339*9c5db199SXin Li attribute and skip applying of dependency labels. 340*9c5db199SXin Li (Default:False) 341*9c5db199SXin Li @param extra_deps: A list of strings which are the extra DEPENDENCIES 342*9c5db199SXin Li to add to each test being scheduled. 343*9c5db199SXin Li @param priority: Integer priority level. Higher is more important. 344*9c5db199SXin Li @param offload_failures_only: Only enable gs_offloading for failed 345*9c5db199SXin Li jobs. 346*9c5db199SXin Li @param test_source_build: Build that contains the server-side test code. 347*9c5db199SXin Li @param job_keyvals: General job keyvals to be inserted into keyval file, 348*9c5db199SXin Li which will be used by tko/parse later. 349*9c5db199SXin Li """ 350*9c5db199SXin Li self._tag = tag 351*9c5db199SXin Li self._builds = builds 352*9c5db199SXin Li self._board = board 353*9c5db199SXin Li self._afe = afe or frontend_wrappers.RetryingAFE(timeout_min=30, 354*9c5db199SXin Li delay_sec=10, 355*9c5db199SXin Li debug=False) 356*9c5db199SXin Li self._max_runtime_mins = max_runtime_mins 357*9c5db199SXin Li self._timeout_mins = timeout_mins 358*9c5db199SXin Li self._suite_job_id = suite_job_id 359*9c5db199SXin Li self._ignore_deps = ignore_deps 360*9c5db199SXin Li self._extra_deps = tuple(extra_deps) 361*9c5db199SXin Li self._priority = priority 362*9c5db199SXin Li self._offload_failures_only = offload_failures_only 363*9c5db199SXin Li self._test_source_build = test_source_build 364*9c5db199SXin Li self._job_keyvals = job_keyvals 365*9c5db199SXin Li 366*9c5db199SXin Li 367*9c5db199SXin Li @property 368*9c5db199SXin Li def cros_build(self): 369*9c5db199SXin Li """Return the CrOS build or the first build in the builds dict.""" 370*9c5db199SXin Li # TODO(ayatane): Note that the builds dict isn't ordered. I'm not 371*9c5db199SXin Li # sure what the implications of this are, but it's probably not a 372*9c5db199SXin Li # good thing. 373*9c5db199SXin Li return self._builds.get(provision.CROS_VERSION_PREFIX, 374*9c5db199SXin Li list(self._builds.values())[0]) 375*9c5db199SXin Li 376*9c5db199SXin Li 377*9c5db199SXin Li def create_job(self, test, retry_for=None): 378*9c5db199SXin Li """ 379*9c5db199SXin Li Thin wrapper around frontend.AFE.create_job(). 380*9c5db199SXin Li 381*9c5db199SXin Li @param test: ControlData object for a test to run. 382*9c5db199SXin Li @param retry_for: If the to-be-created job is a retry for an 383*9c5db199SXin Li old job, the afe_job_id of the old job will 384*9c5db199SXin Li be passed in as |retry_for|, which will be 385*9c5db199SXin Li recorded in the new job's keyvals. 386*9c5db199SXin Li @returns: A frontend.Job object with an added test_name member. 387*9c5db199SXin Li test_name is used to preserve the higher level TEST_NAME 388*9c5db199SXin Li name of the job. 389*9c5db199SXin Li """ 390*9c5db199SXin Li # For a system running multiple suites which share tests, the priority 391*9c5db199SXin Li # overridden may lead to unexpected scheduling order that adds extra 392*9c5db199SXin Li # provision jobs. 393*9c5db199SXin Li test_priority = self._priority 394*9c5db199SXin Li if utils.is_moblab(): 395*9c5db199SXin Li test_priority = max(self._priority, test.priority) 396*9c5db199SXin Li 397*9c5db199SXin Li reboot_before = (model_attributes.RebootBefore.NEVER if test.fast 398*9c5db199SXin Li else None) 399*9c5db199SXin Li 400*9c5db199SXin Li test_obj = self._afe.create_job( 401*9c5db199SXin Li control_file=test.text, 402*9c5db199SXin Li name=tools.create_job_name( 403*9c5db199SXin Li self._test_source_build or self.cros_build, 404*9c5db199SXin Li self._tag, 405*9c5db199SXin Li test.name), 406*9c5db199SXin Li control_type=test.test_type.capitalize(), 407*9c5db199SXin Li meta_hosts=[self._board]*test.sync_count, 408*9c5db199SXin Li dependencies=self._create_job_deps(test), 409*9c5db199SXin Li keyvals=self._create_keyvals_for_test_job(test, retry_for), 410*9c5db199SXin Li max_runtime_mins=self._max_runtime_mins, 411*9c5db199SXin Li timeout_mins=self._timeout_mins, 412*9c5db199SXin Li parent_job_id=self._suite_job_id, 413*9c5db199SXin Li reboot_before=reboot_before, 414*9c5db199SXin Li run_reset=not test.fast, 415*9c5db199SXin Li priority=test_priority, 416*9c5db199SXin Li synch_count=test.sync_count, 417*9c5db199SXin Li require_ssp=test.require_ssp) 418*9c5db199SXin Li 419*9c5db199SXin Li test_obj.test_name = test.name 420*9c5db199SXin Li return test_obj 421*9c5db199SXin Li 422*9c5db199SXin Li 423*9c5db199SXin Li def _create_job_deps(self, test): 424*9c5db199SXin Li """Create job deps list for a test job. 425*9c5db199SXin Li 426*9c5db199SXin Li @returns: A list of dependency strings. 427*9c5db199SXin Li """ 428*9c5db199SXin Li if self._ignore_deps: 429*9c5db199SXin Li job_deps = [] 430*9c5db199SXin Li else: 431*9c5db199SXin Li job_deps = list(test.dependencies) 432*9c5db199SXin Li job_deps.extend(self._extra_deps) 433*9c5db199SXin Li return job_deps 434*9c5db199SXin Li 435*9c5db199SXin Li 436*9c5db199SXin Li def _create_keyvals_for_test_job(self, test, retry_for=None): 437*9c5db199SXin Li """Create keyvals dict for creating a test job. 438*9c5db199SXin Li 439*9c5db199SXin Li @param test: ControlData object for a test to run. 440*9c5db199SXin Li @param retry_for: If the to-be-created job is a retry for an 441*9c5db199SXin Li old job, the afe_job_id of the old job will 442*9c5db199SXin Li be passed in as |retry_for|, which will be 443*9c5db199SXin Li recorded in the new job's keyvals. 444*9c5db199SXin Li @returns: A keyvals dict for creating the test job. 445*9c5db199SXin Li """ 446*9c5db199SXin Li keyvals = { 447*9c5db199SXin Li constants.JOB_BUILD_KEY: self.cros_build, 448*9c5db199SXin Li constants.JOB_SUITE_KEY: self._tag, 449*9c5db199SXin Li constants.JOB_EXPERIMENTAL_KEY: test.experimental, 450*9c5db199SXin Li constants.JOB_BUILDS_KEY: self._builds 451*9c5db199SXin Li } 452*9c5db199SXin Li # test_source_build is saved to job_keyvals so scheduler can retrieve 453*9c5db199SXin Li # the build name from database when compiling autoserv commandline. 454*9c5db199SXin Li # This avoid a database change to add a new field in afe_jobs. 455*9c5db199SXin Li # 456*9c5db199SXin Li # Only add `test_source_build` to job keyvals if the build is different 457*9c5db199SXin Li # from the CrOS build or the job uses more than one build, e.g., both 458*9c5db199SXin Li # firmware and CrOS will be updated in the dut. 459*9c5db199SXin Li # This is for backwards compatibility, so the update Autotest code can 460*9c5db199SXin Li # compile an autoserv command line to run in a SSP container using 461*9c5db199SXin Li # previous builds. 462*9c5db199SXin Li if (self._test_source_build and 463*9c5db199SXin Li (self.cros_build != self._test_source_build or 464*9c5db199SXin Li len(self._builds) > 1)): 465*9c5db199SXin Li keyvals[constants.JOB_TEST_SOURCE_BUILD_KEY] = \ 466*9c5db199SXin Li self._test_source_build 467*9c5db199SXin Li for prefix, build in six.iteritems(self._builds): 468*9c5db199SXin Li if prefix == provision.FW_RW_VERSION_PREFIX: 469*9c5db199SXin Li keyvals[constants.FWRW_BUILD]= build 470*9c5db199SXin Li elif prefix == provision.FW_RO_VERSION_PREFIX: 471*9c5db199SXin Li keyvals[constants.FWRO_BUILD] = build 472*9c5db199SXin Li # Add suite job id to keyvals so tko parser can read it from keyval 473*9c5db199SXin Li # file. 474*9c5db199SXin Li if self._suite_job_id: 475*9c5db199SXin Li keyvals[constants.PARENT_JOB_ID] = self._suite_job_id 476*9c5db199SXin Li # We drop the old job's id in the new job's keyval file so that 477*9c5db199SXin Li # later our tko parser can figure out the retry relationship and 478*9c5db199SXin Li # invalidate the results of the old job in tko database. 479*9c5db199SXin Li if retry_for: 480*9c5db199SXin Li keyvals[constants.RETRY_ORIGINAL_JOB_ID] = retry_for 481*9c5db199SXin Li if self._offload_failures_only: 482*9c5db199SXin Li keyvals[constants.JOB_OFFLOAD_FAILURES_KEY] = True 483*9c5db199SXin Li if self._job_keyvals: 484*9c5db199SXin Li for key in constants.INHERITED_KEYVALS: 485*9c5db199SXin Li if key in self._job_keyvals: 486*9c5db199SXin Li keyvals[key] = self._job_keyvals[key] 487*9c5db199SXin Li return keyvals 488*9c5db199SXin Li 489*9c5db199SXin Li 490*9c5db199SXin Liclass _ControlFileRetriever(object): 491*9c5db199SXin Li """Retrieves control files. 492*9c5db199SXin Li 493*9c5db199SXin Li This returns control data instances, unlike control file getters 494*9c5db199SXin Li which simply return the control file text contents. 495*9c5db199SXin Li """ 496*9c5db199SXin Li 497*9c5db199SXin Li def __init__(self, cf_getter, forgiving_parser=True, run_prod_code=False, 498*9c5db199SXin Li test_args=None): 499*9c5db199SXin Li """Initialize instance. 500*9c5db199SXin Li 501*9c5db199SXin Li @param cf_getter: a control_file_getter.ControlFileGetter used to list 502*9c5db199SXin Li and fetch the content of control files 503*9c5db199SXin Li @param forgiving_parser: If False, will raise ControlVariableExceptions 504*9c5db199SXin Li if any are encountered when parsing control 505*9c5db199SXin Li files. Note that this can raise an exception 506*9c5db199SXin Li for syntax errors in unrelated files, because 507*9c5db199SXin Li we parse them before applying the predicate. 508*9c5db199SXin Li @param run_prod_code: If true, the retrieved tests will run the test 509*9c5db199SXin Li code that lives in prod aka the test code 510*9c5db199SXin Li currently on the lab servers by disabling 511*9c5db199SXin Li SSP for the discovered tests. 512*9c5db199SXin Li @param test_args: A dict of args to be seeded in test control file under 513*9c5db199SXin Li the name |args_dict|. 514*9c5db199SXin Li """ 515*9c5db199SXin Li self._cf_getter = cf_getter 516*9c5db199SXin Li self._forgiving_parser = forgiving_parser 517*9c5db199SXin Li self._run_prod_code = run_prod_code 518*9c5db199SXin Li self._test_args = test_args 519*9c5db199SXin Li 520*9c5db199SXin Li 521*9c5db199SXin Li def retrieve_for_test(self, test_name): 522*9c5db199SXin Li """Retrieve a test's control data. 523*9c5db199SXin Li 524*9c5db199SXin Li This ignores forgiving_parser because we cannot return a 525*9c5db199SXin Li forgiving value. 526*9c5db199SXin Li 527*9c5db199SXin Li @param test_name: Name of test to retrieve. 528*9c5db199SXin Li 529*9c5db199SXin Li @raises ControlVariableException: There is a syntax error in a 530*9c5db199SXin Li control file. 531*9c5db199SXin Li 532*9c5db199SXin Li @returns a ControlData object 533*9c5db199SXin Li """ 534*9c5db199SXin Li return suite_common.retrieve_control_data_for_test( 535*9c5db199SXin Li self._cf_getter, test_name) 536*9c5db199SXin Li 537*9c5db199SXin Li 538*9c5db199SXin Li def retrieve_for_suite(self, suite_name=''): 539*9c5db199SXin Li """Scan through all tests and find all tests. 540*9c5db199SXin Li 541*9c5db199SXin Li @param suite_name: If specified, this method will attempt to restrain 542*9c5db199SXin Li the search space to just this suite's control files. 543*9c5db199SXin Li 544*9c5db199SXin Li @raises ControlVariableException: If forgiving_parser is False and there 545*9c5db199SXin Li is a syntax error in a control file. 546*9c5db199SXin Li 547*9c5db199SXin Li @returns a dictionary of ControlData objects that based on given 548*9c5db199SXin Li parameters. 549*9c5db199SXin Li """ 550*9c5db199SXin Li tests = suite_common.retrieve_for_suite( 551*9c5db199SXin Li self._cf_getter, suite_name, self._forgiving_parser, 552*9c5db199SXin Li self._test_args) 553*9c5db199SXin Li if self._run_prod_code: 554*9c5db199SXin Li for test in six.itervalues(tests): 555*9c5db199SXin Li test.require_ssp = False 556*9c5db199SXin Li 557*9c5db199SXin Li return tests 558*9c5db199SXin Li 559*9c5db199SXin Li 560*9c5db199SXin Lidef list_all_suites(build, devserver, cf_getter=None): 561*9c5db199SXin Li """ 562*9c5db199SXin Li Parses all ControlData objects with a SUITE tag and extracts all 563*9c5db199SXin Li defined suite names. 564*9c5db199SXin Li 565*9c5db199SXin Li @param build: the build on which we're running this suite. 566*9c5db199SXin Li @param devserver: the devserver which contains the build. 567*9c5db199SXin Li @param cf_getter: control_file_getter.ControlFileGetter. Defaults to 568*9c5db199SXin Li using DevServerGetter. 569*9c5db199SXin Li 570*9c5db199SXin Li @return list of suites 571*9c5db199SXin Li """ 572*9c5db199SXin Li if cf_getter is None: 573*9c5db199SXin Li cf_getter = _create_ds_getter(build, devserver) 574*9c5db199SXin Li 575*9c5db199SXin Li suites = set() 576*9c5db199SXin Li predicate = lambda t: True 577*9c5db199SXin Li for test in find_and_parse_tests(cf_getter, predicate): 578*9c5db199SXin Li suites.update(test.suite_tag_parts) 579*9c5db199SXin Li return list(suites) 580*9c5db199SXin Li 581*9c5db199SXin Li 582*9c5db199SXin Lidef test_file_similarity_predicate(test_file_pattern): 583*9c5db199SXin Li """Returns predicate that gets the similarity based on a test's file 584*9c5db199SXin Li name pattern. 585*9c5db199SXin Li 586*9c5db199SXin Li Builds a predicate that takes in a parsed control file (a ControlData) 587*9c5db199SXin Li and returns a tuple of (file path, ratio), where ratio is the 588*9c5db199SXin Li similarity between the test file name and the given test_file_pattern. 589*9c5db199SXin Li 590*9c5db199SXin Li @param test_file_pattern: regular expression (string) to match against 591*9c5db199SXin Li control file names. 592*9c5db199SXin Li @return a callable that takes a ControlData and and returns a tuple of 593*9c5db199SXin Li (file path, ratio), where ratio is the similarity between the 594*9c5db199SXin Li test file name and the given test_file_pattern. 595*9c5db199SXin Li """ 596*9c5db199SXin Li return lambda t: ((None, 0) if not hasattr(t, 'path') else 597*9c5db199SXin Li (t.path, difflib.SequenceMatcher(a=t.path, 598*9c5db199SXin Li b=test_file_pattern).ratio())) 599*9c5db199SXin Li 600*9c5db199SXin Li 601*9c5db199SXin Lidef test_name_similarity_predicate(test_name): 602*9c5db199SXin Li """Returns predicate that matched based on a test's name. 603*9c5db199SXin Li 604*9c5db199SXin Li Builds a predicate that takes in a parsed control file (a ControlData) 605*9c5db199SXin Li and returns a tuple of (test name, ratio), where ratio is the similarity 606*9c5db199SXin Li between the test name and the given test_name. 607*9c5db199SXin Li 608*9c5db199SXin Li @param test_name: the test name to base the predicate on. 609*9c5db199SXin Li @return a callable that takes a ControlData and returns a tuple of 610*9c5db199SXin Li (test name, ratio), where ratio is the similarity between the 611*9c5db199SXin Li test name and the given test_name. 612*9c5db199SXin Li """ 613*9c5db199SXin Li return lambda t: ((None, 0) if not hasattr(t, 'name') else 614*9c5db199SXin Li (t.name, 615*9c5db199SXin Li difflib.SequenceMatcher(a=t.name, b=test_name).ratio())) 616*9c5db199SXin Li 617*9c5db199SXin Li 618*9c5db199SXin Lidef matches_attribute_expression_predicate(test_attr_boolstr): 619*9c5db199SXin Li """Returns predicate that matches based on boolean expression of 620*9c5db199SXin Li attributes. 621*9c5db199SXin Li 622*9c5db199SXin Li Builds a predicate that takes in a parsed control file (a ControlData) 623*9c5db199SXin Li ans returns True if the test attributes satisfy the given attribute 624*9c5db199SXin Li boolean expression. 625*9c5db199SXin Li 626*9c5db199SXin Li @param test_attr_boolstr: boolean expression of the attributes to be 627*9c5db199SXin Li test, like 'system:all and interval:daily'. 628*9c5db199SXin Li 629*9c5db199SXin Li @return a callable that takes a ControlData and returns True if the test 630*9c5db199SXin Li attributes satisfy the given boolean expression. 631*9c5db199SXin Li """ 632*9c5db199SXin Li return lambda t: boolparse_lib.BoolstrResult( 633*9c5db199SXin Li test_attr_boolstr, t.attributes) 634*9c5db199SXin Li 635*9c5db199SXin Li 636*9c5db199SXin Lidef test_file_matches_pattern_predicate(test_file_pattern): 637*9c5db199SXin Li """Returns predicate that matches based on a test's file name pattern. 638*9c5db199SXin Li 639*9c5db199SXin Li Builds a predicate that takes in a parsed control file (a ControlData) 640*9c5db199SXin Li and returns True if the test's control file name matches the given 641*9c5db199SXin Li regular expression. 642*9c5db199SXin Li 643*9c5db199SXin Li @param test_file_pattern: regular expression (string) to match against 644*9c5db199SXin Li control file names. 645*9c5db199SXin Li @return a callable that takes a ControlData and and returns 646*9c5db199SXin Li True if control file name matches the pattern. 647*9c5db199SXin Li """ 648*9c5db199SXin Li return lambda t: hasattr(t, 'path') and re.match(test_file_pattern, 649*9c5db199SXin Li t.path) 650*9c5db199SXin Li 651*9c5db199SXin Li 652*9c5db199SXin Lidef test_name_matches_pattern_predicate(test_name_pattern): 653*9c5db199SXin Li """Returns predicate that matches based on a test's name pattern. 654*9c5db199SXin Li 655*9c5db199SXin Li Builds a predicate that takes in a parsed control file (a ControlData) 656*9c5db199SXin Li and returns True if the test name matches the given regular expression. 657*9c5db199SXin Li 658*9c5db199SXin Li @param test_name_pattern: regular expression (string) to match against 659*9c5db199SXin Li test names. 660*9c5db199SXin Li @return a callable that takes a ControlData and returns 661*9c5db199SXin Li True if the name fields matches the pattern. 662*9c5db199SXin Li """ 663*9c5db199SXin Li return lambda t: hasattr(t, 'name') and re.match(test_name_pattern, 664*9c5db199SXin Li t.name) 665*9c5db199SXin Li 666*9c5db199SXin Li 667*9c5db199SXin Lidef test_name_equals_predicate(test_name): 668*9c5db199SXin Li """Returns predicate that matched based on a test's name. 669*9c5db199SXin Li 670*9c5db199SXin Li Builds a predicate that takes in a parsed control file (a ControlData) 671*9c5db199SXin Li and returns True if the test name is equal to |test_name|. 672*9c5db199SXin Li 673*9c5db199SXin Li @param test_name: the test name to base the predicate on. 674*9c5db199SXin Li @return a callable that takes a ControlData and looks for |test_name| 675*9c5db199SXin Li in that ControlData's name. 676*9c5db199SXin Li """ 677*9c5db199SXin Li return lambda t: hasattr(t, 'name') and test_name == t.name 678*9c5db199SXin Li 679*9c5db199SXin Li 680*9c5db199SXin Lidef name_in_tag_similarity_predicate(name): 681*9c5db199SXin Li """Returns predicate that takes a control file and gets the similarity 682*9c5db199SXin Li of the suites in the control file and the given name. 683*9c5db199SXin Li 684*9c5db199SXin Li Builds a predicate that takes in a parsed control file (a ControlData) 685*9c5db199SXin Li and returns a list of tuples of (suite name, ratio), where suite name 686*9c5db199SXin Li is each suite listed in the control file, and ratio is the similarity 687*9c5db199SXin Li between each suite and the given name. 688*9c5db199SXin Li 689*9c5db199SXin Li @param name: the suite name to base the predicate on. 690*9c5db199SXin Li @return a callable that takes a ControlData and returns a list of tuples 691*9c5db199SXin Li of (suite name, ratio), where suite name is each suite listed in 692*9c5db199SXin Li the control file, and ratio is the similarity between each suite 693*9c5db199SXin Li and the given name. 694*9c5db199SXin Li """ 695*9c5db199SXin Li return lambda t: [(suite, 696*9c5db199SXin Li difflib.SequenceMatcher(a=suite, b=name).ratio()) 697*9c5db199SXin Li for suite in t.suite_tag_parts] or [(None, 0)] 698*9c5db199SXin Li 699*9c5db199SXin Li 700*9c5db199SXin Lidef name_in_tag_predicate(name): 701*9c5db199SXin Li """Returns predicate that takes a control file and looks for |name|. 702*9c5db199SXin Li 703*9c5db199SXin Li Builds a predicate that takes in a parsed control file (a ControlData) 704*9c5db199SXin Li and returns True if the SUITE tag is present and contains |name|. 705*9c5db199SXin Li 706*9c5db199SXin Li @param name: the suite name to base the predicate on. 707*9c5db199SXin Li @return a callable that takes a ControlData and looks for |name| in that 708*9c5db199SXin Li ControlData object's suite member. 709*9c5db199SXin Li """ 710*9c5db199SXin Li return suite_common.name_in_tag_predicate(name) 711*9c5db199SXin Li 712*9c5db199SXin Li 713*9c5db199SXin Lidef create_fs_getter(autotest_dir): 714*9c5db199SXin Li """ 715*9c5db199SXin Li @param autotest_dir: the place to find autotests. 716*9c5db199SXin Li @return a FileSystemGetter instance that looks under |autotest_dir|. 717*9c5db199SXin Li """ 718*9c5db199SXin Li # currently hard-coded places to look for tests. 719*9c5db199SXin Li subpaths = ['server/site_tests', 'client/site_tests', 720*9c5db199SXin Li 'server/tests', 'client/tests'] 721*9c5db199SXin Li directories = [os.path.join(autotest_dir, p) for p in subpaths] 722*9c5db199SXin Li return control_file_getter.FileSystemGetter(directories) 723*9c5db199SXin Li 724*9c5db199SXin Li 725*9c5db199SXin Lidef _create_ds_getter(build, devserver): 726*9c5db199SXin Li """ 727*9c5db199SXin Li @param build: the build on which we're running this suite. 728*9c5db199SXin Li @param devserver: the devserver which contains the build. 729*9c5db199SXin Li @return a FileSystemGetter instance that looks under |autotest_dir|. 730*9c5db199SXin Li """ 731*9c5db199SXin Li return control_file_getter.DevServerGetter(build, devserver) 732*9c5db199SXin Li 733*9c5db199SXin Li 734*9c5db199SXin Lidef _non_experimental_tests_predicate(test_data): 735*9c5db199SXin Li """Test predicate for non-experimental tests.""" 736*9c5db199SXin Li return not test_data.experimental 737*9c5db199SXin Li 738*9c5db199SXin Li 739*9c5db199SXin Lidef find_and_parse_tests(cf_getter, predicate, suite_name='', 740*9c5db199SXin Li add_experimental=False, forgiving_parser=True, 741*9c5db199SXin Li run_prod_code=False, test_args=None): 742*9c5db199SXin Li """ 743*9c5db199SXin Li Function to scan through all tests and find eligible tests. 744*9c5db199SXin Li 745*9c5db199SXin Li Search through all tests based on given cf_getter, suite_name, 746*9c5db199SXin Li add_experimental and forgiving_parser, return the tests that match 747*9c5db199SXin Li given predicate. 748*9c5db199SXin Li 749*9c5db199SXin Li @param cf_getter: a control_file_getter.ControlFileGetter used to list 750*9c5db199SXin Li and fetch the content of control files 751*9c5db199SXin Li @param predicate: a function that should return True when run over a 752*9c5db199SXin Li ControlData representation of a control file that should be in 753*9c5db199SXin Li this Suite. 754*9c5db199SXin Li @param suite_name: If specified, this method will attempt to restrain 755*9c5db199SXin Li the search space to just this suite's control files. 756*9c5db199SXin Li @param add_experimental: add tests with experimental attribute set. 757*9c5db199SXin Li @param forgiving_parser: If False, will raise ControlVariableExceptions 758*9c5db199SXin Li if any are encountered when parsing control 759*9c5db199SXin Li files. Note that this can raise an exception 760*9c5db199SXin Li for syntax errors in unrelated files, because 761*9c5db199SXin Li we parse them before applying the predicate. 762*9c5db199SXin Li @param run_prod_code: If true, the suite will run the test code that 763*9c5db199SXin Li lives in prod aka the test code currently on the 764*9c5db199SXin Li lab servers by disabling SSP for the discovered 765*9c5db199SXin Li tests. 766*9c5db199SXin Li @param test_args: A dict of args to be seeded in test control file. 767*9c5db199SXin Li 768*9c5db199SXin Li @raises ControlVariableException: If forgiving_parser is False and there 769*9c5db199SXin Li is a syntax error in a control file. 770*9c5db199SXin Li 771*9c5db199SXin Li @return list of ControlData objects that should be run, with control 772*9c5db199SXin Li file text added in |text| attribute. Results are sorted based 773*9c5db199SXin Li on the TIME setting in control file, slowest test comes first. 774*9c5db199SXin Li """ 775*9c5db199SXin Li logging.debug('Getting control file list for suite: %s', suite_name) 776*9c5db199SXin Li retriever = _ControlFileRetriever(cf_getter, 777*9c5db199SXin Li forgiving_parser=forgiving_parser, 778*9c5db199SXin Li run_prod_code=run_prod_code, 779*9c5db199SXin Li test_args=test_args) 780*9c5db199SXin Li tests = retriever.retrieve_for_suite(suite_name) 781*9c5db199SXin Li if not add_experimental: 782*9c5db199SXin Li predicate = _ComposedPredicate([predicate, 783*9c5db199SXin Li _non_experimental_tests_predicate]) 784*9c5db199SXin Li return suite_common.filter_tests(tests, predicate) 785*9c5db199SXin Li 786*9c5db199SXin Li 787*9c5db199SXin Lidef find_possible_tests(cf_getter, predicate, suite_name='', count=10): 788*9c5db199SXin Li """ 789*9c5db199SXin Li Function to scan through all tests and find possible tests. 790*9c5db199SXin Li 791*9c5db199SXin Li Search through all tests based on given cf_getter, suite_name, 792*9c5db199SXin Li add_experimental and forgiving_parser. Use the given predicate to 793*9c5db199SXin Li calculate the similarity and return the top 10 matches. 794*9c5db199SXin Li 795*9c5db199SXin Li @param cf_getter: a control_file_getter.ControlFileGetter used to list 796*9c5db199SXin Li and fetch the content of control files 797*9c5db199SXin Li @param predicate: a function that should return a tuple of (name, ratio) 798*9c5db199SXin Li when run over a ControlData representation of a control file that 799*9c5db199SXin Li should be in this Suite. `name` is the key to be compared, e.g., 800*9c5db199SXin Li a suite name or test name. `ratio` is a value between [0,1] 801*9c5db199SXin Li indicating the similarity of `name` and the value to be compared. 802*9c5db199SXin Li @param suite_name: If specified, this method will attempt to restrain 803*9c5db199SXin Li the search space to just this suite's control files. 804*9c5db199SXin Li @param count: Number of suggestions to return, default to 10. 805*9c5db199SXin Li 806*9c5db199SXin Li @return list of top names that similar to the given test, sorted by 807*9c5db199SXin Li match ratio. 808*9c5db199SXin Li """ 809*9c5db199SXin Li logging.debug('Getting control file list for suite: %s', suite_name) 810*9c5db199SXin Li tests = _ControlFileRetriever(cf_getter).retrieve_for_suite(suite_name) 811*9c5db199SXin Li logging.debug('Parsed %s control files.', len(tests)) 812*9c5db199SXin Li similarities = {} 813*9c5db199SXin Li for test in six.itervalues(tests): 814*9c5db199SXin Li ratios = predicate(test) 815*9c5db199SXin Li # Some predicates may return a list of tuples, e.g., 816*9c5db199SXin Li # name_in_tag_similarity_predicate. Convert all returns to a list. 817*9c5db199SXin Li if not isinstance(ratios, list): 818*9c5db199SXin Li ratios = [ratios] 819*9c5db199SXin Li for name, ratio in ratios: 820*9c5db199SXin Li similarities[name] = ratio 821*9c5db199SXin Li return [s[0] for s in 822*9c5db199SXin Li sorted(list(similarities.items()), key=operator.itemgetter(1), 823*9c5db199SXin Li reverse=True)][:count] 824*9c5db199SXin Li 825*9c5db199SXin Li 826*9c5db199SXin Lidef _deprecated_suite_method(func): 827*9c5db199SXin Li """Decorator for deprecated Suite static methods. 828*9c5db199SXin Li 829*9c5db199SXin Li TODO(ayatane): This is used to decorate functions that are called as 830*9c5db199SXin Li static methods on Suite. 831*9c5db199SXin Li """ 832*9c5db199SXin Li @functools.wraps(func) 833*9c5db199SXin Li def wrapper(*args, **kwargs): 834*9c5db199SXin Li """Wraps |func| for warning.""" 835*9c5db199SXin Li warnings.warn('Calling method "%s" from Suite is deprecated' % 836*9c5db199SXin Li func.__name__) 837*9c5db199SXin Li return func(*args, **kwargs) 838*9c5db199SXin Li return staticmethod(wrapper) 839*9c5db199SXin Li 840*9c5db199SXin Li 841*9c5db199SXin Liclass _BaseSuite(object): 842*9c5db199SXin Li """ 843*9c5db199SXin Li A suite of tests, defined by some predicate over control file variables. 844*9c5db199SXin Li 845*9c5db199SXin Li Given a place to search for control files a predicate to match the desired 846*9c5db199SXin Li tests, can gather tests and fire off jobs to run them, and then wait for 847*9c5db199SXin Li results. 848*9c5db199SXin Li 849*9c5db199SXin Li @var _predicate: a function that should return True when run over a 850*9c5db199SXin Li ControlData representation of a control file that should be in 851*9c5db199SXin Li this Suite. 852*9c5db199SXin Li @var _tag: a string with which to tag jobs run in this suite. 853*9c5db199SXin Li @var _builds: the builds on which we're running this suite. 854*9c5db199SXin Li @var _afe: an instance of AFE as defined in server/frontend.py. 855*9c5db199SXin Li @var _tko: an instance of TKO as defined in server/frontend.py. 856*9c5db199SXin Li @var _jobs: currently scheduled jobs, if any. 857*9c5db199SXin Li @var _jobs_to_tests: a dictionary that maps job ids to tests represented 858*9c5db199SXin Li ControlData objects. 859*9c5db199SXin Li @var _retry: a bool value indicating whether jobs should be retried on 860*9c5db199SXin Li failure. 861*9c5db199SXin Li @var _retry_handler: a RetryHandler object. 862*9c5db199SXin Li 863*9c5db199SXin Li """ 864*9c5db199SXin Li 865*9c5db199SXin Li 866*9c5db199SXin Li def __init__( 867*9c5db199SXin Li self, 868*9c5db199SXin Li tests, 869*9c5db199SXin Li tag, 870*9c5db199SXin Li builds, 871*9c5db199SXin Li board, 872*9c5db199SXin Li afe=None, 873*9c5db199SXin Li tko=None, 874*9c5db199SXin Li pool=None, 875*9c5db199SXin Li results_dir=None, 876*9c5db199SXin Li max_runtime_mins=24*60, 877*9c5db199SXin Li timeout_mins=24*60, 878*9c5db199SXin Li file_bugs=False, 879*9c5db199SXin Li suite_job_id=None, 880*9c5db199SXin Li ignore_deps=False, 881*9c5db199SXin Li extra_deps=None, 882*9c5db199SXin Li priority=priorities.Priority.DEFAULT, 883*9c5db199SXin Li wait_for_results=True, 884*9c5db199SXin Li job_retry=False, 885*9c5db199SXin Li max_retries=sys.maxsize, 886*9c5db199SXin Li offload_failures_only=False, 887*9c5db199SXin Li test_source_build=None, 888*9c5db199SXin Li job_keyvals=None, 889*9c5db199SXin Li child_dependencies=(), 890*9c5db199SXin Li result_reporter=None, 891*9c5db199SXin Li ): 892*9c5db199SXin Li """Initialize instance. 893*9c5db199SXin Li 894*9c5db199SXin Li @param tests: Iterable of tests to run. 895*9c5db199SXin Li @param tag: a string with which to tag jobs run in this suite. 896*9c5db199SXin Li @param builds: the builds on which we're running this suite. 897*9c5db199SXin Li @param board: the board on which we're running this suite. 898*9c5db199SXin Li @param afe: an instance of AFE as defined in server/frontend.py. 899*9c5db199SXin Li @param tko: an instance of TKO as defined in server/frontend.py. 900*9c5db199SXin Li @param pool: Specify the pool of machines to use for scheduling 901*9c5db199SXin Li purposes. 902*9c5db199SXin Li @param results_dir: The directory where the job can write results to. 903*9c5db199SXin Li This must be set if you want job_id of sub-jobs 904*9c5db199SXin Li list in the job keyvals. 905*9c5db199SXin Li @param max_runtime_mins: Maximum suite runtime, in minutes. 906*9c5db199SXin Li @param timeout: Maximum job lifetime, in hours. 907*9c5db199SXin Li @param suite_job_id: Job id that will act as parent id to all sub jobs. 908*9c5db199SXin Li Default: None 909*9c5db199SXin Li @param ignore_deps: True if jobs should ignore the DEPENDENCIES 910*9c5db199SXin Li attribute and skip applying of dependency labels. 911*9c5db199SXin Li (Default:False) 912*9c5db199SXin Li @param extra_deps: A list of strings which are the extra DEPENDENCIES 913*9c5db199SXin Li to add to each test being scheduled. 914*9c5db199SXin Li @param priority: Integer priority level. Higher is more important. 915*9c5db199SXin Li @param wait_for_results: Set to False to run the suite job without 916*9c5db199SXin Li waiting for test jobs to finish. Default is 917*9c5db199SXin Li True. 918*9c5db199SXin Li @param job_retry: A bool value indicating whether jobs should be retried 919*9c5db199SXin Li on failure. If True, the field 'JOB_RETRIES' in 920*9c5db199SXin Li control files will be respected. If False, do not 921*9c5db199SXin Li retry. 922*9c5db199SXin Li @param max_retries: Maximum retry limit at suite level. 923*9c5db199SXin Li Regardless how many times each individual test 924*9c5db199SXin Li has been retried, the total number of retries 925*9c5db199SXin Li happening in the suite can't exceed _max_retries. 926*9c5db199SXin Li Default to sys.maxint. 927*9c5db199SXin Li @param offload_failures_only: Only enable gs_offloading for failed 928*9c5db199SXin Li jobs. 929*9c5db199SXin Li @param test_source_build: Build that contains the server-side test code. 930*9c5db199SXin Li @param job_keyvals: General job keyvals to be inserted into keyval file, 931*9c5db199SXin Li which will be used by tko/parse later. 932*9c5db199SXin Li @param child_dependencies: (optional) list of dependency strings 933*9c5db199SXin Li to be added as dependencies to child jobs. 934*9c5db199SXin Li @param result_reporter: A _ResultReporter instance to report results. If 935*9c5db199SXin Li None, an _EmailReporter will be created. 936*9c5db199SXin Li """ 937*9c5db199SXin Li 938*9c5db199SXin Li self.tests = list(tests) 939*9c5db199SXin Li self._tag = tag 940*9c5db199SXin Li self._builds = builds 941*9c5db199SXin Li self._results_dir = results_dir 942*9c5db199SXin Li self._afe = afe or frontend_wrappers.RetryingAFE(timeout_min=30, 943*9c5db199SXin Li delay_sec=10, 944*9c5db199SXin Li debug=False) 945*9c5db199SXin Li self._tko = tko or frontend_wrappers.RetryingTKO(timeout_min=30, 946*9c5db199SXin Li delay_sec=10, 947*9c5db199SXin Li debug=False) 948*9c5db199SXin Li self._jobs = [] 949*9c5db199SXin Li self._jobs_to_tests = {} 950*9c5db199SXin Li 951*9c5db199SXin Li self._file_bugs = file_bugs 952*9c5db199SXin Li self._suite_job_id = suite_job_id 953*9c5db199SXin Li self._job_retry=job_retry 954*9c5db199SXin Li self._max_retries = max_retries 955*9c5db199SXin Li # RetryHandler to be initialized in schedule() 956*9c5db199SXin Li self._retry_handler = None 957*9c5db199SXin Li self.wait_for_results = wait_for_results 958*9c5db199SXin Li self._job_keyvals = job_keyvals 959*9c5db199SXin Li if result_reporter is None: 960*9c5db199SXin Li self._result_reporter = _EmailReporter(self) 961*9c5db199SXin Li else: 962*9c5db199SXin Li self._result_reporter = result_reporter 963*9c5db199SXin Li 964*9c5db199SXin Li if extra_deps is None: 965*9c5db199SXin Li extra_deps = [] 966*9c5db199SXin Li extra_deps.append(board) 967*9c5db199SXin Li if pool: 968*9c5db199SXin Li extra_deps.append(pool) 969*9c5db199SXin Li extra_deps.extend(child_dependencies) 970*9c5db199SXin Li self._dependencies = tuple(extra_deps) 971*9c5db199SXin Li 972*9c5db199SXin Li self._job_creator = _SuiteChildJobCreator( 973*9c5db199SXin Li tag=tag, 974*9c5db199SXin Li builds=builds, 975*9c5db199SXin Li board=board, 976*9c5db199SXin Li afe=afe, 977*9c5db199SXin Li max_runtime_mins=max_runtime_mins, 978*9c5db199SXin Li timeout_mins=timeout_mins, 979*9c5db199SXin Li suite_job_id=suite_job_id, 980*9c5db199SXin Li ignore_deps=ignore_deps, 981*9c5db199SXin Li extra_deps=extra_deps, 982*9c5db199SXin Li priority=priority, 983*9c5db199SXin Li offload_failures_only=offload_failures_only, 984*9c5db199SXin Li test_source_build=test_source_build, 985*9c5db199SXin Li job_keyvals=job_keyvals, 986*9c5db199SXin Li ) 987*9c5db199SXin Li 988*9c5db199SXin Li 989*9c5db199SXin Li def _schedule_test(self, record, test, retry_for=None): 990*9c5db199SXin Li """Schedule a single test and return the job. 991*9c5db199SXin Li 992*9c5db199SXin Li Schedule a single test by creating a job, and then update relevant 993*9c5db199SXin Li data structures that are used to keep track of all running jobs. 994*9c5db199SXin Li 995*9c5db199SXin Li Emits a TEST_NA status log entry if it failed to schedule the test due 996*9c5db199SXin Li to NoEligibleHostException or a non-existent board label. 997*9c5db199SXin Li 998*9c5db199SXin Li Returns a frontend.Job object if the test is successfully scheduled. 999*9c5db199SXin Li If scheduling failed due to NoEligibleHostException or a non-existent 1000*9c5db199SXin Li board label, returns None. 1001*9c5db199SXin Li 1002*9c5db199SXin Li @param record: A callable to use for logging. 1003*9c5db199SXin Li prototype: record(base_job.status_log_entry) 1004*9c5db199SXin Li @param test: ControlData for a test to run. 1005*9c5db199SXin Li @param retry_for: If we are scheduling a test to retry an 1006*9c5db199SXin Li old job, the afe_job_id of the old job 1007*9c5db199SXin Li will be passed in as |retry_for|. 1008*9c5db199SXin Li 1009*9c5db199SXin Li @returns: A frontend.Job object or None 1010*9c5db199SXin Li """ 1011*9c5db199SXin Li msg = 'Scheduling %s' % test.name 1012*9c5db199SXin Li if retry_for: 1013*9c5db199SXin Li msg = msg + ', to retry afe job %d' % retry_for 1014*9c5db199SXin Li logging.debug(msg) 1015*9c5db199SXin Li begin_time_str = datetime.datetime.now().strftime(time_utils.TIME_FMT) 1016*9c5db199SXin Li try: 1017*9c5db199SXin Li job = self._job_creator.create_job(test, retry_for=retry_for) 1018*9c5db199SXin Li except (error.NoEligibleHostException, proxy.ValidationError) as e: 1019*9c5db199SXin Li if (isinstance(e, error.NoEligibleHostException) 1020*9c5db199SXin Li or (isinstance(e, proxy.ValidationError) 1021*9c5db199SXin Li and _is_nonexistent_board_error(e))): 1022*9c5db199SXin Li # Treat a dependency on a non-existent board label the same as 1023*9c5db199SXin Li # a dependency on a board that exists, but for which there's no 1024*9c5db199SXin Li # hardware. 1025*9c5db199SXin Li logging.debug('%s not applicable for this board/pool. ' 1026*9c5db199SXin Li 'Emitting TEST_NA.', test.name) 1027*9c5db199SXin Li Status('TEST_NA', test.name, 1028*9c5db199SXin Li 'Skipping: test not supported on this board/pool.', 1029*9c5db199SXin Li begin_time_str=begin_time_str).record_all(record) 1030*9c5db199SXin Li return None 1031*9c5db199SXin Li else: 1032*9c5db199SXin Li raise e 1033*9c5db199SXin Li except (error.RPCException, proxy.JSONRPCException): 1034*9c5db199SXin Li if retry_for: 1035*9c5db199SXin Li # Mark that we've attempted to retry the old job. 1036*9c5db199SXin Li logging.debug("RPC exception occurred") 1037*9c5db199SXin Li self._retry_handler.set_attempted(job_id=retry_for) 1038*9c5db199SXin Li raise 1039*9c5db199SXin Li else: 1040*9c5db199SXin Li self._jobs.append(job) 1041*9c5db199SXin Li self._jobs_to_tests[job.id] = test 1042*9c5db199SXin Li if retry_for: 1043*9c5db199SXin Li # A retry job was just created, record it. 1044*9c5db199SXin Li self._retry_handler.add_retry( 1045*9c5db199SXin Li old_job_id=retry_for, new_job_id=job.id) 1046*9c5db199SXin Li retry_count = (test.job_retries - 1047*9c5db199SXin Li self._retry_handler.get_retry_max(job.id)) 1048*9c5db199SXin Li logging.debug('Job %d created to retry job %d. ' 1049*9c5db199SXin Li 'Have retried for %d time(s)', 1050*9c5db199SXin Li job.id, retry_for, retry_count) 1051*9c5db199SXin Li self._remember_job_keyval(job) 1052*9c5db199SXin Li return job 1053*9c5db199SXin Li 1054*9c5db199SXin Li def schedule(self, record): 1055*9c5db199SXin Li """ 1056*9c5db199SXin Li Schedule jobs using |self._afe|. 1057*9c5db199SXin Li 1058*9c5db199SXin Li frontend.Job objects representing each scheduled job will be put in 1059*9c5db199SXin Li |self._jobs|. 1060*9c5db199SXin Li 1061*9c5db199SXin Li @param record: A callable to use for logging. 1062*9c5db199SXin Li prototype: record(base_job.status_log_entry) 1063*9c5db199SXin Li @returns: The number of tests that were scheduled. 1064*9c5db199SXin Li """ 1065*9c5db199SXin Li scheduled_test_names = [] 1066*9c5db199SXin Li logging.debug('Discovered %d tests.', len(self.tests)) 1067*9c5db199SXin Li 1068*9c5db199SXin Li Status('INFO', 'Start %s' % self._tag).record_result(record) 1069*9c5db199SXin Li try: 1070*9c5db199SXin Li # Write job_keyvals into keyval file. 1071*9c5db199SXin Li if self._job_keyvals: 1072*9c5db199SXin Li utils.write_keyval(self._results_dir, self._job_keyvals) 1073*9c5db199SXin Li 1074*9c5db199SXin Li # TODO(crbug.com/730885): This is a hack to protect tests that are 1075*9c5db199SXin Li # not usually retried from getting hit by a provision error when run 1076*9c5db199SXin Li # as part of a suite. Remove this hack once provision is separated 1077*9c5db199SXin Li # out in its own suite. 1078*9c5db199SXin Li self._bump_up_test_retries(self.tests) 1079*9c5db199SXin Li for test in self.tests: 1080*9c5db199SXin Li scheduled_job = self._schedule_test(record, test) 1081*9c5db199SXin Li if scheduled_job is not None: 1082*9c5db199SXin Li scheduled_test_names.append(test.name) 1083*9c5db199SXin Li 1084*9c5db199SXin Li # Write the num of scheduled tests and name of them to keyval file. 1085*9c5db199SXin Li logging.debug('Scheduled %d tests, writing the total to keyval.', 1086*9c5db199SXin Li len(scheduled_test_names)) 1087*9c5db199SXin Li utils.write_keyval( 1088*9c5db199SXin Li self._results_dir, 1089*9c5db199SXin Li self._make_scheduled_tests_keyvals(scheduled_test_names)) 1090*9c5db199SXin Li except Exception: 1091*9c5db199SXin Li logging.exception('Exception while scheduling suite') 1092*9c5db199SXin Li Status('FAIL', self._tag, 1093*9c5db199SXin Li 'Exception while scheduling suite').record_result(record) 1094*9c5db199SXin Li 1095*9c5db199SXin Li if self._job_retry: 1096*9c5db199SXin Li logging.debug("Initializing RetryHandler for suite %s.", self._tag) 1097*9c5db199SXin Li self._retry_handler = RetryHandler( 1098*9c5db199SXin Li initial_jobs_to_tests=self._jobs_to_tests, 1099*9c5db199SXin Li max_retries=self._max_retries) 1100*9c5db199SXin Li logging.debug("retry map created: %s ", 1101*9c5db199SXin Li self._retry_handler._retry_map) 1102*9c5db199SXin Li else: 1103*9c5db199SXin Li logging.info("Will not retry jobs from suite %s.", self._tag) 1104*9c5db199SXin Li return len(scheduled_test_names) 1105*9c5db199SXin Li 1106*9c5db199SXin Li 1107*9c5db199SXin Li def _bump_up_test_retries(self, tests): 1108*9c5db199SXin Li """Bump up individual test retries to match suite retry options.""" 1109*9c5db199SXin Li if not self._job_retry: 1110*9c5db199SXin Li return 1111*9c5db199SXin Li 1112*9c5db199SXin Li for test in tests: 1113*9c5db199SXin Li # We do honor if a test insists on JOB_RETRIES = 0. 1114*9c5db199SXin Li if test.job_retries is None: 1115*9c5db199SXin Li logging.debug( 1116*9c5db199SXin Li 'Test %s did not request retries, but suite requires ' 1117*9c5db199SXin Li 'retries. Bumping retries up to 1. ' 1118*9c5db199SXin Li '(See crbug.com/730885)', 1119*9c5db199SXin Li test.name) 1120*9c5db199SXin Li test.job_retries = 1 1121*9c5db199SXin Li 1122*9c5db199SXin Li 1123*9c5db199SXin Li def _make_scheduled_tests_keyvals(self, scheduled_test_names): 1124*9c5db199SXin Li """Make a keyvals dict to write for scheduled test names. 1125*9c5db199SXin Li 1126*9c5db199SXin Li @param scheduled_test_names: A list of scheduled test name strings. 1127*9c5db199SXin Li 1128*9c5db199SXin Li @returns: A keyvals dict. 1129*9c5db199SXin Li """ 1130*9c5db199SXin Li return { 1131*9c5db199SXin Li constants.SCHEDULED_TEST_COUNT_KEY: len(scheduled_test_names), 1132*9c5db199SXin Li constants.SCHEDULED_TEST_NAMES_KEY: repr(scheduled_test_names), 1133*9c5db199SXin Li } 1134*9c5db199SXin Li 1135*9c5db199SXin Li 1136*9c5db199SXin Li def _should_report(self, result): 1137*9c5db199SXin Li """ 1138*9c5db199SXin Li Returns True if this failure requires to be reported. 1139*9c5db199SXin Li 1140*9c5db199SXin Li @param result: A result, encapsulating the status of the failed job. 1141*9c5db199SXin Li @return: True if we should report this failure. 1142*9c5db199SXin Li """ 1143*9c5db199SXin Li return (self._file_bugs and result.test_executed and 1144*9c5db199SXin Li not result.is_testna() and 1145*9c5db199SXin Li result.is_worse_than(job_status.Status('GOOD', '', 'reason'))) 1146*9c5db199SXin Li 1147*9c5db199SXin Li 1148*9c5db199SXin Li def _has_retry(self, result): 1149*9c5db199SXin Li """ 1150*9c5db199SXin Li Return True if this result gets to retry. 1151*9c5db199SXin Li 1152*9c5db199SXin Li @param result: A result, encapsulating the status of the failed job. 1153*9c5db199SXin Li @return: bool 1154*9c5db199SXin Li """ 1155*9c5db199SXin Li return (self._job_retry 1156*9c5db199SXin Li and self._retry_handler.has_following_retry(result)) 1157*9c5db199SXin Li 1158*9c5db199SXin Li 1159*9c5db199SXin Li def wait(self, record): 1160*9c5db199SXin Li """ 1161*9c5db199SXin Li Polls for the job statuses, using |record| to print status when each 1162*9c5db199SXin Li completes. 1163*9c5db199SXin Li 1164*9c5db199SXin Li @param record: callable that records job status. 1165*9c5db199SXin Li prototype: 1166*9c5db199SXin Li record(base_job.status_log_entry) 1167*9c5db199SXin Li """ 1168*9c5db199SXin Li waiter = job_status.JobResultWaiter(self._afe, self._tko) 1169*9c5db199SXin Li try: 1170*9c5db199SXin Li if self._suite_job_id: 1171*9c5db199SXin Li jobs = self._afe.get_jobs(parent_job_id=self._suite_job_id) 1172*9c5db199SXin Li else: 1173*9c5db199SXin Li logging.warning('Unknown suite_job_id, falling back to less ' 1174*9c5db199SXin Li 'efficient results_generator.') 1175*9c5db199SXin Li jobs = self._jobs 1176*9c5db199SXin Li waiter.add_jobs(jobs) 1177*9c5db199SXin Li for result in waiter.wait_for_results(): 1178*9c5db199SXin Li self._handle_result(result=result, record=record, waiter=waiter) 1179*9c5db199SXin Li if self._finished_waiting(): 1180*9c5db199SXin Li break 1181*9c5db199SXin Li except Exception: # pylint: disable=W0703 1182*9c5db199SXin Li logging.exception('Exception waiting for results') 1183*9c5db199SXin Li Status('FAIL', self._tag, 1184*9c5db199SXin Li 'Exception waiting for results').record_result(record) 1185*9c5db199SXin Li 1186*9c5db199SXin Li 1187*9c5db199SXin Li def _finished_waiting(self): 1188*9c5db199SXin Li """Return whether the suite is finished waiting for child jobs.""" 1189*9c5db199SXin Li return False 1190*9c5db199SXin Li 1191*9c5db199SXin Li 1192*9c5db199SXin Li def _handle_result(self, result, record, waiter): 1193*9c5db199SXin Li """ 1194*9c5db199SXin Li Handle a test job result. 1195*9c5db199SXin Li 1196*9c5db199SXin Li @param result: Status instance for job. 1197*9c5db199SXin Li @param record: callable that records job status. 1198*9c5db199SXin Li prototype: 1199*9c5db199SXin Li record(base_job.status_log_entry) 1200*9c5db199SXin Li @param waiter: JobResultsWaiter instance. 1201*9c5db199SXin Li 1202*9c5db199SXin Li @instance_param _result_reporter: _ResultReporter instance. 1203*9c5db199SXin Li """ 1204*9c5db199SXin Li self._record_result(result, record) 1205*9c5db199SXin Li rescheduled = False 1206*9c5db199SXin Li if self._job_retry and self._retry_handler._should_retry(result): 1207*9c5db199SXin Li rescheduled = self._retry_result(result, record, waiter) 1208*9c5db199SXin Li # TODO (crbug.com/751428): If the suite times out before a retry could 1209*9c5db199SXin Li # finish, we would lose the chance to report errors from the original 1210*9c5db199SXin Li # job. 1211*9c5db199SXin Li if self._has_retry(result) and rescheduled: 1212*9c5db199SXin Li return 1213*9c5db199SXin Li 1214*9c5db199SXin Li if self._should_report(result): 1215*9c5db199SXin Li self._result_reporter.report(result) 1216*9c5db199SXin Li 1217*9c5db199SXin Li def _record_result(self, result, record): 1218*9c5db199SXin Li """ 1219*9c5db199SXin Li Record a test job result. 1220*9c5db199SXin Li 1221*9c5db199SXin Li @param result: Status instance for job. 1222*9c5db199SXin Li @param record: callable that records job status. 1223*9c5db199SXin Li prototype: 1224*9c5db199SXin Li record(base_job.status_log_entry) 1225*9c5db199SXin Li """ 1226*9c5db199SXin Li result.record_all(record) 1227*9c5db199SXin Li self._remember_job_keyval(result) 1228*9c5db199SXin Li 1229*9c5db199SXin Li 1230*9c5db199SXin Li def _retry_result(self, result, record, waiter): 1231*9c5db199SXin Li """ 1232*9c5db199SXin Li Retry a test job result. 1233*9c5db199SXin Li 1234*9c5db199SXin Li @param result: Status instance for job. 1235*9c5db199SXin Li @param record: callable that records job status. 1236*9c5db199SXin Li prototype: 1237*9c5db199SXin Li record(base_job.status_log_entry) 1238*9c5db199SXin Li @param waiter: JobResultsWaiter instance. 1239*9c5db199SXin Li @returns: True if a job was scheduled for retry, False otherwise. 1240*9c5db199SXin Li """ 1241*9c5db199SXin Li test = self._jobs_to_tests[result.id] 1242*9c5db199SXin Li try: 1243*9c5db199SXin Li # It only takes effect for CQ retriable job: 1244*9c5db199SXin Li # 1) in first try, test.fast=True. 1245*9c5db199SXin Li # 2) in second try, test will be run in normal mode, so reset 1246*9c5db199SXin Li # test.fast=False. 1247*9c5db199SXin Li test.fast = False 1248*9c5db199SXin Li new_job = self._schedule_test( 1249*9c5db199SXin Li record=record, test=test, retry_for=result.id) 1250*9c5db199SXin Li except (error.RPCException, proxy.JSONRPCException) as e: 1251*9c5db199SXin Li logging.error('Failed to schedule test: %s, Reason: %s', 1252*9c5db199SXin Li test.name, e) 1253*9c5db199SXin Li return False 1254*9c5db199SXin Li else: 1255*9c5db199SXin Li waiter.add_job(new_job) 1256*9c5db199SXin Li return bool(new_job) 1257*9c5db199SXin Li 1258*9c5db199SXin Li @property 1259*9c5db199SXin Li def jobs(self): 1260*9c5db199SXin Li """Give a copy of the associated jobs 1261*9c5db199SXin Li 1262*9c5db199SXin Li @returns: array of jobs""" 1263*9c5db199SXin Li return [job for job in self._jobs] 1264*9c5db199SXin Li 1265*9c5db199SXin Li 1266*9c5db199SXin Li @property 1267*9c5db199SXin Li def _should_file_bugs(self): 1268*9c5db199SXin Li """Return whether bugs should be filed. 1269*9c5db199SXin Li 1270*9c5db199SXin Li @returns: bool 1271*9c5db199SXin Li """ 1272*9c5db199SXin Li # File bug when failure is one of the _FILE_BUG_SUITES, 1273*9c5db199SXin Li # otherwise send an email to the owner anc cc. 1274*9c5db199SXin Li return self._tag in _FILE_BUG_SUITES 1275*9c5db199SXin Li 1276*9c5db199SXin Li 1277*9c5db199SXin Li def abort(self): 1278*9c5db199SXin Li """ 1279*9c5db199SXin Li Abort all scheduled test jobs. 1280*9c5db199SXin Li """ 1281*9c5db199SXin Li if self._jobs: 1282*9c5db199SXin Li job_ids = [job.id for job in self._jobs] 1283*9c5db199SXin Li self._afe.run('abort_host_queue_entries', job__id__in=job_ids) 1284*9c5db199SXin Li 1285*9c5db199SXin Li 1286*9c5db199SXin Li def _remember_job_keyval(self, job): 1287*9c5db199SXin Li """ 1288*9c5db199SXin Li Record provided job as a suite job keyval, for later referencing. 1289*9c5db199SXin Li 1290*9c5db199SXin Li @param job: some representation of a job that has the attributes: 1291*9c5db199SXin Li id, test_name, and owner 1292*9c5db199SXin Li """ 1293*9c5db199SXin Li if self._results_dir and job.id and job.owner and job.test_name: 1294*9c5db199SXin Li job_id_owner = '%s-%s' % (job.id, job.owner) 1295*9c5db199SXin Li logging.debug('Adding job keyval for %s=%s', 1296*9c5db199SXin Li job.test_name, job_id_owner) 1297*9c5db199SXin Li utils.write_keyval( 1298*9c5db199SXin Li self._results_dir, 1299*9c5db199SXin Li {hashlib.md5(job.test_name).hexdigest(): job_id_owner}) 1300*9c5db199SXin Li 1301*9c5db199SXin Li 1302*9c5db199SXin Liclass Suite(_BaseSuite): 1303*9c5db199SXin Li """ 1304*9c5db199SXin Li A suite of tests, defined by some predicate over control file variables. 1305*9c5db199SXin Li 1306*9c5db199SXin Li Given a place to search for control files a predicate to match the desired 1307*9c5db199SXin Li tests, can gather tests and fire off jobs to run them, and then wait for 1308*9c5db199SXin Li results. 1309*9c5db199SXin Li 1310*9c5db199SXin Li @var _predicate: a function that should return True when run over a 1311*9c5db199SXin Li ControlData representation of a control file that should be in 1312*9c5db199SXin Li this Suite. 1313*9c5db199SXin Li @var _tag: a string with which to tag jobs run in this suite. 1314*9c5db199SXin Li @var _builds: the builds on which we're running this suite. 1315*9c5db199SXin Li @var _afe: an instance of AFE as defined in server/frontend.py. 1316*9c5db199SXin Li @var _tko: an instance of TKO as defined in server/frontend.py. 1317*9c5db199SXin Li @var _jobs: currently scheduled jobs, if any. 1318*9c5db199SXin Li @var _jobs_to_tests: a dictionary that maps job ids to tests represented 1319*9c5db199SXin Li ControlData objects. 1320*9c5db199SXin Li @var _cf_getter: a control_file_getter.ControlFileGetter 1321*9c5db199SXin Li @var _retry: a bool value indicating whether jobs should be retried on 1322*9c5db199SXin Li failure. 1323*9c5db199SXin Li @var _retry_handler: a RetryHandler object. 1324*9c5db199SXin Li 1325*9c5db199SXin Li """ 1326*9c5db199SXin Li 1327*9c5db199SXin Li # TODO(ayatane): These methods are kept on the Suite class for 1328*9c5db199SXin Li # backward compatibility. 1329*9c5db199SXin Li find_and_parse_tests = _deprecated_suite_method(find_and_parse_tests) 1330*9c5db199SXin Li find_possible_tests = _deprecated_suite_method(find_possible_tests) 1331*9c5db199SXin Li create_fs_getter = _deprecated_suite_method(create_fs_getter) 1332*9c5db199SXin Li name_in_tag_predicate = _deprecated_suite_method( 1333*9c5db199SXin Li suite_common.name_in_tag_predicate) 1334*9c5db199SXin Li name_in_tag_similarity_predicate = _deprecated_suite_method( 1335*9c5db199SXin Li name_in_tag_similarity_predicate) 1336*9c5db199SXin Li test_name_equals_predicate = _deprecated_suite_method( 1337*9c5db199SXin Li test_name_equals_predicate) 1338*9c5db199SXin Li test_name_in_list_predicate = _deprecated_suite_method( 1339*9c5db199SXin Li suite_common.test_name_in_list_predicate) 1340*9c5db199SXin Li test_name_matches_pattern_predicate = _deprecated_suite_method( 1341*9c5db199SXin Li test_name_matches_pattern_predicate) 1342*9c5db199SXin Li test_file_matches_pattern_predicate = _deprecated_suite_method( 1343*9c5db199SXin Li test_file_matches_pattern_predicate) 1344*9c5db199SXin Li matches_attribute_expression_predicate = _deprecated_suite_method( 1345*9c5db199SXin Li matches_attribute_expression_predicate) 1346*9c5db199SXin Li test_name_similarity_predicate = _deprecated_suite_method( 1347*9c5db199SXin Li test_name_similarity_predicate) 1348*9c5db199SXin Li test_file_similarity_predicate = _deprecated_suite_method( 1349*9c5db199SXin Li test_file_similarity_predicate) 1350*9c5db199SXin Li list_all_suites = _deprecated_suite_method(list_all_suites) 1351*9c5db199SXin Li get_test_source_build = _deprecated_suite_method( 1352*9c5db199SXin Li suite_common.get_test_source_build) 1353*9c5db199SXin Li 1354*9c5db199SXin Li 1355*9c5db199SXin Li @classmethod 1356*9c5db199SXin Li def create_from_predicates(cls, predicates, builds, board, devserver, 1357*9c5db199SXin Li cf_getter=None, name='ad_hoc_suite', 1358*9c5db199SXin Li run_prod_code=False, **dargs): 1359*9c5db199SXin Li """ 1360*9c5db199SXin Li Create a Suite using a given predicate test filters. 1361*9c5db199SXin Li 1362*9c5db199SXin Li Uses supplied predicate(s) to instantiate a Suite. Looks for tests in 1363*9c5db199SXin Li |autotest_dir| and will schedule them using |afe|. Pulls control files 1364*9c5db199SXin Li from the default dev server. Results will be pulled from |tko| upon 1365*9c5db199SXin Li completion. 1366*9c5db199SXin Li 1367*9c5db199SXin Li @param predicates: A list of callables that accept ControlData 1368*9c5db199SXin Li representations of control files. A test will be 1369*9c5db199SXin Li included in suite if all callables in this list 1370*9c5db199SXin Li return True on the given control file. 1371*9c5db199SXin Li @param builds: the builds on which we're running this suite. It's a 1372*9c5db199SXin Li dictionary of version_prefix:build. 1373*9c5db199SXin Li @param board: the board on which we're running this suite. 1374*9c5db199SXin Li @param devserver: the devserver which contains the build. 1375*9c5db199SXin Li @param cf_getter: control_file_getter.ControlFileGetter. Defaults to 1376*9c5db199SXin Li using DevServerGetter. 1377*9c5db199SXin Li @param name: name of suite. Defaults to 'ad_hoc_suite' 1378*9c5db199SXin Li @param run_prod_code: If true, the suite will run the tests that 1379*9c5db199SXin Li lives in prod aka the test code currently on the 1380*9c5db199SXin Li lab servers. 1381*9c5db199SXin Li @param **dargs: Any other Suite constructor parameters, as described 1382*9c5db199SXin Li in Suite.__init__ docstring. 1383*9c5db199SXin Li @return a Suite instance. 1384*9c5db199SXin Li """ 1385*9c5db199SXin Li if cf_getter is None: 1386*9c5db199SXin Li if run_prod_code: 1387*9c5db199SXin Li cf_getter = create_fs_getter(_AUTOTEST_DIR) 1388*9c5db199SXin Li else: 1389*9c5db199SXin Li build = suite_common.get_test_source_build(builds, **dargs) 1390*9c5db199SXin Li cf_getter = _create_ds_getter(build, devserver) 1391*9c5db199SXin Li 1392*9c5db199SXin Li return cls(predicates, 1393*9c5db199SXin Li name, builds, board, cf_getter, run_prod_code, **dargs) 1394*9c5db199SXin Li 1395*9c5db199SXin Li 1396*9c5db199SXin Li @classmethod 1397*9c5db199SXin Li def create_from_name(cls, name, builds, board, devserver, cf_getter=None, 1398*9c5db199SXin Li **dargs): 1399*9c5db199SXin Li """ 1400*9c5db199SXin Li Create a Suite using a predicate based on the SUITE control file var. 1401*9c5db199SXin Li 1402*9c5db199SXin Li Makes a predicate based on |name| and uses it to instantiate a Suite 1403*9c5db199SXin Li that looks for tests in |autotest_dir| and will schedule them using 1404*9c5db199SXin Li |afe|. Pulls control files from the default dev server. 1405*9c5db199SXin Li Results will be pulled from |tko| upon completion. 1406*9c5db199SXin Li 1407*9c5db199SXin Li @param name: a value of the SUITE control file variable to search for. 1408*9c5db199SXin Li @param builds: the builds on which we're running this suite. It's a 1409*9c5db199SXin Li dictionary of version_prefix:build. 1410*9c5db199SXin Li @param board: the board on which we're running this suite. 1411*9c5db199SXin Li @param devserver: the devserver which contains the build. 1412*9c5db199SXin Li @param cf_getter: control_file_getter.ControlFileGetter. Defaults to 1413*9c5db199SXin Li using DevServerGetter. 1414*9c5db199SXin Li @param **dargs: Any other Suite constructor parameters, as described 1415*9c5db199SXin Li in Suite.__init__ docstring. 1416*9c5db199SXin Li @return a Suite instance. 1417*9c5db199SXin Li """ 1418*9c5db199SXin Li if cf_getter is None: 1419*9c5db199SXin Li build = suite_common.get_test_source_build(builds, **dargs) 1420*9c5db199SXin Li cf_getter = _create_ds_getter(build, devserver) 1421*9c5db199SXin Li 1422*9c5db199SXin Li return cls([suite_common.name_in_tag_predicate(name)], 1423*9c5db199SXin Li name, builds, board, cf_getter, **dargs) 1424*9c5db199SXin Li 1425*9c5db199SXin Li 1426*9c5db199SXin Li def __init__( 1427*9c5db199SXin Li self, 1428*9c5db199SXin Li predicates, 1429*9c5db199SXin Li tag, 1430*9c5db199SXin Li builds, 1431*9c5db199SXin Li board, 1432*9c5db199SXin Li cf_getter, 1433*9c5db199SXin Li run_prod_code=False, 1434*9c5db199SXin Li afe=None, 1435*9c5db199SXin Li tko=None, 1436*9c5db199SXin Li pool=None, 1437*9c5db199SXin Li results_dir=None, 1438*9c5db199SXin Li max_runtime_mins=24*60, 1439*9c5db199SXin Li timeout_mins=24*60, 1440*9c5db199SXin Li file_bugs=False, 1441*9c5db199SXin Li suite_job_id=None, 1442*9c5db199SXin Li ignore_deps=False, 1443*9c5db199SXin Li extra_deps=None, 1444*9c5db199SXin Li priority=priorities.Priority.DEFAULT, 1445*9c5db199SXin Li forgiving_parser=True, 1446*9c5db199SXin Li wait_for_results=True, 1447*9c5db199SXin Li job_retry=False, 1448*9c5db199SXin Li max_retries=sys.maxsize, 1449*9c5db199SXin Li offload_failures_only=False, 1450*9c5db199SXin Li test_source_build=None, 1451*9c5db199SXin Li job_keyvals=None, 1452*9c5db199SXin Li test_args=None, 1453*9c5db199SXin Li child_dependencies=(), 1454*9c5db199SXin Li result_reporter=None, 1455*9c5db199SXin Li ): 1456*9c5db199SXin Li """ 1457*9c5db199SXin Li Constructor 1458*9c5db199SXin Li 1459*9c5db199SXin Li @param predicates: A list of callables that accept ControlData 1460*9c5db199SXin Li representations of control files. A test will be 1461*9c5db199SXin Li included in suite if all callables in this list 1462*9c5db199SXin Li return True on the given control file. 1463*9c5db199SXin Li @param tag: a string with which to tag jobs run in this suite. 1464*9c5db199SXin Li @param builds: the builds on which we're running this suite. 1465*9c5db199SXin Li @param board: the board on which we're running this suite. 1466*9c5db199SXin Li @param cf_getter: a control_file_getter.ControlFileGetter 1467*9c5db199SXin Li @param afe: an instance of AFE as defined in server/frontend.py. 1468*9c5db199SXin Li @param tko: an instance of TKO as defined in server/frontend.py. 1469*9c5db199SXin Li @param pool: Specify the pool of machines to use for scheduling 1470*9c5db199SXin Li purposes. 1471*9c5db199SXin Li @param run_prod_code: If true, the suite will run the test code that 1472*9c5db199SXin Li lives in prod aka the test code currently on the 1473*9c5db199SXin Li lab servers. 1474*9c5db199SXin Li @param results_dir: The directory where the job can write results to. 1475*9c5db199SXin Li This must be set if you want job_id of sub-jobs 1476*9c5db199SXin Li list in the job keyvals. 1477*9c5db199SXin Li @param max_runtime_mins: Maximum suite runtime, in minutes. 1478*9c5db199SXin Li @param timeout: Maximum job lifetime, in hours. 1479*9c5db199SXin Li @param suite_job_id: Job id that will act as parent id to all sub jobs. 1480*9c5db199SXin Li Default: None 1481*9c5db199SXin Li @param ignore_deps: True if jobs should ignore the DEPENDENCIES 1482*9c5db199SXin Li attribute and skip applying of dependency labels. 1483*9c5db199SXin Li (Default:False) 1484*9c5db199SXin Li @param extra_deps: A list of strings which are the extra DEPENDENCIES 1485*9c5db199SXin Li to add to each test being scheduled. 1486*9c5db199SXin Li @param priority: Integer priority level. Higher is more important. 1487*9c5db199SXin Li @param wait_for_results: Set to False to run the suite job without 1488*9c5db199SXin Li waiting for test jobs to finish. Default is 1489*9c5db199SXin Li True. 1490*9c5db199SXin Li @param job_retry: A bool value indicating whether jobs should be retried 1491*9c5db199SXin Li on failure. If True, the field 'JOB_RETRIES' in 1492*9c5db199SXin Li control files will be respected. If False, do not 1493*9c5db199SXin Li retry. 1494*9c5db199SXin Li @param max_retries: Maximum retry limit at suite level. 1495*9c5db199SXin Li Regardless how many times each individual test 1496*9c5db199SXin Li has been retried, the total number of retries 1497*9c5db199SXin Li happening in the suite can't exceed _max_retries. 1498*9c5db199SXin Li Default to sys.maxint. 1499*9c5db199SXin Li @param offload_failures_only: Only enable gs_offloading for failed 1500*9c5db199SXin Li jobs. 1501*9c5db199SXin Li @param test_source_build: Build that contains the server-side test code. 1502*9c5db199SXin Li @param job_keyvals: General job keyvals to be inserted into keyval file, 1503*9c5db199SXin Li which will be used by tko/parse later. 1504*9c5db199SXin Li @param test_args: A dict of args passed all the way to each individual 1505*9c5db199SXin Li test that will be actually ran. 1506*9c5db199SXin Li @param child_dependencies: (optional) list of dependency strings 1507*9c5db199SXin Li to be added as dependencies to child jobs. 1508*9c5db199SXin Li @param result_reporter: A _ResultReporter instance to report results. If 1509*9c5db199SXin Li None, an _EmailReporter will be created. 1510*9c5db199SXin Li """ 1511*9c5db199SXin Li tests = find_and_parse_tests( 1512*9c5db199SXin Li cf_getter, 1513*9c5db199SXin Li _ComposedPredicate(predicates), 1514*9c5db199SXin Li tag, 1515*9c5db199SXin Li forgiving_parser=forgiving_parser, 1516*9c5db199SXin Li run_prod_code=run_prod_code, 1517*9c5db199SXin Li test_args=test_args, 1518*9c5db199SXin Li ) 1519*9c5db199SXin Li super(Suite, self).__init__( 1520*9c5db199SXin Li tests=tests, 1521*9c5db199SXin Li tag=tag, 1522*9c5db199SXin Li builds=builds, 1523*9c5db199SXin Li board=board, 1524*9c5db199SXin Li afe=afe, 1525*9c5db199SXin Li tko=tko, 1526*9c5db199SXin Li pool=pool, 1527*9c5db199SXin Li results_dir=results_dir, 1528*9c5db199SXin Li max_runtime_mins=max_runtime_mins, 1529*9c5db199SXin Li timeout_mins=timeout_mins, 1530*9c5db199SXin Li file_bugs=file_bugs, 1531*9c5db199SXin Li suite_job_id=suite_job_id, 1532*9c5db199SXin Li ignore_deps=ignore_deps, 1533*9c5db199SXin Li extra_deps=extra_deps, 1534*9c5db199SXin Li priority=priority, 1535*9c5db199SXin Li wait_for_results=wait_for_results, 1536*9c5db199SXin Li job_retry=job_retry, 1537*9c5db199SXin Li max_retries=max_retries, 1538*9c5db199SXin Li offload_failures_only=offload_failures_only, 1539*9c5db199SXin Li test_source_build=test_source_build, 1540*9c5db199SXin Li job_keyvals=job_keyvals, 1541*9c5db199SXin Li child_dependencies=child_dependencies, 1542*9c5db199SXin Li result_reporter=result_reporter, 1543*9c5db199SXin Li ) 1544*9c5db199SXin Li 1545*9c5db199SXin Li 1546*9c5db199SXin Liclass ProvisionSuite(_BaseSuite): 1547*9c5db199SXin Li """ 1548*9c5db199SXin Li A suite for provisioning DUTs. 1549*9c5db199SXin Li 1550*9c5db199SXin Li This is done by creating stub_Pass tests. 1551*9c5db199SXin Li """ 1552*9c5db199SXin Li 1553*9c5db199SXin Li 1554*9c5db199SXin Li def __init__( 1555*9c5db199SXin Li self, 1556*9c5db199SXin Li tag, 1557*9c5db199SXin Li builds, 1558*9c5db199SXin Li board, 1559*9c5db199SXin Li devserver, 1560*9c5db199SXin Li num_required, 1561*9c5db199SXin Li num_max=float('inf'), 1562*9c5db199SXin Li cf_getter=None, 1563*9c5db199SXin Li run_prod_code=False, 1564*9c5db199SXin Li test_args=None, 1565*9c5db199SXin Li test_source_build=None, 1566*9c5db199SXin Li **kwargs): 1567*9c5db199SXin Li """ 1568*9c5db199SXin Li Constructor 1569*9c5db199SXin Li 1570*9c5db199SXin Li @param tag: a string with which to tag jobs run in this suite. 1571*9c5db199SXin Li @param builds: the builds on which we're running this suite. 1572*9c5db199SXin Li @param board: the board on which we're running this suite. 1573*9c5db199SXin Li @param devserver: the devserver which contains the build. 1574*9c5db199SXin Li @param num_required: number of tests that must pass. This is 1575*9c5db199SXin Li capped by the number of tests that are run. 1576*9c5db199SXin Li @param num_max: max number of tests to make. By default there 1577*9c5db199SXin Li is no cap, a test is created for each eligible host. 1578*9c5db199SXin Li @param cf_getter: a control_file_getter.ControlFileGetter. 1579*9c5db199SXin Li @param test_args: A dict of args passed all the way to each individual 1580*9c5db199SXin Li test that will be actually ran. 1581*9c5db199SXin Li @param test_source_build: Build that contains the server-side test code. 1582*9c5db199SXin Li @param kwargs: Various keyword arguments passed to 1583*9c5db199SXin Li _BaseSuite constructor. 1584*9c5db199SXin Li """ 1585*9c5db199SXin Li super(ProvisionSuite, self).__init__( 1586*9c5db199SXin Li tests=[], 1587*9c5db199SXin Li tag=tag, 1588*9c5db199SXin Li builds=builds, 1589*9c5db199SXin Li board=board, 1590*9c5db199SXin Li **kwargs) 1591*9c5db199SXin Li self._num_successful = 0 1592*9c5db199SXin Li self._num_required = 0 1593*9c5db199SXin Li self.tests = [] 1594*9c5db199SXin Li 1595*9c5db199SXin Li static_deps = [dep for dep in self._dependencies 1596*9c5db199SXin Li if not provision.Provision.acts_on(dep)] 1597*9c5db199SXin Li if 'pool:suites' in static_deps: 1598*9c5db199SXin Li logging.info('Provision suite is disabled on suites pool') 1599*9c5db199SXin Li return 1600*9c5db199SXin Li logging.debug('Looking for hosts matching %r', static_deps) 1601*9c5db199SXin Li hosts = self._afe.get_hosts( 1602*9c5db199SXin Li invalid=False, multiple_labels=static_deps) 1603*9c5db199SXin Li logging.debug('Found %d matching hosts for ProvisionSuite', len(hosts)) 1604*9c5db199SXin Li available_hosts = [h for h in hosts if h.is_available()] 1605*9c5db199SXin Li logging.debug('Found %d available hosts for ProvisionSuite', 1606*9c5db199SXin Li len(available_hosts)) 1607*9c5db199SXin Li dummy_test = _load_dummy_test( 1608*9c5db199SXin Li builds, devserver, cf_getter, 1609*9c5db199SXin Li run_prod_code, test_args, test_source_build) 1610*9c5db199SXin Li self.tests = [dummy_test] * min(len(available_hosts), num_max) 1611*9c5db199SXin Li logging.debug('Made %d tests for ProvisionSuite', len(self.tests)) 1612*9c5db199SXin Li self._num_required = min(num_required, len(self.tests)) 1613*9c5db199SXin Li logging.debug('Expecting %d tests to pass for ProvisionSuite', 1614*9c5db199SXin Li self._num_required) 1615*9c5db199SXin Li 1616*9c5db199SXin Li def _handle_result(self, result, record, waiter): 1617*9c5db199SXin Li super(ProvisionSuite, self)._handle_result(result, record, waiter) 1618*9c5db199SXin Li if result.is_good(): 1619*9c5db199SXin Li self._num_successful += 1 1620*9c5db199SXin Li 1621*9c5db199SXin Li def _finished_waiting(self): 1622*9c5db199SXin Li return self._num_successful >= self._num_required 1623*9c5db199SXin Li 1624*9c5db199SXin Li 1625*9c5db199SXin Lidef _load_dummy_test( 1626*9c5db199SXin Li builds, 1627*9c5db199SXin Li devserver, 1628*9c5db199SXin Li cf_getter=None, 1629*9c5db199SXin Li run_prod_code=False, 1630*9c5db199SXin Li test_args=None, 1631*9c5db199SXin Li test_source_build=None): 1632*9c5db199SXin Li """ 1633*9c5db199SXin Li Load and return the dummy pass test. 1634*9c5db199SXin Li 1635*9c5db199SXin Li @param builds: the builds on which we're running this suite. 1636*9c5db199SXin Li @param devserver: the devserver which contains the build. 1637*9c5db199SXin Li @param cf_getter: a control_file_getter.ControlFileGetter. 1638*9c5db199SXin Li @param test_args: A dict of args passed all the way to each individual 1639*9c5db199SXin Li test that will be actually ran. 1640*9c5db199SXin Li @param test_source_build: Build that contains the server-side test code. 1641*9c5db199SXin Li """ 1642*9c5db199SXin Li if cf_getter is None: 1643*9c5db199SXin Li if run_prod_code: 1644*9c5db199SXin Li cf_getter = create_fs_getter(_AUTOTEST_DIR) 1645*9c5db199SXin Li else: 1646*9c5db199SXin Li build = suite_common.get_test_source_build( 1647*9c5db199SXin Li builds, test_source_build=test_source_build) 1648*9c5db199SXin Li devserver.stage_artifacts(image=build, 1649*9c5db199SXin Li artifacts=['control_files']) 1650*9c5db199SXin Li cf_getter = _create_ds_getter(build, devserver) 1651*9c5db199SXin Li retriever = _ControlFileRetriever(cf_getter, 1652*9c5db199SXin Li run_prod_code=run_prod_code, 1653*9c5db199SXin Li test_args=test_args) 1654*9c5db199SXin Li return retriever.retrieve_for_test('stub_Pass') 1655*9c5db199SXin Li 1656*9c5db199SXin Li 1657*9c5db199SXin Liclass _ComposedPredicate(object): 1658*9c5db199SXin Li """Return the composition of the predicates. 1659*9c5db199SXin Li 1660*9c5db199SXin Li Predicates are functions that take a test control data object and 1661*9c5db199SXin Li return True of that test is to be included. The returned 1662*9c5db199SXin Li predicate's set is the intersection of all of the input predicates' 1663*9c5db199SXin Li sets (it returns True if all predicates return True). 1664*9c5db199SXin Li """ 1665*9c5db199SXin Li 1666*9c5db199SXin Li def __init__(self, predicates): 1667*9c5db199SXin Li """Initialize instance. 1668*9c5db199SXin Li 1669*9c5db199SXin Li @param predicates: Iterable of predicates. 1670*9c5db199SXin Li """ 1671*9c5db199SXin Li self._predicates = list(predicates) 1672*9c5db199SXin Li 1673*9c5db199SXin Li def __repr__(self): 1674*9c5db199SXin Li return '{cls}({this._predicates!r})'.format( 1675*9c5db199SXin Li cls=type(self).__name__, 1676*9c5db199SXin Li this=self, 1677*9c5db199SXin Li ) 1678*9c5db199SXin Li 1679*9c5db199SXin Li def __call__(self, control_data_): 1680*9c5db199SXin Li return all(f(control_data_) for f in self._predicates) 1681*9c5db199SXin Li 1682*9c5db199SXin Li 1683*9c5db199SXin Lidef _is_nonexistent_board_error(e): 1684*9c5db199SXin Li """Return True if error is caused by nonexistent board label. 1685*9c5db199SXin Li 1686*9c5db199SXin Li As of this writing, the particular case we want looks like this: 1687*9c5db199SXin Li 1688*9c5db199SXin Li 1) e.problem_keys is a dictionary 1689*9c5db199SXin Li 2) e.problem_keys['meta_hosts'] exists as the only key 1690*9c5db199SXin Li in the dictionary. 1691*9c5db199SXin Li 3) e.problem_keys['meta_hosts'] matches this pattern: 1692*9c5db199SXin Li "Label "board:.*" not found" 1693*9c5db199SXin Li 1694*9c5db199SXin Li We check for conditions 1) and 2) on the 1695*9c5db199SXin Li theory that they're relatively immutable. 1696*9c5db199SXin Li We don't check condition 3) because it seems 1697*9c5db199SXin Li likely to be a maintenance burden, and for the 1698*9c5db199SXin Li times when we're wrong, being right shouldn't 1699*9c5db199SXin Li matter enough (we _hope_). 1700*9c5db199SXin Li 1701*9c5db199SXin Li @param e: proxy.ValidationError instance 1702*9c5db199SXin Li @returns: boolean 1703*9c5db199SXin Li """ 1704*9c5db199SXin Li return (isinstance(e.problem_keys, dict) 1705*9c5db199SXin Li and len(e.problem_keys) == 1 1706*9c5db199SXin Li and 'meta_hosts' in e.problem_keys) 1707*9c5db199SXin Li 1708*9c5db199SXin Li 1709*9c5db199SXin Liclass _ResultReporter(six.with_metaclass(abc.ABCMeta, object)): 1710*9c5db199SXin Li """Abstract base class for reporting test results. 1711*9c5db199SXin Li 1712*9c5db199SXin Li Usually, this is used to report test failures. 1713*9c5db199SXin Li """ 1714*9c5db199SXin Li 1715*9c5db199SXin Li @abc.abstractmethod 1716*9c5db199SXin Li def report(self, result): 1717*9c5db199SXin Li """Report test result. 1718*9c5db199SXin Li 1719*9c5db199SXin Li @param result: Status instance for job. 1720*9c5db199SXin Li """ 1721*9c5db199SXin Li 1722*9c5db199SXin Li 1723*9c5db199SXin Liclass _EmailReporter(_ResultReporter): 1724*9c5db199SXin Li """Class that emails based on test failures.""" 1725*9c5db199SXin Li 1726*9c5db199SXin Li def __init__(self, suite, bug_template=None): 1727*9c5db199SXin Li self._suite = suite 1728*9c5db199SXin Li self._bug_template = bug_template or {} 1729*9c5db199SXin Li 1730*9c5db199SXin Li def _get_test_bug(self, result): 1731*9c5db199SXin Li """Get TestBug for the given result. 1732*9c5db199SXin Li 1733*9c5db199SXin Li @param result: Status instance for a test job. 1734*9c5db199SXin Li @returns: TestBug instance. 1735*9c5db199SXin Li """ 1736*9c5db199SXin Li # reporting modules have dependency on external packages, e.g., httplib2 1737*9c5db199SXin Li # Such dependency can cause issue to any module tries to import suite.py 1738*9c5db199SXin Li # without building site-packages first. Since the reporting modules are 1739*9c5db199SXin Li # only used in this function, move the imports here avoid the 1740*9c5db199SXin Li # requirement of building site packages to use other functions in this 1741*9c5db199SXin Li # module. 1742*9c5db199SXin Li from autotest_lib.server.cros.dynamic_suite import reporting 1743*9c5db199SXin Li 1744*9c5db199SXin Li job_views = self._suite._tko.run('get_detailed_test_views', 1745*9c5db199SXin Li afe_job_id=result.id) 1746*9c5db199SXin Li return reporting.TestBug(self._suite._job_creator.cros_build, 1747*9c5db199SXin Li utils.get_chrome_version(job_views), 1748*9c5db199SXin Li self._suite._tag, 1749*9c5db199SXin Li result) 1750*9c5db199SXin Li 1751*9c5db199SXin Li def _get_bug_template(self, result): 1752*9c5db199SXin Li """Get BugTemplate for test job. 1753*9c5db199SXin Li 1754*9c5db199SXin Li @param result: Status instance for job. 1755*9c5db199SXin Li @param bug_template: A template dictionary specifying the default bug 1756*9c5db199SXin Li filing options for failures in this suite. 1757*9c5db199SXin Li @returns: BugTemplate instance 1758*9c5db199SXin Li """ 1759*9c5db199SXin Li # reporting modules have dependency on external packages, e.g., httplib2 1760*9c5db199SXin Li # Such dependency can cause issue to any module tries to import suite.py 1761*9c5db199SXin Li # without building site-packages first. Since the reporting modules are 1762*9c5db199SXin Li # only used in this function, move the imports here avoid the 1763*9c5db199SXin Li # requirement of building site packages to use other functions in this 1764*9c5db199SXin Li # module. 1765*9c5db199SXin Li from autotest_lib.server.cros.dynamic_suite import reporting_utils 1766*9c5db199SXin Li 1767*9c5db199SXin Li # Try to merge with bug template in test control file. 1768*9c5db199SXin Li template = reporting_utils.BugTemplate(self._bug_template) 1769*9c5db199SXin Li try: 1770*9c5db199SXin Li test_data = self._suite._jobs_to_tests[result.id] 1771*9c5db199SXin Li return template.finalize_bug_template( 1772*9c5db199SXin Li test_data.bug_template) 1773*9c5db199SXin Li except AttributeError: 1774*9c5db199SXin Li # Test control file does not have bug template defined. 1775*9c5db199SXin Li return template.bug_template 1776*9c5db199SXin Li except reporting_utils.InvalidBugTemplateException as e: 1777*9c5db199SXin Li logging.error('Merging bug templates failed with ' 1778*9c5db199SXin Li 'error: %s An empty bug template will ' 1779*9c5db199SXin Li 'be used.', e) 1780*9c5db199SXin Li return {} 1781*9c5db199SXin Li 1782*9c5db199SXin Li def report(self, result): 1783*9c5db199SXin Li # reporting modules have dependency on external 1784*9c5db199SXin Li # packages, e.g., httplib2 Such dependency can cause 1785*9c5db199SXin Li # issue to any module tries to import suite.py without 1786*9c5db199SXin Li # building site-packages first. Since the reporting 1787*9c5db199SXin Li # modules are only used in this function, move the 1788*9c5db199SXin Li # imports here avoid the requirement of building site 1789*9c5db199SXin Li # packages to use other functions in this module. 1790*9c5db199SXin Li from autotest_lib.server.cros.dynamic_suite import reporting 1791*9c5db199SXin Li 1792*9c5db199SXin Li reporting.send_email( 1793*9c5db199SXin Li self._get_test_bug(result), 1794*9c5db199SXin Li self._get_bug_template(result)) 1795