xref: /aosp_15_r20/external/autotest/server/cros/dynamic_suite/suite.py (revision 9c5db1993ded3edbeafc8092d69fe5de2ee02df7)
1*9c5db199SXin Li# Lint as: python2, python3
2*9c5db199SXin Li# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3*9c5db199SXin Li# Use of this source code is governed by a BSD-style license that can be
4*9c5db199SXin Li# found in the LICENSE file.
5*9c5db199SXin Li
6*9c5db199SXin Lifrom __future__ import absolute_import
7*9c5db199SXin Lifrom __future__ import division
8*9c5db199SXin Lifrom __future__ import print_function
9*9c5db199SXin Li
10*9c5db199SXin Liimport abc
11*9c5db199SXin Liimport datetime
12*9c5db199SXin Liimport difflib
13*9c5db199SXin Liimport functools
14*9c5db199SXin Liimport hashlib
15*9c5db199SXin Liimport logging
16*9c5db199SXin Liimport operator
17*9c5db199SXin Liimport os
18*9c5db199SXin Liimport re
19*9c5db199SXin Liimport six
20*9c5db199SXin Liimport sys
21*9c5db199SXin Liimport warnings
22*9c5db199SXin Li
23*9c5db199SXin Liimport common
24*9c5db199SXin Li
25*9c5db199SXin Lifrom autotest_lib.frontend.afe.json_rpc import proxy
26*9c5db199SXin Lifrom autotest_lib.client.common_lib import autotest_enum
27*9c5db199SXin Lifrom autotest_lib.client.common_lib import error
28*9c5db199SXin Lifrom autotest_lib.client.common_lib import global_config
29*9c5db199SXin Lifrom autotest_lib.client.common_lib import priorities
30*9c5db199SXin Lifrom autotest_lib.client.common_lib import time_utils
31*9c5db199SXin Lifrom autotest_lib.client.common_lib import utils
32*9c5db199SXin Lifrom autotest_lib.frontend.afe import model_attributes
33*9c5db199SXin Lifrom autotest_lib.frontend.afe.json_rpc import proxy
34*9c5db199SXin Lifrom autotest_lib.server.cros import provision
35*9c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import constants
36*9c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import control_file_getter
37*9c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import frontend_wrappers
38*9c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import job_status
39*9c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import suite_common
40*9c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import tools
41*9c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite.job_status import Status
42*9c5db199SXin Li
43*9c5db199SXin Litry:
44*9c5db199SXin Li    from autotest_lib.server.cros.dynamic_suite import boolparse_lib
45*9c5db199SXin Liexcept ImportError as e:
46*9c5db199SXin Li    print('Unable to import boolparse_lib: %s' % (e,))
47*9c5db199SXin Li    print('This script must be either:')
48*9c5db199SXin Li    print('  - Be run in the chroot.')
49*9c5db199SXin Li    print('  - (not yet supported) be run after running ')
50*9c5db199SXin Li    print('    ../utils/build_externals.py')
51*9c5db199SXin Li
52*9c5db199SXin Li_FILE_BUG_SUITES = [
53*9c5db199SXin Li        'au', 'bvt', 'bvt-cq', 'bvt-inline', 'calibration', 'paygen_au_beta',
54*9c5db199SXin Li        'paygen_au_canary', 'paygen_au_dev', 'paygen_au_stable', 'sanity',
55*9c5db199SXin Li        'push_to_prod'
56*9c5db199SXin Li]
57*9c5db199SXin Li_AUTOTEST_DIR = global_config.global_config.get_config_value(
58*9c5db199SXin Li        'SCHEDULER', 'drone_installation_directory')
59*9c5db199SXin Li
60*9c5db199SXin Li
61*9c5db199SXin Liclass RetryHandler(object):
62*9c5db199SXin Li    """Maintain retry information.
63*9c5db199SXin Li
64*9c5db199SXin Li    @var _retry_map: A dictionary that stores retry history.
65*9c5db199SXin Li            The key is afe job id. The value is a dictionary.
66*9c5db199SXin Li            {job_id: {'state':RetryHandler.States, 'retry_max':int}}
67*9c5db199SXin Li            - state:
68*9c5db199SXin Li                The retry state of a job.
69*9c5db199SXin Li                NOT_ATTEMPTED:
70*9c5db199SXin Li                    We haven't done anything about the job.
71*9c5db199SXin Li                ATTEMPTED:
72*9c5db199SXin Li                    We've made an attempt to schedule a retry job. The
73*9c5db199SXin Li                    scheduling may or may not be successful, e.g.
74*9c5db199SXin Li                    it might encounter an rpc error. Note failure
75*9c5db199SXin Li                    in scheduling a retry is different from a retry job failure.
76*9c5db199SXin Li                    For each job, we only attempt to schedule a retry once.
77*9c5db199SXin Li                    For example, assume we have a test with JOB_RETRIES=5 and
78*9c5db199SXin Li                    its second retry job failed. When we attempt to create
79*9c5db199SXin Li                    a third retry job to retry the second, we hit an rpc
80*9c5db199SXin Li                    error. In such case, we will give up on all following
81*9c5db199SXin Li                    retries.
82*9c5db199SXin Li                RETRIED:
83*9c5db199SXin Li                    A retry job has already been successfully
84*9c5db199SXin Li                    scheduled.
85*9c5db199SXin Li            - retry_max:
86*9c5db199SXin Li                The maximum of times the job can still
87*9c5db199SXin Li                be retried, taking into account retries
88*9c5db199SXin Li                that have occurred.
89*9c5db199SXin Li    @var _retry_level: A retry might be triggered only if the result
90*9c5db199SXin Li            is worse than the level.
91*9c5db199SXin Li    @var _max_retries: Maximum retry limit at suite level.
92*9c5db199SXin Li                     Regardless how many times each individual test
93*9c5db199SXin Li                     has been retried, the total number of retries happening in
94*9c5db199SXin Li                     the suite can't exceed _max_retries.
95*9c5db199SXin Li    """
96*9c5db199SXin Li
97*9c5db199SXin Li    States = autotest_enum.AutotestEnum('NOT_ATTEMPTED', 'ATTEMPTED', 'RETRIED',
98*9c5db199SXin Li                                        start_value=1, step=1)
99*9c5db199SXin Li
100*9c5db199SXin Li    def __init__(self, initial_jobs_to_tests, retry_level='WARN',
101*9c5db199SXin Li                 max_retries=None):
102*9c5db199SXin Li        """Initialize RetryHandler.
103*9c5db199SXin Li
104*9c5db199SXin Li        @param initial_jobs_to_tests: A dictionary that maps a job id to
105*9c5db199SXin Li                a ControlData object. This dictionary should contain
106*9c5db199SXin Li                jobs that are originally scheduled by the suite.
107*9c5db199SXin Li        @param retry_level: A retry might be triggered only if the result is
108*9c5db199SXin Li                worse than the level.
109*9c5db199SXin Li        @param max_retries: Integer, maxmium total retries allowed
110*9c5db199SXin Li                                  for the suite. Default to None, no max.
111*9c5db199SXin Li        """
112*9c5db199SXin Li        self._retry_map = {}
113*9c5db199SXin Li        self._retry_level = retry_level
114*9c5db199SXin Li        self._max_retries = (max_retries
115*9c5db199SXin Li                             if max_retries is not None else sys.maxsize)
116*9c5db199SXin Li        for job_id, test in initial_jobs_to_tests.items():
117*9c5db199SXin Li            if test.job_retries > 0:
118*9c5db199SXin Li                self._add_job(new_job_id=job_id,
119*9c5db199SXin Li                              retry_max=test.job_retries)
120*9c5db199SXin Li            else:
121*9c5db199SXin Li                logging.debug("Test %s has no retries", test.name)
122*9c5db199SXin Li
123*9c5db199SXin Li
124*9c5db199SXin Li    def _add_job(self, new_job_id, retry_max):
125*9c5db199SXin Li        """Add a newly-created job to the retry map.
126*9c5db199SXin Li
127*9c5db199SXin Li        @param new_job_id: The afe_job_id of a newly created job.
128*9c5db199SXin Li        @param retry_max: The maximum of times that we could retry
129*9c5db199SXin Li                          the test if the job fails.
130*9c5db199SXin Li
131*9c5db199SXin Li        @raises ValueError if new_job_id is already in retry map.
132*9c5db199SXin Li
133*9c5db199SXin Li        """
134*9c5db199SXin Li        if new_job_id in self._retry_map:
135*9c5db199SXin Li            raise ValueError('add_job called when job is already in retry map.')
136*9c5db199SXin Li
137*9c5db199SXin Li        self._retry_map[new_job_id] = {
138*9c5db199SXin Li                'state': self.States.NOT_ATTEMPTED,
139*9c5db199SXin Li                'retry_max': retry_max}
140*9c5db199SXin Li
141*9c5db199SXin Li
142*9c5db199SXin Li    def _suite_max_reached(self):
143*9c5db199SXin Li        """Return whether maximum retry limit for a suite has been reached."""
144*9c5db199SXin Li        return self._max_retries <= 0
145*9c5db199SXin Li
146*9c5db199SXin Li
147*9c5db199SXin Li    def add_retry(self, old_job_id, new_job_id):
148*9c5db199SXin Li        """Record a retry.
149*9c5db199SXin Li
150*9c5db199SXin Li        Update retry map with the retry information.
151*9c5db199SXin Li
152*9c5db199SXin Li        @param old_job_id: The afe_job_id of the job that is retried.
153*9c5db199SXin Li        @param new_job_id: The afe_job_id of the retry job.
154*9c5db199SXin Li
155*9c5db199SXin Li        @raises KeyError if old_job_id isn't in the retry map.
156*9c5db199SXin Li        @raises ValueError if we have already retried or made an attempt
157*9c5db199SXin Li                to retry the old job.
158*9c5db199SXin Li
159*9c5db199SXin Li        """
160*9c5db199SXin Li        old_record = self._retry_map[old_job_id]
161*9c5db199SXin Li        if old_record['state'] != self.States.NOT_ATTEMPTED:
162*9c5db199SXin Li            raise ValueError(
163*9c5db199SXin Li                    'We have already retried or attempted to retry job %d' %
164*9c5db199SXin Li                    old_job_id)
165*9c5db199SXin Li        old_record['state'] = self.States.RETRIED
166*9c5db199SXin Li        self._add_job(new_job_id=new_job_id,
167*9c5db199SXin Li                      retry_max=old_record['retry_max'] - 1)
168*9c5db199SXin Li        self._max_retries -= 1
169*9c5db199SXin Li
170*9c5db199SXin Li
171*9c5db199SXin Li    def set_attempted(self, job_id):
172*9c5db199SXin Li        """Set the state of the job to ATTEMPTED.
173*9c5db199SXin Li
174*9c5db199SXin Li        @param job_id: afe_job_id of a job.
175*9c5db199SXin Li
176*9c5db199SXin Li        @raises KeyError if job_id isn't in the retry map.
177*9c5db199SXin Li        @raises ValueError if the current state is not NOT_ATTEMPTED.
178*9c5db199SXin Li
179*9c5db199SXin Li        """
180*9c5db199SXin Li        current_state = self._retry_map[job_id]['state']
181*9c5db199SXin Li        if current_state != self.States.NOT_ATTEMPTED:
182*9c5db199SXin Li            # We are supposed to retry or attempt to retry each job
183*9c5db199SXin Li            # only once. Raise an error if this is not the case.
184*9c5db199SXin Li            raise ValueError('Unexpected state transition: %s -> %s' %
185*9c5db199SXin Li                             (self.States.get_string(current_state),
186*9c5db199SXin Li                              self.States.get_string(self.States.ATTEMPTED)))
187*9c5db199SXin Li        else:
188*9c5db199SXin Li            self._retry_map[job_id]['state'] = self.States.ATTEMPTED
189*9c5db199SXin Li
190*9c5db199SXin Li
191*9c5db199SXin Li    def has_following_retry(self, result):
192*9c5db199SXin Li        """Check whether there will be a following retry.
193*9c5db199SXin Li
194*9c5db199SXin Li        We have the following cases for a given job id (result.id),
195*9c5db199SXin Li        - no retry map entry -> retry not required, no following retry
196*9c5db199SXin Li        - has retry map entry:
197*9c5db199SXin Li            - already retried -> has following retry
198*9c5db199SXin Li            - has not retried
199*9c5db199SXin Li                (this branch can be handled by checking should_retry(result))
200*9c5db199SXin Li                - retry_max == 0 --> the last retry job, no more retry
201*9c5db199SXin Li                - retry_max > 0
202*9c5db199SXin Li                   - attempted, but has failed in scheduling a
203*9c5db199SXin Li                     following retry due to rpc error  --> no more retry
204*9c5db199SXin Li                   - has not attempped --> has following retry if test failed.
205*9c5db199SXin Li
206*9c5db199SXin Li        @param result: A result, encapsulating the status of the job.
207*9c5db199SXin Li
208*9c5db199SXin Li        @returns: True, if there will be a following retry.
209*9c5db199SXin Li                  False otherwise.
210*9c5db199SXin Li
211*9c5db199SXin Li        """
212*9c5db199SXin Li        return (result.test_executed
213*9c5db199SXin Li                and result.id in self._retry_map
214*9c5db199SXin Li                and (self._retry_map[result.id]['state'] == self.States.RETRIED
215*9c5db199SXin Li                     or self._should_retry(result)))
216*9c5db199SXin Li
217*9c5db199SXin Li
218*9c5db199SXin Li    def _should_retry(self, result):
219*9c5db199SXin Li        """Check whether we should retry a job based on its result.
220*9c5db199SXin Li
221*9c5db199SXin Li        We will retry the job that corresponds to the result
222*9c5db199SXin Li        when all of the following are true.
223*9c5db199SXin Li        a) The test was actually executed, meaning that if
224*9c5db199SXin Li           a job was aborted before it could ever reach the state
225*9c5db199SXin Li           of 'Running', the job will not be retried.
226*9c5db199SXin Li        b) The result is worse than |self._retry_level| which
227*9c5db199SXin Li           defaults to 'WARN'.
228*9c5db199SXin Li        c) The test requires retry, i.e. the job has an entry in the retry map.
229*9c5db199SXin Li        d) We haven't made any retry attempt yet, i.e. state == NOT_ATTEMPTED
230*9c5db199SXin Li           Note that if a test has JOB_RETRIES=5, and the second time
231*9c5db199SXin Li           it was retried it hit an rpc error, we will give up on
232*9c5db199SXin Li           all following retries.
233*9c5db199SXin Li        e) The job has not reached its retry max, i.e. retry_max > 0
234*9c5db199SXin Li
235*9c5db199SXin Li        @param result: A result, encapsulating the status of the job.
236*9c5db199SXin Li
237*9c5db199SXin Li        @returns: True if we should retry the job.
238*9c5db199SXin Li
239*9c5db199SXin Li        """
240*9c5db199SXin Li        return (
241*9c5db199SXin Li            result.test_executed
242*9c5db199SXin Li            and result.id in self._retry_map
243*9c5db199SXin Li            and not self._suite_max_reached()
244*9c5db199SXin Li            and result.is_worse_than(
245*9c5db199SXin Li                job_status.Status(self._retry_level, '', 'reason'))
246*9c5db199SXin Li            and self._retry_map[result.id]['state'] == self.States.NOT_ATTEMPTED
247*9c5db199SXin Li            and self._retry_map[result.id]['retry_max'] > 0
248*9c5db199SXin Li        )
249*9c5db199SXin Li
250*9c5db199SXin Li    def _should_retry_local_job(self, job_id):
251*9c5db199SXin Li        """Check whether we should retry a job based on information available
252*9c5db199SXin Li        for a local job without a Result object.
253*9c5db199SXin Li
254*9c5db199SXin Li        We will retry the job that corresponds to the result
255*9c5db199SXin Li        when all of the following are true.
256*9c5db199SXin Li        a) The test requires retry, i.e. the job has an entry in the retry map.
257*9c5db199SXin Li        b) We haven't made any retry attempt yet for this job, i.e.
258*9c5db199SXin Li           state == NOT_ATTEMPTED
259*9c5db199SXin Li           If the job is aborted,  we will give up on all following retries,
260*9c5db199SXin Li           regardless of max_retries.
261*9c5db199SXin Li        c) The job has not reached its retry max, i.e. retry_max > 0
262*9c5db199SXin Li
263*9c5db199SXin Li        @param job_id: the id for the job, to look up relevant information.
264*9c5db199SXin Li
265*9c5db199SXin Li        @returns: True if we should retry the job.
266*9c5db199SXin Li
267*9c5db199SXin Li        """
268*9c5db199SXin Li        if self._suite_max_reached():
269*9c5db199SXin Li            logging.debug('suite max_retries reached, not retrying.')
270*9c5db199SXin Li            return False
271*9c5db199SXin Li        if job_id not in self._retry_map:
272*9c5db199SXin Li            logging.debug('job_id not in retry map, not retrying.')
273*9c5db199SXin Li            return False
274*9c5db199SXin Li        if self._retry_map[job_id]['state'] != self.States.NOT_ATTEMPTED:
275*9c5db199SXin Li            logging.debug("job state was %s not 'Not Attempted', not retrying",
276*9c5db199SXin Li                          self._retry_map[job_id]['state'])
277*9c5db199SXin Li            return False
278*9c5db199SXin Li        if self._retry_map[job_id]['retry_max'] <= 0:
279*9c5db199SXin Li            logging.debug('test-level retries exhausted, not retrying')
280*9c5db199SXin Li            return False
281*9c5db199SXin Li        return True
282*9c5db199SXin Li
283*9c5db199SXin Li
284*9c5db199SXin Li    def job_present(self, job_id):
285*9c5db199SXin Li        """Check whether a job id present in the retry map.
286*9c5db199SXin Li
287*9c5db199SXin Li        @param job_id: afe_job_id of a job.
288*9c5db199SXin Li
289*9c5db199SXin Li        @returns: A True if the job is present, False if not.
290*9c5db199SXin Li        """
291*9c5db199SXin Li        return bool(self._retry_map.get(job_id))
292*9c5db199SXin Li
293*9c5db199SXin Li
294*9c5db199SXin Li
295*9c5db199SXin Li    def get_retry_max(self, job_id):
296*9c5db199SXin Li        """Get the maximum times the job can still be retried.
297*9c5db199SXin Li
298*9c5db199SXin Li        @param job_id: afe_job_id of a job.
299*9c5db199SXin Li
300*9c5db199SXin Li        @returns: An int, representing the maximum times the job can still be
301*9c5db199SXin Li                  retried.
302*9c5db199SXin Li        @raises KeyError if job_id isn't in the retry map.
303*9c5db199SXin Li
304*9c5db199SXin Li        """
305*9c5db199SXin Li        return self._retry_map[job_id]['retry_max']
306*9c5db199SXin Li
307*9c5db199SXin Li
308*9c5db199SXin Liclass _SuiteChildJobCreator(object):
309*9c5db199SXin Li    """Create test jobs for a suite."""
310*9c5db199SXin Li
311*9c5db199SXin Li    def __init__(
312*9c5db199SXin Li            self,
313*9c5db199SXin Li            tag,
314*9c5db199SXin Li            builds,
315*9c5db199SXin Li            board,
316*9c5db199SXin Li            afe=None,
317*9c5db199SXin Li            max_runtime_mins=24*60,
318*9c5db199SXin Li            timeout_mins=24*60,
319*9c5db199SXin Li            suite_job_id=None,
320*9c5db199SXin Li            ignore_deps=False,
321*9c5db199SXin Li            extra_deps=(),
322*9c5db199SXin Li            priority=priorities.Priority.DEFAULT,
323*9c5db199SXin Li            offload_failures_only=False,
324*9c5db199SXin Li            test_source_build=None,
325*9c5db199SXin Li            job_keyvals=None,
326*9c5db199SXin Li    ):
327*9c5db199SXin Li        """
328*9c5db199SXin Li        Constructor
329*9c5db199SXin Li
330*9c5db199SXin Li        @param tag: a string with which to tag jobs run in this suite.
331*9c5db199SXin Li        @param builds: the builds on which we're running this suite.
332*9c5db199SXin Li        @param board: the board on which we're running this suite.
333*9c5db199SXin Li        @param afe: an instance of AFE as defined in server/frontend.py.
334*9c5db199SXin Li        @param max_runtime_mins: Maximum suite runtime, in minutes.
335*9c5db199SXin Li        @param timeout_mins: Maximum job lifetime, in minutes.
336*9c5db199SXin Li        @param suite_job_id: Job id that will act as parent id to all sub jobs.
337*9c5db199SXin Li                             Default: None
338*9c5db199SXin Li        @param ignore_deps: True if jobs should ignore the DEPENDENCIES
339*9c5db199SXin Li                            attribute and skip applying of dependency labels.
340*9c5db199SXin Li                            (Default:False)
341*9c5db199SXin Li        @param extra_deps: A list of strings which are the extra DEPENDENCIES
342*9c5db199SXin Li                           to add to each test being scheduled.
343*9c5db199SXin Li        @param priority: Integer priority level.  Higher is more important.
344*9c5db199SXin Li        @param offload_failures_only: Only enable gs_offloading for failed
345*9c5db199SXin Li                                      jobs.
346*9c5db199SXin Li        @param test_source_build: Build that contains the server-side test code.
347*9c5db199SXin Li        @param job_keyvals: General job keyvals to be inserted into keyval file,
348*9c5db199SXin Li                            which will be used by tko/parse later.
349*9c5db199SXin Li        """
350*9c5db199SXin Li        self._tag = tag
351*9c5db199SXin Li        self._builds = builds
352*9c5db199SXin Li        self._board = board
353*9c5db199SXin Li        self._afe = afe or frontend_wrappers.RetryingAFE(timeout_min=30,
354*9c5db199SXin Li                                                         delay_sec=10,
355*9c5db199SXin Li                                                         debug=False)
356*9c5db199SXin Li        self._max_runtime_mins = max_runtime_mins
357*9c5db199SXin Li        self._timeout_mins = timeout_mins
358*9c5db199SXin Li        self._suite_job_id = suite_job_id
359*9c5db199SXin Li        self._ignore_deps = ignore_deps
360*9c5db199SXin Li        self._extra_deps = tuple(extra_deps)
361*9c5db199SXin Li        self._priority = priority
362*9c5db199SXin Li        self._offload_failures_only = offload_failures_only
363*9c5db199SXin Li        self._test_source_build = test_source_build
364*9c5db199SXin Li        self._job_keyvals = job_keyvals
365*9c5db199SXin Li
366*9c5db199SXin Li
367*9c5db199SXin Li    @property
368*9c5db199SXin Li    def cros_build(self):
369*9c5db199SXin Li        """Return the CrOS build or the first build in the builds dict."""
370*9c5db199SXin Li        # TODO(ayatane): Note that the builds dict isn't ordered.  I'm not
371*9c5db199SXin Li        # sure what the implications of this are, but it's probably not a
372*9c5db199SXin Li        # good thing.
373*9c5db199SXin Li        return self._builds.get(provision.CROS_VERSION_PREFIX,
374*9c5db199SXin Li                                list(self._builds.values())[0])
375*9c5db199SXin Li
376*9c5db199SXin Li
377*9c5db199SXin Li    def create_job(self, test, retry_for=None):
378*9c5db199SXin Li        """
379*9c5db199SXin Li        Thin wrapper around frontend.AFE.create_job().
380*9c5db199SXin Li
381*9c5db199SXin Li        @param test: ControlData object for a test to run.
382*9c5db199SXin Li        @param retry_for: If the to-be-created job is a retry for an
383*9c5db199SXin Li                          old job, the afe_job_id of the old job will
384*9c5db199SXin Li                          be passed in as |retry_for|, which will be
385*9c5db199SXin Li                          recorded in the new job's keyvals.
386*9c5db199SXin Li        @returns: A frontend.Job object with an added test_name member.
387*9c5db199SXin Li                  test_name is used to preserve the higher level TEST_NAME
388*9c5db199SXin Li                  name of the job.
389*9c5db199SXin Li        """
390*9c5db199SXin Li        # For a system running multiple suites which share tests, the priority
391*9c5db199SXin Li        # overridden may lead to unexpected scheduling order that adds extra
392*9c5db199SXin Li        # provision jobs.
393*9c5db199SXin Li        test_priority = self._priority
394*9c5db199SXin Li        if utils.is_moblab():
395*9c5db199SXin Li            test_priority = max(self._priority, test.priority)
396*9c5db199SXin Li
397*9c5db199SXin Li        reboot_before = (model_attributes.RebootBefore.NEVER if test.fast
398*9c5db199SXin Li                         else None)
399*9c5db199SXin Li
400*9c5db199SXin Li        test_obj = self._afe.create_job(
401*9c5db199SXin Li            control_file=test.text,
402*9c5db199SXin Li            name=tools.create_job_name(
403*9c5db199SXin Li                    self._test_source_build or self.cros_build,
404*9c5db199SXin Li                    self._tag,
405*9c5db199SXin Li                    test.name),
406*9c5db199SXin Li            control_type=test.test_type.capitalize(),
407*9c5db199SXin Li            meta_hosts=[self._board]*test.sync_count,
408*9c5db199SXin Li            dependencies=self._create_job_deps(test),
409*9c5db199SXin Li            keyvals=self._create_keyvals_for_test_job(test, retry_for),
410*9c5db199SXin Li            max_runtime_mins=self._max_runtime_mins,
411*9c5db199SXin Li            timeout_mins=self._timeout_mins,
412*9c5db199SXin Li            parent_job_id=self._suite_job_id,
413*9c5db199SXin Li            reboot_before=reboot_before,
414*9c5db199SXin Li            run_reset=not test.fast,
415*9c5db199SXin Li            priority=test_priority,
416*9c5db199SXin Li            synch_count=test.sync_count,
417*9c5db199SXin Li            require_ssp=test.require_ssp)
418*9c5db199SXin Li
419*9c5db199SXin Li        test_obj.test_name = test.name
420*9c5db199SXin Li        return test_obj
421*9c5db199SXin Li
422*9c5db199SXin Li
423*9c5db199SXin Li    def _create_job_deps(self, test):
424*9c5db199SXin Li        """Create job deps list for a test job.
425*9c5db199SXin Li
426*9c5db199SXin Li        @returns: A list of dependency strings.
427*9c5db199SXin Li        """
428*9c5db199SXin Li        if self._ignore_deps:
429*9c5db199SXin Li            job_deps = []
430*9c5db199SXin Li        else:
431*9c5db199SXin Li            job_deps = list(test.dependencies)
432*9c5db199SXin Li        job_deps.extend(self._extra_deps)
433*9c5db199SXin Li        return job_deps
434*9c5db199SXin Li
435*9c5db199SXin Li
436*9c5db199SXin Li    def _create_keyvals_for_test_job(self, test, retry_for=None):
437*9c5db199SXin Li        """Create keyvals dict for creating a test job.
438*9c5db199SXin Li
439*9c5db199SXin Li        @param test: ControlData object for a test to run.
440*9c5db199SXin Li        @param retry_for: If the to-be-created job is a retry for an
441*9c5db199SXin Li                          old job, the afe_job_id of the old job will
442*9c5db199SXin Li                          be passed in as |retry_for|, which will be
443*9c5db199SXin Li                          recorded in the new job's keyvals.
444*9c5db199SXin Li        @returns: A keyvals dict for creating the test job.
445*9c5db199SXin Li        """
446*9c5db199SXin Li        keyvals = {
447*9c5db199SXin Li            constants.JOB_BUILD_KEY: self.cros_build,
448*9c5db199SXin Li            constants.JOB_SUITE_KEY: self._tag,
449*9c5db199SXin Li            constants.JOB_EXPERIMENTAL_KEY: test.experimental,
450*9c5db199SXin Li            constants.JOB_BUILDS_KEY: self._builds
451*9c5db199SXin Li        }
452*9c5db199SXin Li        # test_source_build is saved to job_keyvals so scheduler can retrieve
453*9c5db199SXin Li        # the build name from database when compiling autoserv commandline.
454*9c5db199SXin Li        # This avoid a database change to add a new field in afe_jobs.
455*9c5db199SXin Li        #
456*9c5db199SXin Li        # Only add `test_source_build` to job keyvals if the build is different
457*9c5db199SXin Li        # from the CrOS build or the job uses more than one build, e.g., both
458*9c5db199SXin Li        # firmware and CrOS will be updated in the dut.
459*9c5db199SXin Li        # This is for backwards compatibility, so the update Autotest code can
460*9c5db199SXin Li        # compile an autoserv command line to run in a SSP container using
461*9c5db199SXin Li        # previous builds.
462*9c5db199SXin Li        if (self._test_source_build and
463*9c5db199SXin Li            (self.cros_build != self._test_source_build or
464*9c5db199SXin Li             len(self._builds) > 1)):
465*9c5db199SXin Li            keyvals[constants.JOB_TEST_SOURCE_BUILD_KEY] = \
466*9c5db199SXin Li                    self._test_source_build
467*9c5db199SXin Li            for prefix, build in six.iteritems(self._builds):
468*9c5db199SXin Li                if prefix == provision.FW_RW_VERSION_PREFIX:
469*9c5db199SXin Li                    keyvals[constants.FWRW_BUILD]= build
470*9c5db199SXin Li                elif prefix == provision.FW_RO_VERSION_PREFIX:
471*9c5db199SXin Li                    keyvals[constants.FWRO_BUILD] = build
472*9c5db199SXin Li        # Add suite job id to keyvals so tko parser can read it from keyval
473*9c5db199SXin Li        # file.
474*9c5db199SXin Li        if self._suite_job_id:
475*9c5db199SXin Li            keyvals[constants.PARENT_JOB_ID] = self._suite_job_id
476*9c5db199SXin Li        # We drop the old job's id in the new job's keyval file so that
477*9c5db199SXin Li        # later our tko parser can figure out the retry relationship and
478*9c5db199SXin Li        # invalidate the results of the old job in tko database.
479*9c5db199SXin Li        if retry_for:
480*9c5db199SXin Li            keyvals[constants.RETRY_ORIGINAL_JOB_ID] = retry_for
481*9c5db199SXin Li        if self._offload_failures_only:
482*9c5db199SXin Li            keyvals[constants.JOB_OFFLOAD_FAILURES_KEY] = True
483*9c5db199SXin Li        if self._job_keyvals:
484*9c5db199SXin Li            for key in constants.INHERITED_KEYVALS:
485*9c5db199SXin Li                if key in self._job_keyvals:
486*9c5db199SXin Li                    keyvals[key] = self._job_keyvals[key]
487*9c5db199SXin Li        return keyvals
488*9c5db199SXin Li
489*9c5db199SXin Li
490*9c5db199SXin Liclass _ControlFileRetriever(object):
491*9c5db199SXin Li    """Retrieves control files.
492*9c5db199SXin Li
493*9c5db199SXin Li    This returns control data instances, unlike control file getters
494*9c5db199SXin Li    which simply return the control file text contents.
495*9c5db199SXin Li    """
496*9c5db199SXin Li
497*9c5db199SXin Li    def __init__(self, cf_getter, forgiving_parser=True, run_prod_code=False,
498*9c5db199SXin Li                 test_args=None):
499*9c5db199SXin Li        """Initialize instance.
500*9c5db199SXin Li
501*9c5db199SXin Li        @param cf_getter: a control_file_getter.ControlFileGetter used to list
502*9c5db199SXin Li               and fetch the content of control files
503*9c5db199SXin Li        @param forgiving_parser: If False, will raise ControlVariableExceptions
504*9c5db199SXin Li                                 if any are encountered when parsing control
505*9c5db199SXin Li                                 files. Note that this can raise an exception
506*9c5db199SXin Li                                 for syntax errors in unrelated files, because
507*9c5db199SXin Li                                 we parse them before applying the predicate.
508*9c5db199SXin Li        @param run_prod_code: If true, the retrieved tests will run the test
509*9c5db199SXin Li                              code that lives in prod aka the test code
510*9c5db199SXin Li                              currently on the lab servers by disabling
511*9c5db199SXin Li                              SSP for the discovered tests.
512*9c5db199SXin Li        @param test_args: A dict of args to be seeded in test control file under
513*9c5db199SXin Li                          the name |args_dict|.
514*9c5db199SXin Li        """
515*9c5db199SXin Li        self._cf_getter = cf_getter
516*9c5db199SXin Li        self._forgiving_parser = forgiving_parser
517*9c5db199SXin Li        self._run_prod_code = run_prod_code
518*9c5db199SXin Li        self._test_args = test_args
519*9c5db199SXin Li
520*9c5db199SXin Li
521*9c5db199SXin Li    def retrieve_for_test(self, test_name):
522*9c5db199SXin Li        """Retrieve a test's control data.
523*9c5db199SXin Li
524*9c5db199SXin Li        This ignores forgiving_parser because we cannot return a
525*9c5db199SXin Li        forgiving value.
526*9c5db199SXin Li
527*9c5db199SXin Li        @param test_name: Name of test to retrieve.
528*9c5db199SXin Li
529*9c5db199SXin Li        @raises ControlVariableException: There is a syntax error in a
530*9c5db199SXin Li                                          control file.
531*9c5db199SXin Li
532*9c5db199SXin Li        @returns a ControlData object
533*9c5db199SXin Li        """
534*9c5db199SXin Li        return suite_common.retrieve_control_data_for_test(
535*9c5db199SXin Li                self._cf_getter, test_name)
536*9c5db199SXin Li
537*9c5db199SXin Li
538*9c5db199SXin Li    def retrieve_for_suite(self, suite_name=''):
539*9c5db199SXin Li        """Scan through all tests and find all tests.
540*9c5db199SXin Li
541*9c5db199SXin Li        @param suite_name: If specified, this method will attempt to restrain
542*9c5db199SXin Li                           the search space to just this suite's control files.
543*9c5db199SXin Li
544*9c5db199SXin Li        @raises ControlVariableException: If forgiving_parser is False and there
545*9c5db199SXin Li                                          is a syntax error in a control file.
546*9c5db199SXin Li
547*9c5db199SXin Li        @returns a dictionary of ControlData objects that based on given
548*9c5db199SXin Li                 parameters.
549*9c5db199SXin Li        """
550*9c5db199SXin Li        tests = suite_common.retrieve_for_suite(
551*9c5db199SXin Li                self._cf_getter, suite_name, self._forgiving_parser,
552*9c5db199SXin Li                self._test_args)
553*9c5db199SXin Li        if self._run_prod_code:
554*9c5db199SXin Li            for test in six.itervalues(tests):
555*9c5db199SXin Li                test.require_ssp = False
556*9c5db199SXin Li
557*9c5db199SXin Li        return tests
558*9c5db199SXin Li
559*9c5db199SXin Li
560*9c5db199SXin Lidef list_all_suites(build, devserver, cf_getter=None):
561*9c5db199SXin Li    """
562*9c5db199SXin Li    Parses all ControlData objects with a SUITE tag and extracts all
563*9c5db199SXin Li    defined suite names.
564*9c5db199SXin Li
565*9c5db199SXin Li    @param build: the build on which we're running this suite.
566*9c5db199SXin Li    @param devserver: the devserver which contains the build.
567*9c5db199SXin Li    @param cf_getter: control_file_getter.ControlFileGetter. Defaults to
568*9c5db199SXin Li                      using DevServerGetter.
569*9c5db199SXin Li
570*9c5db199SXin Li    @return list of suites
571*9c5db199SXin Li    """
572*9c5db199SXin Li    if cf_getter is None:
573*9c5db199SXin Li        cf_getter = _create_ds_getter(build, devserver)
574*9c5db199SXin Li
575*9c5db199SXin Li    suites = set()
576*9c5db199SXin Li    predicate = lambda t: True
577*9c5db199SXin Li    for test in find_and_parse_tests(cf_getter, predicate):
578*9c5db199SXin Li        suites.update(test.suite_tag_parts)
579*9c5db199SXin Li    return list(suites)
580*9c5db199SXin Li
581*9c5db199SXin Li
582*9c5db199SXin Lidef test_file_similarity_predicate(test_file_pattern):
583*9c5db199SXin Li    """Returns predicate that gets the similarity based on a test's file
584*9c5db199SXin Li    name pattern.
585*9c5db199SXin Li
586*9c5db199SXin Li    Builds a predicate that takes in a parsed control file (a ControlData)
587*9c5db199SXin Li    and returns a tuple of (file path, ratio), where ratio is the
588*9c5db199SXin Li    similarity between the test file name and the given test_file_pattern.
589*9c5db199SXin Li
590*9c5db199SXin Li    @param test_file_pattern: regular expression (string) to match against
591*9c5db199SXin Li                              control file names.
592*9c5db199SXin Li    @return a callable that takes a ControlData and and returns a tuple of
593*9c5db199SXin Li            (file path, ratio), where ratio is the similarity between the
594*9c5db199SXin Li            test file name and the given test_file_pattern.
595*9c5db199SXin Li    """
596*9c5db199SXin Li    return lambda t: ((None, 0) if not hasattr(t, 'path') else
597*9c5db199SXin Li            (t.path, difflib.SequenceMatcher(a=t.path,
598*9c5db199SXin Li                                             b=test_file_pattern).ratio()))
599*9c5db199SXin Li
600*9c5db199SXin Li
601*9c5db199SXin Lidef test_name_similarity_predicate(test_name):
602*9c5db199SXin Li    """Returns predicate that matched based on a test's name.
603*9c5db199SXin Li
604*9c5db199SXin Li    Builds a predicate that takes in a parsed control file (a ControlData)
605*9c5db199SXin Li    and returns a tuple of (test name, ratio), where ratio is the similarity
606*9c5db199SXin Li    between the test name and the given test_name.
607*9c5db199SXin Li
608*9c5db199SXin Li    @param test_name: the test name to base the predicate on.
609*9c5db199SXin Li    @return a callable that takes a ControlData and returns a tuple of
610*9c5db199SXin Li            (test name, ratio), where ratio is the similarity between the
611*9c5db199SXin Li            test name and the given test_name.
612*9c5db199SXin Li    """
613*9c5db199SXin Li    return lambda t: ((None, 0) if not hasattr(t, 'name') else
614*9c5db199SXin Li            (t.name,
615*9c5db199SXin Li             difflib.SequenceMatcher(a=t.name, b=test_name).ratio()))
616*9c5db199SXin Li
617*9c5db199SXin Li
618*9c5db199SXin Lidef matches_attribute_expression_predicate(test_attr_boolstr):
619*9c5db199SXin Li    """Returns predicate that matches based on boolean expression of
620*9c5db199SXin Li    attributes.
621*9c5db199SXin Li
622*9c5db199SXin Li    Builds a predicate that takes in a parsed control file (a ControlData)
623*9c5db199SXin Li    ans returns True if the test attributes satisfy the given attribute
624*9c5db199SXin Li    boolean expression.
625*9c5db199SXin Li
626*9c5db199SXin Li    @param test_attr_boolstr: boolean expression of the attributes to be
627*9c5db199SXin Li                              test, like 'system:all and interval:daily'.
628*9c5db199SXin Li
629*9c5db199SXin Li    @return a callable that takes a ControlData and returns True if the test
630*9c5db199SXin Li            attributes satisfy the given boolean expression.
631*9c5db199SXin Li    """
632*9c5db199SXin Li    return lambda t: boolparse_lib.BoolstrResult(
633*9c5db199SXin Li        test_attr_boolstr, t.attributes)
634*9c5db199SXin Li
635*9c5db199SXin Li
636*9c5db199SXin Lidef test_file_matches_pattern_predicate(test_file_pattern):
637*9c5db199SXin Li    """Returns predicate that matches based on a test's file name pattern.
638*9c5db199SXin Li
639*9c5db199SXin Li    Builds a predicate that takes in a parsed control file (a ControlData)
640*9c5db199SXin Li    and returns True if the test's control file name matches the given
641*9c5db199SXin Li    regular expression.
642*9c5db199SXin Li
643*9c5db199SXin Li    @param test_file_pattern: regular expression (string) to match against
644*9c5db199SXin Li                              control file names.
645*9c5db199SXin Li    @return a callable that takes a ControlData and and returns
646*9c5db199SXin Li            True if control file name matches the pattern.
647*9c5db199SXin Li    """
648*9c5db199SXin Li    return lambda t: hasattr(t, 'path') and re.match(test_file_pattern,
649*9c5db199SXin Li                                                     t.path)
650*9c5db199SXin Li
651*9c5db199SXin Li
652*9c5db199SXin Lidef test_name_matches_pattern_predicate(test_name_pattern):
653*9c5db199SXin Li    """Returns predicate that matches based on a test's name pattern.
654*9c5db199SXin Li
655*9c5db199SXin Li    Builds a predicate that takes in a parsed control file (a ControlData)
656*9c5db199SXin Li    and returns True if the test name matches the given regular expression.
657*9c5db199SXin Li
658*9c5db199SXin Li    @param test_name_pattern: regular expression (string) to match against
659*9c5db199SXin Li                              test names.
660*9c5db199SXin Li    @return a callable that takes a ControlData and returns
661*9c5db199SXin Li            True if the name fields matches the pattern.
662*9c5db199SXin Li    """
663*9c5db199SXin Li    return lambda t: hasattr(t, 'name') and re.match(test_name_pattern,
664*9c5db199SXin Li                                                     t.name)
665*9c5db199SXin Li
666*9c5db199SXin Li
667*9c5db199SXin Lidef test_name_equals_predicate(test_name):
668*9c5db199SXin Li    """Returns predicate that matched based on a test's name.
669*9c5db199SXin Li
670*9c5db199SXin Li    Builds a predicate that takes in a parsed control file (a ControlData)
671*9c5db199SXin Li    and returns True if the test name is equal to |test_name|.
672*9c5db199SXin Li
673*9c5db199SXin Li    @param test_name: the test name to base the predicate on.
674*9c5db199SXin Li    @return a callable that takes a ControlData and looks for |test_name|
675*9c5db199SXin Li            in that ControlData's name.
676*9c5db199SXin Li    """
677*9c5db199SXin Li    return lambda t: hasattr(t, 'name') and test_name == t.name
678*9c5db199SXin Li
679*9c5db199SXin Li
680*9c5db199SXin Lidef name_in_tag_similarity_predicate(name):
681*9c5db199SXin Li    """Returns predicate that takes a control file and gets the similarity
682*9c5db199SXin Li    of the suites in the control file and the given name.
683*9c5db199SXin Li
684*9c5db199SXin Li    Builds a predicate that takes in a parsed control file (a ControlData)
685*9c5db199SXin Li    and returns a list of tuples of (suite name, ratio), where suite name
686*9c5db199SXin Li    is each suite listed in the control file, and ratio is the similarity
687*9c5db199SXin Li    between each suite and the given name.
688*9c5db199SXin Li
689*9c5db199SXin Li    @param name: the suite name to base the predicate on.
690*9c5db199SXin Li    @return a callable that takes a ControlData and returns a list of tuples
691*9c5db199SXin Li            of (suite name, ratio), where suite name is each suite listed in
692*9c5db199SXin Li            the control file, and ratio is the similarity between each suite
693*9c5db199SXin Li            and the given name.
694*9c5db199SXin Li    """
695*9c5db199SXin Li    return lambda t: [(suite,
696*9c5db199SXin Li                       difflib.SequenceMatcher(a=suite, b=name).ratio())
697*9c5db199SXin Li                      for suite in t.suite_tag_parts] or [(None, 0)]
698*9c5db199SXin Li
699*9c5db199SXin Li
700*9c5db199SXin Lidef name_in_tag_predicate(name):
701*9c5db199SXin Li    """Returns predicate that takes a control file and looks for |name|.
702*9c5db199SXin Li
703*9c5db199SXin Li    Builds a predicate that takes in a parsed control file (a ControlData)
704*9c5db199SXin Li    and returns True if the SUITE tag is present and contains |name|.
705*9c5db199SXin Li
706*9c5db199SXin Li    @param name: the suite name to base the predicate on.
707*9c5db199SXin Li    @return a callable that takes a ControlData and looks for |name| in that
708*9c5db199SXin Li            ControlData object's suite member.
709*9c5db199SXin Li    """
710*9c5db199SXin Li    return suite_common.name_in_tag_predicate(name)
711*9c5db199SXin Li
712*9c5db199SXin Li
713*9c5db199SXin Lidef create_fs_getter(autotest_dir):
714*9c5db199SXin Li    """
715*9c5db199SXin Li    @param autotest_dir: the place to find autotests.
716*9c5db199SXin Li    @return a FileSystemGetter instance that looks under |autotest_dir|.
717*9c5db199SXin Li    """
718*9c5db199SXin Li    # currently hard-coded places to look for tests.
719*9c5db199SXin Li    subpaths = ['server/site_tests', 'client/site_tests',
720*9c5db199SXin Li                'server/tests', 'client/tests']
721*9c5db199SXin Li    directories = [os.path.join(autotest_dir, p) for p in subpaths]
722*9c5db199SXin Li    return control_file_getter.FileSystemGetter(directories)
723*9c5db199SXin Li
724*9c5db199SXin Li
725*9c5db199SXin Lidef _create_ds_getter(build, devserver):
726*9c5db199SXin Li    """
727*9c5db199SXin Li    @param build: the build on which we're running this suite.
728*9c5db199SXin Li    @param devserver: the devserver which contains the build.
729*9c5db199SXin Li    @return a FileSystemGetter instance that looks under |autotest_dir|.
730*9c5db199SXin Li    """
731*9c5db199SXin Li    return control_file_getter.DevServerGetter(build, devserver)
732*9c5db199SXin Li
733*9c5db199SXin Li
734*9c5db199SXin Lidef _non_experimental_tests_predicate(test_data):
735*9c5db199SXin Li    """Test predicate for non-experimental tests."""
736*9c5db199SXin Li    return not test_data.experimental
737*9c5db199SXin Li
738*9c5db199SXin Li
739*9c5db199SXin Lidef find_and_parse_tests(cf_getter, predicate, suite_name='',
740*9c5db199SXin Li                         add_experimental=False, forgiving_parser=True,
741*9c5db199SXin Li                         run_prod_code=False, test_args=None):
742*9c5db199SXin Li    """
743*9c5db199SXin Li    Function to scan through all tests and find eligible tests.
744*9c5db199SXin Li
745*9c5db199SXin Li    Search through all tests based on given cf_getter, suite_name,
746*9c5db199SXin Li    add_experimental and forgiving_parser, return the tests that match
747*9c5db199SXin Li    given predicate.
748*9c5db199SXin Li
749*9c5db199SXin Li    @param cf_getter: a control_file_getter.ControlFileGetter used to list
750*9c5db199SXin Li           and fetch the content of control files
751*9c5db199SXin Li    @param predicate: a function that should return True when run over a
752*9c5db199SXin Li           ControlData representation of a control file that should be in
753*9c5db199SXin Li           this Suite.
754*9c5db199SXin Li    @param suite_name: If specified, this method will attempt to restrain
755*9c5db199SXin Li                       the search space to just this suite's control files.
756*9c5db199SXin Li    @param add_experimental: add tests with experimental attribute set.
757*9c5db199SXin Li    @param forgiving_parser: If False, will raise ControlVariableExceptions
758*9c5db199SXin Li                             if any are encountered when parsing control
759*9c5db199SXin Li                             files. Note that this can raise an exception
760*9c5db199SXin Li                             for syntax errors in unrelated files, because
761*9c5db199SXin Li                             we parse them before applying the predicate.
762*9c5db199SXin Li    @param run_prod_code: If true, the suite will run the test code that
763*9c5db199SXin Li                          lives in prod aka the test code currently on the
764*9c5db199SXin Li                          lab servers by disabling SSP for the discovered
765*9c5db199SXin Li                          tests.
766*9c5db199SXin Li    @param test_args: A dict of args to be seeded in test control file.
767*9c5db199SXin Li
768*9c5db199SXin Li    @raises ControlVariableException: If forgiving_parser is False and there
769*9c5db199SXin Li                                      is a syntax error in a control file.
770*9c5db199SXin Li
771*9c5db199SXin Li    @return list of ControlData objects that should be run, with control
772*9c5db199SXin Li            file text added in |text| attribute. Results are sorted based
773*9c5db199SXin Li            on the TIME setting in control file, slowest test comes first.
774*9c5db199SXin Li    """
775*9c5db199SXin Li    logging.debug('Getting control file list for suite: %s', suite_name)
776*9c5db199SXin Li    retriever = _ControlFileRetriever(cf_getter,
777*9c5db199SXin Li                                      forgiving_parser=forgiving_parser,
778*9c5db199SXin Li                                      run_prod_code=run_prod_code,
779*9c5db199SXin Li                                      test_args=test_args)
780*9c5db199SXin Li    tests = retriever.retrieve_for_suite(suite_name)
781*9c5db199SXin Li    if not add_experimental:
782*9c5db199SXin Li        predicate = _ComposedPredicate([predicate,
783*9c5db199SXin Li                                        _non_experimental_tests_predicate])
784*9c5db199SXin Li    return suite_common.filter_tests(tests, predicate)
785*9c5db199SXin Li
786*9c5db199SXin Li
787*9c5db199SXin Lidef find_possible_tests(cf_getter, predicate, suite_name='', count=10):
788*9c5db199SXin Li    """
789*9c5db199SXin Li    Function to scan through all tests and find possible tests.
790*9c5db199SXin Li
791*9c5db199SXin Li    Search through all tests based on given cf_getter, suite_name,
792*9c5db199SXin Li    add_experimental and forgiving_parser. Use the given predicate to
793*9c5db199SXin Li    calculate the similarity and return the top 10 matches.
794*9c5db199SXin Li
795*9c5db199SXin Li    @param cf_getter: a control_file_getter.ControlFileGetter used to list
796*9c5db199SXin Li           and fetch the content of control files
797*9c5db199SXin Li    @param predicate: a function that should return a tuple of (name, ratio)
798*9c5db199SXin Li           when run over a ControlData representation of a control file that
799*9c5db199SXin Li           should be in this Suite. `name` is the key to be compared, e.g.,
800*9c5db199SXin Li           a suite name or test name. `ratio` is a value between [0,1]
801*9c5db199SXin Li           indicating the similarity of `name` and the value to be compared.
802*9c5db199SXin Li    @param suite_name: If specified, this method will attempt to restrain
803*9c5db199SXin Li                       the search space to just this suite's control files.
804*9c5db199SXin Li    @param count: Number of suggestions to return, default to 10.
805*9c5db199SXin Li
806*9c5db199SXin Li    @return list of top names that similar to the given test, sorted by
807*9c5db199SXin Li            match ratio.
808*9c5db199SXin Li    """
809*9c5db199SXin Li    logging.debug('Getting control file list for suite: %s', suite_name)
810*9c5db199SXin Li    tests = _ControlFileRetriever(cf_getter).retrieve_for_suite(suite_name)
811*9c5db199SXin Li    logging.debug('Parsed %s control files.', len(tests))
812*9c5db199SXin Li    similarities = {}
813*9c5db199SXin Li    for test in six.itervalues(tests):
814*9c5db199SXin Li        ratios = predicate(test)
815*9c5db199SXin Li        # Some predicates may return a list of tuples, e.g.,
816*9c5db199SXin Li        # name_in_tag_similarity_predicate. Convert all returns to a list.
817*9c5db199SXin Li        if not isinstance(ratios, list):
818*9c5db199SXin Li            ratios = [ratios]
819*9c5db199SXin Li        for name, ratio in ratios:
820*9c5db199SXin Li            similarities[name] = ratio
821*9c5db199SXin Li    return [s[0] for s in
822*9c5db199SXin Li            sorted(list(similarities.items()), key=operator.itemgetter(1),
823*9c5db199SXin Li                   reverse=True)][:count]
824*9c5db199SXin Li
825*9c5db199SXin Li
826*9c5db199SXin Lidef _deprecated_suite_method(func):
827*9c5db199SXin Li    """Decorator for deprecated Suite static methods.
828*9c5db199SXin Li
829*9c5db199SXin Li    TODO(ayatane): This is used to decorate functions that are called as
830*9c5db199SXin Li    static methods on Suite.
831*9c5db199SXin Li    """
832*9c5db199SXin Li    @functools.wraps(func)
833*9c5db199SXin Li    def wrapper(*args, **kwargs):
834*9c5db199SXin Li        """Wraps |func| for warning."""
835*9c5db199SXin Li        warnings.warn('Calling method "%s" from Suite is deprecated' %
836*9c5db199SXin Li                      func.__name__)
837*9c5db199SXin Li        return func(*args, **kwargs)
838*9c5db199SXin Li    return staticmethod(wrapper)
839*9c5db199SXin Li
840*9c5db199SXin Li
841*9c5db199SXin Liclass _BaseSuite(object):
842*9c5db199SXin Li    """
843*9c5db199SXin Li    A suite of tests, defined by some predicate over control file variables.
844*9c5db199SXin Li
845*9c5db199SXin Li    Given a place to search for control files a predicate to match the desired
846*9c5db199SXin Li    tests, can gather tests and fire off jobs to run them, and then wait for
847*9c5db199SXin Li    results.
848*9c5db199SXin Li
849*9c5db199SXin Li    @var _predicate: a function that should return True when run over a
850*9c5db199SXin Li         ControlData representation of a control file that should be in
851*9c5db199SXin Li         this Suite.
852*9c5db199SXin Li    @var _tag: a string with which to tag jobs run in this suite.
853*9c5db199SXin Li    @var _builds: the builds on which we're running this suite.
854*9c5db199SXin Li    @var _afe: an instance of AFE as defined in server/frontend.py.
855*9c5db199SXin Li    @var _tko: an instance of TKO as defined in server/frontend.py.
856*9c5db199SXin Li    @var _jobs: currently scheduled jobs, if any.
857*9c5db199SXin Li    @var _jobs_to_tests: a dictionary that maps job ids to tests represented
858*9c5db199SXin Li                         ControlData objects.
859*9c5db199SXin Li    @var _retry: a bool value indicating whether jobs should be retried on
860*9c5db199SXin Li                 failure.
861*9c5db199SXin Li    @var _retry_handler: a RetryHandler object.
862*9c5db199SXin Li
863*9c5db199SXin Li    """
864*9c5db199SXin Li
865*9c5db199SXin Li
866*9c5db199SXin Li    def __init__(
867*9c5db199SXin Li            self,
868*9c5db199SXin Li            tests,
869*9c5db199SXin Li            tag,
870*9c5db199SXin Li            builds,
871*9c5db199SXin Li            board,
872*9c5db199SXin Li            afe=None,
873*9c5db199SXin Li            tko=None,
874*9c5db199SXin Li            pool=None,
875*9c5db199SXin Li            results_dir=None,
876*9c5db199SXin Li            max_runtime_mins=24*60,
877*9c5db199SXin Li            timeout_mins=24*60,
878*9c5db199SXin Li            file_bugs=False,
879*9c5db199SXin Li            suite_job_id=None,
880*9c5db199SXin Li            ignore_deps=False,
881*9c5db199SXin Li            extra_deps=None,
882*9c5db199SXin Li            priority=priorities.Priority.DEFAULT,
883*9c5db199SXin Li            wait_for_results=True,
884*9c5db199SXin Li            job_retry=False,
885*9c5db199SXin Li            max_retries=sys.maxsize,
886*9c5db199SXin Li            offload_failures_only=False,
887*9c5db199SXin Li            test_source_build=None,
888*9c5db199SXin Li            job_keyvals=None,
889*9c5db199SXin Li            child_dependencies=(),
890*9c5db199SXin Li            result_reporter=None,
891*9c5db199SXin Li    ):
892*9c5db199SXin Li        """Initialize instance.
893*9c5db199SXin Li
894*9c5db199SXin Li        @param tests: Iterable of tests to run.
895*9c5db199SXin Li        @param tag: a string with which to tag jobs run in this suite.
896*9c5db199SXin Li        @param builds: the builds on which we're running this suite.
897*9c5db199SXin Li        @param board: the board on which we're running this suite.
898*9c5db199SXin Li        @param afe: an instance of AFE as defined in server/frontend.py.
899*9c5db199SXin Li        @param tko: an instance of TKO as defined in server/frontend.py.
900*9c5db199SXin Li        @param pool: Specify the pool of machines to use for scheduling
901*9c5db199SXin Li                purposes.
902*9c5db199SXin Li        @param results_dir: The directory where the job can write results to.
903*9c5db199SXin Li                            This must be set if you want job_id of sub-jobs
904*9c5db199SXin Li                            list in the job keyvals.
905*9c5db199SXin Li        @param max_runtime_mins: Maximum suite runtime, in minutes.
906*9c5db199SXin Li        @param timeout: Maximum job lifetime, in hours.
907*9c5db199SXin Li        @param suite_job_id: Job id that will act as parent id to all sub jobs.
908*9c5db199SXin Li                             Default: None
909*9c5db199SXin Li        @param ignore_deps: True if jobs should ignore the DEPENDENCIES
910*9c5db199SXin Li                            attribute and skip applying of dependency labels.
911*9c5db199SXin Li                            (Default:False)
912*9c5db199SXin Li        @param extra_deps: A list of strings which are the extra DEPENDENCIES
913*9c5db199SXin Li                           to add to each test being scheduled.
914*9c5db199SXin Li        @param priority: Integer priority level.  Higher is more important.
915*9c5db199SXin Li        @param wait_for_results: Set to False to run the suite job without
916*9c5db199SXin Li                                 waiting for test jobs to finish. Default is
917*9c5db199SXin Li                                 True.
918*9c5db199SXin Li        @param job_retry: A bool value indicating whether jobs should be retried
919*9c5db199SXin Li                          on failure. If True, the field 'JOB_RETRIES' in
920*9c5db199SXin Li                          control files will be respected. If False, do not
921*9c5db199SXin Li                          retry.
922*9c5db199SXin Li        @param max_retries: Maximum retry limit at suite level.
923*9c5db199SXin Li                            Regardless how many times each individual test
924*9c5db199SXin Li                            has been retried, the total number of retries
925*9c5db199SXin Li                            happening in the suite can't exceed _max_retries.
926*9c5db199SXin Li                            Default to sys.maxint.
927*9c5db199SXin Li        @param offload_failures_only: Only enable gs_offloading for failed
928*9c5db199SXin Li                                      jobs.
929*9c5db199SXin Li        @param test_source_build: Build that contains the server-side test code.
930*9c5db199SXin Li        @param job_keyvals: General job keyvals to be inserted into keyval file,
931*9c5db199SXin Li                            which will be used by tko/parse later.
932*9c5db199SXin Li        @param child_dependencies: (optional) list of dependency strings
933*9c5db199SXin Li                to be added as dependencies to child jobs.
934*9c5db199SXin Li        @param result_reporter: A _ResultReporter instance to report results. If
935*9c5db199SXin Li                None, an _EmailReporter will be created.
936*9c5db199SXin Li        """
937*9c5db199SXin Li
938*9c5db199SXin Li        self.tests = list(tests)
939*9c5db199SXin Li        self._tag = tag
940*9c5db199SXin Li        self._builds = builds
941*9c5db199SXin Li        self._results_dir = results_dir
942*9c5db199SXin Li        self._afe = afe or frontend_wrappers.RetryingAFE(timeout_min=30,
943*9c5db199SXin Li                                                         delay_sec=10,
944*9c5db199SXin Li                                                         debug=False)
945*9c5db199SXin Li        self._tko = tko or frontend_wrappers.RetryingTKO(timeout_min=30,
946*9c5db199SXin Li                                                         delay_sec=10,
947*9c5db199SXin Li                                                         debug=False)
948*9c5db199SXin Li        self._jobs = []
949*9c5db199SXin Li        self._jobs_to_tests = {}
950*9c5db199SXin Li
951*9c5db199SXin Li        self._file_bugs = file_bugs
952*9c5db199SXin Li        self._suite_job_id = suite_job_id
953*9c5db199SXin Li        self._job_retry=job_retry
954*9c5db199SXin Li        self._max_retries = max_retries
955*9c5db199SXin Li        # RetryHandler to be initialized in schedule()
956*9c5db199SXin Li        self._retry_handler = None
957*9c5db199SXin Li        self.wait_for_results = wait_for_results
958*9c5db199SXin Li        self._job_keyvals = job_keyvals
959*9c5db199SXin Li        if result_reporter is None:
960*9c5db199SXin Li            self._result_reporter = _EmailReporter(self)
961*9c5db199SXin Li        else:
962*9c5db199SXin Li            self._result_reporter = result_reporter
963*9c5db199SXin Li
964*9c5db199SXin Li        if extra_deps is None:
965*9c5db199SXin Li            extra_deps = []
966*9c5db199SXin Li        extra_deps.append(board)
967*9c5db199SXin Li        if pool:
968*9c5db199SXin Li            extra_deps.append(pool)
969*9c5db199SXin Li        extra_deps.extend(child_dependencies)
970*9c5db199SXin Li        self._dependencies = tuple(extra_deps)
971*9c5db199SXin Li
972*9c5db199SXin Li        self._job_creator = _SuiteChildJobCreator(
973*9c5db199SXin Li            tag=tag,
974*9c5db199SXin Li            builds=builds,
975*9c5db199SXin Li            board=board,
976*9c5db199SXin Li            afe=afe,
977*9c5db199SXin Li            max_runtime_mins=max_runtime_mins,
978*9c5db199SXin Li            timeout_mins=timeout_mins,
979*9c5db199SXin Li            suite_job_id=suite_job_id,
980*9c5db199SXin Li            ignore_deps=ignore_deps,
981*9c5db199SXin Li            extra_deps=extra_deps,
982*9c5db199SXin Li            priority=priority,
983*9c5db199SXin Li            offload_failures_only=offload_failures_only,
984*9c5db199SXin Li            test_source_build=test_source_build,
985*9c5db199SXin Li            job_keyvals=job_keyvals,
986*9c5db199SXin Li        )
987*9c5db199SXin Li
988*9c5db199SXin Li
989*9c5db199SXin Li    def _schedule_test(self, record, test, retry_for=None):
990*9c5db199SXin Li        """Schedule a single test and return the job.
991*9c5db199SXin Li
992*9c5db199SXin Li        Schedule a single test by creating a job, and then update relevant
993*9c5db199SXin Li        data structures that are used to keep track of all running jobs.
994*9c5db199SXin Li
995*9c5db199SXin Li        Emits a TEST_NA status log entry if it failed to schedule the test due
996*9c5db199SXin Li        to NoEligibleHostException or a non-existent board label.
997*9c5db199SXin Li
998*9c5db199SXin Li        Returns a frontend.Job object if the test is successfully scheduled.
999*9c5db199SXin Li        If scheduling failed due to NoEligibleHostException or a non-existent
1000*9c5db199SXin Li        board label, returns None.
1001*9c5db199SXin Li
1002*9c5db199SXin Li        @param record: A callable to use for logging.
1003*9c5db199SXin Li                       prototype: record(base_job.status_log_entry)
1004*9c5db199SXin Li        @param test: ControlData for a test to run.
1005*9c5db199SXin Li        @param retry_for: If we are scheduling a test to retry an
1006*9c5db199SXin Li                          old job, the afe_job_id of the old job
1007*9c5db199SXin Li                          will be passed in as |retry_for|.
1008*9c5db199SXin Li
1009*9c5db199SXin Li        @returns: A frontend.Job object or None
1010*9c5db199SXin Li        """
1011*9c5db199SXin Li        msg = 'Scheduling %s' % test.name
1012*9c5db199SXin Li        if retry_for:
1013*9c5db199SXin Li            msg = msg + ', to retry afe job %d' % retry_for
1014*9c5db199SXin Li        logging.debug(msg)
1015*9c5db199SXin Li        begin_time_str = datetime.datetime.now().strftime(time_utils.TIME_FMT)
1016*9c5db199SXin Li        try:
1017*9c5db199SXin Li            job = self._job_creator.create_job(test, retry_for=retry_for)
1018*9c5db199SXin Li        except (error.NoEligibleHostException, proxy.ValidationError) as e:
1019*9c5db199SXin Li            if (isinstance(e, error.NoEligibleHostException)
1020*9c5db199SXin Li                or (isinstance(e, proxy.ValidationError)
1021*9c5db199SXin Li                    and _is_nonexistent_board_error(e))):
1022*9c5db199SXin Li                # Treat a dependency on a non-existent board label the same as
1023*9c5db199SXin Li                # a dependency on a board that exists, but for which there's no
1024*9c5db199SXin Li                # hardware.
1025*9c5db199SXin Li                logging.debug('%s not applicable for this board/pool. '
1026*9c5db199SXin Li                              'Emitting TEST_NA.', test.name)
1027*9c5db199SXin Li                Status('TEST_NA', test.name,
1028*9c5db199SXin Li                       'Skipping:  test not supported on this board/pool.',
1029*9c5db199SXin Li                       begin_time_str=begin_time_str).record_all(record)
1030*9c5db199SXin Li                return None
1031*9c5db199SXin Li            else:
1032*9c5db199SXin Li                raise e
1033*9c5db199SXin Li        except (error.RPCException, proxy.JSONRPCException):
1034*9c5db199SXin Li            if retry_for:
1035*9c5db199SXin Li                # Mark that we've attempted to retry the old job.
1036*9c5db199SXin Li                logging.debug("RPC exception occurred")
1037*9c5db199SXin Li                self._retry_handler.set_attempted(job_id=retry_for)
1038*9c5db199SXin Li            raise
1039*9c5db199SXin Li        else:
1040*9c5db199SXin Li            self._jobs.append(job)
1041*9c5db199SXin Li            self._jobs_to_tests[job.id] = test
1042*9c5db199SXin Li            if retry_for:
1043*9c5db199SXin Li                # A retry job was just created, record it.
1044*9c5db199SXin Li                self._retry_handler.add_retry(
1045*9c5db199SXin Li                        old_job_id=retry_for, new_job_id=job.id)
1046*9c5db199SXin Li                retry_count = (test.job_retries -
1047*9c5db199SXin Li                               self._retry_handler.get_retry_max(job.id))
1048*9c5db199SXin Li                logging.debug('Job %d created to retry job %d. '
1049*9c5db199SXin Li                              'Have retried for %d time(s)',
1050*9c5db199SXin Li                              job.id, retry_for, retry_count)
1051*9c5db199SXin Li            self._remember_job_keyval(job)
1052*9c5db199SXin Li            return job
1053*9c5db199SXin Li
1054*9c5db199SXin Li    def schedule(self, record):
1055*9c5db199SXin Li        """
1056*9c5db199SXin Li        Schedule jobs using |self._afe|.
1057*9c5db199SXin Li
1058*9c5db199SXin Li        frontend.Job objects representing each scheduled job will be put in
1059*9c5db199SXin Li        |self._jobs|.
1060*9c5db199SXin Li
1061*9c5db199SXin Li        @param record: A callable to use for logging.
1062*9c5db199SXin Li                       prototype: record(base_job.status_log_entry)
1063*9c5db199SXin Li        @returns: The number of tests that were scheduled.
1064*9c5db199SXin Li        """
1065*9c5db199SXin Li        scheduled_test_names = []
1066*9c5db199SXin Li        logging.debug('Discovered %d tests.', len(self.tests))
1067*9c5db199SXin Li
1068*9c5db199SXin Li        Status('INFO', 'Start %s' % self._tag).record_result(record)
1069*9c5db199SXin Li        try:
1070*9c5db199SXin Li            # Write job_keyvals into keyval file.
1071*9c5db199SXin Li            if self._job_keyvals:
1072*9c5db199SXin Li                utils.write_keyval(self._results_dir, self._job_keyvals)
1073*9c5db199SXin Li
1074*9c5db199SXin Li            # TODO(crbug.com/730885): This is a hack to protect tests that are
1075*9c5db199SXin Li            # not usually retried from getting hit by a provision error when run
1076*9c5db199SXin Li            # as part of a suite. Remove this hack once provision is separated
1077*9c5db199SXin Li            # out in its own suite.
1078*9c5db199SXin Li            self._bump_up_test_retries(self.tests)
1079*9c5db199SXin Li            for test in self.tests:
1080*9c5db199SXin Li                scheduled_job = self._schedule_test(record, test)
1081*9c5db199SXin Li                if scheduled_job is not None:
1082*9c5db199SXin Li                    scheduled_test_names.append(test.name)
1083*9c5db199SXin Li
1084*9c5db199SXin Li            # Write the num of scheduled tests and name of them to keyval file.
1085*9c5db199SXin Li            logging.debug('Scheduled %d tests, writing the total to keyval.',
1086*9c5db199SXin Li                          len(scheduled_test_names))
1087*9c5db199SXin Li            utils.write_keyval(
1088*9c5db199SXin Li                self._results_dir,
1089*9c5db199SXin Li                self._make_scheduled_tests_keyvals(scheduled_test_names))
1090*9c5db199SXin Li        except Exception:
1091*9c5db199SXin Li            logging.exception('Exception while scheduling suite')
1092*9c5db199SXin Li            Status('FAIL', self._tag,
1093*9c5db199SXin Li                   'Exception while scheduling suite').record_result(record)
1094*9c5db199SXin Li
1095*9c5db199SXin Li        if self._job_retry:
1096*9c5db199SXin Li            logging.debug("Initializing RetryHandler for suite %s.", self._tag)
1097*9c5db199SXin Li            self._retry_handler = RetryHandler(
1098*9c5db199SXin Li                    initial_jobs_to_tests=self._jobs_to_tests,
1099*9c5db199SXin Li                    max_retries=self._max_retries)
1100*9c5db199SXin Li            logging.debug("retry map created: %s ",
1101*9c5db199SXin Li                          self._retry_handler._retry_map)
1102*9c5db199SXin Li        else:
1103*9c5db199SXin Li            logging.info("Will not retry jobs from suite %s.", self._tag)
1104*9c5db199SXin Li        return len(scheduled_test_names)
1105*9c5db199SXin Li
1106*9c5db199SXin Li
1107*9c5db199SXin Li    def _bump_up_test_retries(self, tests):
1108*9c5db199SXin Li        """Bump up individual test retries to match suite retry options."""
1109*9c5db199SXin Li        if not self._job_retry:
1110*9c5db199SXin Li            return
1111*9c5db199SXin Li
1112*9c5db199SXin Li        for test in tests:
1113*9c5db199SXin Li            # We do honor if a test insists on JOB_RETRIES = 0.
1114*9c5db199SXin Li            if test.job_retries is None:
1115*9c5db199SXin Li                logging.debug(
1116*9c5db199SXin Li                        'Test %s did not request retries, but suite requires '
1117*9c5db199SXin Li                        'retries. Bumping retries up to 1. '
1118*9c5db199SXin Li                        '(See crbug.com/730885)',
1119*9c5db199SXin Li                        test.name)
1120*9c5db199SXin Li                test.job_retries = 1
1121*9c5db199SXin Li
1122*9c5db199SXin Li
1123*9c5db199SXin Li    def _make_scheduled_tests_keyvals(self, scheduled_test_names):
1124*9c5db199SXin Li        """Make a keyvals dict to write for scheduled test names.
1125*9c5db199SXin Li
1126*9c5db199SXin Li        @param scheduled_test_names: A list of scheduled test name strings.
1127*9c5db199SXin Li
1128*9c5db199SXin Li        @returns: A keyvals dict.
1129*9c5db199SXin Li        """
1130*9c5db199SXin Li        return {
1131*9c5db199SXin Li            constants.SCHEDULED_TEST_COUNT_KEY: len(scheduled_test_names),
1132*9c5db199SXin Li            constants.SCHEDULED_TEST_NAMES_KEY: repr(scheduled_test_names),
1133*9c5db199SXin Li        }
1134*9c5db199SXin Li
1135*9c5db199SXin Li
1136*9c5db199SXin Li    def _should_report(self, result):
1137*9c5db199SXin Li        """
1138*9c5db199SXin Li        Returns True if this failure requires to be reported.
1139*9c5db199SXin Li
1140*9c5db199SXin Li        @param result: A result, encapsulating the status of the failed job.
1141*9c5db199SXin Li        @return: True if we should report this failure.
1142*9c5db199SXin Li        """
1143*9c5db199SXin Li        return (self._file_bugs and result.test_executed and
1144*9c5db199SXin Li                not result.is_testna() and
1145*9c5db199SXin Li                result.is_worse_than(job_status.Status('GOOD', '', 'reason')))
1146*9c5db199SXin Li
1147*9c5db199SXin Li
1148*9c5db199SXin Li    def _has_retry(self, result):
1149*9c5db199SXin Li        """
1150*9c5db199SXin Li        Return True if this result gets to retry.
1151*9c5db199SXin Li
1152*9c5db199SXin Li        @param result: A result, encapsulating the status of the failed job.
1153*9c5db199SXin Li        @return: bool
1154*9c5db199SXin Li        """
1155*9c5db199SXin Li        return (self._job_retry
1156*9c5db199SXin Li                and self._retry_handler.has_following_retry(result))
1157*9c5db199SXin Li
1158*9c5db199SXin Li
1159*9c5db199SXin Li    def wait(self, record):
1160*9c5db199SXin Li        """
1161*9c5db199SXin Li        Polls for the job statuses, using |record| to print status when each
1162*9c5db199SXin Li        completes.
1163*9c5db199SXin Li
1164*9c5db199SXin Li        @param record: callable that records job status.
1165*9c5db199SXin Li                 prototype:
1166*9c5db199SXin Li                   record(base_job.status_log_entry)
1167*9c5db199SXin Li        """
1168*9c5db199SXin Li        waiter = job_status.JobResultWaiter(self._afe, self._tko)
1169*9c5db199SXin Li        try:
1170*9c5db199SXin Li            if self._suite_job_id:
1171*9c5db199SXin Li                jobs = self._afe.get_jobs(parent_job_id=self._suite_job_id)
1172*9c5db199SXin Li            else:
1173*9c5db199SXin Li                logging.warning('Unknown suite_job_id, falling back to less '
1174*9c5db199SXin Li                                'efficient results_generator.')
1175*9c5db199SXin Li                jobs = self._jobs
1176*9c5db199SXin Li            waiter.add_jobs(jobs)
1177*9c5db199SXin Li            for result in waiter.wait_for_results():
1178*9c5db199SXin Li                self._handle_result(result=result, record=record, waiter=waiter)
1179*9c5db199SXin Li                if self._finished_waiting():
1180*9c5db199SXin Li                    break
1181*9c5db199SXin Li        except Exception:  # pylint: disable=W0703
1182*9c5db199SXin Li            logging.exception('Exception waiting for results')
1183*9c5db199SXin Li            Status('FAIL', self._tag,
1184*9c5db199SXin Li                   'Exception waiting for results').record_result(record)
1185*9c5db199SXin Li
1186*9c5db199SXin Li
1187*9c5db199SXin Li    def _finished_waiting(self):
1188*9c5db199SXin Li        """Return whether the suite is finished waiting for child jobs."""
1189*9c5db199SXin Li        return False
1190*9c5db199SXin Li
1191*9c5db199SXin Li
1192*9c5db199SXin Li    def _handle_result(self, result, record, waiter):
1193*9c5db199SXin Li        """
1194*9c5db199SXin Li        Handle a test job result.
1195*9c5db199SXin Li
1196*9c5db199SXin Li        @param result: Status instance for job.
1197*9c5db199SXin Li        @param record: callable that records job status.
1198*9c5db199SXin Li                 prototype:
1199*9c5db199SXin Li                   record(base_job.status_log_entry)
1200*9c5db199SXin Li        @param waiter: JobResultsWaiter instance.
1201*9c5db199SXin Li
1202*9c5db199SXin Li        @instance_param _result_reporter: _ResultReporter instance.
1203*9c5db199SXin Li        """
1204*9c5db199SXin Li        self._record_result(result, record)
1205*9c5db199SXin Li        rescheduled = False
1206*9c5db199SXin Li        if self._job_retry and self._retry_handler._should_retry(result):
1207*9c5db199SXin Li            rescheduled = self._retry_result(result, record, waiter)
1208*9c5db199SXin Li        # TODO (crbug.com/751428): If the suite times out before a retry could
1209*9c5db199SXin Li        # finish, we would lose the chance to report errors from the original
1210*9c5db199SXin Li        # job.
1211*9c5db199SXin Li        if self._has_retry(result) and rescheduled:
1212*9c5db199SXin Li            return
1213*9c5db199SXin Li
1214*9c5db199SXin Li        if self._should_report(result):
1215*9c5db199SXin Li            self._result_reporter.report(result)
1216*9c5db199SXin Li
1217*9c5db199SXin Li    def _record_result(self, result, record):
1218*9c5db199SXin Li        """
1219*9c5db199SXin Li        Record a test job result.
1220*9c5db199SXin Li
1221*9c5db199SXin Li        @param result: Status instance for job.
1222*9c5db199SXin Li        @param record: callable that records job status.
1223*9c5db199SXin Li                 prototype:
1224*9c5db199SXin Li                   record(base_job.status_log_entry)
1225*9c5db199SXin Li        """
1226*9c5db199SXin Li        result.record_all(record)
1227*9c5db199SXin Li        self._remember_job_keyval(result)
1228*9c5db199SXin Li
1229*9c5db199SXin Li
1230*9c5db199SXin Li    def _retry_result(self, result, record, waiter):
1231*9c5db199SXin Li        """
1232*9c5db199SXin Li        Retry a test job result.
1233*9c5db199SXin Li
1234*9c5db199SXin Li        @param result: Status instance for job.
1235*9c5db199SXin Li        @param record: callable that records job status.
1236*9c5db199SXin Li                 prototype:
1237*9c5db199SXin Li                   record(base_job.status_log_entry)
1238*9c5db199SXin Li        @param waiter: JobResultsWaiter instance.
1239*9c5db199SXin Li        @returns: True if a job was scheduled for retry, False otherwise.
1240*9c5db199SXin Li        """
1241*9c5db199SXin Li        test = self._jobs_to_tests[result.id]
1242*9c5db199SXin Li        try:
1243*9c5db199SXin Li            # It only takes effect for CQ retriable job:
1244*9c5db199SXin Li            #   1) in first try, test.fast=True.
1245*9c5db199SXin Li            #   2) in second try, test will be run in normal mode, so reset
1246*9c5db199SXin Li            #       test.fast=False.
1247*9c5db199SXin Li            test.fast = False
1248*9c5db199SXin Li            new_job = self._schedule_test(
1249*9c5db199SXin Li                    record=record, test=test, retry_for=result.id)
1250*9c5db199SXin Li        except (error.RPCException, proxy.JSONRPCException) as e:
1251*9c5db199SXin Li            logging.error('Failed to schedule test: %s, Reason: %s',
1252*9c5db199SXin Li                          test.name, e)
1253*9c5db199SXin Li            return False
1254*9c5db199SXin Li        else:
1255*9c5db199SXin Li            waiter.add_job(new_job)
1256*9c5db199SXin Li            return bool(new_job)
1257*9c5db199SXin Li
1258*9c5db199SXin Li    @property
1259*9c5db199SXin Li    def jobs(self):
1260*9c5db199SXin Li        """Give a copy of the associated jobs
1261*9c5db199SXin Li
1262*9c5db199SXin Li        @returns: array of jobs"""
1263*9c5db199SXin Li        return [job for job in self._jobs]
1264*9c5db199SXin Li
1265*9c5db199SXin Li
1266*9c5db199SXin Li    @property
1267*9c5db199SXin Li    def _should_file_bugs(self):
1268*9c5db199SXin Li        """Return whether bugs should be filed.
1269*9c5db199SXin Li
1270*9c5db199SXin Li        @returns: bool
1271*9c5db199SXin Li        """
1272*9c5db199SXin Li        # File bug when failure is one of the _FILE_BUG_SUITES,
1273*9c5db199SXin Li        # otherwise send an email to the owner anc cc.
1274*9c5db199SXin Li        return self._tag in _FILE_BUG_SUITES
1275*9c5db199SXin Li
1276*9c5db199SXin Li
1277*9c5db199SXin Li    def abort(self):
1278*9c5db199SXin Li        """
1279*9c5db199SXin Li        Abort all scheduled test jobs.
1280*9c5db199SXin Li        """
1281*9c5db199SXin Li        if self._jobs:
1282*9c5db199SXin Li            job_ids = [job.id for job in self._jobs]
1283*9c5db199SXin Li            self._afe.run('abort_host_queue_entries', job__id__in=job_ids)
1284*9c5db199SXin Li
1285*9c5db199SXin Li
1286*9c5db199SXin Li    def _remember_job_keyval(self, job):
1287*9c5db199SXin Li        """
1288*9c5db199SXin Li        Record provided job as a suite job keyval, for later referencing.
1289*9c5db199SXin Li
1290*9c5db199SXin Li        @param job: some representation of a job that has the attributes:
1291*9c5db199SXin Li                    id, test_name, and owner
1292*9c5db199SXin Li        """
1293*9c5db199SXin Li        if self._results_dir and job.id and job.owner and job.test_name:
1294*9c5db199SXin Li            job_id_owner = '%s-%s' % (job.id, job.owner)
1295*9c5db199SXin Li            logging.debug('Adding job keyval for %s=%s',
1296*9c5db199SXin Li                          job.test_name, job_id_owner)
1297*9c5db199SXin Li            utils.write_keyval(
1298*9c5db199SXin Li                self._results_dir,
1299*9c5db199SXin Li                {hashlib.md5(job.test_name).hexdigest(): job_id_owner})
1300*9c5db199SXin Li
1301*9c5db199SXin Li
1302*9c5db199SXin Liclass Suite(_BaseSuite):
1303*9c5db199SXin Li    """
1304*9c5db199SXin Li    A suite of tests, defined by some predicate over control file variables.
1305*9c5db199SXin Li
1306*9c5db199SXin Li    Given a place to search for control files a predicate to match the desired
1307*9c5db199SXin Li    tests, can gather tests and fire off jobs to run them, and then wait for
1308*9c5db199SXin Li    results.
1309*9c5db199SXin Li
1310*9c5db199SXin Li    @var _predicate: a function that should return True when run over a
1311*9c5db199SXin Li         ControlData representation of a control file that should be in
1312*9c5db199SXin Li         this Suite.
1313*9c5db199SXin Li    @var _tag: a string with which to tag jobs run in this suite.
1314*9c5db199SXin Li    @var _builds: the builds on which we're running this suite.
1315*9c5db199SXin Li    @var _afe: an instance of AFE as defined in server/frontend.py.
1316*9c5db199SXin Li    @var _tko: an instance of TKO as defined in server/frontend.py.
1317*9c5db199SXin Li    @var _jobs: currently scheduled jobs, if any.
1318*9c5db199SXin Li    @var _jobs_to_tests: a dictionary that maps job ids to tests represented
1319*9c5db199SXin Li                         ControlData objects.
1320*9c5db199SXin Li    @var _cf_getter: a control_file_getter.ControlFileGetter
1321*9c5db199SXin Li    @var _retry: a bool value indicating whether jobs should be retried on
1322*9c5db199SXin Li                 failure.
1323*9c5db199SXin Li    @var _retry_handler: a RetryHandler object.
1324*9c5db199SXin Li
1325*9c5db199SXin Li    """
1326*9c5db199SXin Li
1327*9c5db199SXin Li    # TODO(ayatane): These methods are kept on the Suite class for
1328*9c5db199SXin Li    # backward compatibility.
1329*9c5db199SXin Li    find_and_parse_tests = _deprecated_suite_method(find_and_parse_tests)
1330*9c5db199SXin Li    find_possible_tests = _deprecated_suite_method(find_possible_tests)
1331*9c5db199SXin Li    create_fs_getter = _deprecated_suite_method(create_fs_getter)
1332*9c5db199SXin Li    name_in_tag_predicate = _deprecated_suite_method(
1333*9c5db199SXin Li            suite_common.name_in_tag_predicate)
1334*9c5db199SXin Li    name_in_tag_similarity_predicate = _deprecated_suite_method(
1335*9c5db199SXin Li            name_in_tag_similarity_predicate)
1336*9c5db199SXin Li    test_name_equals_predicate = _deprecated_suite_method(
1337*9c5db199SXin Li            test_name_equals_predicate)
1338*9c5db199SXin Li    test_name_in_list_predicate = _deprecated_suite_method(
1339*9c5db199SXin Li            suite_common.test_name_in_list_predicate)
1340*9c5db199SXin Li    test_name_matches_pattern_predicate = _deprecated_suite_method(
1341*9c5db199SXin Li            test_name_matches_pattern_predicate)
1342*9c5db199SXin Li    test_file_matches_pattern_predicate = _deprecated_suite_method(
1343*9c5db199SXin Li            test_file_matches_pattern_predicate)
1344*9c5db199SXin Li    matches_attribute_expression_predicate = _deprecated_suite_method(
1345*9c5db199SXin Li            matches_attribute_expression_predicate)
1346*9c5db199SXin Li    test_name_similarity_predicate = _deprecated_suite_method(
1347*9c5db199SXin Li            test_name_similarity_predicate)
1348*9c5db199SXin Li    test_file_similarity_predicate = _deprecated_suite_method(
1349*9c5db199SXin Li            test_file_similarity_predicate)
1350*9c5db199SXin Li    list_all_suites = _deprecated_suite_method(list_all_suites)
1351*9c5db199SXin Li    get_test_source_build = _deprecated_suite_method(
1352*9c5db199SXin Li            suite_common.get_test_source_build)
1353*9c5db199SXin Li
1354*9c5db199SXin Li
1355*9c5db199SXin Li    @classmethod
1356*9c5db199SXin Li    def create_from_predicates(cls, predicates, builds, board, devserver,
1357*9c5db199SXin Li                               cf_getter=None, name='ad_hoc_suite',
1358*9c5db199SXin Li                               run_prod_code=False, **dargs):
1359*9c5db199SXin Li        """
1360*9c5db199SXin Li        Create a Suite using a given predicate test filters.
1361*9c5db199SXin Li
1362*9c5db199SXin Li        Uses supplied predicate(s) to instantiate a Suite. Looks for tests in
1363*9c5db199SXin Li        |autotest_dir| and will schedule them using |afe|.  Pulls control files
1364*9c5db199SXin Li        from the default dev server. Results will be pulled from |tko| upon
1365*9c5db199SXin Li        completion.
1366*9c5db199SXin Li
1367*9c5db199SXin Li        @param predicates: A list of callables that accept ControlData
1368*9c5db199SXin Li                           representations of control files. A test will be
1369*9c5db199SXin Li                           included in suite if all callables in this list
1370*9c5db199SXin Li                           return True on the given control file.
1371*9c5db199SXin Li        @param builds: the builds on which we're running this suite. It's a
1372*9c5db199SXin Li                       dictionary of version_prefix:build.
1373*9c5db199SXin Li        @param board: the board on which we're running this suite.
1374*9c5db199SXin Li        @param devserver: the devserver which contains the build.
1375*9c5db199SXin Li        @param cf_getter: control_file_getter.ControlFileGetter. Defaults to
1376*9c5db199SXin Li                          using DevServerGetter.
1377*9c5db199SXin Li        @param name: name of suite. Defaults to 'ad_hoc_suite'
1378*9c5db199SXin Li        @param run_prod_code: If true, the suite will run the tests that
1379*9c5db199SXin Li                              lives in prod aka the test code currently on the
1380*9c5db199SXin Li                              lab servers.
1381*9c5db199SXin Li        @param **dargs: Any other Suite constructor parameters, as described
1382*9c5db199SXin Li                        in Suite.__init__ docstring.
1383*9c5db199SXin Li        @return a Suite instance.
1384*9c5db199SXin Li        """
1385*9c5db199SXin Li        if cf_getter is None:
1386*9c5db199SXin Li            if run_prod_code:
1387*9c5db199SXin Li                cf_getter = create_fs_getter(_AUTOTEST_DIR)
1388*9c5db199SXin Li            else:
1389*9c5db199SXin Li                build = suite_common.get_test_source_build(builds, **dargs)
1390*9c5db199SXin Li                cf_getter = _create_ds_getter(build, devserver)
1391*9c5db199SXin Li
1392*9c5db199SXin Li        return cls(predicates,
1393*9c5db199SXin Li                   name, builds, board, cf_getter, run_prod_code, **dargs)
1394*9c5db199SXin Li
1395*9c5db199SXin Li
1396*9c5db199SXin Li    @classmethod
1397*9c5db199SXin Li    def create_from_name(cls, name, builds, board, devserver, cf_getter=None,
1398*9c5db199SXin Li                         **dargs):
1399*9c5db199SXin Li        """
1400*9c5db199SXin Li        Create a Suite using a predicate based on the SUITE control file var.
1401*9c5db199SXin Li
1402*9c5db199SXin Li        Makes a predicate based on |name| and uses it to instantiate a Suite
1403*9c5db199SXin Li        that looks for tests in |autotest_dir| and will schedule them using
1404*9c5db199SXin Li        |afe|.  Pulls control files from the default dev server.
1405*9c5db199SXin Li        Results will be pulled from |tko| upon completion.
1406*9c5db199SXin Li
1407*9c5db199SXin Li        @param name: a value of the SUITE control file variable to search for.
1408*9c5db199SXin Li        @param builds: the builds on which we're running this suite. It's a
1409*9c5db199SXin Li                       dictionary of version_prefix:build.
1410*9c5db199SXin Li        @param board: the board on which we're running this suite.
1411*9c5db199SXin Li        @param devserver: the devserver which contains the build.
1412*9c5db199SXin Li        @param cf_getter: control_file_getter.ControlFileGetter. Defaults to
1413*9c5db199SXin Li                          using DevServerGetter.
1414*9c5db199SXin Li        @param **dargs: Any other Suite constructor parameters, as described
1415*9c5db199SXin Li                        in Suite.__init__ docstring.
1416*9c5db199SXin Li        @return a Suite instance.
1417*9c5db199SXin Li        """
1418*9c5db199SXin Li        if cf_getter is None:
1419*9c5db199SXin Li            build = suite_common.get_test_source_build(builds, **dargs)
1420*9c5db199SXin Li            cf_getter = _create_ds_getter(build, devserver)
1421*9c5db199SXin Li
1422*9c5db199SXin Li        return cls([suite_common.name_in_tag_predicate(name)],
1423*9c5db199SXin Li                   name, builds, board, cf_getter, **dargs)
1424*9c5db199SXin Li
1425*9c5db199SXin Li
1426*9c5db199SXin Li    def __init__(
1427*9c5db199SXin Li            self,
1428*9c5db199SXin Li            predicates,
1429*9c5db199SXin Li            tag,
1430*9c5db199SXin Li            builds,
1431*9c5db199SXin Li            board,
1432*9c5db199SXin Li            cf_getter,
1433*9c5db199SXin Li            run_prod_code=False,
1434*9c5db199SXin Li            afe=None,
1435*9c5db199SXin Li            tko=None,
1436*9c5db199SXin Li            pool=None,
1437*9c5db199SXin Li            results_dir=None,
1438*9c5db199SXin Li            max_runtime_mins=24*60,
1439*9c5db199SXin Li            timeout_mins=24*60,
1440*9c5db199SXin Li            file_bugs=False,
1441*9c5db199SXin Li            suite_job_id=None,
1442*9c5db199SXin Li            ignore_deps=False,
1443*9c5db199SXin Li            extra_deps=None,
1444*9c5db199SXin Li            priority=priorities.Priority.DEFAULT,
1445*9c5db199SXin Li            forgiving_parser=True,
1446*9c5db199SXin Li            wait_for_results=True,
1447*9c5db199SXin Li            job_retry=False,
1448*9c5db199SXin Li            max_retries=sys.maxsize,
1449*9c5db199SXin Li            offload_failures_only=False,
1450*9c5db199SXin Li            test_source_build=None,
1451*9c5db199SXin Li            job_keyvals=None,
1452*9c5db199SXin Li            test_args=None,
1453*9c5db199SXin Li            child_dependencies=(),
1454*9c5db199SXin Li            result_reporter=None,
1455*9c5db199SXin Li    ):
1456*9c5db199SXin Li        """
1457*9c5db199SXin Li        Constructor
1458*9c5db199SXin Li
1459*9c5db199SXin Li        @param predicates: A list of callables that accept ControlData
1460*9c5db199SXin Li                           representations of control files. A test will be
1461*9c5db199SXin Li                           included in suite if all callables in this list
1462*9c5db199SXin Li                           return True on the given control file.
1463*9c5db199SXin Li        @param tag: a string with which to tag jobs run in this suite.
1464*9c5db199SXin Li        @param builds: the builds on which we're running this suite.
1465*9c5db199SXin Li        @param board: the board on which we're running this suite.
1466*9c5db199SXin Li        @param cf_getter: a control_file_getter.ControlFileGetter
1467*9c5db199SXin Li        @param afe: an instance of AFE as defined in server/frontend.py.
1468*9c5db199SXin Li        @param tko: an instance of TKO as defined in server/frontend.py.
1469*9c5db199SXin Li        @param pool: Specify the pool of machines to use for scheduling
1470*9c5db199SXin Li                purposes.
1471*9c5db199SXin Li        @param run_prod_code: If true, the suite will run the test code that
1472*9c5db199SXin Li                              lives in prod aka the test code currently on the
1473*9c5db199SXin Li                              lab servers.
1474*9c5db199SXin Li        @param results_dir: The directory where the job can write results to.
1475*9c5db199SXin Li                            This must be set if you want job_id of sub-jobs
1476*9c5db199SXin Li                            list in the job keyvals.
1477*9c5db199SXin Li        @param max_runtime_mins: Maximum suite runtime, in minutes.
1478*9c5db199SXin Li        @param timeout: Maximum job lifetime, in hours.
1479*9c5db199SXin Li        @param suite_job_id: Job id that will act as parent id to all sub jobs.
1480*9c5db199SXin Li                             Default: None
1481*9c5db199SXin Li        @param ignore_deps: True if jobs should ignore the DEPENDENCIES
1482*9c5db199SXin Li                            attribute and skip applying of dependency labels.
1483*9c5db199SXin Li                            (Default:False)
1484*9c5db199SXin Li        @param extra_deps: A list of strings which are the extra DEPENDENCIES
1485*9c5db199SXin Li                           to add to each test being scheduled.
1486*9c5db199SXin Li        @param priority: Integer priority level.  Higher is more important.
1487*9c5db199SXin Li        @param wait_for_results: Set to False to run the suite job without
1488*9c5db199SXin Li                                 waiting for test jobs to finish. Default is
1489*9c5db199SXin Li                                 True.
1490*9c5db199SXin Li        @param job_retry: A bool value indicating whether jobs should be retried
1491*9c5db199SXin Li                          on failure. If True, the field 'JOB_RETRIES' in
1492*9c5db199SXin Li                          control files will be respected. If False, do not
1493*9c5db199SXin Li                          retry.
1494*9c5db199SXin Li        @param max_retries: Maximum retry limit at suite level.
1495*9c5db199SXin Li                            Regardless how many times each individual test
1496*9c5db199SXin Li                            has been retried, the total number of retries
1497*9c5db199SXin Li                            happening in the suite can't exceed _max_retries.
1498*9c5db199SXin Li                            Default to sys.maxint.
1499*9c5db199SXin Li        @param offload_failures_only: Only enable gs_offloading for failed
1500*9c5db199SXin Li                                      jobs.
1501*9c5db199SXin Li        @param test_source_build: Build that contains the server-side test code.
1502*9c5db199SXin Li        @param job_keyvals: General job keyvals to be inserted into keyval file,
1503*9c5db199SXin Li                            which will be used by tko/parse later.
1504*9c5db199SXin Li        @param test_args: A dict of args passed all the way to each individual
1505*9c5db199SXin Li                          test that will be actually ran.
1506*9c5db199SXin Li        @param child_dependencies: (optional) list of dependency strings
1507*9c5db199SXin Li                to be added as dependencies to child jobs.
1508*9c5db199SXin Li        @param result_reporter: A _ResultReporter instance to report results. If
1509*9c5db199SXin Li                None, an _EmailReporter will be created.
1510*9c5db199SXin Li        """
1511*9c5db199SXin Li        tests = find_and_parse_tests(
1512*9c5db199SXin Li                cf_getter,
1513*9c5db199SXin Li                _ComposedPredicate(predicates),
1514*9c5db199SXin Li                tag,
1515*9c5db199SXin Li                forgiving_parser=forgiving_parser,
1516*9c5db199SXin Li                run_prod_code=run_prod_code,
1517*9c5db199SXin Li                test_args=test_args,
1518*9c5db199SXin Li        )
1519*9c5db199SXin Li        super(Suite, self).__init__(
1520*9c5db199SXin Li                tests=tests,
1521*9c5db199SXin Li                tag=tag,
1522*9c5db199SXin Li                builds=builds,
1523*9c5db199SXin Li                board=board,
1524*9c5db199SXin Li                afe=afe,
1525*9c5db199SXin Li                tko=tko,
1526*9c5db199SXin Li                pool=pool,
1527*9c5db199SXin Li                results_dir=results_dir,
1528*9c5db199SXin Li                max_runtime_mins=max_runtime_mins,
1529*9c5db199SXin Li                timeout_mins=timeout_mins,
1530*9c5db199SXin Li                file_bugs=file_bugs,
1531*9c5db199SXin Li                suite_job_id=suite_job_id,
1532*9c5db199SXin Li                ignore_deps=ignore_deps,
1533*9c5db199SXin Li                extra_deps=extra_deps,
1534*9c5db199SXin Li                priority=priority,
1535*9c5db199SXin Li                wait_for_results=wait_for_results,
1536*9c5db199SXin Li                job_retry=job_retry,
1537*9c5db199SXin Li                max_retries=max_retries,
1538*9c5db199SXin Li                offload_failures_only=offload_failures_only,
1539*9c5db199SXin Li                test_source_build=test_source_build,
1540*9c5db199SXin Li                job_keyvals=job_keyvals,
1541*9c5db199SXin Li                child_dependencies=child_dependencies,
1542*9c5db199SXin Li                result_reporter=result_reporter,
1543*9c5db199SXin Li        )
1544*9c5db199SXin Li
1545*9c5db199SXin Li
1546*9c5db199SXin Liclass ProvisionSuite(_BaseSuite):
1547*9c5db199SXin Li    """
1548*9c5db199SXin Li    A suite for provisioning DUTs.
1549*9c5db199SXin Li
1550*9c5db199SXin Li    This is done by creating stub_Pass tests.
1551*9c5db199SXin Li    """
1552*9c5db199SXin Li
1553*9c5db199SXin Li
1554*9c5db199SXin Li    def __init__(
1555*9c5db199SXin Li            self,
1556*9c5db199SXin Li            tag,
1557*9c5db199SXin Li            builds,
1558*9c5db199SXin Li            board,
1559*9c5db199SXin Li            devserver,
1560*9c5db199SXin Li            num_required,
1561*9c5db199SXin Li            num_max=float('inf'),
1562*9c5db199SXin Li            cf_getter=None,
1563*9c5db199SXin Li            run_prod_code=False,
1564*9c5db199SXin Li            test_args=None,
1565*9c5db199SXin Li            test_source_build=None,
1566*9c5db199SXin Li            **kwargs):
1567*9c5db199SXin Li        """
1568*9c5db199SXin Li        Constructor
1569*9c5db199SXin Li
1570*9c5db199SXin Li        @param tag: a string with which to tag jobs run in this suite.
1571*9c5db199SXin Li        @param builds: the builds on which we're running this suite.
1572*9c5db199SXin Li        @param board: the board on which we're running this suite.
1573*9c5db199SXin Li        @param devserver: the devserver which contains the build.
1574*9c5db199SXin Li        @param num_required: number of tests that must pass.  This is
1575*9c5db199SXin Li                             capped by the number of tests that are run.
1576*9c5db199SXin Li        @param num_max: max number of tests to make.  By default there
1577*9c5db199SXin Li                        is no cap, a test is created for each eligible host.
1578*9c5db199SXin Li        @param cf_getter: a control_file_getter.ControlFileGetter.
1579*9c5db199SXin Li        @param test_args: A dict of args passed all the way to each individual
1580*9c5db199SXin Li                          test that will be actually ran.
1581*9c5db199SXin Li        @param test_source_build: Build that contains the server-side test code.
1582*9c5db199SXin Li        @param kwargs: Various keyword arguments passed to
1583*9c5db199SXin Li                       _BaseSuite constructor.
1584*9c5db199SXin Li        """
1585*9c5db199SXin Li        super(ProvisionSuite, self).__init__(
1586*9c5db199SXin Li                tests=[],
1587*9c5db199SXin Li                tag=tag,
1588*9c5db199SXin Li                builds=builds,
1589*9c5db199SXin Li                board=board,
1590*9c5db199SXin Li                **kwargs)
1591*9c5db199SXin Li        self._num_successful = 0
1592*9c5db199SXin Li        self._num_required = 0
1593*9c5db199SXin Li        self.tests = []
1594*9c5db199SXin Li
1595*9c5db199SXin Li        static_deps = [dep for dep in self._dependencies
1596*9c5db199SXin Li                       if not provision.Provision.acts_on(dep)]
1597*9c5db199SXin Li        if 'pool:suites' in static_deps:
1598*9c5db199SXin Li            logging.info('Provision suite is disabled on suites pool')
1599*9c5db199SXin Li            return
1600*9c5db199SXin Li        logging.debug('Looking for hosts matching %r', static_deps)
1601*9c5db199SXin Li        hosts = self._afe.get_hosts(
1602*9c5db199SXin Li                invalid=False, multiple_labels=static_deps)
1603*9c5db199SXin Li        logging.debug('Found %d matching hosts for ProvisionSuite', len(hosts))
1604*9c5db199SXin Li        available_hosts = [h for h in hosts if h.is_available()]
1605*9c5db199SXin Li        logging.debug('Found %d available hosts for ProvisionSuite',
1606*9c5db199SXin Li                      len(available_hosts))
1607*9c5db199SXin Li        dummy_test = _load_dummy_test(
1608*9c5db199SXin Li                builds, devserver, cf_getter,
1609*9c5db199SXin Li                run_prod_code, test_args, test_source_build)
1610*9c5db199SXin Li        self.tests = [dummy_test] * min(len(available_hosts), num_max)
1611*9c5db199SXin Li        logging.debug('Made %d tests for ProvisionSuite', len(self.tests))
1612*9c5db199SXin Li        self._num_required = min(num_required, len(self.tests))
1613*9c5db199SXin Li        logging.debug('Expecting %d tests to pass for ProvisionSuite',
1614*9c5db199SXin Li                      self._num_required)
1615*9c5db199SXin Li
1616*9c5db199SXin Li    def _handle_result(self, result, record, waiter):
1617*9c5db199SXin Li        super(ProvisionSuite, self)._handle_result(result, record, waiter)
1618*9c5db199SXin Li        if result.is_good():
1619*9c5db199SXin Li            self._num_successful += 1
1620*9c5db199SXin Li
1621*9c5db199SXin Li    def _finished_waiting(self):
1622*9c5db199SXin Li        return self._num_successful >= self._num_required
1623*9c5db199SXin Li
1624*9c5db199SXin Li
1625*9c5db199SXin Lidef _load_dummy_test(
1626*9c5db199SXin Li        builds,
1627*9c5db199SXin Li        devserver,
1628*9c5db199SXin Li        cf_getter=None,
1629*9c5db199SXin Li        run_prod_code=False,
1630*9c5db199SXin Li        test_args=None,
1631*9c5db199SXin Li        test_source_build=None):
1632*9c5db199SXin Li    """
1633*9c5db199SXin Li    Load and return the dummy pass test.
1634*9c5db199SXin Li
1635*9c5db199SXin Li    @param builds: the builds on which we're running this suite.
1636*9c5db199SXin Li    @param devserver: the devserver which contains the build.
1637*9c5db199SXin Li    @param cf_getter: a control_file_getter.ControlFileGetter.
1638*9c5db199SXin Li    @param test_args: A dict of args passed all the way to each individual
1639*9c5db199SXin Li                      test that will be actually ran.
1640*9c5db199SXin Li    @param test_source_build: Build that contains the server-side test code.
1641*9c5db199SXin Li    """
1642*9c5db199SXin Li    if cf_getter is None:
1643*9c5db199SXin Li        if run_prod_code:
1644*9c5db199SXin Li            cf_getter = create_fs_getter(_AUTOTEST_DIR)
1645*9c5db199SXin Li        else:
1646*9c5db199SXin Li            build = suite_common.get_test_source_build(
1647*9c5db199SXin Li                    builds, test_source_build=test_source_build)
1648*9c5db199SXin Li            devserver.stage_artifacts(image=build,
1649*9c5db199SXin Li                                      artifacts=['control_files'])
1650*9c5db199SXin Li            cf_getter = _create_ds_getter(build, devserver)
1651*9c5db199SXin Li    retriever = _ControlFileRetriever(cf_getter,
1652*9c5db199SXin Li                                      run_prod_code=run_prod_code,
1653*9c5db199SXin Li                                      test_args=test_args)
1654*9c5db199SXin Li    return retriever.retrieve_for_test('stub_Pass')
1655*9c5db199SXin Li
1656*9c5db199SXin Li
1657*9c5db199SXin Liclass _ComposedPredicate(object):
1658*9c5db199SXin Li    """Return the composition of the predicates.
1659*9c5db199SXin Li
1660*9c5db199SXin Li    Predicates are functions that take a test control data object and
1661*9c5db199SXin Li    return True of that test is to be included.  The returned
1662*9c5db199SXin Li    predicate's set is the intersection of all of the input predicates'
1663*9c5db199SXin Li    sets (it returns True if all predicates return True).
1664*9c5db199SXin Li    """
1665*9c5db199SXin Li
1666*9c5db199SXin Li    def __init__(self, predicates):
1667*9c5db199SXin Li        """Initialize instance.
1668*9c5db199SXin Li
1669*9c5db199SXin Li        @param predicates: Iterable of predicates.
1670*9c5db199SXin Li        """
1671*9c5db199SXin Li        self._predicates = list(predicates)
1672*9c5db199SXin Li
1673*9c5db199SXin Li    def __repr__(self):
1674*9c5db199SXin Li        return '{cls}({this._predicates!r})'.format(
1675*9c5db199SXin Li            cls=type(self).__name__,
1676*9c5db199SXin Li            this=self,
1677*9c5db199SXin Li        )
1678*9c5db199SXin Li
1679*9c5db199SXin Li    def __call__(self, control_data_):
1680*9c5db199SXin Li        return all(f(control_data_) for f in self._predicates)
1681*9c5db199SXin Li
1682*9c5db199SXin Li
1683*9c5db199SXin Lidef _is_nonexistent_board_error(e):
1684*9c5db199SXin Li    """Return True if error is caused by nonexistent board label.
1685*9c5db199SXin Li
1686*9c5db199SXin Li    As of this writing, the particular case we want looks like this:
1687*9c5db199SXin Li
1688*9c5db199SXin Li     1) e.problem_keys is a dictionary
1689*9c5db199SXin Li     2) e.problem_keys['meta_hosts'] exists as the only key
1690*9c5db199SXin Li        in the dictionary.
1691*9c5db199SXin Li     3) e.problem_keys['meta_hosts'] matches this pattern:
1692*9c5db199SXin Li        "Label "board:.*" not found"
1693*9c5db199SXin Li
1694*9c5db199SXin Li    We check for conditions 1) and 2) on the
1695*9c5db199SXin Li    theory that they're relatively immutable.
1696*9c5db199SXin Li    We don't check condition 3) because it seems
1697*9c5db199SXin Li    likely to be a maintenance burden, and for the
1698*9c5db199SXin Li    times when we're wrong, being right shouldn't
1699*9c5db199SXin Li    matter enough (we _hope_).
1700*9c5db199SXin Li
1701*9c5db199SXin Li    @param e: proxy.ValidationError instance
1702*9c5db199SXin Li    @returns: boolean
1703*9c5db199SXin Li    """
1704*9c5db199SXin Li    return (isinstance(e.problem_keys, dict)
1705*9c5db199SXin Li            and len(e.problem_keys) == 1
1706*9c5db199SXin Li            and 'meta_hosts' in e.problem_keys)
1707*9c5db199SXin Li
1708*9c5db199SXin Li
1709*9c5db199SXin Liclass _ResultReporter(six.with_metaclass(abc.ABCMeta, object)):
1710*9c5db199SXin Li    """Abstract base class for reporting test results.
1711*9c5db199SXin Li
1712*9c5db199SXin Li    Usually, this is used to report test failures.
1713*9c5db199SXin Li    """
1714*9c5db199SXin Li
1715*9c5db199SXin Li    @abc.abstractmethod
1716*9c5db199SXin Li    def report(self, result):
1717*9c5db199SXin Li        """Report test result.
1718*9c5db199SXin Li
1719*9c5db199SXin Li        @param result: Status instance for job.
1720*9c5db199SXin Li        """
1721*9c5db199SXin Li
1722*9c5db199SXin Li
1723*9c5db199SXin Liclass _EmailReporter(_ResultReporter):
1724*9c5db199SXin Li    """Class that emails based on test failures."""
1725*9c5db199SXin Li
1726*9c5db199SXin Li    def __init__(self, suite, bug_template=None):
1727*9c5db199SXin Li        self._suite = suite
1728*9c5db199SXin Li        self._bug_template = bug_template or {}
1729*9c5db199SXin Li
1730*9c5db199SXin Li    def _get_test_bug(self, result):
1731*9c5db199SXin Li        """Get TestBug for the given result.
1732*9c5db199SXin Li
1733*9c5db199SXin Li        @param result: Status instance for a test job.
1734*9c5db199SXin Li        @returns: TestBug instance.
1735*9c5db199SXin Li        """
1736*9c5db199SXin Li        # reporting modules have dependency on external packages, e.g., httplib2
1737*9c5db199SXin Li        # Such dependency can cause issue to any module tries to import suite.py
1738*9c5db199SXin Li        # without building site-packages first. Since the reporting modules are
1739*9c5db199SXin Li        # only used in this function, move the imports here avoid the
1740*9c5db199SXin Li        # requirement of building site packages to use other functions in this
1741*9c5db199SXin Li        # module.
1742*9c5db199SXin Li        from autotest_lib.server.cros.dynamic_suite import reporting
1743*9c5db199SXin Li
1744*9c5db199SXin Li        job_views = self._suite._tko.run('get_detailed_test_views',
1745*9c5db199SXin Li                                         afe_job_id=result.id)
1746*9c5db199SXin Li        return reporting.TestBug(self._suite._job_creator.cros_build,
1747*9c5db199SXin Li                utils.get_chrome_version(job_views),
1748*9c5db199SXin Li                self._suite._tag,
1749*9c5db199SXin Li                result)
1750*9c5db199SXin Li
1751*9c5db199SXin Li    def _get_bug_template(self, result):
1752*9c5db199SXin Li        """Get BugTemplate for test job.
1753*9c5db199SXin Li
1754*9c5db199SXin Li        @param result: Status instance for job.
1755*9c5db199SXin Li        @param bug_template: A template dictionary specifying the default bug
1756*9c5db199SXin Li                             filing options for failures in this suite.
1757*9c5db199SXin Li        @returns: BugTemplate instance
1758*9c5db199SXin Li        """
1759*9c5db199SXin Li        # reporting modules have dependency on external packages, e.g., httplib2
1760*9c5db199SXin Li        # Such dependency can cause issue to any module tries to import suite.py
1761*9c5db199SXin Li        # without building site-packages first. Since the reporting modules are
1762*9c5db199SXin Li        # only used in this function, move the imports here avoid the
1763*9c5db199SXin Li        # requirement of building site packages to use other functions in this
1764*9c5db199SXin Li        # module.
1765*9c5db199SXin Li        from autotest_lib.server.cros.dynamic_suite import reporting_utils
1766*9c5db199SXin Li
1767*9c5db199SXin Li        # Try to merge with bug template in test control file.
1768*9c5db199SXin Li        template = reporting_utils.BugTemplate(self._bug_template)
1769*9c5db199SXin Li        try:
1770*9c5db199SXin Li            test_data = self._suite._jobs_to_tests[result.id]
1771*9c5db199SXin Li            return template.finalize_bug_template(
1772*9c5db199SXin Li                    test_data.bug_template)
1773*9c5db199SXin Li        except AttributeError:
1774*9c5db199SXin Li            # Test control file does not have bug template defined.
1775*9c5db199SXin Li            return template.bug_template
1776*9c5db199SXin Li        except reporting_utils.InvalidBugTemplateException as e:
1777*9c5db199SXin Li            logging.error('Merging bug templates failed with '
1778*9c5db199SXin Li                          'error: %s An empty bug template will '
1779*9c5db199SXin Li                          'be used.', e)
1780*9c5db199SXin Li            return {}
1781*9c5db199SXin Li
1782*9c5db199SXin Li    def report(self, result):
1783*9c5db199SXin Li        # reporting modules have dependency on external
1784*9c5db199SXin Li        # packages, e.g., httplib2 Such dependency can cause
1785*9c5db199SXin Li        # issue to any module tries to import suite.py without
1786*9c5db199SXin Li        # building site-packages first. Since the reporting
1787*9c5db199SXin Li        # modules are only used in this function, move the
1788*9c5db199SXin Li        # imports here avoid the requirement of building site
1789*9c5db199SXin Li        # packages to use other functions in this module.
1790*9c5db199SXin Li        from autotest_lib.server.cros.dynamic_suite import reporting
1791*9c5db199SXin Li
1792*9c5db199SXin Li        reporting.send_email(
1793*9c5db199SXin Li                self._get_test_bug(result),
1794*9c5db199SXin Li                self._get_bug_template(result))
1795