19c5db199SXin Li#!/usr/bin/python3 -u
29c5db199SXin Li# Copyright 2007-2008 Martin J. Bligh <[email protected]>, Google Inc.
39c5db199SXin Li# Released under the GPL v2
49c5db199SXin Li
59c5db199SXin Li"""
69c5db199SXin LiRun a control file through the server side engine
79c5db199SXin Li"""
89c5db199SXin Li
99c5db199SXin Liimport datetime
109c5db199SXin Liimport contextlib
119c5db199SXin Liimport getpass
129c5db199SXin Liimport logging
139c5db199SXin Liimport os
149c5db199SXin Liimport re
159c5db199SXin Liimport shutil
169c5db199SXin Liimport signal
179c5db199SXin Liimport socket
189c5db199SXin Liimport sys
199c5db199SXin Liimport traceback
209c5db199SXin Liimport time
219c5db199SXin Liimport six
229c5db199SXin Lifrom six.moves import urllib
239c5db199SXin Li
249c5db199SXin Liimport common
259c5db199SXin Lifrom autotest_lib.client.bin.result_tools import utils as result_utils
269c5db199SXin Lifrom autotest_lib.client.bin.result_tools import view as result_view
279c5db199SXin Lifrom autotest_lib.client.common_lib import control_data
289c5db199SXin Lifrom autotest_lib.client.common_lib import autotest_enum
299c5db199SXin Lifrom autotest_lib.client.common_lib import error
309c5db199SXin Lifrom autotest_lib.client.common_lib import global_config
319c5db199SXin Lifrom autotest_lib.client.common_lib import host_queue_entry_states
329c5db199SXin Lifrom autotest_lib.client.common_lib import host_states
339c5db199SXin Lifrom autotest_lib.client.common_lib import seven
349c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import suite
359c5db199SXin Li
369c5db199SXin Litry:
379c5db199SXin Li    from autotest_lib.utils.frozen_chromite.lib import metrics
389c5db199SXin Li    from autotest_lib.utils.frozen_chromite.lib import cloud_trace
399c5db199SXin Liexcept ImportError as e:
409c5db199SXin Li    from autotest_lib.client.common_lib import utils as common_utils
419c5db199SXin Li    metrics = common_utils.metrics_mock
429c5db199SXin Li    import mock
439c5db199SXin Li    cloud_trace = mock.MagicMock()
449c5db199SXin Li
459c5db199SXin Li# Number of seconds to wait before returning if testing mode is enabled
469c5db199SXin LiTESTING_MODE_SLEEP_SECS = 1
479c5db199SXin Li
489c5db199SXin Li
499c5db199SXin Lifrom autotest_lib.server import frontend
509c5db199SXin Lifrom autotest_lib.server import server_logging_config
519c5db199SXin Lifrom autotest_lib.server import server_job, utils, autoserv_parser, autotest
529c5db199SXin Lifrom autotest_lib.server import utils as server_utils
539c5db199SXin Lifrom autotest_lib.server import site_utils
549c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import frontend_wrappers
559c5db199SXin Lifrom autotest_lib.site_utils import job_directories
569c5db199SXin Lifrom autotest_lib.site_utils import lxc
579c5db199SXin Lifrom autotest_lib.site_utils.lxc import utils as lxc_utils
589c5db199SXin Lifrom autotest_lib.client.common_lib import pidfile, logging_manager
599c5db199SXin Li
609c5db199SXin Li
619c5db199SXin Li# Control segment to stage server-side package.
629c5db199SXin LiSTAGE_SERVER_SIDE_PACKAGE_CONTROL_FILE = server_job._control_segment_path(
639c5db199SXin Li        'stage_server_side_package')
649c5db199SXin Li
659c5db199SXin Li# Command line to start servod in a moblab.
669c5db199SXin LiSTART_SERVOD_CMD = 'sudo start servod BOARD=%s PORT=%s'
679c5db199SXin LiSTOP_SERVOD_CMD = 'sudo stop servod'
689c5db199SXin Li
699c5db199SXin Li_AUTOTEST_ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
709c5db199SXin Li_CONTROL_FILE_FROM_CONTROL_NAME = 'control.from_control_name'
719c5db199SXin Li
729c5db199SXin Li_LXC_JOB_FOLDER = 'lxc_job_folder'
739c5db199SXin Li
749c5db199SXin Lidef log_alarm(signum, frame):
759c5db199SXin Li    logging.error("Received SIGALARM. Ignoring and continuing on.")
769c5db199SXin Li    sys.exit(1)
779c5db199SXin Li
789c5db199SXin Li
799c5db199SXin Lidef _get_companions(parser):
809c5db199SXin Li    """Get a list of companion devices from command line arg -ch.
819c5db199SXin Li
829c5db199SXin Li    @param parser: Parser for the command line arguments.
839c5db199SXin Li
849c5db199SXin Li    @return: A list of companion devices from command line arg -ch.
859c5db199SXin Li    """
869c5db199SXin Li    if parser.options.companion_hosts:
879c5db199SXin Li        companions = parser.options.companion_hosts.replace(',', ' ').strip().split()
889c5db199SXin Li    else:
899c5db199SXin Li        companions = []
909c5db199SXin Li
919c5db199SXin Li    if companions:
929c5db199SXin Li        for companion in companions:
939c5db199SXin Li            if not companion or re.search('\s', companion):
949c5db199SXin Li                parser.parser.error("Invalid companion: %s" % str(companion))
959c5db199SXin Li        companions = list(set(companions))
969c5db199SXin Li        companions.sort()
979c5db199SXin Li    return companions
989c5db199SXin Li
999c5db199SXin Li
1009c5db199SXin Lidef _get_dutservers(parser):
1019c5db199SXin Li    """Get a list of DUT server addresses from command line arg --dut_servers.
1029c5db199SXin Li
1039c5db199SXin Li    @param parser: Parser for the command line arguments.
1049c5db199SXin Li
1059c5db199SXin Li    @return: A list of DUT server addresses from command line arg
1069c5db199SXin Li             --dut_servers.
1079c5db199SXin Li    """
1089c5db199SXin Li    if parser.options.dut_servers:
1099c5db199SXin Li        dut_servers = parser.options.dut_servers.replace(
1109c5db199SXin Li            ',', ' ').strip().split()
1119c5db199SXin Li    else:
1129c5db199SXin Li        dut_servers = []
1139c5db199SXin Li
1149c5db199SXin Li    if dut_servers:
1159c5db199SXin Li        for dut_server in dut_servers:
1169c5db199SXin Li            if not dut_server or re.search('\s', dut_server):
1179c5db199SXin Li                parser.parser.error(
1189c5db199SXin Li                    "Invalid DUT Server address: %s" % str(dut_server))
1199c5db199SXin Li        dut_servers = list(set(dut_servers))
1209c5db199SXin Li        dut_servers.sort()
1219c5db199SXin Li    return dut_servers
1229c5db199SXin Li
1239c5db199SXin Li
1249c5db199SXin Lidef _get_machines(parser):
1259c5db199SXin Li    """Get a list of machine names from command line arg -m or a file.
1269c5db199SXin Li
1279c5db199SXin Li    @param parser: Parser for the command line arguments.
1289c5db199SXin Li
1299c5db199SXin Li    @return: A list of machine names from command line arg -m or the
1309c5db199SXin Li             machines file specified in the command line arg -M.
1319c5db199SXin Li    """
1329c5db199SXin Li    if parser.options.machines:
1339c5db199SXin Li        machines = parser.options.machines.replace(',', ' ').strip().split()
1349c5db199SXin Li    else:
1359c5db199SXin Li        machines = []
1369c5db199SXin Li    machines_file = parser.options.machines_file
1379c5db199SXin Li    if machines_file:
1389c5db199SXin Li        machines = []
1399c5db199SXin Li        for m in open(machines_file, 'r').readlines():
1409c5db199SXin Li            # remove comments, spaces
1419c5db199SXin Li            m = re.sub('#.*', '', m).strip()
1429c5db199SXin Li            if m:
1439c5db199SXin Li                machines.append(m)
1449c5db199SXin Li        logging.debug('Read list of machines from file: %s', machines_file)
1459c5db199SXin Li        logging.debug('Machines: %s', ','.join(machines))
1469c5db199SXin Li
1479c5db199SXin Li    if machines:
1489c5db199SXin Li        for machine in machines:
1499c5db199SXin Li            if not machine or re.search('\s', machine):
1509c5db199SXin Li                parser.parser.error("Invalid machine: %s" % str(machine))
1519c5db199SXin Li        machines = list(set(machines))
1529c5db199SXin Li        machines.sort()
1539c5db199SXin Li    return machines
1549c5db199SXin Li
1559c5db199SXin Li
1569c5db199SXin Lidef _stage_ssp(parser, resultsdir):
1579c5db199SXin Li    """Stage server-side package.
1589c5db199SXin Li
1599c5db199SXin Li    This function calls a control segment to stage server-side package based on
1609c5db199SXin Li    the job and autoserv command line option. The detail implementation could
1619c5db199SXin Li    be different for each host type. Currently, only CrosHost has
1629c5db199SXin Li    stage_server_side_package function defined.
1639c5db199SXin Li    The script returns None if no server-side package is available. However,
1649c5db199SXin Li    it may raise exception if it failed for reasons other than artifact (the
1659c5db199SXin Li    server-side package) not found.
1669c5db199SXin Li
1679c5db199SXin Li    @param parser: Command line arguments parser passed in the autoserv process.
1689c5db199SXin Li    @param resultsdir: Folder to store results. This could be different from
1699c5db199SXin Li            parser.options.results: parser.options.results  can be set to None
1709c5db199SXin Li            for results to be stored in a temp folder. resultsdir can be None
1719c5db199SXin Li            for autoserv run requires no logging.
1729c5db199SXin Li
1739c5db199SXin Li    @return: url to the autotest server-side package. None in case of errors.
1749c5db199SXin Li    """
1759c5db199SXin Li    machines_list = _get_machines(parser)
1769c5db199SXin Li    machines_list = server_job.get_machine_dicts(
1779c5db199SXin Li            machine_names=machines_list,
1789c5db199SXin Li            store_dir=os.path.join(resultsdir, parser.options.host_info_subdir),
1799c5db199SXin Li            in_lab=parser.options.lab,
1809c5db199SXin Li            use_shadow_store=not parser.options.local_only_host_info,
1819c5db199SXin Li            host_attributes=parser.options.host_attributes,
1829c5db199SXin Li    )
1839c5db199SXin Li
1849c5db199SXin Li    namespace = {'machines': machines_list,
1859c5db199SXin Li                 'image': parser.options.test_source_build}
1869c5db199SXin Li    script_locals = {}
1879c5db199SXin Li
1889c5db199SXin Li    seven.exec_file(
1899c5db199SXin Li        STAGE_SERVER_SIDE_PACKAGE_CONTROL_FILE,
1909c5db199SXin Li        globals_=namespace,
1919c5db199SXin Li        locals_=script_locals,
1929c5db199SXin Li    )
1939c5db199SXin Li    ssp_url = script_locals['ssp_url']
1949c5db199SXin Li    if not ssp_url:
1959c5db199SXin Li        logging.error('Failed to stage SSP package: %s',
1969c5db199SXin Li                      script_locals['error_msg'])
1979c5db199SXin Li        logging.error('This job will fail later, when attempting to run with'
1989c5db199SXin Li                      ' SSP')
1999c5db199SXin Li    return ssp_url
2009c5db199SXin Li
2019c5db199SXin Li
2029c5db199SXin Lidef _run_with_ssp(job, container_id, job_id, results, parser, ssp_url,
2039c5db199SXin Li                  machines):
2049c5db199SXin Li    """Run the server job with server-side packaging.
2059c5db199SXin Li
2069c5db199SXin Li    @param job: The server job object.
2079c5db199SXin Li    @param container_id: ID of the container to run the test.
2089c5db199SXin Li    @param job_id: ID of the test job.
2099c5db199SXin Li    @param results: Folder to store results. This could be different from
2109c5db199SXin Li                    parser.options.results:
2119c5db199SXin Li                    parser.options.results  can be set to None for results to be
2129c5db199SXin Li                    stored in a temp folder.
2139c5db199SXin Li                    results can be None if the autoserv run requires no logging.
2149c5db199SXin Li    @param parser: Command line parser that contains the options.
2159c5db199SXin Li    @param ssp_url: url of the staged server-side package.
2169c5db199SXin Li    @param machines: A list of machines to run the test.
2179c5db199SXin Li    """
2189c5db199SXin Li    if not ssp_url:
2199c5db199SXin Li        job.record('FAIL', None, None,
2209c5db199SXin Li                   'Failed to stage server-side package')
2219c5db199SXin Li        raise error.AutoservError('Failed to stage server-side package')
2229c5db199SXin Li
2239c5db199SXin Li    bucket = lxc.ContainerBucket(
2249c5db199SXin Li            base_name=_ssp_base_image_name_or_default(parser.options))
2259c5db199SXin Li    control = (parser.args[0] if len(parser.args) > 0 and parser.args[0] != ''
2269c5db199SXin Li               else None)
2279c5db199SXin Li    try:
2289c5db199SXin Li        dut_name = machines[0] if len(machines) >= 1 else None
2299c5db199SXin Li        test_container = bucket.setup_test(container_id, job_id, ssp_url,
2309c5db199SXin Li                                           results, control=control,
2319c5db199SXin Li                                           job_folder=_LXC_JOB_FOLDER,
2329c5db199SXin Li                                           dut_name=dut_name)
2339c5db199SXin Li    except Exception as e:
2349c5db199SXin Li        job.record('START', None, None, 'Starting SSP')
2359c5db199SXin Li        job.record('END ABORT', None, None,
2369c5db199SXin Li                   'Failed to setup container for test: %s. Check logs in '
2379c5db199SXin Li                   'ssp_logs folder for more details.' % e)
2389c5db199SXin Li        raise error.AutoservSSPError
2399c5db199SXin Li
2409c5db199SXin Li    args = sys.argv[:]
2419c5db199SXin Li    args.remove('--require-ssp')
2429c5db199SXin Li    # --parent_job_id is only useful in autoserv running in host, not in
2439c5db199SXin Li    # container. Include this argument will cause test to fail for builds before
2449c5db199SXin Li    # CL 286265 was merged.
2459c5db199SXin Li    if '--parent_job_id' in args:
2469c5db199SXin Li        index = args.index('--parent_job_id')
2479c5db199SXin Li        args.remove('--parent_job_id')
2489c5db199SXin Li        # Remove the actual parent job id in command line arg.
2499c5db199SXin Li        del args[index]
2509c5db199SXin Li
2519c5db199SXin Li    # A dictionary of paths to replace in the command line. Key is the path to
2529c5db199SXin Li    # be replaced with the one in value.
2539c5db199SXin Li    paths_to_replace = {}
2549c5db199SXin Li    # Replace the control file path with the one in container.
2559c5db199SXin Li    if control:
2569c5db199SXin Li        container_control_filename = os.path.join(
2579c5db199SXin Li                lxc.CONTROL_TEMP_PATH, os.path.basename(control))
2589c5db199SXin Li        paths_to_replace[control] = container_control_filename
2599c5db199SXin Li    # Update result directory with the one in container.
2609c5db199SXin Li    container_result_dir = os.path.join(lxc.RESULT_DIR_FMT % _LXC_JOB_FOLDER)
2619c5db199SXin Li    if parser.options.results:
2629c5db199SXin Li        paths_to_replace[parser.options.results] = container_result_dir
2639c5db199SXin Li    args = [paths_to_replace.get(arg, arg) for arg in args]
2649c5db199SXin Li
2659c5db199SXin Li    # Apply --use-existing-results, results directory is aready created and
2669c5db199SXin Li    # mounted in container. Apply this arg to avoid exception being raised.
2679c5db199SXin Li    if not '--use-existing-results' in args:
2689c5db199SXin Li        args.append('--use-existing-results')
2699c5db199SXin Li
2709c5db199SXin Li    # Make sure autoserv running in container using a different pid file.
2719c5db199SXin Li    if not '--pidfile-label' in args:
2729c5db199SXin Li        args.extend(['--pidfile-label', 'container_autoserv'])
2739c5db199SXin Li
2749c5db199SXin Li    cmd_line = ' '.join(["'%s'" % arg if ' ' in arg else arg for arg in args])
2759c5db199SXin Li    logging.info('Run command in container: %s', cmd_line)
2769c5db199SXin Li    success = False
2779c5db199SXin Li    try:
2789c5db199SXin Li        test_container.attach_run(cmd_line)
2799c5db199SXin Li        success = True
2809c5db199SXin Li    except Exception as e:
2819c5db199SXin Li        # If the test run inside container fails without generating any log,
2829c5db199SXin Li        # write a message to status.log to help troubleshooting.
2839c5db199SXin Li        debug_files = os.listdir(os.path.join(results, 'debug'))
2849c5db199SXin Li        if not debug_files:
2859c5db199SXin Li            job.record('FAIL', None, None,
2869c5db199SXin Li                       'Failed to run test inside the container: %s. Check '
2879c5db199SXin Li                       'logs in ssp_logs folder for more details.' % e)
2889c5db199SXin Li        raise
2899c5db199SXin Li    finally:
2909c5db199SXin Li        metrics.Counter(
2919c5db199SXin Li            'chromeos/autotest/experimental/execute_job_in_ssp').increment(
2929c5db199SXin Li                fields={'success': success})
2939c5db199SXin Li        test_container.destroy()
2949c5db199SXin Li
2959c5db199SXin Li
2969c5db199SXin Lidef correct_results_folder_permission(results):
2979c5db199SXin Li    """Make sure the results folder has the right permission settings.
2989c5db199SXin Li
2999c5db199SXin Li    For tests running with server-side packaging, the results folder has the
3009c5db199SXin Li    owner of root. This must be changed to the user running the autoserv
3019c5db199SXin Li    process, so parsing job can access the results folder.
3029c5db199SXin Li    TODO(dshi): crbug.com/459344 Remove this function when test container can be
3039c5db199SXin Li    unprivileged container.
3049c5db199SXin Li
3059c5db199SXin Li    @param results: Path to the results folder.
3069c5db199SXin Li
3079c5db199SXin Li    """
3089c5db199SXin Li    if not results:
3099c5db199SXin Li        return
3109c5db199SXin Li
3119c5db199SXin Li    utils.run('sudo -n chown -R %s "%s"' % (os.getuid(), results))
3129c5db199SXin Li    utils.run('sudo -n chgrp -R %s "%s"' % (os.getgid(), results))
3139c5db199SXin Li
3149c5db199SXin Li
3159c5db199SXin Lidef _start_servod(machine):
3169c5db199SXin Li    """Try to start servod in moblab if it's not already running or running with
3179c5db199SXin Li    different board or port.
3189c5db199SXin Li
3199c5db199SXin Li    @param machine: Name of the dut used for test.
3209c5db199SXin Li    """
3219c5db199SXin Li    if not utils.is_moblab():
3229c5db199SXin Li        return
3239c5db199SXin Li
3249c5db199SXin Li    logging.debug('Trying to start servod.')
3259c5db199SXin Li    try:
3269c5db199SXin Li        afe = frontend.AFE()
3279c5db199SXin Li        board = server_utils.get_board_from_afe(machine, afe)
3289c5db199SXin Li        hosts = afe.get_hosts(hostname=machine)
3299c5db199SXin Li        servo_host = hosts[0].attributes.get('servo_host', None)
3309c5db199SXin Li        servo_port = hosts[0].attributes.get('servo_port', 9999)
3319c5db199SXin Li        if not servo_host in ['localhost', '127.0.0.1']:
3329c5db199SXin Li            logging.warning('Starting servod is aborted. The dut\'s servo_host '
3339c5db199SXin Li                         'attribute is not set to localhost.')
3349c5db199SXin Li            return
3359c5db199SXin Li    except (urllib.error.HTTPError, urllib.error.URLError):
3369c5db199SXin Li        # Ignore error if RPC failed to get board
3379c5db199SXin Li        logging.error('Failed to get board name from AFE. Start servod is '
3389c5db199SXin Li                      'aborted')
3399c5db199SXin Li        return
3409c5db199SXin Li
3419c5db199SXin Li    try:
3429c5db199SXin Li        pid = utils.run('pgrep servod').stdout
3439c5db199SXin Li        cmd_line = utils.run('ps -fp %s' % pid).stdout
3449c5db199SXin Li        if ('--board %s' % board in cmd_line and
3459c5db199SXin Li            '--port %s' % servo_port in cmd_line):
3469c5db199SXin Li            logging.debug('Servod is already running with given board and port.'
3479c5db199SXin Li                          ' There is no need to restart servod.')
3489c5db199SXin Li            return
3499c5db199SXin Li        logging.debug('Servod is running with different board or port. '
3509c5db199SXin Li                      'Stopping existing servod.')
3519c5db199SXin Li        utils.run('sudo stop servod')
3529c5db199SXin Li    except error.CmdError:
3539c5db199SXin Li        # servod is not running.
3549c5db199SXin Li        pass
3559c5db199SXin Li
3569c5db199SXin Li    try:
3579c5db199SXin Li        utils.run(START_SERVOD_CMD % (board, servo_port))
3589c5db199SXin Li        logging.debug('Servod is started')
3599c5db199SXin Li    except error.CmdError as e:
3609c5db199SXin Li        logging.error('Servod failed to be started, error: %s', e)
3619c5db199SXin Li
3629c5db199SXin Li
3639c5db199SXin Lidef _control_path_on_disk(control_name):
3649c5db199SXin Li    """Find the control file corresponding to the given control name, on disk.
3659c5db199SXin Li
3669c5db199SXin Li    @param control_name: NAME attribute of the control file to fetch.
3679c5db199SXin Li    @return: Path to the control file.
3689c5db199SXin Li    """
3699c5db199SXin Li    cf_getter = suite.create_fs_getter(_AUTOTEST_ROOT)
3709c5db199SXin Li    control_name_predicate = suite.test_name_matches_pattern_predicate(
3719c5db199SXin Li            '^%s$' % control_name)
3729c5db199SXin Li    tests = suite.find_and_parse_tests(cf_getter, control_name_predicate)
3739c5db199SXin Li    if not tests:
3749c5db199SXin Li        raise error.AutoservError(
3759c5db199SXin Li                'Failed to find any control files with NAME %s' % control_name)
3769c5db199SXin Li    if len(tests) > 1:
3779c5db199SXin Li        logging.error('Found more than one control file with NAME %s: %s',
3789c5db199SXin Li                      control_name, [t.path for t in tests])
3799c5db199SXin Li        raise error.AutoservError(
3809c5db199SXin Li                'Found more than one control file with NAME %s' % control_name)
3819c5db199SXin Li    return tests[0].path
3829c5db199SXin Li
3839c5db199SXin Li
3849c5db199SXin Lidef _stage_control_file(control_name, results_dir):
3859c5db199SXin Li    """Stage the control file to execute from local autotest checkout.
3869c5db199SXin Li
3879c5db199SXin Li    @param control_name: Name of the control file to stage.
3889c5db199SXin Li    @param results_dir: Results directory to stage the control file into.
3899c5db199SXin Li    @return: Absolute path to the staged control file.
3909c5db199SXin Li    """
3919c5db199SXin Li    control_path = _control_path_on_disk(control_name)
3929c5db199SXin Li    new_control = os.path.join(results_dir, _CONTROL_FILE_FROM_CONTROL_NAME)
3939c5db199SXin Li    shutil.copy2(control_path, new_control)
3949c5db199SXin Li    return new_control
3959c5db199SXin Li
3969c5db199SXin Li
3979c5db199SXin Lidef run_autoserv(pid_file_manager, results, parser, ssp_url, use_ssp):
3989c5db199SXin Li    """Run server job with given options.
3999c5db199SXin Li
4009c5db199SXin Li    @param pid_file_manager: PidFileManager used to monitor the autoserv process
4019c5db199SXin Li    @param results: Folder to store results.
4029c5db199SXin Li    @param parser: Parser for the command line arguments.
4039c5db199SXin Li    @param ssp_url: Url to server-side package.
4049c5db199SXin Li    @param use_ssp: Set to True to run with server-side packaging.
4059c5db199SXin Li    """
4069c5db199SXin Li    # send stdin to /dev/null
4079c5db199SXin Li    dev_null = os.open(os.devnull, os.O_RDONLY)
4089c5db199SXin Li    os.dup2(dev_null, sys.stdin.fileno())
4099c5db199SXin Li    os.close(dev_null)
4109c5db199SXin Li
4119c5db199SXin Li    # Create separate process group if the process is not a process group
4129c5db199SXin Li    # leader. This allows autoserv process to keep running after the caller
4139c5db199SXin Li    # process (drone manager call) exits.
4149c5db199SXin Li    if os.getpid() != os.getpgid(0):
4159c5db199SXin Li        os.setsid()
4169c5db199SXin Li
4179c5db199SXin Li    # Container name is predefined so the container can be destroyed in
4189c5db199SXin Li    # handle_sigterm.
4199c5db199SXin Li    job_or_task_id = job_directories.get_job_id_or_task_id(
4209c5db199SXin Li            parser.options.results)
4219c5db199SXin Li    container_id = lxc.ContainerId(job_or_task_id, time.time(), os.getpid())
4229c5db199SXin Li
4239c5db199SXin Li    # Implement SIGTERM handler
4249c5db199SXin Li    def handle_sigterm(signum, frame):
4259c5db199SXin Li        logging.debug('Received SIGTERM')
4269c5db199SXin Li        if pid_file_manager:
4279c5db199SXin Li            pid_file_manager.close_file(1, signal.SIGTERM)
4289c5db199SXin Li        logging.debug('Finished writing to pid_file. Killing process.')
4299c5db199SXin Li
4309c5db199SXin Li        # Update results folder's file permission. This needs to be done ASAP
4319c5db199SXin Li        # before the parsing process tries to access the log.
4329c5db199SXin Li        if use_ssp and results:
4339c5db199SXin Li            correct_results_folder_permission(results)
4349c5db199SXin Li
4359c5db199SXin Li        # This sleep allows the pending output to be logged before the kill
4369c5db199SXin Li        # signal is sent.
4379c5db199SXin Li        time.sleep(.1)
4389c5db199SXin Li        if use_ssp:
4399c5db199SXin Li            logging.debug('Destroy container %s before aborting the autoserv '
4409c5db199SXin Li                          'process.', container_id)
4419c5db199SXin Li            try:
4429c5db199SXin Li                bucket = lxc.ContainerBucket(
4439c5db199SXin Li                        base_name=_ssp_base_image_name_or_default(
4449c5db199SXin Li                                parser.options))
4459c5db199SXin Li                container = bucket.get_container(container_id)
4469c5db199SXin Li                if container:
4479c5db199SXin Li                    container.destroy()
4489c5db199SXin Li                    logging.debug("Container %s destroyed.", container_id)
4499c5db199SXin Li                else:
4509c5db199SXin Li                    logging.debug('Container %s is not found.', container_id)
4519c5db199SXin Li                    bucket.scrub_container_location(container_id)
4529c5db199SXin Li            except:
4539c5db199SXin Li                # Handle any exception so the autoserv process can be aborted.
4549c5db199SXin Li                logging.exception('Failed to destroy container %s.',
4559c5db199SXin Li                                  container_id)
4569c5db199SXin Li            # Try to correct the result file permission again after the
4579c5db199SXin Li            # container is destroyed, as the container might have created some
4589c5db199SXin Li            # new files in the result folder.
4599c5db199SXin Li            if results:
4609c5db199SXin Li                correct_results_folder_permission(results)
4619c5db199SXin Li
4629c5db199SXin Li        os.killpg(os.getpgrp(), signal.SIGKILL)
4639c5db199SXin Li
4649c5db199SXin Li    # Set signal handler
4659c5db199SXin Li    signal.signal(signal.SIGTERM, handle_sigterm)
4669c5db199SXin Li
4679c5db199SXin Li    # faulthandler is only needed to debug in the Lab and is not avaliable to
4689c5db199SXin Li    # be imported in the chroot as part of VMTest, so Try-Except it.
4699c5db199SXin Li    try:
4709c5db199SXin Li        import faulthandler
4719c5db199SXin Li        faulthandler.register(signal.SIGTERM, all_threads=True, chain=True)
4729c5db199SXin Li        logging.debug('faulthandler registered on SIGTERM.')
4739c5db199SXin Li    except ImportError:
4749c5db199SXin Li        # exc_clear() doesn't exist (nor is needed) in python3
4759c5db199SXin Li        if six.PY2:
4769c5db199SXin Li            sys.exc_clear()
4779c5db199SXin Li
4789c5db199SXin Li    # Ignore SIGTTOU's generated by output from forked children.
4799c5db199SXin Li    signal.signal(signal.SIGTTOU, signal.SIG_IGN)
4809c5db199SXin Li
4819c5db199SXin Li    # If we received a SIGALARM, let's be loud about it.
4829c5db199SXin Li    signal.signal(signal.SIGALRM, log_alarm)
4839c5db199SXin Li
4849c5db199SXin Li    # Server side tests that call shell scripts often depend on $USER being set
4859c5db199SXin Li    # but depending on how you launch your autotest scheduler it may not be set.
4869c5db199SXin Li    os.environ['USER'] = getpass.getuser()
4879c5db199SXin Li
4889c5db199SXin Li    label = parser.options.label
4899c5db199SXin Li    group_name = parser.options.group_name
4909c5db199SXin Li    user = parser.options.user
4919c5db199SXin Li    client = parser.options.client
4929c5db199SXin Li    server = parser.options.server
4939c5db199SXin Li    verify = parser.options.verify
4949c5db199SXin Li    repair = parser.options.repair
4959c5db199SXin Li    cleanup = parser.options.cleanup
4969c5db199SXin Li    provision = parser.options.provision
4979c5db199SXin Li    reset = parser.options.reset
4989c5db199SXin Li    job_labels = parser.options.job_labels
4999c5db199SXin Li    no_tee = parser.options.no_tee
5009c5db199SXin Li    execution_tag = parser.options.execution_tag
5019c5db199SXin Li    ssh_user = parser.options.ssh_user
5029c5db199SXin Li    ssh_port = parser.options.ssh_port
5039c5db199SXin Li    ssh_pass = parser.options.ssh_pass
5049c5db199SXin Li    collect_crashinfo = parser.options.collect_crashinfo
5059c5db199SXin Li    control_filename = parser.options.control_filename
5069c5db199SXin Li    verify_job_repo_url = parser.options.verify_job_repo_url
5079c5db199SXin Li    skip_crash_collection = parser.options.skip_crash_collection
5089c5db199SXin Li    ssh_verbosity = int(parser.options.ssh_verbosity)
5099c5db199SXin Li    ssh_options = parser.options.ssh_options
5109c5db199SXin Li    no_use_packaging = parser.options.no_use_packaging
5119c5db199SXin Li    in_lab = bool(parser.options.lab)
5129c5db199SXin Li    companion_hosts = _get_companions(parser)
5139c5db199SXin Li    dut_servers = _get_dutservers(parser)
5149c5db199SXin Li    is_cft = parser.options.cft
5159c5db199SXin Li    force_full_log_collection = parser.options.force_full_log_collection
5169c5db199SXin Li
5179c5db199SXin Li    # can't be both a client and a server side test
5189c5db199SXin Li    if client and server:
5199c5db199SXin Li        parser.parser.error("Can not specify a test as both server and client!")
5209c5db199SXin Li
5219c5db199SXin Li    if provision and client:
5229c5db199SXin Li        parser.parser.error("Cannot specify provisioning and client!")
5239c5db199SXin Li
5249c5db199SXin Li    is_special_task = (verify or repair or cleanup or collect_crashinfo or
5259c5db199SXin Li                       provision or reset)
5269c5db199SXin Li    use_client_trampoline = False
5279c5db199SXin Li    if parser.options.control_name:
5289c5db199SXin Li        if use_ssp:
5299c5db199SXin Li            # When use_ssp is True, autoserv will be re-executed inside a
5309c5db199SXin Li            # container preserving the --control-name argument. Control file
5319c5db199SXin Li            # will be staged inside the rexecuted autoserv.
5329c5db199SXin Li            control = None
5339c5db199SXin Li        else:
5349c5db199SXin Li            try:
5359c5db199SXin Li                control = _stage_control_file(parser.options.control_name,
5369c5db199SXin Li                                              results)
5379c5db199SXin Li            except error.AutoservError as e:
5389c5db199SXin Li                logging.info("Using client trampoline because of: %s", e)
5399c5db199SXin Li                control = parser.options.control_name
5409c5db199SXin Li                use_client_trampoline = True
5419c5db199SXin Li
5429c5db199SXin Li    elif parser.args:
5439c5db199SXin Li        control = parser.args[0]
5449c5db199SXin Li    else:
5459c5db199SXin Li        if not is_special_task:
5469c5db199SXin Li            parser.parser.error("Missing argument: control file")
5479c5db199SXin Li        control = None
5489c5db199SXin Li
5499c5db199SXin Li    if ssh_verbosity > 0:
5509c5db199SXin Li        # ssh_verbosity is an integer between 0 and 3, inclusive
5519c5db199SXin Li        ssh_verbosity_flag = '-' + 'v' * ssh_verbosity
5529c5db199SXin Li    else:
5539c5db199SXin Li        ssh_verbosity_flag = ''
5549c5db199SXin Li
5559c5db199SXin Li    machines = _get_machines(parser)
5569c5db199SXin Li    if group_name and len(machines) < 2:
5579c5db199SXin Li        parser.parser.error('-G %r may only be supplied with more than one '
5589c5db199SXin Li                            'machine.' % group_name)
5599c5db199SXin Li
5609c5db199SXin Li    logging.debug("Parser.args is %r", parser.args)
5619c5db199SXin Li    try:
5629c5db199SXin Li      logging.debug("Parser.options.args is %r", parser.options.args)
5639c5db199SXin Li    except AttributeError:
5649c5db199SXin Li      logging.debug("No Parser.options.args.")
5659c5db199SXin Li
5669c5db199SXin Li    try:
5679c5db199SXin Li      logging.debug("Parser.options is %r", parser.options)
5689c5db199SXin Li    except AttributeError:
5699c5db199SXin Li      logging.debug("No Parser.options.")
5709c5db199SXin Li    job_kwargs = {
5719c5db199SXin Li            'control': control,
5729c5db199SXin Li            'args': parser.args[1:],
5739c5db199SXin Li            'resultdir': results,
5749c5db199SXin Li            'label': label,
5759c5db199SXin Li            'user': user,
5769c5db199SXin Li            'machines': machines,
5779c5db199SXin Li            'machine_dict_list': server_job.get_machine_dicts(
5789c5db199SXin Li                    machine_names=machines,
5799c5db199SXin Li                    store_dir=os.path.join(results,
5809c5db199SXin Li                                           parser.options.host_info_subdir),
5819c5db199SXin Li                    in_lab=in_lab,
5829c5db199SXin Li                    use_shadow_store=not parser.options.local_only_host_info,
5839c5db199SXin Li                    host_attributes=parser.options.host_attributes,
5849c5db199SXin Li            ),
5859c5db199SXin Li            'client': client,
5869c5db199SXin Li            'ssh_user': ssh_user,
5879c5db199SXin Li            'ssh_port': ssh_port,
5889c5db199SXin Li            'ssh_pass': ssh_pass,
5899c5db199SXin Li            'ssh_verbosity_flag': ssh_verbosity_flag,
5909c5db199SXin Li            'ssh_options': ssh_options,
5919c5db199SXin Li            'group_name': group_name,
5929c5db199SXin Li            'tag': execution_tag,
5939c5db199SXin Li            'disable_sysinfo': parser.options.disable_sysinfo,
5949c5db199SXin Li            'in_lab': in_lab,
5959c5db199SXin Li            'use_client_trampoline': use_client_trampoline,
5969c5db199SXin Li            'sync_offload_dir': parser.options.sync_offload_dir,
5979c5db199SXin Li            'companion_hosts': server_job.get_machine_dicts(
5989c5db199SXin Li                    machine_names=companion_hosts,
5999c5db199SXin Li                    store_dir=os.path.join(results,
6009c5db199SXin Li                                           parser.options.host_info_subdir),
6019c5db199SXin Li                    in_lab=in_lab,
6029c5db199SXin Li                    use_shadow_store=not parser.options.local_only_host_info,
6039c5db199SXin Li                    host_attributes=parser.options.host_attributes),
6049c5db199SXin Li            'dut_servers': dut_servers,
6059c5db199SXin Li            'is_cft': is_cft,
6069c5db199SXin Li            'force_full_log_collection': force_full_log_collection
6079c5db199SXin Li    }
6089c5db199SXin Li    if parser.options.parent_job_id:
6099c5db199SXin Li        job_kwargs['parent_job_id'] = int(parser.options.parent_job_id)
6109c5db199SXin Li    if control_filename:
6119c5db199SXin Li        job_kwargs['control_filename'] = control_filename
6129c5db199SXin Li    if parser.options.image_storage_server:
6139c5db199SXin Li        global_config.global_config.override_config_value(
6149c5db199SXin Li            'CROS', 'image_storage_server',
6159c5db199SXin Li            os.path.join(parser.options.image_storage_server, ''))
6169c5db199SXin Li
6179c5db199SXin Li    job = server_job.server_job(**job_kwargs)
6189c5db199SXin Li
6199c5db199SXin Li    job.logging.start_logging()
6209c5db199SXin Li
6219c5db199SXin Li    # perform checks
6229c5db199SXin Li    job.precheck()
6239c5db199SXin Li
6249c5db199SXin Li    # run the job
6259c5db199SXin Li    exit_code = 0
6269c5db199SXin Li    auto_start_servod = global_config.global_config.get_config_value(
6279c5db199SXin Li            'AUTOSERV', 'auto_start_servod', type=bool, default=False)
6289c5db199SXin Li
6299c5db199SXin Li    if not utils.is_in_container():
6309c5db199SXin Li        # crbug.com/1054522 -- ts_mon setup is broken inside the SSP container
6319c5db199SXin Li        # due to a problem in the installed python packages.
6329c5db199SXin Li        # Trying to clean up an incorrectly initialized ts_mon state adds a 5
6339c5db199SXin Li        # second overhead in process teardown, so avoid setting up ts_mon
6349c5db199SXin Li        # entirely inside the SSP container.
6359c5db199SXin Li        site_utils.SetupTsMonGlobalState('autoserv', indirect=False,
6369c5db199SXin Li                                         short_lived=True)
6379c5db199SXin Li    try:
6389c5db199SXin Li        try:
6399c5db199SXin Li            if repair:
6409c5db199SXin Li                if auto_start_servod and len(machines) == 1:
6419c5db199SXin Li                    _start_servod(machines[0])
6429c5db199SXin Li                job.repair(job_labels)
6439c5db199SXin Li            elif verify:
6449c5db199SXin Li                job.verify(job_labels)
6459c5db199SXin Li            elif provision:
6469c5db199SXin Li                job.provision(job_labels)
6479c5db199SXin Li            elif reset:
6489c5db199SXin Li                job.reset(job_labels)
6499c5db199SXin Li            elif cleanup:
6509c5db199SXin Li                job.cleanup(job_labels)
6519c5db199SXin Li            else:
6529c5db199SXin Li                if auto_start_servod and len(machines) == 1:
6539c5db199SXin Li                    _start_servod(machines[0])
6549c5db199SXin Li                if use_ssp:
6559c5db199SXin Li                    try:
6569c5db199SXin Li                        _run_with_ssp(job, container_id, job_or_task_id,
6579c5db199SXin Li                                        results, parser, ssp_url, machines)
6589c5db199SXin Li                    finally:
6599c5db199SXin Li                        # Update the ownership of files in result folder.
6609c5db199SXin Li                        correct_results_folder_permission(results)
6619c5db199SXin Li                else:
6629c5db199SXin Li                    if collect_crashinfo:
6639c5db199SXin Li                        # Update the ownership of files in result folder. If the
6649c5db199SXin Li                        # job to collect crashinfo was running inside container
6659c5db199SXin Li                        # (SSP) and crashed before correcting folder permission,
6669c5db199SXin Li                        # the result folder might have wrong permission setting.
6679c5db199SXin Li                        try:
6689c5db199SXin Li                            correct_results_folder_permission(results)
6699c5db199SXin Li                        except:
6709c5db199SXin Li                            # Ignore any error as the user may not have root
6719c5db199SXin Li                            # permission to run sudo command.
6729c5db199SXin Li                            pass
6739c5db199SXin Li                    metric_name = ('chromeos/autotest/experimental/'
6749c5db199SXin Li                                   'autoserv_job_run_duration')
6759c5db199SXin Li                    f = {'in_container': utils.is_in_container(),
6769c5db199SXin Li                         'success': False}
6779c5db199SXin Li                    with metrics.SecondsTimer(metric_name, fields=f) as c:
6789c5db199SXin Li                        job.run(verify_job_repo_url=verify_job_repo_url,
6799c5db199SXin Li                                only_collect_crashinfo=collect_crashinfo,
6809c5db199SXin Li                                skip_crash_collection=skip_crash_collection,
6819c5db199SXin Li                                job_labels=job_labels,
6829c5db199SXin Li                                use_packaging=(not no_use_packaging))
6839c5db199SXin Li                        c['success'] = True
6849c5db199SXin Li
6859c5db199SXin Li        finally:
6869c5db199SXin Li            job.close()
6879c5db199SXin Li    except error.AutoservSSPError:
6889c5db199SXin Li        # Due to the complexity of the TKO parsing/stainless connection, this
6899c5db199SXin Li        # must be 0 so that the "abort" is actually reflected on stainless.
6909c5db199SXin Li        exit_code = 0
6919c5db199SXin Li        traceback.print_exc()
6929c5db199SXin Li    except:
6939c5db199SXin Li        exit_code = 1
6949c5db199SXin Li        traceback.print_exc()
6959c5db199SXin Li    finally:
6969c5db199SXin Li        metrics.Flush()
6979c5db199SXin Li
6989c5db199SXin Li    sys.exit(exit_code)
6999c5db199SXin Li
7009c5db199SXin Li
7019c5db199SXin Li# Job breakdown statuses
7029c5db199SXin Li_hs = host_states.Status
7039c5db199SXin Li_qs = host_queue_entry_states.Status
7049c5db199SXin Li_status_list = [
7059c5db199SXin Li        _qs.QUEUED, _qs.RESETTING, _qs.VERIFYING,
7069c5db199SXin Li        _qs.PROVISIONING, _hs.REPAIRING, _qs.CLEANING,
7079c5db199SXin Li        _qs.RUNNING, _qs.GATHERING, _qs.PARSING]
7089c5db199SXin Li_JOB_OVERHEAD_STATUS = autotest_enum.AutotestEnum(*_status_list,
7099c5db199SXin Li                                                  string_values=True)
7109c5db199SXin Li
7119c5db199SXin Li
7129c5db199SXin Lidef get_job_status(options):
7139c5db199SXin Li    """Returns the HQE Status for this run.
7149c5db199SXin Li
7159c5db199SXin Li    @param options: parser options.
7169c5db199SXin Li    """
7179c5db199SXin Li    s = _JOB_OVERHEAD_STATUS
7189c5db199SXin Li    task_mapping = {
7199c5db199SXin Li            'reset': s.RESETTING, 'verify': s.VERIFYING,
7209c5db199SXin Li            'provision': s.PROVISIONING, 'repair': s.REPAIRING,
7219c5db199SXin Li            'cleanup': s.CLEANING, 'collect_crashinfo': s.GATHERING}
7229c5db199SXin Li    match = [task for task in task_mapping if getattr(options, task, False)]
7239c5db199SXin Li    return task_mapping[match[0]] if match else s.RUNNING
7249c5db199SXin Li
7259c5db199SXin Li
7269c5db199SXin Lidef _require_ssp_from_control(control_name):
7279c5db199SXin Li    """Read the value of REQUIRE_SSP from test control file.
7289c5db199SXin Li
7299c5db199SXin Li    This reads the control file from the prod checkout of autotest and uses that
7309c5db199SXin Li    to determine whether to even stage the SSP package on a devserver.
7319c5db199SXin Li
7329c5db199SXin Li    This means:
7339c5db199SXin Li    [1] Any change in REQUIRE_SSP directive in a test requires a prod-push to go
7349c5db199SXin Li    live.
7359c5db199SXin Li    [2] This function may find that the control file does not exist but the SSP
7369c5db199SXin Li    package may contain the test file. This function conservatively returns True
7379c5db199SXin Li    in that case.
7389c5db199SXin Li
7399c5db199SXin Li    This function is called very early in autoserv, before logging is setup.
7409c5db199SXin Li    """
7419c5db199SXin Li    if not control_name:
7429c5db199SXin Li        return True
7439c5db199SXin Li    try:
7449c5db199SXin Li        path = _control_path_on_disk(control_name)
7459c5db199SXin Li    except error.AutoservError as e:
7469c5db199SXin Li        sys.stderr.write("autoserv: Could not determine control file path,"
7479c5db199SXin Li                         " assuming we need SSP: %s\n" % e)
7489c5db199SXin Li        sys.stderr.flush()
7499c5db199SXin Li        return True
7509c5db199SXin Li    if not os.path.isfile(path):
7519c5db199SXin Li        return True
7529c5db199SXin Li    control = control_data.parse_control(path)
7539c5db199SXin Li    # There must be explicit directive in the control file to disable SSP.
7549c5db199SXin Li    if not control or control.require_ssp is None:
7559c5db199SXin Li        return True
7569c5db199SXin Li    return control.require_ssp
7579c5db199SXin Li
7589c5db199SXin Li
7599c5db199SXin Lidef _ssp_base_image_name_or_default(options):
7609c5db199SXin Li    """Extract base image name from autoserv options or the global config."""
7619c5db199SXin Li    if options.ssp_base_image_name:
7629c5db199SXin Li        return options.ssp_base_image_name
7639c5db199SXin Li    return global_config.global_config.get_config_value('AUTOSERV',
7649c5db199SXin Li                                                        'container_base_name')
7659c5db199SXin Li
7669c5db199SXin Li
7679c5db199SXin Lidef main():
7689c5db199SXin Li    start_time = datetime.datetime.now()
7699c5db199SXin Li    parser = autoserv_parser.autoserv_parser
7709c5db199SXin Li    parser.parse_args()
7719c5db199SXin Li
7729c5db199SXin Li    if len(sys.argv) == 1:
7739c5db199SXin Li        parser.parser.print_help()
7749c5db199SXin Li        sys.exit(1)
7759c5db199SXin Li
7769c5db199SXin Li    if parser.options.no_logging:
7779c5db199SXin Li        results = None
7789c5db199SXin Li    else:
7799c5db199SXin Li        results = parser.options.results
7809c5db199SXin Li        if not results:
7819c5db199SXin Li            results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S')
7829c5db199SXin Li        results = os.path.abspath(results)
7839c5db199SXin Li        resultdir_exists = False
7849c5db199SXin Li        for filename in ('control.srv', 'status.log', '.autoserv_execute'):
7859c5db199SXin Li            if os.path.exists(os.path.join(results, filename)):
7869c5db199SXin Li                resultdir_exists = True
7879c5db199SXin Li        if not parser.options.use_existing_results and resultdir_exists:
7889c5db199SXin Li            error = "Error: results directory already exists: %s\n" % results
7899c5db199SXin Li            sys.stderr.write(error)
7909c5db199SXin Li            sys.exit(1)
7919c5db199SXin Li
7929c5db199SXin Li        # Now that we certified that there's no leftover results dir from
7939c5db199SXin Li        # previous jobs, lets create the result dir since the logging system
7949c5db199SXin Li        # needs to create the log file in there.
7959c5db199SXin Li        if not os.path.isdir(results):
7969c5db199SXin Li            os.makedirs(results)
7979c5db199SXin Li
7989c5db199SXin Li    if parser.options.require_ssp:
7999c5db199SXin Li        # This is currently only used for skylab (i.e., when --control-name is
8009c5db199SXin Li        # used).
8019c5db199SXin Li        use_ssp = _require_ssp_from_control(parser.options.control_name)
8029c5db199SXin Li    else:
8039c5db199SXin Li        use_ssp = False
8049c5db199SXin Li
8059c5db199SXin Li
8069c5db199SXin Li    if use_ssp:
8079c5db199SXin Li        log_dir = os.path.join(results, 'ssp_logs') if results else None
8089c5db199SXin Li        if log_dir and not os.path.exists(log_dir):
8099c5db199SXin Li            os.makedirs(log_dir)
8109c5db199SXin Li    else:
8119c5db199SXin Li        log_dir = results
8129c5db199SXin Li
8139c5db199SXin Li    logging_manager.configure_logging(
8149c5db199SXin Li            server_logging_config.ServerLoggingConfig(),
8159c5db199SXin Li            results_dir=log_dir,
8169c5db199SXin Li            use_console=not parser.options.no_tee,
8179c5db199SXin Li            verbose=parser.options.verbose,
8189c5db199SXin Li            no_console_prefix=parser.options.no_console_prefix)
8199c5db199SXin Li
8209c5db199SXin Li    logging.debug('autoserv is running in drone %s.', socket.gethostname())
8219c5db199SXin Li    logging.debug('autoserv environment: %r', os.environ)
8229c5db199SXin Li    logging.debug('autoserv command was: %s', ' '.join(sys.argv))
8239c5db199SXin Li    logging.debug('autoserv parsed options: %s', parser.options)
8249c5db199SXin Li    logging.debug('autoserv python version: %s', sys.version)
8259c5db199SXin Li
8269c5db199SXin Li    if use_ssp:
8279c5db199SXin Li        ssp_url = _stage_ssp(parser, results)
8289c5db199SXin Li    else:
8299c5db199SXin Li        ssp_url = None
8309c5db199SXin Li
8319c5db199SXin Li    if results:
8329c5db199SXin Li        logging.info("Results placed in %s" % results)
8339c5db199SXin Li
8349c5db199SXin Li        # wait until now to perform this check, so it get properly logged
8359c5db199SXin Li        if (parser.options.use_existing_results and not resultdir_exists and
8369c5db199SXin Li            not utils.is_in_container()):
8379c5db199SXin Li            logging.error("No existing results directory found: %s", results)
8389c5db199SXin Li            sys.exit(1)
8399c5db199SXin Li
8409c5db199SXin Li    if parser.options.write_pidfile and results:
8419c5db199SXin Li        pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label,
8429c5db199SXin Li                                                  results)
8439c5db199SXin Li        pid_file_manager.open_file()
8449c5db199SXin Li    else:
8459c5db199SXin Li        pid_file_manager = None
8469c5db199SXin Li
8479c5db199SXin Li    autotest.Autotest.set_install_in_tmpdir(
8489c5db199SXin Li        parser.options.install_in_tmpdir)
8499c5db199SXin Li
8509c5db199SXin Li    exit_code = 0
8519c5db199SXin Li    is_task = (parser.options.verify or parser.options.repair or
8529c5db199SXin Li               parser.options.provision or parser.options.reset or
8539c5db199SXin Li               parser.options.cleanup or parser.options.collect_crashinfo)
8549c5db199SXin Li
8559c5db199SXin Li    trace_labels = {
8569c5db199SXin Li            'job_id': job_directories.get_job_id_or_task_id(
8579c5db199SXin Li                    parser.options.results)
8589c5db199SXin Li    }
8599c5db199SXin Li    trace = cloud_trace.SpanStack(
8609c5db199SXin Li            labels=trace_labels,
8619c5db199SXin Li            global_context=parser.options.cloud_trace_context)
8629c5db199SXin Li    trace.enabled = parser.options.cloud_trace_context_enabled == 'True'
8639c5db199SXin Li    try:
8649c5db199SXin Li        try:
8659c5db199SXin Li            with trace.Span(get_job_status(parser.options)):
8669c5db199SXin Li                run_autoserv(pid_file_manager, results, parser, ssp_url,
8679c5db199SXin Li                             use_ssp)
8689c5db199SXin Li        except SystemExit as e:
8699c5db199SXin Li            exit_code = e.code
8709c5db199SXin Li            if exit_code:
8719c5db199SXin Li                logging.exception('Uncaught SystemExit with code %s', exit_code)
8729c5db199SXin Li        except Exception:
8739c5db199SXin Li            # If we don't know what happened, we'll classify it as
8749c5db199SXin Li            # an 'abort' and return 1.
8759c5db199SXin Li            logging.exception('Uncaught Exception, exit_code = 1.')
8769c5db199SXin Li            exit_code = 1
8779c5db199SXin Li    finally:
8789c5db199SXin Li        if pid_file_manager:
8799c5db199SXin Li            pid_file_manager.close_file(exit_code)
8809c5db199SXin Li    sys.exit(exit_code)
8819c5db199SXin Li
8829c5db199SXin Li
8839c5db199SXin Liif __name__ == '__main__':
8849c5db199SXin Li    main()
885