19c5db199SXin Li#!/usr/bin/python3 -u 29c5db199SXin Li# Copyright 2007-2008 Martin J. Bligh <[email protected]>, Google Inc. 39c5db199SXin Li# Released under the GPL v2 49c5db199SXin Li 59c5db199SXin Li""" 69c5db199SXin LiRun a control file through the server side engine 79c5db199SXin Li""" 89c5db199SXin Li 99c5db199SXin Liimport datetime 109c5db199SXin Liimport contextlib 119c5db199SXin Liimport getpass 129c5db199SXin Liimport logging 139c5db199SXin Liimport os 149c5db199SXin Liimport re 159c5db199SXin Liimport shutil 169c5db199SXin Liimport signal 179c5db199SXin Liimport socket 189c5db199SXin Liimport sys 199c5db199SXin Liimport traceback 209c5db199SXin Liimport time 219c5db199SXin Liimport six 229c5db199SXin Lifrom six.moves import urllib 239c5db199SXin Li 249c5db199SXin Liimport common 259c5db199SXin Lifrom autotest_lib.client.bin.result_tools import utils as result_utils 269c5db199SXin Lifrom autotest_lib.client.bin.result_tools import view as result_view 279c5db199SXin Lifrom autotest_lib.client.common_lib import control_data 289c5db199SXin Lifrom autotest_lib.client.common_lib import autotest_enum 299c5db199SXin Lifrom autotest_lib.client.common_lib import error 309c5db199SXin Lifrom autotest_lib.client.common_lib import global_config 319c5db199SXin Lifrom autotest_lib.client.common_lib import host_queue_entry_states 329c5db199SXin Lifrom autotest_lib.client.common_lib import host_states 339c5db199SXin Lifrom autotest_lib.client.common_lib import seven 349c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import suite 359c5db199SXin Li 369c5db199SXin Litry: 379c5db199SXin Li from autotest_lib.utils.frozen_chromite.lib import metrics 389c5db199SXin Li from autotest_lib.utils.frozen_chromite.lib import cloud_trace 399c5db199SXin Liexcept ImportError as e: 409c5db199SXin Li from autotest_lib.client.common_lib import utils as common_utils 419c5db199SXin Li metrics = common_utils.metrics_mock 429c5db199SXin Li import mock 439c5db199SXin Li cloud_trace = mock.MagicMock() 449c5db199SXin Li 459c5db199SXin Li# Number of seconds to wait before returning if testing mode is enabled 469c5db199SXin LiTESTING_MODE_SLEEP_SECS = 1 479c5db199SXin Li 489c5db199SXin Li 499c5db199SXin Lifrom autotest_lib.server import frontend 509c5db199SXin Lifrom autotest_lib.server import server_logging_config 519c5db199SXin Lifrom autotest_lib.server import server_job, utils, autoserv_parser, autotest 529c5db199SXin Lifrom autotest_lib.server import utils as server_utils 539c5db199SXin Lifrom autotest_lib.server import site_utils 549c5db199SXin Lifrom autotest_lib.server.cros.dynamic_suite import frontend_wrappers 559c5db199SXin Lifrom autotest_lib.site_utils import job_directories 569c5db199SXin Lifrom autotest_lib.site_utils import lxc 579c5db199SXin Lifrom autotest_lib.site_utils.lxc import utils as lxc_utils 589c5db199SXin Lifrom autotest_lib.client.common_lib import pidfile, logging_manager 599c5db199SXin Li 609c5db199SXin Li 619c5db199SXin Li# Control segment to stage server-side package. 629c5db199SXin LiSTAGE_SERVER_SIDE_PACKAGE_CONTROL_FILE = server_job._control_segment_path( 639c5db199SXin Li 'stage_server_side_package') 649c5db199SXin Li 659c5db199SXin Li# Command line to start servod in a moblab. 669c5db199SXin LiSTART_SERVOD_CMD = 'sudo start servod BOARD=%s PORT=%s' 679c5db199SXin LiSTOP_SERVOD_CMD = 'sudo stop servod' 689c5db199SXin Li 699c5db199SXin Li_AUTOTEST_ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) 709c5db199SXin Li_CONTROL_FILE_FROM_CONTROL_NAME = 'control.from_control_name' 719c5db199SXin Li 729c5db199SXin Li_LXC_JOB_FOLDER = 'lxc_job_folder' 739c5db199SXin Li 749c5db199SXin Lidef log_alarm(signum, frame): 759c5db199SXin Li logging.error("Received SIGALARM. Ignoring and continuing on.") 769c5db199SXin Li sys.exit(1) 779c5db199SXin Li 789c5db199SXin Li 799c5db199SXin Lidef _get_companions(parser): 809c5db199SXin Li """Get a list of companion devices from command line arg -ch. 819c5db199SXin Li 829c5db199SXin Li @param parser: Parser for the command line arguments. 839c5db199SXin Li 849c5db199SXin Li @return: A list of companion devices from command line arg -ch. 859c5db199SXin Li """ 869c5db199SXin Li if parser.options.companion_hosts: 879c5db199SXin Li companions = parser.options.companion_hosts.replace(',', ' ').strip().split() 889c5db199SXin Li else: 899c5db199SXin Li companions = [] 909c5db199SXin Li 919c5db199SXin Li if companions: 929c5db199SXin Li for companion in companions: 939c5db199SXin Li if not companion or re.search('\s', companion): 949c5db199SXin Li parser.parser.error("Invalid companion: %s" % str(companion)) 959c5db199SXin Li companions = list(set(companions)) 969c5db199SXin Li companions.sort() 979c5db199SXin Li return companions 989c5db199SXin Li 999c5db199SXin Li 1009c5db199SXin Lidef _get_dutservers(parser): 1019c5db199SXin Li """Get a list of DUT server addresses from command line arg --dut_servers. 1029c5db199SXin Li 1039c5db199SXin Li @param parser: Parser for the command line arguments. 1049c5db199SXin Li 1059c5db199SXin Li @return: A list of DUT server addresses from command line arg 1069c5db199SXin Li --dut_servers. 1079c5db199SXin Li """ 1089c5db199SXin Li if parser.options.dut_servers: 1099c5db199SXin Li dut_servers = parser.options.dut_servers.replace( 1109c5db199SXin Li ',', ' ').strip().split() 1119c5db199SXin Li else: 1129c5db199SXin Li dut_servers = [] 1139c5db199SXin Li 1149c5db199SXin Li if dut_servers: 1159c5db199SXin Li for dut_server in dut_servers: 1169c5db199SXin Li if not dut_server or re.search('\s', dut_server): 1179c5db199SXin Li parser.parser.error( 1189c5db199SXin Li "Invalid DUT Server address: %s" % str(dut_server)) 1199c5db199SXin Li dut_servers = list(set(dut_servers)) 1209c5db199SXin Li dut_servers.sort() 1219c5db199SXin Li return dut_servers 1229c5db199SXin Li 1239c5db199SXin Li 1249c5db199SXin Lidef _get_machines(parser): 1259c5db199SXin Li """Get a list of machine names from command line arg -m or a file. 1269c5db199SXin Li 1279c5db199SXin Li @param parser: Parser for the command line arguments. 1289c5db199SXin Li 1299c5db199SXin Li @return: A list of machine names from command line arg -m or the 1309c5db199SXin Li machines file specified in the command line arg -M. 1319c5db199SXin Li """ 1329c5db199SXin Li if parser.options.machines: 1339c5db199SXin Li machines = parser.options.machines.replace(',', ' ').strip().split() 1349c5db199SXin Li else: 1359c5db199SXin Li machines = [] 1369c5db199SXin Li machines_file = parser.options.machines_file 1379c5db199SXin Li if machines_file: 1389c5db199SXin Li machines = [] 1399c5db199SXin Li for m in open(machines_file, 'r').readlines(): 1409c5db199SXin Li # remove comments, spaces 1419c5db199SXin Li m = re.sub('#.*', '', m).strip() 1429c5db199SXin Li if m: 1439c5db199SXin Li machines.append(m) 1449c5db199SXin Li logging.debug('Read list of machines from file: %s', machines_file) 1459c5db199SXin Li logging.debug('Machines: %s', ','.join(machines)) 1469c5db199SXin Li 1479c5db199SXin Li if machines: 1489c5db199SXin Li for machine in machines: 1499c5db199SXin Li if not machine or re.search('\s', machine): 1509c5db199SXin Li parser.parser.error("Invalid machine: %s" % str(machine)) 1519c5db199SXin Li machines = list(set(machines)) 1529c5db199SXin Li machines.sort() 1539c5db199SXin Li return machines 1549c5db199SXin Li 1559c5db199SXin Li 1569c5db199SXin Lidef _stage_ssp(parser, resultsdir): 1579c5db199SXin Li """Stage server-side package. 1589c5db199SXin Li 1599c5db199SXin Li This function calls a control segment to stage server-side package based on 1609c5db199SXin Li the job and autoserv command line option. The detail implementation could 1619c5db199SXin Li be different for each host type. Currently, only CrosHost has 1629c5db199SXin Li stage_server_side_package function defined. 1639c5db199SXin Li The script returns None if no server-side package is available. However, 1649c5db199SXin Li it may raise exception if it failed for reasons other than artifact (the 1659c5db199SXin Li server-side package) not found. 1669c5db199SXin Li 1679c5db199SXin Li @param parser: Command line arguments parser passed in the autoserv process. 1689c5db199SXin Li @param resultsdir: Folder to store results. This could be different from 1699c5db199SXin Li parser.options.results: parser.options.results can be set to None 1709c5db199SXin Li for results to be stored in a temp folder. resultsdir can be None 1719c5db199SXin Li for autoserv run requires no logging. 1729c5db199SXin Li 1739c5db199SXin Li @return: url to the autotest server-side package. None in case of errors. 1749c5db199SXin Li """ 1759c5db199SXin Li machines_list = _get_machines(parser) 1769c5db199SXin Li machines_list = server_job.get_machine_dicts( 1779c5db199SXin Li machine_names=machines_list, 1789c5db199SXin Li store_dir=os.path.join(resultsdir, parser.options.host_info_subdir), 1799c5db199SXin Li in_lab=parser.options.lab, 1809c5db199SXin Li use_shadow_store=not parser.options.local_only_host_info, 1819c5db199SXin Li host_attributes=parser.options.host_attributes, 1829c5db199SXin Li ) 1839c5db199SXin Li 1849c5db199SXin Li namespace = {'machines': machines_list, 1859c5db199SXin Li 'image': parser.options.test_source_build} 1869c5db199SXin Li script_locals = {} 1879c5db199SXin Li 1889c5db199SXin Li seven.exec_file( 1899c5db199SXin Li STAGE_SERVER_SIDE_PACKAGE_CONTROL_FILE, 1909c5db199SXin Li globals_=namespace, 1919c5db199SXin Li locals_=script_locals, 1929c5db199SXin Li ) 1939c5db199SXin Li ssp_url = script_locals['ssp_url'] 1949c5db199SXin Li if not ssp_url: 1959c5db199SXin Li logging.error('Failed to stage SSP package: %s', 1969c5db199SXin Li script_locals['error_msg']) 1979c5db199SXin Li logging.error('This job will fail later, when attempting to run with' 1989c5db199SXin Li ' SSP') 1999c5db199SXin Li return ssp_url 2009c5db199SXin Li 2019c5db199SXin Li 2029c5db199SXin Lidef _run_with_ssp(job, container_id, job_id, results, parser, ssp_url, 2039c5db199SXin Li machines): 2049c5db199SXin Li """Run the server job with server-side packaging. 2059c5db199SXin Li 2069c5db199SXin Li @param job: The server job object. 2079c5db199SXin Li @param container_id: ID of the container to run the test. 2089c5db199SXin Li @param job_id: ID of the test job. 2099c5db199SXin Li @param results: Folder to store results. This could be different from 2109c5db199SXin Li parser.options.results: 2119c5db199SXin Li parser.options.results can be set to None for results to be 2129c5db199SXin Li stored in a temp folder. 2139c5db199SXin Li results can be None if the autoserv run requires no logging. 2149c5db199SXin Li @param parser: Command line parser that contains the options. 2159c5db199SXin Li @param ssp_url: url of the staged server-side package. 2169c5db199SXin Li @param machines: A list of machines to run the test. 2179c5db199SXin Li """ 2189c5db199SXin Li if not ssp_url: 2199c5db199SXin Li job.record('FAIL', None, None, 2209c5db199SXin Li 'Failed to stage server-side package') 2219c5db199SXin Li raise error.AutoservError('Failed to stage server-side package') 2229c5db199SXin Li 2239c5db199SXin Li bucket = lxc.ContainerBucket( 2249c5db199SXin Li base_name=_ssp_base_image_name_or_default(parser.options)) 2259c5db199SXin Li control = (parser.args[0] if len(parser.args) > 0 and parser.args[0] != '' 2269c5db199SXin Li else None) 2279c5db199SXin Li try: 2289c5db199SXin Li dut_name = machines[0] if len(machines) >= 1 else None 2299c5db199SXin Li test_container = bucket.setup_test(container_id, job_id, ssp_url, 2309c5db199SXin Li results, control=control, 2319c5db199SXin Li job_folder=_LXC_JOB_FOLDER, 2329c5db199SXin Li dut_name=dut_name) 2339c5db199SXin Li except Exception as e: 2349c5db199SXin Li job.record('START', None, None, 'Starting SSP') 2359c5db199SXin Li job.record('END ABORT', None, None, 2369c5db199SXin Li 'Failed to setup container for test: %s. Check logs in ' 2379c5db199SXin Li 'ssp_logs folder for more details.' % e) 2389c5db199SXin Li raise error.AutoservSSPError 2399c5db199SXin Li 2409c5db199SXin Li args = sys.argv[:] 2419c5db199SXin Li args.remove('--require-ssp') 2429c5db199SXin Li # --parent_job_id is only useful in autoserv running in host, not in 2439c5db199SXin Li # container. Include this argument will cause test to fail for builds before 2449c5db199SXin Li # CL 286265 was merged. 2459c5db199SXin Li if '--parent_job_id' in args: 2469c5db199SXin Li index = args.index('--parent_job_id') 2479c5db199SXin Li args.remove('--parent_job_id') 2489c5db199SXin Li # Remove the actual parent job id in command line arg. 2499c5db199SXin Li del args[index] 2509c5db199SXin Li 2519c5db199SXin Li # A dictionary of paths to replace in the command line. Key is the path to 2529c5db199SXin Li # be replaced with the one in value. 2539c5db199SXin Li paths_to_replace = {} 2549c5db199SXin Li # Replace the control file path with the one in container. 2559c5db199SXin Li if control: 2569c5db199SXin Li container_control_filename = os.path.join( 2579c5db199SXin Li lxc.CONTROL_TEMP_PATH, os.path.basename(control)) 2589c5db199SXin Li paths_to_replace[control] = container_control_filename 2599c5db199SXin Li # Update result directory with the one in container. 2609c5db199SXin Li container_result_dir = os.path.join(lxc.RESULT_DIR_FMT % _LXC_JOB_FOLDER) 2619c5db199SXin Li if parser.options.results: 2629c5db199SXin Li paths_to_replace[parser.options.results] = container_result_dir 2639c5db199SXin Li args = [paths_to_replace.get(arg, arg) for arg in args] 2649c5db199SXin Li 2659c5db199SXin Li # Apply --use-existing-results, results directory is aready created and 2669c5db199SXin Li # mounted in container. Apply this arg to avoid exception being raised. 2679c5db199SXin Li if not '--use-existing-results' in args: 2689c5db199SXin Li args.append('--use-existing-results') 2699c5db199SXin Li 2709c5db199SXin Li # Make sure autoserv running in container using a different pid file. 2719c5db199SXin Li if not '--pidfile-label' in args: 2729c5db199SXin Li args.extend(['--pidfile-label', 'container_autoserv']) 2739c5db199SXin Li 2749c5db199SXin Li cmd_line = ' '.join(["'%s'" % arg if ' ' in arg else arg for arg in args]) 2759c5db199SXin Li logging.info('Run command in container: %s', cmd_line) 2769c5db199SXin Li success = False 2779c5db199SXin Li try: 2789c5db199SXin Li test_container.attach_run(cmd_line) 2799c5db199SXin Li success = True 2809c5db199SXin Li except Exception as e: 2819c5db199SXin Li # If the test run inside container fails without generating any log, 2829c5db199SXin Li # write a message to status.log to help troubleshooting. 2839c5db199SXin Li debug_files = os.listdir(os.path.join(results, 'debug')) 2849c5db199SXin Li if not debug_files: 2859c5db199SXin Li job.record('FAIL', None, None, 2869c5db199SXin Li 'Failed to run test inside the container: %s. Check ' 2879c5db199SXin Li 'logs in ssp_logs folder for more details.' % e) 2889c5db199SXin Li raise 2899c5db199SXin Li finally: 2909c5db199SXin Li metrics.Counter( 2919c5db199SXin Li 'chromeos/autotest/experimental/execute_job_in_ssp').increment( 2929c5db199SXin Li fields={'success': success}) 2939c5db199SXin Li test_container.destroy() 2949c5db199SXin Li 2959c5db199SXin Li 2969c5db199SXin Lidef correct_results_folder_permission(results): 2979c5db199SXin Li """Make sure the results folder has the right permission settings. 2989c5db199SXin Li 2999c5db199SXin Li For tests running with server-side packaging, the results folder has the 3009c5db199SXin Li owner of root. This must be changed to the user running the autoserv 3019c5db199SXin Li process, so parsing job can access the results folder. 3029c5db199SXin Li TODO(dshi): crbug.com/459344 Remove this function when test container can be 3039c5db199SXin Li unprivileged container. 3049c5db199SXin Li 3059c5db199SXin Li @param results: Path to the results folder. 3069c5db199SXin Li 3079c5db199SXin Li """ 3089c5db199SXin Li if not results: 3099c5db199SXin Li return 3109c5db199SXin Li 3119c5db199SXin Li utils.run('sudo -n chown -R %s "%s"' % (os.getuid(), results)) 3129c5db199SXin Li utils.run('sudo -n chgrp -R %s "%s"' % (os.getgid(), results)) 3139c5db199SXin Li 3149c5db199SXin Li 3159c5db199SXin Lidef _start_servod(machine): 3169c5db199SXin Li """Try to start servod in moblab if it's not already running or running with 3179c5db199SXin Li different board or port. 3189c5db199SXin Li 3199c5db199SXin Li @param machine: Name of the dut used for test. 3209c5db199SXin Li """ 3219c5db199SXin Li if not utils.is_moblab(): 3229c5db199SXin Li return 3239c5db199SXin Li 3249c5db199SXin Li logging.debug('Trying to start servod.') 3259c5db199SXin Li try: 3269c5db199SXin Li afe = frontend.AFE() 3279c5db199SXin Li board = server_utils.get_board_from_afe(machine, afe) 3289c5db199SXin Li hosts = afe.get_hosts(hostname=machine) 3299c5db199SXin Li servo_host = hosts[0].attributes.get('servo_host', None) 3309c5db199SXin Li servo_port = hosts[0].attributes.get('servo_port', 9999) 3319c5db199SXin Li if not servo_host in ['localhost', '127.0.0.1']: 3329c5db199SXin Li logging.warning('Starting servod is aborted. The dut\'s servo_host ' 3339c5db199SXin Li 'attribute is not set to localhost.') 3349c5db199SXin Li return 3359c5db199SXin Li except (urllib.error.HTTPError, urllib.error.URLError): 3369c5db199SXin Li # Ignore error if RPC failed to get board 3379c5db199SXin Li logging.error('Failed to get board name from AFE. Start servod is ' 3389c5db199SXin Li 'aborted') 3399c5db199SXin Li return 3409c5db199SXin Li 3419c5db199SXin Li try: 3429c5db199SXin Li pid = utils.run('pgrep servod').stdout 3439c5db199SXin Li cmd_line = utils.run('ps -fp %s' % pid).stdout 3449c5db199SXin Li if ('--board %s' % board in cmd_line and 3459c5db199SXin Li '--port %s' % servo_port in cmd_line): 3469c5db199SXin Li logging.debug('Servod is already running with given board and port.' 3479c5db199SXin Li ' There is no need to restart servod.') 3489c5db199SXin Li return 3499c5db199SXin Li logging.debug('Servod is running with different board or port. ' 3509c5db199SXin Li 'Stopping existing servod.') 3519c5db199SXin Li utils.run('sudo stop servod') 3529c5db199SXin Li except error.CmdError: 3539c5db199SXin Li # servod is not running. 3549c5db199SXin Li pass 3559c5db199SXin Li 3569c5db199SXin Li try: 3579c5db199SXin Li utils.run(START_SERVOD_CMD % (board, servo_port)) 3589c5db199SXin Li logging.debug('Servod is started') 3599c5db199SXin Li except error.CmdError as e: 3609c5db199SXin Li logging.error('Servod failed to be started, error: %s', e) 3619c5db199SXin Li 3629c5db199SXin Li 3639c5db199SXin Lidef _control_path_on_disk(control_name): 3649c5db199SXin Li """Find the control file corresponding to the given control name, on disk. 3659c5db199SXin Li 3669c5db199SXin Li @param control_name: NAME attribute of the control file to fetch. 3679c5db199SXin Li @return: Path to the control file. 3689c5db199SXin Li """ 3699c5db199SXin Li cf_getter = suite.create_fs_getter(_AUTOTEST_ROOT) 3709c5db199SXin Li control_name_predicate = suite.test_name_matches_pattern_predicate( 3719c5db199SXin Li '^%s$' % control_name) 3729c5db199SXin Li tests = suite.find_and_parse_tests(cf_getter, control_name_predicate) 3739c5db199SXin Li if not tests: 3749c5db199SXin Li raise error.AutoservError( 3759c5db199SXin Li 'Failed to find any control files with NAME %s' % control_name) 3769c5db199SXin Li if len(tests) > 1: 3779c5db199SXin Li logging.error('Found more than one control file with NAME %s: %s', 3789c5db199SXin Li control_name, [t.path for t in tests]) 3799c5db199SXin Li raise error.AutoservError( 3809c5db199SXin Li 'Found more than one control file with NAME %s' % control_name) 3819c5db199SXin Li return tests[0].path 3829c5db199SXin Li 3839c5db199SXin Li 3849c5db199SXin Lidef _stage_control_file(control_name, results_dir): 3859c5db199SXin Li """Stage the control file to execute from local autotest checkout. 3869c5db199SXin Li 3879c5db199SXin Li @param control_name: Name of the control file to stage. 3889c5db199SXin Li @param results_dir: Results directory to stage the control file into. 3899c5db199SXin Li @return: Absolute path to the staged control file. 3909c5db199SXin Li """ 3919c5db199SXin Li control_path = _control_path_on_disk(control_name) 3929c5db199SXin Li new_control = os.path.join(results_dir, _CONTROL_FILE_FROM_CONTROL_NAME) 3939c5db199SXin Li shutil.copy2(control_path, new_control) 3949c5db199SXin Li return new_control 3959c5db199SXin Li 3969c5db199SXin Li 3979c5db199SXin Lidef run_autoserv(pid_file_manager, results, parser, ssp_url, use_ssp): 3989c5db199SXin Li """Run server job with given options. 3999c5db199SXin Li 4009c5db199SXin Li @param pid_file_manager: PidFileManager used to monitor the autoserv process 4019c5db199SXin Li @param results: Folder to store results. 4029c5db199SXin Li @param parser: Parser for the command line arguments. 4039c5db199SXin Li @param ssp_url: Url to server-side package. 4049c5db199SXin Li @param use_ssp: Set to True to run with server-side packaging. 4059c5db199SXin Li """ 4069c5db199SXin Li # send stdin to /dev/null 4079c5db199SXin Li dev_null = os.open(os.devnull, os.O_RDONLY) 4089c5db199SXin Li os.dup2(dev_null, sys.stdin.fileno()) 4099c5db199SXin Li os.close(dev_null) 4109c5db199SXin Li 4119c5db199SXin Li # Create separate process group if the process is not a process group 4129c5db199SXin Li # leader. This allows autoserv process to keep running after the caller 4139c5db199SXin Li # process (drone manager call) exits. 4149c5db199SXin Li if os.getpid() != os.getpgid(0): 4159c5db199SXin Li os.setsid() 4169c5db199SXin Li 4179c5db199SXin Li # Container name is predefined so the container can be destroyed in 4189c5db199SXin Li # handle_sigterm. 4199c5db199SXin Li job_or_task_id = job_directories.get_job_id_or_task_id( 4209c5db199SXin Li parser.options.results) 4219c5db199SXin Li container_id = lxc.ContainerId(job_or_task_id, time.time(), os.getpid()) 4229c5db199SXin Li 4239c5db199SXin Li # Implement SIGTERM handler 4249c5db199SXin Li def handle_sigterm(signum, frame): 4259c5db199SXin Li logging.debug('Received SIGTERM') 4269c5db199SXin Li if pid_file_manager: 4279c5db199SXin Li pid_file_manager.close_file(1, signal.SIGTERM) 4289c5db199SXin Li logging.debug('Finished writing to pid_file. Killing process.') 4299c5db199SXin Li 4309c5db199SXin Li # Update results folder's file permission. This needs to be done ASAP 4319c5db199SXin Li # before the parsing process tries to access the log. 4329c5db199SXin Li if use_ssp and results: 4339c5db199SXin Li correct_results_folder_permission(results) 4349c5db199SXin Li 4359c5db199SXin Li # This sleep allows the pending output to be logged before the kill 4369c5db199SXin Li # signal is sent. 4379c5db199SXin Li time.sleep(.1) 4389c5db199SXin Li if use_ssp: 4399c5db199SXin Li logging.debug('Destroy container %s before aborting the autoserv ' 4409c5db199SXin Li 'process.', container_id) 4419c5db199SXin Li try: 4429c5db199SXin Li bucket = lxc.ContainerBucket( 4439c5db199SXin Li base_name=_ssp_base_image_name_or_default( 4449c5db199SXin Li parser.options)) 4459c5db199SXin Li container = bucket.get_container(container_id) 4469c5db199SXin Li if container: 4479c5db199SXin Li container.destroy() 4489c5db199SXin Li logging.debug("Container %s destroyed.", container_id) 4499c5db199SXin Li else: 4509c5db199SXin Li logging.debug('Container %s is not found.', container_id) 4519c5db199SXin Li bucket.scrub_container_location(container_id) 4529c5db199SXin Li except: 4539c5db199SXin Li # Handle any exception so the autoserv process can be aborted. 4549c5db199SXin Li logging.exception('Failed to destroy container %s.', 4559c5db199SXin Li container_id) 4569c5db199SXin Li # Try to correct the result file permission again after the 4579c5db199SXin Li # container is destroyed, as the container might have created some 4589c5db199SXin Li # new files in the result folder. 4599c5db199SXin Li if results: 4609c5db199SXin Li correct_results_folder_permission(results) 4619c5db199SXin Li 4629c5db199SXin Li os.killpg(os.getpgrp(), signal.SIGKILL) 4639c5db199SXin Li 4649c5db199SXin Li # Set signal handler 4659c5db199SXin Li signal.signal(signal.SIGTERM, handle_sigterm) 4669c5db199SXin Li 4679c5db199SXin Li # faulthandler is only needed to debug in the Lab and is not avaliable to 4689c5db199SXin Li # be imported in the chroot as part of VMTest, so Try-Except it. 4699c5db199SXin Li try: 4709c5db199SXin Li import faulthandler 4719c5db199SXin Li faulthandler.register(signal.SIGTERM, all_threads=True, chain=True) 4729c5db199SXin Li logging.debug('faulthandler registered on SIGTERM.') 4739c5db199SXin Li except ImportError: 4749c5db199SXin Li # exc_clear() doesn't exist (nor is needed) in python3 4759c5db199SXin Li if six.PY2: 4769c5db199SXin Li sys.exc_clear() 4779c5db199SXin Li 4789c5db199SXin Li # Ignore SIGTTOU's generated by output from forked children. 4799c5db199SXin Li signal.signal(signal.SIGTTOU, signal.SIG_IGN) 4809c5db199SXin Li 4819c5db199SXin Li # If we received a SIGALARM, let's be loud about it. 4829c5db199SXin Li signal.signal(signal.SIGALRM, log_alarm) 4839c5db199SXin Li 4849c5db199SXin Li # Server side tests that call shell scripts often depend on $USER being set 4859c5db199SXin Li # but depending on how you launch your autotest scheduler it may not be set. 4869c5db199SXin Li os.environ['USER'] = getpass.getuser() 4879c5db199SXin Li 4889c5db199SXin Li label = parser.options.label 4899c5db199SXin Li group_name = parser.options.group_name 4909c5db199SXin Li user = parser.options.user 4919c5db199SXin Li client = parser.options.client 4929c5db199SXin Li server = parser.options.server 4939c5db199SXin Li verify = parser.options.verify 4949c5db199SXin Li repair = parser.options.repair 4959c5db199SXin Li cleanup = parser.options.cleanup 4969c5db199SXin Li provision = parser.options.provision 4979c5db199SXin Li reset = parser.options.reset 4989c5db199SXin Li job_labels = parser.options.job_labels 4999c5db199SXin Li no_tee = parser.options.no_tee 5009c5db199SXin Li execution_tag = parser.options.execution_tag 5019c5db199SXin Li ssh_user = parser.options.ssh_user 5029c5db199SXin Li ssh_port = parser.options.ssh_port 5039c5db199SXin Li ssh_pass = parser.options.ssh_pass 5049c5db199SXin Li collect_crashinfo = parser.options.collect_crashinfo 5059c5db199SXin Li control_filename = parser.options.control_filename 5069c5db199SXin Li verify_job_repo_url = parser.options.verify_job_repo_url 5079c5db199SXin Li skip_crash_collection = parser.options.skip_crash_collection 5089c5db199SXin Li ssh_verbosity = int(parser.options.ssh_verbosity) 5099c5db199SXin Li ssh_options = parser.options.ssh_options 5109c5db199SXin Li no_use_packaging = parser.options.no_use_packaging 5119c5db199SXin Li in_lab = bool(parser.options.lab) 5129c5db199SXin Li companion_hosts = _get_companions(parser) 5139c5db199SXin Li dut_servers = _get_dutservers(parser) 5149c5db199SXin Li is_cft = parser.options.cft 5159c5db199SXin Li force_full_log_collection = parser.options.force_full_log_collection 5169c5db199SXin Li 5179c5db199SXin Li # can't be both a client and a server side test 5189c5db199SXin Li if client and server: 5199c5db199SXin Li parser.parser.error("Can not specify a test as both server and client!") 5209c5db199SXin Li 5219c5db199SXin Li if provision and client: 5229c5db199SXin Li parser.parser.error("Cannot specify provisioning and client!") 5239c5db199SXin Li 5249c5db199SXin Li is_special_task = (verify or repair or cleanup or collect_crashinfo or 5259c5db199SXin Li provision or reset) 5269c5db199SXin Li use_client_trampoline = False 5279c5db199SXin Li if parser.options.control_name: 5289c5db199SXin Li if use_ssp: 5299c5db199SXin Li # When use_ssp is True, autoserv will be re-executed inside a 5309c5db199SXin Li # container preserving the --control-name argument. Control file 5319c5db199SXin Li # will be staged inside the rexecuted autoserv. 5329c5db199SXin Li control = None 5339c5db199SXin Li else: 5349c5db199SXin Li try: 5359c5db199SXin Li control = _stage_control_file(parser.options.control_name, 5369c5db199SXin Li results) 5379c5db199SXin Li except error.AutoservError as e: 5389c5db199SXin Li logging.info("Using client trampoline because of: %s", e) 5399c5db199SXin Li control = parser.options.control_name 5409c5db199SXin Li use_client_trampoline = True 5419c5db199SXin Li 5429c5db199SXin Li elif parser.args: 5439c5db199SXin Li control = parser.args[0] 5449c5db199SXin Li else: 5459c5db199SXin Li if not is_special_task: 5469c5db199SXin Li parser.parser.error("Missing argument: control file") 5479c5db199SXin Li control = None 5489c5db199SXin Li 5499c5db199SXin Li if ssh_verbosity > 0: 5509c5db199SXin Li # ssh_verbosity is an integer between 0 and 3, inclusive 5519c5db199SXin Li ssh_verbosity_flag = '-' + 'v' * ssh_verbosity 5529c5db199SXin Li else: 5539c5db199SXin Li ssh_verbosity_flag = '' 5549c5db199SXin Li 5559c5db199SXin Li machines = _get_machines(parser) 5569c5db199SXin Li if group_name and len(machines) < 2: 5579c5db199SXin Li parser.parser.error('-G %r may only be supplied with more than one ' 5589c5db199SXin Li 'machine.' % group_name) 5599c5db199SXin Li 5609c5db199SXin Li logging.debug("Parser.args is %r", parser.args) 5619c5db199SXin Li try: 5629c5db199SXin Li logging.debug("Parser.options.args is %r", parser.options.args) 5639c5db199SXin Li except AttributeError: 5649c5db199SXin Li logging.debug("No Parser.options.args.") 5659c5db199SXin Li 5669c5db199SXin Li try: 5679c5db199SXin Li logging.debug("Parser.options is %r", parser.options) 5689c5db199SXin Li except AttributeError: 5699c5db199SXin Li logging.debug("No Parser.options.") 5709c5db199SXin Li job_kwargs = { 5719c5db199SXin Li 'control': control, 5729c5db199SXin Li 'args': parser.args[1:], 5739c5db199SXin Li 'resultdir': results, 5749c5db199SXin Li 'label': label, 5759c5db199SXin Li 'user': user, 5769c5db199SXin Li 'machines': machines, 5779c5db199SXin Li 'machine_dict_list': server_job.get_machine_dicts( 5789c5db199SXin Li machine_names=machines, 5799c5db199SXin Li store_dir=os.path.join(results, 5809c5db199SXin Li parser.options.host_info_subdir), 5819c5db199SXin Li in_lab=in_lab, 5829c5db199SXin Li use_shadow_store=not parser.options.local_only_host_info, 5839c5db199SXin Li host_attributes=parser.options.host_attributes, 5849c5db199SXin Li ), 5859c5db199SXin Li 'client': client, 5869c5db199SXin Li 'ssh_user': ssh_user, 5879c5db199SXin Li 'ssh_port': ssh_port, 5889c5db199SXin Li 'ssh_pass': ssh_pass, 5899c5db199SXin Li 'ssh_verbosity_flag': ssh_verbosity_flag, 5909c5db199SXin Li 'ssh_options': ssh_options, 5919c5db199SXin Li 'group_name': group_name, 5929c5db199SXin Li 'tag': execution_tag, 5939c5db199SXin Li 'disable_sysinfo': parser.options.disable_sysinfo, 5949c5db199SXin Li 'in_lab': in_lab, 5959c5db199SXin Li 'use_client_trampoline': use_client_trampoline, 5969c5db199SXin Li 'sync_offload_dir': parser.options.sync_offload_dir, 5979c5db199SXin Li 'companion_hosts': server_job.get_machine_dicts( 5989c5db199SXin Li machine_names=companion_hosts, 5999c5db199SXin Li store_dir=os.path.join(results, 6009c5db199SXin Li parser.options.host_info_subdir), 6019c5db199SXin Li in_lab=in_lab, 6029c5db199SXin Li use_shadow_store=not parser.options.local_only_host_info, 6039c5db199SXin Li host_attributes=parser.options.host_attributes), 6049c5db199SXin Li 'dut_servers': dut_servers, 6059c5db199SXin Li 'is_cft': is_cft, 6069c5db199SXin Li 'force_full_log_collection': force_full_log_collection 6079c5db199SXin Li } 6089c5db199SXin Li if parser.options.parent_job_id: 6099c5db199SXin Li job_kwargs['parent_job_id'] = int(parser.options.parent_job_id) 6109c5db199SXin Li if control_filename: 6119c5db199SXin Li job_kwargs['control_filename'] = control_filename 6129c5db199SXin Li if parser.options.image_storage_server: 6139c5db199SXin Li global_config.global_config.override_config_value( 6149c5db199SXin Li 'CROS', 'image_storage_server', 6159c5db199SXin Li os.path.join(parser.options.image_storage_server, '')) 6169c5db199SXin Li 6179c5db199SXin Li job = server_job.server_job(**job_kwargs) 6189c5db199SXin Li 6199c5db199SXin Li job.logging.start_logging() 6209c5db199SXin Li 6219c5db199SXin Li # perform checks 6229c5db199SXin Li job.precheck() 6239c5db199SXin Li 6249c5db199SXin Li # run the job 6259c5db199SXin Li exit_code = 0 6269c5db199SXin Li auto_start_servod = global_config.global_config.get_config_value( 6279c5db199SXin Li 'AUTOSERV', 'auto_start_servod', type=bool, default=False) 6289c5db199SXin Li 6299c5db199SXin Li if not utils.is_in_container(): 6309c5db199SXin Li # crbug.com/1054522 -- ts_mon setup is broken inside the SSP container 6319c5db199SXin Li # due to a problem in the installed python packages. 6329c5db199SXin Li # Trying to clean up an incorrectly initialized ts_mon state adds a 5 6339c5db199SXin Li # second overhead in process teardown, so avoid setting up ts_mon 6349c5db199SXin Li # entirely inside the SSP container. 6359c5db199SXin Li site_utils.SetupTsMonGlobalState('autoserv', indirect=False, 6369c5db199SXin Li short_lived=True) 6379c5db199SXin Li try: 6389c5db199SXin Li try: 6399c5db199SXin Li if repair: 6409c5db199SXin Li if auto_start_servod and len(machines) == 1: 6419c5db199SXin Li _start_servod(machines[0]) 6429c5db199SXin Li job.repair(job_labels) 6439c5db199SXin Li elif verify: 6449c5db199SXin Li job.verify(job_labels) 6459c5db199SXin Li elif provision: 6469c5db199SXin Li job.provision(job_labels) 6479c5db199SXin Li elif reset: 6489c5db199SXin Li job.reset(job_labels) 6499c5db199SXin Li elif cleanup: 6509c5db199SXin Li job.cleanup(job_labels) 6519c5db199SXin Li else: 6529c5db199SXin Li if auto_start_servod and len(machines) == 1: 6539c5db199SXin Li _start_servod(machines[0]) 6549c5db199SXin Li if use_ssp: 6559c5db199SXin Li try: 6569c5db199SXin Li _run_with_ssp(job, container_id, job_or_task_id, 6579c5db199SXin Li results, parser, ssp_url, machines) 6589c5db199SXin Li finally: 6599c5db199SXin Li # Update the ownership of files in result folder. 6609c5db199SXin Li correct_results_folder_permission(results) 6619c5db199SXin Li else: 6629c5db199SXin Li if collect_crashinfo: 6639c5db199SXin Li # Update the ownership of files in result folder. If the 6649c5db199SXin Li # job to collect crashinfo was running inside container 6659c5db199SXin Li # (SSP) and crashed before correcting folder permission, 6669c5db199SXin Li # the result folder might have wrong permission setting. 6679c5db199SXin Li try: 6689c5db199SXin Li correct_results_folder_permission(results) 6699c5db199SXin Li except: 6709c5db199SXin Li # Ignore any error as the user may not have root 6719c5db199SXin Li # permission to run sudo command. 6729c5db199SXin Li pass 6739c5db199SXin Li metric_name = ('chromeos/autotest/experimental/' 6749c5db199SXin Li 'autoserv_job_run_duration') 6759c5db199SXin Li f = {'in_container': utils.is_in_container(), 6769c5db199SXin Li 'success': False} 6779c5db199SXin Li with metrics.SecondsTimer(metric_name, fields=f) as c: 6789c5db199SXin Li job.run(verify_job_repo_url=verify_job_repo_url, 6799c5db199SXin Li only_collect_crashinfo=collect_crashinfo, 6809c5db199SXin Li skip_crash_collection=skip_crash_collection, 6819c5db199SXin Li job_labels=job_labels, 6829c5db199SXin Li use_packaging=(not no_use_packaging)) 6839c5db199SXin Li c['success'] = True 6849c5db199SXin Li 6859c5db199SXin Li finally: 6869c5db199SXin Li job.close() 6879c5db199SXin Li except error.AutoservSSPError: 6889c5db199SXin Li # Due to the complexity of the TKO parsing/stainless connection, this 6899c5db199SXin Li # must be 0 so that the "abort" is actually reflected on stainless. 6909c5db199SXin Li exit_code = 0 6919c5db199SXin Li traceback.print_exc() 6929c5db199SXin Li except: 6939c5db199SXin Li exit_code = 1 6949c5db199SXin Li traceback.print_exc() 6959c5db199SXin Li finally: 6969c5db199SXin Li metrics.Flush() 6979c5db199SXin Li 6989c5db199SXin Li sys.exit(exit_code) 6999c5db199SXin Li 7009c5db199SXin Li 7019c5db199SXin Li# Job breakdown statuses 7029c5db199SXin Li_hs = host_states.Status 7039c5db199SXin Li_qs = host_queue_entry_states.Status 7049c5db199SXin Li_status_list = [ 7059c5db199SXin Li _qs.QUEUED, _qs.RESETTING, _qs.VERIFYING, 7069c5db199SXin Li _qs.PROVISIONING, _hs.REPAIRING, _qs.CLEANING, 7079c5db199SXin Li _qs.RUNNING, _qs.GATHERING, _qs.PARSING] 7089c5db199SXin Li_JOB_OVERHEAD_STATUS = autotest_enum.AutotestEnum(*_status_list, 7099c5db199SXin Li string_values=True) 7109c5db199SXin Li 7119c5db199SXin Li 7129c5db199SXin Lidef get_job_status(options): 7139c5db199SXin Li """Returns the HQE Status for this run. 7149c5db199SXin Li 7159c5db199SXin Li @param options: parser options. 7169c5db199SXin Li """ 7179c5db199SXin Li s = _JOB_OVERHEAD_STATUS 7189c5db199SXin Li task_mapping = { 7199c5db199SXin Li 'reset': s.RESETTING, 'verify': s.VERIFYING, 7209c5db199SXin Li 'provision': s.PROVISIONING, 'repair': s.REPAIRING, 7219c5db199SXin Li 'cleanup': s.CLEANING, 'collect_crashinfo': s.GATHERING} 7229c5db199SXin Li match = [task for task in task_mapping if getattr(options, task, False)] 7239c5db199SXin Li return task_mapping[match[0]] if match else s.RUNNING 7249c5db199SXin Li 7259c5db199SXin Li 7269c5db199SXin Lidef _require_ssp_from_control(control_name): 7279c5db199SXin Li """Read the value of REQUIRE_SSP from test control file. 7289c5db199SXin Li 7299c5db199SXin Li This reads the control file from the prod checkout of autotest and uses that 7309c5db199SXin Li to determine whether to even stage the SSP package on a devserver. 7319c5db199SXin Li 7329c5db199SXin Li This means: 7339c5db199SXin Li [1] Any change in REQUIRE_SSP directive in a test requires a prod-push to go 7349c5db199SXin Li live. 7359c5db199SXin Li [2] This function may find that the control file does not exist but the SSP 7369c5db199SXin Li package may contain the test file. This function conservatively returns True 7379c5db199SXin Li in that case. 7389c5db199SXin Li 7399c5db199SXin Li This function is called very early in autoserv, before logging is setup. 7409c5db199SXin Li """ 7419c5db199SXin Li if not control_name: 7429c5db199SXin Li return True 7439c5db199SXin Li try: 7449c5db199SXin Li path = _control_path_on_disk(control_name) 7459c5db199SXin Li except error.AutoservError as e: 7469c5db199SXin Li sys.stderr.write("autoserv: Could not determine control file path," 7479c5db199SXin Li " assuming we need SSP: %s\n" % e) 7489c5db199SXin Li sys.stderr.flush() 7499c5db199SXin Li return True 7509c5db199SXin Li if not os.path.isfile(path): 7519c5db199SXin Li return True 7529c5db199SXin Li control = control_data.parse_control(path) 7539c5db199SXin Li # There must be explicit directive in the control file to disable SSP. 7549c5db199SXin Li if not control or control.require_ssp is None: 7559c5db199SXin Li return True 7569c5db199SXin Li return control.require_ssp 7579c5db199SXin Li 7589c5db199SXin Li 7599c5db199SXin Lidef _ssp_base_image_name_or_default(options): 7609c5db199SXin Li """Extract base image name from autoserv options or the global config.""" 7619c5db199SXin Li if options.ssp_base_image_name: 7629c5db199SXin Li return options.ssp_base_image_name 7639c5db199SXin Li return global_config.global_config.get_config_value('AUTOSERV', 7649c5db199SXin Li 'container_base_name') 7659c5db199SXin Li 7669c5db199SXin Li 7679c5db199SXin Lidef main(): 7689c5db199SXin Li start_time = datetime.datetime.now() 7699c5db199SXin Li parser = autoserv_parser.autoserv_parser 7709c5db199SXin Li parser.parse_args() 7719c5db199SXin Li 7729c5db199SXin Li if len(sys.argv) == 1: 7739c5db199SXin Li parser.parser.print_help() 7749c5db199SXin Li sys.exit(1) 7759c5db199SXin Li 7769c5db199SXin Li if parser.options.no_logging: 7779c5db199SXin Li results = None 7789c5db199SXin Li else: 7799c5db199SXin Li results = parser.options.results 7809c5db199SXin Li if not results: 7819c5db199SXin Li results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S') 7829c5db199SXin Li results = os.path.abspath(results) 7839c5db199SXin Li resultdir_exists = False 7849c5db199SXin Li for filename in ('control.srv', 'status.log', '.autoserv_execute'): 7859c5db199SXin Li if os.path.exists(os.path.join(results, filename)): 7869c5db199SXin Li resultdir_exists = True 7879c5db199SXin Li if not parser.options.use_existing_results and resultdir_exists: 7889c5db199SXin Li error = "Error: results directory already exists: %s\n" % results 7899c5db199SXin Li sys.stderr.write(error) 7909c5db199SXin Li sys.exit(1) 7919c5db199SXin Li 7929c5db199SXin Li # Now that we certified that there's no leftover results dir from 7939c5db199SXin Li # previous jobs, lets create the result dir since the logging system 7949c5db199SXin Li # needs to create the log file in there. 7959c5db199SXin Li if not os.path.isdir(results): 7969c5db199SXin Li os.makedirs(results) 7979c5db199SXin Li 7989c5db199SXin Li if parser.options.require_ssp: 7999c5db199SXin Li # This is currently only used for skylab (i.e., when --control-name is 8009c5db199SXin Li # used). 8019c5db199SXin Li use_ssp = _require_ssp_from_control(parser.options.control_name) 8029c5db199SXin Li else: 8039c5db199SXin Li use_ssp = False 8049c5db199SXin Li 8059c5db199SXin Li 8069c5db199SXin Li if use_ssp: 8079c5db199SXin Li log_dir = os.path.join(results, 'ssp_logs') if results else None 8089c5db199SXin Li if log_dir and not os.path.exists(log_dir): 8099c5db199SXin Li os.makedirs(log_dir) 8109c5db199SXin Li else: 8119c5db199SXin Li log_dir = results 8129c5db199SXin Li 8139c5db199SXin Li logging_manager.configure_logging( 8149c5db199SXin Li server_logging_config.ServerLoggingConfig(), 8159c5db199SXin Li results_dir=log_dir, 8169c5db199SXin Li use_console=not parser.options.no_tee, 8179c5db199SXin Li verbose=parser.options.verbose, 8189c5db199SXin Li no_console_prefix=parser.options.no_console_prefix) 8199c5db199SXin Li 8209c5db199SXin Li logging.debug('autoserv is running in drone %s.', socket.gethostname()) 8219c5db199SXin Li logging.debug('autoserv environment: %r', os.environ) 8229c5db199SXin Li logging.debug('autoserv command was: %s', ' '.join(sys.argv)) 8239c5db199SXin Li logging.debug('autoserv parsed options: %s', parser.options) 8249c5db199SXin Li logging.debug('autoserv python version: %s', sys.version) 8259c5db199SXin Li 8269c5db199SXin Li if use_ssp: 8279c5db199SXin Li ssp_url = _stage_ssp(parser, results) 8289c5db199SXin Li else: 8299c5db199SXin Li ssp_url = None 8309c5db199SXin Li 8319c5db199SXin Li if results: 8329c5db199SXin Li logging.info("Results placed in %s" % results) 8339c5db199SXin Li 8349c5db199SXin Li # wait until now to perform this check, so it get properly logged 8359c5db199SXin Li if (parser.options.use_existing_results and not resultdir_exists and 8369c5db199SXin Li not utils.is_in_container()): 8379c5db199SXin Li logging.error("No existing results directory found: %s", results) 8389c5db199SXin Li sys.exit(1) 8399c5db199SXin Li 8409c5db199SXin Li if parser.options.write_pidfile and results: 8419c5db199SXin Li pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label, 8429c5db199SXin Li results) 8439c5db199SXin Li pid_file_manager.open_file() 8449c5db199SXin Li else: 8459c5db199SXin Li pid_file_manager = None 8469c5db199SXin Li 8479c5db199SXin Li autotest.Autotest.set_install_in_tmpdir( 8489c5db199SXin Li parser.options.install_in_tmpdir) 8499c5db199SXin Li 8509c5db199SXin Li exit_code = 0 8519c5db199SXin Li is_task = (parser.options.verify or parser.options.repair or 8529c5db199SXin Li parser.options.provision or parser.options.reset or 8539c5db199SXin Li parser.options.cleanup or parser.options.collect_crashinfo) 8549c5db199SXin Li 8559c5db199SXin Li trace_labels = { 8569c5db199SXin Li 'job_id': job_directories.get_job_id_or_task_id( 8579c5db199SXin Li parser.options.results) 8589c5db199SXin Li } 8599c5db199SXin Li trace = cloud_trace.SpanStack( 8609c5db199SXin Li labels=trace_labels, 8619c5db199SXin Li global_context=parser.options.cloud_trace_context) 8629c5db199SXin Li trace.enabled = parser.options.cloud_trace_context_enabled == 'True' 8639c5db199SXin Li try: 8649c5db199SXin Li try: 8659c5db199SXin Li with trace.Span(get_job_status(parser.options)): 8669c5db199SXin Li run_autoserv(pid_file_manager, results, parser, ssp_url, 8679c5db199SXin Li use_ssp) 8689c5db199SXin Li except SystemExit as e: 8699c5db199SXin Li exit_code = e.code 8709c5db199SXin Li if exit_code: 8719c5db199SXin Li logging.exception('Uncaught SystemExit with code %s', exit_code) 8729c5db199SXin Li except Exception: 8739c5db199SXin Li # If we don't know what happened, we'll classify it as 8749c5db199SXin Li # an 'abort' and return 1. 8759c5db199SXin Li logging.exception('Uncaught Exception, exit_code = 1.') 8769c5db199SXin Li exit_code = 1 8779c5db199SXin Li finally: 8789c5db199SXin Li if pid_file_manager: 8799c5db199SXin Li pid_file_manager.close_file(exit_code) 8809c5db199SXin Li sys.exit(exit_code) 8819c5db199SXin Li 8829c5db199SXin Li 8839c5db199SXin Liif __name__ == '__main__': 8849c5db199SXin Li main() 885