1# Copyright 2017 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Utility to deploy and run result utils on a DUT. 6 7This module is the one imported by other Autotest code and run result 8throttling. Other modules in result_tools are designed to be copied to DUT and 9executed with command line. That's why other modules (except view.py and 10unittests) don't import the common module. 11""" 12 13import logging 14import os 15 16import common 17from autotest_lib.client.common_lib import error 18from autotest_lib.client.common_lib import global_config 19from autotest_lib.client.common_lib import utils as client_utils 20 21try: 22 from autotest_lib.utils.frozen_chromite.lib import metrics 23except ImportError: 24 metrics = client_utils.metrics_mock 25 26 27CONFIG = global_config.global_config 28ENABLE_RESULT_THROTTLING = CONFIG.get_config_value( 29 'AUTOSERV', 'enable_result_throttling', type=bool, default=False) 30 31_THROTTLE_OPTION_FMT = '-m %s' 32_SUMMARY_CMD= '%s/result_tools/utils.py' 33_BUILD_DIR_SUMMARY_CMD = _SUMMARY_CMD + ' -p %s %s' 34_BUILD_DIR_SUMMARY_TIMEOUT = 120 35_FIND_DIR_SUMMARY_TIMEOUT = 10 36_CLEANUP_DIR_SUMMARY_CMD = _SUMMARY_CMD + ' -p %s -d' 37_CLEANUP_DIR_SUMMARY_TIMEOUT = 10 38 39# Default autotest directory on host 40DEFAULT_AUTOTEST_DIR = '/usr/local/autotest' 41 42# File patterns to be excluded from deploying to the dut. 43_EXCLUDES = ['*.pyc', '*unittest.py', 'common.py', '__init__.py', 'runner.py', 44 'view.py'] 45 46def _deploy_result_tools(host): 47 """Send result tools to the dut. 48 49 @param host: Host to run the result tools. 50 """ 51 with metrics.SecondsTimer( 52 'chromeos/autotest/job/send_result_tools_duration', 53 fields={'dut_host_name': host.hostname}) as fields: 54 try: 55 logging.debug('Always Deploying result utilities to %s', 56 host.hostname) 57 result_tools_dir = os.path.dirname(__file__) 58 host.send_file(result_tools_dir, 59 DEFAULT_AUTOTEST_DIR, 60 excludes=_EXCLUDES) 61 fields['success'] = True 62 except error.AutotestHostRunError: 63 logging.debug('Failed to deploy result tools using `excludes`. Try ' 64 'again without `excludes`.') 65 host.send_file(result_tools_dir, DEFAULT_AUTOTEST_DIR) 66 fields['success'] = False 67 68 69def run_on_client(host, client_results_dir, cleanup_only=False): 70 """Run result utils on the given host. 71 72 @param host: Host to run the result utils. 73 @param client_results_dir: Path to the results directory on the client. 74 @param cleanup_only: True to delete all existing directory summary files in 75 the given directory. 76 @return: True: If the command runs on client without error. 77 False: If the command failed with error in result throttling. 78 """ 79 success = False 80 with metrics.SecondsTimer( 81 'chromeos/autotest/job/dir_summary_collection_duration', 82 fields={'dut_host_name': host.hostname}) as fields: 83 try: 84 _deploy_result_tools(host) 85 86 if cleanup_only: 87 logging.debug('Cleaning up directory summary in %s', 88 client_results_dir) 89 cmd = (_CLEANUP_DIR_SUMMARY_CMD % 90 (DEFAULT_AUTOTEST_DIR, client_results_dir)) 91 host.run(cmd, ignore_status=False, 92 timeout=_CLEANUP_DIR_SUMMARY_TIMEOUT) 93 else: 94 logging.debug('Getting directory summary for %s', 95 client_results_dir) 96 throttle_option = '' 97 if ENABLE_RESULT_THROTTLING: 98 try: 99 throttle_option = (_THROTTLE_OPTION_FMT % 100 host.job.max_result_size_KB) 101 except AttributeError: 102 # In case host job is not set, skip throttling. 103 logging.warning('host object does not have job attribute, ' 104 'skipping result throttling.') 105 cmd = (_BUILD_DIR_SUMMARY_CMD % 106 (DEFAULT_AUTOTEST_DIR, client_results_dir, 107 throttle_option)) 108 host.run(cmd, ignore_status=False, 109 timeout=_BUILD_DIR_SUMMARY_TIMEOUT) 110 success = True 111 fields['success'] = True 112 except error.AutoservRunError: 113 action = 'cleanup' if cleanup_only else 'create' 114 logging.exception( 115 'Non-critical failure: Failed to %s directory summary for ' 116 '%s.', action, client_results_dir) 117 fields['success'] = False 118 119 return success 120 121 122def collect_last_summary(host, source_path, dest_path, 123 skip_summary_collection=False): 124 """Collect the last directory summary next to the given file path. 125 126 If the given source_path is a directory, return without collecting any 127 result summary file, as the summary file should have been collected with the 128 directory. 129 130 @param host: The RemoteHost to collect logs from. 131 @param source_path: The remote path to collect the directory summary file 132 from. If the source_path is a file 133 @param dest_path: A path to write the source_path into. The summary file 134 will be saved to the same folder. 135 @param skip_summary_collection: True to skip summary file collection, only 136 to delete the last summary. This is used in case when result 137 collection in the dut failed. Default is set to False. 138 """ 139 if not os.path.exists(dest_path): 140 logging.debug('Source path %s does not exist, no directory summary ' 141 'will be collected', dest_path) 142 return 143 144 # Test if source_path is a file. 145 try: 146 host.run('test -f %s' % source_path, timeout=_FIND_DIR_SUMMARY_TIMEOUT) 147 is_source_file = True 148 except error.AutoservRunError: 149 is_source_file = False 150 # No need to collect summary files if the source path is a directory, 151 # as the summary files should have been copied over with the directory. 152 # However, the last summary should be cleaned up so it won't affect 153 # later tests. 154 skip_summary_collection = True 155 156 source_dir = os.path.dirname(source_path) if is_source_file else source_path 157 dest_dir = dest_path if os.path.isdir(dest_path) else dest_path 158 159 # Get the latest directory summary file. 160 try: 161 summary_pattern = os.path.join(source_dir, 'dir_summary_*.json') 162 summary_file = host.run( 163 'ls -t %s | head -1' % summary_pattern, 164 timeout=_FIND_DIR_SUMMARY_TIMEOUT).stdout.strip() 165 except error.AutoservRunError: 166 logging.exception( 167 'Non-critical failure: Failed to locate the latest directory ' 168 'summary for %s', source_dir) 169 return 170 171 try: 172 if not skip_summary_collection: 173 host.get_file( 174 summary_file, 175 os.path.join(dest_dir, os.path.basename(summary_file)), 176 preserve_perm=False) 177 finally: 178 # Remove the collected summary file so it won't affect later tests. 179 try: 180 # Check and remove the summary file 181 if host.path_exists(summary_file): 182 host.run('rm %s' % summary_file, 183 timeout=_FIND_DIR_SUMMARY_TIMEOUT).stdout.strip() 184 except error.AutoservRunError: 185 logging.exception( 186 'Non-critical failure: Failed to delete the latest ' 187 'directory summary: %s', summary_file) 188