1# Lint as: python2, python3 2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6from __future__ import absolute_import 7from __future__ import division 8from __future__ import print_function 9 10import logging 11import csv 12import six 13import random 14import re 15import collections 16 17from autotest_lib.client.common_lib.cros import path_utils 18 19class ResourceMonitorRawResult(object): 20 """Encapsulates raw resource_monitor results.""" 21 22 def __init__(self, raw_results_filename): 23 self._raw_results_filename = raw_results_filename 24 25 26 def get_parsed_results(self): 27 """Constructs parsed results from the raw ones. 28 29 @return ResourceMonitorParsedResult object 30 31 """ 32 return ResourceMonitorParsedResult(self.raw_results_filename) 33 34 35 @property 36 def raw_results_filename(self): 37 """@return string filename storing the raw top command output.""" 38 return self._raw_results_filename 39 40 41class IncorrectTopFormat(Exception): 42 """Thrown if top output format is not as expected""" 43 pass 44 45 46def _extract_value_before_single_keyword(line, keyword): 47 """Extract word occurring immediately before the specified keyword. 48 49 @param line string the line in which to search for the keyword. 50 @param keyword string the keyword to look for. Can be a regexp. 51 @return string the word just before the keyword. 52 53 """ 54 pattern = ".*?(\S+) " + keyword 55 matches = re.match(pattern, line) 56 if matches is None or len(matches.groups()) != 1: 57 raise IncorrectTopFormat 58 59 return matches.group(1) 60 61 62def _extract_values_before_keywords(line, *args): 63 """Extract the words occuring immediately before each specified 64 keyword in args. 65 66 @param line string the string to look for the keywords. 67 @param args variable number of string args the keywords to look for. 68 @return string list the words occuring just before each keyword. 69 70 """ 71 line_nocomma = re.sub(",", " ", line) 72 line_singlespace = re.sub("\s+", " ", line_nocomma) 73 74 return [_extract_value_before_single_keyword( 75 line_singlespace, arg) for arg in args] 76 77 78def _find_top_output_identifying_pattern(line): 79 """Return true iff the line looks like the first line of top output. 80 81 @param line string to look for the pattern 82 @return boolean 83 84 """ 85 pattern ="\s*top\s*-.*up.*users.*" 86 matches = re.match(pattern, line) 87 return matches is not None 88 89 90class ResourceMonitorParsedResult(object): 91 """Encapsulates logic to parse and represent top command results.""" 92 93 _columns = ["Time", "UserCPU", "SysCPU", "NCPU", "Idle", 94 "IOWait", "IRQ", "SoftIRQ", "Steal", 95 "MemUnits", "UsedMem", "FreeMem", 96 "SwapUnits", "UsedSwap", "FreeSwap"] 97 UtilValues = collections.namedtuple('UtilValues', ' '.join(_columns)) 98 99 def __init__(self, raw_results_filename): 100 """Construct a ResourceMonitorResult. 101 102 @param raw_results_filename string filename of raw batch top output. 103 104 """ 105 self._raw_results_filename = raw_results_filename 106 self.parse_resource_monitor_results() 107 108 109 def parse_resource_monitor_results(self): 110 """Extract utilization metrics from output file.""" 111 self._utils_over_time = [] 112 113 with open(self._raw_results_filename, "r") as results_file: 114 while True: 115 curr_line = '\n' 116 while curr_line != '' and \ 117 not _find_top_output_identifying_pattern(curr_line): 118 curr_line = results_file.readline() 119 if curr_line == '': 120 break 121 try: 122 time, = _extract_values_before_keywords(curr_line, "up") 123 124 # Ignore one line. 125 _ = results_file.readline() 126 127 # Get the cpu usage. 128 curr_line = results_file.readline() 129 (cpu_user, cpu_sys, cpu_nice, cpu_idle, io_wait, irq, sirq, 130 steal) = _extract_values_before_keywords(curr_line, 131 "us", "sy", "ni", "id", "wa", "hi", "si", "st") 132 133 # Get memory usage. 134 curr_line = results_file.readline() 135 (mem_units, mem_free, 136 mem_used) = _extract_values_before_keywords( 137 curr_line, "Mem", "free", "used") 138 139 # Get swap usage. 140 curr_line = results_file.readline() 141 (swap_units, swap_free, 142 swap_used) = _extract_values_before_keywords( 143 curr_line, "Swap", "free", "used") 144 145 curr_util_values = ResourceMonitorParsedResult.UtilValues( 146 Time=time, UserCPU=cpu_user, 147 SysCPU=cpu_sys, NCPU=cpu_nice, Idle=cpu_idle, 148 IOWait=io_wait, IRQ=irq, SoftIRQ=sirq, Steal=steal, 149 MemUnits=mem_units, UsedMem=mem_used, 150 FreeMem=mem_free, 151 SwapUnits=swap_units, UsedSwap=swap_used, 152 FreeSwap=swap_free) 153 self._utils_over_time.append(curr_util_values) 154 except IncorrectTopFormat: 155 logging.error( 156 "Top output format incorrect. Aborting parse.") 157 return 158 159 160 def __repr__(self): 161 output_stringfile = six.StringIO() 162 self.save_to_file(output_stringfile) 163 return output_stringfile.getvalue() 164 165 166 def save_to_file(self, file): 167 """Save parsed top results to file 168 169 @param file file object to write to 170 171 """ 172 if len(self._utils_over_time) < 1: 173 logging.warning("Tried to save parsed results, but they were " 174 "empty. Skipping the save.") 175 return 176 csvwriter = csv.writer(file, delimiter=',') 177 csvwriter.writerow(self._utils_over_time[0]._fields) 178 for row in self._utils_over_time: 179 csvwriter.writerow(row) 180 181 182 def save_to_filename(self, filename): 183 """Save parsed top results to filename 184 185 @param filename string filepath to write to 186 187 """ 188 out_file = open(filename, "wb") 189 self.save_to_file(out_file) 190 out_file.close() 191 192 193class ResourceMonitorConfig(object): 194 """Defines a single top run.""" 195 196 DEFAULT_MONITOR_PERIOD = 3 197 198 def __init__(self, monitor_period=DEFAULT_MONITOR_PERIOD, 199 rawresult_output_filename=None): 200 """Construct a ResourceMonitorConfig. 201 202 @param monitor_period float seconds between successive top refreshes. 203 @param rawresult_output_filename string filename to output the raw top 204 results to 205 206 """ 207 if monitor_period < 0.1: 208 logging.info('Monitor period must be at least 0.1s.' 209 ' Given: %r. Defaulting to 0.1s', monitor_period) 210 monitor_period = 0.1 211 212 self._monitor_period = monitor_period 213 self._server_outfile = rawresult_output_filename 214 215 216class ResourceMonitor(object): 217 """Delegate to run top on a client. 218 219 Usage example (call from a test): 220 rmc = resource_monitor.ResourceMonitorConfig(monitor_period=1, 221 rawresult_output_filename=os.path.join(self.resultsdir, 222 'topout.txt')) 223 with resource_monitor.ResourceMonitor(self.context.client.host, rmc) as rm: 224 rm.start() 225 <operation_to_monitor> 226 rm_raw_res = rm.stop() 227 rm_res = rm_raw_res.get_parsed_results() 228 rm_res.save_to_filename( 229 os.path.join(self.resultsdir, 'resource_mon.csv')) 230 231 """ 232 233 def __init__(self, client_host, config): 234 """Construct a ResourceMonitor. 235 236 @param client_host: SSHHost object representing a remote ssh host 237 238 """ 239 self._client_host = client_host 240 self._config = config 241 self._command_top = path_utils.must_be_installed( 242 'top', host=self._client_host) 243 self._top_pid = None 244 245 246 def __enter__(self): 247 return self 248 249 250 def __exit__(self, exc_type, exc_value, traceback): 251 if self._top_pid is not None: 252 self._client_host.run('kill %s && rm %s' % 253 (self._top_pid, self._client_outfile), ignore_status=True) 254 return True 255 256 257 def start(self): 258 """Run top and save results to a temp file on the client.""" 259 if self._top_pid is not None: 260 logging.debug("Tried to start monitoring before stopping. " 261 "Ignoring request.") 262 return 263 264 # Decide where to write top's output to (on the client). 265 random_suffix = random.random() 266 self._client_outfile = '/tmp/topcap-%r' % random_suffix 267 268 # Run top on the client. 269 top_command = '%s -b -d%d > %s' % (self._command_top, 270 self._config._monitor_period, self._client_outfile) 271 logging.info('Running top.') 272 self._top_pid = self._client_host.run_background(top_command) 273 logging.info('Top running with pid %s', self._top_pid) 274 275 276 def stop(self): 277 """Stop running top and return the results. 278 279 @return ResourceMonitorRawResult object 280 281 """ 282 logging.debug("Stopping monitor") 283 if self._top_pid is None: 284 logging.debug("Tried to stop monitoring before starting. " 285 "Ignoring request.") 286 return 287 288 # Stop top on the client. 289 self._client_host.run('kill %s' % self._top_pid, ignore_status=True) 290 291 # Get the top output file from the client onto the server. 292 if self._config._server_outfile is None: 293 self._config._server_outfile = self._client_outfile 294 self._client_host.get_file( 295 self._client_outfile, self._config._server_outfile) 296 297 # Delete the top output file from client. 298 self._client_host.run('rm %s' % self._client_outfile, 299 ignore_status=True) 300 301 self._top_pid = None 302 logging.info("Saved resource monitor results at %s", 303 self._config._server_outfile) 304 return ResourceMonitorRawResult(self._config._server_outfile) 305