1# Lint as: python2, python3 2from __future__ import division 3from __future__ import print_function 4 5import json 6import os 7 8from autotest_lib.server.hosts import file_store 9from autotest_lib.client.common_lib import utils 10from autotest_lib.tko import tast 11from autotest_lib.tko import utils as tko_utils 12import six 13 14 15class HostKeyvalError(Exception): 16 """Raised when the host keyval cannot be read.""" 17 18 19class job(object): 20 """Represents a job.""" 21 22 def __init__(self, dir, user, label, machine, queued_time, started_time, 23 finished_time, machine_owner, machine_group, aborted_by, 24 aborted_on, keyval_dict): 25 self.dir = dir 26 self.tests = [] 27 self.user = user 28 self.label = label 29 self.machine = machine 30 self.queued_time = queued_time 31 self.started_time = started_time 32 self.finished_time = finished_time 33 self.machine_owner = machine_owner 34 self.machine_group = machine_group 35 self.aborted_by = aborted_by 36 self.aborted_on = aborted_on 37 self.keyval_dict = keyval_dict 38 self.afe_parent_job_id = None 39 self.build_version = None 40 self.suite = None 41 self.board = None 42 self.job_idx = None 43 # id of the corresponding tko_task_references entry. 44 # This table is used to refer to skylab task / afe job corresponding to 45 # this tko_job. 46 self.task_reference_id = None 47 48 @staticmethod 49 def read_keyval(dir): 50 """ 51 Read job keyval files. 52 53 @param dir: String name of directory containing job keyval files. 54 55 @return A dictionary containing job keyvals. 56 57 """ 58 dir = os.path.normpath(dir) 59 top_dir = tko_utils.find_toplevel_job_dir(dir) 60 if not top_dir: 61 top_dir = dir 62 assert(dir.startswith(top_dir)) 63 64 # Pull in and merge all the keyval files, with higher-level 65 # overriding values in the lower-level ones. 66 keyval = {} 67 while True: 68 try: 69 upper_keyval = utils.read_keyval(dir) 70 # HACK: exclude hostname from the override - this is a special 71 # case where we want lower to override higher. 72 if 'hostname' in upper_keyval and 'hostname' in keyval: 73 del upper_keyval['hostname'] 74 keyval.update(upper_keyval) 75 except IOError: 76 pass # If the keyval can't be read just move on to the next. 77 if dir == top_dir: 78 break 79 else: 80 assert(dir != '/') 81 dir = os.path.dirname(dir) 82 return keyval 83 84 85class kernel(object): 86 """Represents a kernel.""" 87 88 def __init__(self, base, patches, kernel_hash): 89 self.base = base 90 self.patches = patches 91 self.kernel_hash = kernel_hash 92 93 94 @staticmethod 95 def compute_hash(base, hashes): 96 """Compute a hash given the base string and hashes for each patch. 97 98 @param base: A string representing the kernel base. 99 @param hashes: A list of hashes, where each hash is associated with a 100 patch of this kernel. 101 102 @return A string representing the computed hash. 103 104 """ 105 key_string = ','.join([base] + hashes) 106 return utils.hash('md5', key_string).hexdigest() 107 108 109class test(object): 110 """Represents a test.""" 111 112 def __init__(self, subdir, testname, status, reason, test_kernel, 113 machine, started_time, finished_time, iterations, 114 attributes, perf_values, labels): 115 self.subdir = subdir 116 self.testname = testname 117 self.status = status 118 self.reason = reason 119 self.kernel = test_kernel 120 self.machine = machine 121 self.started_time = started_time 122 self.finished_time = finished_time 123 self.iterations = iterations 124 self.attributes = attributes 125 self.perf_values = perf_values 126 self.labels = labels 127 128 129 @staticmethod 130 def load_iterations(keyval_path): 131 """Abstract method to load a list of iterations from a keyval file. 132 133 @param keyval_path: String path to a keyval file. 134 135 @return A list of iteration objects. 136 137 """ 138 raise NotImplementedError 139 140 141 @classmethod 142 def parse_test(cls, job, subdir, testname, status, reason, test_kernel, 143 started_time, finished_time, existing_instance=None): 144 """ 145 Parse test result files to construct a complete test instance. 146 147 Given a job and the basic metadata about the test that can be 148 extracted from the status logs, parse the test result files (keyval 149 files and perf measurement files) and use them to construct a complete 150 test instance. 151 152 @param job: A job object. 153 @param subdir: The string subdirectory name for the given test. 154 @param testname: The name of the test. 155 @param status: The status of the test. 156 @param reason: The reason string for the test. 157 @param test_kernel: The kernel of the test. 158 @param started_time: The start time of the test. 159 @param finished_time: The finish time of the test. 160 @param existing_instance: An existing test instance. 161 162 @return A test instance that has the complete information. 163 164 """ 165 tko_utils.dprint("parsing test %s %s" % (subdir, testname)) 166 167 if tast.is_tast_test(testname): 168 attributes, perf_values = tast.load_tast_test_aux_results(job, 169 testname) 170 iterations = [] 171 elif subdir: 172 # Grab iterations from the results keyval. 173 iteration_keyval = os.path.join(job.dir, subdir, 174 'results', 'keyval') 175 iterations = cls.load_iterations(iteration_keyval) 176 177 # Grab perf values from the perf measurements file. 178 perf_values_file = os.path.join(job.dir, subdir, 179 'results', 'results-chart.json') 180 perf_values = {} 181 if os.path.exists(perf_values_file): 182 with open(perf_values_file, 'r') as fp: 183 contents = fp.read() 184 if contents: 185 perf_values = json.loads(contents) 186 187 # Grab test attributes from the subdir keyval. 188 test_keyval = os.path.join(job.dir, subdir, 'keyval') 189 attributes = test.load_attributes(test_keyval) 190 else: 191 iterations = [] 192 perf_values = {} 193 attributes = {} 194 195 # Grab test+host attributes from the host keyval. 196 host_keyval = cls.parse_host_keyval(job.dir, job.machine) 197 attributes.update(dict(('host-%s' % k, v) 198 for k, v in six.iteritems(host_keyval))) 199 200 if existing_instance: 201 def constructor(*args, **dargs): 202 """Initializes an existing test instance.""" 203 existing_instance.__init__(*args, **dargs) 204 return existing_instance 205 else: 206 constructor = cls 207 208 return constructor(subdir, testname, status, reason, test_kernel, 209 job.machine, started_time, finished_time, 210 iterations, attributes, perf_values, []) 211 212 213 @classmethod 214 def parse_partial_test(cls, job, subdir, testname, reason, test_kernel, 215 started_time): 216 """ 217 Create a test instance representing a partial test result. 218 219 Given a job and the basic metadata available when a test is 220 started, create a test instance representing the partial result. 221 Assume that since the test is not complete there are no results files 222 actually available for parsing. 223 224 @param job: A job object. 225 @param subdir: The string subdirectory name for the given test. 226 @param testname: The name of the test. 227 @param reason: The reason string for the test. 228 @param test_kernel: The kernel of the test. 229 @param started_time: The start time of the test. 230 231 @return A test instance that has partial test information. 232 233 """ 234 tko_utils.dprint('parsing partial test %s %s' % (subdir, testname)) 235 236 return cls(subdir, testname, 'RUNNING', reason, test_kernel, 237 job.machine, started_time, None, [], {}, [], []) 238 239 240 @staticmethod 241 def load_attributes(keyval_path): 242 """ 243 Load test attributes from a test keyval path. 244 245 Load the test attributes into a dictionary from a test 246 keyval path. Does not assume that the path actually exists. 247 248 @param keyval_path: The string path to a keyval file. 249 250 @return A dictionary representing the test keyvals. 251 252 """ 253 if not os.path.exists(keyval_path): 254 return {} 255 return utils.read_keyval(keyval_path) 256 257 258 @staticmethod 259 def _parse_keyval(job_dir, sub_keyval_path): 260 """ 261 Parse a file of keyvals. 262 263 @param job_dir: The string directory name of the associated job. 264 @param sub_keyval_path: Path to a keyval file relative to job_dir. 265 266 @return A dictionary representing the keyvals. 267 268 """ 269 # The "real" job dir may be higher up in the directory tree. 270 job_dir = tko_utils.find_toplevel_job_dir(job_dir) 271 if not job_dir: 272 return {} # We can't find a top-level job dir with job keyvals. 273 274 # The keyval is <job_dir>/`sub_keyval_path` if it exists. 275 keyval_path = os.path.join(job_dir, sub_keyval_path) 276 if os.path.isfile(keyval_path): 277 return utils.read_keyval(keyval_path) 278 else: 279 return {} 280 281 282 @staticmethod 283 def _is_multimachine(job_dir): 284 """ 285 Determine whether the job is a multi-machine job. 286 287 @param job_dir: The string directory name of the associated job. 288 289 @return True, if the job is a multi-machine job, or False if not. 290 291 """ 292 machines_path = os.path.join(job_dir, '.machines') 293 if os.path.exists(machines_path): 294 with open(machines_path, 'r') as fp: 295 line_count = len(fp.read().splitlines()) 296 if line_count > 1: 297 return True 298 return False 299 300 301 @staticmethod 302 def parse_host_keyval(job_dir, hostname): 303 """ 304 Parse host keyvals. 305 306 @param job_dir: The string directory name of the associated job. 307 @param hostname: The string hostname. 308 309 @return A dictionary representing the host keyvals. 310 311 @raises HostKeyvalError if the host keyval is not found. 312 313 """ 314 keyval_path = os.path.join('host_keyvals', hostname) 315 hostinfo_path = os.path.join(job_dir, 'host_info_store', 316 hostname + '.store') 317 # Skylab uses hostinfo. If this is not present, try falling back to the 318 # host keyval file (moblab), or an empty host keyval for multi-machine 319 # tests (jetstream). 320 if os.path.exists(hostinfo_path): 321 tko_utils.dprint('Reading keyvals from hostinfo.') 322 return _parse_hostinfo_keyval(hostinfo_path) 323 elif os.path.exists(os.path.join(job_dir, keyval_path)): 324 tko_utils.dprint('Reading keyvals from %s.' % keyval_path) 325 return test._parse_keyval(job_dir, keyval_path) 326 elif test._is_multimachine(job_dir): 327 tko_utils.dprint('Multimachine job, no keyvals.') 328 return {} 329 raise HostKeyvalError('Host keyval not found') 330 331 332 @staticmethod 333 def parse_job_keyval(job_dir): 334 """ 335 Parse job keyvals. 336 337 @param job_dir: The string directory name of the associated job. 338 339 @return A dictionary representing the job keyvals. 340 341 """ 342 # The job keyval is <job_dir>/keyval if it exists. 343 return test._parse_keyval(job_dir, 'keyval') 344 345 346def _parse_hostinfo_keyval(hostinfo_path): 347 """ 348 Parse host keyvals from hostinfo. 349 350 @param hostinfo_path: The string path to the host info store file. 351 352 @return A dictionary representing the host keyvals. 353 354 """ 355 store = file_store.FileStore(hostinfo_path) 356 hostinfo = store.get() 357 # TODO(ayatane): Investigate if urllib.quote is better. 358 label_string = ','.join(label.replace(':', '%3A') 359 for label in hostinfo.labels) 360 return { 361 'labels': label_string, 362 'platform': hostinfo.model, 363 'board': hostinfo.board 364 } 365 366 367class patch(object): 368 """Represents a patch.""" 369 370 def __init__(self, spec, reference, hash): 371 self.spec = spec 372 self.reference = reference 373 self.hash = hash 374 375 376class iteration(object): 377 """Represents an iteration.""" 378 379 def __init__(self, index, attr_keyval, perf_keyval): 380 self.index = index 381 self.attr_keyval = attr_keyval 382 self.perf_keyval = perf_keyval 383 384 385 @staticmethod 386 def parse_line_into_dicts(line, attr_dict, perf_dict): 387 """ 388 Abstract method to parse a keyval line and insert it into a dictionary. 389 390 @param line: The string line to parse. 391 @param attr_dict: Dictionary of generic iteration attributes. 392 @param perf_dict: Dictionary of iteration performance results. 393 394 """ 395 raise NotImplementedError 396 397 398 @classmethod 399 def load_from_keyval(cls, keyval_path): 400 """ 401 Load a list of iterations from an iteration keyval file. 402 403 Keyval data from separate iterations is separated by blank 404 lines. Makes use of the parse_line_into_dicts method to 405 actually parse the individual lines. 406 407 @param keyval_path: The string path to a keyval file. 408 409 @return A list of iteration objects. 410 411 """ 412 if not os.path.exists(keyval_path): 413 return [] 414 415 iterations = [] 416 index = 1 417 attr, perf = {}, {} 418 with open(keyval_path, 'r') as kp: 419 lines = kp.readlines() 420 for line in lines: 421 line = line.strip() 422 if line: 423 cls.parse_line_into_dicts(line, attr, perf) 424 else: 425 iterations.append(cls(index, attr, perf)) 426 index += 1 427 attr, perf = {}, {} 428 if attr or perf: 429 iterations.append(cls(index, attr, perf)) 430 return iterations 431 432 433class perf_value_iteration(object): 434 """Represents a perf value iteration.""" 435 436 def __init__(self, index, perf_measurements): 437 """ 438 Initializes the perf values for a particular test iteration. 439 440 @param index: The integer iteration number. 441 @param perf_measurements: A list of dictionaries, where each dictionary 442 contains the information for a measured perf metric from the 443 current iteration. 444 445 """ 446 self.index = index 447 self.perf_measurements = perf_measurements 448 449 450 @staticmethod 451 def parse_line_into_dict(line): 452 """ 453 Abstract method to parse an individual perf measurement line. 454 455 @param line: A string line from the perf measurement output file. 456 457 @return A dicionary representing the information for a measured perf 458 metric from one line of the perf measurement output file, or an 459 empty dictionary if the line cannot be parsed successfully. 460 461 """ 462 raise NotImplementedError 463