1# Lint as: python2, python3 2from __future__ import absolute_import 3from __future__ import division 4from __future__ import print_function 5import os, re, db, sys, datetime 6import common 7from autotest_lib.client.common_lib import kernel_versions 8from six.moves import map 9 10MAX_RECORDS = 50000 11MAX_CELLS = 500000 12 13tko = os.path.dirname(os.path.realpath(os.path.abspath(__file__))) 14root_url_file = os.path.join(tko, '.root_url') 15if os.path.exists(root_url_file): 16 html_root = open(root_url_file, 'r').readline().rstrip() 17else: 18 html_root = '/results/' 19 20 21class status_cell: 22 # One cell in the matrix of status data. 23 def __init__(self): 24 # Count is a dictionary: status -> count of tests with status 25 self.status_count = {} 26 self.reasons_list = [] 27 self.job_tag = None 28 self.job_tag_count = 0 29 30 31 def add(self, status, count, job_tags, reasons = None): 32 assert count > 0 33 34 self.job_tag = job_tags 35 self.job_tag_count += count 36 if self.job_tag_count > 1: 37 self.job_tag = None 38 39 self.status_count[status] = count 40 ### status == 6 means 'GOOD' 41 if status != 6: 42 ## None implies sorting problems and extra CRs in a cell 43 if reasons: 44 self.reasons_list.append(reasons) 45 46 47class status_data: 48 def __init__(self, sql_rows, x_field, y_field, query_reasons = False): 49 data = {} 50 y_values = set() 51 52 # Walk through the query, filing all results by x, y info 53 for row in sql_rows: 54 if query_reasons: 55 (x,y, status, count, job_tags, reasons) = row 56 else: 57 (x,y, status, count, job_tags) = row 58 reasons = None 59 if x not in data: 60 data[x] = {} 61 if y not in data[x]: 62 y_values.add(y) 63 data[x][y] = status_cell() 64 data[x][y].add(status, count, job_tags, reasons) 65 66 # 2-d hash of data - [x-value][y-value] 67 self.data = data 68 # List of possible columns (x-values) 69 self.x_values = smart_sort(list(data.keys()), x_field) 70 # List of rows columns (y-values) 71 self.y_values = smart_sort(list(y_values), y_field) 72 nCells = len(self.y_values)*len(self.x_values) 73 if nCells > MAX_CELLS: 74 msg = 'Exceeded allowed number of cells in a table' 75 raise db.MySQLTooManyRows(msg) 76 77 78def get_matrix_data(db_obj, x_axis, y_axis, where = None, 79 query_reasons = False): 80 # Searches on the test_view table - x_axis and y_axis must both be 81 # column names in that table. 82 x_field = test_view_field_dict[x_axis] 83 y_field = test_view_field_dict[y_axis] 84 query_fields_list = [x_field, y_field, 'status','COUNT(status)'] 85 query_fields_list.append("LEFT(GROUP_CONCAT(job_tag),100)") 86 if query_reasons: 87 query_fields_list.append( 88 "LEFT(GROUP_CONCAT(DISTINCT reason SEPARATOR '|'),500)" 89 ) 90 fields = ','.join(query_fields_list) 91 92 group_by = '%s, %s, status' % (x_field, y_field) 93 rows = db_obj.select(fields, 'tko_test_view', 94 where=where, group_by=group_by, max_rows = MAX_RECORDS) 95 return status_data(rows, x_field, y_field, query_reasons) 96 97 98# Dictionary used simply for fast lookups from short reference names for users 99# to fieldnames in test_view 100test_view_field_dict = { 101 'kernel' : 'kernel_printable', 102 'hostname' : 'machine_hostname', 103 'test' : 'test', 104 'label' : 'job_label', 105 'machine_group' : 'machine_group', 106 'reason' : 'reason', 107 'tag' : 'job_tag', 108 'user' : 'job_username', 109 'status' : 'status_word', 110 'time' : 'test_finished_time', 111 'start_time' : 'test_started_time', 112 'time_daily' : 'DATE(test_finished_time)' 113} 114 115 116def smart_sort(list, field): 117 if field == 'kernel_printable': 118 def kernel_encode(kernel): 119 return kernel_versions.version_encode(kernel) 120 list.sort(key = kernel_encode, reverse = True) 121 return list 122 ## old records may contain time=None 123 ## make None comparable with timestamp datetime or date 124 elif field == 'test_finished_time': 125 def convert_None_to_datetime(date_time): 126 if not date_time: 127 return datetime.datetime(1970, 1, 1, 0, 0, 0) 128 else: 129 return date_time 130 list = list(map(convert_None_to_datetime, list)) 131 elif field == 'DATE(test_finished_time)': 132 def convert_None_to_date(date): 133 if not date: 134 return datetime.date(1970, 1, 1) 135 else: 136 return date 137 list = list(map(convert_None_to_date, list)) 138 list.sort() 139 return list 140 141 142class group: 143 @classmethod 144 def select(klass, db): 145 """Return all possible machine groups""" 146 rows = db.select('distinct machine_group', 'tko_machines', 147 'machine_group is not null') 148 groupnames = sorted([row[0] for row in rows]) 149 return [klass(db, groupname) for groupname in groupnames] 150 151 152 def __init__(self, db, name): 153 self.name = name 154 self.db = db 155 156 157 def machines(self): 158 return machine.select(self.db, { 'machine_group' : self.name }) 159 160 161 def tests(self, where = {}): 162 values = [self.name] 163 sql = 't inner join tko_machines m on m.machine_idx=t.machine_idx' 164 sql += ' where m.machine_group=%s' 165 for key in where.keys(): 166 sql += ' and %s=%%s' % key 167 values.append(where[key]) 168 return test.select_sql(self.db, sql, values) 169 170 171class machine: 172 @classmethod 173 def select(klass, db, where = {}): 174 fields = ['machine_idx', 'hostname', 'machine_group', 'owner'] 175 machines = [] 176 for row in db.select(','.join(fields), 'tko_machines', where): 177 machines.append(klass(db, *row)) 178 return machines 179 180 181 def __init__(self, db, idx, hostname, group, owner): 182 self.db = db 183 self.idx = idx 184 self.hostname = hostname 185 self.group = group 186 self.owner = owner 187 188 189class kernel: 190 @classmethod 191 def select(klass, db, where = {}): 192 fields = ['kernel_idx', 'kernel_hash', 'base', 'printable'] 193 rows = db.select(','.join(fields), 'tko_kernels', where) 194 return [klass(db, *row) for row in rows] 195 196 197 def __init__(self, db, idx, hash, base, printable): 198 self.db = db 199 self.idx = idx 200 self.hash = hash 201 self.base = base 202 self.printable = printable 203 self.patches = [] # THIS SHOULD PULL IN PATCHES! 204 205 206class test: 207 @classmethod 208 def select(klass, db, where={}, distinct=False): 209 fields = ['test_idx', 'job_idx', 'test', 'subdir', 210 'kernel_idx', 'status', 'reason', 'machine_idx'] 211 tests = [] 212 for row in db.select(','.join(fields), 'tko_tests', where, 213 distinct): 214 tests.append(klass(db, *row)) 215 return tests 216 217 218 @classmethod 219 def select_sql(klass, db, sql, values): 220 fields = ['test_idx', 'job_idx', 'test', 'subdir', 221 'kernel_idx', 'status', 'reason', 'machine_idx'] 222 fields = ['t.'+field for field in fields] 223 rows = db.select_sql(','.join(fields), 'tko_tests', sql, values) 224 return [klass(db, *row) for row in rows] 225 226 227 def __init__(self, db, test_idx, job_idx, testname, subdir, kernel_idx, 228 status_num, reason, machine_idx): 229 self.idx = test_idx 230 self.job = job(db, job_idx) 231 self.testname = testname 232 self.subdir = subdir 233 self.kernel_idx = kernel_idx 234 self.__kernel = None 235 self.__iterations = None 236 self.machine_idx = machine_idx 237 self.__machine = None 238 self.status_num = status_num 239 self.status_word = db.status_word[status_num] 240 self.reason = reason 241 self.db = db 242 if self.subdir: 243 self.url = html_root + self.job.tag + '/' + self.subdir 244 else: 245 self.url = None 246 247 248 def iterations(self): 249 """ 250 Caching function for iterations 251 """ 252 if not self.__iterations: 253 self.__iterations = {} 254 # A dictionary - dict{key} = [value1, value2, ....] 255 where = {'test_idx' : self.idx} 256 for i in iteration.select(self.db, where): 257 if i.key in self.__iterations: 258 self.__iterations[i.key].append(i.value) 259 else: 260 self.__iterations[i.key] = [i.value] 261 return self.__iterations 262 263 264 def kernel(self): 265 """ 266 Caching function for kernels 267 """ 268 if not self.__kernel: 269 where = {'kernel_idx' : self.kernel_idx} 270 self.__kernel = kernel.select(self.db, where)[0] 271 return self.__kernel 272 273 274 def machine(self): 275 """ 276 Caching function for kernels 277 """ 278 if not self.__machine: 279 where = {'machine_idx' : self.machine_idx} 280 self.__machine = machine.select(self.db, where)[0] 281 return self.__machine 282 283 284class job: 285 def __init__(self, db, job_idx): 286 where = {'job_idx' : job_idx} 287 rows = db.select('tag, machine_idx', 'tko_jobs', where) 288 if rows: 289 self.tag, self.machine_idx = rows[0] 290 self.job_idx = job_idx 291 292 293class iteration: 294 @classmethod 295 def select(klass, db, where): 296 fields = ['iteration', 'attribute', 'value'] 297 iterations = [] 298 rows = db.select(','.join(fields), 'tko_iteration_result', where) 299 for row in rows: 300 iterations.append(klass(*row)) 301 return iterations 302 303 304 def __init__(self, iteration, key, value): 305 self.iteration = iteration 306 self.key = key 307 self.value = value 308 309# class patch: 310# def __init__(self): 311# self.spec = None 312