1*9c5db199SXin Lifrom django.db import models as dbmodels, connection 2*9c5db199SXin Lifrom autotest_lib.frontend.afe import model_logic, readonly_connection 3*9c5db199SXin Li 4*9c5db199SXin Li_quote_name = connection.ops.quote_name 5*9c5db199SXin Li 6*9c5db199SXin Liclass TempManager(model_logic.ExtendedManager): 7*9c5db199SXin Li """A Temp Manager.""" 8*9c5db199SXin Li _GROUP_COUNT_NAME = 'group_count' 9*9c5db199SXin Li 10*9c5db199SXin Li def _get_key_unless_is_function(self, field): 11*9c5db199SXin Li if '(' in field: 12*9c5db199SXin Li return field 13*9c5db199SXin Li return self.get_key_on_this_table(field) 14*9c5db199SXin Li 15*9c5db199SXin Li 16*9c5db199SXin Li def _get_field_names(self, fields, extra_select_fields={}): 17*9c5db199SXin Li field_names = [] 18*9c5db199SXin Li for field in fields: 19*9c5db199SXin Li if field in extra_select_fields: 20*9c5db199SXin Li field_names.append(extra_select_fields[field][0]) 21*9c5db199SXin Li else: 22*9c5db199SXin Li field_names.append(self._get_key_unless_is_function(field)) 23*9c5db199SXin Li return field_names 24*9c5db199SXin Li 25*9c5db199SXin Li 26*9c5db199SXin Li def _get_group_query_sql(self, query, group_by): 27*9c5db199SXin Li compiler = query.query.get_compiler(using=query.db) 28*9c5db199SXin Li sql, params = compiler.as_sql() 29*9c5db199SXin Li 30*9c5db199SXin Li 31*9c5db199SXin Li # insert GROUP BY clause into query 32*9c5db199SXin Li group_fields = self._get_field_names(group_by, query.query.extra_select) 33*9c5db199SXin Li group_by_clause = ' GROUP BY ' + ', '.join(group_fields) 34*9c5db199SXin Li group_by_position = sql.rfind('ORDER BY') 35*9c5db199SXin Li if group_by_position == -1: 36*9c5db199SXin Li group_by_position = len(sql) 37*9c5db199SXin Li sql = (sql[:group_by_position] + 38*9c5db199SXin Li group_by_clause + ' ' + 39*9c5db199SXin Li sql[group_by_position:]) 40*9c5db199SXin Li 41*9c5db199SXin Li return sql, params 42*9c5db199SXin Li 43*9c5db199SXin Li 44*9c5db199SXin Li def _get_column_names(self, cursor): 45*9c5db199SXin Li """Gets the column names from the cursor description. 46*9c5db199SXin Li 47*9c5db199SXin Li This method exists so that it can be mocked in the unit test for 48*9c5db199SXin Li sqlite3 compatibility. 49*9c5db199SXin Li 50*9c5db199SXin Li """ 51*9c5db199SXin Li return [column_info[0] for column_info in cursor.description] 52*9c5db199SXin Li 53*9c5db199SXin Li 54*9c5db199SXin Li def execute_group_query(self, query, group_by): 55*9c5db199SXin Li """Performs the given query grouped by the specified fields. 56*9c5db199SXin Li 57*9c5db199SXin Li The given query's extra select fields are added. 58*9c5db199SXin Li 59*9c5db199SXin Li @param query: The query to perform. 60*9c5db199SXin Li @param group_by: The fields by which to group. 61*9c5db199SXin Li 62*9c5db199SXin Li @return A list of dicts, where each dict corresponds to single row and 63*9c5db199SXin Li contains a key for each grouped field as well as all of the extra 64*9c5db199SXin Li select fields. 65*9c5db199SXin Li 66*9c5db199SXin Li """ 67*9c5db199SXin Li sql, params = self._get_group_query_sql(query, group_by) 68*9c5db199SXin Li cursor = readonly_connection.cursor() 69*9c5db199SXin Li cursor.execute(sql, params) 70*9c5db199SXin Li field_names = self._get_column_names(cursor) 71*9c5db199SXin Li row_dicts = [dict(zip(field_names, row)) for row in cursor.fetchall()] 72*9c5db199SXin Li return row_dicts 73*9c5db199SXin Li 74*9c5db199SXin Li 75*9c5db199SXin Li def get_count_sql(self, query): 76*9c5db199SXin Li """Get SQL to select a per-group count of unique matches for a query. 77*9c5db199SXin Li 78*9c5db199SXin Li @param query: The query to use. 79*9c5db199SXin Li 80*9c5db199SXin Li @return A tuple (field alias, field SQL). 81*9c5db199SXin Li 82*9c5db199SXin Li """ 83*9c5db199SXin Li if query.query.distinct: 84*9c5db199SXin Li pk_field = self.get_key_on_this_table() 85*9c5db199SXin Li count_sql = 'COUNT(DISTINCT %s)' % pk_field 86*9c5db199SXin Li else: 87*9c5db199SXin Li count_sql = 'COUNT(1)' 88*9c5db199SXin Li return self._GROUP_COUNT_NAME, count_sql 89*9c5db199SXin Li 90*9c5db199SXin Li 91*9c5db199SXin Li def _get_num_groups_sql(self, query, group_by): 92*9c5db199SXin Li group_fields = self._get_field_names(group_by, query.query.extra_select) 93*9c5db199SXin Li query = query.order_by() # this can mess up the query and isn't needed 94*9c5db199SXin Li 95*9c5db199SXin Li compiler = query.query.get_compiler(using=query.db) 96*9c5db199SXin Li sql, params = compiler.as_sql() 97*9c5db199SXin Li from_ = sql[sql.find(' FROM'):] 98*9c5db199SXin Li return ('SELECT DISTINCT %s %s' % (','.join(group_fields), 99*9c5db199SXin Li from_), 100*9c5db199SXin Li params) 101*9c5db199SXin Li 102*9c5db199SXin Li 103*9c5db199SXin Li def _cursor_rowcount(self, cursor): 104*9c5db199SXin Li """To be stubbed by tests""" 105*9c5db199SXin Li return cursor.rowcount 106*9c5db199SXin Li 107*9c5db199SXin Li 108*9c5db199SXin Li def get_num_groups(self, query, group_by): 109*9c5db199SXin Li """Gets the number of distinct groups for a query. 110*9c5db199SXin Li 111*9c5db199SXin Li @param query: The query to use. 112*9c5db199SXin Li @param group_by: The fields by which to group. 113*9c5db199SXin Li 114*9c5db199SXin Li @return The number of distinct groups for the given query grouped by 115*9c5db199SXin Li the fields in group_by. 116*9c5db199SXin Li 117*9c5db199SXin Li """ 118*9c5db199SXin Li sql, params = self._get_num_groups_sql(query, group_by) 119*9c5db199SXin Li cursor = readonly_connection.cursor() 120*9c5db199SXin Li cursor.execute(sql, params) 121*9c5db199SXin Li return self._cursor_rowcount(cursor) 122*9c5db199SXin Li 123*9c5db199SXin Li 124*9c5db199SXin Liclass Machine(dbmodels.Model): 125*9c5db199SXin Li """Models a machine.""" 126*9c5db199SXin Li machine_idx = dbmodels.AutoField(primary_key=True) 127*9c5db199SXin Li hostname = dbmodels.CharField(unique=True, max_length=255) 128*9c5db199SXin Li machine_group = dbmodels.CharField(blank=True, max_length=240) 129*9c5db199SXin Li owner = dbmodels.CharField(blank=True, max_length=240) 130*9c5db199SXin Li 131*9c5db199SXin Li class Meta: 132*9c5db199SXin Li """Metadata for class Machine.""" 133*9c5db199SXin Li db_table = 'tko_machines' 134*9c5db199SXin Li 135*9c5db199SXin Li 136*9c5db199SXin Liclass Kernel(dbmodels.Model): 137*9c5db199SXin Li """Models a kernel.""" 138*9c5db199SXin Li kernel_idx = dbmodels.AutoField(primary_key=True) 139*9c5db199SXin Li kernel_hash = dbmodels.CharField(max_length=105, editable=False) 140*9c5db199SXin Li base = dbmodels.CharField(max_length=90) 141*9c5db199SXin Li printable = dbmodels.CharField(max_length=300) 142*9c5db199SXin Li 143*9c5db199SXin Li class Meta: 144*9c5db199SXin Li """Metadata for class Kernel.""" 145*9c5db199SXin Li db_table = 'tko_kernels' 146*9c5db199SXin Li 147*9c5db199SXin Li 148*9c5db199SXin Liclass Patch(dbmodels.Model): 149*9c5db199SXin Li """Models a patch.""" 150*9c5db199SXin Li kernel = dbmodels.ForeignKey(Kernel, db_column='kernel_idx') 151*9c5db199SXin Li name = dbmodels.CharField(blank=True, max_length=240) 152*9c5db199SXin Li url = dbmodels.CharField(blank=True, max_length=900) 153*9c5db199SXin Li the_hash = dbmodels.CharField(blank=True, max_length=105, db_column='hash') 154*9c5db199SXin Li 155*9c5db199SXin Li class Meta: 156*9c5db199SXin Li """Metadata for class Patch.""" 157*9c5db199SXin Li db_table = 'tko_patches' 158*9c5db199SXin Li 159*9c5db199SXin Li 160*9c5db199SXin Liclass Status(dbmodels.Model): 161*9c5db199SXin Li """Models a status.""" 162*9c5db199SXin Li status_idx = dbmodels.AutoField(primary_key=True) 163*9c5db199SXin Li word = dbmodels.CharField(max_length=30) 164*9c5db199SXin Li 165*9c5db199SXin Li class Meta: 166*9c5db199SXin Li """Metadata for class Status.""" 167*9c5db199SXin Li db_table = 'tko_status' 168*9c5db199SXin Li 169*9c5db199SXin Li 170*9c5db199SXin Liclass Job(dbmodels.Model, model_logic.ModelExtensions): 171*9c5db199SXin Li """Models a job.""" 172*9c5db199SXin Li job_idx = dbmodels.AutoField(primary_key=True) 173*9c5db199SXin Li tag = dbmodels.CharField(unique=True, max_length=100) 174*9c5db199SXin Li label = dbmodels.CharField(max_length=300) 175*9c5db199SXin Li username = dbmodels.CharField(max_length=240) 176*9c5db199SXin Li machine = dbmodels.ForeignKey(Machine, db_column='machine_idx') 177*9c5db199SXin Li queued_time = dbmodels.DateTimeField(null=True, blank=True) 178*9c5db199SXin Li started_time = dbmodels.DateTimeField(null=True, blank=True) 179*9c5db199SXin Li finished_time = dbmodels.DateTimeField(null=True, blank=True) 180*9c5db199SXin Li afe_job_id = dbmodels.IntegerField(null=True, default=None) 181*9c5db199SXin Li 182*9c5db199SXin Li objects = model_logic.ExtendedManager() 183*9c5db199SXin Li 184*9c5db199SXin Li class Meta: 185*9c5db199SXin Li """Metadata for class Job.""" 186*9c5db199SXin Li db_table = 'tko_jobs' 187*9c5db199SXin Li 188*9c5db199SXin Li 189*9c5db199SXin Liclass JobKeyval(dbmodels.Model): 190*9c5db199SXin Li """Models a job keyval.""" 191*9c5db199SXin Li job = dbmodels.ForeignKey(Job) 192*9c5db199SXin Li key = dbmodels.CharField(max_length=90) 193*9c5db199SXin Li value = dbmodels.CharField(blank=True, max_length=300) 194*9c5db199SXin Li 195*9c5db199SXin Li class Meta: 196*9c5db199SXin Li """Metadata for class JobKeyval.""" 197*9c5db199SXin Li db_table = 'tko_job_keyvals' 198*9c5db199SXin Li 199*9c5db199SXin Li 200*9c5db199SXin Liclass Test(dbmodels.Model, model_logic.ModelExtensions, 201*9c5db199SXin Li model_logic.ModelWithAttributes): 202*9c5db199SXin Li """Models a test.""" 203*9c5db199SXin Li test_idx = dbmodels.AutoField(primary_key=True) 204*9c5db199SXin Li job = dbmodels.ForeignKey(Job, db_column='job_idx') 205*9c5db199SXin Li test = dbmodels.CharField(max_length=300) 206*9c5db199SXin Li subdir = dbmodels.CharField(blank=True, max_length=300) 207*9c5db199SXin Li kernel = dbmodels.ForeignKey(Kernel, db_column='kernel_idx') 208*9c5db199SXin Li status = dbmodels.ForeignKey(Status, db_column='status') 209*9c5db199SXin Li reason = dbmodels.CharField(blank=True, max_length=3072) 210*9c5db199SXin Li machine = dbmodels.ForeignKey(Machine, db_column='machine_idx') 211*9c5db199SXin Li finished_time = dbmodels.DateTimeField(null=True, blank=True) 212*9c5db199SXin Li started_time = dbmodels.DateTimeField(null=True, blank=True) 213*9c5db199SXin Li invalid = dbmodels.BooleanField(default=False) 214*9c5db199SXin Li invalidates_test = dbmodels.ForeignKey( 215*9c5db199SXin Li 'self', null=True, db_column='invalidates_test_idx', 216*9c5db199SXin Li related_name='invalidates_test_set') 217*9c5db199SXin Li 218*9c5db199SXin Li objects = model_logic.ExtendedManager() 219*9c5db199SXin Li 220*9c5db199SXin Li def _get_attribute_model_and_args(self, attribute): 221*9c5db199SXin Li return TestAttribute, dict(test=self, attribute=attribute, 222*9c5db199SXin Li user_created=True) 223*9c5db199SXin Li 224*9c5db199SXin Li 225*9c5db199SXin Li def set_attribute(self, attribute, value): 226*9c5db199SXin Li # ensure non-user-created attributes remain immutable 227*9c5db199SXin Li try: 228*9c5db199SXin Li TestAttribute.objects.get(test=self, attribute=attribute, 229*9c5db199SXin Li user_created=False) 230*9c5db199SXin Li raise ValueError('Attribute %s already exists for test %s and is ' 231*9c5db199SXin Li 'immutable' % (attribute, self.test_idx)) 232*9c5db199SXin Li except TestAttribute.DoesNotExist: 233*9c5db199SXin Li super(Test, self).set_attribute(attribute, value) 234*9c5db199SXin Li 235*9c5db199SXin Li class Meta: 236*9c5db199SXin Li """Metadata for class Test.""" 237*9c5db199SXin Li db_table = 'tko_tests' 238*9c5db199SXin Li 239*9c5db199SXin Li 240*9c5db199SXin Liclass TestAttribute(dbmodels.Model, model_logic.ModelExtensions): 241*9c5db199SXin Li """Models a test attribute.""" 242*9c5db199SXin Li test = dbmodels.ForeignKey(Test, db_column='test_idx') 243*9c5db199SXin Li attribute = dbmodels.CharField(max_length=90) 244*9c5db199SXin Li value = dbmodels.CharField(blank=True, max_length=300) 245*9c5db199SXin Li user_created = dbmodels.BooleanField(default=False) 246*9c5db199SXin Li 247*9c5db199SXin Li objects = model_logic.ExtendedManager() 248*9c5db199SXin Li 249*9c5db199SXin Li class Meta: 250*9c5db199SXin Li """Metadata for class TestAttribute.""" 251*9c5db199SXin Li db_table = 'tko_test_attributes' 252*9c5db199SXin Li 253*9c5db199SXin Li 254*9c5db199SXin Liclass IterationAttribute(dbmodels.Model, model_logic.ModelExtensions): 255*9c5db199SXin Li """Models an iteration attribute.""" 256*9c5db199SXin Li # This isn't really a primary key, but it's necessary to appease Django 257*9c5db199SXin Li # and is harmless as long as we're careful. 258*9c5db199SXin Li test = dbmodels.ForeignKey(Test, db_column='test_idx', primary_key=True) 259*9c5db199SXin Li iteration = dbmodels.IntegerField() 260*9c5db199SXin Li attribute = dbmodels.CharField(max_length=90) 261*9c5db199SXin Li value = dbmodels.CharField(blank=True, max_length=300) 262*9c5db199SXin Li 263*9c5db199SXin Li objects = model_logic.ExtendedManager() 264*9c5db199SXin Li 265*9c5db199SXin Li class Meta: 266*9c5db199SXin Li """Metadata for class IterationAttribute.""" 267*9c5db199SXin Li db_table = 'tko_iteration_attributes' 268*9c5db199SXin Li 269*9c5db199SXin Li 270*9c5db199SXin Liclass IterationResult(dbmodels.Model, model_logic.ModelExtensions): 271*9c5db199SXin Li """Models an iteration result.""" 272*9c5db199SXin Li # See comment on IterationAttribute regarding primary_key=True. 273*9c5db199SXin Li test = dbmodels.ForeignKey(Test, db_column='test_idx', primary_key=True) 274*9c5db199SXin Li iteration = dbmodels.IntegerField() 275*9c5db199SXin Li attribute = dbmodels.CharField(max_length=256) 276*9c5db199SXin Li value = dbmodels.FloatField(null=True, blank=True) 277*9c5db199SXin Li 278*9c5db199SXin Li objects = model_logic.ExtendedManager() 279*9c5db199SXin Li 280*9c5db199SXin Li class Meta: 281*9c5db199SXin Li """Metadata for class IterationResult.""" 282*9c5db199SXin Li db_table = 'tko_iteration_result' 283*9c5db199SXin Li 284*9c5db199SXin Li 285*9c5db199SXin Liclass TestLabel(dbmodels.Model, model_logic.ModelExtensions): 286*9c5db199SXin Li """Models a test label.""" 287*9c5db199SXin Li name = dbmodels.CharField(max_length=80, unique=True) 288*9c5db199SXin Li description = dbmodels.TextField(blank=True) 289*9c5db199SXin Li tests = dbmodels.ManyToManyField(Test, blank=True, 290*9c5db199SXin Li db_table='tko_test_labels_tests') 291*9c5db199SXin Li 292*9c5db199SXin Li name_field = 'name' 293*9c5db199SXin Li objects = model_logic.ExtendedManager() 294*9c5db199SXin Li 295*9c5db199SXin Li class Meta: 296*9c5db199SXin Li """Metadata for class TestLabel.""" 297*9c5db199SXin Li db_table = 'tko_test_labels' 298*9c5db199SXin Li 299*9c5db199SXin Li 300*9c5db199SXin Liclass SavedQuery(dbmodels.Model, model_logic.ModelExtensions): 301*9c5db199SXin Li """Models a saved query.""" 302*9c5db199SXin Li # TODO: change this to foreign key once DBs are merged. 303*9c5db199SXin Li owner = dbmodels.CharField(max_length=80) 304*9c5db199SXin Li name = dbmodels.CharField(max_length=100) 305*9c5db199SXin Li url_token = dbmodels.TextField() 306*9c5db199SXin Li 307*9c5db199SXin Li class Meta: 308*9c5db199SXin Li """Metadata for class SavedQuery.""" 309*9c5db199SXin Li db_table = 'tko_saved_queries' 310*9c5db199SXin Li 311*9c5db199SXin Li 312*9c5db199SXin Li# Views. 313*9c5db199SXin Li 314*9c5db199SXin Liclass TestViewManager(TempManager): 315*9c5db199SXin Li """A Test View Manager.""" 316*9c5db199SXin Li 317*9c5db199SXin Li def get_query_set(self): 318*9c5db199SXin Li query = super(TestViewManager, self).get_query_set() 319*9c5db199SXin Li 320*9c5db199SXin Li # add extra fields to selects, using the SQL itself as the "alias" 321*9c5db199SXin Li extra_select = dict((sql, sql) 322*9c5db199SXin Li for sql in self.model.extra_fields.iterkeys()) 323*9c5db199SXin Li return query.extra(select=extra_select) 324*9c5db199SXin Li 325*9c5db199SXin Li 326*9c5db199SXin Li def _get_include_exclude_suffix(self, exclude): 327*9c5db199SXin Li if exclude: 328*9c5db199SXin Li return '_exclude' 329*9c5db199SXin Li return '_include' 330*9c5db199SXin Li 331*9c5db199SXin Li 332*9c5db199SXin Li def _add_attribute_join(self, query_set, join_condition, 333*9c5db199SXin Li suffix=None, exclude=False): 334*9c5db199SXin Li if suffix is None: 335*9c5db199SXin Li suffix = self._get_include_exclude_suffix(exclude) 336*9c5db199SXin Li return self.add_join(query_set, 'tko_test_attributes', 337*9c5db199SXin Li join_key='test_idx', 338*9c5db199SXin Li join_condition=join_condition, 339*9c5db199SXin Li suffix=suffix, exclude=exclude) 340*9c5db199SXin Li 341*9c5db199SXin Li 342*9c5db199SXin Li def _add_label_pivot_table_join(self, query_set, suffix, join_condition='', 343*9c5db199SXin Li exclude=False, force_left_join=False): 344*9c5db199SXin Li return self.add_join(query_set, 'tko_test_labels_tests', 345*9c5db199SXin Li join_key='test_id', 346*9c5db199SXin Li join_condition=join_condition, 347*9c5db199SXin Li suffix=suffix, exclude=exclude, 348*9c5db199SXin Li force_left_join=force_left_join) 349*9c5db199SXin Li 350*9c5db199SXin Li 351*9c5db199SXin Li def _add_label_joins(self, query_set, suffix=''): 352*9c5db199SXin Li query_set = self._add_label_pivot_table_join( 353*9c5db199SXin Li query_set, suffix=suffix, force_left_join=True) 354*9c5db199SXin Li 355*9c5db199SXin Li # since we're not joining from the original table, we can't use 356*9c5db199SXin Li # self.add_join() again 357*9c5db199SXin Li second_join_alias = 'tko_test_labels' + suffix 358*9c5db199SXin Li second_join_condition = ('%s.id = %s.testlabel_id' % 359*9c5db199SXin Li (second_join_alias, 360*9c5db199SXin Li 'tko_test_labels_tests' + suffix)) 361*9c5db199SXin Li query_set.query.add_custom_join('tko_test_labels', 362*9c5db199SXin Li second_join_condition, 363*9c5db199SXin Li query_set.query.LOUTER, 364*9c5db199SXin Li alias=second_join_alias) 365*9c5db199SXin Li return query_set 366*9c5db199SXin Li 367*9c5db199SXin Li 368*9c5db199SXin Li def _get_label_ids_from_names(self, label_names): 369*9c5db199SXin Li label_ids = list( # listifying avoids a double query below 370*9c5db199SXin Li TestLabel.objects.filter(name__in=label_names) 371*9c5db199SXin Li .values_list('name', 'id')) 372*9c5db199SXin Li if len(label_ids) < len(set(label_names)): 373*9c5db199SXin Li raise ValueError('Not all labels found: %s' % 374*9c5db199SXin Li ', '.join(label_names)) 375*9c5db199SXin Li return dict(name_and_id for name_and_id in label_ids) 376*9c5db199SXin Li 377*9c5db199SXin Li 378*9c5db199SXin Li def _include_or_exclude_labels(self, query_set, label_names, exclude=False): 379*9c5db199SXin Li label_ids = self._get_label_ids_from_names(label_names).itervalues() 380*9c5db199SXin Li suffix = self._get_include_exclude_suffix(exclude) 381*9c5db199SXin Li condition = ('tko_test_labels_tests%s.testlabel_id IN (%s)' % 382*9c5db199SXin Li (suffix, 383*9c5db199SXin Li ','.join(str(label_id) for label_id in label_ids))) 384*9c5db199SXin Li return self._add_label_pivot_table_join(query_set, 385*9c5db199SXin Li join_condition=condition, 386*9c5db199SXin Li suffix=suffix, 387*9c5db199SXin Li exclude=exclude) 388*9c5db199SXin Li 389*9c5db199SXin Li 390*9c5db199SXin Li def _add_custom_select(self, query_set, select_name, select_sql): 391*9c5db199SXin Li return query_set.extra(select={select_name: select_sql}) 392*9c5db199SXin Li 393*9c5db199SXin Li 394*9c5db199SXin Li def _add_select_value(self, query_set, alias): 395*9c5db199SXin Li return self._add_custom_select(query_set, alias, 396*9c5db199SXin Li _quote_name(alias) + '.value') 397*9c5db199SXin Li 398*9c5db199SXin Li 399*9c5db199SXin Li def _add_select_ifnull(self, query_set, alias, non_null_value): 400*9c5db199SXin Li select_sql = "IF(%s.id IS NOT NULL, '%s', NULL)" % (_quote_name(alias), 401*9c5db199SXin Li non_null_value) 402*9c5db199SXin Li return self._add_custom_select(query_set, alias, select_sql) 403*9c5db199SXin Li 404*9c5db199SXin Li 405*9c5db199SXin Li def _join_test_label_column(self, query_set, label_name, label_id): 406*9c5db199SXin Li alias = 'test_label_' + label_name 407*9c5db199SXin Li label_query = TestLabel.objects.filter(name=label_name) 408*9c5db199SXin Li query_set = Test.objects.join_custom_field(query_set, label_query, 409*9c5db199SXin Li alias) 410*9c5db199SXin Li 411*9c5db199SXin Li query_set = self._add_select_ifnull(query_set, alias, label_name) 412*9c5db199SXin Li return query_set 413*9c5db199SXin Li 414*9c5db199SXin Li 415*9c5db199SXin Li def _join_test_label_columns(self, query_set, label_names): 416*9c5db199SXin Li label_id_map = self._get_label_ids_from_names(label_names) 417*9c5db199SXin Li for label_name in label_names: 418*9c5db199SXin Li query_set = self._join_test_label_column(query_set, label_name, 419*9c5db199SXin Li label_id_map[label_name]) 420*9c5db199SXin Li return query_set 421*9c5db199SXin Li 422*9c5db199SXin Li 423*9c5db199SXin Li def _join_test_attribute(self, query_set, attribute, alias=None, 424*9c5db199SXin Li extra_join_condition=None): 425*9c5db199SXin Li """ 426*9c5db199SXin Li Join the given TestView QuerySet to TestAttribute. The resulting query 427*9c5db199SXin Li has an additional column for the given attribute named 428*9c5db199SXin Li "attribute_<attribute name>". 429*9c5db199SXin Li """ 430*9c5db199SXin Li if not alias: 431*9c5db199SXin Li alias = 'test_attribute_' + attribute 432*9c5db199SXin Li attribute_query = TestAttribute.objects.filter(attribute=attribute) 433*9c5db199SXin Li if extra_join_condition: 434*9c5db199SXin Li attribute_query = attribute_query.extra( 435*9c5db199SXin Li where=[extra_join_condition]) 436*9c5db199SXin Li query_set = Test.objects.join_custom_field(query_set, attribute_query, 437*9c5db199SXin Li alias) 438*9c5db199SXin Li 439*9c5db199SXin Li query_set = self._add_select_value(query_set, alias) 440*9c5db199SXin Li return query_set 441*9c5db199SXin Li 442*9c5db199SXin Li 443*9c5db199SXin Li def _join_machine_label_columns(self, query_set, machine_label_names): 444*9c5db199SXin Li for label_name in machine_label_names: 445*9c5db199SXin Li alias = 'machine_label_' + label_name 446*9c5db199SXin Li condition = "FIND_IN_SET('%s', %s)" % ( 447*9c5db199SXin Li label_name, _quote_name(alias) + '.value') 448*9c5db199SXin Li query_set = self._join_test_attribute( 449*9c5db199SXin Li query_set, 'host-labels', 450*9c5db199SXin Li alias=alias, extra_join_condition=condition) 451*9c5db199SXin Li query_set = self._add_select_ifnull(query_set, alias, label_name) 452*9c5db199SXin Li return query_set 453*9c5db199SXin Li 454*9c5db199SXin Li 455*9c5db199SXin Li def _join_one_iteration_key(self, query_set, result_key, first_alias=None): 456*9c5db199SXin Li alias = 'iteration_result_' + result_key 457*9c5db199SXin Li iteration_query = IterationResult.objects.filter(attribute=result_key) 458*9c5db199SXin Li if first_alias: 459*9c5db199SXin Li # after the first join, we need to match up iteration indices, 460*9c5db199SXin Li # otherwise each join will expand the query by the number of 461*9c5db199SXin Li # iterations and we'll have extraneous rows 462*9c5db199SXin Li iteration_query = iteration_query.extra( 463*9c5db199SXin Li where=['%s.iteration = %s.iteration' 464*9c5db199SXin Li % (_quote_name(alias), _quote_name(first_alias))]) 465*9c5db199SXin Li 466*9c5db199SXin Li query_set = Test.objects.join_custom_field(query_set, iteration_query, 467*9c5db199SXin Li alias, left_join=False) 468*9c5db199SXin Li # select the iteration value and index for this join 469*9c5db199SXin Li query_set = self._add_select_value(query_set, alias) 470*9c5db199SXin Li if not first_alias: 471*9c5db199SXin Li # for first join, add iteration index select too 472*9c5db199SXin Li query_set = self._add_custom_select( 473*9c5db199SXin Li query_set, 'iteration_index', 474*9c5db199SXin Li _quote_name(alias) + '.iteration') 475*9c5db199SXin Li 476*9c5db199SXin Li return query_set, alias 477*9c5db199SXin Li 478*9c5db199SXin Li 479*9c5db199SXin Li def _join_iteration_results(self, test_view_query_set, result_keys): 480*9c5db199SXin Li """Join the given TestView QuerySet to IterationResult for one result. 481*9c5db199SXin Li 482*9c5db199SXin Li The resulting query looks like a TestView query but has one row per 483*9c5db199SXin Li iteration. Each row includes all the attributes of TestView, an 484*9c5db199SXin Li attribute for each key in result_keys and an iteration_index attribute. 485*9c5db199SXin Li 486*9c5db199SXin Li We accomplish this by joining the TestView query to IterationResult 487*9c5db199SXin Li once per result key. Each join is restricted on the result key (and on 488*9c5db199SXin Li the test index, like all one-to-many joins). For the first join, this 489*9c5db199SXin Li is the only restriction, so each TestView row expands to a row per 490*9c5db199SXin Li iteration (per iteration that includes the key, of course). For each 491*9c5db199SXin Li subsequent join, we also restrict the iteration index to match that of 492*9c5db199SXin Li the initial join. This makes each subsequent join produce exactly one 493*9c5db199SXin Li result row for each input row. (This assumes each iteration contains 494*9c5db199SXin Li the same set of keys. Results are undefined if that's not true.) 495*9c5db199SXin Li """ 496*9c5db199SXin Li if not result_keys: 497*9c5db199SXin Li return test_view_query_set 498*9c5db199SXin Li 499*9c5db199SXin Li query_set, first_alias = self._join_one_iteration_key( 500*9c5db199SXin Li test_view_query_set, result_keys[0]) 501*9c5db199SXin Li for result_key in result_keys[1:]: 502*9c5db199SXin Li query_set, _ = self._join_one_iteration_key(query_set, result_key, 503*9c5db199SXin Li first_alias=first_alias) 504*9c5db199SXin Li return query_set 505*9c5db199SXin Li 506*9c5db199SXin Li 507*9c5db199SXin Li def _join_job_keyvals(self, query_set, job_keyvals): 508*9c5db199SXin Li for job_keyval in job_keyvals: 509*9c5db199SXin Li alias = 'job_keyval_' + job_keyval 510*9c5db199SXin Li keyval_query = JobKeyval.objects.filter(key=job_keyval) 511*9c5db199SXin Li query_set = Job.objects.join_custom_field(query_set, keyval_query, 512*9c5db199SXin Li alias) 513*9c5db199SXin Li query_set = self._add_select_value(query_set, alias) 514*9c5db199SXin Li return query_set 515*9c5db199SXin Li 516*9c5db199SXin Li 517*9c5db199SXin Li def _join_iteration_attributes(self, query_set, iteration_attributes): 518*9c5db199SXin Li for attribute in iteration_attributes: 519*9c5db199SXin Li alias = 'iteration_attribute_' + attribute 520*9c5db199SXin Li attribute_query = IterationAttribute.objects.filter( 521*9c5db199SXin Li attribute=attribute) 522*9c5db199SXin Li query_set = Test.objects.join_custom_field(query_set, 523*9c5db199SXin Li attribute_query, alias) 524*9c5db199SXin Li query_set = self._add_select_value(query_set, alias) 525*9c5db199SXin Li return query_set 526*9c5db199SXin Li 527*9c5db199SXin Li 528*9c5db199SXin Li def get_query_set_with_joins(self, filter_data): 529*9c5db199SXin Li """Add joins for querying over test-related items. 530*9c5db199SXin Li 531*9c5db199SXin Li These parameters are supported going forward: 532*9c5db199SXin Li * test_attribute_fields: list of attribute names. Each attribute will 533*9c5db199SXin Li be available as a column attribute_<name>.value. 534*9c5db199SXin Li * test_label_fields: list of label names. Each label will be available 535*9c5db199SXin Li as a column label_<name>.id, non-null iff the label is present. 536*9c5db199SXin Li * iteration_result_fields: list of iteration result names. Each 537*9c5db199SXin Li result will be available as a column iteration_<name>.value. 538*9c5db199SXin Li Note that this changes the semantics to return iterations 539*9c5db199SXin Li instead of tests -- if a test has multiple iterations, a row 540*9c5db199SXin Li will be returned for each one. The iteration index is also 541*9c5db199SXin Li available as iteration_<name>.iteration. 542*9c5db199SXin Li * machine_label_fields: list of machine label names. Each will be 543*9c5db199SXin Li available as a column machine_label_<name>.id, non-null iff the 544*9c5db199SXin Li label is present on the machine used in the test. 545*9c5db199SXin Li * job_keyval_fields: list of job keyval names. Each value will be 546*9c5db199SXin Li available as a column job_keyval_<name>.id, non-null iff the 547*9c5db199SXin Li keyval is present in the AFE job. 548*9c5db199SXin Li * iteration_attribute_fields: list of iteration attribute names. Each 549*9c5db199SXin Li attribute will be available as a column 550*9c5db199SXin Li iteration_attribute<name>.id, non-null iff the attribute is 551*9c5db199SXin Li present. 552*9c5db199SXin Li 553*9c5db199SXin Li These parameters are deprecated: 554*9c5db199SXin Li * include_labels 555*9c5db199SXin Li * exclude_labels 556*9c5db199SXin Li * include_attributes_where 557*9c5db199SXin Li * exclude_attributes_where 558*9c5db199SXin Li 559*9c5db199SXin Li Additionally, this method adds joins if the following strings are 560*9c5db199SXin Li present in extra_where (this is also deprecated): 561*9c5db199SXin Li * test_labels 562*9c5db199SXin Li * test_attributes_host_labels 563*9c5db199SXin Li 564*9c5db199SXin Li @param filter_data: Data by which to filter. 565*9c5db199SXin Li 566*9c5db199SXin Li @return A QuerySet. 567*9c5db199SXin Li 568*9c5db199SXin Li """ 569*9c5db199SXin Li query_set = self.get_query_set() 570*9c5db199SXin Li 571*9c5db199SXin Li test_attributes = filter_data.pop('test_attribute_fields', []) 572*9c5db199SXin Li for attribute in test_attributes: 573*9c5db199SXin Li query_set = self._join_test_attribute(query_set, attribute) 574*9c5db199SXin Li 575*9c5db199SXin Li test_labels = filter_data.pop('test_label_fields', []) 576*9c5db199SXin Li query_set = self._join_test_label_columns(query_set, test_labels) 577*9c5db199SXin Li 578*9c5db199SXin Li machine_labels = filter_data.pop('machine_label_fields', []) 579*9c5db199SXin Li query_set = self._join_machine_label_columns(query_set, machine_labels) 580*9c5db199SXin Li 581*9c5db199SXin Li iteration_keys = filter_data.pop('iteration_result_fields', []) 582*9c5db199SXin Li query_set = self._join_iteration_results(query_set, iteration_keys) 583*9c5db199SXin Li 584*9c5db199SXin Li job_keyvals = filter_data.pop('job_keyval_fields', []) 585*9c5db199SXin Li query_set = self._join_job_keyvals(query_set, job_keyvals) 586*9c5db199SXin Li 587*9c5db199SXin Li iteration_attributes = filter_data.pop('iteration_attribute_fields', []) 588*9c5db199SXin Li query_set = self._join_iteration_attributes(query_set, 589*9c5db199SXin Li iteration_attributes) 590*9c5db199SXin Li 591*9c5db199SXin Li # everything that follows is deprecated behavior 592*9c5db199SXin Li 593*9c5db199SXin Li joined = False 594*9c5db199SXin Li 595*9c5db199SXin Li extra_where = filter_data.get('extra_where', '') 596*9c5db199SXin Li if 'tko_test_labels' in extra_where: 597*9c5db199SXin Li query_set = self._add_label_joins(query_set) 598*9c5db199SXin Li joined = True 599*9c5db199SXin Li 600*9c5db199SXin Li include_labels = filter_data.pop('include_labels', []) 601*9c5db199SXin Li exclude_labels = filter_data.pop('exclude_labels', []) 602*9c5db199SXin Li if include_labels: 603*9c5db199SXin Li query_set = self._include_or_exclude_labels(query_set, 604*9c5db199SXin Li include_labels) 605*9c5db199SXin Li joined = True 606*9c5db199SXin Li if exclude_labels: 607*9c5db199SXin Li query_set = self._include_or_exclude_labels(query_set, 608*9c5db199SXin Li exclude_labels, 609*9c5db199SXin Li exclude=True) 610*9c5db199SXin Li joined = True 611*9c5db199SXin Li 612*9c5db199SXin Li include_attributes_where = filter_data.pop('include_attributes_where', 613*9c5db199SXin Li '') 614*9c5db199SXin Li exclude_attributes_where = filter_data.pop('exclude_attributes_where', 615*9c5db199SXin Li '') 616*9c5db199SXin Li if include_attributes_where: 617*9c5db199SXin Li query_set = self._add_attribute_join( 618*9c5db199SXin Li query_set, 619*9c5db199SXin Li join_condition=self.escape_user_sql(include_attributes_where)) 620*9c5db199SXin Li joined = True 621*9c5db199SXin Li if exclude_attributes_where: 622*9c5db199SXin Li query_set = self._add_attribute_join( 623*9c5db199SXin Li query_set, 624*9c5db199SXin Li join_condition=self.escape_user_sql(exclude_attributes_where), 625*9c5db199SXin Li exclude=True) 626*9c5db199SXin Li joined = True 627*9c5db199SXin Li 628*9c5db199SXin Li if not joined: 629*9c5db199SXin Li filter_data['no_distinct'] = True 630*9c5db199SXin Li 631*9c5db199SXin Li if 'tko_test_attributes_host_labels' in extra_where: 632*9c5db199SXin Li query_set = self._add_attribute_join( 633*9c5db199SXin Li query_set, suffix='_host_labels', 634*9c5db199SXin Li join_condition='tko_test_attributes_host_labels.attribute = ' 635*9c5db199SXin Li '"host-labels"') 636*9c5db199SXin Li 637*9c5db199SXin Li return query_set 638*9c5db199SXin Li 639*9c5db199SXin Li 640*9c5db199SXin Li def query_test_ids(self, filter_data, apply_presentation=True): 641*9c5db199SXin Li """Queries for test IDs. 642*9c5db199SXin Li 643*9c5db199SXin Li @param filter_data: Data by which to filter. 644*9c5db199SXin Li @param apply_presentation: Whether or not to apply presentation 645*9c5db199SXin Li parameters. 646*9c5db199SXin Li 647*9c5db199SXin Li @return A list of test IDs. 648*9c5db199SXin Li 649*9c5db199SXin Li """ 650*9c5db199SXin Li query = self.model.query_objects(filter_data, 651*9c5db199SXin Li apply_presentation=apply_presentation) 652*9c5db199SXin Li dicts = query.values('test_idx') 653*9c5db199SXin Li return [item['test_idx'] for item in dicts] 654*9c5db199SXin Li 655*9c5db199SXin Li 656*9c5db199SXin Li def query_test_label_ids(self, filter_data): 657*9c5db199SXin Li """Queries for test label IDs. 658*9c5db199SXin Li 659*9c5db199SXin Li @param filter_data: Data by which to filter. 660*9c5db199SXin Li 661*9c5db199SXin Li @return A list of test label IDs. 662*9c5db199SXin Li 663*9c5db199SXin Li """ 664*9c5db199SXin Li query_set = self.model.query_objects(filter_data) 665*9c5db199SXin Li query_set = self._add_label_joins(query_set, suffix='_list') 666*9c5db199SXin Li rows = self._custom_select_query(query_set, ['tko_test_labels_list.id']) 667*9c5db199SXin Li return [row[0] for row in rows if row[0] is not None] 668*9c5db199SXin Li 669*9c5db199SXin Li 670*9c5db199SXin Li def escape_user_sql(self, sql): 671*9c5db199SXin Li sql = super(TestViewManager, self).escape_user_sql(sql) 672*9c5db199SXin Li return sql.replace('test_idx', self.get_key_on_this_table('test_idx')) 673*9c5db199SXin Li 674*9c5db199SXin Li 675*9c5db199SXin Liclass TestView(dbmodels.Model, model_logic.ModelExtensions): 676*9c5db199SXin Li """Models a test view.""" 677*9c5db199SXin Li extra_fields = { 678*9c5db199SXin Li 'DATE(job_queued_time)': 'job queued day', 679*9c5db199SXin Li 'DATE(test_finished_time)': 'test finished day', 680*9c5db199SXin Li } 681*9c5db199SXin Li 682*9c5db199SXin Li group_fields = [ 683*9c5db199SXin Li 'test_name', 684*9c5db199SXin Li 'status', 685*9c5db199SXin Li 'kernel', 686*9c5db199SXin Li 'hostname', 687*9c5db199SXin Li 'job_tag', 688*9c5db199SXin Li 'job_name', 689*9c5db199SXin Li 'platform', 690*9c5db199SXin Li 'reason', 691*9c5db199SXin Li 'job_owner', 692*9c5db199SXin Li 'job_queued_time', 693*9c5db199SXin Li 'DATE(job_queued_time)', 694*9c5db199SXin Li 'test_started_time', 695*9c5db199SXin Li 'test_finished_time', 696*9c5db199SXin Li 'DATE(test_finished_time)', 697*9c5db199SXin Li ] 698*9c5db199SXin Li 699*9c5db199SXin Li test_idx = dbmodels.IntegerField('test index', primary_key=True) 700*9c5db199SXin Li job_idx = dbmodels.IntegerField('job index', null=True, blank=True) 701*9c5db199SXin Li test_name = dbmodels.CharField(blank=True, max_length=90) 702*9c5db199SXin Li subdir = dbmodels.CharField('subdirectory', blank=True, max_length=180) 703*9c5db199SXin Li kernel_idx = dbmodels.IntegerField('kernel index') 704*9c5db199SXin Li status_idx = dbmodels.IntegerField('status index') 705*9c5db199SXin Li reason = dbmodels.CharField(blank=True, max_length=3072) 706*9c5db199SXin Li machine_idx = dbmodels.IntegerField('host index') 707*9c5db199SXin Li test_started_time = dbmodels.DateTimeField(null=True, blank=True) 708*9c5db199SXin Li test_finished_time = dbmodels.DateTimeField(null=True, blank=True) 709*9c5db199SXin Li job_tag = dbmodels.CharField(blank=True, max_length=300) 710*9c5db199SXin Li job_name = dbmodels.CharField(blank=True, max_length=300) 711*9c5db199SXin Li job_owner = dbmodels.CharField('owner', blank=True, max_length=240) 712*9c5db199SXin Li job_queued_time = dbmodels.DateTimeField(null=True, blank=True) 713*9c5db199SXin Li job_started_time = dbmodels.DateTimeField(null=True, blank=True) 714*9c5db199SXin Li job_finished_time = dbmodels.DateTimeField(null=True, blank=True) 715*9c5db199SXin Li afe_job_id = dbmodels.IntegerField(null=True) 716*9c5db199SXin Li hostname = dbmodels.CharField(blank=True, max_length=300) 717*9c5db199SXin Li platform = dbmodels.CharField(blank=True, max_length=240) 718*9c5db199SXin Li machine_owner = dbmodels.CharField(blank=True, max_length=240) 719*9c5db199SXin Li kernel_hash = dbmodels.CharField(blank=True, max_length=105) 720*9c5db199SXin Li kernel_base = dbmodels.CharField(blank=True, max_length=90) 721*9c5db199SXin Li kernel = dbmodels.CharField(blank=True, max_length=300) 722*9c5db199SXin Li status = dbmodels.CharField(blank=True, max_length=30) 723*9c5db199SXin Li invalid = dbmodels.BooleanField(blank=True) 724*9c5db199SXin Li invalidates_test_idx = dbmodels.IntegerField(null=True, blank=True) 725*9c5db199SXin Li 726*9c5db199SXin Li objects = TestViewManager() 727*9c5db199SXin Li 728*9c5db199SXin Li def save(self): 729*9c5db199SXin Li raise NotImplementedError('TestView is read-only') 730*9c5db199SXin Li 731*9c5db199SXin Li 732*9c5db199SXin Li def delete(self): 733*9c5db199SXin Li raise NotImplementedError('TestView is read-only') 734*9c5db199SXin Li 735*9c5db199SXin Li 736*9c5db199SXin Li @classmethod 737*9c5db199SXin Li def query_objects(cls, filter_data, initial_query=None, 738*9c5db199SXin Li apply_presentation=True): 739*9c5db199SXin Li if initial_query is None: 740*9c5db199SXin Li initial_query = cls.objects.get_query_set_with_joins(filter_data) 741*9c5db199SXin Li return super(TestView, cls).query_objects( 742*9c5db199SXin Li filter_data, initial_query=initial_query, 743*9c5db199SXin Li apply_presentation=apply_presentation) 744*9c5db199SXin Li 745*9c5db199SXin Li class Meta: 746*9c5db199SXin Li """Metadata for class TestView.""" 747*9c5db199SXin Li db_table = 'tko_test_view_2' 748