xref: /aosp_15_r20/external/autotest/frontend/tko/models.py (revision 9c5db1993ded3edbeafc8092d69fe5de2ee02df7)
1*9c5db199SXin Lifrom django.db import models as dbmodels, connection
2*9c5db199SXin Lifrom autotest_lib.frontend.afe import model_logic, readonly_connection
3*9c5db199SXin Li
4*9c5db199SXin Li_quote_name = connection.ops.quote_name
5*9c5db199SXin Li
6*9c5db199SXin Liclass TempManager(model_logic.ExtendedManager):
7*9c5db199SXin Li    """A Temp Manager."""
8*9c5db199SXin Li    _GROUP_COUNT_NAME = 'group_count'
9*9c5db199SXin Li
10*9c5db199SXin Li    def _get_key_unless_is_function(self, field):
11*9c5db199SXin Li        if '(' in field:
12*9c5db199SXin Li            return field
13*9c5db199SXin Li        return self.get_key_on_this_table(field)
14*9c5db199SXin Li
15*9c5db199SXin Li
16*9c5db199SXin Li    def _get_field_names(self, fields, extra_select_fields={}):
17*9c5db199SXin Li        field_names = []
18*9c5db199SXin Li        for field in fields:
19*9c5db199SXin Li            if field in extra_select_fields:
20*9c5db199SXin Li                field_names.append(extra_select_fields[field][0])
21*9c5db199SXin Li            else:
22*9c5db199SXin Li                field_names.append(self._get_key_unless_is_function(field))
23*9c5db199SXin Li        return field_names
24*9c5db199SXin Li
25*9c5db199SXin Li
26*9c5db199SXin Li    def _get_group_query_sql(self, query, group_by):
27*9c5db199SXin Li        compiler = query.query.get_compiler(using=query.db)
28*9c5db199SXin Li        sql, params = compiler.as_sql()
29*9c5db199SXin Li
30*9c5db199SXin Li
31*9c5db199SXin Li        # insert GROUP BY clause into query
32*9c5db199SXin Li        group_fields = self._get_field_names(group_by, query.query.extra_select)
33*9c5db199SXin Li        group_by_clause = ' GROUP BY ' + ', '.join(group_fields)
34*9c5db199SXin Li        group_by_position = sql.rfind('ORDER BY')
35*9c5db199SXin Li        if group_by_position == -1:
36*9c5db199SXin Li            group_by_position = len(sql)
37*9c5db199SXin Li        sql = (sql[:group_by_position] +
38*9c5db199SXin Li               group_by_clause + ' ' +
39*9c5db199SXin Li               sql[group_by_position:])
40*9c5db199SXin Li
41*9c5db199SXin Li        return sql, params
42*9c5db199SXin Li
43*9c5db199SXin Li
44*9c5db199SXin Li    def _get_column_names(self, cursor):
45*9c5db199SXin Li        """Gets the column names from the cursor description.
46*9c5db199SXin Li
47*9c5db199SXin Li        This method exists so that it can be mocked in the unit test for
48*9c5db199SXin Li        sqlite3 compatibility.
49*9c5db199SXin Li
50*9c5db199SXin Li        """
51*9c5db199SXin Li        return [column_info[0] for column_info in cursor.description]
52*9c5db199SXin Li
53*9c5db199SXin Li
54*9c5db199SXin Li    def execute_group_query(self, query, group_by):
55*9c5db199SXin Li        """Performs the given query grouped by the specified fields.
56*9c5db199SXin Li
57*9c5db199SXin Li        The given query's extra select fields are added.
58*9c5db199SXin Li
59*9c5db199SXin Li        @param query: The query to perform.
60*9c5db199SXin Li        @param group_by: The fields by which to group.
61*9c5db199SXin Li
62*9c5db199SXin Li        @return A list of dicts, where each dict corresponds to single row and
63*9c5db199SXin Li            contains a key for each grouped field as well as all of the extra
64*9c5db199SXin Li            select fields.
65*9c5db199SXin Li
66*9c5db199SXin Li        """
67*9c5db199SXin Li        sql, params = self._get_group_query_sql(query, group_by)
68*9c5db199SXin Li        cursor = readonly_connection.cursor()
69*9c5db199SXin Li        cursor.execute(sql, params)
70*9c5db199SXin Li        field_names = self._get_column_names(cursor)
71*9c5db199SXin Li        row_dicts = [dict(zip(field_names, row)) for row in cursor.fetchall()]
72*9c5db199SXin Li        return row_dicts
73*9c5db199SXin Li
74*9c5db199SXin Li
75*9c5db199SXin Li    def get_count_sql(self, query):
76*9c5db199SXin Li        """Get SQL to select a per-group count of unique matches for a query.
77*9c5db199SXin Li
78*9c5db199SXin Li        @param query: The query to use.
79*9c5db199SXin Li
80*9c5db199SXin Li        @return A tuple (field alias, field SQL).
81*9c5db199SXin Li
82*9c5db199SXin Li        """
83*9c5db199SXin Li        if query.query.distinct:
84*9c5db199SXin Li            pk_field = self.get_key_on_this_table()
85*9c5db199SXin Li            count_sql = 'COUNT(DISTINCT %s)' % pk_field
86*9c5db199SXin Li        else:
87*9c5db199SXin Li            count_sql = 'COUNT(1)'
88*9c5db199SXin Li        return self._GROUP_COUNT_NAME, count_sql
89*9c5db199SXin Li
90*9c5db199SXin Li
91*9c5db199SXin Li    def _get_num_groups_sql(self, query, group_by):
92*9c5db199SXin Li        group_fields = self._get_field_names(group_by, query.query.extra_select)
93*9c5db199SXin Li        query = query.order_by() # this can mess up the query and isn't needed
94*9c5db199SXin Li
95*9c5db199SXin Li        compiler = query.query.get_compiler(using=query.db)
96*9c5db199SXin Li        sql, params = compiler.as_sql()
97*9c5db199SXin Li        from_ = sql[sql.find(' FROM'):]
98*9c5db199SXin Li        return ('SELECT DISTINCT %s %s' % (','.join(group_fields),
99*9c5db199SXin Li                                                  from_),
100*9c5db199SXin Li                params)
101*9c5db199SXin Li
102*9c5db199SXin Li
103*9c5db199SXin Li    def _cursor_rowcount(self, cursor):
104*9c5db199SXin Li        """To be stubbed by tests"""
105*9c5db199SXin Li        return cursor.rowcount
106*9c5db199SXin Li
107*9c5db199SXin Li
108*9c5db199SXin Li    def get_num_groups(self, query, group_by):
109*9c5db199SXin Li        """Gets the number of distinct groups for a query.
110*9c5db199SXin Li
111*9c5db199SXin Li        @param query: The query to use.
112*9c5db199SXin Li        @param group_by: The fields by which to group.
113*9c5db199SXin Li
114*9c5db199SXin Li        @return The number of distinct groups for the given query grouped by
115*9c5db199SXin Li            the fields in group_by.
116*9c5db199SXin Li
117*9c5db199SXin Li        """
118*9c5db199SXin Li        sql, params = self._get_num_groups_sql(query, group_by)
119*9c5db199SXin Li        cursor = readonly_connection.cursor()
120*9c5db199SXin Li        cursor.execute(sql, params)
121*9c5db199SXin Li        return self._cursor_rowcount(cursor)
122*9c5db199SXin Li
123*9c5db199SXin Li
124*9c5db199SXin Liclass Machine(dbmodels.Model):
125*9c5db199SXin Li    """Models a machine."""
126*9c5db199SXin Li    machine_idx = dbmodels.AutoField(primary_key=True)
127*9c5db199SXin Li    hostname = dbmodels.CharField(unique=True, max_length=255)
128*9c5db199SXin Li    machine_group = dbmodels.CharField(blank=True, max_length=240)
129*9c5db199SXin Li    owner = dbmodels.CharField(blank=True, max_length=240)
130*9c5db199SXin Li
131*9c5db199SXin Li    class Meta:
132*9c5db199SXin Li        """Metadata for class Machine."""
133*9c5db199SXin Li        db_table = 'tko_machines'
134*9c5db199SXin Li
135*9c5db199SXin Li
136*9c5db199SXin Liclass Kernel(dbmodels.Model):
137*9c5db199SXin Li    """Models a kernel."""
138*9c5db199SXin Li    kernel_idx = dbmodels.AutoField(primary_key=True)
139*9c5db199SXin Li    kernel_hash = dbmodels.CharField(max_length=105, editable=False)
140*9c5db199SXin Li    base = dbmodels.CharField(max_length=90)
141*9c5db199SXin Li    printable = dbmodels.CharField(max_length=300)
142*9c5db199SXin Li
143*9c5db199SXin Li    class Meta:
144*9c5db199SXin Li        """Metadata for class Kernel."""
145*9c5db199SXin Li        db_table = 'tko_kernels'
146*9c5db199SXin Li
147*9c5db199SXin Li
148*9c5db199SXin Liclass Patch(dbmodels.Model):
149*9c5db199SXin Li    """Models a patch."""
150*9c5db199SXin Li    kernel = dbmodels.ForeignKey(Kernel, db_column='kernel_idx')
151*9c5db199SXin Li    name = dbmodels.CharField(blank=True, max_length=240)
152*9c5db199SXin Li    url = dbmodels.CharField(blank=True, max_length=900)
153*9c5db199SXin Li    the_hash = dbmodels.CharField(blank=True, max_length=105, db_column='hash')
154*9c5db199SXin Li
155*9c5db199SXin Li    class Meta:
156*9c5db199SXin Li        """Metadata for class Patch."""
157*9c5db199SXin Li        db_table = 'tko_patches'
158*9c5db199SXin Li
159*9c5db199SXin Li
160*9c5db199SXin Liclass Status(dbmodels.Model):
161*9c5db199SXin Li    """Models a status."""
162*9c5db199SXin Li    status_idx = dbmodels.AutoField(primary_key=True)
163*9c5db199SXin Li    word = dbmodels.CharField(max_length=30)
164*9c5db199SXin Li
165*9c5db199SXin Li    class Meta:
166*9c5db199SXin Li        """Metadata for class Status."""
167*9c5db199SXin Li        db_table = 'tko_status'
168*9c5db199SXin Li
169*9c5db199SXin Li
170*9c5db199SXin Liclass Job(dbmodels.Model, model_logic.ModelExtensions):
171*9c5db199SXin Li    """Models a job."""
172*9c5db199SXin Li    job_idx = dbmodels.AutoField(primary_key=True)
173*9c5db199SXin Li    tag = dbmodels.CharField(unique=True, max_length=100)
174*9c5db199SXin Li    label = dbmodels.CharField(max_length=300)
175*9c5db199SXin Li    username = dbmodels.CharField(max_length=240)
176*9c5db199SXin Li    machine = dbmodels.ForeignKey(Machine, db_column='machine_idx')
177*9c5db199SXin Li    queued_time = dbmodels.DateTimeField(null=True, blank=True)
178*9c5db199SXin Li    started_time = dbmodels.DateTimeField(null=True, blank=True)
179*9c5db199SXin Li    finished_time = dbmodels.DateTimeField(null=True, blank=True)
180*9c5db199SXin Li    afe_job_id = dbmodels.IntegerField(null=True, default=None)
181*9c5db199SXin Li
182*9c5db199SXin Li    objects = model_logic.ExtendedManager()
183*9c5db199SXin Li
184*9c5db199SXin Li    class Meta:
185*9c5db199SXin Li        """Metadata for class Job."""
186*9c5db199SXin Li        db_table = 'tko_jobs'
187*9c5db199SXin Li
188*9c5db199SXin Li
189*9c5db199SXin Liclass JobKeyval(dbmodels.Model):
190*9c5db199SXin Li    """Models a job keyval."""
191*9c5db199SXin Li    job = dbmodels.ForeignKey(Job)
192*9c5db199SXin Li    key = dbmodels.CharField(max_length=90)
193*9c5db199SXin Li    value = dbmodels.CharField(blank=True, max_length=300)
194*9c5db199SXin Li
195*9c5db199SXin Li    class Meta:
196*9c5db199SXin Li        """Metadata for class JobKeyval."""
197*9c5db199SXin Li        db_table = 'tko_job_keyvals'
198*9c5db199SXin Li
199*9c5db199SXin Li
200*9c5db199SXin Liclass Test(dbmodels.Model, model_logic.ModelExtensions,
201*9c5db199SXin Li           model_logic.ModelWithAttributes):
202*9c5db199SXin Li    """Models a test."""
203*9c5db199SXin Li    test_idx = dbmodels.AutoField(primary_key=True)
204*9c5db199SXin Li    job = dbmodels.ForeignKey(Job, db_column='job_idx')
205*9c5db199SXin Li    test = dbmodels.CharField(max_length=300)
206*9c5db199SXin Li    subdir = dbmodels.CharField(blank=True, max_length=300)
207*9c5db199SXin Li    kernel = dbmodels.ForeignKey(Kernel, db_column='kernel_idx')
208*9c5db199SXin Li    status = dbmodels.ForeignKey(Status, db_column='status')
209*9c5db199SXin Li    reason = dbmodels.CharField(blank=True, max_length=3072)
210*9c5db199SXin Li    machine = dbmodels.ForeignKey(Machine, db_column='machine_idx')
211*9c5db199SXin Li    finished_time = dbmodels.DateTimeField(null=True, blank=True)
212*9c5db199SXin Li    started_time = dbmodels.DateTimeField(null=True, blank=True)
213*9c5db199SXin Li    invalid = dbmodels.BooleanField(default=False)
214*9c5db199SXin Li    invalidates_test = dbmodels.ForeignKey(
215*9c5db199SXin Li            'self', null=True, db_column='invalidates_test_idx',
216*9c5db199SXin Li            related_name='invalidates_test_set')
217*9c5db199SXin Li
218*9c5db199SXin Li    objects = model_logic.ExtendedManager()
219*9c5db199SXin Li
220*9c5db199SXin Li    def _get_attribute_model_and_args(self, attribute):
221*9c5db199SXin Li        return TestAttribute, dict(test=self, attribute=attribute,
222*9c5db199SXin Li                                   user_created=True)
223*9c5db199SXin Li
224*9c5db199SXin Li
225*9c5db199SXin Li    def set_attribute(self, attribute, value):
226*9c5db199SXin Li        # ensure non-user-created attributes remain immutable
227*9c5db199SXin Li        try:
228*9c5db199SXin Li            TestAttribute.objects.get(test=self, attribute=attribute,
229*9c5db199SXin Li                                      user_created=False)
230*9c5db199SXin Li            raise ValueError('Attribute %s already exists for test %s and is '
231*9c5db199SXin Li                             'immutable' % (attribute, self.test_idx))
232*9c5db199SXin Li        except TestAttribute.DoesNotExist:
233*9c5db199SXin Li            super(Test, self).set_attribute(attribute, value)
234*9c5db199SXin Li
235*9c5db199SXin Li    class Meta:
236*9c5db199SXin Li        """Metadata for class Test."""
237*9c5db199SXin Li        db_table = 'tko_tests'
238*9c5db199SXin Li
239*9c5db199SXin Li
240*9c5db199SXin Liclass TestAttribute(dbmodels.Model, model_logic.ModelExtensions):
241*9c5db199SXin Li    """Models a test attribute."""
242*9c5db199SXin Li    test = dbmodels.ForeignKey(Test, db_column='test_idx')
243*9c5db199SXin Li    attribute = dbmodels.CharField(max_length=90)
244*9c5db199SXin Li    value = dbmodels.CharField(blank=True, max_length=300)
245*9c5db199SXin Li    user_created = dbmodels.BooleanField(default=False)
246*9c5db199SXin Li
247*9c5db199SXin Li    objects = model_logic.ExtendedManager()
248*9c5db199SXin Li
249*9c5db199SXin Li    class Meta:
250*9c5db199SXin Li        """Metadata for class TestAttribute."""
251*9c5db199SXin Li        db_table = 'tko_test_attributes'
252*9c5db199SXin Li
253*9c5db199SXin Li
254*9c5db199SXin Liclass IterationAttribute(dbmodels.Model, model_logic.ModelExtensions):
255*9c5db199SXin Li    """Models an iteration attribute."""
256*9c5db199SXin Li    # This isn't really a primary key, but it's necessary to appease Django
257*9c5db199SXin Li    # and is harmless as long as we're careful.
258*9c5db199SXin Li    test = dbmodels.ForeignKey(Test, db_column='test_idx', primary_key=True)
259*9c5db199SXin Li    iteration = dbmodels.IntegerField()
260*9c5db199SXin Li    attribute = dbmodels.CharField(max_length=90)
261*9c5db199SXin Li    value = dbmodels.CharField(blank=True, max_length=300)
262*9c5db199SXin Li
263*9c5db199SXin Li    objects = model_logic.ExtendedManager()
264*9c5db199SXin Li
265*9c5db199SXin Li    class Meta:
266*9c5db199SXin Li        """Metadata for class IterationAttribute."""
267*9c5db199SXin Li        db_table = 'tko_iteration_attributes'
268*9c5db199SXin Li
269*9c5db199SXin Li
270*9c5db199SXin Liclass IterationResult(dbmodels.Model, model_logic.ModelExtensions):
271*9c5db199SXin Li    """Models an iteration result."""
272*9c5db199SXin Li    # See comment on IterationAttribute regarding primary_key=True.
273*9c5db199SXin Li    test = dbmodels.ForeignKey(Test, db_column='test_idx', primary_key=True)
274*9c5db199SXin Li    iteration = dbmodels.IntegerField()
275*9c5db199SXin Li    attribute = dbmodels.CharField(max_length=256)
276*9c5db199SXin Li    value = dbmodels.FloatField(null=True, blank=True)
277*9c5db199SXin Li
278*9c5db199SXin Li    objects = model_logic.ExtendedManager()
279*9c5db199SXin Li
280*9c5db199SXin Li    class Meta:
281*9c5db199SXin Li        """Metadata for class IterationResult."""
282*9c5db199SXin Li        db_table = 'tko_iteration_result'
283*9c5db199SXin Li
284*9c5db199SXin Li
285*9c5db199SXin Liclass TestLabel(dbmodels.Model, model_logic.ModelExtensions):
286*9c5db199SXin Li    """Models a test label."""
287*9c5db199SXin Li    name = dbmodels.CharField(max_length=80, unique=True)
288*9c5db199SXin Li    description = dbmodels.TextField(blank=True)
289*9c5db199SXin Li    tests = dbmodels.ManyToManyField(Test, blank=True,
290*9c5db199SXin Li                                     db_table='tko_test_labels_tests')
291*9c5db199SXin Li
292*9c5db199SXin Li    name_field = 'name'
293*9c5db199SXin Li    objects = model_logic.ExtendedManager()
294*9c5db199SXin Li
295*9c5db199SXin Li    class Meta:
296*9c5db199SXin Li        """Metadata for class TestLabel."""
297*9c5db199SXin Li        db_table = 'tko_test_labels'
298*9c5db199SXin Li
299*9c5db199SXin Li
300*9c5db199SXin Liclass SavedQuery(dbmodels.Model, model_logic.ModelExtensions):
301*9c5db199SXin Li    """Models a saved query."""
302*9c5db199SXin Li    # TODO: change this to foreign key once DBs are merged.
303*9c5db199SXin Li    owner = dbmodels.CharField(max_length=80)
304*9c5db199SXin Li    name = dbmodels.CharField(max_length=100)
305*9c5db199SXin Li    url_token = dbmodels.TextField()
306*9c5db199SXin Li
307*9c5db199SXin Li    class Meta:
308*9c5db199SXin Li        """Metadata for class SavedQuery."""
309*9c5db199SXin Li        db_table = 'tko_saved_queries'
310*9c5db199SXin Li
311*9c5db199SXin Li
312*9c5db199SXin Li# Views.
313*9c5db199SXin Li
314*9c5db199SXin Liclass TestViewManager(TempManager):
315*9c5db199SXin Li    """A Test View Manager."""
316*9c5db199SXin Li
317*9c5db199SXin Li    def get_query_set(self):
318*9c5db199SXin Li        query = super(TestViewManager, self).get_query_set()
319*9c5db199SXin Li
320*9c5db199SXin Li        # add extra fields to selects, using the SQL itself as the "alias"
321*9c5db199SXin Li        extra_select = dict((sql, sql)
322*9c5db199SXin Li                            for sql in self.model.extra_fields.iterkeys())
323*9c5db199SXin Li        return query.extra(select=extra_select)
324*9c5db199SXin Li
325*9c5db199SXin Li
326*9c5db199SXin Li    def _get_include_exclude_suffix(self, exclude):
327*9c5db199SXin Li        if exclude:
328*9c5db199SXin Li            return '_exclude'
329*9c5db199SXin Li        return '_include'
330*9c5db199SXin Li
331*9c5db199SXin Li
332*9c5db199SXin Li    def _add_attribute_join(self, query_set, join_condition,
333*9c5db199SXin Li                            suffix=None, exclude=False):
334*9c5db199SXin Li        if suffix is None:
335*9c5db199SXin Li            suffix = self._get_include_exclude_suffix(exclude)
336*9c5db199SXin Li        return self.add_join(query_set, 'tko_test_attributes',
337*9c5db199SXin Li                             join_key='test_idx',
338*9c5db199SXin Li                             join_condition=join_condition,
339*9c5db199SXin Li                             suffix=suffix, exclude=exclude)
340*9c5db199SXin Li
341*9c5db199SXin Li
342*9c5db199SXin Li    def _add_label_pivot_table_join(self, query_set, suffix, join_condition='',
343*9c5db199SXin Li                                    exclude=False, force_left_join=False):
344*9c5db199SXin Li        return self.add_join(query_set, 'tko_test_labels_tests',
345*9c5db199SXin Li                             join_key='test_id',
346*9c5db199SXin Li                             join_condition=join_condition,
347*9c5db199SXin Li                             suffix=suffix, exclude=exclude,
348*9c5db199SXin Li                             force_left_join=force_left_join)
349*9c5db199SXin Li
350*9c5db199SXin Li
351*9c5db199SXin Li    def _add_label_joins(self, query_set, suffix=''):
352*9c5db199SXin Li        query_set = self._add_label_pivot_table_join(
353*9c5db199SXin Li                query_set, suffix=suffix, force_left_join=True)
354*9c5db199SXin Li
355*9c5db199SXin Li        # since we're not joining from the original table, we can't use
356*9c5db199SXin Li        # self.add_join() again
357*9c5db199SXin Li        second_join_alias = 'tko_test_labels' + suffix
358*9c5db199SXin Li        second_join_condition = ('%s.id = %s.testlabel_id' %
359*9c5db199SXin Li                                 (second_join_alias,
360*9c5db199SXin Li                                  'tko_test_labels_tests' + suffix))
361*9c5db199SXin Li        query_set.query.add_custom_join('tko_test_labels',
362*9c5db199SXin Li                                        second_join_condition,
363*9c5db199SXin Li                                        query_set.query.LOUTER,
364*9c5db199SXin Li                                        alias=second_join_alias)
365*9c5db199SXin Li        return query_set
366*9c5db199SXin Li
367*9c5db199SXin Li
368*9c5db199SXin Li    def _get_label_ids_from_names(self, label_names):
369*9c5db199SXin Li        label_ids = list( # listifying avoids a double query below
370*9c5db199SXin Li                TestLabel.objects.filter(name__in=label_names)
371*9c5db199SXin Li                .values_list('name', 'id'))
372*9c5db199SXin Li        if len(label_ids) < len(set(label_names)):
373*9c5db199SXin Li            raise ValueError('Not all labels found: %s' %
374*9c5db199SXin Li                             ', '.join(label_names))
375*9c5db199SXin Li        return dict(name_and_id for name_and_id in label_ids)
376*9c5db199SXin Li
377*9c5db199SXin Li
378*9c5db199SXin Li    def _include_or_exclude_labels(self, query_set, label_names, exclude=False):
379*9c5db199SXin Li        label_ids = self._get_label_ids_from_names(label_names).itervalues()
380*9c5db199SXin Li        suffix = self._get_include_exclude_suffix(exclude)
381*9c5db199SXin Li        condition = ('tko_test_labels_tests%s.testlabel_id IN (%s)' %
382*9c5db199SXin Li                     (suffix,
383*9c5db199SXin Li                      ','.join(str(label_id) for label_id in label_ids)))
384*9c5db199SXin Li        return self._add_label_pivot_table_join(query_set,
385*9c5db199SXin Li                                                join_condition=condition,
386*9c5db199SXin Li                                                suffix=suffix,
387*9c5db199SXin Li                                                exclude=exclude)
388*9c5db199SXin Li
389*9c5db199SXin Li
390*9c5db199SXin Li    def _add_custom_select(self, query_set, select_name, select_sql):
391*9c5db199SXin Li        return query_set.extra(select={select_name: select_sql})
392*9c5db199SXin Li
393*9c5db199SXin Li
394*9c5db199SXin Li    def _add_select_value(self, query_set, alias):
395*9c5db199SXin Li        return self._add_custom_select(query_set, alias,
396*9c5db199SXin Li                                       _quote_name(alias) + '.value')
397*9c5db199SXin Li
398*9c5db199SXin Li
399*9c5db199SXin Li    def _add_select_ifnull(self, query_set, alias, non_null_value):
400*9c5db199SXin Li        select_sql = "IF(%s.id IS NOT NULL, '%s', NULL)" % (_quote_name(alias),
401*9c5db199SXin Li                                                            non_null_value)
402*9c5db199SXin Li        return self._add_custom_select(query_set, alias, select_sql)
403*9c5db199SXin Li
404*9c5db199SXin Li
405*9c5db199SXin Li    def _join_test_label_column(self, query_set, label_name, label_id):
406*9c5db199SXin Li        alias = 'test_label_' + label_name
407*9c5db199SXin Li        label_query = TestLabel.objects.filter(name=label_name)
408*9c5db199SXin Li        query_set = Test.objects.join_custom_field(query_set, label_query,
409*9c5db199SXin Li                                                   alias)
410*9c5db199SXin Li
411*9c5db199SXin Li        query_set = self._add_select_ifnull(query_set, alias, label_name)
412*9c5db199SXin Li        return query_set
413*9c5db199SXin Li
414*9c5db199SXin Li
415*9c5db199SXin Li    def _join_test_label_columns(self, query_set, label_names):
416*9c5db199SXin Li        label_id_map = self._get_label_ids_from_names(label_names)
417*9c5db199SXin Li        for label_name in label_names:
418*9c5db199SXin Li            query_set = self._join_test_label_column(query_set, label_name,
419*9c5db199SXin Li                                                     label_id_map[label_name])
420*9c5db199SXin Li        return query_set
421*9c5db199SXin Li
422*9c5db199SXin Li
423*9c5db199SXin Li    def _join_test_attribute(self, query_set, attribute, alias=None,
424*9c5db199SXin Li                             extra_join_condition=None):
425*9c5db199SXin Li        """
426*9c5db199SXin Li        Join the given TestView QuerySet to TestAttribute.  The resulting query
427*9c5db199SXin Li        has an additional column for the given attribute named
428*9c5db199SXin Li        "attribute_<attribute name>".
429*9c5db199SXin Li        """
430*9c5db199SXin Li        if not alias:
431*9c5db199SXin Li            alias = 'test_attribute_' + attribute
432*9c5db199SXin Li        attribute_query = TestAttribute.objects.filter(attribute=attribute)
433*9c5db199SXin Li        if extra_join_condition:
434*9c5db199SXin Li            attribute_query = attribute_query.extra(
435*9c5db199SXin Li                    where=[extra_join_condition])
436*9c5db199SXin Li        query_set = Test.objects.join_custom_field(query_set, attribute_query,
437*9c5db199SXin Li                                                   alias)
438*9c5db199SXin Li
439*9c5db199SXin Li        query_set = self._add_select_value(query_set, alias)
440*9c5db199SXin Li        return query_set
441*9c5db199SXin Li
442*9c5db199SXin Li
443*9c5db199SXin Li    def _join_machine_label_columns(self, query_set, machine_label_names):
444*9c5db199SXin Li        for label_name in machine_label_names:
445*9c5db199SXin Li            alias = 'machine_label_' + label_name
446*9c5db199SXin Li            condition = "FIND_IN_SET('%s', %s)" % (
447*9c5db199SXin Li                    label_name, _quote_name(alias) + '.value')
448*9c5db199SXin Li            query_set = self._join_test_attribute(
449*9c5db199SXin Li                    query_set, 'host-labels',
450*9c5db199SXin Li                    alias=alias, extra_join_condition=condition)
451*9c5db199SXin Li            query_set = self._add_select_ifnull(query_set, alias, label_name)
452*9c5db199SXin Li        return query_set
453*9c5db199SXin Li
454*9c5db199SXin Li
455*9c5db199SXin Li    def _join_one_iteration_key(self, query_set, result_key, first_alias=None):
456*9c5db199SXin Li        alias = 'iteration_result_' + result_key
457*9c5db199SXin Li        iteration_query = IterationResult.objects.filter(attribute=result_key)
458*9c5db199SXin Li        if first_alias:
459*9c5db199SXin Li            # after the first join, we need to match up iteration indices,
460*9c5db199SXin Li            # otherwise each join will expand the query by the number of
461*9c5db199SXin Li            # iterations and we'll have extraneous rows
462*9c5db199SXin Li            iteration_query = iteration_query.extra(
463*9c5db199SXin Li                    where=['%s.iteration = %s.iteration'
464*9c5db199SXin Li                           % (_quote_name(alias), _quote_name(first_alias))])
465*9c5db199SXin Li
466*9c5db199SXin Li        query_set = Test.objects.join_custom_field(query_set, iteration_query,
467*9c5db199SXin Li                                                   alias, left_join=False)
468*9c5db199SXin Li        # select the iteration value and index for this join
469*9c5db199SXin Li        query_set = self._add_select_value(query_set, alias)
470*9c5db199SXin Li        if not first_alias:
471*9c5db199SXin Li            # for first join, add iteration index select too
472*9c5db199SXin Li            query_set = self._add_custom_select(
473*9c5db199SXin Li                    query_set, 'iteration_index',
474*9c5db199SXin Li                    _quote_name(alias) + '.iteration')
475*9c5db199SXin Li
476*9c5db199SXin Li        return query_set, alias
477*9c5db199SXin Li
478*9c5db199SXin Li
479*9c5db199SXin Li    def _join_iteration_results(self, test_view_query_set, result_keys):
480*9c5db199SXin Li        """Join the given TestView QuerySet to IterationResult for one result.
481*9c5db199SXin Li
482*9c5db199SXin Li        The resulting query looks like a TestView query but has one row per
483*9c5db199SXin Li        iteration.  Each row includes all the attributes of TestView, an
484*9c5db199SXin Li        attribute for each key in result_keys and an iteration_index attribute.
485*9c5db199SXin Li
486*9c5db199SXin Li        We accomplish this by joining the TestView query to IterationResult
487*9c5db199SXin Li        once per result key.  Each join is restricted on the result key (and on
488*9c5db199SXin Li        the test index, like all one-to-many joins).  For the first join, this
489*9c5db199SXin Li        is the only restriction, so each TestView row expands to a row per
490*9c5db199SXin Li        iteration (per iteration that includes the key, of course).  For each
491*9c5db199SXin Li        subsequent join, we also restrict the iteration index to match that of
492*9c5db199SXin Li        the initial join.  This makes each subsequent join produce exactly one
493*9c5db199SXin Li        result row for each input row.  (This assumes each iteration contains
494*9c5db199SXin Li        the same set of keys.  Results are undefined if that's not true.)
495*9c5db199SXin Li        """
496*9c5db199SXin Li        if not result_keys:
497*9c5db199SXin Li            return test_view_query_set
498*9c5db199SXin Li
499*9c5db199SXin Li        query_set, first_alias = self._join_one_iteration_key(
500*9c5db199SXin Li                test_view_query_set, result_keys[0])
501*9c5db199SXin Li        for result_key in result_keys[1:]:
502*9c5db199SXin Li            query_set, _ = self._join_one_iteration_key(query_set, result_key,
503*9c5db199SXin Li                                                        first_alias=first_alias)
504*9c5db199SXin Li        return query_set
505*9c5db199SXin Li
506*9c5db199SXin Li
507*9c5db199SXin Li    def _join_job_keyvals(self, query_set, job_keyvals):
508*9c5db199SXin Li        for job_keyval in job_keyvals:
509*9c5db199SXin Li            alias = 'job_keyval_' + job_keyval
510*9c5db199SXin Li            keyval_query = JobKeyval.objects.filter(key=job_keyval)
511*9c5db199SXin Li            query_set = Job.objects.join_custom_field(query_set, keyval_query,
512*9c5db199SXin Li                                                       alias)
513*9c5db199SXin Li            query_set = self._add_select_value(query_set, alias)
514*9c5db199SXin Li        return query_set
515*9c5db199SXin Li
516*9c5db199SXin Li
517*9c5db199SXin Li    def _join_iteration_attributes(self, query_set, iteration_attributes):
518*9c5db199SXin Li        for attribute in iteration_attributes:
519*9c5db199SXin Li            alias = 'iteration_attribute_' + attribute
520*9c5db199SXin Li            attribute_query = IterationAttribute.objects.filter(
521*9c5db199SXin Li                    attribute=attribute)
522*9c5db199SXin Li            query_set = Test.objects.join_custom_field(query_set,
523*9c5db199SXin Li                                                       attribute_query, alias)
524*9c5db199SXin Li            query_set = self._add_select_value(query_set, alias)
525*9c5db199SXin Li        return query_set
526*9c5db199SXin Li
527*9c5db199SXin Li
528*9c5db199SXin Li    def get_query_set_with_joins(self, filter_data):
529*9c5db199SXin Li        """Add joins for querying over test-related items.
530*9c5db199SXin Li
531*9c5db199SXin Li        These parameters are supported going forward:
532*9c5db199SXin Li        * test_attribute_fields: list of attribute names.  Each attribute will
533*9c5db199SXin Li                be available as a column attribute_<name>.value.
534*9c5db199SXin Li        * test_label_fields: list of label names.  Each label will be available
535*9c5db199SXin Li                as a column label_<name>.id, non-null iff the label is present.
536*9c5db199SXin Li        * iteration_result_fields: list of iteration result names.  Each
537*9c5db199SXin Li                result will be available as a column iteration_<name>.value.
538*9c5db199SXin Li                Note that this changes the semantics to return iterations
539*9c5db199SXin Li                instead of tests -- if a test has multiple iterations, a row
540*9c5db199SXin Li                will be returned for each one.  The iteration index is also
541*9c5db199SXin Li                available as iteration_<name>.iteration.
542*9c5db199SXin Li        * machine_label_fields: list of machine label names.  Each will be
543*9c5db199SXin Li                available as a column machine_label_<name>.id, non-null iff the
544*9c5db199SXin Li                label is present on the machine used in the test.
545*9c5db199SXin Li        * job_keyval_fields: list of job keyval names. Each value will be
546*9c5db199SXin Li                available as a column job_keyval_<name>.id, non-null iff the
547*9c5db199SXin Li                keyval is present in the AFE job.
548*9c5db199SXin Li        * iteration_attribute_fields: list of iteration attribute names. Each
549*9c5db199SXin Li                attribute will be available as a column
550*9c5db199SXin Li                iteration_attribute<name>.id, non-null iff the attribute is
551*9c5db199SXin Li                present.
552*9c5db199SXin Li
553*9c5db199SXin Li        These parameters are deprecated:
554*9c5db199SXin Li        * include_labels
555*9c5db199SXin Li        * exclude_labels
556*9c5db199SXin Li        * include_attributes_where
557*9c5db199SXin Li        * exclude_attributes_where
558*9c5db199SXin Li
559*9c5db199SXin Li        Additionally, this method adds joins if the following strings are
560*9c5db199SXin Li        present in extra_where (this is also deprecated):
561*9c5db199SXin Li        * test_labels
562*9c5db199SXin Li        * test_attributes_host_labels
563*9c5db199SXin Li
564*9c5db199SXin Li        @param filter_data: Data by which to filter.
565*9c5db199SXin Li
566*9c5db199SXin Li        @return A QuerySet.
567*9c5db199SXin Li
568*9c5db199SXin Li        """
569*9c5db199SXin Li        query_set = self.get_query_set()
570*9c5db199SXin Li
571*9c5db199SXin Li        test_attributes = filter_data.pop('test_attribute_fields', [])
572*9c5db199SXin Li        for attribute in test_attributes:
573*9c5db199SXin Li            query_set = self._join_test_attribute(query_set, attribute)
574*9c5db199SXin Li
575*9c5db199SXin Li        test_labels = filter_data.pop('test_label_fields', [])
576*9c5db199SXin Li        query_set = self._join_test_label_columns(query_set, test_labels)
577*9c5db199SXin Li
578*9c5db199SXin Li        machine_labels = filter_data.pop('machine_label_fields', [])
579*9c5db199SXin Li        query_set = self._join_machine_label_columns(query_set, machine_labels)
580*9c5db199SXin Li
581*9c5db199SXin Li        iteration_keys = filter_data.pop('iteration_result_fields', [])
582*9c5db199SXin Li        query_set = self._join_iteration_results(query_set, iteration_keys)
583*9c5db199SXin Li
584*9c5db199SXin Li        job_keyvals = filter_data.pop('job_keyval_fields', [])
585*9c5db199SXin Li        query_set = self._join_job_keyvals(query_set, job_keyvals)
586*9c5db199SXin Li
587*9c5db199SXin Li        iteration_attributes = filter_data.pop('iteration_attribute_fields', [])
588*9c5db199SXin Li        query_set = self._join_iteration_attributes(query_set,
589*9c5db199SXin Li                                                    iteration_attributes)
590*9c5db199SXin Li
591*9c5db199SXin Li        # everything that follows is deprecated behavior
592*9c5db199SXin Li
593*9c5db199SXin Li        joined = False
594*9c5db199SXin Li
595*9c5db199SXin Li        extra_where = filter_data.get('extra_where', '')
596*9c5db199SXin Li        if 'tko_test_labels' in extra_where:
597*9c5db199SXin Li            query_set = self._add_label_joins(query_set)
598*9c5db199SXin Li            joined = True
599*9c5db199SXin Li
600*9c5db199SXin Li        include_labels = filter_data.pop('include_labels', [])
601*9c5db199SXin Li        exclude_labels = filter_data.pop('exclude_labels', [])
602*9c5db199SXin Li        if include_labels:
603*9c5db199SXin Li            query_set = self._include_or_exclude_labels(query_set,
604*9c5db199SXin Li                                                        include_labels)
605*9c5db199SXin Li            joined = True
606*9c5db199SXin Li        if exclude_labels:
607*9c5db199SXin Li            query_set = self._include_or_exclude_labels(query_set,
608*9c5db199SXin Li                                                        exclude_labels,
609*9c5db199SXin Li                                                        exclude=True)
610*9c5db199SXin Li            joined = True
611*9c5db199SXin Li
612*9c5db199SXin Li        include_attributes_where = filter_data.pop('include_attributes_where',
613*9c5db199SXin Li                                                   '')
614*9c5db199SXin Li        exclude_attributes_where = filter_data.pop('exclude_attributes_where',
615*9c5db199SXin Li                                                   '')
616*9c5db199SXin Li        if include_attributes_where:
617*9c5db199SXin Li            query_set = self._add_attribute_join(
618*9c5db199SXin Li                query_set,
619*9c5db199SXin Li                join_condition=self.escape_user_sql(include_attributes_where))
620*9c5db199SXin Li            joined = True
621*9c5db199SXin Li        if exclude_attributes_where:
622*9c5db199SXin Li            query_set = self._add_attribute_join(
623*9c5db199SXin Li                query_set,
624*9c5db199SXin Li                join_condition=self.escape_user_sql(exclude_attributes_where),
625*9c5db199SXin Li                exclude=True)
626*9c5db199SXin Li            joined = True
627*9c5db199SXin Li
628*9c5db199SXin Li        if not joined:
629*9c5db199SXin Li            filter_data['no_distinct'] = True
630*9c5db199SXin Li
631*9c5db199SXin Li        if 'tko_test_attributes_host_labels' in extra_where:
632*9c5db199SXin Li            query_set = self._add_attribute_join(
633*9c5db199SXin Li                query_set, suffix='_host_labels',
634*9c5db199SXin Li                join_condition='tko_test_attributes_host_labels.attribute = '
635*9c5db199SXin Li                               '"host-labels"')
636*9c5db199SXin Li
637*9c5db199SXin Li        return query_set
638*9c5db199SXin Li
639*9c5db199SXin Li
640*9c5db199SXin Li    def query_test_ids(self, filter_data, apply_presentation=True):
641*9c5db199SXin Li        """Queries for test IDs.
642*9c5db199SXin Li
643*9c5db199SXin Li        @param filter_data: Data by which to filter.
644*9c5db199SXin Li        @param apply_presentation: Whether or not to apply presentation
645*9c5db199SXin Li            parameters.
646*9c5db199SXin Li
647*9c5db199SXin Li        @return A list of test IDs.
648*9c5db199SXin Li
649*9c5db199SXin Li        """
650*9c5db199SXin Li        query = self.model.query_objects(filter_data,
651*9c5db199SXin Li                                         apply_presentation=apply_presentation)
652*9c5db199SXin Li        dicts = query.values('test_idx')
653*9c5db199SXin Li        return [item['test_idx'] for item in dicts]
654*9c5db199SXin Li
655*9c5db199SXin Li
656*9c5db199SXin Li    def query_test_label_ids(self, filter_data):
657*9c5db199SXin Li        """Queries for test label IDs.
658*9c5db199SXin Li
659*9c5db199SXin Li        @param filter_data: Data by which to filter.
660*9c5db199SXin Li
661*9c5db199SXin Li        @return A list of test label IDs.
662*9c5db199SXin Li
663*9c5db199SXin Li        """
664*9c5db199SXin Li        query_set = self.model.query_objects(filter_data)
665*9c5db199SXin Li        query_set = self._add_label_joins(query_set, suffix='_list')
666*9c5db199SXin Li        rows = self._custom_select_query(query_set, ['tko_test_labels_list.id'])
667*9c5db199SXin Li        return [row[0] for row in rows if row[0] is not None]
668*9c5db199SXin Li
669*9c5db199SXin Li
670*9c5db199SXin Li    def escape_user_sql(self, sql):
671*9c5db199SXin Li        sql = super(TestViewManager, self).escape_user_sql(sql)
672*9c5db199SXin Li        return sql.replace('test_idx', self.get_key_on_this_table('test_idx'))
673*9c5db199SXin Li
674*9c5db199SXin Li
675*9c5db199SXin Liclass TestView(dbmodels.Model, model_logic.ModelExtensions):
676*9c5db199SXin Li    """Models a test view."""
677*9c5db199SXin Li    extra_fields = {
678*9c5db199SXin Li            'DATE(job_queued_time)': 'job queued day',
679*9c5db199SXin Li            'DATE(test_finished_time)': 'test finished day',
680*9c5db199SXin Li    }
681*9c5db199SXin Li
682*9c5db199SXin Li    group_fields = [
683*9c5db199SXin Li            'test_name',
684*9c5db199SXin Li            'status',
685*9c5db199SXin Li            'kernel',
686*9c5db199SXin Li            'hostname',
687*9c5db199SXin Li            'job_tag',
688*9c5db199SXin Li            'job_name',
689*9c5db199SXin Li            'platform',
690*9c5db199SXin Li            'reason',
691*9c5db199SXin Li            'job_owner',
692*9c5db199SXin Li            'job_queued_time',
693*9c5db199SXin Li            'DATE(job_queued_time)',
694*9c5db199SXin Li            'test_started_time',
695*9c5db199SXin Li            'test_finished_time',
696*9c5db199SXin Li            'DATE(test_finished_time)',
697*9c5db199SXin Li    ]
698*9c5db199SXin Li
699*9c5db199SXin Li    test_idx = dbmodels.IntegerField('test index', primary_key=True)
700*9c5db199SXin Li    job_idx = dbmodels.IntegerField('job index', null=True, blank=True)
701*9c5db199SXin Li    test_name = dbmodels.CharField(blank=True, max_length=90)
702*9c5db199SXin Li    subdir = dbmodels.CharField('subdirectory', blank=True, max_length=180)
703*9c5db199SXin Li    kernel_idx = dbmodels.IntegerField('kernel index')
704*9c5db199SXin Li    status_idx = dbmodels.IntegerField('status index')
705*9c5db199SXin Li    reason = dbmodels.CharField(blank=True, max_length=3072)
706*9c5db199SXin Li    machine_idx = dbmodels.IntegerField('host index')
707*9c5db199SXin Li    test_started_time = dbmodels.DateTimeField(null=True, blank=True)
708*9c5db199SXin Li    test_finished_time = dbmodels.DateTimeField(null=True, blank=True)
709*9c5db199SXin Li    job_tag = dbmodels.CharField(blank=True, max_length=300)
710*9c5db199SXin Li    job_name = dbmodels.CharField(blank=True, max_length=300)
711*9c5db199SXin Li    job_owner = dbmodels.CharField('owner', blank=True, max_length=240)
712*9c5db199SXin Li    job_queued_time = dbmodels.DateTimeField(null=True, blank=True)
713*9c5db199SXin Li    job_started_time = dbmodels.DateTimeField(null=True, blank=True)
714*9c5db199SXin Li    job_finished_time = dbmodels.DateTimeField(null=True, blank=True)
715*9c5db199SXin Li    afe_job_id = dbmodels.IntegerField(null=True)
716*9c5db199SXin Li    hostname = dbmodels.CharField(blank=True, max_length=300)
717*9c5db199SXin Li    platform = dbmodels.CharField(blank=True, max_length=240)
718*9c5db199SXin Li    machine_owner = dbmodels.CharField(blank=True, max_length=240)
719*9c5db199SXin Li    kernel_hash = dbmodels.CharField(blank=True, max_length=105)
720*9c5db199SXin Li    kernel_base = dbmodels.CharField(blank=True, max_length=90)
721*9c5db199SXin Li    kernel = dbmodels.CharField(blank=True, max_length=300)
722*9c5db199SXin Li    status = dbmodels.CharField(blank=True, max_length=30)
723*9c5db199SXin Li    invalid = dbmodels.BooleanField(blank=True)
724*9c5db199SXin Li    invalidates_test_idx = dbmodels.IntegerField(null=True, blank=True)
725*9c5db199SXin Li
726*9c5db199SXin Li    objects = TestViewManager()
727*9c5db199SXin Li
728*9c5db199SXin Li    def save(self):
729*9c5db199SXin Li        raise NotImplementedError('TestView is read-only')
730*9c5db199SXin Li
731*9c5db199SXin Li
732*9c5db199SXin Li    def delete(self):
733*9c5db199SXin Li        raise NotImplementedError('TestView is read-only')
734*9c5db199SXin Li
735*9c5db199SXin Li
736*9c5db199SXin Li    @classmethod
737*9c5db199SXin Li    def query_objects(cls, filter_data, initial_query=None,
738*9c5db199SXin Li                      apply_presentation=True):
739*9c5db199SXin Li        if initial_query is None:
740*9c5db199SXin Li            initial_query = cls.objects.get_query_set_with_joins(filter_data)
741*9c5db199SXin Li        return super(TestView, cls).query_objects(
742*9c5db199SXin Li                filter_data, initial_query=initial_query,
743*9c5db199SXin Li                apply_presentation=apply_presentation)
744*9c5db199SXin Li
745*9c5db199SXin Li    class Meta:
746*9c5db199SXin Li        """Metadata for class TestView."""
747*9c5db199SXin Li        db_table = 'tko_test_view_2'
748