1# Copyright (C) 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import dataclasses 16from dataclasses import dataclass 17import importlib 18import sys 19from typing import Dict 20from typing import List 21from typing import Set 22from typing import Optional 23from typing import Union 24 25from python.generators.trace_processor_table.public import Alias 26from python.generators.trace_processor_table.public import Column 27from python.generators.trace_processor_table.public import ColumnDoc 28from python.generators.trace_processor_table.public import ColumnFlag 29from python.generators.trace_processor_table.public import CppColumnType 30from python.generators.trace_processor_table.public import CppDouble 31from python.generators.trace_processor_table.public import CppInt32 32from python.generators.trace_processor_table.public import CppInt64 33from python.generators.trace_processor_table.public import CppOptional 34from python.generators.trace_processor_table.public import CppSelfTableId 35from python.generators.trace_processor_table.public import CppString 36from python.generators.trace_processor_table.public import CppTableId 37from python.generators.trace_processor_table.public import CppUint32 38from python.generators.trace_processor_table.public import Table 39 40 41@dataclass 42class ParsedType: 43 """Result of parsing a CppColumnType into its parts.""" 44 cpp_type: str 45 is_optional: bool = False 46 is_alias: bool = False 47 alias_underlying_name: Optional[str] = None 48 is_self_id: bool = False 49 id_table: Optional[Table] = None 50 51 def cpp_type_with_optionality(self) -> str: 52 """Returns the C++ type wrapping with base::Optional if necessary.""" 53 54 # ThreadTable and ProcessTable are special for legacy reasons as they were 55 # around even before the advent of C++ macro tables. Because of this a lot 56 # of code was written assuming that upid and utid were uint32 (e.g. indexing 57 # directly into vectors using them) and it was decided this behaviour was 58 # too expensive in engineering cost to fix given the trivial benefit. For 59 # this reason, continue to maintain this illusion. 60 if self.id_table and self.id_table.class_name in ('ThreadTable', 61 'ProcessTable'): 62 cpp_type = 'uint32_t' 63 else: 64 cpp_type = self.cpp_type 65 if self.is_optional: 66 return f'std::optional<{cpp_type}>' 67 return cpp_type 68 69 70@dataclass(frozen=True) 71class ParsedColumn: 72 """Representation of a column parsed from a Python definition.""" 73 74 column: Column 75 doc: Optional[ColumnDoc] 76 77 # Whether this column is the implicit "id" column which is added by while 78 # parsing the tables rather than by the user. 79 is_implicit_id: bool = False 80 81 # Whether this column is the implicit "type" column which is added by while 82 # parsing the tables rather than by the user. 83 is_implicit_type: bool = False 84 85 # Whether this column comes from copying a column from the ancestor. If this 86 # is set to false, the user explicitly specified it for this table. 87 is_ancestor: bool = False 88 89 90@dataclass(frozen=True) 91class ParsedTable: 92 """Representation of a table parsed from a Python definition.""" 93 94 table: Table 95 columns: List[ParsedColumn] 96 97 98def parse_type_with_cols(table: Table, cols: List[Column], 99 col_type: CppColumnType) -> ParsedType: 100 """Parses a CppColumnType into its constiuent parts.""" 101 102 if isinstance(col_type, CppInt64): 103 return ParsedType('int64_t') 104 if isinstance(col_type, CppInt32): 105 return ParsedType('int32_t') 106 if isinstance(col_type, CppUint32): 107 return ParsedType('uint32_t') 108 if isinstance(col_type, CppDouble): 109 return ParsedType('double') 110 if isinstance(col_type, CppString): 111 return ParsedType('StringPool::Id') 112 113 if isinstance(col_type, Alias): 114 col = next(c for c in cols if c.name == col_type.underlying_column) 115 return ParsedType( 116 parse_type(table, col.type).cpp_type, 117 is_alias=True, 118 alias_underlying_name=col.name) 119 120 if isinstance(col_type, CppTableId): 121 return ParsedType( 122 f'{col_type.table.class_name}::Id', id_table=col_type.table) 123 124 if isinstance(col_type, CppSelfTableId): 125 return ParsedType( 126 f'{table.class_name}::Id', is_self_id=True, id_table=table) 127 128 if isinstance(col_type, CppOptional): 129 inner = parse_type(table, col_type.inner) 130 assert not inner.is_optional, 'Nested optional not allowed' 131 return dataclasses.replace(inner, is_optional=True) 132 133 raise Exception(f'Unknown type {col_type}') 134 135 136def parse_type(table: Table, col_type: CppColumnType) -> ParsedType: 137 """Parses a CppColumnType into its constiuent parts.""" 138 return parse_type_with_cols(table, table.columns, col_type) 139 140 141def typed_column_type(table: Table, col: ParsedColumn) -> str: 142 """Returns the TypedColumn/IdColumn C++ type for a given column.""" 143 144 parsed = parse_type(table, col.column.type) 145 if col.is_implicit_id: 146 return f'IdColumn<{parsed.cpp_type}>' 147 return f'TypedColumn<{parsed.cpp_type_with_optionality()}>' 148 149 150def data_layer_type(table: Table, col: ParsedColumn) -> str: 151 """Returns the DataLayer C++ type for a given column.""" 152 153 parsed = parse_type(table, col.column.type) 154 if col.is_implicit_id: 155 return 'column::IdStorage' 156 if parsed.cpp_type == 'StringPool::Id': 157 return 'column::StringStorage' 158 if ColumnFlag.SET_ID in col.column.flags: 159 return 'column::SetIdStorage' 160 return f'column::NumericStorage<ColumnType::{col.column.name}::non_optional_stored_type>' 161 162 163def find_table_deps(table: Table) -> List[Table]: 164 """Finds all the other table class names this table depends on. 165 166 By "depends", we mean this table in C++ would need the dependency to be 167 defined (or included) before this table is defined.""" 168 169 deps: Dict[str, Table] = {} 170 if table.parent: 171 deps[table.parent.class_name] = table.parent 172 for c in table.columns: 173 # Aliases cannot have dependencies so simply ignore them: trying to parse 174 # them before adding implicit columns can cause issues. 175 if isinstance(c.type, Alias): 176 continue 177 id_table = parse_type(table, c.type).id_table 178 if id_table: 179 deps[id_table.class_name] = id_table 180 return list(deps.values()) 181 182 183def public_sql_name(table: Table) -> str: 184 """Extracts SQL name for the table which should be publicised.""" 185 186 wrapping_view = table.wrapping_sql_view 187 return wrapping_view.view_name if wrapping_view else table.sql_name 188 189 190def _create_implicit_columns_for_root(table: Table) -> List[ParsedColumn]: 191 """Given a root table, returns the implicit id and type columns.""" 192 assert table.parent is None 193 194 sql_name = public_sql_name(table) 195 id_doc = table.tabledoc.columns.get('id') if table.tabledoc else None 196 type_doc = table.tabledoc.columns.get('type') if table.tabledoc else None 197 return [ 198 ParsedColumn( 199 Column('id', CppSelfTableId(), ColumnFlag.SORTED), 200 _to_column_doc(id_doc) if id_doc else ColumnDoc( 201 doc=f'Unique identifier for this {sql_name}.'), 202 is_implicit_id=True), 203 ParsedColumn( 204 Column('type', CppString(), ColumnFlag.NONE), 205 _to_column_doc(type_doc) if type_doc else ColumnDoc(doc=''' 206 The name of the "most-specific" child table containing this 207 row. 208 '''), 209 is_implicit_type=True, 210 ) 211 ] 212 213 214def _topological_sort_table_and_deps(parsed: List[Table]) -> List[Table]: 215 """Topologically sorts a list of tables (i.e. dependenices appear earlier). 216 217 See [1] for information on a topological sort. We do this to allow 218 dependencies to be processed and appear ealier than their dependents. 219 220 [1] https://en.wikipedia.org/wiki/Topological_sorting""" 221 visited: Set[str] = set() 222 result: List[Table] = [] 223 224 # Topological sorting is really just a DFS where we put the nodes in the list 225 # after any dependencies. 226 def dfs(t: Table): 227 if t.class_name in visited: 228 return 229 visited.add(t.class_name) 230 231 for dep in find_table_deps(t): 232 dfs(dep) 233 result.append(t) 234 235 for p in parsed: 236 dfs(p) 237 return result 238 239 240def _to_column_doc(doc: Union[ColumnDoc, str, None]) -> Optional[ColumnDoc]: 241 """Cooerces a user specified ColumnDoc or string into a ColumnDoc.""" 242 243 if doc is None or isinstance(doc, ColumnDoc): 244 return doc 245 return ColumnDoc(doc=doc) 246 247 248def parse_tables_from_modules(modules: List[str]) -> List[ParsedTable]: 249 """Creates a list of tables with the associated paths.""" 250 251 # Create a mapping from the table to a "parsed" version of the table. 252 tables: Dict[str, Table] = {} 253 for module in modules: 254 imported = importlib.import_module(module) 255 run_tables: List[Table] = imported.__dict__['ALL_TABLES'] 256 for table in run_tables: 257 existing_table = tables.get(table.class_name) 258 assert not existing_table or existing_table == table 259 tables[table.class_name] = table 260 261 # Sort all the tables: note that this list may include tables which are not 262 # in |tables| dictionary due to dependencies on tables which live in a file 263 # not covered by |input_paths|. 264 sorted_tables = _topological_sort_table_and_deps(list(tables.values())) 265 266 parsed_tables: Dict[str, ParsedTable] = {} 267 for table in sorted_tables: 268 parsed_columns: List[ParsedColumn] 269 if table.parent: 270 parsed_parent = parsed_tables[table.parent.class_name] 271 parsed_columns = [ 272 dataclasses.replace(c, is_ancestor=True) 273 for c in parsed_parent.columns 274 ] 275 else: 276 parsed_columns = _create_implicit_columns_for_root(table) 277 278 for c in table.columns: 279 doc = table.tabledoc.columns.get(c.name) if table.tabledoc else None 280 parsed_columns.append(ParsedColumn(c, _to_column_doc(doc))) 281 parsed_tables[table.class_name] = ParsedTable(table, parsed_columns) 282 283 # Only return tables which come directly from |input_paths|. This stops us 284 # generating tables which were not requested. 285 return [ 286 parsed_tables[p.class_name] 287 for p in sorted_tables 288 if p.class_name in tables 289 ] 290