1# Copyright (C) 2019 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15# A collection of utilities for extracting build rule information from GN 16# projects. 17 18from __future__ import print_function 19import collections 20from compat import iteritems 21import errno 22import filecmp 23import json 24import os 25import re 26import shutil 27import subprocess 28import sys 29from typing import Dict 30from typing import Optional 31from typing import Set 32from typing import Tuple 33 34BUILDFLAGS_TARGET = '//gn:gen_buildflags' 35GEN_VERSION_TARGET = '//src/base:version_gen_h' 36TARGET_TOOLCHAIN = '//gn/standalone/toolchain:gcc_like_host' 37HOST_TOOLCHAIN = '//gn/standalone/toolchain:gcc_like_host' 38LINKER_UNIT_TYPES = ('executable', 'shared_library', 'static_library') 39 40# TODO(primiano): investigate these, they require further componentization. 41ODR_VIOLATION_IGNORE_TARGETS = { 42 '//test/cts:perfetto_cts_deps', 43 '//:perfetto_integrationtests', 44} 45 46 47def _check_command_output(cmd, cwd): 48 try: 49 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, cwd=cwd) 50 except subprocess.CalledProcessError as e: 51 print( 52 'Command "{}" failed in {}:'.format(' '.join(cmd), cwd), 53 file=sys.stderr) 54 print(e.output.decode(), file=sys.stderr) 55 sys.exit(1) 56 else: 57 return output.decode() 58 59 60def repo_root(): 61 """Returns an absolute path to the repository root.""" 62 return os.path.join( 63 os.path.realpath(os.path.dirname(__file__)), os.path.pardir) 64 65 66def _tool_path(name, system_buildtools=False): 67 # Pass-through to use name if the caller requests to use the system 68 # toolchain. 69 if system_buildtools: 70 return [name] 71 wrapper = os.path.abspath( 72 os.path.join(repo_root(), 'tools', 'run_buildtools_binary.py')) 73 return ['python3', wrapper, name] 74 75 76def prepare_out_directory(gn_args, 77 name, 78 root=repo_root(), 79 system_buildtools=False): 80 """Creates the JSON build description by running GN. 81 82 Returns (path, desc) where |path| is the location of the output directory 83 and |desc| is the JSON build description. 84 """ 85 out = os.path.join(root, 'out', name) 86 try: 87 os.makedirs(out) 88 except OSError as e: 89 if e.errno != errno.EEXIST: 90 raise 91 _check_command_output( 92 _tool_path('gn', system_buildtools) + 93 ['gen', out, '--args=%s' % gn_args], 94 cwd=repo_root()) 95 return out 96 97 98def load_build_description(out, system_buildtools=False): 99 """Creates the JSON build description by running GN.""" 100 desc = _check_command_output( 101 _tool_path('gn', system_buildtools) + 102 ['desc', out, '--format=json', '--all-toolchains', '//*'], 103 cwd=repo_root()) 104 return json.loads(desc) 105 106 107def create_build_description(gn_args, root=repo_root()): 108 """Prepares a GN out directory and loads the build description from it. 109 110 The temporary out directory is automatically deleted. 111 """ 112 out = prepare_out_directory(gn_args, 'tmp.gn_utils', root=root) 113 try: 114 return load_build_description(out) 115 finally: 116 shutil.rmtree(out) 117 118 119def build_targets(out, targets, quiet=False, system_buildtools=False): 120 """Runs ninja to build a list of GN targets in the given out directory. 121 122 Compiling these targets is required so that we can include any generated 123 source files in the amalgamated result. 124 """ 125 targets = [t.replace('//', '') for t in targets] 126 with open(os.devnull, 'w', newline='\n') as devnull: 127 stdout = devnull if quiet else None 128 cmd = _tool_path('ninja', system_buildtools) + targets 129 subprocess.check_call(cmd, cwd=os.path.abspath(out), stdout=stdout) 130 131 132def compute_source_dependencies(out, system_buildtools=False): 133 """For each source file, computes a set of headers it depends on.""" 134 ninja_deps = _check_command_output( 135 _tool_path('ninja', system_buildtools) + ['-t', 'deps'], cwd=out) 136 deps = {} 137 current_source = None 138 for line in ninja_deps.split('\n'): 139 filename = os.path.relpath(os.path.join(out, line.strip()), repo_root()) 140 # Sanitizer builds may have a dependency of ignorelist.txt. Just skip it. 141 if filename.endswith('gn/standalone/sanitizers/ignorelist.txt'): 142 continue 143 if not line or line[0] != ' ': 144 current_source = None 145 continue 146 elif not current_source: 147 # We're assuming the source file is always listed before the 148 # headers. 149 assert os.path.splitext(line)[1] in ['.c', '.cc', '.cpp', '.S'] 150 current_source = filename 151 deps[current_source] = [] 152 else: 153 assert current_source 154 deps[current_source].append(filename) 155 return deps 156 157 158def label_to_path(label): 159 """Turn a GN output label (e.g., //some_dir/file.cc) into a path.""" 160 assert label.startswith('//') 161 return label[2:] 162 163 164def label_without_toolchain(label): 165 """Strips the toolchain from a GN label. 166 167 Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain: 168 gcc_like_host) without the parenthesised toolchain part. 169 """ 170 return label.split('(')[0] 171 172 173def label_to_target_name_with_path(label): 174 """ 175 Turn a GN label into a target name involving the full path. 176 e.g., //src/perfetto:tests -> src_perfetto_tests 177 """ 178 name = re.sub(r'^//:?', '', label) 179 name = re.sub(r'[^a-zA-Z0-9_]', '_', name) 180 return name 181 182 183def gen_buildflags(gn_args, target_file): 184 """Generates the perfetto_build_flags.h for the given config. 185 186 target_file: the path, relative to the repo root, where the generated 187 buildflag header will be copied into. 188 """ 189 tmp_out = prepare_out_directory(gn_args, 'tmp.gen_buildflags') 190 build_targets(tmp_out, [BUILDFLAGS_TARGET], quiet=True) 191 src = os.path.join(tmp_out, 'gen', 'build_config', 'perfetto_build_flags.h') 192 shutil.copy(src, os.path.join(repo_root(), target_file)) 193 shutil.rmtree(tmp_out) 194 195 196def check_or_commit_generated_files(tmp_files, check): 197 """Checks that gen files are unchanged or renames them to the final location 198 199 Takes in input a list of 'xxx.swp' files that have been written. 200 If check == False, it renames xxx.swp -> xxx. 201 If check == True, it just checks that the contents of 'xxx.swp' == 'xxx'. 202 Returns 0 if no diff was detected, 1 otherwise (to be used as exit code). 203 """ 204 res = 0 205 for tmp_file in tmp_files: 206 assert (tmp_file.endswith('.swp')) 207 target_file = os.path.relpath(tmp_file[:-4]) 208 if check: 209 if not filecmp.cmp(tmp_file, target_file): 210 sys.stderr.write('%s needs to be regenerated\n' % target_file) 211 res = 1 212 os.unlink(tmp_file) 213 else: 214 os.replace(tmp_file, target_file) 215 return res 216 217 218class ODRChecker(object): 219 """Detects ODR violations in linker units 220 221 When we turn GN source sets into Soong & Bazel file groups, there is the risk 222 to create ODR violations by including the same file group into different 223 linker unit (this is because other build systems don't have a concept 224 equivalent to GN's source_set). This class navigates the transitive 225 dependencies (mostly static libraries) of a target and detects if multiple 226 paths end up including the same file group. This is to avoid situations like: 227 228 traced.exe -> base(file group) 229 traced.exe -> libperfetto(static lib) -> base(file group) 230 """ 231 232 def __init__(self, gn: 'GnParser', target_name: str): 233 self.gn = gn 234 self.root = gn.get_target(target_name) 235 self.source_sets: Dict[str, Set[str]] = collections.defaultdict(set) 236 self.deps_visited = set() 237 self.source_set_hdr_only = {} 238 239 self._visit(target_name) 240 num_violations = 0 241 if target_name in ODR_VIOLATION_IGNORE_TARGETS: 242 return 243 for sset, paths in self.source_sets.items(): 244 if self.is_header_only(sset): 245 continue 246 if len(paths) != 1: 247 num_violations += 1 248 print( 249 'ODR violation in target %s, multiple paths include %s:\n %s' % 250 (target_name, sset, '\n '.join(paths)), 251 file=sys.stderr) 252 if num_violations > 0: 253 raise Exception('%d ODR violations detected. Build generation aborted' % 254 num_violations) 255 256 def _visit(self, target_name: str, parent_path=''): 257 target = self.gn.get_target(target_name) 258 path = ((parent_path + ' > ') if parent_path else '') + target_name 259 if not target: 260 raise Exception('Cannot find target %s' % target_name) 261 for ssdep in target.transitive_source_set_deps(): 262 name_and_path = '%s (via %s)' % (target_name, path) 263 self.source_sets[ssdep.name].add(name_and_path) 264 deps = set(target.non_proto_or_source_set_deps()).union( 265 target.transitive_proto_deps()) - self.deps_visited 266 for dep in deps: 267 if dep.type == 'executable': 268 continue # Execs are strong boundaries and don't cause ODR violations. 269 # static_library dependencies should reset the path. It doesn't matter if 270 # we get to a source file via: 271 # source_set1 > static_lib > source.cc OR 272 # source_set1 > source_set2 > static_lib > source.cc 273 # This is NOT an ODR violation because source.cc is linked from the same 274 # static library 275 next_parent_path = path if dep.type != 'static_library' else '' 276 self.deps_visited.add(dep.name) 277 self._visit(dep.name, next_parent_path) 278 279 def is_header_only(self, source_set_name: str): 280 cached = self.source_set_hdr_only.get(source_set_name) 281 if cached is not None: 282 return cached 283 target = self.gn.get_target(source_set_name) 284 if target.type != 'source_set': 285 raise TypeError('%s is not a source_set' % source_set_name) 286 res = all(src.endswith('.h') for src in target.sources) 287 self.source_set_hdr_only[source_set_name] = res 288 return res 289 290 291class GnParser(object): 292 """A parser with some cleverness for GN json desc files 293 294 The main goals of this parser are: 295 1) Deal with the fact that other build systems don't have an equivalent 296 notion to GN's source_set. Conversely to Bazel's and Soong's filegroups, 297 GN source_sets expect that dependencies, cflags and other source_set 298 properties propagate up to the linker unit (static_library, executable or 299 shared_library). This parser simulates the same behavior: when a 300 source_set is encountered, some of its variables (cflags and such) are 301 copied up to the dependent targets. This is to allow gen_xxx to create 302 one filegroup for each source_set and then squash all the other flags 303 onto the linker unit. 304 2) Detect and special-case protobuf targets, figuring out the protoc-plugin 305 being used. 306 """ 307 308 class Target(object): 309 """Reperesents A GN target. 310 311 Maked properties are propagated up the dependency chain when a 312 source_set dependency is encountered. 313 """ 314 315 def __init__(self, name, type): 316 self.name = name # e.g. //src/ipc:ipc 317 318 VALID_TYPES = ('static_library', 'shared_library', 'executable', 'group', 319 'action', 'source_set', 'proto_library', 'generated_file') 320 assert (type in VALID_TYPES) 321 self.type = type 322 self.testonly = False 323 self.toolchain = None 324 325 # These are valid only for type == proto_library. 326 # This is typically: 'proto', 'protozero', 'ipc'. 327 self.proto_plugin: Optional[str] = None 328 self.proto_paths = set() 329 self.proto_exports = set() 330 331 self.sources = set() 332 # TODO(primiano): consider whether the public section should be part of 333 # bubbled-up sources. 334 self.public_headers = set() # 'public' 335 336 # These are valid only for type == 'action' 337 self.data = set() 338 self.inputs = set() 339 self.outputs = set() 340 self.script = None 341 self.args = [] 342 self.custom_action_type = None 343 self.python_main = None 344 345 # These variables are propagated up when encountering a dependency 346 # on a source_set target. 347 self.cflags = set() 348 self.defines = set() 349 self.deps: Set[GnParser.Target] = set() 350 self.transitive_deps: Set[GnParser.Target] = set() 351 self.libs = set() 352 self.include_dirs = set() 353 self.ldflags = set() 354 355 # Deps on //gn:xxx have this flag set to True. These dependencies 356 # are special because they pull third_party code from buildtools/. 357 # We don't want to keep recursing into //buildtools in generators, 358 # this flag is used to stop the recursion and create an empty 359 # placeholder target once we hit //gn:protoc or similar. 360 self.is_third_party_dep_ = False 361 362 def non_proto_or_source_set_deps(self): 363 return set(d for d in self.deps 364 if d.type != 'proto_library' and d.type != 'source_set') 365 366 def proto_deps(self): 367 return set(d for d in self.deps if d.type == 'proto_library') 368 369 def transitive_proto_deps(self): 370 return set(d for d in self.transitive_deps if d.type == 'proto_library') 371 372 def transitive_cpp_proto_deps(self): 373 return set( 374 d for d in self.transitive_deps if d.type == 'proto_library' and 375 d.proto_plugin != 'descriptor' and d.proto_plugin != 'source_set') 376 377 def transitive_source_set_deps(self): 378 return set(d for d in self.transitive_deps if d.type == 'source_set') 379 380 def __lt__(self, other): 381 if isinstance(other, self.__class__): 382 return self.name < other.name 383 raise TypeError( 384 '\'<\' not supported between instances of \'%s\' and \'%s\'' % 385 (type(self).__name__, type(other).__name__)) 386 387 def __repr__(self): 388 return json.dumps( 389 { 390 k: (list(sorted(v)) if isinstance(v, set) else v) 391 for (k, v) in iteritems(self.__dict__) 392 }, 393 indent=4, 394 sort_keys=True) 395 396 def update(self, other): 397 for key in ('cflags', 'data', 'defines', 'deps', 'include_dirs', 398 'ldflags', 'transitive_deps', 'libs', 'proto_paths'): 399 self.__dict__[key].update(other.__dict__.get(key, [])) 400 401 def __init__(self, gn_desc): 402 self.gn_desc_ = gn_desc 403 self.all_targets = {} 404 self.linker_units = {} # Executables, shared or static libraries. 405 self.source_sets = {} 406 self.actions = {} 407 self.proto_libs = {} 408 409 def get_target(self, gn_target_name: str) -> Target: 410 """Returns a Target object from the fully qualified GN target name. 411 412 It bubbles up variables from source_set dependencies as described in the 413 class-level comments. 414 """ 415 target = self.all_targets.get(gn_target_name) 416 if target is not None: 417 return target # Target already processed. 418 419 desc = self.gn_desc_.get(gn_target_name) 420 if not desc: 421 return None 422 423 target = GnParser.Target(gn_target_name, desc['type']) 424 target.testonly = desc.get('testonly', False) 425 target.toolchain = desc.get('toolchain', None) 426 self.all_targets[gn_target_name] = target 427 428 # We should never have GN targets directly depend on buidtools. They 429 # should hop via //gn:xxx, so we can give generators an opportunity to 430 # override them. 431 assert (not gn_target_name.startswith('//buildtools')) 432 433 # Don't descend further into third_party targets. Genrators are supposed 434 # to either ignore them or route to other externally-provided targets. 435 if gn_target_name.startswith('//gn'): 436 target.is_third_party_dep_ = True 437 return target 438 439 proto_target_type, proto_desc = self.get_proto_target_type(target) 440 if proto_target_type: 441 assert proto_desc 442 self.proto_libs[target.name] = target 443 target.type = 'proto_library' 444 target.proto_plugin = proto_target_type 445 target.proto_paths.update(self.get_proto_paths(proto_desc)) 446 target.proto_exports.update(self.get_proto_exports(proto_desc)) 447 target.sources.update( 448 self.get_proto_sources(proto_target_type, proto_desc)) 449 assert (all(x.endswith('.proto') for x in target.sources)) 450 elif target.type == 'source_set': 451 self.source_sets[gn_target_name] = target 452 target.sources.update(desc.get('sources', [])) 453 target.inputs.update(desc.get('inputs', [])) 454 elif target.type in LINKER_UNIT_TYPES: 455 self.linker_units[gn_target_name] = target 456 target.sources.update(desc.get('sources', [])) 457 elif target.type == 'action': 458 self.actions[gn_target_name] = target 459 target.data.update(desc.get('metadata', {}).get('perfetto_data', [])) 460 target.inputs.update(desc.get('inputs', [])) 461 target.sources.update(desc.get('sources', [])) 462 outs = [re.sub('^//out/.+?/gen/', '', x) for x in desc['outputs']] 463 target.outputs.update(outs) 464 target.script = desc['script'] 465 # Args are typically relative to the root build dir (../../xxx) 466 # because root build dir is typically out/xxx/). 467 target.args = [re.sub('^../../', '//', x) for x in desc['args']] 468 action_types = desc.get('metadata', 469 {}).get('perfetto_action_type_for_generator', []) 470 target.custom_action_type = action_types[0] if len( 471 action_types) > 0 else None 472 python_main = desc.get('metadata', {}).get('perfetto_python_main', []) 473 target.python_main = python_main[0] if python_main else None 474 475 # Default for 'public' is //* - all headers in 'sources' are public. 476 # TODO(primiano): if a 'public' section is specified (even if empty), then 477 # the rest of 'sources' is considered inaccessible by gn. Consider 478 # emulating that, so that generated build files don't end up with overly 479 # accessible headers. 480 public_headers = [x for x in desc.get('public', []) if x != '*'] 481 target.public_headers.update(public_headers) 482 483 target.cflags.update(desc.get('cflags', []) + desc.get('cflags_cc', [])) 484 target.libs.update(desc.get('libs', [])) 485 target.ldflags.update(desc.get('ldflags', [])) 486 target.defines.update(desc.get('defines', [])) 487 target.include_dirs.update(desc.get('include_dirs', [])) 488 489 # Recurse in dependencies. 490 for dep_name in desc.get('deps', []): 491 dep = self.get_target(dep_name) 492 493 # generated_file targets only exist for GN builds: we can safely ignore 494 # them. 495 if dep.type == 'generated_file': 496 continue 497 498 # When a proto_library depends on an action, that is always the "_gen" 499 # rule of the action which is "private" to the proto_library rule. 500 # therefore, just ignore it for dep tracking purposes. 501 if dep.type == 'action' and proto_target_type is not None: 502 target_no_toolchain = label_without_toolchain(target.name) 503 dep_no_toolchain = label_without_toolchain(dep.name) 504 assert (dep_no_toolchain == f'{target_no_toolchain}_gen') 505 continue 506 507 # Non-third party groups are only used for bubbling cflags etc so don't 508 # add a dep. 509 if dep.type == 'group' and not dep.is_third_party_dep_: 510 target.update(dep) # Bubble up groups's cflags/ldflags etc. 511 continue 512 513 # Linker units act as a hard boundary making all their internal deps 514 # opaque to the outside world. For this reason, do not propogate deps 515 # transitively across them. 516 if dep.type in LINKER_UNIT_TYPES: 517 target.deps.add(dep) 518 continue 519 520 if dep.type == 'source_set': 521 target.update(dep) # Bubble up source set's cflags/ldflags etc. 522 elif dep.type == 'proto_library': 523 target.proto_paths.update(dep.proto_paths) 524 525 target.deps.add(dep) 526 target.transitive_deps.add(dep) 527 target.transitive_deps.update(dep.transitive_deps) 528 529 return target 530 531 def get_proto_exports(self, proto_desc): 532 # exports in metadata will be available for source_set targets. 533 metadata = proto_desc.get('metadata', {}) 534 return metadata.get('exports', []) 535 536 def get_proto_paths(self, proto_desc): 537 metadata = proto_desc.get('metadata', {}) 538 return metadata.get('proto_import_dirs', []) 539 540 def get_proto_sources(self, proto_target_type, proto_desc): 541 if proto_target_type == 'source_set': 542 metadata = proto_desc.get('metadata', {}) 543 return metadata.get('proto_library_sources', []) 544 return proto_desc.get('sources', []) 545 546 def get_proto_target_type( 547 self, target: Target) -> Tuple[Optional[str], Optional[Dict]]: 548 """ Checks if the target is a proto library and return the plugin. 549 550 Returns: 551 (None, None): if the target is not a proto library. 552 (plugin, proto_desc) where |plugin| is 'proto' in the default (lite) 553 case or 'protozero' or 'ipc' or 'descriptor'; |proto_desc| is the GN 554 json desc of the target with the .proto sources (_gen target for 555 non-descriptor types or the target itself for descriptor type). 556 """ 557 parts = target.name.split('(', 1) 558 name = parts[0] 559 toolchain = '(' + parts[1] if len(parts) > 1 else '' 560 561 # Descriptor targets don't have a _gen target; instead we look for the 562 # characteristic flag in the args of the target itself. 563 desc = self.gn_desc_.get(target.name) 564 if '--descriptor_set_out' in desc.get('args', []): 565 return 'descriptor', desc 566 567 # Source set proto targets have a non-empty proto_library_sources in the 568 # metadata of the description. 569 metadata = desc.get('metadata', {}) 570 if 'proto_library_sources' in metadata: 571 return 'source_set', desc 572 573 # In all other cases, we want to look at the _gen target as that has the 574 # important information. 575 gen_desc = self.gn_desc_.get('%s_gen%s' % (name, toolchain)) 576 if gen_desc is None or gen_desc['type'] != 'action': 577 return None, None 578 args = gen_desc.get('args', []) 579 if '/protoc' not in args[0]: 580 return None, None 581 plugin = 'proto' 582 for arg in (arg for arg in args if arg.startswith('--plugin=')): 583 # |arg| at this point looks like: 584 # --plugin=protoc-gen-plugin=gcc_like_host/protozero_plugin 585 # or 586 # --plugin=protoc-gen-plugin=protozero_plugin 587 plugin = arg.split('=')[-1].split('/')[-1].replace('_plugin', '') 588 return plugin, gen_desc 589