1#!/usr/bin/env python3 2# Copyright (C) 2019 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16# This tool uses a collection of BUILD.gn files and build targets to generate 17# an "amalgamated" C++ header and source file pair which compiles to an 18# equivalent program. The tool also outputs the necessary compiler and linker 19# flags needed to compile the resulting source code. 20 21from __future__ import print_function 22import argparse 23import os 24import re 25import shutil 26import subprocess 27import sys 28import tempfile 29 30import gn_utils 31 32# Default targets to include in the result. 33# TODO(primiano): change this script to recurse into target deps when generating 34# headers, but only for proto targets. .pbzero.h files don't include each other 35# and we need to list targets here individually, which is unmaintainable. 36default_targets = [ 37 '//:libperfetto_client_experimental', 38 '//include/perfetto/protozero:protozero', 39 '//protos/perfetto/config:zero', 40 '//protos/perfetto/trace:zero', 41] 42 43# Arguments for the GN output directory (unless overridden from the command 44# line). 45gn_args = ' '.join([ 46 'enable_perfetto_ipc=true', 47 'enable_perfetto_zlib=false', 48 'is_debug=false', 49 'is_perfetto_build_generator=true', 50 'is_perfetto_embedder=true', 51 'perfetto_enable_git_rev_version_header=true', 52 'use_custom_libcxx=false', 53]) 54 55# By default, the amalgamated .h only recurses in #includes but not in the 56# target deps. In the case of protos we want to follow deps even in lieu of 57# direct #includes. This is because, by design, protozero headers don't 58# include each other but rely on forward declarations. The alternative would 59# be adding each proto sub-target individually (e.g. //proto/trace/gpu:zero), 60# but doing that is unmaintainable. We also do this for cpp bindings since some 61# tracing SDK functions depend on them (and the system tracing IPC mechanism 62# does so too). 63recurse_in_header_deps = '^//protos/.*(cpp|zero)$' 64 65# Compiler flags which aren't filtered out. 66cflag_allowlist = r'^-(W.*|fno-exceptions|fPIC|std.*|fvisibility.*)$' 67 68# Linker flags which aren't filtered out. 69ldflag_allowlist = r'^-()$' 70 71# Libraries which are filtered out. 72lib_denylist = r'^(c|gcc_eh)$' 73 74# Macros which aren't filtered out. 75define_allowlist = r'^(PERFETTO.*|GOOGLE_PROTOBUF.*)$' 76 77# Includes which will be removed from the generated source. 78includes_to_remove = r'^(gtest).*$' 79 80# From //gn:default_config (since "gn desc" doesn't describe configs). 81default_includes = [ 82 'include', 83] 84 85default_cflags = [ 86 # Since we're expanding header files into the generated source file, some 87 # constant may remain unused. 88 '-Wno-unused-const-variable' 89] 90 91# Build flags to satisfy a protobuf (lite or full) dependency. 92protobuf_cflags = [ 93 # Note that these point to the local copy of protobuf in buildtools. In 94 # reality the user of the amalgamated result will have to provide a path to 95 # an installed copy of the exact same version of protobuf which was used to 96 # generate the amalgamated build. 97 '-isystembuildtools/protobuf/src', 98 '-Lbuildtools/protobuf/src/.libs', 99 # We also need to disable some warnings for protobuf. 100 '-Wno-missing-prototypes', 101 '-Wno-missing-variable-declarations', 102 '-Wno-sign-conversion', 103 '-Wno-unknown-pragmas', 104 '-Wno-unused-macros', 105] 106 107# A mapping of dependencies to system libraries. Libraries in this map will not 108# be built statically but instead added as dependencies of the amalgamated 109# project. 110system_library_map = { 111 '//buildtools:protobuf_full': { 112 'libs': ['protobuf'], 113 'cflags': protobuf_cflags, 114 }, 115 '//buildtools:protobuf_lite': { 116 'libs': ['protobuf-lite'], 117 'cflags': protobuf_cflags, 118 }, 119 '//buildtools:protoc_lib': { 120 'libs': ['protoc'] 121 }, 122} 123 124# ---------------------------------------------------------------------------- 125# End of configuration. 126# ---------------------------------------------------------------------------- 127 128tool_name = os.path.basename(__file__) 129project_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) 130preamble = """// Copyright (C) 2019 The Android Open Source Project 131// 132// Licensed under the Apache License, Version 2.0 (the "License"); 133// you may not use this file except in compliance with the License. 134// You may obtain a copy of the License at 135// 136// http://www.apache.org/licenses/LICENSE-2.0 137// 138// Unless required by applicable law or agreed to in writing, software 139// distributed under the License is distributed on an "AS IS" BASIS, 140// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 141// See the License for the specific language governing permissions and 142// limitations under the License. 143// 144// This file is automatically generated by %s. Do not edit. 145""" % tool_name 146 147 148def apply_denylist(denylist, items): 149 return [item for item in items if not re.match(denylist, item)] 150 151 152def apply_allowlist(allowlist, items): 153 return [item for item in items if re.match(allowlist, item)] 154 155 156def normalize_path(path): 157 path = os.path.relpath(path, project_root) 158 path = re.sub(r'^out/[^/]+/', '', path) 159 return path 160 161 162class Error(Exception): 163 pass 164 165 166class DependencyNode(object): 167 """A target in a GN build description along with its dependencies.""" 168 169 def __init__(self, target_name): 170 self.target_name = target_name 171 self.dependencies = set() 172 173 def add_dependency(self, target_node): 174 if target_node in self.dependencies: 175 return 176 self.dependencies.add(target_node) 177 178 def iterate_depth_first(self): 179 for node in sorted(self.dependencies, key=lambda n: n.target_name): 180 for node in node.iterate_depth_first(): 181 yield node 182 if self.target_name: 183 yield self 184 185 186class DependencyTree(object): 187 """A tree of GN build target dependencies.""" 188 189 def __init__(self): 190 self.target_to_node_map = {} 191 self.root = self._get_or_create_node(None) 192 193 def _get_or_create_node(self, target_name): 194 if target_name in self.target_to_node_map: 195 return self.target_to_node_map[target_name] 196 node = DependencyNode(target_name) 197 self.target_to_node_map[target_name] = node 198 return node 199 200 def add_dependency(self, from_target, to_target): 201 from_node = self._get_or_create_node(from_target) 202 to_node = self._get_or_create_node(to_target) 203 assert from_node is not to_node 204 from_node.add_dependency(to_node) 205 206 def iterate_depth_first(self): 207 for node in self.root.iterate_depth_first(): 208 yield node 209 210 211class AmalgamatedProject(object): 212 """In-memory representation of an amalgamated source/header pair.""" 213 214 def __init__(self, desc, source_deps, compute_deps_only=False): 215 """Constructor. 216 217 Args: 218 desc: JSON build description. 219 source_deps: A map of (source file, [dependency header]) which is 220 to detect which header files are included by each source file. 221 compute_deps_only: If True, the project will only be used to compute 222 dependency information. Use |get_source_files()| to retrieve 223 the result. 224 """ 225 self.desc = desc 226 self.source_deps = source_deps 227 self.header = [] 228 self.source = [] 229 self.source_defines = [] 230 # Note that we don't support multi-arg flags. 231 self.cflags = set(default_cflags) 232 self.ldflags = set() 233 self.defines = set() 234 self.libs = set() 235 self._dependency_tree = DependencyTree() 236 self._processed_sources = set() 237 self._processed_headers = set() 238 self._processed_header_deps = set() 239 self._processed_source_headers = set() # Header files included from .cc 240 self._include_re = re.compile(r'#include "(.*)"') 241 self._compute_deps_only = compute_deps_only 242 243 def add_target(self, target_name): 244 """Include |target_name| in the amalgamated result.""" 245 self._dependency_tree.add_dependency(None, target_name) 246 self._add_target_dependencies(target_name) 247 self._add_target_flags(target_name) 248 self._add_target_headers(target_name) 249 250 # Recurse into target deps, but only for protos. This generates headers 251 # for all the .{pbzero,gen}.h files, even if they don't #include each other. 252 for _, dep in self._iterate_dep_edges(target_name): 253 if (dep not in self._processed_header_deps and 254 re.match(recurse_in_header_deps, dep)): 255 self._processed_header_deps.add(dep) 256 self.add_target(dep) 257 258 def _iterate_dep_edges(self, target_name): 259 target = self.desc[target_name] 260 for dep in target.get('deps', []): 261 # Ignore system libraries since they will be added as build-time 262 # dependencies. 263 if dep in system_library_map: 264 continue 265 # Don't descend into build action dependencies. 266 if self.desc[dep]['type'] == 'action': 267 continue 268 for sub_target, sub_dep in self._iterate_dep_edges(dep): 269 yield sub_target, sub_dep 270 yield target_name, dep 271 272 def _iterate_target_and_deps(self, target_name): 273 yield target_name 274 for _, dep in self._iterate_dep_edges(target_name): 275 yield dep 276 277 def _add_target_dependencies(self, target_name): 278 for target, dep in self._iterate_dep_edges(target_name): 279 self._dependency_tree.add_dependency(target, dep) 280 281 def process_dep(dep): 282 if dep in system_library_map: 283 self.libs.update(system_library_map[dep].get('libs', [])) 284 self.cflags.update(system_library_map[dep].get('cflags', [])) 285 self.defines.update(system_library_map[dep].get('defines', [])) 286 return True 287 288 def walk_all_deps(target_name): 289 target = self.desc[target_name] 290 for dep in target.get('deps', []): 291 if process_dep(dep): 292 return 293 walk_all_deps(dep) 294 295 walk_all_deps(target_name) 296 297 def _filter_cflags(self, cflags): 298 # Since we want to deduplicate flags, combine two-part switches (e.g., 299 # "-foo bar") into one value ("-foobar") so we can store the result as 300 # a set. 301 result = [] 302 for flag in cflags: 303 if flag.startswith('-'): 304 result.append(flag) 305 else: 306 result[-1] += flag 307 return apply_allowlist(cflag_allowlist, result) 308 309 def _add_target_flags(self, target_name): 310 for target_name in self._iterate_target_and_deps(target_name): 311 target = self.desc[target_name] 312 self.cflags.update(self._filter_cflags(target.get('cflags', []))) 313 self.cflags.update(self._filter_cflags(target.get('cflags_cc', []))) 314 self.ldflags.update( 315 apply_allowlist(ldflag_allowlist, target.get('ldflags', []))) 316 self.libs.update(apply_denylist(lib_denylist, target.get('libs', []))) 317 self.defines.update( 318 apply_allowlist(define_allowlist, target.get('defines', []))) 319 320 def _add_target_headers(self, target_name): 321 target = self.desc[target_name] 322 if not 'sources' in target: 323 return 324 headers = [ 325 gn_utils.label_to_path(s) for s in target['sources'] if s.endswith('.h') 326 ] 327 for header in headers: 328 self._add_header(target_name, header) 329 330 def _get_include_dirs(self, target_name): 331 include_dirs = set(default_includes) 332 for target_name in self._iterate_target_and_deps(target_name): 333 target = self.desc[target_name] 334 if 'include_dirs' in target: 335 include_dirs.update( 336 [gn_utils.label_to_path(d) for d in target['include_dirs']]) 337 return include_dirs 338 339 def _add_source_included_header(self, include_dirs, allowed_files, 340 header_name): 341 for include_dir in include_dirs: 342 rel_path = os.path.join(include_dir, header_name) 343 full_path = os.path.join(gn_utils.repo_root(), rel_path) 344 if os.path.exists(full_path): 345 if not rel_path in allowed_files: 346 return 347 if full_path in self._processed_headers: 348 return 349 if full_path in self._processed_source_headers: 350 return 351 self._processed_source_headers.add(full_path) 352 with open(full_path) as f: 353 self.source.append('// %s begin header: %s' % 354 (tool_name, normalize_path(full_path))) 355 self.source.extend( 356 self._process_source_includes(include_dirs, allowed_files, f)) 357 return 358 if self._compute_deps_only: 359 return 360 msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs) 361 raise Error('Header file %s not found. %s' % (header_name, msg)) 362 363 def _add_source(self, target_name, source_name): 364 if source_name in self._processed_sources: 365 return 366 self._processed_sources.add(source_name) 367 include_dirs = self._get_include_dirs(target_name) 368 deps = self.source_deps[source_name] 369 full_path = os.path.join(gn_utils.repo_root(), source_name) 370 if not os.path.exists(full_path): 371 raise Error('Source file %s not found' % source_name) 372 with open(full_path) as f: 373 self.source.append('// %s begin source: %s' % 374 (tool_name, normalize_path(full_path))) 375 try: 376 self.source.extend( 377 self._patch_source( 378 source_name, 379 self._process_source_includes(include_dirs, deps, f))) 380 except Error as e: 381 raise Error('Failed adding source %s: %s' % (source_name, e)) 382 383 def _add_header_included_header(self, include_dirs, header_name): 384 for include_dir in include_dirs: 385 full_path = os.path.join(gn_utils.repo_root(), include_dir, header_name) 386 if os.path.exists(full_path): 387 if full_path in self._processed_headers: 388 return 389 self._processed_headers.add(full_path) 390 with open(full_path) as f: 391 self.header.append('// %s begin header: %s' % 392 (tool_name, normalize_path(full_path))) 393 self.header.extend(self._process_header_includes(include_dirs, f)) 394 return 395 if self._compute_deps_only: 396 return 397 msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs) 398 raise Error('Header file %s not found. %s' % (header_name, msg)) 399 400 def _add_header(self, target_name, header_name): 401 include_dirs = self._get_include_dirs(target_name) 402 full_path = os.path.join(gn_utils.repo_root(), header_name) 403 if full_path in self._processed_headers: 404 return 405 self._processed_headers.add(full_path) 406 if not os.path.exists(full_path): 407 if self._compute_deps_only: 408 return 409 raise Error('Header file %s not found' % header_name) 410 with open(full_path) as f: 411 self.header.append('// %s begin header: %s' % 412 (tool_name, normalize_path(full_path))) 413 try: 414 self.header.extend(self._process_header_includes(include_dirs, f)) 415 except Error as e: 416 raise Error('Failed adding header %s: %s' % (header_name, e)) 417 418 def _patch_source(self, source_name, lines): 419 result = [] 420 namespace = re.sub(r'[^a-z]', '_', 421 os.path.splitext(os.path.basename(source_name))[0]) 422 for line in lines: 423 # Protobuf generates an identical anonymous function into each 424 # message description. Rename all but the first occurrence to avoid 425 # duplicate symbol definitions. 426 line = line.replace('MergeFromFail', '%s_MergeFromFail' % namespace) 427 result.append(line) 428 return result 429 430 def _process_source_includes(self, include_dirs, allowed_files, file): 431 result = [] 432 for line in file: 433 line = line.rstrip('\n') 434 m = self._include_re.match(line) 435 if not m: 436 result.append(line) 437 continue 438 elif re.match(includes_to_remove, m.group(1)): 439 result.append('// %s removed: %s' % (tool_name, line)) 440 else: 441 result.append('// %s expanded: %s' % (tool_name, line)) 442 self._add_source_included_header(include_dirs, allowed_files, 443 m.group(1)) 444 return result 445 446 def _process_header_includes(self, include_dirs, file): 447 result = [] 448 for line in file: 449 line = line.rstrip('\n') 450 m = self._include_re.match(line) 451 if not m: 452 result.append(line) 453 continue 454 elif re.match(includes_to_remove, m.group(1)): 455 result.append('// %s removed: %s' % (tool_name, line)) 456 else: 457 result.append('// %s expanded: %s' % (tool_name, line)) 458 self._add_header_included_header(include_dirs, m.group(1)) 459 return result 460 461 def generate(self): 462 """Prepares the output for this amalgamated project. 463 464 Call save() to persist the result. 465 """ 466 assert not self._compute_deps_only 467 self.source_defines.append('// %s: predefined macros' % tool_name) 468 469 def add_define(name): 470 # Valued macros aren't supported for now. 471 assert '=' not in name 472 self.source_defines.append('#if !defined(%s)' % name) 473 self.source_defines.append('#define %s' % name) 474 self.source_defines.append('#endif') 475 476 for name in self.defines: 477 add_define(name) 478 for target_name, source_name in self.get_source_files(): 479 self._add_source(target_name, source_name) 480 481 def get_source_files(self): 482 """Return a list of (target, [source file]) that describes the source 483 files pulled in by each target which is a dependency of this project. 484 """ 485 source_files = [] 486 for node in self._dependency_tree.iterate_depth_first(): 487 target = self.desc[node.target_name] 488 if not 'sources' in target: 489 continue 490 sources = [(node.target_name, gn_utils.label_to_path(s)) 491 for s in target['sources'] 492 if s.endswith('.cc')] 493 source_files.extend(sources) 494 return source_files 495 496 def _get_nice_path(self, prefix, format): 497 basename = os.path.basename(prefix) 498 return os.path.join( 499 os.path.relpath(os.path.dirname(prefix)), format % basename) 500 501 def _make_directories(self, directory): 502 if not os.path.isdir(directory): 503 os.makedirs(directory) 504 505 def save(self, output_prefix, system_buildtools=False): 506 """Save the generated header and source file pair. 507 508 Returns a message describing the output with build instructions. 509 """ 510 header_file = self._get_nice_path(output_prefix, '%s.h') 511 source_file = self._get_nice_path(output_prefix, '%s.cc') 512 self._make_directories(os.path.dirname(header_file)) 513 self._make_directories(os.path.dirname(source_file)) 514 with open(header_file, 'w') as f: 515 f.write('\n'.join([preamble] + self.header + ['\n'])) 516 with open(source_file, 'w') as f: 517 include_stmt = '#include "%s"' % os.path.basename(header_file) 518 f.write('\n'.join([preamble] + self.source_defines + [include_stmt] + 519 self.source + ['\n'])) 520 build_cmd = self.get_build_command(output_prefix, system_buildtools) 521 return """Amalgamated project written to %s and %s. 522 523Build settings: 524 - cflags: %s 525 - ldflags: %s 526 - libs: %s 527 528Example build command: 529 530%s 531""" % (header_file, source_file, ' '.join(self.cflags), ' '.join( 532 self.ldflags), ' '.join(self.libs), ' '.join(build_cmd)) 533 534 def get_build_command(self, output_prefix, system_buildtools=False): 535 """Returns an example command line for building the output source.""" 536 source = self._get_nice_path(output_prefix, '%s.cc') 537 library = self._get_nice_path(output_prefix, 'lib%s.so') 538 539 if sys.platform.startswith('linux') and not system_buildtools: 540 llvm_script = os.path.join(gn_utils.repo_root(), 'gn', 'standalone', 541 'toolchain', 'linux_find_llvm.py') 542 cxx = subprocess.check_output([llvm_script]).splitlines()[2].decode() 543 else: 544 cxx = 'clang++' 545 546 build_cmd = [cxx, source, '-o', library, '-shared'] + \ 547 sorted(self.cflags) + sorted(self.ldflags) 548 for lib in sorted(self.libs): 549 build_cmd.append('-l%s' % lib) 550 return build_cmd 551 552 553def main(): 554 parser = argparse.ArgumentParser( 555 description='Generate an amalgamated header/source pair from a GN ' 556 'build description.') 557 parser.add_argument( 558 '--out', 559 help='The name of the temporary build folder in \'out\'', 560 default='tmp.gen_amalgamated.%u' % os.getpid()) 561 parser.add_argument( 562 '--output', 563 help='Base name of files to create. A .cc/.h extension will be added', 564 default=os.path.join(gn_utils.repo_root(), 'out/amalgamated/perfetto')) 565 parser.add_argument( 566 '--gn_args', 567 help='GN arguments used to prepare the output directory', 568 default=gn_args) 569 parser.add_argument( 570 '--keep', 571 help='Don\'t delete the GN output directory at exit', 572 action='store_true') 573 parser.add_argument( 574 '--build', help='Also compile the generated files', action='store_true') 575 parser.add_argument( 576 '--check', help='Don\'t keep the generated files', action='store_true') 577 parser.add_argument('--quiet', help='Only report errors', action='store_true') 578 parser.add_argument( 579 '--dump-deps', 580 help='List all source files that the amalgamated output depends on', 581 action='store_true') 582 parser.add_argument( 583 '--system_buildtools', 584 help='Use the buildtools (e.g. gn) preinstalled in the system instead ' 585 'of the hermetic ones', 586 action='store_true') 587 parser.add_argument( 588 'targets', 589 nargs=argparse.REMAINDER, 590 help='Targets to include in the output (e.g., "//:libperfetto")') 591 args = parser.parse_args() 592 targets = args.targets or default_targets 593 594 # The CHANGELOG mtime triggers the perfetto_version.gen.h genrule. This is 595 # to avoid emitting a stale version information in the remote case of somebody 596 # running gen_amalgamated incrementally after having moved to another commit. 597 changelog_path = os.path.join(project_root, 'CHANGELOG') 598 assert (os.path.exists(changelog_path)) 599 subprocess.check_call(['touch', '-c', changelog_path]) 600 601 output = args.output 602 if args.check: 603 output = os.path.join(tempfile.mkdtemp(), 'perfetto_amalgamated') 604 605 out = gn_utils.prepare_out_directory(args.gn_args, 606 args.out, 607 system_buildtools=args.system_buildtools) 608 if not args.quiet: 609 print('Building project...') 610 try: 611 desc = gn_utils.load_build_description(out, args.system_buildtools) 612 613 # We need to build everything first so that the necessary header 614 # dependencies get generated. However if we are just dumping dependency 615 # information this can be skipped, allowing cross-platform operation. 616 if not args.dump_deps: 617 gn_utils.build_targets(out, targets, 618 system_buildtools=args.system_buildtools) 619 source_deps = gn_utils.compute_source_dependencies(out, 620 args.system_buildtools) 621 project = AmalgamatedProject( 622 desc, source_deps, compute_deps_only=args.dump_deps) 623 624 for target in targets: 625 project.add_target(target) 626 627 if args.dump_deps: 628 source_files = [ 629 source_file for _, source_file in project.get_source_files() 630 ] 631 print('\n'.join(sorted(set(source_files)))) 632 return 633 634 project.generate() 635 result = project.save(output, args.system_buildtools) 636 if not args.quiet: 637 print(result) 638 if args.build: 639 if not args.quiet: 640 sys.stdout.write('Building amalgamated project...') 641 sys.stdout.flush() 642 subprocess.check_call(project.get_build_command(output, 643 args.system_buildtools)) 644 if not args.quiet: 645 print('done') 646 finally: 647 if not args.keep: 648 shutil.rmtree(out) 649 if args.check: 650 shutil.rmtree(os.path.dirname(output)) 651 652 653if __name__ == '__main__': 654 sys.exit(main()) 655