1# Copyright 2023 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import argparse 6import collections 7import contextlib 8import hashlib 9import io 10import json 11import multiprocessing 12import os 13import re 14import shutil 15import subprocess 16import sys 17import tempfile 18 19 20class ArgumentForwarder(object): 21 """Class used to abstract forwarding arguments from to the swiftc compiler. 22 23 Arguments: 24 - arg_name: string corresponding to the argument to pass to the compiler 25 - arg_join: function taking the compiler name and returning whether the 26 argument value is attached to the argument or separated 27 - to_swift: function taking the argument value and returning whether it 28 must be passed to the swift compiler 29 - to_clang: function taking the argument value and returning whether it 30 must be passed to the clang compiler 31 """ 32 33 def __init__(self, arg_name, arg_join, to_swift, to_clang): 34 self._arg_name = arg_name 35 self._arg_join = arg_join 36 self._to_swift = to_swift 37 self._to_clang = to_clang 38 39 def forward(self, swiftc_args, values, target_triple): 40 if not values: 41 return 42 43 is_catalyst = target_triple.endswith('macabi') 44 for value in values: 45 if self._to_swift(value): 46 if self._arg_join('swift'): 47 swiftc_args.append(f'{self._arg_name}{value}') 48 else: 49 swiftc_args.append(self._arg_name) 50 swiftc_args.append(value) 51 52 if self._to_clang(value) and not is_catalyst: 53 if self._arg_join('clang'): 54 swiftc_args.append('-Xcc') 55 swiftc_args.append(f'{self._arg_name}{value}') 56 else: 57 swiftc_args.append('-Xcc') 58 swiftc_args.append(self._arg_name) 59 swiftc_args.append('-Xcc') 60 swiftc_args.append(value) 61 62 63class IncludeArgumentForwarder(ArgumentForwarder): 64 """Argument forwarder for -I and -isystem.""" 65 66 def __init__(self, arg_name): 67 ArgumentForwarder.__init__(self, 68 arg_name, 69 arg_join=lambda _: len(arg_name) == 1, 70 to_swift=lambda _: arg_name != '-isystem', 71 to_clang=lambda _: True) 72 73 74class FrameworkArgumentForwarder(ArgumentForwarder): 75 """Argument forwarder for -F and -Fsystem.""" 76 77 def __init__(self, arg_name): 78 ArgumentForwarder.__init__(self, 79 arg_name, 80 arg_join=lambda _: len(arg_name) == 1, 81 to_swift=lambda _: True, 82 to_clang=lambda _: True) 83 84 85class DefineArgumentForwarder(ArgumentForwarder): 86 """Argument forwarder for -D.""" 87 88 def __init__(self, arg_name): 89 ArgumentForwarder.__init__(self, 90 arg_name, 91 arg_join=lambda _: _ == 'clang', 92 to_swift=lambda _: '=' not in _, 93 to_clang=lambda _: True) 94 95 96# Dictionary mapping argument names to their ArgumentForwarder. 97ARGUMENT_FORWARDER_FOR_ATTR = ( 98 ('include_dirs', IncludeArgumentForwarder('-I')), 99 ('system_include_dirs', IncludeArgumentForwarder('-isystem')), 100 ('framework_dirs', FrameworkArgumentForwarder('-F')), 101 ('system_framework_dirs', FrameworkArgumentForwarder('-Fsystem')), 102 ('defines', DefineArgumentForwarder('-D')), 103) 104 105# Regexp used to parse #import lines. 106IMPORT_LINE_REGEXP = re.compile('#import "([^"]*)"') 107 108 109class FileWriter(contextlib.AbstractContextManager): 110 """ 111 FileWriter is a file-like object that only write data to disk if changed. 112 113 This object implements the context manager protocols and thus can be used 114 in a with-clause. The data is written to disk when the context is exited, 115 and only if the content is different from current file content. 116 117 with FileWriter(path) as stream: 118 stream.write('...') 119 120 If the with-clause ends with an exception, no data is written to the disk 121 and any existing file is left untouched. 122 """ 123 124 def __init__(self, filepath, encoding='utf8'): 125 self._stringio = io.StringIO() 126 self._filepath = filepath 127 self._encoding = encoding 128 129 def __exit__(self, exc_type, exc_value, traceback): 130 if exc_type or exc_value or traceback: 131 return 132 133 new_content = self._stringio.getvalue() 134 if os.path.exists(self._filepath): 135 with open(self._filepath, encoding=self._encoding) as stream: 136 old_content = stream.read() 137 138 if old_content == new_content: 139 return 140 141 with open(self._filepath, 'w', encoding=self._encoding) as stream: 142 stream.write(new_content) 143 144 def write(self, data): 145 self._stringio.write(data) 146 147 148@contextlib.contextmanager 149def existing_directory(path): 150 """Returns a context manager wrapping an existing directory.""" 151 yield path 152 153 154def create_stamp_file(path): 155 """Writes an empty stamp file at path.""" 156 with FileWriter(path) as stream: 157 stream.write('') 158 159 160def create_build_cache_dir(args, build_signature): 161 """Creates the build cache directory according to `args`. 162 163 This function returns an object that implements the context manager 164 protocol and thus can be used in a with-clause. If -derived-data-dir 165 argument is not used, the returned directory is a temporary directory 166 that will be deleted when the with-clause is exited. 167 """ 168 if not args.derived_data_dir: 169 return tempfile.TemporaryDirectory() 170 171 # The derived data cache can be quite large, so delete any obsolete 172 # files or directories. 173 stamp_name = f'{args.module_name}.stamp' 174 if os.path.isdir(args.derived_data_dir): 175 for name in os.listdir(args.derived_data_dir): 176 if name not in (build_signature, stamp_name): 177 path = os.path.join(args.derived_data_dir, name) 178 if os.path.isdir(path): 179 shutil.rmtree(path) 180 else: 181 os.unlink(path) 182 183 ensure_directory(args.derived_data_dir) 184 create_stamp_file(os.path.join(args.derived_data_dir, stamp_name)) 185 186 return existing_directory( 187 ensure_directory(os.path.join(args.derived_data_dir, build_signature))) 188 189 190def ensure_directory(path): 191 """Creates directory at `path` if it does not exists.""" 192 if not os.path.isdir(path): 193 os.makedirs(path) 194 return path 195 196 197def build_signature(env, args): 198 """Generates the build signature from `env` and `args`. 199 200 This allow re-using the derived data dir between builds while still 201 forcing the data to be recreated from scratch in case of significant 202 changes to the build settings (different arguments or tool versions). 203 """ 204 m = hashlib.sha1() 205 for key in sorted(env): 206 if key.endswith('_VERSION') or key == 'DEVELOPER_DIR': 207 m.update(f'{key}={env[key]}'.encode('utf8')) 208 for i, arg in enumerate(args): 209 m.update(f'{i}={arg}'.encode('utf8')) 210 return m.hexdigest() 211 212 213def generate_source_output_file_map_fragment(args, filename): 214 """Generates source OutputFileMap.json fragment according to `args`. 215 216 Create the fragment for a single .swift source file for OutputFileMap. 217 The output depends on whether -whole-module-optimization argument is 218 used or not. 219 """ 220 assert os.path.splitext(filename)[1] == '.swift', filename 221 basename = os.path.splitext(os.path.basename(filename))[0] 222 rel_name = os.path.join(args.target_out_dir, basename) 223 out_name = rel_name 224 225 fragment = { 226 'index-unit-output-path': f'/{rel_name}.o', 227 'object': f'{out_name}.o', 228 } 229 230 if not args.whole_module_optimization: 231 fragment.update({ 232 'const-values': f'{out_name}.swiftconstvalues', 233 'dependencies': f'{out_name}.d', 234 'diagnostics': f'{out_name}.dia', 235 'swift-dependencies': f'{out_name}.swiftdeps', 236 }) 237 238 return fragment 239 240 241def generate_module_output_file_map_fragment(args): 242 """Generates module OutputFileMap.json fragment according to `args`. 243 244 Create the fragment for the module itself for OutputFileMap. The output 245 depends on whether -whole-module-optimization argument is used or not. 246 """ 247 out_name = os.path.join(args.target_out_dir, args.module_name) 248 249 if args.whole_module_optimization: 250 fragment = { 251 'const-values': f'{out_name}.swiftconstvalues', 252 'dependencies': f'{out_name}.d', 253 'diagnostics': f'{out_name}.dia', 254 'swift-dependencies': f'{out_name}.swiftdeps', 255 } 256 else: 257 fragment = { 258 'emit-module-dependencies': f'{out_name}.d', 259 'emit-module-diagnostics': f'{out_name}.dia', 260 'swift-dependencies': f'{out_name}.swiftdeps', 261 } 262 263 return fragment 264 265 266def generate_output_file_map(args): 267 """Generates OutputFileMap.json according to `args`. 268 269 Returns the mapping as a python dictionary that can be serialized to 270 disk as JSON. 271 """ 272 output_file_map = {'': generate_module_output_file_map_fragment(args)} 273 for filename in args.sources: 274 fragment = generate_source_output_file_map_fragment(args, filename) 275 output_file_map[filename] = fragment 276 return output_file_map 277 278 279def fix_generated_header(header_path, output_path, src_dir, gen_dir): 280 """Fix the Objective-C header generated by the Swift compiler. 281 282 The Swift compiler assumes that the generated Objective-C header will be 283 imported from code compiled with module support enabled (-fmodules). The 284 generated code thus uses @import and provides no fallback if modules are 285 not enabled. 286 287 The Swift compiler also uses absolute path when including the bridging 288 header or another module's generated header. This causes issues with the 289 distributed compiler (i.e. reclient or siso) who expects all paths to be 290 relative to the build directory 291 292 This method fix the generated header to use relative path for #import 293 and to use #import instead of @import when using system frameworks. 294 295 The header is read at `header_path` and written to `output_path`. 296 """ 297 298 header_contents = [] 299 with open(header_path, 'r', encoding='utf8') as header_file: 300 301 imports_section = None 302 for line in header_file: 303 # Handle #import lines. 304 match = IMPORT_LINE_REGEXP.match(line) 305 if match: 306 import_path = match.group(1) 307 for root in (gen_dir, src_dir): 308 if import_path.startswith(root): 309 import_path = os.path.relpath(import_path, root) 310 if import_path != match.group(1): 311 span = match.span(1) 312 line = line[:span[0]] + import_path + line[span[1]:] 313 314 # Handle @import lines. 315 if line.startswith('#if __has_feature(objc_modules)'): 316 assert imports_section is None 317 imports_section = (len(header_contents) + 1, 1) 318 elif imports_section: 319 section_start, nesting_level = imports_section 320 if line.startswith('#if'): 321 imports_section = (section_start, nesting_level + 1) 322 elif line.startswith('#endif'): 323 if nesting_level > 1: 324 imports_section = (section_start, nesting_level - 1) 325 else: 326 imports_section = None 327 section_end = len(header_contents) 328 header_contents.append('#else\n') 329 for index in range(section_start, section_end): 330 l = header_contents[index] 331 if l.startswith('@import'): 332 name = l.split()[1].split(';')[0] 333 if name != 'ObjectiveC': 334 header_contents.append(f'#import <{name}/{name}.h>\n') 335 else: 336 header_contents.append(l) 337 338 header_contents.append(line) 339 340 with FileWriter(output_path) as header_file: 341 for line in header_contents: 342 header_file.write(line) 343 344 345def invoke_swift_compiler(args, extras_args, build_cache_dir, output_file_map): 346 """Invokes Swift compiler to compile module according to `args`. 347 348 The `build_cache_dir` and `output_file_map` should be path to existing 349 directory to use for writing intermediate build artifact (optionally 350 a temporary directory) and path to $module-OutputFileMap.json file that 351 lists the outputs to generate for the module and each source file. 352 353 If -fix-module-imports argument is passed, the generated header for the 354 module is written to a temporary location and then modified to replace 355 @import by corresponding #import. 356 """ 357 358 # Write the $module.SwiftFileList file. 359 swift_file_list_path = os.path.join(args.target_out_dir, 360 f'{args.module_name}.SwiftFileList') 361 362 with FileWriter(swift_file_list_path) as stream: 363 for filename in sorted(args.sources): 364 stream.write(f'"{filename}"\n') 365 366 header_path = args.header_path 367 if args.fix_generated_header: 368 header_path = os.path.join(build_cache_dir, os.path.basename(header_path)) 369 370 swiftc_args = [ 371 '-parse-as-library', 372 '-module-name', 373 args.module_name, 374 f'@{swift_file_list_path}', 375 '-sdk', 376 args.sdk_path, 377 '-target', 378 args.target_triple, 379 '-swift-version', 380 args.swift_version, 381 '-c', 382 '-output-file-map', 383 output_file_map, 384 '-save-temps', 385 '-no-color-diagnostics', 386 '-serialize-diagnostics', 387 '-emit-dependencies', 388 '-emit-module', 389 '-emit-module-path', 390 os.path.join(args.target_out_dir, f'{args.module_name}.swiftmodule'), 391 '-emit-objc-header', 392 '-emit-objc-header-path', 393 header_path, 394 '-working-directory', 395 os.getcwd(), 396 '-index-store-path', 397 ensure_directory(os.path.join(build_cache_dir, 'Index.noindex')), 398 '-module-cache-path', 399 ensure_directory(os.path.join(build_cache_dir, 'ModuleCache.noindex')), 400 '-pch-output-dir', 401 ensure_directory(os.path.join(build_cache_dir, 'PrecompiledHeaders')), 402 ] 403 404 # Handle optional -bridge-header flag. 405 if args.bridge_header: 406 swiftc_args.extend(('-import-objc-header', args.bridge_header)) 407 408 # Handle swift const values extraction. 409 swiftc_args.extend(['-emit-const-values']) 410 swiftc_args.extend([ 411 '-Xfrontend', 412 '-const-gather-protocols-file', 413 '-Xfrontend', 414 args.const_gather_protocols_file, 415 ]) 416 417 # Handle -I, -F, -isystem, -Fsystem and -D arguments. 418 for (attr_name, forwarder) in ARGUMENT_FORWARDER_FOR_ATTR: 419 forwarder.forward(swiftc_args, getattr(args, attr_name), args.target_triple) 420 421 # Handle -whole-module-optimization flag. 422 num_threads = max(1, multiprocessing.cpu_count() // 2) 423 if args.whole_module_optimization: 424 swiftc_args.extend([ 425 '-whole-module-optimization', 426 '-no-emit-module-separately-wmo', 427 '-num-threads', 428 f'{num_threads}', 429 ]) 430 else: 431 swiftc_args.extend([ 432 '-enable-batch-mode', 433 '-incremental', 434 '-experimental-emit-module-separately', 435 '-disable-cmo', 436 f'-j{num_threads}', 437 ]) 438 439 swift_toolchain_path = args.swift_toolchain_path 440 if not swift_toolchain_path: 441 swift_toolchain_path = os.path.join(os.path.dirname(args.sdk_path), 442 'XcodeDefault.xctoolchain') 443 if not os.path.isdir(swift_toolchain_path): 444 swift_toolchain_path = '' 445 446 command = [f'{swift_toolchain_path}/usr/bin/swiftc'] + swiftc_args 447 if extras_args: 448 command.extend(extras_args) 449 450 process = subprocess.Popen(command) 451 process.communicate() 452 453 if process.returncode: 454 sys.exit(process.returncode) 455 456 if args.fix_generated_header: 457 fix_generated_header(header_path, 458 args.header_path, 459 src_dir=os.path.abspath(args.src_dir) + os.path.sep, 460 gen_dir=os.path.abspath(args.gen_dir) + os.path.sep) 461 462 463def generate_depfile(args, output_file_map): 464 """Generates compilation depfile according to `args`. 465 466 Parses all intermediate depfile generated by the Swift compiler and 467 replaces absolute path by relative paths (since ninja compares paths 468 as strings and does not resolve relative paths to absolute). 469 470 Converts path to the SDK and toolchain files to the sdk/xcode_link 471 symlinks if possible and available. 472 """ 473 xcode_paths = {} 474 if os.path.islink(args.sdk_path): 475 xcode_links = os.path.dirname(args.sdk_path) 476 for link_name in os.listdir(xcode_links): 477 link_path = os.path.join(xcode_links, link_name) 478 if os.path.islink(link_path): 479 xcode_paths[os.path.realpath(link_path) + os.sep] = link_path + os.sep 480 481 out_dir = os.getcwd() + os.path.sep 482 src_dir = os.path.abspath(args.src_dir) + os.path.sep 483 484 depfile_content = collections.defaultdict(set) 485 for value in output_file_map.values(): 486 partial_depfile_path = value.get('dependencies', None) 487 if partial_depfile_path: 488 with open(partial_depfile_path, encoding='utf8') as stream: 489 for line in stream: 490 output, inputs = line.split(' : ', 2) 491 output = os.path.relpath(output, out_dir) 492 493 # The depfile format uses '\' to quote space in filename. Split the 494 # list of file while respecting this convention. 495 for path in re.split(r'(?<!\\) ', inputs): 496 for xcode_path in xcode_paths: 497 if path.startswith(xcode_path): 498 path = xcode_paths[xcode_path] + path[len(xcode_path):] 499 if path.startswith(src_dir) or path.startswith(out_dir): 500 path = os.path.relpath(path, out_dir) 501 depfile_content[output].add(path) 502 503 with FileWriter(args.depfile_path) as stream: 504 for output, inputs in sorted(depfile_content.items()): 505 stream.write(f'{output}: {" ".join(sorted(inputs))}\n') 506 507 508def compile_module(args, extras_args, build_signature): 509 """Compiles Swift module according to `args`.""" 510 for path in (args.target_out_dir, os.path.dirname(args.header_path)): 511 ensure_directory(path) 512 513 # Write the $module-OutputFileMap.json file. 514 output_file_map = generate_output_file_map(args) 515 output_file_map_path = os.path.join(args.target_out_dir, 516 f'{args.module_name}-OutputFileMap.json') 517 518 with FileWriter(output_file_map_path) as stream: 519 json.dump(output_file_map, stream, indent=' ', sort_keys=True) 520 521 # Invoke Swift compiler. 522 with create_build_cache_dir(args, build_signature) as build_cache_dir: 523 invoke_swift_compiler(args, 524 extras_args, 525 build_cache_dir=build_cache_dir, 526 output_file_map=output_file_map_path) 527 528 # Generate the depfile. 529 generate_depfile(args, output_file_map) 530 531 532def main(args): 533 parser = argparse.ArgumentParser(allow_abbrev=False, add_help=False) 534 535 # Required arguments. 536 parser.add_argument('--module-name', 537 required=True, 538 help='name of the Swift module') 539 540 parser.add_argument('--src-dir', 541 required=True, 542 help='path to the source directory') 543 544 parser.add_argument('--gen-dir', 545 required=True, 546 help='path to the gen directory root') 547 548 parser.add_argument('--target-out-dir', 549 required=True, 550 help='path to the object directory') 551 552 parser.add_argument('--header-path', 553 required=True, 554 help='path to the generated header file') 555 556 parser.add_argument('--bridge-header', 557 required=True, 558 help='path to the Objective-C bridge header file') 559 560 parser.add_argument('--depfile-path', 561 required=True, 562 help='path to the output dependency file') 563 564 parser.add_argument('--const-gather-protocols-file', 565 required=True, 566 help='path to file containing const values protocols') 567 568 # Optional arguments. 569 parser.add_argument('--derived-data-dir', 570 help='path to the derived data directory') 571 572 parser.add_argument('--fix-generated-header', 573 default=False, 574 action='store_true', 575 help='fix imports in generated header') 576 577 parser.add_argument('--swift-toolchain-path', 578 default='', 579 help='path to the Swift toolchain to use') 580 581 parser.add_argument('--whole-module-optimization', 582 default=False, 583 action='store_true', 584 help='enable whole module optimisation') 585 586 # Required arguments (forwarded to the Swift compiler). 587 parser.add_argument('-target', 588 required=True, 589 dest='target_triple', 590 help='generate code for the given target') 591 592 parser.add_argument('-sdk', 593 required=True, 594 dest='sdk_path', 595 help='path to the iOS SDK') 596 597 # Optional arguments (forwarded to the Swift compiler). 598 parser.add_argument('-I', 599 action='append', 600 dest='include_dirs', 601 help='add directory to header search path') 602 603 parser.add_argument('-isystem', 604 action='append', 605 dest='system_include_dirs', 606 help='add directory to system header search path') 607 608 parser.add_argument('-F', 609 action='append', 610 dest='framework_dirs', 611 help='add directory to framework search path') 612 613 parser.add_argument('-Fsystem', 614 action='append', 615 dest='system_framework_dirs', 616 help='add directory to system framework search path') 617 618 parser.add_argument('-D', 619 action='append', 620 dest='defines', 621 help='add preprocessor define') 622 623 parser.add_argument('-swift-version', 624 default='5', 625 help='version of the Swift language') 626 627 parser.add_argument('-file-compilation-dir', 628 help='compilation directory to embed in debug info') 629 630 # Positional arguments. 631 parser.add_argument('sources', 632 nargs='+', 633 help='Swift source files to compile') 634 635 parsed, extras = parser.parse_known_args(args) 636 compile_module(parsed, extras, build_signature(os.environ, args)) 637 638 639if __name__ == '__main__': 640 sys.exit(main(sys.argv[1:])) 641