1# Copyright 2022 The Bazel Authors. All rights reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Common functionality between test/binary executables.""" 15 16load("@bazel_skylib//lib:dicts.bzl", "dicts") 17load("@bazel_skylib//lib:structs.bzl", "structs") 18load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") 19load("@rules_cc//cc:defs.bzl", "cc_common") 20load("//python/private:flags.bzl", "PrecompileAddToRunfilesFlag") 21load("//python/private:py_executable_info.bzl", "PyExecutableInfo") 22load("//python/private:reexports.bzl", "BuiltinPyRuntimeInfo") 23load( 24 "//python/private:toolchain_types.bzl", 25 "EXEC_TOOLS_TOOLCHAIN_TYPE", 26 TOOLCHAIN_TYPE = "TARGET_TOOLCHAIN_TYPE", 27) 28load( 29 ":attributes.bzl", 30 "AGNOSTIC_EXECUTABLE_ATTRS", 31 "COMMON_ATTRS", 32 "PY_SRCS_ATTRS", 33 "PycCollectionAttr", 34 "SRCS_VERSION_ALL_VALUES", 35 "create_srcs_attr", 36 "create_srcs_version_attr", 37) 38load(":cc_helper.bzl", "cc_helper") 39load( 40 ":common.bzl", 41 "check_native_allowed", 42 "collect_imports", 43 "collect_runfiles", 44 "create_instrumented_files_info", 45 "create_output_group_info", 46 "create_py_info", 47 "csv", 48 "filter_to_py_srcs", 49 "target_platform_has_any_constraint", 50 "union_attrs", 51) 52load( 53 ":providers.bzl", 54 "PyCcLinkParamsProvider", 55 "PyInfo", 56 "PyRuntimeInfo", 57) 58load(":py_internal.bzl", "py_internal") 59load( 60 ":semantics.bzl", 61 "ALLOWED_MAIN_EXTENSIONS", 62 "BUILD_DATA_SYMLINK_PATH", 63 "IS_BAZEL", 64 "PY_RUNTIME_ATTR_NAME", 65) 66 67_py_builtins = py_internal 68 69# Bazel 5.4 doesn't have config_common.toolchain_type 70_CC_TOOLCHAINS = [config_common.toolchain_type( 71 "@bazel_tools//tools/cpp:toolchain_type", 72 mandatory = False, 73)] if hasattr(config_common, "toolchain_type") else [] 74 75# Non-Google-specific attributes for executables 76# These attributes are for rules that accept Python sources. 77EXECUTABLE_ATTRS = union_attrs( 78 COMMON_ATTRS, 79 AGNOSTIC_EXECUTABLE_ATTRS, 80 PY_SRCS_ATTRS, 81 { 82 # TODO(b/203567235): In the Java impl, any file is allowed. While marked 83 # label, it is more treated as a string, and doesn't have to refer to 84 # anything that exists because it gets treated as suffix-search string 85 # over `srcs`. 86 "main": attr.label( 87 allow_single_file = True, 88 doc = """\ 89Optional; the name of the source file that is the main entry point of the 90application. This file must also be listed in `srcs`. If left unspecified, 91`name`, with `.py` appended, is used instead. If `name` does not match any 92filename in `srcs`, `main` must be specified. 93""", 94 ), 95 "pyc_collection": attr.string( 96 default = PycCollectionAttr.INHERIT, 97 values = sorted(PycCollectionAttr.__members__.values()), 98 doc = """ 99Determines whether pyc files from dependencies should be manually included. 100 101NOTE: This setting is only useful with {flag}`--precompile_add_to_runfiles=decided_elsewhere`. 102 103Valid values are: 104* `inherit`: Inherit the value from {flag}`--pyc_collection`. 105* `include_pyc`: Add pyc files from dependencies in the binary (from 106 {obj}`PyInfo.transitive_pyc_files`. 107* `disabled`: Don't explicitly add pyc files from dependencies. Note that 108 pyc files may still come from dependencies if a target includes them as 109 part of their runfiles (such as when {obj}`--precompile_add_to_runfiles=always` 110 is used). 111""", 112 ), 113 # TODO(b/203567235): In Google, this attribute is deprecated, and can 114 # only effectively be PY3. Externally, with Bazel, this attribute has 115 # a separate story. 116 "python_version": attr.string( 117 # TODO(b/203567235): In the Java impl, the default comes from 118 # --python_version. Not clear what the Starlark equivalent is. 119 default = "PY3", 120 # NOTE: Some tests care about the order of these values. 121 values = ["PY2", "PY3"], 122 doc = "Defunct, unused, does nothing.", 123 ), 124 "_bootstrap_impl_flag": attr.label( 125 default = "//python/config_settings:bootstrap_impl", 126 providers = [BuildSettingInfo], 127 ), 128 "_pyc_collection_flag": attr.label( 129 default = "//python/config_settings:pyc_collection", 130 providers = [BuildSettingInfo], 131 ), 132 "_windows_constraints": attr.label_list( 133 default = [ 134 "@platforms//os:windows", 135 ], 136 ), 137 }, 138 create_srcs_version_attr(values = SRCS_VERSION_ALL_VALUES), 139 create_srcs_attr(mandatory = True), 140 allow_none = True, 141) 142 143def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = []): 144 """Base rule implementation for a Python executable. 145 146 Google and Bazel call this common base and apply customizations using the 147 semantics object. 148 149 Args: 150 ctx: The rule ctx 151 semantics: BinarySemantics struct; see create_binary_semantics_struct() 152 is_test: bool, True if the rule is a test rule (has `test=True`), 153 False if not (has `executable=True`) 154 inherited_environment: List of str; additional environment variable 155 names that should be inherited from the runtime environment when the 156 executable is run. 157 Returns: 158 DefaultInfo provider for the executable 159 """ 160 _validate_executable(ctx) 161 162 main_py = determine_main(ctx) 163 direct_sources = filter_to_py_srcs(ctx.files.srcs) 164 precompile_result = semantics.maybe_precompile(ctx, direct_sources) 165 166 # Sourceless precompiled builds omit the main py file from outputs, so 167 # main has to be pointed to the precompiled main instead. 168 if main_py not in precompile_result.keep_srcs: 169 main_py = precompile_result.py_to_pyc_map[main_py] 170 direct_pyc_files = depset(precompile_result.pyc_files) 171 172 executable = _declare_executable_file(ctx) 173 default_outputs = [executable] 174 default_outputs.extend(precompile_result.keep_srcs) 175 default_outputs.extend(precompile_result.pyc_files) 176 177 imports = collect_imports(ctx, semantics) 178 179 runtime_details = _get_runtime_details(ctx, semantics) 180 if ctx.configuration.coverage_enabled: 181 extra_deps = semantics.get_coverage_deps(ctx, runtime_details) 182 else: 183 extra_deps = [] 184 185 # The debugger dependency should be prevented by select() config elsewhere, 186 # but just to be safe, also guard against adding it to the output here. 187 if not _is_tool_config(ctx): 188 extra_deps.extend(semantics.get_debugger_deps(ctx, runtime_details)) 189 190 cc_details = semantics.get_cc_details_for_binary(ctx, extra_deps = extra_deps) 191 native_deps_details = _get_native_deps_details( 192 ctx, 193 semantics = semantics, 194 cc_details = cc_details, 195 is_test = is_test, 196 ) 197 runfiles_details = _get_base_runfiles_for_binary( 198 ctx, 199 executable = executable, 200 extra_deps = extra_deps, 201 main_py_files = depset([main_py] + precompile_result.keep_srcs), 202 direct_pyc_files = direct_pyc_files, 203 extra_common_runfiles = [ 204 runtime_details.runfiles, 205 cc_details.extra_runfiles, 206 native_deps_details.runfiles, 207 semantics.get_extra_common_runfiles_for_binary(ctx), 208 ], 209 semantics = semantics, 210 ) 211 exec_result = semantics.create_executable( 212 ctx, 213 executable = executable, 214 main_py = main_py, 215 imports = imports, 216 is_test = is_test, 217 runtime_details = runtime_details, 218 cc_details = cc_details, 219 native_deps_details = native_deps_details, 220 runfiles_details = runfiles_details, 221 ) 222 223 extra_exec_runfiles = exec_result.extra_runfiles.merge( 224 ctx.runfiles(transitive_files = exec_result.extra_files_to_build), 225 ) 226 227 # Copy any existing fields in case of company patches. 228 runfiles_details = struct(**( 229 structs.to_dict(runfiles_details) | dict( 230 default_runfiles = runfiles_details.default_runfiles.merge(extra_exec_runfiles), 231 data_runfiles = runfiles_details.data_runfiles.merge(extra_exec_runfiles), 232 ) 233 )) 234 235 return _create_providers( 236 ctx = ctx, 237 executable = executable, 238 runfiles_details = runfiles_details, 239 main_py = main_py, 240 imports = imports, 241 direct_sources = direct_sources, 242 direct_pyc_files = direct_pyc_files, 243 default_outputs = depset(default_outputs, transitive = [exec_result.extra_files_to_build]), 244 runtime_details = runtime_details, 245 cc_info = cc_details.cc_info_for_propagating, 246 inherited_environment = inherited_environment, 247 semantics = semantics, 248 output_groups = exec_result.output_groups, 249 ) 250 251def _get_build_info(ctx, cc_toolchain): 252 build_info_files = py_internal.cc_toolchain_build_info_files(cc_toolchain) 253 if cc_helper.is_stamping_enabled(ctx): 254 # Makes the target depend on BUILD_INFO_KEY, which helps to discover stamped targets 255 # See b/326620485 for more details. 256 ctx.version_file # buildifier: disable=no-effect 257 return build_info_files.non_redacted_build_info_files.to_list() 258 else: 259 return build_info_files.redacted_build_info_files.to_list() 260 261def _validate_executable(ctx): 262 if ctx.attr.python_version != "PY3": 263 fail("It is not allowed to use Python 2") 264 check_native_allowed(ctx) 265 266def _declare_executable_file(ctx): 267 if target_platform_has_any_constraint(ctx, ctx.attr._windows_constraints): 268 executable = ctx.actions.declare_file(ctx.label.name + ".exe") 269 else: 270 executable = ctx.actions.declare_file(ctx.label.name) 271 272 return executable 273 274def _get_runtime_details(ctx, semantics): 275 """Gets various information about the Python runtime to use. 276 277 While most information comes from the toolchain, various legacy and 278 compatibility behaviors require computing some other information. 279 280 Args: 281 ctx: Rule ctx 282 semantics: A `BinarySemantics` struct; see `create_binary_semantics_struct` 283 284 Returns: 285 A struct; see inline-field comments of the return value for details. 286 """ 287 288 # Bazel has --python_path. This flag has a computed default of "python" when 289 # its actual default is null (see 290 # BazelPythonConfiguration.java#getPythonPath). This flag is only used if 291 # toolchains are not enabled and `--python_top` isn't set. Note that Google 292 # used to have a variant of this named --python_binary, but it has since 293 # been removed. 294 # 295 # TOOD(bazelbuild/bazel#7901): Remove this once --python_path flag is removed. 296 297 if IS_BAZEL: 298 flag_interpreter_path = ctx.fragments.bazel_py.python_path 299 toolchain_runtime, effective_runtime = _maybe_get_runtime_from_ctx(ctx) 300 if not effective_runtime: 301 # Clear these just in case 302 toolchain_runtime = None 303 effective_runtime = None 304 305 else: # Google code path 306 flag_interpreter_path = None 307 toolchain_runtime, effective_runtime = _maybe_get_runtime_from_ctx(ctx) 308 if not effective_runtime: 309 fail("Unable to find Python runtime") 310 311 if effective_runtime: 312 direct = [] # List of files 313 transitive = [] # List of depsets 314 if effective_runtime.interpreter: 315 direct.append(effective_runtime.interpreter) 316 transitive.append(effective_runtime.files) 317 318 if ctx.configuration.coverage_enabled: 319 if effective_runtime.coverage_tool: 320 direct.append(effective_runtime.coverage_tool) 321 if effective_runtime.coverage_files: 322 transitive.append(effective_runtime.coverage_files) 323 runtime_files = depset(direct = direct, transitive = transitive) 324 else: 325 runtime_files = depset() 326 327 executable_interpreter_path = semantics.get_interpreter_path( 328 ctx, 329 runtime = effective_runtime, 330 flag_interpreter_path = flag_interpreter_path, 331 ) 332 333 return struct( 334 # Optional PyRuntimeInfo: The runtime found from toolchain resolution. 335 # This may be None because, within Google, toolchain resolution isn't 336 # yet enabled. 337 toolchain_runtime = toolchain_runtime, 338 # Optional PyRuntimeInfo: The runtime that should be used. When 339 # toolchain resolution is enabled, this is the same as 340 # `toolchain_resolution`. Otherwise, this probably came from the 341 # `_python_top` attribute that the Google implementation still uses. 342 # This is separate from `toolchain_runtime` because toolchain_runtime 343 # is propagated as a provider, while non-toolchain runtimes are not. 344 effective_runtime = effective_runtime, 345 # str; Path to the Python interpreter to use for running the executable 346 # itself (not the bootstrap script). Either an absolute path (which 347 # means it is platform-specific), or a runfiles-relative path (which 348 # means the interpreter should be within `runtime_files`) 349 executable_interpreter_path = executable_interpreter_path, 350 # runfiles: Additional runfiles specific to the runtime that should 351 # be included. For in-build runtimes, this shold include the interpreter 352 # and any supporting files. 353 runfiles = ctx.runfiles(transitive_files = runtime_files), 354 ) 355 356def _maybe_get_runtime_from_ctx(ctx): 357 """Finds the PyRuntimeInfo from the toolchain or attribute, if available. 358 359 Returns: 360 2-tuple of toolchain_runtime, effective_runtime 361 """ 362 if ctx.fragments.py.use_toolchains: 363 toolchain = ctx.toolchains[TOOLCHAIN_TYPE] 364 365 if not hasattr(toolchain, "py3_runtime"): 366 fail("Python toolchain field 'py3_runtime' is missing") 367 if not toolchain.py3_runtime: 368 fail("Python toolchain missing py3_runtime") 369 py3_runtime = toolchain.py3_runtime 370 371 # Hack around the fact that the autodetecting Python toolchain, which is 372 # automatically registered, does not yet support Windows. In this case, 373 # we want to return null so that _get_interpreter_path falls back on 374 # --python_path. See tools/python/toolchain.bzl. 375 # TODO(#7844): Remove this hack when the autodetecting toolchain has a 376 # Windows implementation. 377 if py3_runtime.interpreter_path == "/_magic_pyruntime_sentinel_do_not_use": 378 return None, None 379 380 if py3_runtime.python_version != "PY3": 381 fail("Python toolchain py3_runtime must be python_version=PY3, got {}".format( 382 py3_runtime.python_version, 383 )) 384 toolchain_runtime = toolchain.py3_runtime 385 effective_runtime = toolchain_runtime 386 else: 387 toolchain_runtime = None 388 attr_target = getattr(ctx.attr, PY_RUNTIME_ATTR_NAME) 389 390 # In Bazel, --python_top is null by default. 391 if attr_target and PyRuntimeInfo in attr_target: 392 effective_runtime = attr_target[PyRuntimeInfo] 393 else: 394 return None, None 395 396 return toolchain_runtime, effective_runtime 397 398def _get_base_runfiles_for_binary( 399 ctx, 400 *, 401 executable, 402 extra_deps, 403 main_py_files, 404 direct_pyc_files, 405 extra_common_runfiles, 406 semantics): 407 """Returns the set of runfiles necessary prior to executable creation. 408 409 NOTE: The term "common runfiles" refers to the runfiles that are common to 410 runfiles_without_exe, default_runfiles, and data_runfiles. 411 412 Args: 413 ctx: The rule ctx. 414 executable: The main executable output. 415 extra_deps: List of Targets; additional targets whose runfiles 416 will be added to the common runfiles. 417 main_py_files: depset of File of the default outputs to add into runfiles. 418 direct_pyc_files: depset of File of pyc files directly from this target. 419 extra_common_runfiles: List of runfiles; additional runfiles that 420 will be added to the common runfiles. 421 semantics: A `BinarySemantics` struct; see `create_binary_semantics_struct`. 422 423 Returns: 424 struct with attributes: 425 * default_runfiles: The default runfiles 426 * data_runfiles: The data runfiles 427 * runfiles_without_exe: The default runfiles, but without the executable 428 or files specific to the original program/executable. 429 * build_data_file: A file with build stamp information if stamping is enabled, otherwise 430 None. 431 """ 432 common_runfiles_depsets = [main_py_files] 433 434 if ctx.attr._precompile_add_to_runfiles_flag[BuildSettingInfo].value == PrecompileAddToRunfilesFlag.ALWAYS: 435 common_runfiles_depsets.append(direct_pyc_files) 436 elif PycCollectionAttr.is_pyc_collection_enabled(ctx): 437 common_runfiles_depsets.append(direct_pyc_files) 438 for dep in (ctx.attr.deps + extra_deps): 439 if PyInfo not in dep: 440 continue 441 common_runfiles_depsets.append(dep[PyInfo].transitive_pyc_files) 442 443 common_runfiles = collect_runfiles(ctx, depset( 444 transitive = common_runfiles_depsets, 445 )) 446 if extra_deps: 447 common_runfiles = common_runfiles.merge_all([ 448 t[DefaultInfo].default_runfiles 449 for t in extra_deps 450 ]) 451 common_runfiles = common_runfiles.merge_all(extra_common_runfiles) 452 453 if semantics.should_create_init_files(ctx): 454 common_runfiles = _py_builtins.merge_runfiles_with_generated_inits_empty_files_supplier( 455 ctx = ctx, 456 runfiles = common_runfiles, 457 ) 458 459 # Don't include build_data.txt in the non-exe runfiles. The build data 460 # may contain program-specific content (e.g. target name). 461 runfiles_with_exe = common_runfiles.merge(ctx.runfiles([executable])) 462 463 # Don't include build_data.txt in data runfiles. This allows binaries to 464 # contain other binaries while still using the same fixed location symlink 465 # for the build_data.txt file. Really, the fixed location symlink should be 466 # removed and another way found to locate the underlying build data file. 467 data_runfiles = runfiles_with_exe 468 469 if is_stamping_enabled(ctx, semantics) and semantics.should_include_build_data(ctx): 470 build_data_file, build_data_runfiles = _create_runfiles_with_build_data( 471 ctx, 472 semantics.get_central_uncachable_version_file(ctx), 473 semantics.get_extra_write_build_data_env(ctx), 474 ) 475 default_runfiles = runfiles_with_exe.merge(build_data_runfiles) 476 else: 477 build_data_file = None 478 default_runfiles = runfiles_with_exe 479 480 return struct( 481 runfiles_without_exe = common_runfiles, 482 default_runfiles = default_runfiles, 483 build_data_file = build_data_file, 484 data_runfiles = data_runfiles, 485 ) 486 487def _create_runfiles_with_build_data( 488 ctx, 489 central_uncachable_version_file, 490 extra_write_build_data_env): 491 build_data_file = _write_build_data( 492 ctx, 493 central_uncachable_version_file, 494 extra_write_build_data_env, 495 ) 496 build_data_runfiles = ctx.runfiles(symlinks = { 497 BUILD_DATA_SYMLINK_PATH: build_data_file, 498 }) 499 return build_data_file, build_data_runfiles 500 501def _write_build_data(ctx, central_uncachable_version_file, extra_write_build_data_env): 502 # TODO: Remove this logic when a central file is always available 503 if not central_uncachable_version_file: 504 version_file = ctx.actions.declare_file(ctx.label.name + "-uncachable_version_file.txt") 505 _py_builtins.copy_without_caching( 506 ctx = ctx, 507 read_from = ctx.version_file, 508 write_to = version_file, 509 ) 510 else: 511 version_file = central_uncachable_version_file 512 513 direct_inputs = [ctx.info_file, version_file] 514 515 # A "constant metadata" file is basically a special file that doesn't 516 # support change detection logic and reports that it is unchanged. i.e., it 517 # behaves like ctx.version_file and is ignored when computing "what inputs 518 # changed" (see https://bazel.build/docs/user-manual#workspace-status). 519 # 520 # We do this so that consumers of the final build data file don't have 521 # to transitively rebuild everything -- the `uncachable_version_file` file 522 # isn't cachable, which causes the build data action to always re-run. 523 # 524 # While this technically means a binary could have stale build info, 525 # it ends up not mattering in practice because the volatile information 526 # doesn't meaningfully effect other outputs. 527 # 528 # This is also done for performance and Make It work reasons: 529 # * Passing the transitive dependencies into the action requires passing 530 # the runfiles, but actions don't directly accept runfiles. While 531 # flattening the depsets can be deferred, accessing the 532 # `runfiles.empty_filenames` attribute will will invoke the empty 533 # file supplier a second time, which is too much of a memory and CPU 534 # performance hit. 535 # * Some targets specify a directory in `data`, which is unsound, but 536 # mostly works. Google's RBE, unfortunately, rejects it. 537 # * A binary's transitive closure may be so large that it exceeds 538 # Google RBE limits for action inputs. 539 build_data = _py_builtins.declare_constant_metadata_file( 540 ctx = ctx, 541 name = ctx.label.name + ".build_data.txt", 542 root = ctx.bin_dir, 543 ) 544 545 ctx.actions.run( 546 executable = ctx.executable._build_data_gen, 547 env = dicts.add({ 548 # NOTE: ctx.info_file is undocumented; see 549 # https://github.com/bazelbuild/bazel/issues/9363 550 "INFO_FILE": ctx.info_file.path, 551 "OUTPUT": build_data.path, 552 "PLATFORM": cc_helper.find_cpp_toolchain(ctx).toolchain_id, 553 "TARGET": str(ctx.label), 554 "VERSION_FILE": version_file.path, 555 }, extra_write_build_data_env), 556 inputs = depset( 557 direct = direct_inputs, 558 ), 559 outputs = [build_data], 560 mnemonic = "PyWriteBuildData", 561 progress_message = "Generating %{label} build_data.txt", 562 ) 563 return build_data 564 565def _get_native_deps_details(ctx, *, semantics, cc_details, is_test): 566 if not semantics.should_build_native_deps_dso(ctx): 567 return struct(dso = None, runfiles = ctx.runfiles()) 568 569 cc_info = cc_details.cc_info_for_self_link 570 571 if not cc_info.linking_context.linker_inputs: 572 return struct(dso = None, runfiles = ctx.runfiles()) 573 574 dso = ctx.actions.declare_file(semantics.get_native_deps_dso_name(ctx)) 575 share_native_deps = py_internal.share_native_deps(ctx) 576 cc_feature_config = cc_details.feature_config 577 if share_native_deps: 578 linked_lib = _create_shared_native_deps_dso( 579 ctx, 580 cc_info = cc_info, 581 is_test = is_test, 582 requested_features = cc_feature_config.requested_features, 583 feature_configuration = cc_feature_config.feature_configuration, 584 cc_toolchain = cc_details.cc_toolchain, 585 ) 586 ctx.actions.symlink( 587 output = dso, 588 target_file = linked_lib, 589 progress_message = "Symlinking shared native deps for %{label}", 590 ) 591 else: 592 linked_lib = dso 593 594 # The regular cc_common.link API can't be used because several 595 # args are private-use only; see # private comments 596 py_internal.link( 597 name = ctx.label.name, 598 actions = ctx.actions, 599 linking_contexts = [cc_info.linking_context], 600 output_type = "dynamic_library", 601 never_link = True, # private 602 native_deps = True, # private 603 feature_configuration = cc_feature_config.feature_configuration, 604 cc_toolchain = cc_details.cc_toolchain, 605 test_only_target = is_test, # private 606 stamp = 1 if is_stamping_enabled(ctx, semantics) else 0, 607 main_output = linked_lib, # private 608 use_shareable_artifact_factory = True, # private 609 # NOTE: Only flags not captured by cc_info.linking_context need to 610 # be manually passed 611 user_link_flags = semantics.get_native_deps_user_link_flags(ctx), 612 ) 613 return struct( 614 dso = dso, 615 runfiles = ctx.runfiles(files = [dso]), 616 ) 617 618def _create_shared_native_deps_dso( 619 ctx, 620 *, 621 cc_info, 622 is_test, 623 feature_configuration, 624 requested_features, 625 cc_toolchain): 626 linkstamps = py_internal.linking_context_linkstamps(cc_info.linking_context) 627 628 partially_disabled_thin_lto = ( 629 cc_common.is_enabled( 630 feature_name = "thin_lto_linkstatic_tests_use_shared_nonlto_backends", 631 feature_configuration = feature_configuration, 632 ) and not cc_common.is_enabled( 633 feature_name = "thin_lto_all_linkstatic_use_shared_nonlto_backends", 634 feature_configuration = feature_configuration, 635 ) 636 ) 637 dso_hash = _get_shared_native_deps_hash( 638 linker_inputs = cc_helper.get_static_mode_params_for_dynamic_library_libraries( 639 depset([ 640 lib 641 for linker_input in cc_info.linking_context.linker_inputs.to_list() 642 for lib in linker_input.libraries 643 ]), 644 ), 645 link_opts = [ 646 flag 647 for input in cc_info.linking_context.linker_inputs.to_list() 648 for flag in input.user_link_flags 649 ], 650 linkstamps = [ 651 py_internal.linkstamp_file(linkstamp) 652 for linkstamp in linkstamps.to_list() 653 ], 654 build_info_artifacts = _get_build_info(ctx, cc_toolchain) if linkstamps else [], 655 features = requested_features, 656 is_test_target_partially_disabled_thin_lto = is_test and partially_disabled_thin_lto, 657 ) 658 return py_internal.declare_shareable_artifact(ctx, "_nativedeps/%x.so" % dso_hash) 659 660# This is a minimal version of NativeDepsHelper.getSharedNativeDepsPath, see 661# com.google.devtools.build.lib.rules.nativedeps.NativeDepsHelper#getSharedNativeDepsPath 662# The basic idea is to take all the inputs that affect linking and encode (via 663# hashing) them into the filename. 664# TODO(b/234232820): The settings that affect linking must be kept in sync with the actual 665# C++ link action. For more information, see the large descriptive comment on 666# NativeDepsHelper#getSharedNativeDepsPath. 667def _get_shared_native_deps_hash( 668 *, 669 linker_inputs, 670 link_opts, 671 linkstamps, 672 build_info_artifacts, 673 features, 674 is_test_target_partially_disabled_thin_lto): 675 # NOTE: We use short_path because the build configuration root in which 676 # files are always created already captures the configuration-specific 677 # parts, so no need to include them manually. 678 parts = [] 679 for artifact in linker_inputs: 680 parts.append(artifact.short_path) 681 parts.append(str(len(link_opts))) 682 parts.extend(link_opts) 683 for artifact in linkstamps: 684 parts.append(artifact.short_path) 685 for artifact in build_info_artifacts: 686 parts.append(artifact.short_path) 687 parts.extend(sorted(features)) 688 689 # Sharing of native dependencies may cause an {@link 690 # ActionConflictException} when ThinLTO is disabled for test and test-only 691 # targets that are statically linked, but enabled for other statically 692 # linked targets. This happens in case the artifacts for the shared native 693 # dependency are output by {@link Action}s owned by the non-test and test 694 # targets both. To fix this, we allow creation of multiple artifacts for the 695 # shared native library - one shared among the test and test-only targets 696 # where ThinLTO is disabled, and the other shared among other targets where 697 # ThinLTO is enabled. See b/138118275 698 parts.append("1" if is_test_target_partially_disabled_thin_lto else "0") 699 700 return hash("".join(parts)) 701 702def determine_main(ctx): 703 """Determine the main entry point .py source file. 704 705 Args: 706 ctx: The rule ctx. 707 708 Returns: 709 Artifact; the main file. If one can't be found, an error is raised. 710 """ 711 if ctx.attr.main: 712 proposed_main = ctx.attr.main.label.name 713 if not proposed_main.endswith(tuple(ALLOWED_MAIN_EXTENSIONS)): 714 fail("main must end in '.py'") 715 else: 716 if ctx.label.name.endswith(".py"): 717 fail("name must not end in '.py'") 718 proposed_main = ctx.label.name + ".py" 719 720 main_files = [src for src in ctx.files.srcs if _path_endswith(src.short_path, proposed_main)] 721 if not main_files: 722 if ctx.attr.main: 723 fail("could not find '{}' as specified by 'main' attribute".format(proposed_main)) 724 else: 725 fail(("corresponding default '{}' does not appear in srcs. Add " + 726 "it or override default file name with a 'main' attribute").format( 727 proposed_main, 728 )) 729 730 elif len(main_files) > 1: 731 if ctx.attr.main: 732 fail(("file name '{}' specified by 'main' attributes matches multiple files. " + 733 "Matches: {}").format( 734 proposed_main, 735 csv([f.short_path for f in main_files]), 736 )) 737 else: 738 fail(("default main file '{}' matches multiple files in srcs. Perhaps specify " + 739 "an explicit file with 'main' attribute? Matches were: {}").format( 740 proposed_main, 741 csv([f.short_path for f in main_files]), 742 )) 743 return main_files[0] 744 745def _path_endswith(path, endswith): 746 # Use slash to anchor each path to prevent e.g. 747 # "ab/c.py".endswith("b/c.py") from incorrectly matching. 748 return ("/" + path).endswith("/" + endswith) 749 750def is_stamping_enabled(ctx, semantics): 751 """Tells if stamping is enabled or not. 752 753 Args: 754 ctx: The rule ctx 755 semantics: a semantics struct (see create_semantics_struct). 756 Returns: 757 bool; True if stamping is enabled, False if not. 758 """ 759 if _is_tool_config(ctx): 760 return False 761 762 stamp = ctx.attr.stamp 763 if stamp == 1: 764 return True 765 elif stamp == 0: 766 return False 767 elif stamp == -1: 768 return semantics.get_stamp_flag(ctx) 769 else: 770 fail("Unsupported `stamp` value: {}".format(stamp)) 771 772def _is_tool_config(ctx): 773 # NOTE: The is_tool_configuration() function is only usable by builtins. 774 # See https://github.com/bazelbuild/bazel/issues/14444 for the FR for 775 # a more public API. Until that's available, py_internal to the rescue. 776 return py_internal.is_tool_configuration(ctx) 777 778def _create_providers( 779 *, 780 ctx, 781 executable, 782 main_py, 783 direct_sources, 784 direct_pyc_files, 785 default_outputs, 786 runfiles_details, 787 imports, 788 cc_info, 789 inherited_environment, 790 runtime_details, 791 output_groups, 792 semantics): 793 """Creates the providers an executable should return. 794 795 Args: 796 ctx: The rule ctx. 797 executable: File; the target's executable file. 798 main_py: File; the main .py entry point. 799 direct_sources: list of Files; the direct, raw `.py` sources for the target. 800 This should only be Python source files. It should not include pyc 801 files. 802 direct_pyc_files: depset of File; the direct pyc files for the target. 803 default_outputs: depset of Files; the files for DefaultInfo.files 804 runfiles_details: runfiles that will become the default and data runfiles. 805 imports: depset of strings; the import paths to propagate 806 cc_info: optional CcInfo; Linking information to propagate as 807 PyCcLinkParamsProvider. Note that only the linking information 808 is propagated, not the whole CcInfo. 809 inherited_environment: list of strings; Environment variable names 810 that should be inherited from the environment the executuble 811 is run within. 812 runtime_details: struct of runtime information; see _get_runtime_details() 813 output_groups: dict[str, depset[File]]; used to create OutputGroupInfo 814 semantics: BinarySemantics struct; see create_binary_semantics() 815 816 Returns: 817 A list of modern providers. 818 """ 819 providers = [ 820 DefaultInfo( 821 executable = executable, 822 files = default_outputs, 823 default_runfiles = _py_builtins.make_runfiles_respect_legacy_external_runfiles( 824 ctx, 825 runfiles_details.default_runfiles, 826 ), 827 data_runfiles = _py_builtins.make_runfiles_respect_legacy_external_runfiles( 828 ctx, 829 runfiles_details.data_runfiles, 830 ), 831 ), 832 create_instrumented_files_info(ctx), 833 _create_run_environment_info(ctx, inherited_environment), 834 PyExecutableInfo( 835 main = main_py, 836 runfiles_without_exe = runfiles_details.runfiles_without_exe, 837 build_data_file = runfiles_details.build_data_file, 838 interpreter_path = runtime_details.executable_interpreter_path, 839 ), 840 ] 841 842 # TODO(b/265840007): Make this non-conditional once Google enables 843 # --incompatible_use_python_toolchains. 844 if runtime_details.toolchain_runtime: 845 py_runtime_info = runtime_details.toolchain_runtime 846 providers.append(py_runtime_info) 847 848 # Re-add the builtin PyRuntimeInfo for compatibility to make 849 # transitioning easier, but only if it isn't already added because 850 # returning the same provider type multiple times is an error. 851 # NOTE: The PyRuntimeInfo from the toolchain could be a rules_python 852 # PyRuntimeInfo or a builtin PyRuntimeInfo -- a user could have used the 853 # builtin py_runtime rule or defined their own. We can't directly detect 854 # the type of the provider object, but the rules_python PyRuntimeInfo 855 # object has an extra attribute that the builtin one doesn't. 856 if hasattr(py_runtime_info, "interpreter_version_info"): 857 providers.append(BuiltinPyRuntimeInfo( 858 interpreter_path = py_runtime_info.interpreter_path, 859 interpreter = py_runtime_info.interpreter, 860 files = py_runtime_info.files, 861 coverage_tool = py_runtime_info.coverage_tool, 862 coverage_files = py_runtime_info.coverage_files, 863 python_version = py_runtime_info.python_version, 864 stub_shebang = py_runtime_info.stub_shebang, 865 bootstrap_template = py_runtime_info.bootstrap_template, 866 )) 867 868 # TODO(b/163083591): Remove the PyCcLinkParamsProvider once binaries-in-deps 869 # are cleaned up. 870 if cc_info: 871 providers.append( 872 PyCcLinkParamsProvider(cc_info = cc_info), 873 ) 874 875 py_info, deps_transitive_sources, builtin_py_info = create_py_info( 876 ctx, 877 direct_sources = depset(direct_sources), 878 direct_pyc_files = direct_pyc_files, 879 imports = imports, 880 ) 881 882 # TODO(b/253059598): Remove support for extra actions; https://github.com/bazelbuild/bazel/issues/16455 883 listeners_enabled = _py_builtins.are_action_listeners_enabled(ctx) 884 if listeners_enabled: 885 _py_builtins.add_py_extra_pseudo_action( 886 ctx = ctx, 887 dependency_transitive_python_sources = deps_transitive_sources, 888 ) 889 890 providers.append(py_info) 891 providers.append(builtin_py_info) 892 providers.append(create_output_group_info(py_info.transitive_sources, output_groups)) 893 894 extra_providers = semantics.get_extra_providers( 895 ctx, 896 main_py = main_py, 897 runtime_details = runtime_details, 898 ) 899 providers.extend(extra_providers) 900 return providers 901 902def _create_run_environment_info(ctx, inherited_environment): 903 expanded_env = {} 904 for key, value in ctx.attr.env.items(): 905 expanded_env[key] = _py_builtins.expand_location_and_make_variables( 906 ctx = ctx, 907 attribute_name = "env[{}]".format(key), 908 expression = value, 909 targets = ctx.attr.data, 910 ) 911 return RunEnvironmentInfo( 912 environment = expanded_env, 913 inherited_environment = inherited_environment, 914 ) 915 916def create_base_executable_rule(*, attrs, fragments = [], **kwargs): 917 """Create a function for defining for Python binary/test targets. 918 919 Args: 920 attrs: Rule attributes 921 fragments: List of str; extra config fragments that are required. 922 **kwargs: Additional args to pass onto `rule()` 923 924 Returns: 925 A rule function 926 """ 927 if "py" not in fragments: 928 # The list might be frozen, so use concatentation 929 fragments = fragments + ["py"] 930 kwargs.setdefault("provides", []).append(PyExecutableInfo) 931 return rule( 932 # TODO: add ability to remove attrs, i.e. for imports attr 933 attrs = dicts.add(EXECUTABLE_ATTRS, attrs), 934 toolchains = [ 935 TOOLCHAIN_TYPE, 936 config_common.toolchain_type(EXEC_TOOLS_TOOLCHAIN_TYPE, mandatory = False), 937 ] + _CC_TOOLCHAINS, 938 fragments = fragments, 939 **kwargs 940 ) 941 942def cc_configure_features( 943 ctx, 944 *, 945 cc_toolchain, 946 extra_features, 947 linking_mode = "static_linking_mode"): 948 """Configure C++ features for Python purposes. 949 950 Args: 951 ctx: Rule ctx 952 cc_toolchain: The CcToolchain the target is using. 953 extra_features: list of strings; additional features to request be 954 enabled. 955 linking_mode: str; either "static_linking_mode" or 956 "dynamic_linking_mode". Specifies the linking mode feature for 957 C++ linking. 958 959 Returns: 960 struct of the feature configuration and all requested features. 961 """ 962 requested_features = [linking_mode] 963 requested_features.extend(extra_features) 964 requested_features.extend(ctx.features) 965 if "legacy_whole_archive" not in ctx.disabled_features: 966 requested_features.append("legacy_whole_archive") 967 feature_configuration = cc_common.configure_features( 968 ctx = ctx, 969 cc_toolchain = cc_toolchain, 970 requested_features = requested_features, 971 unsupported_features = ctx.disabled_features, 972 ) 973 return struct( 974 feature_configuration = feature_configuration, 975 requested_features = requested_features, 976 ) 977 978only_exposed_for_google_internal_reason = struct( 979 create_runfiles_with_build_data = _create_runfiles_with_build_data, 980) 981