1# Copyright 2021 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14 15import("//build_overrides/pigweed.gni") 16 17import("$dir_pw_build/input_group.gni") 18import("$dir_pw_build/mirror_tree.gni") 19import("$dir_pw_build/python_action.gni") 20import("$dir_pw_build/python_gn_args.gni") 21import("$dir_pw_protobuf_compiler/toolchain.gni") 22 23declare_args() { 24 # Constraints file selection (arguments to pip install --constraint). 25 # See pip help install. 26 pw_build_PIP_CONSTRAINTS = 27 [ "$dir_pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list" ] 28 29 # Default pip requirements file for all Pigweed based projects. 30 pw_build_PIP_REQUIREMENTS = [] 31 32 # DOCSTAG: [python-static-analysis-tools] 33 # Default set of Python static alaysis tools to run for pw_python_package targets. 34 pw_build_PYTHON_STATIC_ANALYSIS_TOOLS = [ 35 "pylint", 36 "mypy", 37 ] 38 39 # DOCSTAG: [python-static-analysis-tools] 40 41 # If true, GN will run each Python test using the coverage command. A separate 42 # coverage data file for each test will be saved. To generate reports from 43 # this information run: pw presubmit --step gn_python_test_coverage 44 pw_build_PYTHON_TEST_COVERAGE = false 45 46 # Output format for pylint. Options include "text" and "colorized". 47 pw_build_PYLINT_OUTPUT_FORMAT = "colorized" 48 49 # Whether or not to lint/test transitive deps of pw_python_package targets. 50 # 51 # For example: if lib_a depends on lib_b, lib_a.tests will run after first 52 # running lib_b.tests if pw_build_TEST_TRANSITIVE_PYTHON_DEPS is true. 53 # 54 # If pw_build_TEST_TRANSITIVE_PYTHON_DEPS is false, tests for a 55 # pw_python_package will run if you directly build the target (e.g. 56 # lib_b.tests) OR if the pw_python_package is placed in a pw_python_group AND 57 # you build the group.tests target. 58 # 59 # This applies to mypy, pylint, ruff and tests. 60 # 61 # While this defaults to true for compatibility reasons, it's strongly 62 # recommended to turn this off so you're not linting and testing all of your 63 # external dependencies. 64 pw_build_TEST_TRANSITIVE_PYTHON_DEPS = true 65} 66 67# Python packages provide the following targets as $target_name.$subtarget. 68pw_python_package_subtargets = [ 69 "tests", 70 "lint", 71 "lint.mypy", 72 "lint.pylint", 73 "lint.ruff", 74 "install", 75 "wheel", 76 77 # Internal targets that directly depend on one another. 78 "_build_wheel", 79] 80 81# Create aliases for subsargets when the target name matches the directory name. 82# This allows //foo:foo.tests to be accessed as //foo:tests, for example. 83template("_pw_create_aliases_if_name_matches_directory") { 84 not_needed([ "invoker" ]) 85 86 if (get_label_info(":$target_name", "name") == 87 get_path_info(get_label_info(":$target_name", "dir"), "name")) { 88 foreach(subtarget, pw_python_package_subtargets) { 89 group(subtarget) { 90 public_deps = [ ":${invoker.target_name}.$subtarget" ] 91 } 92 } 93 } 94} 95 96# Internal template that runs Mypy. 97template("_pw_python_static_analysis_mypy") { 98 pw_python_action(target_name) { 99 module = "mypy" 100 101 # DOCSTAG: [default-mypy-args] 102 args = [ 103 "--pretty", 104 "--show-error-codes", 105 106 # Use a mypy cache dir for this target only to avoid cache conflicts in 107 # parallel mypy invocations. 108 "--cache-dir", 109 rebase_path(target_out_dir, root_build_dir) + "/.mypy_cache", 110 ] 111 112 # Use this environment variable to force mypy to colorize output. 113 # See https://github.com/python/mypy/issues/7771 114 environment = [ "MYPY_FORCE_COLOR=1" ] 115 116 # DOCSTAG: [default-mypy-args] 117 118 if (defined(invoker.mypy_ini)) { 119 args += 120 [ "--config-file=" + rebase_path(invoker.mypy_ini, root_build_dir) ] 121 inputs = [ invoker.mypy_ini ] 122 } 123 124 args += rebase_path(invoker.sources, root_build_dir) 125 126 stamp = true 127 128 deps = invoker.deps 129 130 if (defined(invoker.python_deps)) { 131 python_deps = invoker.python_deps 132 if (pw_build_TEST_TRANSITIVE_PYTHON_DEPS) { 133 foreach(dep, invoker.python_deps) { 134 deps += [ string_replace(dep, "(", ".lint.mypy(") ] 135 } 136 } 137 } 138 if (defined(invoker.python_metadata_deps)) { 139 python_metadata_deps = invoker.python_metadata_deps 140 } 141 } 142} 143 144# Internal template that runs Pylint. 145template("_pw_python_static_analysis_pylint") { 146 # Create a target to run pylint on each of the Python files in this 147 # package and its dependencies. 148 pw_python_action_foreach(target_name) { 149 module = "pylint" 150 args = [ 151 rebase_path(".", root_build_dir) + "/{{source_target_relative}}", 152 "--jobs=1", 153 "--output-format=$pw_build_PYLINT_OUTPUT_FORMAT", 154 ] 155 156 if (defined(invoker.pylintrc)) { 157 args += [ "--rcfile=" + rebase_path(invoker.pylintrc, root_build_dir) ] 158 inputs = [ invoker.pylintrc ] 159 } 160 161 if (host_os == "win") { 162 # Allow CRLF on Windows, in case Git is set to switch line endings. 163 args += [ "--disable=unexpected-line-ending-format" ] 164 } 165 166 sources = invoker.sources 167 168 stamp = "$target_gen_dir/{{source_target_relative}}.pylint.passed" 169 170 public_deps = invoker.deps 171 172 if (defined(invoker.python_deps)) { 173 python_deps = invoker.python_deps 174 if (pw_build_TEST_TRANSITIVE_PYTHON_DEPS) { 175 foreach(dep, invoker.python_deps) { 176 public_deps += [ string_replace(dep, "(", ".lint.pylint(") ] 177 } 178 } 179 } 180 if (defined(invoker.python_metadata_deps)) { 181 python_metadata_deps = invoker.python_metadata_deps 182 } 183 } 184} 185 186template("_pw_python_static_analysis_ruff") { 187 # Create a target to run ruff on each of the Python files in this 188 # package and its dependencies. 189 pw_python_action_foreach(target_name) { 190 script = "$dir_pw_build/py/pw_build/exec.py" 191 args = [ 192 "--", 193 "ruff", 194 "check", 195 rebase_path(".", root_build_dir) + "/{{source_target_relative}}", 196 ] 197 198 if (defined(invoker.ruff_toml)) { 199 args += [ 200 "--config", 201 rebase_path(invoker.ruff_toml, root_build_dir), 202 ] 203 inputs = [ invoker.ruff_toml ] 204 } 205 206 environment = [ "CLICOLOR_FORCE=1" ] 207 208 sources = invoker.sources 209 210 stamp = "$target_gen_dir/{{source_target_relative}}.ruff.passed" 211 212 public_deps = invoker.deps 213 214 if (defined(invoker.python_deps)) { 215 python_deps = [] 216 foreach(dep, invoker.python_deps) { 217 public_deps += [ string_replace(dep, "(", ".lint.ruff(") ] 218 python_deps += [ dep ] 219 } 220 } 221 if (defined(invoker.python_metadata_deps)) { 222 python_metadata_deps = invoker.python_metadata_deps 223 } 224 } 225} 226 227# Defines a Python package. GN Python packages contain several GN targets: 228# 229# - $name - Provides the Python files in the build, but does not take any 230# actions. All subtargets depend on this target. 231# - $name.lint - Runs static analyis tools on the Python code. This is a group 232# of two subtargets: 233# - $name.lint.mypy - Runs mypy (if enabled). 234# - $name.lint.pylint - Runs pylint (if enabled). 235# - $name.lint.ruff - Runs ruff (if enabled). 236# - $name.tests - Runs all tests for this package. 237# - $name.install - Installs the package in a venv. 238# - $name.wheel - Builds a Python wheel for the package. 239# 240# All Python packages are instantiated with in pw_build_PYTHON_TOOLCHAIN, 241# regardless of the current toolchain. This prevents Python-specific work, like 242# running Pylint, from occurring multiple times in a build. 243# 244# Args: 245# setup: List of setup file paths (setup.py or pyproject.toml & setup.cfg), 246# which must all be in the same directory. 247# generate_setup: As an alternative to 'setup', generate setup files with the 248# keywords in this scope. 'name' is required. 249# sources: Python sources files in the package. 250# tests: Test files for this Python package. 251# python_deps: Dependencies on other pw_python_packages in the GN build. 252# python_test_deps: Test-only pw_python_package dependencies. 253# other_deps: Dependencies on GN targets that are not pw_python_packages. 254# inputs: Other files to track, such as package_data. 255# proto_library: A pw_proto_library target to embed in this Python package. 256# generate_setup is required in place of setup if proto_library is used. 257# static_analysis: List of static analysis tools to run; "*" (default) runs 258# all tools. The supported tools are "mypy", "pylint" and "ruff". 259# pylintrc: Path to a pylintrc configuration file to use. If not 260# provided, Pylint's default rcfile search is used. As this may 261# use the the local user's configuration file, it is highly 262# recommended to pass this option to specify the rcfile explicitly. 263# mypy_ini: Optional path to a mypy configuration file to use. If not 264# provided, mypy's default configuration file search is used. mypy is 265# executed from the package's setup directory, so mypy.ini files in that 266# directory will take precedence over others. 267# ruff_toml: Path to a ruff.toml configuration file to use. 268# 269template("pw_python_package") { 270 # The Python targets are always instantiated in pw_build_PYTHON_TOOLCHAIN. Use 271 # fully qualified labels so that the toolchain is not lost. 272 _other_deps = [] 273 if (defined(invoker.other_deps)) { 274 foreach(dep, invoker.other_deps) { 275 _other_deps += [ get_label_info(dep, "label_with_toolchain") ] 276 } 277 } 278 279 _python_deps = [] 280 if (defined(invoker.python_deps)) { 281 foreach(dep, invoker.python_deps) { 282 _python_deps += [ get_label_info(dep, "label_with_toolchain") ] 283 } 284 } 285 286 # pw_python_script uses pw_python_package, but with a limited set of features. 287 # _pw_standalone signals that this target is actually a pw_python_script. 288 _is_package = !(defined(invoker._pw_standalone) && invoker._pw_standalone) 289 290 _generate_package = false 291 292 _pydeplabel = get_label_info(":$target_name", "label_with_toolchain") 293 294 # If a package does not run static analysis or if it does but doesn't have 295 # any tests then this variable is not used. 296 not_needed([ "_pydeplabel" ]) 297 298 # Check the generate_setup and import_protos args to determine if this package 299 # is generated. 300 if (_is_package) { 301 assert(defined(invoker.generate_setup) != defined(invoker.setup), 302 "Either 'setup' or 'generate_setup' (but not both) must provided") 303 304 if (defined(invoker.proto_library)) { 305 assert(invoker.proto_library != "", "'proto_library' cannot be empty") 306 assert(defined(invoker.generate_setup), 307 "Python packages that import protos with 'proto_library' must " + 308 "use 'generate_setup' instead of 'setup'") 309 310 _import_protos = [ invoker.proto_library ] 311 312 # Depend on the dependencies of the proto library. 313 _proto = get_label_info(invoker.proto_library, "label_no_toolchain") 314 _toolchain = get_label_info(invoker.proto_library, "toolchain") 315 _python_deps += [ "$_proto.python._deps($_toolchain)" ] 316 } else if (defined(invoker.generate_setup)) { 317 _import_protos = [] 318 } 319 320 if (defined(invoker.generate_setup)) { 321 _generate_package = true 322 _setup_dir = "$target_gen_dir/$target_name.generated_python_package" 323 324 if (defined(invoker.strip_prefix)) { 325 _source_root = invoker.strip_prefix 326 } else { 327 _source_root = "." 328 } 329 } else { 330 # Non-generated packages with sources provided need an __init__.py. 331 assert(!defined(invoker.sources) || invoker.sources == [] || 332 filter_include(invoker.sources, [ "*\b__init__.py" ]) != [], 333 "Python packages must have at least one __init__.py file") 334 335 # Get the directories of the setup files. All must be in the same dir. 336 _setup_dirs = get_path_info(invoker.setup, "dir") 337 _setup_dir = _setup_dirs[0] 338 339 foreach(dir, _setup_dirs) { 340 assert(dir == _setup_dir, 341 "All files in 'setup' must be in the same directory") 342 } 343 344 assert(!defined(invoker.strip_prefix), 345 "'strip_prefix' may only be given if 'generate_setup' is provided") 346 } 347 } 348 349 # Process arguments defaults and set defaults. 350 351 _supported_static_analysis_tools = [ 352 "mypy", 353 "pylint", 354 "ruff", 355 ] 356 not_needed([ "_supported_static_analysis_tools" ]) 357 358 # Argument: static_analysis (list of tool names or "*"); default = "*" (all) 359 if (!defined(invoker.static_analysis) || invoker.static_analysis == "*") { 360 _static_analysis = pw_build_PYTHON_STATIC_ANALYSIS_TOOLS 361 } else { 362 _static_analysis = invoker.static_analysis 363 } 364 365 foreach(_tool, _static_analysis) { 366 assert(_supported_static_analysis_tools + [ _tool ] - [ _tool ] != 367 _supported_static_analysis_tools, 368 "'$_tool' is not a supported static analysis tool") 369 } 370 371 # Argument: sources (list) 372 _sources = [] 373 if (defined(invoker.sources)) { 374 if (_generate_package) { 375 foreach(source, rebase_path(invoker.sources, _source_root)) { 376 _sources += [ "$_setup_dir/$source" ] 377 } 378 } else { 379 _sources += invoker.sources 380 } 381 } 382 383 # Argument: tests (list) 384 _test_sources = [] 385 if (defined(invoker.tests)) { 386 if (_generate_package) { 387 foreach(source, rebase_path(invoker.tests, _source_root)) { 388 _test_sources += [ "$_setup_dir/$source" ] 389 } 390 } else { 391 _test_sources += invoker.tests 392 } 393 } 394 395 # Argument: setup (list) 396 _setup_sources = [] 397 if (defined(invoker.setup)) { 398 _setup_sources = invoker.setup 399 } else if (_generate_package) { 400 _setup_sources = [ 401 "$_setup_dir/pyproject.toml", 402 "$_setup_dir/setup.cfg", 403 ] 404 } 405 406 # Argument: python_test_deps (list) 407 _python_test_deps = _python_deps # include all deps in test deps 408 if (defined(invoker.python_test_deps)) { 409 foreach(dep, invoker.python_test_deps) { 410 _python_test_deps += [ get_label_info(dep, "label_with_toolchain") ] 411 } 412 } 413 414 if (_test_sources == []) { 415 assert(!defined(invoker.python_test_deps), 416 "python_test_deps was provided, but there are no tests in " + 417 get_label_info(":$target_name", "label_no_toolchain")) 418 not_needed([ "_python_test_deps" ]) 419 } 420 421 _all_py_files = 422 _sources + _test_sources + filter_include(_setup_sources, [ "*.py" ]) 423 424 # The pw_python_package subtargets are only instantiated in 425 # pw_build_PYTHON_TOOLCHAIN. Targets in other toolchains just refer to the 426 # targets in this toolchain. 427 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) { 428 # Create the package_metadata.json file. This is used by the 429 # pw_python_distribution template. 430 _package_metadata_json_file = 431 "$target_gen_dir/$target_name/package_metadata.json" 432 433 # Get Python package metadata and write to disk as JSON. 434 _package_metadata = { 435 gn_target_name = 436 get_label_info(":${invoker.target_name}", "label_no_toolchain") 437 438 # Get package source files 439 sources = rebase_path(_sources, root_build_dir) 440 441 # Get setup.cfg, pyproject.toml, or setup.py file 442 setup_sources = rebase_path(_setup_sources, root_build_dir) 443 444 # Get test source files 445 tests = rebase_path(_test_sources, root_build_dir) 446 447 # Get package input files (package data) 448 inputs = [] 449 if (defined(invoker.inputs)) { 450 inputs = rebase_path(invoker.inputs, root_build_dir) 451 } 452 453 # Get generate_setup 454 if (defined(invoker.generate_setup)) { 455 generate_setup = invoker.generate_setup 456 } 457 } 458 459 # Finally, write out the json 460 write_file(_package_metadata_json_file, _package_metadata, "json") 461 462 # Create a target group for the Python package metadata only. This is a 463 # python_action so the setup sources can be included as inputs. 464 pw_python_action("$target_name._package_metadata") { 465 metadata = { 466 pw_python_package_metadata_json = [ _package_metadata_json_file ] 467 } 468 469 script = "$dir_pw_build/py/pw_build/nop.py" 470 471 if (_generate_package) { 472 inputs = [ "$_setup_dir/setup.json" ] 473 } else { 474 inputs = _setup_sources 475 } 476 477 _pw_internal_run_in_venv = false 478 stamp = true 479 480 # Forward the package_metadata subtarget for all python_deps. 481 public_deps = [] 482 foreach(dep, _python_deps) { 483 public_deps += [ get_label_info(dep, "label_no_toolchain") + 484 "._package_metadata($pw_build_PYTHON_TOOLCHAIN)" ] 485 } 486 } 487 488 # Declare the main Python package group. This represents the Python files, 489 # but does not take any actions. GN targets can depend on the package name 490 # to run when any files in the package change. 491 if (_generate_package) { 492 # If this package is generated, mirror the sources to the final directory. 493 pw_mirror_tree("$target_name._mirror_sources_to_out_dir") { 494 directory = _setup_dir 495 496 sources = [] 497 if (defined(invoker.sources)) { 498 sources += invoker.sources 499 } 500 if (defined(invoker.tests)) { 501 sources += invoker.tests 502 } 503 if (defined(invoker.inputs)) { 504 sources += invoker.inputs 505 } 506 507 source_root = _source_root 508 public_deps = _python_deps + _other_deps 509 } 510 511 # Get generated_setup scope and write it to disk as JSON. 512 513 # Expected setup.cfg structure: 514 # https://setuptools.readthedocs.io/en/latest/userguide/declarative_config.html 515 _gen_setup = invoker.generate_setup 516 assert(defined(_gen_setup.metadata), 517 "'metadata = {}' is required in generate_package") 518 519 # Get metadata which should contain at least name. 520 _gen_metadata = { 521 } 522 _gen_metadata = _gen_setup.metadata 523 assert( 524 defined(_gen_metadata.name), 525 "metadata = { name = 'package_name' } is required in generate_package") 526 527 # Get options which should not have packages or package_data. 528 if (defined(_gen_setup.options)) { 529 _gen_options = { 530 } 531 _gen_options = _gen_setup.options 532 assert(!defined(_gen_options.packages) && 533 !defined(_gen_options.package_data), 534 "'packages' and 'package_data' may not be provided " + 535 "in 'generate_package' options.") 536 } 537 538 write_file("$_setup_dir/setup.json", _gen_setup, "json") 539 540 # Generate the setup.py, py.typed, and __init__.py files as needed. 541 action(target_name) { 542 metadata = { 543 pw_python_package_metadata_json = [ _package_metadata_json_file ] 544 } 545 546 script = "$dir_pw_build/py/pw_build/generate_python_package.py" 547 args = [ 548 "--label", 549 get_label_info(":$target_name", "label_no_toolchain"), 550 "--generated-root", 551 rebase_path(_setup_dir, root_build_dir), 552 "--setup-json", 553 rebase_path("$_setup_dir/setup.json", root_build_dir), 554 ] + rebase_path(_sources, root_build_dir) 555 556 # Pass in the .json information files for the imported proto libraries. 557 foreach(proto, _import_protos) { 558 _label = get_label_info(proto, "label_no_toolchain") + 559 ".python($pw_protobuf_compiler_TOOLCHAIN)" 560 _file = get_label_info(_label, "target_gen_dir") + "/" + 561 get_label_info(_label, "name") + ".json" 562 args += [ 563 "--proto-library", 564 rebase_path(_file, root_build_dir), 565 ] 566 } 567 568 if (defined(invoker._pw_module_as_package) && 569 invoker._pw_module_as_package) { 570 args += [ "--module-as-package" ] 571 } 572 573 inputs = [ "$_setup_dir/setup.json" ] 574 575 public_deps = [ ":$target_name._mirror_sources_to_out_dir" ] 576 577 outputs = _setup_sources 578 } 579 } else { 580 # If the package is not generated, use an input group for the sources. 581 pw_input_group(target_name) { 582 metadata = { 583 pw_python_package_metadata_json = [ _package_metadata_json_file ] 584 } 585 inputs = _all_py_files 586 if (defined(invoker.inputs)) { 587 inputs += invoker.inputs 588 } 589 590 public_deps = _python_deps + _other_deps 591 } 592 } 593 594 if (_is_package) { 595 # Builds a Python wheel for this package. Records the output directory 596 # in the pw_python_package_wheels metadata key. 597 598 pw_python_action("$target_name._build_wheel") { 599 _wheel_out_dir = "$target_out_dir/$target_name" 600 _wheel_requirement = "$_wheel_out_dir/requirements.txt" 601 metadata = { 602 pw_python_package_wheels = [ _wheel_out_dir ] 603 } 604 605 script = "$dir_pw_build/py/pw_build/generate_python_wheel.py" 606 607 args = [ 608 "--package-dir", 609 rebase_path(_setup_dir, root_build_dir), 610 "--out-dir", 611 rebase_path(_wheel_out_dir, root_build_dir), 612 ] 613 614 # Add hashes to the _wheel_requirement output. 615 if (pw_build_PYTHON_PIP_INSTALL_REQUIRE_HASHES) { 616 args += [ "--generate-hashes" ] 617 } 618 619 deps = [ ":${invoker.target_name}" ] 620 foreach(dep, _python_deps) { 621 deps += [ string_replace(dep, "(", ".wheel(") ] 622 } 623 624 outputs = [ _wheel_requirement ] 625 } 626 } else { 627 # Stub for non-package targets. 628 group("$target_name._build_wheel") { 629 } 630 } 631 632 # Create the .install and .wheel targets. To limit unnecessary pip 633 # executions, non-generated packages are only reinstalled when their 634 # setup.py changes. However, targets that depend on the .install subtarget 635 # re-run whenever any source files change. 636 # 637 # These targets just represent the source files if this isn't a package. 638 group("$target_name.install") { 639 public_deps = [ ":${invoker.target_name}" ] 640 641 foreach(dep, _python_deps) { 642 public_deps += [ string_replace(dep, "(", ".install(") ] 643 } 644 } 645 646 group("$target_name.wheel") { 647 public_deps = [ ":${invoker.target_name}.install" ] 648 649 if (_is_package) { 650 public_deps += [ ":${invoker.target_name}._build_wheel" ] 651 } 652 653 foreach(dep, _python_deps) { 654 public_deps += [ string_replace(dep, "(", ".wheel(") ] 655 } 656 } 657 658 # Define the static analysis targets for this package. 659 group("$target_name.lint") { 660 deps = [] 661 foreach(_tool, _supported_static_analysis_tools) { 662 deps += [ ":${invoker.target_name}.lint.$_tool" ] 663 } 664 } 665 666 if (_static_analysis != [] || _test_sources != []) { 667 # All packages to install for either general use or test running. 668 _test_install_deps = [ ":$target_name.install" ] 669 670 foreach(dep, _python_test_deps) { 671 _test_install_deps += [ string_replace(dep, "(", ".install(") ] 672 _test_install_deps += [ dep ] 673 } 674 } 675 676 # For packages that are not generated, create targets to run mypy and pylint. 677 foreach(_tool, _static_analysis) { 678 # Run lint tools from the setup or target directory so that the tools detect 679 # config files (e.g. pylintrc or mypy.ini) in that directory. Config files 680 # may be explicitly specified with the pylintrc or mypy_ini arguments. 681 target("_pw_python_static_analysis_$_tool", "$target_name.lint.$_tool") { 682 sources = _all_py_files 683 deps = _test_install_deps 684 python_deps = _python_deps + _python_test_deps 685 686 if (_is_package) { 687 python_metadata_deps = [ _pydeplabel ] 688 } 689 690 _optional_variables = [ 691 "mypy_ini", 692 "pylintrc", 693 "ruff_toml", 694 ] 695 forward_variables_from(invoker, _optional_variables) 696 not_needed(_optional_variables) 697 } 698 } 699 700 foreach(_unused_tool, _supported_static_analysis_tools - _static_analysis) { 701 pw_input_group("$target_name.lint.$_unused_tool") { 702 inputs = [] 703 if (defined(invoker.pylintrc)) { 704 inputs += [ invoker.pylintrc ] 705 } 706 if (defined(invoker.mypy_ini)) { 707 inputs += [ invoker.mypy_ini ] 708 } 709 if (defined(invoker.ruff_toml)) { 710 inputs += [ invoker.ruff_toml ] 711 } 712 } 713 714 # Generated packages with linting disabled never need the whole file list. 715 not_needed([ "_all_py_files" ]) 716 } 717 } else { 718 # Create groups with the public target names ($target_name, $target_name.lint, 719 # $target_name.install, etc.). These are actually wrappers around internal 720 # Python actions instantiated with the default toolchain. This ensures there 721 # is only a single copy of each Python action in the build. 722 # 723 # The $target_name.tests group is created separately below. 724 group("$target_name") { 725 deps = [ ":$target_name($pw_build_PYTHON_TOOLCHAIN)" ] 726 } 727 728 foreach(subtarget, pw_python_package_subtargets - [ "tests" ]) { 729 group("$target_name.$subtarget") { 730 deps = 731 [ ":${invoker.target_name}.$subtarget($pw_build_PYTHON_TOOLCHAIN)" ] 732 } 733 } 734 735 # Everything Python-related is only instantiated in the default toolchain. 736 # Silence not-needed warnings except for in the default toolchain. 737 not_needed("*") 738 not_needed(invoker, "*") 739 } 740 741 # Create a target for each test file. 742 _test_targets = [] 743 744 foreach(test, _test_sources) { 745 if (_is_package) { 746 _name = rebase_path(test, _setup_dir) 747 } else { 748 _name = test 749 } 750 751 _test_target = "$target_name.tests." + string_replace(_name, "/", "_") 752 753 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) { 754 pw_python_action(_test_target) { 755 if (pw_build_PYTHON_TEST_COVERAGE) { 756 module = "coverage" 757 working_directory = 758 rebase_path(get_path_info(test, "dir"), root_build_dir) 759 args = [ 760 "run", 761 "--branch", 762 763 # Include all source files in the working_directory when calculating coverage. 764 "--source=.", 765 766 # Test file to run. 767 get_path_info(test, "file"), 768 ] 769 770 # Set the coverage file to a location in out/python/gen/ 771 _coverage_data_file = "$target_gen_dir/$target_name.coverage" 772 outputs = [ _coverage_data_file ] 773 774 # The coverage tool only allows setting the output with an environment variable. 775 environment = 776 [ "COVERAGE_FILE=" + 777 rebase_path(_coverage_data_file, get_path_info(test, "dir")) ] 778 } else { 779 script = test 780 } 781 782 stamp = true 783 784 # Make sure the python test deps are added to the PYTHONPATH. 785 python_metadata_deps = _python_test_deps 786 787 # If this is a test for a package, add it to PYTHONPATH as well. This is 788 # required if the test source file isn't in the same directory as the 789 # folder containing the package sources to allow local Python imports. 790 if (_is_package) { 791 python_metadata_deps += [ _pydeplabel ] 792 } 793 794 deps = _test_install_deps 795 796 if (pw_build_TEST_TRANSITIVE_PYTHON_DEPS) { 797 foreach(dep, _python_test_deps) { 798 deps += [ string_replace(dep, "(", ".tests(") ] 799 } 800 } 801 } 802 } else { 803 # Create a public version of each test target, so tests can be executed as 804 # //path/to:package.tests.foo.py. 805 group(_test_target) { 806 deps = [ ":$_test_target($pw_build_PYTHON_TOOLCHAIN)" ] 807 } 808 } 809 810 _test_targets += [ ":$_test_target" ] 811 } 812 813 group("$target_name.tests") { 814 deps = _test_targets 815 } 816 817 _pw_create_aliases_if_name_matches_directory(target_name) { 818 } 819} 820 821# Declares a group of Python packages or other Python groups. pw_python_groups 822# expose the same set of subtargets as pw_python_package (e.g. 823# "$group_name.lint" and "$group_name.tests"), but these apply to all packages 824# in deps and their dependencies. 825template("pw_python_group") { 826 if (defined(invoker.python_deps)) { 827 _python_deps = invoker.python_deps 828 } else { 829 _python_deps = [] 830 not_needed([ "invoker" ]) # Allow empty groups. 831 } 832 833 group(target_name) { 834 deps = _python_deps 835 836 if (defined(invoker.other_deps)) { 837 deps += invoker.other_deps 838 } 839 } 840 841 # Create a target group for the Python package metadata only. 842 group("$target_name._package_metadata") { 843 # Forward the package_metadata subtarget for all python_deps. 844 public_deps = [] 845 foreach(dep, _python_deps) { 846 public_deps += [ get_label_info(dep, "label_no_toolchain") + 847 "._package_metadata($pw_build_PYTHON_TOOLCHAIN)" ] 848 } 849 } 850 851 foreach(subtarget, pw_python_package_subtargets) { 852 group("$target_name.$subtarget") { 853 public_deps = [] 854 foreach(dep, _python_deps) { 855 # Split out the toolchain to support deps with a toolchain specified. 856 _target = get_label_info(dep, "label_no_toolchain") 857 _toolchain = get_label_info(dep, "toolchain") 858 public_deps += [ "$_target.$subtarget($_toolchain)" ] 859 } 860 } 861 } 862 863 _pw_create_aliases_if_name_matches_directory(target_name) { 864 } 865} 866 867# Declares Python scripts or tests that are not part of a Python package. 868# Similar to pw_python_package, but only supports a subset of its features. 869# 870# pw_python_script accepts the same arguments as pw_python_package, except 871# `setup` cannot be provided. 872# 873# pw_python_script provides the same subtargets as pw_python_package, but 874# $target_name.install and $target_name.wheel only affect the python_deps of 875# this GN target, not the target itself. 876# 877# pw_python_script allows creating a pw_python_action associated with the 878# script. This is provided by passing an 'action' scope to pw_python_script. 879# This functions like a normal action, with a few additions: the action uses the 880# pw_python_script's python_deps and defaults to using the source file as its 881# 'script' argument, if there is only a single source file. 882template("pw_python_script") { 883 _package_variables = [ 884 "sources", 885 "tests", 886 "python_deps", 887 "python_test_deps", 888 "python_metadata_deps", 889 "other_deps", 890 "inputs", 891 "pylintrc", 892 "mypy_ini", 893 "ruff_toml", 894 "static_analysis", 895 ] 896 897 pw_python_package(target_name) { 898 _pw_standalone = true 899 forward_variables_from(invoker, _package_variables) 900 } 901 902 _pw_create_aliases_if_name_matches_directory(target_name) { 903 } 904 905 if (defined(invoker.action)) { 906 pw_python_action("$target_name.action") { 907 forward_variables_from(invoker.action, "*", [ "python_deps" ]) 908 forward_variables_from(invoker, [ "testonly" ]) 909 python_deps = [ ":${invoker.target_name}" ] 910 911 if (!defined(script) && !defined(module) && defined(invoker.sources)) { 912 _sources = invoker.sources 913 assert(_sources != [] && _sources == [ _sources[0] ], 914 "'script' must be specified unless there is only one source " + 915 "in 'sources'") 916 script = _sources[0] 917 } 918 } 919 } 920} 921 922# Represents a list of Python requirements, as in a requirements.txt. 923# 924# Args: 925# files: One or more requirements.txt files. 926# requirements: A list of requirements.txt-style requirements. 927template("pw_python_requirements") { 928 assert(defined(invoker.files) || defined(invoker.requirements), 929 "pw_python_requirements requires a list of requirements.txt files " + 930 "in the 'files' arg or requirements in 'requirements'") 931 932 _requirements_files = [] 933 934 if (defined(invoker.files)) { 935 _requirements_files += invoker.files 936 } 937 938 if (defined(invoker.requirements)) { 939 _requirements_file = "$target_gen_dir/$target_name.requirements.txt" 940 write_file(_requirements_file, invoker.requirements) 941 _requirements_files += [ _requirements_file ] 942 } 943 944 # The default target represents the requirements themselves. 945 pw_input_group(target_name) { 946 inputs = _requirements_files 947 } 948 949 # Use the same subtargets as pw_python_package so these targets can be listed 950 # as python_deps of pw_python_packages. 951 group("$target_name.install") { 952 # TODO: b/232800695 - Remove reliance on this subtarget existing. 953 } 954 955 # Create stubs for the unused subtargets so that pw_python_requirements can be 956 # used as python_deps. 957 foreach(subtarget, pw_python_package_subtargets - [ "install" ]) { 958 group("$target_name.$subtarget") { 959 } 960 } 961 962 # Create a target group for the Python package metadata only. 963 group("$target_name._package_metadata") { 964 # Forward the package_metadata subtarget for all python_deps. 965 public_deps = [] 966 if (defined(invoker.python_deps)) { 967 foreach(dep, invoker.python_deps) { 968 public_deps += [ get_label_info(dep, "label_no_toolchain") + 969 "._package_metadata($pw_build_PYTHON_TOOLCHAIN)" ] 970 } 971 } 972 } 973 974 _pw_create_aliases_if_name_matches_directory(target_name) { 975 } 976} 977