1#!/usr/bin/env python3 2 3# Copyright 2022 gRPC authors. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import argparse 18import collections 19from doctest import SKIP 20import multiprocessing 21import os 22import re 23import sys 24 25import run_buildozer 26 27# find our home 28ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../..")) 29os.chdir(ROOT) 30 31vendors = collections.defaultdict(list) 32scores = collections.defaultdict(int) 33avoidness = collections.defaultdict(int) 34consumes = {} 35no_update = set() 36buildozer_commands = [] 37original_deps = {} 38original_external_deps = {} 39skip_headers = collections.defaultdict(set) 40 41# TODO(ctiller): ideally we wouldn't hardcode a bunch of paths here. 42# We can likely parse out BUILD files from dependencies to generate this index. 43EXTERNAL_DEPS = { 44 "absl/algorithm/container.h": "absl/algorithm:container", 45 "absl/base/attributes.h": "absl/base:core_headers", 46 "absl/base/call_once.h": "absl/base", 47 "absl/base/config.h": "absl/base:config", 48 # TODO(ctiller) remove this 49 "absl/base/internal/endian.h": "absl/base:endian", 50 "absl/base/thread_annotations.h": "absl/base:core_headers", 51 "absl/container/flat_hash_map.h": "absl/container:flat_hash_map", 52 "absl/container/flat_hash_set.h": "absl/container:flat_hash_set", 53 "absl/container/inlined_vector.h": "absl/container:inlined_vector", 54 "absl/cleanup/cleanup.h": "absl/cleanup", 55 "absl/debugging/failure_signal_handler.h": ( 56 "absl/debugging:failure_signal_handler" 57 ), 58 "absl/debugging/stacktrace.h": "absl/debugging:stacktrace", 59 "absl/debugging/symbolize.h": "absl/debugging:symbolize", 60 "absl/flags/flag.h": "absl/flags:flag", 61 "absl/flags/marshalling.h": "absl/flags:marshalling", 62 "absl/flags/parse.h": "absl/flags:parse", 63 "absl/functional/any_invocable.h": "absl/functional:any_invocable", 64 "absl/functional/bind_front.h": "absl/functional:bind_front", 65 "absl/functional/function_ref.h": "absl/functional:function_ref", 66 "absl/hash/hash.h": "absl/hash", 67 "absl/memory/memory.h": "absl/memory", 68 "absl/meta/type_traits.h": "absl/meta:type_traits", 69 "absl/numeric/int128.h": "absl/numeric:int128", 70 "absl/random/random.h": "absl/random", 71 "absl/random/bit_gen_ref.h": "absl/random:bit_gen_ref", 72 "absl/random/mocking_bit_gen.h": "absl/random:mocking_bit_gen", 73 "absl/random/distributions.h": "absl/random:distributions", 74 "absl/random/uniform_int_distribution.h": "absl/random:distributions", 75 "absl/status/status.h": "absl/status", 76 "absl/status/statusor.h": "absl/status:statusor", 77 "absl/strings/ascii.h": "absl/strings", 78 "absl/strings/cord.h": "absl/strings:cord", 79 "absl/strings/escaping.h": "absl/strings", 80 "absl/strings/match.h": "absl/strings", 81 "absl/strings/numbers.h": "absl/strings", 82 "absl/strings/str_cat.h": "absl/strings", 83 "absl/strings/str_format.h": "absl/strings:str_format", 84 "absl/strings/str_join.h": "absl/strings", 85 "absl/strings/str_replace.h": "absl/strings", 86 "absl/strings/str_split.h": "absl/strings", 87 "absl/strings/string_view.h": "absl/strings", 88 "absl/strings/strip.h": "absl/strings", 89 "absl/strings/substitute.h": "absl/strings", 90 "absl/synchronization/mutex.h": "absl/synchronization", 91 "absl/synchronization/notification.h": "absl/synchronization", 92 "absl/time/clock.h": "absl/time", 93 "absl/time/time.h": "absl/time", 94 "absl/types/optional.h": "absl/types:optional", 95 "absl/types/span.h": "absl/types:span", 96 "absl/types/variant.h": "absl/types:variant", 97 "absl/utility/utility.h": "absl/utility", 98 "address_sorting/address_sorting.h": "address_sorting", 99 "google/cloud/opentelemetry/resource_detector.h": "google_cloud_cpp:opentelemetry", 100 "opentelemetry/common/attribute_value.h": "otel/api", 101 "opentelemetry/common/key_value_iterable.h": "otel/api", 102 "opentelemetry/nostd/function_ref.h": "otel/api", 103 "opentelemetry/nostd/string_view.h": "otel/api", 104 "opentelemetry/context/context.h": "otel/api", 105 "opentelemetry/metrics/meter.h": "otel/api", 106 "opentelemetry/metrics/meter_provider.h": "otel/api", 107 "opentelemetry/metrics/provider.h": "otel/api", 108 "opentelemetry/metrics/sync_instruments.h": "otel/api", 109 "opentelemetry/nostd/shared_ptr.h": "otel/api", 110 "opentelemetry/nostd/unique_ptr.h": "otel/api", 111 "opentelemetry/sdk/metrics/meter_provider.h": "otel/sdk/src/metrics", 112 "opentelemetry/sdk/common/attribute_utils.h": "otel/sdk:headers", 113 "opentelemetry/sdk/resource/resource.h": "otel/sdk:headers", 114 "opentelemetry/sdk/resource/resource_detector.h": "otel/sdk:headers", 115 "opentelemetry/sdk/resource/semantic_conventions.h": "otel/sdk:headers", 116 "ares.h": "cares", 117 "fuzztest/fuzztest.h": ["fuzztest", "fuzztest_main"], 118 "google/api/monitored_resource.pb.h": ( 119 "google/api:monitored_resource_cc_proto" 120 ), 121 "google/devtools/cloudtrace/v2/tracing.grpc.pb.h": ( 122 "googleapis_trace_grpc_service" 123 ), 124 "google/logging/v2/logging.grpc.pb.h": "googleapis_logging_grpc_service", 125 "google/logging/v2/logging.pb.h": "googleapis_logging_cc_proto", 126 "google/logging/v2/log_entry.pb.h": "googleapis_logging_cc_proto", 127 "google/monitoring/v3/metric_service.grpc.pb.h": ( 128 "googleapis_monitoring_grpc_service" 129 ), 130 "gmock/gmock.h": "gtest", 131 "gtest/gtest.h": "gtest", 132 "opencensus/exporters/stats/stackdriver/stackdriver_exporter.h": ( 133 "opencensus-stats-stackdriver_exporter" 134 ), 135 "opencensus/exporters/trace/stackdriver/stackdriver_exporter.h": ( 136 "opencensus-trace-stackdriver_exporter" 137 ), 138 "opencensus/trace/context_util.h": "opencensus-trace-context_util", 139 "opencensus/trace/propagation/grpc_trace_bin.h": ( 140 "opencensus-trace-propagation" 141 ), 142 "opencensus/tags/context_util.h": "opencensus-tags-context_util", 143 "opencensus/trace/span_context.h": "opencensus-trace-span_context", 144 "openssl/base.h": "libssl", 145 "openssl/bio.h": "libssl", 146 "openssl/bn.h": "libcrypto", 147 "openssl/buffer.h": "libcrypto", 148 "openssl/crypto.h": "libcrypto", 149 "openssl/digest.h": "libssl", 150 "openssl/engine.h": "libcrypto", 151 "openssl/err.h": "libcrypto", 152 "openssl/evp.h": "libcrypto", 153 "openssl/hmac.h": "libcrypto", 154 "openssl/mem.h": "libcrypto", 155 "openssl/param_build.h": "libcrypto", 156 "openssl/pem.h": "libcrypto", 157 "openssl/rsa.h": "libcrypto", 158 "openssl/sha.h": "libcrypto", 159 "openssl/ssl.h": "libssl", 160 "openssl/tls1.h": "libssl", 161 "openssl/x509.h": "libcrypto", 162 "openssl/x509v3.h": "libcrypto", 163 "re2/re2.h": "re2", 164 "upb/base/status.hpp": "upb_base_lib", 165 "upb/base/string_view.h": "upb_base_lib", 166 "upb/message/map.h": "upb_message_lib", 167 "upb/reflection/def.h": "upb_reflection", 168 "upb/json/encode.h": "upb_json_lib", 169 "upb/mem/arena.h": "upb_mem_lib", 170 "upb/mem/arena.hpp": "upb_mem_lib", 171 "upb/text/encode.h": "upb_textformat_lib", 172 "upb/reflection/def.hpp": "upb_reflection", 173 "upb/upb.h": "upb_amalgamation_lib", 174 "xxhash.h": "xxhash", 175 "zlib.h": "madler_zlib", 176} 177 178INTERNAL_DEPS = { 179 "test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h": ( 180 "//test/core/event_engine/fuzzing_event_engine" 181 ), 182 "test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.pb.h": "//test/core/event_engine/fuzzing_event_engine:fuzzing_event_engine_proto", 183 "test/core/experiments/test_experiments.h": "//test/core/experiments:test_experiments_lib", 184 "google/api/expr/v1alpha1/syntax.upb.h": "google_api_expr_v1alpha1_syntax_upb", 185 "google/rpc/status.upb.h": "google_rpc_status_upb", 186 "google/protobuf/any.upb.h": "protobuf_any_upb", 187 "google/protobuf/duration.upb.h": "protobuf_duration_upb", 188 "google/protobuf/struct.upb.h": "protobuf_struct_upb", 189 "google/protobuf/timestamp.upb.h": "protobuf_timestamp_upb", 190 "google/protobuf/wrappers.upb.h": "protobuf_wrappers_upb", 191 "grpc/status.h": "grpc_public_hdrs", 192 "src/proto/grpc/channelz/channelz.grpc.pb.h": ( 193 "//src/proto/grpc/channelz:channelz_proto" 194 ), 195 "src/proto/grpc/core/stats.pb.h": "//src/proto/grpc/core:stats_proto", 196 "src/proto/grpc/health/v1/health.upb.h": "grpc_health_upb", 197 "src/proto/grpc/lb/v1/load_reporter.grpc.pb.h": ( 198 "//src/proto/grpc/lb/v1:load_reporter_proto" 199 ), 200 "src/proto/grpc/lb/v1/load_balancer.upb.h": "grpc_lb_upb", 201 "src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h": ( 202 "//src/proto/grpc/reflection/v1alpha:reflection_proto" 203 ), 204 "src/proto/grpc/gcp/transport_security_common.upb.h": "alts_upb", 205 "src/proto/grpc/gcp/handshaker.upb.h": "alts_upb", 206 "src/proto/grpc/gcp/altscontext.upb.h": "alts_upb", 207 "src/proto/grpc/lookup/v1/rls.upb.h": "rls_upb", 208 "src/proto/grpc/lookup/v1/rls_config.upb.h": "rls_config_upb", 209 "src/proto/grpc/lookup/v1/rls_config.upbdefs.h": "rls_config_upbdefs", 210 "src/proto/grpc/testing/xds/v3/csds.grpc.pb.h": ( 211 "//src/proto/grpc/testing/xds/v3:csds_proto" 212 ), 213 "xds/data/orca/v3/orca_load_report.upb.h": "xds_orca_upb", 214 "xds/service/orca/v3/orca.upb.h": "xds_orca_service_upb", 215 "xds/type/v3/typed_struct.upb.h": "xds_type_upb", 216} 217 218 219class FakeSelects: 220 def config_setting_group(self, **kwargs): 221 pass 222 223 224num_cc_libraries = 0 225num_opted_out_cc_libraries = 0 226parsing_path = None 227 228 229# Convert the source or header target to a relative path. 230def _get_filename(name, parsing_path): 231 filename = "%s%s" % ( 232 ( 233 parsing_path + "/" 234 if (parsing_path and not name.startswith("//")) 235 else "" 236 ), 237 name, 238 ) 239 filename = filename.replace("//:", "") 240 filename = filename.replace("//src/core:", "src/core/") 241 filename = filename.replace( 242 "//src/cpp/ext/filters/census:", "src/cpp/ext/filters/census/" 243 ) 244 return filename 245 246 247def grpc_cc_library( 248 name, 249 hdrs=[], 250 public_hdrs=[], 251 srcs=[], 252 select_deps=None, 253 tags=[], 254 deps=[], 255 external_deps=[], 256 proto=None, 257 **kwargs, 258): 259 global args 260 global num_cc_libraries 261 global num_opted_out_cc_libraries 262 global parsing_path 263 assert parsing_path is not None 264 name = "//%s:%s" % (parsing_path, name) 265 num_cc_libraries += 1 266 if select_deps or "nofixdeps" in tags: 267 if args.whats_left and not select_deps and "nofixdeps" not in tags: 268 num_opted_out_cc_libraries += 1 269 print("Not opted in: {}".format(name)) 270 no_update.add(name) 271 scores[name] = len(public_hdrs + hdrs) 272 # avoid_dep is the internal way of saying prefer something else 273 # we add grpc_avoid_dep to allow internal grpc-only stuff to avoid each 274 # other, whilst not biasing dependent projects 275 if "avoid_dep" in tags or "grpc_avoid_dep" in tags: 276 avoidness[name] += 10 277 if proto: 278 proto_hdr = "%s%s" % ( 279 (parsing_path + "/" if parsing_path else ""), 280 proto.replace(".proto", ".pb.h"), 281 ) 282 skip_headers[name].add(proto_hdr) 283 284 for hdr in hdrs + public_hdrs: 285 vendors[_get_filename(hdr, parsing_path)].append(name) 286 inc = set() 287 original_deps[name] = frozenset(deps) 288 original_external_deps[name] = frozenset(external_deps) 289 for src in hdrs + public_hdrs + srcs: 290 for line in open(_get_filename(src, parsing_path)): 291 m = re.search(r"^#include <(.*)>", line) 292 if m: 293 inc.add(m.group(1)) 294 m = re.search(r'^#include "(.*)"', line) 295 if m: 296 inc.add(m.group(1)) 297 consumes[name] = list(inc) 298 299 300def grpc_proto_library(name, srcs, **kwargs): 301 global parsing_path 302 assert parsing_path is not None 303 name = "//%s:%s" % (parsing_path, name) 304 for src in srcs: 305 proto_hdr = src.replace(".proto", ".pb.h") 306 vendors[_get_filename(proto_hdr, parsing_path)].append(name) 307 308 309def buildozer(cmd, target): 310 buildozer_commands.append("%s|%s" % (cmd, target)) 311 312 313def buildozer_set_list(name, values, target, via=""): 314 if not values: 315 buildozer("remove %s" % name, target) 316 return 317 adjust = via if via else name 318 buildozer( 319 "set %s %s" % (adjust, " ".join('"%s"' % s for s in values)), target 320 ) 321 if via: 322 buildozer("remove %s" % name, target) 323 buildozer("rename %s %s" % (via, name), target) 324 325 326def score_edit_distance(proposed, existing): 327 """Score a proposed change primarily by edit distance""" 328 sum = 0 329 for p in proposed: 330 if p not in existing: 331 sum += 1 332 for e in existing: 333 if e not in proposed: 334 sum += 1 335 return sum 336 337 338def total_score(proposal): 339 return sum(scores[dep] for dep in proposal) 340 341 342def total_avoidness(proposal): 343 return sum(avoidness[dep] for dep in proposal) 344 345 346def score_list_size(proposed, existing): 347 """Score a proposed change primarily by number of dependencies""" 348 return len(proposed) 349 350 351def score_best(proposed, existing): 352 """Score a proposed change primarily by dependency score""" 353 return 0 354 355 356SCORERS = { 357 "edit_distance": score_edit_distance, 358 "list_size": score_list_size, 359 "best": score_best, 360} 361 362parser = argparse.ArgumentParser(description="Fix build dependencies") 363parser.add_argument("targets", nargs="+", help="targets to fix") 364parser.add_argument( 365 "--score", 366 type=str, 367 default="edit_distance", 368 help="scoring function to use: one of " + ", ".join(SCORERS.keys()), 369) 370parser.add_argument( 371 "--whats_left", 372 action="store_true", 373 default=False, 374 help="show what is left to opt in", 375) 376parser.add_argument( 377 "--explain", 378 action="store_true", 379 default=False, 380 help="try to explain some decisions", 381) 382parser.add_argument( 383 "--why", 384 type=str, 385 default=None, 386 help="with --explain, target why a given dependency is needed", 387) 388args = parser.parse_args() 389 390for dirname in [ 391 "", 392 "src/core", 393 "src/cpp/ext/gcp", 394 "src/cpp/ext/csm", 395 "src/cpp/ext/otel", 396 "test/core/backoff", 397 "test/core/experiments", 398 "test/core/uri", 399 "test/core/util", 400 "test/core/end2end", 401 "test/core/event_engine", 402 "test/core/filters", 403 "test/core/promise", 404 "test/core/resource_quota", 405 "test/core/transport/chaotic_good", 406 "test/core/transport/test_suite", 407 "fuzztest", 408 "fuzztest/core/channel", 409 "fuzztest/core/transport/chttp2", 410]: 411 parsing_path = dirname 412 exec( 413 open("%sBUILD" % (dirname + "/" if dirname else ""), "r").read(), 414 { 415 "load": lambda filename, *args: None, 416 "licenses": lambda licenses: None, 417 "package": lambda **kwargs: None, 418 "exports_files": lambda files, visibility=None: None, 419 "bool_flag": lambda **kwargs: None, 420 "config_setting": lambda **kwargs: None, 421 "selects": FakeSelects(), 422 "python_config_settings": lambda **kwargs: None, 423 "grpc_cc_binary": grpc_cc_library, 424 "grpc_cc_library": grpc_cc_library, 425 "grpc_cc_test": grpc_cc_library, 426 "grpc_core_end2end_test": lambda **kwargs: None, 427 "grpc_transport_test": lambda **kwargs: None, 428 "grpc_fuzzer": grpc_cc_library, 429 "grpc_fuzz_test": grpc_cc_library, 430 "grpc_proto_fuzzer": grpc_cc_library, 431 "grpc_proto_library": grpc_proto_library, 432 "select": lambda d: d["//conditions:default"], 433 "glob": lambda files, **kwargs: None, 434 "grpc_end2end_tests": lambda: None, 435 "grpc_upb_proto_library": lambda name, **kwargs: None, 436 "grpc_upb_proto_reflection_library": lambda name, **kwargs: None, 437 "grpc_generate_one_off_targets": lambda: None, 438 "grpc_generate_one_off_internal_targets": lambda: None, 439 "grpc_package": lambda **kwargs: None, 440 "filegroup": lambda name, **kwargs: None, 441 "sh_library": lambda name, **kwargs: None, 442 "platform": lambda name, **kwargs: None, 443 }, 444 {}, 445 ) 446 parsing_path = None 447 448if args.whats_left: 449 print( 450 "{}/{} libraries are opted in".format( 451 num_cc_libraries - num_opted_out_cc_libraries, num_cc_libraries 452 ) 453 ) 454 455 456def make_relative_path(dep, lib): 457 if lib is None: 458 return dep 459 lib_path = lib[: lib.rfind(":") + 1] 460 if dep.startswith(lib_path): 461 return dep[len(lib_path) :] 462 return dep 463 464 465if args.whats_left: 466 print( 467 "{}/{} libraries are opted in".format( 468 num_cc_libraries - num_opted_out_cc_libraries, num_cc_libraries 469 ) 470 ) 471 472 473# Keeps track of all possible sets of dependencies that could satify the 474# problem. (models the list monad in Haskell!) 475class Choices: 476 def __init__(self, library, substitutions): 477 self.library = library 478 self.to_add = [] 479 self.to_remove = [] 480 self.substitutions = substitutions 481 482 def add_one_of(self, choices, trigger): 483 if not choices: 484 return 485 choices = sum( 486 [self.apply_substitutions(choice) for choice in choices], [] 487 ) 488 if args.explain and (args.why is None or args.why in choices): 489 print( 490 "{}: Adding one of {} for {}".format( 491 self.library, choices, trigger 492 ) 493 ) 494 self.to_add.append( 495 tuple( 496 make_relative_path(choice, self.library) for choice in choices 497 ) 498 ) 499 500 def add(self, choice, trigger): 501 self.add_one_of([choice], trigger) 502 503 def remove(self, remove): 504 for remove in self.apply_substitutions(remove): 505 self.to_remove.append(make_relative_path(remove, self.library)) 506 507 def apply_substitutions(self, dep): 508 if dep in self.substitutions: 509 return self.substitutions[dep] 510 return [dep] 511 512 def best(self, scorer): 513 choices = set() 514 choices.add(frozenset()) 515 516 for add in sorted(set(self.to_add), key=lambda x: (len(x), x)): 517 new_choices = set() 518 for append_choice in add: 519 for choice in choices: 520 new_choices.add(choice.union([append_choice])) 521 choices = new_choices 522 for remove in sorted(set(self.to_remove)): 523 new_choices = set() 524 for choice in choices: 525 new_choices.add(choice.difference([remove])) 526 choices = new_choices 527 528 best = None 529 530 def final_scorer(x): 531 return (total_avoidness(x), scorer(x), total_score(x)) 532 533 for choice in choices: 534 if best is None or final_scorer(choice) < final_scorer(best): 535 best = choice 536 return best 537 538 539def make_library(library): 540 error = False 541 hdrs = sorted(consumes[library]) 542 # we need a little trickery here since grpc_base has channel.cc, which calls grpc_init 543 # which is in grpc, which is illegal but hard to change 544 # once EventEngine lands we can clean this up 545 deps = Choices( 546 library, 547 {"//:grpc_base": ["//:grpc", "//:grpc_unsecure"]} 548 if library.startswith("//test/") 549 else {}, 550 ) 551 external_deps = Choices(None, {}) 552 for hdr in hdrs: 553 if hdr in skip_headers[library]: 554 continue 555 556 if hdr == "systemd/sd-daemon.h": 557 continue 558 559 if hdr == "src/core/lib/profiling/stap_probes.h": 560 continue 561 562 if hdr.startswith("src/libfuzzer/"): 563 continue 564 565 if hdr == "grpc/grpc.h" and library.startswith("//test:"): 566 # not the root build including grpc.h ==> //:grpc 567 deps.add_one_of(["//:grpc", "//:grpc_unsecure"], hdr) 568 continue 569 570 if hdr in INTERNAL_DEPS: 571 dep = INTERNAL_DEPS[hdr] 572 if isinstance(dep, list): 573 for d in dep: 574 deps.add(d, hdr) 575 else: 576 if not ("//" in dep): 577 dep = "//:" + dep 578 deps.add(dep, hdr) 579 continue 580 581 if hdr in vendors: 582 deps.add_one_of(vendors[hdr], hdr) 583 continue 584 585 if "include/" + hdr in vendors: 586 deps.add_one_of(vendors["include/" + hdr], hdr) 587 continue 588 589 if "." not in hdr: 590 # assume a c++ system include 591 continue 592 593 if hdr in EXTERNAL_DEPS: 594 if isinstance(EXTERNAL_DEPS[hdr], list): 595 for dep in EXTERNAL_DEPS[hdr]: 596 external_deps.add(dep, hdr) 597 else: 598 external_deps.add(EXTERNAL_DEPS[hdr], hdr) 599 continue 600 601 if hdr.startswith("opencensus/"): 602 trail = hdr[len("opencensus/") :] 603 trail = trail[: trail.find("/")] 604 external_deps.add("opencensus-" + trail, hdr) 605 continue 606 607 if hdr.startswith("envoy/"): 608 path, file = os.path.split(hdr) 609 file = file.split(".") 610 path = path.split("/") 611 dep = "_".join(path[:-1] + [file[1]]) 612 deps.add(dep, hdr) 613 continue 614 615 if hdr.startswith("google/protobuf/") and not hdr.endswith(".upb.h"): 616 external_deps.add("protobuf_headers", hdr) 617 continue 618 619 if "/" not in hdr: 620 # assume a system include 621 continue 622 623 is_sys_include = False 624 for sys_path in [ 625 "sys", 626 "arpa", 627 "gperftools", 628 "netinet", 629 "linux", 630 "android", 631 "mach", 632 "net", 633 "CoreFoundation", 634 ]: 635 if hdr.startswith(sys_path + "/"): 636 is_sys_include = True 637 break 638 if is_sys_include: 639 # assume a system include 640 continue 641 642 print( 643 "# ERROR: can't categorize header: %s used by %s" % (hdr, library) 644 ) 645 error = True 646 647 deps.remove(library) 648 649 deps = sorted( 650 deps.best(lambda x: SCORERS[args.score](x, original_deps[library])) 651 ) 652 external_deps = sorted( 653 external_deps.best( 654 lambda x: SCORERS[args.score](x, original_external_deps[library]) 655 ) 656 ) 657 658 return (library, error, deps, external_deps) 659 660 661def matches_target(library, target): 662 if not target.startswith("//"): 663 if "/" in target: 664 target = "//" + target 665 else: 666 target = "//:" + target 667 if target == "..." or target == "//...": 668 return True 669 if target.endswith("/..."): 670 return library.startswith(target[:-4]) 671 return library == target 672 673 674def main() -> None: 675 update_libraries = [] 676 for library in sorted(consumes.keys()): 677 if library in no_update: 678 continue 679 for target in args.targets: 680 if matches_target(library, target): 681 update_libraries.append(library) 682 break 683 with multiprocessing.Pool(processes=multiprocessing.cpu_count()) as p: 684 updated_libraries = p.map(make_library, update_libraries, 1) 685 686 error = False 687 for library, lib_error, deps, external_deps in updated_libraries: 688 if lib_error: 689 error = True 690 continue 691 buildozer_set_list("external_deps", external_deps, library, via="deps") 692 buildozer_set_list("deps", deps, library) 693 694 run_buildozer.run_buildozer(buildozer_commands) 695 696 if error: 697 sys.exit(1) 698 699 700if __name__ == "__main__": 701 main() 702