1// Copyright 2017 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package build 16 17import ( 18 "encoding/json" 19 "errors" 20 "fmt" 21 "io/ioutil" 22 "math/rand" 23 "os" 24 "os/exec" 25 "os/user" 26 "path/filepath" 27 "runtime" 28 "strconv" 29 "strings" 30 "syscall" 31 "time" 32 33 "android/soong/finder/fs" 34 "android/soong/shared" 35 "android/soong/ui/metrics" 36 37 "google.golang.org/protobuf/proto" 38 39 smpb "android/soong/ui/metrics/metrics_proto" 40) 41 42const ( 43 envConfigDir = "vendor/google/tools/soong_config" 44 jsonSuffix = "json" 45 abfsSrcDir = "/src" 46) 47 48var ( 49 rbeRandPrefix int 50 googleProdCredsExistCache bool 51) 52 53func init() { 54 rand.Seed(time.Now().UnixNano()) 55 rbeRandPrefix = rand.Intn(1000) 56} 57 58// Which builder are we using? 59type ninjaCommandType = int 60 61const ( 62 _ = iota 63 NINJA_NINJA 64 NINJA_N2 65 NINJA_SISO 66) 67 68type Config struct{ *configImpl } 69 70type configImpl struct { 71 // Some targets that are implemented in soong_build 72 arguments []string 73 goma bool 74 environ *Environment 75 distDir string 76 buildDateTime string 77 logsPrefix string 78 79 // From the arguments 80 parallel int 81 keepGoing int 82 verbose bool 83 checkbuild bool 84 dist bool 85 jsonModuleGraph bool 86 reportMkMetrics bool // Collect and report mk2bp migration progress metrics. 87 soongDocs bool 88 skipConfig bool 89 skipKati bool 90 skipKatiNinja bool 91 skipSoong bool 92 skipNinja bool 93 skipSoongTests bool 94 searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces 95 skipMetricsUpload bool 96 buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time 97 buildFromSourceStub bool 98 incrementalBuildActions bool 99 ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built 100 101 // From the product config 102 katiArgs []string 103 ninjaArgs []string 104 katiSuffix string 105 targetDevice string 106 targetDeviceDir string 107 sandboxConfig *SandboxConfig 108 109 // Autodetected 110 totalRAM uint64 111 systemCpuInfo *metrics.CpuInfo 112 systemMemInfo *metrics.MemInfo 113 114 brokenDupRules bool 115 brokenUsesNetwork bool 116 brokenNinjaEnvVars []string 117 brokenMissingOutputs bool 118 119 pathReplaced bool 120 121 // Set by multiproduct_kati 122 emptyNinjaFile bool 123 124 metricsUploader string 125 126 includeTags []string 127 sourceRootDirs []string 128 129 // Data source to write ninja weight list 130 ninjaWeightListSource NinjaWeightListSource 131 132 // This file is a detailed dump of all soong-defined modules for debugging purposes. 133 // There's quite a bit of overlap with module-info.json and soong module graph. We 134 // could consider merging them. 135 moduleDebugFile string 136 137 // Which builder are we using 138 ninjaCommand ninjaCommandType 139} 140 141type NinjaWeightListSource uint 142 143const ( 144 // ninja doesn't use weight list. 145 NOT_USED NinjaWeightListSource = iota 146 // ninja uses weight list based on previous builds by ninja log 147 NINJA_LOG 148 // ninja thinks every task has the same weight. 149 EVENLY_DISTRIBUTED 150 // ninja uses an external custom weight list 151 EXTERNAL_FILE 152 // ninja uses a prioritized module list from Soong 153 HINT_FROM_SOONG 154 // If ninja log exists, use NINJA_LOG, if not, use HINT_FROM_SOONG instead. 155 // We can assume it is an incremental build if ninja log exists. 156 DEFAULT 157) 158const srcDirFileCheck = "build/soong/root.bp" 159 160var buildFiles = []string{"Android.mk", "Android.bp"} 161 162type BuildAction uint 163 164const ( 165 // Builds all of the modules and their dependencies of a specified directory, relative to the root 166 // directory of the source tree. 167 BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota 168 169 // Builds all of the modules and their dependencies of a list of specified directories. All specified 170 // directories are relative to the root directory of the source tree. 171 BUILD_MODULES_IN_DIRECTORIES 172 173 // Build a list of specified modules. If none was specified, simply build the whole source tree. 174 BUILD_MODULES 175) 176 177// checkTopDir validates that the current directory is at the root directory of the source tree. 178func checkTopDir(ctx Context) { 179 if _, err := os.Stat(srcDirFileCheck); err != nil { 180 if os.IsNotExist(err) { 181 ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck) 182 } 183 ctx.Fatalln("Error verifying tree state:", err) 184 } 185} 186 187func loadEnvConfig(ctx Context, config *configImpl, bc string) error { 188 if bc == "" { 189 return nil 190 } 191 192 configDirs := []string{ 193 config.OutDir(), 194 os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"), 195 envConfigDir, 196 } 197 for _, dir := range configDirs { 198 cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix)) 199 envVarsJSON, err := ioutil.ReadFile(cfgFile) 200 if err != nil { 201 continue 202 } 203 ctx.Verbosef("Loading config file %v\n", cfgFile) 204 var envVars map[string]map[string]string 205 if err := json.Unmarshal(envVarsJSON, &envVars); err != nil { 206 fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error()) 207 continue 208 } 209 for k, v := range envVars["env"] { 210 if os.Getenv(k) != "" { 211 continue 212 } 213 config.environ.Set(k, v) 214 } 215 ctx.Verbosef("Finished loading config file %v\n", cfgFile) 216 break 217 } 218 219 return nil 220} 221 222func NewConfig(ctx Context, args ...string) Config { 223 ret := &configImpl{ 224 environ: OsEnvironment(), 225 sandboxConfig: &SandboxConfig{}, 226 ninjaWeightListSource: DEFAULT, 227 } 228 wd, err := os.Getwd() 229 if err != nil { 230 ctx.Fatalln("Failed to get working directory:", err) 231 } 232 233 // Skip soong tests by default on Linux 234 if runtime.GOOS == "linux" { 235 ret.skipSoongTests = true 236 } 237 238 // Default matching ninja 239 ret.parallel = runtime.NumCPU() + 2 240 ret.keepGoing = 1 241 242 ret.totalRAM = detectTotalRAM(ctx) 243 ret.systemCpuInfo, err = metrics.NewCpuInfo(fs.OsFs) 244 if err != nil { 245 ctx.Fatalln("Failed to get cpuinfo:", err) 246 } 247 ret.systemMemInfo, err = metrics.NewMemInfo(fs.OsFs) 248 if err != nil { 249 ctx.Fatalln("Failed to get meminfo:", err) 250 } 251 ret.parseArgs(ctx, args) 252 253 if ret.ninjaWeightListSource == HINT_FROM_SOONG { 254 ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "always") 255 } else if ret.ninjaWeightListSource == DEFAULT { 256 defaultNinjaWeightListSource := NINJA_LOG 257 if _, err := os.Stat(filepath.Join(ret.OutDir(), ninjaLogFileName)); errors.Is(err, os.ErrNotExist) { 258 ctx.Verboseln("$OUT/.ninja_log doesn't exist, use HINT_FROM_SOONG instead") 259 defaultNinjaWeightListSource = HINT_FROM_SOONG 260 } else { 261 ctx.Verboseln("$OUT/.ninja_log exist, use NINJA_LOG") 262 } 263 ret.ninjaWeightListSource = defaultNinjaWeightListSource 264 // soong_build generates ninja hint depending on ninja log existence. 265 // Set it "depend" to avoid soong re-run due to env variable change. 266 ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "depend") 267 } 268 269 // Make sure OUT_DIR is set appropriately 270 if outDir, ok := ret.environ.Get("OUT_DIR"); ok { 271 ret.environ.Set("OUT_DIR", ret.sandboxPath(wd, filepath.Clean(outDir))) 272 } else { 273 outDir := "out" 274 if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok { 275 outDir = filepath.Join(baseDir, filepath.Base(wd)) 276 } 277 ret.environ.Set("OUT_DIR", ret.sandboxPath(wd, outDir)) 278 } 279 280 // loadEnvConfig needs to know what the OUT_DIR is, so it should 281 // be called after we determine the appropriate out directory. 282 bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG") 283 284 if bc != "" { 285 if err := loadEnvConfig(ctx, ret, bc); err != nil { 286 ctx.Fatalln("Failed to parse env config files: %v", err) 287 } 288 if !ret.canSupportRBE() { 289 // Explicitly set USE_RBE env variable to false when we cannot run 290 // an RBE build to avoid ninja local execution pool issues. 291 ret.environ.Set("USE_RBE", "false") 292 } 293 } 294 295 if distDir, ok := ret.environ.Get("DIST_DIR"); ok { 296 ret.distDir = filepath.Clean(distDir) 297 } else { 298 ret.distDir = filepath.Join(ret.OutDir(), "dist") 299 } 300 301 if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok { 302 ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false") 303 } 304 305 if os.Getenv("GENERATE_SOONG_DEBUG") == "true" { 306 ret.moduleDebugFile, _ = filepath.Abs(shared.JoinPath(ret.SoongOutDir(), "soong-debug-info.json")) 307 } 308 309 // If SOONG_USE_PARTIAL_COMPILE is set, make it one of "true" or the empty string. 310 // This simplifies the generated Ninja rules, so that they only need to check for the empty string. 311 if value, ok := os.LookupEnv("SOONG_USE_PARTIAL_COMPILE"); ok { 312 if value == "true" || value == "1" || value == "y" || value == "yes" { 313 value = "true" 314 } else { 315 value = "" 316 } 317 err = os.Setenv("SOONG_USE_PARTIAL_COMPILE", value) 318 if err != nil { 319 ctx.Fatalln("Failed to set SOONG_USE_PARTIAL_COMPILE: %v", err) 320 } 321 } 322 323 ret.ninjaCommand = NINJA_NINJA 324 switch os.Getenv("SOONG_NINJA") { 325 case "n2": 326 ret.ninjaCommand = NINJA_N2 327 case "siso": 328 ret.ninjaCommand = NINJA_SISO 329 default: 330 if os.Getenv("SOONG_USE_N2") == "true" { 331 ret.ninjaCommand = NINJA_N2 332 } 333 } 334 335 ret.environ.Unset( 336 // We're already using it 337 "USE_SOONG_UI", 338 339 // We should never use GOROOT/GOPATH from the shell environment 340 "GOROOT", 341 "GOPATH", 342 343 // These should only come from Soong, not the environment. 344 "CLANG", 345 "CLANG_CXX", 346 "CCC_CC", 347 "CCC_CXX", 348 349 // Used by the goma compiler wrapper, but should only be set by 350 // gomacc 351 "GOMACC_PATH", 352 353 // We handle this above 354 "OUT_DIR_COMMON_BASE", 355 356 // This is handled above too, and set for individual commands later 357 "DIST_DIR", 358 359 // Variables that have caused problems in the past 360 "BASH_ENV", 361 "CDPATH", 362 "DISPLAY", 363 "GREP_OPTIONS", 364 "JAVAC", 365 "LEX", 366 "NDK_ROOT", 367 "POSIXLY_CORRECT", 368 369 // Drop make flags 370 "MAKEFLAGS", 371 "MAKELEVEL", 372 "MFLAGS", 373 374 // Set in envsetup.sh, reset in makefiles 375 "ANDROID_JAVA_TOOLCHAIN", 376 377 // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional 378 "ANDROID_BUILD_TOP", 379 "ANDROID_HOST_OUT", 380 "ANDROID_PRODUCT_OUT", 381 "ANDROID_HOST_OUT_TESTCASES", 382 "ANDROID_TARGET_OUT_TESTCASES", 383 "ANDROID_TOOLCHAIN", 384 "ANDROID_TOOLCHAIN_2ND_ARCH", 385 "ANDROID_DEV_SCRIPTS", 386 "ANDROID_EMULATOR_PREBUILTS", 387 "ANDROID_PRE_BUILD_PATHS", 388 389 // We read it here already, don't let others share in the fun 390 "GENERATE_SOONG_DEBUG", 391 392 // Use config.ninjaCommand instead. 393 "SOONG_NINJA", 394 "SOONG_USE_N2", 395 ) 396 397 if ret.UseGoma() || ret.ForceUseGoma() { 398 ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.") 399 ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.") 400 } 401 402 // Tell python not to spam the source tree with .pyc files. 403 ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1") 404 405 tmpDir := absPath(ctx, ret.TempDir()) 406 ret.environ.Set("TMPDIR", ret.sandboxPath(wd, tmpDir)) 407 408 // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes 409 symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(), 410 "llvm-binutils-stable/llvm-symbolizer") 411 ret.environ.Set("ASAN_SYMBOLIZER_PATH", ret.sandboxPath(wd, absPath(ctx, symbolizerPath))) 412 413 // Precondition: the current directory is the top of the source tree 414 checkTopDir(ctx) 415 416 srcDir := absPath(ctx, ".") 417 if strings.ContainsRune(srcDir, ' ') { 418 ctx.Println("You are building in a directory whose absolute path contains a space character:") 419 ctx.Println() 420 ctx.Printf("%q\n", srcDir) 421 ctx.Println() 422 ctx.Fatalln("Directory names containing spaces are not supported") 423 } 424 425 ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ) 426 427 if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') { 428 ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:") 429 ctx.Println() 430 ctx.Printf("%q\n", outDir) 431 ctx.Println() 432 ctx.Fatalln("Directory names containing spaces are not supported") 433 } 434 435 if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') { 436 ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:") 437 ctx.Println() 438 ctx.Printf("%q\n", distDir) 439 ctx.Println() 440 ctx.Fatalln("Directory names containing spaces are not supported") 441 } 442 443 // Configure Java-related variables, including adding it to $PATH 444 java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag()) 445 java21Home := filepath.Join("prebuilts/jdk/jdk21", ret.HostPrebuiltTag()) 446 javaHome := func() string { 447 if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok { 448 return override 449 } 450 if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" { 451 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.") 452 } 453 if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" { 454 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.") 455 } 456 if toolchain21, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK21_TOOLCHAIN"); ok && toolchain21 != "true" { 457 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK21_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.") 458 } 459 return java21Home 460 }() 461 absJavaHome := absPath(ctx, javaHome) 462 463 ret.configureLocale(ctx) 464 465 newPath := []string{filepath.Join(absJavaHome, "bin")} 466 if path, ok := ret.environ.Get("PATH"); ok && path != "" { 467 newPath = append(newPath, path) 468 } 469 470 ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME") 471 ret.environ.Set("JAVA_HOME", ret.sandboxPath(wd, absJavaHome)) 472 ret.environ.Set("ANDROID_JAVA_HOME", ret.sandboxPath(wd, javaHome)) 473 ret.environ.Set("ANDROID_JAVA8_HOME", ret.sandboxPath(wd, java8Home)) 474 ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator))) 475 476 // b/286885495, https://bugzilla.redhat.com/show_bug.cgi?id=2227130: some versions of Fedora include patches 477 // to unzip to enable zipbomb detection that incorrectly handle zip64 and data descriptors and fail on large 478 // zip files produced by soong_zip. Disable zipbomb detection. 479 ret.environ.Set("UNZIP_DISABLE_ZIPBOMB_DETECTION", "TRUE") 480 481 outDir := ret.OutDir() 482 buildDateTimeFile := filepath.Join(outDir, "build_date.txt") 483 if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" { 484 ret.buildDateTime = buildDateTime 485 } else { 486 ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10) 487 } 488 489 ret.environ.Set("BUILD_DATETIME_FILE", ret.sandboxPath(wd, buildDateTimeFile)) 490 491 if _, ok := ret.environ.Get("BUILD_USERNAME"); !ok { 492 username := "unknown" 493 if u, err := user.Current(); err == nil { 494 username = u.Username 495 } else { 496 ctx.Println("Failed to get current user:", err) 497 } 498 ret.environ.Set("BUILD_USERNAME", username) 499 } 500 ret.environ.Set("PWD", ret.sandboxPath(wd, wd)) 501 502 if ret.UseRBE() { 503 for k, v := range getRBEVars(ctx, Config{ret}) { 504 ret.environ.Set(k, v) 505 } 506 } 507 508 c := Config{ret} 509 storeConfigMetrics(ctx, c) 510 return c 511} 512 513// NewBuildActionConfig returns a build configuration based on the build action. The arguments are 514// processed based on the build action and extracts any arguments that belongs to the build action. 515func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config { 516 return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...) 517} 518 519// Prepare for getting make variables. For them to be accurate, we need to have 520// obtained PRODUCT_RELEASE_CONFIG_MAPS. 521// 522// Returns: 523// 524// Whether config should be called again. 525// 526// TODO: when converting product config to a declarative language, make sure 527// that PRODUCT_RELEASE_CONFIG_MAPS is properly handled as a separate step in 528// that process. 529func SetProductReleaseConfigMaps(ctx Context, config Config) bool { 530 ctx.BeginTrace(metrics.RunKati, "SetProductReleaseConfigMaps") 531 defer ctx.EndTrace() 532 533 if config.SkipConfig() { 534 // This duplicates the logic from Build to skip product config 535 // if the user has explicitly said to. 536 return false 537 } 538 539 releaseConfigVars := []string{ 540 "PRODUCT_RELEASE_CONFIG_MAPS", 541 } 542 543 origValue, _ := config.environ.Get("PRODUCT_RELEASE_CONFIG_MAPS") 544 // Get the PRODUCT_RELEASE_CONFIG_MAPS for this product, to avoid polluting the environment 545 // when we run product config to get the rest of the make vars. 546 releaseMapVars, err := dumpMakeVars(ctx, config, nil, releaseConfigVars, false, "") 547 if err != nil { 548 ctx.Fatalln("Error getting PRODUCT_RELEASE_CONFIG_MAPS:", err) 549 } 550 productReleaseConfigMaps := releaseMapVars["PRODUCT_RELEASE_CONFIG_MAPS"] 551 os.Setenv("PRODUCT_RELEASE_CONFIG_MAPS", productReleaseConfigMaps) 552 return origValue != productReleaseConfigMaps 553} 554 555// storeConfigMetrics selects a set of configuration information and store in 556// the metrics system for further analysis. 557func storeConfigMetrics(ctx Context, config Config) { 558 if ctx.Metrics == nil { 559 return 560 } 561 562 ctx.Metrics.BuildConfig(buildConfig(config)) 563 564 cpuInfo := &smpb.SystemCpuInfo{ 565 VendorId: proto.String(config.systemCpuInfo.VendorId), 566 ModelName: proto.String(config.systemCpuInfo.ModelName), 567 CpuCores: proto.Int32(config.systemCpuInfo.CpuCores), 568 Flags: proto.String(config.systemCpuInfo.Flags), 569 } 570 memInfo := &smpb.SystemMemInfo{ 571 MemTotal: proto.Uint64(config.systemMemInfo.MemTotal), 572 MemFree: proto.Uint64(config.systemMemInfo.MemFree), 573 MemAvailable: proto.Uint64(config.systemMemInfo.MemAvailable), 574 } 575 576 s := &smpb.SystemResourceInfo{ 577 TotalPhysicalMemory: proto.Uint64(config.TotalRAM()), 578 AvailableCpus: proto.Int32(int32(runtime.NumCPU())), 579 CpuInfo: cpuInfo, 580 MemInfo: memInfo, 581 } 582 ctx.Metrics.SystemResourceInfo(s) 583} 584 585func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig_NinjaWeightListSource { 586 switch s { 587 case NINJA_LOG: 588 return smpb.BuildConfig_NINJA_LOG.Enum() 589 case EVENLY_DISTRIBUTED: 590 return smpb.BuildConfig_EVENLY_DISTRIBUTED.Enum() 591 case EXTERNAL_FILE: 592 return smpb.BuildConfig_EXTERNAL_FILE.Enum() 593 case HINT_FROM_SOONG: 594 return smpb.BuildConfig_HINT_FROM_SOONG.Enum() 595 default: 596 return smpb.BuildConfig_NOT_USED.Enum() 597 } 598} 599 600func buildConfig(config Config) *smpb.BuildConfig { 601 c := &smpb.BuildConfig{ 602 ForceUseGoma: proto.Bool(config.ForceUseGoma()), 603 UseGoma: proto.Bool(config.UseGoma()), 604 UseRbe: proto.Bool(config.UseRBE()), 605 NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()), 606 } 607 c.Targets = append(c.Targets, config.arguments...) 608 609 return c 610} 611 612// getConfigArgs processes the command arguments based on the build action and creates a set of new 613// arguments to be accepted by Config. 614func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string { 615 // The next block of code verifies that the current directory is the root directory of the source 616 // tree. It then finds the relative path of dir based on the root directory of the source tree 617 // and verify that dir is inside of the source tree. 618 checkTopDir(ctx) 619 topDir, err := os.Getwd() 620 if err != nil { 621 ctx.Fatalf("Error retrieving top directory: %v", err) 622 } 623 dir, err = filepath.EvalSymlinks(dir) 624 if err != nil { 625 ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err) 626 } 627 dir, err = filepath.Abs(dir) 628 if err != nil { 629 ctx.Fatalf("Unable to find absolute path %s: %v", dir, err) 630 } 631 relDir, err := filepath.Rel(topDir, dir) 632 if err != nil { 633 ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err) 634 } 635 // If there are ".." in the path, it's not in the source tree. 636 if strings.Contains(relDir, "..") { 637 ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir) 638 } 639 640 configArgs := args[:] 641 642 // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to 643 // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules. 644 targetNamePrefix := "MODULES-IN-" 645 if inList("GET-INSTALL-PATH", configArgs) { 646 targetNamePrefix = "GET-INSTALL-PATH-IN-" 647 configArgs = removeFromList("GET-INSTALL-PATH", configArgs) 648 } 649 650 var targets []string 651 652 switch action { 653 case BUILD_MODULES: 654 // No additional processing is required when building a list of specific modules or all modules. 655 case BUILD_MODULES_IN_A_DIRECTORY: 656 // If dir is the root source tree, all the modules are built of the source tree are built so 657 // no need to find the build file. 658 if topDir == dir { 659 break 660 } 661 662 buildFile := findBuildFile(ctx, relDir) 663 if buildFile == "" { 664 ctx.Fatalf("Build file not found for %s directory", relDir) 665 } 666 targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} 667 case BUILD_MODULES_IN_DIRECTORIES: 668 newConfigArgs, dirs := splitArgs(configArgs) 669 configArgs = newConfigArgs 670 targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix) 671 } 672 673 // Tidy only override all other specified targets. 674 tidyOnly := os.Getenv("WITH_TIDY_ONLY") 675 if tidyOnly == "true" || tidyOnly == "1" { 676 configArgs = append(configArgs, "tidy_only") 677 } else { 678 configArgs = append(configArgs, targets...) 679 } 680 681 return configArgs 682} 683 684// convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name. 685func convertToTarget(dir string, targetNamePrefix string) string { 686 return targetNamePrefix + strings.ReplaceAll(dir, "/", "-") 687} 688 689// hasBuildFile returns true if dir contains an Android build file. 690func hasBuildFile(ctx Context, dir string) bool { 691 for _, buildFile := range buildFiles { 692 _, err := os.Stat(filepath.Join(dir, buildFile)) 693 if err == nil { 694 return true 695 } 696 if !os.IsNotExist(err) { 697 ctx.Fatalf("Error retrieving the build file stats: %v", err) 698 } 699 } 700 return false 701} 702 703// findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file 704// in the current and any sub directory of dir. If a build file is not found, traverse the path 705// up by one directory and repeat again until either a build file is found or reached to the root 706// source tree. The returned filename of build file is "Android.mk". If one was not found, a blank 707// string is returned. 708func findBuildFile(ctx Context, dir string) string { 709 // If the string is empty or ".", assume it is top directory of the source tree. 710 if dir == "" || dir == "." { 711 return "" 712 } 713 714 found := false 715 for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) { 716 err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error { 717 if err != nil { 718 return err 719 } 720 if found { 721 return filepath.SkipDir 722 } 723 if info.IsDir() { 724 return nil 725 } 726 for _, buildFile := range buildFiles { 727 if info.Name() == buildFile { 728 found = true 729 return filepath.SkipDir 730 } 731 } 732 return nil 733 }) 734 if err != nil { 735 ctx.Fatalf("Error finding Android build file: %v", err) 736 } 737 738 if found { 739 return filepath.Join(buildDir, "Android.mk") 740 } 741 } 742 743 return "" 744} 745 746// splitArgs iterates over the arguments list and splits into two lists: arguments and directories. 747func splitArgs(args []string) (newArgs []string, dirs []string) { 748 specialArgs := map[string]bool{ 749 "showcommands": true, 750 "snod": true, 751 "dist": true, 752 "checkbuild": true, 753 } 754 755 newArgs = []string{} 756 dirs = []string{} 757 758 for _, arg := range args { 759 // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory. 760 if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 { 761 newArgs = append(newArgs, arg) 762 continue 763 } 764 765 if _, ok := specialArgs[arg]; ok { 766 newArgs = append(newArgs, arg) 767 continue 768 } 769 770 dirs = append(dirs, arg) 771 } 772 773 return newArgs, dirs 774} 775 776// getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a 777// directory from the dirs list does not exist, a fatal error is raised. relDir is related to the 778// source root tree where the build action command was invoked. Each directory is validated if the 779// build file can be found and follows the format "dir1:target1,target2,...". Target is optional. 780func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) { 781 for _, dir := range dirs { 782 // The directory may have specified specific modules to build. ":" is the separator to separate 783 // the directory and the list of modules. 784 s := strings.Split(dir, ":") 785 l := len(s) 786 if l > 2 { // more than one ":" was specified. 787 ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir) 788 } 789 790 dir = filepath.Join(relDir, s[0]) 791 if _, err := os.Stat(dir); err != nil { 792 ctx.Fatalf("couldn't find directory %s", dir) 793 } 794 795 // Verify that if there are any targets specified after ":". Each target is separated by ",". 796 var newTargets []string 797 if l == 2 && s[1] != "" { 798 newTargets = strings.Split(s[1], ",") 799 if inList("", newTargets) { 800 ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir) 801 } 802 } 803 804 // If there are specified targets to build in dir, an android build file must exist for the one 805 // shot build. For the non-targets case, find the appropriate build file and build all the 806 // modules in dir (or the closest one in the dir path). 807 if len(newTargets) > 0 { 808 if !hasBuildFile(ctx, dir) { 809 ctx.Fatalf("Couldn't locate a build file from %s directory", dir) 810 } 811 } else { 812 buildFile := findBuildFile(ctx, dir) 813 if buildFile == "" { 814 ctx.Fatalf("Build file not found for %s directory", dir) 815 } 816 newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} 817 } 818 819 targets = append(targets, newTargets...) 820 } 821 822 return targets 823} 824 825func (c *configImpl) parseArgs(ctx Context, args []string) { 826 for i := 0; i < len(args); i++ { 827 arg := strings.TrimSpace(args[i]) 828 if arg == "showcommands" { 829 c.verbose = true 830 } else if arg == "--empty-ninja-file" { 831 c.emptyNinjaFile = true 832 } else if arg == "--skip-ninja" { 833 c.skipNinja = true 834 } else if arg == "--skip-make" { 835 // TODO(ccross): deprecate this, it has confusing behaviors. It doesn't run kati, 836 // but it does run a Kati ninja file if the .kati_enabled marker file was created 837 // by a previous build. 838 c.skipConfig = true 839 c.skipKati = true 840 } else if arg == "--soong-only" { 841 c.skipKati = true 842 c.skipKatiNinja = true 843 } else if arg == "--config-only" { 844 c.skipKati = true 845 c.skipKatiNinja = true 846 c.skipSoong = true 847 } else if arg == "--skip-config" { 848 c.skipConfig = true 849 } else if arg == "--skip-soong-tests" { 850 c.skipSoongTests = true 851 } else if arg == "--no-skip-soong-tests" { 852 c.skipSoongTests = false 853 } else if arg == "--skip-metrics-upload" { 854 c.skipMetricsUpload = true 855 } else if arg == "--mk-metrics" { 856 c.reportMkMetrics = true 857 } else if arg == "--search-api-dir" { 858 c.searchApiDir = true 859 } else if strings.HasPrefix(arg, "--ninja_weight_source=") { 860 source := strings.TrimPrefix(arg, "--ninja_weight_source=") 861 if source == "ninja_log" { 862 c.ninjaWeightListSource = NINJA_LOG 863 } else if source == "evenly_distributed" { 864 c.ninjaWeightListSource = EVENLY_DISTRIBUTED 865 } else if source == "not_used" { 866 c.ninjaWeightListSource = NOT_USED 867 } else if source == "soong" { 868 c.ninjaWeightListSource = HINT_FROM_SOONG 869 } else if strings.HasPrefix(source, "file,") { 870 c.ninjaWeightListSource = EXTERNAL_FILE 871 filePath := strings.TrimPrefix(source, "file,") 872 err := validateNinjaWeightList(filePath) 873 if err != nil { 874 ctx.Fatalf("Malformed weight list from %s: %s", filePath, err) 875 } 876 _, err = copyFile(filePath, filepath.Join(c.OutDir(), ".ninja_weight_list")) 877 if err != nil { 878 ctx.Fatalf("Error to copy ninja weight list from %s: %s", filePath, err) 879 } 880 } else { 881 ctx.Fatalf("unknown option for ninja_weight_source: %s", source) 882 } 883 } else if arg == "--build-from-source-stub" { 884 c.buildFromSourceStub = true 885 } else if arg == "--incremental-build-actions" { 886 c.incrementalBuildActions = true 887 } else if strings.HasPrefix(arg, "--build-command=") { 888 buildCmd := strings.TrimPrefix(arg, "--build-command=") 889 // remove quotations 890 buildCmd = strings.TrimPrefix(buildCmd, "\"") 891 buildCmd = strings.TrimSuffix(buildCmd, "\"") 892 ctx.Metrics.SetBuildCommand([]string{buildCmd}) 893 } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") { 894 buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=") 895 val, err := strconv.ParseInt(buildTimeStr, 10, 64) 896 if err == nil { 897 c.buildStartedTime = val 898 } else { 899 ctx.Fatalf("Error parsing build-time-started-unix-millis", err) 900 } 901 } else if arg == "--ensure-allowlist-integrity" { 902 c.ensureAllowlistIntegrity = true 903 } else if len(arg) > 0 && arg[0] == '-' { 904 parseArgNum := func(def int) int { 905 if len(arg) > 2 { 906 p, err := strconv.ParseUint(arg[2:], 10, 31) 907 if err != nil { 908 ctx.Fatalf("Failed to parse %q: %v", arg, err) 909 } 910 return int(p) 911 } else if i+1 < len(args) { 912 p, err := strconv.ParseUint(args[i+1], 10, 31) 913 if err == nil { 914 i++ 915 return int(p) 916 } 917 } 918 return def 919 } 920 921 if len(arg) > 1 && arg[1] == 'j' { 922 c.parallel = parseArgNum(c.parallel) 923 } else if len(arg) > 1 && arg[1] == 'k' { 924 c.keepGoing = parseArgNum(0) 925 } else { 926 ctx.Fatalln("Unknown option:", arg) 927 } 928 } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 { 929 if k == "OUT_DIR" { 930 ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.") 931 } 932 c.environ.Set(k, v) 933 } else if arg == "dist" { 934 c.dist = true 935 } else if arg == "json-module-graph" { 936 c.jsonModuleGraph = true 937 } else if arg == "soong_docs" { 938 c.soongDocs = true 939 } else { 940 if arg == "checkbuild" { 941 c.checkbuild = true 942 } 943 c.arguments = append(c.arguments, arg) 944 } 945 } 946} 947 948func validateNinjaWeightList(weightListFilePath string) (err error) { 949 data, err := os.ReadFile(weightListFilePath) 950 if err != nil { 951 return 952 } 953 lines := strings.Split(strings.TrimSpace(string(data)), "\n") 954 for _, line := range lines { 955 fields := strings.Split(line, ",") 956 if len(fields) != 2 { 957 return fmt.Errorf("wrong format, each line should have two fields, but '%s'", line) 958 } 959 _, err = strconv.Atoi(fields[1]) 960 if err != nil { 961 return 962 } 963 } 964 return 965} 966 967func (c *configImpl) configureLocale(ctx Context) { 968 cmd := Command(ctx, Config{c}, "locale", "locale", "-a") 969 output, err := cmd.Output() 970 971 var locales []string 972 if err == nil { 973 locales = strings.Split(string(output), "\n") 974 } else { 975 // If we're unable to list the locales, let's assume en_US.UTF-8 976 locales = []string{"en_US.UTF-8"} 977 ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales) 978 } 979 980 // gettext uses LANGUAGE, which is passed directly through 981 982 // For LANG and LC_*, only preserve the evaluated version of 983 // LC_MESSAGES 984 userLang := "" 985 if lc_all, ok := c.environ.Get("LC_ALL"); ok { 986 userLang = lc_all 987 } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok { 988 userLang = lc_messages 989 } else if lang, ok := c.environ.Get("LANG"); ok { 990 userLang = lang 991 } 992 993 c.environ.UnsetWithPrefix("LC_") 994 995 if userLang != "" { 996 c.environ.Set("LC_MESSAGES", userLang) 997 } 998 999 // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed 1000 // for others) 1001 if inList("C.UTF-8", locales) { 1002 c.environ.Set("LANG", "C.UTF-8") 1003 } else if inList("C.utf8", locales) { 1004 // These normalize to the same thing 1005 c.environ.Set("LANG", "C.UTF-8") 1006 } else if inList("en_US.UTF-8", locales) { 1007 c.environ.Set("LANG", "en_US.UTF-8") 1008 } else if inList("en_US.utf8", locales) { 1009 // These normalize to the same thing 1010 c.environ.Set("LANG", "en_US.UTF-8") 1011 } else { 1012 ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8") 1013 } 1014} 1015 1016func (c *configImpl) Environment() *Environment { 1017 return c.environ 1018} 1019 1020func (c *configImpl) Arguments() []string { 1021 return c.arguments 1022} 1023 1024func (c *configImpl) SoongBuildInvocationNeeded() bool { 1025 if len(c.Arguments()) > 0 { 1026 // Explicit targets requested that are not special targets like b2pbuild 1027 // or the JSON module graph 1028 return true 1029 } 1030 1031 if !c.JsonModuleGraph() && !c.SoongDocs() { 1032 // Command line was empty, the default Ninja target is built 1033 return true 1034 } 1035 1036 if c.Dist() { 1037 return true 1038 } 1039 1040 // build.ninja doesn't need to be generated 1041 return false 1042} 1043 1044func (c *configImpl) OutDir() string { 1045 if outDir, ok := c.environ.Get("OUT_DIR"); ok { 1046 return outDir 1047 } 1048 return "out" 1049} 1050 1051func (c *configImpl) DistDir() string { 1052 return c.distDir 1053} 1054 1055func (c *configImpl) RealDistDir() string { 1056 return c.distDir 1057} 1058 1059func (c *configImpl) NinjaArgs() []string { 1060 if c.skipKati { 1061 return c.arguments 1062 } 1063 return c.ninjaArgs 1064} 1065 1066func (c *configImpl) SoongOutDir() string { 1067 return filepath.Join(c.OutDir(), "soong") 1068} 1069 1070func (c *configImpl) ApiSurfacesOutDir() string { 1071 return filepath.Join(c.OutDir(), "api_surfaces") 1072} 1073 1074func (c *configImpl) PrebuiltOS() string { 1075 switch runtime.GOOS { 1076 case "linux": 1077 return "linux-x86" 1078 case "darwin": 1079 return "darwin-x86" 1080 default: 1081 panic("Unknown GOOS") 1082 } 1083} 1084 1085func (c *configImpl) HostToolDir() string { 1086 if c.SkipKatiNinja() { 1087 return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin") 1088 } else { 1089 return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin") 1090 } 1091} 1092 1093func (c *configImpl) UsedEnvFile(tag string) string { 1094 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { 1095 return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+v+c.CoverageSuffix()+"."+tag) 1096 } 1097 return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag) 1098} 1099 1100func (c *configImpl) SoongDocsHtml() string { 1101 return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html") 1102} 1103 1104func (c *configImpl) ModuleGraphFile() string { 1105 return shared.JoinPath(c.SoongOutDir(), "module-graph.json") 1106} 1107 1108func (c *configImpl) ModuleActionsFile() string { 1109 return shared.JoinPath(c.SoongOutDir(), "module-actions.json") 1110} 1111 1112func (c *configImpl) TempDir() string { 1113 return shared.TempDirForOutDir(c.SoongOutDir()) 1114} 1115 1116func (c *configImpl) FileListDir() string { 1117 return filepath.Join(c.OutDir(), ".module_paths") 1118} 1119 1120func (c *configImpl) KatiSuffix() string { 1121 if c.katiSuffix != "" { 1122 return c.katiSuffix 1123 } 1124 panic("SetKatiSuffix has not been called") 1125} 1126 1127// Checkbuild returns true if "checkbuild" was one of the build goals, which means that the 1128// user is interested in additional checks at the expense of build time. 1129func (c *configImpl) Checkbuild() bool { 1130 return c.checkbuild 1131} 1132 1133func (c *configImpl) Dist() bool { 1134 return c.dist 1135} 1136 1137func (c *configImpl) JsonModuleGraph() bool { 1138 return c.jsonModuleGraph 1139} 1140 1141func (c *configImpl) SoongDocs() bool { 1142 return c.soongDocs 1143} 1144 1145func (c *configImpl) IsVerbose() bool { 1146 return c.verbose 1147} 1148 1149func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource { 1150 return c.ninjaWeightListSource 1151} 1152 1153func (c *configImpl) SkipKati() bool { 1154 return c.skipKati 1155} 1156 1157func (c *configImpl) SkipKatiNinja() bool { 1158 return c.skipKatiNinja 1159} 1160 1161func (c *configImpl) SkipSoong() bool { 1162 return c.skipSoong 1163} 1164 1165func (c *configImpl) SkipNinja() bool { 1166 return c.skipNinja 1167} 1168 1169func (c *configImpl) SetSkipNinja(v bool) { 1170 c.skipNinja = v 1171} 1172 1173func (c *configImpl) SkipConfig() bool { 1174 return c.skipConfig 1175} 1176 1177func (c *configImpl) BuildFromTextStub() bool { 1178 return !c.buildFromSourceStub 1179} 1180 1181func (c *configImpl) TargetProduct() string { 1182 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { 1183 return v 1184 } 1185 panic("TARGET_PRODUCT is not defined") 1186} 1187 1188func (c *configImpl) TargetProductOrErr() (string, error) { 1189 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { 1190 return v, nil 1191 } 1192 return "", fmt.Errorf("TARGET_PRODUCT is not defined") 1193} 1194 1195func (c *configImpl) CoverageSuffix() string { 1196 if v := c.environ.IsEnvTrue("EMMA_INSTRUMENT"); v { 1197 return ".coverage" 1198 } 1199 return "" 1200} 1201 1202func (c *configImpl) TargetDevice() string { 1203 return c.targetDevice 1204} 1205 1206func (c *configImpl) SetTargetDevice(device string) { 1207 c.targetDevice = device 1208} 1209 1210func (c *configImpl) TargetBuildVariant() string { 1211 if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok { 1212 return v 1213 } 1214 panic("TARGET_BUILD_VARIANT is not defined") 1215} 1216 1217func (c *configImpl) KatiArgs() []string { 1218 return c.katiArgs 1219} 1220 1221func (c *configImpl) Parallel() int { 1222 return c.parallel 1223} 1224 1225func (c *configImpl) GetSourceRootDirs() []string { 1226 return c.sourceRootDirs 1227} 1228 1229func (c *configImpl) SetSourceRootDirs(i []string) { 1230 c.sourceRootDirs = i 1231} 1232 1233func (c *configImpl) GetLogsPrefix() string { 1234 return c.logsPrefix 1235} 1236 1237func (c *configImpl) SetLogsPrefix(prefix string) { 1238 c.logsPrefix = prefix 1239} 1240 1241func (c *configImpl) HighmemParallel() int { 1242 if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok { 1243 return i 1244 } 1245 1246 const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024 1247 parallel := c.Parallel() 1248 if c.UseRemoteBuild() { 1249 // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism 1250 // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs 1251 // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention. 1252 // Return 1/16th of the size of the local pool, rounding up. 1253 return (parallel + 15) / 16 1254 } else if c.totalRAM == 0 { 1255 // Couldn't detect the total RAM, don't restrict highmem processes. 1256 return parallel 1257 } else if c.totalRAM <= 16*1024*1024*1024 { 1258 // Less than 16GB of ram, restrict to 1 highmem processes 1259 return 1 1260 } else if c.totalRAM <= 32*1024*1024*1024 { 1261 // Less than 32GB of ram, restrict to 2 highmem processes 1262 return 2 1263 } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel { 1264 // If less than 8GB total RAM per process, reduce the number of highmem processes 1265 return p 1266 } 1267 // No restriction on highmem processes 1268 return parallel 1269} 1270 1271func (c *configImpl) TotalRAM() uint64 { 1272 return c.totalRAM 1273} 1274 1275// ForceUseGoma determines whether we should override Goma deprecation 1276// and use Goma for the current build or not. 1277func (c *configImpl) ForceUseGoma() bool { 1278 if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok { 1279 v = strings.TrimSpace(v) 1280 if v != "" && v != "false" { 1281 return true 1282 } 1283 } 1284 return false 1285} 1286 1287func (c *configImpl) UseGoma() bool { 1288 if v, ok := c.environ.Get("USE_GOMA"); ok { 1289 v = strings.TrimSpace(v) 1290 if v != "" && v != "false" { 1291 return true 1292 } 1293 } 1294 return false 1295} 1296 1297func (c *configImpl) StartGoma() bool { 1298 if !c.UseGoma() { 1299 return false 1300 } 1301 1302 if v, ok := c.environ.Get("NOSTART_GOMA"); ok { 1303 v = strings.TrimSpace(v) 1304 if v != "" && v != "false" { 1305 return false 1306 } 1307 } 1308 return true 1309} 1310 1311func (c *configImpl) canSupportRBE() bool { 1312 // Only supported on linux 1313 if runtime.GOOS != "linux" { 1314 return false 1315 } 1316 1317 // Do not use RBE with prod credentials in scenarios when stubby doesn't exist, since 1318 // its unlikely that we will be able to obtain necessary creds without stubby. 1319 authType, _ := c.rbeAuth() 1320 if !c.StubbyExists() && strings.Contains(authType, "use_google_prod_creds") { 1321 return false 1322 } 1323 if c.UseABFS() { 1324 return false 1325 } 1326 return true 1327} 1328 1329func (c *configImpl) UseABFS() bool { 1330 if v, ok := c.environ.Get("NO_ABFS"); ok { 1331 v = strings.ToLower(strings.TrimSpace(v)) 1332 if v == "true" || v == "1" { 1333 return false 1334 } 1335 } 1336 1337 abfsBox := c.PrebuiltBuildTool("abfsbox") 1338 err := exec.Command(abfsBox, "hash", srcDirFileCheck).Run() 1339 return err == nil 1340} 1341 1342func (c *configImpl) sandboxPath(base, in string) string { 1343 if !c.UseABFS() { 1344 return in 1345 } 1346 1347 rel, err := filepath.Rel(base, in) 1348 if err != nil { 1349 return in 1350 } 1351 1352 return filepath.Join(abfsSrcDir, rel) 1353} 1354 1355func (c *configImpl) UseRBE() bool { 1356 // These alternate modes of running Soong do not use RBE / reclient. 1357 if c.JsonModuleGraph() { 1358 return false 1359 } 1360 1361 if !c.canSupportRBE() { 1362 return false 1363 } 1364 1365 if v, ok := c.Environment().Get("USE_RBE"); ok { 1366 v = strings.TrimSpace(v) 1367 if v != "" && v != "false" { 1368 return true 1369 } 1370 } 1371 return false 1372} 1373 1374func (c *configImpl) StartRBE() bool { 1375 if !c.UseRBE() { 1376 return false 1377 } 1378 1379 if v, ok := c.environ.Get("NOSTART_RBE"); ok { 1380 v = strings.TrimSpace(v) 1381 if v != "" && v != "false" { 1382 return false 1383 } 1384 } 1385 return true 1386} 1387 1388func (c *configImpl) rbeProxyLogsDir() string { 1389 for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} { 1390 if v, ok := c.environ.Get(f); ok { 1391 return v 1392 } 1393 } 1394 return c.rbeTmpDir() 1395} 1396 1397func (c *configImpl) rbeDownloadTmpDir() string { 1398 for _, f := range []string{"RBE_download_tmp_dir", "FLAG_download_tmp_dir"} { 1399 if v, ok := c.environ.Get(f); ok { 1400 return v 1401 } 1402 } 1403 return c.rbeTmpDir() 1404} 1405 1406func (c *configImpl) rbeTmpDir() string { 1407 return filepath.Join(c.SoongOutDir(), "rbe") 1408} 1409 1410func (c *configImpl) rbeCacheDir() string { 1411 for _, f := range []string{"RBE_cache_dir", "FLAG_cache_dir"} { 1412 if v, ok := c.environ.Get(f); ok { 1413 return v 1414 } 1415 } 1416 return shared.JoinPath(c.SoongOutDir(), "rbe") 1417} 1418 1419func (c *configImpl) shouldCleanupRBELogsDir() bool { 1420 // Perform a log directory cleanup only when the log directory 1421 // is auto created by the build rather than user-specified. 1422 for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} { 1423 if v, ok := c.environ.Get(f); ok { 1424 if v != c.rbeTmpDir() { 1425 return false 1426 } 1427 } 1428 } 1429 return true 1430} 1431 1432func (c *configImpl) rbeExecRoot() string { 1433 for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} { 1434 if v, ok := c.environ.Get(f); ok { 1435 return v 1436 } 1437 } 1438 wd, err := os.Getwd() 1439 if err != nil { 1440 return "" 1441 } 1442 return wd 1443} 1444 1445func (c *configImpl) rbeDir() string { 1446 if v, ok := c.environ.Get("RBE_DIR"); ok { 1447 return v 1448 } 1449 return "prebuilts/remoteexecution-client/live/" 1450} 1451 1452func (c *configImpl) rbeReproxy() string { 1453 for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} { 1454 if v, ok := c.environ.Get(f); ok { 1455 return v 1456 } 1457 } 1458 return filepath.Join(c.rbeDir(), "reproxy") 1459} 1460 1461func (c *configImpl) rbeAuth() (string, string) { 1462 credFlags := []string{ 1463 "use_application_default_credentials", 1464 "use_gce_credentials", 1465 "credential_file", 1466 "use_google_prod_creds", 1467 } 1468 for _, cf := range credFlags { 1469 for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} { 1470 if v, ok := c.environ.Get(f); ok { 1471 v = strings.TrimSpace(v) 1472 if v != "" && v != "false" && v != "0" { 1473 return "RBE_" + cf, v 1474 } 1475 } 1476 } 1477 } 1478 return "RBE_use_application_default_credentials", "true" 1479} 1480 1481func (c *configImpl) rbeSockAddr(dir string) (string, error) { 1482 // Absolute path socket addresses have a prefix of //. This should 1483 // be included in the length limit. 1484 maxNameLen := len(syscall.RawSockaddrUnix{}.Path) - 2 1485 base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix) 1486 1487 name := filepath.Join(dir, base) 1488 if len(name) < maxNameLen { 1489 return name, nil 1490 } 1491 1492 name = filepath.Join("/tmp", base) 1493 if len(name) < maxNameLen { 1494 return name, nil 1495 } 1496 1497 return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen) 1498} 1499 1500// IsGooglerEnvironment returns true if the current build is running 1501// on a Google developer machine and false otherwise. 1502func (c *configImpl) IsGooglerEnvironment() bool { 1503 cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG" 1504 if v, ok := c.environ.Get(cf); ok { 1505 return v == "googler" 1506 } 1507 return false 1508} 1509 1510// GoogleProdCredsExist determine whether credentials exist on the 1511// Googler machine to use remote execution. 1512func (c *configImpl) GoogleProdCredsExist() bool { 1513 if googleProdCredsExistCache { 1514 return googleProdCredsExistCache 1515 } 1516 if _, err := exec.Command("/usr/bin/gcertstatus", "-nocheck_ssh").Output(); err != nil { 1517 return false 1518 } 1519 googleProdCredsExistCache = true 1520 return true 1521} 1522 1523// UseRemoteBuild indicates whether to use a remote build acceleration system 1524// to speed up the build. 1525func (c *configImpl) UseRemoteBuild() bool { 1526 return c.UseGoma() || c.UseRBE() 1527} 1528 1529// StubbyExists checks whether the stubby binary exists on the machine running 1530// the build. 1531func (c *configImpl) StubbyExists() bool { 1532 if _, err := exec.LookPath("stubby"); err != nil { 1533 return false 1534 } 1535 return true 1536} 1537 1538// RemoteParallel controls how many remote jobs (i.e., commands which contain 1539// gomacc) are run in parallel. Note the parallelism of all other jobs is 1540// still limited by Parallel() 1541func (c *configImpl) RemoteParallel() int { 1542 if !c.UseRemoteBuild() { 1543 return 0 1544 } 1545 if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok { 1546 return i 1547 } 1548 return 500 1549} 1550 1551func (c *configImpl) SetKatiArgs(args []string) { 1552 c.katiArgs = args 1553} 1554 1555func (c *configImpl) SetNinjaArgs(args []string) { 1556 c.ninjaArgs = args 1557} 1558 1559func (c *configImpl) SetKatiSuffix(suffix string) { 1560 c.katiSuffix = suffix 1561} 1562 1563func (c *configImpl) LastKatiSuffixFile() string { 1564 return filepath.Join(c.OutDir(), "last_kati_suffix") 1565} 1566 1567func (c *configImpl) HasKatiSuffix() bool { 1568 return c.katiSuffix != "" 1569} 1570 1571func (c *configImpl) KatiEnvFile() string { 1572 return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh") 1573} 1574 1575func (c *configImpl) KatiBuildNinjaFile() string { 1576 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja") 1577} 1578 1579func (c *configImpl) KatiPackageNinjaFile() string { 1580 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja") 1581} 1582 1583func (c *configImpl) SoongVarsFile() string { 1584 targetProduct, err := c.TargetProductOrErr() 1585 if err != nil { 1586 return filepath.Join(c.SoongOutDir(), "soong.variables") 1587 } else { 1588 return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+c.CoverageSuffix()+".variables") 1589 } 1590} 1591 1592func (c *configImpl) SoongExtraVarsFile() string { 1593 targetProduct, err := c.TargetProductOrErr() 1594 if err != nil { 1595 return filepath.Join(c.SoongOutDir(), "soong.extra.variables") 1596 } else { 1597 return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+c.CoverageSuffix()+".extra.variables") 1598 } 1599} 1600 1601func (c *configImpl) SoongNinjaFile() string { 1602 targetProduct, err := c.TargetProductOrErr() 1603 if err != nil { 1604 return filepath.Join(c.SoongOutDir(), "build.ninja") 1605 } else { 1606 return filepath.Join(c.SoongOutDir(), "build."+targetProduct+c.CoverageSuffix()+".ninja") 1607 } 1608} 1609 1610func (c *configImpl) CombinedNinjaFile() string { 1611 if c.katiSuffix == "" { 1612 return filepath.Join(c.OutDir(), "combined.ninja") 1613 } 1614 return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja") 1615} 1616 1617func (c *configImpl) SoongAndroidMk() string { 1618 return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+c.CoverageSuffix()+".mk") 1619} 1620 1621func (c *configImpl) SoongMakeVarsMk() string { 1622 return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+c.CoverageSuffix()+".mk") 1623} 1624 1625func (c *configImpl) SoongBuildMetrics() string { 1626 return filepath.Join(c.LogsDir(), "soong_build_metrics.pb") 1627} 1628 1629func (c *configImpl) ProductOut() string { 1630 return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice()) 1631} 1632 1633func (c *configImpl) DevicePreviousProductConfig() string { 1634 return filepath.Join(c.ProductOut(), "previous_build_config.mk") 1635} 1636 1637func (c *configImpl) KatiPackageMkDir() string { 1638 return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging") 1639} 1640 1641func (c *configImpl) hostOutRoot() string { 1642 return filepath.Join(c.OutDir(), "host") 1643} 1644 1645func (c *configImpl) HostOut() string { 1646 return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag()) 1647} 1648 1649// This probably needs to be multi-valued, so not exporting it for now 1650func (c *configImpl) hostCrossOut() string { 1651 if runtime.GOOS == "linux" { 1652 return filepath.Join(c.hostOutRoot(), "windows-x86") 1653 } else { 1654 return "" 1655 } 1656} 1657 1658func (c *configImpl) HostPrebuiltTag() string { 1659 if runtime.GOOS == "linux" { 1660 return "linux-x86" 1661 } else if runtime.GOOS == "darwin" { 1662 return "darwin-x86" 1663 } else { 1664 panic("Unsupported OS") 1665 } 1666} 1667 1668func (c *configImpl) KatiBin() string { 1669 binName := "ckati" 1670 if c.UseABFS() { 1671 binName = "ckati-wrap" 1672 } 1673 1674 return c.PrebuiltBuildTool(binName) 1675} 1676 1677func (c *configImpl) NinjaBin() string { 1678 binName := "ninja" 1679 if c.UseABFS() { 1680 binName = "ninjago" 1681 } 1682 return c.PrebuiltBuildTool(binName) 1683} 1684 1685func (c *configImpl) N2Bin() string { 1686 path := c.PrebuiltBuildTool("n2") 1687 // Use musl instead of glibc because glibc on the build server is old and has bugs 1688 return strings.ReplaceAll(path, "/linux-x86/", "/linux_musl-x86/") 1689} 1690 1691func (c *configImpl) SisoBin() string { 1692 path := c.PrebuiltBuildTool("siso") 1693 // Use musl instead of glibc because glibc on the build server is old and has bugs 1694 return strings.ReplaceAll(path, "/linux-x86/", "/linux_musl-x86/") 1695} 1696 1697func (c *configImpl) PrebuiltBuildTool(name string) string { 1698 if c.environ.IsEnvTrue("SANITIZE_BUILD_TOOL_PREBUILTS") { 1699 asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name) 1700 if _, err := os.Stat(asan); err == nil { 1701 return asan 1702 } 1703 } 1704 return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name) 1705} 1706 1707func (c *configImpl) SetBuildBrokenDupRules(val bool) { 1708 c.brokenDupRules = val 1709} 1710 1711func (c *configImpl) BuildBrokenDupRules() bool { 1712 return c.brokenDupRules 1713} 1714 1715func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) { 1716 c.brokenUsesNetwork = val 1717} 1718 1719func (c *configImpl) BuildBrokenUsesNetwork() bool { 1720 return c.brokenUsesNetwork 1721} 1722 1723func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) { 1724 c.brokenNinjaEnvVars = val 1725} 1726 1727func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string { 1728 return c.brokenNinjaEnvVars 1729} 1730 1731func (c *configImpl) SetBuildBrokenMissingOutputs(val bool) { 1732 c.brokenMissingOutputs = val 1733} 1734 1735func (c *configImpl) BuildBrokenMissingOutputs() bool { 1736 return c.brokenMissingOutputs 1737} 1738 1739func (c *configImpl) SetTargetDeviceDir(dir string) { 1740 c.targetDeviceDir = dir 1741} 1742 1743func (c *configImpl) TargetDeviceDir() string { 1744 return c.targetDeviceDir 1745} 1746 1747func (c *configImpl) BuildDateTime() string { 1748 return c.buildDateTime 1749} 1750 1751func (c *configImpl) MetricsUploaderApp() string { 1752 return c.metricsUploader 1753} 1754 1755// LogsDir returns the absolute path to the logs directory where build log and 1756// metrics files are located. By default, the logs directory is the out 1757// directory. If the argument dist is specified, the logs directory 1758// is <dist_dir>/logs. 1759func (c *configImpl) LogsDir() string { 1760 dir := c.OutDir() 1761 if c.Dist() { 1762 // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files 1763 dir = filepath.Join(c.RealDistDir(), "logs") 1764 } 1765 absDir, err := filepath.Abs(dir) 1766 if err != nil { 1767 fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error()) 1768 os.Exit(1) 1769 } 1770 return absDir 1771} 1772 1773// MkFileMetrics returns the file path for make-related metrics. 1774func (c *configImpl) MkMetrics() string { 1775 return filepath.Join(c.LogsDir(), "mk_metrics.pb") 1776} 1777 1778func (c *configImpl) SetEmptyNinjaFile(v bool) { 1779 c.emptyNinjaFile = v 1780} 1781 1782func (c *configImpl) EmptyNinjaFile() bool { 1783 return c.emptyNinjaFile 1784} 1785 1786func (c *configImpl) SkipMetricsUpload() bool { 1787 // b/362625275 - Metrics upload sometimes prevents abfs unmount 1788 if c.UseABFS() { 1789 return true 1790 } 1791 1792 return c.skipMetricsUpload 1793} 1794 1795func (c *configImpl) EnsureAllowlistIntegrity() bool { 1796 return c.ensureAllowlistIntegrity 1797} 1798 1799// Returns a Time object if one was passed via a command-line flag. 1800// Otherwise returns the passed default. 1801func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time { 1802 if c.buildStartedTime == 0 { 1803 return defaultTime 1804 } 1805 return time.UnixMilli(c.buildStartedTime) 1806} 1807 1808func GetMetricsUploader(topDir string, env *Environment) string { 1809 if p, ok := env.Get("METRICS_UPLOADER"); ok { 1810 metricsUploader := filepath.Join(topDir, p) 1811 if _, err := os.Stat(metricsUploader); err == nil { 1812 return metricsUploader 1813 } 1814 } 1815 1816 return "" 1817} 1818