1// Copyright 2017 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package build 16 17import ( 18 "android/soong/ui/metrics" 19 "android/soong/ui/status" 20 "crypto/md5" 21 "fmt" 22 "io/ioutil" 23 "os" 24 "os/user" 25 "path/filepath" 26 "strings" 27) 28 29var spaceSlashReplacer = strings.NewReplacer("/", "_", " ", "_") 30 31const katiBuildSuffix = "" 32const katiCleanspecSuffix = "-cleanspec" 33const katiPackageSuffix = "-package" 34 35// genKatiSuffix creates a filename suffix for kati-generated files so that we 36// can cache them based on their inputs. Such files include the generated Ninja 37// files and env.sh environment variable setup files. 38// 39// The filename suffix should encode all common changes to Kati inputs. 40// Currently that includes the TARGET_PRODUCT and kati-processed command line 41// arguments. 42func genKatiSuffix(ctx Context, config Config) { 43 // Construct the base suffix. 44 katiSuffix := "-" + config.TargetProduct() + config.CoverageSuffix() 45 46 // Append kati arguments to the suffix. 47 if args := config.KatiArgs(); len(args) > 0 { 48 katiSuffix += "-" + spaceSlashReplacer.Replace(strings.Join(args, "_")) 49 } 50 51 // If the suffix is too long, replace it with a md5 hash and write a 52 // file that contains the original suffix. 53 if len(katiSuffix) > 64 { 54 shortSuffix := "-" + fmt.Sprintf("%x", md5.Sum([]byte(katiSuffix))) 55 config.SetKatiSuffix(shortSuffix) 56 57 ctx.Verbosef("Kati ninja suffix too long: %q", katiSuffix) 58 ctx.Verbosef("Replacing with: %q", shortSuffix) 59 60 if err := ioutil.WriteFile(strings.TrimSuffix(config.KatiBuildNinjaFile(), "ninja")+"suf", []byte(katiSuffix), 0777); err != nil { 61 ctx.Println("Error writing suffix file:", err) 62 } 63 } else { 64 config.SetKatiSuffix(katiSuffix) 65 } 66} 67 68func writeValueIfChanged(ctx Context, config Config, dir string, filename string, value string) { 69 filePath := filepath.Join(dir, filename) 70 previousValue := "" 71 rawPreviousValue, err := ioutil.ReadFile(filePath) 72 if err == nil { 73 previousValue = string(rawPreviousValue) 74 } 75 76 if previousValue != value { 77 if err = ioutil.WriteFile(filePath, []byte(value), 0666); err != nil { 78 ctx.Fatalf("Failed to write: %v", err) 79 } 80 } 81} 82 83// Base function to construct and run the Kati command line with additional 84// arguments, and a custom function closure to mutate the environment Kati runs 85// in. 86func runKati(ctx Context, config Config, extraSuffix string, args []string, envFunc func(*Environment)) { 87 executable := config.KatiBin() 88 // cKati arguments. 89 args = append([]string{ 90 // Instead of executing commands directly, generate a Ninja file. 91 "--ninja", 92 // Generate Ninja files in the output directory. 93 "--ninja_dir=" + config.OutDir(), 94 // Filename suffix of the generated Ninja file. 95 "--ninja_suffix=" + config.KatiSuffix() + extraSuffix, 96 // Remove common parts at the beginning of a Ninja file, like build_dir, 97 // local_pool and _kati_always_build_. Allows Kati to be run multiple 98 // times, with generated Ninja files combined in a single invocation 99 // using 'include'. 100 "--no_ninja_prelude", 101 // Support declaring phony outputs in AOSP Ninja. 102 "--use_ninja_phony_output", 103 // Regenerate the Ninja file if environment inputs have changed. e.g. 104 // CLI flags, .mk file timestamps, env vars, $(wildcard ..) and some 105 // $(shell ..) results. 106 "--regen", 107 // Skip '-include' directives starting with the specified path. Used to 108 // ignore generated .mk files. 109 "--ignore_optional_include=" + filepath.Join(config.OutDir(), "%.P"), 110 // Detect the use of $(shell echo ...). 111 "--detect_android_echo", 112 // Colorful ANSI-based warning and error messages. 113 "--color_warnings", 114 // Generate all targets, not just the top level requested ones. 115 "--gen_all_targets", 116 // Use the built-in emulator of GNU find for better file finding 117 // performance. Used with $(shell find ...). 118 "--use_find_emulator", 119 // Fail when the find emulator encounters problems. 120 "--werror_find_emulator", 121 // Do not provide any built-in rules. 122 "--no_builtin_rules", 123 // Fail when suffix rules are used. 124 "--werror_suffix_rules", 125 // Fail when a real target depends on a phony target. 126 "--werror_real_to_phony", 127 // Makes real_to_phony checks assume that any top-level or leaf 128 // dependencies that does *not* have a '/' in it is a phony target. 129 "--top_level_phony", 130 // Fail when a phony target contains slashes. 131 "--werror_phony_looks_real", 132 // Fail when writing to a read-only directory. 133 "--werror_writable", 134 // Print Kati's internal statistics, such as the number of variables, 135 // implicit/explicit/suffix rules, and so on. 136 "--kati_stats", 137 }, args...) 138 139 // Generate a minimal Ninja file. 140 // 141 // Used for build_test and multiproduct_kati, which runs Kati several 142 // hundred times for different configurations to test file generation logic. 143 // These can result in generating Ninja files reaching ~1GB or more, 144 // resulting in ~hundreds of GBs of writes. 145 // 146 // Since we don't care about executing the Ninja files in these test cases, 147 // generating the Ninja file content wastes time, so skip writing any 148 // information out with --empty_ninja_file. 149 // 150 // From https://github.com/google/kati/commit/87b8da7af2c8bea28b1d8ab17679453d859f96e5 151 if config.EmptyNinjaFile() { 152 args = append(args, "--empty_ninja_file") 153 } 154 155 // Apply 'local_pool' to to all rules that don't specify a pool. 156 if config.UseRemoteBuild() { 157 args = append(args, "--default_pool=local_pool") 158 } 159 160 cmd := Command(ctx, config, "ckati", executable, args...) 161 162 // Set up the nsjail sandbox. 163 cmd.Sandbox = katiSandbox 164 165 // Set up stdout and stderr. 166 pipe, err := cmd.StdoutPipe() 167 if err != nil { 168 ctx.Fatalln("Error getting output pipe for ckati:", err) 169 } 170 cmd.Stderr = cmd.Stdout 171 172 var username string 173 // Pass on various build environment metadata to Kati. 174 if usernameFromEnv, ok := cmd.Environment.Get("BUILD_USERNAME"); !ok { 175 username = "unknown" 176 if u, err := user.Current(); err == nil { 177 username = u.Username 178 } else { 179 ctx.Println("Failed to get current user:", err) 180 } 181 cmd.Environment.Set("BUILD_USERNAME", username) 182 } else { 183 username = usernameFromEnv 184 } 185 186 // SOONG_USE_PARTIAL_COMPILE may be used in makefiles, but both cases must be supported. 187 // 188 // In general, the partial compile features will be implemented in Soong-based rules. We 189 // also allow them to be used in makefiles. Clear the environment variable when calling 190 // kati so that we avoid reanalysis when the user changes it. We will pass it to Ninja. 191 // As a result, rules where we want to allow the developer to toggle the feature ("use 192 // the partial compile feature" vs "legacy, aka full compile behavior") need to use this 193 // in the rule, since changing it will not cause reanalysis. 194 // 195 // Shell syntax in the rule might look something like this: 196 // if [[ -n ${SOONG_USE_PARTIAL_COMPILE} ]]; then 197 // # partial compile behavior 198 // else 199 // # legacy behavior 200 // fi 201 cmd.Environment.Unset("SOONG_USE_PARTIAL_COMPILE") 202 203 hostname, ok := cmd.Environment.Get("BUILD_HOSTNAME") 204 // Unset BUILD_HOSTNAME during kati run to avoid kati rerun, kati will use BUILD_HOSTNAME from a file. 205 cmd.Environment.Unset("BUILD_HOSTNAME") 206 if !ok { 207 hostname, err = os.Hostname() 208 if err != nil { 209 ctx.Println("Failed to read hostname:", err) 210 hostname = "unknown" 211 } 212 } 213 writeValueIfChanged(ctx, config, config.SoongOutDir(), "build_hostname.txt", hostname) 214 _, ok = cmd.Environment.Get("BUILD_NUMBER") 215 // Unset BUILD_NUMBER during kati run to avoid kati rerun, kati will use BUILD_NUMBER from a file. 216 cmd.Environment.Unset("BUILD_NUMBER") 217 if ok { 218 cmd.Environment.Set("HAS_BUILD_NUMBER", "true") 219 } else { 220 cmd.Environment.Set("HAS_BUILD_NUMBER", "false") 221 } 222 223 // Apply the caller's function closure to mutate the environment variables. 224 envFunc(cmd.Environment) 225 226 cmd.StartOrFatal() 227 // Set up the ToolStatus command line reader for Kati for a consistent UI 228 // for the user. 229 status.KatiReader(ctx.Status.StartTool(), pipe) 230 cmd.WaitOrFatal() 231} 232 233func runKatiBuild(ctx Context, config Config) { 234 ctx.BeginTrace(metrics.RunKati, "kati build") 235 defer ctx.EndTrace() 236 237 args := []string{ 238 // Mark the output directory as writable. 239 "--writable", config.OutDir() + "/", 240 // Fail when encountering implicit rules. e.g. 241 // %.foo: %.bar 242 // cp $< $@ 243 "--werror_implicit_rules", 244 // Entry point for the Kati Ninja file generation. 245 "-f", "build/make/core/main.mk", 246 } 247 248 if !config.BuildBrokenDupRules() { 249 // Fail when redefining / duplicating a target. 250 args = append(args, "--werror_overriding_commands") 251 } 252 253 args = append(args, config.KatiArgs()...) 254 255 args = append(args, 256 // Location of the Make vars .mk file generated by Soong. 257 "SOONG_MAKEVARS_MK="+config.SoongMakeVarsMk(), 258 // Location of the Android.mk file generated by Soong. This 259 // file contains Soong modules represented as Kati modules, 260 // allowing Kati modules to depend on Soong modules. 261 "SOONG_ANDROID_MK="+config.SoongAndroidMk(), 262 // Directory containing outputs for the target device. 263 "TARGET_DEVICE_DIR="+config.TargetDeviceDir(), 264 // Directory containing .mk files for packaging purposes, such as 265 // the dist.mk file, containing dist-for-goals data. 266 "KATI_PACKAGE_MK_DIR="+config.KatiPackageMkDir()) 267 268 runKati(ctx, config, katiBuildSuffix, args, func(env *Environment) {}) 269 270 // compress and dist the main build ninja file. 271 distGzipFile(ctx, config, config.KatiBuildNinjaFile()) 272 273 // Cleanup steps. 274 cleanCopyHeaders(ctx, config) 275 cleanOldInstalledFiles(ctx, config) 276} 277 278// Clean out obsolete header files on the disk that were *not copied* during the 279// build with BUILD_COPY_HEADERS and LOCAL_COPY_HEADERS. 280// 281// These should be increasingly uncommon, as it's a deprecated feature and there 282// isn't an equivalent feature in Soong. 283func cleanCopyHeaders(ctx Context, config Config) { 284 ctx.BeginTrace("clean", "clean copy headers") 285 defer ctx.EndTrace() 286 287 // Read and parse the list of copied headers from a file in the product 288 // output directory. 289 data, err := ioutil.ReadFile(filepath.Join(config.ProductOut(), ".copied_headers_list")) 290 if err != nil { 291 if os.IsNotExist(err) { 292 return 293 } 294 ctx.Fatalf("Failed to read copied headers list: %v", err) 295 } 296 297 headers := strings.Fields(string(data)) 298 if len(headers) < 1 { 299 ctx.Fatal("Failed to parse copied headers list: %q", string(data)) 300 } 301 headerDir := headers[0] 302 headers = headers[1:] 303 304 // Walk the tree and remove any headers that are not in the list of copied 305 // headers in the current build. 306 filepath.Walk(headerDir, 307 func(path string, info os.FileInfo, err error) error { 308 if err != nil { 309 return nil 310 } 311 if info.IsDir() { 312 return nil 313 } 314 if !inList(path, headers) { 315 ctx.Printf("Removing obsolete header %q", path) 316 if err := os.Remove(path); err != nil { 317 ctx.Fatalf("Failed to remove obsolete header %q: %v", path, err) 318 } 319 } 320 return nil 321 }) 322} 323 324// Clean out any previously installed files from the disk that are not installed 325// in the current build. 326func cleanOldInstalledFiles(ctx Context, config Config) { 327 ctx.BeginTrace("clean", "clean old installed files") 328 defer ctx.EndTrace() 329 330 // We shouldn't be removing files from one side of the two-step asan builds 331 var suffix string 332 if v, ok := config.Environment().Get("SANITIZE_TARGET"); ok { 333 if sanitize := strings.Fields(v); inList("address", sanitize) { 334 suffix = "_asan" 335 } 336 } 337 338 cleanOldFiles(ctx, config.ProductOut(), ".installable_files"+suffix) 339 340 cleanOldFiles(ctx, config.HostOut(), ".installable_test_files") 341} 342 343// Generate the Ninja file containing the packaging command lines for the dist 344// dir. 345func runKatiPackage(ctx Context, config Config) { 346 ctx.BeginTrace(metrics.RunKati, "kati package") 347 defer ctx.EndTrace() 348 349 args := []string{ 350 // Mark the dist dir as writable. 351 "--writable", config.DistDir() + "/", 352 // Fail when encountering implicit rules. e.g. 353 "--werror_implicit_rules", 354 // Fail when redefining / duplicating a target. 355 "--werror_overriding_commands", 356 // Entry point. 357 "-f", "build/make/packaging/main.mk", 358 // Directory containing .mk files for packaging purposes, such as 359 // the dist.mk file, containing dist-for-goals data. 360 "KATI_PACKAGE_MK_DIR=" + config.KatiPackageMkDir(), 361 } 362 363 // Run Kati against a restricted set of environment variables. 364 runKati(ctx, config, katiPackageSuffix, args, func(env *Environment) { 365 env.Allow([]string{ 366 // Some generic basics 367 "LANG", 368 "LC_MESSAGES", 369 "PATH", 370 "PWD", 371 "TMPDIR", 372 373 // Tool configs 374 "ASAN_SYMBOLIZER_PATH", 375 "JAVA_HOME", 376 "PYTHONDONTWRITEBYTECODE", 377 378 // Build configuration 379 "ANDROID_BUILD_SHELL", 380 "DIST_DIR", 381 "OUT_DIR", 382 "FILE_NAME_TAG", 383 }...) 384 385 if config.Dist() { 386 env.Set("DIST", "true") 387 env.Set("DIST_DIR", config.DistDir()) 388 } 389 }) 390 391 // Compress and dist the packaging Ninja file. 392 distGzipFile(ctx, config, config.KatiPackageNinjaFile()) 393} 394 395// Run Kati on the cleanspec files to clean the build. 396func runKatiCleanSpec(ctx Context, config Config) { 397 ctx.BeginTrace(metrics.RunKati, "kati cleanspec") 398 defer ctx.EndTrace() 399 400 runKati(ctx, config, katiCleanspecSuffix, []string{ 401 // Fail when encountering implicit rules. e.g. 402 "--werror_implicit_rules", 403 // Fail when redefining / duplicating a target. 404 "--werror_overriding_commands", 405 // Entry point. 406 "-f", "build/make/core/cleanbuild.mk", 407 "SOONG_MAKEVARS_MK=" + config.SoongMakeVarsMk(), 408 "TARGET_DEVICE_DIR=" + config.TargetDeviceDir(), 409 }, func(env *Environment) {}) 410} 411