1// Copyright 2017 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package build 16 17import ( 18 "bytes" 19 "io/ioutil" 20 "os" 21 "path/filepath" 22 "strings" 23 24 "android/soong/finder" 25 "android/soong/finder/fs" 26 "android/soong/ui/logger" 27 28 "android/soong/ui/metrics" 29) 30 31// This file provides an interface to the Finder type for soong_ui. Finder is 32// used to recursively traverse the source tree to gather paths of files, such 33// as Android.bp or Android.mk, and store the lists/database of paths in files 34// under `$OUT_DIR/.module_paths`. This directory can also be dist'd. 35 36// NewSourceFinder returns a new Finder configured to search for source files. 37// Callers of NewSourceFinder should call <f.Shutdown()> when done 38func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { 39 ctx.BeginTrace(metrics.RunSetupTool, "find modules") 40 defer ctx.EndTrace() 41 42 // Set up the working directory for the Finder. 43 dir, err := os.Getwd() 44 if err != nil { 45 ctx.Fatalf("No working directory for module-finder: %v", err.Error()) 46 } 47 filesystem := fs.OsFs 48 49 // .out-dir and .find-ignore are markers for Finder to ignore siblings and 50 // subdirectories of the directory Finder finds them in, hence stopping the 51 // search recursively down those branches. It's possible that these files 52 // are in the root directory, and if they are, then the subsequent error 53 // messages are very confusing, so check for that here. 54 pruneFiles := []string{".out-dir", ".find-ignore"} 55 for _, name := range pruneFiles { 56 prunePath := filepath.Join(dir, name) 57 _, statErr := filesystem.Lstat(prunePath) 58 if statErr == nil { 59 ctx.Fatalf("%v must not exist", prunePath) 60 } 61 } 62 63 // Set up configuration parameters for the Finder cache. 64 cacheParams := finder.CacheParams{ 65 WorkingDirectory: dir, 66 RootDirs: androidBpSearchDirs(config), 67 FollowSymlinks: config.environ.IsEnvTrue("ALLOW_BP_UNDER_SYMLINKS"), 68 ExcludeDirs: []string{".git", ".repo"}, 69 PruneFiles: pruneFiles, 70 IncludeFiles: []string{ 71 // Kati build definitions. 72 "Android.mk", 73 // Product configuration files. 74 "AndroidProducts.mk", 75 // General Soong build definitions, using the Blueprint syntax. 76 "Android.bp", 77 // Kati clean definitions. 78 "CleanSpec.mk", 79 // Ownership definition. 80 "OWNERS", 81 // Test configuration for modules in directories that contain this 82 // file. 83 "TEST_MAPPING", 84 // METADATA file of packages 85 "METADATA", 86 }, 87 // .mk files for product/board configuration. 88 IncludeSuffixes: []string{".mk"}, 89 } 90 dumpDir := config.FileListDir() 91 f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard), 92 filepath.Join(dumpDir, "files.db")) 93 if err != nil { 94 ctx.Fatalf("Could not create module-finder: %v", err) 95 } 96 return f 97} 98 99func androidBpSearchDirs(config Config) []string { 100 dirs := []string{"."} // always search from root of source tree. 101 if config.searchApiDir { 102 // Search in out/api_surfaces 103 dirs = append(dirs, config.ApiSurfacesOutDir()) 104 } 105 return dirs 106} 107 108func findProductAndBoardConfigFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) { 109 matches := []string{} 110 for _, foundName := range entries.FileNames { 111 if foundName != "Android.mk" && 112 foundName != "AndroidProducts.mk" && 113 foundName != "CleanSpec.mk" && 114 strings.HasSuffix(foundName, ".mk") { 115 matches = append(matches, foundName) 116 } 117 } 118 return entries.DirNames, matches 119} 120 121// FindSources searches for source files known to <f> and writes them to the filesystem for 122// use later. 123func FindSources(ctx Context, config Config, f *finder.Finder) { 124 // note that dumpDir in FindSources may be different than dumpDir in NewSourceFinder 125 // if a caller such as multiproduct_kati wants to share one Finder among several builds 126 dumpDir := config.FileListDir() 127 os.MkdirAll(dumpDir, 0777) 128 129 // Stop searching a subdirectory recursively after finding an Android.mk. 130 androidMks := f.FindFirstNamedAt(".", "Android.mk") 131 androidMks = ignoreNdkAndroidMks(androidMks) 132 blockAndroidMks(ctx, androidMks) 133 err := dumpListToFile(ctx, config, androidMks, filepath.Join(dumpDir, "Android.mk.list")) 134 if err != nil { 135 ctx.Fatalf("Could not export module list: %v", err) 136 } 137 138 // Gate collecting/reporting mk metrics on builds that specifically request 139 // it, as identifying the total number of mk files adds 4-5ms onto null 140 // builds. 141 if config.reportMkMetrics { 142 androidMksTotal := f.FindNamedAt(".", "Android.mk") 143 144 ctx.Metrics.SetToplevelMakefiles(len(androidMks)) 145 ctx.Metrics.SetTotalMakefiles(len(androidMksTotal)) 146 ctx.Metrics.DumpMkMetrics(config.MkMetrics()) 147 } 148 149 // Stop searching a subdirectory recursively after finding a CleanSpec.mk. 150 cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk") 151 err = dumpListToFile(ctx, config, cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list")) 152 if err != nil { 153 ctx.Fatalf("Could not export module list: %v", err) 154 } 155 156 // Only consider AndroidProducts.mk in device/, vendor/ and product/, recursively in these directories. 157 androidProductsMks := f.FindNamedAt("device", "AndroidProducts.mk") 158 androidProductsMks = append(androidProductsMks, f.FindNamedAt("vendor", "AndroidProducts.mk")...) 159 androidProductsMks = append(androidProductsMks, f.FindNamedAt("product", "AndroidProducts.mk")...) 160 err = dumpListToFile(ctx, config, androidProductsMks, filepath.Join(dumpDir, "AndroidProducts.mk.list")) 161 if err != nil { 162 ctx.Fatalf("Could not export product list: %v", err) 163 } 164 165 // Recursively look for all OWNERS files. 166 owners := f.FindNamedAt(".", "OWNERS") 167 err = dumpListToFile(ctx, config, owners, filepath.Join(dumpDir, "OWNERS.list")) 168 if err != nil { 169 ctx.Fatalf("Could not find OWNERS: %v", err) 170 } 171 172 // Recursively look for all METADATA files. 173 metadataFiles := f.FindNamedAt(".", "METADATA") 174 metadataFiles = ignoreNonAndroidMetadataFiles(metadataFiles) 175 err = dumpListToFile(ctx, config, metadataFiles, filepath.Join(dumpDir, "METADATA.list")) 176 if err != nil { 177 ctx.Fatalf("Could not find METADATA: %v", err) 178 } 179 180 // Recursively look for all TEST_MAPPING files. 181 testMappings := f.FindNamedAt(".", "TEST_MAPPING") 182 err = dumpListToFile(ctx, config, testMappings, filepath.Join(dumpDir, "TEST_MAPPING.list")) 183 if err != nil { 184 ctx.Fatalf("Could not find TEST_MAPPING: %v", err) 185 } 186 187 // Recursively look for all Android.bp files 188 androidBps := f.FindNamedAt(".", "Android.bp") 189 if len(androidBps) == 0 { 190 ctx.Fatalf("No Android.bp found") 191 } 192 err = dumpListToFile(ctx, config, androidBps, filepath.Join(dumpDir, "Android.bp.list")) 193 if err != nil { 194 ctx.Fatalf("Could not find modules: %v", err) 195 } 196 197 // Recursively look for all product/board config files. 198 configurationFiles := f.FindMatching(".", findProductAndBoardConfigFiles) 199 err = dumpListToFile(ctx, config, configurationFiles, filepath.Join(dumpDir, "configuration.list")) 200 if err != nil { 201 ctx.Fatalf("Could not export product/board configuration list: %v", err) 202 } 203 204 if config.Dist() { 205 f.WaitForDbDump() 206 // Dist the files.db plain text database. 207 distFile(ctx, config, f.DbPath, "module_paths") 208 } 209} 210 211// Write the .list files to disk. 212func dumpListToFile(ctx Context, config Config, list []string, filePath string) (err error) { 213 desiredText := strings.Join(list, "\n") 214 desiredBytes := []byte(desiredText) 215 actualBytes, readErr := ioutil.ReadFile(filePath) 216 if readErr != nil || !bytes.Equal(desiredBytes, actualBytes) { 217 err = ioutil.WriteFile(filePath, desiredBytes, 0777) 218 if err != nil { 219 return err 220 } 221 } 222 223 distFile(ctx, config, filePath, "module_paths") 224 225 return nil 226} 227 228func ignoreNonAndroidMetadataFiles(metadataFiles []string) []string { 229 result := make([]string, 0, len(metadataFiles)) 230 for _, file := range metadataFiles { 231 // Ignore files like prebuilts/clang/host/linux-x86/clang-r536225/python3/lib/python3.11/site-packages/pip-23.1.2.dist-info/METADATA 232 // these METADATA files are from upstream and are not the METADATA files used in Android codebase. 233 if strings.Contains(file, "prebuilts/clang/host/") && strings.Contains(file, "/site-packages/") { 234 continue 235 } 236 result = append(result, file) 237 } 238 return result 239} 240