1// Copyright 2023 The Bazel Authors. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package python 16 17import ( 18 "fmt" 19 "log" 20 "os" 21 "path/filepath" 22 "strings" 23 24 "github.com/bazelbuild/bazel-gazelle/config" 25 "github.com/bazelbuild/bazel-gazelle/label" 26 "github.com/bazelbuild/bazel-gazelle/repo" 27 "github.com/bazelbuild/bazel-gazelle/resolve" 28 "github.com/bazelbuild/bazel-gazelle/rule" 29 bzl "github.com/bazelbuild/buildtools/build" 30 "github.com/emirpasic/gods/sets/treeset" 31 godsutils "github.com/emirpasic/gods/utils" 32 33 "github.com/bazelbuild/rules_python/gazelle/pythonconfig" 34) 35 36const languageName = "py" 37 38const ( 39 // resolvedDepsKey is the attribute key used to pass dependencies that don't 40 // need to be resolved by the dependency resolver in the Resolver step. 41 resolvedDepsKey = "_gazelle_python_resolved_deps" 42) 43 44// Resolver satisfies the resolve.Resolver interface. It resolves dependencies 45// in rules generated by this extension. 46type Resolver struct{} 47 48// Name returns the name of the language. This is the prefix of the kinds of 49// rules generated. E.g. py_library and py_binary. 50func (*Resolver) Name() string { return languageName } 51 52// Imports returns a list of ImportSpecs that can be used to import the rule 53// r. This is used to populate RuleIndex. 54// 55// If nil is returned, the rule will not be indexed. If any non-nil slice is 56// returned, including an empty slice, the rule will be indexed. 57func (py *Resolver) Imports(c *config.Config, r *rule.Rule, f *rule.File) []resolve.ImportSpec { 58 cfgs := c.Exts[languageName].(pythonconfig.Configs) 59 cfg := cfgs[f.Pkg] 60 srcs := r.AttrStrings("srcs") 61 provides := make([]resolve.ImportSpec, 0, len(srcs)+1) 62 for _, src := range srcs { 63 ext := filepath.Ext(src) 64 if ext != ".py" { 65 continue 66 } 67 if cfg.PerFileGeneration() && len(srcs) > 1 && src == pyLibraryEntrypointFilename { 68 // Do not provide import spec from __init__.py when it is being included as 69 // part of another module. 70 continue 71 } 72 pythonProjectRoot := cfg.PythonProjectRoot() 73 provide := importSpecFromSrc(pythonProjectRoot, f.Pkg, src) 74 provides = append(provides, provide) 75 } 76 if len(provides) == 0 { 77 return nil 78 } 79 return provides 80} 81 82// importSpecFromSrc determines the ImportSpec based on the target that contains the src so that 83// the target can be indexed for import statements that match the calculated src relative to the its 84// Python project root. 85func importSpecFromSrc(pythonProjectRoot, bzlPkg, src string) resolve.ImportSpec { 86 pythonPkgDir := filepath.Join(bzlPkg, filepath.Dir(src)) 87 relPythonPkgDir, err := filepath.Rel(pythonProjectRoot, pythonPkgDir) 88 if err != nil { 89 panic(fmt.Errorf("unexpected failure: %v", err)) 90 } 91 if relPythonPkgDir == "." { 92 relPythonPkgDir = "" 93 } 94 pythonPkg := strings.ReplaceAll(relPythonPkgDir, "/", ".") 95 filename := filepath.Base(src) 96 if filename == pyLibraryEntrypointFilename { 97 if pythonPkg != "" { 98 return resolve.ImportSpec{ 99 Lang: languageName, 100 Imp: pythonPkg, 101 } 102 } 103 } 104 moduleName := strings.TrimSuffix(filename, ".py") 105 var imp string 106 if pythonPkg == "" { 107 imp = moduleName 108 } else { 109 imp = fmt.Sprintf("%s.%s", pythonPkg, moduleName) 110 } 111 return resolve.ImportSpec{ 112 Lang: languageName, 113 Imp: imp, 114 } 115} 116 117// Embeds returns a list of labels of rules that the given rule embeds. If 118// a rule is embedded by another importable rule of the same language, only 119// the embedding rule will be indexed. The embedding rule will inherit 120// the imports of the embedded rule. 121func (py *Resolver) Embeds(r *rule.Rule, from label.Label) []label.Label { 122 // TODO(f0rmiga): implement. 123 return make([]label.Label, 0) 124} 125 126// Resolve translates imported libraries for a given rule into Bazel 127// dependencies. Information about imported libraries is returned for each 128// rule generated by language.GenerateRules in 129// language.GenerateResult.Imports. Resolve generates a "deps" attribute (or 130// the appropriate language-specific equivalent) for each import according to 131// language-specific rules and heuristics. 132func (py *Resolver) Resolve( 133 c *config.Config, 134 ix *resolve.RuleIndex, 135 rc *repo.RemoteCache, 136 r *rule.Rule, 137 modulesRaw interface{}, 138 from label.Label, 139) { 140 // TODO(f0rmiga): may need to be defensive here once this Gazelle extension 141 // join with the main Gazelle binary with other rules. It may conflict with 142 // other generators that generate py_* targets. 143 deps := treeset.NewWith(godsutils.StringComparator) 144 if modulesRaw != nil { 145 cfgs := c.Exts[languageName].(pythonconfig.Configs) 146 cfg := cfgs[from.Pkg] 147 pythonProjectRoot := cfg.PythonProjectRoot() 148 modules := modulesRaw.(*treeset.Set) 149 it := modules.Iterator() 150 explainDependency := os.Getenv("EXPLAIN_DEPENDENCY") 151 hasFatalError := false 152 MODULES_LOOP: 153 for it.Next() { 154 mod := it.Value().(module) 155 moduleParts := strings.Split(mod.Name, ".") 156 possibleModules := []string{mod.Name} 157 for len(moduleParts) > 1 { 158 // Iterate back through the possible imports until 159 // a match is found. 160 // For example, "from foo.bar import baz" where baz is a module, we should try `foo.bar.baz` first, then 161 // `foo.bar`, then `foo`. 162 // In the first case, the import could be file `baz.py` in the directory `foo/bar`. 163 // Or, the import could be variable `baz` in file `foo/bar.py`. 164 // The import could also be from a standard module, e.g. `six.moves`, where 165 // the dependency is actually `six`. 166 moduleParts = moduleParts[:len(moduleParts)-1] 167 possibleModules = append(possibleModules, strings.Join(moduleParts, ".")) 168 } 169 errs := []error{} 170 POSSIBLE_MODULE_LOOP: 171 for _, moduleName := range possibleModules { 172 imp := resolve.ImportSpec{Lang: languageName, Imp: moduleName} 173 if override, ok := resolve.FindRuleWithOverride(c, imp, languageName); ok { 174 if override.Repo == "" { 175 override.Repo = from.Repo 176 } 177 if !override.Equal(from) { 178 if override.Repo == from.Repo { 179 override.Repo = "" 180 } 181 dep := override.Rel(from.Repo, from.Pkg).String() 182 deps.Add(dep) 183 if explainDependency == dep { 184 log.Printf("Explaining dependency (%s): "+ 185 "in the target %q, the file %q imports %q at line %d, "+ 186 "which resolves using the \"gazelle:resolve\" directive.\n", 187 explainDependency, from.String(), mod.Filepath, moduleName, mod.LineNumber) 188 } 189 continue MODULES_LOOP 190 } 191 } else { 192 if dep, ok := cfg.FindThirdPartyDependency(moduleName); ok { 193 deps.Add(dep) 194 if explainDependency == dep { 195 log.Printf("Explaining dependency (%s): "+ 196 "in the target %q, the file %q imports %q at line %d, "+ 197 "which resolves from the third-party module %q from the wheel %q.\n", 198 explainDependency, from.String(), mod.Filepath, moduleName, mod.LineNumber, mod.Name, dep) 199 } 200 continue MODULES_LOOP 201 } else { 202 matches := ix.FindRulesByImportWithConfig(c, imp, languageName) 203 if len(matches) == 0 { 204 // Check if the imported module is part of the standard library. 205 if isStdModule(module{Name: moduleName}) { 206 continue MODULES_LOOP 207 } else if cfg.ValidateImportStatements() { 208 err := fmt.Errorf( 209 "%[1]q, line %[2]d: %[3]q is an invalid dependency: possible solutions:\n"+ 210 "\t1. Add it as a dependency in the requirements.txt file.\n"+ 211 "\t2. Use the '# gazelle:resolve py %[3]s TARGET_LABEL' BUILD file directive to resolve to a known dependency.\n"+ 212 "\t3. Ignore it with a comment '# gazelle:ignore %[3]s' in the Python file.\n", 213 mod.Filepath, mod.LineNumber, moduleName, 214 ) 215 errs = append(errs, err) 216 continue POSSIBLE_MODULE_LOOP 217 } 218 } 219 filteredMatches := make([]resolve.FindResult, 0, len(matches)) 220 for _, match := range matches { 221 if match.IsSelfImport(from) { 222 // Prevent from adding itself as a dependency. 223 continue MODULES_LOOP 224 } 225 filteredMatches = append(filteredMatches, match) 226 } 227 if len(filteredMatches) == 0 { 228 continue POSSIBLE_MODULE_LOOP 229 } 230 if len(filteredMatches) > 1 { 231 sameRootMatches := make([]resolve.FindResult, 0, len(filteredMatches)) 232 for _, match := range filteredMatches { 233 if strings.HasPrefix(match.Label.Pkg, pythonProjectRoot) { 234 sameRootMatches = append(sameRootMatches, match) 235 } 236 } 237 if len(sameRootMatches) != 1 { 238 err := fmt.Errorf( 239 "%[1]q, line %[2]d: multiple targets (%[3]s) may be imported with %[4]q: possible solutions:\n"+ 240 "\t1. Disambiguate the above multiple targets by removing duplicate srcs entries.\n"+ 241 "\t2. Use the '# gazelle:resolve py %[4]s TARGET_LABEL' BUILD file directive to resolve to one of the above targets.\n", 242 mod.Filepath, mod.LineNumber, targetListFromResults(filteredMatches), moduleName) 243 errs = append(errs, err) 244 continue POSSIBLE_MODULE_LOOP 245 } 246 filteredMatches = sameRootMatches 247 } 248 matchLabel := filteredMatches[0].Label.Rel(from.Repo, from.Pkg) 249 dep := matchLabel.String() 250 deps.Add(dep) 251 if explainDependency == dep { 252 log.Printf("Explaining dependency (%s): "+ 253 "in the target %q, the file %q imports %q at line %d, "+ 254 "which resolves from the first-party indexed labels.\n", 255 explainDependency, from.String(), mod.Filepath, moduleName, mod.LineNumber) 256 } 257 continue MODULES_LOOP 258 } 259 } 260 } // End possible modules loop. 261 if len(errs) > 0 { 262 // If, after trying all possible modules, we still haven't found anything, error out. 263 joinedErrs := "" 264 for _, err := range errs { 265 joinedErrs = fmt.Sprintf("%s%s\n", joinedErrs, err) 266 } 267 log.Printf("ERROR: failed to validate dependencies for target %q:\n\n%v", from.String(), joinedErrs) 268 hasFatalError = true 269 } 270 } 271 if hasFatalError { 272 os.Exit(1) 273 } 274 } 275 resolvedDeps := r.PrivateAttr(resolvedDepsKey).(*treeset.Set) 276 if !resolvedDeps.Empty() { 277 it := resolvedDeps.Iterator() 278 for it.Next() { 279 deps.Add(it.Value()) 280 } 281 } 282 if !deps.Empty() { 283 r.SetAttr("deps", convertDependencySetToExpr(deps)) 284 } 285} 286 287// targetListFromResults returns a string with the human-readable list of 288// targets contained in the given results. 289func targetListFromResults(results []resolve.FindResult) string { 290 list := make([]string, len(results)) 291 for i, result := range results { 292 list[i] = result.Label.String() 293 } 294 return strings.Join(list, ", ") 295} 296 297// convertDependencySetToExpr converts the given set of dependencies to an 298// expression to be used in the deps attribute. 299func convertDependencySetToExpr(set *treeset.Set) bzl.Expr { 300 deps := make([]bzl.Expr, set.Size()) 301 it := set.Iterator() 302 for it.Next() { 303 dep := it.Value().(string) 304 deps[it.Index()] = &bzl.StringExpr{Value: dep} 305 } 306 return &bzl.ListExpr{List: deps} 307} 308