1#!/usr/bin/env python3 2 3import argparse 4import fnmatch 5import html 6import io 7import json 8import os 9import pathlib 10import subprocess 11import types 12import sys 13 14 15class Graph: 16 def __init__(self, modules): 17 def get_or_make_node(dictionary, id, module): 18 node = dictionary.get(id) 19 if node: 20 if module and not node.module: 21 node.module = module 22 return node 23 node = Node(id, module) 24 dictionary[id] = node 25 return node 26 self.nodes = dict() 27 for module in modules.values(): 28 node = get_or_make_node(self.nodes, module.id, module) 29 for d in module.deps: 30 dep = get_or_make_node(self.nodes, d.id, None) 31 node.deps.add(dep) 32 dep.rdeps.add(node) 33 node.dep_tags.setdefault(dep, list()).append(d) 34 35 def find_paths(self, id1, id2, tag_filter): 36 # Throws KeyError if one of the names isn't found 37 def recurse(node1, node2, visited): 38 result = set() 39 for dep in node1.rdeps: 40 if not matches_tag(dep, node1, tag_filter): 41 continue 42 if dep == node2: 43 result.add(node2) 44 if dep not in visited: 45 visited.add(dep) 46 found = recurse(dep, node2, visited) 47 if found: 48 result |= found 49 result.add(dep) 50 return result 51 node1 = self.nodes[id1] 52 node2 = self.nodes[id2] 53 # Take either direction 54 p = recurse(node1, node2, set()) 55 if p: 56 p.add(node1) 57 return p 58 p = recurse(node2, node1, set()) 59 p.add(node2) 60 return p 61 62 63class Node: 64 def __init__(self, id, module): 65 self.id = id 66 self.module = module 67 self.deps = set() 68 self.rdeps = set() 69 self.dep_tags = {} 70 71 72PROVIDERS = [ 73 "android/soong/java.JarJarProviderData", 74 "android/soong/java.BaseJarJarProviderData", 75] 76 77 78def format_dep_label(node, dep): 79 tags = node.dep_tags.get(dep) 80 labels = [] 81 if tags: 82 labels = [tag.tag_type.split("/")[-1] for tag in tags] 83 labels = sorted(set(labels)) 84 if labels: 85 result = "<<table border=\"0\" cellborder=\"0\" cellspacing=\"0\" cellpadding=\"0\">" 86 for label in labels: 87 result += f"<tr><td>{label}</td></tr>" 88 result += "</table>>" 89 return result 90 91 92def format_node_label(node, module_formatter): 93 result = "<<table border=\"0\" cellborder=\"0\" cellspacing=\"0\" cellpadding=\"0\">" 94 95 # node name 96 result += f"<tr><td><b>{node.module.name if node.module else node.id}</b></td></tr>" 97 98 if node.module: 99 # node_type 100 result += f"<tr><td>{node.module.type}</td></tr>" 101 102 # module_formatter will return a list of rows 103 for row in module_formatter(node.module): 104 row = html.escape(row) 105 result += f"<tr><td><font color=\"#666666\">{row}</font></td></tr>" 106 107 result += "</table>>" 108 return result 109 110 111def format_source_pos(file, lineno): 112 result = file 113 if lineno: 114 result += f":{lineno}" 115 return result 116 117 118STRIP_TYPE_PREFIXES = [ 119 "android/soong/", 120 "github.com/google/", 121] 122 123 124def format_provider(provider): 125 result = "" 126 for prefix in STRIP_TYPE_PREFIXES: 127 if provider.type.startswith(prefix): 128 result = provider.type[len(prefix):] 129 break 130 if not result: 131 result = provider.type 132 if True and provider.debug: 133 result += " (" + provider.debug + ")" 134 return result 135 136 137def load_soong_debug(): 138 # Read the json 139 try: 140 with open(SOONG_DEBUG_DATA_FILENAME) as f: 141 info = json.load(f, object_hook=lambda d: types.SimpleNamespace(**d)) 142 except IOError: 143 sys.stderr.write(f"error: Unable to open {SOONG_DEBUG_DATA_FILENAME}. Make sure you have" 144 + " built with GENERATE_SOONG_DEBUG.\n") 145 sys.exit(1) 146 147 # Construct IDs, which are name + variant if the 148 name_counts = dict() 149 for m in info.modules: 150 name_counts[m.name] = name_counts.get(m.name, 0) + 1 151 def get_id(m): 152 result = m.name 153 if name_counts[m.name] > 1 and m.variant: 154 result += "@@" + m.variant 155 return result 156 for m in info.modules: 157 m.id = get_id(m) 158 for dep in m.deps: 159 dep.id = get_id(dep) 160 161 return info 162 163 164def load_modules(): 165 info = load_soong_debug() 166 167 # Filter out unnamed modules 168 modules = dict() 169 for m in info.modules: 170 if not m.name: 171 continue 172 modules[m.id] = m 173 174 return modules 175 176 177def load_graph(): 178 modules=load_modules() 179 return Graph(modules) 180 181 182def module_selection_args(parser): 183 parser.add_argument("modules", nargs="*", 184 help="Modules to match. Can be glob-style wildcards.") 185 parser.add_argument("--provider", nargs="+", 186 help="Match the given providers.") 187 parser.add_argument("--dep", nargs="+", 188 help="Match the given providers.") 189 190 191def load_and_filter_modules(args): 192 # Which modules are printed 193 matchers = [] 194 if args.modules: 195 matchers.append(lambda m: [True for pattern in args.modules 196 if fnmatch.fnmatchcase(m.name, pattern)]) 197 if args.provider: 198 matchers.append(lambda m: [True for pattern in args.provider 199 if [True for p in m.providers if p.type.endswith(pattern)]]) 200 if args.dep: 201 matchers.append(lambda m: [True for pattern in args.dep 202 if [True for d in m.deps if d.id == pattern]]) 203 204 if not matchers: 205 sys.stderr.write("error: At least one module matcher must be supplied\n") 206 sys.exit(1) 207 208 info = load_soong_debug() 209 for m in sorted(info.modules, key=lambda m: (m.name, m.variant)): 210 if len([matcher for matcher in matchers if matcher(m)]) == len(matchers): 211 yield m 212 213 214def print_args(parser): 215 parser.add_argument("--label", action="append", metavar="JQ_FILTER", 216 help="jq query for each module metadata") 217 parser.add_argument("--deptags", action="store_true", 218 help="show dependency tags (makes the graph much more complex)") 219 parser.add_argument("--tag", action="append", default=[], 220 help="Limit output to these dependency tags.") 221 222 group = parser.add_argument_group("output formats", 223 "If no format is provided, a dot file will be written to" 224 + " stdout.") 225 output = group.add_mutually_exclusive_group() 226 output.add_argument("--dot", type=str, metavar="FILENAME", 227 help="Write the graph to this file as dot (graphviz format)") 228 output.add_argument("--svg", type=str, metavar="FILENAME", 229 help="Write the graph to this file as svg") 230 231 232def print_nodes(args, nodes, module_formatter): 233 # Generate the graphviz 234 dep_tag_id = 0 235 dot = io.StringIO() 236 dot.write("digraph {\n") 237 dot.write("node [shape=box];") 238 239 for node in nodes: 240 dot.write(f"\"{node.id}\" [label={format_node_label(node, module_formatter)}];\n") 241 for dep in node.deps: 242 if dep in nodes: 243 if args.deptags: 244 dot.write(f"\"{node.id}\" -> \"__dep_tag_{dep_tag_id}\" [ arrowhead=none ];\n") 245 dot.write(f"\"__dep_tag_{dep_tag_id}\" -> \"{dep.id}\";\n") 246 dot.write(f"\"__dep_tag_{dep_tag_id}\"" 247 + f"[label={format_dep_label(node, dep)} shape=ellipse" 248 + " color=\"#666666\" fontcolor=\"#666666\"];\n") 249 else: 250 dot.write(f"\"{node.id}\" -> \"{dep.id}\";\n") 251 dep_tag_id += 1 252 dot.write("}\n") 253 text = dot.getvalue() 254 255 # Write it somewhere 256 if args.dot: 257 with open(args.dot, "w") as f: 258 f.write(text) 259 elif args.svg: 260 subprocess.run(["dot", "-Tsvg", "-o", args.svg], 261 input=text, text=True, check=True) 262 else: 263 sys.stdout.write(text) 264 265 266def matches_tag(node, dep, tag_filter): 267 if not tag_filter: 268 return True 269 return not tag_filter.isdisjoint([t.tag_type for t in node.dep_tags[dep]]) 270 271 272def get_deps(nodes, root, maxdepth, reverse, tag_filter): 273 if root in nodes: 274 return 275 nodes.add(root) 276 if maxdepth != 0: 277 for dep in (root.rdeps if reverse else root.deps): 278 if not matches_tag(root, dep, tag_filter): 279 continue 280 get_deps(nodes, dep, maxdepth-1, reverse, tag_filter) 281 282 283def new_module_formatter(args): 284 def module_formatter(module): 285 if not args.label: 286 return [] 287 result = [] 288 text = json.dumps(module, default=lambda o: o.__dict__) 289 for jq_filter in args.label: 290 proc = subprocess.run(["jq", jq_filter], 291 input=text, text=True, check=True, stdout=subprocess.PIPE) 292 if proc.stdout: 293 o = json.loads(proc.stdout) 294 if type(o) == list: 295 for row in o: 296 if row: 297 result.append(row) 298 elif type(o) == dict: 299 result.append(str(proc.stdout).strip()) 300 else: 301 if o: 302 result.append(str(o).strip()) 303 return result 304 return module_formatter 305 306 307class BetweenCommand: 308 help = "Print the module graph between two nodes." 309 310 def args(self, parser): 311 parser.add_argument("module", nargs=2, 312 help="the two modules") 313 print_args(parser) 314 315 def run(self, args): 316 graph = load_graph() 317 print_nodes(args, graph.find_paths(args.module[0], args.module[1], set(args.tag)), 318 new_module_formatter(args)) 319 320 321class DepsCommand: 322 help = "Print the module graph of dependencies of one or more modules" 323 324 def args(self, parser): 325 parser.add_argument("module", nargs="+", 326 help="Module to print dependencies of") 327 parser.add_argument("--reverse", action="store_true", 328 help="traverse reverse dependencies") 329 parser.add_argument("--depth", type=int, default=-1, 330 help="max depth of dependencies (can keep the graph size reasonable)") 331 print_args(parser) 332 333 def run(self, args): 334 graph = load_graph() 335 nodes = set() 336 err = False 337 for id in args.module: 338 root = graph.nodes.get(id) 339 if not root: 340 sys.stderr.write(f"error: Can't find root: {id}\n") 341 err = True 342 continue 343 get_deps(nodes, root, args.depth, args.reverse, set(args.tag)) 344 if err: 345 sys.exit(1) 346 print_nodes(args, nodes, new_module_formatter(args)) 347 348 349class IdCommand: 350 help = "Print the id (name + variant) of matching modules" 351 352 def args(self, parser): 353 module_selection_args(parser) 354 355 def run(self, args): 356 for m in load_and_filter_modules(args): 357 print(m.id) 358 359 360class JsonCommand: 361 help = "Print metadata about modules in json format" 362 363 def args(self, parser): 364 module_selection_args(parser) 365 parser.add_argument("--list", action="store_true", 366 help="Print the results in a json list. If not set and multiple" 367 + " modules are matched, the output won't be valid json.") 368 369 def run(self, args): 370 modules = load_and_filter_modules(args) 371 if args.list: 372 json.dump([m for m in modules], sys.stdout, indent=4, default=lambda o: o.__dict__) 373 else: 374 for m in modules: 375 json.dump(m, sys.stdout, indent=4, default=lambda o: o.__dict__) 376 print() 377 378 379class QueryCommand: 380 help = "Query details about modules" 381 382 def args(self, parser): 383 module_selection_args(parser) 384 385 def run(self, args): 386 for m in load_and_filter_modules(args): 387 print(m.id) 388 print(f" type: {m.type}") 389 print(f" location: {format_source_pos(m.source_file, m.source_line)}") 390 for p in m.providers: 391 print(f" provider: {format_provider(p)}") 392 for d in m.deps: 393 print(f" dep: {d.id}") 394 395 396class StarCommand: 397 help = "Print the dependencies and reverse dependencies of a module" 398 399 def args(self, parser): 400 parser.add_argument("module", nargs="+", 401 help="Module to print dependencies of") 402 parser.add_argument("--depth", type=int, required=True, 403 help="max depth of dependencies") 404 print_args(parser) 405 406 def run(self, args): 407 graph = load_graph() 408 nodes = set() 409 err = False 410 for id in args.module: 411 root = graph.nodes.get(id) 412 if not root: 413 sys.stderr.write(f"error: Can't find root: {id}\n") 414 err = True 415 continue 416 get_deps(nodes, root, args.depth, False, set(args.tag)) 417 nodes.remove(root) # Remove it so get_deps doesn't bail out 418 get_deps(nodes, root, args.depth, True, set(args.tag)) 419 if err: 420 sys.exit(1) 421 print_nodes(args, nodes, new_module_formatter(args)) 422 423 424 425COMMANDS = { 426 "between": BetweenCommand(), 427 "deps": DepsCommand(), 428 "id": IdCommand(), 429 "json": JsonCommand(), 430 "query": QueryCommand(), 431 "star": StarCommand(), 432} 433 434 435def assert_env(name): 436 val = os.getenv(name) 437 if not val: 438 sys.stderr.write(f"{name} not set. please make sure you've run lunch.") 439 return val 440 441ANDROID_BUILD_TOP = assert_env("ANDROID_BUILD_TOP") 442 443TARGET_PRODUCT = assert_env("TARGET_PRODUCT") 444OUT_DIR = os.getenv("OUT_DIR") 445if not OUT_DIR: 446 OUT_DIR = "out" 447if OUT_DIR[0] != "/": 448 OUT_DIR = pathlib.Path(ANDROID_BUILD_TOP).joinpath(OUT_DIR) 449SOONG_DEBUG_DATA_FILENAME = pathlib.Path(OUT_DIR).joinpath("soong/soong-debug-info.json") 450 451 452def main(): 453 parser = argparse.ArgumentParser() 454 subparsers = parser.add_subparsers(required=True, dest="command") 455 for name in sorted(COMMANDS.keys()): 456 command = COMMANDS[name] 457 subparser = subparsers.add_parser(name, help=command.help) 458 command.args(subparser) 459 args = parser.parse_args() 460 COMMANDS[args.command].run(args) 461 sys.exit(0) 462 463 464if __name__ == "__main__": 465 main() 466 467