1#!/usr/bin/env python3 2# 3# Copyright (C) 2022 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License.""" 16"""Helpers pertaining to clang compile actions.""" 17 18import collections 19import pathlib 20import subprocess 21from commands import CommandInfo 22from commands import flag_repr 23from commands import is_flag_starts_with 24from commands import parse_flag_groups 25from diffs.diff import ExtractInfo 26from diffs.context import ContextDiff 27from diffs.nm import NmSymbolDiff 28from diffs.bloaty import BloatyDiff 29 30 31class ClangCompileInfo(CommandInfo): 32 """Contains information about a clang compile action commandline.""" 33 34 def __init__(self, tool, args): 35 CommandInfo.__init__(self, tool, args) 36 37 flag_groups = parse_flag_groups(args, _custom_flag_group) 38 39 misc = [] 40 i_includes = [] 41 iquote_includes = [] 42 isystem_includes = [] 43 defines = [] 44 warnings = [] 45 features = [] 46 libraries = [] 47 linker_args = [] 48 assembler_args = [] 49 file_flags = [] 50 for g in flag_groups: 51 if is_flag_starts_with("D", g) or is_flag_starts_with("U", g): 52 defines += [g] 53 elif is_flag_starts_with("f", g): 54 features += [g] 55 elif is_flag_starts_with("l", g): 56 libraries += [g] 57 elif is_flag_starts_with("Wl", g): 58 linker_args += [g] 59 elif is_flag_starts_with("Wa", g) and not is_flag_starts_with("Wall", g): 60 assembler_args += [g] 61 elif is_flag_starts_with("W", g) or is_flag_starts_with("w", g): 62 warnings += [g] 63 elif is_flag_starts_with("I", g): 64 i_includes += [g] 65 elif is_flag_starts_with("isystem", g): 66 isystem_includes += [g] 67 elif is_flag_starts_with("iquote", g): 68 iquote_includes += [g] 69 elif ( 70 is_flag_starts_with("MF", g) 71 or is_flag_starts_with("o", g) 72 or _is_src_group(g) 73 ): 74 file_flags += [g] 75 else: 76 misc += [g] 77 self.features = features 78 self.defines = _process_defines(defines) 79 self.libraries = libraries 80 self.linker_args = linker_args 81 self.assembler_args = assembler_args 82 self.i_includes = _process_includes(i_includes) 83 self.iquote_includes = _process_includes(iquote_includes) 84 self.isystem_includes = _process_includes(isystem_includes) 85 self.file_flags = file_flags 86 self.warnings = warnings 87 self.misc_flags = sorted(misc, key=flag_repr) 88 89 def _str_for_field(self, field_name, values): 90 s = " " + field_name + ":\n" 91 for x in values: 92 s += " " + flag_repr(x) + "\n" 93 return s 94 95 def __str__(self): 96 s = "ClangCompileInfo:\n" 97 98 for label, fields in { 99 "Features": self.features, 100 "Defines": self.defines, 101 "Libraries": self.libraries, 102 "Linker args": self.linker_args, 103 "Assembler args": self.assembler_args, 104 "Includes (-I,": self.i_includes, 105 "Includes (-iquote,": self.iquote_includes, 106 "Includes (-isystem,": self.isystem_includes, 107 "Files": self.file_flags, 108 "Warnings": self.warnings, 109 "Misc": self.misc_flags, 110 }.items(): 111 if len(fields) > 0: 112 s += self._str_for_field(label, list(set(fields))) 113 114 return s 115 116 def compare(self, other): 117 """computes difference in arguments from another ClangCompileInfo""" 118 diffs = ClangCompileInfo(self.tool, []) 119 diffs.defines = [i for i in self.defines if i not in other.defines] 120 diffs.warnings = [i for i in self.warnings if i not in other.warnings] 121 diffs.features = [i for i in self.features if i not in other.features] 122 diffs.libraries = [i for i in self.libraries if i not in other.libraries] 123 diffs.linker_args = [ 124 i for i in self.linker_args if i not in other.linker_args 125 ] 126 diffs.assembler_args = [ 127 i for i in self.assembler_args if i not in other.assembler_args 128 ] 129 diffs.i_includes = [i for i in self.i_includes if i not in other.i_includes] 130 diffs.iquote_includes = [ 131 i for i in self.iquote_includes if i not in other.iquote_includes 132 ] 133 diffs.isystem_includes = [ 134 i for i in self.isystem_includes if i not in other.isystem_includes 135 ] 136 diffs.file_flags = [i for i in self.file_flags if i not in other.file_flags] 137 diffs.misc_flags = [i for i in self.misc_flags if i not in other.misc_flags] 138 return diffs 139 140 141def _is_src_group(x): 142 """Returns true if the given flag group describes a source file.""" 143 return isinstance(x, str) and x.endswith(".cpp") 144 145 146def _custom_flag_group(x): 147 """Identifies single-arg flag groups for clang compiles. 148 149 Returns a flag group if the given argument corresponds to a single-argument 150 flag group for clang compile. (For example, `-c` is a single-arg flag for 151 clang compiles, but may not be for other tools.) 152 153 See commands.parse_flag_groups documentation for signature details. 154 """ 155 if x.startswith("-I") and len(x) > 2: 156 return ("I", x[2:]) 157 if x.startswith("-W") and len(x) > 2: 158 return x 159 elif x == "-c": 160 return x 161 return None 162 163 164def _process_defines(defs): 165 """Processes and returns deduplicated define flags from all define args.""" 166 # TODO(cparsons): Determine and return effective defines (returning the last 167 # set value). 168 defines_by_var = collections.defaultdict(list) 169 for x in defs: 170 if isinstance(x, tuple): 171 var_name = x[0][2:] 172 else: 173 var_name = x[2:] 174 defines_by_var[var_name].append(x) 175 result = [] 176 for k in sorted(defines_by_var): 177 d = defines_by_var[k] 178 for x in d: 179 result += [x] 180 return result 181 182 183def _process_includes(includes): 184 # Drop genfiles directories; makes diffing easier. 185 result = [] 186 for x in includes: 187 if isinstance(x, tuple): 188 if not x[1].startswith("bazel-out"): 189 result += [x] 190 else: 191 result += [x] 192 return result 193 194 195def _external_tool(*args) -> ExtractInfo: 196 return lambda file: subprocess.run( 197 [*args, str(file)], check=True, capture_output=True, encoding="utf-8" 198 ).stdout.splitlines() 199 200 201# TODO(usta) use nm as a data dependency 202def nm_differences( 203 left_path: pathlib.Path, right_path: pathlib.Path 204) -> list[str]: 205 """Returns differences in symbol tables. 206 207 Returns the empty list if these files are deemed "similar enough". 208 """ 209 return NmSymbolDiff(_external_tool("nm"), "symbol tables").diff( 210 left_path, right_path 211 ) 212 213 214# TODO(usta) use readelf as a data dependency 215def elf_differences( 216 left_path: pathlib.Path, right_path: pathlib.Path 217) -> list[str]: 218 """Returns differences in elf headers. 219 220 Returns the empty list if these files are deemed "similar enough". 221 222 The given files must exist and must be object (.o) files. 223 """ 224 return ContextDiff(_external_tool("readelf", "-h"), "elf headers").diff( 225 left_path, right_path 226 ) 227 228 229# TODO(usta) use bloaty as a data dependency 230def bloaty_differences( 231 left_path: pathlib.Path, right_path: pathlib.Path 232) -> list[str]: 233 """Returns differences in symbol and section tables. 234 235 Returns the empty list if these files are deemed "similar enough". 236 237 The given files must exist and must be object (.o) files. 238 """ 239 return _bloaty_differences(left_path, right_path) 240 241 242# TODO(usta) use bloaty as a data dependency 243def bloaty_differences_compileunits( 244 left_path: pathlib.Path, right_path: pathlib.Path 245) -> list[str]: 246 """Returns differences in symbol and section tables. 247 248 Returns the empty list if these files are deemed "similar enough". 249 250 The given files must exist and must be object (.o) files. 251 """ 252 return _bloaty_differences(left_path, right_path, True) 253 254 255# TODO(usta) use bloaty as a data dependency 256def _bloaty_differences( 257 left_path: pathlib.Path, right_path: pathlib.Path, debug=False 258) -> list[str]: 259 symbols = BloatyDiff( 260 "symbol tables", "symbols", has_debug_symbols=debug 261 ).diff(left_path, right_path) 262 sections = BloatyDiff( 263 "section tables", "sections", has_debug_symbols=debug 264 ).diff(left_path, right_path) 265 segments = BloatyDiff( 266 "segment tables", "segments", has_debug_symbols=debug 267 ).diff(left_path, right_path) 268 return symbols + sections + segments 269