1# Copyright 2020 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4"""Methods related to querying builder information from Buildbucket.""" 5 6from __future__ import print_function 7 8import concurrent.futures 9import json 10import logging 11import os 12import subprocess 13from typing import Any, Dict, Iterable, List, Optional, Set, Tuple 14 15import six 16 17from unexpected_passes_common import constants 18from unexpected_passes_common import data_types 19 20TESTING_BUILDBOT_DIR = os.path.realpath( 21 os.path.join(os.path.dirname(__file__), '..', 'buildbot')) 22INTERNAL_TESTING_BUILDBOT_DIR = os.path.realpath( 23 os.path.join(constants.SRC_INTERNAL_DIR, 'testing', 'buildbot')) 24 25# Public JSON files for internal builders, which should be treated as internal. 26PUBLIC_INTERNAL_JSON_FILES = { 27 'chrome.json', 28 'chrome.gpu.fyi.json', 29 'chromeos.preuprev.json', 30 'internal.chrome.fyi.json', 31 'internal.chromeos.fyi.json', 32} 33 34AUTOGENERATED_JSON_KEY = 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT' 35 36FakeBuildersDict = Dict[data_types.BuilderEntry, Set[data_types.BuilderEntry]] 37 38_registered_instance = None 39 40 41def GetInstance() -> 'Builders': 42 return _registered_instance 43 44 45def RegisterInstance(instance: 'Builders') -> None: 46 global _registered_instance 47 assert _registered_instance is None 48 assert isinstance(instance, Builders) 49 _registered_instance = instance 50 51 52def ClearInstance() -> None: 53 global _registered_instance 54 _registered_instance = None 55 56 57class Builders(): 58 def __init__(self, suite: Optional[str], include_internal_builders: bool): 59 """ 60 Args: 61 suite: A string containing particular suite of interest if applicable, 62 such as for Telemetry-based tests. Can be None if not applicable. 63 include_internal_builders: A boolean indicating whether data from 64 internal builders should be used in addition to external ones. 65 """ 66 self._authenticated = False 67 self._suite = suite 68 self._include_internal_builders = include_internal_builders 69 70 def _ProcessJsonFiles(self, files: List[str], are_internal_files: bool, 71 builder_type: str) -> Set[data_types.BuilderEntry]: 72 builders = set() 73 for filepath in files: 74 if not filepath.endswith('.json'): 75 continue 76 if builder_type == constants.BuilderTypes.CI: 77 if 'tryserver' in filepath: 78 continue 79 elif builder_type == constants.BuilderTypes.TRY: 80 if 'tryserver' not in filepath: 81 continue 82 with open(filepath, encoding='utf-8') as f: 83 buildbot_json = json.load(f) 84 # Skip any JSON files that don't contain builder information. 85 if AUTOGENERATED_JSON_KEY not in buildbot_json: 86 continue 87 88 for builder, test_map in buildbot_json.items(): 89 # Remove the auto-generated comments. 90 if 'AAAA' in builder: 91 continue 92 # Filter out any builders that don't run the suite in question. 93 if not self._BuilderRunsTestOfInterest(test_map): 94 continue 95 builders.add( 96 data_types.BuilderEntry(builder, builder_type, are_internal_files)) 97 return builders 98 99 def GetCiBuilders(self) -> Set[data_types.BuilderEntry]: 100 """Gets the set of CI builders to query. 101 102 Returns: 103 A set of data_types.BuilderEntry, each element corresponding to either a 104 public or internal CI builder to query results from. 105 """ 106 ci_builders = set() 107 108 logging.info('Getting CI builders') 109 ci_builders = self._ProcessJsonFiles(_GetPublicJsonFiles(), False, 110 constants.BuilderTypes.CI) 111 if self._include_internal_builders: 112 ci_builders |= self._ProcessJsonFiles(_GetInternalJsonFiles(), True, 113 constants.BuilderTypes.CI) 114 115 logging.debug('Got %d CI builders after trimming: %s', len(ci_builders), 116 ', '.join([b.name for b in ci_builders])) 117 return ci_builders 118 119 def _BuilderRunsTestOfInterest(self, test_map: Dict[str, Any]) -> bool: 120 """Determines if a builder runs a test of interest. 121 122 Args: 123 test_map: A dict, corresponding to a builder's test spec from a 124 //testing/buildbot JSON file. 125 suite: A string containing particular suite of interest if applicable, 126 such as for Telemetry-based tests. Can be None if not applicable. 127 128 Returns: 129 True if |test_map| contains a test of interest, else False. 130 """ 131 raise NotImplementedError() 132 133 def GetTryBuilders(self, ci_builders: Iterable[data_types.BuilderEntry] 134 ) -> Set[data_types.BuilderEntry]: 135 """Gets the set of try builders to query. 136 137 A try builder is of interest if it mirrors a builder in |ci_builders| or is 138 a dedicated try builder. 139 140 Args: 141 ci_builders: An iterable of data_types.BuilderEntry, each element being a 142 public or internal CI builder that results will be/were queried from. 143 144 Returns: 145 A set of data_types.BuilderEntry, each element being the name of a 146 Chromium try builder to query results from. 147 """ 148 logging.info('Getting try builders') 149 dedicated_try_builders = self._ProcessJsonFiles([ 150 os.path.join(TESTING_BUILDBOT_DIR, f) 151 for f in os.listdir(TESTING_BUILDBOT_DIR) 152 ], False, constants.BuilderTypes.TRY) 153 if self._include_internal_builders: 154 dedicated_try_builders |= self._ProcessJsonFiles([ 155 os.path.join(INTERNAL_TESTING_BUILDBOT_DIR, f) 156 for f in os.listdir(INTERNAL_TESTING_BUILDBOT_DIR) 157 ], True, constants.BuilderTypes.TRY) 158 mirrored_builders = set() 159 no_output_builders = set() 160 161 with concurrent.futures.ThreadPoolExecutor( 162 max_workers=os.cpu_count()) as pool: 163 results_iter = pool.map(self._GetMirroredBuildersForCiBuilder, 164 ci_builders) 165 for (builders, found_mirror) in results_iter: 166 if found_mirror: 167 mirrored_builders |= builders 168 else: 169 no_output_builders |= builders 170 171 if no_output_builders: 172 raise RuntimeError( 173 'Did not get Buildbucket output for the following builders. They may ' 174 'need to be added to the GetFakeCiBuilders or ' 175 'GetNonChromiumBuilders .\n%s' % 176 '\n'.join([b.name for b in no_output_builders])) 177 logging.debug('Got %d try builders: %s', len(mirrored_builders), 178 mirrored_builders) 179 return dedicated_try_builders | mirrored_builders 180 181 def _GetMirroredBuildersForCiBuilder( 182 self, ci_builder: data_types.BuilderEntry 183 ) -> Tuple[Set[data_types.BuilderEntry], bool]: 184 """Gets the set of try builders that mirror a CI builder. 185 186 Args: 187 ci_builder: A data_types.BuilderEntry for a public or internal CI builder. 188 189 Returns: 190 A tuple (builders, found_mirror). |builders| is a set of 191 data_types.BuilderEntry, either the set of try builders that mirror 192 |ci_builder| or |ci_builder|, depending on the value of |found_mirror|. 193 |found_mirror| is True if mirrors were actually found, in which case 194 |builders| contains the try builders. Otherwise, |found_mirror| is False 195 and |builders| contains |ci_builder|. 196 """ 197 mirrored_builders = set() 198 if ci_builder in self.GetNonChromiumBuilders(): 199 logging.debug('%s is a non-Chromium CI builder', ci_builder.name) 200 return mirrored_builders, True 201 202 fake_builders = self.GetFakeCiBuilders() 203 if ci_builder in fake_builders: 204 mirrored_builders |= fake_builders[ci_builder] 205 logging.debug('%s is a fake CI builder mirrored by %s', ci_builder.name, 206 ', '.join(b.name for b in fake_builders[ci_builder])) 207 return mirrored_builders, True 208 209 bb_output = self._GetBuildbucketOutputForCiBuilder(ci_builder) 210 if not bb_output: 211 mirrored_builders.add(ci_builder) 212 logging.debug('Did not get Buildbucket output for builder %s', 213 ci_builder.name) 214 return mirrored_builders, False 215 216 bb_json = json.loads(bb_output) 217 mirrored = bb_json.get('output', {}).get('properties', 218 {}).get('mirrored_builders', []) 219 # The mirror names from Buildbucket include the group separated by :, e.g. 220 # tryserver.chromium.android:gpu-fyi-try-android-m-nexus-5x-64, so only grab 221 # the builder name. 222 for mirror in mirrored: 223 split = mirror.split(':') 224 assert len(split) == 2 225 logging.debug('Got mirrored builder for %s: %s', ci_builder.name, 226 split[1]) 227 mirrored_builders.add( 228 data_types.BuilderEntry(split[1], constants.BuilderTypes.TRY, 229 ci_builder.is_internal_builder)) 230 return mirrored_builders, True 231 232 def _GetBuildbucketOutputForCiBuilder(self, 233 ci_builder: data_types.BuilderEntry 234 ) -> str: 235 # Ensure the user is logged in to bb. 236 if not self._authenticated: 237 try: 238 with open(os.devnull, 'w', newline='', encoding='utf-8') as devnull: 239 subprocess.check_call(['bb', 'auth-info'], 240 stdout=devnull, 241 stderr=devnull) 242 except subprocess.CalledProcessError as e: 243 six.raise_from( 244 RuntimeError('You are not logged into bb - run `bb auth-login`.'), 245 e) 246 self._authenticated = True 247 # Split out for ease of testing. 248 # Get the Buildbucket ID for the most recent completed build for a builder. 249 p = subprocess.Popen([ 250 'bb', 251 'ls', 252 '-id', 253 '-1', 254 '-status', 255 'ended', 256 '%s/ci/%s' % (ci_builder.project, ci_builder.name), 257 ], 258 stdout=subprocess.PIPE) 259 # Use the ID to get the most recent build. 260 bb_output = subprocess.check_output([ 261 'bb', 262 'get', 263 '-A', 264 '-json', 265 ], 266 stdin=p.stdout, 267 text=True) 268 return bb_output 269 270 def GetIsolateNames(self) -> Set[str]: 271 """Gets the isolate names that are relevant to this implementation. 272 273 Returns: 274 A set of strings, each element being the name of an isolate of interest. 275 """ 276 raise NotImplementedError() 277 278 def GetFakeCiBuilders(self) -> FakeBuildersDict: 279 """Gets a mapping of fake CI builders to their mirrored trybots. 280 281 Returns: 282 A dict of data_types.BuilderEntry -> set(data_types.BuilderEntry). Each 283 key is a CI builder that doesn't actually exist and each value is a set of 284 try builders that mirror the CI builder but do exist. 285 """ 286 raise NotImplementedError() 287 288 def GetNonChromiumBuilders(self) -> Set[data_types.BuilderEntry]: 289 """Gets the builders that are not actual Chromium builders. 290 291 These are listed in the Chromium //testing/buildbot files, but aren't under 292 the Chromium Buildbucket project. These don't use the same recipes as 293 Chromium builders, and thus don't have the list of trybot mirrors. 294 295 Returns: 296 A set of data_types.BuilderEntry, each element being a non-Chromium 297 builder. 298 """ 299 raise NotImplementedError() 300 301 302def _GetPublicJsonFiles() -> List[str]: 303 return [ 304 os.path.join(TESTING_BUILDBOT_DIR, f) 305 for f in os.listdir(TESTING_BUILDBOT_DIR) 306 if f not in PUBLIC_INTERNAL_JSON_FILES 307 ] 308 309 310def _GetInternalJsonFiles() -> List[str]: 311 internal_files = [ 312 os.path.join(INTERNAL_TESTING_BUILDBOT_DIR, f) 313 for f in os.listdir(INTERNAL_TESTING_BUILDBOT_DIR) 314 ] 315 public_internal_files = [ 316 os.path.join(TESTING_BUILDBOT_DIR, f) 317 for f in os.listdir(TESTING_BUILDBOT_DIR) 318 if f in PUBLIC_INTERNAL_JSON_FILES 319 ] 320 return internal_files + public_internal_files 321