1# Copyright 2016 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Manage various config files.""" 16 17import configparser 18import functools 19import itertools 20import os 21import shlex 22import sys 23 24_path = os.path.realpath(__file__ + '/../..') 25if sys.path[0] != _path: 26 sys.path.insert(0, _path) 27del _path 28 29# pylint: disable=wrong-import-position 30import rh.hooks 31import rh.shell 32 33 34class Error(Exception): 35 """Base exception class.""" 36 37 38class ValidationError(Error): 39 """Config file has unknown sections/keys or other values.""" 40 41 42# Sentinel so we can handle None-vs-unspecified. 43_UNSET = object() 44 45 46class RawConfigParser(configparser.RawConfigParser): 47 """Like RawConfigParser but with some default helpers.""" 48 49 # pylint doesn't like it when we extend the API. 50 # pylint: disable=arguments-differ 51 52 def options(self, section, default=_UNSET): 53 """Return the options in |section|. 54 55 Args: 56 section: The section to look up. 57 default: What to return if |section| does not exist. 58 """ 59 try: 60 return configparser.RawConfigParser.options(self, section) 61 except configparser.NoSectionError: 62 if default is not _UNSET: 63 return default 64 raise 65 66 def items(self, section=_UNSET, default=_UNSET): 67 """Return a list of (key, value) tuples for the options in |section|.""" 68 if section is _UNSET: 69 return super().items() 70 71 try: 72 return configparser.RawConfigParser.items(self, section) 73 except configparser.NoSectionError: 74 if default is not _UNSET: 75 return default 76 raise 77 78 79class PreUploadConfig(object): 80 """A single (abstract) config used for `repo upload` hooks.""" 81 82 CUSTOM_HOOKS_SECTION = 'Hook Scripts' 83 BUILTIN_HOOKS_SECTION = 'Builtin Hooks' 84 BUILTIN_HOOKS_OPTIONS_SECTION = 'Builtin Hooks Options' 85 BUILTIN_HOOKS_EXCLUDE_SECTION = 'Builtin Hooks Exclude Paths' 86 TOOL_PATHS_SECTION = 'Tool Paths' 87 OPTIONS_SECTION = 'Options' 88 VALID_SECTIONS = { 89 CUSTOM_HOOKS_SECTION, 90 BUILTIN_HOOKS_SECTION, 91 BUILTIN_HOOKS_OPTIONS_SECTION, 92 BUILTIN_HOOKS_EXCLUDE_SECTION, 93 TOOL_PATHS_SECTION, 94 OPTIONS_SECTION, 95 } 96 97 OPTION_IGNORE_MERGED_COMMITS = 'ignore_merged_commits' 98 VALID_OPTIONS = {OPTION_IGNORE_MERGED_COMMITS} 99 100 def __init__(self, config=None, source=None): 101 """Initialize. 102 103 Args: 104 config: A configparse.ConfigParser instance. 105 source: Where this config came from. This is used in error messages to 106 facilitate debugging. It is not necessarily a valid path. 107 """ 108 self.config = config if config else RawConfigParser() 109 self.source = source 110 if config: 111 self._validate() 112 113 @property 114 def custom_hooks(self): 115 """List of custom hooks to run (their keys/names).""" 116 return self.config.options(self.CUSTOM_HOOKS_SECTION, []) 117 118 def custom_hook(self, hook): 119 """The command to execute for |hook|.""" 120 return shlex.split(self.config.get( 121 self.CUSTOM_HOOKS_SECTION, hook, fallback='')) 122 123 @property 124 def builtin_hooks(self): 125 """List of all enabled builtin hooks (their keys/names).""" 126 return [k for k, v in self.config.items(self.BUILTIN_HOOKS_SECTION, ()) 127 if rh.shell.boolean_shell_value(v, None)] 128 129 def builtin_hook_option(self, hook): 130 """The options to pass to |hook|.""" 131 return shlex.split(self.config.get( 132 self.BUILTIN_HOOKS_OPTIONS_SECTION, hook, fallback='')) 133 134 def builtin_hook_exclude_paths(self, hook): 135 """List of paths for which |hook| should not be executed.""" 136 return shlex.split(self.config.get( 137 self.BUILTIN_HOOKS_EXCLUDE_SECTION, hook, fallback='')) 138 139 @property 140 def tool_paths(self): 141 """List of all tool paths.""" 142 return dict(self.config.items(self.TOOL_PATHS_SECTION, ())) 143 144 def callable_custom_hooks(self): 145 """Yield a CallableHook for each hook to be executed.""" 146 scope = rh.hooks.ExclusionScope([]) 147 for hook in self.custom_hooks: 148 options = rh.hooks.HookOptions(hook, 149 self.custom_hook(hook), 150 self.tool_paths) 151 func = functools.partial(rh.hooks.check_custom, options=options) 152 yield rh.hooks.CallableHook(hook, func, scope) 153 154 def callable_builtin_hooks(self): 155 """Yield a CallableHook for each hook to be executed.""" 156 scope = rh.hooks.ExclusionScope([]) 157 for hook in self.builtin_hooks: 158 options = rh.hooks.HookOptions(hook, 159 self.builtin_hook_option(hook), 160 self.tool_paths) 161 func = functools.partial(rh.hooks.BUILTIN_HOOKS[hook], 162 options=options) 163 scope = rh.hooks.ExclusionScope( 164 self.builtin_hook_exclude_paths(hook)) 165 yield rh.hooks.CallableHook(hook, func, scope) 166 167 @property 168 def ignore_merged_commits(self): 169 """Whether to skip hooks for merged commits.""" 170 return rh.shell.boolean_shell_value( 171 self.config.get(self.OPTIONS_SECTION, 172 self.OPTION_IGNORE_MERGED_COMMITS, fallback=None), 173 False) 174 175 def update(self, preupload_config): 176 """Merge settings from |preupload_config| into ourself.""" 177 self.config.read_dict(preupload_config.config) 178 179 def _validate(self): 180 """Run consistency checks on the config settings.""" 181 config = self.config 182 183 # Reject unknown sections. 184 bad_sections = set(config.sections()) - self.VALID_SECTIONS 185 if bad_sections: 186 raise ValidationError( 187 f'{self.source}: unknown sections: {bad_sections}') 188 189 # Reject blank custom hooks. 190 for hook in self.custom_hooks: 191 if not config.get(self.CUSTOM_HOOKS_SECTION, hook): 192 raise ValidationError( 193 f'{self.source}: custom hook "{hook}" cannot be blank') 194 195 # Reject unknown builtin hooks. 196 valid_builtin_hooks = set(rh.hooks.BUILTIN_HOOKS.keys()) 197 if config.has_section(self.BUILTIN_HOOKS_SECTION): 198 hooks = set(config.options(self.BUILTIN_HOOKS_SECTION)) 199 bad_hooks = hooks - valid_builtin_hooks 200 if bad_hooks: 201 raise ValidationError( 202 f'{self.source}: unknown builtin hooks: {bad_hooks}') 203 elif config.has_section(self.BUILTIN_HOOKS_OPTIONS_SECTION): 204 raise ValidationError('Builtin hook options specified, but missing ' 205 'builtin hook settings') 206 207 if config.has_section(self.BUILTIN_HOOKS_OPTIONS_SECTION): 208 hooks = set(config.options(self.BUILTIN_HOOKS_OPTIONS_SECTION)) 209 bad_hooks = hooks - valid_builtin_hooks 210 if bad_hooks: 211 raise ValidationError( 212 f'{self.source}: unknown builtin hook options: {bad_hooks}') 213 214 # Verify hooks are valid shell strings. 215 for hook in self.custom_hooks: 216 try: 217 self.custom_hook(hook) 218 except ValueError as e: 219 raise ValidationError( 220 f'{self.source}: hook "{hook}" command line is invalid: {e}' 221 ) from e 222 223 # Verify hook options are valid shell strings. 224 for hook in self.builtin_hooks: 225 try: 226 self.builtin_hook_option(hook) 227 except ValueError as e: 228 raise ValidationError( 229 f'{self.source}: hook options "{hook}" are invalid: {e}' 230 ) from e 231 232 # Reject unknown tools. 233 valid_tools = set(rh.hooks.TOOL_PATHS.keys()) 234 if config.has_section(self.TOOL_PATHS_SECTION): 235 tools = set(config.options(self.TOOL_PATHS_SECTION)) 236 bad_tools = tools - valid_tools 237 if bad_tools: 238 raise ValidationError( 239 f'{self.source}: unknown tools: {bad_tools}') 240 241 # Reject unknown options. 242 if config.has_section(self.OPTIONS_SECTION): 243 options = set(config.options(self.OPTIONS_SECTION)) 244 bad_options = options - self.VALID_OPTIONS 245 if bad_options: 246 raise ValidationError( 247 f'{self.source}: unknown options: {bad_options}') 248 249 250class PreUploadFile(PreUploadConfig): 251 """A single config (file) used for `repo upload` hooks. 252 253 This is an abstract class that requires subclasses to define the FILENAME 254 constant. 255 256 Attributes: 257 path: The path of the file. 258 """ 259 FILENAME = None 260 261 def __init__(self, path): 262 """Initialize. 263 264 Args: 265 path: The config file to load. 266 """ 267 super().__init__(source=path) 268 269 self.path = path 270 try: 271 self.config.read(path) 272 except configparser.ParsingError as e: 273 raise ValidationError(f'{path}: {e}') from e 274 275 self._validate() 276 277 @classmethod 278 def from_paths(cls, paths): 279 """Search for files within paths that matches the class FILENAME. 280 281 Args: 282 paths: List of directories to look for config files. 283 284 Yields: 285 For each valid file found, an instance is created and returned. 286 """ 287 for path in paths: 288 path = os.path.join(path, cls.FILENAME) 289 if os.path.exists(path): 290 yield cls(path) 291 292 293class LocalPreUploadFile(PreUploadFile): 294 """A single config file for a project (PREUPLOAD.cfg).""" 295 FILENAME = 'PREUPLOAD.cfg' 296 297 def _validate(self): 298 super()._validate() 299 300 # Reject Exclude Paths section for local config. 301 if self.config.has_section(self.BUILTIN_HOOKS_EXCLUDE_SECTION): 302 raise ValidationError( 303 f'{self.path}: [{self.BUILTIN_HOOKS_EXCLUDE_SECTION}] is not ' 304 'valid in local files') 305 306 307class GlobalPreUploadFile(PreUploadFile): 308 """A single config file for a repo (GLOBAL-PREUPLOAD.cfg).""" 309 FILENAME = 'GLOBAL-PREUPLOAD.cfg' 310 311 312class PreUploadSettings(PreUploadConfig): 313 """Settings for `repo upload` hooks. 314 315 This encompasses multiple config files and provides the final (merged) 316 settings for a particular project. 317 """ 318 319 def __init__(self, paths=('',), global_paths=()): 320 """Initialize. 321 322 All the config files found will be merged together in order. 323 324 Args: 325 paths: The directories to look for config files. 326 global_paths: The directories to look for global config files. 327 """ 328 super().__init__() 329 330 self.paths = [] 331 for config in itertools.chain( 332 GlobalPreUploadFile.from_paths(global_paths), 333 LocalPreUploadFile.from_paths(paths)): 334 self.paths.append(config.path) 335 self.update(config) 336 337 338 # We validated configs in isolation, now do one final pass altogether. 339 self.source = '{' + '|'.join(self.paths) + '}' 340 self._validate() 341