1#!/usr/bin/python3 2# Copyright 2014 Google Inc. 3# 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7 8"""Parse a DEPS file and git checkout all of the dependencies. 9 10Args: 11 An optional list of deps_os values. 12 13Environment Variables: 14 GIT_EXECUTABLE: path to "git" binary; if unset, will look for git in 15 your default path. 16 17 GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset, 18 will use the file ../DEPS relative to this script's directory. 19 20 GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages. 21 22Git Config: 23 To disable syncing of a single repository: 24 cd path/to/repository 25 git config sync-deps.disable true 26 27 To re-enable sync: 28 cd path/to/repository 29 git config --unset sync-deps.disable 30""" 31 32 33import os 34import subprocess 35import sys 36import threading 37 38 39def git_executable(): 40 """Find the git executable. 41 42 Returns: 43 A string suitable for passing to subprocess functions, or None. 44 """ 45 envgit = os.environ.get('GIT_EXECUTABLE') 46 searchlist = ['git', 'git.bat'] 47 if envgit: 48 searchlist.insert(0, envgit) 49 with open(os.devnull, 'w') as devnull: 50 for git in searchlist: 51 try: 52 subprocess.call([git, '--version'], stdout=devnull) 53 except (OSError,): 54 continue 55 return git 56 return None 57 58 59DEFAULT_DEPS_PATH = os.path.normpath( 60 os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS')) 61 62 63def usage(deps_file_path = None): 64 sys.stderr.write( 65 'Usage: run to grab dependencies, with optional platform support:\n') 66 sys.stderr.write(' %s %s' % (sys.executable, __file__)) 67 if deps_file_path: 68 parsed_deps = parse_file_to_dict(deps_file_path) 69 if 'deps_os' in parsed_deps: 70 for deps_os in parsed_deps['deps_os']: 71 sys.stderr.write(' [%s]' % deps_os) 72 sys.stderr.write('\n\n') 73 sys.stderr.write(__doc__) 74 75 76def git_repository_sync_is_disabled(git, directory): 77 try: 78 disable = subprocess.check_output( 79 [git, 'config', 'sync-deps.disable'], cwd=directory) 80 return disable.lower().strip() in ['true', '1', 'yes', 'on'] 81 except subprocess.CalledProcessError: 82 return False 83 84 85def is_git_toplevel(git, directory): 86 """Return true iff the directory is the top level of a Git repository. 87 88 Args: 89 git (string) the git executable 90 91 directory (string) the path into which the repository 92 is expected to be checked out. 93 """ 94 try: 95 toplevel = subprocess.check_output( 96 [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip() 97 return (os.path.normcase(os.path.realpath(directory)) == 98 os.path.normcase(os.path.realpath(toplevel.decode()))) 99 except subprocess.CalledProcessError: 100 return False 101 102 103def status(directory, commithash, change): 104 def truncate_beginning(s, length): 105 return s if len(s) <= length else '...' + s[-(length-3):] 106 def truncate_end(s, length): 107 return s if len(s) <= length else s[:(length - 3)] + '...' 108 109 dlen = 36 110 directory = truncate_beginning(directory, dlen) 111 commithash = truncate_end(commithash, 40) 112 symbol = '>' if change else '@' 113 sys.stdout.write('%-*s %s %s\n' % (dlen, directory, symbol, commithash)) 114 115 116def git_checkout_to_directory(git, repo, commithash, directory, shallow, verbose): 117 """Checkout (and clone if needed) a Git repository. 118 119 Args: 120 git (string) the git executable 121 122 repo (string) the location of the repository, suitable 123 for passing to `git clone`. 124 125 commithash (string) a commit, suitable for passing to `git checkout` 126 127 directory (string) the path into which the repository 128 should be checked out. 129 130 verbose (boolean) 131 132 Raises an exception if any calls to git fail. 133 """ 134 if not os.path.isdir(directory): 135 subprocess.check_call( 136 [git, 'clone', '--quiet', *(['--depth=1'] if shallow else []), 137 '--no-checkout', repo, directory]) 138 139 if not is_git_toplevel(git, directory): 140 # if the directory exists, but isn't a git repo, you will modify 141 # the parent repository, which isn't what you want. 142 sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory) 143 return 144 145 # Check to see if this repo is disabled. Quick return. 146 if git_repository_sync_is_disabled(git, directory): 147 sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory) 148 return 149 150 with open(os.devnull, 'w') as devnull: 151 # If this fails, we will fetch before trying again. Don't spam user 152 # with error infomation. 153 if 0 == subprocess.call([git, 'checkout', '--quiet', commithash], 154 cwd=directory, stderr=devnull): 155 # if this succeeds, skip slow `git fetch`. 156 if verbose: 157 status(directory, commithash, False) # Success. 158 return 159 160 # If the repo has changed, always force use of the correct repo. 161 # If origin already points to repo, this is a quick no-op. 162 subprocess.check_call( 163 [git, 'remote', 'set-url', 'origin', repo], cwd=directory) 164 165 subprocess.check_call( 166 [git, 'fetch', '--quiet', 167 *(['--depth=1', repo, commithash] if shallow else [])], 168 cwd=directory) 169 170 subprocess.check_call([git, 'checkout', '--quiet', commithash], cwd=directory) 171 172 if verbose: 173 status(directory, commithash, True) # Success. 174 175 176def parse_file_to_dict(path): 177 dictionary = {} 178 with open(path) as f: 179 exec('def Var(x): return vars[x]\n' + f.read(), dictionary) 180 return dictionary 181 182 183def is_sha1_sum(s): 184 """SHA1 sums are 160 bits, encoded as lowercase hexadecimal.""" 185 return len(s) == 40 and all(c in '0123456789abcdef' for c in s) 186 187 188def git_sync_deps(deps_file_path, command_line_os_requests, shallow, verbose): 189 """Grab dependencies, with optional platform support. 190 191 Args: 192 deps_file_path (string) Path to the DEPS file. 193 194 command_line_os_requests (list of strings) Can be empty list. 195 List of strings that should each be a key in the deps_os 196 dictionary in the DEPS file. 197 198 Raises git Exceptions. 199 """ 200 git = git_executable() 201 assert git 202 203 deps_file_directory = os.path.dirname(deps_file_path) 204 deps_file = parse_file_to_dict(deps_file_path) 205 dependencies = deps_file['deps'].copy() 206 os_specific_dependencies = deps_file.get('deps_os', dict()) 207 if 'all' in command_line_os_requests: 208 for value in os_specific_dependencies.itervalues(): 209 dependencies.update(value) 210 else: 211 for os_name in command_line_os_requests: 212 # Add OS-specific dependencies 213 if os_name in os_specific_dependencies: 214 dependencies.update(os_specific_dependencies[os_name]) 215 for directory in dependencies: 216 for other_dir in dependencies: 217 if directory.startswith(other_dir + '/'): 218 raise Exception('%r is parent of %r' % (other_dir, directory)) 219 list_of_arg_lists = [] 220 for directory in sorted(dependencies): 221 if not isinstance(dependencies[directory], str): 222 if verbose: 223 sys.stdout.write( 'Skipping "%s".\n' % directory) 224 continue 225 if '@' in dependencies[directory]: 226 repo, commithash = dependencies[directory].split('@', 1) 227 else: 228 raise Exception("please specify commit") 229 if not is_sha1_sum(commithash): 230 raise Exception("poorly formed commit hash: %r" % commithash) 231 232 relative_directory = os.path.join(deps_file_directory, directory) 233 234 list_of_arg_lists.append( 235 (git, repo, commithash, relative_directory, shallow, verbose)) 236 237 multithread(git_checkout_to_directory, list_of_arg_lists) 238 239 240def multithread(function, list_of_arg_lists): 241 anything_failed = False 242 threads = [] 243 def hook(args): 244 nonlocal anything_failed 245 anything_failed = True 246 threading.excepthook = hook 247 for args in list_of_arg_lists: 248 thread = threading.Thread(None, function, None, args) 249 thread.start() 250 threads.append(thread) 251 for thread in threads: 252 thread.join() 253 if anything_failed: 254 raise Exception("Thread failure detected") 255 256 257def main(argv): 258 deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH) 259 verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False)) 260 skip_emsdk = bool(os.environ.get('GIT_SYNC_DEPS_SKIP_EMSDK', False)) 261 shallow = not ('--deep' in argv) 262 263 if '--help' in argv or '-h' in argv: 264 usage(deps_file_path) 265 return 1 266 267 git_sync_deps(deps_file_path, argv, shallow, verbose) 268 subprocess.check_call( 269 [sys.executable, 270 os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')]) 271 if not skip_emsdk: 272 subprocess.check_call( 273 [sys.executable, 274 os.path.join(os.path.dirname(deps_file_path), 'bin', 'activate-emsdk')]) 275 return 0 276 277 278if __name__ == '__main__': 279 exit(main(sys.argv[1:])) 280