1#!/usr/bin/env python3
2# Copyright 2014 Google Inc.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#    * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#    * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#    * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30"""Parse a DEPS file and git checkout all of the dependencies.
31"""
32
33EXTRA_HELP = """
34Environment Variables:
35  GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of
36  ['git', 'git.exe', 'git.bat'] in your default path.
37
38  GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset,
39  will use the file ../DEPS relative to this script's directory.
40
41  GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages.
42
43Git Config:
44  To disable syncing of a single repository:
45      cd path/to/repository
46      git config sync-deps.disable true
47
48  To re-enable sync:
49      cd path/to/repository
50      git config --unset sync-deps.disable
51"""
52
53
54import argparse
55import os
56import re
57import subprocess
58import sys
59import threading
60from builtins import bytes
61
62def git_executable():
63  """Find the git executable.
64
65  Returns:
66      A triple:
67        A string suitable for passing to subprocess functions, or None.
68        The major version number
69        The minor version number
70  """
71  envgit = os.environ.get('GIT_EXECUTABLE')
72  searchlist = ['git', 'git.exe', 'git.bat']
73  if envgit:
74    searchlist.insert(0, envgit)
75  with open(os.devnull, 'w') as devnull:
76    for git in searchlist:
77      major=None
78      minor=None
79      try:
80        version_info = subprocess.check_output([git, '--version']).decode('utf-8')
81        match = re.search("^git version (\d+)\.(\d+)",version_info)
82        print("Using {}".format(version_info))
83        if match:
84          major = int(match.group(1))
85          minor = int(match.group(2))
86        else:
87          continue
88      except (OSError,):
89        continue
90      return (git,major,minor)
91  return (None,0,0)
92
93
94DEFAULT_DEPS_PATH = os.path.normpath(
95  os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS'))
96
97def get_deps_os_str(deps_file):
98  parsed_deps = parse_file_to_dict(deps_file)
99  parts = []
100  if 'deps_os' in parsed_deps:
101    for deps_os in parsed_deps['deps_os']:
102      parts.append(' [{}]]'.format(deps_os))
103  return "\n".join(parts)
104
105def looks_like_raw_commit(commit):
106  return re.match('^[a-f0-9]{40}$', commit) is not None
107
108def git_repository_sync_is_disabled(git, directory):
109  try:
110    disable = subprocess.check_output(
111      [git, 'config', 'sync-deps.disable'], cwd=directory)
112    return disable.lower().strip() in ['true', '1', 'yes', 'on']
113  except subprocess.CalledProcessError:
114    return False
115
116
117def is_git_toplevel(git, directory):
118  """Return true iff the directory is the top level of a Git repository.
119
120  Args:
121    git (string) the git executable
122
123    directory (string) the path into which the repository
124              is expected to be checked out.
125  """
126  try:
127    toplevel = subprocess.check_output(
128      [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip()
129    return os.path.realpath(bytes(directory, 'utf8')) == os.path.realpath(toplevel)
130  except subprocess.CalledProcessError:
131    return False
132
133
134def status(directory, checkoutable):
135  def truncate(s, length):
136    return s if len(s) <= length else '...' + s[-(length - 3):]
137  dlen = 36
138  directory = truncate(directory, dlen)
139  checkoutable = truncate(checkoutable, 40)
140  sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable))
141
142
143def git_checkout_to_directory(git, repo, checkoutable, directory, verbose, treeless):
144  """Checkout (and clone if needed) a Git repository.
145
146  Args:
147    git (string) the git executable
148
149    repo (string) the location of the repository, suitable
150         for passing to `git clone`.
151
152    checkoutable (string) a tag, branch, or commit, suitable for
153                 passing to `git checkout`
154
155    directory (string) the path into which the repository
156              should be checked out.
157
158    verbose (boolean): emit status info to stdout
159
160    treeless (boolean): when true, clone without any trees.
161
162  Raises an exception if any calls to git fail.
163  """
164  if not os.path.isdir(directory):
165    # Use blobless or treeless checkouts for faster downloads.
166    # This defers some work to checkout time.
167    # https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/
168    filter = ['--filter=tree:0'] if treeless else ['--filter=blob:none']
169    # If the thing to check out looks like a tag (and not like a commit),
170    # then limit the checkout to that branch.
171    branch = [] if looks_like_raw_commit(checkoutable) else ['--branch={}'.format(checkoutable)]
172    subprocess.check_call(
173        [git, 'clone', '--quiet', '--single-branch'] + filter + branch + [repo, directory])
174
175  if not is_git_toplevel(git, directory):
176    # if the directory exists, but isn't a git repo, you will modify
177    # the parent repostory, which isn't what you want.
178    sys.stdout.write('%s\n  IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory)
179    return
180
181  # Check to see if this repo is disabled.  Quick return.
182  if git_repository_sync_is_disabled(git, directory):
183    sys.stdout.write('%s\n  SYNC IS DISABLED.\n' % directory)
184    return
185
186  with open(os.devnull, 'w') as devnull:
187    # If this fails, we will fetch before trying again.  Don't spam user
188    # with error information.
189    if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable],
190                            cwd=directory, stderr=devnull):
191      # if this succeeds, skip slow `git fetch`.
192      if verbose:
193        status(directory, checkoutable)  # Success.
194      return
195
196  # If the repo has changed, always force use of the correct repo.
197  # If origin already points to repo, this is a quick no-op.
198  subprocess.check_call(
199      [git, 'remote', 'set-url', 'origin', repo], cwd=directory)
200
201  subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory)
202
203  subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory)
204
205  if verbose:
206    status(directory, checkoutable)  # Success.
207
208
209def parse_file_to_dict(path):
210  dictionary = {}
211  contents = open(path).read()
212  # Need to convert Var() to vars[], so that the DEPS is actually Python. Var()
213  # comes from Autoroller using gclient which has a slightly different DEPS
214  # format.
215  contents = re.sub(r"Var\((.*?)\)", r"vars[\1]", contents)
216  exec(contents, dictionary)
217  return dictionary
218
219
220def git_sync_deps(deps_file_path, command_line_os_requests, verbose, treeless):
221  """Grab dependencies, with optional platform support.
222
223  Args:
224    deps_file_path (string) Path to the DEPS file.
225
226    command_line_os_requests (list of strings) Can be empty list.
227        List of strings that should each be a key in the deps_os
228        dictionary in the DEPS file.
229
230    verbose (boolean): emit status info to stdout
231
232    treeless (boolean): when true, clone as treeless instead of blobless
233
234  Raises git Exceptions.
235  """
236  (git,git_major,git_minor) = git_executable()
237  assert git
238
239  # --filter=tree:0 is available in git 2.20 and later
240  if (git_major,git_minor) < (2,20):
241    print("disabling --treeless: git is older than v2.20")
242    treeless = False
243
244  deps_file_directory = os.path.dirname(deps_file_path)
245  deps_file = parse_file_to_dict(deps_file_path)
246  dependencies = deps_file['deps'].copy()
247  os_specific_dependencies = deps_file.get('deps_os', dict())
248  if 'all' in command_line_os_requests:
249    for value in list(os_specific_dependencies.values()):
250      dependencies.update(value)
251  else:
252    for os_name in command_line_os_requests:
253      # Add OS-specific dependencies
254      if os_name in os_specific_dependencies:
255        dependencies.update(os_specific_dependencies[os_name])
256  for directory in dependencies:
257    for other_dir in dependencies:
258      if directory.startswith(other_dir + '/'):
259        raise Exception('%r is parent of %r' % (other_dir, directory))
260  list_of_arg_lists = []
261  for directory in sorted(dependencies):
262    if '@' in dependencies[directory]:
263      repo, checkoutable = dependencies[directory].split('@', 1)
264    else:
265      raise Exception("please specify commit or tag")
266
267    relative_directory = os.path.join(deps_file_directory, directory)
268
269    list_of_arg_lists.append(
270      (git, repo, checkoutable, relative_directory, verbose, treeless))
271
272  multithread(git_checkout_to_directory, list_of_arg_lists)
273
274  for directory in deps_file.get('recursedeps', []):
275    recursive_path = os.path.join(deps_file_directory, directory, 'DEPS')
276    git_sync_deps(recursive_path, command_line_os_requests, verbose)
277
278
279def multithread(function, list_of_arg_lists):
280  # for args in list_of_arg_lists:
281  #   function(*args)
282  # return
283  threads = []
284  for args in list_of_arg_lists:
285    thread = threading.Thread(None, function, None, args)
286    thread.start()
287    threads.append(thread)
288  for thread in threads:
289    thread.join()
290
291
292def main(argv):
293  argparser = argparse.ArgumentParser(
294          prog = "git-sync-deps",
295          description = "Checkout git-based dependencies as specified by the DEPS file",
296          add_help=False # Because we want to print deps_os with -h option
297          )
298  argparser.add_argument("--help", "-h",
299                         action='store_true',
300                         help="show this help message and exit")
301  argparser.add_argument("--deps",
302                         default = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH),
303                         help="location of the the DEPS file")
304  argparser.add_argument("--verbose",
305                         default=not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False)),
306                         action='store_true',
307                         help="be verbose: print status messages")
308  argparser.add_argument("--treeless",
309                         default=False,
310                         action='store_true',
311                         help="""
312    Clone repos without trees (--filter=tree:0).
313    This is the fastest option for a build machine,
314    when you only need a single commit.
315    Defers getting objects until checking out a commit.
316
317    The default is to clone with trees but without blobs.
318
319    Only takes effect if using git 2.20 or later.
320
321    See https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/
322                              """)
323  argparser.add_argument("os_requests",nargs="*",
324                         help="OS requests, as keys in the deps_os dictionariy in the DEPS file")
325
326  args = argparser.parse_args()
327  if args.help:
328    print(argparser.format_help())
329    print(EXTRA_HELP)
330    print(get_deps_os_str(args.deps))
331    return 0
332
333  git_sync_deps(args.deps, args.os_requests, args.verbose, args.treeless)
334  return 0
335
336
337if __name__ == '__main__':
338  exit(main(sys.argv[1:]))
339