1#!/usr/bin/env vpython3 2 3# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 4# 5# Use of this source code is governed by a BSD-style license 6# that can be found in the LICENSE file in the root of the source 7# tree. An additional intellectual property rights grant can be found 8# in the file PATENTS. All contributing project authors may 9# be found in the AUTHORS file in the root of the source tree. 10"""Script to automatically roll dependencies in the WebRTC DEPS file.""" 11 12 13import argparse 14import base64 15import collections 16import logging 17import os 18import re 19import subprocess 20import sys 21import urllib.request 22 23 24def FindSrcDirPath(): 25 """Returns the abs path to the src/ dir of the project.""" 26 src_dir = os.path.dirname(os.path.abspath(__file__)) 27 while os.path.basename(src_dir) != 'src': 28 src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) 29 return src_dir 30 31 32# Skip these dependencies (list without solution name prefix). 33DONT_AUTOROLL_THESE = [ 34 'src/examples/androidtests/third_party/gradle', 35] 36 37# These dependencies are missing in chromium/src/DEPS, either unused or already 38# in-tree. For instance, src/base is a part of the Chromium source git repo, 39# but we pull it through a subtree mirror, so therefore it isn't listed in 40# Chromium's deps but it is in ours. 41WEBRTC_ONLY_DEPS = [ 42 'src/base', 43 'src/build', 44 'src/buildtools', 45 'src/ios', 46 'src/testing', 47 'src/third_party', 48 'src/third_party/gtest-parallel', 49 'src/third_party/pipewire/linux-amd64', 50 'src/tools', 51] 52 53WEBRTC_URL = 'https://webrtc.googlesource.com/src' 54CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src' 55CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s' 56CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' 57CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s' 58 59COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') 60CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'([-0-9a-z]+)\'$') 61ROLL_BRANCH_NAME = 'roll_chromium_revision' 62 63SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 64CHECKOUT_SRC_DIR = FindSrcDirPath() 65CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir)) 66 67# Copied from tools/android/roll/android_deps/.../BuildConfigGenerator.groovy. 68ANDROID_DEPS_START = r'=== ANDROID_DEPS Generated Code Start ===' 69ANDROID_DEPS_END = r'=== ANDROID_DEPS Generated Code End ===' 70# Location of automically gathered android deps. 71ANDROID_DEPS_PATH = 'src/third_party/android_deps/' 72 73NOTIFY_EMAIL = '[email protected]' 74 75sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) 76import find_depot_tools 77 78find_depot_tools.add_depot_tools_to_path() 79 80CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py' 81CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools', 82 'clang', 'scripts', 'update.py') 83 84DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') 85ChangedDep = collections.namedtuple('ChangedDep', 86 'path url current_rev new_rev') 87CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages') 88ChangedCipdPackage = collections.namedtuple( 89 'ChangedCipdPackage', 'path package current_version new_version') 90 91ChromiumRevisionUpdate = collections.namedtuple('ChromiumRevisionUpdate', 92 ('current_chromium_rev ' 93 'new_chromium_rev ')) 94 95 96class RollError(Exception): 97 pass 98 99 100def StrExpansion(): 101 return lambda str_value: str_value 102 103 104def VarLookup(local_scope): 105 return lambda var_name: local_scope['vars'][var_name] 106 107 108def ParseDepsDict(deps_content): 109 local_scope = {} 110 global_scope = { 111 'Str': StrExpansion(), 112 'Var': VarLookup(local_scope), 113 'deps_os': {}, 114 } 115 exec(deps_content, global_scope, local_scope) 116 return local_scope 117 118 119def ParseLocalDepsFile(filename): 120 with open(filename, 'rb') as f: 121 deps_content = f.read().decode('utf-8') 122 return ParseDepsDict(deps_content) 123 124 125def ParseCommitPosition(commit_message): 126 for line in reversed(commit_message.splitlines()): 127 m = COMMIT_POSITION_RE.match(line.strip()) 128 if m: 129 return int(m.group(1)) 130 logging.error('Failed to parse commit position id from:\n%s\n', 131 commit_message) 132 sys.exit(-1) 133 134 135def _RunCommand(command, 136 working_dir=None, 137 ignore_exit_code=False, 138 extra_env=None, 139 input_data=None): 140 """Runs a command and returns the output from that command. 141 142 If the command fails (exit code != 0), the function will exit the process. 143 144 Returns: 145 A tuple containing the stdout and stderr outputs as strings. 146 """ 147 working_dir = working_dir or CHECKOUT_SRC_DIR 148 logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) 149 env = os.environ.copy() 150 if extra_env: 151 assert all(isinstance(value, str) for value in list(extra_env.values())) 152 logging.debug('extra env: %s', extra_env) 153 env.update(extra_env) 154 p = subprocess.Popen(command, 155 stdin=subprocess.PIPE, 156 stdout=subprocess.PIPE, 157 stderr=subprocess.PIPE, 158 env=env, 159 cwd=working_dir, 160 universal_newlines=True) 161 std_output, err_output = p.communicate(input_data) 162 p.stdout.close() 163 p.stderr.close() 164 if not ignore_exit_code and p.returncode != 0: 165 logging.error('Command failed: %s\n' 166 'stdout:\n%s\n' 167 'stderr:\n%s\n', ' '.join(command), std_output, err_output) 168 sys.exit(p.returncode) 169 return std_output, err_output 170 171 172def _GetBranches(): 173 """Returns a tuple of active,branches. 174 175 The 'active' is the name of the currently active branch and 'branches' is a 176 list of all branches. 177 """ 178 lines = _RunCommand(['git', 'branch'])[0].split('\n') 179 branches = [] 180 active = '' 181 for line in lines: 182 if '*' in line: 183 # The assumption is that the first char will always be the '*'. 184 active = line[1:].strip() 185 branches.append(active) 186 else: 187 branch = line.strip() 188 if branch: 189 branches.append(branch) 190 return active, branches 191 192 193def _ReadGitilesContent(url): 194 # Download and decode BASE64 content until 195 # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed. 196 base64_content = ReadUrlContent(url + '?format=TEXT') 197 return base64.b64decode(base64_content[0]).decode('utf-8') 198 199 200def ReadRemoteCrFile(path_below_src, revision): 201 """Reads a remote Chromium file of a specific revision. 202 203 Args: 204 path_below_src: A path to the target file relative to src dir. 205 revision: Revision to read. 206 Returns: 207 A string with file content. 208 """ 209 return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % 210 (revision, path_below_src)) 211 212 213def ReadRemoteCrCommit(revision): 214 """Reads a remote Chromium commit message. Returns a string.""" 215 return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision) 216 217 218def ReadUrlContent(url): 219 """Connect to a remote host and read the contents. 220 221 Args: 222 url: URL to connect to. 223 Returns: 224 A list of lines. 225 """ 226 conn = urllib.request.urlopen(url) 227 try: 228 return conn.readlines() 229 except IOError as e: 230 logging.exception('Error connecting to %s. Error: %s', url, e) 231 raise 232 finally: 233 conn.close() 234 235 236def GetMatchingDepsEntries(depsentry_dict, dir_path): 237 """Gets all deps entries matching the provided path. 238 239 This list may contain more than one DepsEntry object. 240 Example: dir_path='src/testing' would give results containing both 241 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's 242 DEPS. 243 Example 2: dir_path='src/build' should return 'src/build' but not 244 'src/buildtools'. 245 246 Returns: 247 A list of DepsEntry objects. 248 """ 249 result = [] 250 for path, depsentry in list(depsentry_dict.items()): 251 if path == dir_path: 252 result.append(depsentry) 253 else: 254 parts = path.split('/') 255 if all(part == parts[i] for i, part in enumerate(dir_path.split('/'))): 256 result.append(depsentry) 257 return result 258 259 260def BuildDepsentryDict(deps_dict): 261 """Builds a dict of paths to DepsEntry objects from a raw deps dict.""" 262 result = {} 263 264 def AddDepsEntries(deps_subdict): 265 for path, dep in list(deps_subdict.items()): 266 if path in result: 267 continue 268 if not isinstance(dep, dict): 269 dep = {'url': dep} 270 if dep.get('dep_type') == 'cipd': 271 result[path] = CipdDepsEntry(path, dep['packages']) 272 else: 273 if '@' not in dep['url']: 274 continue 275 url, revision = dep['url'].split('@') 276 result[path] = DepsEntry(path, url, revision) 277 278 AddDepsEntries(deps_dict['deps']) 279 for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: 280 AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) 281 return result 282 283 284def _FindChangedCipdPackages(path, old_pkgs, new_pkgs): 285 old_pkgs_names = {p['package'] for p in old_pkgs} 286 new_pkgs_names = {p['package'] for p in new_pkgs} 287 pkgs_equal = (old_pkgs_names == new_pkgs_names) 288 added_pkgs = [p for p in new_pkgs_names if p not in old_pkgs_names] 289 removed_pkgs = [p for p in old_pkgs_names if p not in new_pkgs_names] 290 291 assert pkgs_equal, ('Old: %s\n New: %s.\nYou need to do a manual roll ' 292 'and remove/add entries in DEPS so the old and new ' 293 'list match.\nMost likely, you should add \"%s\" and ' 294 'remove \"%s\"' % 295 (old_pkgs, new_pkgs, added_pkgs, removed_pkgs)) 296 297 for old_pkg in old_pkgs: 298 for new_pkg in new_pkgs: 299 old_version = old_pkg['version'] 300 new_version = new_pkg['version'] 301 if (old_pkg['package'] == new_pkg['package'] 302 and old_version != new_version): 303 logging.debug('Roll dependency %s to %s', path, new_version) 304 yield ChangedCipdPackage(path, old_pkg['package'], old_version, 305 new_version) 306 307 308def _FindNewDeps(old, new): 309 """ Gather dependencies only in `new` and return corresponding paths. """ 310 old_entries = set(BuildDepsentryDict(old)) 311 new_entries = set(BuildDepsentryDict(new)) 312 return [ 313 path for path in new_entries - old_entries 314 if path not in DONT_AUTOROLL_THESE 315 ] 316 317 318def FindAddedDeps(webrtc_deps, new_cr_deps): 319 """ 320 Calculate new deps entries of interest. 321 322 Ideally, that would mean: only appearing in chromium DEPS 323 but transitively used in WebRTC. 324 325 Since it's hard to compute, we restrict ourselves to a well defined subset: 326 deps sitting in `ANDROID_DEPS_PATH`. 327 Otherwise, assumes that's a Chromium-only dependency. 328 329 Args: 330 webrtc_deps: dict of deps as defined in the WebRTC DEPS file. 331 new_cr_deps: dict of deps as defined in the chromium DEPS file. 332 333 Caveat: Doesn't detect a new package in existing dep. 334 335 Returns: 336 A tuple consisting of: 337 A list of paths added dependencies sitting in `ANDROID_DEPS_PATH`. 338 A list of paths for other added dependencies. 339 """ 340 all_added_deps = _FindNewDeps(webrtc_deps, new_cr_deps) 341 generated_android_deps = [ 342 path for path in all_added_deps if path.startswith(ANDROID_DEPS_PATH) 343 ] 344 other_deps = [ 345 path for path in all_added_deps if path not in generated_android_deps 346 ] 347 return generated_android_deps, other_deps 348 349 350def FindRemovedDeps(webrtc_deps, new_cr_deps): 351 """ 352 Calculate obsolete deps entries. 353 354 Ideally, that would mean: no more appearing in chromium DEPS 355 and not used in WebRTC. 356 357 Since it's hard to compute: 358 1/ We restrict ourselves to a well defined subset: 359 deps sitting in `ANDROID_DEPS_PATH`. 360 2/ We rely on existing behavior of CalculateChangeDeps. 361 I.e. Assumes non-CIPD dependencies are WebRTC-only, don't remove them. 362 363 Args: 364 webrtc_deps: dict of deps as defined in the WebRTC DEPS file. 365 new_cr_deps: dict of deps as defined in the chromium DEPS file. 366 367 Caveat: Doesn't detect a deleted package in existing dep. 368 369 Returns: 370 A tuple consisting of: 371 A list of paths of dependencies removed from `ANDROID_DEPS_PATH`. 372 A list of paths of unexpected disappearing dependencies. 373 """ 374 all_removed_deps = _FindNewDeps(new_cr_deps, webrtc_deps) 375 generated_android_deps = sorted( 376 [path for path in all_removed_deps if path.startswith(ANDROID_DEPS_PATH)]) 377 # Webrtc-only dependencies are handled in CalculateChangedDeps. 378 other_deps = sorted([ 379 path for path in all_removed_deps 380 if path not in generated_android_deps and path not in WEBRTC_ONLY_DEPS 381 ]) 382 return generated_android_deps, other_deps 383 384 385def CalculateChangedDeps(webrtc_deps, new_cr_deps): 386 """ 387 Calculate changed deps entries based on entries defined in the WebRTC DEPS 388 file: 389 - If a shared dependency with the Chromium DEPS file: roll it to the same 390 revision as Chromium (i.e. entry in the new_cr_deps dict) 391 - If it's a Chromium sub-directory, roll it to the HEAD revision (notice 392 this means it may be ahead of the chromium_revision, but generally these 393 should be close). 394 - If it's another DEPS entry (not shared with Chromium), roll it to HEAD 395 unless it's configured to be skipped. 396 397 Returns: 398 A list of ChangedDep objects representing the changed deps. 399 """ 400 result = [] 401 webrtc_entries = BuildDepsentryDict(webrtc_deps) 402 new_cr_entries = BuildDepsentryDict(new_cr_deps) 403 for path, webrtc_deps_entry in list(webrtc_entries.items()): 404 if path in DONT_AUTOROLL_THESE: 405 continue 406 cr_deps_entry = new_cr_entries.get(path) 407 if cr_deps_entry: 408 assert type(cr_deps_entry) is type(webrtc_deps_entry) 409 410 if isinstance(cr_deps_entry, CipdDepsEntry): 411 result.extend( 412 _FindChangedCipdPackages(path, webrtc_deps_entry.packages, 413 cr_deps_entry.packages)) 414 continue 415 416 # Use the revision from Chromium's DEPS file. 417 new_rev = cr_deps_entry.revision 418 assert webrtc_deps_entry.url == cr_deps_entry.url, ( 419 'WebRTC DEPS entry %s has a different URL %s than Chromium %s.' % 420 (path, webrtc_deps_entry.url, cr_deps_entry.url)) 421 else: 422 if isinstance(webrtc_deps_entry, DepsEntry): 423 # Use the HEAD of the deps repo. 424 stdout, _ = _RunCommand( 425 ['git', 'ls-remote', webrtc_deps_entry.url, 'HEAD']) 426 new_rev = stdout.strip().split('\t')[0] 427 else: 428 # The dependency has been removed from chromium. 429 # This is handled by FindRemovedDeps. 430 continue 431 432 # Check if an update is necessary. 433 if webrtc_deps_entry.revision != new_rev: 434 logging.debug('Roll dependency %s to %s', path, new_rev) 435 result.append( 436 ChangedDep(path, webrtc_deps_entry.url, webrtc_deps_entry.revision, 437 new_rev)) 438 return sorted(result) 439 440 441def CalculateChangedClang(new_cr_rev): 442 443 def GetClangRev(lines): 444 for line in lines: 445 match = CLANG_REVISION_RE.match(line) 446 if match: 447 return match.group(1) 448 raise RollError('Could not parse Clang revision!') 449 450 with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'r') as f: 451 current_lines = f.readlines() 452 current_rev = GetClangRev(current_lines) 453 454 new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH, 455 new_cr_rev).splitlines() 456 new_rev = GetClangRev(new_clang_update_py) 457 return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev) 458 459 460def GenerateCommitMessage( 461 rev_update, 462 current_commit_pos, 463 new_commit_pos, 464 changed_deps_list, 465 added_deps_paths=None, 466 removed_deps_paths=None, 467 clang_change=None, 468): 469 current_cr_rev = rev_update.current_chromium_rev[0:10] 470 new_cr_rev = rev_update.new_chromium_rev[0:10] 471 rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) 472 git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) 473 474 commit_msg = [ 475 'Roll chromium_revision %s (%s)\n' % (rev_interval, git_number_interval), 476 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval), 477 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % rev_interval) 478 ] 479 480 def Section(adjective, deps): 481 noun = 'dependency' if len(deps) == 1 else 'dependencies' 482 commit_msg.append('%s %s' % (adjective, noun)) 483 484 if changed_deps_list: 485 Section('Changed', changed_deps_list) 486 487 for c in changed_deps_list: 488 if isinstance(c, ChangedCipdPackage): 489 commit_msg.append('* %s: %s..%s' % 490 (c.path, c.current_version, c.new_version)) 491 else: 492 commit_msg.append('* %s: %s/+log/%s..%s' % 493 (c.path, c.url, c.current_rev[0:10], c.new_rev[0:10])) 494 495 if added_deps_paths: 496 Section('Added', added_deps_paths) 497 commit_msg.extend('* %s' % p for p in added_deps_paths) 498 499 if removed_deps_paths: 500 Section('Removed', removed_deps_paths) 501 commit_msg.extend('* %s' % p for p in removed_deps_paths) 502 503 if any([changed_deps_list, added_deps_paths, removed_deps_paths]): 504 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') 505 commit_msg.append('DEPS diff: %s\n' % change_url) 506 else: 507 commit_msg.append('No dependencies changed.') 508 509 if clang_change and clang_change.current_rev != clang_change.new_rev: 510 commit_msg.append('Clang version changed %s:%s' % 511 (clang_change.current_rev, clang_change.new_rev)) 512 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 513 CLANG_UPDATE_SCRIPT_URL_PATH) 514 commit_msg.append('Details: %s\n' % change_url) 515 else: 516 commit_msg.append('No update to Clang.\n') 517 518 commit_msg.append('BUG=None') 519 return '\n'.join(commit_msg) 520 521 522def UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content): 523 """Update the DEPS file with the new revision.""" 524 525 with open(deps_filename, 'rb') as deps_file: 526 deps_content = deps_file.read().decode('utf-8') 527 528 # Update the chromium_revision variable. 529 deps_content = deps_content.replace(rev_update.current_chromium_rev, 530 rev_update.new_chromium_rev) 531 532 # Add and remove dependencies. For now: only generated android deps. 533 # Since gclient cannot add or remove deps, we on the fact that 534 # these android deps are located in one place we can copy/paste. 535 deps_re = re.compile(ANDROID_DEPS_START + '.*' + ANDROID_DEPS_END, re.DOTALL) 536 new_deps = deps_re.search(new_cr_content) 537 old_deps = deps_re.search(deps_content) 538 if not new_deps or not old_deps: 539 faulty = 'Chromium' if not new_deps else 'WebRTC' 540 raise RollError('Was expecting to find "%s" and "%s"\n' 541 'in %s DEPS' % 542 (ANDROID_DEPS_START, ANDROID_DEPS_END, faulty)) 543 deps_content = deps_re.sub(new_deps.group(0), deps_content) 544 545 with open(deps_filename, 'wb') as deps_file: 546 deps_file.write(deps_content.encode('utf-8')) 547 548 # Update each individual DEPS entry. 549 for dep in changed_deps: 550 local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path) 551 if not os.path.isdir(local_dep_dir): 552 raise RollError( 553 'Cannot find local directory %s. Either run\n' 554 'gclient sync --deps=all\n' 555 'or make sure the .gclient file for your solution contains all ' 556 'platforms in the target_os list, i.e.\n' 557 'target_os = ["android", "unix", "mac", "ios", "win"];\n' 558 'Then run "gclient sync" again.' % local_dep_dir) 559 if isinstance(dep, ChangedCipdPackage): 560 package = dep.package.format() # Eliminate double curly brackets 561 update = '%s:%s@%s' % (dep.path, package, dep.new_version) 562 else: 563 update = '%s@%s' % (dep.path, dep.new_rev) 564 _RunCommand(['gclient', 'setdep', '--revision', update], 565 working_dir=CHECKOUT_SRC_DIR) 566 567 568def _IsTreeClean(): 569 stdout, _ = _RunCommand(['git', 'status', '--porcelain']) 570 if len(stdout) == 0: 571 return True 572 573 logging.error('Dirty/unversioned files:\n%s', stdout) 574 return False 575 576 577def _EnsureUpdatedMainBranch(dry_run): 578 current_branch = _RunCommand(['git', 'rev-parse', '--abbrev-ref', 579 'HEAD'])[0].splitlines()[0] 580 if current_branch != 'main': 581 logging.error('Please checkout the main branch and re-run this script.') 582 if not dry_run: 583 sys.exit(-1) 584 585 logging.info('Updating main branch...') 586 _RunCommand(['git', 'pull']) 587 588 589def _CreateRollBranch(dry_run): 590 logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME) 591 if not dry_run: 592 _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) 593 594 595def _RemovePreviousRollBranch(dry_run): 596 active_branch, branches = _GetBranches() 597 if active_branch == ROLL_BRANCH_NAME: 598 active_branch = 'main' 599 if ROLL_BRANCH_NAME in branches: 600 logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME) 601 if not dry_run: 602 _RunCommand(['git', 'checkout', active_branch]) 603 _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) 604 605 606def _LocalCommit(commit_msg, dry_run): 607 logging.info('Committing changes locally.') 608 if not dry_run: 609 _RunCommand(['git', 'add', '--update', '.']) 610 _RunCommand(['git', 'commit', '-m', commit_msg]) 611 612 613def ChooseCQMode(skip_cq, cq_over, current_commit_pos, new_commit_pos): 614 if skip_cq: 615 return 0 616 if (new_commit_pos - current_commit_pos) < cq_over: 617 return 1 618 return 2 619 620 621def _GetCcRecipients(changed_deps_list): 622 """Returns a list of emails to notify based on the changed deps list. 623 """ 624 cc_recipients = [] 625 for c in changed_deps_list: 626 if 'libvpx' in c.path or 'libaom' in c.path: 627 cc_recipients.append('[email protected]') 628 cc_recipients.append('[email protected]') 629 return cc_recipients 630 631 632def _UploadCL(commit_queue_mode, add_cc=None): 633 """Upload the committed changes as a changelist to Gerrit. 634 635 commit_queue_mode: 636 - 2: Submit to commit queue. 637 - 1: Run trybots but do not submit to CQ. 638 - 0: Skip CQ, upload only. 639 640 add_cc: A list of email addresses to add as CC recipients. 641 """ 642 cc_recipients = [NOTIFY_EMAIL] 643 if add_cc: 644 cc_recipients.extend(add_cc) 645 cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks'] 646 if commit_queue_mode >= 2: 647 logging.info('Sending the CL to the CQ...') 648 cmd.extend(['-o', 'label=Bot-Commit+1']) 649 cmd.extend(['-o', 'label=Commit-Queue+2']) 650 cmd.extend(['--send-mail', '--cc', ','.join(cc_recipients)]) 651 elif commit_queue_mode >= 1: 652 logging.info('Starting CQ dry run...') 653 cmd.extend(['-o', 'label=Commit-Queue+1']) 654 extra_env = { 655 'EDITOR': 'true', 656 'SKIP_GCE_AUTH_FOR_GIT': '1', 657 } 658 stdout, stderr = _RunCommand(cmd, extra_env=extra_env) 659 logging.debug('Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s', 660 stdout, stderr) 661 662 663def GetRollRevisionRanges(opts, webrtc_deps): 664 current_cr_rev = webrtc_deps['vars']['chromium_revision'] 665 new_cr_rev = opts.revision 666 if not new_cr_rev: 667 stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) 668 head_rev = stdout.strip().split('\t')[0] 669 logging.info('No revision specified. Using HEAD: %s', head_rev) 670 new_cr_rev = head_rev 671 672 return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev) 673 674 675def main(): 676 p = argparse.ArgumentParser() 677 p.add_argument('--clean', 678 action='store_true', 679 default=False, 680 help='Removes any previous local roll branch.') 681 p.add_argument('-r', 682 '--revision', 683 help=('Chromium Git revision to roll to. Defaults to the ' 684 'Chromium HEAD revision if omitted.')) 685 p.add_argument('--dry-run', 686 action='store_true', 687 default=False, 688 help=('Calculate changes and modify DEPS, but don\'t create ' 689 'any local branch, commit, upload CL or send any ' 690 'tryjobs.')) 691 p.add_argument('-i', 692 '--ignore-unclean-workdir', 693 action='store_true', 694 default=False, 695 help=('Ignore if the current branch is not main or if there ' 696 'are uncommitted changes (default: %(default)s).')) 697 grp = p.add_mutually_exclusive_group() 698 grp.add_argument('--skip-cq', 699 action='store_true', 700 default=False, 701 help='Skip sending the CL to the CQ (default: %(default)s)') 702 grp.add_argument('--cq-over', 703 type=int, 704 default=1, 705 help=('Commit queue dry run if the revision difference ' 706 'is below this number (default: %(default)s)')) 707 p.add_argument('-v', 708 '--verbose', 709 action='store_true', 710 default=False, 711 help='Be extra verbose in printing of log messages.') 712 opts = p.parse_args() 713 714 if opts.verbose: 715 logging.basicConfig(level=logging.DEBUG) 716 else: 717 logging.basicConfig(level=logging.INFO) 718 719 if not opts.ignore_unclean_workdir and not _IsTreeClean(): 720 logging.error('Please clean your local checkout first.') 721 return 1 722 723 if opts.clean: 724 _RemovePreviousRollBranch(opts.dry_run) 725 726 if not opts.ignore_unclean_workdir: 727 _EnsureUpdatedMainBranch(opts.dry_run) 728 729 deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') 730 webrtc_deps = ParseLocalDepsFile(deps_filename) 731 732 rev_update = GetRollRevisionRanges(opts, webrtc_deps) 733 734 current_commit_pos = ParseCommitPosition( 735 ReadRemoteCrCommit(rev_update.current_chromium_rev)) 736 new_commit_pos = ParseCommitPosition( 737 ReadRemoteCrCommit(rev_update.new_chromium_rev)) 738 739 new_cr_content = ReadRemoteCrFile('DEPS', rev_update.new_chromium_rev) 740 new_cr_deps = ParseDepsDict(new_cr_content) 741 changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps) 742 # Discard other deps, assumed to be chromium-only dependencies. 743 new_generated_android_deps, _ = FindAddedDeps(webrtc_deps, new_cr_deps) 744 removed_generated_android_deps, other_deps = FindRemovedDeps( 745 webrtc_deps, new_cr_deps) 746 if other_deps: 747 raise RollError('WebRTC DEPS entries are missing from Chromium: %s.\n' 748 'Remove them or add them to either ' 749 'WEBRTC_ONLY_DEPS or DONT_AUTOROLL_THESE.' % other_deps) 750 clang_change = CalculateChangedClang(rev_update.new_chromium_rev) 751 commit_msg = GenerateCommitMessage( 752 rev_update, 753 current_commit_pos, 754 new_commit_pos, 755 changed_deps, 756 added_deps_paths=new_generated_android_deps, 757 removed_deps_paths=removed_generated_android_deps, 758 clang_change=clang_change) 759 logging.debug('Commit message:\n%s', commit_msg) 760 761 _CreateRollBranch(opts.dry_run) 762 if not opts.dry_run: 763 UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content) 764 if _IsTreeClean(): 765 logging.info("No DEPS changes detected, skipping CL creation.") 766 else: 767 _LocalCommit(commit_msg, opts.dry_run) 768 commit_queue_mode = ChooseCQMode(opts.skip_cq, opts.cq_over, 769 current_commit_pos, new_commit_pos) 770 logging.info('Uploading CL...') 771 if not opts.dry_run: 772 _UploadCL(commit_queue_mode, _GetCcRecipients(changed_deps)) 773 return 0 774 775 776if __name__ == '__main__': 777 sys.exit(main()) 778