# Copyright 2017 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. from recipe_engine import recipe_api UPLOAD_ATTEMPTS = 5 class GSUtilApi(recipe_api.RecipeApi): def __call__(self, step_name, *args): """Run gsutil with the given args. This assumes there exists an executable called gsutil on the PATH. This probably only works for Linux/Mac, but those are the only hosts that we try to upload to GCS from anyway. """ return self.m.step(step_name, cmd=['gsutil'] + list(args)) def cp(self, name, src, dst, extra_gsutil_args=None, extra_args=None, multithread=False): """Attempt to upload or download files to/from Google Cloud Storage (GCS). Args: name: string. Will be used to fill out the step name. src: string. Absolute path for a local file or gcs file (e.g. gs://...) dst: string. Same as src. extra_gsutil_args: optional list of args to be passed to gsutil before the cp command. extra_args: optional list of args to be passed to gsutil cp. e.g. [-Z] asks all files be compressed with gzip after upload and before download. multi_thread: if the -m argument should be used to copy multiple items at once (e.g. gsutil -m cp foo* gs://bar/dir) If the operation fails, it will be retried multiple times. """ cmd = [] if extra_gsutil_args: cmd.extend(extra_gsutil_args) if multithread: cmd.append('-m') cmd.append('cp') if extra_args: cmd.extend(extra_args) cmd.extend([src, dst]) name = 'upload %s' % name for i in range(UPLOAD_ATTEMPTS): step_name = name if i > 0: step_name += ' (attempt %d)' % (i+1) try: self(step_name, *cmd) break except self.m.step.StepFailure: if i == UPLOAD_ATTEMPTS - 1: raise