bazel/gcs_mirror/gcs_mirror.go

// Copyright 2022 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// This executable downloads, verifies, and uploads a given file to the Skia infra Bazel mirror.
// Users should have gsutil installed, on the PATH and authenticated.
// There are two modes of use:
//   - Specify a single file via --url and --sha256.
//   - Copy a JSON array of objects (or Starlark list of dictionaries) via standard in.
//
// This should only need to be called when we add new dependencies or update existing ones. Calling
// it with already archived files should be fine - the mirror is a CAS, so the update should be a
// no-op. The files will be uploaded to the mirror with some metadata about where they came from.
package main

import (
	"crypto/sha256"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/flynn/json5"

	"go.skia.org/infra/go/skerr"
)

const (
	gcsBucketAndPrefix = "gs://skia-world-readable/bazel/"
)

func main() {
	var (
		file          = flag.String("file", "", "A local file on disk to upload. --sha256 must be set.")
		url           = flag.String("url", "", "The single url to mirror. --sha256 must be set.")
		sha256Hash    = flag.String("sha256", "", "The sha256sum of the url to mirror. --url must also be set.")
		jsonFromStdin = flag.Bool("json", false, "If set, read JSON from stdin that consists of a list of objects.")
		noSuffix      = flag.Bool("no_suffix", false, "If true, this is presumed to be a binary which needs no suffix (e.g. executable)")
		addSuffix     = flag.String("add_suffix", "", "If set, this will be the suffix of the file uploaded")
	)
	flag.Parse()

	if (*file != "" && *sha256Hash != "") || (*url != "" && *sha256Hash != "") {
		// ok
	} else if *jsonFromStdin {
		// ok
	} else {
		flag.Usage()
		fatalf("Must specify --url/--file and --sha256 or --json")
	}

	workDir, err := os.MkdirTemp("", "bazel_gcs")
	if err != nil {
		fatalf("Could not make temp directory: %s", err)
	}

	if *jsonFromStdin {
		fmt.Println("Waiting for input on std in. Use Ctrl+D (EOF) when done copying and pasting the array.")
		b, err := io.ReadAll(os.Stdin)
		if err != nil {
			fatalf("Error while reading from stdin: %s", err)
		}
		if err := processJSON(workDir, b); err != nil {
			fatalf("Could not process data from stdin: %s", err)
		}
	} else if *url != "" {
		if err := processOneDownload(workDir, *url, *sha256Hash, *addSuffix, *noSuffix); err != nil {
			fatalf("Error while processing entry: %s", err)
		}
		fmt.Printf("https://storage.googleapis.com/skia-world-readable/bazel/%s%s%s\n", *sha256Hash, getSuffix(*url), *addSuffix)
	} else {
		if err := processOneLocalFile(*file, *sha256Hash); err != nil {
			fatalf("Error while processing entry: %s", err)
		}
		fmt.Printf("https://storage.googleapis.com/skia-world-readable/bazel/%s%s\n", *sha256Hash, getSuffix(*file))
	}
}

type urlEntry struct {
	SHA256 string `json:"sha256"`
	URL    string `json:"url"`
}

func processJSON(workDir string, b []byte) error {
	// We generally will be copying a list from Bazel files, written with Starlark (i.e. Pythonish).
	// As a result, we need to turn the almost valid JSON array of objects into actually valid JSON.
	// It is easier to just do string replacing rather than going line by line to remove the
	// troublesome comments.
	cleaned := fixStarlarkComments(b)
	var entries []urlEntry
	if err := json5.Unmarshal([]byte(cleaned), &entries); err != nil {
		return skerr.Wrapf(err, "unmarshalling JSON")
	}
	for _, entry := range entries {
		if err := processOneDownload(workDir, entry.URL, entry.SHA256, "", false); err != nil {
			return skerr.Wrapf(err, "while processing entry: %+v", entry)
		}
	}
	return nil
}

// fixStarlarkComments replaces the Starlark comment symbol (#) with a JSON comment symbol (//).
func fixStarlarkComments(b []byte) string {
	return strings.ReplaceAll(string(b), "#", "//")
}

func processOneDownload(workDir, url, hash, addSuffix string, noSuffix bool) error {
	suf := getSuffix(url) + addSuffix
	if !noSuffix && suf == "" {
		return skerr.Fmt("%s is not a supported file type", url)
	}
	fmt.Printf("Downloading and verifying %s...\n", url)
	res, err := http.Get(url)
	if err != nil {
		return skerr.Wrapf(err, "downloading %s", url)
	}
	contents, err := io.ReadAll(res.Body)
	if err != nil {
		return skerr.Wrapf(err, "reading %s", url)
	}
	if err := res.Body.Close(); err != nil {
		return skerr.Wrapf(err, "after reading %s", url)
	}
	// Verify
	h := sha256.Sum256(contents)
	if actual := hex.EncodeToString(h[:]); actual != hash {
		return skerr.Fmt("Invalid hash of %s. %s != %s", url, actual, hash)
	}
	fmt.Printf("Uploading %s to GCS...\n", url)
	// Write to disk so gsutil can access it
	tmpFile := filepath.Join(workDir, hash+suf)
	if err := os.WriteFile(tmpFile, contents, 0644); err != nil {
		return skerr.Wrapf(err, "writing %d bytes to %s", len(contents), tmpFile)
	}
	// Upload using gsutil (which is assumed to be properly authed)
	cmd := exec.Command("gsutil",
		// Add custom metadata so we can figure out what the unrecognizable file name was created
		// from. Custom metadata values must start with x-goog-meta-
		"-h", "x-goog-meta-original-url:"+url,
		"cp", tmpFile, gcsBucketAndPrefix+hash+suf)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return skerr.Wrapf(cmd.Run(), "uploading %s to GCS", tmpFile)
}

func processOneLocalFile(file, hash string) error {
	file, err := filepath.Abs(file)
	if err != nil {
		return skerr.Wrap(err)
	}
	suf := getSuffix(file)
	if suf == "" {
		return skerr.Fmt("%s is not a supported file type", file)
	}
	contents, err := os.ReadFile(file)
	if err != nil {
		return skerr.Wrapf(err, "reading %s", file)
	}
	// Verify
	h := sha256.Sum256(contents)
	if actual := hex.EncodeToString(h[:]); actual != hash {
		return skerr.Fmt("Invalid hash of %s. %s != %s", file, actual, hash)
	}
	fmt.Printf("Uploading %s to GCS...\n", file)
	// Upload using gsutil (which is assumed to be properly authed)
	cmd := exec.Command("gsutil",
		// Add custom metadata so we can figure out what the unrecognizable file name was created
		// from. Custom metadata values must start with x-goog-meta-
		"-h", "x-goog-meta-original-file:"+file,
		"cp", file, gcsBucketAndPrefix+hash+suf)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return skerr.Wrapf(cmd.Run(), "uploading %s to GCS", file)
}

var supportedSuffixes = []string{".tar.gz", ".tgz", ".tar.xz", ".deb", ".zip"}

// getSuffix returns the filetype suffix of the file if it is in the list of supported suffixes.
// Otherwise, it returns empty string.
func getSuffix(url string) string {
	for _, suf := range supportedSuffixes {
		if strings.HasSuffix(url, suf) {
			return suf
		}
	}
	return ""
}

func fatalf(format string, args ...interface{}) {
	// Ensure there is a newline at the end of the fatal message.
	format = strings.TrimSuffix(format, "\n") + "\n"
	fmt.Printf(format, args...)
	os.Exit(1)
}