1// Copyright 2020 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package build 16 17// This file contains the functionality to upload data from one location to 18// another. 19 20import ( 21 "fmt" 22 "io/ioutil" 23 "os" 24 "path/filepath" 25 "time" 26 27 "android/soong/ui/metrics" 28 29 "google.golang.org/protobuf/proto" 30 31 upload_proto "android/soong/ui/metrics/upload_proto" 32) 33 34const ( 35 // Used to generate a raw protobuf file that contains information 36 // of the list of metrics files from host to destination storage. 37 uploadPbFilename = ".uploader.pb" 38) 39 40var ( 41 // For testing purpose. 42 tmpDir = ioutil.TempDir 43) 44 45// pruneMetricsFiles iterates the list of paths, checking if a path exist. 46// If a path is a file, it is added to the return list. If the path is a 47// directory, a recursive call is made to add the children files of the 48// path. 49func pruneMetricsFiles(paths []string) []string { 50 var metricsFiles []string 51 for _, p := range paths { 52 fi, err := os.Stat(p) 53 // Some paths passed may not exist. For example, build errors protobuf 54 // file may not exist since the build was successful. 55 if err != nil { 56 continue 57 } 58 59 if fi.IsDir() { 60 if l, err := ioutil.ReadDir(p); err != nil { 61 _, _ = fmt.Fprintf(os.Stderr, "Failed to find files under %s\n", p) 62 } else { 63 files := make([]string, 0, len(l)) 64 for _, fi := range l { 65 files = append(files, filepath.Join(p, fi.Name())) 66 } 67 metricsFiles = append(metricsFiles, pruneMetricsFiles(files)...) 68 } 69 } else { 70 metricsFiles = append(metricsFiles, p) 71 } 72 } 73 return metricsFiles 74} 75 76// UploadMetrics uploads a set of metrics files to a server for analysis. 77// The metrics files are first copied to a temporary directory 78// and the uploader is then executed in the background to allow the user/system 79// to continue working. Soong communicates to the uploader through the 80// upload_proto raw protobuf file. 81func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, paths ...string) { 82 ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics") 83 defer ctx.EndTrace() 84 85 uploader := config.MetricsUploaderApp() 86 if uploader == "" { 87 // If the uploader path was not specified, no metrics shall be uploaded. 88 return 89 } 90 91 // Several of the files might be directories. 92 metricsFiles := pruneMetricsFiles(paths) 93 if len(metricsFiles) == 0 { 94 return 95 } 96 97 // The temporary directory cannot be deleted as the metrics uploader is started 98 // in the background and requires to exist until the operation is done. The 99 // uploader can delete the directory as it is specified in the upload proto. 100 tmpDir, err := tmpDir("", "upload_metrics") 101 if err != nil { 102 ctx.Fatalf("failed to create a temporary directory to store the list of metrics files: %v\n", err) 103 } 104 105 for i, src := range metricsFiles { 106 dst := filepath.Join(tmpDir, filepath.Base(src)) 107 if _, err := copyFile(src, dst); err != nil { 108 ctx.Fatalf("failed to copy %q to %q: %v\n", src, dst, err) 109 } 110 metricsFiles[i] = dst 111 } 112 113 // For platform builds, the branch and target name is hardcoded to specific 114 // values for later extraction of the metrics in the data metrics pipeline. 115 data, err := proto.Marshal(&upload_proto.Upload{ 116 CreationTimestampMs: proto.Uint64(uint64(buildStarted.UnixNano() / int64(time.Millisecond))), 117 CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))), 118 BranchName: proto.String("developer-metrics"), 119 TargetName: proto.String("platform-build-systems-metrics"), 120 MetricsFiles: metricsFiles, 121 DirectoriesToDelete: []string{tmpDir}, 122 }) 123 if err != nil { 124 ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err) 125 } 126 127 pbFile := filepath.Join(tmpDir, uploadPbFilename) 128 if err := ioutil.WriteFile(pbFile, data, 0644); err != nil { 129 ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err) 130 } 131 132 // Start the uploader in the background as it takes several milliseconds to start the uploader 133 // and prepare the metrics for upload. This affects small shell commands like "lunch". 134 cmd := Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile) 135 if simpleOutput { 136 cmd.RunOrFatal() 137 } else { 138 cmd.RunAndStreamOrFatal() 139 } 140} 141