1// Copyright 2021 The Bazel Authors. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package main 16 17import ( 18 "archive/tar" 19 "archive/zip" 20 "compress/gzip" 21 "context" 22 "crypto/sha256" 23 "encoding/hex" 24 "errors" 25 "fmt" 26 "io" 27 "os" 28 "path" 29 "path/filepath" 30 "strings" 31 "sync" 32) 33 34var repoRootState = struct { 35 once sync.Once 36 dir string 37 err error 38}{} 39 40// repoRoot returns the workspace root directory. If this program was invoked 41// with 'bazel run', repoRoot returns the BUILD_WORKSPACE_DIRECTORY environment 42// variable. Otherwise, repoRoot walks up the directory tree and finds a 43// WORKSPACE file. 44func repoRoot() (string, error) { 45 repoRootState.once.Do(func() { 46 if wsDir := os.Getenv("BUILD_WORKSPACE_DIRECTORY"); wsDir != "" { 47 repoRootState.dir = wsDir 48 return 49 } 50 dir, err := os.Getwd() 51 if err != nil { 52 repoRootState.err = err 53 return 54 } 55 for { 56 _, err := os.Stat(filepath.Join(dir, "WORKSPACE")) 57 if err == nil { 58 repoRootState.dir = dir 59 return 60 } 61 if err != os.ErrNotExist { 62 repoRootState.err = err 63 return 64 } 65 parent := filepath.Dir(dir) 66 if parent == dir { 67 repoRootState.err = errors.New("could not find workspace directory") 68 return 69 } 70 dir = parent 71 } 72 }) 73 return repoRootState.dir, repoRootState.err 74} 75 76// extractArchive extracts a zip or tar.gz archive opened in f, into the 77// directory dir, stripping stripPrefix from each entry before extraction. 78// name is the name of the archive, used for error reporting. 79func extractArchive(f *os.File, name, dir, stripPrefix string) (err error) { 80 if strings.HasSuffix(name, ".zip") { 81 return extractZip(f, name, dir, stripPrefix) 82 } 83 if strings.HasSuffix(name, ".tar.gz") { 84 zr, err := gzip.NewReader(f) 85 if err != nil { 86 return fmt.Errorf("extracting %s: %w", name, err) 87 } 88 defer func() { 89 if cerr := zr.Close(); err == nil && cerr != nil { 90 err = cerr 91 } 92 }() 93 return extractTar(zr, name, dir, stripPrefix) 94 } 95 return fmt.Errorf("could not determine archive format from extension: %s", name) 96} 97 98func extractZip(zf *os.File, name, dir, stripPrefix string) (err error) { 99 stripPrefix += "/" 100 fi, err := zf.Stat() 101 if err != nil { 102 return err 103 } 104 defer func() { 105 if err != nil { 106 err = fmt.Errorf("extracting zip %s: %w", name, err) 107 } 108 }() 109 110 zr, err := zip.NewReader(zf, fi.Size()) 111 if err != nil { 112 return err 113 } 114 115 extractFile := func(f *zip.File) (err error) { 116 defer func() { 117 if err != nil { 118 err = fmt.Errorf("extracting %s: %w", f.Name, err) 119 } 120 }() 121 outPath, err := extractedPath(dir, stripPrefix, f.Name) 122 if err != nil { 123 return err 124 } 125 if strings.HasSuffix(f.Name, "/") { 126 return os.MkdirAll(outPath, 0777) 127 } 128 r, err := f.Open() 129 if err != nil { 130 return err 131 } 132 defer r.Close() 133 parent := filepath.Dir(outPath) 134 if err := os.MkdirAll(parent, 0777); err != nil { 135 return err 136 } 137 w, err := os.Create(outPath) 138 if err != nil { 139 return err 140 } 141 defer func() { 142 if cerr := w.Close(); err == nil && cerr != nil { 143 err = cerr 144 } 145 }() 146 _, err = io.Copy(w, r) 147 return err 148 } 149 150 for _, f := range zr.File { 151 if err := extractFile(f); err != nil { 152 return err 153 } 154 } 155 156 return nil 157} 158 159func extractTar(r io.Reader, name, dir, stripPrefix string) (err error) { 160 defer func() { 161 if err != nil { 162 err = fmt.Errorf("extracting tar %s: %w", name, err) 163 } 164 }() 165 166 tr := tar.NewReader(r) 167 extractFile := func(hdr *tar.Header) (err error) { 168 outPath, err := extractedPath(dir, stripPrefix, hdr.Name) 169 if err != nil { 170 return err 171 } 172 switch hdr.Typeflag { 173 case tar.TypeDir: 174 return os.MkdirAll(outPath, 0777) 175 case tar.TypeReg: 176 w, err := os.Create(outPath) 177 if err != nil { 178 return err 179 } 180 defer func() { 181 if cerr := w.Close(); err == nil && cerr != nil { 182 err = cerr 183 } 184 }() 185 _, err = io.Copy(w, tr) 186 return err 187 default: 188 return fmt.Errorf("unsupported file type %x: %q", hdr.Typeflag, hdr.Name) 189 } 190 } 191 192 stripPrefix += "/" 193 for { 194 hdr, err := tr.Next() 195 if err == io.EOF { 196 break 197 } else if err != nil { 198 return err 199 } 200 if err := extractFile(hdr); err != nil { 201 return err 202 } 203 } 204 return nil 205} 206 207// extractedPath returns the file path that a file in an archive should be 208// extracted to. It verifies that entryName starts with stripPrefix and does not 209// point outside dir. 210func extractedPath(dir, stripPrefix, entryName string) (string, error) { 211 if !strings.HasPrefix(entryName, stripPrefix) { 212 return "", fmt.Errorf("entry does not start with prefix %s: %q", stripPrefix, entryName) 213 } 214 entryName = entryName[len(stripPrefix):] 215 if entryName == "" { 216 return dir, nil 217 } 218 if path.IsAbs(entryName) { 219 return "", fmt.Errorf("entry has an absolute path: %q", entryName) 220 } 221 if strings.HasPrefix(entryName, "../") { 222 return "", fmt.Errorf("entry refers to something outside the archive: %q", entryName) 223 } 224 entryName = strings.TrimSuffix(entryName, "/") 225 if path.Clean(entryName) != entryName { 226 return "", fmt.Errorf("entry does not have a clean path: %q", entryName) 227 } 228 return filepath.Join(dir, entryName), nil 229} 230 231// copyDir recursively copies a directory tree. 232func copyDir(toDir, fromDir string) error { 233 if err := os.MkdirAll(toDir, 0777); err != nil { 234 return err 235 } 236 return filepath.Walk(fromDir, func(path string, fi os.FileInfo, err error) error { 237 if err != nil { 238 return err 239 } 240 rel, _ := filepath.Rel(fromDir, path) 241 if rel == "." { 242 return nil 243 } 244 outPath := filepath.Join(toDir, rel) 245 if fi.IsDir() { 246 return os.Mkdir(outPath, 0777) 247 } else { 248 return copyFile(outPath, path) 249 } 250 }) 251} 252 253func copyFile(toFile, fromFile string) (err error) { 254 r, err := os.Open(fromFile) 255 if err != nil { 256 return err 257 } 258 defer r.Close() 259 w, err := os.Create(toFile) 260 if err != nil { 261 return err 262 } 263 defer func() { 264 if cerr := w.Close(); err == nil && cerr != nil { 265 err = cerr 266 } 267 }() 268 _, err = io.Copy(w, r) 269 return err 270} 271 272func sha256SumFile(name string) (string, error) { 273 r, err := os.Open(name) 274 if err != nil { 275 return "", err 276 } 277 defer r.Close() 278 h := sha256.New() 279 if _, err := io.Copy(h, r); err != nil { 280 return "", err 281 } 282 sum := h.Sum(nil) 283 return hex.EncodeToString(sum), nil 284} 285 286// copyFileToMirror uploads a file to the GCS bucket backing mirror.bazel.build. 287// gsutil must be installed, and the user must be authenticated with 288// 'gcloud auth login' and be allowed to write files to the bucket. 289func copyFileToMirror(ctx context.Context, path, fileName string) (err error) { 290 dest := "gs://bazel-mirror/" + path 291 defer func() { 292 if err != nil { 293 err = fmt.Errorf("copying file %s to %s: %w", fileName, dest, err) 294 } 295 }() 296 297 // This function shells out to gsutil instead of using 298 // cloud.google.com/go/storage because that package has a million 299 // dependencies. 300 return runForError(ctx, ".", "gsutil", "cp", "-n", fileName, dest) 301} 302