1// Copyright 2018 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package modfetch 6 7import ( 8 "bytes" 9 "context" 10 "encoding/json" 11 "errors" 12 "fmt" 13 "io" 14 "io/fs" 15 "math/rand" 16 "os" 17 "path/filepath" 18 "strconv" 19 "strings" 20 "sync" 21 22 "cmd/go/internal/base" 23 "cmd/go/internal/cfg" 24 "cmd/go/internal/gover" 25 "cmd/go/internal/lockedfile" 26 "cmd/go/internal/modfetch/codehost" 27 "cmd/go/internal/par" 28 "cmd/go/internal/robustio" 29 "cmd/internal/telemetry/counter" 30 31 "golang.org/x/mod/module" 32 "golang.org/x/mod/semver" 33) 34 35func cacheDir(ctx context.Context, path string) (string, error) { 36 if err := checkCacheDir(ctx); err != nil { 37 return "", err 38 } 39 enc, err := module.EscapePath(path) 40 if err != nil { 41 return "", err 42 } 43 return filepath.Join(cfg.GOMODCACHE, "cache/download", enc, "/@v"), nil 44} 45 46func CachePath(ctx context.Context, m module.Version, suffix string) (string, error) { 47 if gover.IsToolchain(m.Path) { 48 return "", ErrToolchain 49 } 50 dir, err := cacheDir(ctx, m.Path) 51 if err != nil { 52 return "", err 53 } 54 if !gover.ModIsValid(m.Path, m.Version) { 55 return "", fmt.Errorf("non-semver module version %q", m.Version) 56 } 57 if module.CanonicalVersion(m.Version) != m.Version { 58 return "", fmt.Errorf("non-canonical module version %q", m.Version) 59 } 60 encVer, err := module.EscapeVersion(m.Version) 61 if err != nil { 62 return "", err 63 } 64 return filepath.Join(dir, encVer+"."+suffix), nil 65} 66 67// DownloadDir returns the directory to which m should have been downloaded. 68// An error will be returned if the module path or version cannot be escaped. 69// An error satisfying errors.Is(err, fs.ErrNotExist) will be returned 70// along with the directory if the directory does not exist or if the directory 71// is not completely populated. 72func DownloadDir(ctx context.Context, m module.Version) (string, error) { 73 if gover.IsToolchain(m.Path) { 74 return "", ErrToolchain 75 } 76 if err := checkCacheDir(ctx); err != nil { 77 return "", err 78 } 79 enc, err := module.EscapePath(m.Path) 80 if err != nil { 81 return "", err 82 } 83 if !gover.ModIsValid(m.Path, m.Version) { 84 return "", fmt.Errorf("non-semver module version %q", m.Version) 85 } 86 if module.CanonicalVersion(m.Version) != m.Version { 87 return "", fmt.Errorf("non-canonical module version %q", m.Version) 88 } 89 encVer, err := module.EscapeVersion(m.Version) 90 if err != nil { 91 return "", err 92 } 93 94 // Check whether the directory itself exists. 95 dir := filepath.Join(cfg.GOMODCACHE, enc+"@"+encVer) 96 if fi, err := os.Stat(dir); os.IsNotExist(err) { 97 return dir, err 98 } else if err != nil { 99 return dir, &DownloadDirPartialError{dir, err} 100 } else if !fi.IsDir() { 101 return dir, &DownloadDirPartialError{dir, errors.New("not a directory")} 102 } 103 104 // Check if a .partial file exists. This is created at the beginning of 105 // a download and removed after the zip is extracted. 106 partialPath, err := CachePath(ctx, m, "partial") 107 if err != nil { 108 return dir, err 109 } 110 if _, err := os.Stat(partialPath); err == nil { 111 return dir, &DownloadDirPartialError{dir, errors.New("not completely extracted")} 112 } else if !os.IsNotExist(err) { 113 return dir, err 114 } 115 116 // Check if a .ziphash file exists. It should be created before the 117 // zip is extracted, but if it was deleted (by another program?), we need 118 // to re-calculate it. Note that checkMod will repopulate the ziphash 119 // file if it doesn't exist, but if the module is excluded by checks 120 // through GONOSUMDB or GOPRIVATE, that check and repopulation won't happen. 121 ziphashPath, err := CachePath(ctx, m, "ziphash") 122 if err != nil { 123 return dir, err 124 } 125 if _, err := os.Stat(ziphashPath); os.IsNotExist(err) { 126 return dir, &DownloadDirPartialError{dir, errors.New("ziphash file is missing")} 127 } else if err != nil { 128 return dir, err 129 } 130 return dir, nil 131} 132 133// DownloadDirPartialError is returned by DownloadDir if a module directory 134// exists but was not completely populated. 135// 136// DownloadDirPartialError is equivalent to fs.ErrNotExist. 137type DownloadDirPartialError struct { 138 Dir string 139 Err error 140} 141 142func (e *DownloadDirPartialError) Error() string { return fmt.Sprintf("%s: %v", e.Dir, e.Err) } 143func (e *DownloadDirPartialError) Is(err error) bool { return err == fs.ErrNotExist } 144 145// lockVersion locks a file within the module cache that guards the downloading 146// and extraction of the zipfile for the given module version. 147func lockVersion(ctx context.Context, mod module.Version) (unlock func(), err error) { 148 path, err := CachePath(ctx, mod, "lock") 149 if err != nil { 150 return nil, err 151 } 152 if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil { 153 return nil, err 154 } 155 return lockedfile.MutexAt(path).Lock() 156} 157 158// SideLock locks a file within the module cache that previously guarded 159// edits to files outside the cache, such as go.sum and go.mod files in the 160// user's working directory. 161// If err is nil, the caller MUST eventually call the unlock function. 162func SideLock(ctx context.Context) (unlock func(), err error) { 163 if err := checkCacheDir(ctx); err != nil { 164 return nil, err 165 } 166 167 path := filepath.Join(cfg.GOMODCACHE, "cache", "lock") 168 if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil { 169 return nil, fmt.Errorf("failed to create cache directory: %w", err) 170 } 171 172 return lockedfile.MutexAt(path).Lock() 173} 174 175// A cachingRepo is a cache around an underlying Repo, 176// avoiding redundant calls to ModulePath, Versions, Stat, Latest, and GoMod (but not CheckReuse or Zip). 177// It is also safe for simultaneous use by multiple goroutines 178// (so that it can be returned from Lookup multiple times). 179// It serializes calls to the underlying Repo. 180type cachingRepo struct { 181 path string 182 versionsCache par.ErrCache[string, *Versions] 183 statCache par.ErrCache[string, *RevInfo] 184 latestCache par.ErrCache[struct{}, *RevInfo] 185 gomodCache par.ErrCache[string, []byte] 186 187 once sync.Once 188 initRepo func(context.Context) (Repo, error) 189 r Repo 190} 191 192func newCachingRepo(ctx context.Context, path string, initRepo func(context.Context) (Repo, error)) *cachingRepo { 193 return &cachingRepo{ 194 path: path, 195 initRepo: initRepo, 196 } 197} 198 199func (r *cachingRepo) repo(ctx context.Context) Repo { 200 r.once.Do(func() { 201 var err error 202 r.r, err = r.initRepo(ctx) 203 if err != nil { 204 r.r = errRepo{r.path, err} 205 } 206 }) 207 return r.r 208} 209 210func (r *cachingRepo) CheckReuse(ctx context.Context, old *codehost.Origin) error { 211 return r.repo(ctx).CheckReuse(ctx, old) 212} 213 214func (r *cachingRepo) ModulePath() string { 215 return r.path 216} 217 218func (r *cachingRepo) Versions(ctx context.Context, prefix string) (*Versions, error) { 219 v, err := r.versionsCache.Do(prefix, func() (*Versions, error) { 220 return r.repo(ctx).Versions(ctx, prefix) 221 }) 222 223 if err != nil { 224 return nil, err 225 } 226 return &Versions{ 227 Origin: v.Origin, 228 List: append([]string(nil), v.List...), 229 }, nil 230} 231 232type cachedInfo struct { 233 info *RevInfo 234 err error 235} 236 237func (r *cachingRepo) Stat(ctx context.Context, rev string) (*RevInfo, error) { 238 if gover.IsToolchain(r.path) { 239 // Skip disk cache; the underlying golang.org/toolchain repo is cached instead. 240 return r.repo(ctx).Stat(ctx, rev) 241 } 242 info, err := r.statCache.Do(rev, func() (*RevInfo, error) { 243 file, info, err := readDiskStat(ctx, r.path, rev) 244 if err == nil { 245 return info, err 246 } 247 248 info, err = r.repo(ctx).Stat(ctx, rev) 249 if err == nil { 250 // If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78, 251 // then save the information under the proper version, for future use. 252 if info.Version != rev { 253 file, _ = CachePath(ctx, module.Version{Path: r.path, Version: info.Version}, "info") 254 r.statCache.Do(info.Version, func() (*RevInfo, error) { 255 return info, nil 256 }) 257 } 258 259 if err := writeDiskStat(ctx, file, info); err != nil { 260 fmt.Fprintf(os.Stderr, "go: writing stat cache: %v\n", err) 261 } 262 } 263 return info, err 264 }) 265 if info != nil { 266 copy := *info 267 info = © 268 } 269 return info, err 270} 271 272func (r *cachingRepo) Latest(ctx context.Context) (*RevInfo, error) { 273 if gover.IsToolchain(r.path) { 274 // Skip disk cache; the underlying golang.org/toolchain repo is cached instead. 275 return r.repo(ctx).Latest(ctx) 276 } 277 info, err := r.latestCache.Do(struct{}{}, func() (*RevInfo, error) { 278 info, err := r.repo(ctx).Latest(ctx) 279 280 // Save info for likely future Stat call. 281 if err == nil { 282 r.statCache.Do(info.Version, func() (*RevInfo, error) { 283 return info, nil 284 }) 285 if file, _, err := readDiskStat(ctx, r.path, info.Version); err != nil { 286 writeDiskStat(ctx, file, info) 287 } 288 } 289 290 return info, err 291 }) 292 if info != nil { 293 copy := *info 294 info = © 295 } 296 return info, err 297} 298 299func (r *cachingRepo) GoMod(ctx context.Context, version string) ([]byte, error) { 300 if gover.IsToolchain(r.path) { 301 // Skip disk cache; the underlying golang.org/toolchain repo is cached instead. 302 return r.repo(ctx).GoMod(ctx, version) 303 } 304 text, err := r.gomodCache.Do(version, func() ([]byte, error) { 305 file, text, err := readDiskGoMod(ctx, r.path, version) 306 if err == nil { 307 // Note: readDiskGoMod already called checkGoMod. 308 return text, nil 309 } 310 311 text, err = r.repo(ctx).GoMod(ctx, version) 312 if err == nil { 313 if err := checkGoMod(r.path, version, text); err != nil { 314 return text, err 315 } 316 if err := writeDiskGoMod(ctx, file, text); err != nil { 317 fmt.Fprintf(os.Stderr, "go: writing go.mod cache: %v\n", err) 318 } 319 } 320 return text, err 321 }) 322 if err != nil { 323 return nil, err 324 } 325 return append([]byte(nil), text...), nil 326} 327 328func (r *cachingRepo) Zip(ctx context.Context, dst io.Writer, version string) error { 329 if gover.IsToolchain(r.path) { 330 return ErrToolchain 331 } 332 return r.repo(ctx).Zip(ctx, dst, version) 333} 334 335// InfoFile is like Lookup(ctx, path).Stat(version) but also returns the name of the file 336// containing the cached information. 337func InfoFile(ctx context.Context, path, version string) (*RevInfo, string, error) { 338 if !gover.ModIsValid(path, version) { 339 return nil, "", fmt.Errorf("invalid version %q", version) 340 } 341 342 if file, info, err := readDiskStat(ctx, path, version); err == nil { 343 return info, file, nil 344 } 345 346 var info *RevInfo 347 var err2info map[error]*RevInfo 348 err := TryProxies(func(proxy string) error { 349 i, err := Lookup(ctx, proxy, path).Stat(ctx, version) 350 if err == nil { 351 info = i 352 } else { 353 if err2info == nil { 354 err2info = make(map[error]*RevInfo) 355 } 356 err2info[err] = info 357 } 358 return err 359 }) 360 if err != nil { 361 return err2info[err], "", err 362 } 363 364 // Stat should have populated the disk cache for us. 365 file, err := CachePath(ctx, module.Version{Path: path, Version: version}, "info") 366 if err != nil { 367 return nil, "", err 368 } 369 return info, file, nil 370} 371 372// GoMod is like Lookup(ctx, path).GoMod(rev) but avoids the 373// repository path resolution in Lookup if the result is 374// already cached on local disk. 375func GoMod(ctx context.Context, path, rev string) ([]byte, error) { 376 // Convert commit hash to pseudo-version 377 // to increase cache hit rate. 378 if !gover.ModIsValid(path, rev) { 379 if _, info, err := readDiskStat(ctx, path, rev); err == nil { 380 rev = info.Version 381 } else { 382 if errors.Is(err, statCacheErr) { 383 return nil, err 384 } 385 err := TryProxies(func(proxy string) error { 386 info, err := Lookup(ctx, proxy, path).Stat(ctx, rev) 387 if err == nil { 388 rev = info.Version 389 } 390 return err 391 }) 392 if err != nil { 393 return nil, err 394 } 395 } 396 } 397 398 _, data, err := readDiskGoMod(ctx, path, rev) 399 if err == nil { 400 return data, nil 401 } 402 403 err = TryProxies(func(proxy string) (err error) { 404 data, err = Lookup(ctx, proxy, path).GoMod(ctx, rev) 405 return err 406 }) 407 return data, err 408} 409 410// GoModFile is like GoMod but returns the name of the file containing 411// the cached information. 412func GoModFile(ctx context.Context, path, version string) (string, error) { 413 if !gover.ModIsValid(path, version) { 414 return "", fmt.Errorf("invalid version %q", version) 415 } 416 if _, err := GoMod(ctx, path, version); err != nil { 417 return "", err 418 } 419 // GoMod should have populated the disk cache for us. 420 file, err := CachePath(ctx, module.Version{Path: path, Version: version}, "mod") 421 if err != nil { 422 return "", err 423 } 424 return file, nil 425} 426 427// GoModSum returns the go.sum entry for the module version's go.mod file. 428// (That is, it returns the entry listed in go.sum as "path version/go.mod".) 429func GoModSum(ctx context.Context, path, version string) (string, error) { 430 if !gover.ModIsValid(path, version) { 431 return "", fmt.Errorf("invalid version %q", version) 432 } 433 data, err := GoMod(ctx, path, version) 434 if err != nil { 435 return "", err 436 } 437 sum, err := goModSum(data) 438 if err != nil { 439 return "", err 440 } 441 return sum, nil 442} 443 444var errNotCached = fmt.Errorf("not in cache") 445 446// readDiskStat reads a cached stat result from disk, 447// returning the name of the cache file and the result. 448// If the read fails, the caller can use 449// writeDiskStat(file, info) to write a new cache entry. 450func readDiskStat(ctx context.Context, path, rev string) (file string, info *RevInfo, err error) { 451 if gover.IsToolchain(path) { 452 return "", nil, errNotCached 453 } 454 file, data, err := readDiskCache(ctx, path, rev, "info") 455 if err != nil { 456 // If the cache already contains a pseudo-version with the given hash, we 457 // would previously return that pseudo-version without checking upstream. 458 // However, that produced an unfortunate side-effect: if the author added a 459 // tag to the repository, 'go get' would not pick up the effect of that new 460 // tag on the existing commits, and 'go' commands that referred to those 461 // commits would use the previous name instead of the new one. 462 // 463 // That's especially problematic if the original pseudo-version starts with 464 // v0.0.0-, as was the case for all pseudo-versions during vgo development, 465 // since a v0.0.0- pseudo-version has lower precedence than pretty much any 466 // tagged version. 467 // 468 // In practice, we're only looking up by hash during initial conversion of a 469 // legacy config and during an explicit 'go get', and a little extra latency 470 // for those operations seems worth the benefit of picking up more accurate 471 // versions. 472 // 473 // Fall back to this resolution scheme only if the GOPROXY setting prohibits 474 // us from resolving upstream tags. 475 if cfg.GOPROXY == "off" { 476 if file, info, err := readDiskStatByHash(ctx, path, rev); err == nil { 477 return file, info, nil 478 } 479 } 480 return file, nil, err 481 } 482 info = new(RevInfo) 483 if err := json.Unmarshal(data, info); err != nil { 484 return file, nil, errNotCached 485 } 486 // The disk might have stale .info files that have Name and Short fields set. 487 // We want to canonicalize to .info files with those fields omitted. 488 // Remarshal and update the cache file if needed. 489 data2, err := json.Marshal(info) 490 if err == nil && !bytes.Equal(data2, data) { 491 writeDiskCache(ctx, file, data) 492 } 493 return file, info, nil 494} 495 496// readDiskStatByHash is a fallback for readDiskStat for the case 497// where rev is a commit hash instead of a proper semantic version. 498// In that case, we look for a cached pseudo-version that matches 499// the commit hash. If we find one, we use it. 500// This matters most for converting legacy package management 501// configs, when we are often looking up commits by full hash. 502// Without this check we'd be doing network I/O to the remote repo 503// just to find out about a commit we already know about 504// (and have cached under its pseudo-version). 505func readDiskStatByHash(ctx context.Context, path, rev string) (file string, info *RevInfo, err error) { 506 if gover.IsToolchain(path) { 507 return "", nil, errNotCached 508 } 509 if cfg.GOMODCACHE == "" { 510 // Do not download to current directory. 511 return "", nil, errNotCached 512 } 513 514 if !codehost.AllHex(rev) || len(rev) < 12 { 515 return "", nil, errNotCached 516 } 517 rev = rev[:12] 518 cdir, err := cacheDir(ctx, path) 519 if err != nil { 520 return "", nil, errNotCached 521 } 522 dir, err := os.Open(cdir) 523 if err != nil { 524 return "", nil, errNotCached 525 } 526 names, err := dir.Readdirnames(-1) 527 dir.Close() 528 if err != nil { 529 return "", nil, errNotCached 530 } 531 532 // A given commit hash may map to more than one pseudo-version, 533 // depending on which tags are present on the repository. 534 // Take the highest such version. 535 var maxVersion string 536 suffix := "-" + rev + ".info" 537 err = errNotCached 538 for _, name := range names { 539 if strings.HasSuffix(name, suffix) { 540 v := strings.TrimSuffix(name, ".info") 541 if module.IsPseudoVersion(v) && semver.Compare(v, maxVersion) > 0 { 542 maxVersion = v 543 file, info, err = readDiskStat(ctx, path, strings.TrimSuffix(name, ".info")) 544 } 545 } 546 } 547 return file, info, err 548} 549 550// oldVgoPrefix is the prefix in the old auto-generated cached go.mod files. 551// We stopped trying to auto-generate the go.mod files. Now we use a trivial 552// go.mod with only a module line, and we've dropped the version prefix 553// entirely. If we see a version prefix, that means we're looking at an old copy 554// and should ignore it. 555var oldVgoPrefix = []byte("//vgo 0.0.") 556 557// readDiskGoMod reads a cached go.mod file from disk, 558// returning the name of the cache file and the result. 559// If the read fails, the caller can use 560// writeDiskGoMod(file, data) to write a new cache entry. 561func readDiskGoMod(ctx context.Context, path, rev string) (file string, data []byte, err error) { 562 if gover.IsToolchain(path) { 563 return "", nil, errNotCached 564 } 565 file, data, err = readDiskCache(ctx, path, rev, "mod") 566 567 // If the file has an old auto-conversion prefix, pretend it's not there. 568 if bytes.HasPrefix(data, oldVgoPrefix) { 569 err = errNotCached 570 data = nil 571 } 572 573 if err == nil { 574 if err := checkGoMod(path, rev, data); err != nil { 575 return "", nil, err 576 } 577 } 578 579 return file, data, err 580} 581 582// readDiskCache is the generic "read from a cache file" implementation. 583// It takes the revision and an identifying suffix for the kind of data being cached. 584// It returns the name of the cache file and the content of the file. 585// If the read fails, the caller can use 586// writeDiskCache(file, data) to write a new cache entry. 587func readDiskCache(ctx context.Context, path, rev, suffix string) (file string, data []byte, err error) { 588 if gover.IsToolchain(path) { 589 return "", nil, errNotCached 590 } 591 file, err = CachePath(ctx, module.Version{Path: path, Version: rev}, suffix) 592 if err != nil { 593 return "", nil, errNotCached 594 } 595 data, err = robustio.ReadFile(file) 596 if err != nil { 597 return file, nil, errNotCached 598 } 599 return file, data, nil 600} 601 602// writeDiskStat writes a stat result cache entry. 603// The file name must have been returned by a previous call to readDiskStat. 604func writeDiskStat(ctx context.Context, file string, info *RevInfo) error { 605 if file == "" { 606 return nil 607 } 608 609 if info.Origin != nil { 610 // Clean the origin information, which might have too many 611 // validation criteria, for example if we are saving the result of 612 // m@master as m@pseudo-version. 613 clean := *info 614 info = &clean 615 o := *info.Origin 616 info.Origin = &o 617 618 // Tags never matter if you are starting with a semver version, 619 // as we would be when finding this cache entry. 620 o.TagSum = "" 621 o.TagPrefix = "" 622 // Ref doesn't matter if you have a pseudoversion. 623 if module.IsPseudoVersion(info.Version) { 624 o.Ref = "" 625 } 626 } 627 628 js, err := json.Marshal(info) 629 if err != nil { 630 return err 631 } 632 return writeDiskCache(ctx, file, js) 633} 634 635// writeDiskGoMod writes a go.mod cache entry. 636// The file name must have been returned by a previous call to readDiskGoMod. 637func writeDiskGoMod(ctx context.Context, file string, text []byte) error { 638 return writeDiskCache(ctx, file, text) 639} 640 641// writeDiskCache is the generic "write to a cache file" implementation. 642// The file must have been returned by a previous call to readDiskCache. 643func writeDiskCache(ctx context.Context, file string, data []byte) error { 644 if file == "" { 645 return nil 646 } 647 // Make sure directory for file exists. 648 if err := os.MkdirAll(filepath.Dir(file), 0777); err != nil { 649 return err 650 } 651 652 // Write the file to a temporary location, and then rename it to its final 653 // path to reduce the likelihood of a corrupt file existing at that final path. 654 f, err := tempFile(ctx, filepath.Dir(file), filepath.Base(file), 0666) 655 if err != nil { 656 return err 657 } 658 defer func() { 659 // Only call os.Remove on f.Name() if we failed to rename it: otherwise, 660 // some other process may have created a new file with the same name after 661 // the rename completed. 662 if err != nil { 663 f.Close() 664 os.Remove(f.Name()) 665 } 666 }() 667 668 if _, err := f.Write(data); err != nil { 669 return err 670 } 671 if err := f.Close(); err != nil { 672 return err 673 } 674 if err := robustio.Rename(f.Name(), file); err != nil { 675 return err 676 } 677 678 if strings.HasSuffix(file, ".mod") { 679 rewriteVersionList(ctx, filepath.Dir(file)) 680 } 681 return nil 682} 683 684// tempFile creates a new temporary file with given permission bits. 685func tempFile(ctx context.Context, dir, prefix string, perm fs.FileMode) (f *os.File, err error) { 686 for i := 0; i < 10000; i++ { 687 name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+".tmp") 688 f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm) 689 if os.IsExist(err) { 690 if ctx.Err() != nil { 691 return nil, ctx.Err() 692 } 693 continue 694 } 695 break 696 } 697 return 698} 699 700// rewriteVersionList rewrites the version list in dir 701// after a new *.mod file has been written. 702func rewriteVersionList(ctx context.Context, dir string) (err error) { 703 if filepath.Base(dir) != "@v" { 704 base.Fatalf("go: internal error: misuse of rewriteVersionList") 705 } 706 707 listFile := filepath.Join(dir, "list") 708 709 // Lock listfile when writing to it to try to avoid corruption to the file. 710 // Under rare circumstances, for instance, if the system loses power in the 711 // middle of a write it is possible for corrupt data to be written. This is 712 // not a problem for the go command itself, but may be an issue if the 713 // cache is being served by a GOPROXY HTTP server. This will be corrected 714 // the next time a new version of the module is fetched and the file is rewritten. 715 // TODO(matloob): golang.org/issue/43313 covers adding a go mod verify 716 // command that removes module versions that fail checksums. It should also 717 // remove list files that are detected to be corrupt. 718 f, err := lockedfile.Edit(listFile) 719 if err != nil { 720 return err 721 } 722 defer func() { 723 if cerr := f.Close(); cerr != nil && err == nil { 724 err = cerr 725 } 726 }() 727 infos, err := os.ReadDir(dir) 728 if err != nil { 729 return err 730 } 731 var list []string 732 for _, info := range infos { 733 // We look for *.mod files on the theory that if we can't supply 734 // the .mod file then there's no point in listing that version, 735 // since it's unusable. (We can have *.info without *.mod.) 736 // We don't require *.zip files on the theory that for code only 737 // involved in module graph construction, many *.zip files 738 // will never be requested. 739 name := info.Name() 740 if v, found := strings.CutSuffix(name, ".mod"); found { 741 if v != "" && module.CanonicalVersion(v) == v { 742 list = append(list, v) 743 } 744 } 745 } 746 semver.Sort(list) 747 748 var buf bytes.Buffer 749 for _, v := range list { 750 buf.WriteString(v) 751 buf.WriteString("\n") 752 } 753 if fi, err := f.Stat(); err == nil && int(fi.Size()) == buf.Len() { 754 old := make([]byte, buf.Len()+1) 755 if n, err := f.ReadAt(old, 0); err == io.EOF && n == buf.Len() && bytes.Equal(buf.Bytes(), old) { 756 return nil // No edit needed. 757 } 758 } 759 // Remove existing contents, so that when we truncate to the actual size it will zero-fill, 760 // and we will be able to detect (some) incomplete writes as files containing trailing NUL bytes. 761 if err := f.Truncate(0); err != nil { 762 return err 763 } 764 // Reserve the final size and zero-fill. 765 if err := f.Truncate(int64(buf.Len())); err != nil { 766 return err 767 } 768 // Write the actual contents. If this fails partway through, 769 // the remainder of the file should remain as zeroes. 770 if _, err := f.Write(buf.Bytes()); err != nil { 771 f.Truncate(0) 772 return err 773 } 774 775 return nil 776} 777 778var ( 779 statCacheOnce sync.Once 780 statCacheErr error 781 782 counterErrorsGOMODCACHEEntryRelative = counter.New("go/errors:gomodcache-entry-relative") 783) 784 785// checkCacheDir checks if the directory specified by GOMODCACHE exists. An 786// error is returned if it does not. 787func checkCacheDir(ctx context.Context) error { 788 if cfg.GOMODCACHE == "" { 789 // modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE 790 // is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen. 791 return fmt.Errorf("module cache not found: neither GOMODCACHE nor GOPATH is set") 792 } 793 if !filepath.IsAbs(cfg.GOMODCACHE) { 794 counterErrorsGOMODCACHEEntryRelative.Inc() 795 return fmt.Errorf("GOMODCACHE entry is relative; must be absolute path: %q.\n", cfg.GOMODCACHE) 796 } 797 798 // os.Stat is slow on Windows, so we only call it once to prevent unnecessary 799 // I/O every time this function is called. 800 statCacheOnce.Do(func() { 801 fi, err := os.Stat(cfg.GOMODCACHE) 802 if err != nil { 803 if !os.IsNotExist(err) { 804 statCacheErr = fmt.Errorf("could not create module cache: %w", err) 805 return 806 } 807 if err := os.MkdirAll(cfg.GOMODCACHE, 0777); err != nil { 808 statCacheErr = fmt.Errorf("could not create module cache: %w", err) 809 return 810 } 811 return 812 } 813 if !fi.IsDir() { 814 statCacheErr = fmt.Errorf("could not create module cache: %q is not a directory", cfg.GOMODCACHE) 815 return 816 } 817 }) 818 return statCacheErr 819} 820