Source file src/cmd/go/internal/modfetch/fetch.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"archive/zip"
     9  	"bytes"
    10  	"context"
    11  	"crypto/sha256"
    12  	"encoding/base64"
    13  	"errors"
    14  	"fmt"
    15  	"io"
    16  	"io/fs"
    17  	"os"
    18  	"path/filepath"
    19  	"sort"
    20  	"strings"
    21  	"sync"
    22  
    23  	"cmd/go/internal/base"
    24  	"cmd/go/internal/cfg"
    25  	"cmd/go/internal/fsys"
    26  	"cmd/go/internal/gover"
    27  	"cmd/go/internal/lockedfile"
    28  	"cmd/go/internal/str"
    29  	"cmd/go/internal/trace"
    30  	"cmd/internal/par"
    31  	"cmd/internal/robustio"
    32  
    33  	"golang.org/x/mod/module"
    34  	"golang.org/x/mod/sumdb/dirhash"
    35  	modzip "golang.org/x/mod/zip"
    36  )
    37  
    38  var downloadCache par.ErrCache[module.Version, string] // version → directory
    39  
    40  var ErrToolchain = errors.New("internal error: invalid operation on toolchain module")
    41  
    42  // Download downloads the specific module version to the
    43  // local download cache and returns the name of the directory
    44  // corresponding to the root of the module's file tree.
    45  func Download(ctx context.Context, mod module.Version) (dir string, err error) {
    46  	if gover.IsToolchain(mod.Path) {
    47  		return "", ErrToolchain
    48  	}
    49  	if err := checkCacheDir(ctx); err != nil {
    50  		base.Fatal(err)
    51  	}
    52  
    53  	// The par.Cache here avoids duplicate work.
    54  	return downloadCache.Do(mod, func() (string, error) {
    55  		dir, err := download(ctx, mod)
    56  		if err != nil {
    57  			return "", err
    58  		}
    59  		checkMod(ctx, mod)
    60  
    61  		// If go.mod exists (not an old legacy module), check version is not too new.
    62  		if data, err := os.ReadFile(filepath.Join(dir, "go.mod")); err == nil {
    63  			goVersion := gover.GoModLookup(data, "go")
    64  			if gover.Compare(goVersion, gover.Local()) > 0 {
    65  				return "", &gover.TooNewError{What: mod.String(), GoVersion: goVersion}
    66  			}
    67  		} else if !errors.Is(err, fs.ErrNotExist) {
    68  			return "", err
    69  		}
    70  
    71  		return dir, nil
    72  	})
    73  }
    74  
    75  // Unzip is like Download but is given the explicit zip file to use,
    76  // rather than downloading it. This is used for the GOFIPS140 zip files,
    77  // which ship in the Go distribution itself.
    78  func Unzip(ctx context.Context, mod module.Version, zipfile string) (dir string, err error) {
    79  	if err := checkCacheDir(ctx); err != nil {
    80  		base.Fatal(err)
    81  	}
    82  
    83  	return downloadCache.Do(mod, func() (string, error) {
    84  		ctx, span := trace.StartSpan(ctx, "modfetch.Unzip "+mod.String())
    85  		defer span.Done()
    86  
    87  		dir, err = DownloadDir(ctx, mod)
    88  		if err == nil {
    89  			// The directory has already been completely extracted (no .partial file exists).
    90  			return dir, nil
    91  		} else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
    92  			return "", err
    93  		}
    94  
    95  		return unzip(ctx, mod, zipfile)
    96  	})
    97  }
    98  
    99  func download(ctx context.Context, mod module.Version) (dir string, err error) {
   100  	ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String())
   101  	defer span.Done()
   102  
   103  	dir, err = DownloadDir(ctx, mod)
   104  	if err == nil {
   105  		// The directory has already been completely extracted (no .partial file exists).
   106  		return dir, nil
   107  	} else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
   108  		return "", err
   109  	}
   110  
   111  	// To avoid cluttering the cache with extraneous files,
   112  	// DownloadZip uses the same lockfile as Download.
   113  	// Invoke DownloadZip before locking the file.
   114  	zipfile, err := DownloadZip(ctx, mod)
   115  	if err != nil {
   116  		return "", err
   117  	}
   118  
   119  	return unzip(ctx, mod, zipfile)
   120  }
   121  
   122  func unzip(ctx context.Context, mod module.Version, zipfile string) (dir string, err error) {
   123  	unlock, err := lockVersion(ctx, mod)
   124  	if err != nil {
   125  		return "", err
   126  	}
   127  	defer unlock()
   128  
   129  	ctx, span := trace.StartSpan(ctx, "unzip "+zipfile)
   130  	defer span.Done()
   131  
   132  	// Check whether the directory was populated while we were waiting on the lock.
   133  	dir, dirErr := DownloadDir(ctx, mod)
   134  	if dirErr == nil {
   135  		return dir, nil
   136  	}
   137  	_, dirExists := dirErr.(*DownloadDirPartialError)
   138  
   139  	// Clean up any remaining temporary directories created by old versions
   140  	// (before 1.16), as well as partially extracted directories (indicated by
   141  	// DownloadDirPartialError, usually because of a .partial file). This is only
   142  	// safe to do because the lock file ensures that their writers are no longer
   143  	// active.
   144  	parentDir := filepath.Dir(dir)
   145  	tmpPrefix := filepath.Base(dir) + ".tmp-"
   146  	if old, err := filepath.Glob(filepath.Join(str.QuoteGlob(parentDir), str.QuoteGlob(tmpPrefix)+"*")); err == nil {
   147  		for _, path := range old {
   148  			RemoveAll(path) // best effort
   149  		}
   150  	}
   151  	if dirExists {
   152  		if err := RemoveAll(dir); err != nil {
   153  			return "", err
   154  		}
   155  	}
   156  
   157  	partialPath, err := CachePath(ctx, mod, "partial")
   158  	if err != nil {
   159  		return "", err
   160  	}
   161  
   162  	// Extract the module zip directory at its final location.
   163  	//
   164  	// To prevent other processes from reading the directory if we crash,
   165  	// create a .partial file before extracting the directory, and delete
   166  	// the .partial file afterward (all while holding the lock).
   167  	//
   168  	// Before Go 1.16, we extracted to a temporary directory with a random name
   169  	// then renamed it into place with os.Rename. On Windows, this failed with
   170  	// ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner)
   171  	// opened files in the temporary directory.
   172  	//
   173  	// Go 1.14.2 and higher respect .partial files. Older versions may use
   174  	// partially extracted directories. 'go mod verify' can detect this,
   175  	// and 'go clean -modcache' can fix it.
   176  	if err := os.MkdirAll(parentDir, 0777); err != nil {
   177  		return "", err
   178  	}
   179  	if err := os.WriteFile(partialPath, nil, 0666); err != nil {
   180  		return "", err
   181  	}
   182  	if err := modzip.Unzip(dir, mod, zipfile); err != nil {
   183  		fmt.Fprintf(os.Stderr, "-> %s\n", err)
   184  		if rmErr := RemoveAll(dir); rmErr == nil {
   185  			os.Remove(partialPath)
   186  		}
   187  		return "", err
   188  	}
   189  	if err := os.Remove(partialPath); err != nil {
   190  		return "", err
   191  	}
   192  
   193  	if !cfg.ModCacheRW {
   194  		makeDirsReadOnly(dir)
   195  	}
   196  	return dir, nil
   197  }
   198  
   199  var downloadZipCache par.ErrCache[module.Version, string]
   200  
   201  // DownloadZip downloads the specific module version to the
   202  // local zip cache and returns the name of the zip file.
   203  func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) {
   204  	// The par.Cache here avoids duplicate work.
   205  	return downloadZipCache.Do(mod, func() (string, error) {
   206  		zipfile, err := CachePath(ctx, mod, "zip")
   207  		if err != nil {
   208  			return "", err
   209  		}
   210  		ziphashfile := zipfile + "hash"
   211  
   212  		// Return without locking if the zip and ziphash files exist.
   213  		if _, err := os.Stat(zipfile); err == nil {
   214  			if _, err := os.Stat(ziphashfile); err == nil {
   215  				return zipfile, nil
   216  			}
   217  		}
   218  
   219  		// The zip or ziphash file does not exist. Acquire the lock and create them.
   220  		if cfg.CmdName != "mod download" {
   221  			vers := mod.Version
   222  			if mod.Path == "golang.org/toolchain" {
   223  				// Shorten v0.0.1-go1.13.1.darwin-amd64 to go1.13.1.darwin-amd64
   224  				_, vers, _ = strings.Cut(vers, "-")
   225  				if i := strings.LastIndex(vers, "."); i >= 0 {
   226  					goos, goarch, _ := strings.Cut(vers[i+1:], "-")
   227  					vers = vers[:i] + " (" + goos + "/" + goarch + ")"
   228  				}
   229  				fmt.Fprintf(os.Stderr, "go: downloading %s\n", vers)
   230  			} else {
   231  				fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, vers)
   232  			}
   233  		}
   234  		unlock, err := lockVersion(ctx, mod)
   235  		if err != nil {
   236  			return "", err
   237  		}
   238  		defer unlock()
   239  
   240  		if err := downloadZip(ctx, mod, zipfile); err != nil {
   241  			return "", err
   242  		}
   243  		return zipfile, nil
   244  	})
   245  }
   246  
   247  func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) {
   248  	ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile)
   249  	defer span.Done()
   250  
   251  	// Double-check that the zipfile was not created while we were waiting for
   252  	// the lock in DownloadZip.
   253  	ziphashfile := zipfile + "hash"
   254  	var zipExists, ziphashExists bool
   255  	if _, err := os.Stat(zipfile); err == nil {
   256  		zipExists = true
   257  	}
   258  	if _, err := os.Stat(ziphashfile); err == nil {
   259  		ziphashExists = true
   260  	}
   261  	if zipExists && ziphashExists {
   262  		return nil
   263  	}
   264  
   265  	// Create parent directories.
   266  	if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
   267  		return err
   268  	}
   269  
   270  	// Clean up any remaining tempfiles from previous runs.
   271  	// This is only safe to do because the lock file ensures that their
   272  	// writers are no longer active.
   273  	tmpPattern := filepath.Base(zipfile) + "*.tmp"
   274  	if old, err := filepath.Glob(filepath.Join(str.QuoteGlob(filepath.Dir(zipfile)), tmpPattern)); err == nil {
   275  		for _, path := range old {
   276  			os.Remove(path) // best effort
   277  		}
   278  	}
   279  
   280  	// If the zip file exists, the ziphash file must have been deleted
   281  	// or lost after a file system crash. Re-hash the zip without downloading.
   282  	if zipExists {
   283  		return hashZip(mod, zipfile, ziphashfile)
   284  	}
   285  
   286  	// From here to the os.Rename call below is functionally almost equivalent to
   287  	// renameio.WriteToFile, with one key difference: we want to validate the
   288  	// contents of the file (by hashing it) before we commit it. Because the file
   289  	// is zip-compressed, we need an actual file — or at least an io.ReaderAt — to
   290  	// validate it: we can't just tee the stream as we write it.
   291  	f, err := tempFile(ctx, filepath.Dir(zipfile), filepath.Base(zipfile), 0666)
   292  	if err != nil {
   293  		return err
   294  	}
   295  	defer func() {
   296  		if err != nil {
   297  			f.Close()
   298  			os.Remove(f.Name())
   299  		}
   300  	}()
   301  
   302  	var unrecoverableErr error
   303  	err = TryProxies(func(proxy string) error {
   304  		if unrecoverableErr != nil {
   305  			return unrecoverableErr
   306  		}
   307  		repo := Lookup(ctx, proxy, mod.Path)
   308  		err := repo.Zip(ctx, f, mod.Version)
   309  		if err != nil {
   310  			// Zip may have partially written to f before failing.
   311  			// (Perhaps the server crashed while sending the file?)
   312  			// Since we allow fallback on error in some cases, we need to fix up the
   313  			// file to be empty again for the next attempt.
   314  			if _, err := f.Seek(0, io.SeekStart); err != nil {
   315  				unrecoverableErr = err
   316  				return err
   317  			}
   318  			if err := f.Truncate(0); err != nil {
   319  				unrecoverableErr = err
   320  				return err
   321  			}
   322  		}
   323  		return err
   324  	})
   325  	if err != nil {
   326  		return err
   327  	}
   328  
   329  	// Double-check that the paths within the zip file are well-formed.
   330  	//
   331  	// TODO(bcmills): There is a similar check within the Unzip function. Can we eliminate one?
   332  	fi, err := f.Stat()
   333  	if err != nil {
   334  		return err
   335  	}
   336  	z, err := zip.NewReader(f, fi.Size())
   337  	if err != nil {
   338  		return err
   339  	}
   340  	prefix := mod.Path + "@" + mod.Version + "/"
   341  	for _, f := range z.File {
   342  		if !strings.HasPrefix(f.Name, prefix) {
   343  			return fmt.Errorf("zip for %s has unexpected file %s", prefix[:len(prefix)-1], f.Name)
   344  		}
   345  	}
   346  
   347  	if err := f.Close(); err != nil {
   348  		return err
   349  	}
   350  
   351  	// Hash the zip file and check the sum before renaming to the final location.
   352  	if err := hashZip(mod, f.Name(), ziphashfile); err != nil {
   353  		return err
   354  	}
   355  	if err := os.Rename(f.Name(), zipfile); err != nil {
   356  		return err
   357  	}
   358  
   359  	// TODO(bcmills): Should we make the .zip and .ziphash files read-only to discourage tampering?
   360  
   361  	return nil
   362  }
   363  
   364  // hashZip reads the zip file opened in f, then writes the hash to ziphashfile,
   365  // overwriting that file if it exists.
   366  //
   367  // If the hash does not match go.sum (or the sumdb if enabled), hashZip returns
   368  // an error and does not write ziphashfile.
   369  func hashZip(mod module.Version, zipfile, ziphashfile string) (err error) {
   370  	hash, err := dirhash.HashZip(zipfile, dirhash.DefaultHash)
   371  	if err != nil {
   372  		return err
   373  	}
   374  	if err := checkModSum(mod, hash); err != nil {
   375  		return err
   376  	}
   377  	hf, err := lockedfile.Create(ziphashfile)
   378  	if err != nil {
   379  		return err
   380  	}
   381  	defer func() {
   382  		if closeErr := hf.Close(); err == nil && closeErr != nil {
   383  			err = closeErr
   384  		}
   385  	}()
   386  	if err := hf.Truncate(int64(len(hash))); err != nil {
   387  		return err
   388  	}
   389  	if _, err := hf.WriteAt([]byte(hash), 0); err != nil {
   390  		return err
   391  	}
   392  	return nil
   393  }
   394  
   395  // makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
   396  // and its transitive contents.
   397  func makeDirsReadOnly(dir string) {
   398  	type pathMode struct {
   399  		path string
   400  		mode fs.FileMode
   401  	}
   402  	var dirs []pathMode // in lexical order
   403  	filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
   404  		if err == nil && d.IsDir() {
   405  			info, err := d.Info()
   406  			if err == nil && info.Mode()&0222 != 0 {
   407  				dirs = append(dirs, pathMode{path, info.Mode()})
   408  			}
   409  		}
   410  		return nil
   411  	})
   412  
   413  	// Run over list backward to chmod children before parents.
   414  	for i := len(dirs) - 1; i >= 0; i-- {
   415  		os.Chmod(dirs[i].path, dirs[i].mode&^0222)
   416  	}
   417  }
   418  
   419  // RemoveAll removes a directory written by Download or Unzip, first applying
   420  // any permission changes needed to do so.
   421  func RemoveAll(dir string) error {
   422  	// Module cache has 0555 directories; make them writable in order to remove content.
   423  	filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error {
   424  		if err != nil {
   425  			return nil // ignore errors walking in file system
   426  		}
   427  		if info.IsDir() {
   428  			os.Chmod(path, 0777)
   429  		}
   430  		return nil
   431  	})
   432  	return robustio.RemoveAll(dir)
   433  }
   434  
   435  var GoSumFile string             // path to go.sum; set by package modload
   436  var WorkspaceGoSumFiles []string // path to module go.sums in workspace; set by package modload
   437  
   438  type modSum struct {
   439  	mod module.Version
   440  	sum string
   441  }
   442  
   443  var goSum struct {
   444  	mu        sync.Mutex
   445  	m         map[module.Version][]string            // content of go.sum file
   446  	w         map[string]map[module.Version][]string // sum file in workspace -> content of that sum file
   447  	status    map[modSum]modSumStatus                // state of sums in m
   448  	overwrite bool                                   // if true, overwrite go.sum without incorporating its contents
   449  	enabled   bool                                   // whether to use go.sum at all
   450  }
   451  
   452  type modSumStatus struct {
   453  	used, dirty bool
   454  }
   455  
   456  // Reset resets globals in the modfetch package, so previous loads don't affect
   457  // contents of go.sum files.
   458  func Reset() {
   459  	GoSumFile = ""
   460  	WorkspaceGoSumFiles = nil
   461  
   462  	// Uses of lookupCache and downloadCache both can call checkModSum,
   463  	// which in turn sets the used bit on goSum.status for modules.
   464  	// Reset them so used can be computed properly.
   465  	lookupCache = par.Cache[lookupCacheKey, Repo]{}
   466  	downloadCache = par.ErrCache[module.Version, string]{}
   467  
   468  	// Clear all fields on goSum. It will be initialized later
   469  	goSum.mu.Lock()
   470  	goSum.m = nil
   471  	goSum.w = nil
   472  	goSum.status = nil
   473  	goSum.overwrite = false
   474  	goSum.enabled = false
   475  	goSum.mu.Unlock()
   476  }
   477  
   478  // initGoSum initializes the go.sum data.
   479  // The boolean it returns reports whether the
   480  // use of go.sum is now enabled.
   481  // The goSum lock must be held.
   482  func initGoSum() (bool, error) {
   483  	if GoSumFile == "" {
   484  		return false, nil
   485  	}
   486  	if goSum.m != nil {
   487  		return true, nil
   488  	}
   489  
   490  	goSum.m = make(map[module.Version][]string)
   491  	goSum.status = make(map[modSum]modSumStatus)
   492  	goSum.w = make(map[string]map[module.Version][]string)
   493  
   494  	for _, f := range WorkspaceGoSumFiles {
   495  		goSum.w[f] = make(map[module.Version][]string)
   496  		_, err := readGoSumFile(goSum.w[f], f)
   497  		if err != nil {
   498  			return false, err
   499  		}
   500  	}
   501  
   502  	enabled, err := readGoSumFile(goSum.m, GoSumFile)
   503  	goSum.enabled = enabled
   504  	return enabled, err
   505  }
   506  
   507  func readGoSumFile(dst map[module.Version][]string, file string) (bool, error) {
   508  	var (
   509  		data []byte
   510  		err  error
   511  	)
   512  	if fsys.Replaced(file) {
   513  		// Don't lock go.sum if it's part of the overlay.
   514  		// On Plan 9, locking requires chmod, and we don't want to modify any file
   515  		// in the overlay. See #44700.
   516  		data, err = os.ReadFile(fsys.Actual(file))
   517  	} else {
   518  		data, err = lockedfile.Read(file)
   519  	}
   520  	if err != nil && !os.IsNotExist(err) {
   521  		return false, err
   522  	}
   523  	readGoSum(dst, file, data)
   524  
   525  	return true, nil
   526  }
   527  
   528  // emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod.
   529  // A bug caused us to write these into go.sum files for non-modules.
   530  // We detect and remove them.
   531  const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY="
   532  
   533  // readGoSum parses data, which is the content of file,
   534  // and adds it to goSum.m. The goSum lock must be held.
   535  func readGoSum(dst map[module.Version][]string, file string, data []byte) {
   536  	lineno := 0
   537  	for len(data) > 0 {
   538  		var line []byte
   539  		lineno++
   540  		i := bytes.IndexByte(data, '\n')
   541  		if i < 0 {
   542  			line, data = data, nil
   543  		} else {
   544  			line, data = data[:i], data[i+1:]
   545  		}
   546  		f := strings.Fields(string(line))
   547  		if len(f) == 0 {
   548  			// blank line; skip it
   549  			continue
   550  		}
   551  		if len(f) != 3 {
   552  			if cfg.CmdName == "mod tidy" {
   553  				// ignore malformed line so that go mod tidy can fix go.sum
   554  				continue
   555  			} else {
   556  				base.Fatalf("malformed go.sum:\n%s:%d: wrong number of fields %v\n", file, lineno, len(f))
   557  			}
   558  		}
   559  		if f[2] == emptyGoModHash {
   560  			// Old bug; drop it.
   561  			continue
   562  		}
   563  		mod := module.Version{Path: f[0], Version: f[1]}
   564  		dst[mod] = append(dst[mod], f[2])
   565  	}
   566  }
   567  
   568  // HaveSum returns true if the go.sum file contains an entry for mod.
   569  // The entry's hash must be generated with a known hash algorithm.
   570  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   571  // .mod and .zip files.
   572  func HaveSum(mod module.Version) bool {
   573  	goSum.mu.Lock()
   574  	defer goSum.mu.Unlock()
   575  	inited, err := initGoSum()
   576  	if err != nil || !inited {
   577  		return false
   578  	}
   579  	for _, goSums := range goSum.w {
   580  		for _, h := range goSums[mod] {
   581  			if !strings.HasPrefix(h, "h1:") {
   582  				continue
   583  			}
   584  			if !goSum.status[modSum{mod, h}].dirty {
   585  				return true
   586  			}
   587  		}
   588  	}
   589  	for _, h := range goSum.m[mod] {
   590  		if !strings.HasPrefix(h, "h1:") {
   591  			continue
   592  		}
   593  		if !goSum.status[modSum{mod, h}].dirty {
   594  			return true
   595  		}
   596  	}
   597  	return false
   598  }
   599  
   600  // RecordedSum returns the sum if the go.sum file contains an entry for mod.
   601  // The boolean reports true if an entry was found or
   602  // false if no entry found or two conflicting sums are found.
   603  // The entry's hash must be generated with a known hash algorithm.
   604  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   605  // .mod and .zip files.
   606  func RecordedSum(mod module.Version) (sum string, ok bool) {
   607  	goSum.mu.Lock()
   608  	defer goSum.mu.Unlock()
   609  	inited, err := initGoSum()
   610  	foundSum := ""
   611  	if err != nil || !inited {
   612  		return "", false
   613  	}
   614  	for _, goSums := range goSum.w {
   615  		for _, h := range goSums[mod] {
   616  			if !strings.HasPrefix(h, "h1:") {
   617  				continue
   618  			}
   619  			if !goSum.status[modSum{mod, h}].dirty {
   620  				if foundSum != "" && foundSum != h { // conflicting sums exist
   621  					return "", false
   622  				}
   623  				foundSum = h
   624  			}
   625  		}
   626  	}
   627  	for _, h := range goSum.m[mod] {
   628  		if !strings.HasPrefix(h, "h1:") {
   629  			continue
   630  		}
   631  		if !goSum.status[modSum{mod, h}].dirty {
   632  			if foundSum != "" && foundSum != h { // conflicting sums exist
   633  				return "", false
   634  			}
   635  			foundSum = h
   636  		}
   637  	}
   638  	return foundSum, true
   639  }
   640  
   641  // checkMod checks the given module's checksum and Go version.
   642  func checkMod(ctx context.Context, mod module.Version) {
   643  	// Do the file I/O before acquiring the go.sum lock.
   644  	ziphash, err := CachePath(ctx, mod, "ziphash")
   645  	if err != nil {
   646  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   647  	}
   648  	data, err := lockedfile.Read(ziphash)
   649  	if err != nil {
   650  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   651  	}
   652  	data = bytes.TrimSpace(data)
   653  	if !isValidSum(data) {
   654  		// Recreate ziphash file from zip file and use that to check the mod sum.
   655  		zip, err := CachePath(ctx, mod, "zip")
   656  		if err != nil {
   657  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   658  		}
   659  		err = hashZip(mod, zip, ziphash)
   660  		if err != nil {
   661  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   662  		}
   663  		return
   664  	}
   665  	h := string(data)
   666  	if !strings.HasPrefix(h, "h1:") {
   667  		base.Fatalf("verifying %v", module.VersionError(mod, fmt.Errorf("unexpected ziphash: %q", h)))
   668  	}
   669  
   670  	if err := checkModSum(mod, h); err != nil {
   671  		base.Fatalf("%s", err)
   672  	}
   673  }
   674  
   675  // goModSum returns the checksum for the go.mod contents.
   676  func goModSum(data []byte) (string, error) {
   677  	return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
   678  		return io.NopCloser(bytes.NewReader(data)), nil
   679  	})
   680  }
   681  
   682  // checkGoMod checks the given module's go.mod checksum;
   683  // data is the go.mod content.
   684  func checkGoMod(path, version string, data []byte) error {
   685  	h, err := goModSum(data)
   686  	if err != nil {
   687  		return &module.ModuleError{Path: path, Version: version, Err: fmt.Errorf("verifying go.mod: %v", err)}
   688  	}
   689  
   690  	return checkModSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
   691  }
   692  
   693  // checkModSum checks that the recorded checksum for mod is h.
   694  //
   695  // mod.Version may have the additional suffix "/go.mod" to request the checksum
   696  // for the module's go.mod file only.
   697  func checkModSum(mod module.Version, h string) error {
   698  	// We lock goSum when manipulating it,
   699  	// but we arrange to release the lock when calling checkSumDB,
   700  	// so that parallel calls to checkModHash can execute parallel calls
   701  	// to checkSumDB.
   702  
   703  	// Check whether mod+h is listed in go.sum already. If so, we're done.
   704  	goSum.mu.Lock()
   705  	inited, err := initGoSum()
   706  	if err != nil {
   707  		goSum.mu.Unlock()
   708  		return err
   709  	}
   710  	done := inited && haveModSumLocked(mod, h)
   711  	if inited {
   712  		st := goSum.status[modSum{mod, h}]
   713  		st.used = true
   714  		goSum.status[modSum{mod, h}] = st
   715  	}
   716  	goSum.mu.Unlock()
   717  
   718  	if done {
   719  		return nil
   720  	}
   721  
   722  	// Not listed, so we want to add them.
   723  	// Consult checksum database if appropriate.
   724  	if useSumDB(mod) {
   725  		// Calls base.Fatalf if mismatch detected.
   726  		if err := checkSumDB(mod, h); err != nil {
   727  			return err
   728  		}
   729  	}
   730  
   731  	// Add mod+h to go.sum, if it hasn't appeared already.
   732  	if inited {
   733  		goSum.mu.Lock()
   734  		addModSumLocked(mod, h)
   735  		st := goSum.status[modSum{mod, h}]
   736  		st.dirty = true
   737  		goSum.status[modSum{mod, h}] = st
   738  		goSum.mu.Unlock()
   739  	}
   740  	return nil
   741  }
   742  
   743  // haveModSumLocked reports whether the pair mod,h is already listed in go.sum.
   744  // If it finds a conflicting pair instead, it calls base.Fatalf.
   745  // goSum.mu must be locked.
   746  func haveModSumLocked(mod module.Version, h string) bool {
   747  	sumFileName := "go.sum"
   748  	if strings.HasSuffix(GoSumFile, "go.work.sum") {
   749  		sumFileName = "go.work.sum"
   750  	}
   751  	for _, vh := range goSum.m[mod] {
   752  		if h == vh {
   753  			return true
   754  		}
   755  		if strings.HasPrefix(vh, "h1:") {
   756  			base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, sumFileName, vh)
   757  		}
   758  	}
   759  	// Also check workspace sums.
   760  	foundMatch := false
   761  	// Check sums from all files in case there are conflicts between
   762  	// the files.
   763  	for goSumFile, goSums := range goSum.w {
   764  		for _, vh := range goSums[mod] {
   765  			if h == vh {
   766  				foundMatch = true
   767  			} else if strings.HasPrefix(vh, "h1:") {
   768  				base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, goSumFile, vh)
   769  			}
   770  		}
   771  	}
   772  	return foundMatch
   773  }
   774  
   775  // addModSumLocked adds the pair mod,h to go.sum.
   776  // goSum.mu must be locked.
   777  func addModSumLocked(mod module.Version, h string) {
   778  	if haveModSumLocked(mod, h) {
   779  		return
   780  	}
   781  	if len(goSum.m[mod]) > 0 {
   782  		fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h)
   783  	}
   784  	goSum.m[mod] = append(goSum.m[mod], h)
   785  }
   786  
   787  // checkSumDB checks the mod, h pair against the Go checksum database.
   788  // It calls base.Fatalf if the hash is to be rejected.
   789  func checkSumDB(mod module.Version, h string) error {
   790  	modWithoutSuffix := mod
   791  	noun := "module"
   792  	if before, found := strings.CutSuffix(mod.Version, "/go.mod"); found {
   793  		noun = "go.mod"
   794  		modWithoutSuffix.Version = before
   795  	}
   796  
   797  	db, lines, err := lookupSumDB(mod)
   798  	if err != nil {
   799  		return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: %v", noun, err))
   800  	}
   801  
   802  	have := mod.Path + " " + mod.Version + " " + h
   803  	prefix := mod.Path + " " + mod.Version + " h1:"
   804  	for _, line := range lines {
   805  		if line == have {
   806  			return nil
   807  		}
   808  		if strings.HasPrefix(line, prefix) {
   809  			return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, noun, h, db, line[len(prefix)-len("h1:"):]))
   810  		}
   811  	}
   812  	return nil
   813  }
   814  
   815  // Sum returns the checksum for the downloaded copy of the given module,
   816  // if present in the download cache.
   817  func Sum(ctx context.Context, mod module.Version) string {
   818  	if cfg.GOMODCACHE == "" {
   819  		// Do not use current directory.
   820  		return ""
   821  	}
   822  
   823  	ziphash, err := CachePath(ctx, mod, "ziphash")
   824  	if err != nil {
   825  		return ""
   826  	}
   827  	data, err := lockedfile.Read(ziphash)
   828  	if err != nil {
   829  		return ""
   830  	}
   831  	data = bytes.TrimSpace(data)
   832  	if !isValidSum(data) {
   833  		return ""
   834  	}
   835  	return string(data)
   836  }
   837  
   838  // isValidSum returns true if data is the valid contents of a zip hash file.
   839  // Certain critical files are written to disk by first truncating
   840  // then writing the actual bytes, so that if the write fails
   841  // the corrupt file should contain at least one of the null
   842  // bytes written by the truncate operation.
   843  func isValidSum(data []byte) bool {
   844  	if bytes.IndexByte(data, '\000') >= 0 {
   845  		return false
   846  	}
   847  
   848  	if len(data) != len("h1:")+base64.StdEncoding.EncodedLen(sha256.Size) {
   849  		return false
   850  	}
   851  
   852  	return true
   853  }
   854  
   855  var ErrGoSumDirty = errors.New("updates to go.sum needed, disabled by -mod=readonly")
   856  
   857  // WriteGoSum writes the go.sum file if it needs to be updated.
   858  //
   859  // keep is used to check whether a newly added sum should be saved in go.sum.
   860  // It should have entries for both module content sums and go.mod sums
   861  // (version ends with "/go.mod"). Existing sums will be preserved unless they
   862  // have been marked for deletion with TrimGoSum.
   863  func WriteGoSum(ctx context.Context, keep map[module.Version]bool, readonly bool) error {
   864  	goSum.mu.Lock()
   865  	defer goSum.mu.Unlock()
   866  
   867  	// If we haven't read the go.sum file yet, don't bother writing it.
   868  	if !goSum.enabled {
   869  		return nil
   870  	}
   871  
   872  	// Check whether we need to add sums for which keep[m] is true or remove
   873  	// unused sums marked with TrimGoSum. If there are no changes to make,
   874  	// just return without opening go.sum.
   875  	dirty := false
   876  Outer:
   877  	for m, hs := range goSum.m {
   878  		for _, h := range hs {
   879  			st := goSum.status[modSum{m, h}]
   880  			if st.dirty && (!st.used || keep[m]) {
   881  				dirty = true
   882  				break Outer
   883  			}
   884  		}
   885  	}
   886  	if !dirty {
   887  		return nil
   888  	}
   889  	if readonly {
   890  		return ErrGoSumDirty
   891  	}
   892  	if fsys.Replaced(GoSumFile) {
   893  		base.Fatalf("go: updates to go.sum needed, but go.sum is part of the overlay specified with -overlay")
   894  	}
   895  
   896  	// Make a best-effort attempt to acquire the side lock, only to exclude
   897  	// previous versions of the 'go' command from making simultaneous edits.
   898  	if unlock, err := SideLock(ctx); err == nil {
   899  		defer unlock()
   900  	}
   901  
   902  	err := lockedfile.Transform(GoSumFile, func(data []byte) ([]byte, error) {
   903  		tidyGoSum := tidyGoSum(data, keep)
   904  		return tidyGoSum, nil
   905  	})
   906  
   907  	if err != nil {
   908  		return fmt.Errorf("updating go.sum: %w", err)
   909  	}
   910  
   911  	goSum.status = make(map[modSum]modSumStatus)
   912  	goSum.overwrite = false
   913  	return nil
   914  }
   915  
   916  // TidyGoSum returns a tidy version of the go.sum file.
   917  // A missing go.sum file is treated as if empty.
   918  func TidyGoSum(keep map[module.Version]bool) (before, after []byte) {
   919  	goSum.mu.Lock()
   920  	defer goSum.mu.Unlock()
   921  	before, err := lockedfile.Read(GoSumFile)
   922  	if err != nil && !errors.Is(err, fs.ErrNotExist) {
   923  		base.Fatalf("reading go.sum: %v", err)
   924  	}
   925  	after = tidyGoSum(before, keep)
   926  	return before, after
   927  }
   928  
   929  // tidyGoSum returns a tidy version of the go.sum file.
   930  // The goSum lock must be held.
   931  func tidyGoSum(data []byte, keep map[module.Version]bool) []byte {
   932  	if !goSum.overwrite {
   933  		// Incorporate any sums added by other processes in the meantime.
   934  		// Add only the sums that we actually checked: the user may have edited or
   935  		// truncated the file to remove erroneous hashes, and we shouldn't restore
   936  		// them without good reason.
   937  		goSum.m = make(map[module.Version][]string, len(goSum.m))
   938  		readGoSum(goSum.m, GoSumFile, data)
   939  		for ms, st := range goSum.status {
   940  			if st.used && !sumInWorkspaceModulesLocked(ms.mod) {
   941  				addModSumLocked(ms.mod, ms.sum)
   942  			}
   943  		}
   944  	}
   945  
   946  	mods := make([]module.Version, 0, len(goSum.m))
   947  	for m := range goSum.m {
   948  		mods = append(mods, m)
   949  	}
   950  	module.Sort(mods)
   951  
   952  	var buf bytes.Buffer
   953  	for _, m := range mods {
   954  		list := goSum.m[m]
   955  		sort.Strings(list)
   956  		str.Uniq(&list)
   957  		for _, h := range list {
   958  			st := goSum.status[modSum{m, h}]
   959  			if (!st.dirty || (st.used && keep[m])) && !sumInWorkspaceModulesLocked(m) {
   960  				fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
   961  			}
   962  		}
   963  	}
   964  	return buf.Bytes()
   965  }
   966  
   967  func sumInWorkspaceModulesLocked(m module.Version) bool {
   968  	for _, goSums := range goSum.w {
   969  		if _, ok := goSums[m]; ok {
   970  			return true
   971  		}
   972  	}
   973  	return false
   974  }
   975  
   976  // TrimGoSum trims go.sum to contain only the modules needed for reproducible
   977  // builds.
   978  //
   979  // keep is used to check whether a sum should be retained in go.mod. It should
   980  // have entries for both module content sums and go.mod sums (version ends
   981  // with "/go.mod").
   982  func TrimGoSum(keep map[module.Version]bool) {
   983  	goSum.mu.Lock()
   984  	defer goSum.mu.Unlock()
   985  	inited, err := initGoSum()
   986  	if err != nil {
   987  		base.Fatalf("%s", err)
   988  	}
   989  	if !inited {
   990  		return
   991  	}
   992  
   993  	for m, hs := range goSum.m {
   994  		if !keep[m] {
   995  			for _, h := range hs {
   996  				goSum.status[modSum{m, h}] = modSumStatus{used: false, dirty: true}
   997  			}
   998  			goSum.overwrite = true
   999  		}
  1000  	}
  1001  }
  1002  
  1003  const goSumMismatch = `
  1004  
  1005  SECURITY ERROR
  1006  This download does NOT match an earlier download recorded in go.sum.
  1007  The bits may have been replaced on the origin server, or an attacker may
  1008  have intercepted the download attempt.
  1009  
  1010  For more information, see 'go help module-auth'.
  1011  `
  1012  
  1013  const sumdbMismatch = `
  1014  
  1015  SECURITY ERROR
  1016  This download does NOT match the one reported by the checksum server.
  1017  The bits may have been replaced on the origin server, or an attacker may
  1018  have intercepted the download attempt.
  1019  
  1020  For more information, see 'go help module-auth'.
  1021  `
  1022  
  1023  const hashVersionMismatch = `
  1024  
  1025  SECURITY WARNING
  1026  This download is listed in go.sum, but using an unknown hash algorithm.
  1027  The download cannot be verified.
  1028  
  1029  For more information, see 'go help module-auth'.
  1030  
  1031  `
  1032  
  1033  var HelpModuleAuth = &base.Command{
  1034  	UsageLine: "module-auth",
  1035  	Short:     "module authentication using go.sum",
  1036  	Long: `
  1037  When the go command downloads a module zip file or go.mod file into the
  1038  module cache, it computes a cryptographic hash and compares it with a known
  1039  value to verify the file hasn't changed since it was first downloaded. Known
  1040  hashes are stored in a file in the module root directory named go.sum. Hashes
  1041  may also be downloaded from the checksum database depending on the values of
  1042  GOSUMDB, GOPRIVATE, and GONOSUMDB.
  1043  
  1044  For details, see https://golang.org/ref/mod#authenticating.
  1045  `,
  1046  }
  1047  
  1048  var HelpPrivate = &base.Command{
  1049  	UsageLine: "private",
  1050  	Short:     "configuration for downloading non-public code",
  1051  	Long: `
  1052  The go command defaults to downloading modules from the public Go module
  1053  mirror at proxy.golang.org. It also defaults to validating downloaded modules,
  1054  regardless of source, against the public Go checksum database at sum.golang.org.
  1055  These defaults work well for publicly available source code.
  1056  
  1057  The GOPRIVATE environment variable controls which modules the go command
  1058  considers to be private (not available publicly) and should therefore not use
  1059  the proxy or checksum database. The variable is a comma-separated list of
  1060  glob patterns (in the syntax of Go's path.Match) of module path prefixes.
  1061  For example,
  1062  
  1063  	GOPRIVATE=*.corp.example.com,rsc.io/private
  1064  
  1065  causes the go command to treat as private any module with a path prefix
  1066  matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private,
  1067  and rsc.io/private/quux.
  1068  
  1069  For fine-grained control over module download and validation, the GONOPROXY
  1070  and GONOSUMDB environment variables accept the same kind of glob list
  1071  and override GOPRIVATE for the specific decision of whether to use the proxy
  1072  and checksum database, respectively.
  1073  
  1074  For example, if a company ran a module proxy serving private modules,
  1075  users would configure go using:
  1076  
  1077  	GOPRIVATE=*.corp.example.com
  1078  	GOPROXY=proxy.example.com
  1079  	GONOPROXY=none
  1080  
  1081  The GOPRIVATE variable is also used to define the "public" and "private"
  1082  patterns for the GOVCS variable; see 'go help vcs'. For that usage,
  1083  GOPRIVATE applies even in GOPATH mode. In that case, it matches import paths
  1084  instead of module paths.
  1085  
  1086  The 'go env -w' command (see 'go help env') can be used to set these variables
  1087  for future go command invocations.
  1088  
  1089  For more details, see https://golang.org/ref/mod#private-modules.
  1090  `,
  1091  }
  1092  

View as plain text