stdlib_test.go

     1  // Copyright 2013 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // This file tests types2.Check by using it to
     6  // typecheck the standard library and tests.
     7  
     8  package types2_test
     9  
    10  import (
    11  	"bytes"
    12  	"cmd/compile/internal/syntax"
    13  	"errors"
    14  	"fmt"
    15  	"go/build"
    16  	"internal/testenv"
    17  	"os"
    18  	"path/filepath"
    19  	"runtime"
    20  	"slices"
    21  	"strings"
    22  	"sync"
    23  	"testing"
    24  	"time"
    25  
    26  	. "cmd/compile/internal/types2"
    27  )
    28  
    29  var stdLibImporter = defaultImporter()
    30  
    31  func TestStdlib(t *testing.T) {
    32  	if testing.Short() {
    33  		t.Skip("skipping in short mode")
    34  	}
    35  
    36  	testenv.MustHaveGoBuild(t)
    37  
    38  	// Collect non-test files.
    39  	dirFiles := make(map[string][]string)
    40  	root := filepath.Join(testenv.GOROOT(t), "src")
    41  	walkPkgDirs(root, func(dir string, filenames []string) {
    42  		dirFiles[dir] = filenames
    43  	}, t.Error)
    44  
    45  	c := &stdlibChecker{
    46  		dirFiles: dirFiles,
    47  		pkgs:     make(map[string]*futurePackage),
    48  	}
    49  
    50  	start := time.Now()
    51  
    52  	// Though we read files while parsing, type-checking is otherwise CPU bound.
    53  	//
    54  	// This doesn't achieve great CPU utilization as many packages may block
    55  	// waiting for a common import, but in combination with the non-deterministic
    56  	// map iteration below this should provide decent coverage of concurrent
    57  	// type-checking (see golang/go#47729).
    58  	cpulimit := make(chan struct{}, runtime.GOMAXPROCS(0))
    59  	var wg sync.WaitGroup
    60  
    61  	for dir := range dirFiles {
    62  		cpulimit <- struct{}{}
    63  		wg.Add(1)
    64  		go func() {
    65  			defer func() {
    66  				wg.Done()
    67  				<-cpulimit
    68  			}()
    69  
    70  			_, err := c.getDirPackage(dir)
    71  			if err != nil {
    72  				t.Errorf("error checking %s: %v", dir, err)
    73  			}
    74  		}()
    75  	}
    76  
    77  	wg.Wait()
    78  
    79  	if testing.Verbose() {
    80  		fmt.Println(len(dirFiles), "packages typechecked in", time.Since(start))
    81  	}
    82  }
    83  
    84  // stdlibChecker implements concurrent type-checking of the packages defined by
    85  // dirFiles, which must define a closed set of packages (such as GOROOT/src).
    86  type stdlibChecker struct {
    87  	dirFiles map[string][]string // non-test files per directory; must be pre-populated
    88  
    89  	mu   sync.Mutex
    90  	pkgs map[string]*futurePackage // future cache of type-checking results
    91  }
    92  
    93  // A futurePackage is a future result of type-checking.
    94  type futurePackage struct {
    95  	done chan struct{} // guards pkg and err
    96  	pkg  *Package
    97  	err  error
    98  }
    99  
   100  func (c *stdlibChecker) Import(path string) (*Package, error) {
   101  	panic("unimplemented: use ImportFrom")
   102  }
   103  
   104  func (c *stdlibChecker) ImportFrom(path, dir string, _ ImportMode) (*Package, error) {
   105  	if path == "unsafe" {
   106  		// unsafe cannot be type checked normally.
   107  		return Unsafe, nil
   108  	}
   109  
   110  	p, err := build.Default.Import(path, dir, build.FindOnly)
   111  	if err != nil {
   112  		return nil, err
   113  	}
   114  
   115  	pkg, err := c.getDirPackage(p.Dir)
   116  	if pkg != nil {
   117  		// As long as pkg is non-nil, avoid redundant errors related to failed
   118  		// imports. TestStdlib will collect errors once for each package.
   119  		return pkg, nil
   120  	}
   121  	return nil, err
   122  }
   123  
   124  // getDirPackage gets the package defined in dir from the future cache.
   125  //
   126  // If this is the first goroutine requesting the package, getDirPackage
   127  // type-checks.
   128  func (c *stdlibChecker) getDirPackage(dir string) (*Package, error) {
   129  	c.mu.Lock()
   130  	fut, ok := c.pkgs[dir]
   131  	if !ok {
   132  		// First request for this package dir; type check.
   133  		fut = &futurePackage{
   134  			done: make(chan struct{}),
   135  		}
   136  		c.pkgs[dir] = fut
   137  		files, ok := c.dirFiles[dir]
   138  		c.mu.Unlock()
   139  		if !ok {
   140  			fut.err = fmt.Errorf("no files for %s", dir)
   141  		} else {
   142  			// Using dir as the package path here may be inconsistent with the behavior
   143  			// of a normal importer, but is sufficient as dir is by construction unique
   144  			// to this package.
   145  			fut.pkg, fut.err = typecheckFiles(dir, files, c)
   146  		}
   147  		close(fut.done)
   148  	} else {
   149  		// Otherwise, await the result.
   150  		c.mu.Unlock()
   151  		<-fut.done
   152  	}
   153  	return fut.pkg, fut.err
   154  }
   155  
   156  // firstComment returns the contents of the first non-empty comment in
   157  // the given file, "skip", or the empty string. No matter the present
   158  // comments, if any of them contains a build tag, the result is always
   159  // "skip". Only comments within the first 4K of the file are considered.
   160  // TODO(gri) should only read until we see "package" token.
   161  func firstComment(filename string) (first string) {
   162  	f, err := os.Open(filename)
   163  	if err != nil {
   164  		return ""
   165  	}
   166  	defer f.Close()
   167  
   168  	// read at most 4KB
   169  	var buf [4 << 10]byte
   170  	n, _ := f.Read(buf[:])
   171  	src := bytes.NewBuffer(buf[:n])
   172  
   173  	// TODO(gri) we need a better way to terminate CommentsDo
   174  	defer func() {
   175  		if p := recover(); p != nil {
   176  			if s, ok := p.(string); ok {
   177  				first = s
   178  			}
   179  		}
   180  	}()
   181  
   182  	syntax.CommentsDo(src, func(_, _ uint, text string) {
   183  		if text[0] != '/' {
   184  			return // not a comment
   185  		}
   186  
   187  		// extract comment text
   188  		if text[1] == '*' {
   189  			text = text[:len(text)-2]
   190  		}
   191  		text = strings.TrimSpace(text[2:])
   192  
   193  		if strings.HasPrefix(text, "go:build ") {
   194  			panic("skip")
   195  		}
   196  		if first == "" {
   197  			first = text // text may be "" but that's ok
   198  		}
   199  		// continue as we may still see build tags
   200  	})
   201  
   202  	return
   203  }
   204  
   205  func testTestDir(t *testing.T, path string, ignore ...string) {
   206  	files, err := os.ReadDir(path)
   207  	if err != nil {
   208  		// cmd/distpack deletes GOROOT/test, so skip the test if it isn't present.
   209  		// cmd/distpack also requires GOROOT/VERSION to exist, so use that to
   210  		// suppress false-positive skips.
   211  		if _, err := os.Stat(filepath.Join(testenv.GOROOT(t), "test")); os.IsNotExist(err) {
   212  			if _, err := os.Stat(filepath.Join(testenv.GOROOT(t), "VERSION")); err == nil {
   213  				t.Skipf("skipping: GOROOT/test not present")
   214  			}
   215  		}
   216  		t.Fatal(err)
   217  	}
   218  
   219  	excluded := make(map[string]bool)
   220  	for _, filename := range ignore {
   221  		excluded[filename] = true
   222  	}
   223  
   224  	for _, f := range files {
   225  		// filter directory contents
   226  		if f.IsDir() || !strings.HasSuffix(f.Name(), ".go") || excluded[f.Name()] {
   227  			continue
   228  		}
   229  
   230  		// get per-file instructions
   231  		expectErrors := false
   232  		filename := filepath.Join(path, f.Name())
   233  		goVersion := ""
   234  		if comment := firstComment(filename); comment != "" {
   235  			if strings.Contains(comment, "-goexperiment") {
   236  				continue // ignore this file
   237  			}
   238  			fields := strings.Fields(comment)
   239  			switch fields[0] {
   240  			case "skip", "compiledir":
   241  				continue // ignore this file
   242  			case "errorcheck":
   243  				expectErrors = true
   244  				for _, arg := range fields[1:] {
   245  					if arg == "-0" || arg == "-+" || arg == "-std" {
   246  						// Marked explicitly as not expecting errors (-0),
   247  						// or marked as compiling runtime/stdlib, which is only done
   248  						// to trigger runtime/stdlib-only error output.
   249  						// In both cases, the code should typecheck.
   250  						expectErrors = false
   251  						break
   252  					}
   253  					const prefix = "-lang="
   254  					if strings.HasPrefix(arg, prefix) {
   255  						goVersion = arg[len(prefix):]
   256  					}
   257  				}
   258  			}
   259  		}
   260  
   261  		// parse and type-check file
   262  		if testing.Verbose() {
   263  			fmt.Println("\t", filename)
   264  		}
   265  		file, err := syntax.ParseFile(filename, nil, nil, 0)
   266  		if err == nil {
   267  			conf := Config{
   268  				GoVersion: goVersion,
   269  				Importer:  stdLibImporter,
   270  			}
   271  			_, err = conf.Check(filename, []*syntax.File{file}, nil)
   272  		}
   273  
   274  		if expectErrors {
   275  			if err == nil {
   276  				t.Errorf("expected errors but found none in %s", filename)
   277  			}
   278  		} else {
   279  			if err != nil {
   280  				t.Error(err)
   281  			}
   282  		}
   283  	}
   284  }
   285  
   286  func TestStdTest(t *testing.T) {
   287  	testenv.MustHaveGoBuild(t)
   288  
   289  	if testing.Short() && testenv.Builder() == "" {
   290  		t.Skip("skipping in short mode")
   291  	}
   292  
   293  	testTestDir(t, filepath.Join(testenv.GOROOT(t), "test"),
   294  		"cmplxdivide.go", // also needs file cmplxdivide1.go - ignore
   295  		"directive.go",   // tests compiler rejection of bad directive placement - ignore
   296  		"directive2.go",  // tests compiler rejection of bad directive placement - ignore
   297  		"embedfunc.go",   // tests //go:embed
   298  		"embedvers.go",   // tests //go:embed
   299  		"linkname2.go",   // types2 doesn't check validity of //go:xxx directives
   300  		"linkname3.go",   // types2 doesn't check validity of //go:xxx directives
   301  	)
   302  }
   303  
   304  func TestStdFixed(t *testing.T) {
   305  	testenv.MustHaveGoBuild(t)
   306  
   307  	if testing.Short() && testenv.Builder() == "" {
   308  		t.Skip("skipping in short mode")
   309  	}
   310  
   311  	testTestDir(t, filepath.Join(testenv.GOROOT(t), "test", "fixedbugs"),
   312  		"bug248.go", "bug302.go", "bug369.go", // complex test instructions - ignore
   313  		"bug398.go",      // types2 doesn't check for anonymous interface cycles (go.dev/issue/56103)
   314  		"issue6889.go",   // gc-specific test
   315  		"issue11362.go",  // canonical import path check
   316  		"issue16369.go",  // types2 handles this correctly - not an issue
   317  		"issue18459.go",  // types2 doesn't check validity of //go:xxx directives
   318  		"issue18882.go",  // types2 doesn't check validity of //go:xxx directives
   319  		"issue20027.go",  // types2 does not have constraints on channel element size
   320  		"issue20529.go",  // types2 does not have constraints on stack size
   321  		"issue22200.go",  // types2 does not have constraints on stack size
   322  		"issue22200b.go", // types2 does not have constraints on stack size
   323  		"issue25507.go",  // types2 does not have constraints on stack size
   324  		"issue20780.go",  // types2 does not have constraints on stack size
   325  		"issue42058a.go", // types2 does not have constraints on channel element size
   326  		"issue42058b.go", // types2 does not have constraints on channel element size
   327  		"issue48097.go",  // go/types doesn't check validity of //go:xxx directives, and non-init bodyless function
   328  		"issue48230.go",  // go/types doesn't check validity of //go:xxx directives
   329  		"issue49767.go",  // go/types does not have constraints on channel element size
   330  		"issue49814.go",  // go/types does not have constraints on array size
   331  		"issue78355.go",  // types2 does not have constraints on map element size
   332  		"issue56103.go",  // anonymous interface cycles; will be a type checker error in 1.22
   333  		"issue52697.go",  // types2 does not have constraints on stack size
   334  
   335  		// These tests requires runtime/cgo.Incomplete, which is only available on some platforms.
   336  		// However, types2 does not know about build constraints.
   337  		"bug514.go",
   338  		"issue40954.go",
   339  		"issue42032.go",
   340  		"issue42076.go",
   341  		"issue46903.go",
   342  		"issue51733.go",
   343  		"notinheap2.go",
   344  		"notinheap3.go",
   345  	)
   346  }
   347  
   348  func TestStdKen(t *testing.T) {
   349  	testenv.MustHaveGoBuild(t)
   350  
   351  	testTestDir(t, filepath.Join(testenv.GOROOT(t), "test", "ken"))
   352  }
   353  
   354  // Package paths of excluded packages.
   355  var excluded = map[string]bool{
   356  	"builtin":                       true,
   357  	"cmd/compile/internal/ssa/_gen": true,
   358  	"crypto/internal/cryptotest/wycheproof/_schema": true,
   359  	"crypto/internal/cryptotest/x509limbo/_schema":  true,
   360  	"runtime/_mkmalloc":                             true,
   361  	"simd/archsimd/_gen/midway":                     true,
   362  	"simd/archsimd/_gen/sgutil":                     true,
   363  	"simd/archsimd/_gen/simdgen":                    true,
   364  	"simd/archsimd/_gen/simdgen/arm64":              true,
   365  	"simd/archsimd/_gen/tmplgen":                    true,
   366  	"simd/archsimd/_gen/unify":                      true,
   367  	"simd/archsimd/_gen/wasmgen":                    true,
   368  }
   369  
   370  // printPackageMu synchronizes the printing of type-checked package files in
   371  // the typecheckFiles function.
   372  //
   373  // Without synchronization, package files may be interleaved during concurrent
   374  // type-checking.
   375  var printPackageMu sync.Mutex
   376  
   377  // typecheckFiles typechecks the given package files.
   378  func typecheckFiles(path string, filenames []string, importer Importer) (*Package, error) {
   379  	// Parse package files.
   380  	var files []*syntax.File
   381  	for _, filename := range filenames {
   382  		var errs []error
   383  		errh := func(err error) { errs = append(errs, err) }
   384  		file, err := syntax.ParseFile(filename, errh, nil, 0)
   385  		if err != nil {
   386  			return nil, errors.Join(errs...)
   387  		}
   388  
   389  		files = append(files, file)
   390  	}
   391  
   392  	if testing.Verbose() {
   393  		printPackageMu.Lock()
   394  		fmt.Println("package", files[0].PkgName.Value)
   395  		for _, filename := range filenames {
   396  			fmt.Println("\t", filename)
   397  		}
   398  		printPackageMu.Unlock()
   399  	}
   400  
   401  	// Typecheck package files.
   402  	var errs []error
   403  	conf := Config{
   404  		Error: func(err error) {
   405  			errs = append(errs, err)
   406  		},
   407  		Importer: importer,
   408  	}
   409  	info := Info{Uses: make(map[*syntax.Name]Object)}
   410  	pkg, _ := conf.Check(path, files, &info)
   411  	err := errors.Join(errs...)
   412  	if err != nil {
   413  		return pkg, err
   414  	}
   415  
   416  	// Perform checks of API invariants.
   417  
   418  	// All Objects have a package, except predeclared ones.
   419  	errorError := Universe.Lookup("error").Type().Underlying().(*Interface).ExplicitMethod(0) // (error).Error
   420  	for id, obj := range info.Uses {
   421  		predeclared := obj == Universe.Lookup(obj.Name()) || obj == errorError
   422  		if predeclared == (obj.Pkg() != nil) {
   423  			posn := id.Pos()
   424  			if predeclared {
   425  				return nil, fmt.Errorf("%s: predeclared object with package: %s", posn, obj)
   426  			} else {
   427  				return nil, fmt.Errorf("%s: user-defined object without package: %s", posn, obj)
   428  			}
   429  		}
   430  	}
   431  
   432  	return pkg, nil
   433  }
   434  
   435  // pkgFilenames returns the list of package filenames for the given directory.
   436  func pkgFilenames(dir string, includeTest bool) ([]string, error) {
   437  	ctxt := build.Default
   438  	ctxt.CgoEnabled = false
   439  	pkg, err := ctxt.ImportDir(dir, 0)
   440  	if err != nil {
   441  		if _, nogo := err.(*build.NoGoError); nogo {
   442  			return nil, nil // no *.go files, not an error
   443  		}
   444  		return nil, err
   445  	}
   446  	if excluded[pkg.ImportPath] {
   447  		return nil, nil
   448  	}
   449  	if slices.Contains(strings.Split(pkg.ImportPath, "/"), "_asm") {
   450  		// Submodules where not all dependencies are available.
   451  		// See go.dev/issue/46027.
   452  		return nil, nil
   453  	}
   454  	var filenames []string
   455  	for _, name := range pkg.GoFiles {
   456  		filenames = append(filenames, filepath.Join(pkg.Dir, name))
   457  	}
   458  	if includeTest {
   459  		for _, name := range pkg.TestGoFiles {
   460  			filenames = append(filenames, filepath.Join(pkg.Dir, name))
   461  		}
   462  	}
   463  	return filenames, nil
   464  }
   465  
   466  func walkPkgDirs(dir string, pkgh func(dir string, filenames []string), errh func(args ...any)) {
   467  	w := walker{pkgh, errh}
   468  	w.walk(dir)
   469  }
   470  
   471  type walker struct {
   472  	pkgh func(dir string, filenames []string)
   473  	errh func(args ...any)
   474  }
   475  
   476  func (w *walker) walk(dir string) {
   477  	files, err := os.ReadDir(dir)
   478  	if err != nil {
   479  		w.errh(err)
   480  		return
   481  	}
   482  
   483  	// apply pkgh to the files in directory dir
   484  
   485  	// Don't get test files as these packages are imported.
   486  	pkgFiles, err := pkgFilenames(dir, false)
   487  	if err != nil {
   488  		w.errh(err)
   489  		return
   490  	}
   491  	if pkgFiles != nil {
   492  		w.pkgh(dir, pkgFiles)
   493  	}
   494  
   495  	// traverse subdirectories, but don't walk into testdata
   496  	for _, f := range files {
   497  		if f.IsDir() && f.Name() != "testdata" {
   498  			w.walk(filepath.Join(dir, f.Name()))
   499  		}
   500  	}
   501  }
   502
View as plain text