tests.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package tests
     6  
     7  import (
     8  	_ "embed"
     9  	"go/ast"
    10  	"go/token"
    11  	"go/types"
    12  	"regexp"
    13  	"strings"
    14  	"unicode"
    15  	"unicode/utf8"
    16  
    17  	"golang.org/x/tools/go/analysis"
    18  	"golang.org/x/tools/internal/analysis/analyzerutil"
    19  	"golang.org/x/tools/internal/astutil"
    20  	"golang.org/x/tools/internal/typesinternal"
    21  )
    22  
    23  //go:embed doc.go
    24  var doc string
    25  
    26  var Analyzer = &analysis.Analyzer{
    27  	Name: "tests",
    28  	Doc:  analyzerutil.MustExtractDoc(doc, "tests"),
    29  	URL:  "https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/tests",
    30  	Run:  run,
    31  }
    32  
    33  var acceptedFuzzTypes = []types.Type{
    34  	types.Typ[types.String],
    35  	types.Typ[types.Bool],
    36  	types.Typ[types.Float32],
    37  	types.Typ[types.Float64],
    38  	types.Typ[types.Int],
    39  	types.Typ[types.Int8],
    40  	types.Typ[types.Int16],
    41  	types.Typ[types.Int32],
    42  	types.Typ[types.Int64],
    43  	types.Typ[types.Uint],
    44  	types.Typ[types.Uint8],
    45  	types.Typ[types.Uint16],
    46  	types.Typ[types.Uint32],
    47  	types.Typ[types.Uint64],
    48  	types.NewSlice(types.Universe.Lookup("byte").Type()),
    49  }
    50  
    51  func run(pass *analysis.Pass) (any, error) {
    52  	for _, f := range pass.Files {
    53  		if !strings.HasSuffix(pass.Fset.File(f.FileStart).Name(), "_test.go") {
    54  			continue
    55  		}
    56  		for _, decl := range f.Decls {
    57  			fn, ok := decl.(*ast.FuncDecl)
    58  			if !ok || fn.Recv != nil {
    59  				// Ignore non-functions or functions with receivers.
    60  				continue
    61  			}
    62  			switch {
    63  			case strings.HasPrefix(fn.Name.Name, "Example"):
    64  				checkExampleName(pass, fn)
    65  				checkExampleOutput(pass, fn, f.Comments)
    66  			case strings.HasPrefix(fn.Name.Name, "Test"):
    67  				checkTest(pass, fn, "Test")
    68  			case strings.HasPrefix(fn.Name.Name, "Benchmark"):
    69  				checkTest(pass, fn, "Benchmark")
    70  			case strings.HasPrefix(fn.Name.Name, "Fuzz"):
    71  				checkTest(pass, fn, "Fuzz")
    72  				checkFuzz(pass, fn)
    73  			}
    74  		}
    75  	}
    76  	return nil, nil
    77  }
    78  
    79  // checkFuzz checks the contents of a fuzz function.
    80  func checkFuzz(pass *analysis.Pass, fn *ast.FuncDecl) {
    81  	params := checkFuzzCall(pass, fn)
    82  	if params != nil {
    83  		checkAddCalls(pass, fn, params)
    84  	}
    85  }
    86  
    87  // checkFuzzCall checks the arguments of f.Fuzz() calls:
    88  //
    89  //  1. f.Fuzz() should call a function and it should be of type (*testing.F).Fuzz().
    90  //  2. The called function in f.Fuzz(func(){}) should not return result.
    91  //  3. First argument of func() should be of type *testing.T
    92  //  4. Second argument onwards should be of type []byte, string, bool, byte,
    93  //     rune, float32, float64, int, int8, int16, int32, int64, uint, uint8, uint16,
    94  //     uint32, uint64
    95  //  5. func() must not call any *F methods, e.g. (*F).Log, (*F).Error, (*F).Skip
    96  //     The only *F methods that are allowed in the (*F).Fuzz function are (*F).Failed and (*F).Name.
    97  //
    98  // Returns the list of parameters to the fuzz function, if they are valid fuzz parameters.
    99  func checkFuzzCall(pass *analysis.Pass, fn *ast.FuncDecl) (params *types.Tuple) {
   100  	ast.Inspect(fn, func(n ast.Node) bool {
   101  		call, ok := n.(*ast.CallExpr)
   102  		if ok {
   103  			if !isFuzzTargetDotFuzz(pass, call) {
   104  				return true
   105  			}
   106  
   107  			// Only one argument (func) must be passed to (*testing.F).Fuzz.
   108  			if len(call.Args) != 1 {
   109  				return true
   110  			}
   111  			expr := call.Args[0]
   112  			if pass.TypesInfo.Types[expr].Type == nil {
   113  				return true
   114  			}
   115  			t := pass.TypesInfo.Types[expr].Type.Underlying()
   116  			tSign, argOk := t.(*types.Signature)
   117  			// Argument should be a function
   118  			if !argOk {
   119  				pass.ReportRangef(expr, "argument to Fuzz must be a function")
   120  				return false
   121  			}
   122  			// ff Argument function should not return
   123  			if tSign.Results().Len() != 0 {
   124  				pass.ReportRangef(expr, "fuzz target must not return any value")
   125  			}
   126  			// ff Argument function should have 1 or more argument
   127  			if tSign.Params().Len() == 0 {
   128  				pass.ReportRangef(expr, "fuzz target must have 1 or more argument")
   129  				return false
   130  			}
   131  			ok := validateFuzzArgs(pass, tSign.Params(), expr)
   132  			if ok && params == nil {
   133  				params = tSign.Params()
   134  			}
   135  			// Inspect the function that was passed as an argument to make sure that
   136  			// there are no calls to *F methods, except for Name and Failed.
   137  			ast.Inspect(expr, func(n ast.Node) bool {
   138  				if call, ok := n.(*ast.CallExpr); ok {
   139  					if !isFuzzTargetDot(pass, call, "") {
   140  						return true
   141  					}
   142  					if !isFuzzTargetDot(pass, call, "Name") && !isFuzzTargetDot(pass, call, "Failed") {
   143  						pass.ReportRangef(call, "fuzz target must not call any *F methods")
   144  					}
   145  				}
   146  				return true
   147  			})
   148  			// We do not need to look at any calls to f.Fuzz inside of a Fuzz call,
   149  			// since they are not allowed.
   150  			return false
   151  		}
   152  		return true
   153  	})
   154  	return params
   155  }
   156  
   157  // checkAddCalls checks that the arguments of f.Add calls have the same number and type of arguments as
   158  // the signature of the function passed to (*testing.F).Fuzz
   159  func checkAddCalls(pass *analysis.Pass, fn *ast.FuncDecl, params *types.Tuple) {
   160  	ast.Inspect(fn, func(n ast.Node) bool {
   161  		call, ok := n.(*ast.CallExpr)
   162  		if ok {
   163  			if !isFuzzTargetDotAdd(pass, call) {
   164  				return true
   165  			}
   166  
   167  			// The first argument to function passed to (*testing.F).Fuzz is (*testing.T).
   168  			if len(call.Args) != params.Len()-1 {
   169  				pass.ReportRangef(call, "wrong number of values in call to (*testing.F).Add: %d, fuzz target expects %d", len(call.Args), params.Len()-1)
   170  				return true
   171  			}
   172  			var mismatched []int
   173  			for i, expr := range call.Args {
   174  				if pass.TypesInfo.Types[expr].Type == nil {
   175  					return true
   176  				}
   177  				t := pass.TypesInfo.Types[expr].Type
   178  				if !types.Identical(t, params.At(i+1).Type()) {
   179  					mismatched = append(mismatched, i)
   180  				}
   181  			}
   182  			// If just one of the types is mismatched report for that
   183  			// type only. Otherwise report for the whole call to (*testing.F).Add
   184  			if len(mismatched) == 1 {
   185  				i := mismatched[0]
   186  				expr := call.Args[i]
   187  				t := pass.TypesInfo.Types[expr].Type
   188  				pass.ReportRangef(expr, "mismatched type in call to (*testing.F).Add: %v, fuzz target expects %v", t, params.At(i+1).Type())
   189  			} else if len(mismatched) > 1 {
   190  				var gotArgs, wantArgs []types.Type
   191  				for i := 0; i < len(call.Args); i++ {
   192  					gotArgs, wantArgs = append(gotArgs, pass.TypesInfo.Types[call.Args[i]].Type), append(wantArgs, params.At(i+1).Type())
   193  				}
   194  				pass.ReportRangef(call, "mismatched types in call to (*testing.F).Add: %v, fuzz target expects %v", gotArgs, wantArgs)
   195  			}
   196  		}
   197  		return true
   198  	})
   199  }
   200  
   201  // isFuzzTargetDotFuzz reports whether call is (*testing.F).Fuzz().
   202  func isFuzzTargetDotFuzz(pass *analysis.Pass, call *ast.CallExpr) bool {
   203  	return isFuzzTargetDot(pass, call, "Fuzz")
   204  }
   205  
   206  // isFuzzTargetDotAdd reports whether call is (*testing.F).Add().
   207  func isFuzzTargetDotAdd(pass *analysis.Pass, call *ast.CallExpr) bool {
   208  	return isFuzzTargetDot(pass, call, "Add")
   209  }
   210  
   211  // isFuzzTargetDot reports whether call is (*testing.F).<name>().
   212  func isFuzzTargetDot(pass *analysis.Pass, call *ast.CallExpr, name string) bool {
   213  	if selExpr, ok := call.Fun.(*ast.SelectorExpr); ok {
   214  		if !isTestingType(pass.TypesInfo.Types[selExpr.X].Type, "F") {
   215  			return false
   216  		}
   217  		if name == "" || selExpr.Sel.Name == name {
   218  			return true
   219  		}
   220  	}
   221  	return false
   222  }
   223  
   224  // Validate the arguments of fuzz target.
   225  func validateFuzzArgs(pass *analysis.Pass, params *types.Tuple, expr ast.Expr) bool {
   226  	fLit, isFuncLit := expr.(*ast.FuncLit)
   227  	exprRange := expr
   228  	ok := true
   229  	if !isTestingType(params.At(0).Type(), "T") {
   230  		if isFuncLit {
   231  			exprRange = fLit.Type.Params.List[0].Type
   232  		}
   233  		pass.ReportRangef(exprRange, "the first parameter of a fuzz target must be *testing.T")
   234  		ok = false
   235  	}
   236  	for i := 1; i < params.Len(); i++ {
   237  		if !isAcceptedFuzzType(params.At(i).Type()) {
   238  			if isFuncLit {
   239  				curr := 0
   240  				for _, field := range fLit.Type.Params.List {
   241  					curr += len(field.Names)
   242  					if i < curr {
   243  						exprRange = field.Type
   244  						break
   245  					}
   246  				}
   247  			}
   248  			pass.ReportRangef(exprRange, "fuzzing arguments can only have the following types: %s", formatAcceptedFuzzType())
   249  			ok = false
   250  		}
   251  	}
   252  	return ok
   253  }
   254  
   255  func isTestingType(typ types.Type, testingType string) bool {
   256  	// No Unalias here: I doubt "go test" recognizes
   257  	// "type A = *testing.T; func Test(A) {}" as a test.
   258  	ptr, ok := typ.(*types.Pointer)
   259  	if !ok {
   260  		return false
   261  	}
   262  	return typesinternal.IsTypeNamed(ptr.Elem(), "testing", testingType)
   263  }
   264  
   265  // Validate that fuzz target function's arguments are of accepted types.
   266  func isAcceptedFuzzType(paramType types.Type) bool {
   267  	for _, typ := range acceptedFuzzTypes {
   268  		if types.Identical(typ, paramType) {
   269  			return true
   270  		}
   271  	}
   272  	return false
   273  }
   274  
   275  func formatAcceptedFuzzType() string {
   276  	var acceptedFuzzTypesStrings []string
   277  	for _, typ := range acceptedFuzzTypes {
   278  		acceptedFuzzTypesStrings = append(acceptedFuzzTypesStrings, typ.String())
   279  	}
   280  	acceptedFuzzTypesMsg := strings.Join(acceptedFuzzTypesStrings, ", ")
   281  	return acceptedFuzzTypesMsg
   282  }
   283  
   284  func isExampleSuffix(s string) bool {
   285  	r, size := utf8.DecodeRuneInString(s)
   286  	return size > 0 && unicode.IsLower(r)
   287  }
   288  
   289  func isTestSuffix(name string) bool {
   290  	if len(name) == 0 {
   291  		// "Test" is ok.
   292  		return true
   293  	}
   294  	r, _ := utf8.DecodeRuneInString(name)
   295  	return !unicode.IsLower(r)
   296  }
   297  
   298  func isTestParam(typ ast.Expr, wantType string) bool {
   299  	ptr, ok := typ.(*ast.StarExpr)
   300  	if !ok {
   301  		// Not a pointer.
   302  		return false
   303  	}
   304  	// No easy way of making sure it's a *testing.T or *testing.B:
   305  	// ensure the name of the type matches.
   306  	if name, ok := ptr.X.(*ast.Ident); ok {
   307  		return name.Name == wantType
   308  	}
   309  	if sel, ok := ptr.X.(*ast.SelectorExpr); ok {
   310  		return sel.Sel.Name == wantType
   311  	}
   312  	return false
   313  }
   314  
   315  func lookup(pkg *types.Package, name string) []types.Object {
   316  	if o := pkg.Scope().Lookup(name); o != nil {
   317  		return []types.Object{o}
   318  	}
   319  
   320  	var ret []types.Object
   321  	// Search through the imports to see if any of them define name.
   322  	// It's hard to tell in general which package is being tested, so
   323  	// for the purposes of the analysis, allow the object to appear
   324  	// in any of the imports. This guarantees there are no false positives
   325  	// because the example needs to use the object so it must be defined
   326  	// in the package or one if its imports. On the other hand, false
   327  	// negatives are possible, but should be rare.
   328  	for _, imp := range pkg.Imports() {
   329  		if obj := imp.Scope().Lookup(name); obj != nil {
   330  			ret = append(ret, obj)
   331  		}
   332  	}
   333  	return ret
   334  }
   335  
   336  // This pattern is taken from /go/src/go/doc/example.go
   337  var outputRe = regexp.MustCompile(`(?i)^[[:space:]]*(unordered )?output:`)
   338  
   339  type commentMetadata struct {
   340  	isOutput bool
   341  	pos      token.Pos
   342  }
   343  
   344  func checkExampleOutput(pass *analysis.Pass, fn *ast.FuncDecl, fileComments []*ast.CommentGroup) {
   345  	commentsInExample := []commentMetadata{}
   346  	numOutputs := 0
   347  
   348  	// Find the comment blocks that are in the example. These comments are
   349  	// guaranteed to be in order of appearance.
   350  	for _, cg := range fileComments {
   351  		if cg.Pos() < fn.Pos() {
   352  			continue
   353  		} else if cg.End() > fn.End() {
   354  			break
   355  		}
   356  
   357  		isOutput := outputRe.MatchString(cg.Text())
   358  		if isOutput {
   359  			numOutputs++
   360  		}
   361  
   362  		commentsInExample = append(commentsInExample, commentMetadata{
   363  			isOutput: isOutput,
   364  			pos:      cg.Pos(),
   365  		})
   366  	}
   367  
   368  	// Change message based on whether there are multiple output comment blocks.
   369  	msg := "output comment block must be the last comment block"
   370  	if numOutputs > 1 {
   371  		msg = "there can only be one output comment block per example"
   372  	}
   373  
   374  	for i, cg := range commentsInExample {
   375  		// Check for output comments that are not the last comment in the example.
   376  		isLast := (i == len(commentsInExample)-1)
   377  		if cg.isOutput && !isLast {
   378  			pass.Report(
   379  				analysis.Diagnostic{
   380  					Pos:     cg.pos,
   381  					Message: msg,
   382  				},
   383  			)
   384  		}
   385  	}
   386  }
   387  
   388  func checkExampleName(pass *analysis.Pass, fn *ast.FuncDecl) {
   389  	fnName := fn.Name.Name
   390  	if params := fn.Type.Params; len(params.List) != 0 {
   391  		pass.Reportf(fn.Pos(), "%s should be niladic", fnName)
   392  	}
   393  	if results := fn.Type.Results; results != nil && len(results.List) != 0 {
   394  		pass.Reportf(fn.Pos(), "%s should return nothing", fnName)
   395  	}
   396  	if tparams := fn.Type.TypeParams; tparams != nil && len(tparams.List) > 0 {
   397  		pass.Reportf(fn.Pos(), "%s should not have type params", fnName)
   398  	}
   399  
   400  	if fnName == "Example" {
   401  		// Nothing more to do.
   402  		return
   403  	}
   404  
   405  	var (
   406  		exName = strings.TrimPrefix(fnName, "Example")
   407  		elems  = strings.SplitN(exName, "_", 3)
   408  		ident  = elems[0]
   409  		objs   = lookup(pass.Pkg, ident)
   410  	)
   411  	if ident != "" && len(objs) == 0 {
   412  		// Check ExampleFoo and ExampleBadFoo.
   413  		pass.Reportf(fn.Pos(), "%s refers to unknown identifier: %s", fnName, ident)
   414  		// Abort since obj is absent and no subsequent checks can be performed.
   415  		return
   416  	}
   417  	if len(elems) < 2 {
   418  		// Nothing more to do.
   419  		return
   420  	}
   421  
   422  	if ident == "" {
   423  		// Check Example_suffix and Example_BadSuffix.
   424  		if residual := strings.TrimPrefix(exName, "_"); !isExampleSuffix(residual) {
   425  			pass.Reportf(fn.Pos(), "%s has malformed example suffix: %s", fnName, residual)
   426  		}
   427  		return
   428  	}
   429  
   430  	mmbr := elems[1]
   431  	if !isExampleSuffix(mmbr) {
   432  		// Check ExampleFoo_Method and ExampleFoo_BadMethod.
   433  		found := false
   434  		// Check if Foo.Method exists in this package or its imports.
   435  		for _, obj := range objs {
   436  			if obj, _, _ := types.LookupFieldOrMethod(obj.Type(), true, obj.Pkg(), mmbr); obj != nil {
   437  				found = true
   438  				break
   439  			}
   440  		}
   441  		if !found {
   442  			pass.Reportf(fn.Pos(), "%s refers to unknown field or method: %s.%s", fnName, ident, mmbr)
   443  		}
   444  	}
   445  	if len(elems) == 3 && !isExampleSuffix(elems[2]) {
   446  		// Check ExampleFoo_Method_suffix and ExampleFoo_Method_Badsuffix.
   447  		pass.Reportf(fn.Pos(), "%s has malformed example suffix: %s", fnName, elems[2])
   448  	}
   449  }
   450  
   451  func checkTest(pass *analysis.Pass, fn *ast.FuncDecl, prefix string) {
   452  	// Want functions with 0 results and 1 parameter.
   453  	if fn.Type.Results != nil && len(fn.Type.Results.List) > 0 ||
   454  		fn.Type.Params == nil ||
   455  		len(fn.Type.Params.List) != 1 ||
   456  		len(fn.Type.Params.List[0].Names) > 1 {
   457  		return
   458  	}
   459  
   460  	// The param must look like a *testing.T or *testing.B.
   461  	if !isTestParam(fn.Type.Params.List[0].Type, prefix[:1]) {
   462  		return
   463  	}
   464  
   465  	if tparams := fn.Type.TypeParams; tparams != nil && len(tparams.List) > 0 {
   466  		// Note: cmd/go/internal/load also errors about TestXXX and BenchmarkXXX functions with type parameters.
   467  		// We have currently decided to also warn before compilation/package loading. This can help users in IDEs.
   468  		pass.ReportRangef(astutil.RangeOf(tparams.Opening, tparams.Closing),
   469  			"%s has type parameters: it will not be run by go test as a %sXXX function",
   470  			fn.Name.Name, prefix)
   471  	}
   472  
   473  	if !isTestSuffix(fn.Name.Name[len(prefix):]) {
   474  		pass.ReportRangef(fn.Name, "%s has malformed name: first letter after '%s' must not be lowercase", fn.Name.Name, prefix)
   475  	}
   476  }
   477
View as plain text