Source file src/cmd/compile/internal/inline/inl.go

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  //
     5  // The inlining facility makes 2 passes: first CanInline determines which
     6  // functions are suitable for inlining, and for those that are it
     7  // saves a copy of the body. Then InlineCalls walks each function body to
     8  // expand calls to inlinable functions.
     9  //
    10  // The Debug.l flag controls the aggressiveness. Note that main() swaps level 0 and 1,
    11  // making 1 the default and -l disable. Additional levels (beyond -l) may be buggy and
    12  // are not supported.
    13  //      0: disabled
    14  //      1: 80-nodes leaf functions, oneliners, panic, lazy typechecking (default)
    15  //      2: (unassigned)
    16  //      3: (unassigned)
    17  //      4: allow non-leaf functions
    18  //
    19  // At some point this may get another default and become switch-offable with -N.
    20  //
    21  // The -d typcheckinl flag enables early typechecking of all imported bodies,
    22  // which is useful to flush out bugs.
    23  //
    24  // The Debug.m flag enables diagnostic output.  a single -m is useful for verifying
    25  // which calls get inlined or not, more is for debugging, and may go away at any point.
    26  
    27  package inline
    28  
    29  import (
    30  	"fmt"
    31  	"go/constant"
    32  	"internal/buildcfg"
    33  	"strconv"
    34  	"strings"
    35  
    36  	"cmd/compile/internal/base"
    37  	"cmd/compile/internal/inline/inlheur"
    38  	"cmd/compile/internal/ir"
    39  	"cmd/compile/internal/logopt"
    40  	"cmd/compile/internal/pgoir"
    41  	"cmd/compile/internal/typecheck"
    42  	"cmd/compile/internal/types"
    43  	"cmd/internal/obj"
    44  	"cmd/internal/pgo"
    45  	"cmd/internal/src"
    46  )
    47  
    48  // Inlining budget parameters, gathered in one place
    49  const (
    50  	inlineMaxBudget       = 80
    51  	inlineExtraAppendCost = 0
    52  	// default is to inline if there's at most one call. -l=4 overrides this by using 1 instead.
    53  	inlineExtraCallCost  = 57              // 57 was benchmarked to provided most benefit with no bad surprises; see https://github.com/golang/go/issues/19348#issuecomment-439370742
    54  	inlineParamCallCost  = 17              // calling a parameter only costs this much extra (inlining might expose a constant function)
    55  	inlineExtraPanicCost = 1               // do not penalize inlining panics.
    56  	inlineExtraThrowCost = inlineMaxBudget // with current (2018-05/1.11) code, inlining runtime.throw does not help.
    57  
    58  	inlineBigFunctionNodes      = 5000                 // Functions with this many nodes are considered "big".
    59  	inlineBigFunctionMaxCost    = 20                   // Max cost of inlinee when inlining into a "big" function.
    60  	inlineClosureCalledOnceCost = 10 * inlineMaxBudget // if a closure is just called once, inline it.
    61  )
    62  
    63  var (
    64  	// List of all hot callee nodes.
    65  	// TODO(prattmic): Make this non-global.
    66  	candHotCalleeMap = make(map[*pgoir.IRNode]struct{})
    67  
    68  	// Set of functions that contain hot call sites.
    69  	hasHotCall = make(map[*ir.Func]struct{})
    70  
    71  	// List of all hot call sites. CallSiteInfo.Callee is always nil.
    72  	// TODO(prattmic): Make this non-global.
    73  	candHotEdgeMap = make(map[pgoir.CallSiteInfo]struct{})
    74  
    75  	// Threshold in percentage for hot callsite inlining.
    76  	inlineHotCallSiteThresholdPercent float64
    77  
    78  	// Threshold in CDF percentage for hot callsite inlining,
    79  	// that is, for a threshold of X the hottest callsites that
    80  	// make up the top X% of total edge weight will be
    81  	// considered hot for inlining candidates.
    82  	inlineCDFHotCallSiteThresholdPercent = float64(99)
    83  
    84  	// Budget increased due to hotness.
    85  	inlineHotMaxBudget int32 = 2000
    86  )
    87  
    88  func IsPgoHotFunc(fn *ir.Func, profile *pgoir.Profile) bool {
    89  	if profile == nil {
    90  		return false
    91  	}
    92  	if n, ok := profile.WeightedCG.IRNodes[ir.LinkFuncName(fn)]; ok {
    93  		_, ok := candHotCalleeMap[n]
    94  		return ok
    95  	}
    96  	return false
    97  }
    98  
    99  func HasPgoHotInline(fn *ir.Func) bool {
   100  	_, has := hasHotCall[fn]
   101  	return has
   102  }
   103  
   104  // PGOInlinePrologue records the hot callsites from ir-graph.
   105  func PGOInlinePrologue(p *pgoir.Profile) {
   106  	if base.Debug.PGOInlineCDFThreshold != "" {
   107  		if s, err := strconv.ParseFloat(base.Debug.PGOInlineCDFThreshold, 64); err == nil && s >= 0 && s <= 100 {
   108  			inlineCDFHotCallSiteThresholdPercent = s
   109  		} else {
   110  			base.Fatalf("invalid PGOInlineCDFThreshold, must be between 0 and 100")
   111  		}
   112  	}
   113  	var hotCallsites []pgo.NamedCallEdge
   114  	inlineHotCallSiteThresholdPercent, hotCallsites = hotNodesFromCDF(p)
   115  	if base.Debug.PGODebug > 0 {
   116  		fmt.Printf("hot-callsite-thres-from-CDF=%v\n", inlineHotCallSiteThresholdPercent)
   117  	}
   118  
   119  	if x := base.Debug.PGOInlineBudget; x != 0 {
   120  		inlineHotMaxBudget = int32(x)
   121  	}
   122  
   123  	for _, n := range hotCallsites {
   124  		// mark inlineable callees from hot edges
   125  		if callee := p.WeightedCG.IRNodes[n.CalleeName]; callee != nil {
   126  			candHotCalleeMap[callee] = struct{}{}
   127  		}
   128  		// mark hot call sites
   129  		if caller := p.WeightedCG.IRNodes[n.CallerName]; caller != nil && caller.AST != nil {
   130  			csi := pgoir.CallSiteInfo{LineOffset: n.CallSiteOffset, Caller: caller.AST}
   131  			candHotEdgeMap[csi] = struct{}{}
   132  		}
   133  	}
   134  
   135  	if base.Debug.PGODebug >= 3 {
   136  		fmt.Printf("hot-cg before inline in dot format:")
   137  		p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent)
   138  	}
   139  }
   140  
   141  // hotNodesFromCDF computes an edge weight threshold and the list of hot
   142  // nodes that make up the given percentage of the CDF. The threshold, as
   143  // a percent, is the lower bound of weight for nodes to be considered hot
   144  // (currently only used in debug prints) (in case of equal weights,
   145  // comparing with the threshold may not accurately reflect which nodes are
   146  // considered hot).
   147  func hotNodesFromCDF(p *pgoir.Profile) (float64, []pgo.NamedCallEdge) {
   148  	cum := int64(0)
   149  	for i, n := range p.NamedEdgeMap.ByWeight {
   150  		w := p.NamedEdgeMap.Weight[n]
   151  		cum += w
   152  		if pgo.WeightInPercentage(cum, p.TotalWeight) > inlineCDFHotCallSiteThresholdPercent {
   153  			// nodes[:i+1] to include the very last node that makes it to go over the threshold.
   154  			// (Say, if the CDF threshold is 50% and one hot node takes 60% of weight, we want to
   155  			// include that node instead of excluding it.)
   156  			return pgo.WeightInPercentage(w, p.TotalWeight), p.NamedEdgeMap.ByWeight[:i+1]
   157  		}
   158  	}
   159  	return 0, p.NamedEdgeMap.ByWeight
   160  }
   161  
   162  // CanInlineFuncs computes whether a batch of functions are inlinable.
   163  func CanInlineFuncs(funcs []*ir.Func, profile *pgoir.Profile) {
   164  	if profile != nil {
   165  		PGOInlinePrologue(profile)
   166  	}
   167  
   168  	if base.Flag.LowerL == 0 {
   169  		return
   170  	}
   171  
   172  	ir.VisitFuncsBottomUp(funcs, func(funcs []*ir.Func, recursive bool) {
   173  		for _, fn := range funcs {
   174  			CanInline(fn, profile)
   175  			if inlheur.Enabled() {
   176  				analyzeFuncProps(fn, profile)
   177  			}
   178  		}
   179  	})
   180  }
   181  
   182  func simdCreditMultiplier(fn *ir.Func) int32 {
   183  	for _, field := range fn.Type().RecvParamsResults() {
   184  		if field.Type.IsSIMD() {
   185  			return 3
   186  		}
   187  	}
   188  	// Sometimes code uses closures, that do not take simd
   189  	// parameters, to perform repetitive SIMD operations.
   190  	// fn.  These really need to be inlined, or the anticipated
   191  	// awesome SIMD performance will be missed.
   192  	for _, v := range fn.ClosureVars {
   193  		if v.Type().IsSIMD() {
   194  			return 16 // <strike>11</strike> 16 ought to be enough.
   195  		}
   196  	}
   197  
   198  	return 1
   199  }
   200  
   201  // inlineBudget determines the max budget for function 'fn' prior to
   202  // analyzing the hairiness of the body of 'fn'. We pass in the pgo
   203  // profile if available (which can change the budget), also a
   204  // 'relaxed' flag, which expands the budget slightly to allow for the
   205  // possibility that a call to the function might have its score
   206  // adjusted downwards. If 'verbose' is set, then print a remark where
   207  // we boost the budget due to PGO.
   208  // Note that inlineCostOK has the final say on whether an inline will
   209  // happen; changes here merely make inlines possible.
   210  func inlineBudget(fn *ir.Func, profile *pgoir.Profile, relaxed bool, verbose bool) int32 {
   211  	// Update the budget for profile-guided inlining.
   212  	budget := int32(inlineMaxBudget)
   213  
   214  	budget *= simdCreditMultiplier(fn)
   215  
   216  	if strings.HasPrefix(ir.FuncName(fn), "runtime_mapaccess2") &&
   217  		fn.Sym().Pkg.Path == "internal/runtime/maps" {
   218  		// Increase budget for mapaccess2* functions so they could be
   219  		// inlined to mapaccess1* wrappers
   220  		budget = inlineHotMaxBudget
   221  		if verbose {
   222  			fmt.Printf("mapaccess enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn))
   223  		}
   224  	}
   225  
   226  	if IsPgoHotFunc(fn, profile) {
   227  		budget = inlineHotMaxBudget
   228  		if verbose {
   229  			fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn))
   230  		}
   231  	}
   232  	if relaxed {
   233  		budget += inlheur.BudgetExpansion(inlineMaxBudget)
   234  	}
   235  	if fn.ClosureParent != nil {
   236  		// be very liberal here, if the closure is only called once, the budget is large
   237  		budget = max(budget, inlineClosureCalledOnceCost)
   238  	}
   239  
   240  	return budget
   241  }
   242  
   243  // CanInline determines whether fn is inlineable.
   244  // If so, CanInline saves copies of fn.Body and fn.Dcl in fn.Inl.
   245  // fn and fn.Body will already have been typechecked.
   246  func CanInline(fn *ir.Func, profile *pgoir.Profile) {
   247  	if fn.Nname == nil {
   248  		base.Fatalf("CanInline no nname %+v", fn)
   249  	}
   250  
   251  	var reason string // reason, if any, that the function was not inlined
   252  	if base.Flag.LowerM > 1 || logopt.Enabled() {
   253  		defer func() {
   254  			if reason != "" {
   255  				if base.Flag.LowerM > 1 {
   256  					fmt.Printf("%v: cannot inline %v: %s\n", ir.Line(fn), fn.Nname, reason)
   257  				}
   258  				if logopt.Enabled() {
   259  					logopt.LogOpt(fn.Pos(), "cannotInlineFunction", "inline", ir.FuncName(fn), reason)
   260  				}
   261  			}
   262  		}()
   263  	}
   264  
   265  	reason = InlineImpossible(fn)
   266  	if reason != "" {
   267  		return
   268  	}
   269  	if fn.Typecheck() == 0 {
   270  		base.Fatalf("CanInline on non-typechecked function %v", fn)
   271  	}
   272  
   273  	n := fn.Nname
   274  	if n.Func.InlinabilityChecked() {
   275  		return
   276  	}
   277  	defer n.Func.SetInlinabilityChecked(true)
   278  
   279  	cc := int32(inlineExtraCallCost)
   280  	if base.Flag.LowerL == 4 {
   281  		cc = 1 // this appears to yield better performance than 0.
   282  	}
   283  
   284  	// Used a "relaxed" inline budget if the new inliner is enabled.
   285  	relaxed := inlheur.Enabled()
   286  
   287  	// Compute the inline budget for this func.
   288  	budget := inlineBudget(fn, profile, relaxed, base.Debug.PGODebug > 0)
   289  
   290  	// At this point in the game the function we're looking at may
   291  	// have "stale" autos, vars that still appear in the Dcl list, but
   292  	// which no longer have any uses in the function body (due to
   293  	// elimination by deadcode). We'd like to exclude these dead vars
   294  	// when creating the "Inline.Dcl" field below; to accomplish this,
   295  	// the hairyVisitor below builds up a map of used/referenced
   296  	// locals, and we use this map to produce a pruned Inline.Dcl
   297  	// list. See issue 25459 for more context.
   298  
   299  	dbg := ir.MatchAstDump(fn, "inline")
   300  
   301  	visitor := hairyVisitor{
   302  		curFunc:       fn,
   303  		debug:         isDebugFn(fn),
   304  		isBigFunc:     IsBigFunc(fn),
   305  		budget:        budget,
   306  		maxBudget:     budget,
   307  		extraCallCost: cc,
   308  		profile:       profile,
   309  		dbg:           dbg, // Useful for downstream debugging
   310  	}
   311  
   312  	if visitor.tooHairy(fn) {
   313  		reason = visitor.reason
   314  		if dbg {
   315  			ir.AstDump(fn, "inline, too hairy because "+visitor.reason+", "+ir.FuncName(fn))
   316  		}
   317  		return
   318  	} else if dbg {
   319  		ir.AstDump(fn, "inline, OK, "+ir.FuncName(fn))
   320  	}
   321  
   322  	n.Func.Inl = &ir.Inline{
   323  		Cost:            budget - visitor.budget,
   324  		Dcl:             pruneUnusedAutos(n.Func.Dcl, &visitor),
   325  		HaveDcl:         true,
   326  		CanDelayResults: canDelayResults(fn),
   327  	}
   328  	if base.Flag.LowerM != 0 || logopt.Enabled() {
   329  		noteInlinableFunc(n, fn, budget-visitor.budget)
   330  	}
   331  }
   332  
   333  // noteInlinableFunc issues a message to the user that the specified
   334  // function is inlinable.
   335  func noteInlinableFunc(n *ir.Name, fn *ir.Func, cost int32) {
   336  	if base.Flag.LowerM > 1 {
   337  		fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n.DiagName(), cost, fn.Type(), fn.Body)
   338  	} else if base.Flag.LowerM != 0 {
   339  		fmt.Printf("%v: can inline %v\n", ir.Line(fn), n.DiagName())
   340  	}
   341  	// JSON optimization log output.
   342  	if logopt.Enabled() {
   343  		logopt.LogOpt(fn.Pos(), "canInlineFunction", "inline", ir.FuncName(fn), fmt.Sprintf("cost: %d", cost))
   344  	}
   345  }
   346  
   347  // InlineImpossible returns a non-empty reason string if fn is impossible to
   348  // inline regardless of cost or contents.
   349  func InlineImpossible(fn *ir.Func) string {
   350  	var reason string // reason, if any, that the function can not be inlined.
   351  	if fn.Nname == nil {
   352  		reason = "no name"
   353  		return reason
   354  	}
   355  
   356  	// If marked "go:noinline", don't inline.
   357  	if fn.Pragma&ir.Noinline != 0 {
   358  		reason = "marked go:noinline"
   359  		return reason
   360  	}
   361  
   362  	// If marked "go:norace" and -race compilation, don't inline.
   363  	if base.Flag.Race && fn.Pragma&ir.Norace != 0 {
   364  		reason = "marked go:norace with -race compilation"
   365  		return reason
   366  	}
   367  
   368  	// If marked "go:nocheckptr" and -d checkptr compilation, don't inline.
   369  	if base.Debug.Checkptr != 0 && fn.Pragma&ir.NoCheckPtr != 0 {
   370  		reason = "marked go:nocheckptr"
   371  		return reason
   372  	}
   373  
   374  	// If marked "go:cgo_unsafe_args", don't inline, since the function
   375  	// makes assumptions about its argument frame layout.
   376  	if fn.Pragma&ir.CgoUnsafeArgs != 0 {
   377  		reason = "marked go:cgo_unsafe_args"
   378  		return reason
   379  	}
   380  
   381  	// If marked as "go:uintptrkeepalive", don't inline, since the keep
   382  	// alive information is lost during inlining.
   383  	//
   384  	// TODO(prattmic): This is handled on calls during escape analysis,
   385  	// which is after inlining. Move prior to inlining so the keep-alive is
   386  	// maintained after inlining.
   387  	if fn.Pragma&ir.UintptrKeepAlive != 0 {
   388  		reason = "marked as having a keep-alive uintptr argument"
   389  		return reason
   390  	}
   391  
   392  	// If marked as "go:uintptrescapes", don't inline, since the escape
   393  	// information is lost during inlining.
   394  	if fn.Pragma&ir.UintptrEscapes != 0 {
   395  		reason = "marked as having an escaping uintptr argument"
   396  		return reason
   397  	}
   398  
   399  	// The nowritebarrierrec checker currently works at function
   400  	// granularity, so inlining yeswritebarrierrec functions can confuse it
   401  	// (#22342). As a workaround, disallow inlining them for now.
   402  	if fn.Pragma&ir.Yeswritebarrierrec != 0 {
   403  		reason = "marked go:yeswritebarrierrec"
   404  		return reason
   405  	}
   406  
   407  	// If a local function has no fn.Body (is defined outside of Go), cannot inline it.
   408  	// Imported functions don't have fn.Body but might have inline body in fn.Inl.
   409  	if len(fn.Body) == 0 && !typecheck.HaveInlineBody(fn) {
   410  		reason = "no function body"
   411  		return reason
   412  	}
   413  
   414  	return ""
   415  }
   416  
   417  // canDelayResults reports whether inlined calls to fn can delay
   418  // declaring the result parameter until the "return" statement.
   419  func canDelayResults(fn *ir.Func) bool {
   420  	// We can delay declaring+initializing result parameters if:
   421  	// (1) there's exactly one "return" statement in the inlined function;
   422  	// (2) it's not an empty return statement (#44355); and
   423  	// (3) the result parameters aren't named.
   424  
   425  	nreturns := 0
   426  	ir.VisitList(fn.Body, func(n ir.Node) {
   427  		if n, ok := n.(*ir.ReturnStmt); ok {
   428  			nreturns++
   429  			if len(n.Results) == 0 {
   430  				nreturns++ // empty return statement (case 2)
   431  			}
   432  		}
   433  	})
   434  
   435  	if nreturns != 1 {
   436  		return false // not exactly one return statement (case 1)
   437  	}
   438  
   439  	// temporaries for return values.
   440  	for _, param := range fn.Type().Results() {
   441  		if sym := param.Sym; sym != nil && !sym.IsBlank() {
   442  			return false // found a named result parameter (case 3)
   443  		}
   444  	}
   445  
   446  	return true
   447  }
   448  
   449  // hairyVisitor visits a function body to determine its inlining
   450  // hairiness and whether or not it can be inlined.
   451  type hairyVisitor struct {
   452  	// This is needed to access the current caller in the doNode function.
   453  	curFunc       *ir.Func
   454  	isBigFunc     bool
   455  	debug         bool
   456  	budget        int32
   457  	maxBudget     int32
   458  	reason        string
   459  	extraCallCost int32
   460  	usedLocals    ir.NameSet
   461  	do            func(ir.Node) bool
   462  	profile       *pgoir.Profile
   463  	dbg           bool
   464  }
   465  
   466  func isDebugFn(fn *ir.Func) bool {
   467  	// if n := fn.Nname; n != nil {
   468  	// 	if n.Sym().Name == "Int32x8.Transpose8" && n.Sym().Pkg.Path == "simd/archsimd" {
   469  	// 		fmt.Printf("isDebugFn '%s' DOT '%s'\n", n.Sym().Pkg.Path, n.Sym().Name)
   470  	// 		return true
   471  	// 	}
   472  	// }
   473  	return false
   474  }
   475  
   476  func (v *hairyVisitor) tooHairy(fn *ir.Func) bool {
   477  	v.do = v.doNode // cache closure
   478  	if ir.DoChildren(fn, v.do) {
   479  		return true
   480  	}
   481  	if v.budget < 0 {
   482  		v.reason = fmt.Sprintf("function too complex: cost %d exceeds budget %d", v.maxBudget-v.budget, v.maxBudget)
   483  		return true
   484  	}
   485  	return false
   486  }
   487  
   488  // doNode visits n and its children, updates the state in v, and returns true if
   489  // n makes the current function too hairy for inlining.
   490  func (v *hairyVisitor) doNode(n ir.Node) bool {
   491  	if n == nil {
   492  		return false
   493  	}
   494  	if v.debug {
   495  		fmt.Printf("%v: doNode %v budget is %d\n", ir.Line(n), n.Op(), v.budget)
   496  	}
   497  opSwitch:
   498  	switch n.Op() {
   499  	// Call is okay if inlinable and we have the budget for the body.
   500  	case ir.OCALLFUNC:
   501  		n := n.(*ir.CallExpr)
   502  		var cheap bool
   503  		if n.Fun.Op() == ir.ONAME {
   504  			name := n.Fun.(*ir.Name)
   505  			if name.Class == ir.PFUNC {
   506  				s := name.Sym()
   507  				fn := s.Name
   508  				switch s.Pkg.Path {
   509  				case "internal/abi":
   510  					switch fn {
   511  					case "NoEscape":
   512  						// Special case for internal/abi.NoEscape. It does just type
   513  						// conversions to appease the escape analysis, and doesn't
   514  						// generate code.
   515  						cheap = true
   516  					}
   517  					if strings.HasPrefix(fn, "EscapeNonString[") {
   518  						// internal/abi.EscapeNonString[T] is a compiler intrinsic
   519  						// implemented in the escape analysis phase.
   520  						cheap = true
   521  					}
   522  				case "internal/runtime/sys":
   523  					switch fn {
   524  					case "GetCallerPC", "GetCallerSP":
   525  						// Functions that call GetCallerPC/SP can not be inlined
   526  						// because users expect the PC/SP of the logical caller,
   527  						// but GetCallerPC/SP returns the physical caller.
   528  						v.reason = "call to " + fn
   529  						return true
   530  					}
   531  				case "go.runtime":
   532  					switch fn {
   533  					case "throw":
   534  						// runtime.throw is a "cheap call" like panic in normal code.
   535  						v.budget -= inlineExtraThrowCost
   536  						break opSwitch
   537  					case "panicrangestate":
   538  						cheap = true
   539  					case "deferrangefunc":
   540  						v.reason = "defer call in range func"
   541  						return true
   542  					}
   543  				}
   544  			}
   545  			// Special case for coverage counter updates; although
   546  			// these correspond to real operations, we treat them as
   547  			// zero cost for the moment. This is due to the existence
   548  			// of tests that are sensitive to inlining-- if the
   549  			// insertion of coverage instrumentation happens to tip a
   550  			// given function over the threshold and move it from
   551  			// "inlinable" to "not-inlinable", this can cause changes
   552  			// in allocation behavior, which can then result in test
   553  			// failures (a good example is the TestAllocations in
   554  			// crypto/ed25519).
   555  			if isAtomicCoverageCounterUpdate(n) {
   556  				return false
   557  			}
   558  		}
   559  		if n.Fun.Op() == ir.OMETHEXPR {
   560  			if meth := ir.MethodExprName(n.Fun); meth != nil {
   561  				if fn := meth.Func; fn != nil {
   562  					s := fn.Sym()
   563  					if types.RuntimeSymName(s) == "heapBits.nextArena" {
   564  						// Special case: explicitly allow mid-stack inlining of
   565  						// runtime.heapBits.next even though it calls slow-path
   566  						// runtime.heapBits.nextArena.
   567  						cheap = true
   568  					}
   569  					// Special case: on architectures that can do unaligned loads,
   570  					// explicitly mark encoding/binary methods as cheap,
   571  					// because in practice they are, even though our inlining
   572  					// budgeting system does not see that. See issue 42958.
   573  					if base.Ctxt.Arch.CanMergeLoads && s.Pkg.Path == "encoding/binary" {
   574  						switch s.Name {
   575  						case "littleEndian.Uint64", "littleEndian.Uint32", "littleEndian.Uint16",
   576  							"bigEndian.Uint64", "bigEndian.Uint32", "bigEndian.Uint16",
   577  							"littleEndian.PutUint64", "littleEndian.PutUint32", "littleEndian.PutUint16",
   578  							"bigEndian.PutUint64", "bigEndian.PutUint32", "bigEndian.PutUint16",
   579  							"littleEndian.AppendUint64", "littleEndian.AppendUint32", "littleEndian.AppendUint16",
   580  							"bigEndian.AppendUint64", "bigEndian.AppendUint32", "bigEndian.AppendUint16":
   581  							cheap = true
   582  						}
   583  					}
   584  				}
   585  			}
   586  		}
   587  
   588  		// A call to a parameter is optimistically a cheap call, if it's a constant function
   589  		// perhaps it will inline, it also can simplify escape analysis.
   590  		extraCost := v.extraCallCost
   591  
   592  		if n.Fun.Op() == ir.ONAME {
   593  			name := n.Fun.(*ir.Name)
   594  			if name.Class == ir.PFUNC {
   595  				// Special case: on architectures that can do unaligned loads,
   596  				// explicitly mark internal/byteorder methods as cheap,
   597  				// because in practice they are, even though our inlining
   598  				// budgeting system does not see that. See issue 42958.
   599  				if base.Ctxt.Arch.CanMergeLoads && name.Sym().Pkg.Path == "internal/byteorder" {
   600  					switch name.Sym().Name {
   601  					case "LEUint64", "LEUint32", "LEUint16",
   602  						"BEUint64", "BEUint32", "BEUint16",
   603  						"LEPutUint64", "LEPutUint32", "LEPutUint16",
   604  						"BEPutUint64", "BEPutUint32", "BEPutUint16",
   605  						"LEAppendUint64", "LEAppendUint32", "LEAppendUint16",
   606  						"BEAppendUint64", "BEAppendUint32", "BEAppendUint16":
   607  						cheap = true
   608  					}
   609  				}
   610  			}
   611  			if name.Class == ir.PPARAM || name.Class == ir.PAUTOHEAP && name.IsClosureVar() {
   612  				extraCost = min(extraCost, inlineParamCallCost)
   613  			}
   614  		}
   615  
   616  		if cheap {
   617  			if v.debug {
   618  				if ir.IsIntrinsicCall(n) {
   619  					fmt.Printf("%v: cheap call is also intrinsic, %v\n", ir.Line(n), n)
   620  				}
   621  			}
   622  			break // treat like any other node, that is, cost of 1
   623  		}
   624  
   625  		if ir.IsIntrinsicCall(n) {
   626  			if v.debug {
   627  				fmt.Printf("%v: intrinsic call, %v\n", ir.Line(n), n)
   628  			}
   629  			break // Treat like any other node.
   630  		}
   631  
   632  		if callee := inlCallee(v.curFunc, n.Fun, v.profile, false); callee != nil && typecheck.HaveInlineBody(callee) {
   633  			// Check whether we'd actually inline this call. Set
   634  			// log == false since we aren't actually doing inlining
   635  			// yet.
   636  			if ok, _, _ := canInlineCallExpr(v.curFunc, n, callee, v.isBigFunc, false, false); ok {
   637  				// mkinlcall would inline this call [1], so use
   638  				// the cost of the inline body as the cost of
   639  				// the call, as that is what will actually
   640  				// appear in the code.
   641  				//
   642  				// [1] This is almost a perfect match to the
   643  				// mkinlcall logic, except that
   644  				// canInlineCallExpr considers inlining cycles
   645  				// by looking at what has already been inlined.
   646  				// Since we haven't done any inlining yet we
   647  				// will miss those.
   648  				//
   649  				// TODO: in the case of a single-call closure, the inlining budget here is potentially much, much larger.
   650  				//
   651  				v.budget -= callee.Inl.Cost
   652  				break
   653  			}
   654  		}
   655  
   656  		if v.debug {
   657  			fmt.Printf("%v: costly OCALLFUNC %v\n", ir.Line(n), n)
   658  		}
   659  
   660  		// Call cost for non-leaf inlining.
   661  		v.budget -= extraCost
   662  
   663  	case ir.OCALLMETH:
   664  		base.FatalfAt(n.Pos(), "OCALLMETH missed by typecheck")
   665  
   666  	// Things that are too hairy, irrespective of the budget
   667  	case ir.OCALL, ir.OCALLINTER:
   668  		// Call cost for non-leaf inlining.
   669  		if v.debug {
   670  			fmt.Printf("%v: costly OCALL %v\n", ir.Line(n), n)
   671  		}
   672  		v.budget -= v.extraCallCost
   673  
   674  	case ir.OPANIC:
   675  		n := n.(*ir.UnaryExpr)
   676  		if n.X.Op() == ir.OCONVIFACE && n.X.(*ir.ConvExpr).Implicit() {
   677  			// Hack to keep reflect.flag.mustBe inlinable for TestIntendedInlining.
   678  			// Before CL 284412, these conversions were introduced later in the
   679  			// compiler, so they didn't count against inlining budget.
   680  			v.budget++
   681  		}
   682  		v.budget -= inlineExtraPanicCost
   683  
   684  	case ir.ORECOVER:
   685  		// recover matches panics to frames via stack unwinding
   686  		// (including inlined frames), so it is safe to inline.
   687  		v.budget -= v.extraCallCost
   688  
   689  	case ir.OCLOSURE:
   690  		if base.Debug.InlFuncsWithClosures == 0 {
   691  			v.reason = "not inlining functions with closures"
   692  			return true
   693  		}
   694  
   695  		// TODO(danscales): Maybe make budget proportional to number of closure
   696  		// variables, e.g.:
   697  		//v.budget -= int32(len(n.(*ir.ClosureExpr).Func.ClosureVars) * 3)
   698  		// TODO(austin): However, if we're able to inline this closure into
   699  		// v.curFunc, then we actually pay nothing for the closure captures. We
   700  		// should try to account for that if we're going to account for captures.
   701  		v.budget -= 15
   702  
   703  	case ir.OGO, ir.ODEFER, ir.OTAILCALL:
   704  		v.reason = "unhandled op " + n.Op().String()
   705  		return true
   706  
   707  	case ir.OAPPEND:
   708  		v.budget -= inlineExtraAppendCost
   709  
   710  	case ir.OADDR:
   711  		n := n.(*ir.AddrExpr)
   712  		// Make "&s.f" cost 0 when f's offset is zero.
   713  		if dot, ok := n.X.(*ir.SelectorExpr); ok && (dot.Op() == ir.ODOT || dot.Op() == ir.ODOTPTR) {
   714  			if _, ok := dot.X.(*ir.Name); ok && dot.Selection.Offset == 0 {
   715  				v.budget += 2 // undo ir.OADDR+ir.ODOT/ir.ODOTPTR
   716  			}
   717  		}
   718  
   719  	case ir.ODEREF:
   720  		// *(*X)(unsafe.Pointer(&x)) is low-cost
   721  		n := n.(*ir.StarExpr)
   722  
   723  		ptr := n.X
   724  		for ptr.Op() == ir.OCONVNOP {
   725  			ptr = ptr.(*ir.ConvExpr).X
   726  		}
   727  		if ptr.Op() == ir.OADDR {
   728  			v.budget += 1 // undo half of default cost of ir.ODEREF+ir.OADDR
   729  		}
   730  
   731  	case ir.OCONVNOP:
   732  		// This doesn't produce code, but the children might.
   733  		v.budget++ // undo default cost
   734  
   735  	case ir.OFALL, ir.OTYPE:
   736  		// These nodes don't produce code; omit from inlining budget.
   737  		return false
   738  
   739  	case ir.OIF:
   740  		n := n.(*ir.IfStmt)
   741  		if ir.IsConst(n.Cond, constant.Bool) {
   742  			// This if and the condition cost nothing.
   743  			if doList(n.Init(), v.do) {
   744  				return true
   745  			}
   746  			if ir.BoolVal(n.Cond) {
   747  				return doList(n.Body, v.do)
   748  			} else {
   749  				return doList(n.Else, v.do)
   750  			}
   751  		}
   752  
   753  	case ir.ONAME:
   754  		n := n.(*ir.Name)
   755  		if n.Class == ir.PAUTO {
   756  			v.usedLocals.Add(n)
   757  		}
   758  
   759  	case ir.OBLOCK:
   760  		// The only OBLOCK we should see at this point is an empty one.
   761  		// In any event, let the visitList(n.List()) below take care of the statements,
   762  		// and don't charge for the OBLOCK itself. The ++ undoes the -- below.
   763  		v.budget++
   764  
   765  	case ir.OMETHVALUE, ir.OSLICELIT:
   766  		v.budget-- // Hack for toolstash -cmp.
   767  
   768  	case ir.OMETHEXPR:
   769  		v.budget++ // Hack for toolstash -cmp.
   770  
   771  	case ir.OAS2:
   772  		n := n.(*ir.AssignListStmt)
   773  
   774  		// Unified IR unconditionally rewrites:
   775  		//
   776  		//	a, b = f()
   777  		//
   778  		// into:
   779  		//
   780  		//	DCL tmp1
   781  		//	DCL tmp2
   782  		//	tmp1, tmp2 = f()
   783  		//	a, b = tmp1, tmp2
   784  		//
   785  		// so that it can insert implicit conversions as necessary. To
   786  		// minimize impact to the existing inlining heuristics (in
   787  		// particular, to avoid breaking the existing inlinability regress
   788  		// tests), we need to compensate for this here.
   789  		//
   790  		// See also identical logic in IsBigFunc.
   791  		if len(n.Rhs) > 0 {
   792  			if init := n.Rhs[0].Init(); len(init) == 1 {
   793  				if _, ok := init[0].(*ir.AssignListStmt); ok {
   794  					// 4 for each value, because each temporary variable now
   795  					// appears 3 times (DCL, LHS, RHS), plus an extra DCL node.
   796  					//
   797  					// 1 for the extra "tmp1, tmp2 = f()" assignment statement.
   798  					v.budget += 4*int32(len(n.Lhs)) + 1
   799  				}
   800  			}
   801  		}
   802  
   803  	case ir.OAS:
   804  		// Special case for coverage counter updates and coverage
   805  		// function registrations. Although these correspond to real
   806  		// operations, we treat them as zero cost for the moment. This
   807  		// is primarily due to the existence of tests that are
   808  		// sensitive to inlining-- if the insertion of coverage
   809  		// instrumentation happens to tip a given function over the
   810  		// threshold and move it from "inlinable" to "not-inlinable",
   811  		// this can cause changes in allocation behavior, which can
   812  		// then result in test failures (a good example is the
   813  		// TestAllocations in crypto/ed25519).
   814  		n := n.(*ir.AssignStmt)
   815  		if n.X.Op() == ir.OINDEX && isIndexingCoverageCounter(n.X) {
   816  			return false
   817  		}
   818  
   819  	case ir.OSLICE, ir.OSLICEARR, ir.OSLICESTR, ir.OSLICE3, ir.OSLICE3ARR:
   820  		n := n.(*ir.SliceExpr)
   821  
   822  		// Ignore superfluous slicing.
   823  		if n.Low != nil && n.Low.Op() == ir.OLITERAL && ir.Int64Val(n.Low) == 0 {
   824  			v.budget++
   825  		}
   826  		if n.High != nil && n.High.Op() == ir.OLEN && n.High.(*ir.UnaryExpr).X == n.X {
   827  			v.budget += 2
   828  		}
   829  	}
   830  
   831  	v.budget--
   832  
   833  	// When debugging, don't stop early, to get full cost of inlining this function
   834  	if v.budget < 0 && base.Flag.LowerM < 2 && !logopt.Enabled() && !v.debug {
   835  		v.reason = "too expensive"
   836  		return true
   837  	}
   838  
   839  	return ir.DoChildren(n, v.do)
   840  }
   841  
   842  // IsBigFunc reports whether fn is a "big" function.
   843  //
   844  // Note: The criteria for "big" is heuristic and subject to change.
   845  func IsBigFunc(fn *ir.Func) bool {
   846  	budget := inlineBigFunctionNodes
   847  	return ir.Any(fn, func(n ir.Node) bool {
   848  		// See logic in hairyVisitor.doNode, explaining unified IR's
   849  		// handling of "a, b = f()" assignments.
   850  		if n, ok := n.(*ir.AssignListStmt); ok && n.Op() == ir.OAS2 && len(n.Rhs) > 0 {
   851  			if init := n.Rhs[0].Init(); len(init) == 1 {
   852  				if _, ok := init[0].(*ir.AssignListStmt); ok {
   853  					budget += 4*len(n.Lhs) + 1
   854  				}
   855  			}
   856  		}
   857  
   858  		budget--
   859  		return budget <= 0
   860  	})
   861  }
   862  
   863  // inlineCallCheck returns whether a call will never be inlineable
   864  // for basic reasons, and whether the call is an intrinisic call.
   865  // The intrinsic result singles out intrinsic calls for debug logging.
   866  func inlineCallCheck(callerfn *ir.Func, call *ir.CallExpr) (bool, bool) {
   867  	if base.Flag.LowerL == 0 {
   868  		return false, false
   869  	}
   870  	if call.Op() != ir.OCALLFUNC {
   871  		return false, false
   872  	}
   873  	if call.GoDefer || call.NoInline {
   874  		return false, false
   875  	}
   876  
   877  	// Prevent inlining some reflect.Value methods when using checkptr,
   878  	// even when package reflect was compiled without it (#35073).
   879  	if base.Debug.Checkptr != 0 && call.Fun.Op() == ir.OMETHEXPR {
   880  		if method := ir.MethodExprName(call.Fun); method != nil {
   881  			switch types.ReflectSymName(method.Sym()) {
   882  			case "Value.UnsafeAddr", "Value.Pointer":
   883  				return false, false
   884  			}
   885  		}
   886  	}
   887  
   888  	// internal/abi.EscapeNonString[T] is a compiler intrinsic implemented
   889  	// in the escape analysis phase.
   890  	if fn := ir.StaticCalleeName(call.Fun); fn != nil && fn.Sym().Pkg.Path == "internal/abi" &&
   891  		strings.HasPrefix(fn.Sym().Name, "EscapeNonString[") {
   892  		return false, true
   893  	}
   894  
   895  	if ir.IsIntrinsicCall(call) {
   896  		return false, true
   897  	}
   898  	return true, false
   899  }
   900  
   901  // InlineCallTarget returns the resolved-for-inlining target of a call.
   902  // It does not necessarily guarantee that the target can be inlined, though
   903  // obvious exclusions are applied.
   904  func InlineCallTarget(callerfn *ir.Func, call *ir.CallExpr, profile *pgoir.Profile) *ir.Func {
   905  	if mightInline, _ := inlineCallCheck(callerfn, call); !mightInline {
   906  		return nil
   907  	}
   908  	return inlCallee(callerfn, call.Fun, profile, true)
   909  }
   910  
   911  // TryInlineCall returns an inlined call expression for call, or nil
   912  // if inlining is not possible.
   913  func TryInlineCall(callerfn *ir.Func, call *ir.CallExpr, bigCaller bool, profile *pgoir.Profile, closureCalledOnce bool) *ir.InlinedCallExpr {
   914  	mightInline, isIntrinsic := inlineCallCheck(callerfn, call)
   915  
   916  	// Preserve old logging behavior
   917  	if (mightInline || isIntrinsic) && base.Flag.LowerM > 3 {
   918  		fmt.Printf("%v:call to func %+v\n", ir.Line(call), call.Fun)
   919  	}
   920  	if !mightInline {
   921  		return nil
   922  	}
   923  
   924  	if fn := inlCallee(callerfn, call.Fun, profile, false); fn != nil && typecheck.HaveInlineBody(fn) {
   925  		return mkinlcall(callerfn, call, fn, bigCaller, closureCalledOnce, profile)
   926  	}
   927  	return nil
   928  }
   929  
   930  // inlCallee takes a function-typed expression and returns the underlying function ONAME
   931  // that it refers to if statically known. Otherwise, it returns nil.
   932  // resolveOnly skips cost-based inlineability checks for closures; the result may not actually be inlineable.
   933  func inlCallee(caller *ir.Func, fn ir.Node, profile *pgoir.Profile, resolveOnly bool) (res *ir.Func) {
   934  	fn = ir.StaticValue(fn)
   935  	switch fn.Op() {
   936  	case ir.OMETHEXPR:
   937  		fn := fn.(*ir.SelectorExpr)
   938  		n := ir.MethodExprName(fn)
   939  		// Check that receiver type matches fn.X.
   940  		// TODO(mdempsky): Handle implicit dereference
   941  		// of pointer receiver argument?
   942  		if n == nil || !types.Identical(n.Type().Recv().Type, fn.X.Type()) {
   943  			return nil
   944  		}
   945  		return n.Func
   946  	case ir.ONAME:
   947  		fn := fn.(*ir.Name)
   948  		if fn.Class == ir.PFUNC {
   949  			return fn.Func
   950  		}
   951  	case ir.OCLOSURE:
   952  		fn := fn.(*ir.ClosureExpr)
   953  		c := fn.Func
   954  		if len(c.ClosureVars) != 0 && c.ClosureVars[0].Outer.Curfn != caller {
   955  			return nil // inliner doesn't support inlining across closure frames
   956  		}
   957  		if !resolveOnly {
   958  			CanInline(c, profile)
   959  		}
   960  		return c
   961  	}
   962  	return nil
   963  }
   964  
   965  var inlgen int
   966  
   967  // SSADumpInline gives the SSA back end a chance to dump the function
   968  // when producing output for debugging the compiler itself.
   969  var SSADumpInline = func(*ir.Func) {}
   970  
   971  // InlineCall allows the inliner implementation to be overridden.
   972  // If it returns nil, the function will not be inlined.
   973  var InlineCall = func(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlIndex int, profile *pgoir.Profile) *ir.InlinedCallExpr {
   974  	base.Fatalf("inline.InlineCall not overridden")
   975  	panic("unreachable")
   976  }
   977  
   978  // inlineCostOK returns true if call n from caller to callee is cheap enough to
   979  // inline. bigCaller indicates that caller is a big function.
   980  //
   981  // In addition to the "cost OK" boolean, it also returns
   982  //   - the "max cost" limit used to make the decision (which may differ depending on func size)
   983  //   - the score assigned to this specific callsite
   984  //   - whether the inlined function is "hot" according to PGO.
   985  func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller, closureCalledOnce bool) (bool, int32, int32, bool) {
   986  	maxCost := int32(inlineMaxBudget)
   987  
   988  	if strings.HasPrefix(ir.FuncName(caller), "runtime_mapaccess1") && caller.Sym().Pkg.Path == "internal/runtime/maps" &&
   989  		strings.HasPrefix(ir.FuncName(callee), "runtime_mapaccess2") && callee.Sym().Pkg.Path == "internal/runtime/maps" {
   990  		// Raise cost to allow inlining of mapaccess2* functions to mapaccess1* wrappers
   991  		maxCost = inlineHotMaxBudget
   992  	}
   993  
   994  	if bigCaller {
   995  		// We use this to restrict inlining into very big functions.
   996  		// See issue 26546 and 17566.
   997  		maxCost = inlineBigFunctionMaxCost
   998  	}
   999  
  1000  	simdMaxCost := simdCreditMultiplier(callee) * maxCost
  1001  
  1002  	if callee.ClosureParent != nil {
  1003  		maxCost *= 2           // favor inlining closures
  1004  		if closureCalledOnce { // really favor inlining the one call to this closure
  1005  			maxCost = max(maxCost, inlineClosureCalledOnceCost)
  1006  		}
  1007  	}
  1008  
  1009  	maxCost = max(maxCost, simdMaxCost)
  1010  
  1011  	metric := callee.Inl.Cost
  1012  	if inlheur.Enabled() {
  1013  		score, ok := inlheur.GetCallSiteScore(caller, n)
  1014  		if ok {
  1015  			metric = int32(score)
  1016  		}
  1017  	}
  1018  
  1019  	lineOffset := pgoir.NodeLineOffset(n, caller)
  1020  	csi := pgoir.CallSiteInfo{LineOffset: lineOffset, Caller: caller}
  1021  	_, hot := candHotEdgeMap[csi]
  1022  
  1023  	if metric <= maxCost {
  1024  		// Simple case. Function is already cheap enough.
  1025  		return true, 0, metric, hot
  1026  	}
  1027  
  1028  	// We'll also allow inlining of hot functions below inlineHotMaxBudget,
  1029  	// but only in small functions.
  1030  
  1031  	if !hot {
  1032  		// Cold
  1033  		return false, maxCost, metric, false
  1034  	}
  1035  
  1036  	// Hot
  1037  
  1038  	if bigCaller {
  1039  		if base.Debug.PGODebug > 0 {
  1040  			fmt.Printf("hot-big check disallows inlining for call %s (cost %d) at %v in big function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))
  1041  		}
  1042  		return false, maxCost, metric, false
  1043  	}
  1044  
  1045  	if metric > inlineHotMaxBudget {
  1046  		return false, inlineHotMaxBudget, metric, false
  1047  	}
  1048  
  1049  	if !base.PGOHash.MatchPosWithInfo(n.Pos(), "inline", nil) {
  1050  		// De-selected by PGO Hash.
  1051  		return false, maxCost, metric, false
  1052  	}
  1053  
  1054  	if base.Debug.PGODebug > 0 {
  1055  		fmt.Printf("hot-budget check allows inlining for call %s (cost %d) at %v in function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))
  1056  	}
  1057  
  1058  	return true, 0, metric, hot
  1059  }
  1060  
  1061  // parsePos returns all the inlining positions and the innermost position.
  1062  func parsePos(pos src.XPos, posTmp []src.Pos) ([]src.Pos, src.Pos) {
  1063  	ctxt := base.Ctxt
  1064  	ctxt.AllPos(pos, func(p src.Pos) {
  1065  		posTmp = append(posTmp, p)
  1066  	})
  1067  	l := len(posTmp) - 1
  1068  	return posTmp[:l], posTmp[l]
  1069  }
  1070  
  1071  // canInlineCallExpr returns true if the call n from caller to callee
  1072  // can be inlined, plus the score computed for the call expr in question,
  1073  // and whether the callee is hot according to PGO.
  1074  // bigCaller indicates that caller is a big function. log
  1075  // indicates that the 'cannot inline' reason should be logged.
  1076  //
  1077  // Preconditions: CanInline(callee) has already been called.
  1078  func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller, closureCalledOnce bool, log bool) (bool, int32, bool) {
  1079  	if callee.Inl == nil {
  1080  		// callee is never inlinable.
  1081  		if log && logopt.Enabled() {
  1082  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
  1083  				fmt.Sprintf("%s cannot be inlined", ir.PkgFuncName(callee)))
  1084  		}
  1085  		return false, 0, false
  1086  	}
  1087  
  1088  	ok, maxCost, callSiteScore, hot := inlineCostOK(n, callerfn, callee, bigCaller, closureCalledOnce)
  1089  	if !ok {
  1090  		// callee cost too high for this call site.
  1091  		if log && logopt.Enabled() {
  1092  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
  1093  				fmt.Sprintf("cost %d of %s exceeds max caller cost %d", callee.Inl.Cost, ir.PkgFuncName(callee), maxCost))
  1094  		}
  1095  		return false, 0, false
  1096  	}
  1097  
  1098  	callees, calleeInner := parsePos(n.Pos(), make([]src.Pos, 0, 10))
  1099  
  1100  	for _, p := range callees {
  1101  		if p.Line() == calleeInner.Line() && p.Col() == calleeInner.Col() && p.AbsFilename() == calleeInner.AbsFilename() {
  1102  			if log && logopt.Enabled() {
  1103  				logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to %s", ir.FuncName(callerfn)))
  1104  			}
  1105  			return false, 0, false
  1106  		}
  1107  	}
  1108  
  1109  	if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(callee.Sym().Pkg) {
  1110  		// Runtime package must not be instrumented.
  1111  		// Instrument skips runtime package. However, some runtime code can be
  1112  		// inlined into other packages and instrumented there. To avoid this,
  1113  		// we disable inlining of runtime functions when instrumenting.
  1114  		// The example that we observed is inlining of LockOSThread,
  1115  		// which lead to false race reports on m contents.
  1116  		if log && logopt.Enabled() {
  1117  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
  1118  				fmt.Sprintf("call to runtime function %s in instrumented build", ir.PkgFuncName(callee)))
  1119  		}
  1120  		return false, 0, false
  1121  	}
  1122  
  1123  	if base.Flag.Race && types.IsNoRacePkg(callee.Sym().Pkg) {
  1124  		if log && logopt.Enabled() {
  1125  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
  1126  				fmt.Sprintf(`call to into "no-race" package function %s in race build`, ir.PkgFuncName(callee)))
  1127  		}
  1128  		return false, 0, false
  1129  	}
  1130  
  1131  	if base.Debug.Checkptr != 0 && types.IsRuntimePkg(callee.Sym().Pkg) {
  1132  		// We don't instrument runtime packages for checkptr (see base/flag.go).
  1133  		if log && logopt.Enabled() {
  1134  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
  1135  				fmt.Sprintf(`call to into runtime package function %s in -d=checkptr build`, ir.PkgFuncName(callee)))
  1136  		}
  1137  		return false, 0, false
  1138  	}
  1139  
  1140  	// Check if we've already inlined this function at this particular
  1141  	// call site, in order to stop inlining when we reach the beginning
  1142  	// of a recursion cycle again. We don't inline immediately recursive
  1143  	// functions, but allow inlining if there is a recursion cycle of
  1144  	// many functions. Most likely, the inlining will stop before we
  1145  	// even hit the beginning of the cycle again, but this catches the
  1146  	// unusual case.
  1147  	parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
  1148  	sym := callee.Linksym()
  1149  	for inlIndex := parent; inlIndex >= 0; inlIndex = base.Ctxt.InlTree.Parent(inlIndex) {
  1150  		if base.Ctxt.InlTree.InlinedFunction(inlIndex) == sym {
  1151  			if log {
  1152  				if base.Flag.LowerM > 1 {
  1153  					fmt.Printf("%v: cannot inline %v into %v: repeated recursive cycle\n", ir.Line(n), callee, ir.FuncName(callerfn))
  1154  				}
  1155  				if logopt.Enabled() {
  1156  					logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
  1157  						fmt.Sprintf("repeated recursive cycle to %s", ir.PkgFuncName(callee)))
  1158  				}
  1159  			}
  1160  			return false, 0, false
  1161  		}
  1162  	}
  1163  
  1164  	return true, callSiteScore, hot
  1165  }
  1166  
  1167  // mkinlcall returns an OINLCALL node that can replace OCALLFUNC n, or
  1168  // nil if it cannot be inlined. callerfn is the function that contains
  1169  // n, and fn is the function being called.
  1170  //
  1171  // The result of mkinlcall MUST be assigned back to n, e.g.
  1172  //
  1173  //	n.Left = mkinlcall(n.Left, fn, isddd)
  1174  func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller, closureCalledOnce bool, profile *pgoir.Profile) *ir.InlinedCallExpr {
  1175  	ok, score, hot := canInlineCallExpr(callerfn, n, fn, bigCaller, closureCalledOnce, true)
  1176  	if !ok {
  1177  		return nil
  1178  	}
  1179  	if hot {
  1180  		hasHotCall[callerfn] = struct{}{}
  1181  	}
  1182  	typecheck.AssertFixedCall(n)
  1183  
  1184  	parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
  1185  	sym := fn.Linksym()
  1186  	inlIndex := base.Ctxt.InlTree.Add(parent, n.Pos(), sym, ir.FuncName(fn))
  1187  
  1188  	closureInitLSym := func(n *ir.CallExpr, fn *ir.Func) {
  1189  		// The linker needs FuncInfo metadata for all inlined
  1190  		// functions. This is typically handled by gc.enqueueFunc
  1191  		// calling ir.InitLSym for all function declarations in
  1192  		// typecheck.Target.Decls (ir.UseClosure adds all closures to
  1193  		// Decls).
  1194  		//
  1195  		// However, closures in Decls are ignored, and are
  1196  		// instead enqueued when walk of the calling function
  1197  		// discovers them.
  1198  		//
  1199  		// This presents a problem for direct calls to closures.
  1200  		// Inlining will replace the entire closure definition with its
  1201  		// body, which hides the closure from walk and thus suppresses
  1202  		// symbol creation.
  1203  		//
  1204  		// Explicitly create a symbol early in this edge case to ensure
  1205  		// we keep this metadata.
  1206  		//
  1207  		// TODO: Refactor to keep a reference so this can all be done
  1208  		// by enqueueFunc.
  1209  
  1210  		if n.Op() != ir.OCALLFUNC {
  1211  			// Not a standard call.
  1212  			return
  1213  		}
  1214  
  1215  		var nf = n.Fun
  1216  		// Skips ir.OCONVNOPs, see issue #73716.
  1217  		for nf.Op() == ir.OCONVNOP {
  1218  			nf = nf.(*ir.ConvExpr).X
  1219  		}
  1220  		if nf.Op() != ir.OCLOSURE {
  1221  			// Not a direct closure call or one with type conversion.
  1222  			return
  1223  		}
  1224  
  1225  		clo := nf.(*ir.ClosureExpr)
  1226  		if !clo.Func.IsClosure() {
  1227  			// enqueueFunc will handle non closures anyways.
  1228  			return
  1229  		}
  1230  
  1231  		ir.InitLSym(fn, true)
  1232  	}
  1233  
  1234  	closureInitLSym(n, fn)
  1235  
  1236  	if base.Flag.GenDwarfInl > 0 {
  1237  		if !sym.WasInlined() {
  1238  			base.Ctxt.DwFixups.SetPrecursorFunc(sym, fn)
  1239  			sym.Set(obj.AttrWasInlined, true)
  1240  		}
  1241  	}
  1242  
  1243  	if base.Flag.LowerM != 0 {
  1244  		if buildcfg.Experiment.NewInliner {
  1245  			fmt.Printf("%v: inlining call to %v with score %d\n",
  1246  				ir.Line(n), fn.Nname.DiagName(), score)
  1247  		} else {
  1248  			fmt.Printf("%v: inlining call to %v\n", ir.Line(n), fn.Nname.DiagName())
  1249  		}
  1250  	}
  1251  	if base.Flag.LowerM > 2 {
  1252  		fmt.Printf("%v: Before inlining: %+v\n", ir.Line(n), n)
  1253  	}
  1254  
  1255  	res := InlineCall(callerfn, n, fn, inlIndex, profile)
  1256  
  1257  	if res == nil {
  1258  		base.FatalfAt(n.Pos(), "inlining call to %v failed", fn.Nname.DiagName())
  1259  	}
  1260  
  1261  	if base.Flag.LowerM > 2 {
  1262  		fmt.Printf("%v: After inlining %+v\n\n", ir.Line(res), res)
  1263  	}
  1264  
  1265  	if inlheur.Enabled() {
  1266  		inlheur.UpdateCallsiteTable(callerfn, n, res)
  1267  	}
  1268  
  1269  	return res
  1270  }
  1271  
  1272  // CalleeEffects appends any side effects from evaluating callee to init.
  1273  func CalleeEffects(init *ir.Nodes, callee ir.Node) {
  1274  	for {
  1275  		init.Append(ir.TakeInit(callee)...)
  1276  
  1277  		switch callee.Op() {
  1278  		case ir.ONAME, ir.OCLOSURE, ir.OMETHEXPR:
  1279  			return // done
  1280  
  1281  		case ir.OCONVNOP:
  1282  			conv := callee.(*ir.ConvExpr)
  1283  			callee = conv.X
  1284  
  1285  		case ir.OINLCALL:
  1286  			ic := callee.(*ir.InlinedCallExpr)
  1287  			init.Append(ic.Body.Take()...)
  1288  			callee = ic.SingleResult()
  1289  
  1290  		default:
  1291  			base.FatalfAt(callee.Pos(), "unexpected callee expression: %v", callee)
  1292  		}
  1293  	}
  1294  }
  1295  
  1296  func pruneUnusedAutos(ll []*ir.Name, vis *hairyVisitor) []*ir.Name {
  1297  	s := make([]*ir.Name, 0, len(ll))
  1298  	for _, n := range ll {
  1299  		if n.Class == ir.PAUTO {
  1300  			if !vis.usedLocals.Has(n) {
  1301  				// TODO(mdempsky): Simplify code after confident that this
  1302  				// never happens anymore.
  1303  				base.FatalfAt(n.Pos(), "unused auto: %v", n)
  1304  				continue
  1305  			}
  1306  		}
  1307  		s = append(s, n)
  1308  	}
  1309  	return s
  1310  }
  1311  
  1312  func doList(list []ir.Node, do func(ir.Node) bool) bool {
  1313  	for _, x := range list {
  1314  		if x != nil {
  1315  			if do(x) {
  1316  				return true
  1317  			}
  1318  		}
  1319  	}
  1320  	return false
  1321  }
  1322  
  1323  // isIndexingCoverageCounter returns true if the specified node 'n' is indexing
  1324  // into a coverage counter array.
  1325  func isIndexingCoverageCounter(n ir.Node) bool {
  1326  	if n.Op() != ir.OINDEX {
  1327  		return false
  1328  	}
  1329  	ixn := n.(*ir.IndexExpr)
  1330  	if ixn.X.Op() != ir.ONAME || !ixn.X.Type().IsArray() {
  1331  		return false
  1332  	}
  1333  	nn := ixn.X.(*ir.Name)
  1334  	// CoverageAuxVar implies either a coverage counter or a package
  1335  	// ID; since the cover tool never emits code to index into ID vars
  1336  	// this is effectively testing whether nn is a coverage counter.
  1337  	return nn.CoverageAuxVar()
  1338  }
  1339  
  1340  // isAtomicCoverageCounterUpdate examines the specified node to
  1341  // determine whether it represents a call to sync/atomic.AddUint32 to
  1342  // increment a coverage counter.
  1343  func isAtomicCoverageCounterUpdate(cn *ir.CallExpr) bool {
  1344  	if cn.Fun.Op() != ir.ONAME {
  1345  		return false
  1346  	}
  1347  	name := cn.Fun.(*ir.Name)
  1348  	if name.Class != ir.PFUNC {
  1349  		return false
  1350  	}
  1351  	fn := name.Sym().Name
  1352  	if name.Sym().Pkg.Path != "sync/atomic" ||
  1353  		(fn != "AddUint32" && fn != "StoreUint32") {
  1354  		return false
  1355  	}
  1356  	if len(cn.Args) != 2 || cn.Args[0].Op() != ir.OADDR {
  1357  		return false
  1358  	}
  1359  	adn := cn.Args[0].(*ir.AddrExpr)
  1360  	v := isIndexingCoverageCounter(adn.X)
  1361  	return v
  1362  }
  1363  
  1364  func PostProcessCallSites(profile *pgoir.Profile) {
  1365  	if base.Debug.DumpInlCallSiteScores != 0 {
  1366  		budgetCallback := func(fn *ir.Func, prof *pgoir.Profile) (int32, bool) {
  1367  			v := inlineBudget(fn, prof, false, false)
  1368  			return v, v == inlineHotMaxBudget
  1369  		}
  1370  		inlheur.DumpInlCallSiteScores(profile, budgetCallback)
  1371  	}
  1372  }
  1373  
  1374  func analyzeFuncProps(fn *ir.Func, p *pgoir.Profile) {
  1375  	canInline := func(fn *ir.Func) { CanInline(fn, p) }
  1376  	budgetForFunc := func(fn *ir.Func) int32 {
  1377  		return inlineBudget(fn, p, true, false)
  1378  	}
  1379  	inlheur.AnalyzeFunc(fn, canInline, budgetForFunc, inlineMaxBudget)
  1380  }
  1381  

View as plain text