1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27 package inline
28
29 import (
30 "fmt"
31 "go/constant"
32 "internal/buildcfg"
33 "strconv"
34 "strings"
35
36 "cmd/compile/internal/base"
37 "cmd/compile/internal/inline/inlheur"
38 "cmd/compile/internal/ir"
39 "cmd/compile/internal/logopt"
40 "cmd/compile/internal/pgoir"
41 "cmd/compile/internal/typecheck"
42 "cmd/compile/internal/types"
43 "cmd/internal/obj"
44 "cmd/internal/pgo"
45 "cmd/internal/src"
46 )
47
48
49 const (
50 inlineMaxBudget = 80
51 inlineExtraAppendCost = 0
52
53 inlineExtraCallCost = 57
54 inlineParamCallCost = 17
55 inlineExtraPanicCost = 1
56 inlineExtraThrowCost = inlineMaxBudget
57
58 inlineBigFunctionNodes = 5000
59 inlineBigFunctionMaxCost = 20
60 inlineClosureCalledOnceCost = 10 * inlineMaxBudget
61 )
62
63 var (
64
65
66 candHotCalleeMap = make(map[*pgoir.IRNode]struct{})
67
68
69 hasHotCall = make(map[*ir.Func]struct{})
70
71
72
73 candHotEdgeMap = make(map[pgoir.CallSiteInfo]struct{})
74
75
76 inlineHotCallSiteThresholdPercent float64
77
78
79
80
81
82 inlineCDFHotCallSiteThresholdPercent = float64(99)
83
84
85 inlineHotMaxBudget int32 = 2000
86 )
87
88 func IsPgoHotFunc(fn *ir.Func, profile *pgoir.Profile) bool {
89 if profile == nil {
90 return false
91 }
92 if n, ok := profile.WeightedCG.IRNodes[ir.LinkFuncName(fn)]; ok {
93 _, ok := candHotCalleeMap[n]
94 return ok
95 }
96 return false
97 }
98
99 func HasPgoHotInline(fn *ir.Func) bool {
100 _, has := hasHotCall[fn]
101 return has
102 }
103
104
105 func PGOInlinePrologue(p *pgoir.Profile) {
106 if base.Debug.PGOInlineCDFThreshold != "" {
107 if s, err := strconv.ParseFloat(base.Debug.PGOInlineCDFThreshold, 64); err == nil && s >= 0 && s <= 100 {
108 inlineCDFHotCallSiteThresholdPercent = s
109 } else {
110 base.Fatalf("invalid PGOInlineCDFThreshold, must be between 0 and 100")
111 }
112 }
113 var hotCallsites []pgo.NamedCallEdge
114 inlineHotCallSiteThresholdPercent, hotCallsites = hotNodesFromCDF(p)
115 if base.Debug.PGODebug > 0 {
116 fmt.Printf("hot-callsite-thres-from-CDF=%v\n", inlineHotCallSiteThresholdPercent)
117 }
118
119 if x := base.Debug.PGOInlineBudget; x != 0 {
120 inlineHotMaxBudget = int32(x)
121 }
122
123 for _, n := range hotCallsites {
124
125 if callee := p.WeightedCG.IRNodes[n.CalleeName]; callee != nil {
126 candHotCalleeMap[callee] = struct{}{}
127 }
128
129 if caller := p.WeightedCG.IRNodes[n.CallerName]; caller != nil && caller.AST != nil {
130 csi := pgoir.CallSiteInfo{LineOffset: n.CallSiteOffset, Caller: caller.AST}
131 candHotEdgeMap[csi] = struct{}{}
132 }
133 }
134
135 if base.Debug.PGODebug >= 3 {
136 fmt.Printf("hot-cg before inline in dot format:")
137 p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent)
138 }
139 }
140
141
142
143
144
145
146
147 func hotNodesFromCDF(p *pgoir.Profile) (float64, []pgo.NamedCallEdge) {
148 cum := int64(0)
149 for i, n := range p.NamedEdgeMap.ByWeight {
150 w := p.NamedEdgeMap.Weight[n]
151 cum += w
152 if pgo.WeightInPercentage(cum, p.TotalWeight) > inlineCDFHotCallSiteThresholdPercent {
153
154
155
156 return pgo.WeightInPercentage(w, p.TotalWeight), p.NamedEdgeMap.ByWeight[:i+1]
157 }
158 }
159 return 0, p.NamedEdgeMap.ByWeight
160 }
161
162
163 func CanInlineFuncs(funcs []*ir.Func, profile *pgoir.Profile) {
164 if profile != nil {
165 PGOInlinePrologue(profile)
166 }
167
168 if base.Flag.LowerL == 0 {
169 return
170 }
171
172 ir.VisitFuncsBottomUp(funcs, func(funcs []*ir.Func, recursive bool) {
173 for _, fn := range funcs {
174 CanInline(fn, profile)
175 if inlheur.Enabled() {
176 analyzeFuncProps(fn, profile)
177 }
178 }
179 })
180 }
181
182
183
184
185
186
187
188
189 func inlineBudget(fn *ir.Func, profile *pgoir.Profile, relaxed bool, verbose bool) int32 {
190
191 budget := int32(inlineMaxBudget)
192 if IsPgoHotFunc(fn, profile) {
193 budget = inlineHotMaxBudget
194 if verbose {
195 fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn))
196 }
197 }
198 if relaxed {
199 budget += inlheur.BudgetExpansion(inlineMaxBudget)
200 }
201 if fn.ClosureParent != nil {
202
203 budget = max(budget, inlineClosureCalledOnceCost)
204 }
205 return budget
206 }
207
208
209
210
211 func CanInline(fn *ir.Func, profile *pgoir.Profile) {
212 if fn.Nname == nil {
213 base.Fatalf("CanInline no nname %+v", fn)
214 }
215
216 var reason string
217 if base.Flag.LowerM > 1 || logopt.Enabled() {
218 defer func() {
219 if reason != "" {
220 if base.Flag.LowerM > 1 {
221 fmt.Printf("%v: cannot inline %v: %s\n", ir.Line(fn), fn.Nname, reason)
222 }
223 if logopt.Enabled() {
224 logopt.LogOpt(fn.Pos(), "cannotInlineFunction", "inline", ir.FuncName(fn), reason)
225 }
226 }
227 }()
228 }
229
230 reason = InlineImpossible(fn)
231 if reason != "" {
232 return
233 }
234 if fn.Typecheck() == 0 {
235 base.Fatalf("CanInline on non-typechecked function %v", fn)
236 }
237
238 n := fn.Nname
239 if n.Func.InlinabilityChecked() {
240 return
241 }
242 defer n.Func.SetInlinabilityChecked(true)
243
244 cc := int32(inlineExtraCallCost)
245 if base.Flag.LowerL == 4 {
246 cc = 1
247 }
248
249
250 relaxed := inlheur.Enabled()
251
252
253 budget := inlineBudget(fn, profile, relaxed, base.Debug.PGODebug > 0)
254
255
256
257
258
259
260
261
262
263
264 visitor := hairyVisitor{
265 curFunc: fn,
266 isBigFunc: IsBigFunc(fn),
267 budget: budget,
268 maxBudget: budget,
269 extraCallCost: cc,
270 profile: profile,
271 }
272 if visitor.tooHairy(fn) {
273 reason = visitor.reason
274 return
275 }
276
277 n.Func.Inl = &ir.Inline{
278 Cost: budget - visitor.budget,
279 Dcl: pruneUnusedAutos(n.Func.Dcl, &visitor),
280 HaveDcl: true,
281 CanDelayResults: canDelayResults(fn),
282 }
283 if base.Flag.LowerM != 0 || logopt.Enabled() {
284 noteInlinableFunc(n, fn, budget-visitor.budget)
285 }
286 }
287
288
289
290 func noteInlinableFunc(n *ir.Name, fn *ir.Func, cost int32) {
291 if base.Flag.LowerM > 1 {
292 fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n, cost, fn.Type(), ir.Nodes(fn.Body))
293 } else if base.Flag.LowerM != 0 {
294 fmt.Printf("%v: can inline %v\n", ir.Line(fn), n)
295 }
296
297 if logopt.Enabled() {
298 logopt.LogOpt(fn.Pos(), "canInlineFunction", "inline", ir.FuncName(fn), fmt.Sprintf("cost: %d", cost))
299 }
300 }
301
302
303
304 func InlineImpossible(fn *ir.Func) string {
305 var reason string
306 if fn.Nname == nil {
307 reason = "no name"
308 return reason
309 }
310
311
312 if fn.Pragma&ir.Noinline != 0 {
313 reason = "marked go:noinline"
314 return reason
315 }
316
317
318 if base.Flag.Race && fn.Pragma&ir.Norace != 0 {
319 reason = "marked go:norace with -race compilation"
320 return reason
321 }
322
323
324 if base.Debug.Checkptr != 0 && fn.Pragma&ir.NoCheckPtr != 0 {
325 reason = "marked go:nocheckptr"
326 return reason
327 }
328
329
330
331 if fn.Pragma&ir.CgoUnsafeArgs != 0 {
332 reason = "marked go:cgo_unsafe_args"
333 return reason
334 }
335
336
337
338
339
340
341
342 if fn.Pragma&ir.UintptrKeepAlive != 0 {
343 reason = "marked as having a keep-alive uintptr argument"
344 return reason
345 }
346
347
348
349 if fn.Pragma&ir.UintptrEscapes != 0 {
350 reason = "marked as having an escaping uintptr argument"
351 return reason
352 }
353
354
355
356
357 if fn.Pragma&ir.Yeswritebarrierrec != 0 {
358 reason = "marked go:yeswritebarrierrec"
359 return reason
360 }
361
362
363
364 if len(fn.Body) == 0 && !typecheck.HaveInlineBody(fn) {
365 reason = "no function body"
366 return reason
367 }
368
369 return ""
370 }
371
372
373
374 func canDelayResults(fn *ir.Func) bool {
375
376
377
378
379
380 nreturns := 0
381 ir.VisitList(fn.Body, func(n ir.Node) {
382 if n, ok := n.(*ir.ReturnStmt); ok {
383 nreturns++
384 if len(n.Results) == 0 {
385 nreturns++
386 }
387 }
388 })
389
390 if nreturns != 1 {
391 return false
392 }
393
394
395 for _, param := range fn.Type().Results() {
396 if sym := param.Sym; sym != nil && !sym.IsBlank() {
397 return false
398 }
399 }
400
401 return true
402 }
403
404
405
406 type hairyVisitor struct {
407
408 curFunc *ir.Func
409 isBigFunc bool
410 budget int32
411 maxBudget int32
412 reason string
413 extraCallCost int32
414 usedLocals ir.NameSet
415 do func(ir.Node) bool
416 profile *pgoir.Profile
417 }
418
419 func (v *hairyVisitor) tooHairy(fn *ir.Func) bool {
420 v.do = v.doNode
421 if ir.DoChildren(fn, v.do) {
422 return true
423 }
424 if v.budget < 0 {
425 v.reason = fmt.Sprintf("function too complex: cost %d exceeds budget %d", v.maxBudget-v.budget, v.maxBudget)
426 return true
427 }
428 return false
429 }
430
431
432
433 func (v *hairyVisitor) doNode(n ir.Node) bool {
434 if n == nil {
435 return false
436 }
437 opSwitch:
438 switch n.Op() {
439
440 case ir.OCALLFUNC:
441 n := n.(*ir.CallExpr)
442 var cheap bool
443 if n.Fun.Op() == ir.ONAME {
444 name := n.Fun.(*ir.Name)
445 if name.Class == ir.PFUNC {
446 s := name.Sym()
447 fn := s.Name
448 switch s.Pkg.Path {
449 case "internal/abi":
450 switch fn {
451 case "NoEscape":
452
453
454
455 cheap = true
456 }
457 if strings.HasPrefix(fn, "EscapeNonString[") {
458
459
460 cheap = true
461 }
462 case "internal/runtime/sys":
463 switch fn {
464 case "GetCallerPC", "GetCallerSP":
465
466
467
468 v.reason = "call to " + fn
469 return true
470 }
471 case "go.runtime":
472 switch fn {
473 case "throw":
474
475 v.budget -= inlineExtraThrowCost
476 break opSwitch
477 case "panicrangestate":
478 cheap = true
479 }
480 }
481 }
482
483
484
485
486
487
488
489
490
491
492 if isAtomicCoverageCounterUpdate(n) {
493 return false
494 }
495 }
496 if n.Fun.Op() == ir.OMETHEXPR {
497 if meth := ir.MethodExprName(n.Fun); meth != nil {
498 if fn := meth.Func; fn != nil {
499 s := fn.Sym()
500 if types.RuntimeSymName(s) == "heapBits.nextArena" {
501
502
503
504 cheap = true
505 }
506
507
508
509
510 if base.Ctxt.Arch.CanMergeLoads && s.Pkg.Path == "encoding/binary" {
511 switch s.Name {
512 case "littleEndian.Uint64", "littleEndian.Uint32", "littleEndian.Uint16",
513 "bigEndian.Uint64", "bigEndian.Uint32", "bigEndian.Uint16",
514 "littleEndian.PutUint64", "littleEndian.PutUint32", "littleEndian.PutUint16",
515 "bigEndian.PutUint64", "bigEndian.PutUint32", "bigEndian.PutUint16",
516 "littleEndian.AppendUint64", "littleEndian.AppendUint32", "littleEndian.AppendUint16",
517 "bigEndian.AppendUint64", "bigEndian.AppendUint32", "bigEndian.AppendUint16":
518 cheap = true
519 }
520 }
521 }
522 }
523 }
524
525
526
527 extraCost := v.extraCallCost
528
529 if n.Fun.Op() == ir.ONAME {
530 name := n.Fun.(*ir.Name)
531 if name.Class == ir.PFUNC {
532
533
534
535
536 if base.Ctxt.Arch.CanMergeLoads && name.Sym().Pkg.Path == "internal/byteorder" {
537 switch name.Sym().Name {
538 case "LEUint64", "LEUint32", "LEUint16",
539 "BEUint64", "BEUint32", "BEUint16",
540 "LEPutUint64", "LEPutUint32", "LEPutUint16",
541 "BEPutUint64", "BEPutUint32", "BEPutUint16",
542 "LEAppendUint64", "LEAppendUint32", "LEAppendUint16",
543 "BEAppendUint64", "BEAppendUint32", "BEAppendUint16":
544 cheap = true
545 }
546 }
547 }
548 if name.Class == ir.PPARAM || name.Class == ir.PAUTOHEAP && name.IsClosureVar() {
549 extraCost = min(extraCost, inlineParamCallCost)
550 }
551 }
552
553 if cheap {
554 break
555 }
556
557 if ir.IsIntrinsicCall(n) {
558
559 break
560 }
561
562 if callee := inlCallee(v.curFunc, n.Fun, v.profile, false); callee != nil && typecheck.HaveInlineBody(callee) {
563
564
565
566 if ok, _, _ := canInlineCallExpr(v.curFunc, n, callee, v.isBigFunc, false, false); ok {
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581 v.budget -= callee.Inl.Cost
582 break
583 }
584 }
585
586
587 v.budget -= extraCost
588
589 case ir.OCALLMETH:
590 base.FatalfAt(n.Pos(), "OCALLMETH missed by typecheck")
591
592
593 case ir.OCALL, ir.OCALLINTER:
594
595 v.budget -= v.extraCallCost
596
597 case ir.OPANIC:
598 n := n.(*ir.UnaryExpr)
599 if n.X.Op() == ir.OCONVIFACE && n.X.(*ir.ConvExpr).Implicit() {
600
601
602
603 v.budget++
604 }
605 v.budget -= inlineExtraPanicCost
606
607 case ir.ORECOVER:
608 base.FatalfAt(n.Pos(), "ORECOVER missed typecheck")
609 case ir.ORECOVERFP:
610
611
612 v.reason = "call to recover"
613 return true
614
615 case ir.OCLOSURE:
616 if base.Debug.InlFuncsWithClosures == 0 {
617 v.reason = "not inlining functions with closures"
618 return true
619 }
620
621
622
623
624
625
626
627 v.budget -= 15
628
629 case ir.OGO, ir.ODEFER, ir.OTAILCALL:
630 v.reason = "unhandled op " + n.Op().String()
631 return true
632
633 case ir.OAPPEND:
634 v.budget -= inlineExtraAppendCost
635
636 case ir.OADDR:
637 n := n.(*ir.AddrExpr)
638
639 if dot, ok := n.X.(*ir.SelectorExpr); ok && (dot.Op() == ir.ODOT || dot.Op() == ir.ODOTPTR) {
640 if _, ok := dot.X.(*ir.Name); ok && dot.Selection.Offset == 0 {
641 v.budget += 2
642 }
643 }
644
645 case ir.ODEREF:
646
647 n := n.(*ir.StarExpr)
648
649 ptr := n.X
650 for ptr.Op() == ir.OCONVNOP {
651 ptr = ptr.(*ir.ConvExpr).X
652 }
653 if ptr.Op() == ir.OADDR {
654 v.budget += 1
655 }
656
657 case ir.OCONVNOP:
658
659 v.budget++
660
661 case ir.OFALL, ir.OTYPE:
662
663 return false
664
665 case ir.OIF:
666 n := n.(*ir.IfStmt)
667 if ir.IsConst(n.Cond, constant.Bool) {
668
669 if doList(n.Init(), v.do) {
670 return true
671 }
672 if ir.BoolVal(n.Cond) {
673 return doList(n.Body, v.do)
674 } else {
675 return doList(n.Else, v.do)
676 }
677 }
678
679 case ir.ONAME:
680 n := n.(*ir.Name)
681 if n.Class == ir.PAUTO {
682 v.usedLocals.Add(n)
683 }
684
685 case ir.OBLOCK:
686
687
688
689 v.budget++
690
691 case ir.OMETHVALUE, ir.OSLICELIT:
692 v.budget--
693
694 case ir.OMETHEXPR:
695 v.budget++
696
697 case ir.OAS2:
698 n := n.(*ir.AssignListStmt)
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717 if len(n.Rhs) > 0 {
718 if init := n.Rhs[0].Init(); len(init) == 1 {
719 if _, ok := init[0].(*ir.AssignListStmt); ok {
720
721
722
723
724 v.budget += 4*int32(len(n.Lhs)) + 1
725 }
726 }
727 }
728
729 case ir.OAS:
730
731
732
733
734
735
736
737
738
739
740 n := n.(*ir.AssignStmt)
741 if n.X.Op() == ir.OINDEX && isIndexingCoverageCounter(n.X) {
742 return false
743 }
744 }
745
746 v.budget--
747
748
749 if v.budget < 0 && base.Flag.LowerM < 2 && !logopt.Enabled() {
750 v.reason = "too expensive"
751 return true
752 }
753
754 return ir.DoChildren(n, v.do)
755 }
756
757
758
759
760 func IsBigFunc(fn *ir.Func) bool {
761 budget := inlineBigFunctionNodes
762 return ir.Any(fn, func(n ir.Node) bool {
763
764
765 if n, ok := n.(*ir.AssignListStmt); ok && n.Op() == ir.OAS2 && len(n.Rhs) > 0 {
766 if init := n.Rhs[0].Init(); len(init) == 1 {
767 if _, ok := init[0].(*ir.AssignListStmt); ok {
768 budget += 4*len(n.Lhs) + 1
769 }
770 }
771 }
772
773 budget--
774 return budget <= 0
775 })
776 }
777
778
779
780
781 func inlineCallCheck(callerfn *ir.Func, call *ir.CallExpr) (bool, bool) {
782 if base.Flag.LowerL == 0 {
783 return false, false
784 }
785 if call.Op() != ir.OCALLFUNC {
786 return false, false
787 }
788 if call.GoDefer || call.NoInline {
789 return false, false
790 }
791
792
793
794 if base.Debug.Checkptr != 0 && call.Fun.Op() == ir.OMETHEXPR {
795 if method := ir.MethodExprName(call.Fun); method != nil {
796 switch types.ReflectSymName(method.Sym()) {
797 case "Value.UnsafeAddr", "Value.Pointer":
798 return false, false
799 }
800 }
801 }
802
803
804
805 if fn := ir.StaticCalleeName(call.Fun); fn != nil && fn.Sym().Pkg.Path == "internal/abi" &&
806 strings.HasPrefix(fn.Sym().Name, "EscapeNonString[") {
807 return false, true
808 }
809
810 if ir.IsIntrinsicCall(call) {
811 return false, true
812 }
813 return true, false
814 }
815
816
817
818
819 func InlineCallTarget(callerfn *ir.Func, call *ir.CallExpr, profile *pgoir.Profile) *ir.Func {
820 if mightInline, _ := inlineCallCheck(callerfn, call); !mightInline {
821 return nil
822 }
823 return inlCallee(callerfn, call.Fun, profile, true)
824 }
825
826
827
828 func TryInlineCall(callerfn *ir.Func, call *ir.CallExpr, bigCaller bool, profile *pgoir.Profile, closureCalledOnce bool) *ir.InlinedCallExpr {
829 mightInline, isIntrinsic := inlineCallCheck(callerfn, call)
830
831
832 if (mightInline || isIntrinsic) && base.Flag.LowerM > 3 {
833 fmt.Printf("%v:call to func %+v\n", ir.Line(call), call.Fun)
834 }
835 if !mightInline {
836 return nil
837 }
838
839 if fn := inlCallee(callerfn, call.Fun, profile, false); fn != nil && typecheck.HaveInlineBody(fn) {
840 return mkinlcall(callerfn, call, fn, bigCaller, closureCalledOnce)
841 }
842 return nil
843 }
844
845
846
847
848 func inlCallee(caller *ir.Func, fn ir.Node, profile *pgoir.Profile, resolveOnly bool) (res *ir.Func) {
849 fn = ir.StaticValue(fn)
850 switch fn.Op() {
851 case ir.OMETHEXPR:
852 fn := fn.(*ir.SelectorExpr)
853 n := ir.MethodExprName(fn)
854
855
856
857 if n == nil || !types.Identical(n.Type().Recv().Type, fn.X.Type()) {
858 return nil
859 }
860 return n.Func
861 case ir.ONAME:
862 fn := fn.(*ir.Name)
863 if fn.Class == ir.PFUNC {
864 return fn.Func
865 }
866 case ir.OCLOSURE:
867 fn := fn.(*ir.ClosureExpr)
868 c := fn.Func
869 if len(c.ClosureVars) != 0 && c.ClosureVars[0].Outer.Curfn != caller {
870 return nil
871 }
872 if !resolveOnly {
873 CanInline(c, profile)
874 }
875 return c
876 }
877 return nil
878 }
879
880 var inlgen int
881
882
883
884 var SSADumpInline = func(*ir.Func) {}
885
886
887
888 var InlineCall = func(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlIndex int) *ir.InlinedCallExpr {
889 base.Fatalf("inline.InlineCall not overridden")
890 panic("unreachable")
891 }
892
893
894
895
896
897
898
899
900 func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller, closureCalledOnce bool) (bool, int32, int32, bool) {
901 maxCost := int32(inlineMaxBudget)
902
903 if bigCaller {
904
905
906 maxCost = inlineBigFunctionMaxCost
907 }
908
909 if callee.ClosureParent != nil {
910 maxCost *= 2
911 if closureCalledOnce {
912 maxCost = max(maxCost, inlineClosureCalledOnceCost)
913 }
914 }
915
916 metric := callee.Inl.Cost
917 if inlheur.Enabled() {
918 score, ok := inlheur.GetCallSiteScore(caller, n)
919 if ok {
920 metric = int32(score)
921 }
922 }
923
924 lineOffset := pgoir.NodeLineOffset(n, caller)
925 csi := pgoir.CallSiteInfo{LineOffset: lineOffset, Caller: caller}
926 _, hot := candHotEdgeMap[csi]
927
928 if metric <= maxCost {
929
930 return true, 0, metric, hot
931 }
932
933
934
935
936 if !hot {
937
938 return false, maxCost, metric, false
939 }
940
941
942
943 if bigCaller {
944 if base.Debug.PGODebug > 0 {
945 fmt.Printf("hot-big check disallows inlining for call %s (cost %d) at %v in big function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))
946 }
947 return false, maxCost, metric, false
948 }
949
950 if metric > inlineHotMaxBudget {
951 return false, inlineHotMaxBudget, metric, false
952 }
953
954 if !base.PGOHash.MatchPosWithInfo(n.Pos(), "inline", nil) {
955
956 return false, maxCost, metric, false
957 }
958
959 if base.Debug.PGODebug > 0 {
960 fmt.Printf("hot-budget check allows inlining for call %s (cost %d) at %v in function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))
961 }
962
963 return true, 0, metric, hot
964 }
965
966
967 func parsePos(pos src.XPos, posTmp []src.Pos) ([]src.Pos, src.Pos) {
968 ctxt := base.Ctxt
969 ctxt.AllPos(pos, func(p src.Pos) {
970 posTmp = append(posTmp, p)
971 })
972 l := len(posTmp) - 1
973 return posTmp[:l], posTmp[l]
974 }
975
976
977
978
979
980
981
982
983 func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller, closureCalledOnce bool, log bool) (bool, int32, bool) {
984 if callee.Inl == nil {
985
986 if log && logopt.Enabled() {
987 logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
988 fmt.Sprintf("%s cannot be inlined", ir.PkgFuncName(callee)))
989 }
990 return false, 0, false
991 }
992
993 ok, maxCost, callSiteScore, hot := inlineCostOK(n, callerfn, callee, bigCaller, closureCalledOnce)
994 if !ok {
995
996 if log && logopt.Enabled() {
997 logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
998 fmt.Sprintf("cost %d of %s exceeds max caller cost %d", callee.Inl.Cost, ir.PkgFuncName(callee), maxCost))
999 }
1000 return false, 0, false
1001 }
1002
1003 callees, calleeInner := parsePos(n.Pos(), make([]src.Pos, 0, 10))
1004
1005 for _, p := range callees {
1006 if p.Line() == calleeInner.Line() && p.Col() == calleeInner.Col() && p.AbsFilename() == calleeInner.AbsFilename() {
1007 if log && logopt.Enabled() {
1008 logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to %s", ir.FuncName(callerfn)))
1009 }
1010 return false, 0, false
1011 }
1012 }
1013
1014 if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(callee.Sym().Pkg) {
1015
1016
1017
1018
1019
1020
1021 if log && logopt.Enabled() {
1022 logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
1023 fmt.Sprintf("call to runtime function %s in instrumented build", ir.PkgFuncName(callee)))
1024 }
1025 return false, 0, false
1026 }
1027
1028 if base.Flag.Race && types.IsNoRacePkg(callee.Sym().Pkg) {
1029 if log && logopt.Enabled() {
1030 logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
1031 fmt.Sprintf(`call to into "no-race" package function %s in race build`, ir.PkgFuncName(callee)))
1032 }
1033 return false, 0, false
1034 }
1035
1036 if base.Debug.Checkptr != 0 && types.IsRuntimePkg(callee.Sym().Pkg) {
1037
1038 if log && logopt.Enabled() {
1039 logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
1040 fmt.Sprintf(`call to into runtime package function %s in -d=checkptr build`, ir.PkgFuncName(callee)))
1041 }
1042 return false, 0, false
1043 }
1044
1045
1046
1047
1048
1049
1050
1051
1052 parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
1053 sym := callee.Linksym()
1054 for inlIndex := parent; inlIndex >= 0; inlIndex = base.Ctxt.InlTree.Parent(inlIndex) {
1055 if base.Ctxt.InlTree.InlinedFunction(inlIndex) == sym {
1056 if log {
1057 if base.Flag.LowerM > 1 {
1058 fmt.Printf("%v: cannot inline %v into %v: repeated recursive cycle\n", ir.Line(n), callee, ir.FuncName(callerfn))
1059 }
1060 if logopt.Enabled() {
1061 logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
1062 fmt.Sprintf("repeated recursive cycle to %s", ir.PkgFuncName(callee)))
1063 }
1064 }
1065 return false, 0, false
1066 }
1067 }
1068
1069 return true, callSiteScore, hot
1070 }
1071
1072
1073
1074
1075
1076
1077
1078
1079 func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller, closureCalledOnce bool) *ir.InlinedCallExpr {
1080 ok, score, hot := canInlineCallExpr(callerfn, n, fn, bigCaller, closureCalledOnce, true)
1081 if !ok {
1082 return nil
1083 }
1084 if hot {
1085 hasHotCall[callerfn] = struct{}{}
1086 }
1087 typecheck.AssertFixedCall(n)
1088
1089 parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
1090 sym := fn.Linksym()
1091 inlIndex := base.Ctxt.InlTree.Add(parent, n.Pos(), sym, ir.FuncName(fn))
1092
1093 closureInitLSym := func(n *ir.CallExpr, fn *ir.Func) {
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115 if n.Op() != ir.OCALLFUNC {
1116
1117 return
1118 }
1119
1120 var nf = n.Fun
1121
1122 for nf.Op() == ir.OCONVNOP {
1123 nf = nf.(*ir.ConvExpr).X
1124 }
1125 if nf.Op() != ir.OCLOSURE {
1126
1127 return
1128 }
1129
1130 clo := nf.(*ir.ClosureExpr)
1131 if !clo.Func.IsClosure() {
1132
1133 return
1134 }
1135
1136 ir.InitLSym(fn, true)
1137 }
1138
1139 closureInitLSym(n, fn)
1140
1141 if base.Flag.GenDwarfInl > 0 {
1142 if !sym.WasInlined() {
1143 base.Ctxt.DwFixups.SetPrecursorFunc(sym, fn)
1144 sym.Set(obj.AttrWasInlined, true)
1145 }
1146 }
1147
1148 if base.Flag.LowerM != 0 {
1149 if buildcfg.Experiment.NewInliner {
1150 fmt.Printf("%v: inlining call to %v with score %d\n",
1151 ir.Line(n), fn, score)
1152 } else {
1153 fmt.Printf("%v: inlining call to %v\n", ir.Line(n), fn)
1154 }
1155 }
1156 if base.Flag.LowerM > 2 {
1157 fmt.Printf("%v: Before inlining: %+v\n", ir.Line(n), n)
1158 }
1159
1160 res := InlineCall(callerfn, n, fn, inlIndex)
1161
1162 if res == nil {
1163 base.FatalfAt(n.Pos(), "inlining call to %v failed", fn)
1164 }
1165
1166 if base.Flag.LowerM > 2 {
1167 fmt.Printf("%v: After inlining %+v\n\n", ir.Line(res), res)
1168 }
1169
1170 if inlheur.Enabled() {
1171 inlheur.UpdateCallsiteTable(callerfn, n, res)
1172 }
1173
1174 return res
1175 }
1176
1177
1178 func CalleeEffects(init *ir.Nodes, callee ir.Node) {
1179 for {
1180 init.Append(ir.TakeInit(callee)...)
1181
1182 switch callee.Op() {
1183 case ir.ONAME, ir.OCLOSURE, ir.OMETHEXPR:
1184 return
1185
1186 case ir.OCONVNOP:
1187 conv := callee.(*ir.ConvExpr)
1188 callee = conv.X
1189
1190 case ir.OINLCALL:
1191 ic := callee.(*ir.InlinedCallExpr)
1192 init.Append(ic.Body.Take()...)
1193 callee = ic.SingleResult()
1194
1195 default:
1196 base.FatalfAt(callee.Pos(), "unexpected callee expression: %v", callee)
1197 }
1198 }
1199 }
1200
1201 func pruneUnusedAutos(ll []*ir.Name, vis *hairyVisitor) []*ir.Name {
1202 s := make([]*ir.Name, 0, len(ll))
1203 for _, n := range ll {
1204 if n.Class == ir.PAUTO {
1205 if !vis.usedLocals.Has(n) {
1206
1207
1208 base.FatalfAt(n.Pos(), "unused auto: %v", n)
1209 continue
1210 }
1211 }
1212 s = append(s, n)
1213 }
1214 return s
1215 }
1216
1217
1218 func numNonClosures(list []*ir.Func) int {
1219 count := 0
1220 for _, fn := range list {
1221 if fn.OClosure == nil {
1222 count++
1223 }
1224 }
1225 return count
1226 }
1227
1228 func doList(list []ir.Node, do func(ir.Node) bool) bool {
1229 for _, x := range list {
1230 if x != nil {
1231 if do(x) {
1232 return true
1233 }
1234 }
1235 }
1236 return false
1237 }
1238
1239
1240
1241 func isIndexingCoverageCounter(n ir.Node) bool {
1242 if n.Op() != ir.OINDEX {
1243 return false
1244 }
1245 ixn := n.(*ir.IndexExpr)
1246 if ixn.X.Op() != ir.ONAME || !ixn.X.Type().IsArray() {
1247 return false
1248 }
1249 nn := ixn.X.(*ir.Name)
1250
1251
1252
1253 return nn.CoverageAuxVar()
1254 }
1255
1256
1257
1258
1259 func isAtomicCoverageCounterUpdate(cn *ir.CallExpr) bool {
1260 if cn.Fun.Op() != ir.ONAME {
1261 return false
1262 }
1263 name := cn.Fun.(*ir.Name)
1264 if name.Class != ir.PFUNC {
1265 return false
1266 }
1267 fn := name.Sym().Name
1268 if name.Sym().Pkg.Path != "sync/atomic" ||
1269 (fn != "AddUint32" && fn != "StoreUint32") {
1270 return false
1271 }
1272 if len(cn.Args) != 2 || cn.Args[0].Op() != ir.OADDR {
1273 return false
1274 }
1275 adn := cn.Args[0].(*ir.AddrExpr)
1276 v := isIndexingCoverageCounter(adn.X)
1277 return v
1278 }
1279
1280 func PostProcessCallSites(profile *pgoir.Profile) {
1281 if base.Debug.DumpInlCallSiteScores != 0 {
1282 budgetCallback := func(fn *ir.Func, prof *pgoir.Profile) (int32, bool) {
1283 v := inlineBudget(fn, prof, false, false)
1284 return v, v == inlineHotMaxBudget
1285 }
1286 inlheur.DumpInlCallSiteScores(profile, budgetCallback)
1287 }
1288 }
1289
1290 func analyzeFuncProps(fn *ir.Func, p *pgoir.Profile) {
1291 canInline := func(fn *ir.Func) { CanInline(fn, p) }
1292 budgetForFunc := func(fn *ir.Func) int32 {
1293 return inlineBudget(fn, p, true, false)
1294 }
1295 inlheur.AnalyzeFunc(fn, canInline, budgetForFunc, inlineMaxBudget)
1296 }
1297
View as plain text