writebarrier.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssa
     6  
     7  import (
     8  	"cmd/compile/internal/reflectdata"
     9  	"cmd/compile/internal/types"
    10  	"cmd/internal/obj"
    11  	"cmd/internal/objabi"
    12  	"cmd/internal/src"
    13  	"fmt"
    14  	"internal/buildcfg"
    15  )
    16  
    17  // A ZeroRegion records parts of an object which are known to be zero.
    18  // A ZeroRegion only applies to a single memory state.
    19  // Each bit in mask is set if the corresponding pointer-sized word of
    20  // the base object is known to be zero.
    21  // In other words, if mask & (1<<i) != 0, then [base+i*ptrSize, base+(i+1)*ptrSize)
    22  // is known to be zero.
    23  type ZeroRegion struct {
    24  	base *Value
    25  	mask uint64
    26  }
    27  
    28  // mightBeHeapPointer reports whether v might point to the heap.
    29  // v must have pointer type.
    30  func mightBeHeapPointer(v *Value) bool {
    31  	if IsGlobalAddr(v) {
    32  		return false
    33  	}
    34  	return true
    35  }
    36  
    37  // mightContainHeapPointer reports whether the data currently at addresses
    38  // [ptr,ptr+size) might contain heap pointers. "currently" means at memory state mem.
    39  // zeroes contains ZeroRegion data to help make that decision (see computeZeroMap).
    40  func mightContainHeapPointer(ptr *Value, size int64, mem *Value, zeroes map[ID]ZeroRegion) bool {
    41  	if IsReadOnlyGlobalAddr(ptr) {
    42  		// The read-only globals section cannot contain any heap pointers.
    43  		return false
    44  	}
    45  
    46  	// See if we can prove that the queried memory is all zero.
    47  
    48  	// Find base pointer and offset. Hopefully, the base is the result of a new(T).
    49  	var off int64
    50  	for ptr.Op == OpOffPtr {
    51  		off += ptr.AuxInt
    52  		ptr = ptr.Args[0]
    53  	}
    54  
    55  	ptrSize := ptr.Block.Func.Config.PtrSize
    56  	if off%ptrSize != 0 {
    57  		return true // see issue 61187
    58  	}
    59  	if size%ptrSize != 0 {
    60  		ptr.Fatalf("unaligned pointer write")
    61  	}
    62  	if off < 0 || off+size > 64*ptrSize {
    63  		// memory range goes off end of tracked offsets
    64  		return true
    65  	}
    66  	z := zeroes[mem.ID]
    67  	if ptr != z.base {
    68  		// This isn't the object we know about at this memory state.
    69  		return true
    70  	}
    71  	// Mask of bits we're asking about
    72  	m := (uint64(1)<<(size/ptrSize) - 1) << (off / ptrSize)
    73  
    74  	if z.mask&m == m {
    75  		// All locations are known to be zero, so no heap pointers.
    76  		return false
    77  	}
    78  	return true
    79  }
    80  
    81  // needwb reports whether we need write barrier for store op v.
    82  // v must be Store/Move/Zero.
    83  // zeroes provides known zero information (keyed by ID of memory-type values).
    84  func needwb(v *Value, zeroes map[ID]ZeroRegion) bool {
    85  	t, ok := v.Aux.(*types.Type)
    86  	if !ok {
    87  		v.Fatalf("store aux is not a type: %s", v.LongString())
    88  	}
    89  	if !t.HasPointers() {
    90  		return false
    91  	}
    92  	dst := v.Args[0]
    93  	if IsStackAddr(dst) {
    94  		return false // writes into the stack don't need write barrier
    95  	}
    96  	// If we're writing to a place that might have heap pointers, we need
    97  	// the write barrier.
    98  	if mightContainHeapPointer(dst, t.Size(), v.MemoryArg(), zeroes) {
    99  		return true
   100  	}
   101  	// Lastly, check if the values we're writing might be heap pointers.
   102  	// If they aren't, we don't need a write barrier.
   103  	switch v.Op {
   104  	case OpStore:
   105  		if !mightBeHeapPointer(v.Args[1]) {
   106  			return false
   107  		}
   108  	case OpZero:
   109  		return false // nil is not a heap pointer
   110  	case OpMove:
   111  		if !mightContainHeapPointer(v.Args[1], t.Size(), v.Args[2], zeroes) {
   112  			return false
   113  		}
   114  	default:
   115  		v.Fatalf("store op unknown: %s", v.LongString())
   116  	}
   117  	return true
   118  }
   119  
   120  // needWBsrc reports whether GC needs to see v when it is the source of a store.
   121  func needWBsrc(v *Value) bool {
   122  	return !IsGlobalAddr(v)
   123  }
   124  
   125  // needWBdst reports whether GC needs to see what used to be in *ptr when ptr is
   126  // the target of a pointer store.
   127  func needWBdst(ptr, mem *Value, zeroes map[ID]ZeroRegion) bool {
   128  	// Detect storing to zeroed memory.
   129  	var off int64
   130  	for ptr.Op == OpOffPtr {
   131  		off += ptr.AuxInt
   132  		ptr = ptr.Args[0]
   133  	}
   134  	ptrSize := ptr.Block.Func.Config.PtrSize
   135  	if off%ptrSize != 0 {
   136  		return true // see issue 61187
   137  	}
   138  	if off < 0 || off >= 64*ptrSize {
   139  		// write goes off end of tracked offsets
   140  		return true
   141  	}
   142  	z := zeroes[mem.ID]
   143  	if ptr != z.base {
   144  		return true
   145  	}
   146  	// If destination is known to be zeroed, we don't need the write barrier
   147  	// to record the old value in *ptr.
   148  	return z.mask>>uint(off/ptrSize)&1 == 0
   149  }
   150  
   151  // writebarrier pass inserts write barriers for store ops (Store, Move, Zero)
   152  // when necessary (the condition above). It rewrites store ops to branches
   153  // and runtime calls, like
   154  //
   155  //	if writeBarrier.enabled {
   156  //		buf := gcWriteBarrier2()	// Not a regular Go call
   157  //		buf[0] = val
   158  //		buf[1] = *ptr
   159  //	}
   160  //	*ptr = val
   161  //
   162  // A sequence of WB stores for many pointer fields of a single type will
   163  // be emitted together, with a single branch.
   164  func writebarrier(f *Func) {
   165  	if !f.fe.UseWriteBarrier() {
   166  		return
   167  	}
   168  
   169  	// Number of write buffer entries we can request at once.
   170  	// Must match runtime/mwbbuf.go:wbMaxEntriesPerCall.
   171  	// It must also match the number of instances of runtime.gcWriteBarrier{X}.
   172  	const maxEntries = 8
   173  
   174  	var sb, sp, wbaddr, const0 *Value
   175  	var cgoCheckPtrWrite, cgoCheckMemmove *obj.LSym
   176  	var wbZero, wbMove *obj.LSym
   177  	var stores, after []*Value
   178  	var sset, sset2 *sparseSet
   179  	var storeNumber []int32
   180  
   181  	// Compute map from a value to the SelectN [1] value that uses it.
   182  	select1 := f.Cache.allocValueSlice(f.NumValues())
   183  	defer func() { f.Cache.freeValueSlice(select1) }()
   184  	for _, b := range f.Blocks {
   185  		for _, v := range b.Values {
   186  			if v.Op != OpSelectN {
   187  				continue
   188  			}
   189  			if v.AuxInt != 1 {
   190  				continue
   191  			}
   192  			select1[v.Args[0].ID] = v
   193  		}
   194  	}
   195  
   196  	zeroes := f.computeZeroMap(select1)
   197  	for _, b := range f.Blocks { // range loop is safe since the blocks we added contain no stores to expand
   198  		// first, identify all the stores that need to insert a write barrier.
   199  		// mark them with WB ops temporarily. record presence of WB ops.
   200  		nWBops := 0 // count of temporarily created WB ops remaining to be rewritten in the current block
   201  		for _, v := range b.Values {
   202  			switch v.Op {
   203  			case OpStore, OpMove, OpZero:
   204  				if needwb(v, zeroes) {
   205  					switch v.Op {
   206  					case OpStore:
   207  						v.Op = OpStoreWB
   208  					case OpMove:
   209  						v.Op = OpMoveWB
   210  					case OpZero:
   211  						v.Op = OpZeroWB
   212  					}
   213  					nWBops++
   214  				}
   215  			}
   216  		}
   217  		if nWBops == 0 {
   218  			continue
   219  		}
   220  
   221  		if wbaddr == nil {
   222  			// lazily initialize global values for write barrier test and calls
   223  			// find SB and SP values in entry block
   224  			initpos := f.Entry.Pos
   225  			sp, sb = f.spSb()
   226  			wbsym := f.fe.Syslook("writeBarrier")
   227  			wbaddr = f.Entry.NewValue1A(initpos, OpAddr, f.Config.Types.UInt32Ptr, wbsym, sb)
   228  			wbZero = f.fe.Syslook("wbZero")
   229  			wbMove = f.fe.Syslook("wbMove")
   230  			if buildcfg.Experiment.CgoCheck2 {
   231  				cgoCheckPtrWrite = f.fe.Syslook("cgoCheckPtrWrite")
   232  				cgoCheckMemmove = f.fe.Syslook("cgoCheckMemmove")
   233  			}
   234  			const0 = f.ConstInt32(f.Config.Types.UInt32, 0)
   235  
   236  			// allocate auxiliary data structures for computing store order
   237  			sset = f.newSparseSet(f.NumValues())
   238  			defer f.retSparseSet(sset)
   239  			sset2 = f.newSparseSet(f.NumValues())
   240  			defer f.retSparseSet(sset2)
   241  			storeNumber = f.Cache.allocInt32Slice(f.NumValues())
   242  			defer f.Cache.freeInt32Slice(storeNumber)
   243  		}
   244  
   245  		// order values in store order
   246  		b.Values = storeOrder(b.Values, sset, storeNumber)
   247  	again:
   248  		// find the start and end of the last contiguous WB store sequence.
   249  		// a branch will be inserted there. values after it will be moved
   250  		// to a new block.
   251  		var last *Value
   252  		var start, end int
   253  		var nonPtrStores int
   254  		values := b.Values
   255  		hasMove := false
   256  	FindSeq:
   257  		for i := len(values) - 1; i >= 0; i-- {
   258  			w := values[i]
   259  			switch w.Op {
   260  			case OpStoreWB, OpMoveWB, OpZeroWB:
   261  				start = i
   262  				if last == nil {
   263  					last = w
   264  					end = i + 1
   265  				}
   266  				nonPtrStores = 0
   267  				if w.Op == OpMoveWB {
   268  					hasMove = true
   269  				}
   270  			case OpVarDef, OpVarLive:
   271  				continue
   272  			case OpStore:
   273  				if last == nil {
   274  					continue
   275  				}
   276  				nonPtrStores++
   277  				if nonPtrStores > 2 {
   278  					break FindSeq
   279  				}
   280  				if hasMove {
   281  					// We need to ensure that this store happens
   282  					// before we issue a wbMove, as the wbMove might
   283  					// use the result of this store as its source.
   284  					// Even though this store is not write-barrier
   285  					// eligible, it might nevertheless be the store
   286  					// of a pointer to the stack, which is then the
   287  					// source of the move.
   288  					// See issue 71228.
   289  					break FindSeq
   290  				}
   291  			default:
   292  				if last == nil {
   293  					continue
   294  				}
   295  				break FindSeq
   296  			}
   297  		}
   298  		stores = append(stores[:0], b.Values[start:end]...) // copy to avoid aliasing
   299  		after = append(after[:0], b.Values[end:]...)
   300  		b.Values = b.Values[:start]
   301  
   302  		// find the memory before the WB stores
   303  		mem := stores[0].MemoryArg()
   304  		pos := stores[0].Pos
   305  
   306  		// If there is a nil check before the WB store, duplicate it to
   307  		// the two branches, where the store and the WB load occur. So
   308  		// they are more likely be removed by late nilcheck removal (which
   309  		// is block-local).
   310  		var nilcheck, nilcheckThen, nilcheckEnd *Value
   311  		if a := stores[0].Args[0]; a.Op == OpNilCheck && a.Args[1] == mem {
   312  			nilcheck = a
   313  		}
   314  
   315  		// If the source of a MoveWB is volatile (will be clobbered by a
   316  		// function call), we need to copy it to a temporary location, as
   317  		// marshaling the args of wbMove might clobber the value we're
   318  		// trying to move.
   319  		// Look for volatile source, copy it to temporary before we check
   320  		// the write barrier flag.
   321  		// It is unlikely to have more than one of them. Just do a linear
   322  		// search instead of using a map.
   323  		// See issue 15854.
   324  		type volatileCopy struct {
   325  			src *Value // address of original volatile value
   326  			tmp *Value // address of temporary we've copied the volatile value into
   327  		}
   328  		var volatiles []volatileCopy
   329  
   330  		if !(f.ABIDefault == f.ABI1 && len(f.Config.intParamRegs) >= 3) {
   331  			// We don't need to do this if the calls we're going to do take
   332  			// all their arguments in registers.
   333  			// 3 is the magic number because it covers wbZero, wbMove, cgoCheckMemmove.
   334  		copyLoop:
   335  			for _, w := range stores {
   336  				if w.Op == OpMoveWB {
   337  					val := w.Args[1]
   338  					if isVolatile(val) {
   339  						for _, c := range volatiles {
   340  							if val == c.src {
   341  								continue copyLoop // already copied
   342  							}
   343  						}
   344  
   345  						t := val.Type.Elem()
   346  						tmp := f.NewLocal(w.Pos, t)
   347  						mem = b.NewValue1A(w.Pos, OpVarDef, types.TypeMem, tmp, mem)
   348  						tmpaddr := b.NewValue2A(w.Pos, OpLocalAddr, t.PtrTo(), tmp, sp, mem)
   349  						siz := t.Size()
   350  						mem = b.NewValue3I(w.Pos, OpMove, types.TypeMem, siz, tmpaddr, val, mem)
   351  						mem.Aux = t
   352  						volatiles = append(volatiles, volatileCopy{val, tmpaddr})
   353  					}
   354  				}
   355  			}
   356  		}
   357  
   358  		// Build branch point.
   359  		bThen := f.NewBlock(BlockPlain)
   360  		bEnd := f.NewBlock(b.Kind)
   361  		bThen.Pos = pos
   362  		bEnd.Pos = b.Pos
   363  		b.Pos = pos
   364  
   365  		// Set up control flow for end block.
   366  		bEnd.CopyControls(b)
   367  		bEnd.Likely = b.Likely
   368  		for _, e := range b.Succs {
   369  			bEnd.Succs = append(bEnd.Succs, e)
   370  			e.b.Preds[e.i].b = bEnd
   371  		}
   372  
   373  		// set up control flow for write barrier test
   374  		// load word, test word, avoiding partial register write from load byte.
   375  		cfgtypes := &f.Config.Types
   376  		flag := b.NewValue2(pos, OpLoad, cfgtypes.UInt32, wbaddr, mem)
   377  		flag = b.NewValue2(pos, OpNeq32, cfgtypes.Bool, flag, const0)
   378  		b.Kind = BlockIf
   379  		b.SetControl(flag)
   380  		b.Likely = BranchUnlikely
   381  		b.Succs = b.Succs[:0]
   382  		b.AddEdgeTo(bThen)
   383  		b.AddEdgeTo(bEnd)
   384  		bThen.AddEdgeTo(bEnd)
   385  
   386  		// For each write barrier store, append write barrier code to bThen.
   387  		memThen := mem
   388  
   389  		if nilcheck != nil {
   390  			nilcheckThen = bThen.NewValue2(nilcheck.Pos, OpNilCheck, nilcheck.Type, nilcheck.Args[0], memThen)
   391  		}
   392  
   393  		// Note: we can issue the write barrier code in any order. In particular,
   394  		// it doesn't matter if they are in a different order *even if* they end
   395  		// up referring to overlapping memory regions. For instance if an OpStore
   396  		// stores to a location that is later read by an OpMove. In all cases
   397  		// any pointers we must get into the write barrier buffer still make it,
   398  		// possibly in a different order and possibly a different (but definitely
   399  		// more than 0) number of times.
   400  		// In light of that, we process all the OpStoreWBs first. This minimizes
   401  		// the amount of spill/restore code we need around the Zero/Move calls.
   402  
   403  		// srcs contains the value IDs of pointer values we've put in the write barrier buffer.
   404  		srcs := sset
   405  		srcs.clear()
   406  		// dsts contains the value IDs of locations which we've read a pointer out of
   407  		// and put the result in the write barrier buffer.
   408  		dsts := sset2
   409  		dsts.clear()
   410  
   411  		// Buffer up entries that we need to put in the write barrier buffer.
   412  		type write struct {
   413  			ptr *Value   // value to put in write barrier buffer
   414  			pos src.XPos // location to use for the write
   415  		}
   416  		var writeStore [maxEntries]write
   417  		writes := writeStore[:0]
   418  
   419  		flush := func() {
   420  			if len(writes) == 0 {
   421  				return
   422  			}
   423  			// Issue a call to get a write barrier buffer.
   424  			t := types.NewTuple(types.Types[types.TUINTPTR].PtrTo(), types.TypeMem)
   425  			call := bThen.NewValue1I(pos, OpWB, t, int64(len(writes)), memThen)
   426  			curPtr := bThen.NewValue1(pos, OpSelect0, types.Types[types.TUINTPTR].PtrTo(), call)
   427  			memThen = bThen.NewValue1(pos, OpSelect1, types.TypeMem, call)
   428  			// Write each pending pointer to a slot in the buffer.
   429  			for i, write := range writes {
   430  				wbuf := bThen.NewValue1I(write.pos, OpOffPtr, types.Types[types.TUINTPTR].PtrTo(), int64(i)*f.Config.PtrSize, curPtr)
   431  				memThen = bThen.NewValue3A(write.pos, OpStore, types.TypeMem, types.Types[types.TUINTPTR], wbuf, write.ptr, memThen)
   432  			}
   433  			writes = writes[:0]
   434  		}
   435  		addEntry := func(pos src.XPos, ptr *Value) {
   436  			writes = append(writes, write{ptr: ptr, pos: pos})
   437  			if len(writes) == maxEntries {
   438  				flush()
   439  			}
   440  		}
   441  
   442  		// Find all the pointers we need to write to the buffer.
   443  		for _, w := range stores {
   444  			if w.Op != OpStoreWB {
   445  				continue
   446  			}
   447  			pos := w.Pos
   448  			ptr := w.Args[0]
   449  			val := w.Args[1]
   450  			if !srcs.contains(val.ID) && needWBsrc(val) {
   451  				srcs.add(val.ID)
   452  				addEntry(pos, val)
   453  			}
   454  			if !dsts.contains(ptr.ID) && needWBdst(ptr, w.Args[2], zeroes) {
   455  				dsts.add(ptr.ID)
   456  				// Load old value from store target.
   457  				// Note: This turns bad pointer writes into bad
   458  				// pointer reads, which could be confusing. We could avoid
   459  				// reading from obviously bad pointers, which would
   460  				// take care of the vast majority of these. We could
   461  				// patch this up in the signal handler, or use XCHG to
   462  				// combine the read and the write.
   463  				if ptr == nilcheck {
   464  					ptr = nilcheckThen
   465  				}
   466  				oldVal := bThen.NewValue2(pos, OpLoad, types.Types[types.TUINTPTR], ptr, memThen)
   467  				// Save old value to write buffer.
   468  				addEntry(pos, oldVal)
   469  			}
   470  			f.fe.Func().SetWBPos(pos)
   471  			nWBops--
   472  		}
   473  		flush()
   474  
   475  		// Now do the rare cases, Zeros and Moves.
   476  		for _, w := range stores {
   477  			pos := w.Pos
   478  			dst := w.Args[0]
   479  			if dst == nilcheck {
   480  				dst = nilcheckThen
   481  			}
   482  			switch w.Op {
   483  			case OpZeroWB:
   484  				typ := reflectdata.TypeLinksym(w.Aux.(*types.Type))
   485  				// zeroWB(&typ, dst)
   486  				taddr := b.NewValue1A(pos, OpAddr, b.Func.Config.Types.Uintptr, typ, sb)
   487  				memThen = wbcall(pos, bThen, wbZero, sp, memThen, taddr, dst)
   488  				f.fe.Func().SetWBPos(pos)
   489  				nWBops--
   490  			case OpMoveWB:
   491  				src := w.Args[1]
   492  				if isVolatile(src) {
   493  					for _, c := range volatiles {
   494  						if src == c.src {
   495  							src = c.tmp
   496  							break
   497  						}
   498  					}
   499  				}
   500  				typ := reflectdata.TypeLinksym(w.Aux.(*types.Type))
   501  				// moveWB(&typ, dst, src)
   502  				taddr := b.NewValue1A(pos, OpAddr, b.Func.Config.Types.Uintptr, typ, sb)
   503  				memThen = wbcall(pos, bThen, wbMove, sp, memThen, taddr, dst, src)
   504  				f.fe.Func().SetWBPos(pos)
   505  				nWBops--
   506  			}
   507  		}
   508  
   509  		// merge memory
   510  		mem = bEnd.NewValue2(pos, OpPhi, types.TypeMem, mem, memThen)
   511  
   512  		if nilcheck != nil {
   513  			nilcheckEnd = bEnd.NewValue2(nilcheck.Pos, OpNilCheck, nilcheck.Type, nilcheck.Args[0], mem)
   514  		}
   515  
   516  		// Do raw stores after merge point.
   517  		for _, w := range stores {
   518  			pos := w.Pos
   519  			dst := w.Args[0]
   520  			if dst == nilcheck {
   521  				dst = nilcheckEnd
   522  			}
   523  			switch w.Op {
   524  			case OpStoreWB:
   525  				val := w.Args[1]
   526  				if buildcfg.Experiment.CgoCheck2 {
   527  					// Issue cgo checking code.
   528  					mem = wbcall(pos, bEnd, cgoCheckPtrWrite, sp, mem, dst, val)
   529  				}
   530  				mem = bEnd.NewValue3A(pos, OpStore, types.TypeMem, w.Aux, dst, val, mem)
   531  			case OpZeroWB:
   532  				mem = bEnd.NewValue2I(pos, OpZero, types.TypeMem, w.AuxInt, dst, mem)
   533  				mem.Aux = w.Aux
   534  			case OpMoveWB:
   535  				src := w.Args[1]
   536  				if isVolatile(src) {
   537  					for _, c := range volatiles {
   538  						if src == c.src {
   539  							src = c.tmp
   540  							break
   541  						}
   542  					}
   543  				}
   544  				if buildcfg.Experiment.CgoCheck2 {
   545  					// Issue cgo checking code.
   546  					typ := reflectdata.TypeLinksym(w.Aux.(*types.Type))
   547  					taddr := b.NewValue1A(pos, OpAddr, b.Func.Config.Types.Uintptr, typ, sb)
   548  					mem = wbcall(pos, bEnd, cgoCheckMemmove, sp, mem, taddr, dst, src)
   549  				}
   550  				mem = bEnd.NewValue3I(pos, OpMove, types.TypeMem, w.AuxInt, dst, src, mem)
   551  				mem.Aux = w.Aux
   552  			case OpVarDef, OpVarLive:
   553  				mem = bEnd.NewValue1A(pos, w.Op, types.TypeMem, w.Aux, mem)
   554  			case OpStore:
   555  				val := w.Args[1]
   556  				mem = bEnd.NewValue3A(pos, OpStore, types.TypeMem, w.Aux, dst, val, mem)
   557  			}
   558  		}
   559  
   560  		// The last store becomes the WBend marker. This marker is used by the liveness
   561  		// pass to determine what parts of the code are preemption-unsafe.
   562  		// All subsequent memory operations use this memory, so we have to sacrifice the
   563  		// previous last memory op to become this new value.
   564  		bEnd.Values = append(bEnd.Values, last)
   565  		last.Block = bEnd
   566  		last.reset(OpWBend)
   567  		last.Pos = last.Pos.WithNotStmt()
   568  		last.Type = types.TypeMem
   569  		last.AddArg(mem)
   570  
   571  		// Free all the old stores, except last which became the WBend marker.
   572  		for _, w := range stores {
   573  			if w != last {
   574  				w.resetArgs()
   575  			}
   576  		}
   577  		for _, w := range stores {
   578  			if w != last {
   579  				f.freeValue(w)
   580  			}
   581  		}
   582  		if nilcheck != nil && nilcheck.Uses == 0 {
   583  			nilcheck.reset(OpInvalid)
   584  		}
   585  
   586  		// put values after the store sequence into the end block
   587  		bEnd.Values = append(bEnd.Values, after...)
   588  		for _, w := range after {
   589  			w.Block = bEnd
   590  		}
   591  
   592  		// if we have more stores in this block, do this block again
   593  		if nWBops > 0 {
   594  			goto again
   595  		}
   596  	}
   597  }
   598  
   599  // computeZeroMap returns a map from an ID of a memory value to
   600  // a set of locations that are known to be zeroed at that memory value.
   601  func (f *Func) computeZeroMap(select1 []*Value) map[ID]ZeroRegion {
   602  
   603  	ptrSize := f.Config.PtrSize
   604  	// Keep track of which parts of memory are known to be zero.
   605  	// This helps with removing write barriers for various initialization patterns.
   606  	// This analysis is conservative. We only keep track, for each memory state, of
   607  	// which of the first 64 words of a single object are known to be zero.
   608  	zeroes := map[ID]ZeroRegion{}
   609  	// Find new objects.
   610  	for _, b := range f.Blocks {
   611  		for _, v := range b.Values {
   612  			if mem, ok := IsNewObject(v, select1); ok {
   613  				// While compiling package runtime itself, we might see user
   614  				// calls to newobject, which will have result type
   615  				// unsafe.Pointer instead. We can't easily infer how large the
   616  				// allocated memory is, so just skip it.
   617  				if types.LocalPkg.Path == "runtime" && v.Type.IsUnsafePtr() {
   618  					continue
   619  				}
   620  
   621  				nptr := min(64, v.Type.Elem().Size()/ptrSize)
   622  				zeroes[mem.ID] = ZeroRegion{base: v, mask: 1<<uint(nptr) - 1}
   623  			}
   624  		}
   625  	}
   626  	// Find stores to those new objects.
   627  	for {
   628  		changed := false
   629  		for _, b := range f.Blocks {
   630  			// Note: iterating forwards helps convergence, as values are
   631  			// typically (but not always!) in store order.
   632  			for _, v := range b.Values {
   633  				if v.Op != OpStore {
   634  					continue
   635  				}
   636  				z, ok := zeroes[v.MemoryArg().ID]
   637  				if !ok {
   638  					continue
   639  				}
   640  				ptr := v.Args[0]
   641  				var off int64
   642  				size := v.Aux.(*types.Type).Size()
   643  				for ptr.Op == OpOffPtr {
   644  					off += ptr.AuxInt
   645  					ptr = ptr.Args[0]
   646  				}
   647  				if ptr != z.base {
   648  					// Different base object - we don't know anything.
   649  					// We could even be writing to the base object we know
   650  					// about, but through an aliased but offset pointer.
   651  					// So we have to throw all the zero information we have away.
   652  					continue
   653  				}
   654  				// Round to cover any partially written pointer slots.
   655  				// Pointer writes should never be unaligned like this, but non-pointer
   656  				// writes to pointer-containing types will do this.
   657  				if d := off % ptrSize; d != 0 {
   658  					off -= d
   659  					size += d
   660  				}
   661  				if d := size % ptrSize; d != 0 {
   662  					size += ptrSize - d
   663  				}
   664  				// Clip to the 64 words that we track.
   665  				minimum := max(off, 0)
   666  				maximum := min(off+size, 64*ptrSize)
   667  
   668  				// Clear bits for parts that we are writing (and hence
   669  				// will no longer necessarily be zero).
   670  				for i := minimum; i < maximum; i += ptrSize {
   671  					bit := i / ptrSize
   672  					z.mask &^= 1 << uint(bit)
   673  				}
   674  				if z.mask == 0 {
   675  					// No more known zeros - don't bother keeping.
   676  					continue
   677  				}
   678  				// Save updated known zero contents for new store.
   679  				if zeroes[v.ID] != z {
   680  					zeroes[v.ID] = z
   681  					changed = true
   682  				}
   683  			}
   684  		}
   685  		if !changed {
   686  			break
   687  		}
   688  	}
   689  	if f.pass.debug > 0 {
   690  		fmt.Printf("func %s\n", f.Name)
   691  		for mem, z := range zeroes {
   692  			fmt.Printf("  memory=v%d ptr=%v zeromask=%b\n", mem, z.base, z.mask)
   693  		}
   694  	}
   695  	return zeroes
   696  }
   697  
   698  // wbcall emits write barrier runtime call in b, returns memory.
   699  func wbcall(pos src.XPos, b *Block, fn *obj.LSym, sp, mem *Value, args ...*Value) *Value {
   700  	config := b.Func.Config
   701  	typ := config.Types.Uintptr // type of all argument values
   702  	nargs := len(args)
   703  
   704  	// TODO (register args) this is a bit of a hack.
   705  	inRegs := b.Func.ABIDefault == b.Func.ABI1 && len(config.intParamRegs) >= 3
   706  
   707  	if !inRegs {
   708  		// Store arguments to the appropriate stack slot.
   709  		off := config.ctxt.Arch.FixedFrameSize
   710  		for _, arg := range args {
   711  			stkaddr := b.NewValue1I(pos, OpOffPtr, typ.PtrTo(), off, sp)
   712  			mem = b.NewValue3A(pos, OpStore, types.TypeMem, typ, stkaddr, arg, mem)
   713  			off += typ.Size()
   714  		}
   715  		args = args[:0]
   716  	}
   717  
   718  	args = append(args, mem)
   719  
   720  	// issue call
   721  	argTypes := make([]*types.Type, nargs, 3) // at most 3 args; allows stack allocation
   722  	for i := 0; i < nargs; i++ {
   723  		argTypes[i] = typ
   724  	}
   725  	call := b.NewValue0A(pos, OpStaticCall, types.TypeResultMem, StaticAuxCall(fn, b.Func.ABIDefault.ABIAnalyzeTypes(argTypes, nil)))
   726  	call.AddArgs(args...)
   727  	call.AuxInt = int64(nargs) * typ.Size()
   728  	return b.NewValue1I(pos, OpSelectN, types.TypeMem, 0, call)
   729  }
   730  
   731  // IsStackAddr reports whether v is known to be an address of a stack slot.
   732  func IsStackAddr(v *Value) bool {
   733  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy {
   734  		v = v.Args[0]
   735  	}
   736  	switch v.Op {
   737  	case OpSP, OpLocalAddr, OpSelectNAddr, OpGetCallerSP:
   738  		return true
   739  	}
   740  	return false
   741  }
   742  
   743  // IsGlobalAddr reports whether v is known to be an address of a global (or nil).
   744  func IsGlobalAddr(v *Value) bool {
   745  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy {
   746  		v = v.Args[0]
   747  	}
   748  	if v.Op == OpAddr && v.Args[0].Op == OpSB {
   749  		return true // address of a global
   750  	}
   751  	if v.Op == OpConstNil {
   752  		return true
   753  	}
   754  	if v.Op == OpLoad && IsReadOnlyGlobalAddr(v.Args[0]) {
   755  		return true // loading from a read-only global - the resulting address can't be a heap address.
   756  	}
   757  	return false
   758  }
   759  
   760  // IsReadOnlyGlobalAddr reports whether v is known to be an address of a read-only global.
   761  func IsReadOnlyGlobalAddr(v *Value) bool {
   762  	if v.Op == OpConstNil {
   763  		// Nil pointers are read only. See issue 33438.
   764  		return true
   765  	}
   766  	if v.Op == OpAddr && v.Aux != nil && v.Aux.(*obj.LSym).Type == objabi.SRODATA {
   767  		return true
   768  	}
   769  	return false
   770  }
   771  
   772  // IsNewObject reports whether v is a pointer to a freshly allocated & zeroed object,
   773  // if so, also returns the memory state mem at which v is zero.
   774  func IsNewObject(v *Value, select1 []*Value) (mem *Value, ok bool) {
   775  	f := v.Block.Func
   776  	c := f.Config
   777  	if f.ABIDefault == f.ABI1 && len(c.intParamRegs) >= 1 {
   778  		if v.Op != OpSelectN || v.AuxInt != 0 {
   779  			return nil, false
   780  		}
   781  		mem = select1[v.Args[0].ID]
   782  		if mem == nil {
   783  			return nil, false
   784  		}
   785  	} else {
   786  		if v.Op != OpLoad {
   787  			return nil, false
   788  		}
   789  		mem = v.MemoryArg()
   790  		if mem.Op != OpSelectN {
   791  			return nil, false
   792  		}
   793  		if mem.Type != types.TypeMem {
   794  			return nil, false
   795  		} // assume it is the right selection if true
   796  	}
   797  	call := mem.Args[0]
   798  	if call.Op != OpStaticCall {
   799  		return nil, false
   800  	}
   801  	// Check for new object, or for new object calls that have been transformed into size-specialized malloc calls.
   802  	// Calls that have return type unsafe pointer may have originally been produced by flushPendingHeapAllocations
   803  	// in the ssa generator, so may have not originally been newObject calls.
   804  	var numParameters int64
   805  	switch {
   806  	case isNewObject(call.Aux):
   807  		numParameters = 1
   808  	case isSpecializedMalloc(call.Aux) && !v.Type.IsUnsafePtr():
   809  		numParameters = 3
   810  	default:
   811  		return nil, false
   812  	}
   813  	if f.ABIDefault == f.ABI1 && len(c.intParamRegs) >= 1 {
   814  		if v.Args[0] == call {
   815  			return mem, true
   816  		}
   817  		return nil, false
   818  	}
   819  	if v.Args[0].Op != OpOffPtr {
   820  		return nil, false
   821  	}
   822  	if v.Args[0].Args[0].Op != OpSP {
   823  		return nil, false
   824  	}
   825  	if v.Args[0].AuxInt != c.ctxt.Arch.FixedFrameSize+numParameters*c.RegSize { // offset of return value
   826  		return nil, false
   827  	}
   828  	return mem, true
   829  }
   830  
   831  // IsSanitizerSafeAddr reports whether v is known to be an address
   832  // that doesn't need instrumentation.
   833  func IsSanitizerSafeAddr(v *Value) bool {
   834  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy {
   835  		v = v.Args[0]
   836  	}
   837  	switch v.Op {
   838  	case OpSP, OpLocalAddr, OpSelectNAddr:
   839  		// Stack addresses are always safe.
   840  		return true
   841  	case OpITab, OpStringPtr, OpGetClosurePtr:
   842  		// Itabs, string data, and closure fields are
   843  		// read-only once initialized.
   844  		return true
   845  	case OpAddr:
   846  		vt := v.Aux.(*obj.LSym).Type
   847  		return vt == objabi.SRODATA || vt == objabi.SLIBFUZZER_8BIT_COUNTER || vt == objabi.SCOVERAGE_COUNTER || vt == objabi.SCOVERAGE_AUXVAR
   848  	}
   849  	return false
   850  }
   851  
   852  // isVolatile reports whether v is a pointer to argument region on stack which
   853  // will be clobbered by a function call.
   854  func isVolatile(v *Value) bool {
   855  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy || v.Op == OpSelectNAddr {
   856  		v = v.Args[0]
   857  	}
   858  	return v.Op == OpSP
   859  }
   860
View as plain text