writebarrier.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssa
     6  
     7  import (
     8  	"cmd/compile/internal/reflectdata"
     9  	"cmd/compile/internal/types"
    10  	"cmd/internal/obj"
    11  	"cmd/internal/objabi"
    12  	"cmd/internal/src"
    13  	"fmt"
    14  	"internal/buildcfg"
    15  )
    16  
    17  // A ZeroRegion records parts of an object which are known to be zero.
    18  // A ZeroRegion only applies to a single memory state.
    19  // Each bit in mask is set if the corresponding pointer-sized word of
    20  // the base object is known to be zero.
    21  // In other words, if mask & (1<<i) != 0, then [base+i*ptrSize, base+(i+1)*ptrSize)
    22  // is known to be zero.
    23  type ZeroRegion struct {
    24  	base *Value
    25  	mask uint64
    26  }
    27  
    28  // mightBeHeapPointer reports whether v might point to the heap.
    29  // v must have pointer type.
    30  func mightBeHeapPointer(v *Value) bool {
    31  	if IsGlobalAddr(v) {
    32  		return false
    33  	}
    34  	return true
    35  }
    36  
    37  // mightContainHeapPointer reports whether the data currently at addresses
    38  // [ptr,ptr+size) might contain heap pointers. "currently" means at memory state mem.
    39  // zeroes contains ZeroRegion data to help make that decision (see computeZeroMap).
    40  func mightContainHeapPointer(ptr *Value, size int64, mem *Value, zeroes map[ID]ZeroRegion) bool {
    41  	if IsReadOnlyGlobalAddr(ptr) {
    42  		// The read-only globals section cannot contain any heap pointers.
    43  		return false
    44  	}
    45  
    46  	// See if we can prove that the queried memory is all zero.
    47  
    48  	// Find base pointer and offset. Hopefully, the base is the result of a new(T).
    49  	var off int64
    50  	for ptr.Op == OpOffPtr {
    51  		off += ptr.AuxInt
    52  		ptr = ptr.Args[0]
    53  	}
    54  
    55  	ptrSize := ptr.Block.Func.Config.PtrSize
    56  	if off%ptrSize != 0 {
    57  		return true // see issue 61187
    58  	}
    59  	if size%ptrSize != 0 {
    60  		ptr.Fatalf("unaligned pointer write")
    61  	}
    62  	if off < 0 || off+size > 64*ptrSize {
    63  		// memory range goes off end of tracked offsets
    64  		return true
    65  	}
    66  	z := zeroes[mem.ID]
    67  	if ptr != z.base {
    68  		// This isn't the object we know about at this memory state.
    69  		return true
    70  	}
    71  	// Mask of bits we're asking about
    72  	m := (uint64(1)<<(size/ptrSize) - 1) << (off / ptrSize)
    73  
    74  	if z.mask&m == m {
    75  		// All locations are known to be zero, so no heap pointers.
    76  		return false
    77  	}
    78  	return true
    79  }
    80  
    81  // needwb reports whether we need write barrier for store op v.
    82  // v must be Store/Move/Zero.
    83  // zeroes provides known zero information (keyed by ID of memory-type values).
    84  func needwb(v *Value, zeroes map[ID]ZeroRegion) bool {
    85  	t, ok := v.Aux.(*types.Type)
    86  	if !ok {
    87  		v.Fatalf("store aux is not a type: %s", v.LongString())
    88  	}
    89  	if !t.HasPointers() {
    90  		return false
    91  	}
    92  	dst := v.Args[0]
    93  	if IsStackAddr(dst) {
    94  		return false // writes into the stack don't need write barrier
    95  	}
    96  	// If we're writing to a place that might have heap pointers, we need
    97  	// the write barrier.
    98  	if mightContainHeapPointer(dst, t.Size(), v.MemoryArg(), zeroes) {
    99  		return true
   100  	}
   101  	// Lastly, check if the values we're writing might be heap pointers.
   102  	// If they aren't, we don't need a write barrier.
   103  	switch v.Op {
   104  	case OpStore:
   105  		if !mightBeHeapPointer(v.Args[1]) {
   106  			return false
   107  		}
   108  	case OpZero:
   109  		return false // nil is not a heap pointer
   110  	case OpMove:
   111  		if !mightContainHeapPointer(v.Args[1], t.Size(), v.Args[2], zeroes) {
   112  			return false
   113  		}
   114  	default:
   115  		v.Fatalf("store op unknown: %s", v.LongString())
   116  	}
   117  	return true
   118  }
   119  
   120  // needWBsrc reports whether GC needs to see v when it is the source of a store.
   121  func needWBsrc(v *Value) bool {
   122  	return !IsGlobalAddr(v)
   123  }
   124  
   125  // needWBdst reports whether GC needs to see what used to be in *ptr when ptr is
   126  // the target of a pointer store.
   127  func needWBdst(ptr, mem *Value, zeroes map[ID]ZeroRegion) bool {
   128  	// Detect storing to zeroed memory.
   129  	var off int64
   130  	for ptr.Op == OpOffPtr {
   131  		off += ptr.AuxInt
   132  		ptr = ptr.Args[0]
   133  	}
   134  	ptrSize := ptr.Block.Func.Config.PtrSize
   135  	if off%ptrSize != 0 {
   136  		return true // see issue 61187
   137  	}
   138  	if off < 0 || off >= 64*ptrSize {
   139  		// write goes off end of tracked offsets
   140  		return true
   141  	}
   142  	z := zeroes[mem.ID]
   143  	if ptr != z.base {
   144  		return true
   145  	}
   146  	// If destination is known to be zeroed, we don't need the write barrier
   147  	// to record the old value in *ptr.
   148  	return z.mask>>uint(off/ptrSize)&1 == 0
   149  }
   150  
   151  // writebarrier pass inserts write barriers for store ops (Store, Move, Zero)
   152  // when necessary (the condition above). It rewrites store ops to branches
   153  // and runtime calls, like
   154  //
   155  //	if writeBarrier.enabled {
   156  //		buf := gcWriteBarrier2()	// Not a regular Go call
   157  //		buf[0] = val
   158  //		buf[1] = *ptr
   159  //	}
   160  //	*ptr = val
   161  //
   162  // A sequence of WB stores for many pointer fields of a single type will
   163  // be emitted together, with a single branch.
   164  func writebarrier(f *Func) {
   165  	if !f.fe.UseWriteBarrier() {
   166  		return
   167  	}
   168  
   169  	// Number of write buffer entries we can request at once.
   170  	// Must match runtime/mwbbuf.go:wbMaxEntriesPerCall.
   171  	// It must also match the number of instances of runtime.gcWriteBarrier{X}.
   172  	const maxEntries = 8
   173  
   174  	var sb, sp, wbaddr, const0 *Value
   175  	var cgoCheckPtrWrite, cgoCheckMemmove *obj.LSym
   176  	var wbZero, wbMove *obj.LSym
   177  	var stores, after []*Value
   178  	var sset, sset2 *sparseSet
   179  	var storeNumber []int32
   180  
   181  	// Compute map from a value to the SelectN [1] value that uses it.
   182  	select1 := f.Cache.allocValueSlice(f.NumValues())
   183  	defer func() { f.Cache.freeValueSlice(select1) }()
   184  	for _, b := range f.Blocks {
   185  		for _, v := range b.Values {
   186  			if v.Op != OpSelectN {
   187  				continue
   188  			}
   189  			if v.AuxInt != 1 {
   190  				continue
   191  			}
   192  			select1[v.Args[0].ID] = v
   193  		}
   194  	}
   195  
   196  	zeroes := f.computeZeroMap(select1)
   197  	for _, b := range f.Blocks { // range loop is safe since the blocks we added contain no stores to expand
   198  		// first, identify all the stores that need to insert a write barrier.
   199  		// mark them with WB ops temporarily. record presence of WB ops.
   200  		nWBops := 0 // count of temporarily created WB ops remaining to be rewritten in the current block
   201  		for _, v := range b.Values {
   202  			switch v.Op {
   203  			case OpStore, OpMove, OpZero:
   204  				if needwb(v, zeroes) {
   205  					switch v.Op {
   206  					case OpStore:
   207  						v.Op = OpStoreWB
   208  					case OpMove:
   209  						v.Op = OpMoveWB
   210  					case OpZero:
   211  						v.Op = OpZeroWB
   212  					}
   213  					nWBops++
   214  				}
   215  			}
   216  		}
   217  		if nWBops == 0 {
   218  			continue
   219  		}
   220  
   221  		if wbaddr == nil {
   222  			// lazily initialize global values for write barrier test and calls
   223  			// find SB and SP values in entry block
   224  			initpos := f.Entry.Pos
   225  			sp, sb = f.spSb()
   226  			wbsym := f.fe.Syslook("writeBarrier")
   227  			wbaddr = f.Entry.NewValue1A(initpos, OpAddr, f.Config.Types.UInt32Ptr, wbsym, sb)
   228  			wbZero = f.fe.Syslook("wbZero")
   229  			wbMove = f.fe.Syslook("wbMove")
   230  			if buildcfg.Experiment.CgoCheck2 {
   231  				cgoCheckPtrWrite = f.fe.Syslook("cgoCheckPtrWrite")
   232  				cgoCheckMemmove = f.fe.Syslook("cgoCheckMemmove")
   233  			}
   234  			const0 = f.ConstInt32(f.Config.Types.UInt32, 0)
   235  
   236  			// allocate auxiliary data structures for computing store order
   237  			sset = f.newSparseSet(f.NumValues())
   238  			defer f.retSparseSet(sset)
   239  			sset2 = f.newSparseSet(f.NumValues())
   240  			defer f.retSparseSet(sset2)
   241  			storeNumber = f.Cache.allocInt32Slice(f.NumValues())
   242  			defer f.Cache.freeInt32Slice(storeNumber)
   243  		}
   244  
   245  		// order values in store order
   246  		b.Values = storeOrder(b.Values, sset, storeNumber)
   247  	again:
   248  		// find the start and end of the last contiguous WB store sequence.
   249  		// a branch will be inserted there. values after it will be moved
   250  		// to a new block.
   251  		var last *Value
   252  		var start, end int
   253  		var nonPtrStores int
   254  		values := b.Values
   255  		hasMove := false
   256  	FindSeq:
   257  		for i := len(values) - 1; i >= 0; i-- {
   258  			w := values[i]
   259  			switch w.Op {
   260  			case OpStoreWB, OpMoveWB, OpZeroWB:
   261  				start = i
   262  				if last == nil {
   263  					last = w
   264  					end = i + 1
   265  				}
   266  				nonPtrStores = 0
   267  				if w.Op == OpMoveWB {
   268  					hasMove = true
   269  				}
   270  			case OpVarDef, OpVarLive:
   271  				continue
   272  			case OpStore:
   273  				if last == nil {
   274  					continue
   275  				}
   276  				nonPtrStores++
   277  				if nonPtrStores > 2 {
   278  					break FindSeq
   279  				}
   280  				if hasMove {
   281  					// We need to ensure that this store happens
   282  					// before we issue a wbMove, as the wbMove might
   283  					// use the result of this store as its source.
   284  					// Even though this store is not write-barrier
   285  					// eligible, it might nevertheless be the store
   286  					// of a pointer to the stack, which is then the
   287  					// source of the move.
   288  					// See issue 71228.
   289  					break FindSeq
   290  				}
   291  			default:
   292  				if last == nil {
   293  					continue
   294  				}
   295  				break FindSeq
   296  			}
   297  		}
   298  		stores = append(stores[:0], b.Values[start:end]...) // copy to avoid aliasing
   299  		after = append(after[:0], b.Values[end:]...)
   300  		b.Values = b.Values[:start]
   301  
   302  		// find the memory before the WB stores
   303  		mem := stores[0].MemoryArg()
   304  		pos := stores[0].Pos
   305  
   306  		// If the source of a MoveWB is volatile (will be clobbered by a
   307  		// function call), we need to copy it to a temporary location, as
   308  		// marshaling the args of wbMove might clobber the value we're
   309  		// trying to move.
   310  		// Look for volatile source, copy it to temporary before we check
   311  		// the write barrier flag.
   312  		// It is unlikely to have more than one of them. Just do a linear
   313  		// search instead of using a map.
   314  		// See issue 15854.
   315  		type volatileCopy struct {
   316  			src *Value // address of original volatile value
   317  			tmp *Value // address of temporary we've copied the volatile value into
   318  		}
   319  		var volatiles []volatileCopy
   320  
   321  		if !(f.ABIDefault == f.ABI1 && len(f.Config.intParamRegs) >= 3) {
   322  			// We don't need to do this if the calls we're going to do take
   323  			// all their arguments in registers.
   324  			// 3 is the magic number because it covers wbZero, wbMove, cgoCheckMemmove.
   325  		copyLoop:
   326  			for _, w := range stores {
   327  				if w.Op == OpMoveWB {
   328  					val := w.Args[1]
   329  					if isVolatile(val) {
   330  						for _, c := range volatiles {
   331  							if val == c.src {
   332  								continue copyLoop // already copied
   333  							}
   334  						}
   335  
   336  						t := val.Type.Elem()
   337  						tmp := f.NewLocal(w.Pos, t)
   338  						mem = b.NewValue1A(w.Pos, OpVarDef, types.TypeMem, tmp, mem)
   339  						tmpaddr := b.NewValue2A(w.Pos, OpLocalAddr, t.PtrTo(), tmp, sp, mem)
   340  						siz := t.Size()
   341  						mem = b.NewValue3I(w.Pos, OpMove, types.TypeMem, siz, tmpaddr, val, mem)
   342  						mem.Aux = t
   343  						volatiles = append(volatiles, volatileCopy{val, tmpaddr})
   344  					}
   345  				}
   346  			}
   347  		}
   348  
   349  		// Build branch point.
   350  		bThen := f.NewBlock(BlockPlain)
   351  		bEnd := f.NewBlock(b.Kind)
   352  		bThen.Pos = pos
   353  		bEnd.Pos = b.Pos
   354  		b.Pos = pos
   355  
   356  		// Set up control flow for end block.
   357  		bEnd.CopyControls(b)
   358  		bEnd.Likely = b.Likely
   359  		for _, e := range b.Succs {
   360  			bEnd.Succs = append(bEnd.Succs, e)
   361  			e.b.Preds[e.i].b = bEnd
   362  		}
   363  
   364  		// set up control flow for write barrier test
   365  		// load word, test word, avoiding partial register write from load byte.
   366  		cfgtypes := &f.Config.Types
   367  		flag := b.NewValue2(pos, OpLoad, cfgtypes.UInt32, wbaddr, mem)
   368  		flag = b.NewValue2(pos, OpNeq32, cfgtypes.Bool, flag, const0)
   369  		b.Kind = BlockIf
   370  		b.SetControl(flag)
   371  		b.Likely = BranchUnlikely
   372  		b.Succs = b.Succs[:0]
   373  		b.AddEdgeTo(bThen)
   374  		b.AddEdgeTo(bEnd)
   375  		bThen.AddEdgeTo(bEnd)
   376  
   377  		// For each write barrier store, append write barrier code to bThen.
   378  		memThen := mem
   379  
   380  		// Note: we can issue the write barrier code in any order. In particular,
   381  		// it doesn't matter if they are in a different order *even if* they end
   382  		// up referring to overlapping memory regions. For instance if an OpStore
   383  		// stores to a location that is later read by an OpMove. In all cases
   384  		// any pointers we must get into the write barrier buffer still make it,
   385  		// possibly in a different order and possibly a different (but definitely
   386  		// more than 0) number of times.
   387  		// In light of that, we process all the OpStoreWBs first. This minimizes
   388  		// the amount of spill/restore code we need around the Zero/Move calls.
   389  
   390  		// srcs contains the value IDs of pointer values we've put in the write barrier buffer.
   391  		srcs := sset
   392  		srcs.clear()
   393  		// dsts contains the value IDs of locations which we've read a pointer out of
   394  		// and put the result in the write barrier buffer.
   395  		dsts := sset2
   396  		dsts.clear()
   397  
   398  		// Buffer up entries that we need to put in the write barrier buffer.
   399  		type write struct {
   400  			ptr *Value   // value to put in write barrier buffer
   401  			pos src.XPos // location to use for the write
   402  		}
   403  		var writeStore [maxEntries]write
   404  		writes := writeStore[:0]
   405  
   406  		flush := func() {
   407  			if len(writes) == 0 {
   408  				return
   409  			}
   410  			// Issue a call to get a write barrier buffer.
   411  			t := types.NewTuple(types.Types[types.TUINTPTR].PtrTo(), types.TypeMem)
   412  			call := bThen.NewValue1I(pos, OpWB, t, int64(len(writes)), memThen)
   413  			curPtr := bThen.NewValue1(pos, OpSelect0, types.Types[types.TUINTPTR].PtrTo(), call)
   414  			memThen = bThen.NewValue1(pos, OpSelect1, types.TypeMem, call)
   415  			// Write each pending pointer to a slot in the buffer.
   416  			for i, write := range writes {
   417  				wbuf := bThen.NewValue1I(write.pos, OpOffPtr, types.Types[types.TUINTPTR].PtrTo(), int64(i)*f.Config.PtrSize, curPtr)
   418  				memThen = bThen.NewValue3A(write.pos, OpStore, types.TypeMem, types.Types[types.TUINTPTR], wbuf, write.ptr, memThen)
   419  			}
   420  			writes = writes[:0]
   421  		}
   422  		addEntry := func(pos src.XPos, ptr *Value) {
   423  			writes = append(writes, write{ptr: ptr, pos: pos})
   424  			if len(writes) == maxEntries {
   425  				flush()
   426  			}
   427  		}
   428  
   429  		// Find all the pointers we need to write to the buffer.
   430  		for _, w := range stores {
   431  			if w.Op != OpStoreWB {
   432  				continue
   433  			}
   434  			pos := w.Pos
   435  			ptr := w.Args[0]
   436  			val := w.Args[1]
   437  			if !srcs.contains(val.ID) && needWBsrc(val) {
   438  				srcs.add(val.ID)
   439  				addEntry(pos, val)
   440  			}
   441  			if !dsts.contains(ptr.ID) && needWBdst(ptr, w.Args[2], zeroes) {
   442  				dsts.add(ptr.ID)
   443  				// Load old value from store target.
   444  				// Note: This turns bad pointer writes into bad
   445  				// pointer reads, which could be confusing. We could avoid
   446  				// reading from obviously bad pointers, which would
   447  				// take care of the vast majority of these. We could
   448  				// patch this up in the signal handler, or use XCHG to
   449  				// combine the read and the write.
   450  				oldVal := bThen.NewValue2(pos, OpLoad, types.Types[types.TUINTPTR], ptr, memThen)
   451  				// Save old value to write buffer.
   452  				addEntry(pos, oldVal)
   453  			}
   454  			f.fe.Func().SetWBPos(pos)
   455  			nWBops--
   456  		}
   457  		flush()
   458  
   459  		// Now do the rare cases, Zeros and Moves.
   460  		for _, w := range stores {
   461  			pos := w.Pos
   462  			switch w.Op {
   463  			case OpZeroWB:
   464  				dst := w.Args[0]
   465  				typ := reflectdata.TypeLinksym(w.Aux.(*types.Type))
   466  				// zeroWB(&typ, dst)
   467  				taddr := b.NewValue1A(pos, OpAddr, b.Func.Config.Types.Uintptr, typ, sb)
   468  				memThen = wbcall(pos, bThen, wbZero, sp, memThen, taddr, dst)
   469  				f.fe.Func().SetWBPos(pos)
   470  				nWBops--
   471  			case OpMoveWB:
   472  				dst := w.Args[0]
   473  				src := w.Args[1]
   474  				if isVolatile(src) {
   475  					for _, c := range volatiles {
   476  						if src == c.src {
   477  							src = c.tmp
   478  							break
   479  						}
   480  					}
   481  				}
   482  				typ := reflectdata.TypeLinksym(w.Aux.(*types.Type))
   483  				// moveWB(&typ, dst, src)
   484  				taddr := b.NewValue1A(pos, OpAddr, b.Func.Config.Types.Uintptr, typ, sb)
   485  				memThen = wbcall(pos, bThen, wbMove, sp, memThen, taddr, dst, src)
   486  				f.fe.Func().SetWBPos(pos)
   487  				nWBops--
   488  			}
   489  		}
   490  
   491  		// merge memory
   492  		mem = bEnd.NewValue2(pos, OpPhi, types.TypeMem, mem, memThen)
   493  
   494  		// Do raw stores after merge point.
   495  		for _, w := range stores {
   496  			pos := w.Pos
   497  			switch w.Op {
   498  			case OpStoreWB:
   499  				ptr := w.Args[0]
   500  				val := w.Args[1]
   501  				if buildcfg.Experiment.CgoCheck2 {
   502  					// Issue cgo checking code.
   503  					mem = wbcall(pos, bEnd, cgoCheckPtrWrite, sp, mem, ptr, val)
   504  				}
   505  				mem = bEnd.NewValue3A(pos, OpStore, types.TypeMem, w.Aux, ptr, val, mem)
   506  			case OpZeroWB:
   507  				dst := w.Args[0]
   508  				mem = bEnd.NewValue2I(pos, OpZero, types.TypeMem, w.AuxInt, dst, mem)
   509  				mem.Aux = w.Aux
   510  			case OpMoveWB:
   511  				dst := w.Args[0]
   512  				src := w.Args[1]
   513  				if isVolatile(src) {
   514  					for _, c := range volatiles {
   515  						if src == c.src {
   516  							src = c.tmp
   517  							break
   518  						}
   519  					}
   520  				}
   521  				if buildcfg.Experiment.CgoCheck2 {
   522  					// Issue cgo checking code.
   523  					typ := reflectdata.TypeLinksym(w.Aux.(*types.Type))
   524  					taddr := b.NewValue1A(pos, OpAddr, b.Func.Config.Types.Uintptr, typ, sb)
   525  					mem = wbcall(pos, bEnd, cgoCheckMemmove, sp, mem, taddr, dst, src)
   526  				}
   527  				mem = bEnd.NewValue3I(pos, OpMove, types.TypeMem, w.AuxInt, dst, src, mem)
   528  				mem.Aux = w.Aux
   529  			case OpVarDef, OpVarLive:
   530  				mem = bEnd.NewValue1A(pos, w.Op, types.TypeMem, w.Aux, mem)
   531  			case OpStore:
   532  				ptr := w.Args[0]
   533  				val := w.Args[1]
   534  				mem = bEnd.NewValue3A(pos, OpStore, types.TypeMem, w.Aux, ptr, val, mem)
   535  			}
   536  		}
   537  
   538  		// The last store becomes the WBend marker. This marker is used by the liveness
   539  		// pass to determine what parts of the code are preemption-unsafe.
   540  		// All subsequent memory operations use this memory, so we have to sacrifice the
   541  		// previous last memory op to become this new value.
   542  		bEnd.Values = append(bEnd.Values, last)
   543  		last.Block = bEnd
   544  		last.reset(OpWBend)
   545  		last.Pos = last.Pos.WithNotStmt()
   546  		last.Type = types.TypeMem
   547  		last.AddArg(mem)
   548  
   549  		// Free all the old stores, except last which became the WBend marker.
   550  		for _, w := range stores {
   551  			if w != last {
   552  				w.resetArgs()
   553  			}
   554  		}
   555  		for _, w := range stores {
   556  			if w != last {
   557  				f.freeValue(w)
   558  			}
   559  		}
   560  
   561  		// put values after the store sequence into the end block
   562  		bEnd.Values = append(bEnd.Values, after...)
   563  		for _, w := range after {
   564  			w.Block = bEnd
   565  		}
   566  
   567  		// if we have more stores in this block, do this block again
   568  		if nWBops > 0 {
   569  			goto again
   570  		}
   571  	}
   572  }
   573  
   574  // computeZeroMap returns a map from an ID of a memory value to
   575  // a set of locations that are known to be zeroed at that memory value.
   576  func (f *Func) computeZeroMap(select1 []*Value) map[ID]ZeroRegion {
   577  
   578  	ptrSize := f.Config.PtrSize
   579  	// Keep track of which parts of memory are known to be zero.
   580  	// This helps with removing write barriers for various initialization patterns.
   581  	// This analysis is conservative. We only keep track, for each memory state, of
   582  	// which of the first 64 words of a single object are known to be zero.
   583  	zeroes := map[ID]ZeroRegion{}
   584  	// Find new objects.
   585  	for _, b := range f.Blocks {
   586  		for _, v := range b.Values {
   587  			if mem, ok := IsNewObject(v, select1); ok {
   588  				// While compiling package runtime itself, we might see user
   589  				// calls to newobject, which will have result type
   590  				// unsafe.Pointer instead. We can't easily infer how large the
   591  				// allocated memory is, so just skip it.
   592  				if types.LocalPkg.Path == "runtime" && v.Type.IsUnsafePtr() {
   593  					continue
   594  				}
   595  
   596  				nptr := min(64, v.Type.Elem().Size()/ptrSize)
   597  				zeroes[mem.ID] = ZeroRegion{base: v, mask: 1<<uint(nptr) - 1}
   598  			}
   599  		}
   600  	}
   601  	// Find stores to those new objects.
   602  	for {
   603  		changed := false
   604  		for _, b := range f.Blocks {
   605  			// Note: iterating forwards helps convergence, as values are
   606  			// typically (but not always!) in store order.
   607  			for _, v := range b.Values {
   608  				if v.Op != OpStore {
   609  					continue
   610  				}
   611  				z, ok := zeroes[v.MemoryArg().ID]
   612  				if !ok {
   613  					continue
   614  				}
   615  				ptr := v.Args[0]
   616  				var off int64
   617  				size := v.Aux.(*types.Type).Size()
   618  				for ptr.Op == OpOffPtr {
   619  					off += ptr.AuxInt
   620  					ptr = ptr.Args[0]
   621  				}
   622  				if ptr != z.base {
   623  					// Different base object - we don't know anything.
   624  					// We could even be writing to the base object we know
   625  					// about, but through an aliased but offset pointer.
   626  					// So we have to throw all the zero information we have away.
   627  					continue
   628  				}
   629  				// Round to cover any partially written pointer slots.
   630  				// Pointer writes should never be unaligned like this, but non-pointer
   631  				// writes to pointer-containing types will do this.
   632  				if d := off % ptrSize; d != 0 {
   633  					off -= d
   634  					size += d
   635  				}
   636  				if d := size % ptrSize; d != 0 {
   637  					size += ptrSize - d
   638  				}
   639  				// Clip to the 64 words that we track.
   640  				minimum := max(off, 0)
   641  				maximum := min(off+size, 64*ptrSize)
   642  
   643  				// Clear bits for parts that we are writing (and hence
   644  				// will no longer necessarily be zero).
   645  				for i := minimum; i < maximum; i += ptrSize {
   646  					bit := i / ptrSize
   647  					z.mask &^= 1 << uint(bit)
   648  				}
   649  				if z.mask == 0 {
   650  					// No more known zeros - don't bother keeping.
   651  					continue
   652  				}
   653  				// Save updated known zero contents for new store.
   654  				if zeroes[v.ID] != z {
   655  					zeroes[v.ID] = z
   656  					changed = true
   657  				}
   658  			}
   659  		}
   660  		if !changed {
   661  			break
   662  		}
   663  	}
   664  	if f.pass.debug > 0 {
   665  		fmt.Printf("func %s\n", f.Name)
   666  		for mem, z := range zeroes {
   667  			fmt.Printf("  memory=v%d ptr=%v zeromask=%b\n", mem, z.base, z.mask)
   668  		}
   669  	}
   670  	return zeroes
   671  }
   672  
   673  // wbcall emits write barrier runtime call in b, returns memory.
   674  func wbcall(pos src.XPos, b *Block, fn *obj.LSym, sp, mem *Value, args ...*Value) *Value {
   675  	config := b.Func.Config
   676  	typ := config.Types.Uintptr // type of all argument values
   677  	nargs := len(args)
   678  
   679  	// TODO (register args) this is a bit of a hack.
   680  	inRegs := b.Func.ABIDefault == b.Func.ABI1 && len(config.intParamRegs) >= 3
   681  
   682  	if !inRegs {
   683  		// Store arguments to the appropriate stack slot.
   684  		off := config.ctxt.Arch.FixedFrameSize
   685  		for _, arg := range args {
   686  			stkaddr := b.NewValue1I(pos, OpOffPtr, typ.PtrTo(), off, sp)
   687  			mem = b.NewValue3A(pos, OpStore, types.TypeMem, typ, stkaddr, arg, mem)
   688  			off += typ.Size()
   689  		}
   690  		args = args[:0]
   691  	}
   692  
   693  	args = append(args, mem)
   694  
   695  	// issue call
   696  	argTypes := make([]*types.Type, nargs, 3) // at most 3 args; allows stack allocation
   697  	for i := 0; i < nargs; i++ {
   698  		argTypes[i] = typ
   699  	}
   700  	call := b.NewValue0A(pos, OpStaticCall, types.TypeResultMem, StaticAuxCall(fn, b.Func.ABIDefault.ABIAnalyzeTypes(argTypes, nil)))
   701  	call.AddArgs(args...)
   702  	call.AuxInt = int64(nargs) * typ.Size()
   703  	return b.NewValue1I(pos, OpSelectN, types.TypeMem, 0, call)
   704  }
   705  
   706  // IsStackAddr reports whether v is known to be an address of a stack slot.
   707  func IsStackAddr(v *Value) bool {
   708  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy {
   709  		v = v.Args[0]
   710  	}
   711  	switch v.Op {
   712  	case OpSP, OpLocalAddr, OpSelectNAddr, OpGetCallerSP:
   713  		return true
   714  	}
   715  	return false
   716  }
   717  
   718  // IsGlobalAddr reports whether v is known to be an address of a global (or nil).
   719  func IsGlobalAddr(v *Value) bool {
   720  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy {
   721  		v = v.Args[0]
   722  	}
   723  	if v.Op == OpAddr && v.Args[0].Op == OpSB {
   724  		return true // address of a global
   725  	}
   726  	if v.Op == OpConstNil {
   727  		return true
   728  	}
   729  	if v.Op == OpLoad && IsReadOnlyGlobalAddr(v.Args[0]) {
   730  		return true // loading from a read-only global - the resulting address can't be a heap address.
   731  	}
   732  	return false
   733  }
   734  
   735  // IsReadOnlyGlobalAddr reports whether v is known to be an address of a read-only global.
   736  func IsReadOnlyGlobalAddr(v *Value) bool {
   737  	if v.Op == OpConstNil {
   738  		// Nil pointers are read only. See issue 33438.
   739  		return true
   740  	}
   741  	if v.Op == OpAddr && v.Aux != nil && v.Aux.(*obj.LSym).Type == objabi.SRODATA {
   742  		return true
   743  	}
   744  	return false
   745  }
   746  
   747  // IsNewObject reports whether v is a pointer to a freshly allocated & zeroed object,
   748  // if so, also returns the memory state mem at which v is zero.
   749  func IsNewObject(v *Value, select1 []*Value) (mem *Value, ok bool) {
   750  	f := v.Block.Func
   751  	c := f.Config
   752  	if f.ABIDefault == f.ABI1 && len(c.intParamRegs) >= 1 {
   753  		if v.Op != OpSelectN || v.AuxInt != 0 {
   754  			return nil, false
   755  		}
   756  		mem = select1[v.Args[0].ID]
   757  		if mem == nil {
   758  			return nil, false
   759  		}
   760  	} else {
   761  		if v.Op != OpLoad {
   762  			return nil, false
   763  		}
   764  		mem = v.MemoryArg()
   765  		if mem.Op != OpSelectN {
   766  			return nil, false
   767  		}
   768  		if mem.Type != types.TypeMem {
   769  			return nil, false
   770  		} // assume it is the right selection if true
   771  	}
   772  	call := mem.Args[0]
   773  	if call.Op != OpStaticCall {
   774  		return nil, false
   775  	}
   776  	if !isSameCall(call.Aux, "runtime.newobject") {
   777  		return nil, false
   778  	}
   779  	if f.ABIDefault == f.ABI1 && len(c.intParamRegs) >= 1 {
   780  		if v.Args[0] == call {
   781  			return mem, true
   782  		}
   783  		return nil, false
   784  	}
   785  	if v.Args[0].Op != OpOffPtr {
   786  		return nil, false
   787  	}
   788  	if v.Args[0].Args[0].Op != OpSP {
   789  		return nil, false
   790  	}
   791  	if v.Args[0].AuxInt != c.ctxt.Arch.FixedFrameSize+c.RegSize { // offset of return value
   792  		return nil, false
   793  	}
   794  	return mem, true
   795  }
   796  
   797  // IsSanitizerSafeAddr reports whether v is known to be an address
   798  // that doesn't need instrumentation.
   799  func IsSanitizerSafeAddr(v *Value) bool {
   800  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy {
   801  		v = v.Args[0]
   802  	}
   803  	switch v.Op {
   804  	case OpSP, OpLocalAddr, OpSelectNAddr:
   805  		// Stack addresses are always safe.
   806  		return true
   807  	case OpITab, OpStringPtr, OpGetClosurePtr:
   808  		// Itabs, string data, and closure fields are
   809  		// read-only once initialized.
   810  		return true
   811  	case OpAddr:
   812  		vt := v.Aux.(*obj.LSym).Type
   813  		return vt == objabi.SRODATA || vt == objabi.SLIBFUZZER_8BIT_COUNTER || vt == objabi.SCOVERAGE_COUNTER || vt == objabi.SCOVERAGE_AUXVAR
   814  	}
   815  	return false
   816  }
   817  
   818  // isVolatile reports whether v is a pointer to argument region on stack which
   819  // will be clobbered by a function call.
   820  func isVolatile(v *Value) bool {
   821  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy || v.Op == OpSelectNAddr {
   822  		v = v.Args[0]
   823  	}
   824  	return v.Op == OpSP
   825  }
   826
View as plain text