mcheckmark.go

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // GC checkmarks
     6  //
     7  // In a concurrent garbage collector, one worries about failing to mark
     8  // a live object due to mutations without write barriers or bugs in the
     9  // collector implementation. As a sanity check, the GC has a 'checkmark'
    10  // mode that retraverses the object graph with the world stopped, to make
    11  // sure that everything that should be marked is marked.
    12  
    13  package runtime
    14  
    15  import (
    16  	"internal/goarch"
    17  	"internal/runtime/atomic"
    18  	"internal/runtime/sys"
    19  	"unsafe"
    20  )
    21  
    22  // A checkmarksMap stores the GC marks in "checkmarks" mode. It is a
    23  // per-arena bitmap with a bit for every word in the arena. The mark
    24  // is stored on the bit corresponding to the first word of the marked
    25  // allocation.
    26  type checkmarksMap struct {
    27  	_ sys.NotInHeap
    28  	b [heapArenaBytes / goarch.PtrSize / 8]uint8
    29  }
    30  
    31  // If useCheckmark is true, marking of an object uses the checkmark
    32  // bits instead of the standard mark bits.
    33  var useCheckmark = false
    34  
    35  // startCheckmarks prepares for the checkmarks phase.
    36  //
    37  // The world must be stopped.
    38  func startCheckmarks() {
    39  	assertWorldStopped()
    40  
    41  	// Clear all checkmarks.
    42  	clearCheckmarks := func(ai arenaIdx) {
    43  		arena := mheap_.arenas[ai.l1()][ai.l2()]
    44  		bitmap := arena.checkmarks
    45  
    46  		if bitmap == nil {
    47  			// Allocate bitmap on first use.
    48  			bitmap = (*checkmarksMap)(persistentalloc(unsafe.Sizeof(*bitmap), 0, &memstats.gcMiscSys))
    49  			if bitmap == nil {
    50  				throw("out of memory allocating checkmarks bitmap")
    51  			}
    52  			arena.checkmarks = bitmap
    53  		} else {
    54  			// Otherwise clear the existing bitmap.
    55  			clear(bitmap.b[:])
    56  		}
    57  	}
    58  	for _, ai := range mheap_.heapArenas {
    59  		clearCheckmarks(ai)
    60  	}
    61  	for _, ai := range mheap_.userArenaArenas {
    62  		clearCheckmarks(ai)
    63  	}
    64  
    65  	// Enable checkmarking.
    66  	useCheckmark = true
    67  }
    68  
    69  // endCheckmarks ends the checkmarks phase.
    70  func endCheckmarks() {
    71  	if gcMarkWorkAvailable(nil) {
    72  		throw("GC work not flushed")
    73  	}
    74  	useCheckmark = false
    75  }
    76  
    77  // setCheckmark throws if marking object is a checkmarks violation,
    78  // and otherwise sets obj's checkmark. It returns true if obj was
    79  // already checkmarked.
    80  func setCheckmark(obj, base, off uintptr, mbits markBits) bool {
    81  	if !mbits.isMarked() {
    82  		printlock()
    83  		print("runtime: checkmarks found unexpected unmarked object obj=", hex(obj), "\n")
    84  		print("runtime: found obj at *(", hex(base), "+", hex(off), ")\n")
    85  
    86  		// Dump the source (base) object
    87  		gcDumpObject("base", base, off)
    88  
    89  		// Dump the object
    90  		gcDumpObject("obj", obj, ^uintptr(0))
    91  
    92  		getg().m.traceback = 2
    93  		throw("checkmark found unmarked object")
    94  	}
    95  
    96  	ai := arenaIndex(obj)
    97  	arena := mheap_.arenas[ai.l1()][ai.l2()]
    98  	if arena == nil {
    99  		// Non-heap pointer.
   100  		return false
   101  	}
   102  	wordIdx := (obj - alignDown(obj, heapArenaBytes)) / goarch.PtrSize
   103  	arenaWord := wordIdx / 8
   104  	mask := byte(1 << (wordIdx % 8))
   105  	bytep := &arena.checkmarks.b[arenaWord]
   106  
   107  	if atomic.Load8(bytep)&mask != 0 {
   108  		// Already checkmarked.
   109  		return true
   110  	}
   111  
   112  	atomic.Or8(bytep, mask)
   113  	return false
   114  }
   115
View as plain text