Source file src/runtime/mem.go
1 // Copyright 2022 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import "unsafe" 8 9 // OS memory management abstraction layer 10 // 11 // Regions of the address space managed by the runtime may be in one of four 12 // states at any given time: 13 // 1) None - Unreserved and unmapped, the default state of any region. 14 // 2) Reserved - Owned by the runtime, but accessing it would cause a fault. 15 // Does not count against the process' memory footprint. 16 // 3) Prepared - Reserved, intended not to be backed by physical memory (though 17 // an OS may implement this lazily). Can transition efficiently to 18 // Ready. Accessing memory in such a region is undefined (may 19 // fault, may give back unexpected zeroes, etc.). 20 // 4) Ready - may be accessed safely. 21 // 22 // This set of states is more than is strictly necessary to support all the 23 // currently supported platforms. One could get by with just None, Reserved, and 24 // Ready. However, the Prepared state gives us flexibility for performance 25 // purposes. For example, on POSIX-y operating systems, Reserved is usually a 26 // private anonymous mmap'd region with PROT_NONE set, and to transition 27 // to Ready would require setting PROT_READ|PROT_WRITE. However the 28 // underspecification of Prepared lets us use just MADV_FREE to transition from 29 // Ready to Prepared. Thus with the Prepared state we can set the permission 30 // bits just once early on, we can efficiently tell the OS that it's free to 31 // take pages away from us when we don't strictly need them. 32 // 33 // This file defines a cross-OS interface for a common set of helpers 34 // that transition memory regions between these states. The helpers call into 35 // OS-specific implementations that handle errors, while the interface boundary 36 // implements cross-OS functionality, like updating runtime accounting. 37 38 // sysAlloc transitions an OS-chosen region of memory from None to Ready. 39 // More specifically, it obtains a large chunk of zeroed memory from the 40 // operating system, typically on the order of a hundred kilobytes 41 // or a megabyte. This memory is always immediately available for use. 42 // 43 // sysStat must be non-nil. 44 // 45 // Don't split the stack as this function may be invoked without a valid G, 46 // which prevents us from allocating more stack. 47 // 48 //go:nosplit 49 func sysAlloc(n uintptr, sysStat *sysMemStat, vmaName string) unsafe.Pointer { 50 sysStat.add(int64(n)) 51 gcController.mappedReady.Add(int64(n)) 52 p := sysAllocOS(n, vmaName) 53 54 // When using ASAN leak detection, we must tell ASAN about 55 // cases where we store pointers in mmapped memory. 56 if asanenabled { 57 lsanregisterrootregion(p, n) 58 } 59 60 return p 61 } 62 63 // sysUnused transitions a memory region from Ready to Prepared. It notifies the 64 // operating system that the physical pages backing this memory region are no 65 // longer needed and can be reused for other purposes. The contents of a 66 // sysUnused memory region are considered forfeit and the region must not be 67 // accessed again until sysUsed is called. 68 func sysUnused(v unsafe.Pointer, n uintptr) { 69 gcController.mappedReady.Add(-int64(n)) 70 sysUnusedOS(v, n) 71 } 72 73 // sysUsed transitions a memory region from Prepared to Ready. It notifies the 74 // operating system that the memory region is needed and ensures that the region 75 // may be safely accessed. This is typically a no-op on systems that don't have 76 // an explicit commit step and hard over-commit limits, but is critical on 77 // Windows, for example. 78 // 79 // This operation is idempotent for memory already in the Prepared state, so 80 // it is safe to refer, with v and n, to a range of memory that includes both 81 // Prepared and Ready memory. However, the caller must provide the exact amount 82 // of Prepared memory for accounting purposes. 83 func sysUsed(v unsafe.Pointer, n, prepared uintptr) { 84 gcController.mappedReady.Add(int64(prepared)) 85 sysUsedOS(v, n) 86 } 87 88 // sysHugePage does not transition memory regions, but instead provides a 89 // hint to the OS that it would be more efficient to back this memory region 90 // with pages of a larger size transparently. 91 func sysHugePage(v unsafe.Pointer, n uintptr) { 92 sysHugePageOS(v, n) 93 } 94 95 // sysNoHugePage does not transition memory regions, but instead provides a 96 // hint to the OS that it would be less efficient to back this memory region 97 // with pages of a larger size transparently. 98 func sysNoHugePage(v unsafe.Pointer, n uintptr) { 99 sysNoHugePageOS(v, n) 100 } 101 102 // sysHugePageCollapse attempts to immediately back the provided memory region 103 // with huge pages. It is best-effort and may fail silently. 104 func sysHugePageCollapse(v unsafe.Pointer, n uintptr) { 105 sysHugePageCollapseOS(v, n) 106 } 107 108 // sysFree transitions a memory region from any state to None. Therefore, it 109 // returns memory unconditionally. It is used if an out-of-memory error has been 110 // detected midway through an allocation or to carve out an aligned section of 111 // the address space. It is okay if sysFree is a no-op only if sysReserve always 112 // returns a memory region aligned to the heap allocator's alignment 113 // restrictions. 114 // 115 // sysStat must be non-nil. 116 // 117 // Don't split the stack as this function may be invoked without a valid G, 118 // which prevents us from allocating more stack. 119 // 120 //go:nosplit 121 func sysFree(v unsafe.Pointer, n uintptr, sysStat *sysMemStat) { 122 // When using ASAN leak detection, the memory being freed is 123 // known by the sanitizer. We need to unregister it so it's 124 // not accessed by it. 125 if asanenabled { 126 lsanunregisterrootregion(v, n) 127 } 128 129 sysStat.add(-int64(n)) 130 gcController.mappedReady.Add(-int64(n)) 131 sysFreeOS(v, n) 132 } 133 134 // sysFault transitions a memory region from Ready to Reserved. It 135 // marks a region such that it will always fault if accessed. Used only for 136 // debugging the runtime. 137 // 138 // TODO(mknyszek): Currently it's true that all uses of sysFault transition 139 // memory from Ready to Reserved, but this may not be true in the future 140 // since on every platform the operation is much more general than that. 141 // If a transition from Prepared is ever introduced, create a new function 142 // that elides the Ready state accounting. 143 func sysFault(v unsafe.Pointer, n uintptr) { 144 gcController.mappedReady.Add(-int64(n)) 145 sysFaultOS(v, n) 146 } 147 148 // sysReserve transitions a memory region from None to Reserved. It reserves 149 // address space in such a way that it would cause a fatal fault upon access 150 // (either via permissions or not committing the memory). Such a reservation is 151 // thus never backed by physical memory. 152 // 153 // If the pointer passed to it is non-nil, the caller wants the 154 // reservation there, but sysReserve can still choose another 155 // location if that one is unavailable. 156 // 157 // NOTE: sysReserve returns OS-aligned memory, but the heap allocator 158 // may use larger alignment, so the caller must be careful to realign the 159 // memory obtained by sysReserve. 160 func sysReserve(v unsafe.Pointer, n uintptr, vmaName string) unsafe.Pointer { 161 p := sysReserveOS(v, n, vmaName) 162 163 // When using ASAN leak detection, we must tell ASAN about 164 // cases where we store pointers in mmapped memory. 165 if asanenabled { 166 lsanregisterrootregion(p, n) 167 } 168 169 return p 170 } 171 172 // sysMap transitions a memory region from Reserved to Prepared. It ensures the 173 // memory region can be efficiently transitioned to Ready. 174 // 175 // sysStat must be non-nil. 176 func sysMap(v unsafe.Pointer, n uintptr, sysStat *sysMemStat, vmaName string) { 177 sysStat.add(int64(n)) 178 sysMapOS(v, n, vmaName) 179 } 180