mem_linux.go

     1  // Copyright 2010 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/runtime/atomic"
     9  	"unsafe"
    10  )
    11  
    12  const (
    13  	_EACCES = 13
    14  	_EINVAL = 22
    15  )
    16  
    17  // Don't split the stack as this method may be invoked without a valid G, which
    18  // prevents us from allocating more stack.
    19  //
    20  //go:nosplit
    21  func sysAllocOS(n uintptr, vmaName string) unsafe.Pointer {
    22  	p, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
    23  	if err != 0 {
    24  		if err == _EACCES {
    25  			print("runtime: mmap: access denied\n")
    26  			exit(2)
    27  		}
    28  		if err == _EAGAIN {
    29  			print("runtime: mmap: too much locked memory (check 'ulimit -l').\n")
    30  			exit(2)
    31  		}
    32  		return nil
    33  	}
    34  	setVMAName(p, n, vmaName)
    35  	return p
    36  }
    37  
    38  var adviseUnused = uint32(_MADV_FREE)
    39  
    40  const madviseUnsupported = 0
    41  
    42  func sysUnusedOS(v unsafe.Pointer, n uintptr) {
    43  	if uintptr(v)&(physPageSize-1) != 0 || n&(physPageSize-1) != 0 {
    44  		// madvise will round this to any physical page
    45  		// *covered* by this range, so an unaligned madvise
    46  		// will release more memory than intended.
    47  		throw("unaligned sysUnused")
    48  	}
    49  
    50  	advise := atomic.Load(&adviseUnused)
    51  	if debug.madvdontneed != 0 && advise != madviseUnsupported {
    52  		advise = _MADV_DONTNEED
    53  	}
    54  	switch advise {
    55  	case _MADV_FREE:
    56  		if madvise(v, n, _MADV_FREE) == 0 {
    57  			break
    58  		}
    59  		atomic.Store(&adviseUnused, _MADV_DONTNEED)
    60  		fallthrough
    61  	case _MADV_DONTNEED:
    62  		// MADV_FREE was added in Linux 4.5. Fall back on MADV_DONTNEED if it's
    63  		// not supported.
    64  		if madvise(v, n, _MADV_DONTNEED) == 0 {
    65  			break
    66  		}
    67  		atomic.Store(&adviseUnused, madviseUnsupported)
    68  		fallthrough
    69  	case madviseUnsupported:
    70  		// Since Linux 3.18, support for madvise is optional.
    71  		// Fall back on mmap if it's not supported.
    72  		// _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE will unmap all the
    73  		// pages in the old mapping, and remap the memory region.
    74  		p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
    75  		if err == 0 && p != nil {
    76  			setVMAName(p, n, "unused")
    77  		}
    78  	}
    79  
    80  	if debug.harddecommit > 0 {
    81  		p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
    82  		if p != v || err != 0 {
    83  			throw("runtime: cannot disable permissions in address space")
    84  		}
    85  		setVMAName(p, n, "unused")
    86  	}
    87  }
    88  
    89  func sysUsedOS(v unsafe.Pointer, n uintptr) {
    90  	if debug.harddecommit > 0 {
    91  		p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
    92  		if err == _ENOMEM {
    93  			throw("runtime: out of memory")
    94  		}
    95  		if p != v || err != 0 {
    96  			throw("runtime: cannot remap pages in address space")
    97  		}
    98  		setVMAName(p, n, "used")
    99  		return
   100  	}
   101  }
   102  
   103  func sysHugePageOS(v unsafe.Pointer, n uintptr) {
   104  	if physHugePageSize != 0 {
   105  		// Round v up to a huge page boundary.
   106  		beg := alignUp(uintptr(v), physHugePageSize)
   107  		// Round v+n down to a huge page boundary.
   108  		end := alignDown(uintptr(v)+n, physHugePageSize)
   109  
   110  		if beg < end {
   111  			madvise(unsafe.Pointer(beg), end-beg, _MADV_HUGEPAGE)
   112  		}
   113  	}
   114  }
   115  
   116  func sysNoHugePageOS(v unsafe.Pointer, n uintptr) {
   117  	if uintptr(v)&(physPageSize-1) != 0 {
   118  		// The Linux implementation requires that the address
   119  		// addr be page-aligned, and allows length to be zero.
   120  		throw("unaligned sysNoHugePageOS")
   121  	}
   122  	madvise(v, n, _MADV_NOHUGEPAGE)
   123  }
   124  
   125  func sysHugePageCollapseOS(v unsafe.Pointer, n uintptr) {
   126  	if uintptr(v)&(physPageSize-1) != 0 {
   127  		// The Linux implementation requires that the address
   128  		// addr be page-aligned, and allows length to be zero.
   129  		throw("unaligned sysHugePageCollapseOS")
   130  	}
   131  	if physHugePageSize == 0 {
   132  		return
   133  	}
   134  	// N.B. If you find yourself debugging this code, note that
   135  	// this call can fail with EAGAIN because it's best-effort.
   136  	// Also, when it returns an error, it's only for the last
   137  	// huge page in the region requested.
   138  	//
   139  	// It can also sometimes return EINVAL if the corresponding
   140  	// region hasn't been backed by physical memory. This is
   141  	// difficult to guarantee in general, and it also means
   142  	// there's no way to distinguish whether this syscall is
   143  	// actually available. Oops.
   144  	//
   145  	// Anyway, that's why this call just doesn't bother checking
   146  	// any errors.
   147  	madvise(v, n, _MADV_COLLAPSE)
   148  }
   149  
   150  // Don't split the stack as this function may be invoked without a valid G,
   151  // which prevents us from allocating more stack.
   152  //
   153  //go:nosplit
   154  func sysFreeOS(v unsafe.Pointer, n uintptr) {
   155  	munmap(v, n)
   156  }
   157  
   158  func sysFaultOS(v unsafe.Pointer, n uintptr) {
   159  	mprotect(v, n, _PROT_NONE)
   160  	madvise(v, n, _MADV_DONTNEED)
   161  }
   162  
   163  func sysReserveOS(v unsafe.Pointer, n uintptr, vmaName string) unsafe.Pointer {
   164  	p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
   165  	if err != 0 {
   166  		return nil
   167  	}
   168  	setVMAName(p, n, vmaName)
   169  	return p
   170  }
   171  
   172  func sysMapOS(v unsafe.Pointer, n uintptr, vmaName string) {
   173  	p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
   174  	if err == _ENOMEM {
   175  		throw("runtime: out of memory")
   176  	}
   177  	if p != v || err != 0 {
   178  		print("runtime: mmap(", v, ", ", n, ") returned ", p, ", ", err, "\n")
   179  		throw("runtime: cannot map pages in arena address space")
   180  	}
   181  	setVMAName(p, n, vmaName)
   182  
   183  	// Disable huge pages if the GODEBUG for it is set.
   184  	//
   185  	// Note that there are a few sysHugePage calls that can override this, but
   186  	// they're all for GC metadata.
   187  	if debug.disablethp != 0 {
   188  		sysNoHugePageOS(v, n)
   189  	}
   190  }
   191
View as plain text