Text file
src/runtime/asm_riscv64.s
1 // Copyright 2017 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "funcdata.h"
7 #include "textflag.h"
8 #include "cgo/abi_riscv64.h"
9
10
11 // When building with -buildmode=c-shared, this symbol is called when the shared
12 // library is loaded.
13 TEXT _rt0_riscv64_lib(SB),NOSPLIT,$224
14 // Preserve callee-save registers, along with X1 (LR).
15 MOV X1, (8*3)(X2)
16 SAVE_GPR((8*4))
17 SAVE_FPR((8*16))
18
19 // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go
20 MOV X0, g
21
22 MOV A0, _rt0_riscv64_lib_argc<>(SB)
23 MOV A1, _rt0_riscv64_lib_argv<>(SB)
24
25 MOV $runtime·libInit(SB), T1
26 JALR RA, T1
27
28 // Restore callee-save registers, along with X1 (LR).
29 MOV (8*3)(X2), X1
30 RESTORE_GPR((8*4))
31 RESTORE_FPR((8*16))
32
33 RET
34
35 TEXT runtime·rt0_lib_go<ABIInternal>(SB),NOSPLIT,$0
36 MOV _rt0_riscv64_lib_argc<>(SB), A0
37 MOV _rt0_riscv64_lib_argv<>(SB), A1
38 MOV $runtime·rt0_go(SB), T0
39 JALR ZERO, T0
40
41 DATA _rt0_riscv64_lib_argc<>(SB)/8, $0
42 GLOBL _rt0_riscv64_lib_argc<>(SB),NOPTR, $8
43 DATA _rt0_riscv64_lib_argv<>(SB)/8, $0
44 GLOBL _rt0_riscv64_lib_argv<>(SB),NOPTR, $8
45
46 // func rt0_go()
47 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
48 // X2 = stack; A0 = argc; A1 = argv
49 SUB $24, X2
50 MOV A0, 8(X2) // argc
51 MOV A1, 16(X2) // argv
52
53 // create istack out of the given (operating system) stack.
54 // _cgo_init may update stackguard.
55 MOV $runtime·g0(SB), g
56 MOV $(-64*1024), T0
57 ADD T0, X2, T1
58 MOV T1, g_stackguard0(g)
59 MOV T1, g_stackguard1(g)
60 MOV T1, (g_stack+stack_lo)(g)
61 MOV X2, (g_stack+stack_hi)(g)
62
63 // if there is a _cgo_init, call it using the gcc ABI.
64 MOV _cgo_init(SB), T2
65 BEQ T2, ZERO, nocgo
66
67 MOV ZERO, A3 // arg 3: not used
68 MOV ZERO, A2 // arg 2: not used
69 MOV $setg_gcc<>(SB), A1 // arg 1: setg
70 MOV g, A0 // arg 0: G
71 JALR RA, T2
72
73 nocgo:
74 // update stackguard after _cgo_init
75 MOV (g_stack+stack_lo)(g), T0
76 ADD $const_stackGuard, T0
77 MOV T0, g_stackguard0(g)
78 MOV T0, g_stackguard1(g)
79
80 // set the per-goroutine and per-mach "registers"
81 MOV $runtime·m0(SB), T0
82
83 // save m->g0 = g0
84 MOV g, m_g0(T0)
85 // save m0 to g0->m
86 MOV T0, g_m(g)
87
88 CALL runtime·check(SB)
89
90 // args are already prepared
91 CALL runtime·args(SB)
92 CALL runtime·osinit(SB)
93 CALL runtime·schedinit(SB)
94
95 // create a new goroutine to start program
96 MOV $runtime·mainPC(SB), T0 // entry
97 SUB $16, X2
98 MOV T0, 8(X2)
99 MOV ZERO, 0(X2)
100 CALL runtime·newproc(SB)
101 ADD $16, X2
102
103 // start this M
104 CALL runtime·stackcheck(SB) // fault if stack check is wrong
105 CALL runtime·mstart(SB)
106
107 WORD $0 // crash if reached
108 RET
109
110 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
111 CALL runtime·mstart0(SB)
112 RET // not reached
113
114 // void setg_gcc(G*); set g called from gcc with g in A0
115 TEXT setg_gcc<>(SB),NOSPLIT,$0-0
116 MOV A0, g
117 CALL runtime·save_g(SB)
118 RET
119
120 // func cputicks() int64
121 TEXT runtime·cputicks<ABIInternal>(SB),NOSPLIT,$0-0
122 // RDTIME to emulate cpu ticks
123 // RDCYCLE reads counter that is per HART(core) based
124 // according to the riscv manual, see issue 46737
125 RDTIME X10
126 RET
127
128 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
129 // of the G stack. We need to distinguish the routine that
130 // lives at the bottom of the G stack from the one that lives
131 // at the top of the system stack because the one at the top of
132 // the system stack terminates the stack walk (see topofstack()).
133 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
134 UNDEF
135 JALR RA, ZERO // make sure this function is not leaf
136 RET
137
138 // func systemstack(fn func())
139 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
140 MOV fn+0(FP), CTXT // CTXT = fn
141 MOV g_m(g), T0 // T0 = m
142
143 MOV m_gsignal(T0), T1 // T1 = gsignal
144 BEQ g, T1, noswitch
145
146 MOV m_g0(T0), T1 // T1 = g0
147 BEQ g, T1, noswitch
148
149 MOV m_curg(T0), T2
150 BEQ g, T2, switch
151
152 // Bad: g is not gsignal, not g0, not curg. What is it?
153 // Hide call from linker nosplit analysis.
154 MOV $runtime·badsystemstack(SB), T1
155 JALR RA, T1
156
157 switch:
158 // save our state in g->sched. Pretend to
159 // be systemstack_switch if the G stack is scanned.
160 CALL gosave_systemstack_switch<>(SB)
161
162 // switch to g0
163 MOV T1, g
164 CALL runtime·save_g(SB)
165 MOV (g_sched+gobuf_sp)(g), T0
166 MOV T0, X2
167
168 // call target function
169 MOV 0(CTXT), T1 // code pointer
170 JALR RA, T1
171
172 // switch back to g
173 MOV g_m(g), T0
174 MOV m_curg(T0), g
175 CALL runtime·save_g(SB)
176 MOV (g_sched+gobuf_sp)(g), X2
177 MOV ZERO, (g_sched+gobuf_sp)(g)
178 RET
179
180 noswitch:
181 // already on m stack, just call directly
182 // Using a tail call here cleans up tracebacks since we won't stop
183 // at an intermediate systemstack.
184 MOV 0(CTXT), T1 // code pointer
185 ADD $8, X2
186 JMP (T1)
187
188 // func switchToCrashStack0(fn func())
189 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
190 MOV X10, CTXT // context register
191 MOV g_m(g), X11 // curm
192
193 // set g to gcrash
194 MOV $runtime·gcrash(SB), g // g = &gcrash
195 CALL runtime·save_g(SB) // clobbers X31
196 MOV X11, g_m(g) // g.m = curm
197 MOV g, m_g0(X11) // curm.g0 = g
198
199 // switch to crashstack
200 MOV (g_stack+stack_hi)(g), X11
201 SUB $(4*8), X11
202 MOV X11, X2
203
204 // call target function
205 MOV 0(CTXT), X10
206 JALR X1, X10
207
208 // should never return
209 CALL runtime·abort(SB)
210 UNDEF
211
212 /*
213 * support for morestack
214 */
215
216 // Called during function prolog when more stack is needed.
217 // Called with return address (i.e. caller's PC) in X5 (aka T0),
218 // and the LR register contains the caller's LR.
219 //
220 // The traceback routines see morestack on a g0 as being
221 // the top of a stack (for example, morestack calling newstack
222 // calling the scheduler calling newm calling gc), so we must
223 // record an argument size. For that purpose, it has no arguments.
224
225 // func morestack()
226 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
227 // Called from f.
228 // Set g->sched to context in f.
229 MOV X2, (g_sched+gobuf_sp)(g)
230 MOV T0, (g_sched+gobuf_pc)(g)
231 MOV RA, (g_sched+gobuf_lr)(g)
232 MOV CTXT, (g_sched+gobuf_ctxt)(g)
233
234 // Cannot grow scheduler stack (m->g0).
235 MOV g_m(g), A0
236 MOV m_g0(A0), A1
237 BNE g, A1, 3(PC)
238 CALL runtime·badmorestackg0(SB)
239 CALL runtime·abort(SB)
240
241 // Cannot grow signal stack (m->gsignal).
242 MOV m_gsignal(A0), A1
243 BNE g, A1, 3(PC)
244 CALL runtime·badmorestackgsignal(SB)
245 CALL runtime·abort(SB)
246
247 // Called from f.
248 // Set m->morebuf to f's caller.
249 MOV RA, (m_morebuf+gobuf_pc)(A0) // f's caller's PC
250 MOV X2, (m_morebuf+gobuf_sp)(A0) // f's caller's SP
251 MOV g, (m_morebuf+gobuf_g)(A0)
252
253 // Call newstack on m->g0's stack.
254 MOV m_g0(A0), g
255 CALL runtime·save_g(SB)
256 MOV (g_sched+gobuf_sp)(g), X2
257 // Create a stack frame on g0 to call newstack.
258 MOV ZERO, -8(X2) // Zero saved LR in frame
259 SUB $8, X2
260 CALL runtime·newstack(SB)
261
262 // Not reached, but make sure the return PC from the call to newstack
263 // is still in this function, and not the beginning of the next.
264 UNDEF
265
266 // func morestack_noctxt()
267 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
268 // Force SPWRITE. This function doesn't actually write SP,
269 // but it is called with a special calling convention where
270 // the caller doesn't save LR on stack but passes it as a
271 // register, and the unwinder currently doesn't understand.
272 // Make it SPWRITE to stop unwinding. (See issue 54332)
273 MOV X2, X2
274
275 MOV ZERO, CTXT
276 JMP runtime·morestack(SB)
277
278 // check that SP is in range [g->stack.lo, g->stack.hi]
279 TEXT runtime·stackcheck(SB), NOSPLIT|NOFRAME, $0-0
280 MOV (g_stack+stack_hi)(g), A0
281 BGEU A0, X2, 2(PC)
282 CALL runtime·abort(SB)
283
284 MOV (g_stack+stack_lo)(g), A0
285 BGTU X2, A0, 2(PC)
286 CALL runtime·abort(SB)
287 RET
288
289 // restore state from Gobuf; longjmp
290
291 // func gogo(buf *gobuf)
292 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
293 MOV buf+0(FP), T0
294 MOV gobuf_g(T0), T1
295 MOV 0(T1), ZERO // make sure g != nil
296 JMP gogo<>(SB)
297
298 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
299 MOV T1, g
300 CALL runtime·save_g(SB)
301
302 MOV gobuf_sp(T0), X2
303 MOV gobuf_lr(T0), RA
304 MOV gobuf_ctxt(T0), CTXT
305 MOV ZERO, gobuf_sp(T0)
306 MOV ZERO, gobuf_lr(T0)
307 MOV ZERO, gobuf_ctxt(T0)
308 MOV gobuf_pc(T0), T0
309 JALR ZERO, T0
310
311 // func procyieldAsm(cycles uint32)
312 TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0
313 RET
314
315 // Switch to m->g0's stack, call fn(g).
316 // Fn must never return. It should gogo(&g->sched)
317 // to keep running g.
318
319 // func mcall(fn func(*g))
320 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
321 MOV X10, CTXT
322
323 // Save caller state in g->sched
324 MOV X2, (g_sched+gobuf_sp)(g)
325 MOV RA, (g_sched+gobuf_pc)(g)
326 MOV ZERO, (g_sched+gobuf_lr)(g)
327
328 // Switch to m->g0 & its stack, call fn.
329 MOV g, X10
330 MOV g_m(g), T1
331 MOV m_g0(T1), g
332 CALL runtime·save_g(SB)
333 BNE g, X10, 2(PC)
334 JMP runtime·badmcall(SB)
335 MOV 0(CTXT), T1 // code pointer
336 MOV (g_sched+gobuf_sp)(g), X2 // sp = m->g0->sched.sp
337 // we don't need special macro for regabi since arg0(X10) = g
338 SUB $16, X2
339 MOV X10, 8(X2) // setup g
340 MOV ZERO, 0(X2) // clear return address
341 JALR RA, T1
342 JMP runtime·badmcall2(SB)
343
344 // Save state of caller into g->sched,
345 // but using fake PC from systemstack_switch.
346 // Must only be called from functions with no locals ($0)
347 // or else unwinding from systemstack_switch is incorrect.
348 // Smashes X31.
349 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
350 MOV $runtime·systemstack_switch(SB), X31
351 ADD $8, X31 // get past prologue
352 MOV X31, (g_sched+gobuf_pc)(g)
353 MOV X2, (g_sched+gobuf_sp)(g)
354 MOV ZERO, (g_sched+gobuf_lr)(g)
355 // Assert ctxt is zero. See func save.
356 MOV (g_sched+gobuf_ctxt)(g), X31
357 BEQ ZERO, X31, 2(PC)
358 CALL runtime·abort(SB)
359 RET
360
361 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
362 // Call fn(arg) aligned appropriately for the gcc ABI.
363 // Called on a system stack, and there may be no g yet (during needm).
364 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
365 MOV fn+0(FP), X11
366 MOV arg+8(FP), X10
367 MOV X2, X9 // save SP in X9 (callee-saved in C ABI)
368 ANDI $~15, X2 // align SP to 16 bytes per C ABI
369 MOV X0, X8 // clear frame pointer register (see asmcgocall)
370 JALR RA, (X11)
371 MOV X9, X2
372 RET
373
374 // func asmcgocall(fn, arg unsafe.Pointer) int32
375 // Call fn(arg) on the scheduler stack,
376 // aligned appropriately for the gcc ABI.
377 // See cgocall.go for more details.
378 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
379 MOV fn+0(FP), X11
380 MOV arg+8(FP), X10
381
382 MOV X2, X8 // save original stack pointer
383 MOV g, X9
384
385 // Figure out if we need to switch to m->g0 stack.
386 // We get called to create new OS threads too, and those
387 // come in on the m->g0 stack already. Or we might already
388 // be on the m->gsignal stack.
389 BEQZ g, nosave
390 MOV g_m(g), X6
391 MOV m_gsignal(X6), X7
392 BEQ X7, g, g0
393 MOV m_g0(X6), X7
394 BEQ X7, g, g0
395
396 CALL gosave_systemstack_switch<>(SB)
397 MOV X7, g
398 CALL runtime·save_g(SB)
399 MOV (g_sched+gobuf_sp)(g), X2
400
401 // Now on a scheduling stack (a pthread-created stack).
402 g0:
403 ANDI $~15, X2 // align SP to 16 bytes per C ABI
404 // Save room for two of our pointers.
405 SUB $16, X2
406 MOV X9, 0(X2) // save old g on stack
407 MOV (g_stack+stack_hi)(X9), X9
408 SUB X8, X9, X8
409 MOV X8, 8(X2) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
410
411 // Clear the frame pointer register before calling into C.
412 // At least some C unwinder does frame pointer unwinding.
413 // As Go currently doesn't use frame pointer on RISCV64,
414 // the C unwinder may see garbage value and may crash.
415 // Zero the frame pointer to tell the unwinder to stop
416 // (it is a stack switch anyway).
417 // If we enable frame pointers in Go, revisit this.
418 MOV X0, X8
419 JALR RA, (X11)
420
421 // Restore g, stack pointer. X10 is return value.
422 MOV 0(X2), g
423 CALL runtime·save_g(SB)
424 MOV (g_stack+stack_hi)(g), X5
425 MOV 8(X2), X6
426 SUB X6, X5, X6
427 MOV X6, X2
428
429 MOVW X10, ret+16(FP)
430 RET
431
432 nosave:
433 // Running on a system stack, perhaps even without a g.
434 // Having no g can happen during thread creation or thread teardown.
435 MOV fn+0(FP), X11
436 MOV arg+8(FP), X10
437 MOV X2, X8
438 ANDI $~15, X2 // align SP to 16 bytes per C ABI
439 SUB $16, X2
440 MOV ZERO, 0(X2) // Where above code stores g, in case someone looks during debugging.
441 MOV X8, 8(X2) // Save original stack pointer.
442 MOV X0, X8 // Clear frame pointer (see above)
443 JALR RA, (X11)
444 MOV 8(X2), X2 // Restore stack pointer.
445 MOVW X10, ret+16(FP)
446 RET
447
448 // func asminit()
449 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
450 RET
451
452 // reflectcall: call a function with the given argument list
453 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
454 // we don't have variable-sized frames, so we use a small number
455 // of constant-sized-frame functions to encode a few bits of size in the pc.
456 // Caution: ugly multiline assembly macros in your future!
457
458 #define DISPATCH(NAME,MAXSIZE) \
459 MOV $MAXSIZE, T1 \
460 BLTU T1, T0, 3(PC) \
461 MOV $NAME(SB), T2; \
462 JALR ZERO, T2
463 // Note: can't just "BR NAME(SB)" - bad inlining results.
464
465 // func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
466 TEXT reflect·call(SB), NOSPLIT, $0-0
467 JMP ·reflectcall(SB)
468
469 // func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
470 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
471 MOVWU frameSize+32(FP), T0
472 DISPATCH(runtime·call16, 16)
473 DISPATCH(runtime·call32, 32)
474 DISPATCH(runtime·call64, 64)
475 DISPATCH(runtime·call128, 128)
476 DISPATCH(runtime·call256, 256)
477 DISPATCH(runtime·call512, 512)
478 DISPATCH(runtime·call1024, 1024)
479 DISPATCH(runtime·call2048, 2048)
480 DISPATCH(runtime·call4096, 4096)
481 DISPATCH(runtime·call8192, 8192)
482 DISPATCH(runtime·call16384, 16384)
483 DISPATCH(runtime·call32768, 32768)
484 DISPATCH(runtime·call65536, 65536)
485 DISPATCH(runtime·call131072, 131072)
486 DISPATCH(runtime·call262144, 262144)
487 DISPATCH(runtime·call524288, 524288)
488 DISPATCH(runtime·call1048576, 1048576)
489 DISPATCH(runtime·call2097152, 2097152)
490 DISPATCH(runtime·call4194304, 4194304)
491 DISPATCH(runtime·call8388608, 8388608)
492 DISPATCH(runtime·call16777216, 16777216)
493 DISPATCH(runtime·call33554432, 33554432)
494 DISPATCH(runtime·call67108864, 67108864)
495 DISPATCH(runtime·call134217728, 134217728)
496 DISPATCH(runtime·call268435456, 268435456)
497 DISPATCH(runtime·call536870912, 536870912)
498 DISPATCH(runtime·call1073741824, 1073741824)
499 MOV $runtime·badreflectcall(SB), T2
500 JALR ZERO, T2
501
502 #define CALLFN(NAME,MAXSIZE) \
503 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
504 NO_LOCAL_POINTERS; \
505 /* copy arguments to stack */ \
506 MOV stackArgs+16(FP), A1; \
507 MOVWU stackArgsSize+24(FP), A2; \
508 MOV X2, A3; \
509 ADD $8, A3; \
510 ADD A3, A2; \
511 BEQ A3, A2, 6(PC); \
512 MOVBU (A1), A4; \
513 ADD $1, A1; \
514 MOVB A4, (A3); \
515 ADD $1, A3; \
516 JMP -5(PC); \
517 /* set up argument registers */ \
518 MOV regArgs+40(FP), X25; \
519 CALL ·unspillArgs(SB); \
520 /* call function */ \
521 MOV f+8(FP), CTXT; \
522 MOV (CTXT), X25; \
523 PCDATA $PCDATA_StackMapIndex, $0; \
524 JALR RA, X25; \
525 /* copy return values back */ \
526 MOV regArgs+40(FP), X25; \
527 CALL ·spillArgs(SB); \
528 MOV stackArgsType+0(FP), A5; \
529 MOV stackArgs+16(FP), A1; \
530 MOVWU stackArgsSize+24(FP), A2; \
531 MOVWU stackRetOffset+28(FP), A4; \
532 ADD $8, X2, A3; \
533 ADD A4, A3; \
534 ADD A4, A1; \
535 SUB A4, A2; \
536 CALL callRet<>(SB); \
537 RET
538
539 // callRet copies return values back at the end of call*. This is a
540 // separate function so it can allocate stack space for the arguments
541 // to reflectcallmove. It does not follow the Go ABI; it expects its
542 // arguments in registers.
543 TEXT callRet<>(SB), NOSPLIT, $40-0
544 NO_LOCAL_POINTERS
545 MOV A5, 8(X2)
546 MOV A1, 16(X2)
547 MOV A3, 24(X2)
548 MOV A2, 32(X2)
549 MOV X25, 40(X2)
550 CALL runtime·reflectcallmove(SB)
551 RET
552
553 CALLFN(·call16, 16)
554 CALLFN(·call32, 32)
555 CALLFN(·call64, 64)
556 CALLFN(·call128, 128)
557 CALLFN(·call256, 256)
558 CALLFN(·call512, 512)
559 CALLFN(·call1024, 1024)
560 CALLFN(·call2048, 2048)
561 CALLFN(·call4096, 4096)
562 CALLFN(·call8192, 8192)
563 CALLFN(·call16384, 16384)
564 CALLFN(·call32768, 32768)
565 CALLFN(·call65536, 65536)
566 CALLFN(·call131072, 131072)
567 CALLFN(·call262144, 262144)
568 CALLFN(·call524288, 524288)
569 CALLFN(·call1048576, 1048576)
570 CALLFN(·call2097152, 2097152)
571 CALLFN(·call4194304, 4194304)
572 CALLFN(·call8388608, 8388608)
573 CALLFN(·call16777216, 16777216)
574 CALLFN(·call33554432, 33554432)
575 CALLFN(·call67108864, 67108864)
576 CALLFN(·call134217728, 134217728)
577 CALLFN(·call268435456, 268435456)
578 CALLFN(·call536870912, 536870912)
579 CALLFN(·call1073741824, 1073741824)
580
581 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
582 // Must obey the gcc calling convention.
583 TEXT _cgo_topofstack(SB),NOSPLIT,$8
584 // g (X27) and REG_TMP (X31) might be clobbered by load_g.
585 // X27 is callee-save in the gcc calling convention, so save it.
586 MOV g, savedX27-8(SP)
587
588 CALL runtime·load_g(SB)
589 MOV g_m(g), X5
590 MOV m_curg(X5), X5
591 MOV (g_stack+stack_hi)(X5), X10 // return value in X10
592
593 MOV savedX27-8(SP), g
594 RET
595
596 // func goexit(neverCallThisFunction)
597 // The top-most function running on a goroutine, returns to goexit+PCQuantum*2.
598 // Note that the NOPs are written in a manner that will not be compressed,
599 // since the offset must be known by the runtime.
600 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
601 WORD $0x00000013 // NOP
602 JMP runtime·goexit1(SB) // does not return
603 // traceback from goexit1 must hit code range of goexit
604 WORD $0x00000013 // NOP
605
606 // This is called from .init_array and follows the platform, not the Go ABI.
607 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
608 // Use X31 as it is a scratch register in both the Go ABI and psABI.
609 MOV runtime·lastmoduledatap(SB), X31
610 MOV X10, moduledata_next(X31)
611 MOV X10, runtime·lastmoduledatap(SB)
612 RET
613
614 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
615 // See cgocall.go for more details.
616 TEXT ·cgocallback(SB),NOSPLIT,$24-24
617 NO_LOCAL_POINTERS
618
619 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
620 // It is used to dropm while thread is exiting.
621 MOV fn+0(FP), X7
622 BNE ZERO, X7, loadg
623 // Restore the g from frame.
624 MOV frame+8(FP), g
625 JMP dropm
626
627 loadg:
628 // Load m and g from thread-local storage.
629 MOVBU runtime·iscgo(SB), X5
630 BEQ ZERO, X5, nocgo
631 CALL runtime·load_g(SB)
632 nocgo:
633
634 // If g is nil, Go did not create the current thread,
635 // or if this thread never called into Go on pthread platforms.
636 // Call needm to obtain one for temporary use.
637 // In this case, we're running on the thread stack, so there's
638 // lots of space, but the linker doesn't know. Hide the call from
639 // the linker analysis by using an indirect call.
640 BEQ ZERO, g, needm
641
642 MOV g_m(g), X5
643 MOV X5, savedm-8(SP)
644 JMP havem
645
646 needm:
647 MOV g, savedm-8(SP) // g is zero, so is m.
648 MOV $runtime·needAndBindM(SB), X6
649 JALR RA, X6
650
651 // Set m->sched.sp = SP, so that if a panic happens
652 // during the function we are about to execute, it will
653 // have a valid SP to run on the g0 stack.
654 // The next few lines (after the havem label)
655 // will save this SP onto the stack and then write
656 // the same SP back to m->sched.sp. That seems redundant,
657 // but if an unrecovered panic happens, unwindm will
658 // restore the g->sched.sp from the stack location
659 // and then systemstack will try to use it. If we don't set it here,
660 // that restored SP will be uninitialized (typically 0) and
661 // will not be usable.
662 MOV g_m(g), X5
663 MOV m_g0(X5), X6
664 MOV X2, (g_sched+gobuf_sp)(X6)
665
666 havem:
667 // Now there's a valid m, and we're running on its m->g0.
668 // Save current m->g0->sched.sp on stack and then set it to SP.
669 // Save current sp in m->g0->sched.sp in preparation for
670 // switch back to m->curg stack.
671 // NOTE: unwindm knows that the saved g->sched.sp is at 8(X2) aka savedsp-24(SP).
672 MOV m_g0(X5), X6
673 MOV (g_sched+gobuf_sp)(X6), X7
674 MOV X7, savedsp-24(SP) // must match frame size
675 MOV X2, (g_sched+gobuf_sp)(X6)
676
677 // Switch to m->curg stack and call runtime.cgocallbackg.
678 // Because we are taking over the execution of m->curg
679 // but *not* resuming what had been running, we need to
680 // save that information (m->curg->sched) so we can restore it.
681 // We can restore m->curg->sched.sp easily, because calling
682 // runtime.cgocallbackg leaves SP unchanged upon return.
683 // To save m->curg->sched.pc, we push it onto the curg stack and
684 // open a frame the same size as cgocallback's g0 frame.
685 // Once we switch to the curg stack, the pushed PC will appear
686 // to be the return PC of cgocallback, so that the traceback
687 // will seamlessly trace back into the earlier calls.
688 MOV m_curg(X5), g
689 CALL runtime·save_g(SB)
690 MOV (g_sched+gobuf_sp)(g), X6 // prepare stack as X6
691 MOV (g_sched+gobuf_pc)(g), X7
692 MOV X7, -(24+8)(X6) // "saved LR"; must match frame size
693 // Gather our arguments into registers.
694 MOV fn+0(FP), X7
695 MOV frame+8(FP), X8
696 MOV ctxt+16(FP), X9
697 MOV $-(24+8)(X6), X2 // switch stack; must match frame size
698 MOV X7, 8(X2)
699 MOV X8, 16(X2)
700 MOV X9, 24(X2)
701 CALL runtime·cgocallbackg(SB)
702
703 // Restore g->sched (== m->curg->sched) from saved values.
704 MOV 0(X2), X7
705 MOV X7, (g_sched+gobuf_pc)(g)
706 MOV $(24+8)(X2), X6 // must match frame size
707 MOV X6, (g_sched+gobuf_sp)(g)
708
709 // Switch back to m->g0's stack and restore m->g0->sched.sp.
710 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
711 // so we do not have to restore it.)
712 MOV g_m(g), X5
713 MOV m_g0(X5), g
714 CALL runtime·save_g(SB)
715 MOV (g_sched+gobuf_sp)(g), X2
716 MOV savedsp-24(SP), X6 // must match frame size
717 MOV X6, (g_sched+gobuf_sp)(g)
718
719 // If the m on entry was nil, we called needm above to borrow an m,
720 // 1. for the duration of the call on non-pthread platforms,
721 // 2. or the duration of the C thread alive on pthread platforms.
722 // If the m on entry wasn't nil,
723 // 1. the thread might be a Go thread,
724 // 2. or it wasn't the first call from a C thread on pthread platforms,
725 // since then we skip dropm to reuse the m in the first call.
726 MOV savedm-8(SP), X5
727 BNE ZERO, X5, droppedm
728
729 // Skip dropm to reuse it in the next call, when a pthread key has been created.
730 MOV _cgo_pthread_key_created(SB), X5
731 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
732 BEQ ZERO, X5, dropm
733 MOV (X5), X5
734 BNE ZERO, X5, droppedm
735
736 dropm:
737 MOV $runtime·dropm(SB), X6
738 JALR RA, X6
739 droppedm:
740
741 // Done!
742 RET
743
744 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
745 EBREAK
746 RET
747
748 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
749 EBREAK
750 RET
751
752 // void setg(G*); set g. for use by needm.
753 TEXT runtime·setg(SB), NOSPLIT, $0-8
754 MOV gg+0(FP), g
755 // This only happens if iscgo, so jump straight to save_g
756 CALL runtime·save_g(SB)
757 RET
758
759 // spillArgs stores return values from registers to a *internal/abi.RegArgs in X25.
760 TEXT ·spillArgs(SB),NOSPLIT,$0-0
761 MOV X10, (0*8)(X25)
762 MOV X11, (1*8)(X25)
763 MOV X12, (2*8)(X25)
764 MOV X13, (3*8)(X25)
765 MOV X14, (4*8)(X25)
766 MOV X15, (5*8)(X25)
767 MOV X16, (6*8)(X25)
768 MOV X17, (7*8)(X25)
769 MOV X8, (8*8)(X25)
770 MOV X9, (9*8)(X25)
771 MOV X18, (10*8)(X25)
772 MOV X19, (11*8)(X25)
773 MOV X20, (12*8)(X25)
774 MOV X21, (13*8)(X25)
775 MOV X22, (14*8)(X25)
776 MOV X23, (15*8)(X25)
777 MOVD F10, (16*8)(X25)
778 MOVD F11, (17*8)(X25)
779 MOVD F12, (18*8)(X25)
780 MOVD F13, (19*8)(X25)
781 MOVD F14, (20*8)(X25)
782 MOVD F15, (21*8)(X25)
783 MOVD F16, (22*8)(X25)
784 MOVD F17, (23*8)(X25)
785 MOVD F8, (24*8)(X25)
786 MOVD F9, (25*8)(X25)
787 MOVD F18, (26*8)(X25)
788 MOVD F19, (27*8)(X25)
789 MOVD F20, (28*8)(X25)
790 MOVD F21, (29*8)(X25)
791 MOVD F22, (30*8)(X25)
792 MOVD F23, (31*8)(X25)
793 RET
794
795 // unspillArgs loads args into registers from a *internal/abi.RegArgs in X25.
796 TEXT ·unspillArgs(SB),NOSPLIT,$0-0
797 MOV (0*8)(X25), X10
798 MOV (1*8)(X25), X11
799 MOV (2*8)(X25), X12
800 MOV (3*8)(X25), X13
801 MOV (4*8)(X25), X14
802 MOV (5*8)(X25), X15
803 MOV (6*8)(X25), X16
804 MOV (7*8)(X25), X17
805 MOV (8*8)(X25), X8
806 MOV (9*8)(X25), X9
807 MOV (10*8)(X25), X18
808 MOV (11*8)(X25), X19
809 MOV (12*8)(X25), X20
810 MOV (13*8)(X25), X21
811 MOV (14*8)(X25), X22
812 MOV (15*8)(X25), X23
813 MOVD (16*8)(X25), F10
814 MOVD (17*8)(X25), F11
815 MOVD (18*8)(X25), F12
816 MOVD (19*8)(X25), F13
817 MOVD (20*8)(X25), F14
818 MOVD (21*8)(X25), F15
819 MOVD (22*8)(X25), F16
820 MOVD (23*8)(X25), F17
821 MOVD (24*8)(X25), F8
822 MOVD (25*8)(X25), F9
823 MOVD (26*8)(X25), F18
824 MOVD (27*8)(X25), F19
825 MOVD (28*8)(X25), F20
826 MOVD (29*8)(X25), F21
827 MOVD (30*8)(X25), F22
828 MOVD (31*8)(X25), F23
829 RET
830
831 // gcWriteBarrier informs the GC about heap pointer writes.
832 //
833 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
834 // number of bytes of buffer needed in X24, and returns a pointer
835 // to the buffer space in X24.
836 // It clobbers X31 aka T6 (the linker temp register - REG_TMP).
837 // The act of CALLing gcWriteBarrier will clobber RA (LR).
838 // It does not clobber any other general-purpose registers,
839 // but may clobber others (e.g., floating point registers).
840 TEXT gcWriteBarrier<>(SB),NOSPLIT,$208
841 // Save the registers clobbered by the fast path.
842 MOV A0, 24*8(X2)
843 MOV A1, 25*8(X2)
844 retry:
845 MOV g_m(g), A0
846 MOV m_p(A0), A0
847 MOV (p_wbBuf+wbBuf_next)(A0), A1
848 MOV (p_wbBuf+wbBuf_end)(A0), T6 // T6 is linker temp register (REG_TMP)
849 // Increment wbBuf.next position.
850 ADD X24, A1
851 // Is the buffer full?
852 BLTU T6, A1, flush
853 // Commit to the larger buffer.
854 MOV A1, (p_wbBuf+wbBuf_next)(A0)
855 // Make the return value (the original next position)
856 SUB X24, A1, X24
857 // Restore registers.
858 MOV 24*8(X2), A0
859 MOV 25*8(X2), A1
860 RET
861
862 flush:
863 // Save all general purpose registers since these could be
864 // clobbered by wbBufFlush and were not saved by the caller.
865 MOV T0, 1*8(X2)
866 MOV T1, 2*8(X2)
867 // X0 is zero register
868 // X1 is LR, saved by prologue
869 // X2 is SP
870 // X3 is GP
871 // X4 is TP
872 MOV X7, 3*8(X2)
873 MOV X8, 4*8(X2)
874 MOV X9, 5*8(X2)
875 // X10 already saved (A0)
876 // X11 already saved (A1)
877 MOV X12, 6*8(X2)
878 MOV X13, 7*8(X2)
879 MOV X14, 8*8(X2)
880 MOV X15, 9*8(X2)
881 MOV X16, 10*8(X2)
882 MOV X17, 11*8(X2)
883 MOV X18, 12*8(X2)
884 MOV X19, 13*8(X2)
885 MOV X20, 14*8(X2)
886 MOV X21, 15*8(X2)
887 MOV X22, 16*8(X2)
888 MOV X23, 17*8(X2)
889 MOV X24, 18*8(X2)
890 MOV X25, 19*8(X2)
891 MOV X26, 20*8(X2)
892 // X27 is g.
893 MOV X28, 21*8(X2)
894 MOV X29, 22*8(X2)
895 MOV X30, 23*8(X2)
896 // X31 is tmp register.
897
898 CALL runtime·wbBufFlush(SB)
899
900 MOV 1*8(X2), T0
901 MOV 2*8(X2), T1
902 MOV 3*8(X2), X7
903 MOV 4*8(X2), X8
904 MOV 5*8(X2), X9
905 MOV 6*8(X2), X12
906 MOV 7*8(X2), X13
907 MOV 8*8(X2), X14
908 MOV 9*8(X2), X15
909 MOV 10*8(X2), X16
910 MOV 11*8(X2), X17
911 MOV 12*8(X2), X18
912 MOV 13*8(X2), X19
913 MOV 14*8(X2), X20
914 MOV 15*8(X2), X21
915 MOV 16*8(X2), X22
916 MOV 17*8(X2), X23
917 MOV 18*8(X2), X24
918 MOV 19*8(X2), X25
919 MOV 20*8(X2), X26
920 MOV 21*8(X2), X28
921 MOV 22*8(X2), X29
922 MOV 23*8(X2), X30
923
924 JMP retry
925
926 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
927 MOV $8, X24
928 JMP gcWriteBarrier<>(SB)
929 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
930 MOV $16, X24
931 JMP gcWriteBarrier<>(SB)
932 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
933 MOV $24, X24
934 JMP gcWriteBarrier<>(SB)
935 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
936 MOV $32, X24
937 JMP gcWriteBarrier<>(SB)
938 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
939 MOV $40, X24
940 JMP gcWriteBarrier<>(SB)
941 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
942 MOV $48, X24
943 JMP gcWriteBarrier<>(SB)
944 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
945 MOV $56, X24
946 JMP gcWriteBarrier<>(SB)
947 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
948 MOV $64, X24
949 JMP gcWriteBarrier<>(SB)
950
951 TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-0
952 NO_LOCAL_POINTERS
953 // Save all 16 int registers that could have an index in them.
954 // They may be pointers, but if they are they are dead.
955 // Skip X0 aka ZERO, X1 aka LR, X2 aka SP, X3 aka GP, X4 aka TP.
956 MOV X5, 24(X2)
957 MOV X6, 32(X2)
958 MOV X7, 40(X2)
959 MOV X8, 48(X2)
960 MOV X9, 56(X2)
961 MOV X10, 64(X2)
962 MOV X11, 72(X2)
963 MOV X12, 80(X2)
964 MOV X13, 88(X2)
965 MOV X14, 96(X2)
966 MOV X15, 104(X2)
967 MOV X16, 112(X2)
968 MOV X17, 120(X2)
969 MOV X18, 128(X2)
970 MOV X19, 136(X2)
971 MOV X20, 144(X2)
972
973 MOV X1, X10 // PC immediately after call to panicBounds
974 ADD $24, X2, X11 // pointer to save area
975 CALL runtime·panicBounds64<ABIInternal>(SB)
976 RET
977
978 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
979 GLOBL runtime·mainPC(SB),RODATA,$8
980
View as plain text