Text file
src/runtime/asm_riscv64.s
1 // Copyright 2017 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "funcdata.h"
7 #include "textflag.h"
8 #include "cgo/abi_riscv64.h"
9
10
11 // When building with -buildmode=c-shared, this symbol is called when the shared
12 // library is loaded.
13 TEXT _rt0_riscv64_lib(SB),NOSPLIT,$224
14 // Preserve callee-save registers, along with X1 (LR).
15 MOV X1, (8*3)(X2)
16 SAVE_GPR((8*4))
17 SAVE_FPR((8*16))
18
19 // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go
20 MOV X0, g
21
22 MOV A0, _rt0_riscv64_lib_argc<>(SB)
23 MOV A1, _rt0_riscv64_lib_argv<>(SB)
24
25 MOV $runtime·libInit(SB), T1
26 JALR RA, T1
27
28 // Restore callee-save registers, along with X1 (LR).
29 MOV (8*3)(X2), X1
30 RESTORE_GPR((8*4))
31 RESTORE_FPR((8*16))
32
33 RET
34
35 TEXT runtime·rt0_lib_go<ABIInternal>(SB),NOSPLIT,$0
36 MOV _rt0_riscv64_lib_argc<>(SB), A0
37 MOV _rt0_riscv64_lib_argv<>(SB), A1
38 MOV $runtime·rt0_go(SB), T0
39 JALR ZERO, T0
40
41 DATA _rt0_riscv64_lib_argc<>(SB)/8, $0
42 GLOBL _rt0_riscv64_lib_argc<>(SB),NOPTR, $8
43 DATA _rt0_riscv64_lib_argv<>(SB)/8, $0
44 GLOBL _rt0_riscv64_lib_argv<>(SB),NOPTR, $8
45
46 // func rt0_go()
47 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
48 // X2 = stack; A0 = argc; A1 = argv
49 SUB $24, X2
50 MOV A0, 8(X2) // argc
51 MOV A1, 16(X2) // argv
52
53 // create istack out of the given (operating system) stack.
54 // _cgo_init may update stackguard.
55 MOV $runtime·g0(SB), g
56 MOV $(-64*1024), T0
57 ADD T0, X2, T1
58 MOV T1, g_stackguard0(g)
59 MOV T1, g_stackguard1(g)
60 MOV T1, (g_stack+stack_lo)(g)
61 MOV X2, (g_stack+stack_hi)(g)
62
63 // if there is a _cgo_init, call it using the gcc ABI.
64 MOV _cgo_init(SB), T2
65 BEQ T2, ZERO, nocgo
66
67 MOV ZERO, A3 // arg 3: not used
68 MOV ZERO, A2 // arg 2: not used
69 MOV $setg_gcc<>(SB), A1 // arg 1: setg
70 MOV g, A0 // arg 0: G
71 JALR RA, T2
72
73 nocgo:
74 // update stackguard after _cgo_init
75 MOV (g_stack+stack_lo)(g), T0
76 ADD $const_stackGuard, T0
77 MOV T0, g_stackguard0(g)
78 MOV T0, g_stackguard1(g)
79
80 // set the per-goroutine and per-mach "registers"
81 MOV $runtime·m0(SB), T0
82
83 // save m->g0 = g0
84 MOV g, m_g0(T0)
85 // save m0 to g0->m
86 MOV T0, g_m(g)
87
88 CALL runtime·check(SB)
89
90 // args are already prepared
91 CALL runtime·args(SB)
92 CALL runtime·osinit(SB)
93 CALL runtime·schedinit(SB)
94
95 // create a new goroutine to start program
96 MOV $runtime·mainPC(SB), T0 // entry
97 SUB $16, X2
98 MOV T0, 8(X2)
99 MOV ZERO, 0(X2)
100 CALL runtime·newproc(SB)
101 ADD $16, X2
102
103 // start this M
104 CALL runtime·mstart(SB)
105
106 WORD $0 // crash if reached
107 RET
108
109 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
110 CALL runtime·mstart0(SB)
111 RET // not reached
112
113 // void setg_gcc(G*); set g called from gcc with g in A0
114 TEXT setg_gcc<>(SB),NOSPLIT,$0-0
115 MOV A0, g
116 CALL runtime·save_g(SB)
117 RET
118
119 // func cputicks() int64
120 TEXT runtime·cputicks<ABIInternal>(SB),NOSPLIT,$0-0
121 // RDTIME to emulate cpu ticks
122 // RDCYCLE reads counter that is per HART(core) based
123 // according to the riscv manual, see issue 46737
124 RDTIME X10
125 RET
126
127 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
128 // of the G stack. We need to distinguish the routine that
129 // lives at the bottom of the G stack from the one that lives
130 // at the top of the system stack because the one at the top of
131 // the system stack terminates the stack walk (see topofstack()).
132 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
133 UNDEF
134 JALR RA, ZERO // make sure this function is not leaf
135 RET
136
137 // func systemstack(fn func())
138 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
139 MOV fn+0(FP), CTXT // CTXT = fn
140 MOV g_m(g), T0 // T0 = m
141
142 MOV m_gsignal(T0), T1 // T1 = gsignal
143 BEQ g, T1, noswitch
144
145 MOV m_g0(T0), T1 // T1 = g0
146 BEQ g, T1, noswitch
147
148 MOV m_curg(T0), T2
149 BEQ g, T2, switch
150
151 // Bad: g is not gsignal, not g0, not curg. What is it?
152 // Hide call from linker nosplit analysis.
153 MOV $runtime·badsystemstack(SB), T1
154 JALR RA, T1
155
156 switch:
157 // save our state in g->sched. Pretend to
158 // be systemstack_switch if the G stack is scanned.
159 CALL gosave_systemstack_switch<>(SB)
160
161 // switch to g0
162 MOV T1, g
163 CALL runtime·save_g(SB)
164 MOV (g_sched+gobuf_sp)(g), T0
165 MOV T0, X2
166
167 // call target function
168 MOV 0(CTXT), T1 // code pointer
169 JALR RA, T1
170
171 // switch back to g
172 MOV g_m(g), T0
173 MOV m_curg(T0), g
174 CALL runtime·save_g(SB)
175 MOV (g_sched+gobuf_sp)(g), X2
176 MOV ZERO, (g_sched+gobuf_sp)(g)
177 RET
178
179 noswitch:
180 // already on m stack, just call directly
181 // Using a tail call here cleans up tracebacks since we won't stop
182 // at an intermediate systemstack.
183 MOV 0(CTXT), T1 // code pointer
184 ADD $8, X2
185 JMP (T1)
186
187 // func switchToCrashStack0(fn func())
188 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
189 MOV X10, CTXT // context register
190 MOV g_m(g), X11 // curm
191
192 // set g to gcrash
193 MOV $runtime·gcrash(SB), g // g = &gcrash
194 CALL runtime·save_g(SB) // clobbers X31
195 MOV X11, g_m(g) // g.m = curm
196 MOV g, m_g0(X11) // curm.g0 = g
197
198 // switch to crashstack
199 MOV (g_stack+stack_hi)(g), X11
200 SUB $(4*8), X11
201 MOV X11, X2
202
203 // call target function
204 MOV 0(CTXT), X10
205 JALR X1, X10
206
207 // should never return
208 CALL runtime·abort(SB)
209 UNDEF
210
211 /*
212 * support for morestack
213 */
214
215 // Called during function prolog when more stack is needed.
216 // Called with return address (i.e. caller's PC) in X5 (aka T0),
217 // and the LR register contains the caller's LR.
218 //
219 // The traceback routines see morestack on a g0 as being
220 // the top of a stack (for example, morestack calling newstack
221 // calling the scheduler calling newm calling gc), so we must
222 // record an argument size. For that purpose, it has no arguments.
223
224 // func morestack()
225 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
226 // Called from f.
227 // Set g->sched to context in f.
228 MOV X2, (g_sched+gobuf_sp)(g)
229 MOV T0, (g_sched+gobuf_pc)(g)
230 MOV RA, (g_sched+gobuf_lr)(g)
231 MOV CTXT, (g_sched+gobuf_ctxt)(g)
232
233 // Cannot grow scheduler stack (m->g0).
234 MOV g_m(g), A0
235 MOV m_g0(A0), A1
236 BNE g, A1, 3(PC)
237 CALL runtime·badmorestackg0(SB)
238 CALL runtime·abort(SB)
239
240 // Cannot grow signal stack (m->gsignal).
241 MOV m_gsignal(A0), A1
242 BNE g, A1, 3(PC)
243 CALL runtime·badmorestackgsignal(SB)
244 CALL runtime·abort(SB)
245
246 // Called from f.
247 // Set m->morebuf to f's caller.
248 MOV RA, (m_morebuf+gobuf_pc)(A0) // f's caller's PC
249 MOV X2, (m_morebuf+gobuf_sp)(A0) // f's caller's SP
250 MOV g, (m_morebuf+gobuf_g)(A0)
251
252 // Call newstack on m->g0's stack.
253 MOV m_g0(A0), g
254 CALL runtime·save_g(SB)
255 MOV (g_sched+gobuf_sp)(g), X2
256 // Create a stack frame on g0 to call newstack.
257 MOV ZERO, -8(X2) // Zero saved LR in frame
258 SUB $8, X2
259 CALL runtime·newstack(SB)
260
261 // Not reached, but make sure the return PC from the call to newstack
262 // is still in this function, and not the beginning of the next.
263 UNDEF
264
265 // func morestack_noctxt()
266 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
267 // Force SPWRITE. This function doesn't actually write SP,
268 // but it is called with a special calling convention where
269 // the caller doesn't save LR on stack but passes it as a
270 // register, and the unwinder currently doesn't understand.
271 // Make it SPWRITE to stop unwinding. (See issue 54332)
272 MOV X2, X2
273
274 MOV ZERO, CTXT
275 JMP runtime·morestack(SB)
276
277 // restore state from Gobuf; longjmp
278
279 // func gogo(buf *gobuf)
280 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
281 MOV buf+0(FP), T0
282 MOV gobuf_g(T0), T1
283 MOV 0(T1), ZERO // make sure g != nil
284 JMP gogo<>(SB)
285
286 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
287 MOV T1, g
288 CALL runtime·save_g(SB)
289
290 MOV gobuf_sp(T0), X2
291 MOV gobuf_lr(T0), RA
292 MOV gobuf_ctxt(T0), CTXT
293 MOV ZERO, gobuf_sp(T0)
294 MOV ZERO, gobuf_lr(T0)
295 MOV ZERO, gobuf_ctxt(T0)
296 MOV gobuf_pc(T0), T0
297 JALR ZERO, T0
298
299 // func procyieldAsm(cycles uint32)
300 TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0
301 RET
302
303 // Switch to m->g0's stack, call fn(g).
304 // Fn must never return. It should gogo(&g->sched)
305 // to keep running g.
306
307 // func mcall(fn func(*g))
308 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
309 MOV X10, CTXT
310
311 // Save caller state in g->sched
312 MOV X2, (g_sched+gobuf_sp)(g)
313 MOV RA, (g_sched+gobuf_pc)(g)
314 MOV ZERO, (g_sched+gobuf_lr)(g)
315
316 // Switch to m->g0 & its stack, call fn.
317 MOV g, X10
318 MOV g_m(g), T1
319 MOV m_g0(T1), g
320 CALL runtime·save_g(SB)
321 BNE g, X10, 2(PC)
322 JMP runtime·badmcall(SB)
323 MOV 0(CTXT), T1 // code pointer
324 MOV (g_sched+gobuf_sp)(g), X2 // sp = m->g0->sched.sp
325 // we don't need special macro for regabi since arg0(X10) = g
326 SUB $16, X2
327 MOV X10, 8(X2) // setup g
328 MOV ZERO, 0(X2) // clear return address
329 JALR RA, T1
330 JMP runtime·badmcall2(SB)
331
332 // Save state of caller into g->sched,
333 // but using fake PC from systemstack_switch.
334 // Must only be called from functions with no locals ($0)
335 // or else unwinding from systemstack_switch is incorrect.
336 // Smashes X31.
337 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
338 MOV $runtime·systemstack_switch(SB), X31
339 ADD $8, X31 // get past prologue
340 MOV X31, (g_sched+gobuf_pc)(g)
341 MOV X2, (g_sched+gobuf_sp)(g)
342 MOV ZERO, (g_sched+gobuf_lr)(g)
343 // Assert ctxt is zero. See func save.
344 MOV (g_sched+gobuf_ctxt)(g), X31
345 BEQ ZERO, X31, 2(PC)
346 CALL runtime·abort(SB)
347 RET
348
349 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
350 // Call fn(arg) aligned appropriately for the gcc ABI.
351 // Called on a system stack, and there may be no g yet (during needm).
352 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
353 MOV fn+0(FP), X11
354 MOV arg+8(FP), X10
355 JALR RA, (X11)
356 RET
357
358 // func asmcgocall(fn, arg unsafe.Pointer) int32
359 // Call fn(arg) on the scheduler stack,
360 // aligned appropriately for the gcc ABI.
361 // See cgocall.go for more details.
362 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
363 MOV fn+0(FP), X11
364 MOV arg+8(FP), X10
365
366 MOV X2, X8 // save original stack pointer
367 MOV g, X9
368
369 // Figure out if we need to switch to m->g0 stack.
370 // We get called to create new OS threads too, and those
371 // come in on the m->g0 stack already. Or we might already
372 // be on the m->gsignal stack.
373 BEQZ g, nosave
374 MOV g_m(g), X6
375 MOV m_gsignal(X6), X7
376 BEQ X7, g, g0
377 MOV m_g0(X6), X7
378 BEQ X7, g, g0
379
380 CALL gosave_systemstack_switch<>(SB)
381 MOV X7, g
382 CALL runtime·save_g(SB)
383 MOV (g_sched+gobuf_sp)(g), X2
384
385 // Now on a scheduling stack (a pthread-created stack).
386 g0:
387 // Save room for two of our pointers.
388 SUB $16, X2
389 MOV X9, 0(X2) // save old g on stack
390 MOV (g_stack+stack_hi)(X9), X9
391 SUB X8, X9, X8
392 MOV X8, 8(X2) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
393
394 JALR RA, (X11)
395
396 // Restore g, stack pointer. X10 is return value.
397 MOV 0(X2), g
398 CALL runtime·save_g(SB)
399 MOV (g_stack+stack_hi)(g), X5
400 MOV 8(X2), X6
401 SUB X6, X5, X6
402 MOV X6, X2
403
404 MOVW X10, ret+16(FP)
405 RET
406
407 nosave:
408 // Running on a system stack, perhaps even without a g.
409 // Having no g can happen during thread creation or thread teardown.
410 MOV fn+0(FP), X11
411 MOV arg+8(FP), X10
412 MOV X2, X8
413 SUB $16, X2
414 MOV ZERO, 0(X2) // Where above code stores g, in case someone looks during debugging.
415 MOV X8, 8(X2) // Save original stack pointer.
416 JALR RA, (X11)
417 MOV 8(X2), X2 // Restore stack pointer.
418 MOVW X10, ret+16(FP)
419 RET
420
421 // func asminit()
422 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
423 RET
424
425 // reflectcall: call a function with the given argument list
426 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
427 // we don't have variable-sized frames, so we use a small number
428 // of constant-sized-frame functions to encode a few bits of size in the pc.
429 // Caution: ugly multiline assembly macros in your future!
430
431 #define DISPATCH(NAME,MAXSIZE) \
432 MOV $MAXSIZE, T1 \
433 BLTU T1, T0, 3(PC) \
434 MOV $NAME(SB), T2; \
435 JALR ZERO, T2
436 // Note: can't just "BR NAME(SB)" - bad inlining results.
437
438 // func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
439 TEXT reflect·call(SB), NOSPLIT, $0-0
440 JMP ·reflectcall(SB)
441
442 // func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
443 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
444 MOVWU frameSize+32(FP), T0
445 DISPATCH(runtime·call16, 16)
446 DISPATCH(runtime·call32, 32)
447 DISPATCH(runtime·call64, 64)
448 DISPATCH(runtime·call128, 128)
449 DISPATCH(runtime·call256, 256)
450 DISPATCH(runtime·call512, 512)
451 DISPATCH(runtime·call1024, 1024)
452 DISPATCH(runtime·call2048, 2048)
453 DISPATCH(runtime·call4096, 4096)
454 DISPATCH(runtime·call8192, 8192)
455 DISPATCH(runtime·call16384, 16384)
456 DISPATCH(runtime·call32768, 32768)
457 DISPATCH(runtime·call65536, 65536)
458 DISPATCH(runtime·call131072, 131072)
459 DISPATCH(runtime·call262144, 262144)
460 DISPATCH(runtime·call524288, 524288)
461 DISPATCH(runtime·call1048576, 1048576)
462 DISPATCH(runtime·call2097152, 2097152)
463 DISPATCH(runtime·call4194304, 4194304)
464 DISPATCH(runtime·call8388608, 8388608)
465 DISPATCH(runtime·call16777216, 16777216)
466 DISPATCH(runtime·call33554432, 33554432)
467 DISPATCH(runtime·call67108864, 67108864)
468 DISPATCH(runtime·call134217728, 134217728)
469 DISPATCH(runtime·call268435456, 268435456)
470 DISPATCH(runtime·call536870912, 536870912)
471 DISPATCH(runtime·call1073741824, 1073741824)
472 MOV $runtime·badreflectcall(SB), T2
473 JALR ZERO, T2
474
475 #define CALLFN(NAME,MAXSIZE) \
476 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
477 NO_LOCAL_POINTERS; \
478 /* copy arguments to stack */ \
479 MOV stackArgs+16(FP), A1; \
480 MOVWU stackArgsSize+24(FP), A2; \
481 MOV X2, A3; \
482 ADD $8, A3; \
483 ADD A3, A2; \
484 BEQ A3, A2, 6(PC); \
485 MOVBU (A1), A4; \
486 ADD $1, A1; \
487 MOVB A4, (A3); \
488 ADD $1, A3; \
489 JMP -5(PC); \
490 /* set up argument registers */ \
491 MOV regArgs+40(FP), X25; \
492 CALL ·unspillArgs(SB); \
493 /* call function */ \
494 MOV f+8(FP), CTXT; \
495 MOV (CTXT), X25; \
496 PCDATA $PCDATA_StackMapIndex, $0; \
497 JALR RA, X25; \
498 /* copy return values back */ \
499 MOV regArgs+40(FP), X25; \
500 CALL ·spillArgs(SB); \
501 MOV stackArgsType+0(FP), A5; \
502 MOV stackArgs+16(FP), A1; \
503 MOVWU stackArgsSize+24(FP), A2; \
504 MOVWU stackRetOffset+28(FP), A4; \
505 ADD $8, X2, A3; \
506 ADD A4, A3; \
507 ADD A4, A1; \
508 SUB A4, A2; \
509 CALL callRet<>(SB); \
510 RET
511
512 // callRet copies return values back at the end of call*. This is a
513 // separate function so it can allocate stack space for the arguments
514 // to reflectcallmove. It does not follow the Go ABI; it expects its
515 // arguments in registers.
516 TEXT callRet<>(SB), NOSPLIT, $40-0
517 NO_LOCAL_POINTERS
518 MOV A5, 8(X2)
519 MOV A1, 16(X2)
520 MOV A3, 24(X2)
521 MOV A2, 32(X2)
522 MOV X25, 40(X2)
523 CALL runtime·reflectcallmove(SB)
524 RET
525
526 CALLFN(·call16, 16)
527 CALLFN(·call32, 32)
528 CALLFN(·call64, 64)
529 CALLFN(·call128, 128)
530 CALLFN(·call256, 256)
531 CALLFN(·call512, 512)
532 CALLFN(·call1024, 1024)
533 CALLFN(·call2048, 2048)
534 CALLFN(·call4096, 4096)
535 CALLFN(·call8192, 8192)
536 CALLFN(·call16384, 16384)
537 CALLFN(·call32768, 32768)
538 CALLFN(·call65536, 65536)
539 CALLFN(·call131072, 131072)
540 CALLFN(·call262144, 262144)
541 CALLFN(·call524288, 524288)
542 CALLFN(·call1048576, 1048576)
543 CALLFN(·call2097152, 2097152)
544 CALLFN(·call4194304, 4194304)
545 CALLFN(·call8388608, 8388608)
546 CALLFN(·call16777216, 16777216)
547 CALLFN(·call33554432, 33554432)
548 CALLFN(·call67108864, 67108864)
549 CALLFN(·call134217728, 134217728)
550 CALLFN(·call268435456, 268435456)
551 CALLFN(·call536870912, 536870912)
552 CALLFN(·call1073741824, 1073741824)
553
554 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
555 // Must obey the gcc calling convention.
556 TEXT _cgo_topofstack(SB),NOSPLIT,$8
557 // g (X27) and REG_TMP (X31) might be clobbered by load_g.
558 // X27 is callee-save in the gcc calling convention, so save it.
559 MOV g, savedX27-8(SP)
560
561 CALL runtime·load_g(SB)
562 MOV g_m(g), X5
563 MOV m_curg(X5), X5
564 MOV (g_stack+stack_hi)(X5), X10 // return value in X10
565
566 MOV savedX27-8(SP), g
567 RET
568
569 // func goexit(neverCallThisFunction)
570 // The top-most function running on a goroutine, returns to goexit+PCQuantum*2.
571 // Note that the NOPs are written in a manner that will not be compressed,
572 // since the offset must be known by the runtime.
573 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
574 WORD $0x00000013 // NOP
575 JMP runtime·goexit1(SB) // does not return
576 // traceback from goexit1 must hit code range of goexit
577 WORD $0x00000013 // NOP
578
579 // This is called from .init_array and follows the platform, not the Go ABI.
580 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
581 // Use X31 as it is a scratch register in both the Go ABI and psABI.
582 MOV runtime·lastmoduledatap(SB), X31
583 MOV X10, moduledata_next(X31)
584 MOV X10, runtime·lastmoduledatap(SB)
585 RET
586
587 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
588 // See cgocall.go for more details.
589 TEXT ·cgocallback(SB),NOSPLIT,$24-24
590 NO_LOCAL_POINTERS
591
592 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
593 // It is used to dropm while thread is exiting.
594 MOV fn+0(FP), X7
595 BNE ZERO, X7, loadg
596 // Restore the g from frame.
597 MOV frame+8(FP), g
598 JMP dropm
599
600 loadg:
601 // Load m and g from thread-local storage.
602 MOVBU runtime·iscgo(SB), X5
603 BEQ ZERO, X5, nocgo
604 CALL runtime·load_g(SB)
605 nocgo:
606
607 // If g is nil, Go did not create the current thread,
608 // or if this thread never called into Go on pthread platforms.
609 // Call needm to obtain one for temporary use.
610 // In this case, we're running on the thread stack, so there's
611 // lots of space, but the linker doesn't know. Hide the call from
612 // the linker analysis by using an indirect call.
613 BEQ ZERO, g, needm
614
615 MOV g_m(g), X5
616 MOV X5, savedm-8(SP)
617 JMP havem
618
619 needm:
620 MOV g, savedm-8(SP) // g is zero, so is m.
621 MOV $runtime·needAndBindM(SB), X6
622 JALR RA, X6
623
624 // Set m->sched.sp = SP, so that if a panic happens
625 // during the function we are about to execute, it will
626 // have a valid SP to run on the g0 stack.
627 // The next few lines (after the havem label)
628 // will save this SP onto the stack and then write
629 // the same SP back to m->sched.sp. That seems redundant,
630 // but if an unrecovered panic happens, unwindm will
631 // restore the g->sched.sp from the stack location
632 // and then systemstack will try to use it. If we don't set it here,
633 // that restored SP will be uninitialized (typically 0) and
634 // will not be usable.
635 MOV g_m(g), X5
636 MOV m_g0(X5), X6
637 MOV X2, (g_sched+gobuf_sp)(X6)
638
639 havem:
640 // Now there's a valid m, and we're running on its m->g0.
641 // Save current m->g0->sched.sp on stack and then set it to SP.
642 // Save current sp in m->g0->sched.sp in preparation for
643 // switch back to m->curg stack.
644 // NOTE: unwindm knows that the saved g->sched.sp is at 8(X2) aka savedsp-24(SP).
645 MOV m_g0(X5), X6
646 MOV (g_sched+gobuf_sp)(X6), X7
647 MOV X7, savedsp-24(SP) // must match frame size
648 MOV X2, (g_sched+gobuf_sp)(X6)
649
650 // Switch to m->curg stack and call runtime.cgocallbackg.
651 // Because we are taking over the execution of m->curg
652 // but *not* resuming what had been running, we need to
653 // save that information (m->curg->sched) so we can restore it.
654 // We can restore m->curg->sched.sp easily, because calling
655 // runtime.cgocallbackg leaves SP unchanged upon return.
656 // To save m->curg->sched.pc, we push it onto the curg stack and
657 // open a frame the same size as cgocallback's g0 frame.
658 // Once we switch to the curg stack, the pushed PC will appear
659 // to be the return PC of cgocallback, so that the traceback
660 // will seamlessly trace back into the earlier calls.
661 MOV m_curg(X5), g
662 CALL runtime·save_g(SB)
663 MOV (g_sched+gobuf_sp)(g), X6 // prepare stack as X6
664 MOV (g_sched+gobuf_pc)(g), X7
665 MOV X7, -(24+8)(X6) // "saved LR"; must match frame size
666 // Gather our arguments into registers.
667 MOV fn+0(FP), X7
668 MOV frame+8(FP), X8
669 MOV ctxt+16(FP), X9
670 MOV $-(24+8)(X6), X2 // switch stack; must match frame size
671 MOV X7, 8(X2)
672 MOV X8, 16(X2)
673 MOV X9, 24(X2)
674 CALL runtime·cgocallbackg(SB)
675
676 // Restore g->sched (== m->curg->sched) from saved values.
677 MOV 0(X2), X7
678 MOV X7, (g_sched+gobuf_pc)(g)
679 MOV $(24+8)(X2), X6 // must match frame size
680 MOV X6, (g_sched+gobuf_sp)(g)
681
682 // Switch back to m->g0's stack and restore m->g0->sched.sp.
683 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
684 // so we do not have to restore it.)
685 MOV g_m(g), X5
686 MOV m_g0(X5), g
687 CALL runtime·save_g(SB)
688 MOV (g_sched+gobuf_sp)(g), X2
689 MOV savedsp-24(SP), X6 // must match frame size
690 MOV X6, (g_sched+gobuf_sp)(g)
691
692 // If the m on entry was nil, we called needm above to borrow an m,
693 // 1. for the duration of the call on non-pthread platforms,
694 // 2. or the duration of the C thread alive on pthread platforms.
695 // If the m on entry wasn't nil,
696 // 1. the thread might be a Go thread,
697 // 2. or it wasn't the first call from a C thread on pthread platforms,
698 // since then we skip dropm to reuse the m in the first call.
699 MOV savedm-8(SP), X5
700 BNE ZERO, X5, droppedm
701
702 // Skip dropm to reuse it in the next call, when a pthread key has been created.
703 MOV _cgo_pthread_key_created(SB), X5
704 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
705 BEQ ZERO, X5, dropm
706 MOV (X5), X5
707 BNE ZERO, X5, droppedm
708
709 dropm:
710 MOV $runtime·dropm(SB), X6
711 JALR RA, X6
712 droppedm:
713
714 // Done!
715 RET
716
717 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
718 EBREAK
719 RET
720
721 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
722 EBREAK
723 RET
724
725 // void setg(G*); set g. for use by needm.
726 TEXT runtime·setg(SB), NOSPLIT, $0-8
727 MOV gg+0(FP), g
728 // This only happens if iscgo, so jump straight to save_g
729 CALL runtime·save_g(SB)
730 RET
731
732 // spillArgs stores return values from registers to a *internal/abi.RegArgs in X25.
733 TEXT ·spillArgs(SB),NOSPLIT,$0-0
734 MOV X10, (0*8)(X25)
735 MOV X11, (1*8)(X25)
736 MOV X12, (2*8)(X25)
737 MOV X13, (3*8)(X25)
738 MOV X14, (4*8)(X25)
739 MOV X15, (5*8)(X25)
740 MOV X16, (6*8)(X25)
741 MOV X17, (7*8)(X25)
742 MOV X8, (8*8)(X25)
743 MOV X9, (9*8)(X25)
744 MOV X18, (10*8)(X25)
745 MOV X19, (11*8)(X25)
746 MOV X20, (12*8)(X25)
747 MOV X21, (13*8)(X25)
748 MOV X22, (14*8)(X25)
749 MOV X23, (15*8)(X25)
750 MOVD F10, (16*8)(X25)
751 MOVD F11, (17*8)(X25)
752 MOVD F12, (18*8)(X25)
753 MOVD F13, (19*8)(X25)
754 MOVD F14, (20*8)(X25)
755 MOVD F15, (21*8)(X25)
756 MOVD F16, (22*8)(X25)
757 MOVD F17, (23*8)(X25)
758 MOVD F8, (24*8)(X25)
759 MOVD F9, (25*8)(X25)
760 MOVD F18, (26*8)(X25)
761 MOVD F19, (27*8)(X25)
762 MOVD F20, (28*8)(X25)
763 MOVD F21, (29*8)(X25)
764 MOVD F22, (30*8)(X25)
765 MOVD F23, (31*8)(X25)
766 RET
767
768 // unspillArgs loads args into registers from a *internal/abi.RegArgs in X25.
769 TEXT ·unspillArgs(SB),NOSPLIT,$0-0
770 MOV (0*8)(X25), X10
771 MOV (1*8)(X25), X11
772 MOV (2*8)(X25), X12
773 MOV (3*8)(X25), X13
774 MOV (4*8)(X25), X14
775 MOV (5*8)(X25), X15
776 MOV (6*8)(X25), X16
777 MOV (7*8)(X25), X17
778 MOV (8*8)(X25), X8
779 MOV (9*8)(X25), X9
780 MOV (10*8)(X25), X18
781 MOV (11*8)(X25), X19
782 MOV (12*8)(X25), X20
783 MOV (13*8)(X25), X21
784 MOV (14*8)(X25), X22
785 MOV (15*8)(X25), X23
786 MOVD (16*8)(X25), F10
787 MOVD (17*8)(X25), F11
788 MOVD (18*8)(X25), F12
789 MOVD (19*8)(X25), F13
790 MOVD (20*8)(X25), F14
791 MOVD (21*8)(X25), F15
792 MOVD (22*8)(X25), F16
793 MOVD (23*8)(X25), F17
794 MOVD (24*8)(X25), F8
795 MOVD (25*8)(X25), F9
796 MOVD (26*8)(X25), F18
797 MOVD (27*8)(X25), F19
798 MOVD (28*8)(X25), F20
799 MOVD (29*8)(X25), F21
800 MOVD (30*8)(X25), F22
801 MOVD (31*8)(X25), F23
802 RET
803
804 // gcWriteBarrier informs the GC about heap pointer writes.
805 //
806 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
807 // number of bytes of buffer needed in X24, and returns a pointer
808 // to the buffer space in X24.
809 // It clobbers X31 aka T6 (the linker temp register - REG_TMP).
810 // The act of CALLing gcWriteBarrier will clobber RA (LR).
811 // It does not clobber any other general-purpose registers,
812 // but may clobber others (e.g., floating point registers).
813 TEXT gcWriteBarrier<>(SB),NOSPLIT,$208
814 // Save the registers clobbered by the fast path.
815 MOV A0, 24*8(X2)
816 MOV A1, 25*8(X2)
817 retry:
818 MOV g_m(g), A0
819 MOV m_p(A0), A0
820 MOV (p_wbBuf+wbBuf_next)(A0), A1
821 MOV (p_wbBuf+wbBuf_end)(A0), T6 // T6 is linker temp register (REG_TMP)
822 // Increment wbBuf.next position.
823 ADD X24, A1
824 // Is the buffer full?
825 BLTU T6, A1, flush
826 // Commit to the larger buffer.
827 MOV A1, (p_wbBuf+wbBuf_next)(A0)
828 // Make the return value (the original next position)
829 SUB X24, A1, X24
830 // Restore registers.
831 MOV 24*8(X2), A0
832 MOV 25*8(X2), A1
833 RET
834
835 flush:
836 // Save all general purpose registers since these could be
837 // clobbered by wbBufFlush and were not saved by the caller.
838 MOV T0, 1*8(X2)
839 MOV T1, 2*8(X2)
840 // X0 is zero register
841 // X1 is LR, saved by prologue
842 // X2 is SP
843 // X3 is GP
844 // X4 is TP
845 MOV X7, 3*8(X2)
846 MOV X8, 4*8(X2)
847 MOV X9, 5*8(X2)
848 // X10 already saved (A0)
849 // X11 already saved (A1)
850 MOV X12, 6*8(X2)
851 MOV X13, 7*8(X2)
852 MOV X14, 8*8(X2)
853 MOV X15, 9*8(X2)
854 MOV X16, 10*8(X2)
855 MOV X17, 11*8(X2)
856 MOV X18, 12*8(X2)
857 MOV X19, 13*8(X2)
858 MOV X20, 14*8(X2)
859 MOV X21, 15*8(X2)
860 MOV X22, 16*8(X2)
861 MOV X23, 17*8(X2)
862 MOV X24, 18*8(X2)
863 MOV X25, 19*8(X2)
864 MOV X26, 20*8(X2)
865 // X27 is g.
866 MOV X28, 21*8(X2)
867 MOV X29, 22*8(X2)
868 MOV X30, 23*8(X2)
869 // X31 is tmp register.
870
871 CALL runtime·wbBufFlush(SB)
872
873 MOV 1*8(X2), T0
874 MOV 2*8(X2), T1
875 MOV 3*8(X2), X7
876 MOV 4*8(X2), X8
877 MOV 5*8(X2), X9
878 MOV 6*8(X2), X12
879 MOV 7*8(X2), X13
880 MOV 8*8(X2), X14
881 MOV 9*8(X2), X15
882 MOV 10*8(X2), X16
883 MOV 11*8(X2), X17
884 MOV 12*8(X2), X18
885 MOV 13*8(X2), X19
886 MOV 14*8(X2), X20
887 MOV 15*8(X2), X21
888 MOV 16*8(X2), X22
889 MOV 17*8(X2), X23
890 MOV 18*8(X2), X24
891 MOV 19*8(X2), X25
892 MOV 20*8(X2), X26
893 MOV 21*8(X2), X28
894 MOV 22*8(X2), X29
895 MOV 23*8(X2), X30
896
897 JMP retry
898
899 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
900 MOV $8, X24
901 JMP gcWriteBarrier<>(SB)
902 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
903 MOV $16, X24
904 JMP gcWriteBarrier<>(SB)
905 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
906 MOV $24, X24
907 JMP gcWriteBarrier<>(SB)
908 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
909 MOV $32, X24
910 JMP gcWriteBarrier<>(SB)
911 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
912 MOV $40, X24
913 JMP gcWriteBarrier<>(SB)
914 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
915 MOV $48, X24
916 JMP gcWriteBarrier<>(SB)
917 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
918 MOV $56, X24
919 JMP gcWriteBarrier<>(SB)
920 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
921 MOV $64, X24
922 JMP gcWriteBarrier<>(SB)
923
924 TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-0
925 NO_LOCAL_POINTERS
926 // Save all 16 int registers that could have an index in them.
927 // They may be pointers, but if they are they are dead.
928 // Skip X0 aka ZERO, X1 aka LR, X2 aka SP, X3 aka GP, X4 aka TP.
929 MOV X5, 24(X2)
930 MOV X6, 32(X2)
931 MOV X7, 40(X2)
932 MOV X8, 48(X2)
933 MOV X9, 56(X2)
934 MOV X10, 64(X2)
935 MOV X11, 72(X2)
936 MOV X12, 80(X2)
937 MOV X13, 88(X2)
938 MOV X14, 96(X2)
939 MOV X15, 104(X2)
940 MOV X16, 112(X2)
941 MOV X17, 120(X2)
942 MOV X18, 128(X2)
943 MOV X19, 136(X2)
944 MOV X20, 144(X2)
945
946 MOV X1, X10 // PC immediately after call to panicBounds
947 ADD $24, X2, X11 // pointer to save area
948 CALL runtime·panicBounds64<ABIInternal>(SB)
949 RET
950
951 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
952 GLOBL runtime·mainPC(SB),RODATA,$8
953
View as plain text