Text file
src/runtime/asm_riscv64.s
1 // Copyright 2017 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "funcdata.h"
7 #include "textflag.h"
8 #include "cgo/abi_riscv64.h"
9
10
11 // When building with -buildmode=c-shared, this symbol is called when the shared
12 // library is loaded.
13 TEXT _rt0_riscv64_lib(SB),NOSPLIT,$224
14 // Preserve callee-save registers, along with X1 (LR).
15 MOV X1, (8*3)(X2)
16 SAVE_GPR((8*4))
17 SAVE_FPR((8*16))
18
19 // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go
20 MOV X0, g
21
22 MOV A0, _rt0_riscv64_lib_argc<>(SB)
23 MOV A1, _rt0_riscv64_lib_argv<>(SB)
24
25 MOV $runtime·libInit(SB), T1
26 JALR RA, T1
27
28 // Restore callee-save registers, along with X1 (LR).
29 MOV (8*3)(X2), X1
30 RESTORE_GPR((8*4))
31 RESTORE_FPR((8*16))
32
33 RET
34
35 TEXT runtime·rt0_lib_go<ABIInternal>(SB),NOSPLIT,$0
36 MOV _rt0_riscv64_lib_argc<>(SB), A0
37 MOV _rt0_riscv64_lib_argv<>(SB), A1
38 MOV $runtime·rt0_go(SB), T0
39 JALR ZERO, T0
40
41 DATA _rt0_riscv64_lib_argc<>(SB)/8, $0
42 GLOBL _rt0_riscv64_lib_argc<>(SB),NOPTR, $8
43 DATA _rt0_riscv64_lib_argv<>(SB)/8, $0
44 GLOBL _rt0_riscv64_lib_argv<>(SB),NOPTR, $8
45
46 // func rt0_go()
47 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
48 // X2 = stack; A0 = argc; A1 = argv
49 SUB $24, X2
50 MOV A0, 8(X2) // argc
51 MOV A1, 16(X2) // argv
52
53 // create istack out of the given (operating system) stack.
54 // _cgo_init may update stackguard.
55 MOV $runtime·g0(SB), g
56 MOV $(-64*1024), T0
57 ADD T0, X2, T1
58 MOV T1, g_stackguard0(g)
59 MOV T1, g_stackguard1(g)
60 MOV T1, (g_stack+stack_lo)(g)
61 MOV X2, (g_stack+stack_hi)(g)
62
63 // if there is a _cgo_init, call it using the gcc ABI.
64 MOV _cgo_init(SB), T2
65 BEQ T2, ZERO, nocgo
66
67 MOV ZERO, A3 // arg 3: not used
68 MOV ZERO, A2 // arg 2: not used
69 MOV $setg_gcc<>(SB), A1 // arg 1: setg
70 MOV g, A0 // arg 0: G
71 JALR RA, T2
72
73 nocgo:
74 // update stackguard after _cgo_init
75 MOV (g_stack+stack_lo)(g), T0
76 ADD $const_stackGuard, T0
77 MOV T0, g_stackguard0(g)
78 MOV T0, g_stackguard1(g)
79
80 // set the per-goroutine and per-mach "registers"
81 MOV $runtime·m0(SB), T0
82
83 // save m->g0 = g0
84 MOV g, m_g0(T0)
85 // save m0 to g0->m
86 MOV T0, g_m(g)
87
88 CALL runtime·check(SB)
89
90 // args are already prepared
91 CALL runtime·args(SB)
92 CALL runtime·osinit(SB)
93 CALL runtime·schedinit(SB)
94
95 // create a new goroutine to start program
96 MOV $runtime·mainPC(SB), T0 // entry
97 SUB $16, X2
98 MOV T0, 8(X2)
99 MOV ZERO, 0(X2)
100 CALL runtime·newproc(SB)
101 ADD $16, X2
102
103 // start this M
104 CALL runtime·stackcheck(SB) // fault if stack check is wrong
105 CALL runtime·mstart(SB)
106
107 WORD $0 // crash if reached
108 RET
109
110 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
111 CALL runtime·mstart0(SB)
112 RET // not reached
113
114 // void setg_gcc(G*); set g called from gcc with g in A0
115 TEXT setg_gcc<>(SB),NOSPLIT,$0-0
116 MOV A0, g
117 CALL runtime·save_g(SB)
118 RET
119
120 // func cputicks() int64
121 TEXT runtime·cputicks<ABIInternal>(SB),NOSPLIT,$0-0
122 // RDTIME to emulate cpu ticks
123 // RDCYCLE reads counter that is per HART(core) based
124 // according to the riscv manual, see issue 46737
125 RDTIME X10
126 RET
127
128 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
129 // of the G stack. We need to distinguish the routine that
130 // lives at the bottom of the G stack from the one that lives
131 // at the top of the system stack because the one at the top of
132 // the system stack terminates the stack walk (see topofstack()).
133 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
134 UNDEF
135 JALR RA, ZERO // make sure this function is not leaf
136 RET
137
138 // func systemstack(fn func())
139 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
140 MOV fn+0(FP), CTXT // CTXT = fn
141 MOV g_m(g), T0 // T0 = m
142
143 MOV m_gsignal(T0), T1 // T1 = gsignal
144 BEQ g, T1, noswitch
145
146 MOV m_g0(T0), T1 // T1 = g0
147 BEQ g, T1, noswitch
148
149 MOV m_curg(T0), T2
150 BEQ g, T2, switch
151
152 // Bad: g is not gsignal, not g0, not curg. What is it?
153 // Hide call from linker nosplit analysis.
154 MOV $runtime·badsystemstack(SB), T1
155 JALR RA, T1
156
157 switch:
158 // save our state in g->sched. Pretend to
159 // be systemstack_switch if the G stack is scanned.
160 CALL gosave_systemstack_switch<>(SB)
161
162 // switch to g0
163 MOV T1, g
164 CALL runtime·save_g(SB)
165 MOV (g_sched+gobuf_sp)(g), T0
166 MOV T0, X2
167
168 // call target function
169 MOV 0(CTXT), T1 // code pointer
170 JALR RA, T1
171
172 // switch back to g
173 MOV g_m(g), T0
174 MOV m_curg(T0), g
175 CALL runtime·save_g(SB)
176 MOV (g_sched+gobuf_sp)(g), X2
177 MOV ZERO, (g_sched+gobuf_sp)(g)
178 RET
179
180 noswitch:
181 // already on m stack, just call directly
182 // Using a tail call here cleans up tracebacks since we won't stop
183 // at an intermediate systemstack.
184 MOV 0(CTXT), T1 // code pointer
185 ADD $8, X2
186 JMP (T1)
187
188 // func switchToCrashStack0(fn func())
189 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
190 MOV X10, CTXT // context register
191 MOV g_m(g), X11 // curm
192
193 // set g to gcrash
194 MOV $runtime·gcrash(SB), g // g = &gcrash
195 CALL runtime·save_g(SB) // clobbers X31
196 MOV X11, g_m(g) // g.m = curm
197 MOV g, m_g0(X11) // curm.g0 = g
198
199 // switch to crashstack
200 MOV (g_stack+stack_hi)(g), X11
201 SUB $(4*8), X11
202 MOV X11, X2
203
204 // call target function
205 MOV 0(CTXT), X10
206 JALR X1, X10
207
208 // should never return
209 CALL runtime·abort(SB)
210 UNDEF
211
212 /*
213 * support for morestack
214 */
215
216 // Called during function prolog when more stack is needed.
217 // Called with return address (i.e. caller's PC) in X5 (aka T0),
218 // and the LR register contains the caller's LR.
219 //
220 // The traceback routines see morestack on a g0 as being
221 // the top of a stack (for example, morestack calling newstack
222 // calling the scheduler calling newm calling gc), so we must
223 // record an argument size. For that purpose, it has no arguments.
224
225 // func morestack()
226 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
227 // Called from f.
228 // Set g->sched to context in f.
229 MOV X2, (g_sched+gobuf_sp)(g)
230 MOV T0, (g_sched+gobuf_pc)(g)
231 MOV RA, (g_sched+gobuf_lr)(g)
232 MOV CTXT, (g_sched+gobuf_ctxt)(g)
233
234 // Cannot grow scheduler stack (m->g0).
235 MOV g_m(g), A0
236 MOV m_g0(A0), A1
237 BNE g, A1, 3(PC)
238 CALL runtime·badmorestackg0(SB)
239 CALL runtime·abort(SB)
240
241 // Cannot grow signal stack (m->gsignal).
242 MOV m_gsignal(A0), A1
243 BNE g, A1, 3(PC)
244 CALL runtime·badmorestackgsignal(SB)
245 CALL runtime·abort(SB)
246
247 // Called from f.
248 // Set m->morebuf to f's caller.
249 MOV RA, (m_morebuf+gobuf_pc)(A0) // f's caller's PC
250 MOV X2, (m_morebuf+gobuf_sp)(A0) // f's caller's SP
251 MOV g, (m_morebuf+gobuf_g)(A0)
252
253 // Call newstack on m->g0's stack.
254 MOV m_g0(A0), g
255 CALL runtime·save_g(SB)
256 MOV (g_sched+gobuf_sp)(g), X2
257 // Create a stack frame on g0 to call newstack.
258 MOV ZERO, -8(X2) // Zero saved LR in frame
259 SUB $8, X2
260 CALL runtime·newstack(SB)
261
262 // Not reached, but make sure the return PC from the call to newstack
263 // is still in this function, and not the beginning of the next.
264 UNDEF
265
266 // func morestack_noctxt()
267 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
268 // Force SPWRITE. This function doesn't actually write SP,
269 // but it is called with a special calling convention where
270 // the caller doesn't save LR on stack but passes it as a
271 // register, and the unwinder currently doesn't understand.
272 // Make it SPWRITE to stop unwinding. (See issue 54332)
273 MOV X2, X2
274
275 MOV ZERO, CTXT
276 JMP runtime·morestack(SB)
277
278 // check that SP is in range [g->stack.lo, g->stack.hi]
279 TEXT runtime·stackcheck(SB), NOSPLIT|NOFRAME, $0-0
280 MOV (g_stack+stack_hi)(g), A0
281 BGEU A0, X2, 2(PC)
282 CALL runtime·abort(SB)
283
284 MOV (g_stack+stack_lo)(g), A0
285 BGTU X2, A0, 2(PC)
286 CALL runtime·abort(SB)
287 RET
288
289 // restore state from Gobuf; longjmp
290
291 // func gogo(buf *gobuf)
292 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
293 MOV buf+0(FP), T0
294 MOV gobuf_g(T0), T1
295 MOV 0(T1), ZERO // make sure g != nil
296 JMP gogo<>(SB)
297
298 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
299 MOV T1, g
300 CALL runtime·save_g(SB)
301
302 MOV gobuf_sp(T0), X2
303 MOV gobuf_lr(T0), RA
304 MOV gobuf_ctxt(T0), CTXT
305 MOV ZERO, gobuf_sp(T0)
306 MOV ZERO, gobuf_lr(T0)
307 MOV ZERO, gobuf_ctxt(T0)
308 MOV gobuf_pc(T0), T0
309 JALR ZERO, T0
310
311 // func procyieldAsm(cycles uint32)
312 TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0
313 RET
314
315 // Switch to m->g0's stack, call fn(g).
316 // Fn must never return. It should gogo(&g->sched)
317 // to keep running g.
318
319 // func mcall(fn func(*g))
320 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
321 MOV X10, CTXT
322
323 // Save caller state in g->sched
324 MOV X2, (g_sched+gobuf_sp)(g)
325 MOV RA, (g_sched+gobuf_pc)(g)
326 MOV ZERO, (g_sched+gobuf_lr)(g)
327
328 // Switch to m->g0 & its stack, call fn.
329 MOV g, X10
330 MOV g_m(g), T1
331 MOV m_g0(T1), g
332 CALL runtime·save_g(SB)
333 BNE g, X10, 2(PC)
334 JMP runtime·badmcall(SB)
335 MOV 0(CTXT), T1 // code pointer
336 MOV (g_sched+gobuf_sp)(g), X2 // sp = m->g0->sched.sp
337 // we don't need special macro for regabi since arg0(X10) = g
338 SUB $16, X2
339 MOV X10, 8(X2) // setup g
340 MOV ZERO, 0(X2) // clear return address
341 JALR RA, T1
342 JMP runtime·badmcall2(SB)
343
344 // Save state of caller into g->sched,
345 // but using fake PC from systemstack_switch.
346 // Must only be called from functions with no locals ($0)
347 // or else unwinding from systemstack_switch is incorrect.
348 // Smashes X31.
349 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
350 MOV $runtime·systemstack_switch(SB), X31
351 ADD $8, X31 // get past prologue
352 MOV X31, (g_sched+gobuf_pc)(g)
353 MOV X2, (g_sched+gobuf_sp)(g)
354 MOV ZERO, (g_sched+gobuf_lr)(g)
355 // Assert ctxt is zero. See func save.
356 MOV (g_sched+gobuf_ctxt)(g), X31
357 BEQ ZERO, X31, 2(PC)
358 CALL runtime·abort(SB)
359 RET
360
361 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
362 // Call fn(arg) aligned appropriately for the gcc ABI.
363 // Called on a system stack, and there may be no g yet (during needm).
364 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
365 MOV fn+0(FP), X11
366 MOV arg+8(FP), X10
367 JALR RA, (X11)
368 RET
369
370 // func asmcgocall(fn, arg unsafe.Pointer) int32
371 // Call fn(arg) on the scheduler stack,
372 // aligned appropriately for the gcc ABI.
373 // See cgocall.go for more details.
374 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
375 MOV fn+0(FP), X11
376 MOV arg+8(FP), X10
377
378 MOV X2, X8 // save original stack pointer
379 MOV g, X9
380
381 // Figure out if we need to switch to m->g0 stack.
382 // We get called to create new OS threads too, and those
383 // come in on the m->g0 stack already. Or we might already
384 // be on the m->gsignal stack.
385 BEQZ g, nosave
386 MOV g_m(g), X6
387 MOV m_gsignal(X6), X7
388 BEQ X7, g, g0
389 MOV m_g0(X6), X7
390 BEQ X7, g, g0
391
392 CALL gosave_systemstack_switch<>(SB)
393 MOV X7, g
394 CALL runtime·save_g(SB)
395 MOV (g_sched+gobuf_sp)(g), X2
396
397 // Now on a scheduling stack (a pthread-created stack).
398 g0:
399 // Save room for two of our pointers.
400 SUB $16, X2
401 MOV X9, 0(X2) // save old g on stack
402 MOV (g_stack+stack_hi)(X9), X9
403 SUB X8, X9, X8
404 MOV X8, 8(X2) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
405
406 JALR RA, (X11)
407
408 // Restore g, stack pointer. X10 is return value.
409 MOV 0(X2), g
410 CALL runtime·save_g(SB)
411 MOV (g_stack+stack_hi)(g), X5
412 MOV 8(X2), X6
413 SUB X6, X5, X6
414 MOV X6, X2
415
416 MOVW X10, ret+16(FP)
417 RET
418
419 nosave:
420 // Running on a system stack, perhaps even without a g.
421 // Having no g can happen during thread creation or thread teardown.
422 MOV fn+0(FP), X11
423 MOV arg+8(FP), X10
424 MOV X2, X8
425 SUB $16, X2
426 MOV ZERO, 0(X2) // Where above code stores g, in case someone looks during debugging.
427 MOV X8, 8(X2) // Save original stack pointer.
428 JALR RA, (X11)
429 MOV 8(X2), X2 // Restore stack pointer.
430 MOVW X10, ret+16(FP)
431 RET
432
433 // func asminit()
434 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
435 RET
436
437 // reflectcall: call a function with the given argument list
438 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
439 // we don't have variable-sized frames, so we use a small number
440 // of constant-sized-frame functions to encode a few bits of size in the pc.
441 // Caution: ugly multiline assembly macros in your future!
442
443 #define DISPATCH(NAME,MAXSIZE) \
444 MOV $MAXSIZE, T1 \
445 BLTU T1, T0, 3(PC) \
446 MOV $NAME(SB), T2; \
447 JALR ZERO, T2
448 // Note: can't just "BR NAME(SB)" - bad inlining results.
449
450 // func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
451 TEXT reflect·call(SB), NOSPLIT, $0-0
452 JMP ·reflectcall(SB)
453
454 // func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
455 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
456 MOVWU frameSize+32(FP), T0
457 DISPATCH(runtime·call16, 16)
458 DISPATCH(runtime·call32, 32)
459 DISPATCH(runtime·call64, 64)
460 DISPATCH(runtime·call128, 128)
461 DISPATCH(runtime·call256, 256)
462 DISPATCH(runtime·call512, 512)
463 DISPATCH(runtime·call1024, 1024)
464 DISPATCH(runtime·call2048, 2048)
465 DISPATCH(runtime·call4096, 4096)
466 DISPATCH(runtime·call8192, 8192)
467 DISPATCH(runtime·call16384, 16384)
468 DISPATCH(runtime·call32768, 32768)
469 DISPATCH(runtime·call65536, 65536)
470 DISPATCH(runtime·call131072, 131072)
471 DISPATCH(runtime·call262144, 262144)
472 DISPATCH(runtime·call524288, 524288)
473 DISPATCH(runtime·call1048576, 1048576)
474 DISPATCH(runtime·call2097152, 2097152)
475 DISPATCH(runtime·call4194304, 4194304)
476 DISPATCH(runtime·call8388608, 8388608)
477 DISPATCH(runtime·call16777216, 16777216)
478 DISPATCH(runtime·call33554432, 33554432)
479 DISPATCH(runtime·call67108864, 67108864)
480 DISPATCH(runtime·call134217728, 134217728)
481 DISPATCH(runtime·call268435456, 268435456)
482 DISPATCH(runtime·call536870912, 536870912)
483 DISPATCH(runtime·call1073741824, 1073741824)
484 MOV $runtime·badreflectcall(SB), T2
485 JALR ZERO, T2
486
487 #define CALLFN(NAME,MAXSIZE) \
488 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
489 NO_LOCAL_POINTERS; \
490 /* copy arguments to stack */ \
491 MOV stackArgs+16(FP), A1; \
492 MOVWU stackArgsSize+24(FP), A2; \
493 MOV X2, A3; \
494 ADD $8, A3; \
495 ADD A3, A2; \
496 BEQ A3, A2, 6(PC); \
497 MOVBU (A1), A4; \
498 ADD $1, A1; \
499 MOVB A4, (A3); \
500 ADD $1, A3; \
501 JMP -5(PC); \
502 /* set up argument registers */ \
503 MOV regArgs+40(FP), X25; \
504 CALL ·unspillArgs(SB); \
505 /* call function */ \
506 MOV f+8(FP), CTXT; \
507 MOV (CTXT), X25; \
508 PCDATA $PCDATA_StackMapIndex, $0; \
509 JALR RA, X25; \
510 /* copy return values back */ \
511 MOV regArgs+40(FP), X25; \
512 CALL ·spillArgs(SB); \
513 MOV stackArgsType+0(FP), A5; \
514 MOV stackArgs+16(FP), A1; \
515 MOVWU stackArgsSize+24(FP), A2; \
516 MOVWU stackRetOffset+28(FP), A4; \
517 ADD $8, X2, A3; \
518 ADD A4, A3; \
519 ADD A4, A1; \
520 SUB A4, A2; \
521 CALL callRet<>(SB); \
522 RET
523
524 // callRet copies return values back at the end of call*. This is a
525 // separate function so it can allocate stack space for the arguments
526 // to reflectcallmove. It does not follow the Go ABI; it expects its
527 // arguments in registers.
528 TEXT callRet<>(SB), NOSPLIT, $40-0
529 NO_LOCAL_POINTERS
530 MOV A5, 8(X2)
531 MOV A1, 16(X2)
532 MOV A3, 24(X2)
533 MOV A2, 32(X2)
534 MOV X25, 40(X2)
535 CALL runtime·reflectcallmove(SB)
536 RET
537
538 CALLFN(·call16, 16)
539 CALLFN(·call32, 32)
540 CALLFN(·call64, 64)
541 CALLFN(·call128, 128)
542 CALLFN(·call256, 256)
543 CALLFN(·call512, 512)
544 CALLFN(·call1024, 1024)
545 CALLFN(·call2048, 2048)
546 CALLFN(·call4096, 4096)
547 CALLFN(·call8192, 8192)
548 CALLFN(·call16384, 16384)
549 CALLFN(·call32768, 32768)
550 CALLFN(·call65536, 65536)
551 CALLFN(·call131072, 131072)
552 CALLFN(·call262144, 262144)
553 CALLFN(·call524288, 524288)
554 CALLFN(·call1048576, 1048576)
555 CALLFN(·call2097152, 2097152)
556 CALLFN(·call4194304, 4194304)
557 CALLFN(·call8388608, 8388608)
558 CALLFN(·call16777216, 16777216)
559 CALLFN(·call33554432, 33554432)
560 CALLFN(·call67108864, 67108864)
561 CALLFN(·call134217728, 134217728)
562 CALLFN(·call268435456, 268435456)
563 CALLFN(·call536870912, 536870912)
564 CALLFN(·call1073741824, 1073741824)
565
566 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
567 // Must obey the gcc calling convention.
568 TEXT _cgo_topofstack(SB),NOSPLIT,$8
569 // g (X27) and REG_TMP (X31) might be clobbered by load_g.
570 // X27 is callee-save in the gcc calling convention, so save it.
571 MOV g, savedX27-8(SP)
572
573 CALL runtime·load_g(SB)
574 MOV g_m(g), X5
575 MOV m_curg(X5), X5
576 MOV (g_stack+stack_hi)(X5), X10 // return value in X10
577
578 MOV savedX27-8(SP), g
579 RET
580
581 // func goexit(neverCallThisFunction)
582 // The top-most function running on a goroutine, returns to goexit+PCQuantum*2.
583 // Note that the NOPs are written in a manner that will not be compressed,
584 // since the offset must be known by the runtime.
585 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
586 WORD $0x00000013 // NOP
587 JMP runtime·goexit1(SB) // does not return
588 // traceback from goexit1 must hit code range of goexit
589 WORD $0x00000013 // NOP
590
591 // This is called from .init_array and follows the platform, not the Go ABI.
592 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
593 // Use X31 as it is a scratch register in both the Go ABI and psABI.
594 MOV runtime·lastmoduledatap(SB), X31
595 MOV X10, moduledata_next(X31)
596 MOV X10, runtime·lastmoduledatap(SB)
597 RET
598
599 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
600 // See cgocall.go for more details.
601 TEXT ·cgocallback(SB),NOSPLIT,$24-24
602 NO_LOCAL_POINTERS
603
604 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
605 // It is used to dropm while thread is exiting.
606 MOV fn+0(FP), X7
607 BNE ZERO, X7, loadg
608 // Restore the g from frame.
609 MOV frame+8(FP), g
610 JMP dropm
611
612 loadg:
613 // Load m and g from thread-local storage.
614 MOVBU runtime·iscgo(SB), X5
615 BEQ ZERO, X5, nocgo
616 CALL runtime·load_g(SB)
617 nocgo:
618
619 // If g is nil, Go did not create the current thread,
620 // or if this thread never called into Go on pthread platforms.
621 // Call needm to obtain one for temporary use.
622 // In this case, we're running on the thread stack, so there's
623 // lots of space, but the linker doesn't know. Hide the call from
624 // the linker analysis by using an indirect call.
625 BEQ ZERO, g, needm
626
627 MOV g_m(g), X5
628 MOV X5, savedm-8(SP)
629 JMP havem
630
631 needm:
632 MOV g, savedm-8(SP) // g is zero, so is m.
633 MOV $runtime·needAndBindM(SB), X6
634 JALR RA, X6
635
636 // Set m->sched.sp = SP, so that if a panic happens
637 // during the function we are about to execute, it will
638 // have a valid SP to run on the g0 stack.
639 // The next few lines (after the havem label)
640 // will save this SP onto the stack and then write
641 // the same SP back to m->sched.sp. That seems redundant,
642 // but if an unrecovered panic happens, unwindm will
643 // restore the g->sched.sp from the stack location
644 // and then systemstack will try to use it. If we don't set it here,
645 // that restored SP will be uninitialized (typically 0) and
646 // will not be usable.
647 MOV g_m(g), X5
648 MOV m_g0(X5), X6
649 MOV X2, (g_sched+gobuf_sp)(X6)
650
651 havem:
652 // Now there's a valid m, and we're running on its m->g0.
653 // Save current m->g0->sched.sp on stack and then set it to SP.
654 // Save current sp in m->g0->sched.sp in preparation for
655 // switch back to m->curg stack.
656 // NOTE: unwindm knows that the saved g->sched.sp is at 8(X2) aka savedsp-24(SP).
657 MOV m_g0(X5), X6
658 MOV (g_sched+gobuf_sp)(X6), X7
659 MOV X7, savedsp-24(SP) // must match frame size
660 MOV X2, (g_sched+gobuf_sp)(X6)
661
662 // Switch to m->curg stack and call runtime.cgocallbackg.
663 // Because we are taking over the execution of m->curg
664 // but *not* resuming what had been running, we need to
665 // save that information (m->curg->sched) so we can restore it.
666 // We can restore m->curg->sched.sp easily, because calling
667 // runtime.cgocallbackg leaves SP unchanged upon return.
668 // To save m->curg->sched.pc, we push it onto the curg stack and
669 // open a frame the same size as cgocallback's g0 frame.
670 // Once we switch to the curg stack, the pushed PC will appear
671 // to be the return PC of cgocallback, so that the traceback
672 // will seamlessly trace back into the earlier calls.
673 MOV m_curg(X5), g
674 CALL runtime·save_g(SB)
675 MOV (g_sched+gobuf_sp)(g), X6 // prepare stack as X6
676 MOV (g_sched+gobuf_pc)(g), X7
677 MOV X7, -(24+8)(X6) // "saved LR"; must match frame size
678 // Gather our arguments into registers.
679 MOV fn+0(FP), X7
680 MOV frame+8(FP), X8
681 MOV ctxt+16(FP), X9
682 MOV $-(24+8)(X6), X2 // switch stack; must match frame size
683 MOV X7, 8(X2)
684 MOV X8, 16(X2)
685 MOV X9, 24(X2)
686 CALL runtime·cgocallbackg(SB)
687
688 // Restore g->sched (== m->curg->sched) from saved values.
689 MOV 0(X2), X7
690 MOV X7, (g_sched+gobuf_pc)(g)
691 MOV $(24+8)(X2), X6 // must match frame size
692 MOV X6, (g_sched+gobuf_sp)(g)
693
694 // Switch back to m->g0's stack and restore m->g0->sched.sp.
695 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
696 // so we do not have to restore it.)
697 MOV g_m(g), X5
698 MOV m_g0(X5), g
699 CALL runtime·save_g(SB)
700 MOV (g_sched+gobuf_sp)(g), X2
701 MOV savedsp-24(SP), X6 // must match frame size
702 MOV X6, (g_sched+gobuf_sp)(g)
703
704 // If the m on entry was nil, we called needm above to borrow an m,
705 // 1. for the duration of the call on non-pthread platforms,
706 // 2. or the duration of the C thread alive on pthread platforms.
707 // If the m on entry wasn't nil,
708 // 1. the thread might be a Go thread,
709 // 2. or it wasn't the first call from a C thread on pthread platforms,
710 // since then we skip dropm to reuse the m in the first call.
711 MOV savedm-8(SP), X5
712 BNE ZERO, X5, droppedm
713
714 // Skip dropm to reuse it in the next call, when a pthread key has been created.
715 MOV _cgo_pthread_key_created(SB), X5
716 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
717 BEQ ZERO, X5, dropm
718 MOV (X5), X5
719 BNE ZERO, X5, droppedm
720
721 dropm:
722 MOV $runtime·dropm(SB), X6
723 JALR RA, X6
724 droppedm:
725
726 // Done!
727 RET
728
729 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
730 EBREAK
731 RET
732
733 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
734 EBREAK
735 RET
736
737 // void setg(G*); set g. for use by needm.
738 TEXT runtime·setg(SB), NOSPLIT, $0-8
739 MOV gg+0(FP), g
740 // This only happens if iscgo, so jump straight to save_g
741 CALL runtime·save_g(SB)
742 RET
743
744 // spillArgs stores return values from registers to a *internal/abi.RegArgs in X25.
745 TEXT ·spillArgs(SB),NOSPLIT,$0-0
746 MOV X10, (0*8)(X25)
747 MOV X11, (1*8)(X25)
748 MOV X12, (2*8)(X25)
749 MOV X13, (3*8)(X25)
750 MOV X14, (4*8)(X25)
751 MOV X15, (5*8)(X25)
752 MOV X16, (6*8)(X25)
753 MOV X17, (7*8)(X25)
754 MOV X8, (8*8)(X25)
755 MOV X9, (9*8)(X25)
756 MOV X18, (10*8)(X25)
757 MOV X19, (11*8)(X25)
758 MOV X20, (12*8)(X25)
759 MOV X21, (13*8)(X25)
760 MOV X22, (14*8)(X25)
761 MOV X23, (15*8)(X25)
762 MOVD F10, (16*8)(X25)
763 MOVD F11, (17*8)(X25)
764 MOVD F12, (18*8)(X25)
765 MOVD F13, (19*8)(X25)
766 MOVD F14, (20*8)(X25)
767 MOVD F15, (21*8)(X25)
768 MOVD F16, (22*8)(X25)
769 MOVD F17, (23*8)(X25)
770 MOVD F8, (24*8)(X25)
771 MOVD F9, (25*8)(X25)
772 MOVD F18, (26*8)(X25)
773 MOVD F19, (27*8)(X25)
774 MOVD F20, (28*8)(X25)
775 MOVD F21, (29*8)(X25)
776 MOVD F22, (30*8)(X25)
777 MOVD F23, (31*8)(X25)
778 RET
779
780 // unspillArgs loads args into registers from a *internal/abi.RegArgs in X25.
781 TEXT ·unspillArgs(SB),NOSPLIT,$0-0
782 MOV (0*8)(X25), X10
783 MOV (1*8)(X25), X11
784 MOV (2*8)(X25), X12
785 MOV (3*8)(X25), X13
786 MOV (4*8)(X25), X14
787 MOV (5*8)(X25), X15
788 MOV (6*8)(X25), X16
789 MOV (7*8)(X25), X17
790 MOV (8*8)(X25), X8
791 MOV (9*8)(X25), X9
792 MOV (10*8)(X25), X18
793 MOV (11*8)(X25), X19
794 MOV (12*8)(X25), X20
795 MOV (13*8)(X25), X21
796 MOV (14*8)(X25), X22
797 MOV (15*8)(X25), X23
798 MOVD (16*8)(X25), F10
799 MOVD (17*8)(X25), F11
800 MOVD (18*8)(X25), F12
801 MOVD (19*8)(X25), F13
802 MOVD (20*8)(X25), F14
803 MOVD (21*8)(X25), F15
804 MOVD (22*8)(X25), F16
805 MOVD (23*8)(X25), F17
806 MOVD (24*8)(X25), F8
807 MOVD (25*8)(X25), F9
808 MOVD (26*8)(X25), F18
809 MOVD (27*8)(X25), F19
810 MOVD (28*8)(X25), F20
811 MOVD (29*8)(X25), F21
812 MOVD (30*8)(X25), F22
813 MOVD (31*8)(X25), F23
814 RET
815
816 // gcWriteBarrier informs the GC about heap pointer writes.
817 //
818 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
819 // number of bytes of buffer needed in X24, and returns a pointer
820 // to the buffer space in X24.
821 // It clobbers X31 aka T6 (the linker temp register - REG_TMP).
822 // The act of CALLing gcWriteBarrier will clobber RA (LR).
823 // It does not clobber any other general-purpose registers,
824 // but may clobber others (e.g., floating point registers).
825 TEXT gcWriteBarrier<>(SB),NOSPLIT,$208
826 // Save the registers clobbered by the fast path.
827 MOV A0, 24*8(X2)
828 MOV A1, 25*8(X2)
829 retry:
830 MOV g_m(g), A0
831 MOV m_p(A0), A0
832 MOV (p_wbBuf+wbBuf_next)(A0), A1
833 MOV (p_wbBuf+wbBuf_end)(A0), T6 // T6 is linker temp register (REG_TMP)
834 // Increment wbBuf.next position.
835 ADD X24, A1
836 // Is the buffer full?
837 BLTU T6, A1, flush
838 // Commit to the larger buffer.
839 MOV A1, (p_wbBuf+wbBuf_next)(A0)
840 // Make the return value (the original next position)
841 SUB X24, A1, X24
842 // Restore registers.
843 MOV 24*8(X2), A0
844 MOV 25*8(X2), A1
845 RET
846
847 flush:
848 // Save all general purpose registers since these could be
849 // clobbered by wbBufFlush and were not saved by the caller.
850 MOV T0, 1*8(X2)
851 MOV T1, 2*8(X2)
852 // X0 is zero register
853 // X1 is LR, saved by prologue
854 // X2 is SP
855 // X3 is GP
856 // X4 is TP
857 MOV X7, 3*8(X2)
858 MOV X8, 4*8(X2)
859 MOV X9, 5*8(X2)
860 // X10 already saved (A0)
861 // X11 already saved (A1)
862 MOV X12, 6*8(X2)
863 MOV X13, 7*8(X2)
864 MOV X14, 8*8(X2)
865 MOV X15, 9*8(X2)
866 MOV X16, 10*8(X2)
867 MOV X17, 11*8(X2)
868 MOV X18, 12*8(X2)
869 MOV X19, 13*8(X2)
870 MOV X20, 14*8(X2)
871 MOV X21, 15*8(X2)
872 MOV X22, 16*8(X2)
873 MOV X23, 17*8(X2)
874 MOV X24, 18*8(X2)
875 MOV X25, 19*8(X2)
876 MOV X26, 20*8(X2)
877 // X27 is g.
878 MOV X28, 21*8(X2)
879 MOV X29, 22*8(X2)
880 MOV X30, 23*8(X2)
881 // X31 is tmp register.
882
883 CALL runtime·wbBufFlush(SB)
884
885 MOV 1*8(X2), T0
886 MOV 2*8(X2), T1
887 MOV 3*8(X2), X7
888 MOV 4*8(X2), X8
889 MOV 5*8(X2), X9
890 MOV 6*8(X2), X12
891 MOV 7*8(X2), X13
892 MOV 8*8(X2), X14
893 MOV 9*8(X2), X15
894 MOV 10*8(X2), X16
895 MOV 11*8(X2), X17
896 MOV 12*8(X2), X18
897 MOV 13*8(X2), X19
898 MOV 14*8(X2), X20
899 MOV 15*8(X2), X21
900 MOV 16*8(X2), X22
901 MOV 17*8(X2), X23
902 MOV 18*8(X2), X24
903 MOV 19*8(X2), X25
904 MOV 20*8(X2), X26
905 MOV 21*8(X2), X28
906 MOV 22*8(X2), X29
907 MOV 23*8(X2), X30
908
909 JMP retry
910
911 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
912 MOV $8, X24
913 JMP gcWriteBarrier<>(SB)
914 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
915 MOV $16, X24
916 JMP gcWriteBarrier<>(SB)
917 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
918 MOV $24, X24
919 JMP gcWriteBarrier<>(SB)
920 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
921 MOV $32, X24
922 JMP gcWriteBarrier<>(SB)
923 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
924 MOV $40, X24
925 JMP gcWriteBarrier<>(SB)
926 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
927 MOV $48, X24
928 JMP gcWriteBarrier<>(SB)
929 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
930 MOV $56, X24
931 JMP gcWriteBarrier<>(SB)
932 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
933 MOV $64, X24
934 JMP gcWriteBarrier<>(SB)
935
936 TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-0
937 NO_LOCAL_POINTERS
938 // Save all 16 int registers that could have an index in them.
939 // They may be pointers, but if they are they are dead.
940 // Skip X0 aka ZERO, X1 aka LR, X2 aka SP, X3 aka GP, X4 aka TP.
941 MOV X5, 24(X2)
942 MOV X6, 32(X2)
943 MOV X7, 40(X2)
944 MOV X8, 48(X2)
945 MOV X9, 56(X2)
946 MOV X10, 64(X2)
947 MOV X11, 72(X2)
948 MOV X12, 80(X2)
949 MOV X13, 88(X2)
950 MOV X14, 96(X2)
951 MOV X15, 104(X2)
952 MOV X16, 112(X2)
953 MOV X17, 120(X2)
954 MOV X18, 128(X2)
955 MOV X19, 136(X2)
956 MOV X20, 144(X2)
957
958 MOV X1, X10 // PC immediately after call to panicBounds
959 ADD $24, X2, X11 // pointer to save area
960 CALL runtime·panicBounds64<ABIInternal>(SB)
961 RET
962
963 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
964 GLOBL runtime·mainPC(SB),RODATA,$8
965
View as plain text