Text file
src/runtime/asm_riscv64.s
1 // Copyright 2017 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "funcdata.h"
7 #include "textflag.h"
8
9 // func rt0_go()
10 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
11 // X2 = stack; A0 = argc; A1 = argv
12 SUB $24, X2
13 MOV A0, 8(X2) // argc
14 MOV A1, 16(X2) // argv
15
16 // create istack out of the given (operating system) stack.
17 // _cgo_init may update stackguard.
18 MOV $runtime·g0(SB), g
19 MOV $(-64*1024), T0
20 ADD T0, X2, T1
21 MOV T1, g_stackguard0(g)
22 MOV T1, g_stackguard1(g)
23 MOV T1, (g_stack+stack_lo)(g)
24 MOV X2, (g_stack+stack_hi)(g)
25
26 // if there is a _cgo_init, call it using the gcc ABI.
27 MOV _cgo_init(SB), T0
28 BEQ T0, ZERO, nocgo
29
30 MOV ZERO, A3 // arg 3: not used
31 MOV ZERO, A2 // arg 2: not used
32 MOV $setg_gcc<>(SB), A1 // arg 1: setg
33 MOV g, A0 // arg 0: G
34 JALR RA, T0
35
36 nocgo:
37 // update stackguard after _cgo_init
38 MOV (g_stack+stack_lo)(g), T0
39 ADD $const_stackGuard, T0
40 MOV T0, g_stackguard0(g)
41 MOV T0, g_stackguard1(g)
42
43 // set the per-goroutine and per-mach "registers"
44 MOV $runtime·m0(SB), T0
45
46 // save m->g0 = g0
47 MOV g, m_g0(T0)
48 // save m0 to g0->m
49 MOV T0, g_m(g)
50
51 CALL runtime·check(SB)
52
53 // args are already prepared
54 CALL runtime·args(SB)
55 CALL runtime·osinit(SB)
56 CALL runtime·schedinit(SB)
57
58 // create a new goroutine to start program
59 MOV $runtime·mainPC(SB), T0 // entry
60 SUB $16, X2
61 MOV T0, 8(X2)
62 MOV ZERO, 0(X2)
63 CALL runtime·newproc(SB)
64 ADD $16, X2
65
66 // start this M
67 CALL runtime·mstart(SB)
68
69 WORD $0 // crash if reached
70 RET
71
72 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
73 CALL runtime·mstart0(SB)
74 RET // not reached
75
76 // void setg_gcc(G*); set g called from gcc with g in A0
77 TEXT setg_gcc<>(SB),NOSPLIT,$0-0
78 MOV A0, g
79 CALL runtime·save_g(SB)
80 RET
81
82 // func cputicks() int64
83 TEXT runtime·cputicks<ABIInternal>(SB),NOSPLIT,$0-0
84 // RDTIME to emulate cpu ticks
85 // RDCYCLE reads counter that is per HART(core) based
86 // according to the riscv manual, see issue 46737
87 RDTIME X10
88 RET
89
90 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
91 // of the G stack. We need to distinguish the routine that
92 // lives at the bottom of the G stack from the one that lives
93 // at the top of the system stack because the one at the top of
94 // the system stack terminates the stack walk (see topofstack()).
95 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
96 UNDEF
97 JALR RA, ZERO // make sure this function is not leaf
98 RET
99
100 // func systemstack(fn func())
101 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
102 MOV fn+0(FP), CTXT // CTXT = fn
103 MOV g_m(g), T0 // T0 = m
104
105 MOV m_gsignal(T0), T1 // T1 = gsignal
106 BEQ g, T1, noswitch
107
108 MOV m_g0(T0), T1 // T1 = g0
109 BEQ g, T1, noswitch
110
111 MOV m_curg(T0), T2
112 BEQ g, T2, switch
113
114 // Bad: g is not gsignal, not g0, not curg. What is it?
115 // Hide call from linker nosplit analysis.
116 MOV $runtime·badsystemstack(SB), T1
117 JALR RA, T1
118
119 switch:
120 // save our state in g->sched. Pretend to
121 // be systemstack_switch if the G stack is scanned.
122 CALL gosave_systemstack_switch<>(SB)
123
124 // switch to g0
125 MOV T1, g
126 CALL runtime·save_g(SB)
127 MOV (g_sched+gobuf_sp)(g), T0
128 MOV T0, X2
129
130 // call target function
131 MOV 0(CTXT), T1 // code pointer
132 JALR RA, T1
133
134 // switch back to g
135 MOV g_m(g), T0
136 MOV m_curg(T0), g
137 CALL runtime·save_g(SB)
138 MOV (g_sched+gobuf_sp)(g), X2
139 MOV ZERO, (g_sched+gobuf_sp)(g)
140 RET
141
142 noswitch:
143 // already on m stack, just call directly
144 // Using a tail call here cleans up tracebacks since we won't stop
145 // at an intermediate systemstack.
146 MOV 0(CTXT), T1 // code pointer
147 ADD $8, X2
148 JMP (T1)
149
150 // func switchToCrashStack0(fn func())
151 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
152 MOV X10, CTXT // context register
153 MOV g_m(g), X11 // curm
154
155 // set g to gcrash
156 MOV $runtime·gcrash(SB), g // g = &gcrash
157 CALL runtime·save_g(SB) // clobbers X31
158 MOV X11, g_m(g) // g.m = curm
159 MOV g, m_g0(X11) // curm.g0 = g
160
161 // switch to crashstack
162 MOV (g_stack+stack_hi)(g), X11
163 SUB $(4*8), X11
164 MOV X11, X2
165
166 // call target function
167 MOV 0(CTXT), X10
168 JALR X1, X10
169
170 // should never return
171 CALL runtime·abort(SB)
172 UNDEF
173
174 /*
175 * support for morestack
176 */
177
178 // Called during function prolog when more stack is needed.
179 // Called with return address (i.e. caller's PC) in X5 (aka T0),
180 // and the LR register contains the caller's LR.
181 //
182 // The traceback routines see morestack on a g0 as being
183 // the top of a stack (for example, morestack calling newstack
184 // calling the scheduler calling newm calling gc), so we must
185 // record an argument size. For that purpose, it has no arguments.
186
187 // func morestack()
188 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
189 // Called from f.
190 // Set g->sched to context in f.
191 MOV X2, (g_sched+gobuf_sp)(g)
192 MOV T0, (g_sched+gobuf_pc)(g)
193 MOV RA, (g_sched+gobuf_lr)(g)
194 MOV CTXT, (g_sched+gobuf_ctxt)(g)
195
196 // Cannot grow scheduler stack (m->g0).
197 MOV g_m(g), A0
198 MOV m_g0(A0), A1
199 BNE g, A1, 3(PC)
200 CALL runtime·badmorestackg0(SB)
201 CALL runtime·abort(SB)
202
203 // Cannot grow signal stack (m->gsignal).
204 MOV m_gsignal(A0), A1
205 BNE g, A1, 3(PC)
206 CALL runtime·badmorestackgsignal(SB)
207 CALL runtime·abort(SB)
208
209 // Called from f.
210 // Set m->morebuf to f's caller.
211 MOV RA, (m_morebuf+gobuf_pc)(A0) // f's caller's PC
212 MOV X2, (m_morebuf+gobuf_sp)(A0) // f's caller's SP
213 MOV g, (m_morebuf+gobuf_g)(A0)
214
215 // Call newstack on m->g0's stack.
216 MOV m_g0(A0), g
217 CALL runtime·save_g(SB)
218 MOV (g_sched+gobuf_sp)(g), X2
219 // Create a stack frame on g0 to call newstack.
220 MOV ZERO, -8(X2) // Zero saved LR in frame
221 SUB $8, X2
222 CALL runtime·newstack(SB)
223
224 // Not reached, but make sure the return PC from the call to newstack
225 // is still in this function, and not the beginning of the next.
226 UNDEF
227
228 // func morestack_noctxt()
229 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
230 // Force SPWRITE. This function doesn't actually write SP,
231 // but it is called with a special calling convention where
232 // the caller doesn't save LR on stack but passes it as a
233 // register, and the unwinder currently doesn't understand.
234 // Make it SPWRITE to stop unwinding. (See issue 54332)
235 MOV X2, X2
236
237 MOV ZERO, CTXT
238 JMP runtime·morestack(SB)
239
240 // AES hashing not implemented for riscv64
241 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
242 JMP runtime·memhashFallback<ABIInternal>(SB)
243 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
244 JMP runtime·strhashFallback<ABIInternal>(SB)
245 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
246 JMP runtime·memhash32Fallback<ABIInternal>(SB)
247 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
248 JMP runtime·memhash64Fallback<ABIInternal>(SB)
249
250 // restore state from Gobuf; longjmp
251
252 // func gogo(buf *gobuf)
253 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
254 MOV buf+0(FP), T0
255 MOV gobuf_g(T0), T1
256 MOV 0(T1), ZERO // make sure g != nil
257 JMP gogo<>(SB)
258
259 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
260 MOV T1, g
261 CALL runtime·save_g(SB)
262
263 MOV gobuf_sp(T0), X2
264 MOV gobuf_lr(T0), RA
265 MOV gobuf_ctxt(T0), CTXT
266 MOV ZERO, gobuf_sp(T0)
267 MOV ZERO, gobuf_lr(T0)
268 MOV ZERO, gobuf_ctxt(T0)
269 MOV gobuf_pc(T0), T0
270 JALR ZERO, T0
271
272 // func procyield(cycles uint32)
273 TEXT runtime·procyield(SB),NOSPLIT,$0-0
274 RET
275
276 // Switch to m->g0's stack, call fn(g).
277 // Fn must never return. It should gogo(&g->sched)
278 // to keep running g.
279
280 // func mcall(fn func(*g))
281 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
282 MOV X10, CTXT
283
284 // Save caller state in g->sched
285 MOV X2, (g_sched+gobuf_sp)(g)
286 MOV RA, (g_sched+gobuf_pc)(g)
287 MOV ZERO, (g_sched+gobuf_lr)(g)
288
289 // Switch to m->g0 & its stack, call fn.
290 MOV g, X10
291 MOV g_m(g), T1
292 MOV m_g0(T1), g
293 CALL runtime·save_g(SB)
294 BNE g, X10, 2(PC)
295 JMP runtime·badmcall(SB)
296 MOV 0(CTXT), T1 // code pointer
297 MOV (g_sched+gobuf_sp)(g), X2 // sp = m->g0->sched.sp
298 // we don't need special macro for regabi since arg0(X10) = g
299 SUB $16, X2
300 MOV X10, 8(X2) // setup g
301 MOV ZERO, 0(X2) // clear return address
302 JALR RA, T1
303 JMP runtime·badmcall2(SB)
304
305 // Save state of caller into g->sched,
306 // but using fake PC from systemstack_switch.
307 // Must only be called from functions with no locals ($0)
308 // or else unwinding from systemstack_switch is incorrect.
309 // Smashes X31.
310 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
311 MOV $runtime·systemstack_switch(SB), X31
312 ADD $8, X31 // get past prologue
313 MOV X31, (g_sched+gobuf_pc)(g)
314 MOV X2, (g_sched+gobuf_sp)(g)
315 MOV ZERO, (g_sched+gobuf_lr)(g)
316 // Assert ctxt is zero. See func save.
317 MOV (g_sched+gobuf_ctxt)(g), X31
318 BEQ ZERO, X31, 2(PC)
319 CALL runtime·abort(SB)
320 RET
321
322 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
323 // Call fn(arg) aligned appropriately for the gcc ABI.
324 // Called on a system stack, and there may be no g yet (during needm).
325 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
326 MOV fn+0(FP), X5
327 MOV arg+8(FP), X10
328 JALR RA, (X5)
329 RET
330
331 // func asmcgocall(fn, arg unsafe.Pointer) int32
332 // Call fn(arg) on the scheduler stack,
333 // aligned appropriately for the gcc ABI.
334 // See cgocall.go for more details.
335 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
336 MOV fn+0(FP), X5
337 MOV arg+8(FP), X10
338
339 MOV X2, X8 // save original stack pointer
340 MOV g, X9
341
342 // Figure out if we need to switch to m->g0 stack.
343 // We get called to create new OS threads too, and those
344 // come in on the m->g0 stack already. Or we might already
345 // be on the m->gsignal stack.
346 MOV g_m(g), X6
347 MOV m_gsignal(X6), X7
348 BEQ X7, g, g0
349 MOV m_g0(X6), X7
350 BEQ X7, g, g0
351
352 CALL gosave_systemstack_switch<>(SB)
353 MOV X7, g
354 CALL runtime·save_g(SB)
355 MOV (g_sched+gobuf_sp)(g), X2
356
357 // Now on a scheduling stack (a pthread-created stack).
358 g0:
359 // Save room for two of our pointers.
360 SUB $16, X2
361 MOV X9, 0(X2) // save old g on stack
362 MOV (g_stack+stack_hi)(X9), X9
363 SUB X8, X9, X8
364 MOV X8, 8(X2) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
365
366 JALR RA, (X5)
367
368 // Restore g, stack pointer. X10 is return value.
369 MOV 0(X2), g
370 CALL runtime·save_g(SB)
371 MOV (g_stack+stack_hi)(g), X5
372 MOV 8(X2), X6
373 SUB X6, X5, X6
374 MOV X6, X2
375
376 MOVW X10, ret+16(FP)
377 RET
378
379 // func asminit()
380 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
381 RET
382
383 // reflectcall: call a function with the given argument list
384 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
385 // we don't have variable-sized frames, so we use a small number
386 // of constant-sized-frame functions to encode a few bits of size in the pc.
387 // Caution: ugly multiline assembly macros in your future!
388
389 #define DISPATCH(NAME,MAXSIZE) \
390 MOV $MAXSIZE, T1 \
391 BLTU T1, T0, 3(PC) \
392 MOV $NAME(SB), T2; \
393 JALR ZERO, T2
394 // Note: can't just "BR NAME(SB)" - bad inlining results.
395
396 // func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
397 TEXT reflect·call(SB), NOSPLIT, $0-0
398 JMP ·reflectcall(SB)
399
400 // func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
401 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
402 MOVWU frameSize+32(FP), T0
403 DISPATCH(runtime·call16, 16)
404 DISPATCH(runtime·call32, 32)
405 DISPATCH(runtime·call64, 64)
406 DISPATCH(runtime·call128, 128)
407 DISPATCH(runtime·call256, 256)
408 DISPATCH(runtime·call512, 512)
409 DISPATCH(runtime·call1024, 1024)
410 DISPATCH(runtime·call2048, 2048)
411 DISPATCH(runtime·call4096, 4096)
412 DISPATCH(runtime·call8192, 8192)
413 DISPATCH(runtime·call16384, 16384)
414 DISPATCH(runtime·call32768, 32768)
415 DISPATCH(runtime·call65536, 65536)
416 DISPATCH(runtime·call131072, 131072)
417 DISPATCH(runtime·call262144, 262144)
418 DISPATCH(runtime·call524288, 524288)
419 DISPATCH(runtime·call1048576, 1048576)
420 DISPATCH(runtime·call2097152, 2097152)
421 DISPATCH(runtime·call4194304, 4194304)
422 DISPATCH(runtime·call8388608, 8388608)
423 DISPATCH(runtime·call16777216, 16777216)
424 DISPATCH(runtime·call33554432, 33554432)
425 DISPATCH(runtime·call67108864, 67108864)
426 DISPATCH(runtime·call134217728, 134217728)
427 DISPATCH(runtime·call268435456, 268435456)
428 DISPATCH(runtime·call536870912, 536870912)
429 DISPATCH(runtime·call1073741824, 1073741824)
430 MOV $runtime·badreflectcall(SB), T2
431 JALR ZERO, T2
432
433 #define CALLFN(NAME,MAXSIZE) \
434 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
435 NO_LOCAL_POINTERS; \
436 /* copy arguments to stack */ \
437 MOV stackArgs+16(FP), A1; \
438 MOVWU stackArgsSize+24(FP), A2; \
439 MOV X2, A3; \
440 ADD $8, A3; \
441 ADD A3, A2; \
442 BEQ A3, A2, 6(PC); \
443 MOVBU (A1), A4; \
444 ADD $1, A1; \
445 MOVB A4, (A3); \
446 ADD $1, A3; \
447 JMP -5(PC); \
448 /* set up argument registers */ \
449 MOV regArgs+40(FP), X25; \
450 CALL ·unspillArgs(SB); \
451 /* call function */ \
452 MOV f+8(FP), CTXT; \
453 MOV (CTXT), X25; \
454 PCDATA $PCDATA_StackMapIndex, $0; \
455 JALR RA, X25; \
456 /* copy return values back */ \
457 MOV regArgs+40(FP), X25; \
458 CALL ·spillArgs(SB); \
459 MOV stackArgsType+0(FP), A5; \
460 MOV stackArgs+16(FP), A1; \
461 MOVWU stackArgsSize+24(FP), A2; \
462 MOVWU stackRetOffset+28(FP), A4; \
463 ADD $8, X2, A3; \
464 ADD A4, A3; \
465 ADD A4, A1; \
466 SUB A4, A2; \
467 CALL callRet<>(SB); \
468 RET
469
470 // callRet copies return values back at the end of call*. This is a
471 // separate function so it can allocate stack space for the arguments
472 // to reflectcallmove. It does not follow the Go ABI; it expects its
473 // arguments in registers.
474 TEXT callRet<>(SB), NOSPLIT, $40-0
475 NO_LOCAL_POINTERS
476 MOV A5, 8(X2)
477 MOV A1, 16(X2)
478 MOV A3, 24(X2)
479 MOV A2, 32(X2)
480 MOV X25, 40(X2)
481 CALL runtime·reflectcallmove(SB)
482 RET
483
484 CALLFN(·call16, 16)
485 CALLFN(·call32, 32)
486 CALLFN(·call64, 64)
487 CALLFN(·call128, 128)
488 CALLFN(·call256, 256)
489 CALLFN(·call512, 512)
490 CALLFN(·call1024, 1024)
491 CALLFN(·call2048, 2048)
492 CALLFN(·call4096, 4096)
493 CALLFN(·call8192, 8192)
494 CALLFN(·call16384, 16384)
495 CALLFN(·call32768, 32768)
496 CALLFN(·call65536, 65536)
497 CALLFN(·call131072, 131072)
498 CALLFN(·call262144, 262144)
499 CALLFN(·call524288, 524288)
500 CALLFN(·call1048576, 1048576)
501 CALLFN(·call2097152, 2097152)
502 CALLFN(·call4194304, 4194304)
503 CALLFN(·call8388608, 8388608)
504 CALLFN(·call16777216, 16777216)
505 CALLFN(·call33554432, 33554432)
506 CALLFN(·call67108864, 67108864)
507 CALLFN(·call134217728, 134217728)
508 CALLFN(·call268435456, 268435456)
509 CALLFN(·call536870912, 536870912)
510 CALLFN(·call1073741824, 1073741824)
511
512 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
513 // Must obey the gcc calling convention.
514 TEXT _cgo_topofstack(SB),NOSPLIT,$8
515 // g (X27) and REG_TMP (X31) might be clobbered by load_g.
516 // X27 is callee-save in the gcc calling convention, so save it.
517 MOV g, savedX27-8(SP)
518
519 CALL runtime·load_g(SB)
520 MOV g_m(g), X5
521 MOV m_curg(X5), X5
522 MOV (g_stack+stack_hi)(X5), X10 // return value in X10
523
524 MOV savedX27-8(SP), g
525 RET
526
527 // func goexit(neverCallThisFunction)
528 // The top-most function running on a goroutine
529 // returns to goexit+PCQuantum.
530 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
531 MOV ZERO, ZERO // NOP
532 JMP runtime·goexit1(SB) // does not return
533 // traceback from goexit1 must hit code range of goexit
534 MOV ZERO, ZERO // NOP
535
536
537 // This is called from .init_array and follows the platform, not the Go ABI.
538 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
539 // Use X31 as it is a scratch register in both the Go ABI and psABI.
540 MOV runtime·lastmoduledatap(SB), X31
541 MOV X10, moduledata_next(X31)
542 MOV X10, runtime·lastmoduledatap(SB)
543 RET
544
545 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
546 // See cgocall.go for more details.
547 TEXT ·cgocallback(SB),NOSPLIT,$24-24
548 NO_LOCAL_POINTERS
549
550 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
551 // It is used to dropm while thread is exiting.
552 MOV fn+0(FP), X7
553 BNE ZERO, X7, loadg
554 // Restore the g from frame.
555 MOV frame+8(FP), g
556 JMP dropm
557
558 loadg:
559 // Load m and g from thread-local storage.
560 MOVBU runtime·iscgo(SB), X5
561 BEQ ZERO, X5, nocgo
562 CALL runtime·load_g(SB)
563 nocgo:
564
565 // If g is nil, Go did not create the current thread,
566 // or if this thread never called into Go on pthread platforms.
567 // Call needm to obtain one for temporary use.
568 // In this case, we're running on the thread stack, so there's
569 // lots of space, but the linker doesn't know. Hide the call from
570 // the linker analysis by using an indirect call.
571 BEQ ZERO, g, needm
572
573 MOV g_m(g), X5
574 MOV X5, savedm-8(SP)
575 JMP havem
576
577 needm:
578 MOV g, savedm-8(SP) // g is zero, so is m.
579 MOV $runtime·needAndBindM(SB), X6
580 JALR RA, X6
581
582 // Set m->sched.sp = SP, so that if a panic happens
583 // during the function we are about to execute, it will
584 // have a valid SP to run on the g0 stack.
585 // The next few lines (after the havem label)
586 // will save this SP onto the stack and then write
587 // the same SP back to m->sched.sp. That seems redundant,
588 // but if an unrecovered panic happens, unwindm will
589 // restore the g->sched.sp from the stack location
590 // and then systemstack will try to use it. If we don't set it here,
591 // that restored SP will be uninitialized (typically 0) and
592 // will not be usable.
593 MOV g_m(g), X5
594 MOV m_g0(X5), X6
595 MOV X2, (g_sched+gobuf_sp)(X6)
596
597 havem:
598 // Now there's a valid m, and we're running on its m->g0.
599 // Save current m->g0->sched.sp on stack and then set it to SP.
600 // Save current sp in m->g0->sched.sp in preparation for
601 // switch back to m->curg stack.
602 // NOTE: unwindm knows that the saved g->sched.sp is at 8(X2) aka savedsp-24(SP).
603 MOV m_g0(X5), X6
604 MOV (g_sched+gobuf_sp)(X6), X7
605 MOV X7, savedsp-24(SP) // must match frame size
606 MOV X2, (g_sched+gobuf_sp)(X6)
607
608 // Switch to m->curg stack and call runtime.cgocallbackg.
609 // Because we are taking over the execution of m->curg
610 // but *not* resuming what had been running, we need to
611 // save that information (m->curg->sched) so we can restore it.
612 // We can restore m->curg->sched.sp easily, because calling
613 // runtime.cgocallbackg leaves SP unchanged upon return.
614 // To save m->curg->sched.pc, we push it onto the curg stack and
615 // open a frame the same size as cgocallback's g0 frame.
616 // Once we switch to the curg stack, the pushed PC will appear
617 // to be the return PC of cgocallback, so that the traceback
618 // will seamlessly trace back into the earlier calls.
619 MOV m_curg(X5), g
620 CALL runtime·save_g(SB)
621 MOV (g_sched+gobuf_sp)(g), X6 // prepare stack as X6
622 MOV (g_sched+gobuf_pc)(g), X7
623 MOV X7, -(24+8)(X6) // "saved LR"; must match frame size
624 // Gather our arguments into registers.
625 MOV fn+0(FP), X7
626 MOV frame+8(FP), X8
627 MOV ctxt+16(FP), X9
628 MOV $-(24+8)(X6), X2 // switch stack; must match frame size
629 MOV X7, 8(X2)
630 MOV X8, 16(X2)
631 MOV X9, 24(X2)
632 CALL runtime·cgocallbackg(SB)
633
634 // Restore g->sched (== m->curg->sched) from saved values.
635 MOV 0(X2), X7
636 MOV X7, (g_sched+gobuf_pc)(g)
637 MOV $(24+8)(X2), X6 // must match frame size
638 MOV X6, (g_sched+gobuf_sp)(g)
639
640 // Switch back to m->g0's stack and restore m->g0->sched.sp.
641 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
642 // so we do not have to restore it.)
643 MOV g_m(g), X5
644 MOV m_g0(X5), g
645 CALL runtime·save_g(SB)
646 MOV (g_sched+gobuf_sp)(g), X2
647 MOV savedsp-24(SP), X6 // must match frame size
648 MOV X6, (g_sched+gobuf_sp)(g)
649
650 // If the m on entry was nil, we called needm above to borrow an m,
651 // 1. for the duration of the call on non-pthread platforms,
652 // 2. or the duration of the C thread alive on pthread platforms.
653 // If the m on entry wasn't nil,
654 // 1. the thread might be a Go thread,
655 // 2. or it wasn't the first call from a C thread on pthread platforms,
656 // since then we skip dropm to reuse the m in the first call.
657 MOV savedm-8(SP), X5
658 BNE ZERO, X5, droppedm
659
660 // Skip dropm to reuse it in the next call, when a pthread key has been created.
661 MOV _cgo_pthread_key_created(SB), X5
662 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
663 BEQ ZERO, X5, dropm
664 MOV (X5), X5
665 BNE ZERO, X5, droppedm
666
667 dropm:
668 MOV $runtime·dropm(SB), X6
669 JALR RA, X6
670 droppedm:
671
672 // Done!
673 RET
674
675 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
676 EBREAK
677 RET
678
679 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
680 EBREAK
681 RET
682
683 // void setg(G*); set g. for use by needm.
684 TEXT runtime·setg(SB), NOSPLIT, $0-8
685 MOV gg+0(FP), g
686 // This only happens if iscgo, so jump straight to save_g
687 CALL runtime·save_g(SB)
688 RET
689
690 TEXT ·checkASM(SB),NOSPLIT,$0-1
691 MOV $1, T0
692 MOV T0, ret+0(FP)
693 RET
694
695 // spillArgs stores return values from registers to a *internal/abi.RegArgs in X25.
696 TEXT ·spillArgs(SB),NOSPLIT,$0-0
697 MOV X10, (0*8)(X25)
698 MOV X11, (1*8)(X25)
699 MOV X12, (2*8)(X25)
700 MOV X13, (3*8)(X25)
701 MOV X14, (4*8)(X25)
702 MOV X15, (5*8)(X25)
703 MOV X16, (6*8)(X25)
704 MOV X17, (7*8)(X25)
705 MOV X8, (8*8)(X25)
706 MOV X9, (9*8)(X25)
707 MOV X18, (10*8)(X25)
708 MOV X19, (11*8)(X25)
709 MOV X20, (12*8)(X25)
710 MOV X21, (13*8)(X25)
711 MOV X22, (14*8)(X25)
712 MOV X23, (15*8)(X25)
713 MOVD F10, (16*8)(X25)
714 MOVD F11, (17*8)(X25)
715 MOVD F12, (18*8)(X25)
716 MOVD F13, (19*8)(X25)
717 MOVD F14, (20*8)(X25)
718 MOVD F15, (21*8)(X25)
719 MOVD F16, (22*8)(X25)
720 MOVD F17, (23*8)(X25)
721 MOVD F8, (24*8)(X25)
722 MOVD F9, (25*8)(X25)
723 MOVD F18, (26*8)(X25)
724 MOVD F19, (27*8)(X25)
725 MOVD F20, (28*8)(X25)
726 MOVD F21, (29*8)(X25)
727 MOVD F22, (30*8)(X25)
728 MOVD F23, (31*8)(X25)
729 RET
730
731 // unspillArgs loads args into registers from a *internal/abi.RegArgs in X25.
732 TEXT ·unspillArgs(SB),NOSPLIT,$0-0
733 MOV (0*8)(X25), X10
734 MOV (1*8)(X25), X11
735 MOV (2*8)(X25), X12
736 MOV (3*8)(X25), X13
737 MOV (4*8)(X25), X14
738 MOV (5*8)(X25), X15
739 MOV (6*8)(X25), X16
740 MOV (7*8)(X25), X17
741 MOV (8*8)(X25), X8
742 MOV (9*8)(X25), X9
743 MOV (10*8)(X25), X18
744 MOV (11*8)(X25), X19
745 MOV (12*8)(X25), X20
746 MOV (13*8)(X25), X21
747 MOV (14*8)(X25), X22
748 MOV (15*8)(X25), X23
749 MOVD (16*8)(X25), F10
750 MOVD (17*8)(X25), F11
751 MOVD (18*8)(X25), F12
752 MOVD (19*8)(X25), F13
753 MOVD (20*8)(X25), F14
754 MOVD (21*8)(X25), F15
755 MOVD (22*8)(X25), F16
756 MOVD (23*8)(X25), F17
757 MOVD (24*8)(X25), F8
758 MOVD (25*8)(X25), F9
759 MOVD (26*8)(X25), F18
760 MOVD (27*8)(X25), F19
761 MOVD (28*8)(X25), F20
762 MOVD (29*8)(X25), F21
763 MOVD (30*8)(X25), F22
764 MOVD (31*8)(X25), F23
765 RET
766
767 // gcWriteBarrier informs the GC about heap pointer writes.
768 //
769 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
770 // number of bytes of buffer needed in X24, and returns a pointer
771 // to the buffer space in X24.
772 // It clobbers X31 aka T6 (the linker temp register - REG_TMP).
773 // The act of CALLing gcWriteBarrier will clobber RA (LR).
774 // It does not clobber any other general-purpose registers,
775 // but may clobber others (e.g., floating point registers).
776 TEXT gcWriteBarrier<>(SB),NOSPLIT,$208
777 // Save the registers clobbered by the fast path.
778 MOV A0, 24*8(X2)
779 MOV A1, 25*8(X2)
780 retry:
781 MOV g_m(g), A0
782 MOV m_p(A0), A0
783 MOV (p_wbBuf+wbBuf_next)(A0), A1
784 MOV (p_wbBuf+wbBuf_end)(A0), T6 // T6 is linker temp register (REG_TMP)
785 // Increment wbBuf.next position.
786 ADD X24, A1
787 // Is the buffer full?
788 BLTU T6, A1, flush
789 // Commit to the larger buffer.
790 MOV A1, (p_wbBuf+wbBuf_next)(A0)
791 // Make the return value (the original next position)
792 SUB X24, A1, X24
793 // Restore registers.
794 MOV 24*8(X2), A0
795 MOV 25*8(X2), A1
796 RET
797
798 flush:
799 // Save all general purpose registers since these could be
800 // clobbered by wbBufFlush and were not saved by the caller.
801 MOV T0, 1*8(X2)
802 MOV T1, 2*8(X2)
803 // X0 is zero register
804 // X1 is LR, saved by prologue
805 // X2 is SP
806 // X3 is GP
807 // X4 is TP
808 MOV X7, 3*8(X2)
809 MOV X8, 4*8(X2)
810 MOV X9, 5*8(X2)
811 // X10 already saved (A0)
812 // X11 already saved (A1)
813 MOV X12, 6*8(X2)
814 MOV X13, 7*8(X2)
815 MOV X14, 8*8(X2)
816 MOV X15, 9*8(X2)
817 MOV X16, 10*8(X2)
818 MOV X17, 11*8(X2)
819 MOV X18, 12*8(X2)
820 MOV X19, 13*8(X2)
821 MOV X20, 14*8(X2)
822 MOV X21, 15*8(X2)
823 MOV X22, 16*8(X2)
824 MOV X23, 17*8(X2)
825 MOV X24, 18*8(X2)
826 MOV X25, 19*8(X2)
827 MOV X26, 20*8(X2)
828 // X27 is g.
829 MOV X28, 21*8(X2)
830 MOV X29, 22*8(X2)
831 MOV X30, 23*8(X2)
832 // X31 is tmp register.
833
834 CALL runtime·wbBufFlush(SB)
835
836 MOV 1*8(X2), T0
837 MOV 2*8(X2), T1
838 MOV 3*8(X2), X7
839 MOV 4*8(X2), X8
840 MOV 5*8(X2), X9
841 MOV 6*8(X2), X12
842 MOV 7*8(X2), X13
843 MOV 8*8(X2), X14
844 MOV 9*8(X2), X15
845 MOV 10*8(X2), X16
846 MOV 11*8(X2), X17
847 MOV 12*8(X2), X18
848 MOV 13*8(X2), X19
849 MOV 14*8(X2), X20
850 MOV 15*8(X2), X21
851 MOV 16*8(X2), X22
852 MOV 17*8(X2), X23
853 MOV 18*8(X2), X24
854 MOV 19*8(X2), X25
855 MOV 20*8(X2), X26
856 MOV 21*8(X2), X28
857 MOV 22*8(X2), X29
858 MOV 23*8(X2), X30
859
860 JMP retry
861
862 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
863 MOV $8, X24
864 JMP gcWriteBarrier<>(SB)
865 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
866 MOV $16, X24
867 JMP gcWriteBarrier<>(SB)
868 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
869 MOV $24, X24
870 JMP gcWriteBarrier<>(SB)
871 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
872 MOV $32, X24
873 JMP gcWriteBarrier<>(SB)
874 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
875 MOV $40, X24
876 JMP gcWriteBarrier<>(SB)
877 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
878 MOV $48, X24
879 JMP gcWriteBarrier<>(SB)
880 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
881 MOV $56, X24
882 JMP gcWriteBarrier<>(SB)
883 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
884 MOV $64, X24
885 JMP gcWriteBarrier<>(SB)
886
887 // Note: these functions use a special calling convention to save generated code space.
888 // Arguments are passed in registers (ssa/gen/RISCV64Ops.go), but the space for those
889 // arguments are allocated in the caller's stack frame.
890 // These stubs write the args into that stack space and then tail call to the
891 // corresponding runtime handler.
892 // The tail call makes these stubs disappear in backtraces.
893 TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
894 MOV T0, X10
895 MOV T1, X11
896 JMP runtime·goPanicIndex<ABIInternal>(SB)
897 TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
898 MOV T0, X10
899 MOV T1, X11
900 JMP runtime·goPanicIndexU<ABIInternal>(SB)
901 TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
902 MOV T1, X10
903 MOV T2, X11
904 JMP runtime·goPanicSliceAlen<ABIInternal>(SB)
905 TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
906 MOV T1, X10
907 MOV T2, X11
908 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB)
909 TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
910 MOV T1, X10
911 MOV T2, X11
912 JMP runtime·goPanicSliceAcap<ABIInternal>(SB)
913 TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
914 MOV T1, X10
915 MOV T2, X11
916 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB)
917 TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
918 MOV T0, X10
919 MOV T1, X11
920 JMP runtime·goPanicSliceB<ABIInternal>(SB)
921 TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
922 MOV T0, X10
923 MOV T1, X11
924 JMP runtime·goPanicSliceBU<ABIInternal>(SB)
925 TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
926 MOV T2, X10
927 MOV T3, X11
928 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB)
929 TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
930 MOV T2, X10
931 MOV T3, X11
932 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB)
933 TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
934 MOV T2, X10
935 MOV T3, X11
936 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB)
937 TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
938 MOV T2, X10
939 MOV T3, X11
940 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB)
941 TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
942 MOV T1, X10
943 MOV T2, X11
944 JMP runtime·goPanicSlice3B<ABIInternal>(SB)
945 TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
946 MOV T1, X10
947 MOV T2, X11
948 JMP runtime·goPanicSlice3BU<ABIInternal>(SB)
949 TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
950 MOV T0, X10
951 MOV T1, X11
952 JMP runtime·goPanicSlice3C<ABIInternal>(SB)
953 TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
954 MOV T0, X10
955 MOV T1, X11
956 JMP runtime·goPanicSlice3CU<ABIInternal>(SB)
957 TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
958 MOV T2, X10
959 MOV T3, X11
960 JMP runtime·goPanicSliceConvert<ABIInternal>(SB)
961
962 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
963 GLOBL runtime·mainPC(SB),RODATA,$8
964
View as plain text