Text file
src/runtime/asm_ppc64x.s
1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build ppc64 || ppc64le
6
7 #include "go_asm.h"
8 #include "go_tls.h"
9 #include "funcdata.h"
10 #include "textflag.h"
11 #include "asm_ppc64x.h"
12
13 #ifdef GOOS_aix
14 #define cgoCalleeStackSize 48
15 #else
16 #define cgoCalleeStackSize 32
17 #endif
18
19 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
20 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer
21
22 // initialize essential registers
23 BL runtime·reginit(SB)
24
25 SUB $(FIXED_FRAME+16), R1
26 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame
27 MOVW R3, FIXED_FRAME+0(R1) // argc
28 MOVD R4, FIXED_FRAME+8(R1) // argv
29
30 // create istack out of the given (operating system) stack.
31 // _cgo_init may update stackguard.
32 MOVD $runtime·g0(SB), g
33 BL runtime·save_g(SB)
34 MOVD $(-64*1024), R31
35 ADD R31, R1, R3
36 MOVD R3, g_stackguard0(g)
37 MOVD R3, g_stackguard1(g)
38 MOVD R3, (g_stack+stack_lo)(g)
39 MOVD R1, (g_stack+stack_hi)(g)
40
41 // If there is a _cgo_init, call it using the gcc ABI.
42 MOVD _cgo_init(SB), R12
43 CMP R12, $0
44 BEQ nocgo
45
46 #ifdef GO_PPC64X_HAS_FUNCDESC
47 // Load the real entry address from the first slot of the function descriptor.
48 MOVD 8(R12), R2
49 MOVD (R12), R12
50 #endif
51 MOVD R12, CTR // r12 = "global function entry point"
52 MOVD R13, R5 // arg 2: TLS base pointer
53 MOVD $setg_gcc<>(SB), R4 // arg 1: setg
54 MOVD g, R3 // arg 0: G
55 // C functions expect 32 (48 for AIX) bytes of space on caller
56 // stack frame and a 16-byte aligned R1
57 MOVD R1, R14 // save current stack
58 SUB $cgoCalleeStackSize, R1 // reserve the callee area
59 RLDCR $0, R1, $~15, R1 // 16-byte align
60 BL (CTR) // may clobber R0, R3-R12
61 MOVD R14, R1 // restore stack
62 #ifndef GOOS_aix
63 MOVD 24(R1), R2
64 #endif
65 XOR R0, R0 // fix R0
66
67 nocgo:
68 // update stackguard after _cgo_init
69 MOVD (g_stack+stack_lo)(g), R3
70 ADD $const_stackGuard, R3
71 MOVD R3, g_stackguard0(g)
72 MOVD R3, g_stackguard1(g)
73
74 // set the per-goroutine and per-mach "registers"
75 MOVD $runtime·m0(SB), R3
76
77 // save m->g0 = g0
78 MOVD g, m_g0(R3)
79 // save m0 to g0->m
80 MOVD R3, g_m(g)
81
82 BL runtime·check(SB)
83
84 // args are already prepared
85 BL runtime·args(SB)
86 BL runtime·osinit(SB)
87 BL runtime·schedinit(SB)
88
89 // create a new goroutine to start program
90 MOVD $runtime·mainPC(SB), R3 // entry
91 MOVDU R3, -8(R1)
92 MOVDU R0, -8(R1)
93 MOVDU R0, -8(R1)
94 MOVDU R0, -8(R1)
95 MOVDU R0, -8(R1)
96 BL runtime·newproc(SB)
97 ADD $(8+FIXED_FRAME), R1
98
99 // start this M
100 BL runtime·mstart(SB)
101 // Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are
102 // intended to be called by debuggers.
103 #ifdef GOARCH_ppc64le
104 MOVD $runtime·debugPinnerV1<ABIInternal>(SB), R31
105 MOVD $runtime·debugCallV2<ABIInternal>(SB), R31
106 #endif
107 MOVD R0, 0(R0)
108 RET
109
110 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
111 GLOBL runtime·mainPC(SB),RODATA,$8
112
113 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
114 TW $31, R0, R0
115 RET
116
117 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
118 RET
119
120 // Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall_ppc64
121 TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0
122 // crosscall_ppc64 and crosscall2 need to reginit, but can't
123 // get at the 'runtime.reginit' symbol.
124 BR runtime·reginit(SB)
125
126 TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0
127 // set R0 to zero, it's expected by the toolchain
128 XOR R0, R0
129 RET
130
131 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
132 BL runtime·mstart0(SB)
133 RET // not reached
134
135 /*
136 * go-routine
137 */
138
139 // void gogo(Gobuf*)
140 // restore state from Gobuf; longjmp
141 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
142 MOVD buf+0(FP), R5
143 MOVD gobuf_g(R5), R6
144 MOVD 0(R6), R4 // make sure g != nil
145 BR gogo<>(SB)
146
147 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
148 MOVD R6, g
149 BL runtime·save_g(SB)
150
151 MOVD gobuf_sp(R5), R1
152 MOVD gobuf_lr(R5), R31
153 #ifndef GOOS_aix
154 MOVD 24(R1), R2 // restore R2
155 #endif
156 MOVD R31, LR
157 MOVD gobuf_ctxt(R5), R11
158 MOVD R0, gobuf_sp(R5)
159 MOVD R0, gobuf_lr(R5)
160 MOVD R0, gobuf_ctxt(R5)
161 CMP R0, R0 // set condition codes for == test, needed by stack split
162 MOVD gobuf_pc(R5), R12
163 MOVD R12, CTR
164 BR (CTR)
165
166 // void mcall(fn func(*g))
167 // Switch to m->g0's stack, call fn(g).
168 // Fn must never return. It should gogo(&g->sched)
169 // to keep running g.
170 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
171 // Save caller state in g->sched
172 // R11 should be safe across save_g??
173 MOVD R3, R11
174 MOVD R1, (g_sched+gobuf_sp)(g)
175 MOVD LR, R31
176 MOVD R31, (g_sched+gobuf_pc)(g)
177 MOVD R0, (g_sched+gobuf_lr)(g)
178
179 // Switch to m->g0 & its stack, call fn.
180 MOVD g, R3
181 MOVD g_m(g), R8
182 MOVD m_g0(R8), g
183 BL runtime·save_g(SB)
184 CMP g, R3
185 BNE 2(PC)
186 BR runtime·badmcall(SB)
187 MOVD 0(R11), R12 // code pointer
188 MOVD R12, CTR
189 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp
190 // Don't need to do anything special for regabiargs here
191 // R3 is g; stack is set anyway
192 MOVDU R3, -8(R1)
193 MOVDU R0, -8(R1)
194 MOVDU R0, -8(R1)
195 MOVDU R0, -8(R1)
196 MOVDU R0, -8(R1)
197 BL (CTR)
198 MOVD 24(R1), R2
199 BR runtime·badmcall2(SB)
200
201 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
202 // of the G stack. We need to distinguish the routine that
203 // lives at the bottom of the G stack from the one that lives
204 // at the top of the system stack because the one at the top of
205 // the system stack terminates the stack walk (see topofstack()).
206 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
207 // We have several undefs here so that 16 bytes past
208 // $runtime·systemstack_switch lies within them whether or not the
209 // instructions that derive r2 from r12 are there.
210 UNDEF
211 UNDEF
212 UNDEF
213 BL (LR) // make sure this function is not leaf
214 RET
215
216 // func systemstack(fn func())
217 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
218 MOVD fn+0(FP), R3 // R3 = fn
219 MOVD R3, R11 // context
220 MOVD g_m(g), R4 // R4 = m
221
222 MOVD m_gsignal(R4), R5 // R5 = gsignal
223 CMP g, R5
224 BEQ noswitch
225
226 MOVD m_g0(R4), R5 // R5 = g0
227 CMP g, R5
228 BEQ noswitch
229
230 MOVD m_curg(R4), R6
231 CMP g, R6
232 BEQ switch
233
234 // Bad: g is not gsignal, not g0, not curg. What is it?
235 // Hide call from linker nosplit analysis.
236 MOVD $runtime·badsystemstack(SB), R12
237 MOVD R12, CTR
238 BL (CTR)
239 BL runtime·abort(SB)
240
241 switch:
242 // save our state in g->sched. Pretend to
243 // be systemstack_switch if the G stack is scanned.
244 BL gosave_systemstack_switch<>(SB)
245
246 // switch to g0
247 MOVD R5, g
248 BL runtime·save_g(SB)
249 MOVD (g_sched+gobuf_sp)(g), R1
250
251 // call target function
252 MOVD 0(R11), R12 // code pointer
253 MOVD R12, CTR
254 BL (CTR)
255
256 // restore TOC pointer. It seems unlikely that we will use systemstack
257 // to call a function defined in another module, but the results of
258 // doing so would be so confusing that it's worth doing this.
259 MOVD g_m(g), R3
260 MOVD m_curg(R3), g
261 MOVD (g_sched+gobuf_sp)(g), R3
262 #ifndef GOOS_aix
263 MOVD 24(R3), R2
264 #endif
265 // switch back to g
266 MOVD g_m(g), R3
267 MOVD m_curg(R3), g
268 BL runtime·save_g(SB)
269 MOVD (g_sched+gobuf_sp)(g), R1
270 MOVD R0, (g_sched+gobuf_sp)(g)
271 RET
272
273 noswitch:
274 // already on m stack, just call directly
275 // On other arches we do a tail call here, but it appears to be
276 // impossible to tail call a function pointer in shared mode on
277 // ppc64 because the caller is responsible for restoring the TOC.
278 MOVD 0(R11), R12 // code pointer
279 MOVD R12, CTR
280 BL (CTR)
281 #ifndef GOOS_aix
282 MOVD 24(R1), R2
283 #endif
284 RET
285
286 // func switchToCrashStack0(fn func())
287 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
288 MOVD R3, R11 // context register
289 MOVD g_m(g), R3 // curm
290
291 // set g to gcrash
292 MOVD $runtime·gcrash(SB), g // g = &gcrash
293 CALL runtime·save_g(SB) // clobbers R31
294 MOVD R3, g_m(g) // g.m = curm
295 MOVD g, m_g0(R3) // curm.g0 = g
296
297 // switch to crashstack
298 MOVD (g_stack+stack_hi)(g), R3
299 SUB $(4*8), R3
300 MOVD R3, R1
301
302 // call target function
303 MOVD 0(R11), R12 // code pointer
304 MOVD R12, CTR
305 BL (CTR)
306
307 // should never return
308 CALL runtime·abort(SB)
309 UNDEF
310
311 /*
312 * support for morestack
313 */
314
315 // Called during function prolog when more stack is needed.
316 // Caller has already loaded:
317 // R3: framesize, R4: argsize, R5: LR
318 //
319 // The traceback routines see morestack on a g0 as being
320 // the top of a stack (for example, morestack calling newstack
321 // calling the scheduler calling newm calling gc), so we must
322 // record an argument size. For that purpose, it has no arguments.
323 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
324 // Called from f.
325 // Set g->sched to context in f.
326 MOVD R1, (g_sched+gobuf_sp)(g)
327 MOVD LR, R8
328 MOVD R8, (g_sched+gobuf_pc)(g)
329 MOVD R5, (g_sched+gobuf_lr)(g)
330 MOVD R11, (g_sched+gobuf_ctxt)(g)
331
332 // Cannot grow scheduler stack (m->g0).
333 MOVD g_m(g), R7
334 MOVD m_g0(R7), R8
335 CMP g, R8
336 BNE 3(PC)
337 BL runtime·badmorestackg0(SB)
338 BL runtime·abort(SB)
339
340 // Cannot grow signal stack (m->gsignal).
341 MOVD m_gsignal(R7), R8
342 CMP g, R8
343 BNE 3(PC)
344 BL runtime·badmorestackgsignal(SB)
345 BL runtime·abort(SB)
346
347 // Called from f.
348 // Set m->morebuf to f's caller.
349 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
350 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
351 MOVD g, (m_morebuf+gobuf_g)(R7)
352
353 // Call newstack on m->g0's stack.
354 MOVD m_g0(R7), g
355 BL runtime·save_g(SB)
356 MOVD (g_sched+gobuf_sp)(g), R1
357 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0
358 BL runtime·newstack(SB)
359
360 // Not reached, but make sure the return PC from the call to newstack
361 // is still in this function, and not the beginning of the next.
362 UNDEF
363
364 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
365 // Force SPWRITE. This function doesn't actually write SP,
366 // but it is called with a special calling convention where
367 // the caller doesn't save LR on stack but passes it as a
368 // register (R5), and the unwinder currently doesn't understand.
369 // Make it SPWRITE to stop unwinding. (See issue 54332)
370 // Use OR R0, R1 instead of MOVD R1, R1 as the MOVD instruction
371 // has a special affect on Power8,9,10 by lowering the thread
372 // priority and causing a slowdown in execution time
373
374 OR R0, R1
375 MOVD R0, R11
376 BR runtime·morestack(SB)
377
378 // reflectcall: call a function with the given argument list
379 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
380 // we don't have variable-sized frames, so we use a small number
381 // of constant-sized-frame functions to encode a few bits of size in the pc.
382 // Caution: ugly multiline assembly macros in your future!
383
384 #define DISPATCH(NAME,MAXSIZE) \
385 MOVD $MAXSIZE, R31; \
386 CMP R3, R31; \
387 BGT 4(PC); \
388 MOVD $NAME(SB), R12; \
389 MOVD R12, CTR; \
390 BR (CTR)
391 // Note: can't just "BR NAME(SB)" - bad inlining results.
392
393 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
394 MOVWZ frameSize+32(FP), R3
395 DISPATCH(runtime·call16, 16)
396 DISPATCH(runtime·call32, 32)
397 DISPATCH(runtime·call64, 64)
398 DISPATCH(runtime·call128, 128)
399 DISPATCH(runtime·call256, 256)
400 DISPATCH(runtime·call512, 512)
401 DISPATCH(runtime·call1024, 1024)
402 DISPATCH(runtime·call2048, 2048)
403 DISPATCH(runtime·call4096, 4096)
404 DISPATCH(runtime·call8192, 8192)
405 DISPATCH(runtime·call16384, 16384)
406 DISPATCH(runtime·call32768, 32768)
407 DISPATCH(runtime·call65536, 65536)
408 DISPATCH(runtime·call131072, 131072)
409 DISPATCH(runtime·call262144, 262144)
410 DISPATCH(runtime·call524288, 524288)
411 DISPATCH(runtime·call1048576, 1048576)
412 DISPATCH(runtime·call2097152, 2097152)
413 DISPATCH(runtime·call4194304, 4194304)
414 DISPATCH(runtime·call8388608, 8388608)
415 DISPATCH(runtime·call16777216, 16777216)
416 DISPATCH(runtime·call33554432, 33554432)
417 DISPATCH(runtime·call67108864, 67108864)
418 DISPATCH(runtime·call134217728, 134217728)
419 DISPATCH(runtime·call268435456, 268435456)
420 DISPATCH(runtime·call536870912, 536870912)
421 DISPATCH(runtime·call1073741824, 1073741824)
422 MOVD $runtime·badreflectcall(SB), R12
423 MOVD R12, CTR
424 BR (CTR)
425
426 #define CALLFN(NAME,MAXSIZE) \
427 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
428 NO_LOCAL_POINTERS; \
429 /* copy arguments to stack */ \
430 MOVD stackArgs+16(FP), R3; \
431 MOVWZ stackArgsSize+24(FP), R4; \
432 MOVD R1, R5; \
433 CMP R4, $8; \
434 BLT tailsetup; \
435 /* copy 8 at a time if possible */ \
436 ADD $(FIXED_FRAME-8), R5; \
437 SUB $8, R3; \
438 top: \
439 MOVDU 8(R3), R7; \
440 MOVDU R7, 8(R5); \
441 SUB $8, R4; \
442 CMP R4, $8; \
443 BGE top; \
444 /* handle remaining bytes */ \
445 CMP $0, R4; \
446 BEQ callfn; \
447 ADD $7, R3; \
448 ADD $7, R5; \
449 BR tail; \
450 tailsetup: \
451 CMP $0, R4; \
452 BEQ callfn; \
453 ADD $(FIXED_FRAME-1), R5; \
454 SUB $1, R3; \
455 tail: \
456 MOVBU 1(R3), R6; \
457 MOVBU R6, 1(R5); \
458 SUB $1, R4; \
459 CMP $0, R4; \
460 BGT tail; \
461 callfn: \
462 /* call function */ \
463 MOVD f+8(FP), R11; \
464 #ifdef GOOS_aix \
465 /* AIX won't trigger a SIGSEGV if R11 = nil */ \
466 /* So it manually triggers it */ \
467 CMP R11, $0 \
468 BNE 2(PC) \
469 MOVD R0, 0(R0) \
470 #endif \
471 MOVD regArgs+40(FP), R20; \
472 BL runtime·unspillArgs(SB); \
473 MOVD (R11), R12; \
474 MOVD R12, CTR; \
475 PCDATA $PCDATA_StackMapIndex, $0; \
476 BL (CTR); \
477 #ifndef GOOS_aix \
478 MOVD 24(R1), R2; \
479 #endif \
480 /* copy return values back */ \
481 MOVD regArgs+40(FP), R20; \
482 BL runtime·spillArgs(SB); \
483 MOVD stackArgsType+0(FP), R7; \
484 MOVD stackArgs+16(FP), R3; \
485 MOVWZ stackArgsSize+24(FP), R4; \
486 MOVWZ stackRetOffset+28(FP), R6; \
487 ADD $FIXED_FRAME, R1, R5; \
488 ADD R6, R5; \
489 ADD R6, R3; \
490 SUB R6, R4; \
491 BL callRet<>(SB); \
492 RET
493
494 // callRet copies return values back at the end of call*. This is a
495 // separate function so it can allocate stack space for the arguments
496 // to reflectcallmove. It does not follow the Go ABI; it expects its
497 // arguments in registers.
498 TEXT callRet<>(SB), NOSPLIT, $40-0
499 NO_LOCAL_POINTERS
500 MOVD R7, FIXED_FRAME+0(R1)
501 MOVD R3, FIXED_FRAME+8(R1)
502 MOVD R5, FIXED_FRAME+16(R1)
503 MOVD R4, FIXED_FRAME+24(R1)
504 MOVD R20, FIXED_FRAME+32(R1)
505 BL runtime·reflectcallmove(SB)
506 RET
507
508 CALLFN(·call16, 16)
509 CALLFN(·call32, 32)
510 CALLFN(·call64, 64)
511 CALLFN(·call128, 128)
512 CALLFN(·call256, 256)
513 CALLFN(·call512, 512)
514 CALLFN(·call1024, 1024)
515 CALLFN(·call2048, 2048)
516 CALLFN(·call4096, 4096)
517 CALLFN(·call8192, 8192)
518 CALLFN(·call16384, 16384)
519 CALLFN(·call32768, 32768)
520 CALLFN(·call65536, 65536)
521 CALLFN(·call131072, 131072)
522 CALLFN(·call262144, 262144)
523 CALLFN(·call524288, 524288)
524 CALLFN(·call1048576, 1048576)
525 CALLFN(·call2097152, 2097152)
526 CALLFN(·call4194304, 4194304)
527 CALLFN(·call8388608, 8388608)
528 CALLFN(·call16777216, 16777216)
529 CALLFN(·call33554432, 33554432)
530 CALLFN(·call67108864, 67108864)
531 CALLFN(·call134217728, 134217728)
532 CALLFN(·call268435456, 268435456)
533 CALLFN(·call536870912, 536870912)
534 CALLFN(·call1073741824, 1073741824)
535
536 TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0-4
537 MOVW cycles+0(FP), R7
538 // POWER does not have a pause/yield instruction equivalent.
539 // Instead, we can lower the program priority by setting the
540 // Program Priority Register prior to the wait loop and set it
541 // back to default afterwards. On Linux, the default priority is
542 // medium-low. For details, see page 837 of the ISA 3.0.
543 OR R1, R1, R1 // Set PPR priority to low
544 again:
545 SUB $1, R7
546 CMP $0, R7
547 BNE again
548 OR R6, R6, R6 // Set PPR priority back to medium-low
549 RET
550
551 // Save state of caller into g->sched,
552 // but using fake PC from systemstack_switch.
553 // Must only be called from functions with no locals ($0)
554 // or else unwinding from systemstack_switch is incorrect.
555 // Smashes R31.
556 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
557 MOVD $runtime·systemstack_switch(SB), R31
558 ADD $16, R31 // get past prologue (including r2-setting instructions when they're there)
559 MOVD R31, (g_sched+gobuf_pc)(g)
560 MOVD R1, (g_sched+gobuf_sp)(g)
561 MOVD R0, (g_sched+gobuf_lr)(g)
562 // Assert ctxt is zero. See func save.
563 MOVD (g_sched+gobuf_ctxt)(g), R31
564 CMP R31, $0
565 BEQ 2(PC)
566 BL runtime·abort(SB)
567 RET
568
569 #ifdef GOOS_aix
570 #define asmcgocallSaveOffset cgoCalleeStackSize + 8
571 #else
572 #define asmcgocallSaveOffset cgoCalleeStackSize
573 #endif
574
575 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
576 // Call fn(arg) aligned appropriately for the gcc ABI.
577 // Called on a system stack, and there may be no g yet (during needm).
578 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
579 MOVD fn+0(FP), R3
580 MOVD arg+8(FP), R4
581
582 MOVD R1, R15
583 SUB $(asmcgocallSaveOffset+8), R1
584 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
585 MOVD R15, asmcgocallSaveOffset(R1)
586
587 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
588
589 // This is a "global call", so put the global entry point in r12
590 MOVD R3, R12
591
592 #ifdef GO_PPC64X_HAS_FUNCDESC
593 // Load the real entry address from the first slot of the function descriptor.
594 MOVD 8(R12), R2
595 MOVD (R12), R12
596 #endif
597 MOVD R12, CTR
598 MOVD R4, R3 // arg in r3
599 BL (CTR)
600
601 // C code can clobber R0, so set it back to 0. F27-F31 are
602 // callee save, so we don't need to recover those.
603 XOR R0, R0
604
605 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
606 #ifndef GOOS_aix
607 MOVD 24(R1), R2
608 #endif
609
610 RET
611
612 // func asmcgocall(fn, arg unsafe.Pointer) int32
613 // Call fn(arg) on the scheduler stack,
614 // aligned appropriately for the gcc ABI.
615 // See cgocall.go for more details.
616 TEXT ·asmcgocall<ABIInternal>(SB),NOSPLIT,$0-20
617 // R3 = fn
618 // R4 = arg
619
620 MOVD R1, R7 // save original stack pointer
621 CMP $0, g
622 BEQ nosave
623 MOVD g, R5
624
625 // Figure out if we need to switch to m->g0 stack.
626 // We get called to create new OS threads too, and those
627 // come in on the m->g0 stack already. Or we might already
628 // be on the m->gsignal stack.
629 MOVD g_m(g), R8
630 MOVD m_gsignal(R8), R6
631 CMP R6, g
632 BEQ nosave
633 MOVD m_g0(R8), R6
634 CMP R6, g
635 BEQ nosave
636
637 BL gosave_systemstack_switch<>(SB)
638 MOVD R6, g
639 BL runtime·save_g(SB)
640 MOVD (g_sched+gobuf_sp)(g), R1
641
642 // Now on a scheduling stack (a pthread-created stack).
643 #ifdef GOOS_aix
644 // Create a fake LR to improve backtrace.
645 MOVD $runtime·asmcgocall(SB), R6
646 MOVD R6, 16(R1)
647 // AIX also saves one argument on the stack.
648 SUB $8, R1
649 #endif
650 // Save room for two of our pointers, plus the callee
651 // save area that lives on the caller stack.
652 // Do arithmetics in R10 to hide from the assembler
653 // counting it as SP delta, which is irrelevant as we are
654 // on the system stack.
655 SUB $(asmcgocallSaveOffset+16), R1, R10
656 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
657 MOVD R5, (asmcgocallSaveOffset+8)(R1) // save old g on stack
658 MOVD (g_stack+stack_hi)(R5), R5
659 SUB R7, R5
660 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
661 #ifdef GOOS_aix
662 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb
663 #else
664 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
665 #endif
666 // This is a "global call", so put the global entry point in r12
667 MOVD R3, R12
668
669 #ifdef GO_PPC64X_HAS_FUNCDESC
670 // Load the real entry address from the first slot of the function descriptor.
671 MOVD 8(R12), R2
672 MOVD (R12), R12
673 #endif
674 MOVD R12, CTR
675 MOVD R4, R3 // arg in r3
676 BL (CTR)
677
678 // Reinitialise zero value register.
679 XOR R0, R0
680
681 // Restore g, stack pointer, toc pointer.
682 // R3 is errno, so don't touch it
683 MOVD (asmcgocallSaveOffset+8)(R1), g
684 MOVD (g_stack+stack_hi)(g), R5
685 MOVD asmcgocallSaveOffset(R1), R6
686 SUB R6, R5
687 #ifndef GOOS_aix
688 MOVD 24(R5), R2
689 #endif
690 MOVD R5, R1
691 BL runtime·save_g(SB)
692
693 // ret = R3
694 RET
695
696 nosave:
697 // Running on a system stack, perhaps even without a g.
698 // Having no g can happen during thread creation or thread teardown.
699 // This code is like the above sequence but without saving/restoring g
700 // and without worrying about the stack moving out from under us
701 // (because we're on a system stack, not a goroutine stack).
702 // The above code could be used directly if already on a system stack,
703 // but then the only path through this code would be a rare case.
704 // Using this code for all "already on system stack" calls exercises it more,
705 // which should help keep it correct.
706
707 SUB $(asmcgocallSaveOffset+8), R1, R10
708 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
709 MOVD R7, asmcgocallSaveOffset(R1) // Save original stack pointer.
710
711 MOVD R3, R12 // fn
712 #ifdef GO_PPC64X_HAS_FUNCDESC
713 // Load the real entry address from the first slot of the function descriptor.
714 MOVD 8(R12), R2
715 MOVD (R12), R12
716 #endif
717 MOVD R12, CTR
718 MOVD R4, R3 // arg
719 BL (CTR)
720
721 // Reinitialise zero value register.
722 XOR R0, R0
723
724 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
725 #ifndef GOOS_aix
726 MOVD 24(R1), R2
727 #endif
728 // ret = R3
729 RET
730
731 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
732 // See cgocall.go for more details.
733 TEXT ·cgocallback(SB),NOSPLIT,$24-24
734 NO_LOCAL_POINTERS
735
736 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
737 // It is used to dropm while thread is exiting.
738 MOVD fn+0(FP), R5
739 CMP R5, $0
740 BNE loadg
741 // Restore the g from frame.
742 MOVD frame+8(FP), g
743 BR dropm
744
745 loadg:
746 // Load m and g from thread-local storage.
747 #ifndef GOOS_openbsd
748 MOVBZ runtime·iscgo(SB), R3
749 CMP R3, $0
750 BEQ nocgo
751 #endif
752 BL runtime·load_g(SB)
753 nocgo:
754
755 // If g is nil, Go did not create the current thread,
756 // or if this thread never called into Go on pthread platforms.
757 // Call needm to obtain one for temporary use.
758 // In this case, we're running on the thread stack, so there's
759 // lots of space, but the linker doesn't know. Hide the call from
760 // the linker analysis by using an indirect call.
761 CMP g, $0
762 BEQ needm
763
764 MOVD g_m(g), R8
765 MOVD R8, savedm-8(SP)
766 BR havem
767
768 needm:
769 MOVD g, savedm-8(SP) // g is zero, so is m.
770 MOVD $runtime·needAndBindM(SB), R12
771 MOVD R12, CTR
772 BL (CTR)
773
774 // Set m->sched.sp = SP, so that if a panic happens
775 // during the function we are about to execute, it will
776 // have a valid SP to run on the g0 stack.
777 // The next few lines (after the havem label)
778 // will save this SP onto the stack and then write
779 // the same SP back to m->sched.sp. That seems redundant,
780 // but if an unrecovered panic happens, unwindm will
781 // restore the g->sched.sp from the stack location
782 // and then systemstack will try to use it. If we don't set it here,
783 // that restored SP will be uninitialized (typically 0) and
784 // will not be usable.
785 MOVD g_m(g), R8
786 MOVD m_g0(R8), R3
787 MOVD R1, (g_sched+gobuf_sp)(R3)
788
789 havem:
790 // Now there's a valid m, and we're running on its m->g0.
791 // Save current m->g0->sched.sp on stack and then set it to SP.
792 // Save current sp in m->g0->sched.sp in preparation for
793 // switch back to m->curg stack.
794 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
795 MOVD m_g0(R8), R3
796 MOVD (g_sched+gobuf_sp)(R3), R4
797 MOVD R4, savedsp-24(SP) // must match frame size
798 MOVD R1, (g_sched+gobuf_sp)(R3)
799
800 // Switch to m->curg stack and call runtime.cgocallbackg.
801 // Because we are taking over the execution of m->curg
802 // but *not* resuming what had been running, we need to
803 // save that information (m->curg->sched) so we can restore it.
804 // We can restore m->curg->sched.sp easily, because calling
805 // runtime.cgocallbackg leaves SP unchanged upon return.
806 // To save m->curg->sched.pc, we push it onto the curg stack and
807 // open a frame the same size as cgocallback's g0 frame.
808 // Once we switch to the curg stack, the pushed PC will appear
809 // to be the return PC of cgocallback, so that the traceback
810 // will seamlessly trace back into the earlier calls.
811 MOVD m_curg(R8), g
812 BL runtime·save_g(SB)
813 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
814 MOVD (g_sched+gobuf_pc)(g), R5
815 MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size
816 // Gather our arguments into registers.
817 MOVD fn+0(FP), R5
818 MOVD frame+8(FP), R6
819 MOVD ctxt+16(FP), R7
820 MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size
821 MOVD R5, FIXED_FRAME+0(R1)
822 MOVD R6, FIXED_FRAME+8(R1)
823 MOVD R7, FIXED_FRAME+16(R1)
824
825 MOVD $runtime·cgocallbackg(SB), R12
826 MOVD R12, CTR
827 CALL (CTR) // indirect call to bypass nosplit check. We're on a different stack now.
828
829 // Restore g->sched (== m->curg->sched) from saved values.
830 MOVD 0(R1), R5
831 MOVD R5, (g_sched+gobuf_pc)(g)
832 MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size
833 MOVD R4, (g_sched+gobuf_sp)(g)
834
835 // Switch back to m->g0's stack and restore m->g0->sched.sp.
836 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
837 // so we do not have to restore it.)
838 MOVD g_m(g), R8
839 MOVD m_g0(R8), g
840 BL runtime·save_g(SB)
841 MOVD (g_sched+gobuf_sp)(g), R1
842 MOVD savedsp-24(SP), R4 // must match frame size
843 MOVD R4, (g_sched+gobuf_sp)(g)
844
845 // If the m on entry was nil, we called needm above to borrow an m,
846 // 1. for the duration of the call on non-pthread platforms,
847 // 2. or the duration of the C thread alive on pthread platforms.
848 // If the m on entry wasn't nil,
849 // 1. the thread might be a Go thread,
850 // 2. or it wasn't the first call from a C thread on pthread platforms,
851 // since then we skip dropm to reuse the m in the first call.
852 MOVD savedm-8(SP), R6
853 CMP R6, $0
854 BNE droppedm
855
856 // Skip dropm to reuse it in the next call, when a pthread key has been created.
857 MOVD _cgo_pthread_key_created(SB), R6
858 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
859 CMP R6, $0
860 BEQ dropm
861 MOVD (R6), R6
862 CMP R6, $0
863 BNE droppedm
864
865 dropm:
866 MOVD $runtime·dropm(SB), R12
867 MOVD R12, CTR
868 BL (CTR)
869 droppedm:
870
871 // Done!
872 RET
873
874 // void setg(G*); set g. for use by needm.
875 TEXT runtime·setg(SB), NOSPLIT, $0-8
876 MOVD gg+0(FP), g
877 // This only happens if iscgo, so jump straight to save_g
878 BL runtime·save_g(SB)
879 RET
880
881 #ifdef GO_PPC64X_HAS_FUNCDESC
882 DEFINE_PPC64X_FUNCDESC(setg_gcc<>, _setg_gcc<>)
883 TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
884 #else
885 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
886 #endif
887 // The standard prologue clobbers R31, which is callee-save in
888 // the C ABI, so we have to use $-8-0 and save LR ourselves.
889 MOVD LR, R4
890 // Also save g and R31, since they're callee-save in C ABI
891 MOVD R31, R5
892 MOVD g, R6
893
894 MOVD R3, g
895 BL runtime·save_g(SB)
896
897 MOVD R6, g
898 MOVD R5, R31
899 MOVD R4, LR
900 RET
901
902 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
903 MOVW (R0), R0
904 UNDEF
905
906 #define TBR 268
907
908 // int64 runtime·cputicks(void)
909 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
910 MOVD SPR(TBR), R3
911 MOVD R3, ret+0(FP)
912 RET
913
914 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
915 TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
916 MOVD R3, 0(R20)
917 MOVD R4, 8(R20)
918 MOVD R5, 16(R20)
919 MOVD R6, 24(R20)
920 MOVD R7, 32(R20)
921 MOVD R8, 40(R20)
922 MOVD R9, 48(R20)
923 MOVD R10, 56(R20)
924 MOVD R14, 64(R20)
925 MOVD R15, 72(R20)
926 MOVD R16, 80(R20)
927 MOVD R17, 88(R20)
928 FMOVD F1, 96(R20)
929 FMOVD F2, 104(R20)
930 FMOVD F3, 112(R20)
931 FMOVD F4, 120(R20)
932 FMOVD F5, 128(R20)
933 FMOVD F6, 136(R20)
934 FMOVD F7, 144(R20)
935 FMOVD F8, 152(R20)
936 FMOVD F9, 160(R20)
937 FMOVD F10, 168(R20)
938 FMOVD F11, 176(R20)
939 FMOVD F12, 184(R20)
940 RET
941
942 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
943 TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
944 MOVD 0(R20), R3
945 MOVD 8(R20), R4
946 MOVD 16(R20), R5
947 MOVD 24(R20), R6
948 MOVD 32(R20), R7
949 MOVD 40(R20), R8
950 MOVD 48(R20), R9
951 MOVD 56(R20), R10
952 MOVD 64(R20), R14
953 MOVD 72(R20), R15
954 MOVD 80(R20), R16
955 MOVD 88(R20), R17
956 FMOVD 96(R20), F1
957 FMOVD 104(R20), F2
958 FMOVD 112(R20), F3
959 FMOVD 120(R20), F4
960 FMOVD 128(R20), F5
961 FMOVD 136(R20), F6
962 FMOVD 144(R20), F7
963 FMOVD 152(R20), F8
964 FMOVD 160(R20), F9
965 FMOVD 168(R20), F10
966 FMOVD 176(R20), F11
967 FMOVD 184(R20), F12
968 RET
969
970 // AES hashing not implemented for ppc64
971 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
972 JMP runtime·memhashFallback<ABIInternal>(SB)
973 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
974 JMP runtime·strhashFallback<ABIInternal>(SB)
975 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
976 JMP runtime·memhash32Fallback<ABIInternal>(SB)
977 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
978 JMP runtime·memhash64Fallback<ABIInternal>(SB)
979
980 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
981 // Must obey the gcc calling convention.
982 #ifdef GOOS_aix
983 // On AIX, _cgo_topofstack is defined in runtime/cgo, because it must
984 // be a longcall in order to prevent trampolines from ld.
985 TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
986 #else
987 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
988 #endif
989 // g (R30) and R31 are callee-save in the C ABI, so save them
990 MOVD g, R4
991 MOVD R31, R5
992 MOVD LR, R6
993
994 BL runtime·load_g(SB) // clobbers g (R30), R31
995 MOVD g_m(g), R3
996 MOVD m_curg(R3), R3
997 MOVD (g_stack+stack_hi)(R3), R3
998
999 MOVD R4, g
1000 MOVD R5, R31
1001 MOVD R6, LR
1002 RET
1003
1004 // The top-most function running on a goroutine
1005 // returns to goexit+PCQuantum.
1006 //
1007 // When dynamically linking Go, it can be returned to from a function
1008 // implemented in a different module and so needs to reload the TOC pointer
1009 // from the stack (although this function declares that it does not set up x-a
1010 // frame, newproc1 does in fact allocate one for goexit and saves the TOC
1011 // pointer in the correct place).
1012 // goexit+_PCQuantum is halfway through the usual global entry point prologue
1013 // that derives r2 from r12 which is a bit silly, but not harmful.
1014 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
1015 MOVD 24(R1), R2
1016 BL runtime·goexit1(SB) // does not return
1017 // traceback from goexit1 must hit code range of goexit
1018 MOVD R0, R0 // NOP
1019
1020 // prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the
1021 // module containing runtime) to the frame that goexit will execute in when
1022 // the goroutine exits. It's implemented in assembly mainly because that's the
1023 // easiest way to get access to R2.
1024 TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8
1025 MOVD sp+0(FP), R3
1026 MOVD R2, 24(R3)
1027 RET
1028
1029 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
1030 ADD $-8, R1
1031 MOVD R31, 0(R1)
1032 MOVD runtime·lastmoduledatap(SB), R4
1033 MOVD R3, moduledata_next(R4)
1034 MOVD R3, runtime·lastmoduledatap(SB)
1035 MOVD 0(R1), R31
1036 ADD $8, R1
1037 RET
1038
1039 TEXT ·checkASM(SB),NOSPLIT,$0-1
1040 MOVW $1, R3
1041 MOVB R3, ret+0(FP)
1042 RET
1043
1044 // gcWriteBarrier informs the GC about heap pointer writes.
1045 //
1046 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
1047 // number of bytes of buffer needed in R29, and returns a pointer
1048 // to the buffer space in R29.
1049 // It clobbers condition codes.
1050 // It does not clobber R0 through R17 (except special registers),
1051 // but may clobber any other register, *including* R31.
1052 TEXT gcWriteBarrier<>(SB),NOSPLIT,$120
1053 // The standard prologue clobbers R31.
1054 // We use R18, R19, and R31 as scratch registers.
1055 retry:
1056 MOVD g_m(g), R18
1057 MOVD m_p(R18), R18
1058 MOVD (p_wbBuf+wbBuf_next)(R18), R19
1059 MOVD (p_wbBuf+wbBuf_end)(R18), R31
1060 // Increment wbBuf.next position.
1061 ADD R29, R19
1062 // Is the buffer full?
1063 CMPU R31, R19
1064 BLT flush
1065 // Commit to the larger buffer.
1066 MOVD R19, (p_wbBuf+wbBuf_next)(R18)
1067 // Make return value (the original next position)
1068 SUB R29, R19, R29
1069 RET
1070
1071 flush:
1072 // Save registers R0 through R15 since these were not saved by the caller.
1073 // We don't save all registers on ppc64 because it takes too much space.
1074 MOVD R20, (FIXED_FRAME+0)(R1)
1075 MOVD R21, (FIXED_FRAME+8)(R1)
1076 // R0 is always 0, so no need to spill.
1077 // R1 is SP.
1078 // R2 is SB.
1079 MOVD R3, (FIXED_FRAME+16)(R1)
1080 MOVD R4, (FIXED_FRAME+24)(R1)
1081 MOVD R5, (FIXED_FRAME+32)(R1)
1082 MOVD R6, (FIXED_FRAME+40)(R1)
1083 MOVD R7, (FIXED_FRAME+48)(R1)
1084 MOVD R8, (FIXED_FRAME+56)(R1)
1085 MOVD R9, (FIXED_FRAME+64)(R1)
1086 MOVD R10, (FIXED_FRAME+72)(R1)
1087 // R11, R12 may be clobbered by external-linker-inserted trampoline
1088 // R13 is REGTLS
1089 MOVD R14, (FIXED_FRAME+80)(R1)
1090 MOVD R15, (FIXED_FRAME+88)(R1)
1091 MOVD R16, (FIXED_FRAME+96)(R1)
1092 MOVD R17, (FIXED_FRAME+104)(R1)
1093 MOVD R29, (FIXED_FRAME+112)(R1)
1094
1095 CALL runtime·wbBufFlush(SB)
1096
1097 MOVD (FIXED_FRAME+0)(R1), R20
1098 MOVD (FIXED_FRAME+8)(R1), R21
1099 MOVD (FIXED_FRAME+16)(R1), R3
1100 MOVD (FIXED_FRAME+24)(R1), R4
1101 MOVD (FIXED_FRAME+32)(R1), R5
1102 MOVD (FIXED_FRAME+40)(R1), R6
1103 MOVD (FIXED_FRAME+48)(R1), R7
1104 MOVD (FIXED_FRAME+56)(R1), R8
1105 MOVD (FIXED_FRAME+64)(R1), R9
1106 MOVD (FIXED_FRAME+72)(R1), R10
1107 MOVD (FIXED_FRAME+80)(R1), R14
1108 MOVD (FIXED_FRAME+88)(R1), R15
1109 MOVD (FIXED_FRAME+96)(R1), R16
1110 MOVD (FIXED_FRAME+104)(R1), R17
1111 MOVD (FIXED_FRAME+112)(R1), R29
1112 JMP retry
1113
1114 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
1115 MOVD $8, R29
1116 JMP gcWriteBarrier<>(SB)
1117 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
1118 MOVD $16, R29
1119 JMP gcWriteBarrier<>(SB)
1120 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
1121 MOVD $24, R29
1122 JMP gcWriteBarrier<>(SB)
1123 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
1124 MOVD $32, R29
1125 JMP gcWriteBarrier<>(SB)
1126 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
1127 MOVD $40, R29
1128 JMP gcWriteBarrier<>(SB)
1129 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
1130 MOVD $48, R29
1131 JMP gcWriteBarrier<>(SB)
1132 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
1133 MOVD $56, R29
1134 JMP gcWriteBarrier<>(SB)
1135 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
1136 MOVD $64, R29
1137 JMP gcWriteBarrier<>(SB)
1138
1139 DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1140 GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1141
1142 // debugCallV2 is the entry point for debugger-injected function
1143 // calls on running goroutines. It informs the runtime that a
1144 // debug call has been injected and creates a call frame for the
1145 // debugger to fill in.
1146 //
1147 // To inject a function call, a debugger should:
1148 // 1. Check that the goroutine is in state _Grunning and that
1149 // there are at least 320 bytes free on the stack.
1150 // 2. Set SP as SP-32.
1151 // 3. Store the current LR in (SP) (using the SP after step 2).
1152 // 4. Store the current PC in the LR register.
1153 // 5. Write the desired argument frame size at SP-32
1154 // 6. Save all machine registers (including flags and floating point registers)
1155 // so they can be restored later by the debugger.
1156 // 7. Set the PC to debugCallV2 and resume execution.
1157 //
1158 // If the goroutine is in state _Grunnable, then it's not generally
1159 // safe to inject a call because it may return out via other runtime
1160 // operations. Instead, the debugger should unwind the stack to find
1161 // the return to non-runtime code, add a temporary breakpoint there,
1162 // and inject the call once that breakpoint is hit.
1163 //
1164 // If the goroutine is in any other state, it's not safe to inject a call.
1165 //
1166 // This function communicates back to the debugger by setting R20 and
1167 // invoking TW to raise a breakpoint signal. Note that the signal PC of
1168 // the signal triggered by the TW instruction is the PC where the signal
1169 // is trapped, not the next PC, so to resume execution, the debugger needs
1170 // to set the signal PC to PC+4. See the comments in the implementation for
1171 // the protocol the debugger is expected to follow. InjectDebugCall in the
1172 // runtime tests demonstrates this protocol.
1173 // The debugger must ensure that any pointers passed to the function
1174 // obey escape analysis requirements. Specifically, it must not pass
1175 // a stack pointer to an escaping argument. debugCallV2 cannot check
1176 // this invariant.
1177 //
1178 // This is ABIInternal because Go code injects its PC directly into new
1179 // goroutine stacks.
1180 #ifdef GOARCH_ppc64le
1181 TEXT runtime·debugCallV2<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-0
1182 // save scratch register R31 first
1183 MOVD R31, -184(R1)
1184 MOVD 0(R1), R31
1185 // save caller LR
1186 MOVD R31, -304(R1)
1187 MOVD -32(R1), R31
1188 // save argument frame size
1189 MOVD R31, -192(R1)
1190 MOVD LR, R31
1191 MOVD R31, -320(R1)
1192 ADD $-320, R1
1193 // save all registers that can contain pointers
1194 // and the CR register
1195 MOVW CR, R31
1196 MOVD R31, 8(R1)
1197 MOVD R2, 24(R1)
1198 MOVD R3, 56(R1)
1199 MOVD R4, 64(R1)
1200 MOVD R5, 72(R1)
1201 MOVD R6, 80(R1)
1202 MOVD R7, 88(R1)
1203 MOVD R8, 96(R1)
1204 MOVD R9, 104(R1)
1205 MOVD R10, 112(R1)
1206 MOVD R11, 120(R1)
1207 MOVD R12, 144(R1)
1208 MOVD R13, 152(R1)
1209 MOVD R14, 160(R1)
1210 MOVD R15, 168(R1)
1211 MOVD R16, 176(R1)
1212 MOVD R17, 184(R1)
1213 MOVD R18, 192(R1)
1214 MOVD R19, 200(R1)
1215 MOVD R20, 208(R1)
1216 MOVD R21, 216(R1)
1217 MOVD R22, 224(R1)
1218 MOVD R23, 232(R1)
1219 MOVD R24, 240(R1)
1220 MOVD R25, 248(R1)
1221 MOVD R26, 256(R1)
1222 MOVD R27, 264(R1)
1223 MOVD R28, 272(R1)
1224 MOVD R29, 280(R1)
1225 MOVD g, 288(R1)
1226 MOVD LR, R31
1227 MOVD R31, 32(R1)
1228 CALL runtime·debugCallCheck(SB)
1229 MOVD 40(R1), R22
1230 XOR R0, R0
1231 CMP R22, $0
1232 BEQ good
1233 MOVD 48(R1), R22
1234 MOVD $8, R20
1235 TW $31, R0, R0
1236
1237 BR restore
1238
1239 good:
1240 #define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1241 MOVD $MAXSIZE, R23; \
1242 CMP R26, R23; \
1243 BGT 5(PC); \
1244 MOVD $NAME(SB), R26; \
1245 MOVD R26, 32(R1); \
1246 CALL runtime·debugCallWrap(SB); \
1247 BR restore
1248
1249 // the argument frame size
1250 MOVD 128(R1), R26
1251
1252 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1253 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1254 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1255 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1256 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1257 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1258 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1259 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1260 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1261 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1262 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1263 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1264 // The frame size is too large. Report the error.
1265 MOVD $debugCallFrameTooLarge<>(SB), R22
1266 MOVD R22, 32(R1)
1267 MOVD $20, R22
1268 // length of debugCallFrameTooLarge string
1269 MOVD R22, 40(R1)
1270 MOVD $8, R20
1271 TW $31, R0, R0
1272 BR restore
1273 restore:
1274 MOVD $16, R20
1275 TW $31, R0, R0
1276 // restore all registers that can contain
1277 // pointers including CR
1278 MOVD 8(R1), R31
1279 MOVW R31, CR
1280 MOVD 24(R1), R2
1281 MOVD 56(R1), R3
1282 MOVD 64(R1), R4
1283 MOVD 72(R1), R5
1284 MOVD 80(R1), R6
1285 MOVD 88(R1), R7
1286 MOVD 96(R1), R8
1287 MOVD 104(R1), R9
1288 MOVD 112(R1), R10
1289 MOVD 120(R1), R11
1290 MOVD 144(R1), R12
1291 MOVD 152(R1), R13
1292 MOVD 160(R1), R14
1293 MOVD 168(R1), R15
1294 MOVD 176(R1), R16
1295 MOVD 184(R1), R17
1296 MOVD 192(R1), R18
1297 MOVD 200(R1), R19
1298 MOVD 208(R1), R20
1299 MOVD 216(R1), R21
1300 MOVD 224(R1), R22
1301 MOVD 232(R1), R23
1302 MOVD 240(R1), R24
1303 MOVD 248(R1), R25
1304 MOVD 256(R1), R26
1305 MOVD 264(R1), R27
1306 MOVD 272(R1), R28
1307 MOVD 280(R1), R29
1308 MOVD 288(R1), g
1309 MOVD 16(R1), R31
1310 // restore old LR
1311 MOVD R31, LR
1312 // restore caller PC
1313 MOVD 0(R1), CTR
1314 MOVD 136(R1), R31
1315 // Add 32 bytes more to compensate for SP change in saveSigContext
1316 ADD $352, R1
1317 JMP (CTR)
1318 #endif
1319 #define DEBUG_CALL_FN(NAME,MAXSIZE) \
1320 TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1321 NO_LOCAL_POINTERS; \
1322 MOVD $0, R20; \
1323 TW $31, R0, R0 \
1324 MOVD $1, R20; \
1325 TW $31, R0, R0 \
1326 RET
1327 DEBUG_CALL_FN(debugCall32<>, 32)
1328 DEBUG_CALL_FN(debugCall64<>, 64)
1329 DEBUG_CALL_FN(debugCall128<>, 128)
1330 DEBUG_CALL_FN(debugCall256<>, 256)
1331 DEBUG_CALL_FN(debugCall512<>, 512)
1332 DEBUG_CALL_FN(debugCall1024<>, 1024)
1333 DEBUG_CALL_FN(debugCall2048<>, 2048)
1334 DEBUG_CALL_FN(debugCall4096<>, 4096)
1335 DEBUG_CALL_FN(debugCall8192<>, 8192)
1336 DEBUG_CALL_FN(debugCall16384<>, 16384)
1337 DEBUG_CALL_FN(debugCall32768<>, 32768)
1338 DEBUG_CALL_FN(debugCall65536<>, 65536)
1339
1340 #ifdef GOARCH_ppc64le
1341 // func debugCallPanicked(val interface{})
1342 TEXT runtime·debugCallPanicked(SB),NOSPLIT,$32-16
1343 // Copy the panic value to the top of stack at SP+32.
1344 MOVD val_type+0(FP), R31
1345 MOVD R31, 32(R1)
1346 MOVD val_data+8(FP), R31
1347 MOVD R31, 40(R1)
1348 MOVD $2, R20
1349 TW $31, R0, R0
1350 RET
1351 #endif
1352 // Note: these functions use a special calling convention to save generated code space.
1353 // Arguments are passed in registers, but the space for those arguments are allocated
1354 // in the caller's stack frame. These stubs write the args into that stack space and
1355 // then tail call to the corresponding runtime handler.
1356 // The tail call makes these stubs disappear in backtraces.
1357 TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
1358 JMP runtime·goPanicIndex<ABIInternal>(SB)
1359 TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
1360 JMP runtime·goPanicIndexU<ABIInternal>(SB)
1361 TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
1362 MOVD R4, R3
1363 MOVD R5, R4
1364 JMP runtime·goPanicSliceAlen<ABIInternal>(SB)
1365 TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
1366 MOVD R4, R3
1367 MOVD R5, R4
1368 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB)
1369 TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
1370 MOVD R4, R3
1371 MOVD R5, R4
1372 JMP runtime·goPanicSliceAcap<ABIInternal>(SB)
1373 TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
1374 MOVD R4, R3
1375 MOVD R5, R4
1376 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB)
1377 TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
1378 JMP runtime·goPanicSliceB<ABIInternal>(SB)
1379 TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
1380 JMP runtime·goPanicSliceBU<ABIInternal>(SB)
1381 TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
1382 MOVD R5, R3
1383 MOVD R6, R4
1384 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB)
1385 TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
1386 MOVD R5, R3
1387 MOVD R6, R4
1388 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB)
1389 TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
1390 MOVD R5, R3
1391 MOVD R6, R4
1392 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB)
1393 TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
1394 MOVD R5, R3
1395 MOVD R6, R4
1396 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB)
1397 TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
1398 MOVD R4, R3
1399 MOVD R5, R4
1400 JMP runtime·goPanicSlice3B<ABIInternal>(SB)
1401 TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
1402 MOVD R4, R3
1403 MOVD R5, R4
1404 JMP runtime·goPanicSlice3BU<ABIInternal>(SB)
1405 TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
1406 JMP runtime·goPanicSlice3C<ABIInternal>(SB)
1407 TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
1408 JMP runtime·goPanicSlice3CU<ABIInternal>(SB)
1409 TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
1410 MOVD R5, R3
1411 MOVD R6, R4
1412 JMP runtime·goPanicSliceConvert<ABIInternal>(SB)
1413
1414 // These functions are used when internal linking cgo with external
1415 // objects compiled with the -Os on gcc. They reduce prologue/epilogue
1416 // size by deferring preservation of callee-save registers to a shared
1417 // function. These are defined in PPC64 ELFv2 2.3.3 (but also present
1418 // in ELFv1)
1419 //
1420 // These appear unused, but the linker will redirect calls to functions
1421 // like _savegpr0_14 or _restgpr1_14 to runtime.elf_savegpr0 or
1422 // runtime.elf_restgpr1 with an appropriate offset based on the number
1423 // register operations required when linking external objects which
1424 // make these calls. For GPR/FPR saves, the minimum register value is
1425 // 14, for VR it is 20.
1426 //
1427 // These are only used when linking such cgo code internally. Note, R12
1428 // and R0 may be used in different ways than regular ELF compliant
1429 // functions.
1430 TEXT runtime·elf_savegpr0(SB),NOSPLIT|NOFRAME,$0
1431 // R0 holds the LR of the caller's caller, R1 holds save location
1432 MOVD R14, -144(R1)
1433 MOVD R15, -136(R1)
1434 MOVD R16, -128(R1)
1435 MOVD R17, -120(R1)
1436 MOVD R18, -112(R1)
1437 MOVD R19, -104(R1)
1438 MOVD R20, -96(R1)
1439 MOVD R21, -88(R1)
1440 MOVD R22, -80(R1)
1441 MOVD R23, -72(R1)
1442 MOVD R24, -64(R1)
1443 MOVD R25, -56(R1)
1444 MOVD R26, -48(R1)
1445 MOVD R27, -40(R1)
1446 MOVD R28, -32(R1)
1447 MOVD R29, -24(R1)
1448 MOVD g, -16(R1)
1449 MOVD R31, -8(R1)
1450 MOVD R0, 16(R1)
1451 RET
1452 TEXT runtime·elf_restgpr0(SB),NOSPLIT|NOFRAME,$0
1453 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1454 MOVD -144(R1), R14
1455 MOVD -136(R1), R15
1456 MOVD -128(R1), R16
1457 MOVD -120(R1), R17
1458 MOVD -112(R1), R18
1459 MOVD -104(R1), R19
1460 MOVD -96(R1), R20
1461 MOVD -88(R1), R21
1462 MOVD -80(R1), R22
1463 MOVD -72(R1), R23
1464 MOVD -64(R1), R24
1465 MOVD -56(R1), R25
1466 MOVD -48(R1), R26
1467 MOVD -40(R1), R27
1468 MOVD -32(R1), R28
1469 MOVD -24(R1), R29
1470 MOVD -16(R1), g
1471 MOVD -8(R1), R31
1472 MOVD 16(R1), R0 // Load and return to saved LR
1473 MOVD R0, LR
1474 RET
1475 TEXT runtime·elf_savegpr1(SB),NOSPLIT|NOFRAME,$0
1476 // R12 holds the save location
1477 MOVD R14, -144(R12)
1478 MOVD R15, -136(R12)
1479 MOVD R16, -128(R12)
1480 MOVD R17, -120(R12)
1481 MOVD R18, -112(R12)
1482 MOVD R19, -104(R12)
1483 MOVD R20, -96(R12)
1484 MOVD R21, -88(R12)
1485 MOVD R22, -80(R12)
1486 MOVD R23, -72(R12)
1487 MOVD R24, -64(R12)
1488 MOVD R25, -56(R12)
1489 MOVD R26, -48(R12)
1490 MOVD R27, -40(R12)
1491 MOVD R28, -32(R12)
1492 MOVD R29, -24(R12)
1493 MOVD g, -16(R12)
1494 MOVD R31, -8(R12)
1495 RET
1496 TEXT runtime·elf_restgpr1(SB),NOSPLIT|NOFRAME,$0
1497 // R12 holds the save location
1498 MOVD -144(R12), R14
1499 MOVD -136(R12), R15
1500 MOVD -128(R12), R16
1501 MOVD -120(R12), R17
1502 MOVD -112(R12), R18
1503 MOVD -104(R12), R19
1504 MOVD -96(R12), R20
1505 MOVD -88(R12), R21
1506 MOVD -80(R12), R22
1507 MOVD -72(R12), R23
1508 MOVD -64(R12), R24
1509 MOVD -56(R12), R25
1510 MOVD -48(R12), R26
1511 MOVD -40(R12), R27
1512 MOVD -32(R12), R28
1513 MOVD -24(R12), R29
1514 MOVD -16(R12), g
1515 MOVD -8(R12), R31
1516 RET
1517 TEXT runtime·elf_savefpr(SB),NOSPLIT|NOFRAME,$0
1518 // R0 holds the LR of the caller's caller, R1 holds save location
1519 FMOVD F14, -144(R1)
1520 FMOVD F15, -136(R1)
1521 FMOVD F16, -128(R1)
1522 FMOVD F17, -120(R1)
1523 FMOVD F18, -112(R1)
1524 FMOVD F19, -104(R1)
1525 FMOVD F20, -96(R1)
1526 FMOVD F21, -88(R1)
1527 FMOVD F22, -80(R1)
1528 FMOVD F23, -72(R1)
1529 FMOVD F24, -64(R1)
1530 FMOVD F25, -56(R1)
1531 FMOVD F26, -48(R1)
1532 FMOVD F27, -40(R1)
1533 FMOVD F28, -32(R1)
1534 FMOVD F29, -24(R1)
1535 FMOVD F30, -16(R1)
1536 FMOVD F31, -8(R1)
1537 MOVD R0, 16(R1)
1538 RET
1539 TEXT runtime·elf_restfpr(SB),NOSPLIT|NOFRAME,$0
1540 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1541 FMOVD -144(R1), F14
1542 FMOVD -136(R1), F15
1543 FMOVD -128(R1), F16
1544 FMOVD -120(R1), F17
1545 FMOVD -112(R1), F18
1546 FMOVD -104(R1), F19
1547 FMOVD -96(R1), F20
1548 FMOVD -88(R1), F21
1549 FMOVD -80(R1), F22
1550 FMOVD -72(R1), F23
1551 FMOVD -64(R1), F24
1552 FMOVD -56(R1), F25
1553 FMOVD -48(R1), F26
1554 FMOVD -40(R1), F27
1555 FMOVD -32(R1), F28
1556 FMOVD -24(R1), F29
1557 FMOVD -16(R1), F30
1558 FMOVD -8(R1), F31
1559 MOVD 16(R1), R0 // Load and return to saved LR
1560 MOVD R0, LR
1561 RET
1562 TEXT runtime·elf_savevr(SB),NOSPLIT|NOFRAME,$0
1563 // R0 holds the save location, R12 is clobbered
1564 MOVD $-192, R12
1565 STVX V20, (R0+R12)
1566 MOVD $-176, R12
1567 STVX V21, (R0+R12)
1568 MOVD $-160, R12
1569 STVX V22, (R0+R12)
1570 MOVD $-144, R12
1571 STVX V23, (R0+R12)
1572 MOVD $-128, R12
1573 STVX V24, (R0+R12)
1574 MOVD $-112, R12
1575 STVX V25, (R0+R12)
1576 MOVD $-96, R12
1577 STVX V26, (R0+R12)
1578 MOVD $-80, R12
1579 STVX V27, (R0+R12)
1580 MOVD $-64, R12
1581 STVX V28, (R0+R12)
1582 MOVD $-48, R12
1583 STVX V29, (R0+R12)
1584 MOVD $-32, R12
1585 STVX V30, (R0+R12)
1586 MOVD $-16, R12
1587 STVX V31, (R0+R12)
1588 RET
1589 TEXT runtime·elf_restvr(SB),NOSPLIT|NOFRAME,$0
1590 // R0 holds the save location, R12 is clobbered
1591 MOVD $-192, R12
1592 LVX (R0+R12), V20
1593 MOVD $-176, R12
1594 LVX (R0+R12), V21
1595 MOVD $-160, R12
1596 LVX (R0+R12), V22
1597 MOVD $-144, R12
1598 LVX (R0+R12), V23
1599 MOVD $-128, R12
1600 LVX (R0+R12), V24
1601 MOVD $-112, R12
1602 LVX (R0+R12), V25
1603 MOVD $-96, R12
1604 LVX (R0+R12), V26
1605 MOVD $-80, R12
1606 LVX (R0+R12), V27
1607 MOVD $-64, R12
1608 LVX (R0+R12), V28
1609 MOVD $-48, R12
1610 LVX (R0+R12), V29
1611 MOVD $-32, R12
1612 LVX (R0+R12), V30
1613 MOVD $-16, R12
1614 LVX (R0+R12), V31
1615 RET
1616
View as plain text