Text file
src/runtime/asm_ppc64x.s
1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build ppc64 || ppc64le
6
7 #include "go_asm.h"
8 #include "go_tls.h"
9 #include "funcdata.h"
10 #include "textflag.h"
11 #include "asm_ppc64x.h"
12 #include "cgo/abi_ppc64x.h"
13
14 // This is called using the host ABI. argc and argv arguments
15 // should be in R3 and R4 respectively.
16 TEXT _rt0_ppc64x_lib(SB),NOSPLIT|NOFRAME,$0
17 // Convert to Go ABI, and Allocate argument storage for call to newosproc0.
18 STACK_AND_SAVE_HOST_TO_GO_ABI(16)
19
20 MOVD R3, _rt0_ppc64x_lib_argc<>(SB)
21 MOVD R4, _rt0_ppc64x_lib_argv<>(SB)
22
23 // Synchronous initialization.
24 MOVD $runtime·reginit(SB), R12
25 MOVD R12, CTR
26 BL (CTR)
27 MOVD $runtime·libpreinit(SB), R12
28 MOVD R12, CTR
29 BL (CTR)
30
31 // Create a new thread to do the runtime initialization and return.
32 MOVD _cgo_sys_thread_create(SB), R12
33 CMP $0, R12
34 BEQ nocgo
35 MOVD $_rt0_ppc64x_lib_go(SB), R3
36 MOVD $0, R4
37 MOVD R12, CTR
38 BL (CTR)
39 BR done
40
41 nocgo:
42 MOVD $0x800000, R12 // stacksize = 8192KB
43 MOVD R12, 8+FIXED_FRAME(R1)
44 MOVD $_rt0_ppc64x_lib_go(SB), R12
45 MOVD R12, 16+FIXED_FRAME(R1)
46 MOVD $runtime·newosproc0(SB),R12
47 MOVD R12, CTR
48 BL (CTR)
49
50 done:
51 UNSTACK_AND_RESTORE_GO_TO_HOST_ABI(16)
52 RET
53
54 #ifdef GO_PPC64X_HAS_FUNCDESC
55 DEFINE_PPC64X_FUNCDESC(_rt0_ppc64x_lib_go, __rt0_ppc64x_lib_go)
56 TEXT __rt0_ppc64x_lib_go(SB),NOSPLIT,$0
57 #else
58 TEXT _rt0_ppc64x_lib_go(SB),NOSPLIT,$0
59 #endif
60 MOVD _rt0_ppc64x_lib_argc<>(SB), R3
61 MOVD _rt0_ppc64x_lib_argv<>(SB), R4
62 MOVD $runtime·rt0_go(SB), R12
63 MOVD R12, CTR
64 BR (CTR)
65
66 DATA _rt0_ppc64x_lib_argc<>(SB)/8, $0
67 GLOBL _rt0_ppc64x_lib_argc<>(SB),NOPTR, $8
68 DATA _rt0_ppc64x_lib_argv<>(SB)/8, $0
69 GLOBL _rt0_ppc64x_lib_argv<>(SB),NOPTR, $8
70
71
72 #ifdef GOOS_aix
73 #define cgoCalleeStackSize 48
74 #else
75 #define cgoCalleeStackSize 32
76 #endif
77
78 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
79 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer
80
81 // initialize essential registers
82 BL runtime·reginit(SB)
83
84 SUB $(FIXED_FRAME+16), R1
85 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame
86 MOVW R3, FIXED_FRAME+0(R1) // argc
87 MOVD R4, FIXED_FRAME+8(R1) // argv
88
89 // create istack out of the given (operating system) stack.
90 // _cgo_init may update stackguard.
91 MOVD $runtime·g0(SB), g
92 BL runtime·save_g(SB)
93 MOVD $(-64*1024), R31
94 ADD R31, R1, R3
95 MOVD R3, g_stackguard0(g)
96 MOVD R3, g_stackguard1(g)
97 MOVD R3, (g_stack+stack_lo)(g)
98 MOVD R1, (g_stack+stack_hi)(g)
99
100 // If there is a _cgo_init, call it using the gcc ABI.
101 MOVD _cgo_init(SB), R12
102 CMP R12, $0
103 BEQ nocgo
104
105 #ifdef GO_PPC64X_HAS_FUNCDESC
106 // Load the real entry address from the first slot of the function descriptor.
107 MOVD 8(R12), R2
108 MOVD (R12), R12
109 #endif
110 MOVD R12, CTR // r12 = "global function entry point"
111 MOVD R13, R5 // arg 2: TLS base pointer
112 MOVD $setg_gcc<>(SB), R4 // arg 1: setg
113 MOVD g, R3 // arg 0: G
114 // C functions expect 32 (48 for AIX) bytes of space on caller
115 // stack frame and a 16-byte aligned R1
116 MOVD R1, R14 // save current stack
117 SUB $cgoCalleeStackSize, R1 // reserve the callee area
118 RLDCR $0, R1, $~15, R1 // 16-byte align
119 BL (CTR) // may clobber R0, R3-R12
120 MOVD R14, R1 // restore stack
121 #ifndef GOOS_aix
122 MOVD 24(R1), R2
123 #endif
124 XOR R0, R0 // fix R0
125
126 nocgo:
127 // update stackguard after _cgo_init
128 MOVD (g_stack+stack_lo)(g), R3
129 ADD $const_stackGuard, R3
130 MOVD R3, g_stackguard0(g)
131 MOVD R3, g_stackguard1(g)
132
133 // set the per-goroutine and per-mach "registers"
134 MOVD $runtime·m0(SB), R3
135
136 // save m->g0 = g0
137 MOVD g, m_g0(R3)
138 // save m0 to g0->m
139 MOVD R3, g_m(g)
140
141 BL runtime·check(SB)
142
143 // args are already prepared
144 BL runtime·args(SB)
145 BL runtime·osinit(SB)
146 BL runtime·schedinit(SB)
147
148 // create a new goroutine to start program
149 MOVD $runtime·mainPC(SB), R3 // entry
150 MOVDU R3, -8(R1)
151 MOVDU R0, -8(R1)
152 MOVDU R0, -8(R1)
153 MOVDU R0, -8(R1)
154 MOVDU R0, -8(R1)
155 BL runtime·newproc(SB)
156 ADD $(8+FIXED_FRAME), R1
157
158 // start this M
159 BL runtime·mstart(SB)
160 // Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are
161 // intended to be called by debuggers.
162 #ifdef GOARCH_ppc64le
163 MOVD $runtime·debugPinnerV1<ABIInternal>(SB), R31
164 MOVD $runtime·debugCallV2<ABIInternal>(SB), R31
165 #endif
166 MOVD R0, 0(R0)
167 RET
168
169 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
170 GLOBL runtime·mainPC(SB),RODATA,$8
171
172 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
173 TW $31, R0, R0
174 RET
175
176 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
177 RET
178
179 // Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall_ppc64
180 TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0
181 // crosscall_ppc64 and crosscall2 need to reginit, but can't
182 // get at the 'runtime.reginit' symbol.
183 BR runtime·reginit(SB)
184
185 TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0
186 // set R0 to zero, it's expected by the toolchain
187 XOR R0, R0
188 RET
189
190 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
191 BL runtime·mstart0(SB)
192 RET // not reached
193
194 /*
195 * go-routine
196 */
197
198 // void gogo(Gobuf*)
199 // restore state from Gobuf; longjmp
200 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
201 MOVD buf+0(FP), R5
202 MOVD gobuf_g(R5), R6
203 MOVD 0(R6), R4 // make sure g != nil
204 BR gogo<>(SB)
205
206 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
207 MOVD R6, g
208 BL runtime·save_g(SB)
209
210 MOVD gobuf_sp(R5), R1
211 MOVD gobuf_lr(R5), R31
212 #ifndef GOOS_aix
213 MOVD 24(R1), R2 // restore R2
214 #endif
215 MOVD R31, LR
216 MOVD gobuf_ctxt(R5), R11
217 MOVD R0, gobuf_sp(R5)
218 MOVD R0, gobuf_lr(R5)
219 MOVD R0, gobuf_ctxt(R5)
220 CMP R0, R0 // set condition codes for == test, needed by stack split
221 MOVD gobuf_pc(R5), R12
222 MOVD R12, CTR
223 BR (CTR)
224
225 // void mcall(fn func(*g))
226 // Switch to m->g0's stack, call fn(g).
227 // Fn must never return. It should gogo(&g->sched)
228 // to keep running g.
229 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
230 // Save caller state in g->sched
231 // R11 should be safe across save_g??
232 MOVD R3, R11
233 MOVD R1, (g_sched+gobuf_sp)(g)
234 MOVD LR, R31
235 MOVD R31, (g_sched+gobuf_pc)(g)
236 MOVD R0, (g_sched+gobuf_lr)(g)
237
238 // Switch to m->g0 & its stack, call fn.
239 MOVD g, R3
240 MOVD g_m(g), R8
241 MOVD m_g0(R8), g
242 BL runtime·save_g(SB)
243 CMP g, R3
244 BNE 2(PC)
245 BR runtime·badmcall(SB)
246 MOVD 0(R11), R12 // code pointer
247 MOVD R12, CTR
248 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp
249 // Don't need to do anything special for regabiargs here
250 // R3 is g; stack is set anyway
251 MOVDU R3, -8(R1)
252 MOVDU R0, -8(R1)
253 MOVDU R0, -8(R1)
254 MOVDU R0, -8(R1)
255 MOVDU R0, -8(R1)
256 BL (CTR)
257 MOVD 24(R1), R2
258 BR runtime·badmcall2(SB)
259
260 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
261 // of the G stack. We need to distinguish the routine that
262 // lives at the bottom of the G stack from the one that lives
263 // at the top of the system stack because the one at the top of
264 // the system stack terminates the stack walk (see topofstack()).
265 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
266 // We have several undefs here so that 16 bytes past
267 // $runtime·systemstack_switch lies within them whether or not the
268 // instructions that derive r2 from r12 are there.
269 UNDEF
270 UNDEF
271 UNDEF
272 BL (LR) // make sure this function is not leaf
273 RET
274
275 // func systemstack(fn func())
276 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
277 MOVD fn+0(FP), R3 // R3 = fn
278 MOVD R3, R11 // context
279 MOVD g_m(g), R4 // R4 = m
280
281 MOVD m_gsignal(R4), R5 // R5 = gsignal
282 CMP g, R5
283 BEQ noswitch
284
285 MOVD m_g0(R4), R5 // R5 = g0
286 CMP g, R5
287 BEQ noswitch
288
289 MOVD m_curg(R4), R6
290 CMP g, R6
291 BEQ switch
292
293 // Bad: g is not gsignal, not g0, not curg. What is it?
294 // Hide call from linker nosplit analysis.
295 MOVD $runtime·badsystemstack(SB), R12
296 MOVD R12, CTR
297 BL (CTR)
298 BL runtime·abort(SB)
299
300 switch:
301 // save our state in g->sched. Pretend to
302 // be systemstack_switch if the G stack is scanned.
303 BL gosave_systemstack_switch<>(SB)
304
305 // switch to g0
306 MOVD R5, g
307 BL runtime·save_g(SB)
308 MOVD (g_sched+gobuf_sp)(g), R1
309
310 // call target function
311 MOVD 0(R11), R12 // code pointer
312 MOVD R12, CTR
313 BL (CTR)
314
315 // restore TOC pointer. It seems unlikely that we will use systemstack
316 // to call a function defined in another module, but the results of
317 // doing so would be so confusing that it's worth doing this.
318 MOVD g_m(g), R3
319 MOVD m_curg(R3), g
320 MOVD (g_sched+gobuf_sp)(g), R3
321 #ifndef GOOS_aix
322 MOVD 24(R3), R2
323 #endif
324 // switch back to g
325 MOVD g_m(g), R3
326 MOVD m_curg(R3), g
327 BL runtime·save_g(SB)
328 MOVD (g_sched+gobuf_sp)(g), R1
329 MOVD R0, (g_sched+gobuf_sp)(g)
330 RET
331
332 noswitch:
333 // already on m stack, just call directly
334 // On other arches we do a tail call here, but it appears to be
335 // impossible to tail call a function pointer in shared mode on
336 // ppc64 because the caller is responsible for restoring the TOC.
337 MOVD 0(R11), R12 // code pointer
338 MOVD R12, CTR
339 BL (CTR)
340 #ifndef GOOS_aix
341 MOVD 24(R1), R2
342 #endif
343 RET
344
345 // func switchToCrashStack0(fn func())
346 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
347 MOVD R3, R11 // context register
348 MOVD g_m(g), R3 // curm
349
350 // set g to gcrash
351 MOVD $runtime·gcrash(SB), g // g = &gcrash
352 CALL runtime·save_g(SB) // clobbers R31
353 MOVD R3, g_m(g) // g.m = curm
354 MOVD g, m_g0(R3) // curm.g0 = g
355
356 // switch to crashstack
357 MOVD (g_stack+stack_hi)(g), R3
358 SUB $(4*8), R3
359 MOVD R3, R1
360
361 // call target function
362 MOVD 0(R11), R12 // code pointer
363 MOVD R12, CTR
364 BL (CTR)
365
366 // should never return
367 CALL runtime·abort(SB)
368 UNDEF
369
370 /*
371 * support for morestack
372 */
373
374 // Called during function prolog when more stack is needed.
375 // Caller has already loaded:
376 // R3: framesize, R4: argsize, R5: LR
377 //
378 // The traceback routines see morestack on a g0 as being
379 // the top of a stack (for example, morestack calling newstack
380 // calling the scheduler calling newm calling gc), so we must
381 // record an argument size. For that purpose, it has no arguments.
382 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
383 // Called from f.
384 // Set g->sched to context in f.
385 MOVD R1, (g_sched+gobuf_sp)(g)
386 MOVD LR, R8
387 MOVD R8, (g_sched+gobuf_pc)(g)
388 MOVD R5, (g_sched+gobuf_lr)(g)
389 MOVD R11, (g_sched+gobuf_ctxt)(g)
390
391 // Cannot grow scheduler stack (m->g0).
392 MOVD g_m(g), R7
393 MOVD m_g0(R7), R8
394 CMP g, R8
395 BNE 3(PC)
396 BL runtime·badmorestackg0(SB)
397 BL runtime·abort(SB)
398
399 // Cannot grow signal stack (m->gsignal).
400 MOVD m_gsignal(R7), R8
401 CMP g, R8
402 BNE 3(PC)
403 BL runtime·badmorestackgsignal(SB)
404 BL runtime·abort(SB)
405
406 // Called from f.
407 // Set m->morebuf to f's caller.
408 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
409 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
410 MOVD g, (m_morebuf+gobuf_g)(R7)
411
412 // Call newstack on m->g0's stack.
413 MOVD m_g0(R7), g
414 BL runtime·save_g(SB)
415 MOVD (g_sched+gobuf_sp)(g), R1
416 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0
417 BL runtime·newstack(SB)
418
419 // Not reached, but make sure the return PC from the call to newstack
420 // is still in this function, and not the beginning of the next.
421 UNDEF
422
423 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
424 // Force SPWRITE. This function doesn't actually write SP,
425 // but it is called with a special calling convention where
426 // the caller doesn't save LR on stack but passes it as a
427 // register (R5), and the unwinder currently doesn't understand.
428 // Make it SPWRITE to stop unwinding. (See issue 54332)
429 // Use OR R0, R1 instead of MOVD R1, R1 as the MOVD instruction
430 // has a special affect on Power8,9,10 by lowering the thread
431 // priority and causing a slowdown in execution time
432
433 OR R0, R1
434 MOVD R0, R11
435 BR runtime·morestack(SB)
436
437 // reflectcall: call a function with the given argument list
438 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
439 // we don't have variable-sized frames, so we use a small number
440 // of constant-sized-frame functions to encode a few bits of size in the pc.
441 // Caution: ugly multiline assembly macros in your future!
442
443 #define DISPATCH(NAME,MAXSIZE) \
444 MOVD $MAXSIZE, R31; \
445 CMP R3, R31; \
446 BGT 4(PC); \
447 MOVD $NAME(SB), R12; \
448 MOVD R12, CTR; \
449 BR (CTR)
450 // Note: can't just "BR NAME(SB)" - bad inlining results.
451
452 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
453 MOVWZ frameSize+32(FP), R3
454 DISPATCH(runtime·call16, 16)
455 DISPATCH(runtime·call32, 32)
456 DISPATCH(runtime·call64, 64)
457 DISPATCH(runtime·call128, 128)
458 DISPATCH(runtime·call256, 256)
459 DISPATCH(runtime·call512, 512)
460 DISPATCH(runtime·call1024, 1024)
461 DISPATCH(runtime·call2048, 2048)
462 DISPATCH(runtime·call4096, 4096)
463 DISPATCH(runtime·call8192, 8192)
464 DISPATCH(runtime·call16384, 16384)
465 DISPATCH(runtime·call32768, 32768)
466 DISPATCH(runtime·call65536, 65536)
467 DISPATCH(runtime·call131072, 131072)
468 DISPATCH(runtime·call262144, 262144)
469 DISPATCH(runtime·call524288, 524288)
470 DISPATCH(runtime·call1048576, 1048576)
471 DISPATCH(runtime·call2097152, 2097152)
472 DISPATCH(runtime·call4194304, 4194304)
473 DISPATCH(runtime·call8388608, 8388608)
474 DISPATCH(runtime·call16777216, 16777216)
475 DISPATCH(runtime·call33554432, 33554432)
476 DISPATCH(runtime·call67108864, 67108864)
477 DISPATCH(runtime·call134217728, 134217728)
478 DISPATCH(runtime·call268435456, 268435456)
479 DISPATCH(runtime·call536870912, 536870912)
480 DISPATCH(runtime·call1073741824, 1073741824)
481 MOVD $runtime·badreflectcall(SB), R12
482 MOVD R12, CTR
483 BR (CTR)
484
485 #define CALLFN(NAME,MAXSIZE) \
486 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
487 NO_LOCAL_POINTERS; \
488 /* copy arguments to stack */ \
489 MOVD stackArgs+16(FP), R3; \
490 MOVWZ stackArgsSize+24(FP), R4; \
491 MOVD R1, R5; \
492 CMP R4, $8; \
493 BLT tailsetup; \
494 /* copy 8 at a time if possible */ \
495 ADD $(FIXED_FRAME-8), R5; \
496 SUB $8, R3; \
497 top: \
498 MOVDU 8(R3), R7; \
499 MOVDU R7, 8(R5); \
500 SUB $8, R4; \
501 CMP R4, $8; \
502 BGE top; \
503 /* handle remaining bytes */ \
504 CMP $0, R4; \
505 BEQ callfn; \
506 ADD $7, R3; \
507 ADD $7, R5; \
508 BR tail; \
509 tailsetup: \
510 CMP $0, R4; \
511 BEQ callfn; \
512 ADD $(FIXED_FRAME-1), R5; \
513 SUB $1, R3; \
514 tail: \
515 MOVBU 1(R3), R6; \
516 MOVBU R6, 1(R5); \
517 SUB $1, R4; \
518 CMP $0, R4; \
519 BGT tail; \
520 callfn: \
521 /* call function */ \
522 MOVD f+8(FP), R11; \
523 #ifdef GOOS_aix \
524 /* AIX won't trigger a SIGSEGV if R11 = nil */ \
525 /* So it manually triggers it */ \
526 CMP R11, $0 \
527 BNE 2(PC) \
528 MOVD R0, 0(R0) \
529 #endif \
530 MOVD regArgs+40(FP), R20; \
531 BL runtime·unspillArgs(SB); \
532 MOVD (R11), R12; \
533 MOVD R12, CTR; \
534 PCDATA $PCDATA_StackMapIndex, $0; \
535 BL (CTR); \
536 #ifndef GOOS_aix \
537 MOVD 24(R1), R2; \
538 #endif \
539 /* copy return values back */ \
540 MOVD regArgs+40(FP), R20; \
541 BL runtime·spillArgs(SB); \
542 MOVD stackArgsType+0(FP), R7; \
543 MOVD stackArgs+16(FP), R3; \
544 MOVWZ stackArgsSize+24(FP), R4; \
545 MOVWZ stackRetOffset+28(FP), R6; \
546 ADD $FIXED_FRAME, R1, R5; \
547 ADD R6, R5; \
548 ADD R6, R3; \
549 SUB R6, R4; \
550 BL callRet<>(SB); \
551 RET
552
553 // callRet copies return values back at the end of call*. This is a
554 // separate function so it can allocate stack space for the arguments
555 // to reflectcallmove. It does not follow the Go ABI; it expects its
556 // arguments in registers.
557 TEXT callRet<>(SB), NOSPLIT, $40-0
558 NO_LOCAL_POINTERS
559 MOVD R7, FIXED_FRAME+0(R1)
560 MOVD R3, FIXED_FRAME+8(R1)
561 MOVD R5, FIXED_FRAME+16(R1)
562 MOVD R4, FIXED_FRAME+24(R1)
563 MOVD R20, FIXED_FRAME+32(R1)
564 BL runtime·reflectcallmove(SB)
565 RET
566
567 CALLFN(·call16, 16)
568 CALLFN(·call32, 32)
569 CALLFN(·call64, 64)
570 CALLFN(·call128, 128)
571 CALLFN(·call256, 256)
572 CALLFN(·call512, 512)
573 CALLFN(·call1024, 1024)
574 CALLFN(·call2048, 2048)
575 CALLFN(·call4096, 4096)
576 CALLFN(·call8192, 8192)
577 CALLFN(·call16384, 16384)
578 CALLFN(·call32768, 32768)
579 CALLFN(·call65536, 65536)
580 CALLFN(·call131072, 131072)
581 CALLFN(·call262144, 262144)
582 CALLFN(·call524288, 524288)
583 CALLFN(·call1048576, 1048576)
584 CALLFN(·call2097152, 2097152)
585 CALLFN(·call4194304, 4194304)
586 CALLFN(·call8388608, 8388608)
587 CALLFN(·call16777216, 16777216)
588 CALLFN(·call33554432, 33554432)
589 CALLFN(·call67108864, 67108864)
590 CALLFN(·call134217728, 134217728)
591 CALLFN(·call268435456, 268435456)
592 CALLFN(·call536870912, 536870912)
593 CALLFN(·call1073741824, 1073741824)
594
595 TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0-4
596 MOVW cycles+0(FP), R7
597 // POWER does not have a pause/yield instruction equivalent.
598 // Instead, we can lower the program priority by setting the
599 // Program Priority Register prior to the wait loop and set it
600 // back to default afterwards. On Linux, the default priority is
601 // medium-low. For details, see page 837 of the ISA 3.0.
602 OR R1, R1, R1 // Set PPR priority to low
603 again:
604 SUB $1, R7
605 CMP $0, R7
606 BNE again
607 OR R6, R6, R6 // Set PPR priority back to medium-low
608 RET
609
610 // Save state of caller into g->sched,
611 // but using fake PC from systemstack_switch.
612 // Must only be called from functions with no locals ($0)
613 // or else unwinding from systemstack_switch is incorrect.
614 // Smashes R31.
615 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
616 MOVD $runtime·systemstack_switch(SB), R31
617 ADD $16, R31 // get past prologue (including r2-setting instructions when they're there)
618 MOVD R31, (g_sched+gobuf_pc)(g)
619 MOVD R1, (g_sched+gobuf_sp)(g)
620 MOVD R0, (g_sched+gobuf_lr)(g)
621 // Assert ctxt is zero. See func save.
622 MOVD (g_sched+gobuf_ctxt)(g), R31
623 CMP R31, $0
624 BEQ 2(PC)
625 BL runtime·abort(SB)
626 RET
627
628 #ifdef GOOS_aix
629 #define asmcgocallSaveOffset cgoCalleeStackSize + 8
630 #else
631 #define asmcgocallSaveOffset cgoCalleeStackSize
632 #endif
633
634 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
635 // Call fn(arg) aligned appropriately for the gcc ABI.
636 // Called on a system stack, and there may be no g yet (during needm).
637 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
638 MOVD fn+0(FP), R3
639 MOVD arg+8(FP), R4
640
641 MOVD R1, R15
642 SUB $(asmcgocallSaveOffset+8), R1
643 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
644 MOVD R15, asmcgocallSaveOffset(R1)
645
646 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
647
648 // This is a "global call", so put the global entry point in r12
649 MOVD R3, R12
650
651 #ifdef GO_PPC64X_HAS_FUNCDESC
652 // Load the real entry address from the first slot of the function descriptor.
653 MOVD 8(R12), R2
654 MOVD (R12), R12
655 #endif
656 MOVD R12, CTR
657 MOVD R4, R3 // arg in r3
658 BL (CTR)
659
660 // C code can clobber R0, so set it back to 0. F27-F31 are
661 // callee save, so we don't need to recover those.
662 XOR R0, R0
663
664 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
665 #ifndef GOOS_aix
666 MOVD 24(R1), R2
667 #endif
668
669 RET
670
671 // func asmcgocall(fn, arg unsafe.Pointer) int32
672 // Call fn(arg) on the scheduler stack,
673 // aligned appropriately for the gcc ABI.
674 // See cgocall.go for more details.
675 TEXT ·asmcgocall<ABIInternal>(SB),NOSPLIT,$0-20
676 // R3 = fn
677 // R4 = arg
678
679 MOVD R1, R7 // save original stack pointer
680 CMP $0, g
681 BEQ nosave
682 MOVD g, R5
683
684 // Figure out if we need to switch to m->g0 stack.
685 // We get called to create new OS threads too, and those
686 // come in on the m->g0 stack already. Or we might already
687 // be on the m->gsignal stack.
688 MOVD g_m(g), R8
689 MOVD m_gsignal(R8), R6
690 CMP R6, g
691 BEQ nosave
692 MOVD m_g0(R8), R6
693 CMP R6, g
694 BEQ nosave
695
696 BL gosave_systemstack_switch<>(SB)
697 MOVD R6, g
698 BL runtime·save_g(SB)
699 MOVD (g_sched+gobuf_sp)(g), R1
700
701 // Now on a scheduling stack (a pthread-created stack).
702 #ifdef GOOS_aix
703 // Create a fake LR to improve backtrace.
704 MOVD $runtime·asmcgocall(SB), R6
705 MOVD R6, 16(R1)
706 // AIX also saves one argument on the stack.
707 SUB $8, R1
708 #endif
709 // Save room for two of our pointers, plus the callee
710 // save area that lives on the caller stack.
711 // Do arithmetics in R10 to hide from the assembler
712 // counting it as SP delta, which is irrelevant as we are
713 // on the system stack.
714 SUB $(asmcgocallSaveOffset+16), R1, R10
715 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
716 MOVD R5, (asmcgocallSaveOffset+8)(R1) // save old g on stack
717 MOVD (g_stack+stack_hi)(R5), R5
718 SUB R7, R5
719 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
720 #ifdef GOOS_aix
721 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb
722 #else
723 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
724 #endif
725 // This is a "global call", so put the global entry point in r12
726 MOVD R3, R12
727
728 #ifdef GO_PPC64X_HAS_FUNCDESC
729 // Load the real entry address from the first slot of the function descriptor.
730 MOVD 8(R12), R2
731 MOVD (R12), R12
732 #endif
733 MOVD R12, CTR
734 MOVD R4, R3 // arg in r3
735 BL (CTR)
736
737 // Reinitialise zero value register.
738 XOR R0, R0
739
740 // Restore g, stack pointer, toc pointer.
741 // R3 is errno, so don't touch it
742 MOVD (asmcgocallSaveOffset+8)(R1), g
743 MOVD (g_stack+stack_hi)(g), R5
744 MOVD asmcgocallSaveOffset(R1), R6
745 SUB R6, R5
746 #ifndef GOOS_aix
747 MOVD 24(R5), R2
748 #endif
749 MOVD R5, R1
750 BL runtime·save_g(SB)
751
752 // ret = R3
753 RET
754
755 nosave:
756 // Running on a system stack, perhaps even without a g.
757 // Having no g can happen during thread creation or thread teardown.
758 // This code is like the above sequence but without saving/restoring g
759 // and without worrying about the stack moving out from under us
760 // (because we're on a system stack, not a goroutine stack).
761 // The above code could be used directly if already on a system stack,
762 // but then the only path through this code would be a rare case.
763 // Using this code for all "already on system stack" calls exercises it more,
764 // which should help keep it correct.
765
766 SUB $(asmcgocallSaveOffset+8), R1, R10
767 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
768 MOVD R7, asmcgocallSaveOffset(R1) // Save original stack pointer.
769
770 MOVD R3, R12 // fn
771 #ifdef GO_PPC64X_HAS_FUNCDESC
772 // Load the real entry address from the first slot of the function descriptor.
773 MOVD 8(R12), R2
774 MOVD (R12), R12
775 #endif
776 MOVD R12, CTR
777 MOVD R4, R3 // arg
778 BL (CTR)
779
780 // Reinitialise zero value register.
781 XOR R0, R0
782
783 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
784 #ifndef GOOS_aix
785 MOVD 24(R1), R2
786 #endif
787 // ret = R3
788 RET
789
790 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
791 // See cgocall.go for more details.
792 TEXT ·cgocallback(SB),NOSPLIT,$24-24
793 NO_LOCAL_POINTERS
794
795 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
796 // It is used to dropm while thread is exiting.
797 MOVD fn+0(FP), R5
798 CMP R5, $0
799 BNE loadg
800 // Restore the g from frame.
801 MOVD frame+8(FP), g
802 BR dropm
803
804 loadg:
805 // Load m and g from thread-local storage.
806 #ifndef GOOS_openbsd
807 MOVBZ runtime·iscgo(SB), R3
808 CMP R3, $0
809 BEQ nocgo
810 #endif
811 BL runtime·load_g(SB)
812 nocgo:
813
814 // If g is nil, Go did not create the current thread,
815 // or if this thread never called into Go on pthread platforms.
816 // Call needm to obtain one for temporary use.
817 // In this case, we're running on the thread stack, so there's
818 // lots of space, but the linker doesn't know. Hide the call from
819 // the linker analysis by using an indirect call.
820 CMP g, $0
821 BEQ needm
822
823 MOVD g_m(g), R8
824 MOVD R8, savedm-8(SP)
825 BR havem
826
827 needm:
828 MOVD g, savedm-8(SP) // g is zero, so is m.
829 MOVD $runtime·needAndBindM(SB), R12
830 MOVD R12, CTR
831 BL (CTR)
832
833 // Set m->sched.sp = SP, so that if a panic happens
834 // during the function we are about to execute, it will
835 // have a valid SP to run on the g0 stack.
836 // The next few lines (after the havem label)
837 // will save this SP onto the stack and then write
838 // the same SP back to m->sched.sp. That seems redundant,
839 // but if an unrecovered panic happens, unwindm will
840 // restore the g->sched.sp from the stack location
841 // and then systemstack will try to use it. If we don't set it here,
842 // that restored SP will be uninitialized (typically 0) and
843 // will not be usable.
844 MOVD g_m(g), R8
845 MOVD m_g0(R8), R3
846 MOVD R1, (g_sched+gobuf_sp)(R3)
847
848 havem:
849 // Now there's a valid m, and we're running on its m->g0.
850 // Save current m->g0->sched.sp on stack and then set it to SP.
851 // Save current sp in m->g0->sched.sp in preparation for
852 // switch back to m->curg stack.
853 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
854 MOVD m_g0(R8), R3
855 MOVD (g_sched+gobuf_sp)(R3), R4
856 MOVD R4, savedsp-24(SP) // must match frame size
857 MOVD R1, (g_sched+gobuf_sp)(R3)
858
859 // Switch to m->curg stack and call runtime.cgocallbackg.
860 // Because we are taking over the execution of m->curg
861 // but *not* resuming what had been running, we need to
862 // save that information (m->curg->sched) so we can restore it.
863 // We can restore m->curg->sched.sp easily, because calling
864 // runtime.cgocallbackg leaves SP unchanged upon return.
865 // To save m->curg->sched.pc, we push it onto the curg stack and
866 // open a frame the same size as cgocallback's g0 frame.
867 // Once we switch to the curg stack, the pushed PC will appear
868 // to be the return PC of cgocallback, so that the traceback
869 // will seamlessly trace back into the earlier calls.
870 MOVD m_curg(R8), g
871 BL runtime·save_g(SB)
872 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
873 MOVD (g_sched+gobuf_pc)(g), R5
874 MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size
875 // Gather our arguments into registers.
876 MOVD fn+0(FP), R5
877 MOVD frame+8(FP), R6
878 MOVD ctxt+16(FP), R7
879 MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size
880 MOVD R5, FIXED_FRAME+0(R1)
881 MOVD R6, FIXED_FRAME+8(R1)
882 MOVD R7, FIXED_FRAME+16(R1)
883
884 MOVD $runtime·cgocallbackg(SB), R12
885 MOVD R12, CTR
886 CALL (CTR) // indirect call to bypass nosplit check. We're on a different stack now.
887
888 // Restore g->sched (== m->curg->sched) from saved values.
889 MOVD 0(R1), R5
890 MOVD R5, (g_sched+gobuf_pc)(g)
891 MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size
892 MOVD R4, (g_sched+gobuf_sp)(g)
893
894 // Switch back to m->g0's stack and restore m->g0->sched.sp.
895 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
896 // so we do not have to restore it.)
897 MOVD g_m(g), R8
898 MOVD m_g0(R8), g
899 BL runtime·save_g(SB)
900 MOVD (g_sched+gobuf_sp)(g), R1
901 MOVD savedsp-24(SP), R4 // must match frame size
902 MOVD R4, (g_sched+gobuf_sp)(g)
903
904 // If the m on entry was nil, we called needm above to borrow an m,
905 // 1. for the duration of the call on non-pthread platforms,
906 // 2. or the duration of the C thread alive on pthread platforms.
907 // If the m on entry wasn't nil,
908 // 1. the thread might be a Go thread,
909 // 2. or it wasn't the first call from a C thread on pthread platforms,
910 // since then we skip dropm to reuse the m in the first call.
911 MOVD savedm-8(SP), R6
912 CMP R6, $0
913 BNE droppedm
914
915 // Skip dropm to reuse it in the next call, when a pthread key has been created.
916 MOVD _cgo_pthread_key_created(SB), R6
917 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
918 CMP R6, $0
919 BEQ dropm
920 MOVD (R6), R6
921 CMP R6, $0
922 BNE droppedm
923
924 dropm:
925 MOVD $runtime·dropm(SB), R12
926 MOVD R12, CTR
927 BL (CTR)
928 droppedm:
929
930 // Done!
931 RET
932
933 // void setg(G*); set g. for use by needm.
934 TEXT runtime·setg(SB), NOSPLIT, $0-8
935 MOVD gg+0(FP), g
936 // This only happens if iscgo, so jump straight to save_g
937 BL runtime·save_g(SB)
938 RET
939
940 #ifdef GO_PPC64X_HAS_FUNCDESC
941 DEFINE_PPC64X_FUNCDESC(setg_gcc<>, _setg_gcc<>)
942 TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
943 #else
944 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
945 #endif
946 // The standard prologue clobbers R31, which is callee-save in
947 // the C ABI, so we have to use $-8-0 and save LR ourselves.
948 MOVD LR, R4
949 // Also save g and R31, since they're callee-save in C ABI
950 MOVD R31, R5
951 MOVD g, R6
952
953 MOVD R3, g
954 BL runtime·save_g(SB)
955
956 MOVD R6, g
957 MOVD R5, R31
958 MOVD R4, LR
959 RET
960
961 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
962 MOVW (R0), R0
963 UNDEF
964
965 #define TBR 268
966
967 // int64 runtime·cputicks(void)
968 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
969 MOVD SPR(TBR), R3
970 MOVD R3, ret+0(FP)
971 RET
972
973 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
974 TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
975 MOVD R3, 0(R20)
976 MOVD R4, 8(R20)
977 MOVD R5, 16(R20)
978 MOVD R6, 24(R20)
979 MOVD R7, 32(R20)
980 MOVD R8, 40(R20)
981 MOVD R9, 48(R20)
982 MOVD R10, 56(R20)
983 MOVD R14, 64(R20)
984 MOVD R15, 72(R20)
985 MOVD R16, 80(R20)
986 MOVD R17, 88(R20)
987 FMOVD F1, 96(R20)
988 FMOVD F2, 104(R20)
989 FMOVD F3, 112(R20)
990 FMOVD F4, 120(R20)
991 FMOVD F5, 128(R20)
992 FMOVD F6, 136(R20)
993 FMOVD F7, 144(R20)
994 FMOVD F8, 152(R20)
995 FMOVD F9, 160(R20)
996 FMOVD F10, 168(R20)
997 FMOVD F11, 176(R20)
998 FMOVD F12, 184(R20)
999 RET
1000
1001 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
1002 TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
1003 MOVD 0(R20), R3
1004 MOVD 8(R20), R4
1005 MOVD 16(R20), R5
1006 MOVD 24(R20), R6
1007 MOVD 32(R20), R7
1008 MOVD 40(R20), R8
1009 MOVD 48(R20), R9
1010 MOVD 56(R20), R10
1011 MOVD 64(R20), R14
1012 MOVD 72(R20), R15
1013 MOVD 80(R20), R16
1014 MOVD 88(R20), R17
1015 FMOVD 96(R20), F1
1016 FMOVD 104(R20), F2
1017 FMOVD 112(R20), F3
1018 FMOVD 120(R20), F4
1019 FMOVD 128(R20), F5
1020 FMOVD 136(R20), F6
1021 FMOVD 144(R20), F7
1022 FMOVD 152(R20), F8
1023 FMOVD 160(R20), F9
1024 FMOVD 168(R20), F10
1025 FMOVD 176(R20), F11
1026 FMOVD 184(R20), F12
1027 RET
1028
1029 // AES hashing not implemented for ppc64
1030 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
1031 JMP runtime·memhashFallback<ABIInternal>(SB)
1032 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1033 JMP runtime·strhashFallback<ABIInternal>(SB)
1034 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1035 JMP runtime·memhash32Fallback<ABIInternal>(SB)
1036 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
1037 JMP runtime·memhash64Fallback<ABIInternal>(SB)
1038
1039 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
1040 // Must obey the gcc calling convention.
1041 #ifdef GOOS_aix
1042 // On AIX, _cgo_topofstack is defined in runtime/cgo, because it must
1043 // be a longcall in order to prevent trampolines from ld.
1044 TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
1045 #else
1046 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
1047 #endif
1048 // g (R30) and R31 are callee-save in the C ABI, so save them
1049 MOVD g, R4
1050 MOVD R31, R5
1051 MOVD LR, R6
1052
1053 BL runtime·load_g(SB) // clobbers g (R30), R31
1054 MOVD g_m(g), R3
1055 MOVD m_curg(R3), R3
1056 MOVD (g_stack+stack_hi)(R3), R3
1057
1058 MOVD R4, g
1059 MOVD R5, R31
1060 MOVD R6, LR
1061 RET
1062
1063 // The top-most function running on a goroutine
1064 // returns to goexit+PCQuantum.
1065 //
1066 // When dynamically linking Go, it can be returned to from a function
1067 // implemented in a different module and so needs to reload the TOC pointer
1068 // from the stack (although this function declares that it does not set up x-a
1069 // frame, newproc1 does in fact allocate one for goexit and saves the TOC
1070 // pointer in the correct place).
1071 // goexit+_PCQuantum is halfway through the usual global entry point prologue
1072 // that derives r2 from r12 which is a bit silly, but not harmful.
1073 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
1074 MOVD 24(R1), R2
1075 BL runtime·goexit1(SB) // does not return
1076 // traceback from goexit1 must hit code range of goexit
1077 MOVD R0, R0 // NOP
1078
1079 // prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the
1080 // module containing runtime) to the frame that goexit will execute in when
1081 // the goroutine exits. It's implemented in assembly mainly because that's the
1082 // easiest way to get access to R2.
1083 TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8
1084 MOVD sp+0(FP), R3
1085 MOVD R2, 24(R3)
1086 RET
1087
1088 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
1089 ADD $-8, R1
1090 MOVD R31, 0(R1)
1091 MOVD runtime·lastmoduledatap(SB), R4
1092 MOVD R3, moduledata_next(R4)
1093 MOVD R3, runtime·lastmoduledatap(SB)
1094 MOVD 0(R1), R31
1095 ADD $8, R1
1096 RET
1097
1098 TEXT ·checkASM(SB),NOSPLIT,$0-1
1099 MOVW $1, R3
1100 MOVB R3, ret+0(FP)
1101 RET
1102
1103 // gcWriteBarrier informs the GC about heap pointer writes.
1104 //
1105 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
1106 // number of bytes of buffer needed in R29, and returns a pointer
1107 // to the buffer space in R29.
1108 // It clobbers condition codes.
1109 // It does not clobber R0 through R17 (except special registers),
1110 // but may clobber any other register, *including* R31.
1111 TEXT gcWriteBarrier<>(SB),NOSPLIT,$120
1112 // The standard prologue clobbers R31.
1113 // We use R18, R19, and R31 as scratch registers.
1114 retry:
1115 MOVD g_m(g), R18
1116 MOVD m_p(R18), R18
1117 MOVD (p_wbBuf+wbBuf_next)(R18), R19
1118 MOVD (p_wbBuf+wbBuf_end)(R18), R31
1119 // Increment wbBuf.next position.
1120 ADD R29, R19
1121 // Is the buffer full?
1122 CMPU R31, R19
1123 BLT flush
1124 // Commit to the larger buffer.
1125 MOVD R19, (p_wbBuf+wbBuf_next)(R18)
1126 // Make return value (the original next position)
1127 SUB R29, R19, R29
1128 RET
1129
1130 flush:
1131 // Save registers R0 through R15 since these were not saved by the caller.
1132 // We don't save all registers on ppc64 because it takes too much space.
1133 MOVD R20, (FIXED_FRAME+0)(R1)
1134 MOVD R21, (FIXED_FRAME+8)(R1)
1135 // R0 is always 0, so no need to spill.
1136 // R1 is SP.
1137 // R2 is SB.
1138 MOVD R3, (FIXED_FRAME+16)(R1)
1139 MOVD R4, (FIXED_FRAME+24)(R1)
1140 MOVD R5, (FIXED_FRAME+32)(R1)
1141 MOVD R6, (FIXED_FRAME+40)(R1)
1142 MOVD R7, (FIXED_FRAME+48)(R1)
1143 MOVD R8, (FIXED_FRAME+56)(R1)
1144 MOVD R9, (FIXED_FRAME+64)(R1)
1145 MOVD R10, (FIXED_FRAME+72)(R1)
1146 // R11, R12 may be clobbered by external-linker-inserted trampoline
1147 // R13 is REGTLS
1148 MOVD R14, (FIXED_FRAME+80)(R1)
1149 MOVD R15, (FIXED_FRAME+88)(R1)
1150 MOVD R16, (FIXED_FRAME+96)(R1)
1151 MOVD R17, (FIXED_FRAME+104)(R1)
1152 MOVD R29, (FIXED_FRAME+112)(R1)
1153
1154 CALL runtime·wbBufFlush(SB)
1155
1156 MOVD (FIXED_FRAME+0)(R1), R20
1157 MOVD (FIXED_FRAME+8)(R1), R21
1158 MOVD (FIXED_FRAME+16)(R1), R3
1159 MOVD (FIXED_FRAME+24)(R1), R4
1160 MOVD (FIXED_FRAME+32)(R1), R5
1161 MOVD (FIXED_FRAME+40)(R1), R6
1162 MOVD (FIXED_FRAME+48)(R1), R7
1163 MOVD (FIXED_FRAME+56)(R1), R8
1164 MOVD (FIXED_FRAME+64)(R1), R9
1165 MOVD (FIXED_FRAME+72)(R1), R10
1166 MOVD (FIXED_FRAME+80)(R1), R14
1167 MOVD (FIXED_FRAME+88)(R1), R15
1168 MOVD (FIXED_FRAME+96)(R1), R16
1169 MOVD (FIXED_FRAME+104)(R1), R17
1170 MOVD (FIXED_FRAME+112)(R1), R29
1171 JMP retry
1172
1173 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
1174 MOVD $8, R29
1175 JMP gcWriteBarrier<>(SB)
1176 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
1177 MOVD $16, R29
1178 JMP gcWriteBarrier<>(SB)
1179 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
1180 MOVD $24, R29
1181 JMP gcWriteBarrier<>(SB)
1182 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
1183 MOVD $32, R29
1184 JMP gcWriteBarrier<>(SB)
1185 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
1186 MOVD $40, R29
1187 JMP gcWriteBarrier<>(SB)
1188 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
1189 MOVD $48, R29
1190 JMP gcWriteBarrier<>(SB)
1191 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
1192 MOVD $56, R29
1193 JMP gcWriteBarrier<>(SB)
1194 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
1195 MOVD $64, R29
1196 JMP gcWriteBarrier<>(SB)
1197
1198 DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1199 GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1200
1201 // debugCallV2 is the entry point for debugger-injected function
1202 // calls on running goroutines. It informs the runtime that a
1203 // debug call has been injected and creates a call frame for the
1204 // debugger to fill in.
1205 //
1206 // To inject a function call, a debugger should:
1207 // 1. Check that the goroutine is in state _Grunning and that
1208 // there are at least 320 bytes free on the stack.
1209 // 2. Set SP as SP-32.
1210 // 3. Store the current LR in (SP) (using the SP after step 2).
1211 // 4. Store the current PC in the LR register.
1212 // 5. Write the desired argument frame size at SP-32
1213 // 6. Save all machine registers (including flags and floating point registers)
1214 // so they can be restored later by the debugger.
1215 // 7. Set the PC to debugCallV2 and resume execution.
1216 //
1217 // If the goroutine is in state _Grunnable, then it's not generally
1218 // safe to inject a call because it may return out via other runtime
1219 // operations. Instead, the debugger should unwind the stack to find
1220 // the return to non-runtime code, add a temporary breakpoint there,
1221 // and inject the call once that breakpoint is hit.
1222 //
1223 // If the goroutine is in any other state, it's not safe to inject a call.
1224 //
1225 // This function communicates back to the debugger by setting R20 and
1226 // invoking TW to raise a breakpoint signal. Note that the signal PC of
1227 // the signal triggered by the TW instruction is the PC where the signal
1228 // is trapped, not the next PC, so to resume execution, the debugger needs
1229 // to set the signal PC to PC+4. See the comments in the implementation for
1230 // the protocol the debugger is expected to follow. InjectDebugCall in the
1231 // runtime tests demonstrates this protocol.
1232 // The debugger must ensure that any pointers passed to the function
1233 // obey escape analysis requirements. Specifically, it must not pass
1234 // a stack pointer to an escaping argument. debugCallV2 cannot check
1235 // this invariant.
1236 //
1237 // This is ABIInternal because Go code injects its PC directly into new
1238 // goroutine stacks.
1239 #ifdef GOARCH_ppc64le
1240 TEXT runtime·debugCallV2<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-0
1241 // save scratch register R31 first
1242 MOVD R31, -184(R1)
1243 MOVD 0(R1), R31
1244 // save caller LR
1245 MOVD R31, -304(R1)
1246 MOVD -32(R1), R31
1247 // save argument frame size
1248 MOVD R31, -192(R1)
1249 MOVD LR, R31
1250 MOVD R31, -320(R1)
1251 ADD $-320, R1
1252 // save all registers that can contain pointers
1253 // and the CR register
1254 MOVW CR, R31
1255 MOVD R31, 8(R1)
1256 MOVD R2, 24(R1)
1257 MOVD R3, 56(R1)
1258 MOVD R4, 64(R1)
1259 MOVD R5, 72(R1)
1260 MOVD R6, 80(R1)
1261 MOVD R7, 88(R1)
1262 MOVD R8, 96(R1)
1263 MOVD R9, 104(R1)
1264 MOVD R10, 112(R1)
1265 MOVD R11, 120(R1)
1266 MOVD R12, 144(R1)
1267 MOVD R13, 152(R1)
1268 MOVD R14, 160(R1)
1269 MOVD R15, 168(R1)
1270 MOVD R16, 176(R1)
1271 MOVD R17, 184(R1)
1272 MOVD R18, 192(R1)
1273 MOVD R19, 200(R1)
1274 MOVD R20, 208(R1)
1275 MOVD R21, 216(R1)
1276 MOVD R22, 224(R1)
1277 MOVD R23, 232(R1)
1278 MOVD R24, 240(R1)
1279 MOVD R25, 248(R1)
1280 MOVD R26, 256(R1)
1281 MOVD R27, 264(R1)
1282 MOVD R28, 272(R1)
1283 MOVD R29, 280(R1)
1284 MOVD g, 288(R1)
1285 MOVD LR, R31
1286 MOVD R31, 32(R1)
1287 CALL runtime·debugCallCheck(SB)
1288 MOVD 40(R1), R22
1289 XOR R0, R0
1290 CMP R22, $0
1291 BEQ good
1292 MOVD 48(R1), R22
1293 MOVD $8, R20
1294 TW $31, R0, R0
1295
1296 BR restore
1297
1298 good:
1299 #define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1300 MOVD $MAXSIZE, R23; \
1301 CMP R26, R23; \
1302 BGT 5(PC); \
1303 MOVD $NAME(SB), R26; \
1304 MOVD R26, 32(R1); \
1305 CALL runtime·debugCallWrap(SB); \
1306 BR restore
1307
1308 // the argument frame size
1309 MOVD 128(R1), R26
1310
1311 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1312 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1313 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1314 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1315 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1316 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1317 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1318 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1319 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1320 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1321 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1322 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1323 // The frame size is too large. Report the error.
1324 MOVD $debugCallFrameTooLarge<>(SB), R22
1325 MOVD R22, 32(R1)
1326 MOVD $20, R22
1327 // length of debugCallFrameTooLarge string
1328 MOVD R22, 40(R1)
1329 MOVD $8, R20
1330 TW $31, R0, R0
1331 BR restore
1332 restore:
1333 MOVD $16, R20
1334 TW $31, R0, R0
1335 // restore all registers that can contain
1336 // pointers including CR
1337 MOVD 8(R1), R31
1338 MOVW R31, CR
1339 MOVD 24(R1), R2
1340 MOVD 56(R1), R3
1341 MOVD 64(R1), R4
1342 MOVD 72(R1), R5
1343 MOVD 80(R1), R6
1344 MOVD 88(R1), R7
1345 MOVD 96(R1), R8
1346 MOVD 104(R1), R9
1347 MOVD 112(R1), R10
1348 MOVD 120(R1), R11
1349 MOVD 144(R1), R12
1350 MOVD 152(R1), R13
1351 MOVD 160(R1), R14
1352 MOVD 168(R1), R15
1353 MOVD 176(R1), R16
1354 MOVD 184(R1), R17
1355 MOVD 192(R1), R18
1356 MOVD 200(R1), R19
1357 MOVD 208(R1), R20
1358 MOVD 216(R1), R21
1359 MOVD 224(R1), R22
1360 MOVD 232(R1), R23
1361 MOVD 240(R1), R24
1362 MOVD 248(R1), R25
1363 MOVD 256(R1), R26
1364 MOVD 264(R1), R27
1365 MOVD 272(R1), R28
1366 MOVD 280(R1), R29
1367 MOVD 288(R1), g
1368 MOVD 16(R1), R31
1369 // restore old LR
1370 MOVD R31, LR
1371 // restore caller PC
1372 MOVD 0(R1), CTR
1373 MOVD 136(R1), R31
1374 // Add 32 bytes more to compensate for SP change in saveSigContext
1375 ADD $352, R1
1376 JMP (CTR)
1377 #endif
1378 #define DEBUG_CALL_FN(NAME,MAXSIZE) \
1379 TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1380 NO_LOCAL_POINTERS; \
1381 MOVD $0, R20; \
1382 TW $31, R0, R0 \
1383 MOVD $1, R20; \
1384 TW $31, R0, R0 \
1385 RET
1386 DEBUG_CALL_FN(debugCall32<>, 32)
1387 DEBUG_CALL_FN(debugCall64<>, 64)
1388 DEBUG_CALL_FN(debugCall128<>, 128)
1389 DEBUG_CALL_FN(debugCall256<>, 256)
1390 DEBUG_CALL_FN(debugCall512<>, 512)
1391 DEBUG_CALL_FN(debugCall1024<>, 1024)
1392 DEBUG_CALL_FN(debugCall2048<>, 2048)
1393 DEBUG_CALL_FN(debugCall4096<>, 4096)
1394 DEBUG_CALL_FN(debugCall8192<>, 8192)
1395 DEBUG_CALL_FN(debugCall16384<>, 16384)
1396 DEBUG_CALL_FN(debugCall32768<>, 32768)
1397 DEBUG_CALL_FN(debugCall65536<>, 65536)
1398
1399 #ifdef GOARCH_ppc64le
1400 // func debugCallPanicked(val interface{})
1401 TEXT runtime·debugCallPanicked(SB),NOSPLIT,$32-16
1402 // Copy the panic value to the top of stack at SP+32.
1403 MOVD val_type+0(FP), R31
1404 MOVD R31, 32(R1)
1405 MOVD val_data+8(FP), R31
1406 MOVD R31, 40(R1)
1407 MOVD $2, R20
1408 TW $31, R0, R0
1409 RET
1410 #endif
1411
1412 TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$88-0
1413 // Note: frame size is 16 bytes larger than necessary
1414 // in order to pacify vet. Vet doesn't understand ppc64
1415 // layout properly.
1416 NO_LOCAL_POINTERS
1417 // Save all 7 int registers that could have an index in them.
1418 // They may be pointers, but if so they are dead.
1419 // Skip R0 aka ZERO, R1 aka SP, R2 aka SB
1420 MOVD R3, 48(R1)
1421 MOVD R4, 56(R1)
1422 MOVD R5, 64(R1)
1423 MOVD R6, 72(R1)
1424 MOVD R7, 80(R1)
1425 MOVD R8, 88(R1)
1426 MOVD R9, 96(R1)
1427 // Note: we only save 7 registers to keep under nosplit stack limit
1428 // Also, R11 is clobbered in dynamic linking situations
1429
1430 MOVD LR, R3 // PC immediately after call to panicBounds
1431 ADD $48, R1, R4 // pointer to save area
1432 CALL runtime·panicBounds64<ABIInternal>(SB)
1433 RET
1434
1435 // These functions are used when internal linking cgo with external
1436 // objects compiled with the -Os on gcc. They reduce prologue/epilogue
1437 // size by deferring preservation of callee-save registers to a shared
1438 // function. These are defined in PPC64 ELFv2 2.3.3 (but also present
1439 // in ELFv1)
1440 //
1441 // These appear unused, but the linker will redirect calls to functions
1442 // like _savegpr0_14 or _restgpr1_14 to runtime.elf_savegpr0 or
1443 // runtime.elf_restgpr1 with an appropriate offset based on the number
1444 // register operations required when linking external objects which
1445 // make these calls. For GPR/FPR saves, the minimum register value is
1446 // 14, for VR it is 20.
1447 //
1448 // These are only used when linking such cgo code internally. Note, R12
1449 // and R0 may be used in different ways than regular ELF compliant
1450 // functions.
1451 TEXT runtime·elf_savegpr0(SB),NOSPLIT|NOFRAME,$0
1452 // R0 holds the LR of the caller's caller, R1 holds save location
1453 MOVD R14, -144(R1)
1454 MOVD R15, -136(R1)
1455 MOVD R16, -128(R1)
1456 MOVD R17, -120(R1)
1457 MOVD R18, -112(R1)
1458 MOVD R19, -104(R1)
1459 MOVD R20, -96(R1)
1460 MOVD R21, -88(R1)
1461 MOVD R22, -80(R1)
1462 MOVD R23, -72(R1)
1463 MOVD R24, -64(R1)
1464 MOVD R25, -56(R1)
1465 MOVD R26, -48(R1)
1466 MOVD R27, -40(R1)
1467 MOVD R28, -32(R1)
1468 MOVD R29, -24(R1)
1469 MOVD g, -16(R1)
1470 MOVD R31, -8(R1)
1471 MOVD R0, 16(R1)
1472 RET
1473 TEXT runtime·elf_restgpr0(SB),NOSPLIT|NOFRAME,$0
1474 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1475 MOVD -144(R1), R14
1476 MOVD -136(R1), R15
1477 MOVD -128(R1), R16
1478 MOVD -120(R1), R17
1479 MOVD -112(R1), R18
1480 MOVD -104(R1), R19
1481 MOVD -96(R1), R20
1482 MOVD -88(R1), R21
1483 MOVD -80(R1), R22
1484 MOVD -72(R1), R23
1485 MOVD -64(R1), R24
1486 MOVD -56(R1), R25
1487 MOVD -48(R1), R26
1488 MOVD -40(R1), R27
1489 MOVD -32(R1), R28
1490 MOVD -24(R1), R29
1491 MOVD -16(R1), g
1492 MOVD -8(R1), R31
1493 MOVD 16(R1), R0 // Load and return to saved LR
1494 MOVD R0, LR
1495 RET
1496 TEXT runtime·elf_savegpr1(SB),NOSPLIT|NOFRAME,$0
1497 // R12 holds the save location
1498 MOVD R14, -144(R12)
1499 MOVD R15, -136(R12)
1500 MOVD R16, -128(R12)
1501 MOVD R17, -120(R12)
1502 MOVD R18, -112(R12)
1503 MOVD R19, -104(R12)
1504 MOVD R20, -96(R12)
1505 MOVD R21, -88(R12)
1506 MOVD R22, -80(R12)
1507 MOVD R23, -72(R12)
1508 MOVD R24, -64(R12)
1509 MOVD R25, -56(R12)
1510 MOVD R26, -48(R12)
1511 MOVD R27, -40(R12)
1512 MOVD R28, -32(R12)
1513 MOVD R29, -24(R12)
1514 MOVD g, -16(R12)
1515 MOVD R31, -8(R12)
1516 RET
1517 TEXT runtime·elf_restgpr1(SB),NOSPLIT|NOFRAME,$0
1518 // R12 holds the save location
1519 MOVD -144(R12), R14
1520 MOVD -136(R12), R15
1521 MOVD -128(R12), R16
1522 MOVD -120(R12), R17
1523 MOVD -112(R12), R18
1524 MOVD -104(R12), R19
1525 MOVD -96(R12), R20
1526 MOVD -88(R12), R21
1527 MOVD -80(R12), R22
1528 MOVD -72(R12), R23
1529 MOVD -64(R12), R24
1530 MOVD -56(R12), R25
1531 MOVD -48(R12), R26
1532 MOVD -40(R12), R27
1533 MOVD -32(R12), R28
1534 MOVD -24(R12), R29
1535 MOVD -16(R12), g
1536 MOVD -8(R12), R31
1537 RET
1538 TEXT runtime·elf_savefpr(SB),NOSPLIT|NOFRAME,$0
1539 // R0 holds the LR of the caller's caller, R1 holds save location
1540 FMOVD F14, -144(R1)
1541 FMOVD F15, -136(R1)
1542 FMOVD F16, -128(R1)
1543 FMOVD F17, -120(R1)
1544 FMOVD F18, -112(R1)
1545 FMOVD F19, -104(R1)
1546 FMOVD F20, -96(R1)
1547 FMOVD F21, -88(R1)
1548 FMOVD F22, -80(R1)
1549 FMOVD F23, -72(R1)
1550 FMOVD F24, -64(R1)
1551 FMOVD F25, -56(R1)
1552 FMOVD F26, -48(R1)
1553 FMOVD F27, -40(R1)
1554 FMOVD F28, -32(R1)
1555 FMOVD F29, -24(R1)
1556 FMOVD F30, -16(R1)
1557 FMOVD F31, -8(R1)
1558 MOVD R0, 16(R1)
1559 RET
1560 TEXT runtime·elf_restfpr(SB),NOSPLIT|NOFRAME,$0
1561 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1562 FMOVD -144(R1), F14
1563 FMOVD -136(R1), F15
1564 FMOVD -128(R1), F16
1565 FMOVD -120(R1), F17
1566 FMOVD -112(R1), F18
1567 FMOVD -104(R1), F19
1568 FMOVD -96(R1), F20
1569 FMOVD -88(R1), F21
1570 FMOVD -80(R1), F22
1571 FMOVD -72(R1), F23
1572 FMOVD -64(R1), F24
1573 FMOVD -56(R1), F25
1574 FMOVD -48(R1), F26
1575 FMOVD -40(R1), F27
1576 FMOVD -32(R1), F28
1577 FMOVD -24(R1), F29
1578 FMOVD -16(R1), F30
1579 FMOVD -8(R1), F31
1580 MOVD 16(R1), R0 // Load and return to saved LR
1581 MOVD R0, LR
1582 RET
1583 TEXT runtime·elf_savevr(SB),NOSPLIT|NOFRAME,$0
1584 // R0 holds the save location, R12 is clobbered
1585 MOVD $-192, R12
1586 STVX V20, (R0+R12)
1587 MOVD $-176, R12
1588 STVX V21, (R0+R12)
1589 MOVD $-160, R12
1590 STVX V22, (R0+R12)
1591 MOVD $-144, R12
1592 STVX V23, (R0+R12)
1593 MOVD $-128, R12
1594 STVX V24, (R0+R12)
1595 MOVD $-112, R12
1596 STVX V25, (R0+R12)
1597 MOVD $-96, R12
1598 STVX V26, (R0+R12)
1599 MOVD $-80, R12
1600 STVX V27, (R0+R12)
1601 MOVD $-64, R12
1602 STVX V28, (R0+R12)
1603 MOVD $-48, R12
1604 STVX V29, (R0+R12)
1605 MOVD $-32, R12
1606 STVX V30, (R0+R12)
1607 MOVD $-16, R12
1608 STVX V31, (R0+R12)
1609 RET
1610 TEXT runtime·elf_restvr(SB),NOSPLIT|NOFRAME,$0
1611 // R0 holds the save location, R12 is clobbered
1612 MOVD $-192, R12
1613 LVX (R0+R12), V20
1614 MOVD $-176, R12
1615 LVX (R0+R12), V21
1616 MOVD $-160, R12
1617 LVX (R0+R12), V22
1618 MOVD $-144, R12
1619 LVX (R0+R12), V23
1620 MOVD $-128, R12
1621 LVX (R0+R12), V24
1622 MOVD $-112, R12
1623 LVX (R0+R12), V25
1624 MOVD $-96, R12
1625 LVX (R0+R12), V26
1626 MOVD $-80, R12
1627 LVX (R0+R12), V27
1628 MOVD $-64, R12
1629 LVX (R0+R12), V28
1630 MOVD $-48, R12
1631 LVX (R0+R12), V29
1632 MOVD $-32, R12
1633 LVX (R0+R12), V30
1634 MOVD $-16, R12
1635 LVX (R0+R12), V31
1636 RET
1637
View as plain text