Text file
src/runtime/asm_s390x.s
1 // Copyright 2016 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "funcdata.h"
8 #include "textflag.h"
9
10 // _rt0_s390x_lib is common startup code for s390x systems when
11 // using -buildmode=c-archive or -buildmode=c-shared. The linker will
12 // arrange to invoke this function as a global constructor (for
13 // c-archive) or when the shared library is loaded (for c-shared).
14 // We expect argc and argv to be passed in the usual C ABI registers
15 // R2 and R3.
16 TEXT _rt0_s390x_lib(SB), NOSPLIT|NOFRAME, $0
17 STMG R6, R15, 48(R15)
18 MOVD R2, _rt0_s390x_lib_argc<>(SB)
19 MOVD R3, _rt0_s390x_lib_argv<>(SB)
20
21 // Save R6-R15 in the register save area of the calling function.
22 STMG R6, R15, 48(R15)
23
24 // Allocate 80 bytes on the stack.
25 MOVD $-80(R15), R15
26
27 // Save F8-F15 in our stack frame.
28 FMOVD F8, 16(R15)
29 FMOVD F9, 24(R15)
30 FMOVD F10, 32(R15)
31 FMOVD F11, 40(R15)
32 FMOVD F12, 48(R15)
33 FMOVD F13, 56(R15)
34 FMOVD F14, 64(R15)
35 FMOVD F15, 72(R15)
36
37 // Synchronous initialization.
38 MOVD $runtime·libpreinit(SB), R1
39 BL R1
40
41 // Create a new thread to finish Go runtime initialization.
42 MOVD _cgo_sys_thread_create(SB), R1
43 CMP R1, $0
44 BEQ nocgo
45 MOVD $_rt0_s390x_lib_go(SB), R2
46 MOVD $0, R3
47 BL R1
48 BR restore
49
50 nocgo:
51 MOVD $0x800000, R1 // stacksize
52 MOVD R1, 0(R15)
53 MOVD $_rt0_s390x_lib_go(SB), R1
54 MOVD R1, 8(R15) // fn
55 MOVD $runtime·newosproc(SB), R1
56 BL R1
57
58 restore:
59 // Restore F8-F15 from our stack frame.
60 FMOVD 16(R15), F8
61 FMOVD 24(R15), F9
62 FMOVD 32(R15), F10
63 FMOVD 40(R15), F11
64 FMOVD 48(R15), F12
65 FMOVD 56(R15), F13
66 FMOVD 64(R15), F14
67 FMOVD 72(R15), F15
68 MOVD $80(R15), R15
69
70 // Restore R6-R15.
71 LMG 48(R15), R6, R15
72 RET
73
74 // _rt0_s390x_lib_go initializes the Go runtime.
75 // This is started in a separate thread by _rt0_s390x_lib.
76 TEXT _rt0_s390x_lib_go(SB), NOSPLIT|NOFRAME, $0
77 MOVD _rt0_s390x_lib_argc<>(SB), R2
78 MOVD _rt0_s390x_lib_argv<>(SB), R3
79 MOVD $runtime·rt0_go(SB), R1
80 BR R1
81
82 DATA _rt0_s390x_lib_argc<>(SB)/8, $0
83 GLOBL _rt0_s390x_lib_argc<>(SB), NOPTR, $8
84 DATA _rt0_s90x_lib_argv<>(SB)/8, $0
85 GLOBL _rt0_s390x_lib_argv<>(SB), NOPTR, $8
86
87 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
88 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
89 // C TLS base pointer in AR0:AR1
90
91 // initialize essential registers
92 XOR R0, R0
93
94 SUB $24, R15
95 MOVW R2, 8(R15) // argc
96 MOVD R3, 16(R15) // argv
97
98 // create istack out of the given (operating system) stack.
99 // _cgo_init may update stackguard.
100 MOVD $runtime·g0(SB), g
101 MOVD R15, R11
102 SUB $(64*1024), R11
103 MOVD R11, g_stackguard0(g)
104 MOVD R11, g_stackguard1(g)
105 MOVD R11, (g_stack+stack_lo)(g)
106 MOVD R15, (g_stack+stack_hi)(g)
107
108 // if there is a _cgo_init, call it using the gcc ABI.
109 MOVD _cgo_init(SB), R11
110 CMPBEQ R11, $0, nocgo
111 MOVW AR0, R4 // (AR0 << 32 | AR1) is the TLS base pointer; MOVD is translated to EAR
112 SLD $32, R4, R4
113 MOVW AR1, R4 // arg 2: TLS base pointer
114 MOVD $setg_gcc<>(SB), R3 // arg 1: setg
115 MOVD g, R2 // arg 0: G
116 // C functions expect 160 bytes of space on caller stack frame
117 // and an 8-byte aligned stack pointer
118 MOVD R15, R9 // save current stack (R9 is preserved in the Linux ABI)
119 SUB $160, R15 // reserve 160 bytes
120 MOVD $~7, R6
121 AND R6, R15 // 8-byte align
122 BL R11 // this call clobbers volatile registers according to Linux ABI (R0-R5, R14)
123 MOVD R9, R15 // restore stack
124 XOR R0, R0 // zero R0
125
126 nocgo:
127 // update stackguard after _cgo_init
128 MOVD (g_stack+stack_lo)(g), R2
129 ADD $const_stackGuard, R2
130 MOVD R2, g_stackguard0(g)
131 MOVD R2, g_stackguard1(g)
132
133 // set the per-goroutine and per-mach "registers"
134 MOVD $runtime·m0(SB), R2
135
136 // save m->g0 = g0
137 MOVD g, m_g0(R2)
138 // save m0 to g0->m
139 MOVD R2, g_m(g)
140
141 BL runtime·check(SB)
142
143 // argc/argv are already prepared on stack
144 BL runtime·args(SB)
145 BL runtime·checkS390xCPU(SB)
146 BL runtime·osinit(SB)
147 BL runtime·schedinit(SB)
148
149 // create a new goroutine to start program
150 MOVD $runtime·mainPC(SB), R2 // entry
151 SUB $16, R15
152 MOVD R2, 8(R15)
153 MOVD $0, 0(R15)
154 BL runtime·newproc(SB)
155 ADD $16, R15
156
157 // start this M
158 BL runtime·mstart(SB)
159
160 MOVD $0, 1(R0)
161 RET
162
163 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB)
164 GLOBL runtime·mainPC(SB),RODATA,$8
165
166 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
167 BRRK
168 RET
169
170 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
171 RET
172
173 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
174 CALL runtime·mstart0(SB)
175 RET // not reached
176
177 /*
178 * go-routine
179 */
180
181 // void gogo(Gobuf*)
182 // restore state from Gobuf; longjmp
183 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
184 MOVD buf+0(FP), R5
185 MOVD gobuf_g(R5), R6
186 MOVD 0(R6), R7 // make sure g != nil
187 BR gogo<>(SB)
188
189 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
190 MOVD R6, g
191 BL runtime·save_g(SB)
192
193 MOVD 0(g), R4
194 MOVD gobuf_sp(R5), R15
195 MOVD gobuf_lr(R5), LR
196 MOVD gobuf_ctxt(R5), R12
197 MOVD $0, gobuf_sp(R5)
198 MOVD $0, gobuf_lr(R5)
199 MOVD $0, gobuf_ctxt(R5)
200 CMP R0, R0 // set condition codes for == test, needed by stack split
201 MOVD gobuf_pc(R5), R6
202 BR (R6)
203
204 // void mcall(fn func(*g))
205 // Switch to m->g0's stack, call fn(g).
206 // Fn must never return. It should gogo(&g->sched)
207 // to keep running g.
208 TEXT runtime·mcall(SB), NOSPLIT, $-8-8
209 // Save caller state in g->sched
210 MOVD R15, (g_sched+gobuf_sp)(g)
211 MOVD LR, (g_sched+gobuf_pc)(g)
212 MOVD $0, (g_sched+gobuf_lr)(g)
213
214 // Switch to m->g0 & its stack, call fn.
215 MOVD g, R3
216 MOVD g_m(g), R8
217 MOVD m_g0(R8), g
218 BL runtime·save_g(SB)
219 CMP g, R3
220 BNE 2(PC)
221 BR runtime·badmcall(SB)
222 MOVD fn+0(FP), R12 // context
223 MOVD 0(R12), R4 // code pointer
224 MOVD (g_sched+gobuf_sp)(g), R15 // sp = m->g0->sched.sp
225 SUB $16, R15
226 MOVD R3, 8(R15)
227 MOVD $0, 0(R15)
228 BL (R4)
229 BR runtime·badmcall2(SB)
230
231 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
232 // of the G stack. We need to distinguish the routine that
233 // lives at the bottom of the G stack from the one that lives
234 // at the top of the system stack because the one at the top of
235 // the system stack terminates the stack walk (see topofstack()).
236 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
237 UNDEF
238 BL (LR) // make sure this function is not leaf
239 RET
240
241 // func systemstack(fn func())
242 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
243 MOVD fn+0(FP), R3 // R3 = fn
244 MOVD R3, R12 // context
245 MOVD g_m(g), R4 // R4 = m
246
247 MOVD m_gsignal(R4), R5 // R5 = gsignal
248 CMPBEQ g, R5, noswitch
249
250 MOVD m_g0(R4), R5 // R5 = g0
251 CMPBEQ g, R5, noswitch
252
253 MOVD m_curg(R4), R6
254 CMPBEQ g, R6, switch
255
256 // Bad: g is not gsignal, not g0, not curg. What is it?
257 // Hide call from linker nosplit analysis.
258 MOVD $runtime·badsystemstack(SB), R3
259 BL (R3)
260 BL runtime·abort(SB)
261
262 switch:
263 // save our state in g->sched. Pretend to
264 // be systemstack_switch if the G stack is scanned.
265 BL gosave_systemstack_switch<>(SB)
266
267 // switch to g0
268 MOVD R5, g
269 BL runtime·save_g(SB)
270 MOVD (g_sched+gobuf_sp)(g), R15
271
272 // call target function
273 MOVD 0(R12), R3 // code pointer
274 BL (R3)
275
276 // switch back to g
277 MOVD g_m(g), R3
278 MOVD m_curg(R3), g
279 BL runtime·save_g(SB)
280 MOVD (g_sched+gobuf_sp)(g), R15
281 MOVD $0, (g_sched+gobuf_sp)(g)
282 RET
283
284 noswitch:
285 // already on m stack, just call directly
286 // Using a tail call here cleans up tracebacks since we won't stop
287 // at an intermediate systemstack.
288 MOVD 0(R12), R3 // code pointer
289 MOVD 0(R15), LR // restore LR
290 ADD $8, R15
291 BR (R3)
292
293 // func switchToCrashStack0(fn func())
294 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
295 MOVD fn+0(FP), R12 // context
296 MOVD g_m(g), R4 // curm
297
298 // set g to gcrash
299 MOVD $runtime·gcrash(SB), g // g = &gcrash
300 BL runtime·save_g(SB)
301 MOVD R4, g_m(g) // g.m = curm
302 MOVD g, m_g0(R4) // curm.g0 = g
303
304 // switch to crashstack
305 MOVD (g_stack+stack_hi)(g), R4
306 ADD $(-4*8), R4, R15
307
308 // call target function
309 MOVD 0(R12), R3 // code pointer
310 BL (R3)
311
312 // should never return
313 BL runtime·abort(SB)
314 UNDEF
315
316 /*
317 * support for morestack
318 */
319
320 // Called during function prolog when more stack is needed.
321 // Caller has already loaded:
322 // R3: framesize, R4: argsize, R5: LR
323 //
324 // The traceback routines see morestack on a g0 as being
325 // the top of a stack (for example, morestack calling newstack
326 // calling the scheduler calling newm calling gc), so we must
327 // record an argument size. For that purpose, it has no arguments.
328 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
329 // Called from f.
330 // Set g->sched to context in f.
331 MOVD R15, (g_sched+gobuf_sp)(g)
332 MOVD LR, R8
333 MOVD R8, (g_sched+gobuf_pc)(g)
334 MOVD R5, (g_sched+gobuf_lr)(g)
335 MOVD R12, (g_sched+gobuf_ctxt)(g)
336
337 // Cannot grow scheduler stack (m->g0).
338 MOVD g_m(g), R7
339 MOVD m_g0(R7), R8
340 CMPBNE g, R8, 3(PC)
341 BL runtime·badmorestackg0(SB)
342 BL runtime·abort(SB)
343
344 // Cannot grow signal stack (m->gsignal).
345 MOVD m_gsignal(R7), R8
346 CMP g, R8
347 BNE 3(PC)
348 BL runtime·badmorestackgsignal(SB)
349 BL runtime·abort(SB)
350
351 // Called from f.
352 // Set m->morebuf to f's caller.
353 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
354 MOVD R15, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
355 MOVD g, (m_morebuf+gobuf_g)(R7)
356
357 // Call newstack on m->g0's stack.
358 MOVD m_g0(R7), g
359 BL runtime·save_g(SB)
360 MOVD (g_sched+gobuf_sp)(g), R15
361 // Create a stack frame on g0 to call newstack.
362 MOVD $0, -8(R15) // Zero saved LR in frame
363 SUB $8, R15
364 BL runtime·newstack(SB)
365
366 // Not reached, but make sure the return PC from the call to newstack
367 // is still in this function, and not the beginning of the next.
368 UNDEF
369
370 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
371 // Force SPWRITE. This function doesn't actually write SP,
372 // but it is called with a special calling convention where
373 // the caller doesn't save LR on stack but passes it as a
374 // register (R5), and the unwinder currently doesn't understand.
375 // Make it SPWRITE to stop unwinding. (See issue 54332)
376 MOVD R15, R15
377
378 MOVD $0, R12
379 BR runtime·morestack(SB)
380
381 // reflectcall: call a function with the given argument list
382 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
383 // we don't have variable-sized frames, so we use a small number
384 // of constant-sized-frame functions to encode a few bits of size in the pc.
385 // Caution: ugly multiline assembly macros in your future!
386
387 #define DISPATCH(NAME,MAXSIZE) \
388 MOVD $MAXSIZE, R4; \
389 CMP R3, R4; \
390 BGT 3(PC); \
391 MOVD $NAME(SB), R5; \
392 BR (R5)
393 // Note: can't just "BR NAME(SB)" - bad inlining results.
394
395 TEXT ·reflectcall(SB), NOSPLIT, $-8-48
396 MOVWZ frameSize+32(FP), R3
397 DISPATCH(runtime·call16, 16)
398 DISPATCH(runtime·call32, 32)
399 DISPATCH(runtime·call64, 64)
400 DISPATCH(runtime·call128, 128)
401 DISPATCH(runtime·call256, 256)
402 DISPATCH(runtime·call512, 512)
403 DISPATCH(runtime·call1024, 1024)
404 DISPATCH(runtime·call2048, 2048)
405 DISPATCH(runtime·call4096, 4096)
406 DISPATCH(runtime·call8192, 8192)
407 DISPATCH(runtime·call16384, 16384)
408 DISPATCH(runtime·call32768, 32768)
409 DISPATCH(runtime·call65536, 65536)
410 DISPATCH(runtime·call131072, 131072)
411 DISPATCH(runtime·call262144, 262144)
412 DISPATCH(runtime·call524288, 524288)
413 DISPATCH(runtime·call1048576, 1048576)
414 DISPATCH(runtime·call2097152, 2097152)
415 DISPATCH(runtime·call4194304, 4194304)
416 DISPATCH(runtime·call8388608, 8388608)
417 DISPATCH(runtime·call16777216, 16777216)
418 DISPATCH(runtime·call33554432, 33554432)
419 DISPATCH(runtime·call67108864, 67108864)
420 DISPATCH(runtime·call134217728, 134217728)
421 DISPATCH(runtime·call268435456, 268435456)
422 DISPATCH(runtime·call536870912, 536870912)
423 DISPATCH(runtime·call1073741824, 1073741824)
424 MOVD $runtime·badreflectcall(SB), R5
425 BR (R5)
426
427 #define CALLFN(NAME,MAXSIZE) \
428 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
429 NO_LOCAL_POINTERS; \
430 /* copy arguments to stack */ \
431 MOVD stackArgs+16(FP), R4; \
432 MOVWZ stackArgsSize+24(FP), R5; \
433 MOVD $stack-MAXSIZE(SP), R6; \
434 loopArgs: /* copy 256 bytes at a time */ \
435 CMP R5, $256; \
436 BLT tailArgs; \
437 SUB $256, R5; \
438 MVC $256, 0(R4), 0(R6); \
439 MOVD $256(R4), R4; \
440 MOVD $256(R6), R6; \
441 BR loopArgs; \
442 tailArgs: /* copy remaining bytes */ \
443 CMP R5, $0; \
444 BEQ callFunction; \
445 SUB $1, R5; \
446 EXRL $callfnMVC<>(SB), R5; \
447 callFunction: \
448 MOVD f+8(FP), R12; \
449 MOVD (R12), R8; \
450 PCDATA $PCDATA_StackMapIndex, $0; \
451 BL (R8); \
452 /* copy return values back */ \
453 MOVD stackArgsType+0(FP), R7; \
454 MOVD stackArgs+16(FP), R6; \
455 MOVWZ stackArgsSize+24(FP), R5; \
456 MOVD $stack-MAXSIZE(SP), R4; \
457 MOVWZ stackRetOffset+28(FP), R1; \
458 ADD R1, R4; \
459 ADD R1, R6; \
460 SUB R1, R5; \
461 BL callRet<>(SB); \
462 RET
463
464 // callRet copies return values back at the end of call*. This is a
465 // separate function so it can allocate stack space for the arguments
466 // to reflectcallmove. It does not follow the Go ABI; it expects its
467 // arguments in registers.
468 TEXT callRet<>(SB), NOSPLIT, $40-0
469 MOVD R7, 8(R15)
470 MOVD R6, 16(R15)
471 MOVD R4, 24(R15)
472 MOVD R5, 32(R15)
473 MOVD $0, 40(R15)
474 BL runtime·reflectcallmove(SB)
475 RET
476
477 CALLFN(·call16, 16)
478 CALLFN(·call32, 32)
479 CALLFN(·call64, 64)
480 CALLFN(·call128, 128)
481 CALLFN(·call256, 256)
482 CALLFN(·call512, 512)
483 CALLFN(·call1024, 1024)
484 CALLFN(·call2048, 2048)
485 CALLFN(·call4096, 4096)
486 CALLFN(·call8192, 8192)
487 CALLFN(·call16384, 16384)
488 CALLFN(·call32768, 32768)
489 CALLFN(·call65536, 65536)
490 CALLFN(·call131072, 131072)
491 CALLFN(·call262144, 262144)
492 CALLFN(·call524288, 524288)
493 CALLFN(·call1048576, 1048576)
494 CALLFN(·call2097152, 2097152)
495 CALLFN(·call4194304, 4194304)
496 CALLFN(·call8388608, 8388608)
497 CALLFN(·call16777216, 16777216)
498 CALLFN(·call33554432, 33554432)
499 CALLFN(·call67108864, 67108864)
500 CALLFN(·call134217728, 134217728)
501 CALLFN(·call268435456, 268435456)
502 CALLFN(·call536870912, 536870912)
503 CALLFN(·call1073741824, 1073741824)
504
505 // Not a function: target for EXRL (execute relative long) instruction.
506 TEXT callfnMVC<>(SB),NOSPLIT|NOFRAME,$0-0
507 MVC $1, 0(R4), 0(R6)
508
509 TEXT runtime·procyield(SB),NOSPLIT,$0-0
510 RET
511
512 // Save state of caller into g->sched,
513 // but using fake PC from systemstack_switch.
514 // Must only be called from functions with no locals ($0)
515 // or else unwinding from systemstack_switch is incorrect.
516 // Smashes R1.
517 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
518 MOVD $runtime·systemstack_switch(SB), R1
519 ADD $16, R1 // get past prologue
520 MOVD R1, (g_sched+gobuf_pc)(g)
521 MOVD R15, (g_sched+gobuf_sp)(g)
522 MOVD $0, (g_sched+gobuf_lr)(g)
523 // Assert ctxt is zero. See func save.
524 MOVD (g_sched+gobuf_ctxt)(g), R1
525 CMPBEQ R1, $0, 2(PC)
526 BL runtime·abort(SB)
527 RET
528
529 // func asmcgocall(fn, arg unsafe.Pointer) int32
530 // Call fn(arg) on the scheduler stack,
531 // aligned appropriately for the gcc ABI.
532 // See cgocall.go for more details.
533 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
534 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
535 // C TLS base pointer in AR0:AR1
536 MOVD fn+0(FP), R3
537 MOVD arg+8(FP), R4
538
539 MOVD R15, R2 // save original stack pointer
540 MOVD g, R5
541
542 // Figure out if we need to switch to m->g0 stack.
543 // We get called to create new OS threads too, and those
544 // come in on the m->g0 stack already. Or we might already
545 // be on the m->gsignal stack.
546 MOVD g_m(g), R6
547 MOVD m_gsignal(R6), R7
548 CMPBEQ R7, g, g0
549 MOVD m_g0(R6), R7
550 CMPBEQ R7, g, g0
551 BL gosave_systemstack_switch<>(SB)
552 MOVD R7, g
553 BL runtime·save_g(SB)
554 MOVD (g_sched+gobuf_sp)(g), R15
555
556 // Now on a scheduling stack (a pthread-created stack).
557 g0:
558 // Save room for two of our pointers, plus 160 bytes of callee
559 // save area that lives on the caller stack.
560 SUB $176, R15
561 MOVD $~7, R6
562 AND R6, R15 // 8-byte alignment for gcc ABI
563 MOVD R5, 168(R15) // save old g on stack
564 MOVD (g_stack+stack_hi)(R5), R5
565 SUB R2, R5
566 MOVD R5, 160(R15) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
567 MOVD $0, 0(R15) // clear back chain pointer (TODO can we give it real back trace information?)
568 MOVD R4, R2 // arg in R2
569 BL R3 // can clobber: R0-R5, R14, F0-F3, F5, F7-F15
570
571 XOR R0, R0 // set R0 back to 0.
572 // Restore g, stack pointer.
573 MOVD 168(R15), g
574 BL runtime·save_g(SB)
575 MOVD (g_stack+stack_hi)(g), R5
576 MOVD 160(R15), R6
577 SUB R6, R5
578 MOVD R5, R15
579
580 MOVW R2, ret+16(FP)
581 RET
582
583 // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
584 // See cgocall.go for more details.
585 TEXT ·cgocallback(SB),NOSPLIT,$24-24
586 NO_LOCAL_POINTERS
587
588 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
589 // It is used to dropm while thread is exiting.
590 MOVD fn+0(FP), R1
591 CMPBNE R1, $0, loadg
592 // Restore the g from frame.
593 MOVD frame+8(FP), g
594 BR dropm
595
596 loadg:
597 // Load m and g from thread-local storage.
598 MOVB runtime·iscgo(SB), R3
599 CMPBEQ R3, $0, nocgo
600 BL runtime·load_g(SB)
601
602 nocgo:
603 // If g is nil, Go did not create the current thread,
604 // or if this thread never called into Go on pthread platforms.
605 // Call needm to obtain one for temporary use.
606 // In this case, we're running on the thread stack, so there's
607 // lots of space, but the linker doesn't know. Hide the call from
608 // the linker analysis by using an indirect call.
609 CMPBEQ g, $0, needm
610
611 MOVD g_m(g), R8
612 MOVD R8, savedm-8(SP)
613 BR havem
614
615 needm:
616 MOVD g, savedm-8(SP) // g is zero, so is m.
617 MOVD $runtime·needAndBindM(SB), R3
618 BL (R3)
619
620 // Set m->sched.sp = SP, so that if a panic happens
621 // during the function we are about to execute, it will
622 // have a valid SP to run on the g0 stack.
623 // The next few lines (after the havem label)
624 // will save this SP onto the stack and then write
625 // the same SP back to m->sched.sp. That seems redundant,
626 // but if an unrecovered panic happens, unwindm will
627 // restore the g->sched.sp from the stack location
628 // and then systemstack will try to use it. If we don't set it here,
629 // that restored SP will be uninitialized (typically 0) and
630 // will not be usable.
631 MOVD g_m(g), R8
632 MOVD m_g0(R8), R3
633 MOVD R15, (g_sched+gobuf_sp)(R3)
634
635 havem:
636 // Now there's a valid m, and we're running on its m->g0.
637 // Save current m->g0->sched.sp on stack and then set it to SP.
638 // Save current sp in m->g0->sched.sp in preparation for
639 // switch back to m->curg stack.
640 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
641 MOVD m_g0(R8), R3
642 MOVD (g_sched+gobuf_sp)(R3), R4
643 MOVD R4, savedsp-24(SP) // must match frame size
644 MOVD R15, (g_sched+gobuf_sp)(R3)
645
646 // Switch to m->curg stack and call runtime.cgocallbackg.
647 // Because we are taking over the execution of m->curg
648 // but *not* resuming what had been running, we need to
649 // save that information (m->curg->sched) so we can restore it.
650 // We can restore m->curg->sched.sp easily, because calling
651 // runtime.cgocallbackg leaves SP unchanged upon return.
652 // To save m->curg->sched.pc, we push it onto the curg stack and
653 // open a frame the same size as cgocallback's g0 frame.
654 // Once we switch to the curg stack, the pushed PC will appear
655 // to be the return PC of cgocallback, so that the traceback
656 // will seamlessly trace back into the earlier calls.
657 MOVD m_curg(R8), g
658 BL runtime·save_g(SB)
659 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
660 MOVD (g_sched+gobuf_pc)(g), R5
661 MOVD R5, -(24+8)(R4) // "saved LR"; must match frame size
662 // Gather our arguments into registers.
663 MOVD fn+0(FP), R1
664 MOVD frame+8(FP), R2
665 MOVD ctxt+16(FP), R3
666 MOVD $-(24+8)(R4), R15 // switch stack; must match frame size
667 MOVD R1, 8(R15)
668 MOVD R2, 16(R15)
669 MOVD R3, 24(R15)
670 BL runtime·cgocallbackg(SB)
671
672 // Restore g->sched (== m->curg->sched) from saved values.
673 MOVD 0(R15), R5
674 MOVD R5, (g_sched+gobuf_pc)(g)
675 MOVD $(24+8)(R15), R4 // must match frame size
676 MOVD R4, (g_sched+gobuf_sp)(g)
677
678 // Switch back to m->g0's stack and restore m->g0->sched.sp.
679 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
680 // so we do not have to restore it.)
681 MOVD g_m(g), R8
682 MOVD m_g0(R8), g
683 BL runtime·save_g(SB)
684 MOVD (g_sched+gobuf_sp)(g), R15
685 MOVD savedsp-24(SP), R4 // must match frame size
686 MOVD R4, (g_sched+gobuf_sp)(g)
687
688 // If the m on entry was nil, we called needm above to borrow an m,
689 // 1. for the duration of the call on non-pthread platforms,
690 // 2. or the duration of the C thread alive on pthread platforms.
691 // If the m on entry wasn't nil,
692 // 1. the thread might be a Go thread,
693 // 2. or it wasn't the first call from a C thread on pthread platforms,
694 // since then we skip dropm to reuse the m in the first call.
695 MOVD savedm-8(SP), R6
696 CMPBNE R6, $0, droppedm
697
698 // Skip dropm to reuse it in the next call, when a pthread key has been created.
699 MOVD _cgo_pthread_key_created(SB), R6
700 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
701 CMPBEQ R6, $0, dropm
702 MOVD (R6), R6
703 CMPBNE R6, $0, droppedm
704
705 dropm:
706 MOVD $runtime·dropm(SB), R3
707 BL (R3)
708 droppedm:
709
710 // Done!
711 RET
712
713 // void setg(G*); set g. for use by needm.
714 TEXT runtime·setg(SB), NOSPLIT, $0-8
715 MOVD gg+0(FP), g
716 // This only happens if iscgo, so jump straight to save_g
717 BL runtime·save_g(SB)
718 RET
719
720 // void setg_gcc(G*); set g in C TLS.
721 // Must obey the gcc calling convention.
722 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
723 // The standard prologue clobbers LR (R14), which is callee-save in
724 // the C ABI, so we have to use NOFRAME and save LR ourselves.
725 MOVD LR, R1
726 // Also save g, R10, and R11 since they're callee-save in C ABI
727 MOVD R10, R3
728 MOVD g, R4
729 MOVD R11, R5
730
731 MOVD R2, g
732 BL runtime·save_g(SB)
733
734 MOVD R5, R11
735 MOVD R4, g
736 MOVD R3, R10
737 MOVD R1, LR
738 RET
739
740 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
741 MOVW (R0), R0
742 UNDEF
743
744 // int64 runtime·cputicks(void)
745 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
746 // The TOD clock on s390 counts from the year 1900 in ~250ps intervals.
747 // This means that since about 1972 the msb has been set, making the
748 // result of a call to STORE CLOCK (stck) a negative number.
749 // We clear the msb to make it positive.
750 STCK ret+0(FP) // serialises before and after call
751 MOVD ret+0(FP), R3 // R3 will wrap to 0 in the year 2043
752 SLD $1, R3
753 SRD $1, R3
754 MOVD R3, ret+0(FP)
755 RET
756
757 // AES hashing not implemented for s390x
758 TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32
759 JMP runtime·memhashFallback(SB)
760 TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24
761 JMP runtime·strhashFallback(SB)
762 TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24
763 JMP runtime·memhash32Fallback(SB)
764 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24
765 JMP runtime·memhash64Fallback(SB)
766
767 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
768 // Must obey the gcc calling convention.
769 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
770 // g (R13), R10, R11 and LR (R14) are callee-save in the C ABI, so save them
771 MOVD g, R1
772 MOVD R10, R3
773 MOVD LR, R4
774 MOVD R11, R5
775
776 BL runtime·load_g(SB) // clobbers g (R13), R10, R11
777 MOVD g_m(g), R2
778 MOVD m_curg(R2), R2
779 MOVD (g_stack+stack_hi)(R2), R2
780
781 MOVD R1, g
782 MOVD R3, R10
783 MOVD R4, LR
784 MOVD R5, R11
785 RET
786
787 // The top-most function running on a goroutine
788 // returns to goexit+PCQuantum.
789 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
790 BYTE $0x07; BYTE $0x00; // 2-byte nop
791 BL runtime·goexit1(SB) // does not return
792 // traceback from goexit1 must hit code range of goexit
793 BYTE $0x07; BYTE $0x00; // 2-byte nop
794
795 TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
796 // Stores are already ordered on s390x, so this is just a
797 // compile barrier.
798 RET
799
800 // This is called from .init_array and follows the platform, not Go, ABI.
801 // We are overly conservative. We could only save the registers we use.
802 // However, since this function is only called once per loaded module
803 // performance is unimportant.
804 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
805 // Save R6-R15 in the register save area of the calling function.
806 // Don't bother saving F8-F15 as we aren't doing any calls.
807 STMG R6, R15, 48(R15)
808
809 // append the argument (passed in R2, as per the ELF ABI) to the
810 // moduledata linked list.
811 MOVD runtime·lastmoduledatap(SB), R1
812 MOVD R2, moduledata_next(R1)
813 MOVD R2, runtime·lastmoduledatap(SB)
814
815 // Restore R6-R15.
816 LMG 48(R15), R6, R15
817 RET
818
819 TEXT ·checkASM(SB),NOSPLIT,$0-1
820 MOVB $1, ret+0(FP)
821 RET
822
823 // gcWriteBarrier informs the GC about heap pointer writes.
824 //
825 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
826 // number of bytes of buffer needed in R9, and returns a pointer
827 // to the buffer space in R9.
828 // It clobbers R10 (the temp register) and R1 (used by PLT stub).
829 // It does not clobber any other general-purpose registers,
830 // but may clobber others (e.g., floating point registers).
831 TEXT gcWriteBarrier<>(SB),NOSPLIT,$96
832 // Save the registers clobbered by the fast path.
833 MOVD R4, 96(R15)
834 retry:
835 MOVD g_m(g), R1
836 MOVD m_p(R1), R1
837 // Increment wbBuf.next position.
838 MOVD R9, R4
839 ADD (p_wbBuf+wbBuf_next)(R1), R4
840 // Is the buffer full?
841 MOVD (p_wbBuf+wbBuf_end)(R1), R10
842 CMPUBGT R4, R10, flush
843 // Commit to the larger buffer.
844 MOVD R4, (p_wbBuf+wbBuf_next)(R1)
845 // Make return value (the original next position)
846 SUB R9, R4, R9
847 // Restore registers.
848 MOVD 96(R15), R4
849 RET
850
851 flush:
852 // Save all general purpose registers since these could be
853 // clobbered by wbBufFlush and were not saved by the caller.
854 STMG R2, R3, 8(R15)
855 MOVD R0, 24(R15)
856 // R1 already saved.
857 // R4 already saved.
858 STMG R5, R12, 32(R15) // save R5 - R12
859 // R13 is g.
860 // R14 is LR.
861 // R15 is SP.
862
863 CALL runtime·wbBufFlush(SB)
864
865 LMG 8(R15), R2, R3 // restore R2 - R3
866 MOVD 24(R15), R0 // restore R0
867 LMG 32(R15), R5, R12 // restore R5 - R12
868 JMP retry
869
870 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
871 MOVD $8, R9
872 JMP gcWriteBarrier<>(SB)
873 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
874 MOVD $16, R9
875 JMP gcWriteBarrier<>(SB)
876 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
877 MOVD $24, R9
878 JMP gcWriteBarrier<>(SB)
879 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
880 MOVD $32, R9
881 JMP gcWriteBarrier<>(SB)
882 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
883 MOVD $40, R9
884 JMP gcWriteBarrier<>(SB)
885 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
886 MOVD $48, R9
887 JMP gcWriteBarrier<>(SB)
888 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
889 MOVD $56, R9
890 JMP gcWriteBarrier<>(SB)
891 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
892 MOVD $64, R9
893 JMP gcWriteBarrier<>(SB)
894
895 // Note: these functions use a special calling convention to save generated code space.
896 // Arguments are passed in registers, but the space for those arguments are allocated
897 // in the caller's stack frame. These stubs write the args into that stack space and
898 // then tail call to the corresponding runtime handler.
899 // The tail call makes these stubs disappear in backtraces.
900 TEXT runtime·panicIndex(SB),NOSPLIT,$0-16
901 MOVD R0, x+0(FP)
902 MOVD R1, y+8(FP)
903 JMP runtime·goPanicIndex(SB)
904 TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16
905 MOVD R0, x+0(FP)
906 MOVD R1, y+8(FP)
907 JMP runtime·goPanicIndexU(SB)
908 TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16
909 MOVD R1, x+0(FP)
910 MOVD R2, y+8(FP)
911 JMP runtime·goPanicSliceAlen(SB)
912 TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16
913 MOVD R1, x+0(FP)
914 MOVD R2, y+8(FP)
915 JMP runtime·goPanicSliceAlenU(SB)
916 TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16
917 MOVD R1, x+0(FP)
918 MOVD R2, y+8(FP)
919 JMP runtime·goPanicSliceAcap(SB)
920 TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16
921 MOVD R1, x+0(FP)
922 MOVD R2, y+8(FP)
923 JMP runtime·goPanicSliceAcapU(SB)
924 TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16
925 MOVD R0, x+0(FP)
926 MOVD R1, y+8(FP)
927 JMP runtime·goPanicSliceB(SB)
928 TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16
929 MOVD R0, x+0(FP)
930 MOVD R1, y+8(FP)
931 JMP runtime·goPanicSliceBU(SB)
932 TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16
933 MOVD R2, x+0(FP)
934 MOVD R3, y+8(FP)
935 JMP runtime·goPanicSlice3Alen(SB)
936 TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16
937 MOVD R2, x+0(FP)
938 MOVD R3, y+8(FP)
939 JMP runtime·goPanicSlice3AlenU(SB)
940 TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16
941 MOVD R2, x+0(FP)
942 MOVD R3, y+8(FP)
943 JMP runtime·goPanicSlice3Acap(SB)
944 TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16
945 MOVD R2, x+0(FP)
946 MOVD R3, y+8(FP)
947 JMP runtime·goPanicSlice3AcapU(SB)
948 TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16
949 MOVD R1, x+0(FP)
950 MOVD R2, y+8(FP)
951 JMP runtime·goPanicSlice3B(SB)
952 TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16
953 MOVD R1, x+0(FP)
954 MOVD R2, y+8(FP)
955 JMP runtime·goPanicSlice3BU(SB)
956 TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16
957 MOVD R0, x+0(FP)
958 MOVD R1, y+8(FP)
959 JMP runtime·goPanicSlice3C(SB)
960 TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16
961 MOVD R0, x+0(FP)
962 MOVD R1, y+8(FP)
963 JMP runtime·goPanicSlice3CU(SB)
964 TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16
965 MOVD R2, x+0(FP)
966 MOVD R3, y+8(FP)
967 JMP runtime·goPanicSliceConvert(SB)
968
View as plain text