Text file
src/runtime/asm_wasm.s
1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "funcdata.h"
8 #include "textflag.h"
9
10 TEXT runtime·rt0_go(SB), NOSPLIT|NOFRAME|TOPFRAME, $0
11 // save m->g0 = g0
12 MOVD $runtime·g0(SB), runtime·m0+m_g0(SB)
13 // save m0 to g0->m
14 MOVD $runtime·m0(SB), runtime·g0+g_m(SB)
15 // set g to g0
16 MOVD $runtime·g0(SB), g
17 CALLNORESUME runtime·check(SB)
18 #ifdef GOOS_js
19 CALLNORESUME runtime·args(SB)
20 #endif
21 CALLNORESUME runtime·osinit(SB)
22 CALLNORESUME runtime·schedinit(SB)
23 MOVD $runtime·mainPC(SB), 0(SP)
24 CALLNORESUME runtime·newproc(SB)
25 CALL runtime·mstart(SB) // WebAssembly stack will unwind when switching to another goroutine
26 UNDEF
27
28 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
29 CALL runtime·mstart0(SB)
30 RET // not reached
31
32 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB)
33 GLOBL runtime·mainPC(SB),RODATA,$8
34
35 // func checkASM() bool
36 TEXT ·checkASM(SB), NOSPLIT, $0-1
37 MOVB $1, ret+0(FP)
38 RET
39
40 TEXT runtime·gogo(SB), NOSPLIT, $0-8
41 MOVD buf+0(FP), R0
42 MOVD gobuf_g(R0), R1
43 MOVD 0(R1), R2 // make sure g != nil
44 MOVD R1, g
45 MOVD gobuf_sp(R0), SP
46
47 // Put target PC at -8(SP), wasm_pc_f_loop will pick it up
48 Get SP
49 I32Const $8
50 I32Sub
51 I64Load gobuf_pc(R0)
52 I64Store $0
53
54 MOVD gobuf_ctxt(R0), CTXT
55 // clear to help garbage collector
56 MOVD $0, gobuf_sp(R0)
57 MOVD $0, gobuf_ctxt(R0)
58
59 I32Const $1
60 Return
61
62 // func mcall(fn func(*g))
63 // Switch to m->g0's stack, call fn(g).
64 // Fn must never return. It should gogo(&g->sched)
65 // to keep running g.
66 TEXT runtime·mcall(SB), NOSPLIT, $0-8
67 // CTXT = fn
68 MOVD fn+0(FP), CTXT
69 // R1 = g.m
70 MOVD g_m(g), R1
71 // R2 = g0
72 MOVD m_g0(R1), R2
73
74 // save state in g->sched
75 MOVD 0(SP), g_sched+gobuf_pc(g) // caller's PC
76 MOVD $fn+0(FP), g_sched+gobuf_sp(g) // caller's SP
77
78 // if g == g0 call badmcall
79 Get g
80 Get R2
81 I64Eq
82 If
83 JMP runtime·badmcall(SB)
84 End
85
86 // switch to g0's stack
87 I64Load (g_sched+gobuf_sp)(R2)
88 I64Const $8
89 I64Sub
90 I32WrapI64
91 Set SP
92
93 // set arg to current g
94 MOVD g, 0(SP)
95
96 // switch to g0
97 MOVD R2, g
98
99 // call fn
100 Get CTXT
101 I32WrapI64
102 I64Load $0
103 CALL
104
105 Get SP
106 I32Const $8
107 I32Add
108 Set SP
109
110 JMP runtime·badmcall2(SB)
111
112 // func systemstack(fn func())
113 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
114 // R0 = fn
115 MOVD fn+0(FP), R0
116 // R1 = g.m
117 MOVD g_m(g), R1
118 // R2 = g0
119 MOVD m_g0(R1), R2
120
121 // if g == g0
122 Get g
123 Get R2
124 I64Eq
125 If
126 // no switch:
127 MOVD R0, CTXT
128
129 Get CTXT
130 I32WrapI64
131 I64Load $0
132 JMP
133 End
134
135 // if g != m.curg
136 Get g
137 I64Load m_curg(R1)
138 I64Ne
139 If
140 CALLNORESUME runtime·badsystemstack(SB)
141 CALLNORESUME runtime·abort(SB)
142 End
143
144 // switch:
145
146 // save state in g->sched. Pretend to
147 // be systemstack_switch if the G stack is scanned.
148 MOVD $runtime·systemstack_switch(SB), g_sched+gobuf_pc(g)
149
150 MOVD SP, g_sched+gobuf_sp(g)
151
152 // switch to g0
153 MOVD R2, g
154
155 // make it look like mstart called systemstack on g0, to stop traceback
156 I64Load (g_sched+gobuf_sp)(R2)
157 I64Const $8
158 I64Sub
159 Set R3
160
161 MOVD $runtime·mstart(SB), 0(R3)
162 MOVD R3, SP
163
164 // call fn
165 MOVD R0, CTXT
166
167 Get CTXT
168 I32WrapI64
169 I64Load $0
170 CALL
171
172 // switch back to g
173 MOVD g_m(g), R1
174 MOVD m_curg(R1), R2
175 MOVD R2, g
176 MOVD g_sched+gobuf_sp(R2), SP
177 MOVD $0, g_sched+gobuf_sp(R2)
178 RET
179
180 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
181 RET
182
183 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
184 UNDEF
185
186 // AES hashing not implemented for wasm
187 TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32
188 JMP runtime·memhashFallback(SB)
189 TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24
190 JMP runtime·strhashFallback(SB)
191 TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24
192 JMP runtime·memhash32Fallback(SB)
193 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24
194 JMP runtime·memhash64Fallback(SB)
195
196 TEXT runtime·asminit(SB), NOSPLIT, $0-0
197 // No per-thread init.
198 RET
199
200 TEXT ·publicationBarrier(SB), NOSPLIT, $0-0
201 RET
202
203 TEXT runtime·procyield(SB), NOSPLIT, $0-0 // FIXME
204 RET
205
206 TEXT runtime·breakpoint(SB), NOSPLIT, $0-0
207 UNDEF
208
209 // func switchToCrashStack0(fn func())
210 TEXT runtime·switchToCrashStack0(SB), NOSPLIT, $0-8
211 MOVD fn+0(FP), CTXT // context register
212 MOVD g_m(g), R2 // curm
213
214 // set g to gcrash
215 MOVD $runtime·gcrash(SB), g // g = &gcrash
216 MOVD R2, g_m(g) // g.m = curm
217 MOVD g, m_g0(R2) // curm.g0 = g
218
219 // switch to crashstack
220 I64Load (g_stack+stack_hi)(g)
221 I64Const $(-4*8)
222 I64Add
223 I32WrapI64
224 Set SP
225
226 // call target function
227 Get CTXT
228 I32WrapI64
229 I64Load $0
230 CALL
231
232 // should never return
233 CALL runtime·abort(SB)
234 UNDEF
235
236 // Called during function prolog when more stack is needed.
237 //
238 // The traceback routines see morestack on a g0 as being
239 // the top of a stack (for example, morestack calling newstack
240 // calling the scheduler calling newm calling gc), so we must
241 // record an argument size. For that purpose, it has no arguments.
242 TEXT runtime·morestack(SB), NOSPLIT, $0-0
243 // R1 = g.m
244 MOVD g_m(g), R1
245
246 // R2 = g0
247 MOVD m_g0(R1), R2
248
249 // Set g->sched to context in f.
250 NOP SP // tell vet SP changed - stop checking offsets
251 MOVD 0(SP), g_sched+gobuf_pc(g)
252 MOVD $8(SP), g_sched+gobuf_sp(g) // f's SP
253 MOVD CTXT, g_sched+gobuf_ctxt(g)
254
255 // Cannot grow scheduler stack (m->g0).
256 Get g
257 Get R2
258 I64Eq
259 If
260 CALLNORESUME runtime·badmorestackg0(SB)
261 CALLNORESUME runtime·abort(SB)
262 End
263
264 // Cannot grow signal stack (m->gsignal).
265 Get g
266 I64Load m_gsignal(R1)
267 I64Eq
268 If
269 CALLNORESUME runtime·badmorestackgsignal(SB)
270 CALLNORESUME runtime·abort(SB)
271 End
272
273 // Called from f.
274 // Set m->morebuf to f's caller.
275 MOVD 8(SP), m_morebuf+gobuf_pc(R1)
276 MOVD $16(SP), m_morebuf+gobuf_sp(R1) // f's caller's SP
277 MOVD g, m_morebuf+gobuf_g(R1)
278
279 // Call newstack on m->g0's stack.
280 MOVD R2, g
281 MOVD g_sched+gobuf_sp(R2), SP
282 CALL runtime·newstack(SB)
283 UNDEF // crash if newstack returns
284
285 // morestack but not preserving ctxt.
286 TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
287 MOVD $0, CTXT
288 JMP runtime·morestack(SB)
289
290 TEXT ·asmcgocall(SB), NOSPLIT, $0-0
291 UNDEF
292
293 #define DISPATCH(NAME, MAXSIZE) \
294 Get R0; \
295 I64Const $MAXSIZE; \
296 I64LeU; \
297 If; \
298 JMP NAME(SB); \
299 End
300
301 TEXT ·reflectcall(SB), NOSPLIT, $0-48
302 I64Load fn+8(FP)
303 I64Eqz
304 If
305 CALLNORESUME runtime·sigpanic<ABIInternal>(SB)
306 End
307
308 MOVW frameSize+32(FP), R0
309
310 DISPATCH(runtime·call16, 16)
311 DISPATCH(runtime·call32, 32)
312 DISPATCH(runtime·call64, 64)
313 DISPATCH(runtime·call128, 128)
314 DISPATCH(runtime·call256, 256)
315 DISPATCH(runtime·call512, 512)
316 DISPATCH(runtime·call1024, 1024)
317 DISPATCH(runtime·call2048, 2048)
318 DISPATCH(runtime·call4096, 4096)
319 DISPATCH(runtime·call8192, 8192)
320 DISPATCH(runtime·call16384, 16384)
321 DISPATCH(runtime·call32768, 32768)
322 DISPATCH(runtime·call65536, 65536)
323 DISPATCH(runtime·call131072, 131072)
324 DISPATCH(runtime·call262144, 262144)
325 DISPATCH(runtime·call524288, 524288)
326 DISPATCH(runtime·call1048576, 1048576)
327 DISPATCH(runtime·call2097152, 2097152)
328 DISPATCH(runtime·call4194304, 4194304)
329 DISPATCH(runtime·call8388608, 8388608)
330 DISPATCH(runtime·call16777216, 16777216)
331 DISPATCH(runtime·call33554432, 33554432)
332 DISPATCH(runtime·call67108864, 67108864)
333 DISPATCH(runtime·call134217728, 134217728)
334 DISPATCH(runtime·call268435456, 268435456)
335 DISPATCH(runtime·call536870912, 536870912)
336 DISPATCH(runtime·call1073741824, 1073741824)
337 JMP runtime·badreflectcall(SB)
338
339 #define CALLFN(NAME, MAXSIZE) \
340 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
341 NO_LOCAL_POINTERS; \
342 MOVW stackArgsSize+24(FP), R0; \
343 \
344 Get R0; \
345 I64Eqz; \
346 Not; \
347 If; \
348 Get SP; \
349 I64Load stackArgs+16(FP); \
350 I32WrapI64; \
351 I64Load stackArgsSize+24(FP); \
352 I32WrapI64; \
353 MemoryCopy; \
354 End; \
355 \
356 MOVD f+8(FP), CTXT; \
357 Get CTXT; \
358 I32WrapI64; \
359 I64Load $0; \
360 CALL; \
361 \
362 I64Load32U stackRetOffset+28(FP); \
363 Set R0; \
364 \
365 MOVD stackArgsType+0(FP), RET0; \
366 \
367 I64Load stackArgs+16(FP); \
368 Get R0; \
369 I64Add; \
370 Set RET1; \
371 \
372 Get SP; \
373 I64ExtendI32U; \
374 Get R0; \
375 I64Add; \
376 Set RET2; \
377 \
378 I64Load32U stackArgsSize+24(FP); \
379 Get R0; \
380 I64Sub; \
381 Set RET3; \
382 \
383 CALL callRet<>(SB); \
384 RET
385
386 // callRet copies return values back at the end of call*. This is a
387 // separate function so it can allocate stack space for the arguments
388 // to reflectcallmove. It does not follow the Go ABI; it expects its
389 // arguments in registers.
390 TEXT callRet<>(SB), NOSPLIT, $40-0
391 NO_LOCAL_POINTERS
392 MOVD RET0, 0(SP)
393 MOVD RET1, 8(SP)
394 MOVD RET2, 16(SP)
395 MOVD RET3, 24(SP)
396 MOVD $0, 32(SP)
397 CALL runtime·reflectcallmove(SB)
398 RET
399
400 CALLFN(·call16, 16)
401 CALLFN(·call32, 32)
402 CALLFN(·call64, 64)
403 CALLFN(·call128, 128)
404 CALLFN(·call256, 256)
405 CALLFN(·call512, 512)
406 CALLFN(·call1024, 1024)
407 CALLFN(·call2048, 2048)
408 CALLFN(·call4096, 4096)
409 CALLFN(·call8192, 8192)
410 CALLFN(·call16384, 16384)
411 CALLFN(·call32768, 32768)
412 CALLFN(·call65536, 65536)
413 CALLFN(·call131072, 131072)
414 CALLFN(·call262144, 262144)
415 CALLFN(·call524288, 524288)
416 CALLFN(·call1048576, 1048576)
417 CALLFN(·call2097152, 2097152)
418 CALLFN(·call4194304, 4194304)
419 CALLFN(·call8388608, 8388608)
420 CALLFN(·call16777216, 16777216)
421 CALLFN(·call33554432, 33554432)
422 CALLFN(·call67108864, 67108864)
423 CALLFN(·call134217728, 134217728)
424 CALLFN(·call268435456, 268435456)
425 CALLFN(·call536870912, 536870912)
426 CALLFN(·call1073741824, 1073741824)
427
428 TEXT runtime·goexit(SB), NOSPLIT|TOPFRAME, $0-0
429 NOP // first PC of goexit is skipped
430 CALL runtime·goexit1(SB) // does not return
431 UNDEF
432
433 TEXT runtime·cgocallback(SB), NOSPLIT, $0-24
434 UNDEF
435
436 // gcWriteBarrier informs the GC about heap pointer writes.
437 //
438 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
439 // number of bytes of buffer needed as a wasm argument
440 // (put on the TOS by the caller, lives in local R0 in this body)
441 // and returns a pointer to the buffer space as a wasm result
442 // (left on the TOS in this body, appears on the wasm stack
443 // in the caller).
444 TEXT gcWriteBarrier<>(SB), NOSPLIT, $0
445 Loop
446 // R3 = g.m
447 MOVD g_m(g), R3
448 // R4 = p
449 MOVD m_p(R3), R4
450 // R5 = wbBuf.next
451 MOVD p_wbBuf+wbBuf_next(R4), R5
452
453 // Increment wbBuf.next
454 Get R5
455 Get R0
456 I64Add
457 Set R5
458
459 // Is the buffer full?
460 Get R5
461 I64Load (p_wbBuf+wbBuf_end)(R4)
462 I64LeU
463 If
464 // Commit to the larger buffer.
465 MOVD R5, p_wbBuf+wbBuf_next(R4)
466
467 // Make return value (the original next position)
468 Get R5
469 Get R0
470 I64Sub
471
472 Return
473 End
474
475 // Flush
476 CALLNORESUME runtime·wbBufFlush(SB)
477
478 // Retry
479 Br $0
480 End
481
482 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
483 I64Const $8
484 Call gcWriteBarrier<>(SB)
485 Return
486 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
487 I64Const $16
488 Call gcWriteBarrier<>(SB)
489 Return
490 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
491 I64Const $24
492 Call gcWriteBarrier<>(SB)
493 Return
494 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
495 I64Const $32
496 Call gcWriteBarrier<>(SB)
497 Return
498 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
499 I64Const $40
500 Call gcWriteBarrier<>(SB)
501 Return
502 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
503 I64Const $48
504 Call gcWriteBarrier<>(SB)
505 Return
506 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
507 I64Const $56
508 Call gcWriteBarrier<>(SB)
509 Return
510 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
511 I64Const $64
512 Call gcWriteBarrier<>(SB)
513 Return
514
515 TEXT wasm_pc_f_loop(SB),NOSPLIT,$0
516 // Call the function for the current PC_F. Repeat until PAUSE != 0 indicates pause or exit.
517 // The WebAssembly stack may unwind, e.g. when switching goroutines.
518 // The Go stack on the linear memory is then used to jump to the correct functions
519 // with this loop, without having to restore the full WebAssembly stack.
520 // It is expected to have a pending call before entering the loop, so check PAUSE first.
521 Get PAUSE
522 I32Eqz
523 If
524 loop:
525 Loop
526 // Get PC_B & PC_F from -8(SP)
527 Get SP
528 I32Const $8
529 I32Sub
530 I32Load16U $0 // PC_B
531
532 Get SP
533 I32Const $8
534 I32Sub
535 I32Load16U $2 // PC_F
536
537 CallIndirect $0
538 Drop
539
540 Get PAUSE
541 I32Eqz
542 BrIf loop
543 End
544 End
545
546 I32Const $0
547 Set PAUSE
548
549 Return
550
551 // wasm_pc_f_loop_export is like wasm_pc_f_loop, except that this takes an
552 // argument (on Wasm stack) that is a PC_F, and the loop stops when we get
553 // to that PC in a normal return (not unwinding).
554 // This is for handling an wasmexport function when it needs to switch the
555 // stack.
556 TEXT wasm_pc_f_loop_export(SB),NOSPLIT,$0
557 Get PAUSE
558 I32Eqz
559 outer:
560 If
561 // R1 is whether a function return normally (0) or unwinding (1).
562 // Start with unwinding.
563 I32Const $1
564 Set R1
565 loop:
566 Loop
567 // Get PC_F & PC_B from -8(SP)
568 Get SP
569 I32Const $8
570 I32Sub
571 I32Load16U $2 // PC_F
572 Tee R2
573
574 Get R0
575 I32Eq
576 If // PC_F == R0, we're at the stop PC
577 Get R1
578 I32Eqz
579 // Break if it is a normal return
580 BrIf outer // actually jump to after the corresponding End
581 End
582
583 Get SP
584 I32Const $8
585 I32Sub
586 I32Load16U $0 // PC_B
587
588 Get R2 // PC_F
589 CallIndirect $0
590 Set R1 // save return/unwinding state for next iteration
591
592 Get PAUSE
593 I32Eqz
594 BrIf loop
595 End
596 End
597
598 I32Const $0
599 Set PAUSE
600
601 Return
602
603 TEXT wasm_export_lib(SB),NOSPLIT,$0
604 UNDEF
605
606 TEXT runtime·pause(SB), NOSPLIT, $0-8
607 MOVD newsp+0(FP), SP
608 I32Const $1
609 Set PAUSE
610 RETUNWIND
611
612 // Called if a wasmexport function is called before runtime initialization
613 TEXT runtime·notInitialized(SB), NOSPLIT, $0
614 MOVD $runtime·wasmStack+(m0Stack__size-16-8)(SB), SP
615 I32Const $0 // entry PC_B
616 Call runtime·notInitialized1(SB)
617 Drop
618 I32Const $0 // entry PC_B
619 Call runtime·abort(SB)
620 UNDEF
621
View as plain text