1 // Copyright 2022 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "textflag.h"
7
8 // bool cas(uint32 *ptr, uint32 old, uint32 new)
9 // Atomically:
10 // if(*ptr == old){
11 // *ptr = new;
12 // return 1;
13 // } else
14 // return 0;
15 TEXT ·Cas(SB), NOSPLIT, $0-17
16 MOVV ptr+0(FP), R4
17 MOVW old+8(FP), R5
18 MOVW new+12(FP), R6
19
20 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8
21 BEQ R8, cas_again
22 MOVV R5, R7 // backup old value
23 AMCASDBW R6, (R4), R5
24 BNE R7, R5, cas_fail0
25 MOVV $1, R4
26 MOVB R4, ret+16(FP)
27 RET
28 cas_fail0:
29 MOVB R0, ret+16(FP)
30 RET
31
32 // Implemented using the ll-sc instruction pair
33 DBAR $0x14 // LoadAcquire barrier
34 cas_again:
35 MOVV R6, R7
36 LL (R4), R8
37 BNE R5, R8, cas_fail1
38 SC R7, (R4)
39 BEQ R7, cas_again
40 MOVV $1, R4
41 MOVB R4, ret+16(FP)
42 DBAR $0x12 // StoreRelease barrier
43 RET
44 cas_fail1:
45 MOVV $0, R4
46 JMP -4(PC)
47
48 // bool cas64(uint64 *ptr, uint64 old, uint64 new)
49 // Atomically:
50 // if(*ptr == old){
51 // *ptr = new;
52 // return 1;
53 // } else {
54 // return 0;
55 // }
56 TEXT ·Cas64(SB), NOSPLIT, $0-25
57 MOVV ptr+0(FP), R4
58 MOVV old+8(FP), R5
59 MOVV new+16(FP), R6
60
61 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8
62 BEQ R8, cas64_again
63 MOVV R5, R7 // backup old value
64 AMCASDBV R6, (R4), R5
65 BNE R7, R5, cas64_fail0
66 MOVV $1, R4
67 MOVB R4, ret+24(FP)
68 RET
69 cas64_fail0:
70 MOVB R0, ret+24(FP)
71 RET
72
73 // Implemented using the ll-sc instruction pair
74 DBAR $0x14
75 cas64_again:
76 MOVV R6, R7
77 LLV (R4), R8
78 BNE R5, R8, cas64_fail1
79 SCV R7, (R4)
80 BEQ R7, cas64_again
81 MOVV $1, R4
82 MOVB R4, ret+24(FP)
83 DBAR $0x12
84 RET
85 cas64_fail1:
86 MOVV $0, R4
87 JMP -4(PC)
88
89 TEXT ·Casint32(SB),NOSPLIT,$0-17
90 JMP ·Cas(SB)
91
92 TEXT ·Casint64(SB),NOSPLIT,$0-25
93 JMP ·Cas64(SB)
94
95 TEXT ·Casuintptr(SB), NOSPLIT, $0-25
96 JMP ·Cas64(SB)
97
98 TEXT ·CasRel(SB), NOSPLIT, $0-17
99 JMP ·Cas(SB)
100
101 TEXT ·Loaduintptr(SB), NOSPLIT|NOFRAME, $0-16
102 JMP ·Load64(SB)
103
104 TEXT ·Loaduint(SB), NOSPLIT|NOFRAME, $0-16
105 JMP ·Load64(SB)
106
107 TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
108 JMP ·Store64(SB)
109
110 TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
111 JMP ·Xadd64(SB)
112
113 TEXT ·Loadint64(SB), NOSPLIT, $0-16
114 JMP ·Load64(SB)
115
116 TEXT ·Xaddint32(SB),NOSPLIT,$0-20
117 JMP ·Xadd(SB)
118
119 TEXT ·Xaddint64(SB), NOSPLIT, $0-24
120 JMP ·Xadd64(SB)
121
122 // bool casp(void **val, void *old, void *new)
123 // Atomically:
124 // if(*val == old){
125 // *val = new;
126 // return 1;
127 // } else
128 // return 0;
129 TEXT ·Casp1(SB), NOSPLIT, $0-25
130 JMP ·Cas64(SB)
131
132 // uint32 Xadd(uint32 volatile *ptr, int32 delta)
133 // Atomically:
134 // *val += delta;
135 // return *val;
136 TEXT ·Xadd(SB), NOSPLIT, $0-20
137 MOVV ptr+0(FP), R4
138 MOVW delta+8(FP), R5
139 AMADDDBW R5, (R4), R6
140 ADDV R6, R5, R4
141 MOVW R4, ret+16(FP)
142 RET
143
144 // func Xadd64(ptr *uint64, delta int64) uint64
145 TEXT ·Xadd64(SB), NOSPLIT, $0-24
146 MOVV ptr+0(FP), R4
147 MOVV delta+8(FP), R5
148 AMADDDBV R5, (R4), R6
149 ADDV R6, R5, R4
150 MOVV R4, ret+16(FP)
151 RET
152
153 // uint8 Xchg8(ptr *uint8, new uint8)
154 // Atomically:
155 // old := *ptr;
156 // *ptr = new;
157 // return old;
158 TEXT ·Xchg8(SB), NOSPLIT, $0-17
159 MOVV ptr+0(FP), R4
160 MOVBU new+8(FP), R5
161
162 // R6 = ((ptr & 3) * 8)
163 AND $3, R4, R6
164 SLLV $3, R6
165
166 // R7 = ((0xFF) << R6) ^ (-1)
167 MOVV $0xFF, R8
168 SLLV R6, R8, R7
169 XOR $-1, R7
170
171 // R4 = ptr & (~3)
172 MOVV $~3, R8
173 AND R8, R4
174
175 // R5 = ((val) << R6)
176 SLLV R6, R5
177
178 DBAR $0x14 // LoadAcquire barrier
179 _xchg8_again:
180 LL (R4), R8
181 MOVV R8, R9 // backup old val
182 AND R7, R8
183 OR R5, R8
184 SC R8, (R4)
185 BEQ R8, _xchg8_again
186 DBAR $0x12 // StoreRelease barrier
187 SRLV R6, R9, R9
188 MOVBU R9, ret+16(FP)
189 RET
190
191 // func Xchg(ptr *uint32, new uint32) uint32
192 TEXT ·Xchg(SB), NOSPLIT, $0-20
193 MOVV ptr+0(FP), R4
194 MOVW new+8(FP), R5
195 AMSWAPDBW R5, (R4), R6
196 MOVW R6, ret+16(FP)
197 RET
198
199 // func Xchg64(ptr *uint64, new uint64) uint64
200 TEXT ·Xchg64(SB), NOSPLIT, $0-24
201 MOVV ptr+0(FP), R4
202 MOVV new+8(FP), R5
203 AMSWAPDBV R5, (R4), R6
204 MOVV R6, ret+16(FP)
205 RET
206
207 TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
208 JMP ·Xchg64(SB)
209
210 // func Xchgint32(ptr *int32, new int32) int32
211 TEXT ·Xchgint32(SB), NOSPLIT, $0-20
212 JMP ·Xchg(SB)
213
214 // func Xchgint64(ptr *int64, new int64) int64
215 TEXT ·Xchgint64(SB), NOSPLIT, $0-24
216 JMP ·Xchg64(SB)
217
218 TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
219 JMP ·Store64(SB)
220
221 TEXT ·StoreRel(SB), NOSPLIT, $0-12
222 JMP ·Store(SB)
223
224 TEXT ·StoreRel64(SB), NOSPLIT, $0-16
225 JMP ·Store64(SB)
226
227 TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
228 JMP ·Store64(SB)
229
230 TEXT ·Store(SB), NOSPLIT, $0-12
231 MOVV ptr+0(FP), R4
232 MOVW val+8(FP), R5
233 AMSWAPDBW R5, (R4), R0
234 RET
235
236 TEXT ·Store8(SB), NOSPLIT, $0-9
237 MOVV ptr+0(FP), R4
238 MOVB val+8(FP), R5
239 MOVBU internal∕cpu·Loong64+const_offsetLoong64HasLAM_BH(SB), R6
240 BEQ R6, _legacy_store8_
241 AMSWAPDBB R5, (R4), R0
242 RET
243 _legacy_store8_:
244 // StoreRelease barrier
245 DBAR $0x12
246 MOVB R5, 0(R4)
247 DBAR $0x18
248 RET
249
250 TEXT ·Store64(SB), NOSPLIT, $0-16
251 MOVV ptr+0(FP), R4
252 MOVV val+8(FP), R5
253 AMSWAPDBV R5, (R4), R0
254 RET
255
256 // void Or8(byte volatile*, byte);
257 TEXT ·Or8(SB), NOSPLIT, $0-9
258 MOVV ptr+0(FP), R4
259 MOVBU val+8(FP), R5
260 // R6 = ptr & (~3)
261 MOVV $~3, R6
262 AND R4, R6
263 // R7 = ((ptr & 3) * 8)
264 AND $3, R4, R7
265 SLLV $3, R7
266 // R5 = val << R7
267 SLLV R7, R5
268 AMORDBW R5, (R6), R0
269 RET
270
271 // void And8(byte volatile*, byte);
272 TEXT ·And8(SB), NOSPLIT, $0-9
273 MOVV ptr+0(FP), R4
274 MOVBU val+8(FP), R5
275 // R6 = ptr & (~3)
276 MOVV $~3, R6
277 AND R4, R6
278 // R7 = ((ptr & 3) * 8)
279 AND $3, R4, R7
280 SLLV $3, R7
281 // R5 = ((val ^ 0xFF) << R7) ^ (-1)
282 XOR $255, R5
283 SLLV R7, R5
284 XOR $-1, R5
285 AMANDDBW R5, (R6), R0
286 RET
287
288 // func Or(addr *uint32, v uint32)
289 TEXT ·Or(SB), NOSPLIT, $0-12
290 MOVV ptr+0(FP), R4
291 MOVW val+8(FP), R5
292 AMORDBW R5, (R4), R0
293 RET
294
295 // func And(addr *uint32, v uint32)
296 TEXT ·And(SB), NOSPLIT, $0-12
297 MOVV ptr+0(FP), R4
298 MOVW val+8(FP), R5
299 AMANDDBW R5, (R4), R0
300 RET
301
302 // func Or32(addr *uint32, v uint32) old uint32
303 TEXT ·Or32(SB), NOSPLIT, $0-20
304 MOVV ptr+0(FP), R4
305 MOVW val+8(FP), R5
306 AMORDBW R5, (R4), R6
307 MOVW R6, ret+16(FP)
308 RET
309
310 // func And32(addr *uint32, v uint32) old uint32
311 TEXT ·And32(SB), NOSPLIT, $0-20
312 MOVV ptr+0(FP), R4
313 MOVW val+8(FP), R5
314 AMANDDBW R5, (R4), R6
315 MOVW R6, ret+16(FP)
316 RET
317
318 // func Or64(addr *uint64, v uint64) old uint64
319 TEXT ·Or64(SB), NOSPLIT, $0-24
320 MOVV ptr+0(FP), R4
321 MOVV val+8(FP), R5
322 AMORDBV R5, (R4), R6
323 MOVV R6, ret+16(FP)
324 RET
325
326 // func And64(addr *uint64, v uint64) old uint64
327 TEXT ·And64(SB), NOSPLIT, $0-24
328 MOVV ptr+0(FP), R4
329 MOVV val+8(FP), R5
330 AMANDDBV R5, (R4), R6
331 MOVV R6, ret+16(FP)
332 RET
333
334 // func Anduintptr(addr *uintptr, v uintptr) old uintptr
335 TEXT ·Anduintptr(SB), NOSPLIT, $0-24
336 JMP ·And64(SB)
337
338 // func Oruintptr(addr *uintptr, v uintptr) old uintptr
339 TEXT ·Oruintptr(SB), NOSPLIT, $0-24
340 JMP ·Or64(SB)
341
342 // uint32 internal∕runtime∕atomic·Load(uint32 volatile* ptr)
343 TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12
344 MOVV ptr+0(FP), R19
345 MOVWU 0(R19), R19
346 DBAR $0x14 // LoadAcquire barrier
347 MOVW R19, ret+8(FP)
348 RET
349
350 // uint8 internal∕runtime∕atomic·Load8(uint8 volatile* ptr)
351 TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9
352 MOVV ptr+0(FP), R19
353 MOVBU 0(R19), R19
354 DBAR $0x14
355 MOVB R19, ret+8(FP)
356 RET
357
358 // uint64 internal∕runtime∕atomic·Load64(uint64 volatile* ptr)
359 TEXT ·Load64(SB),NOSPLIT|NOFRAME,$0-16
360 MOVV ptr+0(FP), R19
361 MOVV 0(R19), R19
362 DBAR $0x14
363 MOVV R19, ret+8(FP)
364 RET
365
366 // void *internal∕runtime∕atomic·Loadp(void *volatile *ptr)
367 TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-16
368 JMP ·Load64(SB)
369
370 // uint32 internal∕runtime∕atomic·LoadAcq(uint32 volatile* ptr)
371 TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12
372 JMP ·Load(SB)
373
374 // uint64 ·LoadAcq64(uint64 volatile* ptr)
375 TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$0-16
376 JMP ·Load64(SB)
377
378 // uintptr ·LoadAcquintptr(uintptr volatile* ptr)
379 TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-16
380 JMP ·Load64(SB)
381
382
View as plain text