1// Copyright 2014 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// RISC-V's atomic operations have two bits, aq ("acquire") and rl ("release"),
6// which may be toggled on and off. Their precise semantics are defined in
7// section 6.3 of the specification, but the basic idea is as follows:
8//
9//   - If neither aq nor rl is set, the CPU may reorder the atomic arbitrarily.
10//     It guarantees only that it will execute atomically.
11//
12//   - If aq is set, the CPU may move the instruction backward, but not forward.
13//
14//   - If rl is set, the CPU may move the instruction forward, but not backward.
15//
16//   - If both are set, the CPU may not reorder the instruction at all.
17//
18// These four modes correspond to other well-known memory models on other CPUs.
19// On ARM, aq corresponds to a dmb ishst, aq+rl corresponds to a dmb ish. On
20// Intel, aq corresponds to an lfence, rl to an sfence, and aq+rl to an mfence
21// (or a lock prefix).
22//
23// Go's memory model requires that
24//   - if a read happens after a write, the read must observe the write, and
25//     that
26//   - if a read happens concurrently with a write, the read may observe the
27//     write.
28// aq is sufficient to guarantee this, so that's what we use here. (This jibes
29// with ARM, which uses dmb ishst.)
30
31#include "textflag.h"
32
33// func Cas(ptr *uint64, old, new uint64) bool
34// Atomically:
35//      if(*val == old){
36//              *val = new;
37//              return 1;
38//      } else {
39//              return 0;
40//      }
41TEXT ·Cas(SB), NOSPLIT, $0-17
42	MOV	ptr+0(FP), A0
43	MOVW	old+8(FP), A1
44	MOVW	new+12(FP), A2
45cas_again:
46	LRW	(A0), A3
47	BNE	A3, A1, cas_fail
48	SCW	A2, (A0), A4
49	BNE	A4, ZERO, cas_again
50	MOV	$1, A0
51	MOVB	A0, ret+16(FP)
52	RET
53cas_fail:
54	MOV	$0, A0
55	MOV	A0, ret+16(FP)
56	RET
57
58// func Cas64(ptr *uint64, old, new uint64) bool
59TEXT ·Cas64(SB), NOSPLIT, $0-25
60	MOV	ptr+0(FP), A0
61	MOV	old+8(FP), A1
62	MOV	new+16(FP), A2
63cas_again:
64	LRD	(A0), A3
65	BNE	A3, A1, cas_fail
66	SCD	A2, (A0), A4
67	BNE	A4, ZERO, cas_again
68	MOV	$1, A0
69	MOVB	A0, ret+24(FP)
70	RET
71cas_fail:
72	MOVB	ZERO, ret+24(FP)
73	RET
74
75// func Load(ptr *uint32) uint32
76TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12
77	MOV	ptr+0(FP), A0
78	LRW	(A0), A0
79	MOVW	A0, ret+8(FP)
80	RET
81
82// func Load8(ptr *uint8) uint8
83TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9
84	MOV	ptr+0(FP), A0
85	FENCE
86	MOVBU	(A0), A1
87	FENCE
88	MOVB	A1, ret+8(FP)
89	RET
90
91// func Load64(ptr *uint64) uint64
92TEXT ·Load64(SB),NOSPLIT|NOFRAME,$0-16
93	MOV	ptr+0(FP), A0
94	LRD	(A0), A0
95	MOV	A0, ret+8(FP)
96	RET
97
98// func Store(ptr *uint32, val uint32)
99TEXT ·Store(SB), NOSPLIT, $0-12
100	MOV	ptr+0(FP), A0
101	MOVW	val+8(FP), A1
102	AMOSWAPW A1, (A0), ZERO
103	RET
104
105// func Store8(ptr *uint8, val uint8)
106TEXT ·Store8(SB), NOSPLIT, $0-9
107	MOV	ptr+0(FP), A0
108	MOVBU	val+8(FP), A1
109	FENCE
110	MOVB	A1, (A0)
111	FENCE
112	RET
113
114// func Store64(ptr *uint64, val uint64)
115TEXT ·Store64(SB), NOSPLIT, $0-16
116	MOV	ptr+0(FP), A0
117	MOV	val+8(FP), A1
118	AMOSWAPD A1, (A0), ZERO
119	RET
120
121TEXT ·Casp1(SB), NOSPLIT, $0-25
122	JMP	·Cas64(SB)
123
124TEXT ·Casint32(SB),NOSPLIT,$0-17
125	JMP	·Cas(SB)
126
127TEXT ·Casint64(SB),NOSPLIT,$0-25
128	JMP	·Cas64(SB)
129
130TEXT ·Casuintptr(SB),NOSPLIT,$0-25
131	JMP	·Cas64(SB)
132
133TEXT ·CasRel(SB), NOSPLIT, $0-17
134	JMP	·Cas(SB)
135
136TEXT ·Loaduintptr(SB),NOSPLIT,$0-16
137	JMP	·Load64(SB)
138
139TEXT ·Storeint32(SB),NOSPLIT,$0-12
140	JMP	·Store(SB)
141
142TEXT ·Storeint64(SB),NOSPLIT,$0-16
143	JMP	·Store64(SB)
144
145TEXT ·Storeuintptr(SB),NOSPLIT,$0-16
146	JMP	·Store64(SB)
147
148TEXT ·Loaduint(SB),NOSPLIT,$0-16
149	JMP ·Loaduintptr(SB)
150
151TEXT ·Loadint32(SB),NOSPLIT,$0-12
152	JMP ·Load(SB)
153
154TEXT ·Loadint64(SB),NOSPLIT,$0-16
155	JMP ·Load64(SB)
156
157TEXT ·Xaddint32(SB),NOSPLIT,$0-20
158	JMP ·Xadd(SB)
159
160TEXT ·Xaddint64(SB),NOSPLIT,$0-24
161	MOV	ptr+0(FP), A0
162	MOV	delta+8(FP), A1
163	AMOADDD A1, (A0), A0
164	ADD	A0, A1, A0
165	MOVW	A0, ret+16(FP)
166	RET
167
168TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12
169	JMP	·Load(SB)
170
171TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$0-16
172	JMP	·Load64(SB)
173
174TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-16
175	JMP	·Load64(SB)
176
177// func Loadp(ptr unsafe.Pointer) unsafe.Pointer
178TEXT ·Loadp(SB),NOSPLIT,$0-16
179	JMP	·Load64(SB)
180
181// func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer)
182TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
183	JMP	·Store64(SB)
184
185TEXT ·StoreRel(SB), NOSPLIT, $0-12
186	JMP	·Store(SB)
187
188TEXT ·StoreRel64(SB), NOSPLIT, $0-16
189	JMP	·Store64(SB)
190
191TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
192	JMP	·Store64(SB)
193
194// func Xchg(ptr *uint32, new uint32) uint32
195TEXT ·Xchg(SB), NOSPLIT, $0-20
196	MOV	ptr+0(FP), A0
197	MOVW	new+8(FP), A1
198	AMOSWAPW A1, (A0), A1
199	MOVW	A1, ret+16(FP)
200	RET
201
202// func Xchg64(ptr *uint64, new uint64) uint64
203TEXT ·Xchg64(SB), NOSPLIT, $0-24
204	MOV	ptr+0(FP), A0
205	MOV	new+8(FP), A1
206	AMOSWAPD A1, (A0), A1
207	MOV	A1, ret+16(FP)
208	RET
209
210// Atomically:
211//      *val += delta;
212//      return *val;
213
214// func Xadd(ptr *uint32, delta int32) uint32
215TEXT ·Xadd(SB), NOSPLIT, $0-20
216	MOV	ptr+0(FP), A0
217	MOVW	delta+8(FP), A1
218	AMOADDW A1, (A0), A2
219	ADD	A2,A1,A0
220	MOVW	A0, ret+16(FP)
221	RET
222
223// func Xadd64(ptr *uint64, delta int64) uint64
224TEXT ·Xadd64(SB), NOSPLIT, $0-24
225	MOV	ptr+0(FP), A0
226	MOV	delta+8(FP), A1
227	AMOADDD A1, (A0), A2
228	ADD	A2, A1, A0
229	MOV	A0, ret+16(FP)
230	RET
231
232// func Xadduintptr(ptr *uintptr, delta uintptr) uintptr
233TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
234	JMP	·Xadd64(SB)
235
236// func Xchgint32(ptr *int32, new int32) int32
237TEXT ·Xchgint32(SB), NOSPLIT, $0-20
238	JMP	·Xchg(SB)
239
240// func Xchgint64(ptr *int64, new int64) int64
241TEXT ·Xchgint64(SB), NOSPLIT, $0-24
242	JMP	·Xchg64(SB)
243
244// func Xchguintptr(ptr *uintptr, new uintptr) uintptr
245TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
246	JMP	·Xchg64(SB)
247
248// func And8(ptr *uint8, val uint8)
249TEXT ·And8(SB), NOSPLIT, $0-9
250	MOV	ptr+0(FP), A0
251	MOVBU	val+8(FP), A1
252	AND	$3, A0, A2
253	AND	$-4, A0
254	SLL	$3, A2
255	XOR	$255, A1
256	SLL	A2, A1
257	XOR	$-1, A1
258	AMOANDW A1, (A0), ZERO
259	RET
260
261// func Or8(ptr *uint8, val uint8)
262TEXT ·Or8(SB), NOSPLIT, $0-9
263	MOV	ptr+0(FP), A0
264	MOVBU	val+8(FP), A1
265	AND	$3, A0, A2
266	AND	$-4, A0
267	SLL	$3, A2
268	SLL	A2, A1
269	AMOORW	A1, (A0), ZERO
270	RET
271
272// func And(ptr *uint32, val uint32)
273TEXT ·And(SB), NOSPLIT, $0-12
274	MOV	ptr+0(FP), A0
275	MOVW	val+8(FP), A1
276	AMOANDW	A1, (A0), ZERO
277	RET
278
279// func Or(ptr *uint32, val uint32)
280TEXT ·Or(SB), NOSPLIT, $0-12
281	MOV	ptr+0(FP), A0
282	MOVW	val+8(FP), A1
283	AMOORW	A1, (A0), ZERO
284	RET
285
286// func Or32(ptr *uint32, val uint32) uint32
287TEXT ·Or32(SB), NOSPLIT, $0-20
288	MOV	ptr+0(FP), A0
289	MOVW	val+8(FP), A1
290	AMOORW	A1, (A0), A2
291	MOVW	A2, ret+16(FP)
292	RET
293
294// func And32(ptr *uint32, val uint32) uint32
295TEXT ·And32(SB), NOSPLIT, $0-20
296	MOV	ptr+0(FP), A0
297	MOVW	val+8(FP), A1
298	AMOANDW	A1, (A0), A2
299	MOVW	A2, ret+16(FP)
300	RET
301
302// func Or64(ptr *uint64, val uint64) uint64
303TEXT ·Or64(SB), NOSPLIT, $0-24
304	MOV	ptr+0(FP), A0
305	MOV	val+8(FP), A1
306	AMOORD	A1, (A0), A2
307	MOV	A2, ret+16(FP)
308	RET
309
310// func And64(ptr *uint64, val uint64) uint64
311TEXT ·And64(SB), NOSPLIT, $0-24
312	MOV	ptr+0(FP), A0
313	MOV	val+8(FP), A1
314	AMOANDD	A1, (A0), A2
315	MOV	A2, ret+16(FP)
316	RET
317
318// func Anduintptr(ptr *uintptr, val uintptr) uintptr
319TEXT ·Anduintptr(SB), NOSPLIT, $0-24
320	JMP	·And64(SB)
321
322// func Oruintptr(ptr *uintptr, val uintptr) uintptr
323TEXT ·Oruintptr(SB), NOSPLIT, $0-24
324	JMP	·Or64(SB)
325