Skip to content

Commit 305d454

Browse files
wildea01ctmarinas
wildea01
authored and
ctmarinas
committed
arm64: atomics: implement native {relaxed, acquire, release} atomics
Commit 654672d ("locking/atomics: Add _{acquire|release|relaxed}() variants of some atomic operation") introduced a relaxed atomic API to Linux that maps nicely onto the arm64 memory model, including the new ARMv8.1 atomic instructions. This patch hooks up the API to our relaxed atomic instructions, rather than have them all expand to the full-barrier variants as they do currently. Signed-off-by: Will Deacon <[email protected]> Signed-off-by: Catalin Marinas <[email protected]>
1 parent e8f3010 commit 305d454

File tree

4 files changed

+371
-262
lines changed

4 files changed

+371
-262
lines changed

‎arch/arm64/include/asm/atomic.h‎

Lines changed: 59 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,42 @@
5555

5656
#define atomic_read(v) READ_ONCE((v)->counter)
5757
#define atomic_set(v, i) (((v)->counter) = (i))
58+
59+
#define atomic_add_return_relaxed atomic_add_return_relaxed
60+
#define atomic_add_return_acquire atomic_add_return_acquire
61+
#define atomic_add_return_release atomic_add_return_release
62+
#define atomic_add_return atomic_add_return
63+
64+
#define atomic_inc_return_relaxed(v) atomic_add_return_relaxed(1, (v))
65+
#define atomic_inc_return_acquire(v) atomic_add_return_acquire(1, (v))
66+
#define atomic_inc_return_release(v) atomic_add_return_release(1, (v))
67+
#define atomic_inc_return(v) atomic_add_return(1, (v))
68+
69+
#define atomic_sub_return_relaxed atomic_sub_return_relaxed
70+
#define atomic_sub_return_acquire atomic_sub_return_acquire
71+
#define atomic_sub_return_release atomic_sub_return_release
72+
#define atomic_sub_return atomic_sub_return
73+
74+
#define atomic_dec_return_relaxed(v) atomic_sub_return_relaxed(1, (v))
75+
#define atomic_dec_return_acquire(v) atomic_sub_return_acquire(1, (v))
76+
#define atomic_dec_return_release(v) atomic_sub_return_release(1, (v))
77+
#define atomic_dec_return(v) atomic_sub_return(1, (v))
78+
79+
#define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
80+
#define atomic_xchg_acquire(v, new) xchg_acquire(&((v)->counter), (new))
81+
#define atomic_xchg_release(v, new) xchg_release(&((v)->counter), (new))
5882
#define atomic_xchg(v, new) xchg(&((v)->counter), (new))
83+
84+
#define atomic_cmpxchg_relaxed(v, old, new) \
85+
cmpxchg_relaxed(&((v)->counter), (old), (new))
86+
#define atomic_cmpxchg_acquire(v, old, new) \
87+
cmpxchg_acquire(&((v)->counter), (old), (new))
88+
#define atomic_cmpxchg_release(v, old, new) \
89+
cmpxchg_release(&((v)->counter), (old), (new))
5990
#define atomic_cmpxchg(v, old, new) cmpxchg(&((v)->counter), (old), (new))
6091

6192
#define atomic_inc(v) atomic_add(1, (v))
6293
#define atomic_dec(v) atomic_sub(1, (v))
63-
#define atomic_inc_return(v) atomic_add_return(1, (v))
64-
#define atomic_dec_return(v) atomic_sub_return(1, (v))
6594
#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
6695
#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0)
6796
#define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0)
@@ -75,13 +104,39 @@
75104
#define ATOMIC64_INIT ATOMIC_INIT
76105
#define atomic64_read atomic_read
77106
#define atomic64_set atomic_set
107+
108+
#define atomic64_add_return_relaxed atomic64_add_return_relaxed
109+
#define atomic64_add_return_acquire atomic64_add_return_acquire
110+
#define atomic64_add_return_release atomic64_add_return_release
111+
#define atomic64_add_return atomic64_add_return
112+
113+
#define atomic64_inc_return_relaxed(v) atomic64_add_return_relaxed(1, (v))
114+
#define atomic64_inc_return_acquire(v) atomic64_add_return_acquire(1, (v))
115+
#define atomic64_inc_return_release(v) atomic64_add_return_release(1, (v))
116+
#define atomic64_inc_return(v) atomic64_add_return(1, (v))
117+
118+
#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
119+
#define atomic64_sub_return_acquire atomic64_sub_return_acquire
120+
#define atomic64_sub_return_release atomic64_sub_return_release
121+
#define atomic64_sub_return atomic64_sub_return
122+
123+
#define atomic64_dec_return_relaxed(v) atomic64_sub_return_relaxed(1, (v))
124+
#define atomic64_dec_return_acquire(v) atomic64_sub_return_acquire(1, (v))
125+
#define atomic64_dec_return_release(v) atomic64_sub_return_release(1, (v))
126+
#define atomic64_dec_return(v) atomic64_sub_return(1, (v))
127+
128+
#define atomic64_xchg_relaxed atomic_xchg_relaxed
129+
#define atomic64_xchg_acquire atomic_xchg_acquire
130+
#define atomic64_xchg_release atomic_xchg_release
78131
#define atomic64_xchg atomic_xchg
132+
133+
#define atomic64_cmpxchg_relaxed atomic_cmpxchg_relaxed
134+
#define atomic64_cmpxchg_acquire atomic_cmpxchg_acquire
135+
#define atomic64_cmpxchg_release atomic_cmpxchg_release
79136
#define atomic64_cmpxchg atomic_cmpxchg
80137

81138
#define atomic64_inc(v) atomic64_add(1, (v))
82139
#define atomic64_dec(v) atomic64_sub(1, (v))
83-
#define atomic64_inc_return(v) atomic64_add_return(1, (v))
84-
#define atomic64_dec_return(v) atomic64_sub_return(1, (v))
85140
#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
86141
#define atomic64_dec_and_test(v) (atomic64_dec_return(v) == 0)
87142
#define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0)

‎arch/arm64/include/asm/atomic_ll_sc.h‎

Lines changed: 60 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -55,40 +55,47 @@ __LL_SC_PREFIX(atomic_##op(int i, atomic_t *v)) \
5555
} \
5656
__LL_SC_EXPORT(atomic_##op);
5757

58-
#define ATOMIC_OP_RETURN(op, asm_op) \
58+
#define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \
5959
__LL_SC_INLINE int \
60-
__LL_SC_PREFIX(atomic_##op##_return(int i, atomic_t *v)) \
60+
__LL_SC_PREFIX(atomic_##op##_return##name(int i, atomic_t *v)) \
6161
{ \
6262
unsigned long tmp; \
6363
int result; \
6464
\
65-
asm volatile("// atomic_" #op "_return\n" \
65+
asm volatile("// atomic_" #op "_return" #name "\n" \
6666
" prfm pstl1strm, %2\n" \
67-
"1: ldxr %w0, %2\n" \
67+
"1: ld" #acq "xr %w0, %2\n" \
6868
" " #asm_op " %w0, %w0, %w3\n" \
69-
" stlxr %w1, %w0, %2\n" \
70-
" cbnz %w1, 1b" \
69+
" st" #rel "xr %w1, %w0, %2\n" \
70+
" cbnz %w1, 1b\n" \
71+
" " #mb \
7172
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
7273
: "Ir" (i) \
73-
: "memory"); \
74+
: cl); \
7475
\
75-
smp_mb(); \
7676
return result; \
7777
} \
78-
__LL_SC_EXPORT(atomic_##op##_return);
78+
__LL_SC_EXPORT(atomic_##op##_return##name);
79+
80+
#define ATOMIC_OPS(...) \
81+
ATOMIC_OP(__VA_ARGS__) \
82+
ATOMIC_OP_RETURN( , dmb ish, , l, "memory", __VA_ARGS__)
7983

80-
#define ATOMIC_OPS(op, asm_op) \
81-
ATOMIC_OP(op, asm_op) \
82-
ATOMIC_OP_RETURN(op, asm_op)
84+
#define ATOMIC_OPS_RLX(...) \
85+
ATOMIC_OPS(__VA_ARGS__) \
86+
ATOMIC_OP_RETURN(_relaxed, , , , , __VA_ARGS__)\
87+
ATOMIC_OP_RETURN(_acquire, , a, , "memory", __VA_ARGS__)\
88+
ATOMIC_OP_RETURN(_release, , , l, "memory", __VA_ARGS__)
8389

84-
ATOMIC_OPS(add, add)
85-
ATOMIC_OPS(sub, sub)
90+
ATOMIC_OPS_RLX(add, add)
91+
ATOMIC_OPS_RLX(sub, sub)
8692

8793
ATOMIC_OP(and, and)
8894
ATOMIC_OP(andnot, bic)
8995
ATOMIC_OP(or, orr)
9096
ATOMIC_OP(xor, eor)
9197

98+
#undef ATOMIC_OPS_RLX
9299
#undef ATOMIC_OPS
93100
#undef ATOMIC_OP_RETURN
94101
#undef ATOMIC_OP
@@ -111,40 +118,47 @@ __LL_SC_PREFIX(atomic64_##op(long i, atomic64_t *v)) \
111118
} \
112119
__LL_SC_EXPORT(atomic64_##op);
113120

114-
#define ATOMIC64_OP_RETURN(op, asm_op) \
121+
#define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \
115122
__LL_SC_INLINE long \
116-
__LL_SC_PREFIX(atomic64_##op##_return(long i, atomic64_t *v)) \
123+
__LL_SC_PREFIX(atomic64_##op##_return##name(long i, atomic64_t *v)) \
117124
{ \
118125
long result; \
119126
unsigned long tmp; \
120127
\
121-
asm volatile("// atomic64_" #op "_return\n" \
128+
asm volatile("// atomic64_" #op "_return" #name "\n" \
122129
" prfm pstl1strm, %2\n" \
123-
"1: ldxr %0, %2\n" \
130+
"1: ld" #acq "xr %0, %2\n" \
124131
" " #asm_op " %0, %0, %3\n" \
125-
" stlxr %w1, %0, %2\n" \
126-
" cbnz %w1, 1b" \
132+
" st" #rel "xr %w1, %0, %2\n" \
133+
" cbnz %w1, 1b\n" \
134+
" " #mb \
127135
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
128136
: "Ir" (i) \
129-
: "memory"); \
137+
: cl); \
130138
\
131-
smp_mb(); \
132139
return result; \
133140
} \
134-
__LL_SC_EXPORT(atomic64_##op##_return);
141+
__LL_SC_EXPORT(atomic64_##op##_return##name);
142+
143+
#define ATOMIC64_OPS(...) \
144+
ATOMIC64_OP(__VA_ARGS__) \
145+
ATOMIC64_OP_RETURN(, dmb ish, , l, "memory", __VA_ARGS__)
135146

136-
#define ATOMIC64_OPS(op, asm_op) \
137-
ATOMIC64_OP(op, asm_op) \
138-
ATOMIC64_OP_RETURN(op, asm_op)
147+
#define ATOMIC64_OPS_RLX(...) \
148+
ATOMIC64_OPS(__VA_ARGS__) \
149+
ATOMIC64_OP_RETURN(_relaxed,, , , , __VA_ARGS__) \
150+
ATOMIC64_OP_RETURN(_acquire,, a, , "memory", __VA_ARGS__) \
151+
ATOMIC64_OP_RETURN(_release,, , l, "memory", __VA_ARGS__)
139152

140-
ATOMIC64_OPS(add, add)
141-
ATOMIC64_OPS(sub, sub)
153+
ATOMIC64_OPS_RLX(add, add)
154+
ATOMIC64_OPS_RLX(sub, sub)
142155

143156
ATOMIC64_OP(and, and)
144157
ATOMIC64_OP(andnot, bic)
145158
ATOMIC64_OP(or, orr)
146159
ATOMIC64_OP(xor, eor)
147160

161+
#undef ATOMIC64_OPS_RLX
148162
#undef ATOMIC64_OPS
149163
#undef ATOMIC64_OP_RETURN
150164
#undef ATOMIC64_OP
@@ -172,7 +186,7 @@ __LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
172186
}
173187
__LL_SC_EXPORT(atomic64_dec_if_positive);
174188

175-
#define __CMPXCHG_CASE(w, sz, name, mb, rel, cl) \
189+
#define __CMPXCHG_CASE(w, sz, name, mb, acq, rel, cl) \
176190
__LL_SC_INLINE unsigned long \
177191
__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
178192
unsigned long old, \
@@ -182,7 +196,7 @@ __LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
182196
\
183197
asm volatile( \
184198
" prfm pstl1strm, %[v]\n" \
185-
"1: ldxr" #sz "\t%" #w "[oldval], %[v]\n" \
199+
"1: ld" #acq "xr" #sz "\t%" #w "[oldval], %[v]\n" \
186200
" eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \
187201
" cbnz %" #w "[tmp], 2f\n" \
188202
" st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \
@@ -199,14 +213,22 @@ __LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
199213
} \
200214
__LL_SC_EXPORT(__cmpxchg_case_##name);
201215

202-
__CMPXCHG_CASE(w, b, 1, , , )
203-
__CMPXCHG_CASE(w, h, 2, , , )
204-
__CMPXCHG_CASE(w, , 4, , , )
205-
__CMPXCHG_CASE( , , 8, , , )
206-
__CMPXCHG_CASE(w, b, mb_1, dmb ish, l, "memory")
207-
__CMPXCHG_CASE(w, h, mb_2, dmb ish, l, "memory")
208-
__CMPXCHG_CASE(w, , mb_4, dmb ish, l, "memory")
209-
__CMPXCHG_CASE( , , mb_8, dmb ish, l, "memory")
216+
__CMPXCHG_CASE(w, b, 1, , , , )
217+
__CMPXCHG_CASE(w, h, 2, , , , )
218+
__CMPXCHG_CASE(w, , 4, , , , )
219+
__CMPXCHG_CASE( , , 8, , , , )
220+
__CMPXCHG_CASE(w, b, acq_1, , a, , "memory")
221+
__CMPXCHG_CASE(w, h, acq_2, , a, , "memory")
222+
__CMPXCHG_CASE(w, , acq_4, , a, , "memory")
223+
__CMPXCHG_CASE( , , acq_8, , a, , "memory")
224+
__CMPXCHG_CASE(w, b, rel_1, , , l, "memory")
225+
__CMPXCHG_CASE(w, h, rel_2, , , l, "memory")
226+
__CMPXCHG_CASE(w, , rel_4, , , l, "memory")
227+
__CMPXCHG_CASE( , , rel_8, , , l, "memory")
228+
__CMPXCHG_CASE(w, b, mb_1, dmb ish, , l, "memory")
229+
__CMPXCHG_CASE(w, h, mb_2, dmb ish, , l, "memory")
230+
__CMPXCHG_CASE(w, , mb_4, dmb ish, , l, "memory")
231+
__CMPXCHG_CASE( , , mb_8, dmb ish, , l, "memory")
210232

211233
#undef __CMPXCHG_CASE
212234

0 commit comments

Comments
 (0)