1#ifndef __ARCH_S390_ATOMIC__
2#define __ARCH_S390_ATOMIC__
3
4#include <linux/compiler.h>
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26typedef struct {
27 int counter;
28} __attribute__ ((aligned (4))) atomic_t;
29#define ATOMIC_INIT(i) { (i) }
30
31#ifdef __KERNEL__
32
33#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
34
35#define __CS_LOOP(ptr, op_val, op_string) ({ \
36 typeof(ptr->counter) old_val, new_val; \
37 asm volatile( \
38 " l %0,%2\n" \
39 "0: lr %1,%0\n" \
40 op_string " %1,%3\n" \
41 " cs %0,%1,%2\n" \
42 " jl 0b" \
43 : "=&d" (old_val), "=&d" (new_val), \
44 "=Q" (((atomic_t *)(ptr))->counter) \
45 : "d" (op_val), "Q" (((atomic_t *)(ptr))->counter) \
46 : "cc", "memory"); \
47 new_val; \
48})
49
50#else
51
52#define __CS_LOOP(ptr, op_val, op_string) ({ \
53 typeof(ptr->counter) old_val, new_val; \
54 asm volatile( \
55 " l %0,0(%3)\n" \
56 "0: lr %1,%0\n" \
57 op_string " %1,%4\n" \
58 " cs %0,%1,0(%3)\n" \
59 " jl 0b" \
60 : "=&d" (old_val), "=&d" (new_val), \
61 "=m" (((atomic_t *)(ptr))->counter) \
62 : "a" (ptr), "d" (op_val), \
63 "m" (((atomic_t *)(ptr))->counter) \
64 : "cc", "memory"); \
65 new_val; \
66})
67
68#endif
69
70static inline int atomic_read(const atomic_t *v)
71{
72 barrier();
73 return v->counter;
74}
75
76static inline void atomic_set(atomic_t *v, int i)
77{
78 v->counter = i;
79 barrier();
80}
81
82static __inline__ int atomic_add_return(int i, atomic_t * v)
83{
84 return __CS_LOOP(v, i, "ar");
85}
86#define atomic_add(_i, _v) atomic_add_return(_i, _v)
87#define atomic_add_negative(_i, _v) (atomic_add_return(_i, _v) < 0)
88#define atomic_inc(_v) atomic_add_return(1, _v)
89#define atomic_inc_return(_v) atomic_add_return(1, _v)
90#define atomic_inc_and_test(_v) (atomic_add_return(1, _v) == 0)
91
92static __inline__ int atomic_sub_return(int i, atomic_t * v)
93{
94 return __CS_LOOP(v, i, "sr");
95}
96#define atomic_sub(_i, _v) atomic_sub_return(_i, _v)
97#define atomic_sub_and_test(_i, _v) (atomic_sub_return(_i, _v) == 0)
98#define atomic_dec(_v) atomic_sub_return(1, _v)
99#define atomic_dec_return(_v) atomic_sub_return(1, _v)
100#define atomic_dec_and_test(_v) (atomic_sub_return(1, _v) == 0)
101
102static __inline__ void atomic_clear_mask(unsigned long mask, atomic_t * v)
103{
104 __CS_LOOP(v, ~mask, "nr");
105}
106
107static __inline__ void atomic_set_mask(unsigned long mask, atomic_t * v)
108{
109 __CS_LOOP(v, mask, "or");
110}
111
112#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
113
114static __inline__ int atomic_cmpxchg(atomic_t *v, int old, int new)
115{
116#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
117 asm volatile(
118 " cs %0,%2,%1"
119 : "+d" (old), "=Q" (v->counter)
120 : "d" (new), "Q" (v->counter)
121 : "cc", "memory");
122#else
123 asm volatile(
124 " cs %0,%3,0(%2)"
125 : "+d" (old), "=m" (v->counter)
126 : "a" (v), "d" (new), "m" (v->counter)
127 : "cc", "memory");
128#endif
129 return old;
130}
131
132static __inline__ int atomic_add_unless(atomic_t *v, int a, int u)
133{
134 int c, old;
135 c = atomic_read(v);
136 for (;;) {
137 if (unlikely(c == u))
138 break;
139 old = atomic_cmpxchg(v, c, c + a);
140 if (likely(old == c))
141 break;
142 c = old;
143 }
144 return c != u;
145}
146
147#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
148
149#undef __CS_LOOP
150
151#ifdef __s390x__
152typedef struct {
153 long long counter;
154} __attribute__ ((aligned (8))) atomic64_t;
155#define ATOMIC64_INIT(i) { (i) }
156
157#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
158
159#define __CSG_LOOP(ptr, op_val, op_string) ({ \
160 typeof(ptr->counter) old_val, new_val; \
161 asm volatile( \
162 " lg %0,%2\n" \
163 "0: lgr %1,%0\n" \
164 op_string " %1,%3\n" \
165 " csg %0,%1,%2\n" \
166 " jl 0b" \
167 : "=&d" (old_val), "=&d" (new_val), \
168 "=Q" (((atomic_t *)(ptr))->counter) \
169 : "d" (op_val), "Q" (((atomic_t *)(ptr))->counter) \
170 : "cc", "memory" ); \
171 new_val; \
172})
173
174#else
175
176#define __CSG_LOOP(ptr, op_val, op_string) ({ \
177 typeof(ptr->counter) old_val, new_val; \
178 asm volatile( \
179 " lg %0,0(%3)\n" \
180 "0: lgr %1,%0\n" \
181 op_string " %1,%4\n" \
182 " csg %0,%1,0(%3)\n" \
183 " jl 0b" \
184 : "=&d" (old_val), "=&d" (new_val), \
185 "=m" (((atomic_t *)(ptr))->counter) \
186 : "a" (ptr), "d" (op_val), \
187 "m" (((atomic_t *)(ptr))->counter) \
188 : "cc", "memory" ); \
189 new_val; \
190})
191
192#endif
193
194static inline long long atomic64_read(const atomic64_t *v)
195{
196 barrier();
197 return v->counter;
198}
199
200static inline void atomic64_set(atomic64_t *v, long long i)
201{
202 v->counter = i;
203 barrier();
204}
205
206static __inline__ long long atomic64_add_return(long long i, atomic64_t * v)
207{
208 return __CSG_LOOP(v, i, "agr");
209}
210#define atomic64_add(_i, _v) atomic64_add_return(_i, _v)
211#define atomic64_add_negative(_i, _v) (atomic64_add_return(_i, _v) < 0)
212#define atomic64_inc(_v) atomic64_add_return(1, _v)
213#define atomic64_inc_return(_v) atomic64_add_return(1, _v)
214#define atomic64_inc_and_test(_v) (atomic64_add_return(1, _v) == 0)
215
216static __inline__ long long atomic64_sub_return(long long i, atomic64_t * v)
217{
218 return __CSG_LOOP(v, i, "sgr");
219}
220#define atomic64_sub(_i, _v) atomic64_sub_return(_i, _v)
221#define atomic64_sub_and_test(_i, _v) (atomic64_sub_return(_i, _v) == 0)
222#define atomic64_dec(_v) atomic64_sub_return(1, _v)
223#define atomic64_dec_return(_v) atomic64_sub_return(1, _v)
224#define atomic64_dec_and_test(_v) (atomic64_sub_return(1, _v) == 0)
225
226static __inline__ void atomic64_clear_mask(unsigned long mask, atomic64_t * v)
227{
228 __CSG_LOOP(v, ~mask, "ngr");
229}
230
231static __inline__ void atomic64_set_mask(unsigned long mask, atomic64_t * v)
232{
233 __CSG_LOOP(v, mask, "ogr");
234}
235
236#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
237
238static __inline__ long long atomic64_cmpxchg(atomic64_t *v,
239 long long old, long long new)
240{
241#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
242 asm volatile(
243 " csg %0,%2,%1"
244 : "+d" (old), "=Q" (v->counter)
245 : "d" (new), "Q" (v->counter)
246 : "cc", "memory");
247#else
248 asm volatile(
249 " csg %0,%3,0(%2)"
250 : "+d" (old), "=m" (v->counter)
251 : "a" (v), "d" (new), "m" (v->counter)
252 : "cc", "memory");
253#endif
254 return old;
255}
256
257static __inline__ int atomic64_add_unless(atomic64_t *v,
258 long long a, long long u)
259{
260 long long c, old;
261 c = atomic64_read(v);
262 for (;;) {
263 if (unlikely(c == u))
264 break;
265 old = atomic64_cmpxchg(v, c, c + a);
266 if (likely(old == c))
267 break;
268 c = old;
269 }
270 return c != u;
271}
272
273#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
274
275#undef __CSG_LOOP
276#endif
277
278#define smp_mb__before_atomic_dec() smp_mb()
279#define smp_mb__after_atomic_dec() smp_mb()
280#define smp_mb__before_atomic_inc() smp_mb()
281#define smp_mb__after_atomic_inc() smp_mb()
282
283#include <asm-generic/atomic.h>
284#endif
285#endif
286
