1#ifndef __ASM_SPINLOCK_H
2#define __ASM_SPINLOCK_H
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <asm/memory.h>
19#include <asm/hvcall.h>
20
21
22
23
24
25
26
27
28typedef struct {
29 volatile unsigned long lock;
30} spinlock_t;
31
32#ifdef __KERNEL__
33#define SPIN_LOCK_UNLOCKED (spinlock_t) { 0 }
34
35#define spin_is_locked(x) ((x)->lock != 0)
36
37static __inline__ int spin_trylock(spinlock_t *lock)
38{
39 unsigned long tmp;
40
41 __asm__ __volatile__(
42"1: ldarx %0,0,%1 # spin_trylock\n\
43 cmpdi 0,%0,0\n\
44 li %0,0\n\
45 bne- 2f\n\
46 li %0,1\n\
47 stdcx. 13,0,%1\n\
48 bne- 1b\n\
49 isync\n\
502:" : "=&r"(tmp)
51 : "r"(&lock->lock)
52 : "cr0", "memory");
53
54 return tmp;
55}
56
57
58
59
60
61
62#ifdef CONFIG_PPC_ISERIES
63static __inline__ void spin_lock(spinlock_t *lock)
64{
65 unsigned long tmp, tmp2;
66
67 __asm__ __volatile__(
68 "b 2f # spin_lock\n\
691:"
70 HMT_LOW
71" ldx %0,0,%2 # load the lock value\n\
72 cmpdi 0,%0,0 # if not locked, try to acquire\n\
73 beq- 2f\n\
74 lwz 5,0x280(%0) # load yield counter\n\
75 andi. %1,5,1 # if even then spin\n\
76 beq 1b\n\
77 lwsync # if odd, give up cycles\n\
78 ldx %1,0,%2 # reverify the lock holder\n\
79 cmpd %0,%1\n\
80 bne 1b # new holder so restart\n\
81 li 3,0x25 # yield hcall 0x8-12 \n\
82 rotrdi 3,3,1 # put the bits in the right spot\n\
83 lhz 4,0x18(%0) # processor number\n\
84 sldi 4,4,32 # move into top half of word\n\
85 or 5,5,4 # r5 has yield cnt - or it in\n\
86 li 4,2 # yield to processor\n\
87 li 0,-1 # indicate an hcall\n\
88 sc # do the hcall \n\
89 b 1b\n\
902: \n"
91 HMT_MEDIUM
92" ldarx %0,0,%2\n\
93 cmpdi 0,%0,0\n\
94 bne- 1b\n\
95 stdcx. 13,0,%2\n\
96 bne- 2b\n\
97 isync"
98 : "=&r"(tmp), "=&r"(tmp2)
99 : "r"(&lock->lock)
100 : "r0", "r3", "r4", "r5", "ctr", "cr0", "cr1", "cr2", "cr3", "cr4",
101 "xer", "memory");
102}
103#else
104#ifdef SPLPAR_LOCKS
105static __inline__ void spin_lock(spinlock_t *lock)
106{
107 unsigned long tmp, tmp2;
108
109 __asm__ __volatile__(
110 "b 2f # spin_lock\n\
1111:"
112 HMT_LOW
113" ldx %0,0,%2 # load the lock value\n\
114 cmpdi 0,%0,0 # if not locked, try to acquire\n\
115 beq- 2f\n\
116 lwz 5,0x280(%0) # load dispatch counter\n\
117 andi. %1,5,1 # if even then spin\n\
118 beq 1b\n\
119 lwsync # if odd, give up cycles\n\
120 ldx %1,0,%2 # reverify the lock holder\n\
121 cmpd %0,%1\n\
122 bne 1b # new holder so restart\n\
123 li 3,0xE4 # give up the cycles H_CONFER\n\
124 lhz 4,0x18(%0) # processor number\n\
125 # r5 has dispatch cnt already\n"
126 HVSC
127" b 1b\n\
1282: \n"
129 HMT_MEDIUM
130" ldarx %0,0,%2\n\
131 cmpdi 0,%0,0\n\
132 bne- 1b\n\
133 stdcx. 13,0,%2\n\
134 bne- 2b\n\
135 isync"
136 : "=&r"(tmp), "=&r"(tmp2)
137 : "r"(&lock->lock)
138 : "r3", "r4", "r5", "cr0", "cr1", "ctr", "xer", "memory");
139}
140#else
141static __inline__ void spin_lock(spinlock_t *lock)
142{
143 unsigned long tmp;
144
145 __asm__ __volatile__(
146 "b 2f # spin_lock\n\
1471:"
148 HMT_LOW
149" ldx %0,0,%1 # load the lock value\n\
150 cmpdi 0,%0,0 # if not locked, try to acquire\n\
151 bne+ 1b\n\
1522: \n"
153 HMT_MEDIUM
154" ldarx %0,0,%1\n\
155 cmpdi 0,%0,0\n\
156 bne- 1b\n\
157 stdcx. 13,0,%1\n\
158 bne- 2b\n\
159 isync"
160 : "=&r"(tmp)
161 : "r"(&lock->lock)
162 : "cr0", "memory");
163}
164#endif
165#endif
166
167static __inline__ void spin_unlock(spinlock_t *lock)
168{
169 __asm__ __volatile__("lwsync # spin_unlock": : :"memory");
170 lock->lock = 0;
171}
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192typedef struct {
193 volatile signed long lock;
194} rwlock_t;
195
196#define RW_LOCK_UNLOCKED (rwlock_t) { 0 }
197
198static __inline__ int read_trylock(rwlock_t *rw)
199{
200 unsigned long tmp;
201 unsigned int ret;
202
203 __asm__ __volatile__(
204"1: ldarx %0,0,%2 # read_trylock\n\
205 li %1,0\n\
206 addic. %0,%0,1\n\
207 ble- 2f\n\
208 stdcx. %0,0,%2\n\
209 bne- 1b\n\
210 li %1,1\n\
211 isync\n\
2122:" : "=&r"(tmp), "=&r"(ret)
213 : "r"(&rw->lock)
214 : "cr0", "memory");
215
216 return ret;
217}
218
219#ifdef CONFIG_PPC_ISERIES
220static __inline__ void read_lock(rwlock_t *rw)
221{
222 unsigned long tmp, tmp2;
223
224 __asm__ __volatile__(
225 "b 2f # read_lock\n\
2261:"
227 HMT_LOW
228" ldx %0,0,%2\n\
229 cmpdi 0,%0,0\n\
230 bge- 2f\n\
231 lwz 5,0x280(%0) # load yield counter\n\
232 andi. %1,5,1 # if even then spin\n\
233 beq 1b\n\
234 lwsync # if odd, give up cycles\n\
235 ldx %1,0,%2 # reverify the lock holder\n\
236 cmpd %0,%1\n\
237 bne 1b # new holder so restart\n\
238 li 3,0x25 # yield hcall 0x8-12 \n\
239 rotrdi 3,3,1 # put the bits in the right spot\n\
240 lhz 4,0x18(%0) # processor number\n\
241 sldi 4,4,32 # move into top half of word\n\
242 or 5,5,4 # r5 has yield cnt - or it in\n\
243 li 4,2 # yield to processor\n\
244 li 0,-1 # indicate an hcall\n\
245 sc # do the hcall \n\
2462: \n"
247 HMT_MEDIUM
248" ldarx %0,0,%2\n\
249 addic. %0,%0,1\n\
250 ble- 1b\n\
251 stdcx. %0,0,%2\n\
252 bne- 2b\n\
253 isync"
254 : "=&r"(tmp), "=&r"(tmp2)
255 : "r"(&rw->lock)
256 : "r0", "r3", "r4", "r5", "ctr", "cr0", "cr1", "cr2", "cr3", "cr4",
257 "xer", "memory");
258}
259#else
260#ifdef SPLPAR_LOCKS
261static __inline__ void read_lock(rwlock_t *rw)
262{
263 unsigned long tmp, tmp2;
264
265 __asm__ __volatile__(
266 "b 2f # read_lock\n\
2671:"
268 HMT_LOW
269" ldx %0,0,%2\n\
270 cmpdi 0,%0,0\n\
271 bge- 2f\n\
272 lwz 5,0x280(%0) # load dispatch counter\n\
273 andi. %1,5,1 # if even then spin\n\
274 beq 1b\n\
275 lwsync # if odd, give up cycles\n\
276 ldx %1,0,%2 # reverify the lock holder\n\
277 cmpd %0,%1\n\
278 bne 1b # new holder so restart\n\
279 li 3,0xE4 # give up the cycles H_CONFER\n\
280 lhz 4,0x18(%0) # processor number\n\
281 # r5 has dispatch cnt already\n"
282 HVSC
283"2: \n"
284 HMT_MEDIUM
285" ldarx %0,0,%2\n\
286 addic. %0,%0,1\n\
287 ble- 1b\n\
288 stdcx. %0,0,%2\n\
289 bne- 2b\n\
290 isync"
291 : "=&r"(tmp), "=&r"(tmp2)
292 : "r"(&rw->lock)
293 : "r3", "r4", "r5", "cr0", "cr1", "ctr", "xer", "memory");
294}
295#else
296static __inline__ void read_lock(rwlock_t *rw)
297{
298 unsigned long tmp;
299
300 __asm__ __volatile__(
301 "b 2f # read_lock\n\
3021:"
303 HMT_LOW
304" ldx %0,0,%1\n\
305 cmpdi 0,%0,0\n\
306 blt+ 1b\n\
3072: \n"
308 HMT_MEDIUM
309" ldarx %0,0,%1\n\
310 addic. %0,%0,1\n\
311 ble- 1b\n\
312 stdcx. %0,0,%1\n\
313 bne- 2b\n\
314 isync"
315 : "=&r"(tmp)
316 : "r"(&rw->lock)
317 : "cr0", "memory");
318}
319#endif
320#endif
321
322static __inline__ void read_unlock(rwlock_t *rw)
323{
324 unsigned long tmp;
325
326 __asm__ __volatile__(
327 "eieio # read_unlock\n\
3281: ldarx %0,0,%1\n\
329 addic %0,%0,-1\n\
330 stdcx. %0,0,%1\n\
331 bne- 1b"
332 : "=&r"(tmp)
333 : "r"(&rw->lock)
334 : "cr0", "memory");
335}
336
337static __inline__ int write_trylock(rwlock_t *rw)
338{
339 unsigned long tmp;
340 unsigned long ret;
341
342 __asm__ __volatile__(
343"1: ldarx %0,0,%2 # write_trylock\n\
344 cmpdi 0,%0,0\n\
345 li %1,0\n\
346 bne- 2f\n\
347 stdcx. 13,0,%2\n\
348 bne- 1b\n\
349 li %1,1\n\
350 isync\n\
3512:" : "=&r"(tmp), "=&r"(ret)
352 : "r"(&rw->lock)
353 : "cr0", "memory");
354
355 return ret;
356}
357
358#ifdef CONFIG_PPC_ISERIES
359static __inline__ void write_lock(rwlock_t *rw)
360{
361 unsigned long tmp, tmp2;
362
363 __asm__ __volatile__(
364 "b 2f # spin_lock\n\
3651:"
366 HMT_LOW
367" ldx %0,0,%2 # load the lock value\n\
368 cmpdi 0,%0,0 # if not locked(0), try to acquire\n\
369 beq- 2f\n\
370 bgt 1b # negative(0xc..)->cycles to holder\n"
371"3: lwz 5,0x280(%0) # load yield counter\n\
372 andi. %1,5,1 # if even then spin\n\
373 beq 1b\n\
374 lwsync # if odd, give up cycles\n\
375 ldx %1,0,%2 # reverify the lock holder\n\
376 cmpd %0,%1\n\
377 bne 1b # new holder so restart\n\
378 lhz 4,0x18(%0) # processor number\n\
379 sldi 4,4,32 # move into top half of word\n\
380 or 5,5,4 # r5 has yield cnt - or it in\n\
381 li 3,0x25 # yield hcall 0x8-12 \n\
382 rotrdi 3,3,1 # put the bits in the right spot\n\
383 li 4,2 # yield to processor\n\
384 li 0,-1 # indicate an hcall\n\
385 sc # do the hcall \n\
3862: \n"
387 HMT_MEDIUM
388" ldarx %0,0,%2\n\
389 cmpdi 0,%0,0\n\
390 bne- 1b\n\
391 stdcx. 13,0,%2\n\
392 bne- 2b\n\
393 isync"
394 : "=&r"(tmp), "=&r"(tmp2)
395 : "r"(&rw->lock)
396 : "r0", "r3", "r4", "r5", "ctr", "cr0", "cr1", "cr2", "cr3", "cr4",
397 "xer", "memory");
398}
399#else
400#ifdef SPLPAR_LOCKS
401static __inline__ void write_lock(rwlock_t *rw)
402{
403 unsigned long tmp, tmp2;
404
405 __asm__ __volatile__(
406 "b 2f # spin_lock\n\
4071:"
408 HMT_LOW
409" ldx %0,0,%2 # load the lock value\n\
410 li 3,0xE4 # give up the cycles H_CONFER\n\
411 cmpdi 0,%0,0 # if not locked(0), try to acquire\n\
412 beq- 2f\n\
413 blt 3f # negative(0xc..)->confer to holder\n\
414 b 1b\n"
415"3: lwz 5,0x280(%0) # load dispatch counter\n\
416 andi. %1,5,1 # if even then spin\n\
417 beq 1b\n\
418 lwsync # if odd, give up cycles\n\
419 ldx %1,0,%2 # reverify the lock holder\n\
420 cmpd %0,%1\n\
421 bne 1b # new holder so restart\n\
422 lhz 4,0x18(%0) # processor number\n\
423 # r5 has dispatch cnt already\n"
424 HVSC
425" b 1b\n\
4262: \n"
427 HMT_MEDIUM
428" ldarx %0,0,%2\n\
429 cmpdi 0,%0,0\n\
430 bne- 1b\n\
431 stdcx. 13,0,%2\n\
432 bne- 2b\n\
433 isync"
434 : "=&r"(tmp), "=&r"(tmp2)
435 : "r"(&rw->lock)
436 : "r3", "r4", "r5", "cr0", "cr1", "ctr", "xer", "memory");
437}
438#else
439static __inline__ void write_lock(rwlock_t *rw)
440{
441 unsigned long tmp;
442
443 __asm__ __volatile__(
444 "b 2f # spin_lock\n\
4451:"
446 HMT_LOW
447" ldx %0,0,%1 # load the lock value\n\
448 cmpdi 0,%0,0 # if not locked(0), try to acquire\n\
449 bne+ 1b\n\
4502: \n"
451 HMT_MEDIUM
452" ldarx %0,0,%1\n\
453 cmpdi 0,%0,0\n\
454 bne- 1b\n\
455 stdcx. 13,0,%1\n\
456 bne- 2b\n\
457 isync"
458 : "=&r"(tmp)
459 : "r"(&rw->lock)
460 : "cr0", "memory");
461}
462#endif
463#endif
464
465static __inline__ void write_unlock(rwlock_t *rw)
466{
467 __asm__ __volatile__("lwsync # write_unlock": : :"memory");
468 rw->lock = 0;
469}
470
471static __inline__ int is_read_locked(rwlock_t *rw)
472{
473 return rw->lock > 0;
474}
475
476static __inline__ int is_write_locked(rwlock_t *rw)
477{
478 return rw->lock < 0;
479}
480
481#define spin_lock_init(x) do { *(x) = SPIN_LOCK_UNLOCKED; } while(0)
482#define spin_unlock_wait(x) do { barrier(); } while(spin_is_locked(x))
483
484#define rwlock_init(x) do { *(x) = RW_LOCK_UNLOCKED; } while(0)
485
486#endif
487#endif
488
