diff options
Diffstat (limited to 'arch')
70 files changed, 110 insertions, 297 deletions
diff --git a/arch/alpha/include/asm/atomic.h b/arch/alpha/include/asm/atomic.h index 78b03ef39f6f..ed60a1ee1ed3 100644 --- a/arch/alpha/include/asm/atomic.h +++ b/arch/alpha/include/asm/atomic.h @@ -292,9 +292,4 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) #define atomic_dec(v) atomic_sub(1,(v)) #define atomic64_dec(v) atomic64_sub(1,(v)) -#define smp_mb__before_atomic_dec() smp_mb() -#define smp_mb__after_atomic_dec() smp_mb() -#define smp_mb__before_atomic_inc() smp_mb() -#define smp_mb__after_atomic_inc() smp_mb() - #endif /* _ALPHA_ATOMIC_H */ diff --git a/arch/alpha/include/asm/bitops.h b/arch/alpha/include/asm/bitops.h index a19ba5efea4c..4bdfbd444e63 100644 --- a/arch/alpha/include/asm/bitops.h +++ b/arch/alpha/include/asm/bitops.h @@ -53,9 +53,6 @@ __set_bit(unsigned long nr, volatile void * addr) *m |= 1 << (nr & 31); } -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() - static inline void clear_bit(unsigned long nr, volatile void * addr) { diff --git a/arch/arc/include/asm/atomic.h b/arch/arc/include/asm/atomic.h index 03e494f695d1..83f03ca6caf6 100644 --- a/arch/arc/include/asm/atomic.h +++ b/arch/arc/include/asm/atomic.h @@ -190,11 +190,6 @@ static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr) #endif /* !CONFIG_ARC_HAS_LLSC */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - /** * __atomic_add_unless - add unless the number is a given value * @v: pointer of type atomic_t diff --git a/arch/arc/include/asm/bitops.h b/arch/arc/include/asm/bitops.h index 647a83a8e756..ebc0cf3164dc 100644 --- a/arch/arc/include/asm/bitops.h +++ b/arch/arc/include/asm/bitops.h @@ -19,6 +19,7 @@ #include <linux/types.h> #include <linux/compiler.h> +#include <asm/barrier.h> /* * Hardware assisted read-modify-write using ARC700 LLOCK/SCOND insns. @@ -496,10 +497,6 @@ static inline __attribute__ ((const)) int __ffs(unsigned long word) */ #define ffz(x) __ffs(~(x)) -/* TODO does this affect uni-processor code */ -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() - #include <asm-generic/bitops/hweight.h> #include <asm-generic/bitops/fls64.h> #include <asm-generic/bitops/sched.h> diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index 9a92fd7864a8..3040359094d9 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -241,11 +241,6 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) #define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0) -#define smp_mb__before_atomic_dec() smp_mb() -#define smp_mb__after_atomic_dec() smp_mb() -#define smp_mb__before_atomic_inc() smp_mb() -#define smp_mb__after_atomic_inc() smp_mb() - #ifndef CONFIG_GENERIC_ATOMIC64 typedef struct { long long counter; diff --git a/arch/arm/include/asm/barrier.h b/arch/arm/include/asm/barrier.h index 2f59f7443396..c6a3e73a6e24 100644 --- a/arch/arm/include/asm/barrier.h +++ b/arch/arm/include/asm/barrier.h @@ -79,5 +79,8 @@ do { \ #define set_mb(var, value) do { var = value; smp_mb(); } while (0) +#define smp_mb__before_atomic() smp_mb() +#define smp_mb__after_atomic() smp_mb() + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_BARRIER_H */ diff --git a/arch/arm/include/asm/bitops.h b/arch/arm/include/asm/bitops.h index b2e298a90d76..56380995f4c3 100644 --- a/arch/arm/include/asm/bitops.h +++ b/arch/arm/include/asm/bitops.h @@ -25,9 +25,7 @@ #include <linux/compiler.h> #include <linux/irqflags.h> - -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() +#include <asm/barrier.h> /* * These functions are the basis of our bit ops. diff --git a/arch/arm64/include/asm/atomic.h b/arch/arm64/include/asm/atomic.h index 0237f0867e37..57e8cb49824c 100644 --- a/arch/arm64/include/asm/atomic.h +++ b/arch/arm64/include/asm/atomic.h @@ -152,11 +152,6 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) #define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0) -#define smp_mb__before_atomic_dec() smp_mb() -#define smp_mb__after_atomic_dec() smp_mb() -#define smp_mb__before_atomic_inc() smp_mb() -#define smp_mb__after_atomic_inc() smp_mb() - /* * 64-bit atomic operations. */ diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index 66eb7648043b..48b9e704af7c 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -98,6 +98,9 @@ do { \ #define set_mb(var, value) do { var = value; smp_mb(); } while (0) #define nop() asm volatile("nop"); +#define smp_mb__before_atomic() smp_mb() +#define smp_mb__after_atomic() smp_mb() + #endif /* __ASSEMBLY__ */ #endif /* __ASM_BARRIER_H */ diff --git a/arch/arm64/include/asm/bitops.h b/arch/arm64/include/asm/bitops.h index aa5b59d6ba43..9c19594ce7cb 100644 --- a/arch/arm64/include/asm/bitops.h +++ b/arch/arm64/include/asm/bitops.h @@ -17,17 +17,8 @@ #define __ASM_BITOPS_H #include <linux/compiler.h> - #include <asm/barrier.h> -/* - * clear_bit may not imply a memory barrier - */ -#ifndef smp_mb__before_clear_bit -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() -#endif - #ifndef _LINUX_BITOPS_H #error only <linux/bitops.h> can be included directly #endif diff --git a/arch/avr32/include/asm/atomic.h b/arch/avr32/include/asm/atomic.h index 61407279208a..0780f3f2415b 100644 --- a/arch/avr32/include/asm/atomic.h +++ b/arch/avr32/include/asm/atomic.h @@ -183,9 +183,4 @@ static inline int atomic_sub_if_positive(int i, atomic_t *v) #define atomic_dec_if_positive(v) atomic_sub_if_positive(1, v) -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #endif /* __ASM_AVR32_ATOMIC_H */ diff --git a/arch/avr32/include/asm/bitops.h b/arch/avr32/include/asm/bitops.h index ebe7ad3f490b..910d5374ce59 100644 --- a/arch/avr32/include/asm/bitops.h +++ b/arch/avr32/include/asm/bitops.h @@ -13,12 +13,7 @@ #endif #include <asm/byteorder.h> - -/* - * clear_bit() doesn't provide any barrier for the compiler - */ -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() +#include <asm/barrier.h> /* * set_bit - Atomically set a bit in memory @@ -67,7 +62,7 @@ static inline void set_bit(int nr, volatile void * addr) * * clear_bit() is atomic and may not be reordered. However, it does * not contain a memory barrier, so if it is used for locking purposes, - * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() + * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic() * in order to ensure changes are visible on other processors. */ static inline void clear_bit(int nr, volatile void * addr) diff --git a/arch/blackfin/include/asm/barrier.h b/arch/blackfin/include/asm/barrier.h index 19283a16ac08..420006877998 100644 --- a/arch/blackfin/include/asm/barrier.h +++ b/arch/blackfin/include/asm/barrier.h @@ -27,6 +27,9 @@ #endif /* !CONFIG_SMP */ +#define smp_mb__before_atomic() barrier() +#define smp_mb__after_atomic() barrier() + #include <asm-generic/barrier.h> #endif /* _BLACKFIN_BARRIER_H */ diff --git a/arch/blackfin/include/asm/bitops.h b/arch/blackfin/include/asm/bitops.h index 0ca40dd44724..b298b654a26f 100644 --- a/arch/blackfin/include/asm/bitops.h +++ b/arch/blackfin/include/asm/bitops.h @@ -27,21 +27,17 @@ #include <asm-generic/bitops/ext2-atomic.h> +#include <asm/barrier.h> + #ifndef CONFIG_SMP #include <linux/irqflags.h> - /* * clear_bit may not imply a memory barrier */ -#ifndef smp_mb__before_clear_bit -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() -#endif #include <asm-generic/bitops/atomic.h> #include <asm-generic/bitops/non-atomic.h> #else -#include <asm/barrier.h> #include <asm/byteorder.h> /* swab32 */ #include <linux/linkage.h> @@ -101,12 +97,6 @@ static inline int test_and_change_bit(int nr, volatile unsigned long *addr) return __raw_bit_test_toggle_asm(a, nr & 0x1f); } -/* - * clear_bit() doesn't provide any barrier for the compiler. - */ -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() - #define test_bit __skip_test_bit #include <asm-generic/bitops/non-atomic.h> #undef test_bit diff --git a/arch/c6x/include/asm/bitops.h b/arch/c6x/include/asm/bitops.h index 0bec7e5036a8..f0ab012401b6 100644 --- a/arch/c6x/include/asm/bitops.h +++ b/arch/c6x/include/asm/bitops.h @@ -14,14 +14,8 @@ #ifdef __KERNEL__ #include <linux/bitops.h> - #include <asm/byteorder.h> - -/* - * clear_bit() doesn't provide any barrier for the compiler. - */ -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() +#include <asm/barrier.h> /* * We are lucky, DSP is perfect for bitops: do it in 3 cycles diff --git a/arch/cris/include/asm/atomic.h b/arch/cris/include/asm/atomic.h index 1056a5dfe04f..aa429baebaf9 100644 --- a/arch/cris/include/asm/atomic.h +++ b/arch/cris/include/asm/atomic.h @@ -7,6 +7,8 @@ #include <linux/types.h> #include <asm/cmpxchg.h> #include <arch/atomic.h> +#include <arch/system.h> +#include <asm/barrier.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -151,10 +153,4 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) return ret; } -/* Atomic operations are already serializing */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #endif diff --git a/arch/cris/include/asm/bitops.h b/arch/cris/include/asm/bitops.h index 053c17b36559..bd49a546f4f5 100644 --- a/arch/cris/include/asm/bitops.h +++ b/arch/cris/include/asm/bitops.h @@ -21,6 +21,7 @@ #include <arch/bitops.h> #include <linux/atomic.h> #include <linux/compiler.h> +#include <asm/barrier.h> /* * set_bit - Atomically set a bit in memory @@ -42,7 +43,7 @@ * * clear_bit() is atomic and may not be reordered. However, it does * not contain a memory barrier, so if it is used for locking purposes, - * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() + * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic() * in order to ensure changes are visible on other processors. */ @@ -84,12 +85,6 @@ static inline int test_and_set_bit(int nr, volatile unsigned long *addr) return retval; } -/* - * clear_bit() doesn't provide any barrier for the compiler. - */ -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() - /** * test_and_clear_bit - Clear a bit and return its old value * @nr: Bit to clear diff --git a/arch/frv/include/asm/atomic.h b/arch/frv/include/asm/atomic.h index b86329d0e316..f6c3a1690101 100644 --- a/arch/frv/include/asm/atomic.h +++ b/arch/frv/include/asm/atomic.h @@ -17,6 +17,7 @@ #include <linux/types.h> #include <asm/spr-regs.h> #include <asm/cmpxchg.h> +#include <asm/barrier.h> #ifdef CONFIG_SMP #error not SMP safe @@ -29,12 +30,6 @@ * We do not have SMP systems, so we don't have to deal with that. */ -/* Atomic operations are already serializing */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #define ATOMIC_INIT(i) { (i) } #define atomic_read(v) (*(volatile int *)&(v)->counter) #define atomic_set(v, i) (((v)->counter) = (i)) diff --git a/arch/frv/include/asm/bitops.h b/arch/frv/include/asm/bitops.h index 57bf85db893f..96de220ef131 100644 --- a/arch/frv/include/asm/bitops.h +++ b/arch/frv/include/asm/bitops.h @@ -25,12 +25,6 @@ #include <asm-generic/bitops/ffz.h> -/* - * clear_bit() doesn't provide any barrier for the compiler. - */ -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() - #ifndef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS static inline unsigned long atomic_test_and_ANDNOT_mask(unsigned long mask, volatile unsigned long *v) diff --git a/arch/hexagon/include/asm/atomic.h b/arch/hexagon/include/asm/atomic.h index 17dc63780c06..de916b11bff5 100644 --- a/arch/hexagon/include/asm/atomic.h +++ b/arch/hexagon/include/asm/atomic.h @@ -24,6 +24,7 @@ #include <linux/types.h> #include <asm/cmpxchg.h> +#include <asm/barrier.h> #define ATOMIC_INIT(i) { (i) } @@ -176,9 +177,4 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) #define atomic_inc_return(v) (atomic_add_return(1, v)) #define atomic_dec_return(v) (atomic_sub_return(1, v)) -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #endif diff --git a/arch/hexagon/include/asm/bitops.h b/arch/hexagon/include/asm/bitops.h index 9b1e4afbab3c..5e4a59b3ec1b 100644 --- a/arch/hexagon/include/asm/bitops.h +++ b/arch/hexagon/include/asm/bitops.h @@ -25,12 +25,10 @@ #include <linux/compiler.h> #include <asm/byteorder.h> #include <asm/atomic.h> +#include <asm/barrier.h> #ifdef __KERNEL__ -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() - /* * The offset calculations for these are based on BITS_PER_LONG == 32 * (i.e. I get to shift by #5-2 (32 bits per long, 4 bytes per access), diff --git a/arch/ia64/include/asm/atomic.h b/arch/ia64/include/asm/atomic.h index 6e6fe1839f5d..0f8bf48dadf3 100644 --- a/arch/ia64/include/asm/atomic.h +++ b/arch/ia64/include/asm/atomic.h @@ -15,6 +15,7 @@ #include <linux/types.h> #include <asm/intrinsics.h> +#include <asm/barrier.h> #define ATOMIC_INIT(i) { (i) } @@ -208,10 +209,4 @@ atomic64_add_negative (__s64 i, atomic64_t *v) #define atomic64_inc(v) atomic64_add(1, (v)) #define atomic64_dec(v) atomic64_sub(1, (v)) -/* Atomic operations are already serializing */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #endif /* _ASM_IA64_ATOMIC_H */ diff --git a/arch/ia64/include/asm/barrier.h b/arch/ia64/include/asm/barrier.h index d0a69aa35e27..a48957c7b445 100644 --- a/arch/ia64/include/asm/barrier.h +++ b/arch/ia64/include/asm/barrier.h @@ -55,6 +55,9 @@ #endif +#define smp_mb__before_atomic() barrier() +#define smp_mb__after_atomic() barrier() + /* * IA64 GCC turns volatile stores into st.rel and volatile loads into ld.acq no * need for asm trickery! diff --git a/arch/ia64/include/asm/bitops.h b/arch/ia64/include/asm/bitops.h index c27eccd33349..71e8145243ee 100644 --- a/arch/ia64/include/asm/bitops.h +++ b/arch/ia64/include/asm/bitops.h @@ -16,6 +16,7 @@ #include <linux/compiler.h> #include <linux/types.h> #include <asm/intrinsics.h> +#include <asm/barrier.h> /** * set_bit - Atomically set a bit in memory @@ -65,12 +66,6 @@ __set_bit (int nr, volatile void *addr) *((__u32 *) addr + (nr >> 5)) |= (1 << (nr & 31)); } -/* - * clear_bit() has "acquire" semantics. - */ -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() do { /* skip */; } while (0) - /** * clear_bit - Clears a bit in memory * @nr: Bit to clear @@ -78,7 +73,7 @@ __set_bit (int nr, volatile void *addr) * * clear_bit() is atomic and may not be reordered. However, it does * not contain a memory barrier, so if it is used for locking purposes, - * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() + * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic() * in order to ensure changes are visible on other processors. */ static __inline__ void diff --git a/arch/ia64/include/uapi/asm/cmpxchg.h b/arch/ia64/include/uapi/asm/cmpxchg.h index 4f37dbbb8640..f35109b1d907 100644 --- a/arch/ia64/include/uapi/asm/cmpxchg.h +++ b/arch/ia64/include/uapi/asm/cmpxchg.h @@ -118,6 +118,15 @@ extern long ia64_cmpxchg_called_with_bad_pointer(void); #define cmpxchg_rel(ptr, o, n) \ ia64_cmpxchg(rel, (ptr), (o), (n), sizeof(*(ptr))) +/* + * Worse still - early processor implementations actually just ignored + * the acquire/release and did a full fence all the time. Unfortunately + * this meant a lot of badly written code that used .acq when they really + * wanted .rel became legacy out in the wild - so when we made a cpu + * that strictly did the .acq or .rel ... all that code started breaking - so + * we had to back-pedal and keep the "legacy" behavior of a full fence :-( + */ + /* for compatibility with other platforms: */ #define cmpxchg(ptr, o, n) cmpxchg_acq((ptr), (o), (n)) #define cmpxchg64(ptr, o, n) cmpxchg_acq((ptr), (o), (n)) diff --git a/arch/m32r/include/asm/atomic.h b/arch/m32r/include/asm/atomic.h index 0d81697c326c..8ad0ed4182a5 100644 --- a/arch/m32r/include/asm/atomic.h +++ b/arch/m32r/include/asm/atomic.h @@ -13,6 +13,7 @@ #include <asm/assembler.h> #include <asm/cmpxchg.h> #include <asm/dcache_clear.h> +#include <asm/barrier.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -308,10 +309,4 @@ static __inline__ void atomic_set_mask(unsigned long mask, atomic_t *addr) local_irq_restore(flags); } -/* Atomic operations are already serializing on m32r */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #endif /* _ASM_M32R_ATOMIC_H */ diff --git a/arch/m32r/include/asm/bitops.h b/arch/m32r/include/asm/bitops.h index d3dea9ac7d4e..86ba2b42a6cf 100644 --- a/arch/m32r/include/asm/bitops.h +++ b/arch/m32r/include/asm/bitops.h @@ -21,6 +21,7 @@ #include <asm/byteorder.h> #include <asm/dcache_clear.h> #include <asm/types.h> +#include <asm/barrier.h> /* * These have to be done with inline assembly: that way the bit-setting @@ -73,7 +74,7 @@ static __inline__ void set_bit(int nr, volatile void * addr) * * clear_bit() is atomic and may not be reordered. However, it does * not contain a memory barrier, so if it is used for locking purposes, - * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() + * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic() * in order to ensure changes are visible on other processors. */ static __inline__ void clear_bit(int nr, volatile void * addr) @@ -103,9 +104,6 @@ static __inline__ void clear_bit(int nr, volatile void * addr) local_irq_restore(flags); } -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() - /** * change_bit - Toggle a bit in memory * @nr: Bit to clear diff --git a/arch/m68k/include/asm/atomic.h b/arch/m68k/include/asm/atomic.h index f4e32de263a7..55695212a2ae 100644 --- a/arch/m68k/include/asm/atomic.h +++ b/arch/m68k/include/asm/atomic.h @@ -4,6 +4,7 @@ #include <linux/types.h> #include <linux/irqflags.h> #include <asm/cmpxchg.h> +#include <asm/barrier.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -209,11 +210,4 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) return c; } - -/* Atomic operations are already serializing */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #endif /* __ARCH_M68K_ATOMIC __ */ diff --git a/arch/m68k/include/asm/bitops.h b/arch/m68k/include/asm/bitops.h index c6baa913592a..b4a9b0d5928d 100644 --- a/arch/m68k/include/asm/bitops.h +++ b/arch/m68k/include/asm/bitops.h @@ -13,6 +13,7 @@ #endif #include <linux/compiler.h> +#include <asm/barrier.h> /* * Bit access functions vary across the ColdFire and 68k families. @@ -67,12 +68,6 @@ static inline void bfset_mem_set_bit(int nr, volatile unsigned long *vaddr) #define __set_bit(nr, vaddr) set_bit(nr, vaddr) -/* - * clear_bit() doesn't provide any barrier for the compiler. - */ -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() - static inline void bclr_reg_clear_bit(int nr, volatile unsigned long *vaddr) { char *p = (char *)vaddr + (nr ^ 31) / 8; diff --git a/arch/metag/include/asm/atomic.h b/arch/metag/include/asm/atomic.h index 307ecd2bd9a1..470e365f04ea 100644 --- a/arch/metag/include/asm/atomic.h +++ b/arch/metag/include/asm/atomic.h @@ -4,6 +4,7 @@ #include <linux/compiler.h> #include <linux/types.h> #include <asm/cmpxchg.h> +#include <asm/barrier.h> #if defined(CONFIG_METAG_ATOMICITY_IRQSOFF) /* The simple UP case. */ @@ -39,11 +40,6 @@ #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #endif #define atomic_dec_if_positive(v) atomic_sub_if_positive(1, v) diff --git a/arch/metag/include/asm/barrier.h b/arch/metag/include/asm/barrier.h index 2d6f0de77325..c7591e80067c 100644 --- a/arch/metag/include/asm/barrier.h +++ b/arch/metag/include/asm/barrier.h @@ -100,4 +100,7 @@ do { \ ___p1; \ }) +#define smp_mb__before_atomic() barrier() +#define smp_mb__after_atomic() barrier() + #endif /* _ASM_METAG_BARRIER_H */ diff --git a/arch/metag/include/asm/bitops.h b/arch/metag/include/asm/bitops.h index c0d0df0d1378..2671134ee745 100644 --- a/arch/metag/include/asm/bitops.h +++ b/arch/metag/include/asm/bitops.h @@ -5,12 +5,6 @@ #include <asm/barrier.h> #include <asm/global_lock.h> -/* - * clear_bit() doesn't provide any barrier for the compiler. - */ -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() - #ifdef CONFIG_SMP /* * These functions are the basis of our bit ops. diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h index e8eb3d53a241..37b2befe651a 100644 --- a/arch/mips/include/asm/atomic.h +++ b/arch/mips/include/asm/atomic.h @@ -761,13 +761,4 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) #endif /* CONFIG_64BIT */ -/* - * atomic*_return operations are serializing but not the non-*_return - * versions. - */ -#define smp_mb__before_atomic_dec() smp_mb__before_llsc() -#define smp_mb__after_atomic_dec() smp_llsc_mb() -#define smp_mb__before_atomic_inc() smp_mb__before_llsc() -#define smp_mb__after_atomic_inc() smp_llsc_mb() - #endif /* _ASM_ATOMIC_H */ diff --git a/arch/mips/include/asm/barrier.h b/arch/mips/include/asm/barrier.h index e1aa4e4c2984..d0101dd0575e 100644 --- a/arch/mips/include/asm/barrier.h +++ b/arch/mips/include/asm/barrier.h @@ -195,4 +195,7 @@ do { \ ___p1; \ }) +#define smp_mb__before_atomic() smp_mb__before_llsc() +#define smp_mb__after_atomic() smp_llsc_mb() + #endif /* __ASM_BARRIER_H */ diff --git a/arch/mips/include/asm/bitops.h b/arch/mips/include/asm/bitops.h index 6a65d49e2c0d..7c8816f7b7c4 100644 --- a/arch/mips/include/asm/bitops.h +++ b/arch/mips/include/asm/bitops.h @@ -38,13 +38,6 @@ #endif /* - * clear_bit() doesn't provide any barrier for the compiler. - */ -#define smp_mb__before_clear_bit() smp_mb__before_llsc() -#define smp_mb__after_clear_bit() smp_llsc_mb() - - -/* * These are the "slower" versions of the functions and are in bitops.c. * These functions call raw_local_irq_{save,restore}(). */ @@ -120,7 +113,7 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr) * * clear_bit() is atomic and may not be reordered. However, it does * not contain a memory barrier, so if it is used for locking purposes, - * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() + * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic() * in order to ensure changes are visible on other processors. */ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) @@ -175,7 +168,7 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) */ static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr) { - smp_mb__before_clear_bit(); + smp_mb__before_atomic(); clear_bit(nr, addr); } diff --git a/arch/mips/kernel/irq.c b/arch/mips/kernel/irq.c index d1fea7a054be..1818da4dbb85 100644 --- a/arch/mips/kernel/irq.c +++ b/arch/mips/kernel/irq.c @@ -62,9 +62,9 @@ void __init alloc_legacy_irqno(void) void free_irqno(unsigned int irq) { - smp_mb__before_clear_bit(); + smp_mb__before_atomic(); clear_bit(irq, irq_map); - smp_mb__after_clear_bit(); + smp_mb__after_atomic(); } /* diff --git a/arch/mn10300/include/asm/atomic.h b/arch/mn10300/include/asm/atomic.h index 975e1841ca64..cadeb1e2cdfc 100644 --- a/arch/mn10300/include/asm/atomic.h +++ b/arch/mn10300/include/asm/atomic.h @@ -13,6 +13,7 @@ #include <asm/irqflags.h> #include <asm/cmpxchg.h> +#include <asm/barrier.h> #ifndef CONFIG_SMP #include <asm-generic/atomic.h> @@ -234,12 +235,6 @@ static inline void atomic_set_mask(unsigned long mask, unsigned long *addr) #endif } -/* Atomic operations are already serializing on MN10300??? */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #endif /* __KERNEL__ */ #endif /* CONFIG_SMP */ #endif /* _ASM_ATOMIC_H */ diff --git a/arch/mn10300/include/asm/bitops.h b/arch/mn10300/include/asm/bitops.h index 596bb2706d81..fe6f8e2c3617 100644 --- a/arch/mn10300/include/asm/bitops.h +++ b/arch/mn10300/include/asm/bitops.h @@ -18,9 +18,7 @@ #define __ASM_BITOPS_H #include <asm/cpu-regs.h> - -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() +#include <asm/barrier.h> /* * set bit diff --git a/arch/mn10300/mm/tlb-smp.c b/arch/mn10300/mm/tlb-smp.c index 3e57faf04083..e5d0ef722bfa 100644 --- a/arch/mn10300/mm/tlb-smp.c +++ b/arch/mn10300/mm/tlb-smp.c @@ -78,9 +78,9 @@ void smp_flush_tlb(void *unused) else local_flush_tlb_page(flush_mm, flush_va); - smp_mb__before_clear_bit(); + smp_mb__before_atomic(); cpumask_clear_cpu(cpu_id, &flush_cpumask); - smp_mb__after_clear_bit(); + smp_mb__after_atomic(); out: put_cpu(); } diff --git a/arch/openrisc/include/asm/bitops.h b/arch/openrisc/include/asm/bitops.h index 2c64f2228dc7..3003cdad561b 100644 --- a/arch/openrisc/include/asm/bitops.h +++ b/arch/openrisc/include/asm/bitops.h @@ -27,14 +27,7 @@ #include <linux/irqflags.h> #include <linux/compiler.h> - -/* - * clear_bit may not imply a memory barrier - */ -#ifndef smp_mb__before_clear_bit -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() -#endif +#include <asm/barrier.h> #include <asm/bitops/__ffs.h> #include <asm-generic/bitops/ffz.h> diff --git a/arch/parisc/include/asm/atomic.h b/arch/parisc/include/asm/atomic.h index 472886ceab1d..0be2db2c7d44 100644 --- a/arch/parisc/include/asm/atomic.h +++ b/arch/parisc/include/asm/atomic.h @@ -7,6 +7,7 @@ #include <linux/types.h> #include <asm/cmpxchg.h> +#include <asm/barrier.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -143,11 +144,6 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) #define ATOMIC_INIT(i) { (i) } -#define smp_mb__before_atomic_dec() smp_mb() -#define smp_mb__after_atomic_dec() smp_mb() -#define smp_mb__before_atomic_inc() smp_mb() -#define smp_mb__after_atomic_inc() smp_mb() - #ifdef CONFIG_64BIT #define ATOMIC64_INIT(i) { (i) } diff --git a/arch/parisc/include/asm/bitops.h b/arch/parisc/include/asm/bitops.h index 8c9b631d2a78..3f9406d9b9d6 100644 --- a/arch/parisc/include/asm/bitops.h +++ b/arch/parisc/include/asm/bitops.h @@ -8,6 +8,7 @@ #include <linux/compiler.h> #include <asm/types.h> /* for BITS_PER_LONG/SHIFT_PER_LONG */ #include <asm/byteorder.h> +#include <asm/barrier.h> #include <linux/atomic.h> /* @@ -19,9 +20,6 @@ #define CHOP_SHIFTCOUNT(x) (((unsigned long) (x)) & (BITS_PER_LONG - 1)) -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() - /* See http://marc.theaimsgroup.com/?t=108826637900003 for discussion * on use of volatile and __*_bit() (set/clear/change): * *_bit() want use of volatile. diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h index e3b1d41c89be..28992d012926 100644 --- a/arch/powerpc/include/asm/atomic.h +++ b/arch/powerpc/include/asm/atomic.h @@ -8,6 +8,7 @@ #ifdef __KERNEL__ #include <linux/types.h> #include <asm/cmpxchg.h> +#include <asm/barrier.h> #define ATOMIC_INIT(i) { (i) } @@ -270,11 +271,6 @@ static __inline__ int atomic_dec_if_positive(atomic_t *v) } #define atomic_dec_if_positive atomic_dec_if_positive -#define smp_mb__before_atomic_dec() smp_mb() -#define smp_mb__after_atomic_dec() smp_mb() -#define smp_mb__before_atomic_inc() smp_mb() -#define smp_mb__after_atomic_inc() smp_mb() - #ifdef __powerpc64__ #define ATOMIC64_INIT(i) { (i) } diff --git a/arch/powerpc/include/asm/barrier.h b/arch/powerpc/include/asm/barrier.h index f89da808ce31..bab79a110c7b 100644 --- a/arch/powerpc/include/asm/barrier.h +++ b/arch/powerpc/include/asm/barrier.h @@ -84,4 +84,7 @@ do { \ ___p1; \ }) +#define smp_mb__before_atomic() smp_mb() +#define smp_mb__after_atomic() smp_mb() + #endif /* _ASM_POWERPC_BARRIER_H */ diff --git a/arch/powerpc/include/asm/bitops.h b/arch/powerpc/include/asm/bitops.h index a5e9a7d494d8..bd3bd573d0ae 100644 --- a/arch/powerpc/include/asm/bitops.h +++ b/arch/powerpc/include/asm/bitops.h @@ -51,11 +51,7 @@ #define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit)) #define PPC_BITMASK(bs, be) ((PPC_BIT(bs) - PPC_BIT(be)) | PPC_BIT(bs)) -/* - * clear_bit doesn't imply a memory barrier - */ -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() +#include <asm/barrier.h> /* Macro for generating the ***_bits() functions */ #define DEFINE_BITOP(fn, op, prefix) \ diff --git a/arch/powerpc/kernel/crash.c b/arch/powerpc/kernel/crash.c index 18d7c80ddeb9..51dbace3269b 100644 --- a/arch/powerpc/kernel/crash.c +++ b/arch/powerpc/kernel/crash.c @@ -81,7 +81,7 @@ void crash_ipi_callback(struct pt_regs *regs) } atomic_inc(&cpus_in_crash); - smp_mb__after_atomic_inc(); + smp_mb__after_atomic(); /* * Starting the kdump boot. diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index 1d4706114a45..fa934fe080c1 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -412,9 +412,4 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v) #define atomic64_dec_and_test(_v) (atomic64_sub_return(1, _v) == 0) #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) -#define smp_mb__before_atomic_dec() smp_mb() -#define smp_mb__after_atomic_dec() smp_mb() -#define smp_mb__before_atomic_inc() smp_mb() -#define smp_mb__after_atomic_inc() smp_mb() - #endif /* __ARCH_S390_ATOMIC__ */ diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h index 578680f6207a..19ff956b752b 100644 --- a/arch/s390/include/asm/barrier.h +++ b/arch/s390/include/asm/barrier.h @@ -27,8 +27,9 @@ #define smp_rmb() rmb() #define smp_wmb() wmb() #define smp_read_barrier_depends() read_barrier_depends() -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() + +#define smp_mb__before_atomic() smp_mb() +#define smp_mb__after_atomic() smp_mb() #define set_mb(var, value) do { var = value; mb(); } while (0) diff --git a/arch/score/include/asm/bitops.h b/arch/score/include/asm/bitops.h index a304096b1894..c1bf8d6d0fb0 100644 --- a/arch/score/include/asm/bitops.h +++ b/arch/score/include/asm/bitops.h @@ -2,12 +2,7 @@ #define _ASM_SCORE_BITOPS_H #include <asm/byteorder.h> /* swab32 */ - -/* - * clear_bit() doesn't provide any barrier for the compiler. - */ -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() +#include <asm/barrier.h> #include <asm-generic/bitops.h> #include <asm-generic/bitops/__fls.h> diff --git a/arch/sh/include/asm/atomic.h b/arch/sh/include/asm/atomic.h index f4c1c20bcdf6..f57b8a6743b3 100644 --- a/arch/sh/include/asm/atomic.h +++ b/arch/sh/include/asm/atomic.h @@ -10,6 +10,7 @@ #include <linux/compiler.h> #include <linux/types.h> #include <asm/cmpxchg.h> +#include <asm/barrier.h> #define ATOMIC_INIT(i) { (i) } @@ -62,9 +63,4 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) return c; } -#define smp_mb__before_atomic_dec() smp_mb() -#define smp_mb__after_atomic_dec() smp_mb() -#define smp_mb__before_atomic_inc() smp_mb() -#define smp_mb__after_atomic_inc() smp_mb() - #endif /* __ASM_SH_ATOMIC_H */ diff --git a/arch/sh/include/asm/bitops.h b/arch/sh/include/asm/bitops.h index ea8706d94f08..fc8e652cf173 100644 --- a/arch/sh/include/asm/bitops.h +++ b/arch/sh/include/asm/bitops.h @@ -9,6 +9,7 @@ /* For __swab32 */ #include <asm/byteorder.h> +#include <asm/barrier.h> #ifdef CONFIG_GUSA_RB #include <asm/bitops-grb.h> @@ -22,12 +23,6 @@ #include <asm-generic/bitops/non-atomic.h> #endif -/* - * clear_bit() doesn't provide any barrier for the compiler. - */ -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() - #ifdef CONFIG_SUPERH32 static inline unsigned long ffz(unsigned long word) { diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h index 905832aa9e9e..f08fe51b264d 100644 --- a/arch/sparc/include/asm/atomic_32.h +++ b/arch/sparc/include/asm/atomic_32.h @@ -14,6 +14,7 @@ #include <linux/types.h> #include <asm/cmpxchg.h> +#include <asm/barrier.h> #include <asm-generic/atomic64.h> @@ -52,10 +53,4 @@ extern void atomic_set(atomic_t *, int); #define atomic_dec_and_test(v) (atomic_dec_return(v) == 0) #define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) -/* Atomic operations are already serializing */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #endif /* !(__ARCH_SPARC_ATOMIC__) */ diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h index be56a244c9cf..8b2f1bde2889 100644 --- a/arch/sparc/include/asm/atomic_64.h +++ b/arch/sparc/include/asm/atomic_64.h @@ -9,6 +9,7 @@ #include <linux/types.h> #include <asm/cmpxchg.h> +#include <asm/barrier.h> #define ATOMIC_INIT(i) { (i) } #define ATOMIC64_INIT(i) { (i) } @@ -108,10 +109,4 @@ static inline long atomic64_add_unless(atomic64_t *v, long a, long u) extern long atomic64_dec_if_positive(atomic64_t *v); -/* Atomic operations are already serializing */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #endif /* !(__ARCH_SPARC64_ATOMIC__) */ diff --git a/arch/sparc/include/asm/barrier_64.h b/arch/sparc/include/asm/barrier_64.h index b5aad964558e..305dcc3dc721 100644 --- a/arch/sparc/include/asm/barrier_64.h +++ b/arch/sparc/include/asm/barrier_64.h @@ -68,4 +68,7 @@ do { \ ___p1; \ }) +#define smp_mb__before_atomic() barrier() +#define smp_mb__after_atomic() barrier() + #endif /* !(__SPARC64_BARRIER_H) */ diff --git a/arch/sparc/include/asm/bitops_32.h b/arch/sparc/include/asm/bitops_32.h index 25a676653d45..88c9a962502c 100644 --- a/arch/sparc/include/asm/bitops_32.h +++ b/arch/sparc/include/asm/bitops_32.h @@ -90,9 +90,6 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr) #include <asm-generic/bitops/non-atomic.h> -#define smp_mb__before_clear_bit() do { } while(0) -#define smp_mb__after_clear_bit() do { } while(0) - #include <asm-generic/bitops/ffz.h> #include <asm-generic/bitops/__ffs.h> #include <asm-generic/bitops/sched.h> diff --git a/arch/sparc/include/asm/bitops_64.h b/arch/sparc/include/asm/bitops_64.h index 29011cc0e4be..f1a051ca301a 100644 --- a/arch/sparc/include/asm/bitops_64.h +++ b/arch/sparc/include/asm/bitops_64.h @@ -13,6 +13,7 @@ #include <linux/compiler.h> #include <asm/byteorder.h> +#include <asm/barrier.h> extern int test_and_set_bit(unsigned long nr, volatile unsigned long *addr); extern int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr); @@ -23,9 +24,6 @@ extern void change_bit(unsigned long nr, volatile unsigned long *addr); #include <asm-generic/bitops/non-atomic.h> -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() - #include <asm-generic/bitops/fls.h> #include <asm-generic/bitops/__fls.h> #include <asm-generic/bitops/fls64.h> diff --git a/arch/tile/include/asm/atomic_32.h b/arch/tile/include/asm/atomic_32.h index 1ad4a1f7d42b..1b109fad9fff 100644 --- a/arch/tile/include/asm/atomic_32.h +++ b/arch/tile/include/asm/atomic_32.h @@ -169,16 +169,6 @@ static inline void atomic64_set(atomic64_t *v, long long n) #define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL) -/* - * We need to barrier before modifying the word, since the _atomic_xxx() - * routines just tns the lock and then read/modify/write of the word. - * But after the word is updated, the routine issues an "mf" before returning, - * and since it's a function call, we don't even need a compiler barrier. - */ -#define smp_mb__before_atomic_dec() smp_mb() -#define smp_mb__before_atomic_inc() smp_mb() -#define smp_mb__after_atomic_dec() do { } while (0) -#define smp_mb__after_atomic_inc() do { } while (0) #endif /* !__ASSEMBLY__ */ diff --git a/arch/tile/include/asm/atomic_64.h b/arch/tile/include/asm/atomic_64.h index ad220eed05fc..7b11c5fadd42 100644 --- a/arch/tile/include/asm/atomic_64.h +++ b/arch/tile/include/asm/atomic_64.h @@ -105,12 +105,6 @@ static inline long atomic64_add_unless(atomic64_t *v, long a, long u) #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) -/* Atomic dec and inc don't implement barrier, so provide them if needed. */ -#define smp_mb__before_atomic_dec() smp_mb() -#define smp_mb__after_atomic_dec() smp_mb() -#define smp_mb__before_atomic_inc() smp_mb() -#define smp_mb__after_atomic_inc() smp_mb() - /* Define this to indicate that cmpxchg is an efficient operation. */ #define __HAVE_ARCH_CMPXCHG diff --git a/arch/tile/include/asm/barrier.h b/arch/tile/include/asm/barrier.h index b5a05d050a8f..96a42ae79f4d 100644 --- a/arch/tile/include/asm/barrier.h +++ b/arch/tile/include/asm/barrier.h @@ -72,6 +72,20 @@ mb_incoherent(void) #define mb() fast_mb() #define iob() fast_iob() +#ifndef __tilegx__ /* 32 bit */ +/* + * We need to barrier before modifying the word, since the _atomic_xxx() + * routines just tns the lock and then read/modify/write of the word. + * But after the word is updated, the routine issues an "mf" before returning, + * and since it's a function call, we don't even need a compiler barrier. + */ +#define smp_mb__before_atomic() smp_mb() +#define smp_mb__after_atomic() do { } while (0) +#else /* 64 bit */ +#define smp_mb__before_atomic() smp_mb() +#define smp_mb__after_atomic() smp_mb() +#endif + #include <asm-generic/barrier.h> #endif /* !__ASSEMBLY__ */ diff --git a/arch/tile/include/asm/bitops.h b/arch/tile/include/asm/bitops.h index d5a206865036..20caa346ac06 100644 --- a/arch/tile/include/asm/bitops.h +++ b/arch/tile/include/asm/bitops.h @@ -17,6 +17,7 @@ #define _ASM_TILE_BITOPS_H #include <linux/types.h> +#include <asm/barrier.h> #ifndef _LINUX_BITOPS_H #error only <linux/bitops.h> can be included directly diff --git a/arch/tile/include/asm/bitops_32.h b/arch/tile/include/asm/bitops_32.h index 386865ad2f55..bbf7b666f21d 100644 --- a/arch/tile/include/asm/bitops_32.h +++ b/arch/tile/include/asm/bitops_32.h @@ -49,8 +49,8 @@ static inline void set_bit(unsigned nr, volatile unsigned long *addr) * restricted to acting on a single-word quantity. * * clear_bit() may not contain a memory barrier, so if it is used for - * locking purposes, you should call smp_mb__before_clear_bit() and/or - * smp_mb__after_clear_bit() to ensure changes are visible on other cpus. + * locking purposes, you should call smp_mb__before_atomic() and/or + * smp_mb__after_atomic() to ensure changes are visible on other cpus. */ static inline void clear_bit(unsigned nr, volatile unsigned long *addr) { @@ -121,10 +121,6 @@ static inline int test_and_change_bit(unsigned nr, return (_atomic_xor(addr, mask) & mask) != 0; } -/* See discussion at smp_mb__before_atomic_dec() in <asm/atomic_32.h>. */ -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() do {} while (0) - #include <asm-generic/bitops/ext2-atomic.h> #endif /* _ASM_TILE_BITOPS_32_H */ diff --git a/arch/tile/include/asm/bitops_64.h b/arch/tile/include/asm/bitops_64.h index ad34cd056085..bb1a29221fcd 100644 --- a/arch/tile/include/asm/bitops_64.h +++ b/arch/tile/include/asm/bitops_64.h @@ -32,10 +32,6 @@ static inline void clear_bit(unsigned nr, volatile unsigned long *addr) __insn_fetchand((void *)(addr + nr / BITS_PER_LONG), ~mask); } -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() - - static inline void change_bit(unsigned nr, volatile unsigned long *addr) { unsigned long mask = (1UL << (nr % BITS_PER_LONG)); diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h index b17f4f48ecd7..6dd1c7dd0473 100644 --- a/arch/x86/include/asm/atomic.h +++ b/arch/x86/include/asm/atomic.h @@ -7,6 +7,7 @@ #include <asm/alternative.h> #include <asm/cmpxchg.h> #include <asm/rmwcc.h> +#include <asm/barrier.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -243,12 +244,6 @@ static inline void atomic_or_long(unsigned long *v1, unsigned long v2) : : "r" ((unsigned)(mask)), "m" (*(addr)) \ : "memory") -/* Atomic operations are already serializing on x86 */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #ifdef CONFIG_X86_32 # include <asm/atomic64_32.h> #else diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h index 69bbb4845020..5c7198cca5ed 100644 --- a/arch/x86/include/asm/barrier.h +++ b/arch/x86/include/asm/barrier.h @@ -137,6 +137,10 @@ do { \ #endif +/* Atomic operations are already serializing on x86 */ +#define smp_mb__before_atomic() barrier() +#define smp_mb__after_atomic() barrier() + /* * Stop RDTSC speculation. This is needed when you need to use RDTSC * (or get_cycles or vread that possibly accesses the TSC) in a defined diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h index 9fc1af74dc83..afcd35d331de 100644 --- a/arch/x86/include/asm/bitops.h +++ b/arch/x86/include/asm/bitops.h @@ -15,6 +15,7 @@ #include <linux/compiler.h> #include <asm/alternative.h> #include <asm/rmwcc.h> +#include <asm/barrier.h> #if BITS_PER_LONG == 32 # define _BITOPS_LONG_SHIFT 5 @@ -102,7 +103,7 @@ static inline void __set_bit(long nr, volatile unsigned long *addr) * * clear_bit() is atomic and may not be reordered. However, it does * not contain a memory barrier, so if it is used for locking purposes, - * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() + * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic() * in order to ensure changes are visible on other processors. */ static __always_inline void @@ -156,9 +157,6 @@ static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr) __clear_bit(nr, addr); } -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() - /** * __change_bit - Toggle a bit in memory * @nr: the bit to change diff --git a/arch/x86/include/asm/sync_bitops.h b/arch/x86/include/asm/sync_bitops.h index 05af3b31d522..f28a24b51dc7 100644 --- a/arch/x86/include/asm/sync_bitops.h +++ b/arch/x86/include/asm/sync_bitops.h @@ -41,7 +41,7 @@ static inline void sync_set_bit(long nr, volatile unsigned long *addr) * * sync_clear_bit() is atomic and may not be reordered. However, it does * not contain a memory barrier, so if it is used for locking purposes, - * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() + * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic() * in order to ensure changes are visible on other processors. */ static inline void sync_clear_bit(long nr, volatile unsigned long *addr) diff --git a/arch/x86/kernel/apic/hw_nmi.c b/arch/x86/kernel/apic/hw_nmi.c index a698d7165c96..eab67047dec3 100644 --- a/arch/x86/kernel/apic/hw_nmi.c +++ b/arch/x86/kernel/apic/hw_nmi.c @@ -57,7 +57,7 @@ void arch_trigger_all_cpu_backtrace(void) } clear_bit(0, &backtrace_flag); - smp_mb__after_clear_bit(); + smp_mb__after_atomic(); } static int __kprobes diff --git a/arch/xtensa/include/asm/atomic.h b/arch/xtensa/include/asm/atomic.h index e7fb447bce8e..e5103b47a8ce 100644 --- a/arch/xtensa/include/asm/atomic.h +++ b/arch/xtensa/include/asm/atomic.h @@ -19,6 +19,7 @@ #ifdef __KERNEL__ #include <asm/processor.h> #include <asm/cmpxchg.h> +#include <asm/barrier.h> #define ATOMIC_INIT(i) { (i) } @@ -387,12 +388,6 @@ static inline void atomic_set_mask(unsigned int mask, atomic_t *v) #endif } -/* Atomic operations are already serializing */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #endif /* __KERNEL__ */ #endif /* _XTENSA_ATOMIC_H */ diff --git a/arch/xtensa/include/asm/barrier.h b/arch/xtensa/include/asm/barrier.h index 0a24b04d6b21..5b88774c75ab 100644 --- a/arch/xtensa/include/asm/barrier.h +++ b/arch/xtensa/include/asm/barrier.h @@ -13,6 +13,9 @@ #define rmb() barrier() #define wmb() mb() +#define smp_mb__before_atomic() barrier() +#define smp_mb__after_atomic() barrier() + #include <asm-generic/barrier.h> #endif /* _XTENSA_SYSTEM_H */ diff --git a/arch/xtensa/include/asm/bitops.h b/arch/xtensa/include/asm/bitops.h index 7b6873ae84c2..3f44fa2a53e9 100644 --- a/arch/xtensa/include/asm/bitops.h +++ b/arch/xtensa/include/asm/bitops.h @@ -21,9 +21,7 @@ #include <asm/processor.h> #include <asm/byteorder.h> - -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() +#include <asm/barrier.h> #include <asm-generic/bitops/non-atomic.h> |
