diff options
Diffstat (limited to 'arch/mips/include/asm/cmpxchg.h')
| -rw-r--r-- | arch/mips/include/asm/cmpxchg.h | 76 |
1 files changed, 46 insertions, 30 deletions
diff --git a/arch/mips/include/asm/cmpxchg.h b/arch/mips/include/asm/cmpxchg.h index f345a873742d..5b0b3a6777ea 100644 --- a/arch/mips/include/asm/cmpxchg.h +++ b/arch/mips/include/asm/cmpxchg.h @@ -11,20 +11,11 @@ #include <linux/bug.h> #include <linux/irqflags.h> #include <asm/compiler.h> +#include <asm/llsc.h> +#include <asm/sync.h> #include <asm/war.h> /* - * Using a branch-likely instruction to check the result of an sc instruction - * works around a bug present in R10000 CPUs prior to revision 3.0 that could - * cause ll-sc sequences to execute non-atomically. - */ -#if R10000_LLSC_WAR -# define __scbeqz "beqzl" -#else -# define __scbeqz "beqz" -#endif - -/* * These functions doesn't exist, so if they are called you'll either: * * - Get an error at compile-time due to __compiletime_error, if supported by @@ -51,16 +42,17 @@ extern unsigned long __xchg_called_with_bad_pointer(void) " .set noat \n" \ " .set push \n" \ " .set " MIPS_ISA_ARCH_LEVEL " \n" \ + " " __SYNC(full, loongson3_war) " \n" \ "1: " ld " %0, %2 # __xchg_asm \n" \ " .set pop \n" \ " move $1, %z3 \n" \ " .set " MIPS_ISA_ARCH_LEVEL " \n" \ " " st " $1, %1 \n" \ - "\t" __scbeqz " $1, 1b \n" \ + "\t" __SC_BEQZ "$1, 1b \n" \ " .set pop \n" \ : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \ : GCC_OFF_SMALL_ASM() (*m), "Jr" (val) \ - : "memory"); \ + : __LLSC_CLOBBER); \ } else { \ unsigned long __flags; \ \ @@ -76,8 +68,8 @@ extern unsigned long __xchg_called_with_bad_pointer(void) extern unsigned long __xchg_small(volatile void *ptr, unsigned long val, unsigned int size); -static inline unsigned long __xchg(volatile void *ptr, unsigned long x, - int size) +static __always_inline +unsigned long __xchg(volatile void *ptr, unsigned long x, int size) { switch (size) { case 1: @@ -102,7 +94,13 @@ static inline unsigned long __xchg(volatile void *ptr, unsigned long x, ({ \ __typeof__(*(ptr)) __res; \ \ - smp_mb__before_llsc(); \ + /* \ + * In the Loongson3 workaround case __xchg_asm() already \ + * contains a completion barrier prior to the LL, so we don't \ + * need to emit an extra one here. \ + */ \ + if (!__SYNC_loongson3_war) \ + smp_mb__before_llsc(); \ \ __res = (__typeof__(*(ptr))) \ __xchg((ptr), (unsigned long)(x), sizeof(*(ptr))); \ @@ -122,18 +120,19 @@ static inline unsigned long __xchg(volatile void *ptr, unsigned long x, " .set noat \n" \ " .set push \n" \ " .set "MIPS_ISA_ARCH_LEVEL" \n" \ + " " __SYNC(full, loongson3_war) " \n" \ "1: " ld " %0, %2 # __cmpxchg_asm \n" \ " bne %0, %z3, 2f \n" \ " .set pop \n" \ " move $1, %z4 \n" \ " .set "MIPS_ISA_ARCH_LEVEL" \n" \ " " st " $1, %1 \n" \ - "\t" __scbeqz " $1, 1b \n" \ + "\t" __SC_BEQZ "$1, 1b \n" \ " .set pop \n" \ - "2: \n" \ + "2: " __SYNC(full, loongson3_war) " \n" \ : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \ - : GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \ - : "memory"); \ + : GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \ + : __LLSC_CLOBBER); \ } else { \ unsigned long __flags; \ \ @@ -150,8 +149,9 @@ static inline unsigned long __xchg(volatile void *ptr, unsigned long x, extern unsigned long __cmpxchg_small(volatile void *ptr, unsigned long old, unsigned long new, unsigned int size); -static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, - unsigned long new, unsigned int size) +static __always_inline +unsigned long __cmpxchg(volatile void *ptr, unsigned long old, + unsigned long new, unsigned int size) { switch (size) { case 1: @@ -186,9 +186,23 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, ({ \ __typeof__(*(ptr)) __res; \ \ - smp_mb__before_llsc(); \ + /* \ + * In the Loongson3 workaround case __cmpxchg_asm() already \ + * contains a completion barrier prior to the LL, so we don't \ + * need to emit an extra one here. \ + */ \ + if (!__SYNC_loongson3_war) \ + smp_mb__before_llsc(); \ + \ __res = cmpxchg_local((ptr), (old), (new)); \ - smp_llsc_mb(); \ + \ + /* \ + * In the Loongson3 workaround case __cmpxchg_asm() already \ + * contains a completion barrier after the SC, so we don't \ + * need to emit an extra one here. \ + */ \ + if (!__SYNC_loongson3_war) \ + smp_llsc_mb(); \ \ __res; \ }) @@ -233,6 +247,7 @@ static inline unsigned long __cmpxchg64(volatile void *ptr, " .set push \n" " .set " MIPS_ISA_ARCH_LEVEL " \n" /* Load 64 bits from ptr */ + " " __SYNC(full, loongson3_war) " \n" "1: lld %L0, %3 # __cmpxchg64 \n" /* * Split the 64 bit value we loaded into the 2 registers that hold the @@ -264,9 +279,9 @@ static inline unsigned long __cmpxchg64(volatile void *ptr, /* Attempt to store new at ptr */ " scd %L1, %2 \n" /* If we failed, loop! */ - "\t" __scbeqz " %L1, 1b \n" + "\t" __SC_BEQZ "%L1, 1b \n" " .set pop \n" - "2: \n" + "2: " __SYNC(full, loongson3_war) " \n" : "=&r"(ret), "=&r"(tmp), "=" GCC_OFF_SMALL_ASM() (*(unsigned long long *)ptr) @@ -290,10 +305,13 @@ static inline unsigned long __cmpxchg64(volatile void *ptr, * will cause a build error unless cpu_has_64bits is a \ * compile-time constant 1. \ */ \ - if (cpu_has_64bits && kernel_uses_llsc) \ + if (cpu_has_64bits && kernel_uses_llsc) { \ + smp_mb__before_llsc(); \ __res = __cmpxchg64((ptr), __old, __new); \ - else \ + smp_llsc_mb(); \ + } else { \ __res = __cmpxchg64_unsupported(); \ + } \ \ __res; \ }) @@ -303,6 +321,4 @@ static inline unsigned long __cmpxchg64(volatile void *ptr, # endif /* !CONFIG_SMP */ #endif /* !CONFIG_64BIT */ -#undef __scbeqz - #endif /* __ASM_CMPXCHG_H */ |
