From 4e14a4d17a8cd66ccab180d32c977091922cfbed Mon Sep 17 00:00:00 2001 From: Anton Blanchard Date: Wed, 10 Feb 2010 00:57:28 +0000 Subject: powerpc: Use lwarx hint in spinlocks Recent versions of the PowerPC architecture added a hint bit to the larx instructions to differentiate between an atomic operation and a lock operation: > 0 Other programs might attempt to modify the word in storage addressed by EA > even if the subsequent Store Conditional succeeds. > > 1 Other programs will not attempt to modify the word in storage addressed by > EA until the program that has acquired the lock performs a subsequent store > releasing the lock. To avoid a binutils dependency this patch create macros for the extended lwarx format and uses it in the spinlock code. To test this change I used a simple test case that acquires and releases a global pthread mutex: pthread_mutex_lock(&mutex); pthread_mutex_unlock(&mutex); On a 32 core POWER6, running 32 test threads we spend almost all our time in the futex spinlock code: 94.37% perf [kernel] [k] ._raw_spin_lock | |--99.95%-- ._raw_spin_lock | | | |--63.29%-- .futex_wake | | | |--36.64%-- .futex_wait_setup Which is a good test for this patch. The results (in lock/unlock operations per second) are: before: 1538203 ops/sec after: 2189219 ops/sec An improvement of 42% A 32 core POWER7 improves even more: before: 1279529 ops/sec after: 2282076 ops/sec An improvement of 78% Signed-off-by: Anton Blanchard Signed-off-by: Benjamin Herrenschmidt --- arch/powerpc/include/asm/spinlock.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'arch/powerpc/include/asm/spinlock.h') diff --git a/arch/powerpc/include/asm/spinlock.h b/arch/powerpc/include/asm/spinlock.h index 764094cff681..1c35b59f6f30 100644 --- a/arch/powerpc/include/asm/spinlock.h +++ b/arch/powerpc/include/asm/spinlock.h @@ -27,6 +27,7 @@ #endif #include #include +#include #define arch_spin_is_locked(x) ((x)->slock != 0) @@ -60,7 +61,7 @@ static inline unsigned long __arch_spin_trylock(arch_spinlock_t *lock) token = LOCK_TOKEN; __asm__ __volatile__( -"1: lwarx %0,0,%2\n\ +"1: " PPC_LWARX(%0,0,%2,1) "\n\ cmpwi 0,%0,0\n\ bne- 2f\n\ stwcx. %1,0,%2\n\ @@ -186,7 +187,7 @@ static inline long __arch_read_trylock(arch_rwlock_t *rw) long tmp; __asm__ __volatile__( -"1: lwarx %0,0,%1\n" +"1: " PPC_LWARX(%0,0,%1,1) "\n" __DO_SIGN_EXTEND " addic. %0,%0,1\n\ ble- 2f\n" @@ -211,7 +212,7 @@ static inline long __arch_write_trylock(arch_rwlock_t *rw) token = WRLOCK_TOKEN; __asm__ __volatile__( -"1: lwarx %0,0,%2\n\ +"1: " PPC_LWARX(%0,0,%2,1) "\n\ cmpwi 0,%0,0\n\ bne- 2f\n" PPC405_ERR77(0,%1) -- cgit v1.2.1 From f10e2e5b4b4c9937de596f96ffe028be3a565598 Mon Sep 17 00:00:00 2001 From: Anton Blanchard Date: Wed, 10 Feb 2010 01:04:06 +0000 Subject: powerpc: Rename LWSYNC_ON_SMP to PPC_RELEASE_BARRIER, ISYNC_ON_SMP to PPC_ACQUIRE_BARRIER For performance reasons we are about to change ISYNC_ON_SMP to sometimes be lwsync. Now that the macro name doesn't make sense, change it and LWSYNC_ON_SMP to better explain what the barriers are doing. Signed-off-by: Anton Blanchard Signed-off-by: Benjamin Herrenschmidt --- arch/powerpc/include/asm/spinlock.h | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) (limited to 'arch/powerpc/include/asm/spinlock.h') diff --git a/arch/powerpc/include/asm/spinlock.h b/arch/powerpc/include/asm/spinlock.h index 1c35b59f6f30..f9611bd69ed2 100644 --- a/arch/powerpc/include/asm/spinlock.h +++ b/arch/powerpc/include/asm/spinlock.h @@ -65,9 +65,10 @@ static inline unsigned long __arch_spin_trylock(arch_spinlock_t *lock) cmpwi 0,%0,0\n\ bne- 2f\n\ stwcx. %1,0,%2\n\ - bne- 1b\n\ - isync\n\ -2:" : "=&r" (tmp) + bne- 1b\n" + PPC_ACQUIRE_BARRIER +"2:" + : "=&r" (tmp) : "r" (token), "r" (&lock->slock) : "cr0", "memory"); @@ -145,7 +146,7 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) { SYNC_IO; __asm__ __volatile__("# arch_spin_unlock\n\t" - LWSYNC_ON_SMP: : :"memory"); + PPC_RELEASE_BARRIER: : :"memory"); lock->slock = 0; } @@ -193,9 +194,9 @@ static inline long __arch_read_trylock(arch_rwlock_t *rw) ble- 2f\n" PPC405_ERR77(0,%1) " stwcx. %0,0,%1\n\ - bne- 1b\n\ - isync\n\ -2:" : "=&r" (tmp) + bne- 1b\n" + PPC_ACQUIRE_BARRIER +"2:" : "=&r" (tmp) : "r" (&rw->lock) : "cr0", "xer", "memory"); @@ -217,9 +218,9 @@ static inline long __arch_write_trylock(arch_rwlock_t *rw) bne- 2f\n" PPC405_ERR77(0,%1) " stwcx. %1,0,%2\n\ - bne- 1b\n\ - isync\n\ -2:" : "=&r" (tmp) + bne- 1b\n" + PPC_ACQUIRE_BARRIER +"2:" : "=&r" (tmp) : "r" (token), "r" (&rw->lock) : "cr0", "memory"); @@ -270,7 +271,7 @@ static inline void arch_read_unlock(arch_rwlock_t *rw) __asm__ __volatile__( "# read_unlock\n\t" - LWSYNC_ON_SMP + PPC_RELEASE_BARRIER "1: lwarx %0,0,%1\n\ addic %0,%0,-1\n" PPC405_ERR77(0,%1) @@ -284,7 +285,7 @@ static inline void arch_read_unlock(arch_rwlock_t *rw) static inline void arch_write_unlock(arch_rwlock_t *rw) { __asm__ __volatile__("# write_unlock\n\t" - LWSYNC_ON_SMP: : :"memory"); + PPC_RELEASE_BARRIER: : :"memory"); rw->lock = 0; } -- cgit v1.2.1