dlan17/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

locking/rwlock: Provide RT variant

Browse Source 
Similar to rw_semaphores, on RT the rwlock substitution is not writer fair,
because it's not feasible to have a writer inherit its priority to
multiple readers. Readers blocked on a writer follow the normal rules of
priority inheritance. Like RT spinlocks, RT rwlocks are state preserving
across the slow lock operations (contended case).

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20210815211303.882793524@linutronix.de
This commit is contained in:
Thomas Gleixner
2021-08-17 17:50:51 +02:00
committed by Ingo Molnar
parent 0f383b6dc9
commit 8282947f67
7 changed files with 323 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/Kconfig.locks
+1 -1
View File
@@ -251,7 +251,7 @@ config ARCH_USE_QUEUED_RWLOCKS
config QUEUED_RWLOCKS
def_bool y if ARCH_USE_QUEUED_RWLOCKS
depends on SMP
depends on SMP && !PREEMPT_RT
config ARCH_HAS_MMIOWB
bool
kernel/locking/spinlock.c
+7
View File
@@ -124,8 +124,11 @@ void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \
* __[spin|read|write]_lock_bh()
*/
BUILD_LOCK_OPS(spin, raw_spinlock);
#ifndef CONFIG_PREEMPT_RT
BUILD_LOCK_OPS(read, rwlock);
BUILD_LOCK_OPS(write, rwlock);
#endif
#endif
@@ -209,6 +212,8 @@ void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock)
EXPORT_SYMBOL(_raw_spin_unlock_bh);
#endif
#ifndef CONFIG_PREEMPT_RT
#ifndef CONFIG_INLINE_READ_TRYLOCK
int __lockfunc _raw_read_trylock(rwlock_t *lock)
{
@@ -353,6 +358,8 @@ void __lockfunc _raw_write_unlock_bh(rwlock_t *lock)
EXPORT_SYMBOL(_raw_write_unlock_bh);
#endif
#endif /* !CONFIG_PREEMPT_RT */
#ifdef CONFIG_DEBUG_LOCK_ALLOC
void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass)
kernel/locking/spinlock_debug.c
+5
View File
@@ -31,6 +31,7 @@ void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
EXPORT_SYMBOL(__raw_spin_lock_init);
#ifndef CONFIG_PREEMPT_RT
void __rwlock_init(rwlock_t *lock, const char *name,
struct lock_class_key *key)
{
@@ -48,6 +49,7 @@ void __rwlock_init(rwlock_t *lock, const char *name,
}
EXPORT_SYMBOL(__rwlock_init);
#endif
static void spin_dump(raw_spinlock_t *lock, const char *msg)
{
@@ -139,6 +141,7 @@ void do_raw_spin_unlock(raw_spinlock_t *lock)
arch_spin_unlock(&lock->raw_lock);
}
#ifndef CONFIG_PREEMPT_RT
static void rwlock_bug(rwlock_t *lock, const char *msg)
{
if (!debug_locks_off())
@@ -228,3 +231,5 @@ void do_raw_write_unlock(rwlock_t *lock)
debug_write_unlock(lock);
arch_write_unlock(&lock->raw_lock);
}
#endif /* !CONFIG_PREEMPT_RT */
kernel/locking/spinlock_rt.c
+131
View File
@@ -127,3 +127,134 @@ void __rt_spin_lock_init(spinlock_t *lock, const char *name,
}
EXPORT_SYMBOL(__rt_spin_lock_init);
#endif
/*
* RT-specific reader/writer locks
*/
#define rwbase_set_and_save_current_state(state) \
current_save_and_set_rtlock_wait_state()
#define rwbase_restore_current_state() \
current_restore_rtlock_saved_state()
static __always_inline int
rwbase_rtmutex_lock_state(struct rt_mutex_base *rtm, unsigned int state)
{
if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
rtlock_slowlock(rtm);
return 0;
}
static __always_inline int
rwbase_rtmutex_slowlock_locked(struct rt_mutex_base *rtm, unsigned int state)
{
rtlock_slowlock_locked(rtm);
return 0;
}
static __always_inline void rwbase_rtmutex_unlock(struct rt_mutex_base *rtm)
{
if (likely(rt_mutex_cmpxchg_acquire(rtm, current, NULL)))
return;
rt_mutex_slowunlock(rtm);
}
static __always_inline int rwbase_rtmutex_trylock(struct rt_mutex_base *rtm)
{
if (likely(rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
return 1;
return rt_mutex_slowtrylock(rtm);
}
#define rwbase_signal_pending_state(state, current) (0)
#define rwbase_schedule() \
schedule_rtlock()
#include "rwbase_rt.c"
/*
* The common functions which get wrapped into the rwlock API.
*/
int __sched rt_read_trylock(rwlock_t *rwlock)
{
int ret;
ret = rwbase_read_trylock(&rwlock->rwbase);
if (ret) {
rwlock_acquire_read(&rwlock->dep_map, 0, 1, _RET_IP_);
rcu_read_lock();
migrate_disable();
}
return ret;
}
EXPORT_SYMBOL(rt_read_trylock);
int __sched rt_write_trylock(rwlock_t *rwlock)
{
int ret;
ret = rwbase_write_trylock(&rwlock->rwbase);
if (ret) {
rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_);
rcu_read_lock();
migrate_disable();
}
return ret;
}
EXPORT_SYMBOL(rt_write_trylock);
void __sched rt_read_lock(rwlock_t *rwlock)
{
___might_sleep(__FILE__, __LINE__, 0);
rwlock_acquire_read(&rwlock->dep_map, 0, 0, _RET_IP_);
rwbase_read_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
rcu_read_lock();
migrate_disable();
}
EXPORT_SYMBOL(rt_read_lock);
void __sched rt_write_lock(rwlock_t *rwlock)
{
___might_sleep(__FILE__, __LINE__, 0);
rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_);
rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
rcu_read_lock();
migrate_disable();
}
EXPORT_SYMBOL(rt_write_lock);
void __sched rt_read_unlock(rwlock_t *rwlock)
{
rwlock_release(&rwlock->dep_map, _RET_IP_);
migrate_enable();
rcu_read_unlock();
rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
}
EXPORT_SYMBOL(rt_read_unlock);
void __sched rt_write_unlock(rwlock_t *rwlock)
{
rwlock_release(&rwlock->dep_map, _RET_IP_);
rcu_read_unlock();
migrate_enable();
rwbase_write_unlock(&rwlock->rwbase);
}
EXPORT_SYMBOL(rt_write_unlock);
int __sched rt_rwlock_is_contended(rwlock_t *rwlock)
{
return rw_base_is_contended(&rwlock->rwbase);
}
EXPORT_SYMBOL(rt_rwlock_is_contended);
#ifdef CONFIG_DEBUG_LOCK_ALLOC
void __rt_rwlock_init(rwlock_t *rwlock, const char *name,
struct lock_class_key *key)
{
debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock));
lockdep_init_map_wait(&rwlock->dep_map, name, key, 0, LD_WAIT_CONFIG);
}
EXPORT_SYMBOL(__rt_rwlock_init);
#endif