Messages in this thread Patch in this message | | | Date | Mon, 3 Oct 2022 21:44:51 +0000 | Subject | [RFC PATCH 01/11] locking/ww_mutex: Remove wakeups from under mutex::wait_lock | From | "Connor O'Brien" <> |
| |
From: Peter Zijlstra <peterz@infradead.org>
In preparation to nest mutex::wait_lock under rq::lock we need to remove wakeups from under it.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Connor O'Brien <connoro@google.com> --- include/linux/ww_mutex.h | 3 +++ kernel/locking/mutex.c | 8 ++++++++ kernel/locking/ww_mutex.h | 9 +++++++-- kernel/sched/core.c | 7 +++++++ 4 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/include/linux/ww_mutex.h b/include/linux/ww_mutex.h index bb763085479a..9335b2202017 100644 --- a/include/linux/ww_mutex.h +++ b/include/linux/ww_mutex.h @@ -19,6 +19,7 @@ #include <linux/mutex.h> #include <linux/rtmutex.h> +#include <linux/sched/wake_q.h> #if defined(CONFIG_DEBUG_MUTEXES) || \ (defined(CONFIG_PREEMPT_RT) && defined(CONFIG_DEBUG_RT_MUTEXES)) @@ -58,6 +59,7 @@ struct ww_acquire_ctx { unsigned int acquired; unsigned short wounded; unsigned short is_wait_die; + struct wake_q_head wake_q; #ifdef DEBUG_WW_MUTEXES unsigned int done_acquire; struct ww_class *ww_class; @@ -137,6 +139,7 @@ static inline void ww_acquire_init(struct ww_acquire_ctx *ctx, ctx->acquired = 0; ctx->wounded = false; ctx->is_wait_die = ww_class->is_wait_die; + wake_q_init(&ctx->wake_q); #ifdef DEBUG_WW_MUTEXES ctx->ww_class = ww_class; ctx->done_acquire = 0; diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index d973fe6041bf..1582756914df 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -676,6 +676,8 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas } raw_spin_unlock(&lock->wait_lock); + if (ww_ctx) + ww_ctx_wake(ww_ctx); schedule_preempt_disabled(); first = __mutex_waiter_is_first(lock, &waiter); @@ -725,6 +727,8 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas ww_mutex_lock_acquired(ww, ww_ctx); raw_spin_unlock(&lock->wait_lock); + if (ww_ctx) + ww_ctx_wake(ww_ctx); preempt_enable(); return 0; @@ -736,6 +740,8 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas raw_spin_unlock(&lock->wait_lock); debug_mutex_free_waiter(&waiter); mutex_release(&lock->dep_map, ip); + if (ww_ctx) + ww_ctx_wake(ww_ctx); preempt_enable(); return ret; } @@ -946,9 +952,11 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne if (owner & MUTEX_FLAG_HANDOFF) __mutex_handoff(lock, next); + preempt_disable(); raw_spin_unlock(&lock->wait_lock); wake_up_q(&wake_q); + preempt_enable(); } #ifndef CONFIG_DEBUG_LOCK_ALLOC diff --git a/kernel/locking/ww_mutex.h b/kernel/locking/ww_mutex.h index 56f139201f24..dfc174cd96c6 100644 --- a/kernel/locking/ww_mutex.h +++ b/kernel/locking/ww_mutex.h @@ -161,6 +161,11 @@ static inline void lockdep_assert_wait_lock_held(struct rt_mutex *lock) #endif /* WW_RT */ +void ww_ctx_wake(struct ww_acquire_ctx *ww_ctx) +{ + wake_up_q(&ww_ctx->wake_q); +} + /* * Wait-Die: * The newer transactions are killed when: @@ -284,7 +289,7 @@ __ww_mutex_die(struct MUTEX *lock, struct MUTEX_WAITER *waiter, #ifndef WW_RT debug_mutex_wake_waiter(lock, waiter); #endif - wake_up_process(waiter->task); + wake_q_add(&ww_ctx->wake_q, waiter->task); } return true; @@ -331,7 +336,7 @@ static bool __ww_mutex_wound(struct MUTEX *lock, * wakeup pending to re-read the wounded state. */ if (owner != current) - wake_up_process(owner); + wake_q_add(&ww_ctx->wake_q, owner); return true; } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index ee28253c9ac0..617e737392be 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1013,6 +1013,13 @@ void wake_up_q(struct wake_q_head *head) wake_up_process(task); put_task_struct(task); } + /* + * XXX connoro: seems this is needed now that ww_ctx_wake() passes in a + * wake_q_head that is embedded in struct ww_acquire_ctx rather than + * declared locally. + */ + head->first = node; + head->lastp = &head->first; } /* -- 2.38.0.rc1.362.ged0d419d3c-goog
| |