Messages in this thread Patch in this message | | | From | Waiman Long <> | Subject | [PATCH v2 6/8] locking/rwsem: Unify wait loop | Date | Mon, 27 Mar 2023 16:24:11 -0400 |
| |
From: Peter Zijlstra <peterz@infradead.org>
Now that the reader and writer wait loops are identical, share the code.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- kernel/locking/rwsem.c | 121 +++++++++++++++++------------------------ 1 file changed, 51 insertions(+), 70 deletions(-)
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index 0bc262dc77fd..ee8861effcc2 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -650,13 +650,11 @@ static void rwsem_mark_wake(struct rw_semaphore *sem, * optionally wake up waiters before it returns. */ static inline void -rwsem_del_wake_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter, - struct wake_q_head *wake_q) +rwsem_del_wake_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter) __releases(&sem->wait_lock) { bool first = rwsem_first_waiter(sem) == waiter; - - wake_q_init(wake_q); + DEFINE_WAKE_Q(wake_q); /* * If the wait_list isn't empty and the waiter to be deleted is @@ -664,10 +662,10 @@ rwsem_del_wake_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter, * be eligible to acquire or spin on the lock. */ if (rwsem_del_waiter(sem, waiter) && first) - rwsem_mark_wake(sem, RWSEM_WAKE_ANY, wake_q); + rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); raw_spin_unlock_irq(&sem->wait_lock); - if (!wake_q_empty(wake_q)) - wake_up_q(wake_q); + if (!wake_q_empty(&wake_q)) + wake_up_q(&wake_q); } /* @@ -997,6 +995,50 @@ static inline void rwsem_cond_wake_waiter(struct rw_semaphore *sem, long count, rwsem_mark_wake(sem, wake_type, wake_q); } +#define lockevent_rw_inc(rd, evr, evw) do { \ + lockevent_cond_inc(evr, (rd)); \ + lockevent_cond_inc(evw, !(rd)); \ +} while (0) + +static __always_inline struct rw_semaphore * +rwsem_waiter_wait(struct rw_semaphore *sem, struct rwsem_waiter *waiter, + int state, bool reader) +{ + trace_contention_begin(sem, reader ? LCB_F_READ : LCB_F_WRITE); + + /* wait to be given the lock */ + for (;;) { + set_current_state(state); + if (!smp_load_acquire(&waiter->task)) { + /* Matches rwsem_waiter_wake()'s smp_store_release(). */ + break; + } + if (signal_pending_state(state, current)) { + raw_spin_lock_irq(&sem->wait_lock); + if (waiter->task) + goto out_nolock; + raw_spin_unlock_irq(&sem->wait_lock); + /* Ordered by sem->wait_lock against rwsem_mark_wake(). */ + break; + } + schedule_preempt_disabled(); + lockevent_rw_inc(reader, rwsem_sleep_reader, rwsem_sleep_writer); + } + + __set_current_state(TASK_RUNNING); + + lockevent_rw_inc(reader, rwsem_rlock, rwsem_wlock); + trace_contention_end(sem, 0); + return sem; + +out_nolock: + rwsem_del_wake_waiter(sem, waiter); + __set_current_state(TASK_RUNNING); + lockevent_rw_inc(reader, rwsem_rlock_fail, rwsem_wlock_fail); + trace_contention_end(sem, -EINTR); + return ERR_PTR(-EINTR); +} + /* * Wait for the read lock to be granted */ @@ -1074,38 +1116,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat if (!wake_q_empty(&wake_q)) wake_up_q(&wake_q); - trace_contention_begin(sem, LCB_F_READ); - - /* wait to be given the lock */ - for (;;) { - set_current_state(state); - if (!smp_load_acquire(&waiter.task)) { - /* Matches rwsem_waiter_wake()'s smp_store_release(). */ - break; - } - if (signal_pending_state(state, current)) { - raw_spin_lock_irq(&sem->wait_lock); - if (waiter.task) - goto out_nolock; - raw_spin_unlock_irq(&sem->wait_lock); - /* Ordered by sem->wait_lock against rwsem_mark_wake(). */ - break; - } - schedule_preempt_disabled(); - lockevent_inc(rwsem_sleep_reader); - } - - __set_current_state(TASK_RUNNING); - lockevent_inc(rwsem_rlock); - trace_contention_end(sem, 0); - return sem; - -out_nolock: - rwsem_del_wake_waiter(sem, &waiter, &wake_q); - __set_current_state(TASK_RUNNING); - lockevent_inc(rwsem_rlock_fail); - trace_contention_end(sem, -EINTR); - return ERR_PTR(-EINTR); + return rwsem_waiter_wait(sem, &waiter, state, true); } /* @@ -1155,37 +1166,7 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) } raw_spin_unlock_irq(&sem->wait_lock); - /* wait until we successfully acquire the lock */ - trace_contention_begin(sem, LCB_F_WRITE); - - for (;;) { - set_current_state(state); - if (!smp_load_acquire(&waiter.task)) { - /* Matches rwsem_waiter_wake()'s smp_store_release(). */ - break; - } - if (signal_pending_state(state, current)) { - raw_spin_lock_irq(&sem->wait_lock); - if (waiter.task) - goto out_nolock; - raw_spin_unlock_irq(&sem->wait_lock); - /* Ordered by sem->wait_lock against rwsem_mark_wake(). */ - break; - } - schedule_preempt_disabled(); - lockevent_inc(rwsem_sleep_writer); - } - __set_current_state(TASK_RUNNING); - lockevent_inc(rwsem_wlock); - trace_contention_end(sem, 0); - return sem; - -out_nolock: - rwsem_del_wake_waiter(sem, &waiter, &wake_q); - __set_current_state(TASK_RUNNING); - lockevent_inc(rwsem_wlock_fail); - trace_contention_end(sem, -EINTR); - return ERR_PTR(-EINTR); + return rwsem_waiter_wait(sem, &waiter, state, false); } /* -- 2.31.1
| |