Messages in this thread Patch in this message | | | Date | Sat, 19 Aug 2023 06:08:48 +0000 | Subject | [PATCH v5 14/19] sched: Add a very simple proxy() function | From | John Stultz <> |
| |
This adds a very simple proxy() function so if we select a blocked task to run, we will deactivate it and pick again. The exception being if it has become unblocked after proxy() was called.
Greatly simplified from patch by: Peter Zijlstra (Intel) <peterz@infradead.org> Juri Lelli <juri.lelli@redhat.com> Valentin Schneider <valentin.schneider@arm.com> Connor O'Brien <connoro@google.com>
Cc: Joel Fernandes <joelaf@google.com> Cc: Qais Yousef <qyousef@google.com> Cc: Ingo Molnar <mingo@redhat.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Juri Lelli <juri.lelli@redhat.com> Cc: Vincent Guittot <vincent.guittot@linaro.org> Cc: Dietmar Eggemann <dietmar.eggemann@arm.com> Cc: Valentin Schneider <vschneid@redhat.com> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Ben Segall <bsegall@google.com> Cc: Zimuzo Ezeozue <zezeozue@google.com> Cc: Youssef Esmat <youssefesmat@google.com> Cc: Mel Gorman <mgorman@suse.de> Cc: Daniel Bristot de Oliveira <bristot@redhat.com> Cc: Will Deacon <will@kernel.org> Cc: Waiman Long <longman@redhat.com> Cc: Boqun Feng <boqun.feng@gmail.com> Cc: "Paul E . McKenney" <paulmck@kernel.org> Cc: kernel-team@android.com [jstultz: Split out from larger proxy patch and simplified for review and testing.] Signed-off-by: John Stultz <jstultz@google.com> --- v5: * Split out from larger proxy patch --- kernel/sched/core.c | 89 +++++++++++++++++++++++++++++++++++++++++++-- kernel/sched/rt.c | 19 +++++++++- 2 files changed, 102 insertions(+), 6 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 76a42f21dda7..72d0803c7d47 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6562,11 +6562,12 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) # define SM_MASK_PREEMPT SM_PREEMPT #endif -bool try_to_deactivate_task(struct rq *rq, struct task_struct *p, unsigned long state) +bool try_to_deactivate_task(struct rq *rq, struct task_struct *p, + unsigned long state, bool deactivate_cond) { if (signal_pending_state(state, p)) { WRITE_ONCE(p->__state, TASK_RUNNING); - } else { + } else if (deactivate_cond) { p->sched_contributes_to_load = (state & TASK_UNINTERRUPTIBLE) && !(state & TASK_NOLOAD) && @@ -6597,6 +6598,74 @@ bool try_to_deactivate_task(struct rq *rq, struct task_struct *p, unsigned long return false; } +#ifdef CONFIG_PROXY_EXEC +/* + * Initial simple proxy that just returns the task if its waking + * or deactivates the blocked task so we can pick something that + * isn't blocked. + */ +static struct task_struct * +proxy(struct rq *rq, struct task_struct *next, struct rq_flags *rf) +{ + struct task_struct *p = next; + struct mutex *mutex; + unsigned long state; + + mutex = p->blocked_on; + /* Something changed in the chain, pick_again */ + if (!mutex) + return NULL; + /* + * By taking mutex->wait_lock we hold off concurrent mutex_unlock() + * and ensure @owner sticks around. + */ + raw_spin_lock(&mutex->wait_lock); + raw_spin_lock(&p->blocked_lock); + + /* Check again that p is blocked with blocked_lock held */ + if (!task_is_blocked(p) || mutex != p->blocked_on) { + /* + * Something changed in the blocked_on chain and + * we don't know if only at this level. So, let's + * just bail out completely and let __schedule + * figure things out (pick_again loop). + */ + raw_spin_unlock(&p->blocked_lock); + raw_spin_unlock(&mutex->wait_lock); + return NULL; + } + + state = READ_ONCE(p->__state); + /* Don't deactivate if the state has been changed to TASK_RUNNING */ + if (!state) { + raw_spin_unlock(&p->blocked_lock); + raw_spin_unlock(&mutex->wait_lock); + return p; + } + + try_to_deactivate_task(rq, next, state, true); + + /* + * If next is the selected task, then remove lingering + * references to it from rq and sched_class structs after + * dequeueing. + */ + put_prev_task(rq, next); + rq_set_selected(rq, rq->idle); + resched_curr(rq); + raw_spin_unlock(&p->blocked_lock); + raw_spin_unlock(&mutex->wait_lock); + return NULL; +} +#else /* PROXY_EXEC */ +static struct task_struct * +proxy(struct rq *rq, struct task_struct *next, struct rq_flags *rf) +{ + BUG(); // This should never be called in the !PROXY case + return next; +} +#endif /* PROXY_EXEC */ + /* * __schedule() is the main scheduler function. * @@ -6687,12 +6756,24 @@ static void __sched notrace __schedule(unsigned int sched_mode) */ prev_state = READ_ONCE(prev->__state); if (!(sched_mode & SM_MASK_PREEMPT) && prev_state) { - try_to_deactivate_task(rq, prev, prev_state); + try_to_deactivate_task(rq, prev, prev_state, + !task_is_blocked(prev)); switch_count = &prev->nvcsw; } - next = pick_next_task(rq, prev, &rf); +pick_again: + next = pick_next_task(rq, rq_selected(rq), &rf); rq_set_selected(rq, next); + if (unlikely(task_is_blocked(next))) { + next = proxy(rq, next, &rf); + if (!next) { + rq_unpin_lock(rq, &rf); + __balance_callbacks(rq); + rq_repin_lock(rq, &rf); + goto pick_again; + } + } + clear_tsk_need_resched(prev); clear_preempt_need_resched(); #ifdef CONFIG_SCHED_DEBUG diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 6d9036547c1d..cfbf3925e595 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1537,8 +1537,19 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags) enqueue_rt_entity(rt_se, flags); - if (!task_current(rq, p) && p->nr_cpus_allowed > 1) - enqueue_pushable_task(rq, p); + /* + * Current can't be pushed away. Selected is tied to current, + * so don't push it either. + */ + if (task_current(rq, p) || task_current_selected(rq, p)) + return; + /* + * Pinned tasks can't be pushed. + */ + if (p->nr_cpus_allowed == 1) + return; + + enqueue_pushable_task(rq, p); } static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags) @@ -1825,6 +1836,10 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p) update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1); + /* Avoid marking selected as pushable */ + if (task_current_selected(rq, p)) + return; + /* * The previous task needs to be made eligible for pushing * if it is still active -- 2.42.0.rc1.204.g551eb34607-goog
| |