Messages in this thread Patch in this message | | | From | Chengming Zhou <> | Subject | [PATCH v3 08/10] sched/fair: refactor detach/attach_entity_cfs_rq using update_load_avg() | Date | Mon, 1 Aug 2022 12:27:43 +0800 |
| |
Since update_load_avg() now support DO_ATTACH and DO_DETACH flags to attach or detach entity sched_avg to the cfs_rq, we can using it to refactor detach/attach_entity_cfs_rq() functions.
Note we can attach a task with last_update_time!=0 from switched_to_fair() since we want to decay sched_avg when running in !fair class.
So this patch move last_update_time condition check to enqueue_entity() for task which migrate CPU or change cgroup.
Signed-off-by: Chengming Zhou <zhouchengming@bytedance.com> --- kernel/sched/fair.c | 68 ++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 41 deletions(-)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ad20a939227d..b8cb826bd755 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4019,21 +4019,10 @@ static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s decayed = update_cfs_rq_load_avg(now, cfs_rq); decayed |= propagate_entity_load_avg(se); - if (!se->avg.last_update_time && (flags & DO_ATTACH)) { - /* - * DO_ATTACH means we're here from enqueue_entity(). - * !last_update_time means we've passed through - * migrate_task_rq_fair() indicating we migrated. - * - * IOW we're enqueueing a task on a new CPU. - */ + if (flags & DO_ATTACH) { attach_entity_load_avg(cfs_rq, se); update_tg_load_avg(cfs_rq); } else if (flags & DO_DETACH) { - /* - * DO_DETACH means we're here from dequeue_entity() - * and we are migrating task out of the CPU. - */ detach_entity_load_avg(cfs_rq, se); update_tg_load_avg(cfs_rq); } else if (decayed) { @@ -4082,44 +4071,31 @@ static void remove_entity_load_avg(struct sched_entity *se) raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags); } -#ifdef CONFIG_FAIR_GROUP_SCHED /* * Propagate the changes of the sched_entity across the tg tree to make it * visible to the root */ -static void propagate_entity_cfs_rq(struct sched_entity *se) +static void propagate_entity_cfs_rq(struct sched_entity *se, int flags) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - - if (cfs_rq_throttled(cfs_rq)) - return; - - if (!throttled_hierarchy(cfs_rq)) - list_add_leaf_cfs_rq(cfs_rq); - - /* Start to propagate at parent */ - se = se->parent; + struct cfs_rq *cfs_rq; for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); - update_load_avg(cfs_rq, se, UPDATE_TG); + update_load_avg(cfs_rq, se, flags); if (cfs_rq_throttled(cfs_rq)) break; if (!throttled_hierarchy(cfs_rq)) list_add_leaf_cfs_rq(cfs_rq); + + flags = UPDATE_TG; } } -#else -static void propagate_entity_cfs_rq(struct sched_entity *se) { } -#endif static void detach_entity_cfs_rq(struct sched_entity *se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - /* * In case the task sched_avg hasn't been attached: * - A forked task which hasn't been woken up by wake_up_new_task(). @@ -4130,21 +4106,18 @@ static void detach_entity_cfs_rq(struct sched_entity *se) return; /* Catch up with the cfs_rq and remove our load when we leave */ - update_load_avg(cfs_rq, se, 0); - detach_entity_load_avg(cfs_rq, se); - update_tg_load_avg(cfs_rq); - propagate_entity_cfs_rq(se); + propagate_entity_cfs_rq(se, DO_DETACH | UPDATE_TG); } static void attach_entity_cfs_rq(struct sched_entity *se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); + int flags = DO_ATTACH | UPDATE_TG; + + if (!sched_feat(ATTACH_AGE_LOAD)) + flags |= SKIP_AGE_LOAD; - /* Synchronize entity with its cfs_rq */ - update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD); - attach_entity_load_avg(cfs_rq, se); - update_tg_load_avg(cfs_rq); - propagate_entity_cfs_rq(se); + /* Synchronize entity with its cfs_rq and attach our load */ + propagate_entity_cfs_rq(se, flags); } static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq) @@ -4479,6 +4452,15 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { bool renorm = !(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED); bool curr = cfs_rq->curr == se; + int action = UPDATE_TG; + + /* + * !last_update_time means we've passed through migrate_task_rq_fair() + * or task_change_group_fair() indicating we migrated cfs_rq. IOW we're + * enqueueing a task on a new CPU or moving task to a new cgroup. + */ + if (!se->avg.last_update_time) + action |= DO_ATTACH; /* * If we're the current task, we must renormalise before calling @@ -4507,7 +4489,7 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) * its group cfs_rq * - Add its new weight to cfs_rq->load.weight */ - update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH); + update_load_avg(cfs_rq, se, action); se_update_runnable(se); update_cfs_group(se); account_entity_enqueue(cfs_rq, se); @@ -4581,6 +4563,10 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { int action = UPDATE_TG; + /* + * When we are migrating task out of the CPU, we should + * detach entity sched_avg from the cfs_rq. + */ if (entity_is_task(se) && task_on_rq_migrating(task_of(se))) action |= DO_DETACH; -- 2.36.1
| |