diff --git a/Makefile b/Makefile index 04a2186a427696..c458fa4a33dfd2 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ VERSION = 4 PATCHLEVEL = 4 SUBLEVEL = 43 -EXTRAVERSION = +EXTRAVERSION = -HCBS-Demotion NAME = Blurry Fish Butt # *DOCUMENTATION* diff --git a/include/linux/sched.h b/include/linux/sched.h index 06d46907931235..9b0fcff296c7f1 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1362,11 +1362,11 @@ struct sched_rt_entity { struct sched_rt_entity *back; #ifdef CONFIG_RT_GROUP_SCHED - struct sched_rt_entity *parent; + struct sched_dl_entity *parent; /* rq on which this entity is (to be) queued: */ struct rt_rq *rt_rq; - /* rq "owned" by this entity/group: */ - struct rt_rq *my_q; + struct list_head cfs_throttled_task; + struct rt_rq *cfs_throttle_rt_rq; #endif }; @@ -1413,6 +1413,17 @@ struct sched_dl_entity { * own bandwidth to be enforced, thus we need one timer per task. */ struct hrtimer dl_timer; + +/* + * An instance of a sched_dl_entity may represent a group of tasks, therefore + * it requires: + * - dl_rq: the rq on which this entity is queued; + * - rt_rq: the rq owned by this entity; + */ +#ifdef CONFIG_RT_GROUP_SCHED + struct dl_rq *dl_rq; + struct rt_rq *my_q; +#endif }; union rcu_special { diff --git a/include/linux/sched/rt.h b/include/linux/sched/rt.h index a30b172df6e1a7..70d1bcb2fb3d58 100644 --- a/include/linux/sched/rt.h +++ b/include/linux/sched/rt.h @@ -15,6 +15,11 @@ static inline int rt_task(struct task_struct *p) return rt_prio(p->prio); } +static inline int rt_throttled(struct task_struct *p) +{ + return !list_empty(&p->rt.cfs_throttled_task); +} + #ifdef CONFIG_RT_MUTEXES extern int rt_mutex_getprio(struct task_struct *p); extern void rt_mutex_setprio(struct task_struct *p, int prio); diff --git a/kernel/sched/auto_group.c b/kernel/sched/auto_group.c index 750ed601ddf78e..9e49b24898827f 100644 --- a/kernel/sched/auto_group.c +++ b/kernel/sched/auto_group.c @@ -30,7 +30,7 @@ static inline void autogroup_destroy(struct kref *kref) #ifdef CONFIG_RT_GROUP_SCHED /* We've redirected RT tasks to the root task group... */ - ag->tg->rt_se = NULL; + ag->tg->dl_se = NULL; ag->tg->rt_rq = NULL; #endif sched_offline_group(ag->tg); @@ -88,7 +88,7 @@ static inline struct autogroup *autogroup_create(void) * the policy change to proceed. */ free_rt_sched_group(tg); - tg->rt_se = root_task_group.rt_se; + tg->dl_se = root_task_group.dl_se; tg->rt_rq = root_task_group.rt_rq; #endif tg->autogroup = ag; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index fb9e27ea455e91..9942bdc8cb2bd6 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1107,6 +1107,9 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) { const struct sched_class *class; + if (is_dl_group(rt_rq_of_se(&p->rt)) && task_has_rt_policy(p)) + resched_curr(rq); + if (p->sched_class == rq->curr->sched_class) { rq->curr->sched_class->check_preempt_curr(rq, p, flags); } else { @@ -2228,6 +2231,9 @@ void __dl_clear_params(struct task_struct *p) dl_se->dl_throttled = 0; dl_se->dl_yielded = 0; +#ifdef CONFIG_RT_GROUP_SCHED + dl_se->my_q = NULL; +#endif } /* @@ -2262,6 +2268,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) p->rt.time_slice = sched_rr_timeslice; p->rt.on_rq = 0; p->rt.on_list = 0; +#ifdef CONFIG_RT_GROUP_SCHED + INIT_LIST_HEAD(&p->rt.cfs_throttled_task); +#endif #ifdef CONFIG_PREEMPT_NOTIFIERS INIT_HLIST_HEAD(&p->preempt_notifiers); @@ -3623,6 +3632,98 @@ int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags, } EXPORT_SYMBOL(default_wake_function); +void __setprio_other(struct rq *rq, struct task_struct *p) +{ + int oldprio, queued, running; + const struct sched_class *prev_class; + + lockdep_assert_held(&rq->lock); + + oldprio = p->prio; + prev_class = p->sched_class; + queued = task_on_rq_queued(p); + running = task_current(rq, p); + BUG_ON(!rt_throttled(p)); + + if (queued) + dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE); +/* + if (running) + put_prev_task(rq, p); +*/ + p->rt.cfs_throttle_rt_rq = rt_rq_of_se(&p->rt); + p->sched_class = &fair_sched_class; + p->prio = DEFAULT_PRIO; + +#ifdef CONFIG_FAIR_GROUP_SCHED + /* + * As in attach_task_cfs_rq, since the real-depth could have been + * changed (only FAIR class maintain depth value), reset depth + * properly. + */ + p->se.depth = p->se.parent ? p->se.parent->depth + 1 : 0; +#endif + + if (running) + p->sched_class->set_curr_task(rq); + if (queued) + enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_MOVE); + + check_class_changed(rq, p, prev_class, oldprio); +} + +void __setprio_fifo(struct rq *rq, struct task_struct *p) +{ + int oldprio, queued, running, cpu; + const struct sched_class *prev_class; + unsigned int count = 0; + + lockdep_assert_held(&rq->lock); + + /* + * p might have migrated while hanging out in OTHER. We will need its + * current rq lock for dequeue_task/put_prev_task. + */ +again: + cpu = task_cpu(p); + if (cpu != cpu_of(rq)) { + double_lock_balance(rq, cpu_rq(cpu)); + if (cpu != task_cpu(p)) { + double_unlock_balance(rq, cpu_rq(cpu)); + count++; + BUG_ON(count > 10); + goto again; + } + } + + BUG_ON(p->sched_class == &rt_sched_class); + + oldprio = p->prio; + prev_class = p->sched_class; + queued = task_on_rq_queued(p); + running = task_current(cpu_rq(cpu), p); + BUG_ON(rt_throttled(p)); + + if (queued) + dequeue_task(cpu_rq(cpu), p, DEQUEUE_SAVE | DEQUEUE_MOVE); + if (running) + put_prev_task(cpu_rq(cpu), p); + + p->rt.cfs_throttle_rt_rq = NULL; + p->sched_class = &rt_sched_class; + p->prio = (MAX_RT_PRIO - 1) - p->rt_priority; + + if (running) + p->sched_class->set_curr_task(cpu_rq(cpu)); + if (queued) + enqueue_task(cpu_rq(cpu), p, ENQUEUE_REPLENISH | ENQUEUE_MOVE | ENQUEUE_RESTORE); + + check_class_changed(cpu_rq(cpu), p, prev_class, oldprio); +out: + if (cpu != cpu_of(rq)) + double_unlock_balance(rq, cpu_rq(cpu)); +} + #ifdef CONFIG_RT_MUTEXES /* @@ -3664,6 +3765,9 @@ void rt_mutex_setprio(struct task_struct *p, int prio) WARN_ON(p->pi_blocked_on); goto out_unlock; } + + if (rt_throttled(p)) + goto out_unlock; trace_sched_pi_setprio(p, prio); oldprio = p->prio; @@ -3971,7 +4075,7 @@ static void __setscheduler(struct rq *rq, struct task_struct *p, if (dl_prio(p->prio)) p->sched_class = &dl_sched_class; - else if (rt_prio(p->prio)) + else if (rt_prio(p->prio) && !rt_throttled(p)) p->sched_class = &rt_sched_class; else p->sched_class = &fair_sched_class; @@ -4201,8 +4305,8 @@ static int __sched_setscheduler(struct task_struct *p, * Do not allow realtime tasks into groups that have no runtime * assigned. */ - if (rt_bandwidth_enabled() && rt_policy(policy) && - task_group(p)->rt_bandwidth.rt_runtime == 0 && + if (dl_bandwidth_enabled() && rt_policy(policy) && + task_group(p)->dl_bandwidth.dl_runtime == 0 && !task_group_is_autogroup(task_group(p))) { task_rq_unlock(rq, p, &rf); return -EPERM; @@ -4259,7 +4363,7 @@ static int __sched_setscheduler(struct task_struct *p, queue_flags &= ~DEQUEUE_MOVE; } - queued = task_on_rq_queued(p); + queued = task_on_rq_queued(p) && !rt_throttled(p); running = task_current(rq, p); if (queued) dequeue_task(rq, p, queue_flags); @@ -7816,7 +7920,7 @@ void __init sched_init(void) #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_RT_GROUP_SCHED - root_task_group.rt_se = (struct sched_rt_entity **)ptr; + root_task_group.dl_se = (struct sched_dl_entity **)ptr; ptr += nr_cpu_ids * sizeof(void **); root_task_group.rt_rq = (struct rt_rq **)ptr; @@ -7831,8 +7935,6 @@ void __init sched_init(void) } #endif /* CONFIG_CPUMASK_OFFSTACK */ - init_rt_bandwidth(&def_rt_bandwidth, - global_rt_period(), global_rt_runtime()); init_dl_bandwidth(&def_dl_bandwidth, global_rt_period(), global_rt_runtime()); @@ -7841,10 +7943,10 @@ void __init sched_init(void) #endif #ifdef CONFIG_RT_GROUP_SCHED - init_rt_bandwidth(&root_task_group.rt_bandwidth, + init_dl_bandwidth(&root_task_group.dl_bandwidth, global_rt_period(), global_rt_runtime()); #endif /* CONFIG_RT_GROUP_SCHED */ - + #ifdef CONFIG_CGROUP_SCHED list_add(&root_task_group.list, &task_groups); INIT_LIST_HEAD(&root_task_group.children); @@ -7890,7 +7992,6 @@ void __init sched_init(void) init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); #endif /* CONFIG_FAIR_GROUP_SCHED */ - rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; #ifdef CONFIG_RT_GROUP_SCHED init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); #endif @@ -8299,9 +8400,10 @@ static int tg_rt_schedulable(struct task_group *tg, void *data) struct task_group *child; unsigned long total, sum = 0; u64 period, runtime; + unsigned long flags; - period = ktime_to_ns(tg->rt_bandwidth.rt_period); - runtime = tg->rt_bandwidth.rt_runtime; + period = tg->dl_bandwidth.dl_period; + runtime = tg->dl_bandwidth.dl_runtime; if (tg == d->tg) { period = d->rt_period; @@ -8317,7 +8419,7 @@ static int tg_rt_schedulable(struct task_group *tg, void *data) /* * Ensure we don't starve existing RT tasks. */ - if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg)) + if (dl_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg)) return -EBUSY; total = to_ratio(period, runtime); @@ -8328,12 +8430,27 @@ static int tg_rt_schedulable(struct task_group *tg, void *data) if (total > to_ratio(global_rt_period(), global_rt_runtime())) return -EINVAL; + if (tg == &root_task_group) { + int cpus = num_online_cpus(); + struct dl_bw *dl_b = dl_bw_of(smp_processor_id()); + + raw_spin_lock_irqsave(&dl_b->lock, flags); + + if (dl_b->bw != -1 && + dl_b->bw * cpus < dl_b->total_bw + total * cpus) { + raw_spin_unlock_irqrestore(&dl_b->lock, flags); + return -EBUSY; + } + + raw_spin_unlock_irqrestore(&dl_b->lock, flags); + } + /* * The sum of our children's runtime should not exceed our own. */ list_for_each_entry_rcu(child, &tg->children, siblings) { - period = ktime_to_ns(child->rt_bandwidth.rt_period); - runtime = child->rt_bandwidth.rt_runtime; + period = child->dl_bandwidth.dl_period; + runtime = child->dl_bandwidth.dl_runtime; if (child == d->tg) { period = d->rt_period; @@ -8388,18 +8505,33 @@ static int tg_set_rt_bandwidth(struct task_group *tg, if (err) goto unlock; - raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock); - tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period); - tg->rt_bandwidth.rt_runtime = rt_runtime; + raw_spin_lock_irq(&tg->dl_bandwidth.dl_runtime_lock); + tg->dl_bandwidth.dl_period = rt_period; + tg->dl_bandwidth.dl_runtime = rt_runtime; + + if (tg == &root_task_group) + goto unlock_bandwidth; for_each_possible_cpu(i) { - struct rt_rq *rt_rq = tg->rt_rq[i]; + struct sched_dl_entity *dl_se = tg->dl_se[i]; + struct rq *rq = container_of(dl_se->dl_rq, struct rq, dl); + + raw_spin_lock_irq(&rq->lock); + dl_se->dl_runtime = rt_runtime; + dl_se->dl_period = rt_period; + dl_se->dl_deadline = dl_se->dl_period; + dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); + + if (!((s64)(rt_period - rt_runtime) >= 0) || + !(rt_runtime >= (2 << (DL_SCALE - 1)))) { + raw_spin_unlock_irq(&rq->lock); + continue; + } - raw_spin_lock(&rt_rq->rt_runtime_lock); - rt_rq->rt_runtime = rt_runtime; - raw_spin_unlock(&rt_rq->rt_runtime_lock); + raw_spin_unlock_irq(&rq->lock); } - raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock); +unlock_bandwidth: + raw_spin_unlock_irq(&tg->dl_bandwidth.dl_runtime_lock); unlock: read_unlock(&tasklist_lock); mutex_unlock(&rt_constraints_mutex); @@ -8411,7 +8543,7 @@ static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us) { u64 rt_runtime, rt_period; - rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period); + rt_period = tg->dl_bandwidth.dl_period; rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC; if (rt_runtime_us < 0) rt_runtime = RUNTIME_INF; @@ -8423,10 +8555,10 @@ static long sched_group_rt_runtime(struct task_group *tg) { u64 rt_runtime_us; - if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF) + if (tg->dl_bandwidth.dl_runtime == RUNTIME_INF) return -1; - rt_runtime_us = tg->rt_bandwidth.rt_runtime; + rt_runtime_us = tg->dl_bandwidth.dl_runtime; do_div(rt_runtime_us, NSEC_PER_USEC); return rt_runtime_us; } @@ -8436,7 +8568,7 @@ static int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us) u64 rt_runtime, rt_period; rt_period = rt_period_us * NSEC_PER_USEC; - rt_runtime = tg->rt_bandwidth.rt_runtime; + rt_runtime = tg->dl_bandwidth.dl_runtime; return tg_set_rt_bandwidth(tg, rt_period, rt_runtime); } @@ -8445,7 +8577,7 @@ static long sched_group_rt_period(struct task_group *tg) { u64 rt_period_us; - rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period); + rt_period_us = tg->dl_bandwidth.dl_period; do_div(rt_period_us, NSEC_PER_USEC); return rt_period_us; } @@ -8468,7 +8600,7 @@ static int sched_rt_global_constraints(void) static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk) { /* Don't accept realtime tasks when there is no way for them to run */ - if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0) + if (rt_task(tsk) && tg->dl_bandwidth.dl_runtime == 0) return 0; return 1; @@ -8477,20 +8609,7 @@ static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk) #else /* !CONFIG_RT_GROUP_SCHED */ static int sched_rt_global_constraints(void) { - unsigned long flags; - int i, ret = 0; - - raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); - for_each_possible_cpu(i) { - struct rt_rq *rt_rq = &cpu_rq(i)->rt; - - raw_spin_lock(&rt_rq->rt_runtime_lock); - rt_rq->rt_runtime = global_rt_runtime(); - raw_spin_unlock(&rt_rq->rt_runtime_lock); - } - raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); - - return ret; + return 0; } #endif /* CONFIG_RT_GROUP_SCHED */ @@ -8570,12 +8689,6 @@ static int sched_rt_global_validate(void) return 0; } -static void sched_rt_do_global(void) -{ - def_rt_bandwidth.rt_runtime = global_rt_runtime(); - def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period()); -} - int sched_rt_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) @@ -8603,7 +8716,6 @@ int sched_rt_handler(struct ctl_table *table, int write, if (ret) goto undo; - sched_rt_do_global(); sched_dl_do_global(); } if (0) { @@ -8653,6 +8765,12 @@ cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) return &root_task_group.css; } + /* Do not allow cpu_cgroup hierachies with depth greater than 2. */ +#ifdef CONFIG_RT_GROUP_SCHED + if (parent != &root_task_group) + return ERR_PTR(-EINVAL); +#endif + tg = sched_create_group(parent); if (IS_ERR(tg)) return ERR_PTR(-ENOMEM); diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index db5a1fab8bda25..5c716897f56197 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -20,8 +20,17 @@ struct dl_bandwidth def_dl_bandwidth; +#ifdef CONFIG_RT_GROUP_SCHED +#define dl_entity_is_task(dl_se) (!(dl_se)->my_q) +#define rt_rq_of_dl_entity(dl_se) ((dl_se)->my_q) +#else +#define dl_entity_is_task(dl_se) (1) +#define rt_rq_of_dl_entity(dl_se) (NULL) +#endif + static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se) { + BUG_ON(!dl_entity_is_task(dl_se)); return container_of(dl_se, struct task_struct, dl); } @@ -30,6 +39,14 @@ static inline struct rq *rq_of_dl_rq(struct dl_rq *dl_rq) return container_of(dl_rq, struct rq, dl); } +#ifdef CONFIG_RT_GROUP_SCHED +static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se) +{ + return dl_se->dl_rq; +} + +#else /* CONFIG_RT_GROUP_SCHED */ + static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se) { struct task_struct *p = dl_task_of(dl_se); @@ -37,6 +54,7 @@ static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se) return &rq->dl; } +#endif static inline int on_dl_rq(struct sched_dl_entity *dl_se) { @@ -119,7 +137,11 @@ static inline void dl_clear_overload(struct rq *rq) static void update_dl_migration(struct dl_rq *dl_rq) { +#ifdef CONFIG_RT_GROUP_SCHED + if (dl_rq->dl_nr_migratory && dl_rq->dl_nr_total > 1) { +#else if (dl_rq->dl_nr_migratory && dl_rq->dl_nr_running > 1) { +#endif if (!dl_rq->overloaded) { dl_set_overload(rq_of_dl_rq(dl_rq)); dl_rq->overloaded = 1; @@ -515,11 +537,11 @@ static void update_dl_entity(struct sched_dl_entity *dl_se, * actually started or not (i.e., the replenishment instant is in * the future or in the past). */ -static int start_dl_timer(struct task_struct *p) +int start_dl_timer(struct sched_dl_entity *dl_se) { - struct sched_dl_entity *dl_se = &p->dl; struct hrtimer *timer = &dl_se->dl_timer; - struct rq *rq = task_rq(p); + struct dl_rq *dl_rq = dl_rq_of_se(dl_se); + struct rq *rq = rq_of_dl_rq(dl_rq); ktime_t now, act; s64 delta; @@ -553,7 +575,11 @@ static int start_dl_timer(struct task_struct *p) * and observe our state. */ if (!hrtimer_is_queued(timer)) { - get_task_struct(p); + if (dl_entity_is_task(dl_se)) { + struct task_struct *p = dl_task_of(dl_se); + + get_task_struct(p); + } hrtimer_start(timer, act, HRTIMER_MODE_ABS); } @@ -578,10 +604,46 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) struct sched_dl_entity *dl_se = container_of(timer, struct sched_dl_entity, dl_timer); - struct task_struct *p = dl_task_of(dl_se); + struct task_struct *p; struct rq_flags rf; struct rq *rq; +#ifdef CONFIG_RT_GROUP_SCHED + /* Replenish dl group and check for preemption. */ + if (!dl_entity_is_task(dl_se)) { + struct rt_rq *rt_rq = rt_rq_of_dl_entity(dl_se); + + rq = rq_of_dl_rq(dl_rq_of_se(dl_se)); + + raw_spin_lock(&rq->lock); + + + sched_clock_tick(); + update_rq_clock(rq); + + cfs_unthrottle_rt_tasks(rt_rq); + dl_se->dl_throttled = 0; + if (rt_rq->rt_nr_running) { + enqueue_dl_entity(dl_se, dl_se, ENQUEUE_REPLENISH); + + resched_curr(rq); +/* +#ifdef CONFIG_SMP + if (has_pushable_dl_tasks(rq)) + push_dl_task(rq); +#endif +*/ + } else { + replenish_dl_entity(dl_se, dl_se); + } + + raw_spin_unlock(&rq->lock); + + return HRTIMER_NORESTART; + } +#endif /* CONFIG_RT_GROUP_SCHED */ + + p = dl_task_of(dl_se); rq = task_rq_lock(p, &rf); /* @@ -689,14 +751,11 @@ void init_dl_task_timer(struct sched_dl_entity *dl_se) timer->function = dl_task_timer; } -static int dl_runtime_exceeded(struct sched_dl_entity *dl_se) { return (dl_se->runtime <= 0); } -extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq); - /* * Update the current task's runtime statistics (provided it is still * a -deadline task and has not been removed from the dl_rq). @@ -743,37 +802,12 @@ static void update_curr_dl(struct rq *rq) if (dl_runtime_exceeded(dl_se) || dl_se->dl_yielded) { dl_se->dl_throttled = 1; __dequeue_task_dl(rq, curr, 0); - if (unlikely(dl_se->dl_boosted || !start_dl_timer(curr))) + if (unlikely(dl_se->dl_boosted || !start_dl_timer(&curr->dl))) enqueue_task_dl(rq, curr, ENQUEUE_REPLENISH); if (!is_leftmost(curr, &rq->dl)) resched_curr(rq); } - - /* - * Because -- for now -- we share the rt bandwidth, we need to - * account our runtime there too, otherwise actual rt tasks - * would be able to exceed the shared quota. - * - * Account to the root rt group for now. - * - * The solution we're working towards is having the RT groups scheduled - * using deadline servers -- however there's a few nasties to figure - * out before that can happen. - */ - if (rt_bandwidth_enabled()) { - struct rt_rq *rt_rq = &rq->rt; - - raw_spin_lock(&rt_rq->rt_runtime_lock); - /* - * We'll let actual RT tasks worry about the overflow here, we - * have our own CBS to keep us inline; only account when RT - * bandwidth is relevant. - */ - if (sched_rt_bandwidth_account(rt_rq)) - rt_rq->rt_time += delta_exec; - raw_spin_unlock(&rt_rq->rt_runtime_lock); - } } #ifdef CONFIG_SMP @@ -821,29 +855,39 @@ static inline void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {} static inline void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) { - int prio = dl_task_of(dl_se)->prio; u64 deadline = dl_se->deadline; - WARN_ON(!dl_prio(prio)); - dl_rq->dl_nr_running++; - add_nr_running(rq_of_dl_rq(dl_rq), 1); + if (dl_entity_is_task(dl_se)) { + dl_rq->dl_nr_running++; + add_nr_running(rq_of_dl_rq(dl_rq), 1); + inc_dl_migration(dl_se, dl_rq); + } else { + struct rt_rq *rt_rq = rt_rq_of_dl_entity(dl_se); + + add_nr_running(rq_of_dl_rq(dl_rq), rt_rq->rt_nr_running); + } inc_dl_deadline(dl_rq, deadline); - inc_dl_migration(dl_se, dl_rq); } static inline void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) { - int prio = dl_task_of(dl_se)->prio; +#ifdef CONFIG_RT_GROUP_SCHED + WARN_ON(!dl_rq->dl_nr_total); +#endif - WARN_ON(!dl_prio(prio)); - WARN_ON(!dl_rq->dl_nr_running); - dl_rq->dl_nr_running--; - sub_nr_running(rq_of_dl_rq(dl_rq), 1); + if (dl_entity_is_task(dl_se)) { + dl_rq->dl_nr_running--; + sub_nr_running(rq_of_dl_rq(dl_rq), 1); + dec_dl_migration(dl_se, dl_rq); + } else { + struct rt_rq *rt_rq = rt_rq_of_dl_entity(dl_se); + + sub_nr_running(rq_of_dl_rq(dl_rq), rt_rq->rt_nr_running); + } dec_dl_deadline(dl_rq, dl_se->deadline); - dec_dl_migration(dl_se, dl_rq); } static void __enqueue_dl_entity(struct sched_dl_entity *dl_se) @@ -872,7 +916,9 @@ static void __enqueue_dl_entity(struct sched_dl_entity *dl_se) rb_link_node(&dl_se->rb_node, parent, link); rb_insert_color(&dl_se->rb_node, &dl_rq->rb_root); - +#ifdef CONFIG_RT_GROUP_SCHED + dl_rq->dl_nr_total++; +#endif inc_dl_tasks(dl_se, dl_rq); } @@ -894,9 +940,12 @@ static void __dequeue_dl_entity(struct sched_dl_entity *dl_se) RB_CLEAR_NODE(&dl_se->rb_node); dec_dl_tasks(dl_se, dl_rq); +#ifdef CONFIG_RT_GROUP_SCHED + dl_rq->dl_nr_total--; +#endif } -static void +void enqueue_dl_entity(struct sched_dl_entity *dl_se, struct sched_dl_entity *pi_se, int flags) { @@ -915,7 +964,7 @@ enqueue_dl_entity(struct sched_dl_entity *dl_se, __enqueue_dl_entity(dl_se); } -static void dequeue_dl_entity(struct sched_dl_entity *dl_se) +void dequeue_dl_entity(struct sched_dl_entity *dl_se) { __dequeue_dl_entity(dl_se); } @@ -1094,12 +1143,12 @@ static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p, } #ifdef CONFIG_SCHED_HRTICK -static void start_hrtick_dl(struct rq *rq, struct task_struct *p) +void start_hrtick_dl(struct rq *rq, struct sched_dl_entity *dl_se) { - hrtick_start(rq, p->dl.runtime); + hrtick_start(rq, dl_se->runtime); } #else /* !CONFIG_SCHED_HRTICK */ -static void start_hrtick_dl(struct rq *rq, struct task_struct *p) +void start_hrtick_dl(struct rq *rq, struct sched_dl_entity *dl_se) { } #endif @@ -1150,14 +1199,38 @@ pick_next_task_dl(struct rq *rq, struct task_struct *prev, struct pin_cookie coo if (prev->sched_class == &dl_sched_class) update_curr_dl(rq); +again: +#ifdef CONFIG_RT_GROUP_SCHED + if (unlikely(!dl_rq->dl_nr_total)) + return NULL; +#else if (unlikely(!dl_rq->dl_nr_running)) return NULL; - - put_prev_task(rq, prev); +#endif dl_se = pick_next_dl_entity(rq, dl_rq); BUG_ON(!dl_se); + put_prev_task(rq, prev); + if (dl_se->dl_throttled) + goto again; + + if (!dl_entity_is_task(dl_se)) { + struct rt_rq *rt_rq = rt_rq_of_dl_entity(dl_se); + struct sched_rt_entity *rt_se; + + rt_se = pick_next_rt_entity(rq, rt_rq); + p = container_of(rt_se, struct task_struct, rt); + p->se.exec_start = rq_clock_task(rq); + + dequeue_pushable_task(rt_rq_of_se(&p->rt), p); + + if (hrtick_enabled(rq)) + start_hrtick_dl(rq, dl_se); + + return p; + } + p = dl_task_of(dl_se); p->se.exec_start = rq_clock_task(rq); @@ -1165,7 +1238,7 @@ pick_next_task_dl(struct rq *rq, struct task_struct *prev, struct pin_cookie coo dequeue_pushable_dl_task(rq, p); if (hrtick_enabled(rq)) - start_hrtick_dl(rq, p); + start_hrtick_dl(rq, &p->dl); queue_push_tasks(rq); @@ -1191,7 +1264,7 @@ static void task_tick_dl(struct rq *rq, struct task_struct *p, int queued) */ if (hrtick_enabled(rq) && queued && p->dl.runtime > 0 && is_leftmost(p, &rq->dl)) - start_hrtick_dl(rq, p); + start_hrtick_dl(rq, &p->dl); } static void task_fork_dl(struct task_struct *p) @@ -1607,14 +1680,21 @@ static void pull_dl_task(struct rq *this_rq) */ static void task_woken_dl(struct rq *rq, struct task_struct *p) { - if (!task_running(rq, p) && - !test_tsk_need_resched(rq->curr) && - tsk_nr_cpus_allowed(p) > 1 && - dl_task(rq->curr) && + if (task_running(rq, p) || + test_tsk_need_resched(rq->curr) || + p->nr_cpus_allowed <= 1) + return; + + if (dl_task(rq->curr) && (tsk_nr_cpus_allowed(rq->curr) < 2 || !dl_entity_preempt(&p->dl, &rq->curr->dl))) { push_dl_tasks(rq); } + +#ifdef CONFIG_RT_GROUP_SCHED + if (rt_task(rq->curr) && is_dl_group(rq->curr->rt.rt_rq)) + push_dl_tasks(rq); +#endif } static void set_cpus_allowed_dl(struct task_struct *p, @@ -1690,7 +1770,7 @@ static void switched_from_dl(struct rq *rq, struct task_struct *p) * SCHED_DEADLINE until the deadline passes, the timer will reset the * task. */ - if (!start_dl_timer(p)) + if (!start_dl_timer(&p->dl)) __dl_clear_params(p); /* @@ -1698,10 +1778,15 @@ static void switched_from_dl(struct rq *rq, struct task_struct *p) * this is the right place to try to pull some other one * from an overloaded cpu, if any. */ +#ifdef CONFIG_RT_GROUP_SCHED + if (!rq->dl.dl_nr_total) + queue_pull_task(rq); +#else if (!task_on_rq_queued(p) || rq->dl.dl_nr_running) return; queue_pull_task(rq); +#endif } /* diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 641511771ae6a6..22a3bd4d7dc749 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -248,9 +248,6 @@ void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq) SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rt_rq->x)) P(rt_nr_running); - P(rt_throttled); - PN(rt_time); - PN(rt_runtime); #undef PN #undef P diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index aa9cc1015f6d84..15c80219dc8175 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6088,9 +6088,36 @@ static void migrate_task_rq_fair(struct task_struct *p) p->se.exec_start = 0; } +#ifdef CONFIG_RT_GROUP_SCHED +static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) +{ + return rt_rq->rq; +} +#else +static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) +{ + return container_of(rt_rq, struct rq, rt); +} +#endif /* CONFIG_RT_GROUP_SCHED */ + static void task_dead_fair(struct task_struct *p) { remove_entity_load_avg(&p->se); + + /* + * p got killed while hanging out in RT. + * Remove it from throttled_task list. + */ + if (rt_throttled(p)) { + struct sched_rt_entity *rt_se = &p->rt; + struct rt_rq *rt_rq = rt_se->cfs_throttle_rt_rq; + unsigned long flags = 0; + + raw_spin_lock_irqsave(&rq_of_rt_rq(rt_rq)->lock, flags); + lockdep_assert_held(&rq_of_rt_rq(rt_rq)->lock); + list_del_init(&rt_se->cfs_throttled_task); + raw_spin_unlock_irqrestore(&rq_of_rt_rq(rt_rq)->lock, flags); + } } #else #define task_fits_max(p, cpu) true @@ -9296,6 +9323,21 @@ static void switched_from_fair(struct rq *rq, struct task_struct *p) static void switched_to_fair(struct rq *rq, struct task_struct *p) { + if (!rt_prio(p->normal_prio) && rt_throttled(p)) { + struct sched_rt_entity *rt_se = &p->rt; + struct rt_rq *rt_rq = rt_se->cfs_throttle_rt_rq; + + if (cpu_rq(task_cpu(p)) != rq_of_rt_rq(rt_rq)) + double_lock_balance(cpu_rq(task_cpu(p)), rq_of_rt_rq(rt_rq)); + + lockdep_assert_held(&rq_of_rt_rq(rt_rq)->lock); + list_del_init(&rt_se->cfs_throttled_task); + + rt_se->cfs_throttle_rt_rq = NULL; + if (cpu_rq(task_cpu(p)) != rq_of_rt_rq(rt_rq)) + double_unlock_balance(cpu_rq(task_cpu(p)), rq_of_rt_rq(rt_rq)); + } + attach_task_cfs_rq(p); if (task_on_rq_queued(p)) { diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 63d87fb7666fac..f36839537f4a3e 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -12,60 +12,6 @@ int sched_rr_timeslice = RR_TIMESLICE; -static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun); - -struct rt_bandwidth def_rt_bandwidth; - -static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer) -{ - struct rt_bandwidth *rt_b = - container_of(timer, struct rt_bandwidth, rt_period_timer); - int idle = 0; - int overrun; - - raw_spin_lock(&rt_b->rt_runtime_lock); - for (;;) { - overrun = hrtimer_forward_now(timer, rt_b->rt_period); - if (!overrun) - break; - - raw_spin_unlock(&rt_b->rt_runtime_lock); - idle = do_sched_rt_period_timer(rt_b, overrun); - raw_spin_lock(&rt_b->rt_runtime_lock); - } - if (idle) - rt_b->rt_period_active = 0; - raw_spin_unlock(&rt_b->rt_runtime_lock); - - return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; -} - -void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime) -{ - rt_b->rt_period = ns_to_ktime(period); - rt_b->rt_runtime = runtime; - - raw_spin_lock_init(&rt_b->rt_runtime_lock); - - hrtimer_init(&rt_b->rt_period_timer, - CLOCK_MONOTONIC, HRTIMER_MODE_REL); - rt_b->rt_period_timer.function = sched_rt_period_timer; -} - -static void start_rt_bandwidth(struct rt_bandwidth *rt_b) -{ - if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF) - return; - - raw_spin_lock(&rt_b->rt_runtime_lock); - if (!rt_b->rt_period_active) { - rt_b->rt_period_active = 1; - hrtimer_forward_now(&rt_b->rt_period_timer, rt_b->rt_period); - hrtimer_start_expires(&rt_b->rt_period_timer, HRTIMER_MODE_ABS_PINNED); - } - raw_spin_unlock(&rt_b->rt_runtime_lock); -} - #if defined(CONFIG_SMP) && defined(HAVE_RT_PUSH_IPI) static void push_irq_work_func(struct irq_work *work); #endif @@ -97,108 +43,77 @@ void init_rt_rq(struct rt_rq *rt_rq) init_irq_work(&rt_rq->push_work, push_irq_work_func); #endif #endif /* CONFIG_SMP */ - /* We start is dequeued state, because no RT tasks are queued */ - rt_rq->rt_queued = 0; - - rt_rq->rt_time = 0; - rt_rq->rt_throttled = 0; - rt_rq->rt_runtime = 0; - raw_spin_lock_init(&rt_rq->rt_runtime_lock); -} - #ifdef CONFIG_RT_GROUP_SCHED -static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b) -{ - hrtimer_cancel(&rt_b->rt_period_timer); -} - -#define rt_entity_is_task(rt_se) (!(rt_se)->my_q) - -static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) -{ -#ifdef CONFIG_SCHED_DEBUG - WARN_ON_ONCE(!rt_entity_is_task(rt_se)); + INIT_LIST_HEAD(&rt_rq->cfs_throttled_tasks); #endif - return container_of(rt_se, struct task_struct, rt); } -static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) -{ - return rt_rq->rq; -} - -static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) -{ - return rt_se->rt_rq; -} +#ifdef CONFIG_RT_GROUP_SCHED -static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) +static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) { - struct rt_rq *rt_rq = rt_se->rt_rq; - return rt_rq->rq; } void free_rt_sched_group(struct task_group *tg) { + unsigned long flags; int i; - if (tg->rt_se) - destroy_rt_bandwidth(&tg->rt_bandwidth); - for_each_possible_cpu(i) { if (tg->rt_rq) kfree(tg->rt_rq[i]); - if (tg->rt_se) - kfree(tg->rt_se[i]); + if (tg->dl_se) { + raw_spin_lock_irqsave(&cpu_rq(i)->lock, flags); + if (!tg->dl_se[i]->dl_throttled) + dequeue_dl_entity(tg->dl_se[i]); + raw_spin_unlock_irqrestore(&cpu_rq(i)->lock, flags); + + hrtimer_cancel(&tg->dl_se[i]->dl_timer); + kfree(tg->dl_se[i]); + } } kfree(tg->rt_rq); - kfree(tg->rt_se); + kfree(tg->dl_se); } void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq, - struct sched_rt_entity *rt_se, int cpu, - struct sched_rt_entity *parent) + struct sched_dl_entity *dl_se, int cpu, + struct sched_dl_entity *parent) { struct rq *rq = cpu_rq(cpu); rt_rq->highest_prio.curr = MAX_RT_PRIO; - rt_rq->rt_nr_boosted = 0; rt_rq->rq = rq; rt_rq->tg = tg; tg->rt_rq[cpu] = rt_rq; - tg->rt_se[cpu] = rt_se; + tg->dl_se[cpu] = dl_se; - if (!rt_se) + if (!dl_se) return; - if (!parent) - rt_se->rt_rq = &rq->rt; - else - rt_se->rt_rq = parent->my_q; - - rt_se->my_q = rt_rq; - rt_se->parent = parent; - INIT_LIST_HEAD(&rt_se->run_list); + dl_se->dl_rq = &rq->dl; + dl_se->my_q = rt_rq; + RB_CLEAR_NODE(&dl_se->rb_node); } int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) { struct rt_rq *rt_rq; - struct sched_rt_entity *rt_se; + struct sched_dl_entity *dl_se; int i; - tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL); + tg->rt_rq = kcalloc(nr_cpu_ids, sizeof(rt_rq), GFP_KERNEL); if (!tg->rt_rq) goto err; - tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL); - if (!tg->rt_se) + tg->dl_se = kcalloc(nr_cpu_ids, sizeof(dl_se), GFP_KERNEL); + if (!tg->dl_se) goto err; - init_rt_bandwidth(&tg->rt_bandwidth, - ktime_to_ns(def_rt_bandwidth.rt_period), 0); + init_dl_bandwidth(&tg->dl_bandwidth, + def_dl_bandwidth.dl_period, 0); for_each_possible_cpu(i) { rt_rq = kzalloc_node(sizeof(struct rt_rq), @@ -206,14 +121,24 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) if (!rt_rq) goto err; - rt_se = kzalloc_node(sizeof(struct sched_rt_entity), + dl_se = kzalloc_node(sizeof(struct sched_dl_entity), GFP_KERNEL, cpu_to_node(i)); - if (!rt_se) + if (!dl_se) goto err_free_rq; init_rt_rq(rt_rq); - rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime; - init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]); + rt_rq->rq = cpu_rq(i); + + init_dl_task_timer(dl_se); + + dl_se->dl_runtime = tg->dl_bandwidth.dl_runtime; + dl_se->dl_period = tg->dl_bandwidth.dl_period; + dl_se->dl_deadline = dl_se->dl_period; + dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); + + dl_se->dl_throttled = 0; + + init_tg_rt_entry(tg, rt_rq, dl_se, i, parent->dl_se[i]); } return 1; @@ -226,32 +151,11 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) #else /* CONFIG_RT_GROUP_SCHED */ -#define rt_entity_is_task(rt_se) (1) - -static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) -{ - return container_of(rt_se, struct task_struct, rt); -} - static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) { return container_of(rt_rq, struct rq, rt); } -static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) -{ - struct task_struct *p = rt_task_of(rt_se); - - return task_rq(p); -} - -static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) -{ - struct rq *rq = rq_of_rt_se(rt_se); - - return &rq->rt; -} - void free_rt_sched_group(struct task_group *tg) { } int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) @@ -306,7 +210,7 @@ static inline void rt_clear_overload(struct rq *rq) static void update_rt_migration(struct rt_rq *rt_rq) { - if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) { + if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_running > 1) { if (!rt_rq->overloaded) { rt_set_overload(rq_of_rt_rq(rt_rq)); rt_rq->overloaded = 1; @@ -321,13 +225,8 @@ static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) { struct task_struct *p; - if (!rt_entity_is_task(rt_se)) - return; - p = rt_task_of(rt_se); - rt_rq = &rq_of_rt_rq(rt_rq)->rt; - rt_rq->rt_nr_total++; if (tsk_nr_cpus_allowed(p) > 1) rt_rq->rt_nr_migratory++; @@ -338,22 +237,17 @@ static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) { struct task_struct *p; - if (!rt_entity_is_task(rt_se)) - return; - p = rt_task_of(rt_se); - rt_rq = &rq_of_rt_rq(rt_rq)->rt; - rt_rq->rt_nr_total--; if (tsk_nr_cpus_allowed(p) > 1) rt_rq->rt_nr_migratory--; update_rt_migration(rt_rq); } -static inline int has_pushable_tasks(struct rq *rq) +static inline int has_pushable_tasks(struct rt_rq *rt_rq) { - return !plist_head_empty(&rq->rt.pushable_tasks); + return !plist_head_empty(&rt_rq->pushable_tasks); } static DEFINE_PER_CPU(struct callback_head, rt_push_head); @@ -364,7 +258,7 @@ static void pull_rt_task(struct rq *); static inline void queue_push_tasks(struct rq *rq) { - if (!has_pushable_tasks(rq)) + if (!has_pushable_tasks(&rq->rt)) return; queue_balance_callback(rq, &per_cpu(rt_push_head, rq->cpu), push_rt_tasks); @@ -375,37 +269,35 @@ static inline void queue_pull_task(struct rq *rq) queue_balance_callback(rq, &per_cpu(rt_pull_head, rq->cpu), pull_rt_task); } -static void enqueue_pushable_task(struct rq *rq, struct task_struct *p) +static void enqueue_pushable_task(struct rt_rq *rt_rq, struct task_struct *p) { - plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); + plist_del(&p->pushable_tasks, &rt_rq->pushable_tasks); plist_node_init(&p->pushable_tasks, p->prio); - plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks); + plist_add(&p->pushable_tasks, &rt_rq->pushable_tasks); /* Update the highest prio pushable task */ - if (p->prio < rq->rt.highest_prio.next) - rq->rt.highest_prio.next = p->prio; + if (p->prio < rt_rq->highest_prio.next) + rt_rq->highest_prio.next = p->prio; } -static void dequeue_pushable_task(struct rq *rq, struct task_struct *p) +#ifdef CONFIG_RT_GROUP_SCHED +void dequeue_pushable_task(struct rt_rq *rt_rq, struct task_struct *p) { - plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); + plist_del(&p->pushable_tasks, &rt_rq->pushable_tasks); /* Update the new highest prio pushable task */ - if (has_pushable_tasks(rq)) { - p = plist_first_entry(&rq->rt.pushable_tasks, + if (has_pushable_tasks(rt_rq)) { + p = plist_first_entry(&rt_rq->pushable_tasks, struct task_struct, pushable_tasks); - rq->rt.highest_prio.next = p->prio; + rt_rq->highest_prio.next = p->prio; } else - rq->rt.highest_prio.next = MAX_RT_PRIO; + rt_rq->highest_prio.next = MAX_RT_PRIO; } - +#endif #else -static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p) -{ -} - -static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p) +static inline +void enqueue_pushable_task(struct rt_rq *rt_rq, struct task_struct *p) { } @@ -431,558 +323,77 @@ static inline void pull_rt_task(struct rq *this_rq) static inline void queue_push_tasks(struct rq *rq) { } -#endif /* CONFIG_SMP */ - -static void enqueue_top_rt_rq(struct rt_rq *rt_rq); -static void dequeue_top_rt_rq(struct rt_rq *rt_rq); - -static inline int on_rt_rq(struct sched_rt_entity *rt_se) -{ - return rt_se->on_rq; -} - -#ifdef CONFIG_RT_GROUP_SCHED - -static inline u64 sched_rt_runtime(struct rt_rq *rt_rq) -{ - if (!rt_rq->tg) - return RUNTIME_INF; - - return rt_rq->rt_runtime; -} - -static inline u64 sched_rt_period(struct rt_rq *rt_rq) -{ - return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period); -} - -typedef struct task_group *rt_rq_iter_t; - -static inline struct task_group *next_task_group(struct task_group *tg) -{ - do { - tg = list_entry_rcu(tg->list.next, - typeof(struct task_group), list); - } while (&tg->list != &task_groups && task_group_is_autogroup(tg)); - - if (&tg->list == &task_groups) - tg = NULL; - - return tg; -} - -#define for_each_rt_rq(rt_rq, iter, rq) \ - for (iter = container_of(&task_groups, typeof(*iter), list); \ - (iter = next_task_group(iter)) && \ - (rt_rq = iter->rt_rq[cpu_of(rq)]);) - -#define for_each_sched_rt_entity(rt_se) \ - for (; rt_se; rt_se = rt_se->parent) - -static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) -{ - return rt_se->my_q; -} - -static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags); -static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags); - -static void sched_rt_rq_enqueue(struct rt_rq *rt_rq) -{ - struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr; - struct rq *rq = rq_of_rt_rq(rt_rq); - struct sched_rt_entity *rt_se; - - int cpu = cpu_of(rq); - - rt_se = rt_rq->tg->rt_se[cpu]; - - if (rt_rq->rt_nr_running) { - if (!rt_se) - enqueue_top_rt_rq(rt_rq); - else if (!on_rt_rq(rt_se)) - enqueue_rt_entity(rt_se, 0); - - if (rt_rq->highest_prio.curr < curr->prio) - resched_curr(rq); - } -} - -static void sched_rt_rq_dequeue(struct rt_rq *rt_rq) -{ - struct sched_rt_entity *rt_se; - int cpu = cpu_of(rq_of_rt_rq(rt_rq)); - - rt_se = rt_rq->tg->rt_se[cpu]; - - if (!rt_se) - dequeue_top_rt_rq(rt_rq); - else if (on_rt_rq(rt_se)) - dequeue_rt_entity(rt_se, 0); -} - -static inline int rt_rq_throttled(struct rt_rq *rt_rq) -{ - return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted; -} - -static int rt_se_boosted(struct sched_rt_entity *rt_se) -{ - struct rt_rq *rt_rq = group_rt_rq(rt_se); - struct task_struct *p; - - if (rt_rq) - return !!rt_rq->rt_nr_boosted; - - p = rt_task_of(rt_se); - return p->prio != p->normal_prio; -} - -#ifdef CONFIG_SMP -static inline const struct cpumask *sched_rt_period_mask(void) -{ - return this_rq()->rd->span; -} -#else -static inline const struct cpumask *sched_rt_period_mask(void) -{ - return cpu_online_mask; -} -#endif - -static inline -struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu) -{ - return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu]; -} - -static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq) -{ - return &rt_rq->tg->rt_bandwidth; -} - -#else /* !CONFIG_RT_GROUP_SCHED */ - -static inline u64 sched_rt_runtime(struct rt_rq *rt_rq) -{ - return rt_rq->rt_runtime; -} - -static inline u64 sched_rt_period(struct rt_rq *rt_rq) -{ - return ktime_to_ns(def_rt_bandwidth.rt_period); -} - -typedef struct rt_rq *rt_rq_iter_t; - -#define for_each_rt_rq(rt_rq, iter, rq) \ - for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL) - -#define for_each_sched_rt_entity(rt_se) \ - for (; rt_se; rt_se = NULL) - -static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) -{ - return NULL; -} - -static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq) -{ - struct rq *rq = rq_of_rt_rq(rt_rq); - - if (!rt_rq->rt_nr_running) - return; - - enqueue_top_rt_rq(rt_rq); - resched_curr(rq); -} - -static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq) -{ - dequeue_top_rt_rq(rt_rq); -} - -static inline int rt_rq_throttled(struct rt_rq *rt_rq) -{ - return rt_rq->rt_throttled; -} - -static inline const struct cpumask *sched_rt_period_mask(void) -{ - return cpu_online_mask; -} - -static inline -struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu) -{ - return &cpu_rq(cpu)->rt; -} - -static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq) -{ - return &def_rt_bandwidth; -} - -#endif /* CONFIG_RT_GROUP_SCHED */ - -bool sched_rt_bandwidth_account(struct rt_rq *rt_rq) -{ - struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); - - return (hrtimer_active(&rt_b->rt_period_timer) || - rt_rq->rt_time < rt_b->rt_runtime); -} - -#ifdef CONFIG_SMP -/* - * We ran out of runtime, see if we can borrow some from our neighbours. - */ -static void do_balance_runtime(struct rt_rq *rt_rq) -{ - struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); - struct root_domain *rd = rq_of_rt_rq(rt_rq)->rd; - int i, weight; - u64 rt_period; - - weight = cpumask_weight(rd->span); - - raw_spin_lock(&rt_b->rt_runtime_lock); - rt_period = ktime_to_ns(rt_b->rt_period); - for_each_cpu(i, rd->span) { - struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); - s64 diff; - - if (iter == rt_rq) - continue; - - raw_spin_lock(&iter->rt_runtime_lock); - /* - * Either all rqs have inf runtime and there's nothing to steal - * or __disable_runtime() below sets a specific rq to inf to - * indicate its been disabled and disalow stealing. - */ - if (iter->rt_runtime == RUNTIME_INF) - goto next; - - /* - * From runqueues with spare time, take 1/n part of their - * spare time, but no more than our period. - */ - diff = iter->rt_runtime - iter->rt_time; - if (diff > 0) { - diff = div_u64((u64)diff, weight); - if (rt_rq->rt_runtime + diff > rt_period) - diff = rt_period - rt_rq->rt_runtime; - iter->rt_runtime -= diff; - rt_rq->rt_runtime += diff; - if (rt_rq->rt_runtime == rt_period) { - raw_spin_unlock(&iter->rt_runtime_lock); - break; - } - } -next: - raw_spin_unlock(&iter->rt_runtime_lock); - } - raw_spin_unlock(&rt_b->rt_runtime_lock); -} - -/* - * Ensure this RQ takes back all the runtime it lend to its neighbours. - */ -static void __disable_runtime(struct rq *rq) -{ - struct root_domain *rd = rq->rd; - rt_rq_iter_t iter; - struct rt_rq *rt_rq; - - if (unlikely(!scheduler_running)) - return; - - for_each_rt_rq(rt_rq, iter, rq) { - struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); - s64 want; - int i; - - raw_spin_lock(&rt_b->rt_runtime_lock); - raw_spin_lock(&rt_rq->rt_runtime_lock); - /* - * Either we're all inf and nobody needs to borrow, or we're - * already disabled and thus have nothing to do, or we have - * exactly the right amount of runtime to take out. - */ - if (rt_rq->rt_runtime == RUNTIME_INF || - rt_rq->rt_runtime == rt_b->rt_runtime) - goto balanced; - raw_spin_unlock(&rt_rq->rt_runtime_lock); - - /* - * Calculate the difference between what we started out with - * and what we current have, that's the amount of runtime - * we lend and now have to reclaim. - */ - want = rt_b->rt_runtime - rt_rq->rt_runtime; - - /* - * Greedy reclaim, take back as much as we can. - */ - for_each_cpu(i, rd->span) { - struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); - s64 diff; - - /* - * Can't reclaim from ourselves or disabled runqueues. - */ - if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF) - continue; - - raw_spin_lock(&iter->rt_runtime_lock); - if (want > 0) { - diff = min_t(s64, iter->rt_runtime, want); - iter->rt_runtime -= diff; - want -= diff; - } else { - iter->rt_runtime -= want; - want -= want; - } - raw_spin_unlock(&iter->rt_runtime_lock); - - if (!want) - break; - } - - raw_spin_lock(&rt_rq->rt_runtime_lock); - /* - * We cannot be left wanting - that would mean some runtime - * leaked out of the system. - */ - BUG_ON(want); -balanced: - /* - * Disable all the borrow logic by pretending we have inf - * runtime - in which case borrowing doesn't make sense. - */ - rt_rq->rt_runtime = RUNTIME_INF; - rt_rq->rt_throttled = 0; - raw_spin_unlock(&rt_rq->rt_runtime_lock); - raw_spin_unlock(&rt_b->rt_runtime_lock); - - /* Make rt_rq available for pick_next_task() */ - sched_rt_rq_enqueue(rt_rq); - } -} - -static void __enable_runtime(struct rq *rq) -{ - rt_rq_iter_t iter; - struct rt_rq *rt_rq; - - if (unlikely(!scheduler_running)) - return; - - /* - * Reset each runqueue's bandwidth settings - */ - for_each_rt_rq(rt_rq, iter, rq) { - struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); - - raw_spin_lock(&rt_b->rt_runtime_lock); - raw_spin_lock(&rt_rq->rt_runtime_lock); - rt_rq->rt_runtime = rt_b->rt_runtime; - rt_rq->rt_time = 0; - rt_rq->rt_throttled = 0; - raw_spin_unlock(&rt_rq->rt_runtime_lock); - raw_spin_unlock(&rt_b->rt_runtime_lock); - } -} -static void balance_runtime(struct rt_rq *rt_rq) +static inline void queue_pull_task(struct rq *rq) { - if (!sched_feat(RT_RUNTIME_SHARE)) - return; - - if (rt_rq->rt_time > rt_rq->rt_runtime) { - raw_spin_unlock(&rt_rq->rt_runtime_lock); - do_balance_runtime(rt_rq); - raw_spin_lock(&rt_rq->rt_runtime_lock); - } } -#else /* !CONFIG_SMP */ -static inline void balance_runtime(struct rt_rq *rt_rq) {} #endif /* CONFIG_SMP */ -static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) +static inline int on_rt_rq(struct sched_rt_entity *rt_se) { - int i, idle = 1, throttled = 0; - const struct cpumask *span; - - span = sched_rt_period_mask(); -#ifdef CONFIG_RT_GROUP_SCHED - /* - * FIXME: isolated CPUs should really leave the root task group, - * whether they are isolcpus or were isolated via cpusets, lest - * the timer run on a CPU which does not service all runqueues, - * potentially leaving other CPUs indefinitely throttled. If - * isolation is really required, the user will turn the throttle - * off to kill the perturbations it causes anyway. Meanwhile, - * this maintains functionality for boot and/or troubleshooting. - */ - if (rt_b == &root_task_group.rt_bandwidth) - span = cpu_online_mask; -#endif - for_each_cpu(i, span) { - int enqueue = 0; - struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i); - struct rq *rq = rq_of_rt_rq(rt_rq); - - raw_spin_lock(&rq->lock); - if (rt_rq->rt_time) { - u64 runtime; - - raw_spin_lock(&rt_rq->rt_runtime_lock); - if (rt_rq->rt_throttled) - balance_runtime(rt_rq); - runtime = rt_rq->rt_runtime; - rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime); - if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) { - rt_rq->rt_throttled = 0; - enqueue = 1; - - /* - * When we're idle and a woken (rt) task is - * throttled check_preempt_curr() will set - * skip_update and the time between the wakeup - * and this unthrottle will get accounted as - * 'runtime'. - */ - if (rt_rq->rt_nr_running && rq->curr == rq->idle) - rq_clock_skip_update(rq, false); - } - if (rt_rq->rt_time || rt_rq->rt_nr_running) - idle = 0; - raw_spin_unlock(&rt_rq->rt_runtime_lock); - } else if (rt_rq->rt_nr_running) { - idle = 0; - if (!rt_rq_throttled(rt_rq)) - enqueue = 1; - } - if (rt_rq->rt_throttled) - throttled = 1; - - if (enqueue) - sched_rt_rq_enqueue(rt_rq); - raw_spin_unlock(&rq->lock); - } - - if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)) - return 1; - - return idle; + return rt_se->on_rq; } static inline int rt_se_prio(struct sched_rt_entity *rt_se) { -#ifdef CONFIG_RT_GROUP_SCHED - struct rt_rq *rt_rq = group_rt_rq(rt_se); - - if (rt_rq) - return rt_rq->highest_prio.curr; -#endif - return rt_task_of(rt_se)->prio; } -static void dump_throttled_rt_tasks(struct rt_rq *rt_rq) +/* + * Iterates through all the tasks on @rt_rq and, depending on @enqueue, moves + * them between FIFO and OTHER. + */ +static void cfs_throttle_rt_tasks(struct rt_rq *rt_rq) { struct rt_prio_array *array = &rt_rq->active; - struct sched_rt_entity *rt_se; - char buf[500]; - char *pos = buf; - char *end = buf + sizeof(buf); + struct rq *rq = rq_of_rt_rq(rt_rq); int idx; - - pos += snprintf(pos, sizeof(buf), - "sched: RT throttling activated for rt_rq %p (cpu %d)\n", - rt_rq, cpu_of(rq_of_rt_rq(rt_rq))); + struct list_head *pos, *n; if (bitmap_empty(array->bitmap, MAX_RT_PRIO)) - goto out; + return; - pos += snprintf(pos, end - pos, "potential CPU hogs:\n"); idx = sched_find_first_bit(array->bitmap); while (idx < MAX_RT_PRIO) { - list_for_each_entry(rt_se, array->queue + idx, run_list) { + list_for_each_safe(pos, n, array->queue + idx) { + struct sched_rt_entity *rt_se; struct task_struct *p; - if (!rt_entity_is_task(rt_se)) - continue; + rt_se = list_entry(pos, struct sched_rt_entity, run_list); p = rt_task_of(rt_se); - if (pos < end) - pos += snprintf(pos, end - pos, "\t%s (%d)\n", - p->comm, p->pid); + /* + * Don't enqueue in fair if the task is going + * to sleep. We'll handle the transition at + * wakeup time eventually. + */ + if (p->state != TASK_RUNNING) + continue; + + list_add(&rt_se->cfs_throttled_task, + &rt_rq->cfs_throttled_tasks); + __setprio_other(rq, p); } idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx + 1); } -out: -#ifdef CONFIG_PANIC_ON_RT_THROTTLING - /* - * Use pr_err() in the BUG() case since printk_sched() will - * not get flushed and deadlock is not a concern. - */ - pr_err("%s", buf); - BUG(); -#else - printk_deferred("%s", buf); -#endif } -static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) +void cfs_unthrottle_rt_tasks(struct rt_rq *rt_rq) { - u64 runtime = sched_rt_runtime(rt_rq); - - if (rt_rq->rt_throttled) - return rt_rq_throttled(rt_rq); - - if (runtime >= sched_rt_period(rt_rq)) - return 0; - - balance_runtime(rt_rq); - runtime = sched_rt_runtime(rt_rq); - if (runtime == RUNTIME_INF) - return 0; - - if (rt_rq->rt_time > runtime) { - struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); + struct rq *rq = rq_of_rt_rq(rt_rq); - /* - * Don't actually throttle groups that have no runtime assigned - * but accrue some time due to boosting. - */ - if (likely(rt_b->rt_runtime)) { - static bool once = false; + while (!list_empty(&rt_rq->cfs_throttled_tasks)) { + struct sched_rt_entity *rt_se; + struct task_struct *p; - rt_rq->rt_throttled = 1; + rt_se = list_first_entry(&rt_rq->cfs_throttled_tasks, + struct sched_rt_entity, + cfs_throttled_task); - if (!once) { - once = true; - dump_throttled_rt_tasks(rt_rq); - } - } else { - /* - * In case we did anyway, make it go away, - * replenishment is a joke, since it will replenish us - * with exactly 0 ns. - */ - rt_rq->rt_time = 0; - } - - if (rt_rq_throttled(rt_rq)) { - sched_rt_rq_dequeue(rt_rq); - return 1; - } + p = rt_task_of(rt_se); + list_del_init(&rt_se->cfs_throttled_task); + __setprio_fifo(rq, p); } - - return 0; } /* @@ -992,7 +403,7 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) static void update_curr_rt(struct rq *rq) { struct task_struct *curr = rq->curr; - struct sched_rt_entity *rt_se = &curr->rt; + struct rt_rq *rt_rq = rt_rq_of_se(&curr->rt); u64 delta_exec; if (curr->sched_class != &rt_sched_class) @@ -1016,52 +427,34 @@ static void update_curr_rt(struct rq *rq) sched_rt_avg_update(rq, delta_exec); - if (!rt_bandwidth_enabled()) + if (!dl_bandwidth_enabled()) return; - for_each_sched_rt_entity(rt_se) { - struct rt_rq *rt_rq = rt_rq_of_se(rt_se); + if (is_dl_group(rt_rq)) { + struct sched_dl_entity *dl_se = dl_group_of(rt_rq); - if (sched_rt_runtime(rt_rq) != RUNTIME_INF) { - raw_spin_lock(&rt_rq->rt_runtime_lock); - rt_rq->rt_time += delta_exec; - if (sched_rt_runtime_exceeded(rt_rq)) - resched_curr(rq); - raw_spin_unlock(&rt_rq->rt_runtime_lock); + if (dl_se->dl_throttled) { + resched_curr(rq); + return; } - } -} - -static void -dequeue_top_rt_rq(struct rt_rq *rt_rq) -{ - struct rq *rq = rq_of_rt_rq(rt_rq); - BUG_ON(&rq->rt != rt_rq); + BUG_ON(rt_rq->rt_nr_running > rq->nr_running); + dl_se->runtime -= delta_exec; - if (!rt_rq->rt_queued) - return; - - BUG_ON(!rq->nr_running); - - sub_nr_running(rq, rt_rq->rt_nr_running); - rt_rq->rt_queued = 0; -} - -static void -enqueue_top_rt_rq(struct rt_rq *rt_rq) -{ - struct rq *rq = rq_of_rt_rq(rt_rq); + /* A group exhausts the budget. */ + if (dl_runtime_exceeded(dl_se)) { + dequeue_dl_entity(dl_se); - BUG_ON(&rq->rt != rt_rq); + if (likely(start_dl_timer(dl_se))) { + dl_se->dl_throttled = 1; + cfs_throttle_rt_tasks(rt_rq); + } else + enqueue_dl_entity(dl_se, dl_se, + ENQUEUE_REPLENISH); - if (rt_rq->rt_queued) - return; - if (rt_rq_throttled(rt_rq) || !rt_rq->rt_nr_running) - return; - - add_nr_running(rq, rt_rq->rt_nr_running); - rt_rq->rt_queued = 1; + resched_curr(rq); + } + } } #if defined CONFIG_SMP @@ -1071,13 +464,9 @@ inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) { struct rq *rq = rq_of_rt_rq(rt_rq); -#ifdef CONFIG_RT_GROUP_SCHED - /* - * Change rq's cpupri only if rt_rq is the top queue. - */ - if (&rq->rt != rt_rq) + if (is_dl_group(rt_rq)) return; -#endif + if (rq->online && prio < prev_prio) cpupri_set(&rq->rd->cpupri, rq->cpu, prio); } @@ -1087,13 +476,9 @@ dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) { struct rq *rq = rq_of_rt_rq(rt_rq); -#ifdef CONFIG_RT_GROUP_SCHED - /* - * Change rq's cpupri only if rt_rq is the top queue. - */ - if (&rq->rt != rt_rq) + if (is_dl_group(rt_rq)) return; -#endif + if (rq->online && rt_rq->highest_prio.curr != prev_prio) cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr); } @@ -1107,12 +492,15 @@ void dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {} #endif /* CONFIG_SMP */ -#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED +#if defined(CONFIG_SMP) static void inc_rt_prio(struct rt_rq *rt_rq, int prio) { int prev_prio = rt_rq->highest_prio.curr; + if (is_dl_group(rt_rq)) + return; + if (prio < prev_prio) rt_rq->highest_prio.curr = prio; @@ -1124,6 +512,9 @@ dec_rt_prio(struct rt_rq *rt_rq, int prio) { int prev_prio = rt_rq->highest_prio.curr; + if (is_dl_group(rt_rq)) + return; + if (rt_rq->rt_nr_running) { WARN_ON(prio < prev_prio); @@ -1150,51 +541,12 @@ dec_rt_prio(struct rt_rq *rt_rq, int prio) static inline void inc_rt_prio(struct rt_rq *rt_rq, int prio) {} static inline void dec_rt_prio(struct rt_rq *rt_rq, int prio) {} -#endif /* CONFIG_SMP || CONFIG_RT_GROUP_SCHED */ - -#ifdef CONFIG_RT_GROUP_SCHED - -static void -inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) -{ - if (rt_se_boosted(rt_se)) - rt_rq->rt_nr_boosted++; - - if (rt_rq->tg) - start_rt_bandwidth(&rt_rq->tg->rt_bandwidth); -} - -static void -dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) -{ - if (rt_se_boosted(rt_se)) - rt_rq->rt_nr_boosted--; - - WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted); -} - -#else /* CONFIG_RT_GROUP_SCHED */ - -static void -inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) -{ - start_rt_bandwidth(&def_rt_bandwidth); -} - -static inline -void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {} - -#endif /* CONFIG_RT_GROUP_SCHED */ +#endif /* CONFIG_SMP && !CONFIG_RT_GROUP_SCHED */ static inline unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se) { - struct rt_rq *group_rq = group_rt_rq(rt_se); - - if (group_rq) - return group_rq->rt_nr_running; - else - return 1; + return 1; } static inline @@ -1206,20 +558,35 @@ void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) rt_rq->rt_nr_running += rt_se_nr_running(rt_se); inc_rt_prio(rt_rq, prio); + + if (is_dl_group(rt_rq)) { + struct sched_dl_entity *dl_se = dl_group_of(rt_rq); + + if (!dl_se->dl_throttled) + add_nr_running(rq_of_rt_rq(rt_rq), 1); + } else { + add_nr_running(rq_of_rt_rq(rt_rq), 1); + } + inc_rt_migration(rt_se, rt_rq); - inc_rt_group(rt_se, rt_rq); } static inline void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) { WARN_ON(!rt_prio(rt_se_prio(rt_se))); - WARN_ON(!rt_rq->rt_nr_running); rt_rq->rt_nr_running -= rt_se_nr_running(rt_se); dec_rt_prio(rt_rq, rt_se_prio(rt_se)); + if (is_dl_group(rt_rq)) { + struct sched_dl_entity *dl_se = dl_group_of(rt_rq); + + if (!dl_se->dl_throttled) + sub_nr_running(rq_of_rt_rq(rt_rq), 1); + } else { + sub_nr_running(rq_of_rt_rq(rt_rq), 1); + } dec_rt_migration(rt_se, rt_rq); - dec_rt_group(rt_se, rt_rq); } /* @@ -1245,25 +612,12 @@ static void __delist_rt_entity(struct sched_rt_entity *rt_se, struct rt_prio_arr rt_se->on_list = 0; } -static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) +static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) { struct rt_rq *rt_rq = rt_rq_of_se(rt_se); struct rt_prio_array *array = &rt_rq->active; - struct rt_rq *group_rq = group_rt_rq(rt_se); struct list_head *queue = array->queue + rt_se_prio(rt_se); - /* - * Don't enqueue the group if its throttled, or when empty. - * The latter is a consequence of the former when a child group - * get throttled and the current group doesn't have any other - * active members. - */ - if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running)) { - if (rt_se->on_list) - __delist_rt_entity(rt_se, array); - return; - } - if (move_entity(flags)) { WARN_ON_ONCE(rt_se->on_list); if (flags & ENQUEUE_HEAD) @@ -1279,7 +633,7 @@ static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flag inc_rt_tasks(rt_se, rt_rq); } -static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) +static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) { struct rt_rq *rt_rq = rt_rq_of_se(rt_se); struct rt_prio_array *array = &rt_rq->active; @@ -1293,52 +647,6 @@ static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flag dec_rt_tasks(rt_se, rt_rq); } -/* - * Because the prio of an upper entry depends on the lower - * entries, we must remove entries top - down. - */ -static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags) -{ - struct sched_rt_entity *back = NULL; - - for_each_sched_rt_entity(rt_se) { - rt_se->back = back; - back = rt_se; - } - - dequeue_top_rt_rq(rt_rq_of_se(back)); - - for (rt_se = back; rt_se; rt_se = rt_se->back) { - if (on_rt_rq(rt_se)) - __dequeue_rt_entity(rt_se, flags); - } -} - -static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) -{ - struct rq *rq = rq_of_rt_se(rt_se); - - dequeue_rt_stack(rt_se, flags); - for_each_sched_rt_entity(rt_se) - __enqueue_rt_entity(rt_se, flags); - enqueue_top_rt_rq(&rq->rt); -} - -static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags) -{ - struct rq *rq = rq_of_rt_se(rt_se); - - dequeue_rt_stack(rt_se, flags); - - for_each_sched_rt_entity(rt_se) { - struct rt_rq *rt_rq = group_rt_rq(rt_se); - - if (rt_rq && rt_rq->rt_nr_running) - __enqueue_rt_entity(rt_se, flags); - } - enqueue_top_rt_rq(&rq->rt); -} - /* * Adding/removing a task to/from a priority array: */ @@ -1346,26 +654,92 @@ static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags) { struct sched_rt_entity *rt_se = &p->rt; + struct rt_rq *rt_rq = rt_rq_of_se(rt_se); + if (is_dl_group(rt_rq)) { + BUG_ON( (rt_rq->rt_nr_running == 0) && + (!RB_EMPTY_NODE(&dl_group_of(rt_rq)->rb_node)) + ); + } + if (flags & ENQUEUE_WAKEUP) rt_se->timeout = 0; + /* Task arriving in an idle group of tasks. */ + if (is_dl_group(rt_rq) && (rt_rq->rt_nr_running == 0)) { + struct sched_dl_entity *dl_se = dl_group_of(rt_rq); + + if (!dl_se->dl_throttled) { + enqueue_dl_entity(dl_se, dl_se, flags); + resched_curr(rq); + } else { + BUG_ON(rt_throttled(p)); + /* + * rt_se's group was throttled while this task was + * sleeping/blocked/migrated. + * + * Do the transition towards OTHER now. + */ + if ((flags & ENQUEUE_REPLENISH) == 0) { + BUG_ON(on_rt_rq(rt_se)); + lockdep_assert_held(&rq->lock); + + list_add(&rt_se->cfs_throttled_task, + &rt_rq->cfs_throttled_tasks); + rt_se->cfs_throttle_rt_rq = rt_rq; + p->sched_class = &fair_sched_class; + p->prio = DEFAULT_PRIO; + if (task_current(rq, p)) + p->sched_class->set_curr_task(rq); +#ifdef CONFIG_FAIR_GROUP_SCHED + p->se.depth = p->se.parent ? p->se.parent->depth + 1 : 0; +#endif + + p->sched_class->enqueue_task(rq, p, flags); + if ((flags & ENQUEUE_RESTORE) == 0) + p->sched_class->switched_to(rq, p); + + return; + } + } + } + + BUG_ON(p->sched_class != &rt_sched_class); enqueue_rt_entity(rt_se, flags); walt_inc_cumulative_runnable_avg(rq, p); if (!task_current(rq, p) && tsk_nr_cpus_allowed(p) > 1) - enqueue_pushable_task(rq, p); + enqueue_pushable_task(rt_rq, p); } static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags) { struct sched_rt_entity *rt_se = &p->rt; + struct rt_rq *rt_rq = rt_rq_of_se(rt_se); - update_curr_rt(rq); + if (!rt_throttled(p)) + update_curr_rt(rq); + if (p->sched_class != &rt_sched_class) { + p->sched_class->dequeue_task(rq, p, flags); + return; + } dequeue_rt_entity(rt_se, flags); walt_dec_cumulative_runnable_avg(rq, p); - dequeue_pushable_task(rq, p); + dequeue_pushable_task(rt_rq_of_se(rt_se), p); + + /* Last task of the task group. */ + if (is_dl_group(rt_rq) && !rt_rq->rt_nr_running) { + struct sched_dl_entity *dl_se = dl_group_of(rt_rq); + +#ifndef CONFIG_RT_GROUP_SCHED + queue_pull_task(rq); +#endif + if (!rt_rq->rt_nr_running) { + dequeue_dl_entity(dl_se); + resched_curr(rq); + } + } } /* @@ -1384,6 +758,12 @@ requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head) else list_move_tail(&rt_se->run_list, queue); } + + if (is_dl_group(rt_rq)) { + BUG_ON( (rt_rq->rt_nr_running == 0) && + (!RB_EMPTY_NODE(&dl_group_of(rt_rq)->rb_node)) + ); + } } static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head) @@ -1391,9 +771,13 @@ static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head) struct sched_rt_entity *rt_se = &p->rt; struct rt_rq *rt_rq; - for_each_sched_rt_entity(rt_se) { - rt_rq = rt_rq_of_se(rt_se); - requeue_rt_entity(rt_rq, rt_se, head); + rt_rq = rt_rq_of_se(rt_se); + requeue_rt_entity(rt_rq, rt_se, head); + + if (is_dl_group(rt_rq)) { + BUG_ON( (rt_rq->rt_nr_running == 0) && + (!RB_EMPTY_NODE(&dl_group_of(rt_rq)->rb_node)) + ); } } @@ -1495,6 +879,30 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) */ static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags) { + if (is_dl_group(rt_rq_of_se(&p->rt)) && + is_dl_group(rt_rq_of_se(&rq->curr->rt))) { + struct sched_dl_entity *dl_se, *curr_dl_se; + + dl_se = dl_group_of(rt_rq_of_se(&p->rt)); + curr_dl_se = dl_group_of(rt_rq_of_se(&rq->curr->rt)); + + if (dl_entity_preempt(dl_se, curr_dl_se)) { + resched_curr(rq); + return; + } else if (!dl_entity_preempt(curr_dl_se, dl_se)) { + if (p->prio < rq->curr->prio) { + resched_curr(rq); + return; + } + } + return; + } else if (is_dl_group(rt_rq_of_se(&p->rt))) { + resched_curr(rq); + return; + } else if (is_dl_group(rt_rq_of_se(&rq->curr->rt))) { + return; + } + if (p->prio < rq->curr->prio) { resched_curr(rq); return; @@ -1553,7 +961,7 @@ static inline void sched_rt_update_capacity_req(struct rq *rq) #endif -static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq, +struct sched_rt_entity *pick_next_rt_entity(struct rq *rq, struct rt_rq *rt_rq) { struct rt_prio_array *array = &rt_rq->active; @@ -1576,11 +984,8 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq) struct task_struct *p; struct rt_rq *rt_rq = &rq->rt; - do { - rt_se = pick_next_rt_entity(rq, rt_rq); - BUG_ON(!rt_se); - rt_rq = group_rt_rq(rt_se); - } while (rt_rq); + rt_se = pick_next_rt_entity(rq, rt_rq); + BUG_ON(!rt_se); p = rt_task_of(rt_se); p->se.exec_start = rq_clock_task(rq); @@ -1621,7 +1026,7 @@ pick_next_task_rt(struct rq *rq, struct task_struct *prev, struct pin_cookie coo if (prev->sched_class == &rt_sched_class) update_curr_rt(rq); - if (!rt_rq->rt_queued) { + if (!rt_rq->rt_nr_running) { /* * The next task to be picked on this rq will have a lower * priority than rt tasks so we can spend some time to update @@ -1638,7 +1043,7 @@ pick_next_task_rt(struct rq *rq, struct task_struct *prev, struct pin_cookie coo p = _pick_next_task_rt(rq); /* The running task is never eligible for pushing */ - dequeue_pushable_task(rq, p); + dequeue_pushable_task(rt_rq, p); queue_push_tasks(rq); @@ -1647,6 +1052,8 @@ pick_next_task_rt(struct rq *rq, struct task_struct *prev, struct pin_cookie coo static void put_prev_task_rt(struct rq *rq, struct task_struct *p) { + struct rt_rq *rt_rq = rt_rq_of_se(&p->rt); + update_curr_rt(rq); /* @@ -1654,7 +1061,7 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p) * if it is still active */ if (on_rt_rq(&p->rt) && tsk_nr_cpus_allowed(p) > 1) - enqueue_pushable_task(rq, p); + enqueue_pushable_task(rt_rq, p); } #ifdef CONFIG_SMP @@ -1662,9 +1069,9 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p) /* Only try algorithms three times */ #define RT_MAX_TRIES 3 -static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu) +static int pick_rt_task(struct rt_rq *rt_rq, struct task_struct *p, int cpu) { - if (!task_running(rq, p) && + if (!task_running(rq_of_rt_rq(rt_rq), p) && cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) return 1; return 0; @@ -1674,16 +1081,17 @@ static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu) * Return the highest pushable rq's task, which is suitable to be executed * on the cpu, NULL otherwise */ -static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu) +static +struct task_struct *pick_highest_pushable_task(struct rt_rq *rt_rq, int cpu) { - struct plist_head *head = &rq->rt.pushable_tasks; + struct plist_head *head = &rt_rq->pushable_tasks; struct task_struct *p; - if (!has_pushable_tasks(rq)) + if (!has_pushable_tasks(rt_rq)) return NULL; plist_for_each_entry(p, head, pushable_tasks) { - if (pick_rt_task(rq, p, cpu)) + if (pick_rt_task(rt_rq, p, cpu)) return p; } @@ -1824,14 +1232,15 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq) return lowest_rq; } -static struct task_struct *pick_next_pushable_task(struct rq *rq) +static struct task_struct *pick_next_pushable_task(struct rt_rq *rt_rq) { + struct rq *rq = rq_of_rt_rq(rt_rq); struct task_struct *p; - if (!has_pushable_tasks(rq)) + if (!has_pushable_tasks(rt_rq)) return NULL; - p = plist_first_entry(&rq->rt.pushable_tasks, + p = plist_first_entry(&rt_rq->pushable_tasks, struct task_struct, pushable_tasks); BUG_ON(rq->cpu != task_cpu(p)); @@ -1858,7 +1267,7 @@ static int push_rt_task(struct rq *rq) if (!rq->rt.overloaded) return 0; - next_task = pick_next_pushable_task(rq); + next_task = pick_next_pushable_task(&rq->rt); if (!next_task) return 0; @@ -1893,7 +1302,7 @@ static int push_rt_task(struct rq *rq) * run-queue and is also still the next task eligible for * pushing. */ - task = pick_next_pushable_task(rq); + task = pick_next_pushable_task(&rq->rt); if (task_cpu(next_task) == rq->cpu && task == next_task) { /* * The task hasn't migrated, and is still the next @@ -2050,7 +1459,7 @@ static void try_to_push_tasks(void *arg) src_rq = rq_of_rt_rq(rt_rq); again: - if (has_pushable_tasks(rq)) { + if (has_pushable_tasks(&rq->rt)) { raw_spin_lock(&rq->lock); push_rt_task(rq); raw_spin_unlock(&rq->lock); @@ -2101,6 +1510,7 @@ static void pull_rt_task(struct rq *this_rq) int this_cpu = this_rq->cpu, cpu; bool resched = false; struct task_struct *p; + struct rt_rq *src_rt_rq; struct rq *src_rq; if (likely(!rt_overloaded(this_rq))) @@ -2124,6 +1534,7 @@ static void pull_rt_task(struct rq *this_rq) continue; src_rq = cpu_rq(cpu); + src_rt_rq = &src_rq->rt; /* * Don't bother taking the src_rq->lock if the next highest @@ -2132,7 +1543,7 @@ static void pull_rt_task(struct rq *this_rq) * logically higher, the src_rq will push this task away. * And if its going logically lower, we do not care */ - if (src_rq->rt.highest_prio.next >= + if (src_rt_rq->highest_prio.next >= this_rq->rt.highest_prio.curr) continue; @@ -2147,7 +1558,7 @@ static void pull_rt_task(struct rq *this_rq) * We can pull only a task, which is pushable * on its rq, and no others. */ - p = pick_highest_pushable_task(src_rq, this_cpu); + p = pick_highest_pushable_task(src_rt_rq, this_cpu); /* * Do we have an RT task that preempts @@ -2188,19 +1599,44 @@ static void pull_rt_task(struct rq *this_rq) resched_curr(this_rq); } +#ifdef CONFIG_RT_GROUP_SCHED +int group_push_rt_task(struct rt_rq *rt_rq) +{ + struct rq *rq = rq_of_rt_rq(rt_rq); + + if (is_dl_group(rt_rq)) + return 0; + + return push_rt_task(rq); +} + +void group_push_rt_tasks(struct rt_rq *rt_rq) +{ + while (group_push_rt_task(rt_rq)) + ; +} +#else +void group_push_rt_tasks(struct rt_rq *rt_rq) +{ + push_rt_tasks(rq_of_rt_rq(rt_rq)); +} +#endif + /* * If we are not running and we are not going to reschedule soon, we should * try to push tasks away now */ static void task_woken_rt(struct rq *rq, struct task_struct *p) { + struct rt_rq *rt_rq = rt_rq_of_se(&p->rt); + if (!task_running(rq, p) && !test_tsk_need_resched(rq->curr) && tsk_nr_cpus_allowed(p) > 1 && (dl_task(rq->curr) || rt_task(rq->curr)) && (tsk_nr_cpus_allowed(rq->curr) < 2 || rq->curr->prio <= p->prio)) - push_rt_tasks(rq); + group_push_rt_tasks(rt_rq); } /* Assumes rq->lock is held */ @@ -2209,8 +1645,6 @@ static void rq_online_rt(struct rq *rq) if (rq->rt.overloaded) rt_set_overload(rq); - __enable_runtime(rq); - cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr); } @@ -2220,8 +1654,6 @@ static void rq_offline_rt(struct rq *rq) if (rq->rt.overloaded) rt_clear_overload(rq); - __disable_runtime(rq); - cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID); } @@ -2231,6 +1663,10 @@ static void rq_offline_rt(struct rq *rq) */ static void switched_from_rt(struct rq *rq, struct task_struct *p) { + struct rt_rq *rt_rq = rt_rq_of_se(&p->rt); + + BUG_ON(task_cpu(p) != cpu_of(rq)); + /* * If there are other RT tasks then we will reschedule * and the scheduling of the other RT tasks will handle @@ -2238,10 +1674,12 @@ static void switched_from_rt(struct rq *rq, struct task_struct *p) * we may need to handle the pulling of RT tasks * now. */ - if (!task_on_rq_queued(p) || rq->rt.rt_nr_running) + if (!task_on_rq_queued(p) || rt_rq->rt_nr_running) return; +#ifndef CONFIG_RT_GROUP_SCHED queue_pull_task(rq); +#endif } void __init init_sched_rt_class(void) @@ -2271,8 +1709,16 @@ static void switched_to_rt(struct rq *rq, struct task_struct *p) */ if (task_on_rq_queued(p) && rq->curr != p) { #ifdef CONFIG_SMP +#ifndef CONFIG_RT_GROUP_SCHED if (tsk_nr_cpus_allowed(p) > 1 && rq->rt.overloaded) queue_push_tasks(rq); +#else + if (rt_rq_of_se(&p->rt)->overloaded) { + } else { + if (p->prio < rq->curr->prio) + resched_curr(rq); + } +#endif #else if (p->prio < rq->curr->prio) resched_curr(rq); @@ -2287,6 +1733,10 @@ static void switched_to_rt(struct rq *rq, struct task_struct *p) static void prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio) { +#ifdef CONFIG_SMP + struct rt_rq *rt_rq = rt_rq_of_se(&p->rt); +#endif + if (!task_on_rq_queued(p)) return; @@ -2297,13 +1747,14 @@ prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio) * may need to pull tasks to this runqueue. */ if (oldprio < p->prio) +#ifndef CONFIG_RT_GROUP_SCHED queue_pull_task(rq); - +#endif /* * If there's a higher priority task waiting to run * then reschedule. */ - if (p->prio > rq->rt.highest_prio.curr) + if (p->prio > rt_rq->highest_prio.curr) resched_curr(rq); #else /* For UP simply resched on drop of prio */ @@ -2348,6 +1799,14 @@ static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued) struct sched_rt_entity *rt_se = &p->rt; update_curr_rt(rq); +#ifdef CONFIG_RT_GROUP_SCHED + if (is_dl_group(&rq->rt)) { + struct sched_dl_entity *dl_se = dl_group_of(&rq->rt); + + if (hrtick_enabled(rq) && queued && dl_se->runtime > 0) + start_hrtick_dl(rq, dl_se); + } +#endif if (rq->rt.rt_nr_running) sched_rt_update_capacity_req(rq); @@ -2370,23 +1829,22 @@ static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued) * Requeue to the end of queue if we (and all of our ancestors) are not * the only element on the queue */ - for_each_sched_rt_entity(rt_se) { - if (rt_se->run_list.prev != rt_se->run_list.next) { - requeue_task_rt(rq, p, 0); - resched_curr(rq); - return; - } + if (rt_se->run_list.prev != rt_se->run_list.next) { + requeue_task_rt(rq, p, 0); + set_tsk_need_resched(p); + return; } } static void set_curr_task_rt(struct rq *rq) { struct task_struct *p = rq->curr; + struct rt_rq *rt_rq = rt_rq_of_se(&p->rt); p->se.exec_start = rq_clock_task(rq); /* The running task is never eligible for pushing */ - dequeue_pushable_task(rq, p); + dequeue_pushable_task(rt_rq, p); } static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task) @@ -2437,12 +1895,5 @@ extern void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq); void print_rt_stats(struct seq_file *m, int cpu) { - rt_rq_iter_t iter; - struct rt_rq *rt_rq; - - rcu_read_lock(); - for_each_rt_rq(rt_rq, iter, cpu_rq(cpu)) - print_rt_rq(m, cpu, rt_rq); - rcu_read_unlock(); } #endif /* CONFIG_SCHED_DEBUG */ diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index e35d273ea4d0b0..698f9436a9ff9a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -75,6 +75,8 @@ static inline void update_cpu_load_active(struct rq *this_rq) { } */ #define DL_SCALE (10) +unsigned long to_ratio(u64 period, u64 runtime); + /* * These are the 'tuning knobs' of the scheduler: */ @@ -135,15 +137,6 @@ struct rt_prio_array { struct list_head queue[MAX_RT_PRIO]; }; -struct rt_bandwidth { - /* nests inside the rq lock: */ - raw_spinlock_t rt_runtime_lock; - ktime_t rt_period; - u64 rt_runtime; - struct hrtimer rt_period_timer; - unsigned int rt_period_active; -}; - void __dl_clear_params(struct task_struct *p); /* @@ -200,12 +193,6 @@ void __dl_add(struct dl_bw *dl_b, u64 tsk_bw) dl_b->total_bw += tsk_bw; } -static inline -bool __dl_overflow(struct dl_bw *dl_b, int cpus, u64 old_bw, u64 new_bw) -{ - return dl_b->bw != -1 && - dl_b->bw * cpus < dl_b->total_bw - old_bw + new_bw; -} extern struct mutex sched_domains_mutex; @@ -253,10 +240,14 @@ struct task_group { #endif #ifdef CONFIG_RT_GROUP_SCHED - struct sched_rt_entity **rt_se; + /* + * The scheduling entities for the task group are managed as a single + * sched_dl_entity, each of them sharing the same dl_bandwidth. + */ + struct sched_dl_entity **dl_se; struct rt_rq **rt_rq; - struct rt_bandwidth rt_bandwidth; + struct dl_bandwidth dl_bandwidth; #endif struct rcu_head rcu; @@ -322,8 +313,8 @@ extern void unthrottle_cfs_rq(struct cfs_rq *cfs_rq); extern void free_rt_sched_group(struct task_group *tg); extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent); extern void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq, - struct sched_rt_entity *rt_se, int cpu, - struct sched_rt_entity *parent); + struct sched_dl_entity *rt_se, int cpu, + struct sched_dl_entity *parent); extern struct task_group *sched_create_group(struct task_group *parent); extern void sched_online_group(struct task_group *tg, @@ -343,6 +334,21 @@ struct cfs_bandwidth { }; #endif /* CONFIG_CGROUP_SCHED */ +static inline +bool __dl_overflow(struct dl_bw *dl_b, int cpus, u64 old_bw, u64 new_bw) +{ + u64 dl_groups_root = 0; + +#ifdef CONFIG_RT_GROUP_SCHED + dl_groups_root = to_ratio(root_task_group.dl_bandwidth.dl_period, + root_task_group.dl_bandwidth.dl_runtime); +#endif + return dl_b->bw != -1 && + dl_b->bw * cpus < dl_b->total_bw - old_bw + new_bw + + dl_groups_root * cpus; +} + + /* CFS-related fields in a runqueue */ struct cfs_rq { struct load_weight load; @@ -427,11 +433,6 @@ struct cfs_rq { #endif /* CONFIG_FAIR_GROUP_SCHED */ }; -static inline int rt_bandwidth_enabled(void) -{ - return sysctl_sched_rt_runtime >= 0; -} - /* RT IPI pull logic requires IRQ_WORK */ #ifdef CONFIG_IRQ_WORK # define HAVE_RT_PUSH_IPI @@ -451,7 +452,6 @@ struct rt_rq { #endif #ifdef CONFIG_SMP unsigned long rt_nr_migratory; - unsigned long rt_nr_total; int overloaded; struct plist_head pushable_tasks; #ifdef HAVE_RT_PUSH_IPI @@ -461,20 +461,13 @@ struct rt_rq { raw_spinlock_t push_lock; #endif #endif /* CONFIG_SMP */ - int rt_queued; - - int rt_throttled; - u64 rt_time; - u64 rt_runtime; - /* Nests inside the rq lock: */ - raw_spinlock_t rt_runtime_lock; #ifdef CONFIG_RT_GROUP_SCHED - unsigned long rt_nr_boosted; - - struct rq *rq; struct task_group *tg; + + struct list_head cfs_throttled_tasks; #endif + struct rq *rq; }; /* Deadline class' related fields in a runqueue */ @@ -485,6 +478,12 @@ struct dl_rq { unsigned long dl_nr_running; +#ifdef CONFIG_RT_GROUP_SCHED + unsigned long dl_nr_total; + struct rt_rq *rq_to_push_from; + struct rt_rq *rq_to_pull_to; +#endif + #ifdef CONFIG_SMP /* * Deadline values of the currently executing and the @@ -786,6 +785,10 @@ static inline void rq_clock_skip_update(struct rq *rq, bool skip) rq->clock_skip_update &= ~RQCF_REQ_SKIP; } +void __setprio_other(struct rq *rq, struct task_struct *p); +void __setprio_fifo(struct rq *rq, struct task_struct *p); +void cfs_unthrottle_rt_tasks(struct rt_rq *rt_rq); + #ifdef CONFIG_NUMA enum numa_topology_type { NUMA_DIRECT, @@ -993,7 +996,8 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) #ifdef CONFIG_RT_GROUP_SCHED p->rt.rt_rq = tg->rt_rq[cpu]; - p->rt.parent = tg->rt_se[cpu]; + p->rt.parent = tg->dl_se[cpu]; + p->dl.dl_rq = &cpu_rq(cpu)->dl; #endif } @@ -1398,15 +1402,10 @@ extern void init_sched_fair_class(void); extern void resched_curr(struct rq *rq); extern void resched_cpu(int cpu); -extern struct rt_bandwidth def_rt_bandwidth; -extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime); - extern struct dl_bandwidth def_dl_bandwidth; extern void init_dl_bandwidth(struct dl_bandwidth *dl_b, u64 period, u64 runtime); extern void init_dl_task_timer(struct sched_dl_entity *dl_se); -unsigned long to_ratio(u64 period, u64 runtime); - extern void init_entity_runnable_average(struct sched_entity *se); static inline void __add_nr_running(struct rq *rq, unsigned count) @@ -1439,6 +1438,7 @@ static inline void __add_nr_running(struct rq *rq, unsigned count) static inline void __sub_nr_running(struct rq *rq, unsigned count) { + BUG_ON(rq->nr_running < count); rq->nr_running -= count; } @@ -1911,6 +1911,71 @@ static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2) #endif +#ifdef CONFIG_RT_GROUP_SCHED +static inline int is_dl_group(struct rt_rq *rt_rq) +{ + return rt_rq->tg != &root_task_group; +} + +/* + * Return the scheduling entity of this group of tasks. + */ +static inline struct sched_dl_entity *dl_group_of(struct rt_rq *rt_rq) +{ + BUG_ON(!is_dl_group(rt_rq)); + + return rt_rq->tg->dl_se[cpu_of(rt_rq->rq)]; +} + +static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) +{ + return container_of(rt_se, struct task_struct, rt); +} + +static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) +{ + struct rt_rq *rt_rq = rt_se->rt_rq; + + return rt_rq->rq; +} + +static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) +{ + return rt_se->rt_rq; +} +#else +static inline int is_dl_group(struct rt_rq *rt_rq) +{ + return 0; +} + +static inline struct sched_dl_entity *dl_group_of(struct rt_rq *rt_rq) +{ + return NULL; +} + +static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) +{ + return container_of(rt_se, struct task_struct, rt); +} + +static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) +{ + struct task_struct *p = rt_task_of(rt_se); + + return task_rq(p); +} + +static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) +{ + struct rq *rq = rq_of_rt_se(rt_se); + + return &rq->rt; +} +#endif + + + extern struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq); extern struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq); @@ -2059,3 +2124,28 @@ static inline void cpufreq_update_this_cpu(struct rq *rq, unsigned int flags) {} #else /* arch_scale_freq_capacity */ #define arch_scale_freq_invariant() (false) #endif + +int group_pull_rt_task(struct rt_rq *rt_rq); +int group_push_rt_task(struct rt_rq *rt_rq); + +struct sched_rt_entity *pick_next_rt_entity(struct rq *rq, struct rt_rq *rt_rq); +#if defined(CONFIG_RT_GROUP_SCHED) && defined(CONFIG_SMP) +void dequeue_pushable_task(struct rt_rq *rt_rq, struct task_struct *p); +#else +static inline +void dequeue_pushable_task(struct rt_rq *rt_rq, struct task_struct *p) +{ +} +#endif + +int is_dl_group(struct rt_rq *rt_rq); + +void dequeue_dl_entity(struct sched_dl_entity *dl_se); + +void init_dl_timer(struct sched_dl_entity *dl_se); + +void enqueue_dl_entity(struct sched_dl_entity *dl_se, + struct sched_dl_entity *pi_se, int flags); +int dl_runtime_exceeded(struct sched_dl_entity *dl_se); +int start_dl_timer(struct sched_dl_entity *dl_se); +void start_hrtick_dl(struct rq *rq, struct sched_dl_entity *dl_se);