sched/deadline: Fix reset_on_fork reporting of DL tasks

[linux-2.6-microblaze.git] / kernel / sched / sched.h

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 4a898ab..75a5c12 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -227,6 +227,8 @@ static inline void update_avg(u64 *avg, u64 sample)
  */
 #define SCHED_FLAG_SUGOV       0x10000000
 
+#define SCHED_DL_FLAGS (SCHED_FLAG_RECLAIM | SCHED_FLAG_DL_OVERRUN | SCHED_FLAG_SUGOV)
+
 static inline bool dl_entity_is_special(struct sched_dl_entity *dl_se)
 {
 #ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL
@@ -366,6 +368,7 @@ struct cfs_bandwidth {
        ktime_t                 period;
        u64                     quota;
        u64                     runtime;
+       u64                     burst;
        s64                     hierarchical_quota;
 
        u8                      idle;
@@ -636,8 +639,8 @@ struct rt_rq {
        } highest_prio;
 #endif
 #ifdef CONFIG_SMP
-       unsigned long           rt_nr_migratory;
-       unsigned long           rt_nr_total;
+       unsigned int            rt_nr_migratory;
+       unsigned int            rt_nr_total;
        int                     overloaded;
        struct plist_head       pushable_tasks;
 
@@ -651,7 +654,7 @@ struct rt_rq {
        raw_spinlock_t          rt_runtime_lock;
 
 #ifdef CONFIG_RT_GROUP_SCHED
-       unsigned long           rt_nr_boosted;
+       unsigned int            rt_nr_boosted;
 
        struct rq               *rq;
        struct task_group       *tg;
@@ -668,7 +671,7 @@ struct dl_rq {
        /* runqueue is an rbtree, ordered by deadline */
        struct rb_root_cached   root;
 
-       unsigned long           dl_nr_running;
+       unsigned int            dl_nr_running;
 
 #ifdef CONFIG_SMP
        /*
@@ -682,7 +685,7 @@ struct dl_rq {
                u64             next;
        } earliest_dl;
 
-       unsigned long           dl_nr_migratory;
+       unsigned int            dl_nr_migratory;
        int                     overloaded;
 
        /*
@@ -960,7 +963,7 @@ struct rq {
         * one CPU and if it got migrated afterwards it may decrease
         * it on another CPU. Always updated under the runqueue lock:
         */
-       unsigned long           nr_uninterruptible;
+       unsigned int            nr_uninterruptible;
 
        struct task_struct __rcu        *curr;
        struct task_struct      *idle;
@@ -1022,6 +1025,9 @@ struct rq {
        u64                     idle_stamp;
        u64                     avg_idle;
 
+       unsigned long           wake_stamp;
+       u64                     wake_avg_idle;
+
        /* This is used to determine avg_idle's max value */
        u64                     max_idle_balance_cost;
 
@@ -1134,7 +1140,9 @@ static inline bool is_migration_disabled(struct task_struct *p)
 #endif
 }
 
+struct sched_group;
 #ifdef CONFIG_SCHED_CORE
+static inline struct cpumask *sched_group_span(struct sched_group *sg);
 
 DECLARE_STATIC_KEY_FALSE(__sched_core_enabled);
 
@@ -1170,6 +1178,79 @@ static inline raw_spinlock_t *__rq_lockp(struct rq *rq)
 
 bool cfs_prio_less(struct task_struct *a, struct task_struct *b, bool fi);
 
+/*
+ * Helpers to check if the CPU's core cookie matches with the task's cookie
+ * when core scheduling is enabled.
+ * A special case is that the task's cookie always matches with CPU's core
+ * cookie if the CPU is in an idle core.
+ */
+static inline bool sched_cpu_cookie_match(struct rq *rq, struct task_struct *p)
+{
+       /* Ignore cookie match if core scheduler is not enabled on the CPU. */
+       if (!sched_core_enabled(rq))
+               return true;
+
+       return rq->core->core_cookie == p->core_cookie;
+}
+
+static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p)
+{
+       bool idle_core = true;
+       int cpu;
+
+       /* Ignore cookie match if core scheduler is not enabled on the CPU. */
+       if (!sched_core_enabled(rq))
+               return true;
+
+       for_each_cpu(cpu, cpu_smt_mask(cpu_of(rq))) {
+               if (!available_idle_cpu(cpu)) {
+                       idle_core = false;
+                       break;
+               }
+       }
+
+       /*
+        * A CPU in an idle core is always the best choice for tasks with
+        * cookies.
+        */
+       return idle_core || rq->core->core_cookie == p->core_cookie;
+}
+
+static inline bool sched_group_cookie_match(struct rq *rq,
+                                           struct task_struct *p,
+                                           struct sched_group *group)
+{
+       int cpu;
+
+       /* Ignore cookie match if core scheduler is not enabled on the CPU. */
+       if (!sched_core_enabled(rq))
+               return true;
+
+       for_each_cpu_and(cpu, sched_group_span(group), p->cpus_ptr) {
+               if (sched_core_cookie_match(rq, p))
+                       return true;
+       }
+       return false;
+}
+
+extern void queue_core_balance(struct rq *rq);
+
+static inline bool sched_core_enqueued(struct task_struct *p)
+{
+       return !RB_EMPTY_NODE(&p->core_node);
+}
+
+extern void sched_core_enqueue(struct rq *rq, struct task_struct *p);
+extern void sched_core_dequeue(struct rq *rq, struct task_struct *p);
+
+extern void sched_core_get(void);
+extern void sched_core_put(void);
+
+extern unsigned long sched_core_alloc_cookie(void);
+extern void sched_core_put_cookie(unsigned long cookie);
+extern unsigned long sched_core_get_cookie(unsigned long cookie);
+extern unsigned long sched_core_update_cookie(struct task_struct *p, unsigned long cookie);
+
 #else /* !CONFIG_SCHED_CORE */
 
 static inline bool sched_core_enabled(struct rq *rq)
@@ -1192,6 +1273,26 @@ static inline raw_spinlock_t *__rq_lockp(struct rq *rq)
        return &rq->__lock;
 }
 
+static inline void queue_core_balance(struct rq *rq)
+{
+}
+
+static inline bool sched_cpu_cookie_match(struct rq *rq, struct task_struct *p)
+{
+       return true;
+}
+
+static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p)
+{
+       return true;
+}
+
+static inline bool sched_group_cookie_match(struct rq *rq,
+                                           struct task_struct *p,
+                                           struct sched_group *group)
+{
+       return true;
+}
 #endif /* CONFIG_SCHED_CORE */
 
 static inline void lockdep_assert_rq_held(struct rq *rq)
@@ -2286,6 +2387,21 @@ extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags);
 extern const_debug unsigned int sysctl_sched_nr_migrate;
 extern const_debug unsigned int sysctl_sched_migration_cost;
 
+#ifdef CONFIG_SCHED_DEBUG
+extern unsigned int sysctl_sched_latency;
+extern unsigned int sysctl_sched_min_granularity;
+extern unsigned int sysctl_sched_wakeup_granularity;
+extern int sysctl_resched_latency_warn_ms;
+extern int sysctl_resched_latency_warn_once;
+
+extern unsigned int sysctl_sched_tunable_scaling;
+
+extern unsigned int sysctl_numa_balancing_scan_delay;
+extern unsigned int sysctl_numa_balancing_scan_period_min;
+extern unsigned int sysctl_numa_balancing_scan_period_max;
+extern unsigned int sysctl_numa_balancing_scan_size;
+#endif
+
 #ifdef CONFIG_SCHED_HRTICK
 
 /*