kernel/sched/isolation.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Housekeeping management. Manage the targets for routine code that can run on
   4  *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
   5  *
   6  * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
   7  * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
   8  *
   9  */
  10 #include "sched.h"
  11
  12 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
  13 EXPORT_SYMBOL_GPL(housekeeping_overridden);
  14 static cpumask_var_t housekeeping_mask;
  15 static unsigned int housekeeping_flags;
  16
  17 bool housekeeping_enabled(enum hk_flags flags)
  18 {
  19         return !!(housekeeping_flags & flags);
  20 }
  21 EXPORT_SYMBOL_GPL(housekeeping_enabled);
  22
  23 int housekeeping_any_cpu(enum hk_flags flags)
  24 {
  25         int cpu;
  26
  27         if (static_branch_unlikely(&housekeeping_overridden)) {
  28                 if (housekeeping_flags & flags) {
  29                         cpu = sched_numa_find_closest(housekeeping_mask, smp_processor_id());
  30                         if (cpu < nr_cpu_ids)
  31                                 return cpu;
  32
  33                         return cpumask_any_and(housekeeping_mask, cpu_online_mask);
  34                 }
  35         }
  36         return smp_processor_id();
  37 }
  38 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
  39
  40 const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
  41 {
  42         if (static_branch_unlikely(&housekeeping_overridden))
  43                 if (housekeeping_flags & flags)
  44                         return housekeeping_mask;
  45         return cpu_possible_mask;
  46 }
  47 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
  48
  49 void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
  50 {
  51         if (static_branch_unlikely(&housekeeping_overridden))
  52                 if (housekeeping_flags & flags)
  53                         set_cpus_allowed_ptr(t, housekeeping_mask);
  54 }
  55 EXPORT_SYMBOL_GPL(housekeeping_affine);
  56
  57 bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
  58 {
  59         if (static_branch_unlikely(&housekeeping_overridden))
  60                 if (housekeeping_flags & flags)
  61                         return cpumask_test_cpu(cpu, housekeeping_mask);
  62         return true;
  63 }
  64 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
  65
  66 void __init housekeeping_init(void)
  67 {
  68         if (!housekeeping_flags)
  69                 return;
  70
  71         static_branch_enable(&housekeeping_overridden);
  72
  73         if (housekeeping_flags & HK_FLAG_TICK)
  74                 sched_tick_offload_init();
  75
  76         /* We need at least one CPU to handle housekeeping work */
  77         WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
  78 }
  79
  80 static int __init housekeeping_setup(char *str, enum hk_flags flags)
  81 {
  82         cpumask_var_t non_housekeeping_mask;
  83         cpumask_var_t tmp;
  84         int err;
  85
  86         alloc_bootmem_cpumask_var(&non_housekeeping_mask);
  87         err = cpulist_parse(str, non_housekeeping_mask);
  88         if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
  89                 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
  90                 free_bootmem_cpumask_var(non_housekeeping_mask);
  91                 return 0;
  92         }
  93
  94         alloc_bootmem_cpumask_var(&tmp);
  95         if (!housekeeping_flags) {
  96                 alloc_bootmem_cpumask_var(&housekeeping_mask);
  97                 cpumask_andnot(housekeeping_mask,
  98                                cpu_possible_mask, non_housekeeping_mask);
  99
 100                 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
 101                 if (cpumask_empty(tmp)) {
 102                         pr_warn("Housekeeping: must include one present CPU, "
 103                                 "using boot CPU:%d\n", smp_processor_id());
 104                         __cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
 105                         __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
 106                 }
 107         } else {
 108                 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
 109                 if (cpumask_empty(tmp))
 110                         __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
 111                 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
 112                 if (!cpumask_equal(tmp, housekeeping_mask)) {
 113                         pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
 114                         free_bootmem_cpumask_var(tmp);
 115                         free_bootmem_cpumask_var(non_housekeeping_mask);
 116                         return 0;
 117                 }
 118         }
 119         free_bootmem_cpumask_var(tmp);
 120
 121         if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
 122                 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
 123                         tick_nohz_full_setup(non_housekeeping_mask);
 124                 } else {
 125                         pr_warn("Housekeeping: nohz unsupported."
 126                                 " Build with CONFIG_NO_HZ_FULL\n");
 127                         free_bootmem_cpumask_var(non_housekeeping_mask);
 128                         return 0;
 129                 }
 130         }
 131
 132         housekeeping_flags |= flags;
 133
 134         free_bootmem_cpumask_var(non_housekeeping_mask);
 135
 136         return 1;
 137 }
 138
 139 static int __init housekeeping_nohz_full_setup(char *str)
 140 {
 141         unsigned int flags;
 142
 143         flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
 144                 HK_FLAG_MISC | HK_FLAG_KTHREAD;
 145
 146         return housekeeping_setup(str, flags);
 147 }
 148 __setup("nohz_full=", housekeeping_nohz_full_setup);
 149
 150 static int __init housekeeping_isolcpus_setup(char *str)
 151 {
 152         unsigned int flags = 0;
 153         bool illegal = false;
 154         char *par;
 155         int len;
 156
 157         while (isalpha(*str)) {
 158                 if (!strncmp(str, "nohz,", 5)) {
 159                         str += 5;
 160                         flags |= HK_FLAG_TICK;
 161                         continue;
 162                 }
 163
 164                 if (!strncmp(str, "domain,", 7)) {
 165                         str += 7;
 166                         flags |= HK_FLAG_DOMAIN;
 167                         continue;
 168                 }
 169
 170                 if (!strncmp(str, "managed_irq,", 12)) {
 171                         str += 12;
 172                         flags |= HK_FLAG_MANAGED_IRQ;
 173                         continue;
 174                 }
 175
 176                 /*
 177                  * Skip unknown sub-parameter and validate that it is not
 178                  * containing an invalid character.
 179                  */
 180                 for (par = str, len = 0; *str && *str != ','; str++, len++) {
 181                         if (!isalpha(*str) && *str != '_')
 182                                 illegal = true;
 183                 }
 184
 185                 if (illegal) {
 186                         pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
 187                         return 0;
 188                 }
 189
 190                 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
 191                 str++;
 192         }
 193
 194         /* Default behaviour for isolcpus without flags */
 195         if (!flags)
 196                 flags |= HK_FLAG_DOMAIN;
 197
 198         return housekeeping_setup(str, flags);
 199 }
 200 __setup("isolcpus=", housekeeping_isolcpus_setup);