arch/x86/kernel/apic/x2apic_cluster.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <linux/cpuhotplug.h>
   4 #include <linux/cpumask.h>
   5 #include <linux/slab.h>
   6 #include <linux/mm.h>
   7
   8 #include <asm/apic.h>
   9
  10 #include "local.h"
  11
  12 #define apic_cluster(apicid) ((apicid) >> 4)
  13
  14 /*
  15  * __x2apic_send_IPI_mask() possibly needs to read
  16  * x86_cpu_to_logical_apicid for all online cpus in a sequential way.
  17  * Using per cpu variable would cost one cache line per cpu.
  18  */
  19 static u32 *x86_cpu_to_logical_apicid __read_mostly;
  20
  21 static DEFINE_PER_CPU(cpumask_var_t, ipi_mask);
  22 static DEFINE_PER_CPU_READ_MOSTLY(struct cpumask *, cluster_masks);
  23
  24 static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
  25 {
  26         return x2apic_enabled();
  27 }
  28
  29 static void x2apic_send_IPI(int cpu, int vector)
  30 {
  31         u32 dest = x86_cpu_to_logical_apicid[cpu];
  32
  33         /* x2apic MSRs are special and need a special fence: */
  34         weak_wrmsr_fence();
  35         __x2apic_send_IPI_dest(dest, vector, APIC_DEST_LOGICAL);
  36 }
  37
  38 static void
  39 __x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
  40 {
  41         unsigned int cpu, clustercpu;
  42         struct cpumask *tmpmsk;
  43         unsigned long flags;
  44         u32 dest;
  45
  46         /* x2apic MSRs are special and need a special fence: */
  47         weak_wrmsr_fence();
  48         local_irq_save(flags);
  49
  50         tmpmsk = this_cpu_cpumask_var_ptr(ipi_mask);
  51         cpumask_copy(tmpmsk, mask);
  52         /* If IPI should not be sent to self, clear current CPU */
  53         if (apic_dest != APIC_DEST_ALLINC)
  54                 __cpumask_clear_cpu(smp_processor_id(), tmpmsk);
  55
  56         /* Collapse cpus in a cluster so a single IPI per cluster is sent */
  57         for_each_cpu(cpu, tmpmsk) {
  58                 struct cpumask *cmsk = per_cpu(cluster_masks, cpu);
  59
  60                 dest = 0;
  61                 for_each_cpu_and(clustercpu, tmpmsk, cmsk)
  62                         dest |= x86_cpu_to_logical_apicid[clustercpu];
  63
  64                 if (!dest)
  65                         continue;
  66
  67                 __x2apic_send_IPI_dest(dest, vector, APIC_DEST_LOGICAL);
  68                 /* Remove cluster CPUs from tmpmask */
  69                 cpumask_andnot(tmpmsk, tmpmsk, cmsk);
  70         }
  71
  72         local_irq_restore(flags);
  73 }
  74
  75 static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
  76 {
  77         __x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLINC);
  78 }
  79
  80 static void
  81 x2apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector)
  82 {
  83         __x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLBUT);
  84 }
  85
  86 static void x2apic_send_IPI_allbutself(int vector)
  87 {
  88         __x2apic_send_IPI_shorthand(vector, APIC_DEST_ALLBUT);
  89 }
  90
  91 static void x2apic_send_IPI_all(int vector)
  92 {
  93         __x2apic_send_IPI_shorthand(vector, APIC_DEST_ALLINC);
  94 }
  95
  96 static u32 x2apic_calc_apicid(unsigned int cpu)
  97 {
  98         return x86_cpu_to_logical_apicid[cpu];
  99 }
 100
 101 static void init_x2apic_ldr(void)
 102 {
 103         struct cpumask *cmsk = this_cpu_read(cluster_masks);
 104
 105         BUG_ON(!cmsk);
 106
 107         cpumask_set_cpu(smp_processor_id(), cmsk);
 108 }
 109
 110 /*
 111  * As an optimisation during boot, set the cluster_mask for all present
 112  * CPUs at once, to prevent each of them having to iterate over the others
 113  * to find the existing cluster_mask.
 114  */
 115 static void prefill_clustermask(struct cpumask *cmsk, unsigned int cpu, u32 cluster)
 116 {
 117         int cpu_i;
 118
 119         for_each_present_cpu(cpu_i) {
 120                 struct cpumask **cpu_cmsk = &per_cpu(cluster_masks, cpu_i);
 121                 u32 apicid = apic->cpu_present_to_apicid(cpu_i);
 122
 123                 if (apicid == BAD_APICID || cpu_i == cpu || apic_cluster(apicid) != cluster)
 124                         continue;
 125
 126                 if (WARN_ON_ONCE(*cpu_cmsk == cmsk))
 127                         continue;
 128
 129                 BUG_ON(*cpu_cmsk);
 130                 *cpu_cmsk = cmsk;
 131         }
 132 }
 133
 134 static int alloc_clustermask(unsigned int cpu, u32 cluster, int node)
 135 {
 136         struct cpumask *cmsk = NULL;
 137         unsigned int cpu_i;
 138
 139         /*
 140          * At boot time, the CPU present mask is stable. The cluster mask is
 141          * allocated for the first CPU in the cluster and propagated to all
 142          * present siblings in the cluster. If the cluster mask is already set
 143          * on entry to this function for a given CPU, there is nothing to do.
 144          */
 145         if (per_cpu(cluster_masks, cpu))
 146                 return 0;
 147
 148         if (system_state < SYSTEM_RUNNING)
 149                 goto alloc;
 150
 151         /*
 152          * On post boot hotplug for a CPU which was not present at boot time,
 153          * iterate over all possible CPUs (even those which are not present
 154          * any more) to find any existing cluster mask.
 155          */
 156         for_each_possible_cpu(cpu_i) {
 157                 u32 apicid = apic->cpu_present_to_apicid(cpu_i);
 158
 159                 if (apicid != BAD_APICID && apic_cluster(apicid) == cluster) {
 160                         cmsk = per_cpu(cluster_masks, cpu_i);
 161                         /*
 162                          * If the cluster is already initialized, just store
 163                          * the mask and return. There's no need to propagate.
 164                          */
 165                         if (cmsk) {
 166                                 per_cpu(cluster_masks, cpu) = cmsk;
 167                                 return 0;
 168                         }
 169                 }
 170         }
 171         /*
 172          * No CPU in the cluster has ever been initialized, so fall through to
 173          * the boot time code which will also populate the cluster mask for any
 174          * other CPU in the cluster which is (now) present.
 175          */
 176 alloc:
 177         cmsk = kzalloc_node(sizeof(*cmsk), GFP_KERNEL, node);
 178         if (!cmsk)
 179                 return -ENOMEM;
 180         per_cpu(cluster_masks, cpu) = cmsk;
 181         prefill_clustermask(cmsk, cpu, cluster);
 182
 183         return 0;
 184 }
 185
 186 static int x2apic_prepare_cpu(unsigned int cpu)
 187 {
 188         u32 phys_apicid = apic->cpu_present_to_apicid(cpu);
 189         u32 cluster = apic_cluster(phys_apicid);
 190         u32 logical_apicid = (cluster << 16) | (1 << (phys_apicid & 0xf));
 191
 192         x86_cpu_to_logical_apicid[cpu] = logical_apicid;
 193
 194         if (alloc_clustermask(cpu, cluster, cpu_to_node(cpu)) < 0)
 195                 return -ENOMEM;
 196         if (!zalloc_cpumask_var(&per_cpu(ipi_mask, cpu), GFP_KERNEL))
 197                 return -ENOMEM;
 198         return 0;
 199 }
 200
 201 static int x2apic_dead_cpu(unsigned int dead_cpu)
 202 {
 203         struct cpumask *cmsk = per_cpu(cluster_masks, dead_cpu);
 204
 205         if (cmsk)
 206                 cpumask_clear_cpu(dead_cpu, cmsk);
 207         free_cpumask_var(per_cpu(ipi_mask, dead_cpu));
 208         return 0;
 209 }
 210
 211 static int x2apic_cluster_probe(void)
 212 {
 213         u32 slots;
 214
 215         if (!x2apic_mode)
 216                 return 0;
 217
 218         slots = max_t(u32, L1_CACHE_BYTES/sizeof(u32), nr_cpu_ids);
 219         x86_cpu_to_logical_apicid = kcalloc(slots, sizeof(u32), GFP_KERNEL);
 220         if (!x86_cpu_to_logical_apicid)
 221                 return 0;
 222
 223         if (cpuhp_setup_state(CPUHP_X2APIC_PREPARE, "x86/x2apic:prepare",
 224                               x2apic_prepare_cpu, x2apic_dead_cpu) < 0) {
 225                 pr_err("Failed to register X2APIC_PREPARE\n");
 226                 kfree(x86_cpu_to_logical_apicid);
 227                 x86_cpu_to_logical_apicid = NULL;
 228                 return 0;
 229         }
 230         init_x2apic_ldr();
 231         return 1;
 232 }
 233
 234 static struct apic apic_x2apic_cluster __ro_after_init = {
 235
 236         .name                           = "cluster x2apic",
 237         .probe                          = x2apic_cluster_probe,
 238         .acpi_madt_oem_check            = x2apic_acpi_madt_oem_check,
 239         .apic_id_valid                  = x2apic_apic_id_valid,
 240         .apic_id_registered             = x2apic_apic_id_registered,
 241
 242         .delivery_mode                  = APIC_DELIVERY_MODE_FIXED,
 243         .dest_mode_logical              = true,
 244
 245         .disable_esr                    = 0,
 246
 247         .check_apicid_used              = NULL,
 248         .init_apic_ldr                  = init_x2apic_ldr,
 249         .ioapic_phys_id_map             = NULL,
 250         .setup_apic_routing             = NULL,
 251         .cpu_present_to_apicid          = default_cpu_present_to_apicid,
 252         .apicid_to_cpu_present          = NULL,
 253         .check_phys_apicid_present      = default_check_phys_apicid_present,
 254         .phys_pkg_id                    = x2apic_phys_pkg_id,
 255
 256         .get_apic_id                    = x2apic_get_apic_id,
 257         .set_apic_id                    = x2apic_set_apic_id,
 258
 259         .calc_dest_apicid               = x2apic_calc_apicid,
 260
 261         .send_IPI                       = x2apic_send_IPI,
 262         .send_IPI_mask                  = x2apic_send_IPI_mask,
 263         .send_IPI_mask_allbutself       = x2apic_send_IPI_mask_allbutself,
 264         .send_IPI_allbutself            = x2apic_send_IPI_allbutself,
 265         .send_IPI_all                   = x2apic_send_IPI_all,
 266         .send_IPI_self                  = x2apic_send_IPI_self,
 267
 268         .read                           = native_apic_msr_read,
 269         .write                          = native_apic_msr_write,
 270         .eoi_write                      = native_apic_msr_eoi_write,
 271         .icr_read                       = native_x2apic_icr_read,
 272         .icr_write                      = native_x2apic_icr_write,
 273         .wait_icr_idle                  = native_x2apic_wait_icr_idle,
 274         .safe_wait_icr_idle             = native_safe_x2apic_wait_icr_idle,
 275 };
 276
 277 apic_driver(apic_x2apic_cluster);