arch/x86/kernel/amd_nb.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Shared support code for AMD K8 northbridges and derivates.
   4  * Copyright 2006 Andi Kleen, SUSE Labs.
   5  */
   6
   7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   8
   9 #include <linux/types.h>
  10 #include <linux/slab.h>
  11 #include <linux/init.h>
  12 #include <linux/errno.h>
  13 #include <linux/export.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/pci_ids.h>
  16 #include <asm/amd_nb.h>
  17
  18 #define PCI_DEVICE_ID_AMD_17H_ROOT      0x1450
  19 #define PCI_DEVICE_ID_AMD_17H_M10H_ROOT 0x15d0
  20 #define PCI_DEVICE_ID_AMD_17H_M30H_ROOT 0x1480
  21 #define PCI_DEVICE_ID_AMD_17H_DF_F4     0x1464
  22 #define PCI_DEVICE_ID_AMD_17H_M10H_DF_F4 0x15ec
  23 #define PCI_DEVICE_ID_AMD_17H_M30H_DF_F4 0x1494
  24 #define PCI_DEVICE_ID_AMD_17H_M70H_DF_F4 0x1444
  25 #define PCI_DEVICE_ID_AMD_19H_DF_F4     0x1654
  26
  27 /* Protect the PCI config register pairs used for SMN and DF indirect access. */
  28 static DEFINE_MUTEX(smn_mutex);
  29
  30 static u32 *flush_words;
  31
  32 static const struct pci_device_id amd_root_ids[] = {
  33         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_ROOT) },
  34         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M10H_ROOT) },
  35         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M30H_ROOT) },
  36         {}
  37 };
  38
  39 #define PCI_DEVICE_ID_AMD_CNB17H_F4     0x1704
  40
  41 static const struct pci_device_id amd_nb_misc_ids[] = {
  42         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB_MISC) },
  43         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_10H_NB_MISC) },
  44         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_NB_F3) },
  45         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_M10H_F3) },
  46         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_M30H_NB_F3) },
  47         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_M60H_NB_F3) },
  48         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_NB_F3) },
  49         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_M30H_NB_F3) },
  50         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_DF_F3) },
  51         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M10H_DF_F3) },
  52         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M30H_DF_F3) },
  53         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CNB17H_F3) },
  54         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F3) },
  55         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_DF_F3) },
  56         {}
  57 };
  58
  59 static const struct pci_device_id amd_nb_link_ids[] = {
  60         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_NB_F4) },
  61         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_M30H_NB_F4) },
  62         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_M60H_NB_F4) },
  63         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_NB_F4) },
  64         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_M30H_NB_F4) },
  65         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_DF_F4) },
  66         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M10H_DF_F4) },
  67         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M30H_DF_F4) },
  68         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F4) },
  69         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_DF_F4) },
  70         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CNB17H_F4) },
  71         {}
  72 };
  73
  74 static const struct pci_device_id hygon_root_ids[] = {
  75         { PCI_DEVICE(PCI_VENDOR_ID_HYGON, PCI_DEVICE_ID_AMD_17H_ROOT) },
  76         {}
  77 };
  78
  79 static const struct pci_device_id hygon_nb_misc_ids[] = {
  80         { PCI_DEVICE(PCI_VENDOR_ID_HYGON, PCI_DEVICE_ID_AMD_17H_DF_F3) },
  81         {}
  82 };
  83
  84 static const struct pci_device_id hygon_nb_link_ids[] = {
  85         { PCI_DEVICE(PCI_VENDOR_ID_HYGON, PCI_DEVICE_ID_AMD_17H_DF_F4) },
  86         {}
  87 };
  88
  89 const struct amd_nb_bus_dev_range amd_nb_bus_dev_ranges[] __initconst = {
  90         { 0x00, 0x18, 0x20 },
  91         { 0xff, 0x00, 0x20 },
  92         { 0xfe, 0x00, 0x20 },
  93         { }
  94 };
  95
  96 static struct amd_northbridge_info amd_northbridges;
  97
  98 u16 amd_nb_num(void)
  99 {
 100         return amd_northbridges.num;
 101 }
 102 EXPORT_SYMBOL_GPL(amd_nb_num);
 103
 104 bool amd_nb_has_feature(unsigned int feature)
 105 {
 106         return ((amd_northbridges.flags & feature) == feature);
 107 }
 108 EXPORT_SYMBOL_GPL(amd_nb_has_feature);
 109
 110 struct amd_northbridge *node_to_amd_nb(int node)
 111 {
 112         return (node < amd_northbridges.num) ? &amd_northbridges.nb[node] : NULL;
 113 }
 114 EXPORT_SYMBOL_GPL(node_to_amd_nb);
 115
 116 static struct pci_dev *next_northbridge(struct pci_dev *dev,
 117                                         const struct pci_device_id *ids)
 118 {
 119         do {
 120                 dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev);
 121                 if (!dev)
 122                         break;
 123         } while (!pci_match_id(ids, dev));
 124         return dev;
 125 }
 126
 127 static int __amd_smn_rw(u16 node, u32 address, u32 *value, bool write)
 128 {
 129         struct pci_dev *root;
 130         int err = -ENODEV;
 131
 132         if (node >= amd_northbridges.num)
 133                 goto out;
 134
 135         root = node_to_amd_nb(node)->root;
 136         if (!root)
 137                 goto out;
 138
 139         mutex_lock(&smn_mutex);
 140
 141         err = pci_write_config_dword(root, 0x60, address);
 142         if (err) {
 143                 pr_warn("Error programming SMN address 0x%x.\n", address);
 144                 goto out_unlock;
 145         }
 146
 147         err = (write ? pci_write_config_dword(root, 0x64, *value)
 148                      : pci_read_config_dword(root, 0x64, value));
 149         if (err)
 150                 pr_warn("Error %s SMN address 0x%x.\n",
 151                         (write ? "writing to" : "reading from"), address);
 152
 153 out_unlock:
 154         mutex_unlock(&smn_mutex);
 155
 156 out:
 157         return err;
 158 }
 159
 160 int amd_smn_read(u16 node, u32 address, u32 *value)
 161 {
 162         return __amd_smn_rw(node, address, value, false);
 163 }
 164 EXPORT_SYMBOL_GPL(amd_smn_read);
 165
 166 int amd_smn_write(u16 node, u32 address, u32 value)
 167 {
 168         return __amd_smn_rw(node, address, &value, true);
 169 }
 170 EXPORT_SYMBOL_GPL(amd_smn_write);
 171
 172 /*
 173  * Data Fabric Indirect Access uses FICAA/FICAD.
 174  *
 175  * Fabric Indirect Configuration Access Address (FICAA): Constructed based
 176  * on the device's Instance Id and the PCI function and register offset of
 177  * the desired register.
 178  *
 179  * Fabric Indirect Configuration Access Data (FICAD): There are FICAD LO
 180  * and FICAD HI registers but so far we only need the LO register.
 181  */
 182 int amd_df_indirect_read(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo)
 183 {
 184         struct pci_dev *F4;
 185         u32 ficaa;
 186         int err = -ENODEV;
 187
 188         if (node >= amd_northbridges.num)
 189                 goto out;
 190
 191         F4 = node_to_amd_nb(node)->link;
 192         if (!F4)
 193                 goto out;
 194
 195         ficaa  = 1;
 196         ficaa |= reg & 0x3FC;
 197         ficaa |= (func & 0x7) << 11;
 198         ficaa |= instance_id << 16;
 199
 200         mutex_lock(&smn_mutex);
 201
 202         err = pci_write_config_dword(F4, 0x5C, ficaa);
 203         if (err) {
 204                 pr_warn("Error writing DF Indirect FICAA, FICAA=0x%x\n", ficaa);
 205                 goto out_unlock;
 206         }
 207
 208         err = pci_read_config_dword(F4, 0x98, lo);
 209         if (err)
 210                 pr_warn("Error reading DF Indirect FICAD LO, FICAA=0x%x.\n", ficaa);
 211
 212 out_unlock:
 213         mutex_unlock(&smn_mutex);
 214
 215 out:
 216         return err;
 217 }
 218 EXPORT_SYMBOL_GPL(amd_df_indirect_read);
 219
 220 int amd_cache_northbridges(void)
 221 {
 222         const struct pci_device_id *misc_ids = amd_nb_misc_ids;
 223         const struct pci_device_id *link_ids = amd_nb_link_ids;
 224         const struct pci_device_id *root_ids = amd_root_ids;
 225         struct pci_dev *root, *misc, *link;
 226         struct amd_northbridge *nb;
 227         u16 roots_per_misc = 0;
 228         u16 misc_count = 0;
 229         u16 root_count = 0;
 230         u16 i, j;
 231
 232         if (amd_northbridges.num)
 233                 return 0;
 234
 235         if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
 236                 root_ids = hygon_root_ids;
 237                 misc_ids = hygon_nb_misc_ids;
 238                 link_ids = hygon_nb_link_ids;
 239         }
 240
 241         misc = NULL;
 242         while ((misc = next_northbridge(misc, misc_ids)) != NULL)
 243                 misc_count++;
 244
 245         if (!misc_count)
 246                 return -ENODEV;
 247
 248         root = NULL;
 249         while ((root = next_northbridge(root, root_ids)) != NULL)
 250                 root_count++;
 251
 252         if (root_count) {
 253                 roots_per_misc = root_count / misc_count;
 254
 255                 /*
 256                  * There should be _exactly_ N roots for each DF/SMN
 257                  * interface.
 258                  */
 259                 if (!roots_per_misc || (root_count % roots_per_misc)) {
 260                         pr_info("Unsupported AMD DF/PCI configuration found\n");
 261                         return -ENODEV;
 262                 }
 263         }
 264
 265         nb = kcalloc(misc_count, sizeof(struct amd_northbridge), GFP_KERNEL);
 266         if (!nb)
 267                 return -ENOMEM;
 268
 269         amd_northbridges.nb = nb;
 270         amd_northbridges.num = misc_count;
 271
 272         link = misc = root = NULL;
 273         for (i = 0; i < amd_northbridges.num; i++) {
 274                 node_to_amd_nb(i)->root = root =
 275                         next_northbridge(root, root_ids);
 276                 node_to_amd_nb(i)->misc = misc =
 277                         next_northbridge(misc, misc_ids);
 278                 node_to_amd_nb(i)->link = link =
 279                         next_northbridge(link, link_ids);
 280
 281                 /*
 282                  * If there are more PCI root devices than data fabric/
 283                  * system management network interfaces, then the (N)
 284                  * PCI roots per DF/SMN interface are functionally the
 285                  * same (for DF/SMN access) and N-1 are redundant.  N-1
 286                  * PCI roots should be skipped per DF/SMN interface so
 287                  * the following DF/SMN interfaces get mapped to
 288                  * correct PCI roots.
 289                  */
 290                 for (j = 1; j < roots_per_misc; j++)
 291                         root = next_northbridge(root, root_ids);
 292         }
 293
 294         if (amd_gart_present())
 295                 amd_northbridges.flags |= AMD_NB_GART;
 296
 297         /*
 298          * Check for L3 cache presence.
 299          */
 300         if (!cpuid_edx(0x80000006))
 301                 return 0;
 302
 303         /*
 304          * Some CPU families support L3 Cache Index Disable. There are some
 305          * limitations because of E382 and E388 on family 0x10.
 306          */
 307         if (boot_cpu_data.x86 == 0x10 &&
 308             boot_cpu_data.x86_model >= 0x8 &&
 309             (boot_cpu_data.x86_model > 0x9 ||
 310              boot_cpu_data.x86_stepping >= 0x1))
 311                 amd_northbridges.flags |= AMD_NB_L3_INDEX_DISABLE;
 312
 313         if (boot_cpu_data.x86 == 0x15)
 314                 amd_northbridges.flags |= AMD_NB_L3_INDEX_DISABLE;
 315
 316         /* L3 cache partitioning is supported on family 0x15 */
 317         if (boot_cpu_data.x86 == 0x15)
 318                 amd_northbridges.flags |= AMD_NB_L3_PARTITIONING;
 319
 320         return 0;
 321 }
 322 EXPORT_SYMBOL_GPL(amd_cache_northbridges);
 323
 324 /*
 325  * Ignores subdevice/subvendor but as far as I can figure out
 326  * they're useless anyways
 327  */
 328 bool __init early_is_amd_nb(u32 device)
 329 {
 330         const struct pci_device_id *misc_ids = amd_nb_misc_ids;
 331         const struct pci_device_id *id;
 332         u32 vendor = device & 0xffff;
 333
 334         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
 335             boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
 336                 return false;
 337
 338         if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
 339                 misc_ids = hygon_nb_misc_ids;
 340
 341         device >>= 16;
 342         for (id = misc_ids; id->vendor; id++)
 343                 if (vendor == id->vendor && device == id->device)
 344                         return true;
 345         return false;
 346 }
 347
 348 struct resource *amd_get_mmconfig_range(struct resource *res)
 349 {
 350         u32 address;
 351         u64 base, msr;
 352         unsigned int segn_busn_bits;
 353
 354         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
 355             boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
 356                 return NULL;
 357
 358         /* assume all cpus from fam10h have mmconfig */
 359         if (boot_cpu_data.x86 < 0x10)
 360                 return NULL;
 361
 362         address = MSR_FAM10H_MMIO_CONF_BASE;
 363         rdmsrl(address, msr);
 364
 365         /* mmconfig is not enabled */
 366         if (!(msr & FAM10H_MMIO_CONF_ENABLE))
 367                 return NULL;
 368
 369         base = msr & (FAM10H_MMIO_CONF_BASE_MASK<<FAM10H_MMIO_CONF_BASE_SHIFT);
 370
 371         segn_busn_bits = (msr >> FAM10H_MMIO_CONF_BUSRANGE_SHIFT) &
 372                          FAM10H_MMIO_CONF_BUSRANGE_MASK;
 373
 374         res->flags = IORESOURCE_MEM;
 375         res->start = base;
 376         res->end = base + (1ULL<<(segn_busn_bits + 20)) - 1;
 377         return res;
 378 }
 379
 380 int amd_get_subcaches(int cpu)
 381 {
 382         struct pci_dev *link = node_to_amd_nb(amd_get_nb_id(cpu))->link;
 383         unsigned int mask;
 384
 385         if (!amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
 386                 return 0;
 387
 388         pci_read_config_dword(link, 0x1d4, &mask);
 389
 390         return (mask >> (4 * cpu_data(cpu).cpu_core_id)) & 0xf;
 391 }
 392
 393 int amd_set_subcaches(int cpu, unsigned long mask)
 394 {
 395         static unsigned int reset, ban;
 396         struct amd_northbridge *nb = node_to_amd_nb(amd_get_nb_id(cpu));
 397         unsigned int reg;
 398         int cuid;
 399
 400         if (!amd_nb_has_feature(AMD_NB_L3_PARTITIONING) || mask > 0xf)
 401                 return -EINVAL;
 402
 403         /* if necessary, collect reset state of L3 partitioning and BAN mode */
 404         if (reset == 0) {
 405                 pci_read_config_dword(nb->link, 0x1d4, &reset);
 406                 pci_read_config_dword(nb->misc, 0x1b8, &ban);
 407                 ban &= 0x180000;
 408         }
 409
 410         /* deactivate BAN mode if any subcaches are to be disabled */
 411         if (mask != 0xf) {
 412                 pci_read_config_dword(nb->misc, 0x1b8, &reg);
 413                 pci_write_config_dword(nb->misc, 0x1b8, reg & ~0x180000);
 414         }
 415
 416         cuid = cpu_data(cpu).cpu_core_id;
 417         mask <<= 4 * cuid;
 418         mask |= (0xf ^ (1 << cuid)) << 26;
 419
 420         pci_write_config_dword(nb->link, 0x1d4, mask);
 421
 422         /* reset BAN mode if L3 partitioning returned to reset state */
 423         pci_read_config_dword(nb->link, 0x1d4, &reg);
 424         if (reg == reset) {
 425                 pci_read_config_dword(nb->misc, 0x1b8, &reg);
 426                 reg &= ~0x180000;
 427                 pci_write_config_dword(nb->misc, 0x1b8, reg | ban);
 428         }
 429
 430         return 0;
 431 }
 432
 433 static void amd_cache_gart(void)
 434 {
 435         u16 i;
 436
 437         if (!amd_nb_has_feature(AMD_NB_GART))
 438                 return;
 439
 440         flush_words = kmalloc_array(amd_northbridges.num, sizeof(u32), GFP_KERNEL);
 441         if (!flush_words) {
 442                 amd_northbridges.flags &= ~AMD_NB_GART;
 443                 pr_notice("Cannot initialize GART flush words, GART support disabled\n");
 444                 return;
 445         }
 446
 447         for (i = 0; i != amd_northbridges.num; i++)
 448                 pci_read_config_dword(node_to_amd_nb(i)->misc, 0x9c, &flush_words[i]);
 449 }
 450
 451 void amd_flush_garts(void)
 452 {
 453         int flushed, i;
 454         unsigned long flags;
 455         static DEFINE_SPINLOCK(gart_lock);
 456
 457         if (!amd_nb_has_feature(AMD_NB_GART))
 458                 return;
 459
 460         /*
 461          * Avoid races between AGP and IOMMU. In theory it's not needed
 462          * but I'm not sure if the hardware won't lose flush requests
 463          * when another is pending. This whole thing is so expensive anyways
 464          * that it doesn't matter to serialize more. -AK
 465          */
 466         spin_lock_irqsave(&gart_lock, flags);
 467         flushed = 0;
 468         for (i = 0; i < amd_northbridges.num; i++) {
 469                 pci_write_config_dword(node_to_amd_nb(i)->misc, 0x9c,
 470                                        flush_words[i] | 1);
 471                 flushed++;
 472         }
 473         for (i = 0; i < amd_northbridges.num; i++) {
 474                 u32 w;
 475                 /* Make sure the hardware actually executed the flush*/
 476                 for (;;) {
 477                         pci_read_config_dword(node_to_amd_nb(i)->misc,
 478                                               0x9c, &w);
 479                         if (!(w & 1))
 480                                 break;
 481                         cpu_relax();
 482                 }
 483         }
 484         spin_unlock_irqrestore(&gart_lock, flags);
 485         if (!flushed)
 486                 pr_notice("nothing to flush?\n");
 487 }
 488 EXPORT_SYMBOL_GPL(amd_flush_garts);
 489
 490 static void __fix_erratum_688(void *info)
 491 {
 492 #define MSR_AMD64_IC_CFG 0xC0011021
 493
 494         msr_set_bit(MSR_AMD64_IC_CFG, 3);
 495         msr_set_bit(MSR_AMD64_IC_CFG, 14);
 496 }
 497
 498 /* Apply erratum 688 fix so machines without a BIOS fix work. */
 499 static __init void fix_erratum_688(void)
 500 {
 501         struct pci_dev *F4;
 502         u32 val;
 503
 504         if (boot_cpu_data.x86 != 0x14)
 505                 return;
 506
 507         if (!amd_northbridges.num)
 508                 return;
 509
 510         F4 = node_to_amd_nb(0)->link;
 511         if (!F4)
 512                 return;
 513
 514         if (pci_read_config_dword(F4, 0x164, &val))
 515                 return;
 516
 517         if (val & BIT(2))
 518                 return;
 519
 520         on_each_cpu(__fix_erratum_688, NULL, 0);
 521
 522         pr_info("x86/cpu/AMD: CPU erratum 688 worked around\n");
 523 }
 524
 525 static __init int init_amd_nbs(void)
 526 {
 527         amd_cache_northbridges();
 528         amd_cache_gart();
 529
 530         fix_erratum_688();
 531
 532         return 0;
 533 }
 534
 535 /* This has to go after the PCI subsystem */
 536 fs_initcall(init_amd_nbs);