arch/x86/kernel/amd_nb.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Shared support code for AMD K8 northbridges and derivates.
   4  * Copyright 2006 Andi Kleen, SUSE Labs.
   5  */
   6
   7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   8
   9 #include <linux/types.h>
  10 #include <linux/slab.h>
  11 #include <linux/init.h>
  12 #include <linux/errno.h>
  13 #include <linux/export.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/pci_ids.h>
  16 #include <asm/amd_nb.h>
  17
  18 #define PCI_DEVICE_ID_AMD_17H_ROOT      0x1450
  19 #define PCI_DEVICE_ID_AMD_17H_M10H_ROOT 0x15d0
  20 #define PCI_DEVICE_ID_AMD_17H_M30H_ROOT 0x1480
  21 #define PCI_DEVICE_ID_AMD_17H_DF_F4     0x1464
  22 #define PCI_DEVICE_ID_AMD_17H_M10H_DF_F4 0x15ec
  23 #define PCI_DEVICE_ID_AMD_17H_M30H_DF_F4 0x1494
  24 #define PCI_DEVICE_ID_AMD_17H_M70H_DF_F4 0x1444
  25 #define PCI_DEVICE_ID_AMD_19H_DF_F4     0x1654
  26
  27 /* Protect the PCI config register pairs used for SMN and DF indirect access. */
  28 static DEFINE_MUTEX(smn_mutex);
  29
  30 static u32 *flush_words;
  31
  32 static const struct pci_device_id amd_root_ids[] = {
  33         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_ROOT) },
  34         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M10H_ROOT) },
  35         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M30H_ROOT) },
  36         {}
  37 };
  38
  39
  40 #define PCI_DEVICE_ID_AMD_CNB17H_F4     0x1704
  41
  42 const struct pci_device_id amd_nb_misc_ids[] = {
  43         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB_MISC) },
  44         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_10H_NB_MISC) },
  45         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_NB_F3) },
  46         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_M10H_F3) },
  47         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_M30H_NB_F3) },
  48         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_M60H_NB_F3) },
  49         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_NB_F3) },
  50         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_M30H_NB_F3) },
  51         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_DF_F3) },
  52         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M10H_DF_F3) },
  53         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M30H_DF_F3) },
  54         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CNB17H_F3) },
  55         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F3) },
  56         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_DF_F3) },
  57         {}
  58 };
  59 EXPORT_SYMBOL_GPL(amd_nb_misc_ids);
  60
  61 static const struct pci_device_id amd_nb_link_ids[] = {
  62         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_NB_F4) },
  63         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_M30H_NB_F4) },
  64         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_M60H_NB_F4) },
  65         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_NB_F4) },
  66         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_M30H_NB_F4) },
  67         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_DF_F4) },
  68         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M10H_DF_F4) },
  69         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M30H_DF_F4) },
  70         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F4) },
  71         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_DF_F4) },
  72         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CNB17H_F4) },
  73         {}
  74 };
  75
  76 static const struct pci_device_id hygon_root_ids[] = {
  77         { PCI_DEVICE(PCI_VENDOR_ID_HYGON, PCI_DEVICE_ID_AMD_17H_ROOT) },
  78         {}
  79 };
  80
  81 static const struct pci_device_id hygon_nb_misc_ids[] = {
  82         { PCI_DEVICE(PCI_VENDOR_ID_HYGON, PCI_DEVICE_ID_AMD_17H_DF_F3) },
  83         {}
  84 };
  85
  86 static const struct pci_device_id hygon_nb_link_ids[] = {
  87         { PCI_DEVICE(PCI_VENDOR_ID_HYGON, PCI_DEVICE_ID_AMD_17H_DF_F4) },
  88         {}
  89 };
  90
  91 const struct amd_nb_bus_dev_range amd_nb_bus_dev_ranges[] __initconst = {
  92         { 0x00, 0x18, 0x20 },
  93         { 0xff, 0x00, 0x20 },
  94         { 0xfe, 0x00, 0x20 },
  95         { }
  96 };
  97
  98 static struct amd_northbridge_info amd_northbridges;
  99
 100 u16 amd_nb_num(void)
 101 {
 102         return amd_northbridges.num;
 103 }
 104 EXPORT_SYMBOL_GPL(amd_nb_num);
 105
 106 bool amd_nb_has_feature(unsigned int feature)
 107 {
 108         return ((amd_northbridges.flags & feature) == feature);
 109 }
 110 EXPORT_SYMBOL_GPL(amd_nb_has_feature);
 111
 112 struct amd_northbridge *node_to_amd_nb(int node)
 113 {
 114         return (node < amd_northbridges.num) ? &amd_northbridges.nb[node] : NULL;
 115 }
 116 EXPORT_SYMBOL_GPL(node_to_amd_nb);
 117
 118 static struct pci_dev *next_northbridge(struct pci_dev *dev,
 119                                         const struct pci_device_id *ids)
 120 {
 121         do {
 122                 dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev);
 123                 if (!dev)
 124                         break;
 125         } while (!pci_match_id(ids, dev));
 126         return dev;
 127 }
 128
 129 static int __amd_smn_rw(u16 node, u32 address, u32 *value, bool write)
 130 {
 131         struct pci_dev *root;
 132         int err = -ENODEV;
 133
 134         if (node >= amd_northbridges.num)
 135                 goto out;
 136
 137         root = node_to_amd_nb(node)->root;
 138         if (!root)
 139                 goto out;
 140
 141         mutex_lock(&smn_mutex);
 142
 143         err = pci_write_config_dword(root, 0x60, address);
 144         if (err) {
 145                 pr_warn("Error programming SMN address 0x%x.\n", address);
 146                 goto out_unlock;
 147         }
 148
 149         err = (write ? pci_write_config_dword(root, 0x64, *value)
 150                      : pci_read_config_dword(root, 0x64, value));
 151         if (err)
 152                 pr_warn("Error %s SMN address 0x%x.\n",
 153                         (write ? "writing to" : "reading from"), address);
 154
 155 out_unlock:
 156         mutex_unlock(&smn_mutex);
 157
 158 out:
 159         return err;
 160 }
 161
 162 int amd_smn_read(u16 node, u32 address, u32 *value)
 163 {
 164         return __amd_smn_rw(node, address, value, false);
 165 }
 166 EXPORT_SYMBOL_GPL(amd_smn_read);
 167
 168 int amd_smn_write(u16 node, u32 address, u32 value)
 169 {
 170         return __amd_smn_rw(node, address, &value, true);
 171 }
 172 EXPORT_SYMBOL_GPL(amd_smn_write);
 173
 174 /*
 175  * Data Fabric Indirect Access uses FICAA/FICAD.
 176  *
 177  * Fabric Indirect Configuration Access Address (FICAA): Constructed based
 178  * on the device's Instance Id and the PCI function and register offset of
 179  * the desired register.
 180  *
 181  * Fabric Indirect Configuration Access Data (FICAD): There are FICAD LO
 182  * and FICAD HI registers but so far we only need the LO register.
 183  */
 184 int amd_df_indirect_read(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo)
 185 {
 186         struct pci_dev *F4;
 187         u32 ficaa;
 188         int err = -ENODEV;
 189
 190         if (node >= amd_northbridges.num)
 191                 goto out;
 192
 193         F4 = node_to_amd_nb(node)->link;
 194         if (!F4)
 195                 goto out;
 196
 197         ficaa  = 1;
 198         ficaa |= reg & 0x3FC;
 199         ficaa |= (func & 0x7) << 11;
 200         ficaa |= instance_id << 16;
 201
 202         mutex_lock(&smn_mutex);
 203
 204         err = pci_write_config_dword(F4, 0x5C, ficaa);
 205         if (err) {
 206                 pr_warn("Error writing DF Indirect FICAA, FICAA=0x%x\n", ficaa);
 207                 goto out_unlock;
 208         }
 209
 210         err = pci_read_config_dword(F4, 0x98, lo);
 211         if (err)
 212                 pr_warn("Error reading DF Indirect FICAD LO, FICAA=0x%x.\n", ficaa);
 213
 214 out_unlock:
 215         mutex_unlock(&smn_mutex);
 216
 217 out:
 218         return err;
 219 }
 220 EXPORT_SYMBOL_GPL(amd_df_indirect_read);
 221
 222 int amd_cache_northbridges(void)
 223 {
 224         const struct pci_device_id *misc_ids = amd_nb_misc_ids;
 225         const struct pci_device_id *link_ids = amd_nb_link_ids;
 226         const struct pci_device_id *root_ids = amd_root_ids;
 227         struct pci_dev *root, *misc, *link;
 228         struct amd_northbridge *nb;
 229         u16 roots_per_misc = 0;
 230         u16 misc_count = 0;
 231         u16 root_count = 0;
 232         u16 i, j;
 233
 234         if (amd_northbridges.num)
 235                 return 0;
 236
 237         if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
 238                 root_ids = hygon_root_ids;
 239                 misc_ids = hygon_nb_misc_ids;
 240                 link_ids = hygon_nb_link_ids;
 241         }
 242
 243         misc = NULL;
 244         while ((misc = next_northbridge(misc, misc_ids)) != NULL)
 245                 misc_count++;
 246
 247         if (!misc_count)
 248                 return -ENODEV;
 249
 250         root = NULL;
 251         while ((root = next_northbridge(root, root_ids)) != NULL)
 252                 root_count++;
 253
 254         if (root_count) {
 255                 roots_per_misc = root_count / misc_count;
 256
 257                 /*
 258                  * There should be _exactly_ N roots for each DF/SMN
 259                  * interface.
 260                  */
 261                 if (!roots_per_misc || (root_count % roots_per_misc)) {
 262                         pr_info("Unsupported AMD DF/PCI configuration found\n");
 263                         return -ENODEV;
 264                 }
 265         }
 266
 267         nb = kcalloc(misc_count, sizeof(struct amd_northbridge), GFP_KERNEL);
 268         if (!nb)
 269                 return -ENOMEM;
 270
 271         amd_northbridges.nb = nb;
 272         amd_northbridges.num = misc_count;
 273
 274         link = misc = root = NULL;
 275         for (i = 0; i < amd_northbridges.num; i++) {
 276                 node_to_amd_nb(i)->root = root =
 277                         next_northbridge(root, root_ids);
 278                 node_to_amd_nb(i)->misc = misc =
 279                         next_northbridge(misc, misc_ids);
 280                 node_to_amd_nb(i)->link = link =
 281                         next_northbridge(link, link_ids);
 282
 283                 /*
 284                  * If there are more PCI root devices than data fabric/
 285                  * system management network interfaces, then the (N)
 286                  * PCI roots per DF/SMN interface are functionally the
 287                  * same (for DF/SMN access) and N-1 are redundant.  N-1
 288                  * PCI roots should be skipped per DF/SMN interface so
 289                  * the following DF/SMN interfaces get mapped to
 290                  * correct PCI roots.
 291                  */
 292                 for (j = 1; j < roots_per_misc; j++)
 293                         root = next_northbridge(root, root_ids);
 294         }
 295
 296         if (amd_gart_present())
 297                 amd_northbridges.flags |= AMD_NB_GART;
 298
 299         /*
 300          * Check for L3 cache presence.
 301          */
 302         if (!cpuid_edx(0x80000006))
 303                 return 0;
 304
 305         /*
 306          * Some CPU families support L3 Cache Index Disable. There are some
 307          * limitations because of E382 and E388 on family 0x10.
 308          */
 309         if (boot_cpu_data.x86 == 0x10 &&
 310             boot_cpu_data.x86_model >= 0x8 &&
 311             (boot_cpu_data.x86_model > 0x9 ||
 312              boot_cpu_data.x86_stepping >= 0x1))
 313                 amd_northbridges.flags |= AMD_NB_L3_INDEX_DISABLE;
 314
 315         if (boot_cpu_data.x86 == 0x15)
 316                 amd_northbridges.flags |= AMD_NB_L3_INDEX_DISABLE;
 317
 318         /* L3 cache partitioning is supported on family 0x15 */
 319         if (boot_cpu_data.x86 == 0x15)
 320                 amd_northbridges.flags |= AMD_NB_L3_PARTITIONING;
 321
 322         return 0;
 323 }
 324 EXPORT_SYMBOL_GPL(amd_cache_northbridges);
 325
 326 /*
 327  * Ignores subdevice/subvendor but as far as I can figure out
 328  * they're useless anyways
 329  */
 330 bool __init early_is_amd_nb(u32 device)
 331 {
 332         const struct pci_device_id *misc_ids = amd_nb_misc_ids;
 333         const struct pci_device_id *id;
 334         u32 vendor = device & 0xffff;
 335
 336         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
 337             boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
 338                 return false;
 339
 340         if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
 341                 misc_ids = hygon_nb_misc_ids;
 342
 343         device >>= 16;
 344         for (id = misc_ids; id->vendor; id++)
 345                 if (vendor == id->vendor && device == id->device)
 346                         return true;
 347         return false;
 348 }
 349
 350 struct resource *amd_get_mmconfig_range(struct resource *res)
 351 {
 352         u32 address;
 353         u64 base, msr;
 354         unsigned int segn_busn_bits;
 355
 356         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
 357             boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
 358                 return NULL;
 359
 360         /* assume all cpus from fam10h have mmconfig */
 361         if (boot_cpu_data.x86 < 0x10)
 362                 return NULL;
 363
 364         address = MSR_FAM10H_MMIO_CONF_BASE;
 365         rdmsrl(address, msr);
 366
 367         /* mmconfig is not enabled */
 368         if (!(msr & FAM10H_MMIO_CONF_ENABLE))
 369                 return NULL;
 370
 371         base = msr & (FAM10H_MMIO_CONF_BASE_MASK<<FAM10H_MMIO_CONF_BASE_SHIFT);
 372
 373         segn_busn_bits = (msr >> FAM10H_MMIO_CONF_BUSRANGE_SHIFT) &
 374                          FAM10H_MMIO_CONF_BUSRANGE_MASK;
 375
 376         res->flags = IORESOURCE_MEM;
 377         res->start = base;
 378         res->end = base + (1ULL<<(segn_busn_bits + 20)) - 1;
 379         return res;
 380 }
 381
 382 int amd_get_subcaches(int cpu)
 383 {
 384         struct pci_dev *link = node_to_amd_nb(amd_get_nb_id(cpu))->link;
 385         unsigned int mask;
 386
 387         if (!amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
 388                 return 0;
 389
 390         pci_read_config_dword(link, 0x1d4, &mask);
 391
 392         return (mask >> (4 * cpu_data(cpu).cpu_core_id)) & 0xf;
 393 }
 394
 395 int amd_set_subcaches(int cpu, unsigned long mask)
 396 {
 397         static unsigned int reset, ban;
 398         struct amd_northbridge *nb = node_to_amd_nb(amd_get_nb_id(cpu));
 399         unsigned int reg;
 400         int cuid;
 401
 402         if (!amd_nb_has_feature(AMD_NB_L3_PARTITIONING) || mask > 0xf)
 403                 return -EINVAL;
 404
 405         /* if necessary, collect reset state of L3 partitioning and BAN mode */
 406         if (reset == 0) {
 407                 pci_read_config_dword(nb->link, 0x1d4, &reset);
 408                 pci_read_config_dword(nb->misc, 0x1b8, &ban);
 409                 ban &= 0x180000;
 410         }
 411
 412         /* deactivate BAN mode if any subcaches are to be disabled */
 413         if (mask != 0xf) {
 414                 pci_read_config_dword(nb->misc, 0x1b8, &reg);
 415                 pci_write_config_dword(nb->misc, 0x1b8, reg & ~0x180000);
 416         }
 417
 418         cuid = cpu_data(cpu).cpu_core_id;
 419         mask <<= 4 * cuid;
 420         mask |= (0xf ^ (1 << cuid)) << 26;
 421
 422         pci_write_config_dword(nb->link, 0x1d4, mask);
 423
 424         /* reset BAN mode if L3 partitioning returned to reset state */
 425         pci_read_config_dword(nb->link, 0x1d4, &reg);
 426         if (reg == reset) {
 427                 pci_read_config_dword(nb->misc, 0x1b8, &reg);
 428                 reg &= ~0x180000;
 429                 pci_write_config_dword(nb->misc, 0x1b8, reg | ban);
 430         }
 431
 432         return 0;
 433 }
 434
 435 static void amd_cache_gart(void)
 436 {
 437         u16 i;
 438
 439         if (!amd_nb_has_feature(AMD_NB_GART))
 440                 return;
 441
 442         flush_words = kmalloc_array(amd_northbridges.num, sizeof(u32), GFP_KERNEL);
 443         if (!flush_words) {
 444                 amd_northbridges.flags &= ~AMD_NB_GART;
 445                 pr_notice("Cannot initialize GART flush words, GART support disabled\n");
 446                 return;
 447         }
 448
 449         for (i = 0; i != amd_northbridges.num; i++)
 450                 pci_read_config_dword(node_to_amd_nb(i)->misc, 0x9c, &flush_words[i]);
 451 }
 452
 453 void amd_flush_garts(void)
 454 {
 455         int flushed, i;
 456         unsigned long flags;
 457         static DEFINE_SPINLOCK(gart_lock);
 458
 459         if (!amd_nb_has_feature(AMD_NB_GART))
 460                 return;
 461
 462         /*
 463          * Avoid races between AGP and IOMMU. In theory it's not needed
 464          * but I'm not sure if the hardware won't lose flush requests
 465          * when another is pending. This whole thing is so expensive anyways
 466          * that it doesn't matter to serialize more. -AK
 467          */
 468         spin_lock_irqsave(&gart_lock, flags);
 469         flushed = 0;
 470         for (i = 0; i < amd_northbridges.num; i++) {
 471                 pci_write_config_dword(node_to_amd_nb(i)->misc, 0x9c,
 472                                        flush_words[i] | 1);
 473                 flushed++;
 474         }
 475         for (i = 0; i < amd_northbridges.num; i++) {
 476                 u32 w;
 477                 /* Make sure the hardware actually executed the flush*/
 478                 for (;;) {
 479                         pci_read_config_dword(node_to_amd_nb(i)->misc,
 480                                               0x9c, &w);
 481                         if (!(w & 1))
 482                                 break;
 483                         cpu_relax();
 484                 }
 485         }
 486         spin_unlock_irqrestore(&gart_lock, flags);
 487         if (!flushed)
 488                 pr_notice("nothing to flush?\n");
 489 }
 490 EXPORT_SYMBOL_GPL(amd_flush_garts);
 491
 492 static void __fix_erratum_688(void *info)
 493 {
 494 #define MSR_AMD64_IC_CFG 0xC0011021
 495
 496         msr_set_bit(MSR_AMD64_IC_CFG, 3);
 497         msr_set_bit(MSR_AMD64_IC_CFG, 14);
 498 }
 499
 500 /* Apply erratum 688 fix so machines without a BIOS fix work. */
 501 static __init void fix_erratum_688(void)
 502 {
 503         struct pci_dev *F4;
 504         u32 val;
 505
 506         if (boot_cpu_data.x86 != 0x14)
 507                 return;
 508
 509         if (!amd_northbridges.num)
 510                 return;
 511
 512         F4 = node_to_amd_nb(0)->link;
 513         if (!F4)
 514                 return;
 515
 516         if (pci_read_config_dword(F4, 0x164, &val))
 517                 return;
 518
 519         if (val & BIT(2))
 520                 return;
 521
 522         on_each_cpu(__fix_erratum_688, NULL, 0);
 523
 524         pr_info("x86/cpu/AMD: CPU erratum 688 worked around\n");
 525 }
 526
 527 static __init int init_amd_nbs(void)
 528 {
 529         amd_cache_northbridges();
 530         amd_cache_gart();
 531
 532         fix_erratum_688();
 533
 534         return 0;
 535 }
 536
 537 /* This has to go after the PCI subsystem */
 538 fs_initcall(init_amd_nbs);