drivers/edac/i10nm_base.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Driver for Intel(R) 10nm server memory controller.
   4  * Copyright (c) 2019, Intel Corporation.
   5  *
   6  */
   7
   8 #include <linux/kernel.h>
   9 #include <linux/io.h>
  10 #include <asm/cpu_device_id.h>
  11 #include <asm/intel-family.h>
  12 #include <asm/mce.h>
  13 #include "edac_module.h"
  14 #include "skx_common.h"
  15
  16 #define I10NM_REVISION  "v0.0.5"
  17 #define EDAC_MOD_STR    "i10nm_edac"
  18
  19 /* Debug macros */
  20 #define i10nm_printk(level, fmt, arg...)        \
  21         edac_printk(level, "i10nm", fmt, ##arg)
  22
  23 #define I10NM_GET_SCK_BAR(d, reg)       \
  24         pci_read_config_dword((d)->uracu, 0xd0, &(reg))
  25 #define I10NM_GET_IMC_BAR(d, i, reg)    \
  26         pci_read_config_dword((d)->uracu, 0xd8 + (i) * 4, &(reg))
  27 #define I10NM_GET_SAD(d, offset, i, reg)\
  28         pci_read_config_dword((d)->sad_all, (offset) + (i) * 8, &(reg))
  29 #define I10NM_GET_HBM_IMC_BAR(d, reg)   \
  30         pci_read_config_dword((d)->uracu, 0xd4, &(reg))
  31 #define I10NM_GET_CAPID3_CFG(d, reg)    \
  32         pci_read_config_dword((d)->pcu_cr3, 0x90, &(reg))
  33 #define I10NM_GET_DIMMMTR(m, i, j)      \
  34         readl((m)->mbase + ((m)->hbm_mc ? 0x80c : 0x2080c) + \
  35         (i) * (m)->chan_mmio_sz + (j) * 4)
  36 #define I10NM_GET_MCDDRTCFG(m, i, j)    \
  37         readl((m)->mbase + ((m)->hbm_mc ? 0x970 : 0x20970) + \
  38         (i) * (m)->chan_mmio_sz + (j) * 4)
  39 #define I10NM_GET_MCMTR(m, i)           \
  40         readl((m)->mbase + ((m)->hbm_mc ? 0xef8 : 0x20ef8) + \
  41         (i) * (m)->chan_mmio_sz)
  42 #define I10NM_GET_AMAP(m, i)            \
  43         readl((m)->mbase + ((m)->hbm_mc ? 0x814 : 0x20814) + \
  44         (i) * (m)->chan_mmio_sz)
  45
  46 #define I10NM_GET_SCK_MMIO_BASE(reg)    (GET_BITFIELD(reg, 0, 28) << 23)
  47 #define I10NM_GET_IMC_MMIO_OFFSET(reg)  (GET_BITFIELD(reg, 0, 10) << 12)
  48 #define I10NM_GET_IMC_MMIO_SIZE(reg)    ((GET_BITFIELD(reg, 13, 23) - \
  49                                          GET_BITFIELD(reg, 0, 10) + 1) << 12)
  50 #define I10NM_GET_HBM_IMC_MMIO_OFFSET(reg)      \
  51         ((GET_BITFIELD(reg, 0, 10) << 12) + 0x140000)
  52
  53 #define I10NM_HBM_IMC_MMIO_SIZE         0x9000
  54 #define I10NM_IS_HBM_PRESENT(reg)       GET_BITFIELD(reg, 27, 30)
  55 #define I10NM_IS_HBM_IMC(reg)           GET_BITFIELD(reg, 29, 29)
  56
  57 #define I10NM_MAX_SAD                   16
  58 #define I10NM_SAD_ENABLE(reg)           GET_BITFIELD(reg, 0, 0)
  59 #define I10NM_SAD_NM_CACHEABLE(reg)     GET_BITFIELD(reg, 5, 5)
  60
  61 static struct list_head *i10nm_edac_list;
  62
  63 static struct pci_dev *pci_get_dev_wrapper(int dom, unsigned int bus,
  64                                            unsigned int dev, unsigned int fun)
  65 {
  66         struct pci_dev *pdev;
  67
  68         pdev = pci_get_domain_bus_and_slot(dom, bus, PCI_DEVFN(dev, fun));
  69         if (!pdev) {
  70                 edac_dbg(2, "No device %02x:%02x.%x\n",
  71                          bus, dev, fun);
  72                 return NULL;
  73         }
  74
  75         if (unlikely(pci_enable_device(pdev) < 0)) {
  76                 edac_dbg(2, "Failed to enable device %02x:%02x.%x\n",
  77                          bus, dev, fun);
  78                 return NULL;
  79         }
  80
  81         pci_dev_get(pdev);
  82
  83         return pdev;
  84 }
  85
  86 static bool i10nm_check_2lm(struct res_config *cfg)
  87 {
  88         struct skx_dev *d;
  89         u32 reg;
  90         int i;
  91
  92         list_for_each_entry(d, i10nm_edac_list, list) {
  93                 d->sad_all = pci_get_dev_wrapper(d->seg, d->bus[1],
  94                                                  PCI_SLOT(cfg->sad_all_devfn),
  95                                                  PCI_FUNC(cfg->sad_all_devfn));
  96                 if (!d->sad_all)
  97                         continue;
  98
  99                 for (i = 0; i < I10NM_MAX_SAD; i++) {
 100                         I10NM_GET_SAD(d, cfg->sad_all_offset, i, reg);
 101                         if (I10NM_SAD_ENABLE(reg) && I10NM_SAD_NM_CACHEABLE(reg)) {
 102                                 edac_dbg(2, "2-level memory configuration.\n");
 103                                 return true;
 104                         }
 105                 }
 106         }
 107
 108         return false;
 109 }
 110
 111 static int i10nm_get_ddr_munits(void)
 112 {
 113         struct pci_dev *mdev;
 114         void __iomem *mbase;
 115         unsigned long size;
 116         struct skx_dev *d;
 117         int i, j = 0;
 118         u32 reg, off;
 119         u64 base;
 120
 121         list_for_each_entry(d, i10nm_edac_list, list) {
 122                 d->util_all = pci_get_dev_wrapper(d->seg, d->bus[1], 29, 1);
 123                 if (!d->util_all)
 124                         return -ENODEV;
 125
 126                 d->uracu = pci_get_dev_wrapper(d->seg, d->bus[0], 0, 1);
 127                 if (!d->uracu)
 128                         return -ENODEV;
 129
 130                 if (I10NM_GET_SCK_BAR(d, reg)) {
 131                         i10nm_printk(KERN_ERR, "Failed to socket bar\n");
 132                         return -ENODEV;
 133                 }
 134
 135                 base = I10NM_GET_SCK_MMIO_BASE(reg);
 136                 edac_dbg(2, "socket%d mmio base 0x%llx (reg 0x%x)\n",
 137                          j++, base, reg);
 138
 139                 for (i = 0; i < I10NM_NUM_DDR_IMC; i++) {
 140                         mdev = pci_get_dev_wrapper(d->seg, d->bus[0],
 141                                                    12 + i, 0);
 142                         if (i == 0 && !mdev) {
 143                                 i10nm_printk(KERN_ERR, "No IMC found\n");
 144                                 return -ENODEV;
 145                         }
 146                         if (!mdev)
 147                                 continue;
 148
 149                         d->imc[i].mdev = mdev;
 150
 151                         if (I10NM_GET_IMC_BAR(d, i, reg)) {
 152                                 i10nm_printk(KERN_ERR, "Failed to get mc bar\n");
 153                                 return -ENODEV;
 154                         }
 155
 156                         off  = I10NM_GET_IMC_MMIO_OFFSET(reg);
 157                         size = I10NM_GET_IMC_MMIO_SIZE(reg);
 158                         edac_dbg(2, "mc%d mmio base 0x%llx size 0x%lx (reg 0x%x)\n",
 159                                  i, base + off, size, reg);
 160
 161                         mbase = ioremap(base + off, size);
 162                         if (!mbase) {
 163                                 i10nm_printk(KERN_ERR, "Failed to ioremap 0x%llx\n",
 164                                              base + off);
 165                                 return -ENODEV;
 166                         }
 167
 168                         d->imc[i].mbase = mbase;
 169                 }
 170         }
 171
 172         return 0;
 173 }
 174
 175 static bool i10nm_check_hbm_imc(struct skx_dev *d)
 176 {
 177         u32 reg;
 178
 179         if (I10NM_GET_CAPID3_CFG(d, reg)) {
 180                 i10nm_printk(KERN_ERR, "Failed to get capid3_cfg\n");
 181                 return false;
 182         }
 183
 184         return I10NM_IS_HBM_PRESENT(reg) != 0;
 185 }
 186
 187 static int i10nm_get_hbm_munits(void)
 188 {
 189         struct pci_dev *mdev;
 190         void __iomem *mbase;
 191         u32 reg, off, mcmtr;
 192         struct skx_dev *d;
 193         int i, lmc;
 194         u64 base;
 195
 196         list_for_each_entry(d, i10nm_edac_list, list) {
 197                 d->pcu_cr3 = pci_get_dev_wrapper(d->seg, d->bus[1], 30, 3);
 198                 if (!d->pcu_cr3)
 199                         return -ENODEV;
 200
 201                 if (!i10nm_check_hbm_imc(d)) {
 202                         i10nm_printk(KERN_DEBUG, "No hbm memory\n");
 203                         return -ENODEV;
 204                 }
 205
 206                 if (I10NM_GET_SCK_BAR(d, reg)) {
 207                         i10nm_printk(KERN_ERR, "Failed to get socket bar\n");
 208                         return -ENODEV;
 209                 }
 210                 base = I10NM_GET_SCK_MMIO_BASE(reg);
 211
 212                 if (I10NM_GET_HBM_IMC_BAR(d, reg)) {
 213                         i10nm_printk(KERN_ERR, "Failed to get hbm mc bar\n");
 214                         return -ENODEV;
 215                 }
 216                 base += I10NM_GET_HBM_IMC_MMIO_OFFSET(reg);
 217
 218                 lmc = I10NM_NUM_DDR_IMC;
 219
 220                 for (i = 0; i < I10NM_NUM_HBM_IMC; i++) {
 221                         mdev = pci_get_dev_wrapper(d->seg, d->bus[0],
 222                                                    12 + i / 4, 1 + i % 4);
 223                         if (i == 0 && !mdev) {
 224                                 i10nm_printk(KERN_ERR, "No hbm mc found\n");
 225                                 return -ENODEV;
 226                         }
 227                         if (!mdev)
 228                                 continue;
 229
 230                         d->imc[lmc].mdev = mdev;
 231                         off = i * I10NM_HBM_IMC_MMIO_SIZE;
 232
 233                         edac_dbg(2, "hbm mc%d mmio base 0x%llx size 0x%x\n",
 234                                  lmc, base + off, I10NM_HBM_IMC_MMIO_SIZE);
 235
 236                         mbase = ioremap(base + off, I10NM_HBM_IMC_MMIO_SIZE);
 237                         if (!mbase) {
 238                                 i10nm_printk(KERN_ERR, "Failed to ioremap for hbm mc 0x%llx\n",
 239                                              base + off);
 240                                 return -ENOMEM;
 241                         }
 242
 243                         d->imc[lmc].mbase = mbase;
 244                         d->imc[lmc].hbm_mc = true;
 245
 246                         mcmtr = I10NM_GET_MCMTR(&d->imc[lmc], 0);
 247                         if (!I10NM_IS_HBM_IMC(mcmtr)) {
 248                                 i10nm_printk(KERN_ERR, "This isn't an hbm mc!\n");
 249                                 return -ENODEV;
 250                         }
 251
 252                         lmc++;
 253                 }
 254         }
 255
 256         return 0;
 257 }
 258
 259 static struct res_config i10nm_cfg0 = {
 260         .type                   = I10NM,
 261         .decs_did               = 0x3452,
 262         .busno_cfg_offset       = 0xcc,
 263         .ddr_chan_mmio_sz       = 0x4000,
 264         .sad_all_devfn          = PCI_DEVFN(29, 0),
 265         .sad_all_offset         = 0x108,
 266 };
 267
 268 static struct res_config i10nm_cfg1 = {
 269         .type                   = I10NM,
 270         .decs_did               = 0x3452,
 271         .busno_cfg_offset       = 0xd0,
 272         .ddr_chan_mmio_sz       = 0x4000,
 273         .sad_all_devfn          = PCI_DEVFN(29, 0),
 274         .sad_all_offset         = 0x108,
 275 };
 276
 277 static struct res_config spr_cfg = {
 278         .type                   = SPR,
 279         .decs_did               = 0x3252,
 280         .busno_cfg_offset       = 0xd0,
 281         .ddr_chan_mmio_sz       = 0x8000,
 282         .hbm_chan_mmio_sz       = 0x4000,
 283         .support_ddr5           = true,
 284         .sad_all_devfn          = PCI_DEVFN(10, 0),
 285         .sad_all_offset         = 0x300,
 286 };
 287
 288 static const struct x86_cpu_id i10nm_cpuids[] = {
 289         X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ATOM_TREMONT_D,    X86_STEPPINGS(0x0, 0x3), &i10nm_cfg0),
 290         X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ATOM_TREMONT_D,    X86_STEPPINGS(0x4, 0xf), &i10nm_cfg1),
 291         X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_X,         X86_STEPPINGS(0x0, 0x3), &i10nm_cfg0),
 292         X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_X,         X86_STEPPINGS(0x4, 0xf), &i10nm_cfg1),
 293         X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_D,         X86_STEPPINGS(0x0, 0xf), &i10nm_cfg1),
 294         X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SAPPHIRERAPIDS_X,  X86_STEPPINGS(0x0, 0xf), &spr_cfg),
 295         {}
 296 };
 297 MODULE_DEVICE_TABLE(x86cpu, i10nm_cpuids);
 298
 299 static bool i10nm_check_ecc(struct skx_imc *imc, int chan)
 300 {
 301         u32 mcmtr;
 302
 303         mcmtr = I10NM_GET_MCMTR(imc, chan);
 304         edac_dbg(1, "ch%d mcmtr reg %x\n", chan, mcmtr);
 305
 306         return !!GET_BITFIELD(mcmtr, 2, 2);
 307 }
 308
 309 static int i10nm_get_dimm_config(struct mem_ctl_info *mci,
 310                                  struct res_config *cfg)
 311 {
 312         struct skx_pvt *pvt = mci->pvt_info;
 313         struct skx_imc *imc = pvt->imc;
 314         u32 mtr, amap, mcddrtcfg;
 315         struct dimm_info *dimm;
 316         int i, j, ndimms;
 317
 318         for (i = 0; i < imc->num_channels; i++) {
 319                 if (!imc->mbase)
 320                         continue;
 321
 322                 ndimms = 0;
 323                 amap = I10NM_GET_AMAP(imc, i);
 324                 for (j = 0; j < imc->num_dimms; j++) {
 325                         dimm = edac_get_dimm(mci, i, j, 0);
 326                         mtr = I10NM_GET_DIMMMTR(imc, i, j);
 327                         mcddrtcfg = I10NM_GET_MCDDRTCFG(imc, i, j);
 328                         edac_dbg(1, "dimmmtr 0x%x mcddrtcfg 0x%x (mc%d ch%d dimm%d)\n",
 329                                  mtr, mcddrtcfg, imc->mc, i, j);
 330
 331                         if (IS_DIMM_PRESENT(mtr))
 332                                 ndimms += skx_get_dimm_info(mtr, 0, amap, dimm,
 333                                                             imc, i, j, cfg);
 334                         else if (IS_NVDIMM_PRESENT(mcddrtcfg, j))
 335                                 ndimms += skx_get_nvdimm_info(dimm, imc, i, j,
 336                                                               EDAC_MOD_STR);
 337                 }
 338                 if (ndimms && !i10nm_check_ecc(imc, i)) {
 339                         i10nm_printk(KERN_ERR, "ECC is disabled on imc %d channel %d\n",
 340                                      imc->mc, i);
 341                         return -ENODEV;
 342                 }
 343         }
 344
 345         return 0;
 346 }
 347
 348 static struct notifier_block i10nm_mce_dec = {
 349         .notifier_call  = skx_mce_check_error,
 350         .priority       = MCE_PRIO_EDAC,
 351 };
 352
 353 #ifdef CONFIG_EDAC_DEBUG
 354 /*
 355  * Debug feature.
 356  * Exercise the address decode logic by writing an address to
 357  * /sys/kernel/debug/edac/i10nm_test/addr.
 358  */
 359 static struct dentry *i10nm_test;
 360
 361 static int debugfs_u64_set(void *data, u64 val)
 362 {
 363         struct mce m;
 364
 365         pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val);
 366
 367         memset(&m, 0, sizeof(m));
 368         /* ADDRV + MemRd + Unknown channel */
 369         m.status = MCI_STATUS_ADDRV + 0x90;
 370         /* One corrected error */
 371         m.status |= BIT_ULL(MCI_STATUS_CEC_SHIFT);
 372         m.addr = val;
 373         skx_mce_check_error(NULL, 0, &m);
 374
 375         return 0;
 376 }
 377 DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n");
 378
 379 static void setup_i10nm_debug(void)
 380 {
 381         i10nm_test = edac_debugfs_create_dir("i10nm_test");
 382         if (!i10nm_test)
 383                 return;
 384
 385         if (!edac_debugfs_create_file("addr", 0200, i10nm_test,
 386                                       NULL, &fops_u64_wo)) {
 387                 debugfs_remove(i10nm_test);
 388                 i10nm_test = NULL;
 389         }
 390 }
 391
 392 static void teardown_i10nm_debug(void)
 393 {
 394         debugfs_remove_recursive(i10nm_test);
 395 }
 396 #else
 397 static inline void setup_i10nm_debug(void) {}
 398 static inline void teardown_i10nm_debug(void) {}
 399 #endif /*CONFIG_EDAC_DEBUG*/
 400
 401 static int __init i10nm_init(void)
 402 {
 403         u8 mc = 0, src_id = 0, node_id = 0;
 404         const struct x86_cpu_id *id;
 405         struct res_config *cfg;
 406         const char *owner;
 407         struct skx_dev *d;
 408         int rc, i, off[3] = {0xd0, 0xc8, 0xcc};
 409         u64 tolm, tohm;
 410
 411         edac_dbg(2, "\n");
 412
 413         owner = edac_get_owner();
 414         if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
 415                 return -EBUSY;
 416
 417         if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
 418                 return -ENODEV;
 419
 420         id = x86_match_cpu(i10nm_cpuids);
 421         if (!id)
 422                 return -ENODEV;
 423
 424         cfg = (struct res_config *)id->driver_data;
 425
 426         rc = skx_get_hi_lo(0x09a2, off, &tolm, &tohm);
 427         if (rc)
 428                 return rc;
 429
 430         rc = skx_get_all_bus_mappings(cfg, &i10nm_edac_list);
 431         if (rc < 0)
 432                 goto fail;
 433         if (rc == 0) {
 434                 i10nm_printk(KERN_ERR, "No memory controllers found\n");
 435                 return -ENODEV;
 436         }
 437
 438         skx_set_mem_cfg(i10nm_check_2lm(cfg));
 439
 440         rc = i10nm_get_ddr_munits();
 441
 442         if (i10nm_get_hbm_munits() && rc)
 443                 goto fail;
 444
 445         list_for_each_entry(d, i10nm_edac_list, list) {
 446                 rc = skx_get_src_id(d, 0xf8, &src_id);
 447                 if (rc < 0)
 448                         goto fail;
 449
 450                 rc = skx_get_node_id(d, &node_id);
 451                 if (rc < 0)
 452                         goto fail;
 453
 454                 edac_dbg(2, "src_id = %d node_id = %d\n", src_id, node_id);
 455                 for (i = 0; i < I10NM_NUM_IMC; i++) {
 456                         if (!d->imc[i].mdev)
 457                                 continue;
 458
 459                         d->imc[i].mc  = mc++;
 460                         d->imc[i].lmc = i;
 461                         d->imc[i].src_id  = src_id;
 462                         d->imc[i].node_id = node_id;
 463                         if (d->imc[i].hbm_mc) {
 464                                 d->imc[i].chan_mmio_sz = cfg->hbm_chan_mmio_sz;
 465                                 d->imc[i].num_channels = I10NM_NUM_HBM_CHANNELS;
 466                                 d->imc[i].num_dimms    = I10NM_NUM_HBM_DIMMS;
 467                         } else {
 468                                 d->imc[i].chan_mmio_sz = cfg->ddr_chan_mmio_sz;
 469                                 d->imc[i].num_channels = I10NM_NUM_DDR_CHANNELS;
 470                                 d->imc[i].num_dimms    = I10NM_NUM_DDR_DIMMS;
 471                         }
 472
 473                         rc = skx_register_mci(&d->imc[i], d->imc[i].mdev,
 474                                               "Intel_10nm Socket", EDAC_MOD_STR,
 475                                               i10nm_get_dimm_config, cfg);
 476                         if (rc < 0)
 477                                 goto fail;
 478                 }
 479         }
 480
 481         rc = skx_adxl_get();
 482         if (rc)
 483                 goto fail;
 484
 485         opstate_init();
 486         mce_register_decode_chain(&i10nm_mce_dec);
 487         setup_i10nm_debug();
 488
 489         i10nm_printk(KERN_INFO, "%s\n", I10NM_REVISION);
 490
 491         return 0;
 492 fail:
 493         skx_remove();
 494         return rc;
 495 }
 496
 497 static void __exit i10nm_exit(void)
 498 {
 499         edac_dbg(2, "\n");
 500         teardown_i10nm_debug();
 501         mce_unregister_decode_chain(&i10nm_mce_dec);
 502         skx_adxl_put();
 503         skx_remove();
 504 }
 505
 506 module_init(i10nm_init);
 507 module_exit(i10nm_exit);
 508
 509 MODULE_LICENSE("GPL v2");
 510 MODULE_DESCRIPTION("MC Driver for Intel 10nm server processors");