drivers/cxl/core.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
   3 #include <linux/io-64-nonatomic-lo-hi.h>
   4 #include <linux/device.h>
   5 #include <linux/module.h>
   6 #include <linux/pci.h>
   7 #include <linux/slab.h>
   8 #include <linux/idr.h>
   9 #include "cxl.h"
  10 #include "mem.h"
  11
  12 /**
  13  * DOC: cxl core
  14  *
  15  * The CXL core provides a sysfs hierarchy for control devices and a rendezvous
  16  * point for cross-device interleave coordination through cxl ports.
  17  */
  18
  19 static DEFINE_IDA(cxl_port_ida);
  20
  21 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
  22                             char *buf)
  23 {
  24         return sysfs_emit(buf, "%s\n", dev->type->name);
  25 }
  26 static DEVICE_ATTR_RO(devtype);
  27
  28 static struct attribute *cxl_base_attributes[] = {
  29         &dev_attr_devtype.attr,
  30         NULL,
  31 };
  32
  33 static struct attribute_group cxl_base_attribute_group = {
  34         .attrs = cxl_base_attributes,
  35 };
  36
  37 static ssize_t start_show(struct device *dev, struct device_attribute *attr,
  38                           char *buf)
  39 {
  40         struct cxl_decoder *cxld = to_cxl_decoder(dev);
  41
  42         return sysfs_emit(buf, "%#llx\n", cxld->range.start);
  43 }
  44 static DEVICE_ATTR_RO(start);
  45
  46 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
  47                         char *buf)
  48 {
  49         struct cxl_decoder *cxld = to_cxl_decoder(dev);
  50
  51         return sysfs_emit(buf, "%#llx\n", range_len(&cxld->range));
  52 }
  53 static DEVICE_ATTR_RO(size);
  54
  55 #define CXL_DECODER_FLAG_ATTR(name, flag)                            \
  56 static ssize_t name##_show(struct device *dev,                       \
  57                            struct device_attribute *attr, char *buf) \
  58 {                                                                    \
  59         struct cxl_decoder *cxld = to_cxl_decoder(dev);              \
  60                                                                      \
  61         return sysfs_emit(buf, "%s\n",                               \
  62                           (cxld->flags & (flag)) ? "1" : "0");       \
  63 }                                                                    \
  64 static DEVICE_ATTR_RO(name)
  65
  66 CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
  67 CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
  68 CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
  69 CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
  70 CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
  71
  72 static ssize_t target_type_show(struct device *dev,
  73                                 struct device_attribute *attr, char *buf)
  74 {
  75         struct cxl_decoder *cxld = to_cxl_decoder(dev);
  76
  77         switch (cxld->target_type) {
  78         case CXL_DECODER_ACCELERATOR:
  79                 return sysfs_emit(buf, "accelerator\n");
  80         case CXL_DECODER_EXPANDER:
  81                 return sysfs_emit(buf, "expander\n");
  82         }
  83         return -ENXIO;
  84 }
  85 static DEVICE_ATTR_RO(target_type);
  86
  87 static ssize_t target_list_show(struct device *dev,
  88                                struct device_attribute *attr, char *buf)
  89 {
  90         struct cxl_decoder *cxld = to_cxl_decoder(dev);
  91         ssize_t offset = 0;
  92         int i, rc = 0;
  93
  94         device_lock(dev);
  95         for (i = 0; i < cxld->interleave_ways; i++) {
  96                 struct cxl_dport *dport = cxld->target[i];
  97                 struct cxl_dport *next = NULL;
  98
  99                 if (!dport)
 100                         break;
 101
 102                 if (i + 1 < cxld->interleave_ways)
 103                         next = cxld->target[i + 1];
 104                 rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
 105                                    next ? "," : "");
 106                 if (rc < 0)
 107                         break;
 108                 offset += rc;
 109         }
 110         device_unlock(dev);
 111
 112         if (rc < 0)
 113                 return rc;
 114
 115         rc = sysfs_emit_at(buf, offset, "\n");
 116         if (rc < 0)
 117                 return rc;
 118
 119         return offset + rc;
 120 }
 121 static DEVICE_ATTR_RO(target_list);
 122
 123 static struct attribute *cxl_decoder_base_attrs[] = {
 124         &dev_attr_start.attr,
 125         &dev_attr_size.attr,
 126         &dev_attr_locked.attr,
 127         &dev_attr_target_list.attr,
 128         NULL,
 129 };
 130
 131 static struct attribute_group cxl_decoder_base_attribute_group = {
 132         .attrs = cxl_decoder_base_attrs,
 133 };
 134
 135 static struct attribute *cxl_decoder_root_attrs[] = {
 136         &dev_attr_cap_pmem.attr,
 137         &dev_attr_cap_ram.attr,
 138         &dev_attr_cap_type2.attr,
 139         &dev_attr_cap_type3.attr,
 140         NULL,
 141 };
 142
 143 static struct attribute_group cxl_decoder_root_attribute_group = {
 144         .attrs = cxl_decoder_root_attrs,
 145 };
 146
 147 static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
 148         &cxl_decoder_root_attribute_group,
 149         &cxl_decoder_base_attribute_group,
 150         &cxl_base_attribute_group,
 151         NULL,
 152 };
 153
 154 static struct attribute *cxl_decoder_switch_attrs[] = {
 155         &dev_attr_target_type.attr,
 156         NULL,
 157 };
 158
 159 static struct attribute_group cxl_decoder_switch_attribute_group = {
 160         .attrs = cxl_decoder_switch_attrs,
 161 };
 162
 163 static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
 164         &cxl_decoder_switch_attribute_group,
 165         &cxl_decoder_base_attribute_group,
 166         &cxl_base_attribute_group,
 167         NULL,
 168 };
 169
 170 static void cxl_decoder_release(struct device *dev)
 171 {
 172         struct cxl_decoder *cxld = to_cxl_decoder(dev);
 173         struct cxl_port *port = to_cxl_port(dev->parent);
 174
 175         ida_free(&port->decoder_ida, cxld->id);
 176         kfree(cxld);
 177 }
 178
 179 static const struct device_type cxl_decoder_switch_type = {
 180         .name = "cxl_decoder_switch",
 181         .release = cxl_decoder_release,
 182         .groups = cxl_decoder_switch_attribute_groups,
 183 };
 184
 185 static const struct device_type cxl_decoder_root_type = {
 186         .name = "cxl_decoder_root",
 187         .release = cxl_decoder_release,
 188         .groups = cxl_decoder_root_attribute_groups,
 189 };
 190
 191 bool is_root_decoder(struct device *dev)
 192 {
 193         return dev->type == &cxl_decoder_root_type;
 194 }
 195 EXPORT_SYMBOL_GPL(is_root_decoder);
 196
 197 struct cxl_decoder *to_cxl_decoder(struct device *dev)
 198 {
 199         if (dev_WARN_ONCE(dev, dev->type->release != cxl_decoder_release,
 200                           "not a cxl_decoder device\n"))
 201                 return NULL;
 202         return container_of(dev, struct cxl_decoder, dev);
 203 }
 204 EXPORT_SYMBOL_GPL(to_cxl_decoder);
 205
 206 static void cxl_dport_release(struct cxl_dport *dport)
 207 {
 208         list_del(&dport->list);
 209         put_device(dport->dport);
 210         kfree(dport);
 211 }
 212
 213 static void cxl_port_release(struct device *dev)
 214 {
 215         struct cxl_port *port = to_cxl_port(dev);
 216         struct cxl_dport *dport, *_d;
 217
 218         device_lock(dev);
 219         list_for_each_entry_safe(dport, _d, &port->dports, list)
 220                 cxl_dport_release(dport);
 221         device_unlock(dev);
 222         ida_free(&cxl_port_ida, port->id);
 223         kfree(port);
 224 }
 225
 226 static const struct attribute_group *cxl_port_attribute_groups[] = {
 227         &cxl_base_attribute_group,
 228         NULL,
 229 };
 230
 231 static const struct device_type cxl_port_type = {
 232         .name = "cxl_port",
 233         .release = cxl_port_release,
 234         .groups = cxl_port_attribute_groups,
 235 };
 236
 237 struct cxl_port *to_cxl_port(struct device *dev)
 238 {
 239         if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
 240                           "not a cxl_port device\n"))
 241                 return NULL;
 242         return container_of(dev, struct cxl_port, dev);
 243 }
 244
 245 static void unregister_port(void *_port)
 246 {
 247         struct cxl_port *port = _port;
 248         struct cxl_dport *dport;
 249
 250         device_lock(&port->dev);
 251         list_for_each_entry(dport, &port->dports, list) {
 252                 char link_name[CXL_TARGET_STRLEN];
 253
 254                 if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d",
 255                              dport->port_id) >= CXL_TARGET_STRLEN)
 256                         continue;
 257                 sysfs_remove_link(&port->dev.kobj, link_name);
 258         }
 259         device_unlock(&port->dev);
 260         device_unregister(&port->dev);
 261 }
 262
 263 static void cxl_unlink_uport(void *_port)
 264 {
 265         struct cxl_port *port = _port;
 266
 267         sysfs_remove_link(&port->dev.kobj, "uport");
 268 }
 269
 270 static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
 271 {
 272         int rc;
 273
 274         rc = sysfs_create_link(&port->dev.kobj, &port->uport->kobj, "uport");
 275         if (rc)
 276                 return rc;
 277         return devm_add_action_or_reset(host, cxl_unlink_uport, port);
 278 }
 279
 280 static struct cxl_port *cxl_port_alloc(struct device *uport,
 281                                        resource_size_t component_reg_phys,
 282                                        struct cxl_port *parent_port)
 283 {
 284         struct cxl_port *port;
 285         struct device *dev;
 286         int rc;
 287
 288         port = kzalloc(sizeof(*port), GFP_KERNEL);
 289         if (!port)
 290                 return ERR_PTR(-ENOMEM);
 291
 292         rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
 293         if (rc < 0)
 294                 goto err;
 295         port->id = rc;
 296
 297         /*
 298          * The top-level cxl_port "cxl_root" does not have a cxl_port as
 299          * its parent and it does not have any corresponding component
 300          * registers as its decode is described by a fixed platform
 301          * description.
 302          */
 303         dev = &port->dev;
 304         if (parent_port)
 305                 dev->parent = &parent_port->dev;
 306         else
 307                 dev->parent = uport;
 308
 309         port->uport = uport;
 310         port->component_reg_phys = component_reg_phys;
 311         ida_init(&port->decoder_ida);
 312         INIT_LIST_HEAD(&port->dports);
 313
 314         device_initialize(dev);
 315         device_set_pm_not_required(dev);
 316         dev->bus = &cxl_bus_type;
 317         dev->type = &cxl_port_type;
 318
 319         return port;
 320
 321 err:
 322         kfree(port);
 323         return ERR_PTR(rc);
 324 }
 325
 326 /**
 327  * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
 328  * @host: host device for devm operations
 329  * @uport: "physical" device implementing this upstream port
 330  * @component_reg_phys: (optional) for configurable cxl_port instances
 331  * @parent_port: next hop up in the CXL memory decode hierarchy
 332  */
 333 struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport,
 334                                    resource_size_t component_reg_phys,
 335                                    struct cxl_port *parent_port)
 336 {
 337         struct cxl_port *port;
 338         struct device *dev;
 339         int rc;
 340
 341         port = cxl_port_alloc(uport, component_reg_phys, parent_port);
 342         if (IS_ERR(port))
 343                 return port;
 344
 345         dev = &port->dev;
 346         if (parent_port)
 347                 rc = dev_set_name(dev, "port%d", port->id);
 348         else
 349                 rc = dev_set_name(dev, "root%d", port->id);
 350         if (rc)
 351                 goto err;
 352
 353         rc = device_add(dev);
 354         if (rc)
 355                 goto err;
 356
 357         rc = devm_add_action_or_reset(host, unregister_port, port);
 358         if (rc)
 359                 return ERR_PTR(rc);
 360
 361         rc = devm_cxl_link_uport(host, port);
 362         if (rc)
 363                 return ERR_PTR(rc);
 364
 365         return port;
 366
 367 err:
 368         put_device(dev);
 369         return ERR_PTR(rc);
 370 }
 371 EXPORT_SYMBOL_GPL(devm_cxl_add_port);
 372
 373 static struct cxl_dport *find_dport(struct cxl_port *port, int id)
 374 {
 375         struct cxl_dport *dport;
 376
 377         device_lock_assert(&port->dev);
 378         list_for_each_entry (dport, &port->dports, list)
 379                 if (dport->port_id == id)
 380                         return dport;
 381         return NULL;
 382 }
 383
 384 static int add_dport(struct cxl_port *port, struct cxl_dport *new)
 385 {
 386         struct cxl_dport *dup;
 387
 388         device_lock(&port->dev);
 389         dup = find_dport(port, new->port_id);
 390         if (dup)
 391                 dev_err(&port->dev,
 392                         "unable to add dport%d-%s non-unique port id (%s)\n",
 393                         new->port_id, dev_name(new->dport),
 394                         dev_name(dup->dport));
 395         else
 396                 list_add_tail(&new->list, &port->dports);
 397         device_unlock(&port->dev);
 398
 399         return dup ? -EEXIST : 0;
 400 }
 401
 402 /**
 403  * cxl_add_dport - append downstream port data to a cxl_port
 404  * @port: the cxl_port that references this dport
 405  * @dport_dev: firmware or PCI device representing the dport
 406  * @port_id: identifier for this dport in a decoder's target list
 407  * @component_reg_phys: optional location of CXL component registers
 408  *
 409  * Note that all allocations and links are undone by cxl_port deletion
 410  * and release.
 411  */
 412 int cxl_add_dport(struct cxl_port *port, struct device *dport_dev, int port_id,
 413                   resource_size_t component_reg_phys)
 414 {
 415         char link_name[CXL_TARGET_STRLEN];
 416         struct cxl_dport *dport;
 417         int rc;
 418
 419         if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
 420             CXL_TARGET_STRLEN)
 421                 return -EINVAL;
 422
 423         dport = kzalloc(sizeof(*dport), GFP_KERNEL);
 424         if (!dport)
 425                 return -ENOMEM;
 426
 427         INIT_LIST_HEAD(&dport->list);
 428         dport->dport = get_device(dport_dev);
 429         dport->port_id = port_id;
 430         dport->component_reg_phys = component_reg_phys;
 431         dport->port = port;
 432
 433         rc = add_dport(port, dport);
 434         if (rc)
 435                 goto err;
 436
 437         rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
 438         if (rc)
 439                 goto err;
 440
 441         return 0;
 442 err:
 443         cxl_dport_release(dport);
 444         return rc;
 445 }
 446 EXPORT_SYMBOL_GPL(cxl_add_dport);
 447
 448 static struct cxl_decoder *
 449 cxl_decoder_alloc(struct cxl_port *port, int nr_targets, resource_size_t base,
 450                   resource_size_t len, int interleave_ways,
 451                   int interleave_granularity, enum cxl_decoder_type type,
 452                   unsigned long flags)
 453 {
 454         struct cxl_decoder *cxld;
 455         struct device *dev;
 456         int rc = 0;
 457
 458         if (interleave_ways < 1)
 459                 return ERR_PTR(-EINVAL);
 460
 461         device_lock(&port->dev);
 462         if (list_empty(&port->dports))
 463                 rc = -EINVAL;
 464         device_unlock(&port->dev);
 465         if (rc)
 466                 return ERR_PTR(rc);
 467
 468         cxld = kzalloc(struct_size(cxld, target, nr_targets), GFP_KERNEL);
 469         if (!cxld)
 470                 return ERR_PTR(-ENOMEM);
 471
 472         rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
 473         if (rc < 0)
 474                 goto err;
 475
 476         *cxld = (struct cxl_decoder) {
 477                 .id = rc,
 478                 .range = {
 479                         .start = base,
 480                         .end = base + len - 1,
 481                 },
 482                 .flags = flags,
 483                 .interleave_ways = interleave_ways,
 484                 .interleave_granularity = interleave_granularity,
 485                 .target_type = type,
 486         };
 487
 488         /* handle implied target_list */
 489         if (interleave_ways == 1)
 490                 cxld->target[0] =
 491                         list_first_entry(&port->dports, struct cxl_dport, list);
 492         dev = &cxld->dev;
 493         device_initialize(dev);
 494         device_set_pm_not_required(dev);
 495         dev->parent = &port->dev;
 496         dev->bus = &cxl_bus_type;
 497
 498         /* root ports do not have a cxl_port_type parent */
 499         if (port->dev.parent->type == &cxl_port_type)
 500                 dev->type = &cxl_decoder_switch_type;
 501         else
 502                 dev->type = &cxl_decoder_root_type;
 503
 504         return cxld;
 505 err:
 506         kfree(cxld);
 507         return ERR_PTR(rc);
 508 }
 509
 510 static void unregister_dev(void *dev)
 511 {
 512         device_unregister(dev);
 513 }
 514
 515 struct cxl_decoder *
 516 devm_cxl_add_decoder(struct device *host, struct cxl_port *port, int nr_targets,
 517                      resource_size_t base, resource_size_t len,
 518                      int interleave_ways, int interleave_granularity,
 519                      enum cxl_decoder_type type, unsigned long flags)
 520 {
 521         struct cxl_decoder *cxld;
 522         struct device *dev;
 523         int rc;
 524
 525         cxld = cxl_decoder_alloc(port, nr_targets, base, len, interleave_ways,
 526                                  interleave_granularity, type, flags);
 527         if (IS_ERR(cxld))
 528                 return cxld;
 529
 530         dev = &cxld->dev;
 531         rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
 532         if (rc)
 533                 goto err;
 534
 535         rc = device_add(dev);
 536         if (rc)
 537                 goto err;
 538
 539         rc = devm_add_action_or_reset(host, unregister_dev, dev);
 540         if (rc)
 541                 return ERR_PTR(rc);
 542         return cxld;
 543
 544 err:
 545         put_device(dev);
 546         return ERR_PTR(rc);
 547 }
 548 EXPORT_SYMBOL_GPL(devm_cxl_add_decoder);
 549
 550 /**
 551  * cxl_probe_component_regs() - Detect CXL Component register blocks
 552  * @dev: Host device of the @base mapping
 553  * @base: Mapping containing the HDM Decoder Capability Header
 554  * @map: Map object describing the register block information found
 555  *
 556  * See CXL 2.0 8.2.4 Component Register Layout and Definition
 557  * See CXL 2.0 8.2.5.5 CXL Device Register Interface
 558  *
 559  * Probe for component register information and return it in map object.
 560  */
 561 void cxl_probe_component_regs(struct device *dev, void __iomem *base,
 562                               struct cxl_component_reg_map *map)
 563 {
 564         int cap, cap_count;
 565         u64 cap_array;
 566
 567         *map = (struct cxl_component_reg_map) { 0 };
 568
 569         /*
 570          * CXL.cache and CXL.mem registers are at offset 0x1000 as defined in
 571          * CXL 2.0 8.2.4 Table 141.
 572          */
 573         base += CXL_CM_OFFSET;
 574
 575         cap_array = readq(base + CXL_CM_CAP_HDR_OFFSET);
 576
 577         if (FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, cap_array) != CM_CAP_HDR_CAP_ID) {
 578                 dev_err(dev,
 579                         "Couldn't locate the CXL.cache and CXL.mem capability array header./n");
 580                 return;
 581         }
 582
 583         /* It's assumed that future versions will be backward compatible */
 584         cap_count = FIELD_GET(CXL_CM_CAP_HDR_ARRAY_SIZE_MASK, cap_array);
 585
 586         for (cap = 1; cap <= cap_count; cap++) {
 587                 void __iomem *register_block;
 588                 u32 hdr;
 589                 int decoder_cnt;
 590                 u16 cap_id, offset;
 591                 u32 length;
 592
 593                 hdr = readl(base + cap * 0x4);
 594
 595                 cap_id = FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, hdr);
 596                 offset = FIELD_GET(CXL_CM_CAP_PTR_MASK, hdr);
 597                 register_block = base + offset;
 598
 599                 switch (cap_id) {
 600                 case CXL_CM_CAP_CAP_ID_HDM:
 601                         dev_dbg(dev, "found HDM decoder capability (0x%x)\n",
 602                                 offset);
 603
 604                         hdr = readl(register_block);
 605
 606                         decoder_cnt = cxl_hdm_decoder_count(hdr);
 607                         length = 0x20 * decoder_cnt + 0x10;
 608
 609                         map->hdm_decoder.valid = true;
 610                         map->hdm_decoder.offset = CXL_CM_OFFSET + offset;
 611                         map->hdm_decoder.size = length;
 612                         break;
 613                 default:
 614                         dev_dbg(dev, "Unknown CM cap ID: %d (0x%x)\n", cap_id,
 615                                 offset);
 616                         break;
 617                 }
 618         }
 619 }
 620 EXPORT_SYMBOL_GPL(cxl_probe_component_regs);
 621
 622 static void cxl_nvdimm_bridge_release(struct device *dev)
 623 {
 624         struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
 625
 626         kfree(cxl_nvb);
 627 }
 628
 629 static const struct attribute_group *cxl_nvdimm_bridge_attribute_groups[] = {
 630         &cxl_base_attribute_group,
 631         NULL,
 632 };
 633
 634 static const struct device_type cxl_nvdimm_bridge_type = {
 635         .name = "cxl_nvdimm_bridge",
 636         .release = cxl_nvdimm_bridge_release,
 637         .groups = cxl_nvdimm_bridge_attribute_groups,
 638 };
 639
 640 struct cxl_nvdimm_bridge *to_cxl_nvdimm_bridge(struct device *dev)
 641 {
 642         if (dev_WARN_ONCE(dev, dev->type != &cxl_nvdimm_bridge_type,
 643                           "not a cxl_nvdimm_bridge device\n"))
 644                 return NULL;
 645         return container_of(dev, struct cxl_nvdimm_bridge, dev);
 646 }
 647 EXPORT_SYMBOL_GPL(to_cxl_nvdimm_bridge);
 648
 649 static struct cxl_nvdimm_bridge *
 650 cxl_nvdimm_bridge_alloc(struct cxl_port *port)
 651 {
 652         struct cxl_nvdimm_bridge *cxl_nvb;
 653         struct device *dev;
 654
 655         cxl_nvb = kzalloc(sizeof(*cxl_nvb), GFP_KERNEL);
 656         if (!cxl_nvb)
 657                 return ERR_PTR(-ENOMEM);
 658
 659         dev = &cxl_nvb->dev;
 660         cxl_nvb->port = port;
 661         cxl_nvb->state = CXL_NVB_NEW;
 662         device_initialize(dev);
 663         device_set_pm_not_required(dev);
 664         dev->parent = &port->dev;
 665         dev->bus = &cxl_bus_type;
 666         dev->type = &cxl_nvdimm_bridge_type;
 667
 668         return cxl_nvb;
 669 }
 670
 671 static void unregister_nvb(void *_cxl_nvb)
 672 {
 673         struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb;
 674         bool flush;
 675
 676         /*
 677          * If the bridge was ever activated then there might be in-flight state
 678          * work to flush. Once the state has been changed to 'dead' then no new
 679          * work can be queued by user-triggered bind.
 680          */
 681         device_lock(&cxl_nvb->dev);
 682         flush = cxl_nvb->state != CXL_NVB_NEW;
 683         cxl_nvb->state = CXL_NVB_DEAD;
 684         device_unlock(&cxl_nvb->dev);
 685
 686         /*
 687          * Even though the device core will trigger device_release_driver()
 688          * before the unregister, it does not know about the fact that
 689          * cxl_nvdimm_bridge_driver defers ->remove() work. So, do the driver
 690          * release not and flush it before tearing down the nvdimm device
 691          * hierarchy.
 692          */
 693         device_release_driver(&cxl_nvb->dev);
 694         if (flush)
 695                 flush_work(&cxl_nvb->state_work);
 696         device_unregister(&cxl_nvb->dev);
 697 }
 698
 699 struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host,
 700                                                      struct cxl_port *port)
 701 {
 702         struct cxl_nvdimm_bridge *cxl_nvb;
 703         struct device *dev;
 704         int rc;
 705
 706         if (!IS_ENABLED(CONFIG_CXL_PMEM))
 707                 return ERR_PTR(-ENXIO);
 708
 709         cxl_nvb = cxl_nvdimm_bridge_alloc(port);
 710         if (IS_ERR(cxl_nvb))
 711                 return cxl_nvb;
 712
 713         dev = &cxl_nvb->dev;
 714         rc = dev_set_name(dev, "nvdimm-bridge");
 715         if (rc)
 716                 goto err;
 717
 718         rc = device_add(dev);
 719         if (rc)
 720                 goto err;
 721
 722         rc = devm_add_action_or_reset(host, unregister_nvb, cxl_nvb);
 723         if (rc)
 724                 return ERR_PTR(rc);
 725
 726         return cxl_nvb;
 727
 728 err:
 729         put_device(dev);
 730         return ERR_PTR(rc);
 731 }
 732 EXPORT_SYMBOL_GPL(devm_cxl_add_nvdimm_bridge);
 733
 734 static void cxl_nvdimm_release(struct device *dev)
 735 {
 736         struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
 737
 738         kfree(cxl_nvd);
 739 }
 740
 741 static const struct attribute_group *cxl_nvdimm_attribute_groups[] = {
 742         &cxl_base_attribute_group,
 743         NULL,
 744 };
 745
 746 static const struct device_type cxl_nvdimm_type = {
 747         .name = "cxl_nvdimm",
 748         .release = cxl_nvdimm_release,
 749         .groups = cxl_nvdimm_attribute_groups,
 750 };
 751
 752 bool is_cxl_nvdimm(struct device *dev)
 753 {
 754         return dev->type == &cxl_nvdimm_type;
 755 }
 756 EXPORT_SYMBOL_GPL(is_cxl_nvdimm);
 757
 758 struct cxl_nvdimm *to_cxl_nvdimm(struct device *dev)
 759 {
 760         if (dev_WARN_ONCE(dev, !is_cxl_nvdimm(dev),
 761                           "not a cxl_nvdimm device\n"))
 762                 return NULL;
 763         return container_of(dev, struct cxl_nvdimm, dev);
 764 }
 765 EXPORT_SYMBOL_GPL(to_cxl_nvdimm);
 766
 767 static struct cxl_nvdimm *cxl_nvdimm_alloc(struct cxl_memdev *cxlmd)
 768 {
 769         struct cxl_nvdimm *cxl_nvd;
 770         struct device *dev;
 771
 772         cxl_nvd = kzalloc(sizeof(*cxl_nvd), GFP_KERNEL);
 773         if (!cxl_nvd)
 774                 return ERR_PTR(-ENOMEM);
 775
 776         dev = &cxl_nvd->dev;
 777         cxl_nvd->cxlmd = cxlmd;
 778         device_initialize(dev);
 779         device_set_pm_not_required(dev);
 780         dev->parent = &cxlmd->dev;
 781         dev->bus = &cxl_bus_type;
 782         dev->type = &cxl_nvdimm_type;
 783
 784         return cxl_nvd;
 785 }
 786
 787 int devm_cxl_add_nvdimm(struct device *host, struct cxl_memdev *cxlmd)
 788 {
 789         struct cxl_nvdimm *cxl_nvd;
 790         struct device *dev;
 791         int rc;
 792
 793         cxl_nvd = cxl_nvdimm_alloc(cxlmd);
 794         if (IS_ERR(cxl_nvd))
 795                 return PTR_ERR(cxl_nvd);
 796
 797         dev = &cxl_nvd->dev;
 798         rc = dev_set_name(dev, "pmem%d", cxlmd->id);
 799         if (rc)
 800                 goto err;
 801
 802         rc = device_add(dev);
 803         if (rc)
 804                 goto err;
 805
 806         dev_dbg(host, "%s: register %s\n", dev_name(dev->parent),
 807                 dev_name(dev));
 808
 809         return devm_add_action_or_reset(host, unregister_dev, dev);
 810
 811 err:
 812         put_device(dev);
 813         return rc;
 814 }
 815 EXPORT_SYMBOL_GPL(devm_cxl_add_nvdimm);
 816
 817 /**
 818  * cxl_probe_device_regs() - Detect CXL Device register blocks
 819  * @dev: Host device of the @base mapping
 820  * @base: Mapping of CXL 2.0 8.2.8 CXL Device Register Interface
 821  * @map: Map object describing the register block information found
 822  *
 823  * Probe for device register information and return it in map object.
 824  */
 825 void cxl_probe_device_regs(struct device *dev, void __iomem *base,
 826                            struct cxl_device_reg_map *map)
 827 {
 828         int cap, cap_count;
 829         u64 cap_array;
 830
 831         *map = (struct cxl_device_reg_map){ 0 };
 832
 833         cap_array = readq(base + CXLDEV_CAP_ARRAY_OFFSET);
 834         if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) !=
 835             CXLDEV_CAP_ARRAY_CAP_ID)
 836                 return;
 837
 838         cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array);
 839
 840         for (cap = 1; cap <= cap_count; cap++) {
 841                 u32 offset, length;
 842                 u16 cap_id;
 843
 844                 cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK,
 845                                    readl(base + cap * 0x10));
 846                 offset = readl(base + cap * 0x10 + 0x4);
 847                 length = readl(base + cap * 0x10 + 0x8);
 848
 849                 switch (cap_id) {
 850                 case CXLDEV_CAP_CAP_ID_DEVICE_STATUS:
 851                         dev_dbg(dev, "found Status capability (0x%x)\n", offset);
 852
 853                         map->status.valid = true;
 854                         map->status.offset = offset;
 855                         map->status.size = length;
 856                         break;
 857                 case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX:
 858                         dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset);
 859                         map->mbox.valid = true;
 860                         map->mbox.offset = offset;
 861                         map->mbox.size = length;
 862                         break;
 863                 case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX:
 864                         dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset);
 865                         break;
 866                 case CXLDEV_CAP_CAP_ID_MEMDEV:
 867                         dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset);
 868                         map->memdev.valid = true;
 869                         map->memdev.offset = offset;
 870                         map->memdev.size = length;
 871                         break;
 872                 default:
 873                         if (cap_id >= 0x8000)
 874                                 dev_dbg(dev, "Vendor cap ID: %#x offset: %#x\n", cap_id, offset);
 875                         else
 876                                 dev_dbg(dev, "Unknown cap ID: %#x offset: %#x\n", cap_id, offset);
 877                         break;
 878                 }
 879         }
 880 }
 881 EXPORT_SYMBOL_GPL(cxl_probe_device_regs);
 882
 883 static void __iomem *devm_cxl_iomap_block(struct device *dev,
 884                                           resource_size_t addr,
 885                                           resource_size_t length)
 886 {
 887         void __iomem *ret_val;
 888         struct resource *res;
 889
 890         res = devm_request_mem_region(dev, addr, length, dev_name(dev));
 891         if (!res) {
 892                 resource_size_t end = addr + length - 1;
 893
 894                 dev_err(dev, "Failed to request region %pa-%pa\n", &addr, &end);
 895                 return NULL;
 896         }
 897
 898         ret_val = devm_ioremap(dev, addr, length);
 899         if (!ret_val)
 900                 dev_err(dev, "Failed to map region %pr\n", res);
 901
 902         return ret_val;
 903 }
 904
 905 int cxl_map_component_regs(struct pci_dev *pdev,
 906                            struct cxl_component_regs *regs,
 907                            struct cxl_register_map *map)
 908 {
 909         struct device *dev = &pdev->dev;
 910         resource_size_t phys_addr;
 911         resource_size_t length;
 912
 913         phys_addr = pci_resource_start(pdev, map->barno);
 914         phys_addr += map->block_offset;
 915
 916         phys_addr += map->component_map.hdm_decoder.offset;
 917         length = map->component_map.hdm_decoder.size;
 918         regs->hdm_decoder = devm_cxl_iomap_block(dev, phys_addr, length);
 919         if (!regs->hdm_decoder)
 920                 return -ENOMEM;
 921
 922         return 0;
 923 }
 924 EXPORT_SYMBOL_GPL(cxl_map_component_regs);
 925
 926 int cxl_map_device_regs(struct pci_dev *pdev,
 927                         struct cxl_device_regs *regs,
 928                         struct cxl_register_map *map)
 929 {
 930         struct device *dev = &pdev->dev;
 931         resource_size_t phys_addr;
 932
 933         phys_addr = pci_resource_start(pdev, map->barno);
 934         phys_addr += map->block_offset;
 935
 936         if (map->device_map.status.valid) {
 937                 resource_size_t addr;
 938                 resource_size_t length;
 939
 940                 addr = phys_addr + map->device_map.status.offset;
 941                 length = map->device_map.status.size;
 942                 regs->status = devm_cxl_iomap_block(dev, addr, length);
 943                 if (!regs->status)
 944                         return -ENOMEM;
 945         }
 946
 947         if (map->device_map.mbox.valid) {
 948                 resource_size_t addr;
 949                 resource_size_t length;
 950
 951                 addr = phys_addr + map->device_map.mbox.offset;
 952                 length = map->device_map.mbox.size;
 953                 regs->mbox = devm_cxl_iomap_block(dev, addr, length);
 954                 if (!regs->mbox)
 955                         return -ENOMEM;
 956         }
 957
 958         if (map->device_map.memdev.valid) {
 959                 resource_size_t addr;
 960                 resource_size_t length;
 961
 962                 addr = phys_addr + map->device_map.memdev.offset;
 963                 length = map->device_map.memdev.size;
 964                 regs->memdev = devm_cxl_iomap_block(dev, addr, length);
 965                 if (!regs->memdev)
 966                         return -ENOMEM;
 967         }
 968
 969         return 0;
 970 }
 971 EXPORT_SYMBOL_GPL(cxl_map_device_regs);
 972
 973 /**
 974  * __cxl_driver_register - register a driver for the cxl bus
 975  * @cxl_drv: cxl driver structure to attach
 976  * @owner: owning module/driver
 977  * @modname: KBUILD_MODNAME for parent driver
 978  */
 979 int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
 980                           const char *modname)
 981 {
 982         if (!cxl_drv->probe) {
 983                 pr_debug("%s ->probe() must be specified\n", modname);
 984                 return -EINVAL;
 985         }
 986
 987         if (!cxl_drv->name) {
 988                 pr_debug("%s ->name must be specified\n", modname);
 989                 return -EINVAL;
 990         }
 991
 992         if (!cxl_drv->id) {
 993                 pr_debug("%s ->id must be specified\n", modname);
 994                 return -EINVAL;
 995         }
 996
 997         cxl_drv->drv.bus = &cxl_bus_type;
 998         cxl_drv->drv.owner = owner;
 999         cxl_drv->drv.mod_name = modname;
1000         cxl_drv->drv.name = cxl_drv->name;
1001
1002         return driver_register(&cxl_drv->drv);
1003 }
1004 EXPORT_SYMBOL_GPL(__cxl_driver_register);
1005
1006 void cxl_driver_unregister(struct cxl_driver *cxl_drv)
1007 {
1008         driver_unregister(&cxl_drv->drv);
1009 }
1010 EXPORT_SYMBOL_GPL(cxl_driver_unregister);
1011
1012 static int cxl_device_id(struct device *dev)
1013 {
1014         if (dev->type == &cxl_nvdimm_bridge_type)
1015                 return CXL_DEVICE_NVDIMM_BRIDGE;
1016         if (dev->type == &cxl_nvdimm_type)
1017                 return CXL_DEVICE_NVDIMM;
1018         return 0;
1019 }
1020
1021 static int cxl_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
1022 {
1023         return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,
1024                               cxl_device_id(dev));
1025 }
1026
1027 static int cxl_bus_match(struct device *dev, struct device_driver *drv)
1028 {
1029         return cxl_device_id(dev) == to_cxl_drv(drv)->id;
1030 }
1031
1032 static int cxl_bus_probe(struct device *dev)
1033 {
1034         return to_cxl_drv(dev->driver)->probe(dev);
1035 }
1036
1037 static void cxl_bus_remove(struct device *dev)
1038 {
1039         struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
1040
1041         if (cxl_drv->remove)
1042                 cxl_drv->remove(dev);
1043 }
1044
1045 struct bus_type cxl_bus_type = {
1046         .name = "cxl",
1047         .uevent = cxl_bus_uevent,
1048         .match = cxl_bus_match,
1049         .probe = cxl_bus_probe,
1050         .remove = cxl_bus_remove,
1051 };
1052 EXPORT_SYMBOL_GPL(cxl_bus_type);
1053
1054 static __init int cxl_core_init(void)
1055 {
1056         return bus_register(&cxl_bus_type);
1057 }
1058
1059 static void cxl_core_exit(void)
1060 {
1061         bus_unregister(&cxl_bus_type);
1062 }
1063
1064 module_init(cxl_core_init);
1065 module_exit(cxl_core_exit);
1066 MODULE_LICENSE("GPL v2");