1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3 #include <linux/io-64-nonatomic-lo-hi.h>
4 #include <linux/workqueue.h>
5 #include <linux/device.h>
6 #include <linux/module.h>
8 #include <linux/slab.h>
18 * The CXL core provides a set of interfaces that can be consumed by CXL aware
19 * drivers. The interfaces allow for creation, modification, and destruction of
20 * regions, memory devices, ports, and decoders. CXL aware drivers must register
21 * with the CXL core via these interfaces in order to be able to participate in
22 * cross-device interleave coordination. The CXL core also establishes and
23 * maintains the bridge to the nvdimm subsystem.
25 * CXL core introduces sysfs hierarchy to control the devices that are
26 * instantiated by the core.
29 static DEFINE_IDA(cxl_port_ida);
30 static DEFINE_XARRAY(cxl_root_buses);
32 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
35 return sysfs_emit(buf, "%s\n", dev->type->name);
37 static DEVICE_ATTR_RO(devtype);
39 static int cxl_device_id(struct device *dev)
41 if (dev->type == &cxl_nvdimm_bridge_type)
42 return CXL_DEVICE_NVDIMM_BRIDGE;
43 if (dev->type == &cxl_nvdimm_type)
44 return CXL_DEVICE_NVDIMM;
45 if (is_cxl_port(dev)) {
46 if (is_cxl_root(to_cxl_port(dev)))
47 return CXL_DEVICE_ROOT;
48 return CXL_DEVICE_PORT;
50 if (is_cxl_memdev(dev))
51 return CXL_DEVICE_MEMORY_EXPANDER;
55 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
58 return sysfs_emit(buf, CXL_MODALIAS_FMT "\n", cxl_device_id(dev));
60 static DEVICE_ATTR_RO(modalias);
62 static struct attribute *cxl_base_attributes[] = {
63 &dev_attr_devtype.attr,
64 &dev_attr_modalias.attr,
68 struct attribute_group cxl_base_attribute_group = {
69 .attrs = cxl_base_attributes,
72 static ssize_t start_show(struct device *dev, struct device_attribute *attr,
75 struct cxl_decoder *cxld = to_cxl_decoder(dev);
78 if (is_root_decoder(dev))
79 start = cxld->platform_res.start;
81 start = cxld->decoder_range.start;
83 return sysfs_emit(buf, "%#llx\n", start);
85 static DEVICE_ATTR_ADMIN_RO(start);
87 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
90 struct cxl_decoder *cxld = to_cxl_decoder(dev);
93 if (is_root_decoder(dev))
94 size = resource_size(&cxld->platform_res);
96 size = range_len(&cxld->decoder_range);
98 return sysfs_emit(buf, "%#llx\n", size);
100 static DEVICE_ATTR_RO(size);
102 #define CXL_DECODER_FLAG_ATTR(name, flag) \
103 static ssize_t name##_show(struct device *dev, \
104 struct device_attribute *attr, char *buf) \
106 struct cxl_decoder *cxld = to_cxl_decoder(dev); \
108 return sysfs_emit(buf, "%s\n", \
109 (cxld->flags & (flag)) ? "1" : "0"); \
111 static DEVICE_ATTR_RO(name)
113 CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
114 CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
115 CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
116 CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
117 CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
119 static ssize_t target_type_show(struct device *dev,
120 struct device_attribute *attr, char *buf)
122 struct cxl_decoder *cxld = to_cxl_decoder(dev);
124 switch (cxld->target_type) {
125 case CXL_DECODER_ACCELERATOR:
126 return sysfs_emit(buf, "accelerator\n");
127 case CXL_DECODER_EXPANDER:
128 return sysfs_emit(buf, "expander\n");
132 static DEVICE_ATTR_RO(target_type);
134 static ssize_t emit_target_list(struct cxl_decoder *cxld, char *buf)
139 for (i = 0; i < cxld->interleave_ways; i++) {
140 struct cxl_dport *dport = cxld->target[i];
141 struct cxl_dport *next = NULL;
146 if (i + 1 < cxld->interleave_ways)
147 next = cxld->target[i + 1];
148 rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
158 static ssize_t target_list_show(struct device *dev,
159 struct device_attribute *attr, char *buf)
161 struct cxl_decoder *cxld = to_cxl_decoder(dev);
167 seq = read_seqbegin(&cxld->target_lock);
168 rc = emit_target_list(cxld, buf);
169 } while (read_seqretry(&cxld->target_lock, seq));
175 rc = sysfs_emit_at(buf, offset, "\n");
181 static DEVICE_ATTR_RO(target_list);
183 static struct attribute *cxl_decoder_base_attrs[] = {
184 &dev_attr_start.attr,
186 &dev_attr_locked.attr,
190 static struct attribute_group cxl_decoder_base_attribute_group = {
191 .attrs = cxl_decoder_base_attrs,
194 static struct attribute *cxl_decoder_root_attrs[] = {
195 &dev_attr_cap_pmem.attr,
196 &dev_attr_cap_ram.attr,
197 &dev_attr_cap_type2.attr,
198 &dev_attr_cap_type3.attr,
199 &dev_attr_target_list.attr,
203 static struct attribute_group cxl_decoder_root_attribute_group = {
204 .attrs = cxl_decoder_root_attrs,
207 static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
208 &cxl_decoder_root_attribute_group,
209 &cxl_decoder_base_attribute_group,
210 &cxl_base_attribute_group,
214 static struct attribute *cxl_decoder_switch_attrs[] = {
215 &dev_attr_target_type.attr,
216 &dev_attr_target_list.attr,
220 static struct attribute_group cxl_decoder_switch_attribute_group = {
221 .attrs = cxl_decoder_switch_attrs,
224 static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
225 &cxl_decoder_switch_attribute_group,
226 &cxl_decoder_base_attribute_group,
227 &cxl_base_attribute_group,
231 static struct attribute *cxl_decoder_endpoint_attrs[] = {
232 &dev_attr_target_type.attr,
236 static struct attribute_group cxl_decoder_endpoint_attribute_group = {
237 .attrs = cxl_decoder_endpoint_attrs,
240 static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = {
241 &cxl_decoder_base_attribute_group,
242 &cxl_decoder_endpoint_attribute_group,
243 &cxl_base_attribute_group,
247 static void cxl_decoder_release(struct device *dev)
249 struct cxl_decoder *cxld = to_cxl_decoder(dev);
250 struct cxl_port *port = to_cxl_port(dev->parent);
252 ida_free(&port->decoder_ida, cxld->id);
254 put_device(&port->dev);
257 static const struct device_type cxl_decoder_endpoint_type = {
258 .name = "cxl_decoder_endpoint",
259 .release = cxl_decoder_release,
260 .groups = cxl_decoder_endpoint_attribute_groups,
263 static const struct device_type cxl_decoder_switch_type = {
264 .name = "cxl_decoder_switch",
265 .release = cxl_decoder_release,
266 .groups = cxl_decoder_switch_attribute_groups,
269 static const struct device_type cxl_decoder_root_type = {
270 .name = "cxl_decoder_root",
271 .release = cxl_decoder_release,
272 .groups = cxl_decoder_root_attribute_groups,
275 bool is_endpoint_decoder(struct device *dev)
277 return dev->type == &cxl_decoder_endpoint_type;
280 bool is_root_decoder(struct device *dev)
282 return dev->type == &cxl_decoder_root_type;
284 EXPORT_SYMBOL_NS_GPL(is_root_decoder, CXL);
286 bool is_cxl_decoder(struct device *dev)
288 return dev->type && dev->type->release == cxl_decoder_release;
290 EXPORT_SYMBOL_NS_GPL(is_cxl_decoder, CXL);
292 struct cxl_decoder *to_cxl_decoder(struct device *dev)
294 if (dev_WARN_ONCE(dev, dev->type->release != cxl_decoder_release,
295 "not a cxl_decoder device\n"))
297 return container_of(dev, struct cxl_decoder, dev);
299 EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, CXL);
301 static void cxl_ep_release(struct cxl_ep *ep)
310 static void cxl_port_release(struct device *dev)
312 struct cxl_port *port = to_cxl_port(dev);
313 struct cxl_ep *ep, *_e;
316 list_for_each_entry_safe(ep, _e, &port->endpoints, list)
319 ida_free(&cxl_port_ida, port->id);
323 static const struct attribute_group *cxl_port_attribute_groups[] = {
324 &cxl_base_attribute_group,
328 static const struct device_type cxl_port_type = {
330 .release = cxl_port_release,
331 .groups = cxl_port_attribute_groups,
334 bool is_cxl_port(struct device *dev)
336 return dev->type == &cxl_port_type;
338 EXPORT_SYMBOL_NS_GPL(is_cxl_port, CXL);
340 struct cxl_port *to_cxl_port(struct device *dev)
342 if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
343 "not a cxl_port device\n"))
345 return container_of(dev, struct cxl_port, dev);
347 EXPORT_SYMBOL_NS_GPL(to_cxl_port, CXL);
349 static void unregister_port(void *_port)
351 struct cxl_port *port = _port;
352 struct cxl_port *parent;
353 struct device *lock_dev;
355 if (is_cxl_root(port))
358 parent = to_cxl_port(port->dev.parent);
361 * CXL root port's and the first level of ports are unregistered
362 * under the platform firmware device lock, all other ports are
363 * unregistered while holding their parent port lock.
366 lock_dev = port->uport;
367 else if (is_cxl_root(parent))
368 lock_dev = parent->uport;
370 lock_dev = &parent->dev;
372 device_lock_assert(lock_dev);
374 device_unregister(&port->dev);
377 static void cxl_unlink_uport(void *_port)
379 struct cxl_port *port = _port;
381 sysfs_remove_link(&port->dev.kobj, "uport");
384 static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
388 rc = sysfs_create_link(&port->dev.kobj, &port->uport->kobj, "uport");
391 return devm_add_action_or_reset(host, cxl_unlink_uport, port);
394 static struct lock_class_key cxl_port_key;
396 static struct cxl_port *cxl_port_alloc(struct device *uport,
397 resource_size_t component_reg_phys,
398 struct cxl_port *parent_port)
400 struct cxl_port *port;
404 port = kzalloc(sizeof(*port), GFP_KERNEL);
406 return ERR_PTR(-ENOMEM);
408 rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
414 * The top-level cxl_port "cxl_root" does not have a cxl_port as
415 * its parent and it does not have any corresponding component
416 * registers as its decode is described by a fixed platform
421 dev->parent = &parent_port->dev;
422 port->depth = parent_port->depth + 1;
427 port->component_reg_phys = component_reg_phys;
428 ida_init(&port->decoder_ida);
429 INIT_LIST_HEAD(&port->dports);
430 INIT_LIST_HEAD(&port->endpoints);
432 device_initialize(dev);
433 lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
434 device_set_pm_not_required(dev);
435 dev->bus = &cxl_bus_type;
436 dev->type = &cxl_port_type;
446 * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
447 * @host: host device for devm operations
448 * @uport: "physical" device implementing this upstream port
449 * @component_reg_phys: (optional) for configurable cxl_port instances
450 * @parent_port: next hop up in the CXL memory decode hierarchy
452 struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport,
453 resource_size_t component_reg_phys,
454 struct cxl_port *parent_port)
456 struct cxl_port *port;
460 port = cxl_port_alloc(uport, component_reg_phys, parent_port);
465 if (is_cxl_memdev(uport))
466 rc = dev_set_name(dev, "endpoint%d", port->id);
467 else if (parent_port)
468 rc = dev_set_name(dev, "port%d", port->id);
470 rc = dev_set_name(dev, "root%d", port->id);
474 rc = device_add(dev);
478 rc = devm_add_action_or_reset(host, unregister_port, port);
482 rc = devm_cxl_link_uport(host, port);
492 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, CXL);
494 struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port)
496 /* There is no pci_bus associated with a CXL platform-root port */
497 if (is_cxl_root(port))
500 if (dev_is_pci(port->uport)) {
501 struct pci_dev *pdev = to_pci_dev(port->uport);
503 return pdev->subordinate;
506 return xa_load(&cxl_root_buses, (unsigned long)port->uport);
508 EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, CXL);
510 static void unregister_pci_bus(void *uport)
512 xa_erase(&cxl_root_buses, (unsigned long)uport);
515 int devm_cxl_register_pci_bus(struct device *host, struct device *uport,
520 if (dev_is_pci(uport))
523 rc = xa_insert(&cxl_root_buses, (unsigned long)uport, bus, GFP_KERNEL);
526 return devm_add_action_or_reset(host, unregister_pci_bus, uport);
528 EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, CXL);
530 static bool dev_is_cxl_root_child(struct device *dev)
532 struct cxl_port *port, *parent;
534 if (!is_cxl_port(dev))
537 port = to_cxl_port(dev);
538 if (is_cxl_root(port))
541 parent = to_cxl_port(port->dev.parent);
542 if (is_cxl_root(parent))
548 /* Find a 2nd level CXL port that has a dport that is an ancestor of @match */
549 static int match_root_child(struct device *dev, const void *match)
551 const struct device *iter = NULL;
552 struct cxl_dport *dport;
553 struct cxl_port *port;
555 if (!dev_is_cxl_root_child(dev))
558 port = to_cxl_port(dev);
560 list_for_each_entry(dport, &port->dports, list) {
563 if (iter == dport->dport)
574 struct cxl_port *find_cxl_root(struct device *dev)
576 struct device *port_dev;
577 struct cxl_port *root;
579 port_dev = bus_find_device(&cxl_bus_type, NULL, dev, match_root_child);
583 root = to_cxl_port(port_dev->parent);
584 get_device(&root->dev);
585 put_device(port_dev);
588 EXPORT_SYMBOL_NS_GPL(find_cxl_root, CXL);
590 static struct cxl_dport *find_dport(struct cxl_port *port, int id)
592 struct cxl_dport *dport;
594 device_lock_assert(&port->dev);
595 list_for_each_entry (dport, &port->dports, list)
596 if (dport->port_id == id)
601 static int add_dport(struct cxl_port *port, struct cxl_dport *new)
603 struct cxl_dport *dup;
605 device_lock_assert(&port->dev);
606 dup = find_dport(port, new->port_id);
609 "unable to add dport%d-%s non-unique port id (%s)\n",
610 new->port_id, dev_name(new->dport),
611 dev_name(dup->dport));
613 list_add_tail(&new->list, &port->dports);
615 return dup ? -EEXIST : 0;
619 * Since root-level CXL dports cannot be enumerated by PCI they are not
620 * enumerated by the common port driver that acquires the port lock over
621 * dport add/remove. Instead, root dports are manually added by a
622 * platform driver and cond_cxl_root_lock() is used to take the missing
623 * port lock in that case.
625 static void cond_cxl_root_lock(struct cxl_port *port)
627 if (is_cxl_root(port))
628 device_lock(&port->dev);
631 static void cond_cxl_root_unlock(struct cxl_port *port)
633 if (is_cxl_root(port))
634 device_unlock(&port->dev);
637 static void cxl_dport_remove(void *data)
639 struct cxl_dport *dport = data;
640 struct cxl_port *port = dport->port;
642 put_device(dport->dport);
643 cond_cxl_root_lock(port);
644 list_del(&dport->list);
645 cond_cxl_root_unlock(port);
648 static void cxl_dport_unlink(void *data)
650 struct cxl_dport *dport = data;
651 struct cxl_port *port = dport->port;
652 char link_name[CXL_TARGET_STRLEN];
654 sprintf(link_name, "dport%d", dport->port_id);
655 sysfs_remove_link(&port->dev.kobj, link_name);
659 * devm_cxl_add_dport - append downstream port data to a cxl_port
660 * @port: the cxl_port that references this dport
661 * @dport_dev: firmware or PCI device representing the dport
662 * @port_id: identifier for this dport in a decoder's target list
663 * @component_reg_phys: optional location of CXL component registers
665 * Note that dports are appended to the devm release action's of the
666 * either the port's host (for root ports), or the port itself (for
669 struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
670 struct device *dport_dev, int port_id,
671 resource_size_t component_reg_phys)
673 char link_name[CXL_TARGET_STRLEN];
674 struct cxl_dport *dport;
678 if (is_cxl_root(port))
684 dev_WARN_ONCE(&port->dev, 1, "dport:%s bad devm context\n",
685 dev_name(dport_dev));
686 return ERR_PTR(-ENXIO);
689 if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
691 return ERR_PTR(-EINVAL);
693 dport = devm_kzalloc(host, sizeof(*dport), GFP_KERNEL);
695 return ERR_PTR(-ENOMEM);
697 INIT_LIST_HEAD(&dport->list);
698 dport->dport = dport_dev;
699 dport->port_id = port_id;
700 dport->component_reg_phys = component_reg_phys;
703 cond_cxl_root_lock(port);
704 rc = add_dport(port, dport);
705 cond_cxl_root_unlock(port);
709 get_device(dport_dev);
710 rc = devm_add_action_or_reset(host, cxl_dport_remove, dport);
714 rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
718 rc = devm_add_action_or_reset(host, cxl_dport_unlink, dport);
724 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL);
726 static struct cxl_ep *find_ep(struct cxl_port *port, struct device *ep_dev)
730 device_lock_assert(&port->dev);
731 list_for_each_entry(ep, &port->endpoints, list)
732 if (ep->ep == ep_dev)
737 static int add_ep(struct cxl_port *port, struct cxl_ep *new)
741 device_lock(&port->dev);
743 device_unlock(&port->dev);
746 dup = find_ep(port, new->ep);
748 list_add_tail(&new->list, &port->endpoints);
749 device_unlock(&port->dev);
751 return dup ? -EEXIST : 0;
755 * cxl_add_ep - register an endpoint's interest in a port
756 * @port: a port in the endpoint's topology ancestry
757 * @ep_dev: device representing the endpoint
759 * Intermediate CXL ports are scanned based on the arrival of endpoints.
760 * When those endpoints depart the port can be destroyed once all
761 * endpoints that care about that port have been removed.
763 static int cxl_add_ep(struct cxl_port *port, struct device *ep_dev)
768 ep = kzalloc(sizeof(*ep), GFP_KERNEL);
772 INIT_LIST_HEAD(&ep->list);
773 ep->ep = get_device(ep_dev);
775 rc = add_ep(port, ep);
781 struct cxl_find_port_ctx {
782 const struct device *dport_dev;
783 const struct cxl_port *parent_port;
786 static int match_port_by_dport(struct device *dev, const void *data)
788 const struct cxl_find_port_ctx *ctx = data;
789 struct cxl_port *port;
791 if (!is_cxl_port(dev))
793 if (ctx->parent_port && dev->parent != &ctx->parent_port->dev)
796 port = to_cxl_port(dev);
797 return cxl_find_dport_by_dev(port, ctx->dport_dev) != NULL;
800 static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx)
807 dev = bus_find_device(&cxl_bus_type, NULL, ctx, match_port_by_dport);
809 return to_cxl_port(dev);
813 static struct cxl_port *find_cxl_port(struct device *dport_dev)
815 struct cxl_find_port_ctx ctx = {
816 .dport_dev = dport_dev,
819 return __find_cxl_port(&ctx);
822 static struct cxl_port *find_cxl_port_at(struct cxl_port *parent_port,
823 struct device *dport_dev)
825 struct cxl_find_port_ctx ctx = {
826 .dport_dev = dport_dev,
827 .parent_port = parent_port,
830 return __find_cxl_port(&ctx);
834 * All users of grandparent() are using it to walk PCIe-like swich port
835 * hierarchy. A PCIe switch is comprised of a bridge device representing the
836 * upstream switch port and N bridges representing downstream switch ports. When
837 * bridges stack the grand-parent of a downstream switch port is another
838 * downstream switch port in the immediate ancestor switch.
840 static struct device *grandparent(struct device *dev)
842 if (dev && dev->parent)
843 return dev->parent->parent;
847 static void delete_endpoint(void *data)
849 struct cxl_memdev *cxlmd = data;
850 struct cxl_port *endpoint = dev_get_drvdata(&cxlmd->dev);
851 struct cxl_port *parent_port;
852 struct device *parent;
854 parent_port = cxl_mem_find_port(cxlmd);
857 parent = &parent_port->dev;
860 if (parent->driver && endpoint->uport) {
861 devm_release_action(parent, cxl_unlink_uport, endpoint);
862 devm_release_action(parent, unregister_port, endpoint);
864 device_unlock(parent);
867 put_device(&endpoint->dev);
870 int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
872 struct device *dev = &cxlmd->dev;
874 get_device(&endpoint->dev);
875 dev_set_drvdata(dev, endpoint);
876 return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
878 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, CXL);
881 * The natural end of life of a non-root 'cxl_port' is when its parent port goes
882 * through a ->remove() event ("top-down" unregistration). The unnatural trigger
883 * for a port to be unregistered is when all memdevs beneath that port have gone
884 * through ->remove(). This "bottom-up" removal selectively removes individual
885 * child ports manually. This depends on devm_cxl_add_port() to not change is
886 * devm action registration order.
888 static void delete_switch_port(struct cxl_port *port, struct list_head *dports)
890 struct cxl_dport *dport, *_d;
892 list_for_each_entry_safe(dport, _d, dports, list) {
893 devm_release_action(&port->dev, cxl_dport_unlink, dport);
894 devm_release_action(&port->dev, cxl_dport_remove, dport);
895 devm_kfree(&port->dev, dport);
897 devm_release_action(port->dev.parent, cxl_unlink_uport, port);
898 devm_release_action(port->dev.parent, unregister_port, port);
901 static void cxl_detach_ep(void *data)
903 struct cxl_memdev *cxlmd = data;
906 for (iter = &cxlmd->dev; iter; iter = grandparent(iter)) {
907 struct device *dport_dev = grandparent(iter);
908 struct cxl_port *port, *parent_port;
909 LIST_HEAD(reap_dports);
915 port = find_cxl_port(dport_dev);
919 if (is_cxl_root(port)) {
920 put_device(&port->dev);
924 parent_port = to_cxl_port(port->dev.parent);
925 device_lock(&parent_port->dev);
926 if (!parent_port->dev.driver) {
928 * The bottom-up race to delete the port lost to a
929 * top-down port disable, give up here, because the
930 * parent_port ->remove() will have cleaned up all
933 device_unlock(&parent_port->dev);
934 put_device(&port->dev);
938 device_lock(&port->dev);
939 ep = find_ep(port, &cxlmd->dev);
940 dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
941 ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
943 if (ep && !port->dead && list_empty(&port->endpoints) &&
944 !is_cxl_root(parent_port)) {
946 * This was the last ep attached to a dynamically
947 * enumerated port. Block new cxl_add_ep() and garbage
951 list_splice_init(&port->dports, &reap_dports);
953 device_unlock(&port->dev);
955 if (!list_empty(&reap_dports)) {
956 dev_dbg(&cxlmd->dev, "delete %s\n",
957 dev_name(&port->dev));
958 delete_switch_port(port, &reap_dports);
960 put_device(&port->dev);
961 device_unlock(&parent_port->dev);
965 static resource_size_t find_component_registers(struct device *dev)
967 struct cxl_register_map map;
968 struct pci_dev *pdev;
971 * Theoretically, CXL component registers can be hosted on a
972 * non-PCI device, in practice, only cxl_test hits this case.
974 if (!dev_is_pci(dev))
975 return CXL_RESOURCE_NONE;
977 pdev = to_pci_dev(dev);
979 cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
980 return cxl_regmap_to_base(pdev, &map);
983 static int add_port_attach_ep(struct cxl_memdev *cxlmd,
984 struct device *uport_dev,
985 struct device *dport_dev)
987 struct device *dparent = grandparent(dport_dev);
988 struct cxl_port *port, *parent_port = NULL;
989 resource_size_t component_reg_phys;
994 * The iteration reached the topology root without finding the
995 * CXL-root 'cxl_port' on a previous iteration, fail for now to
996 * be re-probed after platform driver attaches.
998 dev_dbg(&cxlmd->dev, "%s is a root dport\n",
999 dev_name(dport_dev));
1003 parent_port = find_cxl_port(dparent);
1005 /* iterate to create this parent_port */
1009 device_lock(&parent_port->dev);
1010 if (!parent_port->dev.driver) {
1011 dev_warn(&cxlmd->dev,
1012 "port %s:%s disabled, failed to enumerate CXL.mem\n",
1013 dev_name(&parent_port->dev), dev_name(uport_dev));
1014 port = ERR_PTR(-ENXIO);
1018 port = find_cxl_port_at(parent_port, dport_dev);
1020 component_reg_phys = find_component_registers(uport_dev);
1021 port = devm_cxl_add_port(&parent_port->dev, uport_dev,
1022 component_reg_phys, parent_port);
1024 get_device(&port->dev);
1027 device_unlock(&parent_port->dev);
1032 dev_dbg(&cxlmd->dev, "add to new port %s:%s\n",
1033 dev_name(&port->dev), dev_name(port->uport));
1034 rc = cxl_add_ep(port, &cxlmd->dev);
1035 if (rc == -EEXIST) {
1037 * "can't" happen, but this error code means
1038 * something to the caller, so translate it.
1042 put_device(&port->dev);
1045 put_device(&parent_port->dev);
1049 int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
1051 struct device *dev = &cxlmd->dev;
1052 struct device *iter;
1055 rc = devm_add_action_or_reset(&cxlmd->dev, cxl_detach_ep, cxlmd);
1060 * Scan for and add all cxl_ports in this device's ancestry.
1061 * Repeat until no more ports are added. Abort if a port add
1065 for (iter = dev; iter; iter = grandparent(iter)) {
1066 struct device *dport_dev = grandparent(iter);
1067 struct device *uport_dev;
1068 struct cxl_port *port;
1073 uport_dev = dport_dev->parent;
1075 dev_warn(dev, "at %s no parent for dport: %s\n",
1076 dev_name(iter), dev_name(dport_dev));
1080 dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
1081 dev_name(iter), dev_name(dport_dev),
1082 dev_name(uport_dev));
1083 port = find_cxl_port(dport_dev);
1085 dev_dbg(&cxlmd->dev,
1086 "found already registered port %s:%s\n",
1087 dev_name(&port->dev), dev_name(port->uport));
1088 rc = cxl_add_ep(port, &cxlmd->dev);
1091 * If the endpoint already exists in the port's list,
1092 * that's ok, it was added on a previous pass.
1093 * Otherwise, retry in add_port_attach_ep() after taking
1094 * the parent_port lock as the current port may be being
1097 if (rc && rc != -EEXIST) {
1098 put_device(&port->dev);
1102 /* Any more ports to add between this one and the root? */
1103 if (!dev_is_cxl_root_child(&port->dev)) {
1104 put_device(&port->dev);
1108 put_device(&port->dev);
1112 rc = add_port_attach_ep(cxlmd, uport_dev, dport_dev);
1113 /* port missing, try to add parent */
1116 /* failed to add ep or port */
1119 /* port added, new descendants possible, start over */
1125 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL);
1127 struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd)
1129 return find_cxl_port(grandparent(&cxlmd->dev));
1131 EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, CXL);
1133 struct cxl_dport *cxl_find_dport_by_dev(struct cxl_port *port,
1134 const struct device *dev)
1136 struct cxl_dport *dport;
1138 device_lock(&port->dev);
1139 list_for_each_entry(dport, &port->dports, list)
1140 if (dport->dport == dev) {
1141 device_unlock(&port->dev);
1145 device_unlock(&port->dev);
1148 EXPORT_SYMBOL_NS_GPL(cxl_find_dport_by_dev, CXL);
1150 static int decoder_populate_targets(struct cxl_decoder *cxld,
1151 struct cxl_port *port, int *target_map)
1158 device_lock_assert(&port->dev);
1160 if (list_empty(&port->dports))
1163 write_seqlock(&cxld->target_lock);
1164 for (i = 0; i < cxld->nr_targets; i++) {
1165 struct cxl_dport *dport = find_dport(port, target_map[i]);
1171 cxld->target[i] = dport;
1173 write_sequnlock(&cxld->target_lock);
1178 static struct lock_class_key cxl_decoder_key;
1181 * cxl_decoder_alloc - Allocate a new CXL decoder
1182 * @port: owning port of this decoder
1183 * @nr_targets: downstream targets accessible by this decoder. All upstream
1184 * ports and root ports must have at least 1 target. Endpoint
1185 * devices will have 0 targets. Callers wishing to register an
1186 * endpoint device should specify 0.
1188 * A port should contain one or more decoders. Each of those decoders enable
1189 * some address space for CXL.mem utilization. A decoder is expected to be
1190 * configured by the caller before registering.
1192 * Return: A new cxl decoder to be registered by cxl_decoder_add(). The decoder
1193 * is initialized to be a "passthrough" decoder.
1195 static struct cxl_decoder *cxl_decoder_alloc(struct cxl_port *port,
1196 unsigned int nr_targets)
1198 struct cxl_decoder *cxld;
1202 if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1203 return ERR_PTR(-EINVAL);
1205 cxld = kzalloc(struct_size(cxld, target, nr_targets), GFP_KERNEL);
1207 return ERR_PTR(-ENOMEM);
1209 rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
1213 /* need parent to stick around to release the id */
1214 get_device(&port->dev);
1217 cxld->nr_targets = nr_targets;
1218 seqlock_init(&cxld->target_lock);
1220 device_initialize(dev);
1221 lockdep_set_class(&dev->mutex, &cxl_decoder_key);
1222 device_set_pm_not_required(dev);
1223 dev->parent = &port->dev;
1224 dev->bus = &cxl_bus_type;
1225 if (is_cxl_root(port))
1226 cxld->dev.type = &cxl_decoder_root_type;
1227 else if (is_cxl_endpoint(port))
1228 cxld->dev.type = &cxl_decoder_endpoint_type;
1230 cxld->dev.type = &cxl_decoder_switch_type;
1232 /* Pre initialize an "empty" decoder */
1233 cxld->interleave_ways = 1;
1234 cxld->interleave_granularity = PAGE_SIZE;
1235 cxld->target_type = CXL_DECODER_EXPANDER;
1236 cxld->platform_res = (struct resource)DEFINE_RES_MEM(0, 0);
1245 * cxl_root_decoder_alloc - Allocate a root level decoder
1246 * @port: owning CXL root of this decoder
1247 * @nr_targets: static number of downstream targets
1249 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1250 * 'CXL root' decoder is one that decodes from a top-level / static platform
1251 * firmware description of CXL resources into a CXL standard decode
1254 struct cxl_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
1255 unsigned int nr_targets)
1257 if (!is_cxl_root(port))
1258 return ERR_PTR(-EINVAL);
1260 return cxl_decoder_alloc(port, nr_targets);
1262 EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL);
1265 * cxl_switch_decoder_alloc - Allocate a switch level decoder
1266 * @port: owning CXL switch port of this decoder
1267 * @nr_targets: max number of dynamically addressable downstream targets
1269 * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1270 * 'switch' decoder is any decoder that can be enumerated by PCIe
1271 * topology and the HDM Decoder Capability. This includes the decoders
1272 * that sit between Switch Upstream Ports / Switch Downstream Ports and
1273 * Host Bridges / Root Ports.
1275 struct cxl_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
1276 unsigned int nr_targets)
1278 if (is_cxl_root(port) || is_cxl_endpoint(port))
1279 return ERR_PTR(-EINVAL);
1281 return cxl_decoder_alloc(port, nr_targets);
1283 EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, CXL);
1286 * cxl_endpoint_decoder_alloc - Allocate an endpoint decoder
1287 * @port: owning port of this decoder
1289 * Return: A new cxl decoder to be registered by cxl_decoder_add()
1291 struct cxl_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port)
1293 if (!is_cxl_endpoint(port))
1294 return ERR_PTR(-EINVAL);
1296 return cxl_decoder_alloc(port, 0);
1298 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, CXL);
1301 * cxl_decoder_add_locked - Add a decoder with targets
1302 * @cxld: The cxl decoder allocated by cxl_decoder_alloc()
1303 * @target_map: A list of downstream ports that this decoder can direct memory
1304 * traffic to. These numbers should correspond with the port number
1305 * in the PCIe Link Capabilities structure.
1307 * Certain types of decoders may not have any targets. The main example of this
1308 * is an endpoint device. A more awkward example is a hostbridge whose root
1309 * ports get hot added (technically possible, though unlikely).
1311 * This is the locked variant of cxl_decoder_add().
1313 * Context: Process context. Expects the device lock of the port that owns the
1316 * Return: Negative error code if the decoder wasn't properly configured; else
1319 int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map)
1321 struct cxl_port *port;
1325 if (WARN_ON_ONCE(!cxld))
1328 if (WARN_ON_ONCE(IS_ERR(cxld)))
1329 return PTR_ERR(cxld);
1331 if (cxld->interleave_ways < 1)
1336 port = to_cxl_port(cxld->dev.parent);
1337 if (!is_endpoint_decoder(dev)) {
1338 rc = decoder_populate_targets(cxld, port, target_map);
1339 if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {
1341 "Failed to populate active decoder targets\n");
1346 rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
1351 * Platform decoder resources should show up with a reasonable name. All
1352 * other resources are just sub ranges within the main decoder resource.
1354 if (is_root_decoder(dev))
1355 cxld->platform_res.name = dev_name(dev);
1357 return device_add(dev);
1359 EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, CXL);
1362 * cxl_decoder_add - Add a decoder with targets
1363 * @cxld: The cxl decoder allocated by cxl_decoder_alloc()
1364 * @target_map: A list of downstream ports that this decoder can direct memory
1365 * traffic to. These numbers should correspond with the port number
1366 * in the PCIe Link Capabilities structure.
1368 * This is the unlocked variant of cxl_decoder_add_locked().
1369 * See cxl_decoder_add_locked().
1371 * Context: Process context. Takes and releases the device lock of the port that
1374 int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map)
1376 struct cxl_port *port;
1379 if (WARN_ON_ONCE(!cxld))
1382 if (WARN_ON_ONCE(IS_ERR(cxld)))
1383 return PTR_ERR(cxld);
1385 port = to_cxl_port(cxld->dev.parent);
1387 device_lock(&port->dev);
1388 rc = cxl_decoder_add_locked(cxld, target_map);
1389 device_unlock(&port->dev);
1393 EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, CXL);
1395 static void cxld_unregister(void *dev)
1397 device_unregister(dev);
1400 int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
1402 return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
1404 EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, CXL);
1407 * __cxl_driver_register - register a driver for the cxl bus
1408 * @cxl_drv: cxl driver structure to attach
1409 * @owner: owning module/driver
1410 * @modname: KBUILD_MODNAME for parent driver
1412 int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
1413 const char *modname)
1415 if (!cxl_drv->probe) {
1416 pr_debug("%s ->probe() must be specified\n", modname);
1420 if (!cxl_drv->name) {
1421 pr_debug("%s ->name must be specified\n", modname);
1426 pr_debug("%s ->id must be specified\n", modname);
1430 cxl_drv->drv.bus = &cxl_bus_type;
1431 cxl_drv->drv.owner = owner;
1432 cxl_drv->drv.mod_name = modname;
1433 cxl_drv->drv.name = cxl_drv->name;
1435 return driver_register(&cxl_drv->drv);
1437 EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, CXL);
1439 void cxl_driver_unregister(struct cxl_driver *cxl_drv)
1441 driver_unregister(&cxl_drv->drv);
1443 EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, CXL);
1445 static int cxl_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
1447 return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,
1448 cxl_device_id(dev));
1451 static int cxl_bus_match(struct device *dev, struct device_driver *drv)
1453 return cxl_device_id(dev) == to_cxl_drv(drv)->id;
1456 static int cxl_bus_probe(struct device *dev)
1460 rc = to_cxl_drv(dev->driver)->probe(dev);
1461 dev_dbg(dev, "probe: %d\n", rc);
1465 static void cxl_bus_remove(struct device *dev)
1467 struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
1469 if (cxl_drv->remove)
1470 cxl_drv->remove(dev);
1473 static struct workqueue_struct *cxl_bus_wq;
1475 int cxl_bus_rescan(void)
1477 return bus_rescan_devices(&cxl_bus_type);
1479 EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, CXL);
1481 bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
1483 return queue_work(cxl_bus_wq, &cxlmd->detach_work);
1485 EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL);
1487 /* for user tooling to ensure port disable work has completed */
1488 static ssize_t flush_store(struct bus_type *bus, const char *buf, size_t count)
1490 if (sysfs_streq(buf, "1")) {
1491 flush_workqueue(cxl_bus_wq);
1498 static BUS_ATTR_WO(flush);
1500 static struct attribute *cxl_bus_attributes[] = {
1501 &bus_attr_flush.attr,
1505 static struct attribute_group cxl_bus_attribute_group = {
1506 .attrs = cxl_bus_attributes,
1509 static const struct attribute_group *cxl_bus_attribute_groups[] = {
1510 &cxl_bus_attribute_group,
1514 struct bus_type cxl_bus_type = {
1516 .uevent = cxl_bus_uevent,
1517 .match = cxl_bus_match,
1518 .probe = cxl_bus_probe,
1519 .remove = cxl_bus_remove,
1520 .bus_groups = cxl_bus_attribute_groups,
1522 EXPORT_SYMBOL_NS_GPL(cxl_bus_type, CXL);
1524 static __init int cxl_core_init(void)
1530 rc = cxl_memdev_init();
1534 cxl_bus_wq = alloc_ordered_workqueue("cxl_port", 0);
1540 rc = bus_register(&cxl_bus_type);
1547 destroy_workqueue(cxl_bus_wq);
1554 static void cxl_core_exit(void)
1556 bus_unregister(&cxl_bus_type);
1557 destroy_workqueue(cxl_bus_wq);
1562 module_init(cxl_core_init);
1563 module_exit(cxl_core_exit);
1564 MODULE_LICENSE("GPL v2");