1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3 #include <linux/io-64-nonatomic-lo-hi.h>
4 #include <linux/device.h>
5 #include <linux/module.h>
7 #include <linux/slab.h>
16 * The CXL core provides a set of interfaces that can be consumed by CXL aware
17 * drivers. The interfaces allow for creation, modification, and destruction of
18 * regions, memory devices, ports, and decoders. CXL aware drivers must register
19 * with the CXL core via these interfaces in order to be able to participate in
20 * cross-device interleave coordination. The CXL core also establishes and
21 * maintains the bridge to the nvdimm subsystem.
23 * CXL core introduces sysfs hierarchy to control the devices that are
24 * instantiated by the core.
27 static DEFINE_IDA(cxl_port_ida);
29 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
32 return sysfs_emit(buf, "%s\n", dev->type->name);
34 static DEVICE_ATTR_RO(devtype);
36 static struct attribute *cxl_base_attributes[] = {
37 &dev_attr_devtype.attr,
41 struct attribute_group cxl_base_attribute_group = {
42 .attrs = cxl_base_attributes,
45 static ssize_t start_show(struct device *dev, struct device_attribute *attr,
48 struct cxl_decoder *cxld = to_cxl_decoder(dev);
50 return sysfs_emit(buf, "%#llx\n", cxld->range.start);
52 static DEVICE_ATTR_RO(start);
54 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
57 struct cxl_decoder *cxld = to_cxl_decoder(dev);
59 return sysfs_emit(buf, "%#llx\n", range_len(&cxld->range));
61 static DEVICE_ATTR_RO(size);
63 #define CXL_DECODER_FLAG_ATTR(name, flag) \
64 static ssize_t name##_show(struct device *dev, \
65 struct device_attribute *attr, char *buf) \
67 struct cxl_decoder *cxld = to_cxl_decoder(dev); \
69 return sysfs_emit(buf, "%s\n", \
70 (cxld->flags & (flag)) ? "1" : "0"); \
72 static DEVICE_ATTR_RO(name)
74 CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
75 CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
76 CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
77 CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
78 CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
80 static ssize_t target_type_show(struct device *dev,
81 struct device_attribute *attr, char *buf)
83 struct cxl_decoder *cxld = to_cxl_decoder(dev);
85 switch (cxld->target_type) {
86 case CXL_DECODER_ACCELERATOR:
87 return sysfs_emit(buf, "accelerator\n");
88 case CXL_DECODER_EXPANDER:
89 return sysfs_emit(buf, "expander\n");
93 static DEVICE_ATTR_RO(target_type);
95 static ssize_t target_list_show(struct device *dev,
96 struct device_attribute *attr, char *buf)
98 struct cxl_decoder *cxld = to_cxl_decoder(dev);
103 for (i = 0; i < cxld->interleave_ways; i++) {
104 struct cxl_dport *dport = cxld->target[i];
105 struct cxl_dport *next = NULL;
110 if (i + 1 < cxld->interleave_ways)
111 next = cxld->target[i + 1];
112 rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
123 rc = sysfs_emit_at(buf, offset, "\n");
129 static DEVICE_ATTR_RO(target_list);
131 static struct attribute *cxl_decoder_base_attrs[] = {
132 &dev_attr_start.attr,
134 &dev_attr_locked.attr,
135 &dev_attr_target_list.attr,
139 static struct attribute_group cxl_decoder_base_attribute_group = {
140 .attrs = cxl_decoder_base_attrs,
143 static struct attribute *cxl_decoder_root_attrs[] = {
144 &dev_attr_cap_pmem.attr,
145 &dev_attr_cap_ram.attr,
146 &dev_attr_cap_type2.attr,
147 &dev_attr_cap_type3.attr,
151 static struct attribute_group cxl_decoder_root_attribute_group = {
152 .attrs = cxl_decoder_root_attrs,
155 static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
156 &cxl_decoder_root_attribute_group,
157 &cxl_decoder_base_attribute_group,
158 &cxl_base_attribute_group,
162 static struct attribute *cxl_decoder_switch_attrs[] = {
163 &dev_attr_target_type.attr,
167 static struct attribute_group cxl_decoder_switch_attribute_group = {
168 .attrs = cxl_decoder_switch_attrs,
171 static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
172 &cxl_decoder_switch_attribute_group,
173 &cxl_decoder_base_attribute_group,
174 &cxl_base_attribute_group,
178 static void cxl_decoder_release(struct device *dev)
180 struct cxl_decoder *cxld = to_cxl_decoder(dev);
181 struct cxl_port *port = to_cxl_port(dev->parent);
183 ida_free(&port->decoder_ida, cxld->id);
187 static const struct device_type cxl_decoder_switch_type = {
188 .name = "cxl_decoder_switch",
189 .release = cxl_decoder_release,
190 .groups = cxl_decoder_switch_attribute_groups,
193 static const struct device_type cxl_decoder_root_type = {
194 .name = "cxl_decoder_root",
195 .release = cxl_decoder_release,
196 .groups = cxl_decoder_root_attribute_groups,
199 bool is_root_decoder(struct device *dev)
201 return dev->type == &cxl_decoder_root_type;
203 EXPORT_SYMBOL_GPL(is_root_decoder);
205 struct cxl_decoder *to_cxl_decoder(struct device *dev)
207 if (dev_WARN_ONCE(dev, dev->type->release != cxl_decoder_release,
208 "not a cxl_decoder device\n"))
210 return container_of(dev, struct cxl_decoder, dev);
212 EXPORT_SYMBOL_GPL(to_cxl_decoder);
214 static void cxl_dport_release(struct cxl_dport *dport)
216 list_del(&dport->list);
217 put_device(dport->dport);
221 static void cxl_port_release(struct device *dev)
223 struct cxl_port *port = to_cxl_port(dev);
224 struct cxl_dport *dport, *_d;
227 list_for_each_entry_safe(dport, _d, &port->dports, list)
228 cxl_dport_release(dport);
230 ida_free(&cxl_port_ida, port->id);
234 static const struct attribute_group *cxl_port_attribute_groups[] = {
235 &cxl_base_attribute_group,
239 static const struct device_type cxl_port_type = {
241 .release = cxl_port_release,
242 .groups = cxl_port_attribute_groups,
245 struct cxl_port *to_cxl_port(struct device *dev)
247 if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
248 "not a cxl_port device\n"))
250 return container_of(dev, struct cxl_port, dev);
253 static void unregister_port(void *_port)
255 struct cxl_port *port = _port;
256 struct cxl_dport *dport;
258 device_lock(&port->dev);
259 list_for_each_entry(dport, &port->dports, list) {
260 char link_name[CXL_TARGET_STRLEN];
262 if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d",
263 dport->port_id) >= CXL_TARGET_STRLEN)
265 sysfs_remove_link(&port->dev.kobj, link_name);
267 device_unlock(&port->dev);
268 device_unregister(&port->dev);
271 static void cxl_unlink_uport(void *_port)
273 struct cxl_port *port = _port;
275 sysfs_remove_link(&port->dev.kobj, "uport");
278 static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
282 rc = sysfs_create_link(&port->dev.kobj, &port->uport->kobj, "uport");
285 return devm_add_action_or_reset(host, cxl_unlink_uport, port);
288 static struct cxl_port *cxl_port_alloc(struct device *uport,
289 resource_size_t component_reg_phys,
290 struct cxl_port *parent_port)
292 struct cxl_port *port;
296 port = kzalloc(sizeof(*port), GFP_KERNEL);
298 return ERR_PTR(-ENOMEM);
300 rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
306 * The top-level cxl_port "cxl_root" does not have a cxl_port as
307 * its parent and it does not have any corresponding component
308 * registers as its decode is described by a fixed platform
313 dev->parent = &parent_port->dev;
318 port->component_reg_phys = component_reg_phys;
319 ida_init(&port->decoder_ida);
320 INIT_LIST_HEAD(&port->dports);
322 device_initialize(dev);
323 device_set_pm_not_required(dev);
324 dev->bus = &cxl_bus_type;
325 dev->type = &cxl_port_type;
335 * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
336 * @host: host device for devm operations
337 * @uport: "physical" device implementing this upstream port
338 * @component_reg_phys: (optional) for configurable cxl_port instances
339 * @parent_port: next hop up in the CXL memory decode hierarchy
341 struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport,
342 resource_size_t component_reg_phys,
343 struct cxl_port *parent_port)
345 struct cxl_port *port;
349 port = cxl_port_alloc(uport, component_reg_phys, parent_port);
355 rc = dev_set_name(dev, "port%d", port->id);
357 rc = dev_set_name(dev, "root%d", port->id);
361 rc = device_add(dev);
365 rc = devm_add_action_or_reset(host, unregister_port, port);
369 rc = devm_cxl_link_uport(host, port);
379 EXPORT_SYMBOL_GPL(devm_cxl_add_port);
381 static struct cxl_dport *find_dport(struct cxl_port *port, int id)
383 struct cxl_dport *dport;
385 device_lock_assert(&port->dev);
386 list_for_each_entry (dport, &port->dports, list)
387 if (dport->port_id == id)
392 static int add_dport(struct cxl_port *port, struct cxl_dport *new)
394 struct cxl_dport *dup;
396 device_lock(&port->dev);
397 dup = find_dport(port, new->port_id);
400 "unable to add dport%d-%s non-unique port id (%s)\n",
401 new->port_id, dev_name(new->dport),
402 dev_name(dup->dport));
404 list_add_tail(&new->list, &port->dports);
405 device_unlock(&port->dev);
407 return dup ? -EEXIST : 0;
411 * cxl_add_dport - append downstream port data to a cxl_port
412 * @port: the cxl_port that references this dport
413 * @dport_dev: firmware or PCI device representing the dport
414 * @port_id: identifier for this dport in a decoder's target list
415 * @component_reg_phys: optional location of CXL component registers
417 * Note that all allocations and links are undone by cxl_port deletion
420 int cxl_add_dport(struct cxl_port *port, struct device *dport_dev, int port_id,
421 resource_size_t component_reg_phys)
423 char link_name[CXL_TARGET_STRLEN];
424 struct cxl_dport *dport;
427 if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
431 dport = kzalloc(sizeof(*dport), GFP_KERNEL);
435 INIT_LIST_HEAD(&dport->list);
436 dport->dport = get_device(dport_dev);
437 dport->port_id = port_id;
438 dport->component_reg_phys = component_reg_phys;
441 rc = add_dport(port, dport);
445 rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
451 cxl_dport_release(dport);
454 EXPORT_SYMBOL_GPL(cxl_add_dport);
456 static struct cxl_decoder *
457 cxl_decoder_alloc(struct cxl_port *port, int nr_targets, resource_size_t base,
458 resource_size_t len, int interleave_ways,
459 int interleave_granularity, enum cxl_decoder_type type,
462 struct cxl_decoder *cxld;
466 if (interleave_ways < 1)
467 return ERR_PTR(-EINVAL);
469 device_lock(&port->dev);
470 if (list_empty(&port->dports))
472 device_unlock(&port->dev);
476 cxld = kzalloc(struct_size(cxld, target, nr_targets), GFP_KERNEL);
478 return ERR_PTR(-ENOMEM);
480 rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
484 *cxld = (struct cxl_decoder) {
488 .end = base + len - 1,
491 .interleave_ways = interleave_ways,
492 .interleave_granularity = interleave_granularity,
496 /* handle implied target_list */
497 if (interleave_ways == 1)
499 list_first_entry(&port->dports, struct cxl_dport, list);
501 device_initialize(dev);
502 device_set_pm_not_required(dev);
503 dev->parent = &port->dev;
504 dev->bus = &cxl_bus_type;
506 /* root ports do not have a cxl_port_type parent */
507 if (port->dev.parent->type == &cxl_port_type)
508 dev->type = &cxl_decoder_switch_type;
510 dev->type = &cxl_decoder_root_type;
519 devm_cxl_add_decoder(struct device *host, struct cxl_port *port, int nr_targets,
520 resource_size_t base, resource_size_t len,
521 int interleave_ways, int interleave_granularity,
522 enum cxl_decoder_type type, unsigned long flags)
524 struct cxl_decoder *cxld;
528 cxld = cxl_decoder_alloc(port, nr_targets, base, len, interleave_ways,
529 interleave_granularity, type, flags);
534 rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
538 rc = device_add(dev);
542 rc = devm_add_action_or_reset(host, unregister_cxl_dev, dev);
551 EXPORT_SYMBOL_GPL(devm_cxl_add_decoder);
554 * __cxl_driver_register - register a driver for the cxl bus
555 * @cxl_drv: cxl driver structure to attach
556 * @owner: owning module/driver
557 * @modname: KBUILD_MODNAME for parent driver
559 int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
562 if (!cxl_drv->probe) {
563 pr_debug("%s ->probe() must be specified\n", modname);
567 if (!cxl_drv->name) {
568 pr_debug("%s ->name must be specified\n", modname);
573 pr_debug("%s ->id must be specified\n", modname);
577 cxl_drv->drv.bus = &cxl_bus_type;
578 cxl_drv->drv.owner = owner;
579 cxl_drv->drv.mod_name = modname;
580 cxl_drv->drv.name = cxl_drv->name;
582 return driver_register(&cxl_drv->drv);
584 EXPORT_SYMBOL_GPL(__cxl_driver_register);
586 void cxl_driver_unregister(struct cxl_driver *cxl_drv)
588 driver_unregister(&cxl_drv->drv);
590 EXPORT_SYMBOL_GPL(cxl_driver_unregister);
592 static int cxl_device_id(struct device *dev)
594 if (dev->type == &cxl_nvdimm_bridge_type)
595 return CXL_DEVICE_NVDIMM_BRIDGE;
596 if (dev->type == &cxl_nvdimm_type)
597 return CXL_DEVICE_NVDIMM;
601 static int cxl_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
603 return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,
607 static int cxl_bus_match(struct device *dev, struct device_driver *drv)
609 return cxl_device_id(dev) == to_cxl_drv(drv)->id;
612 static int cxl_bus_probe(struct device *dev)
614 return to_cxl_drv(dev->driver)->probe(dev);
617 static void cxl_bus_remove(struct device *dev)
619 struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
622 cxl_drv->remove(dev);
625 struct bus_type cxl_bus_type = {
627 .uevent = cxl_bus_uevent,
628 .match = cxl_bus_match,
629 .probe = cxl_bus_probe,
630 .remove = cxl_bus_remove,
632 EXPORT_SYMBOL_GPL(cxl_bus_type);
634 static __init int cxl_core_init(void)
638 rc = cxl_memdev_init();
642 rc = bus_register(&cxl_bus_type);
652 static void cxl_core_exit(void)
654 bus_unregister(&cxl_bus_type);
658 module_init(cxl_core_init);
659 module_exit(cxl_core_exit);
660 MODULE_LICENSE("GPL v2");