drivers/cxl/pmem.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2021 Intel Corporation. All rights reserved. */
   3 #include <linux/libnvdimm.h>
   4 #include <asm/unaligned.h>
   5 #include <linux/device.h>
   6 #include <linux/module.h>
   7 #include <linux/ndctl.h>
   8 #include <linux/async.h>
   9 #include <linux/slab.h>
  10 #include <linux/nd.h>
  11 #include "cxlmem.h"
  12 #include "cxl.h"
  13
  14 extern const struct nvdimm_security_ops *cxl_security_ops;
  15
  16 static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
  17
  18 static void clear_exclusive(void *cxlds)
  19 {
  20         clear_exclusive_cxl_commands(cxlds, exclusive_cmds);
  21 }
  22
  23 static void unregister_nvdimm(void *nvdimm)
  24 {
  25         nvdimm_delete(nvdimm);
  26 }
  27
  28 static ssize_t provider_show(struct device *dev, struct device_attribute *attr, char *buf)
  29 {
  30         struct nvdimm *nvdimm = to_nvdimm(dev);
  31         struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
  32
  33         return sysfs_emit(buf, "%s\n", dev_name(&cxl_nvd->dev));
  34 }
  35 static DEVICE_ATTR_RO(provider);
  36
  37 static ssize_t id_show(struct device *dev, struct device_attribute *attr, char *buf)
  38 {
  39         struct nvdimm *nvdimm = to_nvdimm(dev);
  40         struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
  41         struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds;
  42
  43         return sysfs_emit(buf, "%lld\n", cxlds->serial);
  44 }
  45 static DEVICE_ATTR_RO(id);
  46
  47 static struct attribute *cxl_dimm_attributes[] = {
  48         &dev_attr_id.attr,
  49         &dev_attr_provider.attr,
  50         NULL
  51 };
  52
  53 static const struct attribute_group cxl_dimm_attribute_group = {
  54         .name = "cxl",
  55         .attrs = cxl_dimm_attributes,
  56 };
  57
  58 static const struct attribute_group *cxl_dimm_attribute_groups[] = {
  59         &cxl_dimm_attribute_group,
  60         NULL
  61 };
  62
  63 static int cxl_nvdimm_probe(struct device *dev)
  64 {
  65         struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
  66         struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
  67         struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb;
  68         unsigned long flags = 0, cmd_mask = 0;
  69         struct cxl_dev_state *cxlds = cxlmd->cxlds;
  70         struct nvdimm *nvdimm;
  71         int rc;
  72
  73         set_exclusive_cxl_commands(cxlds, exclusive_cmds);
  74         rc = devm_add_action_or_reset(dev, clear_exclusive, cxlds);
  75         if (rc)
  76                 return rc;
  77
  78         set_bit(NDD_LABELING, &flags);
  79         set_bit(NDD_REGISTER_SYNC, &flags);
  80         set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
  81         set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
  82         set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
  83         nvdimm = __nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd,
  84                                  cxl_dimm_attribute_groups, flags,
  85                                  cmd_mask, 0, NULL, cxl_nvd->dev_id,
  86                                  cxl_security_ops, NULL);
  87         if (!nvdimm)
  88                 return -ENOMEM;
  89
  90         dev_set_drvdata(dev, nvdimm);
  91         return devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
  92 }
  93
  94 static struct cxl_driver cxl_nvdimm_driver = {
  95         .name = "cxl_nvdimm",
  96         .probe = cxl_nvdimm_probe,
  97         .id = CXL_DEVICE_NVDIMM,
  98         .drv = {
  99                 .suppress_bind_attrs = true,
 100         },
 101 };
 102
 103 static int cxl_pmem_get_config_size(struct cxl_dev_state *cxlds,
 104                                     struct nd_cmd_get_config_size *cmd,
 105                                     unsigned int buf_len)
 106 {
 107         if (sizeof(*cmd) > buf_len)
 108                 return -EINVAL;
 109
 110         *cmd = (struct nd_cmd_get_config_size) {
 111                  .config_size = cxlds->lsa_size,
 112                  .max_xfer = cxlds->payload_size - sizeof(struct cxl_mbox_set_lsa),
 113         };
 114
 115         return 0;
 116 }
 117
 118 static int cxl_pmem_get_config_data(struct cxl_dev_state *cxlds,
 119                                     struct nd_cmd_get_config_data_hdr *cmd,
 120                                     unsigned int buf_len)
 121 {
 122         struct cxl_mbox_get_lsa get_lsa;
 123         struct cxl_mbox_cmd mbox_cmd;
 124         int rc;
 125
 126         if (sizeof(*cmd) > buf_len)
 127                 return -EINVAL;
 128         if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
 129                 return -EINVAL;
 130
 131         get_lsa = (struct cxl_mbox_get_lsa) {
 132                 .offset = cpu_to_le32(cmd->in_offset),
 133                 .length = cpu_to_le32(cmd->in_length),
 134         };
 135         mbox_cmd = (struct cxl_mbox_cmd) {
 136                 .opcode = CXL_MBOX_OP_GET_LSA,
 137                 .payload_in = &get_lsa,
 138                 .size_in = sizeof(get_lsa),
 139                 .size_out = cmd->in_length,
 140                 .payload_out = cmd->out_buf,
 141         };
 142
 143         rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
 144         cmd->status = 0;
 145
 146         return rc;
 147 }
 148
 149 static int cxl_pmem_set_config_data(struct cxl_dev_state *cxlds,
 150                                     struct nd_cmd_set_config_hdr *cmd,
 151                                     unsigned int buf_len)
 152 {
 153         struct cxl_mbox_set_lsa *set_lsa;
 154         struct cxl_mbox_cmd mbox_cmd;
 155         int rc;
 156
 157         if (sizeof(*cmd) > buf_len)
 158                 return -EINVAL;
 159
 160         /* 4-byte status follows the input data in the payload */
 161         if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len)
 162                 return -EINVAL;
 163
 164         set_lsa =
 165                 kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);
 166         if (!set_lsa)
 167                 return -ENOMEM;
 168
 169         *set_lsa = (struct cxl_mbox_set_lsa) {
 170                 .offset = cpu_to_le32(cmd->in_offset),
 171         };
 172         memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
 173         mbox_cmd = (struct cxl_mbox_cmd) {
 174                 .opcode = CXL_MBOX_OP_SET_LSA,
 175                 .payload_in = set_lsa,
 176                 .size_in = struct_size(set_lsa, data, cmd->in_length),
 177         };
 178
 179         rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
 180
 181         /*
 182          * Set "firmware" status (4-packed bytes at the end of the input
 183          * payload.
 184          */
 185         put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
 186         kvfree(set_lsa);
 187
 188         return rc;
 189 }
 190
 191 static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
 192                                void *buf, unsigned int buf_len)
 193 {
 194         struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
 195         unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
 196         struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
 197         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 198
 199         if (!test_bit(cmd, &cmd_mask))
 200                 return -ENOTTY;
 201
 202         switch (cmd) {
 203         case ND_CMD_GET_CONFIG_SIZE:
 204                 return cxl_pmem_get_config_size(cxlds, buf, buf_len);
 205         case ND_CMD_GET_CONFIG_DATA:
 206                 return cxl_pmem_get_config_data(cxlds, buf, buf_len);
 207         case ND_CMD_SET_CONFIG_DATA:
 208                 return cxl_pmem_set_config_data(cxlds, buf, buf_len);
 209         default:
 210                 return -ENOTTY;
 211         }
 212 }
 213
 214 static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
 215                         struct nvdimm *nvdimm, unsigned int cmd, void *buf,
 216                         unsigned int buf_len, int *cmd_rc)
 217 {
 218         /*
 219          * No firmware response to translate, let the transport error
 220          * code take precedence.
 221          */
 222         *cmd_rc = 0;
 223
 224         if (!nvdimm)
 225                 return -ENOTTY;
 226         return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
 227 }
 228
 229 static int detach_nvdimm(struct device *dev, void *data)
 230 {
 231         struct cxl_nvdimm *cxl_nvd;
 232         bool release = false;
 233
 234         if (!is_cxl_nvdimm(dev))
 235                 return 0;
 236
 237         device_lock(dev);
 238         if (!dev->driver)
 239                 goto out;
 240
 241         cxl_nvd = to_cxl_nvdimm(dev);
 242         if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data)
 243                 release = true;
 244 out:
 245         device_unlock(dev);
 246         if (release)
 247                 device_release_driver(dev);
 248         return 0;
 249 }
 250
 251 static void unregister_nvdimm_bus(void *_cxl_nvb)
 252 {
 253         struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb;
 254         struct nvdimm_bus *nvdimm_bus = cxl_nvb->nvdimm_bus;
 255
 256         bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb, detach_nvdimm);
 257
 258         cxl_nvb->nvdimm_bus = NULL;
 259         nvdimm_bus_unregister(nvdimm_bus);
 260 }
 261
 262 static int cxl_nvdimm_bridge_probe(struct device *dev)
 263 {
 264         struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
 265
 266         cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {
 267                 .provider_name = "CXL",
 268                 .module = THIS_MODULE,
 269                 .ndctl = cxl_pmem_ctl,
 270         };
 271
 272         cxl_nvb->nvdimm_bus =
 273                 nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);
 274
 275         if (!cxl_nvb->nvdimm_bus)
 276                 return -ENOMEM;
 277
 278         return devm_add_action_or_reset(dev, unregister_nvdimm_bus, cxl_nvb);
 279 }
 280
 281 static struct cxl_driver cxl_nvdimm_bridge_driver = {
 282         .name = "cxl_nvdimm_bridge",
 283         .probe = cxl_nvdimm_bridge_probe,
 284         .id = CXL_DEVICE_NVDIMM_BRIDGE,
 285         .drv = {
 286                 .suppress_bind_attrs = true,
 287         },
 288 };
 289
 290 static void unregister_nvdimm_region(void *nd_region)
 291 {
 292         nvdimm_region_delete(nd_region);
 293 }
 294
 295 static void cxlr_pmem_remove_resource(void *res)
 296 {
 297         remove_resource(res);
 298 }
 299
 300 struct cxl_pmem_region_info {
 301         u64 offset;
 302         u64 serial;
 303 };
 304
 305 static int cxl_pmem_region_probe(struct device *dev)
 306 {
 307         struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE];
 308         struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
 309         struct cxl_region *cxlr = cxlr_pmem->cxlr;
 310         struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
 311         struct cxl_pmem_region_info *info = NULL;
 312         struct nd_interleave_set *nd_set;
 313         struct nd_region_desc ndr_desc;
 314         struct cxl_nvdimm *cxl_nvd;
 315         struct nvdimm *nvdimm;
 316         struct resource *res;
 317         int rc, i = 0;
 318
 319         memset(&mappings, 0, sizeof(mappings));
 320         memset(&ndr_desc, 0, sizeof(ndr_desc));
 321
 322         res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
 323         if (!res)
 324                 return -ENOMEM;
 325
 326         res->name = "Persistent Memory";
 327         res->start = cxlr_pmem->hpa_range.start;
 328         res->end = cxlr_pmem->hpa_range.end;
 329         res->flags = IORESOURCE_MEM;
 330         res->desc = IORES_DESC_PERSISTENT_MEMORY;
 331
 332         rc = insert_resource(&iomem_resource, res);
 333         if (rc)
 334                 return rc;
 335
 336         rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res);
 337         if (rc)
 338                 return rc;
 339
 340         ndr_desc.res = res;
 341         ndr_desc.provider_data = cxlr_pmem;
 342
 343         ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start);
 344         ndr_desc.target_node = phys_to_target_node(res->start);
 345         if (ndr_desc.target_node == NUMA_NO_NODE) {
 346                 ndr_desc.target_node = ndr_desc.numa_node;
 347                 dev_dbg(&cxlr->dev, "changing target node from %d to %d",
 348                         NUMA_NO_NODE, ndr_desc.target_node);
 349         }
 350
 351         nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
 352         if (!nd_set)
 353                 return -ENOMEM;
 354
 355         ndr_desc.memregion = cxlr->id;
 356         set_bit(ND_REGION_CXL, &ndr_desc.flags);
 357         set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags);
 358
 359         info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL);
 360         if (!info)
 361                 return -ENOMEM;
 362
 363         for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
 364                 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
 365                 struct cxl_memdev *cxlmd = m->cxlmd;
 366                 struct cxl_dev_state *cxlds = cxlmd->cxlds;
 367
 368                 cxl_nvd = cxlmd->cxl_nvd;
 369                 nvdimm = dev_get_drvdata(&cxl_nvd->dev);
 370                 if (!nvdimm) {
 371                         dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i,
 372                                 dev_name(&cxlmd->dev));
 373                         rc = -ENODEV;
 374                         goto out_nvd;
 375                 }
 376
 377                 m->cxl_nvd = cxl_nvd;
 378                 mappings[i] = (struct nd_mapping_desc) {
 379                         .nvdimm = nvdimm,
 380                         .start = m->start,
 381                         .size = m->size,
 382                         .position = i,
 383                 };
 384                 info[i].offset = m->start;
 385                 info[i].serial = cxlds->serial;
 386         }
 387         ndr_desc.num_mappings = cxlr_pmem->nr_mappings;
 388         ndr_desc.mapping = mappings;
 389
 390         /*
 391          * TODO enable CXL labels which skip the need for 'interleave-set cookie'
 392          */
 393         nd_set->cookie1 =
 394                 nd_fletcher64(info, sizeof(*info) * cxlr_pmem->nr_mappings, 0);
 395         nd_set->cookie2 = nd_set->cookie1;
 396         ndr_desc.nd_set = nd_set;
 397
 398         cxlr_pmem->nd_region =
 399                 nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc);
 400         if (!cxlr_pmem->nd_region) {
 401                 rc = -ENOMEM;
 402                 goto out_nvd;
 403         }
 404
 405         rc = devm_add_action_or_reset(dev, unregister_nvdimm_region,
 406                                       cxlr_pmem->nd_region);
 407 out_nvd:
 408         kfree(info);
 409
 410         return rc;
 411 }
 412
 413 static struct cxl_driver cxl_pmem_region_driver = {
 414         .name = "cxl_pmem_region",
 415         .probe = cxl_pmem_region_probe,
 416         .id = CXL_DEVICE_PMEM_REGION,
 417         .drv = {
 418                 .suppress_bind_attrs = true,
 419         },
 420 };
 421
 422 static __init int cxl_pmem_init(void)
 423 {
 424         int rc;
 425
 426         set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
 427         set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);
 428
 429         rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);
 430         if (rc)
 431                 return rc;
 432
 433         rc = cxl_driver_register(&cxl_nvdimm_driver);
 434         if (rc)
 435                 goto err_nvdimm;
 436
 437         rc = cxl_driver_register(&cxl_pmem_region_driver);
 438         if (rc)
 439                 goto err_region;
 440
 441         return 0;
 442
 443 err_region:
 444         cxl_driver_unregister(&cxl_nvdimm_driver);
 445 err_nvdimm:
 446         cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
 447         return rc;
 448 }
 449
 450 static __exit void cxl_pmem_exit(void)
 451 {
 452         cxl_driver_unregister(&cxl_pmem_region_driver);
 453         cxl_driver_unregister(&cxl_nvdimm_driver);
 454         cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
 455 }
 456
 457 MODULE_LICENSE("GPL v2");
 458 module_init(cxl_pmem_init);
 459 module_exit(cxl_pmem_exit);
 460 MODULE_IMPORT_NS(CXL);
 461 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
 462 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
 463 MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION);