drivers/acpi/nfit.c

   1 /*
   2  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  */
  13 #include <linux/list_sort.h>
  14 #include <linux/libnvdimm.h>
  15 #include <linux/module.h>
  16 #include <linux/mutex.h>
  17 #include <linux/ndctl.h>
  18 #include <linux/list.h>
  19 #include <linux/acpi.h>
  20 #include <linux/sort.h>
  21 #include <linux/pmem.h>
  22 #include <linux/io.h>
  23 #include <asm/cacheflush.h>
  24 #include "nfit.h"
  25
  26 /*
  27  * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
  28  * irrelevant.
  29  */
  30 #include <linux/io-64-nonatomic-hi-lo.h>
  31
  32 static bool force_enable_dimms;
  33 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR);
  34 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status");
  35
  36 struct nfit_table_prev {
  37         struct list_head spas;
  38         struct list_head memdevs;
  39         struct list_head dcrs;
  40         struct list_head bdws;
  41         struct list_head idts;
  42         struct list_head flushes;
  43 };
  44
  45 static u8 nfit_uuid[NFIT_UUID_MAX][16];
  46
  47 const u8 *to_nfit_uuid(enum nfit_uuids id)
  48 {
  49         return nfit_uuid[id];
  50 }
  51 EXPORT_SYMBOL(to_nfit_uuid);
  52
  53 static struct acpi_nfit_desc *to_acpi_nfit_desc(
  54                 struct nvdimm_bus_descriptor *nd_desc)
  55 {
  56         return container_of(nd_desc, struct acpi_nfit_desc, nd_desc);
  57 }
  58
  59 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc)
  60 {
  61         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
  62
  63         /*
  64          * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
  65          * acpi_device.
  66          */
  67         if (!nd_desc->provider_name
  68                         || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0)
  69                 return NULL;
  70
  71         return to_acpi_device(acpi_desc->dev);
  72 }
  73
  74 static int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc,
  75                 struct nvdimm *nvdimm, unsigned int cmd, void *buf,
  76                 unsigned int buf_len)
  77 {
  78         struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
  79         const struct nd_cmd_desc *desc = NULL;
  80         union acpi_object in_obj, in_buf, *out_obj;
  81         struct device *dev = acpi_desc->dev;
  82         const char *cmd_name, *dimm_name;
  83         unsigned long dsm_mask;
  84         acpi_handle handle;
  85         const u8 *uuid;
  86         u32 offset;
  87         int rc, i;
  88
  89         if (nvdimm) {
  90                 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
  91                 struct acpi_device *adev = nfit_mem->adev;
  92
  93                 if (!adev)
  94                         return -ENOTTY;
  95                 dimm_name = nvdimm_name(nvdimm);
  96                 cmd_name = nvdimm_cmd_name(cmd);
  97                 dsm_mask = nfit_mem->dsm_mask;
  98                 desc = nd_cmd_dimm_desc(cmd);
  99                 uuid = to_nfit_uuid(NFIT_DEV_DIMM);
 100                 handle = adev->handle;
 101         } else {
 102                 struct acpi_device *adev = to_acpi_dev(acpi_desc);
 103
 104                 cmd_name = nvdimm_bus_cmd_name(cmd);
 105                 dsm_mask = nd_desc->dsm_mask;
 106                 desc = nd_cmd_bus_desc(cmd);
 107                 uuid = to_nfit_uuid(NFIT_DEV_BUS);
 108                 handle = adev->handle;
 109                 dimm_name = "bus";
 110         }
 111
 112         if (!desc || (cmd && (desc->out_num + desc->in_num == 0)))
 113                 return -ENOTTY;
 114
 115         if (!test_bit(cmd, &dsm_mask))
 116                 return -ENOTTY;
 117
 118         in_obj.type = ACPI_TYPE_PACKAGE;
 119         in_obj.package.count = 1;
 120         in_obj.package.elements = &in_buf;
 121         in_buf.type = ACPI_TYPE_BUFFER;
 122         in_buf.buffer.pointer = buf;
 123         in_buf.buffer.length = 0;
 124
 125         /* libnvdimm has already validated the input envelope */
 126         for (i = 0; i < desc->in_num; i++)
 127                 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc,
 128                                 i, buf);
 129
 130         if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) {
 131                 dev_dbg(dev, "%s:%s cmd: %s input length: %d\n", __func__,
 132                                 dimm_name, cmd_name, in_buf.buffer.length);
 133                 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4,
 134                                 4, in_buf.buffer.pointer, min_t(u32, 128,
 135                                         in_buf.buffer.length), true);
 136         }
 137
 138         out_obj = acpi_evaluate_dsm(handle, uuid, 1, cmd, &in_obj);
 139         if (!out_obj) {
 140                 dev_dbg(dev, "%s:%s _DSM failed cmd: %s\n", __func__, dimm_name,
 141                                 cmd_name);
 142                 return -EINVAL;
 143         }
 144
 145         if (out_obj->package.type != ACPI_TYPE_BUFFER) {
 146                 dev_dbg(dev, "%s:%s unexpected output object type cmd: %s type: %d\n",
 147                                 __func__, dimm_name, cmd_name, out_obj->type);
 148                 rc = -EINVAL;
 149                 goto out;
 150         }
 151
 152         if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) {
 153                 dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__,
 154                                 dimm_name, cmd_name, out_obj->buffer.length);
 155                 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4,
 156                                 4, out_obj->buffer.pointer, min_t(u32, 128,
 157                                         out_obj->buffer.length), true);
 158         }
 159
 160         for (i = 0, offset = 0; i < desc->out_num; i++) {
 161                 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf,
 162                                 (u32 *) out_obj->buffer.pointer);
 163
 164                 if (offset + out_size > out_obj->buffer.length) {
 165                         dev_dbg(dev, "%s:%s output object underflow cmd: %s field: %d\n",
 166                                         __func__, dimm_name, cmd_name, i);
 167                         break;
 168                 }
 169
 170                 if (in_buf.buffer.length + offset + out_size > buf_len) {
 171                         dev_dbg(dev, "%s:%s output overrun cmd: %s field: %d\n",
 172                                         __func__, dimm_name, cmd_name, i);
 173                         rc = -ENXIO;
 174                         goto out;
 175                 }
 176                 memcpy(buf + in_buf.buffer.length + offset,
 177                                 out_obj->buffer.pointer + offset, out_size);
 178                 offset += out_size;
 179         }
 180         if (offset + in_buf.buffer.length < buf_len) {
 181                 if (i >= 1) {
 182                         /*
 183                          * status valid, return the number of bytes left
 184                          * unfilled in the output buffer
 185                          */
 186                         rc = buf_len - offset - in_buf.buffer.length;
 187                 } else {
 188                         dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
 189                                         __func__, dimm_name, cmd_name, buf_len,
 190                                         offset);
 191                         rc = -ENXIO;
 192                 }
 193         } else
 194                 rc = 0;
 195
 196  out:
 197         ACPI_FREE(out_obj);
 198
 199         return rc;
 200 }
 201
 202 static const char *spa_type_name(u16 type)
 203 {
 204         static const char *to_name[] = {
 205                 [NFIT_SPA_VOLATILE] = "volatile",
 206                 [NFIT_SPA_PM] = "pmem",
 207                 [NFIT_SPA_DCR] = "dimm-control-region",
 208                 [NFIT_SPA_BDW] = "block-data-window",
 209                 [NFIT_SPA_VDISK] = "volatile-disk",
 210                 [NFIT_SPA_VCD] = "volatile-cd",
 211                 [NFIT_SPA_PDISK] = "persistent-disk",
 212                 [NFIT_SPA_PCD] = "persistent-cd",
 213
 214         };
 215
 216         if (type > NFIT_SPA_PCD)
 217                 return "unknown";
 218
 219         return to_name[type];
 220 }
 221
 222 static int nfit_spa_type(struct acpi_nfit_system_address *spa)
 223 {
 224         int i;
 225
 226         for (i = 0; i < NFIT_UUID_MAX; i++)
 227                 if (memcmp(to_nfit_uuid(i), spa->range_guid, 16) == 0)
 228                         return i;
 229         return -1;
 230 }
 231
 232 static bool add_spa(struct acpi_nfit_desc *acpi_desc,
 233                 struct nfit_table_prev *prev,
 234                 struct acpi_nfit_system_address *spa)
 235 {
 236         struct device *dev = acpi_desc->dev;
 237         struct nfit_spa *nfit_spa;
 238
 239         list_for_each_entry(nfit_spa, &prev->spas, list) {
 240                 if (memcmp(nfit_spa->spa, spa, sizeof(*spa)) == 0) {
 241                         list_move_tail(&nfit_spa->list, &acpi_desc->spas);
 242                         return true;
 243                 }
 244         }
 245
 246         nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa), GFP_KERNEL);
 247         if (!nfit_spa)
 248                 return false;
 249         INIT_LIST_HEAD(&nfit_spa->list);
 250         nfit_spa->spa = spa;
 251         list_add_tail(&nfit_spa->list, &acpi_desc->spas);
 252         dev_dbg(dev, "%s: spa index: %d type: %s\n", __func__,
 253                         spa->range_index,
 254                         spa_type_name(nfit_spa_type(spa)));
 255         return true;
 256 }
 257
 258 static bool add_memdev(struct acpi_nfit_desc *acpi_desc,
 259                 struct nfit_table_prev *prev,
 260                 struct acpi_nfit_memory_map *memdev)
 261 {
 262         struct device *dev = acpi_desc->dev;
 263         struct nfit_memdev *nfit_memdev;
 264
 265         list_for_each_entry(nfit_memdev, &prev->memdevs, list)
 266                 if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) {
 267                         list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs);
 268                         return true;
 269                 }
 270
 271         nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev), GFP_KERNEL);
 272         if (!nfit_memdev)
 273                 return false;
 274         INIT_LIST_HEAD(&nfit_memdev->list);
 275         nfit_memdev->memdev = memdev;
 276         list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs);
 277         dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d\n",
 278                         __func__, memdev->device_handle, memdev->range_index,
 279                         memdev->region_index);
 280         return true;
 281 }
 282
 283 static bool add_dcr(struct acpi_nfit_desc *acpi_desc,
 284                 struct nfit_table_prev *prev,
 285                 struct acpi_nfit_control_region *dcr)
 286 {
 287         struct device *dev = acpi_desc->dev;
 288         struct nfit_dcr *nfit_dcr;
 289
 290         list_for_each_entry(nfit_dcr, &prev->dcrs, list)
 291                 if (memcmp(nfit_dcr->dcr, dcr, sizeof(*dcr)) == 0) {
 292                         list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs);
 293                         return true;
 294                 }
 295
 296         nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr), GFP_KERNEL);
 297         if (!nfit_dcr)
 298                 return false;
 299         INIT_LIST_HEAD(&nfit_dcr->list);
 300         nfit_dcr->dcr = dcr;
 301         list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs);
 302         dev_dbg(dev, "%s: dcr index: %d windows: %d\n", __func__,
 303                         dcr->region_index, dcr->windows);
 304         return true;
 305 }
 306
 307 static bool add_bdw(struct acpi_nfit_desc *acpi_desc,
 308                 struct nfit_table_prev *prev,
 309                 struct acpi_nfit_data_region *bdw)
 310 {
 311         struct device *dev = acpi_desc->dev;
 312         struct nfit_bdw *nfit_bdw;
 313
 314         list_for_each_entry(nfit_bdw, &prev->bdws, list)
 315                 if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) {
 316                         list_move_tail(&nfit_bdw->list, &acpi_desc->bdws);
 317                         return true;
 318                 }
 319
 320         nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw), GFP_KERNEL);
 321         if (!nfit_bdw)
 322                 return false;
 323         INIT_LIST_HEAD(&nfit_bdw->list);
 324         nfit_bdw->bdw = bdw;
 325         list_add_tail(&nfit_bdw->list, &acpi_desc->bdws);
 326         dev_dbg(dev, "%s: bdw dcr: %d windows: %d\n", __func__,
 327                         bdw->region_index, bdw->windows);
 328         return true;
 329 }
 330
 331 static bool add_idt(struct acpi_nfit_desc *acpi_desc,
 332                 struct nfit_table_prev *prev,
 333                 struct acpi_nfit_interleave *idt)
 334 {
 335         struct device *dev = acpi_desc->dev;
 336         struct nfit_idt *nfit_idt;
 337
 338         list_for_each_entry(nfit_idt, &prev->idts, list)
 339                 if (memcmp(nfit_idt->idt, idt, sizeof(*idt)) == 0) {
 340                         list_move_tail(&nfit_idt->list, &acpi_desc->idts);
 341                         return true;
 342                 }
 343
 344         nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt), GFP_KERNEL);
 345         if (!nfit_idt)
 346                 return false;
 347         INIT_LIST_HEAD(&nfit_idt->list);
 348         nfit_idt->idt = idt;
 349         list_add_tail(&nfit_idt->list, &acpi_desc->idts);
 350         dev_dbg(dev, "%s: idt index: %d num_lines: %d\n", __func__,
 351                         idt->interleave_index, idt->line_count);
 352         return true;
 353 }
 354
 355 static bool add_flush(struct acpi_nfit_desc *acpi_desc,
 356                 struct nfit_table_prev *prev,
 357                 struct acpi_nfit_flush_address *flush)
 358 {
 359         struct device *dev = acpi_desc->dev;
 360         struct nfit_flush *nfit_flush;
 361
 362         list_for_each_entry(nfit_flush, &prev->flushes, list)
 363                 if (memcmp(nfit_flush->flush, flush, sizeof(*flush)) == 0) {
 364                         list_move_tail(&nfit_flush->list, &acpi_desc->flushes);
 365                         return true;
 366                 }
 367
 368         nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush), GFP_KERNEL);
 369         if (!nfit_flush)
 370                 return false;
 371         INIT_LIST_HEAD(&nfit_flush->list);
 372         nfit_flush->flush = flush;
 373         list_add_tail(&nfit_flush->list, &acpi_desc->flushes);
 374         dev_dbg(dev, "%s: nfit_flush handle: %d hint_count: %d\n", __func__,
 375                         flush->device_handle, flush->hint_count);
 376         return true;
 377 }
 378
 379 static void *add_table(struct acpi_nfit_desc *acpi_desc,
 380                 struct nfit_table_prev *prev, void *table, const void *end)
 381 {
 382         struct device *dev = acpi_desc->dev;
 383         struct acpi_nfit_header *hdr;
 384         void *err = ERR_PTR(-ENOMEM);
 385
 386         if (table >= end)
 387                 return NULL;
 388
 389         hdr = table;
 390         if (!hdr->length) {
 391                 dev_warn(dev, "found a zero length table '%d' parsing nfit\n",
 392                         hdr->type);
 393                 return NULL;
 394         }
 395
 396         switch (hdr->type) {
 397         case ACPI_NFIT_TYPE_SYSTEM_ADDRESS:
 398                 if (!add_spa(acpi_desc, prev, table))
 399                         return err;
 400                 break;
 401         case ACPI_NFIT_TYPE_MEMORY_MAP:
 402                 if (!add_memdev(acpi_desc, prev, table))
 403                         return err;
 404                 break;
 405         case ACPI_NFIT_TYPE_CONTROL_REGION:
 406                 if (!add_dcr(acpi_desc, prev, table))
 407                         return err;
 408                 break;
 409         case ACPI_NFIT_TYPE_DATA_REGION:
 410                 if (!add_bdw(acpi_desc, prev, table))
 411                         return err;
 412                 break;
 413         case ACPI_NFIT_TYPE_INTERLEAVE:
 414                 if (!add_idt(acpi_desc, prev, table))
 415                         return err;
 416                 break;
 417         case ACPI_NFIT_TYPE_FLUSH_ADDRESS:
 418                 if (!add_flush(acpi_desc, prev, table))
 419                         return err;
 420                 break;
 421         case ACPI_NFIT_TYPE_SMBIOS:
 422                 dev_dbg(dev, "%s: smbios\n", __func__);
 423                 break;
 424         default:
 425                 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type);
 426                 break;
 427         }
 428
 429         return table + hdr->length;
 430 }
 431
 432 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc,
 433                 struct nfit_mem *nfit_mem)
 434 {
 435         u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
 436         u16 dcr = nfit_mem->dcr->region_index;
 437         struct nfit_spa *nfit_spa;
 438
 439         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
 440                 u16 range_index = nfit_spa->spa->range_index;
 441                 int type = nfit_spa_type(nfit_spa->spa);
 442                 struct nfit_memdev *nfit_memdev;
 443
 444                 if (type != NFIT_SPA_BDW)
 445                         continue;
 446
 447                 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
 448                         if (nfit_memdev->memdev->range_index != range_index)
 449                                 continue;
 450                         if (nfit_memdev->memdev->device_handle != device_handle)
 451                                 continue;
 452                         if (nfit_memdev->memdev->region_index != dcr)
 453                                 continue;
 454
 455                         nfit_mem->spa_bdw = nfit_spa->spa;
 456                         return;
 457                 }
 458         }
 459
 460         dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n",
 461                         nfit_mem->spa_dcr->range_index);
 462         nfit_mem->bdw = NULL;
 463 }
 464
 465 static int nfit_mem_add(struct acpi_nfit_desc *acpi_desc,
 466                 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa)
 467 {
 468         u16 dcr = __to_nfit_memdev(nfit_mem)->region_index;
 469         struct nfit_memdev *nfit_memdev;
 470         struct nfit_flush *nfit_flush;
 471         struct nfit_dcr *nfit_dcr;
 472         struct nfit_bdw *nfit_bdw;
 473         struct nfit_idt *nfit_idt;
 474         u16 idt_idx, range_index;
 475
 476         list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
 477                 if (nfit_dcr->dcr->region_index != dcr)
 478                         continue;
 479                 nfit_mem->dcr = nfit_dcr->dcr;
 480                 break;
 481         }
 482
 483         if (!nfit_mem->dcr) {
 484                 dev_dbg(acpi_desc->dev, "SPA %d missing:%s%s\n",
 485                                 spa->range_index, __to_nfit_memdev(nfit_mem)
 486                                 ? "" : " MEMDEV", nfit_mem->dcr ? "" : " DCR");
 487                 return -ENODEV;
 488         }
 489
 490         /*
 491          * We've found enough to create an nvdimm, optionally
 492          * find an associated BDW
 493          */
 494         list_add(&nfit_mem->list, &acpi_desc->dimms);
 495
 496         list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) {
 497                 if (nfit_bdw->bdw->region_index != dcr)
 498                         continue;
 499                 nfit_mem->bdw = nfit_bdw->bdw;
 500                 break;
 501         }
 502
 503         if (!nfit_mem->bdw)
 504                 return 0;
 505
 506         nfit_mem_find_spa_bdw(acpi_desc, nfit_mem);
 507
 508         if (!nfit_mem->spa_bdw)
 509                 return 0;
 510
 511         range_index = nfit_mem->spa_bdw->range_index;
 512         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
 513                 if (nfit_memdev->memdev->range_index != range_index ||
 514                                 nfit_memdev->memdev->region_index != dcr)
 515                         continue;
 516                 nfit_mem->memdev_bdw = nfit_memdev->memdev;
 517                 idt_idx = nfit_memdev->memdev->interleave_index;
 518                 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
 519                         if (nfit_idt->idt->interleave_index != idt_idx)
 520                                 continue;
 521                         nfit_mem->idt_bdw = nfit_idt->idt;
 522                         break;
 523                 }
 524
 525                 list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) {
 526                         if (nfit_flush->flush->device_handle !=
 527                                         nfit_memdev->memdev->device_handle)
 528                                 continue;
 529                         nfit_mem->nfit_flush = nfit_flush;
 530                         break;
 531                 }
 532                 break;
 533         }
 534
 535         return 0;
 536 }
 537
 538 static int nfit_mem_dcr_init(struct acpi_nfit_desc *acpi_desc,
 539                 struct acpi_nfit_system_address *spa)
 540 {
 541         struct nfit_mem *nfit_mem, *found;
 542         struct nfit_memdev *nfit_memdev;
 543         int type = nfit_spa_type(spa);
 544         u16 dcr;
 545
 546         switch (type) {
 547         case NFIT_SPA_DCR:
 548         case NFIT_SPA_PM:
 549                 break;
 550         default:
 551                 return 0;
 552         }
 553
 554         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
 555                 int rc;
 556
 557                 if (nfit_memdev->memdev->range_index != spa->range_index)
 558                         continue;
 559                 found = NULL;
 560                 dcr = nfit_memdev->memdev->region_index;
 561                 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
 562                         if (__to_nfit_memdev(nfit_mem)->region_index == dcr) {
 563                                 found = nfit_mem;
 564                                 break;
 565                         }
 566
 567                 if (found)
 568                         nfit_mem = found;
 569                 else {
 570                         nfit_mem = devm_kzalloc(acpi_desc->dev,
 571                                         sizeof(*nfit_mem), GFP_KERNEL);
 572                         if (!nfit_mem)
 573                                 return -ENOMEM;
 574                         INIT_LIST_HEAD(&nfit_mem->list);
 575                 }
 576
 577                 if (type == NFIT_SPA_DCR) {
 578                         struct nfit_idt *nfit_idt;
 579                         u16 idt_idx;
 580
 581                         /* multiple dimms may share a SPA when interleaved */
 582                         nfit_mem->spa_dcr = spa;
 583                         nfit_mem->memdev_dcr = nfit_memdev->memdev;
 584                         idt_idx = nfit_memdev->memdev->interleave_index;
 585                         list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
 586                                 if (nfit_idt->idt->interleave_index != idt_idx)
 587                                         continue;
 588                                 nfit_mem->idt_dcr = nfit_idt->idt;
 589                                 break;
 590                         }
 591                 } else {
 592                         /*
 593                          * A single dimm may belong to multiple SPA-PM
 594                          * ranges, record at least one in addition to
 595                          * any SPA-DCR range.
 596                          */
 597                         nfit_mem->memdev_pmem = nfit_memdev->memdev;
 598                 }
 599
 600                 if (found)
 601                         continue;
 602
 603                 rc = nfit_mem_add(acpi_desc, nfit_mem, spa);
 604                 if (rc)
 605                         return rc;
 606         }
 607
 608         return 0;
 609 }
 610
 611 static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b)
 612 {
 613         struct nfit_mem *a = container_of(_a, typeof(*a), list);
 614         struct nfit_mem *b = container_of(_b, typeof(*b), list);
 615         u32 handleA, handleB;
 616
 617         handleA = __to_nfit_memdev(a)->device_handle;
 618         handleB = __to_nfit_memdev(b)->device_handle;
 619         if (handleA < handleB)
 620                 return -1;
 621         else if (handleA > handleB)
 622                 return 1;
 623         return 0;
 624 }
 625
 626 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc)
 627 {
 628         struct nfit_spa *nfit_spa;
 629
 630         /*
 631          * For each SPA-DCR or SPA-PMEM address range find its
 632          * corresponding MEMDEV(s).  From each MEMDEV find the
 633          * corresponding DCR.  Then, if we're operating on a SPA-DCR,
 634          * try to find a SPA-BDW and a corresponding BDW that references
 635          * the DCR.  Throw it all into an nfit_mem object.  Note, that
 636          * BDWs are optional.
 637          */
 638         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
 639                 int rc;
 640
 641                 rc = nfit_mem_dcr_init(acpi_desc, nfit_spa->spa);
 642                 if (rc)
 643                         return rc;
 644         }
 645
 646         list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp);
 647
 648         return 0;
 649 }
 650
 651 static ssize_t revision_show(struct device *dev,
 652                 struct device_attribute *attr, char *buf)
 653 {
 654         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
 655         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
 656         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
 657
 658         return sprintf(buf, "%d\n", acpi_desc->nfit->header.revision);
 659 }
 660 static DEVICE_ATTR_RO(revision);
 661
 662 static struct attribute *acpi_nfit_attributes[] = {
 663         &dev_attr_revision.attr,
 664         NULL,
 665 };
 666
 667 static struct attribute_group acpi_nfit_attribute_group = {
 668         .name = "nfit",
 669         .attrs = acpi_nfit_attributes,
 670 };
 671
 672 const struct attribute_group *acpi_nfit_attribute_groups[] = {
 673         &nvdimm_bus_attribute_group,
 674         &acpi_nfit_attribute_group,
 675         NULL,
 676 };
 677 EXPORT_SYMBOL_GPL(acpi_nfit_attribute_groups);
 678
 679 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev)
 680 {
 681         struct nvdimm *nvdimm = to_nvdimm(dev);
 682         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 683
 684         return __to_nfit_memdev(nfit_mem);
 685 }
 686
 687 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev)
 688 {
 689         struct nvdimm *nvdimm = to_nvdimm(dev);
 690         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 691
 692         return nfit_mem->dcr;
 693 }
 694
 695 static ssize_t handle_show(struct device *dev,
 696                 struct device_attribute *attr, char *buf)
 697 {
 698         struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
 699
 700         return sprintf(buf, "%#x\n", memdev->device_handle);
 701 }
 702 static DEVICE_ATTR_RO(handle);
 703
 704 static ssize_t phys_id_show(struct device *dev,
 705                 struct device_attribute *attr, char *buf)
 706 {
 707         struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
 708
 709         return sprintf(buf, "%#x\n", memdev->physical_id);
 710 }
 711 static DEVICE_ATTR_RO(phys_id);
 712
 713 static ssize_t vendor_show(struct device *dev,
 714                 struct device_attribute *attr, char *buf)
 715 {
 716         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 717
 718         return sprintf(buf, "%#x\n", dcr->vendor_id);
 719 }
 720 static DEVICE_ATTR_RO(vendor);
 721
 722 static ssize_t rev_id_show(struct device *dev,
 723                 struct device_attribute *attr, char *buf)
 724 {
 725         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 726
 727         return sprintf(buf, "%#x\n", dcr->revision_id);
 728 }
 729 static DEVICE_ATTR_RO(rev_id);
 730
 731 static ssize_t device_show(struct device *dev,
 732                 struct device_attribute *attr, char *buf)
 733 {
 734         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 735
 736         return sprintf(buf, "%#x\n", dcr->device_id);
 737 }
 738 static DEVICE_ATTR_RO(device);
 739
 740 static ssize_t format_show(struct device *dev,
 741                 struct device_attribute *attr, char *buf)
 742 {
 743         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 744
 745         return sprintf(buf, "%#x\n", dcr->code);
 746 }
 747 static DEVICE_ATTR_RO(format);
 748
 749 static ssize_t serial_show(struct device *dev,
 750                 struct device_attribute *attr, char *buf)
 751 {
 752         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 753
 754         return sprintf(buf, "%#x\n", dcr->serial_number);
 755 }
 756 static DEVICE_ATTR_RO(serial);
 757
 758 static ssize_t flags_show(struct device *dev,
 759                 struct device_attribute *attr, char *buf)
 760 {
 761         u16 flags = to_nfit_memdev(dev)->flags;
 762
 763         return sprintf(buf, "%s%s%s%s%s\n",
 764                 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "",
 765                 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "",
 766                 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "",
 767                 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "",
 768                 flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "");
 769 }
 770 static DEVICE_ATTR_RO(flags);
 771
 772 static struct attribute *acpi_nfit_dimm_attributes[] = {
 773         &dev_attr_handle.attr,
 774         &dev_attr_phys_id.attr,
 775         &dev_attr_vendor.attr,
 776         &dev_attr_device.attr,
 777         &dev_attr_format.attr,
 778         &dev_attr_serial.attr,
 779         &dev_attr_rev_id.attr,
 780         &dev_attr_flags.attr,
 781         NULL,
 782 };
 783
 784 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj,
 785                 struct attribute *a, int n)
 786 {
 787         struct device *dev = container_of(kobj, struct device, kobj);
 788
 789         if (to_nfit_dcr(dev))
 790                 return a->mode;
 791         else
 792                 return 0;
 793 }
 794
 795 static struct attribute_group acpi_nfit_dimm_attribute_group = {
 796         .name = "nfit",
 797         .attrs = acpi_nfit_dimm_attributes,
 798         .is_visible = acpi_nfit_dimm_attr_visible,
 799 };
 800
 801 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = {
 802         &nvdimm_attribute_group,
 803         &nd_device_attribute_group,
 804         &acpi_nfit_dimm_attribute_group,
 805         NULL,
 806 };
 807
 808 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc,
 809                 u32 device_handle)
 810 {
 811         struct nfit_mem *nfit_mem;
 812
 813         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
 814                 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle)
 815                         return nfit_mem->nvdimm;
 816
 817         return NULL;
 818 }
 819
 820 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
 821                 struct nfit_mem *nfit_mem, u32 device_handle)
 822 {
 823         struct acpi_device *adev, *adev_dimm;
 824         struct device *dev = acpi_desc->dev;
 825         const u8 *uuid = to_nfit_uuid(NFIT_DEV_DIMM);
 826         int i;
 827
 828         nfit_mem->dsm_mask = acpi_desc->dimm_dsm_force_en;
 829         adev = to_acpi_dev(acpi_desc);
 830         if (!adev)
 831                 return 0;
 832
 833         adev_dimm = acpi_find_child_device(adev, device_handle, false);
 834         nfit_mem->adev = adev_dimm;
 835         if (!adev_dimm) {
 836                 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n",
 837                                 device_handle);
 838                 return force_enable_dimms ? 0 : -ENODEV;
 839         }
 840
 841         for (i = ND_CMD_SMART; i <= ND_CMD_VENDOR; i++)
 842                 if (acpi_check_dsm(adev_dimm->handle, uuid, 1, 1ULL << i))
 843                         set_bit(i, &nfit_mem->dsm_mask);
 844
 845         return 0;
 846 }
 847
 848 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
 849 {
 850         struct nfit_mem *nfit_mem;
 851         int dimm_count = 0;
 852
 853         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
 854                 struct nvdimm *nvdimm;
 855                 unsigned long flags = 0;
 856                 u32 device_handle;
 857                 u16 mem_flags;
 858                 int rc;
 859
 860                 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
 861                 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle);
 862                 if (nvdimm) {
 863                         dimm_count++;
 864                         continue;
 865                 }
 866
 867                 if (nfit_mem->bdw && nfit_mem->memdev_pmem)
 868                         flags |= NDD_ALIASING;
 869
 870                 mem_flags = __to_nfit_memdev(nfit_mem)->flags;
 871                 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED)
 872                         flags |= NDD_UNARMED;
 873
 874                 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle);
 875                 if (rc)
 876                         continue;
 877
 878                 nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem,
 879                                 acpi_nfit_dimm_attribute_groups,
 880                                 flags, &nfit_mem->dsm_mask);
 881                 if (!nvdimm)
 882                         return -ENOMEM;
 883
 884                 nfit_mem->nvdimm = nvdimm;
 885                 dimm_count++;
 886
 887                 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0)
 888                         continue;
 889
 890                 dev_info(acpi_desc->dev, "%s flags:%s%s%s%s\n",
 891                                 nvdimm_name(nvdimm),
 892                   mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "",
 893                   mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"",
 894                   mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "",
 895                   mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "");
 896
 897         }
 898
 899         return nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count);
 900 }
 901
 902 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc)
 903 {
 904         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
 905         const u8 *uuid = to_nfit_uuid(NFIT_DEV_BUS);
 906         struct acpi_device *adev;
 907         int i;
 908
 909         nd_desc->dsm_mask = acpi_desc->bus_dsm_force_en;
 910         adev = to_acpi_dev(acpi_desc);
 911         if (!adev)
 912                 return;
 913
 914         for (i = ND_CMD_ARS_CAP; i <= ND_CMD_ARS_STATUS; i++)
 915                 if (acpi_check_dsm(adev->handle, uuid, 1, 1ULL << i))
 916                         set_bit(i, &nd_desc->dsm_mask);
 917 }
 918
 919 static ssize_t range_index_show(struct device *dev,
 920                 struct device_attribute *attr, char *buf)
 921 {
 922         struct nd_region *nd_region = to_nd_region(dev);
 923         struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region);
 924
 925         return sprintf(buf, "%d\n", nfit_spa->spa->range_index);
 926 }
 927 static DEVICE_ATTR_RO(range_index);
 928
 929 static struct attribute *acpi_nfit_region_attributes[] = {
 930         &dev_attr_range_index.attr,
 931         NULL,
 932 };
 933
 934 static struct attribute_group acpi_nfit_region_attribute_group = {
 935         .name = "nfit",
 936         .attrs = acpi_nfit_region_attributes,
 937 };
 938
 939 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = {
 940         &nd_region_attribute_group,
 941         &nd_mapping_attribute_group,
 942         &nd_device_attribute_group,
 943         &nd_numa_attribute_group,
 944         &acpi_nfit_region_attribute_group,
 945         NULL,
 946 };
 947
 948 /* enough info to uniquely specify an interleave set */
 949 struct nfit_set_info {
 950         struct nfit_set_info_map {
 951                 u64 region_offset;
 952                 u32 serial_number;
 953                 u32 pad;
 954         } mapping[0];
 955 };
 956
 957 static size_t sizeof_nfit_set_info(int num_mappings)
 958 {
 959         return sizeof(struct nfit_set_info)
 960                 + num_mappings * sizeof(struct nfit_set_info_map);
 961 }
 962
 963 static int cmp_map(const void *m0, const void *m1)
 964 {
 965         const struct nfit_set_info_map *map0 = m0;
 966         const struct nfit_set_info_map *map1 = m1;
 967
 968         return memcmp(&map0->region_offset, &map1->region_offset,
 969                         sizeof(u64));
 970 }
 971
 972 /* Retrieve the nth entry referencing this spa */
 973 static struct acpi_nfit_memory_map *memdev_from_spa(
 974                 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n)
 975 {
 976         struct nfit_memdev *nfit_memdev;
 977
 978         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list)
 979                 if (nfit_memdev->memdev->range_index == range_index)
 980                         if (n-- == 0)
 981                                 return nfit_memdev->memdev;
 982         return NULL;
 983 }
 984
 985 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
 986                 struct nd_region_desc *ndr_desc,
 987                 struct acpi_nfit_system_address *spa)
 988 {
 989         int i, spa_type = nfit_spa_type(spa);
 990         struct device *dev = acpi_desc->dev;
 991         struct nd_interleave_set *nd_set;
 992         u16 nr = ndr_desc->num_mappings;
 993         struct nfit_set_info *info;
 994
 995         if (spa_type == NFIT_SPA_PM || spa_type == NFIT_SPA_VOLATILE)
 996                 /* pass */;
 997         else
 998                 return 0;
 999
1000         nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
1001         if (!nd_set)
1002                 return -ENOMEM;
1003
1004         info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL);
1005         if (!info)
1006                 return -ENOMEM;
1007         for (i = 0; i < nr; i++) {
1008                 struct nd_mapping *nd_mapping = &ndr_desc->nd_mapping[i];
1009                 struct nfit_set_info_map *map = &info->mapping[i];
1010                 struct nvdimm *nvdimm = nd_mapping->nvdimm;
1011                 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1012                 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc,
1013                                 spa->range_index, i);
1014
1015                 if (!memdev || !nfit_mem->dcr) {
1016                         dev_err(dev, "%s: failed to find DCR\n", __func__);
1017                         return -ENODEV;
1018                 }
1019
1020                 map->region_offset = memdev->region_offset;
1021                 map->serial_number = nfit_mem->dcr->serial_number;
1022         }
1023
1024         sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
1025                         cmp_map, NULL);
1026         nd_set->cookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
1027         ndr_desc->nd_set = nd_set;
1028         devm_kfree(dev, info);
1029
1030         return 0;
1031 }
1032
1033 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio)
1034 {
1035         struct acpi_nfit_interleave *idt = mmio->idt;
1036         u32 sub_line_offset, line_index, line_offset;
1037         u64 line_no, table_skip_count, table_offset;
1038
1039         line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset);
1040         table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index);
1041         line_offset = idt->line_offset[line_index]
1042                 * mmio->line_size;
1043         table_offset = table_skip_count * mmio->table_size;
1044
1045         return mmio->base_offset + line_offset + table_offset + sub_line_offset;
1046 }
1047
1048 static void wmb_blk(struct nfit_blk *nfit_blk)
1049 {
1050
1051         if (nfit_blk->nvdimm_flush) {
1052                 /*
1053                  * The first wmb() is needed to 'sfence' all previous writes
1054                  * such that they are architecturally visible for the platform
1055                  * buffer flush.  Note that we've already arranged for pmem
1056                  * writes to avoid the cache via arch_memcpy_to_pmem().  The
1057                  * final wmb() ensures ordering for the NVDIMM flush write.
1058                  */
1059                 wmb();
1060                 writeq(1, nfit_blk->nvdimm_flush);
1061                 wmb();
1062         } else
1063                 wmb_pmem();
1064 }
1065
1066 static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw)
1067 {
1068         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
1069         u64 offset = nfit_blk->stat_offset + mmio->size * bw;
1070
1071         if (mmio->num_lines)
1072                 offset = to_interleave_offset(offset, mmio);
1073
1074         return readl(mmio->addr.base + offset);
1075 }
1076
1077 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,
1078                 resource_size_t dpa, unsigned int len, unsigned int write)
1079 {
1080         u64 cmd, offset;
1081         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
1082
1083         enum {
1084                 BCW_OFFSET_MASK = (1ULL << 48)-1,
1085                 BCW_LEN_SHIFT = 48,
1086                 BCW_LEN_MASK = (1ULL << 8) - 1,
1087                 BCW_CMD_SHIFT = 56,
1088         };
1089
1090         cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK;
1091         len = len >> L1_CACHE_SHIFT;
1092         cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT;
1093         cmd |= ((u64) write) << BCW_CMD_SHIFT;
1094
1095         offset = nfit_blk->cmd_offset + mmio->size * bw;
1096         if (mmio->num_lines)
1097                 offset = to_interleave_offset(offset, mmio);
1098
1099         writeq(cmd, mmio->addr.base + offset);
1100         wmb_blk(nfit_blk);
1101
1102         if (nfit_blk->dimm_flags & ND_BLK_DCR_LATCH)
1103                 readq(mmio->addr.base + offset);
1104 }
1105
1106 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
1107                 resource_size_t dpa, void *iobuf, size_t len, int rw,
1108                 unsigned int lane)
1109 {
1110         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
1111         unsigned int copied = 0;
1112         u64 base_offset;
1113         int rc;
1114
1115         base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES
1116                 + lane * mmio->size;
1117         write_blk_ctl(nfit_blk, lane, dpa, len, rw);
1118         while (len) {
1119                 unsigned int c;
1120                 u64 offset;
1121
1122                 if (mmio->num_lines) {
1123                         u32 line_offset;
1124
1125                         offset = to_interleave_offset(base_offset + copied,
1126                                         mmio);
1127                         div_u64_rem(offset, mmio->line_size, &line_offset);
1128                         c = min_t(size_t, len, mmio->line_size - line_offset);
1129                 } else {
1130                         offset = base_offset + nfit_blk->bdw_offset;
1131                         c = len;
1132                 }
1133
1134                 if (rw)
1135                         memcpy_to_pmem(mmio->addr.aperture + offset,
1136                                         iobuf + copied, c);
1137                 else {
1138                         if (nfit_blk->dimm_flags & ND_BLK_READ_FLUSH)
1139                                 mmio_flush_range((void __force *)
1140                                         mmio->addr.aperture + offset, c);
1141
1142                         memcpy_from_pmem(iobuf + copied,
1143                                         mmio->addr.aperture + offset, c);
1144                 }
1145
1146                 copied += c;
1147                 len -= c;
1148         }
1149
1150         if (rw)
1151                 wmb_blk(nfit_blk);
1152
1153         rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0;
1154         return rc;
1155 }
1156
1157 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr,
1158                 resource_size_t dpa, void *iobuf, u64 len, int rw)
1159 {
1160         struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
1161         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
1162         struct nd_region *nd_region = nfit_blk->nd_region;
1163         unsigned int lane, copied = 0;
1164         int rc = 0;
1165
1166         lane = nd_region_acquire_lane(nd_region);
1167         while (len) {
1168                 u64 c = min(len, mmio->size);
1169
1170                 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied,
1171                                 iobuf + copied, c, rw, lane);
1172                 if (rc)
1173                         break;
1174
1175                 copied += c;
1176                 len -= c;
1177         }
1178         nd_region_release_lane(nd_region, lane);
1179
1180         return rc;
1181 }
1182
1183 static void nfit_spa_mapping_release(struct kref *kref)
1184 {
1185         struct nfit_spa_mapping *spa_map = to_spa_map(kref);
1186         struct acpi_nfit_system_address *spa = spa_map->spa;
1187         struct acpi_nfit_desc *acpi_desc = spa_map->acpi_desc;
1188
1189         WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1190         dev_dbg(acpi_desc->dev, "%s: SPA%d\n", __func__, spa->range_index);
1191         if (spa_map->type == SPA_MAP_APERTURE)
1192                 memunmap((void __force *)spa_map->addr.aperture);
1193         else
1194                 iounmap(spa_map->addr.base);
1195         release_mem_region(spa->address, spa->length);
1196         list_del(&spa_map->list);
1197         kfree(spa_map);
1198 }
1199
1200 static struct nfit_spa_mapping *find_spa_mapping(
1201                 struct acpi_nfit_desc *acpi_desc,
1202                 struct acpi_nfit_system_address *spa)
1203 {
1204         struct nfit_spa_mapping *spa_map;
1205
1206         WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1207         list_for_each_entry(spa_map, &acpi_desc->spa_maps, list)
1208                 if (spa_map->spa == spa)
1209                         return spa_map;
1210
1211         return NULL;
1212 }
1213
1214 static void nfit_spa_unmap(struct acpi_nfit_desc *acpi_desc,
1215                 struct acpi_nfit_system_address *spa)
1216 {
1217         struct nfit_spa_mapping *spa_map;
1218
1219         mutex_lock(&acpi_desc->spa_map_mutex);
1220         spa_map = find_spa_mapping(acpi_desc, spa);
1221
1222         if (spa_map)
1223                 kref_put(&spa_map->kref, nfit_spa_mapping_release);
1224         mutex_unlock(&acpi_desc->spa_map_mutex);
1225 }
1226
1227 static void __iomem *__nfit_spa_map(struct acpi_nfit_desc *acpi_desc,
1228                 struct acpi_nfit_system_address *spa, enum spa_map_type type)
1229 {
1230         resource_size_t start = spa->address;
1231         resource_size_t n = spa->length;
1232         struct nfit_spa_mapping *spa_map;
1233         struct resource *res;
1234
1235         WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1236
1237         spa_map = find_spa_mapping(acpi_desc, spa);
1238         if (spa_map) {
1239                 kref_get(&spa_map->kref);
1240                 return spa_map->addr.base;
1241         }
1242
1243         spa_map = kzalloc(sizeof(*spa_map), GFP_KERNEL);
1244         if (!spa_map)
1245                 return NULL;
1246
1247         INIT_LIST_HEAD(&spa_map->list);
1248         spa_map->spa = spa;
1249         kref_init(&spa_map->kref);
1250         spa_map->acpi_desc = acpi_desc;
1251
1252         res = request_mem_region(start, n, dev_name(acpi_desc->dev));
1253         if (!res)
1254                 goto err_mem;
1255
1256         spa_map->type = type;
1257         if (type == SPA_MAP_APERTURE)
1258                 spa_map->addr.aperture = (void __pmem *)memremap(start, n,
1259                                                         ARCH_MEMREMAP_PMEM);
1260         else
1261                 spa_map->addr.base = ioremap_nocache(start, n);
1262
1263
1264         if (!spa_map->addr.base)
1265                 goto err_map;
1266
1267         list_add_tail(&spa_map->list, &acpi_desc->spa_maps);
1268         return spa_map->addr.base;
1269
1270  err_map:
1271         release_mem_region(start, n);
1272  err_mem:
1273         kfree(spa_map);
1274         return NULL;
1275 }
1276
1277 /**
1278  * nfit_spa_map - interleave-aware managed-mappings of acpi_nfit_system_address ranges
1279  * @nvdimm_bus: NFIT-bus that provided the spa table entry
1280  * @nfit_spa: spa table to map
1281  * @type: aperture or control region
1282  *
1283  * In the case where block-data-window apertures and
1284  * dimm-control-regions are interleaved they will end up sharing a
1285  * single request_mem_region() + ioremap() for the address range.  In
1286  * the style of devm nfit_spa_map() mappings are automatically dropped
1287  * when all region devices referencing the same mapping are disabled /
1288  * unbound.
1289  */
1290 static void __iomem *nfit_spa_map(struct acpi_nfit_desc *acpi_desc,
1291                 struct acpi_nfit_system_address *spa, enum spa_map_type type)
1292 {
1293         void __iomem *iomem;
1294
1295         mutex_lock(&acpi_desc->spa_map_mutex);
1296         iomem = __nfit_spa_map(acpi_desc, spa, type);
1297         mutex_unlock(&acpi_desc->spa_map_mutex);
1298
1299         return iomem;
1300 }
1301
1302 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio,
1303                 struct acpi_nfit_interleave *idt, u16 interleave_ways)
1304 {
1305         if (idt) {
1306                 mmio->num_lines = idt->line_count;
1307                 mmio->line_size = idt->line_size;
1308                 if (interleave_ways == 0)
1309                         return -ENXIO;
1310                 mmio->table_size = mmio->num_lines * interleave_ways
1311                         * mmio->line_size;
1312         }
1313
1314         return 0;
1315 }
1316
1317 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc,
1318                 struct nvdimm *nvdimm, struct nfit_blk *nfit_blk)
1319 {
1320         struct nd_cmd_dimm_flags flags;
1321         int rc;
1322
1323         memset(&flags, 0, sizeof(flags));
1324         rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags,
1325                         sizeof(flags));
1326
1327         if (rc >= 0 && flags.status == 0)
1328                 nfit_blk->dimm_flags = flags.flags;
1329         else if (rc == -ENOTTY) {
1330                 /* fall back to a conservative default */
1331                 nfit_blk->dimm_flags = ND_BLK_DCR_LATCH | ND_BLK_READ_FLUSH;
1332                 rc = 0;
1333         } else
1334                 rc = -ENXIO;
1335
1336         return rc;
1337 }
1338
1339 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus,
1340                 struct device *dev)
1341 {
1342         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1343         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1344         struct nd_blk_region *ndbr = to_nd_blk_region(dev);
1345         struct nfit_flush *nfit_flush;
1346         struct nfit_blk_mmio *mmio;
1347         struct nfit_blk *nfit_blk;
1348         struct nfit_mem *nfit_mem;
1349         struct nvdimm *nvdimm;
1350         int rc;
1351
1352         nvdimm = nd_blk_region_to_dimm(ndbr);
1353         nfit_mem = nvdimm_provider_data(nvdimm);
1354         if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) {
1355                 dev_dbg(dev, "%s: missing%s%s%s\n", __func__,
1356                                 nfit_mem ? "" : " nfit_mem",
1357                                 (nfit_mem && nfit_mem->dcr) ? "" : " dcr",
1358                                 (nfit_mem && nfit_mem->bdw) ? "" : " bdw");
1359                 return -ENXIO;
1360         }
1361
1362         nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL);
1363         if (!nfit_blk)
1364                 return -ENOMEM;
1365         nd_blk_region_set_provider_data(ndbr, nfit_blk);
1366         nfit_blk->nd_region = to_nd_region(dev);
1367
1368         /* map block aperture memory */
1369         nfit_blk->bdw_offset = nfit_mem->bdw->offset;
1370         mmio = &nfit_blk->mmio[BDW];
1371         mmio->addr.base = nfit_spa_map(acpi_desc, nfit_mem->spa_bdw,
1372                         SPA_MAP_APERTURE);
1373         if (!mmio->addr.base) {
1374                 dev_dbg(dev, "%s: %s failed to map bdw\n", __func__,
1375                                 nvdimm_name(nvdimm));
1376                 return -ENOMEM;
1377         }
1378         mmio->size = nfit_mem->bdw->size;
1379         mmio->base_offset = nfit_mem->memdev_bdw->region_offset;
1380         mmio->idt = nfit_mem->idt_bdw;
1381         mmio->spa = nfit_mem->spa_bdw;
1382         rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw,
1383                         nfit_mem->memdev_bdw->interleave_ways);
1384         if (rc) {
1385                 dev_dbg(dev, "%s: %s failed to init bdw interleave\n",
1386                                 __func__, nvdimm_name(nvdimm));
1387                 return rc;
1388         }
1389
1390         /* map block control memory */
1391         nfit_blk->cmd_offset = nfit_mem->dcr->command_offset;
1392         nfit_blk->stat_offset = nfit_mem->dcr->status_offset;
1393         mmio = &nfit_blk->mmio[DCR];
1394         mmio->addr.base = nfit_spa_map(acpi_desc, nfit_mem->spa_dcr,
1395                         SPA_MAP_CONTROL);
1396         if (!mmio->addr.base) {
1397                 dev_dbg(dev, "%s: %s failed to map dcr\n", __func__,
1398                                 nvdimm_name(nvdimm));
1399                 return -ENOMEM;
1400         }
1401         mmio->size = nfit_mem->dcr->window_size;
1402         mmio->base_offset = nfit_mem->memdev_dcr->region_offset;
1403         mmio->idt = nfit_mem->idt_dcr;
1404         mmio->spa = nfit_mem->spa_dcr;
1405         rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr,
1406                         nfit_mem->memdev_dcr->interleave_ways);
1407         if (rc) {
1408                 dev_dbg(dev, "%s: %s failed to init dcr interleave\n",
1409                                 __func__, nvdimm_name(nvdimm));
1410                 return rc;
1411         }
1412
1413         rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk);
1414         if (rc < 0) {
1415                 dev_dbg(dev, "%s: %s failed get DIMM flags\n",
1416                                 __func__, nvdimm_name(nvdimm));
1417                 return rc;
1418         }
1419
1420         nfit_flush = nfit_mem->nfit_flush;
1421         if (nfit_flush && nfit_flush->flush->hint_count != 0) {
1422                 nfit_blk->nvdimm_flush = devm_ioremap_nocache(dev,
1423                                 nfit_flush->flush->hint_address[0], 8);
1424                 if (!nfit_blk->nvdimm_flush)
1425                         return -ENOMEM;
1426         }
1427
1428         if (!arch_has_wmb_pmem() && !nfit_blk->nvdimm_flush)
1429                 dev_warn(dev, "unable to guarantee persistence of writes\n");
1430
1431         if (mmio->line_size == 0)
1432                 return 0;
1433
1434         if ((u32) nfit_blk->cmd_offset % mmio->line_size
1435                         + 8 > mmio->line_size) {
1436                 dev_dbg(dev, "cmd_offset crosses interleave boundary\n");
1437                 return -ENXIO;
1438         } else if ((u32) nfit_blk->stat_offset % mmio->line_size
1439                         + 8 > mmio->line_size) {
1440                 dev_dbg(dev, "stat_offset crosses interleave boundary\n");
1441                 return -ENXIO;
1442         }
1443
1444         return 0;
1445 }
1446
1447 static void acpi_nfit_blk_region_disable(struct nvdimm_bus *nvdimm_bus,
1448                 struct device *dev)
1449 {
1450         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1451         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1452         struct nd_blk_region *ndbr = to_nd_blk_region(dev);
1453         struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
1454         int i;
1455
1456         if (!nfit_blk)
1457                 return; /* never enabled */
1458
1459         /* auto-free BLK spa mappings */
1460         for (i = 0; i < 2; i++) {
1461                 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[i];
1462
1463                 if (mmio->addr.base)
1464                         nfit_spa_unmap(acpi_desc, mmio->spa);
1465         }
1466         nd_blk_region_set_provider_data(ndbr, NULL);
1467         /* devm will free nfit_blk */
1468 }
1469
1470 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc,
1471                 struct nd_mapping *nd_mapping, struct nd_region_desc *ndr_desc,
1472                 struct acpi_nfit_memory_map *memdev,
1473                 struct acpi_nfit_system_address *spa)
1474 {
1475         struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc,
1476                         memdev->device_handle);
1477         struct nd_blk_region_desc *ndbr_desc;
1478         struct nfit_mem *nfit_mem;
1479         int blk_valid = 0;
1480
1481         if (!nvdimm) {
1482                 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n",
1483                                 spa->range_index, memdev->device_handle);
1484                 return -ENODEV;
1485         }
1486
1487         nd_mapping->nvdimm = nvdimm;
1488         switch (nfit_spa_type(spa)) {
1489         case NFIT_SPA_PM:
1490         case NFIT_SPA_VOLATILE:
1491                 nd_mapping->start = memdev->address;
1492                 nd_mapping->size = memdev->region_size;
1493                 break;
1494         case NFIT_SPA_DCR:
1495                 nfit_mem = nvdimm_provider_data(nvdimm);
1496                 if (!nfit_mem || !nfit_mem->bdw) {
1497                         dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n",
1498                                         spa->range_index, nvdimm_name(nvdimm));
1499                 } else {
1500                         nd_mapping->size = nfit_mem->bdw->capacity;
1501                         nd_mapping->start = nfit_mem->bdw->start_address;
1502                         ndr_desc->num_lanes = nfit_mem->bdw->windows;
1503                         blk_valid = 1;
1504                 }
1505
1506                 ndr_desc->nd_mapping = nd_mapping;
1507                 ndr_desc->num_mappings = blk_valid;
1508                 ndbr_desc = to_blk_region_desc(ndr_desc);
1509                 ndbr_desc->enable = acpi_nfit_blk_region_enable;
1510                 ndbr_desc->disable = acpi_nfit_blk_region_disable;
1511                 ndbr_desc->do_io = acpi_desc->blk_do_io;
1512                 if (!nvdimm_blk_region_create(acpi_desc->nvdimm_bus, ndr_desc))
1513                         return -ENOMEM;
1514                 break;
1515         }
1516
1517         return 0;
1518 }
1519
1520 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
1521                 struct nfit_spa *nfit_spa)
1522 {
1523         static struct nd_mapping nd_mappings[ND_MAX_MAPPINGS];
1524         struct acpi_nfit_system_address *spa = nfit_spa->spa;
1525         struct nd_blk_region_desc ndbr_desc;
1526         struct nd_region_desc *ndr_desc;
1527         struct nfit_memdev *nfit_memdev;
1528         struct nvdimm_bus *nvdimm_bus;
1529         struct resource res;
1530         int count = 0, rc;
1531
1532         if (nfit_spa->is_registered)
1533                 return 0;
1534
1535         if (spa->range_index == 0) {
1536                 dev_dbg(acpi_desc->dev, "%s: detected invalid spa index\n",
1537                                 __func__);
1538                 return 0;
1539         }
1540
1541         memset(&res, 0, sizeof(res));
1542         memset(&nd_mappings, 0, sizeof(nd_mappings));
1543         memset(&ndbr_desc, 0, sizeof(ndbr_desc));
1544         res.start = spa->address;
1545         res.end = res.start + spa->length - 1;
1546         ndr_desc = &ndbr_desc.ndr_desc;
1547         ndr_desc->res = &res;
1548         ndr_desc->provider_data = nfit_spa;
1549         ndr_desc->attr_groups = acpi_nfit_region_attribute_groups;
1550         if (spa->flags & ACPI_NFIT_PROXIMITY_VALID)
1551                 ndr_desc->numa_node = acpi_map_pxm_to_online_node(
1552                                                 spa->proximity_domain);
1553         else
1554                 ndr_desc->numa_node = NUMA_NO_NODE;
1555
1556         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1557                 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
1558                 struct nd_mapping *nd_mapping;
1559
1560                 if (memdev->range_index != spa->range_index)
1561                         continue;
1562                 if (count >= ND_MAX_MAPPINGS) {
1563                         dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n",
1564                                         spa->range_index, ND_MAX_MAPPINGS);
1565                         return -ENXIO;
1566                 }
1567                 nd_mapping = &nd_mappings[count++];
1568                 rc = acpi_nfit_init_mapping(acpi_desc, nd_mapping, ndr_desc,
1569                                 memdev, spa);
1570                 if (rc)
1571                         return rc;
1572         }
1573
1574         ndr_desc->nd_mapping = nd_mappings;
1575         ndr_desc->num_mappings = count;
1576         rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
1577         if (rc)
1578                 return rc;
1579
1580         nvdimm_bus = acpi_desc->nvdimm_bus;
1581         if (nfit_spa_type(spa) == NFIT_SPA_PM) {
1582                 if (!nvdimm_pmem_region_create(nvdimm_bus, ndr_desc))
1583                         return -ENOMEM;
1584         } else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) {
1585                 if (!nvdimm_volatile_region_create(nvdimm_bus, ndr_desc))
1586                         return -ENOMEM;
1587         }
1588
1589         nfit_spa->is_registered = 1;
1590         return 0;
1591 }
1592
1593 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc)
1594 {
1595         struct nfit_spa *nfit_spa;
1596
1597         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
1598                 int rc = acpi_nfit_register_region(acpi_desc, nfit_spa);
1599
1600                 if (rc)
1601                         return rc;
1602         }
1603         return 0;
1604 }
1605
1606 static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc,
1607                 struct nfit_table_prev *prev)
1608 {
1609         struct device *dev = acpi_desc->dev;
1610
1611         if (!list_empty(&prev->spas) ||
1612                         !list_empty(&prev->memdevs) ||
1613                         !list_empty(&prev->dcrs) ||
1614                         !list_empty(&prev->bdws) ||
1615                         !list_empty(&prev->idts) ||
1616                         !list_empty(&prev->flushes)) {
1617                 dev_err(dev, "new nfit deletes entries (unsupported)\n");
1618                 return -ENXIO;
1619         }
1620         return 0;
1621 }
1622
1623 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, acpi_size sz)
1624 {
1625         struct device *dev = acpi_desc->dev;
1626         struct nfit_table_prev prev;
1627         const void *end;
1628         u8 *data;
1629         int rc;
1630
1631         mutex_lock(&acpi_desc->init_mutex);
1632
1633         INIT_LIST_HEAD(&prev.spas);
1634         INIT_LIST_HEAD(&prev.memdevs);
1635         INIT_LIST_HEAD(&prev.dcrs);
1636         INIT_LIST_HEAD(&prev.bdws);
1637         INIT_LIST_HEAD(&prev.idts);
1638         INIT_LIST_HEAD(&prev.flushes);
1639
1640         list_cut_position(&prev.spas, &acpi_desc->spas,
1641                                 acpi_desc->spas.prev);
1642         list_cut_position(&prev.memdevs, &acpi_desc->memdevs,
1643                                 acpi_desc->memdevs.prev);
1644         list_cut_position(&prev.dcrs, &acpi_desc->dcrs,
1645                                 acpi_desc->dcrs.prev);
1646         list_cut_position(&prev.bdws, &acpi_desc->bdws,
1647                                 acpi_desc->bdws.prev);
1648         list_cut_position(&prev.idts, &acpi_desc->idts,
1649                                 acpi_desc->idts.prev);
1650         list_cut_position(&prev.flushes, &acpi_desc->flushes,
1651                                 acpi_desc->flushes.prev);
1652
1653         data = (u8 *) acpi_desc->nfit;
1654         end = data + sz;
1655         data += sizeof(struct acpi_table_nfit);
1656         while (!IS_ERR_OR_NULL(data))
1657                 data = add_table(acpi_desc, &prev, data, end);
1658
1659         if (IS_ERR(data)) {
1660                 dev_dbg(dev, "%s: nfit table parsing error: %ld\n", __func__,
1661                                 PTR_ERR(data));
1662                 rc = PTR_ERR(data);
1663                 goto out_unlock;
1664         }
1665
1666         rc = acpi_nfit_check_deletions(acpi_desc, &prev);
1667         if (rc)
1668                 goto out_unlock;
1669
1670         if (nfit_mem_init(acpi_desc) != 0) {
1671                 rc = -ENOMEM;
1672                 goto out_unlock;
1673         }
1674
1675         acpi_nfit_init_dsms(acpi_desc);
1676
1677         rc = acpi_nfit_register_dimms(acpi_desc);
1678         if (rc)
1679                 goto out_unlock;
1680
1681         rc = acpi_nfit_register_regions(acpi_desc);
1682
1683  out_unlock:
1684         mutex_unlock(&acpi_desc->init_mutex);
1685         return rc;
1686 }
1687 EXPORT_SYMBOL_GPL(acpi_nfit_init);
1688
1689 static struct acpi_nfit_desc *acpi_nfit_desc_init(struct acpi_device *adev)
1690 {
1691         struct nvdimm_bus_descriptor *nd_desc;
1692         struct acpi_nfit_desc *acpi_desc;
1693         struct device *dev = &adev->dev;
1694
1695         acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
1696         if (!acpi_desc)
1697                 return ERR_PTR(-ENOMEM);
1698
1699         dev_set_drvdata(dev, acpi_desc);
1700         acpi_desc->dev = dev;
1701         acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io;
1702         nd_desc = &acpi_desc->nd_desc;
1703         nd_desc->provider_name = "ACPI.NFIT";
1704         nd_desc->ndctl = acpi_nfit_ctl;
1705         nd_desc->attr_groups = acpi_nfit_attribute_groups;
1706
1707         acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, nd_desc);
1708         if (!acpi_desc->nvdimm_bus) {
1709                 devm_kfree(dev, acpi_desc);
1710                 return ERR_PTR(-ENXIO);
1711         }
1712
1713         INIT_LIST_HEAD(&acpi_desc->spa_maps);
1714         INIT_LIST_HEAD(&acpi_desc->spas);
1715         INIT_LIST_HEAD(&acpi_desc->dcrs);
1716         INIT_LIST_HEAD(&acpi_desc->bdws);
1717         INIT_LIST_HEAD(&acpi_desc->idts);
1718         INIT_LIST_HEAD(&acpi_desc->flushes);
1719         INIT_LIST_HEAD(&acpi_desc->memdevs);
1720         INIT_LIST_HEAD(&acpi_desc->dimms);
1721         mutex_init(&acpi_desc->spa_map_mutex);
1722         mutex_init(&acpi_desc->init_mutex);
1723
1724         return acpi_desc;
1725 }
1726
1727 static int acpi_nfit_add(struct acpi_device *adev)
1728 {
1729         struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
1730         struct acpi_nfit_desc *acpi_desc;
1731         struct device *dev = &adev->dev;
1732         struct acpi_table_header *tbl;
1733         acpi_status status = AE_OK;
1734         acpi_size sz;
1735         int rc;
1736
1737         status = acpi_get_table_with_size("NFIT", 0, &tbl, &sz);
1738         if (ACPI_FAILURE(status)) {
1739                 /* This is ok, we could have an nvdimm hotplugged later */
1740                 dev_dbg(dev, "failed to find NFIT at startup\n");
1741                 return 0;
1742         }
1743
1744         acpi_desc = acpi_nfit_desc_init(adev);
1745         if (IS_ERR(acpi_desc)) {
1746                 dev_err(dev, "%s: error initializing acpi_desc: %ld\n",
1747                                 __func__, PTR_ERR(acpi_desc));
1748                 return PTR_ERR(acpi_desc);
1749         }
1750
1751         acpi_desc->nfit = (struct acpi_table_nfit *) tbl;
1752
1753         /* Evaluate _FIT and override with that if present */
1754         status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf);
1755         if (ACPI_SUCCESS(status) && buf.length > 0) {
1756                 acpi_desc->nfit = (struct acpi_table_nfit *)buf.pointer;
1757                 sz = buf.length;
1758         }
1759
1760         rc = acpi_nfit_init(acpi_desc, sz);
1761         if (rc) {
1762                 nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
1763                 return rc;
1764         }
1765         return 0;
1766 }
1767
1768 static int acpi_nfit_remove(struct acpi_device *adev)
1769 {
1770         struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(&adev->dev);
1771
1772         nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
1773         return 0;
1774 }
1775
1776 static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
1777 {
1778         struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(&adev->dev);
1779         struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
1780         struct acpi_table_nfit *nfit_saved;
1781         struct device *dev = &adev->dev;
1782         acpi_status status;
1783         int ret;
1784
1785         dev_dbg(dev, "%s: event: %d\n", __func__, event);
1786
1787         device_lock(dev);
1788         if (!dev->driver) {
1789                 /* dev->driver may be null if we're being removed */
1790                 dev_dbg(dev, "%s: no driver found for dev\n", __func__);
1791                 return;
1792         }
1793
1794         if (!acpi_desc) {
1795                 acpi_desc = acpi_nfit_desc_init(adev);
1796                 if (IS_ERR(acpi_desc)) {
1797                         dev_err(dev, "%s: error initializing acpi_desc: %ld\n",
1798                                 __func__, PTR_ERR(acpi_desc));
1799                         goto out_unlock;
1800                 }
1801         }
1802
1803         /* Evaluate _FIT */
1804         status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf);
1805         if (ACPI_FAILURE(status)) {
1806                 dev_err(dev, "failed to evaluate _FIT\n");
1807                 goto out_unlock;
1808         }
1809
1810         nfit_saved = acpi_desc->nfit;
1811         acpi_desc->nfit = (struct acpi_table_nfit *)buf.pointer;
1812         ret = acpi_nfit_init(acpi_desc, buf.length);
1813         if (!ret) {
1814                 /* Merge failed, restore old nfit, and exit */
1815                 acpi_desc->nfit = nfit_saved;
1816                 dev_err(dev, "failed to merge updated NFIT\n");
1817         }
1818         kfree(buf.pointer);
1819
1820  out_unlock:
1821         device_unlock(dev);
1822 }
1823
1824 static const struct acpi_device_id acpi_nfit_ids[] = {
1825         { "ACPI0012", 0 },
1826         { "", 0 },
1827 };
1828 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids);
1829
1830 static struct acpi_driver acpi_nfit_driver = {
1831         .name = KBUILD_MODNAME,
1832         .ids = acpi_nfit_ids,
1833         .ops = {
1834                 .add = acpi_nfit_add,
1835                 .remove = acpi_nfit_remove,
1836                 .notify = acpi_nfit_notify,
1837         },
1838 };
1839
1840 static __init int nfit_init(void)
1841 {
1842         BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40);
1843         BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56);
1844         BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48);
1845         BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20);
1846         BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9);
1847         BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80);
1848         BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40);
1849
1850         acpi_str_to_uuid(UUID_VOLATILE_MEMORY, nfit_uuid[NFIT_SPA_VOLATILE]);
1851         acpi_str_to_uuid(UUID_PERSISTENT_MEMORY, nfit_uuid[NFIT_SPA_PM]);
1852         acpi_str_to_uuid(UUID_CONTROL_REGION, nfit_uuid[NFIT_SPA_DCR]);
1853         acpi_str_to_uuid(UUID_DATA_REGION, nfit_uuid[NFIT_SPA_BDW]);
1854         acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_VDISK]);
1855         acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_CD, nfit_uuid[NFIT_SPA_VCD]);
1856         acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_PDISK]);
1857         acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_CD, nfit_uuid[NFIT_SPA_PCD]);
1858         acpi_str_to_uuid(UUID_NFIT_BUS, nfit_uuid[NFIT_DEV_BUS]);
1859         acpi_str_to_uuid(UUID_NFIT_DIMM, nfit_uuid[NFIT_DEV_DIMM]);
1860
1861         return acpi_bus_register_driver(&acpi_nfit_driver);
1862 }
1863
1864 static __exit void nfit_exit(void)
1865 {
1866         acpi_bus_unregister_driver(&acpi_nfit_driver);
1867 }
1868
1869 module_init(nfit_init);
1870 module_exit(nfit_exit);
1871 MODULE_LICENSE("GPL v2");
1872 MODULE_AUTHOR("Intel Corporation");