1 // SPDX-License-Identifier: GPL-2.0+
2 // Copyright 2017 IBM Corp.
4 #include <asm/pnv-ocxl.h>
6 #include <misc/ocxl-config.h>
8 #define EXTRACT_BIT(val, bit) (!!(val & BIT(bit)))
9 #define EXTRACT_BITS(val, s, e) ((val & GENMASK(e, s)) >> s)
11 #define OCXL_DVSEC_AFU_IDX_MASK GENMASK(5, 0)
12 #define OCXL_DVSEC_ACTAG_MASK GENMASK(11, 0)
13 #define OCXL_DVSEC_PASID_MASK GENMASK(19, 0)
14 #define OCXL_DVSEC_PASID_LOG_MASK GENMASK(4, 0)
16 #define OCXL_DVSEC_TEMPL_VERSION 0x0
17 #define OCXL_DVSEC_TEMPL_NAME 0x4
18 #define OCXL_DVSEC_TEMPL_AFU_VERSION 0x1C
19 #define OCXL_DVSEC_TEMPL_MMIO_GLOBAL 0x20
20 #define OCXL_DVSEC_TEMPL_MMIO_GLOBAL_SZ 0x28
21 #define OCXL_DVSEC_TEMPL_MMIO_PP 0x30
22 #define OCXL_DVSEC_TEMPL_MMIO_PP_SZ 0x38
23 #define OCXL_DVSEC_TEMPL_MEM_SZ 0x3C
24 #define OCXL_DVSEC_TEMPL_WWID 0x40
26 #define OCXL_MAX_AFU_PER_FUNCTION 64
27 #define OCXL_TEMPL_LEN 0x58
28 #define OCXL_TEMPL_NAME_LEN 24
29 #define OCXL_CFG_TIMEOUT 3
31 static int find_dvsec(struct pci_dev *dev, int dvsec_id)
36 while ((vsec = pci_find_next_ext_capability(dev, vsec,
37 OCXL_EXT_CAP_ID_DVSEC))) {
38 pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET,
40 pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id);
41 if (vendor == PCI_VENDOR_ID_IBM && id == dvsec_id)
47 static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx)
53 while ((vsec = pci_find_next_ext_capability(dev, vsec,
54 OCXL_EXT_CAP_ID_DVSEC))) {
55 pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET,
57 pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id);
59 if (vendor == PCI_VENDOR_ID_IBM &&
60 id == OCXL_DVSEC_AFU_CTRL_ID) {
61 pci_read_config_byte(dev,
62 vsec + OCXL_DVSEC_AFU_CTRL_AFU_IDX,
71 static int read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn)
76 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PASID);
79 * PASID capability is not mandatory, but there
80 * shouldn't be any AFU
82 dev_dbg(&dev->dev, "Function doesn't require any PASID\n");
83 fn->max_pasid_log = -1;
86 pci_read_config_word(dev, pos + PCI_PASID_CAP, &val);
87 fn->max_pasid_log = EXTRACT_BITS(val, 8, 12);
90 dev_dbg(&dev->dev, "PASID capability:\n");
91 dev_dbg(&dev->dev, " Max PASID log = %d\n", fn->max_pasid_log);
95 static int read_dvsec_tl(struct pci_dev *dev, struct ocxl_fn_config *fn)
99 pos = find_dvsec(dev, OCXL_DVSEC_TL_ID);
100 if (!pos && PCI_FUNC(dev->devfn) == 0) {
101 dev_err(&dev->dev, "Can't find TL DVSEC\n");
104 if (pos && PCI_FUNC(dev->devfn) != 0) {
105 dev_err(&dev->dev, "TL DVSEC is only allowed on function 0\n");
108 fn->dvsec_tl_pos = pos;
112 static int read_dvsec_function(struct pci_dev *dev, struct ocxl_fn_config *fn)
114 int pos, afu_present;
117 pos = find_dvsec(dev, OCXL_DVSEC_FUNC_ID);
119 dev_err(&dev->dev, "Can't find function DVSEC\n");
122 fn->dvsec_function_pos = pos;
124 pci_read_config_dword(dev, pos + OCXL_DVSEC_FUNC_OFF_INDEX, &val);
125 afu_present = EXTRACT_BIT(val, 31);
127 fn->max_afu_index = -1;
128 dev_dbg(&dev->dev, "Function doesn't define any AFU\n");
131 fn->max_afu_index = EXTRACT_BITS(val, 24, 29);
134 dev_dbg(&dev->dev, "Function DVSEC:\n");
135 dev_dbg(&dev->dev, " Max AFU index = %d\n", fn->max_afu_index);
139 static int read_dvsec_afu_info(struct pci_dev *dev, struct ocxl_fn_config *fn)
143 if (fn->max_afu_index < 0) {
144 fn->dvsec_afu_info_pos = -1;
148 pos = find_dvsec(dev, OCXL_DVSEC_AFU_INFO_ID);
150 dev_err(&dev->dev, "Can't find AFU information DVSEC\n");
153 fn->dvsec_afu_info_pos = pos;
157 static int read_dvsec_vendor(struct pci_dev *dev)
163 * vendor specific DVSEC is optional
165 * It's currently only used on function 0 to specify the
166 * version of some logic blocks. Some older images may not
167 * even have it so we ignore any errors
169 if (PCI_FUNC(dev->devfn) != 0)
172 pos = find_dvsec(dev, OCXL_DVSEC_VENDOR_ID);
176 pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_CFG_VERS, &cfg);
177 pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_TLX_VERS, &tlx);
178 pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_DLX_VERS, &dlx);
180 dev_dbg(&dev->dev, "Vendor specific DVSEC:\n");
181 dev_dbg(&dev->dev, " CFG version = 0x%x\n", cfg);
182 dev_dbg(&dev->dev, " TLX version = 0x%x\n", tlx);
183 dev_dbg(&dev->dev, " DLX version = 0x%x\n", dlx);
187 static int validate_function(struct pci_dev *dev, struct ocxl_fn_config *fn)
189 if (fn->max_pasid_log == -1 && fn->max_afu_index >= 0) {
191 "AFUs are defined but no PASIDs are requested\n");
195 if (fn->max_afu_index > OCXL_MAX_AFU_PER_FUNCTION) {
197 "Max AFU index out of architectural limit (%d vs %d)\n",
198 fn->max_afu_index, OCXL_MAX_AFU_PER_FUNCTION);
204 int ocxl_config_read_function(struct pci_dev *dev, struct ocxl_fn_config *fn)
208 rc = read_pasid(dev, fn);
210 dev_err(&dev->dev, "Invalid PASID configuration: %d\n", rc);
214 rc = read_dvsec_tl(dev, fn);
217 "Invalid Transaction Layer DVSEC configuration: %d\n",
222 rc = read_dvsec_function(dev, fn);
225 "Invalid Function DVSEC configuration: %d\n", rc);
229 rc = read_dvsec_afu_info(dev, fn);
231 dev_err(&dev->dev, "Invalid AFU configuration: %d\n", rc);
235 rc = read_dvsec_vendor(dev);
238 "Invalid vendor specific DVSEC configuration: %d\n",
243 rc = validate_function(dev, fn);
246 EXPORT_SYMBOL_GPL(ocxl_config_read_function);
248 static int read_afu_info(struct pci_dev *dev, struct ocxl_fn_config *fn,
249 int offset, u32 *data)
252 unsigned long timeout = jiffies + (HZ * OCXL_CFG_TIMEOUT);
253 int pos = fn->dvsec_afu_info_pos;
255 /* Protect 'data valid' bit */
256 if (EXTRACT_BIT(offset, 31)) {
257 dev_err(&dev->dev, "Invalid offset in AFU info DVSEC\n");
261 pci_write_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_OFF, offset);
262 pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_OFF, &val);
263 while (!EXTRACT_BIT(val, 31)) {
264 if (time_after_eq(jiffies, timeout)) {
266 "Timeout while reading AFU info DVSEC (offset=%d)\n",
271 pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_OFF, &val);
273 pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_DATA, data);
277 int ocxl_config_check_afu_index(struct pci_dev *dev,
278 struct ocxl_fn_config *fn, int afu_idx)
281 int rc, templ_major, templ_minor, len;
283 pci_write_config_byte(dev,
284 fn->dvsec_afu_info_pos + OCXL_DVSEC_AFU_INFO_AFU_IDX,
286 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_VERSION, &val);
290 /* AFU index map can have holes */
294 templ_major = EXTRACT_BITS(val, 8, 15);
295 templ_minor = EXTRACT_BITS(val, 0, 7);
296 dev_dbg(&dev->dev, "AFU descriptor template version %d.%d\n",
297 templ_major, templ_minor);
299 len = EXTRACT_BITS(val, 16, 31);
300 if (len != OCXL_TEMPL_LEN) {
302 "Unexpected template length in AFU information (%#x)\n",
307 EXPORT_SYMBOL_GPL(ocxl_config_check_afu_index);
309 static int read_afu_name(struct pci_dev *dev, struct ocxl_fn_config *fn,
310 struct ocxl_afu_config *afu)
315 BUILD_BUG_ON(OCXL_AFU_NAME_SZ < OCXL_TEMPL_NAME_LEN);
316 for (i = 0; i < OCXL_TEMPL_NAME_LEN; i += 4) {
317 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_NAME + i, &val);
320 ptr = (u32 *) &afu->name[i];
321 *ptr = le32_to_cpu((__force __le32) val);
323 afu->name[OCXL_AFU_NAME_SZ - 1] = '\0'; /* play safe */
327 static int read_afu_mmio(struct pci_dev *dev, struct ocxl_fn_config *fn,
328 struct ocxl_afu_config *afu)
336 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL, &val);
339 afu->global_mmio_bar = EXTRACT_BITS(val, 0, 2);
340 afu->global_mmio_offset = EXTRACT_BITS(val, 16, 31) << 16;
342 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL + 4, &val);
345 afu->global_mmio_offset += (u64) val << 32;
347 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL_SZ, &val);
350 afu->global_mmio_size = val;
355 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP, &val);
358 afu->pp_mmio_bar = EXTRACT_BITS(val, 0, 2);
359 afu->pp_mmio_offset = EXTRACT_BITS(val, 16, 31) << 16;
361 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP + 4, &val);
364 afu->pp_mmio_offset += (u64) val << 32;
366 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP_SZ, &val);
369 afu->pp_mmio_stride = val;
374 static int read_afu_control(struct pci_dev *dev, struct ocxl_afu_config *afu)
380 pos = find_dvsec_afu_ctrl(dev, afu->idx);
382 dev_err(&dev->dev, "Can't find AFU control DVSEC for AFU %d\n",
386 afu->dvsec_afu_control_pos = pos;
388 pci_read_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_SUP, &val8);
389 afu->pasid_supported_log = EXTRACT_BITS(val8, 0, 4);
391 pci_read_config_word(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_SUP, &val16);
392 afu->actag_supported = EXTRACT_BITS(val16, 0, 11);
396 static bool char_allowed(int c)
399 * Permitted Characters : Alphanumeric, hyphen, underscore, comma
401 if ((c >= 0x30 && c <= 0x39) /* digits */ ||
402 (c >= 0x41 && c <= 0x5A) /* upper case */ ||
403 (c >= 0x61 && c <= 0x7A) /* lower case */ ||
412 static int validate_afu(struct pci_dev *dev, struct ocxl_afu_config *afu)
417 dev_err(&dev->dev, "Empty AFU name\n");
420 for (i = 0; i < OCXL_TEMPL_NAME_LEN; i++) {
421 if (!char_allowed(afu->name[i])) {
423 "Invalid character in AFU name\n");
428 if (afu->global_mmio_bar != 0 &&
429 afu->global_mmio_bar != 2 &&
430 afu->global_mmio_bar != 4) {
431 dev_err(&dev->dev, "Invalid global MMIO bar number\n");
434 if (afu->pp_mmio_bar != 0 &&
435 afu->pp_mmio_bar != 2 &&
436 afu->pp_mmio_bar != 4) {
437 dev_err(&dev->dev, "Invalid per-process MMIO bar number\n");
443 int ocxl_config_read_afu(struct pci_dev *dev, struct ocxl_fn_config *fn,
444 struct ocxl_afu_config *afu, u8 afu_idx)
450 * First, we need to write the AFU idx for the AFU we want to
453 WARN_ON((afu_idx & OCXL_DVSEC_AFU_IDX_MASK) != afu_idx);
455 pci_write_config_byte(dev,
456 fn->dvsec_afu_info_pos + OCXL_DVSEC_AFU_INFO_AFU_IDX,
459 rc = read_afu_name(dev, fn, afu);
463 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_AFU_VERSION, &val32);
466 afu->version_major = EXTRACT_BITS(val32, 24, 31);
467 afu->version_minor = EXTRACT_BITS(val32, 16, 23);
468 afu->afuc_type = EXTRACT_BITS(val32, 14, 15);
469 afu->afum_type = EXTRACT_BITS(val32, 12, 13);
470 afu->profile = EXTRACT_BITS(val32, 0, 7);
472 rc = read_afu_mmio(dev, fn, afu);
476 rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MEM_SZ, &val32);
479 afu->log_mem_size = EXTRACT_BITS(val32, 0, 7);
481 rc = read_afu_control(dev, afu);
485 dev_dbg(&dev->dev, "AFU configuration:\n");
486 dev_dbg(&dev->dev, " name = %s\n", afu->name);
487 dev_dbg(&dev->dev, " version = %d.%d\n", afu->version_major,
489 dev_dbg(&dev->dev, " global mmio bar = %hhu\n", afu->global_mmio_bar);
490 dev_dbg(&dev->dev, " global mmio offset = %#llx\n",
491 afu->global_mmio_offset);
492 dev_dbg(&dev->dev, " global mmio size = %#x\n", afu->global_mmio_size);
493 dev_dbg(&dev->dev, " pp mmio bar = %hhu\n", afu->pp_mmio_bar);
494 dev_dbg(&dev->dev, " pp mmio offset = %#llx\n", afu->pp_mmio_offset);
495 dev_dbg(&dev->dev, " pp mmio stride = %#x\n", afu->pp_mmio_stride);
496 dev_dbg(&dev->dev, " mem size (log) = %hhu\n", afu->log_mem_size);
497 dev_dbg(&dev->dev, " pasid supported (log) = %u\n",
498 afu->pasid_supported_log);
499 dev_dbg(&dev->dev, " actag supported = %u\n",
500 afu->actag_supported);
502 rc = validate_afu(dev, afu);
505 EXPORT_SYMBOL_GPL(ocxl_config_read_afu);
507 int ocxl_config_get_actag_info(struct pci_dev *dev, u16 *base, u16 *enabled,
513 * This is really a simple wrapper for the kernel API, to
514 * avoid an external driver using ocxl as a library to call
515 * platform-dependent code
517 rc = pnv_ocxl_get_actag(dev, base, enabled, supported);
519 dev_err(&dev->dev, "Can't get actag for device: %d\n", rc);
524 EXPORT_SYMBOL_GPL(ocxl_config_get_actag_info);
526 void ocxl_config_set_afu_actag(struct pci_dev *dev, int pos, int actag_base,
531 val = actag_count & OCXL_DVSEC_ACTAG_MASK;
532 pci_write_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_EN, val);
534 val = actag_base & OCXL_DVSEC_ACTAG_MASK;
535 pci_write_config_dword(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_BASE, val);
537 EXPORT_SYMBOL_GPL(ocxl_config_set_afu_actag);
539 int ocxl_config_get_pasid_info(struct pci_dev *dev, int *count)
541 return pnv_ocxl_get_pasid_count(dev, count);
543 EXPORT_SYMBOL_GPL(ocxl_config_get_pasid_info);
545 void ocxl_config_set_afu_pasid(struct pci_dev *dev, int pos, int pasid_base,
551 val8 = pasid_count_log & OCXL_DVSEC_PASID_LOG_MASK;
552 pci_write_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_EN, val8);
554 pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_BASE,
556 val32 &= ~OCXL_DVSEC_PASID_MASK;
557 val32 |= pasid_base & OCXL_DVSEC_PASID_MASK;
558 pci_write_config_dword(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_BASE,
561 EXPORT_SYMBOL_GPL(ocxl_config_set_afu_pasid);
563 void ocxl_config_set_afu_state(struct pci_dev *dev, int pos, int enable)
567 pci_read_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_ENABLE, &val);
572 pci_write_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_ENABLE, val);
574 EXPORT_SYMBOL_GPL(ocxl_config_set_afu_state);
576 int ocxl_config_set_TL(struct pci_dev *dev, int tl_dvsec)
586 * Skip on function != 0, as the TL can only be defined on 0
588 if (PCI_FUNC(dev->devfn) != 0)
591 recv_rate = kzalloc(PNV_OCXL_TL_RATE_BUF_SIZE, GFP_KERNEL);
595 * The spec defines 64 templates for messages in the
596 * Transaction Layer (TL).
598 * The host and device each support a subset, so we need to
599 * configure the transmitters on each side to send only
600 * templates the receiver understands, at a rate the receiver
601 * can process. Per the spec, template 0 must be supported by
602 * everybody. That's the template which has been used by the
603 * host and device so far.
605 * The sending rate limit must be set before the template is
612 rc = pnv_ocxl_get_tl_cap(dev, &recv_cap, recv_rate,
613 PNV_OCXL_TL_RATE_BUF_SIZE);
617 for (i = 0; i < PNV_OCXL_TL_RATE_BUF_SIZE; i += 4) {
618 be32ptr = (__be32 *) &recv_rate[i];
619 pci_write_config_dword(dev,
620 tl_dvsec + OCXL_DVSEC_TL_SEND_RATE + i,
621 be32_to_cpu(*be32ptr));
623 val = recv_cap >> 32;
624 pci_write_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_SEND_CAP, val);
625 val = recv_cap & GENMASK(31, 0);
626 pci_write_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_SEND_CAP + 4, val);
631 for (i = 0; i < PNV_OCXL_TL_RATE_BUF_SIZE; i += 4) {
632 pci_read_config_dword(dev,
633 tl_dvsec + OCXL_DVSEC_TL_RECV_RATE + i,
635 be32ptr = (__be32 *) &recv_rate[i];
636 *be32ptr = cpu_to_be32(val);
638 pci_read_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_RECV_CAP, &val);
639 recv_cap = (long) val << 32;
640 pci_read_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_RECV_CAP + 4, &val);
643 rc = pnv_ocxl_set_tl_conf(dev, recv_cap, __pa(recv_rate),
644 PNV_OCXL_TL_RATE_BUF_SIZE);
649 * Opencapi commands needing to be retried are classified per
650 * the TL in 2 groups: short and long commands.
652 * The short back off timer it not used for now. It will be
655 * The long back off timer is typically used when an AFU hits
656 * a page fault but the NPU is already processing one. So the
657 * AFU needs to wait before it can resubmit. Having a value
658 * too low doesn't break anything, but can generate extra
659 * traffic on the link.
660 * We set it to 1.6 us for now. It's shorter than, but in the
661 * same order of magnitude as the time spent to process a page
664 timers = 0x2 << 4; /* long timer = 1.6 us */
665 pci_write_config_byte(dev, tl_dvsec + OCXL_DVSEC_TL_BACKOFF_TIMERS,
673 EXPORT_SYMBOL_GPL(ocxl_config_set_TL);
675 int ocxl_config_terminate_pasid(struct pci_dev *dev, int afu_control, int pasid)
678 unsigned long timeout;
680 pci_read_config_dword(dev, afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
682 if (EXTRACT_BIT(val, 20)) {
684 "Can't terminate PASID %#x, previous termination didn't complete\n",
689 val &= ~OCXL_DVSEC_PASID_MASK;
690 val |= pasid & OCXL_DVSEC_PASID_MASK;
692 pci_write_config_dword(dev,
693 afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
696 timeout = jiffies + (HZ * OCXL_CFG_TIMEOUT);
697 pci_read_config_dword(dev, afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
699 while (EXTRACT_BIT(val, 20)) {
700 if (time_after_eq(jiffies, timeout)) {
702 "Timeout while waiting for AFU to terminate PASID %#x\n",
707 pci_read_config_dword(dev,
708 afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
713 EXPORT_SYMBOL_GPL(ocxl_config_terminate_pasid);
715 void ocxl_config_set_actag(struct pci_dev *dev, int func_dvsec, u32 tag_first,
720 val = (tag_first & OCXL_DVSEC_ACTAG_MASK) << 16;
721 val |= tag_count & OCXL_DVSEC_ACTAG_MASK;
722 pci_write_config_dword(dev, func_dvsec + OCXL_DVSEC_FUNC_OFF_ACTAG,
725 EXPORT_SYMBOL_GPL(ocxl_config_set_actag);