1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright 2016-17 IBM Corp.
6 #define pr_fmt(fmt) "vas: " fmt
8 #include <linux/types.h>
9 #include <linux/mutex.h>
10 #include <linux/slab.h>
12 #include <linux/log2.h>
13 #include <linux/rcupdate.h>
14 #include <linux/cred.h>
15 #include <linux/sched/mm.h>
16 #include <linux/mmu_context.h>
17 #include <asm/switch_to.h>
18 #include <asm/ppc-opcode.h>
20 #include "copy-paste.h"
22 #define CREATE_TRACE_POINTS
23 #include "vas-trace.h"
26 * Compute the paste address region for the window @window using the
27 * ->paste_base_addr and ->paste_win_id_shift we got from device tree.
29 void vas_win_paste_addr(struct vas_window *window, u64 *addr, int *len)
34 base = window->vinst->paste_base_addr;
35 shift = window->vinst->paste_win_id_shift;
36 winid = window->winid;
38 *addr = base + (winid << shift);
42 pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr);
45 static inline void get_hvwc_mmio_bar(struct vas_window *window,
50 pbaddr = window->vinst->hvwc_bar_start;
51 *start = pbaddr + window->winid * VAS_HVWC_SIZE;
55 static inline void get_uwc_mmio_bar(struct vas_window *window,
60 pbaddr = window->vinst->uwc_bar_start;
61 *start = pbaddr + window->winid * VAS_UWC_SIZE;
66 * Map the paste bus address of the given send window into kernel address
67 * space. Unlike MMIO regions (map_mmio_region() below), paste region must
68 * be mapped cache-able and is only applicable to send windows.
70 static void *map_paste_region(struct vas_window *txwin)
77 name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id,
82 txwin->paste_addr_name = name;
83 vas_win_paste_addr(txwin, &start, &len);
85 if (!request_mem_region(start, len, name)) {
86 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
87 __func__, start, len);
91 map = ioremap_cache(start, len);
93 pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__,
98 pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map);
103 return ERR_PTR(-ENOMEM);
106 static void *map_mmio_region(char *name, u64 start, int len)
110 if (!request_mem_region(start, len, name)) {
111 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
112 __func__, start, len);
116 map = ioremap(start, len);
118 pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start,
126 static void unmap_region(void *addr, u64 start, int len)
129 release_mem_region((phys_addr_t)start, len);
133 * Unmap the paste address region for a window.
135 static void unmap_paste_region(struct vas_window *window)
140 if (window->paste_kaddr) {
141 vas_win_paste_addr(window, &busaddr_start, &len);
142 unmap_region(window->paste_kaddr, busaddr_start, len);
143 window->paste_kaddr = NULL;
144 kfree(window->paste_addr_name);
145 window->paste_addr_name = NULL;
150 * Unmap the MMIO regions for a window. Hold the vas_mutex so we don't
151 * unmap when the window's debugfs dir is in use. This serializes close
152 * of a window even on another VAS instance but since its not a critical
153 * path, just minimize the time we hold the mutex for now. We can add
154 * a per-instance mutex later if necessary.
156 static void unmap_winctx_mmio_bars(struct vas_window *window)
163 mutex_lock(&vas_mutex);
165 hvwc_map = window->hvwc_map;
166 window->hvwc_map = NULL;
168 uwc_map = window->uwc_map;
169 window->uwc_map = NULL;
171 mutex_unlock(&vas_mutex);
174 get_hvwc_mmio_bar(window, &busaddr_start, &len);
175 unmap_region(hvwc_map, busaddr_start, len);
179 get_uwc_mmio_bar(window, &busaddr_start, &len);
180 unmap_region(uwc_map, busaddr_start, len);
185 * Find the Hypervisor Window Context (HVWC) MMIO Base Address Region and the
186 * OS/User Window Context (UWC) MMIO Base Address Region for the given window.
187 * Map these bus addresses and save the mapped kernel addresses in @window.
189 int map_winctx_mmio_bars(struct vas_window *window)
194 get_hvwc_mmio_bar(window, &start, &len);
195 window->hvwc_map = map_mmio_region("HVWCM_Window", start, len);
197 get_uwc_mmio_bar(window, &start, &len);
198 window->uwc_map = map_mmio_region("UWCM_Window", start, len);
200 if (!window->hvwc_map || !window->uwc_map) {
201 unmap_winctx_mmio_bars(window);
209 * Reset all valid registers in the HV and OS/User Window Contexts for
210 * the window identified by @window.
212 * NOTE: We cannot really use a for loop to reset window context. Not all
213 * offsets in a window context are valid registers and the valid
214 * registers are not sequential. And, we can only write to offsets
215 * with valid registers.
217 void reset_window_regs(struct vas_window *window)
219 write_hvwc_reg(window, VREG(LPID), 0ULL);
220 write_hvwc_reg(window, VREG(PID), 0ULL);
221 write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL);
222 write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL);
223 write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL);
224 write_hvwc_reg(window, VREG(AMR), 0ULL);
225 write_hvwc_reg(window, VREG(SEIDR), 0ULL);
226 write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL);
227 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
228 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL);
229 write_hvwc_reg(window, VREG(PSWID), 0ULL);
230 write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL);
231 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL);
232 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL);
233 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
234 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
235 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
236 write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL);
237 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
238 write_hvwc_reg(window, VREG(TX_WCRED), 0ULL);
239 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
240 write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL);
241 write_hvwc_reg(window, VREG(WINCTL), 0ULL);
242 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
243 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL);
244 write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL);
245 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL);
246 write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL);
247 write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL);
248 write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL);
249 write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL);
250 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL);
251 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
253 /* Skip read-only registers: NX_UTIL and NX_UTIL_SE */
256 * The send and receive window credit adder registers are also
257 * accessible from HVWC and have been initialized above. We don't
258 * need to initialize from the OS/User Window Context, so skip
261 * write_uwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
262 * write_uwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
267 * Initialize window context registers related to Address Translation.
268 * These registers are common to send/receive windows although they
269 * differ for user/kernel windows. As we resolve the TODOs we may
270 * want to add fields to vas_winctx and move the initialization to
271 * init_vas_winctx_regs().
273 static void init_xlate_regs(struct vas_window *window, bool user_win)
278 * MSR_TA, MSR_US are false for both kernel and user.
279 * MSR_DR and MSR_PR are false for kernel.
282 val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1);
283 val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1);
285 val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1);
286 val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1);
288 write_hvwc_reg(window, VREG(XLATE_MSR), val);
290 lpcr = mfspr(SPRN_LPCR);
293 * NOTE: From Section 5.7.8.1 Segment Lookaside Buffer of the
294 * Power ISA, v3.0B, Page size encoding is 0 = 4KB, 5 = 64KB.
296 * NOTE: From Section 1.3.1, Address Translation Context of the
297 * Nest MMU Workbook, LPCR_SC should be 0 for Power9.
299 val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5);
300 val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL);
301 val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC);
302 val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0);
303 write_hvwc_reg(window, VREG(XLATE_LPCR), val);
306 * Section 1.3.1 (Address translation Context) of NMMU workbook.
307 * 0b00 Hashed Page Table mode
310 * 0b11 Radix on Radix
313 val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2);
314 write_hvwc_reg(window, VREG(XLATE_CTL), val);
317 * TODO: Can we mfspr(AMR) even for user windows?
320 val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR));
321 write_hvwc_reg(window, VREG(AMR), val);
324 val = SET_FIELD(VAS_SEIDR, val, 0);
325 write_hvwc_reg(window, VREG(SEIDR), val);
329 * Initialize Reserved Send Buffer Count for the send window. It involves
330 * writing to the register, reading it back to confirm that the hardware
331 * has enough buffers to reserve. See section 1.3.1.2.1 of VAS workbook.
333 * Since we can only make a best-effort attempt to fulfill the request,
334 * we don't return any errors if we cannot.
336 * TODO: Reserved (aka dedicated) send buffers are not supported yet.
338 static void init_rsvd_tx_buf_count(struct vas_window *txwin,
339 struct vas_winctx *winctx)
341 write_hvwc_reg(txwin, VREG(TX_RSVD_BUF_COUNT), 0ULL);
346 * Initialize window context registers for a receive window.
347 * Except for caching control and marking window open, the registers
348 * are initialized in the order listed in Section 3.1.4 (Window Context
349 * Cache Register Details) of the VAS workbook although they don't need
352 * Design note: For NX receive windows, NX allocates the FIFO buffer in OPAL
353 * (so that it can get a large contiguous area) and passes that buffer
354 * to kernel via device tree. We now write that buffer address to the
355 * FIFO BAR. Would it make sense to do this all in OPAL? i.e have OPAL
356 * write the per-chip RX FIFO addresses to the windows during boot-up
357 * as a one-time task? That could work for NX but what about other
358 * receivers? Let the receivers tell us the rx-fifo buffers for now.
360 int init_winctx_regs(struct vas_window *window, struct vas_winctx *winctx)
365 reset_window_regs(window);
368 val = SET_FIELD(VAS_LPID, val, winctx->lpid);
369 write_hvwc_reg(window, VREG(LPID), val);
372 val = SET_FIELD(VAS_PID_ID, val, winctx->pidr);
373 write_hvwc_reg(window, VREG(PID), val);
375 init_xlate_regs(window, winctx->user_win);
378 val = SET_FIELD(VAS_FAULT_TX_WIN, val, winctx->fault_win_id);
379 write_hvwc_reg(window, VREG(FAULT_TX_WIN), val);
381 /* In PowerNV, interrupts go to HV. */
382 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
385 val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port);
386 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val);
389 val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid);
390 write_hvwc_reg(window, VREG(PSWID), val);
392 write_hvwc_reg(window, VREG(SPARE1), 0ULL);
393 write_hvwc_reg(window, VREG(SPARE2), 0ULL);
394 write_hvwc_reg(window, VREG(SPARE3), 0ULL);
397 * NOTE: VAS expects the FIFO address to be copied into the LFIFO_BAR
398 * register as is - do NOT shift the address into VAS_LFIFO_BAR
399 * bit fields! Ok to set the page migration select fields -
400 * VAS ignores the lower 10+ bits in the address anyway, because
401 * the minimum FIFO size is 1K?
403 * See also: Design note in function header.
405 val = __pa(winctx->rx_fifo);
406 val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0);
407 write_hvwc_reg(window, VREG(LFIFO_BAR), val);
410 val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp);
411 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val);
414 val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type);
415 val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable);
416 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val);
418 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
419 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
420 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
423 val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max);
424 write_hvwc_reg(window, VREG(LRX_WCRED), val);
427 val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max);
428 write_hvwc_reg(window, VREG(TX_WCRED), val);
430 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
431 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
433 fifo_size = winctx->rx_fifo_size / 1024;
436 val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size));
437 write_hvwc_reg(window, VREG(LFIFO_SIZE), val);
439 /* Update window control and caching control registers last so
440 * we mark the window open only after fully initializing it and
441 * pushing context to cache.
444 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
446 init_rsvd_tx_buf_count(window, winctx);
448 /* for a send window, point to the matching receive window */
450 val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id);
451 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val);
453 write_hvwc_reg(window, VREG(SPARE4), 0ULL);
456 val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable);
457 val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable);
458 val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early);
459 val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg);
460 write_hvwc_reg(window, VREG(LNOTIFY_CTL), val);
463 val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid);
464 write_hvwc_reg(window, VREG(LNOTIFY_PID), val);
467 val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid);
468 write_hvwc_reg(window, VREG(LNOTIFY_LPID), val);
471 val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid);
472 write_hvwc_reg(window, VREG(LNOTIFY_TID), val);
475 val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope);
476 val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope);
477 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val);
479 /* Skip read-only registers NX_UTIL and NX_UTIL_SE */
481 write_hvwc_reg(window, VREG(SPARE5), 0ULL);
482 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
483 write_hvwc_reg(window, VREG(SPARE6), 0ULL);
485 /* Finally, push window context to memory and... */
487 val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1);
488 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val);
490 /* ... mark the window open for business */
492 val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit);
493 val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win);
494 val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode);
495 val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode);
496 val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode);
497 val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode);
498 val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win);
499 val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win);
500 val = SET_FIELD(VAS_WINCTL_OPEN, val, 1);
501 write_hvwc_reg(window, VREG(WINCTL), val);
506 static void vas_release_window_id(struct ida *ida, int winid)
508 ida_free(ida, winid);
511 static int vas_assign_window_id(struct ida *ida)
513 int winid = ida_alloc_max(ida, VAS_WINDOWS_PER_CHIP - 1, GFP_KERNEL);
515 if (winid == -ENOSPC) {
516 pr_err("Too many (%d) open windows\n", VAS_WINDOWS_PER_CHIP);
523 static void vas_window_free(struct vas_window *window)
525 int winid = window->winid;
526 struct vas_instance *vinst = window->vinst;
528 unmap_winctx_mmio_bars(window);
530 vas_window_free_dbgdir(window);
534 vas_release_window_id(&vinst->ida, winid);
537 static struct vas_window *vas_window_alloc(struct vas_instance *vinst)
540 struct vas_window *window;
542 winid = vas_assign_window_id(&vinst->ida);
544 return ERR_PTR(winid);
546 window = kzalloc(sizeof(*window), GFP_KERNEL);
550 window->vinst = vinst;
551 window->winid = winid;
553 if (map_winctx_mmio_bars(window))
556 vas_window_init_dbgdir(window);
562 vas_release_window_id(&vinst->ida, winid);
563 return ERR_PTR(-ENOMEM);
566 static void put_rx_win(struct vas_window *rxwin)
568 /* Better not be a send window! */
569 WARN_ON_ONCE(rxwin->tx_win);
571 atomic_dec(&rxwin->num_txwins);
575 * Find the user space receive window given the @pswid.
576 * - We must have a valid vasid and it must belong to this instance.
577 * (so both send and receive windows are on the same VAS instance)
578 * - The window must refer to an OPEN, FTW, RECEIVE window.
580 * NOTE: We access ->windows[] table and assume that vinst->mutex is held.
582 static struct vas_window *get_user_rxwin(struct vas_instance *vinst, u32 pswid)
585 struct vas_window *rxwin;
587 decode_pswid(pswid, &vasid, &winid);
589 if (vinst->vas_id != vasid)
590 return ERR_PTR(-EINVAL);
592 rxwin = vinst->windows[winid];
594 if (!rxwin || rxwin->tx_win || rxwin->cop != VAS_COP_TYPE_FTW)
595 return ERR_PTR(-EINVAL);
601 * Get the VAS receive window associated with NX engine identified
602 * by @cop and if applicable, @pswid.
604 * See also function header of set_vinst_win().
606 static struct vas_window *get_vinst_rxwin(struct vas_instance *vinst,
607 enum vas_cop_type cop, u32 pswid)
609 struct vas_window *rxwin;
611 mutex_lock(&vinst->mutex);
613 if (cop == VAS_COP_TYPE_FTW)
614 rxwin = get_user_rxwin(vinst, pswid);
616 rxwin = vinst->rxwin[cop] ?: ERR_PTR(-EINVAL);
619 atomic_inc(&rxwin->num_txwins);
621 mutex_unlock(&vinst->mutex);
627 * We have two tables of windows in a VAS instance. The first one,
628 * ->windows[], contains all the windows in the instance and allows
629 * looking up a window by its id. It is used to look up send windows
630 * during fault handling and receive windows when pairing user space
631 * send/receive windows.
633 * The second table, ->rxwin[], contains receive windows that are
634 * associated with NX engines. This table has VAS_COP_TYPE_MAX
635 * entries and is used to look up a receive window by its
638 * Here, we save @window in the ->windows[] table. If it is a receive
639 * window, we also save the window in the ->rxwin[] table.
641 static void set_vinst_win(struct vas_instance *vinst,
642 struct vas_window *window)
644 int id = window->winid;
646 mutex_lock(&vinst->mutex);
649 * There should only be one receive window for a coprocessor type
650 * unless its a user (FTW) window.
652 if (!window->user_win && !window->tx_win) {
653 WARN_ON_ONCE(vinst->rxwin[window->cop]);
654 vinst->rxwin[window->cop] = window;
657 WARN_ON_ONCE(vinst->windows[id] != NULL);
658 vinst->windows[id] = window;
660 mutex_unlock(&vinst->mutex);
664 * Clear this window from the table(s) of windows for this VAS instance.
665 * See also function header of set_vinst_win().
667 static void clear_vinst_win(struct vas_window *window)
669 int id = window->winid;
670 struct vas_instance *vinst = window->vinst;
672 mutex_lock(&vinst->mutex);
674 if (!window->user_win && !window->tx_win) {
675 WARN_ON_ONCE(!vinst->rxwin[window->cop]);
676 vinst->rxwin[window->cop] = NULL;
679 WARN_ON_ONCE(vinst->windows[id] != window);
680 vinst->windows[id] = NULL;
682 mutex_unlock(&vinst->mutex);
685 static void init_winctx_for_rxwin(struct vas_window *rxwin,
686 struct vas_rx_win_attr *rxattr,
687 struct vas_winctx *winctx)
690 * We first zero (memset()) all fields and only set non-zero fields.
691 * Following fields are 0/false but maybe deserve a comment:
693 * ->notify_os_intr_reg In powerNV, send intrs to HV
694 * ->notify_disable False for NX windows
695 * ->intr_disable False for Fault Windows
696 * ->xtra_write False for NX windows
697 * ->notify_early NA for NX windows
698 * ->rsvd_txbuf_count NA for Rx windows
699 * ->lpid, ->pid, ->tid NA for Rx windows
702 memset(winctx, 0, sizeof(struct vas_winctx));
704 winctx->rx_fifo = rxattr->rx_fifo;
705 winctx->rx_fifo_size = rxattr->rx_fifo_size;
706 winctx->wcreds_max = rxwin->wcreds_max;
707 winctx->pin_win = rxattr->pin_win;
709 winctx->nx_win = rxattr->nx_win;
710 winctx->fault_win = rxattr->fault_win;
711 winctx->user_win = rxattr->user_win;
712 winctx->rej_no_credit = rxattr->rej_no_credit;
713 winctx->rx_word_mode = rxattr->rx_win_ord_mode;
714 winctx->tx_word_mode = rxattr->tx_win_ord_mode;
715 winctx->rx_wcred_mode = rxattr->rx_wcred_mode;
716 winctx->tx_wcred_mode = rxattr->tx_wcred_mode;
717 winctx->notify_early = rxattr->notify_early;
719 if (winctx->nx_win) {
720 winctx->data_stamp = true;
721 winctx->intr_disable = true;
722 winctx->pin_win = true;
724 WARN_ON_ONCE(winctx->fault_win);
725 WARN_ON_ONCE(!winctx->rx_word_mode);
726 WARN_ON_ONCE(!winctx->tx_word_mode);
727 WARN_ON_ONCE(winctx->notify_after_count);
728 } else if (winctx->fault_win) {
729 winctx->notify_disable = true;
730 } else if (winctx->user_win) {
732 * Section 1.8.1 Low Latency Core-Core Wake up of
735 * - disable credit checks ([tr]x_wcred_mode = false)
736 * - disable FIFO writes
737 * - enable ASB_Notify, disable interrupt
739 winctx->fifo_disable = true;
740 winctx->intr_disable = true;
741 winctx->rx_fifo = NULL;
744 winctx->lnotify_lpid = rxattr->lnotify_lpid;
745 winctx->lnotify_pid = rxattr->lnotify_pid;
746 winctx->lnotify_tid = rxattr->lnotify_tid;
747 winctx->pswid = rxattr->pswid;
748 winctx->dma_type = VAS_DMA_TYPE_INJECT;
749 winctx->tc_mode = rxattr->tc_mode;
751 winctx->min_scope = VAS_SCOPE_LOCAL;
752 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
753 if (rxwin->vinst->virq)
754 winctx->irq_port = rxwin->vinst->irq_port;
757 static bool rx_win_args_valid(enum vas_cop_type cop,
758 struct vas_rx_win_attr *attr)
760 pr_debug("Rxattr: fault %d, notify %d, intr %d, early %d, fifo %d\n",
761 attr->fault_win, attr->notify_disable,
762 attr->intr_disable, attr->notify_early,
765 if (cop >= VAS_COP_TYPE_MAX)
768 if (cop != VAS_COP_TYPE_FTW &&
769 attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN)
772 if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX)
775 if (!attr->wcreds_max)
779 /* cannot be fault or user window if it is nx */
780 if (attr->fault_win || attr->user_win)
783 * Section 3.1.4.32: NX Windows must not disable notification,
784 * and must not enable interrupts or early notification.
786 if (attr->notify_disable || !attr->intr_disable ||
789 } else if (attr->fault_win) {
790 /* cannot be both fault and user window */
795 * Section 3.1.4.32: Fault windows must disable notification
796 * but not interrupts.
798 if (!attr->notify_disable || attr->intr_disable)
801 } else if (attr->user_win) {
803 * User receive windows are only for fast-thread-wakeup
804 * (FTW). They don't need a FIFO and must disable interrupts
806 if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable)
809 /* Rx window must be one of NX or Fault or User window. */
816 void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop)
818 memset(rxattr, 0, sizeof(*rxattr));
820 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
821 cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
822 rxattr->pin_win = true;
823 rxattr->nx_win = true;
824 rxattr->fault_win = false;
825 rxattr->intr_disable = true;
826 rxattr->rx_wcred_mode = true;
827 rxattr->tx_wcred_mode = true;
828 rxattr->rx_win_ord_mode = true;
829 rxattr->tx_win_ord_mode = true;
830 } else if (cop == VAS_COP_TYPE_FAULT) {
831 rxattr->pin_win = true;
832 rxattr->fault_win = true;
833 rxattr->notify_disable = true;
834 rxattr->rx_wcred_mode = true;
835 rxattr->rx_win_ord_mode = true;
836 rxattr->rej_no_credit = true;
837 rxattr->tc_mode = VAS_THRESH_DISABLED;
838 } else if (cop == VAS_COP_TYPE_FTW) {
839 rxattr->user_win = true;
840 rxattr->intr_disable = true;
843 * As noted in the VAS Workbook we disable credit checks.
844 * If we enable credit checks in the future, we must also
845 * implement a mechanism to return the user credits or new
846 * paste operations will fail.
850 EXPORT_SYMBOL_GPL(vas_init_rx_win_attr);
852 struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop,
853 struct vas_rx_win_attr *rxattr)
855 struct vas_window *rxwin;
856 struct vas_winctx winctx;
857 struct vas_instance *vinst;
859 trace_vas_rx_win_open(current, vasid, cop, rxattr);
861 if (!rx_win_args_valid(cop, rxattr))
862 return ERR_PTR(-EINVAL);
864 vinst = find_vas_instance(vasid);
866 pr_devel("vasid %d not found!\n", vasid);
867 return ERR_PTR(-EINVAL);
869 pr_devel("Found instance %d\n", vasid);
871 rxwin = vas_window_alloc(vinst);
873 pr_devel("Unable to allocate memory for Rx window\n");
877 rxwin->tx_win = false;
878 rxwin->nx_win = rxattr->nx_win;
879 rxwin->user_win = rxattr->user_win;
881 rxwin->wcreds_max = rxattr->wcreds_max;
883 init_winctx_for_rxwin(rxwin, rxattr, &winctx);
884 init_winctx_regs(rxwin, &winctx);
886 set_vinst_win(vinst, rxwin);
890 EXPORT_SYMBOL_GPL(vas_rx_win_open);
892 void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop)
894 memset(txattr, 0, sizeof(*txattr));
896 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
897 cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
898 txattr->rej_no_credit = false;
899 txattr->rx_wcred_mode = true;
900 txattr->tx_wcred_mode = true;
901 txattr->rx_win_ord_mode = true;
902 txattr->tx_win_ord_mode = true;
903 } else if (cop == VAS_COP_TYPE_FTW) {
904 txattr->user_win = true;
907 EXPORT_SYMBOL_GPL(vas_init_tx_win_attr);
909 static void init_winctx_for_txwin(struct vas_window *txwin,
910 struct vas_tx_win_attr *txattr,
911 struct vas_winctx *winctx)
914 * We first zero all fields and only set non-zero ones. Following
915 * are some fields set to 0/false for the stated reason:
917 * ->notify_os_intr_reg In powernv, send intrs to HV
918 * ->rsvd_txbuf_count Not supported yet.
919 * ->notify_disable False for NX windows
920 * ->xtra_write False for NX windows
921 * ->notify_early NA for NX windows
922 * ->lnotify_lpid NA for Tx windows
923 * ->lnotify_pid NA for Tx windows
924 * ->lnotify_tid NA for Tx windows
925 * ->tx_win_cred_mode Ignore for now for NX windows
926 * ->rx_win_cred_mode Ignore for now for NX windows
928 memset(winctx, 0, sizeof(struct vas_winctx));
930 winctx->wcreds_max = txwin->wcreds_max;
932 winctx->user_win = txattr->user_win;
933 winctx->nx_win = txwin->rxwin->nx_win;
934 winctx->pin_win = txattr->pin_win;
935 winctx->rej_no_credit = txattr->rej_no_credit;
936 winctx->rsvd_txbuf_enable = txattr->rsvd_txbuf_enable;
938 winctx->rx_wcred_mode = txattr->rx_wcred_mode;
939 winctx->tx_wcred_mode = txattr->tx_wcred_mode;
940 winctx->rx_word_mode = txattr->rx_win_ord_mode;
941 winctx->tx_word_mode = txattr->tx_win_ord_mode;
942 winctx->rsvd_txbuf_count = txattr->rsvd_txbuf_count;
944 winctx->intr_disable = true;
946 winctx->data_stamp = true;
948 winctx->lpid = txattr->lpid;
949 winctx->pidr = txattr->pidr;
950 winctx->rx_win_id = txwin->rxwin->winid;
952 * IRQ and fault window setup is successful. Set fault window
953 * for the send window so that ready to handle faults.
955 if (txwin->vinst->virq)
956 winctx->fault_win_id = txwin->vinst->fault_win->winid;
958 winctx->dma_type = VAS_DMA_TYPE_INJECT;
959 winctx->tc_mode = txattr->tc_mode;
960 winctx->min_scope = VAS_SCOPE_LOCAL;
961 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
962 if (txwin->vinst->virq)
963 winctx->irq_port = txwin->vinst->irq_port;
965 winctx->pswid = txattr->pswid ? txattr->pswid :
966 encode_pswid(txwin->vinst->vas_id, txwin->winid);
969 static bool tx_win_args_valid(enum vas_cop_type cop,
970 struct vas_tx_win_attr *attr)
972 if (attr->tc_mode != VAS_THRESH_DISABLED)
975 if (cop > VAS_COP_TYPE_MAX)
978 if (attr->wcreds_max > VAS_TX_WCREDS_MAX)
981 if (attr->user_win) {
982 if (attr->rsvd_txbuf_count)
985 if (cop != VAS_COP_TYPE_FTW && cop != VAS_COP_TYPE_GZIP &&
986 cop != VAS_COP_TYPE_GZIP_HIPRI)
993 struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
994 struct vas_tx_win_attr *attr)
997 struct vas_window *txwin;
998 struct vas_window *rxwin;
999 struct vas_winctx winctx;
1000 struct vas_instance *vinst;
1002 trace_vas_tx_win_open(current, vasid, cop, attr);
1004 if (!tx_win_args_valid(cop, attr))
1005 return ERR_PTR(-EINVAL);
1008 * If caller did not specify a vasid but specified the PSWID of a
1009 * receive window (applicable only to FTW windows), use the vasid
1010 * from that receive window.
1012 if (vasid == -1 && attr->pswid)
1013 decode_pswid(attr->pswid, &vasid, NULL);
1015 vinst = find_vas_instance(vasid);
1017 pr_devel("vasid %d not found!\n", vasid);
1018 return ERR_PTR(-EINVAL);
1021 rxwin = get_vinst_rxwin(vinst, cop, attr->pswid);
1022 if (IS_ERR(rxwin)) {
1023 pr_devel("No RxWin for vasid %d, cop %d\n", vasid, cop);
1027 txwin = vas_window_alloc(vinst);
1028 if (IS_ERR(txwin)) {
1029 rc = PTR_ERR(txwin);
1035 txwin->rxwin = rxwin;
1036 txwin->nx_win = txwin->rxwin->nx_win;
1037 txwin->user_win = attr->user_win;
1038 txwin->wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT;
1040 init_winctx_for_txwin(txwin, attr, &winctx);
1042 init_winctx_regs(txwin, &winctx);
1045 * If its a kernel send window, map the window address into the
1046 * kernel's address space. For user windows, user must issue an
1047 * mmap() to map the window into their address space.
1049 * NOTE: If kernel ever resubmits a user CRB after handling a page
1050 * fault, we will need to map this into kernel as well.
1052 if (!txwin->user_win) {
1053 txwin->paste_kaddr = map_paste_region(txwin);
1054 if (IS_ERR(txwin->paste_kaddr)) {
1055 rc = PTR_ERR(txwin->paste_kaddr);
1060 * Interrupt hanlder or fault window setup failed. Means
1061 * NX can not generate fault for page fault. So not
1062 * opening for user space tx window.
1070 * Window opened by a child thread may not be closed when
1071 * it exits. So take reference to its pid and release it
1072 * when the window is free by parent thread.
1073 * Acquire a reference to the task's pid to make sure
1074 * pid will not be re-used - needed only for multithread
1077 txwin->pid = get_task_pid(current, PIDTYPE_PID);
1079 * Acquire a reference to the task's mm.
1081 txwin->mm = get_task_mm(current);
1084 put_pid(txwin->pid);
1085 pr_err("VAS: pid(%d): mm_struct is not found\n",
1093 mm_context_add_vas_window(txwin->mm);
1095 * Process closes window during exit. In the case of
1096 * multithread application, the child thread can open
1097 * window and can exit without closing it. Expects parent
1098 * thread to use and close the window. So do not need
1099 * to take pid reference for parent thread.
1101 txwin->tgid = find_get_pid(task_tgid_vnr(current));
1103 * Even a process that has no foreign real address mapping can
1104 * use an unpaired COPY instruction (to no real effect). Issue
1105 * CP_ABORT to clear any pending COPY and prevent a covert
1108 * __switch_to() will issue CP_ABORT on future context switches
1109 * if process / thread has any open VAS window (Use
1110 * current->mm->context.vas_windows).
1112 asm volatile(PPC_CP_ABORT);
1115 set_vinst_win(vinst, txwin);
1120 vas_window_free(txwin);
1127 EXPORT_SYMBOL_GPL(vas_tx_win_open);
1129 int vas_copy_crb(void *crb, int offset)
1131 return vas_copy(crb, offset);
1133 EXPORT_SYMBOL_GPL(vas_copy_crb);
1135 #define RMA_LSMP_REPORT_ENABLE PPC_BIT(53)
1136 int vas_paste_crb(struct vas_window *txwin, int offset, bool re)
1142 trace_vas_paste_crb(current, txwin);
1145 * Only NX windows are supported for now and hardware assumes
1146 * report-enable flag is set for NX windows. Ensure software
1149 WARN_ON_ONCE(txwin->nx_win && !re);
1151 addr = txwin->paste_kaddr;
1154 * Set the REPORT_ENABLE bit (equivalent to writing
1155 * to 1K offset of the paste address)
1157 val = SET_FIELD(RMA_LSMP_REPORT_ENABLE, 0ULL, 1);
1162 * Map the raw CR value from vas_paste() to an error code (there
1163 * is just pass or fail for now though).
1165 rc = vas_paste(addr, offset);
1171 pr_debug("Txwin #%d: Msg count %llu\n", txwin->winid,
1172 read_hvwc_reg(txwin, VREG(LRFIFO_PUSH)));
1176 EXPORT_SYMBOL_GPL(vas_paste_crb);
1179 * If credit checking is enabled for this window, poll for the return
1180 * of window credits (i.e for NX engines to process any outstanding CRBs).
1181 * Since NX-842 waits for the CRBs to be processed before closing the
1182 * window, we should not have to wait for too long.
1184 * TODO: We retry in 10ms intervals now. We could/should probably peek at
1185 * the VAS_LRFIFO_PUSH_OFFSET register to get an estimate of pending
1186 * CRBs on the FIFO and compute the delay dynamically on each retry.
1187 * But that is not really needed until we support NX-GZIP access from
1188 * user space. (NX-842 driver waits for CSB and Fast thread-wakeup
1189 * doesn't use credit checking).
1191 static void poll_window_credits(struct vas_window *window)
1197 val = read_hvwc_reg(window, VREG(WINCTL));
1199 mode = GET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val);
1201 mode = GET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val);
1206 if (window->tx_win) {
1207 val = read_hvwc_reg(window, VREG(TX_WCRED));
1208 creds = GET_FIELD(VAS_TX_WCRED, val);
1210 val = read_hvwc_reg(window, VREG(LRX_WCRED));
1211 creds = GET_FIELD(VAS_LRX_WCRED, val);
1215 * Takes around few milliseconds to complete all pending requests
1216 * and return credits.
1217 * TODO: Scan fault FIFO and invalidate CRBs points to this window
1218 * and issue CRB Kill to stop all pending requests. Need only
1219 * if there is a bug in NX or fault handling in kernel.
1221 if (creds < window->wcreds_max) {
1223 set_current_state(TASK_UNINTERRUPTIBLE);
1224 schedule_timeout(msecs_to_jiffies(10));
1227 * Process can not close send window until all credits are
1230 if (!(count % 1000))
1231 pr_warn_ratelimited("VAS: pid %d stuck. Waiting for credits returned for Window(%d). creds %d, Retries %d\n",
1232 vas_window_pid(window), window->winid,
1240 * Wait for the window to go to "not-busy" state. It should only take a
1241 * short time to queue a CRB, so window should not be busy for too long.
1242 * Trying 5ms intervals.
1244 static void poll_window_busy_state(struct vas_window *window)
1251 val = read_hvwc_reg(window, VREG(WIN_STATUS));
1252 busy = GET_FIELD(VAS_WIN_BUSY, val);
1255 set_current_state(TASK_UNINTERRUPTIBLE);
1256 schedule_timeout(msecs_to_jiffies(10));
1259 * Takes around few milliseconds to process all pending
1262 if (!(count % 1000))
1263 pr_warn_ratelimited("VAS: pid %d stuck. Window (ID=%d) is in busy state. Retries %d\n",
1264 vas_window_pid(window), window->winid, count);
1271 * Have the hardware cast a window out of cache and wait for it to
1274 * NOTE: It can take a relatively long time to cast the window context
1275 * out of the cache. It is not strictly necessary to cast out if:
1277 * - we clear the "Pin Window" bit (so hardware is free to evict)
1279 * - we re-initialize the window context when it is reassigned.
1281 * We do the former in vas_win_close() and latter in vas_win_open().
1282 * So, ignoring the cast-out for now. We can add it as needed. If
1283 * casting out becomes necessary we should consider offloading the
1284 * job to a worker thread, so the window close can proceed quickly.
1286 static void poll_window_castout(struct vas_window *window)
1292 * Unpin and close a window so no new requests are accepted and the
1293 * hardware can evict this window from cache if necessary.
1295 static void unpin_close_window(struct vas_window *window)
1299 val = read_hvwc_reg(window, VREG(WINCTL));
1300 val = SET_FIELD(VAS_WINCTL_PIN, val, 0);
1301 val = SET_FIELD(VAS_WINCTL_OPEN, val, 0);
1302 write_hvwc_reg(window, VREG(WINCTL), val);
1308 * See Section 1.12.1 of VAS workbook v1.05 for details on closing window:
1309 * - Disable new paste operations (unmap paste address)
1310 * - Poll for the "Window Busy" bit to be cleared
1311 * - Clear the Open/Enable bit for the Window.
1312 * - Poll for return of window Credits (implies FIFO empty for Rx win?)
1313 * - Unpin and cast window context out of cache
1315 * Besides the hardware, kernel has some bookkeeping of course.
1317 int vas_win_close(struct vas_window *window)
1322 if (!window->tx_win && atomic_read(&window->num_txwins) != 0) {
1323 pr_devel("Attempting to close an active Rx window!\n");
1328 unmap_paste_region(window);
1330 poll_window_busy_state(window);
1332 unpin_close_window(window);
1334 poll_window_credits(window);
1336 clear_vinst_win(window);
1338 poll_window_castout(window);
1340 /* if send window, drop reference to matching receive window */
1341 if (window->tx_win) {
1342 if (window->user_win) {
1343 /* Drop references to pid and mm */
1344 put_pid(window->pid);
1346 mm_context_remove_vas_window(window->mm);
1350 put_rx_win(window->rxwin);
1353 vas_window_free(window);
1357 EXPORT_SYMBOL_GPL(vas_win_close);
1360 * Return credit for the given window.
1361 * Send windows and fault window uses credit mechanism as follows:
1364 * - The default number of credits available for each send window is
1365 * 1024. It means 1024 requests can be issued asynchronously at the
1366 * same time. If the credit is not available, that request will be
1367 * returned with RMA_Busy.
1368 * - One credit is taken when NX request is issued.
1369 * - This credit is returned after NX processed that request.
1370 * - If NX encounters translation error, kernel will return the
1371 * credit on the specific send window after processing the fault CRB.
1374 * - The total number credits available is FIFO_SIZE/CRB_SIZE.
1375 * Means 4MB/128 in the current implementation. If credit is not
1376 * available, RMA_Reject is returned.
1377 * - A credit is taken when NX pastes CRB in fault FIFO.
1378 * - The kernel with return credit on fault window after reading entry
1381 void vas_return_credit(struct vas_window *window, bool tx)
1386 if (tx) { /* send window */
1387 val = SET_FIELD(VAS_TX_WCRED, val, 1);
1388 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), val);
1390 val = SET_FIELD(VAS_LRX_WCRED, val, 1);
1391 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), val);
1395 struct vas_window *vas_pswid_to_window(struct vas_instance *vinst,
1398 struct vas_window *window;
1402 pr_devel("%s: called for pswid 0!\n", __func__);
1403 return ERR_PTR(-ESRCH);
1406 decode_pswid(pswid, NULL, &winid);
1408 if (winid >= VAS_WINDOWS_PER_CHIP)
1409 return ERR_PTR(-ESRCH);
1412 * If application closes the window before the hardware
1413 * returns the fault CRB, we should wait in vas_win_close()
1414 * for the pending requests. so the window must be active
1415 * and the process alive.
1417 * If its a kernel process, we should not get any faults and
1418 * should not get here.
1420 window = vinst->windows[winid];
1423 pr_err("PSWID decode: Could not find window for winid %d pswid %d vinst 0x%p\n",
1424 winid, pswid, vinst);
1429 * Do some sanity checks on the decoded window. Window should be
1430 * NX GZIP user send window. FTW windows should not incur faults
1431 * since their CRBs are ignored (not queued on FIFO or processed
1434 if (!window->tx_win || !window->user_win || !window->nx_win ||
1435 window->cop == VAS_COP_TYPE_FAULT ||
1436 window->cop == VAS_COP_TYPE_FTW) {
1437 pr_err("PSWID decode: id %d, tx %d, user %d, nx %d, cop %d\n",
1438 winid, window->tx_win, window->user_win,
1439 window->nx_win, window->cop);