1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright 2016-17 IBM Corp.
4 */
5
6#define pr_fmt(fmt) "vas: " fmt
7
8#include <linux/types.h>
9#include <linux/mutex.h>
10#include <linux/slab.h>
11#include <linux/io.h>
12#include <linux/log2.h>
13#include <linux/rcupdate.h>
14#include <linux/cred.h>
15#include <linux/sched/mm.h>
16#include <linux/mmu_context.h>
17#include <asm/switch_to.h>
18#include <asm/ppc-opcode.h>
19#include <asm/vas.h>
20#include "vas.h"
21#include "copy-paste.h"
22
23#define CREATE_TRACE_POINTS
24#include "vas-trace.h"
25
26/*
27 * Compute the paste address region for the window @window using the
28 * ->paste_base_addr and ->paste_win_id_shift we got from device tree.
29 */
30void vas_win_paste_addr(struct pnv_vas_window *window, u64 *addr, int *len)
31{
32 int winid;
33 u64 base, shift;
34
35 base = window->vinst->paste_base_addr;
36 shift = window->vinst->paste_win_id_shift;
37 winid = window->vas_win.winid;
38
39 *addr = base + (winid << shift);
40 if (len)
41 *len = PAGE_SIZE;
42
43 pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr);
44}
45
46static inline void get_hvwc_mmio_bar(struct pnv_vas_window *window,
47 u64 *start, int *len)
48{
49 u64 pbaddr;
50
51 pbaddr = window->vinst->hvwc_bar_start;
52 *start = pbaddr + window->vas_win.winid * VAS_HVWC_SIZE;
53 *len = VAS_HVWC_SIZE;
54}
55
56static inline void get_uwc_mmio_bar(struct pnv_vas_window *window,
57 u64 *start, int *len)
58{
59 u64 pbaddr;
60
61 pbaddr = window->vinst->uwc_bar_start;
62 *start = pbaddr + window->vas_win.winid * VAS_UWC_SIZE;
63 *len = VAS_UWC_SIZE;
64}
65
66/*
67 * Map the paste bus address of the given send window into kernel address
68 * space. Unlike MMIO regions (map_mmio_region() below), paste region must
69 * be mapped cache-able and is only applicable to send windows.
70 */
71static void *map_paste_region(struct pnv_vas_window *txwin)
72{
73 int len;
74 void *map;
75 char *name;
76 u64 start;
77
78 name = kasprintf(GFP_KERNEL, fmt: "window-v%d-w%d", txwin->vinst->vas_id,
79 txwin->vas_win.winid);
80 if (!name)
81 goto free_name;
82
83 txwin->paste_addr_name = name;
84 vas_win_paste_addr(window: txwin, addr: &start, len: &len);
85
86 if (!request_mem_region(start, len, name)) {
87 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
88 __func__, start, len);
89 goto free_name;
90 }
91
92 map = ioremap_cache(offset: start, size: len);
93 if (!map) {
94 pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__,
95 start, len);
96 goto free_name;
97 }
98
99 pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map);
100 return map;
101
102free_name:
103 kfree(objp: name);
104 return ERR_PTR(error: -ENOMEM);
105}
106
107static void *map_mmio_region(char *name, u64 start, int len)
108{
109 void *map;
110
111 if (!request_mem_region(start, len, name)) {
112 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
113 __func__, start, len);
114 return NULL;
115 }
116
117 map = ioremap(offset: start, size: len);
118 if (!map) {
119 pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start,
120 len);
121 return NULL;
122 }
123
124 return map;
125}
126
127static void unmap_region(void *addr, u64 start, int len)
128{
129 iounmap(addr);
130 release_mem_region((phys_addr_t)start, len);
131}
132
133/*
134 * Unmap the paste address region for a window.
135 */
136static void unmap_paste_region(struct pnv_vas_window *window)
137{
138 int len;
139 u64 busaddr_start;
140
141 if (window->paste_kaddr) {
142 vas_win_paste_addr(window, addr: &busaddr_start, len: &len);
143 unmap_region(addr: window->paste_kaddr, start: busaddr_start, len);
144 window->paste_kaddr = NULL;
145 kfree(objp: window->paste_addr_name);
146 window->paste_addr_name = NULL;
147 }
148}
149
150/*
151 * Unmap the MMIO regions for a window. Hold the vas_mutex so we don't
152 * unmap when the window's debugfs dir is in use. This serializes close
153 * of a window even on another VAS instance but since its not a critical
154 * path, just minimize the time we hold the mutex for now. We can add
155 * a per-instance mutex later if necessary.
156 */
157static void unmap_winctx_mmio_bars(struct pnv_vas_window *window)
158{
159 int len;
160 void *uwc_map;
161 void *hvwc_map;
162 u64 busaddr_start;
163
164 mutex_lock(&vas_mutex);
165
166 hvwc_map = window->hvwc_map;
167 window->hvwc_map = NULL;
168
169 uwc_map = window->uwc_map;
170 window->uwc_map = NULL;
171
172 mutex_unlock(lock: &vas_mutex);
173
174 if (hvwc_map) {
175 get_hvwc_mmio_bar(window, start: &busaddr_start, len: &len);
176 unmap_region(addr: hvwc_map, start: busaddr_start, len);
177 }
178
179 if (uwc_map) {
180 get_uwc_mmio_bar(window, start: &busaddr_start, len: &len);
181 unmap_region(addr: uwc_map, start: busaddr_start, len);
182 }
183}
184
185/*
186 * Find the Hypervisor Window Context (HVWC) MMIO Base Address Region and the
187 * OS/User Window Context (UWC) MMIO Base Address Region for the given window.
188 * Map these bus addresses and save the mapped kernel addresses in @window.
189 */
190static int map_winctx_mmio_bars(struct pnv_vas_window *window)
191{
192 int len;
193 u64 start;
194
195 get_hvwc_mmio_bar(window, start: &start, len: &len);
196 window->hvwc_map = map_mmio_region(name: "HVWCM_Window", start, len);
197
198 get_uwc_mmio_bar(window, start: &start, len: &len);
199 window->uwc_map = map_mmio_region(name: "UWCM_Window", start, len);
200
201 if (!window->hvwc_map || !window->uwc_map) {
202 unmap_winctx_mmio_bars(window);
203 return -1;
204 }
205
206 return 0;
207}
208
209/*
210 * Reset all valid registers in the HV and OS/User Window Contexts for
211 * the window identified by @window.
212 *
213 * NOTE: We cannot really use a for loop to reset window context. Not all
214 * offsets in a window context are valid registers and the valid
215 * registers are not sequential. And, we can only write to offsets
216 * with valid registers.
217 */
218static void reset_window_regs(struct pnv_vas_window *window)
219{
220 write_hvwc_reg(win: window, VREG(LPID), val: 0ULL);
221 write_hvwc_reg(win: window, VREG(PID), val: 0ULL);
222 write_hvwc_reg(win: window, VREG(XLATE_MSR), val: 0ULL);
223 write_hvwc_reg(win: window, VREG(XLATE_LPCR), val: 0ULL);
224 write_hvwc_reg(win: window, VREG(XLATE_CTL), val: 0ULL);
225 write_hvwc_reg(win: window, VREG(AMR), val: 0ULL);
226 write_hvwc_reg(win: window, VREG(SEIDR), val: 0ULL);
227 write_hvwc_reg(win: window, VREG(FAULT_TX_WIN), val: 0ULL);
228 write_hvwc_reg(win: window, VREG(OSU_INTR_SRC_RA), val: 0ULL);
229 write_hvwc_reg(win: window, VREG(HV_INTR_SRC_RA), val: 0ULL);
230 write_hvwc_reg(win: window, VREG(PSWID), val: 0ULL);
231 write_hvwc_reg(win: window, VREG(LFIFO_BAR), val: 0ULL);
232 write_hvwc_reg(win: window, VREG(LDATA_STAMP_CTL), val: 0ULL);
233 write_hvwc_reg(win: window, VREG(LDMA_CACHE_CTL), val: 0ULL);
234 write_hvwc_reg(win: window, VREG(LRFIFO_PUSH), val: 0ULL);
235 write_hvwc_reg(win: window, VREG(CURR_MSG_COUNT), val: 0ULL);
236 write_hvwc_reg(win: window, VREG(LNOTIFY_AFTER_COUNT), val: 0ULL);
237 write_hvwc_reg(win: window, VREG(LRX_WCRED), val: 0ULL);
238 write_hvwc_reg(win: window, VREG(LRX_WCRED_ADDER), val: 0ULL);
239 write_hvwc_reg(win: window, VREG(TX_WCRED), val: 0ULL);
240 write_hvwc_reg(win: window, VREG(TX_WCRED_ADDER), val: 0ULL);
241 write_hvwc_reg(win: window, VREG(LFIFO_SIZE), val: 0ULL);
242 write_hvwc_reg(win: window, VREG(WINCTL), val: 0ULL);
243 write_hvwc_reg(win: window, VREG(WIN_STATUS), val: 0ULL);
244 write_hvwc_reg(win: window, VREG(WIN_CTX_CACHING_CTL), val: 0ULL);
245 write_hvwc_reg(win: window, VREG(TX_RSVD_BUF_COUNT), val: 0ULL);
246 write_hvwc_reg(win: window, VREG(LRFIFO_WIN_PTR), val: 0ULL);
247 write_hvwc_reg(win: window, VREG(LNOTIFY_CTL), val: 0ULL);
248 write_hvwc_reg(win: window, VREG(LNOTIFY_PID), val: 0ULL);
249 write_hvwc_reg(win: window, VREG(LNOTIFY_LPID), val: 0ULL);
250 write_hvwc_reg(win: window, VREG(LNOTIFY_TID), val: 0ULL);
251 write_hvwc_reg(win: window, VREG(LNOTIFY_SCOPE), val: 0ULL);
252 write_hvwc_reg(win: window, VREG(NX_UTIL_ADDER), val: 0ULL);
253
254 /* Skip read-only registers: NX_UTIL and NX_UTIL_SE */
255
256 /*
257 * The send and receive window credit adder registers are also
258 * accessible from HVWC and have been initialized above. We don't
259 * need to initialize from the OS/User Window Context, so skip
260 * following calls:
261 *
262 * write_uwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
263 * write_uwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
264 */
265}
266
267/*
268 * Initialize window context registers related to Address Translation.
269 * These registers are common to send/receive windows although they
270 * differ for user/kernel windows. As we resolve the TODOs we may
271 * want to add fields to vas_winctx and move the initialization to
272 * init_vas_winctx_regs().
273 */
274static void init_xlate_regs(struct pnv_vas_window *window, bool user_win)
275{
276 u64 lpcr, val;
277
278 /*
279 * MSR_TA, MSR_US are false for both kernel and user.
280 * MSR_DR and MSR_PR are false for kernel.
281 */
282 val = 0ULL;
283 val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1);
284 val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1);
285 if (user_win) {
286 val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1);
287 val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1);
288 }
289 write_hvwc_reg(win: window, VREG(XLATE_MSR), val);
290
291 lpcr = mfspr(SPRN_LPCR);
292 val = 0ULL;
293 /*
294 * NOTE: From Section 5.7.8.1 Segment Lookaside Buffer of the
295 * Power ISA, v3.0B, Page size encoding is 0 = 4KB, 5 = 64KB.
296 *
297 * NOTE: From Section 1.3.1, Address Translation Context of the
298 * Nest MMU Workbook, LPCR_SC should be 0 for Power9.
299 */
300 val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5);
301 val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL);
302 val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC);
303 val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0);
304 write_hvwc_reg(win: window, VREG(XLATE_LPCR), val);
305
306 /*
307 * Section 1.3.1 (Address translation Context) of NMMU workbook.
308 * 0b00 Hashed Page Table mode
309 * 0b01 Reserved
310 * 0b10 Radix on HPT
311 * 0b11 Radix on Radix
312 */
313 val = 0ULL;
314 val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2);
315 write_hvwc_reg(win: window, VREG(XLATE_CTL), val);
316
317 /*
318 * TODO: Can we mfspr(AMR) even for user windows?
319 */
320 val = 0ULL;
321 val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR));
322 write_hvwc_reg(win: window, VREG(AMR), val);
323
324 val = 0ULL;
325 val = SET_FIELD(VAS_SEIDR, val, 0);
326 write_hvwc_reg(win: window, VREG(SEIDR), val);
327}
328
329/*
330 * Initialize Reserved Send Buffer Count for the send window. It involves
331 * writing to the register, reading it back to confirm that the hardware
332 * has enough buffers to reserve. See section 1.3.1.2.1 of VAS workbook.
333 *
334 * Since we can only make a best-effort attempt to fulfill the request,
335 * we don't return any errors if we cannot.
336 *
337 * TODO: Reserved (aka dedicated) send buffers are not supported yet.
338 */
339static void init_rsvd_tx_buf_count(struct pnv_vas_window *txwin,
340 struct vas_winctx *winctx)
341{
342 write_hvwc_reg(win: txwin, VREG(TX_RSVD_BUF_COUNT), val: 0ULL);
343}
344
345/*
346 * init_winctx_regs()
347 * Initialize window context registers for a receive window.
348 * Except for caching control and marking window open, the registers
349 * are initialized in the order listed in Section 3.1.4 (Window Context
350 * Cache Register Details) of the VAS workbook although they don't need
351 * to be.
352 *
353 * Design note: For NX receive windows, NX allocates the FIFO buffer in OPAL
354 * (so that it can get a large contiguous area) and passes that buffer
355 * to kernel via device tree. We now write that buffer address to the
356 * FIFO BAR. Would it make sense to do this all in OPAL? i.e have OPAL
357 * write the per-chip RX FIFO addresses to the windows during boot-up
358 * as a one-time task? That could work for NX but what about other
359 * receivers? Let the receivers tell us the rx-fifo buffers for now.
360 */
361static void init_winctx_regs(struct pnv_vas_window *window,
362 struct vas_winctx *winctx)
363{
364 u64 val;
365 int fifo_size;
366
367 reset_window_regs(window);
368
369 val = 0ULL;
370 val = SET_FIELD(VAS_LPID, val, winctx->lpid);
371 write_hvwc_reg(win: window, VREG(LPID), val);
372
373 val = 0ULL;
374 val = SET_FIELD(VAS_PID_ID, val, winctx->pidr);
375 write_hvwc_reg(win: window, VREG(PID), val);
376
377 init_xlate_regs(window, user_win: winctx->user_win);
378
379 val = 0ULL;
380 val = SET_FIELD(VAS_FAULT_TX_WIN, val, winctx->fault_win_id);
381 write_hvwc_reg(win: window, VREG(FAULT_TX_WIN), val);
382
383 /* In PowerNV, interrupts go to HV. */
384 write_hvwc_reg(win: window, VREG(OSU_INTR_SRC_RA), val: 0ULL);
385
386 val = 0ULL;
387 val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port);
388 write_hvwc_reg(win: window, VREG(HV_INTR_SRC_RA), val);
389
390 val = 0ULL;
391 val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid);
392 write_hvwc_reg(win: window, VREG(PSWID), val);
393
394 write_hvwc_reg(win: window, VREG(SPARE1), val: 0ULL);
395 write_hvwc_reg(win: window, VREG(SPARE2), val: 0ULL);
396 write_hvwc_reg(win: window, VREG(SPARE3), val: 0ULL);
397
398 /*
399 * NOTE: VAS expects the FIFO address to be copied into the LFIFO_BAR
400 * register as is - do NOT shift the address into VAS_LFIFO_BAR
401 * bit fields! Ok to set the page migration select fields -
402 * VAS ignores the lower 10+ bits in the address anyway, because
403 * the minimum FIFO size is 1K?
404 *
405 * See also: Design note in function header.
406 */
407 val = winctx->rx_fifo;
408 val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0);
409 write_hvwc_reg(win: window, VREG(LFIFO_BAR), val);
410
411 val = 0ULL;
412 val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp);
413 write_hvwc_reg(win: window, VREG(LDATA_STAMP_CTL), val);
414
415 val = 0ULL;
416 val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type);
417 val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable);
418 write_hvwc_reg(win: window, VREG(LDMA_CACHE_CTL), val);
419
420 write_hvwc_reg(win: window, VREG(LRFIFO_PUSH), val: 0ULL);
421 write_hvwc_reg(win: window, VREG(CURR_MSG_COUNT), val: 0ULL);
422 write_hvwc_reg(win: window, VREG(LNOTIFY_AFTER_COUNT), val: 0ULL);
423
424 val = 0ULL;
425 val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max);
426 write_hvwc_reg(win: window, VREG(LRX_WCRED), val);
427
428 val = 0ULL;
429 val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max);
430 write_hvwc_reg(win: window, VREG(TX_WCRED), val);
431
432 write_hvwc_reg(win: window, VREG(LRX_WCRED_ADDER), val: 0ULL);
433 write_hvwc_reg(win: window, VREG(TX_WCRED_ADDER), val: 0ULL);
434
435 fifo_size = winctx->rx_fifo_size / 1024;
436
437 val = 0ULL;
438 val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size));
439 write_hvwc_reg(win: window, VREG(LFIFO_SIZE), val);
440
441 /* Update window control and caching control registers last so
442 * we mark the window open only after fully initializing it and
443 * pushing context to cache.
444 */
445
446 write_hvwc_reg(win: window, VREG(WIN_STATUS), val: 0ULL);
447
448 init_rsvd_tx_buf_count(txwin: window, winctx);
449
450 /* for a send window, point to the matching receive window */
451 val = 0ULL;
452 val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id);
453 write_hvwc_reg(win: window, VREG(LRFIFO_WIN_PTR), val);
454
455 write_hvwc_reg(win: window, VREG(SPARE4), val: 0ULL);
456
457 val = 0ULL;
458 val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable);
459 val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable);
460 val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early);
461 val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg);
462 write_hvwc_reg(win: window, VREG(LNOTIFY_CTL), val);
463
464 val = 0ULL;
465 val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid);
466 write_hvwc_reg(win: window, VREG(LNOTIFY_PID), val);
467
468 val = 0ULL;
469 val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid);
470 write_hvwc_reg(win: window, VREG(LNOTIFY_LPID), val);
471
472 val = 0ULL;
473 val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid);
474 write_hvwc_reg(win: window, VREG(LNOTIFY_TID), val);
475
476 val = 0ULL;
477 val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope);
478 val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope);
479 write_hvwc_reg(win: window, VREG(LNOTIFY_SCOPE), val);
480
481 /* Skip read-only registers NX_UTIL and NX_UTIL_SE */
482
483 write_hvwc_reg(win: window, VREG(SPARE5), val: 0ULL);
484 write_hvwc_reg(win: window, VREG(NX_UTIL_ADDER), val: 0ULL);
485 write_hvwc_reg(win: window, VREG(SPARE6), val: 0ULL);
486
487 /* Finally, push window context to memory and... */
488 val = 0ULL;
489 val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1);
490 write_hvwc_reg(win: window, VREG(WIN_CTX_CACHING_CTL), val);
491
492 /* ... mark the window open for business */
493 val = 0ULL;
494 val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit);
495 val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win);
496 val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode);
497 val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode);
498 val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode);
499 val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode);
500 val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win);
501 val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win);
502 val = SET_FIELD(VAS_WINCTL_OPEN, val, 1);
503 write_hvwc_reg(win: window, VREG(WINCTL), val);
504}
505
506static void vas_release_window_id(struct ida *ida, int winid)
507{
508 ida_free(ida, id: winid);
509}
510
511static int vas_assign_window_id(struct ida *ida)
512{
513 int winid = ida_alloc_max(ida, VAS_WINDOWS_PER_CHIP - 1, GFP_KERNEL);
514
515 if (winid == -ENOSPC) {
516 pr_err("Too many (%d) open windows\n", VAS_WINDOWS_PER_CHIP);
517 return -EAGAIN;
518 }
519
520 return winid;
521}
522
523static void vas_window_free(struct pnv_vas_window *window)
524{
525 struct vas_instance *vinst = window->vinst;
526 int winid = window->vas_win.winid;
527
528 unmap_winctx_mmio_bars(window);
529
530 vas_window_free_dbgdir(win: window);
531
532 kfree(objp: window);
533
534 vas_release_window_id(ida: &vinst->ida, winid);
535}
536
537static struct pnv_vas_window *vas_window_alloc(struct vas_instance *vinst)
538{
539 int winid;
540 struct pnv_vas_window *window;
541
542 winid = vas_assign_window_id(ida: &vinst->ida);
543 if (winid < 0)
544 return ERR_PTR(error: winid);
545
546 window = kzalloc(sizeof(*window), GFP_KERNEL);
547 if (!window)
548 goto out_free;
549
550 window->vinst = vinst;
551 window->vas_win.winid = winid;
552
553 if (map_winctx_mmio_bars(window))
554 goto out_free;
555
556 vas_window_init_dbgdir(win: window);
557
558 return window;
559
560out_free:
561 kfree(objp: window);
562 vas_release_window_id(ida: &vinst->ida, winid);
563 return ERR_PTR(error: -ENOMEM);
564}
565
566static void put_rx_win(struct pnv_vas_window *rxwin)
567{
568 /* Better not be a send window! */
569 WARN_ON_ONCE(rxwin->tx_win);
570
571 atomic_dec(v: &rxwin->num_txwins);
572}
573
574/*
575 * Find the user space receive window given the @pswid.
576 * - We must have a valid vasid and it must belong to this instance.
577 * (so both send and receive windows are on the same VAS instance)
578 * - The window must refer to an OPEN, FTW, RECEIVE window.
579 *
580 * NOTE: We access ->windows[] table and assume that vinst->mutex is held.
581 */
582static struct pnv_vas_window *get_user_rxwin(struct vas_instance *vinst,
583 u32 pswid)
584{
585 int vasid, winid;
586 struct pnv_vas_window *rxwin;
587
588 decode_pswid(pswid, vasid: &vasid, winid: &winid);
589
590 if (vinst->vas_id != vasid)
591 return ERR_PTR(error: -EINVAL);
592
593 rxwin = vinst->windows[winid];
594
595 if (!rxwin || rxwin->tx_win || rxwin->vas_win.cop != VAS_COP_TYPE_FTW)
596 return ERR_PTR(error: -EINVAL);
597
598 return rxwin;
599}
600
601/*
602 * Get the VAS receive window associated with NX engine identified
603 * by @cop and if applicable, @pswid.
604 *
605 * See also function header of set_vinst_win().
606 */
607static struct pnv_vas_window *get_vinst_rxwin(struct vas_instance *vinst,
608 enum vas_cop_type cop, u32 pswid)
609{
610 struct pnv_vas_window *rxwin;
611
612 mutex_lock(&vinst->mutex);
613
614 if (cop == VAS_COP_TYPE_FTW)
615 rxwin = get_user_rxwin(vinst, pswid);
616 else
617 rxwin = vinst->rxwin[cop] ?: ERR_PTR(error: -EINVAL);
618
619 if (!IS_ERR(ptr: rxwin))
620 atomic_inc(v: &rxwin->num_txwins);
621
622 mutex_unlock(lock: &vinst->mutex);
623
624 return rxwin;
625}
626
627/*
628 * We have two tables of windows in a VAS instance. The first one,
629 * ->windows[], contains all the windows in the instance and allows
630 * looking up a window by its id. It is used to look up send windows
631 * during fault handling and receive windows when pairing user space
632 * send/receive windows.
633 *
634 * The second table, ->rxwin[], contains receive windows that are
635 * associated with NX engines. This table has VAS_COP_TYPE_MAX
636 * entries and is used to look up a receive window by its
637 * coprocessor type.
638 *
639 * Here, we save @window in the ->windows[] table. If it is a receive
640 * window, we also save the window in the ->rxwin[] table.
641 */
642static void set_vinst_win(struct vas_instance *vinst,
643 struct pnv_vas_window *window)
644{
645 int id = window->vas_win.winid;
646
647 mutex_lock(&vinst->mutex);
648
649 /*
650 * There should only be one receive window for a coprocessor type
651 * unless its a user (FTW) window.
652 */
653 if (!window->user_win && !window->tx_win) {
654 WARN_ON_ONCE(vinst->rxwin[window->vas_win.cop]);
655 vinst->rxwin[window->vas_win.cop] = window;
656 }
657
658 WARN_ON_ONCE(vinst->windows[id] != NULL);
659 vinst->windows[id] = window;
660
661 mutex_unlock(lock: &vinst->mutex);
662}
663
664/*
665 * Clear this window from the table(s) of windows for this VAS instance.
666 * See also function header of set_vinst_win().
667 */
668static void clear_vinst_win(struct pnv_vas_window *window)
669{
670 int id = window->vas_win.winid;
671 struct vas_instance *vinst = window->vinst;
672
673 mutex_lock(&vinst->mutex);
674
675 if (!window->user_win && !window->tx_win) {
676 WARN_ON_ONCE(!vinst->rxwin[window->vas_win.cop]);
677 vinst->rxwin[window->vas_win.cop] = NULL;
678 }
679
680 WARN_ON_ONCE(vinst->windows[id] != window);
681 vinst->windows[id] = NULL;
682
683 mutex_unlock(lock: &vinst->mutex);
684}
685
686static void init_winctx_for_rxwin(struct pnv_vas_window *rxwin,
687 struct vas_rx_win_attr *rxattr,
688 struct vas_winctx *winctx)
689{
690 /*
691 * We first zero (memset()) all fields and only set non-zero fields.
692 * Following fields are 0/false but maybe deserve a comment:
693 *
694 * ->notify_os_intr_reg In powerNV, send intrs to HV
695 * ->notify_disable False for NX windows
696 * ->intr_disable False for Fault Windows
697 * ->xtra_write False for NX windows
698 * ->notify_early NA for NX windows
699 * ->rsvd_txbuf_count NA for Rx windows
700 * ->lpid, ->pid, ->tid NA for Rx windows
701 */
702
703 memset(winctx, 0, sizeof(struct vas_winctx));
704
705 winctx->rx_fifo = rxattr->rx_fifo;
706 winctx->rx_fifo_size = rxattr->rx_fifo_size;
707 winctx->wcreds_max = rxwin->vas_win.wcreds_max;
708 winctx->pin_win = rxattr->pin_win;
709
710 winctx->nx_win = rxattr->nx_win;
711 winctx->fault_win = rxattr->fault_win;
712 winctx->user_win = rxattr->user_win;
713 winctx->rej_no_credit = rxattr->rej_no_credit;
714 winctx->rx_word_mode = rxattr->rx_win_ord_mode;
715 winctx->tx_word_mode = rxattr->tx_win_ord_mode;
716 winctx->rx_wcred_mode = rxattr->rx_wcred_mode;
717 winctx->tx_wcred_mode = rxattr->tx_wcred_mode;
718 winctx->notify_early = rxattr->notify_early;
719
720 if (winctx->nx_win) {
721 winctx->data_stamp = true;
722 winctx->intr_disable = true;
723 winctx->pin_win = true;
724
725 WARN_ON_ONCE(winctx->fault_win);
726 WARN_ON_ONCE(!winctx->rx_word_mode);
727 WARN_ON_ONCE(!winctx->tx_word_mode);
728 WARN_ON_ONCE(winctx->notify_after_count);
729 } else if (winctx->fault_win) {
730 winctx->notify_disable = true;
731 } else if (winctx->user_win) {
732 /*
733 * Section 1.8.1 Low Latency Core-Core Wake up of
734 * the VAS workbook:
735 *
736 * - disable credit checks ([tr]x_wcred_mode = false)
737 * - disable FIFO writes
738 * - enable ASB_Notify, disable interrupt
739 */
740 winctx->fifo_disable = true;
741 winctx->intr_disable = true;
742 winctx->rx_fifo = 0;
743 }
744
745 winctx->lnotify_lpid = rxattr->lnotify_lpid;
746 winctx->lnotify_pid = rxattr->lnotify_pid;
747 winctx->lnotify_tid = rxattr->lnotify_tid;
748 winctx->pswid = rxattr->pswid;
749 winctx->dma_type = VAS_DMA_TYPE_INJECT;
750 winctx->tc_mode = rxattr->tc_mode;
751
752 winctx->min_scope = VAS_SCOPE_LOCAL;
753 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
754 if (rxwin->vinst->virq)
755 winctx->irq_port = rxwin->vinst->irq_port;
756}
757
758static bool rx_win_args_valid(enum vas_cop_type cop,
759 struct vas_rx_win_attr *attr)
760{
761 pr_debug("Rxattr: fault %d, notify %d, intr %d, early %d, fifo %d\n",
762 attr->fault_win, attr->notify_disable,
763 attr->intr_disable, attr->notify_early,
764 attr->rx_fifo_size);
765
766 if (cop >= VAS_COP_TYPE_MAX)
767 return false;
768
769 if (cop != VAS_COP_TYPE_FTW &&
770 attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN)
771 return false;
772
773 if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX)
774 return false;
775
776 if (!attr->wcreds_max)
777 return false;
778
779 if (attr->nx_win) {
780 /* cannot be fault or user window if it is nx */
781 if (attr->fault_win || attr->user_win)
782 return false;
783 /*
784 * Section 3.1.4.32: NX Windows must not disable notification,
785 * and must not enable interrupts or early notification.
786 */
787 if (attr->notify_disable || !attr->intr_disable ||
788 attr->notify_early)
789 return false;
790 } else if (attr->fault_win) {
791 /* cannot be both fault and user window */
792 if (attr->user_win)
793 return false;
794
795 /*
796 * Section 3.1.4.32: Fault windows must disable notification
797 * but not interrupts.
798 */
799 if (!attr->notify_disable || attr->intr_disable)
800 return false;
801
802 } else if (attr->user_win) {
803 /*
804 * User receive windows are only for fast-thread-wakeup
805 * (FTW). They don't need a FIFO and must disable interrupts
806 */
807 if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable)
808 return false;
809 } else {
810 /* Rx window must be one of NX or Fault or User window. */
811 return false;
812 }
813
814 return true;
815}
816
817void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop)
818{
819 memset(rxattr, 0, sizeof(*rxattr));
820
821 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
822 cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
823 rxattr->pin_win = true;
824 rxattr->nx_win = true;
825 rxattr->fault_win = false;
826 rxattr->intr_disable = true;
827 rxattr->rx_wcred_mode = true;
828 rxattr->tx_wcred_mode = true;
829 rxattr->rx_win_ord_mode = true;
830 rxattr->tx_win_ord_mode = true;
831 } else if (cop == VAS_COP_TYPE_FAULT) {
832 rxattr->pin_win = true;
833 rxattr->fault_win = true;
834 rxattr->notify_disable = true;
835 rxattr->rx_wcred_mode = true;
836 rxattr->rx_win_ord_mode = true;
837 rxattr->rej_no_credit = true;
838 rxattr->tc_mode = VAS_THRESH_DISABLED;
839 } else if (cop == VAS_COP_TYPE_FTW) {
840 rxattr->user_win = true;
841 rxattr->intr_disable = true;
842
843 /*
844 * As noted in the VAS Workbook we disable credit checks.
845 * If we enable credit checks in the future, we must also
846 * implement a mechanism to return the user credits or new
847 * paste operations will fail.
848 */
849 }
850}
851EXPORT_SYMBOL_GPL(vas_init_rx_win_attr);
852
853struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop,
854 struct vas_rx_win_attr *rxattr)
855{
856 struct pnv_vas_window *rxwin;
857 struct vas_winctx winctx;
858 struct vas_instance *vinst;
859
860 trace_vas_rx_win_open(current, vasid, cop: cop, rxattr);
861
862 if (!rx_win_args_valid(cop: cop, attr: rxattr))
863 return ERR_PTR(error: -EINVAL);
864
865 vinst = find_vas_instance(vasid);
866 if (!vinst) {
867 pr_devel("vasid %d not found!\n", vasid);
868 return ERR_PTR(error: -EINVAL);
869 }
870 pr_devel("Found instance %d\n", vasid);
871
872 rxwin = vas_window_alloc(vinst);
873 if (IS_ERR(ptr: rxwin)) {
874 pr_devel("Unable to allocate memory for Rx window\n");
875 return (struct vas_window *)rxwin;
876 }
877
878 rxwin->tx_win = false;
879 rxwin->nx_win = rxattr->nx_win;
880 rxwin->user_win = rxattr->user_win;
881 rxwin->vas_win.cop = cop;
882 rxwin->vas_win.wcreds_max = rxattr->wcreds_max;
883
884 init_winctx_for_rxwin(rxwin, rxattr, winctx: &winctx);
885 init_winctx_regs(window: rxwin, winctx: &winctx);
886
887 set_vinst_win(vinst, window: rxwin);
888
889 return &rxwin->vas_win;
890}
891EXPORT_SYMBOL_GPL(vas_rx_win_open);
892
893void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop)
894{
895 memset(txattr, 0, sizeof(*txattr));
896
897 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
898 cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
899 txattr->rej_no_credit = false;
900 txattr->rx_wcred_mode = true;
901 txattr->tx_wcred_mode = true;
902 txattr->rx_win_ord_mode = true;
903 txattr->tx_win_ord_mode = true;
904 } else if (cop == VAS_COP_TYPE_FTW) {
905 txattr->user_win = true;
906 }
907}
908EXPORT_SYMBOL_GPL(vas_init_tx_win_attr);
909
910static void init_winctx_for_txwin(struct pnv_vas_window *txwin,
911 struct vas_tx_win_attr *txattr,
912 struct vas_winctx *winctx)
913{
914 /*
915 * We first zero all fields and only set non-zero ones. Following
916 * are some fields set to 0/false for the stated reason:
917 *
918 * ->notify_os_intr_reg In powernv, send intrs to HV
919 * ->rsvd_txbuf_count Not supported yet.
920 * ->notify_disable False for NX windows
921 * ->xtra_write False for NX windows
922 * ->notify_early NA for NX windows
923 * ->lnotify_lpid NA for Tx windows
924 * ->lnotify_pid NA for Tx windows
925 * ->lnotify_tid NA for Tx windows
926 * ->tx_win_cred_mode Ignore for now for NX windows
927 * ->rx_win_cred_mode Ignore for now for NX windows
928 */
929 memset(winctx, 0, sizeof(struct vas_winctx));
930
931 winctx->wcreds_max = txwin->vas_win.wcreds_max;
932
933 winctx->user_win = txattr->user_win;
934 winctx->nx_win = txwin->rxwin->nx_win;
935 winctx->pin_win = txattr->pin_win;
936 winctx->rej_no_credit = txattr->rej_no_credit;
937 winctx->rsvd_txbuf_enable = txattr->rsvd_txbuf_enable;
938
939 winctx->rx_wcred_mode = txattr->rx_wcred_mode;
940 winctx->tx_wcred_mode = txattr->tx_wcred_mode;
941 winctx->rx_word_mode = txattr->rx_win_ord_mode;
942 winctx->tx_word_mode = txattr->tx_win_ord_mode;
943 winctx->rsvd_txbuf_count = txattr->rsvd_txbuf_count;
944
945 winctx->intr_disable = true;
946 if (winctx->nx_win)
947 winctx->data_stamp = true;
948
949 winctx->lpid = txattr->lpid;
950 winctx->pidr = txattr->pidr;
951 winctx->rx_win_id = txwin->rxwin->vas_win.winid;
952 /*
953 * IRQ and fault window setup is successful. Set fault window
954 * for the send window so that ready to handle faults.
955 */
956 if (txwin->vinst->virq)
957 winctx->fault_win_id = txwin->vinst->fault_win->vas_win.winid;
958
959 winctx->dma_type = VAS_DMA_TYPE_INJECT;
960 winctx->tc_mode = txattr->tc_mode;
961 winctx->min_scope = VAS_SCOPE_LOCAL;
962 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
963 if (txwin->vinst->virq)
964 winctx->irq_port = txwin->vinst->irq_port;
965
966 winctx->pswid = txattr->pswid ? txattr->pswid :
967 encode_pswid(vasid: txwin->vinst->vas_id,
968 winid: txwin->vas_win.winid);
969}
970
971static bool tx_win_args_valid(enum vas_cop_type cop,
972 struct vas_tx_win_attr *attr)
973{
974 if (attr->tc_mode != VAS_THRESH_DISABLED)
975 return false;
976
977 if (cop > VAS_COP_TYPE_MAX)
978 return false;
979
980 if (attr->wcreds_max > VAS_TX_WCREDS_MAX)
981 return false;
982
983 if (attr->user_win) {
984 if (attr->rsvd_txbuf_count)
985 return false;
986
987 if (cop != VAS_COP_TYPE_FTW && cop != VAS_COP_TYPE_GZIP &&
988 cop != VAS_COP_TYPE_GZIP_HIPRI)
989 return false;
990 }
991
992 return true;
993}
994
995struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
996 struct vas_tx_win_attr *attr)
997{
998 int rc;
999 struct pnv_vas_window *txwin;
1000 struct pnv_vas_window *rxwin;
1001 struct vas_winctx winctx;
1002 struct vas_instance *vinst;
1003
1004 trace_vas_tx_win_open(current, vasid, cop: cop, txattr: attr);
1005
1006 if (!tx_win_args_valid(cop: cop, attr))
1007 return ERR_PTR(error: -EINVAL);
1008
1009 /*
1010 * If caller did not specify a vasid but specified the PSWID of a
1011 * receive window (applicable only to FTW windows), use the vasid
1012 * from that receive window.
1013 */
1014 if (vasid == -1 && attr->pswid)
1015 decode_pswid(pswid: attr->pswid, vasid: &vasid, NULL);
1016
1017 vinst = find_vas_instance(vasid);
1018 if (!vinst) {
1019 pr_devel("vasid %d not found!\n", vasid);
1020 return ERR_PTR(error: -EINVAL);
1021 }
1022
1023 rxwin = get_vinst_rxwin(vinst, cop: cop, pswid: attr->pswid);
1024 if (IS_ERR(ptr: rxwin)) {
1025 pr_devel("No RxWin for vasid %d, cop %d\n", vasid, cop);
1026 return (struct vas_window *)rxwin;
1027 }
1028
1029 txwin = vas_window_alloc(vinst);
1030 if (IS_ERR(ptr: txwin)) {
1031 rc = PTR_ERR(ptr: txwin);
1032 goto put_rxwin;
1033 }
1034
1035 txwin->vas_win.cop = cop;
1036 txwin->tx_win = 1;
1037 txwin->rxwin = rxwin;
1038 txwin->nx_win = txwin->rxwin->nx_win;
1039 txwin->user_win = attr->user_win;
1040 txwin->vas_win.wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT;
1041
1042 init_winctx_for_txwin(txwin, txattr: attr, winctx: &winctx);
1043
1044 init_winctx_regs(window: txwin, winctx: &winctx);
1045
1046 /*
1047 * If its a kernel send window, map the window address into the
1048 * kernel's address space. For user windows, user must issue an
1049 * mmap() to map the window into their address space.
1050 *
1051 * NOTE: If kernel ever resubmits a user CRB after handling a page
1052 * fault, we will need to map this into kernel as well.
1053 */
1054 if (!txwin->user_win) {
1055 txwin->paste_kaddr = map_paste_region(txwin);
1056 if (IS_ERR(ptr: txwin->paste_kaddr)) {
1057 rc = PTR_ERR(ptr: txwin->paste_kaddr);
1058 goto free_window;
1059 }
1060 } else {
1061 /*
1062 * Interrupt handler or fault window setup failed. Means
1063 * NX can not generate fault for page fault. So not
1064 * opening for user space tx window.
1065 */
1066 if (!vinst->virq) {
1067 rc = -ENODEV;
1068 goto free_window;
1069 }
1070 rc = get_vas_user_win_ref(&txwin->vas_win.task_ref);
1071 if (rc)
1072 goto free_window;
1073
1074 vas_user_win_add_mm_context(&txwin->vas_win.task_ref);
1075 }
1076
1077 set_vinst_win(vinst, window: txwin);
1078
1079 return &txwin->vas_win;
1080
1081free_window:
1082 vas_window_free(window: txwin);
1083
1084put_rxwin:
1085 put_rx_win(rxwin);
1086 return ERR_PTR(error: rc);
1087
1088}
1089EXPORT_SYMBOL_GPL(vas_tx_win_open);
1090
1091int vas_copy_crb(void *crb, int offset)
1092{
1093 return vas_copy(crb, offset);
1094}
1095EXPORT_SYMBOL_GPL(vas_copy_crb);
1096
1097#define RMA_LSMP_REPORT_ENABLE PPC_BIT(53)
1098int vas_paste_crb(struct vas_window *vwin, int offset, bool re)
1099{
1100 struct pnv_vas_window *txwin;
1101 int rc;
1102 void *addr;
1103 uint64_t val;
1104
1105 txwin = container_of(vwin, struct pnv_vas_window, vas_win);
1106 trace_vas_paste_crb(current, win: txwin);
1107
1108 /*
1109 * Only NX windows are supported for now and hardware assumes
1110 * report-enable flag is set for NX windows. Ensure software
1111 * complies too.
1112 */
1113 WARN_ON_ONCE(txwin->nx_win && !re);
1114
1115 addr = txwin->paste_kaddr;
1116 if (re) {
1117 /*
1118 * Set the REPORT_ENABLE bit (equivalent to writing
1119 * to 1K offset of the paste address)
1120 */
1121 val = SET_FIELD(RMA_LSMP_REPORT_ENABLE, 0ULL, 1);
1122 addr += val;
1123 }
1124
1125 /*
1126 * Map the raw CR value from vas_paste() to an error code (there
1127 * is just pass or fail for now though).
1128 */
1129 rc = vas_paste(paste_address: addr, offset);
1130 if (rc == 2)
1131 rc = 0;
1132 else
1133 rc = -EINVAL;
1134
1135 pr_debug("Txwin #%d: Msg count %llu\n", txwin->vas_win.winid,
1136 read_hvwc_reg(txwin, VREG(LRFIFO_PUSH)));
1137
1138 return rc;
1139}
1140EXPORT_SYMBOL_GPL(vas_paste_crb);
1141
1142/*
1143 * If credit checking is enabled for this window, poll for the return
1144 * of window credits (i.e for NX engines to process any outstanding CRBs).
1145 * Since NX-842 waits for the CRBs to be processed before closing the
1146 * window, we should not have to wait for too long.
1147 *
1148 * TODO: We retry in 10ms intervals now. We could/should probably peek at
1149 * the VAS_LRFIFO_PUSH_OFFSET register to get an estimate of pending
1150 * CRBs on the FIFO and compute the delay dynamically on each retry.
1151 * But that is not really needed until we support NX-GZIP access from
1152 * user space. (NX-842 driver waits for CSB and Fast thread-wakeup
1153 * doesn't use credit checking).
1154 */
1155static void poll_window_credits(struct pnv_vas_window *window)
1156{
1157 u64 val;
1158 int creds, mode;
1159 int count = 0;
1160
1161 val = read_hvwc_reg(win: window, VREG(WINCTL));
1162 if (window->tx_win)
1163 mode = GET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val);
1164 else
1165 mode = GET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val);
1166
1167 if (!mode)
1168 return;
1169retry:
1170 if (window->tx_win) {
1171 val = read_hvwc_reg(win: window, VREG(TX_WCRED));
1172 creds = GET_FIELD(VAS_TX_WCRED, val);
1173 } else {
1174 val = read_hvwc_reg(win: window, VREG(LRX_WCRED));
1175 creds = GET_FIELD(VAS_LRX_WCRED, val);
1176 }
1177
1178 /*
1179 * Takes around few milliseconds to complete all pending requests
1180 * and return credits.
1181 * TODO: Scan fault FIFO and invalidate CRBs points to this window
1182 * and issue CRB Kill to stop all pending requests. Need only
1183 * if there is a bug in NX or fault handling in kernel.
1184 */
1185 if (creds < window->vas_win.wcreds_max) {
1186 val = 0;
1187 set_current_state(TASK_UNINTERRUPTIBLE);
1188 schedule_timeout(timeout: msecs_to_jiffies(m: 10));
1189 count++;
1190 /*
1191 * Process can not close send window until all credits are
1192 * returned.
1193 */
1194 if (!(count % 1000))
1195 pr_warn_ratelimited("VAS: pid %d stuck. Waiting for credits returned for Window(%d). creds %d, Retries %d\n",
1196 vas_window_pid(&window->vas_win),
1197 window->vas_win.winid,
1198 creds, count);
1199
1200 goto retry;
1201 }
1202}
1203
1204/*
1205 * Wait for the window to go to "not-busy" state. It should only take a
1206 * short time to queue a CRB, so window should not be busy for too long.
1207 * Trying 5ms intervals.
1208 */
1209static void poll_window_busy_state(struct pnv_vas_window *window)
1210{
1211 int busy;
1212 u64 val;
1213 int count = 0;
1214
1215retry:
1216 val = read_hvwc_reg(win: window, VREG(WIN_STATUS));
1217 busy = GET_FIELD(VAS_WIN_BUSY, val);
1218 if (busy) {
1219 val = 0;
1220 set_current_state(TASK_UNINTERRUPTIBLE);
1221 schedule_timeout(timeout: msecs_to_jiffies(m: 10));
1222 count++;
1223 /*
1224 * Takes around few milliseconds to process all pending
1225 * requests.
1226 */
1227 if (!(count % 1000))
1228 pr_warn_ratelimited("VAS: pid %d stuck. Window (ID=%d) is in busy state. Retries %d\n",
1229 vas_window_pid(&window->vas_win),
1230 window->vas_win.winid, count);
1231
1232 goto retry;
1233 }
1234}
1235
1236/*
1237 * Have the hardware cast a window out of cache and wait for it to
1238 * be completed.
1239 *
1240 * NOTE: It can take a relatively long time to cast the window context
1241 * out of the cache. It is not strictly necessary to cast out if:
1242 *
1243 * - we clear the "Pin Window" bit (so hardware is free to evict)
1244 *
1245 * - we re-initialize the window context when it is reassigned.
1246 *
1247 * We do the former in vas_win_close() and latter in vas_win_open().
1248 * So, ignoring the cast-out for now. We can add it as needed. If
1249 * casting out becomes necessary we should consider offloading the
1250 * job to a worker thread, so the window close can proceed quickly.
1251 */
1252static void poll_window_castout(struct pnv_vas_window *window)
1253{
1254 /* stub for now */
1255}
1256
1257/*
1258 * Unpin and close a window so no new requests are accepted and the
1259 * hardware can evict this window from cache if necessary.
1260 */
1261static void unpin_close_window(struct pnv_vas_window *window)
1262{
1263 u64 val;
1264
1265 val = read_hvwc_reg(win: window, VREG(WINCTL));
1266 val = SET_FIELD(VAS_WINCTL_PIN, val, 0);
1267 val = SET_FIELD(VAS_WINCTL_OPEN, val, 0);
1268 write_hvwc_reg(win: window, VREG(WINCTL), val);
1269}
1270
1271/*
1272 * Close a window.
1273 *
1274 * See Section 1.12.1 of VAS workbook v1.05 for details on closing window:
1275 * - Disable new paste operations (unmap paste address)
1276 * - Poll for the "Window Busy" bit to be cleared
1277 * - Clear the Open/Enable bit for the Window.
1278 * - Poll for return of window Credits (implies FIFO empty for Rx win?)
1279 * - Unpin and cast window context out of cache
1280 *
1281 * Besides the hardware, kernel has some bookkeeping of course.
1282 */
1283int vas_win_close(struct vas_window *vwin)
1284{
1285 struct pnv_vas_window *window;
1286
1287 if (!vwin)
1288 return 0;
1289
1290 window = container_of(vwin, struct pnv_vas_window, vas_win);
1291
1292 if (!window->tx_win && atomic_read(v: &window->num_txwins) != 0) {
1293 pr_devel("Attempting to close an active Rx window!\n");
1294 WARN_ON_ONCE(1);
1295 return -EBUSY;
1296 }
1297
1298 unmap_paste_region(window);
1299
1300 poll_window_busy_state(window);
1301
1302 unpin_close_window(window);
1303
1304 poll_window_credits(window);
1305
1306 clear_vinst_win(window);
1307
1308 poll_window_castout(window);
1309
1310 /* if send window, drop reference to matching receive window */
1311 if (window->tx_win) {
1312 if (window->user_win) {
1313 mm_context_remove_vas_window(vwin->task_ref.mm);
1314 put_vas_user_win_ref(&vwin->task_ref);
1315 }
1316 put_rx_win(rxwin: window->rxwin);
1317 }
1318
1319 vas_window_free(window);
1320
1321 return 0;
1322}
1323EXPORT_SYMBOL_GPL(vas_win_close);
1324
1325/*
1326 * Return credit for the given window.
1327 * Send windows and fault window uses credit mechanism as follows:
1328 *
1329 * Send windows:
1330 * - The default number of credits available for each send window is
1331 * 1024. It means 1024 requests can be issued asynchronously at the
1332 * same time. If the credit is not available, that request will be
1333 * returned with RMA_Busy.
1334 * - One credit is taken when NX request is issued.
1335 * - This credit is returned after NX processed that request.
1336 * - If NX encounters translation error, kernel will return the
1337 * credit on the specific send window after processing the fault CRB.
1338 *
1339 * Fault window:
1340 * - The total number credits available is FIFO_SIZE/CRB_SIZE.
1341 * Means 4MB/128 in the current implementation. If credit is not
1342 * available, RMA_Reject is returned.
1343 * - A credit is taken when NX pastes CRB in fault FIFO.
1344 * - The kernel with return credit on fault window after reading entry
1345 * from fault FIFO.
1346 */
1347void vas_return_credit(struct pnv_vas_window *window, bool tx)
1348{
1349 uint64_t val;
1350
1351 val = 0ULL;
1352 if (tx) { /* send window */
1353 val = SET_FIELD(VAS_TX_WCRED, val, 1);
1354 write_hvwc_reg(win: window, VREG(TX_WCRED_ADDER), val);
1355 } else {
1356 val = SET_FIELD(VAS_LRX_WCRED, val, 1);
1357 write_hvwc_reg(win: window, VREG(LRX_WCRED_ADDER), val);
1358 }
1359}
1360
1361struct pnv_vas_window *vas_pswid_to_window(struct vas_instance *vinst,
1362 uint32_t pswid)
1363{
1364 struct pnv_vas_window *window;
1365 int winid;
1366
1367 if (!pswid) {
1368 pr_devel("%s: called for pswid 0!\n", __func__);
1369 return ERR_PTR(error: -ESRCH);
1370 }
1371
1372 decode_pswid(pswid, NULL, winid: &winid);
1373
1374 if (winid >= VAS_WINDOWS_PER_CHIP)
1375 return ERR_PTR(error: -ESRCH);
1376
1377 /*
1378 * If application closes the window before the hardware
1379 * returns the fault CRB, we should wait in vas_win_close()
1380 * for the pending requests. so the window must be active
1381 * and the process alive.
1382 *
1383 * If its a kernel process, we should not get any faults and
1384 * should not get here.
1385 */
1386 window = vinst->windows[winid];
1387
1388 if (!window) {
1389 pr_err("PSWID decode: Could not find window for winid %d pswid %d vinst 0x%p\n",
1390 winid, pswid, vinst);
1391 return NULL;
1392 }
1393
1394 /*
1395 * Do some sanity checks on the decoded window. Window should be
1396 * NX GZIP user send window. FTW windows should not incur faults
1397 * since their CRBs are ignored (not queued on FIFO or processed
1398 * by NX).
1399 */
1400 if (!window->tx_win || !window->user_win || !window->nx_win ||
1401 window->vas_win.cop == VAS_COP_TYPE_FAULT ||
1402 window->vas_win.cop == VAS_COP_TYPE_FTW) {
1403 pr_err("PSWID decode: id %d, tx %d, user %d, nx %d, cop %d\n",
1404 winid, window->tx_win, window->user_win,
1405 window->nx_win, window->vas_win.cop);
1406 WARN_ON(1);
1407 }
1408
1409 return window;
1410}
1411
1412static struct vas_window *vas_user_win_open(int vas_id, u64 flags,
1413 enum vas_cop_type cop_type)
1414{
1415 struct vas_tx_win_attr txattr = {};
1416
1417 vas_init_tx_win_attr(txattr: &txattr, cop: cop_type);
1418
1419 txattr.lpid = mfspr(SPRN_LPID);
1420 txattr.pidr = mfspr(SPRN_PID);
1421 txattr.user_win = true;
1422 txattr.rsvd_txbuf_count = false;
1423 txattr.pswid = false;
1424
1425 pr_devel("Pid %d: Opening txwin, PIDR %ld\n", txattr.pidr,
1426 mfspr(SPRN_PID));
1427
1428 return vas_tx_win_open(vasid: vas_id, cop: cop_type, attr: &txattr);
1429}
1430
1431static u64 vas_user_win_paste_addr(struct vas_window *txwin)
1432{
1433 struct pnv_vas_window *win;
1434 u64 paste_addr;
1435
1436 win = container_of(txwin, struct pnv_vas_window, vas_win);
1437 vas_win_paste_addr(window: win, addr: &paste_addr, NULL);
1438
1439 return paste_addr;
1440}
1441
1442static int vas_user_win_close(struct vas_window *txwin)
1443{
1444 vas_win_close(vwin: txwin);
1445
1446 return 0;
1447}
1448
1449static const struct vas_user_win_ops vops = {
1450 .open_win = vas_user_win_open,
1451 .paste_addr = vas_user_win_paste_addr,
1452 .close_win = vas_user_win_close,
1453};
1454
1455/*
1456 * Supporting only nx-gzip coprocessor type now, but this API code
1457 * extended to other coprocessor types later.
1458 */
1459int vas_register_api_powernv(struct module *mod, enum vas_cop_type cop_type,
1460 const char *name)
1461{
1462
1463 return vas_register_coproc_api(mod, cop_type, name, &vops);
1464}
1465EXPORT_SYMBOL_GPL(vas_register_api_powernv);
1466
1467void vas_unregister_api_powernv(void)
1468{
1469 vas_unregister_coproc_api();
1470}
1471EXPORT_SYMBOL_GPL(vas_unregister_api_powernv);
1472

source code of linux/arch/powerpc/platforms/powernv/vas-window.c