1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * IBM Accelerator Family 'GenWQE' |
4 | * |
5 | * (C) Copyright IBM Corp. 2013 |
6 | * |
7 | * Author: Frank Haverkamp <haver@linux.vnet.ibm.com> |
8 | * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com> |
9 | * Author: Michael Jung <mijung@gmx.net> |
10 | * Author: Michael Ruettger <michael@ibmra.de> |
11 | */ |
12 | |
13 | /* |
14 | * Device Driver Control Block (DDCB) queue support. Definition of |
15 | * interrupt handlers for queue support as well as triggering the |
16 | * health monitor code in case of problems. The current hardware uses |
17 | * an MSI interrupt which is shared between error handling and |
18 | * functional code. |
19 | */ |
20 | |
21 | #include <linux/types.h> |
22 | #include <linux/sched.h> |
23 | #include <linux/wait.h> |
24 | #include <linux/pci.h> |
25 | #include <linux/string.h> |
26 | #include <linux/dma-mapping.h> |
27 | #include <linux/delay.h> |
28 | #include <linux/module.h> |
29 | #include <linux/interrupt.h> |
30 | #include <linux/crc-itu-t.h> |
31 | |
32 | #include "card_base.h" |
33 | #include "card_ddcb.h" |
34 | |
35 | /* |
36 | * N: next DDCB, this is where the next DDCB will be put. |
37 | * A: active DDCB, this is where the code will look for the next completion. |
38 | * x: DDCB is enqueued, we are waiting for its completion. |
39 | |
40 | * Situation (1): Empty queue |
41 | * +---+---+---+---+---+---+---+---+ |
42 | * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | |
43 | * | | | | | | | | | |
44 | * +---+---+---+---+---+---+---+---+ |
45 | * A/N |
46 | * enqueued_ddcbs = A - N = 2 - 2 = 0 |
47 | * |
48 | * Situation (2): Wrapped, N > A |
49 | * +---+---+---+---+---+---+---+---+ |
50 | * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | |
51 | * | | | x | x | | | | | |
52 | * +---+---+---+---+---+---+---+---+ |
53 | * A N |
54 | * enqueued_ddcbs = N - A = 4 - 2 = 2 |
55 | * |
56 | * Situation (3): Queue wrapped, A > N |
57 | * +---+---+---+---+---+---+---+---+ |
58 | * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | |
59 | * | x | x | | | x | x | x | x | |
60 | * +---+---+---+---+---+---+---+---+ |
61 | * N A |
62 | * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 2) = 6 |
63 | * |
64 | * Situation (4a): Queue full N > A |
65 | * +---+---+---+---+---+---+---+---+ |
66 | * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | |
67 | * | x | x | x | x | x | x | x | | |
68 | * +---+---+---+---+---+---+---+---+ |
69 | * A N |
70 | * |
71 | * enqueued_ddcbs = N - A = 7 - 0 = 7 |
72 | * |
73 | * Situation (4a): Queue full A > N |
74 | * +---+---+---+---+---+---+---+---+ |
75 | * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | |
76 | * | x | x | x | | x | x | x | x | |
77 | * +---+---+---+---+---+---+---+---+ |
78 | * N A |
79 | * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7 |
80 | */ |
81 | |
82 | static int queue_empty(struct ddcb_queue *queue) |
83 | { |
84 | return queue->ddcb_next == queue->ddcb_act; |
85 | } |
86 | |
87 | static int queue_enqueued_ddcbs(struct ddcb_queue *queue) |
88 | { |
89 | if (queue->ddcb_next >= queue->ddcb_act) |
90 | return queue->ddcb_next - queue->ddcb_act; |
91 | |
92 | return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next); |
93 | } |
94 | |
95 | static int queue_free_ddcbs(struct ddcb_queue *queue) |
96 | { |
97 | int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1; |
98 | |
99 | if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */ |
100 | return 0; |
101 | } |
102 | return free_ddcbs; |
103 | } |
104 | |
105 | /* |
106 | * Use of the PRIV field in the DDCB for queue debugging: |
107 | * |
108 | * (1) Trying to get rid of a DDCB which saw a timeout: |
109 | * pddcb->priv[6] = 0xcc; # cleared |
110 | * |
111 | * (2) Append a DDCB via NEXT bit: |
112 | * pddcb->priv[7] = 0xaa; # appended |
113 | * |
114 | * (3) DDCB needed tapping: |
115 | * pddcb->priv[7] = 0xbb; # tapped |
116 | * |
117 | * (4) DDCB marked as correctly finished: |
118 | * pddcb->priv[6] = 0xff; # finished |
119 | */ |
120 | |
121 | static inline void ddcb_mark_tapped(struct ddcb *pddcb) |
122 | { |
123 | pddcb->priv[7] = 0xbb; /* tapped */ |
124 | } |
125 | |
126 | static inline void ddcb_mark_appended(struct ddcb *pddcb) |
127 | { |
128 | pddcb->priv[7] = 0xaa; /* appended */ |
129 | } |
130 | |
131 | static inline void ddcb_mark_cleared(struct ddcb *pddcb) |
132 | { |
133 | pddcb->priv[6] = 0xcc; /* cleared */ |
134 | } |
135 | |
136 | static inline void ddcb_mark_finished(struct ddcb *pddcb) |
137 | { |
138 | pddcb->priv[6] = 0xff; /* finished */ |
139 | } |
140 | |
141 | static inline void ddcb_mark_unused(struct ddcb *pddcb) |
142 | { |
143 | pddcb->priv_64 = cpu_to_be64(0); /* not tapped */ |
144 | } |
145 | |
146 | /** |
147 | * genwqe_crc16() - Generate 16-bit crc as required for DDCBs |
148 | * @buff: pointer to data buffer |
149 | * @len: length of data for calculation |
150 | * @init: initial crc (0xffff at start) |
151 | * |
152 | * Polynomial = x^16 + x^12 + x^5 + 1 (0x1021) |
153 | * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff |
154 | * should result in a crc16 of 0x89c3 |
155 | * |
156 | * Return: crc16 checksum in big endian format ! |
157 | */ |
158 | static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init) |
159 | { |
160 | return crc_itu_t(crc: init, buffer: buff, len); |
161 | } |
162 | |
163 | static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue) |
164 | { |
165 | int i; |
166 | struct ddcb *pddcb; |
167 | unsigned long flags; |
168 | struct pci_dev *pci_dev = cd->pci_dev; |
169 | |
170 | spin_lock_irqsave(&cd->print_lock, flags); |
171 | |
172 | dev_info(&pci_dev->dev, |
173 | "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n" , |
174 | cd->card_idx, queue->ddcb_act, queue->ddcb_next); |
175 | |
176 | pddcb = queue->ddcb_vaddr; |
177 | for (i = 0; i < queue->ddcb_max; i++) { |
178 | dev_err(&pci_dev->dev, |
179 | " %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n" , |
180 | i == queue->ddcb_act ? '>' : ' ', |
181 | i, |
182 | be16_to_cpu(pddcb->retc_16), |
183 | be16_to_cpu(pddcb->seqnum_16), |
184 | pddcb->hsi, |
185 | pddcb->shi, |
186 | be64_to_cpu(pddcb->priv_64), |
187 | pddcb->cmd); |
188 | pddcb++; |
189 | } |
190 | spin_unlock_irqrestore(lock: &cd->print_lock, flags); |
191 | } |
192 | |
193 | struct genwqe_ddcb_cmd *ddcb_requ_alloc(void) |
194 | { |
195 | struct ddcb_requ *req; |
196 | |
197 | req = kzalloc(size: sizeof(*req), GFP_KERNEL); |
198 | if (!req) |
199 | return NULL; |
200 | |
201 | return &req->cmd; |
202 | } |
203 | |
204 | void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd) |
205 | { |
206 | struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd); |
207 | |
208 | kfree(objp: req); |
209 | } |
210 | |
211 | static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req) |
212 | { |
213 | return req->req_state; |
214 | } |
215 | |
216 | static inline void ddcb_requ_set_state(struct ddcb_requ *req, |
217 | enum genwqe_requ_state new_state) |
218 | { |
219 | req->req_state = new_state; |
220 | } |
221 | |
222 | static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req) |
223 | { |
224 | return req->cmd.ddata_addr != 0x0; |
225 | } |
226 | |
227 | /** |
228 | * ddcb_requ_finished() - Returns the hardware state of the associated DDCB |
229 | * @cd: pointer to genwqe device descriptor |
230 | * @req: DDCB work request |
231 | * |
232 | * Status of ddcb_requ mirrors this hardware state, but is copied in |
233 | * the ddcb_requ on interrupt/polling function. The lowlevel code |
234 | * should check the hardware state directly, the higher level code |
235 | * should check the copy. |
236 | * |
237 | * This function will also return true if the state of the queue is |
238 | * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the |
239 | * shutdown case. |
240 | */ |
241 | static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req) |
242 | { |
243 | return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) || |
244 | (cd->card_state != GENWQE_CARD_USED); |
245 | } |
246 | |
247 | #define RET_DDCB_APPENDED 1 |
248 | #define RET_DDCB_TAPPED 2 |
249 | /** |
250 | * enqueue_ddcb() - Enqueue a DDCB |
251 | * @cd: pointer to genwqe device descriptor |
252 | * @queue: queue this operation should be done on |
253 | * @pddcb: pointer to ddcb structure |
254 | * @ddcb_no: pointer to ddcb number being tapped |
255 | * |
256 | * Start execution of DDCB by tapping or append to queue via NEXT |
257 | * bit. This is done by an atomic 'compare and swap' instruction and |
258 | * checking SHI and HSI of the previous DDCB. |
259 | * |
260 | * This function must only be called with ddcb_lock held. |
261 | * |
262 | * Return: 1 if new DDCB is appended to previous |
263 | * 2 if DDCB queue is tapped via register/simulation |
264 | */ |
265 | static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue, |
266 | struct ddcb *pddcb, int ddcb_no) |
267 | { |
268 | unsigned int try; |
269 | int prev_no; |
270 | struct ddcb *prev_ddcb; |
271 | __be32 old, new, icrc_hsi_shi; |
272 | u64 num; |
273 | |
274 | /* |
275 | * For performance checks a Dispatch Timestamp can be put into |
276 | * DDCB It is supposed to use the SLU's free running counter, |
277 | * but this requires PCIe cycles. |
278 | */ |
279 | ddcb_mark_unused(pddcb); |
280 | |
281 | /* check previous DDCB if already fetched */ |
282 | prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1; |
283 | prev_ddcb = &queue->ddcb_vaddr[prev_no]; |
284 | |
285 | /* |
286 | * It might have happened that the HSI.FETCHED bit is |
287 | * set. Retry in this case. Therefore I expect maximum 2 times |
288 | * trying. |
289 | */ |
290 | ddcb_mark_appended(pddcb); |
291 | for (try = 0; try < 2; try++) { |
292 | old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */ |
293 | |
294 | /* try to append via NEXT bit if prev DDCB is not completed */ |
295 | if ((old & DDCB_COMPLETED_BE32) != 0x00000000) |
296 | break; |
297 | |
298 | new = (old | DDCB_NEXT_BE32); |
299 | |
300 | wmb(); /* need to ensure write ordering */ |
301 | icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new); |
302 | |
303 | if (icrc_hsi_shi == old) |
304 | return RET_DDCB_APPENDED; /* appended to queue */ |
305 | } |
306 | |
307 | /* Queue must be re-started by updating QUEUE_OFFSET */ |
308 | ddcb_mark_tapped(pddcb); |
309 | num = (u64)ddcb_no << 8; |
310 | |
311 | wmb(); /* need to ensure write ordering */ |
312 | __genwqe_writeq(cd, byte_offs: queue->IO_QUEUE_OFFSET, val: num); /* start queue */ |
313 | |
314 | return RET_DDCB_TAPPED; |
315 | } |
316 | |
317 | /** |
318 | * copy_ddcb_results() - Copy output state from real DDCB to request |
319 | * @req: pointer to requested DDCB parameters |
320 | * @ddcb_no: pointer to ddcb number being tapped |
321 | * |
322 | * Copy DDCB ASV to request struct. There is no endian |
323 | * conversion made, since data structure in ASV is still |
324 | * unknown here. |
325 | * |
326 | * This is needed by: |
327 | * - genwqe_purge_ddcb() |
328 | * - genwqe_check_ddcb_queue() |
329 | */ |
330 | static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no) |
331 | { |
332 | struct ddcb_queue *queue = req->queue; |
333 | struct ddcb *pddcb = &queue->ddcb_vaddr[req->num]; |
334 | |
335 | memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH); |
336 | |
337 | /* copy status flags of the variant part */ |
338 | req->cmd.vcrc = be16_to_cpu(pddcb->vcrc_16); |
339 | req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64); |
340 | req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64); |
341 | |
342 | req->cmd.attn = be16_to_cpu(pddcb->attn_16); |
343 | req->cmd.progress = be32_to_cpu(pddcb->progress_32); |
344 | req->cmd.retc = be16_to_cpu(pddcb->retc_16); |
345 | |
346 | if (ddcb_requ_collect_debug_data(req)) { |
347 | int prev_no = (ddcb_no == 0) ? |
348 | queue->ddcb_max - 1 : ddcb_no - 1; |
349 | struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no]; |
350 | |
351 | memcpy(&req->debug_data.ddcb_finished, pddcb, |
352 | sizeof(req->debug_data.ddcb_finished)); |
353 | memcpy(&req->debug_data.ddcb_prev, prev_pddcb, |
354 | sizeof(req->debug_data.ddcb_prev)); |
355 | } |
356 | } |
357 | |
358 | /** |
359 | * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work requests. |
360 | * @cd: pointer to genwqe device descriptor |
361 | * @queue: queue to be checked |
362 | * |
363 | * Return: Number of DDCBs which were finished |
364 | */ |
365 | static int genwqe_check_ddcb_queue(struct genwqe_dev *cd, |
366 | struct ddcb_queue *queue) |
367 | { |
368 | unsigned long flags; |
369 | int ddcbs_finished = 0; |
370 | struct pci_dev *pci_dev = cd->pci_dev; |
371 | |
372 | spin_lock_irqsave(&queue->ddcb_lock, flags); |
373 | |
374 | /* FIXME avoid soft locking CPU */ |
375 | while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) { |
376 | |
377 | struct ddcb *pddcb; |
378 | struct ddcb_requ *req; |
379 | u16 vcrc, vcrc_16, retc_16; |
380 | |
381 | pddcb = &queue->ddcb_vaddr[queue->ddcb_act]; |
382 | |
383 | if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == |
384 | 0x00000000) |
385 | goto go_home; /* not completed, continue waiting */ |
386 | |
387 | wmb(); /* Add sync to decouple prev. read operations */ |
388 | |
389 | /* Note: DDCB could be purged */ |
390 | req = queue->ddcb_req[queue->ddcb_act]; |
391 | if (req == NULL) { |
392 | /* this occurs if DDCB is purged, not an error */ |
393 | /* Move active DDCB further; Nothing to do anymore. */ |
394 | goto pick_next_one; |
395 | } |
396 | |
397 | /* |
398 | * HSI=0x44 (fetched and completed), but RETC is |
399 | * 0x101, or even worse 0x000. |
400 | * |
401 | * In case of seeing the queue in inconsistent state |
402 | * we read the errcnts and the queue status to provide |
403 | * a trigger for our PCIe analyzer stop capturing. |
404 | */ |
405 | retc_16 = be16_to_cpu(pddcb->retc_16); |
406 | if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) { |
407 | u64 errcnts, status; |
408 | u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr; |
409 | |
410 | errcnts = __genwqe_readq(cd, byte_offs: queue->IO_QUEUE_ERRCNTS); |
411 | status = __genwqe_readq(cd, byte_offs: queue->IO_QUEUE_STATUS); |
412 | |
413 | dev_err(&pci_dev->dev, |
414 | "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n" , |
415 | __func__, be16_to_cpu(pddcb->seqnum_16), |
416 | pddcb->hsi, retc_16, errcnts, status, |
417 | queue->ddcb_daddr + ddcb_offs); |
418 | } |
419 | |
420 | copy_ddcb_results(req, ddcb_no: queue->ddcb_act); |
421 | queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */ |
422 | |
423 | dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n" , req->num); |
424 | genwqe_hexdump(pci_dev, buff: pddcb, size: sizeof(*pddcb)); |
425 | |
426 | ddcb_mark_finished(pddcb); |
427 | |
428 | /* calculate CRC_16 to see if VCRC is correct */ |
429 | vcrc = genwqe_crc16(buff: pddcb->asv, |
430 | VCRC_LENGTH(req->cmd.asv_length), |
431 | init: 0xffff); |
432 | vcrc_16 = be16_to_cpu(pddcb->vcrc_16); |
433 | if (vcrc != vcrc_16) { |
434 | printk_ratelimited(KERN_ERR |
435 | "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n" , |
436 | GENWQE_DEVNAME, dev_name(&pci_dev->dev), |
437 | pddcb->pre, VCRC_LENGTH(req->cmd.asv_length), |
438 | vcrc, vcrc_16); |
439 | } |
440 | |
441 | ddcb_requ_set_state(req, new_state: GENWQE_REQU_FINISHED); |
442 | queue->ddcbs_completed++; |
443 | queue->ddcbs_in_flight--; |
444 | |
445 | /* wake up process waiting for this DDCB, and |
446 | processes on the busy queue */ |
447 | wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]); |
448 | wake_up_interruptible(&queue->busy_waitq); |
449 | |
450 | pick_next_one: |
451 | queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max; |
452 | ddcbs_finished++; |
453 | } |
454 | |
455 | go_home: |
456 | spin_unlock_irqrestore(lock: &queue->ddcb_lock, flags); |
457 | return ddcbs_finished; |
458 | } |
459 | |
460 | /** |
461 | * __genwqe_wait_ddcb(): Waits until DDCB is completed |
462 | * @cd: pointer to genwqe device descriptor |
463 | * @req: pointer to requsted DDCB parameters |
464 | * |
465 | * The Service Layer will update the RETC in DDCB when processing is |
466 | * pending or done. |
467 | * |
468 | * Return: > 0 remaining jiffies, DDCB completed |
469 | * -ETIMEDOUT when timeout |
470 | * -ERESTARTSYS when ^C |
471 | * -EINVAL when unknown error condition |
472 | * |
473 | * When an error is returned the called needs to ensure that |
474 | * purge_ddcb() is being called to get the &req removed from the |
475 | * queue. |
476 | */ |
477 | int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) |
478 | { |
479 | int rc; |
480 | unsigned int ddcb_no; |
481 | struct ddcb_queue *queue; |
482 | struct pci_dev *pci_dev = cd->pci_dev; |
483 | |
484 | if (req == NULL) |
485 | return -EINVAL; |
486 | |
487 | queue = req->queue; |
488 | if (queue == NULL) |
489 | return -EINVAL; |
490 | |
491 | ddcb_no = req->num; |
492 | if (ddcb_no >= queue->ddcb_max) |
493 | return -EINVAL; |
494 | |
495 | rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no], |
496 | ddcb_requ_finished(cd, req), |
497 | GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ); |
498 | |
499 | /* |
500 | * We need to distinguish 3 cases here: |
501 | * 1. rc == 0 timeout occurred |
502 | * 2. rc == -ERESTARTSYS signal received |
503 | * 3. rc > 0 remaining jiffies condition is true |
504 | */ |
505 | if (rc == 0) { |
506 | struct ddcb_queue *queue = req->queue; |
507 | struct ddcb *pddcb; |
508 | |
509 | /* |
510 | * Timeout may be caused by long task switching time. |
511 | * When timeout happens, check if the request has |
512 | * meanwhile completed. |
513 | */ |
514 | genwqe_check_ddcb_queue(cd, queue: req->queue); |
515 | if (ddcb_requ_finished(cd, req)) |
516 | return rc; |
517 | |
518 | dev_err(&pci_dev->dev, |
519 | "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n" , |
520 | __func__, req->num, rc, ddcb_requ_get_state(req), |
521 | req); |
522 | dev_err(&pci_dev->dev, |
523 | "[%s] IO_QUEUE_STATUS=0x%016llx\n" , __func__, |
524 | __genwqe_readq(cd, queue->IO_QUEUE_STATUS)); |
525 | |
526 | pddcb = &queue->ddcb_vaddr[req->num]; |
527 | genwqe_hexdump(pci_dev, buff: pddcb, size: sizeof(*pddcb)); |
528 | |
529 | print_ddcb_info(cd, queue: req->queue); |
530 | return -ETIMEDOUT; |
531 | |
532 | } else if (rc == -ERESTARTSYS) { |
533 | return rc; |
534 | /* |
535 | * EINTR: Stops the application |
536 | * ERESTARTSYS: Restartable systemcall; called again |
537 | */ |
538 | |
539 | } else if (rc < 0) { |
540 | dev_err(&pci_dev->dev, |
541 | "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n" , |
542 | __func__, req->num, rc, ddcb_requ_get_state(req)); |
543 | return -EINVAL; |
544 | } |
545 | |
546 | /* Severe error occured. Driver is forced to stop operation */ |
547 | if (cd->card_state != GENWQE_CARD_USED) { |
548 | dev_err(&pci_dev->dev, |
549 | "[%s] err: DDCB#%d forced to stop (rc=%d)\n" , |
550 | __func__, req->num, rc); |
551 | return -EIO; |
552 | } |
553 | return rc; |
554 | } |
555 | |
556 | /** |
557 | * get_next_ddcb() - Get next available DDCB |
558 | * @cd: pointer to genwqe device descriptor |
559 | * @queue: DDCB queue |
560 | * @num: internal DDCB number |
561 | * |
562 | * DDCB's content is completely cleared but presets for PRE and |
563 | * SEQNUM. This function must only be called when ddcb_lock is held. |
564 | * |
565 | * Return: NULL if no empty DDCB available otherwise ptr to next DDCB. |
566 | */ |
567 | static struct ddcb *get_next_ddcb(struct genwqe_dev *cd, |
568 | struct ddcb_queue *queue, |
569 | int *num) |
570 | { |
571 | u64 *pu64; |
572 | struct ddcb *pddcb; |
573 | |
574 | if (queue_free_ddcbs(queue) == 0) /* queue is full */ |
575 | return NULL; |
576 | |
577 | /* find new ddcb */ |
578 | pddcb = &queue->ddcb_vaddr[queue->ddcb_next]; |
579 | |
580 | /* if it is not completed, we are not allowed to use it */ |
581 | /* barrier(); */ |
582 | if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000) |
583 | return NULL; |
584 | |
585 | *num = queue->ddcb_next; /* internal DDCB number */ |
586 | queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max; |
587 | |
588 | /* clear important DDCB fields */ |
589 | pu64 = (u64 *)pddcb; |
590 | pu64[0] = 0ULL; /* offs 0x00 (ICRC,HSI,SHI,...) */ |
591 | pu64[1] = 0ULL; /* offs 0x01 (ACFUNC,CMD...) */ |
592 | |
593 | /* destroy previous results in ASV */ |
594 | pu64[0x80/8] = 0ULL; /* offs 0x80 (ASV + 0) */ |
595 | pu64[0x88/8] = 0ULL; /* offs 0x88 (ASV + 0x08) */ |
596 | pu64[0x90/8] = 0ULL; /* offs 0x90 (ASV + 0x10) */ |
597 | pu64[0x98/8] = 0ULL; /* offs 0x98 (ASV + 0x18) */ |
598 | pu64[0xd0/8] = 0ULL; /* offs 0xd0 (RETC,ATTN...) */ |
599 | |
600 | pddcb->pre = DDCB_PRESET_PRE; /* 128 */ |
601 | pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++); |
602 | return pddcb; |
603 | } |
604 | |
605 | /** |
606 | * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue |
607 | * @cd: genwqe device descriptor |
608 | * @req: DDCB request |
609 | * |
610 | * This will fail when the request was already FETCHED. In this case |
611 | * we need to wait until it is finished. Else the DDCB can be |
612 | * reused. This function also ensures that the request data structure |
613 | * is removed from ddcb_req[]. |
614 | * |
615 | * Do not forget to call this function when genwqe_wait_ddcb() fails, |
616 | * such that the request gets really removed from ddcb_req[]. |
617 | * |
618 | * Return: 0 success |
619 | */ |
620 | int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req) |
621 | { |
622 | struct ddcb *pddcb = NULL; |
623 | unsigned int t; |
624 | unsigned long flags; |
625 | struct ddcb_queue *queue = req->queue; |
626 | struct pci_dev *pci_dev = cd->pci_dev; |
627 | u64 queue_status; |
628 | __be32 icrc_hsi_shi = 0x0000; |
629 | __be32 old, new; |
630 | |
631 | /* unsigned long flags; */ |
632 | if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) { |
633 | dev_err(&pci_dev->dev, |
634 | "[%s] err: software timeout is not set!\n" , __func__); |
635 | return -EFAULT; |
636 | } |
637 | |
638 | pddcb = &queue->ddcb_vaddr[req->num]; |
639 | |
640 | for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) { |
641 | |
642 | spin_lock_irqsave(&queue->ddcb_lock, flags); |
643 | |
644 | /* Check if req was meanwhile finished */ |
645 | if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) |
646 | goto go_home; |
647 | |
648 | /* try to set PURGE bit if FETCHED/COMPLETED are not set */ |
649 | old = pddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */ |
650 | if ((old & DDCB_FETCHED_BE32) == 0x00000000) { |
651 | |
652 | new = (old | DDCB_PURGE_BE32); |
653 | icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32, |
654 | old, new); |
655 | if (icrc_hsi_shi == old) |
656 | goto finish_ddcb; |
657 | } |
658 | |
659 | /* normal finish with HSI bit */ |
660 | barrier(); |
661 | icrc_hsi_shi = pddcb->icrc_hsi_shi_32; |
662 | if (icrc_hsi_shi & DDCB_COMPLETED_BE32) |
663 | goto finish_ddcb; |
664 | |
665 | spin_unlock_irqrestore(lock: &queue->ddcb_lock, flags); |
666 | |
667 | /* |
668 | * Here the check_ddcb() function will most likely |
669 | * discover this DDCB to be finished some point in |
670 | * time. It will mark the req finished and free it up |
671 | * in the list. |
672 | */ |
673 | |
674 | copy_ddcb_results(req, ddcb_no: req->num); /* for the failing case */ |
675 | msleep(msecs: 100); /* sleep for 1/10 second and try again */ |
676 | continue; |
677 | |
678 | finish_ddcb: |
679 | copy_ddcb_results(req, ddcb_no: req->num); |
680 | ddcb_requ_set_state(req, new_state: GENWQE_REQU_FINISHED); |
681 | queue->ddcbs_in_flight--; |
682 | queue->ddcb_req[req->num] = NULL; /* delete from array */ |
683 | ddcb_mark_cleared(pddcb); |
684 | |
685 | /* Move active DDCB further; Nothing to do here anymore. */ |
686 | |
687 | /* |
688 | * We need to ensure that there is at least one free |
689 | * DDCB in the queue. To do that, we must update |
690 | * ddcb_act only if the COMPLETED bit is set for the |
691 | * DDCB we are working on else we treat that DDCB even |
692 | * if we PURGED it as occupied (hardware is supposed |
693 | * to set the COMPLETED bit yet!). |
694 | */ |
695 | icrc_hsi_shi = pddcb->icrc_hsi_shi_32; |
696 | if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) && |
697 | (queue->ddcb_act == req->num)) { |
698 | queue->ddcb_act = ((queue->ddcb_act + 1) % |
699 | queue->ddcb_max); |
700 | } |
701 | go_home: |
702 | spin_unlock_irqrestore(lock: &queue->ddcb_lock, flags); |
703 | return 0; |
704 | } |
705 | |
706 | /* |
707 | * If the card is dead and the queue is forced to stop, we |
708 | * might see this in the queue status register. |
709 | */ |
710 | queue_status = __genwqe_readq(cd, byte_offs: queue->IO_QUEUE_STATUS); |
711 | |
712 | dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n" , req->num); |
713 | genwqe_hexdump(pci_dev, buff: pddcb, size: sizeof(*pddcb)); |
714 | |
715 | dev_err(&pci_dev->dev, |
716 | "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n" , |
717 | __func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT, |
718 | queue_status); |
719 | |
720 | print_ddcb_info(cd, queue: req->queue); |
721 | |
722 | return -EFAULT; |
723 | } |
724 | |
725 | int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d) |
726 | { |
727 | int len; |
728 | struct pci_dev *pci_dev = cd->pci_dev; |
729 | |
730 | if (d == NULL) { |
731 | dev_err(&pci_dev->dev, |
732 | "[%s] err: invalid memory for debug data!\n" , |
733 | __func__); |
734 | return -EFAULT; |
735 | } |
736 | |
737 | len = sizeof(d->driver_version); |
738 | snprintf(buf: d->driver_version, size: len, fmt: "%s" , DRV_VERSION); |
739 | d->slu_unitcfg = cd->slu_unitcfg; |
740 | d->app_unitcfg = cd->app_unitcfg; |
741 | return 0; |
742 | } |
743 | |
744 | /** |
745 | * __genwqe_enqueue_ddcb() - Enqueue a DDCB |
746 | * @cd: pointer to genwqe device descriptor |
747 | * @req: pointer to DDCB execution request |
748 | * @f_flags: file mode: blocking, non-blocking |
749 | * |
750 | * Return: 0 if enqueuing succeeded |
751 | * -EIO if card is unusable/PCIe problems |
752 | * -EBUSY if enqueuing failed |
753 | */ |
754 | int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req, |
755 | unsigned int f_flags) |
756 | { |
757 | struct ddcb *pddcb; |
758 | unsigned long flags; |
759 | struct ddcb_queue *queue; |
760 | struct pci_dev *pci_dev = cd->pci_dev; |
761 | u16 icrc; |
762 | |
763 | retry: |
764 | if (cd->card_state != GENWQE_CARD_USED) { |
765 | printk_ratelimited(KERN_ERR |
766 | "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n" , |
767 | GENWQE_DEVNAME, dev_name(&pci_dev->dev), |
768 | __func__, req->num); |
769 | return -EIO; |
770 | } |
771 | |
772 | queue = req->queue = &cd->queue; |
773 | |
774 | /* FIXME circumvention to improve performance when no irq is |
775 | * there. |
776 | */ |
777 | if (GENWQE_POLLING_ENABLED) |
778 | genwqe_check_ddcb_queue(cd, queue); |
779 | |
780 | /* |
781 | * It must be ensured to process all DDCBs in successive |
782 | * order. Use a lock here in order to prevent nested DDCB |
783 | * enqueuing. |
784 | */ |
785 | spin_lock_irqsave(&queue->ddcb_lock, flags); |
786 | |
787 | pddcb = get_next_ddcb(cd, queue, num: &req->num); /* get ptr and num */ |
788 | if (pddcb == NULL) { |
789 | int rc; |
790 | |
791 | spin_unlock_irqrestore(lock: &queue->ddcb_lock, flags); |
792 | |
793 | if (f_flags & O_NONBLOCK) { |
794 | queue->return_on_busy++; |
795 | return -EBUSY; |
796 | } |
797 | |
798 | queue->wait_on_busy++; |
799 | rc = wait_event_interruptible(queue->busy_waitq, |
800 | queue_free_ddcbs(queue) != 0); |
801 | dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n" , |
802 | __func__, rc); |
803 | if (rc == -ERESTARTSYS) |
804 | return rc; /* interrupted by a signal */ |
805 | |
806 | goto retry; |
807 | } |
808 | |
809 | if (queue->ddcb_req[req->num] != NULL) { |
810 | spin_unlock_irqrestore(lock: &queue->ddcb_lock, flags); |
811 | |
812 | dev_err(&pci_dev->dev, |
813 | "[%s] picked DDCB %d with req=%p still in use!!\n" , |
814 | __func__, req->num, req); |
815 | return -EFAULT; |
816 | } |
817 | ddcb_requ_set_state(req, new_state: GENWQE_REQU_ENQUEUED); |
818 | queue->ddcb_req[req->num] = req; |
819 | |
820 | pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts); |
821 | pddcb->cmd = req->cmd.cmd; |
822 | pddcb->acfunc = req->cmd.acfunc; /* functional unit */ |
823 | |
824 | /* |
825 | * We know that we can get retc 0x104 with CRC error, do not |
826 | * stop the queue in those cases for this command. XDIR = 1 |
827 | * does not work for old SLU versions. |
828 | * |
829 | * Last bitstream with the old XDIR behavior had SLU_ID |
830 | * 0x34199. |
831 | */ |
832 | if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull) |
833 | pddcb->xdir = 0x1; |
834 | else |
835 | pddcb->xdir = 0x0; |
836 | |
837 | |
838 | pddcb->psp = (((req->cmd.asiv_length / 8) << 4) | |
839 | ((req->cmd.asv_length / 8))); |
840 | pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts); |
841 | |
842 | /* |
843 | * If copying the whole DDCB_ASIV_LENGTH is impacting |
844 | * performance we need to change it to |
845 | * req->cmd.asiv_length. But simulation benefits from some |
846 | * non-architectured bits behind the architectured content. |
847 | * |
848 | * How much data is copied depends on the availability of the |
849 | * ATS field, which was introduced late. If the ATS field is |
850 | * supported ASIV is 8 bytes shorter than it used to be. Since |
851 | * the ATS field is copied too, the code should do exactly |
852 | * what it did before, but I wanted to make copying of the ATS |
853 | * field very explicit. |
854 | */ |
855 | if (genwqe_get_slu_id(cd) <= 0x2) { |
856 | memcpy(&pddcb->__asiv[0], /* destination */ |
857 | &req->cmd.__asiv[0], /* source */ |
858 | DDCB_ASIV_LENGTH); /* req->cmd.asiv_length */ |
859 | } else { |
860 | pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats); |
861 | memcpy(&pddcb->n.asiv[0], /* destination */ |
862 | &req->cmd.asiv[0], /* source */ |
863 | DDCB_ASIV_LENGTH_ATS); /* req->cmd.asiv_length */ |
864 | } |
865 | |
866 | pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */ |
867 | |
868 | /* |
869 | * Calculate CRC_16 for corresponding range PSP(7:4). Include |
870 | * empty 4 bytes prior to the data. |
871 | */ |
872 | icrc = genwqe_crc16(buff: (const u8 *)pddcb, |
873 | ICRC_LENGTH(req->cmd.asiv_length), init: 0xffff); |
874 | pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16); |
875 | |
876 | /* enable DDCB completion irq */ |
877 | if (!GENWQE_POLLING_ENABLED) |
878 | pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32; |
879 | |
880 | dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n" , req->num); |
881 | genwqe_hexdump(pci_dev, buff: pddcb, size: sizeof(*pddcb)); |
882 | |
883 | if (ddcb_requ_collect_debug_data(req)) { |
884 | /* use the kernel copy of debug data. copying back to |
885 | user buffer happens later */ |
886 | |
887 | genwqe_init_debug_data(cd, d: &req->debug_data); |
888 | memcpy(&req->debug_data.ddcb_before, pddcb, |
889 | sizeof(req->debug_data.ddcb_before)); |
890 | } |
891 | |
892 | enqueue_ddcb(cd, queue, pddcb, ddcb_no: req->num); |
893 | queue->ddcbs_in_flight++; |
894 | |
895 | if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight) |
896 | queue->ddcbs_max_in_flight = queue->ddcbs_in_flight; |
897 | |
898 | ddcb_requ_set_state(req, new_state: GENWQE_REQU_TAPPED); |
899 | spin_unlock_irqrestore(lock: &queue->ddcb_lock, flags); |
900 | wake_up_interruptible(&cd->queue_waitq); |
901 | |
902 | return 0; |
903 | } |
904 | |
905 | /** |
906 | * __genwqe_execute_raw_ddcb() - Setup and execute DDCB |
907 | * @cd: pointer to genwqe device descriptor |
908 | * @cmd: user provided DDCB command |
909 | * @f_flags: file mode: blocking, non-blocking |
910 | */ |
911 | int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd, |
912 | struct genwqe_ddcb_cmd *cmd, |
913 | unsigned int f_flags) |
914 | { |
915 | int rc = 0; |
916 | struct pci_dev *pci_dev = cd->pci_dev; |
917 | struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd); |
918 | |
919 | if (cmd->asiv_length > DDCB_ASIV_LENGTH) { |
920 | dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n" , |
921 | __func__, cmd->asiv_length); |
922 | return -EINVAL; |
923 | } |
924 | if (cmd->asv_length > DDCB_ASV_LENGTH) { |
925 | dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n" , |
926 | __func__, cmd->asiv_length); |
927 | return -EINVAL; |
928 | } |
929 | rc = __genwqe_enqueue_ddcb(cd, req, f_flags); |
930 | if (rc != 0) |
931 | return rc; |
932 | |
933 | rc = __genwqe_wait_ddcb(cd, req); |
934 | if (rc < 0) /* error or signal interrupt */ |
935 | goto err_exit; |
936 | |
937 | if (ddcb_requ_collect_debug_data(req)) { |
938 | if (copy_to_user(to: (struct genwqe_debug_data __user *) |
939 | (unsigned long)cmd->ddata_addr, |
940 | from: &req->debug_data, |
941 | n: sizeof(struct genwqe_debug_data))) |
942 | return -EFAULT; |
943 | } |
944 | |
945 | /* |
946 | * Higher values than 0x102 indicate completion with faults, |
947 | * lower values than 0x102 indicate processing faults. Note |
948 | * that DDCB might have been purged. E.g. Cntl+C. |
949 | */ |
950 | if (cmd->retc != DDCB_RETC_COMPLETE) { |
951 | /* This might happen e.g. flash read, and needs to be |
952 | handled by the upper layer code. */ |
953 | rc = -EBADMSG; /* not processed/error retc */ |
954 | } |
955 | |
956 | return rc; |
957 | |
958 | err_exit: |
959 | __genwqe_purge_ddcb(cd, req); |
960 | |
961 | if (ddcb_requ_collect_debug_data(req)) { |
962 | if (copy_to_user(to: (struct genwqe_debug_data __user *) |
963 | (unsigned long)cmd->ddata_addr, |
964 | from: &req->debug_data, |
965 | n: sizeof(struct genwqe_debug_data))) |
966 | return -EFAULT; |
967 | } |
968 | return rc; |
969 | } |
970 | |
971 | /** |
972 | * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished |
973 | * @cd: pointer to genwqe device descriptor |
974 | * |
975 | * We use this as condition for our wait-queue code. |
976 | */ |
977 | static int genwqe_next_ddcb_ready(struct genwqe_dev *cd) |
978 | { |
979 | unsigned long flags; |
980 | struct ddcb *pddcb; |
981 | struct ddcb_queue *queue = &cd->queue; |
982 | |
983 | spin_lock_irqsave(&queue->ddcb_lock, flags); |
984 | |
985 | if (queue_empty(queue)) { /* empty queue */ |
986 | spin_unlock_irqrestore(lock: &queue->ddcb_lock, flags); |
987 | return 0; |
988 | } |
989 | |
990 | pddcb = &queue->ddcb_vaddr[queue->ddcb_act]; |
991 | if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */ |
992 | spin_unlock_irqrestore(lock: &queue->ddcb_lock, flags); |
993 | return 1; |
994 | } |
995 | |
996 | spin_unlock_irqrestore(lock: &queue->ddcb_lock, flags); |
997 | return 0; |
998 | } |
999 | |
1000 | /** |
1001 | * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight |
1002 | * @cd: pointer to genwqe device descriptor |
1003 | * |
1004 | * Keep track on the number of DDCBs which ware currently in the |
1005 | * queue. This is needed for statistics as well as condition if we want |
1006 | * to wait or better do polling in case of no interrupts available. |
1007 | */ |
1008 | int genwqe_ddcbs_in_flight(struct genwqe_dev *cd) |
1009 | { |
1010 | unsigned long flags; |
1011 | int ddcbs_in_flight = 0; |
1012 | struct ddcb_queue *queue = &cd->queue; |
1013 | |
1014 | spin_lock_irqsave(&queue->ddcb_lock, flags); |
1015 | ddcbs_in_flight += queue->ddcbs_in_flight; |
1016 | spin_unlock_irqrestore(lock: &queue->ddcb_lock, flags); |
1017 | |
1018 | return ddcbs_in_flight; |
1019 | } |
1020 | |
1021 | static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) |
1022 | { |
1023 | int rc, i; |
1024 | struct ddcb *pddcb; |
1025 | u64 val64; |
1026 | unsigned int queue_size; |
1027 | struct pci_dev *pci_dev = cd->pci_dev; |
1028 | |
1029 | if (GENWQE_DDCB_MAX < 2) |
1030 | return -EINVAL; |
1031 | |
1032 | queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE); |
1033 | |
1034 | queue->ddcbs_in_flight = 0; /* statistics */ |
1035 | queue->ddcbs_max_in_flight = 0; |
1036 | queue->ddcbs_completed = 0; |
1037 | queue->return_on_busy = 0; |
1038 | queue->wait_on_busy = 0; |
1039 | |
1040 | queue->ddcb_seq = 0x100; /* start sequence number */ |
1041 | queue->ddcb_max = GENWQE_DDCB_MAX; |
1042 | queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, size: queue_size, |
1043 | dma_handle: &queue->ddcb_daddr); |
1044 | if (queue->ddcb_vaddr == NULL) { |
1045 | dev_err(&pci_dev->dev, |
1046 | "[%s] **err: could not allocate DDCB **\n" , __func__); |
1047 | return -ENOMEM; |
1048 | } |
1049 | queue->ddcb_req = kcalloc(n: queue->ddcb_max, size: sizeof(struct ddcb_requ *), |
1050 | GFP_KERNEL); |
1051 | if (!queue->ddcb_req) { |
1052 | rc = -ENOMEM; |
1053 | goto free_ddcbs; |
1054 | } |
1055 | |
1056 | queue->ddcb_waitqs = kcalloc(n: queue->ddcb_max, |
1057 | size: sizeof(wait_queue_head_t), |
1058 | GFP_KERNEL); |
1059 | if (!queue->ddcb_waitqs) { |
1060 | rc = -ENOMEM; |
1061 | goto free_requs; |
1062 | } |
1063 | |
1064 | for (i = 0; i < queue->ddcb_max; i++) { |
1065 | pddcb = &queue->ddcb_vaddr[i]; /* DDCBs */ |
1066 | pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32; |
1067 | pddcb->retc_16 = cpu_to_be16(0xfff); |
1068 | |
1069 | queue->ddcb_req[i] = NULL; /* requests */ |
1070 | init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */ |
1071 | } |
1072 | |
1073 | queue->ddcb_act = 0; |
1074 | queue->ddcb_next = 0; /* queue is empty */ |
1075 | |
1076 | spin_lock_init(&queue->ddcb_lock); |
1077 | init_waitqueue_head(&queue->busy_waitq); |
1078 | |
1079 | val64 = ((u64)(queue->ddcb_max - 1) << 8); /* lastptr */ |
1080 | __genwqe_writeq(cd, byte_offs: queue->IO_QUEUE_CONFIG, val: 0x07); /* iCRC/vCRC */ |
1081 | __genwqe_writeq(cd, byte_offs: queue->IO_QUEUE_SEGMENT, val: queue->ddcb_daddr); |
1082 | __genwqe_writeq(cd, byte_offs: queue->IO_QUEUE_INITSQN, val: queue->ddcb_seq); |
1083 | __genwqe_writeq(cd, byte_offs: queue->IO_QUEUE_WRAP, val: val64); |
1084 | return 0; |
1085 | |
1086 | free_requs: |
1087 | kfree(objp: queue->ddcb_req); |
1088 | queue->ddcb_req = NULL; |
1089 | free_ddcbs: |
1090 | __genwqe_free_consistent(cd, size: queue_size, vaddr: queue->ddcb_vaddr, |
1091 | dma_handle: queue->ddcb_daddr); |
1092 | queue->ddcb_vaddr = NULL; |
1093 | queue->ddcb_daddr = 0ull; |
1094 | return rc; |
1095 | |
1096 | } |
1097 | |
1098 | static int ddcb_queue_initialized(struct ddcb_queue *queue) |
1099 | { |
1100 | return queue->ddcb_vaddr != NULL; |
1101 | } |
1102 | |
1103 | static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue) |
1104 | { |
1105 | unsigned int queue_size; |
1106 | |
1107 | queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE); |
1108 | |
1109 | kfree(objp: queue->ddcb_req); |
1110 | queue->ddcb_req = NULL; |
1111 | |
1112 | if (queue->ddcb_vaddr) { |
1113 | __genwqe_free_consistent(cd, size: queue_size, vaddr: queue->ddcb_vaddr, |
1114 | dma_handle: queue->ddcb_daddr); |
1115 | queue->ddcb_vaddr = NULL; |
1116 | queue->ddcb_daddr = 0ull; |
1117 | } |
1118 | } |
1119 | |
1120 | static irqreturn_t genwqe_pf_isr(int irq, void *dev_id) |
1121 | { |
1122 | u64 gfir; |
1123 | struct genwqe_dev *cd = (struct genwqe_dev *)dev_id; |
1124 | struct pci_dev *pci_dev = cd->pci_dev; |
1125 | |
1126 | /* |
1127 | * In case of fatal FIR error the queue is stopped, such that |
1128 | * we can safely check it without risking anything. |
1129 | */ |
1130 | cd->irqs_processed++; |
1131 | wake_up_interruptible(&cd->queue_waitq); |
1132 | |
1133 | /* |
1134 | * Checking for errors before kicking the queue might be |
1135 | * safer, but slower for the good-case ... See above. |
1136 | */ |
1137 | gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR); |
1138 | if (((gfir & GFIR_ERR_TRIGGER) != 0x0) && |
1139 | !pci_channel_offline(pdev: pci_dev)) { |
1140 | |
1141 | if (cd->use_platform_recovery) { |
1142 | /* |
1143 | * Since we use raw accessors, EEH errors won't be |
1144 | * detected by the platform until we do a non-raw |
1145 | * MMIO or config space read |
1146 | */ |
1147 | readq(addr: cd->mmio + IO_SLC_CFGREG_GFIR); |
1148 | |
1149 | /* Don't do anything if the PCI channel is frozen */ |
1150 | if (pci_channel_offline(pdev: pci_dev)) |
1151 | goto exit; |
1152 | } |
1153 | |
1154 | wake_up_interruptible(&cd->health_waitq); |
1155 | |
1156 | /* |
1157 | * By default GFIRs causes recovery actions. This |
1158 | * count is just for debug when recovery is masked. |
1159 | */ |
1160 | dev_err_ratelimited(&pci_dev->dev, |
1161 | "[%s] GFIR=%016llx\n" , |
1162 | __func__, gfir); |
1163 | } |
1164 | |
1165 | exit: |
1166 | return IRQ_HANDLED; |
1167 | } |
1168 | |
1169 | static irqreturn_t genwqe_vf_isr(int irq, void *dev_id) |
1170 | { |
1171 | struct genwqe_dev *cd = (struct genwqe_dev *)dev_id; |
1172 | |
1173 | cd->irqs_processed++; |
1174 | wake_up_interruptible(&cd->queue_waitq); |
1175 | |
1176 | return IRQ_HANDLED; |
1177 | } |
1178 | |
1179 | /** |
1180 | * genwqe_card_thread() - Work thread for the DDCB queue |
1181 | * @data: pointer to genwqe device descriptor |
1182 | * |
1183 | * The idea is to check if there are DDCBs in processing. If there are |
1184 | * some finished DDCBs, we process them and wakeup the |
1185 | * requestors. Otherwise we give other processes time using |
1186 | * cond_resched(). |
1187 | */ |
1188 | static int genwqe_card_thread(void *data) |
1189 | { |
1190 | int should_stop = 0; |
1191 | struct genwqe_dev *cd = (struct genwqe_dev *)data; |
1192 | |
1193 | while (!kthread_should_stop()) { |
1194 | |
1195 | genwqe_check_ddcb_queue(cd, queue: &cd->queue); |
1196 | |
1197 | if (GENWQE_POLLING_ENABLED) { |
1198 | wait_event_interruptible_timeout( |
1199 | cd->queue_waitq, |
1200 | genwqe_ddcbs_in_flight(cd) || |
1201 | (should_stop = kthread_should_stop()), 1); |
1202 | } else { |
1203 | wait_event_interruptible_timeout( |
1204 | cd->queue_waitq, |
1205 | genwqe_next_ddcb_ready(cd) || |
1206 | (should_stop = kthread_should_stop()), HZ); |
1207 | } |
1208 | if (should_stop) |
1209 | break; |
1210 | |
1211 | /* |
1212 | * Avoid soft lockups on heavy loads; we do not want |
1213 | * to disable our interrupts. |
1214 | */ |
1215 | cond_resched(); |
1216 | } |
1217 | return 0; |
1218 | } |
1219 | |
1220 | /** |
1221 | * genwqe_setup_service_layer() - Setup DDCB queue |
1222 | * @cd: pointer to genwqe device descriptor |
1223 | * |
1224 | * Allocate DDCBs. Configure Service Layer Controller (SLC). |
1225 | * |
1226 | * Return: 0 success |
1227 | */ |
1228 | int genwqe_setup_service_layer(struct genwqe_dev *cd) |
1229 | { |
1230 | int rc; |
1231 | struct ddcb_queue *queue; |
1232 | struct pci_dev *pci_dev = cd->pci_dev; |
1233 | |
1234 | if (genwqe_is_privileged(cd)) { |
1235 | rc = genwqe_card_reset(cd); |
1236 | if (rc < 0) { |
1237 | dev_err(&pci_dev->dev, |
1238 | "[%s] err: reset failed.\n" , __func__); |
1239 | return rc; |
1240 | } |
1241 | genwqe_read_softreset(cd); |
1242 | } |
1243 | |
1244 | queue = &cd->queue; |
1245 | queue->IO_QUEUE_CONFIG = IO_SLC_QUEUE_CONFIG; |
1246 | queue->IO_QUEUE_STATUS = IO_SLC_QUEUE_STATUS; |
1247 | queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT; |
1248 | queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN; |
1249 | queue->IO_QUEUE_OFFSET = IO_SLC_QUEUE_OFFSET; |
1250 | queue->IO_QUEUE_WRAP = IO_SLC_QUEUE_WRAP; |
1251 | queue->IO_QUEUE_WTIME = IO_SLC_QUEUE_WTIME; |
1252 | queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS; |
1253 | queue->IO_QUEUE_LRW = IO_SLC_QUEUE_LRW; |
1254 | |
1255 | rc = setup_ddcb_queue(cd, queue); |
1256 | if (rc != 0) { |
1257 | rc = -ENODEV; |
1258 | goto err_out; |
1259 | } |
1260 | |
1261 | init_waitqueue_head(&cd->queue_waitq); |
1262 | cd->card_thread = kthread_run(genwqe_card_thread, cd, |
1263 | GENWQE_DEVNAME "%d_thread" , |
1264 | cd->card_idx); |
1265 | if (IS_ERR(ptr: cd->card_thread)) { |
1266 | rc = PTR_ERR(ptr: cd->card_thread); |
1267 | cd->card_thread = NULL; |
1268 | goto stop_free_queue; |
1269 | } |
1270 | |
1271 | rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS); |
1272 | if (rc) |
1273 | goto stop_kthread; |
1274 | |
1275 | /* |
1276 | * We must have all wait-queues initialized when we enable the |
1277 | * interrupts. Otherwise we might crash if we get an early |
1278 | * irq. |
1279 | */ |
1280 | init_waitqueue_head(&cd->health_waitq); |
1281 | |
1282 | if (genwqe_is_privileged(cd)) { |
1283 | rc = request_irq(irq: pci_dev->irq, handler: genwqe_pf_isr, IRQF_SHARED, |
1284 | GENWQE_DEVNAME, dev: cd); |
1285 | } else { |
1286 | rc = request_irq(irq: pci_dev->irq, handler: genwqe_vf_isr, IRQF_SHARED, |
1287 | GENWQE_DEVNAME, dev: cd); |
1288 | } |
1289 | if (rc < 0) { |
1290 | dev_err(&pci_dev->dev, "irq %d not free.\n" , pci_dev->irq); |
1291 | goto stop_irq_cap; |
1292 | } |
1293 | |
1294 | cd->card_state = GENWQE_CARD_USED; |
1295 | return 0; |
1296 | |
1297 | stop_irq_cap: |
1298 | genwqe_reset_interrupt_capability(cd); |
1299 | stop_kthread: |
1300 | kthread_stop(k: cd->card_thread); |
1301 | cd->card_thread = NULL; |
1302 | stop_free_queue: |
1303 | free_ddcb_queue(cd, queue); |
1304 | err_out: |
1305 | return rc; |
1306 | } |
1307 | |
1308 | /** |
1309 | * queue_wake_up_all() - Handles fatal error case |
1310 | * @cd: pointer to genwqe device descriptor |
1311 | * |
1312 | * The PCI device got unusable and we have to stop all pending |
1313 | * requests as fast as we can. The code after this must purge the |
1314 | * DDCBs in question and ensure that all mappings are freed. |
1315 | */ |
1316 | static int queue_wake_up_all(struct genwqe_dev *cd) |
1317 | { |
1318 | unsigned int i; |
1319 | unsigned long flags; |
1320 | struct ddcb_queue *queue = &cd->queue; |
1321 | |
1322 | spin_lock_irqsave(&queue->ddcb_lock, flags); |
1323 | |
1324 | for (i = 0; i < queue->ddcb_max; i++) |
1325 | wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]); |
1326 | |
1327 | wake_up_interruptible(&queue->busy_waitq); |
1328 | spin_unlock_irqrestore(lock: &queue->ddcb_lock, flags); |
1329 | |
1330 | return 0; |
1331 | } |
1332 | |
1333 | /** |
1334 | * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces |
1335 | * @cd: pointer to genwqe device descriptor |
1336 | * |
1337 | * Relies on the pre-condition that there are no users of the card |
1338 | * device anymore e.g. with open file-descriptors. |
1339 | * |
1340 | * This function must be robust enough to be called twice. |
1341 | */ |
1342 | int genwqe_finish_queue(struct genwqe_dev *cd) |
1343 | { |
1344 | int i, rc = 0, in_flight; |
1345 | int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT; |
1346 | struct pci_dev *pci_dev = cd->pci_dev; |
1347 | struct ddcb_queue *queue = &cd->queue; |
1348 | |
1349 | if (!ddcb_queue_initialized(queue)) |
1350 | return 0; |
1351 | |
1352 | /* Do not wipe out the error state. */ |
1353 | if (cd->card_state == GENWQE_CARD_USED) |
1354 | cd->card_state = GENWQE_CARD_UNUSED; |
1355 | |
1356 | /* Wake up all requests in the DDCB queue such that they |
1357 | should be removed nicely. */ |
1358 | queue_wake_up_all(cd); |
1359 | |
1360 | /* We must wait to get rid of the DDCBs in flight */ |
1361 | for (i = 0; i < waitmax; i++) { |
1362 | in_flight = genwqe_ddcbs_in_flight(cd); |
1363 | |
1364 | if (in_flight == 0) |
1365 | break; |
1366 | |
1367 | dev_dbg(&pci_dev->dev, |
1368 | " DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n" , |
1369 | i, waitmax, in_flight); |
1370 | |
1371 | /* |
1372 | * Severe severe error situation: The card itself has |
1373 | * 16 DDCB queues, each queue has e.g. 32 entries, |
1374 | * each DDBC has a hardware timeout of currently 250 |
1375 | * msec but the PFs have a hardware timeout of 8 sec |
1376 | * ... so I take something large. |
1377 | */ |
1378 | msleep(msecs: 1000); |
1379 | } |
1380 | if (i == waitmax) { |
1381 | dev_err(&pci_dev->dev, " [%s] err: queue is not empty!!\n" , |
1382 | __func__); |
1383 | rc = -EIO; |
1384 | } |
1385 | return rc; |
1386 | } |
1387 | |
1388 | /** |
1389 | * genwqe_release_service_layer() - Shutdown DDCB queue |
1390 | * @cd: genwqe device descriptor |
1391 | * |
1392 | * This function must be robust enough to be called twice. |
1393 | */ |
1394 | int genwqe_release_service_layer(struct genwqe_dev *cd) |
1395 | { |
1396 | struct pci_dev *pci_dev = cd->pci_dev; |
1397 | |
1398 | if (!ddcb_queue_initialized(queue: &cd->queue)) |
1399 | return 1; |
1400 | |
1401 | free_irq(pci_dev->irq, cd); |
1402 | genwqe_reset_interrupt_capability(cd); |
1403 | |
1404 | if (cd->card_thread != NULL) { |
1405 | kthread_stop(k: cd->card_thread); |
1406 | cd->card_thread = NULL; |
1407 | } |
1408 | |
1409 | free_ddcb_queue(cd, queue: &cd->queue); |
1410 | return 0; |
1411 | } |
1412 | |