1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | /* |
3 | * CAAM/SEC 4.x transport/backend driver |
4 | * JobR backend functionality |
5 | * |
6 | * Copyright 2008-2012 Freescale Semiconductor, Inc. |
7 | * Copyright 2019, 2023 NXP |
8 | */ |
9 | |
10 | #include <linux/of_irq.h> |
11 | #include <linux/of_address.h> |
12 | #include <linux/platform_device.h> |
13 | |
14 | #include "compat.h" |
15 | #include "ctrl.h" |
16 | #include "regs.h" |
17 | #include "jr.h" |
18 | #include "desc.h" |
19 | #include "intern.h" |
20 | |
21 | struct jr_driver_data { |
22 | /* List of Physical JobR's with the Driver */ |
23 | struct list_head jr_list; |
24 | spinlock_t jr_alloc_lock; /* jr_list lock */ |
25 | } ____cacheline_aligned; |
26 | |
27 | static struct jr_driver_data driver_data; |
28 | static DEFINE_MUTEX(algs_lock); |
29 | static unsigned int active_devs; |
30 | |
31 | static void register_algs(struct caam_drv_private_jr *jrpriv, |
32 | struct device *dev) |
33 | { |
34 | mutex_lock(&algs_lock); |
35 | |
36 | if (++active_devs != 1) |
37 | goto algs_unlock; |
38 | |
39 | caam_algapi_init(dev); |
40 | caam_algapi_hash_init(dev); |
41 | caam_pkc_init(dev); |
42 | jrpriv->hwrng = !caam_rng_init(dev); |
43 | caam_prng_register(dev); |
44 | caam_qi_algapi_init(dev); |
45 | |
46 | algs_unlock: |
47 | mutex_unlock(lock: &algs_lock); |
48 | } |
49 | |
50 | static void unregister_algs(void) |
51 | { |
52 | mutex_lock(&algs_lock); |
53 | |
54 | if (--active_devs != 0) |
55 | goto algs_unlock; |
56 | |
57 | caam_qi_algapi_exit(); |
58 | caam_prng_unregister(NULL); |
59 | caam_pkc_exit(); |
60 | caam_algapi_hash_exit(); |
61 | caam_algapi_exit(); |
62 | |
63 | algs_unlock: |
64 | mutex_unlock(lock: &algs_lock); |
65 | } |
66 | |
67 | static void caam_jr_crypto_engine_exit(void *data) |
68 | { |
69 | struct device *jrdev = data; |
70 | struct caam_drv_private_jr *jrpriv = dev_get_drvdata(dev: jrdev); |
71 | |
72 | /* Free the resources of crypto-engine */ |
73 | crypto_engine_exit(engine: jrpriv->engine); |
74 | } |
75 | |
76 | /* |
77 | * Put the CAAM in quiesce, ie stop |
78 | * |
79 | * Must be called with itr disabled |
80 | */ |
81 | static int caam_jr_stop_processing(struct device *dev, u32 jrcr_bits) |
82 | { |
83 | struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); |
84 | unsigned int timeout = 100000; |
85 | |
86 | /* Check the current status */ |
87 | if (rd_reg32(reg: &jrp->rregs->jrintstatus) & JRINT_ERR_HALT_INPROGRESS) |
88 | goto wait_quiesce_completion; |
89 | |
90 | /* Reset the field */ |
91 | clrsetbits_32(reg: &jrp->rregs->jrintstatus, JRINT_ERR_HALT_MASK, set: 0); |
92 | |
93 | /* initiate flush / park (required prior to reset) */ |
94 | wr_reg32(reg: &jrp->rregs->jrcommand, data: jrcr_bits); |
95 | |
96 | wait_quiesce_completion: |
97 | while (((rd_reg32(reg: &jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) == |
98 | JRINT_ERR_HALT_INPROGRESS) && --timeout) |
99 | cpu_relax(); |
100 | |
101 | if ((rd_reg32(reg: &jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) != |
102 | JRINT_ERR_HALT_COMPLETE || timeout == 0) { |
103 | dev_err(dev, "failed to flush job ring %d\n" , jrp->ridx); |
104 | return -EIO; |
105 | } |
106 | |
107 | return 0; |
108 | } |
109 | |
110 | /* |
111 | * Flush the job ring, so the jobs running will be stopped, jobs queued will be |
112 | * invalidated and the CAAM will no longer fetch fron input ring. |
113 | * |
114 | * Must be called with itr disabled |
115 | */ |
116 | static int caam_jr_flush(struct device *dev) |
117 | { |
118 | return caam_jr_stop_processing(dev, JRCR_RESET); |
119 | } |
120 | |
121 | /* The resume can be used after a park or a flush if CAAM has not been reset */ |
122 | static int caam_jr_restart_processing(struct device *dev) |
123 | { |
124 | struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); |
125 | u32 halt_status = rd_reg32(reg: &jrp->rregs->jrintstatus) & |
126 | JRINT_ERR_HALT_MASK; |
127 | |
128 | /* Check that the flush/park is completed */ |
129 | if (halt_status != JRINT_ERR_HALT_COMPLETE) |
130 | return -1; |
131 | |
132 | /* Resume processing of jobs */ |
133 | clrsetbits_32(reg: &jrp->rregs->jrintstatus, clear: 0, JRINT_ERR_HALT_COMPLETE); |
134 | |
135 | return 0; |
136 | } |
137 | |
138 | static int caam_reset_hw_jr(struct device *dev) |
139 | { |
140 | struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); |
141 | unsigned int timeout = 100000; |
142 | int err; |
143 | /* |
144 | * mask interrupts since we are going to poll |
145 | * for reset completion status |
146 | */ |
147 | clrsetbits_32(reg: &jrp->rregs->rconfig_lo, clear: 0, JRCFG_IMSK); |
148 | err = caam_jr_flush(dev); |
149 | if (err) |
150 | return err; |
151 | |
152 | /* initiate reset */ |
153 | wr_reg32(reg: &jrp->rregs->jrcommand, JRCR_RESET); |
154 | while ((rd_reg32(reg: &jrp->rregs->jrcommand) & JRCR_RESET) && --timeout) |
155 | cpu_relax(); |
156 | |
157 | if (timeout == 0) { |
158 | dev_err(dev, "failed to reset job ring %d\n" , jrp->ridx); |
159 | return -EIO; |
160 | } |
161 | |
162 | /* unmask interrupts */ |
163 | clrsetbits_32(reg: &jrp->rregs->rconfig_lo, JRCFG_IMSK, set: 0); |
164 | |
165 | return 0; |
166 | } |
167 | |
168 | /* |
169 | * Shutdown JobR independent of platform property code |
170 | */ |
171 | static int caam_jr_shutdown(struct device *dev) |
172 | { |
173 | struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); |
174 | int ret; |
175 | |
176 | ret = caam_reset_hw_jr(dev); |
177 | |
178 | tasklet_kill(t: &jrp->irqtask); |
179 | |
180 | return ret; |
181 | } |
182 | |
183 | static void caam_jr_remove(struct platform_device *pdev) |
184 | { |
185 | int ret; |
186 | struct device *jrdev; |
187 | struct caam_drv_private_jr *jrpriv; |
188 | |
189 | jrdev = &pdev->dev; |
190 | jrpriv = dev_get_drvdata(dev: jrdev); |
191 | |
192 | if (jrpriv->hwrng) |
193 | caam_rng_exit(dev: jrdev->parent); |
194 | |
195 | /* |
196 | * If a job ring is still allocated there is trouble ahead. Once |
197 | * caam_jr_remove() returned, jrpriv will be freed and the registers |
198 | * will get unmapped. So any user of such a job ring will probably |
199 | * crash. |
200 | */ |
201 | if (atomic_read(v: &jrpriv->tfm_count)) { |
202 | dev_alert(jrdev, "Device is busy; consumers might start to crash\n" ); |
203 | return; |
204 | } |
205 | |
206 | /* Unregister JR-based RNG & crypto algorithms */ |
207 | unregister_algs(); |
208 | |
209 | /* Remove the node from Physical JobR list maintained by driver */ |
210 | spin_lock(lock: &driver_data.jr_alloc_lock); |
211 | list_del(entry: &jrpriv->list_node); |
212 | spin_unlock(lock: &driver_data.jr_alloc_lock); |
213 | |
214 | /* Release ring */ |
215 | ret = caam_jr_shutdown(dev: jrdev); |
216 | if (ret) |
217 | dev_err(jrdev, "Failed to shut down job ring\n" ); |
218 | } |
219 | |
220 | /* Main per-ring interrupt handler */ |
221 | static irqreturn_t caam_jr_interrupt(int irq, void *st_dev) |
222 | { |
223 | struct device *dev = st_dev; |
224 | struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); |
225 | u32 irqstate; |
226 | |
227 | /* |
228 | * Check the output ring for ready responses, kick |
229 | * tasklet if jobs done. |
230 | */ |
231 | irqstate = rd_reg32(reg: &jrp->rregs->jrintstatus); |
232 | if (!(irqstate & JRINT_JR_INT)) |
233 | return IRQ_NONE; |
234 | |
235 | /* |
236 | * If JobR error, we got more development work to do |
237 | * Flag a bug now, but we really need to shut down and |
238 | * restart the queue (and fix code). |
239 | */ |
240 | if (irqstate & JRINT_JR_ERROR) { |
241 | dev_err(dev, "job ring error: irqstate: %08x\n" , irqstate); |
242 | BUG(); |
243 | } |
244 | |
245 | /* mask valid interrupts */ |
246 | clrsetbits_32(reg: &jrp->rregs->rconfig_lo, clear: 0, JRCFG_IMSK); |
247 | |
248 | /* Have valid interrupt at this point, just ACK and trigger */ |
249 | wr_reg32(reg: &jrp->rregs->jrintstatus, data: irqstate); |
250 | |
251 | preempt_disable(); |
252 | tasklet_schedule(t: &jrp->irqtask); |
253 | preempt_enable(); |
254 | |
255 | return IRQ_HANDLED; |
256 | } |
257 | |
258 | /* Deferred service handler, run as interrupt-fired tasklet */ |
259 | static void caam_jr_dequeue(unsigned long devarg) |
260 | { |
261 | int hw_idx, sw_idx, i, head, tail; |
262 | struct caam_jr_dequeue_params *params = (void *)devarg; |
263 | struct device *dev = params->dev; |
264 | struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); |
265 | void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg); |
266 | u32 *userdesc, userstatus; |
267 | void *userarg; |
268 | u32 outring_used = 0; |
269 | |
270 | while (outring_used || |
271 | (outring_used = rd_reg32(reg: &jrp->rregs->outring_used))) { |
272 | |
273 | head = READ_ONCE(jrp->head); |
274 | |
275 | sw_idx = tail = jrp->tail; |
276 | hw_idx = jrp->out_ring_read_index; |
277 | |
278 | for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) { |
279 | sw_idx = (tail + i) & (JOBR_DEPTH - 1); |
280 | |
281 | if (jr_outentry_desc(jrp->outring, hw_idx) == |
282 | caam_dma_to_cpu(jrp->entinfo[sw_idx].desc_addr_dma)) |
283 | break; /* found */ |
284 | } |
285 | /* we should never fail to find a matching descriptor */ |
286 | BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0); |
287 | |
288 | /* Unmap just-run descriptor so we can post-process */ |
289 | dma_unmap_single(dev, |
290 | caam_dma_to_cpu(jr_outentry_desc(jrp->outring, |
291 | hw_idx)), |
292 | jrp->entinfo[sw_idx].desc_size, |
293 | DMA_TO_DEVICE); |
294 | |
295 | /* mark completed, avoid matching on a recycled desc addr */ |
296 | jrp->entinfo[sw_idx].desc_addr_dma = 0; |
297 | |
298 | /* Stash callback params */ |
299 | usercall = jrp->entinfo[sw_idx].callbk; |
300 | userarg = jrp->entinfo[sw_idx].cbkarg; |
301 | userdesc = jrp->entinfo[sw_idx].desc_addr_virt; |
302 | userstatus = caam32_to_cpu(val: jr_outentry_jrstatus(outring: jrp->outring, |
303 | hw_idx)); |
304 | |
305 | /* |
306 | * Make sure all information from the job has been obtained |
307 | * before telling CAAM that the job has been removed from the |
308 | * output ring. |
309 | */ |
310 | mb(); |
311 | |
312 | /* set done */ |
313 | wr_reg32(reg: &jrp->rregs->outring_rmvd, data: 1); |
314 | |
315 | jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) & |
316 | (JOBR_DEPTH - 1); |
317 | |
318 | /* |
319 | * if this job completed out-of-order, do not increment |
320 | * the tail. Otherwise, increment tail by 1 plus the |
321 | * number of subsequent jobs already completed out-of-order |
322 | */ |
323 | if (sw_idx == tail) { |
324 | do { |
325 | tail = (tail + 1) & (JOBR_DEPTH - 1); |
326 | } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 && |
327 | jrp->entinfo[tail].desc_addr_dma == 0); |
328 | |
329 | jrp->tail = tail; |
330 | } |
331 | |
332 | /* Finally, execute user's callback */ |
333 | usercall(dev, userdesc, userstatus, userarg); |
334 | outring_used--; |
335 | } |
336 | |
337 | if (params->enable_itr) |
338 | /* reenable / unmask IRQs */ |
339 | clrsetbits_32(reg: &jrp->rregs->rconfig_lo, JRCFG_IMSK, set: 0); |
340 | } |
341 | |
342 | /** |
343 | * caam_jr_alloc() - Alloc a job ring for someone to use as needed. |
344 | * |
345 | * returns : pointer to the newly allocated physical |
346 | * JobR dev can be written to if successful. |
347 | **/ |
348 | struct device *caam_jr_alloc(void) |
349 | { |
350 | struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL; |
351 | struct device *dev = ERR_PTR(error: -ENODEV); |
352 | int min_tfm_cnt = INT_MAX; |
353 | int tfm_cnt; |
354 | |
355 | spin_lock(lock: &driver_data.jr_alloc_lock); |
356 | |
357 | if (list_empty(head: &driver_data.jr_list)) { |
358 | spin_unlock(lock: &driver_data.jr_alloc_lock); |
359 | return ERR_PTR(error: -ENODEV); |
360 | } |
361 | |
362 | list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) { |
363 | tfm_cnt = atomic_read(v: &jrpriv->tfm_count); |
364 | if (tfm_cnt < min_tfm_cnt) { |
365 | min_tfm_cnt = tfm_cnt; |
366 | min_jrpriv = jrpriv; |
367 | } |
368 | if (!min_tfm_cnt) |
369 | break; |
370 | } |
371 | |
372 | if (min_jrpriv) { |
373 | atomic_inc(v: &min_jrpriv->tfm_count); |
374 | dev = min_jrpriv->dev; |
375 | } |
376 | spin_unlock(lock: &driver_data.jr_alloc_lock); |
377 | |
378 | return dev; |
379 | } |
380 | EXPORT_SYMBOL(caam_jr_alloc); |
381 | |
382 | /** |
383 | * caam_jr_free() - Free the Job Ring |
384 | * @rdev: points to the dev that identifies the Job ring to |
385 | * be released. |
386 | **/ |
387 | void caam_jr_free(struct device *rdev) |
388 | { |
389 | struct caam_drv_private_jr *jrpriv = dev_get_drvdata(dev: rdev); |
390 | |
391 | atomic_dec(v: &jrpriv->tfm_count); |
392 | } |
393 | EXPORT_SYMBOL(caam_jr_free); |
394 | |
395 | /** |
396 | * caam_jr_enqueue() - Enqueue a job descriptor head. Returns -EINPROGRESS |
397 | * if OK, -ENOSPC if the queue is full, -EIO if it cannot map the caller's |
398 | * descriptor. |
399 | * @dev: struct device of the job ring to be used |
400 | * @desc: points to a job descriptor that execute our request. All |
401 | * descriptors (and all referenced data) must be in a DMAable |
402 | * region, and all data references must be physical addresses |
403 | * accessible to CAAM (i.e. within a PAMU window granted |
404 | * to it). |
405 | * @cbk: pointer to a callback function to be invoked upon completion |
406 | * of this request. This has the form: |
407 | * callback(struct device *dev, u32 *desc, u32 stat, void *arg) |
408 | * where: |
409 | * dev: contains the job ring device that processed this |
410 | * response. |
411 | * desc: descriptor that initiated the request, same as |
412 | * "desc" being argued to caam_jr_enqueue(). |
413 | * status: untranslated status received from CAAM. See the |
414 | * reference manual for a detailed description of |
415 | * error meaning, or see the JRSTA definitions in the |
416 | * register header file |
417 | * areq: optional pointer to an argument passed with the |
418 | * original request |
419 | * @areq: optional pointer to a user argument for use at callback |
420 | * time. |
421 | **/ |
422 | int caam_jr_enqueue(struct device *dev, u32 *desc, |
423 | void (*cbk)(struct device *dev, u32 *desc, |
424 | u32 status, void *areq), |
425 | void *areq) |
426 | { |
427 | struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); |
428 | struct caam_jrentry_info *head_entry; |
429 | int head, tail, desc_size; |
430 | dma_addr_t desc_dma; |
431 | |
432 | desc_size = (caam32_to_cpu(val: *desc) & HDR_JD_LENGTH_MASK) * sizeof(u32); |
433 | desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE); |
434 | if (dma_mapping_error(dev, dma_addr: desc_dma)) { |
435 | dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n" ); |
436 | return -EIO; |
437 | } |
438 | |
439 | spin_lock_bh(lock: &jrp->inplock); |
440 | |
441 | head = jrp->head; |
442 | tail = READ_ONCE(jrp->tail); |
443 | |
444 | if (!jrp->inpring_avail || |
445 | CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) { |
446 | spin_unlock_bh(lock: &jrp->inplock); |
447 | dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE); |
448 | return -ENOSPC; |
449 | } |
450 | |
451 | head_entry = &jrp->entinfo[head]; |
452 | head_entry->desc_addr_virt = desc; |
453 | head_entry->desc_size = desc_size; |
454 | head_entry->callbk = (void *)cbk; |
455 | head_entry->cbkarg = areq; |
456 | head_entry->desc_addr_dma = desc_dma; |
457 | |
458 | jr_inpentry_set(inpring: jrp->inpring, hw_idx: head, val: cpu_to_caam_dma(value: desc_dma)); |
459 | |
460 | /* |
461 | * Guarantee that the descriptor's DMA address has been written to |
462 | * the next slot in the ring before the write index is updated, since |
463 | * other cores may update this index independently. |
464 | * |
465 | * Under heavy DDR load, smp_wmb() or dma_wmb() fail to make the input |
466 | * ring be updated before the CAAM starts reading it. So, CAAM will |
467 | * process, again, an old descriptor address and will put it in the |
468 | * output ring. This will make caam_jr_dequeue() to fail, since this |
469 | * old descriptor is not in the software ring. |
470 | * To fix this, use wmb() which works on the full system instead of |
471 | * inner/outer shareable domains. |
472 | */ |
473 | wmb(); |
474 | |
475 | jrp->head = (head + 1) & (JOBR_DEPTH - 1); |
476 | |
477 | /* |
478 | * Ensure that all job information has been written before |
479 | * notifying CAAM that a new job was added to the input ring |
480 | * using a memory barrier. The wr_reg32() uses api iowrite32() |
481 | * to do the register write. iowrite32() issues a memory barrier |
482 | * before the write operation. |
483 | */ |
484 | |
485 | wr_reg32(reg: &jrp->rregs->inpring_jobadd, data: 1); |
486 | |
487 | jrp->inpring_avail--; |
488 | if (!jrp->inpring_avail) |
489 | jrp->inpring_avail = rd_reg32(reg: &jrp->rregs->inpring_avail); |
490 | |
491 | spin_unlock_bh(lock: &jrp->inplock); |
492 | |
493 | return -EINPROGRESS; |
494 | } |
495 | EXPORT_SYMBOL(caam_jr_enqueue); |
496 | |
497 | static void caam_jr_init_hw(struct device *dev, dma_addr_t inpbusaddr, |
498 | dma_addr_t outbusaddr) |
499 | { |
500 | struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); |
501 | |
502 | wr_reg64(reg: &jrp->rregs->inpring_base, data: inpbusaddr); |
503 | wr_reg64(reg: &jrp->rregs->outring_base, data: outbusaddr); |
504 | wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH); |
505 | wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH); |
506 | |
507 | /* Select interrupt coalescing parameters */ |
508 | clrsetbits_32(reg: &jrp->rregs->rconfig_lo, clear: 0, JOBR_INTC | |
509 | (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) | |
510 | (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT)); |
511 | } |
512 | |
513 | static void caam_jr_reset_index(struct caam_drv_private_jr *jrp) |
514 | { |
515 | jrp->out_ring_read_index = 0; |
516 | jrp->head = 0; |
517 | jrp->tail = 0; |
518 | } |
519 | |
520 | /* |
521 | * Init JobR independent of platform property detection |
522 | */ |
523 | static int caam_jr_init(struct device *dev) |
524 | { |
525 | struct caam_drv_private_jr *jrp; |
526 | dma_addr_t inpbusaddr, outbusaddr; |
527 | int i, error; |
528 | |
529 | jrp = dev_get_drvdata(dev); |
530 | |
531 | error = caam_reset_hw_jr(dev); |
532 | if (error) |
533 | return error; |
534 | |
535 | jrp->inpring = dmam_alloc_coherent(dev, SIZEOF_JR_INPENTRY * |
536 | JOBR_DEPTH, &inpbusaddr, |
537 | GFP_KERNEL); |
538 | if (!jrp->inpring) |
539 | return -ENOMEM; |
540 | |
541 | jrp->outring = dmam_alloc_coherent(dev, SIZEOF_JR_OUTENTRY * |
542 | JOBR_DEPTH, &outbusaddr, |
543 | GFP_KERNEL); |
544 | if (!jrp->outring) |
545 | return -ENOMEM; |
546 | |
547 | jrp->entinfo = devm_kcalloc(dev, JOBR_DEPTH, sizeof(*jrp->entinfo), |
548 | GFP_KERNEL); |
549 | if (!jrp->entinfo) |
550 | return -ENOMEM; |
551 | |
552 | for (i = 0; i < JOBR_DEPTH; i++) |
553 | jrp->entinfo[i].desc_addr_dma = !0; |
554 | |
555 | /* Setup rings */ |
556 | caam_jr_reset_index(jrp); |
557 | jrp->inpring_avail = JOBR_DEPTH; |
558 | caam_jr_init_hw(dev, inpbusaddr, outbusaddr); |
559 | |
560 | spin_lock_init(&jrp->inplock); |
561 | |
562 | jrp->tasklet_params.dev = dev; |
563 | jrp->tasklet_params.enable_itr = 1; |
564 | tasklet_init(t: &jrp->irqtask, func: caam_jr_dequeue, |
565 | data: (unsigned long)&jrp->tasklet_params); |
566 | |
567 | /* Connect job ring interrupt handler. */ |
568 | error = devm_request_irq(dev, irq: jrp->irq, handler: caam_jr_interrupt, IRQF_SHARED, |
569 | devname: dev_name(dev), dev_id: dev); |
570 | if (error) { |
571 | dev_err(dev, "can't connect JobR %d interrupt (%d)\n" , |
572 | jrp->ridx, jrp->irq); |
573 | tasklet_kill(t: &jrp->irqtask); |
574 | } |
575 | |
576 | return error; |
577 | } |
578 | |
579 | static void caam_jr_irq_dispose_mapping(void *data) |
580 | { |
581 | irq_dispose_mapping(virq: (unsigned long)data); |
582 | } |
583 | |
584 | /* |
585 | * Probe routine for each detected JobR subsystem. |
586 | */ |
587 | static int caam_jr_probe(struct platform_device *pdev) |
588 | { |
589 | struct device *jrdev; |
590 | struct device_node *nprop; |
591 | struct caam_job_ring __iomem *ctrl; |
592 | struct caam_drv_private_jr *jrpriv; |
593 | static int total_jobrs; |
594 | struct resource *r; |
595 | int error; |
596 | |
597 | jrdev = &pdev->dev; |
598 | jrpriv = devm_kzalloc(dev: jrdev, size: sizeof(*jrpriv), GFP_KERNEL); |
599 | if (!jrpriv) |
600 | return -ENOMEM; |
601 | |
602 | dev_set_drvdata(dev: jrdev, data: jrpriv); |
603 | |
604 | /* save ring identity relative to detection */ |
605 | jrpriv->ridx = total_jobrs++; |
606 | |
607 | nprop = pdev->dev.of_node; |
608 | /* Get configuration properties from device tree */ |
609 | /* First, get register page */ |
610 | r = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
611 | if (!r) { |
612 | dev_err(jrdev, "platform_get_resource() failed\n" ); |
613 | return -ENOMEM; |
614 | } |
615 | |
616 | ctrl = devm_ioremap(dev: jrdev, offset: r->start, size: resource_size(res: r)); |
617 | if (!ctrl) { |
618 | dev_err(jrdev, "devm_ioremap() failed\n" ); |
619 | return -ENOMEM; |
620 | } |
621 | |
622 | jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl; |
623 | |
624 | error = dma_set_mask_and_coherent(dev: jrdev, mask: caam_get_dma_mask(dev: jrdev)); |
625 | if (error) { |
626 | dev_err(jrdev, "dma_set_mask_and_coherent failed (%d)\n" , |
627 | error); |
628 | return error; |
629 | } |
630 | |
631 | /* Initialize crypto engine */ |
632 | jrpriv->engine = crypto_engine_alloc_init_and_set(jrdev, true, NULL, |
633 | false, |
634 | CRYPTO_ENGINE_MAX_QLEN); |
635 | if (!jrpriv->engine) { |
636 | dev_err(jrdev, "Could not init crypto-engine\n" ); |
637 | return -ENOMEM; |
638 | } |
639 | |
640 | error = devm_add_action_or_reset(jrdev, caam_jr_crypto_engine_exit, |
641 | jrdev); |
642 | if (error) |
643 | return error; |
644 | |
645 | /* Start crypto engine */ |
646 | error = crypto_engine_start(engine: jrpriv->engine); |
647 | if (error) { |
648 | dev_err(jrdev, "Could not start crypto-engine\n" ); |
649 | return error; |
650 | } |
651 | |
652 | /* Identify the interrupt */ |
653 | jrpriv->irq = irq_of_parse_and_map(node: nprop, index: 0); |
654 | if (!jrpriv->irq) { |
655 | dev_err(jrdev, "irq_of_parse_and_map failed\n" ); |
656 | return -EINVAL; |
657 | } |
658 | |
659 | error = devm_add_action_or_reset(jrdev, caam_jr_irq_dispose_mapping, |
660 | (void *)(unsigned long)jrpriv->irq); |
661 | if (error) |
662 | return error; |
663 | |
664 | /* Now do the platform independent part */ |
665 | error = caam_jr_init(dev: jrdev); /* now turn on hardware */ |
666 | if (error) |
667 | return error; |
668 | |
669 | jrpriv->dev = jrdev; |
670 | spin_lock(lock: &driver_data.jr_alloc_lock); |
671 | list_add_tail(new: &jrpriv->list_node, head: &driver_data.jr_list); |
672 | spin_unlock(lock: &driver_data.jr_alloc_lock); |
673 | |
674 | atomic_set(v: &jrpriv->tfm_count, i: 0); |
675 | |
676 | device_init_wakeup(dev: &pdev->dev, enable: 1); |
677 | device_set_wakeup_enable(dev: &pdev->dev, enable: false); |
678 | |
679 | register_algs(jrpriv, dev: jrdev->parent); |
680 | |
681 | return 0; |
682 | } |
683 | |
684 | static void caam_jr_get_hw_state(struct device *dev) |
685 | { |
686 | struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); |
687 | |
688 | jrp->state.inpbusaddr = rd_reg64(reg: &jrp->rregs->inpring_base); |
689 | jrp->state.outbusaddr = rd_reg64(reg: &jrp->rregs->outring_base); |
690 | } |
691 | |
692 | static int caam_jr_suspend(struct device *dev) |
693 | { |
694 | struct platform_device *pdev = to_platform_device(dev); |
695 | struct caam_drv_private_jr *jrpriv = platform_get_drvdata(pdev); |
696 | struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev: dev->parent); |
697 | struct caam_jr_dequeue_params suspend_params = { |
698 | .dev = dev, |
699 | .enable_itr = 0, |
700 | }; |
701 | |
702 | /* Remove the node from Physical JobR list maintained by driver */ |
703 | spin_lock(lock: &driver_data.jr_alloc_lock); |
704 | list_del(entry: &jrpriv->list_node); |
705 | spin_unlock(lock: &driver_data.jr_alloc_lock); |
706 | |
707 | if (jrpriv->hwrng) |
708 | caam_rng_exit(dev: dev->parent); |
709 | |
710 | if (ctrlpriv->caam_off_during_pm) { |
711 | int err; |
712 | |
713 | tasklet_disable(t: &jrpriv->irqtask); |
714 | |
715 | /* mask itr to call flush */ |
716 | clrsetbits_32(reg: &jrpriv->rregs->rconfig_lo, clear: 0, JRCFG_IMSK); |
717 | |
718 | /* Invalid job in process */ |
719 | err = caam_jr_flush(dev); |
720 | if (err) { |
721 | dev_err(dev, "Failed to flush\n" ); |
722 | return err; |
723 | } |
724 | |
725 | /* Dequeing jobs flushed */ |
726 | caam_jr_dequeue(devarg: (unsigned long)&suspend_params); |
727 | |
728 | /* Save state */ |
729 | caam_jr_get_hw_state(dev); |
730 | } else if (device_may_wakeup(dev: &pdev->dev)) { |
731 | enable_irq_wake(irq: jrpriv->irq); |
732 | } |
733 | |
734 | return 0; |
735 | } |
736 | |
737 | static int caam_jr_resume(struct device *dev) |
738 | { |
739 | struct platform_device *pdev = to_platform_device(dev); |
740 | struct caam_drv_private_jr *jrpriv = platform_get_drvdata(pdev); |
741 | struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev: dev->parent); |
742 | |
743 | if (ctrlpriv->caam_off_during_pm) { |
744 | u64 inp_addr; |
745 | int err; |
746 | |
747 | /* |
748 | * Check if the CAAM has been resetted checking the address of |
749 | * the input ring |
750 | */ |
751 | inp_addr = rd_reg64(reg: &jrpriv->rregs->inpring_base); |
752 | if (inp_addr != 0) { |
753 | /* JR still has some configuration */ |
754 | if (inp_addr == jrpriv->state.inpbusaddr) { |
755 | /* JR has not been resetted */ |
756 | err = caam_jr_restart_processing(dev); |
757 | if (err) { |
758 | dev_err(dev, |
759 | "Restart processing failed\n" ); |
760 | return err; |
761 | } |
762 | |
763 | tasklet_enable(t: &jrpriv->irqtask); |
764 | |
765 | clrsetbits_32(reg: &jrpriv->rregs->rconfig_lo, |
766 | JRCFG_IMSK, set: 0); |
767 | |
768 | goto add_jr; |
769 | } else if (ctrlpriv->optee_en) { |
770 | /* JR has been used by OPTEE, reset it */ |
771 | err = caam_reset_hw_jr(dev); |
772 | if (err) { |
773 | dev_err(dev, "Failed to reset JR\n" ); |
774 | return err; |
775 | } |
776 | } else { |
777 | /* No explanation, return error */ |
778 | return -EIO; |
779 | } |
780 | } |
781 | |
782 | caam_jr_reset_index(jrp: jrpriv); |
783 | caam_jr_init_hw(dev, inpbusaddr: jrpriv->state.inpbusaddr, |
784 | outbusaddr: jrpriv->state.outbusaddr); |
785 | |
786 | tasklet_enable(t: &jrpriv->irqtask); |
787 | } else if (device_may_wakeup(dev: &pdev->dev)) { |
788 | disable_irq_wake(irq: jrpriv->irq); |
789 | } |
790 | |
791 | add_jr: |
792 | spin_lock(lock: &driver_data.jr_alloc_lock); |
793 | list_add_tail(new: &jrpriv->list_node, head: &driver_data.jr_list); |
794 | spin_unlock(lock: &driver_data.jr_alloc_lock); |
795 | |
796 | if (jrpriv->hwrng) |
797 | jrpriv->hwrng = !caam_rng_init(dev: dev->parent); |
798 | |
799 | return 0; |
800 | } |
801 | |
802 | static DEFINE_SIMPLE_DEV_PM_OPS(caam_jr_pm_ops, caam_jr_suspend, caam_jr_resume); |
803 | |
804 | static const struct of_device_id caam_jr_match[] = { |
805 | { |
806 | .compatible = "fsl,sec-v4.0-job-ring" , |
807 | }, |
808 | { |
809 | .compatible = "fsl,sec4.0-job-ring" , |
810 | }, |
811 | {}, |
812 | }; |
813 | MODULE_DEVICE_TABLE(of, caam_jr_match); |
814 | |
815 | static struct platform_driver caam_jr_driver = { |
816 | .driver = { |
817 | .name = "caam_jr" , |
818 | .of_match_table = caam_jr_match, |
819 | .pm = pm_ptr(&caam_jr_pm_ops), |
820 | }, |
821 | .probe = caam_jr_probe, |
822 | .remove_new = caam_jr_remove, |
823 | .shutdown = caam_jr_remove, |
824 | }; |
825 | |
826 | static int __init jr_driver_init(void) |
827 | { |
828 | spin_lock_init(&driver_data.jr_alloc_lock); |
829 | INIT_LIST_HEAD(list: &driver_data.jr_list); |
830 | return platform_driver_register(&caam_jr_driver); |
831 | } |
832 | |
833 | static void __exit jr_driver_exit(void) |
834 | { |
835 | platform_driver_unregister(&caam_jr_driver); |
836 | } |
837 | |
838 | module_init(jr_driver_init); |
839 | module_exit(jr_driver_exit); |
840 | |
841 | MODULE_LICENSE("GPL" ); |
842 | MODULE_DESCRIPTION("FSL CAAM JR request backend" ); |
843 | MODULE_AUTHOR("Freescale Semiconductor - NMG/STC" ); |
844 | |