1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Altera TSE SGDMA and MSGDMA Linux driver |
3 | * Copyright (C) 2014 Altera Corporation. All rights reserved |
4 | */ |
5 | |
6 | #include <linux/list.h> |
7 | #include "altera_utils.h" |
8 | #include "altera_tse.h" |
9 | #include "altera_sgdmahw.h" |
10 | #include "altera_sgdma.h" |
11 | |
12 | static void sgdma_setup_descrip(struct sgdma_descrip __iomem *desc, |
13 | struct sgdma_descrip __iomem *ndesc, |
14 | dma_addr_t ndesc_phys, |
15 | dma_addr_t raddr, |
16 | dma_addr_t waddr, |
17 | u16 length, |
18 | int generate_eop, |
19 | int rfixed, |
20 | int wfixed); |
21 | |
22 | static int sgdma_async_write(struct altera_tse_private *priv, |
23 | struct sgdma_descrip __iomem *desc); |
24 | |
25 | static int sgdma_async_read(struct altera_tse_private *priv); |
26 | |
27 | static dma_addr_t |
28 | sgdma_txphysaddr(struct altera_tse_private *priv, |
29 | struct sgdma_descrip __iomem *desc); |
30 | |
31 | static dma_addr_t |
32 | sgdma_rxphysaddr(struct altera_tse_private *priv, |
33 | struct sgdma_descrip __iomem *desc); |
34 | |
35 | static int sgdma_txbusy(struct altera_tse_private *priv); |
36 | |
37 | static int sgdma_rxbusy(struct altera_tse_private *priv); |
38 | |
39 | static void |
40 | queue_tx(struct altera_tse_private *priv, struct tse_buffer *buffer); |
41 | |
42 | static void |
43 | queue_rx(struct altera_tse_private *priv, struct tse_buffer *buffer); |
44 | |
45 | static struct tse_buffer * |
46 | dequeue_tx(struct altera_tse_private *priv); |
47 | |
48 | static struct tse_buffer * |
49 | dequeue_rx(struct altera_tse_private *priv); |
50 | |
51 | static struct tse_buffer * |
52 | queue_rx_peekhead(struct altera_tse_private *priv); |
53 | |
54 | int sgdma_initialize(struct altera_tse_private *priv) |
55 | { |
56 | priv->txctrlreg = SGDMA_CTRLREG_ILASTD | |
57 | SGDMA_CTRLREG_INTEN; |
58 | |
59 | priv->rxctrlreg = SGDMA_CTRLREG_IDESCRIP | |
60 | SGDMA_CTRLREG_INTEN | |
61 | SGDMA_CTRLREG_ILASTD; |
62 | |
63 | INIT_LIST_HEAD(list: &priv->txlisthd); |
64 | INIT_LIST_HEAD(list: &priv->rxlisthd); |
65 | |
66 | priv->rxdescphys = (dma_addr_t) 0; |
67 | priv->txdescphys = (dma_addr_t) 0; |
68 | |
69 | priv->rxdescphys = dma_map_single(priv->device, |
70 | (void __force *)priv->rx_dma_desc, |
71 | priv->rxdescmem, DMA_BIDIRECTIONAL); |
72 | |
73 | if (dma_mapping_error(dev: priv->device, dma_addr: priv->rxdescphys)) { |
74 | sgdma_uninitialize(priv); |
75 | netdev_err(dev: priv->dev, format: "error mapping rx descriptor memory\n" ); |
76 | return -EINVAL; |
77 | } |
78 | |
79 | priv->txdescphys = dma_map_single(priv->device, |
80 | (void __force *)priv->tx_dma_desc, |
81 | priv->txdescmem, DMA_TO_DEVICE); |
82 | |
83 | if (dma_mapping_error(dev: priv->device, dma_addr: priv->txdescphys)) { |
84 | sgdma_uninitialize(priv); |
85 | netdev_err(dev: priv->dev, format: "error mapping tx descriptor memory\n" ); |
86 | return -EINVAL; |
87 | } |
88 | |
89 | /* Initialize descriptor memory to all 0's, sync memory to cache */ |
90 | memset_io(priv->tx_dma_desc, 0, priv->txdescmem); |
91 | memset_io(priv->rx_dma_desc, 0, priv->rxdescmem); |
92 | |
93 | dma_sync_single_for_device(dev: priv->device, addr: priv->txdescphys, |
94 | size: priv->txdescmem, dir: DMA_TO_DEVICE); |
95 | |
96 | dma_sync_single_for_device(dev: priv->device, addr: priv->rxdescphys, |
97 | size: priv->rxdescmem, dir: DMA_TO_DEVICE); |
98 | |
99 | return 0; |
100 | } |
101 | |
102 | void sgdma_uninitialize(struct altera_tse_private *priv) |
103 | { |
104 | if (priv->rxdescphys) |
105 | dma_unmap_single(priv->device, priv->rxdescphys, |
106 | priv->rxdescmem, DMA_BIDIRECTIONAL); |
107 | |
108 | if (priv->txdescphys) |
109 | dma_unmap_single(priv->device, priv->txdescphys, |
110 | priv->txdescmem, DMA_TO_DEVICE); |
111 | } |
112 | |
113 | /* This function resets the SGDMA controller and clears the |
114 | * descriptor memory used for transmits and receives. |
115 | */ |
116 | void sgdma_reset(struct altera_tse_private *priv) |
117 | { |
118 | /* Initialize descriptor memory to 0 */ |
119 | memset_io(priv->tx_dma_desc, 0, priv->txdescmem); |
120 | memset_io(priv->rx_dma_desc, 0, priv->rxdescmem); |
121 | |
122 | csrwr32(SGDMA_CTRLREG_RESET, mac: priv->tx_dma_csr, sgdma_csroffs(control)); |
123 | csrwr32(val: 0, mac: priv->tx_dma_csr, sgdma_csroffs(control)); |
124 | |
125 | csrwr32(SGDMA_CTRLREG_RESET, mac: priv->rx_dma_csr, sgdma_csroffs(control)); |
126 | csrwr32(val: 0, mac: priv->rx_dma_csr, sgdma_csroffs(control)); |
127 | } |
128 | |
129 | /* For SGDMA, interrupts remain enabled after initially enabling, |
130 | * so no need to provide implementations for abstract enable |
131 | * and disable |
132 | */ |
133 | |
134 | void sgdma_enable_rxirq(struct altera_tse_private *priv) |
135 | { |
136 | } |
137 | |
138 | void sgdma_enable_txirq(struct altera_tse_private *priv) |
139 | { |
140 | } |
141 | |
142 | void sgdma_disable_rxirq(struct altera_tse_private *priv) |
143 | { |
144 | } |
145 | |
146 | void sgdma_disable_txirq(struct altera_tse_private *priv) |
147 | { |
148 | } |
149 | |
150 | void sgdma_clear_rxirq(struct altera_tse_private *priv) |
151 | { |
152 | tse_set_bit(ioaddr: priv->rx_dma_csr, sgdma_csroffs(control), |
153 | SGDMA_CTRLREG_CLRINT); |
154 | } |
155 | |
156 | void sgdma_clear_txirq(struct altera_tse_private *priv) |
157 | { |
158 | tse_set_bit(ioaddr: priv->tx_dma_csr, sgdma_csroffs(control), |
159 | SGDMA_CTRLREG_CLRINT); |
160 | } |
161 | |
162 | /* transmits buffer through SGDMA. Returns number of buffers |
163 | * transmitted, 0 if not possible. |
164 | * |
165 | * tx_lock is held by the caller |
166 | */ |
167 | int sgdma_tx_buffer(struct altera_tse_private *priv, struct tse_buffer *buffer) |
168 | { |
169 | struct sgdma_descrip __iomem *descbase = |
170 | (struct sgdma_descrip __iomem *)priv->tx_dma_desc; |
171 | |
172 | struct sgdma_descrip __iomem *cdesc = &descbase[0]; |
173 | struct sgdma_descrip __iomem *ndesc = &descbase[1]; |
174 | |
175 | /* wait 'til the tx sgdma is ready for the next transmit request */ |
176 | if (sgdma_txbusy(priv)) |
177 | return 0; |
178 | |
179 | sgdma_setup_descrip(desc: cdesc, /* current descriptor */ |
180 | ndesc, /* next descriptor */ |
181 | ndesc_phys: sgdma_txphysaddr(priv, desc: ndesc), |
182 | raddr: buffer->dma_addr, /* address of packet to xmit */ |
183 | waddr: 0, /* write addr 0 for tx dma */ |
184 | length: buffer->len, /* length of packet */ |
185 | SGDMA_CONTROL_EOP, /* Generate EOP */ |
186 | rfixed: 0, /* read fixed */ |
187 | SGDMA_CONTROL_WR_FIXED); /* Generate SOP */ |
188 | |
189 | sgdma_async_write(priv, desc: cdesc); |
190 | |
191 | /* enqueue the request to the pending transmit queue */ |
192 | queue_tx(priv, buffer); |
193 | |
194 | return 1; |
195 | } |
196 | |
197 | |
198 | /* tx_lock held to protect access to queued tx list |
199 | */ |
200 | u32 sgdma_tx_completions(struct altera_tse_private *priv) |
201 | { |
202 | u32 ready = 0; |
203 | |
204 | if (!sgdma_txbusy(priv) && |
205 | ((csrrd8(mac: priv->tx_dma_desc, sgdma_descroffs(control)) |
206 | & SGDMA_CONTROL_HW_OWNED) == 0) && |
207 | (dequeue_tx(priv))) { |
208 | ready = 1; |
209 | } |
210 | |
211 | return ready; |
212 | } |
213 | |
214 | void sgdma_start_rxdma(struct altera_tse_private *priv) |
215 | { |
216 | sgdma_async_read(priv); |
217 | } |
218 | |
219 | void sgdma_add_rx_desc(struct altera_tse_private *priv, |
220 | struct tse_buffer *rxbuffer) |
221 | { |
222 | queue_rx(priv, buffer: rxbuffer); |
223 | } |
224 | |
225 | /* status is returned on upper 16 bits, |
226 | * length is returned in lower 16 bits |
227 | */ |
228 | u32 sgdma_rx_status(struct altera_tse_private *priv) |
229 | { |
230 | struct sgdma_descrip __iomem *base = |
231 | (struct sgdma_descrip __iomem *)priv->rx_dma_desc; |
232 | struct sgdma_descrip __iomem *desc = NULL; |
233 | struct tse_buffer *rxbuffer = NULL; |
234 | unsigned int rxstatus = 0; |
235 | |
236 | u32 sts = csrrd32(mac: priv->rx_dma_csr, sgdma_csroffs(status)); |
237 | |
238 | desc = &base[0]; |
239 | if (sts & SGDMA_STSREG_EOP) { |
240 | unsigned int pktlength = 0; |
241 | unsigned int pktstatus = 0; |
242 | dma_sync_single_for_cpu(dev: priv->device, |
243 | addr: priv->rxdescphys, |
244 | SGDMA_DESC_LEN, |
245 | dir: DMA_FROM_DEVICE); |
246 | |
247 | pktlength = csrrd16(mac: desc, sgdma_descroffs(bytes_xferred)); |
248 | pktstatus = csrrd8(mac: desc, sgdma_descroffs(status)); |
249 | rxstatus = pktstatus & ~SGDMA_STATUS_EOP; |
250 | rxstatus = rxstatus << 16; |
251 | rxstatus |= (pktlength & 0xffff); |
252 | |
253 | if (rxstatus) { |
254 | csrwr8(val: 0, mac: desc, sgdma_descroffs(status)); |
255 | |
256 | rxbuffer = dequeue_rx(priv); |
257 | if (rxbuffer == NULL) |
258 | netdev_info(dev: priv->dev, |
259 | format: "sgdma rx and rx queue empty!\n" ); |
260 | |
261 | /* Clear control */ |
262 | csrwr32(val: 0, mac: priv->rx_dma_csr, sgdma_csroffs(control)); |
263 | /* clear status */ |
264 | csrwr32(val: 0xf, mac: priv->rx_dma_csr, sgdma_csroffs(status)); |
265 | |
266 | /* kick the rx sgdma after reaping this descriptor */ |
267 | sgdma_async_read(priv); |
268 | |
269 | } else { |
270 | /* If the SGDMA indicated an end of packet on recv, |
271 | * then it's expected that the rxstatus from the |
272 | * descriptor is non-zero - meaning a valid packet |
273 | * with a nonzero length, or an error has been |
274 | * indicated. if not, then all we can do is signal |
275 | * an error and return no packet received. Most likely |
276 | * there is a system design error, or an error in the |
277 | * underlying kernel (cache or cache management problem) |
278 | */ |
279 | netdev_err(dev: priv->dev, |
280 | format: "SGDMA RX Error Info: %x, %x, %x\n" , |
281 | sts, csrrd8(mac: desc, sgdma_descroffs(status)), |
282 | rxstatus); |
283 | } |
284 | } else if (sts == 0) { |
285 | sgdma_async_read(priv); |
286 | } |
287 | |
288 | return rxstatus; |
289 | } |
290 | |
291 | |
292 | /* Private functions */ |
293 | static void sgdma_setup_descrip(struct sgdma_descrip __iomem *desc, |
294 | struct sgdma_descrip __iomem *ndesc, |
295 | dma_addr_t ndesc_phys, |
296 | dma_addr_t raddr, |
297 | dma_addr_t waddr, |
298 | u16 length, |
299 | int generate_eop, |
300 | int rfixed, |
301 | int wfixed) |
302 | { |
303 | /* Clear the next descriptor as not owned by hardware */ |
304 | |
305 | u32 ctrl = csrrd8(mac: ndesc, sgdma_descroffs(control)); |
306 | ctrl &= ~SGDMA_CONTROL_HW_OWNED; |
307 | csrwr8(val: ctrl, mac: ndesc, sgdma_descroffs(control)); |
308 | |
309 | ctrl = SGDMA_CONTROL_HW_OWNED; |
310 | ctrl |= generate_eop; |
311 | ctrl |= rfixed; |
312 | ctrl |= wfixed; |
313 | |
314 | /* Channel is implicitly zero, initialized to 0 by default */ |
315 | csrwr32(lower_32_bits(raddr), mac: desc, sgdma_descroffs(raddr)); |
316 | csrwr32(lower_32_bits(waddr), mac: desc, sgdma_descroffs(waddr)); |
317 | |
318 | csrwr32(val: 0, mac: desc, sgdma_descroffs(pad1)); |
319 | csrwr32(val: 0, mac: desc, sgdma_descroffs(pad2)); |
320 | csrwr32(lower_32_bits(ndesc_phys), mac: desc, sgdma_descroffs(next)); |
321 | |
322 | csrwr8(val: ctrl, mac: desc, sgdma_descroffs(control)); |
323 | csrwr8(val: 0, mac: desc, sgdma_descroffs(status)); |
324 | csrwr8(val: 0, mac: desc, sgdma_descroffs(wburst)); |
325 | csrwr8(val: 0, mac: desc, sgdma_descroffs(rburst)); |
326 | csrwr16(val: length, mac: desc, sgdma_descroffs(bytes)); |
327 | csrwr16(val: 0, mac: desc, sgdma_descroffs(bytes_xferred)); |
328 | } |
329 | |
330 | /* If hardware is busy, don't restart async read. |
331 | * if status register is 0 - meaning initial state, restart async read, |
332 | * probably for the first time when populating a receive buffer. |
333 | * If read status indicate not busy and a status, restart the async |
334 | * DMA read. |
335 | */ |
336 | static int sgdma_async_read(struct altera_tse_private *priv) |
337 | { |
338 | struct sgdma_descrip __iomem *descbase = |
339 | (struct sgdma_descrip __iomem *)priv->rx_dma_desc; |
340 | |
341 | struct sgdma_descrip __iomem *cdesc = &descbase[0]; |
342 | struct sgdma_descrip __iomem *ndesc = &descbase[1]; |
343 | struct tse_buffer *rxbuffer = NULL; |
344 | |
345 | if (!sgdma_rxbusy(priv)) { |
346 | rxbuffer = queue_rx_peekhead(priv); |
347 | if (rxbuffer == NULL) { |
348 | netdev_err(dev: priv->dev, format: "no rx buffers available\n" ); |
349 | return 0; |
350 | } |
351 | |
352 | sgdma_setup_descrip(desc: cdesc, /* current descriptor */ |
353 | ndesc, /* next descriptor */ |
354 | ndesc_phys: sgdma_rxphysaddr(priv, desc: ndesc), |
355 | raddr: 0, /* read addr 0 for rx dma */ |
356 | waddr: rxbuffer->dma_addr, /* write addr for rx dma */ |
357 | length: 0, /* read 'til EOP */ |
358 | generate_eop: 0, /* EOP: NA for rx dma */ |
359 | rfixed: 0, /* read fixed: NA for rx dma */ |
360 | wfixed: 0); /* SOP: NA for rx DMA */ |
361 | |
362 | dma_sync_single_for_device(dev: priv->device, |
363 | addr: priv->rxdescphys, |
364 | SGDMA_DESC_LEN, |
365 | dir: DMA_TO_DEVICE); |
366 | |
367 | csrwr32(lower_32_bits(sgdma_rxphysaddr(priv, cdesc)), |
368 | mac: priv->rx_dma_csr, |
369 | sgdma_csroffs(next_descrip)); |
370 | |
371 | csrwr32(val: (priv->rxctrlreg | SGDMA_CTRLREG_START), |
372 | mac: priv->rx_dma_csr, |
373 | sgdma_csroffs(control)); |
374 | |
375 | return 1; |
376 | } |
377 | |
378 | return 0; |
379 | } |
380 | |
381 | static int sgdma_async_write(struct altera_tse_private *priv, |
382 | struct sgdma_descrip __iomem *desc) |
383 | { |
384 | if (sgdma_txbusy(priv)) |
385 | return 0; |
386 | |
387 | /* clear control and status */ |
388 | csrwr32(val: 0, mac: priv->tx_dma_csr, sgdma_csroffs(control)); |
389 | csrwr32(val: 0x1f, mac: priv->tx_dma_csr, sgdma_csroffs(status)); |
390 | |
391 | dma_sync_single_for_device(dev: priv->device, addr: priv->txdescphys, |
392 | SGDMA_DESC_LEN, dir: DMA_TO_DEVICE); |
393 | |
394 | csrwr32(lower_32_bits(sgdma_txphysaddr(priv, desc)), |
395 | mac: priv->tx_dma_csr, |
396 | sgdma_csroffs(next_descrip)); |
397 | |
398 | csrwr32(val: (priv->txctrlreg | SGDMA_CTRLREG_START), |
399 | mac: priv->tx_dma_csr, |
400 | sgdma_csroffs(control)); |
401 | |
402 | return 1; |
403 | } |
404 | |
405 | static dma_addr_t |
406 | sgdma_txphysaddr(struct altera_tse_private *priv, |
407 | struct sgdma_descrip __iomem *desc) |
408 | { |
409 | dma_addr_t paddr = priv->txdescmem_busaddr; |
410 | uintptr_t offs = (uintptr_t)desc - (uintptr_t)priv->tx_dma_desc; |
411 | return (dma_addr_t)((uintptr_t)paddr + offs); |
412 | } |
413 | |
414 | static dma_addr_t |
415 | sgdma_rxphysaddr(struct altera_tse_private *priv, |
416 | struct sgdma_descrip __iomem *desc) |
417 | { |
418 | dma_addr_t paddr = priv->rxdescmem_busaddr; |
419 | uintptr_t offs = (uintptr_t)desc - (uintptr_t)priv->rx_dma_desc; |
420 | return (dma_addr_t)((uintptr_t)paddr + offs); |
421 | } |
422 | |
423 | #define list_remove_head(list, entry, type, member) \ |
424 | do { \ |
425 | entry = NULL; \ |
426 | if (!list_empty(list)) { \ |
427 | entry = list_entry((list)->next, type, member); \ |
428 | list_del_init(&entry->member); \ |
429 | } \ |
430 | } while (0) |
431 | |
432 | #define list_peek_head(list, entry, type, member) \ |
433 | do { \ |
434 | entry = NULL; \ |
435 | if (!list_empty(list)) { \ |
436 | entry = list_entry((list)->next, type, member); \ |
437 | } \ |
438 | } while (0) |
439 | |
440 | /* adds a tse_buffer to the tail of a tx buffer list. |
441 | * assumes the caller is managing and holding a mutual exclusion |
442 | * primitive to avoid simultaneous pushes/pops to the list. |
443 | */ |
444 | static void |
445 | queue_tx(struct altera_tse_private *priv, struct tse_buffer *buffer) |
446 | { |
447 | list_add_tail(new: &buffer->lh, head: &priv->txlisthd); |
448 | } |
449 | |
450 | |
451 | /* adds a tse_buffer to the tail of a rx buffer list |
452 | * assumes the caller is managing and holding a mutual exclusion |
453 | * primitive to avoid simultaneous pushes/pops to the list. |
454 | */ |
455 | static void |
456 | queue_rx(struct altera_tse_private *priv, struct tse_buffer *buffer) |
457 | { |
458 | list_add_tail(new: &buffer->lh, head: &priv->rxlisthd); |
459 | } |
460 | |
461 | /* dequeues a tse_buffer from the transmit buffer list, otherwise |
462 | * returns NULL if empty. |
463 | * assumes the caller is managing and holding a mutual exclusion |
464 | * primitive to avoid simultaneous pushes/pops to the list. |
465 | */ |
466 | static struct tse_buffer * |
467 | dequeue_tx(struct altera_tse_private *priv) |
468 | { |
469 | struct tse_buffer *buffer = NULL; |
470 | list_remove_head(&priv->txlisthd, buffer, struct tse_buffer, lh); |
471 | return buffer; |
472 | } |
473 | |
474 | /* dequeues a tse_buffer from the receive buffer list, otherwise |
475 | * returns NULL if empty |
476 | * assumes the caller is managing and holding a mutual exclusion |
477 | * primitive to avoid simultaneous pushes/pops to the list. |
478 | */ |
479 | static struct tse_buffer * |
480 | dequeue_rx(struct altera_tse_private *priv) |
481 | { |
482 | struct tse_buffer *buffer = NULL; |
483 | list_remove_head(&priv->rxlisthd, buffer, struct tse_buffer, lh); |
484 | return buffer; |
485 | } |
486 | |
487 | /* dequeues a tse_buffer from the receive buffer list, otherwise |
488 | * returns NULL if empty |
489 | * assumes the caller is managing and holding a mutual exclusion |
490 | * primitive to avoid simultaneous pushes/pops to the list while the |
491 | * head is being examined. |
492 | */ |
493 | static struct tse_buffer * |
494 | queue_rx_peekhead(struct altera_tse_private *priv) |
495 | { |
496 | struct tse_buffer *buffer = NULL; |
497 | list_peek_head(&priv->rxlisthd, buffer, struct tse_buffer, lh); |
498 | return buffer; |
499 | } |
500 | |
501 | /* check and return rx sgdma status without polling |
502 | */ |
503 | static int sgdma_rxbusy(struct altera_tse_private *priv) |
504 | { |
505 | return csrrd32(mac: priv->rx_dma_csr, sgdma_csroffs(status)) |
506 | & SGDMA_STSREG_BUSY; |
507 | } |
508 | |
509 | /* waits for the tx sgdma to finish it's current operation, returns 0 |
510 | * when it transitions to nonbusy, returns 1 if the operation times out |
511 | */ |
512 | static int sgdma_txbusy(struct altera_tse_private *priv) |
513 | { |
514 | int delay = 0; |
515 | |
516 | /* if DMA is busy, wait for current transaction to finish */ |
517 | while ((csrrd32(mac: priv->tx_dma_csr, sgdma_csroffs(status)) |
518 | & SGDMA_STSREG_BUSY) && (delay++ < 100)) |
519 | udelay(1); |
520 | |
521 | if (csrrd32(mac: priv->tx_dma_csr, sgdma_csroffs(status)) |
522 | & SGDMA_STSREG_BUSY) { |
523 | netdev_err(dev: priv->dev, format: "timeout waiting for tx dma\n" ); |
524 | return 1; |
525 | } |
526 | return 0; |
527 | } |
528 | |