1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * copy offload engine support |
4 | * |
5 | * Copyright © 2006, Intel Corporation. |
6 | * |
7 | * Dan Williams <dan.j.williams@intel.com> |
8 | * |
9 | * with architecture considerations by: |
10 | * Neil Brown <neilb@suse.de> |
11 | * Jeff Garzik <jeff@garzik.org> |
12 | */ |
13 | #include <linux/kernel.h> |
14 | #include <linux/highmem.h> |
15 | #include <linux/module.h> |
16 | #include <linux/mm.h> |
17 | #include <linux/dma-mapping.h> |
18 | #include <linux/async_tx.h> |
19 | |
20 | /** |
21 | * async_memcpy - attempt to copy memory with a dma engine. |
22 | * @dest: destination page |
23 | * @src: src page |
24 | * @dest_offset: offset into 'dest' to start transaction |
25 | * @src_offset: offset into 'src' to start transaction |
26 | * @len: length in bytes |
27 | * @submit: submission / completion modifiers |
28 | * |
29 | * honored flags: ASYNC_TX_ACK |
30 | */ |
31 | struct dma_async_tx_descriptor * |
32 | async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, |
33 | unsigned int src_offset, size_t len, |
34 | struct async_submit_ctl *submit) |
35 | { |
36 | struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMCPY, |
37 | &dest, 1, &src, 1, len); |
38 | struct dma_device *device = chan ? chan->device : NULL; |
39 | struct dma_async_tx_descriptor *tx = NULL; |
40 | struct dmaengine_unmap_data *unmap = NULL; |
41 | |
42 | if (device) |
43 | unmap = dmaengine_get_unmap_data(dev: device->dev, nr: 2, GFP_NOWAIT); |
44 | |
45 | if (unmap && is_dma_copy_aligned(dev: device, off1: src_offset, off2: dest_offset, len)) { |
46 | unsigned long dma_prep_flags = 0; |
47 | |
48 | if (submit->cb_fn) |
49 | dma_prep_flags |= DMA_PREP_INTERRUPT; |
50 | if (submit->flags & ASYNC_TX_FENCE) |
51 | dma_prep_flags |= DMA_PREP_FENCE; |
52 | |
53 | unmap->to_cnt = 1; |
54 | unmap->addr[0] = dma_map_page(device->dev, src, src_offset, len, |
55 | DMA_TO_DEVICE); |
56 | unmap->from_cnt = 1; |
57 | unmap->addr[1] = dma_map_page(device->dev, dest, dest_offset, len, |
58 | DMA_FROM_DEVICE); |
59 | unmap->len = len; |
60 | |
61 | tx = device->device_prep_dma_memcpy(chan, unmap->addr[1], |
62 | unmap->addr[0], len, |
63 | dma_prep_flags); |
64 | } |
65 | |
66 | if (tx) { |
67 | pr_debug("%s: (async) len: %zu\n" , __func__, len); |
68 | |
69 | dma_set_unmap(tx, unmap); |
70 | async_tx_submit(chan, tx, submit); |
71 | } else { |
72 | void *dest_buf, *src_buf; |
73 | pr_debug("%s: (sync) len: %zu\n" , __func__, len); |
74 | |
75 | /* wait for any prerequisite operations */ |
76 | async_tx_quiesce(tx: &submit->depend_tx); |
77 | |
78 | dest_buf = kmap_atomic(page: dest) + dest_offset; |
79 | src_buf = kmap_atomic(page: src) + src_offset; |
80 | |
81 | memcpy(dest_buf, src_buf, len); |
82 | |
83 | kunmap_atomic(src_buf); |
84 | kunmap_atomic(dest_buf); |
85 | |
86 | async_tx_sync_epilog(submit); |
87 | } |
88 | |
89 | dmaengine_unmap_put(unmap); |
90 | |
91 | return tx; |
92 | } |
93 | EXPORT_SYMBOL_GPL(async_memcpy); |
94 | |
95 | MODULE_AUTHOR("Intel Corporation" ); |
96 | MODULE_DESCRIPTION("asynchronous memcpy api" ); |
97 | MODULE_LICENSE("GPL" ); |
98 | |