1 | /* |
2 | * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. |
3 | * |
4 | * This software is available to you under a choice of one of two |
5 | * licenses. You may choose to be licensed under the terms of the GNU |
6 | * General Public License (GPL) Version 2, available from the file |
7 | * COPYING in the main directory of this source tree, or the |
8 | * OpenIB.org BSD license below: |
9 | * |
10 | * Redistribution and use in source and binary forms, with or |
11 | * without modification, are permitted provided that the following |
12 | * conditions are met: |
13 | * |
14 | * - Redistributions of source code must retain the above |
15 | * copyright notice, this list of conditions and the following |
16 | * disclaimer. |
17 | * |
18 | * - Redistributions in binary form must reproduce the above |
19 | * copyright notice, this list of conditions and the following |
20 | * disclaimer in the documentation and/or other materials |
21 | * provided with the distribution. |
22 | * |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
30 | * SOFTWARE. |
31 | */ |
32 | |
33 | #include <linux/errno.h> |
34 | #include <linux/slab.h> |
35 | #include <linux/mm.h> |
36 | #include <linux/export.h> |
37 | #include <linux/bitmap.h> |
38 | #include <linux/dma-mapping.h> |
39 | #include <linux/vmalloc.h> |
40 | #include <linux/mlx5/driver.h> |
41 | |
42 | #include "mlx5_core.h" |
43 | |
44 | struct mlx5_db_pgdir { |
45 | struct list_head list; |
46 | unsigned long *bitmap; |
47 | __be32 *db_page; |
48 | dma_addr_t db_dma; |
49 | }; |
50 | |
51 | /* Handling for queue buffers -- we allocate a bunch of memory and |
52 | * register it in a memory region at HCA virtual address 0. |
53 | */ |
54 | |
55 | static void *mlx5_dma_zalloc_coherent_node(struct mlx5_core_dev *dev, |
56 | size_t size, dma_addr_t *dma_handle, |
57 | int node) |
58 | { |
59 | struct device *device = mlx5_core_dma_dev(dev); |
60 | struct mlx5_priv *priv = &dev->priv; |
61 | int original_node; |
62 | void *cpu_handle; |
63 | |
64 | mutex_lock(&priv->alloc_mutex); |
65 | original_node = dev_to_node(dev: device); |
66 | set_dev_node(dev: device, node); |
67 | cpu_handle = dma_alloc_coherent(dev: device, size, dma_handle, |
68 | GFP_KERNEL); |
69 | set_dev_node(dev: device, node: original_node); |
70 | mutex_unlock(lock: &priv->alloc_mutex); |
71 | return cpu_handle; |
72 | } |
73 | |
74 | int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size, |
75 | struct mlx5_frag_buf *buf, int node) |
76 | { |
77 | int i; |
78 | |
79 | buf->size = size; |
80 | buf->npages = DIV_ROUND_UP(size, PAGE_SIZE); |
81 | buf->page_shift = PAGE_SHIFT; |
82 | buf->frags = kcalloc(n: buf->npages, size: sizeof(struct mlx5_buf_list), |
83 | GFP_KERNEL); |
84 | if (!buf->frags) |
85 | goto err_out; |
86 | |
87 | for (i = 0; i < buf->npages; i++) { |
88 | struct mlx5_buf_list *frag = &buf->frags[i]; |
89 | int frag_sz = min_t(int, size, PAGE_SIZE); |
90 | |
91 | frag->buf = mlx5_dma_zalloc_coherent_node(dev, size: frag_sz, |
92 | dma_handle: &frag->map, node); |
93 | if (!frag->buf) |
94 | goto err_free_buf; |
95 | if (frag->map & ((1 << buf->page_shift) - 1)) { |
96 | dma_free_coherent(dev: mlx5_core_dma_dev(dev), size: frag_sz, |
97 | cpu_addr: buf->frags[i].buf, dma_handle: buf->frags[i].map); |
98 | mlx5_core_warn(dev, "unexpected map alignment: %pad, page_shift=%d\n" , |
99 | &frag->map, buf->page_shift); |
100 | goto err_free_buf; |
101 | } |
102 | size -= frag_sz; |
103 | } |
104 | |
105 | return 0; |
106 | |
107 | err_free_buf: |
108 | while (i--) |
109 | dma_free_coherent(dev: mlx5_core_dma_dev(dev), PAGE_SIZE, cpu_addr: buf->frags[i].buf, |
110 | dma_handle: buf->frags[i].map); |
111 | kfree(objp: buf->frags); |
112 | err_out: |
113 | return -ENOMEM; |
114 | } |
115 | EXPORT_SYMBOL_GPL(mlx5_frag_buf_alloc_node); |
116 | |
117 | void mlx5_frag_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf) |
118 | { |
119 | int size = buf->size; |
120 | int i; |
121 | |
122 | for (i = 0; i < buf->npages; i++) { |
123 | int frag_sz = min_t(int, size, PAGE_SIZE); |
124 | |
125 | dma_free_coherent(dev: mlx5_core_dma_dev(dev), size: frag_sz, cpu_addr: buf->frags[i].buf, |
126 | dma_handle: buf->frags[i].map); |
127 | size -= frag_sz; |
128 | } |
129 | kfree(objp: buf->frags); |
130 | } |
131 | EXPORT_SYMBOL_GPL(mlx5_frag_buf_free); |
132 | |
133 | static struct mlx5_db_pgdir *mlx5_alloc_db_pgdir(struct mlx5_core_dev *dev, |
134 | int node) |
135 | { |
136 | u32 db_per_page = PAGE_SIZE / cache_line_size(); |
137 | struct mlx5_db_pgdir *pgdir; |
138 | |
139 | pgdir = kzalloc_node(size: sizeof(*pgdir), GFP_KERNEL, node); |
140 | if (!pgdir) |
141 | return NULL; |
142 | |
143 | pgdir->bitmap = bitmap_zalloc_node(nbits: db_per_page, GFP_KERNEL, node); |
144 | if (!pgdir->bitmap) { |
145 | kfree(objp: pgdir); |
146 | return NULL; |
147 | } |
148 | |
149 | bitmap_fill(dst: pgdir->bitmap, nbits: db_per_page); |
150 | |
151 | pgdir->db_page = mlx5_dma_zalloc_coherent_node(dev, PAGE_SIZE, |
152 | dma_handle: &pgdir->db_dma, node); |
153 | if (!pgdir->db_page) { |
154 | bitmap_free(bitmap: pgdir->bitmap); |
155 | kfree(objp: pgdir); |
156 | return NULL; |
157 | } |
158 | |
159 | return pgdir; |
160 | } |
161 | |
162 | static int mlx5_alloc_db_from_pgdir(struct mlx5_db_pgdir *pgdir, |
163 | struct mlx5_db *db) |
164 | { |
165 | u32 db_per_page = PAGE_SIZE / cache_line_size(); |
166 | int offset; |
167 | int i; |
168 | |
169 | i = find_first_bit(addr: pgdir->bitmap, size: db_per_page); |
170 | if (i >= db_per_page) |
171 | return -ENOMEM; |
172 | |
173 | __clear_bit(i, pgdir->bitmap); |
174 | |
175 | db->u.pgdir = pgdir; |
176 | db->index = i; |
177 | offset = db->index * cache_line_size(); |
178 | db->db = pgdir->db_page + offset / sizeof(*pgdir->db_page); |
179 | db->dma = pgdir->db_dma + offset; |
180 | |
181 | db->db[0] = 0; |
182 | db->db[1] = 0; |
183 | |
184 | return 0; |
185 | } |
186 | |
187 | int mlx5_db_alloc_node(struct mlx5_core_dev *dev, struct mlx5_db *db, int node) |
188 | { |
189 | struct mlx5_db_pgdir *pgdir; |
190 | int ret = 0; |
191 | |
192 | mutex_lock(&dev->priv.pgdir_mutex); |
193 | |
194 | list_for_each_entry(pgdir, &dev->priv.pgdir_list, list) |
195 | if (!mlx5_alloc_db_from_pgdir(pgdir, db)) |
196 | goto out; |
197 | |
198 | pgdir = mlx5_alloc_db_pgdir(dev, node); |
199 | if (!pgdir) { |
200 | ret = -ENOMEM; |
201 | goto out; |
202 | } |
203 | |
204 | list_add(new: &pgdir->list, head: &dev->priv.pgdir_list); |
205 | |
206 | /* This should never fail -- we just allocated an empty page: */ |
207 | WARN_ON(mlx5_alloc_db_from_pgdir(pgdir, db)); |
208 | |
209 | out: |
210 | mutex_unlock(lock: &dev->priv.pgdir_mutex); |
211 | |
212 | return ret; |
213 | } |
214 | EXPORT_SYMBOL_GPL(mlx5_db_alloc_node); |
215 | |
216 | void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db) |
217 | { |
218 | u32 db_per_page = PAGE_SIZE / cache_line_size(); |
219 | |
220 | mutex_lock(&dev->priv.pgdir_mutex); |
221 | |
222 | __set_bit(db->index, db->u.pgdir->bitmap); |
223 | |
224 | if (bitmap_full(src: db->u.pgdir->bitmap, nbits: db_per_page)) { |
225 | dma_free_coherent(dev: mlx5_core_dma_dev(dev), PAGE_SIZE, |
226 | cpu_addr: db->u.pgdir->db_page, dma_handle: db->u.pgdir->db_dma); |
227 | list_del(entry: &db->u.pgdir->list); |
228 | bitmap_free(bitmap: db->u.pgdir->bitmap); |
229 | kfree(objp: db->u.pgdir); |
230 | } |
231 | |
232 | mutex_unlock(lock: &dev->priv.pgdir_mutex); |
233 | } |
234 | EXPORT_SYMBOL_GPL(mlx5_db_free); |
235 | |
236 | void mlx5_fill_page_frag_array_perm(struct mlx5_frag_buf *buf, __be64 *pas, u8 perm) |
237 | { |
238 | int i; |
239 | |
240 | WARN_ON(perm & 0xfc); |
241 | for (i = 0; i < buf->npages; i++) |
242 | pas[i] = cpu_to_be64(buf->frags[i].map | perm); |
243 | } |
244 | EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array_perm); |
245 | |
246 | void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas) |
247 | { |
248 | mlx5_fill_page_frag_array_perm(buf, pas, 0); |
249 | } |
250 | EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array); |
251 | |