1 | /* |
2 | * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. |
3 | * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. |
4 | * |
5 | * This software is available to you under a choice of one of two |
6 | * licenses. You may choose to be licensed under the terms of the GNU |
7 | * General Public License (GPL) Version 2, available from the file |
8 | * COPYING in the main directory of this source tree, or the |
9 | * OpenIB.org BSD license below: |
10 | * |
11 | * Redistribution and use in source and binary forms, with or |
12 | * without modification, are permitted provided that the following |
13 | * conditions are met: |
14 | * |
15 | * - Redistributions of source code must retain the above |
16 | * copyright notice, this list of conditions and the following |
17 | * disclaimer. |
18 | * |
19 | * - Redistributions in binary form must reproduce the above |
20 | * copyright notice, this list of conditions and the following |
21 | * disclaimer in the documentation and/or other materials |
22 | * provided with the distribution. |
23 | * |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
31 | * SOFTWARE. |
32 | */ |
33 | |
34 | #include <linux/errno.h> |
35 | #include <linux/slab.h> |
36 | #include <linux/mm.h> |
37 | #include <linux/export.h> |
38 | #include <linux/bitmap.h> |
39 | #include <linux/dma-mapping.h> |
40 | #include <linux/vmalloc.h> |
41 | |
42 | #include "mlx4.h" |
43 | |
44 | u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) |
45 | { |
46 | u32 obj; |
47 | |
48 | spin_lock(lock: &bitmap->lock); |
49 | |
50 | obj = find_next_zero_bit(addr: bitmap->table, size: bitmap->max, offset: bitmap->last); |
51 | if (obj >= bitmap->max) { |
52 | bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) |
53 | & bitmap->mask; |
54 | obj = find_first_zero_bit(addr: bitmap->table, size: bitmap->max); |
55 | } |
56 | |
57 | if (obj < bitmap->max) { |
58 | set_bit(nr: obj, addr: bitmap->table); |
59 | bitmap->last = (obj + 1); |
60 | if (bitmap->last == bitmap->max) |
61 | bitmap->last = 0; |
62 | obj |= bitmap->top; |
63 | } else |
64 | obj = -1; |
65 | |
66 | if (obj != -1) |
67 | --bitmap->avail; |
68 | |
69 | spin_unlock(lock: &bitmap->lock); |
70 | |
71 | return obj; |
72 | } |
73 | |
74 | void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr) |
75 | { |
76 | mlx4_bitmap_free_range(bitmap, obj, cnt: 1, use_rr); |
77 | } |
78 | |
79 | static unsigned long find_aligned_range(unsigned long *bitmap, |
80 | u32 start, u32 nbits, |
81 | int len, int align, u32 skip_mask) |
82 | { |
83 | unsigned long end, i; |
84 | |
85 | again: |
86 | start = ALIGN(start, align); |
87 | |
88 | while ((start < nbits) && (test_bit(start, bitmap) || |
89 | (start & skip_mask))) |
90 | start += align; |
91 | |
92 | if (start >= nbits) |
93 | return -1; |
94 | |
95 | end = start+len; |
96 | if (end > nbits) |
97 | return -1; |
98 | |
99 | for (i = start + 1; i < end; i++) { |
100 | if (test_bit(i, bitmap) || ((u32)i & skip_mask)) { |
101 | start = i + 1; |
102 | goto again; |
103 | } |
104 | } |
105 | |
106 | return start; |
107 | } |
108 | |
109 | u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, |
110 | int align, u32 skip_mask) |
111 | { |
112 | u32 obj; |
113 | |
114 | if (likely(cnt == 1 && align == 1 && !skip_mask)) |
115 | return mlx4_bitmap_alloc(bitmap); |
116 | |
117 | spin_lock(lock: &bitmap->lock); |
118 | |
119 | obj = find_aligned_range(bitmap: bitmap->table, start: bitmap->last, |
120 | nbits: bitmap->max, len: cnt, align, skip_mask); |
121 | if (obj >= bitmap->max) { |
122 | bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) |
123 | & bitmap->mask; |
124 | obj = find_aligned_range(bitmap: bitmap->table, start: 0, nbits: bitmap->max, |
125 | len: cnt, align, skip_mask); |
126 | } |
127 | |
128 | if (obj < bitmap->max) { |
129 | bitmap_set(map: bitmap->table, start: obj, nbits: cnt); |
130 | if (obj == bitmap->last) { |
131 | bitmap->last = (obj + cnt); |
132 | if (bitmap->last >= bitmap->max) |
133 | bitmap->last = 0; |
134 | } |
135 | obj |= bitmap->top; |
136 | } else |
137 | obj = -1; |
138 | |
139 | if (obj != -1) |
140 | bitmap->avail -= cnt; |
141 | |
142 | spin_unlock(lock: &bitmap->lock); |
143 | |
144 | return obj; |
145 | } |
146 | |
147 | u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) |
148 | { |
149 | return bitmap->avail; |
150 | } |
151 | |
152 | static u32 mlx4_bitmap_masked_value(struct mlx4_bitmap *bitmap, u32 obj) |
153 | { |
154 | return obj & (bitmap->max + bitmap->reserved_top - 1); |
155 | } |
156 | |
157 | void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, |
158 | int use_rr) |
159 | { |
160 | obj &= bitmap->max + bitmap->reserved_top - 1; |
161 | |
162 | spin_lock(lock: &bitmap->lock); |
163 | if (!use_rr) { |
164 | bitmap->last = min(bitmap->last, obj); |
165 | bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) |
166 | & bitmap->mask; |
167 | } |
168 | bitmap_clear(map: bitmap->table, start: obj, nbits: cnt); |
169 | bitmap->avail += cnt; |
170 | spin_unlock(lock: &bitmap->lock); |
171 | } |
172 | |
173 | int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, |
174 | u32 reserved_bot, u32 reserved_top) |
175 | { |
176 | /* num must be a power of 2 */ |
177 | if (num != roundup_pow_of_two(num)) |
178 | return -EINVAL; |
179 | |
180 | bitmap->last = 0; |
181 | bitmap->top = 0; |
182 | bitmap->max = num - reserved_top; |
183 | bitmap->mask = mask; |
184 | bitmap->reserved_top = reserved_top; |
185 | bitmap->avail = num - reserved_top - reserved_bot; |
186 | bitmap->effective_len = bitmap->avail; |
187 | spin_lock_init(&bitmap->lock); |
188 | bitmap->table = bitmap_zalloc(nbits: bitmap->max, GFP_KERNEL); |
189 | if (!bitmap->table) |
190 | return -ENOMEM; |
191 | |
192 | bitmap_set(map: bitmap->table, start: 0, nbits: reserved_bot); |
193 | |
194 | return 0; |
195 | } |
196 | |
197 | void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) |
198 | { |
199 | bitmap_free(bitmap: bitmap->table); |
200 | } |
201 | |
202 | struct mlx4_zone_allocator { |
203 | struct list_head entries; |
204 | struct list_head prios; |
205 | u32 last_uid; |
206 | u32 mask; |
207 | /* protect the zone_allocator from concurrent accesses */ |
208 | spinlock_t lock; |
209 | enum mlx4_zone_alloc_flags flags; |
210 | }; |
211 | |
212 | struct mlx4_zone_entry { |
213 | struct list_head list; |
214 | struct list_head prio_list; |
215 | u32 uid; |
216 | struct mlx4_zone_allocator *allocator; |
217 | struct mlx4_bitmap *bitmap; |
218 | int use_rr; |
219 | int priority; |
220 | int offset; |
221 | enum mlx4_zone_flags flags; |
222 | }; |
223 | |
224 | struct mlx4_zone_allocator *mlx4_zone_allocator_create(enum mlx4_zone_alloc_flags flags) |
225 | { |
226 | struct mlx4_zone_allocator *zones = kmalloc(size: sizeof(*zones), GFP_KERNEL); |
227 | |
228 | if (NULL == zones) |
229 | return NULL; |
230 | |
231 | INIT_LIST_HEAD(list: &zones->entries); |
232 | INIT_LIST_HEAD(list: &zones->prios); |
233 | spin_lock_init(&zones->lock); |
234 | zones->last_uid = 0; |
235 | zones->mask = 0; |
236 | zones->flags = flags; |
237 | |
238 | return zones; |
239 | } |
240 | |
241 | int mlx4_zone_add_one(struct mlx4_zone_allocator *zone_alloc, |
242 | struct mlx4_bitmap *bitmap, |
243 | u32 flags, |
244 | int priority, |
245 | int offset, |
246 | u32 *puid) |
247 | { |
248 | u32 mask = mlx4_bitmap_masked_value(bitmap, obj: (u32)-1); |
249 | struct mlx4_zone_entry *it; |
250 | struct mlx4_zone_entry *zone = kmalloc(size: sizeof(*zone), GFP_KERNEL); |
251 | |
252 | if (NULL == zone) |
253 | return -ENOMEM; |
254 | |
255 | zone->flags = flags; |
256 | zone->bitmap = bitmap; |
257 | zone->use_rr = (flags & MLX4_ZONE_USE_RR) ? MLX4_USE_RR : 0; |
258 | zone->priority = priority; |
259 | zone->offset = offset; |
260 | |
261 | spin_lock(lock: &zone_alloc->lock); |
262 | |
263 | zone->uid = zone_alloc->last_uid++; |
264 | zone->allocator = zone_alloc; |
265 | |
266 | if (zone_alloc->mask < mask) |
267 | zone_alloc->mask = mask; |
268 | |
269 | list_for_each_entry(it, &zone_alloc->prios, prio_list) |
270 | if (it->priority >= priority) |
271 | break; |
272 | |
273 | if (&it->prio_list == &zone_alloc->prios || it->priority > priority) |
274 | list_add_tail(new: &zone->prio_list, head: &it->prio_list); |
275 | list_add_tail(new: &zone->list, head: &it->list); |
276 | |
277 | spin_unlock(lock: &zone_alloc->lock); |
278 | |
279 | *puid = zone->uid; |
280 | |
281 | return 0; |
282 | } |
283 | |
284 | /* Should be called under a lock */ |
285 | static void __mlx4_zone_remove_one_entry(struct mlx4_zone_entry *entry) |
286 | { |
287 | struct mlx4_zone_allocator *zone_alloc = entry->allocator; |
288 | |
289 | if (!list_empty(head: &entry->prio_list)) { |
290 | /* Check if we need to add an alternative node to the prio list */ |
291 | if (!list_is_last(list: &entry->list, head: &zone_alloc->entries)) { |
292 | struct mlx4_zone_entry *next = list_first_entry(&entry->list, |
293 | typeof(*next), |
294 | list); |
295 | |
296 | if (next->priority == entry->priority) |
297 | list_add_tail(new: &next->prio_list, head: &entry->prio_list); |
298 | } |
299 | |
300 | list_del(entry: &entry->prio_list); |
301 | } |
302 | |
303 | list_del(entry: &entry->list); |
304 | |
305 | if (zone_alloc->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP) { |
306 | u32 mask = 0; |
307 | struct mlx4_zone_entry *it; |
308 | |
309 | list_for_each_entry(it, &zone_alloc->prios, prio_list) { |
310 | u32 cur_mask = mlx4_bitmap_masked_value(bitmap: it->bitmap, obj: (u32)-1); |
311 | |
312 | if (mask < cur_mask) |
313 | mask = cur_mask; |
314 | } |
315 | zone_alloc->mask = mask; |
316 | } |
317 | } |
318 | |
319 | void mlx4_zone_allocator_destroy(struct mlx4_zone_allocator *zone_alloc) |
320 | { |
321 | struct mlx4_zone_entry *zone, *tmp; |
322 | |
323 | spin_lock(lock: &zone_alloc->lock); |
324 | |
325 | list_for_each_entry_safe(zone, tmp, &zone_alloc->entries, list) { |
326 | list_del(entry: &zone->list); |
327 | list_del(entry: &zone->prio_list); |
328 | kfree(objp: zone); |
329 | } |
330 | |
331 | spin_unlock(lock: &zone_alloc->lock); |
332 | kfree(objp: zone_alloc); |
333 | } |
334 | |
335 | /* Should be called under a lock */ |
336 | static u32 __mlx4_alloc_from_zone(struct mlx4_zone_entry *zone, int count, |
337 | int align, u32 skip_mask, u32 *puid) |
338 | { |
339 | u32 uid = 0; |
340 | u32 res; |
341 | struct mlx4_zone_allocator *zone_alloc = zone->allocator; |
342 | struct mlx4_zone_entry *curr_node; |
343 | |
344 | res = mlx4_bitmap_alloc_range(bitmap: zone->bitmap, cnt: count, |
345 | align, skip_mask); |
346 | |
347 | if (res != (u32)-1) { |
348 | res += zone->offset; |
349 | uid = zone->uid; |
350 | goto out; |
351 | } |
352 | |
353 | list_for_each_entry(curr_node, &zone_alloc->prios, prio_list) { |
354 | if (unlikely(curr_node->priority == zone->priority)) |
355 | break; |
356 | } |
357 | |
358 | if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO) { |
359 | struct mlx4_zone_entry *it = curr_node; |
360 | |
361 | list_for_each_entry_continue_reverse(it, &zone_alloc->entries, list) { |
362 | res = mlx4_bitmap_alloc_range(bitmap: it->bitmap, cnt: count, |
363 | align, skip_mask); |
364 | if (res != (u32)-1) { |
365 | res += it->offset; |
366 | uid = it->uid; |
367 | goto out; |
368 | } |
369 | } |
370 | } |
371 | |
372 | if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO) { |
373 | struct mlx4_zone_entry *it = curr_node; |
374 | |
375 | list_for_each_entry_from(it, &zone_alloc->entries, list) { |
376 | if (unlikely(it == zone)) |
377 | continue; |
378 | |
379 | if (unlikely(it->priority != curr_node->priority)) |
380 | break; |
381 | |
382 | res = mlx4_bitmap_alloc_range(bitmap: it->bitmap, cnt: count, |
383 | align, skip_mask); |
384 | if (res != (u32)-1) { |
385 | res += it->offset; |
386 | uid = it->uid; |
387 | goto out; |
388 | } |
389 | } |
390 | } |
391 | |
392 | if (zone->flags & MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO) { |
393 | if (list_is_last(list: &curr_node->prio_list, head: &zone_alloc->prios)) |
394 | goto out; |
395 | |
396 | curr_node = list_first_entry(&curr_node->prio_list, |
397 | typeof(*curr_node), |
398 | prio_list); |
399 | |
400 | list_for_each_entry_from(curr_node, &zone_alloc->entries, list) { |
401 | res = mlx4_bitmap_alloc_range(bitmap: curr_node->bitmap, cnt: count, |
402 | align, skip_mask); |
403 | if (res != (u32)-1) { |
404 | res += curr_node->offset; |
405 | uid = curr_node->uid; |
406 | goto out; |
407 | } |
408 | } |
409 | } |
410 | |
411 | out: |
412 | if (NULL != puid && res != (u32)-1) |
413 | *puid = uid; |
414 | return res; |
415 | } |
416 | |
417 | /* Should be called under a lock */ |
418 | static void __mlx4_free_from_zone(struct mlx4_zone_entry *zone, u32 obj, |
419 | u32 count) |
420 | { |
421 | mlx4_bitmap_free_range(bitmap: zone->bitmap, obj: obj - zone->offset, cnt: count, use_rr: zone->use_rr); |
422 | } |
423 | |
424 | /* Should be called under a lock */ |
425 | static struct mlx4_zone_entry *__mlx4_find_zone_by_uid( |
426 | struct mlx4_zone_allocator *zones, u32 uid) |
427 | { |
428 | struct mlx4_zone_entry *zone; |
429 | |
430 | list_for_each_entry(zone, &zones->entries, list) { |
431 | if (zone->uid == uid) |
432 | return zone; |
433 | } |
434 | |
435 | return NULL; |
436 | } |
437 | |
438 | struct mlx4_bitmap *mlx4_zone_get_bitmap(struct mlx4_zone_allocator *zones, u32 uid) |
439 | { |
440 | struct mlx4_zone_entry *zone; |
441 | struct mlx4_bitmap *bitmap; |
442 | |
443 | spin_lock(lock: &zones->lock); |
444 | |
445 | zone = __mlx4_find_zone_by_uid(zones, uid); |
446 | |
447 | bitmap = zone == NULL ? NULL : zone->bitmap; |
448 | |
449 | spin_unlock(lock: &zones->lock); |
450 | |
451 | return bitmap; |
452 | } |
453 | |
454 | int mlx4_zone_remove_one(struct mlx4_zone_allocator *zones, u32 uid) |
455 | { |
456 | struct mlx4_zone_entry *zone; |
457 | int res = 0; |
458 | |
459 | spin_lock(lock: &zones->lock); |
460 | |
461 | zone = __mlx4_find_zone_by_uid(zones, uid); |
462 | |
463 | if (NULL == zone) { |
464 | res = -1; |
465 | goto out; |
466 | } |
467 | |
468 | __mlx4_zone_remove_one_entry(entry: zone); |
469 | |
470 | out: |
471 | spin_unlock(lock: &zones->lock); |
472 | kfree(objp: zone); |
473 | |
474 | return res; |
475 | } |
476 | |
477 | /* Should be called under a lock */ |
478 | static struct mlx4_zone_entry *__mlx4_find_zone_by_uid_unique( |
479 | struct mlx4_zone_allocator *zones, u32 obj) |
480 | { |
481 | struct mlx4_zone_entry *zone, *zone_candidate = NULL; |
482 | u32 dist = (u32)-1; |
483 | |
484 | /* Search for the smallest zone that this obj could be |
485 | * allocated from. This is done in order to handle |
486 | * situations when small bitmaps are allocated from bigger |
487 | * bitmaps (and the allocated space is marked as reserved in |
488 | * the bigger bitmap. |
489 | */ |
490 | list_for_each_entry(zone, &zones->entries, list) { |
491 | if (obj >= zone->offset) { |
492 | u32 mobj = (obj - zone->offset) & zones->mask; |
493 | |
494 | if (mobj < zone->bitmap->max) { |
495 | u32 curr_dist = zone->bitmap->effective_len; |
496 | |
497 | if (curr_dist < dist) { |
498 | dist = curr_dist; |
499 | zone_candidate = zone; |
500 | } |
501 | } |
502 | } |
503 | } |
504 | |
505 | return zone_candidate; |
506 | } |
507 | |
508 | u32 mlx4_zone_alloc_entries(struct mlx4_zone_allocator *zones, u32 uid, int count, |
509 | int align, u32 skip_mask, u32 *puid) |
510 | { |
511 | struct mlx4_zone_entry *zone; |
512 | int res = -1; |
513 | |
514 | spin_lock(lock: &zones->lock); |
515 | |
516 | zone = __mlx4_find_zone_by_uid(zones, uid); |
517 | |
518 | if (NULL == zone) |
519 | goto out; |
520 | |
521 | res = __mlx4_alloc_from_zone(zone, count, align, skip_mask, puid); |
522 | |
523 | out: |
524 | spin_unlock(lock: &zones->lock); |
525 | |
526 | return res; |
527 | } |
528 | |
529 | u32 mlx4_zone_free_entries(struct mlx4_zone_allocator *zones, u32 uid, u32 obj, u32 count) |
530 | { |
531 | struct mlx4_zone_entry *zone; |
532 | int res = 0; |
533 | |
534 | spin_lock(lock: &zones->lock); |
535 | |
536 | zone = __mlx4_find_zone_by_uid(zones, uid); |
537 | |
538 | if (NULL == zone) { |
539 | res = -1; |
540 | goto out; |
541 | } |
542 | |
543 | __mlx4_free_from_zone(zone, obj, count); |
544 | |
545 | out: |
546 | spin_unlock(lock: &zones->lock); |
547 | |
548 | return res; |
549 | } |
550 | |
551 | u32 mlx4_zone_free_entries_unique(struct mlx4_zone_allocator *zones, u32 obj, u32 count) |
552 | { |
553 | struct mlx4_zone_entry *zone; |
554 | int res; |
555 | |
556 | if (!(zones->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP)) |
557 | return -EFAULT; |
558 | |
559 | spin_lock(lock: &zones->lock); |
560 | |
561 | zone = __mlx4_find_zone_by_uid_unique(zones, obj); |
562 | |
563 | if (NULL == zone) { |
564 | res = -1; |
565 | goto out; |
566 | } |
567 | |
568 | __mlx4_free_from_zone(zone, obj, count); |
569 | res = 0; |
570 | |
571 | out: |
572 | spin_unlock(lock: &zones->lock); |
573 | |
574 | return res; |
575 | } |
576 | |
577 | static int mlx4_buf_direct_alloc(struct mlx4_dev *dev, int size, |
578 | struct mlx4_buf *buf) |
579 | { |
580 | dma_addr_t t; |
581 | |
582 | buf->nbufs = 1; |
583 | buf->npages = 1; |
584 | buf->page_shift = get_order(size) + PAGE_SHIFT; |
585 | buf->direct.buf = |
586 | dma_alloc_coherent(dev: &dev->persist->pdev->dev, size, dma_handle: &t, |
587 | GFP_KERNEL); |
588 | if (!buf->direct.buf) |
589 | return -ENOMEM; |
590 | |
591 | buf->direct.map = t; |
592 | |
593 | while (t & ((1 << buf->page_shift) - 1)) { |
594 | --buf->page_shift; |
595 | buf->npages *= 2; |
596 | } |
597 | |
598 | return 0; |
599 | } |
600 | |
601 | /* Handling for queue buffers -- we allocate a bunch of memory and |
602 | * register it in a memory region at HCA virtual address 0. If the |
603 | * requested size is > max_direct, we split the allocation into |
604 | * multiple pages, so we don't require too much contiguous memory. |
605 | */ |
606 | int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, |
607 | struct mlx4_buf *buf) |
608 | { |
609 | if (size <= max_direct) { |
610 | return mlx4_buf_direct_alloc(dev, size, buf); |
611 | } else { |
612 | dma_addr_t t; |
613 | int i; |
614 | |
615 | buf->direct.buf = NULL; |
616 | buf->nbufs = DIV_ROUND_UP(size, PAGE_SIZE); |
617 | buf->npages = buf->nbufs; |
618 | buf->page_shift = PAGE_SHIFT; |
619 | buf->page_list = kcalloc(n: buf->nbufs, size: sizeof(*buf->page_list), |
620 | GFP_KERNEL); |
621 | if (!buf->page_list) |
622 | return -ENOMEM; |
623 | |
624 | for (i = 0; i < buf->nbufs; ++i) { |
625 | buf->page_list[i].buf = |
626 | dma_alloc_coherent(dev: &dev->persist->pdev->dev, |
627 | PAGE_SIZE, dma_handle: &t, GFP_KERNEL); |
628 | if (!buf->page_list[i].buf) |
629 | goto err_free; |
630 | |
631 | buf->page_list[i].map = t; |
632 | } |
633 | } |
634 | |
635 | return 0; |
636 | |
637 | err_free: |
638 | mlx4_buf_free(dev, size, buf); |
639 | |
640 | return -ENOMEM; |
641 | } |
642 | EXPORT_SYMBOL_GPL(mlx4_buf_alloc); |
643 | |
644 | void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) |
645 | { |
646 | if (buf->nbufs == 1) { |
647 | dma_free_coherent(dev: &dev->persist->pdev->dev, size, |
648 | cpu_addr: buf->direct.buf, dma_handle: buf->direct.map); |
649 | } else { |
650 | int i; |
651 | |
652 | for (i = 0; i < buf->nbufs; ++i) |
653 | if (buf->page_list[i].buf) |
654 | dma_free_coherent(dev: &dev->persist->pdev->dev, |
655 | PAGE_SIZE, |
656 | cpu_addr: buf->page_list[i].buf, |
657 | dma_handle: buf->page_list[i].map); |
658 | kfree(objp: buf->page_list); |
659 | } |
660 | } |
661 | EXPORT_SYMBOL_GPL(mlx4_buf_free); |
662 | |
663 | static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device) |
664 | { |
665 | struct mlx4_db_pgdir *pgdir; |
666 | |
667 | pgdir = kzalloc(size: sizeof(*pgdir), GFP_KERNEL); |
668 | if (!pgdir) |
669 | return NULL; |
670 | |
671 | bitmap_fill(dst: pgdir->order1, nbits: MLX4_DB_PER_PAGE / 2); |
672 | pgdir->bits[0] = pgdir->order0; |
673 | pgdir->bits[1] = pgdir->order1; |
674 | pgdir->db_page = dma_alloc_coherent(dev: dma_device, PAGE_SIZE, |
675 | dma_handle: &pgdir->db_dma, GFP_KERNEL); |
676 | if (!pgdir->db_page) { |
677 | kfree(objp: pgdir); |
678 | return NULL; |
679 | } |
680 | |
681 | return pgdir; |
682 | } |
683 | |
684 | static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, |
685 | struct mlx4_db *db, int order) |
686 | { |
687 | int o; |
688 | int i; |
689 | |
690 | for (o = order; o <= 1; ++o) { |
691 | i = find_first_bit(addr: pgdir->bits[o], size: MLX4_DB_PER_PAGE >> o); |
692 | if (i < MLX4_DB_PER_PAGE >> o) |
693 | goto found; |
694 | } |
695 | |
696 | return -ENOMEM; |
697 | |
698 | found: |
699 | clear_bit(nr: i, addr: pgdir->bits[o]); |
700 | |
701 | i <<= o; |
702 | |
703 | if (o > order) |
704 | set_bit(nr: i ^ 1, addr: pgdir->bits[order]); |
705 | |
706 | db->u.pgdir = pgdir; |
707 | db->index = i; |
708 | db->db = pgdir->db_page + db->index; |
709 | db->dma = pgdir->db_dma + db->index * 4; |
710 | db->order = order; |
711 | |
712 | return 0; |
713 | } |
714 | |
715 | int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order) |
716 | { |
717 | struct mlx4_priv *priv = mlx4_priv(dev); |
718 | struct mlx4_db_pgdir *pgdir; |
719 | int ret = 0; |
720 | |
721 | mutex_lock(&priv->pgdir_mutex); |
722 | |
723 | list_for_each_entry(pgdir, &priv->pgdir_list, list) |
724 | if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) |
725 | goto out; |
726 | |
727 | pgdir = mlx4_alloc_db_pgdir(dma_device: &dev->persist->pdev->dev); |
728 | if (!pgdir) { |
729 | ret = -ENOMEM; |
730 | goto out; |
731 | } |
732 | |
733 | list_add(new: &pgdir->list, head: &priv->pgdir_list); |
734 | |
735 | /* This should never fail -- we just allocated an empty page: */ |
736 | WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); |
737 | |
738 | out: |
739 | mutex_unlock(lock: &priv->pgdir_mutex); |
740 | |
741 | return ret; |
742 | } |
743 | EXPORT_SYMBOL_GPL(mlx4_db_alloc); |
744 | |
745 | void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) |
746 | { |
747 | struct mlx4_priv *priv = mlx4_priv(dev); |
748 | int o; |
749 | int i; |
750 | |
751 | mutex_lock(&priv->pgdir_mutex); |
752 | |
753 | o = db->order; |
754 | i = db->index; |
755 | |
756 | if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { |
757 | clear_bit(nr: i ^ 1, addr: db->u.pgdir->order0); |
758 | ++o; |
759 | } |
760 | i >>= o; |
761 | set_bit(nr: i, addr: db->u.pgdir->bits[o]); |
762 | |
763 | if (bitmap_full(src: db->u.pgdir->order1, nbits: MLX4_DB_PER_PAGE / 2)) { |
764 | dma_free_coherent(dev: &dev->persist->pdev->dev, PAGE_SIZE, |
765 | cpu_addr: db->u.pgdir->db_page, dma_handle: db->u.pgdir->db_dma); |
766 | list_del(entry: &db->u.pgdir->list); |
767 | kfree(objp: db->u.pgdir); |
768 | } |
769 | |
770 | mutex_unlock(lock: &priv->pgdir_mutex); |
771 | } |
772 | EXPORT_SYMBOL_GPL(mlx4_db_free); |
773 | |
774 | int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, |
775 | int size) |
776 | { |
777 | int err; |
778 | |
779 | err = mlx4_db_alloc(dev, &wqres->db, 1); |
780 | if (err) |
781 | return err; |
782 | |
783 | *wqres->db.db = 0; |
784 | |
785 | err = mlx4_buf_direct_alloc(dev, size, buf: &wqres->buf); |
786 | if (err) |
787 | goto err_db; |
788 | |
789 | err = mlx4_mtt_init(dev, npages: wqres->buf.npages, page_shift: wqres->buf.page_shift, |
790 | mtt: &wqres->mtt); |
791 | if (err) |
792 | goto err_buf; |
793 | |
794 | err = mlx4_buf_write_mtt(dev, mtt: &wqres->mtt, buf: &wqres->buf); |
795 | if (err) |
796 | goto err_mtt; |
797 | |
798 | return 0; |
799 | |
800 | err_mtt: |
801 | mlx4_mtt_cleanup(dev, mtt: &wqres->mtt); |
802 | err_buf: |
803 | mlx4_buf_free(dev, size, &wqres->buf); |
804 | err_db: |
805 | mlx4_db_free(dev, &wqres->db); |
806 | |
807 | return err; |
808 | } |
809 | EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); |
810 | |
811 | void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, |
812 | int size) |
813 | { |
814 | mlx4_mtt_cleanup(dev, mtt: &wqres->mtt); |
815 | mlx4_buf_free(dev, size, &wqres->buf); |
816 | mlx4_db_free(dev, &wqres->db); |
817 | } |
818 | EXPORT_SYMBOL_GPL(mlx4_free_hwq_res); |
819 | |