1/*
2 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#include <linux/errno.h>
35#include <linux/slab.h>
36#include <linux/mm.h>
37#include <linux/export.h>
38#include <linux/bitmap.h>
39#include <linux/dma-mapping.h>
40#include <linux/vmalloc.h>
41
42#include "mlx4.h"
43
44u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap)
45{
46 u32 obj;
47
48 spin_lock(lock: &bitmap->lock);
49
50 obj = find_next_zero_bit(addr: bitmap->table, size: bitmap->max, offset: bitmap->last);
51 if (obj >= bitmap->max) {
52 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
53 & bitmap->mask;
54 obj = find_first_zero_bit(addr: bitmap->table, size: bitmap->max);
55 }
56
57 if (obj < bitmap->max) {
58 set_bit(nr: obj, addr: bitmap->table);
59 bitmap->last = (obj + 1);
60 if (bitmap->last == bitmap->max)
61 bitmap->last = 0;
62 obj |= bitmap->top;
63 } else
64 obj = -1;
65
66 if (obj != -1)
67 --bitmap->avail;
68
69 spin_unlock(lock: &bitmap->lock);
70
71 return obj;
72}
73
74void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr)
75{
76 mlx4_bitmap_free_range(bitmap, obj, cnt: 1, use_rr);
77}
78
79static unsigned long find_aligned_range(unsigned long *bitmap,
80 u32 start, u32 nbits,
81 int len, int align, u32 skip_mask)
82{
83 unsigned long end, i;
84
85again:
86 start = ALIGN(start, align);
87
88 while ((start < nbits) && (test_bit(start, bitmap) ||
89 (start & skip_mask)))
90 start += align;
91
92 if (start >= nbits)
93 return -1;
94
95 end = start+len;
96 if (end > nbits)
97 return -1;
98
99 for (i = start + 1; i < end; i++) {
100 if (test_bit(i, bitmap) || ((u32)i & skip_mask)) {
101 start = i + 1;
102 goto again;
103 }
104 }
105
106 return start;
107}
108
109u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt,
110 int align, u32 skip_mask)
111{
112 u32 obj;
113
114 if (likely(cnt == 1 && align == 1 && !skip_mask))
115 return mlx4_bitmap_alloc(bitmap);
116
117 spin_lock(lock: &bitmap->lock);
118
119 obj = find_aligned_range(bitmap: bitmap->table, start: bitmap->last,
120 nbits: bitmap->max, len: cnt, align, skip_mask);
121 if (obj >= bitmap->max) {
122 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
123 & bitmap->mask;
124 obj = find_aligned_range(bitmap: bitmap->table, start: 0, nbits: bitmap->max,
125 len: cnt, align, skip_mask);
126 }
127
128 if (obj < bitmap->max) {
129 bitmap_set(map: bitmap->table, start: obj, nbits: cnt);
130 if (obj == bitmap->last) {
131 bitmap->last = (obj + cnt);
132 if (bitmap->last >= bitmap->max)
133 bitmap->last = 0;
134 }
135 obj |= bitmap->top;
136 } else
137 obj = -1;
138
139 if (obj != -1)
140 bitmap->avail -= cnt;
141
142 spin_unlock(lock: &bitmap->lock);
143
144 return obj;
145}
146
147u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap)
148{
149 return bitmap->avail;
150}
151
152static u32 mlx4_bitmap_masked_value(struct mlx4_bitmap *bitmap, u32 obj)
153{
154 return obj & (bitmap->max + bitmap->reserved_top - 1);
155}
156
157void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt,
158 int use_rr)
159{
160 obj &= bitmap->max + bitmap->reserved_top - 1;
161
162 spin_lock(lock: &bitmap->lock);
163 if (!use_rr) {
164 bitmap->last = min(bitmap->last, obj);
165 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
166 & bitmap->mask;
167 }
168 bitmap_clear(map: bitmap->table, start: obj, nbits: cnt);
169 bitmap->avail += cnt;
170 spin_unlock(lock: &bitmap->lock);
171}
172
173int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask,
174 u32 reserved_bot, u32 reserved_top)
175{
176 /* num must be a power of 2 */
177 if (num != roundup_pow_of_two(num))
178 return -EINVAL;
179
180 bitmap->last = 0;
181 bitmap->top = 0;
182 bitmap->max = num - reserved_top;
183 bitmap->mask = mask;
184 bitmap->reserved_top = reserved_top;
185 bitmap->avail = num - reserved_top - reserved_bot;
186 bitmap->effective_len = bitmap->avail;
187 spin_lock_init(&bitmap->lock);
188 bitmap->table = bitmap_zalloc(nbits: bitmap->max, GFP_KERNEL);
189 if (!bitmap->table)
190 return -ENOMEM;
191
192 bitmap_set(map: bitmap->table, start: 0, nbits: reserved_bot);
193
194 return 0;
195}
196
197void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap)
198{
199 bitmap_free(bitmap: bitmap->table);
200}
201
202struct mlx4_zone_allocator {
203 struct list_head entries;
204 struct list_head prios;
205 u32 last_uid;
206 u32 mask;
207 /* protect the zone_allocator from concurrent accesses */
208 spinlock_t lock;
209 enum mlx4_zone_alloc_flags flags;
210};
211
212struct mlx4_zone_entry {
213 struct list_head list;
214 struct list_head prio_list;
215 u32 uid;
216 struct mlx4_zone_allocator *allocator;
217 struct mlx4_bitmap *bitmap;
218 int use_rr;
219 int priority;
220 int offset;
221 enum mlx4_zone_flags flags;
222};
223
224struct mlx4_zone_allocator *mlx4_zone_allocator_create(enum mlx4_zone_alloc_flags flags)
225{
226 struct mlx4_zone_allocator *zones = kmalloc(size: sizeof(*zones), GFP_KERNEL);
227
228 if (NULL == zones)
229 return NULL;
230
231 INIT_LIST_HEAD(list: &zones->entries);
232 INIT_LIST_HEAD(list: &zones->prios);
233 spin_lock_init(&zones->lock);
234 zones->last_uid = 0;
235 zones->mask = 0;
236 zones->flags = flags;
237
238 return zones;
239}
240
241int mlx4_zone_add_one(struct mlx4_zone_allocator *zone_alloc,
242 struct mlx4_bitmap *bitmap,
243 u32 flags,
244 int priority,
245 int offset,
246 u32 *puid)
247{
248 u32 mask = mlx4_bitmap_masked_value(bitmap, obj: (u32)-1);
249 struct mlx4_zone_entry *it;
250 struct mlx4_zone_entry *zone = kmalloc(size: sizeof(*zone), GFP_KERNEL);
251
252 if (NULL == zone)
253 return -ENOMEM;
254
255 zone->flags = flags;
256 zone->bitmap = bitmap;
257 zone->use_rr = (flags & MLX4_ZONE_USE_RR) ? MLX4_USE_RR : 0;
258 zone->priority = priority;
259 zone->offset = offset;
260
261 spin_lock(lock: &zone_alloc->lock);
262
263 zone->uid = zone_alloc->last_uid++;
264 zone->allocator = zone_alloc;
265
266 if (zone_alloc->mask < mask)
267 zone_alloc->mask = mask;
268
269 list_for_each_entry(it, &zone_alloc->prios, prio_list)
270 if (it->priority >= priority)
271 break;
272
273 if (&it->prio_list == &zone_alloc->prios || it->priority > priority)
274 list_add_tail(new: &zone->prio_list, head: &it->prio_list);
275 list_add_tail(new: &zone->list, head: &it->list);
276
277 spin_unlock(lock: &zone_alloc->lock);
278
279 *puid = zone->uid;
280
281 return 0;
282}
283
284/* Should be called under a lock */
285static void __mlx4_zone_remove_one_entry(struct mlx4_zone_entry *entry)
286{
287 struct mlx4_zone_allocator *zone_alloc = entry->allocator;
288
289 if (!list_empty(head: &entry->prio_list)) {
290 /* Check if we need to add an alternative node to the prio list */
291 if (!list_is_last(list: &entry->list, head: &zone_alloc->entries)) {
292 struct mlx4_zone_entry *next = list_first_entry(&entry->list,
293 typeof(*next),
294 list);
295
296 if (next->priority == entry->priority)
297 list_add_tail(new: &next->prio_list, head: &entry->prio_list);
298 }
299
300 list_del(entry: &entry->prio_list);
301 }
302
303 list_del(entry: &entry->list);
304
305 if (zone_alloc->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP) {
306 u32 mask = 0;
307 struct mlx4_zone_entry *it;
308
309 list_for_each_entry(it, &zone_alloc->prios, prio_list) {
310 u32 cur_mask = mlx4_bitmap_masked_value(bitmap: it->bitmap, obj: (u32)-1);
311
312 if (mask < cur_mask)
313 mask = cur_mask;
314 }
315 zone_alloc->mask = mask;
316 }
317}
318
319void mlx4_zone_allocator_destroy(struct mlx4_zone_allocator *zone_alloc)
320{
321 struct mlx4_zone_entry *zone, *tmp;
322
323 spin_lock(lock: &zone_alloc->lock);
324
325 list_for_each_entry_safe(zone, tmp, &zone_alloc->entries, list) {
326 list_del(entry: &zone->list);
327 list_del(entry: &zone->prio_list);
328 kfree(objp: zone);
329 }
330
331 spin_unlock(lock: &zone_alloc->lock);
332 kfree(objp: zone_alloc);
333}
334
335/* Should be called under a lock */
336static u32 __mlx4_alloc_from_zone(struct mlx4_zone_entry *zone, int count,
337 int align, u32 skip_mask, u32 *puid)
338{
339 u32 uid = 0;
340 u32 res;
341 struct mlx4_zone_allocator *zone_alloc = zone->allocator;
342 struct mlx4_zone_entry *curr_node;
343
344 res = mlx4_bitmap_alloc_range(bitmap: zone->bitmap, cnt: count,
345 align, skip_mask);
346
347 if (res != (u32)-1) {
348 res += zone->offset;
349 uid = zone->uid;
350 goto out;
351 }
352
353 list_for_each_entry(curr_node, &zone_alloc->prios, prio_list) {
354 if (unlikely(curr_node->priority == zone->priority))
355 break;
356 }
357
358 if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO) {
359 struct mlx4_zone_entry *it = curr_node;
360
361 list_for_each_entry_continue_reverse(it, &zone_alloc->entries, list) {
362 res = mlx4_bitmap_alloc_range(bitmap: it->bitmap, cnt: count,
363 align, skip_mask);
364 if (res != (u32)-1) {
365 res += it->offset;
366 uid = it->uid;
367 goto out;
368 }
369 }
370 }
371
372 if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO) {
373 struct mlx4_zone_entry *it = curr_node;
374
375 list_for_each_entry_from(it, &zone_alloc->entries, list) {
376 if (unlikely(it == zone))
377 continue;
378
379 if (unlikely(it->priority != curr_node->priority))
380 break;
381
382 res = mlx4_bitmap_alloc_range(bitmap: it->bitmap, cnt: count,
383 align, skip_mask);
384 if (res != (u32)-1) {
385 res += it->offset;
386 uid = it->uid;
387 goto out;
388 }
389 }
390 }
391
392 if (zone->flags & MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO) {
393 if (list_is_last(list: &curr_node->prio_list, head: &zone_alloc->prios))
394 goto out;
395
396 curr_node = list_first_entry(&curr_node->prio_list,
397 typeof(*curr_node),
398 prio_list);
399
400 list_for_each_entry_from(curr_node, &zone_alloc->entries, list) {
401 res = mlx4_bitmap_alloc_range(bitmap: curr_node->bitmap, cnt: count,
402 align, skip_mask);
403 if (res != (u32)-1) {
404 res += curr_node->offset;
405 uid = curr_node->uid;
406 goto out;
407 }
408 }
409 }
410
411out:
412 if (NULL != puid && res != (u32)-1)
413 *puid = uid;
414 return res;
415}
416
417/* Should be called under a lock */
418static void __mlx4_free_from_zone(struct mlx4_zone_entry *zone, u32 obj,
419 u32 count)
420{
421 mlx4_bitmap_free_range(bitmap: zone->bitmap, obj: obj - zone->offset, cnt: count, use_rr: zone->use_rr);
422}
423
424/* Should be called under a lock */
425static struct mlx4_zone_entry *__mlx4_find_zone_by_uid(
426 struct mlx4_zone_allocator *zones, u32 uid)
427{
428 struct mlx4_zone_entry *zone;
429
430 list_for_each_entry(zone, &zones->entries, list) {
431 if (zone->uid == uid)
432 return zone;
433 }
434
435 return NULL;
436}
437
438struct mlx4_bitmap *mlx4_zone_get_bitmap(struct mlx4_zone_allocator *zones, u32 uid)
439{
440 struct mlx4_zone_entry *zone;
441 struct mlx4_bitmap *bitmap;
442
443 spin_lock(lock: &zones->lock);
444
445 zone = __mlx4_find_zone_by_uid(zones, uid);
446
447 bitmap = zone == NULL ? NULL : zone->bitmap;
448
449 spin_unlock(lock: &zones->lock);
450
451 return bitmap;
452}
453
454int mlx4_zone_remove_one(struct mlx4_zone_allocator *zones, u32 uid)
455{
456 struct mlx4_zone_entry *zone;
457 int res = 0;
458
459 spin_lock(lock: &zones->lock);
460
461 zone = __mlx4_find_zone_by_uid(zones, uid);
462
463 if (NULL == zone) {
464 res = -1;
465 goto out;
466 }
467
468 __mlx4_zone_remove_one_entry(entry: zone);
469
470out:
471 spin_unlock(lock: &zones->lock);
472 kfree(objp: zone);
473
474 return res;
475}
476
477/* Should be called under a lock */
478static struct mlx4_zone_entry *__mlx4_find_zone_by_uid_unique(
479 struct mlx4_zone_allocator *zones, u32 obj)
480{
481 struct mlx4_zone_entry *zone, *zone_candidate = NULL;
482 u32 dist = (u32)-1;
483
484 /* Search for the smallest zone that this obj could be
485 * allocated from. This is done in order to handle
486 * situations when small bitmaps are allocated from bigger
487 * bitmaps (and the allocated space is marked as reserved in
488 * the bigger bitmap.
489 */
490 list_for_each_entry(zone, &zones->entries, list) {
491 if (obj >= zone->offset) {
492 u32 mobj = (obj - zone->offset) & zones->mask;
493
494 if (mobj < zone->bitmap->max) {
495 u32 curr_dist = zone->bitmap->effective_len;
496
497 if (curr_dist < dist) {
498 dist = curr_dist;
499 zone_candidate = zone;
500 }
501 }
502 }
503 }
504
505 return zone_candidate;
506}
507
508u32 mlx4_zone_alloc_entries(struct mlx4_zone_allocator *zones, u32 uid, int count,
509 int align, u32 skip_mask, u32 *puid)
510{
511 struct mlx4_zone_entry *zone;
512 int res = -1;
513
514 spin_lock(lock: &zones->lock);
515
516 zone = __mlx4_find_zone_by_uid(zones, uid);
517
518 if (NULL == zone)
519 goto out;
520
521 res = __mlx4_alloc_from_zone(zone, count, align, skip_mask, puid);
522
523out:
524 spin_unlock(lock: &zones->lock);
525
526 return res;
527}
528
529u32 mlx4_zone_free_entries(struct mlx4_zone_allocator *zones, u32 uid, u32 obj, u32 count)
530{
531 struct mlx4_zone_entry *zone;
532 int res = 0;
533
534 spin_lock(lock: &zones->lock);
535
536 zone = __mlx4_find_zone_by_uid(zones, uid);
537
538 if (NULL == zone) {
539 res = -1;
540 goto out;
541 }
542
543 __mlx4_free_from_zone(zone, obj, count);
544
545out:
546 spin_unlock(lock: &zones->lock);
547
548 return res;
549}
550
551u32 mlx4_zone_free_entries_unique(struct mlx4_zone_allocator *zones, u32 obj, u32 count)
552{
553 struct mlx4_zone_entry *zone;
554 int res;
555
556 if (!(zones->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP))
557 return -EFAULT;
558
559 spin_lock(lock: &zones->lock);
560
561 zone = __mlx4_find_zone_by_uid_unique(zones, obj);
562
563 if (NULL == zone) {
564 res = -1;
565 goto out;
566 }
567
568 __mlx4_free_from_zone(zone, obj, count);
569 res = 0;
570
571out:
572 spin_unlock(lock: &zones->lock);
573
574 return res;
575}
576
577static int mlx4_buf_direct_alloc(struct mlx4_dev *dev, int size,
578 struct mlx4_buf *buf)
579{
580 dma_addr_t t;
581
582 buf->nbufs = 1;
583 buf->npages = 1;
584 buf->page_shift = get_order(size) + PAGE_SHIFT;
585 buf->direct.buf =
586 dma_alloc_coherent(dev: &dev->persist->pdev->dev, size, dma_handle: &t,
587 GFP_KERNEL);
588 if (!buf->direct.buf)
589 return -ENOMEM;
590
591 buf->direct.map = t;
592
593 while (t & ((1 << buf->page_shift) - 1)) {
594 --buf->page_shift;
595 buf->npages *= 2;
596 }
597
598 return 0;
599}
600
601/* Handling for queue buffers -- we allocate a bunch of memory and
602 * register it in a memory region at HCA virtual address 0. If the
603 * requested size is > max_direct, we split the allocation into
604 * multiple pages, so we don't require too much contiguous memory.
605 */
606int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,
607 struct mlx4_buf *buf)
608{
609 if (size <= max_direct) {
610 return mlx4_buf_direct_alloc(dev, size, buf);
611 } else {
612 dma_addr_t t;
613 int i;
614
615 buf->direct.buf = NULL;
616 buf->nbufs = DIV_ROUND_UP(size, PAGE_SIZE);
617 buf->npages = buf->nbufs;
618 buf->page_shift = PAGE_SHIFT;
619 buf->page_list = kcalloc(n: buf->nbufs, size: sizeof(*buf->page_list),
620 GFP_KERNEL);
621 if (!buf->page_list)
622 return -ENOMEM;
623
624 for (i = 0; i < buf->nbufs; ++i) {
625 buf->page_list[i].buf =
626 dma_alloc_coherent(dev: &dev->persist->pdev->dev,
627 PAGE_SIZE, dma_handle: &t, GFP_KERNEL);
628 if (!buf->page_list[i].buf)
629 goto err_free;
630
631 buf->page_list[i].map = t;
632 }
633 }
634
635 return 0;
636
637err_free:
638 mlx4_buf_free(dev, size, buf);
639
640 return -ENOMEM;
641}
642EXPORT_SYMBOL_GPL(mlx4_buf_alloc);
643
644void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf)
645{
646 if (buf->nbufs == 1) {
647 dma_free_coherent(dev: &dev->persist->pdev->dev, size,
648 cpu_addr: buf->direct.buf, dma_handle: buf->direct.map);
649 } else {
650 int i;
651
652 for (i = 0; i < buf->nbufs; ++i)
653 if (buf->page_list[i].buf)
654 dma_free_coherent(dev: &dev->persist->pdev->dev,
655 PAGE_SIZE,
656 cpu_addr: buf->page_list[i].buf,
657 dma_handle: buf->page_list[i].map);
658 kfree(objp: buf->page_list);
659 }
660}
661EXPORT_SYMBOL_GPL(mlx4_buf_free);
662
663static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device)
664{
665 struct mlx4_db_pgdir *pgdir;
666
667 pgdir = kzalloc(size: sizeof(*pgdir), GFP_KERNEL);
668 if (!pgdir)
669 return NULL;
670
671 bitmap_fill(dst: pgdir->order1, nbits: MLX4_DB_PER_PAGE / 2);
672 pgdir->bits[0] = pgdir->order0;
673 pgdir->bits[1] = pgdir->order1;
674 pgdir->db_page = dma_alloc_coherent(dev: dma_device, PAGE_SIZE,
675 dma_handle: &pgdir->db_dma, GFP_KERNEL);
676 if (!pgdir->db_page) {
677 kfree(objp: pgdir);
678 return NULL;
679 }
680
681 return pgdir;
682}
683
684static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir,
685 struct mlx4_db *db, int order)
686{
687 int o;
688 int i;
689
690 for (o = order; o <= 1; ++o) {
691 i = find_first_bit(addr: pgdir->bits[o], size: MLX4_DB_PER_PAGE >> o);
692 if (i < MLX4_DB_PER_PAGE >> o)
693 goto found;
694 }
695
696 return -ENOMEM;
697
698found:
699 clear_bit(nr: i, addr: pgdir->bits[o]);
700
701 i <<= o;
702
703 if (o > order)
704 set_bit(nr: i ^ 1, addr: pgdir->bits[order]);
705
706 db->u.pgdir = pgdir;
707 db->index = i;
708 db->db = pgdir->db_page + db->index;
709 db->dma = pgdir->db_dma + db->index * 4;
710 db->order = order;
711
712 return 0;
713}
714
715int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order)
716{
717 struct mlx4_priv *priv = mlx4_priv(dev);
718 struct mlx4_db_pgdir *pgdir;
719 int ret = 0;
720
721 mutex_lock(&priv->pgdir_mutex);
722
723 list_for_each_entry(pgdir, &priv->pgdir_list, list)
724 if (!mlx4_alloc_db_from_pgdir(pgdir, db, order))
725 goto out;
726
727 pgdir = mlx4_alloc_db_pgdir(dma_device: &dev->persist->pdev->dev);
728 if (!pgdir) {
729 ret = -ENOMEM;
730 goto out;
731 }
732
733 list_add(new: &pgdir->list, head: &priv->pgdir_list);
734
735 /* This should never fail -- we just allocated an empty page: */
736 WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order));
737
738out:
739 mutex_unlock(lock: &priv->pgdir_mutex);
740
741 return ret;
742}
743EXPORT_SYMBOL_GPL(mlx4_db_alloc);
744
745void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db)
746{
747 struct mlx4_priv *priv = mlx4_priv(dev);
748 int o;
749 int i;
750
751 mutex_lock(&priv->pgdir_mutex);
752
753 o = db->order;
754 i = db->index;
755
756 if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {
757 clear_bit(nr: i ^ 1, addr: db->u.pgdir->order0);
758 ++o;
759 }
760 i >>= o;
761 set_bit(nr: i, addr: db->u.pgdir->bits[o]);
762
763 if (bitmap_full(src: db->u.pgdir->order1, nbits: MLX4_DB_PER_PAGE / 2)) {
764 dma_free_coherent(dev: &dev->persist->pdev->dev, PAGE_SIZE,
765 cpu_addr: db->u.pgdir->db_page, dma_handle: db->u.pgdir->db_dma);
766 list_del(entry: &db->u.pgdir->list);
767 kfree(objp: db->u.pgdir);
768 }
769
770 mutex_unlock(lock: &priv->pgdir_mutex);
771}
772EXPORT_SYMBOL_GPL(mlx4_db_free);
773
774int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres,
775 int size)
776{
777 int err;
778
779 err = mlx4_db_alloc(dev, &wqres->db, 1);
780 if (err)
781 return err;
782
783 *wqres->db.db = 0;
784
785 err = mlx4_buf_direct_alloc(dev, size, buf: &wqres->buf);
786 if (err)
787 goto err_db;
788
789 err = mlx4_mtt_init(dev, npages: wqres->buf.npages, page_shift: wqres->buf.page_shift,
790 mtt: &wqres->mtt);
791 if (err)
792 goto err_buf;
793
794 err = mlx4_buf_write_mtt(dev, mtt: &wqres->mtt, buf: &wqres->buf);
795 if (err)
796 goto err_mtt;
797
798 return 0;
799
800err_mtt:
801 mlx4_mtt_cleanup(dev, mtt: &wqres->mtt);
802err_buf:
803 mlx4_buf_free(dev, size, &wqres->buf);
804err_db:
805 mlx4_db_free(dev, &wqres->db);
806
807 return err;
808}
809EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res);
810
811void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres,
812 int size)
813{
814 mlx4_mtt_cleanup(dev, mtt: &wqres->mtt);
815 mlx4_buf_free(dev, size, &wqres->buf);
816 mlx4_db_free(dev, &wqres->db);
817}
818EXPORT_SYMBOL_GPL(mlx4_free_hwq_res);
819

source code of linux/drivers/net/ethernet/mellanox/mlx4/alloc.c