1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2020 Mellanox Technologies Ltd. */
3
4#include <linux/vhost_types.h>
5#include <linux/vdpa.h>
6#include <linux/gcd.h>
7#include <linux/string.h>
8#include <linux/mlx5/qp.h>
9#include "mlx5_vdpa.h"
10
11/* DIV_ROUND_UP where the divider is a power of 2 give by its log base 2 value */
12#define MLX5_DIV_ROUND_UP_POW2(_n, _s) \
13({ \
14 u64 __s = _s; \
15 u64 _res; \
16 _res = (((_n) + (1 << (__s)) - 1) >> (__s)); \
17 _res; \
18})
19
20static int get_octo_len(u64 len, int page_shift)
21{
22 u64 page_size = 1ULL << page_shift;
23 int npages;
24
25 npages = ALIGN(len, page_size) >> page_shift;
26 return (npages + 1) / 2;
27}
28
29static void mlx5_set_access_mode(void *mkc, int mode)
30{
31 MLX5_SET(mkc, mkc, access_mode_1_0, mode & 0x3);
32 MLX5_SET(mkc, mkc, access_mode_4_2, mode >> 2);
33}
34
35static void populate_mtts(struct mlx5_vdpa_direct_mr *mr, __be64 *mtt)
36{
37 struct scatterlist *sg;
38 int nsg = mr->nsg;
39 u64 dma_addr;
40 u64 dma_len;
41 int j = 0;
42 int i;
43
44 for_each_sg(mr->sg_head.sgl, sg, mr->nent, i) {
45 for (dma_addr = sg_dma_address(sg), dma_len = sg_dma_len(sg);
46 nsg && dma_len;
47 nsg--, dma_addr += BIT(mr->log_size), dma_len -= BIT(mr->log_size))
48 mtt[j++] = cpu_to_be64(dma_addr);
49 }
50}
51
52static int create_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)
53{
54 int inlen;
55 void *mkc;
56 void *in;
57 int err;
58
59 inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + roundup(MLX5_ST_SZ_BYTES(mtt) * mr->nsg, 16);
60 in = kvzalloc(size: inlen, GFP_KERNEL);
61 if (!in)
62 return -ENOMEM;
63
64 MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
65 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
66 MLX5_SET(mkc, mkc, lw, !!(mr->perm & VHOST_MAP_WO));
67 MLX5_SET(mkc, mkc, lr, !!(mr->perm & VHOST_MAP_RO));
68 mlx5_set_access_mode(mkc, mode: MLX5_MKC_ACCESS_MODE_MTT);
69 MLX5_SET(mkc, mkc, qpn, 0xffffff);
70 MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);
71 MLX5_SET64(mkc, mkc, start_addr, mr->offset);
72 MLX5_SET64(mkc, mkc, len, mr->end - mr->start);
73 MLX5_SET(mkc, mkc, log_page_size, mr->log_size);
74 MLX5_SET(mkc, mkc, translations_octword_size,
75 get_octo_len(mr->end - mr->start, mr->log_size));
76 MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
77 get_octo_len(mr->end - mr->start, mr->log_size));
78 populate_mtts(mr, MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt));
79 err = mlx5_vdpa_create_mkey(mvdev, mkey: &mr->mr, in, inlen);
80 kvfree(addr: in);
81 if (err) {
82 mlx5_vdpa_warn(mvdev, "Failed to create direct MR\n");
83 return err;
84 }
85
86 return 0;
87}
88
89static void destroy_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)
90{
91 mlx5_vdpa_destroy_mkey(mvdev, mkey: mr->mr);
92}
93
94static u64 map_start(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr)
95{
96 return max_t(u64, map->start, mr->start);
97}
98
99static u64 map_end(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr)
100{
101 return min_t(u64, map->last + 1, mr->end);
102}
103
104static u64 maplen(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr)
105{
106 return map_end(map, mr) - map_start(map, mr);
107}
108
109#define MLX5_VDPA_INVALID_START_ADDR ((u64)-1)
110#define MLX5_VDPA_INVALID_LEN ((u64)-1)
111
112static u64 indir_start_addr(struct mlx5_vdpa_mr *mkey)
113{
114 struct mlx5_vdpa_direct_mr *s;
115
116 s = list_first_entry_or_null(&mkey->head, struct mlx5_vdpa_direct_mr, list);
117 if (!s)
118 return MLX5_VDPA_INVALID_START_ADDR;
119
120 return s->start;
121}
122
123static u64 indir_len(struct mlx5_vdpa_mr *mkey)
124{
125 struct mlx5_vdpa_direct_mr *s;
126 struct mlx5_vdpa_direct_mr *e;
127
128 s = list_first_entry_or_null(&mkey->head, struct mlx5_vdpa_direct_mr, list);
129 if (!s)
130 return MLX5_VDPA_INVALID_LEN;
131
132 e = list_last_entry(&mkey->head, struct mlx5_vdpa_direct_mr, list);
133
134 return e->end - s->start;
135}
136
137#define LOG_MAX_KLM_SIZE 30
138#define MAX_KLM_SIZE BIT(LOG_MAX_KLM_SIZE)
139
140static u32 klm_bcount(u64 size)
141{
142 return (u32)size;
143}
144
145static void fill_indir(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mkey, void *in)
146{
147 struct mlx5_vdpa_direct_mr *dmr;
148 struct mlx5_klm *klmarr;
149 struct mlx5_klm *klm;
150 bool first = true;
151 u64 preve;
152 int i;
153
154 klmarr = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
155 i = 0;
156 list_for_each_entry(dmr, &mkey->head, list) {
157again:
158 klm = &klmarr[i++];
159 if (first) {
160 preve = dmr->start;
161 first = false;
162 }
163
164 if (preve == dmr->start) {
165 klm->key = cpu_to_be32(dmr->mr);
166 klm->bcount = cpu_to_be32(klm_bcount(dmr->end - dmr->start));
167 preve = dmr->end;
168 } else {
169 klm->key = cpu_to_be32(mvdev->res.null_mkey);
170 klm->bcount = cpu_to_be32(klm_bcount(dmr->start - preve));
171 preve = dmr->start;
172 goto again;
173 }
174 }
175}
176
177static int klm_byte_size(int nklms)
178{
179 return 16 * ALIGN(nklms, 4);
180}
181
182static int create_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
183{
184 int inlen;
185 void *mkc;
186 void *in;
187 int err;
188 u64 start;
189 u64 len;
190
191 start = indir_start_addr(mkey: mr);
192 len = indir_len(mkey: mr);
193 if (start == MLX5_VDPA_INVALID_START_ADDR || len == MLX5_VDPA_INVALID_LEN)
194 return -EINVAL;
195
196 inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + klm_byte_size(nklms: mr->num_klms);
197 in = kzalloc(size: inlen, GFP_KERNEL);
198 if (!in)
199 return -ENOMEM;
200
201 MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
202 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
203 MLX5_SET(mkc, mkc, lw, 1);
204 MLX5_SET(mkc, mkc, lr, 1);
205 mlx5_set_access_mode(mkc, mode: MLX5_MKC_ACCESS_MODE_KLMS);
206 MLX5_SET(mkc, mkc, qpn, 0xffffff);
207 MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);
208 MLX5_SET64(mkc, mkc, start_addr, start);
209 MLX5_SET64(mkc, mkc, len, len);
210 MLX5_SET(mkc, mkc, translations_octword_size, klm_byte_size(mr->num_klms) / 16);
211 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, mr->num_klms);
212 fill_indir(mvdev, mkey: mr, in);
213 err = mlx5_vdpa_create_mkey(mvdev, mkey: &mr->mkey, in, inlen);
214 kfree(objp: in);
215 return err;
216}
217
218static void destroy_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mkey)
219{
220 mlx5_vdpa_destroy_mkey(mvdev, mkey: mkey->mkey);
221}
222
223static int map_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr,
224 struct vhost_iotlb *iotlb)
225{
226 struct vhost_iotlb_map *map;
227 unsigned long lgcd = 0;
228 int log_entity_size;
229 unsigned long size;
230 u64 start = 0;
231 int err;
232 struct page *pg;
233 unsigned int nsg;
234 int sglen;
235 u64 pa;
236 u64 paend;
237 struct scatterlist *sg;
238 struct device *dma = mvdev->vdev.dma_dev;
239
240 for (map = vhost_iotlb_itree_first(iotlb, start: mr->start, last: mr->end - 1);
241 map; map = vhost_iotlb_itree_next(map, start, last: mr->end - 1)) {
242 size = maplen(map, mr);
243 lgcd = gcd(a: lgcd, b: size);
244 start += size;
245 }
246 log_entity_size = ilog2(lgcd);
247
248 sglen = 1 << log_entity_size;
249 nsg = MLX5_DIV_ROUND_UP_POW2(mr->end - mr->start, log_entity_size);
250
251 err = sg_alloc_table(&mr->sg_head, nsg, GFP_KERNEL);
252 if (err)
253 return err;
254
255 sg = mr->sg_head.sgl;
256 for (map = vhost_iotlb_itree_first(iotlb, start: mr->start, last: mr->end - 1);
257 map; map = vhost_iotlb_itree_next(map, start: mr->start, last: mr->end - 1)) {
258 paend = map->addr + maplen(map, mr);
259 for (pa = map->addr; pa < paend; pa += sglen) {
260 pg = pfn_to_page(__phys_to_pfn(pa));
261 if (!sg) {
262 mlx5_vdpa_warn(mvdev, "sg null. start 0x%llx, end 0x%llx\n",
263 map->start, map->last + 1);
264 err = -ENOMEM;
265 goto err_map;
266 }
267 sg_set_page(sg, page: pg, len: sglen, offset: 0);
268 sg = sg_next(sg);
269 if (!sg)
270 goto done;
271 }
272 }
273done:
274 mr->log_size = log_entity_size;
275 mr->nsg = nsg;
276 mr->nent = dma_map_sg_attrs(dev: dma, sg: mr->sg_head.sgl, nents: mr->nsg, dir: DMA_BIDIRECTIONAL, attrs: 0);
277 if (!mr->nent) {
278 err = -ENOMEM;
279 goto err_map;
280 }
281
282 err = create_direct_mr(mvdev, mr);
283 if (err)
284 goto err_direct;
285
286 return 0;
287
288err_direct:
289 dma_unmap_sg_attrs(dev: dma, sg: mr->sg_head.sgl, nents: mr->nsg, dir: DMA_BIDIRECTIONAL, attrs: 0);
290err_map:
291 sg_free_table(&mr->sg_head);
292 return err;
293}
294
295static void unmap_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)
296{
297 struct device *dma = mvdev->vdev.dma_dev;
298
299 destroy_direct_mr(mvdev, mr);
300 dma_unmap_sg_attrs(dev: dma, sg: mr->sg_head.sgl, nents: mr->nsg, dir: DMA_BIDIRECTIONAL, attrs: 0);
301 sg_free_table(&mr->sg_head);
302}
303
304static int add_direct_chain(struct mlx5_vdpa_dev *mvdev,
305 struct mlx5_vdpa_mr *mr,
306 u64 start,
307 u64 size,
308 u8 perm,
309 struct vhost_iotlb *iotlb)
310{
311 struct mlx5_vdpa_direct_mr *dmr;
312 struct mlx5_vdpa_direct_mr *n;
313 LIST_HEAD(tmp);
314 u64 st;
315 u64 sz;
316 int err;
317
318 st = start;
319 while (size) {
320 sz = (u32)min_t(u64, MAX_KLM_SIZE, size);
321 dmr = kzalloc(size: sizeof(*dmr), GFP_KERNEL);
322 if (!dmr) {
323 err = -ENOMEM;
324 goto err_alloc;
325 }
326
327 dmr->start = st;
328 dmr->end = st + sz;
329 dmr->perm = perm;
330 err = map_direct_mr(mvdev, mr: dmr, iotlb);
331 if (err) {
332 kfree(objp: dmr);
333 goto err_alloc;
334 }
335
336 list_add_tail(new: &dmr->list, head: &tmp);
337 size -= sz;
338 mr->num_directs++;
339 mr->num_klms++;
340 st += sz;
341 }
342 list_splice_tail(list: &tmp, head: &mr->head);
343 return 0;
344
345err_alloc:
346 list_for_each_entry_safe(dmr, n, &mr->head, list) {
347 list_del_init(entry: &dmr->list);
348 unmap_direct_mr(mvdev, mr: dmr);
349 kfree(objp: dmr);
350 }
351 return err;
352}
353
354/* The iotlb pointer contains a list of maps. Go over the maps, possibly
355 * merging mergeable maps, and create direct memory keys that provide the
356 * device access to memory. The direct mkeys are then referred to by the
357 * indirect memory key that provides access to the enitre address space given
358 * by iotlb.
359 */
360static int create_user_mr(struct mlx5_vdpa_dev *mvdev,
361 struct mlx5_vdpa_mr *mr,
362 struct vhost_iotlb *iotlb)
363{
364 struct mlx5_vdpa_direct_mr *dmr;
365 struct mlx5_vdpa_direct_mr *n;
366 struct vhost_iotlb_map *map;
367 u32 pperm = U16_MAX;
368 u64 last = U64_MAX;
369 u64 ps = U64_MAX;
370 u64 pe = U64_MAX;
371 u64 start = 0;
372 int err = 0;
373 int nnuls;
374
375 INIT_LIST_HEAD(list: &mr->head);
376 for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
377 map = vhost_iotlb_itree_next(map, start, last)) {
378 start = map->start;
379 if (pe == map->start && pperm == map->perm) {
380 pe = map->last + 1;
381 } else {
382 if (ps != U64_MAX) {
383 if (pe < map->start) {
384 /* We have a hole in the map. Check how
385 * many null keys are required to fill it.
386 */
387 nnuls = MLX5_DIV_ROUND_UP_POW2(map->start - pe,
388 LOG_MAX_KLM_SIZE);
389 mr->num_klms += nnuls;
390 }
391 err = add_direct_chain(mvdev, mr, start: ps, size: pe - ps, perm: pperm, iotlb);
392 if (err)
393 goto err_chain;
394 }
395 ps = map->start;
396 pe = map->last + 1;
397 pperm = map->perm;
398 }
399 }
400 err = add_direct_chain(mvdev, mr, start: ps, size: pe - ps, perm: pperm, iotlb);
401 if (err)
402 goto err_chain;
403
404 /* Create the memory key that defines the guests's address space. This
405 * memory key refers to the direct keys that contain the MTT
406 * translations
407 */
408 err = create_indirect_key(mvdev, mr);
409 if (err)
410 goto err_chain;
411
412 mr->user_mr = true;
413 return 0;
414
415err_chain:
416 list_for_each_entry_safe_reverse(dmr, n, &mr->head, list) {
417 list_del_init(entry: &dmr->list);
418 unmap_direct_mr(mvdev, mr: dmr);
419 kfree(objp: dmr);
420 }
421 return err;
422}
423
424static int create_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
425{
426 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
427 void *mkc;
428 u32 *in;
429 int err;
430
431 in = kzalloc(size: inlen, GFP_KERNEL);
432 if (!in)
433 return -ENOMEM;
434
435 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
436
437 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
438 MLX5_SET(mkc, mkc, length64, 1);
439 MLX5_SET(mkc, mkc, lw, 1);
440 MLX5_SET(mkc, mkc, lr, 1);
441 MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);
442 MLX5_SET(mkc, mkc, qpn, 0xffffff);
443
444 err = mlx5_vdpa_create_mkey(mvdev, mkey: &mr->mkey, in, inlen);
445 if (!err)
446 mr->user_mr = false;
447
448 kfree(objp: in);
449 return err;
450}
451
452static void destroy_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
453{
454 mlx5_vdpa_destroy_mkey(mvdev, mkey: mr->mkey);
455}
456
457static int dup_iotlb(struct vhost_iotlb *dst, struct vhost_iotlb *src)
458{
459 struct vhost_iotlb_map *map;
460 u64 start = 0, last = ULLONG_MAX;
461 int err;
462
463 if (dst == src)
464 return -EINVAL;
465
466 if (!src) {
467 err = vhost_iotlb_add_range(iotlb: dst, start, last, addr: start, VHOST_ACCESS_RW);
468 return err;
469 }
470
471 for (map = vhost_iotlb_itree_first(iotlb: src, start, last); map;
472 map = vhost_iotlb_itree_next(map, start, last)) {
473 err = vhost_iotlb_add_range(iotlb: dst, start: map->start, last: map->last,
474 addr: map->addr, perm: map->perm);
475 if (err)
476 return err;
477 }
478 return 0;
479}
480
481static void prune_iotlb(struct vhost_iotlb *iotlb)
482{
483 vhost_iotlb_del_range(iotlb, start: 0, ULLONG_MAX);
484}
485
486static void destroy_user_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
487{
488 struct mlx5_vdpa_direct_mr *dmr;
489 struct mlx5_vdpa_direct_mr *n;
490
491 destroy_indirect_key(mvdev, mkey: mr);
492 list_for_each_entry_safe_reverse(dmr, n, &mr->head, list) {
493 list_del_init(entry: &dmr->list);
494 unmap_direct_mr(mvdev, mr: dmr);
495 kfree(objp: dmr);
496 }
497}
498
499static void _mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
500{
501 if (WARN_ON(!mr))
502 return;
503
504 if (mr->user_mr)
505 destroy_user_mr(mvdev, mr);
506 else
507 destroy_dma_mr(mvdev, mr);
508
509 vhost_iotlb_free(iotlb: mr->iotlb);
510
511 list_del(entry: &mr->mr_list);
512
513 kfree(objp: mr);
514}
515
516static void _mlx5_vdpa_put_mr(struct mlx5_vdpa_dev *mvdev,
517 struct mlx5_vdpa_mr *mr)
518{
519 if (!mr)
520 return;
521
522 if (refcount_dec_and_test(r: &mr->refcount))
523 _mlx5_vdpa_destroy_mr(mvdev, mr);
524}
525
526void mlx5_vdpa_put_mr(struct mlx5_vdpa_dev *mvdev,
527 struct mlx5_vdpa_mr *mr)
528{
529 mutex_lock(&mvdev->mr_mtx);
530 _mlx5_vdpa_put_mr(mvdev, mr);
531 mutex_unlock(lock: &mvdev->mr_mtx);
532}
533
534static void _mlx5_vdpa_get_mr(struct mlx5_vdpa_dev *mvdev,
535 struct mlx5_vdpa_mr *mr)
536{
537 if (!mr)
538 return;
539
540 refcount_inc(r: &mr->refcount);
541}
542
543void mlx5_vdpa_get_mr(struct mlx5_vdpa_dev *mvdev,
544 struct mlx5_vdpa_mr *mr)
545{
546 mutex_lock(&mvdev->mr_mtx);
547 _mlx5_vdpa_get_mr(mvdev, mr);
548 mutex_unlock(lock: &mvdev->mr_mtx);
549}
550
551void mlx5_vdpa_update_mr(struct mlx5_vdpa_dev *mvdev,
552 struct mlx5_vdpa_mr *new_mr,
553 unsigned int asid)
554{
555 struct mlx5_vdpa_mr *old_mr = mvdev->mr[asid];
556
557 mutex_lock(&mvdev->mr_mtx);
558
559 _mlx5_vdpa_put_mr(mvdev, mr: old_mr);
560 mvdev->mr[asid] = new_mr;
561
562 mutex_unlock(lock: &mvdev->mr_mtx);
563}
564
565static void mlx5_vdpa_show_mr_leaks(struct mlx5_vdpa_dev *mvdev)
566{
567 struct mlx5_vdpa_mr *mr;
568
569 mutex_lock(&mvdev->mr_mtx);
570
571 list_for_each_entry(mr, &mvdev->mr_list_head, mr_list) {
572
573 mlx5_vdpa_warn(mvdev, "mkey still alive after resource delete: "
574 "mr: %p, mkey: 0x%x, refcount: %u\n",
575 mr, mr->mkey, refcount_read(&mr->refcount));
576 }
577
578 mutex_unlock(lock: &mvdev->mr_mtx);
579
580}
581
582void mlx5_vdpa_destroy_mr_resources(struct mlx5_vdpa_dev *mvdev)
583{
584 for (int i = 0; i < MLX5_VDPA_NUM_AS; i++)
585 mlx5_vdpa_update_mr(mvdev, NULL, asid: i);
586
587 prune_iotlb(iotlb: mvdev->cvq.iotlb);
588
589 mlx5_vdpa_show_mr_leaks(mvdev);
590}
591
592static int _mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev,
593 struct mlx5_vdpa_mr *mr,
594 struct vhost_iotlb *iotlb)
595{
596 int err;
597
598 if (iotlb)
599 err = create_user_mr(mvdev, mr, iotlb);
600 else
601 err = create_dma_mr(mvdev, mr);
602
603 if (err)
604 return err;
605
606 mr->iotlb = vhost_iotlb_alloc(limit: 0, flags: 0);
607 if (!mr->iotlb) {
608 err = -ENOMEM;
609 goto err_mr;
610 }
611
612 err = dup_iotlb(dst: mr->iotlb, src: iotlb);
613 if (err)
614 goto err_iotlb;
615
616 list_add_tail(new: &mr->mr_list, head: &mvdev->mr_list_head);
617
618 return 0;
619
620err_iotlb:
621 vhost_iotlb_free(iotlb: mr->iotlb);
622
623err_mr:
624 if (iotlb)
625 destroy_user_mr(mvdev, mr);
626 else
627 destroy_dma_mr(mvdev, mr);
628
629 return err;
630}
631
632struct mlx5_vdpa_mr *mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev,
633 struct vhost_iotlb *iotlb)
634{
635 struct mlx5_vdpa_mr *mr;
636 int err;
637
638 mr = kzalloc(size: sizeof(*mr), GFP_KERNEL);
639 if (!mr)
640 return ERR_PTR(error: -ENOMEM);
641
642 mutex_lock(&mvdev->mr_mtx);
643 err = _mlx5_vdpa_create_mr(mvdev, mr, iotlb);
644 mutex_unlock(lock: &mvdev->mr_mtx);
645
646 if (err)
647 goto out_err;
648
649 refcount_set(r: &mr->refcount, n: 1);
650
651 return mr;
652
653out_err:
654 kfree(objp: mr);
655 return ERR_PTR(error: err);
656}
657
658int mlx5_vdpa_update_cvq_iotlb(struct mlx5_vdpa_dev *mvdev,
659 struct vhost_iotlb *iotlb,
660 unsigned int asid)
661{
662 int err;
663
664 if (mvdev->group2asid[MLX5_VDPA_CVQ_GROUP] != asid)
665 return 0;
666
667 spin_lock(lock: &mvdev->cvq.iommu_lock);
668
669 prune_iotlb(iotlb: mvdev->cvq.iotlb);
670 err = dup_iotlb(dst: mvdev->cvq.iotlb, src: iotlb);
671
672 spin_unlock(lock: &mvdev->cvq.iommu_lock);
673
674 return err;
675}
676
677int mlx5_vdpa_create_dma_mr(struct mlx5_vdpa_dev *mvdev)
678{
679 struct mlx5_vdpa_mr *mr;
680
681 mr = mlx5_vdpa_create_mr(mvdev, NULL);
682 if (IS_ERR(ptr: mr))
683 return PTR_ERR(ptr: mr);
684
685 mlx5_vdpa_update_mr(mvdev, new_mr: mr, asid: 0);
686
687 return mlx5_vdpa_update_cvq_iotlb(mvdev, NULL, asid: 0);
688}
689
690int mlx5_vdpa_reset_mr(struct mlx5_vdpa_dev *mvdev, unsigned int asid)
691{
692 if (asid >= MLX5_VDPA_NUM_AS)
693 return -EINVAL;
694
695 mlx5_vdpa_update_mr(mvdev, NULL, asid);
696
697 if (asid == 0 && MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) {
698 if (mlx5_vdpa_create_dma_mr(mvdev))
699 mlx5_vdpa_warn(mvdev, "create DMA MR failed\n");
700 } else {
701 mlx5_vdpa_update_cvq_iotlb(mvdev, NULL, asid);
702 }
703
704 return 0;
705}
706

source code of linux/drivers/vdpa/mlx5/core/mr.c