1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
4 */
5
6#include "mana_ib.h"
7
8#define VALID_MR_FLAGS (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |\
9 IB_ACCESS_REMOTE_ATOMIC | IB_ZERO_BASED)
10
11#define VALID_DMA_MR_FLAGS (IB_ACCESS_LOCAL_WRITE)
12
13static enum gdma_mr_access_flags
14mana_ib_verbs_to_gdma_access_flags(int access_flags)
15{
16 enum gdma_mr_access_flags flags = GDMA_ACCESS_FLAG_LOCAL_READ;
17
18 if (access_flags & IB_ACCESS_LOCAL_WRITE)
19 flags |= GDMA_ACCESS_FLAG_LOCAL_WRITE;
20
21 if (access_flags & IB_ACCESS_REMOTE_WRITE)
22 flags |= GDMA_ACCESS_FLAG_REMOTE_WRITE;
23
24 if (access_flags & IB_ACCESS_REMOTE_READ)
25 flags |= GDMA_ACCESS_FLAG_REMOTE_READ;
26
27 if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
28 flags |= GDMA_ACCESS_FLAG_REMOTE_ATOMIC;
29
30 return flags;
31}
32
33static int mana_ib_gd_create_mr(struct mana_ib_dev *dev, struct mana_ib_mr *mr,
34 struct gdma_create_mr_params *mr_params)
35{
36 struct gdma_create_mr_response resp = {};
37 struct gdma_create_mr_request req = {};
38 struct gdma_context *gc = mdev_to_gc(mdev: dev);
39 int err;
40
41 mana_gd_init_req_hdr(hdr: &req.hdr, code: GDMA_CREATE_MR, req_size: sizeof(req),
42 resp_size: sizeof(resp));
43 req.pd_handle = mr_params->pd_handle;
44 req.mr_type = mr_params->mr_type;
45
46 switch (mr_params->mr_type) {
47 case GDMA_MR_TYPE_GPA:
48 break;
49 case GDMA_MR_TYPE_GVA:
50 req.gva.dma_region_handle = mr_params->gva.dma_region_handle;
51 req.gva.virtual_address = mr_params->gva.virtual_address;
52 req.gva.access_flags = mr_params->gva.access_flags;
53 break;
54 case GDMA_MR_TYPE_ZBVA:
55 req.zbva.dma_region_handle = mr_params->zbva.dma_region_handle;
56 req.zbva.access_flags = mr_params->zbva.access_flags;
57 break;
58 default:
59 ibdev_dbg(&dev->ib_dev,
60 "invalid param (GDMA_MR_TYPE) passed, type %d\n",
61 req.mr_type);
62 return -EINVAL;
63 }
64
65 err = mana_gd_send_request(gc, req_len: sizeof(req), req: &req, resp_len: sizeof(resp), resp: &resp);
66
67 if (err || resp.hdr.status) {
68 ibdev_dbg(&dev->ib_dev, "Failed to create mr %d, %u", err,
69 resp.hdr.status);
70 if (!err)
71 err = -EPROTO;
72
73 return err;
74 }
75
76 mr->ibmr.lkey = resp.lkey;
77 mr->ibmr.rkey = resp.rkey;
78 mr->mr_handle = resp.mr_handle;
79
80 return 0;
81}
82
83static int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)
84{
85 struct gdma_destroy_mr_response resp = {};
86 struct gdma_destroy_mr_request req = {};
87 struct gdma_context *gc = mdev_to_gc(mdev: dev);
88 int err;
89
90 mana_gd_init_req_hdr(hdr: &req.hdr, code: GDMA_DESTROY_MR, req_size: sizeof(req),
91 resp_size: sizeof(resp));
92
93 req.mr_handle = mr_handle;
94
95 err = mana_gd_send_request(gc, req_len: sizeof(req), req: &req, resp_len: sizeof(resp), resp: &resp);
96 if (err || resp.hdr.status) {
97 dev_err(gc->dev, "Failed to destroy MR: %d, 0x%x\n", err,
98 resp.hdr.status);
99 if (!err)
100 err = -EPROTO;
101 return err;
102 }
103
104 return 0;
105}
106
107struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
108 u64 iova, int access_flags,
109 struct ib_dmah *dmah,
110 struct ib_udata *udata)
111{
112 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
113 struct gdma_create_mr_params mr_params = {};
114 struct ib_device *ibdev = ibpd->device;
115 struct mana_ib_dev *dev;
116 struct mana_ib_mr *mr;
117 u64 dma_region_handle;
118 int err;
119
120 if (dmah)
121 return ERR_PTR(error: -EOPNOTSUPP);
122
123 dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
124
125 ibdev_dbg(ibdev,
126 "start 0x%llx, iova 0x%llx length 0x%llx access_flags 0x%x",
127 start, iova, length, access_flags);
128
129 access_flags &= ~IB_ACCESS_OPTIONAL;
130 if (access_flags & ~VALID_MR_FLAGS)
131 return ERR_PTR(error: -EINVAL);
132
133 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
134 if (!mr)
135 return ERR_PTR(error: -ENOMEM);
136
137 mr->umem = ib_umem_get(device: ibdev, addr: start, size: length, access: access_flags);
138 if (IS_ERR(ptr: mr->umem)) {
139 err = PTR_ERR(ptr: mr->umem);
140 ibdev_dbg(ibdev,
141 "Failed to get umem for register user-mr, %pe\n",
142 mr->umem);
143 goto err_free;
144 }
145
146 err = mana_ib_create_dma_region(dev, umem: mr->umem, gdma_region: &dma_region_handle, virt: iova);
147 if (err) {
148 ibdev_dbg(ibdev, "Failed create dma region for user-mr, %d\n",
149 err);
150 goto err_umem;
151 }
152
153 ibdev_dbg(ibdev,
154 "created dma region for user-mr 0x%llx\n",
155 dma_region_handle);
156
157 mr_params.pd_handle = pd->pd_handle;
158 if (access_flags & IB_ZERO_BASED) {
159 mr_params.mr_type = GDMA_MR_TYPE_ZBVA;
160 mr_params.zbva.dma_region_handle = dma_region_handle;
161 mr_params.zbva.access_flags =
162 mana_ib_verbs_to_gdma_access_flags(access_flags);
163 } else {
164 mr_params.mr_type = GDMA_MR_TYPE_GVA;
165 mr_params.gva.dma_region_handle = dma_region_handle;
166 mr_params.gva.virtual_address = iova;
167 mr_params.gva.access_flags =
168 mana_ib_verbs_to_gdma_access_flags(access_flags);
169 }
170
171 err = mana_ib_gd_create_mr(dev, mr, mr_params: &mr_params);
172 if (err)
173 goto err_dma_region;
174
175 /*
176 * There is no need to keep track of dma_region_handle after MR is
177 * successfully created. The dma_region_handle is tracked in the PF
178 * as part of the lifecycle of this MR.
179 */
180
181 return &mr->ibmr;
182
183err_dma_region:
184 mana_gd_destroy_dma_region(gc: mdev_to_gc(mdev: dev), dma_region_handle);
185
186err_umem:
187 ib_umem_release(umem: mr->umem);
188
189err_free:
190 kfree(objp: mr);
191 return ERR_PTR(error: err);
192}
193
194struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length,
195 u64 iova, int fd, int access_flags,
196 struct ib_dmah *dmah,
197 struct uverbs_attr_bundle *attrs)
198{
199 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
200 struct gdma_create_mr_params mr_params = {};
201 struct ib_device *ibdev = ibpd->device;
202 struct ib_umem_dmabuf *umem_dmabuf;
203 struct mana_ib_dev *dev;
204 struct mana_ib_mr *mr;
205 u64 dma_region_handle;
206 int err;
207
208 if (dmah)
209 return ERR_PTR(error: -EOPNOTSUPP);
210
211 dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
212
213 access_flags &= ~IB_ACCESS_OPTIONAL;
214 if (access_flags & ~VALID_MR_FLAGS)
215 return ERR_PTR(error: -EOPNOTSUPP);
216
217 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
218 if (!mr)
219 return ERR_PTR(error: -ENOMEM);
220
221 umem_dmabuf = ib_umem_dmabuf_get_pinned(device: ibdev, offset: start, size: length, fd, access: access_flags);
222 if (IS_ERR(ptr: umem_dmabuf)) {
223 err = PTR_ERR(ptr: umem_dmabuf);
224 ibdev_dbg(ibdev, "Failed to get dmabuf umem, %pe\n",
225 umem_dmabuf);
226 goto err_free;
227 }
228
229 mr->umem = &umem_dmabuf->umem;
230
231 err = mana_ib_create_dma_region(dev, umem: mr->umem, gdma_region: &dma_region_handle, virt: iova);
232 if (err) {
233 ibdev_dbg(ibdev, "Failed create dma region for user-mr, %d\n",
234 err);
235 goto err_umem;
236 }
237
238 mr_params.pd_handle = pd->pd_handle;
239 mr_params.mr_type = GDMA_MR_TYPE_GVA;
240 mr_params.gva.dma_region_handle = dma_region_handle;
241 mr_params.gva.virtual_address = iova;
242 mr_params.gva.access_flags =
243 mana_ib_verbs_to_gdma_access_flags(access_flags);
244
245 err = mana_ib_gd_create_mr(dev, mr, mr_params: &mr_params);
246 if (err)
247 goto err_dma_region;
248
249 /*
250 * There is no need to keep track of dma_region_handle after MR is
251 * successfully created. The dma_region_handle is tracked in the PF
252 * as part of the lifecycle of this MR.
253 */
254
255 return &mr->ibmr;
256
257err_dma_region:
258 mana_gd_destroy_dma_region(gc: mdev_to_gc(mdev: dev), dma_region_handle);
259
260err_umem:
261 ib_umem_release(umem: mr->umem);
262
263err_free:
264 kfree(objp: mr);
265 return ERR_PTR(error: err);
266}
267
268struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags)
269{
270 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
271 struct gdma_create_mr_params mr_params = {};
272 struct ib_device *ibdev = ibpd->device;
273 struct mana_ib_dev *dev;
274 struct mana_ib_mr *mr;
275 int err;
276
277 dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
278
279 if (access_flags & ~VALID_DMA_MR_FLAGS)
280 return ERR_PTR(error: -EINVAL);
281
282 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
283 if (!mr)
284 return ERR_PTR(error: -ENOMEM);
285
286 mr_params.pd_handle = pd->pd_handle;
287 mr_params.mr_type = GDMA_MR_TYPE_GPA;
288
289 err = mana_ib_gd_create_mr(dev, mr, mr_params: &mr_params);
290 if (err)
291 goto err_free;
292
293 return &mr->ibmr;
294
295err_free:
296 kfree(objp: mr);
297 return ERR_PTR(error: err);
298}
299
300int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
301{
302 struct mana_ib_mr *mr = container_of(ibmr, struct mana_ib_mr, ibmr);
303 struct ib_device *ibdev = ibmr->device;
304 struct mana_ib_dev *dev;
305 int err;
306
307 dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
308
309 err = mana_ib_gd_destroy_mr(dev, mr_handle: mr->mr_handle);
310 if (err)
311 return err;
312
313 if (mr->umem)
314 ib_umem_release(umem: mr->umem);
315
316 kfree(objp: mr);
317
318 return 0;
319}
320

source code of linux/drivers/infiniband/hw/mana/mr.c