1 | /* |
2 | * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. |
3 | * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. |
4 | * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. |
5 | * Copyright (c) 2005 Mellanox Technologies. All rights reserved. |
6 | * Copyright (c) 2004 Voltaire, Inc. All rights reserved. |
7 | * |
8 | * This software is available to you under a choice of one of two |
9 | * licenses. You may choose to be licensed under the terms of the GNU |
10 | * General Public License (GPL) Version 2, available from the file |
11 | * COPYING in the main directory of this source tree, or the |
12 | * OpenIB.org BSD license below: |
13 | * |
14 | * Redistribution and use in source and binary forms, with or |
15 | * without modification, are permitted provided that the following |
16 | * conditions are met: |
17 | * |
18 | * - Redistributions of source code must retain the above |
19 | * copyright notice, this list of conditions and the following |
20 | * disclaimer. |
21 | * |
22 | * - Redistributions in binary form must reproduce the above |
23 | * copyright notice, this list of conditions and the following |
24 | * disclaimer in the documentation and/or other materials |
25 | * provided with the distribution. |
26 | * |
27 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
28 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
29 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
30 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
31 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
32 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
33 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
34 | * SOFTWARE. |
35 | */ |
36 | |
37 | #include <rdma/ib_smi.h> |
38 | #include <rdma/ib_umem.h> |
39 | #include <rdma/ib_user_verbs.h> |
40 | #include <rdma/uverbs_ioctl.h> |
41 | |
42 | #include <linux/sched.h> |
43 | #include <linux/slab.h> |
44 | #include <linux/stat.h> |
45 | #include <linux/mm.h> |
46 | #include <linux/export.h> |
47 | |
48 | #include "mthca_dev.h" |
49 | #include "mthca_cmd.h" |
50 | #include <rdma/mthca-abi.h> |
51 | #include "mthca_memfree.h" |
52 | |
53 | static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *props, |
54 | struct ib_udata *uhw) |
55 | { |
56 | struct ib_smp *in_mad; |
57 | struct ib_smp *out_mad; |
58 | int err = -ENOMEM; |
59 | struct mthca_dev *mdev = to_mdev(ibdev); |
60 | |
61 | if (uhw->inlen || uhw->outlen) |
62 | return -EINVAL; |
63 | |
64 | in_mad = kzalloc(size: sizeof *in_mad, GFP_KERNEL); |
65 | out_mad = kmalloc(size: sizeof *out_mad, GFP_KERNEL); |
66 | if (!in_mad || !out_mad) |
67 | goto out; |
68 | |
69 | memset(props, 0, sizeof *props); |
70 | |
71 | props->fw_ver = mdev->fw_ver; |
72 | |
73 | ib_init_query_mad(mad: in_mad); |
74 | in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; |
75 | |
76 | err = mthca_MAD_IFC(dev: mdev, ignore_mkey: 1, ignore_bkey: 1, |
77 | port: 1, NULL, NULL, in_mad, response_mad: out_mad); |
78 | if (err) |
79 | goto out; |
80 | |
81 | props->device_cap_flags = mdev->device_cap_flags; |
82 | props->vendor_id = be32_to_cpup(p: (__be32 *) (out_mad->data + 36)) & |
83 | 0xffffff; |
84 | props->vendor_part_id = be16_to_cpup(p: (__be16 *) (out_mad->data + 30)); |
85 | props->hw_ver = be32_to_cpup(p: (__be32 *) (out_mad->data + 32)); |
86 | memcpy(&props->sys_image_guid, out_mad->data + 4, 8); |
87 | |
88 | props->max_mr_size = ~0ull; |
89 | props->page_size_cap = mdev->limits.page_size_cap; |
90 | props->max_qp = mdev->limits.num_qps - mdev->limits.reserved_qps; |
91 | props->max_qp_wr = mdev->limits.max_wqes; |
92 | props->max_send_sge = mdev->limits.max_sg; |
93 | props->max_recv_sge = mdev->limits.max_sg; |
94 | props->max_sge_rd = mdev->limits.max_sg; |
95 | props->max_cq = mdev->limits.num_cqs - mdev->limits.reserved_cqs; |
96 | props->max_cqe = mdev->limits.max_cqes; |
97 | props->max_mr = mdev->limits.num_mpts - mdev->limits.reserved_mrws; |
98 | props->max_pd = mdev->limits.num_pds - mdev->limits.reserved_pds; |
99 | props->max_qp_rd_atom = 1 << mdev->qp_table.rdb_shift; |
100 | props->max_qp_init_rd_atom = mdev->limits.max_qp_init_rdma; |
101 | props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; |
102 | props->max_srq = mdev->limits.num_srqs - mdev->limits.reserved_srqs; |
103 | props->max_srq_wr = mdev->limits.max_srq_wqes; |
104 | props->max_srq_sge = mdev->limits.max_srq_sge; |
105 | props->local_ca_ack_delay = mdev->limits.local_ca_ack_delay; |
106 | props->atomic_cap = mdev->limits.flags & DEV_LIM_FLAG_ATOMIC ? |
107 | IB_ATOMIC_HCA : IB_ATOMIC_NONE; |
108 | props->max_pkeys = mdev->limits.pkey_table_len; |
109 | props->max_mcast_grp = mdev->limits.num_mgms + mdev->limits.num_amgms; |
110 | props->max_mcast_qp_attach = MTHCA_QP_PER_MGM; |
111 | props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * |
112 | props->max_mcast_grp; |
113 | |
114 | err = 0; |
115 | out: |
116 | kfree(objp: in_mad); |
117 | kfree(objp: out_mad); |
118 | return err; |
119 | } |
120 | |
121 | static int mthca_query_port(struct ib_device *ibdev, |
122 | u32 port, struct ib_port_attr *props) |
123 | { |
124 | struct ib_smp *in_mad; |
125 | struct ib_smp *out_mad; |
126 | int err = -ENOMEM; |
127 | |
128 | in_mad = kzalloc(size: sizeof *in_mad, GFP_KERNEL); |
129 | out_mad = kmalloc(size: sizeof *out_mad, GFP_KERNEL); |
130 | if (!in_mad || !out_mad) |
131 | goto out; |
132 | |
133 | /* props being zeroed by the caller, avoid zeroing it here */ |
134 | |
135 | ib_init_query_mad(mad: in_mad); |
136 | in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; |
137 | in_mad->attr_mod = cpu_to_be32(port); |
138 | |
139 | err = mthca_MAD_IFC(dev: to_mdev(ibdev), ignore_mkey: 1, ignore_bkey: 1, |
140 | port, NULL, NULL, in_mad, response_mad: out_mad); |
141 | if (err) |
142 | goto out; |
143 | |
144 | props->lid = be16_to_cpup(p: (__be16 *) (out_mad->data + 16)); |
145 | props->lmc = out_mad->data[34] & 0x7; |
146 | props->sm_lid = be16_to_cpup(p: (__be16 *) (out_mad->data + 18)); |
147 | props->sm_sl = out_mad->data[36] & 0xf; |
148 | props->state = out_mad->data[32] & 0xf; |
149 | props->phys_state = out_mad->data[33] >> 4; |
150 | props->port_cap_flags = be32_to_cpup(p: (__be32 *) (out_mad->data + 20)); |
151 | props->gid_tbl_len = to_mdev(ibdev)->limits.gid_table_len; |
152 | props->max_msg_sz = 0x80000000; |
153 | props->pkey_tbl_len = to_mdev(ibdev)->limits.pkey_table_len; |
154 | props->bad_pkey_cntr = be16_to_cpup(p: (__be16 *) (out_mad->data + 46)); |
155 | props->qkey_viol_cntr = be16_to_cpup(p: (__be16 *) (out_mad->data + 48)); |
156 | props->active_width = out_mad->data[31] & 0xf; |
157 | props->active_speed = out_mad->data[35] >> 4; |
158 | props->max_mtu = out_mad->data[41] & 0xf; |
159 | props->active_mtu = out_mad->data[36] >> 4; |
160 | props->subnet_timeout = out_mad->data[51] & 0x1f; |
161 | props->max_vl_num = out_mad->data[37] >> 4; |
162 | props->init_type_reply = out_mad->data[41] >> 4; |
163 | |
164 | out: |
165 | kfree(objp: in_mad); |
166 | kfree(objp: out_mad); |
167 | return err; |
168 | } |
169 | |
170 | static int mthca_modify_device(struct ib_device *ibdev, |
171 | int mask, |
172 | struct ib_device_modify *props) |
173 | { |
174 | if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) |
175 | return -EOPNOTSUPP; |
176 | |
177 | if (mask & IB_DEVICE_MODIFY_NODE_DESC) { |
178 | if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex)) |
179 | return -ERESTARTSYS; |
180 | memcpy(ibdev->node_desc, props->node_desc, |
181 | IB_DEVICE_NODE_DESC_MAX); |
182 | mutex_unlock(lock: &to_mdev(ibdev)->cap_mask_mutex); |
183 | } |
184 | |
185 | return 0; |
186 | } |
187 | |
188 | static int mthca_modify_port(struct ib_device *ibdev, |
189 | u32 port, int port_modify_mask, |
190 | struct ib_port_modify *props) |
191 | { |
192 | struct mthca_set_ib_param set_ib; |
193 | struct ib_port_attr attr; |
194 | int err; |
195 | |
196 | if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex)) |
197 | return -ERESTARTSYS; |
198 | |
199 | err = ib_query_port(device: ibdev, port_num: port, port_attr: &attr); |
200 | if (err) |
201 | goto out; |
202 | |
203 | set_ib.set_si_guid = 0; |
204 | set_ib.reset_qkey_viol = !!(port_modify_mask & IB_PORT_RESET_QKEY_CNTR); |
205 | |
206 | set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) & |
207 | ~props->clr_port_cap_mask; |
208 | |
209 | err = mthca_SET_IB(dev: to_mdev(ibdev), param: &set_ib, port); |
210 | if (err) |
211 | goto out; |
212 | out: |
213 | mutex_unlock(lock: &to_mdev(ibdev)->cap_mask_mutex); |
214 | return err; |
215 | } |
216 | |
217 | static int mthca_query_pkey(struct ib_device *ibdev, |
218 | u32 port, u16 index, u16 *pkey) |
219 | { |
220 | struct ib_smp *in_mad; |
221 | struct ib_smp *out_mad; |
222 | int err = -ENOMEM; |
223 | |
224 | in_mad = kzalloc(size: sizeof *in_mad, GFP_KERNEL); |
225 | out_mad = kmalloc(size: sizeof *out_mad, GFP_KERNEL); |
226 | if (!in_mad || !out_mad) |
227 | goto out; |
228 | |
229 | ib_init_query_mad(mad: in_mad); |
230 | in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE; |
231 | in_mad->attr_mod = cpu_to_be32(index / 32); |
232 | |
233 | err = mthca_MAD_IFC(dev: to_mdev(ibdev), ignore_mkey: 1, ignore_bkey: 1, |
234 | port, NULL, NULL, in_mad, response_mad: out_mad); |
235 | if (err) |
236 | goto out; |
237 | |
238 | *pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]); |
239 | |
240 | out: |
241 | kfree(objp: in_mad); |
242 | kfree(objp: out_mad); |
243 | return err; |
244 | } |
245 | |
246 | static int mthca_query_gid(struct ib_device *ibdev, u32 port, |
247 | int index, union ib_gid *gid) |
248 | { |
249 | struct ib_smp *in_mad; |
250 | struct ib_smp *out_mad; |
251 | int err = -ENOMEM; |
252 | |
253 | in_mad = kzalloc(size: sizeof *in_mad, GFP_KERNEL); |
254 | out_mad = kmalloc(size: sizeof *out_mad, GFP_KERNEL); |
255 | if (!in_mad || !out_mad) |
256 | goto out; |
257 | |
258 | ib_init_query_mad(mad: in_mad); |
259 | in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; |
260 | in_mad->attr_mod = cpu_to_be32(port); |
261 | |
262 | err = mthca_MAD_IFC(dev: to_mdev(ibdev), ignore_mkey: 1, ignore_bkey: 1, |
263 | port, NULL, NULL, in_mad, response_mad: out_mad); |
264 | if (err) |
265 | goto out; |
266 | |
267 | memcpy(gid->raw, out_mad->data + 8, 8); |
268 | |
269 | ib_init_query_mad(mad: in_mad); |
270 | in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; |
271 | in_mad->attr_mod = cpu_to_be32(index / 8); |
272 | |
273 | err = mthca_MAD_IFC(dev: to_mdev(ibdev), ignore_mkey: 1, ignore_bkey: 1, |
274 | port, NULL, NULL, in_mad, response_mad: out_mad); |
275 | if (err) |
276 | goto out; |
277 | |
278 | memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8); |
279 | |
280 | out: |
281 | kfree(objp: in_mad); |
282 | kfree(objp: out_mad); |
283 | return err; |
284 | } |
285 | |
286 | static int mthca_alloc_ucontext(struct ib_ucontext *uctx, |
287 | struct ib_udata *udata) |
288 | { |
289 | struct ib_device *ibdev = uctx->device; |
290 | struct mthca_alloc_ucontext_resp uresp = {}; |
291 | struct mthca_ucontext *context = to_mucontext(ibucontext: uctx); |
292 | int err; |
293 | |
294 | if (!(to_mdev(ibdev)->active)) |
295 | return -EAGAIN; |
296 | |
297 | uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps; |
298 | if (mthca_is_memfree(dev: to_mdev(ibdev))) |
299 | uresp.uarc_size = to_mdev(ibdev)->uar_table.uarc_size; |
300 | else |
301 | uresp.uarc_size = 0; |
302 | |
303 | err = mthca_uar_alloc(dev: to_mdev(ibdev), uar: &context->uar); |
304 | if (err) |
305 | return err; |
306 | |
307 | context->db_tab = mthca_init_user_db_tab(dev: to_mdev(ibdev)); |
308 | if (IS_ERR(ptr: context->db_tab)) { |
309 | err = PTR_ERR(ptr: context->db_tab); |
310 | mthca_uar_free(dev: to_mdev(ibdev), uar: &context->uar); |
311 | return err; |
312 | } |
313 | |
314 | if (ib_copy_to_udata(udata, src: &uresp, len: sizeof(uresp))) { |
315 | mthca_cleanup_user_db_tab(dev: to_mdev(ibdev), uar: &context->uar, db_tab: context->db_tab); |
316 | mthca_uar_free(dev: to_mdev(ibdev), uar: &context->uar); |
317 | return -EFAULT; |
318 | } |
319 | |
320 | context->reg_mr_warned = 0; |
321 | |
322 | return 0; |
323 | } |
324 | |
325 | static void mthca_dealloc_ucontext(struct ib_ucontext *context) |
326 | { |
327 | mthca_cleanup_user_db_tab(dev: to_mdev(ibdev: context->device), uar: &to_mucontext(ibucontext: context)->uar, |
328 | db_tab: to_mucontext(ibucontext: context)->db_tab); |
329 | mthca_uar_free(dev: to_mdev(ibdev: context->device), uar: &to_mucontext(ibucontext: context)->uar); |
330 | } |
331 | |
332 | static int mthca_mmap_uar(struct ib_ucontext *context, |
333 | struct vm_area_struct *vma) |
334 | { |
335 | if (vma->vm_end - vma->vm_start != PAGE_SIZE) |
336 | return -EINVAL; |
337 | |
338 | vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); |
339 | |
340 | if (io_remap_pfn_range(vma, addr: vma->vm_start, |
341 | pfn: to_mucontext(ibucontext: context)->uar.pfn, |
342 | PAGE_SIZE, prot: vma->vm_page_prot)) |
343 | return -EAGAIN; |
344 | |
345 | return 0; |
346 | } |
347 | |
348 | static int mthca_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) |
349 | { |
350 | struct ib_device *ibdev = ibpd->device; |
351 | struct mthca_pd *pd = to_mpd(ibpd); |
352 | int err; |
353 | |
354 | err = mthca_pd_alloc(dev: to_mdev(ibdev), privileged: !udata, pd); |
355 | if (err) |
356 | return err; |
357 | |
358 | if (udata) { |
359 | if (ib_copy_to_udata(udata, src: &pd->pd_num, len: sizeof (__u32))) { |
360 | mthca_pd_free(dev: to_mdev(ibdev), pd); |
361 | return -EFAULT; |
362 | } |
363 | } |
364 | |
365 | return 0; |
366 | } |
367 | |
368 | static int mthca_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) |
369 | { |
370 | mthca_pd_free(dev: to_mdev(ibdev: pd->device), pd: to_mpd(ibpd: pd)); |
371 | return 0; |
372 | } |
373 | |
374 | static int mthca_ah_create(struct ib_ah *ibah, |
375 | struct rdma_ah_init_attr *init_attr, |
376 | struct ib_udata *udata) |
377 | |
378 | { |
379 | struct mthca_ah *ah = to_mah(ibah); |
380 | |
381 | return mthca_create_ah(dev: to_mdev(ibdev: ibah->device), pd: to_mpd(ibpd: ibah->pd), |
382 | ah_attr: init_attr->ah_attr, ah); |
383 | } |
384 | |
385 | static int mthca_ah_destroy(struct ib_ah *ah, u32 flags) |
386 | { |
387 | mthca_destroy_ah(dev: to_mdev(ibdev: ah->device), ah: to_mah(ibah: ah)); |
388 | return 0; |
389 | } |
390 | |
391 | static int mthca_create_srq(struct ib_srq *ibsrq, |
392 | struct ib_srq_init_attr *init_attr, |
393 | struct ib_udata *udata) |
394 | { |
395 | struct mthca_create_srq ucmd; |
396 | struct mthca_ucontext *context = rdma_udata_to_drv_context( |
397 | udata, struct mthca_ucontext, ibucontext); |
398 | struct mthca_srq *srq = to_msrq(ibsrq); |
399 | int err; |
400 | |
401 | if (init_attr->srq_type != IB_SRQT_BASIC) |
402 | return -EOPNOTSUPP; |
403 | |
404 | if (udata) { |
405 | if (ib_copy_from_udata(dest: &ucmd, udata, len: sizeof(ucmd))) |
406 | return -EFAULT; |
407 | |
408 | err = mthca_map_user_db(dev: to_mdev(ibdev: ibsrq->device), uar: &context->uar, |
409 | db_tab: context->db_tab, index: ucmd.db_index, |
410 | uaddr: ucmd.db_page); |
411 | |
412 | if (err) |
413 | return err; |
414 | |
415 | srq->mr.ibmr.lkey = ucmd.lkey; |
416 | srq->db_index = ucmd.db_index; |
417 | } |
418 | |
419 | err = mthca_alloc_srq(dev: to_mdev(ibdev: ibsrq->device), pd: to_mpd(ibpd: ibsrq->pd), |
420 | attr: &init_attr->attr, srq, udata); |
421 | |
422 | if (err && udata) |
423 | mthca_unmap_user_db(dev: to_mdev(ibdev: ibsrq->device), uar: &context->uar, |
424 | db_tab: context->db_tab, index: ucmd.db_index); |
425 | |
426 | if (err) |
427 | return err; |
428 | |
429 | if (context && ib_copy_to_udata(udata, src: &srq->srqn, len: sizeof(__u32))) { |
430 | mthca_free_srq(dev: to_mdev(ibdev: ibsrq->device), srq); |
431 | return -EFAULT; |
432 | } |
433 | |
434 | return 0; |
435 | } |
436 | |
437 | static int mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) |
438 | { |
439 | if (udata) { |
440 | struct mthca_ucontext *context = |
441 | rdma_udata_to_drv_context( |
442 | udata, |
443 | struct mthca_ucontext, |
444 | ibucontext); |
445 | |
446 | mthca_unmap_user_db(dev: to_mdev(ibdev: srq->device), uar: &context->uar, |
447 | db_tab: context->db_tab, index: to_msrq(ibsrq: srq)->db_index); |
448 | } |
449 | |
450 | mthca_free_srq(dev: to_mdev(ibdev: srq->device), srq: to_msrq(ibsrq: srq)); |
451 | return 0; |
452 | } |
453 | |
454 | static int mthca_create_qp(struct ib_qp *ibqp, |
455 | struct ib_qp_init_attr *init_attr, |
456 | struct ib_udata *udata) |
457 | { |
458 | struct mthca_ucontext *context = rdma_udata_to_drv_context( |
459 | udata, struct mthca_ucontext, ibucontext); |
460 | struct mthca_create_qp ucmd; |
461 | struct mthca_qp *qp = to_mqp(ibqp); |
462 | struct mthca_dev *dev = to_mdev(ibdev: ibqp->device); |
463 | int err; |
464 | |
465 | if (init_attr->create_flags) |
466 | return -EOPNOTSUPP; |
467 | |
468 | switch (init_attr->qp_type) { |
469 | case IB_QPT_RC: |
470 | case IB_QPT_UC: |
471 | case IB_QPT_UD: |
472 | { |
473 | if (udata) { |
474 | if (ib_copy_from_udata(dest: &ucmd, udata, len: sizeof(ucmd))) |
475 | return -EFAULT; |
476 | |
477 | err = mthca_map_user_db(dev, uar: &context->uar, |
478 | db_tab: context->db_tab, |
479 | index: ucmd.sq_db_index, |
480 | uaddr: ucmd.sq_db_page); |
481 | if (err) |
482 | return err; |
483 | |
484 | err = mthca_map_user_db(dev, uar: &context->uar, |
485 | db_tab: context->db_tab, |
486 | index: ucmd.rq_db_index, |
487 | uaddr: ucmd.rq_db_page); |
488 | if (err) { |
489 | mthca_unmap_user_db(dev, uar: &context->uar, |
490 | db_tab: context->db_tab, |
491 | index: ucmd.sq_db_index); |
492 | return err; |
493 | } |
494 | |
495 | qp->mr.ibmr.lkey = ucmd.lkey; |
496 | qp->sq.db_index = ucmd.sq_db_index; |
497 | qp->rq.db_index = ucmd.rq_db_index; |
498 | } |
499 | |
500 | err = mthca_alloc_qp(dev, pd: to_mpd(ibpd: ibqp->pd), |
501 | send_cq: to_mcq(ibcq: init_attr->send_cq), |
502 | recv_cq: to_mcq(ibcq: init_attr->recv_cq), |
503 | type: init_attr->qp_type, send_policy: init_attr->sq_sig_type, |
504 | cap: &init_attr->cap, qp, udata); |
505 | |
506 | if (err && udata) { |
507 | mthca_unmap_user_db(dev, uar: &context->uar, db_tab: context->db_tab, |
508 | index: ucmd.sq_db_index); |
509 | mthca_unmap_user_db(dev, uar: &context->uar, db_tab: context->db_tab, |
510 | index: ucmd.rq_db_index); |
511 | } |
512 | |
513 | qp->ibqp.qp_num = qp->qpn; |
514 | break; |
515 | } |
516 | case IB_QPT_SMI: |
517 | case IB_QPT_GSI: |
518 | { |
519 | qp->sqp = kzalloc(size: sizeof(struct mthca_sqp), GFP_KERNEL); |
520 | if (!qp->sqp) |
521 | return -ENOMEM; |
522 | |
523 | qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1; |
524 | |
525 | err = mthca_alloc_sqp(dev, pd: to_mpd(ibpd: ibqp->pd), |
526 | send_cq: to_mcq(ibcq: init_attr->send_cq), |
527 | recv_cq: to_mcq(ibcq: init_attr->recv_cq), |
528 | send_policy: init_attr->sq_sig_type, cap: &init_attr->cap, |
529 | qpn: qp->ibqp.qp_num, port: init_attr->port_num, qp, |
530 | udata); |
531 | break; |
532 | } |
533 | default: |
534 | /* Don't support raw QPs */ |
535 | return -EOPNOTSUPP; |
536 | } |
537 | |
538 | if (err) { |
539 | kfree(objp: qp->sqp); |
540 | return err; |
541 | } |
542 | |
543 | init_attr->cap.max_send_wr = qp->sq.max; |
544 | init_attr->cap.max_recv_wr = qp->rq.max; |
545 | init_attr->cap.max_send_sge = qp->sq.max_gs; |
546 | init_attr->cap.max_recv_sge = qp->rq.max_gs; |
547 | init_attr->cap.max_inline_data = qp->max_inline_data; |
548 | |
549 | return 0; |
550 | } |
551 | |
552 | static int mthca_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) |
553 | { |
554 | if (udata) { |
555 | struct mthca_ucontext *context = |
556 | rdma_udata_to_drv_context( |
557 | udata, |
558 | struct mthca_ucontext, |
559 | ibucontext); |
560 | |
561 | mthca_unmap_user_db(dev: to_mdev(ibdev: qp->device), |
562 | uar: &context->uar, |
563 | db_tab: context->db_tab, |
564 | index: to_mqp(ibqp: qp)->sq.db_index); |
565 | mthca_unmap_user_db(dev: to_mdev(ibdev: qp->device), |
566 | uar: &context->uar, |
567 | db_tab: context->db_tab, |
568 | index: to_mqp(ibqp: qp)->rq.db_index); |
569 | } |
570 | mthca_free_qp(dev: to_mdev(ibdev: qp->device), qp: to_mqp(ibqp: qp)); |
571 | kfree(objp: to_mqp(ibqp: qp)->sqp); |
572 | return 0; |
573 | } |
574 | |
575 | static int mthca_create_cq(struct ib_cq *ibcq, |
576 | const struct ib_cq_init_attr *attr, |
577 | struct ib_udata *udata) |
578 | { |
579 | struct ib_device *ibdev = ibcq->device; |
580 | int entries = attr->cqe; |
581 | struct mthca_create_cq ucmd; |
582 | struct mthca_cq *cq; |
583 | int nent; |
584 | int err; |
585 | struct mthca_ucontext *context = rdma_udata_to_drv_context( |
586 | udata, struct mthca_ucontext, ibucontext); |
587 | |
588 | if (attr->flags) |
589 | return -EOPNOTSUPP; |
590 | |
591 | if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes) |
592 | return -EINVAL; |
593 | |
594 | if (udata) { |
595 | if (ib_copy_from_udata(dest: &ucmd, udata, len: sizeof(ucmd))) |
596 | return -EFAULT; |
597 | |
598 | err = mthca_map_user_db(dev: to_mdev(ibdev), uar: &context->uar, |
599 | db_tab: context->db_tab, index: ucmd.set_db_index, |
600 | uaddr: ucmd.set_db_page); |
601 | if (err) |
602 | return err; |
603 | |
604 | err = mthca_map_user_db(dev: to_mdev(ibdev), uar: &context->uar, |
605 | db_tab: context->db_tab, index: ucmd.arm_db_index, |
606 | uaddr: ucmd.arm_db_page); |
607 | if (err) |
608 | goto err_unmap_set; |
609 | } |
610 | |
611 | cq = to_mcq(ibcq); |
612 | |
613 | if (udata) { |
614 | cq->buf.mr.ibmr.lkey = ucmd.lkey; |
615 | cq->set_ci_db_index = ucmd.set_db_index; |
616 | cq->arm_db_index = ucmd.arm_db_index; |
617 | } |
618 | |
619 | for (nent = 1; nent <= entries; nent <<= 1) |
620 | ; /* nothing */ |
621 | |
622 | err = mthca_init_cq(dev: to_mdev(ibdev), nent, ctx: context, |
623 | pdn: udata ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num, |
624 | cq); |
625 | if (err) |
626 | goto err_unmap_arm; |
627 | |
628 | if (udata && ib_copy_to_udata(udata, src: &cq->cqn, len: sizeof(__u32))) { |
629 | mthca_free_cq(dev: to_mdev(ibdev), cq); |
630 | err = -EFAULT; |
631 | goto err_unmap_arm; |
632 | } |
633 | |
634 | cq->resize_buf = NULL; |
635 | |
636 | return 0; |
637 | |
638 | err_unmap_arm: |
639 | if (udata) |
640 | mthca_unmap_user_db(dev: to_mdev(ibdev), uar: &context->uar, |
641 | db_tab: context->db_tab, index: ucmd.arm_db_index); |
642 | |
643 | err_unmap_set: |
644 | if (udata) |
645 | mthca_unmap_user_db(dev: to_mdev(ibdev), uar: &context->uar, |
646 | db_tab: context->db_tab, index: ucmd.set_db_index); |
647 | |
648 | return err; |
649 | } |
650 | |
651 | static int mthca_alloc_resize_buf(struct mthca_dev *dev, struct mthca_cq *cq, |
652 | int entries) |
653 | { |
654 | int ret; |
655 | |
656 | spin_lock_irq(lock: &cq->lock); |
657 | if (cq->resize_buf) { |
658 | ret = -EBUSY; |
659 | goto unlock; |
660 | } |
661 | |
662 | cq->resize_buf = kmalloc(size: sizeof *cq->resize_buf, GFP_ATOMIC); |
663 | if (!cq->resize_buf) { |
664 | ret = -ENOMEM; |
665 | goto unlock; |
666 | } |
667 | |
668 | cq->resize_buf->state = CQ_RESIZE_ALLOC; |
669 | |
670 | ret = 0; |
671 | |
672 | unlock: |
673 | spin_unlock_irq(lock: &cq->lock); |
674 | |
675 | if (ret) |
676 | return ret; |
677 | |
678 | ret = mthca_alloc_cq_buf(dev, buf: &cq->resize_buf->buf, nent: entries); |
679 | if (ret) { |
680 | spin_lock_irq(lock: &cq->lock); |
681 | kfree(objp: cq->resize_buf); |
682 | cq->resize_buf = NULL; |
683 | spin_unlock_irq(lock: &cq->lock); |
684 | return ret; |
685 | } |
686 | |
687 | cq->resize_buf->cqe = entries - 1; |
688 | |
689 | spin_lock_irq(lock: &cq->lock); |
690 | cq->resize_buf->state = CQ_RESIZE_READY; |
691 | spin_unlock_irq(lock: &cq->lock); |
692 | |
693 | return 0; |
694 | } |
695 | |
696 | static int mthca_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) |
697 | { |
698 | struct mthca_dev *dev = to_mdev(ibdev: ibcq->device); |
699 | struct mthca_cq *cq = to_mcq(ibcq); |
700 | struct mthca_resize_cq ucmd; |
701 | u32 lkey; |
702 | int ret; |
703 | |
704 | if (entries < 1 || entries > dev->limits.max_cqes) |
705 | return -EINVAL; |
706 | |
707 | mutex_lock(&cq->mutex); |
708 | |
709 | entries = roundup_pow_of_two(entries + 1); |
710 | if (entries == ibcq->cqe + 1) { |
711 | ret = 0; |
712 | goto out; |
713 | } |
714 | |
715 | if (cq->is_kernel) { |
716 | ret = mthca_alloc_resize_buf(dev, cq, entries); |
717 | if (ret) |
718 | goto out; |
719 | lkey = cq->resize_buf->buf.mr.ibmr.lkey; |
720 | } else { |
721 | if (ib_copy_from_udata(dest: &ucmd, udata, len: sizeof ucmd)) { |
722 | ret = -EFAULT; |
723 | goto out; |
724 | } |
725 | lkey = ucmd.lkey; |
726 | } |
727 | |
728 | ret = mthca_RESIZE_CQ(dev, cq_num: cq->cqn, lkey, ilog2(entries)); |
729 | |
730 | if (ret) { |
731 | if (cq->resize_buf) { |
732 | mthca_free_cq_buf(dev, buf: &cq->resize_buf->buf, |
733 | cqe: cq->resize_buf->cqe); |
734 | kfree(objp: cq->resize_buf); |
735 | spin_lock_irq(lock: &cq->lock); |
736 | cq->resize_buf = NULL; |
737 | spin_unlock_irq(lock: &cq->lock); |
738 | } |
739 | goto out; |
740 | } |
741 | |
742 | if (cq->is_kernel) { |
743 | struct mthca_cq_buf tbuf; |
744 | int tcqe; |
745 | |
746 | spin_lock_irq(lock: &cq->lock); |
747 | if (cq->resize_buf->state == CQ_RESIZE_READY) { |
748 | mthca_cq_resize_copy_cqes(cq); |
749 | tbuf = cq->buf; |
750 | tcqe = cq->ibcq.cqe; |
751 | cq->buf = cq->resize_buf->buf; |
752 | cq->ibcq.cqe = cq->resize_buf->cqe; |
753 | } else { |
754 | tbuf = cq->resize_buf->buf; |
755 | tcqe = cq->resize_buf->cqe; |
756 | } |
757 | |
758 | kfree(objp: cq->resize_buf); |
759 | cq->resize_buf = NULL; |
760 | spin_unlock_irq(lock: &cq->lock); |
761 | |
762 | mthca_free_cq_buf(dev, buf: &tbuf, cqe: tcqe); |
763 | } else |
764 | ibcq->cqe = entries - 1; |
765 | |
766 | out: |
767 | mutex_unlock(lock: &cq->mutex); |
768 | |
769 | return ret; |
770 | } |
771 | |
772 | static int mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) |
773 | { |
774 | if (udata) { |
775 | struct mthca_ucontext *context = |
776 | rdma_udata_to_drv_context( |
777 | udata, |
778 | struct mthca_ucontext, |
779 | ibucontext); |
780 | |
781 | mthca_unmap_user_db(dev: to_mdev(ibdev: cq->device), |
782 | uar: &context->uar, |
783 | db_tab: context->db_tab, |
784 | index: to_mcq(ibcq: cq)->arm_db_index); |
785 | mthca_unmap_user_db(dev: to_mdev(ibdev: cq->device), |
786 | uar: &context->uar, |
787 | db_tab: context->db_tab, |
788 | index: to_mcq(ibcq: cq)->set_ci_db_index); |
789 | } |
790 | mthca_free_cq(dev: to_mdev(ibdev: cq->device), cq: to_mcq(ibcq: cq)); |
791 | return 0; |
792 | } |
793 | |
794 | static inline u32 convert_access(int acc) |
795 | { |
796 | return (acc & IB_ACCESS_REMOTE_ATOMIC ? MTHCA_MPT_FLAG_ATOMIC : 0) | |
797 | (acc & IB_ACCESS_REMOTE_WRITE ? MTHCA_MPT_FLAG_REMOTE_WRITE : 0) | |
798 | (acc & IB_ACCESS_REMOTE_READ ? MTHCA_MPT_FLAG_REMOTE_READ : 0) | |
799 | (acc & IB_ACCESS_LOCAL_WRITE ? MTHCA_MPT_FLAG_LOCAL_WRITE : 0) | |
800 | MTHCA_MPT_FLAG_LOCAL_READ; |
801 | } |
802 | |
803 | static struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, int acc) |
804 | { |
805 | struct mthca_mr *mr; |
806 | int err; |
807 | |
808 | mr = kmalloc(size: sizeof *mr, GFP_KERNEL); |
809 | if (!mr) |
810 | return ERR_PTR(error: -ENOMEM); |
811 | |
812 | err = mthca_mr_alloc_notrans(dev: to_mdev(ibdev: pd->device), |
813 | pd: to_mpd(ibpd: pd)->pd_num, |
814 | access: convert_access(acc), mr); |
815 | |
816 | if (err) { |
817 | kfree(objp: mr); |
818 | return ERR_PTR(error: err); |
819 | } |
820 | |
821 | mr->umem = NULL; |
822 | |
823 | return &mr->ibmr; |
824 | } |
825 | |
826 | static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, |
827 | u64 virt, int acc, struct ib_udata *udata) |
828 | { |
829 | struct mthca_dev *dev = to_mdev(ibdev: pd->device); |
830 | struct ib_block_iter biter; |
831 | struct mthca_ucontext *context = rdma_udata_to_drv_context( |
832 | udata, struct mthca_ucontext, ibucontext); |
833 | struct mthca_mr *mr; |
834 | struct mthca_reg_mr ucmd; |
835 | u64 *pages; |
836 | int n, i; |
837 | int err = 0; |
838 | int write_mtt_size; |
839 | |
840 | if (udata->inlen < sizeof ucmd) { |
841 | if (!context->reg_mr_warned) { |
842 | mthca_warn(dev, "Process '%s' did not pass in MR attrs.\n" , |
843 | current->comm); |
844 | mthca_warn(dev, " Update libmthca to fix this.\n" ); |
845 | } |
846 | ++context->reg_mr_warned; |
847 | ucmd.mr_attrs = 0; |
848 | } else if (ib_copy_from_udata(dest: &ucmd, udata, len: sizeof ucmd)) |
849 | return ERR_PTR(error: -EFAULT); |
850 | |
851 | mr = kmalloc(size: sizeof *mr, GFP_KERNEL); |
852 | if (!mr) |
853 | return ERR_PTR(error: -ENOMEM); |
854 | |
855 | mr->umem = ib_umem_get(device: pd->device, addr: start, size: length, access: acc); |
856 | if (IS_ERR(ptr: mr->umem)) { |
857 | err = PTR_ERR(ptr: mr->umem); |
858 | goto err; |
859 | } |
860 | |
861 | n = ib_umem_num_dma_blocks(umem: mr->umem, PAGE_SIZE); |
862 | |
863 | mr->mtt = mthca_alloc_mtt(dev, size: n); |
864 | if (IS_ERR(ptr: mr->mtt)) { |
865 | err = PTR_ERR(ptr: mr->mtt); |
866 | goto err_umem; |
867 | } |
868 | |
869 | pages = (u64 *) __get_free_page(GFP_KERNEL); |
870 | if (!pages) { |
871 | err = -ENOMEM; |
872 | goto err_mtt; |
873 | } |
874 | |
875 | i = n = 0; |
876 | |
877 | write_mtt_size = min(mthca_write_mtt_size(dev), (int) (PAGE_SIZE / sizeof *pages)); |
878 | |
879 | rdma_umem_for_each_dma_block(mr->umem, &biter, PAGE_SIZE) { |
880 | pages[i++] = rdma_block_iter_dma_address(biter: &biter); |
881 | |
882 | /* |
883 | * Be friendly to write_mtt and pass it chunks |
884 | * of appropriate size. |
885 | */ |
886 | if (i == write_mtt_size) { |
887 | err = mthca_write_mtt(dev, mtt: mr->mtt, start_index: n, buffer_list: pages, list_len: i); |
888 | if (err) |
889 | goto mtt_done; |
890 | n += i; |
891 | i = 0; |
892 | } |
893 | } |
894 | |
895 | if (i) |
896 | err = mthca_write_mtt(dev, mtt: mr->mtt, start_index: n, buffer_list: pages, list_len: i); |
897 | mtt_done: |
898 | free_page((unsigned long) pages); |
899 | if (err) |
900 | goto err_mtt; |
901 | |
902 | err = mthca_mr_alloc(dev, pd: to_mpd(ibpd: pd)->pd_num, PAGE_SHIFT, iova: virt, total_size: length, |
903 | access: convert_access(acc), mr); |
904 | |
905 | if (err) |
906 | goto err_mtt; |
907 | |
908 | return &mr->ibmr; |
909 | |
910 | err_mtt: |
911 | mthca_free_mtt(dev, mtt: mr->mtt); |
912 | |
913 | err_umem: |
914 | ib_umem_release(umem: mr->umem); |
915 | |
916 | err: |
917 | kfree(objp: mr); |
918 | return ERR_PTR(error: err); |
919 | } |
920 | |
921 | static int mthca_dereg_mr(struct ib_mr *mr, struct ib_udata *udata) |
922 | { |
923 | struct mthca_mr *mmr = to_mmr(ibmr: mr); |
924 | |
925 | mthca_free_mr(dev: to_mdev(ibdev: mr->device), mr: mmr); |
926 | ib_umem_release(umem: mmr->umem); |
927 | kfree(objp: mmr); |
928 | |
929 | return 0; |
930 | } |
931 | |
932 | static ssize_t hw_rev_show(struct device *device, |
933 | struct device_attribute *attr, char *buf) |
934 | { |
935 | struct mthca_dev *dev = |
936 | rdma_device_to_drv_device(device, struct mthca_dev, ib_dev); |
937 | |
938 | return sysfs_emit(buf, fmt: "%x\n" , dev->rev_id); |
939 | } |
940 | static DEVICE_ATTR_RO(hw_rev); |
941 | |
942 | static const char *hca_type_string(int hca_type) |
943 | { |
944 | switch (hca_type) { |
945 | case PCI_DEVICE_ID_MELLANOX_TAVOR: |
946 | return "MT23108" ; |
947 | case PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT: |
948 | return "MT25208 (MT23108 compat mode)" ; |
949 | case PCI_DEVICE_ID_MELLANOX_ARBEL: |
950 | return "MT25208" ; |
951 | case PCI_DEVICE_ID_MELLANOX_SINAI: |
952 | case PCI_DEVICE_ID_MELLANOX_SINAI_OLD: |
953 | return "MT25204" ; |
954 | } |
955 | |
956 | return "unknown" ; |
957 | } |
958 | |
959 | static ssize_t hca_type_show(struct device *device, |
960 | struct device_attribute *attr, char *buf) |
961 | { |
962 | struct mthca_dev *dev = |
963 | rdma_device_to_drv_device(device, struct mthca_dev, ib_dev); |
964 | |
965 | return sysfs_emit(buf, fmt: "%s\n" , hca_type_string(hca_type: dev->pdev->device)); |
966 | } |
967 | static DEVICE_ATTR_RO(hca_type); |
968 | |
969 | static ssize_t board_id_show(struct device *device, |
970 | struct device_attribute *attr, char *buf) |
971 | { |
972 | struct mthca_dev *dev = |
973 | rdma_device_to_drv_device(device, struct mthca_dev, ib_dev); |
974 | |
975 | return sysfs_emit(buf, fmt: "%.*s\n" , MTHCA_BOARD_ID_LEN, dev->board_id); |
976 | } |
977 | static DEVICE_ATTR_RO(board_id); |
978 | |
979 | static struct attribute *mthca_dev_attributes[] = { |
980 | &dev_attr_hw_rev.attr, |
981 | &dev_attr_hca_type.attr, |
982 | &dev_attr_board_id.attr, |
983 | NULL |
984 | }; |
985 | |
986 | static const struct attribute_group mthca_attr_group = { |
987 | .attrs = mthca_dev_attributes, |
988 | }; |
989 | |
990 | static int mthca_init_node_data(struct mthca_dev *dev) |
991 | { |
992 | struct ib_smp *in_mad; |
993 | struct ib_smp *out_mad; |
994 | int err = -ENOMEM; |
995 | |
996 | in_mad = kzalloc(size: sizeof *in_mad, GFP_KERNEL); |
997 | out_mad = kmalloc(size: sizeof *out_mad, GFP_KERNEL); |
998 | if (!in_mad || !out_mad) |
999 | goto out; |
1000 | |
1001 | ib_init_query_mad(mad: in_mad); |
1002 | in_mad->attr_id = IB_SMP_ATTR_NODE_DESC; |
1003 | |
1004 | err = mthca_MAD_IFC(dev, ignore_mkey: 1, ignore_bkey: 1, |
1005 | port: 1, NULL, NULL, in_mad, response_mad: out_mad); |
1006 | if (err) |
1007 | goto out; |
1008 | |
1009 | memcpy(dev->ib_dev.node_desc, out_mad->data, IB_DEVICE_NODE_DESC_MAX); |
1010 | |
1011 | in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; |
1012 | |
1013 | err = mthca_MAD_IFC(dev, ignore_mkey: 1, ignore_bkey: 1, |
1014 | port: 1, NULL, NULL, in_mad, response_mad: out_mad); |
1015 | if (err) |
1016 | goto out; |
1017 | |
1018 | if (mthca_is_memfree(dev)) |
1019 | dev->rev_id = be32_to_cpup(p: (__be32 *) (out_mad->data + 32)); |
1020 | memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8); |
1021 | |
1022 | out: |
1023 | kfree(objp: in_mad); |
1024 | kfree(objp: out_mad); |
1025 | return err; |
1026 | } |
1027 | |
1028 | static int mthca_port_immutable(struct ib_device *ibdev, u32 port_num, |
1029 | struct ib_port_immutable *immutable) |
1030 | { |
1031 | struct ib_port_attr attr; |
1032 | int err; |
1033 | |
1034 | immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB; |
1035 | |
1036 | err = ib_query_port(device: ibdev, port_num, port_attr: &attr); |
1037 | if (err) |
1038 | return err; |
1039 | |
1040 | immutable->pkey_tbl_len = attr.pkey_tbl_len; |
1041 | immutable->gid_tbl_len = attr.gid_tbl_len; |
1042 | immutable->max_mad_size = IB_MGMT_MAD_SIZE; |
1043 | |
1044 | return 0; |
1045 | } |
1046 | |
1047 | static void get_dev_fw_str(struct ib_device *device, char *str) |
1048 | { |
1049 | struct mthca_dev *dev = |
1050 | container_of(device, struct mthca_dev, ib_dev); |
1051 | snprintf(buf: str, IB_FW_VERSION_NAME_MAX, fmt: "%d.%d.%d" , |
1052 | (int) (dev->fw_ver >> 32), |
1053 | (int) (dev->fw_ver >> 16) & 0xffff, |
1054 | (int) dev->fw_ver & 0xffff); |
1055 | } |
1056 | |
1057 | static const struct ib_device_ops mthca_dev_ops = { |
1058 | .owner = THIS_MODULE, |
1059 | .driver_id = RDMA_DRIVER_MTHCA, |
1060 | .uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION, |
1061 | .uverbs_no_driver_id_binding = 1, |
1062 | |
1063 | .alloc_pd = mthca_alloc_pd, |
1064 | .alloc_ucontext = mthca_alloc_ucontext, |
1065 | .attach_mcast = mthca_multicast_attach, |
1066 | .create_ah = mthca_ah_create, |
1067 | .create_cq = mthca_create_cq, |
1068 | .create_qp = mthca_create_qp, |
1069 | .dealloc_pd = mthca_dealloc_pd, |
1070 | .dealloc_ucontext = mthca_dealloc_ucontext, |
1071 | .dereg_mr = mthca_dereg_mr, |
1072 | .destroy_ah = mthca_ah_destroy, |
1073 | .destroy_cq = mthca_destroy_cq, |
1074 | .destroy_qp = mthca_destroy_qp, |
1075 | .detach_mcast = mthca_multicast_detach, |
1076 | .device_group = &mthca_attr_group, |
1077 | .get_dev_fw_str = get_dev_fw_str, |
1078 | .get_dma_mr = mthca_get_dma_mr, |
1079 | .get_port_immutable = mthca_port_immutable, |
1080 | .mmap = mthca_mmap_uar, |
1081 | .modify_device = mthca_modify_device, |
1082 | .modify_port = mthca_modify_port, |
1083 | .modify_qp = mthca_modify_qp, |
1084 | .poll_cq = mthca_poll_cq, |
1085 | .process_mad = mthca_process_mad, |
1086 | .query_ah = mthca_ah_query, |
1087 | .query_device = mthca_query_device, |
1088 | .query_gid = mthca_query_gid, |
1089 | .query_pkey = mthca_query_pkey, |
1090 | .query_port = mthca_query_port, |
1091 | .query_qp = mthca_query_qp, |
1092 | .reg_user_mr = mthca_reg_user_mr, |
1093 | .resize_cq = mthca_resize_cq, |
1094 | |
1095 | INIT_RDMA_OBJ_SIZE(ib_ah, mthca_ah, ibah), |
1096 | INIT_RDMA_OBJ_SIZE(ib_cq, mthca_cq, ibcq), |
1097 | INIT_RDMA_OBJ_SIZE(ib_pd, mthca_pd, ibpd), |
1098 | INIT_RDMA_OBJ_SIZE(ib_qp, mthca_qp, ibqp), |
1099 | INIT_RDMA_OBJ_SIZE(ib_ucontext, mthca_ucontext, ibucontext), |
1100 | }; |
1101 | |
1102 | static const struct ib_device_ops mthca_dev_arbel_srq_ops = { |
1103 | .create_srq = mthca_create_srq, |
1104 | .destroy_srq = mthca_destroy_srq, |
1105 | .modify_srq = mthca_modify_srq, |
1106 | .post_srq_recv = mthca_arbel_post_srq_recv, |
1107 | .query_srq = mthca_query_srq, |
1108 | |
1109 | INIT_RDMA_OBJ_SIZE(ib_srq, mthca_srq, ibsrq), |
1110 | }; |
1111 | |
1112 | static const struct ib_device_ops mthca_dev_tavor_srq_ops = { |
1113 | .create_srq = mthca_create_srq, |
1114 | .destroy_srq = mthca_destroy_srq, |
1115 | .modify_srq = mthca_modify_srq, |
1116 | .post_srq_recv = mthca_tavor_post_srq_recv, |
1117 | .query_srq = mthca_query_srq, |
1118 | |
1119 | INIT_RDMA_OBJ_SIZE(ib_srq, mthca_srq, ibsrq), |
1120 | }; |
1121 | |
1122 | static const struct ib_device_ops mthca_dev_arbel_ops = { |
1123 | .post_recv = mthca_arbel_post_receive, |
1124 | .post_send = mthca_arbel_post_send, |
1125 | .req_notify_cq = mthca_arbel_arm_cq, |
1126 | }; |
1127 | |
1128 | static const struct ib_device_ops mthca_dev_tavor_ops = { |
1129 | .post_recv = mthca_tavor_post_receive, |
1130 | .post_send = mthca_tavor_post_send, |
1131 | .req_notify_cq = mthca_tavor_arm_cq, |
1132 | }; |
1133 | |
1134 | int mthca_register_device(struct mthca_dev *dev) |
1135 | { |
1136 | int ret; |
1137 | |
1138 | ret = mthca_init_node_data(dev); |
1139 | if (ret) |
1140 | return ret; |
1141 | |
1142 | dev->ib_dev.node_type = RDMA_NODE_IB_CA; |
1143 | dev->ib_dev.phys_port_cnt = dev->limits.num_ports; |
1144 | dev->ib_dev.num_comp_vectors = 1; |
1145 | dev->ib_dev.dev.parent = &dev->pdev->dev; |
1146 | |
1147 | if (dev->mthca_flags & MTHCA_FLAG_SRQ) { |
1148 | if (mthca_is_memfree(dev)) |
1149 | ib_set_device_ops(device: &dev->ib_dev, |
1150 | ops: &mthca_dev_arbel_srq_ops); |
1151 | else |
1152 | ib_set_device_ops(device: &dev->ib_dev, |
1153 | ops: &mthca_dev_tavor_srq_ops); |
1154 | } |
1155 | |
1156 | ib_set_device_ops(device: &dev->ib_dev, ops: &mthca_dev_ops); |
1157 | |
1158 | if (mthca_is_memfree(dev)) |
1159 | ib_set_device_ops(device: &dev->ib_dev, ops: &mthca_dev_arbel_ops); |
1160 | else |
1161 | ib_set_device_ops(device: &dev->ib_dev, ops: &mthca_dev_tavor_ops); |
1162 | |
1163 | mutex_init(&dev->cap_mask_mutex); |
1164 | |
1165 | ret = ib_register_device(device: &dev->ib_dev, name: "mthca%d" , dma_device: &dev->pdev->dev); |
1166 | if (ret) |
1167 | return ret; |
1168 | |
1169 | mthca_start_catas_poll(dev); |
1170 | |
1171 | return 0; |
1172 | } |
1173 | |
1174 | void mthca_unregister_device(struct mthca_dev *dev) |
1175 | { |
1176 | mthca_stop_catas_poll(dev); |
1177 | ib_unregister_device(device: &dev->ib_dev); |
1178 | } |
1179 | |