1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2/*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7#ifndef RXE_VERBS_H
8#define RXE_VERBS_H
9
10#include <linux/interrupt.h>
11#include <linux/workqueue.h>
12#include "rxe_pool.h"
13#include "rxe_task.h"
14#include "rxe_hw_counters.h"
15
16static inline int pkey_match(u16 key1, u16 key2)
17{
18 return (((key1 & 0x7fff) != 0) &&
19 ((key1 & 0x7fff) == (key2 & 0x7fff)) &&
20 ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0;
21}
22
23/* Return >0 if psn_a > psn_b
24 * 0 if psn_a == psn_b
25 * <0 if psn_a < psn_b
26 */
27static inline int psn_compare(u32 psn_a, u32 psn_b)
28{
29 s32 diff;
30
31 diff = (psn_a - psn_b) << 8;
32 return diff;
33}
34
35struct rxe_ucontext {
36 struct ib_ucontext ibuc;
37 struct rxe_pool_elem elem;
38};
39
40struct rxe_pd {
41 struct ib_pd ibpd;
42 struct rxe_pool_elem elem;
43};
44
45struct rxe_ah {
46 struct ib_ah ibah;
47 struct rxe_pool_elem elem;
48 struct rxe_av av;
49 bool is_user;
50 int ah_num;
51};
52
53struct rxe_cqe {
54 union {
55 struct ib_wc ibwc;
56 struct ib_uverbs_wc uibwc;
57 };
58};
59
60struct rxe_cq {
61 struct ib_cq ibcq;
62 struct rxe_pool_elem elem;
63 struct rxe_queue *queue;
64 spinlock_t cq_lock;
65 u8 notify;
66 bool is_user;
67 atomic_t num_wq;
68};
69
70enum wqe_state {
71 wqe_state_posted,
72 wqe_state_processing,
73 wqe_state_pending,
74 wqe_state_done,
75 wqe_state_error,
76};
77
78struct rxe_sq {
79 int max_wr;
80 int max_sge;
81 int max_inline;
82 spinlock_t sq_lock; /* guard queue */
83 struct rxe_queue *queue;
84};
85
86struct rxe_rq {
87 int max_wr;
88 int max_sge;
89 spinlock_t producer_lock; /* guard queue producer */
90 spinlock_t consumer_lock; /* guard queue consumer */
91 struct rxe_queue *queue;
92};
93
94struct rxe_srq {
95 struct ib_srq ibsrq;
96 struct rxe_pool_elem elem;
97 struct rxe_pd *pd;
98 struct rxe_rq rq;
99 u32 srq_num;
100
101 int limit;
102 int error;
103};
104
105struct rxe_req_info {
106 int wqe_index;
107 u32 psn;
108 int opcode;
109 atomic_t rd_atomic;
110 int wait_fence;
111 int need_rd_atomic;
112 int wait_psn;
113 int need_retry;
114 int wait_for_rnr_timer;
115 int noack_pkts;
116 int again;
117};
118
119struct rxe_comp_info {
120 u32 psn;
121 int opcode;
122 int timeout;
123 int timeout_retry;
124 int started_retry;
125 u32 retry_cnt;
126 u32 rnr_retry;
127};
128
129/* responder states */
130enum resp_states {
131 RESPST_NONE,
132 RESPST_GET_REQ,
133 RESPST_CHK_PSN,
134 RESPST_CHK_OP_SEQ,
135 RESPST_CHK_OP_VALID,
136 RESPST_CHK_RESOURCE,
137 RESPST_CHK_LENGTH,
138 RESPST_CHK_RKEY,
139 RESPST_EXECUTE,
140 RESPST_READ_REPLY,
141 RESPST_ATOMIC_REPLY,
142 RESPST_ATOMIC_WRITE_REPLY,
143 RESPST_PROCESS_FLUSH,
144 RESPST_COMPLETE,
145 RESPST_ACKNOWLEDGE,
146 RESPST_CLEANUP,
147 RESPST_DUPLICATE_REQUEST,
148 RESPST_ERR_MALFORMED_WQE,
149 RESPST_ERR_UNSUPPORTED_OPCODE,
150 RESPST_ERR_MISALIGNED_ATOMIC,
151 RESPST_ERR_PSN_OUT_OF_SEQ,
152 RESPST_ERR_MISSING_OPCODE_FIRST,
153 RESPST_ERR_MISSING_OPCODE_LAST_C,
154 RESPST_ERR_MISSING_OPCODE_LAST_D1E,
155 RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
156 RESPST_ERR_RNR,
157 RESPST_ERR_RKEY_VIOLATION,
158 RESPST_ERR_INVALIDATE_RKEY,
159 RESPST_ERR_LENGTH,
160 RESPST_ERR_CQ_OVERFLOW,
161 RESPST_ERROR,
162 RESPST_DONE,
163 RESPST_EXIT,
164};
165
166enum rdatm_res_state {
167 rdatm_res_state_next,
168 rdatm_res_state_new,
169 rdatm_res_state_replay,
170};
171
172struct resp_res {
173 int type;
174 int replay;
175 u32 first_psn;
176 u32 last_psn;
177 u32 cur_psn;
178 enum rdatm_res_state state;
179
180 union {
181 struct {
182 u64 orig_val;
183 } atomic;
184 struct {
185 u64 va_org;
186 u32 rkey;
187 u32 length;
188 u64 va;
189 u32 resid;
190 } read;
191 struct {
192 u32 length;
193 u64 va;
194 u8 type;
195 u8 level;
196 } flush;
197 };
198};
199
200struct rxe_resp_info {
201 u32 msn;
202 u32 psn;
203 u32 ack_psn;
204 int opcode;
205 int drop_msg;
206 int goto_error;
207 int sent_psn_nak;
208 enum ib_wc_status status;
209 u8 aeth_syndrome;
210
211 /* Receive only */
212 struct rxe_recv_wqe *wqe;
213
214 /* RDMA read / atomic only */
215 u64 va;
216 u64 offset;
217 struct rxe_mr *mr;
218 u32 resid;
219 u32 rkey;
220 u32 length;
221
222 /* SRQ only */
223 struct {
224 struct rxe_recv_wqe wqe;
225 struct ib_sge sge[RXE_MAX_SGE];
226 } srq_wqe;
227
228 /* Responder resources. It's a circular list where the oldest
229 * resource is dropped first.
230 */
231 struct resp_res *resources;
232 unsigned int res_head;
233 unsigned int res_tail;
234 struct resp_res *res;
235};
236
237struct rxe_qp {
238 struct ib_qp ibqp;
239 struct rxe_pool_elem elem;
240 struct ib_qp_attr attr;
241 unsigned int valid;
242 unsigned int mtu;
243 bool is_user;
244
245 struct rxe_pd *pd;
246 struct rxe_srq *srq;
247 struct rxe_cq *scq;
248 struct rxe_cq *rcq;
249
250 enum ib_sig_type sq_sig_type;
251
252 struct rxe_sq sq;
253 struct rxe_rq rq;
254
255 struct socket *sk;
256 u32 dst_cookie;
257 u16 src_port;
258
259 struct rxe_av pri_av;
260 struct rxe_av alt_av;
261
262 atomic_t mcg_num;
263
264 struct sk_buff_head req_pkts;
265 struct sk_buff_head resp_pkts;
266
267 struct rxe_task send_task;
268 struct rxe_task recv_task;
269
270 struct rxe_req_info req;
271 struct rxe_comp_info comp;
272 struct rxe_resp_info resp;
273
274 atomic_t ssn;
275 atomic_t skb_out;
276 int need_req_skb;
277
278 /* Timer for retranmitting packet when ACKs have been lost. RC
279 * only. The requester sets it when it is not already
280 * started. The responder resets it whenever an ack is
281 * received.
282 */
283 struct timer_list retrans_timer;
284 u64 qp_timeout_jiffies;
285
286 /* Timer for handling RNR NAKS. */
287 struct timer_list rnr_nak_timer;
288
289 spinlock_t state_lock; /* guard requester and completer */
290
291 struct execute_work cleanup_work;
292};
293
294enum {
295 RXE_ACCESS_REMOTE = IB_ACCESS_REMOTE_READ
296 | IB_ACCESS_REMOTE_WRITE
297 | IB_ACCESS_REMOTE_ATOMIC,
298 RXE_ACCESS_SUPPORTED_MR = RXE_ACCESS_REMOTE
299 | IB_ACCESS_LOCAL_WRITE
300 | IB_ACCESS_MW_BIND
301 | IB_ACCESS_ON_DEMAND
302 | IB_ACCESS_FLUSH_GLOBAL
303 | IB_ACCESS_FLUSH_PERSISTENT
304 | IB_ACCESS_OPTIONAL,
305 RXE_ACCESS_SUPPORTED_QP = RXE_ACCESS_SUPPORTED_MR,
306 RXE_ACCESS_SUPPORTED_MW = RXE_ACCESS_SUPPORTED_MR
307 | IB_ZERO_BASED,
308};
309
310enum rxe_mr_state {
311 RXE_MR_STATE_INVALID,
312 RXE_MR_STATE_FREE,
313 RXE_MR_STATE_VALID,
314};
315
316enum rxe_mr_copy_dir {
317 RXE_TO_MR_OBJ,
318 RXE_FROM_MR_OBJ,
319};
320
321enum rxe_mr_lookup_type {
322 RXE_LOOKUP_LOCAL,
323 RXE_LOOKUP_REMOTE,
324};
325
326enum rxe_rereg {
327 RXE_MR_REREG_SUPPORTED = IB_MR_REREG_PD
328 | IB_MR_REREG_ACCESS,
329};
330
331static inline int rkey_is_mw(u32 rkey)
332{
333 u32 index = rkey >> 8;
334
335 return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX);
336}
337
338struct rxe_mr {
339 struct rxe_pool_elem elem;
340 struct ib_mr ibmr;
341
342 struct ib_umem *umem;
343
344 u32 lkey;
345 u32 rkey;
346 enum rxe_mr_state state;
347 int access;
348 atomic_t num_mw;
349
350 unsigned int page_offset;
351 unsigned int page_shift;
352 u64 page_mask;
353
354 u32 num_buf;
355 u32 nbuf;
356
357 struct xarray page_list;
358};
359
360static inline unsigned int mr_page_size(struct rxe_mr *mr)
361{
362 return mr ? mr->ibmr.page_size : PAGE_SIZE;
363}
364
365enum rxe_mw_state {
366 RXE_MW_STATE_INVALID = RXE_MR_STATE_INVALID,
367 RXE_MW_STATE_FREE = RXE_MR_STATE_FREE,
368 RXE_MW_STATE_VALID = RXE_MR_STATE_VALID,
369};
370
371struct rxe_mw {
372 struct ib_mw ibmw;
373 struct rxe_pool_elem elem;
374 spinlock_t lock;
375 enum rxe_mw_state state;
376 struct rxe_qp *qp; /* Type 2 only */
377 struct rxe_mr *mr;
378 u32 rkey;
379 int access;
380 u64 addr;
381 u64 length;
382};
383
384struct rxe_mcg {
385 struct rb_node node;
386 struct kref ref_cnt;
387 struct rxe_dev *rxe;
388 struct list_head qp_list;
389 union ib_gid mgid;
390 atomic_t qp_num;
391 u32 qkey;
392 u16 pkey;
393};
394
395struct rxe_mca {
396 struct list_head qp_list;
397 struct rxe_qp *qp;
398};
399
400struct rxe_port {
401 struct ib_port_attr attr;
402 __be64 port_guid;
403 __be64 subnet_prefix;
404 spinlock_t port_lock; /* guard port */
405 unsigned int mtu_cap;
406 /* special QPs */
407 u32 qp_gsi_index;
408};
409
410#define RXE_PORT 1
411struct rxe_dev {
412 struct ib_device ib_dev;
413 struct ib_device_attr attr;
414 int max_ucontext;
415 int max_inline_data;
416 struct mutex usdev_lock;
417
418 char raw_gid[ETH_ALEN];
419
420 struct rxe_pool uc_pool;
421 struct rxe_pool pd_pool;
422 struct rxe_pool ah_pool;
423 struct rxe_pool srq_pool;
424 struct rxe_pool qp_pool;
425 struct rxe_pool cq_pool;
426 struct rxe_pool mr_pool;
427 struct rxe_pool mw_pool;
428
429 /* multicast support */
430 spinlock_t mcg_lock;
431 struct rb_root mcg_tree;
432 atomic_t mcg_num;
433 atomic_t mcg_attach;
434
435 spinlock_t pending_lock; /* guard pending_mmaps */
436 struct list_head pending_mmaps;
437
438 spinlock_t mmap_offset_lock; /* guard mmap_offset */
439 u64 mmap_offset;
440
441 atomic64_t stats_counters[RXE_NUM_OF_COUNTERS];
442
443 struct rxe_port port;
444};
445
446static inline struct net_device *rxe_ib_device_get_netdev(struct ib_device *dev)
447{
448 return ib_device_get_netdev(ib_dev: dev, RXE_PORT);
449}
450
451static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
452{
453 atomic64_inc(v: &rxe->stats_counters[index]);
454}
455
456static inline struct rxe_dev *to_rdev(struct ib_device *dev)
457{
458 return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
459}
460
461static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc)
462{
463 return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL;
464}
465
466static inline struct rxe_pd *to_rpd(struct ib_pd *pd)
467{
468 return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL;
469}
470
471static inline struct rxe_ah *to_rah(struct ib_ah *ah)
472{
473 return ah ? container_of(ah, struct rxe_ah, ibah) : NULL;
474}
475
476static inline struct rxe_srq *to_rsrq(struct ib_srq *srq)
477{
478 return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL;
479}
480
481static inline struct rxe_qp *to_rqp(struct ib_qp *qp)
482{
483 return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL;
484}
485
486static inline struct rxe_cq *to_rcq(struct ib_cq *cq)
487{
488 return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL;
489}
490
491static inline struct rxe_mr *to_rmr(struct ib_mr *mr)
492{
493 return mr ? container_of(mr, struct rxe_mr, ibmr) : NULL;
494}
495
496static inline struct rxe_mw *to_rmw(struct ib_mw *mw)
497{
498 return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL;
499}
500
501static inline struct rxe_pd *rxe_ah_pd(struct rxe_ah *ah)
502{
503 return to_rpd(pd: ah->ibah.pd);
504}
505
506static inline struct rxe_pd *mr_pd(struct rxe_mr *mr)
507{
508 return to_rpd(pd: mr->ibmr.pd);
509}
510
511static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)
512{
513 return to_rpd(pd: mw->ibmw.pd);
514}
515
516int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name,
517 struct net_device *ndev);
518
519#endif /* RXE_VERBS_H */
520

source code of linux/drivers/infiniband/sw/rxe/rxe_verbs.h