| 1 | /* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ |
|---|
| 2 | /* |
|---|
| 3 | * Copyright(c) 2018 Intel Corporation. |
|---|
| 4 | * |
|---|
| 5 | */ |
|---|
| 6 | #ifndef _HFI1_OPFN_H |
|---|
| 7 | #define _HFI1_OPFN_H |
|---|
| 8 | |
|---|
| 9 | /** |
|---|
| 10 | * DOC: Omni Path Feature Negotion (OPFN) |
|---|
| 11 | * |
|---|
| 12 | * OPFN is a discovery protocol for Intel Omni-Path fabric that |
|---|
| 13 | * allows two RC QPs to negotiate a common feature that both QPs |
|---|
| 14 | * can support. Currently, the only OPA feature that OPFN |
|---|
| 15 | * supports is TID RDMA. |
|---|
| 16 | * |
|---|
| 17 | * Architecture |
|---|
| 18 | * |
|---|
| 19 | * OPFN involves the communication between two QPs on the HFI |
|---|
| 20 | * level on an Omni-Path fabric, and ULPs have no knowledge of |
|---|
| 21 | * OPFN at all. |
|---|
| 22 | * |
|---|
| 23 | * Implementation |
|---|
| 24 | * |
|---|
| 25 | * OPFN extends the existing IB RC protocol with the following |
|---|
| 26 | * changes: |
|---|
| 27 | * -- Uses Bit 24 (reserved) of DWORD 1 of Base Transport |
|---|
| 28 | * Header (BTH1) to indicate that the RC QP supports OPFN; |
|---|
| 29 | * -- Uses a combination of RC COMPARE_SWAP opcode (0x13) and |
|---|
| 30 | * the address U64_MAX (0xFFFFFFFFFFFFFFFF) as an OPFN |
|---|
| 31 | * request; The 64-bit data carried with the request/response |
|---|
| 32 | * contains the parameters for negotiation and will be |
|---|
| 33 | * defined in tid_rdma.c file; |
|---|
| 34 | * -- Defines IB_WR_RESERVED3 as IB_WR_OPFN. |
|---|
| 35 | * |
|---|
| 36 | * The OPFN communication will be triggered when an RC QP |
|---|
| 37 | * receives a request with Bit 24 of BTH1 set. The responder QP |
|---|
| 38 | * will then post send an OPFN request with its local |
|---|
| 39 | * parameters, which will be sent to the requester QP once all |
|---|
| 40 | * existing requests on the responder QP side have been sent. |
|---|
| 41 | * Once the requester QP receives the OPFN request, it will |
|---|
| 42 | * keep a copy of the responder QP's parameters, and return a |
|---|
| 43 | * response packet with its own local parameters. The responder |
|---|
| 44 | * QP receives the response packet and keeps a copy of the requester |
|---|
| 45 | * QP's parameters. After this exchange, each side has the parameters |
|---|
| 46 | * for both sides and therefore can select the right parameters |
|---|
| 47 | * for future transactions |
|---|
| 48 | */ |
|---|
| 49 | |
|---|
| 50 | #include <linux/workqueue.h> |
|---|
| 51 | #include <rdma/ib_verbs.h> |
|---|
| 52 | #include <rdma/rdmavt_qp.h> |
|---|
| 53 | |
|---|
| 54 | /* STL Verbs Extended */ |
|---|
| 55 | #define IB_BTHE_E_SHIFT 24 |
|---|
| 56 | #define HFI1_VERBS_E_ATOMIC_VADDR U64_MAX |
|---|
| 57 | |
|---|
| 58 | enum hfi1_opfn_codes { |
|---|
| 59 | STL_VERBS_EXTD_NONE = 0, |
|---|
| 60 | STL_VERBS_EXTD_TID_RDMA, |
|---|
| 61 | STL_VERBS_EXTD_MAX |
|---|
| 62 | }; |
|---|
| 63 | |
|---|
| 64 | struct hfi1_opfn_data { |
|---|
| 65 | u8 extended; |
|---|
| 66 | u16 requested; |
|---|
| 67 | u16 completed; |
|---|
| 68 | enum hfi1_opfn_codes curr; |
|---|
| 69 | /* serialize opfn function calls */ |
|---|
| 70 | spinlock_t lock; |
|---|
| 71 | struct work_struct opfn_work; |
|---|
| 72 | }; |
|---|
| 73 | |
|---|
| 74 | /* WR opcode for OPFN */ |
|---|
| 75 | #define IB_WR_OPFN IB_WR_RESERVED3 |
|---|
| 76 | |
|---|
| 77 | void opfn_send_conn_request(struct work_struct *work); |
|---|
| 78 | void opfn_conn_response(struct rvt_qp *qp, struct rvt_ack_entry *e, |
|---|
| 79 | struct ib_atomic_eth *ateth); |
|---|
| 80 | void opfn_conn_reply(struct rvt_qp *qp, u64 data); |
|---|
| 81 | void opfn_conn_error(struct rvt_qp *qp); |
|---|
| 82 | void opfn_qp_init(struct rvt_qp *qp, struct ib_qp_attr *attr, int attr_mask); |
|---|
| 83 | void opfn_trigger_conn_request(struct rvt_qp *qp, u32 bth1); |
|---|
| 84 | int opfn_init(void); |
|---|
| 85 | void opfn_exit(void); |
|---|
| 86 | |
|---|
| 87 | #endif /* _HFI1_OPFN_H */ |
|---|
| 88 | |
|---|
