1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */ |
3 | |
4 | #include "tir.h" |
5 | #include "params.h" |
6 | #include <linux/mlx5/transobj.h> |
7 | |
8 | #define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ (64 * 1024) |
9 | |
10 | /* max() doesn't work inside square brackets. */ |
11 | #define MLX5E_TIR_CMD_IN_SZ_DW ( \ |
12 | MLX5_ST_SZ_DW(create_tir_in) > MLX5_ST_SZ_DW(modify_tir_in) ? \ |
13 | MLX5_ST_SZ_DW(create_tir_in) : MLX5_ST_SZ_DW(modify_tir_in) \ |
14 | ) |
15 | |
16 | struct mlx5e_tir_builder { |
17 | u32 in[MLX5E_TIR_CMD_IN_SZ_DW]; |
18 | bool modify; |
19 | }; |
20 | |
21 | struct mlx5e_tir_builder *mlx5e_tir_builder_alloc(bool modify) |
22 | { |
23 | struct mlx5e_tir_builder *builder; |
24 | |
25 | builder = kvzalloc(size: sizeof(*builder), GFP_KERNEL); |
26 | builder->modify = modify; |
27 | |
28 | return builder; |
29 | } |
30 | |
31 | void mlx5e_tir_builder_free(struct mlx5e_tir_builder *builder) |
32 | { |
33 | kvfree(addr: builder); |
34 | } |
35 | |
36 | void mlx5e_tir_builder_clear(struct mlx5e_tir_builder *builder) |
37 | { |
38 | memset(builder->in, 0, sizeof(builder->in)); |
39 | } |
40 | |
41 | static void *mlx5e_tir_builder_get_tirc(struct mlx5e_tir_builder *builder) |
42 | { |
43 | if (builder->modify) |
44 | return MLX5_ADDR_OF(modify_tir_in, builder->in, ctx); |
45 | return MLX5_ADDR_OF(create_tir_in, builder->in, ctx); |
46 | } |
47 | |
48 | void mlx5e_tir_builder_build_inline(struct mlx5e_tir_builder *builder, u32 tdn, u32 rqn) |
49 | { |
50 | void *tirc = mlx5e_tir_builder_get_tirc(builder); |
51 | |
52 | WARN_ON(builder->modify); |
53 | |
54 | MLX5_SET(tirc, tirc, transport_domain, tdn); |
55 | MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT); |
56 | MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_NONE); |
57 | MLX5_SET(tirc, tirc, inline_rqn, rqn); |
58 | } |
59 | |
60 | void mlx5e_tir_builder_build_rqt(struct mlx5e_tir_builder *builder, u32 tdn, |
61 | u32 rqtn, bool inner_ft_support) |
62 | { |
63 | void *tirc = mlx5e_tir_builder_get_tirc(builder); |
64 | |
65 | WARN_ON(builder->modify); |
66 | |
67 | MLX5_SET(tirc, tirc, transport_domain, tdn); |
68 | MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); |
69 | MLX5_SET(tirc, tirc, indirect_table, rqtn); |
70 | MLX5_SET(tirc, tirc, tunneled_offload_en, inner_ft_support); |
71 | } |
72 | |
73 | void mlx5e_tir_builder_build_packet_merge(struct mlx5e_tir_builder *builder, |
74 | const struct mlx5e_packet_merge_param *pkt_merge_param) |
75 | { |
76 | void *tirc = mlx5e_tir_builder_get_tirc(builder); |
77 | const unsigned int rough_max_l2_l3_hdr_sz = 256; |
78 | |
79 | if (builder->modify) |
80 | MLX5_SET(modify_tir_in, builder->in, bitmask.packet_merge, 1); |
81 | |
82 | switch (pkt_merge_param->type) { |
83 | case MLX5E_PACKET_MERGE_LRO: |
84 | MLX5_SET(tirc, tirc, packet_merge_mask, |
85 | MLX5_TIRC_PACKET_MERGE_MASK_IPV4_LRO | |
86 | MLX5_TIRC_PACKET_MERGE_MASK_IPV6_LRO); |
87 | MLX5_SET(tirc, tirc, lro_max_ip_payload_size, |
88 | (MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ - rough_max_l2_l3_hdr_sz) >> 8); |
89 | MLX5_SET(tirc, tirc, lro_timeout_period_usecs, pkt_merge_param->timeout); |
90 | break; |
91 | default: |
92 | break; |
93 | } |
94 | } |
95 | |
96 | static int mlx5e_hfunc_to_hw(u8 hfunc) |
97 | { |
98 | switch (hfunc) { |
99 | case ETH_RSS_HASH_TOP: |
100 | return MLX5_RX_HASH_FN_TOEPLITZ; |
101 | case ETH_RSS_HASH_XOR: |
102 | return MLX5_RX_HASH_FN_INVERTED_XOR8; |
103 | default: |
104 | return MLX5_RX_HASH_FN_NONE; |
105 | } |
106 | } |
107 | |
108 | void (struct mlx5e_tir_builder *builder, |
109 | const struct mlx5e_rss_params_hash *, |
110 | const struct mlx5e_rss_params_traffic_type *, |
111 | bool inner) |
112 | { |
113 | void *tirc = mlx5e_tir_builder_get_tirc(builder); |
114 | void *hfso; |
115 | |
116 | if (builder->modify) |
117 | MLX5_SET(modify_tir_in, builder->in, bitmask.hash, 1); |
118 | |
119 | MLX5_SET(tirc, tirc, rx_hash_fn, mlx5e_hfunc_to_hw(rss_hash->hfunc)); |
120 | if (rss_hash->hfunc == ETH_RSS_HASH_TOP) { |
121 | const size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key); |
122 | void * = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); |
123 | |
124 | MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); |
125 | memcpy(rss_key, rss_hash->toeplitz_hash_key, len); |
126 | } |
127 | |
128 | if (inner) |
129 | hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner); |
130 | else |
131 | hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); |
132 | MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, rss_tt->l3_prot_type); |
133 | MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, rss_tt->l4_prot_type); |
134 | MLX5_SET(rx_hash_field_select, hfso, selected_fields, rss_tt->rx_hash_fields); |
135 | } |
136 | |
137 | void mlx5e_tir_builder_build_direct(struct mlx5e_tir_builder *builder) |
138 | { |
139 | void *tirc = mlx5e_tir_builder_get_tirc(builder); |
140 | |
141 | WARN_ON(builder->modify); |
142 | |
143 | MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8); |
144 | } |
145 | |
146 | void mlx5e_tir_builder_build_tls(struct mlx5e_tir_builder *builder) |
147 | { |
148 | void *tirc = mlx5e_tir_builder_get_tirc(builder); |
149 | |
150 | WARN_ON(builder->modify); |
151 | |
152 | MLX5_SET(tirc, tirc, tls_en, 1); |
153 | MLX5_SET(tirc, tirc, self_lb_block, |
154 | MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST | |
155 | MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST); |
156 | } |
157 | |
158 | int mlx5e_tir_init(struct mlx5e_tir *tir, struct mlx5e_tir_builder *builder, |
159 | struct mlx5_core_dev *mdev, bool reg) |
160 | { |
161 | int err; |
162 | |
163 | tir->mdev = mdev; |
164 | |
165 | err = mlx5_core_create_tir(dev: tir->mdev, in: builder->in, tirn: &tir->tirn); |
166 | if (err) |
167 | return err; |
168 | |
169 | if (reg) { |
170 | struct mlx5e_hw_objs *res = &tir->mdev->mlx5e_res.hw_objs; |
171 | |
172 | mutex_lock(&res->td.list_lock); |
173 | list_add(new: &tir->list, head: &res->td.tirs_list); |
174 | mutex_unlock(lock: &res->td.list_lock); |
175 | } else { |
176 | INIT_LIST_HEAD(list: &tir->list); |
177 | } |
178 | |
179 | return 0; |
180 | } |
181 | |
182 | void mlx5e_tir_destroy(struct mlx5e_tir *tir) |
183 | { |
184 | struct mlx5e_hw_objs *res = &tir->mdev->mlx5e_res.hw_objs; |
185 | |
186 | /* Skip mutex if list_del is no-op (the TIR wasn't registered in the |
187 | * list). list_empty will never return true for an item of tirs_list, |
188 | * and READ_ONCE/WRITE_ONCE in list_empty/list_del guarantee consistency |
189 | * of the list->next value. |
190 | */ |
191 | if (!list_empty(head: &tir->list)) { |
192 | mutex_lock(&res->td.list_lock); |
193 | list_del(entry: &tir->list); |
194 | mutex_unlock(lock: &res->td.list_lock); |
195 | } |
196 | |
197 | mlx5_core_destroy_tir(dev: tir->mdev, tirn: tir->tirn); |
198 | } |
199 | |
200 | int mlx5e_tir_modify(struct mlx5e_tir *tir, struct mlx5e_tir_builder *builder) |
201 | { |
202 | return mlx5_core_modify_tir(dev: tir->mdev, tirn: tir->tirn, in: builder->in); |
203 | } |
204 | |