1 | // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 |
2 | /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */ |
3 | |
4 | #include <linux/kernel.h> |
5 | #include <linux/err.h> |
6 | #include <linux/errno.h> |
7 | #include <linux/gfp.h> |
8 | #include <linux/refcount.h> |
9 | #include <linux/rhashtable.h> |
10 | #define CREATE_TRACE_POINTS |
11 | #include <trace/events/mlxsw.h> |
12 | |
13 | #include "reg.h" |
14 | #include "core.h" |
15 | #include "spectrum.h" |
16 | #include "spectrum_acl_tcam.h" |
17 | #include "core_acl_flex_keys.h" |
18 | |
19 | #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START 0 |
20 | #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END 5 |
21 | |
22 | struct mlxsw_sp_acl_atcam_lkey_id_ht_key { |
23 | char enc_key[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN]; /* MSB blocks */ |
24 | u8 erp_id; |
25 | }; |
26 | |
27 | struct mlxsw_sp_acl_atcam_lkey_id { |
28 | struct rhash_head ht_node; |
29 | struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key; |
30 | refcount_t refcnt; |
31 | u32 id; |
32 | }; |
33 | |
34 | struct mlxsw_sp_acl_atcam_region_ops { |
35 | int (*init)(struct mlxsw_sp_acl_atcam_region *aregion); |
36 | void (*fini)(struct mlxsw_sp_acl_atcam_region *aregion); |
37 | struct mlxsw_sp_acl_atcam_lkey_id * |
38 | (*lkey_id_get)(struct mlxsw_sp_acl_atcam_region *aregion, |
39 | char *enc_key, u8 erp_id); |
40 | void (*lkey_id_put)(struct mlxsw_sp_acl_atcam_region *aregion, |
41 | struct mlxsw_sp_acl_atcam_lkey_id *lkey_id); |
42 | }; |
43 | |
44 | struct mlxsw_sp_acl_atcam_region_generic { |
45 | struct mlxsw_sp_acl_atcam_lkey_id dummy_lkey_id; |
46 | }; |
47 | |
48 | struct mlxsw_sp_acl_atcam_region_12kb { |
49 | struct rhashtable lkey_ht; |
50 | unsigned int max_lkey_id; |
51 | unsigned long *used_lkey_id; |
52 | }; |
53 | |
54 | static const struct rhashtable_params mlxsw_sp_acl_atcam_lkey_id_ht_params = { |
55 | .key_len = sizeof(struct mlxsw_sp_acl_atcam_lkey_id_ht_key), |
56 | .key_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_key), |
57 | .head_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_node), |
58 | }; |
59 | |
60 | static const struct rhashtable_params mlxsw_sp_acl_atcam_entries_ht_params = { |
61 | .key_len = sizeof(struct mlxsw_sp_acl_atcam_entry_ht_key), |
62 | .key_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_key), |
63 | .head_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_node), |
64 | }; |
65 | |
66 | static bool |
67 | mlxsw_sp_acl_atcam_is_centry(const struct mlxsw_sp_acl_atcam_entry *aentry) |
68 | { |
69 | return mlxsw_sp_acl_erp_mask_is_ctcam(erp_mask: aentry->erp_mask); |
70 | } |
71 | |
72 | static int |
73 | mlxsw_sp_acl_atcam_region_generic_init(struct mlxsw_sp_acl_atcam_region *aregion) |
74 | { |
75 | struct mlxsw_sp_acl_atcam_region_generic *region_generic; |
76 | |
77 | region_generic = kzalloc(size: sizeof(*region_generic), GFP_KERNEL); |
78 | if (!region_generic) |
79 | return -ENOMEM; |
80 | |
81 | refcount_set(r: ®ion_generic->dummy_lkey_id.refcnt, n: 1); |
82 | aregion->priv = region_generic; |
83 | |
84 | return 0; |
85 | } |
86 | |
87 | static void |
88 | mlxsw_sp_acl_atcam_region_generic_fini(struct mlxsw_sp_acl_atcam_region *aregion) |
89 | { |
90 | kfree(objp: aregion->priv); |
91 | } |
92 | |
93 | static struct mlxsw_sp_acl_atcam_lkey_id * |
94 | mlxsw_sp_acl_atcam_generic_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion, |
95 | char *enc_key, u8 erp_id) |
96 | { |
97 | struct mlxsw_sp_acl_atcam_region_generic *region_generic; |
98 | |
99 | region_generic = aregion->priv; |
100 | return ®ion_generic->dummy_lkey_id; |
101 | } |
102 | |
103 | static void |
104 | mlxsw_sp_acl_atcam_generic_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion, |
105 | struct mlxsw_sp_acl_atcam_lkey_id *lkey_id) |
106 | { |
107 | } |
108 | |
109 | static const struct mlxsw_sp_acl_atcam_region_ops |
110 | mlxsw_sp_acl_atcam_region_generic_ops = { |
111 | .init = mlxsw_sp_acl_atcam_region_generic_init, |
112 | .fini = mlxsw_sp_acl_atcam_region_generic_fini, |
113 | .lkey_id_get = mlxsw_sp_acl_atcam_generic_lkey_id_get, |
114 | .lkey_id_put = mlxsw_sp_acl_atcam_generic_lkey_id_put, |
115 | }; |
116 | |
117 | static int |
118 | mlxsw_sp_acl_atcam_region_12kb_init(struct mlxsw_sp_acl_atcam_region *aregion) |
119 | { |
120 | struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; |
121 | struct mlxsw_sp_acl_atcam_region_12kb *region_12kb; |
122 | u64 max_lkey_id; |
123 | int err; |
124 | |
125 | if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID)) |
126 | return -EIO; |
127 | |
128 | max_lkey_id = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID); |
129 | region_12kb = kzalloc(size: sizeof(*region_12kb), GFP_KERNEL); |
130 | if (!region_12kb) |
131 | return -ENOMEM; |
132 | |
133 | region_12kb->used_lkey_id = bitmap_zalloc(nbits: max_lkey_id, GFP_KERNEL); |
134 | if (!region_12kb->used_lkey_id) { |
135 | err = -ENOMEM; |
136 | goto err_used_lkey_id_alloc; |
137 | } |
138 | |
139 | err = rhashtable_init(ht: ®ion_12kb->lkey_ht, |
140 | params: &mlxsw_sp_acl_atcam_lkey_id_ht_params); |
141 | if (err) |
142 | goto err_rhashtable_init; |
143 | |
144 | region_12kb->max_lkey_id = max_lkey_id; |
145 | aregion->priv = region_12kb; |
146 | |
147 | return 0; |
148 | |
149 | err_rhashtable_init: |
150 | bitmap_free(bitmap: region_12kb->used_lkey_id); |
151 | err_used_lkey_id_alloc: |
152 | kfree(objp: region_12kb); |
153 | return err; |
154 | } |
155 | |
156 | static void |
157 | mlxsw_sp_acl_atcam_region_12kb_fini(struct mlxsw_sp_acl_atcam_region *aregion) |
158 | { |
159 | struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv; |
160 | |
161 | rhashtable_destroy(ht: ®ion_12kb->lkey_ht); |
162 | bitmap_free(bitmap: region_12kb->used_lkey_id); |
163 | kfree(objp: region_12kb); |
164 | } |
165 | |
166 | static struct mlxsw_sp_acl_atcam_lkey_id * |
167 | mlxsw_sp_acl_atcam_lkey_id_create(struct mlxsw_sp_acl_atcam_region *aregion, |
168 | struct mlxsw_sp_acl_atcam_lkey_id_ht_key *ht_key) |
169 | { |
170 | struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv; |
171 | struct mlxsw_sp_acl_atcam_lkey_id *lkey_id; |
172 | u32 id; |
173 | int err; |
174 | |
175 | id = find_first_zero_bit(addr: region_12kb->used_lkey_id, |
176 | size: region_12kb->max_lkey_id); |
177 | if (id < region_12kb->max_lkey_id) |
178 | __set_bit(id, region_12kb->used_lkey_id); |
179 | else |
180 | return ERR_PTR(error: -ENOBUFS); |
181 | |
182 | lkey_id = kzalloc(size: sizeof(*lkey_id), GFP_KERNEL); |
183 | if (!lkey_id) { |
184 | err = -ENOMEM; |
185 | goto err_lkey_id_alloc; |
186 | } |
187 | |
188 | lkey_id->id = id; |
189 | memcpy(&lkey_id->ht_key, ht_key, sizeof(*ht_key)); |
190 | refcount_set(r: &lkey_id->refcnt, n: 1); |
191 | |
192 | err = rhashtable_insert_fast(ht: ®ion_12kb->lkey_ht, |
193 | obj: &lkey_id->ht_node, |
194 | params: mlxsw_sp_acl_atcam_lkey_id_ht_params); |
195 | if (err) |
196 | goto err_rhashtable_insert; |
197 | |
198 | return lkey_id; |
199 | |
200 | err_rhashtable_insert: |
201 | kfree(objp: lkey_id); |
202 | err_lkey_id_alloc: |
203 | __clear_bit(id, region_12kb->used_lkey_id); |
204 | return ERR_PTR(error: err); |
205 | } |
206 | |
207 | static void |
208 | mlxsw_sp_acl_atcam_lkey_id_destroy(struct mlxsw_sp_acl_atcam_region *aregion, |
209 | struct mlxsw_sp_acl_atcam_lkey_id *lkey_id) |
210 | { |
211 | struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv; |
212 | u32 id = lkey_id->id; |
213 | |
214 | rhashtable_remove_fast(ht: ®ion_12kb->lkey_ht, obj: &lkey_id->ht_node, |
215 | params: mlxsw_sp_acl_atcam_lkey_id_ht_params); |
216 | kfree(objp: lkey_id); |
217 | __clear_bit(id, region_12kb->used_lkey_id); |
218 | } |
219 | |
220 | static struct mlxsw_sp_acl_atcam_lkey_id * |
221 | mlxsw_sp_acl_atcam_12kb_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion, |
222 | char *enc_key, u8 erp_id) |
223 | { |
224 | struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv; |
225 | struct mlxsw_sp_acl_tcam_region *region = aregion->region; |
226 | struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key = {{ 0 } }; |
227 | struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp; |
228 | struct mlxsw_afk *afk = mlxsw_sp_acl_afk(acl: mlxsw_sp->acl); |
229 | struct mlxsw_sp_acl_atcam_lkey_id *lkey_id; |
230 | |
231 | memcpy(ht_key.enc_key, enc_key, sizeof(ht_key.enc_key)); |
232 | mlxsw_afk_clear(mlxsw_afk: afk, key: ht_key.enc_key, |
233 | MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START, |
234 | MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END); |
235 | ht_key.erp_id = erp_id; |
236 | lkey_id = rhashtable_lookup_fast(ht: ®ion_12kb->lkey_ht, key: &ht_key, |
237 | params: mlxsw_sp_acl_atcam_lkey_id_ht_params); |
238 | if (lkey_id) { |
239 | refcount_inc(r: &lkey_id->refcnt); |
240 | return lkey_id; |
241 | } |
242 | |
243 | return mlxsw_sp_acl_atcam_lkey_id_create(aregion, ht_key: &ht_key); |
244 | } |
245 | |
246 | static void |
247 | mlxsw_sp_acl_atcam_12kb_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion, |
248 | struct mlxsw_sp_acl_atcam_lkey_id *lkey_id) |
249 | { |
250 | if (refcount_dec_and_test(r: &lkey_id->refcnt)) |
251 | mlxsw_sp_acl_atcam_lkey_id_destroy(aregion, lkey_id); |
252 | } |
253 | |
254 | static const struct mlxsw_sp_acl_atcam_region_ops |
255 | mlxsw_sp_acl_atcam_region_12kb_ops = { |
256 | .init = mlxsw_sp_acl_atcam_region_12kb_init, |
257 | .fini = mlxsw_sp_acl_atcam_region_12kb_fini, |
258 | .lkey_id_get = mlxsw_sp_acl_atcam_12kb_lkey_id_get, |
259 | .lkey_id_put = mlxsw_sp_acl_atcam_12kb_lkey_id_put, |
260 | }; |
261 | |
262 | static const struct mlxsw_sp_acl_atcam_region_ops * |
263 | mlxsw_sp_acl_atcam_region_ops_arr[] = { |
264 | [MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] = |
265 | &mlxsw_sp_acl_atcam_region_generic_ops, |
266 | [MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] = |
267 | &mlxsw_sp_acl_atcam_region_generic_ops, |
268 | [MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] = |
269 | &mlxsw_sp_acl_atcam_region_generic_ops, |
270 | [MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] = |
271 | &mlxsw_sp_acl_atcam_region_12kb_ops, |
272 | }; |
273 | |
274 | int mlxsw_sp_acl_atcam_region_associate(struct mlxsw_sp *mlxsw_sp, |
275 | u16 region_id) |
276 | { |
277 | char perar_pl[MLXSW_REG_PERAR_LEN]; |
278 | /* For now, just assume that every region has 12 key blocks */ |
279 | u16 hw_region = region_id * 3; |
280 | u64 max_regions; |
281 | |
282 | max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS); |
283 | if (hw_region >= max_regions) |
284 | return -ENOBUFS; |
285 | |
286 | mlxsw_reg_perar_pack(payload: perar_pl, region_id, hw_region); |
287 | return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(perar), payload: perar_pl); |
288 | } |
289 | |
290 | static void |
291 | mlxsw_sp_acl_atcam_region_type_init(struct mlxsw_sp_acl_atcam_region *aregion) |
292 | { |
293 | struct mlxsw_sp_acl_tcam_region *region = aregion->region; |
294 | enum mlxsw_sp_acl_atcam_region_type region_type; |
295 | unsigned int blocks_count; |
296 | |
297 | /* We already know the blocks count can not exceed the maximum |
298 | * blocks count. |
299 | */ |
300 | blocks_count = mlxsw_afk_key_info_blocks_count_get(key_info: region->key_info); |
301 | if (blocks_count <= 2) |
302 | region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB; |
303 | else if (blocks_count <= 4) |
304 | region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB; |
305 | else if (blocks_count <= 8) |
306 | region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB; |
307 | else |
308 | region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB; |
309 | |
310 | aregion->type = region_type; |
311 | aregion->ops = mlxsw_sp_acl_atcam_region_ops_arr[region_type]; |
312 | } |
313 | |
314 | int |
315 | mlxsw_sp_acl_atcam_region_init(struct mlxsw_sp *mlxsw_sp, |
316 | struct mlxsw_sp_acl_atcam *atcam, |
317 | struct mlxsw_sp_acl_atcam_region *aregion, |
318 | struct mlxsw_sp_acl_tcam_region *region, |
319 | void *hints_priv, |
320 | const struct mlxsw_sp_acl_ctcam_region_ops *ops) |
321 | { |
322 | int err; |
323 | |
324 | aregion->region = region; |
325 | aregion->atcam = atcam; |
326 | mlxsw_sp_acl_atcam_region_type_init(aregion); |
327 | INIT_LIST_HEAD(list: &aregion->entries_list); |
328 | |
329 | err = rhashtable_init(ht: &aregion->entries_ht, |
330 | params: &mlxsw_sp_acl_atcam_entries_ht_params); |
331 | if (err) |
332 | return err; |
333 | err = aregion->ops->init(aregion); |
334 | if (err) |
335 | goto err_ops_init; |
336 | err = mlxsw_sp_acl_erp_region_init(aregion, hints_priv); |
337 | if (err) |
338 | goto err_erp_region_init; |
339 | err = mlxsw_sp_acl_ctcam_region_init(mlxsw_sp, cregion: &aregion->cregion, |
340 | region, ops); |
341 | if (err) |
342 | goto err_ctcam_region_init; |
343 | |
344 | return 0; |
345 | |
346 | err_ctcam_region_init: |
347 | mlxsw_sp_acl_erp_region_fini(aregion); |
348 | err_erp_region_init: |
349 | aregion->ops->fini(aregion); |
350 | err_ops_init: |
351 | rhashtable_destroy(ht: &aregion->entries_ht); |
352 | return err; |
353 | } |
354 | |
355 | void mlxsw_sp_acl_atcam_region_fini(struct mlxsw_sp_acl_atcam_region *aregion) |
356 | { |
357 | mlxsw_sp_acl_ctcam_region_fini(cregion: &aregion->cregion); |
358 | mlxsw_sp_acl_erp_region_fini(aregion); |
359 | aregion->ops->fini(aregion); |
360 | rhashtable_destroy(ht: &aregion->entries_ht); |
361 | WARN_ON(!list_empty(&aregion->entries_list)); |
362 | } |
363 | |
364 | void mlxsw_sp_acl_atcam_chunk_init(struct mlxsw_sp_acl_atcam_region *aregion, |
365 | struct mlxsw_sp_acl_atcam_chunk *achunk, |
366 | unsigned int priority) |
367 | { |
368 | mlxsw_sp_acl_ctcam_chunk_init(cregion: &aregion->cregion, cchunk: &achunk->cchunk, |
369 | priority); |
370 | } |
371 | |
372 | void mlxsw_sp_acl_atcam_chunk_fini(struct mlxsw_sp_acl_atcam_chunk *achunk) |
373 | { |
374 | mlxsw_sp_acl_ctcam_chunk_fini(cchunk: &achunk->cchunk); |
375 | } |
376 | |
377 | static int |
378 | mlxsw_sp_acl_atcam_region_entry_insert(struct mlxsw_sp *mlxsw_sp, |
379 | struct mlxsw_sp_acl_atcam_region *aregion, |
380 | struct mlxsw_sp_acl_atcam_entry *aentry, |
381 | struct mlxsw_sp_acl_rule_info *rulei) |
382 | { |
383 | struct mlxsw_sp_acl_tcam_region *region = aregion->region; |
384 | u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(erp_mask: aentry->erp_mask); |
385 | struct mlxsw_sp_acl_atcam_lkey_id *lkey_id; |
386 | char ptce3_pl[MLXSW_REG_PTCE3_LEN]; |
387 | u32 kvdl_index, priority; |
388 | int err; |
389 | |
390 | err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, priority: &priority, fillup_priority: true); |
391 | if (err) |
392 | return err; |
393 | |
394 | lkey_id = aregion->ops->lkey_id_get(aregion, aentry->enc_key, erp_id); |
395 | if (IS_ERR(ptr: lkey_id)) |
396 | return PTR_ERR(ptr: lkey_id); |
397 | aentry->lkey_id = lkey_id; |
398 | |
399 | kvdl_index = mlxsw_afa_block_first_kvdl_index(block: rulei->act_block); |
400 | mlxsw_reg_ptce3_pack(payload: ptce3_pl, valid: true, op: MLXSW_REG_PTCE3_OP_WRITE_WRITE, |
401 | priority, tcam_region_info: region->tcam_region_info, |
402 | key: aentry->enc_key, erp_id, |
403 | delta_start: aentry->delta_info.start, |
404 | delta_mask: aentry->delta_info.mask, |
405 | delta_value: aentry->delta_info.value, |
406 | large_exists: refcount_read(r: &lkey_id->refcnt) != 1, lkey_id: lkey_id->id, |
407 | action_pointer: kvdl_index); |
408 | err = mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ptce3), payload: ptce3_pl); |
409 | if (err) |
410 | goto err_ptce3_write; |
411 | |
412 | return 0; |
413 | |
414 | err_ptce3_write: |
415 | aregion->ops->lkey_id_put(aregion, lkey_id); |
416 | return err; |
417 | } |
418 | |
419 | static void |
420 | mlxsw_sp_acl_atcam_region_entry_remove(struct mlxsw_sp *mlxsw_sp, |
421 | struct mlxsw_sp_acl_atcam_region *aregion, |
422 | struct mlxsw_sp_acl_atcam_entry *aentry) |
423 | { |
424 | struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id; |
425 | struct mlxsw_sp_acl_tcam_region *region = aregion->region; |
426 | u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(erp_mask: aentry->erp_mask); |
427 | char ptce3_pl[MLXSW_REG_PTCE3_LEN]; |
428 | |
429 | mlxsw_reg_ptce3_pack(payload: ptce3_pl, valid: false, op: MLXSW_REG_PTCE3_OP_WRITE_WRITE, priority: 0, |
430 | tcam_region_info: region->tcam_region_info, |
431 | key: aentry->enc_key, erp_id, |
432 | delta_start: aentry->delta_info.start, |
433 | delta_mask: aentry->delta_info.mask, |
434 | delta_value: aentry->delta_info.value, |
435 | large_exists: refcount_read(r: &lkey_id->refcnt) != 1, |
436 | lkey_id: lkey_id->id, action_pointer: 0); |
437 | mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ptce3), payload: ptce3_pl); |
438 | aregion->ops->lkey_id_put(aregion, lkey_id); |
439 | } |
440 | |
441 | static int |
442 | mlxsw_sp_acl_atcam_region_entry_action_replace(struct mlxsw_sp *mlxsw_sp, |
443 | struct mlxsw_sp_acl_atcam_region *aregion, |
444 | struct mlxsw_sp_acl_atcam_entry *aentry, |
445 | struct mlxsw_sp_acl_rule_info *rulei) |
446 | { |
447 | struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id; |
448 | u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(erp_mask: aentry->erp_mask); |
449 | struct mlxsw_sp_acl_tcam_region *region = aregion->region; |
450 | char ptce3_pl[MLXSW_REG_PTCE3_LEN]; |
451 | u32 kvdl_index, priority; |
452 | int err; |
453 | |
454 | err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, priority: &priority, fillup_priority: true); |
455 | if (err) |
456 | return err; |
457 | kvdl_index = mlxsw_afa_block_first_kvdl_index(block: rulei->act_block); |
458 | mlxsw_reg_ptce3_pack(payload: ptce3_pl, valid: true, op: MLXSW_REG_PTCE3_OP_WRITE_UPDATE, |
459 | priority, tcam_region_info: region->tcam_region_info, |
460 | key: aentry->enc_key, erp_id, |
461 | delta_start: aentry->delta_info.start, |
462 | delta_mask: aentry->delta_info.mask, |
463 | delta_value: aentry->delta_info.value, |
464 | large_exists: refcount_read(r: &lkey_id->refcnt) != 1, lkey_id: lkey_id->id, |
465 | action_pointer: kvdl_index); |
466 | return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ptce3), payload: ptce3_pl); |
467 | } |
468 | |
469 | static int |
470 | __mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp, |
471 | struct mlxsw_sp_acl_atcam_region *aregion, |
472 | struct mlxsw_sp_acl_atcam_entry *aentry, |
473 | struct mlxsw_sp_acl_rule_info *rulei) |
474 | { |
475 | struct mlxsw_sp_acl_tcam_region *region = aregion->region; |
476 | char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 }; |
477 | struct mlxsw_afk *afk = mlxsw_sp_acl_afk(acl: mlxsw_sp->acl); |
478 | const struct mlxsw_sp_acl_erp_delta *delta; |
479 | struct mlxsw_sp_acl_erp_mask *erp_mask; |
480 | int err; |
481 | |
482 | mlxsw_afk_encode(mlxsw_afk: afk, key_info: region->key_info, values: &rulei->values, |
483 | key: aentry->ht_key.full_enc_key, mask); |
484 | |
485 | erp_mask = mlxsw_sp_acl_erp_mask_get(aregion, mask, ctcam: false); |
486 | if (IS_ERR(ptr: erp_mask)) |
487 | return PTR_ERR(ptr: erp_mask); |
488 | aentry->erp_mask = erp_mask; |
489 | aentry->ht_key.erp_id = mlxsw_sp_acl_erp_mask_erp_id(erp_mask); |
490 | memcpy(aentry->enc_key, aentry->ht_key.full_enc_key, |
491 | sizeof(aentry->enc_key)); |
492 | |
493 | /* Compute all needed delta information and clear the delta bits |
494 | * from the encrypted key. |
495 | */ |
496 | delta = mlxsw_sp_acl_erp_delta(erp_mask: aentry->erp_mask); |
497 | aentry->delta_info.start = mlxsw_sp_acl_erp_delta_start(delta); |
498 | aentry->delta_info.mask = mlxsw_sp_acl_erp_delta_mask(delta); |
499 | aentry->delta_info.value = |
500 | mlxsw_sp_acl_erp_delta_value(delta, |
501 | enc_key: aentry->ht_key.full_enc_key); |
502 | mlxsw_sp_acl_erp_delta_clear(delta, enc_key: aentry->enc_key); |
503 | |
504 | /* Add rule to the list of A-TCAM rules, assuming this |
505 | * rule is intended to A-TCAM. In case this rule does |
506 | * not fit into A-TCAM it will be removed from the list. |
507 | */ |
508 | list_add(new: &aentry->list, head: &aregion->entries_list); |
509 | |
510 | /* We can't insert identical rules into the A-TCAM, so fail and |
511 | * let the rule spill into C-TCAM |
512 | */ |
513 | err = rhashtable_lookup_insert_fast(ht: &aregion->entries_ht, |
514 | obj: &aentry->ht_node, |
515 | params: mlxsw_sp_acl_atcam_entries_ht_params); |
516 | if (err) |
517 | goto err_rhashtable_insert; |
518 | |
519 | /* Bloom filter must be updated here, before inserting the rule into |
520 | * the A-TCAM. |
521 | */ |
522 | err = mlxsw_sp_acl_erp_bf_insert(mlxsw_sp, aregion, erp_mask, aentry); |
523 | if (err) |
524 | goto err_bf_insert; |
525 | |
526 | err = mlxsw_sp_acl_atcam_region_entry_insert(mlxsw_sp, aregion, aentry, |
527 | rulei); |
528 | if (err) |
529 | goto err_rule_insert; |
530 | |
531 | return 0; |
532 | |
533 | err_rule_insert: |
534 | mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, erp_mask, aentry); |
535 | err_bf_insert: |
536 | rhashtable_remove_fast(ht: &aregion->entries_ht, obj: &aentry->ht_node, |
537 | params: mlxsw_sp_acl_atcam_entries_ht_params); |
538 | err_rhashtable_insert: |
539 | list_del(entry: &aentry->list); |
540 | mlxsw_sp_acl_erp_mask_put(aregion, erp_mask); |
541 | return err; |
542 | } |
543 | |
544 | static void |
545 | __mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp, |
546 | struct mlxsw_sp_acl_atcam_region *aregion, |
547 | struct mlxsw_sp_acl_atcam_entry *aentry) |
548 | { |
549 | mlxsw_sp_acl_atcam_region_entry_remove(mlxsw_sp, aregion, aentry); |
550 | mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, erp_mask: aentry->erp_mask, aentry); |
551 | rhashtable_remove_fast(ht: &aregion->entries_ht, obj: &aentry->ht_node, |
552 | params: mlxsw_sp_acl_atcam_entries_ht_params); |
553 | list_del(entry: &aentry->list); |
554 | mlxsw_sp_acl_erp_mask_put(aregion, erp_mask: aentry->erp_mask); |
555 | } |
556 | |
557 | static int |
558 | __mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp, |
559 | struct mlxsw_sp_acl_atcam_region *aregion, |
560 | struct mlxsw_sp_acl_atcam_entry *aentry, |
561 | struct mlxsw_sp_acl_rule_info *rulei) |
562 | { |
563 | return mlxsw_sp_acl_atcam_region_entry_action_replace(mlxsw_sp, aregion, |
564 | aentry, rulei); |
565 | } |
566 | |
567 | int mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp, |
568 | struct mlxsw_sp_acl_atcam_region *aregion, |
569 | struct mlxsw_sp_acl_atcam_chunk *achunk, |
570 | struct mlxsw_sp_acl_atcam_entry *aentry, |
571 | struct mlxsw_sp_acl_rule_info *rulei) |
572 | { |
573 | int err; |
574 | |
575 | err = __mlxsw_sp_acl_atcam_entry_add(mlxsw_sp, aregion, aentry, rulei); |
576 | if (!err) |
577 | return 0; |
578 | |
579 | /* It is possible we failed to add the rule to the A-TCAM due to |
580 | * exceeded number of masks. Try to spill into C-TCAM. |
581 | */ |
582 | trace_mlxsw_sp_acl_atcam_entry_add_ctcam_spill(mlxsw_sp, aregion); |
583 | err = mlxsw_sp_acl_ctcam_entry_add(mlxsw_sp, cregion: &aregion->cregion, |
584 | cchunk: &achunk->cchunk, centry: &aentry->centry, |
585 | rulei, fillup_priority: true); |
586 | if (!err) |
587 | return 0; |
588 | |
589 | return err; |
590 | } |
591 | |
592 | void mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp, |
593 | struct mlxsw_sp_acl_atcam_region *aregion, |
594 | struct mlxsw_sp_acl_atcam_chunk *achunk, |
595 | struct mlxsw_sp_acl_atcam_entry *aentry) |
596 | { |
597 | if (mlxsw_sp_acl_atcam_is_centry(aentry)) |
598 | mlxsw_sp_acl_ctcam_entry_del(mlxsw_sp, cregion: &aregion->cregion, |
599 | cchunk: &achunk->cchunk, centry: &aentry->centry); |
600 | else |
601 | __mlxsw_sp_acl_atcam_entry_del(mlxsw_sp, aregion, aentry); |
602 | } |
603 | |
604 | int |
605 | mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp, |
606 | struct mlxsw_sp_acl_atcam_region *aregion, |
607 | struct mlxsw_sp_acl_atcam_entry *aentry, |
608 | struct mlxsw_sp_acl_rule_info *rulei) |
609 | { |
610 | int err; |
611 | |
612 | if (mlxsw_sp_acl_atcam_is_centry(aentry)) |
613 | err = mlxsw_sp_acl_ctcam_entry_action_replace(mlxsw_sp, |
614 | cregion: &aregion->cregion, |
615 | centry: &aentry->centry, |
616 | rulei); |
617 | else |
618 | err = __mlxsw_sp_acl_atcam_entry_action_replace(mlxsw_sp, |
619 | aregion, aentry, |
620 | rulei); |
621 | |
622 | return err; |
623 | } |
624 | |
625 | int mlxsw_sp_acl_atcam_init(struct mlxsw_sp *mlxsw_sp, |
626 | struct mlxsw_sp_acl_atcam *atcam) |
627 | { |
628 | return mlxsw_sp_acl_erps_init(mlxsw_sp, atcam); |
629 | } |
630 | |
631 | void mlxsw_sp_acl_atcam_fini(struct mlxsw_sp *mlxsw_sp, |
632 | struct mlxsw_sp_acl_atcam *atcam) |
633 | { |
634 | mlxsw_sp_acl_erps_fini(mlxsw_sp, atcam); |
635 | } |
636 | |
637 | void * |
638 | mlxsw_sp_acl_atcam_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion) |
639 | { |
640 | return mlxsw_sp_acl_erp_rehash_hints_get(aregion); |
641 | } |
642 | |
643 | void mlxsw_sp_acl_atcam_rehash_hints_put(void *hints_priv) |
644 | { |
645 | mlxsw_sp_acl_erp_rehash_hints_put(hints_priv); |
646 | } |
647 | |