1	// SPDX-License-Identifier: GPL-2.0+
2	/* Microchip VCAP API
3	*
4	* Copyright (c) 2022 Microchip Technology Inc. and its subsidiaries.
5	*/
6
7	#include <linux/types.h>
8
9	#include "vcap_api_private.h"
10
11	static int keyfield_size_table[] = {
12	[VCAP_FIELD_BIT] = sizeof(struct vcap_u1_key),
13	[VCAP_FIELD_U32] = sizeof(struct vcap_u32_key),
14	[VCAP_FIELD_U48] = sizeof(struct vcap_u48_key),
15	[VCAP_FIELD_U56] = sizeof(struct vcap_u56_key),
16	[VCAP_FIELD_U64] = sizeof(struct vcap_u64_key),
17	[VCAP_FIELD_U72] = sizeof(struct vcap_u72_key),
18	[VCAP_FIELD_U112] = sizeof(struct vcap_u112_key),
19	[VCAP_FIELD_U128] = sizeof(struct vcap_u128_key),
20	};
21
22	static int actionfield_size_table[] = {
23	[VCAP_FIELD_BIT] = sizeof(struct vcap_u1_action),
24	[VCAP_FIELD_U32] = sizeof(struct vcap_u32_action),
25	[VCAP_FIELD_U48] = sizeof(struct vcap_u48_action),
26	[VCAP_FIELD_U56] = sizeof(struct vcap_u56_action),
27	[VCAP_FIELD_U64] = sizeof(struct vcap_u64_action),
28	[VCAP_FIELD_U72] = sizeof(struct vcap_u72_action),
29	[VCAP_FIELD_U112] = sizeof(struct vcap_u112_action),
30	[VCAP_FIELD_U128] = sizeof(struct vcap_u128_action),
31	};
32
33	/* Moving a rule in the VCAP address space */
34	struct vcap_rule_move {
35	int addr; /* address to move */
36	int offset; /* change in address */
37	int count; /* blocksize of addresses to move */
38	};
39
40	/* Stores the filter cookie and chain id that enabled the port */
41	struct vcap_enabled_port {
42	struct list_head list; /* for insertion in enabled ports list */
43	struct net_device *ndev; /* the enabled port */
44	unsigned long cookie; /* filter that enabled the port */
45	int src_cid; /* source chain id */
46	int dst_cid; /* destination chain id */
47	};
48
49	void vcap_iter_set(struct vcap_stream_iter *itr, int sw_width,
50	const struct vcap_typegroup *tg, u32 offset)
51	{
52	memset(itr, 0, sizeof(*itr));
53	itr->offset = offset;
54	itr->sw_width = sw_width;
55	itr->regs_per_sw = DIV_ROUND_UP(sw_width, 32);
56	itr->tg = tg;
57	}
58
59	static void vcap_iter_skip_tg(struct vcap_stream_iter *itr)
60	{
61	/* Compensate the field offset for preceding typegroups.
62	* A typegroup table ends with an all-zero terminator.
63	*/
64	while (itr->tg->width && itr->offset >= itr->tg->offset) {
65	itr->offset += itr->tg->width;
66	itr->tg++; /* next typegroup */
67	}
68	}
69
70	void vcap_iter_update(struct vcap_stream_iter *itr)
71	{
72	int sw_idx, sw_bitpos;
73
74	/* Calculate the subword index and bitposition for current bit */
75	sw_idx = itr->offset / itr->sw_width;
76	sw_bitpos = itr->offset % itr->sw_width;
77	/* Calculate the register index and bitposition for current bit */
78	itr->reg_idx = (sw_idx * itr->regs_per_sw) + (sw_bitpos / 32);
79	itr->reg_bitpos = sw_bitpos % 32;
80	}
81
82	void vcap_iter_init(struct vcap_stream_iter *itr, int sw_width,
83	const struct vcap_typegroup *tg, u32 offset)
84	{
85	vcap_iter_set(itr, sw_width, tg, offset);
86	vcap_iter_skip_tg(itr);
87	vcap_iter_update(itr);
88	}
89
90	void vcap_iter_next(struct vcap_stream_iter *itr)
91	{
92	itr->offset++;
93	vcap_iter_skip_tg(itr);
94	vcap_iter_update(itr);
95	}
96
97	static void vcap_set_bit(u32 *stream, struct vcap_stream_iter *itr, bool value)
98	{
99	u32 mask = BIT(itr->reg_bitpos);
100	u32 *p = &stream[itr->reg_idx];
101
102	if (value)
103	*p |= mask;
104	else
105	*p &= ~mask;
106	}
107
108	static void vcap_encode_bit(u32 *stream, struct vcap_stream_iter *itr, bool val)
109	{
110	/* When intersected by a type group field, stream the type group bits
111	* before continuing with the value bit
112	*/
113	while (itr->tg->width &&
114	itr->offset >= itr->tg->offset &&
115	itr->offset < itr->tg->offset + itr->tg->width) {
116	int tg_bitpos = itr->tg->offset - itr->offset;
117
118	vcap_set_bit(stream, itr, value: (itr->tg->value >> tg_bitpos) & 0x1);
119	itr->offset++;
120	vcap_iter_update(itr);
121	}
122	vcap_set_bit(stream, itr, value: val);
123	}
124
125	static void vcap_encode_field(u32 *stream, struct vcap_stream_iter *itr,
126	int width, const u8 *value)
127	{
128	int idx;
129
130	/* Loop over the field value bits and add the value bits one by one to
131	* the output stream.
132	*/
133	for (idx = 0; idx < width; idx++) {
134	u8 bidx = idx & GENMASK(2, 0);
135
136	/* Encode one field value bit */
137	vcap_encode_bit(stream, itr, val: (value[idx / 8] >> bidx) & 0x1);
138	vcap_iter_next(itr);
139	}
140	}
141
142	static void vcap_encode_typegroups(u32 *stream, int sw_width,
143	const struct vcap_typegroup *tg,
144	bool mask)
145	{
146	struct vcap_stream_iter iter;
147	int idx;
148
149	/* Mask bits must be set to zeros (inverted later when writing to the
150	* mask cache register), so that the mask typegroup bits consist of
151	* match-1 or match-0, or both
152	*/
153	vcap_iter_set(itr: &iter, sw_width, tg, offset: 0);
154	while (iter.tg->width) {
155	/* Set position to current typegroup bit */
156	iter.offset = iter.tg->offset;
157	vcap_iter_update(itr: &iter);
158	for (idx = 0; idx < iter.tg->width; idx++) {
159	/* Iterate over current typegroup bits. Mask typegroup
160	* bits are always set
161	*/
162	if (mask)
163	vcap_set_bit(stream, itr: &iter, value: 0x1);
164	else
165	vcap_set_bit(stream, itr: &iter,
166	value: (iter.tg->value >> idx) & 0x1);
167	iter.offset++;
168	vcap_iter_update(itr: &iter);
169	}
170	iter.tg++; /* next typegroup */
171	}
172	}
173
174	static bool vcap_bitarray_zero(int width, u8 *value)
175	{
176	int bytes = DIV_ROUND_UP(width, BITS_PER_BYTE);
177	u8 total = 0, bmask = 0xff;
178	int rwidth = width;
179	int idx;
180
181	for (idx = 0; idx < bytes; ++idx, rwidth -= BITS_PER_BYTE) {
182	if (rwidth && rwidth < BITS_PER_BYTE)
183	bmask = (1 << rwidth) - 1;
184	total += value[idx] & bmask;
185	}
186	return total == 0;
187	}
188
189	static bool vcap_get_bit(u32 *stream, struct vcap_stream_iter *itr)
190	{
191	u32 mask = BIT(itr->reg_bitpos);
192	u32 *p = &stream[itr->reg_idx];
193
194	return !!(*p & mask);
195	}
196
197	static void vcap_decode_field(u32 *stream, struct vcap_stream_iter *itr,
198	int width, u8 *value)
199	{
200	int idx;
201
202	/* Loop over the field value bits and get the field bits and
203	* set them in the output value byte array
204	*/
205	for (idx = 0; idx < width; idx++) {
206	u8 bidx = idx & 0x7;
207
208	/* Decode one field value bit */
209	if (vcap_get_bit(stream, itr))
210	*value |= 1 << bidx;
211	vcap_iter_next(itr);
212	if (bidx == 7)
213	value++;
214	}
215	}
216
217	/* Verify that the type id in the stream matches the type id of the keyset */
218	static bool vcap_verify_keystream_keyset(struct vcap_control *vctrl,
219	enum vcap_type vt,
220	u32 *keystream,
221	u32 *mskstream,
222	enum vcap_keyfield_set keyset)
223	{
224	const struct vcap_info *vcap = &vctrl->vcaps[vt];
225	const struct vcap_field *typefld;
226	const struct vcap_typegroup *tgt;
227	const struct vcap_field *fields;
228	struct vcap_stream_iter iter;
229	const struct vcap_set *info;
230	u32 value = 0;
231	u32 mask = 0;
232
233	if (vcap_keyfield_count(vctrl, vt, keyset) == 0)
234	return false;
235
236	info = vcap_keyfieldset(vctrl, vt, keyset);
237	/* Check that the keyset is valid */
238	if (!info)
239	return false;
240
241	/* a type_id of value -1 means that there is no type field */
242	if (info->type_id == (u8)-1)
243	return true;
244
245	/* Get a valid typegroup for the specific keyset */
246	tgt = vcap_keyfield_typegroup(vctrl, vt, keyset);
247	if (!tgt)
248	return false;
249
250	fields = vcap_keyfields(vctrl, vt, keyset);
251	if (!fields)
252	return false;
253
254	typefld = &fields[VCAP_KF_TYPE];
255	vcap_iter_init(itr: &iter, sw_width: vcap->sw_width, tg: tgt, offset: typefld->offset);
256	vcap_decode_field(stream: mskstream, itr: &iter, width: typefld->width, value: (u8 *)&mask);
257	/* no type info if there are no mask bits */
258	if (vcap_bitarray_zero(width: typefld->width, value: (u8 *)&mask))
259	return false;
260
261	/* Get the value of the type field in the stream and compare to the
262	* one define in the vcap keyset
263	*/
264	vcap_iter_init(itr: &iter, sw_width: vcap->sw_width, tg: tgt, offset: typefld->offset);
265	vcap_decode_field(stream: keystream, itr: &iter, width: typefld->width, value: (u8 *)&value);
266
267	return (value & mask) == (info->type_id & mask);
268	}
269
270	/* Verify that the typegroup bits have the correct values */
271	static int vcap_verify_typegroups(u32 *stream, int sw_width,
272	const struct vcap_typegroup *tgt, bool mask,
273	int sw_max)
274	{
275	struct vcap_stream_iter iter;
276	int sw_cnt, idx;
277
278	vcap_iter_set(itr: &iter, sw_width, tg: tgt, offset: 0);
279	sw_cnt = 0;
280	while (iter.tg->width) {
281	u32 value = 0;
282	u32 tg_value = iter.tg->value;
283
284	if (mask)
285	tg_value = (1 << iter.tg->width) - 1;
286	/* Set position to current typegroup bit */
287	iter.offset = iter.tg->offset;
288	vcap_iter_update(itr: &iter);
289	for (idx = 0; idx < iter.tg->width; idx++) {
290	/* Decode one typegroup bit */
291	if (vcap_get_bit(stream, itr: &iter))
292	value |= 1 << idx;
293	iter.offset++;
294	vcap_iter_update(itr: &iter);
295	}
296	if (value != tg_value)
297	return -EINVAL;
298	iter.tg++; /* next typegroup */
299	sw_cnt++;
300	/* Stop checking more typegroups */
301	if (sw_max && sw_cnt >= sw_max)
302	break;
303	}
304	return 0;
305	}
306
307	/* Find the subword width of the key typegroup that matches the stream data */
308	static int vcap_find_keystream_typegroup_sw(struct vcap_control *vctrl,
309	enum vcap_type vt, u32 *stream,
310	bool mask, int sw_max)
311	{
312	const struct vcap_typegroup **tgt;
313	int sw_idx, res;
314
315	tgt = vctrl->vcaps[vt].keyfield_set_typegroups;
316	/* Try the longest subword match first */
317	for (sw_idx = vctrl->vcaps[vt].sw_count; sw_idx >= 0; sw_idx--) {
318	if (!tgt[sw_idx])
319	continue;
320
321	res = vcap_verify_typegroups(stream, sw_width: vctrl->vcaps[vt].sw_width,
322	tgt: tgt[sw_idx], mask, sw_max);
323	if (res == 0)
324	return sw_idx;
325	}
326	return -EINVAL;
327	}
328
329	/* Verify that the typegroup information, subword count, keyset and type id
330	* are in sync and correct, return the list of matchin keysets
331	*/
332	int
333	vcap_find_keystream_keysets(struct vcap_control *vctrl,
334	enum vcap_type vt,
335	u32 *keystream,
336	u32 *mskstream,
337	bool mask, int sw_max,
338	struct vcap_keyset_list *kslist)
339	{
340	const struct vcap_set *keyfield_set;
341	int sw_count, idx;
342
343	sw_count = vcap_find_keystream_typegroup_sw(vctrl, vt, stream: keystream, mask,
344	sw_max);
345	if (sw_count < 0)
346	return sw_count;
347
348	keyfield_set = vctrl->vcaps[vt].keyfield_set;
349	for (idx = 0; idx < vctrl->vcaps[vt].keyfield_set_size; ++idx) {
350	if (keyfield_set[idx].sw_per_item != sw_count)
351	continue;
352
353	if (vcap_verify_keystream_keyset(vctrl, vt, keystream,
354	mskstream, keyset: idx))
355	vcap_keyset_list_add(keysetlist: kslist, keyset: idx);
356	}
357	if (kslist->cnt > 0)
358	return 0;
359	return -EINVAL;
360	}
361	EXPORT_SYMBOL_GPL(vcap_find_keystream_keysets);
362
363	/* Read key data from a VCAP address and discover if there are any rule keysets
364	* here
365	*/
366	int vcap_addr_keysets(struct vcap_control *vctrl,
367	struct net_device *ndev,
368	struct vcap_admin *admin,
369	int addr,
370	struct vcap_keyset_list *kslist)
371	{
372	enum vcap_type vt = admin->vtype;
373	int keyset_sw_regs, idx;
374	u32 key = 0, mask = 0;
375
376	/* Read the cache at the specified address */
377	keyset_sw_regs = DIV_ROUND_UP(vctrl->vcaps[vt].sw_width, 32);
378	vctrl->ops->update(ndev, admin, VCAP_CMD_READ, VCAP_SEL_ALL, addr);
379	vctrl->ops->cache_read(ndev, admin, VCAP_SEL_ENTRY, 0,
380	keyset_sw_regs);
381	/* Skip uninitialized key/mask entries */
382	for (idx = 0; idx < keyset_sw_regs; ++idx) {
383	key |= ~admin->cache.keystream[idx];
384	mask |= admin->cache.maskstream[idx];
385	}
386	if (key == 0 && mask == 0)
387	return -EINVAL;
388	/* Decode and locate the keysets */
389	return vcap_find_keystream_keysets(vctrl, vt, admin->cache.keystream,
390	admin->cache.maskstream, false, 0,
391	kslist);
392	}
393	EXPORT_SYMBOL_GPL(vcap_addr_keysets);
394
395	/* Return the list of keyfields for the keyset */
396	const struct vcap_field *vcap_keyfields(struct vcap_control *vctrl,
397	enum vcap_type vt,
398	enum vcap_keyfield_set keyset)
399	{
400	/* Check that the keyset exists in the vcap keyset list */
401	if (keyset >= vctrl->vcaps[vt].keyfield_set_size)
402	return NULL;
403	return vctrl->vcaps[vt].keyfield_set_map[keyset];
404	}
405
406	/* Return the keyset information for the keyset */
407	const struct vcap_set *vcap_keyfieldset(struct vcap_control *vctrl,
408	enum vcap_type vt,
409	enum vcap_keyfield_set keyset)
410	{
411	const struct vcap_set *kset;
412
413	/* Check that the keyset exists in the vcap keyset list */
414	if (keyset >= vctrl->vcaps[vt].keyfield_set_size)
415	return NULL;
416	kset = &vctrl->vcaps[vt].keyfield_set[keyset];
417	if (kset->sw_per_item == 0 || kset->sw_per_item > vctrl->vcaps[vt].sw_count)
418	return NULL;
419	return kset;
420	}
421	EXPORT_SYMBOL_GPL(vcap_keyfieldset);
422
423	/* Return the typegroup table for the matching keyset (using subword size) */
424	const struct vcap_typegroup *
425	vcap_keyfield_typegroup(struct vcap_control *vctrl,
426	enum vcap_type vt, enum vcap_keyfield_set keyset)
427	{
428	const struct vcap_set *kset = vcap_keyfieldset(vctrl, vt, keyset);
429
430	/* Check that the keyset is valid */
431	if (!kset)
432	return NULL;
433	return vctrl->vcaps[vt].keyfield_set_typegroups[kset->sw_per_item];
434	}
435
436	/* Return the number of keyfields in the keyset */
437	int vcap_keyfield_count(struct vcap_control *vctrl,
438	enum vcap_type vt, enum vcap_keyfield_set keyset)
439	{
440	/* Check that the keyset exists in the vcap keyset list */
441	if (keyset >= vctrl->vcaps[vt].keyfield_set_size)
442	return 0;
443	return vctrl->vcaps[vt].keyfield_set_map_size[keyset];
444	}
445
446	static void vcap_encode_keyfield(struct vcap_rule_internal *ri,
447	const struct vcap_client_keyfield *kf,
448	const struct vcap_field *rf,
449	const struct vcap_typegroup *tgt)
450	{
451	int sw_width = ri->vctrl->vcaps[ri->admin->vtype].sw_width;
452	struct vcap_cache_data *cache = &ri->admin->cache;
453	struct vcap_stream_iter iter;
454	const u8 *value, *mask;
455
456	/* Encode the fields for the key and the mask in their respective
457	* streams, respecting the subword width.
458	*/
459	switch (kf->ctrl.type) {
460	case VCAP_FIELD_BIT:
461	value = &kf->data.u1.value;
462	mask = &kf->data.u1.mask;
463	break;
464	case VCAP_FIELD_U32:
465	value = (const u8 *)&kf->data.u32.value;
466	mask = (const u8 *)&kf->data.u32.mask;
467	break;
468	case VCAP_FIELD_U48:
469	value = kf->data.u48.value;
470	mask = kf->data.u48.mask;
471	break;
472	case VCAP_FIELD_U56:
473	value = kf->data.u56.value;
474	mask = kf->data.u56.mask;
475	break;
476	case VCAP_FIELD_U64:
477	value = kf->data.u64.value;
478	mask = kf->data.u64.mask;
479	break;
480	case VCAP_FIELD_U72:
481	value = kf->data.u72.value;
482	mask = kf->data.u72.mask;
483	break;
484	case VCAP_FIELD_U112:
485	value = kf->data.u112.value;
486	mask = kf->data.u112.mask;
487	break;
488	case VCAP_FIELD_U128:
489	value = kf->data.u128.value;
490	mask = kf->data.u128.mask;
491	break;
492	}
493	vcap_iter_init(itr: &iter, sw_width, tg: tgt, offset: rf->offset);
494	vcap_encode_field(stream: cache->keystream, itr: &iter, width: rf->width, value);
495	vcap_iter_init(itr: &iter, sw_width, tg: tgt, offset: rf->offset);
496	vcap_encode_field(stream: cache->maskstream, itr: &iter, width: rf->width, value: mask);
497	}
498
499	static void vcap_encode_keyfield_typegroups(struct vcap_control *vctrl,
500	struct vcap_rule_internal *ri,
501	const struct vcap_typegroup *tgt)
502	{
503	int sw_width = vctrl->vcaps[ri->admin->vtype].sw_width;
504	struct vcap_cache_data *cache = &ri->admin->cache;
505
506	/* Encode the typegroup bits for the key and the mask in their streams,
507	* respecting the subword width.
508	*/
509	vcap_encode_typegroups(stream: cache->keystream, sw_width, tg: tgt, mask: false);
510	vcap_encode_typegroups(stream: cache->maskstream, sw_width, tg: tgt, mask: true);
511	}
512
513	/* Copy data from src to dst but reverse the data in chunks of 32bits.
514	* For example if src is 00:11:22:33:44:55 where 55 is LSB the dst will
515	* have the value 22:33:44:55:00:11.
516	*/
517	static void vcap_copy_to_w32be(u8 *dst, const u8 *src, int size)
518	{
519	for (int idx = 0; idx < size; ++idx) {
520	int first_byte_index = 0;
521	int nidx;
522
523	first_byte_index = size - (((idx >> 2) + 1) << 2);
524	if (first_byte_index < 0)
525	first_byte_index = 0;
526	nidx = idx + first_byte_index - (idx & ~0x3);
527	dst[nidx] = src[idx];
528	}
529	}
530
531	static void
532	vcap_copy_from_client_keyfield(struct vcap_rule *rule,
533	struct vcap_client_keyfield *dst,
534	const struct vcap_client_keyfield *src)
535	{
536	struct vcap_rule_internal *ri = to_intrule(rule);
537	const struct vcap_client_keyfield_data *sdata;
538	struct vcap_client_keyfield_data *ddata;
539	int size;
540
541	dst->ctrl.type = src->ctrl.type;
542	dst->ctrl.key = src->ctrl.key;
543	INIT_LIST_HEAD(list: &dst->ctrl.list);
544	sdata = &src->data;
545	ddata = &dst->data;
546
547	if (!ri->admin->w32be) {
548	memcpy(ddata, sdata, sizeof(dst->data));
549	return;
550	}
551
552	size = keyfield_size_table[dst->ctrl.type] / 2;
553
554	switch (dst->ctrl.type) {
555	case VCAP_FIELD_BIT:
556	case VCAP_FIELD_U32:
557	memcpy(ddata, sdata, sizeof(dst->data));
558	break;
559	case VCAP_FIELD_U48:
560	vcap_copy_to_w32be(dst: ddata->u48.value, src: src->data.u48.value, size);
561	vcap_copy_to_w32be(dst: ddata->u48.mask, src: src->data.u48.mask, size);
562	break;
563	case VCAP_FIELD_U56:
564	vcap_copy_to_w32be(dst: ddata->u56.value, src: sdata->u56.value, size);
565	vcap_copy_to_w32be(dst: ddata->u56.mask, src: sdata->u56.mask, size);
566	break;
567	case VCAP_FIELD_U64:
568	vcap_copy_to_w32be(dst: ddata->u64.value, src: sdata->u64.value, size);
569	vcap_copy_to_w32be(dst: ddata->u64.mask, src: sdata->u64.mask, size);
570	break;
571	case VCAP_FIELD_U72:
572	vcap_copy_to_w32be(dst: ddata->u72.value, src: sdata->u72.value, size);
573	vcap_copy_to_w32be(dst: ddata->u72.mask, src: sdata->u72.mask, size);
574	break;
575	case VCAP_FIELD_U112:
576	vcap_copy_to_w32be(dst: ddata->u112.value, src: sdata->u112.value, size);
577	vcap_copy_to_w32be(dst: ddata->u112.mask, src: sdata->u112.mask, size);
578	break;
579	case VCAP_FIELD_U128:
580	vcap_copy_to_w32be(dst: ddata->u128.value, src: sdata->u128.value, size);
581	vcap_copy_to_w32be(dst: ddata->u128.mask, src: sdata->u128.mask, size);
582	break;
583	}
584	}
585
586	static void
587	vcap_copy_from_client_actionfield(struct vcap_rule *rule,
588	struct vcap_client_actionfield *dst,
589	const struct vcap_client_actionfield *src)
590	{
591	struct vcap_rule_internal *ri = to_intrule(rule);
592	const struct vcap_client_actionfield_data *sdata;
593	struct vcap_client_actionfield_data *ddata;
594	int size;
595
596	dst->ctrl.type = src->ctrl.type;
597	dst->ctrl.action = src->ctrl.action;
598	INIT_LIST_HEAD(list: &dst->ctrl.list);
599	sdata = &src->data;
600	ddata = &dst->data;
601
602	if (!ri->admin->w32be) {
603	memcpy(ddata, sdata, sizeof(dst->data));
604	return;
605	}
606
607	size = actionfield_size_table[dst->ctrl.type];
608
609	switch (dst->ctrl.type) {
610	case VCAP_FIELD_BIT:
611	case VCAP_FIELD_U32:
612	memcpy(ddata, sdata, sizeof(dst->data));
613	break;
614	case VCAP_FIELD_U48:
615	vcap_copy_to_w32be(dst: ddata->u48.value, src: sdata->u48.value, size);
616	break;
617	case VCAP_FIELD_U56:
618	vcap_copy_to_w32be(dst: ddata->u56.value, src: sdata->u56.value, size);
619	break;
620	case VCAP_FIELD_U64:
621	vcap_copy_to_w32be(dst: ddata->u64.value, src: sdata->u64.value, size);
622	break;
623	case VCAP_FIELD_U72:
624	vcap_copy_to_w32be(dst: ddata->u72.value, src: sdata->u72.value, size);
625	break;
626	case VCAP_FIELD_U112:
627	vcap_copy_to_w32be(dst: ddata->u112.value, src: sdata->u112.value, size);
628	break;
629	case VCAP_FIELD_U128:
630	vcap_copy_to_w32be(dst: ddata->u128.value, src: sdata->u128.value, size);
631	break;
632	}
633	}
634
635	static int vcap_encode_rule_keyset(struct vcap_rule_internal *ri)
636	{
637	const struct vcap_client_keyfield *ckf;
638	const struct vcap_typegroup *tg_table;
639	struct vcap_client_keyfield tempkf;
640	const struct vcap_field *kf_table;
641	int keyset_size;
642
643	/* Get a valid set of fields for the specific keyset */
644	kf_table = vcap_keyfields(vctrl: ri->vctrl, vt: ri->admin->vtype, keyset: ri->data.keyset);
645	if (!kf_table) {
646	pr_err("%s:%d: no fields available for this keyset: %d\n",
647	__func__, __LINE__, ri->data.keyset);
648	return -EINVAL;
649	}
650	/* Get a valid typegroup for the specific keyset */
651	tg_table = vcap_keyfield_typegroup(vctrl: ri->vctrl, vt: ri->admin->vtype,
652	keyset: ri->data.keyset);
653	if (!tg_table) {
654	pr_err("%s:%d: no typegroups available for this keyset: %d\n",
655	__func__, __LINE__, ri->data.keyset);
656	return -EINVAL;
657	}
658	/* Get a valid size for the specific keyset */
659	keyset_size = vcap_keyfield_count(vctrl: ri->vctrl, vt: ri->admin->vtype,
660	keyset: ri->data.keyset);
661	if (keyset_size == 0) {
662	pr_err("%s:%d: zero field count for this keyset: %d\n",
663	__func__, __LINE__, ri->data.keyset);
664	return -EINVAL;
665	}
666	/* Iterate over the keyfields (key, mask) in the rule
667	* and encode these bits
668	*/
669	if (list_empty(head: &ri->data.keyfields)) {
670	pr_err("%s:%d: no keyfields in the rule\n", __func__, __LINE__);
671	return -EINVAL;
672	}
673	list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list) {
674	/* Check that the client entry exists in the keyset */
675	if (ckf->ctrl.key >= keyset_size) {
676	pr_err("%s:%d: key %d is not in vcap\n",
677	__func__, __LINE__, ckf->ctrl.key);
678	return -EINVAL;
679	}
680	vcap_copy_from_client_keyfield(rule: &ri->data, dst: &tempkf, src: ckf);
681	vcap_encode_keyfield(ri, kf: &tempkf, rf: &kf_table[ckf->ctrl.key],
682	tgt: tg_table);
683	}
684	/* Add typegroup bits to the key/mask bitstreams */
685	vcap_encode_keyfield_typegroups(vctrl: ri->vctrl, ri, tgt: tg_table);
686	return 0;
687	}
688
689	/* Return the list of actionfields for the actionset */
690	const struct vcap_field *
691	vcap_actionfields(struct vcap_control *vctrl,
692	enum vcap_type vt, enum vcap_actionfield_set actionset)
693	{
694	/* Check that the actionset exists in the vcap actionset list */
695	if (actionset >= vctrl->vcaps[vt].actionfield_set_size)
696	return NULL;
697	return vctrl->vcaps[vt].actionfield_set_map[actionset];
698	}
699
700	const struct vcap_set *
701	vcap_actionfieldset(struct vcap_control *vctrl,
702	enum vcap_type vt, enum vcap_actionfield_set actionset)
703	{
704	const struct vcap_set *aset;
705
706	/* Check that the actionset exists in the vcap actionset list */
707	if (actionset >= vctrl->vcaps[vt].actionfield_set_size)
708	return NULL;
709	aset = &vctrl->vcaps[vt].actionfield_set[actionset];
710	if (aset->sw_per_item == 0 || aset->sw_per_item > vctrl->vcaps[vt].sw_count)
711	return NULL;
712	return aset;
713	}
714
715	/* Return the typegroup table for the matching actionset (using subword size) */
716	const struct vcap_typegroup *
717	vcap_actionfield_typegroup(struct vcap_control *vctrl,
718	enum vcap_type vt, enum vcap_actionfield_set actionset)
719	{
720	const struct vcap_set *aset = vcap_actionfieldset(vctrl, vt, actionset);
721
722	/* Check that the actionset is valid */
723	if (!aset)
724	return NULL;
725	return vctrl->vcaps[vt].actionfield_set_typegroups[aset->sw_per_item];
726	}
727
728	/* Return the number of actionfields in the actionset */
729	int vcap_actionfield_count(struct vcap_control *vctrl,
730	enum vcap_type vt,
731	enum vcap_actionfield_set actionset)
732	{
733	/* Check that the actionset exists in the vcap actionset list */
734	if (actionset >= vctrl->vcaps[vt].actionfield_set_size)
735	return 0;
736	return vctrl->vcaps[vt].actionfield_set_map_size[actionset];
737	}
738
739	static void vcap_encode_actionfield(struct vcap_rule_internal *ri,
740	const struct vcap_client_actionfield *af,
741	const struct vcap_field *rf,
742	const struct vcap_typegroup *tgt)
743	{
744	int act_width = ri->vctrl->vcaps[ri->admin->vtype].act_width;
745
746	struct vcap_cache_data *cache = &ri->admin->cache;
747	struct vcap_stream_iter iter;
748	const u8 *value;
749
750	/* Encode the action field in the stream, respecting the subword width */
751	switch (af->ctrl.type) {
752	case VCAP_FIELD_BIT:
753	value = &af->data.u1.value;
754	break;
755	case VCAP_FIELD_U32:
756	value = (const u8 *)&af->data.u32.value;
757	break;
758	case VCAP_FIELD_U48:
759	value = af->data.u48.value;
760	break;
761	case VCAP_FIELD_U56:
762	value = af->data.u56.value;
763	break;
764	case VCAP_FIELD_U64:
765	value = af->data.u64.value;
766	break;
767	case VCAP_FIELD_U72:
768	value = af->data.u72.value;
769	break;
770	case VCAP_FIELD_U112:
771	value = af->data.u112.value;
772	break;
773	case VCAP_FIELD_U128:
774	value = af->data.u128.value;
775	break;
776	}
777	vcap_iter_init(itr: &iter, sw_width: act_width, tg: tgt, offset: rf->offset);
778	vcap_encode_field(stream: cache->actionstream, itr: &iter, width: rf->width, value);
779	}
780
781	static void vcap_encode_actionfield_typegroups(struct vcap_rule_internal *ri,
782	const struct vcap_typegroup *tgt)
783	{
784	int sw_width = ri->vctrl->vcaps[ri->admin->vtype].act_width;
785	struct vcap_cache_data *cache = &ri->admin->cache;
786
787	/* Encode the typegroup bits for the actionstream respecting the subword
788	* width.
789	*/
790	vcap_encode_typegroups(stream: cache->actionstream, sw_width, tg: tgt, mask: false);
791	}
792
793	static int vcap_encode_rule_actionset(struct vcap_rule_internal *ri)
794	{
795	const struct vcap_client_actionfield *caf;
796	const struct vcap_typegroup *tg_table;
797	struct vcap_client_actionfield tempaf;
798	const struct vcap_field *af_table;
799	int actionset_size;
800
801	/* Get a valid set of actionset fields for the specific actionset */
802	af_table = vcap_actionfields(vctrl: ri->vctrl, vt: ri->admin->vtype,
803	actionset: ri->data.actionset);
804	if (!af_table) {
805	pr_err("%s:%d: no fields available for this actionset: %d\n",
806	__func__, __LINE__, ri->data.actionset);
807	return -EINVAL;
808	}
809	/* Get a valid typegroup for the specific actionset */
810	tg_table = vcap_actionfield_typegroup(vctrl: ri->vctrl, vt: ri->admin->vtype,
811	actionset: ri->data.actionset);
812	if (!tg_table) {
813	pr_err("%s:%d: no typegroups available for this actionset: %d\n",
814	__func__, __LINE__, ri->data.actionset);
815	return -EINVAL;
816	}
817	/* Get a valid actionset size for the specific actionset */
818	actionset_size = vcap_actionfield_count(vctrl: ri->vctrl, vt: ri->admin->vtype,
819	actionset: ri->data.actionset);
820	if (actionset_size == 0) {
821	pr_err("%s:%d: zero field count for this actionset: %d\n",
822	__func__, __LINE__, ri->data.actionset);
823	return -EINVAL;
824	}
825	/* Iterate over the actionfields in the rule
826	* and encode these bits
827	*/
828	if (list_empty(head: &ri->data.actionfields))
829	pr_warn("%s:%d: no actionfields in the rule\n",
830	__func__, __LINE__);
831	list_for_each_entry(caf, &ri->data.actionfields, ctrl.list) {
832	/* Check that the client action exists in the actionset */
833	if (caf->ctrl.action >= actionset_size) {
834	pr_err("%s:%d: action %d is not in vcap\n",
835	__func__, __LINE__, caf->ctrl.action);
836	return -EINVAL;
837	}
838	vcap_copy_from_client_actionfield(rule: &ri->data, dst: &tempaf, src: caf);
839	vcap_encode_actionfield(ri, af: &tempaf,
840	rf: &af_table[caf->ctrl.action], tgt: tg_table);
841	}
842	/* Add typegroup bits to the entry bitstreams */
843	vcap_encode_actionfield_typegroups(ri, tgt: tg_table);
844	return 0;
845	}
846
847	static int vcap_encode_rule(struct vcap_rule_internal *ri)
848	{
849	int err;
850
851	err = vcap_encode_rule_keyset(ri);
852	if (err)
853	return err;
854	err = vcap_encode_rule_actionset(ri);
855	if (err)
856	return err;
857	return 0;
858	}
859
860	int vcap_api_check(struct vcap_control *ctrl)
861	{
862	if (!ctrl) {
863	pr_err("%s:%d: vcap control is missing\n", __func__, __LINE__);
864	return -EINVAL;
865	}
866	if (!ctrl->ops || !ctrl->ops->validate_keyset ||
867	!ctrl->ops->add_default_fields || !ctrl->ops->cache_erase ||
868	!ctrl->ops->cache_write || !ctrl->ops->cache_read ||
869	!ctrl->ops->init || !ctrl->ops->update || !ctrl->ops->move ||
870	!ctrl->ops->port_info) {
871	pr_err("%s:%d: client operations are missing\n",
872	__func__, __LINE__);
873	return -ENOENT;
874	}
875	return 0;
876	}
877
878	void vcap_erase_cache(struct vcap_rule_internal *ri)
879	{
880	ri->vctrl->ops->cache_erase(ri->admin);
881	}
882
883	/* Update the keyset for the rule */
884	int vcap_set_rule_set_keyset(struct vcap_rule *rule,
885	enum vcap_keyfield_set keyset)
886	{
887	struct vcap_rule_internal *ri = to_intrule(rule);
888	const struct vcap_set *kset;
889	int sw_width;
890
891	kset = vcap_keyfieldset(ri->vctrl, ri->admin->vtype, keyset);
892	/* Check that the keyset is valid */
893	if (!kset)
894	return -EINVAL;
895	ri->keyset_sw = kset->sw_per_item;
896	sw_width = ri->vctrl->vcaps[ri->admin->vtype].sw_width;
897	ri->keyset_sw_regs = DIV_ROUND_UP(sw_width, 32);
898	ri->data.keyset = keyset;
899	return 0;
900	}
901	EXPORT_SYMBOL_GPL(vcap_set_rule_set_keyset);
902
903	/* Update the actionset for the rule */
904	int vcap_set_rule_set_actionset(struct vcap_rule *rule,
905	enum vcap_actionfield_set actionset)
906	{
907	struct vcap_rule_internal *ri = to_intrule(rule);
908	const struct vcap_set *aset;
909	int act_width;
910
911	aset = vcap_actionfieldset(vctrl: ri->vctrl, vt: ri->admin->vtype, actionset);
912	/* Check that the actionset is valid */
913	if (!aset)
914	return -EINVAL;
915	ri->actionset_sw = aset->sw_per_item;
916	act_width = ri->vctrl->vcaps[ri->admin->vtype].act_width;
917	ri->actionset_sw_regs = DIV_ROUND_UP(act_width, 32);
918	ri->data.actionset = actionset;
919	return 0;
920	}
921	EXPORT_SYMBOL_GPL(vcap_set_rule_set_actionset);
922
923	/* Check if a rule with this id exists */
924	static bool vcap_rule_exists(struct vcap_control *vctrl, u32 id)
925	{
926	struct vcap_rule_internal *ri;
927	struct vcap_admin *admin;
928
929	/* Look for the rule id in all vcaps */
930	list_for_each_entry(admin, &vctrl->list, list)
931	list_for_each_entry(ri, &admin->rules, list)
932	if (ri->data.id == id)
933	return true;
934	return false;
935	}
936
937	/* Find a rule with a provided rule id return a locked vcap */
938	static struct vcap_rule_internal *
939	vcap_get_locked_rule(struct vcap_control *vctrl, u32 id)
940	{
941	struct vcap_rule_internal *ri;
942	struct vcap_admin *admin;
943
944	/* Look for the rule id in all vcaps */
945	list_for_each_entry(admin, &vctrl->list, list) {
946	mutex_lock(&admin->lock);
947	list_for_each_entry(ri, &admin->rules, list)
948	if (ri->data.id == id)
949	return ri;
950	mutex_unlock(lock: &admin->lock);
951	}
952	return NULL;
953	}
954
955	/* Find a rule id with a provided cookie */
956	int vcap_lookup_rule_by_cookie(struct vcap_control *vctrl, u64 cookie)
957	{
958	struct vcap_rule_internal *ri;
959	struct vcap_admin *admin;
960	int id = 0;
961
962	/* Look for the rule id in all vcaps */
963	list_for_each_entry(admin, &vctrl->list, list) {
964	mutex_lock(&admin->lock);
965	list_for_each_entry(ri, &admin->rules, list) {
966	if (ri->data.cookie == cookie) {
967	id = ri->data.id;
968	break;
969	}
970	}
971	mutex_unlock(lock: &admin->lock);
972	if (id)
973	return id;
974	}
975	return -ENOENT;
976	}
977	EXPORT_SYMBOL_GPL(vcap_lookup_rule_by_cookie);
978
979	/* Get number of rules in a vcap instance lookup chain id range */
980	int vcap_admin_rule_count(struct vcap_admin *admin, int cid)
981	{
982	int max_cid = roundup(cid + 1, VCAP_CID_LOOKUP_SIZE);
983	int min_cid = rounddown(cid, VCAP_CID_LOOKUP_SIZE);
984	struct vcap_rule_internal *elem;
985	int count = 0;
986
987	list_for_each_entry(elem, &admin->rules, list) {
988	mutex_lock(&admin->lock);
989	if (elem->data.vcap_chain_id >= min_cid &&
990	elem->data.vcap_chain_id < max_cid)
991	++count;
992	mutex_unlock(lock: &admin->lock);
993	}
994	return count;
995	}
996	EXPORT_SYMBOL_GPL(vcap_admin_rule_count);
997
998	/* Make a copy of the rule, shallow or full */
999	static struct vcap_rule_internal *vcap_dup_rule(struct vcap_rule_internal *ri,
1000	bool full)
1001	{
1002	struct vcap_client_actionfield *caf, *newcaf;
1003	struct vcap_client_keyfield *ckf, *newckf;
1004	struct vcap_rule_internal *duprule;
1005
1006	/* Allocate the client part */
1007	duprule = kzalloc(size: sizeof(*duprule), GFP_KERNEL);
1008	if (!duprule)
1009	return ERR_PTR(error: -ENOMEM);
1010	*duprule = *ri;
1011	/* Not inserted in the VCAP */
1012	INIT_LIST_HEAD(list: &duprule->list);
1013	/* No elements in these lists */
1014	INIT_LIST_HEAD(list: &duprule->data.keyfields);
1015	INIT_LIST_HEAD(list: &duprule->data.actionfields);
1016
1017	/* A full rule copy includes keys and actions */
1018	if (!full)
1019	return duprule;
1020
1021	list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list) {
1022	newckf = kmemdup(p: ckf, size: sizeof(*newckf), GFP_KERNEL);
1023	if (!newckf)
1024	goto err;
1025	list_add_tail(new: &newckf->ctrl.list, head: &duprule->data.keyfields);
1026	}
1027
1028	list_for_each_entry(caf, &ri->data.actionfields, ctrl.list) {
1029	newcaf = kmemdup(p: caf, size: sizeof(*newcaf), GFP_KERNEL);
1030	if (!newcaf)
1031	goto err;
1032	list_add_tail(new: &newcaf->ctrl.list, head: &duprule->data.actionfields);
1033	}
1034
1035	return duprule;
1036
1037	err:
1038	list_for_each_entry_safe(ckf, newckf, &duprule->data.keyfields, ctrl.list) {
1039	list_del(entry: &ckf->ctrl.list);
1040	kfree(objp: ckf);
1041	}
1042
1043	list_for_each_entry_safe(caf, newcaf, &duprule->data.actionfields, ctrl.list) {
1044	list_del(entry: &caf->ctrl.list);
1045	kfree(objp: caf);
1046	}
1047
1048	kfree(objp: duprule);
1049	return ERR_PTR(error: -ENOMEM);
1050	}
1051
1052	static void vcap_apply_width(u8 *dst, int width, int bytes)
1053	{
1054	u8 bmask;
1055	int idx;
1056
1057	for (idx = 0; idx < bytes; idx++) {
1058	if (width > 0)
1059	if (width < 8)
1060	bmask = (1 << width) - 1;
1061	else
1062	bmask = ~0;
1063	else
1064	bmask = 0;
1065	dst[idx] &= bmask;
1066	width -= 8;
1067	}
1068	}
1069
1070	static void vcap_copy_from_w32be(u8 *dst, u8 *src, int size, int width)
1071	{
1072	int idx, ridx, wstart, nidx;
1073	int tail_bytes = (((size + 4) >> 2) << 2) - size;
1074
1075	for (idx = 0, ridx = size - 1; idx < size; ++idx, --ridx) {
1076	wstart = (idx >> 2) << 2;
1077	nidx = wstart + 3 - (idx & 0x3);
1078	if (nidx >= size)
1079	nidx -= tail_bytes;
1080	dst[nidx] = src[ridx];
1081	}
1082
1083	vcap_apply_width(dst, width, bytes: size);
1084	}
1085
1086	static void vcap_copy_action_bit_field(struct vcap_u1_action *field, u8 *value)
1087	{
1088	field->value = (*value) & 0x1;
1089	}
1090
1091	static void vcap_copy_limited_actionfield(u8 *dstvalue, u8 *srcvalue,
1092	int width, int bytes)
1093	{
1094	memcpy(dstvalue, srcvalue, bytes);
1095	vcap_apply_width(dst: dstvalue, width, bytes);
1096	}
1097
1098	static void vcap_copy_to_client_actionfield(struct vcap_rule_internal *ri,
1099	struct vcap_client_actionfield *field,
1100	u8 *value, u16 width)
1101	{
1102	int field_size = actionfield_size_table[field->ctrl.type];
1103
1104	if (ri->admin->w32be) {
1105	switch (field->ctrl.type) {
1106	case VCAP_FIELD_BIT:
1107	vcap_copy_action_bit_field(field: &field->data.u1, value);
1108	break;
1109	case VCAP_FIELD_U32:
1110	vcap_copy_limited_actionfield(dstvalue: (u8 *)&field->data.u32.value,
1111	srcvalue: value,
1112	width, bytes: field_size);
1113	break;
1114	case VCAP_FIELD_U48:
1115	vcap_copy_from_w32be(dst: field->data.u48.value, src: value,
1116	size: field_size, width);
1117	break;
1118	case VCAP_FIELD_U56:
1119	vcap_copy_from_w32be(dst: field->data.u56.value, src: value,
1120	size: field_size, width);
1121	break;
1122	case VCAP_FIELD_U64:
1123	vcap_copy_from_w32be(dst: field->data.u64.value, src: value,
1124	size: field_size, width);
1125	break;
1126	case VCAP_FIELD_U72:
1127	vcap_copy_from_w32be(dst: field->data.u72.value, src: value,
1128	size: field_size, width);
1129	break;
1130	case VCAP_FIELD_U112:
1131	vcap_copy_from_w32be(dst: field->data.u112.value, src: value,
1132	size: field_size, width);
1133	break;
1134	case VCAP_FIELD_U128:
1135	vcap_copy_from_w32be(dst: field->data.u128.value, src: value,
1136	size: field_size, width);
1137	break;
1138	}
1139	} else {
1140	switch (field->ctrl.type) {
1141	case VCAP_FIELD_BIT:
1142	vcap_copy_action_bit_field(field: &field->data.u1, value);
1143	break;
1144	case VCAP_FIELD_U32:
1145	vcap_copy_limited_actionfield(dstvalue: (u8 *)&field->data.u32.value,
1146	srcvalue: value,
1147	width, bytes: field_size);
1148	break;
1149	case VCAP_FIELD_U48:
1150	vcap_copy_limited_actionfield(dstvalue: field->data.u48.value,
1151	srcvalue: value,
1152	width, bytes: field_size);
1153	break;
1154	case VCAP_FIELD_U56:
1155	vcap_copy_limited_actionfield(dstvalue: field->data.u56.value,
1156	srcvalue: value,
1157	width, bytes: field_size);
1158	break;
1159	case VCAP_FIELD_U64:
1160	vcap_copy_limited_actionfield(dstvalue: field->data.u64.value,
1161	srcvalue: value,
1162	width, bytes: field_size);
1163	break;
1164	case VCAP_FIELD_U72:
1165	vcap_copy_limited_actionfield(dstvalue: field->data.u72.value,
1166	srcvalue: value,
1167	width, bytes: field_size);
1168	break;
1169	case VCAP_FIELD_U112:
1170	vcap_copy_limited_actionfield(dstvalue: field->data.u112.value,
1171	srcvalue: value,
1172	width, bytes: field_size);
1173	break;
1174	case VCAP_FIELD_U128:
1175	vcap_copy_limited_actionfield(dstvalue: field->data.u128.value,
1176	srcvalue: value,
1177	width, bytes: field_size);
1178	break;
1179	}
1180	}
1181	}
1182
1183	static void vcap_copy_key_bit_field(struct vcap_u1_key *field,
1184	u8 *value, u8 *mask)
1185	{
1186	field->value = (*value) & 0x1;
1187	field->mask = (*mask) & 0x1;
1188	}
1189
1190	static void vcap_copy_limited_keyfield(u8 *dstvalue, u8 *dstmask,
1191	u8 *srcvalue, u8 *srcmask,
1192	int width, int bytes)
1193	{
1194	memcpy(dstvalue, srcvalue, bytes);
1195	vcap_apply_width(dst: dstvalue, width, bytes);
1196	memcpy(dstmask, srcmask, bytes);
1197	vcap_apply_width(dst: dstmask, width, bytes);
1198	}
1199
1200	static void vcap_copy_to_client_keyfield(struct vcap_rule_internal *ri,
1201	struct vcap_client_keyfield *field,
1202	u8 *value, u8 *mask, u16 width)
1203	{
1204	int field_size = keyfield_size_table[field->ctrl.type] / 2;
1205
1206	if (ri->admin->w32be) {
1207	switch (field->ctrl.type) {
1208	case VCAP_FIELD_BIT:
1209	vcap_copy_key_bit_field(field: &field->data.u1, value, mask);
1210	break;
1211	case VCAP_FIELD_U32:
1212	vcap_copy_limited_keyfield(dstvalue: (u8 *)&field->data.u32.value,
1213	dstmask: (u8 *)&field->data.u32.mask,
1214	srcvalue: value, srcmask: mask,
1215	width, bytes: field_size);
1216	break;
1217	case VCAP_FIELD_U48:
1218	vcap_copy_from_w32be(dst: field->data.u48.value, src: value,
1219	size: field_size, width);
1220	vcap_copy_from_w32be(dst: field->data.u48.mask, src: mask,
1221	size: field_size, width);
1222	break;
1223	case VCAP_FIELD_U56:
1224	vcap_copy_from_w32be(dst: field->data.u56.value, src: value,
1225	size: field_size, width);
1226	vcap_copy_from_w32be(dst: field->data.u56.mask, src: mask,
1227	size: field_size, width);
1228	break;
1229	case VCAP_FIELD_U64:
1230	vcap_copy_from_w32be(dst: field->data.u64.value, src: value,
1231	size: field_size, width);
1232	vcap_copy_from_w32be(dst: field->data.u64.mask, src: mask,
1233	size: field_size, width);
1234	break;
1235	case VCAP_FIELD_U72:
1236	vcap_copy_from_w32be(dst: field->data.u72.value, src: value,
1237	size: field_size, width);
1238	vcap_copy_from_w32be(dst: field->data.u72.mask, src: mask,
1239	size: field_size, width);
1240	break;
1241	case VCAP_FIELD_U112:
1242	vcap_copy_from_w32be(dst: field->data.u112.value, src: value,
1243	size: field_size, width);
1244	vcap_copy_from_w32be(dst: field->data.u112.mask, src: mask,
1245	size: field_size, width);
1246	break;
1247	case VCAP_FIELD_U128:
1248	vcap_copy_from_w32be(dst: field->data.u128.value, src: value,
1249	size: field_size, width);
1250	vcap_copy_from_w32be(dst: field->data.u128.mask, src: mask,
1251	size: field_size, width);
1252	break;
1253	}
1254	} else {
1255	switch (field->ctrl.type) {
1256	case VCAP_FIELD_BIT:
1257	vcap_copy_key_bit_field(field: &field->data.u1, value, mask);
1258	break;
1259	case VCAP_FIELD_U32:
1260	vcap_copy_limited_keyfield(dstvalue: (u8 *)&field->data.u32.value,
1261	dstmask: (u8 *)&field->data.u32.mask,
1262	srcvalue: value, srcmask: mask,
1263	width, bytes: field_size);
1264	break;
1265	case VCAP_FIELD_U48:
1266	vcap_copy_limited_keyfield(dstvalue: field->data.u48.value,
1267	dstmask: field->data.u48.mask,
1268	srcvalue: value, srcmask: mask,
1269	width, bytes: field_size);
1270	break;
1271	case VCAP_FIELD_U56:
1272	vcap_copy_limited_keyfield(dstvalue: field->data.u56.value,
1273	dstmask: field->data.u56.mask,
1274	srcvalue: value, srcmask: mask,
1275	width, bytes: field_size);
1276	break;
1277	case VCAP_FIELD_U64:
1278	vcap_copy_limited_keyfield(dstvalue: field->data.u64.value,
1279	dstmask: field->data.u64.mask,
1280	srcvalue: value, srcmask: mask,
1281	width, bytes: field_size);
1282	break;
1283	case VCAP_FIELD_U72:
1284	vcap_copy_limited_keyfield(dstvalue: field->data.u72.value,
1285	dstmask: field->data.u72.mask,
1286	srcvalue: value, srcmask: mask,
1287	width, bytes: field_size);
1288	break;
1289	case VCAP_FIELD_U112:
1290	vcap_copy_limited_keyfield(dstvalue: field->data.u112.value,
1291	dstmask: field->data.u112.mask,
1292	srcvalue: value, srcmask: mask,
1293	width, bytes: field_size);
1294	break;
1295	case VCAP_FIELD_U128:
1296	vcap_copy_limited_keyfield(dstvalue: field->data.u128.value,
1297	dstmask: field->data.u128.mask,
1298	srcvalue: value, srcmask: mask,
1299	width, bytes: field_size);
1300	break;
1301	}
1302	}
1303	}
1304
1305	static void vcap_rule_alloc_keyfield(struct vcap_rule_internal *ri,
1306	const struct vcap_field *keyfield,
1307	enum vcap_key_field key,
1308	u8 *value, u8 *mask)
1309	{
1310	struct vcap_client_keyfield *field;
1311
1312	field = kzalloc(size: sizeof(*field), GFP_KERNEL);
1313	if (!field)
1314	return;
1315	INIT_LIST_HEAD(list: &field->ctrl.list);
1316	field->ctrl.key = key;
1317	field->ctrl.type = keyfield->type;
1318	vcap_copy_to_client_keyfield(ri, field, value, mask, width: keyfield->width);
1319	list_add_tail(new: &field->ctrl.list, head: &ri->data.keyfields);
1320	}
1321
1322	/* Read key data from a VCAP address and discover if there is a rule keyset
1323	* here
1324	*/
1325	static bool
1326	vcap_verify_actionstream_actionset(struct vcap_control *vctrl,
1327	enum vcap_type vt,
1328	u32 *actionstream,
1329	enum vcap_actionfield_set actionset)
1330	{
1331	const struct vcap_typegroup *tgt;
1332	const struct vcap_field *fields;
1333	const struct vcap_set *info;
1334
1335	if (vcap_actionfield_count(vctrl, vt, actionset) == 0)
1336	return false;
1337
1338	info = vcap_actionfieldset(vctrl, vt, actionset);
1339	/* Check that the actionset is valid */
1340	if (!info)
1341	return false;
1342
1343	/* a type_id of value -1 means that there is no type field */
1344	if (info->type_id == (u8)-1)
1345	return true;
1346
1347	/* Get a valid typegroup for the specific actionset */
1348	tgt = vcap_actionfield_typegroup(vctrl, vt, actionset);
1349	if (!tgt)
1350	return false;
1351
1352	fields = vcap_actionfields(vctrl, vt, actionset);
1353	if (!fields)
1354	return false;
1355
1356	/* Later this will be expanded with a check of the type id */
1357	return true;
1358	}
1359
1360	/* Find the subword width of the action typegroup that matches the stream data
1361	*/
1362	static int vcap_find_actionstream_typegroup_sw(struct vcap_control *vctrl,
1363	enum vcap_type vt, u32 *stream,
1364	int sw_max)
1365	{
1366	const struct vcap_typegroup **tgt;
1367	int sw_idx, res;
1368
1369	tgt = vctrl->vcaps[vt].actionfield_set_typegroups;
1370	/* Try the longest subword match first */
1371	for (sw_idx = vctrl->vcaps[vt].sw_count; sw_idx >= 0; sw_idx--) {
1372	if (!tgt[sw_idx])
1373	continue;
1374	res = vcap_verify_typegroups(stream, sw_width: vctrl->vcaps[vt].act_width,
1375	tgt: tgt[sw_idx], mask: false, sw_max);
1376	if (res == 0)
1377	return sw_idx;
1378	}
1379	return -EINVAL;
1380	}
1381
1382	/* Verify that the typegroup information, subword count, actionset and type id
1383	* are in sync and correct, return the actionset
1384	*/
1385	static enum vcap_actionfield_set
1386	vcap_find_actionstream_actionset(struct vcap_control *vctrl,
1387	enum vcap_type vt,
1388	u32 *stream,
1389	int sw_max)
1390	{
1391	const struct vcap_set *actionfield_set;
1392	int sw_count, idx;
1393	bool res;
1394
1395	sw_count = vcap_find_actionstream_typegroup_sw(vctrl, vt, stream,
1396	sw_max);
1397	if (sw_count < 0)
1398	return sw_count;
1399
1400	actionfield_set = vctrl->vcaps[vt].actionfield_set;
1401	for (idx = 0; idx < vctrl->vcaps[vt].actionfield_set_size; ++idx) {
1402	if (actionfield_set[idx].sw_per_item != sw_count)
1403	continue;
1404
1405	res = vcap_verify_actionstream_actionset(vctrl, vt,
1406	actionstream: stream, actionset: idx);
1407	if (res)
1408	return idx;
1409	}
1410	return -EINVAL;
1411	}
1412
1413	/* Store action value in an element in a list for the client */
1414	static void vcap_rule_alloc_actionfield(struct vcap_rule_internal *ri,
1415	const struct vcap_field *actionfield,
1416	enum vcap_action_field action,
1417	u8 *value)
1418	{
1419	struct vcap_client_actionfield *field;
1420
1421	field = kzalloc(size: sizeof(*field), GFP_KERNEL);
1422	if (!field)
1423	return;
1424	INIT_LIST_HEAD(list: &field->ctrl.list);
1425	field->ctrl.action = action;
1426	field->ctrl.type = actionfield->type;
1427	vcap_copy_to_client_actionfield(ri, field, value, width: actionfield->width);
1428	list_add_tail(new: &field->ctrl.list, head: &ri->data.actionfields);
1429	}
1430
1431	static int vcap_decode_actionset(struct vcap_rule_internal *ri)
1432	{
1433	struct vcap_control *vctrl = ri->vctrl;
1434	struct vcap_admin *admin = ri->admin;
1435	const struct vcap_field *actionfield;
1436	enum vcap_actionfield_set actionset;
1437	enum vcap_type vt = admin->vtype;
1438	const struct vcap_typegroup *tgt;
1439	struct vcap_stream_iter iter;
1440	int idx, res, actfield_count;
1441	u32 *actstream;
1442	u8 value[16];
1443
1444	actstream = admin->cache.actionstream;
1445	res = vcap_find_actionstream_actionset(vctrl, vt, stream: actstream, sw_max: 0);
1446	if (res < 0) {
1447	pr_err("%s:%d: could not find valid actionset: %d\n",
1448	__func__, __LINE__, res);
1449	return -EINVAL;
1450	}
1451	actionset = res;
1452	actfield_count = vcap_actionfield_count(vctrl, vt, actionset);
1453	actionfield = vcap_actionfields(vctrl, vt, actionset);
1454	tgt = vcap_actionfield_typegroup(vctrl, vt, actionset);
1455	/* Start decoding the stream */
1456	for (idx = 0; idx < actfield_count; ++idx) {
1457	if (actionfield[idx].width <= 0)
1458	continue;
1459	/* Get the action */
1460	memset(value, 0, DIV_ROUND_UP(actionfield[idx].width, 8));
1461	vcap_iter_init(itr: &iter, sw_width: vctrl->vcaps[vt].act_width, tg: tgt,
1462	offset: actionfield[idx].offset);
1463	vcap_decode_field(stream: actstream, itr: &iter, width: actionfield[idx].width,
1464	value);
1465	/* Skip if no bits are set */
1466	if (vcap_bitarray_zero(width: actionfield[idx].width, value))
1467	continue;
1468	vcap_rule_alloc_actionfield(ri, actionfield: &actionfield[idx], action: idx, value);
1469	/* Later the action id will also be checked */
1470	}
1471	return vcap_set_rule_set_actionset((struct vcap_rule *)ri, actionset);
1472	}
1473
1474	static int vcap_decode_keyset(struct vcap_rule_internal *ri)
1475	{
1476	struct vcap_control *vctrl = ri->vctrl;
1477	struct vcap_stream_iter kiter, miter;
1478	struct vcap_admin *admin = ri->admin;
1479	enum vcap_keyfield_set keysets[10];
1480	const struct vcap_field *keyfield;
1481	enum vcap_type vt = admin->vtype;
1482	const struct vcap_typegroup *tgt;
1483	struct vcap_keyset_list matches;
1484	enum vcap_keyfield_set keyset;
1485	int idx, res, keyfield_count;
1486	u32 *maskstream;
1487	u32 *keystream;
1488	u8 value[16];
1489	u8 mask[16];
1490
1491	keystream = admin->cache.keystream;
1492	maskstream = admin->cache.maskstream;
1493	matches.keysets = keysets;
1494	matches.cnt = 0;
1495	matches.max = ARRAY_SIZE(keysets);
1496	res = vcap_find_keystream_keysets(vctrl, vt, keystream, maskstream,
1497	false, 0, &matches);
1498	if (res < 0) {
1499	pr_err("%s:%d: could not find valid keysets: %d\n",
1500	__func__, __LINE__, res);
1501	return -EINVAL;
1502	}
1503	keyset = matches.keysets[0];
1504	keyfield_count = vcap_keyfield_count(vctrl, vt, keyset);
1505	keyfield = vcap_keyfields(vctrl, vt, keyset);
1506	tgt = vcap_keyfield_typegroup(vctrl, vt, keyset);
1507	/* Start decoding the streams */
1508	for (idx = 0; idx < keyfield_count; ++idx) {
1509	if (keyfield[idx].width <= 0)
1510	continue;
1511	/* First get the mask */
1512	memset(mask, 0, DIV_ROUND_UP(keyfield[idx].width, 8));
1513	vcap_iter_init(itr: &miter, sw_width: vctrl->vcaps[vt].sw_width, tg: tgt,
1514	offset: keyfield[idx].offset);
1515	vcap_decode_field(stream: maskstream, itr: &miter, width: keyfield[idx].width,
1516	value: mask);
1517	/* Skip if no mask bits are set */
1518	if (vcap_bitarray_zero(width: keyfield[idx].width, value: mask))
1519	continue;
1520	/* Get the key */
1521	memset(value, 0, DIV_ROUND_UP(keyfield[idx].width, 8));
1522	vcap_iter_init(itr: &kiter, sw_width: vctrl->vcaps[vt].sw_width, tg: tgt,
1523	offset: keyfield[idx].offset);
1524	vcap_decode_field(stream: keystream, itr: &kiter, width: keyfield[idx].width,
1525	value);
1526	vcap_rule_alloc_keyfield(ri, keyfield: &keyfield[idx], key: idx, value, mask);
1527	}
1528	return vcap_set_rule_set_keyset((struct vcap_rule *)ri, keyset);
1529	}
1530
1531	/* Read VCAP content into the VCAP cache */
1532	static int vcap_read_rule(struct vcap_rule_internal *ri)
1533	{
1534	struct vcap_admin *admin = ri->admin;
1535	int sw_idx, ent_idx = 0, act_idx = 0;
1536	u32 addr = ri->addr;
1537
1538	if (!ri->size || !ri->keyset_sw_regs || !ri->actionset_sw_regs) {
1539	pr_err("%s:%d: rule is empty\n", __func__, __LINE__);
1540	return -EINVAL;
1541	}
1542	vcap_erase_cache(ri);
1543	/* Use the values in the streams to read the VCAP cache */
1544	for (sw_idx = 0; sw_idx < ri->size; sw_idx++, addr++) {
1545	ri->vctrl->ops->update(ri->ndev, admin, VCAP_CMD_READ,
1546	VCAP_SEL_ALL, addr);
1547	ri->vctrl->ops->cache_read(ri->ndev, admin,
1548	VCAP_SEL_ENTRY, ent_idx,
1549	ri->keyset_sw_regs);
1550	ri->vctrl->ops->cache_read(ri->ndev, admin,
1551	VCAP_SEL_ACTION, act_idx,
1552	ri->actionset_sw_regs);
1553	if (sw_idx == 0)
1554	ri->vctrl->ops->cache_read(ri->ndev, admin,
1555	VCAP_SEL_COUNTER,
1556	ri->counter_id, 0);
1557	ent_idx += ri->keyset_sw_regs;
1558	act_idx += ri->actionset_sw_regs;
1559	}
1560	return 0;
1561	}
1562
1563	/* Write VCAP cache content to the VCAP HW instance */
1564	static int vcap_write_rule(struct vcap_rule_internal *ri)
1565	{
1566	struct vcap_admin *admin = ri->admin;
1567	int sw_idx, ent_idx = 0, act_idx = 0;
1568	u32 addr = ri->addr;
1569
1570	if (!ri->size || !ri->keyset_sw_regs || !ri->actionset_sw_regs) {
1571	pr_err("%s:%d: rule is empty\n", __func__, __LINE__);
1572	return -EINVAL;
1573	}
1574	/* Use the values in the streams to write the VCAP cache */
1575	for (sw_idx = 0; sw_idx < ri->size; sw_idx++, addr++) {
1576	ri->vctrl->ops->cache_write(ri->ndev, admin,
1577	VCAP_SEL_ENTRY, ent_idx,
1578	ri->keyset_sw_regs);
1579	ri->vctrl->ops->cache_write(ri->ndev, admin,
1580	VCAP_SEL_ACTION, act_idx,
1581	ri->actionset_sw_regs);
1582	ri->vctrl->ops->update(ri->ndev, admin, VCAP_CMD_WRITE,
1583	VCAP_SEL_ALL, addr);
1584	ent_idx += ri->keyset_sw_regs;
1585	act_idx += ri->actionset_sw_regs;
1586	}
1587	return 0;
1588	}
1589
1590	static int vcap_write_counter(struct vcap_rule_internal *ri,
1591	struct vcap_counter *ctr)
1592	{
1593	struct vcap_admin *admin = ri->admin;
1594
1595	admin->cache.counter = ctr->value;
1596	admin->cache.sticky = ctr->sticky;
1597	ri->vctrl->ops->cache_write(ri->ndev, admin, VCAP_SEL_COUNTER,
1598	ri->counter_id, 0);
1599	ri->vctrl->ops->update(ri->ndev, admin, VCAP_CMD_WRITE,
1600	VCAP_SEL_COUNTER, ri->addr);
1601	return 0;
1602	}
1603
1604	/* Convert a chain id to a VCAP lookup index */
1605	int vcap_chain_id_to_lookup(struct vcap_admin *admin, int cur_cid)
1606	{
1607	int lookup_first = admin->vinst * admin->lookups_per_instance;
1608	int lookup_last = lookup_first + admin->lookups_per_instance;
1609	int cid_next = admin->first_cid + VCAP_CID_LOOKUP_SIZE;
1610	int cid = admin->first_cid;
1611	int lookup;
1612
1613	for (lookup = lookup_first; lookup < lookup_last; ++lookup,
1614	cid += VCAP_CID_LOOKUP_SIZE, cid_next += VCAP_CID_LOOKUP_SIZE)
1615	if (cur_cid >= cid && cur_cid < cid_next)
1616	return lookup;
1617	return 0;
1618	}
1619	EXPORT_SYMBOL_GPL(vcap_chain_id_to_lookup);
1620
1621	/* Lookup a vcap instance using chain id */
1622	struct vcap_admin *vcap_find_admin(struct vcap_control *vctrl, int cid)
1623	{
1624	struct vcap_admin *admin;
1625
1626	if (vcap_api_check(ctrl: vctrl))
1627	return NULL;
1628
1629	list_for_each_entry(admin, &vctrl->list, list) {
1630	if (cid >= admin->first_cid && cid <= admin->last_cid)
1631	return admin;
1632	}
1633	return NULL;
1634	}
1635	EXPORT_SYMBOL_GPL(vcap_find_admin);
1636
1637	/* Is this the last admin instance ordered by chain id and direction */
1638	static bool vcap_admin_is_last(struct vcap_control *vctrl,
1639	struct vcap_admin *admin,
1640	bool ingress)
1641	{
1642	struct vcap_admin *iter, *last = NULL;
1643	int max_cid = 0;
1644
1645	list_for_each_entry(iter, &vctrl->list, list) {
1646	if (iter->first_cid > max_cid &&
1647	iter->ingress == ingress) {
1648	last = iter;
1649	max_cid = iter->first_cid;
1650	}
1651	}
1652	if (!last)
1653	return false;
1654
1655	return admin == last;
1656	}
1657
1658	/* Calculate the value used for chaining VCAP rules */
1659	int vcap_chain_offset(struct vcap_control *vctrl, int from_cid, int to_cid)
1660	{
1661	int diff = to_cid - from_cid;
1662
1663	if (diff < 0) /* Wrong direction */
1664	return diff;
1665	to_cid %= VCAP_CID_LOOKUP_SIZE;
1666	if (to_cid == 0) /* Destination aligned to a lookup == no chaining */
1667	return 0;
1668	diff %= VCAP_CID_LOOKUP_SIZE; /* Limit to a value within a lookup */
1669	return diff;
1670	}
1671	EXPORT_SYMBOL_GPL(vcap_chain_offset);
1672
1673	/* Is the next chain id in one of the following lookups
1674	* For now this does not support filters linked to other filters using
1675	* keys and actions. That will be added later.
1676	*/
1677	bool vcap_is_next_lookup(struct vcap_control *vctrl, int src_cid, int dst_cid)
1678	{
1679	struct vcap_admin *admin;
1680	int next_cid;
1681
1682	if (vcap_api_check(ctrl: vctrl))
1683	return false;
1684
1685	/* The offset must be at least one lookup so round up one chain */
1686	next_cid = roundup(src_cid + 1, VCAP_CID_LOOKUP_SIZE);
1687
1688	if (dst_cid < next_cid)
1689	return false;
1690
1691	admin = vcap_find_admin(vctrl, dst_cid);
1692	if (!admin)
1693	return false;
1694
1695	return true;
1696	}
1697	EXPORT_SYMBOL_GPL(vcap_is_next_lookup);
1698
1699	/* Check if there is room for a new rule */
1700	static int vcap_rule_space(struct vcap_admin *admin, int size)
1701	{
1702	if (admin->last_used_addr - size < admin->first_valid_addr) {
1703	pr_err("%s:%d: No room for rule size: %u, %u\n",
1704	__func__, __LINE__, size, admin->first_valid_addr);
1705	return -ENOSPC;
1706	}
1707	return 0;
1708	}
1709
1710	/* Add the keyset typefield to the list of rule keyfields */
1711	static int vcap_add_type_keyfield(struct vcap_rule *rule)
1712	{
1713	struct vcap_rule_internal *ri = to_intrule(rule);
1714	enum vcap_keyfield_set keyset = rule->keyset;
1715	enum vcap_type vt = ri->admin->vtype;
1716	const struct vcap_field *fields;
1717	const struct vcap_set *kset;
1718	int ret = -EINVAL;
1719
1720	kset = vcap_keyfieldset(ri->vctrl, vt, keyset);
1721	if (!kset)
1722	return ret;
1723	if (kset->type_id == (u8)-1) /* No type field is needed */
1724	return 0;
1725
1726	fields = vcap_keyfields(vctrl: ri->vctrl, vt, keyset);
1727	if (!fields)
1728	return -EINVAL;
1729	if (fields[VCAP_KF_TYPE].width > 1) {
1730	ret = vcap_rule_add_key_u32(rule, key: VCAP_KF_TYPE,
1731	value: kset->type_id, mask: 0xff);
1732	} else {
1733	if (kset->type_id)
1734	ret = vcap_rule_add_key_bit(rule, key: VCAP_KF_TYPE,
1735	val: VCAP_BIT_1);
1736	else
1737	ret = vcap_rule_add_key_bit(rule, key: VCAP_KF_TYPE,
1738	val: VCAP_BIT_0);
1739	}
1740	return 0;
1741	}
1742
1743	/* Add the actionset typefield to the list of rule actionfields */
1744	static int vcap_add_type_actionfield(struct vcap_rule *rule)
1745	{
1746	enum vcap_actionfield_set actionset = rule->actionset;
1747	struct vcap_rule_internal *ri = to_intrule(rule);
1748	enum vcap_type vt = ri->admin->vtype;
1749	const struct vcap_field *fields;
1750	const struct vcap_set *aset;
1751	int ret = -EINVAL;
1752
1753	aset = vcap_actionfieldset(vctrl: ri->vctrl, vt, actionset);
1754	if (!aset)
1755	return ret;
1756	if (aset->type_id == (u8)-1) /* No type field is needed */
1757	return 0;
1758
1759	fields = vcap_actionfields(vctrl: ri->vctrl, vt, actionset);
1760	if (!fields)
1761	return -EINVAL;
1762	if (fields[VCAP_AF_TYPE].width > 1) {
1763	ret = vcap_rule_add_action_u32(rule, action: VCAP_AF_TYPE,
1764	value: aset->type_id);
1765	} else {
1766	if (aset->type_id)
1767	ret = vcap_rule_add_action_bit(rule, action: VCAP_AF_TYPE,
1768	val: VCAP_BIT_1);
1769	else
1770	ret = vcap_rule_add_action_bit(rule, action: VCAP_AF_TYPE,
1771	val: VCAP_BIT_0);
1772	}
1773	return ret;
1774	}
1775
1776	/* Add a keyset to a keyset list */
1777	bool vcap_keyset_list_add(struct vcap_keyset_list *keysetlist,
1778	enum vcap_keyfield_set keyset)
1779	{
1780	int idx;
1781
1782	if (keysetlist->cnt < keysetlist->max) {
1783	/* Avoid duplicates */
1784	for (idx = 0; idx < keysetlist->cnt; ++idx)
1785	if (keysetlist->keysets[idx] == keyset)
1786	return keysetlist->cnt < keysetlist->max;
1787	keysetlist->keysets[keysetlist->cnt++] = keyset;
1788	}
1789	return keysetlist->cnt < keysetlist->max;
1790	}
1791	EXPORT_SYMBOL_GPL(vcap_keyset_list_add);
1792
1793	/* Add a actionset to a actionset list */
1794	static bool vcap_actionset_list_add(struct vcap_actionset_list *actionsetlist,
1795	enum vcap_actionfield_set actionset)
1796	{
1797	int idx;
1798
1799	if (actionsetlist->cnt < actionsetlist->max) {
1800	/* Avoid duplicates */
1801	for (idx = 0; idx < actionsetlist->cnt; ++idx)
1802	if (actionsetlist->actionsets[idx] == actionset)
1803	return actionsetlist->cnt < actionsetlist->max;
1804	actionsetlist->actionsets[actionsetlist->cnt++] = actionset;
1805	}
1806	return actionsetlist->cnt < actionsetlist->max;
1807	}
1808
1809	/* map keyset id to a string with the keyset name */
1810	const char *vcap_keyset_name(struct vcap_control *vctrl,
1811	enum vcap_keyfield_set keyset)
1812	{
1813	return vctrl->stats->keyfield_set_names[keyset];
1814	}
1815	EXPORT_SYMBOL_GPL(vcap_keyset_name);
1816
1817	/* map key field id to a string with the key name */
1818	const char *vcap_keyfield_name(struct vcap_control *vctrl,
1819	enum vcap_key_field key)
1820	{
1821	return vctrl->stats->keyfield_names[key];
1822	}
1823	EXPORT_SYMBOL_GPL(vcap_keyfield_name);
1824
1825	/* map actionset id to a string with the actionset name */
1826	const char *vcap_actionset_name(struct vcap_control *vctrl,
1827	enum vcap_actionfield_set actionset)
1828	{
1829	return vctrl->stats->actionfield_set_names[actionset];
1830	}
1831
1832	/* map action field id to a string with the action name */
1833	const char *vcap_actionfield_name(struct vcap_control *vctrl,
1834	enum vcap_action_field action)
1835	{
1836	return vctrl->stats->actionfield_names[action];
1837	}
1838
1839	/* Return the keyfield that matches a key in a keyset */
1840	static const struct vcap_field *
1841	vcap_find_keyset_keyfield(struct vcap_control *vctrl,
1842	enum vcap_type vtype,
1843	enum vcap_keyfield_set keyset,
1844	enum vcap_key_field key)
1845	{
1846	const struct vcap_field *fields;
1847	int idx, count;
1848
1849	fields = vcap_keyfields(vctrl, vt: vtype, keyset);
1850	if (!fields)
1851	return NULL;
1852
1853	/* Iterate the keyfields of the keyset */
1854	count = vcap_keyfield_count(vctrl, vt: vtype, keyset);
1855	for (idx = 0; idx < count; ++idx) {
1856	if (fields[idx].width == 0)
1857	continue;
1858
1859	if (key == idx)
1860	return &fields[idx];
1861	}
1862
1863	return NULL;
1864	}
1865
1866	/* Match a list of keys against the keysets available in a vcap type */
1867	static bool _vcap_rule_find_keysets(struct vcap_rule_internal *ri,
1868	struct vcap_keyset_list *matches)
1869	{
1870	const struct vcap_client_keyfield *ckf;
1871	int keyset, found, keycount, map_size;
1872	const struct vcap_field **map;
1873	enum vcap_type vtype;
1874
1875	vtype = ri->admin->vtype;
1876	map = ri->vctrl->vcaps[vtype].keyfield_set_map;
1877	map_size = ri->vctrl->vcaps[vtype].keyfield_set_size;
1878
1879	/* Get a count of the keyfields we want to match */
1880	keycount = 0;
1881	list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list)
1882	++keycount;
1883
1884	matches->cnt = 0;
1885	/* Iterate the keysets of the VCAP */
1886	for (keyset = 0; keyset < map_size; ++keyset) {
1887	if (!map[keyset])
1888	continue;
1889
1890	/* Iterate the keys in the rule */
1891	found = 0;
1892	list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list)
1893	if (vcap_find_keyset_keyfield(vctrl: ri->vctrl, vtype,
1894	keyset, key: ckf->ctrl.key))
1895	++found;
1896
1897	/* Save the keyset if all keyfields were found */
1898	if (found == keycount)
1899	if (!vcap_keyset_list_add(matches, keyset))
1900	/* bail out when the quota is filled */
1901	break;
1902	}
1903
1904	return matches->cnt > 0;
1905	}
1906
1907	/* Match a list of keys against the keysets available in a vcap type */
1908	bool vcap_rule_find_keysets(struct vcap_rule *rule,
1909	struct vcap_keyset_list *matches)
1910	{
1911	struct vcap_rule_internal *ri = to_intrule(rule);
1912
1913	return _vcap_rule_find_keysets(ri, matches);
1914	}
1915	EXPORT_SYMBOL_GPL(vcap_rule_find_keysets);
1916
1917	/* Return the actionfield that matches a action in a actionset */
1918	static const struct vcap_field *
1919	vcap_find_actionset_actionfield(struct vcap_control *vctrl,
1920	enum vcap_type vtype,
1921	enum vcap_actionfield_set actionset,
1922	enum vcap_action_field action)
1923	{
1924	const struct vcap_field *fields;
1925	int idx, count;
1926
1927	fields = vcap_actionfields(vctrl, vt: vtype, actionset);
1928	if (!fields)
1929	return NULL;
1930
1931	/* Iterate the actionfields of the actionset */
1932	count = vcap_actionfield_count(vctrl, vt: vtype, actionset);
1933	for (idx = 0; idx < count; ++idx) {
1934	if (fields[idx].width == 0)
1935	continue;
1936
1937	if (action == idx)
1938	return &fields[idx];
1939	}
1940
1941	return NULL;
1942	}
1943
1944	/* Match a list of actions against the actionsets available in a vcap type */
1945	static bool vcap_rule_find_actionsets(struct vcap_rule_internal *ri,
1946	struct vcap_actionset_list *matches)
1947	{
1948	int actionset, found, actioncount, map_size;
1949	const struct vcap_client_actionfield *ckf;
1950	const struct vcap_field **map;
1951	enum vcap_type vtype;
1952
1953	vtype = ri->admin->vtype;
1954	map = ri->vctrl->vcaps[vtype].actionfield_set_map;
1955	map_size = ri->vctrl->vcaps[vtype].actionfield_set_size;
1956
1957	/* Get a count of the actionfields we want to match */
1958	actioncount = 0;
1959	list_for_each_entry(ckf, &ri->data.actionfields, ctrl.list)
1960	++actioncount;
1961
1962	matches->cnt = 0;
1963	/* Iterate the actionsets of the VCAP */
1964	for (actionset = 0; actionset < map_size; ++actionset) {
1965	if (!map[actionset])
1966	continue;
1967
1968	/* Iterate the actions in the rule */
1969	found = 0;
1970	list_for_each_entry(ckf, &ri->data.actionfields, ctrl.list)
1971	if (vcap_find_actionset_actionfield(vctrl: ri->vctrl, vtype,
1972	actionset,
1973	action: ckf->ctrl.action))
1974	++found;
1975
1976	/* Save the actionset if all actionfields were found */
1977	if (found == actioncount)
1978	if (!vcap_actionset_list_add(actionsetlist: matches, actionset))
1979	/* bail out when the quota is filled */
1980	break;
1981	}
1982
1983	return matches->cnt > 0;
1984	}
1985
1986	/* Validate a rule with respect to available port keys */
1987	int vcap_val_rule(struct vcap_rule *rule, u16 l3_proto)
1988	{
1989	struct vcap_rule_internal *ri = to_intrule(rule);
1990	struct vcap_keyset_list matches = {};
1991	enum vcap_keyfield_set keysets[10];
1992	int ret;
1993
1994	ret = vcap_api_check(ctrl: ri->vctrl);
1995	if (ret)
1996	return ret;
1997	if (!ri->admin) {
1998	ri->data.exterr = VCAP_ERR_NO_ADMIN;
1999	return -EINVAL;
2000	}
2001	if (!ri->ndev) {
2002	ri->data.exterr = VCAP_ERR_NO_NETDEV;
2003	return -EINVAL;
2004	}
2005
2006	matches.keysets = keysets;
2007	matches.max = ARRAY_SIZE(keysets);
2008	if (ri->data.keyset == VCAP_KFS_NO_VALUE) {
2009	/* Iterate over rule keyfields and select keysets that fits */
2010	if (!_vcap_rule_find_keysets(ri, matches: &matches)) {
2011	ri->data.exterr = VCAP_ERR_NO_KEYSET_MATCH;
2012	return -EINVAL;
2013	}
2014	} else {
2015	/* prepare for keyset validation */
2016	keysets[0] = ri->data.keyset;
2017	matches.cnt = 1;
2018	}
2019
2020	/* Pick a keyset that is supported in the port lookups */
2021	ret = ri->vctrl->ops->validate_keyset(ri->ndev, ri->admin, rule,
2022	&matches, l3_proto);
2023	if (ret < 0) {
2024	pr_err("%s:%d: keyset validation failed: %d\n",
2025	__func__, __LINE__, ret);
2026	ri->data.exterr = VCAP_ERR_NO_PORT_KEYSET_MATCH;
2027	return ret;
2028	}
2029	/* use the keyset that is supported in the port lookups */
2030	ret = vcap_set_rule_set_keyset(rule, ret);
2031	if (ret < 0) {
2032	pr_err("%s:%d: keyset was not updated: %d\n",
2033	__func__, __LINE__, ret);
2034	return ret;
2035	}
2036	if (ri->data.actionset == VCAP_AFS_NO_VALUE) {
2037	struct vcap_actionset_list matches = {};
2038	enum vcap_actionfield_set actionsets[10];
2039
2040	matches.actionsets = actionsets;
2041	matches.max = ARRAY_SIZE(actionsets);
2042
2043	/* Find an actionset that fits the rule actions */
2044	if (!vcap_rule_find_actionsets(ri, matches: &matches)) {
2045	ri->data.exterr = VCAP_ERR_NO_ACTIONSET_MATCH;
2046	return -EINVAL;
2047	}
2048	ret = vcap_set_rule_set_actionset(rule, actionsets[0]);
2049	if (ret < 0) {
2050	pr_err("%s:%d: actionset was not updated: %d\n",
2051	__func__, __LINE__, ret);
2052	return ret;
2053	}
2054	}
2055	vcap_add_type_keyfield(rule);
2056	vcap_add_type_actionfield(rule);
2057	/* Add default fields to this rule */
2058	ri->vctrl->ops->add_default_fields(ri->ndev, ri->admin, rule);
2059
2060	/* Rule size is the maximum of the entry and action subword count */
2061	ri->size = max(ri->keyset_sw, ri->actionset_sw);
2062
2063	/* Finally check if there is room for the rule in the VCAP */
2064	return vcap_rule_space(admin: ri->admin, size: ri->size);
2065	}
2066	EXPORT_SYMBOL_GPL(vcap_val_rule);
2067
2068	/* Entries are sorted with increasing values of sort_key.
2069	* I.e. Lowest numerical sort_key is first in list.
2070	* In order to locate largest keys first in list we negate the key size with
2071	* (max_size - size).
2072	*/
2073	static u32 vcap_sort_key(u32 max_size, u32 size, u8 user, u16 prio)
2074	{
2075	return ((max_size - size) << 24) | (user << 16) | prio;
2076	}
2077
2078	/* calculate the address of the next rule after this (lower address and prio) */
2079	static u32 vcap_next_rule_addr(u32 addr, struct vcap_rule_internal *ri)
2080	{
2081	return ((addr - ri->size) / ri->size) * ri->size;
2082	}
2083
2084	/* Assign a unique rule id and autogenerate one if id == 0 */
2085	static u32 vcap_set_rule_id(struct vcap_rule_internal *ri)
2086	{
2087	if (ri->data.id != 0)
2088	return ri->data.id;
2089
2090	for (u32 next_id = 1; next_id < ~0; ++next_id) {
2091	if (!vcap_rule_exists(vctrl: ri->vctrl, id: next_id)) {
2092	ri->data.id = next_id;
2093	break;
2094	}
2095	}
2096	return ri->data.id;
2097	}
2098
2099	static int vcap_insert_rule(struct vcap_rule_internal *ri,
2100	struct vcap_rule_move *move)
2101	{
2102	int sw_count = ri->vctrl->vcaps[ri->admin->vtype].sw_count;
2103	struct vcap_rule_internal *duprule, *iter, *elem = NULL;
2104	struct vcap_admin *admin = ri->admin;
2105	u32 addr;
2106
2107	ri->sort_key = vcap_sort_key(max_size: sw_count, size: ri->size, user: ri->data.user,
2108	prio: ri->data.priority);
2109
2110	/* Insert the new rule in the list of rule based on the sort key
2111	* If the rule needs to be inserted between existing rules then move
2112	* these rules to make room for the new rule and update their start
2113	* address.
2114	*/
2115	list_for_each_entry(iter, &admin->rules, list) {
2116	if (ri->sort_key < iter->sort_key) {
2117	elem = iter;
2118	break;
2119	}
2120	}
2121
2122	if (!elem) {
2123	ri->addr = vcap_next_rule_addr(addr: admin->last_used_addr, ri);
2124	admin->last_used_addr = ri->addr;
2125
2126	/* Add a copy of the rule to the VCAP list */
2127	duprule = vcap_dup_rule(ri, full: ri->state == VCAP_RS_DISABLED);
2128	if (IS_ERR(ptr: duprule))
2129	return PTR_ERR(ptr: duprule);
2130
2131	list_add_tail(new: &duprule->list, head: &admin->rules);
2132	return 0;
2133	}
2134
2135	/* Reuse the space of the current rule */
2136	addr = elem->addr + elem->size;
2137	ri->addr = vcap_next_rule_addr(addr, ri);
2138	addr = ri->addr;
2139
2140	/* Add a copy of the rule to the VCAP list */
2141	duprule = vcap_dup_rule(ri, full: ri->state == VCAP_RS_DISABLED);
2142	if (IS_ERR(ptr: duprule))
2143	return PTR_ERR(ptr: duprule);
2144
2145	/* Add before the current entry */
2146	list_add_tail(new: &duprule->list, head: &elem->list);
2147
2148	/* Update the current rule */
2149	elem->addr = vcap_next_rule_addr(addr, ri: elem);
2150	addr = elem->addr;
2151
2152	/* Update the address in the remaining rules in the list */
2153	list_for_each_entry_continue(elem, &admin->rules, list) {
2154	elem->addr = vcap_next_rule_addr(addr, ri: elem);
2155	addr = elem->addr;
2156	}
2157
2158	/* Update the move info */
2159	move->addr = admin->last_used_addr;
2160	move->count = ri->addr - addr;
2161	move->offset = admin->last_used_addr - addr;
2162	admin->last_used_addr = addr;
2163	return 0;
2164	}
2165
2166	static void vcap_move_rules(struct vcap_rule_internal *ri,
2167	struct vcap_rule_move *move)
2168	{
2169	ri->vctrl->ops->move(ri->ndev, ri->admin, move->addr,
2170	move->offset, move->count);
2171	}
2172
2173	/* Check if the chain is already used to enable a VCAP lookup for this port */
2174	static bool vcap_is_chain_used(struct vcap_control *vctrl,
2175	struct net_device *ndev, int src_cid)
2176	{
2177	struct vcap_enabled_port *eport;
2178	struct vcap_admin *admin;
2179
2180	list_for_each_entry(admin, &vctrl->list, list)
2181	list_for_each_entry(eport, &admin->enabled, list)
2182	if (eport->src_cid == src_cid && eport->ndev == ndev)
2183	return true;
2184
2185	return false;
2186	}
2187
2188	/* Fetch the next chain in the enabled list for the port */
2189	static int vcap_get_next_chain(struct vcap_control *vctrl,
2190	struct net_device *ndev,
2191	int dst_cid)
2192	{
2193	struct vcap_enabled_port *eport;
2194	struct vcap_admin *admin;
2195
2196	list_for_each_entry(admin, &vctrl->list, list) {
2197	list_for_each_entry(eport, &admin->enabled, list) {
2198	if (eport->ndev != ndev)
2199	continue;
2200	if (eport->src_cid == dst_cid)
2201	return eport->dst_cid;
2202	}
2203	}
2204
2205	return 0;
2206	}
2207
2208	static bool vcap_path_exist(struct vcap_control *vctrl, struct net_device *ndev,
2209	int dst_cid)
2210	{
2211	int cid = rounddown(dst_cid, VCAP_CID_LOOKUP_SIZE);
2212	struct vcap_enabled_port *eport = NULL;
2213	struct vcap_enabled_port *elem;
2214	struct vcap_admin *admin;
2215	int tmp;
2216
2217	if (cid == 0) /* Chain zero is always available */
2218	return true;
2219
2220	/* Find first entry that starts from chain 0*/
2221	list_for_each_entry(admin, &vctrl->list, list) {
2222	list_for_each_entry(elem, &admin->enabled, list) {
2223	if (elem->src_cid == 0 && elem->ndev == ndev) {
2224	eport = elem;
2225	break;
2226	}
2227	}
2228	if (eport)
2229	break;
2230	}
2231
2232	if (!eport)
2233	return false;
2234
2235	tmp = eport->dst_cid;
2236	while (tmp != cid && tmp != 0)
2237	tmp = vcap_get_next_chain(vctrl, ndev, dst_cid: tmp);
2238
2239	return !!tmp;
2240	}
2241
2242	/* Internal clients can always store their rules in HW
2243	* External clients can store their rules if the chain is enabled all
2244	* the way from chain 0, otherwise the rule will be cached until
2245	* the chain is enabled.
2246	*/
2247	static void vcap_rule_set_state(struct vcap_rule_internal *ri)
2248	{
2249	if (ri->data.user <= VCAP_USER_QOS)
2250	ri->state = VCAP_RS_PERMANENT;
2251	else if (vcap_path_exist(vctrl: ri->vctrl, ndev: ri->ndev, dst_cid: ri->data.vcap_chain_id))
2252	ri->state = VCAP_RS_ENABLED;
2253	else
2254	ri->state = VCAP_RS_DISABLED;
2255	}
2256
2257	/* Encode and write a validated rule to the VCAP */
2258	int vcap_add_rule(struct vcap_rule *rule)
2259	{
2260	struct vcap_rule_internal *ri = to_intrule(rule);
2261	struct vcap_rule_move move = {0};
2262	struct vcap_counter ctr = {0};
2263	int ret;
2264
2265	ret = vcap_api_check(ctrl: ri->vctrl);
2266	if (ret)
2267	return ret;
2268	/* Insert the new rule in the list of vcap rules */
2269	mutex_lock(&ri->admin->lock);
2270
2271	vcap_rule_set_state(ri);
2272	ret = vcap_insert_rule(ri, move: &move);
2273	if (ret < 0) {
2274	pr_err("%s:%d: could not insert rule in vcap list: %d\n",
2275	__func__, __LINE__, ret);
2276	goto out;
2277	}
2278	if (move.count > 0)
2279	vcap_move_rules(ri, move: &move);
2280
2281	/* Set the counter to zero */
2282	ret = vcap_write_counter(ri, ctr: &ctr);
2283	if (ret)
2284	goto out;
2285
2286	if (ri->state == VCAP_RS_DISABLED) {
2287	/* Erase the rule area */
2288	ri->vctrl->ops->init(ri->ndev, ri->admin, ri->addr, ri->size);
2289	goto out;
2290	}
2291
2292	vcap_erase_cache(ri);
2293	ret = vcap_encode_rule(ri);
2294	if (ret) {
2295	pr_err("%s:%d: rule encoding error: %d\n", __func__, __LINE__, ret);
2296	goto out;
2297	}
2298
2299	ret = vcap_write_rule(ri);
2300	if (ret) {
2301	pr_err("%s:%d: rule write error: %d\n", __func__, __LINE__, ret);
2302	goto out;
2303	}
2304	out:
2305	mutex_unlock(lock: &ri->admin->lock);
2306	return ret;
2307	}
2308	EXPORT_SYMBOL_GPL(vcap_add_rule);
2309
2310	/* Allocate a new rule with the provided arguments */
2311	struct vcap_rule *vcap_alloc_rule(struct vcap_control *vctrl,
2312	struct net_device *ndev, int vcap_chain_id,
2313	enum vcap_user user, u16 priority,
2314	u32 id)
2315	{
2316	struct vcap_rule_internal *ri;
2317	struct vcap_admin *admin;
2318	int err, maxsize;
2319
2320	err = vcap_api_check(ctrl: vctrl);
2321	if (err)
2322	return ERR_PTR(error: err);
2323	if (!ndev)
2324	return ERR_PTR(error: -ENODEV);
2325	/* Get the VCAP instance */
2326	admin = vcap_find_admin(vctrl, vcap_chain_id);
2327	if (!admin)
2328	return ERR_PTR(error: -ENOENT);
2329	/* Sanity check that this VCAP is supported on this platform */
2330	if (vctrl->vcaps[admin->vtype].rows == 0)
2331	return ERR_PTR(error: -EINVAL);
2332
2333	mutex_lock(&admin->lock);
2334	/* Check if a rule with this id already exists */
2335	if (vcap_rule_exists(vctrl, id)) {
2336	err = -EINVAL;
2337	goto out_unlock;
2338	}
2339
2340	/* Check if there is room for the rule in the block(s) of the VCAP */
2341	maxsize = vctrl->vcaps[admin->vtype].sw_count; /* worst case rule size */
2342	if (vcap_rule_space(admin, size: maxsize)) {
2343	err = -ENOSPC;
2344	goto out_unlock;
2345	}
2346
2347	/* Create a container for the rule and return it */
2348	ri = kzalloc(size: sizeof(*ri), GFP_KERNEL);
2349	if (!ri) {
2350	err = -ENOMEM;
2351	goto out_unlock;
2352	}
2353
2354	ri->data.vcap_chain_id = vcap_chain_id;
2355	ri->data.user = user;
2356	ri->data.priority = priority;
2357	ri->data.id = id;
2358	ri->data.keyset = VCAP_KFS_NO_VALUE;
2359	ri->data.actionset = VCAP_AFS_NO_VALUE;
2360	INIT_LIST_HEAD(list: &ri->list);
2361	INIT_LIST_HEAD(list: &ri->data.keyfields);
2362	INIT_LIST_HEAD(list: &ri->data.actionfields);
2363	ri->ndev = ndev;
2364	ri->admin = admin; /* refer to the vcap instance */
2365	ri->vctrl = vctrl; /* refer to the client */
2366
2367	if (vcap_set_rule_id(ri) == 0) {
2368	err = -EINVAL;
2369	goto out_free;
2370	}
2371
2372	mutex_unlock(lock: &admin->lock);
2373	return (struct vcap_rule *)ri;
2374
2375	out_free:
2376	kfree(objp: ri);
2377	out_unlock:
2378	mutex_unlock(lock: &admin->lock);
2379	return ERR_PTR(error: err);
2380
2381	}
2382	EXPORT_SYMBOL_GPL(vcap_alloc_rule);
2383
2384	/* Free mem of a rule owned by client after the rule as been added to the VCAP */
2385	void vcap_free_rule(struct vcap_rule *rule)
2386	{
2387	struct vcap_rule_internal *ri = to_intrule(rule);
2388	struct vcap_client_actionfield *caf, *next_caf;
2389	struct vcap_client_keyfield *ckf, *next_ckf;
2390
2391	/* Deallocate the list of keys and actions */
2392	list_for_each_entry_safe(ckf, next_ckf, &ri->data.keyfields, ctrl.list) {
2393	list_del(entry: &ckf->ctrl.list);
2394	kfree(objp: ckf);
2395	}
2396	list_for_each_entry_safe(caf, next_caf, &ri->data.actionfields, ctrl.list) {
2397	list_del(entry: &caf->ctrl.list);
2398	kfree(objp: caf);
2399	}
2400	/* Deallocate the rule */
2401	kfree(objp: rule);
2402	}
2403	EXPORT_SYMBOL_GPL(vcap_free_rule);
2404
2405	/* Decode a rule from the VCAP cache and return a copy */
2406	struct vcap_rule *vcap_decode_rule(struct vcap_rule_internal *elem)
2407	{
2408	struct vcap_rule_internal *ri;
2409	int err;
2410
2411	ri = vcap_dup_rule(ri: elem, full: elem->state == VCAP_RS_DISABLED);
2412	if (IS_ERR(ptr: ri))
2413	return ERR_CAST(ptr: ri);
2414
2415	if (ri->state == VCAP_RS_DISABLED)
2416	goto out;
2417
2418	err = vcap_read_rule(ri);
2419	if (err)
2420	return ERR_PTR(error: err);
2421
2422	err = vcap_decode_keyset(ri);
2423	if (err)
2424	return ERR_PTR(error: err);
2425
2426	err = vcap_decode_actionset(ri);
2427	if (err)
2428	return ERR_PTR(error: err);
2429
2430	out:
2431	return &ri->data;
2432	}
2433
2434	struct vcap_rule *vcap_get_rule(struct vcap_control *vctrl, u32 id)
2435	{
2436	struct vcap_rule_internal *elem;
2437	struct vcap_rule *rule;
2438	int err;
2439
2440	err = vcap_api_check(ctrl: vctrl);
2441	if (err)
2442	return ERR_PTR(error: err);
2443
2444	elem = vcap_get_locked_rule(vctrl, id);
2445	if (!elem)
2446	return ERR_PTR(error: -ENOENT);
2447
2448	rule = vcap_decode_rule(elem);
2449	mutex_unlock(lock: &elem->admin->lock);
2450	return rule;
2451	}
2452	EXPORT_SYMBOL_GPL(vcap_get_rule);
2453
2454	/* Update existing rule */
2455	int vcap_mod_rule(struct vcap_rule *rule)
2456	{
2457	struct vcap_rule_internal *ri = to_intrule(rule);
2458	struct vcap_counter ctr;
2459	int err;
2460
2461	err = vcap_api_check(ctrl: ri->vctrl);
2462	if (err)
2463	return err;
2464
2465	if (!vcap_get_locked_rule(vctrl: ri->vctrl, id: ri->data.id))
2466	return -ENOENT;
2467
2468	vcap_rule_set_state(ri);
2469	if (ri->state == VCAP_RS_DISABLED)
2470	goto out;
2471
2472	/* Encode the bitstreams to the VCAP cache */
2473	vcap_erase_cache(ri);
2474	err = vcap_encode_rule(ri);
2475	if (err)
2476	goto out;
2477
2478	err = vcap_write_rule(ri);
2479	if (err)
2480	goto out;
2481
2482	memset(&ctr, 0, sizeof(ctr));
2483	err = vcap_write_counter(ri, ctr: &ctr);
2484
2485	out:
2486	mutex_unlock(lock: &ri->admin->lock);
2487	return err;
2488	}
2489	EXPORT_SYMBOL_GPL(vcap_mod_rule);
2490
2491	/* Return the alignment offset for a new rule address */
2492	static int vcap_valid_rule_move(struct vcap_rule_internal *el, int offset)
2493	{
2494	return (el->addr + offset) % el->size;
2495	}
2496
2497	/* Update the rule address with an offset */
2498	static void vcap_adjust_rule_addr(struct vcap_rule_internal *el, int offset)
2499	{
2500	el->addr += offset;
2501	}
2502
2503	/* Rules needs to be moved to fill the gap of the deleted rule */
2504	static int vcap_fill_rule_gap(struct vcap_rule_internal *ri)
2505	{
2506	struct vcap_admin *admin = ri->admin;
2507	struct vcap_rule_internal *elem;
2508	struct vcap_rule_move move;
2509	int gap = 0, offset = 0;
2510
2511	/* If the first rule is deleted: Move other rules to the top */
2512	if (list_is_first(list: &ri->list, head: &admin->rules))
2513	offset = admin->last_valid_addr + 1 - ri->addr - ri->size;
2514
2515	/* Locate gaps between odd size rules and adjust the move */
2516	elem = ri;
2517	list_for_each_entry_continue(elem, &admin->rules, list)
2518	gap += vcap_valid_rule_move(el: elem, offset: ri->size);
2519
2520	/* Update the address in the remaining rules in the list */
2521	elem = ri;
2522	list_for_each_entry_continue(elem, &admin->rules, list)
2523	vcap_adjust_rule_addr(el: elem, offset: ri->size + gap + offset);
2524
2525	/* Update the move info */
2526	move.addr = admin->last_used_addr;
2527	move.count = ri->addr - admin->last_used_addr - gap;
2528	move.offset = -(ri->size + gap + offset);
2529
2530	/* Do the actual move operation */
2531	vcap_move_rules(ri, move: &move);
2532
2533	return gap + offset;
2534	}
2535
2536	/* Delete rule in a VCAP instance */
2537	int vcap_del_rule(struct vcap_control *vctrl, struct net_device *ndev, u32 id)
2538	{
2539	struct vcap_rule_internal *ri, *elem;
2540	struct vcap_admin *admin;
2541	int gap = 0, err;
2542
2543	/* This will later also handle rule moving */
2544	if (!ndev)
2545	return -ENODEV;
2546	err = vcap_api_check(ctrl: vctrl);
2547	if (err)
2548	return err;
2549	/* Look for the rule id in all vcaps */
2550	ri = vcap_get_locked_rule(vctrl, id);
2551	if (!ri)
2552	return -ENOENT;
2553
2554	admin = ri->admin;
2555
2556	if (ri->addr > admin->last_used_addr)
2557	gap = vcap_fill_rule_gap(ri);
2558
2559	/* Delete the rule from the list of rules and the cache */
2560	list_del(entry: &ri->list);
2561	vctrl->ops->init(ndev, admin, admin->last_used_addr, ri->size + gap);
2562	vcap_free_rule(&ri->data);
2563
2564	/* Update the last used address, set to default when no rules */
2565	if (list_empty(head: &admin->rules)) {
2566	admin->last_used_addr = admin->last_valid_addr + 1;
2567	} else {
2568	elem = list_last_entry(&admin->rules, struct vcap_rule_internal,
2569	list);
2570	admin->last_used_addr = elem->addr;
2571	}
2572
2573	mutex_unlock(lock: &admin->lock);
2574	return err;
2575	}
2576	EXPORT_SYMBOL_GPL(vcap_del_rule);
2577
2578	/* Delete all rules in the VCAP instance */
2579	int vcap_del_rules(struct vcap_control *vctrl, struct vcap_admin *admin)
2580	{
2581	struct vcap_enabled_port *eport, *next_eport;
2582	struct vcap_rule_internal *ri, *next_ri;
2583	int ret = vcap_api_check(ctrl: vctrl);
2584
2585	if (ret)
2586	return ret;
2587
2588	mutex_lock(&admin->lock);
2589	list_for_each_entry_safe(ri, next_ri, &admin->rules, list) {
2590	vctrl->ops->init(ri->ndev, admin, ri->addr, ri->size);
2591	list_del(entry: &ri->list);
2592	vcap_free_rule(&ri->data);
2593	}
2594	admin->last_used_addr = admin->last_valid_addr;
2595
2596	/* Remove list of enabled ports */
2597	list_for_each_entry_safe(eport, next_eport, &admin->enabled, list) {
2598	list_del(entry: &eport->list);
2599	kfree(objp: eport);
2600	}
2601	mutex_unlock(lock: &admin->lock);
2602
2603	return 0;
2604	}
2605	EXPORT_SYMBOL_GPL(vcap_del_rules);
2606
2607	/* Find a client key field in a rule */
2608	static struct vcap_client_keyfield *
2609	vcap_find_keyfield(struct vcap_rule *rule, enum vcap_key_field key)
2610	{
2611	struct vcap_rule_internal *ri = to_intrule(rule);
2612	struct vcap_client_keyfield *ckf;
2613
2614	list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list)
2615	if (ckf->ctrl.key == key)
2616	return ckf;
2617	return NULL;
2618	}
2619
2620	/* Find information on a key field in a rule */
2621	const struct vcap_field *vcap_lookup_keyfield(struct vcap_rule *rule,
2622	enum vcap_key_field key)
2623	{
2624	struct vcap_rule_internal *ri = to_intrule(rule);
2625	enum vcap_keyfield_set keyset = rule->keyset;
2626	enum vcap_type vt = ri->admin->vtype;
2627	const struct vcap_field *fields;
2628
2629	if (keyset == VCAP_KFS_NO_VALUE)
2630	return NULL;
2631	fields = vcap_keyfields(vctrl: ri->vctrl, vt, keyset);
2632	if (!fields)
2633	return NULL;
2634	return &fields[key];
2635	}
2636	EXPORT_SYMBOL_GPL(vcap_lookup_keyfield);
2637
2638	/* Check if the keyfield is already in the rule */
2639	static bool vcap_keyfield_unique(struct vcap_rule *rule,
2640	enum vcap_key_field key)
2641	{
2642	struct vcap_rule_internal *ri = to_intrule(rule);
2643	const struct vcap_client_keyfield *ckf;
2644
2645	list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list)
2646	if (ckf->ctrl.key == key)
2647	return false;
2648	return true;
2649	}
2650
2651	/* Check if the keyfield is in the keyset */
2652	static bool vcap_keyfield_match_keyset(struct vcap_rule *rule,
2653	enum vcap_key_field key)
2654	{
2655	struct vcap_rule_internal *ri = to_intrule(rule);
2656	enum vcap_keyfield_set keyset = rule->keyset;
2657	enum vcap_type vt = ri->admin->vtype;
2658	const struct vcap_field *fields;
2659
2660	/* the field is accepted if the rule has no keyset yet */
2661	if (keyset == VCAP_KFS_NO_VALUE)
2662	return true;
2663	fields = vcap_keyfields(vctrl: ri->vctrl, vt, keyset);
2664	if (!fields)
2665	return false;
2666	/* if there is a width there is a way */
2667	return fields[key].width > 0;
2668	}
2669
2670	static int vcap_rule_add_key(struct vcap_rule *rule,
2671	enum vcap_key_field key,
2672	enum vcap_field_type ftype,
2673	struct vcap_client_keyfield_data *data)
2674	{
2675	struct vcap_rule_internal *ri = to_intrule(rule);
2676	struct vcap_client_keyfield *field;
2677
2678	if (!vcap_keyfield_unique(rule, key)) {
2679	pr_warn("%s:%d: keyfield %s is already in the rule\n",
2680	__func__, __LINE__,
2681	vcap_keyfield_name(ri->vctrl, key));
2682	return -EINVAL;
2683	}
2684
2685	if (!vcap_keyfield_match_keyset(rule, key)) {
2686	pr_err("%s:%d: keyfield %s does not belong in the rule keyset\n",
2687	__func__, __LINE__,
2688	vcap_keyfield_name(ri->vctrl, key));
2689	return -EINVAL;
2690	}
2691
2692	field = kzalloc(size: sizeof(*field), GFP_KERNEL);
2693	if (!field)
2694	return -ENOMEM;
2695	memcpy(&field->data, data, sizeof(field->data));
2696	field->ctrl.key = key;
2697	field->ctrl.type = ftype;
2698	list_add_tail(new: &field->ctrl.list, head: &rule->keyfields);
2699	return 0;
2700	}
2701
2702	static void vcap_rule_set_key_bitsize(struct vcap_u1_key *u1, enum vcap_bit val)
2703	{
2704	switch (val) {
2705	case VCAP_BIT_0:
2706	u1->value = 0;
2707	u1->mask = 1;
2708	break;
2709	case VCAP_BIT_1:
2710	u1->value = 1;
2711	u1->mask = 1;
2712	break;
2713	case VCAP_BIT_ANY:
2714	u1->value = 0;
2715	u1->mask = 0;
2716	break;
2717	}
2718	}
2719
2720	/* Add a bit key with value and mask to the rule */
2721	int vcap_rule_add_key_bit(struct vcap_rule *rule, enum vcap_key_field key,
2722	enum vcap_bit val)
2723	{
2724	struct vcap_client_keyfield_data data;
2725
2726	vcap_rule_set_key_bitsize(u1: &data.u1, val);
2727	return vcap_rule_add_key(rule, key, ftype: VCAP_FIELD_BIT, data: &data);
2728	}
2729	EXPORT_SYMBOL_GPL(vcap_rule_add_key_bit);
2730
2731	/* Add a 32 bit key field with value and mask to the rule */
2732	int vcap_rule_add_key_u32(struct vcap_rule *rule, enum vcap_key_field key,
2733	u32 value, u32 mask)
2734	{
2735	struct vcap_client_keyfield_data data;
2736
2737	data.u32.value = value;
2738	data.u32.mask = mask;
2739	return vcap_rule_add_key(rule, key, ftype: VCAP_FIELD_U32, data: &data);
2740	}
2741	EXPORT_SYMBOL_GPL(vcap_rule_add_key_u32);
2742
2743	/* Add a 48 bit key with value and mask to the rule */
2744	int vcap_rule_add_key_u48(struct vcap_rule *rule, enum vcap_key_field key,
2745	struct vcap_u48_key *fieldval)
2746	{
2747	struct vcap_client_keyfield_data data;
2748
2749	memcpy(&data.u48, fieldval, sizeof(data.u48));
2750	return vcap_rule_add_key(rule, key, ftype: VCAP_FIELD_U48, data: &data);
2751	}
2752	EXPORT_SYMBOL_GPL(vcap_rule_add_key_u48);
2753
2754	/* Add a 72 bit key with value and mask to the rule */
2755	int vcap_rule_add_key_u72(struct vcap_rule *rule, enum vcap_key_field key,
2756	struct vcap_u72_key *fieldval)
2757	{
2758	struct vcap_client_keyfield_data data;
2759
2760	memcpy(&data.u72, fieldval, sizeof(data.u72));
2761	return vcap_rule_add_key(rule, key, ftype: VCAP_FIELD_U72, data: &data);
2762	}
2763	EXPORT_SYMBOL_GPL(vcap_rule_add_key_u72);
2764
2765	/* Add a 128 bit key with value and mask to the rule */
2766	int vcap_rule_add_key_u128(struct vcap_rule *rule, enum vcap_key_field key,
2767	struct vcap_u128_key *fieldval)
2768	{
2769	struct vcap_client_keyfield_data data;
2770
2771	memcpy(&data.u128, fieldval, sizeof(data.u128));
2772	return vcap_rule_add_key(rule, key, ftype: VCAP_FIELD_U128, data: &data);
2773	}
2774	EXPORT_SYMBOL_GPL(vcap_rule_add_key_u128);
2775
2776	int vcap_rule_get_key_u32(struct vcap_rule *rule, enum vcap_key_field key,
2777	u32 *value, u32 *mask)
2778	{
2779	struct vcap_client_keyfield *ckf;
2780
2781	ckf = vcap_find_keyfield(rule, key);
2782	if (!ckf)
2783	return -ENOENT;
2784
2785	*value = ckf->data.u32.value;
2786	*mask = ckf->data.u32.mask;
2787
2788	return 0;
2789	}
2790	EXPORT_SYMBOL_GPL(vcap_rule_get_key_u32);
2791
2792	/* Find a client action field in a rule */
2793	struct vcap_client_actionfield *
2794	vcap_find_actionfield(struct vcap_rule *rule, enum vcap_action_field act)
2795	{
2796	struct vcap_rule_internal *ri = (struct vcap_rule_internal *)rule;
2797	struct vcap_client_actionfield *caf;
2798
2799	list_for_each_entry(caf, &ri->data.actionfields, ctrl.list)
2800	if (caf->ctrl.action == act)
2801	return caf;
2802	return NULL;
2803	}
2804	EXPORT_SYMBOL_GPL(vcap_find_actionfield);
2805
2806	/* Check if the actionfield is already in the rule */
2807	static bool vcap_actionfield_unique(struct vcap_rule *rule,
2808	enum vcap_action_field act)
2809	{
2810	struct vcap_rule_internal *ri = to_intrule(rule);
2811	const struct vcap_client_actionfield *caf;
2812
2813	list_for_each_entry(caf, &ri->data.actionfields, ctrl.list)
2814	if (caf->ctrl.action == act)
2815	return false;
2816	return true;
2817	}
2818
2819	/* Check if the actionfield is in the actionset */
2820	static bool vcap_actionfield_match_actionset(struct vcap_rule *rule,
2821	enum vcap_action_field action)
2822	{
2823	enum vcap_actionfield_set actionset = rule->actionset;
2824	struct vcap_rule_internal *ri = to_intrule(rule);
2825	enum vcap_type vt = ri->admin->vtype;
2826	const struct vcap_field *fields;
2827
2828	/* the field is accepted if the rule has no actionset yet */
2829	if (actionset == VCAP_AFS_NO_VALUE)
2830	return true;
2831	fields = vcap_actionfields(vctrl: ri->vctrl, vt, actionset);
2832	if (!fields)
2833	return false;
2834	/* if there is a width there is a way */
2835	return fields[action].width > 0;
2836	}
2837
2838	static int vcap_rule_add_action(struct vcap_rule *rule,
2839	enum vcap_action_field action,
2840	enum vcap_field_type ftype,
2841	struct vcap_client_actionfield_data *data)
2842	{
2843	struct vcap_rule_internal *ri = to_intrule(rule);
2844	struct vcap_client_actionfield *field;
2845
2846	if (!vcap_actionfield_unique(rule, act: action)) {
2847	pr_warn("%s:%d: actionfield %s is already in the rule\n",
2848	__func__, __LINE__,
2849	vcap_actionfield_name(ri->vctrl, action));
2850	return -EINVAL;
2851	}
2852
2853	if (!vcap_actionfield_match_actionset(rule, action)) {
2854	pr_err("%s:%d: actionfield %s does not belong in the rule actionset\n",
2855	__func__, __LINE__,
2856	vcap_actionfield_name(ri->vctrl, action));
2857	return -EINVAL;
2858	}
2859
2860	field = kzalloc(size: sizeof(*field), GFP_KERNEL);
2861	if (!field)
2862	return -ENOMEM;
2863	memcpy(&field->data, data, sizeof(field->data));
2864	field->ctrl.action = action;
2865	field->ctrl.type = ftype;
2866	list_add_tail(new: &field->ctrl.list, head: &rule->actionfields);
2867	return 0;
2868	}
2869
2870	static void vcap_rule_set_action_bitsize(struct vcap_u1_action *u1,
2871	enum vcap_bit val)
2872	{
2873	switch (val) {
2874	case VCAP_BIT_0:
2875	u1->value = 0;
2876	break;
2877	case VCAP_BIT_1:
2878	u1->value = 1;
2879	break;
2880	case VCAP_BIT_ANY:
2881	u1->value = 0;
2882	break;
2883	}
2884	}
2885
2886	/* Add a bit action with value to the rule */
2887	int vcap_rule_add_action_bit(struct vcap_rule *rule,
2888	enum vcap_action_field action,
2889	enum vcap_bit val)
2890	{
2891	struct vcap_client_actionfield_data data;
2892
2893	vcap_rule_set_action_bitsize(u1: &data.u1, val);
2894	return vcap_rule_add_action(rule, action, ftype: VCAP_FIELD_BIT, data: &data);
2895	}
2896	EXPORT_SYMBOL_GPL(vcap_rule_add_action_bit);
2897
2898	/* Add a 32 bit action field with value to the rule */
2899	int vcap_rule_add_action_u32(struct vcap_rule *rule,
2900	enum vcap_action_field action,
2901	u32 value)
2902	{
2903	struct vcap_client_actionfield_data data;
2904
2905	data.u32.value = value;
2906	return vcap_rule_add_action(rule, action, ftype: VCAP_FIELD_U32, data: &data);
2907	}
2908	EXPORT_SYMBOL_GPL(vcap_rule_add_action_u32);
2909
2910	static int vcap_read_counter(struct vcap_rule_internal *ri,
2911	struct vcap_counter *ctr)
2912	{
2913	struct vcap_admin *admin = ri->admin;
2914
2915	ri->vctrl->ops->update(ri->ndev, admin, VCAP_CMD_READ, VCAP_SEL_COUNTER,
2916	ri->addr);
2917	ri->vctrl->ops->cache_read(ri->ndev, admin, VCAP_SEL_COUNTER,
2918	ri->counter_id, 0);
2919	ctr->value = admin->cache.counter;
2920	ctr->sticky = admin->cache.sticky;
2921	return 0;
2922	}
2923
2924	/* Copy to host byte order */
2925	void vcap_netbytes_copy(u8 *dst, u8 *src, int count)
2926	{
2927	int idx;
2928
2929	for (idx = 0; idx < count; ++idx, ++dst)
2930	*dst = src[count - idx - 1];
2931	}
2932	EXPORT_SYMBOL_GPL(vcap_netbytes_copy);
2933
2934	/* Convert validation error code into tc extact error message */
2935	void vcap_set_tc_exterr(struct flow_cls_offload *fco, struct vcap_rule *vrule)
2936	{
2937	switch (vrule->exterr) {
2938	case VCAP_ERR_NONE:
2939	break;
2940	case VCAP_ERR_NO_ADMIN:
2941	NL_SET_ERR_MSG_MOD(fco->common.extack,
2942	"Missing VCAP instance");
2943	break;
2944	case VCAP_ERR_NO_NETDEV:
2945	NL_SET_ERR_MSG_MOD(fco->common.extack,
2946	"Missing network interface");
2947	break;
2948	case VCAP_ERR_NO_KEYSET_MATCH:
2949	NL_SET_ERR_MSG_MOD(fco->common.extack,
2950	"No keyset matched the filter keys");
2951	break;
2952	case VCAP_ERR_NO_ACTIONSET_MATCH:
2953	NL_SET_ERR_MSG_MOD(fco->common.extack,
2954	"No actionset matched the filter actions");
2955	break;
2956	case VCAP_ERR_NO_PORT_KEYSET_MATCH:
2957	NL_SET_ERR_MSG_MOD(fco->common.extack,
2958	"No port keyset matched the filter keys");
2959	break;
2960	}
2961	}
2962	EXPORT_SYMBOL_GPL(vcap_set_tc_exterr);
2963
2964	/* Write a rule to VCAP HW to enable it */
2965	static int vcap_enable_rule(struct vcap_rule_internal *ri)
2966	{
2967	struct vcap_client_actionfield *af, *naf;
2968	struct vcap_client_keyfield *kf, *nkf;
2969	int err;
2970
2971	vcap_erase_cache(ri);
2972	err = vcap_encode_rule(ri);
2973	if (err)
2974	goto out;
2975	err = vcap_write_rule(ri);
2976	if (err)
2977	goto out;
2978
2979	/* Deallocate the list of keys and actions */
2980	list_for_each_entry_safe(kf, nkf, &ri->data.keyfields, ctrl.list) {
2981	list_del(entry: &kf->ctrl.list);
2982	kfree(objp: kf);
2983	}
2984	list_for_each_entry_safe(af, naf, &ri->data.actionfields, ctrl.list) {
2985	list_del(entry: &af->ctrl.list);
2986	kfree(objp: af);
2987	}
2988	ri->state = VCAP_RS_ENABLED;
2989	out:
2990	return err;
2991	}
2992
2993	/* Enable all disabled rules for a specific chain/port in the VCAP HW */
2994	static int vcap_enable_rules(struct vcap_control *vctrl,
2995	struct net_device *ndev, int chain)
2996	{
2997	int next_chain = chain + VCAP_CID_LOOKUP_SIZE;
2998	struct vcap_rule_internal *ri;
2999	struct vcap_admin *admin;
3000	int err = 0;
3001
3002	list_for_each_entry(admin, &vctrl->list, list) {
3003	if (!(chain >= admin->first_cid && chain <= admin->last_cid))
3004	continue;
3005
3006	/* Found the admin, now find the offloadable rules */
3007	mutex_lock(&admin->lock);
3008	list_for_each_entry(ri, &admin->rules, list) {
3009	/* Is the rule in the lookup defined by the chain */
3010	if (!(ri->data.vcap_chain_id >= chain &&
3011	ri->data.vcap_chain_id < next_chain)) {
3012	continue;
3013	}
3014
3015	if (ri->ndev != ndev)
3016	continue;
3017
3018	if (ri->state != VCAP_RS_DISABLED)
3019	continue;
3020
3021	err = vcap_enable_rule(ri);
3022	if (err)
3023	break;
3024	}
3025	mutex_unlock(lock: &admin->lock);
3026	if (err)
3027	break;
3028	}
3029	return err;
3030	}
3031
3032	/* Read and erase a rule from VCAP HW to disable it */
3033	static int vcap_disable_rule(struct vcap_rule_internal *ri)
3034	{
3035	int err;
3036
3037	err = vcap_read_rule(ri);
3038	if (err)
3039	return err;
3040	err = vcap_decode_keyset(ri);
3041	if (err)
3042	return err;
3043	err = vcap_decode_actionset(ri);
3044	if (err)
3045	return err;
3046
3047	ri->state = VCAP_RS_DISABLED;
3048	ri->vctrl->ops->init(ri->ndev, ri->admin, ri->addr, ri->size);
3049	return 0;
3050	}
3051
3052	/* Disable all enabled rules for a specific chain/port in the VCAP HW */
3053	static int vcap_disable_rules(struct vcap_control *vctrl,
3054	struct net_device *ndev, int chain)
3055	{
3056	struct vcap_rule_internal *ri;
3057	struct vcap_admin *admin;
3058	int err = 0;
3059
3060	list_for_each_entry(admin, &vctrl->list, list) {
3061	if (!(chain >= admin->first_cid && chain <= admin->last_cid))
3062	continue;
3063
3064	/* Found the admin, now find the rules on the chain */
3065	mutex_lock(&admin->lock);
3066	list_for_each_entry(ri, &admin->rules, list) {
3067	if (ri->data.vcap_chain_id != chain)
3068	continue;
3069
3070	if (ri->ndev != ndev)
3071	continue;
3072
3073	if (ri->state != VCAP_RS_ENABLED)
3074	continue;
3075
3076	err = vcap_disable_rule(ri);
3077	if (err)
3078	break;
3079	}
3080	mutex_unlock(lock: &admin->lock);
3081	if (err)
3082	break;
3083	}
3084	return err;
3085	}
3086
3087	/* Check if this port is already enabled for this VCAP instance */
3088	static bool vcap_is_enabled(struct vcap_control *vctrl, struct net_device *ndev,
3089	int dst_cid)
3090	{
3091	struct vcap_enabled_port *eport;
3092	struct vcap_admin *admin;
3093
3094	list_for_each_entry(admin, &vctrl->list, list)
3095	list_for_each_entry(eport, &admin->enabled, list)
3096	if (eport->dst_cid == dst_cid && eport->ndev == ndev)
3097	return true;
3098
3099	return false;
3100	}
3101
3102	/* Enable this port and chain id in a VCAP instance */
3103	static int vcap_enable(struct vcap_control *vctrl, struct net_device *ndev,
3104	unsigned long cookie, int src_cid, int dst_cid)
3105	{
3106	struct vcap_enabled_port *eport;
3107	struct vcap_admin *admin;
3108
3109	if (src_cid >= dst_cid)
3110	return -EFAULT;
3111
3112	admin = vcap_find_admin(vctrl, dst_cid);
3113	if (!admin)
3114	return -ENOENT;
3115
3116	eport = kzalloc(size: sizeof(*eport), GFP_KERNEL);
3117	if (!eport)
3118	return -ENOMEM;
3119
3120	eport->ndev = ndev;
3121	eport->cookie = cookie;
3122	eport->src_cid = src_cid;
3123	eport->dst_cid = dst_cid;
3124	mutex_lock(&admin->lock);
3125	list_add_tail(new: &eport->list, head: &admin->enabled);
3126	mutex_unlock(lock: &admin->lock);
3127
3128	if (vcap_path_exist(vctrl, ndev, dst_cid: src_cid)) {
3129	/* Enable chained lookups */
3130	while (dst_cid) {
3131	admin = vcap_find_admin(vctrl, dst_cid);
3132	if (!admin)
3133	return -ENOENT;
3134
3135	vcap_enable_rules(vctrl, ndev, chain: dst_cid);
3136	dst_cid = vcap_get_next_chain(vctrl, ndev, dst_cid);
3137	}
3138	}
3139	return 0;
3140	}
3141
3142	/* Disable this port and chain id for a VCAP instance */
3143	static int vcap_disable(struct vcap_control *vctrl, struct net_device *ndev,
3144	unsigned long cookie)
3145	{
3146	struct vcap_enabled_port *elem, *eport = NULL;
3147	struct vcap_admin *found = NULL, *admin;
3148	int dst_cid;
3149
3150	list_for_each_entry(admin, &vctrl->list, list) {
3151	list_for_each_entry(elem, &admin->enabled, list) {
3152	if (elem->cookie == cookie && elem->ndev == ndev) {
3153	eport = elem;
3154	found = admin;
3155	break;
3156	}
3157	}
3158	if (eport)
3159	break;
3160	}
3161
3162	if (!eport)
3163	return -ENOENT;
3164
3165	/* Disable chained lookups */
3166	dst_cid = eport->dst_cid;
3167	while (dst_cid) {
3168	admin = vcap_find_admin(vctrl, dst_cid);
3169	if (!admin)
3170	return -ENOENT;
3171
3172	vcap_disable_rules(vctrl, ndev, chain: dst_cid);
3173	dst_cid = vcap_get_next_chain(vctrl, ndev, dst_cid);
3174	}
3175
3176	mutex_lock(&found->lock);
3177	list_del(entry: &eport->list);
3178	mutex_unlock(lock: &found->lock);
3179	kfree(objp: eport);
3180	return 0;
3181	}
3182
3183	/* Enable/Disable the VCAP instance lookups */
3184	int vcap_enable_lookups(struct vcap_control *vctrl, struct net_device *ndev,
3185	int src_cid, int dst_cid, unsigned long cookie,
3186	bool enable)
3187	{
3188	int err;
3189
3190	err = vcap_api_check(ctrl: vctrl);
3191	if (err)
3192	return err;
3193
3194	if (!ndev)
3195	return -ENODEV;
3196
3197	/* Source and destination must be the first chain in a lookup */
3198	if (src_cid % VCAP_CID_LOOKUP_SIZE)
3199	return -EFAULT;
3200	if (dst_cid % VCAP_CID_LOOKUP_SIZE)
3201	return -EFAULT;
3202
3203	if (enable) {
3204	if (vcap_is_enabled(vctrl, ndev, dst_cid))
3205	return -EADDRINUSE;
3206	if (vcap_is_chain_used(vctrl, ndev, src_cid))
3207	return -EADDRNOTAVAIL;
3208	err = vcap_enable(vctrl, ndev, cookie, src_cid, dst_cid);
3209	} else {
3210	err = vcap_disable(vctrl, ndev, cookie);
3211	}
3212
3213	return err;
3214	}
3215	EXPORT_SYMBOL_GPL(vcap_enable_lookups);
3216
3217	/* Is this chain id the last lookup of all VCAPs */
3218	bool vcap_is_last_chain(struct vcap_control *vctrl, int cid, bool ingress)
3219	{
3220	struct vcap_admin *admin;
3221	int lookup;
3222
3223	if (vcap_api_check(ctrl: vctrl))
3224	return false;
3225
3226	admin = vcap_find_admin(vctrl, cid);
3227	if (!admin)
3228	return false;
3229
3230	if (!vcap_admin_is_last(vctrl, admin, ingress))
3231	return false;
3232
3233	/* This must be the last lookup in this VCAP type */
3234	lookup = vcap_chain_id_to_lookup(admin, cid);
3235	return lookup == admin->lookups - 1;
3236	}
3237	EXPORT_SYMBOL_GPL(vcap_is_last_chain);
3238
3239	/* Set a rule counter id (for certain vcaps only) */
3240	void vcap_rule_set_counter_id(struct vcap_rule *rule, u32 counter_id)
3241	{
3242	struct vcap_rule_internal *ri = to_intrule(rule);
3243
3244	ri->counter_id = counter_id;
3245	}
3246	EXPORT_SYMBOL_GPL(vcap_rule_set_counter_id);
3247
3248	int vcap_rule_set_counter(struct vcap_rule *rule, struct vcap_counter *ctr)
3249	{
3250	struct vcap_rule_internal *ri = to_intrule(rule);
3251	int err;
3252
3253	err = vcap_api_check(ctrl: ri->vctrl);
3254	if (err)
3255	return err;
3256	if (!ctr) {
3257	pr_err("%s:%d: counter is missing\n", __func__, __LINE__);
3258	return -EINVAL;
3259	}
3260
3261	mutex_lock(&ri->admin->lock);
3262	err = vcap_write_counter(ri, ctr);
3263	mutex_unlock(lock: &ri->admin->lock);
3264
3265	return err;
3266	}
3267	EXPORT_SYMBOL_GPL(vcap_rule_set_counter);
3268
3269	int vcap_rule_get_counter(struct vcap_rule *rule, struct vcap_counter *ctr)
3270	{
3271	struct vcap_rule_internal *ri = to_intrule(rule);
3272	int err;
3273
3274	err = vcap_api_check(ctrl: ri->vctrl);
3275	if (err)
3276	return err;
3277	if (!ctr) {
3278	pr_err("%s:%d: counter is missing\n", __func__, __LINE__);
3279	return -EINVAL;
3280	}
3281
3282	mutex_lock(&ri->admin->lock);
3283	err = vcap_read_counter(ri, ctr);
3284	mutex_unlock(lock: &ri->admin->lock);
3285
3286	return err;
3287	}
3288	EXPORT_SYMBOL_GPL(vcap_rule_get_counter);
3289
3290	/* Get a copy of a client key field */
3291	static int vcap_rule_get_key(struct vcap_rule *rule,
3292	enum vcap_key_field key,
3293	struct vcap_client_keyfield *ckf)
3294	{
3295	struct vcap_client_keyfield *field;
3296
3297	field = vcap_find_keyfield(rule, key);
3298	if (!field)
3299	return -EINVAL;
3300	memcpy(ckf, field, sizeof(*ckf));
3301	INIT_LIST_HEAD(list: &ckf->ctrl.list);
3302	return 0;
3303	}
3304
3305	/* Find a keyset having the same size as the provided rule, where the keyset
3306	* does not have a type id.
3307	*/
3308	static int vcap_rule_get_untyped_keyset(struct vcap_rule_internal *ri,
3309	struct vcap_keyset_list *matches)
3310	{
3311	struct vcap_control *vctrl = ri->vctrl;
3312	enum vcap_type vt = ri->admin->vtype;
3313	const struct vcap_set *keyfield_set;
3314	int idx;
3315
3316	keyfield_set = vctrl->vcaps[vt].keyfield_set;
3317	for (idx = 0; idx < vctrl->vcaps[vt].keyfield_set_size; ++idx) {
3318	if (keyfield_set[idx].sw_per_item == ri->keyset_sw &&
3319	keyfield_set[idx].type_id == (u8)-1) {
3320	vcap_keyset_list_add(matches, idx);
3321	return 0;
3322	}
3323	}
3324	return -EINVAL;
3325	}
3326
3327	/* Get the keysets that matches the rule key type/mask */
3328	int vcap_rule_get_keysets(struct vcap_rule_internal *ri,
3329	struct vcap_keyset_list *matches)
3330	{
3331	struct vcap_control *vctrl = ri->vctrl;
3332	enum vcap_type vt = ri->admin->vtype;
3333	const struct vcap_set *keyfield_set;
3334	struct vcap_client_keyfield kf = {};
3335	u32 value, mask;
3336	int err, idx;
3337
3338	err = vcap_rule_get_key(rule: &ri->data, key: VCAP_KF_TYPE, ckf: &kf);
3339	if (err)
3340	return vcap_rule_get_untyped_keyset(ri, matches);
3341
3342	if (kf.ctrl.type == VCAP_FIELD_BIT) {
3343	value = kf.data.u1.value;
3344	mask = kf.data.u1.mask;
3345	} else if (kf.ctrl.type == VCAP_FIELD_U32) {
3346	value = kf.data.u32.value;
3347	mask = kf.data.u32.mask;
3348	} else {
3349	return -EINVAL;
3350	}
3351
3352	keyfield_set = vctrl->vcaps[vt].keyfield_set;
3353	for (idx = 0; idx < vctrl->vcaps[vt].keyfield_set_size; ++idx) {
3354	if (keyfield_set[idx].sw_per_item != ri->keyset_sw)
3355	continue;
3356
3357	if (keyfield_set[idx].type_id == (u8)-1) {
3358	vcap_keyset_list_add(matches, idx);
3359	continue;
3360	}
3361
3362	if ((keyfield_set[idx].type_id & mask) == value)
3363	vcap_keyset_list_add(matches, idx);
3364	}
3365	if (matches->cnt > 0)
3366	return 0;
3367
3368	return -EINVAL;
3369	}
3370
3371	/* Collect packet counts from all rules with the same cookie */
3372	int vcap_get_rule_count_by_cookie(struct vcap_control *vctrl,
3373	struct vcap_counter *ctr, u64 cookie)
3374	{
3375	struct vcap_rule_internal *ri;
3376	struct vcap_counter temp = {};
3377	struct vcap_admin *admin;
3378	int err;
3379
3380	err = vcap_api_check(ctrl: vctrl);
3381	if (err)
3382	return err;
3383
3384	/* Iterate all rules in each VCAP instance */
3385	list_for_each_entry(admin, &vctrl->list, list) {
3386	mutex_lock(&admin->lock);
3387	list_for_each_entry(ri, &admin->rules, list) {
3388	if (ri->data.cookie != cookie)
3389	continue;
3390
3391	err = vcap_read_counter(ri, ctr: &temp);
3392	if (err)
3393	goto unlock;
3394	ctr->value += temp.value;
3395
3396	/* Reset the rule counter */
3397	temp.value = 0;
3398	temp.sticky = 0;
3399	err = vcap_write_counter(ri, ctr: &temp);
3400	if (err)
3401	goto unlock;
3402	}
3403	mutex_unlock(lock: &admin->lock);
3404	}
3405	return err;
3406
3407	unlock:
3408	mutex_unlock(lock: &admin->lock);
3409	return err;
3410	}
3411	EXPORT_SYMBOL_GPL(vcap_get_rule_count_by_cookie);
3412
3413	static int vcap_rule_mod_key(struct vcap_rule *rule,
3414	enum vcap_key_field key,
3415	enum vcap_field_type ftype,
3416	struct vcap_client_keyfield_data *data)
3417	{
3418	struct vcap_client_keyfield *field;
3419
3420	field = vcap_find_keyfield(rule, key);
3421	if (!field)
3422	return vcap_rule_add_key(rule, key, ftype, data);
3423	memcpy(&field->data, data, sizeof(field->data));
3424	return 0;
3425	}
3426
3427	/* Modify a 32 bit key field with value and mask in the rule */
3428	int vcap_rule_mod_key_u32(struct vcap_rule *rule, enum vcap_key_field key,
3429	u32 value, u32 mask)
3430	{
3431	struct vcap_client_keyfield_data data;
3432
3433	data.u32.value = value;
3434	data.u32.mask = mask;
3435	return vcap_rule_mod_key(rule, key, ftype: VCAP_FIELD_U32, data: &data);
3436	}
3437	EXPORT_SYMBOL_GPL(vcap_rule_mod_key_u32);
3438
3439	/* Remove a key field with value and mask in the rule */
3440	int vcap_rule_rem_key(struct vcap_rule *rule, enum vcap_key_field key)
3441	{
3442	struct vcap_rule_internal *ri = to_intrule(rule);
3443	struct vcap_client_keyfield *field;
3444
3445	field = vcap_find_keyfield(rule, key);
3446	if (!field) {
3447	pr_err("%s:%d: key %s is not in the rule\n",
3448	__func__, __LINE__, vcap_keyfield_name(ri->vctrl, key));
3449	return -EINVAL;
3450	}
3451	/* Deallocate the key field */
3452	list_del(entry: &field->ctrl.list);
3453	kfree(objp: field);
3454	return 0;
3455	}
3456	EXPORT_SYMBOL_GPL(vcap_rule_rem_key);
3457
3458	static int vcap_rule_mod_action(struct vcap_rule *rule,
3459	enum vcap_action_field action,
3460	enum vcap_field_type ftype,
3461	struct vcap_client_actionfield_data *data)
3462	{
3463	struct vcap_client_actionfield *field;
3464
3465	field = vcap_find_actionfield(rule, action);
3466	if (!field)
3467	return vcap_rule_add_action(rule, action, ftype, data);
3468	memcpy(&field->data, data, sizeof(field->data));
3469	return 0;
3470	}
3471
3472	/* Modify a 32 bit action field with value in the rule */
3473	int vcap_rule_mod_action_u32(struct vcap_rule *rule,
3474	enum vcap_action_field action,
3475	u32 value)
3476	{
3477	struct vcap_client_actionfield_data data;
3478
3479	data.u32.value = value;
3480	return vcap_rule_mod_action(rule, action, ftype: VCAP_FIELD_U32, data: &data);
3481	}
3482	EXPORT_SYMBOL_GPL(vcap_rule_mod_action_u32);
3483
3484	/* Drop keys in a keylist and any keys that are not supported by the keyset */
3485	int vcap_filter_rule_keys(struct vcap_rule *rule,
3486	enum vcap_key_field keylist[], int length,
3487	bool drop_unsupported)
3488	{
3489	struct vcap_rule_internal *ri = to_intrule(rule);
3490	struct vcap_client_keyfield *ckf, *next_ckf;
3491	const struct vcap_field *fields;
3492	enum vcap_key_field key;
3493	int err = 0;
3494	int idx;
3495
3496	if (length > 0) {
3497	err = -EEXIST;
3498	list_for_each_entry_safe(ckf, next_ckf,
3499	&ri->data.keyfields, ctrl.list) {
3500	key = ckf->ctrl.key;
3501	for (idx = 0; idx < length; ++idx)
3502	if (key == keylist[idx]) {
3503	list_del(entry: &ckf->ctrl.list);
3504	kfree(objp: ckf);
3505	idx++;
3506	err = 0;
3507	}
3508	}
3509	}
3510	if (drop_unsupported) {
3511	err = -EEXIST;
3512	fields = vcap_keyfields(vctrl: ri->vctrl, vt: ri->admin->vtype,
3513	keyset: rule->keyset);
3514	if (!fields)
3515	return err;
3516	list_for_each_entry_safe(ckf, next_ckf,
3517	&ri->data.keyfields, ctrl.list) {
3518	key = ckf->ctrl.key;
3519	if (fields[key].width == 0) {
3520	list_del(entry: &ckf->ctrl.list);
3521	kfree(objp: ckf);
3522	err = 0;
3523	}
3524	}
3525	}
3526	return err;
3527	}
3528	EXPORT_SYMBOL_GPL(vcap_filter_rule_keys);
3529
3530	/* Select the keyset from the list that results in the smallest rule size */
3531	enum vcap_keyfield_set
3532	vcap_select_min_rule_keyset(struct vcap_control *vctrl,
3533	enum vcap_type vtype,
3534	struct vcap_keyset_list *kslist)
3535	{
3536	enum vcap_keyfield_set ret = VCAP_KFS_NO_VALUE;
3537	const struct vcap_set *kset;
3538	int max = 100, idx;
3539
3540	for (idx = 0; idx < kslist->cnt; ++idx) {
3541	kset = vcap_keyfieldset(vctrl, vtype, kslist->keysets[idx]);
3542	if (!kset)
3543	continue;
3544	if (kset->sw_per_item >= max)
3545	continue;
3546	max = kset->sw_per_item;
3547	ret = kslist->keysets[idx];
3548	}
3549	return ret;
3550	}
3551	EXPORT_SYMBOL_GPL(vcap_select_min_rule_keyset);
3552
3553	/* Make a full copy of an existing rule with a new rule id */
3554	struct vcap_rule *vcap_copy_rule(struct vcap_rule *erule)
3555	{
3556	struct vcap_rule_internal *ri = to_intrule(erule);
3557	struct vcap_client_actionfield *caf;
3558	struct vcap_client_keyfield *ckf;
3559	struct vcap_rule *rule;
3560	int err;
3561
3562	err = vcap_api_check(ctrl: ri->vctrl);
3563	if (err)
3564	return ERR_PTR(error: err);
3565
3566	rule = vcap_alloc_rule(ri->vctrl, ri->ndev, ri->data.vcap_chain_id,
3567	ri->data.user, ri->data.priority, 0);
3568	if (IS_ERR(ptr: rule))
3569	return rule;
3570
3571	list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list) {
3572	/* Add a key duplicate in the new rule */
3573	err = vcap_rule_add_key(rule,
3574	key: ckf->ctrl.key,
3575	ftype: ckf->ctrl.type,
3576	data: &ckf->data);
3577	if (err)
3578	goto err;
3579	}
3580
3581	list_for_each_entry(caf, &ri->data.actionfields, ctrl.list) {
3582	/* Add a action duplicate in the new rule */
3583	err = vcap_rule_add_action(rule,
3584	action: caf->ctrl.action,
3585	ftype: caf->ctrl.type,
3586	data: &caf->data);
3587	if (err)
3588	goto err;
3589	}
3590	return rule;
3591	err:
3592	vcap_free_rule(rule);
3593	return ERR_PTR(error: err);
3594	}
3595	EXPORT_SYMBOL_GPL(vcap_copy_rule);
3596
3597	#ifdef CONFIG_VCAP_KUNIT_TEST
3598	#include "vcap_api_kunit.c"
3599	#endif
3600