1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* net/sched/cls_api.c Packet classifier API.
4	*
5	* Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
6	*
7	* Changes:
8	*
9	* Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
10	*/
11
12	#include <linux/module.h>
13	#include <linux/types.h>
14	#include <linux/kernel.h>
15	#include <linux/string.h>
16	#include <linux/errno.h>
17	#include <linux/err.h>
18	#include <linux/skbuff.h>
19	#include <linux/init.h>
20	#include <linux/kmod.h>
21	#include <linux/slab.h>
22	#include <linux/idr.h>
23	#include <linux/jhash.h>
24	#include <linux/rculist.h>
25	#include <linux/rhashtable.h>
26	#include <net/net_namespace.h>
27	#include <net/sock.h>
28	#include <net/netlink.h>
29	#include <net/pkt_sched.h>
30	#include <net/pkt_cls.h>
31	#include <net/tc_act/tc_pedit.h>
32	#include <net/tc_act/tc_mirred.h>
33	#include <net/tc_act/tc_vlan.h>
34	#include <net/tc_act/tc_tunnel_key.h>
35	#include <net/tc_act/tc_csum.h>
36	#include <net/tc_act/tc_gact.h>
37	#include <net/tc_act/tc_police.h>
38	#include <net/tc_act/tc_sample.h>
39	#include <net/tc_act/tc_skbedit.h>
40	#include <net/tc_act/tc_ct.h>
41	#include <net/tc_act/tc_mpls.h>
42	#include <net/tc_act/tc_gate.h>
43	#include <net/flow_offload.h>
44	#include <net/tc_wrapper.h>
45
46	/* The list of all installed classifier types */
47	static LIST_HEAD(tcf_proto_base);
48
49	/* Protects list of registered TC modules. It is pure SMP lock. */
50	static DEFINE_RWLOCK(cls_mod_lock);
51
52	static struct xarray tcf_exts_miss_cookies_xa;
53	struct tcf_exts_miss_cookie_node {
54	const struct tcf_chain *chain;
55	const struct tcf_proto *tp;
56	const struct tcf_exts *exts;
57	u32 chain_index;
58	u32 tp_prio;
59	u32 handle;
60	u32 miss_cookie_base;
61	struct rcu_head rcu;
62	};
63
64	/* Each tc action entry cookie will be comprised of 32bit miss_cookie_base +
65	* action index in the exts tc actions array.
66	*/
67	union tcf_exts_miss_cookie {
68	struct {
69	u32 miss_cookie_base;
70	u32 act_index;
71	};
72	u64 miss_cookie;
73	};
74
75	#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
76	static int
77	tcf_exts_miss_cookie_base_alloc(struct tcf_exts *exts, struct tcf_proto *tp,
78	u32 handle)
79	{
80	struct tcf_exts_miss_cookie_node *n;
81	static u32 next;
82	int err;
83
84	if (WARN_ON(!handle || !tp->ops->get_exts))
85	return -EINVAL;
86
87	n = kzalloc(size: sizeof(*n), GFP_KERNEL);
88	if (!n)
89	return -ENOMEM;
90
91	n->chain_index = tp->chain->index;
92	n->chain = tp->chain;
93	n->tp_prio = tp->prio;
94	n->tp = tp;
95	n->exts = exts;
96	n->handle = handle;
97
98	err = xa_alloc_cyclic(xa: &tcf_exts_miss_cookies_xa, id: &n->miss_cookie_base,
99	entry: n, xa_limit_32b, next: &next, GFP_KERNEL);
100	if (err)
101	goto err_xa_alloc;
102
103	exts->miss_cookie_node = n;
104	return 0;
105
106	err_xa_alloc:
107	kfree(objp: n);
108	return err;
109	}
110
111	static void tcf_exts_miss_cookie_base_destroy(struct tcf_exts *exts)
112	{
113	struct tcf_exts_miss_cookie_node *n;
114
115	if (!exts->miss_cookie_node)
116	return;
117
118	n = exts->miss_cookie_node;
119	xa_erase(&tcf_exts_miss_cookies_xa, index: n->miss_cookie_base);
120	kfree_rcu(n, rcu);
121	}
122
123	static struct tcf_exts_miss_cookie_node *
124	tcf_exts_miss_cookie_lookup(u64 miss_cookie, int *act_index)
125	{
126	union tcf_exts_miss_cookie mc = { .miss_cookie = miss_cookie, };
127
128	*act_index = mc.act_index;
129	return xa_load(&tcf_exts_miss_cookies_xa, index: mc.miss_cookie_base);
130	}
131	#else /* IS_ENABLED(CONFIG_NET_TC_SKB_EXT) */
132	static int
133	tcf_exts_miss_cookie_base_alloc(struct tcf_exts *exts, struct tcf_proto *tp,
134	u32 handle)
135	{
136	return 0;
137	}
138
139	static void tcf_exts_miss_cookie_base_destroy(struct tcf_exts *exts)
140	{
141	}
142	#endif /* IS_ENABLED(CONFIG_NET_TC_SKB_EXT) */
143
144	static u64 tcf_exts_miss_cookie_get(u32 miss_cookie_base, int act_index)
145	{
146	union tcf_exts_miss_cookie mc = { .act_index = act_index, };
147
148	if (!miss_cookie_base)
149	return 0;
150
151	mc.miss_cookie_base = miss_cookie_base;
152	return mc.miss_cookie;
153	}
154
155	#ifdef CONFIG_NET_CLS_ACT
156	DEFINE_STATIC_KEY_FALSE(tc_skb_ext_tc);
157	EXPORT_SYMBOL(tc_skb_ext_tc);
158
159	void tc_skb_ext_tc_enable(void)
160	{
161	static_branch_inc(&tc_skb_ext_tc);
162	}
163	EXPORT_SYMBOL(tc_skb_ext_tc_enable);
164
165	void tc_skb_ext_tc_disable(void)
166	{
167	static_branch_dec(&tc_skb_ext_tc);
168	}
169	EXPORT_SYMBOL(tc_skb_ext_tc_disable);
170	#endif
171
172	static u32 destroy_obj_hashfn(const struct tcf_proto *tp)
173	{
174	return jhash_3words(a: tp->chain->index, b: tp->prio,
175	c: (__force __u32)tp->protocol, initval: 0);
176	}
177
178	static void tcf_proto_signal_destroying(struct tcf_chain *chain,
179	struct tcf_proto *tp)
180	{
181	struct tcf_block *block = chain->block;
182
183	mutex_lock(&block->proto_destroy_lock);
184	hash_add_rcu(block->proto_destroy_ht, &tp->destroy_ht_node,
185	destroy_obj_hashfn(tp));
186	mutex_unlock(lock: &block->proto_destroy_lock);
187	}
188
189	static bool tcf_proto_cmp(const struct tcf_proto *tp1,
190	const struct tcf_proto *tp2)
191	{
192	return tp1->chain->index == tp2->chain->index &&
193	tp1->prio == tp2->prio &&
194	tp1->protocol == tp2->protocol;
195	}
196
197	static bool tcf_proto_exists_destroying(struct tcf_chain *chain,
198	struct tcf_proto *tp)
199	{
200	u32 hash = destroy_obj_hashfn(tp);
201	struct tcf_proto *iter;
202	bool found = false;
203
204	rcu_read_lock();
205	hash_for_each_possible_rcu(chain->block->proto_destroy_ht, iter,
206	destroy_ht_node, hash) {
207	if (tcf_proto_cmp(tp1: tp, tp2: iter)) {
208	found = true;
209	break;
210	}
211	}
212	rcu_read_unlock();
213
214	return found;
215	}
216
217	static void
218	tcf_proto_signal_destroyed(struct tcf_chain *chain, struct tcf_proto *tp)
219	{
220	struct tcf_block *block = chain->block;
221
222	mutex_lock(&block->proto_destroy_lock);
223	if (hash_hashed(node: &tp->destroy_ht_node))
224	hash_del_rcu(node: &tp->destroy_ht_node);
225	mutex_unlock(lock: &block->proto_destroy_lock);
226	}
227
228	/* Find classifier type by string name */
229
230	static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind)
231	{
232	const struct tcf_proto_ops *t, *res = NULL;
233
234	if (kind) {
235	read_lock(&cls_mod_lock);
236	list_for_each_entry(t, &tcf_proto_base, head) {
237	if (strcmp(kind, t->kind) == 0) {
238	if (try_module_get(module: t->owner))
239	res = t;
240	break;
241	}
242	}
243	read_unlock(&cls_mod_lock);
244	}
245	return res;
246	}
247
248	static const struct tcf_proto_ops *
249	tcf_proto_lookup_ops(const char *kind, bool rtnl_held,
250	struct netlink_ext_ack *extack)
251	{
252	const struct tcf_proto_ops *ops;
253
254	ops = __tcf_proto_lookup_ops(kind);
255	if (ops)
256	return ops;
257	#ifdef CONFIG_MODULES
258	if (rtnl_held)
259	rtnl_unlock();
260	request_module(NET_CLS_ALIAS_PREFIX "%s", kind);
261	if (rtnl_held)
262	rtnl_lock();
263	ops = __tcf_proto_lookup_ops(kind);
264	/* We dropped the RTNL semaphore in order to perform
265	* the module load. So, even if we succeeded in loading
266	* the module we have to replay the request. We indicate
267	* this using -EAGAIN.
268	*/
269	if (ops) {
270	module_put(module: ops->owner);
271	return ERR_PTR(error: -EAGAIN);
272	}
273	#endif
274	NL_SET_ERR_MSG(extack, "TC classifier not found");
275	return ERR_PTR(error: -ENOENT);
276	}
277
278	/* Register(unregister) new classifier type */
279
280	int register_tcf_proto_ops(struct tcf_proto_ops *ops)
281	{
282	struct tcf_proto_ops *t;
283	int rc = -EEXIST;
284
285	write_lock(&cls_mod_lock);
286	list_for_each_entry(t, &tcf_proto_base, head)
287	if (!strcmp(ops->kind, t->kind))
288	goto out;
289
290	list_add_tail(new: &ops->head, head: &tcf_proto_base);
291	rc = 0;
292	out:
293	write_unlock(&cls_mod_lock);
294	return rc;
295	}
296	EXPORT_SYMBOL(register_tcf_proto_ops);
297
298	static struct workqueue_struct *tc_filter_wq;
299
300	void unregister_tcf_proto_ops(struct tcf_proto_ops *ops)
301	{
302	struct tcf_proto_ops *t;
303	int rc = -ENOENT;
304
305	/* Wait for outstanding call_rcu()s, if any, from a
306	* tcf_proto_ops's destroy() handler.
307	*/
308	rcu_barrier();
309	flush_workqueue(tc_filter_wq);
310
311	write_lock(&cls_mod_lock);
312	list_for_each_entry(t, &tcf_proto_base, head) {
313	if (t == ops) {
314	list_del(entry: &t->head);
315	rc = 0;
316	break;
317	}
318	}
319	write_unlock(&cls_mod_lock);
320
321	WARN(rc, "unregister tc filter kind(%s) failed %d\n", ops->kind, rc);
322	}
323	EXPORT_SYMBOL(unregister_tcf_proto_ops);
324
325	bool tcf_queue_work(struct rcu_work *rwork, work_func_t func)
326	{
327	INIT_RCU_WORK(rwork, func);
328	return queue_rcu_work(wq: tc_filter_wq, rwork);
329	}
330	EXPORT_SYMBOL(tcf_queue_work);
331
332	/* Select new prio value from the range, managed by kernel. */
333
334	static inline u32 tcf_auto_prio(struct tcf_proto *tp)
335	{
336	u32 first = TC_H_MAKE(0xC0000000U, 0U);
337
338	if (tp)
339	first = tp->prio - 1;
340
341	return TC_H_MAJ(first);
342	}
343
344	static bool tcf_proto_check_kind(struct nlattr *kind, char *name)
345	{
346	if (kind)
347	return nla_strscpy(dst: name, nla: kind, IFNAMSIZ) < 0;
348	memset(name, 0, IFNAMSIZ);
349	return false;
350	}
351
352	static bool tcf_proto_is_unlocked(const char *kind)
353	{
354	const struct tcf_proto_ops *ops;
355	bool ret;
356
357	if (strlen(kind) == 0)
358	return false;
359
360	ops = tcf_proto_lookup_ops(kind, rtnl_held: false, NULL);
361	/* On error return false to take rtnl lock. Proto lookup/create
362	* functions will perform lookup again and properly handle errors.
363	*/
364	if (IS_ERR(ptr: ops))
365	return false;
366
367	ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED);
368	module_put(module: ops->owner);
369	return ret;
370	}
371
372	static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol,
373	u32 prio, struct tcf_chain *chain,
374	bool rtnl_held,
375	struct netlink_ext_ack *extack)
376	{
377	struct tcf_proto *tp;
378	int err;
379
380	tp = kzalloc(size: sizeof(*tp), GFP_KERNEL);
381	if (!tp)
382	return ERR_PTR(error: -ENOBUFS);
383
384	tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack);
385	if (IS_ERR(ptr: tp->ops)) {
386	err = PTR_ERR(ptr: tp->ops);
387	goto errout;
388	}
389	tp->classify = tp->ops->classify;
390	tp->protocol = protocol;
391	tp->prio = prio;
392	tp->chain = chain;
393	spin_lock_init(&tp->lock);
394	refcount_set(r: &tp->refcnt, n: 1);
395
396	err = tp->ops->init(tp);
397	if (err) {
398	module_put(module: tp->ops->owner);
399	goto errout;
400	}
401	return tp;
402
403	errout:
404	kfree(objp: tp);
405	return ERR_PTR(error: err);
406	}
407
408	static void tcf_proto_get(struct tcf_proto *tp)
409	{
410	refcount_inc(r: &tp->refcnt);
411	}
412
413	static void tcf_chain_put(struct tcf_chain *chain);
414
415	static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held,
416	bool sig_destroy, struct netlink_ext_ack *extack)
417	{
418	tp->ops->destroy(tp, rtnl_held, extack);
419	if (sig_destroy)
420	tcf_proto_signal_destroyed(chain: tp->chain, tp);
421	tcf_chain_put(chain: tp->chain);
422	module_put(module: tp->ops->owner);
423	kfree_rcu(tp, rcu);
424	}
425
426	static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held,
427	struct netlink_ext_ack *extack)
428	{
429	if (refcount_dec_and_test(r: &tp->refcnt))
430	tcf_proto_destroy(tp, rtnl_held, sig_destroy: true, extack);
431	}
432
433	static bool tcf_proto_check_delete(struct tcf_proto *tp)
434	{
435	if (tp->ops->delete_empty)
436	return tp->ops->delete_empty(tp);
437
438	tp->deleting = true;
439	return tp->deleting;
440	}
441
442	static void tcf_proto_mark_delete(struct tcf_proto *tp)
443	{
444	spin_lock(lock: &tp->lock);
445	tp->deleting = true;
446	spin_unlock(lock: &tp->lock);
447	}
448
449	static bool tcf_proto_is_deleting(struct tcf_proto *tp)
450	{
451	bool deleting;
452
453	spin_lock(lock: &tp->lock);
454	deleting = tp->deleting;
455	spin_unlock(lock: &tp->lock);
456
457	return deleting;
458	}
459
460	#define ASSERT_BLOCK_LOCKED(block) \
461	lockdep_assert_held(&(block)->lock)
462
463	struct tcf_filter_chain_list_item {
464	struct list_head list;
465	tcf_chain_head_change_t *chain_head_change;
466	void *chain_head_change_priv;
467	};
468
469	static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
470	u32 chain_index)
471	{
472	struct tcf_chain *chain;
473
474	ASSERT_BLOCK_LOCKED(block);
475
476	chain = kzalloc(size: sizeof(*chain), GFP_KERNEL);
477	if (!chain)
478	return NULL;
479	list_add_tail_rcu(new: &chain->list, head: &block->chain_list);
480	mutex_init(&chain->filter_chain_lock);
481	chain->block = block;
482	chain->index = chain_index;
483	chain->refcnt = 1;
484	if (!chain->index)
485	block->chain0.chain = chain;
486	return chain;
487	}
488
489	static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
490	struct tcf_proto *tp_head)
491	{
492	if (item->chain_head_change)
493	item->chain_head_change(tp_head, item->chain_head_change_priv);
494	}
495
496	static void tcf_chain0_head_change(struct tcf_chain *chain,
497	struct tcf_proto *tp_head)
498	{
499	struct tcf_filter_chain_list_item *item;
500	struct tcf_block *block = chain->block;
501
502	if (chain->index)
503	return;
504
505	mutex_lock(&block->lock);
506	list_for_each_entry(item, &block->chain0.filter_chain_list, list)
507	tcf_chain_head_change_item(item, tp_head);
508	mutex_unlock(lock: &block->lock);
509	}
510
511	/* Returns true if block can be safely freed. */
512
513	static bool tcf_chain_detach(struct tcf_chain *chain)
514	{
515	struct tcf_block *block = chain->block;
516
517	ASSERT_BLOCK_LOCKED(block);
518
519	list_del_rcu(entry: &chain->list);
520	if (!chain->index)
521	block->chain0.chain = NULL;
522
523	if (list_empty(head: &block->chain_list) &&
524	refcount_read(r: &block->refcnt) == 0)
525	return true;
526
527	return false;
528	}
529
530	static void tcf_block_destroy(struct tcf_block *block)
531	{
532	mutex_destroy(lock: &block->lock);
533	mutex_destroy(lock: &block->proto_destroy_lock);
534	xa_destroy(&block->ports);
535	kfree_rcu(block, rcu);
536	}
537
538	static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
539	{
540	struct tcf_block *block = chain->block;
541
542	mutex_destroy(lock: &chain->filter_chain_lock);
543	kfree_rcu(chain, rcu);
544	if (free_block)
545	tcf_block_destroy(block);
546	}
547
548	static void tcf_chain_hold(struct tcf_chain *chain)
549	{
550	ASSERT_BLOCK_LOCKED(chain->block);
551
552	++chain->refcnt;
553	}
554
555	static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
556	{
557	ASSERT_BLOCK_LOCKED(chain->block);
558
559	/* In case all the references are action references, this
560	* chain should not be shown to the user.
561	*/
562	return chain->refcnt == chain->action_refcnt;
563	}
564
565	static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
566	u32 chain_index)
567	{
568	struct tcf_chain *chain;
569
570	ASSERT_BLOCK_LOCKED(block);
571
572	list_for_each_entry(chain, &block->chain_list, list) {
573	if (chain->index == chain_index)
574	return chain;
575	}
576	return NULL;
577	}
578
579	#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
580	static struct tcf_chain *tcf_chain_lookup_rcu(const struct tcf_block *block,
581	u32 chain_index)
582	{
583	struct tcf_chain *chain;
584
585	list_for_each_entry_rcu(chain, &block->chain_list, list) {
586	if (chain->index == chain_index)
587	return chain;
588	}
589	return NULL;
590	}
591	#endif
592
593	static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
594	u32 seq, u16 flags, int event, bool unicast,
595	struct netlink_ext_ack *extack);
596
597	static struct tcf_chain *__tcf_chain_get(struct tcf_block *block,
598	u32 chain_index, bool create,
599	bool by_act)
600	{
601	struct tcf_chain *chain = NULL;
602	bool is_first_reference;
603
604	mutex_lock(&block->lock);
605	chain = tcf_chain_lookup(block, chain_index);
606	if (chain) {
607	tcf_chain_hold(chain);
608	} else {
609	if (!create)
610	goto errout;
611	chain = tcf_chain_create(block, chain_index);
612	if (!chain)
613	goto errout;
614	}
615
616	if (by_act)
617	++chain->action_refcnt;
618	is_first_reference = chain->refcnt - chain->action_refcnt == 1;
619	mutex_unlock(lock: &block->lock);
620
621	/* Send notification only in case we got the first
622	* non-action reference. Until then, the chain acts only as
623	* a placeholder for actions pointing to it and user ought
624	* not know about them.
625	*/
626	if (is_first_reference && !by_act)
627	tc_chain_notify(chain, NULL, seq: 0, NLM_F_CREATE | NLM_F_EXCL,
628	RTM_NEWCHAIN, unicast: false, NULL);
629
630	return chain;
631
632	errout:
633	mutex_unlock(lock: &block->lock);
634	return chain;
635	}
636
637	static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index,
638	bool create)
639	{
640	return __tcf_chain_get(block, chain_index, create, by_act: false);
641	}
642
643	struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index)
644	{
645	return __tcf_chain_get(block, chain_index, create: true, by_act: true);
646	}
647	EXPORT_SYMBOL(tcf_chain_get_by_act);
648
649	static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
650	void *tmplt_priv);
651	static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
652	void *tmplt_priv, u32 chain_index,
653	struct tcf_block *block, struct sk_buff *oskb,
654	u32 seq, u16 flags);
655
656	static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
657	bool explicitly_created)
658	{
659	struct tcf_block *block = chain->block;
660	const struct tcf_proto_ops *tmplt_ops;
661	unsigned int refcnt, non_act_refcnt;
662	bool free_block = false;
663	void *tmplt_priv;
664
665	mutex_lock(&block->lock);
666	if (explicitly_created) {
667	if (!chain->explicitly_created) {
668	mutex_unlock(lock: &block->lock);
669	return;
670	}
671	chain->explicitly_created = false;
672	}
673
674	if (by_act)
675	chain->action_refcnt--;
676
677	/* tc_chain_notify_delete can't be called while holding block lock.
678	* However, when block is unlocked chain can be changed concurrently, so
679	* save these to temporary variables.
680	*/
681	refcnt = --chain->refcnt;
682	non_act_refcnt = refcnt - chain->action_refcnt;
683	tmplt_ops = chain->tmplt_ops;
684	tmplt_priv = chain->tmplt_priv;
685
686	if (non_act_refcnt == chain->explicitly_created && !by_act) {
687	if (non_act_refcnt == 0)
688	tc_chain_notify_delete(tmplt_ops, tmplt_priv,
689	chain_index: chain->index, block, NULL, seq: 0, flags: 0);
690	/* Last reference to chain, no need to lock. */
691	chain->flushing = false;
692	}
693
694	if (refcnt == 0)
695	free_block = tcf_chain_detach(chain);
696	mutex_unlock(lock: &block->lock);
697
698	if (refcnt == 0) {
699	tc_chain_tmplt_del(tmplt_ops, tmplt_priv);
700	tcf_chain_destroy(chain, free_block);
701	}
702	}
703
704	static void tcf_chain_put(struct tcf_chain *chain)
705	{
706	__tcf_chain_put(chain, by_act: false, explicitly_created: false);
707	}
708
709	void tcf_chain_put_by_act(struct tcf_chain *chain)
710	{
711	__tcf_chain_put(chain, by_act: true, explicitly_created: false);
712	}
713	EXPORT_SYMBOL(tcf_chain_put_by_act);
714
715	static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
716	{
717	__tcf_chain_put(chain, by_act: false, explicitly_created: true);
718	}
719
720	static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held)
721	{
722	struct tcf_proto *tp, *tp_next;
723
724	mutex_lock(&chain->filter_chain_lock);
725	tp = tcf_chain_dereference(chain->filter_chain, chain);
726	while (tp) {
727	tp_next = rcu_dereference_protected(tp->next, 1);
728	tcf_proto_signal_destroying(chain, tp);
729	tp = tp_next;
730	}
731	tp = tcf_chain_dereference(chain->filter_chain, chain);
732	RCU_INIT_POINTER(chain->filter_chain, NULL);
733	tcf_chain0_head_change(chain, NULL);
734	chain->flushing = true;
735	mutex_unlock(lock: &chain->filter_chain_lock);
736
737	while (tp) {
738	tp_next = rcu_dereference_protected(tp->next, 1);
739	tcf_proto_put(tp, rtnl_held, NULL);
740	tp = tp_next;
741	}
742	}
743
744	static int tcf_block_setup(struct tcf_block *block,
745	struct flow_block_offload *bo);
746
747	static void tcf_block_offload_init(struct flow_block_offload *bo,
748	struct net_device *dev, struct Qdisc *sch,
749	enum flow_block_command command,
750	enum flow_block_binder_type binder_type,
751	struct flow_block *flow_block,
752	bool shared, struct netlink_ext_ack *extack)
753	{
754	bo->net = dev_net(dev);
755	bo->command = command;
756	bo->binder_type = binder_type;
757	bo->block = flow_block;
758	bo->block_shared = shared;
759	bo->extack = extack;
760	bo->sch = sch;
761	bo->cb_list_head = &flow_block->cb_list;
762	INIT_LIST_HEAD(list: &bo->cb_list);
763	}
764
765	static void tcf_block_unbind(struct tcf_block *block,
766	struct flow_block_offload *bo);
767
768	static void tc_block_indr_cleanup(struct flow_block_cb *block_cb)
769	{
770	struct tcf_block *block = block_cb->indr.data;
771	struct net_device *dev = block_cb->indr.dev;
772	struct Qdisc *sch = block_cb->indr.sch;
773	struct netlink_ext_ack extack = {};
774	struct flow_block_offload bo = {};
775
776	tcf_block_offload_init(bo: &bo, dev, sch, command: FLOW_BLOCK_UNBIND,
777	binder_type: block_cb->indr.binder_type,
778	flow_block: &block->flow_block, shared: tcf_block_shared(block),
779	extack: &extack);
780	rtnl_lock();
781	down_write(sem: &block->cb_lock);
782	list_del(entry: &block_cb->driver_list);
783	list_move(list: &block_cb->list, head: &bo.cb_list);
784	tcf_block_unbind(block, bo: &bo);
785	up_write(sem: &block->cb_lock);
786	rtnl_unlock();
787	}
788
789	static bool tcf_block_offload_in_use(struct tcf_block *block)
790	{
791	return atomic_read(v: &block->offloadcnt);
792	}
793
794	static int tcf_block_offload_cmd(struct tcf_block *block,
795	struct net_device *dev, struct Qdisc *sch,
796	struct tcf_block_ext_info *ei,
797	enum flow_block_command command,
798	struct netlink_ext_ack *extack)
799	{
800	struct flow_block_offload bo = {};
801
802	tcf_block_offload_init(bo: &bo, dev, sch, command, binder_type: ei->binder_type,
803	flow_block: &block->flow_block, shared: tcf_block_shared(block),
804	extack);
805
806	if (dev->netdev_ops->ndo_setup_tc) {
807	int err;
808
809	err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
810	if (err < 0) {
811	if (err != -EOPNOTSUPP)
812	NL_SET_ERR_MSG(extack, "Driver ndo_setup_tc failed");
813	return err;
814	}
815
816	return tcf_block_setup(block, bo: &bo);
817	}
818
819	flow_indr_dev_setup_offload(dev, sch, type: TC_SETUP_BLOCK, data: block, bo: &bo,
820	cleanup: tc_block_indr_cleanup);
821	tcf_block_setup(block, bo: &bo);
822
823	return -EOPNOTSUPP;
824	}
825
826	static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q,
827	struct tcf_block_ext_info *ei,
828	struct netlink_ext_ack *extack)
829	{
830	struct net_device *dev = q->dev_queue->dev;
831	int err;
832
833	down_write(sem: &block->cb_lock);
834
835	/* If tc offload feature is disabled and the block we try to bind
836	* to already has some offloaded filters, forbid to bind.
837	*/
838	if (dev->netdev_ops->ndo_setup_tc &&
839	!tc_can_offload(dev) &&
840	tcf_block_offload_in_use(block)) {
841	NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled");
842	err = -EOPNOTSUPP;
843	goto err_unlock;
844	}
845
846	err = tcf_block_offload_cmd(block, dev, sch: q, ei, command: FLOW_BLOCK_BIND, extack);
847	if (err == -EOPNOTSUPP)
848	goto no_offload_dev_inc;
849	if (err)
850	goto err_unlock;
851
852	up_write(sem: &block->cb_lock);
853	return 0;
854
855	no_offload_dev_inc:
856	if (tcf_block_offload_in_use(block))
857	goto err_unlock;
858
859	err = 0;
860	block->nooffloaddevcnt++;
861	err_unlock:
862	up_write(sem: &block->cb_lock);
863	return err;
864	}
865
866	static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
867	struct tcf_block_ext_info *ei)
868	{
869	struct net_device *dev = q->dev_queue->dev;
870	int err;
871
872	down_write(sem: &block->cb_lock);
873	err = tcf_block_offload_cmd(block, dev, sch: q, ei, command: FLOW_BLOCK_UNBIND, NULL);
874	if (err == -EOPNOTSUPP)
875	goto no_offload_dev_dec;
876	up_write(sem: &block->cb_lock);
877	return;
878
879	no_offload_dev_dec:
880	WARN_ON(block->nooffloaddevcnt-- == 0);
881	up_write(sem: &block->cb_lock);
882	}
883
884	static int
885	tcf_chain0_head_change_cb_add(struct tcf_block *block,
886	struct tcf_block_ext_info *ei,
887	struct netlink_ext_ack *extack)
888	{
889	struct tcf_filter_chain_list_item *item;
890	struct tcf_chain *chain0;
891
892	item = kmalloc(size: sizeof(*item), GFP_KERNEL);
893	if (!item) {
894	NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed");
895	return -ENOMEM;
896	}
897	item->chain_head_change = ei->chain_head_change;
898	item->chain_head_change_priv = ei->chain_head_change_priv;
899
900	mutex_lock(&block->lock);
901	chain0 = block->chain0.chain;
902	if (chain0)
903	tcf_chain_hold(chain: chain0);
904	else
905	list_add(new: &item->list, head: &block->chain0.filter_chain_list);
906	mutex_unlock(lock: &block->lock);
907
908	if (chain0) {
909	struct tcf_proto *tp_head;
910
911	mutex_lock(&chain0->filter_chain_lock);
912
913	tp_head = tcf_chain_dereference(chain0->filter_chain, chain0);
914	if (tp_head)
915	tcf_chain_head_change_item(item, tp_head);
916
917	mutex_lock(&block->lock);
918	list_add(new: &item->list, head: &block->chain0.filter_chain_list);
919	mutex_unlock(lock: &block->lock);
920
921	mutex_unlock(lock: &chain0->filter_chain_lock);
922	tcf_chain_put(chain: chain0);
923	}
924
925	return 0;
926	}
927
928	static void
929	tcf_chain0_head_change_cb_del(struct tcf_block *block,
930	struct tcf_block_ext_info *ei)
931	{
932	struct tcf_filter_chain_list_item *item;
933
934	mutex_lock(&block->lock);
935	list_for_each_entry(item, &block->chain0.filter_chain_list, list) {
936	if ((!ei->chain_head_change && !ei->chain_head_change_priv) ||
937	(item->chain_head_change == ei->chain_head_change &&
938	item->chain_head_change_priv == ei->chain_head_change_priv)) {
939	if (block->chain0.chain)
940	tcf_chain_head_change_item(item, NULL);
941	list_del(entry: &item->list);
942	mutex_unlock(lock: &block->lock);
943
944	kfree(objp: item);
945	return;
946	}
947	}
948	mutex_unlock(lock: &block->lock);
949	WARN_ON(1);
950	}
951
952	struct tcf_net {
953	spinlock_t idr_lock; /* Protects idr */
954	struct idr idr;
955	};
956
957	static unsigned int tcf_net_id;
958
959	static int tcf_block_insert(struct tcf_block *block, struct net *net,
960	struct netlink_ext_ack *extack)
961	{
962	struct tcf_net *tn = net_generic(net, id: tcf_net_id);
963	int err;
964
965	idr_preload(GFP_KERNEL);
966	spin_lock(lock: &tn->idr_lock);
967	err = idr_alloc_u32(&tn->idr, ptr: block, id: &block->index, max: block->index,
968	GFP_NOWAIT);
969	spin_unlock(lock: &tn->idr_lock);
970	idr_preload_end();
971
972	return err;
973	}
974
975	static void tcf_block_remove(struct tcf_block *block, struct net *net)
976	{
977	struct tcf_net *tn = net_generic(net, id: tcf_net_id);
978
979	spin_lock(lock: &tn->idr_lock);
980	idr_remove(&tn->idr, id: block->index);
981	spin_unlock(lock: &tn->idr_lock);
982	}
983
984	static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q,
985	u32 block_index,
986	struct netlink_ext_ack *extack)
987	{
988	struct tcf_block *block;
989
990	block = kzalloc(size: sizeof(*block), GFP_KERNEL);
991	if (!block) {
992	NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
993	return ERR_PTR(error: -ENOMEM);
994	}
995	mutex_init(&block->lock);
996	mutex_init(&block->proto_destroy_lock);
997	init_rwsem(&block->cb_lock);
998	flow_block_init(flow_block: &block->flow_block);
999	INIT_LIST_HEAD(list: &block->chain_list);
1000	INIT_LIST_HEAD(list: &block->owner_list);
1001	INIT_LIST_HEAD(list: &block->chain0.filter_chain_list);
1002
1003	refcount_set(r: &block->refcnt, n: 1);
1004	block->net = net;
1005	block->index = block_index;
1006	xa_init(xa: &block->ports);
1007
1008	/* Don't store q pointer for blocks which are shared */
1009	if (!tcf_block_shared(block))
1010	block->q = q;
1011	return block;
1012	}
1013
1014	struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index)
1015	{
1016	struct tcf_net *tn = net_generic(net, id: tcf_net_id);
1017
1018	return idr_find(&tn->idr, id: block_index);
1019	}
1020	EXPORT_SYMBOL(tcf_block_lookup);
1021
1022	static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index)
1023	{
1024	struct tcf_block *block;
1025
1026	rcu_read_lock();
1027	block = tcf_block_lookup(net, block_index);
1028	if (block && !refcount_inc_not_zero(r: &block->refcnt))
1029	block = NULL;
1030	rcu_read_unlock();
1031
1032	return block;
1033	}
1034
1035	static struct tcf_chain *
1036	__tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
1037	{
1038	mutex_lock(&block->lock);
1039	if (chain)
1040	chain = list_is_last(list: &chain->list, head: &block->chain_list) ?
1041	NULL : list_next_entry(chain, list);
1042	else
1043	chain = list_first_entry_or_null(&block->chain_list,
1044	struct tcf_chain, list);
1045
1046	/* skip all action-only chains */
1047	while (chain && tcf_chain_held_by_acts_only(chain))
1048	chain = list_is_last(list: &chain->list, head: &block->chain_list) ?
1049	NULL : list_next_entry(chain, list);
1050
1051	if (chain)
1052	tcf_chain_hold(chain);
1053	mutex_unlock(lock: &block->lock);
1054
1055	return chain;
1056	}
1057
1058	/* Function to be used by all clients that want to iterate over all chains on
1059	* block. It properly obtains block->lock and takes reference to chain before
1060	* returning it. Users of this function must be tolerant to concurrent chain
1061	* insertion/deletion or ensure that no concurrent chain modification is
1062	* possible. Note that all netlink dump callbacks cannot guarantee to provide
1063	* consistent dump because rtnl lock is released each time skb is filled with
1064	* data and sent to user-space.
1065	*/
1066
1067	struct tcf_chain *
1068	tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
1069	{
1070	struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain);
1071
1072	if (chain)
1073	tcf_chain_put(chain);
1074
1075	return chain_next;
1076	}
1077	EXPORT_SYMBOL(tcf_get_next_chain);
1078
1079	static struct tcf_proto *
1080	__tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
1081	{
1082	u32 prio = 0;
1083
1084	ASSERT_RTNL();
1085	mutex_lock(&chain->filter_chain_lock);
1086
1087	if (!tp) {
1088	tp = tcf_chain_dereference(chain->filter_chain, chain);
1089	} else if (tcf_proto_is_deleting(tp)) {
1090	/* 'deleting' flag is set and chain->filter_chain_lock was
1091	* unlocked, which means next pointer could be invalid. Restart
1092	* search.
1093	*/
1094	prio = tp->prio + 1;
1095	tp = tcf_chain_dereference(chain->filter_chain, chain);
1096
1097	for (; tp; tp = tcf_chain_dereference(tp->next, chain))
1098	if (!tp->deleting && tp->prio >= prio)
1099	break;
1100	} else {
1101	tp = tcf_chain_dereference(tp->next, chain);
1102	}
1103
1104	if (tp)
1105	tcf_proto_get(tp);
1106
1107	mutex_unlock(lock: &chain->filter_chain_lock);
1108
1109	return tp;
1110	}
1111
1112	/* Function to be used by all clients that want to iterate over all tp's on
1113	* chain. Users of this function must be tolerant to concurrent tp
1114	* insertion/deletion or ensure that no concurrent chain modification is
1115	* possible. Note that all netlink dump callbacks cannot guarantee to provide
1116	* consistent dump because rtnl lock is released each time skb is filled with
1117	* data and sent to user-space.
1118	*/
1119
1120	struct tcf_proto *
1121	tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
1122	{
1123	struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp);
1124
1125	if (tp)
1126	tcf_proto_put(tp, rtnl_held: true, NULL);
1127
1128	return tp_next;
1129	}
1130	EXPORT_SYMBOL(tcf_get_next_proto);
1131
1132	static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held)
1133	{
1134	struct tcf_chain *chain;
1135
1136	/* Last reference to block. At this point chains cannot be added or
1137	* removed concurrently.
1138	*/
1139	for (chain = tcf_get_next_chain(block, NULL);
1140	chain;
1141	chain = tcf_get_next_chain(block, chain)) {
1142	tcf_chain_put_explicitly_created(chain);
1143	tcf_chain_flush(chain, rtnl_held);
1144	}
1145	}
1146
1147	/* Lookup Qdisc and increments its reference counter.
1148	* Set parent, if necessary.
1149	*/
1150
1151	static int __tcf_qdisc_find(struct net *net, struct Qdisc **q,
1152	u32 *parent, int ifindex, bool rtnl_held,
1153	struct netlink_ext_ack *extack)
1154	{
1155	const struct Qdisc_class_ops *cops;
1156	struct net_device *dev;
1157	int err = 0;
1158
1159	if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1160	return 0;
1161
1162	rcu_read_lock();
1163
1164	/* Find link */
1165	dev = dev_get_by_index_rcu(net, ifindex);
1166	if (!dev) {
1167	rcu_read_unlock();
1168	return -ENODEV;
1169	}
1170
1171	/* Find qdisc */
1172	if (!*parent) {
1173	*q = rcu_dereference(dev->qdisc);
1174	*parent = (*q)->handle;
1175	} else {
1176	*q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent));
1177	if (!*q) {
1178	NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1179	err = -EINVAL;
1180	goto errout_rcu;
1181	}
1182	}
1183
1184	*q = qdisc_refcount_inc_nz(qdisc: *q);
1185	if (!*q) {
1186	NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1187	err = -EINVAL;
1188	goto errout_rcu;
1189	}
1190
1191	/* Is it classful? */
1192	cops = (*q)->ops->cl_ops;
1193	if (!cops) {
1194	NL_SET_ERR_MSG(extack, "Qdisc not classful");
1195	err = -EINVAL;
1196	goto errout_qdisc;
1197	}
1198
1199	if (!cops->tcf_block) {
1200	NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
1201	err = -EOPNOTSUPP;
1202	goto errout_qdisc;
1203	}
1204
1205	errout_rcu:
1206	/* At this point we know that qdisc is not noop_qdisc,
1207	* which means that qdisc holds a reference to net_device
1208	* and we hold a reference to qdisc, so it is safe to release
1209	* rcu read lock.
1210	*/
1211	rcu_read_unlock();
1212	return err;
1213
1214	errout_qdisc:
1215	rcu_read_unlock();
1216
1217	if (rtnl_held)
1218	qdisc_put(qdisc: *q);
1219	else
1220	qdisc_put_unlocked(qdisc: *q);
1221	*q = NULL;
1222
1223	return err;
1224	}
1225
1226	static int __tcf_qdisc_cl_find(struct Qdisc *q, u32 parent, unsigned long *cl,
1227	int ifindex, struct netlink_ext_ack *extack)
1228	{
1229	if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1230	return 0;
1231
1232	/* Do we search for filter, attached to class? */
1233	if (TC_H_MIN(parent)) {
1234	const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1235
1236	*cl = cops->find(q, parent);
1237	if (*cl == 0) {
1238	NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
1239	return -ENOENT;
1240	}
1241	}
1242
1243	return 0;
1244	}
1245
1246	static struct tcf_block *__tcf_block_find(struct net *net, struct Qdisc *q,
1247	unsigned long cl, int ifindex,
1248	u32 block_index,
1249	struct netlink_ext_ack *extack)
1250	{
1251	struct tcf_block *block;
1252
1253	if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
1254	block = tcf_block_refcnt_get(net, block_index);
1255	if (!block) {
1256	NL_SET_ERR_MSG(extack, "Block of given index was not found");
1257	return ERR_PTR(error: -EINVAL);
1258	}
1259	} else {
1260	const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1261
1262	block = cops->tcf_block(q, cl, extack);
1263	if (!block)
1264	return ERR_PTR(error: -EINVAL);
1265
1266	if (tcf_block_shared(block)) {
1267	NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
1268	return ERR_PTR(error: -EOPNOTSUPP);
1269	}
1270
1271	/* Always take reference to block in order to support execution
1272	* of rules update path of cls API without rtnl lock. Caller
1273	* must release block when it is finished using it. 'if' block
1274	* of this conditional obtain reference to block by calling
1275	* tcf_block_refcnt_get().
1276	*/
1277	refcount_inc(r: &block->refcnt);
1278	}
1279
1280	return block;
1281	}
1282
1283	static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
1284	struct tcf_block_ext_info *ei, bool rtnl_held)
1285	{
1286	if (refcount_dec_and_mutex_lock(r: &block->refcnt, lock: &block->lock)) {
1287	/* Flushing/putting all chains will cause the block to be
1288	* deallocated when last chain is freed. However, if chain_list
1289	* is empty, block has to be manually deallocated. After block
1290	* reference counter reached 0, it is no longer possible to
1291	* increment it or add new chains to block.
1292	*/
1293	bool free_block = list_empty(head: &block->chain_list);
1294
1295	mutex_unlock(lock: &block->lock);
1296	if (tcf_block_shared(block))
1297	tcf_block_remove(block, net: block->net);
1298
1299	if (q)
1300	tcf_block_offload_unbind(block, q, ei);
1301
1302	if (free_block)
1303	tcf_block_destroy(block);
1304	else
1305	tcf_block_flush_all_chains(block, rtnl_held);
1306	} else if (q) {
1307	tcf_block_offload_unbind(block, q, ei);
1308	}
1309	}
1310
1311	static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held)
1312	{
1313	__tcf_block_put(block, NULL, NULL, rtnl_held);
1314	}
1315
1316	/* Find tcf block.
1317	* Set q, parent, cl when appropriate.
1318	*/
1319
1320	static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q,
1321	u32 *parent, unsigned long *cl,
1322	int ifindex, u32 block_index,
1323	struct netlink_ext_ack *extack)
1324	{
1325	struct tcf_block *block;
1326	int err = 0;
1327
1328	ASSERT_RTNL();
1329
1330	err = __tcf_qdisc_find(net, q, parent, ifindex, rtnl_held: true, extack);
1331	if (err)
1332	goto errout;
1333
1334	err = __tcf_qdisc_cl_find(q: *q, parent: *parent, cl, ifindex, extack);
1335	if (err)
1336	goto errout_qdisc;
1337
1338	block = __tcf_block_find(net, q: *q, cl: *cl, ifindex, block_index, extack);
1339	if (IS_ERR(ptr: block)) {
1340	err = PTR_ERR(ptr: block);
1341	goto errout_qdisc;
1342	}
1343
1344	return block;
1345
1346	errout_qdisc:
1347	if (*q)
1348	qdisc_put(qdisc: *q);
1349	errout:
1350	*q = NULL;
1351	return ERR_PTR(error: err);
1352	}
1353
1354	static void tcf_block_release(struct Qdisc *q, struct tcf_block *block,
1355	bool rtnl_held)
1356	{
1357	if (!IS_ERR_OR_NULL(ptr: block))
1358	tcf_block_refcnt_put(block, rtnl_held);
1359
1360	if (q) {
1361	if (rtnl_held)
1362	qdisc_put(qdisc: q);
1363	else
1364	qdisc_put_unlocked(qdisc: q);
1365	}
1366	}
1367
1368	struct tcf_block_owner_item {
1369	struct list_head list;
1370	struct Qdisc *q;
1371	enum flow_block_binder_type binder_type;
1372	};
1373
1374	static void
1375	tcf_block_owner_netif_keep_dst(struct tcf_block *block,
1376	struct Qdisc *q,
1377	enum flow_block_binder_type binder_type)
1378	{
1379	if (block->keep_dst &&
1380	binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
1381	binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
1382	netif_keep_dst(dev: qdisc_dev(qdisc: q));
1383	}
1384
1385	void tcf_block_netif_keep_dst(struct tcf_block *block)
1386	{
1387	struct tcf_block_owner_item *item;
1388
1389	block->keep_dst = true;
1390	list_for_each_entry(item, &block->owner_list, list)
1391	tcf_block_owner_netif_keep_dst(block, q: item->q,
1392	binder_type: item->binder_type);
1393	}
1394	EXPORT_SYMBOL(tcf_block_netif_keep_dst);
1395
1396	static int tcf_block_owner_add(struct tcf_block *block,
1397	struct Qdisc *q,
1398	enum flow_block_binder_type binder_type)
1399	{
1400	struct tcf_block_owner_item *item;
1401
1402	item = kmalloc(size: sizeof(*item), GFP_KERNEL);
1403	if (!item)
1404	return -ENOMEM;
1405	item->q = q;
1406	item->binder_type = binder_type;
1407	list_add(new: &item->list, head: &block->owner_list);
1408	return 0;
1409	}
1410
1411	static void tcf_block_owner_del(struct tcf_block *block,
1412	struct Qdisc *q,
1413	enum flow_block_binder_type binder_type)
1414	{
1415	struct tcf_block_owner_item *item;
1416
1417	list_for_each_entry(item, &block->owner_list, list) {
1418	if (item->q == q && item->binder_type == binder_type) {
1419	list_del(entry: &item->list);
1420	kfree(objp: item);
1421	return;
1422	}
1423	}
1424	WARN_ON(1);
1425	}
1426
1427	static bool tcf_block_tracks_dev(struct tcf_block *block,
1428	struct tcf_block_ext_info *ei)
1429	{
1430	return tcf_block_shared(block) &&
1431	(ei->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS ||
1432	ei->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS);
1433	}
1434
1435	int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
1436	struct tcf_block_ext_info *ei,
1437	struct netlink_ext_ack *extack)
1438	{
1439	struct net_device *dev = qdisc_dev(qdisc: q);
1440	struct net *net = qdisc_net(q);
1441	struct tcf_block *block = NULL;
1442	int err;
1443
1444	if (ei->block_index)
1445	/* block_index not 0 means the shared block is requested */
1446	block = tcf_block_refcnt_get(net, block_index: ei->block_index);
1447
1448	if (!block) {
1449	block = tcf_block_create(net, q, block_index: ei->block_index, extack);
1450	if (IS_ERR(ptr: block))
1451	return PTR_ERR(ptr: block);
1452	if (tcf_block_shared(block)) {
1453	err = tcf_block_insert(block, net, extack);
1454	if (err)
1455	goto err_block_insert;
1456	}
1457	}
1458
1459	err = tcf_block_owner_add(block, q, binder_type: ei->binder_type);
1460	if (err)
1461	goto err_block_owner_add;
1462
1463	tcf_block_owner_netif_keep_dst(block, q, binder_type: ei->binder_type);
1464
1465	err = tcf_chain0_head_change_cb_add(block, ei, extack);
1466	if (err)
1467	goto err_chain0_head_change_cb_add;
1468
1469	err = tcf_block_offload_bind(block, q, ei, extack);
1470	if (err)
1471	goto err_block_offload_bind;
1472
1473	if (tcf_block_tracks_dev(block, ei)) {
1474	err = xa_insert(xa: &block->ports, index: dev->ifindex, entry: dev, GFP_KERNEL);
1475	if (err) {
1476	NL_SET_ERR_MSG(extack, "block dev insert failed");
1477	goto err_dev_insert;
1478	}
1479	}
1480
1481	*p_block = block;
1482	return 0;
1483
1484	err_dev_insert:
1485	err_block_offload_bind:
1486	tcf_chain0_head_change_cb_del(block, ei);
1487	err_chain0_head_change_cb_add:
1488	tcf_block_owner_del(block, q, binder_type: ei->binder_type);
1489	err_block_owner_add:
1490	err_block_insert:
1491	tcf_block_refcnt_put(block, rtnl_held: true);
1492	return err;
1493	}
1494	EXPORT_SYMBOL(tcf_block_get_ext);
1495
1496	static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
1497	{
1498	struct tcf_proto __rcu **p_filter_chain = priv;
1499
1500	rcu_assign_pointer(*p_filter_chain, tp_head);
1501	}
1502
1503	int tcf_block_get(struct tcf_block **p_block,
1504	struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
1505	struct netlink_ext_ack *extack)
1506	{
1507	struct tcf_block_ext_info ei = {
1508	.chain_head_change = tcf_chain_head_change_dflt,
1509	.chain_head_change_priv = p_filter_chain,
1510	};
1511
1512	WARN_ON(!p_filter_chain);
1513	return tcf_block_get_ext(p_block, q, &ei, extack);
1514	}
1515	EXPORT_SYMBOL(tcf_block_get);
1516
1517	/* XXX: Standalone actions are not allowed to jump to any chain, and bound
1518	* actions should be all removed after flushing.
1519	*/
1520	void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
1521	struct tcf_block_ext_info *ei)
1522	{
1523	struct net_device *dev = qdisc_dev(qdisc: q);
1524
1525	if (!block)
1526	return;
1527	if (tcf_block_tracks_dev(block, ei))
1528	xa_erase(&block->ports, index: dev->ifindex);
1529	tcf_chain0_head_change_cb_del(block, ei);
1530	tcf_block_owner_del(block, q, binder_type: ei->binder_type);
1531
1532	__tcf_block_put(block, q, ei, rtnl_held: true);
1533	}
1534	EXPORT_SYMBOL(tcf_block_put_ext);
1535
1536	void tcf_block_put(struct tcf_block *block)
1537	{
1538	struct tcf_block_ext_info ei = {0, };
1539
1540	if (!block)
1541	return;
1542	tcf_block_put_ext(block, block->q, &ei);
1543	}
1544
1545	EXPORT_SYMBOL(tcf_block_put);
1546
1547	static int
1548	tcf_block_playback_offloads(struct tcf_block *block, flow_setup_cb_t *cb,
1549	void *cb_priv, bool add, bool offload_in_use,
1550	struct netlink_ext_ack *extack)
1551	{
1552	struct tcf_chain *chain, *chain_prev;
1553	struct tcf_proto *tp, *tp_prev;
1554	int err;
1555
1556	lockdep_assert_held(&block->cb_lock);
1557
1558	for (chain = __tcf_get_next_chain(block, NULL);
1559	chain;
1560	chain_prev = chain,
1561	chain = __tcf_get_next_chain(block, chain),
1562	tcf_chain_put(chain: chain_prev)) {
1563	if (chain->tmplt_ops && add)
1564	chain->tmplt_ops->tmplt_reoffload(chain, true, cb,
1565	cb_priv);
1566	for (tp = __tcf_get_next_proto(chain, NULL); tp;
1567	tp_prev = tp,
1568	tp = __tcf_get_next_proto(chain, tp),
1569	tcf_proto_put(tp: tp_prev, rtnl_held: true, NULL)) {
1570	if (tp->ops->reoffload) {
1571	err = tp->ops->reoffload(tp, add, cb, cb_priv,
1572	extack);
1573	if (err && add)
1574	goto err_playback_remove;
1575	} else if (add && offload_in_use) {
1576	err = -EOPNOTSUPP;
1577	NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support");
1578	goto err_playback_remove;
1579	}
1580	}
1581	if (chain->tmplt_ops && !add)
1582	chain->tmplt_ops->tmplt_reoffload(chain, false, cb,
1583	cb_priv);
1584	}
1585
1586	return 0;
1587
1588	err_playback_remove:
1589	tcf_proto_put(tp, rtnl_held: true, NULL);
1590	tcf_chain_put(chain);
1591	tcf_block_playback_offloads(block, cb, cb_priv, add: false, offload_in_use,
1592	extack);
1593	return err;
1594	}
1595
1596	static int tcf_block_bind(struct tcf_block *block,
1597	struct flow_block_offload *bo)
1598	{
1599	struct flow_block_cb *block_cb, *next;
1600	int err, i = 0;
1601
1602	lockdep_assert_held(&block->cb_lock);
1603
1604	list_for_each_entry(block_cb, &bo->cb_list, list) {
1605	err = tcf_block_playback_offloads(block, cb: block_cb->cb,
1606	cb_priv: block_cb->cb_priv, add: true,
1607	offload_in_use: tcf_block_offload_in_use(block),
1608	extack: bo->extack);
1609	if (err)
1610	goto err_unroll;
1611	if (!bo->unlocked_driver_cb)
1612	block->lockeddevcnt++;
1613
1614	i++;
1615	}
1616	list_splice(list: &bo->cb_list, head: &block->flow_block.cb_list);
1617
1618	return 0;
1619
1620	err_unroll:
1621	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1622	list_del(entry: &block_cb->driver_list);
1623	if (i-- > 0) {
1624	list_del(entry: &block_cb->list);
1625	tcf_block_playback_offloads(block, cb: block_cb->cb,
1626	cb_priv: block_cb->cb_priv, add: false,
1627	offload_in_use: tcf_block_offload_in_use(block),
1628	NULL);
1629	if (!bo->unlocked_driver_cb)
1630	block->lockeddevcnt--;
1631	}
1632	flow_block_cb_free(block_cb);
1633	}
1634
1635	return err;
1636	}
1637
1638	static void tcf_block_unbind(struct tcf_block *block,
1639	struct flow_block_offload *bo)
1640	{
1641	struct flow_block_cb *block_cb, *next;
1642
1643	lockdep_assert_held(&block->cb_lock);
1644
1645	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1646	tcf_block_playback_offloads(block, cb: block_cb->cb,
1647	cb_priv: block_cb->cb_priv, add: false,
1648	offload_in_use: tcf_block_offload_in_use(block),
1649	NULL);
1650	list_del(entry: &block_cb->list);
1651	flow_block_cb_free(block_cb);
1652	if (!bo->unlocked_driver_cb)
1653	block->lockeddevcnt--;
1654	}
1655	}
1656
1657	static int tcf_block_setup(struct tcf_block *block,
1658	struct flow_block_offload *bo)
1659	{
1660	int err;
1661
1662	switch (bo->command) {
1663	case FLOW_BLOCK_BIND:
1664	err = tcf_block_bind(block, bo);
1665	break;
1666	case FLOW_BLOCK_UNBIND:
1667	err = 0;
1668	tcf_block_unbind(block, bo);
1669	break;
1670	default:
1671	WARN_ON_ONCE(1);
1672	err = -EOPNOTSUPP;
1673	}
1674
1675	return err;
1676	}
1677
1678	/* Main classifier routine: scans classifier chain attached
1679	* to this qdisc, (optionally) tests for protocol and asks
1680	* specific classifiers.
1681	*/
1682	static inline int __tcf_classify(struct sk_buff *skb,
1683	const struct tcf_proto *tp,
1684	const struct tcf_proto *orig_tp,
1685	struct tcf_result *res,
1686	bool compat_mode,
1687	struct tcf_exts_miss_cookie_node *n,
1688	int act_index,
1689	u32 *last_executed_chain)
1690	{
1691	#ifdef CONFIG_NET_CLS_ACT
1692	const int max_reclassify_loop = 16;
1693	const struct tcf_proto *first_tp;
1694	int limit = 0;
1695
1696	reclassify:
1697	#endif
1698	for (; tp; tp = rcu_dereference_bh(tp->next)) {
1699	__be16 protocol = skb_protocol(skb, skip_vlan: false);
1700	int err = 0;
1701
1702	if (n) {
1703	struct tcf_exts *exts;
1704
1705	if (n->tp_prio != tp->prio)
1706	continue;
1707
1708	/* We re-lookup the tp and chain based on index instead
1709	* of having hard refs and locks to them, so do a sanity
1710	* check if any of tp,chain,exts was replaced by the
1711	* time we got here with a cookie from hardware.
1712	*/
1713	if (unlikely(n->tp != tp || n->tp->chain != n->chain ||
1714	!tp->ops->get_exts)) {
1715	tcf_set_drop_reason(skb,
1716	reason: SKB_DROP_REASON_TC_COOKIE_ERROR);
1717	return TC_ACT_SHOT;
1718	}
1719
1720	exts = tp->ops->get_exts(tp, n->handle);
1721	if (unlikely(!exts || n->exts != exts)) {
1722	tcf_set_drop_reason(skb,
1723	reason: SKB_DROP_REASON_TC_COOKIE_ERROR);
1724	return TC_ACT_SHOT;
1725	}
1726
1727	n = NULL;
1728	err = tcf_exts_exec_ex(skb, exts, act_index, res);
1729	} else {
1730	if (tp->protocol != protocol &&
1731	tp->protocol != htons(ETH_P_ALL))
1732	continue;
1733
1734	err = tc_classify(skb, tp, res);
1735	}
1736	#ifdef CONFIG_NET_CLS_ACT
1737	if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
1738	first_tp = orig_tp;
1739	*last_executed_chain = first_tp->chain->index;
1740	goto reset;
1741	} else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) {
1742	first_tp = res->goto_tp;
1743	*last_executed_chain = err & TC_ACT_EXT_VAL_MASK;
1744	goto reset;
1745	}
1746	#endif
1747	if (err >= 0)
1748	return err;
1749	}
1750
1751	if (unlikely(n)) {
1752	tcf_set_drop_reason(skb,
1753	reason: SKB_DROP_REASON_TC_COOKIE_ERROR);
1754	return TC_ACT_SHOT;
1755	}
1756
1757	return TC_ACT_UNSPEC; /* signal: continue lookup */
1758	#ifdef CONFIG_NET_CLS_ACT
1759	reset:
1760	if (unlikely(limit++ >= max_reclassify_loop)) {
1761	net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n",
1762	tp->chain->block->index,
1763	tp->prio & 0xffff,
1764	ntohs(tp->protocol));
1765	tcf_set_drop_reason(skb,
1766	reason: SKB_DROP_REASON_TC_RECLASSIFY_LOOP);
1767	return TC_ACT_SHOT;
1768	}
1769
1770	tp = first_tp;
1771	goto reclassify;
1772	#endif
1773	}
1774
1775	int tcf_classify(struct sk_buff *skb,
1776	const struct tcf_block *block,
1777	const struct tcf_proto *tp,
1778	struct tcf_result *res, bool compat_mode)
1779	{
1780	#if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
1781	u32 last_executed_chain = 0;
1782
1783	return __tcf_classify(skb, tp, tp, res, compat_mode, NULL, 0,
1784	&last_executed_chain);
1785	#else
1786	u32 last_executed_chain = tp ? tp->chain->index : 0;
1787	struct tcf_exts_miss_cookie_node *n = NULL;
1788	const struct tcf_proto *orig_tp = tp;
1789	struct tc_skb_ext *ext;
1790	int act_index = 0;
1791	int ret;
1792
1793	if (block) {
1794	ext = skb_ext_find(skb, id: TC_SKB_EXT);
1795
1796	if (ext && (ext->chain || ext->act_miss)) {
1797	struct tcf_chain *fchain;
1798	u32 chain;
1799
1800	if (ext->act_miss) {
1801	n = tcf_exts_miss_cookie_lookup(miss_cookie: ext->act_miss_cookie,
1802	act_index: &act_index);
1803	if (!n) {
1804	tcf_set_drop_reason(skb,
1805	reason: SKB_DROP_REASON_TC_COOKIE_ERROR);
1806	return TC_ACT_SHOT;
1807	}
1808
1809	chain = n->chain_index;
1810	} else {
1811	chain = ext->chain;
1812	}
1813
1814	fchain = tcf_chain_lookup_rcu(block, chain_index: chain);
1815	if (!fchain) {
1816	tcf_set_drop_reason(skb,
1817	reason: SKB_DROP_REASON_TC_CHAIN_NOTFOUND);
1818
1819	return TC_ACT_SHOT;
1820	}
1821
1822	/* Consume, so cloned/redirect skbs won't inherit ext */
1823	skb_ext_del(skb, id: TC_SKB_EXT);
1824
1825	tp = rcu_dereference_bh(fchain->filter_chain);
1826	last_executed_chain = fchain->index;
1827	}
1828	}
1829
1830	ret = __tcf_classify(skb, tp, orig_tp, res, compat_mode, n, act_index,
1831	last_executed_chain: &last_executed_chain);
1832
1833	if (tc_skb_ext_tc_enabled()) {
1834	/* If we missed on some chain */
1835	if (ret == TC_ACT_UNSPEC && last_executed_chain) {
1836	struct tc_skb_cb *cb = tc_skb_cb(skb);
1837
1838	ext = tc_skb_ext_alloc(skb);
1839	if (WARN_ON_ONCE(!ext)) {
1840	tcf_set_drop_reason(skb, reason: SKB_DROP_REASON_NOMEM);
1841	return TC_ACT_SHOT;
1842	}
1843	ext->chain = last_executed_chain;
1844	ext->mru = cb->mru;
1845	ext->post_ct = cb->post_ct;
1846	ext->post_ct_snat = cb->post_ct_snat;
1847	ext->post_ct_dnat = cb->post_ct_dnat;
1848	ext->zone = cb->zone;
1849	}
1850	}
1851
1852	return ret;
1853	#endif
1854	}
1855	EXPORT_SYMBOL(tcf_classify);
1856
1857	struct tcf_chain_info {
1858	struct tcf_proto __rcu **pprev;
1859	struct tcf_proto __rcu *next;
1860	};
1861
1862	static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain,
1863	struct tcf_chain_info *chain_info)
1864	{
1865	return tcf_chain_dereference(*chain_info->pprev, chain);
1866	}
1867
1868	static int tcf_chain_tp_insert(struct tcf_chain *chain,
1869	struct tcf_chain_info *chain_info,
1870	struct tcf_proto *tp)
1871	{
1872	if (chain->flushing)
1873	return -EAGAIN;
1874
1875	RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
1876	if (*chain_info->pprev == chain->filter_chain)
1877	tcf_chain0_head_change(chain, tp_head: tp);
1878	tcf_proto_get(tp);
1879	rcu_assign_pointer(*chain_info->pprev, tp);
1880
1881	return 0;
1882	}
1883
1884	static void tcf_chain_tp_remove(struct tcf_chain *chain,
1885	struct tcf_chain_info *chain_info,
1886	struct tcf_proto *tp)
1887	{
1888	struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain);
1889
1890	tcf_proto_mark_delete(tp);
1891	if (tp == chain->filter_chain)
1892	tcf_chain0_head_change(chain, tp_head: next);
1893	RCU_INIT_POINTER(*chain_info->pprev, next);
1894	}
1895
1896	static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1897	struct tcf_chain_info *chain_info,
1898	u32 protocol, u32 prio,
1899	bool prio_allocate);
1900
1901	/* Try to insert new proto.
1902	* If proto with specified priority already exists, free new proto
1903	* and return existing one.
1904	*/
1905
1906	static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain,
1907	struct tcf_proto *tp_new,
1908	u32 protocol, u32 prio,
1909	bool rtnl_held)
1910	{
1911	struct tcf_chain_info chain_info;
1912	struct tcf_proto *tp;
1913	int err = 0;
1914
1915	mutex_lock(&chain->filter_chain_lock);
1916
1917	if (tcf_proto_exists_destroying(chain, tp: tp_new)) {
1918	mutex_unlock(lock: &chain->filter_chain_lock);
1919	tcf_proto_destroy(tp: tp_new, rtnl_held, sig_destroy: false, NULL);
1920	return ERR_PTR(error: -EAGAIN);
1921	}
1922
1923	tp = tcf_chain_tp_find(chain, chain_info: &chain_info,
1924	protocol, prio, prio_allocate: false);
1925	if (!tp)
1926	err = tcf_chain_tp_insert(chain, chain_info: &chain_info, tp: tp_new);
1927	mutex_unlock(lock: &chain->filter_chain_lock);
1928
1929	if (tp) {
1930	tcf_proto_destroy(tp: tp_new, rtnl_held, sig_destroy: false, NULL);
1931	tp_new = tp;
1932	} else if (err) {
1933	tcf_proto_destroy(tp: tp_new, rtnl_held, sig_destroy: false, NULL);
1934	tp_new = ERR_PTR(error: err);
1935	}
1936
1937	return tp_new;
1938	}
1939
1940	static void tcf_chain_tp_delete_empty(struct tcf_chain *chain,
1941	struct tcf_proto *tp, bool rtnl_held,
1942	struct netlink_ext_ack *extack)
1943	{
1944	struct tcf_chain_info chain_info;
1945	struct tcf_proto *tp_iter;
1946	struct tcf_proto **pprev;
1947	struct tcf_proto *next;
1948
1949	mutex_lock(&chain->filter_chain_lock);
1950
1951	/* Atomically find and remove tp from chain. */
1952	for (pprev = &chain->filter_chain;
1953	(tp_iter = tcf_chain_dereference(*pprev, chain));
1954	pprev = &tp_iter->next) {
1955	if (tp_iter == tp) {
1956	chain_info.pprev = pprev;
1957	chain_info.next = tp_iter->next;
1958	WARN_ON(tp_iter->deleting);
1959	break;
1960	}
1961	}
1962	/* Verify that tp still exists and no new filters were inserted
1963	* concurrently.
1964	* Mark tp for deletion if it is empty.
1965	*/
1966	if (!tp_iter || !tcf_proto_check_delete(tp)) {
1967	mutex_unlock(lock: &chain->filter_chain_lock);
1968	return;
1969	}
1970
1971	tcf_proto_signal_destroying(chain, tp);
1972	next = tcf_chain_dereference(chain_info.next, chain);
1973	if (tp == chain->filter_chain)
1974	tcf_chain0_head_change(chain, tp_head: next);
1975	RCU_INIT_POINTER(*chain_info.pprev, next);
1976	mutex_unlock(lock: &chain->filter_chain_lock);
1977
1978	tcf_proto_put(tp, rtnl_held, extack);
1979	}
1980
1981	static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1982	struct tcf_chain_info *chain_info,
1983	u32 protocol, u32 prio,
1984	bool prio_allocate)
1985	{
1986	struct tcf_proto **pprev;
1987	struct tcf_proto *tp;
1988
1989	/* Check the chain for existence of proto-tcf with this priority */
1990	for (pprev = &chain->filter_chain;
1991	(tp = tcf_chain_dereference(*pprev, chain));
1992	pprev = &tp->next) {
1993	if (tp->prio >= prio) {
1994	if (tp->prio == prio) {
1995	if (prio_allocate ||
1996	(tp->protocol != protocol && protocol))
1997	return ERR_PTR(error: -EINVAL);
1998	} else {
1999	tp = NULL;
2000	}
2001	break;
2002	}
2003	}
2004	chain_info->pprev = pprev;
2005	if (tp) {
2006	chain_info->next = tp->next;
2007	tcf_proto_get(tp);
2008	} else {
2009	chain_info->next = NULL;
2010	}
2011	return tp;
2012	}
2013
2014	static int tcf_fill_node(struct net *net, struct sk_buff *skb,
2015	struct tcf_proto *tp, struct tcf_block *block,
2016	struct Qdisc *q, u32 parent, void *fh,
2017	u32 portid, u32 seq, u16 flags, int event,
2018	bool terse_dump, bool rtnl_held,
2019	struct netlink_ext_ack *extack)
2020	{
2021	struct tcmsg *tcm;
2022	struct nlmsghdr *nlh;
2023	unsigned char *b = skb_tail_pointer(skb);
2024
2025	nlh = nlmsg_put(skb, portid, seq, type: event, payload: sizeof(*tcm), flags);
2026	if (!nlh)
2027	goto out_nlmsg_trim;
2028	tcm = nlmsg_data(nlh);
2029	tcm->tcm_family = AF_UNSPEC;
2030	tcm->tcm__pad1 = 0;
2031	tcm->tcm__pad2 = 0;
2032	if (q) {
2033	tcm->tcm_ifindex = qdisc_dev(qdisc: q)->ifindex;
2034	tcm->tcm_parent = parent;
2035	} else {
2036	tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
2037	tcm->tcm_block_index = block->index;
2038	}
2039	tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
2040	if (nla_put_string(skb, attrtype: TCA_KIND, str: tp->ops->kind))
2041	goto nla_put_failure;
2042	if (nla_put_u32(skb, attrtype: TCA_CHAIN, value: tp->chain->index))
2043	goto nla_put_failure;
2044	if (!fh) {
2045	tcm->tcm_handle = 0;
2046	} else if (terse_dump) {
2047	if (tp->ops->terse_dump) {
2048	if (tp->ops->terse_dump(net, tp, fh, skb, tcm,
2049	rtnl_held) < 0)
2050	goto nla_put_failure;
2051	} else {
2052	goto cls_op_not_supp;
2053	}
2054	} else {
2055	if (tp->ops->dump &&
2056	tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0)
2057	goto nla_put_failure;
2058	}
2059
2060	if (extack && extack->_msg &&
2061	nla_put_string(skb, attrtype: TCA_EXT_WARN_MSG, str: extack->_msg))
2062	goto nla_put_failure;
2063
2064	nlh->nlmsg_len = skb_tail_pointer(skb) - b;
2065
2066	return skb->len;
2067
2068	out_nlmsg_trim:
2069	nla_put_failure:
2070	cls_op_not_supp:
2071	nlmsg_trim(skb, mark: b);
2072	return -1;
2073	}
2074
2075	static int tfilter_notify(struct net *net, struct sk_buff *oskb,
2076	struct nlmsghdr *n, struct tcf_proto *tp,
2077	struct tcf_block *block, struct Qdisc *q,
2078	u32 parent, void *fh, int event, bool unicast,
2079	bool rtnl_held, struct netlink_ext_ack *extack)
2080	{
2081	struct sk_buff *skb;
2082	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2083	int err = 0;
2084
2085	if (!unicast && !rtnl_notify_needed(net, nlflags: n->nlmsg_flags, RTNLGRP_TC))
2086	return 0;
2087
2088	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2089	if (!skb)
2090	return -ENOBUFS;
2091
2092	if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
2093	seq: n->nlmsg_seq, flags: n->nlmsg_flags, event,
2094	terse_dump: false, rtnl_held, extack) <= 0) {
2095	kfree_skb(skb);
2096	return -EINVAL;
2097	}
2098
2099	if (unicast)
2100	err = rtnl_unicast(skb, net, pid: portid);
2101	else
2102	err = rtnetlink_send(skb, net, pid: portid, RTNLGRP_TC,
2103	echo: n->nlmsg_flags & NLM_F_ECHO);
2104	return err;
2105	}
2106
2107	static int tfilter_del_notify(struct net *net, struct sk_buff *oskb,
2108	struct nlmsghdr *n, struct tcf_proto *tp,
2109	struct tcf_block *block, struct Qdisc *q,
2110	u32 parent, void *fh, bool *last, bool rtnl_held,
2111	struct netlink_ext_ack *extack)
2112	{
2113	struct sk_buff *skb;
2114	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2115	int err;
2116
2117	if (!rtnl_notify_needed(net, nlflags: n->nlmsg_flags, RTNLGRP_TC))
2118	return tp->ops->delete(tp, fh, last, rtnl_held, extack);
2119
2120	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2121	if (!skb)
2122	return -ENOBUFS;
2123
2124	if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
2125	seq: n->nlmsg_seq, flags: n->nlmsg_flags, RTM_DELTFILTER,
2126	terse_dump: false, rtnl_held, extack) <= 0) {
2127	NL_SET_ERR_MSG(extack, "Failed to build del event notification");
2128	kfree_skb(skb);
2129	return -EINVAL;
2130	}
2131
2132	err = tp->ops->delete(tp, fh, last, rtnl_held, extack);
2133	if (err) {
2134	kfree_skb(skb);
2135	return err;
2136	}
2137
2138	err = rtnetlink_send(skb, net, pid: portid, RTNLGRP_TC,
2139	echo: n->nlmsg_flags & NLM_F_ECHO);
2140	if (err < 0)
2141	NL_SET_ERR_MSG(extack, "Failed to send filter delete notification");
2142
2143	return err;
2144	}
2145
2146	static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
2147	struct tcf_block *block, struct Qdisc *q,
2148	u32 parent, struct nlmsghdr *n,
2149	struct tcf_chain *chain, int event,
2150	struct netlink_ext_ack *extack)
2151	{
2152	struct tcf_proto *tp;
2153
2154	for (tp = tcf_get_next_proto(chain, NULL);
2155	tp; tp = tcf_get_next_proto(chain, tp))
2156	tfilter_notify(net, oskb, n, tp, block, q, parent, NULL,
2157	event, unicast: false, rtnl_held: true, extack);
2158	}
2159
2160	static void tfilter_put(struct tcf_proto *tp, void *fh)
2161	{
2162	if (tp->ops->put && fh)
2163	tp->ops->put(tp, fh);
2164	}
2165
2166	static bool is_qdisc_ingress(__u32 classid)
2167	{
2168	return (TC_H_MIN(classid) == TC_H_MIN(TC_H_MIN_INGRESS));
2169	}
2170
2171	static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2172	struct netlink_ext_ack *extack)
2173	{
2174	struct net *net = sock_net(sk: skb->sk);
2175	struct nlattr *tca[TCA_MAX + 1];
2176	char name[IFNAMSIZ];
2177	struct tcmsg *t;
2178	u32 protocol;
2179	u32 prio;
2180	bool prio_allocate;
2181	u32 parent;
2182	u32 chain_index;
2183	struct Qdisc *q;
2184	struct tcf_chain_info chain_info;
2185	struct tcf_chain *chain;
2186	struct tcf_block *block;
2187	struct tcf_proto *tp;
2188	unsigned long cl;
2189	void *fh;
2190	int err;
2191	int tp_created;
2192	bool rtnl_held = false;
2193	u32 flags;
2194
2195	replay:
2196	tp_created = 0;
2197
2198	err = nlmsg_parse_deprecated(nlh: n, hdrlen: sizeof(*t), tb: tca, TCA_MAX,
2199	policy: rtm_tca_policy, extack);
2200	if (err < 0)
2201	return err;
2202
2203	t = nlmsg_data(nlh: n);
2204	protocol = TC_H_MIN(t->tcm_info);
2205	prio = TC_H_MAJ(t->tcm_info);
2206	prio_allocate = false;
2207	parent = t->tcm_parent;
2208	tp = NULL;
2209	cl = 0;
2210	block = NULL;
2211	q = NULL;
2212	chain = NULL;
2213	flags = 0;
2214
2215	if (prio == 0) {
2216	/* If no priority is provided by the user,
2217	* we allocate one.
2218	*/
2219	if (n->nlmsg_flags & NLM_F_CREATE) {
2220	prio = TC_H_MAKE(0x80000000U, 0U);
2221	prio_allocate = true;
2222	} else {
2223	NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
2224	return -ENOENT;
2225	}
2226	}
2227
2228	/* Find head of filter chain. */
2229
2230	err = __tcf_qdisc_find(net, q: &q, parent: &parent, ifindex: t->tcm_ifindex, rtnl_held: false, extack);
2231	if (err)
2232	return err;
2233
2234	if (tcf_proto_check_kind(kind: tca[TCA_KIND], name)) {
2235	NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2236	err = -EINVAL;
2237	goto errout;
2238	}
2239
2240	/* Take rtnl mutex if rtnl_held was set to true on previous iteration,
2241	* block is shared (no qdisc found), qdisc is not unlocked, classifier
2242	* type is not specified, classifier is not unlocked.
2243	*/
2244	if (rtnl_held ||
2245	(q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2246	!tcf_proto_is_unlocked(kind: name)) {
2247	rtnl_held = true;
2248	rtnl_lock();
2249	}
2250
2251	err = __tcf_qdisc_cl_find(q, parent, cl: &cl, ifindex: t->tcm_ifindex, extack);
2252	if (err)
2253	goto errout;
2254
2255	block = __tcf_block_find(net, q, cl, ifindex: t->tcm_ifindex, block_index: t->tcm_block_index,
2256	extack);
2257	if (IS_ERR(ptr: block)) {
2258	err = PTR_ERR(ptr: block);
2259	goto errout;
2260	}
2261	block->classid = parent;
2262
2263	chain_index = tca[TCA_CHAIN] ? nla_get_u32(nla: tca[TCA_CHAIN]) : 0;
2264	if (chain_index > TC_ACT_EXT_VAL_MASK) {
2265	NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2266	err = -EINVAL;
2267	goto errout;
2268	}
2269	chain = tcf_chain_get(block, chain_index, create: true);
2270	if (!chain) {
2271	NL_SET_ERR_MSG(extack, "Cannot create specified filter chain");
2272	err = -ENOMEM;
2273	goto errout;
2274	}
2275
2276	mutex_lock(&chain->filter_chain_lock);
2277	tp = tcf_chain_tp_find(chain, chain_info: &chain_info, protocol,
2278	prio, prio_allocate);
2279	if (IS_ERR(ptr: tp)) {
2280	NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2281	err = PTR_ERR(ptr: tp);
2282	goto errout_locked;
2283	}
2284
2285	if (tp == NULL) {
2286	struct tcf_proto *tp_new = NULL;
2287
2288	if (chain->flushing) {
2289	err = -EAGAIN;
2290	goto errout_locked;
2291	}
2292
2293	/* Proto-tcf does not exist, create new one */
2294
2295	if (tca[TCA_KIND] == NULL || !protocol) {
2296	NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
2297	err = -EINVAL;
2298	goto errout_locked;
2299	}
2300
2301	if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2302	NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
2303	err = -ENOENT;
2304	goto errout_locked;
2305	}
2306
2307	if (prio_allocate)
2308	prio = tcf_auto_prio(tp: tcf_chain_tp_prev(chain,
2309	chain_info: &chain_info));
2310
2311	mutex_unlock(lock: &chain->filter_chain_lock);
2312	tp_new = tcf_proto_create(kind: name, protocol, prio, chain,
2313	rtnl_held, extack);
2314	if (IS_ERR(ptr: tp_new)) {
2315	err = PTR_ERR(ptr: tp_new);
2316	goto errout_tp;
2317	}
2318
2319	tp_created = 1;
2320	tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio,
2321	rtnl_held);
2322	if (IS_ERR(ptr: tp)) {
2323	err = PTR_ERR(ptr: tp);
2324	goto errout_tp;
2325	}
2326	} else {
2327	mutex_unlock(lock: &chain->filter_chain_lock);
2328	}
2329
2330	if (tca[TCA_KIND] && nla_strcmp(nla: tca[TCA_KIND], str: tp->ops->kind)) {
2331	NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2332	err = -EINVAL;
2333	goto errout;
2334	}
2335
2336	fh = tp->ops->get(tp, t->tcm_handle);
2337
2338	if (!fh) {
2339	if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2340	NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
2341	err = -ENOENT;
2342	goto errout;
2343	}
2344	} else if (n->nlmsg_flags & NLM_F_EXCL) {
2345	tfilter_put(tp, fh);
2346	NL_SET_ERR_MSG(extack, "Filter already exists");
2347	err = -EEXIST;
2348	goto errout;
2349	}
2350
2351	if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
2352	tfilter_put(tp, fh);
2353	NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind");
2354	err = -EINVAL;
2355	goto errout;
2356	}
2357
2358	if (!(n->nlmsg_flags & NLM_F_CREATE))
2359	flags |= TCA_ACT_FLAGS_REPLACE;
2360	if (!rtnl_held)
2361	flags |= TCA_ACT_FLAGS_NO_RTNL;
2362	if (is_qdisc_ingress(classid: parent))
2363	flags |= TCA_ACT_FLAGS_AT_INGRESS;
2364	err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
2365	flags, extack);
2366	if (err == 0) {
2367	tfilter_notify(net, oskb: skb, n, tp, block, q, parent, fh,
2368	RTM_NEWTFILTER, unicast: false, rtnl_held, extack);
2369	tfilter_put(tp, fh);
2370	/* q pointer is NULL for shared blocks */
2371	if (q)
2372	q->flags &= ~TCQ_F_CAN_BYPASS;
2373	}
2374
2375	errout:
2376	if (err && tp_created)
2377	tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL);
2378	errout_tp:
2379	if (chain) {
2380	if (tp && !IS_ERR(ptr: tp))
2381	tcf_proto_put(tp, rtnl_held, NULL);
2382	if (!tp_created)
2383	tcf_chain_put(chain);
2384	}
2385	tcf_block_release(q, block, rtnl_held);
2386
2387	if (rtnl_held)
2388	rtnl_unlock();
2389
2390	if (err == -EAGAIN) {
2391	/* Take rtnl lock in case EAGAIN is caused by concurrent flush
2392	* of target chain.
2393	*/
2394	rtnl_held = true;
2395	/* Replay the request. */
2396	goto replay;
2397	}
2398	return err;
2399
2400	errout_locked:
2401	mutex_unlock(lock: &chain->filter_chain_lock);
2402	goto errout;
2403	}
2404
2405	static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2406	struct netlink_ext_ack *extack)
2407	{
2408	struct net *net = sock_net(sk: skb->sk);
2409	struct nlattr *tca[TCA_MAX + 1];
2410	char name[IFNAMSIZ];
2411	struct tcmsg *t;
2412	u32 protocol;
2413	u32 prio;
2414	u32 parent;
2415	u32 chain_index;
2416	struct Qdisc *q = NULL;
2417	struct tcf_chain_info chain_info;
2418	struct tcf_chain *chain = NULL;
2419	struct tcf_block *block = NULL;
2420	struct tcf_proto *tp = NULL;
2421	unsigned long cl = 0;
2422	void *fh = NULL;
2423	int err;
2424	bool rtnl_held = false;
2425
2426	err = nlmsg_parse_deprecated(nlh: n, hdrlen: sizeof(*t), tb: tca, TCA_MAX,
2427	policy: rtm_tca_policy, extack);
2428	if (err < 0)
2429	return err;
2430
2431	t = nlmsg_data(nlh: n);
2432	protocol = TC_H_MIN(t->tcm_info);
2433	prio = TC_H_MAJ(t->tcm_info);
2434	parent = t->tcm_parent;
2435
2436	if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) {
2437	NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set");
2438	return -ENOENT;
2439	}
2440
2441	/* Find head of filter chain. */
2442
2443	err = __tcf_qdisc_find(net, q: &q, parent: &parent, ifindex: t->tcm_ifindex, rtnl_held: false, extack);
2444	if (err)
2445	return err;
2446
2447	if (tcf_proto_check_kind(kind: tca[TCA_KIND], name)) {
2448	NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2449	err = -EINVAL;
2450	goto errout;
2451	}
2452	/* Take rtnl mutex if flushing whole chain, block is shared (no qdisc
2453	* found), qdisc is not unlocked, classifier type is not specified,
2454	* classifier is not unlocked.
2455	*/
2456	if (!prio ||
2457	(q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2458	!tcf_proto_is_unlocked(kind: name)) {
2459	rtnl_held = true;
2460	rtnl_lock();
2461	}
2462
2463	err = __tcf_qdisc_cl_find(q, parent, cl: &cl, ifindex: t->tcm_ifindex, extack);
2464	if (err)
2465	goto errout;
2466
2467	block = __tcf_block_find(net, q, cl, ifindex: t->tcm_ifindex, block_index: t->tcm_block_index,
2468	extack);
2469	if (IS_ERR(ptr: block)) {
2470	err = PTR_ERR(ptr: block);
2471	goto errout;
2472	}
2473
2474	chain_index = tca[TCA_CHAIN] ? nla_get_u32(nla: tca[TCA_CHAIN]) : 0;
2475	if (chain_index > TC_ACT_EXT_VAL_MASK) {
2476	NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2477	err = -EINVAL;
2478	goto errout;
2479	}
2480	chain = tcf_chain_get(block, chain_index, create: false);
2481	if (!chain) {
2482	/* User requested flush on non-existent chain. Nothing to do,
2483	* so just return success.
2484	*/
2485	if (prio == 0) {
2486	err = 0;
2487	goto errout;
2488	}
2489	NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2490	err = -ENOENT;
2491	goto errout;
2492	}
2493
2494	if (prio == 0) {
2495	tfilter_notify_chain(net, oskb: skb, block, q, parent, n,
2496	chain, RTM_DELTFILTER, extack);
2497	tcf_chain_flush(chain, rtnl_held);
2498	err = 0;
2499	goto errout;
2500	}
2501
2502	mutex_lock(&chain->filter_chain_lock);
2503	tp = tcf_chain_tp_find(chain, chain_info: &chain_info, protocol,
2504	prio, prio_allocate: false);
2505	if (!tp || IS_ERR(ptr: tp)) {
2506	NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2507	err = tp ? PTR_ERR(ptr: tp) : -ENOENT;
2508	goto errout_locked;
2509	} else if (tca[TCA_KIND] && nla_strcmp(nla: tca[TCA_KIND], str: tp->ops->kind)) {
2510	NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2511	err = -EINVAL;
2512	goto errout_locked;
2513	} else if (t->tcm_handle == 0) {
2514	tcf_proto_signal_destroying(chain, tp);
2515	tcf_chain_tp_remove(chain, chain_info: &chain_info, tp);
2516	mutex_unlock(lock: &chain->filter_chain_lock);
2517
2518	tcf_proto_put(tp, rtnl_held, NULL);
2519	tfilter_notify(net, oskb: skb, n, tp, block, q, parent, fh,
2520	RTM_DELTFILTER, unicast: false, rtnl_held, extack);
2521	err = 0;
2522	goto errout;
2523	}
2524	mutex_unlock(lock: &chain->filter_chain_lock);
2525
2526	fh = tp->ops->get(tp, t->tcm_handle);
2527
2528	if (!fh) {
2529	NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2530	err = -ENOENT;
2531	} else {
2532	bool last;
2533
2534	err = tfilter_del_notify(net, oskb: skb, n, tp, block, q, parent, fh,
2535	last: &last, rtnl_held, extack);
2536
2537	if (err)
2538	goto errout;
2539	if (last)
2540	tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack);
2541	}
2542
2543	errout:
2544	if (chain) {
2545	if (tp && !IS_ERR(ptr: tp))
2546	tcf_proto_put(tp, rtnl_held, NULL);
2547	tcf_chain_put(chain);
2548	}
2549	tcf_block_release(q, block, rtnl_held);
2550
2551	if (rtnl_held)
2552	rtnl_unlock();
2553
2554	return err;
2555
2556	errout_locked:
2557	mutex_unlock(lock: &chain->filter_chain_lock);
2558	goto errout;
2559	}
2560
2561	static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2562	struct netlink_ext_ack *extack)
2563	{
2564	struct net *net = sock_net(sk: skb->sk);
2565	struct nlattr *tca[TCA_MAX + 1];
2566	char name[IFNAMSIZ];
2567	struct tcmsg *t;
2568	u32 protocol;
2569	u32 prio;
2570	u32 parent;
2571	u32 chain_index;
2572	struct Qdisc *q = NULL;
2573	struct tcf_chain_info chain_info;
2574	struct tcf_chain *chain = NULL;
2575	struct tcf_block *block = NULL;
2576	struct tcf_proto *tp = NULL;
2577	unsigned long cl = 0;
2578	void *fh = NULL;
2579	int err;
2580	bool rtnl_held = false;
2581
2582	err = nlmsg_parse_deprecated(nlh: n, hdrlen: sizeof(*t), tb: tca, TCA_MAX,
2583	policy: rtm_tca_policy, extack);
2584	if (err < 0)
2585	return err;
2586
2587	t = nlmsg_data(nlh: n);
2588	protocol = TC_H_MIN(t->tcm_info);
2589	prio = TC_H_MAJ(t->tcm_info);
2590	parent = t->tcm_parent;
2591
2592	if (prio == 0) {
2593	NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
2594	return -ENOENT;
2595	}
2596
2597	/* Find head of filter chain. */
2598
2599	err = __tcf_qdisc_find(net, q: &q, parent: &parent, ifindex: t->tcm_ifindex, rtnl_held: false, extack);
2600	if (err)
2601	return err;
2602
2603	if (tcf_proto_check_kind(kind: tca[TCA_KIND], name)) {
2604	NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2605	err = -EINVAL;
2606	goto errout;
2607	}
2608	/* Take rtnl mutex if block is shared (no qdisc found), qdisc is not
2609	* unlocked, classifier type is not specified, classifier is not
2610	* unlocked.
2611	*/
2612	if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2613	!tcf_proto_is_unlocked(kind: name)) {
2614	rtnl_held = true;
2615	rtnl_lock();
2616	}
2617
2618	err = __tcf_qdisc_cl_find(q, parent, cl: &cl, ifindex: t->tcm_ifindex, extack);
2619	if (err)
2620	goto errout;
2621
2622	block = __tcf_block_find(net, q, cl, ifindex: t->tcm_ifindex, block_index: t->tcm_block_index,
2623	extack);
2624	if (IS_ERR(ptr: block)) {
2625	err = PTR_ERR(ptr: block);
2626	goto errout;
2627	}
2628
2629	chain_index = tca[TCA_CHAIN] ? nla_get_u32(nla: tca[TCA_CHAIN]) : 0;
2630	if (chain_index > TC_ACT_EXT_VAL_MASK) {
2631	NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2632	err = -EINVAL;
2633	goto errout;
2634	}
2635	chain = tcf_chain_get(block, chain_index, create: false);
2636	if (!chain) {
2637	NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2638	err = -EINVAL;
2639	goto errout;
2640	}
2641
2642	mutex_lock(&chain->filter_chain_lock);
2643	tp = tcf_chain_tp_find(chain, chain_info: &chain_info, protocol,
2644	prio, prio_allocate: false);
2645	mutex_unlock(lock: &chain->filter_chain_lock);
2646	if (!tp || IS_ERR(ptr: tp)) {
2647	NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2648	err = tp ? PTR_ERR(ptr: tp) : -ENOENT;
2649	goto errout;
2650	} else if (tca[TCA_KIND] && nla_strcmp(nla: tca[TCA_KIND], str: tp->ops->kind)) {
2651	NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2652	err = -EINVAL;
2653	goto errout;
2654	}
2655
2656	fh = tp->ops->get(tp, t->tcm_handle);
2657
2658	if (!fh) {
2659	NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2660	err = -ENOENT;
2661	} else {
2662	err = tfilter_notify(net, oskb: skb, n, tp, block, q, parent,
2663	fh, RTM_NEWTFILTER, unicast: true, rtnl_held, NULL);
2664	if (err < 0)
2665	NL_SET_ERR_MSG(extack, "Failed to send filter notify message");
2666	}
2667
2668	tfilter_put(tp, fh);
2669	errout:
2670	if (chain) {
2671	if (tp && !IS_ERR(ptr: tp))
2672	tcf_proto_put(tp, rtnl_held, NULL);
2673	tcf_chain_put(chain);
2674	}
2675	tcf_block_release(q, block, rtnl_held);
2676
2677	if (rtnl_held)
2678	rtnl_unlock();
2679
2680	return err;
2681	}
2682
2683	struct tcf_dump_args {
2684	struct tcf_walker w;
2685	struct sk_buff *skb;
2686	struct netlink_callback *cb;
2687	struct tcf_block *block;
2688	struct Qdisc *q;
2689	u32 parent;
2690	bool terse_dump;
2691	};
2692
2693	static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
2694	{
2695	struct tcf_dump_args *a = (void *)arg;
2696	struct net *net = sock_net(sk: a->skb->sk);
2697
2698	return tcf_fill_node(net, skb: a->skb, tp, block: a->block, q: a->q, parent: a->parent,
2699	fh: n, NETLINK_CB(a->cb->skb).portid,
2700	seq: a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
2701	RTM_NEWTFILTER, terse_dump: a->terse_dump, rtnl_held: true, NULL);
2702	}
2703
2704	static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
2705	struct sk_buff *skb, struct netlink_callback *cb,
2706	long index_start, long *p_index, bool terse)
2707	{
2708	struct net *net = sock_net(sk: skb->sk);
2709	struct tcf_block *block = chain->block;
2710	struct tcmsg *tcm = nlmsg_data(nlh: cb->nlh);
2711	struct tcf_proto *tp, *tp_prev;
2712	struct tcf_dump_args arg;
2713
2714	for (tp = __tcf_get_next_proto(chain, NULL);
2715	tp;
2716	tp_prev = tp,
2717	tp = __tcf_get_next_proto(chain, tp),
2718	tcf_proto_put(tp: tp_prev, rtnl_held: true, NULL),
2719	(*p_index)++) {
2720	if (*p_index < index_start)
2721	continue;
2722	if (TC_H_MAJ(tcm->tcm_info) &&
2723	TC_H_MAJ(tcm->tcm_info) != tp->prio)
2724	continue;
2725	if (TC_H_MIN(tcm->tcm_info) &&
2726	TC_H_MIN(tcm->tcm_info) != tp->protocol)
2727	continue;
2728	if (*p_index > index_start)
2729	memset(&cb->args[1], 0,
2730	sizeof(cb->args) - sizeof(cb->args[0]));
2731	if (cb->args[1] == 0) {
2732	if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
2733	NETLINK_CB(cb->skb).portid,
2734	seq: cb->nlh->nlmsg_seq, NLM_F_MULTI,
2735	RTM_NEWTFILTER, terse_dump: false, rtnl_held: true, NULL) <= 0)
2736	goto errout;
2737	cb->args[1] = 1;
2738	}
2739	if (!tp->ops->walk)
2740	continue;
2741	arg.w.fn = tcf_node_dump;
2742	arg.skb = skb;
2743	arg.cb = cb;
2744	arg.block = block;
2745	arg.q = q;
2746	arg.parent = parent;
2747	arg.w.stop = 0;
2748	arg.w.skip = cb->args[1] - 1;
2749	arg.w.count = 0;
2750	arg.w.cookie = cb->args[2];
2751	arg.terse_dump = terse;
2752	tp->ops->walk(tp, &arg.w, true);
2753	cb->args[2] = arg.w.cookie;
2754	cb->args[1] = arg.w.count + 1;
2755	if (arg.w.stop)
2756	goto errout;
2757	}
2758	return true;
2759
2760	errout:
2761	tcf_proto_put(tp, rtnl_held: true, NULL);
2762	return false;
2763	}
2764
2765	static const struct nla_policy tcf_tfilter_dump_policy[TCA_MAX + 1] = {
2766	[TCA_CHAIN] = { .type = NLA_U32 },
2767	[TCA_DUMP_FLAGS] = NLA_POLICY_BITFIELD32(TCA_DUMP_FLAGS_TERSE),
2768	};
2769
2770	/* called with RTNL */
2771	static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
2772	{
2773	struct tcf_chain *chain, *chain_prev;
2774	struct net *net = sock_net(sk: skb->sk);
2775	struct nlattr *tca[TCA_MAX + 1];
2776	struct Qdisc *q = NULL;
2777	struct tcf_block *block;
2778	struct tcmsg *tcm = nlmsg_data(nlh: cb->nlh);
2779	bool terse_dump = false;
2780	long index_start;
2781	long index;
2782	u32 parent;
2783	int err;
2784
2785	if (nlmsg_len(nlh: cb->nlh) < sizeof(*tcm))
2786	return skb->len;
2787
2788	err = nlmsg_parse_deprecated(nlh: cb->nlh, hdrlen: sizeof(*tcm), tb: tca, TCA_MAX,
2789	policy: tcf_tfilter_dump_policy, extack: cb->extack);
2790	if (err)
2791	return err;
2792
2793	if (tca[TCA_DUMP_FLAGS]) {
2794	struct nla_bitfield32 flags =
2795	nla_get_bitfield32(nla: tca[TCA_DUMP_FLAGS]);
2796
2797	terse_dump = flags.value & TCA_DUMP_FLAGS_TERSE;
2798	}
2799
2800	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2801	block = tcf_block_refcnt_get(net, block_index: tcm->tcm_block_index);
2802	if (!block)
2803	goto out;
2804	/* If we work with block index, q is NULL and parent value
2805	* will never be used in the following code. The check
2806	* in tcf_fill_node prevents it. However, compiler does not
2807	* see that far, so set parent to zero to silence the warning
2808	* about parent being uninitialized.
2809	*/
2810	parent = 0;
2811	} else {
2812	const struct Qdisc_class_ops *cops;
2813	struct net_device *dev;
2814	unsigned long cl = 0;
2815
2816	dev = __dev_get_by_index(net, ifindex: tcm->tcm_ifindex);
2817	if (!dev)
2818	return skb->len;
2819
2820	parent = tcm->tcm_parent;
2821	if (!parent)
2822	q = rtnl_dereference(dev->qdisc);
2823	else
2824	q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2825	if (!q)
2826	goto out;
2827	cops = q->ops->cl_ops;
2828	if (!cops)
2829	goto out;
2830	if (!cops->tcf_block)
2831	goto out;
2832	if (TC_H_MIN(tcm->tcm_parent)) {
2833	cl = cops->find(q, tcm->tcm_parent);
2834	if (cl == 0)
2835	goto out;
2836	}
2837	block = cops->tcf_block(q, cl, NULL);
2838	if (!block)
2839	goto out;
2840	parent = block->classid;
2841	if (tcf_block_shared(block))
2842	q = NULL;
2843	}
2844
2845	index_start = cb->args[0];
2846	index = 0;
2847
2848	for (chain = __tcf_get_next_chain(block, NULL);
2849	chain;
2850	chain_prev = chain,
2851	chain = __tcf_get_next_chain(block, chain),
2852	tcf_chain_put(chain: chain_prev)) {
2853	if (tca[TCA_CHAIN] &&
2854	nla_get_u32(nla: tca[TCA_CHAIN]) != chain->index)
2855	continue;
2856	if (!tcf_chain_dump(chain, q, parent, skb, cb,
2857	index_start, p_index: &index, terse: terse_dump)) {
2858	tcf_chain_put(chain);
2859	err = -EMSGSIZE;
2860	break;
2861	}
2862	}
2863
2864	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
2865	tcf_block_refcnt_put(block, rtnl_held: true);
2866	cb->args[0] = index;
2867
2868	out:
2869	/* If we did no progress, the error (EMSGSIZE) is real */
2870	if (skb->len == 0 && err)
2871	return err;
2872	return skb->len;
2873	}
2874
2875	static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops,
2876	void *tmplt_priv, u32 chain_index,
2877	struct net *net, struct sk_buff *skb,
2878	struct tcf_block *block,
2879	u32 portid, u32 seq, u16 flags, int event,
2880	struct netlink_ext_ack *extack)
2881	{
2882	unsigned char *b = skb_tail_pointer(skb);
2883	const struct tcf_proto_ops *ops;
2884	struct nlmsghdr *nlh;
2885	struct tcmsg *tcm;
2886	void *priv;
2887
2888	ops = tmplt_ops;
2889	priv = tmplt_priv;
2890
2891	nlh = nlmsg_put(skb, portid, seq, type: event, payload: sizeof(*tcm), flags);
2892	if (!nlh)
2893	goto out_nlmsg_trim;
2894	tcm = nlmsg_data(nlh);
2895	tcm->tcm_family = AF_UNSPEC;
2896	tcm->tcm__pad1 = 0;
2897	tcm->tcm__pad2 = 0;
2898	tcm->tcm_handle = 0;
2899	if (block->q) {
2900	tcm->tcm_ifindex = qdisc_dev(qdisc: block->q)->ifindex;
2901	tcm->tcm_parent = block->q->handle;
2902	} else {
2903	tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
2904	tcm->tcm_block_index = block->index;
2905	}
2906
2907	if (nla_put_u32(skb, attrtype: TCA_CHAIN, value: chain_index))
2908	goto nla_put_failure;
2909
2910	if (ops) {
2911	if (nla_put_string(skb, attrtype: TCA_KIND, str: ops->kind))
2912	goto nla_put_failure;
2913	if (ops->tmplt_dump(skb, net, priv) < 0)
2914	goto nla_put_failure;
2915	}
2916
2917	if (extack && extack->_msg &&
2918	nla_put_string(skb, attrtype: TCA_EXT_WARN_MSG, str: extack->_msg))
2919	goto out_nlmsg_trim;
2920
2921	nlh->nlmsg_len = skb_tail_pointer(skb) - b;
2922
2923	return skb->len;
2924
2925	out_nlmsg_trim:
2926	nla_put_failure:
2927	nlmsg_trim(skb, mark: b);
2928	return -EMSGSIZE;
2929	}
2930
2931	static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
2932	u32 seq, u16 flags, int event, bool unicast,
2933	struct netlink_ext_ack *extack)
2934	{
2935	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2936	struct tcf_block *block = chain->block;
2937	struct net *net = block->net;
2938	struct sk_buff *skb;
2939	int err = 0;
2940
2941	if (!unicast && !rtnl_notify_needed(net, nlflags: flags, RTNLGRP_TC))
2942	return 0;
2943
2944	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2945	if (!skb)
2946	return -ENOBUFS;
2947
2948	if (tc_chain_fill_node(tmplt_ops: chain->tmplt_ops, tmplt_priv: chain->tmplt_priv,
2949	chain_index: chain->index, net, skb, block, portid,
2950	seq, flags, event, extack) <= 0) {
2951	kfree_skb(skb);
2952	return -EINVAL;
2953	}
2954
2955	if (unicast)
2956	err = rtnl_unicast(skb, net, pid: portid);
2957	else
2958	err = rtnetlink_send(skb, net, pid: portid, RTNLGRP_TC,
2959	echo: flags & NLM_F_ECHO);
2960
2961	return err;
2962	}
2963
2964	static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
2965	void *tmplt_priv, u32 chain_index,
2966	struct tcf_block *block, struct sk_buff *oskb,
2967	u32 seq, u16 flags)
2968	{
2969	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2970	struct net *net = block->net;
2971	struct sk_buff *skb;
2972
2973	if (!rtnl_notify_needed(net, nlflags: flags, RTNLGRP_TC))
2974	return 0;
2975
2976	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2977	if (!skb)
2978	return -ENOBUFS;
2979
2980	if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb,
2981	block, portid, seq, flags, RTM_DELCHAIN, NULL) <= 0) {
2982	kfree_skb(skb);
2983	return -EINVAL;
2984	}
2985
2986	return rtnetlink_send(skb, net, pid: portid, RTNLGRP_TC, echo: flags & NLM_F_ECHO);
2987	}
2988
2989	static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
2990	struct nlattr **tca,
2991	struct netlink_ext_ack *extack)
2992	{
2993	const struct tcf_proto_ops *ops;
2994	char name[IFNAMSIZ];
2995	void *tmplt_priv;
2996
2997	/* If kind is not set, user did not specify template. */
2998	if (!tca[TCA_KIND])
2999	return 0;
3000
3001	if (tcf_proto_check_kind(kind: tca[TCA_KIND], name)) {
3002	NL_SET_ERR_MSG(extack, "Specified TC chain template name too long");
3003	return -EINVAL;
3004	}
3005
3006	ops = tcf_proto_lookup_ops(kind: name, rtnl_held: true, extack);
3007	if (IS_ERR(ptr: ops))
3008	return PTR_ERR(ptr: ops);
3009	if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump ||
3010	!ops->tmplt_reoffload) {
3011	NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier");
3012	module_put(module: ops->owner);
3013	return -EOPNOTSUPP;
3014	}
3015
3016	tmplt_priv = ops->tmplt_create(net, chain, tca, extack);
3017	if (IS_ERR(ptr: tmplt_priv)) {
3018	module_put(module: ops->owner);
3019	return PTR_ERR(ptr: tmplt_priv);
3020	}
3021	chain->tmplt_ops = ops;
3022	chain->tmplt_priv = tmplt_priv;
3023	return 0;
3024	}
3025
3026	static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
3027	void *tmplt_priv)
3028	{
3029	/* If template ops are set, no work to do for us. */
3030	if (!tmplt_ops)
3031	return;
3032
3033	tmplt_ops->tmplt_destroy(tmplt_priv);
3034	module_put(module: tmplt_ops->owner);
3035	}
3036
3037	/* Add/delete/get a chain */
3038
3039	static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
3040	struct netlink_ext_ack *extack)
3041	{
3042	struct net *net = sock_net(sk: skb->sk);
3043	struct nlattr *tca[TCA_MAX + 1];
3044	struct tcmsg *t;
3045	u32 parent;
3046	u32 chain_index;
3047	struct Qdisc *q;
3048	struct tcf_chain *chain;
3049	struct tcf_block *block;
3050	unsigned long cl;
3051	int err;
3052
3053	replay:
3054	q = NULL;
3055	err = nlmsg_parse_deprecated(nlh: n, hdrlen: sizeof(*t), tb: tca, TCA_MAX,
3056	policy: rtm_tca_policy, extack);
3057	if (err < 0)
3058	return err;
3059
3060	t = nlmsg_data(nlh: n);
3061	parent = t->tcm_parent;
3062	cl = 0;
3063
3064	block = tcf_block_find(net, q: &q, parent: &parent, cl: &cl,
3065	ifindex: t->tcm_ifindex, block_index: t->tcm_block_index, extack);
3066	if (IS_ERR(ptr: block))
3067	return PTR_ERR(ptr: block);
3068
3069	chain_index = tca[TCA_CHAIN] ? nla_get_u32(nla: tca[TCA_CHAIN]) : 0;
3070	if (chain_index > TC_ACT_EXT_VAL_MASK) {
3071	NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
3072	err = -EINVAL;
3073	goto errout_block;
3074	}
3075
3076	mutex_lock(&block->lock);
3077	chain = tcf_chain_lookup(block, chain_index);
3078	if (n->nlmsg_type == RTM_NEWCHAIN) {
3079	if (chain) {
3080	if (tcf_chain_held_by_acts_only(chain)) {
3081	/* The chain exists only because there is
3082	* some action referencing it.
3083	*/
3084	tcf_chain_hold(chain);
3085	} else {
3086	NL_SET_ERR_MSG(extack, "Filter chain already exists");
3087	err = -EEXIST;
3088	goto errout_block_locked;
3089	}
3090	} else {
3091	if (!(n->nlmsg_flags & NLM_F_CREATE)) {
3092	NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
3093	err = -ENOENT;
3094	goto errout_block_locked;
3095	}
3096	chain = tcf_chain_create(block, chain_index);
3097	if (!chain) {
3098	NL_SET_ERR_MSG(extack, "Failed to create filter chain");
3099	err = -ENOMEM;
3100	goto errout_block_locked;
3101	}
3102	}
3103	} else {
3104	if (!chain || tcf_chain_held_by_acts_only(chain)) {
3105	NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
3106	err = -EINVAL;
3107	goto errout_block_locked;
3108	}
3109	tcf_chain_hold(chain);
3110	}
3111
3112	if (n->nlmsg_type == RTM_NEWCHAIN) {
3113	/* Modifying chain requires holding parent block lock. In case
3114	* the chain was successfully added, take a reference to the
3115	* chain. This ensures that an empty chain does not disappear at
3116	* the end of this function.
3117	*/
3118	tcf_chain_hold(chain);
3119	chain->explicitly_created = true;
3120	}
3121	mutex_unlock(lock: &block->lock);
3122
3123	switch (n->nlmsg_type) {
3124	case RTM_NEWCHAIN:
3125	err = tc_chain_tmplt_add(chain, net, tca, extack);
3126	if (err) {
3127	tcf_chain_put_explicitly_created(chain);
3128	goto errout;
3129	}
3130
3131	tc_chain_notify(chain, NULL, seq: 0, NLM_F_CREATE | NLM_F_EXCL,
3132	RTM_NEWCHAIN, unicast: false, extack);
3133	break;
3134	case RTM_DELCHAIN:
3135	tfilter_notify_chain(net, oskb: skb, block, q, parent, n,
3136	chain, RTM_DELTFILTER, extack);
3137	/* Flush the chain first as the user requested chain removal. */
3138	tcf_chain_flush(chain, rtnl_held: true);
3139	/* In case the chain was successfully deleted, put a reference
3140	* to the chain previously taken during addition.
3141	*/
3142	tcf_chain_put_explicitly_created(chain);
3143	break;
3144	case RTM_GETCHAIN:
3145	err = tc_chain_notify(chain, oskb: skb, seq: n->nlmsg_seq,
3146	flags: n->nlmsg_flags, event: n->nlmsg_type, unicast: true, extack);
3147	if (err < 0)
3148	NL_SET_ERR_MSG(extack, "Failed to send chain notify message");
3149	break;
3150	default:
3151	err = -EOPNOTSUPP;
3152	NL_SET_ERR_MSG(extack, "Unsupported message type");
3153	goto errout;
3154	}
3155
3156	errout:
3157	tcf_chain_put(chain);
3158	errout_block:
3159	tcf_block_release(q, block, rtnl_held: true);
3160	if (err == -EAGAIN)
3161	/* Replay the request. */
3162	goto replay;
3163	return err;
3164
3165	errout_block_locked:
3166	mutex_unlock(lock: &block->lock);
3167	goto errout_block;
3168	}
3169
3170	/* called with RTNL */
3171	static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
3172	{
3173	struct net *net = sock_net(sk: skb->sk);
3174	struct nlattr *tca[TCA_MAX + 1];
3175	struct Qdisc *q = NULL;
3176	struct tcf_block *block;
3177	struct tcmsg *tcm = nlmsg_data(nlh: cb->nlh);
3178	struct tcf_chain *chain;
3179	long index_start;
3180	long index;
3181	int err;
3182
3183	if (nlmsg_len(nlh: cb->nlh) < sizeof(*tcm))
3184	return skb->len;
3185
3186	err = nlmsg_parse_deprecated(nlh: cb->nlh, hdrlen: sizeof(*tcm), tb: tca, TCA_MAX,
3187	policy: rtm_tca_policy, extack: cb->extack);
3188	if (err)
3189	return err;
3190
3191	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
3192	block = tcf_block_refcnt_get(net, block_index: tcm->tcm_block_index);
3193	if (!block)
3194	goto out;
3195	} else {
3196	const struct Qdisc_class_ops *cops;
3197	struct net_device *dev;
3198	unsigned long cl = 0;
3199
3200	dev = __dev_get_by_index(net, ifindex: tcm->tcm_ifindex);
3201	if (!dev)
3202	return skb->len;
3203
3204	if (!tcm->tcm_parent)
3205	q = rtnl_dereference(dev->qdisc);
3206	else
3207	q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
3208
3209	if (!q)
3210	goto out;
3211	cops = q->ops->cl_ops;
3212	if (!cops)
3213	goto out;
3214	if (!cops->tcf_block)
3215	goto out;
3216	if (TC_H_MIN(tcm->tcm_parent)) {
3217	cl = cops->find(q, tcm->tcm_parent);
3218	if (cl == 0)
3219	goto out;
3220	}
3221	block = cops->tcf_block(q, cl, NULL);
3222	if (!block)
3223	goto out;
3224	if (tcf_block_shared(block))
3225	q = NULL;
3226	}
3227
3228	index_start = cb->args[0];
3229	index = 0;
3230
3231	mutex_lock(&block->lock);
3232	list_for_each_entry(chain, &block->chain_list, list) {
3233	if ((tca[TCA_CHAIN] &&
3234	nla_get_u32(nla: tca[TCA_CHAIN]) != chain->index))
3235	continue;
3236	if (index < index_start) {
3237	index++;
3238	continue;
3239	}
3240	if (tcf_chain_held_by_acts_only(chain))
3241	continue;
3242	err = tc_chain_fill_node(tmplt_ops: chain->tmplt_ops, tmplt_priv: chain->tmplt_priv,
3243	chain_index: chain->index, net, skb, block,
3244	NETLINK_CB(cb->skb).portid,
3245	seq: cb->nlh->nlmsg_seq, NLM_F_MULTI,
3246	RTM_NEWCHAIN, NULL);
3247	if (err <= 0)
3248	break;
3249	index++;
3250	}
3251	mutex_unlock(lock: &block->lock);
3252
3253	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
3254	tcf_block_refcnt_put(block, rtnl_held: true);
3255	cb->args[0] = index;
3256
3257	out:
3258	/* If we did no progress, the error (EMSGSIZE) is real */
3259	if (skb->len == 0 && err)
3260	return err;
3261	return skb->len;
3262	}
3263
3264	int tcf_exts_init_ex(struct tcf_exts *exts, struct net *net, int action,
3265	int police, struct tcf_proto *tp, u32 handle,
3266	bool use_action_miss)
3267	{
3268	int err = 0;
3269
3270	#ifdef CONFIG_NET_CLS_ACT
3271	exts->type = 0;
3272	exts->nr_actions = 0;
3273	exts->miss_cookie_node = NULL;
3274	/* Note: we do not own yet a reference on net.
3275	* This reference might be taken later from tcf_exts_get_net().
3276	*/
3277	exts->net = net;
3278	exts->actions = kcalloc(TCA_ACT_MAX_PRIO, size: sizeof(struct tc_action *),
3279	GFP_KERNEL);
3280	if (!exts->actions)
3281	return -ENOMEM;
3282	#endif
3283
3284	exts->action = action;
3285	exts->police = police;
3286
3287	if (!use_action_miss)
3288	return 0;
3289
3290	err = tcf_exts_miss_cookie_base_alloc(exts, tp, handle);
3291	if (err)
3292	goto err_miss_alloc;
3293
3294	return 0;
3295
3296	err_miss_alloc:
3297	tcf_exts_destroy(exts);
3298	#ifdef CONFIG_NET_CLS_ACT
3299	exts->actions = NULL;
3300	#endif
3301	return err;
3302	}
3303	EXPORT_SYMBOL(tcf_exts_init_ex);
3304
3305	void tcf_exts_destroy(struct tcf_exts *exts)
3306	{
3307	tcf_exts_miss_cookie_base_destroy(exts);
3308
3309	#ifdef CONFIG_NET_CLS_ACT
3310	if (exts->actions) {
3311	tcf_action_destroy(actions: exts->actions, TCA_ACT_UNBIND);
3312	kfree(objp: exts->actions);
3313	}
3314	exts->nr_actions = 0;
3315	#endif
3316	}
3317	EXPORT_SYMBOL(tcf_exts_destroy);
3318
3319	int tcf_exts_validate_ex(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
3320	struct nlattr *rate_tlv, struct tcf_exts *exts,
3321	u32 flags, u32 fl_flags, struct netlink_ext_ack *extack)
3322	{
3323	#ifdef CONFIG_NET_CLS_ACT
3324	{
3325	int init_res[TCA_ACT_MAX_PRIO] = {};
3326	struct tc_action *act;
3327	size_t attr_size = 0;
3328
3329	if (exts->police && tb[exts->police]) {
3330	struct tc_action_ops *a_o;
3331
3332	flags |= TCA_ACT_FLAGS_POLICE | TCA_ACT_FLAGS_BIND;
3333	a_o = tc_action_load_ops(nla: tb[exts->police], flags,
3334	extack);
3335	if (IS_ERR(ptr: a_o))
3336	return PTR_ERR(ptr: a_o);
3337	act = tcf_action_init_1(net, tp, nla: tb[exts->police],
3338	est: rate_tlv, a_o, init_res, flags,
3339	extack);
3340	module_put(module: a_o->owner);
3341	if (IS_ERR(ptr: act))
3342	return PTR_ERR(ptr: act);
3343
3344	act->type = exts->type = TCA_OLD_COMPAT;
3345	exts->actions[0] = act;
3346	exts->nr_actions = 1;
3347	tcf_idr_insert_many(actions: exts->actions, init_res);
3348	} else if (exts->action && tb[exts->action]) {
3349	int err;
3350
3351	flags |= TCA_ACT_FLAGS_BIND;
3352	err = tcf_action_init(net, tp, nla: tb[exts->action],
3353	est: rate_tlv, actions: exts->actions, init_res,
3354	attr_size: &attr_size, flags, fl_flags,
3355	extack);
3356	if (err < 0)
3357	return err;
3358	exts->nr_actions = err;
3359	}
3360	}
3361	#else
3362	if ((exts->action && tb[exts->action]) ||
3363	(exts->police && tb[exts->police])) {
3364	NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)");
3365	return -EOPNOTSUPP;
3366	}
3367	#endif
3368
3369	return 0;
3370	}
3371	EXPORT_SYMBOL(tcf_exts_validate_ex);
3372
3373	int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
3374	struct nlattr *rate_tlv, struct tcf_exts *exts,
3375	u32 flags, struct netlink_ext_ack *extack)
3376	{
3377	return tcf_exts_validate_ex(net, tp, tb, rate_tlv, exts,
3378	flags, 0, extack);
3379	}
3380	EXPORT_SYMBOL(tcf_exts_validate);
3381
3382	void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src)
3383	{
3384	#ifdef CONFIG_NET_CLS_ACT
3385	struct tcf_exts old = *dst;
3386
3387	*dst = *src;
3388	tcf_exts_destroy(&old);
3389	#endif
3390	}
3391	EXPORT_SYMBOL(tcf_exts_change);
3392
3393	#ifdef CONFIG_NET_CLS_ACT
3394	static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts)
3395	{
3396	if (exts->nr_actions == 0)
3397	return NULL;
3398	else
3399	return exts->actions[0];
3400	}
3401	#endif
3402
3403	int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts)
3404	{
3405	#ifdef CONFIG_NET_CLS_ACT
3406	struct nlattr *nest;
3407
3408	if (exts->action && tcf_exts_has_actions(exts)) {
3409	/*
3410	* again for backward compatible mode - we want
3411	* to work with both old and new modes of entering
3412	* tc data even if iproute2 was newer - jhs
3413	*/
3414	if (exts->type != TCA_OLD_COMPAT) {
3415	nest = nla_nest_start_noflag(skb, attrtype: exts->action);
3416	if (nest == NULL)
3417	goto nla_put_failure;
3418
3419	if (tcf_action_dump(skb, actions: exts->actions, bind: 0, ref: 0, terse: false)
3420	< 0)
3421	goto nla_put_failure;
3422	nla_nest_end(skb, start: nest);
3423	} else if (exts->police) {
3424	struct tc_action *act = tcf_exts_first_act(exts);
3425	nest = nla_nest_start_noflag(skb, attrtype: exts->police);
3426	if (nest == NULL || !act)
3427	goto nla_put_failure;
3428	if (tcf_action_dump_old(skb, a: act, 0, 0) < 0)
3429	goto nla_put_failure;
3430	nla_nest_end(skb, start: nest);
3431	}
3432	}
3433	return 0;
3434
3435	nla_put_failure:
3436	nla_nest_cancel(skb, start: nest);
3437	return -1;
3438	#else
3439	return 0;
3440	#endif
3441	}
3442	EXPORT_SYMBOL(tcf_exts_dump);
3443
3444	int tcf_exts_terse_dump(struct sk_buff *skb, struct tcf_exts *exts)
3445	{
3446	#ifdef CONFIG_NET_CLS_ACT
3447	struct nlattr *nest;
3448
3449	if (!exts->action || !tcf_exts_has_actions(exts))
3450	return 0;
3451
3452	nest = nla_nest_start_noflag(skb, attrtype: exts->action);
3453	if (!nest)
3454	goto nla_put_failure;
3455
3456	if (tcf_action_dump(skb, actions: exts->actions, bind: 0, ref: 0, terse: true) < 0)
3457	goto nla_put_failure;
3458	nla_nest_end(skb, start: nest);
3459	return 0;
3460
3461	nla_put_failure:
3462	nla_nest_cancel(skb, start: nest);
3463	return -1;
3464	#else
3465	return 0;
3466	#endif
3467	}
3468	EXPORT_SYMBOL(tcf_exts_terse_dump);
3469
3470	int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts)
3471	{
3472	#ifdef CONFIG_NET_CLS_ACT
3473	struct tc_action *a = tcf_exts_first_act(exts);
3474	if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0)
3475	return -1;
3476	#endif
3477	return 0;
3478	}
3479	EXPORT_SYMBOL(tcf_exts_dump_stats);
3480
3481	static void tcf_block_offload_inc(struct tcf_block *block, u32 *flags)
3482	{
3483	if (*flags & TCA_CLS_FLAGS_IN_HW)
3484	return;
3485	*flags |= TCA_CLS_FLAGS_IN_HW;
3486	atomic_inc(v: &block->offloadcnt);
3487	}
3488
3489	static void tcf_block_offload_dec(struct tcf_block *block, u32 *flags)
3490	{
3491	if (!(*flags & TCA_CLS_FLAGS_IN_HW))
3492	return;
3493	*flags &= ~TCA_CLS_FLAGS_IN_HW;
3494	atomic_dec(v: &block->offloadcnt);
3495	}
3496
3497	static void tc_cls_offload_cnt_update(struct tcf_block *block,
3498	struct tcf_proto *tp, u32 *cnt,
3499	u32 *flags, u32 diff, bool add)
3500	{
3501	lockdep_assert_held(&block->cb_lock);
3502
3503	spin_lock(lock: &tp->lock);
3504	if (add) {
3505	if (!*cnt)
3506	tcf_block_offload_inc(block, flags);
3507	*cnt += diff;
3508	} else {
3509	*cnt -= diff;
3510	if (!*cnt)
3511	tcf_block_offload_dec(block, flags);
3512	}
3513	spin_unlock(lock: &tp->lock);
3514	}
3515
3516	static void
3517	tc_cls_offload_cnt_reset(struct tcf_block *block, struct tcf_proto *tp,
3518	u32 *cnt, u32 *flags)
3519	{
3520	lockdep_assert_held(&block->cb_lock);
3521
3522	spin_lock(lock: &tp->lock);
3523	tcf_block_offload_dec(block, flags);
3524	*cnt = 0;
3525	spin_unlock(lock: &tp->lock);
3526	}
3527
3528	static int
3529	__tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3530	void *type_data, bool err_stop)
3531	{
3532	struct flow_block_cb *block_cb;
3533	int ok_count = 0;
3534	int err;
3535
3536	list_for_each_entry(block_cb, &block->flow_block.cb_list, list) {
3537	err = block_cb->cb(type, type_data, block_cb->cb_priv);
3538	if (err) {
3539	if (err_stop)
3540	return err;
3541	} else {
3542	ok_count++;
3543	}
3544	}
3545	return ok_count;
3546	}
3547
3548	int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3549	void *type_data, bool err_stop, bool rtnl_held)
3550	{
3551	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3552	int ok_count;
3553
3554	retry:
3555	if (take_rtnl)
3556	rtnl_lock();
3557	down_read(sem: &block->cb_lock);
3558	/* Need to obtain rtnl lock if block is bound to devs that require it.
3559	* In block bind code cb_lock is obtained while holding rtnl, so we must
3560	* obtain the locks in same order here.
3561	*/
3562	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3563	up_read(sem: &block->cb_lock);
3564	take_rtnl = true;
3565	goto retry;
3566	}
3567
3568	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3569
3570	up_read(sem: &block->cb_lock);
3571	if (take_rtnl)
3572	rtnl_unlock();
3573	return ok_count;
3574	}
3575	EXPORT_SYMBOL(tc_setup_cb_call);
3576
3577	/* Non-destructive filter add. If filter that wasn't already in hardware is
3578	* successfully offloaded, increment block offloads counter. On failure,
3579	* previously offloaded filter is considered to be intact and offloads counter
3580	* is not decremented.
3581	*/
3582
3583	int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp,
3584	enum tc_setup_type type, void *type_data, bool err_stop,
3585	u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
3586	{
3587	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3588	int ok_count;
3589
3590	retry:
3591	if (take_rtnl)
3592	rtnl_lock();
3593	down_read(sem: &block->cb_lock);
3594	/* Need to obtain rtnl lock if block is bound to devs that require it.
3595	* In block bind code cb_lock is obtained while holding rtnl, so we must
3596	* obtain the locks in same order here.
3597	*/
3598	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3599	up_read(sem: &block->cb_lock);
3600	take_rtnl = true;
3601	goto retry;
3602	}
3603
3604	/* Make sure all netdevs sharing this block are offload-capable. */
3605	if (block->nooffloaddevcnt && err_stop) {
3606	ok_count = -EOPNOTSUPP;
3607	goto err_unlock;
3608	}
3609
3610	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3611	if (ok_count < 0)
3612	goto err_unlock;
3613
3614	if (tp->ops->hw_add)
3615	tp->ops->hw_add(tp, type_data);
3616	if (ok_count > 0)
3617	tc_cls_offload_cnt_update(block, tp, cnt: in_hw_count, flags,
3618	diff: ok_count, add: true);
3619	err_unlock:
3620	up_read(sem: &block->cb_lock);
3621	if (take_rtnl)
3622	rtnl_unlock();
3623	return min(ok_count, 0);
3624	}
3625	EXPORT_SYMBOL(tc_setup_cb_add);
3626
3627	/* Destructive filter replace. If filter that wasn't already in hardware is
3628	* successfully offloaded, increment block offload counter. On failure,
3629	* previously offloaded filter is considered to be destroyed and offload counter
3630	* is decremented.
3631	*/
3632
3633	int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp,
3634	enum tc_setup_type type, void *type_data, bool err_stop,
3635	u32 *old_flags, unsigned int *old_in_hw_count,
3636	u32 *new_flags, unsigned int *new_in_hw_count,
3637	bool rtnl_held)
3638	{
3639	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3640	int ok_count;
3641
3642	retry:
3643	if (take_rtnl)
3644	rtnl_lock();
3645	down_read(sem: &block->cb_lock);
3646	/* Need to obtain rtnl lock if block is bound to devs that require it.
3647	* In block bind code cb_lock is obtained while holding rtnl, so we must
3648	* obtain the locks in same order here.
3649	*/
3650	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3651	up_read(sem: &block->cb_lock);
3652	take_rtnl = true;
3653	goto retry;
3654	}
3655
3656	/* Make sure all netdevs sharing this block are offload-capable. */
3657	if (block->nooffloaddevcnt && err_stop) {
3658	ok_count = -EOPNOTSUPP;
3659	goto err_unlock;
3660	}
3661
3662	tc_cls_offload_cnt_reset(block, tp, cnt: old_in_hw_count, flags: old_flags);
3663	if (tp->ops->hw_del)
3664	tp->ops->hw_del(tp, type_data);
3665
3666	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3667	if (ok_count < 0)
3668	goto err_unlock;
3669
3670	if (tp->ops->hw_add)
3671	tp->ops->hw_add(tp, type_data);
3672	if (ok_count > 0)
3673	tc_cls_offload_cnt_update(block, tp, cnt: new_in_hw_count,
3674	flags: new_flags, diff: ok_count, add: true);
3675	err_unlock:
3676	up_read(sem: &block->cb_lock);
3677	if (take_rtnl)
3678	rtnl_unlock();
3679	return min(ok_count, 0);
3680	}
3681	EXPORT_SYMBOL(tc_setup_cb_replace);
3682
3683	/* Destroy filter and decrement block offload counter, if filter was previously
3684	* offloaded.
3685	*/
3686
3687	int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp,
3688	enum tc_setup_type type, void *type_data, bool err_stop,
3689	u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
3690	{
3691	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3692	int ok_count;
3693
3694	retry:
3695	if (take_rtnl)
3696	rtnl_lock();
3697	down_read(sem: &block->cb_lock);
3698	/* Need to obtain rtnl lock if block is bound to devs that require it.
3699	* In block bind code cb_lock is obtained while holding rtnl, so we must
3700	* obtain the locks in same order here.
3701	*/
3702	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3703	up_read(sem: &block->cb_lock);
3704	take_rtnl = true;
3705	goto retry;
3706	}
3707
3708	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3709
3710	tc_cls_offload_cnt_reset(block, tp, cnt: in_hw_count, flags);
3711	if (tp->ops->hw_del)
3712	tp->ops->hw_del(tp, type_data);
3713
3714	up_read(sem: &block->cb_lock);
3715	if (take_rtnl)
3716	rtnl_unlock();
3717	return min(ok_count, 0);
3718	}
3719	EXPORT_SYMBOL(tc_setup_cb_destroy);
3720
3721	int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp,
3722	bool add, flow_setup_cb_t *cb,
3723	enum tc_setup_type type, void *type_data,
3724	void *cb_priv, u32 *flags, unsigned int *in_hw_count)
3725	{
3726	int err = cb(type, type_data, cb_priv);
3727
3728	if (err) {
3729	if (add && tc_skip_sw(flags: *flags))
3730	return err;
3731	} else {
3732	tc_cls_offload_cnt_update(block, tp, cnt: in_hw_count, flags, diff: 1,
3733	add);
3734	}
3735
3736	return 0;
3737	}
3738	EXPORT_SYMBOL(tc_setup_cb_reoffload);
3739
3740	static int tcf_act_get_user_cookie(struct flow_action_entry *entry,
3741	const struct tc_action *act)
3742	{
3743	struct tc_cookie *user_cookie;
3744	int err = 0;
3745
3746	rcu_read_lock();
3747	user_cookie = rcu_dereference(act->user_cookie);
3748	if (user_cookie) {
3749	entry->user_cookie = flow_action_cookie_create(data: user_cookie->data,
3750	len: user_cookie->len,
3751	GFP_ATOMIC);
3752	if (!entry->user_cookie)
3753	err = -ENOMEM;
3754	}
3755	rcu_read_unlock();
3756	return err;
3757	}
3758
3759	static void tcf_act_put_user_cookie(struct flow_action_entry *entry)
3760	{
3761	flow_action_cookie_destroy(cookie: entry->user_cookie);
3762	}
3763
3764	void tc_cleanup_offload_action(struct flow_action *flow_action)
3765	{
3766	struct flow_action_entry *entry;
3767	int i;
3768
3769	flow_action_for_each(i, entry, flow_action) {
3770	tcf_act_put_user_cookie(entry);
3771	if (entry->destructor)
3772	entry->destructor(entry->destructor_priv);
3773	}
3774	}
3775	EXPORT_SYMBOL(tc_cleanup_offload_action);
3776
3777	static int tc_setup_offload_act(struct tc_action *act,
3778	struct flow_action_entry *entry,
3779	u32 *index_inc,
3780	struct netlink_ext_ack *extack)
3781	{
3782	#ifdef CONFIG_NET_CLS_ACT
3783	if (act->ops->offload_act_setup) {
3784	return act->ops->offload_act_setup(act, entry, index_inc, true,
3785	extack);
3786	} else {
3787	NL_SET_ERR_MSG(extack, "Action does not support offload");
3788	return -EOPNOTSUPP;
3789	}
3790	#else
3791	return 0;
3792	#endif
3793	}
3794
3795	int tc_setup_action(struct flow_action *flow_action,
3796	struct tc_action *actions[],
3797	u32 miss_cookie_base,
3798	struct netlink_ext_ack *extack)
3799	{
3800	int i, j, k, index, err = 0;
3801	struct tc_action *act;
3802
3803	BUILD_BUG_ON(TCA_ACT_HW_STATS_ANY != FLOW_ACTION_HW_STATS_ANY);
3804	BUILD_BUG_ON(TCA_ACT_HW_STATS_IMMEDIATE != FLOW_ACTION_HW_STATS_IMMEDIATE);
3805	BUILD_BUG_ON(TCA_ACT_HW_STATS_DELAYED != FLOW_ACTION_HW_STATS_DELAYED);
3806
3807	if (!actions)
3808	return 0;
3809
3810	j = 0;
3811	tcf_act_for_each_action(i, act, actions) {
3812	struct flow_action_entry *entry;
3813
3814	entry = &flow_action->entries[j];
3815	spin_lock_bh(lock: &act->tcfa_lock);
3816	err = tcf_act_get_user_cookie(entry, act);
3817	if (err)
3818	goto err_out_locked;
3819
3820	index = 0;
3821	err = tc_setup_offload_act(act, entry, index_inc: &index, extack);
3822	if (err)
3823	goto err_out_locked;
3824
3825	for (k = 0; k < index ; k++) {
3826	entry[k].hw_stats = tc_act_hw_stats(hw_stats: act->hw_stats);
3827	entry[k].hw_index = act->tcfa_index;
3828	entry[k].cookie = (unsigned long)act;
3829	entry[k].miss_cookie =
3830	tcf_exts_miss_cookie_get(miss_cookie_base, act_index: i);
3831	}
3832
3833	j += index;
3834
3835	spin_unlock_bh(lock: &act->tcfa_lock);
3836	}
3837
3838	err_out:
3839	if (err)
3840	tc_cleanup_offload_action(flow_action);
3841
3842	return err;
3843	err_out_locked:
3844	spin_unlock_bh(lock: &act->tcfa_lock);
3845	goto err_out;
3846	}
3847
3848	int tc_setup_offload_action(struct flow_action *flow_action,
3849	const struct tcf_exts *exts,
3850	struct netlink_ext_ack *extack)
3851	{
3852	#ifdef CONFIG_NET_CLS_ACT
3853	u32 miss_cookie_base;
3854
3855	if (!exts)
3856	return 0;
3857
3858	miss_cookie_base = exts->miss_cookie_node ?
3859	exts->miss_cookie_node->miss_cookie_base : 0;
3860	return tc_setup_action(flow_action, actions: exts->actions, miss_cookie_base,
3861	extack);
3862	#else
3863	return 0;
3864	#endif
3865	}
3866	EXPORT_SYMBOL(tc_setup_offload_action);
3867
3868	unsigned int tcf_exts_num_actions(struct tcf_exts *exts)
3869	{
3870	unsigned int num_acts = 0;
3871	struct tc_action *act;
3872	int i;
3873
3874	tcf_exts_for_each_action(i, act, exts) {
3875	if (is_tcf_pedit(a: act))
3876	num_acts += tcf_pedit_nkeys(a: act);
3877	else
3878	num_acts++;
3879	}
3880	return num_acts;
3881	}
3882	EXPORT_SYMBOL(tcf_exts_num_actions);
3883
3884	#ifdef CONFIG_NET_CLS_ACT
3885	static int tcf_qevent_parse_block_index(struct nlattr *block_index_attr,
3886	u32 *p_block_index,
3887	struct netlink_ext_ack *extack)
3888	{
3889	*p_block_index = nla_get_u32(nla: block_index_attr);
3890	if (!*p_block_index) {
3891	NL_SET_ERR_MSG(extack, "Block number may not be zero");
3892	return -EINVAL;
3893	}
3894
3895	return 0;
3896	}
3897
3898	int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch,
3899	enum flow_block_binder_type binder_type,
3900	struct nlattr *block_index_attr,
3901	struct netlink_ext_ack *extack)
3902	{
3903	u32 block_index;
3904	int err;
3905
3906	if (!block_index_attr)
3907	return 0;
3908
3909	err = tcf_qevent_parse_block_index(block_index_attr, p_block_index: &block_index, extack);
3910	if (err)
3911	return err;
3912
3913	qe->info.binder_type = binder_type;
3914	qe->info.chain_head_change = tcf_chain_head_change_dflt;
3915	qe->info.chain_head_change_priv = &qe->filter_chain;
3916	qe->info.block_index = block_index;
3917
3918	return tcf_block_get_ext(&qe->block, sch, &qe->info, extack);
3919	}
3920	EXPORT_SYMBOL(tcf_qevent_init);
3921
3922	void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch)
3923	{
3924	if (qe->info.block_index)
3925	tcf_block_put_ext(qe->block, sch, &qe->info);
3926	}
3927	EXPORT_SYMBOL(tcf_qevent_destroy);
3928
3929	int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr,
3930	struct netlink_ext_ack *extack)
3931	{
3932	u32 block_index;
3933	int err;
3934
3935	if (!block_index_attr)
3936	return 0;
3937
3938	err = tcf_qevent_parse_block_index(block_index_attr, p_block_index: &block_index, extack);
3939	if (err)
3940	return err;
3941
3942	/* Bounce newly-configured block or change in block. */
3943	if (block_index != qe->info.block_index) {
3944	NL_SET_ERR_MSG(extack, "Change of blocks is not supported");
3945	return -EINVAL;
3946	}
3947
3948	return 0;
3949	}
3950	EXPORT_SYMBOL(tcf_qevent_validate_change);
3951
3952	struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb,
3953	struct sk_buff **to_free, int *ret)
3954	{
3955	struct tcf_result cl_res;
3956	struct tcf_proto *fl;
3957
3958	if (!qe->info.block_index)
3959	return skb;
3960
3961	fl = rcu_dereference_bh(qe->filter_chain);
3962
3963	switch (tcf_classify(skb, NULL, fl, &cl_res, false)) {
3964	case TC_ACT_SHOT:
3965	qdisc_qstats_drop(sch);
3966	__qdisc_drop(skb, to_free);
3967	*ret = __NET_XMIT_BYPASS;
3968	return NULL;
3969	case TC_ACT_STOLEN:
3970	case TC_ACT_QUEUED:
3971	case TC_ACT_TRAP:
3972	__qdisc_drop(skb, to_free);
3973	*ret = __NET_XMIT_STOLEN;
3974	return NULL;
3975	case TC_ACT_REDIRECT:
3976	skb_do_redirect(skb);
3977	*ret = __NET_XMIT_STOLEN;
3978	return NULL;
3979	}
3980
3981	return skb;
3982	}
3983	EXPORT_SYMBOL(tcf_qevent_handle);
3984
3985	int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe)
3986	{
3987	if (!qe->info.block_index)
3988	return 0;
3989	return nla_put_u32(skb, attrtype: attr_name, value: qe->info.block_index);
3990	}
3991	EXPORT_SYMBOL(tcf_qevent_dump);
3992	#endif
3993
3994	static __net_init int tcf_net_init(struct net *net)
3995	{
3996	struct tcf_net *tn = net_generic(net, id: tcf_net_id);
3997
3998	spin_lock_init(&tn->idr_lock);
3999	idr_init(idr: &tn->idr);
4000	return 0;
4001	}
4002
4003	static void __net_exit tcf_net_exit(struct net *net)
4004	{
4005	struct tcf_net *tn = net_generic(net, id: tcf_net_id);
4006
4007	idr_destroy(&tn->idr);
4008	}
4009
4010	static struct pernet_operations tcf_net_ops = {
4011	.init = tcf_net_init,
4012	.exit = tcf_net_exit,
4013	.id = &tcf_net_id,
4014	.size = sizeof(struct tcf_net),
4015	};
4016
4017	static int __init tc_filter_init(void)
4018	{
4019	int err;
4020
4021	tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0);
4022	if (!tc_filter_wq)
4023	return -ENOMEM;
4024
4025	err = register_pernet_subsys(&tcf_net_ops);
4026	if (err)
4027	goto err_register_pernet_subsys;
4028
4029	xa_init_flags(xa: &tcf_exts_miss_cookies_xa, XA_FLAGS_ALLOC1);
4030
4031	rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL,
4032	flags: RTNL_FLAG_DOIT_UNLOCKED);
4033	rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL,
4034	flags: RTNL_FLAG_DOIT_UNLOCKED);
4035	rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter,
4036	tc_dump_tfilter, flags: RTNL_FLAG_DOIT_UNLOCKED);
4037	rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, flags: 0);
4038	rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, flags: 0);
4039	rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain,
4040	tc_dump_chain, flags: 0);
4041
4042	return 0;
4043
4044	err_register_pernet_subsys:
4045	destroy_workqueue(wq: tc_filter_wq);
4046	return err;
4047	}
4048
4049	subsys_initcall(tc_filter_init);
4050