1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* x_tables core - Backend for {ip,ip6,arp}_tables
4	*
5	* Copyright (C) 2006-2006 Harald Welte <laforge@netfilter.org>
6	* Copyright (C) 2006-2012 Patrick McHardy <kaber@trash.net>
7	*
8	* Based on existing ip_tables code which is
9	* Copyright (C) 1999 Paul `Rusty' Russell & Michael J. Neuling
10	* Copyright (C) 2000-2005 Netfilter Core Team <coreteam@netfilter.org>
11	*/
12	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
13	#include <linux/kernel.h>
14	#include <linux/module.h>
15	#include <linux/socket.h>
16	#include <linux/net.h>
17	#include <linux/proc_fs.h>
18	#include <linux/seq_file.h>
19	#include <linux/string.h>
20	#include <linux/vmalloc.h>
21	#include <linux/mutex.h>
22	#include <linux/mm.h>
23	#include <linux/slab.h>
24	#include <linux/audit.h>
25	#include <linux/user_namespace.h>
26	#include <net/net_namespace.h>
27	#include <net/netns/generic.h>
28
29	#include <linux/netfilter/x_tables.h>
30	#include <linux/netfilter_arp.h>
31	#include <linux/netfilter_ipv4/ip_tables.h>
32	#include <linux/netfilter_ipv6/ip6_tables.h>
33	#include <linux/netfilter_arp/arp_tables.h>
34
35	MODULE_LICENSE("GPL");
36	MODULE_AUTHOR("Harald Welte <laforge@netfilter.org>");
37	MODULE_DESCRIPTION("{ip,ip6,arp,eb}_tables backend module");
38
39	#define XT_PCPU_BLOCK_SIZE 4096
40	#define XT_MAX_TABLE_SIZE (512 * 1024 * 1024)
41
42	struct xt_template {
43	struct list_head list;
44
45	/* called when table is needed in the given netns */
46	int (*table_init)(struct net *net);
47
48	struct module *me;
49
50	/* A unique name... */
51	char name[XT_TABLE_MAXNAMELEN];
52	};
53
54	static struct list_head xt_templates[NFPROTO_NUMPROTO];
55
56	struct xt_pernet {
57	struct list_head tables[NFPROTO_NUMPROTO];
58	};
59
60	struct compat_delta {
61	unsigned int offset; /* offset in kernel */
62	int delta; /* delta in 32bit user land */
63	};
64
65	struct xt_af {
66	struct mutex mutex;
67	struct list_head match;
68	struct list_head target;
69	#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
70	struct mutex compat_mutex;
71	struct compat_delta *compat_tab;
72	unsigned int number; /* number of slots in compat_tab[] */
73	unsigned int cur; /* number of used slots in compat_tab[] */
74	#endif
75	};
76
77	static unsigned int xt_pernet_id __read_mostly;
78	static struct xt_af *xt __read_mostly;
79
80	static const char *const xt_prefix[NFPROTO_NUMPROTO] = {
81	[NFPROTO_UNSPEC] = "x",
82	[NFPROTO_IPV4] = "ip",
83	[NFPROTO_ARP] = "arp",
84	[NFPROTO_BRIDGE] = "eb",
85	[NFPROTO_IPV6] = "ip6",
86	};
87
88	/* Registration hooks for targets. */
89	int xt_register_target(struct xt_target *target)
90	{
91	u_int8_t af = target->family;
92
93	mutex_lock(&xt[af].mutex);
94	list_add(new: &target->list, head: &xt[af].target);
95	mutex_unlock(lock: &xt[af].mutex);
96	return 0;
97	}
98	EXPORT_SYMBOL(xt_register_target);
99
100	void
101	xt_unregister_target(struct xt_target *target)
102	{
103	u_int8_t af = target->family;
104
105	mutex_lock(&xt[af].mutex);
106	list_del(entry: &target->list);
107	mutex_unlock(lock: &xt[af].mutex);
108	}
109	EXPORT_SYMBOL(xt_unregister_target);
110
111	int
112	xt_register_targets(struct xt_target *target, unsigned int n)
113	{
114	unsigned int i;
115	int err = 0;
116
117	for (i = 0; i < n; i++) {
118	err = xt_register_target(&target[i]);
119	if (err)
120	goto err;
121	}
122	return err;
123
124	err:
125	if (i > 0)
126	xt_unregister_targets(target, n: i);
127	return err;
128	}
129	EXPORT_SYMBOL(xt_register_targets);
130
131	void
132	xt_unregister_targets(struct xt_target *target, unsigned int n)
133	{
134	while (n-- > 0)
135	xt_unregister_target(&target[n]);
136	}
137	EXPORT_SYMBOL(xt_unregister_targets);
138
139	int xt_register_match(struct xt_match *match)
140	{
141	u_int8_t af = match->family;
142
143	mutex_lock(&xt[af].mutex);
144	list_add(new: &match->list, head: &xt[af].match);
145	mutex_unlock(lock: &xt[af].mutex);
146	return 0;
147	}
148	EXPORT_SYMBOL(xt_register_match);
149
150	void
151	xt_unregister_match(struct xt_match *match)
152	{
153	u_int8_t af = match->family;
154
155	mutex_lock(&xt[af].mutex);
156	list_del(entry: &match->list);
157	mutex_unlock(lock: &xt[af].mutex);
158	}
159	EXPORT_SYMBOL(xt_unregister_match);
160
161	int
162	xt_register_matches(struct xt_match *match, unsigned int n)
163	{
164	unsigned int i;
165	int err = 0;
166
167	for (i = 0; i < n; i++) {
168	err = xt_register_match(&match[i]);
169	if (err)
170	goto err;
171	}
172	return err;
173
174	err:
175	if (i > 0)
176	xt_unregister_matches(match, n: i);
177	return err;
178	}
179	EXPORT_SYMBOL(xt_register_matches);
180
181	void
182	xt_unregister_matches(struct xt_match *match, unsigned int n)
183	{
184	while (n-- > 0)
185	xt_unregister_match(&match[n]);
186	}
187	EXPORT_SYMBOL(xt_unregister_matches);
188
189
190	/*
191	* These are weird, but module loading must not be done with mutex
192	* held (since they will register), and we have to have a single
193	* function to use.
194	*/
195
196	/* Find match, grabs ref. Returns ERR_PTR() on error. */
197	struct xt_match *xt_find_match(u8 af, const char *name, u8 revision)
198	{
199	struct xt_match *m;
200	int err = -ENOENT;
201
202	if (strnlen(p: name, XT_EXTENSION_MAXNAMELEN) == XT_EXTENSION_MAXNAMELEN)
203	return ERR_PTR(error: -EINVAL);
204
205	mutex_lock(&xt[af].mutex);
206	list_for_each_entry(m, &xt[af].match, list) {
207	if (strcmp(m->name, name) == 0) {
208	if (m->revision == revision) {
209	if (try_module_get(module: m->me)) {
210	mutex_unlock(lock: &xt[af].mutex);
211	return m;
212	}
213	} else
214	err = -EPROTOTYPE; /* Found something. */
215	}
216	}
217	mutex_unlock(lock: &xt[af].mutex);
218
219	if (af != NFPROTO_UNSPEC)
220	/* Try searching again in the family-independent list */
221	return xt_find_match(af: NFPROTO_UNSPEC, name, revision);
222
223	return ERR_PTR(error: err);
224	}
225	EXPORT_SYMBOL(xt_find_match);
226
227	struct xt_match *
228	xt_request_find_match(uint8_t nfproto, const char *name, uint8_t revision)
229	{
230	struct xt_match *match;
231
232	if (strnlen(p: name, XT_EXTENSION_MAXNAMELEN) == XT_EXTENSION_MAXNAMELEN)
233	return ERR_PTR(error: -EINVAL);
234
235	match = xt_find_match(nfproto, name, revision);
236	if (IS_ERR(ptr: match)) {
237	request_module("%st_%s", xt_prefix[nfproto], name);
238	match = xt_find_match(nfproto, name, revision);
239	}
240
241	return match;
242	}
243	EXPORT_SYMBOL_GPL(xt_request_find_match);
244
245	/* Find target, grabs ref. Returns ERR_PTR() on error. */
246	static struct xt_target *xt_find_target(u8 af, const char *name, u8 revision)
247	{
248	struct xt_target *t;
249	int err = -ENOENT;
250
251	if (strnlen(p: name, XT_EXTENSION_MAXNAMELEN) == XT_EXTENSION_MAXNAMELEN)
252	return ERR_PTR(error: -EINVAL);
253
254	mutex_lock(&xt[af].mutex);
255	list_for_each_entry(t, &xt[af].target, list) {
256	if (strcmp(t->name, name) == 0) {
257	if (t->revision == revision) {
258	if (try_module_get(module: t->me)) {
259	mutex_unlock(lock: &xt[af].mutex);
260	return t;
261	}
262	} else
263	err = -EPROTOTYPE; /* Found something. */
264	}
265	}
266	mutex_unlock(lock: &xt[af].mutex);
267
268	if (af != NFPROTO_UNSPEC)
269	/* Try searching again in the family-independent list */
270	return xt_find_target(af: NFPROTO_UNSPEC, name, revision);
271
272	return ERR_PTR(error: err);
273	}
274
275	struct xt_target *xt_request_find_target(u8 af, const char *name, u8 revision)
276	{
277	struct xt_target *target;
278
279	if (strnlen(p: name, XT_EXTENSION_MAXNAMELEN) == XT_EXTENSION_MAXNAMELEN)
280	return ERR_PTR(error: -EINVAL);
281
282	target = xt_find_target(af, name, revision);
283	if (IS_ERR(ptr: target)) {
284	request_module("%st_%s", xt_prefix[af], name);
285	target = xt_find_target(af, name, revision);
286	}
287
288	return target;
289	}
290	EXPORT_SYMBOL_GPL(xt_request_find_target);
291
292
293	static int xt_obj_to_user(u16 __user *psize, u16 size,
294	void __user *pname, const char *name,
295	u8 __user *prev, u8 rev)
296	{
297	if (put_user(size, psize))
298	return -EFAULT;
299	if (copy_to_user(to: pname, from: name, strlen(name) + 1))
300	return -EFAULT;
301	if (put_user(rev, prev))
302	return -EFAULT;
303
304	return 0;
305	}
306
307	#define XT_OBJ_TO_USER(U, K, TYPE, C_SIZE) \
308	xt_obj_to_user(&U->u.TYPE##_size, C_SIZE ? : K->u.TYPE##_size, \
309	U->u.user.name, K->u.kernel.TYPE->name, \
310	&U->u.user.revision, K->u.kernel.TYPE->revision)
311
312	int xt_data_to_user(void __user *dst, const void *src,
313	int usersize, int size, int aligned_size)
314	{
315	usersize = usersize ? : size;
316	if (copy_to_user(to: dst, from: src, n: usersize))
317	return -EFAULT;
318	if (usersize != aligned_size &&
319	clear_user(to: dst + usersize, n: aligned_size - usersize))
320	return -EFAULT;
321
322	return 0;
323	}
324	EXPORT_SYMBOL_GPL(xt_data_to_user);
325
326	#define XT_DATA_TO_USER(U, K, TYPE) \
327	xt_data_to_user(U->data, K->data, \
328	K->u.kernel.TYPE->usersize, \
329	K->u.kernel.TYPE->TYPE##size, \
330	XT_ALIGN(K->u.kernel.TYPE->TYPE##size))
331
332	int xt_match_to_user(const struct xt_entry_match *m,
333	struct xt_entry_match __user *u)
334	{
335	return XT_OBJ_TO_USER(u, m, match, 0) ||
336	XT_DATA_TO_USER(u, m, match);
337	}
338	EXPORT_SYMBOL_GPL(xt_match_to_user);
339
340	int xt_target_to_user(const struct xt_entry_target *t,
341	struct xt_entry_target __user *u)
342	{
343	return XT_OBJ_TO_USER(u, t, target, 0) ||
344	XT_DATA_TO_USER(u, t, target);
345	}
346	EXPORT_SYMBOL_GPL(xt_target_to_user);
347
348	static int match_revfn(u8 af, const char *name, u8 revision, int *bestp)
349	{
350	const struct xt_match *m;
351	int have_rev = 0;
352
353	mutex_lock(&xt[af].mutex);
354	list_for_each_entry(m, &xt[af].match, list) {
355	if (strcmp(m->name, name) == 0) {
356	if (m->revision > *bestp)
357	*bestp = m->revision;
358	if (m->revision == revision)
359	have_rev = 1;
360	}
361	}
362	mutex_unlock(lock: &xt[af].mutex);
363
364	if (af != NFPROTO_UNSPEC && !have_rev)
365	return match_revfn(af: NFPROTO_UNSPEC, name, revision, bestp);
366
367	return have_rev;
368	}
369
370	static int target_revfn(u8 af, const char *name, u8 revision, int *bestp)
371	{
372	const struct xt_target *t;
373	int have_rev = 0;
374
375	mutex_lock(&xt[af].mutex);
376	list_for_each_entry(t, &xt[af].target, list) {
377	if (strcmp(t->name, name) == 0) {
378	if (t->revision > *bestp)
379	*bestp = t->revision;
380	if (t->revision == revision)
381	have_rev = 1;
382	}
383	}
384	mutex_unlock(lock: &xt[af].mutex);
385
386	if (af != NFPROTO_UNSPEC && !have_rev)
387	return target_revfn(af: NFPROTO_UNSPEC, name, revision, bestp);
388
389	return have_rev;
390	}
391
392	/* Returns true or false (if no such extension at all) */
393	int xt_find_revision(u8 af, const char *name, u8 revision, int target,
394	int *err)
395	{
396	int have_rev, best = -1;
397
398	if (target == 1)
399	have_rev = target_revfn(af, name, revision, bestp: &best);
400	else
401	have_rev = match_revfn(af, name, revision, bestp: &best);
402
403	/* Nothing at all? Return 0 to try loading module. */
404	if (best == -1) {
405	*err = -ENOENT;
406	return 0;
407	}
408
409	*err = best;
410	if (!have_rev)
411	*err = -EPROTONOSUPPORT;
412	return 1;
413	}
414	EXPORT_SYMBOL_GPL(xt_find_revision);
415
416	static char *
417	textify_hooks(char *buf, size_t size, unsigned int mask, uint8_t nfproto)
418	{
419	static const char *const inetbr_names[] = {
420	"PREROUTING", "INPUT", "FORWARD",
421	"OUTPUT", "POSTROUTING", "BROUTING",
422	};
423	static const char *const arp_names[] = {
424	"INPUT", "FORWARD", "OUTPUT",
425	};
426	const char *const *names;
427	unsigned int i, max;
428	char *p = buf;
429	bool np = false;
430	int res;
431
432	names = (nfproto == NFPROTO_ARP) ? arp_names : inetbr_names;
433	max = (nfproto == NFPROTO_ARP) ? ARRAY_SIZE(arp_names) :
434	ARRAY_SIZE(inetbr_names);
435	*p = '\0';
436	for (i = 0; i < max; ++i) {
437	if (!(mask & (1 << i)))
438	continue;
439	res = snprintf(buf: p, size, fmt: "%s%s", np ? "/" : "", names[i]);
440	if (res > 0) {
441	size -= res;
442	p += res;
443	}
444	np = true;
445	}
446
447	return buf;
448	}
449
450	/**
451	* xt_check_proc_name - check that name is suitable for /proc file creation
452	*
453	* @name: file name candidate
454	* @size: length of buffer
455	*
456	* some x_tables modules wish to create a file in /proc.
457	* This function makes sure that the name is suitable for this
458	* purpose, it checks that name is NUL terminated and isn't a 'special'
459	* name, like "..".
460	*
461	* returns negative number on error or 0 if name is useable.
462	*/
463	int xt_check_proc_name(const char *name, unsigned int size)
464	{
465	if (name[0] == '\0')
466	return -EINVAL;
467
468	if (strnlen(p: name, maxlen: size) == size)
469	return -ENAMETOOLONG;
470
471	if (strcmp(name, ".") == 0 ||
472	strcmp(name, "..") == 0 ||
473	strchr(name, '/'))
474	return -EINVAL;
475
476	return 0;
477	}
478	EXPORT_SYMBOL(xt_check_proc_name);
479
480	int xt_check_match(struct xt_mtchk_param *par,
481	unsigned int size, u16 proto, bool inv_proto)
482	{
483	int ret;
484
485	if (XT_ALIGN(par->match->matchsize) != size &&
486	par->match->matchsize != -1) {
487	/*
488	* ebt_among is exempt from centralized matchsize checking
489	* because it uses a dynamic-size data set.
490	*/
491	pr_err_ratelimited("%s_tables: %s.%u match: invalid size %u (kernel) != (user) %u\n",
492	xt_prefix[par->family], par->match->name,
493	par->match->revision,
494	XT_ALIGN(par->match->matchsize), size);
495	return -EINVAL;
496	}
497	if (par->match->table != NULL &&
498	strcmp(par->match->table, par->table) != 0) {
499	pr_info_ratelimited("%s_tables: %s match: only valid in %s table, not %s\n",
500	xt_prefix[par->family], par->match->name,
501	par->match->table, par->table);
502	return -EINVAL;
503	}
504	if (par->match->hooks && (par->hook_mask & ~par->match->hooks) != 0) {
505	char used[64], allow[64];
506
507	pr_info_ratelimited("%s_tables: %s match: used from hooks %s, but only valid from %s\n",
508	xt_prefix[par->family], par->match->name,
509	textify_hooks(used, sizeof(used),
510	par->hook_mask, par->family),
511	textify_hooks(allow, sizeof(allow),
512	par->match->hooks,
513	par->family));
514	return -EINVAL;
515	}
516	if (par->match->proto && (par->match->proto != proto || inv_proto)) {
517	pr_info_ratelimited("%s_tables: %s match: only valid for protocol %u\n",
518	xt_prefix[par->family], par->match->name,
519	par->match->proto);
520	return -EINVAL;
521	}
522	if (par->match->checkentry != NULL) {
523	ret = par->match->checkentry(par);
524	if (ret < 0)
525	return ret;
526	else if (ret > 0)
527	/* Flag up potential errors. */
528	return -EIO;
529	}
530	return 0;
531	}
532	EXPORT_SYMBOL_GPL(xt_check_match);
533
534	/** xt_check_entry_match - check that matches end before start of target
535	*
536	* @match: beginning of xt_entry_match
537	* @target: beginning of this rules target (alleged end of matches)
538	* @alignment: alignment requirement of match structures
539	*
540	* Validates that all matches add up to the beginning of the target,
541	* and that each match covers at least the base structure size.
542	*
543	* Return: 0 on success, negative errno on failure.
544	*/
545	static int xt_check_entry_match(const char *match, const char *target,
546	const size_t alignment)
547	{
548	const struct xt_entry_match *pos;
549	int length = target - match;
550
551	if (length == 0) /* no matches */
552	return 0;
553
554	pos = (struct xt_entry_match *)match;
555	do {
556	if ((unsigned long)pos % alignment)
557	return -EINVAL;
558
559	if (length < (int)sizeof(struct xt_entry_match))
560	return -EINVAL;
561
562	if (pos->u.match_size < sizeof(struct xt_entry_match))
563	return -EINVAL;
564
565	if (pos->u.match_size > length)
566	return -EINVAL;
567
568	length -= pos->u.match_size;
569	pos = ((void *)((char *)(pos) + (pos)->u.match_size));
570	} while (length > 0);
571
572	return 0;
573	}
574
575	/** xt_check_table_hooks - check hook entry points are sane
576	*
577	* @info xt_table_info to check
578	* @valid_hooks - hook entry points that we can enter from
579	*
580	* Validates that the hook entry and underflows points are set up.
581	*
582	* Return: 0 on success, negative errno on failure.
583	*/
584	int xt_check_table_hooks(const struct xt_table_info *info, unsigned int valid_hooks)
585	{
586	const char *err = "unsorted underflow";
587	unsigned int i, max_uflow, max_entry;
588	bool check_hooks = false;
589
590	BUILD_BUG_ON(ARRAY_SIZE(info->hook_entry) != ARRAY_SIZE(info->underflow));
591
592	max_entry = 0;
593	max_uflow = 0;
594
595	for (i = 0; i < ARRAY_SIZE(info->hook_entry); i++) {
596	if (!(valid_hooks & (1 << i)))
597	continue;
598
599	if (info->hook_entry[i] == 0xFFFFFFFF)
600	return -EINVAL;
601	if (info->underflow[i] == 0xFFFFFFFF)
602	return -EINVAL;
603
604	if (check_hooks) {
605	if (max_uflow > info->underflow[i])
606	goto error;
607
608	if (max_uflow == info->underflow[i]) {
609	err = "duplicate underflow";
610	goto error;
611	}
612	if (max_entry > info->hook_entry[i]) {
613	err = "unsorted entry";
614	goto error;
615	}
616	if (max_entry == info->hook_entry[i]) {
617	err = "duplicate entry";
618	goto error;
619	}
620	}
621	max_entry = info->hook_entry[i];
622	max_uflow = info->underflow[i];
623	check_hooks = true;
624	}
625
626	return 0;
627	error:
628	pr_err_ratelimited("%s at hook %d\n", err, i);
629	return -EINVAL;
630	}
631	EXPORT_SYMBOL(xt_check_table_hooks);
632
633	static bool verdict_ok(int verdict)
634	{
635	if (verdict > 0)
636	return true;
637
638	if (verdict < 0) {
639	int v = -verdict - 1;
640
641	if (verdict == XT_RETURN)
642	return true;
643
644	switch (v) {
645	case NF_ACCEPT: return true;
646	case NF_DROP: return true;
647	case NF_QUEUE: return true;
648	default:
649	break;
650	}
651
652	return false;
653	}
654
655	return false;
656	}
657
658	static bool error_tg_ok(unsigned int usersize, unsigned int kernsize,
659	const char *msg, unsigned int msglen)
660	{
661	return usersize == kernsize && strnlen(p: msg, maxlen: msglen) < msglen;
662	}
663
664	#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
665	int xt_compat_add_offset(u_int8_t af, unsigned int offset, int delta)
666	{
667	struct xt_af *xp = &xt[af];
668
669	WARN_ON(!mutex_is_locked(&xt[af].compat_mutex));
670
671	if (WARN_ON(!xp->compat_tab))
672	return -ENOMEM;
673
674	if (xp->cur >= xp->number)
675	return -EINVAL;
676
677	if (xp->cur)
678	delta += xp->compat_tab[xp->cur - 1].delta;
679	xp->compat_tab[xp->cur].offset = offset;
680	xp->compat_tab[xp->cur].delta = delta;
681	xp->cur++;
682	return 0;
683	}
684	EXPORT_SYMBOL_GPL(xt_compat_add_offset);
685
686	void xt_compat_flush_offsets(u_int8_t af)
687	{
688	WARN_ON(!mutex_is_locked(&xt[af].compat_mutex));
689
690	if (xt[af].compat_tab) {
691	vfree(addr: xt[af].compat_tab);
692	xt[af].compat_tab = NULL;
693	xt[af].number = 0;
694	xt[af].cur = 0;
695	}
696	}
697	EXPORT_SYMBOL_GPL(xt_compat_flush_offsets);
698
699	int xt_compat_calc_jump(u_int8_t af, unsigned int offset)
700	{
701	struct compat_delta *tmp = xt[af].compat_tab;
702	int mid, left = 0, right = xt[af].cur - 1;
703
704	while (left <= right) {
705	mid = (left + right) >> 1;
706	if (offset > tmp[mid].offset)
707	left = mid + 1;
708	else if (offset < tmp[mid].offset)
709	right = mid - 1;
710	else
711	return mid ? tmp[mid - 1].delta : 0;
712	}
713	return left ? tmp[left - 1].delta : 0;
714	}
715	EXPORT_SYMBOL_GPL(xt_compat_calc_jump);
716
717	int xt_compat_init_offsets(u8 af, unsigned int number)
718	{
719	size_t mem;
720
721	WARN_ON(!mutex_is_locked(&xt[af].compat_mutex));
722
723	if (!number || number > (INT_MAX / sizeof(struct compat_delta)))
724	return -EINVAL;
725
726	if (WARN_ON(xt[af].compat_tab))
727	return -EINVAL;
728
729	mem = sizeof(struct compat_delta) * number;
730	if (mem > XT_MAX_TABLE_SIZE)
731	return -ENOMEM;
732
733	xt[af].compat_tab = vmalloc(size: mem);
734	if (!xt[af].compat_tab)
735	return -ENOMEM;
736
737	xt[af].number = number;
738	xt[af].cur = 0;
739
740	return 0;
741	}
742	EXPORT_SYMBOL(xt_compat_init_offsets);
743
744	int xt_compat_match_offset(const struct xt_match *match)
745	{
746	u_int16_t csize = match->compatsize ? : match->matchsize;
747	return XT_ALIGN(match->matchsize) - COMPAT_XT_ALIGN(csize);
748	}
749	EXPORT_SYMBOL_GPL(xt_compat_match_offset);
750
751	void xt_compat_match_from_user(struct xt_entry_match *m, void **dstptr,
752	unsigned int *size)
753	{
754	const struct xt_match *match = m->u.kernel.match;
755	struct compat_xt_entry_match *cm = (struct compat_xt_entry_match *)m;
756	int off = xt_compat_match_offset(match);
757	u_int16_t msize = cm->u.user.match_size;
758	char name[sizeof(m->u.user.name)];
759
760	m = *dstptr;
761	memcpy(m, cm, sizeof(*cm));
762	if (match->compat_from_user)
763	match->compat_from_user(m->data, cm->data);
764	else
765	memcpy(m->data, cm->data, msize - sizeof(*cm));
766
767	msize += off;
768	m->u.user.match_size = msize;
769	strscpy(name, match->name, sizeof(name));
770	module_put(module: match->me);
771	strscpy_pad(m->u.user.name, name, sizeof(m->u.user.name));
772
773	*size += off;
774	*dstptr += msize;
775	}
776	EXPORT_SYMBOL_GPL(xt_compat_match_from_user);
777
778	#define COMPAT_XT_DATA_TO_USER(U, K, TYPE, C_SIZE) \
779	xt_data_to_user(U->data, K->data, \
780	K->u.kernel.TYPE->usersize, \
781	C_SIZE, \
782	COMPAT_XT_ALIGN(C_SIZE))
783
784	int xt_compat_match_to_user(const struct xt_entry_match *m,
785	void __user **dstptr, unsigned int *size)
786	{
787	const struct xt_match *match = m->u.kernel.match;
788	struct compat_xt_entry_match __user *cm = *dstptr;
789	int off = xt_compat_match_offset(match);
790	u_int16_t msize = m->u.user.match_size - off;
791
792	if (XT_OBJ_TO_USER(cm, m, match, msize))
793	return -EFAULT;
794
795	if (match->compat_to_user) {
796	if (match->compat_to_user((void __user *)cm->data, m->data))
797	return -EFAULT;
798	} else {
799	if (COMPAT_XT_DATA_TO_USER(cm, m, match, msize - sizeof(*cm)))
800	return -EFAULT;
801	}
802
803	*size -= off;
804	*dstptr += msize;
805	return 0;
806	}
807	EXPORT_SYMBOL_GPL(xt_compat_match_to_user);
808
809	/* non-compat version may have padding after verdict */
810	struct compat_xt_standard_target {
811	struct compat_xt_entry_target t;
812	compat_uint_t verdict;
813	};
814
815	struct compat_xt_error_target {
816	struct compat_xt_entry_target t;
817	char errorname[XT_FUNCTION_MAXNAMELEN];
818	};
819
820	int xt_compat_check_entry_offsets(const void *base, const char *elems,
821	unsigned int target_offset,
822	unsigned int next_offset)
823	{
824	long size_of_base_struct = elems - (const char *)base;
825	const struct compat_xt_entry_target *t;
826	const char *e = base;
827
828	if (target_offset < size_of_base_struct)
829	return -EINVAL;
830
831	if (target_offset + sizeof(*t) > next_offset)
832	return -EINVAL;
833
834	t = (void *)(e + target_offset);
835	if (t->u.target_size < sizeof(*t))
836	return -EINVAL;
837
838	if (target_offset + t->u.target_size > next_offset)
839	return -EINVAL;
840
841	if (strcmp(t->u.user.name, XT_STANDARD_TARGET) == 0) {
842	const struct compat_xt_standard_target *st = (const void *)t;
843
844	if (COMPAT_XT_ALIGN(target_offset + sizeof(*st)) != next_offset)
845	return -EINVAL;
846
847	if (!verdict_ok(verdict: st->verdict))
848	return -EINVAL;
849	} else if (strcmp(t->u.user.name, XT_ERROR_TARGET) == 0) {
850	const struct compat_xt_error_target *et = (const void *)t;
851
852	if (!error_tg_ok(usersize: t->u.target_size, kernsize: sizeof(*et),
853	msg: et->errorname, msglen: sizeof(et->errorname)))
854	return -EINVAL;
855	}
856
857	/* compat_xt_entry match has less strict alignment requirements,
858	* otherwise they are identical. In case of padding differences
859	* we need to add compat version of xt_check_entry_match.
860	*/
861	BUILD_BUG_ON(sizeof(struct compat_xt_entry_match) != sizeof(struct xt_entry_match));
862
863	return xt_check_entry_match(match: elems, target: base + target_offset,
864	alignment: __alignof__(struct compat_xt_entry_match));
865	}
866	EXPORT_SYMBOL(xt_compat_check_entry_offsets);
867	#endif /* CONFIG_NETFILTER_XTABLES_COMPAT */
868
869	/**
870	* xt_check_entry_offsets - validate arp/ip/ip6t_entry
871	*
872	* @base: pointer to arp/ip/ip6t_entry
873	* @elems: pointer to first xt_entry_match, i.e. ip(6)t_entry->elems
874	* @target_offset: the arp/ip/ip6_t->target_offset
875	* @next_offset: the arp/ip/ip6_t->next_offset
876	*
877	* validates that target_offset and next_offset are sane and that all
878	* match sizes (if any) align with the target offset.
879	*
880	* This function does not validate the targets or matches themselves, it
881	* only tests that all the offsets and sizes are correct, that all
882	* match structures are aligned, and that the last structure ends where
883	* the target structure begins.
884	*
885	* Also see xt_compat_check_entry_offsets for CONFIG_NETFILTER_XTABLES_COMPAT version.
886	*
887	* The arp/ip/ip6t_entry structure @base must have passed following tests:
888	* - it must point to a valid memory location
889	* - base to base + next_offset must be accessible, i.e. not exceed allocated
890	* length.
891	*
892	* A well-formed entry looks like this:
893	*
894	* ip(6)t_entry match [mtdata] match [mtdata] target [tgdata] ip(6)t_entry
895	* e->elems[]-----' | |
896	* matchsize | |
897	* matchsize | |
898	* | |
899	* target_offset---------------------------------' |
900	* next_offset---------------------------------------------------'
901	*
902	* elems[]: flexible array member at end of ip(6)/arpt_entry struct.
903	* This is where matches (if any) and the target reside.
904	* target_offset: beginning of target.
905	* next_offset: start of the next rule; also: size of this rule.
906	* Since targets have a minimum size, target_offset + minlen <= next_offset.
907	*
908	* Every match stores its size, sum of sizes must not exceed target_offset.
909	*
910	* Return: 0 on success, negative errno on failure.
911	*/
912	int xt_check_entry_offsets(const void *base,
913	const char *elems,
914	unsigned int target_offset,
915	unsigned int next_offset)
916	{
917	long size_of_base_struct = elems - (const char *)base;
918	const struct xt_entry_target *t;
919	const char *e = base;
920
921	/* target start is within the ip/ip6/arpt_entry struct */
922	if (target_offset < size_of_base_struct)
923	return -EINVAL;
924
925	if (target_offset + sizeof(*t) > next_offset)
926	return -EINVAL;
927
928	t = (void *)(e + target_offset);
929	if (t->u.target_size < sizeof(*t))
930	return -EINVAL;
931
932	if (target_offset + t->u.target_size > next_offset)
933	return -EINVAL;
934
935	if (strcmp(t->u.user.name, XT_STANDARD_TARGET) == 0) {
936	const struct xt_standard_target *st = (const void *)t;
937
938	if (XT_ALIGN(target_offset + sizeof(*st)) != next_offset)
939	return -EINVAL;
940
941	if (!verdict_ok(verdict: st->verdict))
942	return -EINVAL;
943	} else if (strcmp(t->u.user.name, XT_ERROR_TARGET) == 0) {
944	const struct xt_error_target *et = (const void *)t;
945
946	if (!error_tg_ok(usersize: t->u.target_size, kernsize: sizeof(*et),
947	msg: et->errorname, msglen: sizeof(et->errorname)))
948	return -EINVAL;
949	}
950
951	return xt_check_entry_match(match: elems, target: base + target_offset,
952	alignment: __alignof__(struct xt_entry_match));
953	}
954	EXPORT_SYMBOL(xt_check_entry_offsets);
955
956	/**
957	* xt_alloc_entry_offsets - allocate array to store rule head offsets
958	*
959	* @size: number of entries
960	*
961	* Return: NULL or zeroed kmalloc'd or vmalloc'd array
962	*/
963	unsigned int *xt_alloc_entry_offsets(unsigned int size)
964	{
965	if (size > XT_MAX_TABLE_SIZE / sizeof(unsigned int))
966	return NULL;
967
968	return kvcalloc(n: size, size: sizeof(unsigned int), GFP_KERNEL);
969
970	}
971	EXPORT_SYMBOL(xt_alloc_entry_offsets);
972
973	/**
974	* xt_find_jump_offset - check if target is a valid jump offset
975	*
976	* @offsets: array containing all valid rule start offsets of a rule blob
977	* @target: the jump target to search for
978	* @size: entries in @offset
979	*/
980	bool xt_find_jump_offset(const unsigned int *offsets,
981	unsigned int target, unsigned int size)
982	{
983	int m, low = 0, hi = size;
984
985	while (hi > low) {
986	m = (low + hi) / 2u;
987
988	if (offsets[m] > target)
989	hi = m;
990	else if (offsets[m] < target)
991	low = m + 1;
992	else
993	return true;
994	}
995
996	return false;
997	}
998	EXPORT_SYMBOL(xt_find_jump_offset);
999
1000	int xt_check_target(struct xt_tgchk_param *par,
1001	unsigned int size, u16 proto, bool inv_proto)
1002	{
1003	int ret;
1004
1005	if (XT_ALIGN(par->target->targetsize) != size) {
1006	pr_err_ratelimited("%s_tables: %s.%u target: invalid size %u (kernel) != (user) %u\n",
1007	xt_prefix[par->family], par->target->name,
1008	par->target->revision,
1009	XT_ALIGN(par->target->targetsize), size);
1010	return -EINVAL;
1011	}
1012	if (par->target->table != NULL &&
1013	strcmp(par->target->table, par->table) != 0) {
1014	pr_info_ratelimited("%s_tables: %s target: only valid in %s table, not %s\n",
1015	xt_prefix[par->family], par->target->name,
1016	par->target->table, par->table);
1017	return -EINVAL;
1018	}
1019	if (par->target->hooks && (par->hook_mask & ~par->target->hooks) != 0) {
1020	char used[64], allow[64];
1021
1022	pr_info_ratelimited("%s_tables: %s target: used from hooks %s, but only usable from %s\n",
1023	xt_prefix[par->family], par->target->name,
1024	textify_hooks(used, sizeof(used),
1025	par->hook_mask, par->family),
1026	textify_hooks(allow, sizeof(allow),
1027	par->target->hooks,
1028	par->family));
1029	return -EINVAL;
1030	}
1031	if (par->target->proto && (par->target->proto != proto || inv_proto)) {
1032	pr_info_ratelimited("%s_tables: %s target: only valid for protocol %u\n",
1033	xt_prefix[par->family], par->target->name,
1034	par->target->proto);
1035	return -EINVAL;
1036	}
1037	if (par->target->checkentry != NULL) {
1038	ret = par->target->checkentry(par);
1039	if (ret < 0)
1040	return ret;
1041	else if (ret > 0)
1042	/* Flag up potential errors. */
1043	return -EIO;
1044	}
1045	return 0;
1046	}
1047	EXPORT_SYMBOL_GPL(xt_check_target);
1048
1049	/**
1050	* xt_copy_counters - copy counters and metadata from a sockptr_t
1051	*
1052	* @arg: src sockptr
1053	* @len: alleged size of userspace memory
1054	* @info: where to store the xt_counters_info metadata
1055	*
1056	* Copies counter meta data from @user and stores it in @info.
1057	*
1058	* vmallocs memory to hold the counters, then copies the counter data
1059	* from @user to the new memory and returns a pointer to it.
1060	*
1061	* If called from a compat syscall, @info gets converted automatically to the
1062	* 64bit representation.
1063	*
1064	* The metadata associated with the counters is stored in @info.
1065	*
1066	* Return: returns pointer that caller has to test via IS_ERR().
1067	* If IS_ERR is false, caller has to vfree the pointer.
1068	*/
1069	void *xt_copy_counters(sockptr_t arg, unsigned int len,
1070	struct xt_counters_info *info)
1071	{
1072	size_t offset;
1073	void *mem;
1074	u64 size;
1075
1076	#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
1077	if (in_compat_syscall()) {
1078	/* structures only differ in size due to alignment */
1079	struct compat_xt_counters_info compat_tmp;
1080
1081	if (len <= sizeof(compat_tmp))
1082	return ERR_PTR(error: -EINVAL);
1083
1084	len -= sizeof(compat_tmp);
1085	if (copy_from_sockptr(dst: &compat_tmp, src: arg, size: sizeof(compat_tmp)) != 0)
1086	return ERR_PTR(error: -EFAULT);
1087
1088	memcpy(info->name, compat_tmp.name, sizeof(info->name) - 1);
1089	info->num_counters = compat_tmp.num_counters;
1090	offset = sizeof(compat_tmp);
1091	} else
1092	#endif
1093	{
1094	if (len <= sizeof(*info))
1095	return ERR_PTR(error: -EINVAL);
1096
1097	len -= sizeof(*info);
1098	if (copy_from_sockptr(dst: info, src: arg, size: sizeof(*info)) != 0)
1099	return ERR_PTR(error: -EFAULT);
1100
1101	offset = sizeof(*info);
1102	}
1103	info->name[sizeof(info->name) - 1] = '\0';
1104
1105	size = sizeof(struct xt_counters);
1106	size *= info->num_counters;
1107
1108	if (size != (u64)len)
1109	return ERR_PTR(error: -EINVAL);
1110
1111	mem = vmalloc(size: len);
1112	if (!mem)
1113	return ERR_PTR(error: -ENOMEM);
1114
1115	if (copy_from_sockptr_offset(dst: mem, src: arg, offset, size: len) == 0)
1116	return mem;
1117
1118	vfree(addr: mem);
1119	return ERR_PTR(error: -EFAULT);
1120	}
1121	EXPORT_SYMBOL_GPL(xt_copy_counters);
1122
1123	#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
1124	int xt_compat_target_offset(const struct xt_target *target)
1125	{
1126	u_int16_t csize = target->compatsize ? : target->targetsize;
1127	return XT_ALIGN(target->targetsize) - COMPAT_XT_ALIGN(csize);
1128	}
1129	EXPORT_SYMBOL_GPL(xt_compat_target_offset);
1130
1131	void xt_compat_target_from_user(struct xt_entry_target *t, void **dstptr,
1132	unsigned int *size)
1133	{
1134	const struct xt_target *target = t->u.kernel.target;
1135	struct compat_xt_entry_target *ct = (struct compat_xt_entry_target *)t;
1136	int off = xt_compat_target_offset(target);
1137	u_int16_t tsize = ct->u.user.target_size;
1138	char name[sizeof(t->u.user.name)];
1139
1140	t = *dstptr;
1141	memcpy(t, ct, sizeof(*ct));
1142	if (target->compat_from_user)
1143	target->compat_from_user(t->data, ct->data);
1144	else
1145	unsafe_memcpy(t->data, ct->data, tsize - sizeof(*ct),
1146	/* UAPI 0-sized destination */);
1147
1148	tsize += off;
1149	t->u.user.target_size = tsize;
1150	strscpy(name, target->name, sizeof(name));
1151	module_put(module: target->me);
1152	strscpy_pad(t->u.user.name, name, sizeof(t->u.user.name));
1153
1154	*size += off;
1155	*dstptr += tsize;
1156	}
1157	EXPORT_SYMBOL_GPL(xt_compat_target_from_user);
1158
1159	int xt_compat_target_to_user(const struct xt_entry_target *t,
1160	void __user **dstptr, unsigned int *size)
1161	{
1162	const struct xt_target *target = t->u.kernel.target;
1163	struct compat_xt_entry_target __user *ct = *dstptr;
1164	int off = xt_compat_target_offset(target);
1165	u_int16_t tsize = t->u.user.target_size - off;
1166
1167	if (XT_OBJ_TO_USER(ct, t, target, tsize))
1168	return -EFAULT;
1169
1170	if (target->compat_to_user) {
1171	if (target->compat_to_user((void __user *)ct->data, t->data))
1172	return -EFAULT;
1173	} else {
1174	if (COMPAT_XT_DATA_TO_USER(ct, t, target, tsize - sizeof(*ct)))
1175	return -EFAULT;
1176	}
1177
1178	*size -= off;
1179	*dstptr += tsize;
1180	return 0;
1181	}
1182	EXPORT_SYMBOL_GPL(xt_compat_target_to_user);
1183	#endif
1184
1185	struct xt_table_info *xt_alloc_table_info(unsigned int size)
1186	{
1187	struct xt_table_info *info = NULL;
1188	size_t sz = sizeof(*info) + size;
1189
1190	if (sz < sizeof(*info) || sz >= XT_MAX_TABLE_SIZE)
1191	return NULL;
1192
1193	info = kvmalloc(size: sz, GFP_KERNEL_ACCOUNT);
1194	if (!info)
1195	return NULL;
1196
1197	memset(info, 0, sizeof(*info));
1198	info->size = size;
1199	return info;
1200	}
1201	EXPORT_SYMBOL(xt_alloc_table_info);
1202
1203	void xt_free_table_info(struct xt_table_info *info)
1204	{
1205	int cpu;
1206
1207	if (info->jumpstack != NULL) {
1208	for_each_possible_cpu(cpu)
1209	kvfree(addr: info->jumpstack[cpu]);
1210	kvfree(addr: info->jumpstack);
1211	}
1212
1213	kvfree(addr: info);
1214	}
1215	EXPORT_SYMBOL(xt_free_table_info);
1216
1217	struct xt_table *xt_find_table(struct net *net, u8 af, const char *name)
1218	{
1219	struct xt_pernet *xt_net = net_generic(net, id: xt_pernet_id);
1220	struct xt_table *t;
1221
1222	mutex_lock(&xt[af].mutex);
1223	list_for_each_entry(t, &xt_net->tables[af], list) {
1224	if (strcmp(t->name, name) == 0) {
1225	mutex_unlock(lock: &xt[af].mutex);
1226	return t;
1227	}
1228	}
1229	mutex_unlock(lock: &xt[af].mutex);
1230	return NULL;
1231	}
1232	EXPORT_SYMBOL(xt_find_table);
1233
1234	/* Find table by name, grabs mutex & ref. Returns ERR_PTR on error. */
1235	struct xt_table *xt_find_table_lock(struct net *net, u_int8_t af,
1236	const char *name)
1237	{
1238	struct xt_pernet *xt_net = net_generic(net, id: xt_pernet_id);
1239	struct module *owner = NULL;
1240	struct xt_template *tmpl;
1241	struct xt_table *t;
1242
1243	mutex_lock(&xt[af].mutex);
1244	list_for_each_entry(t, &xt_net->tables[af], list)
1245	if (strcmp(t->name, name) == 0 && try_module_get(module: t->me))
1246	return t;
1247
1248	/* Table doesn't exist in this netns, check larval list */
1249	list_for_each_entry(tmpl, &xt_templates[af], list) {
1250	int err;
1251
1252	if (strcmp(tmpl->name, name))
1253	continue;
1254	if (!try_module_get(module: tmpl->me))
1255	goto out;
1256
1257	owner = tmpl->me;
1258
1259	mutex_unlock(lock: &xt[af].mutex);
1260	err = tmpl->table_init(net);
1261	if (err < 0) {
1262	module_put(module: owner);
1263	return ERR_PTR(error: err);
1264	}
1265
1266	mutex_lock(&xt[af].mutex);
1267	break;
1268	}
1269
1270	/* and once again: */
1271	list_for_each_entry(t, &xt_net->tables[af], list)
1272	if (strcmp(t->name, name) == 0)
1273	return t;
1274
1275	module_put(module: owner);
1276	out:
1277	mutex_unlock(lock: &xt[af].mutex);
1278	return ERR_PTR(error: -ENOENT);
1279	}
1280	EXPORT_SYMBOL_GPL(xt_find_table_lock);
1281
1282	struct xt_table *xt_request_find_table_lock(struct net *net, u_int8_t af,
1283	const char *name)
1284	{
1285	struct xt_table *t = xt_find_table_lock(net, af, name);
1286
1287	#ifdef CONFIG_MODULES
1288	if (IS_ERR(ptr: t)) {
1289	int err = request_module("%stable_%s", xt_prefix[af], name);
1290	if (err < 0)
1291	return ERR_PTR(error: err);
1292	t = xt_find_table_lock(net, af, name);
1293	}
1294	#endif
1295
1296	return t;
1297	}
1298	EXPORT_SYMBOL_GPL(xt_request_find_table_lock);
1299
1300	void xt_table_unlock(struct xt_table *table)
1301	{
1302	mutex_unlock(lock: &xt[table->af].mutex);
1303	}
1304	EXPORT_SYMBOL_GPL(xt_table_unlock);
1305
1306	#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
1307	void xt_compat_lock(u_int8_t af)
1308	{
1309	mutex_lock(&xt[af].compat_mutex);
1310	}
1311	EXPORT_SYMBOL_GPL(xt_compat_lock);
1312
1313	void xt_compat_unlock(u_int8_t af)
1314	{
1315	mutex_unlock(lock: &xt[af].compat_mutex);
1316	}
1317	EXPORT_SYMBOL_GPL(xt_compat_unlock);
1318	#endif
1319
1320	DEFINE_PER_CPU(seqcount_t, xt_recseq);
1321	EXPORT_PER_CPU_SYMBOL_GPL(xt_recseq);
1322
1323	struct static_key xt_tee_enabled __read_mostly;
1324	EXPORT_SYMBOL_GPL(xt_tee_enabled);
1325
1326	static int xt_jumpstack_alloc(struct xt_table_info *i)
1327	{
1328	unsigned int size;
1329	int cpu;
1330
1331	size = sizeof(void **) * nr_cpu_ids;
1332	if (size > PAGE_SIZE)
1333	i->jumpstack = kvzalloc(size, GFP_KERNEL);
1334	else
1335	i->jumpstack = kzalloc(size, GFP_KERNEL);
1336	if (i->jumpstack == NULL)
1337	return -ENOMEM;
1338
1339	/* ruleset without jumps -- no stack needed */
1340	if (i->stacksize == 0)
1341	return 0;
1342
1343	/* Jumpstack needs to be able to record two full callchains, one
1344	* from the first rule set traversal, plus one table reentrancy
1345	* via -j TEE without clobbering the callchain that brought us to
1346	* TEE target.
1347	*
1348	* This is done by allocating two jumpstacks per cpu, on reentry
1349	* the upper half of the stack is used.
1350	*
1351	* see the jumpstack setup in ipt_do_table() for more details.
1352	*/
1353	size = sizeof(void *) * i->stacksize * 2u;
1354	for_each_possible_cpu(cpu) {
1355	i->jumpstack[cpu] = kvmalloc_node(size, GFP_KERNEL,
1356	cpu_to_node(cpu));
1357	if (i->jumpstack[cpu] == NULL)
1358	/*
1359	* Freeing will be done later on by the callers. The
1360	* chain is: xt_replace_table -> __do_replace ->
1361	* do_replace -> xt_free_table_info.
1362	*/
1363	return -ENOMEM;
1364	}
1365
1366	return 0;
1367	}
1368
1369	struct xt_counters *xt_counters_alloc(unsigned int counters)
1370	{
1371	struct xt_counters *mem;
1372
1373	if (counters == 0 || counters > INT_MAX / sizeof(*mem))
1374	return NULL;
1375
1376	counters *= sizeof(*mem);
1377	if (counters > XT_MAX_TABLE_SIZE)
1378	return NULL;
1379
1380	return vzalloc(size: counters);
1381	}
1382	EXPORT_SYMBOL(xt_counters_alloc);
1383
1384	struct xt_table_info *
1385	xt_replace_table(struct xt_table *table,
1386	unsigned int num_counters,
1387	struct xt_table_info *newinfo,
1388	int *error)
1389	{
1390	struct xt_table_info *private;
1391	unsigned int cpu;
1392	int ret;
1393
1394	ret = xt_jumpstack_alloc(i: newinfo);
1395	if (ret < 0) {
1396	*error = ret;
1397	return NULL;
1398	}
1399
1400	/* Do the substitution. */
1401	local_bh_disable();
1402	private = table->private;
1403
1404	/* Check inside lock: is the old number correct? */
1405	if (num_counters != private->number) {
1406	pr_debug("num_counters != table->private->number (%u/%u)\n",
1407	num_counters, private->number);
1408	local_bh_enable();
1409	*error = -EAGAIN;
1410	return NULL;
1411	}
1412
1413	newinfo->initial_entries = private->initial_entries;
1414	/*
1415	* Ensure contents of newinfo are visible before assigning to
1416	* private.
1417	*/
1418	smp_wmb();
1419	table->private = newinfo;
1420
1421	/* make sure all cpus see new ->private value */
1422	smp_mb();
1423
1424	/*
1425	* Even though table entries have now been swapped, other CPU's
1426	* may still be using the old entries...
1427	*/
1428	local_bh_enable();
1429
1430	/* ... so wait for even xt_recseq on all cpus */
1431	for_each_possible_cpu(cpu) {
1432	seqcount_t *s = &per_cpu(xt_recseq, cpu);
1433	u32 seq = raw_read_seqcount(s);
1434
1435	if (seq & 1) {
1436	do {
1437	cond_resched();
1438	cpu_relax();
1439	} while (seq == raw_read_seqcount(s));
1440	}
1441	}
1442
1443	audit_log_nfcfg(name: table->name, af: table->af, nentries: private->number,
1444	op: !private->number ? AUDIT_XT_OP_REGISTER :
1445	AUDIT_XT_OP_REPLACE,
1446	GFP_KERNEL);
1447	return private;
1448	}
1449	EXPORT_SYMBOL_GPL(xt_replace_table);
1450
1451	struct xt_table *xt_register_table(struct net *net,
1452	const struct xt_table *input_table,
1453	struct xt_table_info *bootstrap,
1454	struct xt_table_info *newinfo)
1455	{
1456	struct xt_pernet *xt_net = net_generic(net, id: xt_pernet_id);
1457	struct xt_table_info *private;
1458	struct xt_table *t, *table;
1459	int ret;
1460
1461	/* Don't add one object to multiple lists. */
1462	table = kmemdup(p: input_table, size: sizeof(struct xt_table), GFP_KERNEL);
1463	if (!table) {
1464	ret = -ENOMEM;
1465	goto out;
1466	}
1467
1468	mutex_lock(&xt[table->af].mutex);
1469	/* Don't autoload: we'd eat our tail... */
1470	list_for_each_entry(t, &xt_net->tables[table->af], list) {
1471	if (strcmp(t->name, table->name) == 0) {
1472	ret = -EEXIST;
1473	goto unlock;
1474	}
1475	}
1476
1477	/* Simplifies replace_table code. */
1478	table->private = bootstrap;
1479
1480	if (!xt_replace_table(table, 0, newinfo, &ret))
1481	goto unlock;
1482
1483	private = table->private;
1484	pr_debug("table->private->number = %u\n", private->number);
1485
1486	/* save number of initial entries */
1487	private->initial_entries = private->number;
1488
1489	list_add(new: &table->list, head: &xt_net->tables[table->af]);
1490	mutex_unlock(lock: &xt[table->af].mutex);
1491	return table;
1492
1493	unlock:
1494	mutex_unlock(lock: &xt[table->af].mutex);
1495	kfree(objp: table);
1496	out:
1497	return ERR_PTR(error: ret);
1498	}
1499	EXPORT_SYMBOL_GPL(xt_register_table);
1500
1501	void *xt_unregister_table(struct xt_table *table)
1502	{
1503	struct xt_table_info *private;
1504
1505	mutex_lock(&xt[table->af].mutex);
1506	private = table->private;
1507	list_del(entry: &table->list);
1508	mutex_unlock(lock: &xt[table->af].mutex);
1509	audit_log_nfcfg(name: table->name, af: table->af, nentries: private->number,
1510	op: AUDIT_XT_OP_UNREGISTER, GFP_KERNEL);
1511	kfree(objp: table->ops);
1512	kfree(objp: table);
1513
1514	return private;
1515	}
1516	EXPORT_SYMBOL_GPL(xt_unregister_table);
1517
1518	#ifdef CONFIG_PROC_FS
1519	static void *xt_table_seq_start(struct seq_file *seq, loff_t *pos)
1520	{
1521	u8 af = (unsigned long)pde_data(inode: file_inode(f: seq->file));
1522	struct net *net = seq_file_net(seq);
1523	struct xt_pernet *xt_net;
1524
1525	xt_net = net_generic(net, id: xt_pernet_id);
1526
1527	mutex_lock(&xt[af].mutex);
1528	return seq_list_start(head: &xt_net->tables[af], pos: *pos);
1529	}
1530
1531	static void *xt_table_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1532	{
1533	u8 af = (unsigned long)pde_data(inode: file_inode(f: seq->file));
1534	struct net *net = seq_file_net(seq);
1535	struct xt_pernet *xt_net;
1536
1537	xt_net = net_generic(net, id: xt_pernet_id);
1538
1539	return seq_list_next(v, head: &xt_net->tables[af], ppos: pos);
1540	}
1541
1542	static void xt_table_seq_stop(struct seq_file *seq, void *v)
1543	{
1544	u_int8_t af = (unsigned long)pde_data(inode: file_inode(f: seq->file));
1545
1546	mutex_unlock(lock: &xt[af].mutex);
1547	}
1548
1549	static int xt_table_seq_show(struct seq_file *seq, void *v)
1550	{
1551	struct xt_table *table = list_entry(v, struct xt_table, list);
1552
1553	if (*table->name)
1554	seq_printf(m: seq, fmt: "%s\n", table->name);
1555	return 0;
1556	}
1557
1558	static const struct seq_operations xt_table_seq_ops = {
1559	.start = xt_table_seq_start,
1560	.next = xt_table_seq_next,
1561	.stop = xt_table_seq_stop,
1562	.show = xt_table_seq_show,
1563	};
1564
1565	/*
1566	* Traverse state for ip{,6}_{tables,matches} for helping crossing
1567	* the multi-AF mutexes.
1568	*/
1569	struct nf_mttg_trav {
1570	struct list_head *head, *curr;
1571	uint8_t class;
1572	};
1573
1574	enum {
1575	MTTG_TRAV_INIT,
1576	MTTG_TRAV_NFP_UNSPEC,
1577	MTTG_TRAV_NFP_SPEC,
1578	MTTG_TRAV_DONE,
1579	};
1580
1581	static void *xt_mttg_seq_next(struct seq_file *seq, void *v, loff_t *ppos,
1582	bool is_target)
1583	{
1584	static const uint8_t next_class[] = {
1585	[MTTG_TRAV_NFP_UNSPEC] = MTTG_TRAV_NFP_SPEC,
1586	[MTTG_TRAV_NFP_SPEC] = MTTG_TRAV_DONE,
1587	};
1588	uint8_t nfproto = (unsigned long)pde_data(inode: file_inode(f: seq->file));
1589	struct nf_mttg_trav *trav = seq->private;
1590
1591	if (ppos != NULL)
1592	++(*ppos);
1593
1594	switch (trav->class) {
1595	case MTTG_TRAV_INIT:
1596	trav->class = MTTG_TRAV_NFP_UNSPEC;
1597	mutex_lock(&xt[NFPROTO_UNSPEC].mutex);
1598	trav->head = trav->curr = is_target ?
1599	&xt[NFPROTO_UNSPEC].target : &xt[NFPROTO_UNSPEC].match;
1600	break;
1601	case MTTG_TRAV_NFP_UNSPEC:
1602	trav->curr = trav->curr->next;
1603	if (trav->curr != trav->head)
1604	break;
1605	mutex_unlock(lock: &xt[NFPROTO_UNSPEC].mutex);
1606	mutex_lock(&xt[nfproto].mutex);
1607	trav->head = trav->curr = is_target ?
1608	&xt[nfproto].target : &xt[nfproto].match;
1609	trav->class = next_class[trav->class];
1610	break;
1611	case MTTG_TRAV_NFP_SPEC:
1612	trav->curr = trav->curr->next;
1613	if (trav->curr != trav->head)
1614	break;
1615	fallthrough;
1616	default:
1617	return NULL;
1618	}
1619	return trav;
1620	}
1621
1622	static void *xt_mttg_seq_start(struct seq_file *seq, loff_t *pos,
1623	bool is_target)
1624	{
1625	struct nf_mttg_trav *trav = seq->private;
1626	unsigned int j;
1627
1628	trav->class = MTTG_TRAV_INIT;
1629	for (j = 0; j < *pos; ++j)
1630	if (xt_mttg_seq_next(seq, NULL, NULL, is_target) == NULL)
1631	return NULL;
1632	return trav;
1633	}
1634
1635	static void xt_mttg_seq_stop(struct seq_file *seq, void *v)
1636	{
1637	uint8_t nfproto = (unsigned long)pde_data(inode: file_inode(f: seq->file));
1638	struct nf_mttg_trav *trav = seq->private;
1639
1640	switch (trav->class) {
1641	case MTTG_TRAV_NFP_UNSPEC:
1642	mutex_unlock(lock: &xt[NFPROTO_UNSPEC].mutex);
1643	break;
1644	case MTTG_TRAV_NFP_SPEC:
1645	mutex_unlock(lock: &xt[nfproto].mutex);
1646	break;
1647	}
1648	}
1649
1650	static void *xt_match_seq_start(struct seq_file *seq, loff_t *pos)
1651	{
1652	return xt_mttg_seq_start(seq, pos, is_target: false);
1653	}
1654
1655	static void *xt_match_seq_next(struct seq_file *seq, void *v, loff_t *ppos)
1656	{
1657	return xt_mttg_seq_next(seq, v, ppos, is_target: false);
1658	}
1659
1660	static int xt_match_seq_show(struct seq_file *seq, void *v)
1661	{
1662	const struct nf_mttg_trav *trav = seq->private;
1663	const struct xt_match *match;
1664
1665	switch (trav->class) {
1666	case MTTG_TRAV_NFP_UNSPEC:
1667	case MTTG_TRAV_NFP_SPEC:
1668	if (trav->curr == trav->head)
1669	return 0;
1670	match = list_entry(trav->curr, struct xt_match, list);
1671	if (*match->name)
1672	seq_printf(m: seq, fmt: "%s\n", match->name);
1673	}
1674	return 0;
1675	}
1676
1677	static const struct seq_operations xt_match_seq_ops = {
1678	.start = xt_match_seq_start,
1679	.next = xt_match_seq_next,
1680	.stop = xt_mttg_seq_stop,
1681	.show = xt_match_seq_show,
1682	};
1683
1684	static void *xt_target_seq_start(struct seq_file *seq, loff_t *pos)
1685	{
1686	return xt_mttg_seq_start(seq, pos, is_target: true);
1687	}
1688
1689	static void *xt_target_seq_next(struct seq_file *seq, void *v, loff_t *ppos)
1690	{
1691	return xt_mttg_seq_next(seq, v, ppos, is_target: true);
1692	}
1693
1694	static int xt_target_seq_show(struct seq_file *seq, void *v)
1695	{
1696	const struct nf_mttg_trav *trav = seq->private;
1697	const struct xt_target *target;
1698
1699	switch (trav->class) {
1700	case MTTG_TRAV_NFP_UNSPEC:
1701	case MTTG_TRAV_NFP_SPEC:
1702	if (trav->curr == trav->head)
1703	return 0;
1704	target = list_entry(trav->curr, struct xt_target, list);
1705	if (*target->name)
1706	seq_printf(m: seq, fmt: "%s\n", target->name);
1707	}
1708	return 0;
1709	}
1710
1711	static const struct seq_operations xt_target_seq_ops = {
1712	.start = xt_target_seq_start,
1713	.next = xt_target_seq_next,
1714	.stop = xt_mttg_seq_stop,
1715	.show = xt_target_seq_show,
1716	};
1717
1718	#define FORMAT_TABLES "_tables_names"
1719	#define FORMAT_MATCHES "_tables_matches"
1720	#define FORMAT_TARGETS "_tables_targets"
1721
1722	#endif /* CONFIG_PROC_FS */
1723
1724	/**
1725	* xt_hook_ops_alloc - set up hooks for a new table
1726	* @table: table with metadata needed to set up hooks
1727	* @fn: Hook function
1728	*
1729	* This function will create the nf_hook_ops that the x_table needs
1730	* to hand to xt_hook_link_net().
1731	*/
1732	struct nf_hook_ops *
1733	xt_hook_ops_alloc(const struct xt_table *table, nf_hookfn *fn)
1734	{
1735	unsigned int hook_mask = table->valid_hooks;
1736	uint8_t i, num_hooks = hweight32(hook_mask);
1737	uint8_t hooknum;
1738	struct nf_hook_ops *ops;
1739
1740	if (!num_hooks)
1741	return ERR_PTR(error: -EINVAL);
1742
1743	ops = kcalloc(n: num_hooks, size: sizeof(*ops), GFP_KERNEL);
1744	if (ops == NULL)
1745	return ERR_PTR(error: -ENOMEM);
1746
1747	for (i = 0, hooknum = 0; i < num_hooks && hook_mask != 0;
1748	hook_mask >>= 1, ++hooknum) {
1749	if (!(hook_mask & 1))
1750	continue;
1751	ops[i].hook = fn;
1752	ops[i].pf = table->af;
1753	ops[i].hooknum = hooknum;
1754	ops[i].priority = table->priority;
1755	++i;
1756	}
1757
1758	return ops;
1759	}
1760	EXPORT_SYMBOL_GPL(xt_hook_ops_alloc);
1761
1762	int xt_register_template(const struct xt_table *table,
1763	int (*table_init)(struct net *net))
1764	{
1765	int ret = -EEXIST, af = table->af;
1766	struct xt_template *t;
1767
1768	mutex_lock(&xt[af].mutex);
1769
1770	list_for_each_entry(t, &xt_templates[af], list) {
1771	if (WARN_ON_ONCE(strcmp(table->name, t->name) == 0))
1772	goto out_unlock;
1773	}
1774
1775	ret = -ENOMEM;
1776	t = kzalloc(size: sizeof(*t), GFP_KERNEL);
1777	if (!t)
1778	goto out_unlock;
1779
1780	BUILD_BUG_ON(sizeof(t->name) != sizeof(table->name));
1781
1782	strscpy(t->name, table->name, sizeof(t->name));
1783	t->table_init = table_init;
1784	t->me = table->me;
1785	list_add(new: &t->list, head: &xt_templates[af]);
1786	ret = 0;
1787	out_unlock:
1788	mutex_unlock(lock: &xt[af].mutex);
1789	return ret;
1790	}
1791	EXPORT_SYMBOL_GPL(xt_register_template);
1792
1793	void xt_unregister_template(const struct xt_table *table)
1794	{
1795	struct xt_template *t;
1796	int af = table->af;
1797
1798	mutex_lock(&xt[af].mutex);
1799	list_for_each_entry(t, &xt_templates[af], list) {
1800	if (strcmp(table->name, t->name))
1801	continue;
1802
1803	list_del(entry: &t->list);
1804	mutex_unlock(lock: &xt[af].mutex);
1805	kfree(objp: t);
1806	return;
1807	}
1808
1809	mutex_unlock(lock: &xt[af].mutex);
1810	WARN_ON_ONCE(1);
1811	}
1812	EXPORT_SYMBOL_GPL(xt_unregister_template);
1813
1814	int xt_proto_init(struct net *net, u_int8_t af)
1815	{
1816	#ifdef CONFIG_PROC_FS
1817	char buf[XT_FUNCTION_MAXNAMELEN];
1818	struct proc_dir_entry *proc;
1819	kuid_t root_uid;
1820	kgid_t root_gid;
1821	#endif
1822
1823	if (af >= ARRAY_SIZE(xt_prefix))
1824	return -EINVAL;
1825
1826
1827	#ifdef CONFIG_PROC_FS
1828	root_uid = make_kuid(from: net->user_ns, uid: 0);
1829	root_gid = make_kgid(from: net->user_ns, gid: 0);
1830
1831	strscpy(buf, xt_prefix[af], sizeof(buf));
1832	strlcat(p: buf, FORMAT_TABLES, avail: sizeof(buf));
1833	proc = proc_create_net_data(name: buf, mode: 0440, parent: net->proc_net, ops: &xt_table_seq_ops,
1834	state_size: sizeof(struct seq_net_private),
1835	data: (void *)(unsigned long)af);
1836	if (!proc)
1837	goto out;
1838	if (uid_valid(uid: root_uid) && gid_valid(gid: root_gid))
1839	proc_set_user(proc, root_uid, root_gid);
1840
1841	strscpy(buf, xt_prefix[af], sizeof(buf));
1842	strlcat(p: buf, FORMAT_MATCHES, avail: sizeof(buf));
1843	proc = proc_create_seq_private(name: buf, mode: 0440, parent: net->proc_net,
1844	ops: &xt_match_seq_ops, state_size: sizeof(struct nf_mttg_trav),
1845	data: (void *)(unsigned long)af);
1846	if (!proc)
1847	goto out_remove_tables;
1848	if (uid_valid(uid: root_uid) && gid_valid(gid: root_gid))
1849	proc_set_user(proc, root_uid, root_gid);
1850
1851	strscpy(buf, xt_prefix[af], sizeof(buf));
1852	strlcat(p: buf, FORMAT_TARGETS, avail: sizeof(buf));
1853	proc = proc_create_seq_private(name: buf, mode: 0440, parent: net->proc_net,
1854	ops: &xt_target_seq_ops, state_size: sizeof(struct nf_mttg_trav),
1855	data: (void *)(unsigned long)af);
1856	if (!proc)
1857	goto out_remove_matches;
1858	if (uid_valid(uid: root_uid) && gid_valid(gid: root_gid))
1859	proc_set_user(proc, root_uid, root_gid);
1860	#endif
1861
1862	return 0;
1863
1864	#ifdef CONFIG_PROC_FS
1865	out_remove_matches:
1866	strscpy(buf, xt_prefix[af], sizeof(buf));
1867	strlcat(p: buf, FORMAT_MATCHES, avail: sizeof(buf));
1868	remove_proc_entry(buf, net->proc_net);
1869
1870	out_remove_tables:
1871	strscpy(buf, xt_prefix[af], sizeof(buf));
1872	strlcat(p: buf, FORMAT_TABLES, avail: sizeof(buf));
1873	remove_proc_entry(buf, net->proc_net);
1874	out:
1875	return -1;
1876	#endif
1877	}
1878	EXPORT_SYMBOL_GPL(xt_proto_init);
1879
1880	void xt_proto_fini(struct net *net, u_int8_t af)
1881	{
1882	#ifdef CONFIG_PROC_FS
1883	char buf[XT_FUNCTION_MAXNAMELEN];
1884
1885	strscpy(buf, xt_prefix[af], sizeof(buf));
1886	strlcat(p: buf, FORMAT_TABLES, avail: sizeof(buf));
1887	remove_proc_entry(buf, net->proc_net);
1888
1889	strscpy(buf, xt_prefix[af], sizeof(buf));
1890	strlcat(p: buf, FORMAT_TARGETS, avail: sizeof(buf));
1891	remove_proc_entry(buf, net->proc_net);
1892
1893	strscpy(buf, xt_prefix[af], sizeof(buf));
1894	strlcat(p: buf, FORMAT_MATCHES, avail: sizeof(buf));
1895	remove_proc_entry(buf, net->proc_net);
1896	#endif /*CONFIG_PROC_FS*/
1897	}
1898	EXPORT_SYMBOL_GPL(xt_proto_fini);
1899
1900	/**
1901	* xt_percpu_counter_alloc - allocate x_tables rule counter
1902	*
1903	* @state: pointer to xt_percpu allocation state
1904	* @counter: pointer to counter struct inside the ip(6)/arpt_entry struct
1905	*
1906	* On SMP, the packet counter [ ip(6)t_entry->counters.pcnt ] will then
1907	* contain the address of the real (percpu) counter.
1908	*
1909	* Rule evaluation needs to use xt_get_this_cpu_counter() helper
1910	* to fetch the real percpu counter.
1911	*
1912	* To speed up allocation and improve data locality, a 4kb block is
1913	* allocated. Freeing any counter may free an entire block, so all
1914	* counters allocated using the same state must be freed at the same
1915	* time.
1916	*
1917	* xt_percpu_counter_alloc_state contains the base address of the
1918	* allocated page and the current sub-offset.
1919	*
1920	* returns false on error.
1921	*/
1922	bool xt_percpu_counter_alloc(struct xt_percpu_counter_alloc_state *state,
1923	struct xt_counters *counter)
1924	{
1925	BUILD_BUG_ON(XT_PCPU_BLOCK_SIZE < (sizeof(*counter) * 2));
1926
1927	if (nr_cpu_ids <= 1)
1928	return true;
1929
1930	if (!state->mem) {
1931	state->mem = __alloc_percpu(XT_PCPU_BLOCK_SIZE,
1932	XT_PCPU_BLOCK_SIZE);
1933	if (!state->mem)
1934	return false;
1935	}
1936	counter->pcnt = (__force unsigned long)(state->mem + state->off);
1937	state->off += sizeof(*counter);
1938	if (state->off > (XT_PCPU_BLOCK_SIZE - sizeof(*counter))) {
1939	state->mem = NULL;
1940	state->off = 0;
1941	}
1942	return true;
1943	}
1944	EXPORT_SYMBOL_GPL(xt_percpu_counter_alloc);
1945
1946	void xt_percpu_counter_free(struct xt_counters *counters)
1947	{
1948	unsigned long pcnt = counters->pcnt;
1949
1950	if (nr_cpu_ids > 1 && (pcnt & (XT_PCPU_BLOCK_SIZE - 1)) == 0)
1951	free_percpu(pdata: (void __percpu *)pcnt);
1952	}
1953	EXPORT_SYMBOL_GPL(xt_percpu_counter_free);
1954
1955	static int __net_init xt_net_init(struct net *net)
1956	{
1957	struct xt_pernet *xt_net = net_generic(net, id: xt_pernet_id);
1958	int i;
1959
1960	for (i = 0; i < NFPROTO_NUMPROTO; i++)
1961	INIT_LIST_HEAD(list: &xt_net->tables[i]);
1962	return 0;
1963	}
1964
1965	static void __net_exit xt_net_exit(struct net *net)
1966	{
1967	struct xt_pernet *xt_net = net_generic(net, id: xt_pernet_id);
1968	int i;
1969
1970	for (i = 0; i < NFPROTO_NUMPROTO; i++)
1971	WARN_ON_ONCE(!list_empty(&xt_net->tables[i]));
1972	}
1973
1974	static struct pernet_operations xt_net_ops = {
1975	.init = xt_net_init,
1976	.exit = xt_net_exit,
1977	.id = &xt_pernet_id,
1978	.size = sizeof(struct xt_pernet),
1979	};
1980
1981	static int __init xt_init(void)
1982	{
1983	unsigned int i;
1984	int rv;
1985
1986	for_each_possible_cpu(i) {
1987	seqcount_init(&per_cpu(xt_recseq, i));
1988	}
1989
1990	xt = kcalloc(n: NFPROTO_NUMPROTO, size: sizeof(struct xt_af), GFP_KERNEL);
1991	if (!xt)
1992	return -ENOMEM;
1993
1994	for (i = 0; i < NFPROTO_NUMPROTO; i++) {
1995	mutex_init(&xt[i].mutex);
1996	#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
1997	mutex_init(&xt[i].compat_mutex);
1998	xt[i].compat_tab = NULL;
1999	#endif
2000	INIT_LIST_HEAD(list: &xt[i].target);
2001	INIT_LIST_HEAD(list: &xt[i].match);
2002	INIT_LIST_HEAD(list: &xt_templates[i]);
2003	}
2004	rv = register_pernet_subsys(&xt_net_ops);
2005	if (rv < 0)
2006	kfree(objp: xt);
2007	return rv;
2008	}
2009
2010	static void __exit xt_fini(void)
2011	{
2012	unregister_pernet_subsys(&xt_net_ops);
2013	kfree(objp: xt);
2014	}
2015
2016	module_init(xt_init);
2017	module_exit(xt_fini);
2018