1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* xfrm_state.c
4	*
5	* Changes:
6	* Mitsuru KANDA @USAGI
7	* Kazunori MIYAZAWA @USAGI
8	* Kunihiro Ishiguro <kunihiro@ipinfusion.com>
9	* IPv6 support
10	* YOSHIFUJI Hideaki @USAGI
11	* Split up af-specific functions
12	* Derek Atkins <derek@ihtfp.com>
13	* Add UDP Encapsulation
14	*
15	*/
16
17	#include <linux/compat.h>
18	#include <linux/workqueue.h>
19	#include <net/xfrm.h>
20	#include <linux/pfkeyv2.h>
21	#include <linux/ipsec.h>
22	#include <linux/module.h>
23	#include <linux/cache.h>
24	#include <linux/audit.h>
25	#include <linux/uaccess.h>
26	#include <linux/ktime.h>
27	#include <linux/slab.h>
28	#include <linux/interrupt.h>
29	#include <linux/kernel.h>
30
31	#include <crypto/aead.h>
32
33	#include "xfrm_hash.h"
34
35	#define xfrm_state_deref_prot(table, net) \
36	rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
37
38	static void xfrm_state_gc_task(struct work_struct *work);
39
40	/* Each xfrm_state may be linked to two tables:
41
42	1. Hash table by (spi,daddr,ah/esp) to find SA by SPI. (input,ctl)
43	2. Hash table by (daddr,family,reqid) to find what SAs exist for given
44	destination/tunnel endpoint. (output)
45	*/
46
47	static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024;
48	static struct kmem_cache *xfrm_state_cache __ro_after_init;
49
50	static DECLARE_WORK(xfrm_state_gc_work, xfrm_state_gc_task);
51	static HLIST_HEAD(xfrm_state_gc_list);
52
53	static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu *x)
54	{
55	return refcount_inc_not_zero(r: &x->refcnt);
56	}
57
58	static inline unsigned int xfrm_dst_hash(struct net *net,
59	const xfrm_address_t *daddr,
60	const xfrm_address_t *saddr,
61	u32 reqid,
62	unsigned short family)
63	{
64	return __xfrm_dst_hash(daddr, saddr, reqid, family, hmask: net->xfrm.state_hmask);
65	}
66
67	static inline unsigned int xfrm_src_hash(struct net *net,
68	const xfrm_address_t *daddr,
69	const xfrm_address_t *saddr,
70	unsigned short family)
71	{
72	return __xfrm_src_hash(daddr, saddr, family, hmask: net->xfrm.state_hmask);
73	}
74
75	static inline unsigned int
76	xfrm_spi_hash(struct net *net, const xfrm_address_t *daddr,
77	__be32 spi, u8 proto, unsigned short family)
78	{
79	return __xfrm_spi_hash(daddr, spi, proto, family, hmask: net->xfrm.state_hmask);
80	}
81
82	static unsigned int xfrm_seq_hash(struct net *net, u32 seq)
83	{
84	return __xfrm_seq_hash(seq, hmask: net->xfrm.state_hmask);
85	}
86
87	#define XFRM_STATE_INSERT(by, _n, _h, _type) \
88	{ \
89	struct xfrm_state *_x = NULL; \
90	\
91	if (_type != XFRM_DEV_OFFLOAD_PACKET) { \
92	hlist_for_each_entry_rcu(_x, _h, by) { \
93	if (_x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
94	continue; \
95	break; \
96	} \
97	} \
98	\
99	if (!_x || _x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
100	/* SAD is empty or consist from HW SAs only */ \
101	hlist_add_head_rcu(_n, _h); \
102	else \
103	hlist_add_before_rcu(_n, &_x->by); \
104	}
105
106	static void xfrm_hash_transfer(struct hlist_head *list,
107	struct hlist_head *ndsttable,
108	struct hlist_head *nsrctable,
109	struct hlist_head *nspitable,
110	struct hlist_head *nseqtable,
111	unsigned int nhashmask)
112	{
113	struct hlist_node *tmp;
114	struct xfrm_state *x;
115
116	hlist_for_each_entry_safe(x, tmp, list, bydst) {
117	unsigned int h;
118
119	h = __xfrm_dst_hash(daddr: &x->id.daddr, saddr: &x->props.saddr,
120	reqid: x->props.reqid, family: x->props.family,
121	hmask: nhashmask);
122	XFRM_STATE_INSERT(bydst, &x->bydst, ndsttable + h, x->xso.type);
123
124	h = __xfrm_src_hash(daddr: &x->id.daddr, saddr: &x->props.saddr,
125	family: x->props.family,
126	hmask: nhashmask);
127	XFRM_STATE_INSERT(bysrc, &x->bysrc, nsrctable + h, x->xso.type);
128
129	if (x->id.spi) {
130	h = __xfrm_spi_hash(daddr: &x->id.daddr, spi: x->id.spi,
131	proto: x->id.proto, family: x->props.family,
132	hmask: nhashmask);
133	XFRM_STATE_INSERT(byspi, &x->byspi, nspitable + h,
134	x->xso.type);
135	}
136
137	if (x->km.seq) {
138	h = __xfrm_seq_hash(seq: x->km.seq, hmask: nhashmask);
139	XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h,
140	x->xso.type);
141	}
142	}
143	}
144
145	static unsigned long xfrm_hash_new_size(unsigned int state_hmask)
146	{
147	return ((state_hmask + 1) << 1) * sizeof(struct hlist_head);
148	}
149
150	static void xfrm_hash_resize(struct work_struct *work)
151	{
152	struct net *net = container_of(work, struct net, xfrm.state_hash_work);
153	struct hlist_head *ndst, *nsrc, *nspi, *nseq, *odst, *osrc, *ospi, *oseq;
154	unsigned long nsize, osize;
155	unsigned int nhashmask, ohashmask;
156	int i;
157
158	nsize = xfrm_hash_new_size(state_hmask: net->xfrm.state_hmask);
159	ndst = xfrm_hash_alloc(sz: nsize);
160	if (!ndst)
161	return;
162	nsrc = xfrm_hash_alloc(sz: nsize);
163	if (!nsrc) {
164	xfrm_hash_free(n: ndst, sz: nsize);
165	return;
166	}
167	nspi = xfrm_hash_alloc(sz: nsize);
168	if (!nspi) {
169	xfrm_hash_free(n: ndst, sz: nsize);
170	xfrm_hash_free(n: nsrc, sz: nsize);
171	return;
172	}
173	nseq = xfrm_hash_alloc(sz: nsize);
174	if (!nseq) {
175	xfrm_hash_free(n: ndst, sz: nsize);
176	xfrm_hash_free(n: nsrc, sz: nsize);
177	xfrm_hash_free(n: nspi, sz: nsize);
178	return;
179	}
180
181	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
182	write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
183
184	nhashmask = (nsize / sizeof(struct hlist_head)) - 1U;
185	odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net);
186	for (i = net->xfrm.state_hmask; i >= 0; i--)
187	xfrm_hash_transfer(list: odst + i, ndsttable: ndst, nsrctable: nsrc, nspitable: nspi, nseqtable: nseq, nhashmask);
188
189	osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net);
190	ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net);
191	oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net);
192	ohashmask = net->xfrm.state_hmask;
193
194	rcu_assign_pointer(net->xfrm.state_bydst, ndst);
195	rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);
196	rcu_assign_pointer(net->xfrm.state_byspi, nspi);
197	rcu_assign_pointer(net->xfrm.state_byseq, nseq);
198	net->xfrm.state_hmask = nhashmask;
199
200	write_seqcount_end(&net->xfrm.xfrm_state_hash_generation);
201	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
202
203	osize = (ohashmask + 1) * sizeof(struct hlist_head);
204
205	synchronize_rcu();
206
207	xfrm_hash_free(n: odst, sz: osize);
208	xfrm_hash_free(n: osrc, sz: osize);
209	xfrm_hash_free(n: ospi, sz: osize);
210	xfrm_hash_free(n: oseq, sz: osize);
211	}
212
213	static DEFINE_SPINLOCK(xfrm_state_afinfo_lock);
214	static struct xfrm_state_afinfo __rcu *xfrm_state_afinfo[NPROTO];
215
216	static DEFINE_SPINLOCK(xfrm_state_gc_lock);
217
218	int __xfrm_state_delete(struct xfrm_state *x);
219
220	int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol);
221	static bool km_is_alive(const struct km_event *c);
222	void km_state_expired(struct xfrm_state *x, int hard, u32 portid);
223
224	int xfrm_register_type(const struct xfrm_type *type, unsigned short family)
225	{
226	struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
227	int err = 0;
228
229	if (!afinfo)
230	return -EAFNOSUPPORT;
231
232	#define X(afi, T, name) do { \
233	WARN_ON((afi)->type_ ## name); \
234	(afi)->type_ ## name = (T); \
235	} while (0)
236
237	switch (type->proto) {
238	case IPPROTO_COMP:
239	X(afinfo, type, comp);
240	break;
241	case IPPROTO_AH:
242	X(afinfo, type, ah);
243	break;
244	case IPPROTO_ESP:
245	X(afinfo, type, esp);
246	break;
247	case IPPROTO_IPIP:
248	X(afinfo, type, ipip);
249	break;
250	case IPPROTO_DSTOPTS:
251	X(afinfo, type, dstopts);
252	break;
253	case IPPROTO_ROUTING:
254	X(afinfo, type, routing);
255	break;
256	case IPPROTO_IPV6:
257	X(afinfo, type, ipip6);
258	break;
259	default:
260	WARN_ON(1);
261	err = -EPROTONOSUPPORT;
262	break;
263	}
264	#undef X
265	rcu_read_unlock();
266	return err;
267	}
268	EXPORT_SYMBOL(xfrm_register_type);
269
270	void xfrm_unregister_type(const struct xfrm_type *type, unsigned short family)
271	{
272	struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
273
274	if (unlikely(afinfo == NULL))
275	return;
276
277	#define X(afi, T, name) do { \
278	WARN_ON((afi)->type_ ## name != (T)); \
279	(afi)->type_ ## name = NULL; \
280	} while (0)
281
282	switch (type->proto) {
283	case IPPROTO_COMP:
284	X(afinfo, type, comp);
285	break;
286	case IPPROTO_AH:
287	X(afinfo, type, ah);
288	break;
289	case IPPROTO_ESP:
290	X(afinfo, type, esp);
291	break;
292	case IPPROTO_IPIP:
293	X(afinfo, type, ipip);
294	break;
295	case IPPROTO_DSTOPTS:
296	X(afinfo, type, dstopts);
297	break;
298	case IPPROTO_ROUTING:
299	X(afinfo, type, routing);
300	break;
301	case IPPROTO_IPV6:
302	X(afinfo, type, ipip6);
303	break;
304	default:
305	WARN_ON(1);
306	break;
307	}
308	#undef X
309	rcu_read_unlock();
310	}
311	EXPORT_SYMBOL(xfrm_unregister_type);
312
313	static const struct xfrm_type *xfrm_get_type(u8 proto, unsigned short family)
314	{
315	const struct xfrm_type *type = NULL;
316	struct xfrm_state_afinfo *afinfo;
317	int modload_attempted = 0;
318
319	retry:
320	afinfo = xfrm_state_get_afinfo(family);
321	if (unlikely(afinfo == NULL))
322	return NULL;
323
324	switch (proto) {
325	case IPPROTO_COMP:
326	type = afinfo->type_comp;
327	break;
328	case IPPROTO_AH:
329	type = afinfo->type_ah;
330	break;
331	case IPPROTO_ESP:
332	type = afinfo->type_esp;
333	break;
334	case IPPROTO_IPIP:
335	type = afinfo->type_ipip;
336	break;
337	case IPPROTO_DSTOPTS:
338	type = afinfo->type_dstopts;
339	break;
340	case IPPROTO_ROUTING:
341	type = afinfo->type_routing;
342	break;
343	case IPPROTO_IPV6:
344	type = afinfo->type_ipip6;
345	break;
346	default:
347	break;
348	}
349
350	if (unlikely(type && !try_module_get(type->owner)))
351	type = NULL;
352
353	rcu_read_unlock();
354
355	if (!type && !modload_attempted) {
356	request_module("xfrm-type-%d-%d", family, proto);
357	modload_attempted = 1;
358	goto retry;
359	}
360
361	return type;
362	}
363
364	static void xfrm_put_type(const struct xfrm_type *type)
365	{
366	module_put(module: type->owner);
367	}
368
369	int xfrm_register_type_offload(const struct xfrm_type_offload *type,
370	unsigned short family)
371	{
372	struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
373	int err = 0;
374
375	if (unlikely(afinfo == NULL))
376	return -EAFNOSUPPORT;
377
378	switch (type->proto) {
379	case IPPROTO_ESP:
380	WARN_ON(afinfo->type_offload_esp);
381	afinfo->type_offload_esp = type;
382	break;
383	default:
384	WARN_ON(1);
385	err = -EPROTONOSUPPORT;
386	break;
387	}
388
389	rcu_read_unlock();
390	return err;
391	}
392	EXPORT_SYMBOL(xfrm_register_type_offload);
393
394	void xfrm_unregister_type_offload(const struct xfrm_type_offload *type,
395	unsigned short family)
396	{
397	struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
398
399	if (unlikely(afinfo == NULL))
400	return;
401
402	switch (type->proto) {
403	case IPPROTO_ESP:
404	WARN_ON(afinfo->type_offload_esp != type);
405	afinfo->type_offload_esp = NULL;
406	break;
407	default:
408	WARN_ON(1);
409	break;
410	}
411	rcu_read_unlock();
412	}
413	EXPORT_SYMBOL(xfrm_unregister_type_offload);
414
415	static const struct xfrm_type_offload *
416	xfrm_get_type_offload(u8 proto, unsigned short family, bool try_load)
417	{
418	const struct xfrm_type_offload *type = NULL;
419	struct xfrm_state_afinfo *afinfo;
420
421	retry:
422	afinfo = xfrm_state_get_afinfo(family);
423	if (unlikely(afinfo == NULL))
424	return NULL;
425
426	switch (proto) {
427	case IPPROTO_ESP:
428	type = afinfo->type_offload_esp;
429	break;
430	default:
431	break;
432	}
433
434	if ((type && !try_module_get(module: type->owner)))
435	type = NULL;
436
437	rcu_read_unlock();
438
439	if (!type && try_load) {
440	request_module("xfrm-offload-%d-%d", family, proto);
441	try_load = false;
442	goto retry;
443	}
444
445	return type;
446	}
447
448	static void xfrm_put_type_offload(const struct xfrm_type_offload *type)
449	{
450	module_put(module: type->owner);
451	}
452
453	static const struct xfrm_mode xfrm4_mode_map[XFRM_MODE_MAX] = {
454	[XFRM_MODE_BEET] = {
455	.encap = XFRM_MODE_BEET,
456	.flags = XFRM_MODE_FLAG_TUNNEL,
457	.family = AF_INET,
458	},
459	[XFRM_MODE_TRANSPORT] = {
460	.encap = XFRM_MODE_TRANSPORT,
461	.family = AF_INET,
462	},
463	[XFRM_MODE_TUNNEL] = {
464	.encap = XFRM_MODE_TUNNEL,
465	.flags = XFRM_MODE_FLAG_TUNNEL,
466	.family = AF_INET,
467	},
468	};
469
470	static const struct xfrm_mode xfrm6_mode_map[XFRM_MODE_MAX] = {
471	[XFRM_MODE_BEET] = {
472	.encap = XFRM_MODE_BEET,
473	.flags = XFRM_MODE_FLAG_TUNNEL,
474	.family = AF_INET6,
475	},
476	[XFRM_MODE_ROUTEOPTIMIZATION] = {
477	.encap = XFRM_MODE_ROUTEOPTIMIZATION,
478	.family = AF_INET6,
479	},
480	[XFRM_MODE_TRANSPORT] = {
481	.encap = XFRM_MODE_TRANSPORT,
482	.family = AF_INET6,
483	},
484	[XFRM_MODE_TUNNEL] = {
485	.encap = XFRM_MODE_TUNNEL,
486	.flags = XFRM_MODE_FLAG_TUNNEL,
487	.family = AF_INET6,
488	},
489	};
490
491	static const struct xfrm_mode *xfrm_get_mode(unsigned int encap, int family)
492	{
493	const struct xfrm_mode *mode;
494
495	if (unlikely(encap >= XFRM_MODE_MAX))
496	return NULL;
497
498	switch (family) {
499	case AF_INET:
500	mode = &xfrm4_mode_map[encap];
501	if (mode->family == family)
502	return mode;
503	break;
504	case AF_INET6:
505	mode = &xfrm6_mode_map[encap];
506	if (mode->family == family)
507	return mode;
508	break;
509	default:
510	break;
511	}
512
513	return NULL;
514	}
515
516	void xfrm_state_free(struct xfrm_state *x)
517	{
518	kmem_cache_free(s: xfrm_state_cache, objp: x);
519	}
520	EXPORT_SYMBOL(xfrm_state_free);
521
522	static void ___xfrm_state_destroy(struct xfrm_state *x)
523	{
524	hrtimer_cancel(timer: &x->mtimer);
525	del_timer_sync(timer: &x->rtimer);
526	kfree(objp: x->aead);
527	kfree(objp: x->aalg);
528	kfree(objp: x->ealg);
529	kfree(objp: x->calg);
530	kfree(objp: x->encap);
531	kfree(objp: x->coaddr);
532	kfree(objp: x->replay_esn);
533	kfree(objp: x->preplay_esn);
534	if (x->type_offload)
535	xfrm_put_type_offload(type: x->type_offload);
536	if (x->type) {
537	x->type->destructor(x);
538	xfrm_put_type(type: x->type);
539	}
540	if (x->xfrag.page)
541	put_page(page: x->xfrag.page);
542	xfrm_dev_state_free(x);
543	security_xfrm_state_free(x);
544	xfrm_state_free(x);
545	}
546
547	static void xfrm_state_gc_task(struct work_struct *work)
548	{
549	struct xfrm_state *x;
550	struct hlist_node *tmp;
551	struct hlist_head gc_list;
552
553	spin_lock_bh(lock: &xfrm_state_gc_lock);
554	hlist_move_list(old: &xfrm_state_gc_list, new: &gc_list);
555	spin_unlock_bh(lock: &xfrm_state_gc_lock);
556
557	synchronize_rcu();
558
559	hlist_for_each_entry_safe(x, tmp, &gc_list, gclist)
560	___xfrm_state_destroy(x);
561	}
562
563	static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me)
564	{
565	struct xfrm_state *x = container_of(me, struct xfrm_state, mtimer);
566	enum hrtimer_restart ret = HRTIMER_NORESTART;
567	time64_t now = ktime_get_real_seconds();
568	time64_t next = TIME64_MAX;
569	int warn = 0;
570	int err = 0;
571
572	spin_lock(lock: &x->lock);
573	xfrm_dev_state_update_stats(x);
574
575	if (x->km.state == XFRM_STATE_DEAD)
576	goto out;
577	if (x->km.state == XFRM_STATE_EXPIRED)
578	goto expired;
579	if (x->lft.hard_add_expires_seconds) {
580	time64_t tmo = x->lft.hard_add_expires_seconds +
581	x->curlft.add_time - now;
582	if (tmo <= 0) {
583	if (x->xflags & XFRM_SOFT_EXPIRE) {
584	/* enter hard expire without soft expire first?!
585	* setting a new date could trigger this.
586	* workaround: fix x->curflt.add_time by below:
587	*/
588	x->curlft.add_time = now - x->saved_tmo - 1;
589	tmo = x->lft.hard_add_expires_seconds - x->saved_tmo;
590	} else
591	goto expired;
592	}
593	if (tmo < next)
594	next = tmo;
595	}
596	if (x->lft.hard_use_expires_seconds) {
597	time64_t tmo = x->lft.hard_use_expires_seconds +
598	(READ_ONCE(x->curlft.use_time) ? : now) - now;
599	if (tmo <= 0)
600	goto expired;
601	if (tmo < next)
602	next = tmo;
603	}
604	if (x->km.dying)
605	goto resched;
606	if (x->lft.soft_add_expires_seconds) {
607	time64_t tmo = x->lft.soft_add_expires_seconds +
608	x->curlft.add_time - now;
609	if (tmo <= 0) {
610	warn = 1;
611	x->xflags &= ~XFRM_SOFT_EXPIRE;
612	} else if (tmo < next) {
613	next = tmo;
614	x->xflags |= XFRM_SOFT_EXPIRE;
615	x->saved_tmo = tmo;
616	}
617	}
618	if (x->lft.soft_use_expires_seconds) {
619	time64_t tmo = x->lft.soft_use_expires_seconds +
620	(READ_ONCE(x->curlft.use_time) ? : now) - now;
621	if (tmo <= 0)
622	warn = 1;
623	else if (tmo < next)
624	next = tmo;
625	}
626
627	x->km.dying = warn;
628	if (warn)
629	km_state_expired(x, hard: 0, portid: 0);
630	resched:
631	if (next != TIME64_MAX) {
632	hrtimer_forward_now(timer: &x->mtimer, interval: ktime_set(secs: next, nsecs: 0));
633	ret = HRTIMER_RESTART;
634	}
635
636	goto out;
637
638	expired:
639	if (x->km.state == XFRM_STATE_ACQ && x->id.spi == 0)
640	x->km.state = XFRM_STATE_EXPIRED;
641
642	err = __xfrm_state_delete(x);
643	if (!err)
644	km_state_expired(x, hard: 1, portid: 0);
645
646	xfrm_audit_state_delete(x, result: err ? 0 : 1, task_valid: true);
647
648	out:
649	spin_unlock(lock: &x->lock);
650	return ret;
651	}
652
653	static void xfrm_replay_timer_handler(struct timer_list *t);
654
655	struct xfrm_state *xfrm_state_alloc(struct net *net)
656	{
657	struct xfrm_state *x;
658
659	x = kmem_cache_zalloc(k: xfrm_state_cache, GFP_ATOMIC);
660
661	if (x) {
662	write_pnet(pnet: &x->xs_net, net);
663	refcount_set(r: &x->refcnt, n: 1);
664	atomic_set(v: &x->tunnel_users, i: 0);
665	INIT_LIST_HEAD(list: &x->km.all);
666	INIT_HLIST_NODE(h: &x->bydst);
667	INIT_HLIST_NODE(h: &x->bysrc);
668	INIT_HLIST_NODE(h: &x->byspi);
669	INIT_HLIST_NODE(h: &x->byseq);
670	hrtimer_init(timer: &x->mtimer, CLOCK_BOOTTIME, mode: HRTIMER_MODE_ABS_SOFT);
671	x->mtimer.function = xfrm_timer_handler;
672	timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0);
673	x->curlft.add_time = ktime_get_real_seconds();
674	x->lft.soft_byte_limit = XFRM_INF;
675	x->lft.soft_packet_limit = XFRM_INF;
676	x->lft.hard_byte_limit = XFRM_INF;
677	x->lft.hard_packet_limit = XFRM_INF;
678	x->replay_maxage = 0;
679	x->replay_maxdiff = 0;
680	spin_lock_init(&x->lock);
681	}
682	return x;
683	}
684	EXPORT_SYMBOL(xfrm_state_alloc);
685
686	void __xfrm_state_destroy(struct xfrm_state *x, bool sync)
687	{
688	WARN_ON(x->km.state != XFRM_STATE_DEAD);
689
690	if (sync) {
691	synchronize_rcu();
692	___xfrm_state_destroy(x);
693	} else {
694	spin_lock_bh(lock: &xfrm_state_gc_lock);
695	hlist_add_head(n: &x->gclist, h: &xfrm_state_gc_list);
696	spin_unlock_bh(lock: &xfrm_state_gc_lock);
697	schedule_work(work: &xfrm_state_gc_work);
698	}
699	}
700	EXPORT_SYMBOL(__xfrm_state_destroy);
701
702	int __xfrm_state_delete(struct xfrm_state *x)
703	{
704	struct net *net = xs_net(x);
705	int err = -ESRCH;
706
707	if (x->km.state != XFRM_STATE_DEAD) {
708	x->km.state = XFRM_STATE_DEAD;
709	spin_lock(lock: &net->xfrm.xfrm_state_lock);
710	list_del(entry: &x->km.all);
711	hlist_del_rcu(n: &x->bydst);
712	hlist_del_rcu(n: &x->bysrc);
713	if (x->km.seq)
714	hlist_del_rcu(n: &x->byseq);
715	if (x->id.spi)
716	hlist_del_rcu(n: &x->byspi);
717	net->xfrm.state_num--;
718	spin_unlock(lock: &net->xfrm.xfrm_state_lock);
719
720	if (x->encap_sk)
721	sock_put(rcu_dereference_raw(x->encap_sk));
722
723	xfrm_dev_state_delete(x);
724
725	/* All xfrm_state objects are created by xfrm_state_alloc.
726	* The xfrm_state_alloc call gives a reference, and that
727	* is what we are dropping here.
728	*/
729	xfrm_state_put(x);
730	err = 0;
731	}
732
733	return err;
734	}
735	EXPORT_SYMBOL(__xfrm_state_delete);
736
737	int xfrm_state_delete(struct xfrm_state *x)
738	{
739	int err;
740
741	spin_lock_bh(lock: &x->lock);
742	err = __xfrm_state_delete(x);
743	spin_unlock_bh(lock: &x->lock);
744
745	return err;
746	}
747	EXPORT_SYMBOL(xfrm_state_delete);
748
749	#ifdef CONFIG_SECURITY_NETWORK_XFRM
750	static inline int
751	xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
752	{
753	int i, err = 0;
754
755	for (i = 0; i <= net->xfrm.state_hmask; i++) {
756	struct xfrm_state *x;
757
758	hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
759	if (xfrm_id_proto_match(proto: x->id.proto, userproto: proto) &&
760	(err = security_xfrm_state_delete(x)) != 0) {
761	xfrm_audit_state_delete(x, result: 0, task_valid);
762	return err;
763	}
764	}
765	}
766
767	return err;
768	}
769
770	static inline int
771	xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid)
772	{
773	int i, err = 0;
774
775	for (i = 0; i <= net->xfrm.state_hmask; i++) {
776	struct xfrm_state *x;
777	struct xfrm_dev_offload *xso;
778
779	hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
780	xso = &x->xso;
781
782	if (xso->dev == dev &&
783	(err = security_xfrm_state_delete(x)) != 0) {
784	xfrm_audit_state_delete(x, result: 0, task_valid);
785	return err;
786	}
787	}
788	}
789
790	return err;
791	}
792	#else
793	static inline int
794	xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
795	{
796	return 0;
797	}
798
799	static inline int
800	xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid)
801	{
802	return 0;
803	}
804	#endif
805
806	int xfrm_state_flush(struct net *net, u8 proto, bool task_valid, bool sync)
807	{
808	int i, err = 0, cnt = 0;
809
810	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
811	err = xfrm_state_flush_secctx_check(net, proto, task_valid);
812	if (err)
813	goto out;
814
815	err = -ESRCH;
816	for (i = 0; i <= net->xfrm.state_hmask; i++) {
817	struct xfrm_state *x;
818	restart:
819	hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
820	if (!xfrm_state_kern(x) &&
821	xfrm_id_proto_match(proto: x->id.proto, userproto: proto)) {
822	xfrm_state_hold(x);
823	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
824
825	err = xfrm_state_delete(x);
826	xfrm_audit_state_delete(x, result: err ? 0 : 1,
827	task_valid);
828	if (sync)
829	xfrm_state_put_sync(x);
830	else
831	xfrm_state_put(x);
832	if (!err)
833	cnt++;
834
835	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
836	goto restart;
837	}
838	}
839	}
840	out:
841	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
842	if (cnt)
843	err = 0;
844
845	return err;
846	}
847	EXPORT_SYMBOL(xfrm_state_flush);
848
849	int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid)
850	{
851	int i, err = 0, cnt = 0;
852
853	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
854	err = xfrm_dev_state_flush_secctx_check(net, dev, task_valid);
855	if (err)
856	goto out;
857
858	err = -ESRCH;
859	for (i = 0; i <= net->xfrm.state_hmask; i++) {
860	struct xfrm_state *x;
861	struct xfrm_dev_offload *xso;
862	restart:
863	hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
864	xso = &x->xso;
865
866	if (!xfrm_state_kern(x) && xso->dev == dev) {
867	xfrm_state_hold(x);
868	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
869
870	err = xfrm_state_delete(x);
871	xfrm_audit_state_delete(x, result: err ? 0 : 1,
872	task_valid);
873	xfrm_state_put(x);
874	if (!err)
875	cnt++;
876
877	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
878	goto restart;
879	}
880	}
881	}
882	if (cnt)
883	err = 0;
884
885	out:
886	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
887	return err;
888	}
889	EXPORT_SYMBOL(xfrm_dev_state_flush);
890
891	void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si)
892	{
893	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
894	si->sadcnt = net->xfrm.state_num;
895	si->sadhcnt = net->xfrm.state_hmask + 1;
896	si->sadhmcnt = xfrm_state_hashmax;
897	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
898	}
899	EXPORT_SYMBOL(xfrm_sad_getinfo);
900
901	static void
902	__xfrm4_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl)
903	{
904	const struct flowi4 *fl4 = &fl->u.ip4;
905
906	sel->daddr.a4 = fl4->daddr;
907	sel->saddr.a4 = fl4->saddr;
908	sel->dport = xfrm_flowi_dport(fl, uli: &fl4->uli);
909	sel->dport_mask = htons(0xffff);
910	sel->sport = xfrm_flowi_sport(fl, uli: &fl4->uli);
911	sel->sport_mask = htons(0xffff);
912	sel->family = AF_INET;
913	sel->prefixlen_d = 32;
914	sel->prefixlen_s = 32;
915	sel->proto = fl4->flowi4_proto;
916	sel->ifindex = fl4->flowi4_oif;
917	}
918
919	static void
920	__xfrm6_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl)
921	{
922	const struct flowi6 *fl6 = &fl->u.ip6;
923
924	/* Initialize temporary selector matching only to current session. */
925	*(struct in6_addr *)&sel->daddr = fl6->daddr;
926	*(struct in6_addr *)&sel->saddr = fl6->saddr;
927	sel->dport = xfrm_flowi_dport(fl, uli: &fl6->uli);
928	sel->dport_mask = htons(0xffff);
929	sel->sport = xfrm_flowi_sport(fl, uli: &fl6->uli);
930	sel->sport_mask = htons(0xffff);
931	sel->family = AF_INET6;
932	sel->prefixlen_d = 128;
933	sel->prefixlen_s = 128;
934	sel->proto = fl6->flowi6_proto;
935	sel->ifindex = fl6->flowi6_oif;
936	}
937
938	static void
939	xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl,
940	const struct xfrm_tmpl *tmpl,
941	const xfrm_address_t *daddr, const xfrm_address_t *saddr,
942	unsigned short family)
943	{
944	switch (family) {
945	case AF_INET:
946	__xfrm4_init_tempsel(sel: &x->sel, fl);
947	break;
948	case AF_INET6:
949	__xfrm6_init_tempsel(sel: &x->sel, fl);
950	break;
951	}
952
953	x->id = tmpl->id;
954
955	switch (tmpl->encap_family) {
956	case AF_INET:
957	if (x->id.daddr.a4 == 0)
958	x->id.daddr.a4 = daddr->a4;
959	x->props.saddr = tmpl->saddr;
960	if (x->props.saddr.a4 == 0)
961	x->props.saddr.a4 = saddr->a4;
962	break;
963	case AF_INET6:
964	if (ipv6_addr_any(a: (struct in6_addr *)&x->id.daddr))
965	memcpy(&x->id.daddr, daddr, sizeof(x->sel.daddr));
966	memcpy(&x->props.saddr, &tmpl->saddr, sizeof(x->props.saddr));
967	if (ipv6_addr_any(a: (struct in6_addr *)&x->props.saddr))
968	memcpy(&x->props.saddr, saddr, sizeof(x->props.saddr));
969	break;
970	}
971
972	x->props.mode = tmpl->mode;
973	x->props.reqid = tmpl->reqid;
974	x->props.family = tmpl->encap_family;
975	}
976
977	static struct xfrm_state *__xfrm_state_lookup_all(struct net *net, u32 mark,
978	const xfrm_address_t *daddr,
979	__be32 spi, u8 proto,
980	unsigned short family,
981	struct xfrm_dev_offload *xdo)
982	{
983	unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family);
984	struct xfrm_state *x;
985
986	hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) {
987	#ifdef CONFIG_XFRM_OFFLOAD
988	if (xdo->type == XFRM_DEV_OFFLOAD_PACKET) {
989	if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
990	/* HW states are in the head of list, there is
991	* no need to iterate further.
992	*/
993	break;
994
995	/* Packet offload: both policy and SA should
996	* have same device.
997	*/
998	if (xdo->dev != x->xso.dev)
999	continue;
1000	} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1001	/* Skip HW policy for SW lookups */
1002	continue;
1003	#endif
1004	if (x->props.family != family ||
1005	x->id.spi != spi ||
1006	x->id.proto != proto ||
1007	!xfrm_addr_equal(a: &x->id.daddr, b: daddr, family))
1008	continue;
1009
1010	if ((mark & x->mark.m) != x->mark.v)
1011	continue;
1012	if (!xfrm_state_hold_rcu(x))
1013	continue;
1014	return x;
1015	}
1016
1017	return NULL;
1018	}
1019
1020	static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark,
1021	const xfrm_address_t *daddr,
1022	__be32 spi, u8 proto,
1023	unsigned short family)
1024	{
1025	unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family);
1026	struct xfrm_state *x;
1027
1028	hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) {
1029	if (x->props.family != family ||
1030	x->id.spi != spi ||
1031	x->id.proto != proto ||
1032	!xfrm_addr_equal(a: &x->id.daddr, b: daddr, family))
1033	continue;
1034
1035	if ((mark & x->mark.m) != x->mark.v)
1036	continue;
1037	if (!xfrm_state_hold_rcu(x))
1038	continue;
1039	return x;
1040	}
1041
1042	return NULL;
1043	}
1044
1045	static struct xfrm_state *__xfrm_state_lookup_byaddr(struct net *net, u32 mark,
1046	const xfrm_address_t *daddr,
1047	const xfrm_address_t *saddr,
1048	u8 proto, unsigned short family)
1049	{
1050	unsigned int h = xfrm_src_hash(net, daddr, saddr, family);
1051	struct xfrm_state *x;
1052
1053	hlist_for_each_entry_rcu(x, net->xfrm.state_bysrc + h, bysrc) {
1054	if (x->props.family != family ||
1055	x->id.proto != proto ||
1056	!xfrm_addr_equal(a: &x->id.daddr, b: daddr, family) ||
1057	!xfrm_addr_equal(a: &x->props.saddr, b: saddr, family))
1058	continue;
1059
1060	if ((mark & x->mark.m) != x->mark.v)
1061	continue;
1062	if (!xfrm_state_hold_rcu(x))
1063	continue;
1064	return x;
1065	}
1066
1067	return NULL;
1068	}
1069
1070	static inline struct xfrm_state *
1071	__xfrm_state_locate(struct xfrm_state *x, int use_spi, int family)
1072	{
1073	struct net *net = xs_net(x);
1074	u32 mark = x->mark.v & x->mark.m;
1075
1076	if (use_spi)
1077	return __xfrm_state_lookup(net, mark, daddr: &x->id.daddr,
1078	spi: x->id.spi, proto: x->id.proto, family);
1079	else
1080	return __xfrm_state_lookup_byaddr(net, mark,
1081	daddr: &x->id.daddr,
1082	saddr: &x->props.saddr,
1083	proto: x->id.proto, family);
1084	}
1085
1086	static void xfrm_hash_grow_check(struct net *net, int have_hash_collision)
1087	{
1088	if (have_hash_collision &&
1089	(net->xfrm.state_hmask + 1) < xfrm_state_hashmax &&
1090	net->xfrm.state_num > net->xfrm.state_hmask)
1091	schedule_work(work: &net->xfrm.state_hash_work);
1092	}
1093
1094	static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x,
1095	const struct flowi *fl, unsigned short family,
1096	struct xfrm_state **best, int *acq_in_progress,
1097	int *error)
1098	{
1099	/* Resolution logic:
1100	* 1. There is a valid state with matching selector. Done.
1101	* 2. Valid state with inappropriate selector. Skip.
1102	*
1103	* Entering area of "sysdeps".
1104	*
1105	* 3. If state is not valid, selector is temporary, it selects
1106	* only session which triggered previous resolution. Key
1107	* manager will do something to install a state with proper
1108	* selector.
1109	*/
1110	if (x->km.state == XFRM_STATE_VALID) {
1111	if ((x->sel.family &&
1112	(x->sel.family != family ||
1113	!xfrm_selector_match(sel: &x->sel, fl, family))) ||
1114	!security_xfrm_state_pol_flow_match(x, xp: pol,
1115	flic: &fl->u.__fl_common))
1116	return;
1117
1118	if (!*best ||
1119	(*best)->km.dying > x->km.dying ||
1120	((*best)->km.dying == x->km.dying &&
1121	(*best)->curlft.add_time < x->curlft.add_time))
1122	*best = x;
1123	} else if (x->km.state == XFRM_STATE_ACQ) {
1124	*acq_in_progress = 1;
1125	} else if (x->km.state == XFRM_STATE_ERROR ||
1126	x->km.state == XFRM_STATE_EXPIRED) {
1127	if ((!x->sel.family ||
1128	(x->sel.family == family &&
1129	xfrm_selector_match(sel: &x->sel, fl, family))) &&
1130	security_xfrm_state_pol_flow_match(x, xp: pol,
1131	flic: &fl->u.__fl_common))
1132	*error = -ESRCH;
1133	}
1134	}
1135
1136	struct xfrm_state *
1137	xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1138	const struct flowi *fl, struct xfrm_tmpl *tmpl,
1139	struct xfrm_policy *pol, int *err,
1140	unsigned short family, u32 if_id)
1141	{
1142	static xfrm_address_t saddr_wildcard = { };
1143	struct net *net = xp_net(xp: pol);
1144	unsigned int h, h_wildcard;
1145	struct xfrm_state *x, *x0, *to_put;
1146	int acquire_in_progress = 0;
1147	int error = 0;
1148	struct xfrm_state *best = NULL;
1149	u32 mark = pol->mark.v & pol->mark.m;
1150	unsigned short encap_family = tmpl->encap_family;
1151	unsigned int sequence;
1152	struct km_event c;
1153
1154	to_put = NULL;
1155
1156	sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
1157
1158	rcu_read_lock();
1159	h = xfrm_dst_hash(net, daddr, saddr, reqid: tmpl->reqid, family: encap_family);
1160	hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h, bydst) {
1161	#ifdef CONFIG_XFRM_OFFLOAD
1162	if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1163	if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1164	/* HW states are in the head of list, there is
1165	* no need to iterate further.
1166	*/
1167	break;
1168
1169	/* Packet offload: both policy and SA should
1170	* have same device.
1171	*/
1172	if (pol->xdo.dev != x->xso.dev)
1173	continue;
1174	} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1175	/* Skip HW policy for SW lookups */
1176	continue;
1177	#endif
1178	if (x->props.family == encap_family &&
1179	x->props.reqid == tmpl->reqid &&
1180	(mark & x->mark.m) == x->mark.v &&
1181	x->if_id == if_id &&
1182	!(x->props.flags & XFRM_STATE_WILDRECV) &&
1183	xfrm_state_addr_check(x, daddr, saddr, family: encap_family) &&
1184	tmpl->mode == x->props.mode &&
1185	tmpl->id.proto == x->id.proto &&
1186	(tmpl->id.spi == x->id.spi || !tmpl->id.spi))
1187	xfrm_state_look_at(pol, x, fl, family,
1188	best: &best, acq_in_progress: &acquire_in_progress, error: &error);
1189	}
1190	if (best || acquire_in_progress)
1191	goto found;
1192
1193	h_wildcard = xfrm_dst_hash(net, daddr, saddr: &saddr_wildcard, reqid: tmpl->reqid, family: encap_family);
1194	hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h_wildcard, bydst) {
1195	#ifdef CONFIG_XFRM_OFFLOAD
1196	if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1197	if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1198	/* HW states are in the head of list, there is
1199	* no need to iterate further.
1200	*/
1201	break;
1202
1203	/* Packet offload: both policy and SA should
1204	* have same device.
1205	*/
1206	if (pol->xdo.dev != x->xso.dev)
1207	continue;
1208	} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1209	/* Skip HW policy for SW lookups */
1210	continue;
1211	#endif
1212	if (x->props.family == encap_family &&
1213	x->props.reqid == tmpl->reqid &&
1214	(mark & x->mark.m) == x->mark.v &&
1215	x->if_id == if_id &&
1216	!(x->props.flags & XFRM_STATE_WILDRECV) &&
1217	xfrm_addr_equal(a: &x->id.daddr, b: daddr, family: encap_family) &&
1218	tmpl->mode == x->props.mode &&
1219	tmpl->id.proto == x->id.proto &&
1220	(tmpl->id.spi == x->id.spi || !tmpl->id.spi))
1221	xfrm_state_look_at(pol, x, fl, family,
1222	best: &best, acq_in_progress: &acquire_in_progress, error: &error);
1223	}
1224
1225	found:
1226	x = best;
1227	if (!x && !error && !acquire_in_progress) {
1228	if (tmpl->id.spi &&
1229	(x0 = __xfrm_state_lookup_all(net, mark, daddr,
1230	spi: tmpl->id.spi, proto: tmpl->id.proto,
1231	family: encap_family,
1232	xdo: &pol->xdo)) != NULL) {
1233	to_put = x0;
1234	error = -EEXIST;
1235	goto out;
1236	}
1237
1238	c.net = net;
1239	/* If the KMs have no listeners (yet...), avoid allocating an SA
1240	* for each and every packet - garbage collection might not
1241	* handle the flood.
1242	*/
1243	if (!km_is_alive(c: &c)) {
1244	error = -ESRCH;
1245	goto out;
1246	}
1247
1248	x = xfrm_state_alloc(net);
1249	if (x == NULL) {
1250	error = -ENOMEM;
1251	goto out;
1252	}
1253	/* Initialize temporary state matching only
1254	* to current session. */
1255	xfrm_init_tempstate(x, fl, tmpl, daddr, saddr, family);
1256	memcpy(&x->mark, &pol->mark, sizeof(x->mark));
1257	x->if_id = if_id;
1258
1259	error = security_xfrm_state_alloc_acquire(x, polsec: pol->security, secid: fl->flowi_secid);
1260	if (error) {
1261	x->km.state = XFRM_STATE_DEAD;
1262	to_put = x;
1263	x = NULL;
1264	goto out;
1265	}
1266	#ifdef CONFIG_XFRM_OFFLOAD
1267	if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1268	struct xfrm_dev_offload *xdo = &pol->xdo;
1269	struct xfrm_dev_offload *xso = &x->xso;
1270
1271	xso->type = XFRM_DEV_OFFLOAD_PACKET;
1272	xso->dir = xdo->dir;
1273	xso->dev = xdo->dev;
1274	xso->real_dev = xdo->real_dev;
1275	xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ;
1276	netdev_tracker_alloc(dev: xso->dev, tracker: &xso->dev_tracker,
1277	GFP_ATOMIC);
1278	error = xso->dev->xfrmdev_ops->xdo_dev_state_add(x, NULL);
1279	if (error) {
1280	xso->dir = 0;
1281	netdev_put(dev: xso->dev, tracker: &xso->dev_tracker);
1282	xso->dev = NULL;
1283	xso->real_dev = NULL;
1284	xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
1285	x->km.state = XFRM_STATE_DEAD;
1286	to_put = x;
1287	x = NULL;
1288	goto out;
1289	}
1290	}
1291	#endif
1292	if (km_query(x, t: tmpl, pol) == 0) {
1293	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1294	x->km.state = XFRM_STATE_ACQ;
1295	list_add(new: &x->km.all, head: &net->xfrm.state_all);
1296	XFRM_STATE_INSERT(bydst, &x->bydst,
1297	net->xfrm.state_bydst + h,
1298	x->xso.type);
1299	h = xfrm_src_hash(net, daddr, saddr, family: encap_family);
1300	XFRM_STATE_INSERT(bysrc, &x->bysrc,
1301	net->xfrm.state_bysrc + h,
1302	x->xso.type);
1303	if (x->id.spi) {
1304	h = xfrm_spi_hash(net, daddr: &x->id.daddr, spi: x->id.spi, proto: x->id.proto, family: encap_family);
1305	XFRM_STATE_INSERT(byspi, &x->byspi,
1306	net->xfrm.state_byspi + h,
1307	x->xso.type);
1308	}
1309	if (x->km.seq) {
1310	h = xfrm_seq_hash(net, seq: x->km.seq);
1311	XFRM_STATE_INSERT(byseq, &x->byseq,
1312	net->xfrm.state_byseq + h,
1313	x->xso.type);
1314	}
1315	x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
1316	hrtimer_start(timer: &x->mtimer,
1317	tim: ktime_set(secs: net->xfrm.sysctl_acq_expires, nsecs: 0),
1318	mode: HRTIMER_MODE_REL_SOFT);
1319	net->xfrm.state_num++;
1320	xfrm_hash_grow_check(net, have_hash_collision: x->bydst.next != NULL);
1321	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1322	} else {
1323	#ifdef CONFIG_XFRM_OFFLOAD
1324	struct xfrm_dev_offload *xso = &x->xso;
1325
1326	if (xso->type == XFRM_DEV_OFFLOAD_PACKET) {
1327	xfrm_dev_state_delete(x);
1328	xfrm_dev_state_free(x);
1329	}
1330	#endif
1331	x->km.state = XFRM_STATE_DEAD;
1332	to_put = x;
1333	x = NULL;
1334	error = -ESRCH;
1335	}
1336	}
1337	out:
1338	if (x) {
1339	if (!xfrm_state_hold_rcu(x)) {
1340	*err = -EAGAIN;
1341	x = NULL;
1342	}
1343	} else {
1344	*err = acquire_in_progress ? -EAGAIN : error;
1345	}
1346	rcu_read_unlock();
1347	if (to_put)
1348	xfrm_state_put(x: to_put);
1349
1350	if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) {
1351	*err = -EAGAIN;
1352	if (x) {
1353	xfrm_state_put(x);
1354	x = NULL;
1355	}
1356	}
1357
1358	return x;
1359	}
1360
1361	struct xfrm_state *
1362	xfrm_stateonly_find(struct net *net, u32 mark, u32 if_id,
1363	xfrm_address_t *daddr, xfrm_address_t *saddr,
1364	unsigned short family, u8 mode, u8 proto, u32 reqid)
1365	{
1366	unsigned int h;
1367	struct xfrm_state *rx = NULL, *x = NULL;
1368
1369	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1370	h = xfrm_dst_hash(net, daddr, saddr, reqid, family);
1371	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1372	if (x->props.family == family &&
1373	x->props.reqid == reqid &&
1374	(mark & x->mark.m) == x->mark.v &&
1375	x->if_id == if_id &&
1376	!(x->props.flags & XFRM_STATE_WILDRECV) &&
1377	xfrm_state_addr_check(x, daddr, saddr, family) &&
1378	mode == x->props.mode &&
1379	proto == x->id.proto &&
1380	x->km.state == XFRM_STATE_VALID) {
1381	rx = x;
1382	break;
1383	}
1384	}
1385
1386	if (rx)
1387	xfrm_state_hold(x: rx);
1388	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1389
1390
1391	return rx;
1392	}
1393	EXPORT_SYMBOL(xfrm_stateonly_find);
1394
1395	struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi,
1396	unsigned short family)
1397	{
1398	struct xfrm_state *x;
1399	struct xfrm_state_walk *w;
1400
1401	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1402	list_for_each_entry(w, &net->xfrm.state_all, all) {
1403	x = container_of(w, struct xfrm_state, km);
1404	if (x->props.family != family ||
1405	x->id.spi != spi)
1406	continue;
1407
1408	xfrm_state_hold(x);
1409	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1410	return x;
1411	}
1412	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1413	return NULL;
1414	}
1415	EXPORT_SYMBOL(xfrm_state_lookup_byspi);
1416
1417	static void __xfrm_state_insert(struct xfrm_state *x)
1418	{
1419	struct net *net = xs_net(x);
1420	unsigned int h;
1421
1422	list_add(new: &x->km.all, head: &net->xfrm.state_all);
1423
1424	h = xfrm_dst_hash(net, daddr: &x->id.daddr, saddr: &x->props.saddr,
1425	reqid: x->props.reqid, family: x->props.family);
1426	XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,
1427	x->xso.type);
1428
1429	h = xfrm_src_hash(net, daddr: &x->id.daddr, saddr: &x->props.saddr, family: x->props.family);
1430	XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,
1431	x->xso.type);
1432
1433	if (x->id.spi) {
1434	h = xfrm_spi_hash(net, daddr: &x->id.daddr, spi: x->id.spi, proto: x->id.proto,
1435	family: x->props.family);
1436
1437	XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,
1438	x->xso.type);
1439	}
1440
1441	if (x->km.seq) {
1442	h = xfrm_seq_hash(net, seq: x->km.seq);
1443
1444	XFRM_STATE_INSERT(byseq, &x->byseq, net->xfrm.state_byseq + h,
1445	x->xso.type);
1446	}
1447
1448	hrtimer_start(timer: &x->mtimer, tim: ktime_set(secs: 1, nsecs: 0), mode: HRTIMER_MODE_REL_SOFT);
1449	if (x->replay_maxage)
1450	mod_timer(timer: &x->rtimer, expires: jiffies + x->replay_maxage);
1451
1452	net->xfrm.state_num++;
1453
1454	xfrm_hash_grow_check(net, have_hash_collision: x->bydst.next != NULL);
1455	}
1456
1457	/* net->xfrm.xfrm_state_lock is held */
1458	static void __xfrm_state_bump_genids(struct xfrm_state *xnew)
1459	{
1460	struct net *net = xs_net(x: xnew);
1461	unsigned short family = xnew->props.family;
1462	u32 reqid = xnew->props.reqid;
1463	struct xfrm_state *x;
1464	unsigned int h;
1465	u32 mark = xnew->mark.v & xnew->mark.m;
1466	u32 if_id = xnew->if_id;
1467
1468	h = xfrm_dst_hash(net, daddr: &xnew->id.daddr, saddr: &xnew->props.saddr, reqid, family);
1469	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1470	if (x->props.family == family &&
1471	x->props.reqid == reqid &&
1472	x->if_id == if_id &&
1473	(mark & x->mark.m) == x->mark.v &&
1474	xfrm_addr_equal(a: &x->id.daddr, b: &xnew->id.daddr, family) &&
1475	xfrm_addr_equal(a: &x->props.saddr, b: &xnew->props.saddr, family))
1476	x->genid++;
1477	}
1478	}
1479
1480	void xfrm_state_insert(struct xfrm_state *x)
1481	{
1482	struct net *net = xs_net(x);
1483
1484	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1485	__xfrm_state_bump_genids(xnew: x);
1486	__xfrm_state_insert(x);
1487	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1488	}
1489	EXPORT_SYMBOL(xfrm_state_insert);
1490
1491	/* net->xfrm.xfrm_state_lock is held */
1492	static struct xfrm_state *__find_acq_core(struct net *net,
1493	const struct xfrm_mark *m,
1494	unsigned short family, u8 mode,
1495	u32 reqid, u32 if_id, u8 proto,
1496	const xfrm_address_t *daddr,
1497	const xfrm_address_t *saddr,
1498	int create)
1499	{
1500	unsigned int h = xfrm_dst_hash(net, daddr, saddr, reqid, family);
1501	struct xfrm_state *x;
1502	u32 mark = m->v & m->m;
1503
1504	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1505	if (x->props.reqid != reqid ||
1506	x->props.mode != mode ||
1507	x->props.family != family ||
1508	x->km.state != XFRM_STATE_ACQ ||
1509	x->id.spi != 0 ||
1510	x->id.proto != proto ||
1511	(mark & x->mark.m) != x->mark.v ||
1512	!xfrm_addr_equal(a: &x->id.daddr, b: daddr, family) ||
1513	!xfrm_addr_equal(a: &x->props.saddr, b: saddr, family))
1514	continue;
1515
1516	xfrm_state_hold(x);
1517	return x;
1518	}
1519
1520	if (!create)
1521	return NULL;
1522
1523	x = xfrm_state_alloc(net);
1524	if (likely(x)) {
1525	switch (family) {
1526	case AF_INET:
1527	x->sel.daddr.a4 = daddr->a4;
1528	x->sel.saddr.a4 = saddr->a4;
1529	x->sel.prefixlen_d = 32;
1530	x->sel.prefixlen_s = 32;
1531	x->props.saddr.a4 = saddr->a4;
1532	x->id.daddr.a4 = daddr->a4;
1533	break;
1534
1535	case AF_INET6:
1536	x->sel.daddr.in6 = daddr->in6;
1537	x->sel.saddr.in6 = saddr->in6;
1538	x->sel.prefixlen_d = 128;
1539	x->sel.prefixlen_s = 128;
1540	x->props.saddr.in6 = saddr->in6;
1541	x->id.daddr.in6 = daddr->in6;
1542	break;
1543	}
1544
1545	x->km.state = XFRM_STATE_ACQ;
1546	x->id.proto = proto;
1547	x->props.family = family;
1548	x->props.mode = mode;
1549	x->props.reqid = reqid;
1550	x->if_id = if_id;
1551	x->mark.v = m->v;
1552	x->mark.m = m->m;
1553	x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
1554	xfrm_state_hold(x);
1555	hrtimer_start(timer: &x->mtimer,
1556	tim: ktime_set(secs: net->xfrm.sysctl_acq_expires, nsecs: 0),
1557	mode: HRTIMER_MODE_REL_SOFT);
1558	list_add(new: &x->km.all, head: &net->xfrm.state_all);
1559	XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,
1560	x->xso.type);
1561	h = xfrm_src_hash(net, daddr, saddr, family);
1562	XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,
1563	x->xso.type);
1564
1565	net->xfrm.state_num++;
1566
1567	xfrm_hash_grow_check(net, have_hash_collision: x->bydst.next != NULL);
1568	}
1569
1570	return x;
1571	}
1572
1573	static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq);
1574
1575	int xfrm_state_add(struct xfrm_state *x)
1576	{
1577	struct net *net = xs_net(x);
1578	struct xfrm_state *x1, *to_put;
1579	int family;
1580	int err;
1581	u32 mark = x->mark.v & x->mark.m;
1582	int use_spi = xfrm_id_proto_match(proto: x->id.proto, IPSEC_PROTO_ANY);
1583
1584	family = x->props.family;
1585
1586	to_put = NULL;
1587
1588	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1589
1590	x1 = __xfrm_state_locate(x, use_spi, family);
1591	if (x1) {
1592	to_put = x1;
1593	x1 = NULL;
1594	err = -EEXIST;
1595	goto out;
1596	}
1597
1598	if (use_spi && x->km.seq) {
1599	x1 = __xfrm_find_acq_byseq(net, mark, seq: x->km.seq);
1600	if (x1 && ((x1->id.proto != x->id.proto) ||
1601	!xfrm_addr_equal(a: &x1->id.daddr, b: &x->id.daddr, family))) {
1602	to_put = x1;
1603	x1 = NULL;
1604	}
1605	}
1606
1607	if (use_spi && !x1)
1608	x1 = __find_acq_core(net, m: &x->mark, family, mode: x->props.mode,
1609	reqid: x->props.reqid, if_id: x->if_id, proto: x->id.proto,
1610	daddr: &x->id.daddr, saddr: &x->props.saddr, create: 0);
1611
1612	__xfrm_state_bump_genids(xnew: x);
1613	__xfrm_state_insert(x);
1614	err = 0;
1615
1616	out:
1617	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1618
1619	if (x1) {
1620	xfrm_state_delete(x1);
1621	xfrm_state_put(x: x1);
1622	}
1623
1624	if (to_put)
1625	xfrm_state_put(x: to_put);
1626
1627	return err;
1628	}
1629	EXPORT_SYMBOL(xfrm_state_add);
1630
1631	#ifdef CONFIG_XFRM_MIGRATE
1632	static inline int clone_security(struct xfrm_state *x, struct xfrm_sec_ctx *security)
1633	{
1634	struct xfrm_user_sec_ctx *uctx;
1635	int size = sizeof(*uctx) + security->ctx_len;
1636	int err;
1637
1638	uctx = kmalloc(size, GFP_KERNEL);
1639	if (!uctx)
1640	return -ENOMEM;
1641
1642	uctx->exttype = XFRMA_SEC_CTX;
1643	uctx->len = size;
1644	uctx->ctx_doi = security->ctx_doi;
1645	uctx->ctx_alg = security->ctx_alg;
1646	uctx->ctx_len = security->ctx_len;
1647	memcpy(uctx + 1, security->ctx_str, security->ctx_len);
1648	err = security_xfrm_state_alloc(x, sec_ctx: uctx);
1649	kfree(objp: uctx);
1650	if (err)
1651	return err;
1652
1653	return 0;
1654	}
1655
1656	static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig,
1657	struct xfrm_encap_tmpl *encap)
1658	{
1659	struct net *net = xs_net(x: orig);
1660	struct xfrm_state *x = xfrm_state_alloc(net);
1661	if (!x)
1662	goto out;
1663
1664	memcpy(&x->id, &orig->id, sizeof(x->id));
1665	memcpy(&x->sel, &orig->sel, sizeof(x->sel));
1666	memcpy(&x->lft, &orig->lft, sizeof(x->lft));
1667	x->props.mode = orig->props.mode;
1668	x->props.replay_window = orig->props.replay_window;
1669	x->props.reqid = orig->props.reqid;
1670	x->props.family = orig->props.family;
1671	x->props.saddr = orig->props.saddr;
1672
1673	if (orig->aalg) {
1674	x->aalg = xfrm_algo_auth_clone(orig: orig->aalg);
1675	if (!x->aalg)
1676	goto error;
1677	}
1678	x->props.aalgo = orig->props.aalgo;
1679
1680	if (orig->aead) {
1681	x->aead = xfrm_algo_aead_clone(orig: orig->aead);
1682	x->geniv = orig->geniv;
1683	if (!x->aead)
1684	goto error;
1685	}
1686	if (orig->ealg) {
1687	x->ealg = xfrm_algo_clone(orig: orig->ealg);
1688	if (!x->ealg)
1689	goto error;
1690	}
1691	x->props.ealgo = orig->props.ealgo;
1692
1693	if (orig->calg) {
1694	x->calg = xfrm_algo_clone(orig: orig->calg);
1695	if (!x->calg)
1696	goto error;
1697	}
1698	x->props.calgo = orig->props.calgo;
1699
1700	if (encap || orig->encap) {
1701	if (encap)
1702	x->encap = kmemdup(p: encap, size: sizeof(*x->encap),
1703	GFP_KERNEL);
1704	else
1705	x->encap = kmemdup(p: orig->encap, size: sizeof(*x->encap),
1706	GFP_KERNEL);
1707
1708	if (!x->encap)
1709	goto error;
1710	}
1711
1712	if (orig->security)
1713	if (clone_security(x, security: orig->security))
1714	goto error;
1715
1716	if (orig->coaddr) {
1717	x->coaddr = kmemdup(p: orig->coaddr, size: sizeof(*x->coaddr),
1718	GFP_KERNEL);
1719	if (!x->coaddr)
1720	goto error;
1721	}
1722
1723	if (orig->replay_esn) {
1724	if (xfrm_replay_clone(x, orig))
1725	goto error;
1726	}
1727
1728	memcpy(&x->mark, &orig->mark, sizeof(x->mark));
1729	memcpy(&x->props.smark, &orig->props.smark, sizeof(x->props.smark));
1730
1731	x->props.flags = orig->props.flags;
1732	x->props.extra_flags = orig->props.extra_flags;
1733
1734	x->if_id = orig->if_id;
1735	x->tfcpad = orig->tfcpad;
1736	x->replay_maxdiff = orig->replay_maxdiff;
1737	x->replay_maxage = orig->replay_maxage;
1738	memcpy(&x->curlft, &orig->curlft, sizeof(x->curlft));
1739	x->km.state = orig->km.state;
1740	x->km.seq = orig->km.seq;
1741	x->replay = orig->replay;
1742	x->preplay = orig->preplay;
1743	x->mapping_maxage = orig->mapping_maxage;
1744	x->lastused = orig->lastused;
1745	x->new_mapping = 0;
1746	x->new_mapping_sport = 0;
1747
1748	return x;
1749
1750	error:
1751	xfrm_state_put(x);
1752	out:
1753	return NULL;
1754	}
1755
1756	struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net,
1757	u32 if_id)
1758	{
1759	unsigned int h;
1760	struct xfrm_state *x = NULL;
1761
1762	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1763
1764	if (m->reqid) {
1765	h = xfrm_dst_hash(net, daddr: &m->old_daddr, saddr: &m->old_saddr,
1766	reqid: m->reqid, family: m->old_family);
1767	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1768	if (x->props.mode != m->mode ||
1769	x->id.proto != m->proto)
1770	continue;
1771	if (m->reqid && x->props.reqid != m->reqid)
1772	continue;
1773	if (if_id != 0 && x->if_id != if_id)
1774	continue;
1775	if (!xfrm_addr_equal(a: &x->id.daddr, b: &m->old_daddr,
1776	family: m->old_family) ||
1777	!xfrm_addr_equal(a: &x->props.saddr, b: &m->old_saddr,
1778	family: m->old_family))
1779	continue;
1780	xfrm_state_hold(x);
1781	break;
1782	}
1783	} else {
1784	h = xfrm_src_hash(net, daddr: &m->old_daddr, saddr: &m->old_saddr,
1785	family: m->old_family);
1786	hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) {
1787	if (x->props.mode != m->mode ||
1788	x->id.proto != m->proto)
1789	continue;
1790	if (if_id != 0 && x->if_id != if_id)
1791	continue;
1792	if (!xfrm_addr_equal(a: &x->id.daddr, b: &m->old_daddr,
1793	family: m->old_family) ||
1794	!xfrm_addr_equal(a: &x->props.saddr, b: &m->old_saddr,
1795	family: m->old_family))
1796	continue;
1797	xfrm_state_hold(x);
1798	break;
1799	}
1800	}
1801
1802	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1803
1804	return x;
1805	}
1806	EXPORT_SYMBOL(xfrm_migrate_state_find);
1807
1808	struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x,
1809	struct xfrm_migrate *m,
1810	struct xfrm_encap_tmpl *encap)
1811	{
1812	struct xfrm_state *xc;
1813
1814	xc = xfrm_state_clone(orig: x, encap);
1815	if (!xc)
1816	return NULL;
1817
1818	xc->props.family = m->new_family;
1819
1820	if (xfrm_init_state(x: xc) < 0)
1821	goto error;
1822
1823	memcpy(&xc->id.daddr, &m->new_daddr, sizeof(xc->id.daddr));
1824	memcpy(&xc->props.saddr, &m->new_saddr, sizeof(xc->props.saddr));
1825
1826	/* add state */
1827	if (xfrm_addr_equal(a: &x->id.daddr, b: &m->new_daddr, family: m->new_family)) {
1828	/* a care is needed when the destination address of the
1829	state is to be updated as it is a part of triplet */
1830	xfrm_state_insert(xc);
1831	} else {
1832	if (xfrm_state_add(xc) < 0)
1833	goto error;
1834	}
1835
1836	return xc;
1837	error:
1838	xfrm_state_put(x: xc);
1839	return NULL;
1840	}
1841	EXPORT_SYMBOL(xfrm_state_migrate);
1842	#endif
1843
1844	int xfrm_state_update(struct xfrm_state *x)
1845	{
1846	struct xfrm_state *x1, *to_put;
1847	int err;
1848	int use_spi = xfrm_id_proto_match(proto: x->id.proto, IPSEC_PROTO_ANY);
1849	struct net *net = xs_net(x);
1850
1851	to_put = NULL;
1852
1853	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1854	x1 = __xfrm_state_locate(x, use_spi, family: x->props.family);
1855
1856	err = -ESRCH;
1857	if (!x1)
1858	goto out;
1859
1860	if (xfrm_state_kern(x: x1)) {
1861	to_put = x1;
1862	err = -EEXIST;
1863	goto out;
1864	}
1865
1866	if (x1->km.state == XFRM_STATE_ACQ) {
1867	__xfrm_state_insert(x);
1868	x = NULL;
1869	}
1870	err = 0;
1871
1872	out:
1873	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1874
1875	if (to_put)
1876	xfrm_state_put(x: to_put);
1877
1878	if (err)
1879	return err;
1880
1881	if (!x) {
1882	xfrm_state_delete(x1);
1883	xfrm_state_put(x: x1);
1884	return 0;
1885	}
1886
1887	err = -EINVAL;
1888	spin_lock_bh(lock: &x1->lock);
1889	if (likely(x1->km.state == XFRM_STATE_VALID)) {
1890	if (x->encap && x1->encap &&
1891	x->encap->encap_type == x1->encap->encap_type)
1892	memcpy(x1->encap, x->encap, sizeof(*x1->encap));
1893	else if (x->encap || x1->encap)
1894	goto fail;
1895
1896	if (x->coaddr && x1->coaddr) {
1897	memcpy(x1->coaddr, x->coaddr, sizeof(*x1->coaddr));
1898	}
1899	if (!use_spi && memcmp(p: &x1->sel, q: &x->sel, size: sizeof(x1->sel)))
1900	memcpy(&x1->sel, &x->sel, sizeof(x1->sel));
1901	memcpy(&x1->lft, &x->lft, sizeof(x1->lft));
1902	x1->km.dying = 0;
1903
1904	hrtimer_start(timer: &x1->mtimer, tim: ktime_set(secs: 1, nsecs: 0),
1905	mode: HRTIMER_MODE_REL_SOFT);
1906	if (READ_ONCE(x1->curlft.use_time))
1907	xfrm_state_check_expire(x: x1);
1908
1909	if (x->props.smark.m || x->props.smark.v || x->if_id) {
1910	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1911
1912	if (x->props.smark.m || x->props.smark.v)
1913	x1->props.smark = x->props.smark;
1914
1915	if (x->if_id)
1916	x1->if_id = x->if_id;
1917
1918	__xfrm_state_bump_genids(xnew: x1);
1919	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1920	}
1921
1922	err = 0;
1923	x->km.state = XFRM_STATE_DEAD;
1924	__xfrm_state_put(x);
1925	}
1926
1927	fail:
1928	spin_unlock_bh(lock: &x1->lock);
1929
1930	xfrm_state_put(x: x1);
1931
1932	return err;
1933	}
1934	EXPORT_SYMBOL(xfrm_state_update);
1935
1936	int xfrm_state_check_expire(struct xfrm_state *x)
1937	{
1938	xfrm_dev_state_update_stats(x);
1939
1940	if (!READ_ONCE(x->curlft.use_time))
1941	WRITE_ONCE(x->curlft.use_time, ktime_get_real_seconds());
1942
1943	if (x->curlft.bytes >= x->lft.hard_byte_limit ||
1944	x->curlft.packets >= x->lft.hard_packet_limit) {
1945	x->km.state = XFRM_STATE_EXPIRED;
1946	hrtimer_start(timer: &x->mtimer, tim: 0, mode: HRTIMER_MODE_REL_SOFT);
1947	return -EINVAL;
1948	}
1949
1950	if (!x->km.dying &&
1951	(x->curlft.bytes >= x->lft.soft_byte_limit ||
1952	x->curlft.packets >= x->lft.soft_packet_limit)) {
1953	x->km.dying = 1;
1954	km_state_expired(x, hard: 0, portid: 0);
1955	}
1956	return 0;
1957	}
1958	EXPORT_SYMBOL(xfrm_state_check_expire);
1959
1960	void xfrm_state_update_stats(struct net *net)
1961	{
1962	struct xfrm_state *x;
1963	int i;
1964
1965	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1966	for (i = 0; i <= net->xfrm.state_hmask; i++) {
1967	hlist_for_each_entry(x, net->xfrm.state_bydst + i, bydst)
1968	xfrm_dev_state_update_stats(x);
1969	}
1970	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1971	}
1972
1973	struct xfrm_state *
1974	xfrm_state_lookup(struct net *net, u32 mark, const xfrm_address_t *daddr, __be32 spi,
1975	u8 proto, unsigned short family)
1976	{
1977	struct xfrm_state *x;
1978
1979	rcu_read_lock();
1980	x = __xfrm_state_lookup(net, mark, daddr, spi, proto, family);
1981	rcu_read_unlock();
1982	return x;
1983	}
1984	EXPORT_SYMBOL(xfrm_state_lookup);
1985
1986	struct xfrm_state *
1987	xfrm_state_lookup_byaddr(struct net *net, u32 mark,
1988	const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1989	u8 proto, unsigned short family)
1990	{
1991	struct xfrm_state *x;
1992
1993	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1994	x = __xfrm_state_lookup_byaddr(net, mark, daddr, saddr, proto, family);
1995	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1996	return x;
1997	}
1998	EXPORT_SYMBOL(xfrm_state_lookup_byaddr);
1999
2000	struct xfrm_state *
2001	xfrm_find_acq(struct net *net, const struct xfrm_mark *mark, u8 mode, u32 reqid,
2002	u32 if_id, u8 proto, const xfrm_address_t *daddr,
2003	const xfrm_address_t *saddr, int create, unsigned short family)
2004	{
2005	struct xfrm_state *x;
2006
2007	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
2008	x = __find_acq_core(net, m: mark, family, mode, reqid, if_id, proto, daddr, saddr, create);
2009	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
2010
2011	return x;
2012	}
2013	EXPORT_SYMBOL(xfrm_find_acq);
2014
2015	#ifdef CONFIG_XFRM_SUB_POLICY
2016	#if IS_ENABLED(CONFIG_IPV6)
2017	/* distribution counting sort function for xfrm_state and xfrm_tmpl */
2018	static void
2019	__xfrm6_sort(void **dst, void **src, int n,
2020	int (*cmp)(const void *p), int maxclass)
2021	{
2022	int count[XFRM_MAX_DEPTH] = { };
2023	int class[XFRM_MAX_DEPTH];
2024	int i;
2025
2026	for (i = 0; i < n; i++) {
2027	int c = cmp(src[i]);
2028
2029	class[i] = c;
2030	count[c]++;
2031	}
2032
2033	for (i = 2; i < maxclass; i++)
2034	count[i] += count[i - 1];
2035
2036	for (i = 0; i < n; i++) {
2037	dst[count[class[i] - 1]++] = src[i];
2038	src[i] = NULL;
2039	}
2040	}
2041
2042	/* Rule for xfrm_state:
2043	*
2044	* rule 1: select IPsec transport except AH
2045	* rule 2: select MIPv6 RO or inbound trigger
2046	* rule 3: select IPsec transport AH
2047	* rule 4: select IPsec tunnel
2048	* rule 5: others
2049	*/
2050	static int __xfrm6_state_sort_cmp(const void *p)
2051	{
2052	const struct xfrm_state *v = p;
2053
2054	switch (v->props.mode) {
2055	case XFRM_MODE_TRANSPORT:
2056	if (v->id.proto != IPPROTO_AH)
2057	return 1;
2058	else
2059	return 3;
2060	#if IS_ENABLED(CONFIG_IPV6_MIP6)
2061	case XFRM_MODE_ROUTEOPTIMIZATION:
2062	case XFRM_MODE_IN_TRIGGER:
2063	return 2;
2064	#endif
2065	case XFRM_MODE_TUNNEL:
2066	case XFRM_MODE_BEET:
2067	return 4;
2068	}
2069	return 5;
2070	}
2071
2072	/* Rule for xfrm_tmpl:
2073	*
2074	* rule 1: select IPsec transport
2075	* rule 2: select MIPv6 RO or inbound trigger
2076	* rule 3: select IPsec tunnel
2077	* rule 4: others
2078	*/
2079	static int __xfrm6_tmpl_sort_cmp(const void *p)
2080	{
2081	const struct xfrm_tmpl *v = p;
2082
2083	switch (v->mode) {
2084	case XFRM_MODE_TRANSPORT:
2085	return 1;
2086	#if IS_ENABLED(CONFIG_IPV6_MIP6)
2087	case XFRM_MODE_ROUTEOPTIMIZATION:
2088	case XFRM_MODE_IN_TRIGGER:
2089	return 2;
2090	#endif
2091	case XFRM_MODE_TUNNEL:
2092	case XFRM_MODE_BEET:
2093	return 3;
2094	}
2095	return 4;
2096	}
2097	#else
2098	static inline int __xfrm6_state_sort_cmp(const void *p) { return 5; }
2099	static inline int __xfrm6_tmpl_sort_cmp(const void *p) { return 4; }
2100
2101	static inline void
2102	__xfrm6_sort(void **dst, void **src, int n,
2103	int (*cmp)(const void *p), int maxclass)
2104	{
2105	int i;
2106
2107	for (i = 0; i < n; i++)
2108	dst[i] = src[i];
2109	}
2110	#endif /* CONFIG_IPV6 */
2111
2112	void
2113	xfrm_tmpl_sort(struct xfrm_tmpl **dst, struct xfrm_tmpl **src, int n,
2114	unsigned short family)
2115	{
2116	int i;
2117
2118	if (family == AF_INET6)
2119	__xfrm6_sort(dst: (void **)dst, src: (void **)src, n,
2120	cmp: __xfrm6_tmpl_sort_cmp, maxclass: 5);
2121	else
2122	for (i = 0; i < n; i++)
2123	dst[i] = src[i];
2124	}
2125
2126	void
2127	xfrm_state_sort(struct xfrm_state **dst, struct xfrm_state **src, int n,
2128	unsigned short family)
2129	{
2130	int i;
2131
2132	if (family == AF_INET6)
2133	__xfrm6_sort(dst: (void **)dst, src: (void **)src, n,
2134	cmp: __xfrm6_state_sort_cmp, maxclass: 6);
2135	else
2136	for (i = 0; i < n; i++)
2137	dst[i] = src[i];
2138	}
2139	#endif
2140
2141	/* Silly enough, but I'm lazy to build resolution list */
2142
2143	static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq)
2144	{
2145	unsigned int h = xfrm_seq_hash(net, seq);
2146	struct xfrm_state *x;
2147
2148	hlist_for_each_entry_rcu(x, net->xfrm.state_byseq + h, byseq) {
2149	if (x->km.seq == seq &&
2150	(mark & x->mark.m) == x->mark.v &&
2151	x->km.state == XFRM_STATE_ACQ) {
2152	xfrm_state_hold(x);
2153	return x;
2154	}
2155	}
2156
2157	return NULL;
2158	}
2159
2160	struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq)
2161	{
2162	struct xfrm_state *x;
2163
2164	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
2165	x = __xfrm_find_acq_byseq(net, mark, seq);
2166	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
2167	return x;
2168	}
2169	EXPORT_SYMBOL(xfrm_find_acq_byseq);
2170
2171	u32 xfrm_get_acqseq(void)
2172	{
2173	u32 res;
2174	static atomic_t acqseq;
2175
2176	do {
2177	res = atomic_inc_return(v: &acqseq);
2178	} while (!res);
2179
2180	return res;
2181	}
2182	EXPORT_SYMBOL(xfrm_get_acqseq);
2183
2184	int verify_spi_info(u8 proto, u32 min, u32 max, struct netlink_ext_ack *extack)
2185	{
2186	switch (proto) {
2187	case IPPROTO_AH:
2188	case IPPROTO_ESP:
2189	break;
2190
2191	case IPPROTO_COMP:
2192	/* IPCOMP spi is 16-bits. */
2193	if (max >= 0x10000) {
2194	NL_SET_ERR_MSG(extack, "IPCOMP SPI must be <= 65535");
2195	return -EINVAL;
2196	}
2197	break;
2198
2199	default:
2200	NL_SET_ERR_MSG(extack, "Invalid protocol, must be one of AH, ESP, IPCOMP");
2201	return -EINVAL;
2202	}
2203
2204	if (min > max) {
2205	NL_SET_ERR_MSG(extack, "Invalid SPI range: min > max");
2206	return -EINVAL;
2207	}
2208
2209	return 0;
2210	}
2211	EXPORT_SYMBOL(verify_spi_info);
2212
2213	int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high,
2214	struct netlink_ext_ack *extack)
2215	{
2216	struct net *net = xs_net(x);
2217	unsigned int h;
2218	struct xfrm_state *x0;
2219	int err = -ENOENT;
2220	__be32 minspi = htonl(low);
2221	__be32 maxspi = htonl(high);
2222	__be32 newspi = 0;
2223	u32 mark = x->mark.v & x->mark.m;
2224
2225	spin_lock_bh(lock: &x->lock);
2226	if (x->km.state == XFRM_STATE_DEAD) {
2227	NL_SET_ERR_MSG(extack, "Target ACQUIRE is in DEAD state");
2228	goto unlock;
2229	}
2230
2231	err = 0;
2232	if (x->id.spi)
2233	goto unlock;
2234
2235	err = -ENOENT;
2236
2237	if (minspi == maxspi) {
2238	x0 = xfrm_state_lookup(net, mark, &x->id.daddr, minspi, x->id.proto, x->props.family);
2239	if (x0) {
2240	NL_SET_ERR_MSG(extack, "Requested SPI is already in use");
2241	xfrm_state_put(x: x0);
2242	goto unlock;
2243	}
2244	newspi = minspi;
2245	} else {
2246	u32 spi = 0;
2247	for (h = 0; h < high-low+1; h++) {
2248	spi = get_random_u32_inclusive(floor: low, ceil: high);
2249	x0 = xfrm_state_lookup(net, mark, &x->id.daddr, htonl(spi), x->id.proto, x->props.family);
2250	if (x0 == NULL) {
2251	newspi = htonl(spi);
2252	break;
2253	}
2254	xfrm_state_put(x: x0);
2255	}
2256	}
2257	if (newspi) {
2258	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
2259	x->id.spi = newspi;
2260	h = xfrm_spi_hash(net, daddr: &x->id.daddr, spi: x->id.spi, proto: x->id.proto, family: x->props.family);
2261	XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,
2262	x->xso.type);
2263	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
2264
2265	err = 0;
2266	} else {
2267	NL_SET_ERR_MSG(extack, "No SPI available in the requested range");
2268	}
2269
2270	unlock:
2271	spin_unlock_bh(lock: &x->lock);
2272
2273	return err;
2274	}
2275	EXPORT_SYMBOL(xfrm_alloc_spi);
2276
2277	static bool __xfrm_state_filter_match(struct xfrm_state *x,
2278	struct xfrm_address_filter *filter)
2279	{
2280	if (filter) {
2281	if ((filter->family == AF_INET ||
2282	filter->family == AF_INET6) &&
2283	x->props.family != filter->family)
2284	return false;
2285
2286	return addr_match(token1: &x->props.saddr, token2: &filter->saddr,
2287	prefixlen: filter->splen) &&
2288	addr_match(token1: &x->id.daddr, token2: &filter->daddr,
2289	prefixlen: filter->dplen);
2290	}
2291	return true;
2292	}
2293
2294	int xfrm_state_walk(struct net *net, struct xfrm_state_walk *walk,
2295	int (*func)(struct xfrm_state *, int, void*),
2296	void *data)
2297	{
2298	struct xfrm_state *state;
2299	struct xfrm_state_walk *x;
2300	int err = 0;
2301
2302	if (walk->seq != 0 && list_empty(head: &walk->all))
2303	return 0;
2304
2305	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
2306	if (list_empty(head: &walk->all))
2307	x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all);
2308	else
2309	x = list_first_entry(&walk->all, struct xfrm_state_walk, all);
2310	list_for_each_entry_from(x, &net->xfrm.state_all, all) {
2311	if (x->state == XFRM_STATE_DEAD)
2312	continue;
2313	state = container_of(x, struct xfrm_state, km);
2314	if (!xfrm_id_proto_match(proto: state->id.proto, userproto: walk->proto))
2315	continue;
2316	if (!__xfrm_state_filter_match(x: state, filter: walk->filter))
2317	continue;
2318	err = func(state, walk->seq, data);
2319	if (err) {
2320	list_move_tail(list: &walk->all, head: &x->all);
2321	goto out;
2322	}
2323	walk->seq++;
2324	}
2325	if (walk->seq == 0) {
2326	err = -ENOENT;
2327	goto out;
2328	}
2329	list_del_init(entry: &walk->all);
2330	out:
2331	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
2332	return err;
2333	}
2334	EXPORT_SYMBOL(xfrm_state_walk);
2335
2336	void xfrm_state_walk_init(struct xfrm_state_walk *walk, u8 proto,
2337	struct xfrm_address_filter *filter)
2338	{
2339	INIT_LIST_HEAD(list: &walk->all);
2340	walk->proto = proto;
2341	walk->state = XFRM_STATE_DEAD;
2342	walk->seq = 0;
2343	walk->filter = filter;
2344	}
2345	EXPORT_SYMBOL(xfrm_state_walk_init);
2346
2347	void xfrm_state_walk_done(struct xfrm_state_walk *walk, struct net *net)
2348	{
2349	kfree(objp: walk->filter);
2350
2351	if (list_empty(head: &walk->all))
2352	return;
2353
2354	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
2355	list_del(entry: &walk->all);
2356	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
2357	}
2358	EXPORT_SYMBOL(xfrm_state_walk_done);
2359
2360	static void xfrm_replay_timer_handler(struct timer_list *t)
2361	{
2362	struct xfrm_state *x = from_timer(x, t, rtimer);
2363
2364	spin_lock(lock: &x->lock);
2365
2366	if (x->km.state == XFRM_STATE_VALID) {
2367	if (xfrm_aevent_is_on(net: xs_net(x)))
2368	xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT);
2369	else
2370	x->xflags |= XFRM_TIME_DEFER;
2371	}
2372
2373	spin_unlock(lock: &x->lock);
2374	}
2375
2376	static LIST_HEAD(xfrm_km_list);
2377
2378	void km_policy_notify(struct xfrm_policy *xp, int dir, const struct km_event *c)
2379	{
2380	struct xfrm_mgr *km;
2381
2382	rcu_read_lock();
2383	list_for_each_entry_rcu(km, &xfrm_km_list, list)
2384	if (km->notify_policy)
2385	km->notify_policy(xp, dir, c);
2386	rcu_read_unlock();
2387	}
2388
2389	void km_state_notify(struct xfrm_state *x, const struct km_event *c)
2390	{
2391	struct xfrm_mgr *km;
2392	rcu_read_lock();
2393	list_for_each_entry_rcu(km, &xfrm_km_list, list)
2394	if (km->notify)
2395	km->notify(x, c);
2396	rcu_read_unlock();
2397	}
2398
2399	EXPORT_SYMBOL(km_policy_notify);
2400	EXPORT_SYMBOL(km_state_notify);
2401
2402	void km_state_expired(struct xfrm_state *x, int hard, u32 portid)
2403	{
2404	struct km_event c;
2405
2406	c.data.hard = hard;
2407	c.portid = portid;
2408	c.event = XFRM_MSG_EXPIRE;
2409	km_state_notify(x, &c);
2410	}
2411
2412	EXPORT_SYMBOL(km_state_expired);
2413	/*
2414	* We send to all registered managers regardless of failure
2415	* We are happy with one success
2416	*/
2417	int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol)
2418	{
2419	int err = -EINVAL, acqret;
2420	struct xfrm_mgr *km;
2421
2422	rcu_read_lock();
2423	list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2424	acqret = km->acquire(x, t, pol);
2425	if (!acqret)
2426	err = acqret;
2427	}
2428	rcu_read_unlock();
2429	return err;
2430	}
2431	EXPORT_SYMBOL(km_query);
2432
2433	static int __km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
2434	{
2435	int err = -EINVAL;
2436	struct xfrm_mgr *km;
2437
2438	rcu_read_lock();
2439	list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2440	if (km->new_mapping)
2441	err = km->new_mapping(x, ipaddr, sport);
2442	if (!err)
2443	break;
2444	}
2445	rcu_read_unlock();
2446	return err;
2447	}
2448
2449	int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
2450	{
2451	int ret = 0;
2452
2453	if (x->mapping_maxage) {
2454	if ((jiffies / HZ - x->new_mapping) > x->mapping_maxage ||
2455	x->new_mapping_sport != sport) {
2456	x->new_mapping_sport = sport;
2457	x->new_mapping = jiffies / HZ;
2458	ret = __km_new_mapping(x, ipaddr, sport);
2459	}
2460	} else {
2461	ret = __km_new_mapping(x, ipaddr, sport);
2462	}
2463
2464	return ret;
2465	}
2466	EXPORT_SYMBOL(km_new_mapping);
2467
2468	void km_policy_expired(struct xfrm_policy *pol, int dir, int hard, u32 portid)
2469	{
2470	struct km_event c;
2471
2472	c.data.hard = hard;
2473	c.portid = portid;
2474	c.event = XFRM_MSG_POLEXPIRE;
2475	km_policy_notify(pol, dir, &c);
2476	}
2477	EXPORT_SYMBOL(km_policy_expired);
2478
2479	#ifdef CONFIG_XFRM_MIGRATE
2480	int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
2481	const struct xfrm_migrate *m, int num_migrate,
2482	const struct xfrm_kmaddress *k,
2483	const struct xfrm_encap_tmpl *encap)
2484	{
2485	int err = -EINVAL;
2486	int ret;
2487	struct xfrm_mgr *km;
2488
2489	rcu_read_lock();
2490	list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2491	if (km->migrate) {
2492	ret = km->migrate(sel, dir, type, m, num_migrate, k,
2493	encap);
2494	if (!ret)
2495	err = ret;
2496	}
2497	}
2498	rcu_read_unlock();
2499	return err;
2500	}
2501	EXPORT_SYMBOL(km_migrate);
2502	#endif
2503
2504	int km_report(struct net *net, u8 proto, struct xfrm_selector *sel, xfrm_address_t *addr)
2505	{
2506	int err = -EINVAL;
2507	int ret;
2508	struct xfrm_mgr *km;
2509
2510	rcu_read_lock();
2511	list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2512	if (km->report) {
2513	ret = km->report(net, proto, sel, addr);
2514	if (!ret)
2515	err = ret;
2516	}
2517	}
2518	rcu_read_unlock();
2519	return err;
2520	}
2521	EXPORT_SYMBOL(km_report);
2522
2523	static bool km_is_alive(const struct km_event *c)
2524	{
2525	struct xfrm_mgr *km;
2526	bool is_alive = false;
2527
2528	rcu_read_lock();
2529	list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2530	if (km->is_alive && km->is_alive(c)) {
2531	is_alive = true;
2532	break;
2533	}
2534	}
2535	rcu_read_unlock();
2536
2537	return is_alive;
2538	}
2539
2540	#if IS_ENABLED(CONFIG_XFRM_USER_COMPAT)
2541	static DEFINE_SPINLOCK(xfrm_translator_lock);
2542	static struct xfrm_translator __rcu *xfrm_translator;
2543
2544	struct xfrm_translator *xfrm_get_translator(void)
2545	{
2546	struct xfrm_translator *xtr;
2547
2548	rcu_read_lock();
2549	xtr = rcu_dereference(xfrm_translator);
2550	if (unlikely(!xtr))
2551	goto out;
2552	if (!try_module_get(module: xtr->owner))
2553	xtr = NULL;
2554	out:
2555	rcu_read_unlock();
2556	return xtr;
2557	}
2558	EXPORT_SYMBOL_GPL(xfrm_get_translator);
2559
2560	void xfrm_put_translator(struct xfrm_translator *xtr)
2561	{
2562	module_put(module: xtr->owner);
2563	}
2564	EXPORT_SYMBOL_GPL(xfrm_put_translator);
2565
2566	int xfrm_register_translator(struct xfrm_translator *xtr)
2567	{
2568	int err = 0;
2569
2570	spin_lock_bh(lock: &xfrm_translator_lock);
2571	if (unlikely(xfrm_translator != NULL))
2572	err = -EEXIST;
2573	else
2574	rcu_assign_pointer(xfrm_translator, xtr);
2575	spin_unlock_bh(lock: &xfrm_translator_lock);
2576
2577	return err;
2578	}
2579	EXPORT_SYMBOL_GPL(xfrm_register_translator);
2580
2581	int xfrm_unregister_translator(struct xfrm_translator *xtr)
2582	{
2583	int err = 0;
2584
2585	spin_lock_bh(lock: &xfrm_translator_lock);
2586	if (likely(xfrm_translator != NULL)) {
2587	if (rcu_access_pointer(xfrm_translator) != xtr)
2588	err = -EINVAL;
2589	else
2590	RCU_INIT_POINTER(xfrm_translator, NULL);
2591	}
2592	spin_unlock_bh(lock: &xfrm_translator_lock);
2593	synchronize_rcu();
2594
2595	return err;
2596	}
2597	EXPORT_SYMBOL_GPL(xfrm_unregister_translator);
2598	#endif
2599
2600	int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen)
2601	{
2602	int err;
2603	u8 *data;
2604	struct xfrm_mgr *km;
2605	struct xfrm_policy *pol = NULL;
2606
2607	if (sockptr_is_null(sockptr: optval) && !optlen) {
2608	xfrm_sk_policy_insert(sk, dir: XFRM_POLICY_IN, NULL);
2609	xfrm_sk_policy_insert(sk, dir: XFRM_POLICY_OUT, NULL);
2610	__sk_dst_reset(sk);
2611	return 0;
2612	}
2613
2614	if (optlen <= 0 || optlen > PAGE_SIZE)
2615	return -EMSGSIZE;
2616
2617	data = memdup_sockptr(src: optval, len: optlen);
2618	if (IS_ERR(ptr: data))
2619	return PTR_ERR(ptr: data);
2620
2621	if (in_compat_syscall()) {
2622	struct xfrm_translator *xtr = xfrm_get_translator();
2623
2624	if (!xtr) {
2625	kfree(objp: data);
2626	return -EOPNOTSUPP;
2627	}
2628
2629	err = xtr->xlate_user_policy_sockptr(&data, optlen);
2630	xfrm_put_translator(xtr);
2631	if (err) {
2632	kfree(objp: data);
2633	return err;
2634	}
2635	}
2636
2637	err = -EINVAL;
2638	rcu_read_lock();
2639	list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2640	pol = km->compile_policy(sk, optname, data,
2641	optlen, &err);
2642	if (err >= 0)
2643	break;
2644	}
2645	rcu_read_unlock();
2646
2647	if (err >= 0) {
2648	xfrm_sk_policy_insert(sk, dir: err, pol);
2649	xfrm_pol_put(policy: pol);
2650	__sk_dst_reset(sk);
2651	err = 0;
2652	}
2653
2654	kfree(objp: data);
2655	return err;
2656	}
2657	EXPORT_SYMBOL(xfrm_user_policy);
2658
2659	static DEFINE_SPINLOCK(xfrm_km_lock);
2660
2661	void xfrm_register_km(struct xfrm_mgr *km)
2662	{
2663	spin_lock_bh(lock: &xfrm_km_lock);
2664	list_add_tail_rcu(new: &km->list, head: &xfrm_km_list);
2665	spin_unlock_bh(lock: &xfrm_km_lock);
2666	}
2667	EXPORT_SYMBOL(xfrm_register_km);
2668
2669	void xfrm_unregister_km(struct xfrm_mgr *km)
2670	{
2671	spin_lock_bh(lock: &xfrm_km_lock);
2672	list_del_rcu(entry: &km->list);
2673	spin_unlock_bh(lock: &xfrm_km_lock);
2674	synchronize_rcu();
2675	}
2676	EXPORT_SYMBOL(xfrm_unregister_km);
2677
2678	int xfrm_state_register_afinfo(struct xfrm_state_afinfo *afinfo)
2679	{
2680	int err = 0;
2681
2682	if (WARN_ON(afinfo->family >= NPROTO))
2683	return -EAFNOSUPPORT;
2684
2685	spin_lock_bh(lock: &xfrm_state_afinfo_lock);
2686	if (unlikely(xfrm_state_afinfo[afinfo->family] != NULL))
2687	err = -EEXIST;
2688	else
2689	rcu_assign_pointer(xfrm_state_afinfo[afinfo->family], afinfo);
2690	spin_unlock_bh(lock: &xfrm_state_afinfo_lock);
2691	return err;
2692	}
2693	EXPORT_SYMBOL(xfrm_state_register_afinfo);
2694
2695	int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo *afinfo)
2696	{
2697	int err = 0, family = afinfo->family;
2698
2699	if (WARN_ON(family >= NPROTO))
2700	return -EAFNOSUPPORT;
2701
2702	spin_lock_bh(lock: &xfrm_state_afinfo_lock);
2703	if (likely(xfrm_state_afinfo[afinfo->family] != NULL)) {
2704	if (rcu_access_pointer(xfrm_state_afinfo[family]) != afinfo)
2705	err = -EINVAL;
2706	else
2707	RCU_INIT_POINTER(xfrm_state_afinfo[afinfo->family], NULL);
2708	}
2709	spin_unlock_bh(lock: &xfrm_state_afinfo_lock);
2710	synchronize_rcu();
2711	return err;
2712	}
2713	EXPORT_SYMBOL(xfrm_state_unregister_afinfo);
2714
2715	struct xfrm_state_afinfo *xfrm_state_afinfo_get_rcu(unsigned int family)
2716	{
2717	if (unlikely(family >= NPROTO))
2718	return NULL;
2719
2720	return rcu_dereference(xfrm_state_afinfo[family]);
2721	}
2722	EXPORT_SYMBOL_GPL(xfrm_state_afinfo_get_rcu);
2723
2724	struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned int family)
2725	{
2726	struct xfrm_state_afinfo *afinfo;
2727	if (unlikely(family >= NPROTO))
2728	return NULL;
2729	rcu_read_lock();
2730	afinfo = rcu_dereference(xfrm_state_afinfo[family]);
2731	if (unlikely(!afinfo))
2732	rcu_read_unlock();
2733	return afinfo;
2734	}
2735
2736	void xfrm_flush_gc(void)
2737	{
2738	flush_work(work: &xfrm_state_gc_work);
2739	}
2740	EXPORT_SYMBOL(xfrm_flush_gc);
2741
2742	/* Temporarily located here until net/xfrm/xfrm_tunnel.c is created */
2743	void xfrm_state_delete_tunnel(struct xfrm_state *x)
2744	{
2745	if (x->tunnel) {
2746	struct xfrm_state *t = x->tunnel;
2747
2748	if (atomic_read(v: &t->tunnel_users) == 2)
2749	xfrm_state_delete(t);
2750	atomic_dec(v: &t->tunnel_users);
2751	xfrm_state_put_sync(x: t);
2752	x->tunnel = NULL;
2753	}
2754	}
2755	EXPORT_SYMBOL(xfrm_state_delete_tunnel);
2756
2757	u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
2758	{
2759	const struct xfrm_type *type = READ_ONCE(x->type);
2760	struct crypto_aead *aead;
2761	u32 blksize, net_adj = 0;
2762
2763	if (x->km.state != XFRM_STATE_VALID ||
2764	!type || type->proto != IPPROTO_ESP)
2765	return mtu - x->props.header_len;
2766
2767	aead = x->data;
2768	blksize = ALIGN(crypto_aead_blocksize(aead), 4);
2769
2770	switch (x->props.mode) {
2771	case XFRM_MODE_TRANSPORT:
2772	case XFRM_MODE_BEET:
2773	if (x->props.family == AF_INET)
2774	net_adj = sizeof(struct iphdr);
2775	else if (x->props.family == AF_INET6)
2776	net_adj = sizeof(struct ipv6hdr);
2777	break;
2778	case XFRM_MODE_TUNNEL:
2779	break;
2780	default:
2781	WARN_ON_ONCE(1);
2782	break;
2783	}
2784
2785	return ((mtu - x->props.header_len - crypto_aead_authsize(tfm: aead) -
2786	net_adj) & ~(blksize - 1)) + net_adj - 2;
2787	}
2788	EXPORT_SYMBOL_GPL(xfrm_state_mtu);
2789
2790	int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload,
2791	struct netlink_ext_ack *extack)
2792	{
2793	const struct xfrm_mode *inner_mode;
2794	const struct xfrm_mode *outer_mode;
2795	int family = x->props.family;
2796	int err;
2797
2798	if (family == AF_INET &&
2799	READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc))
2800	x->props.flags |= XFRM_STATE_NOPMTUDISC;
2801
2802	err = -EPROTONOSUPPORT;
2803
2804	if (x->sel.family != AF_UNSPEC) {
2805	inner_mode = xfrm_get_mode(encap: x->props.mode, family: x->sel.family);
2806	if (inner_mode == NULL) {
2807	NL_SET_ERR_MSG(extack, "Requested mode not found");
2808	goto error;
2809	}
2810
2811	if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) &&
2812	family != x->sel.family) {
2813	NL_SET_ERR_MSG(extack, "Only tunnel modes can accommodate a change of family");
2814	goto error;
2815	}
2816
2817	x->inner_mode = *inner_mode;
2818	} else {
2819	const struct xfrm_mode *inner_mode_iaf;
2820	int iafamily = AF_INET;
2821
2822	inner_mode = xfrm_get_mode(encap: x->props.mode, family: x->props.family);
2823	if (inner_mode == NULL) {
2824	NL_SET_ERR_MSG(extack, "Requested mode not found");
2825	goto error;
2826	}
2827
2828	x->inner_mode = *inner_mode;
2829
2830	if (x->props.family == AF_INET)
2831	iafamily = AF_INET6;
2832
2833	inner_mode_iaf = xfrm_get_mode(encap: x->props.mode, family: iafamily);
2834	if (inner_mode_iaf) {
2835	if (inner_mode_iaf->flags & XFRM_MODE_FLAG_TUNNEL)
2836	x->inner_mode_iaf = *inner_mode_iaf;
2837	}
2838	}
2839
2840	x->type = xfrm_get_type(proto: x->id.proto, family);
2841	if (x->type == NULL) {
2842	NL_SET_ERR_MSG(extack, "Requested type not found");
2843	goto error;
2844	}
2845
2846	x->type_offload = xfrm_get_type_offload(proto: x->id.proto, family, try_load: offload);
2847
2848	err = x->type->init_state(x, extack);
2849	if (err)
2850	goto error;
2851
2852	outer_mode = xfrm_get_mode(encap: x->props.mode, family);
2853	if (!outer_mode) {
2854	NL_SET_ERR_MSG(extack, "Requested mode not found");
2855	err = -EPROTONOSUPPORT;
2856	goto error;
2857	}
2858
2859	x->outer_mode = *outer_mode;
2860	if (init_replay) {
2861	err = xfrm_init_replay(x, extack);
2862	if (err)
2863	goto error;
2864	}
2865
2866	error:
2867	return err;
2868	}
2869
2870	EXPORT_SYMBOL(__xfrm_init_state);
2871
2872	int xfrm_init_state(struct xfrm_state *x)
2873	{
2874	int err;
2875
2876	err = __xfrm_init_state(x, true, false, NULL);
2877	if (!err)
2878	x->km.state = XFRM_STATE_VALID;
2879
2880	return err;
2881	}
2882
2883	EXPORT_SYMBOL(xfrm_init_state);
2884
2885	int __net_init xfrm_state_init(struct net *net)
2886	{
2887	unsigned int sz;
2888
2889	if (net_eq(net1: net, net2: &init_net))
2890	xfrm_state_cache = KMEM_CACHE(xfrm_state,
2891	SLAB_HWCACHE_ALIGN | SLAB_PANIC);
2892
2893	INIT_LIST_HEAD(list: &net->xfrm.state_all);
2894
2895	sz = sizeof(struct hlist_head) * 8;
2896
2897	net->xfrm.state_bydst = xfrm_hash_alloc(sz);
2898	if (!net->xfrm.state_bydst)
2899	goto out_bydst;
2900	net->xfrm.state_bysrc = xfrm_hash_alloc(sz);
2901	if (!net->xfrm.state_bysrc)
2902	goto out_bysrc;
2903	net->xfrm.state_byspi = xfrm_hash_alloc(sz);
2904	if (!net->xfrm.state_byspi)
2905	goto out_byspi;
2906	net->xfrm.state_byseq = xfrm_hash_alloc(sz);
2907	if (!net->xfrm.state_byseq)
2908	goto out_byseq;
2909	net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1);
2910
2911	net->xfrm.state_num = 0;
2912	INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize);
2913	spin_lock_init(&net->xfrm.xfrm_state_lock);
2914	seqcount_spinlock_init(&net->xfrm.xfrm_state_hash_generation,
2915	&net->xfrm.xfrm_state_lock);
2916	return 0;
2917
2918	out_byseq:
2919	xfrm_hash_free(n: net->xfrm.state_byspi, sz);
2920	out_byspi:
2921	xfrm_hash_free(n: net->xfrm.state_bysrc, sz);
2922	out_bysrc:
2923	xfrm_hash_free(n: net->xfrm.state_bydst, sz);
2924	out_bydst:
2925	return -ENOMEM;
2926	}
2927
2928	void xfrm_state_fini(struct net *net)
2929	{
2930	unsigned int sz;
2931
2932	flush_work(work: &net->xfrm.state_hash_work);
2933	flush_work(work: &xfrm_state_gc_work);
2934	xfrm_state_flush(net, 0, false, true);
2935
2936	WARN_ON(!list_empty(&net->xfrm.state_all));
2937
2938	sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head);
2939	WARN_ON(!hlist_empty(net->xfrm.state_byseq));
2940	xfrm_hash_free(n: net->xfrm.state_byseq, sz);
2941	WARN_ON(!hlist_empty(net->xfrm.state_byspi));
2942	xfrm_hash_free(n: net->xfrm.state_byspi, sz);
2943	WARN_ON(!hlist_empty(net->xfrm.state_bysrc));
2944	xfrm_hash_free(n: net->xfrm.state_bysrc, sz);
2945	WARN_ON(!hlist_empty(net->xfrm.state_bydst));
2946	xfrm_hash_free(n: net->xfrm.state_bydst, sz);
2947	}
2948
2949	#ifdef CONFIG_AUDITSYSCALL
2950	static void xfrm_audit_helper_sainfo(struct xfrm_state *x,
2951	struct audit_buffer *audit_buf)
2952	{
2953	struct xfrm_sec_ctx *ctx = x->security;
2954	u32 spi = ntohl(x->id.spi);
2955
2956	if (ctx)
2957	audit_log_format(ab: audit_buf, fmt: " sec_alg=%u sec_doi=%u sec_obj=%s",
2958	ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
2959
2960	switch (x->props.family) {
2961	case AF_INET:
2962	audit_log_format(ab: audit_buf, fmt: " src=%pI4 dst=%pI4",
2963	&x->props.saddr.a4, &x->id.daddr.a4);
2964	break;
2965	case AF_INET6:
2966	audit_log_format(ab: audit_buf, fmt: " src=%pI6 dst=%pI6",
2967	x->props.saddr.a6, x->id.daddr.a6);
2968	break;
2969	}
2970
2971	audit_log_format(ab: audit_buf, fmt: " spi=%u(0x%x)", spi, spi);
2972	}
2973
2974	static void xfrm_audit_helper_pktinfo(struct sk_buff *skb, u16 family,
2975	struct audit_buffer *audit_buf)
2976	{
2977	const struct iphdr *iph4;
2978	const struct ipv6hdr *iph6;
2979
2980	switch (family) {
2981	case AF_INET:
2982	iph4 = ip_hdr(skb);
2983	audit_log_format(ab: audit_buf, fmt: " src=%pI4 dst=%pI4",
2984	&iph4->saddr, &iph4->daddr);
2985	break;
2986	case AF_INET6:
2987	iph6 = ipv6_hdr(skb);
2988	audit_log_format(ab: audit_buf,
2989	fmt: " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x",
2990	&iph6->saddr, &iph6->daddr,
2991	iph6->flow_lbl[0] & 0x0f,
2992	iph6->flow_lbl[1],
2993	iph6->flow_lbl[2]);
2994	break;
2995	}
2996	}
2997
2998	void xfrm_audit_state_add(struct xfrm_state *x, int result, bool task_valid)
2999	{
3000	struct audit_buffer *audit_buf;
3001
3002	audit_buf = xfrm_audit_start(op: "SAD-add");
3003	if (audit_buf == NULL)
3004	return;
3005	xfrm_audit_helper_usrinfo(task_valid, audit_buf);
3006	xfrm_audit_helper_sainfo(x, audit_buf);
3007	audit_log_format(ab: audit_buf, fmt: " res=%u", result);
3008	audit_log_end(ab: audit_buf);
3009	}
3010	EXPORT_SYMBOL_GPL(xfrm_audit_state_add);
3011
3012	void xfrm_audit_state_delete(struct xfrm_state *x, int result, bool task_valid)
3013	{
3014	struct audit_buffer *audit_buf;
3015
3016	audit_buf = xfrm_audit_start(op: "SAD-delete");
3017	if (audit_buf == NULL)
3018	return;
3019	xfrm_audit_helper_usrinfo(task_valid, audit_buf);
3020	xfrm_audit_helper_sainfo(x, audit_buf);
3021	audit_log_format(ab: audit_buf, fmt: " res=%u", result);
3022	audit_log_end(ab: audit_buf);
3023	}
3024	EXPORT_SYMBOL_GPL(xfrm_audit_state_delete);
3025
3026	void xfrm_audit_state_replay_overflow(struct xfrm_state *x,
3027	struct sk_buff *skb)
3028	{
3029	struct audit_buffer *audit_buf;
3030	u32 spi;
3031
3032	audit_buf = xfrm_audit_start(op: "SA-replay-overflow");
3033	if (audit_buf == NULL)
3034	return;
3035	xfrm_audit_helper_pktinfo(skb, family: x->props.family, audit_buf);
3036	/* don't record the sequence number because it's inherent in this kind
3037	* of audit message */
3038	spi = ntohl(x->id.spi);
3039	audit_log_format(ab: audit_buf, fmt: " spi=%u(0x%x)", spi, spi);
3040	audit_log_end(ab: audit_buf);
3041	}
3042	EXPORT_SYMBOL_GPL(xfrm_audit_state_replay_overflow);
3043
3044	void xfrm_audit_state_replay(struct xfrm_state *x,
3045	struct sk_buff *skb, __be32 net_seq)
3046	{
3047	struct audit_buffer *audit_buf;
3048	u32 spi;
3049
3050	audit_buf = xfrm_audit_start(op: "SA-replayed-pkt");
3051	if (audit_buf == NULL)
3052	return;
3053	xfrm_audit_helper_pktinfo(skb, family: x->props.family, audit_buf);
3054	spi = ntohl(x->id.spi);
3055	audit_log_format(ab: audit_buf, fmt: " spi=%u(0x%x) seqno=%u",
3056	spi, spi, ntohl(net_seq));
3057	audit_log_end(ab: audit_buf);
3058	}
3059	EXPORT_SYMBOL_GPL(xfrm_audit_state_replay);
3060
3061	void xfrm_audit_state_notfound_simple(struct sk_buff *skb, u16 family)
3062	{
3063	struct audit_buffer *audit_buf;
3064
3065	audit_buf = xfrm_audit_start(op: "SA-notfound");
3066	if (audit_buf == NULL)
3067	return;
3068	xfrm_audit_helper_pktinfo(skb, family, audit_buf);
3069	audit_log_end(ab: audit_buf);
3070	}
3071	EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound_simple);
3072
3073	void xfrm_audit_state_notfound(struct sk_buff *skb, u16 family,
3074	__be32 net_spi, __be32 net_seq)
3075	{
3076	struct audit_buffer *audit_buf;
3077	u32 spi;
3078
3079	audit_buf = xfrm_audit_start(op: "SA-notfound");
3080	if (audit_buf == NULL)
3081	return;
3082	xfrm_audit_helper_pktinfo(skb, family, audit_buf);
3083	spi = ntohl(net_spi);
3084	audit_log_format(ab: audit_buf, fmt: " spi=%u(0x%x) seqno=%u",
3085	spi, spi, ntohl(net_seq));
3086	audit_log_end(ab: audit_buf);
3087	}
3088	EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound);
3089
3090	void xfrm_audit_state_icvfail(struct xfrm_state *x,
3091	struct sk_buff *skb, u8 proto)
3092	{
3093	struct audit_buffer *audit_buf;
3094	__be32 net_spi;
3095	__be32 net_seq;
3096
3097	audit_buf = xfrm_audit_start(op: "SA-icv-failure");
3098	if (audit_buf == NULL)
3099	return;
3100	xfrm_audit_helper_pktinfo(skb, family: x->props.family, audit_buf);
3101	if (xfrm_parse_spi(skb, nexthdr: proto, spi: &net_spi, seq: &net_seq) == 0) {
3102	u32 spi = ntohl(net_spi);
3103	audit_log_format(ab: audit_buf, fmt: " spi=%u(0x%x) seqno=%u",
3104	spi, spi, ntohl(net_seq));
3105	}
3106	audit_log_end(ab: audit_buf);
3107	}
3108	EXPORT_SYMBOL_GPL(xfrm_audit_state_icvfail);
3109	#endif /* CONFIG_AUDITSYSCALL */
3110