1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* drivers/net/macsec.c - MACsec device
4	*
5	* Copyright (c) 2015 Sabrina Dubroca <sd@queasysnail.net>
6	*/
7
8	#include <linux/types.h>
9	#include <linux/skbuff.h>
10	#include <linux/socket.h>
11	#include <linux/module.h>
12	#include <crypto/aead.h>
13	#include <linux/etherdevice.h>
14	#include <linux/netdevice.h>
15	#include <linux/rtnetlink.h>
16	#include <linux/refcount.h>
17	#include <net/genetlink.h>
18	#include <net/sock.h>
19	#include <net/gro_cells.h>
20	#include <net/macsec.h>
21	#include <net/dst_metadata.h>
22	#include <net/netdev_lock.h>
23	#include <linux/phy.h>
24	#include <linux/byteorder/generic.h>
25	#include <linux/if_arp.h>
26
27	#include <uapi/linux/if_macsec.h>
28
29	/* SecTAG length = macsec_eth_header without the optional SCI */
30	#define MACSEC_TAG_LEN 6
31
32	struct macsec_eth_header {
33	struct ethhdr eth;
34	/* SecTAG */
35	u8 tci_an;
36	#if defined(__LITTLE_ENDIAN_BITFIELD)
37	u8 short_length:6,
38	unused:2;
39	#elif defined(__BIG_ENDIAN_BITFIELD)
40	u8 unused:2,
41	short_length:6;
42	#else
43	#error "Please fix <asm/byteorder.h>"
44	#endif
45	__be32 packet_number;
46	u8 secure_channel_id[8]; /* optional */
47	} __packed;
48
49	/* minimum secure data length deemed "not short", see IEEE 802.1AE-2006 9.7 */
50	#define MIN_NON_SHORT_LEN 48
51
52	#define GCM_AES_IV_LEN 12
53
54	#define for_each_rxsc(secy, sc) \
55	for (sc = rcu_dereference_bh(secy->rx_sc); \
56	sc; \
57	sc = rcu_dereference_bh(sc->next))
58	#define for_each_rxsc_rtnl(secy, sc) \
59	for (sc = rtnl_dereference(secy->rx_sc); \
60	sc; \
61	sc = rtnl_dereference(sc->next))
62
63	#define pn_same_half(pn1, pn2) (!(((pn1) >> 31) ^ ((pn2) >> 31)))
64
65	struct gcm_iv_xpn {
66	union {
67	u8 short_secure_channel_id[4];
68	ssci_t ssci;
69	};
70	__be64 pn;
71	} __packed;
72
73	struct gcm_iv {
74	union {
75	u8 secure_channel_id[8];
76	sci_t sci;
77	};
78	__be32 pn;
79	};
80
81	#define MACSEC_VALIDATE_DEFAULT MACSEC_VALIDATE_STRICT
82
83	struct pcpu_secy_stats {
84	struct macsec_dev_stats stats;
85	struct u64_stats_sync syncp;
86	};
87
88	/**
89	* struct macsec_dev - private data
90	* @secy: SecY config
91	* @real_dev: pointer to underlying netdevice
92	* @dev_tracker: refcount tracker for @real_dev reference
93	* @stats: MACsec device stats
94	* @secys: linked list of SecY's on the underlying device
95	* @gro_cells: pointer to the Generic Receive Offload cell
96	* @offload: status of offloading on the MACsec device
97	* @insert_tx_tag: when offloading, device requires to insert an
98	* additional tag
99	*/
100	struct macsec_dev {
101	struct macsec_secy secy;
102	struct net_device *real_dev;
103	netdevice_tracker dev_tracker;
104	struct pcpu_secy_stats __percpu *stats;
105	struct list_head secys;
106	struct gro_cells gro_cells;
107	enum macsec_offload offload;
108	bool insert_tx_tag;
109	};
110
111	/**
112	* struct macsec_rxh_data - rx_handler private argument
113	* @secys: linked list of SecY's on this underlying device
114	*/
115	struct macsec_rxh_data {
116	struct list_head secys;
117	};
118
119	static struct macsec_dev *macsec_priv(const struct net_device *dev)
120	{
121	return (struct macsec_dev *)netdev_priv(dev);
122	}
123
124	static struct macsec_rxh_data *macsec_data_rcu(const struct net_device *dev)
125	{
126	return rcu_dereference_bh(dev->rx_handler_data);
127	}
128
129	static struct macsec_rxh_data *macsec_data_rtnl(const struct net_device *dev)
130	{
131	return rtnl_dereference(dev->rx_handler_data);
132	}
133
134	struct macsec_cb {
135	struct aead_request *req;
136	union {
137	struct macsec_tx_sa *tx_sa;
138	struct macsec_rx_sa *rx_sa;
139	};
140	u8 assoc_num;
141	bool valid;
142	bool has_sci;
143	};
144
145	static struct macsec_rx_sa *macsec_rxsa_get(struct macsec_rx_sa __rcu *ptr)
146	{
147	struct macsec_rx_sa *sa = rcu_dereference_bh(ptr);
148
149	if (!sa || !sa->active)
150	return NULL;
151
152	if (!refcount_inc_not_zero(r: &sa->refcnt))
153	return NULL;
154
155	return sa;
156	}
157
158	static void free_rx_sc_rcu(struct rcu_head *head)
159	{
160	struct macsec_rx_sc *rx_sc = container_of(head, struct macsec_rx_sc, rcu_head);
161
162	free_percpu(pdata: rx_sc->stats);
163	kfree(objp: rx_sc);
164	}
165
166	static struct macsec_rx_sc *macsec_rxsc_get(struct macsec_rx_sc *sc)
167	{
168	return refcount_inc_not_zero(r: &sc->refcnt) ? sc : NULL;
169	}
170
171	static void macsec_rxsc_put(struct macsec_rx_sc *sc)
172	{
173	if (refcount_dec_and_test(r: &sc->refcnt))
174	call_rcu(head: &sc->rcu_head, func: free_rx_sc_rcu);
175	}
176
177	static void free_rxsa(struct rcu_head *head)
178	{
179	struct macsec_rx_sa *sa = container_of(head, struct macsec_rx_sa, rcu);
180
181	crypto_free_aead(tfm: sa->key.tfm);
182	free_percpu(pdata: sa->stats);
183	kfree(objp: sa);
184	}
185
186	static void macsec_rxsa_put(struct macsec_rx_sa *sa)
187	{
188	if (refcount_dec_and_test(r: &sa->refcnt))
189	call_rcu(head: &sa->rcu, func: free_rxsa);
190	}
191
192	static struct macsec_tx_sa *macsec_txsa_get(struct macsec_tx_sa __rcu *ptr)
193	{
194	struct macsec_tx_sa *sa = rcu_dereference_bh(ptr);
195
196	if (!sa || !sa->active)
197	return NULL;
198
199	if (!refcount_inc_not_zero(r: &sa->refcnt))
200	return NULL;
201
202	return sa;
203	}
204
205	static void free_txsa(struct rcu_head *head)
206	{
207	struct macsec_tx_sa *sa = container_of(head, struct macsec_tx_sa, rcu);
208
209	crypto_free_aead(tfm: sa->key.tfm);
210	free_percpu(pdata: sa->stats);
211	kfree(objp: sa);
212	}
213
214	static void macsec_txsa_put(struct macsec_tx_sa *sa)
215	{
216	if (refcount_dec_and_test(r: &sa->refcnt))
217	call_rcu(head: &sa->rcu, func: free_txsa);
218	}
219
220	static struct macsec_cb *macsec_skb_cb(struct sk_buff *skb)
221	{
222	BUILD_BUG_ON(sizeof(struct macsec_cb) > sizeof(skb->cb));
223	return (struct macsec_cb *)skb->cb;
224	}
225
226	#define MACSEC_PORT_SCB (0x0000)
227	#define MACSEC_UNDEF_SCI ((__force sci_t)0xffffffffffffffffULL)
228	#define MACSEC_UNDEF_SSCI ((__force ssci_t)0xffffffff)
229
230	#define MACSEC_GCM_AES_128_SAK_LEN 16
231	#define MACSEC_GCM_AES_256_SAK_LEN 32
232
233	#define DEFAULT_SAK_LEN MACSEC_GCM_AES_128_SAK_LEN
234	#define DEFAULT_XPN false
235	#define DEFAULT_SEND_SCI true
236	#define DEFAULT_ENCRYPT false
237	#define DEFAULT_ENCODING_SA 0
238	#define MACSEC_XPN_MAX_REPLAY_WINDOW (((1 << 30) - 1))
239
240	static sci_t make_sci(const u8 *addr, __be16 port)
241	{
242	sci_t sci;
243
244	memcpy(&sci, addr, ETH_ALEN);
245	memcpy(((char *)&sci) + ETH_ALEN, &port, sizeof(port));
246
247	return sci;
248	}
249
250	static sci_t macsec_frame_sci(struct macsec_eth_header *hdr, bool sci_present)
251	{
252	sci_t sci;
253
254	if (sci_present)
255	memcpy(&sci, hdr->secure_channel_id,
256	sizeof(hdr->secure_channel_id));
257	else
258	sci = make_sci(addr: hdr->eth.h_source, MACSEC_PORT_ES);
259
260	return sci;
261	}
262
263	static unsigned int macsec_sectag_len(bool sci_present)
264	{
265	return MACSEC_TAG_LEN + (sci_present ? MACSEC_SCI_LEN : 0);
266	}
267
268	static unsigned int macsec_hdr_len(bool sci_present)
269	{
270	return macsec_sectag_len(sci_present) + ETH_HLEN;
271	}
272
273	static unsigned int macsec_extra_len(bool sci_present)
274	{
275	return macsec_sectag_len(sci_present) + sizeof(__be16);
276	}
277
278	/* Fill SecTAG according to IEEE 802.1AE-2006 10.5.3 */
279	static void macsec_fill_sectag(struct macsec_eth_header *h,
280	const struct macsec_secy *secy, u32 pn,
281	bool sci_present)
282	{
283	const struct macsec_tx_sc *tx_sc = &secy->tx_sc;
284
285	memset(&h->tci_an, 0, macsec_sectag_len(sci_present));
286	h->eth.h_proto = htons(ETH_P_MACSEC);
287
288	if (sci_present) {
289	h->tci_an |= MACSEC_TCI_SC;
290	memcpy(&h->secure_channel_id, &secy->sci,
291	sizeof(h->secure_channel_id));
292	} else {
293	if (tx_sc->end_station)
294	h->tci_an |= MACSEC_TCI_ES;
295	if (tx_sc->scb)
296	h->tci_an |= MACSEC_TCI_SCB;
297	}
298
299	h->packet_number = htonl(pn);
300
301	/* with GCM, C/E clear for !encrypt, both set for encrypt */
302	if (tx_sc->encrypt)
303	h->tci_an |= MACSEC_TCI_CONFID;
304	else if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN)
305	h->tci_an |= MACSEC_TCI_C;
306
307	h->tci_an |= tx_sc->encoding_sa;
308	}
309
310	static void macsec_set_shortlen(struct macsec_eth_header *h, size_t data_len)
311	{
312	if (data_len < MIN_NON_SHORT_LEN)
313	h->short_length = data_len;
314	}
315
316	/* Checks if a MACsec interface is being offloaded to an hardware engine */
317	static bool macsec_is_offloaded(struct macsec_dev *macsec)
318	{
319	if (macsec->offload == MACSEC_OFFLOAD_MAC ||
320	macsec->offload == MACSEC_OFFLOAD_PHY)
321	return true;
322
323	return false;
324	}
325
326	/* Checks if underlying layers implement MACsec offloading functions. */
327	static bool macsec_check_offload(enum macsec_offload offload,
328	struct macsec_dev *macsec)
329	{
330	if (!macsec || !macsec->real_dev)
331	return false;
332
333	if (offload == MACSEC_OFFLOAD_PHY)
334	return macsec->real_dev->phydev &&
335	macsec->real_dev->phydev->macsec_ops;
336	else if (offload == MACSEC_OFFLOAD_MAC)
337	return macsec->real_dev->features & NETIF_F_HW_MACSEC &&
338	macsec->real_dev->macsec_ops;
339
340	return false;
341	}
342
343	static const struct macsec_ops *__macsec_get_ops(enum macsec_offload offload,
344	struct macsec_dev *macsec,
345	struct macsec_context *ctx)
346	{
347	if (ctx) {
348	memset(ctx, 0, sizeof(*ctx));
349	ctx->offload = offload;
350
351	if (offload == MACSEC_OFFLOAD_PHY)
352	ctx->phydev = macsec->real_dev->phydev;
353	else if (offload == MACSEC_OFFLOAD_MAC)
354	ctx->netdev = macsec->real_dev;
355	}
356
357	if (offload == MACSEC_OFFLOAD_PHY)
358	return macsec->real_dev->phydev->macsec_ops;
359	else
360	return macsec->real_dev->macsec_ops;
361	}
362
363	/* Returns a pointer to the MACsec ops struct if any and updates the MACsec
364	* context device reference if provided.
365	*/
366	static const struct macsec_ops *macsec_get_ops(struct macsec_dev *macsec,
367	struct macsec_context *ctx)
368	{
369	if (!macsec_check_offload(offload: macsec->offload, macsec))
370	return NULL;
371
372	return __macsec_get_ops(offload: macsec->offload, macsec, ctx);
373	}
374
375	/* validate MACsec packet according to IEEE 802.1AE-2018 9.12 */
376	static bool macsec_validate_skb(struct sk_buff *skb, u16 icv_len, bool xpn)
377	{
378	struct macsec_eth_header *h = (struct macsec_eth_header *)skb->data;
379	int len = skb->len - 2 * ETH_ALEN;
380	int extra_len = macsec_extra_len(sci_present: !!(h->tci_an & MACSEC_TCI_SC)) + icv_len;
381
382	/* a) It comprises at least 17 octets */
383	if (skb->len <= 16)
384	return false;
385
386	/* b) MACsec EtherType: already checked */
387
388	/* c) V bit is clear */
389	if (h->tci_an & MACSEC_TCI_VERSION)
390	return false;
391
392	/* d) ES or SCB => !SC */
393	if ((h->tci_an & MACSEC_TCI_ES || h->tci_an & MACSEC_TCI_SCB) &&
394	(h->tci_an & MACSEC_TCI_SC))
395	return false;
396
397	/* e) Bits 7 and 8 of octet 4 of the SecTAG are clear */
398	if (h->unused)
399	return false;
400
401	/* rx.pn != 0 if not XPN (figure 10-5 with 802.11AEbw-2013 amendment) */
402	if (!h->packet_number && !xpn)
403	return false;
404
405	/* length check, f) g) h) i) */
406	if (h->short_length)
407	return len == extra_len + h->short_length;
408	return len >= extra_len + MIN_NON_SHORT_LEN;
409	}
410
411	#define MACSEC_NEEDED_HEADROOM (macsec_extra_len(true))
412	#define MACSEC_NEEDED_TAILROOM MACSEC_STD_ICV_LEN
413
414	static void macsec_fill_iv_xpn(unsigned char *iv, ssci_t ssci, u64 pn,
415	salt_t salt)
416	{
417	struct gcm_iv_xpn *gcm_iv = (struct gcm_iv_xpn *)iv;
418
419	gcm_iv->ssci = ssci ^ salt.ssci;
420	gcm_iv->pn = cpu_to_be64(pn) ^ salt.pn;
421	}
422
423	static void macsec_fill_iv(unsigned char *iv, sci_t sci, u32 pn)
424	{
425	struct gcm_iv *gcm_iv = (struct gcm_iv *)iv;
426
427	gcm_iv->sci = sci;
428	gcm_iv->pn = htonl(pn);
429	}
430
431	static struct macsec_eth_header *macsec_ethhdr(struct sk_buff *skb)
432	{
433	return (struct macsec_eth_header *)skb_mac_header(skb);
434	}
435
436	static void __macsec_pn_wrapped(struct macsec_secy *secy,
437	struct macsec_tx_sa *tx_sa)
438	{
439	pr_debug("PN wrapped, transitioning to !oper\n");
440	tx_sa->active = false;
441	if (secy->protect_frames)
442	secy->operational = false;
443	}
444
445	void macsec_pn_wrapped(struct macsec_secy *secy, struct macsec_tx_sa *tx_sa)
446	{
447	spin_lock_bh(lock: &tx_sa->lock);
448	__macsec_pn_wrapped(secy, tx_sa);
449	spin_unlock_bh(lock: &tx_sa->lock);
450	}
451	EXPORT_SYMBOL_GPL(macsec_pn_wrapped);
452
453	static pn_t tx_sa_update_pn(struct macsec_tx_sa *tx_sa,
454	struct macsec_secy *secy)
455	{
456	pn_t pn;
457
458	spin_lock_bh(lock: &tx_sa->lock);
459
460	pn = tx_sa->next_pn_halves;
461	if (secy->xpn)
462	tx_sa->next_pn++;
463	else
464	tx_sa->next_pn_halves.lower++;
465
466	if (tx_sa->next_pn == 0)
467	__macsec_pn_wrapped(secy, tx_sa);
468	spin_unlock_bh(lock: &tx_sa->lock);
469
470	return pn;
471	}
472
473	static void macsec_encrypt_finish(struct sk_buff *skb, struct net_device *dev)
474	{
475	struct macsec_dev *macsec = netdev_priv(dev);
476
477	skb->dev = macsec->real_dev;
478	skb_reset_mac_header(skb);
479	skb->protocol = eth_hdr(skb)->h_proto;
480	}
481
482	static unsigned int macsec_msdu_len(struct sk_buff *skb)
483	{
484	struct macsec_dev *macsec = macsec_priv(dev: skb->dev);
485	struct macsec_secy *secy = &macsec->secy;
486	bool sci_present = macsec_skb_cb(skb)->has_sci;
487
488	return skb->len - macsec_hdr_len(sci_present) - secy->icv_len;
489	}
490
491	static void macsec_count_tx(struct sk_buff *skb, struct macsec_tx_sc *tx_sc,
492	struct macsec_tx_sa *tx_sa)
493	{
494	unsigned int msdu_len = macsec_msdu_len(skb);
495	struct pcpu_tx_sc_stats *txsc_stats = this_cpu_ptr(tx_sc->stats);
496
497	u64_stats_update_begin(syncp: &txsc_stats->syncp);
498	if (tx_sc->encrypt) {
499	txsc_stats->stats.OutOctetsEncrypted += msdu_len;
500	txsc_stats->stats.OutPktsEncrypted++;
501	this_cpu_inc(tx_sa->stats->OutPktsEncrypted);
502	} else {
503	txsc_stats->stats.OutOctetsProtected += msdu_len;
504	txsc_stats->stats.OutPktsProtected++;
505	this_cpu_inc(tx_sa->stats->OutPktsProtected);
506	}
507	u64_stats_update_end(syncp: &txsc_stats->syncp);
508	}
509
510	static void count_tx(struct net_device *dev, int ret, int len)
511	{
512	if (likely(ret == NET_XMIT_SUCCESS || ret == NET_XMIT_CN))
513	dev_sw_netstats_tx_add(dev, packets: 1, len);
514	}
515
516	static void macsec_encrypt_done(void *data, int err)
517	{
518	struct sk_buff *skb = data;
519	struct net_device *dev = skb->dev;
520	struct macsec_dev *macsec = macsec_priv(dev);
521	struct macsec_tx_sa *sa = macsec_skb_cb(skb)->tx_sa;
522	int len, ret;
523
524	aead_request_free(req: macsec_skb_cb(skb)->req);
525
526	rcu_read_lock_bh();
527	macsec_count_tx(skb, tx_sc: &macsec->secy.tx_sc, tx_sa: macsec_skb_cb(skb)->tx_sa);
528	/* packet is encrypted/protected so tx_bytes must be calculated */
529	len = macsec_msdu_len(skb) + 2 * ETH_ALEN;
530	macsec_encrypt_finish(skb, dev);
531	ret = dev_queue_xmit(skb);
532	count_tx(dev, ret, len);
533	rcu_read_unlock_bh();
534
535	macsec_txsa_put(sa);
536	dev_put(dev);
537	}
538
539	static struct aead_request *macsec_alloc_req(struct crypto_aead *tfm,
540	unsigned char **iv,
541	struct scatterlist **sg,
542	int num_frags)
543	{
544	size_t size, iv_offset, sg_offset;
545	struct aead_request *req;
546	void *tmp;
547
548	size = sizeof(struct aead_request) + crypto_aead_reqsize(tfm);
549	iv_offset = size;
550	size += GCM_AES_IV_LEN;
551
552	size = ALIGN(size, __alignof__(struct scatterlist));
553	sg_offset = size;
554	size += sizeof(struct scatterlist) * num_frags;
555
556	tmp = kmalloc(size, GFP_ATOMIC);
557	if (!tmp)
558	return NULL;
559
560	*iv = (unsigned char *)(tmp + iv_offset);
561	*sg = (struct scatterlist *)(tmp + sg_offset);
562	req = tmp;
563
564	aead_request_set_tfm(req, tfm);
565
566	return req;
567	}
568
569	static struct sk_buff *macsec_encrypt(struct sk_buff *skb,
570	struct net_device *dev)
571	{
572	int ret;
573	struct scatterlist *sg;
574	struct sk_buff *trailer;
575	unsigned char *iv;
576	struct ethhdr *eth;
577	struct macsec_eth_header *hh;
578	size_t unprotected_len;
579	struct aead_request *req;
580	struct macsec_secy *secy;
581	struct macsec_tx_sc *tx_sc;
582	struct macsec_tx_sa *tx_sa;
583	struct macsec_dev *macsec = macsec_priv(dev);
584	bool sci_present;
585	pn_t pn;
586
587	secy = &macsec->secy;
588	tx_sc = &secy->tx_sc;
589
590	/* 10.5.1 TX SA assignment */
591	tx_sa = macsec_txsa_get(ptr: tx_sc->sa[tx_sc->encoding_sa]);
592	if (!tx_sa) {
593	secy->operational = false;
594	kfree_skb(skb);
595	return ERR_PTR(error: -EINVAL);
596	}
597
598	if (unlikely(skb_headroom(skb) < MACSEC_NEEDED_HEADROOM ||
599	skb_tailroom(skb) < MACSEC_NEEDED_TAILROOM)) {
600	struct sk_buff *nskb = skb_copy_expand(skb,
601	MACSEC_NEEDED_HEADROOM,
602	MACSEC_NEEDED_TAILROOM,
603	GFP_ATOMIC);
604	if (likely(nskb)) {
605	consume_skb(skb);
606	skb = nskb;
607	} else {
608	macsec_txsa_put(sa: tx_sa);
609	kfree_skb(skb);
610	return ERR_PTR(error: -ENOMEM);
611	}
612	} else {
613	skb = skb_unshare(skb, GFP_ATOMIC);
614	if (!skb) {
615	macsec_txsa_put(sa: tx_sa);
616	return ERR_PTR(error: -ENOMEM);
617	}
618	}
619
620	unprotected_len = skb->len;
621	eth = eth_hdr(skb);
622	sci_present = macsec_send_sci(secy);
623	hh = skb_push(skb, len: macsec_extra_len(sci_present));
624	memmove(hh, eth, 2 * ETH_ALEN);
625
626	pn = tx_sa_update_pn(tx_sa, secy);
627	if (pn.full64 == 0) {
628	macsec_txsa_put(sa: tx_sa);
629	kfree_skb(skb);
630	return ERR_PTR(error: -ENOLINK);
631	}
632	macsec_fill_sectag(h: hh, secy, pn: pn.lower, sci_present);
633	macsec_set_shortlen(h: hh, data_len: unprotected_len - 2 * ETH_ALEN);
634
635	skb_put(skb, len: secy->icv_len);
636
637	if (skb->len - ETH_HLEN > macsec_priv(dev)->real_dev->mtu) {
638	struct pcpu_secy_stats *secy_stats = this_cpu_ptr(macsec->stats);
639
640	u64_stats_update_begin(syncp: &secy_stats->syncp);
641	secy_stats->stats.OutPktsTooLong++;
642	u64_stats_update_end(syncp: &secy_stats->syncp);
643
644	macsec_txsa_put(sa: tx_sa);
645	kfree_skb(skb);
646	return ERR_PTR(error: -EINVAL);
647	}
648
649	ret = skb_cow_data(skb, tailbits: 0, trailer: &trailer);
650	if (unlikely(ret < 0)) {
651	macsec_txsa_put(sa: tx_sa);
652	kfree_skb(skb);
653	return ERR_PTR(error: ret);
654	}
655
656	req = macsec_alloc_req(tfm: tx_sa->key.tfm, iv: &iv, sg: &sg, num_frags: ret);
657	if (!req) {
658	macsec_txsa_put(sa: tx_sa);
659	kfree_skb(skb);
660	return ERR_PTR(error: -ENOMEM);
661	}
662
663	if (secy->xpn)
664	macsec_fill_iv_xpn(iv, ssci: tx_sa->ssci, pn: pn.full64, salt: tx_sa->key.salt);
665	else
666	macsec_fill_iv(iv, sci: secy->sci, pn: pn.lower);
667
668	sg_init_table(sg, ret);
669	ret = skb_to_sgvec(skb, sg, offset: 0, len: skb->len);
670	if (unlikely(ret < 0)) {
671	aead_request_free(req);
672	macsec_txsa_put(sa: tx_sa);
673	kfree_skb(skb);
674	return ERR_PTR(error: ret);
675	}
676
677	if (tx_sc->encrypt) {
678	int len = skb->len - macsec_hdr_len(sci_present) -
679	secy->icv_len;
680	aead_request_set_crypt(req, src: sg, dst: sg, cryptlen: len, iv);
681	aead_request_set_ad(req, assoclen: macsec_hdr_len(sci_present));
682	} else {
683	aead_request_set_crypt(req, src: sg, dst: sg, cryptlen: 0, iv);
684	aead_request_set_ad(req, assoclen: skb->len - secy->icv_len);
685	}
686
687	macsec_skb_cb(skb)->req = req;
688	macsec_skb_cb(skb)->tx_sa = tx_sa;
689	macsec_skb_cb(skb)->has_sci = sci_present;
690	aead_request_set_callback(req, flags: 0, compl: macsec_encrypt_done, data: skb);
691
692	dev_hold(dev: skb->dev);
693	ret = crypto_aead_encrypt(req);
694	if (ret == -EINPROGRESS) {
695	return ERR_PTR(error: ret);
696	} else if (ret != 0) {
697	dev_put(dev: skb->dev);
698	kfree_skb(skb);
699	aead_request_free(req);
700	macsec_txsa_put(sa: tx_sa);
701	return ERR_PTR(error: -EINVAL);
702	}
703
704	dev_put(dev: skb->dev);
705	aead_request_free(req);
706	macsec_txsa_put(sa: tx_sa);
707
708	return skb;
709	}
710
711	static bool macsec_post_decrypt(struct sk_buff *skb, struct macsec_secy *secy, u32 pn)
712	{
713	struct macsec_rx_sa *rx_sa = macsec_skb_cb(skb)->rx_sa;
714	struct pcpu_rx_sc_stats *rxsc_stats = this_cpu_ptr(rx_sa->sc->stats);
715	struct macsec_eth_header *hdr = macsec_ethhdr(skb);
716	u32 lowest_pn = 0;
717
718	spin_lock(lock: &rx_sa->lock);
719	if (rx_sa->next_pn_halves.lower >= secy->replay_window)
720	lowest_pn = rx_sa->next_pn_halves.lower - secy->replay_window;
721
722	/* Now perform replay protection check again
723	* (see IEEE 802.1AE-2006 figure 10-5)
724	*/
725	if (secy->replay_protect && pn < lowest_pn &&
726	(!secy->xpn || pn_same_half(pn, lowest_pn))) {
727	spin_unlock(lock: &rx_sa->lock);
728	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
729	rxsc_stats->stats.InPktsLate++;
730	u64_stats_update_end(syncp: &rxsc_stats->syncp);
731	DEV_STATS_INC(secy->netdev, rx_dropped);
732	return false;
733	}
734
735	if (secy->validate_frames != MACSEC_VALIDATE_DISABLED) {
736	unsigned int msdu_len = macsec_msdu_len(skb);
737	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
738	if (hdr->tci_an & MACSEC_TCI_E)
739	rxsc_stats->stats.InOctetsDecrypted += msdu_len;
740	else
741	rxsc_stats->stats.InOctetsValidated += msdu_len;
742	u64_stats_update_end(syncp: &rxsc_stats->syncp);
743	}
744
745	if (!macsec_skb_cb(skb)->valid) {
746	spin_unlock(lock: &rx_sa->lock);
747
748	/* 10.6.5 */
749	if (hdr->tci_an & MACSEC_TCI_C ||
750	secy->validate_frames == MACSEC_VALIDATE_STRICT) {
751	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
752	rxsc_stats->stats.InPktsNotValid++;
753	u64_stats_update_end(syncp: &rxsc_stats->syncp);
754	this_cpu_inc(rx_sa->stats->InPktsNotValid);
755	DEV_STATS_INC(secy->netdev, rx_errors);
756	return false;
757	}
758
759	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
760	if (secy->validate_frames == MACSEC_VALIDATE_CHECK) {
761	rxsc_stats->stats.InPktsInvalid++;
762	this_cpu_inc(rx_sa->stats->InPktsInvalid);
763	} else if (pn < lowest_pn) {
764	rxsc_stats->stats.InPktsDelayed++;
765	} else {
766	rxsc_stats->stats.InPktsUnchecked++;
767	}
768	u64_stats_update_end(syncp: &rxsc_stats->syncp);
769	} else {
770	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
771	if (pn < lowest_pn) {
772	rxsc_stats->stats.InPktsDelayed++;
773	} else {
774	rxsc_stats->stats.InPktsOK++;
775	this_cpu_inc(rx_sa->stats->InPktsOK);
776	}
777	u64_stats_update_end(syncp: &rxsc_stats->syncp);
778
779	// Instead of "pn >=" - to support pn overflow in xpn
780	if (pn + 1 > rx_sa->next_pn_halves.lower) {
781	rx_sa->next_pn_halves.lower = pn + 1;
782	} else if (secy->xpn &&
783	!pn_same_half(pn, rx_sa->next_pn_halves.lower)) {
784	rx_sa->next_pn_halves.upper++;
785	rx_sa->next_pn_halves.lower = pn + 1;
786	}
787
788	spin_unlock(lock: &rx_sa->lock);
789	}
790
791	return true;
792	}
793
794	static void macsec_reset_skb(struct sk_buff *skb, struct net_device *dev)
795	{
796	skb->pkt_type = PACKET_HOST;
797	skb->protocol = eth_type_trans(skb, dev);
798
799	skb_reset_network_header(skb);
800	if (!skb_transport_header_was_set(skb))
801	skb_reset_transport_header(skb);
802	skb_reset_mac_len(skb);
803	}
804
805	static void macsec_finalize_skb(struct sk_buff *skb, u8 icv_len, u8 hdr_len)
806	{
807	skb->ip_summed = CHECKSUM_NONE;
808	memmove(skb->data + hdr_len, skb->data, 2 * ETH_ALEN);
809	skb_pull(skb, len: hdr_len);
810	pskb_trim_unique(skb, len: skb->len - icv_len);
811	}
812
813	static void count_rx(struct net_device *dev, int len)
814	{
815	dev_sw_netstats_rx_add(dev, len);
816	}
817
818	static void macsec_decrypt_done(void *data, int err)
819	{
820	struct sk_buff *skb = data;
821	struct net_device *dev = skb->dev;
822	struct macsec_dev *macsec = macsec_priv(dev);
823	struct macsec_rx_sa *rx_sa = macsec_skb_cb(skb)->rx_sa;
824	struct macsec_rx_sc *rx_sc = rx_sa->sc;
825	int len;
826	u32 pn;
827
828	aead_request_free(req: macsec_skb_cb(skb)->req);
829
830	if (!err)
831	macsec_skb_cb(skb)->valid = true;
832
833	rcu_read_lock_bh();
834	pn = ntohl(macsec_ethhdr(skb)->packet_number);
835	if (!macsec_post_decrypt(skb, secy: &macsec->secy, pn)) {
836	rcu_read_unlock_bh();
837	kfree_skb(skb);
838	goto out;
839	}
840
841	macsec_finalize_skb(skb, icv_len: macsec->secy.icv_len,
842	hdr_len: macsec_extra_len(sci_present: macsec_skb_cb(skb)->has_sci));
843	len = skb->len;
844	macsec_reset_skb(skb, dev: macsec->secy.netdev);
845
846	if (gro_cells_receive(gcells: &macsec->gro_cells, skb) == NET_RX_SUCCESS)
847	count_rx(dev, len);
848
849	rcu_read_unlock_bh();
850
851	out:
852	macsec_rxsa_put(sa: rx_sa);
853	macsec_rxsc_put(sc: rx_sc);
854	dev_put(dev);
855	}
856
857	static struct sk_buff *macsec_decrypt(struct sk_buff *skb,
858	struct net_device *dev,
859	struct macsec_rx_sa *rx_sa,
860	sci_t sci,
861	struct macsec_secy *secy)
862	{
863	int ret;
864	struct scatterlist *sg;
865	struct sk_buff *trailer;
866	unsigned char *iv;
867	struct aead_request *req;
868	struct macsec_eth_header *hdr;
869	u32 hdr_pn;
870	u16 icv_len = secy->icv_len;
871
872	macsec_skb_cb(skb)->valid = false;
873	skb = skb_share_check(skb, GFP_ATOMIC);
874	if (!skb)
875	return ERR_PTR(error: -ENOMEM);
876
877	ret = skb_cow_data(skb, tailbits: 0, trailer: &trailer);
878	if (unlikely(ret < 0)) {
879	kfree_skb(skb);
880	return ERR_PTR(error: ret);
881	}
882	req = macsec_alloc_req(tfm: rx_sa->key.tfm, iv: &iv, sg: &sg, num_frags: ret);
883	if (!req) {
884	kfree_skb(skb);
885	return ERR_PTR(error: -ENOMEM);
886	}
887
888	hdr = (struct macsec_eth_header *)skb->data;
889	hdr_pn = ntohl(hdr->packet_number);
890
891	if (secy->xpn) {
892	pn_t recovered_pn = rx_sa->next_pn_halves;
893
894	recovered_pn.lower = hdr_pn;
895	if (hdr_pn < rx_sa->next_pn_halves.lower &&
896	!pn_same_half(hdr_pn, rx_sa->next_pn_halves.lower))
897	recovered_pn.upper++;
898
899	macsec_fill_iv_xpn(iv, ssci: rx_sa->ssci, pn: recovered_pn.full64,
900	salt: rx_sa->key.salt);
901	} else {
902	macsec_fill_iv(iv, sci, pn: hdr_pn);
903	}
904
905	sg_init_table(sg, ret);
906	ret = skb_to_sgvec(skb, sg, offset: 0, len: skb->len);
907	if (unlikely(ret < 0)) {
908	aead_request_free(req);
909	kfree_skb(skb);
910	return ERR_PTR(error: ret);
911	}
912
913	if (hdr->tci_an & MACSEC_TCI_E) {
914	/* confidentiality: ethernet + macsec header
915	* authenticated, encrypted payload
916	*/
917	int len = skb->len - macsec_hdr_len(sci_present: macsec_skb_cb(skb)->has_sci);
918
919	aead_request_set_crypt(req, src: sg, dst: sg, cryptlen: len, iv);
920	aead_request_set_ad(req, assoclen: macsec_hdr_len(sci_present: macsec_skb_cb(skb)->has_sci));
921	skb = skb_unshare(skb, GFP_ATOMIC);
922	if (!skb) {
923	aead_request_free(req);
924	return ERR_PTR(error: -ENOMEM);
925	}
926	} else {
927	/* integrity only: all headers + data authenticated */
928	aead_request_set_crypt(req, src: sg, dst: sg, cryptlen: icv_len, iv);
929	aead_request_set_ad(req, assoclen: skb->len - icv_len);
930	}
931
932	macsec_skb_cb(skb)->req = req;
933	skb->dev = dev;
934	aead_request_set_callback(req, flags: 0, compl: macsec_decrypt_done, data: skb);
935
936	dev_hold(dev);
937	ret = crypto_aead_decrypt(req);
938	if (ret == -EINPROGRESS) {
939	return ERR_PTR(error: ret);
940	} else if (ret != 0) {
941	/* decryption/authentication failed
942	* 10.6 if validateFrames is disabled, deliver anyway
943	*/
944	if (ret != -EBADMSG) {
945	kfree_skb(skb);
946	skb = ERR_PTR(error: ret);
947	}
948	} else {
949	macsec_skb_cb(skb)->valid = true;
950	}
951	dev_put(dev);
952
953	aead_request_free(req);
954
955	return skb;
956	}
957
958	static struct macsec_rx_sc *find_rx_sc(struct macsec_secy *secy, sci_t sci)
959	{
960	struct macsec_rx_sc *rx_sc;
961
962	for_each_rxsc(secy, rx_sc) {
963	if (rx_sc->sci == sci)
964	return rx_sc;
965	}
966
967	return NULL;
968	}
969
970	static struct macsec_rx_sc *find_rx_sc_rtnl(struct macsec_secy *secy, sci_t sci)
971	{
972	struct macsec_rx_sc *rx_sc;
973
974	for_each_rxsc_rtnl(secy, rx_sc) {
975	if (rx_sc->sci == sci)
976	return rx_sc;
977	}
978
979	return NULL;
980	}
981
982	static enum rx_handler_result handle_not_macsec(struct sk_buff *skb)
983	{
984	/* Deliver to the uncontrolled port by default */
985	enum rx_handler_result ret = RX_HANDLER_PASS;
986	struct ethhdr *hdr = eth_hdr(skb);
987	struct metadata_dst *md_dst;
988	struct macsec_rxh_data *rxd;
989	struct macsec_dev *macsec;
990	bool is_macsec_md_dst;
991
992	rcu_read_lock();
993	rxd = macsec_data_rcu(dev: skb->dev);
994	md_dst = skb_metadata_dst(skb);
995	is_macsec_md_dst = md_dst && md_dst->type == METADATA_MACSEC;
996
997	list_for_each_entry_rcu(macsec, &rxd->secys, secys) {
998	struct sk_buff *nskb;
999	struct pcpu_secy_stats *secy_stats = this_cpu_ptr(macsec->stats);
1000	struct net_device *ndev = macsec->secy.netdev;
1001
1002	/* If h/w offloading is enabled, HW decodes frames and strips
1003	* the SecTAG, so we have to deduce which port to deliver to.
1004	*/
1005	if (macsec_is_offloaded(macsec) && netif_running(dev: ndev)) {
1006	const struct macsec_ops *ops;
1007
1008	ops = macsec_get_ops(macsec, NULL);
1009
1010	if (ops->rx_uses_md_dst && !is_macsec_md_dst)
1011	continue;
1012
1013	if (is_macsec_md_dst) {
1014	struct macsec_rx_sc *rx_sc;
1015
1016	/* All drivers that implement MACsec offload
1017	* support using skb metadata destinations must
1018	* indicate that they do so.
1019	*/
1020	DEBUG_NET_WARN_ON_ONCE(!ops->rx_uses_md_dst);
1021	rx_sc = find_rx_sc(secy: &macsec->secy,
1022	sci: md_dst->u.macsec_info.sci);
1023	if (!rx_sc)
1024	continue;
1025	/* device indicated macsec offload occurred */
1026	skb->dev = ndev;
1027	skb->pkt_type = PACKET_HOST;
1028	eth_skb_pkt_type(skb, dev: ndev);
1029	ret = RX_HANDLER_ANOTHER;
1030	goto out;
1031	}
1032
1033	/* This datapath is insecure because it is unable to
1034	* enforce isolation of broadcast/multicast traffic and
1035	* unicast traffic with promiscuous mode on the macsec
1036	* netdev. Since the core stack has no mechanism to
1037	* check that the hardware did indeed receive MACsec
1038	* traffic, it is possible that the response handling
1039	* done by the MACsec port was to a plaintext packet.
1040	* This violates the MACsec protocol standard.
1041	*/
1042	if (ether_addr_equal_64bits(addr1: hdr->h_dest,
1043	addr2: ndev->dev_addr)) {
1044	/* exact match, divert skb to this port */
1045	skb->dev = ndev;
1046	skb->pkt_type = PACKET_HOST;
1047	ret = RX_HANDLER_ANOTHER;
1048	goto out;
1049	} else if (is_multicast_ether_addr_64bits(
1050	addr: hdr->h_dest)) {
1051	/* multicast frame, deliver on this port too */
1052	nskb = skb_clone(skb, GFP_ATOMIC);
1053	if (!nskb)
1054	break;
1055
1056	nskb->dev = ndev;
1057	eth_skb_pkt_type(skb: nskb, dev: ndev);
1058
1059	__netif_rx(skb: nskb);
1060	} else if (ndev->flags & IFF_PROMISC) {
1061	skb->dev = ndev;
1062	skb->pkt_type = PACKET_HOST;
1063	ret = RX_HANDLER_ANOTHER;
1064	goto out;
1065	}
1066
1067	continue;
1068	}
1069
1070	/* 10.6 If the management control validateFrames is not
1071	* Strict, frames without a SecTAG are received, counted, and
1072	* delivered to the Controlled Port
1073	*/
1074	if (macsec->secy.validate_frames == MACSEC_VALIDATE_STRICT) {
1075	u64_stats_update_begin(syncp: &secy_stats->syncp);
1076	secy_stats->stats.InPktsNoTag++;
1077	u64_stats_update_end(syncp: &secy_stats->syncp);
1078	DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1079	continue;
1080	}
1081
1082	/* deliver on this port */
1083	nskb = skb_clone(skb, GFP_ATOMIC);
1084	if (!nskb)
1085	break;
1086
1087	nskb->dev = ndev;
1088
1089	if (__netif_rx(skb: nskb) == NET_RX_SUCCESS) {
1090	u64_stats_update_begin(syncp: &secy_stats->syncp);
1091	secy_stats->stats.InPktsUntagged++;
1092	u64_stats_update_end(syncp: &secy_stats->syncp);
1093	}
1094	}
1095
1096	out:
1097	rcu_read_unlock();
1098	return ret;
1099	}
1100
1101	static rx_handler_result_t macsec_handle_frame(struct sk_buff **pskb)
1102	{
1103	struct sk_buff *skb = *pskb;
1104	struct net_device *dev = skb->dev;
1105	struct macsec_eth_header *hdr;
1106	struct macsec_secy *secy = NULL;
1107	struct macsec_rx_sc *rx_sc;
1108	struct macsec_rx_sa *rx_sa;
1109	struct macsec_rxh_data *rxd;
1110	struct macsec_dev *macsec;
1111	unsigned int len;
1112	sci_t sci;
1113	u32 hdr_pn;
1114	bool cbit;
1115	struct pcpu_rx_sc_stats *rxsc_stats;
1116	struct pcpu_secy_stats *secy_stats;
1117	bool pulled_sci;
1118	int ret;
1119
1120	if (skb_headroom(skb) < ETH_HLEN)
1121	goto drop_direct;
1122
1123	hdr = macsec_ethhdr(skb);
1124	if (hdr->eth.h_proto != htons(ETH_P_MACSEC))
1125	return handle_not_macsec(skb);
1126
1127	skb = skb_unshare(skb, GFP_ATOMIC);
1128	*pskb = skb;
1129	if (!skb)
1130	return RX_HANDLER_CONSUMED;
1131
1132	pulled_sci = pskb_may_pull(skb, len: macsec_extra_len(sci_present: true));
1133	if (!pulled_sci) {
1134	if (!pskb_may_pull(skb, len: macsec_extra_len(sci_present: false)))
1135	goto drop_direct;
1136	}
1137
1138	hdr = macsec_ethhdr(skb);
1139
1140	/* Frames with a SecTAG that has the TCI E bit set but the C
1141	* bit clear are discarded, as this reserved encoding is used
1142	* to identify frames with a SecTAG that are not to be
1143	* delivered to the Controlled Port.
1144	*/
1145	if ((hdr->tci_an & (MACSEC_TCI_C | MACSEC_TCI_E)) == MACSEC_TCI_E)
1146	return RX_HANDLER_PASS;
1147
1148	/* now, pull the extra length */
1149	if (hdr->tci_an & MACSEC_TCI_SC) {
1150	if (!pulled_sci)
1151	goto drop_direct;
1152	}
1153
1154	/* ethernet header is part of crypto processing */
1155	skb_push(skb, ETH_HLEN);
1156
1157	macsec_skb_cb(skb)->has_sci = !!(hdr->tci_an & MACSEC_TCI_SC);
1158	macsec_skb_cb(skb)->assoc_num = hdr->tci_an & MACSEC_AN_MASK;
1159	sci = macsec_frame_sci(hdr, sci_present: macsec_skb_cb(skb)->has_sci);
1160
1161	rcu_read_lock();
1162	rxd = macsec_data_rcu(dev: skb->dev);
1163
1164	list_for_each_entry_rcu(macsec, &rxd->secys, secys) {
1165	struct macsec_rx_sc *sc = find_rx_sc(secy: &macsec->secy, sci);
1166
1167	sc = sc ? macsec_rxsc_get(sc) : NULL;
1168
1169	if (sc) {
1170	secy = &macsec->secy;
1171	rx_sc = sc;
1172	break;
1173	}
1174	}
1175
1176	if (!secy)
1177	goto nosci;
1178
1179	dev = secy->netdev;
1180	macsec = macsec_priv(dev);
1181	secy_stats = this_cpu_ptr(macsec->stats);
1182	rxsc_stats = this_cpu_ptr(rx_sc->stats);
1183
1184	if (!macsec_validate_skb(skb, icv_len: secy->icv_len, xpn: secy->xpn)) {
1185	u64_stats_update_begin(syncp: &secy_stats->syncp);
1186	secy_stats->stats.InPktsBadTag++;
1187	u64_stats_update_end(syncp: &secy_stats->syncp);
1188	DEV_STATS_INC(secy->netdev, rx_errors);
1189	goto drop_nosa;
1190	}
1191
1192	rx_sa = macsec_rxsa_get(ptr: rx_sc->sa[macsec_skb_cb(skb)->assoc_num]);
1193	if (!rx_sa) {
1194	/* 10.6.1 if the SA is not in use */
1195
1196	/* If validateFrames is Strict or the C bit in the
1197	* SecTAG is set, discard
1198	*/
1199	if (hdr->tci_an & MACSEC_TCI_C ||
1200	secy->validate_frames == MACSEC_VALIDATE_STRICT) {
1201	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
1202	rxsc_stats->stats.InPktsNotUsingSA++;
1203	u64_stats_update_end(syncp: &rxsc_stats->syncp);
1204	DEV_STATS_INC(secy->netdev, rx_errors);
1205	goto drop_nosa;
1206	}
1207
1208	/* not Strict, the frame (with the SecTAG and ICV
1209	* removed) is delivered to the Controlled Port.
1210	*/
1211	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
1212	rxsc_stats->stats.InPktsUnusedSA++;
1213	u64_stats_update_end(syncp: &rxsc_stats->syncp);
1214	goto deliver;
1215	}
1216
1217	/* First, PN check to avoid decrypting obviously wrong packets */
1218	hdr_pn = ntohl(hdr->packet_number);
1219	if (secy->replay_protect) {
1220	bool late;
1221
1222	spin_lock(lock: &rx_sa->lock);
1223	late = rx_sa->next_pn_halves.lower >= secy->replay_window &&
1224	hdr_pn < (rx_sa->next_pn_halves.lower - secy->replay_window);
1225
1226	if (secy->xpn)
1227	late = late && pn_same_half(rx_sa->next_pn_halves.lower, hdr_pn);
1228	spin_unlock(lock: &rx_sa->lock);
1229
1230	if (late) {
1231	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
1232	rxsc_stats->stats.InPktsLate++;
1233	u64_stats_update_end(syncp: &rxsc_stats->syncp);
1234	DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1235	goto drop;
1236	}
1237	}
1238
1239	macsec_skb_cb(skb)->rx_sa = rx_sa;
1240
1241	/* Disabled && !changed text => skip validation */
1242	if (hdr->tci_an & MACSEC_TCI_C ||
1243	secy->validate_frames != MACSEC_VALIDATE_DISABLED)
1244	skb = macsec_decrypt(skb, dev, rx_sa, sci, secy);
1245
1246	if (IS_ERR(ptr: skb)) {
1247	/* the decrypt callback needs the reference */
1248	if (PTR_ERR(ptr: skb) != -EINPROGRESS) {
1249	macsec_rxsa_put(sa: rx_sa);
1250	macsec_rxsc_put(sc: rx_sc);
1251	}
1252	rcu_read_unlock();
1253	*pskb = NULL;
1254	return RX_HANDLER_CONSUMED;
1255	}
1256
1257	if (!macsec_post_decrypt(skb, secy, pn: hdr_pn))
1258	goto drop;
1259
1260	deliver:
1261	macsec_finalize_skb(skb, icv_len: secy->icv_len,
1262	hdr_len: macsec_extra_len(sci_present: macsec_skb_cb(skb)->has_sci));
1263	len = skb->len;
1264	macsec_reset_skb(skb, dev: secy->netdev);
1265
1266	if (rx_sa)
1267	macsec_rxsa_put(sa: rx_sa);
1268	macsec_rxsc_put(sc: rx_sc);
1269
1270	skb_orphan(skb);
1271	ret = gro_cells_receive(gcells: &macsec->gro_cells, skb);
1272	if (ret == NET_RX_SUCCESS)
1273	count_rx(dev, len);
1274	else
1275	DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1276
1277	rcu_read_unlock();
1278
1279	*pskb = NULL;
1280	return RX_HANDLER_CONSUMED;
1281
1282	drop:
1283	macsec_rxsa_put(sa: rx_sa);
1284	drop_nosa:
1285	macsec_rxsc_put(sc: rx_sc);
1286	rcu_read_unlock();
1287	drop_direct:
1288	kfree_skb(skb);
1289	*pskb = NULL;
1290	return RX_HANDLER_CONSUMED;
1291
1292	nosci:
1293	/* 10.6.1 if the SC is not found */
1294	cbit = !!(hdr->tci_an & MACSEC_TCI_C);
1295	if (!cbit)
1296	macsec_finalize_skb(skb, MACSEC_DEFAULT_ICV_LEN,
1297	hdr_len: macsec_extra_len(sci_present: macsec_skb_cb(skb)->has_sci));
1298
1299	list_for_each_entry_rcu(macsec, &rxd->secys, secys) {
1300	struct sk_buff *nskb;
1301
1302	secy_stats = this_cpu_ptr(macsec->stats);
1303
1304	/* If validateFrames is Strict or the C bit in the
1305	* SecTAG is set, discard
1306	*/
1307	if (cbit ||
1308	macsec->secy.validate_frames == MACSEC_VALIDATE_STRICT) {
1309	u64_stats_update_begin(syncp: &secy_stats->syncp);
1310	secy_stats->stats.InPktsNoSCI++;
1311	u64_stats_update_end(syncp: &secy_stats->syncp);
1312	DEV_STATS_INC(macsec->secy.netdev, rx_errors);
1313	continue;
1314	}
1315
1316	/* not strict, the frame (with the SecTAG and ICV
1317	* removed) is delivered to the Controlled Port.
1318	*/
1319	nskb = skb_clone(skb, GFP_ATOMIC);
1320	if (!nskb)
1321	break;
1322
1323	macsec_reset_skb(skb: nskb, dev: macsec->secy.netdev);
1324
1325	ret = __netif_rx(skb: nskb);
1326	if (ret == NET_RX_SUCCESS) {
1327	u64_stats_update_begin(syncp: &secy_stats->syncp);
1328	secy_stats->stats.InPktsUnknownSCI++;
1329	u64_stats_update_end(syncp: &secy_stats->syncp);
1330	} else {
1331	DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1332	}
1333	}
1334
1335	rcu_read_unlock();
1336	*pskb = skb;
1337	return RX_HANDLER_PASS;
1338	}
1339
1340	static struct crypto_aead *macsec_alloc_tfm(char *key, int key_len, int icv_len)
1341	{
1342	struct crypto_aead *tfm;
1343	int ret;
1344
1345	tfm = crypto_alloc_aead(alg_name: "gcm(aes)", type: 0, mask: 0);
1346
1347	if (IS_ERR(ptr: tfm))
1348	return tfm;
1349
1350	ret = crypto_aead_setkey(tfm, key, keylen: key_len);
1351	if (ret < 0)
1352	goto fail;
1353
1354	ret = crypto_aead_setauthsize(tfm, authsize: icv_len);
1355	if (ret < 0)
1356	goto fail;
1357
1358	return tfm;
1359	fail:
1360	crypto_free_aead(tfm);
1361	return ERR_PTR(error: ret);
1362	}
1363
1364	static int init_rx_sa(struct macsec_rx_sa *rx_sa, char *sak, int key_len,
1365	int icv_len)
1366	{
1367	rx_sa->stats = alloc_percpu(struct macsec_rx_sa_stats);
1368	if (!rx_sa->stats)
1369	return -ENOMEM;
1370
1371	rx_sa->key.tfm = macsec_alloc_tfm(key: sak, key_len, icv_len);
1372	if (IS_ERR(ptr: rx_sa->key.tfm)) {
1373	free_percpu(pdata: rx_sa->stats);
1374	return PTR_ERR(ptr: rx_sa->key.tfm);
1375	}
1376
1377	rx_sa->ssci = MACSEC_UNDEF_SSCI;
1378	rx_sa->active = false;
1379	rx_sa->next_pn = 1;
1380	refcount_set(r: &rx_sa->refcnt, n: 1);
1381	spin_lock_init(&rx_sa->lock);
1382
1383	return 0;
1384	}
1385
1386	static void clear_rx_sa(struct macsec_rx_sa *rx_sa)
1387	{
1388	rx_sa->active = false;
1389
1390	macsec_rxsa_put(sa: rx_sa);
1391	}
1392
1393	static void free_rx_sc(struct macsec_rx_sc *rx_sc)
1394	{
1395	int i;
1396
1397	for (i = 0; i < MACSEC_NUM_AN; i++) {
1398	struct macsec_rx_sa *sa = rtnl_dereference(rx_sc->sa[i]);
1399
1400	RCU_INIT_POINTER(rx_sc->sa[i], NULL);
1401	if (sa)
1402	clear_rx_sa(rx_sa: sa);
1403	}
1404
1405	macsec_rxsc_put(sc: rx_sc);
1406	}
1407
1408	static struct macsec_rx_sc *del_rx_sc(struct macsec_secy *secy, sci_t sci)
1409	{
1410	struct macsec_rx_sc *rx_sc, __rcu **rx_scp;
1411
1412	for (rx_scp = &secy->rx_sc, rx_sc = rtnl_dereference(*rx_scp);
1413	rx_sc;
1414	rx_scp = &rx_sc->next, rx_sc = rtnl_dereference(*rx_scp)) {
1415	if (rx_sc->sci == sci) {
1416	if (rx_sc->active)
1417	secy->n_rx_sc--;
1418	rcu_assign_pointer(*rx_scp, rx_sc->next);
1419	return rx_sc;
1420	}
1421	}
1422
1423	return NULL;
1424	}
1425
1426	static struct macsec_rx_sc *create_rx_sc(struct net_device *dev, sci_t sci,
1427	bool active)
1428	{
1429	struct macsec_rx_sc *rx_sc;
1430	struct macsec_dev *macsec;
1431	struct net_device *real_dev = macsec_priv(dev)->real_dev;
1432	struct macsec_rxh_data *rxd = macsec_data_rtnl(dev: real_dev);
1433	struct macsec_secy *secy;
1434
1435	list_for_each_entry(macsec, &rxd->secys, secys) {
1436	if (find_rx_sc_rtnl(secy: &macsec->secy, sci))
1437	return ERR_PTR(error: -EEXIST);
1438	}
1439
1440	rx_sc = kzalloc(sizeof(*rx_sc), GFP_KERNEL);
1441	if (!rx_sc)
1442	return ERR_PTR(error: -ENOMEM);
1443
1444	rx_sc->stats = netdev_alloc_pcpu_stats(struct pcpu_rx_sc_stats);
1445	if (!rx_sc->stats) {
1446	kfree(objp: rx_sc);
1447	return ERR_PTR(error: -ENOMEM);
1448	}
1449
1450	rx_sc->sci = sci;
1451	rx_sc->active = active;
1452	refcount_set(r: &rx_sc->refcnt, n: 1);
1453
1454	secy = &macsec_priv(dev)->secy;
1455	rcu_assign_pointer(rx_sc->next, secy->rx_sc);
1456	rcu_assign_pointer(secy->rx_sc, rx_sc);
1457
1458	if (rx_sc->active)
1459	secy->n_rx_sc++;
1460
1461	return rx_sc;
1462	}
1463
1464	static int init_tx_sa(struct macsec_tx_sa *tx_sa, char *sak, int key_len,
1465	int icv_len)
1466	{
1467	tx_sa->stats = alloc_percpu(struct macsec_tx_sa_stats);
1468	if (!tx_sa->stats)
1469	return -ENOMEM;
1470
1471	tx_sa->key.tfm = macsec_alloc_tfm(key: sak, key_len, icv_len);
1472	if (IS_ERR(ptr: tx_sa->key.tfm)) {
1473	free_percpu(pdata: tx_sa->stats);
1474	return PTR_ERR(ptr: tx_sa->key.tfm);
1475	}
1476
1477	tx_sa->ssci = MACSEC_UNDEF_SSCI;
1478	tx_sa->active = false;
1479	refcount_set(r: &tx_sa->refcnt, n: 1);
1480	spin_lock_init(&tx_sa->lock);
1481
1482	return 0;
1483	}
1484
1485	static void clear_tx_sa(struct macsec_tx_sa *tx_sa)
1486	{
1487	tx_sa->active = false;
1488
1489	macsec_txsa_put(sa: tx_sa);
1490	}
1491
1492	static struct genl_family macsec_fam;
1493
1494	static struct net_device *get_dev_from_nl(struct net *net,
1495	struct nlattr **attrs)
1496	{
1497	int ifindex = nla_get_u32(nla: attrs[MACSEC_ATTR_IFINDEX]);
1498	struct net_device *dev;
1499
1500	dev = __dev_get_by_index(net, ifindex);
1501	if (!dev)
1502	return ERR_PTR(error: -ENODEV);
1503
1504	if (!netif_is_macsec(dev))
1505	return ERR_PTR(error: -ENODEV);
1506
1507	return dev;
1508	}
1509
1510	static enum macsec_offload nla_get_offload(const struct nlattr *nla)
1511	{
1512	return (__force enum macsec_offload)nla_get_u8(nla);
1513	}
1514
1515	static sci_t nla_get_sci(const struct nlattr *nla)
1516	{
1517	return (__force sci_t)nla_get_u64(nla);
1518	}
1519
1520	static int nla_put_sci(struct sk_buff *skb, int attrtype, sci_t value,
1521	int padattr)
1522	{
1523	return nla_put_u64_64bit(skb, attrtype, value: (__force u64)value, padattr);
1524	}
1525
1526	static ssci_t nla_get_ssci(const struct nlattr *nla)
1527	{
1528	return (__force ssci_t)nla_get_u32(nla);
1529	}
1530
1531	static int nla_put_ssci(struct sk_buff *skb, int attrtype, ssci_t value)
1532	{
1533	return nla_put_u32(skb, attrtype, value: (__force u64)value);
1534	}
1535
1536	static struct macsec_tx_sa *get_txsa_from_nl(struct net *net,
1537	struct nlattr **attrs,
1538	struct nlattr **tb_sa,
1539	struct net_device **devp,
1540	struct macsec_secy **secyp,
1541	struct macsec_tx_sc **scp,
1542	u8 *assoc_num)
1543	{
1544	struct net_device *dev;
1545	struct macsec_secy *secy;
1546	struct macsec_tx_sc *tx_sc;
1547	struct macsec_tx_sa *tx_sa;
1548
1549	if (!tb_sa[MACSEC_SA_ATTR_AN])
1550	return ERR_PTR(error: -EINVAL);
1551
1552	*assoc_num = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_AN]);
1553
1554	dev = get_dev_from_nl(net, attrs);
1555	if (IS_ERR(ptr: dev))
1556	return ERR_CAST(ptr: dev);
1557
1558	if (*assoc_num >= MACSEC_NUM_AN)
1559	return ERR_PTR(error: -EINVAL);
1560
1561	secy = &macsec_priv(dev)->secy;
1562	tx_sc = &secy->tx_sc;
1563
1564	tx_sa = rtnl_dereference(tx_sc->sa[*assoc_num]);
1565	if (!tx_sa)
1566	return ERR_PTR(error: -ENODEV);
1567
1568	*devp = dev;
1569	*scp = tx_sc;
1570	*secyp = secy;
1571	return tx_sa;
1572	}
1573
1574	static struct macsec_rx_sc *get_rxsc_from_nl(struct net *net,
1575	struct nlattr **attrs,
1576	struct nlattr **tb_rxsc,
1577	struct net_device **devp,
1578	struct macsec_secy **secyp)
1579	{
1580	struct net_device *dev;
1581	struct macsec_secy *secy;
1582	struct macsec_rx_sc *rx_sc;
1583	sci_t sci;
1584
1585	dev = get_dev_from_nl(net, attrs);
1586	if (IS_ERR(ptr: dev))
1587	return ERR_CAST(ptr: dev);
1588
1589	secy = &macsec_priv(dev)->secy;
1590
1591	if (!tb_rxsc[MACSEC_RXSC_ATTR_SCI])
1592	return ERR_PTR(error: -EINVAL);
1593
1594	sci = nla_get_sci(nla: tb_rxsc[MACSEC_RXSC_ATTR_SCI]);
1595	rx_sc = find_rx_sc_rtnl(secy, sci);
1596	if (!rx_sc)
1597	return ERR_PTR(error: -ENODEV);
1598
1599	*secyp = secy;
1600	*devp = dev;
1601
1602	return rx_sc;
1603	}
1604
1605	static struct macsec_rx_sa *get_rxsa_from_nl(struct net *net,
1606	struct nlattr **attrs,
1607	struct nlattr **tb_rxsc,
1608	struct nlattr **tb_sa,
1609	struct net_device **devp,
1610	struct macsec_secy **secyp,
1611	struct macsec_rx_sc **scp,
1612	u8 *assoc_num)
1613	{
1614	struct macsec_rx_sc *rx_sc;
1615	struct macsec_rx_sa *rx_sa;
1616
1617	if (!tb_sa[MACSEC_SA_ATTR_AN])
1618	return ERR_PTR(error: -EINVAL);
1619
1620	*assoc_num = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_AN]);
1621	if (*assoc_num >= MACSEC_NUM_AN)
1622	return ERR_PTR(error: -EINVAL);
1623
1624	rx_sc = get_rxsc_from_nl(net, attrs, tb_rxsc, devp, secyp);
1625	if (IS_ERR(ptr: rx_sc))
1626	return ERR_CAST(ptr: rx_sc);
1627
1628	rx_sa = rtnl_dereference(rx_sc->sa[*assoc_num]);
1629	if (!rx_sa)
1630	return ERR_PTR(error: -ENODEV);
1631
1632	*scp = rx_sc;
1633	return rx_sa;
1634	}
1635
1636	static const struct nla_policy macsec_genl_policy[NUM_MACSEC_ATTR] = {
1637	[MACSEC_ATTR_IFINDEX] = { .type = NLA_U32 },
1638	[MACSEC_ATTR_RXSC_CONFIG] = { .type = NLA_NESTED },
1639	[MACSEC_ATTR_SA_CONFIG] = { .type = NLA_NESTED },
1640	[MACSEC_ATTR_OFFLOAD] = { .type = NLA_NESTED },
1641	};
1642
1643	static const struct nla_policy macsec_genl_rxsc_policy[NUM_MACSEC_RXSC_ATTR] = {
1644	[MACSEC_RXSC_ATTR_SCI] = { .type = NLA_U64 },
1645	[MACSEC_RXSC_ATTR_ACTIVE] = { .type = NLA_U8 },
1646	};
1647
1648	static const struct nla_policy macsec_genl_sa_policy[NUM_MACSEC_SA_ATTR] = {
1649	[MACSEC_SA_ATTR_AN] = { .type = NLA_U8 },
1650	[MACSEC_SA_ATTR_ACTIVE] = { .type = NLA_U8 },
1651	[MACSEC_SA_ATTR_PN] = NLA_POLICY_MIN_LEN(4),
1652	[MACSEC_SA_ATTR_KEYID] = { .type = NLA_BINARY,
1653	.len = MACSEC_KEYID_LEN, },
1654	[MACSEC_SA_ATTR_KEY] = { .type = NLA_BINARY,
1655	.len = MACSEC_MAX_KEY_LEN, },
1656	[MACSEC_SA_ATTR_SSCI] = { .type = NLA_U32 },
1657	[MACSEC_SA_ATTR_SALT] = { .type = NLA_BINARY,
1658	.len = MACSEC_SALT_LEN, },
1659	};
1660
1661	static const struct nla_policy macsec_genl_offload_policy[NUM_MACSEC_OFFLOAD_ATTR] = {
1662	[MACSEC_OFFLOAD_ATTR_TYPE] = { .type = NLA_U8 },
1663	};
1664
1665	/* Offloads an operation to a device driver */
1666	static int macsec_offload(int (* const func)(struct macsec_context *),
1667	struct macsec_context *ctx)
1668	{
1669	int ret;
1670
1671	if (unlikely(!func))
1672	return 0;
1673
1674	if (ctx->offload == MACSEC_OFFLOAD_PHY)
1675	mutex_lock(&ctx->phydev->lock);
1676
1677	ret = (*func)(ctx);
1678
1679	if (ctx->offload == MACSEC_OFFLOAD_PHY)
1680	mutex_unlock(lock: &ctx->phydev->lock);
1681
1682	return ret;
1683	}
1684
1685	static int parse_sa_config(struct nlattr **attrs, struct nlattr **tb_sa)
1686	{
1687	if (!attrs[MACSEC_ATTR_SA_CONFIG])
1688	return -EINVAL;
1689
1690	if (nla_parse_nested_deprecated(tb: tb_sa, maxtype: MACSEC_SA_ATTR_MAX, nla: attrs[MACSEC_ATTR_SA_CONFIG], policy: macsec_genl_sa_policy, NULL))
1691	return -EINVAL;
1692
1693	return 0;
1694	}
1695
1696	static int parse_rxsc_config(struct nlattr **attrs, struct nlattr **tb_rxsc)
1697	{
1698	if (!attrs[MACSEC_ATTR_RXSC_CONFIG])
1699	return -EINVAL;
1700
1701	if (nla_parse_nested_deprecated(tb: tb_rxsc, maxtype: MACSEC_RXSC_ATTR_MAX, nla: attrs[MACSEC_ATTR_RXSC_CONFIG], policy: macsec_genl_rxsc_policy, NULL))
1702	return -EINVAL;
1703
1704	return 0;
1705	}
1706
1707	static bool validate_add_rxsa(struct nlattr **attrs)
1708	{
1709	if (!attrs[MACSEC_SA_ATTR_AN] ||
1710	!attrs[MACSEC_SA_ATTR_KEY] ||
1711	!attrs[MACSEC_SA_ATTR_KEYID])
1712	return false;
1713
1714	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_AN]) >= MACSEC_NUM_AN)
1715	return false;
1716
1717	if (attrs[MACSEC_SA_ATTR_PN] &&
1718	nla_get_u64(nla: attrs[MACSEC_SA_ATTR_PN]) == 0)
1719	return false;
1720
1721	if (attrs[MACSEC_SA_ATTR_ACTIVE]) {
1722	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_ACTIVE]) > 1)
1723	return false;
1724	}
1725
1726	if (nla_len(nla: attrs[MACSEC_SA_ATTR_KEYID]) != MACSEC_KEYID_LEN)
1727	return false;
1728
1729	return true;
1730	}
1731
1732	static int macsec_add_rxsa(struct sk_buff *skb, struct genl_info *info)
1733	{
1734	struct net_device *dev;
1735	struct nlattr **attrs = info->attrs;
1736	struct macsec_secy *secy;
1737	struct macsec_rx_sc *rx_sc;
1738	struct macsec_rx_sa *rx_sa;
1739	unsigned char assoc_num;
1740	int pn_len;
1741	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
1742	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
1743	int err;
1744
1745	if (!attrs[MACSEC_ATTR_IFINDEX])
1746	return -EINVAL;
1747
1748	if (parse_sa_config(attrs, tb_sa))
1749	return -EINVAL;
1750
1751	if (parse_rxsc_config(attrs, tb_rxsc))
1752	return -EINVAL;
1753
1754	if (!validate_add_rxsa(attrs: tb_sa))
1755	return -EINVAL;
1756
1757	rtnl_lock();
1758	rx_sc = get_rxsc_from_nl(net: genl_info_net(info), attrs, tb_rxsc, devp: &dev, secyp: &secy);
1759	if (IS_ERR(ptr: rx_sc)) {
1760	rtnl_unlock();
1761	return PTR_ERR(ptr: rx_sc);
1762	}
1763
1764	assoc_num = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_AN]);
1765
1766	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_KEY]) != secy->key_len) {
1767	pr_notice("macsec: nl: add_rxsa: bad key length: %d != %d\n",
1768	nla_len(tb_sa[MACSEC_SA_ATTR_KEY]), secy->key_len);
1769	rtnl_unlock();
1770	return -EINVAL;
1771	}
1772
1773	pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
1774	if (tb_sa[MACSEC_SA_ATTR_PN] &&
1775	nla_len(nla: tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
1776	pr_notice("macsec: nl: add_rxsa: bad pn length: %d != %d\n",
1777	nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
1778	rtnl_unlock();
1779	return -EINVAL;
1780	}
1781
1782	if (secy->xpn) {
1783	if (!tb_sa[MACSEC_SA_ATTR_SSCI] || !tb_sa[MACSEC_SA_ATTR_SALT]) {
1784	rtnl_unlock();
1785	return -EINVAL;
1786	}
1787
1788	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_SALT]) != MACSEC_SALT_LEN) {
1789	pr_notice("macsec: nl: add_rxsa: bad salt length: %d != %d\n",
1790	nla_len(tb_sa[MACSEC_SA_ATTR_SALT]),
1791	MACSEC_SALT_LEN);
1792	rtnl_unlock();
1793	return -EINVAL;
1794	}
1795	}
1796
1797	rx_sa = rtnl_dereference(rx_sc->sa[assoc_num]);
1798	if (rx_sa) {
1799	rtnl_unlock();
1800	return -EBUSY;
1801	}
1802
1803	rx_sa = kmalloc(sizeof(*rx_sa), GFP_KERNEL);
1804	if (!rx_sa) {
1805	rtnl_unlock();
1806	return -ENOMEM;
1807	}
1808
1809	err = init_rx_sa(rx_sa, sak: nla_data(nla: tb_sa[MACSEC_SA_ATTR_KEY]),
1810	key_len: secy->key_len, icv_len: secy->icv_len);
1811	if (err < 0) {
1812	kfree(objp: rx_sa);
1813	rtnl_unlock();
1814	return err;
1815	}
1816
1817	if (tb_sa[MACSEC_SA_ATTR_PN]) {
1818	spin_lock_bh(lock: &rx_sa->lock);
1819	rx_sa->next_pn = nla_get_u64(nla: tb_sa[MACSEC_SA_ATTR_PN]);
1820	spin_unlock_bh(lock: &rx_sa->lock);
1821	}
1822
1823	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
1824	rx_sa->active = !!nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_ACTIVE]);
1825
1826	rx_sa->sc = rx_sc;
1827
1828	if (secy->xpn) {
1829	rx_sa->ssci = nla_get_ssci(nla: tb_sa[MACSEC_SA_ATTR_SSCI]);
1830	nla_memcpy(dest: rx_sa->key.salt.bytes, src: tb_sa[MACSEC_SA_ATTR_SALT],
1831	MACSEC_SALT_LEN);
1832	}
1833
1834	/* If h/w offloading is available, propagate to the device */
1835	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
1836	const struct macsec_ops *ops;
1837	struct macsec_context ctx;
1838
1839	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
1840	if (!ops) {
1841	err = -EOPNOTSUPP;
1842	goto cleanup;
1843	}
1844
1845	ctx.sa.assoc_num = assoc_num;
1846	ctx.sa.rx_sa = rx_sa;
1847	ctx.secy = secy;
1848	memcpy(ctx.sa.key, nla_data(tb_sa[MACSEC_SA_ATTR_KEY]),
1849	secy->key_len);
1850
1851	err = macsec_offload(func: ops->mdo_add_rxsa, ctx: &ctx);
1852	memzero_explicit(s: ctx.sa.key, count: secy->key_len);
1853	if (err)
1854	goto cleanup;
1855	}
1856
1857	nla_memcpy(dest: rx_sa->key.id, src: tb_sa[MACSEC_SA_ATTR_KEYID], MACSEC_KEYID_LEN);
1858	rcu_assign_pointer(rx_sc->sa[assoc_num], rx_sa);
1859
1860	rtnl_unlock();
1861
1862	return 0;
1863
1864	cleanup:
1865	macsec_rxsa_put(sa: rx_sa);
1866	rtnl_unlock();
1867	return err;
1868	}
1869
1870	static bool validate_add_rxsc(struct nlattr **attrs)
1871	{
1872	if (!attrs[MACSEC_RXSC_ATTR_SCI])
1873	return false;
1874
1875	if (attrs[MACSEC_RXSC_ATTR_ACTIVE]) {
1876	if (nla_get_u8(nla: attrs[MACSEC_RXSC_ATTR_ACTIVE]) > 1)
1877	return false;
1878	}
1879
1880	return true;
1881	}
1882
1883	static int macsec_add_rxsc(struct sk_buff *skb, struct genl_info *info)
1884	{
1885	struct net_device *dev;
1886	sci_t sci = MACSEC_UNDEF_SCI;
1887	struct nlattr **attrs = info->attrs;
1888	struct macsec_rx_sc *rx_sc;
1889	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
1890	struct macsec_secy *secy;
1891	bool active = true;
1892	int ret;
1893
1894	if (!attrs[MACSEC_ATTR_IFINDEX])
1895	return -EINVAL;
1896
1897	if (parse_rxsc_config(attrs, tb_rxsc))
1898	return -EINVAL;
1899
1900	if (!validate_add_rxsc(attrs: tb_rxsc))
1901	return -EINVAL;
1902
1903	rtnl_lock();
1904	dev = get_dev_from_nl(net: genl_info_net(info), attrs);
1905	if (IS_ERR(ptr: dev)) {
1906	rtnl_unlock();
1907	return PTR_ERR(ptr: dev);
1908	}
1909
1910	secy = &macsec_priv(dev)->secy;
1911	sci = nla_get_sci(nla: tb_rxsc[MACSEC_RXSC_ATTR_SCI]);
1912
1913	if (tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE])
1914	active = nla_get_u8(nla: tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]);
1915
1916	rx_sc = create_rx_sc(dev, sci, active);
1917	if (IS_ERR(ptr: rx_sc)) {
1918	rtnl_unlock();
1919	return PTR_ERR(ptr: rx_sc);
1920	}
1921
1922	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
1923	const struct macsec_ops *ops;
1924	struct macsec_context ctx;
1925
1926	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
1927	if (!ops) {
1928	ret = -EOPNOTSUPP;
1929	goto cleanup;
1930	}
1931
1932	ctx.rx_sc = rx_sc;
1933	ctx.secy = secy;
1934
1935	ret = macsec_offload(func: ops->mdo_add_rxsc, ctx: &ctx);
1936	if (ret)
1937	goto cleanup;
1938	}
1939
1940	rtnl_unlock();
1941
1942	return 0;
1943
1944	cleanup:
1945	del_rx_sc(secy, sci);
1946	free_rx_sc(rx_sc);
1947	rtnl_unlock();
1948	return ret;
1949	}
1950
1951	static bool validate_add_txsa(struct nlattr **attrs)
1952	{
1953	if (!attrs[MACSEC_SA_ATTR_AN] ||
1954	!attrs[MACSEC_SA_ATTR_PN] ||
1955	!attrs[MACSEC_SA_ATTR_KEY] ||
1956	!attrs[MACSEC_SA_ATTR_KEYID])
1957	return false;
1958
1959	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_AN]) >= MACSEC_NUM_AN)
1960	return false;
1961
1962	if (nla_get_u64(nla: attrs[MACSEC_SA_ATTR_PN]) == 0)
1963	return false;
1964
1965	if (attrs[MACSEC_SA_ATTR_ACTIVE]) {
1966	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_ACTIVE]) > 1)
1967	return false;
1968	}
1969
1970	if (nla_len(nla: attrs[MACSEC_SA_ATTR_KEYID]) != MACSEC_KEYID_LEN)
1971	return false;
1972
1973	return true;
1974	}
1975
1976	static int macsec_add_txsa(struct sk_buff *skb, struct genl_info *info)
1977	{
1978	struct net_device *dev;
1979	struct nlattr **attrs = info->attrs;
1980	struct macsec_secy *secy;
1981	struct macsec_tx_sc *tx_sc;
1982	struct macsec_tx_sa *tx_sa;
1983	unsigned char assoc_num;
1984	int pn_len;
1985	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
1986	bool was_operational;
1987	int err;
1988
1989	if (!attrs[MACSEC_ATTR_IFINDEX])
1990	return -EINVAL;
1991
1992	if (parse_sa_config(attrs, tb_sa))
1993	return -EINVAL;
1994
1995	if (!validate_add_txsa(attrs: tb_sa))
1996	return -EINVAL;
1997
1998	rtnl_lock();
1999	dev = get_dev_from_nl(net: genl_info_net(info), attrs);
2000	if (IS_ERR(ptr: dev)) {
2001	rtnl_unlock();
2002	return PTR_ERR(ptr: dev);
2003	}
2004
2005	secy = &macsec_priv(dev)->secy;
2006	tx_sc = &secy->tx_sc;
2007
2008	assoc_num = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_AN]);
2009
2010	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_KEY]) != secy->key_len) {
2011	pr_notice("macsec: nl: add_txsa: bad key length: %d != %d\n",
2012	nla_len(tb_sa[MACSEC_SA_ATTR_KEY]), secy->key_len);
2013	rtnl_unlock();
2014	return -EINVAL;
2015	}
2016
2017	pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
2018	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
2019	pr_notice("macsec: nl: add_txsa: bad pn length: %d != %d\n",
2020	nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
2021	rtnl_unlock();
2022	return -EINVAL;
2023	}
2024
2025	if (secy->xpn) {
2026	if (!tb_sa[MACSEC_SA_ATTR_SSCI] || !tb_sa[MACSEC_SA_ATTR_SALT]) {
2027	rtnl_unlock();
2028	return -EINVAL;
2029	}
2030
2031	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_SALT]) != MACSEC_SALT_LEN) {
2032	pr_notice("macsec: nl: add_txsa: bad salt length: %d != %d\n",
2033	nla_len(tb_sa[MACSEC_SA_ATTR_SALT]),
2034	MACSEC_SALT_LEN);
2035	rtnl_unlock();
2036	return -EINVAL;
2037	}
2038	}
2039
2040	tx_sa = rtnl_dereference(tx_sc->sa[assoc_num]);
2041	if (tx_sa) {
2042	rtnl_unlock();
2043	return -EBUSY;
2044	}
2045
2046	tx_sa = kmalloc(sizeof(*tx_sa), GFP_KERNEL);
2047	if (!tx_sa) {
2048	rtnl_unlock();
2049	return -ENOMEM;
2050	}
2051
2052	err = init_tx_sa(tx_sa, sak: nla_data(nla: tb_sa[MACSEC_SA_ATTR_KEY]),
2053	key_len: secy->key_len, icv_len: secy->icv_len);
2054	if (err < 0) {
2055	kfree(objp: tx_sa);
2056	rtnl_unlock();
2057	return err;
2058	}
2059
2060	spin_lock_bh(lock: &tx_sa->lock);
2061	tx_sa->next_pn = nla_get_u64(nla: tb_sa[MACSEC_SA_ATTR_PN]);
2062	spin_unlock_bh(lock: &tx_sa->lock);
2063
2064	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
2065	tx_sa->active = !!nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_ACTIVE]);
2066
2067	was_operational = secy->operational;
2068	if (assoc_num == tx_sc->encoding_sa && tx_sa->active)
2069	secy->operational = true;
2070
2071	if (secy->xpn) {
2072	tx_sa->ssci = nla_get_ssci(nla: tb_sa[MACSEC_SA_ATTR_SSCI]);
2073	nla_memcpy(dest: tx_sa->key.salt.bytes, src: tb_sa[MACSEC_SA_ATTR_SALT],
2074	MACSEC_SALT_LEN);
2075	}
2076
2077	/* If h/w offloading is available, propagate to the device */
2078	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2079	const struct macsec_ops *ops;
2080	struct macsec_context ctx;
2081
2082	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2083	if (!ops) {
2084	err = -EOPNOTSUPP;
2085	goto cleanup;
2086	}
2087
2088	ctx.sa.assoc_num = assoc_num;
2089	ctx.sa.tx_sa = tx_sa;
2090	ctx.secy = secy;
2091	memcpy(ctx.sa.key, nla_data(tb_sa[MACSEC_SA_ATTR_KEY]),
2092	secy->key_len);
2093
2094	err = macsec_offload(func: ops->mdo_add_txsa, ctx: &ctx);
2095	memzero_explicit(s: ctx.sa.key, count: secy->key_len);
2096	if (err)
2097	goto cleanup;
2098	}
2099
2100	nla_memcpy(dest: tx_sa->key.id, src: tb_sa[MACSEC_SA_ATTR_KEYID], MACSEC_KEYID_LEN);
2101	rcu_assign_pointer(tx_sc->sa[assoc_num], tx_sa);
2102
2103	rtnl_unlock();
2104
2105	return 0;
2106
2107	cleanup:
2108	secy->operational = was_operational;
2109	macsec_txsa_put(sa: tx_sa);
2110	rtnl_unlock();
2111	return err;
2112	}
2113
2114	static int macsec_del_rxsa(struct sk_buff *skb, struct genl_info *info)
2115	{
2116	struct nlattr **attrs = info->attrs;
2117	struct net_device *dev;
2118	struct macsec_secy *secy;
2119	struct macsec_rx_sc *rx_sc;
2120	struct macsec_rx_sa *rx_sa;
2121	u8 assoc_num;
2122	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2123	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2124	int ret;
2125
2126	if (!attrs[MACSEC_ATTR_IFINDEX])
2127	return -EINVAL;
2128
2129	if (parse_sa_config(attrs, tb_sa))
2130	return -EINVAL;
2131
2132	if (parse_rxsc_config(attrs, tb_rxsc))
2133	return -EINVAL;
2134
2135	rtnl_lock();
2136	rx_sa = get_rxsa_from_nl(net: genl_info_net(info), attrs, tb_rxsc, tb_sa,
2137	devp: &dev, secyp: &secy, scp: &rx_sc, assoc_num: &assoc_num);
2138	if (IS_ERR(ptr: rx_sa)) {
2139	rtnl_unlock();
2140	return PTR_ERR(ptr: rx_sa);
2141	}
2142
2143	if (rx_sa->active) {
2144	rtnl_unlock();
2145	return -EBUSY;
2146	}
2147
2148	/* If h/w offloading is available, propagate to the device */
2149	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2150	const struct macsec_ops *ops;
2151	struct macsec_context ctx;
2152
2153	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2154	if (!ops) {
2155	ret = -EOPNOTSUPP;
2156	goto cleanup;
2157	}
2158
2159	ctx.sa.assoc_num = assoc_num;
2160	ctx.sa.rx_sa = rx_sa;
2161	ctx.secy = secy;
2162
2163	ret = macsec_offload(func: ops->mdo_del_rxsa, ctx: &ctx);
2164	if (ret)
2165	goto cleanup;
2166	}
2167
2168	RCU_INIT_POINTER(rx_sc->sa[assoc_num], NULL);
2169	clear_rx_sa(rx_sa);
2170
2171	rtnl_unlock();
2172
2173	return 0;
2174
2175	cleanup:
2176	rtnl_unlock();
2177	return ret;
2178	}
2179
2180	static int macsec_del_rxsc(struct sk_buff *skb, struct genl_info *info)
2181	{
2182	struct nlattr **attrs = info->attrs;
2183	struct net_device *dev;
2184	struct macsec_secy *secy;
2185	struct macsec_rx_sc *rx_sc;
2186	sci_t sci;
2187	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2188	int ret;
2189
2190	if (!attrs[MACSEC_ATTR_IFINDEX])
2191	return -EINVAL;
2192
2193	if (parse_rxsc_config(attrs, tb_rxsc))
2194	return -EINVAL;
2195
2196	if (!tb_rxsc[MACSEC_RXSC_ATTR_SCI])
2197	return -EINVAL;
2198
2199	rtnl_lock();
2200	dev = get_dev_from_nl(net: genl_info_net(info), attrs: info->attrs);
2201	if (IS_ERR(ptr: dev)) {
2202	rtnl_unlock();
2203	return PTR_ERR(ptr: dev);
2204	}
2205
2206	secy = &macsec_priv(dev)->secy;
2207	sci = nla_get_sci(nla: tb_rxsc[MACSEC_RXSC_ATTR_SCI]);
2208
2209	rx_sc = del_rx_sc(secy, sci);
2210	if (!rx_sc) {
2211	rtnl_unlock();
2212	return -ENODEV;
2213	}
2214
2215	/* If h/w offloading is available, propagate to the device */
2216	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2217	const struct macsec_ops *ops;
2218	struct macsec_context ctx;
2219
2220	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2221	if (!ops) {
2222	ret = -EOPNOTSUPP;
2223	goto cleanup;
2224	}
2225
2226	ctx.rx_sc = rx_sc;
2227	ctx.secy = secy;
2228	ret = macsec_offload(func: ops->mdo_del_rxsc, ctx: &ctx);
2229	if (ret)
2230	goto cleanup;
2231	}
2232
2233	free_rx_sc(rx_sc);
2234	rtnl_unlock();
2235
2236	return 0;
2237
2238	cleanup:
2239	rtnl_unlock();
2240	return ret;
2241	}
2242
2243	static int macsec_del_txsa(struct sk_buff *skb, struct genl_info *info)
2244	{
2245	struct nlattr **attrs = info->attrs;
2246	struct net_device *dev;
2247	struct macsec_secy *secy;
2248	struct macsec_tx_sc *tx_sc;
2249	struct macsec_tx_sa *tx_sa;
2250	u8 assoc_num;
2251	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2252	int ret;
2253
2254	if (!attrs[MACSEC_ATTR_IFINDEX])
2255	return -EINVAL;
2256
2257	if (parse_sa_config(attrs, tb_sa))
2258	return -EINVAL;
2259
2260	rtnl_lock();
2261	tx_sa = get_txsa_from_nl(net: genl_info_net(info), attrs, tb_sa,
2262	devp: &dev, secyp: &secy, scp: &tx_sc, assoc_num: &assoc_num);
2263	if (IS_ERR(ptr: tx_sa)) {
2264	rtnl_unlock();
2265	return PTR_ERR(ptr: tx_sa);
2266	}
2267
2268	if (tx_sa->active) {
2269	rtnl_unlock();
2270	return -EBUSY;
2271	}
2272
2273	/* If h/w offloading is available, propagate to the device */
2274	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2275	const struct macsec_ops *ops;
2276	struct macsec_context ctx;
2277
2278	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2279	if (!ops) {
2280	ret = -EOPNOTSUPP;
2281	goto cleanup;
2282	}
2283
2284	ctx.sa.assoc_num = assoc_num;
2285	ctx.sa.tx_sa = tx_sa;
2286	ctx.secy = secy;
2287
2288	ret = macsec_offload(func: ops->mdo_del_txsa, ctx: &ctx);
2289	if (ret)
2290	goto cleanup;
2291	}
2292
2293	RCU_INIT_POINTER(tx_sc->sa[assoc_num], NULL);
2294	clear_tx_sa(tx_sa);
2295
2296	rtnl_unlock();
2297
2298	return 0;
2299
2300	cleanup:
2301	rtnl_unlock();
2302	return ret;
2303	}
2304
2305	static bool validate_upd_sa(struct nlattr **attrs)
2306	{
2307	if (!attrs[MACSEC_SA_ATTR_AN] ||
2308	attrs[MACSEC_SA_ATTR_KEY] ||
2309	attrs[MACSEC_SA_ATTR_KEYID] ||
2310	attrs[MACSEC_SA_ATTR_SSCI] ||
2311	attrs[MACSEC_SA_ATTR_SALT])
2312	return false;
2313
2314	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_AN]) >= MACSEC_NUM_AN)
2315	return false;
2316
2317	if (attrs[MACSEC_SA_ATTR_PN] && nla_get_u64(nla: attrs[MACSEC_SA_ATTR_PN]) == 0)
2318	return false;
2319
2320	if (attrs[MACSEC_SA_ATTR_ACTIVE]) {
2321	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_ACTIVE]) > 1)
2322	return false;
2323	}
2324
2325	return true;
2326	}
2327
2328	static int macsec_upd_txsa(struct sk_buff *skb, struct genl_info *info)
2329	{
2330	struct nlattr **attrs = info->attrs;
2331	struct net_device *dev;
2332	struct macsec_secy *secy;
2333	struct macsec_tx_sc *tx_sc;
2334	struct macsec_tx_sa *tx_sa;
2335	u8 assoc_num;
2336	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2337	bool was_operational, was_active;
2338	pn_t prev_pn;
2339	int ret = 0;
2340
2341	prev_pn.full64 = 0;
2342
2343	if (!attrs[MACSEC_ATTR_IFINDEX])
2344	return -EINVAL;
2345
2346	if (parse_sa_config(attrs, tb_sa))
2347	return -EINVAL;
2348
2349	if (!validate_upd_sa(attrs: tb_sa))
2350	return -EINVAL;
2351
2352	rtnl_lock();
2353	tx_sa = get_txsa_from_nl(net: genl_info_net(info), attrs, tb_sa,
2354	devp: &dev, secyp: &secy, scp: &tx_sc, assoc_num: &assoc_num);
2355	if (IS_ERR(ptr: tx_sa)) {
2356	rtnl_unlock();
2357	return PTR_ERR(ptr: tx_sa);
2358	}
2359
2360	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2361	int pn_len;
2362
2363	pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
2364	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
2365	pr_notice("macsec: nl: upd_txsa: bad pn length: %d != %d\n",
2366	nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
2367	rtnl_unlock();
2368	return -EINVAL;
2369	}
2370
2371	spin_lock_bh(lock: &tx_sa->lock);
2372	prev_pn = tx_sa->next_pn_halves;
2373	tx_sa->next_pn = nla_get_u64(nla: tb_sa[MACSEC_SA_ATTR_PN]);
2374	spin_unlock_bh(lock: &tx_sa->lock);
2375	}
2376
2377	was_active = tx_sa->active;
2378	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
2379	tx_sa->active = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_ACTIVE]);
2380
2381	was_operational = secy->operational;
2382	if (assoc_num == tx_sc->encoding_sa)
2383	secy->operational = tx_sa->active;
2384
2385	/* If h/w offloading is available, propagate to the device */
2386	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2387	const struct macsec_ops *ops;
2388	struct macsec_context ctx;
2389
2390	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2391	if (!ops) {
2392	ret = -EOPNOTSUPP;
2393	goto cleanup;
2394	}
2395
2396	ctx.sa.assoc_num = assoc_num;
2397	ctx.sa.tx_sa = tx_sa;
2398	ctx.sa.update_pn = !!prev_pn.full64;
2399	ctx.secy = secy;
2400
2401	ret = macsec_offload(func: ops->mdo_upd_txsa, ctx: &ctx);
2402	if (ret)
2403	goto cleanup;
2404	}
2405
2406	rtnl_unlock();
2407
2408	return 0;
2409
2410	cleanup:
2411	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2412	spin_lock_bh(lock: &tx_sa->lock);
2413	tx_sa->next_pn_halves = prev_pn;
2414	spin_unlock_bh(lock: &tx_sa->lock);
2415	}
2416	tx_sa->active = was_active;
2417	secy->operational = was_operational;
2418	rtnl_unlock();
2419	return ret;
2420	}
2421
2422	static int macsec_upd_rxsa(struct sk_buff *skb, struct genl_info *info)
2423	{
2424	struct nlattr **attrs = info->attrs;
2425	struct net_device *dev;
2426	struct macsec_secy *secy;
2427	struct macsec_rx_sc *rx_sc;
2428	struct macsec_rx_sa *rx_sa;
2429	u8 assoc_num;
2430	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2431	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2432	bool was_active;
2433	pn_t prev_pn;
2434	int ret = 0;
2435
2436	prev_pn.full64 = 0;
2437
2438	if (!attrs[MACSEC_ATTR_IFINDEX])
2439	return -EINVAL;
2440
2441	if (parse_rxsc_config(attrs, tb_rxsc))
2442	return -EINVAL;
2443
2444	if (parse_sa_config(attrs, tb_sa))
2445	return -EINVAL;
2446
2447	if (!validate_upd_sa(attrs: tb_sa))
2448	return -EINVAL;
2449
2450	rtnl_lock();
2451	rx_sa = get_rxsa_from_nl(net: genl_info_net(info), attrs, tb_rxsc, tb_sa,
2452	devp: &dev, secyp: &secy, scp: &rx_sc, assoc_num: &assoc_num);
2453	if (IS_ERR(ptr: rx_sa)) {
2454	rtnl_unlock();
2455	return PTR_ERR(ptr: rx_sa);
2456	}
2457
2458	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2459	int pn_len;
2460
2461	pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
2462	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
2463	pr_notice("macsec: nl: upd_rxsa: bad pn length: %d != %d\n",
2464	nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
2465	rtnl_unlock();
2466	return -EINVAL;
2467	}
2468
2469	spin_lock_bh(lock: &rx_sa->lock);
2470	prev_pn = rx_sa->next_pn_halves;
2471	rx_sa->next_pn = nla_get_u64(nla: tb_sa[MACSEC_SA_ATTR_PN]);
2472	spin_unlock_bh(lock: &rx_sa->lock);
2473	}
2474
2475	was_active = rx_sa->active;
2476	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
2477	rx_sa->active = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_ACTIVE]);
2478
2479	/* If h/w offloading is available, propagate to the device */
2480	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2481	const struct macsec_ops *ops;
2482	struct macsec_context ctx;
2483
2484	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2485	if (!ops) {
2486	ret = -EOPNOTSUPP;
2487	goto cleanup;
2488	}
2489
2490	ctx.sa.assoc_num = assoc_num;
2491	ctx.sa.rx_sa = rx_sa;
2492	ctx.sa.update_pn = !!prev_pn.full64;
2493	ctx.secy = secy;
2494
2495	ret = macsec_offload(func: ops->mdo_upd_rxsa, ctx: &ctx);
2496	if (ret)
2497	goto cleanup;
2498	}
2499
2500	rtnl_unlock();
2501	return 0;
2502
2503	cleanup:
2504	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2505	spin_lock_bh(lock: &rx_sa->lock);
2506	rx_sa->next_pn_halves = prev_pn;
2507	spin_unlock_bh(lock: &rx_sa->lock);
2508	}
2509	rx_sa->active = was_active;
2510	rtnl_unlock();
2511	return ret;
2512	}
2513
2514	static int macsec_upd_rxsc(struct sk_buff *skb, struct genl_info *info)
2515	{
2516	struct nlattr **attrs = info->attrs;
2517	struct net_device *dev;
2518	struct macsec_secy *secy;
2519	struct macsec_rx_sc *rx_sc;
2520	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2521	unsigned int prev_n_rx_sc;
2522	bool was_active;
2523	int ret;
2524
2525	if (!attrs[MACSEC_ATTR_IFINDEX])
2526	return -EINVAL;
2527
2528	if (parse_rxsc_config(attrs, tb_rxsc))
2529	return -EINVAL;
2530
2531	if (!validate_add_rxsc(attrs: tb_rxsc))
2532	return -EINVAL;
2533
2534	rtnl_lock();
2535	rx_sc = get_rxsc_from_nl(net: genl_info_net(info), attrs, tb_rxsc, devp: &dev, secyp: &secy);
2536	if (IS_ERR(ptr: rx_sc)) {
2537	rtnl_unlock();
2538	return PTR_ERR(ptr: rx_sc);
2539	}
2540
2541	was_active = rx_sc->active;
2542	prev_n_rx_sc = secy->n_rx_sc;
2543	if (tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]) {
2544	bool new = !!nla_get_u8(nla: tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]);
2545
2546	if (rx_sc->active != new)
2547	secy->n_rx_sc += new ? 1 : -1;
2548
2549	rx_sc->active = new;
2550	}
2551
2552	/* If h/w offloading is available, propagate to the device */
2553	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2554	const struct macsec_ops *ops;
2555	struct macsec_context ctx;
2556
2557	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2558	if (!ops) {
2559	ret = -EOPNOTSUPP;
2560	goto cleanup;
2561	}
2562
2563	ctx.rx_sc = rx_sc;
2564	ctx.secy = secy;
2565
2566	ret = macsec_offload(func: ops->mdo_upd_rxsc, ctx: &ctx);
2567	if (ret)
2568	goto cleanup;
2569	}
2570
2571	rtnl_unlock();
2572
2573	return 0;
2574
2575	cleanup:
2576	secy->n_rx_sc = prev_n_rx_sc;
2577	rx_sc->active = was_active;
2578	rtnl_unlock();
2579	return ret;
2580	}
2581
2582	static bool macsec_is_configured(struct macsec_dev *macsec)
2583	{
2584	struct macsec_secy *secy = &macsec->secy;
2585	struct macsec_tx_sc *tx_sc = &secy->tx_sc;
2586	int i;
2587
2588	if (secy->rx_sc)
2589	return true;
2590
2591	for (i = 0; i < MACSEC_NUM_AN; i++)
2592	if (tx_sc->sa[i])
2593	return true;
2594
2595	return false;
2596	}
2597
2598	static bool macsec_needs_tx_tag(struct macsec_dev *macsec,
2599	const struct macsec_ops *ops)
2600	{
2601	return macsec->offload == MACSEC_OFFLOAD_PHY &&
2602	ops->mdo_insert_tx_tag;
2603	}
2604
2605	static void macsec_set_head_tail_room(struct net_device *dev)
2606	{
2607	struct macsec_dev *macsec = macsec_priv(dev);
2608	struct net_device *real_dev = macsec->real_dev;
2609	int needed_headroom, needed_tailroom;
2610	const struct macsec_ops *ops;
2611
2612	ops = macsec_get_ops(macsec, NULL);
2613	if (ops) {
2614	needed_headroom = ops->needed_headroom;
2615	needed_tailroom = ops->needed_tailroom;
2616	} else {
2617	needed_headroom = MACSEC_NEEDED_HEADROOM;
2618	needed_tailroom = MACSEC_NEEDED_TAILROOM;
2619	}
2620
2621	dev->needed_headroom = real_dev->needed_headroom + needed_headroom;
2622	dev->needed_tailroom = real_dev->needed_tailroom + needed_tailroom;
2623	}
2624
2625	static void macsec_inherit_tso_max(struct net_device *dev)
2626	{
2627	struct macsec_dev *macsec = macsec_priv(dev);
2628
2629	/* if macsec is offloaded, we need to follow the lower
2630	* device's capabilities. otherwise, we can ignore them.
2631	*/
2632	if (macsec_is_offloaded(macsec))
2633	netif_inherit_tso_max(to: dev, from: macsec->real_dev);
2634	}
2635
2636	static int macsec_update_offload(struct net_device *dev, enum macsec_offload offload)
2637	{
2638	enum macsec_offload prev_offload;
2639	const struct macsec_ops *ops;
2640	struct macsec_context ctx;
2641	struct macsec_dev *macsec;
2642	int ret = 0;
2643
2644	macsec = macsec_priv(dev);
2645
2646	/* Check if the offloading mode is supported by the underlying layers */
2647	if (offload != MACSEC_OFFLOAD_OFF &&
2648	!macsec_check_offload(offload, macsec))
2649	return -EOPNOTSUPP;
2650
2651	/* Check if the net device is busy. */
2652	if (netif_running(dev))
2653	return -EBUSY;
2654
2655	/* Check if the device already has rules configured: we do not support
2656	* rules migration.
2657	*/
2658	if (macsec_is_configured(macsec))
2659	return -EBUSY;
2660
2661	prev_offload = macsec->offload;
2662
2663	ops = __macsec_get_ops(offload: offload == MACSEC_OFFLOAD_OFF ? prev_offload : offload,
2664	macsec, ctx: &ctx);
2665	if (!ops)
2666	return -EOPNOTSUPP;
2667
2668	macsec->offload = offload;
2669
2670	ctx.secy = &macsec->secy;
2671	ret = offload == MACSEC_OFFLOAD_OFF ? macsec_offload(func: ops->mdo_del_secy, ctx: &ctx)
2672	: macsec_offload(func: ops->mdo_add_secy, ctx: &ctx);
2673	if (ret) {
2674	macsec->offload = prev_offload;
2675	return ret;
2676	}
2677
2678	macsec_set_head_tail_room(dev);
2679	macsec->insert_tx_tag = macsec_needs_tx_tag(macsec, ops);
2680
2681	macsec_inherit_tso_max(dev);
2682
2683	netdev_update_features(dev);
2684
2685	return ret;
2686	}
2687
2688	static int macsec_upd_offload(struct sk_buff *skb, struct genl_info *info)
2689	{
2690	struct nlattr *tb_offload[MACSEC_OFFLOAD_ATTR_MAX + 1];
2691	struct nlattr **attrs = info->attrs;
2692	enum macsec_offload offload;
2693	struct macsec_dev *macsec;
2694	struct net_device *dev;
2695	int ret = 0;
2696
2697	if (!attrs[MACSEC_ATTR_IFINDEX])
2698	return -EINVAL;
2699
2700	if (!attrs[MACSEC_ATTR_OFFLOAD])
2701	return -EINVAL;
2702
2703	if (nla_parse_nested_deprecated(tb: tb_offload, maxtype: MACSEC_OFFLOAD_ATTR_MAX,
2704	nla: attrs[MACSEC_ATTR_OFFLOAD],
2705	policy: macsec_genl_offload_policy, NULL))
2706	return -EINVAL;
2707
2708	rtnl_lock();
2709
2710	dev = get_dev_from_nl(net: genl_info_net(info), attrs);
2711	if (IS_ERR(ptr: dev)) {
2712	ret = PTR_ERR(ptr: dev);
2713	goto out;
2714	}
2715	macsec = macsec_priv(dev);
2716
2717	if (!tb_offload[MACSEC_OFFLOAD_ATTR_TYPE]) {
2718	ret = -EINVAL;
2719	goto out;
2720	}
2721
2722	offload = nla_get_u8(nla: tb_offload[MACSEC_OFFLOAD_ATTR_TYPE]);
2723
2724	if (macsec->offload != offload)
2725	ret = macsec_update_offload(dev, offload);
2726	out:
2727	rtnl_unlock();
2728	return ret;
2729	}
2730
2731	static void get_tx_sa_stats(struct net_device *dev, int an,
2732	struct macsec_tx_sa *tx_sa,
2733	struct macsec_tx_sa_stats *sum)
2734	{
2735	struct macsec_dev *macsec = macsec_priv(dev);
2736	int cpu;
2737
2738	/* If h/w offloading is available, propagate to the device */
2739	if (macsec_is_offloaded(macsec)) {
2740	const struct macsec_ops *ops;
2741	struct macsec_context ctx;
2742
2743	ops = macsec_get_ops(macsec, ctx: &ctx);
2744	if (ops) {
2745	ctx.sa.assoc_num = an;
2746	ctx.sa.tx_sa = tx_sa;
2747	ctx.stats.tx_sa_stats = sum;
2748	ctx.secy = &macsec_priv(dev)->secy;
2749	macsec_offload(func: ops->mdo_get_tx_sa_stats, ctx: &ctx);
2750	}
2751	return;
2752	}
2753
2754	for_each_possible_cpu(cpu) {
2755	const struct macsec_tx_sa_stats *stats =
2756	per_cpu_ptr(tx_sa->stats, cpu);
2757
2758	sum->OutPktsProtected += stats->OutPktsProtected;
2759	sum->OutPktsEncrypted += stats->OutPktsEncrypted;
2760	}
2761	}
2762
2763	static int copy_tx_sa_stats(struct sk_buff *skb, struct macsec_tx_sa_stats *sum)
2764	{
2765	if (nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_OUT_PKTS_PROTECTED,
2766	value: sum->OutPktsProtected) ||
2767	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_OUT_PKTS_ENCRYPTED,
2768	value: sum->OutPktsEncrypted))
2769	return -EMSGSIZE;
2770
2771	return 0;
2772	}
2773
2774	static void get_rx_sa_stats(struct net_device *dev,
2775	struct macsec_rx_sc *rx_sc, int an,
2776	struct macsec_rx_sa *rx_sa,
2777	struct macsec_rx_sa_stats *sum)
2778	{
2779	struct macsec_dev *macsec = macsec_priv(dev);
2780	int cpu;
2781
2782	/* If h/w offloading is available, propagate to the device */
2783	if (macsec_is_offloaded(macsec)) {
2784	const struct macsec_ops *ops;
2785	struct macsec_context ctx;
2786
2787	ops = macsec_get_ops(macsec, ctx: &ctx);
2788	if (ops) {
2789	ctx.sa.assoc_num = an;
2790	ctx.sa.rx_sa = rx_sa;
2791	ctx.stats.rx_sa_stats = sum;
2792	ctx.secy = &macsec_priv(dev)->secy;
2793	ctx.rx_sc = rx_sc;
2794	macsec_offload(func: ops->mdo_get_rx_sa_stats, ctx: &ctx);
2795	}
2796	return;
2797	}
2798
2799	for_each_possible_cpu(cpu) {
2800	const struct macsec_rx_sa_stats *stats =
2801	per_cpu_ptr(rx_sa->stats, cpu);
2802
2803	sum->InPktsOK += stats->InPktsOK;
2804	sum->InPktsInvalid += stats->InPktsInvalid;
2805	sum->InPktsNotValid += stats->InPktsNotValid;
2806	sum->InPktsNotUsingSA += stats->InPktsNotUsingSA;
2807	sum->InPktsUnusedSA += stats->InPktsUnusedSA;
2808	}
2809	}
2810
2811	static int copy_rx_sa_stats(struct sk_buff *skb,
2812	struct macsec_rx_sa_stats *sum)
2813	{
2814	if (nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_OK, value: sum->InPktsOK) ||
2815	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_INVALID,
2816	value: sum->InPktsInvalid) ||
2817	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_NOT_VALID,
2818	value: sum->InPktsNotValid) ||
2819	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_NOT_USING_SA,
2820	value: sum->InPktsNotUsingSA) ||
2821	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_UNUSED_SA,
2822	value: sum->InPktsUnusedSA))
2823	return -EMSGSIZE;
2824
2825	return 0;
2826	}
2827
2828	static void get_rx_sc_stats(struct net_device *dev,
2829	struct macsec_rx_sc *rx_sc,
2830	struct macsec_rx_sc_stats *sum)
2831	{
2832	struct macsec_dev *macsec = macsec_priv(dev);
2833	int cpu;
2834
2835	/* If h/w offloading is available, propagate to the device */
2836	if (macsec_is_offloaded(macsec)) {
2837	const struct macsec_ops *ops;
2838	struct macsec_context ctx;
2839
2840	ops = macsec_get_ops(macsec, ctx: &ctx);
2841	if (ops) {
2842	ctx.stats.rx_sc_stats = sum;
2843	ctx.secy = &macsec_priv(dev)->secy;
2844	ctx.rx_sc = rx_sc;
2845	macsec_offload(func: ops->mdo_get_rx_sc_stats, ctx: &ctx);
2846	}
2847	return;
2848	}
2849
2850	for_each_possible_cpu(cpu) {
2851	const struct pcpu_rx_sc_stats *stats;
2852	struct macsec_rx_sc_stats tmp;
2853	unsigned int start;
2854
2855	stats = per_cpu_ptr(rx_sc->stats, cpu);
2856	do {
2857	start = u64_stats_fetch_begin(syncp: &stats->syncp);
2858	memcpy(&tmp, &stats->stats, sizeof(tmp));
2859	} while (u64_stats_fetch_retry(syncp: &stats->syncp, start));
2860
2861	sum->InOctetsValidated += tmp.InOctetsValidated;
2862	sum->InOctetsDecrypted += tmp.InOctetsDecrypted;
2863	sum->InPktsUnchecked += tmp.InPktsUnchecked;
2864	sum->InPktsDelayed += tmp.InPktsDelayed;
2865	sum->InPktsOK += tmp.InPktsOK;
2866	sum->InPktsInvalid += tmp.InPktsInvalid;
2867	sum->InPktsLate += tmp.InPktsLate;
2868	sum->InPktsNotValid += tmp.InPktsNotValid;
2869	sum->InPktsNotUsingSA += tmp.InPktsNotUsingSA;
2870	sum->InPktsUnusedSA += tmp.InPktsUnusedSA;
2871	}
2872	}
2873
2874	static int copy_rx_sc_stats(struct sk_buff *skb, struct macsec_rx_sc_stats *sum)
2875	{
2876	if (nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_OCTETS_VALIDATED,
2877	value: sum->InOctetsValidated,
2878	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2879	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_OCTETS_DECRYPTED,
2880	value: sum->InOctetsDecrypted,
2881	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2882	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_UNCHECKED,
2883	value: sum->InPktsUnchecked,
2884	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2885	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_DELAYED,
2886	value: sum->InPktsDelayed,
2887	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2888	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_OK,
2889	value: sum->InPktsOK,
2890	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2891	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_INVALID,
2892	value: sum->InPktsInvalid,
2893	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2894	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_LATE,
2895	value: sum->InPktsLate,
2896	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2897	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_NOT_VALID,
2898	value: sum->InPktsNotValid,
2899	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2900	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_NOT_USING_SA,
2901	value: sum->InPktsNotUsingSA,
2902	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2903	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_UNUSED_SA,
2904	value: sum->InPktsUnusedSA,
2905	padattr: MACSEC_RXSC_STATS_ATTR_PAD))
2906	return -EMSGSIZE;
2907
2908	return 0;
2909	}
2910
2911	static void get_tx_sc_stats(struct net_device *dev,
2912	struct macsec_tx_sc_stats *sum)
2913	{
2914	struct macsec_dev *macsec = macsec_priv(dev);
2915	int cpu;
2916
2917	/* If h/w offloading is available, propagate to the device */
2918	if (macsec_is_offloaded(macsec)) {
2919	const struct macsec_ops *ops;
2920	struct macsec_context ctx;
2921
2922	ops = macsec_get_ops(macsec, ctx: &ctx);
2923	if (ops) {
2924	ctx.stats.tx_sc_stats = sum;
2925	ctx.secy = &macsec_priv(dev)->secy;
2926	macsec_offload(func: ops->mdo_get_tx_sc_stats, ctx: &ctx);
2927	}
2928	return;
2929	}
2930
2931	for_each_possible_cpu(cpu) {
2932	const struct pcpu_tx_sc_stats *stats;
2933	struct macsec_tx_sc_stats tmp;
2934	unsigned int start;
2935
2936	stats = per_cpu_ptr(macsec_priv(dev)->secy.tx_sc.stats, cpu);
2937	do {
2938	start = u64_stats_fetch_begin(syncp: &stats->syncp);
2939	memcpy(&tmp, &stats->stats, sizeof(tmp));
2940	} while (u64_stats_fetch_retry(syncp: &stats->syncp, start));
2941
2942	sum->OutPktsProtected += tmp.OutPktsProtected;
2943	sum->OutPktsEncrypted += tmp.OutPktsEncrypted;
2944	sum->OutOctetsProtected += tmp.OutOctetsProtected;
2945	sum->OutOctetsEncrypted += tmp.OutOctetsEncrypted;
2946	}
2947	}
2948
2949	static int copy_tx_sc_stats(struct sk_buff *skb, struct macsec_tx_sc_stats *sum)
2950	{
2951	if (nla_put_u64_64bit(skb, attrtype: MACSEC_TXSC_STATS_ATTR_OUT_PKTS_PROTECTED,
2952	value: sum->OutPktsProtected,
2953	padattr: MACSEC_TXSC_STATS_ATTR_PAD) ||
2954	nla_put_u64_64bit(skb, attrtype: MACSEC_TXSC_STATS_ATTR_OUT_PKTS_ENCRYPTED,
2955	value: sum->OutPktsEncrypted,
2956	padattr: MACSEC_TXSC_STATS_ATTR_PAD) ||
2957	nla_put_u64_64bit(skb, attrtype: MACSEC_TXSC_STATS_ATTR_OUT_OCTETS_PROTECTED,
2958	value: sum->OutOctetsProtected,
2959	padattr: MACSEC_TXSC_STATS_ATTR_PAD) ||
2960	nla_put_u64_64bit(skb, attrtype: MACSEC_TXSC_STATS_ATTR_OUT_OCTETS_ENCRYPTED,
2961	value: sum->OutOctetsEncrypted,
2962	padattr: MACSEC_TXSC_STATS_ATTR_PAD))
2963	return -EMSGSIZE;
2964
2965	return 0;
2966	}
2967
2968	static void get_secy_stats(struct net_device *dev, struct macsec_dev_stats *sum)
2969	{
2970	struct macsec_dev *macsec = macsec_priv(dev);
2971	int cpu;
2972
2973	/* If h/w offloading is available, propagate to the device */
2974	if (macsec_is_offloaded(macsec)) {
2975	const struct macsec_ops *ops;
2976	struct macsec_context ctx;
2977
2978	ops = macsec_get_ops(macsec, ctx: &ctx);
2979	if (ops) {
2980	ctx.stats.dev_stats = sum;
2981	ctx.secy = &macsec_priv(dev)->secy;
2982	macsec_offload(func: ops->mdo_get_dev_stats, ctx: &ctx);
2983	}
2984	return;
2985	}
2986
2987	for_each_possible_cpu(cpu) {
2988	const struct pcpu_secy_stats *stats;
2989	struct macsec_dev_stats tmp;
2990	unsigned int start;
2991
2992	stats = per_cpu_ptr(macsec_priv(dev)->stats, cpu);
2993	do {
2994	start = u64_stats_fetch_begin(syncp: &stats->syncp);
2995	memcpy(&tmp, &stats->stats, sizeof(tmp));
2996	} while (u64_stats_fetch_retry(syncp: &stats->syncp, start));
2997
2998	sum->OutPktsUntagged += tmp.OutPktsUntagged;
2999	sum->InPktsUntagged += tmp.InPktsUntagged;
3000	sum->OutPktsTooLong += tmp.OutPktsTooLong;
3001	sum->InPktsNoTag += tmp.InPktsNoTag;
3002	sum->InPktsBadTag += tmp.InPktsBadTag;
3003	sum->InPktsUnknownSCI += tmp.InPktsUnknownSCI;
3004	sum->InPktsNoSCI += tmp.InPktsNoSCI;
3005	sum->InPktsOverrun += tmp.InPktsOverrun;
3006	}
3007	}
3008
3009	static int copy_secy_stats(struct sk_buff *skb, struct macsec_dev_stats *sum)
3010	{
3011	if (nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_OUT_PKTS_UNTAGGED,
3012	value: sum->OutPktsUntagged,
3013	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
3014	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_UNTAGGED,
3015	value: sum->InPktsUntagged,
3016	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
3017	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_OUT_PKTS_TOO_LONG,
3018	value: sum->OutPktsTooLong,
3019	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
3020	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_NO_TAG,
3021	value: sum->InPktsNoTag,
3022	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
3023	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_BAD_TAG,
3024	value: sum->InPktsBadTag,
3025	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
3026	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_UNKNOWN_SCI,
3027	value: sum->InPktsUnknownSCI,
3028	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
3029	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_NO_SCI,
3030	value: sum->InPktsNoSCI,
3031	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
3032	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_OVERRUN,
3033	value: sum->InPktsOverrun,
3034	padattr: MACSEC_SECY_STATS_ATTR_PAD))
3035	return -EMSGSIZE;
3036
3037	return 0;
3038	}
3039
3040	static int nla_put_secy(struct macsec_secy *secy, struct sk_buff *skb)
3041	{
3042	struct macsec_tx_sc *tx_sc = &secy->tx_sc;
3043	struct nlattr *secy_nest = nla_nest_start_noflag(skb,
3044	attrtype: MACSEC_ATTR_SECY);
3045	u64 csid;
3046
3047	if (!secy_nest)
3048	return 1;
3049
3050	switch (secy->key_len) {
3051	case MACSEC_GCM_AES_128_SAK_LEN:
3052	csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_128 : MACSEC_DEFAULT_CIPHER_ID;
3053	break;
3054	case MACSEC_GCM_AES_256_SAK_LEN:
3055	csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_256 : MACSEC_CIPHER_ID_GCM_AES_256;
3056	break;
3057	default:
3058	goto cancel;
3059	}
3060
3061	if (nla_put_sci(skb, attrtype: MACSEC_SECY_ATTR_SCI, value: secy->sci,
3062	padattr: MACSEC_SECY_ATTR_PAD) ||
3063	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_ATTR_CIPHER_SUITE,
3064	value: csid, padattr: MACSEC_SECY_ATTR_PAD) ||
3065	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_ICV_LEN, value: secy->icv_len) ||
3066	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_OPER, value: secy->operational) ||
3067	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_PROTECT, value: secy->protect_frames) ||
3068	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_REPLAY, value: secy->replay_protect) ||
3069	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_VALIDATE, value: secy->validate_frames) ||
3070	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_ENCRYPT, value: tx_sc->encrypt) ||
3071	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_INC_SCI, value: tx_sc->send_sci) ||
3072	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_ES, value: tx_sc->end_station) ||
3073	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_SCB, value: tx_sc->scb) ||
3074	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_ENCODING_SA, value: tx_sc->encoding_sa))
3075	goto cancel;
3076
3077	if (secy->replay_protect) {
3078	if (nla_put_u32(skb, attrtype: MACSEC_SECY_ATTR_WINDOW, value: secy->replay_window))
3079	goto cancel;
3080	}
3081
3082	nla_nest_end(skb, start: secy_nest);
3083	return 0;
3084
3085	cancel:
3086	nla_nest_cancel(skb, start: secy_nest);
3087	return 1;
3088	}
3089
3090	static noinline_for_stack int
3091	dump_secy(struct macsec_secy *secy, struct net_device *dev,
3092	struct sk_buff *skb, struct netlink_callback *cb)
3093	{
3094	struct macsec_tx_sc_stats tx_sc_stats = {0, };
3095	struct macsec_tx_sa_stats tx_sa_stats = {0, };
3096	struct macsec_rx_sc_stats rx_sc_stats = {0, };
3097	struct macsec_rx_sa_stats rx_sa_stats = {0, };
3098	struct macsec_dev *macsec = netdev_priv(dev);
3099	struct macsec_dev_stats dev_stats = {0, };
3100	struct macsec_tx_sc *tx_sc = &secy->tx_sc;
3101	struct nlattr *txsa_list, *rxsc_list;
3102	struct macsec_rx_sc *rx_sc;
3103	struct nlattr *attr;
3104	void *hdr;
3105	int i, j;
3106
3107	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, seq: cb->nlh->nlmsg_seq,
3108	family: &macsec_fam, NLM_F_MULTI, cmd: MACSEC_CMD_GET_TXSC);
3109	if (!hdr)
3110	return -EMSGSIZE;
3111
3112	genl_dump_check_consistent(cb, user_hdr: hdr);
3113
3114	if (nla_put_u32(skb, attrtype: MACSEC_ATTR_IFINDEX, value: dev->ifindex))
3115	goto nla_put_failure;
3116
3117	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_OFFLOAD);
3118	if (!attr)
3119	goto nla_put_failure;
3120	if (nla_put_u8(skb, attrtype: MACSEC_OFFLOAD_ATTR_TYPE, value: macsec->offload))
3121	goto nla_put_failure;
3122	nla_nest_end(skb, start: attr);
3123
3124	if (nla_put_secy(secy, skb))
3125	goto nla_put_failure;
3126
3127	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_TXSC_STATS);
3128	if (!attr)
3129	goto nla_put_failure;
3130
3131	get_tx_sc_stats(dev, sum: &tx_sc_stats);
3132	if (copy_tx_sc_stats(skb, sum: &tx_sc_stats)) {
3133	nla_nest_cancel(skb, start: attr);
3134	goto nla_put_failure;
3135	}
3136	nla_nest_end(skb, start: attr);
3137
3138	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_SECY_STATS);
3139	if (!attr)
3140	goto nla_put_failure;
3141	get_secy_stats(dev, sum: &dev_stats);
3142	if (copy_secy_stats(skb, sum: &dev_stats)) {
3143	nla_nest_cancel(skb, start: attr);
3144	goto nla_put_failure;
3145	}
3146	nla_nest_end(skb, start: attr);
3147
3148	txsa_list = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_TXSA_LIST);
3149	if (!txsa_list)
3150	goto nla_put_failure;
3151	for (i = 0, j = 1; i < MACSEC_NUM_AN; i++) {
3152	struct macsec_tx_sa *tx_sa = rtnl_dereference(tx_sc->sa[i]);
3153	struct nlattr *txsa_nest;
3154	u64 pn;
3155	int pn_len;
3156
3157	if (!tx_sa)
3158	continue;
3159
3160	txsa_nest = nla_nest_start_noflag(skb, attrtype: j++);
3161	if (!txsa_nest) {
3162	nla_nest_cancel(skb, start: txsa_list);
3163	goto nla_put_failure;
3164	}
3165
3166	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_SA_ATTR_STATS);
3167	if (!attr) {
3168	nla_nest_cancel(skb, start: txsa_nest);
3169	nla_nest_cancel(skb, start: txsa_list);
3170	goto nla_put_failure;
3171	}
3172	memset(&tx_sa_stats, 0, sizeof(tx_sa_stats));
3173	get_tx_sa_stats(dev, an: i, tx_sa, sum: &tx_sa_stats);
3174	if (copy_tx_sa_stats(skb, sum: &tx_sa_stats)) {
3175	nla_nest_cancel(skb, start: attr);
3176	nla_nest_cancel(skb, start: txsa_nest);
3177	nla_nest_cancel(skb, start: txsa_list);
3178	goto nla_put_failure;
3179	}
3180	nla_nest_end(skb, start: attr);
3181
3182	if (secy->xpn) {
3183	pn = tx_sa->next_pn;
3184	pn_len = MACSEC_XPN_PN_LEN;
3185	} else {
3186	pn = tx_sa->next_pn_halves.lower;
3187	pn_len = MACSEC_DEFAULT_PN_LEN;
3188	}
3189
3190	if (nla_put_u8(skb, attrtype: MACSEC_SA_ATTR_AN, value: i) ||
3191	nla_put(skb, attrtype: MACSEC_SA_ATTR_PN, attrlen: pn_len, data: &pn) ||
3192	nla_put(skb, attrtype: MACSEC_SA_ATTR_KEYID, MACSEC_KEYID_LEN, data: tx_sa->key.id) ||
3193	(secy->xpn && nla_put_ssci(skb, attrtype: MACSEC_SA_ATTR_SSCI, value: tx_sa->ssci)) ||
3194	nla_put_u8(skb, attrtype: MACSEC_SA_ATTR_ACTIVE, value: tx_sa->active)) {
3195	nla_nest_cancel(skb, start: txsa_nest);
3196	nla_nest_cancel(skb, start: txsa_list);
3197	goto nla_put_failure;
3198	}
3199
3200	nla_nest_end(skb, start: txsa_nest);
3201	}
3202	nla_nest_end(skb, start: txsa_list);
3203
3204	rxsc_list = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_RXSC_LIST);
3205	if (!rxsc_list)
3206	goto nla_put_failure;
3207
3208	j = 1;
3209	for_each_rxsc_rtnl(secy, rx_sc) {
3210	int k;
3211	struct nlattr *rxsa_list;
3212	struct nlattr *rxsc_nest = nla_nest_start_noflag(skb, attrtype: j++);
3213
3214	if (!rxsc_nest) {
3215	nla_nest_cancel(skb, start: rxsc_list);
3216	goto nla_put_failure;
3217	}
3218
3219	if (nla_put_u8(skb, attrtype: MACSEC_RXSC_ATTR_ACTIVE, value: rx_sc->active) ||
3220	nla_put_sci(skb, attrtype: MACSEC_RXSC_ATTR_SCI, value: rx_sc->sci,
3221	padattr: MACSEC_RXSC_ATTR_PAD)) {
3222	nla_nest_cancel(skb, start: rxsc_nest);
3223	nla_nest_cancel(skb, start: rxsc_list);
3224	goto nla_put_failure;
3225	}
3226
3227	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_RXSC_ATTR_STATS);
3228	if (!attr) {
3229	nla_nest_cancel(skb, start: rxsc_nest);
3230	nla_nest_cancel(skb, start: rxsc_list);
3231	goto nla_put_failure;
3232	}
3233	memset(&rx_sc_stats, 0, sizeof(rx_sc_stats));
3234	get_rx_sc_stats(dev, rx_sc, sum: &rx_sc_stats);
3235	if (copy_rx_sc_stats(skb, sum: &rx_sc_stats)) {
3236	nla_nest_cancel(skb, start: attr);
3237	nla_nest_cancel(skb, start: rxsc_nest);
3238	nla_nest_cancel(skb, start: rxsc_list);
3239	goto nla_put_failure;
3240	}
3241	nla_nest_end(skb, start: attr);
3242
3243	rxsa_list = nla_nest_start_noflag(skb,
3244	attrtype: MACSEC_RXSC_ATTR_SA_LIST);
3245	if (!rxsa_list) {
3246	nla_nest_cancel(skb, start: rxsc_nest);
3247	nla_nest_cancel(skb, start: rxsc_list);
3248	goto nla_put_failure;
3249	}
3250
3251	for (i = 0, k = 1; i < MACSEC_NUM_AN; i++) {
3252	struct macsec_rx_sa *rx_sa = rtnl_dereference(rx_sc->sa[i]);
3253	struct nlattr *rxsa_nest;
3254	u64 pn;
3255	int pn_len;
3256
3257	if (!rx_sa)
3258	continue;
3259
3260	rxsa_nest = nla_nest_start_noflag(skb, attrtype: k++);
3261	if (!rxsa_nest) {
3262	nla_nest_cancel(skb, start: rxsa_list);
3263	nla_nest_cancel(skb, start: rxsc_nest);
3264	nla_nest_cancel(skb, start: rxsc_list);
3265	goto nla_put_failure;
3266	}
3267
3268	attr = nla_nest_start_noflag(skb,
3269	attrtype: MACSEC_SA_ATTR_STATS);
3270	if (!attr) {
3271	nla_nest_cancel(skb, start: rxsa_list);
3272	nla_nest_cancel(skb, start: rxsc_nest);
3273	nla_nest_cancel(skb, start: rxsc_list);
3274	goto nla_put_failure;
3275	}
3276	memset(&rx_sa_stats, 0, sizeof(rx_sa_stats));
3277	get_rx_sa_stats(dev, rx_sc, an: i, rx_sa, sum: &rx_sa_stats);
3278	if (copy_rx_sa_stats(skb, sum: &rx_sa_stats)) {
3279	nla_nest_cancel(skb, start: attr);
3280	nla_nest_cancel(skb, start: rxsa_list);
3281	nla_nest_cancel(skb, start: rxsc_nest);
3282	nla_nest_cancel(skb, start: rxsc_list);
3283	goto nla_put_failure;
3284	}
3285	nla_nest_end(skb, start: attr);
3286
3287	if (secy->xpn) {
3288	pn = rx_sa->next_pn;
3289	pn_len = MACSEC_XPN_PN_LEN;
3290	} else {
3291	pn = rx_sa->next_pn_halves.lower;
3292	pn_len = MACSEC_DEFAULT_PN_LEN;
3293	}
3294
3295	if (nla_put_u8(skb, attrtype: MACSEC_SA_ATTR_AN, value: i) ||
3296	nla_put(skb, attrtype: MACSEC_SA_ATTR_PN, attrlen: pn_len, data: &pn) ||
3297	nla_put(skb, attrtype: MACSEC_SA_ATTR_KEYID, MACSEC_KEYID_LEN, data: rx_sa->key.id) ||
3298	(secy->xpn && nla_put_ssci(skb, attrtype: MACSEC_SA_ATTR_SSCI, value: rx_sa->ssci)) ||
3299	nla_put_u8(skb, attrtype: MACSEC_SA_ATTR_ACTIVE, value: rx_sa->active)) {
3300	nla_nest_cancel(skb, start: rxsa_nest);
3301	nla_nest_cancel(skb, start: rxsc_nest);
3302	nla_nest_cancel(skb, start: rxsc_list);
3303	goto nla_put_failure;
3304	}
3305	nla_nest_end(skb, start: rxsa_nest);
3306	}
3307
3308	nla_nest_end(skb, start: rxsa_list);
3309	nla_nest_end(skb, start: rxsc_nest);
3310	}
3311
3312	nla_nest_end(skb, start: rxsc_list);
3313
3314	genlmsg_end(skb, hdr);
3315
3316	return 0;
3317
3318	nla_put_failure:
3319	genlmsg_cancel(skb, hdr);
3320	return -EMSGSIZE;
3321	}
3322
3323	static int macsec_generation = 1; /* protected by RTNL */
3324
3325	static int macsec_dump_txsc(struct sk_buff *skb, struct netlink_callback *cb)
3326	{
3327	struct net *net = sock_net(sk: skb->sk);
3328	struct net_device *dev;
3329	int dev_idx, d;
3330
3331	dev_idx = cb->args[0];
3332
3333	d = 0;
3334	rtnl_lock();
3335
3336	cb->seq = macsec_generation;
3337
3338	for_each_netdev(net, dev) {
3339	struct macsec_secy *secy;
3340
3341	if (d < dev_idx)
3342	goto next;
3343
3344	if (!netif_is_macsec(dev))
3345	goto next;
3346
3347	secy = &macsec_priv(dev)->secy;
3348	if (dump_secy(secy, dev, skb, cb) < 0)
3349	goto done;
3350	next:
3351	d++;
3352	}
3353
3354	done:
3355	rtnl_unlock();
3356	cb->args[0] = d;
3357	return skb->len;
3358	}
3359
3360	static const struct genl_small_ops macsec_genl_ops[] = {
3361	{
3362	.cmd = MACSEC_CMD_GET_TXSC,
3363	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3364	.dumpit = macsec_dump_txsc,
3365	},
3366	{
3367	.cmd = MACSEC_CMD_ADD_RXSC,
3368	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3369	.doit = macsec_add_rxsc,
3370	.flags = GENL_ADMIN_PERM,
3371	},
3372	{
3373	.cmd = MACSEC_CMD_DEL_RXSC,
3374	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3375	.doit = macsec_del_rxsc,
3376	.flags = GENL_ADMIN_PERM,
3377	},
3378	{
3379	.cmd = MACSEC_CMD_UPD_RXSC,
3380	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3381	.doit = macsec_upd_rxsc,
3382	.flags = GENL_ADMIN_PERM,
3383	},
3384	{
3385	.cmd = MACSEC_CMD_ADD_TXSA,
3386	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3387	.doit = macsec_add_txsa,
3388	.flags = GENL_ADMIN_PERM,
3389	},
3390	{
3391	.cmd = MACSEC_CMD_DEL_TXSA,
3392	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3393	.doit = macsec_del_txsa,
3394	.flags = GENL_ADMIN_PERM,
3395	},
3396	{
3397	.cmd = MACSEC_CMD_UPD_TXSA,
3398	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3399	.doit = macsec_upd_txsa,
3400	.flags = GENL_ADMIN_PERM,
3401	},
3402	{
3403	.cmd = MACSEC_CMD_ADD_RXSA,
3404	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3405	.doit = macsec_add_rxsa,
3406	.flags = GENL_ADMIN_PERM,
3407	},
3408	{
3409	.cmd = MACSEC_CMD_DEL_RXSA,
3410	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3411	.doit = macsec_del_rxsa,
3412	.flags = GENL_ADMIN_PERM,
3413	},
3414	{
3415	.cmd = MACSEC_CMD_UPD_RXSA,
3416	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3417	.doit = macsec_upd_rxsa,
3418	.flags = GENL_ADMIN_PERM,
3419	},
3420	{
3421	.cmd = MACSEC_CMD_UPD_OFFLOAD,
3422	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3423	.doit = macsec_upd_offload,
3424	.flags = GENL_ADMIN_PERM,
3425	},
3426	};
3427
3428	static struct genl_family macsec_fam __ro_after_init = {
3429	.name = MACSEC_GENL_NAME,
3430	.hdrsize = 0,
3431	.version = MACSEC_GENL_VERSION,
3432	.maxattr = MACSEC_ATTR_MAX,
3433	.policy = macsec_genl_policy,
3434	.netnsok = true,
3435	.module = THIS_MODULE,
3436	.small_ops = macsec_genl_ops,
3437	.n_small_ops = ARRAY_SIZE(macsec_genl_ops),
3438	.resv_start_op = MACSEC_CMD_UPD_OFFLOAD + 1,
3439	};
3440
3441	static struct sk_buff *macsec_insert_tx_tag(struct sk_buff *skb,
3442	struct net_device *dev)
3443	{
3444	struct macsec_dev *macsec = macsec_priv(dev);
3445	const struct macsec_ops *ops;
3446	struct phy_device *phydev;
3447	struct macsec_context ctx;
3448	int skb_final_len;
3449	int err;
3450
3451	ops = macsec_get_ops(macsec, ctx: &ctx);
3452	skb_final_len = skb->len - ETH_HLEN + ops->needed_headroom +
3453	ops->needed_tailroom;
3454	if (unlikely(skb_final_len > macsec->real_dev->mtu)) {
3455	err = -EINVAL;
3456	goto cleanup;
3457	}
3458
3459	phydev = macsec->real_dev->phydev;
3460
3461	err = skb_ensure_writable_head_tail(skb, dev);
3462	if (unlikely(err < 0))
3463	goto cleanup;
3464
3465	err = ops->mdo_insert_tx_tag(phydev, skb);
3466	if (unlikely(err))
3467	goto cleanup;
3468
3469	return skb;
3470	cleanup:
3471	kfree_skb(skb);
3472	return ERR_PTR(error: err);
3473	}
3474
3475	static netdev_tx_t macsec_start_xmit(struct sk_buff *skb,
3476	struct net_device *dev)
3477	{
3478	struct macsec_dev *macsec = netdev_priv(dev);
3479	struct macsec_secy *secy = &macsec->secy;
3480	struct pcpu_secy_stats *secy_stats;
3481	int ret, len;
3482
3483	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
3484	struct metadata_dst *md_dst = secy->tx_sc.md_dst;
3485
3486	skb_dst_drop(skb);
3487	dst_hold(dst: &md_dst->dst);
3488	skb_dst_set(skb, dst: &md_dst->dst);
3489
3490	if (macsec->insert_tx_tag) {
3491	skb = macsec_insert_tx_tag(skb, dev);
3492	if (IS_ERR(ptr: skb)) {
3493	DEV_STATS_INC(dev, tx_dropped);
3494	return NETDEV_TX_OK;
3495	}
3496	}
3497
3498	skb->dev = macsec->real_dev;
3499	return dev_queue_xmit(skb);
3500	}
3501
3502	/* 10.5 */
3503	if (!secy->protect_frames) {
3504	secy_stats = this_cpu_ptr(macsec->stats);
3505	u64_stats_update_begin(syncp: &secy_stats->syncp);
3506	secy_stats->stats.OutPktsUntagged++;
3507	u64_stats_update_end(syncp: &secy_stats->syncp);
3508	skb->dev = macsec->real_dev;
3509	len = skb->len;
3510	ret = dev_queue_xmit(skb);
3511	count_tx(dev, ret, len);
3512	return ret;
3513	}
3514
3515	if (!secy->operational) {
3516	kfree_skb(skb);
3517	DEV_STATS_INC(dev, tx_dropped);
3518	return NETDEV_TX_OK;
3519	}
3520
3521	len = skb->len;
3522	skb = macsec_encrypt(skb, dev);
3523	if (IS_ERR(ptr: skb)) {
3524	if (PTR_ERR(ptr: skb) != -EINPROGRESS)
3525	DEV_STATS_INC(dev, tx_dropped);
3526	return NETDEV_TX_OK;
3527	}
3528
3529	macsec_count_tx(skb, tx_sc: &macsec->secy.tx_sc, tx_sa: macsec_skb_cb(skb)->tx_sa);
3530
3531	macsec_encrypt_finish(skb, dev);
3532	ret = dev_queue_xmit(skb);
3533	count_tx(dev, ret, len);
3534	return ret;
3535	}
3536
3537	#define MACSEC_FEATURES \
3538	(NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST)
3539
3540	#define MACSEC_OFFLOAD_FEATURES \
3541	(MACSEC_FEATURES | NETIF_F_GSO_SOFTWARE | NETIF_F_SOFT_FEATURES | \
3542	NETIF_F_LRO | NETIF_F_RXHASH | NETIF_F_CSUM_MASK | NETIF_F_RXCSUM)
3543
3544	static int macsec_dev_init(struct net_device *dev)
3545	{
3546	struct macsec_dev *macsec = macsec_priv(dev);
3547	struct net_device *real_dev = macsec->real_dev;
3548	int err;
3549
3550	err = gro_cells_init(gcells: &macsec->gro_cells, dev);
3551	if (err)
3552	return err;
3553
3554	macsec_inherit_tso_max(dev);
3555
3556	dev->hw_features = real_dev->hw_features & MACSEC_OFFLOAD_FEATURES;
3557	dev->hw_features |= NETIF_F_GSO_SOFTWARE;
3558
3559	dev->features = real_dev->features & MACSEC_OFFLOAD_FEATURES;
3560	dev->features |= NETIF_F_GSO_SOFTWARE;
3561	dev->lltx = true;
3562	dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
3563
3564	macsec_set_head_tail_room(dev);
3565
3566	if (is_zero_ether_addr(addr: dev->dev_addr))
3567	eth_hw_addr_inherit(dst: dev, src: real_dev);
3568	if (is_zero_ether_addr(addr: dev->broadcast))
3569	memcpy(dev->broadcast, real_dev->broadcast, dev->addr_len);
3570
3571	/* Get macsec's reference to real_dev */
3572	netdev_hold(dev: real_dev, tracker: &macsec->dev_tracker, GFP_KERNEL);
3573
3574	return 0;
3575	}
3576
3577	static void macsec_dev_uninit(struct net_device *dev)
3578	{
3579	struct macsec_dev *macsec = macsec_priv(dev);
3580
3581	gro_cells_destroy(gcells: &macsec->gro_cells);
3582	}
3583
3584	static netdev_features_t macsec_fix_features(struct net_device *dev,
3585	netdev_features_t features)
3586	{
3587	struct macsec_dev *macsec = macsec_priv(dev);
3588	struct net_device *real_dev = macsec->real_dev;
3589	netdev_features_t mask;
3590
3591	mask = macsec_is_offloaded(macsec) ? MACSEC_OFFLOAD_FEATURES
3592	: MACSEC_FEATURES;
3593
3594	features &= (real_dev->features & mask) |
3595	NETIF_F_GSO_SOFTWARE | NETIF_F_SOFT_FEATURES;
3596
3597	return features;
3598	}
3599
3600	static int macsec_dev_open(struct net_device *dev)
3601	{
3602	struct macsec_dev *macsec = macsec_priv(dev);
3603	struct net_device *real_dev = macsec->real_dev;
3604	int err;
3605
3606	err = dev_uc_add(dev: real_dev, addr: dev->dev_addr);
3607	if (err < 0)
3608	return err;
3609
3610	if (dev->flags & IFF_ALLMULTI) {
3611	err = dev_set_allmulti(dev: real_dev, inc: 1);
3612	if (err < 0)
3613	goto del_unicast;
3614	}
3615
3616	if (dev->flags & IFF_PROMISC) {
3617	err = dev_set_promiscuity(dev: real_dev, inc: 1);
3618	if (err < 0)
3619	goto clear_allmulti;
3620	}
3621
3622	/* If h/w offloading is available, propagate to the device */
3623	if (macsec_is_offloaded(macsec)) {
3624	const struct macsec_ops *ops;
3625	struct macsec_context ctx;
3626
3627	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
3628	if (!ops) {
3629	err = -EOPNOTSUPP;
3630	goto clear_allmulti;
3631	}
3632
3633	ctx.secy = &macsec->secy;
3634	err = macsec_offload(func: ops->mdo_dev_open, ctx: &ctx);
3635	if (err)
3636	goto clear_allmulti;
3637	}
3638
3639	if (netif_carrier_ok(dev: real_dev))
3640	netif_carrier_on(dev);
3641
3642	return 0;
3643	clear_allmulti:
3644	if (dev->flags & IFF_ALLMULTI)
3645	dev_set_allmulti(dev: real_dev, inc: -1);
3646	del_unicast:
3647	dev_uc_del(dev: real_dev, addr: dev->dev_addr);
3648	netif_carrier_off(dev);
3649	return err;
3650	}
3651
3652	static int macsec_dev_stop(struct net_device *dev)
3653	{
3654	struct macsec_dev *macsec = macsec_priv(dev);
3655	struct net_device *real_dev = macsec->real_dev;
3656
3657	netif_carrier_off(dev);
3658
3659	/* If h/w offloading is available, propagate to the device */
3660	if (macsec_is_offloaded(macsec)) {
3661	const struct macsec_ops *ops;
3662	struct macsec_context ctx;
3663
3664	ops = macsec_get_ops(macsec, ctx: &ctx);
3665	if (ops) {
3666	ctx.secy = &macsec->secy;
3667	macsec_offload(func: ops->mdo_dev_stop, ctx: &ctx);
3668	}
3669	}
3670
3671	dev_mc_unsync(to: real_dev, from: dev);
3672	dev_uc_unsync(to: real_dev, from: dev);
3673
3674	if (dev->flags & IFF_ALLMULTI)
3675	dev_set_allmulti(dev: real_dev, inc: -1);
3676
3677	if (dev->flags & IFF_PROMISC)
3678	dev_set_promiscuity(dev: real_dev, inc: -1);
3679
3680	dev_uc_del(dev: real_dev, addr: dev->dev_addr);
3681
3682	return 0;
3683	}
3684
3685	static void macsec_dev_change_rx_flags(struct net_device *dev, int change)
3686	{
3687	struct net_device *real_dev = macsec_priv(dev)->real_dev;
3688
3689	if (!(dev->flags & IFF_UP))
3690	return;
3691
3692	if (change & IFF_ALLMULTI)
3693	dev_set_allmulti(dev: real_dev, inc: dev->flags & IFF_ALLMULTI ? 1 : -1);
3694
3695	if (change & IFF_PROMISC)
3696	dev_set_promiscuity(dev: real_dev,
3697	inc: dev->flags & IFF_PROMISC ? 1 : -1);
3698	}
3699
3700	static void macsec_dev_set_rx_mode(struct net_device *dev)
3701	{
3702	struct net_device *real_dev = macsec_priv(dev)->real_dev;
3703
3704	dev_mc_sync(to: real_dev, from: dev);
3705	dev_uc_sync(to: real_dev, from: dev);
3706	}
3707
3708	static int macsec_set_mac_address(struct net_device *dev, void *p)
3709	{
3710	struct macsec_dev *macsec = macsec_priv(dev);
3711	struct net_device *real_dev = macsec->real_dev;
3712	struct sockaddr *addr = p;
3713	u8 old_addr[ETH_ALEN];
3714	int err;
3715
3716	if (!is_valid_ether_addr(addr: addr->sa_data))
3717	return -EADDRNOTAVAIL;
3718
3719	if (dev->flags & IFF_UP) {
3720	err = dev_uc_add(dev: real_dev, addr: addr->sa_data);
3721	if (err < 0)
3722	return err;
3723	}
3724
3725	ether_addr_copy(dst: old_addr, src: dev->dev_addr);
3726	eth_hw_addr_set(dev, addr: addr->sa_data);
3727
3728	/* If h/w offloading is available, propagate to the device */
3729	if (macsec_is_offloaded(macsec)) {
3730	const struct macsec_ops *ops;
3731	struct macsec_context ctx;
3732
3733	ops = macsec_get_ops(macsec, ctx: &ctx);
3734	if (!ops) {
3735	err = -EOPNOTSUPP;
3736	goto restore_old_addr;
3737	}
3738
3739	ctx.secy = &macsec->secy;
3740	err = macsec_offload(func: ops->mdo_upd_secy, ctx: &ctx);
3741	if (err)
3742	goto restore_old_addr;
3743	}
3744
3745	if (dev->flags & IFF_UP)
3746	dev_uc_del(dev: real_dev, addr: old_addr);
3747
3748	return 0;
3749
3750	restore_old_addr:
3751	if (dev->flags & IFF_UP)
3752	dev_uc_del(dev: real_dev, addr: addr->sa_data);
3753
3754	eth_hw_addr_set(dev, addr: old_addr);
3755
3756	return err;
3757	}
3758
3759	static int macsec_change_mtu(struct net_device *dev, int new_mtu)
3760	{
3761	struct macsec_dev *macsec = macsec_priv(dev);
3762	unsigned int extra = macsec->secy.icv_len + macsec_extra_len(sci_present: true);
3763
3764	if (macsec->real_dev->mtu - extra < new_mtu)
3765	return -ERANGE;
3766
3767	WRITE_ONCE(dev->mtu, new_mtu);
3768
3769	return 0;
3770	}
3771
3772	static void macsec_get_stats64(struct net_device *dev,
3773	struct rtnl_link_stats64 *s)
3774	{
3775	if (!dev->tstats)
3776	return;
3777
3778	dev_fetch_sw_netstats(s, netstats: dev->tstats);
3779
3780	s->rx_dropped = DEV_STATS_READ(dev, rx_dropped);
3781	s->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
3782	s->rx_errors = DEV_STATS_READ(dev, rx_errors);
3783	}
3784
3785	static int macsec_get_iflink(const struct net_device *dev)
3786	{
3787	return READ_ONCE(macsec_priv(dev)->real_dev->ifindex);
3788	}
3789
3790	static const struct net_device_ops macsec_netdev_ops = {
3791	.ndo_init = macsec_dev_init,
3792	.ndo_uninit = macsec_dev_uninit,
3793	.ndo_open = macsec_dev_open,
3794	.ndo_stop = macsec_dev_stop,
3795	.ndo_fix_features = macsec_fix_features,
3796	.ndo_change_mtu = macsec_change_mtu,
3797	.ndo_set_rx_mode = macsec_dev_set_rx_mode,
3798	.ndo_change_rx_flags = macsec_dev_change_rx_flags,
3799	.ndo_set_mac_address = macsec_set_mac_address,
3800	.ndo_start_xmit = macsec_start_xmit,
3801	.ndo_get_stats64 = macsec_get_stats64,
3802	.ndo_get_iflink = macsec_get_iflink,
3803	};
3804
3805	static const struct device_type macsec_type = {
3806	.name = "macsec",
3807	};
3808
3809	static const struct nla_policy macsec_rtnl_policy[IFLA_MACSEC_MAX + 1] = {
3810	[IFLA_MACSEC_SCI] = { .type = NLA_U64 },
3811	[IFLA_MACSEC_PORT] = { .type = NLA_U16 },
3812	[IFLA_MACSEC_ICV_LEN] = { .type = NLA_U8 },
3813	[IFLA_MACSEC_CIPHER_SUITE] = { .type = NLA_U64 },
3814	[IFLA_MACSEC_WINDOW] = { .type = NLA_U32 },
3815	[IFLA_MACSEC_ENCODING_SA] = { .type = NLA_U8 },
3816	[IFLA_MACSEC_ENCRYPT] = { .type = NLA_U8 },
3817	[IFLA_MACSEC_PROTECT] = { .type = NLA_U8 },
3818	[IFLA_MACSEC_INC_SCI] = { .type = NLA_U8 },
3819	[IFLA_MACSEC_ES] = { .type = NLA_U8 },
3820	[IFLA_MACSEC_SCB] = { .type = NLA_U8 },
3821	[IFLA_MACSEC_REPLAY_PROTECT] = { .type = NLA_U8 },
3822	[IFLA_MACSEC_VALIDATION] = { .type = NLA_U8 },
3823	[IFLA_MACSEC_OFFLOAD] = { .type = NLA_U8 },
3824	};
3825
3826	static void macsec_free_netdev(struct net_device *dev)
3827	{
3828	struct macsec_dev *macsec = macsec_priv(dev);
3829
3830	dst_release(dst: &macsec->secy.tx_sc.md_dst->dst);
3831	free_percpu(pdata: macsec->stats);
3832	free_percpu(pdata: macsec->secy.tx_sc.stats);
3833
3834	/* Get rid of the macsec's reference to real_dev */
3835	netdev_put(dev: macsec->real_dev, tracker: &macsec->dev_tracker);
3836	}
3837
3838	static void macsec_setup(struct net_device *dev)
3839	{
3840	ether_setup(dev);
3841	dev->min_mtu = 0;
3842	dev->max_mtu = ETH_MAX_MTU;
3843	dev->priv_flags |= IFF_NO_QUEUE;
3844	dev->netdev_ops = &macsec_netdev_ops;
3845	dev->needs_free_netdev = true;
3846	dev->priv_destructor = macsec_free_netdev;
3847	SET_NETDEV_DEVTYPE(dev, &macsec_type);
3848
3849	eth_zero_addr(addr: dev->broadcast);
3850	}
3851
3852	static int macsec_changelink_common(struct net_device *dev,
3853	struct nlattr *data[])
3854	{
3855	struct macsec_secy *secy;
3856	struct macsec_tx_sc *tx_sc;
3857
3858	secy = &macsec_priv(dev)->secy;
3859	tx_sc = &secy->tx_sc;
3860
3861	if (data[IFLA_MACSEC_ENCODING_SA]) {
3862	struct macsec_tx_sa *tx_sa;
3863
3864	tx_sc->encoding_sa = nla_get_u8(nla: data[IFLA_MACSEC_ENCODING_SA]);
3865	tx_sa = rtnl_dereference(tx_sc->sa[tx_sc->encoding_sa]);
3866
3867	secy->operational = tx_sa && tx_sa->active;
3868	}
3869
3870	if (data[IFLA_MACSEC_ENCRYPT])
3871	tx_sc->encrypt = !!nla_get_u8(nla: data[IFLA_MACSEC_ENCRYPT]);
3872
3873	if (data[IFLA_MACSEC_PROTECT])
3874	secy->protect_frames = !!nla_get_u8(nla: data[IFLA_MACSEC_PROTECT]);
3875
3876	if (data[IFLA_MACSEC_INC_SCI])
3877	tx_sc->send_sci = !!nla_get_u8(nla: data[IFLA_MACSEC_INC_SCI]);
3878
3879	if (data[IFLA_MACSEC_ES])
3880	tx_sc->end_station = !!nla_get_u8(nla: data[IFLA_MACSEC_ES]);
3881
3882	if (data[IFLA_MACSEC_SCB])
3883	tx_sc->scb = !!nla_get_u8(nla: data[IFLA_MACSEC_SCB]);
3884
3885	if (data[IFLA_MACSEC_REPLAY_PROTECT])
3886	secy->replay_protect = !!nla_get_u8(nla: data[IFLA_MACSEC_REPLAY_PROTECT]);
3887
3888	if (data[IFLA_MACSEC_VALIDATION])
3889	secy->validate_frames = nla_get_u8(nla: data[IFLA_MACSEC_VALIDATION]);
3890
3891	if (data[IFLA_MACSEC_CIPHER_SUITE]) {
3892	switch (nla_get_u64(nla: data[IFLA_MACSEC_CIPHER_SUITE])) {
3893	case MACSEC_CIPHER_ID_GCM_AES_128:
3894	case MACSEC_DEFAULT_CIPHER_ID:
3895	secy->key_len = MACSEC_GCM_AES_128_SAK_LEN;
3896	secy->xpn = false;
3897	break;
3898	case MACSEC_CIPHER_ID_GCM_AES_256:
3899	secy->key_len = MACSEC_GCM_AES_256_SAK_LEN;
3900	secy->xpn = false;
3901	break;
3902	case MACSEC_CIPHER_ID_GCM_AES_XPN_128:
3903	secy->key_len = MACSEC_GCM_AES_128_SAK_LEN;
3904	secy->xpn = true;
3905	break;
3906	case MACSEC_CIPHER_ID_GCM_AES_XPN_256:
3907	secy->key_len = MACSEC_GCM_AES_256_SAK_LEN;
3908	secy->xpn = true;
3909	break;
3910	default:
3911	return -EINVAL;
3912	}
3913	}
3914
3915	if (data[IFLA_MACSEC_WINDOW]) {
3916	secy->replay_window = nla_get_u32(nla: data[IFLA_MACSEC_WINDOW]);
3917
3918	/* IEEE 802.1AEbw-2013 10.7.8 - maximum replay window
3919	* for XPN cipher suites */
3920	if (secy->xpn &&
3921	secy->replay_window > MACSEC_XPN_MAX_REPLAY_WINDOW)
3922	return -EINVAL;
3923	}
3924
3925	return 0;
3926	}
3927
3928	static int macsec_changelink(struct net_device *dev, struct nlattr *tb[],
3929	struct nlattr *data[],
3930	struct netlink_ext_ack *extack)
3931	{
3932	struct macsec_dev *macsec = macsec_priv(dev);
3933	bool macsec_offload_state_change = false;
3934	enum macsec_offload offload;
3935	struct macsec_tx_sc tx_sc;
3936	struct macsec_secy secy;
3937	int ret;
3938
3939	if (!data)
3940	return 0;
3941
3942	if (data[IFLA_MACSEC_CIPHER_SUITE] ||
3943	data[IFLA_MACSEC_ICV_LEN] ||
3944	data[IFLA_MACSEC_SCI] ||
3945	data[IFLA_MACSEC_PORT])
3946	return -EINVAL;
3947
3948	/* Keep a copy of unmodified secy and tx_sc, in case the offload
3949	* propagation fails, to revert macsec_changelink_common.
3950	*/
3951	memcpy(&secy, &macsec->secy, sizeof(secy));
3952	memcpy(&tx_sc, &macsec->secy.tx_sc, sizeof(tx_sc));
3953
3954	ret = macsec_changelink_common(dev, data);
3955	if (ret)
3956	goto cleanup;
3957
3958	if (data[IFLA_MACSEC_OFFLOAD]) {
3959	offload = nla_get_u8(nla: data[IFLA_MACSEC_OFFLOAD]);
3960	if (macsec->offload != offload) {
3961	macsec_offload_state_change = true;
3962	ret = macsec_update_offload(dev, offload);
3963	if (ret)
3964	goto cleanup;
3965	}
3966	}
3967
3968	/* If h/w offloading is available, propagate to the device */
3969	if (!macsec_offload_state_change && macsec_is_offloaded(macsec)) {
3970	const struct macsec_ops *ops;
3971	struct macsec_context ctx;
3972
3973	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
3974	if (!ops) {
3975	ret = -EOPNOTSUPP;
3976	goto cleanup;
3977	}
3978
3979	ctx.secy = &macsec->secy;
3980	ret = macsec_offload(func: ops->mdo_upd_secy, ctx: &ctx);
3981	if (ret)
3982	goto cleanup;
3983	}
3984
3985	return 0;
3986
3987	cleanup:
3988	memcpy(&macsec->secy.tx_sc, &tx_sc, sizeof(tx_sc));
3989	memcpy(&macsec->secy, &secy, sizeof(secy));
3990
3991	return ret;
3992	}
3993
3994	static void macsec_del_dev(struct macsec_dev *macsec)
3995	{
3996	int i;
3997
3998	while (macsec->secy.rx_sc) {
3999	struct macsec_rx_sc *rx_sc = rtnl_dereference(macsec->secy.rx_sc);
4000
4001	rcu_assign_pointer(macsec->secy.rx_sc, rx_sc->next);
4002	free_rx_sc(rx_sc);
4003	}
4004
4005	for (i = 0; i < MACSEC_NUM_AN; i++) {
4006	struct macsec_tx_sa *sa = rtnl_dereference(macsec->secy.tx_sc.sa[i]);
4007
4008	if (sa) {
4009	RCU_INIT_POINTER(macsec->secy.tx_sc.sa[i], NULL);
4010	clear_tx_sa(tx_sa: sa);
4011	}
4012	}
4013	}
4014
4015	static void macsec_common_dellink(struct net_device *dev, struct list_head *head)
4016	{
4017	struct macsec_dev *macsec = macsec_priv(dev);
4018	struct net_device *real_dev = macsec->real_dev;
4019
4020	/* If h/w offloading is available, propagate to the device */
4021	if (macsec_is_offloaded(macsec)) {
4022	const struct macsec_ops *ops;
4023	struct macsec_context ctx;
4024
4025	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
4026	if (ops) {
4027	ctx.secy = &macsec->secy;
4028	macsec_offload(func: ops->mdo_del_secy, ctx: &ctx);
4029	}
4030	}
4031
4032	unregister_netdevice_queue(dev, head);
4033	list_del_rcu(entry: &macsec->secys);
4034	macsec_del_dev(macsec);
4035	netdev_upper_dev_unlink(dev: real_dev, upper_dev: dev);
4036
4037	macsec_generation++;
4038	}
4039
4040	static void macsec_dellink(struct net_device *dev, struct list_head *head)
4041	{
4042	struct macsec_dev *macsec = macsec_priv(dev);
4043	struct net_device *real_dev = macsec->real_dev;
4044	struct macsec_rxh_data *rxd = macsec_data_rtnl(dev: real_dev);
4045
4046	macsec_common_dellink(dev, head);
4047
4048	if (list_empty(head: &rxd->secys)) {
4049	netdev_rx_handler_unregister(dev: real_dev);
4050	kfree(objp: rxd);
4051	}
4052	}
4053
4054	static int register_macsec_dev(struct net_device *real_dev,
4055	struct net_device *dev)
4056	{
4057	struct macsec_dev *macsec = macsec_priv(dev);
4058	struct macsec_rxh_data *rxd = macsec_data_rtnl(dev: real_dev);
4059
4060	if (!rxd) {
4061	int err;
4062
4063	rxd = kmalloc(sizeof(*rxd), GFP_KERNEL);
4064	if (!rxd)
4065	return -ENOMEM;
4066
4067	INIT_LIST_HEAD(list: &rxd->secys);
4068
4069	err = netdev_rx_handler_register(dev: real_dev, rx_handler: macsec_handle_frame,
4070	rx_handler_data: rxd);
4071	if (err < 0) {
4072	kfree(objp: rxd);
4073	return err;
4074	}
4075	}
4076
4077	list_add_tail_rcu(new: &macsec->secys, head: &rxd->secys);
4078	return 0;
4079	}
4080
4081	static bool sci_exists(struct net_device *dev, sci_t sci)
4082	{
4083	struct macsec_rxh_data *rxd = macsec_data_rtnl(dev);
4084	struct macsec_dev *macsec;
4085
4086	list_for_each_entry(macsec, &rxd->secys, secys) {
4087	if (macsec->secy.sci == sci)
4088	return true;
4089	}
4090
4091	return false;
4092	}
4093
4094	static sci_t dev_to_sci(struct net_device *dev, __be16 port)
4095	{
4096	return make_sci(addr: dev->dev_addr, port);
4097	}
4098
4099	static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len)
4100	{
4101	struct macsec_dev *macsec = macsec_priv(dev);
4102	struct macsec_secy *secy = &macsec->secy;
4103
4104	macsec->stats = netdev_alloc_pcpu_stats(struct pcpu_secy_stats);
4105	if (!macsec->stats)
4106	return -ENOMEM;
4107
4108	secy->tx_sc.stats = netdev_alloc_pcpu_stats(struct pcpu_tx_sc_stats);
4109	if (!secy->tx_sc.stats)
4110	return -ENOMEM;
4111
4112	secy->tx_sc.md_dst = metadata_dst_alloc(optslen: 0, type: METADATA_MACSEC, GFP_KERNEL);
4113	if (!secy->tx_sc.md_dst)
4114	/* macsec and secy percpu stats will be freed when unregistering
4115	* net_device in macsec_free_netdev()
4116	*/
4117	return -ENOMEM;
4118
4119	if (sci == MACSEC_UNDEF_SCI)
4120	sci = dev_to_sci(dev, MACSEC_PORT_ES);
4121
4122	secy->netdev = dev;
4123	secy->operational = true;
4124	secy->key_len = DEFAULT_SAK_LEN;
4125	secy->icv_len = icv_len;
4126	secy->validate_frames = MACSEC_VALIDATE_DEFAULT;
4127	secy->protect_frames = true;
4128	secy->replay_protect = false;
4129	secy->xpn = DEFAULT_XPN;
4130
4131	secy->sci = sci;
4132	secy->tx_sc.md_dst->u.macsec_info.sci = sci;
4133	secy->tx_sc.active = true;
4134	secy->tx_sc.encoding_sa = DEFAULT_ENCODING_SA;
4135	secy->tx_sc.encrypt = DEFAULT_ENCRYPT;
4136	secy->tx_sc.send_sci = DEFAULT_SEND_SCI;
4137	secy->tx_sc.end_station = false;
4138	secy->tx_sc.scb = false;
4139
4140	return 0;
4141	}
4142
4143	static struct lock_class_key macsec_netdev_addr_lock_key;
4144
4145	static int macsec_newlink(struct net_device *dev,
4146	struct rtnl_newlink_params *params,
4147	struct netlink_ext_ack *extack)
4148	{
4149	struct net *link_net = rtnl_newlink_link_net(p: params);
4150	struct macsec_dev *macsec = macsec_priv(dev);
4151	struct nlattr **data = params->data;
4152	struct nlattr **tb = params->tb;
4153	rx_handler_func_t *rx_handler;
4154	u8 icv_len = MACSEC_DEFAULT_ICV_LEN;
4155	struct net_device *real_dev;
4156	int err, mtu;
4157	sci_t sci;
4158
4159	if (!tb[IFLA_LINK])
4160	return -EINVAL;
4161	real_dev = __dev_get_by_index(net: link_net, ifindex: nla_get_u32(nla: tb[IFLA_LINK]));
4162	if (!real_dev)
4163	return -ENODEV;
4164	if (real_dev->type != ARPHRD_ETHER)
4165	return -EINVAL;
4166
4167	dev->priv_flags |= IFF_MACSEC;
4168
4169	macsec->real_dev = real_dev;
4170
4171	if (data && data[IFLA_MACSEC_OFFLOAD])
4172	macsec->offload = nla_get_offload(nla: data[IFLA_MACSEC_OFFLOAD]);
4173	else
4174	/* MACsec offloading is off by default */
4175	macsec->offload = MACSEC_OFFLOAD_OFF;
4176
4177	/* Check if the offloading mode is supported by the underlying layers */
4178	if (macsec->offload != MACSEC_OFFLOAD_OFF &&
4179	!macsec_check_offload(offload: macsec->offload, macsec))
4180	return -EOPNOTSUPP;
4181
4182	/* send_sci must be set to true when transmit sci explicitly is set */
4183	if ((data && data[IFLA_MACSEC_SCI]) &&
4184	(data && data[IFLA_MACSEC_INC_SCI])) {
4185	u8 send_sci = !!nla_get_u8(nla: data[IFLA_MACSEC_INC_SCI]);
4186
4187	if (!send_sci)
4188	return -EINVAL;
4189	}
4190
4191	if (data && data[IFLA_MACSEC_ICV_LEN])
4192	icv_len = nla_get_u8(nla: data[IFLA_MACSEC_ICV_LEN]);
4193	mtu = real_dev->mtu - icv_len - macsec_extra_len(sci_present: true);
4194	if (mtu < 0)
4195	dev->mtu = 0;
4196	else
4197	dev->mtu = mtu;
4198
4199	rx_handler = rtnl_dereference(real_dev->rx_handler);
4200	if (rx_handler && rx_handler != macsec_handle_frame)
4201	return -EBUSY;
4202
4203	err = register_netdevice(dev);
4204	if (err < 0)
4205	return err;
4206
4207	netdev_lockdep_set_classes(dev);
4208	lockdep_set_class(&dev->addr_list_lock,
4209	&macsec_netdev_addr_lock_key);
4210
4211	err = netdev_upper_dev_link(dev: real_dev, upper_dev: dev, extack);
4212	if (err < 0)
4213	goto unregister;
4214
4215	/* need to be already registered so that ->init has run and
4216	* the MAC addr is set
4217	*/
4218	if (data && data[IFLA_MACSEC_SCI])
4219	sci = nla_get_sci(nla: data[IFLA_MACSEC_SCI]);
4220	else if (data && data[IFLA_MACSEC_PORT])
4221	sci = dev_to_sci(dev, port: nla_get_be16(nla: data[IFLA_MACSEC_PORT]));
4222	else
4223	sci = dev_to_sci(dev, MACSEC_PORT_ES);
4224
4225	if (rx_handler && sci_exists(dev: real_dev, sci)) {
4226	err = -EBUSY;
4227	goto unlink;
4228	}
4229
4230	err = macsec_add_dev(dev, sci, icv_len);
4231	if (err)
4232	goto unlink;
4233
4234	if (data) {
4235	err = macsec_changelink_common(dev, data);
4236	if (err)
4237	goto del_dev;
4238	}
4239
4240	/* If h/w offloading is available, propagate to the device */
4241	if (macsec_is_offloaded(macsec)) {
4242	const struct macsec_ops *ops;
4243	struct macsec_context ctx;
4244
4245	ops = macsec_get_ops(macsec, ctx: &ctx);
4246	if (ops) {
4247	ctx.secy = &macsec->secy;
4248	err = macsec_offload(func: ops->mdo_add_secy, ctx: &ctx);
4249	if (err)
4250	goto del_dev;
4251
4252	macsec->insert_tx_tag =
4253	macsec_needs_tx_tag(macsec, ops);
4254	}
4255	}
4256
4257	err = register_macsec_dev(real_dev, dev);
4258	if (err < 0)
4259	goto del_dev;
4260
4261	netif_stacked_transfer_operstate(rootdev: real_dev, dev);
4262	linkwatch_fire_event(dev);
4263
4264	macsec_generation++;
4265
4266	return 0;
4267
4268	del_dev:
4269	macsec_del_dev(macsec);
4270	unlink:
4271	netdev_upper_dev_unlink(dev: real_dev, upper_dev: dev);
4272	unregister:
4273	unregister_netdevice(dev);
4274	return err;
4275	}
4276
4277	static int macsec_validate_attr(struct nlattr *tb[], struct nlattr *data[],
4278	struct netlink_ext_ack *extack)
4279	{
4280	u64 csid = MACSEC_DEFAULT_CIPHER_ID;
4281	u8 icv_len = MACSEC_DEFAULT_ICV_LEN;
4282	int flag;
4283	bool es, scb, sci;
4284
4285	if (!data)
4286	return 0;
4287
4288	if (data[IFLA_MACSEC_CIPHER_SUITE])
4289	csid = nla_get_u64(nla: data[IFLA_MACSEC_CIPHER_SUITE]);
4290
4291	if (data[IFLA_MACSEC_ICV_LEN]) {
4292	icv_len = nla_get_u8(nla: data[IFLA_MACSEC_ICV_LEN]);
4293	if (icv_len != MACSEC_DEFAULT_ICV_LEN) {
4294	char dummy_key[DEFAULT_SAK_LEN] = { 0 };
4295	struct crypto_aead *dummy_tfm;
4296
4297	dummy_tfm = macsec_alloc_tfm(key: dummy_key,
4298	DEFAULT_SAK_LEN,
4299	icv_len);
4300	if (IS_ERR(ptr: dummy_tfm))
4301	return PTR_ERR(ptr: dummy_tfm);
4302	crypto_free_aead(tfm: dummy_tfm);
4303	}
4304	}
4305
4306	switch (csid) {
4307	case MACSEC_CIPHER_ID_GCM_AES_128:
4308	case MACSEC_CIPHER_ID_GCM_AES_256:
4309	case MACSEC_CIPHER_ID_GCM_AES_XPN_128:
4310	case MACSEC_CIPHER_ID_GCM_AES_XPN_256:
4311	case MACSEC_DEFAULT_CIPHER_ID:
4312	if (icv_len < MACSEC_MIN_ICV_LEN ||
4313	icv_len > MACSEC_STD_ICV_LEN)
4314	return -EINVAL;
4315	break;
4316	default:
4317	return -EINVAL;
4318	}
4319
4320	if (data[IFLA_MACSEC_ENCODING_SA]) {
4321	if (nla_get_u8(nla: data[IFLA_MACSEC_ENCODING_SA]) >= MACSEC_NUM_AN)
4322	return -EINVAL;
4323	}
4324
4325	for (flag = IFLA_MACSEC_ENCODING_SA + 1;
4326	flag < IFLA_MACSEC_VALIDATION;
4327	flag++) {
4328	if (data[flag]) {
4329	if (nla_get_u8(nla: data[flag]) > 1)
4330	return -EINVAL;
4331	}
4332	}
4333
4334	es = nla_get_u8_default(nla: data[IFLA_MACSEC_ES], defvalue: false);
4335	sci = nla_get_u8_default(nla: data[IFLA_MACSEC_INC_SCI], defvalue: false);
4336	scb = nla_get_u8_default(nla: data[IFLA_MACSEC_SCB], defvalue: false);
4337
4338	if ((sci && (scb || es)) || (scb && es))
4339	return -EINVAL;
4340
4341	if (data[IFLA_MACSEC_VALIDATION] &&
4342	nla_get_u8(nla: data[IFLA_MACSEC_VALIDATION]) > MACSEC_VALIDATE_MAX)
4343	return -EINVAL;
4344
4345	if ((data[IFLA_MACSEC_REPLAY_PROTECT] &&
4346	nla_get_u8(nla: data[IFLA_MACSEC_REPLAY_PROTECT])) &&
4347	!data[IFLA_MACSEC_WINDOW])
4348	return -EINVAL;
4349
4350	return 0;
4351	}
4352
4353	static struct net *macsec_get_link_net(const struct net_device *dev)
4354	{
4355	return dev_net(dev: macsec_priv(dev)->real_dev);
4356	}
4357
4358	struct net_device *macsec_get_real_dev(const struct net_device *dev)
4359	{
4360	return macsec_priv(dev)->real_dev;
4361	}
4362	EXPORT_SYMBOL_GPL(macsec_get_real_dev);
4363
4364	bool macsec_netdev_is_offloaded(struct net_device *dev)
4365	{
4366	return macsec_is_offloaded(macsec: macsec_priv(dev));
4367	}
4368	EXPORT_SYMBOL_GPL(macsec_netdev_is_offloaded);
4369
4370	static size_t macsec_get_size(const struct net_device *dev)
4371	{
4372	return nla_total_size_64bit(payload: 8) + /* IFLA_MACSEC_SCI */
4373	nla_total_size(payload: 1) + /* IFLA_MACSEC_ICV_LEN */
4374	nla_total_size_64bit(payload: 8) + /* IFLA_MACSEC_CIPHER_SUITE */
4375	nla_total_size(payload: 4) + /* IFLA_MACSEC_WINDOW */
4376	nla_total_size(payload: 1) + /* IFLA_MACSEC_ENCODING_SA */
4377	nla_total_size(payload: 1) + /* IFLA_MACSEC_ENCRYPT */
4378	nla_total_size(payload: 1) + /* IFLA_MACSEC_PROTECT */
4379	nla_total_size(payload: 1) + /* IFLA_MACSEC_INC_SCI */
4380	nla_total_size(payload: 1) + /* IFLA_MACSEC_ES */
4381	nla_total_size(payload: 1) + /* IFLA_MACSEC_SCB */
4382	nla_total_size(payload: 1) + /* IFLA_MACSEC_REPLAY_PROTECT */
4383	nla_total_size(payload: 1) + /* IFLA_MACSEC_VALIDATION */
4384	nla_total_size(payload: 1) + /* IFLA_MACSEC_OFFLOAD */
4385	0;
4386	}
4387
4388	static int macsec_fill_info(struct sk_buff *skb,
4389	const struct net_device *dev)
4390	{
4391	struct macsec_tx_sc *tx_sc;
4392	struct macsec_dev *macsec;
4393	struct macsec_secy *secy;
4394	u64 csid;
4395
4396	macsec = macsec_priv(dev);
4397	secy = &macsec->secy;
4398	tx_sc = &secy->tx_sc;
4399
4400	switch (secy->key_len) {
4401	case MACSEC_GCM_AES_128_SAK_LEN:
4402	csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_128 : MACSEC_DEFAULT_CIPHER_ID;
4403	break;
4404	case MACSEC_GCM_AES_256_SAK_LEN:
4405	csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_256 : MACSEC_CIPHER_ID_GCM_AES_256;
4406	break;
4407	default:
4408	goto nla_put_failure;
4409	}
4410
4411	if (nla_put_sci(skb, attrtype: IFLA_MACSEC_SCI, value: secy->sci,
4412	padattr: IFLA_MACSEC_PAD) ||
4413	nla_put_u8(skb, attrtype: IFLA_MACSEC_ICV_LEN, value: secy->icv_len) ||
4414	nla_put_u64_64bit(skb, attrtype: IFLA_MACSEC_CIPHER_SUITE,
4415	value: csid, padattr: IFLA_MACSEC_PAD) ||
4416	nla_put_u8(skb, attrtype: IFLA_MACSEC_ENCODING_SA, value: tx_sc->encoding_sa) ||
4417	nla_put_u8(skb, attrtype: IFLA_MACSEC_ENCRYPT, value: tx_sc->encrypt) ||
4418	nla_put_u8(skb, attrtype: IFLA_MACSEC_PROTECT, value: secy->protect_frames) ||
4419	nla_put_u8(skb, attrtype: IFLA_MACSEC_INC_SCI, value: tx_sc->send_sci) ||
4420	nla_put_u8(skb, attrtype: IFLA_MACSEC_ES, value: tx_sc->end_station) ||
4421	nla_put_u8(skb, attrtype: IFLA_MACSEC_SCB, value: tx_sc->scb) ||
4422	nla_put_u8(skb, attrtype: IFLA_MACSEC_REPLAY_PROTECT, value: secy->replay_protect) ||
4423	nla_put_u8(skb, attrtype: IFLA_MACSEC_VALIDATION, value: secy->validate_frames) ||
4424	nla_put_u8(skb, attrtype: IFLA_MACSEC_OFFLOAD, value: macsec->offload) ||
4425	0)
4426	goto nla_put_failure;
4427
4428	if (secy->replay_protect) {
4429	if (nla_put_u32(skb, attrtype: IFLA_MACSEC_WINDOW, value: secy->replay_window))
4430	goto nla_put_failure;
4431	}
4432
4433	return 0;
4434
4435	nla_put_failure:
4436	return -EMSGSIZE;
4437	}
4438
4439	static struct rtnl_link_ops macsec_link_ops __read_mostly = {
4440	.kind = "macsec",
4441	.priv_size = sizeof(struct macsec_dev),
4442	.maxtype = IFLA_MACSEC_MAX,
4443	.policy = macsec_rtnl_policy,
4444	.setup = macsec_setup,
4445	.validate = macsec_validate_attr,
4446	.newlink = macsec_newlink,
4447	.changelink = macsec_changelink,
4448	.dellink = macsec_dellink,
4449	.get_size = macsec_get_size,
4450	.fill_info = macsec_fill_info,
4451	.get_link_net = macsec_get_link_net,
4452	};
4453
4454	static bool is_macsec_master(struct net_device *dev)
4455	{
4456	return rcu_access_pointer(dev->rx_handler) == macsec_handle_frame;
4457	}
4458
4459	static int macsec_notify(struct notifier_block *this, unsigned long event,
4460	void *ptr)
4461	{
4462	struct net_device *real_dev = netdev_notifier_info_to_dev(info: ptr);
4463	struct macsec_rxh_data *rxd;
4464	struct macsec_dev *m, *n;
4465	LIST_HEAD(head);
4466
4467	if (!is_macsec_master(dev: real_dev))
4468	return NOTIFY_DONE;
4469
4470	rxd = macsec_data_rtnl(dev: real_dev);
4471
4472	switch (event) {
4473	case NETDEV_DOWN:
4474	case NETDEV_UP:
4475	case NETDEV_CHANGE:
4476	list_for_each_entry_safe(m, n, &rxd->secys, secys) {
4477	struct net_device *dev = m->secy.netdev;
4478
4479	netif_stacked_transfer_operstate(rootdev: real_dev, dev);
4480	}
4481	break;
4482	case NETDEV_UNREGISTER:
4483	list_for_each_entry_safe(m, n, &rxd->secys, secys) {
4484	macsec_common_dellink(dev: m->secy.netdev, head: &head);
4485	}
4486
4487	netdev_rx_handler_unregister(dev: real_dev);
4488	kfree(objp: rxd);
4489
4490	unregister_netdevice_many(head: &head);
4491	break;
4492	case NETDEV_CHANGEMTU:
4493	list_for_each_entry(m, &rxd->secys, secys) {
4494	struct net_device *dev = m->secy.netdev;
4495	unsigned int mtu = real_dev->mtu - (m->secy.icv_len +
4496	macsec_extra_len(sci_present: true));
4497
4498	if (dev->mtu > mtu)
4499	dev_set_mtu(dev, mtu);
4500	}
4501	break;
4502	case NETDEV_FEAT_CHANGE:
4503	list_for_each_entry(m, &rxd->secys, secys) {
4504	macsec_inherit_tso_max(dev: m->secy.netdev);
4505	netdev_update_features(dev: m->secy.netdev);
4506	}
4507	break;
4508	}
4509
4510	return NOTIFY_OK;
4511	}
4512
4513	static struct notifier_block macsec_notifier = {
4514	.notifier_call = macsec_notify,
4515	};
4516
4517	static int __init macsec_init(void)
4518	{
4519	int err;
4520
4521	pr_info("MACsec IEEE 802.1AE\n");
4522	err = register_netdevice_notifier(nb: &macsec_notifier);
4523	if (err)
4524	return err;
4525
4526	err = rtnl_link_register(ops: &macsec_link_ops);
4527	if (err)
4528	goto notifier;
4529
4530	err = genl_register_family(family: &macsec_fam);
4531	if (err)
4532	goto rtnl;
4533
4534	return 0;
4535
4536	rtnl:
4537	rtnl_link_unregister(ops: &macsec_link_ops);
4538	notifier:
4539	unregister_netdevice_notifier(nb: &macsec_notifier);
4540	return err;
4541	}
4542
4543	static void __exit macsec_exit(void)
4544	{
4545	genl_unregister_family(family: &macsec_fam);
4546	rtnl_link_unregister(ops: &macsec_link_ops);
4547	unregister_netdevice_notifier(nb: &macsec_notifier);
4548	rcu_barrier();
4549	}
4550
4551	module_init(macsec_init);
4552	module_exit(macsec_exit);
4553
4554	MODULE_ALIAS_RTNL_LINK("macsec");
4555	MODULE_ALIAS_GENL_FAMILY("macsec");
4556
4557	MODULE_DESCRIPTION("MACsec IEEE 802.1AE");
4558	MODULE_LICENSE("GPL v2");
4559