1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* drivers/net/macsec.c - MACsec device
4	*
5	* Copyright (c) 2015 Sabrina Dubroca <sd@queasysnail.net>
6	*/
7
8	#include <linux/types.h>
9	#include <linux/skbuff.h>
10	#include <linux/socket.h>
11	#include <linux/module.h>
12	#include <crypto/aead.h>
13	#include <linux/etherdevice.h>
14	#include <linux/netdevice.h>
15	#include <linux/rtnetlink.h>
16	#include <linux/refcount.h>
17	#include <net/genetlink.h>
18	#include <net/sock.h>
19	#include <net/gro_cells.h>
20	#include <net/macsec.h>
21	#include <net/dst_metadata.h>
22	#include <linux/phy.h>
23	#include <linux/byteorder/generic.h>
24	#include <linux/if_arp.h>
25
26	#include <uapi/linux/if_macsec.h>
27
28	/* SecTAG length = macsec_eth_header without the optional SCI */
29	#define MACSEC_TAG_LEN 6
30
31	struct macsec_eth_header {
32	struct ethhdr eth;
33	/* SecTAG */
34	u8 tci_an;
35	#if defined(__LITTLE_ENDIAN_BITFIELD)
36	u8 short_length:6,
37	unused:2;
38	#elif defined(__BIG_ENDIAN_BITFIELD)
39	u8 unused:2,
40	short_length:6;
41	#else
42	#error "Please fix <asm/byteorder.h>"
43	#endif
44	__be32 packet_number;
45	u8 secure_channel_id[8]; /* optional */
46	} __packed;
47
48	/* minimum secure data length deemed "not short", see IEEE 802.1AE-2006 9.7 */
49	#define MIN_NON_SHORT_LEN 48
50
51	#define GCM_AES_IV_LEN 12
52
53	#define for_each_rxsc(secy, sc) \
54	for (sc = rcu_dereference_bh(secy->rx_sc); \
55	sc; \
56	sc = rcu_dereference_bh(sc->next))
57	#define for_each_rxsc_rtnl(secy, sc) \
58	for (sc = rtnl_dereference(secy->rx_sc); \
59	sc; \
60	sc = rtnl_dereference(sc->next))
61
62	#define pn_same_half(pn1, pn2) (!(((pn1) >> 31) ^ ((pn2) >> 31)))
63
64	struct gcm_iv_xpn {
65	union {
66	u8 short_secure_channel_id[4];
67	ssci_t ssci;
68	};
69	__be64 pn;
70	} __packed;
71
72	struct gcm_iv {
73	union {
74	u8 secure_channel_id[8];
75	sci_t sci;
76	};
77	__be32 pn;
78	};
79
80	#define MACSEC_VALIDATE_DEFAULT MACSEC_VALIDATE_STRICT
81
82	struct pcpu_secy_stats {
83	struct macsec_dev_stats stats;
84	struct u64_stats_sync syncp;
85	};
86
87	/**
88	* struct macsec_dev - private data
89	* @secy: SecY config
90	* @real_dev: pointer to underlying netdevice
91	* @dev_tracker: refcount tracker for @real_dev reference
92	* @stats: MACsec device stats
93	* @secys: linked list of SecY's on the underlying device
94	* @gro_cells: pointer to the Generic Receive Offload cell
95	* @offload: status of offloading on the MACsec device
96	*/
97	struct macsec_dev {
98	struct macsec_secy secy;
99	struct net_device *real_dev;
100	netdevice_tracker dev_tracker;
101	struct pcpu_secy_stats __percpu *stats;
102	struct list_head secys;
103	struct gro_cells gro_cells;
104	enum macsec_offload offload;
105	};
106
107	/**
108	* struct macsec_rxh_data - rx_handler private argument
109	* @secys: linked list of SecY's on this underlying device
110	*/
111	struct macsec_rxh_data {
112	struct list_head secys;
113	};
114
115	static struct macsec_dev *macsec_priv(const struct net_device *dev)
116	{
117	return (struct macsec_dev *)netdev_priv(dev);
118	}
119
120	static struct macsec_rxh_data *macsec_data_rcu(const struct net_device *dev)
121	{
122	return rcu_dereference_bh(dev->rx_handler_data);
123	}
124
125	static struct macsec_rxh_data *macsec_data_rtnl(const struct net_device *dev)
126	{
127	return rtnl_dereference(dev->rx_handler_data);
128	}
129
130	struct macsec_cb {
131	struct aead_request *req;
132	union {
133	struct macsec_tx_sa *tx_sa;
134	struct macsec_rx_sa *rx_sa;
135	};
136	u8 assoc_num;
137	bool valid;
138	bool has_sci;
139	};
140
141	static struct macsec_rx_sa *macsec_rxsa_get(struct macsec_rx_sa __rcu *ptr)
142	{
143	struct macsec_rx_sa *sa = rcu_dereference_bh(ptr);
144
145	if (!sa || !sa->active)
146	return NULL;
147
148	if (!refcount_inc_not_zero(r: &sa->refcnt))
149	return NULL;
150
151	return sa;
152	}
153
154	static struct macsec_rx_sa *macsec_active_rxsa_get(struct macsec_rx_sc *rx_sc)
155	{
156	struct macsec_rx_sa *sa = NULL;
157	int an;
158
159	for (an = 0; an < MACSEC_NUM_AN; an++) {
160	sa = macsec_rxsa_get(ptr: rx_sc->sa[an]);
161	if (sa)
162	break;
163	}
164	return sa;
165	}
166
167	static void free_rx_sc_rcu(struct rcu_head *head)
168	{
169	struct macsec_rx_sc *rx_sc = container_of(head, struct macsec_rx_sc, rcu_head);
170
171	free_percpu(pdata: rx_sc->stats);
172	kfree(objp: rx_sc);
173	}
174
175	static struct macsec_rx_sc *macsec_rxsc_get(struct macsec_rx_sc *sc)
176	{
177	return refcount_inc_not_zero(r: &sc->refcnt) ? sc : NULL;
178	}
179
180	static void macsec_rxsc_put(struct macsec_rx_sc *sc)
181	{
182	if (refcount_dec_and_test(r: &sc->refcnt))
183	call_rcu(head: &sc->rcu_head, func: free_rx_sc_rcu);
184	}
185
186	static void free_rxsa(struct rcu_head *head)
187	{
188	struct macsec_rx_sa *sa = container_of(head, struct macsec_rx_sa, rcu);
189
190	crypto_free_aead(tfm: sa->key.tfm);
191	free_percpu(pdata: sa->stats);
192	kfree(objp: sa);
193	}
194
195	static void macsec_rxsa_put(struct macsec_rx_sa *sa)
196	{
197	if (refcount_dec_and_test(r: &sa->refcnt))
198	call_rcu(head: &sa->rcu, func: free_rxsa);
199	}
200
201	static struct macsec_tx_sa *macsec_txsa_get(struct macsec_tx_sa __rcu *ptr)
202	{
203	struct macsec_tx_sa *sa = rcu_dereference_bh(ptr);
204
205	if (!sa || !sa->active)
206	return NULL;
207
208	if (!refcount_inc_not_zero(r: &sa->refcnt))
209	return NULL;
210
211	return sa;
212	}
213
214	static void free_txsa(struct rcu_head *head)
215	{
216	struct macsec_tx_sa *sa = container_of(head, struct macsec_tx_sa, rcu);
217
218	crypto_free_aead(tfm: sa->key.tfm);
219	free_percpu(pdata: sa->stats);
220	kfree(objp: sa);
221	}
222
223	static void macsec_txsa_put(struct macsec_tx_sa *sa)
224	{
225	if (refcount_dec_and_test(r: &sa->refcnt))
226	call_rcu(head: &sa->rcu, func: free_txsa);
227	}
228
229	static struct macsec_cb *macsec_skb_cb(struct sk_buff *skb)
230	{
231	BUILD_BUG_ON(sizeof(struct macsec_cb) > sizeof(skb->cb));
232	return (struct macsec_cb *)skb->cb;
233	}
234
235	#define MACSEC_PORT_SCB (0x0000)
236	#define MACSEC_UNDEF_SCI ((__force sci_t)0xffffffffffffffffULL)
237	#define MACSEC_UNDEF_SSCI ((__force ssci_t)0xffffffff)
238
239	#define MACSEC_GCM_AES_128_SAK_LEN 16
240	#define MACSEC_GCM_AES_256_SAK_LEN 32
241
242	#define DEFAULT_SAK_LEN MACSEC_GCM_AES_128_SAK_LEN
243	#define DEFAULT_XPN false
244	#define DEFAULT_SEND_SCI true
245	#define DEFAULT_ENCRYPT false
246	#define DEFAULT_ENCODING_SA 0
247	#define MACSEC_XPN_MAX_REPLAY_WINDOW (((1 << 30) - 1))
248
249	static sci_t make_sci(const u8 *addr, __be16 port)
250	{
251	sci_t sci;
252
253	memcpy(&sci, addr, ETH_ALEN);
254	memcpy(((char *)&sci) + ETH_ALEN, &port, sizeof(port));
255
256	return sci;
257	}
258
259	static sci_t macsec_frame_sci(struct macsec_eth_header *hdr, bool sci_present)
260	{
261	sci_t sci;
262
263	if (sci_present)
264	memcpy(&sci, hdr->secure_channel_id,
265	sizeof(hdr->secure_channel_id));
266	else
267	sci = make_sci(addr: hdr->eth.h_source, MACSEC_PORT_ES);
268
269	return sci;
270	}
271
272	static unsigned int macsec_sectag_len(bool sci_present)
273	{
274	return MACSEC_TAG_LEN + (sci_present ? MACSEC_SCI_LEN : 0);
275	}
276
277	static unsigned int macsec_hdr_len(bool sci_present)
278	{
279	return macsec_sectag_len(sci_present) + ETH_HLEN;
280	}
281
282	static unsigned int macsec_extra_len(bool sci_present)
283	{
284	return macsec_sectag_len(sci_present) + sizeof(__be16);
285	}
286
287	/* Fill SecTAG according to IEEE 802.1AE-2006 10.5.3 */
288	static void macsec_fill_sectag(struct macsec_eth_header *h,
289	const struct macsec_secy *secy, u32 pn,
290	bool sci_present)
291	{
292	const struct macsec_tx_sc *tx_sc = &secy->tx_sc;
293
294	memset(&h->tci_an, 0, macsec_sectag_len(sci_present));
295	h->eth.h_proto = htons(ETH_P_MACSEC);
296
297	if (sci_present) {
298	h->tci_an |= MACSEC_TCI_SC;
299	memcpy(&h->secure_channel_id, &secy->sci,
300	sizeof(h->secure_channel_id));
301	} else {
302	if (tx_sc->end_station)
303	h->tci_an |= MACSEC_TCI_ES;
304	if (tx_sc->scb)
305	h->tci_an |= MACSEC_TCI_SCB;
306	}
307
308	h->packet_number = htonl(pn);
309
310	/* with GCM, C/E clear for !encrypt, both set for encrypt */
311	if (tx_sc->encrypt)
312	h->tci_an |= MACSEC_TCI_CONFID;
313	else if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN)
314	h->tci_an |= MACSEC_TCI_C;
315
316	h->tci_an |= tx_sc->encoding_sa;
317	}
318
319	static void macsec_set_shortlen(struct macsec_eth_header *h, size_t data_len)
320	{
321	if (data_len < MIN_NON_SHORT_LEN)
322	h->short_length = data_len;
323	}
324
325	/* Checks if a MACsec interface is being offloaded to an hardware engine */
326	static bool macsec_is_offloaded(struct macsec_dev *macsec)
327	{
328	if (macsec->offload == MACSEC_OFFLOAD_MAC ||
329	macsec->offload == MACSEC_OFFLOAD_PHY)
330	return true;
331
332	return false;
333	}
334
335	/* Checks if underlying layers implement MACsec offloading functions. */
336	static bool macsec_check_offload(enum macsec_offload offload,
337	struct macsec_dev *macsec)
338	{
339	if (!macsec || !macsec->real_dev)
340	return false;
341
342	if (offload == MACSEC_OFFLOAD_PHY)
343	return macsec->real_dev->phydev &&
344	macsec->real_dev->phydev->macsec_ops;
345	else if (offload == MACSEC_OFFLOAD_MAC)
346	return macsec->real_dev->features & NETIF_F_HW_MACSEC &&
347	macsec->real_dev->macsec_ops;
348
349	return false;
350	}
351
352	static const struct macsec_ops *__macsec_get_ops(enum macsec_offload offload,
353	struct macsec_dev *macsec,
354	struct macsec_context *ctx)
355	{
356	if (ctx) {
357	memset(ctx, 0, sizeof(*ctx));
358	ctx->offload = offload;
359
360	if (offload == MACSEC_OFFLOAD_PHY)
361	ctx->phydev = macsec->real_dev->phydev;
362	else if (offload == MACSEC_OFFLOAD_MAC)
363	ctx->netdev = macsec->real_dev;
364	}
365
366	if (offload == MACSEC_OFFLOAD_PHY)
367	return macsec->real_dev->phydev->macsec_ops;
368	else
369	return macsec->real_dev->macsec_ops;
370	}
371
372	/* Returns a pointer to the MACsec ops struct if any and updates the MACsec
373	* context device reference if provided.
374	*/
375	static const struct macsec_ops *macsec_get_ops(struct macsec_dev *macsec,
376	struct macsec_context *ctx)
377	{
378	if (!macsec_check_offload(offload: macsec->offload, macsec))
379	return NULL;
380
381	return __macsec_get_ops(offload: macsec->offload, macsec, ctx);
382	}
383
384	/* validate MACsec packet according to IEEE 802.1AE-2018 9.12 */
385	static bool macsec_validate_skb(struct sk_buff *skb, u16 icv_len, bool xpn)
386	{
387	struct macsec_eth_header *h = (struct macsec_eth_header *)skb->data;
388	int len = skb->len - 2 * ETH_ALEN;
389	int extra_len = macsec_extra_len(sci_present: !!(h->tci_an & MACSEC_TCI_SC)) + icv_len;
390
391	/* a) It comprises at least 17 octets */
392	if (skb->len <= 16)
393	return false;
394
395	/* b) MACsec EtherType: already checked */
396
397	/* c) V bit is clear */
398	if (h->tci_an & MACSEC_TCI_VERSION)
399	return false;
400
401	/* d) ES or SCB => !SC */
402	if ((h->tci_an & MACSEC_TCI_ES || h->tci_an & MACSEC_TCI_SCB) &&
403	(h->tci_an & MACSEC_TCI_SC))
404	return false;
405
406	/* e) Bits 7 and 8 of octet 4 of the SecTAG are clear */
407	if (h->unused)
408	return false;
409
410	/* rx.pn != 0 if not XPN (figure 10-5 with 802.11AEbw-2013 amendment) */
411	if (!h->packet_number && !xpn)
412	return false;
413
414	/* length check, f) g) h) i) */
415	if (h->short_length)
416	return len == extra_len + h->short_length;
417	return len >= extra_len + MIN_NON_SHORT_LEN;
418	}
419
420	#define MACSEC_NEEDED_HEADROOM (macsec_extra_len(true))
421	#define MACSEC_NEEDED_TAILROOM MACSEC_STD_ICV_LEN
422
423	static void macsec_fill_iv_xpn(unsigned char *iv, ssci_t ssci, u64 pn,
424	salt_t salt)
425	{
426	struct gcm_iv_xpn *gcm_iv = (struct gcm_iv_xpn *)iv;
427
428	gcm_iv->ssci = ssci ^ salt.ssci;
429	gcm_iv->pn = cpu_to_be64(pn) ^ salt.pn;
430	}
431
432	static void macsec_fill_iv(unsigned char *iv, sci_t sci, u32 pn)
433	{
434	struct gcm_iv *gcm_iv = (struct gcm_iv *)iv;
435
436	gcm_iv->sci = sci;
437	gcm_iv->pn = htonl(pn);
438	}
439
440	static struct macsec_eth_header *macsec_ethhdr(struct sk_buff *skb)
441	{
442	return (struct macsec_eth_header *)skb_mac_header(skb);
443	}
444
445	static void __macsec_pn_wrapped(struct macsec_secy *secy,
446	struct macsec_tx_sa *tx_sa)
447	{
448	pr_debug("PN wrapped, transitioning to !oper\n");
449	tx_sa->active = false;
450	if (secy->protect_frames)
451	secy->operational = false;
452	}
453
454	void macsec_pn_wrapped(struct macsec_secy *secy, struct macsec_tx_sa *tx_sa)
455	{
456	spin_lock_bh(lock: &tx_sa->lock);
457	__macsec_pn_wrapped(secy, tx_sa);
458	spin_unlock_bh(lock: &tx_sa->lock);
459	}
460	EXPORT_SYMBOL_GPL(macsec_pn_wrapped);
461
462	static pn_t tx_sa_update_pn(struct macsec_tx_sa *tx_sa,
463	struct macsec_secy *secy)
464	{
465	pn_t pn;
466
467	spin_lock_bh(lock: &tx_sa->lock);
468
469	pn = tx_sa->next_pn_halves;
470	if (secy->xpn)
471	tx_sa->next_pn++;
472	else
473	tx_sa->next_pn_halves.lower++;
474
475	if (tx_sa->next_pn == 0)
476	__macsec_pn_wrapped(secy, tx_sa);
477	spin_unlock_bh(lock: &tx_sa->lock);
478
479	return pn;
480	}
481
482	static void macsec_encrypt_finish(struct sk_buff *skb, struct net_device *dev)
483	{
484	struct macsec_dev *macsec = netdev_priv(dev);
485
486	skb->dev = macsec->real_dev;
487	skb_reset_mac_header(skb);
488	skb->protocol = eth_hdr(skb)->h_proto;
489	}
490
491	static unsigned int macsec_msdu_len(struct sk_buff *skb)
492	{
493	struct macsec_dev *macsec = macsec_priv(dev: skb->dev);
494	struct macsec_secy *secy = &macsec->secy;
495	bool sci_present = macsec_skb_cb(skb)->has_sci;
496
497	return skb->len - macsec_hdr_len(sci_present) - secy->icv_len;
498	}
499
500	static void macsec_count_tx(struct sk_buff *skb, struct macsec_tx_sc *tx_sc,
501	struct macsec_tx_sa *tx_sa)
502	{
503	unsigned int msdu_len = macsec_msdu_len(skb);
504	struct pcpu_tx_sc_stats *txsc_stats = this_cpu_ptr(tx_sc->stats);
505
506	u64_stats_update_begin(syncp: &txsc_stats->syncp);
507	if (tx_sc->encrypt) {
508	txsc_stats->stats.OutOctetsEncrypted += msdu_len;
509	txsc_stats->stats.OutPktsEncrypted++;
510	this_cpu_inc(tx_sa->stats->OutPktsEncrypted);
511	} else {
512	txsc_stats->stats.OutOctetsProtected += msdu_len;
513	txsc_stats->stats.OutPktsProtected++;
514	this_cpu_inc(tx_sa->stats->OutPktsProtected);
515	}
516	u64_stats_update_end(syncp: &txsc_stats->syncp);
517	}
518
519	static void count_tx(struct net_device *dev, int ret, int len)
520	{
521	if (likely(ret == NET_XMIT_SUCCESS || ret == NET_XMIT_CN))
522	dev_sw_netstats_tx_add(dev, packets: 1, len);
523	}
524
525	static void macsec_encrypt_done(void *data, int err)
526	{
527	struct sk_buff *skb = data;
528	struct net_device *dev = skb->dev;
529	struct macsec_dev *macsec = macsec_priv(dev);
530	struct macsec_tx_sa *sa = macsec_skb_cb(skb)->tx_sa;
531	int len, ret;
532
533	aead_request_free(req: macsec_skb_cb(skb)->req);
534
535	rcu_read_lock_bh();
536	macsec_count_tx(skb, tx_sc: &macsec->secy.tx_sc, tx_sa: macsec_skb_cb(skb)->tx_sa);
537	/* packet is encrypted/protected so tx_bytes must be calculated */
538	len = macsec_msdu_len(skb) + 2 * ETH_ALEN;
539	macsec_encrypt_finish(skb, dev);
540	ret = dev_queue_xmit(skb);
541	count_tx(dev, ret, len);
542	rcu_read_unlock_bh();
543
544	macsec_txsa_put(sa);
545	dev_put(dev);
546	}
547
548	static struct aead_request *macsec_alloc_req(struct crypto_aead *tfm,
549	unsigned char **iv,
550	struct scatterlist **sg,
551	int num_frags)
552	{
553	size_t size, iv_offset, sg_offset;
554	struct aead_request *req;
555	void *tmp;
556
557	size = sizeof(struct aead_request) + crypto_aead_reqsize(tfm);
558	iv_offset = size;
559	size += GCM_AES_IV_LEN;
560
561	size = ALIGN(size, __alignof__(struct scatterlist));
562	sg_offset = size;
563	size += sizeof(struct scatterlist) * num_frags;
564
565	tmp = kmalloc(size, GFP_ATOMIC);
566	if (!tmp)
567	return NULL;
568
569	*iv = (unsigned char *)(tmp + iv_offset);
570	*sg = (struct scatterlist *)(tmp + sg_offset);
571	req = tmp;
572
573	aead_request_set_tfm(req, tfm);
574
575	return req;
576	}
577
578	static struct sk_buff *macsec_encrypt(struct sk_buff *skb,
579	struct net_device *dev)
580	{
581	int ret;
582	struct scatterlist *sg;
583	struct sk_buff *trailer;
584	unsigned char *iv;
585	struct ethhdr *eth;
586	struct macsec_eth_header *hh;
587	size_t unprotected_len;
588	struct aead_request *req;
589	struct macsec_secy *secy;
590	struct macsec_tx_sc *tx_sc;
591	struct macsec_tx_sa *tx_sa;
592	struct macsec_dev *macsec = macsec_priv(dev);
593	bool sci_present;
594	pn_t pn;
595
596	secy = &macsec->secy;
597	tx_sc = &secy->tx_sc;
598
599	/* 10.5.1 TX SA assignment */
600	tx_sa = macsec_txsa_get(ptr: tx_sc->sa[tx_sc->encoding_sa]);
601	if (!tx_sa) {
602	secy->operational = false;
603	kfree_skb(skb);
604	return ERR_PTR(error: -EINVAL);
605	}
606
607	if (unlikely(skb_headroom(skb) < MACSEC_NEEDED_HEADROOM ||
608	skb_tailroom(skb) < MACSEC_NEEDED_TAILROOM)) {
609	struct sk_buff *nskb = skb_copy_expand(skb,
610	MACSEC_NEEDED_HEADROOM,
611	MACSEC_NEEDED_TAILROOM,
612	GFP_ATOMIC);
613	if (likely(nskb)) {
614	consume_skb(skb);
615	skb = nskb;
616	} else {
617	macsec_txsa_put(sa: tx_sa);
618	kfree_skb(skb);
619	return ERR_PTR(error: -ENOMEM);
620	}
621	} else {
622	skb = skb_unshare(skb, GFP_ATOMIC);
623	if (!skb) {
624	macsec_txsa_put(sa: tx_sa);
625	return ERR_PTR(error: -ENOMEM);
626	}
627	}
628
629	unprotected_len = skb->len;
630	eth = eth_hdr(skb);
631	sci_present = macsec_send_sci(secy);
632	hh = skb_push(skb, len: macsec_extra_len(sci_present));
633	memmove(hh, eth, 2 * ETH_ALEN);
634
635	pn = tx_sa_update_pn(tx_sa, secy);
636	if (pn.full64 == 0) {
637	macsec_txsa_put(sa: tx_sa);
638	kfree_skb(skb);
639	return ERR_PTR(error: -ENOLINK);
640	}
641	macsec_fill_sectag(h: hh, secy, pn: pn.lower, sci_present);
642	macsec_set_shortlen(h: hh, data_len: unprotected_len - 2 * ETH_ALEN);
643
644	skb_put(skb, len: secy->icv_len);
645
646	if (skb->len - ETH_HLEN > macsec_priv(dev)->real_dev->mtu) {
647	struct pcpu_secy_stats *secy_stats = this_cpu_ptr(macsec->stats);
648
649	u64_stats_update_begin(syncp: &secy_stats->syncp);
650	secy_stats->stats.OutPktsTooLong++;
651	u64_stats_update_end(syncp: &secy_stats->syncp);
652
653	macsec_txsa_put(sa: tx_sa);
654	kfree_skb(skb);
655	return ERR_PTR(error: -EINVAL);
656	}
657
658	ret = skb_cow_data(skb, tailbits: 0, trailer: &trailer);
659	if (unlikely(ret < 0)) {
660	macsec_txsa_put(sa: tx_sa);
661	kfree_skb(skb);
662	return ERR_PTR(error: ret);
663	}
664
665	req = macsec_alloc_req(tfm: tx_sa->key.tfm, iv: &iv, sg: &sg, num_frags: ret);
666	if (!req) {
667	macsec_txsa_put(sa: tx_sa);
668	kfree_skb(skb);
669	return ERR_PTR(error: -ENOMEM);
670	}
671
672	if (secy->xpn)
673	macsec_fill_iv_xpn(iv, ssci: tx_sa->ssci, pn: pn.full64, salt: tx_sa->key.salt);
674	else
675	macsec_fill_iv(iv, sci: secy->sci, pn: pn.lower);
676
677	sg_init_table(sg, ret);
678	ret = skb_to_sgvec(skb, sg, offset: 0, len: skb->len);
679	if (unlikely(ret < 0)) {
680	aead_request_free(req);
681	macsec_txsa_put(sa: tx_sa);
682	kfree_skb(skb);
683	return ERR_PTR(error: ret);
684	}
685
686	if (tx_sc->encrypt) {
687	int len = skb->len - macsec_hdr_len(sci_present) -
688	secy->icv_len;
689	aead_request_set_crypt(req, src: sg, dst: sg, cryptlen: len, iv);
690	aead_request_set_ad(req, assoclen: macsec_hdr_len(sci_present));
691	} else {
692	aead_request_set_crypt(req, src: sg, dst: sg, cryptlen: 0, iv);
693	aead_request_set_ad(req, assoclen: skb->len - secy->icv_len);
694	}
695
696	macsec_skb_cb(skb)->req = req;
697	macsec_skb_cb(skb)->tx_sa = tx_sa;
698	macsec_skb_cb(skb)->has_sci = sci_present;
699	aead_request_set_callback(req, flags: 0, compl: macsec_encrypt_done, data: skb);
700
701	dev_hold(dev: skb->dev);
702	ret = crypto_aead_encrypt(req);
703	if (ret == -EINPROGRESS) {
704	return ERR_PTR(error: ret);
705	} else if (ret != 0) {
706	dev_put(dev: skb->dev);
707	kfree_skb(skb);
708	aead_request_free(req);
709	macsec_txsa_put(sa: tx_sa);
710	return ERR_PTR(error: -EINVAL);
711	}
712
713	dev_put(dev: skb->dev);
714	aead_request_free(req);
715	macsec_txsa_put(sa: tx_sa);
716
717	return skb;
718	}
719
720	static bool macsec_post_decrypt(struct sk_buff *skb, struct macsec_secy *secy, u32 pn)
721	{
722	struct macsec_rx_sa *rx_sa = macsec_skb_cb(skb)->rx_sa;
723	struct pcpu_rx_sc_stats *rxsc_stats = this_cpu_ptr(rx_sa->sc->stats);
724	struct macsec_eth_header *hdr = macsec_ethhdr(skb);
725	u32 lowest_pn = 0;
726
727	spin_lock(lock: &rx_sa->lock);
728	if (rx_sa->next_pn_halves.lower >= secy->replay_window)
729	lowest_pn = rx_sa->next_pn_halves.lower - secy->replay_window;
730
731	/* Now perform replay protection check again
732	* (see IEEE 802.1AE-2006 figure 10-5)
733	*/
734	if (secy->replay_protect && pn < lowest_pn &&
735	(!secy->xpn || pn_same_half(pn, lowest_pn))) {
736	spin_unlock(lock: &rx_sa->lock);
737	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
738	rxsc_stats->stats.InPktsLate++;
739	u64_stats_update_end(syncp: &rxsc_stats->syncp);
740	DEV_STATS_INC(secy->netdev, rx_dropped);
741	return false;
742	}
743
744	if (secy->validate_frames != MACSEC_VALIDATE_DISABLED) {
745	unsigned int msdu_len = macsec_msdu_len(skb);
746	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
747	if (hdr->tci_an & MACSEC_TCI_E)
748	rxsc_stats->stats.InOctetsDecrypted += msdu_len;
749	else
750	rxsc_stats->stats.InOctetsValidated += msdu_len;
751	u64_stats_update_end(syncp: &rxsc_stats->syncp);
752	}
753
754	if (!macsec_skb_cb(skb)->valid) {
755	spin_unlock(lock: &rx_sa->lock);
756
757	/* 10.6.5 */
758	if (hdr->tci_an & MACSEC_TCI_C ||
759	secy->validate_frames == MACSEC_VALIDATE_STRICT) {
760	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
761	rxsc_stats->stats.InPktsNotValid++;
762	u64_stats_update_end(syncp: &rxsc_stats->syncp);
763	this_cpu_inc(rx_sa->stats->InPktsNotValid);
764	DEV_STATS_INC(secy->netdev, rx_errors);
765	return false;
766	}
767
768	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
769	if (secy->validate_frames == MACSEC_VALIDATE_CHECK) {
770	rxsc_stats->stats.InPktsInvalid++;
771	this_cpu_inc(rx_sa->stats->InPktsInvalid);
772	} else if (pn < lowest_pn) {
773	rxsc_stats->stats.InPktsDelayed++;
774	} else {
775	rxsc_stats->stats.InPktsUnchecked++;
776	}
777	u64_stats_update_end(syncp: &rxsc_stats->syncp);
778	} else {
779	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
780	if (pn < lowest_pn) {
781	rxsc_stats->stats.InPktsDelayed++;
782	} else {
783	rxsc_stats->stats.InPktsOK++;
784	this_cpu_inc(rx_sa->stats->InPktsOK);
785	}
786	u64_stats_update_end(syncp: &rxsc_stats->syncp);
787
788	// Instead of "pn >=" - to support pn overflow in xpn
789	if (pn + 1 > rx_sa->next_pn_halves.lower) {
790	rx_sa->next_pn_halves.lower = pn + 1;
791	} else if (secy->xpn &&
792	!pn_same_half(pn, rx_sa->next_pn_halves.lower)) {
793	rx_sa->next_pn_halves.upper++;
794	rx_sa->next_pn_halves.lower = pn + 1;
795	}
796
797	spin_unlock(lock: &rx_sa->lock);
798	}
799
800	return true;
801	}
802
803	static void macsec_reset_skb(struct sk_buff *skb, struct net_device *dev)
804	{
805	skb->pkt_type = PACKET_HOST;
806	skb->protocol = eth_type_trans(skb, dev);
807
808	skb_reset_network_header(skb);
809	if (!skb_transport_header_was_set(skb))
810	skb_reset_transport_header(skb);
811	skb_reset_mac_len(skb);
812	}
813
814	static void macsec_finalize_skb(struct sk_buff *skb, u8 icv_len, u8 hdr_len)
815	{
816	skb->ip_summed = CHECKSUM_NONE;
817	memmove(skb->data + hdr_len, skb->data, 2 * ETH_ALEN);
818	skb_pull(skb, len: hdr_len);
819	pskb_trim_unique(skb, len: skb->len - icv_len);
820	}
821
822	static void count_rx(struct net_device *dev, int len)
823	{
824	dev_sw_netstats_rx_add(dev, len);
825	}
826
827	static void macsec_decrypt_done(void *data, int err)
828	{
829	struct sk_buff *skb = data;
830	struct net_device *dev = skb->dev;
831	struct macsec_dev *macsec = macsec_priv(dev);
832	struct macsec_rx_sa *rx_sa = macsec_skb_cb(skb)->rx_sa;
833	struct macsec_rx_sc *rx_sc = rx_sa->sc;
834	int len;
835	u32 pn;
836
837	aead_request_free(req: macsec_skb_cb(skb)->req);
838
839	if (!err)
840	macsec_skb_cb(skb)->valid = true;
841
842	rcu_read_lock_bh();
843	pn = ntohl(macsec_ethhdr(skb)->packet_number);
844	if (!macsec_post_decrypt(skb, secy: &macsec->secy, pn)) {
845	rcu_read_unlock_bh();
846	kfree_skb(skb);
847	goto out;
848	}
849
850	macsec_finalize_skb(skb, icv_len: macsec->secy.icv_len,
851	hdr_len: macsec_extra_len(sci_present: macsec_skb_cb(skb)->has_sci));
852	len = skb->len;
853	macsec_reset_skb(skb, dev: macsec->secy.netdev);
854
855	if (gro_cells_receive(gcells: &macsec->gro_cells, skb) == NET_RX_SUCCESS)
856	count_rx(dev, len);
857
858	rcu_read_unlock_bh();
859
860	out:
861	macsec_rxsa_put(sa: rx_sa);
862	macsec_rxsc_put(sc: rx_sc);
863	dev_put(dev);
864	}
865
866	static struct sk_buff *macsec_decrypt(struct sk_buff *skb,
867	struct net_device *dev,
868	struct macsec_rx_sa *rx_sa,
869	sci_t sci,
870	struct macsec_secy *secy)
871	{
872	int ret;
873	struct scatterlist *sg;
874	struct sk_buff *trailer;
875	unsigned char *iv;
876	struct aead_request *req;
877	struct macsec_eth_header *hdr;
878	u32 hdr_pn;
879	u16 icv_len = secy->icv_len;
880
881	macsec_skb_cb(skb)->valid = false;
882	skb = skb_share_check(skb, GFP_ATOMIC);
883	if (!skb)
884	return ERR_PTR(error: -ENOMEM);
885
886	ret = skb_cow_data(skb, tailbits: 0, trailer: &trailer);
887	if (unlikely(ret < 0)) {
888	kfree_skb(skb);
889	return ERR_PTR(error: ret);
890	}
891	req = macsec_alloc_req(tfm: rx_sa->key.tfm, iv: &iv, sg: &sg, num_frags: ret);
892	if (!req) {
893	kfree_skb(skb);
894	return ERR_PTR(error: -ENOMEM);
895	}
896
897	hdr = (struct macsec_eth_header *)skb->data;
898	hdr_pn = ntohl(hdr->packet_number);
899
900	if (secy->xpn) {
901	pn_t recovered_pn = rx_sa->next_pn_halves;
902
903	recovered_pn.lower = hdr_pn;
904	if (hdr_pn < rx_sa->next_pn_halves.lower &&
905	!pn_same_half(hdr_pn, rx_sa->next_pn_halves.lower))
906	recovered_pn.upper++;
907
908	macsec_fill_iv_xpn(iv, ssci: rx_sa->ssci, pn: recovered_pn.full64,
909	salt: rx_sa->key.salt);
910	} else {
911	macsec_fill_iv(iv, sci, pn: hdr_pn);
912	}
913
914	sg_init_table(sg, ret);
915	ret = skb_to_sgvec(skb, sg, offset: 0, len: skb->len);
916	if (unlikely(ret < 0)) {
917	aead_request_free(req);
918	kfree_skb(skb);
919	return ERR_PTR(error: ret);
920	}
921
922	if (hdr->tci_an & MACSEC_TCI_E) {
923	/* confidentiality: ethernet + macsec header
924	* authenticated, encrypted payload
925	*/
926	int len = skb->len - macsec_hdr_len(sci_present: macsec_skb_cb(skb)->has_sci);
927
928	aead_request_set_crypt(req, src: sg, dst: sg, cryptlen: len, iv);
929	aead_request_set_ad(req, assoclen: macsec_hdr_len(sci_present: macsec_skb_cb(skb)->has_sci));
930	skb = skb_unshare(skb, GFP_ATOMIC);
931	if (!skb) {
932	aead_request_free(req);
933	return ERR_PTR(error: -ENOMEM);
934	}
935	} else {
936	/* integrity only: all headers + data authenticated */
937	aead_request_set_crypt(req, src: sg, dst: sg, cryptlen: icv_len, iv);
938	aead_request_set_ad(req, assoclen: skb->len - icv_len);
939	}
940
941	macsec_skb_cb(skb)->req = req;
942	skb->dev = dev;
943	aead_request_set_callback(req, flags: 0, compl: macsec_decrypt_done, data: skb);
944
945	dev_hold(dev);
946	ret = crypto_aead_decrypt(req);
947	if (ret == -EINPROGRESS) {
948	return ERR_PTR(error: ret);
949	} else if (ret != 0) {
950	/* decryption/authentication failed
951	* 10.6 if validateFrames is disabled, deliver anyway
952	*/
953	if (ret != -EBADMSG) {
954	kfree_skb(skb);
955	skb = ERR_PTR(error: ret);
956	}
957	} else {
958	macsec_skb_cb(skb)->valid = true;
959	}
960	dev_put(dev);
961
962	aead_request_free(req);
963
964	return skb;
965	}
966
967	static struct macsec_rx_sc *find_rx_sc(struct macsec_secy *secy, sci_t sci)
968	{
969	struct macsec_rx_sc *rx_sc;
970
971	for_each_rxsc(secy, rx_sc) {
972	if (rx_sc->sci == sci)
973	return rx_sc;
974	}
975
976	return NULL;
977	}
978
979	static struct macsec_rx_sc *find_rx_sc_rtnl(struct macsec_secy *secy, sci_t sci)
980	{
981	struct macsec_rx_sc *rx_sc;
982
983	for_each_rxsc_rtnl(secy, rx_sc) {
984	if (rx_sc->sci == sci)
985	return rx_sc;
986	}
987
988	return NULL;
989	}
990
991	static enum rx_handler_result handle_not_macsec(struct sk_buff *skb)
992	{
993	/* Deliver to the uncontrolled port by default */
994	enum rx_handler_result ret = RX_HANDLER_PASS;
995	struct ethhdr *hdr = eth_hdr(skb);
996	struct metadata_dst *md_dst;
997	struct macsec_rxh_data *rxd;
998	struct macsec_dev *macsec;
999
1000	rcu_read_lock();
1001	rxd = macsec_data_rcu(dev: skb->dev);
1002	md_dst = skb_metadata_dst(skb);
1003
1004	list_for_each_entry_rcu(macsec, &rxd->secys, secys) {
1005	struct sk_buff *nskb;
1006	struct pcpu_secy_stats *secy_stats = this_cpu_ptr(macsec->stats);
1007	struct net_device *ndev = macsec->secy.netdev;
1008
1009	/* If h/w offloading is enabled, HW decodes frames and strips
1010	* the SecTAG, so we have to deduce which port to deliver to.
1011	*/
1012	if (macsec_is_offloaded(macsec) && netif_running(dev: ndev)) {
1013	struct macsec_rx_sc *rx_sc = NULL;
1014
1015	if (md_dst && md_dst->type == METADATA_MACSEC)
1016	rx_sc = find_rx_sc(secy: &macsec->secy, sci: md_dst->u.macsec_info.sci);
1017
1018	if (md_dst && md_dst->type == METADATA_MACSEC && !rx_sc)
1019	continue;
1020
1021	if (ether_addr_equal_64bits(addr1: hdr->h_dest,
1022	addr2: ndev->dev_addr)) {
1023	/* exact match, divert skb to this port */
1024	skb->dev = ndev;
1025	skb->pkt_type = PACKET_HOST;
1026	ret = RX_HANDLER_ANOTHER;
1027	goto out;
1028	} else if (is_multicast_ether_addr_64bits(
1029	addr: hdr->h_dest)) {
1030	/* multicast frame, deliver on this port too */
1031	nskb = skb_clone(skb, GFP_ATOMIC);
1032	if (!nskb)
1033	break;
1034
1035	nskb->dev = ndev;
1036	if (ether_addr_equal_64bits(addr1: hdr->h_dest,
1037	addr2: ndev->broadcast))
1038	nskb->pkt_type = PACKET_BROADCAST;
1039	else
1040	nskb->pkt_type = PACKET_MULTICAST;
1041
1042	__netif_rx(skb: nskb);
1043	} else if (rx_sc || ndev->flags & IFF_PROMISC) {
1044	skb->dev = ndev;
1045	skb->pkt_type = PACKET_HOST;
1046	ret = RX_HANDLER_ANOTHER;
1047	goto out;
1048	}
1049
1050	continue;
1051	}
1052
1053	/* 10.6 If the management control validateFrames is not
1054	* Strict, frames without a SecTAG are received, counted, and
1055	* delivered to the Controlled Port
1056	*/
1057	if (macsec->secy.validate_frames == MACSEC_VALIDATE_STRICT) {
1058	u64_stats_update_begin(syncp: &secy_stats->syncp);
1059	secy_stats->stats.InPktsNoTag++;
1060	u64_stats_update_end(syncp: &secy_stats->syncp);
1061	DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1062	continue;
1063	}
1064
1065	/* deliver on this port */
1066	nskb = skb_clone(skb, GFP_ATOMIC);
1067	if (!nskb)
1068	break;
1069
1070	nskb->dev = ndev;
1071
1072	if (__netif_rx(skb: nskb) == NET_RX_SUCCESS) {
1073	u64_stats_update_begin(syncp: &secy_stats->syncp);
1074	secy_stats->stats.InPktsUntagged++;
1075	u64_stats_update_end(syncp: &secy_stats->syncp);
1076	}
1077	}
1078
1079	out:
1080	rcu_read_unlock();
1081	return ret;
1082	}
1083
1084	static rx_handler_result_t macsec_handle_frame(struct sk_buff **pskb)
1085	{
1086	struct sk_buff *skb = *pskb;
1087	struct net_device *dev = skb->dev;
1088	struct macsec_eth_header *hdr;
1089	struct macsec_secy *secy = NULL;
1090	struct macsec_rx_sc *rx_sc;
1091	struct macsec_rx_sa *rx_sa;
1092	struct macsec_rxh_data *rxd;
1093	struct macsec_dev *macsec;
1094	unsigned int len;
1095	sci_t sci;
1096	u32 hdr_pn;
1097	bool cbit;
1098	struct pcpu_rx_sc_stats *rxsc_stats;
1099	struct pcpu_secy_stats *secy_stats;
1100	bool pulled_sci;
1101	int ret;
1102
1103	if (skb_headroom(skb) < ETH_HLEN)
1104	goto drop_direct;
1105
1106	hdr = macsec_ethhdr(skb);
1107	if (hdr->eth.h_proto != htons(ETH_P_MACSEC))
1108	return handle_not_macsec(skb);
1109
1110	skb = skb_unshare(skb, GFP_ATOMIC);
1111	*pskb = skb;
1112	if (!skb)
1113	return RX_HANDLER_CONSUMED;
1114
1115	pulled_sci = pskb_may_pull(skb, len: macsec_extra_len(sci_present: true));
1116	if (!pulled_sci) {
1117	if (!pskb_may_pull(skb, len: macsec_extra_len(sci_present: false)))
1118	goto drop_direct;
1119	}
1120
1121	hdr = macsec_ethhdr(skb);
1122
1123	/* Frames with a SecTAG that has the TCI E bit set but the C
1124	* bit clear are discarded, as this reserved encoding is used
1125	* to identify frames with a SecTAG that are not to be
1126	* delivered to the Controlled Port.
1127	*/
1128	if ((hdr->tci_an & (MACSEC_TCI_C | MACSEC_TCI_E)) == MACSEC_TCI_E)
1129	return RX_HANDLER_PASS;
1130
1131	/* now, pull the extra length */
1132	if (hdr->tci_an & MACSEC_TCI_SC) {
1133	if (!pulled_sci)
1134	goto drop_direct;
1135	}
1136
1137	/* ethernet header is part of crypto processing */
1138	skb_push(skb, ETH_HLEN);
1139
1140	macsec_skb_cb(skb)->has_sci = !!(hdr->tci_an & MACSEC_TCI_SC);
1141	macsec_skb_cb(skb)->assoc_num = hdr->tci_an & MACSEC_AN_MASK;
1142	sci = macsec_frame_sci(hdr, sci_present: macsec_skb_cb(skb)->has_sci);
1143
1144	rcu_read_lock();
1145	rxd = macsec_data_rcu(dev: skb->dev);
1146
1147	list_for_each_entry_rcu(macsec, &rxd->secys, secys) {
1148	struct macsec_rx_sc *sc = find_rx_sc(secy: &macsec->secy, sci);
1149
1150	sc = sc ? macsec_rxsc_get(sc) : NULL;
1151
1152	if (sc) {
1153	secy = &macsec->secy;
1154	rx_sc = sc;
1155	break;
1156	}
1157	}
1158
1159	if (!secy)
1160	goto nosci;
1161
1162	dev = secy->netdev;
1163	macsec = macsec_priv(dev);
1164	secy_stats = this_cpu_ptr(macsec->stats);
1165	rxsc_stats = this_cpu_ptr(rx_sc->stats);
1166
1167	if (!macsec_validate_skb(skb, icv_len: secy->icv_len, xpn: secy->xpn)) {
1168	u64_stats_update_begin(syncp: &secy_stats->syncp);
1169	secy_stats->stats.InPktsBadTag++;
1170	u64_stats_update_end(syncp: &secy_stats->syncp);
1171	DEV_STATS_INC(secy->netdev, rx_errors);
1172	goto drop_nosa;
1173	}
1174
1175	rx_sa = macsec_rxsa_get(ptr: rx_sc->sa[macsec_skb_cb(skb)->assoc_num]);
1176	if (!rx_sa) {
1177	/* 10.6.1 if the SA is not in use */
1178
1179	/* If validateFrames is Strict or the C bit in the
1180	* SecTAG is set, discard
1181	*/
1182	struct macsec_rx_sa *active_rx_sa = macsec_active_rxsa_get(rx_sc);
1183	if (hdr->tci_an & MACSEC_TCI_C ||
1184	secy->validate_frames == MACSEC_VALIDATE_STRICT) {
1185	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
1186	rxsc_stats->stats.InPktsNotUsingSA++;
1187	u64_stats_update_end(syncp: &rxsc_stats->syncp);
1188	DEV_STATS_INC(secy->netdev, rx_errors);
1189	if (active_rx_sa)
1190	this_cpu_inc(active_rx_sa->stats->InPktsNotUsingSA);
1191	goto drop_nosa;
1192	}
1193
1194	/* not Strict, the frame (with the SecTAG and ICV
1195	* removed) is delivered to the Controlled Port.
1196	*/
1197	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
1198	rxsc_stats->stats.InPktsUnusedSA++;
1199	u64_stats_update_end(syncp: &rxsc_stats->syncp);
1200	if (active_rx_sa)
1201	this_cpu_inc(active_rx_sa->stats->InPktsUnusedSA);
1202	goto deliver;
1203	}
1204
1205	/* First, PN check to avoid decrypting obviously wrong packets */
1206	hdr_pn = ntohl(hdr->packet_number);
1207	if (secy->replay_protect) {
1208	bool late;
1209
1210	spin_lock(lock: &rx_sa->lock);
1211	late = rx_sa->next_pn_halves.lower >= secy->replay_window &&
1212	hdr_pn < (rx_sa->next_pn_halves.lower - secy->replay_window);
1213
1214	if (secy->xpn)
1215	late = late && pn_same_half(rx_sa->next_pn_halves.lower, hdr_pn);
1216	spin_unlock(lock: &rx_sa->lock);
1217
1218	if (late) {
1219	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
1220	rxsc_stats->stats.InPktsLate++;
1221	u64_stats_update_end(syncp: &rxsc_stats->syncp);
1222	DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1223	goto drop;
1224	}
1225	}
1226
1227	macsec_skb_cb(skb)->rx_sa = rx_sa;
1228
1229	/* Disabled && !changed text => skip validation */
1230	if (hdr->tci_an & MACSEC_TCI_C ||
1231	secy->validate_frames != MACSEC_VALIDATE_DISABLED)
1232	skb = macsec_decrypt(skb, dev, rx_sa, sci, secy);
1233
1234	if (IS_ERR(ptr: skb)) {
1235	/* the decrypt callback needs the reference */
1236	if (PTR_ERR(ptr: skb) != -EINPROGRESS) {
1237	macsec_rxsa_put(sa: rx_sa);
1238	macsec_rxsc_put(sc: rx_sc);
1239	}
1240	rcu_read_unlock();
1241	*pskb = NULL;
1242	return RX_HANDLER_CONSUMED;
1243	}
1244
1245	if (!macsec_post_decrypt(skb, secy, pn: hdr_pn))
1246	goto drop;
1247
1248	deliver:
1249	macsec_finalize_skb(skb, icv_len: secy->icv_len,
1250	hdr_len: macsec_extra_len(sci_present: macsec_skb_cb(skb)->has_sci));
1251	len = skb->len;
1252	macsec_reset_skb(skb, dev: secy->netdev);
1253
1254	if (rx_sa)
1255	macsec_rxsa_put(sa: rx_sa);
1256	macsec_rxsc_put(sc: rx_sc);
1257
1258	skb_orphan(skb);
1259	ret = gro_cells_receive(gcells: &macsec->gro_cells, skb);
1260	if (ret == NET_RX_SUCCESS)
1261	count_rx(dev, len);
1262	else
1263	DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1264
1265	rcu_read_unlock();
1266
1267	*pskb = NULL;
1268	return RX_HANDLER_CONSUMED;
1269
1270	drop:
1271	macsec_rxsa_put(sa: rx_sa);
1272	drop_nosa:
1273	macsec_rxsc_put(sc: rx_sc);
1274	rcu_read_unlock();
1275	drop_direct:
1276	kfree_skb(skb);
1277	*pskb = NULL;
1278	return RX_HANDLER_CONSUMED;
1279
1280	nosci:
1281	/* 10.6.1 if the SC is not found */
1282	cbit = !!(hdr->tci_an & MACSEC_TCI_C);
1283	if (!cbit)
1284	macsec_finalize_skb(skb, MACSEC_DEFAULT_ICV_LEN,
1285	hdr_len: macsec_extra_len(sci_present: macsec_skb_cb(skb)->has_sci));
1286
1287	list_for_each_entry_rcu(macsec, &rxd->secys, secys) {
1288	struct sk_buff *nskb;
1289
1290	secy_stats = this_cpu_ptr(macsec->stats);
1291
1292	/* If validateFrames is Strict or the C bit in the
1293	* SecTAG is set, discard
1294	*/
1295	if (cbit ||
1296	macsec->secy.validate_frames == MACSEC_VALIDATE_STRICT) {
1297	u64_stats_update_begin(syncp: &secy_stats->syncp);
1298	secy_stats->stats.InPktsNoSCI++;
1299	u64_stats_update_end(syncp: &secy_stats->syncp);
1300	DEV_STATS_INC(macsec->secy.netdev, rx_errors);
1301	continue;
1302	}
1303
1304	/* not strict, the frame (with the SecTAG and ICV
1305	* removed) is delivered to the Controlled Port.
1306	*/
1307	nskb = skb_clone(skb, GFP_ATOMIC);
1308	if (!nskb)
1309	break;
1310
1311	macsec_reset_skb(skb: nskb, dev: macsec->secy.netdev);
1312
1313	ret = __netif_rx(skb: nskb);
1314	if (ret == NET_RX_SUCCESS) {
1315	u64_stats_update_begin(syncp: &secy_stats->syncp);
1316	secy_stats->stats.InPktsUnknownSCI++;
1317	u64_stats_update_end(syncp: &secy_stats->syncp);
1318	} else {
1319	DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1320	}
1321	}
1322
1323	rcu_read_unlock();
1324	*pskb = skb;
1325	return RX_HANDLER_PASS;
1326	}
1327
1328	static struct crypto_aead *macsec_alloc_tfm(char *key, int key_len, int icv_len)
1329	{
1330	struct crypto_aead *tfm;
1331	int ret;
1332
1333	tfm = crypto_alloc_aead(alg_name: "gcm(aes)", type: 0, mask: 0);
1334
1335	if (IS_ERR(ptr: tfm))
1336	return tfm;
1337
1338	ret = crypto_aead_setkey(tfm, key, keylen: key_len);
1339	if (ret < 0)
1340	goto fail;
1341
1342	ret = crypto_aead_setauthsize(tfm, authsize: icv_len);
1343	if (ret < 0)
1344	goto fail;
1345
1346	return tfm;
1347	fail:
1348	crypto_free_aead(tfm);
1349	return ERR_PTR(error: ret);
1350	}
1351
1352	static int init_rx_sa(struct macsec_rx_sa *rx_sa, char *sak, int key_len,
1353	int icv_len)
1354	{
1355	rx_sa->stats = alloc_percpu(struct macsec_rx_sa_stats);
1356	if (!rx_sa->stats)
1357	return -ENOMEM;
1358
1359	rx_sa->key.tfm = macsec_alloc_tfm(key: sak, key_len, icv_len);
1360	if (IS_ERR(ptr: rx_sa->key.tfm)) {
1361	free_percpu(pdata: rx_sa->stats);
1362	return PTR_ERR(ptr: rx_sa->key.tfm);
1363	}
1364
1365	rx_sa->ssci = MACSEC_UNDEF_SSCI;
1366	rx_sa->active = false;
1367	rx_sa->next_pn = 1;
1368	refcount_set(r: &rx_sa->refcnt, n: 1);
1369	spin_lock_init(&rx_sa->lock);
1370
1371	return 0;
1372	}
1373
1374	static void clear_rx_sa(struct macsec_rx_sa *rx_sa)
1375	{
1376	rx_sa->active = false;
1377
1378	macsec_rxsa_put(sa: rx_sa);
1379	}
1380
1381	static void free_rx_sc(struct macsec_rx_sc *rx_sc)
1382	{
1383	int i;
1384
1385	for (i = 0; i < MACSEC_NUM_AN; i++) {
1386	struct macsec_rx_sa *sa = rtnl_dereference(rx_sc->sa[i]);
1387
1388	RCU_INIT_POINTER(rx_sc->sa[i], NULL);
1389	if (sa)
1390	clear_rx_sa(rx_sa: sa);
1391	}
1392
1393	macsec_rxsc_put(sc: rx_sc);
1394	}
1395
1396	static struct macsec_rx_sc *del_rx_sc(struct macsec_secy *secy, sci_t sci)
1397	{
1398	struct macsec_rx_sc *rx_sc, __rcu **rx_scp;
1399
1400	for (rx_scp = &secy->rx_sc, rx_sc = rtnl_dereference(*rx_scp);
1401	rx_sc;
1402	rx_scp = &rx_sc->next, rx_sc = rtnl_dereference(*rx_scp)) {
1403	if (rx_sc->sci == sci) {
1404	if (rx_sc->active)
1405	secy->n_rx_sc--;
1406	rcu_assign_pointer(*rx_scp, rx_sc->next);
1407	return rx_sc;
1408	}
1409	}
1410
1411	return NULL;
1412	}
1413
1414	static struct macsec_rx_sc *create_rx_sc(struct net_device *dev, sci_t sci,
1415	bool active)
1416	{
1417	struct macsec_rx_sc *rx_sc;
1418	struct macsec_dev *macsec;
1419	struct net_device *real_dev = macsec_priv(dev)->real_dev;
1420	struct macsec_rxh_data *rxd = macsec_data_rtnl(dev: real_dev);
1421	struct macsec_secy *secy;
1422
1423	list_for_each_entry(macsec, &rxd->secys, secys) {
1424	if (find_rx_sc_rtnl(secy: &macsec->secy, sci))
1425	return ERR_PTR(error: -EEXIST);
1426	}
1427
1428	rx_sc = kzalloc(size: sizeof(*rx_sc), GFP_KERNEL);
1429	if (!rx_sc)
1430	return ERR_PTR(error: -ENOMEM);
1431
1432	rx_sc->stats = netdev_alloc_pcpu_stats(struct pcpu_rx_sc_stats);
1433	if (!rx_sc->stats) {
1434	kfree(objp: rx_sc);
1435	return ERR_PTR(error: -ENOMEM);
1436	}
1437
1438	rx_sc->sci = sci;
1439	rx_sc->active = active;
1440	refcount_set(r: &rx_sc->refcnt, n: 1);
1441
1442	secy = &macsec_priv(dev)->secy;
1443	rcu_assign_pointer(rx_sc->next, secy->rx_sc);
1444	rcu_assign_pointer(secy->rx_sc, rx_sc);
1445
1446	if (rx_sc->active)
1447	secy->n_rx_sc++;
1448
1449	return rx_sc;
1450	}
1451
1452	static int init_tx_sa(struct macsec_tx_sa *tx_sa, char *sak, int key_len,
1453	int icv_len)
1454	{
1455	tx_sa->stats = alloc_percpu(struct macsec_tx_sa_stats);
1456	if (!tx_sa->stats)
1457	return -ENOMEM;
1458
1459	tx_sa->key.tfm = macsec_alloc_tfm(key: sak, key_len, icv_len);
1460	if (IS_ERR(ptr: tx_sa->key.tfm)) {
1461	free_percpu(pdata: tx_sa->stats);
1462	return PTR_ERR(ptr: tx_sa->key.tfm);
1463	}
1464
1465	tx_sa->ssci = MACSEC_UNDEF_SSCI;
1466	tx_sa->active = false;
1467	refcount_set(r: &tx_sa->refcnt, n: 1);
1468	spin_lock_init(&tx_sa->lock);
1469
1470	return 0;
1471	}
1472
1473	static void clear_tx_sa(struct macsec_tx_sa *tx_sa)
1474	{
1475	tx_sa->active = false;
1476
1477	macsec_txsa_put(sa: tx_sa);
1478	}
1479
1480	static struct genl_family macsec_fam;
1481
1482	static struct net_device *get_dev_from_nl(struct net *net,
1483	struct nlattr **attrs)
1484	{
1485	int ifindex = nla_get_u32(nla: attrs[MACSEC_ATTR_IFINDEX]);
1486	struct net_device *dev;
1487
1488	dev = __dev_get_by_index(net, ifindex);
1489	if (!dev)
1490	return ERR_PTR(error: -ENODEV);
1491
1492	if (!netif_is_macsec(dev))
1493	return ERR_PTR(error: -ENODEV);
1494
1495	return dev;
1496	}
1497
1498	static enum macsec_offload nla_get_offload(const struct nlattr *nla)
1499	{
1500	return (__force enum macsec_offload)nla_get_u8(nla);
1501	}
1502
1503	static sci_t nla_get_sci(const struct nlattr *nla)
1504	{
1505	return (__force sci_t)nla_get_u64(nla);
1506	}
1507
1508	static int nla_put_sci(struct sk_buff *skb, int attrtype, sci_t value,
1509	int padattr)
1510	{
1511	return nla_put_u64_64bit(skb, attrtype, value: (__force u64)value, padattr);
1512	}
1513
1514	static ssci_t nla_get_ssci(const struct nlattr *nla)
1515	{
1516	return (__force ssci_t)nla_get_u32(nla);
1517	}
1518
1519	static int nla_put_ssci(struct sk_buff *skb, int attrtype, ssci_t value)
1520	{
1521	return nla_put_u32(skb, attrtype, value: (__force u64)value);
1522	}
1523
1524	static struct macsec_tx_sa *get_txsa_from_nl(struct net *net,
1525	struct nlattr **attrs,
1526	struct nlattr **tb_sa,
1527	struct net_device **devp,
1528	struct macsec_secy **secyp,
1529	struct macsec_tx_sc **scp,
1530	u8 *assoc_num)
1531	{
1532	struct net_device *dev;
1533	struct macsec_secy *secy;
1534	struct macsec_tx_sc *tx_sc;
1535	struct macsec_tx_sa *tx_sa;
1536
1537	if (!tb_sa[MACSEC_SA_ATTR_AN])
1538	return ERR_PTR(error: -EINVAL);
1539
1540	*assoc_num = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_AN]);
1541
1542	dev = get_dev_from_nl(net, attrs);
1543	if (IS_ERR(ptr: dev))
1544	return ERR_CAST(ptr: dev);
1545
1546	if (*assoc_num >= MACSEC_NUM_AN)
1547	return ERR_PTR(error: -EINVAL);
1548
1549	secy = &macsec_priv(dev)->secy;
1550	tx_sc = &secy->tx_sc;
1551
1552	tx_sa = rtnl_dereference(tx_sc->sa[*assoc_num]);
1553	if (!tx_sa)
1554	return ERR_PTR(error: -ENODEV);
1555
1556	*devp = dev;
1557	*scp = tx_sc;
1558	*secyp = secy;
1559	return tx_sa;
1560	}
1561
1562	static struct macsec_rx_sc *get_rxsc_from_nl(struct net *net,
1563	struct nlattr **attrs,
1564	struct nlattr **tb_rxsc,
1565	struct net_device **devp,
1566	struct macsec_secy **secyp)
1567	{
1568	struct net_device *dev;
1569	struct macsec_secy *secy;
1570	struct macsec_rx_sc *rx_sc;
1571	sci_t sci;
1572
1573	dev = get_dev_from_nl(net, attrs);
1574	if (IS_ERR(ptr: dev))
1575	return ERR_CAST(ptr: dev);
1576
1577	secy = &macsec_priv(dev)->secy;
1578
1579	if (!tb_rxsc[MACSEC_RXSC_ATTR_SCI])
1580	return ERR_PTR(error: -EINVAL);
1581
1582	sci = nla_get_sci(nla: tb_rxsc[MACSEC_RXSC_ATTR_SCI]);
1583	rx_sc = find_rx_sc_rtnl(secy, sci);
1584	if (!rx_sc)
1585	return ERR_PTR(error: -ENODEV);
1586
1587	*secyp = secy;
1588	*devp = dev;
1589
1590	return rx_sc;
1591	}
1592
1593	static struct macsec_rx_sa *get_rxsa_from_nl(struct net *net,
1594	struct nlattr **attrs,
1595	struct nlattr **tb_rxsc,
1596	struct nlattr **tb_sa,
1597	struct net_device **devp,
1598	struct macsec_secy **secyp,
1599	struct macsec_rx_sc **scp,
1600	u8 *assoc_num)
1601	{
1602	struct macsec_rx_sc *rx_sc;
1603	struct macsec_rx_sa *rx_sa;
1604
1605	if (!tb_sa[MACSEC_SA_ATTR_AN])
1606	return ERR_PTR(error: -EINVAL);
1607
1608	*assoc_num = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_AN]);
1609	if (*assoc_num >= MACSEC_NUM_AN)
1610	return ERR_PTR(error: -EINVAL);
1611
1612	rx_sc = get_rxsc_from_nl(net, attrs, tb_rxsc, devp, secyp);
1613	if (IS_ERR(ptr: rx_sc))
1614	return ERR_CAST(ptr: rx_sc);
1615
1616	rx_sa = rtnl_dereference(rx_sc->sa[*assoc_num]);
1617	if (!rx_sa)
1618	return ERR_PTR(error: -ENODEV);
1619
1620	*scp = rx_sc;
1621	return rx_sa;
1622	}
1623
1624	static const struct nla_policy macsec_genl_policy[NUM_MACSEC_ATTR] = {
1625	[MACSEC_ATTR_IFINDEX] = { .type = NLA_U32 },
1626	[MACSEC_ATTR_RXSC_CONFIG] = { .type = NLA_NESTED },
1627	[MACSEC_ATTR_SA_CONFIG] = { .type = NLA_NESTED },
1628	[MACSEC_ATTR_OFFLOAD] = { .type = NLA_NESTED },
1629	};
1630
1631	static const struct nla_policy macsec_genl_rxsc_policy[NUM_MACSEC_RXSC_ATTR] = {
1632	[MACSEC_RXSC_ATTR_SCI] = { .type = NLA_U64 },
1633	[MACSEC_RXSC_ATTR_ACTIVE] = { .type = NLA_U8 },
1634	};
1635
1636	static const struct nla_policy macsec_genl_sa_policy[NUM_MACSEC_SA_ATTR] = {
1637	[MACSEC_SA_ATTR_AN] = { .type = NLA_U8 },
1638	[MACSEC_SA_ATTR_ACTIVE] = { .type = NLA_U8 },
1639	[MACSEC_SA_ATTR_PN] = NLA_POLICY_MIN_LEN(4),
1640	[MACSEC_SA_ATTR_KEYID] = { .type = NLA_BINARY,
1641	.len = MACSEC_KEYID_LEN, },
1642	[MACSEC_SA_ATTR_KEY] = { .type = NLA_BINARY,
1643	.len = MACSEC_MAX_KEY_LEN, },
1644	[MACSEC_SA_ATTR_SSCI] = { .type = NLA_U32 },
1645	[MACSEC_SA_ATTR_SALT] = { .type = NLA_BINARY,
1646	.len = MACSEC_SALT_LEN, },
1647	};
1648
1649	static const struct nla_policy macsec_genl_offload_policy[NUM_MACSEC_OFFLOAD_ATTR] = {
1650	[MACSEC_OFFLOAD_ATTR_TYPE] = { .type = NLA_U8 },
1651	};
1652
1653	/* Offloads an operation to a device driver */
1654	static int macsec_offload(int (* const func)(struct macsec_context *),
1655	struct macsec_context *ctx)
1656	{
1657	int ret;
1658
1659	if (unlikely(!func))
1660	return 0;
1661
1662	if (ctx->offload == MACSEC_OFFLOAD_PHY)
1663	mutex_lock(&ctx->phydev->lock);
1664
1665	ret = (*func)(ctx);
1666
1667	if (ctx->offload == MACSEC_OFFLOAD_PHY)
1668	mutex_unlock(lock: &ctx->phydev->lock);
1669
1670	return ret;
1671	}
1672
1673	static int parse_sa_config(struct nlattr **attrs, struct nlattr **tb_sa)
1674	{
1675	if (!attrs[MACSEC_ATTR_SA_CONFIG])
1676	return -EINVAL;
1677
1678	if (nla_parse_nested_deprecated(tb: tb_sa, maxtype: MACSEC_SA_ATTR_MAX, nla: attrs[MACSEC_ATTR_SA_CONFIG], policy: macsec_genl_sa_policy, NULL))
1679	return -EINVAL;
1680
1681	return 0;
1682	}
1683
1684	static int parse_rxsc_config(struct nlattr **attrs, struct nlattr **tb_rxsc)
1685	{
1686	if (!attrs[MACSEC_ATTR_RXSC_CONFIG])
1687	return -EINVAL;
1688
1689	if (nla_parse_nested_deprecated(tb: tb_rxsc, maxtype: MACSEC_RXSC_ATTR_MAX, nla: attrs[MACSEC_ATTR_RXSC_CONFIG], policy: macsec_genl_rxsc_policy, NULL))
1690	return -EINVAL;
1691
1692	return 0;
1693	}
1694
1695	static bool validate_add_rxsa(struct nlattr **attrs)
1696	{
1697	if (!attrs[MACSEC_SA_ATTR_AN] ||
1698	!attrs[MACSEC_SA_ATTR_KEY] ||
1699	!attrs[MACSEC_SA_ATTR_KEYID])
1700	return false;
1701
1702	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_AN]) >= MACSEC_NUM_AN)
1703	return false;
1704
1705	if (attrs[MACSEC_SA_ATTR_PN] &&
1706	nla_get_u64(nla: attrs[MACSEC_SA_ATTR_PN]) == 0)
1707	return false;
1708
1709	if (attrs[MACSEC_SA_ATTR_ACTIVE]) {
1710	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_ACTIVE]) > 1)
1711	return false;
1712	}
1713
1714	if (nla_len(nla: attrs[MACSEC_SA_ATTR_KEYID]) != MACSEC_KEYID_LEN)
1715	return false;
1716
1717	return true;
1718	}
1719
1720	static int macsec_add_rxsa(struct sk_buff *skb, struct genl_info *info)
1721	{
1722	struct net_device *dev;
1723	struct nlattr **attrs = info->attrs;
1724	struct macsec_secy *secy;
1725	struct macsec_rx_sc *rx_sc;
1726	struct macsec_rx_sa *rx_sa;
1727	unsigned char assoc_num;
1728	int pn_len;
1729	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
1730	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
1731	int err;
1732
1733	if (!attrs[MACSEC_ATTR_IFINDEX])
1734	return -EINVAL;
1735
1736	if (parse_sa_config(attrs, tb_sa))
1737	return -EINVAL;
1738
1739	if (parse_rxsc_config(attrs, tb_rxsc))
1740	return -EINVAL;
1741
1742	if (!validate_add_rxsa(attrs: tb_sa))
1743	return -EINVAL;
1744
1745	rtnl_lock();
1746	rx_sc = get_rxsc_from_nl(net: genl_info_net(info), attrs, tb_rxsc, devp: &dev, secyp: &secy);
1747	if (IS_ERR(ptr: rx_sc)) {
1748	rtnl_unlock();
1749	return PTR_ERR(ptr: rx_sc);
1750	}
1751
1752	assoc_num = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_AN]);
1753
1754	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_KEY]) != secy->key_len) {
1755	pr_notice("macsec: nl: add_rxsa: bad key length: %d != %d\n",
1756	nla_len(tb_sa[MACSEC_SA_ATTR_KEY]), secy->key_len);
1757	rtnl_unlock();
1758	return -EINVAL;
1759	}
1760
1761	pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
1762	if (tb_sa[MACSEC_SA_ATTR_PN] &&
1763	nla_len(nla: tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
1764	pr_notice("macsec: nl: add_rxsa: bad pn length: %d != %d\n",
1765	nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
1766	rtnl_unlock();
1767	return -EINVAL;
1768	}
1769
1770	if (secy->xpn) {
1771	if (!tb_sa[MACSEC_SA_ATTR_SSCI] || !tb_sa[MACSEC_SA_ATTR_SALT]) {
1772	rtnl_unlock();
1773	return -EINVAL;
1774	}
1775
1776	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_SALT]) != MACSEC_SALT_LEN) {
1777	pr_notice("macsec: nl: add_rxsa: bad salt length: %d != %d\n",
1778	nla_len(tb_sa[MACSEC_SA_ATTR_SALT]),
1779	MACSEC_SALT_LEN);
1780	rtnl_unlock();
1781	return -EINVAL;
1782	}
1783	}
1784
1785	rx_sa = rtnl_dereference(rx_sc->sa[assoc_num]);
1786	if (rx_sa) {
1787	rtnl_unlock();
1788	return -EBUSY;
1789	}
1790
1791	rx_sa = kmalloc(size: sizeof(*rx_sa), GFP_KERNEL);
1792	if (!rx_sa) {
1793	rtnl_unlock();
1794	return -ENOMEM;
1795	}
1796
1797	err = init_rx_sa(rx_sa, sak: nla_data(nla: tb_sa[MACSEC_SA_ATTR_KEY]),
1798	key_len: secy->key_len, icv_len: secy->icv_len);
1799	if (err < 0) {
1800	kfree(objp: rx_sa);
1801	rtnl_unlock();
1802	return err;
1803	}
1804
1805	if (tb_sa[MACSEC_SA_ATTR_PN]) {
1806	spin_lock_bh(lock: &rx_sa->lock);
1807	rx_sa->next_pn = nla_get_u64(nla: tb_sa[MACSEC_SA_ATTR_PN]);
1808	spin_unlock_bh(lock: &rx_sa->lock);
1809	}
1810
1811	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
1812	rx_sa->active = !!nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_ACTIVE]);
1813
1814	rx_sa->sc = rx_sc;
1815
1816	if (secy->xpn) {
1817	rx_sa->ssci = nla_get_ssci(nla: tb_sa[MACSEC_SA_ATTR_SSCI]);
1818	nla_memcpy(dest: rx_sa->key.salt.bytes, src: tb_sa[MACSEC_SA_ATTR_SALT],
1819	MACSEC_SALT_LEN);
1820	}
1821
1822	/* If h/w offloading is available, propagate to the device */
1823	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
1824	const struct macsec_ops *ops;
1825	struct macsec_context ctx;
1826
1827	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
1828	if (!ops) {
1829	err = -EOPNOTSUPP;
1830	goto cleanup;
1831	}
1832
1833	ctx.sa.assoc_num = assoc_num;
1834	ctx.sa.rx_sa = rx_sa;
1835	ctx.secy = secy;
1836	memcpy(ctx.sa.key, nla_data(tb_sa[MACSEC_SA_ATTR_KEY]),
1837	secy->key_len);
1838
1839	err = macsec_offload(func: ops->mdo_add_rxsa, ctx: &ctx);
1840	memzero_explicit(s: ctx.sa.key, count: secy->key_len);
1841	if (err)
1842	goto cleanup;
1843	}
1844
1845	nla_memcpy(dest: rx_sa->key.id, src: tb_sa[MACSEC_SA_ATTR_KEYID], MACSEC_KEYID_LEN);
1846	rcu_assign_pointer(rx_sc->sa[assoc_num], rx_sa);
1847
1848	rtnl_unlock();
1849
1850	return 0;
1851
1852	cleanup:
1853	macsec_rxsa_put(sa: rx_sa);
1854	rtnl_unlock();
1855	return err;
1856	}
1857
1858	static bool validate_add_rxsc(struct nlattr **attrs)
1859	{
1860	if (!attrs[MACSEC_RXSC_ATTR_SCI])
1861	return false;
1862
1863	if (attrs[MACSEC_RXSC_ATTR_ACTIVE]) {
1864	if (nla_get_u8(nla: attrs[MACSEC_RXSC_ATTR_ACTIVE]) > 1)
1865	return false;
1866	}
1867
1868	return true;
1869	}
1870
1871	static int macsec_add_rxsc(struct sk_buff *skb, struct genl_info *info)
1872	{
1873	struct net_device *dev;
1874	sci_t sci = MACSEC_UNDEF_SCI;
1875	struct nlattr **attrs = info->attrs;
1876	struct macsec_rx_sc *rx_sc;
1877	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
1878	struct macsec_secy *secy;
1879	bool active = true;
1880	int ret;
1881
1882	if (!attrs[MACSEC_ATTR_IFINDEX])
1883	return -EINVAL;
1884
1885	if (parse_rxsc_config(attrs, tb_rxsc))
1886	return -EINVAL;
1887
1888	if (!validate_add_rxsc(attrs: tb_rxsc))
1889	return -EINVAL;
1890
1891	rtnl_lock();
1892	dev = get_dev_from_nl(net: genl_info_net(info), attrs);
1893	if (IS_ERR(ptr: dev)) {
1894	rtnl_unlock();
1895	return PTR_ERR(ptr: dev);
1896	}
1897
1898	secy = &macsec_priv(dev)->secy;
1899	sci = nla_get_sci(nla: tb_rxsc[MACSEC_RXSC_ATTR_SCI]);
1900
1901	if (tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE])
1902	active = nla_get_u8(nla: tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]);
1903
1904	rx_sc = create_rx_sc(dev, sci, active);
1905	if (IS_ERR(ptr: rx_sc)) {
1906	rtnl_unlock();
1907	return PTR_ERR(ptr: rx_sc);
1908	}
1909
1910	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
1911	const struct macsec_ops *ops;
1912	struct macsec_context ctx;
1913
1914	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
1915	if (!ops) {
1916	ret = -EOPNOTSUPP;
1917	goto cleanup;
1918	}
1919
1920	ctx.rx_sc = rx_sc;
1921	ctx.secy = secy;
1922
1923	ret = macsec_offload(func: ops->mdo_add_rxsc, ctx: &ctx);
1924	if (ret)
1925	goto cleanup;
1926	}
1927
1928	rtnl_unlock();
1929
1930	return 0;
1931
1932	cleanup:
1933	del_rx_sc(secy, sci);
1934	free_rx_sc(rx_sc);
1935	rtnl_unlock();
1936	return ret;
1937	}
1938
1939	static bool validate_add_txsa(struct nlattr **attrs)
1940	{
1941	if (!attrs[MACSEC_SA_ATTR_AN] ||
1942	!attrs[MACSEC_SA_ATTR_PN] ||
1943	!attrs[MACSEC_SA_ATTR_KEY] ||
1944	!attrs[MACSEC_SA_ATTR_KEYID])
1945	return false;
1946
1947	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_AN]) >= MACSEC_NUM_AN)
1948	return false;
1949
1950	if (nla_get_u64(nla: attrs[MACSEC_SA_ATTR_PN]) == 0)
1951	return false;
1952
1953	if (attrs[MACSEC_SA_ATTR_ACTIVE]) {
1954	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_ACTIVE]) > 1)
1955	return false;
1956	}
1957
1958	if (nla_len(nla: attrs[MACSEC_SA_ATTR_KEYID]) != MACSEC_KEYID_LEN)
1959	return false;
1960
1961	return true;
1962	}
1963
1964	static int macsec_add_txsa(struct sk_buff *skb, struct genl_info *info)
1965	{
1966	struct net_device *dev;
1967	struct nlattr **attrs = info->attrs;
1968	struct macsec_secy *secy;
1969	struct macsec_tx_sc *tx_sc;
1970	struct macsec_tx_sa *tx_sa;
1971	unsigned char assoc_num;
1972	int pn_len;
1973	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
1974	bool was_operational;
1975	int err;
1976
1977	if (!attrs[MACSEC_ATTR_IFINDEX])
1978	return -EINVAL;
1979
1980	if (parse_sa_config(attrs, tb_sa))
1981	return -EINVAL;
1982
1983	if (!validate_add_txsa(attrs: tb_sa))
1984	return -EINVAL;
1985
1986	rtnl_lock();
1987	dev = get_dev_from_nl(net: genl_info_net(info), attrs);
1988	if (IS_ERR(ptr: dev)) {
1989	rtnl_unlock();
1990	return PTR_ERR(ptr: dev);
1991	}
1992
1993	secy = &macsec_priv(dev)->secy;
1994	tx_sc = &secy->tx_sc;
1995
1996	assoc_num = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_AN]);
1997
1998	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_KEY]) != secy->key_len) {
1999	pr_notice("macsec: nl: add_txsa: bad key length: %d != %d\n",
2000	nla_len(tb_sa[MACSEC_SA_ATTR_KEY]), secy->key_len);
2001	rtnl_unlock();
2002	return -EINVAL;
2003	}
2004
2005	pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
2006	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
2007	pr_notice("macsec: nl: add_txsa: bad pn length: %d != %d\n",
2008	nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
2009	rtnl_unlock();
2010	return -EINVAL;
2011	}
2012
2013	if (secy->xpn) {
2014	if (!tb_sa[MACSEC_SA_ATTR_SSCI] || !tb_sa[MACSEC_SA_ATTR_SALT]) {
2015	rtnl_unlock();
2016	return -EINVAL;
2017	}
2018
2019	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_SALT]) != MACSEC_SALT_LEN) {
2020	pr_notice("macsec: nl: add_txsa: bad salt length: %d != %d\n",
2021	nla_len(tb_sa[MACSEC_SA_ATTR_SALT]),
2022	MACSEC_SALT_LEN);
2023	rtnl_unlock();
2024	return -EINVAL;
2025	}
2026	}
2027
2028	tx_sa = rtnl_dereference(tx_sc->sa[assoc_num]);
2029	if (tx_sa) {
2030	rtnl_unlock();
2031	return -EBUSY;
2032	}
2033
2034	tx_sa = kmalloc(size: sizeof(*tx_sa), GFP_KERNEL);
2035	if (!tx_sa) {
2036	rtnl_unlock();
2037	return -ENOMEM;
2038	}
2039
2040	err = init_tx_sa(tx_sa, sak: nla_data(nla: tb_sa[MACSEC_SA_ATTR_KEY]),
2041	key_len: secy->key_len, icv_len: secy->icv_len);
2042	if (err < 0) {
2043	kfree(objp: tx_sa);
2044	rtnl_unlock();
2045	return err;
2046	}
2047
2048	spin_lock_bh(lock: &tx_sa->lock);
2049	tx_sa->next_pn = nla_get_u64(nla: tb_sa[MACSEC_SA_ATTR_PN]);
2050	spin_unlock_bh(lock: &tx_sa->lock);
2051
2052	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
2053	tx_sa->active = !!nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_ACTIVE]);
2054
2055	was_operational = secy->operational;
2056	if (assoc_num == tx_sc->encoding_sa && tx_sa->active)
2057	secy->operational = true;
2058
2059	if (secy->xpn) {
2060	tx_sa->ssci = nla_get_ssci(nla: tb_sa[MACSEC_SA_ATTR_SSCI]);
2061	nla_memcpy(dest: tx_sa->key.salt.bytes, src: tb_sa[MACSEC_SA_ATTR_SALT],
2062	MACSEC_SALT_LEN);
2063	}
2064
2065	/* If h/w offloading is available, propagate to the device */
2066	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2067	const struct macsec_ops *ops;
2068	struct macsec_context ctx;
2069
2070	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2071	if (!ops) {
2072	err = -EOPNOTSUPP;
2073	goto cleanup;
2074	}
2075
2076	ctx.sa.assoc_num = assoc_num;
2077	ctx.sa.tx_sa = tx_sa;
2078	ctx.secy = secy;
2079	memcpy(ctx.sa.key, nla_data(tb_sa[MACSEC_SA_ATTR_KEY]),
2080	secy->key_len);
2081
2082	err = macsec_offload(func: ops->mdo_add_txsa, ctx: &ctx);
2083	memzero_explicit(s: ctx.sa.key, count: secy->key_len);
2084	if (err)
2085	goto cleanup;
2086	}
2087
2088	nla_memcpy(dest: tx_sa->key.id, src: tb_sa[MACSEC_SA_ATTR_KEYID], MACSEC_KEYID_LEN);
2089	rcu_assign_pointer(tx_sc->sa[assoc_num], tx_sa);
2090
2091	rtnl_unlock();
2092
2093	return 0;
2094
2095	cleanup:
2096	secy->operational = was_operational;
2097	macsec_txsa_put(sa: tx_sa);
2098	rtnl_unlock();
2099	return err;
2100	}
2101
2102	static int macsec_del_rxsa(struct sk_buff *skb, struct genl_info *info)
2103	{
2104	struct nlattr **attrs = info->attrs;
2105	struct net_device *dev;
2106	struct macsec_secy *secy;
2107	struct macsec_rx_sc *rx_sc;
2108	struct macsec_rx_sa *rx_sa;
2109	u8 assoc_num;
2110	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2111	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2112	int ret;
2113
2114	if (!attrs[MACSEC_ATTR_IFINDEX])
2115	return -EINVAL;
2116
2117	if (parse_sa_config(attrs, tb_sa))
2118	return -EINVAL;
2119
2120	if (parse_rxsc_config(attrs, tb_rxsc))
2121	return -EINVAL;
2122
2123	rtnl_lock();
2124	rx_sa = get_rxsa_from_nl(net: genl_info_net(info), attrs, tb_rxsc, tb_sa,
2125	devp: &dev, secyp: &secy, scp: &rx_sc, assoc_num: &assoc_num);
2126	if (IS_ERR(ptr: rx_sa)) {
2127	rtnl_unlock();
2128	return PTR_ERR(ptr: rx_sa);
2129	}
2130
2131	if (rx_sa->active) {
2132	rtnl_unlock();
2133	return -EBUSY;
2134	}
2135
2136	/* If h/w offloading is available, propagate to the device */
2137	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2138	const struct macsec_ops *ops;
2139	struct macsec_context ctx;
2140
2141	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2142	if (!ops) {
2143	ret = -EOPNOTSUPP;
2144	goto cleanup;
2145	}
2146
2147	ctx.sa.assoc_num = assoc_num;
2148	ctx.sa.rx_sa = rx_sa;
2149	ctx.secy = secy;
2150
2151	ret = macsec_offload(func: ops->mdo_del_rxsa, ctx: &ctx);
2152	if (ret)
2153	goto cleanup;
2154	}
2155
2156	RCU_INIT_POINTER(rx_sc->sa[assoc_num], NULL);
2157	clear_rx_sa(rx_sa);
2158
2159	rtnl_unlock();
2160
2161	return 0;
2162
2163	cleanup:
2164	rtnl_unlock();
2165	return ret;
2166	}
2167
2168	static int macsec_del_rxsc(struct sk_buff *skb, struct genl_info *info)
2169	{
2170	struct nlattr **attrs = info->attrs;
2171	struct net_device *dev;
2172	struct macsec_secy *secy;
2173	struct macsec_rx_sc *rx_sc;
2174	sci_t sci;
2175	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2176	int ret;
2177
2178	if (!attrs[MACSEC_ATTR_IFINDEX])
2179	return -EINVAL;
2180
2181	if (parse_rxsc_config(attrs, tb_rxsc))
2182	return -EINVAL;
2183
2184	if (!tb_rxsc[MACSEC_RXSC_ATTR_SCI])
2185	return -EINVAL;
2186
2187	rtnl_lock();
2188	dev = get_dev_from_nl(net: genl_info_net(info), attrs: info->attrs);
2189	if (IS_ERR(ptr: dev)) {
2190	rtnl_unlock();
2191	return PTR_ERR(ptr: dev);
2192	}
2193
2194	secy = &macsec_priv(dev)->secy;
2195	sci = nla_get_sci(nla: tb_rxsc[MACSEC_RXSC_ATTR_SCI]);
2196
2197	rx_sc = del_rx_sc(secy, sci);
2198	if (!rx_sc) {
2199	rtnl_unlock();
2200	return -ENODEV;
2201	}
2202
2203	/* If h/w offloading is available, propagate to the device */
2204	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2205	const struct macsec_ops *ops;
2206	struct macsec_context ctx;
2207
2208	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2209	if (!ops) {
2210	ret = -EOPNOTSUPP;
2211	goto cleanup;
2212	}
2213
2214	ctx.rx_sc = rx_sc;
2215	ctx.secy = secy;
2216	ret = macsec_offload(func: ops->mdo_del_rxsc, ctx: &ctx);
2217	if (ret)
2218	goto cleanup;
2219	}
2220
2221	free_rx_sc(rx_sc);
2222	rtnl_unlock();
2223
2224	return 0;
2225
2226	cleanup:
2227	rtnl_unlock();
2228	return ret;
2229	}
2230
2231	static int macsec_del_txsa(struct sk_buff *skb, struct genl_info *info)
2232	{
2233	struct nlattr **attrs = info->attrs;
2234	struct net_device *dev;
2235	struct macsec_secy *secy;
2236	struct macsec_tx_sc *tx_sc;
2237	struct macsec_tx_sa *tx_sa;
2238	u8 assoc_num;
2239	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2240	int ret;
2241
2242	if (!attrs[MACSEC_ATTR_IFINDEX])
2243	return -EINVAL;
2244
2245	if (parse_sa_config(attrs, tb_sa))
2246	return -EINVAL;
2247
2248	rtnl_lock();
2249	tx_sa = get_txsa_from_nl(net: genl_info_net(info), attrs, tb_sa,
2250	devp: &dev, secyp: &secy, scp: &tx_sc, assoc_num: &assoc_num);
2251	if (IS_ERR(ptr: tx_sa)) {
2252	rtnl_unlock();
2253	return PTR_ERR(ptr: tx_sa);
2254	}
2255
2256	if (tx_sa->active) {
2257	rtnl_unlock();
2258	return -EBUSY;
2259	}
2260
2261	/* If h/w offloading is available, propagate to the device */
2262	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2263	const struct macsec_ops *ops;
2264	struct macsec_context ctx;
2265
2266	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2267	if (!ops) {
2268	ret = -EOPNOTSUPP;
2269	goto cleanup;
2270	}
2271
2272	ctx.sa.assoc_num = assoc_num;
2273	ctx.sa.tx_sa = tx_sa;
2274	ctx.secy = secy;
2275
2276	ret = macsec_offload(func: ops->mdo_del_txsa, ctx: &ctx);
2277	if (ret)
2278	goto cleanup;
2279	}
2280
2281	RCU_INIT_POINTER(tx_sc->sa[assoc_num], NULL);
2282	clear_tx_sa(tx_sa);
2283
2284	rtnl_unlock();
2285
2286	return 0;
2287
2288	cleanup:
2289	rtnl_unlock();
2290	return ret;
2291	}
2292
2293	static bool validate_upd_sa(struct nlattr **attrs)
2294	{
2295	if (!attrs[MACSEC_SA_ATTR_AN] ||
2296	attrs[MACSEC_SA_ATTR_KEY] ||
2297	attrs[MACSEC_SA_ATTR_KEYID] ||
2298	attrs[MACSEC_SA_ATTR_SSCI] ||
2299	attrs[MACSEC_SA_ATTR_SALT])
2300	return false;
2301
2302	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_AN]) >= MACSEC_NUM_AN)
2303	return false;
2304
2305	if (attrs[MACSEC_SA_ATTR_PN] && nla_get_u64(nla: attrs[MACSEC_SA_ATTR_PN]) == 0)
2306	return false;
2307
2308	if (attrs[MACSEC_SA_ATTR_ACTIVE]) {
2309	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_ACTIVE]) > 1)
2310	return false;
2311	}
2312
2313	return true;
2314	}
2315
2316	static int macsec_upd_txsa(struct sk_buff *skb, struct genl_info *info)
2317	{
2318	struct nlattr **attrs = info->attrs;
2319	struct net_device *dev;
2320	struct macsec_secy *secy;
2321	struct macsec_tx_sc *tx_sc;
2322	struct macsec_tx_sa *tx_sa;
2323	u8 assoc_num;
2324	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2325	bool was_operational, was_active;
2326	pn_t prev_pn;
2327	int ret = 0;
2328
2329	prev_pn.full64 = 0;
2330
2331	if (!attrs[MACSEC_ATTR_IFINDEX])
2332	return -EINVAL;
2333
2334	if (parse_sa_config(attrs, tb_sa))
2335	return -EINVAL;
2336
2337	if (!validate_upd_sa(attrs: tb_sa))
2338	return -EINVAL;
2339
2340	rtnl_lock();
2341	tx_sa = get_txsa_from_nl(net: genl_info_net(info), attrs, tb_sa,
2342	devp: &dev, secyp: &secy, scp: &tx_sc, assoc_num: &assoc_num);
2343	if (IS_ERR(ptr: tx_sa)) {
2344	rtnl_unlock();
2345	return PTR_ERR(ptr: tx_sa);
2346	}
2347
2348	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2349	int pn_len;
2350
2351	pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
2352	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
2353	pr_notice("macsec: nl: upd_txsa: bad pn length: %d != %d\n",
2354	nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
2355	rtnl_unlock();
2356	return -EINVAL;
2357	}
2358
2359	spin_lock_bh(lock: &tx_sa->lock);
2360	prev_pn = tx_sa->next_pn_halves;
2361	tx_sa->next_pn = nla_get_u64(nla: tb_sa[MACSEC_SA_ATTR_PN]);
2362	spin_unlock_bh(lock: &tx_sa->lock);
2363	}
2364
2365	was_active = tx_sa->active;
2366	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
2367	tx_sa->active = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_ACTIVE]);
2368
2369	was_operational = secy->operational;
2370	if (assoc_num == tx_sc->encoding_sa)
2371	secy->operational = tx_sa->active;
2372
2373	/* If h/w offloading is available, propagate to the device */
2374	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2375	const struct macsec_ops *ops;
2376	struct macsec_context ctx;
2377
2378	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2379	if (!ops) {
2380	ret = -EOPNOTSUPP;
2381	goto cleanup;
2382	}
2383
2384	ctx.sa.assoc_num = assoc_num;
2385	ctx.sa.tx_sa = tx_sa;
2386	ctx.sa.update_pn = !!prev_pn.full64;
2387	ctx.secy = secy;
2388
2389	ret = macsec_offload(func: ops->mdo_upd_txsa, ctx: &ctx);
2390	if (ret)
2391	goto cleanup;
2392	}
2393
2394	rtnl_unlock();
2395
2396	return 0;
2397
2398	cleanup:
2399	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2400	spin_lock_bh(lock: &tx_sa->lock);
2401	tx_sa->next_pn_halves = prev_pn;
2402	spin_unlock_bh(lock: &tx_sa->lock);
2403	}
2404	tx_sa->active = was_active;
2405	secy->operational = was_operational;
2406	rtnl_unlock();
2407	return ret;
2408	}
2409
2410	static int macsec_upd_rxsa(struct sk_buff *skb, struct genl_info *info)
2411	{
2412	struct nlattr **attrs = info->attrs;
2413	struct net_device *dev;
2414	struct macsec_secy *secy;
2415	struct macsec_rx_sc *rx_sc;
2416	struct macsec_rx_sa *rx_sa;
2417	u8 assoc_num;
2418	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2419	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2420	bool was_active;
2421	pn_t prev_pn;
2422	int ret = 0;
2423
2424	prev_pn.full64 = 0;
2425
2426	if (!attrs[MACSEC_ATTR_IFINDEX])
2427	return -EINVAL;
2428
2429	if (parse_rxsc_config(attrs, tb_rxsc))
2430	return -EINVAL;
2431
2432	if (parse_sa_config(attrs, tb_sa))
2433	return -EINVAL;
2434
2435	if (!validate_upd_sa(attrs: tb_sa))
2436	return -EINVAL;
2437
2438	rtnl_lock();
2439	rx_sa = get_rxsa_from_nl(net: genl_info_net(info), attrs, tb_rxsc, tb_sa,
2440	devp: &dev, secyp: &secy, scp: &rx_sc, assoc_num: &assoc_num);
2441	if (IS_ERR(ptr: rx_sa)) {
2442	rtnl_unlock();
2443	return PTR_ERR(ptr: rx_sa);
2444	}
2445
2446	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2447	int pn_len;
2448
2449	pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
2450	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
2451	pr_notice("macsec: nl: upd_rxsa: bad pn length: %d != %d\n",
2452	nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
2453	rtnl_unlock();
2454	return -EINVAL;
2455	}
2456
2457	spin_lock_bh(lock: &rx_sa->lock);
2458	prev_pn = rx_sa->next_pn_halves;
2459	rx_sa->next_pn = nla_get_u64(nla: tb_sa[MACSEC_SA_ATTR_PN]);
2460	spin_unlock_bh(lock: &rx_sa->lock);
2461	}
2462
2463	was_active = rx_sa->active;
2464	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
2465	rx_sa->active = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_ACTIVE]);
2466
2467	/* If h/w offloading is available, propagate to the device */
2468	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2469	const struct macsec_ops *ops;
2470	struct macsec_context ctx;
2471
2472	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2473	if (!ops) {
2474	ret = -EOPNOTSUPP;
2475	goto cleanup;
2476	}
2477
2478	ctx.sa.assoc_num = assoc_num;
2479	ctx.sa.rx_sa = rx_sa;
2480	ctx.sa.update_pn = !!prev_pn.full64;
2481	ctx.secy = secy;
2482
2483	ret = macsec_offload(func: ops->mdo_upd_rxsa, ctx: &ctx);
2484	if (ret)
2485	goto cleanup;
2486	}
2487
2488	rtnl_unlock();
2489	return 0;
2490
2491	cleanup:
2492	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2493	spin_lock_bh(lock: &rx_sa->lock);
2494	rx_sa->next_pn_halves = prev_pn;
2495	spin_unlock_bh(lock: &rx_sa->lock);
2496	}
2497	rx_sa->active = was_active;
2498	rtnl_unlock();
2499	return ret;
2500	}
2501
2502	static int macsec_upd_rxsc(struct sk_buff *skb, struct genl_info *info)
2503	{
2504	struct nlattr **attrs = info->attrs;
2505	struct net_device *dev;
2506	struct macsec_secy *secy;
2507	struct macsec_rx_sc *rx_sc;
2508	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2509	unsigned int prev_n_rx_sc;
2510	bool was_active;
2511	int ret;
2512
2513	if (!attrs[MACSEC_ATTR_IFINDEX])
2514	return -EINVAL;
2515
2516	if (parse_rxsc_config(attrs, tb_rxsc))
2517	return -EINVAL;
2518
2519	if (!validate_add_rxsc(attrs: tb_rxsc))
2520	return -EINVAL;
2521
2522	rtnl_lock();
2523	rx_sc = get_rxsc_from_nl(net: genl_info_net(info), attrs, tb_rxsc, devp: &dev, secyp: &secy);
2524	if (IS_ERR(ptr: rx_sc)) {
2525	rtnl_unlock();
2526	return PTR_ERR(ptr: rx_sc);
2527	}
2528
2529	was_active = rx_sc->active;
2530	prev_n_rx_sc = secy->n_rx_sc;
2531	if (tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]) {
2532	bool new = !!nla_get_u8(nla: tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]);
2533
2534	if (rx_sc->active != new)
2535	secy->n_rx_sc += new ? 1 : -1;
2536
2537	rx_sc->active = new;
2538	}
2539
2540	/* If h/w offloading is available, propagate to the device */
2541	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2542	const struct macsec_ops *ops;
2543	struct macsec_context ctx;
2544
2545	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2546	if (!ops) {
2547	ret = -EOPNOTSUPP;
2548	goto cleanup;
2549	}
2550
2551	ctx.rx_sc = rx_sc;
2552	ctx.secy = secy;
2553
2554	ret = macsec_offload(func: ops->mdo_upd_rxsc, ctx: &ctx);
2555	if (ret)
2556	goto cleanup;
2557	}
2558
2559	rtnl_unlock();
2560
2561	return 0;
2562
2563	cleanup:
2564	secy->n_rx_sc = prev_n_rx_sc;
2565	rx_sc->active = was_active;
2566	rtnl_unlock();
2567	return ret;
2568	}
2569
2570	static bool macsec_is_configured(struct macsec_dev *macsec)
2571	{
2572	struct macsec_secy *secy = &macsec->secy;
2573	struct macsec_tx_sc *tx_sc = &secy->tx_sc;
2574	int i;
2575
2576	if (secy->rx_sc)
2577	return true;
2578
2579	for (i = 0; i < MACSEC_NUM_AN; i++)
2580	if (tx_sc->sa[i])
2581	return true;
2582
2583	return false;
2584	}
2585
2586	static int macsec_update_offload(struct net_device *dev, enum macsec_offload offload)
2587	{
2588	enum macsec_offload prev_offload;
2589	const struct macsec_ops *ops;
2590	struct macsec_context ctx;
2591	struct macsec_dev *macsec;
2592	int ret = 0;
2593
2594	macsec = macsec_priv(dev);
2595
2596	/* Check if the offloading mode is supported by the underlying layers */
2597	if (offload != MACSEC_OFFLOAD_OFF &&
2598	!macsec_check_offload(offload, macsec))
2599	return -EOPNOTSUPP;
2600
2601	/* Check if the net device is busy. */
2602	if (netif_running(dev))
2603	return -EBUSY;
2604
2605	/* Check if the device already has rules configured: we do not support
2606	* rules migration.
2607	*/
2608	if (macsec_is_configured(macsec))
2609	return -EBUSY;
2610
2611	prev_offload = macsec->offload;
2612
2613	ops = __macsec_get_ops(offload: offload == MACSEC_OFFLOAD_OFF ? prev_offload : offload,
2614	macsec, ctx: &ctx);
2615	if (!ops)
2616	return -EOPNOTSUPP;
2617
2618	macsec->offload = offload;
2619
2620	ctx.secy = &macsec->secy;
2621	ret = offload == MACSEC_OFFLOAD_OFF ? macsec_offload(func: ops->mdo_del_secy, ctx: &ctx)
2622	: macsec_offload(func: ops->mdo_add_secy, ctx: &ctx);
2623	if (ret)
2624	macsec->offload = prev_offload;
2625
2626	return ret;
2627	}
2628
2629	static int macsec_upd_offload(struct sk_buff *skb, struct genl_info *info)
2630	{
2631	struct nlattr *tb_offload[MACSEC_OFFLOAD_ATTR_MAX + 1];
2632	struct nlattr **attrs = info->attrs;
2633	enum macsec_offload offload;
2634	struct macsec_dev *macsec;
2635	struct net_device *dev;
2636	int ret = 0;
2637
2638	if (!attrs[MACSEC_ATTR_IFINDEX])
2639	return -EINVAL;
2640
2641	if (!attrs[MACSEC_ATTR_OFFLOAD])
2642	return -EINVAL;
2643
2644	if (nla_parse_nested_deprecated(tb: tb_offload, maxtype: MACSEC_OFFLOAD_ATTR_MAX,
2645	nla: attrs[MACSEC_ATTR_OFFLOAD],
2646	policy: macsec_genl_offload_policy, NULL))
2647	return -EINVAL;
2648
2649	rtnl_lock();
2650
2651	dev = get_dev_from_nl(net: genl_info_net(info), attrs);
2652	if (IS_ERR(ptr: dev)) {
2653	ret = PTR_ERR(ptr: dev);
2654	goto out;
2655	}
2656	macsec = macsec_priv(dev);
2657
2658	if (!tb_offload[MACSEC_OFFLOAD_ATTR_TYPE]) {
2659	ret = -EINVAL;
2660	goto out;
2661	}
2662
2663	offload = nla_get_u8(nla: tb_offload[MACSEC_OFFLOAD_ATTR_TYPE]);
2664
2665	if (macsec->offload != offload)
2666	ret = macsec_update_offload(dev, offload);
2667	out:
2668	rtnl_unlock();
2669	return ret;
2670	}
2671
2672	static void get_tx_sa_stats(struct net_device *dev, int an,
2673	struct macsec_tx_sa *tx_sa,
2674	struct macsec_tx_sa_stats *sum)
2675	{
2676	struct macsec_dev *macsec = macsec_priv(dev);
2677	int cpu;
2678
2679	/* If h/w offloading is available, propagate to the device */
2680	if (macsec_is_offloaded(macsec)) {
2681	const struct macsec_ops *ops;
2682	struct macsec_context ctx;
2683
2684	ops = macsec_get_ops(macsec, ctx: &ctx);
2685	if (ops) {
2686	ctx.sa.assoc_num = an;
2687	ctx.sa.tx_sa = tx_sa;
2688	ctx.stats.tx_sa_stats = sum;
2689	ctx.secy = &macsec_priv(dev)->secy;
2690	macsec_offload(func: ops->mdo_get_tx_sa_stats, ctx: &ctx);
2691	}
2692	return;
2693	}
2694
2695	for_each_possible_cpu(cpu) {
2696	const struct macsec_tx_sa_stats *stats =
2697	per_cpu_ptr(tx_sa->stats, cpu);
2698
2699	sum->OutPktsProtected += stats->OutPktsProtected;
2700	sum->OutPktsEncrypted += stats->OutPktsEncrypted;
2701	}
2702	}
2703
2704	static int copy_tx_sa_stats(struct sk_buff *skb, struct macsec_tx_sa_stats *sum)
2705	{
2706	if (nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_OUT_PKTS_PROTECTED,
2707	value: sum->OutPktsProtected) ||
2708	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_OUT_PKTS_ENCRYPTED,
2709	value: sum->OutPktsEncrypted))
2710	return -EMSGSIZE;
2711
2712	return 0;
2713	}
2714
2715	static void get_rx_sa_stats(struct net_device *dev,
2716	struct macsec_rx_sc *rx_sc, int an,
2717	struct macsec_rx_sa *rx_sa,
2718	struct macsec_rx_sa_stats *sum)
2719	{
2720	struct macsec_dev *macsec = macsec_priv(dev);
2721	int cpu;
2722
2723	/* If h/w offloading is available, propagate to the device */
2724	if (macsec_is_offloaded(macsec)) {
2725	const struct macsec_ops *ops;
2726	struct macsec_context ctx;
2727
2728	ops = macsec_get_ops(macsec, ctx: &ctx);
2729	if (ops) {
2730	ctx.sa.assoc_num = an;
2731	ctx.sa.rx_sa = rx_sa;
2732	ctx.stats.rx_sa_stats = sum;
2733	ctx.secy = &macsec_priv(dev)->secy;
2734	ctx.rx_sc = rx_sc;
2735	macsec_offload(func: ops->mdo_get_rx_sa_stats, ctx: &ctx);
2736	}
2737	return;
2738	}
2739
2740	for_each_possible_cpu(cpu) {
2741	const struct macsec_rx_sa_stats *stats =
2742	per_cpu_ptr(rx_sa->stats, cpu);
2743
2744	sum->InPktsOK += stats->InPktsOK;
2745	sum->InPktsInvalid += stats->InPktsInvalid;
2746	sum->InPktsNotValid += stats->InPktsNotValid;
2747	sum->InPktsNotUsingSA += stats->InPktsNotUsingSA;
2748	sum->InPktsUnusedSA += stats->InPktsUnusedSA;
2749	}
2750	}
2751
2752	static int copy_rx_sa_stats(struct sk_buff *skb,
2753	struct macsec_rx_sa_stats *sum)
2754	{
2755	if (nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_OK, value: sum->InPktsOK) ||
2756	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_INVALID,
2757	value: sum->InPktsInvalid) ||
2758	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_NOT_VALID,
2759	value: sum->InPktsNotValid) ||
2760	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_NOT_USING_SA,
2761	value: sum->InPktsNotUsingSA) ||
2762	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_UNUSED_SA,
2763	value: sum->InPktsUnusedSA))
2764	return -EMSGSIZE;
2765
2766	return 0;
2767	}
2768
2769	static void get_rx_sc_stats(struct net_device *dev,
2770	struct macsec_rx_sc *rx_sc,
2771	struct macsec_rx_sc_stats *sum)
2772	{
2773	struct macsec_dev *macsec = macsec_priv(dev);
2774	int cpu;
2775
2776	/* If h/w offloading is available, propagate to the device */
2777	if (macsec_is_offloaded(macsec)) {
2778	const struct macsec_ops *ops;
2779	struct macsec_context ctx;
2780
2781	ops = macsec_get_ops(macsec, ctx: &ctx);
2782	if (ops) {
2783	ctx.stats.rx_sc_stats = sum;
2784	ctx.secy = &macsec_priv(dev)->secy;
2785	ctx.rx_sc = rx_sc;
2786	macsec_offload(func: ops->mdo_get_rx_sc_stats, ctx: &ctx);
2787	}
2788	return;
2789	}
2790
2791	for_each_possible_cpu(cpu) {
2792	const struct pcpu_rx_sc_stats *stats;
2793	struct macsec_rx_sc_stats tmp;
2794	unsigned int start;
2795
2796	stats = per_cpu_ptr(rx_sc->stats, cpu);
2797	do {
2798	start = u64_stats_fetch_begin(syncp: &stats->syncp);
2799	memcpy(&tmp, &stats->stats, sizeof(tmp));
2800	} while (u64_stats_fetch_retry(syncp: &stats->syncp, start));
2801
2802	sum->InOctetsValidated += tmp.InOctetsValidated;
2803	sum->InOctetsDecrypted += tmp.InOctetsDecrypted;
2804	sum->InPktsUnchecked += tmp.InPktsUnchecked;
2805	sum->InPktsDelayed += tmp.InPktsDelayed;
2806	sum->InPktsOK += tmp.InPktsOK;
2807	sum->InPktsInvalid += tmp.InPktsInvalid;
2808	sum->InPktsLate += tmp.InPktsLate;
2809	sum->InPktsNotValid += tmp.InPktsNotValid;
2810	sum->InPktsNotUsingSA += tmp.InPktsNotUsingSA;
2811	sum->InPktsUnusedSA += tmp.InPktsUnusedSA;
2812	}
2813	}
2814
2815	static int copy_rx_sc_stats(struct sk_buff *skb, struct macsec_rx_sc_stats *sum)
2816	{
2817	if (nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_OCTETS_VALIDATED,
2818	value: sum->InOctetsValidated,
2819	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2820	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_OCTETS_DECRYPTED,
2821	value: sum->InOctetsDecrypted,
2822	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2823	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_UNCHECKED,
2824	value: sum->InPktsUnchecked,
2825	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2826	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_DELAYED,
2827	value: sum->InPktsDelayed,
2828	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2829	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_OK,
2830	value: sum->InPktsOK,
2831	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2832	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_INVALID,
2833	value: sum->InPktsInvalid,
2834	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2835	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_LATE,
2836	value: sum->InPktsLate,
2837	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2838	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_NOT_VALID,
2839	value: sum->InPktsNotValid,
2840	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2841	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_NOT_USING_SA,
2842	value: sum->InPktsNotUsingSA,
2843	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2844	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_UNUSED_SA,
2845	value: sum->InPktsUnusedSA,
2846	padattr: MACSEC_RXSC_STATS_ATTR_PAD))
2847	return -EMSGSIZE;
2848
2849	return 0;
2850	}
2851
2852	static void get_tx_sc_stats(struct net_device *dev,
2853	struct macsec_tx_sc_stats *sum)
2854	{
2855	struct macsec_dev *macsec = macsec_priv(dev);
2856	int cpu;
2857
2858	/* If h/w offloading is available, propagate to the device */
2859	if (macsec_is_offloaded(macsec)) {
2860	const struct macsec_ops *ops;
2861	struct macsec_context ctx;
2862
2863	ops = macsec_get_ops(macsec, ctx: &ctx);
2864	if (ops) {
2865	ctx.stats.tx_sc_stats = sum;
2866	ctx.secy = &macsec_priv(dev)->secy;
2867	macsec_offload(func: ops->mdo_get_tx_sc_stats, ctx: &ctx);
2868	}
2869	return;
2870	}
2871
2872	for_each_possible_cpu(cpu) {
2873	const struct pcpu_tx_sc_stats *stats;
2874	struct macsec_tx_sc_stats tmp;
2875	unsigned int start;
2876
2877	stats = per_cpu_ptr(macsec_priv(dev)->secy.tx_sc.stats, cpu);
2878	do {
2879	start = u64_stats_fetch_begin(syncp: &stats->syncp);
2880	memcpy(&tmp, &stats->stats, sizeof(tmp));
2881	} while (u64_stats_fetch_retry(syncp: &stats->syncp, start));
2882
2883	sum->OutPktsProtected += tmp.OutPktsProtected;
2884	sum->OutPktsEncrypted += tmp.OutPktsEncrypted;
2885	sum->OutOctetsProtected += tmp.OutOctetsProtected;
2886	sum->OutOctetsEncrypted += tmp.OutOctetsEncrypted;
2887	}
2888	}
2889
2890	static int copy_tx_sc_stats(struct sk_buff *skb, struct macsec_tx_sc_stats *sum)
2891	{
2892	if (nla_put_u64_64bit(skb, attrtype: MACSEC_TXSC_STATS_ATTR_OUT_PKTS_PROTECTED,
2893	value: sum->OutPktsProtected,
2894	padattr: MACSEC_TXSC_STATS_ATTR_PAD) ||
2895	nla_put_u64_64bit(skb, attrtype: MACSEC_TXSC_STATS_ATTR_OUT_PKTS_ENCRYPTED,
2896	value: sum->OutPktsEncrypted,
2897	padattr: MACSEC_TXSC_STATS_ATTR_PAD) ||
2898	nla_put_u64_64bit(skb, attrtype: MACSEC_TXSC_STATS_ATTR_OUT_OCTETS_PROTECTED,
2899	value: sum->OutOctetsProtected,
2900	padattr: MACSEC_TXSC_STATS_ATTR_PAD) ||
2901	nla_put_u64_64bit(skb, attrtype: MACSEC_TXSC_STATS_ATTR_OUT_OCTETS_ENCRYPTED,
2902	value: sum->OutOctetsEncrypted,
2903	padattr: MACSEC_TXSC_STATS_ATTR_PAD))
2904	return -EMSGSIZE;
2905
2906	return 0;
2907	}
2908
2909	static void get_secy_stats(struct net_device *dev, struct macsec_dev_stats *sum)
2910	{
2911	struct macsec_dev *macsec = macsec_priv(dev);
2912	int cpu;
2913
2914	/* If h/w offloading is available, propagate to the device */
2915	if (macsec_is_offloaded(macsec)) {
2916	const struct macsec_ops *ops;
2917	struct macsec_context ctx;
2918
2919	ops = macsec_get_ops(macsec, ctx: &ctx);
2920	if (ops) {
2921	ctx.stats.dev_stats = sum;
2922	ctx.secy = &macsec_priv(dev)->secy;
2923	macsec_offload(func: ops->mdo_get_dev_stats, ctx: &ctx);
2924	}
2925	return;
2926	}
2927
2928	for_each_possible_cpu(cpu) {
2929	const struct pcpu_secy_stats *stats;
2930	struct macsec_dev_stats tmp;
2931	unsigned int start;
2932
2933	stats = per_cpu_ptr(macsec_priv(dev)->stats, cpu);
2934	do {
2935	start = u64_stats_fetch_begin(syncp: &stats->syncp);
2936	memcpy(&tmp, &stats->stats, sizeof(tmp));
2937	} while (u64_stats_fetch_retry(syncp: &stats->syncp, start));
2938
2939	sum->OutPktsUntagged += tmp.OutPktsUntagged;
2940	sum->InPktsUntagged += tmp.InPktsUntagged;
2941	sum->OutPktsTooLong += tmp.OutPktsTooLong;
2942	sum->InPktsNoTag += tmp.InPktsNoTag;
2943	sum->InPktsBadTag += tmp.InPktsBadTag;
2944	sum->InPktsUnknownSCI += tmp.InPktsUnknownSCI;
2945	sum->InPktsNoSCI += tmp.InPktsNoSCI;
2946	sum->InPktsOverrun += tmp.InPktsOverrun;
2947	}
2948	}
2949
2950	static int copy_secy_stats(struct sk_buff *skb, struct macsec_dev_stats *sum)
2951	{
2952	if (nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_OUT_PKTS_UNTAGGED,
2953	value: sum->OutPktsUntagged,
2954	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
2955	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_UNTAGGED,
2956	value: sum->InPktsUntagged,
2957	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
2958	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_OUT_PKTS_TOO_LONG,
2959	value: sum->OutPktsTooLong,
2960	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
2961	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_NO_TAG,
2962	value: sum->InPktsNoTag,
2963	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
2964	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_BAD_TAG,
2965	value: sum->InPktsBadTag,
2966	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
2967	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_UNKNOWN_SCI,
2968	value: sum->InPktsUnknownSCI,
2969	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
2970	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_NO_SCI,
2971	value: sum->InPktsNoSCI,
2972	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
2973	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_OVERRUN,
2974	value: sum->InPktsOverrun,
2975	padattr: MACSEC_SECY_STATS_ATTR_PAD))
2976	return -EMSGSIZE;
2977
2978	return 0;
2979	}
2980
2981	static int nla_put_secy(struct macsec_secy *secy, struct sk_buff *skb)
2982	{
2983	struct macsec_tx_sc *tx_sc = &secy->tx_sc;
2984	struct nlattr *secy_nest = nla_nest_start_noflag(skb,
2985	attrtype: MACSEC_ATTR_SECY);
2986	u64 csid;
2987
2988	if (!secy_nest)
2989	return 1;
2990
2991	switch (secy->key_len) {
2992	case MACSEC_GCM_AES_128_SAK_LEN:
2993	csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_128 : MACSEC_DEFAULT_CIPHER_ID;
2994	break;
2995	case MACSEC_GCM_AES_256_SAK_LEN:
2996	csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_256 : MACSEC_CIPHER_ID_GCM_AES_256;
2997	break;
2998	default:
2999	goto cancel;
3000	}
3001
3002	if (nla_put_sci(skb, attrtype: MACSEC_SECY_ATTR_SCI, value: secy->sci,
3003	padattr: MACSEC_SECY_ATTR_PAD) ||
3004	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_ATTR_CIPHER_SUITE,
3005	value: csid, padattr: MACSEC_SECY_ATTR_PAD) ||
3006	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_ICV_LEN, value: secy->icv_len) ||
3007	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_OPER, value: secy->operational) ||
3008	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_PROTECT, value: secy->protect_frames) ||
3009	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_REPLAY, value: secy->replay_protect) ||
3010	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_VALIDATE, value: secy->validate_frames) ||
3011	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_ENCRYPT, value: tx_sc->encrypt) ||
3012	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_INC_SCI, value: tx_sc->send_sci) ||
3013	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_ES, value: tx_sc->end_station) ||
3014	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_SCB, value: tx_sc->scb) ||
3015	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_ENCODING_SA, value: tx_sc->encoding_sa))
3016	goto cancel;
3017
3018	if (secy->replay_protect) {
3019	if (nla_put_u32(skb, attrtype: MACSEC_SECY_ATTR_WINDOW, value: secy->replay_window))
3020	goto cancel;
3021	}
3022
3023	nla_nest_end(skb, start: secy_nest);
3024	return 0;
3025
3026	cancel:
3027	nla_nest_cancel(skb, start: secy_nest);
3028	return 1;
3029	}
3030
3031	static noinline_for_stack int
3032	dump_secy(struct macsec_secy *secy, struct net_device *dev,
3033	struct sk_buff *skb, struct netlink_callback *cb)
3034	{
3035	struct macsec_tx_sc_stats tx_sc_stats = {0, };
3036	struct macsec_tx_sa_stats tx_sa_stats = {0, };
3037	struct macsec_rx_sc_stats rx_sc_stats = {0, };
3038	struct macsec_rx_sa_stats rx_sa_stats = {0, };
3039	struct macsec_dev *macsec = netdev_priv(dev);
3040	struct macsec_dev_stats dev_stats = {0, };
3041	struct macsec_tx_sc *tx_sc = &secy->tx_sc;
3042	struct nlattr *txsa_list, *rxsc_list;
3043	struct macsec_rx_sc *rx_sc;
3044	struct nlattr *attr;
3045	void *hdr;
3046	int i, j;
3047
3048	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, seq: cb->nlh->nlmsg_seq,
3049	family: &macsec_fam, NLM_F_MULTI, cmd: MACSEC_CMD_GET_TXSC);
3050	if (!hdr)
3051	return -EMSGSIZE;
3052
3053	genl_dump_check_consistent(cb, user_hdr: hdr);
3054
3055	if (nla_put_u32(skb, attrtype: MACSEC_ATTR_IFINDEX, value: dev->ifindex))
3056	goto nla_put_failure;
3057
3058	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_OFFLOAD);
3059	if (!attr)
3060	goto nla_put_failure;
3061	if (nla_put_u8(skb, attrtype: MACSEC_OFFLOAD_ATTR_TYPE, value: macsec->offload))
3062	goto nla_put_failure;
3063	nla_nest_end(skb, start: attr);
3064
3065	if (nla_put_secy(secy, skb))
3066	goto nla_put_failure;
3067
3068	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_TXSC_STATS);
3069	if (!attr)
3070	goto nla_put_failure;
3071
3072	get_tx_sc_stats(dev, sum: &tx_sc_stats);
3073	if (copy_tx_sc_stats(skb, sum: &tx_sc_stats)) {
3074	nla_nest_cancel(skb, start: attr);
3075	goto nla_put_failure;
3076	}
3077	nla_nest_end(skb, start: attr);
3078
3079	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_SECY_STATS);
3080	if (!attr)
3081	goto nla_put_failure;
3082	get_secy_stats(dev, sum: &dev_stats);
3083	if (copy_secy_stats(skb, sum: &dev_stats)) {
3084	nla_nest_cancel(skb, start: attr);
3085	goto nla_put_failure;
3086	}
3087	nla_nest_end(skb, start: attr);
3088
3089	txsa_list = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_TXSA_LIST);
3090	if (!txsa_list)
3091	goto nla_put_failure;
3092	for (i = 0, j = 1; i < MACSEC_NUM_AN; i++) {
3093	struct macsec_tx_sa *tx_sa = rtnl_dereference(tx_sc->sa[i]);
3094	struct nlattr *txsa_nest;
3095	u64 pn;
3096	int pn_len;
3097
3098	if (!tx_sa)
3099	continue;
3100
3101	txsa_nest = nla_nest_start_noflag(skb, attrtype: j++);
3102	if (!txsa_nest) {
3103	nla_nest_cancel(skb, start: txsa_list);
3104	goto nla_put_failure;
3105	}
3106
3107	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_SA_ATTR_STATS);
3108	if (!attr) {
3109	nla_nest_cancel(skb, start: txsa_nest);
3110	nla_nest_cancel(skb, start: txsa_list);
3111	goto nla_put_failure;
3112	}
3113	memset(&tx_sa_stats, 0, sizeof(tx_sa_stats));
3114	get_tx_sa_stats(dev, an: i, tx_sa, sum: &tx_sa_stats);
3115	if (copy_tx_sa_stats(skb, sum: &tx_sa_stats)) {
3116	nla_nest_cancel(skb, start: attr);
3117	nla_nest_cancel(skb, start: txsa_nest);
3118	nla_nest_cancel(skb, start: txsa_list);
3119	goto nla_put_failure;
3120	}
3121	nla_nest_end(skb, start: attr);
3122
3123	if (secy->xpn) {
3124	pn = tx_sa->next_pn;
3125	pn_len = MACSEC_XPN_PN_LEN;
3126	} else {
3127	pn = tx_sa->next_pn_halves.lower;
3128	pn_len = MACSEC_DEFAULT_PN_LEN;
3129	}
3130
3131	if (nla_put_u8(skb, attrtype: MACSEC_SA_ATTR_AN, value: i) ||
3132	nla_put(skb, attrtype: MACSEC_SA_ATTR_PN, attrlen: pn_len, data: &pn) ||
3133	nla_put(skb, attrtype: MACSEC_SA_ATTR_KEYID, MACSEC_KEYID_LEN, data: tx_sa->key.id) ||
3134	(secy->xpn && nla_put_ssci(skb, attrtype: MACSEC_SA_ATTR_SSCI, value: tx_sa->ssci)) ||
3135	nla_put_u8(skb, attrtype: MACSEC_SA_ATTR_ACTIVE, value: tx_sa->active)) {
3136	nla_nest_cancel(skb, start: txsa_nest);
3137	nla_nest_cancel(skb, start: txsa_list);
3138	goto nla_put_failure;
3139	}
3140
3141	nla_nest_end(skb, start: txsa_nest);
3142	}
3143	nla_nest_end(skb, start: txsa_list);
3144
3145	rxsc_list = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_RXSC_LIST);
3146	if (!rxsc_list)
3147	goto nla_put_failure;
3148
3149	j = 1;
3150	for_each_rxsc_rtnl(secy, rx_sc) {
3151	int k;
3152	struct nlattr *rxsa_list;
3153	struct nlattr *rxsc_nest = nla_nest_start_noflag(skb, attrtype: j++);
3154
3155	if (!rxsc_nest) {
3156	nla_nest_cancel(skb, start: rxsc_list);
3157	goto nla_put_failure;
3158	}
3159
3160	if (nla_put_u8(skb, attrtype: MACSEC_RXSC_ATTR_ACTIVE, value: rx_sc->active) ||
3161	nla_put_sci(skb, attrtype: MACSEC_RXSC_ATTR_SCI, value: rx_sc->sci,
3162	padattr: MACSEC_RXSC_ATTR_PAD)) {
3163	nla_nest_cancel(skb, start: rxsc_nest);
3164	nla_nest_cancel(skb, start: rxsc_list);
3165	goto nla_put_failure;
3166	}
3167
3168	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_RXSC_ATTR_STATS);
3169	if (!attr) {
3170	nla_nest_cancel(skb, start: rxsc_nest);
3171	nla_nest_cancel(skb, start: rxsc_list);
3172	goto nla_put_failure;
3173	}
3174	memset(&rx_sc_stats, 0, sizeof(rx_sc_stats));
3175	get_rx_sc_stats(dev, rx_sc, sum: &rx_sc_stats);
3176	if (copy_rx_sc_stats(skb, sum: &rx_sc_stats)) {
3177	nla_nest_cancel(skb, start: attr);
3178	nla_nest_cancel(skb, start: rxsc_nest);
3179	nla_nest_cancel(skb, start: rxsc_list);
3180	goto nla_put_failure;
3181	}
3182	nla_nest_end(skb, start: attr);
3183
3184	rxsa_list = nla_nest_start_noflag(skb,
3185	attrtype: MACSEC_RXSC_ATTR_SA_LIST);
3186	if (!rxsa_list) {
3187	nla_nest_cancel(skb, start: rxsc_nest);
3188	nla_nest_cancel(skb, start: rxsc_list);
3189	goto nla_put_failure;
3190	}
3191
3192	for (i = 0, k = 1; i < MACSEC_NUM_AN; i++) {
3193	struct macsec_rx_sa *rx_sa = rtnl_dereference(rx_sc->sa[i]);
3194	struct nlattr *rxsa_nest;
3195	u64 pn;
3196	int pn_len;
3197
3198	if (!rx_sa)
3199	continue;
3200
3201	rxsa_nest = nla_nest_start_noflag(skb, attrtype: k++);
3202	if (!rxsa_nest) {
3203	nla_nest_cancel(skb, start: rxsa_list);
3204	nla_nest_cancel(skb, start: rxsc_nest);
3205	nla_nest_cancel(skb, start: rxsc_list);
3206	goto nla_put_failure;
3207	}
3208
3209	attr = nla_nest_start_noflag(skb,
3210	attrtype: MACSEC_SA_ATTR_STATS);
3211	if (!attr) {
3212	nla_nest_cancel(skb, start: rxsa_list);
3213	nla_nest_cancel(skb, start: rxsc_nest);
3214	nla_nest_cancel(skb, start: rxsc_list);
3215	goto nla_put_failure;
3216	}
3217	memset(&rx_sa_stats, 0, sizeof(rx_sa_stats));
3218	get_rx_sa_stats(dev, rx_sc, an: i, rx_sa, sum: &rx_sa_stats);
3219	if (copy_rx_sa_stats(skb, sum: &rx_sa_stats)) {
3220	nla_nest_cancel(skb, start: attr);
3221	nla_nest_cancel(skb, start: rxsa_list);
3222	nla_nest_cancel(skb, start: rxsc_nest);
3223	nla_nest_cancel(skb, start: rxsc_list);
3224	goto nla_put_failure;
3225	}
3226	nla_nest_end(skb, start: attr);
3227
3228	if (secy->xpn) {
3229	pn = rx_sa->next_pn;
3230	pn_len = MACSEC_XPN_PN_LEN;
3231	} else {
3232	pn = rx_sa->next_pn_halves.lower;
3233	pn_len = MACSEC_DEFAULT_PN_LEN;
3234	}
3235
3236	if (nla_put_u8(skb, attrtype: MACSEC_SA_ATTR_AN, value: i) ||
3237	nla_put(skb, attrtype: MACSEC_SA_ATTR_PN, attrlen: pn_len, data: &pn) ||
3238	nla_put(skb, attrtype: MACSEC_SA_ATTR_KEYID, MACSEC_KEYID_LEN, data: rx_sa->key.id) ||
3239	(secy->xpn && nla_put_ssci(skb, attrtype: MACSEC_SA_ATTR_SSCI, value: rx_sa->ssci)) ||
3240	nla_put_u8(skb, attrtype: MACSEC_SA_ATTR_ACTIVE, value: rx_sa->active)) {
3241	nla_nest_cancel(skb, start: rxsa_nest);
3242	nla_nest_cancel(skb, start: rxsc_nest);
3243	nla_nest_cancel(skb, start: rxsc_list);
3244	goto nla_put_failure;
3245	}
3246	nla_nest_end(skb, start: rxsa_nest);
3247	}
3248
3249	nla_nest_end(skb, start: rxsa_list);
3250	nla_nest_end(skb, start: rxsc_nest);
3251	}
3252
3253	nla_nest_end(skb, start: rxsc_list);
3254
3255	genlmsg_end(skb, hdr);
3256
3257	return 0;
3258
3259	nla_put_failure:
3260	genlmsg_cancel(skb, hdr);
3261	return -EMSGSIZE;
3262	}
3263
3264	static int macsec_generation = 1; /* protected by RTNL */
3265
3266	static int macsec_dump_txsc(struct sk_buff *skb, struct netlink_callback *cb)
3267	{
3268	struct net *net = sock_net(sk: skb->sk);
3269	struct net_device *dev;
3270	int dev_idx, d;
3271
3272	dev_idx = cb->args[0];
3273
3274	d = 0;
3275	rtnl_lock();
3276
3277	cb->seq = macsec_generation;
3278
3279	for_each_netdev(net, dev) {
3280	struct macsec_secy *secy;
3281
3282	if (d < dev_idx)
3283	goto next;
3284
3285	if (!netif_is_macsec(dev))
3286	goto next;
3287
3288	secy = &macsec_priv(dev)->secy;
3289	if (dump_secy(secy, dev, skb, cb) < 0)
3290	goto done;
3291	next:
3292	d++;
3293	}
3294
3295	done:
3296	rtnl_unlock();
3297	cb->args[0] = d;
3298	return skb->len;
3299	}
3300
3301	static const struct genl_small_ops macsec_genl_ops[] = {
3302	{
3303	.cmd = MACSEC_CMD_GET_TXSC,
3304	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3305	.dumpit = macsec_dump_txsc,
3306	},
3307	{
3308	.cmd = MACSEC_CMD_ADD_RXSC,
3309	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3310	.doit = macsec_add_rxsc,
3311	.flags = GENL_ADMIN_PERM,
3312	},
3313	{
3314	.cmd = MACSEC_CMD_DEL_RXSC,
3315	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3316	.doit = macsec_del_rxsc,
3317	.flags = GENL_ADMIN_PERM,
3318	},
3319	{
3320	.cmd = MACSEC_CMD_UPD_RXSC,
3321	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3322	.doit = macsec_upd_rxsc,
3323	.flags = GENL_ADMIN_PERM,
3324	},
3325	{
3326	.cmd = MACSEC_CMD_ADD_TXSA,
3327	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3328	.doit = macsec_add_txsa,
3329	.flags = GENL_ADMIN_PERM,
3330	},
3331	{
3332	.cmd = MACSEC_CMD_DEL_TXSA,
3333	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3334	.doit = macsec_del_txsa,
3335	.flags = GENL_ADMIN_PERM,
3336	},
3337	{
3338	.cmd = MACSEC_CMD_UPD_TXSA,
3339	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3340	.doit = macsec_upd_txsa,
3341	.flags = GENL_ADMIN_PERM,
3342	},
3343	{
3344	.cmd = MACSEC_CMD_ADD_RXSA,
3345	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3346	.doit = macsec_add_rxsa,
3347	.flags = GENL_ADMIN_PERM,
3348	},
3349	{
3350	.cmd = MACSEC_CMD_DEL_RXSA,
3351	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3352	.doit = macsec_del_rxsa,
3353	.flags = GENL_ADMIN_PERM,
3354	},
3355	{
3356	.cmd = MACSEC_CMD_UPD_RXSA,
3357	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3358	.doit = macsec_upd_rxsa,
3359	.flags = GENL_ADMIN_PERM,
3360	},
3361	{
3362	.cmd = MACSEC_CMD_UPD_OFFLOAD,
3363	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3364	.doit = macsec_upd_offload,
3365	.flags = GENL_ADMIN_PERM,
3366	},
3367	};
3368
3369	static struct genl_family macsec_fam __ro_after_init = {
3370	.name = MACSEC_GENL_NAME,
3371	.hdrsize = 0,
3372	.version = MACSEC_GENL_VERSION,
3373	.maxattr = MACSEC_ATTR_MAX,
3374	.policy = macsec_genl_policy,
3375	.netnsok = true,
3376	.module = THIS_MODULE,
3377	.small_ops = macsec_genl_ops,
3378	.n_small_ops = ARRAY_SIZE(macsec_genl_ops),
3379	.resv_start_op = MACSEC_CMD_UPD_OFFLOAD + 1,
3380	};
3381
3382	static netdev_tx_t macsec_start_xmit(struct sk_buff *skb,
3383	struct net_device *dev)
3384	{
3385	struct macsec_dev *macsec = netdev_priv(dev);
3386	struct macsec_secy *secy = &macsec->secy;
3387	struct pcpu_secy_stats *secy_stats;
3388	int ret, len;
3389
3390	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
3391	struct metadata_dst *md_dst = secy->tx_sc.md_dst;
3392
3393	skb_dst_drop(skb);
3394	dst_hold(dst: &md_dst->dst);
3395	skb_dst_set(skb, dst: &md_dst->dst);
3396	skb->dev = macsec->real_dev;
3397	return dev_queue_xmit(skb);
3398	}
3399
3400	/* 10.5 */
3401	if (!secy->protect_frames) {
3402	secy_stats = this_cpu_ptr(macsec->stats);
3403	u64_stats_update_begin(syncp: &secy_stats->syncp);
3404	secy_stats->stats.OutPktsUntagged++;
3405	u64_stats_update_end(syncp: &secy_stats->syncp);
3406	skb->dev = macsec->real_dev;
3407	len = skb->len;
3408	ret = dev_queue_xmit(skb);
3409	count_tx(dev, ret, len);
3410	return ret;
3411	}
3412
3413	if (!secy->operational) {
3414	kfree_skb(skb);
3415	DEV_STATS_INC(dev, tx_dropped);
3416	return NETDEV_TX_OK;
3417	}
3418
3419	len = skb->len;
3420	skb = macsec_encrypt(skb, dev);
3421	if (IS_ERR(ptr: skb)) {
3422	if (PTR_ERR(ptr: skb) != -EINPROGRESS)
3423	DEV_STATS_INC(dev, tx_dropped);
3424	return NETDEV_TX_OK;
3425	}
3426
3427	macsec_count_tx(skb, tx_sc: &macsec->secy.tx_sc, tx_sa: macsec_skb_cb(skb)->tx_sa);
3428
3429	macsec_encrypt_finish(skb, dev);
3430	ret = dev_queue_xmit(skb);
3431	count_tx(dev, ret, len);
3432	return ret;
3433	}
3434
3435	#define MACSEC_FEATURES \
3436	(NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST)
3437
3438	static int macsec_dev_init(struct net_device *dev)
3439	{
3440	struct macsec_dev *macsec = macsec_priv(dev);
3441	struct net_device *real_dev = macsec->real_dev;
3442	int err;
3443
3444	dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
3445	if (!dev->tstats)
3446	return -ENOMEM;
3447
3448	err = gro_cells_init(gcells: &macsec->gro_cells, dev);
3449	if (err) {
3450	free_percpu(pdata: dev->tstats);
3451	return err;
3452	}
3453
3454	dev->features = real_dev->features & MACSEC_FEATURES;
3455	dev->features |= NETIF_F_LLTX | NETIF_F_GSO_SOFTWARE;
3456
3457	dev->needed_headroom = real_dev->needed_headroom +
3458	MACSEC_NEEDED_HEADROOM;
3459	dev->needed_tailroom = real_dev->needed_tailroom +
3460	MACSEC_NEEDED_TAILROOM;
3461
3462	if (is_zero_ether_addr(addr: dev->dev_addr))
3463	eth_hw_addr_inherit(dst: dev, src: real_dev);
3464	if (is_zero_ether_addr(addr: dev->broadcast))
3465	memcpy(dev->broadcast, real_dev->broadcast, dev->addr_len);
3466
3467	/* Get macsec's reference to real_dev */
3468	netdev_hold(dev: real_dev, tracker: &macsec->dev_tracker, GFP_KERNEL);
3469
3470	return 0;
3471	}
3472
3473	static void macsec_dev_uninit(struct net_device *dev)
3474	{
3475	struct macsec_dev *macsec = macsec_priv(dev);
3476
3477	gro_cells_destroy(gcells: &macsec->gro_cells);
3478	free_percpu(pdata: dev->tstats);
3479	}
3480
3481	static netdev_features_t macsec_fix_features(struct net_device *dev,
3482	netdev_features_t features)
3483	{
3484	struct macsec_dev *macsec = macsec_priv(dev);
3485	struct net_device *real_dev = macsec->real_dev;
3486
3487	features &= (real_dev->features & MACSEC_FEATURES) |
3488	NETIF_F_GSO_SOFTWARE | NETIF_F_SOFT_FEATURES;
3489	features |= NETIF_F_LLTX;
3490
3491	return features;
3492	}
3493
3494	static int macsec_dev_open(struct net_device *dev)
3495	{
3496	struct macsec_dev *macsec = macsec_priv(dev);
3497	struct net_device *real_dev = macsec->real_dev;
3498	int err;
3499
3500	err = dev_uc_add(dev: real_dev, addr: dev->dev_addr);
3501	if (err < 0)
3502	return err;
3503
3504	if (dev->flags & IFF_ALLMULTI) {
3505	err = dev_set_allmulti(dev: real_dev, inc: 1);
3506	if (err < 0)
3507	goto del_unicast;
3508	}
3509
3510	if (dev->flags & IFF_PROMISC) {
3511	err = dev_set_promiscuity(dev: real_dev, inc: 1);
3512	if (err < 0)
3513	goto clear_allmulti;
3514	}
3515
3516	/* If h/w offloading is available, propagate to the device */
3517	if (macsec_is_offloaded(macsec)) {
3518	const struct macsec_ops *ops;
3519	struct macsec_context ctx;
3520
3521	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
3522	if (!ops) {
3523	err = -EOPNOTSUPP;
3524	goto clear_allmulti;
3525	}
3526
3527	ctx.secy = &macsec->secy;
3528	err = macsec_offload(func: ops->mdo_dev_open, ctx: &ctx);
3529	if (err)
3530	goto clear_allmulti;
3531	}
3532
3533	if (netif_carrier_ok(dev: real_dev))
3534	netif_carrier_on(dev);
3535
3536	return 0;
3537	clear_allmulti:
3538	if (dev->flags & IFF_ALLMULTI)
3539	dev_set_allmulti(dev: real_dev, inc: -1);
3540	del_unicast:
3541	dev_uc_del(dev: real_dev, addr: dev->dev_addr);
3542	netif_carrier_off(dev);
3543	return err;
3544	}
3545
3546	static int macsec_dev_stop(struct net_device *dev)
3547	{
3548	struct macsec_dev *macsec = macsec_priv(dev);
3549	struct net_device *real_dev = macsec->real_dev;
3550
3551	netif_carrier_off(dev);
3552
3553	/* If h/w offloading is available, propagate to the device */
3554	if (macsec_is_offloaded(macsec)) {
3555	const struct macsec_ops *ops;
3556	struct macsec_context ctx;
3557
3558	ops = macsec_get_ops(macsec, ctx: &ctx);
3559	if (ops) {
3560	ctx.secy = &macsec->secy;
3561	macsec_offload(func: ops->mdo_dev_stop, ctx: &ctx);
3562	}
3563	}
3564
3565	dev_mc_unsync(to: real_dev, from: dev);
3566	dev_uc_unsync(to: real_dev, from: dev);
3567
3568	if (dev->flags & IFF_ALLMULTI)
3569	dev_set_allmulti(dev: real_dev, inc: -1);
3570
3571	if (dev->flags & IFF_PROMISC)
3572	dev_set_promiscuity(dev: real_dev, inc: -1);
3573
3574	dev_uc_del(dev: real_dev, addr: dev->dev_addr);
3575
3576	return 0;
3577	}
3578
3579	static void macsec_dev_change_rx_flags(struct net_device *dev, int change)
3580	{
3581	struct net_device *real_dev = macsec_priv(dev)->real_dev;
3582
3583	if (!(dev->flags & IFF_UP))
3584	return;
3585
3586	if (change & IFF_ALLMULTI)
3587	dev_set_allmulti(dev: real_dev, inc: dev->flags & IFF_ALLMULTI ? 1 : -1);
3588
3589	if (change & IFF_PROMISC)
3590	dev_set_promiscuity(dev: real_dev,
3591	inc: dev->flags & IFF_PROMISC ? 1 : -1);
3592	}
3593
3594	static void macsec_dev_set_rx_mode(struct net_device *dev)
3595	{
3596	struct net_device *real_dev = macsec_priv(dev)->real_dev;
3597
3598	dev_mc_sync(to: real_dev, from: dev);
3599	dev_uc_sync(to: real_dev, from: dev);
3600	}
3601
3602	static int macsec_set_mac_address(struct net_device *dev, void *p)
3603	{
3604	struct macsec_dev *macsec = macsec_priv(dev);
3605	struct net_device *real_dev = macsec->real_dev;
3606	struct sockaddr *addr = p;
3607	int err;
3608
3609	if (!is_valid_ether_addr(addr: addr->sa_data))
3610	return -EADDRNOTAVAIL;
3611
3612	if (!(dev->flags & IFF_UP))
3613	goto out;
3614
3615	err = dev_uc_add(dev: real_dev, addr: addr->sa_data);
3616	if (err < 0)
3617	return err;
3618
3619	dev_uc_del(dev: real_dev, addr: dev->dev_addr);
3620
3621	out:
3622	eth_hw_addr_set(dev, addr: addr->sa_data);
3623
3624	/* If h/w offloading is available, propagate to the device */
3625	if (macsec_is_offloaded(macsec)) {
3626	const struct macsec_ops *ops;
3627	struct macsec_context ctx;
3628
3629	ops = macsec_get_ops(macsec, ctx: &ctx);
3630	if (ops) {
3631	ctx.secy = &macsec->secy;
3632	macsec_offload(func: ops->mdo_upd_secy, ctx: &ctx);
3633	}
3634	}
3635
3636	return 0;
3637	}
3638
3639	static int macsec_change_mtu(struct net_device *dev, int new_mtu)
3640	{
3641	struct macsec_dev *macsec = macsec_priv(dev);
3642	unsigned int extra = macsec->secy.icv_len + macsec_extra_len(sci_present: true);
3643
3644	if (macsec->real_dev->mtu - extra < new_mtu)
3645	return -ERANGE;
3646
3647	dev->mtu = new_mtu;
3648
3649	return 0;
3650	}
3651
3652	static void macsec_get_stats64(struct net_device *dev,
3653	struct rtnl_link_stats64 *s)
3654	{
3655	if (!dev->tstats)
3656	return;
3657
3658	dev_fetch_sw_netstats(s, netstats: dev->tstats);
3659
3660	s->rx_dropped = DEV_STATS_READ(dev, rx_dropped);
3661	s->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
3662	s->rx_errors = DEV_STATS_READ(dev, rx_errors);
3663	}
3664
3665	static int macsec_get_iflink(const struct net_device *dev)
3666	{
3667	return macsec_priv(dev)->real_dev->ifindex;
3668	}
3669
3670	static const struct net_device_ops macsec_netdev_ops = {
3671	.ndo_init = macsec_dev_init,
3672	.ndo_uninit = macsec_dev_uninit,
3673	.ndo_open = macsec_dev_open,
3674	.ndo_stop = macsec_dev_stop,
3675	.ndo_fix_features = macsec_fix_features,
3676	.ndo_change_mtu = macsec_change_mtu,
3677	.ndo_set_rx_mode = macsec_dev_set_rx_mode,
3678	.ndo_change_rx_flags = macsec_dev_change_rx_flags,
3679	.ndo_set_mac_address = macsec_set_mac_address,
3680	.ndo_start_xmit = macsec_start_xmit,
3681	.ndo_get_stats64 = macsec_get_stats64,
3682	.ndo_get_iflink = macsec_get_iflink,
3683	};
3684
3685	static const struct device_type macsec_type = {
3686	.name = "macsec",
3687	};
3688
3689	static const struct nla_policy macsec_rtnl_policy[IFLA_MACSEC_MAX + 1] = {
3690	[IFLA_MACSEC_SCI] = { .type = NLA_U64 },
3691	[IFLA_MACSEC_PORT] = { .type = NLA_U16 },
3692	[IFLA_MACSEC_ICV_LEN] = { .type = NLA_U8 },
3693	[IFLA_MACSEC_CIPHER_SUITE] = { .type = NLA_U64 },
3694	[IFLA_MACSEC_WINDOW] = { .type = NLA_U32 },
3695	[IFLA_MACSEC_ENCODING_SA] = { .type = NLA_U8 },
3696	[IFLA_MACSEC_ENCRYPT] = { .type = NLA_U8 },
3697	[IFLA_MACSEC_PROTECT] = { .type = NLA_U8 },
3698	[IFLA_MACSEC_INC_SCI] = { .type = NLA_U8 },
3699	[IFLA_MACSEC_ES] = { .type = NLA_U8 },
3700	[IFLA_MACSEC_SCB] = { .type = NLA_U8 },
3701	[IFLA_MACSEC_REPLAY_PROTECT] = { .type = NLA_U8 },
3702	[IFLA_MACSEC_VALIDATION] = { .type = NLA_U8 },
3703	[IFLA_MACSEC_OFFLOAD] = { .type = NLA_U8 },
3704	};
3705
3706	static void macsec_free_netdev(struct net_device *dev)
3707	{
3708	struct macsec_dev *macsec = macsec_priv(dev);
3709
3710	if (macsec->secy.tx_sc.md_dst)
3711	metadata_dst_free(macsec->secy.tx_sc.md_dst);
3712	free_percpu(pdata: macsec->stats);
3713	free_percpu(pdata: macsec->secy.tx_sc.stats);
3714
3715	/* Get rid of the macsec's reference to real_dev */
3716	netdev_put(dev: macsec->real_dev, tracker: &macsec->dev_tracker);
3717	}
3718
3719	static void macsec_setup(struct net_device *dev)
3720	{
3721	ether_setup(dev);
3722	dev->min_mtu = 0;
3723	dev->max_mtu = ETH_MAX_MTU;
3724	dev->priv_flags |= IFF_NO_QUEUE;
3725	dev->netdev_ops = &macsec_netdev_ops;
3726	dev->needs_free_netdev = true;
3727	dev->priv_destructor = macsec_free_netdev;
3728	SET_NETDEV_DEVTYPE(dev, &macsec_type);
3729
3730	eth_zero_addr(addr: dev->broadcast);
3731	}
3732
3733	static int macsec_changelink_common(struct net_device *dev,
3734	struct nlattr *data[])
3735	{
3736	struct macsec_secy *secy;
3737	struct macsec_tx_sc *tx_sc;
3738
3739	secy = &macsec_priv(dev)->secy;
3740	tx_sc = &secy->tx_sc;
3741
3742	if (data[IFLA_MACSEC_ENCODING_SA]) {
3743	struct macsec_tx_sa *tx_sa;
3744
3745	tx_sc->encoding_sa = nla_get_u8(nla: data[IFLA_MACSEC_ENCODING_SA]);
3746	tx_sa = rtnl_dereference(tx_sc->sa[tx_sc->encoding_sa]);
3747
3748	secy->operational = tx_sa && tx_sa->active;
3749	}
3750
3751	if (data[IFLA_MACSEC_ENCRYPT])
3752	tx_sc->encrypt = !!nla_get_u8(nla: data[IFLA_MACSEC_ENCRYPT]);
3753
3754	if (data[IFLA_MACSEC_PROTECT])
3755	secy->protect_frames = !!nla_get_u8(nla: data[IFLA_MACSEC_PROTECT]);
3756
3757	if (data[IFLA_MACSEC_INC_SCI])
3758	tx_sc->send_sci = !!nla_get_u8(nla: data[IFLA_MACSEC_INC_SCI]);
3759
3760	if (data[IFLA_MACSEC_ES])
3761	tx_sc->end_station = !!nla_get_u8(nla: data[IFLA_MACSEC_ES]);
3762
3763	if (data[IFLA_MACSEC_SCB])
3764	tx_sc->scb = !!nla_get_u8(nla: data[IFLA_MACSEC_SCB]);
3765
3766	if (data[IFLA_MACSEC_REPLAY_PROTECT])
3767	secy->replay_protect = !!nla_get_u8(nla: data[IFLA_MACSEC_REPLAY_PROTECT]);
3768
3769	if (data[IFLA_MACSEC_VALIDATION])
3770	secy->validate_frames = nla_get_u8(nla: data[IFLA_MACSEC_VALIDATION]);
3771
3772	if (data[IFLA_MACSEC_CIPHER_SUITE]) {
3773	switch (nla_get_u64(nla: data[IFLA_MACSEC_CIPHER_SUITE])) {
3774	case MACSEC_CIPHER_ID_GCM_AES_128:
3775	case MACSEC_DEFAULT_CIPHER_ID:
3776	secy->key_len = MACSEC_GCM_AES_128_SAK_LEN;
3777	secy->xpn = false;
3778	break;
3779	case MACSEC_CIPHER_ID_GCM_AES_256:
3780	secy->key_len = MACSEC_GCM_AES_256_SAK_LEN;
3781	secy->xpn = false;
3782	break;
3783	case MACSEC_CIPHER_ID_GCM_AES_XPN_128:
3784	secy->key_len = MACSEC_GCM_AES_128_SAK_LEN;
3785	secy->xpn = true;
3786	break;
3787	case MACSEC_CIPHER_ID_GCM_AES_XPN_256:
3788	secy->key_len = MACSEC_GCM_AES_256_SAK_LEN;
3789	secy->xpn = true;
3790	break;
3791	default:
3792	return -EINVAL;
3793	}
3794	}
3795
3796	if (data[IFLA_MACSEC_WINDOW]) {
3797	secy->replay_window = nla_get_u32(nla: data[IFLA_MACSEC_WINDOW]);
3798
3799	/* IEEE 802.1AEbw-2013 10.7.8 - maximum replay window
3800	* for XPN cipher suites */
3801	if (secy->xpn &&
3802	secy->replay_window > MACSEC_XPN_MAX_REPLAY_WINDOW)
3803	return -EINVAL;
3804	}
3805
3806	return 0;
3807	}
3808
3809	static int macsec_changelink(struct net_device *dev, struct nlattr *tb[],
3810	struct nlattr *data[],
3811	struct netlink_ext_ack *extack)
3812	{
3813	struct macsec_dev *macsec = macsec_priv(dev);
3814	bool macsec_offload_state_change = false;
3815	enum macsec_offload offload;
3816	struct macsec_tx_sc tx_sc;
3817	struct macsec_secy secy;
3818	int ret;
3819
3820	if (!data)
3821	return 0;
3822
3823	if (data[IFLA_MACSEC_CIPHER_SUITE] ||
3824	data[IFLA_MACSEC_ICV_LEN] ||
3825	data[IFLA_MACSEC_SCI] ||
3826	data[IFLA_MACSEC_PORT])
3827	return -EINVAL;
3828
3829	/* Keep a copy of unmodified secy and tx_sc, in case the offload
3830	* propagation fails, to revert macsec_changelink_common.
3831	*/
3832	memcpy(&secy, &macsec->secy, sizeof(secy));
3833	memcpy(&tx_sc, &macsec->secy.tx_sc, sizeof(tx_sc));
3834
3835	ret = macsec_changelink_common(dev, data);
3836	if (ret)
3837	goto cleanup;
3838
3839	if (data[IFLA_MACSEC_OFFLOAD]) {
3840	offload = nla_get_u8(nla: data[IFLA_MACSEC_OFFLOAD]);
3841	if (macsec->offload != offload) {
3842	macsec_offload_state_change = true;
3843	ret = macsec_update_offload(dev, offload);
3844	if (ret)
3845	goto cleanup;
3846	}
3847	}
3848
3849	/* If h/w offloading is available, propagate to the device */
3850	if (!macsec_offload_state_change && macsec_is_offloaded(macsec)) {
3851	const struct macsec_ops *ops;
3852	struct macsec_context ctx;
3853
3854	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
3855	if (!ops) {
3856	ret = -EOPNOTSUPP;
3857	goto cleanup;
3858	}
3859
3860	ctx.secy = &macsec->secy;
3861	ret = macsec_offload(func: ops->mdo_upd_secy, ctx: &ctx);
3862	if (ret)
3863	goto cleanup;
3864	}
3865
3866	return 0;
3867
3868	cleanup:
3869	memcpy(&macsec->secy.tx_sc, &tx_sc, sizeof(tx_sc));
3870	memcpy(&macsec->secy, &secy, sizeof(secy));
3871
3872	return ret;
3873	}
3874
3875	static void macsec_del_dev(struct macsec_dev *macsec)
3876	{
3877	int i;
3878
3879	while (macsec->secy.rx_sc) {
3880	struct macsec_rx_sc *rx_sc = rtnl_dereference(macsec->secy.rx_sc);
3881
3882	rcu_assign_pointer(macsec->secy.rx_sc, rx_sc->next);
3883	free_rx_sc(rx_sc);
3884	}
3885
3886	for (i = 0; i < MACSEC_NUM_AN; i++) {
3887	struct macsec_tx_sa *sa = rtnl_dereference(macsec->secy.tx_sc.sa[i]);
3888
3889	if (sa) {
3890	RCU_INIT_POINTER(macsec->secy.tx_sc.sa[i], NULL);
3891	clear_tx_sa(tx_sa: sa);
3892	}
3893	}
3894	}
3895
3896	static void macsec_common_dellink(struct net_device *dev, struct list_head *head)
3897	{
3898	struct macsec_dev *macsec = macsec_priv(dev);
3899	struct net_device *real_dev = macsec->real_dev;
3900
3901	/* If h/w offloading is available, propagate to the device */
3902	if (macsec_is_offloaded(macsec)) {
3903	const struct macsec_ops *ops;
3904	struct macsec_context ctx;
3905
3906	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
3907	if (ops) {
3908	ctx.secy = &macsec->secy;
3909	macsec_offload(func: ops->mdo_del_secy, ctx: &ctx);
3910	}
3911	}
3912
3913	unregister_netdevice_queue(dev, head);
3914	list_del_rcu(entry: &macsec->secys);
3915	macsec_del_dev(macsec);
3916	netdev_upper_dev_unlink(dev: real_dev, upper_dev: dev);
3917
3918	macsec_generation++;
3919	}
3920
3921	static void macsec_dellink(struct net_device *dev, struct list_head *head)
3922	{
3923	struct macsec_dev *macsec = macsec_priv(dev);
3924	struct net_device *real_dev = macsec->real_dev;
3925	struct macsec_rxh_data *rxd = macsec_data_rtnl(dev: real_dev);
3926
3927	macsec_common_dellink(dev, head);
3928
3929	if (list_empty(head: &rxd->secys)) {
3930	netdev_rx_handler_unregister(dev: real_dev);
3931	kfree(objp: rxd);
3932	}
3933	}
3934
3935	static int register_macsec_dev(struct net_device *real_dev,
3936	struct net_device *dev)
3937	{
3938	struct macsec_dev *macsec = macsec_priv(dev);
3939	struct macsec_rxh_data *rxd = macsec_data_rtnl(dev: real_dev);
3940
3941	if (!rxd) {
3942	int err;
3943
3944	rxd = kmalloc(size: sizeof(*rxd), GFP_KERNEL);
3945	if (!rxd)
3946	return -ENOMEM;
3947
3948	INIT_LIST_HEAD(list: &rxd->secys);
3949
3950	err = netdev_rx_handler_register(dev: real_dev, rx_handler: macsec_handle_frame,
3951	rx_handler_data: rxd);
3952	if (err < 0) {
3953	kfree(objp: rxd);
3954	return err;
3955	}
3956	}
3957
3958	list_add_tail_rcu(new: &macsec->secys, head: &rxd->secys);
3959	return 0;
3960	}
3961
3962	static bool sci_exists(struct net_device *dev, sci_t sci)
3963	{
3964	struct macsec_rxh_data *rxd = macsec_data_rtnl(dev);
3965	struct macsec_dev *macsec;
3966
3967	list_for_each_entry(macsec, &rxd->secys, secys) {
3968	if (macsec->secy.sci == sci)
3969	return true;
3970	}
3971
3972	return false;
3973	}
3974
3975	static sci_t dev_to_sci(struct net_device *dev, __be16 port)
3976	{
3977	return make_sci(addr: dev->dev_addr, port);
3978	}
3979
3980	static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len)
3981	{
3982	struct macsec_dev *macsec = macsec_priv(dev);
3983	struct macsec_secy *secy = &macsec->secy;
3984
3985	macsec->stats = netdev_alloc_pcpu_stats(struct pcpu_secy_stats);
3986	if (!macsec->stats)
3987	return -ENOMEM;
3988
3989	secy->tx_sc.stats = netdev_alloc_pcpu_stats(struct pcpu_tx_sc_stats);
3990	if (!secy->tx_sc.stats)
3991	return -ENOMEM;
3992
3993	secy->tx_sc.md_dst = metadata_dst_alloc(optslen: 0, type: METADATA_MACSEC, GFP_KERNEL);
3994	if (!secy->tx_sc.md_dst)
3995	/* macsec and secy percpu stats will be freed when unregistering
3996	* net_device in macsec_free_netdev()
3997	*/
3998	return -ENOMEM;
3999
4000	if (sci == MACSEC_UNDEF_SCI)
4001	sci = dev_to_sci(dev, MACSEC_PORT_ES);
4002
4003	secy->netdev = dev;
4004	secy->operational = true;
4005	secy->key_len = DEFAULT_SAK_LEN;
4006	secy->icv_len = icv_len;
4007	secy->validate_frames = MACSEC_VALIDATE_DEFAULT;
4008	secy->protect_frames = true;
4009	secy->replay_protect = false;
4010	secy->xpn = DEFAULT_XPN;
4011
4012	secy->sci = sci;
4013	secy->tx_sc.md_dst->u.macsec_info.sci = sci;
4014	secy->tx_sc.active = true;
4015	secy->tx_sc.encoding_sa = DEFAULT_ENCODING_SA;
4016	secy->tx_sc.encrypt = DEFAULT_ENCRYPT;
4017	secy->tx_sc.send_sci = DEFAULT_SEND_SCI;
4018	secy->tx_sc.end_station = false;
4019	secy->tx_sc.scb = false;
4020
4021	return 0;
4022	}
4023
4024	static struct lock_class_key macsec_netdev_addr_lock_key;
4025
4026	static int macsec_newlink(struct net *net, struct net_device *dev,
4027	struct nlattr *tb[], struct nlattr *data[],
4028	struct netlink_ext_ack *extack)
4029	{
4030	struct macsec_dev *macsec = macsec_priv(dev);
4031	rx_handler_func_t *rx_handler;
4032	u8 icv_len = MACSEC_DEFAULT_ICV_LEN;
4033	struct net_device *real_dev;
4034	int err, mtu;
4035	sci_t sci;
4036
4037	if (!tb[IFLA_LINK])
4038	return -EINVAL;
4039	real_dev = __dev_get_by_index(net, ifindex: nla_get_u32(nla: tb[IFLA_LINK]));
4040	if (!real_dev)
4041	return -ENODEV;
4042	if (real_dev->type != ARPHRD_ETHER)
4043	return -EINVAL;
4044
4045	dev->priv_flags |= IFF_MACSEC;
4046
4047	macsec->real_dev = real_dev;
4048
4049	if (data && data[IFLA_MACSEC_OFFLOAD])
4050	macsec->offload = nla_get_offload(nla: data[IFLA_MACSEC_OFFLOAD]);
4051	else
4052	/* MACsec offloading is off by default */
4053	macsec->offload = MACSEC_OFFLOAD_OFF;
4054
4055	/* Check if the offloading mode is supported by the underlying layers */
4056	if (macsec->offload != MACSEC_OFFLOAD_OFF &&
4057	!macsec_check_offload(offload: macsec->offload, macsec))
4058	return -EOPNOTSUPP;
4059
4060	/* send_sci must be set to true when transmit sci explicitly is set */
4061	if ((data && data[IFLA_MACSEC_SCI]) &&
4062	(data && data[IFLA_MACSEC_INC_SCI])) {
4063	u8 send_sci = !!nla_get_u8(nla: data[IFLA_MACSEC_INC_SCI]);
4064
4065	if (!send_sci)
4066	return -EINVAL;
4067	}
4068
4069	if (data && data[IFLA_MACSEC_ICV_LEN])
4070	icv_len = nla_get_u8(nla: data[IFLA_MACSEC_ICV_LEN]);
4071	mtu = real_dev->mtu - icv_len - macsec_extra_len(sci_present: true);
4072	if (mtu < 0)
4073	dev->mtu = 0;
4074	else
4075	dev->mtu = mtu;
4076
4077	rx_handler = rtnl_dereference(real_dev->rx_handler);
4078	if (rx_handler && rx_handler != macsec_handle_frame)
4079	return -EBUSY;
4080
4081	err = register_netdevice(dev);
4082	if (err < 0)
4083	return err;
4084
4085	netdev_lockdep_set_classes(dev);
4086	lockdep_set_class(&dev->addr_list_lock,
4087	&macsec_netdev_addr_lock_key);
4088
4089	err = netdev_upper_dev_link(dev: real_dev, upper_dev: dev, extack);
4090	if (err < 0)
4091	goto unregister;
4092
4093	/* need to be already registered so that ->init has run and
4094	* the MAC addr is set
4095	*/
4096	if (data && data[IFLA_MACSEC_SCI])
4097	sci = nla_get_sci(nla: data[IFLA_MACSEC_SCI]);
4098	else if (data && data[IFLA_MACSEC_PORT])
4099	sci = dev_to_sci(dev, port: nla_get_be16(nla: data[IFLA_MACSEC_PORT]));
4100	else
4101	sci = dev_to_sci(dev, MACSEC_PORT_ES);
4102
4103	if (rx_handler && sci_exists(dev: real_dev, sci)) {
4104	err = -EBUSY;
4105	goto unlink;
4106	}
4107
4108	err = macsec_add_dev(dev, sci, icv_len);
4109	if (err)
4110	goto unlink;
4111
4112	if (data) {
4113	err = macsec_changelink_common(dev, data);
4114	if (err)
4115	goto del_dev;
4116	}
4117
4118	/* If h/w offloading is available, propagate to the device */
4119	if (macsec_is_offloaded(macsec)) {
4120	const struct macsec_ops *ops;
4121	struct macsec_context ctx;
4122
4123	ops = macsec_get_ops(macsec, ctx: &ctx);
4124	if (ops) {
4125	ctx.secy = &macsec->secy;
4126	err = macsec_offload(func: ops->mdo_add_secy, ctx: &ctx);
4127	if (err)
4128	goto del_dev;
4129	}
4130	}
4131
4132	err = register_macsec_dev(real_dev, dev);
4133	if (err < 0)
4134	goto del_dev;
4135
4136	netif_stacked_transfer_operstate(rootdev: real_dev, dev);
4137	linkwatch_fire_event(dev);
4138
4139	macsec_generation++;
4140
4141	return 0;
4142
4143	del_dev:
4144	macsec_del_dev(macsec);
4145	unlink:
4146	netdev_upper_dev_unlink(dev: real_dev, upper_dev: dev);
4147	unregister:
4148	unregister_netdevice(dev);
4149	return err;
4150	}
4151
4152	static int macsec_validate_attr(struct nlattr *tb[], struct nlattr *data[],
4153	struct netlink_ext_ack *extack)
4154	{
4155	u64 csid = MACSEC_DEFAULT_CIPHER_ID;
4156	u8 icv_len = MACSEC_DEFAULT_ICV_LEN;
4157	int flag;
4158	bool es, scb, sci;
4159
4160	if (!data)
4161	return 0;
4162
4163	if (data[IFLA_MACSEC_CIPHER_SUITE])
4164	csid = nla_get_u64(nla: data[IFLA_MACSEC_CIPHER_SUITE]);
4165
4166	if (data[IFLA_MACSEC_ICV_LEN]) {
4167	icv_len = nla_get_u8(nla: data[IFLA_MACSEC_ICV_LEN]);
4168	if (icv_len != MACSEC_DEFAULT_ICV_LEN) {
4169	char dummy_key[DEFAULT_SAK_LEN] = { 0 };
4170	struct crypto_aead *dummy_tfm;
4171
4172	dummy_tfm = macsec_alloc_tfm(key: dummy_key,
4173	DEFAULT_SAK_LEN,
4174	icv_len);
4175	if (IS_ERR(ptr: dummy_tfm))
4176	return PTR_ERR(ptr: dummy_tfm);
4177	crypto_free_aead(tfm: dummy_tfm);
4178	}
4179	}
4180
4181	switch (csid) {
4182	case MACSEC_CIPHER_ID_GCM_AES_128:
4183	case MACSEC_CIPHER_ID_GCM_AES_256:
4184	case MACSEC_CIPHER_ID_GCM_AES_XPN_128:
4185	case MACSEC_CIPHER_ID_GCM_AES_XPN_256:
4186	case MACSEC_DEFAULT_CIPHER_ID:
4187	if (icv_len < MACSEC_MIN_ICV_LEN ||
4188	icv_len > MACSEC_STD_ICV_LEN)
4189	return -EINVAL;
4190	break;
4191	default:
4192	return -EINVAL;
4193	}
4194
4195	if (data[IFLA_MACSEC_ENCODING_SA]) {
4196	if (nla_get_u8(nla: data[IFLA_MACSEC_ENCODING_SA]) >= MACSEC_NUM_AN)
4197	return -EINVAL;
4198	}
4199
4200	for (flag = IFLA_MACSEC_ENCODING_SA + 1;
4201	flag < IFLA_MACSEC_VALIDATION;
4202	flag++) {
4203	if (data[flag]) {
4204	if (nla_get_u8(nla: data[flag]) > 1)
4205	return -EINVAL;
4206	}
4207	}
4208
4209	es = data[IFLA_MACSEC_ES] ? nla_get_u8(nla: data[IFLA_MACSEC_ES]) : false;
4210	sci = data[IFLA_MACSEC_INC_SCI] ? nla_get_u8(nla: data[IFLA_MACSEC_INC_SCI]) : false;
4211	scb = data[IFLA_MACSEC_SCB] ? nla_get_u8(nla: data[IFLA_MACSEC_SCB]) : false;
4212
4213	if ((sci && (scb || es)) || (scb && es))
4214	return -EINVAL;
4215
4216	if (data[IFLA_MACSEC_VALIDATION] &&
4217	nla_get_u8(nla: data[IFLA_MACSEC_VALIDATION]) > MACSEC_VALIDATE_MAX)
4218	return -EINVAL;
4219
4220	if ((data[IFLA_MACSEC_REPLAY_PROTECT] &&
4221	nla_get_u8(nla: data[IFLA_MACSEC_REPLAY_PROTECT])) &&
4222	!data[IFLA_MACSEC_WINDOW])
4223	return -EINVAL;
4224
4225	return 0;
4226	}
4227
4228	static struct net *macsec_get_link_net(const struct net_device *dev)
4229	{
4230	return dev_net(dev: macsec_priv(dev)->real_dev);
4231	}
4232
4233	struct net_device *macsec_get_real_dev(const struct net_device *dev)
4234	{
4235	return macsec_priv(dev)->real_dev;
4236	}
4237	EXPORT_SYMBOL_GPL(macsec_get_real_dev);
4238
4239	bool macsec_netdev_is_offloaded(struct net_device *dev)
4240	{
4241	return macsec_is_offloaded(macsec: macsec_priv(dev));
4242	}
4243	EXPORT_SYMBOL_GPL(macsec_netdev_is_offloaded);
4244
4245	static size_t macsec_get_size(const struct net_device *dev)
4246	{
4247	return nla_total_size_64bit(payload: 8) + /* IFLA_MACSEC_SCI */
4248	nla_total_size(payload: 1) + /* IFLA_MACSEC_ICV_LEN */
4249	nla_total_size_64bit(payload: 8) + /* IFLA_MACSEC_CIPHER_SUITE */
4250	nla_total_size(payload: 4) + /* IFLA_MACSEC_WINDOW */
4251	nla_total_size(payload: 1) + /* IFLA_MACSEC_ENCODING_SA */
4252	nla_total_size(payload: 1) + /* IFLA_MACSEC_ENCRYPT */
4253	nla_total_size(payload: 1) + /* IFLA_MACSEC_PROTECT */
4254	nla_total_size(payload: 1) + /* IFLA_MACSEC_INC_SCI */
4255	nla_total_size(payload: 1) + /* IFLA_MACSEC_ES */
4256	nla_total_size(payload: 1) + /* IFLA_MACSEC_SCB */
4257	nla_total_size(payload: 1) + /* IFLA_MACSEC_REPLAY_PROTECT */
4258	nla_total_size(payload: 1) + /* IFLA_MACSEC_VALIDATION */
4259	nla_total_size(payload: 1) + /* IFLA_MACSEC_OFFLOAD */
4260	0;
4261	}
4262
4263	static int macsec_fill_info(struct sk_buff *skb,
4264	const struct net_device *dev)
4265	{
4266	struct macsec_tx_sc *tx_sc;
4267	struct macsec_dev *macsec;
4268	struct macsec_secy *secy;
4269	u64 csid;
4270
4271	macsec = macsec_priv(dev);
4272	secy = &macsec->secy;
4273	tx_sc = &secy->tx_sc;
4274
4275	switch (secy->key_len) {
4276	case MACSEC_GCM_AES_128_SAK_LEN:
4277	csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_128 : MACSEC_DEFAULT_CIPHER_ID;
4278	break;
4279	case MACSEC_GCM_AES_256_SAK_LEN:
4280	csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_256 : MACSEC_CIPHER_ID_GCM_AES_256;
4281	break;
4282	default:
4283	goto nla_put_failure;
4284	}
4285
4286	if (nla_put_sci(skb, attrtype: IFLA_MACSEC_SCI, value: secy->sci,
4287	padattr: IFLA_MACSEC_PAD) ||
4288	nla_put_u8(skb, attrtype: IFLA_MACSEC_ICV_LEN, value: secy->icv_len) ||
4289	nla_put_u64_64bit(skb, attrtype: IFLA_MACSEC_CIPHER_SUITE,
4290	value: csid, padattr: IFLA_MACSEC_PAD) ||
4291	nla_put_u8(skb, attrtype: IFLA_MACSEC_ENCODING_SA, value: tx_sc->encoding_sa) ||
4292	nla_put_u8(skb, attrtype: IFLA_MACSEC_ENCRYPT, value: tx_sc->encrypt) ||
4293	nla_put_u8(skb, attrtype: IFLA_MACSEC_PROTECT, value: secy->protect_frames) ||
4294	nla_put_u8(skb, attrtype: IFLA_MACSEC_INC_SCI, value: tx_sc->send_sci) ||
4295	nla_put_u8(skb, attrtype: IFLA_MACSEC_ES, value: tx_sc->end_station) ||
4296	nla_put_u8(skb, attrtype: IFLA_MACSEC_SCB, value: tx_sc->scb) ||
4297	nla_put_u8(skb, attrtype: IFLA_MACSEC_REPLAY_PROTECT, value: secy->replay_protect) ||
4298	nla_put_u8(skb, attrtype: IFLA_MACSEC_VALIDATION, value: secy->validate_frames) ||
4299	nla_put_u8(skb, attrtype: IFLA_MACSEC_OFFLOAD, value: macsec->offload) ||
4300	0)
4301	goto nla_put_failure;
4302
4303	if (secy->replay_protect) {
4304	if (nla_put_u32(skb, attrtype: IFLA_MACSEC_WINDOW, value: secy->replay_window))
4305	goto nla_put_failure;
4306	}
4307
4308	return 0;
4309
4310	nla_put_failure:
4311	return -EMSGSIZE;
4312	}
4313
4314	static struct rtnl_link_ops macsec_link_ops __read_mostly = {
4315	.kind = "macsec",
4316	.priv_size = sizeof(struct macsec_dev),
4317	.maxtype = IFLA_MACSEC_MAX,
4318	.policy = macsec_rtnl_policy,
4319	.setup = macsec_setup,
4320	.validate = macsec_validate_attr,
4321	.newlink = macsec_newlink,
4322	.changelink = macsec_changelink,
4323	.dellink = macsec_dellink,
4324	.get_size = macsec_get_size,
4325	.fill_info = macsec_fill_info,
4326	.get_link_net = macsec_get_link_net,
4327	};
4328
4329	static bool is_macsec_master(struct net_device *dev)
4330	{
4331	return rcu_access_pointer(dev->rx_handler) == macsec_handle_frame;
4332	}
4333
4334	static int macsec_notify(struct notifier_block *this, unsigned long event,
4335	void *ptr)
4336	{
4337	struct net_device *real_dev = netdev_notifier_info_to_dev(info: ptr);
4338	LIST_HEAD(head);
4339
4340	if (!is_macsec_master(dev: real_dev))
4341	return NOTIFY_DONE;
4342
4343	switch (event) {
4344	case NETDEV_DOWN:
4345	case NETDEV_UP:
4346	case NETDEV_CHANGE: {
4347	struct macsec_dev *m, *n;
4348	struct macsec_rxh_data *rxd;
4349
4350	rxd = macsec_data_rtnl(dev: real_dev);
4351	list_for_each_entry_safe(m, n, &rxd->secys, secys) {
4352	struct net_device *dev = m->secy.netdev;
4353
4354	netif_stacked_transfer_operstate(rootdev: real_dev, dev);
4355	}
4356	break;
4357	}
4358	case NETDEV_UNREGISTER: {
4359	struct macsec_dev *m, *n;
4360	struct macsec_rxh_data *rxd;
4361
4362	rxd = macsec_data_rtnl(dev: real_dev);
4363	list_for_each_entry_safe(m, n, &rxd->secys, secys) {
4364	macsec_common_dellink(dev: m->secy.netdev, head: &head);
4365	}
4366
4367	netdev_rx_handler_unregister(dev: real_dev);
4368	kfree(objp: rxd);
4369
4370	unregister_netdevice_many(head: &head);
4371	break;
4372	}
4373	case NETDEV_CHANGEMTU: {
4374	struct macsec_dev *m;
4375	struct macsec_rxh_data *rxd;
4376
4377	rxd = macsec_data_rtnl(dev: real_dev);
4378	list_for_each_entry(m, &rxd->secys, secys) {
4379	struct net_device *dev = m->secy.netdev;
4380	unsigned int mtu = real_dev->mtu - (m->secy.icv_len +
4381	macsec_extra_len(sci_present: true));
4382
4383	if (dev->mtu > mtu)
4384	dev_set_mtu(dev, mtu);
4385	}
4386	}
4387	}
4388
4389	return NOTIFY_OK;
4390	}
4391
4392	static struct notifier_block macsec_notifier = {
4393	.notifier_call = macsec_notify,
4394	};
4395
4396	static int __init macsec_init(void)
4397	{
4398	int err;
4399
4400	pr_info("MACsec IEEE 802.1AE\n");
4401	err = register_netdevice_notifier(nb: &macsec_notifier);
4402	if (err)
4403	return err;
4404
4405	err = rtnl_link_register(ops: &macsec_link_ops);
4406	if (err)
4407	goto notifier;
4408
4409	err = genl_register_family(family: &macsec_fam);
4410	if (err)
4411	goto rtnl;
4412
4413	return 0;
4414
4415	rtnl:
4416	rtnl_link_unregister(ops: &macsec_link_ops);
4417	notifier:
4418	unregister_netdevice_notifier(nb: &macsec_notifier);
4419	return err;
4420	}
4421
4422	static void __exit macsec_exit(void)
4423	{
4424	genl_unregister_family(family: &macsec_fam);
4425	rtnl_link_unregister(ops: &macsec_link_ops);
4426	unregister_netdevice_notifier(nb: &macsec_notifier);
4427	rcu_barrier();
4428	}
4429
4430	module_init(macsec_init);
4431	module_exit(macsec_exit);
4432
4433	MODULE_ALIAS_RTNL_LINK("macsec");
4434	MODULE_ALIAS_GENL_FAMILY("macsec");
4435
4436	MODULE_DESCRIPTION("MACsec IEEE 802.1AE");
4437	MODULE_LICENSE("GPL v2");
4438