1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* drivers/net/macsec.c - MACsec device
4	*
5	* Copyright (c) 2015 Sabrina Dubroca <sd@queasysnail.net>
6	*/
7
8	#include <linux/types.h>
9	#include <linux/skbuff.h>
10	#include <linux/socket.h>
11	#include <linux/module.h>
12	#include <crypto/aead.h>
13	#include <linux/etherdevice.h>
14	#include <linux/netdevice.h>
15	#include <linux/rtnetlink.h>
16	#include <linux/refcount.h>
17	#include <net/genetlink.h>
18	#include <net/sock.h>
19	#include <net/gro_cells.h>
20	#include <net/macsec.h>
21	#include <net/dst_metadata.h>
22	#include <linux/phy.h>
23	#include <linux/byteorder/generic.h>
24	#include <linux/if_arp.h>
25
26	#include <uapi/linux/if_macsec.h>
27
28	/* SecTAG length = macsec_eth_header without the optional SCI */
29	#define MACSEC_TAG_LEN 6
30
31	struct macsec_eth_header {
32	struct ethhdr eth;
33	/* SecTAG */
34	u8 tci_an;
35	#if defined(__LITTLE_ENDIAN_BITFIELD)
36	u8 short_length:6,
37	unused:2;
38	#elif defined(__BIG_ENDIAN_BITFIELD)
39	u8 unused:2,
40	short_length:6;
41	#else
42	#error "Please fix <asm/byteorder.h>"
43	#endif
44	__be32 packet_number;
45	u8 secure_channel_id[8]; /* optional */
46	} __packed;
47
48	/* minimum secure data length deemed "not short", see IEEE 802.1AE-2006 9.7 */
49	#define MIN_NON_SHORT_LEN 48
50
51	#define GCM_AES_IV_LEN 12
52
53	#define for_each_rxsc(secy, sc) \
54	for (sc = rcu_dereference_bh(secy->rx_sc); \
55	sc; \
56	sc = rcu_dereference_bh(sc->next))
57	#define for_each_rxsc_rtnl(secy, sc) \
58	for (sc = rtnl_dereference(secy->rx_sc); \
59	sc; \
60	sc = rtnl_dereference(sc->next))
61
62	#define pn_same_half(pn1, pn2) (!(((pn1) >> 31) ^ ((pn2) >> 31)))
63
64	struct gcm_iv_xpn {
65	union {
66	u8 short_secure_channel_id[4];
67	ssci_t ssci;
68	};
69	__be64 pn;
70	} __packed;
71
72	struct gcm_iv {
73	union {
74	u8 secure_channel_id[8];
75	sci_t sci;
76	};
77	__be32 pn;
78	};
79
80	#define MACSEC_VALIDATE_DEFAULT MACSEC_VALIDATE_STRICT
81
82	struct pcpu_secy_stats {
83	struct macsec_dev_stats stats;
84	struct u64_stats_sync syncp;
85	};
86
87	/**
88	* struct macsec_dev - private data
89	* @secy: SecY config
90	* @real_dev: pointer to underlying netdevice
91	* @dev_tracker: refcount tracker for @real_dev reference
92	* @stats: MACsec device stats
93	* @secys: linked list of SecY's on the underlying device
94	* @gro_cells: pointer to the Generic Receive Offload cell
95	* @offload: status of offloading on the MACsec device
96	* @insert_tx_tag: when offloading, device requires to insert an
97	* additional tag
98	*/
99	struct macsec_dev {
100	struct macsec_secy secy;
101	struct net_device *real_dev;
102	netdevice_tracker dev_tracker;
103	struct pcpu_secy_stats __percpu *stats;
104	struct list_head secys;
105	struct gro_cells gro_cells;
106	enum macsec_offload offload;
107	bool insert_tx_tag;
108	};
109
110	/**
111	* struct macsec_rxh_data - rx_handler private argument
112	* @secys: linked list of SecY's on this underlying device
113	*/
114	struct macsec_rxh_data {
115	struct list_head secys;
116	};
117
118	static struct macsec_dev *macsec_priv(const struct net_device *dev)
119	{
120	return (struct macsec_dev *)netdev_priv(dev);
121	}
122
123	static struct macsec_rxh_data *macsec_data_rcu(const struct net_device *dev)
124	{
125	return rcu_dereference_bh(dev->rx_handler_data);
126	}
127
128	static struct macsec_rxh_data *macsec_data_rtnl(const struct net_device *dev)
129	{
130	return rtnl_dereference(dev->rx_handler_data);
131	}
132
133	struct macsec_cb {
134	struct aead_request *req;
135	union {
136	struct macsec_tx_sa *tx_sa;
137	struct macsec_rx_sa *rx_sa;
138	};
139	u8 assoc_num;
140	bool valid;
141	bool has_sci;
142	};
143
144	static struct macsec_rx_sa *macsec_rxsa_get(struct macsec_rx_sa __rcu *ptr)
145	{
146	struct macsec_rx_sa *sa = rcu_dereference_bh(ptr);
147
148	if (!sa || !sa->active)
149	return NULL;
150
151	if (!refcount_inc_not_zero(r: &sa->refcnt))
152	return NULL;
153
154	return sa;
155	}
156
157	static struct macsec_rx_sa *macsec_active_rxsa_get(struct macsec_rx_sc *rx_sc)
158	{
159	struct macsec_rx_sa *sa = NULL;
160	int an;
161
162	for (an = 0; an < MACSEC_NUM_AN; an++) {
163	sa = macsec_rxsa_get(ptr: rx_sc->sa[an]);
164	if (sa)
165	break;
166	}
167	return sa;
168	}
169
170	static void free_rx_sc_rcu(struct rcu_head *head)
171	{
172	struct macsec_rx_sc *rx_sc = container_of(head, struct macsec_rx_sc, rcu_head);
173
174	free_percpu(pdata: rx_sc->stats);
175	kfree(objp: rx_sc);
176	}
177
178	static struct macsec_rx_sc *macsec_rxsc_get(struct macsec_rx_sc *sc)
179	{
180	return refcount_inc_not_zero(r: &sc->refcnt) ? sc : NULL;
181	}
182
183	static void macsec_rxsc_put(struct macsec_rx_sc *sc)
184	{
185	if (refcount_dec_and_test(r: &sc->refcnt))
186	call_rcu(head: &sc->rcu_head, func: free_rx_sc_rcu);
187	}
188
189	static void free_rxsa(struct rcu_head *head)
190	{
191	struct macsec_rx_sa *sa = container_of(head, struct macsec_rx_sa, rcu);
192
193	crypto_free_aead(tfm: sa->key.tfm);
194	free_percpu(pdata: sa->stats);
195	kfree(objp: sa);
196	}
197
198	static void macsec_rxsa_put(struct macsec_rx_sa *sa)
199	{
200	if (refcount_dec_and_test(r: &sa->refcnt))
201	call_rcu(head: &sa->rcu, func: free_rxsa);
202	}
203
204	static struct macsec_tx_sa *macsec_txsa_get(struct macsec_tx_sa __rcu *ptr)
205	{
206	struct macsec_tx_sa *sa = rcu_dereference_bh(ptr);
207
208	if (!sa || !sa->active)
209	return NULL;
210
211	if (!refcount_inc_not_zero(r: &sa->refcnt))
212	return NULL;
213
214	return sa;
215	}
216
217	static void free_txsa(struct rcu_head *head)
218	{
219	struct macsec_tx_sa *sa = container_of(head, struct macsec_tx_sa, rcu);
220
221	crypto_free_aead(tfm: sa->key.tfm);
222	free_percpu(pdata: sa->stats);
223	kfree(objp: sa);
224	}
225
226	static void macsec_txsa_put(struct macsec_tx_sa *sa)
227	{
228	if (refcount_dec_and_test(r: &sa->refcnt))
229	call_rcu(head: &sa->rcu, func: free_txsa);
230	}
231
232	static struct macsec_cb *macsec_skb_cb(struct sk_buff *skb)
233	{
234	BUILD_BUG_ON(sizeof(struct macsec_cb) > sizeof(skb->cb));
235	return (struct macsec_cb *)skb->cb;
236	}
237
238	#define MACSEC_PORT_SCB (0x0000)
239	#define MACSEC_UNDEF_SCI ((__force sci_t)0xffffffffffffffffULL)
240	#define MACSEC_UNDEF_SSCI ((__force ssci_t)0xffffffff)
241
242	#define MACSEC_GCM_AES_128_SAK_LEN 16
243	#define MACSEC_GCM_AES_256_SAK_LEN 32
244
245	#define DEFAULT_SAK_LEN MACSEC_GCM_AES_128_SAK_LEN
246	#define DEFAULT_XPN false
247	#define DEFAULT_SEND_SCI true
248	#define DEFAULT_ENCRYPT false
249	#define DEFAULT_ENCODING_SA 0
250	#define MACSEC_XPN_MAX_REPLAY_WINDOW (((1 << 30) - 1))
251
252	static sci_t make_sci(const u8 *addr, __be16 port)
253	{
254	sci_t sci;
255
256	memcpy(&sci, addr, ETH_ALEN);
257	memcpy(((char *)&sci) + ETH_ALEN, &port, sizeof(port));
258
259	return sci;
260	}
261
262	static sci_t macsec_frame_sci(struct macsec_eth_header *hdr, bool sci_present)
263	{
264	sci_t sci;
265
266	if (sci_present)
267	memcpy(&sci, hdr->secure_channel_id,
268	sizeof(hdr->secure_channel_id));
269	else
270	sci = make_sci(addr: hdr->eth.h_source, MACSEC_PORT_ES);
271
272	return sci;
273	}
274
275	static unsigned int macsec_sectag_len(bool sci_present)
276	{
277	return MACSEC_TAG_LEN + (sci_present ? MACSEC_SCI_LEN : 0);
278	}
279
280	static unsigned int macsec_hdr_len(bool sci_present)
281	{
282	return macsec_sectag_len(sci_present) + ETH_HLEN;
283	}
284
285	static unsigned int macsec_extra_len(bool sci_present)
286	{
287	return macsec_sectag_len(sci_present) + sizeof(__be16);
288	}
289
290	/* Fill SecTAG according to IEEE 802.1AE-2006 10.5.3 */
291	static void macsec_fill_sectag(struct macsec_eth_header *h,
292	const struct macsec_secy *secy, u32 pn,
293	bool sci_present)
294	{
295	const struct macsec_tx_sc *tx_sc = &secy->tx_sc;
296
297	memset(&h->tci_an, 0, macsec_sectag_len(sci_present));
298	h->eth.h_proto = htons(ETH_P_MACSEC);
299
300	if (sci_present) {
301	h->tci_an |= MACSEC_TCI_SC;
302	memcpy(&h->secure_channel_id, &secy->sci,
303	sizeof(h->secure_channel_id));
304	} else {
305	if (tx_sc->end_station)
306	h->tci_an |= MACSEC_TCI_ES;
307	if (tx_sc->scb)
308	h->tci_an |= MACSEC_TCI_SCB;
309	}
310
311	h->packet_number = htonl(pn);
312
313	/* with GCM, C/E clear for !encrypt, both set for encrypt */
314	if (tx_sc->encrypt)
315	h->tci_an |= MACSEC_TCI_CONFID;
316	else if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN)
317	h->tci_an |= MACSEC_TCI_C;
318
319	h->tci_an |= tx_sc->encoding_sa;
320	}
321
322	static void macsec_set_shortlen(struct macsec_eth_header *h, size_t data_len)
323	{
324	if (data_len < MIN_NON_SHORT_LEN)
325	h->short_length = data_len;
326	}
327
328	/* Checks if a MACsec interface is being offloaded to an hardware engine */
329	static bool macsec_is_offloaded(struct macsec_dev *macsec)
330	{
331	if (macsec->offload == MACSEC_OFFLOAD_MAC ||
332	macsec->offload == MACSEC_OFFLOAD_PHY)
333	return true;
334
335	return false;
336	}
337
338	/* Checks if underlying layers implement MACsec offloading functions. */
339	static bool macsec_check_offload(enum macsec_offload offload,
340	struct macsec_dev *macsec)
341	{
342	if (!macsec || !macsec->real_dev)
343	return false;
344
345	if (offload == MACSEC_OFFLOAD_PHY)
346	return macsec->real_dev->phydev &&
347	macsec->real_dev->phydev->macsec_ops;
348	else if (offload == MACSEC_OFFLOAD_MAC)
349	return macsec->real_dev->features & NETIF_F_HW_MACSEC &&
350	macsec->real_dev->macsec_ops;
351
352	return false;
353	}
354
355	static const struct macsec_ops *__macsec_get_ops(enum macsec_offload offload,
356	struct macsec_dev *macsec,
357	struct macsec_context *ctx)
358	{
359	if (ctx) {
360	memset(ctx, 0, sizeof(*ctx));
361	ctx->offload = offload;
362
363	if (offload == MACSEC_OFFLOAD_PHY)
364	ctx->phydev = macsec->real_dev->phydev;
365	else if (offload == MACSEC_OFFLOAD_MAC)
366	ctx->netdev = macsec->real_dev;
367	}
368
369	if (offload == MACSEC_OFFLOAD_PHY)
370	return macsec->real_dev->phydev->macsec_ops;
371	else
372	return macsec->real_dev->macsec_ops;
373	}
374
375	/* Returns a pointer to the MACsec ops struct if any and updates the MACsec
376	* context device reference if provided.
377	*/
378	static const struct macsec_ops *macsec_get_ops(struct macsec_dev *macsec,
379	struct macsec_context *ctx)
380	{
381	if (!macsec_check_offload(offload: macsec->offload, macsec))
382	return NULL;
383
384	return __macsec_get_ops(offload: macsec->offload, macsec, ctx);
385	}
386
387	/* validate MACsec packet according to IEEE 802.1AE-2018 9.12 */
388	static bool macsec_validate_skb(struct sk_buff *skb, u16 icv_len, bool xpn)
389	{
390	struct macsec_eth_header *h = (struct macsec_eth_header *)skb->data;
391	int len = skb->len - 2 * ETH_ALEN;
392	int extra_len = macsec_extra_len(sci_present: !!(h->tci_an & MACSEC_TCI_SC)) + icv_len;
393
394	/* a) It comprises at least 17 octets */
395	if (skb->len <= 16)
396	return false;
397
398	/* b) MACsec EtherType: already checked */
399
400	/* c) V bit is clear */
401	if (h->tci_an & MACSEC_TCI_VERSION)
402	return false;
403
404	/* d) ES or SCB => !SC */
405	if ((h->tci_an & MACSEC_TCI_ES || h->tci_an & MACSEC_TCI_SCB) &&
406	(h->tci_an & MACSEC_TCI_SC))
407	return false;
408
409	/* e) Bits 7 and 8 of octet 4 of the SecTAG are clear */
410	if (h->unused)
411	return false;
412
413	/* rx.pn != 0 if not XPN (figure 10-5 with 802.11AEbw-2013 amendment) */
414	if (!h->packet_number && !xpn)
415	return false;
416
417	/* length check, f) g) h) i) */
418	if (h->short_length)
419	return len == extra_len + h->short_length;
420	return len >= extra_len + MIN_NON_SHORT_LEN;
421	}
422
423	#define MACSEC_NEEDED_HEADROOM (macsec_extra_len(true))
424	#define MACSEC_NEEDED_TAILROOM MACSEC_STD_ICV_LEN
425
426	static void macsec_fill_iv_xpn(unsigned char *iv, ssci_t ssci, u64 pn,
427	salt_t salt)
428	{
429	struct gcm_iv_xpn *gcm_iv = (struct gcm_iv_xpn *)iv;
430
431	gcm_iv->ssci = ssci ^ salt.ssci;
432	gcm_iv->pn = cpu_to_be64(pn) ^ salt.pn;
433	}
434
435	static void macsec_fill_iv(unsigned char *iv, sci_t sci, u32 pn)
436	{
437	struct gcm_iv *gcm_iv = (struct gcm_iv *)iv;
438
439	gcm_iv->sci = sci;
440	gcm_iv->pn = htonl(pn);
441	}
442
443	static struct macsec_eth_header *macsec_ethhdr(struct sk_buff *skb)
444	{
445	return (struct macsec_eth_header *)skb_mac_header(skb);
446	}
447
448	static void __macsec_pn_wrapped(struct macsec_secy *secy,
449	struct macsec_tx_sa *tx_sa)
450	{
451	pr_debug("PN wrapped, transitioning to !oper\n");
452	tx_sa->active = false;
453	if (secy->protect_frames)
454	secy->operational = false;
455	}
456
457	void macsec_pn_wrapped(struct macsec_secy *secy, struct macsec_tx_sa *tx_sa)
458	{
459	spin_lock_bh(lock: &tx_sa->lock);
460	__macsec_pn_wrapped(secy, tx_sa);
461	spin_unlock_bh(lock: &tx_sa->lock);
462	}
463	EXPORT_SYMBOL_GPL(macsec_pn_wrapped);
464
465	static pn_t tx_sa_update_pn(struct macsec_tx_sa *tx_sa,
466	struct macsec_secy *secy)
467	{
468	pn_t pn;
469
470	spin_lock_bh(lock: &tx_sa->lock);
471
472	pn = tx_sa->next_pn_halves;
473	if (secy->xpn)
474	tx_sa->next_pn++;
475	else
476	tx_sa->next_pn_halves.lower++;
477
478	if (tx_sa->next_pn == 0)
479	__macsec_pn_wrapped(secy, tx_sa);
480	spin_unlock_bh(lock: &tx_sa->lock);
481
482	return pn;
483	}
484
485	static void macsec_encrypt_finish(struct sk_buff *skb, struct net_device *dev)
486	{
487	struct macsec_dev *macsec = netdev_priv(dev);
488
489	skb->dev = macsec->real_dev;
490	skb_reset_mac_header(skb);
491	skb->protocol = eth_hdr(skb)->h_proto;
492	}
493
494	static unsigned int macsec_msdu_len(struct sk_buff *skb)
495	{
496	struct macsec_dev *macsec = macsec_priv(dev: skb->dev);
497	struct macsec_secy *secy = &macsec->secy;
498	bool sci_present = macsec_skb_cb(skb)->has_sci;
499
500	return skb->len - macsec_hdr_len(sci_present) - secy->icv_len;
501	}
502
503	static void macsec_count_tx(struct sk_buff *skb, struct macsec_tx_sc *tx_sc,
504	struct macsec_tx_sa *tx_sa)
505	{
506	unsigned int msdu_len = macsec_msdu_len(skb);
507	struct pcpu_tx_sc_stats *txsc_stats = this_cpu_ptr(tx_sc->stats);
508
509	u64_stats_update_begin(syncp: &txsc_stats->syncp);
510	if (tx_sc->encrypt) {
511	txsc_stats->stats.OutOctetsEncrypted += msdu_len;
512	txsc_stats->stats.OutPktsEncrypted++;
513	this_cpu_inc(tx_sa->stats->OutPktsEncrypted);
514	} else {
515	txsc_stats->stats.OutOctetsProtected += msdu_len;
516	txsc_stats->stats.OutPktsProtected++;
517	this_cpu_inc(tx_sa->stats->OutPktsProtected);
518	}
519	u64_stats_update_end(syncp: &txsc_stats->syncp);
520	}
521
522	static void count_tx(struct net_device *dev, int ret, int len)
523	{
524	if (likely(ret == NET_XMIT_SUCCESS || ret == NET_XMIT_CN))
525	dev_sw_netstats_tx_add(dev, packets: 1, len);
526	}
527
528	static void macsec_encrypt_done(void *data, int err)
529	{
530	struct sk_buff *skb = data;
531	struct net_device *dev = skb->dev;
532	struct macsec_dev *macsec = macsec_priv(dev);
533	struct macsec_tx_sa *sa = macsec_skb_cb(skb)->tx_sa;
534	int len, ret;
535
536	aead_request_free(req: macsec_skb_cb(skb)->req);
537
538	rcu_read_lock_bh();
539	macsec_count_tx(skb, tx_sc: &macsec->secy.tx_sc, tx_sa: macsec_skb_cb(skb)->tx_sa);
540	/* packet is encrypted/protected so tx_bytes must be calculated */
541	len = macsec_msdu_len(skb) + 2 * ETH_ALEN;
542	macsec_encrypt_finish(skb, dev);
543	ret = dev_queue_xmit(skb);
544	count_tx(dev, ret, len);
545	rcu_read_unlock_bh();
546
547	macsec_txsa_put(sa);
548	dev_put(dev);
549	}
550
551	static struct aead_request *macsec_alloc_req(struct crypto_aead *tfm,
552	unsigned char **iv,
553	struct scatterlist **sg,
554	int num_frags)
555	{
556	size_t size, iv_offset, sg_offset;
557	struct aead_request *req;
558	void *tmp;
559
560	size = sizeof(struct aead_request) + crypto_aead_reqsize(tfm);
561	iv_offset = size;
562	size += GCM_AES_IV_LEN;
563
564	size = ALIGN(size, __alignof__(struct scatterlist));
565	sg_offset = size;
566	size += sizeof(struct scatterlist) * num_frags;
567
568	tmp = kmalloc(size, GFP_ATOMIC);
569	if (!tmp)
570	return NULL;
571
572	*iv = (unsigned char *)(tmp + iv_offset);
573	*sg = (struct scatterlist *)(tmp + sg_offset);
574	req = tmp;
575
576	aead_request_set_tfm(req, tfm);
577
578	return req;
579	}
580
581	static struct sk_buff *macsec_encrypt(struct sk_buff *skb,
582	struct net_device *dev)
583	{
584	int ret;
585	struct scatterlist *sg;
586	struct sk_buff *trailer;
587	unsigned char *iv;
588	struct ethhdr *eth;
589	struct macsec_eth_header *hh;
590	size_t unprotected_len;
591	struct aead_request *req;
592	struct macsec_secy *secy;
593	struct macsec_tx_sc *tx_sc;
594	struct macsec_tx_sa *tx_sa;
595	struct macsec_dev *macsec = macsec_priv(dev);
596	bool sci_present;
597	pn_t pn;
598
599	secy = &macsec->secy;
600	tx_sc = &secy->tx_sc;
601
602	/* 10.5.1 TX SA assignment */
603	tx_sa = macsec_txsa_get(ptr: tx_sc->sa[tx_sc->encoding_sa]);
604	if (!tx_sa) {
605	secy->operational = false;
606	kfree_skb(skb);
607	return ERR_PTR(error: -EINVAL);
608	}
609
610	if (unlikely(skb_headroom(skb) < MACSEC_NEEDED_HEADROOM ||
611	skb_tailroom(skb) < MACSEC_NEEDED_TAILROOM)) {
612	struct sk_buff *nskb = skb_copy_expand(skb,
613	MACSEC_NEEDED_HEADROOM,
614	MACSEC_NEEDED_TAILROOM,
615	GFP_ATOMIC);
616	if (likely(nskb)) {
617	consume_skb(skb);
618	skb = nskb;
619	} else {
620	macsec_txsa_put(sa: tx_sa);
621	kfree_skb(skb);
622	return ERR_PTR(error: -ENOMEM);
623	}
624	} else {
625	skb = skb_unshare(skb, GFP_ATOMIC);
626	if (!skb) {
627	macsec_txsa_put(sa: tx_sa);
628	return ERR_PTR(error: -ENOMEM);
629	}
630	}
631
632	unprotected_len = skb->len;
633	eth = eth_hdr(skb);
634	sci_present = macsec_send_sci(secy);
635	hh = skb_push(skb, len: macsec_extra_len(sci_present));
636	memmove(hh, eth, 2 * ETH_ALEN);
637
638	pn = tx_sa_update_pn(tx_sa, secy);
639	if (pn.full64 == 0) {
640	macsec_txsa_put(sa: tx_sa);
641	kfree_skb(skb);
642	return ERR_PTR(error: -ENOLINK);
643	}
644	macsec_fill_sectag(h: hh, secy, pn: pn.lower, sci_present);
645	macsec_set_shortlen(h: hh, data_len: unprotected_len - 2 * ETH_ALEN);
646
647	skb_put(skb, len: secy->icv_len);
648
649	if (skb->len - ETH_HLEN > macsec_priv(dev)->real_dev->mtu) {
650	struct pcpu_secy_stats *secy_stats = this_cpu_ptr(macsec->stats);
651
652	u64_stats_update_begin(syncp: &secy_stats->syncp);
653	secy_stats->stats.OutPktsTooLong++;
654	u64_stats_update_end(syncp: &secy_stats->syncp);
655
656	macsec_txsa_put(sa: tx_sa);
657	kfree_skb(skb);
658	return ERR_PTR(error: -EINVAL);
659	}
660
661	ret = skb_cow_data(skb, tailbits: 0, trailer: &trailer);
662	if (unlikely(ret < 0)) {
663	macsec_txsa_put(sa: tx_sa);
664	kfree_skb(skb);
665	return ERR_PTR(error: ret);
666	}
667
668	req = macsec_alloc_req(tfm: tx_sa->key.tfm, iv: &iv, sg: &sg, num_frags: ret);
669	if (!req) {
670	macsec_txsa_put(sa: tx_sa);
671	kfree_skb(skb);
672	return ERR_PTR(error: -ENOMEM);
673	}
674
675	if (secy->xpn)
676	macsec_fill_iv_xpn(iv, ssci: tx_sa->ssci, pn: pn.full64, salt: tx_sa->key.salt);
677	else
678	macsec_fill_iv(iv, sci: secy->sci, pn: pn.lower);
679
680	sg_init_table(sg, ret);
681	ret = skb_to_sgvec(skb, sg, offset: 0, len: skb->len);
682	if (unlikely(ret < 0)) {
683	aead_request_free(req);
684	macsec_txsa_put(sa: tx_sa);
685	kfree_skb(skb);
686	return ERR_PTR(error: ret);
687	}
688
689	if (tx_sc->encrypt) {
690	int len = skb->len - macsec_hdr_len(sci_present) -
691	secy->icv_len;
692	aead_request_set_crypt(req, src: sg, dst: sg, cryptlen: len, iv);
693	aead_request_set_ad(req, assoclen: macsec_hdr_len(sci_present));
694	} else {
695	aead_request_set_crypt(req, src: sg, dst: sg, cryptlen: 0, iv);
696	aead_request_set_ad(req, assoclen: skb->len - secy->icv_len);
697	}
698
699	macsec_skb_cb(skb)->req = req;
700	macsec_skb_cb(skb)->tx_sa = tx_sa;
701	macsec_skb_cb(skb)->has_sci = sci_present;
702	aead_request_set_callback(req, flags: 0, compl: macsec_encrypt_done, data: skb);
703
704	dev_hold(dev: skb->dev);
705	ret = crypto_aead_encrypt(req);
706	if (ret == -EINPROGRESS) {
707	return ERR_PTR(error: ret);
708	} else if (ret != 0) {
709	dev_put(dev: skb->dev);
710	kfree_skb(skb);
711	aead_request_free(req);
712	macsec_txsa_put(sa: tx_sa);
713	return ERR_PTR(error: -EINVAL);
714	}
715
716	dev_put(dev: skb->dev);
717	aead_request_free(req);
718	macsec_txsa_put(sa: tx_sa);
719
720	return skb;
721	}
722
723	static bool macsec_post_decrypt(struct sk_buff *skb, struct macsec_secy *secy, u32 pn)
724	{
725	struct macsec_rx_sa *rx_sa = macsec_skb_cb(skb)->rx_sa;
726	struct pcpu_rx_sc_stats *rxsc_stats = this_cpu_ptr(rx_sa->sc->stats);
727	struct macsec_eth_header *hdr = macsec_ethhdr(skb);
728	u32 lowest_pn = 0;
729
730	spin_lock(lock: &rx_sa->lock);
731	if (rx_sa->next_pn_halves.lower >= secy->replay_window)
732	lowest_pn = rx_sa->next_pn_halves.lower - secy->replay_window;
733
734	/* Now perform replay protection check again
735	* (see IEEE 802.1AE-2006 figure 10-5)
736	*/
737	if (secy->replay_protect && pn < lowest_pn &&
738	(!secy->xpn || pn_same_half(pn, lowest_pn))) {
739	spin_unlock(lock: &rx_sa->lock);
740	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
741	rxsc_stats->stats.InPktsLate++;
742	u64_stats_update_end(syncp: &rxsc_stats->syncp);
743	DEV_STATS_INC(secy->netdev, rx_dropped);
744	return false;
745	}
746
747	if (secy->validate_frames != MACSEC_VALIDATE_DISABLED) {
748	unsigned int msdu_len = macsec_msdu_len(skb);
749	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
750	if (hdr->tci_an & MACSEC_TCI_E)
751	rxsc_stats->stats.InOctetsDecrypted += msdu_len;
752	else
753	rxsc_stats->stats.InOctetsValidated += msdu_len;
754	u64_stats_update_end(syncp: &rxsc_stats->syncp);
755	}
756
757	if (!macsec_skb_cb(skb)->valid) {
758	spin_unlock(lock: &rx_sa->lock);
759
760	/* 10.6.5 */
761	if (hdr->tci_an & MACSEC_TCI_C ||
762	secy->validate_frames == MACSEC_VALIDATE_STRICT) {
763	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
764	rxsc_stats->stats.InPktsNotValid++;
765	u64_stats_update_end(syncp: &rxsc_stats->syncp);
766	this_cpu_inc(rx_sa->stats->InPktsNotValid);
767	DEV_STATS_INC(secy->netdev, rx_errors);
768	return false;
769	}
770
771	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
772	if (secy->validate_frames == MACSEC_VALIDATE_CHECK) {
773	rxsc_stats->stats.InPktsInvalid++;
774	this_cpu_inc(rx_sa->stats->InPktsInvalid);
775	} else if (pn < lowest_pn) {
776	rxsc_stats->stats.InPktsDelayed++;
777	} else {
778	rxsc_stats->stats.InPktsUnchecked++;
779	}
780	u64_stats_update_end(syncp: &rxsc_stats->syncp);
781	} else {
782	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
783	if (pn < lowest_pn) {
784	rxsc_stats->stats.InPktsDelayed++;
785	} else {
786	rxsc_stats->stats.InPktsOK++;
787	this_cpu_inc(rx_sa->stats->InPktsOK);
788	}
789	u64_stats_update_end(syncp: &rxsc_stats->syncp);
790
791	// Instead of "pn >=" - to support pn overflow in xpn
792	if (pn + 1 > rx_sa->next_pn_halves.lower) {
793	rx_sa->next_pn_halves.lower = pn + 1;
794	} else if (secy->xpn &&
795	!pn_same_half(pn, rx_sa->next_pn_halves.lower)) {
796	rx_sa->next_pn_halves.upper++;
797	rx_sa->next_pn_halves.lower = pn + 1;
798	}
799
800	spin_unlock(lock: &rx_sa->lock);
801	}
802
803	return true;
804	}
805
806	static void macsec_reset_skb(struct sk_buff *skb, struct net_device *dev)
807	{
808	skb->pkt_type = PACKET_HOST;
809	skb->protocol = eth_type_trans(skb, dev);
810
811	skb_reset_network_header(skb);
812	if (!skb_transport_header_was_set(skb))
813	skb_reset_transport_header(skb);
814	skb_reset_mac_len(skb);
815	}
816
817	static void macsec_finalize_skb(struct sk_buff *skb, u8 icv_len, u8 hdr_len)
818	{
819	skb->ip_summed = CHECKSUM_NONE;
820	memmove(skb->data + hdr_len, skb->data, 2 * ETH_ALEN);
821	skb_pull(skb, len: hdr_len);
822	pskb_trim_unique(skb, len: skb->len - icv_len);
823	}
824
825	static void count_rx(struct net_device *dev, int len)
826	{
827	dev_sw_netstats_rx_add(dev, len);
828	}
829
830	static void macsec_decrypt_done(void *data, int err)
831	{
832	struct sk_buff *skb = data;
833	struct net_device *dev = skb->dev;
834	struct macsec_dev *macsec = macsec_priv(dev);
835	struct macsec_rx_sa *rx_sa = macsec_skb_cb(skb)->rx_sa;
836	struct macsec_rx_sc *rx_sc = rx_sa->sc;
837	int len;
838	u32 pn;
839
840	aead_request_free(req: macsec_skb_cb(skb)->req);
841
842	if (!err)
843	macsec_skb_cb(skb)->valid = true;
844
845	rcu_read_lock_bh();
846	pn = ntohl(macsec_ethhdr(skb)->packet_number);
847	if (!macsec_post_decrypt(skb, secy: &macsec->secy, pn)) {
848	rcu_read_unlock_bh();
849	kfree_skb(skb);
850	goto out;
851	}
852
853	macsec_finalize_skb(skb, icv_len: macsec->secy.icv_len,
854	hdr_len: macsec_extra_len(sci_present: macsec_skb_cb(skb)->has_sci));
855	len = skb->len;
856	macsec_reset_skb(skb, dev: macsec->secy.netdev);
857
858	if (gro_cells_receive(gcells: &macsec->gro_cells, skb) == NET_RX_SUCCESS)
859	count_rx(dev, len);
860
861	rcu_read_unlock_bh();
862
863	out:
864	macsec_rxsa_put(sa: rx_sa);
865	macsec_rxsc_put(sc: rx_sc);
866	dev_put(dev);
867	}
868
869	static struct sk_buff *macsec_decrypt(struct sk_buff *skb,
870	struct net_device *dev,
871	struct macsec_rx_sa *rx_sa,
872	sci_t sci,
873	struct macsec_secy *secy)
874	{
875	int ret;
876	struct scatterlist *sg;
877	struct sk_buff *trailer;
878	unsigned char *iv;
879	struct aead_request *req;
880	struct macsec_eth_header *hdr;
881	u32 hdr_pn;
882	u16 icv_len = secy->icv_len;
883
884	macsec_skb_cb(skb)->valid = false;
885	skb = skb_share_check(skb, GFP_ATOMIC);
886	if (!skb)
887	return ERR_PTR(error: -ENOMEM);
888
889	ret = skb_cow_data(skb, tailbits: 0, trailer: &trailer);
890	if (unlikely(ret < 0)) {
891	kfree_skb(skb);
892	return ERR_PTR(error: ret);
893	}
894	req = macsec_alloc_req(tfm: rx_sa->key.tfm, iv: &iv, sg: &sg, num_frags: ret);
895	if (!req) {
896	kfree_skb(skb);
897	return ERR_PTR(error: -ENOMEM);
898	}
899
900	hdr = (struct macsec_eth_header *)skb->data;
901	hdr_pn = ntohl(hdr->packet_number);
902
903	if (secy->xpn) {
904	pn_t recovered_pn = rx_sa->next_pn_halves;
905
906	recovered_pn.lower = hdr_pn;
907	if (hdr_pn < rx_sa->next_pn_halves.lower &&
908	!pn_same_half(hdr_pn, rx_sa->next_pn_halves.lower))
909	recovered_pn.upper++;
910
911	macsec_fill_iv_xpn(iv, ssci: rx_sa->ssci, pn: recovered_pn.full64,
912	salt: rx_sa->key.salt);
913	} else {
914	macsec_fill_iv(iv, sci, pn: hdr_pn);
915	}
916
917	sg_init_table(sg, ret);
918	ret = skb_to_sgvec(skb, sg, offset: 0, len: skb->len);
919	if (unlikely(ret < 0)) {
920	aead_request_free(req);
921	kfree_skb(skb);
922	return ERR_PTR(error: ret);
923	}
924
925	if (hdr->tci_an & MACSEC_TCI_E) {
926	/* confidentiality: ethernet + macsec header
927	* authenticated, encrypted payload
928	*/
929	int len = skb->len - macsec_hdr_len(sci_present: macsec_skb_cb(skb)->has_sci);
930
931	aead_request_set_crypt(req, src: sg, dst: sg, cryptlen: len, iv);
932	aead_request_set_ad(req, assoclen: macsec_hdr_len(sci_present: macsec_skb_cb(skb)->has_sci));
933	skb = skb_unshare(skb, GFP_ATOMIC);
934	if (!skb) {
935	aead_request_free(req);
936	return ERR_PTR(error: -ENOMEM);
937	}
938	} else {
939	/* integrity only: all headers + data authenticated */
940	aead_request_set_crypt(req, src: sg, dst: sg, cryptlen: icv_len, iv);
941	aead_request_set_ad(req, assoclen: skb->len - icv_len);
942	}
943
944	macsec_skb_cb(skb)->req = req;
945	skb->dev = dev;
946	aead_request_set_callback(req, flags: 0, compl: macsec_decrypt_done, data: skb);
947
948	dev_hold(dev);
949	ret = crypto_aead_decrypt(req);
950	if (ret == -EINPROGRESS) {
951	return ERR_PTR(error: ret);
952	} else if (ret != 0) {
953	/* decryption/authentication failed
954	* 10.6 if validateFrames is disabled, deliver anyway
955	*/
956	if (ret != -EBADMSG) {
957	kfree_skb(skb);
958	skb = ERR_PTR(error: ret);
959	}
960	} else {
961	macsec_skb_cb(skb)->valid = true;
962	}
963	dev_put(dev);
964
965	aead_request_free(req);
966
967	return skb;
968	}
969
970	static struct macsec_rx_sc *find_rx_sc(struct macsec_secy *secy, sci_t sci)
971	{
972	struct macsec_rx_sc *rx_sc;
973
974	for_each_rxsc(secy, rx_sc) {
975	if (rx_sc->sci == sci)
976	return rx_sc;
977	}
978
979	return NULL;
980	}
981
982	static struct macsec_rx_sc *find_rx_sc_rtnl(struct macsec_secy *secy, sci_t sci)
983	{
984	struct macsec_rx_sc *rx_sc;
985
986	for_each_rxsc_rtnl(secy, rx_sc) {
987	if (rx_sc->sci == sci)
988	return rx_sc;
989	}
990
991	return NULL;
992	}
993
994	static enum rx_handler_result handle_not_macsec(struct sk_buff *skb)
995	{
996	/* Deliver to the uncontrolled port by default */
997	enum rx_handler_result ret = RX_HANDLER_PASS;
998	struct ethhdr *hdr = eth_hdr(skb);
999	struct metadata_dst *md_dst;
1000	struct macsec_rxh_data *rxd;
1001	struct macsec_dev *macsec;
1002
1003	rcu_read_lock();
1004	rxd = macsec_data_rcu(dev: skb->dev);
1005	md_dst = skb_metadata_dst(skb);
1006
1007	list_for_each_entry_rcu(macsec, &rxd->secys, secys) {
1008	struct sk_buff *nskb;
1009	struct pcpu_secy_stats *secy_stats = this_cpu_ptr(macsec->stats);
1010	struct net_device *ndev = macsec->secy.netdev;
1011
1012	/* If h/w offloading is enabled, HW decodes frames and strips
1013	* the SecTAG, so we have to deduce which port to deliver to.
1014	*/
1015	if (macsec_is_offloaded(macsec) && netif_running(dev: ndev)) {
1016	struct macsec_rx_sc *rx_sc = NULL;
1017
1018	if (md_dst && md_dst->type == METADATA_MACSEC)
1019	rx_sc = find_rx_sc(secy: &macsec->secy, sci: md_dst->u.macsec_info.sci);
1020
1021	if (md_dst && md_dst->type == METADATA_MACSEC && !rx_sc)
1022	continue;
1023
1024	if (ether_addr_equal_64bits(addr1: hdr->h_dest,
1025	addr2: ndev->dev_addr)) {
1026	/* exact match, divert skb to this port */
1027	skb->dev = ndev;
1028	skb->pkt_type = PACKET_HOST;
1029	ret = RX_HANDLER_ANOTHER;
1030	goto out;
1031	} else if (is_multicast_ether_addr_64bits(
1032	addr: hdr->h_dest)) {
1033	/* multicast frame, deliver on this port too */
1034	nskb = skb_clone(skb, GFP_ATOMIC);
1035	if (!nskb)
1036	break;
1037
1038	nskb->dev = ndev;
1039	if (ether_addr_equal_64bits(addr1: hdr->h_dest,
1040	addr2: ndev->broadcast))
1041	nskb->pkt_type = PACKET_BROADCAST;
1042	else
1043	nskb->pkt_type = PACKET_MULTICAST;
1044
1045	__netif_rx(skb: nskb);
1046	} else if (rx_sc || ndev->flags & IFF_PROMISC) {
1047	skb->dev = ndev;
1048	skb->pkt_type = PACKET_HOST;
1049	ret = RX_HANDLER_ANOTHER;
1050	goto out;
1051	}
1052
1053	continue;
1054	}
1055
1056	/* 10.6 If the management control validateFrames is not
1057	* Strict, frames without a SecTAG are received, counted, and
1058	* delivered to the Controlled Port
1059	*/
1060	if (macsec->secy.validate_frames == MACSEC_VALIDATE_STRICT) {
1061	u64_stats_update_begin(syncp: &secy_stats->syncp);
1062	secy_stats->stats.InPktsNoTag++;
1063	u64_stats_update_end(syncp: &secy_stats->syncp);
1064	DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1065	continue;
1066	}
1067
1068	/* deliver on this port */
1069	nskb = skb_clone(skb, GFP_ATOMIC);
1070	if (!nskb)
1071	break;
1072
1073	nskb->dev = ndev;
1074
1075	if (__netif_rx(skb: nskb) == NET_RX_SUCCESS) {
1076	u64_stats_update_begin(syncp: &secy_stats->syncp);
1077	secy_stats->stats.InPktsUntagged++;
1078	u64_stats_update_end(syncp: &secy_stats->syncp);
1079	}
1080	}
1081
1082	out:
1083	rcu_read_unlock();
1084	return ret;
1085	}
1086
1087	static rx_handler_result_t macsec_handle_frame(struct sk_buff **pskb)
1088	{
1089	struct sk_buff *skb = *pskb;
1090	struct net_device *dev = skb->dev;
1091	struct macsec_eth_header *hdr;
1092	struct macsec_secy *secy = NULL;
1093	struct macsec_rx_sc *rx_sc;
1094	struct macsec_rx_sa *rx_sa;
1095	struct macsec_rxh_data *rxd;
1096	struct macsec_dev *macsec;
1097	unsigned int len;
1098	sci_t sci;
1099	u32 hdr_pn;
1100	bool cbit;
1101	struct pcpu_rx_sc_stats *rxsc_stats;
1102	struct pcpu_secy_stats *secy_stats;
1103	bool pulled_sci;
1104	int ret;
1105
1106	if (skb_headroom(skb) < ETH_HLEN)
1107	goto drop_direct;
1108
1109	hdr = macsec_ethhdr(skb);
1110	if (hdr->eth.h_proto != htons(ETH_P_MACSEC))
1111	return handle_not_macsec(skb);
1112
1113	skb = skb_unshare(skb, GFP_ATOMIC);
1114	*pskb = skb;
1115	if (!skb)
1116	return RX_HANDLER_CONSUMED;
1117
1118	pulled_sci = pskb_may_pull(skb, len: macsec_extra_len(sci_present: true));
1119	if (!pulled_sci) {
1120	if (!pskb_may_pull(skb, len: macsec_extra_len(sci_present: false)))
1121	goto drop_direct;
1122	}
1123
1124	hdr = macsec_ethhdr(skb);
1125
1126	/* Frames with a SecTAG that has the TCI E bit set but the C
1127	* bit clear are discarded, as this reserved encoding is used
1128	* to identify frames with a SecTAG that are not to be
1129	* delivered to the Controlled Port.
1130	*/
1131	if ((hdr->tci_an & (MACSEC_TCI_C | MACSEC_TCI_E)) == MACSEC_TCI_E)
1132	return RX_HANDLER_PASS;
1133
1134	/* now, pull the extra length */
1135	if (hdr->tci_an & MACSEC_TCI_SC) {
1136	if (!pulled_sci)
1137	goto drop_direct;
1138	}
1139
1140	/* ethernet header is part of crypto processing */
1141	skb_push(skb, ETH_HLEN);
1142
1143	macsec_skb_cb(skb)->has_sci = !!(hdr->tci_an & MACSEC_TCI_SC);
1144	macsec_skb_cb(skb)->assoc_num = hdr->tci_an & MACSEC_AN_MASK;
1145	sci = macsec_frame_sci(hdr, sci_present: macsec_skb_cb(skb)->has_sci);
1146
1147	rcu_read_lock();
1148	rxd = macsec_data_rcu(dev: skb->dev);
1149
1150	list_for_each_entry_rcu(macsec, &rxd->secys, secys) {
1151	struct macsec_rx_sc *sc = find_rx_sc(secy: &macsec->secy, sci);
1152
1153	sc = sc ? macsec_rxsc_get(sc) : NULL;
1154
1155	if (sc) {
1156	secy = &macsec->secy;
1157	rx_sc = sc;
1158	break;
1159	}
1160	}
1161
1162	if (!secy)
1163	goto nosci;
1164
1165	dev = secy->netdev;
1166	macsec = macsec_priv(dev);
1167	secy_stats = this_cpu_ptr(macsec->stats);
1168	rxsc_stats = this_cpu_ptr(rx_sc->stats);
1169
1170	if (!macsec_validate_skb(skb, icv_len: secy->icv_len, xpn: secy->xpn)) {
1171	u64_stats_update_begin(syncp: &secy_stats->syncp);
1172	secy_stats->stats.InPktsBadTag++;
1173	u64_stats_update_end(syncp: &secy_stats->syncp);
1174	DEV_STATS_INC(secy->netdev, rx_errors);
1175	goto drop_nosa;
1176	}
1177
1178	rx_sa = macsec_rxsa_get(ptr: rx_sc->sa[macsec_skb_cb(skb)->assoc_num]);
1179	if (!rx_sa) {
1180	/* 10.6.1 if the SA is not in use */
1181
1182	/* If validateFrames is Strict or the C bit in the
1183	* SecTAG is set, discard
1184	*/
1185	struct macsec_rx_sa *active_rx_sa = macsec_active_rxsa_get(rx_sc);
1186	if (hdr->tci_an & MACSEC_TCI_C ||
1187	secy->validate_frames == MACSEC_VALIDATE_STRICT) {
1188	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
1189	rxsc_stats->stats.InPktsNotUsingSA++;
1190	u64_stats_update_end(syncp: &rxsc_stats->syncp);
1191	DEV_STATS_INC(secy->netdev, rx_errors);
1192	if (active_rx_sa)
1193	this_cpu_inc(active_rx_sa->stats->InPktsNotUsingSA);
1194	goto drop_nosa;
1195	}
1196
1197	/* not Strict, the frame (with the SecTAG and ICV
1198	* removed) is delivered to the Controlled Port.
1199	*/
1200	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
1201	rxsc_stats->stats.InPktsUnusedSA++;
1202	u64_stats_update_end(syncp: &rxsc_stats->syncp);
1203	if (active_rx_sa)
1204	this_cpu_inc(active_rx_sa->stats->InPktsUnusedSA);
1205	goto deliver;
1206	}
1207
1208	/* First, PN check to avoid decrypting obviously wrong packets */
1209	hdr_pn = ntohl(hdr->packet_number);
1210	if (secy->replay_protect) {
1211	bool late;
1212
1213	spin_lock(lock: &rx_sa->lock);
1214	late = rx_sa->next_pn_halves.lower >= secy->replay_window &&
1215	hdr_pn < (rx_sa->next_pn_halves.lower - secy->replay_window);
1216
1217	if (secy->xpn)
1218	late = late && pn_same_half(rx_sa->next_pn_halves.lower, hdr_pn);
1219	spin_unlock(lock: &rx_sa->lock);
1220
1221	if (late) {
1222	u64_stats_update_begin(syncp: &rxsc_stats->syncp);
1223	rxsc_stats->stats.InPktsLate++;
1224	u64_stats_update_end(syncp: &rxsc_stats->syncp);
1225	DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1226	goto drop;
1227	}
1228	}
1229
1230	macsec_skb_cb(skb)->rx_sa = rx_sa;
1231
1232	/* Disabled && !changed text => skip validation */
1233	if (hdr->tci_an & MACSEC_TCI_C ||
1234	secy->validate_frames != MACSEC_VALIDATE_DISABLED)
1235	skb = macsec_decrypt(skb, dev, rx_sa, sci, secy);
1236
1237	if (IS_ERR(ptr: skb)) {
1238	/* the decrypt callback needs the reference */
1239	if (PTR_ERR(ptr: skb) != -EINPROGRESS) {
1240	macsec_rxsa_put(sa: rx_sa);
1241	macsec_rxsc_put(sc: rx_sc);
1242	}
1243	rcu_read_unlock();
1244	*pskb = NULL;
1245	return RX_HANDLER_CONSUMED;
1246	}
1247
1248	if (!macsec_post_decrypt(skb, secy, pn: hdr_pn))
1249	goto drop;
1250
1251	deliver:
1252	macsec_finalize_skb(skb, icv_len: secy->icv_len,
1253	hdr_len: macsec_extra_len(sci_present: macsec_skb_cb(skb)->has_sci));
1254	len = skb->len;
1255	macsec_reset_skb(skb, dev: secy->netdev);
1256
1257	if (rx_sa)
1258	macsec_rxsa_put(sa: rx_sa);
1259	macsec_rxsc_put(sc: rx_sc);
1260
1261	skb_orphan(skb);
1262	ret = gro_cells_receive(gcells: &macsec->gro_cells, skb);
1263	if (ret == NET_RX_SUCCESS)
1264	count_rx(dev, len);
1265	else
1266	DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1267
1268	rcu_read_unlock();
1269
1270	*pskb = NULL;
1271	return RX_HANDLER_CONSUMED;
1272
1273	drop:
1274	macsec_rxsa_put(sa: rx_sa);
1275	drop_nosa:
1276	macsec_rxsc_put(sc: rx_sc);
1277	rcu_read_unlock();
1278	drop_direct:
1279	kfree_skb(skb);
1280	*pskb = NULL;
1281	return RX_HANDLER_CONSUMED;
1282
1283	nosci:
1284	/* 10.6.1 if the SC is not found */
1285	cbit = !!(hdr->tci_an & MACSEC_TCI_C);
1286	if (!cbit)
1287	macsec_finalize_skb(skb, MACSEC_DEFAULT_ICV_LEN,
1288	hdr_len: macsec_extra_len(sci_present: macsec_skb_cb(skb)->has_sci));
1289
1290	list_for_each_entry_rcu(macsec, &rxd->secys, secys) {
1291	struct sk_buff *nskb;
1292
1293	secy_stats = this_cpu_ptr(macsec->stats);
1294
1295	/* If validateFrames is Strict or the C bit in the
1296	* SecTAG is set, discard
1297	*/
1298	if (cbit ||
1299	macsec->secy.validate_frames == MACSEC_VALIDATE_STRICT) {
1300	u64_stats_update_begin(syncp: &secy_stats->syncp);
1301	secy_stats->stats.InPktsNoSCI++;
1302	u64_stats_update_end(syncp: &secy_stats->syncp);
1303	DEV_STATS_INC(macsec->secy.netdev, rx_errors);
1304	continue;
1305	}
1306
1307	/* not strict, the frame (with the SecTAG and ICV
1308	* removed) is delivered to the Controlled Port.
1309	*/
1310	nskb = skb_clone(skb, GFP_ATOMIC);
1311	if (!nskb)
1312	break;
1313
1314	macsec_reset_skb(skb: nskb, dev: macsec->secy.netdev);
1315
1316	ret = __netif_rx(skb: nskb);
1317	if (ret == NET_RX_SUCCESS) {
1318	u64_stats_update_begin(syncp: &secy_stats->syncp);
1319	secy_stats->stats.InPktsUnknownSCI++;
1320	u64_stats_update_end(syncp: &secy_stats->syncp);
1321	} else {
1322	DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1323	}
1324	}
1325
1326	rcu_read_unlock();
1327	*pskb = skb;
1328	return RX_HANDLER_PASS;
1329	}
1330
1331	static struct crypto_aead *macsec_alloc_tfm(char *key, int key_len, int icv_len)
1332	{
1333	struct crypto_aead *tfm;
1334	int ret;
1335
1336	tfm = crypto_alloc_aead(alg_name: "gcm(aes)", type: 0, mask: 0);
1337
1338	if (IS_ERR(ptr: tfm))
1339	return tfm;
1340
1341	ret = crypto_aead_setkey(tfm, key, keylen: key_len);
1342	if (ret < 0)
1343	goto fail;
1344
1345	ret = crypto_aead_setauthsize(tfm, authsize: icv_len);
1346	if (ret < 0)
1347	goto fail;
1348
1349	return tfm;
1350	fail:
1351	crypto_free_aead(tfm);
1352	return ERR_PTR(error: ret);
1353	}
1354
1355	static int init_rx_sa(struct macsec_rx_sa *rx_sa, char *sak, int key_len,
1356	int icv_len)
1357	{
1358	rx_sa->stats = alloc_percpu(struct macsec_rx_sa_stats);
1359	if (!rx_sa->stats)
1360	return -ENOMEM;
1361
1362	rx_sa->key.tfm = macsec_alloc_tfm(key: sak, key_len, icv_len);
1363	if (IS_ERR(ptr: rx_sa->key.tfm)) {
1364	free_percpu(pdata: rx_sa->stats);
1365	return PTR_ERR(ptr: rx_sa->key.tfm);
1366	}
1367
1368	rx_sa->ssci = MACSEC_UNDEF_SSCI;
1369	rx_sa->active = false;
1370	rx_sa->next_pn = 1;
1371	refcount_set(r: &rx_sa->refcnt, n: 1);
1372	spin_lock_init(&rx_sa->lock);
1373
1374	return 0;
1375	}
1376
1377	static void clear_rx_sa(struct macsec_rx_sa *rx_sa)
1378	{
1379	rx_sa->active = false;
1380
1381	macsec_rxsa_put(sa: rx_sa);
1382	}
1383
1384	static void free_rx_sc(struct macsec_rx_sc *rx_sc)
1385	{
1386	int i;
1387
1388	for (i = 0; i < MACSEC_NUM_AN; i++) {
1389	struct macsec_rx_sa *sa = rtnl_dereference(rx_sc->sa[i]);
1390
1391	RCU_INIT_POINTER(rx_sc->sa[i], NULL);
1392	if (sa)
1393	clear_rx_sa(rx_sa: sa);
1394	}
1395
1396	macsec_rxsc_put(sc: rx_sc);
1397	}
1398
1399	static struct macsec_rx_sc *del_rx_sc(struct macsec_secy *secy, sci_t sci)
1400	{
1401	struct macsec_rx_sc *rx_sc, __rcu **rx_scp;
1402
1403	for (rx_scp = &secy->rx_sc, rx_sc = rtnl_dereference(*rx_scp);
1404	rx_sc;
1405	rx_scp = &rx_sc->next, rx_sc = rtnl_dereference(*rx_scp)) {
1406	if (rx_sc->sci == sci) {
1407	if (rx_sc->active)
1408	secy->n_rx_sc--;
1409	rcu_assign_pointer(*rx_scp, rx_sc->next);
1410	return rx_sc;
1411	}
1412	}
1413
1414	return NULL;
1415	}
1416
1417	static struct macsec_rx_sc *create_rx_sc(struct net_device *dev, sci_t sci,
1418	bool active)
1419	{
1420	struct macsec_rx_sc *rx_sc;
1421	struct macsec_dev *macsec;
1422	struct net_device *real_dev = macsec_priv(dev)->real_dev;
1423	struct macsec_rxh_data *rxd = macsec_data_rtnl(dev: real_dev);
1424	struct macsec_secy *secy;
1425
1426	list_for_each_entry(macsec, &rxd->secys, secys) {
1427	if (find_rx_sc_rtnl(secy: &macsec->secy, sci))
1428	return ERR_PTR(error: -EEXIST);
1429	}
1430
1431	rx_sc = kzalloc(size: sizeof(*rx_sc), GFP_KERNEL);
1432	if (!rx_sc)
1433	return ERR_PTR(error: -ENOMEM);
1434
1435	rx_sc->stats = netdev_alloc_pcpu_stats(struct pcpu_rx_sc_stats);
1436	if (!rx_sc->stats) {
1437	kfree(objp: rx_sc);
1438	return ERR_PTR(error: -ENOMEM);
1439	}
1440
1441	rx_sc->sci = sci;
1442	rx_sc->active = active;
1443	refcount_set(r: &rx_sc->refcnt, n: 1);
1444
1445	secy = &macsec_priv(dev)->secy;
1446	rcu_assign_pointer(rx_sc->next, secy->rx_sc);
1447	rcu_assign_pointer(secy->rx_sc, rx_sc);
1448
1449	if (rx_sc->active)
1450	secy->n_rx_sc++;
1451
1452	return rx_sc;
1453	}
1454
1455	static int init_tx_sa(struct macsec_tx_sa *tx_sa, char *sak, int key_len,
1456	int icv_len)
1457	{
1458	tx_sa->stats = alloc_percpu(struct macsec_tx_sa_stats);
1459	if (!tx_sa->stats)
1460	return -ENOMEM;
1461
1462	tx_sa->key.tfm = macsec_alloc_tfm(key: sak, key_len, icv_len);
1463	if (IS_ERR(ptr: tx_sa->key.tfm)) {
1464	free_percpu(pdata: tx_sa->stats);
1465	return PTR_ERR(ptr: tx_sa->key.tfm);
1466	}
1467
1468	tx_sa->ssci = MACSEC_UNDEF_SSCI;
1469	tx_sa->active = false;
1470	refcount_set(r: &tx_sa->refcnt, n: 1);
1471	spin_lock_init(&tx_sa->lock);
1472
1473	return 0;
1474	}
1475
1476	static void clear_tx_sa(struct macsec_tx_sa *tx_sa)
1477	{
1478	tx_sa->active = false;
1479
1480	macsec_txsa_put(sa: tx_sa);
1481	}
1482
1483	static struct genl_family macsec_fam;
1484
1485	static struct net_device *get_dev_from_nl(struct net *net,
1486	struct nlattr **attrs)
1487	{
1488	int ifindex = nla_get_u32(nla: attrs[MACSEC_ATTR_IFINDEX]);
1489	struct net_device *dev;
1490
1491	dev = __dev_get_by_index(net, ifindex);
1492	if (!dev)
1493	return ERR_PTR(error: -ENODEV);
1494
1495	if (!netif_is_macsec(dev))
1496	return ERR_PTR(error: -ENODEV);
1497
1498	return dev;
1499	}
1500
1501	static enum macsec_offload nla_get_offload(const struct nlattr *nla)
1502	{
1503	return (__force enum macsec_offload)nla_get_u8(nla);
1504	}
1505
1506	static sci_t nla_get_sci(const struct nlattr *nla)
1507	{
1508	return (__force sci_t)nla_get_u64(nla);
1509	}
1510
1511	static int nla_put_sci(struct sk_buff *skb, int attrtype, sci_t value,
1512	int padattr)
1513	{
1514	return nla_put_u64_64bit(skb, attrtype, value: (__force u64)value, padattr);
1515	}
1516
1517	static ssci_t nla_get_ssci(const struct nlattr *nla)
1518	{
1519	return (__force ssci_t)nla_get_u32(nla);
1520	}
1521
1522	static int nla_put_ssci(struct sk_buff *skb, int attrtype, ssci_t value)
1523	{
1524	return nla_put_u32(skb, attrtype, value: (__force u64)value);
1525	}
1526
1527	static struct macsec_tx_sa *get_txsa_from_nl(struct net *net,
1528	struct nlattr **attrs,
1529	struct nlattr **tb_sa,
1530	struct net_device **devp,
1531	struct macsec_secy **secyp,
1532	struct macsec_tx_sc **scp,
1533	u8 *assoc_num)
1534	{
1535	struct net_device *dev;
1536	struct macsec_secy *secy;
1537	struct macsec_tx_sc *tx_sc;
1538	struct macsec_tx_sa *tx_sa;
1539
1540	if (!tb_sa[MACSEC_SA_ATTR_AN])
1541	return ERR_PTR(error: -EINVAL);
1542
1543	*assoc_num = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_AN]);
1544
1545	dev = get_dev_from_nl(net, attrs);
1546	if (IS_ERR(ptr: dev))
1547	return ERR_CAST(ptr: dev);
1548
1549	if (*assoc_num >= MACSEC_NUM_AN)
1550	return ERR_PTR(error: -EINVAL);
1551
1552	secy = &macsec_priv(dev)->secy;
1553	tx_sc = &secy->tx_sc;
1554
1555	tx_sa = rtnl_dereference(tx_sc->sa[*assoc_num]);
1556	if (!tx_sa)
1557	return ERR_PTR(error: -ENODEV);
1558
1559	*devp = dev;
1560	*scp = tx_sc;
1561	*secyp = secy;
1562	return tx_sa;
1563	}
1564
1565	static struct macsec_rx_sc *get_rxsc_from_nl(struct net *net,
1566	struct nlattr **attrs,
1567	struct nlattr **tb_rxsc,
1568	struct net_device **devp,
1569	struct macsec_secy **secyp)
1570	{
1571	struct net_device *dev;
1572	struct macsec_secy *secy;
1573	struct macsec_rx_sc *rx_sc;
1574	sci_t sci;
1575
1576	dev = get_dev_from_nl(net, attrs);
1577	if (IS_ERR(ptr: dev))
1578	return ERR_CAST(ptr: dev);
1579
1580	secy = &macsec_priv(dev)->secy;
1581
1582	if (!tb_rxsc[MACSEC_RXSC_ATTR_SCI])
1583	return ERR_PTR(error: -EINVAL);
1584
1585	sci = nla_get_sci(nla: tb_rxsc[MACSEC_RXSC_ATTR_SCI]);
1586	rx_sc = find_rx_sc_rtnl(secy, sci);
1587	if (!rx_sc)
1588	return ERR_PTR(error: -ENODEV);
1589
1590	*secyp = secy;
1591	*devp = dev;
1592
1593	return rx_sc;
1594	}
1595
1596	static struct macsec_rx_sa *get_rxsa_from_nl(struct net *net,
1597	struct nlattr **attrs,
1598	struct nlattr **tb_rxsc,
1599	struct nlattr **tb_sa,
1600	struct net_device **devp,
1601	struct macsec_secy **secyp,
1602	struct macsec_rx_sc **scp,
1603	u8 *assoc_num)
1604	{
1605	struct macsec_rx_sc *rx_sc;
1606	struct macsec_rx_sa *rx_sa;
1607
1608	if (!tb_sa[MACSEC_SA_ATTR_AN])
1609	return ERR_PTR(error: -EINVAL);
1610
1611	*assoc_num = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_AN]);
1612	if (*assoc_num >= MACSEC_NUM_AN)
1613	return ERR_PTR(error: -EINVAL);
1614
1615	rx_sc = get_rxsc_from_nl(net, attrs, tb_rxsc, devp, secyp);
1616	if (IS_ERR(ptr: rx_sc))
1617	return ERR_CAST(ptr: rx_sc);
1618
1619	rx_sa = rtnl_dereference(rx_sc->sa[*assoc_num]);
1620	if (!rx_sa)
1621	return ERR_PTR(error: -ENODEV);
1622
1623	*scp = rx_sc;
1624	return rx_sa;
1625	}
1626
1627	static const struct nla_policy macsec_genl_policy[NUM_MACSEC_ATTR] = {
1628	[MACSEC_ATTR_IFINDEX] = { .type = NLA_U32 },
1629	[MACSEC_ATTR_RXSC_CONFIG] = { .type = NLA_NESTED },
1630	[MACSEC_ATTR_SA_CONFIG] = { .type = NLA_NESTED },
1631	[MACSEC_ATTR_OFFLOAD] = { .type = NLA_NESTED },
1632	};
1633
1634	static const struct nla_policy macsec_genl_rxsc_policy[NUM_MACSEC_RXSC_ATTR] = {
1635	[MACSEC_RXSC_ATTR_SCI] = { .type = NLA_U64 },
1636	[MACSEC_RXSC_ATTR_ACTIVE] = { .type = NLA_U8 },
1637	};
1638
1639	static const struct nla_policy macsec_genl_sa_policy[NUM_MACSEC_SA_ATTR] = {
1640	[MACSEC_SA_ATTR_AN] = { .type = NLA_U8 },
1641	[MACSEC_SA_ATTR_ACTIVE] = { .type = NLA_U8 },
1642	[MACSEC_SA_ATTR_PN] = NLA_POLICY_MIN_LEN(4),
1643	[MACSEC_SA_ATTR_KEYID] = { .type = NLA_BINARY,
1644	.len = MACSEC_KEYID_LEN, },
1645	[MACSEC_SA_ATTR_KEY] = { .type = NLA_BINARY,
1646	.len = MACSEC_MAX_KEY_LEN, },
1647	[MACSEC_SA_ATTR_SSCI] = { .type = NLA_U32 },
1648	[MACSEC_SA_ATTR_SALT] = { .type = NLA_BINARY,
1649	.len = MACSEC_SALT_LEN, },
1650	};
1651
1652	static const struct nla_policy macsec_genl_offload_policy[NUM_MACSEC_OFFLOAD_ATTR] = {
1653	[MACSEC_OFFLOAD_ATTR_TYPE] = { .type = NLA_U8 },
1654	};
1655
1656	/* Offloads an operation to a device driver */
1657	static int macsec_offload(int (* const func)(struct macsec_context *),
1658	struct macsec_context *ctx)
1659	{
1660	int ret;
1661
1662	if (unlikely(!func))
1663	return 0;
1664
1665	if (ctx->offload == MACSEC_OFFLOAD_PHY)
1666	mutex_lock(&ctx->phydev->lock);
1667
1668	ret = (*func)(ctx);
1669
1670	if (ctx->offload == MACSEC_OFFLOAD_PHY)
1671	mutex_unlock(lock: &ctx->phydev->lock);
1672
1673	return ret;
1674	}
1675
1676	static int parse_sa_config(struct nlattr **attrs, struct nlattr **tb_sa)
1677	{
1678	if (!attrs[MACSEC_ATTR_SA_CONFIG])
1679	return -EINVAL;
1680
1681	if (nla_parse_nested_deprecated(tb: tb_sa, maxtype: MACSEC_SA_ATTR_MAX, nla: attrs[MACSEC_ATTR_SA_CONFIG], policy: macsec_genl_sa_policy, NULL))
1682	return -EINVAL;
1683
1684	return 0;
1685	}
1686
1687	static int parse_rxsc_config(struct nlattr **attrs, struct nlattr **tb_rxsc)
1688	{
1689	if (!attrs[MACSEC_ATTR_RXSC_CONFIG])
1690	return -EINVAL;
1691
1692	if (nla_parse_nested_deprecated(tb: tb_rxsc, maxtype: MACSEC_RXSC_ATTR_MAX, nla: attrs[MACSEC_ATTR_RXSC_CONFIG], policy: macsec_genl_rxsc_policy, NULL))
1693	return -EINVAL;
1694
1695	return 0;
1696	}
1697
1698	static bool validate_add_rxsa(struct nlattr **attrs)
1699	{
1700	if (!attrs[MACSEC_SA_ATTR_AN] ||
1701	!attrs[MACSEC_SA_ATTR_KEY] ||
1702	!attrs[MACSEC_SA_ATTR_KEYID])
1703	return false;
1704
1705	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_AN]) >= MACSEC_NUM_AN)
1706	return false;
1707
1708	if (attrs[MACSEC_SA_ATTR_PN] &&
1709	nla_get_u64(nla: attrs[MACSEC_SA_ATTR_PN]) == 0)
1710	return false;
1711
1712	if (attrs[MACSEC_SA_ATTR_ACTIVE]) {
1713	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_ACTIVE]) > 1)
1714	return false;
1715	}
1716
1717	if (nla_len(nla: attrs[MACSEC_SA_ATTR_KEYID]) != MACSEC_KEYID_LEN)
1718	return false;
1719
1720	return true;
1721	}
1722
1723	static int macsec_add_rxsa(struct sk_buff *skb, struct genl_info *info)
1724	{
1725	struct net_device *dev;
1726	struct nlattr **attrs = info->attrs;
1727	struct macsec_secy *secy;
1728	struct macsec_rx_sc *rx_sc;
1729	struct macsec_rx_sa *rx_sa;
1730	unsigned char assoc_num;
1731	int pn_len;
1732	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
1733	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
1734	int err;
1735
1736	if (!attrs[MACSEC_ATTR_IFINDEX])
1737	return -EINVAL;
1738
1739	if (parse_sa_config(attrs, tb_sa))
1740	return -EINVAL;
1741
1742	if (parse_rxsc_config(attrs, tb_rxsc))
1743	return -EINVAL;
1744
1745	if (!validate_add_rxsa(attrs: tb_sa))
1746	return -EINVAL;
1747
1748	rtnl_lock();
1749	rx_sc = get_rxsc_from_nl(net: genl_info_net(info), attrs, tb_rxsc, devp: &dev, secyp: &secy);
1750	if (IS_ERR(ptr: rx_sc)) {
1751	rtnl_unlock();
1752	return PTR_ERR(ptr: rx_sc);
1753	}
1754
1755	assoc_num = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_AN]);
1756
1757	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_KEY]) != secy->key_len) {
1758	pr_notice("macsec: nl: add_rxsa: bad key length: %d != %d\n",
1759	nla_len(tb_sa[MACSEC_SA_ATTR_KEY]), secy->key_len);
1760	rtnl_unlock();
1761	return -EINVAL;
1762	}
1763
1764	pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
1765	if (tb_sa[MACSEC_SA_ATTR_PN] &&
1766	nla_len(nla: tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
1767	pr_notice("macsec: nl: add_rxsa: bad pn length: %d != %d\n",
1768	nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
1769	rtnl_unlock();
1770	return -EINVAL;
1771	}
1772
1773	if (secy->xpn) {
1774	if (!tb_sa[MACSEC_SA_ATTR_SSCI] || !tb_sa[MACSEC_SA_ATTR_SALT]) {
1775	rtnl_unlock();
1776	return -EINVAL;
1777	}
1778
1779	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_SALT]) != MACSEC_SALT_LEN) {
1780	pr_notice("macsec: nl: add_rxsa: bad salt length: %d != %d\n",
1781	nla_len(tb_sa[MACSEC_SA_ATTR_SALT]),
1782	MACSEC_SALT_LEN);
1783	rtnl_unlock();
1784	return -EINVAL;
1785	}
1786	}
1787
1788	rx_sa = rtnl_dereference(rx_sc->sa[assoc_num]);
1789	if (rx_sa) {
1790	rtnl_unlock();
1791	return -EBUSY;
1792	}
1793
1794	rx_sa = kmalloc(size: sizeof(*rx_sa), GFP_KERNEL);
1795	if (!rx_sa) {
1796	rtnl_unlock();
1797	return -ENOMEM;
1798	}
1799
1800	err = init_rx_sa(rx_sa, sak: nla_data(nla: tb_sa[MACSEC_SA_ATTR_KEY]),
1801	key_len: secy->key_len, icv_len: secy->icv_len);
1802	if (err < 0) {
1803	kfree(objp: rx_sa);
1804	rtnl_unlock();
1805	return err;
1806	}
1807
1808	if (tb_sa[MACSEC_SA_ATTR_PN]) {
1809	spin_lock_bh(lock: &rx_sa->lock);
1810	rx_sa->next_pn = nla_get_u64(nla: tb_sa[MACSEC_SA_ATTR_PN]);
1811	spin_unlock_bh(lock: &rx_sa->lock);
1812	}
1813
1814	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
1815	rx_sa->active = !!nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_ACTIVE]);
1816
1817	rx_sa->sc = rx_sc;
1818
1819	if (secy->xpn) {
1820	rx_sa->ssci = nla_get_ssci(nla: tb_sa[MACSEC_SA_ATTR_SSCI]);
1821	nla_memcpy(dest: rx_sa->key.salt.bytes, src: tb_sa[MACSEC_SA_ATTR_SALT],
1822	MACSEC_SALT_LEN);
1823	}
1824
1825	/* If h/w offloading is available, propagate to the device */
1826	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
1827	const struct macsec_ops *ops;
1828	struct macsec_context ctx;
1829
1830	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
1831	if (!ops) {
1832	err = -EOPNOTSUPP;
1833	goto cleanup;
1834	}
1835
1836	ctx.sa.assoc_num = assoc_num;
1837	ctx.sa.rx_sa = rx_sa;
1838	ctx.secy = secy;
1839	memcpy(ctx.sa.key, nla_data(tb_sa[MACSEC_SA_ATTR_KEY]),
1840	secy->key_len);
1841
1842	err = macsec_offload(func: ops->mdo_add_rxsa, ctx: &ctx);
1843	memzero_explicit(s: ctx.sa.key, count: secy->key_len);
1844	if (err)
1845	goto cleanup;
1846	}
1847
1848	nla_memcpy(dest: rx_sa->key.id, src: tb_sa[MACSEC_SA_ATTR_KEYID], MACSEC_KEYID_LEN);
1849	rcu_assign_pointer(rx_sc->sa[assoc_num], rx_sa);
1850
1851	rtnl_unlock();
1852
1853	return 0;
1854
1855	cleanup:
1856	macsec_rxsa_put(sa: rx_sa);
1857	rtnl_unlock();
1858	return err;
1859	}
1860
1861	static bool validate_add_rxsc(struct nlattr **attrs)
1862	{
1863	if (!attrs[MACSEC_RXSC_ATTR_SCI])
1864	return false;
1865
1866	if (attrs[MACSEC_RXSC_ATTR_ACTIVE]) {
1867	if (nla_get_u8(nla: attrs[MACSEC_RXSC_ATTR_ACTIVE]) > 1)
1868	return false;
1869	}
1870
1871	return true;
1872	}
1873
1874	static int macsec_add_rxsc(struct sk_buff *skb, struct genl_info *info)
1875	{
1876	struct net_device *dev;
1877	sci_t sci = MACSEC_UNDEF_SCI;
1878	struct nlattr **attrs = info->attrs;
1879	struct macsec_rx_sc *rx_sc;
1880	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
1881	struct macsec_secy *secy;
1882	bool active = true;
1883	int ret;
1884
1885	if (!attrs[MACSEC_ATTR_IFINDEX])
1886	return -EINVAL;
1887
1888	if (parse_rxsc_config(attrs, tb_rxsc))
1889	return -EINVAL;
1890
1891	if (!validate_add_rxsc(attrs: tb_rxsc))
1892	return -EINVAL;
1893
1894	rtnl_lock();
1895	dev = get_dev_from_nl(net: genl_info_net(info), attrs);
1896	if (IS_ERR(ptr: dev)) {
1897	rtnl_unlock();
1898	return PTR_ERR(ptr: dev);
1899	}
1900
1901	secy = &macsec_priv(dev)->secy;
1902	sci = nla_get_sci(nla: tb_rxsc[MACSEC_RXSC_ATTR_SCI]);
1903
1904	if (tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE])
1905	active = nla_get_u8(nla: tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]);
1906
1907	rx_sc = create_rx_sc(dev, sci, active);
1908	if (IS_ERR(ptr: rx_sc)) {
1909	rtnl_unlock();
1910	return PTR_ERR(ptr: rx_sc);
1911	}
1912
1913	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
1914	const struct macsec_ops *ops;
1915	struct macsec_context ctx;
1916
1917	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
1918	if (!ops) {
1919	ret = -EOPNOTSUPP;
1920	goto cleanup;
1921	}
1922
1923	ctx.rx_sc = rx_sc;
1924	ctx.secy = secy;
1925
1926	ret = macsec_offload(func: ops->mdo_add_rxsc, ctx: &ctx);
1927	if (ret)
1928	goto cleanup;
1929	}
1930
1931	rtnl_unlock();
1932
1933	return 0;
1934
1935	cleanup:
1936	del_rx_sc(secy, sci);
1937	free_rx_sc(rx_sc);
1938	rtnl_unlock();
1939	return ret;
1940	}
1941
1942	static bool validate_add_txsa(struct nlattr **attrs)
1943	{
1944	if (!attrs[MACSEC_SA_ATTR_AN] ||
1945	!attrs[MACSEC_SA_ATTR_PN] ||
1946	!attrs[MACSEC_SA_ATTR_KEY] ||
1947	!attrs[MACSEC_SA_ATTR_KEYID])
1948	return false;
1949
1950	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_AN]) >= MACSEC_NUM_AN)
1951	return false;
1952
1953	if (nla_get_u64(nla: attrs[MACSEC_SA_ATTR_PN]) == 0)
1954	return false;
1955
1956	if (attrs[MACSEC_SA_ATTR_ACTIVE]) {
1957	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_ACTIVE]) > 1)
1958	return false;
1959	}
1960
1961	if (nla_len(nla: attrs[MACSEC_SA_ATTR_KEYID]) != MACSEC_KEYID_LEN)
1962	return false;
1963
1964	return true;
1965	}
1966
1967	static int macsec_add_txsa(struct sk_buff *skb, struct genl_info *info)
1968	{
1969	struct net_device *dev;
1970	struct nlattr **attrs = info->attrs;
1971	struct macsec_secy *secy;
1972	struct macsec_tx_sc *tx_sc;
1973	struct macsec_tx_sa *tx_sa;
1974	unsigned char assoc_num;
1975	int pn_len;
1976	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
1977	bool was_operational;
1978	int err;
1979
1980	if (!attrs[MACSEC_ATTR_IFINDEX])
1981	return -EINVAL;
1982
1983	if (parse_sa_config(attrs, tb_sa))
1984	return -EINVAL;
1985
1986	if (!validate_add_txsa(attrs: tb_sa))
1987	return -EINVAL;
1988
1989	rtnl_lock();
1990	dev = get_dev_from_nl(net: genl_info_net(info), attrs);
1991	if (IS_ERR(ptr: dev)) {
1992	rtnl_unlock();
1993	return PTR_ERR(ptr: dev);
1994	}
1995
1996	secy = &macsec_priv(dev)->secy;
1997	tx_sc = &secy->tx_sc;
1998
1999	assoc_num = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_AN]);
2000
2001	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_KEY]) != secy->key_len) {
2002	pr_notice("macsec: nl: add_txsa: bad key length: %d != %d\n",
2003	nla_len(tb_sa[MACSEC_SA_ATTR_KEY]), secy->key_len);
2004	rtnl_unlock();
2005	return -EINVAL;
2006	}
2007
2008	pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
2009	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
2010	pr_notice("macsec: nl: add_txsa: bad pn length: %d != %d\n",
2011	nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
2012	rtnl_unlock();
2013	return -EINVAL;
2014	}
2015
2016	if (secy->xpn) {
2017	if (!tb_sa[MACSEC_SA_ATTR_SSCI] || !tb_sa[MACSEC_SA_ATTR_SALT]) {
2018	rtnl_unlock();
2019	return -EINVAL;
2020	}
2021
2022	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_SALT]) != MACSEC_SALT_LEN) {
2023	pr_notice("macsec: nl: add_txsa: bad salt length: %d != %d\n",
2024	nla_len(tb_sa[MACSEC_SA_ATTR_SALT]),
2025	MACSEC_SALT_LEN);
2026	rtnl_unlock();
2027	return -EINVAL;
2028	}
2029	}
2030
2031	tx_sa = rtnl_dereference(tx_sc->sa[assoc_num]);
2032	if (tx_sa) {
2033	rtnl_unlock();
2034	return -EBUSY;
2035	}
2036
2037	tx_sa = kmalloc(size: sizeof(*tx_sa), GFP_KERNEL);
2038	if (!tx_sa) {
2039	rtnl_unlock();
2040	return -ENOMEM;
2041	}
2042
2043	err = init_tx_sa(tx_sa, sak: nla_data(nla: tb_sa[MACSEC_SA_ATTR_KEY]),
2044	key_len: secy->key_len, icv_len: secy->icv_len);
2045	if (err < 0) {
2046	kfree(objp: tx_sa);
2047	rtnl_unlock();
2048	return err;
2049	}
2050
2051	spin_lock_bh(lock: &tx_sa->lock);
2052	tx_sa->next_pn = nla_get_u64(nla: tb_sa[MACSEC_SA_ATTR_PN]);
2053	spin_unlock_bh(lock: &tx_sa->lock);
2054
2055	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
2056	tx_sa->active = !!nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_ACTIVE]);
2057
2058	was_operational = secy->operational;
2059	if (assoc_num == tx_sc->encoding_sa && tx_sa->active)
2060	secy->operational = true;
2061
2062	if (secy->xpn) {
2063	tx_sa->ssci = nla_get_ssci(nla: tb_sa[MACSEC_SA_ATTR_SSCI]);
2064	nla_memcpy(dest: tx_sa->key.salt.bytes, src: tb_sa[MACSEC_SA_ATTR_SALT],
2065	MACSEC_SALT_LEN);
2066	}
2067
2068	/* If h/w offloading is available, propagate to the device */
2069	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2070	const struct macsec_ops *ops;
2071	struct macsec_context ctx;
2072
2073	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2074	if (!ops) {
2075	err = -EOPNOTSUPP;
2076	goto cleanup;
2077	}
2078
2079	ctx.sa.assoc_num = assoc_num;
2080	ctx.sa.tx_sa = tx_sa;
2081	ctx.secy = secy;
2082	memcpy(ctx.sa.key, nla_data(tb_sa[MACSEC_SA_ATTR_KEY]),
2083	secy->key_len);
2084
2085	err = macsec_offload(func: ops->mdo_add_txsa, ctx: &ctx);
2086	memzero_explicit(s: ctx.sa.key, count: secy->key_len);
2087	if (err)
2088	goto cleanup;
2089	}
2090
2091	nla_memcpy(dest: tx_sa->key.id, src: tb_sa[MACSEC_SA_ATTR_KEYID], MACSEC_KEYID_LEN);
2092	rcu_assign_pointer(tx_sc->sa[assoc_num], tx_sa);
2093
2094	rtnl_unlock();
2095
2096	return 0;
2097
2098	cleanup:
2099	secy->operational = was_operational;
2100	macsec_txsa_put(sa: tx_sa);
2101	rtnl_unlock();
2102	return err;
2103	}
2104
2105	static int macsec_del_rxsa(struct sk_buff *skb, struct genl_info *info)
2106	{
2107	struct nlattr **attrs = info->attrs;
2108	struct net_device *dev;
2109	struct macsec_secy *secy;
2110	struct macsec_rx_sc *rx_sc;
2111	struct macsec_rx_sa *rx_sa;
2112	u8 assoc_num;
2113	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2114	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2115	int ret;
2116
2117	if (!attrs[MACSEC_ATTR_IFINDEX])
2118	return -EINVAL;
2119
2120	if (parse_sa_config(attrs, tb_sa))
2121	return -EINVAL;
2122
2123	if (parse_rxsc_config(attrs, tb_rxsc))
2124	return -EINVAL;
2125
2126	rtnl_lock();
2127	rx_sa = get_rxsa_from_nl(net: genl_info_net(info), attrs, tb_rxsc, tb_sa,
2128	devp: &dev, secyp: &secy, scp: &rx_sc, assoc_num: &assoc_num);
2129	if (IS_ERR(ptr: rx_sa)) {
2130	rtnl_unlock();
2131	return PTR_ERR(ptr: rx_sa);
2132	}
2133
2134	if (rx_sa->active) {
2135	rtnl_unlock();
2136	return -EBUSY;
2137	}
2138
2139	/* If h/w offloading is available, propagate to the device */
2140	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2141	const struct macsec_ops *ops;
2142	struct macsec_context ctx;
2143
2144	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2145	if (!ops) {
2146	ret = -EOPNOTSUPP;
2147	goto cleanup;
2148	}
2149
2150	ctx.sa.assoc_num = assoc_num;
2151	ctx.sa.rx_sa = rx_sa;
2152	ctx.secy = secy;
2153
2154	ret = macsec_offload(func: ops->mdo_del_rxsa, ctx: &ctx);
2155	if (ret)
2156	goto cleanup;
2157	}
2158
2159	RCU_INIT_POINTER(rx_sc->sa[assoc_num], NULL);
2160	clear_rx_sa(rx_sa);
2161
2162	rtnl_unlock();
2163
2164	return 0;
2165
2166	cleanup:
2167	rtnl_unlock();
2168	return ret;
2169	}
2170
2171	static int macsec_del_rxsc(struct sk_buff *skb, struct genl_info *info)
2172	{
2173	struct nlattr **attrs = info->attrs;
2174	struct net_device *dev;
2175	struct macsec_secy *secy;
2176	struct macsec_rx_sc *rx_sc;
2177	sci_t sci;
2178	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2179	int ret;
2180
2181	if (!attrs[MACSEC_ATTR_IFINDEX])
2182	return -EINVAL;
2183
2184	if (parse_rxsc_config(attrs, tb_rxsc))
2185	return -EINVAL;
2186
2187	if (!tb_rxsc[MACSEC_RXSC_ATTR_SCI])
2188	return -EINVAL;
2189
2190	rtnl_lock();
2191	dev = get_dev_from_nl(net: genl_info_net(info), attrs: info->attrs);
2192	if (IS_ERR(ptr: dev)) {
2193	rtnl_unlock();
2194	return PTR_ERR(ptr: dev);
2195	}
2196
2197	secy = &macsec_priv(dev)->secy;
2198	sci = nla_get_sci(nla: tb_rxsc[MACSEC_RXSC_ATTR_SCI]);
2199
2200	rx_sc = del_rx_sc(secy, sci);
2201	if (!rx_sc) {
2202	rtnl_unlock();
2203	return -ENODEV;
2204	}
2205
2206	/* If h/w offloading is available, propagate to the device */
2207	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2208	const struct macsec_ops *ops;
2209	struct macsec_context ctx;
2210
2211	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2212	if (!ops) {
2213	ret = -EOPNOTSUPP;
2214	goto cleanup;
2215	}
2216
2217	ctx.rx_sc = rx_sc;
2218	ctx.secy = secy;
2219	ret = macsec_offload(func: ops->mdo_del_rxsc, ctx: &ctx);
2220	if (ret)
2221	goto cleanup;
2222	}
2223
2224	free_rx_sc(rx_sc);
2225	rtnl_unlock();
2226
2227	return 0;
2228
2229	cleanup:
2230	rtnl_unlock();
2231	return ret;
2232	}
2233
2234	static int macsec_del_txsa(struct sk_buff *skb, struct genl_info *info)
2235	{
2236	struct nlattr **attrs = info->attrs;
2237	struct net_device *dev;
2238	struct macsec_secy *secy;
2239	struct macsec_tx_sc *tx_sc;
2240	struct macsec_tx_sa *tx_sa;
2241	u8 assoc_num;
2242	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2243	int ret;
2244
2245	if (!attrs[MACSEC_ATTR_IFINDEX])
2246	return -EINVAL;
2247
2248	if (parse_sa_config(attrs, tb_sa))
2249	return -EINVAL;
2250
2251	rtnl_lock();
2252	tx_sa = get_txsa_from_nl(net: genl_info_net(info), attrs, tb_sa,
2253	devp: &dev, secyp: &secy, scp: &tx_sc, assoc_num: &assoc_num);
2254	if (IS_ERR(ptr: tx_sa)) {
2255	rtnl_unlock();
2256	return PTR_ERR(ptr: tx_sa);
2257	}
2258
2259	if (tx_sa->active) {
2260	rtnl_unlock();
2261	return -EBUSY;
2262	}
2263
2264	/* If h/w offloading is available, propagate to the device */
2265	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2266	const struct macsec_ops *ops;
2267	struct macsec_context ctx;
2268
2269	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2270	if (!ops) {
2271	ret = -EOPNOTSUPP;
2272	goto cleanup;
2273	}
2274
2275	ctx.sa.assoc_num = assoc_num;
2276	ctx.sa.tx_sa = tx_sa;
2277	ctx.secy = secy;
2278
2279	ret = macsec_offload(func: ops->mdo_del_txsa, ctx: &ctx);
2280	if (ret)
2281	goto cleanup;
2282	}
2283
2284	RCU_INIT_POINTER(tx_sc->sa[assoc_num], NULL);
2285	clear_tx_sa(tx_sa);
2286
2287	rtnl_unlock();
2288
2289	return 0;
2290
2291	cleanup:
2292	rtnl_unlock();
2293	return ret;
2294	}
2295
2296	static bool validate_upd_sa(struct nlattr **attrs)
2297	{
2298	if (!attrs[MACSEC_SA_ATTR_AN] ||
2299	attrs[MACSEC_SA_ATTR_KEY] ||
2300	attrs[MACSEC_SA_ATTR_KEYID] ||
2301	attrs[MACSEC_SA_ATTR_SSCI] ||
2302	attrs[MACSEC_SA_ATTR_SALT])
2303	return false;
2304
2305	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_AN]) >= MACSEC_NUM_AN)
2306	return false;
2307
2308	if (attrs[MACSEC_SA_ATTR_PN] && nla_get_u64(nla: attrs[MACSEC_SA_ATTR_PN]) == 0)
2309	return false;
2310
2311	if (attrs[MACSEC_SA_ATTR_ACTIVE]) {
2312	if (nla_get_u8(nla: attrs[MACSEC_SA_ATTR_ACTIVE]) > 1)
2313	return false;
2314	}
2315
2316	return true;
2317	}
2318
2319	static int macsec_upd_txsa(struct sk_buff *skb, struct genl_info *info)
2320	{
2321	struct nlattr **attrs = info->attrs;
2322	struct net_device *dev;
2323	struct macsec_secy *secy;
2324	struct macsec_tx_sc *tx_sc;
2325	struct macsec_tx_sa *tx_sa;
2326	u8 assoc_num;
2327	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2328	bool was_operational, was_active;
2329	pn_t prev_pn;
2330	int ret = 0;
2331
2332	prev_pn.full64 = 0;
2333
2334	if (!attrs[MACSEC_ATTR_IFINDEX])
2335	return -EINVAL;
2336
2337	if (parse_sa_config(attrs, tb_sa))
2338	return -EINVAL;
2339
2340	if (!validate_upd_sa(attrs: tb_sa))
2341	return -EINVAL;
2342
2343	rtnl_lock();
2344	tx_sa = get_txsa_from_nl(net: genl_info_net(info), attrs, tb_sa,
2345	devp: &dev, secyp: &secy, scp: &tx_sc, assoc_num: &assoc_num);
2346	if (IS_ERR(ptr: tx_sa)) {
2347	rtnl_unlock();
2348	return PTR_ERR(ptr: tx_sa);
2349	}
2350
2351	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2352	int pn_len;
2353
2354	pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
2355	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
2356	pr_notice("macsec: nl: upd_txsa: bad pn length: %d != %d\n",
2357	nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
2358	rtnl_unlock();
2359	return -EINVAL;
2360	}
2361
2362	spin_lock_bh(lock: &tx_sa->lock);
2363	prev_pn = tx_sa->next_pn_halves;
2364	tx_sa->next_pn = nla_get_u64(nla: tb_sa[MACSEC_SA_ATTR_PN]);
2365	spin_unlock_bh(lock: &tx_sa->lock);
2366	}
2367
2368	was_active = tx_sa->active;
2369	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
2370	tx_sa->active = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_ACTIVE]);
2371
2372	was_operational = secy->operational;
2373	if (assoc_num == tx_sc->encoding_sa)
2374	secy->operational = tx_sa->active;
2375
2376	/* If h/w offloading is available, propagate to the device */
2377	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2378	const struct macsec_ops *ops;
2379	struct macsec_context ctx;
2380
2381	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2382	if (!ops) {
2383	ret = -EOPNOTSUPP;
2384	goto cleanup;
2385	}
2386
2387	ctx.sa.assoc_num = assoc_num;
2388	ctx.sa.tx_sa = tx_sa;
2389	ctx.sa.update_pn = !!prev_pn.full64;
2390	ctx.secy = secy;
2391
2392	ret = macsec_offload(func: ops->mdo_upd_txsa, ctx: &ctx);
2393	if (ret)
2394	goto cleanup;
2395	}
2396
2397	rtnl_unlock();
2398
2399	return 0;
2400
2401	cleanup:
2402	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2403	spin_lock_bh(lock: &tx_sa->lock);
2404	tx_sa->next_pn_halves = prev_pn;
2405	spin_unlock_bh(lock: &tx_sa->lock);
2406	}
2407	tx_sa->active = was_active;
2408	secy->operational = was_operational;
2409	rtnl_unlock();
2410	return ret;
2411	}
2412
2413	static int macsec_upd_rxsa(struct sk_buff *skb, struct genl_info *info)
2414	{
2415	struct nlattr **attrs = info->attrs;
2416	struct net_device *dev;
2417	struct macsec_secy *secy;
2418	struct macsec_rx_sc *rx_sc;
2419	struct macsec_rx_sa *rx_sa;
2420	u8 assoc_num;
2421	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2422	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2423	bool was_active;
2424	pn_t prev_pn;
2425	int ret = 0;
2426
2427	prev_pn.full64 = 0;
2428
2429	if (!attrs[MACSEC_ATTR_IFINDEX])
2430	return -EINVAL;
2431
2432	if (parse_rxsc_config(attrs, tb_rxsc))
2433	return -EINVAL;
2434
2435	if (parse_sa_config(attrs, tb_sa))
2436	return -EINVAL;
2437
2438	if (!validate_upd_sa(attrs: tb_sa))
2439	return -EINVAL;
2440
2441	rtnl_lock();
2442	rx_sa = get_rxsa_from_nl(net: genl_info_net(info), attrs, tb_rxsc, tb_sa,
2443	devp: &dev, secyp: &secy, scp: &rx_sc, assoc_num: &assoc_num);
2444	if (IS_ERR(ptr: rx_sa)) {
2445	rtnl_unlock();
2446	return PTR_ERR(ptr: rx_sa);
2447	}
2448
2449	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2450	int pn_len;
2451
2452	pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
2453	if (nla_len(nla: tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
2454	pr_notice("macsec: nl: upd_rxsa: bad pn length: %d != %d\n",
2455	nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
2456	rtnl_unlock();
2457	return -EINVAL;
2458	}
2459
2460	spin_lock_bh(lock: &rx_sa->lock);
2461	prev_pn = rx_sa->next_pn_halves;
2462	rx_sa->next_pn = nla_get_u64(nla: tb_sa[MACSEC_SA_ATTR_PN]);
2463	spin_unlock_bh(lock: &rx_sa->lock);
2464	}
2465
2466	was_active = rx_sa->active;
2467	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
2468	rx_sa->active = nla_get_u8(nla: tb_sa[MACSEC_SA_ATTR_ACTIVE]);
2469
2470	/* If h/w offloading is available, propagate to the device */
2471	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2472	const struct macsec_ops *ops;
2473	struct macsec_context ctx;
2474
2475	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2476	if (!ops) {
2477	ret = -EOPNOTSUPP;
2478	goto cleanup;
2479	}
2480
2481	ctx.sa.assoc_num = assoc_num;
2482	ctx.sa.rx_sa = rx_sa;
2483	ctx.sa.update_pn = !!prev_pn.full64;
2484	ctx.secy = secy;
2485
2486	ret = macsec_offload(func: ops->mdo_upd_rxsa, ctx: &ctx);
2487	if (ret)
2488	goto cleanup;
2489	}
2490
2491	rtnl_unlock();
2492	return 0;
2493
2494	cleanup:
2495	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2496	spin_lock_bh(lock: &rx_sa->lock);
2497	rx_sa->next_pn_halves = prev_pn;
2498	spin_unlock_bh(lock: &rx_sa->lock);
2499	}
2500	rx_sa->active = was_active;
2501	rtnl_unlock();
2502	return ret;
2503	}
2504
2505	static int macsec_upd_rxsc(struct sk_buff *skb, struct genl_info *info)
2506	{
2507	struct nlattr **attrs = info->attrs;
2508	struct net_device *dev;
2509	struct macsec_secy *secy;
2510	struct macsec_rx_sc *rx_sc;
2511	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2512	unsigned int prev_n_rx_sc;
2513	bool was_active;
2514	int ret;
2515
2516	if (!attrs[MACSEC_ATTR_IFINDEX])
2517	return -EINVAL;
2518
2519	if (parse_rxsc_config(attrs, tb_rxsc))
2520	return -EINVAL;
2521
2522	if (!validate_add_rxsc(attrs: tb_rxsc))
2523	return -EINVAL;
2524
2525	rtnl_lock();
2526	rx_sc = get_rxsc_from_nl(net: genl_info_net(info), attrs, tb_rxsc, devp: &dev, secyp: &secy);
2527	if (IS_ERR(ptr: rx_sc)) {
2528	rtnl_unlock();
2529	return PTR_ERR(ptr: rx_sc);
2530	}
2531
2532	was_active = rx_sc->active;
2533	prev_n_rx_sc = secy->n_rx_sc;
2534	if (tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]) {
2535	bool new = !!nla_get_u8(nla: tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]);
2536
2537	if (rx_sc->active != new)
2538	secy->n_rx_sc += new ? 1 : -1;
2539
2540	rx_sc->active = new;
2541	}
2542
2543	/* If h/w offloading is available, propagate to the device */
2544	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
2545	const struct macsec_ops *ops;
2546	struct macsec_context ctx;
2547
2548	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
2549	if (!ops) {
2550	ret = -EOPNOTSUPP;
2551	goto cleanup;
2552	}
2553
2554	ctx.rx_sc = rx_sc;
2555	ctx.secy = secy;
2556
2557	ret = macsec_offload(func: ops->mdo_upd_rxsc, ctx: &ctx);
2558	if (ret)
2559	goto cleanup;
2560	}
2561
2562	rtnl_unlock();
2563
2564	return 0;
2565
2566	cleanup:
2567	secy->n_rx_sc = prev_n_rx_sc;
2568	rx_sc->active = was_active;
2569	rtnl_unlock();
2570	return ret;
2571	}
2572
2573	static bool macsec_is_configured(struct macsec_dev *macsec)
2574	{
2575	struct macsec_secy *secy = &macsec->secy;
2576	struct macsec_tx_sc *tx_sc = &secy->tx_sc;
2577	int i;
2578
2579	if (secy->rx_sc)
2580	return true;
2581
2582	for (i = 0; i < MACSEC_NUM_AN; i++)
2583	if (tx_sc->sa[i])
2584	return true;
2585
2586	return false;
2587	}
2588
2589	static bool macsec_needs_tx_tag(struct macsec_dev *macsec,
2590	const struct macsec_ops *ops)
2591	{
2592	return macsec->offload == MACSEC_OFFLOAD_PHY &&
2593	ops->mdo_insert_tx_tag;
2594	}
2595
2596	static void macsec_set_head_tail_room(struct net_device *dev)
2597	{
2598	struct macsec_dev *macsec = macsec_priv(dev);
2599	struct net_device *real_dev = macsec->real_dev;
2600	int needed_headroom, needed_tailroom;
2601	const struct macsec_ops *ops;
2602
2603	ops = macsec_get_ops(macsec, NULL);
2604	if (ops) {
2605	needed_headroom = ops->needed_headroom;
2606	needed_tailroom = ops->needed_tailroom;
2607	} else {
2608	needed_headroom = MACSEC_NEEDED_HEADROOM;
2609	needed_tailroom = MACSEC_NEEDED_TAILROOM;
2610	}
2611
2612	dev->needed_headroom = real_dev->needed_headroom + needed_headroom;
2613	dev->needed_tailroom = real_dev->needed_tailroom + needed_tailroom;
2614	}
2615
2616	static int macsec_update_offload(struct net_device *dev, enum macsec_offload offload)
2617	{
2618	enum macsec_offload prev_offload;
2619	const struct macsec_ops *ops;
2620	struct macsec_context ctx;
2621	struct macsec_dev *macsec;
2622	int ret = 0;
2623
2624	macsec = macsec_priv(dev);
2625
2626	/* Check if the offloading mode is supported by the underlying layers */
2627	if (offload != MACSEC_OFFLOAD_OFF &&
2628	!macsec_check_offload(offload, macsec))
2629	return -EOPNOTSUPP;
2630
2631	/* Check if the net device is busy. */
2632	if (netif_running(dev))
2633	return -EBUSY;
2634
2635	/* Check if the device already has rules configured: we do not support
2636	* rules migration.
2637	*/
2638	if (macsec_is_configured(macsec))
2639	return -EBUSY;
2640
2641	prev_offload = macsec->offload;
2642
2643	ops = __macsec_get_ops(offload: offload == MACSEC_OFFLOAD_OFF ? prev_offload : offload,
2644	macsec, ctx: &ctx);
2645	if (!ops)
2646	return -EOPNOTSUPP;
2647
2648	macsec->offload = offload;
2649
2650	ctx.secy = &macsec->secy;
2651	ret = offload == MACSEC_OFFLOAD_OFF ? macsec_offload(func: ops->mdo_del_secy, ctx: &ctx)
2652	: macsec_offload(func: ops->mdo_add_secy, ctx: &ctx);
2653	if (ret) {
2654	macsec->offload = prev_offload;
2655	return ret;
2656	}
2657
2658	macsec_set_head_tail_room(dev);
2659	macsec->insert_tx_tag = macsec_needs_tx_tag(macsec, ops);
2660
2661	return ret;
2662	}
2663
2664	static int macsec_upd_offload(struct sk_buff *skb, struct genl_info *info)
2665	{
2666	struct nlattr *tb_offload[MACSEC_OFFLOAD_ATTR_MAX + 1];
2667	struct nlattr **attrs = info->attrs;
2668	enum macsec_offload offload;
2669	struct macsec_dev *macsec;
2670	struct net_device *dev;
2671	int ret = 0;
2672
2673	if (!attrs[MACSEC_ATTR_IFINDEX])
2674	return -EINVAL;
2675
2676	if (!attrs[MACSEC_ATTR_OFFLOAD])
2677	return -EINVAL;
2678
2679	if (nla_parse_nested_deprecated(tb: tb_offload, maxtype: MACSEC_OFFLOAD_ATTR_MAX,
2680	nla: attrs[MACSEC_ATTR_OFFLOAD],
2681	policy: macsec_genl_offload_policy, NULL))
2682	return -EINVAL;
2683
2684	rtnl_lock();
2685
2686	dev = get_dev_from_nl(net: genl_info_net(info), attrs);
2687	if (IS_ERR(ptr: dev)) {
2688	ret = PTR_ERR(ptr: dev);
2689	goto out;
2690	}
2691	macsec = macsec_priv(dev);
2692
2693	if (!tb_offload[MACSEC_OFFLOAD_ATTR_TYPE]) {
2694	ret = -EINVAL;
2695	goto out;
2696	}
2697
2698	offload = nla_get_u8(nla: tb_offload[MACSEC_OFFLOAD_ATTR_TYPE]);
2699
2700	if (macsec->offload != offload)
2701	ret = macsec_update_offload(dev, offload);
2702	out:
2703	rtnl_unlock();
2704	return ret;
2705	}
2706
2707	static void get_tx_sa_stats(struct net_device *dev, int an,
2708	struct macsec_tx_sa *tx_sa,
2709	struct macsec_tx_sa_stats *sum)
2710	{
2711	struct macsec_dev *macsec = macsec_priv(dev);
2712	int cpu;
2713
2714	/* If h/w offloading is available, propagate to the device */
2715	if (macsec_is_offloaded(macsec)) {
2716	const struct macsec_ops *ops;
2717	struct macsec_context ctx;
2718
2719	ops = macsec_get_ops(macsec, ctx: &ctx);
2720	if (ops) {
2721	ctx.sa.assoc_num = an;
2722	ctx.sa.tx_sa = tx_sa;
2723	ctx.stats.tx_sa_stats = sum;
2724	ctx.secy = &macsec_priv(dev)->secy;
2725	macsec_offload(func: ops->mdo_get_tx_sa_stats, ctx: &ctx);
2726	}
2727	return;
2728	}
2729
2730	for_each_possible_cpu(cpu) {
2731	const struct macsec_tx_sa_stats *stats =
2732	per_cpu_ptr(tx_sa->stats, cpu);
2733
2734	sum->OutPktsProtected += stats->OutPktsProtected;
2735	sum->OutPktsEncrypted += stats->OutPktsEncrypted;
2736	}
2737	}
2738
2739	static int copy_tx_sa_stats(struct sk_buff *skb, struct macsec_tx_sa_stats *sum)
2740	{
2741	if (nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_OUT_PKTS_PROTECTED,
2742	value: sum->OutPktsProtected) ||
2743	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_OUT_PKTS_ENCRYPTED,
2744	value: sum->OutPktsEncrypted))
2745	return -EMSGSIZE;
2746
2747	return 0;
2748	}
2749
2750	static void get_rx_sa_stats(struct net_device *dev,
2751	struct macsec_rx_sc *rx_sc, int an,
2752	struct macsec_rx_sa *rx_sa,
2753	struct macsec_rx_sa_stats *sum)
2754	{
2755	struct macsec_dev *macsec = macsec_priv(dev);
2756	int cpu;
2757
2758	/* If h/w offloading is available, propagate to the device */
2759	if (macsec_is_offloaded(macsec)) {
2760	const struct macsec_ops *ops;
2761	struct macsec_context ctx;
2762
2763	ops = macsec_get_ops(macsec, ctx: &ctx);
2764	if (ops) {
2765	ctx.sa.assoc_num = an;
2766	ctx.sa.rx_sa = rx_sa;
2767	ctx.stats.rx_sa_stats = sum;
2768	ctx.secy = &macsec_priv(dev)->secy;
2769	ctx.rx_sc = rx_sc;
2770	macsec_offload(func: ops->mdo_get_rx_sa_stats, ctx: &ctx);
2771	}
2772	return;
2773	}
2774
2775	for_each_possible_cpu(cpu) {
2776	const struct macsec_rx_sa_stats *stats =
2777	per_cpu_ptr(rx_sa->stats, cpu);
2778
2779	sum->InPktsOK += stats->InPktsOK;
2780	sum->InPktsInvalid += stats->InPktsInvalid;
2781	sum->InPktsNotValid += stats->InPktsNotValid;
2782	sum->InPktsNotUsingSA += stats->InPktsNotUsingSA;
2783	sum->InPktsUnusedSA += stats->InPktsUnusedSA;
2784	}
2785	}
2786
2787	static int copy_rx_sa_stats(struct sk_buff *skb,
2788	struct macsec_rx_sa_stats *sum)
2789	{
2790	if (nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_OK, value: sum->InPktsOK) ||
2791	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_INVALID,
2792	value: sum->InPktsInvalid) ||
2793	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_NOT_VALID,
2794	value: sum->InPktsNotValid) ||
2795	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_NOT_USING_SA,
2796	value: sum->InPktsNotUsingSA) ||
2797	nla_put_u32(skb, attrtype: MACSEC_SA_STATS_ATTR_IN_PKTS_UNUSED_SA,
2798	value: sum->InPktsUnusedSA))
2799	return -EMSGSIZE;
2800
2801	return 0;
2802	}
2803
2804	static void get_rx_sc_stats(struct net_device *dev,
2805	struct macsec_rx_sc *rx_sc,
2806	struct macsec_rx_sc_stats *sum)
2807	{
2808	struct macsec_dev *macsec = macsec_priv(dev);
2809	int cpu;
2810
2811	/* If h/w offloading is available, propagate to the device */
2812	if (macsec_is_offloaded(macsec)) {
2813	const struct macsec_ops *ops;
2814	struct macsec_context ctx;
2815
2816	ops = macsec_get_ops(macsec, ctx: &ctx);
2817	if (ops) {
2818	ctx.stats.rx_sc_stats = sum;
2819	ctx.secy = &macsec_priv(dev)->secy;
2820	ctx.rx_sc = rx_sc;
2821	macsec_offload(func: ops->mdo_get_rx_sc_stats, ctx: &ctx);
2822	}
2823	return;
2824	}
2825
2826	for_each_possible_cpu(cpu) {
2827	const struct pcpu_rx_sc_stats *stats;
2828	struct macsec_rx_sc_stats tmp;
2829	unsigned int start;
2830
2831	stats = per_cpu_ptr(rx_sc->stats, cpu);
2832	do {
2833	start = u64_stats_fetch_begin(syncp: &stats->syncp);
2834	memcpy(&tmp, &stats->stats, sizeof(tmp));
2835	} while (u64_stats_fetch_retry(syncp: &stats->syncp, start));
2836
2837	sum->InOctetsValidated += tmp.InOctetsValidated;
2838	sum->InOctetsDecrypted += tmp.InOctetsDecrypted;
2839	sum->InPktsUnchecked += tmp.InPktsUnchecked;
2840	sum->InPktsDelayed += tmp.InPktsDelayed;
2841	sum->InPktsOK += tmp.InPktsOK;
2842	sum->InPktsInvalid += tmp.InPktsInvalid;
2843	sum->InPktsLate += tmp.InPktsLate;
2844	sum->InPktsNotValid += tmp.InPktsNotValid;
2845	sum->InPktsNotUsingSA += tmp.InPktsNotUsingSA;
2846	sum->InPktsUnusedSA += tmp.InPktsUnusedSA;
2847	}
2848	}
2849
2850	static int copy_rx_sc_stats(struct sk_buff *skb, struct macsec_rx_sc_stats *sum)
2851	{
2852	if (nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_OCTETS_VALIDATED,
2853	value: sum->InOctetsValidated,
2854	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2855	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_OCTETS_DECRYPTED,
2856	value: sum->InOctetsDecrypted,
2857	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2858	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_UNCHECKED,
2859	value: sum->InPktsUnchecked,
2860	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2861	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_DELAYED,
2862	value: sum->InPktsDelayed,
2863	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2864	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_OK,
2865	value: sum->InPktsOK,
2866	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2867	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_INVALID,
2868	value: sum->InPktsInvalid,
2869	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2870	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_LATE,
2871	value: sum->InPktsLate,
2872	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2873	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_NOT_VALID,
2874	value: sum->InPktsNotValid,
2875	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2876	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_NOT_USING_SA,
2877	value: sum->InPktsNotUsingSA,
2878	padattr: MACSEC_RXSC_STATS_ATTR_PAD) ||
2879	nla_put_u64_64bit(skb, attrtype: MACSEC_RXSC_STATS_ATTR_IN_PKTS_UNUSED_SA,
2880	value: sum->InPktsUnusedSA,
2881	padattr: MACSEC_RXSC_STATS_ATTR_PAD))
2882	return -EMSGSIZE;
2883
2884	return 0;
2885	}
2886
2887	static void get_tx_sc_stats(struct net_device *dev,
2888	struct macsec_tx_sc_stats *sum)
2889	{
2890	struct macsec_dev *macsec = macsec_priv(dev);
2891	int cpu;
2892
2893	/* If h/w offloading is available, propagate to the device */
2894	if (macsec_is_offloaded(macsec)) {
2895	const struct macsec_ops *ops;
2896	struct macsec_context ctx;
2897
2898	ops = macsec_get_ops(macsec, ctx: &ctx);
2899	if (ops) {
2900	ctx.stats.tx_sc_stats = sum;
2901	ctx.secy = &macsec_priv(dev)->secy;
2902	macsec_offload(func: ops->mdo_get_tx_sc_stats, ctx: &ctx);
2903	}
2904	return;
2905	}
2906
2907	for_each_possible_cpu(cpu) {
2908	const struct pcpu_tx_sc_stats *stats;
2909	struct macsec_tx_sc_stats tmp;
2910	unsigned int start;
2911
2912	stats = per_cpu_ptr(macsec_priv(dev)->secy.tx_sc.stats, cpu);
2913	do {
2914	start = u64_stats_fetch_begin(syncp: &stats->syncp);
2915	memcpy(&tmp, &stats->stats, sizeof(tmp));
2916	} while (u64_stats_fetch_retry(syncp: &stats->syncp, start));
2917
2918	sum->OutPktsProtected += tmp.OutPktsProtected;
2919	sum->OutPktsEncrypted += tmp.OutPktsEncrypted;
2920	sum->OutOctetsProtected += tmp.OutOctetsProtected;
2921	sum->OutOctetsEncrypted += tmp.OutOctetsEncrypted;
2922	}
2923	}
2924
2925	static int copy_tx_sc_stats(struct sk_buff *skb, struct macsec_tx_sc_stats *sum)
2926	{
2927	if (nla_put_u64_64bit(skb, attrtype: MACSEC_TXSC_STATS_ATTR_OUT_PKTS_PROTECTED,
2928	value: sum->OutPktsProtected,
2929	padattr: MACSEC_TXSC_STATS_ATTR_PAD) ||
2930	nla_put_u64_64bit(skb, attrtype: MACSEC_TXSC_STATS_ATTR_OUT_PKTS_ENCRYPTED,
2931	value: sum->OutPktsEncrypted,
2932	padattr: MACSEC_TXSC_STATS_ATTR_PAD) ||
2933	nla_put_u64_64bit(skb, attrtype: MACSEC_TXSC_STATS_ATTR_OUT_OCTETS_PROTECTED,
2934	value: sum->OutOctetsProtected,
2935	padattr: MACSEC_TXSC_STATS_ATTR_PAD) ||
2936	nla_put_u64_64bit(skb, attrtype: MACSEC_TXSC_STATS_ATTR_OUT_OCTETS_ENCRYPTED,
2937	value: sum->OutOctetsEncrypted,
2938	padattr: MACSEC_TXSC_STATS_ATTR_PAD))
2939	return -EMSGSIZE;
2940
2941	return 0;
2942	}
2943
2944	static void get_secy_stats(struct net_device *dev, struct macsec_dev_stats *sum)
2945	{
2946	struct macsec_dev *macsec = macsec_priv(dev);
2947	int cpu;
2948
2949	/* If h/w offloading is available, propagate to the device */
2950	if (macsec_is_offloaded(macsec)) {
2951	const struct macsec_ops *ops;
2952	struct macsec_context ctx;
2953
2954	ops = macsec_get_ops(macsec, ctx: &ctx);
2955	if (ops) {
2956	ctx.stats.dev_stats = sum;
2957	ctx.secy = &macsec_priv(dev)->secy;
2958	macsec_offload(func: ops->mdo_get_dev_stats, ctx: &ctx);
2959	}
2960	return;
2961	}
2962
2963	for_each_possible_cpu(cpu) {
2964	const struct pcpu_secy_stats *stats;
2965	struct macsec_dev_stats tmp;
2966	unsigned int start;
2967
2968	stats = per_cpu_ptr(macsec_priv(dev)->stats, cpu);
2969	do {
2970	start = u64_stats_fetch_begin(syncp: &stats->syncp);
2971	memcpy(&tmp, &stats->stats, sizeof(tmp));
2972	} while (u64_stats_fetch_retry(syncp: &stats->syncp, start));
2973
2974	sum->OutPktsUntagged += tmp.OutPktsUntagged;
2975	sum->InPktsUntagged += tmp.InPktsUntagged;
2976	sum->OutPktsTooLong += tmp.OutPktsTooLong;
2977	sum->InPktsNoTag += tmp.InPktsNoTag;
2978	sum->InPktsBadTag += tmp.InPktsBadTag;
2979	sum->InPktsUnknownSCI += tmp.InPktsUnknownSCI;
2980	sum->InPktsNoSCI += tmp.InPktsNoSCI;
2981	sum->InPktsOverrun += tmp.InPktsOverrun;
2982	}
2983	}
2984
2985	static int copy_secy_stats(struct sk_buff *skb, struct macsec_dev_stats *sum)
2986	{
2987	if (nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_OUT_PKTS_UNTAGGED,
2988	value: sum->OutPktsUntagged,
2989	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
2990	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_UNTAGGED,
2991	value: sum->InPktsUntagged,
2992	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
2993	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_OUT_PKTS_TOO_LONG,
2994	value: sum->OutPktsTooLong,
2995	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
2996	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_NO_TAG,
2997	value: sum->InPktsNoTag,
2998	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
2999	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_BAD_TAG,
3000	value: sum->InPktsBadTag,
3001	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
3002	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_UNKNOWN_SCI,
3003	value: sum->InPktsUnknownSCI,
3004	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
3005	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_NO_SCI,
3006	value: sum->InPktsNoSCI,
3007	padattr: MACSEC_SECY_STATS_ATTR_PAD) ||
3008	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_STATS_ATTR_IN_PKTS_OVERRUN,
3009	value: sum->InPktsOverrun,
3010	padattr: MACSEC_SECY_STATS_ATTR_PAD))
3011	return -EMSGSIZE;
3012
3013	return 0;
3014	}
3015
3016	static int nla_put_secy(struct macsec_secy *secy, struct sk_buff *skb)
3017	{
3018	struct macsec_tx_sc *tx_sc = &secy->tx_sc;
3019	struct nlattr *secy_nest = nla_nest_start_noflag(skb,
3020	attrtype: MACSEC_ATTR_SECY);
3021	u64 csid;
3022
3023	if (!secy_nest)
3024	return 1;
3025
3026	switch (secy->key_len) {
3027	case MACSEC_GCM_AES_128_SAK_LEN:
3028	csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_128 : MACSEC_DEFAULT_CIPHER_ID;
3029	break;
3030	case MACSEC_GCM_AES_256_SAK_LEN:
3031	csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_256 : MACSEC_CIPHER_ID_GCM_AES_256;
3032	break;
3033	default:
3034	goto cancel;
3035	}
3036
3037	if (nla_put_sci(skb, attrtype: MACSEC_SECY_ATTR_SCI, value: secy->sci,
3038	padattr: MACSEC_SECY_ATTR_PAD) ||
3039	nla_put_u64_64bit(skb, attrtype: MACSEC_SECY_ATTR_CIPHER_SUITE,
3040	value: csid, padattr: MACSEC_SECY_ATTR_PAD) ||
3041	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_ICV_LEN, value: secy->icv_len) ||
3042	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_OPER, value: secy->operational) ||
3043	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_PROTECT, value: secy->protect_frames) ||
3044	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_REPLAY, value: secy->replay_protect) ||
3045	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_VALIDATE, value: secy->validate_frames) ||
3046	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_ENCRYPT, value: tx_sc->encrypt) ||
3047	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_INC_SCI, value: tx_sc->send_sci) ||
3048	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_ES, value: tx_sc->end_station) ||
3049	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_SCB, value: tx_sc->scb) ||
3050	nla_put_u8(skb, attrtype: MACSEC_SECY_ATTR_ENCODING_SA, value: tx_sc->encoding_sa))
3051	goto cancel;
3052
3053	if (secy->replay_protect) {
3054	if (nla_put_u32(skb, attrtype: MACSEC_SECY_ATTR_WINDOW, value: secy->replay_window))
3055	goto cancel;
3056	}
3057
3058	nla_nest_end(skb, start: secy_nest);
3059	return 0;
3060
3061	cancel:
3062	nla_nest_cancel(skb, start: secy_nest);
3063	return 1;
3064	}
3065
3066	static noinline_for_stack int
3067	dump_secy(struct macsec_secy *secy, struct net_device *dev,
3068	struct sk_buff *skb, struct netlink_callback *cb)
3069	{
3070	struct macsec_tx_sc_stats tx_sc_stats = {0, };
3071	struct macsec_tx_sa_stats tx_sa_stats = {0, };
3072	struct macsec_rx_sc_stats rx_sc_stats = {0, };
3073	struct macsec_rx_sa_stats rx_sa_stats = {0, };
3074	struct macsec_dev *macsec = netdev_priv(dev);
3075	struct macsec_dev_stats dev_stats = {0, };
3076	struct macsec_tx_sc *tx_sc = &secy->tx_sc;
3077	struct nlattr *txsa_list, *rxsc_list;
3078	struct macsec_rx_sc *rx_sc;
3079	struct nlattr *attr;
3080	void *hdr;
3081	int i, j;
3082
3083	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, seq: cb->nlh->nlmsg_seq,
3084	family: &macsec_fam, NLM_F_MULTI, cmd: MACSEC_CMD_GET_TXSC);
3085	if (!hdr)
3086	return -EMSGSIZE;
3087
3088	genl_dump_check_consistent(cb, user_hdr: hdr);
3089
3090	if (nla_put_u32(skb, attrtype: MACSEC_ATTR_IFINDEX, value: dev->ifindex))
3091	goto nla_put_failure;
3092
3093	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_OFFLOAD);
3094	if (!attr)
3095	goto nla_put_failure;
3096	if (nla_put_u8(skb, attrtype: MACSEC_OFFLOAD_ATTR_TYPE, value: macsec->offload))
3097	goto nla_put_failure;
3098	nla_nest_end(skb, start: attr);
3099
3100	if (nla_put_secy(secy, skb))
3101	goto nla_put_failure;
3102
3103	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_TXSC_STATS);
3104	if (!attr)
3105	goto nla_put_failure;
3106
3107	get_tx_sc_stats(dev, sum: &tx_sc_stats);
3108	if (copy_tx_sc_stats(skb, sum: &tx_sc_stats)) {
3109	nla_nest_cancel(skb, start: attr);
3110	goto nla_put_failure;
3111	}
3112	nla_nest_end(skb, start: attr);
3113
3114	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_SECY_STATS);
3115	if (!attr)
3116	goto nla_put_failure;
3117	get_secy_stats(dev, sum: &dev_stats);
3118	if (copy_secy_stats(skb, sum: &dev_stats)) {
3119	nla_nest_cancel(skb, start: attr);
3120	goto nla_put_failure;
3121	}
3122	nla_nest_end(skb, start: attr);
3123
3124	txsa_list = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_TXSA_LIST);
3125	if (!txsa_list)
3126	goto nla_put_failure;
3127	for (i = 0, j = 1; i < MACSEC_NUM_AN; i++) {
3128	struct macsec_tx_sa *tx_sa = rtnl_dereference(tx_sc->sa[i]);
3129	struct nlattr *txsa_nest;
3130	u64 pn;
3131	int pn_len;
3132
3133	if (!tx_sa)
3134	continue;
3135
3136	txsa_nest = nla_nest_start_noflag(skb, attrtype: j++);
3137	if (!txsa_nest) {
3138	nla_nest_cancel(skb, start: txsa_list);
3139	goto nla_put_failure;
3140	}
3141
3142	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_SA_ATTR_STATS);
3143	if (!attr) {
3144	nla_nest_cancel(skb, start: txsa_nest);
3145	nla_nest_cancel(skb, start: txsa_list);
3146	goto nla_put_failure;
3147	}
3148	memset(&tx_sa_stats, 0, sizeof(tx_sa_stats));
3149	get_tx_sa_stats(dev, an: i, tx_sa, sum: &tx_sa_stats);
3150	if (copy_tx_sa_stats(skb, sum: &tx_sa_stats)) {
3151	nla_nest_cancel(skb, start: attr);
3152	nla_nest_cancel(skb, start: txsa_nest);
3153	nla_nest_cancel(skb, start: txsa_list);
3154	goto nla_put_failure;
3155	}
3156	nla_nest_end(skb, start: attr);
3157
3158	if (secy->xpn) {
3159	pn = tx_sa->next_pn;
3160	pn_len = MACSEC_XPN_PN_LEN;
3161	} else {
3162	pn = tx_sa->next_pn_halves.lower;
3163	pn_len = MACSEC_DEFAULT_PN_LEN;
3164	}
3165
3166	if (nla_put_u8(skb, attrtype: MACSEC_SA_ATTR_AN, value: i) ||
3167	nla_put(skb, attrtype: MACSEC_SA_ATTR_PN, attrlen: pn_len, data: &pn) ||
3168	nla_put(skb, attrtype: MACSEC_SA_ATTR_KEYID, MACSEC_KEYID_LEN, data: tx_sa->key.id) ||
3169	(secy->xpn && nla_put_ssci(skb, attrtype: MACSEC_SA_ATTR_SSCI, value: tx_sa->ssci)) ||
3170	nla_put_u8(skb, attrtype: MACSEC_SA_ATTR_ACTIVE, value: tx_sa->active)) {
3171	nla_nest_cancel(skb, start: txsa_nest);
3172	nla_nest_cancel(skb, start: txsa_list);
3173	goto nla_put_failure;
3174	}
3175
3176	nla_nest_end(skb, start: txsa_nest);
3177	}
3178	nla_nest_end(skb, start: txsa_list);
3179
3180	rxsc_list = nla_nest_start_noflag(skb, attrtype: MACSEC_ATTR_RXSC_LIST);
3181	if (!rxsc_list)
3182	goto nla_put_failure;
3183
3184	j = 1;
3185	for_each_rxsc_rtnl(secy, rx_sc) {
3186	int k;
3187	struct nlattr *rxsa_list;
3188	struct nlattr *rxsc_nest = nla_nest_start_noflag(skb, attrtype: j++);
3189
3190	if (!rxsc_nest) {
3191	nla_nest_cancel(skb, start: rxsc_list);
3192	goto nla_put_failure;
3193	}
3194
3195	if (nla_put_u8(skb, attrtype: MACSEC_RXSC_ATTR_ACTIVE, value: rx_sc->active) ||
3196	nla_put_sci(skb, attrtype: MACSEC_RXSC_ATTR_SCI, value: rx_sc->sci,
3197	padattr: MACSEC_RXSC_ATTR_PAD)) {
3198	nla_nest_cancel(skb, start: rxsc_nest);
3199	nla_nest_cancel(skb, start: rxsc_list);
3200	goto nla_put_failure;
3201	}
3202
3203	attr = nla_nest_start_noflag(skb, attrtype: MACSEC_RXSC_ATTR_STATS);
3204	if (!attr) {
3205	nla_nest_cancel(skb, start: rxsc_nest);
3206	nla_nest_cancel(skb, start: rxsc_list);
3207	goto nla_put_failure;
3208	}
3209	memset(&rx_sc_stats, 0, sizeof(rx_sc_stats));
3210	get_rx_sc_stats(dev, rx_sc, sum: &rx_sc_stats);
3211	if (copy_rx_sc_stats(skb, sum: &rx_sc_stats)) {
3212	nla_nest_cancel(skb, start: attr);
3213	nla_nest_cancel(skb, start: rxsc_nest);
3214	nla_nest_cancel(skb, start: rxsc_list);
3215	goto nla_put_failure;
3216	}
3217	nla_nest_end(skb, start: attr);
3218
3219	rxsa_list = nla_nest_start_noflag(skb,
3220	attrtype: MACSEC_RXSC_ATTR_SA_LIST);
3221	if (!rxsa_list) {
3222	nla_nest_cancel(skb, start: rxsc_nest);
3223	nla_nest_cancel(skb, start: rxsc_list);
3224	goto nla_put_failure;
3225	}
3226
3227	for (i = 0, k = 1; i < MACSEC_NUM_AN; i++) {
3228	struct macsec_rx_sa *rx_sa = rtnl_dereference(rx_sc->sa[i]);
3229	struct nlattr *rxsa_nest;
3230	u64 pn;
3231	int pn_len;
3232
3233	if (!rx_sa)
3234	continue;
3235
3236	rxsa_nest = nla_nest_start_noflag(skb, attrtype: k++);
3237	if (!rxsa_nest) {
3238	nla_nest_cancel(skb, start: rxsa_list);
3239	nla_nest_cancel(skb, start: rxsc_nest);
3240	nla_nest_cancel(skb, start: rxsc_list);
3241	goto nla_put_failure;
3242	}
3243
3244	attr = nla_nest_start_noflag(skb,
3245	attrtype: MACSEC_SA_ATTR_STATS);
3246	if (!attr) {
3247	nla_nest_cancel(skb, start: rxsa_list);
3248	nla_nest_cancel(skb, start: rxsc_nest);
3249	nla_nest_cancel(skb, start: rxsc_list);
3250	goto nla_put_failure;
3251	}
3252	memset(&rx_sa_stats, 0, sizeof(rx_sa_stats));
3253	get_rx_sa_stats(dev, rx_sc, an: i, rx_sa, sum: &rx_sa_stats);
3254	if (copy_rx_sa_stats(skb, sum: &rx_sa_stats)) {
3255	nla_nest_cancel(skb, start: attr);
3256	nla_nest_cancel(skb, start: rxsa_list);
3257	nla_nest_cancel(skb, start: rxsc_nest);
3258	nla_nest_cancel(skb, start: rxsc_list);
3259	goto nla_put_failure;
3260	}
3261	nla_nest_end(skb, start: attr);
3262
3263	if (secy->xpn) {
3264	pn = rx_sa->next_pn;
3265	pn_len = MACSEC_XPN_PN_LEN;
3266	} else {
3267	pn = rx_sa->next_pn_halves.lower;
3268	pn_len = MACSEC_DEFAULT_PN_LEN;
3269	}
3270
3271	if (nla_put_u8(skb, attrtype: MACSEC_SA_ATTR_AN, value: i) ||
3272	nla_put(skb, attrtype: MACSEC_SA_ATTR_PN, attrlen: pn_len, data: &pn) ||
3273	nla_put(skb, attrtype: MACSEC_SA_ATTR_KEYID, MACSEC_KEYID_LEN, data: rx_sa->key.id) ||
3274	(secy->xpn && nla_put_ssci(skb, attrtype: MACSEC_SA_ATTR_SSCI, value: rx_sa->ssci)) ||
3275	nla_put_u8(skb, attrtype: MACSEC_SA_ATTR_ACTIVE, value: rx_sa->active)) {
3276	nla_nest_cancel(skb, start: rxsa_nest);
3277	nla_nest_cancel(skb, start: rxsc_nest);
3278	nla_nest_cancel(skb, start: rxsc_list);
3279	goto nla_put_failure;
3280	}
3281	nla_nest_end(skb, start: rxsa_nest);
3282	}
3283
3284	nla_nest_end(skb, start: rxsa_list);
3285	nla_nest_end(skb, start: rxsc_nest);
3286	}
3287
3288	nla_nest_end(skb, start: rxsc_list);
3289
3290	genlmsg_end(skb, hdr);
3291
3292	return 0;
3293
3294	nla_put_failure:
3295	genlmsg_cancel(skb, hdr);
3296	return -EMSGSIZE;
3297	}
3298
3299	static int macsec_generation = 1; /* protected by RTNL */
3300
3301	static int macsec_dump_txsc(struct sk_buff *skb, struct netlink_callback *cb)
3302	{
3303	struct net *net = sock_net(sk: skb->sk);
3304	struct net_device *dev;
3305	int dev_idx, d;
3306
3307	dev_idx = cb->args[0];
3308
3309	d = 0;
3310	rtnl_lock();
3311
3312	cb->seq = macsec_generation;
3313
3314	for_each_netdev(net, dev) {
3315	struct macsec_secy *secy;
3316
3317	if (d < dev_idx)
3318	goto next;
3319
3320	if (!netif_is_macsec(dev))
3321	goto next;
3322
3323	secy = &macsec_priv(dev)->secy;
3324	if (dump_secy(secy, dev, skb, cb) < 0)
3325	goto done;
3326	next:
3327	d++;
3328	}
3329
3330	done:
3331	rtnl_unlock();
3332	cb->args[0] = d;
3333	return skb->len;
3334	}
3335
3336	static const struct genl_small_ops macsec_genl_ops[] = {
3337	{
3338	.cmd = MACSEC_CMD_GET_TXSC,
3339	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3340	.dumpit = macsec_dump_txsc,
3341	},
3342	{
3343	.cmd = MACSEC_CMD_ADD_RXSC,
3344	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3345	.doit = macsec_add_rxsc,
3346	.flags = GENL_ADMIN_PERM,
3347	},
3348	{
3349	.cmd = MACSEC_CMD_DEL_RXSC,
3350	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3351	.doit = macsec_del_rxsc,
3352	.flags = GENL_ADMIN_PERM,
3353	},
3354	{
3355	.cmd = MACSEC_CMD_UPD_RXSC,
3356	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3357	.doit = macsec_upd_rxsc,
3358	.flags = GENL_ADMIN_PERM,
3359	},
3360	{
3361	.cmd = MACSEC_CMD_ADD_TXSA,
3362	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3363	.doit = macsec_add_txsa,
3364	.flags = GENL_ADMIN_PERM,
3365	},
3366	{
3367	.cmd = MACSEC_CMD_DEL_TXSA,
3368	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3369	.doit = macsec_del_txsa,
3370	.flags = GENL_ADMIN_PERM,
3371	},
3372	{
3373	.cmd = MACSEC_CMD_UPD_TXSA,
3374	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3375	.doit = macsec_upd_txsa,
3376	.flags = GENL_ADMIN_PERM,
3377	},
3378	{
3379	.cmd = MACSEC_CMD_ADD_RXSA,
3380	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3381	.doit = macsec_add_rxsa,
3382	.flags = GENL_ADMIN_PERM,
3383	},
3384	{
3385	.cmd = MACSEC_CMD_DEL_RXSA,
3386	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3387	.doit = macsec_del_rxsa,
3388	.flags = GENL_ADMIN_PERM,
3389	},
3390	{
3391	.cmd = MACSEC_CMD_UPD_RXSA,
3392	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3393	.doit = macsec_upd_rxsa,
3394	.flags = GENL_ADMIN_PERM,
3395	},
3396	{
3397	.cmd = MACSEC_CMD_UPD_OFFLOAD,
3398	.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3399	.doit = macsec_upd_offload,
3400	.flags = GENL_ADMIN_PERM,
3401	},
3402	};
3403
3404	static struct genl_family macsec_fam __ro_after_init = {
3405	.name = MACSEC_GENL_NAME,
3406	.hdrsize = 0,
3407	.version = MACSEC_GENL_VERSION,
3408	.maxattr = MACSEC_ATTR_MAX,
3409	.policy = macsec_genl_policy,
3410	.netnsok = true,
3411	.module = THIS_MODULE,
3412	.small_ops = macsec_genl_ops,
3413	.n_small_ops = ARRAY_SIZE(macsec_genl_ops),
3414	.resv_start_op = MACSEC_CMD_UPD_OFFLOAD + 1,
3415	};
3416
3417	static struct sk_buff *macsec_insert_tx_tag(struct sk_buff *skb,
3418	struct net_device *dev)
3419	{
3420	struct macsec_dev *macsec = macsec_priv(dev);
3421	const struct macsec_ops *ops;
3422	struct phy_device *phydev;
3423	struct macsec_context ctx;
3424	int skb_final_len;
3425	int err;
3426
3427	ops = macsec_get_ops(macsec, ctx: &ctx);
3428	skb_final_len = skb->len - ETH_HLEN + ops->needed_headroom +
3429	ops->needed_tailroom;
3430	if (unlikely(skb_final_len > macsec->real_dev->mtu)) {
3431	err = -EINVAL;
3432	goto cleanup;
3433	}
3434
3435	phydev = macsec->real_dev->phydev;
3436
3437	err = skb_ensure_writable_head_tail(skb, dev);
3438	if (unlikely(err < 0))
3439	goto cleanup;
3440
3441	err = ops->mdo_insert_tx_tag(phydev, skb);
3442	if (unlikely(err))
3443	goto cleanup;
3444
3445	return skb;
3446	cleanup:
3447	kfree_skb(skb);
3448	return ERR_PTR(error: err);
3449	}
3450
3451	static netdev_tx_t macsec_start_xmit(struct sk_buff *skb,
3452	struct net_device *dev)
3453	{
3454	struct macsec_dev *macsec = netdev_priv(dev);
3455	struct macsec_secy *secy = &macsec->secy;
3456	struct pcpu_secy_stats *secy_stats;
3457	int ret, len;
3458
3459	if (macsec_is_offloaded(macsec: netdev_priv(dev))) {
3460	struct metadata_dst *md_dst = secy->tx_sc.md_dst;
3461
3462	skb_dst_drop(skb);
3463	dst_hold(dst: &md_dst->dst);
3464	skb_dst_set(skb, dst: &md_dst->dst);
3465
3466	if (macsec->insert_tx_tag) {
3467	skb = macsec_insert_tx_tag(skb, dev);
3468	if (IS_ERR(ptr: skb)) {
3469	DEV_STATS_INC(dev, tx_dropped);
3470	return NETDEV_TX_OK;
3471	}
3472	}
3473
3474	skb->dev = macsec->real_dev;
3475	return dev_queue_xmit(skb);
3476	}
3477
3478	/* 10.5 */
3479	if (!secy->protect_frames) {
3480	secy_stats = this_cpu_ptr(macsec->stats);
3481	u64_stats_update_begin(syncp: &secy_stats->syncp);
3482	secy_stats->stats.OutPktsUntagged++;
3483	u64_stats_update_end(syncp: &secy_stats->syncp);
3484	skb->dev = macsec->real_dev;
3485	len = skb->len;
3486	ret = dev_queue_xmit(skb);
3487	count_tx(dev, ret, len);
3488	return ret;
3489	}
3490
3491	if (!secy->operational) {
3492	kfree_skb(skb);
3493	DEV_STATS_INC(dev, tx_dropped);
3494	return NETDEV_TX_OK;
3495	}
3496
3497	len = skb->len;
3498	skb = macsec_encrypt(skb, dev);
3499	if (IS_ERR(ptr: skb)) {
3500	if (PTR_ERR(ptr: skb) != -EINPROGRESS)
3501	DEV_STATS_INC(dev, tx_dropped);
3502	return NETDEV_TX_OK;
3503	}
3504
3505	macsec_count_tx(skb, tx_sc: &macsec->secy.tx_sc, tx_sa: macsec_skb_cb(skb)->tx_sa);
3506
3507	macsec_encrypt_finish(skb, dev);
3508	ret = dev_queue_xmit(skb);
3509	count_tx(dev, ret, len);
3510	return ret;
3511	}
3512
3513	#define MACSEC_FEATURES \
3514	(NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST)
3515
3516	static int macsec_dev_init(struct net_device *dev)
3517	{
3518	struct macsec_dev *macsec = macsec_priv(dev);
3519	struct net_device *real_dev = macsec->real_dev;
3520	int err;
3521
3522	err = gro_cells_init(gcells: &macsec->gro_cells, dev);
3523	if (err)
3524	return err;
3525
3526	dev->features = real_dev->features & MACSEC_FEATURES;
3527	dev->features |= NETIF_F_LLTX | NETIF_F_GSO_SOFTWARE;
3528	dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
3529
3530	macsec_set_head_tail_room(dev);
3531
3532	if (is_zero_ether_addr(addr: dev->dev_addr))
3533	eth_hw_addr_inherit(dst: dev, src: real_dev);
3534	if (is_zero_ether_addr(addr: dev->broadcast))
3535	memcpy(dev->broadcast, real_dev->broadcast, dev->addr_len);
3536
3537	/* Get macsec's reference to real_dev */
3538	netdev_hold(dev: real_dev, tracker: &macsec->dev_tracker, GFP_KERNEL);
3539
3540	return 0;
3541	}
3542
3543	static void macsec_dev_uninit(struct net_device *dev)
3544	{
3545	struct macsec_dev *macsec = macsec_priv(dev);
3546
3547	gro_cells_destroy(gcells: &macsec->gro_cells);
3548	}
3549
3550	static netdev_features_t macsec_fix_features(struct net_device *dev,
3551	netdev_features_t features)
3552	{
3553	struct macsec_dev *macsec = macsec_priv(dev);
3554	struct net_device *real_dev = macsec->real_dev;
3555
3556	features &= (real_dev->features & MACSEC_FEATURES) |
3557	NETIF_F_GSO_SOFTWARE | NETIF_F_SOFT_FEATURES;
3558	features |= NETIF_F_LLTX;
3559
3560	return features;
3561	}
3562
3563	static int macsec_dev_open(struct net_device *dev)
3564	{
3565	struct macsec_dev *macsec = macsec_priv(dev);
3566	struct net_device *real_dev = macsec->real_dev;
3567	int err;
3568
3569	err = dev_uc_add(dev: real_dev, addr: dev->dev_addr);
3570	if (err < 0)
3571	return err;
3572
3573	if (dev->flags & IFF_ALLMULTI) {
3574	err = dev_set_allmulti(dev: real_dev, inc: 1);
3575	if (err < 0)
3576	goto del_unicast;
3577	}
3578
3579	if (dev->flags & IFF_PROMISC) {
3580	err = dev_set_promiscuity(dev: real_dev, inc: 1);
3581	if (err < 0)
3582	goto clear_allmulti;
3583	}
3584
3585	/* If h/w offloading is available, propagate to the device */
3586	if (macsec_is_offloaded(macsec)) {
3587	const struct macsec_ops *ops;
3588	struct macsec_context ctx;
3589
3590	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
3591	if (!ops) {
3592	err = -EOPNOTSUPP;
3593	goto clear_allmulti;
3594	}
3595
3596	ctx.secy = &macsec->secy;
3597	err = macsec_offload(func: ops->mdo_dev_open, ctx: &ctx);
3598	if (err)
3599	goto clear_allmulti;
3600	}
3601
3602	if (netif_carrier_ok(dev: real_dev))
3603	netif_carrier_on(dev);
3604
3605	return 0;
3606	clear_allmulti:
3607	if (dev->flags & IFF_ALLMULTI)
3608	dev_set_allmulti(dev: real_dev, inc: -1);
3609	del_unicast:
3610	dev_uc_del(dev: real_dev, addr: dev->dev_addr);
3611	netif_carrier_off(dev);
3612	return err;
3613	}
3614
3615	static int macsec_dev_stop(struct net_device *dev)
3616	{
3617	struct macsec_dev *macsec = macsec_priv(dev);
3618	struct net_device *real_dev = macsec->real_dev;
3619
3620	netif_carrier_off(dev);
3621
3622	/* If h/w offloading is available, propagate to the device */
3623	if (macsec_is_offloaded(macsec)) {
3624	const struct macsec_ops *ops;
3625	struct macsec_context ctx;
3626
3627	ops = macsec_get_ops(macsec, ctx: &ctx);
3628	if (ops) {
3629	ctx.secy = &macsec->secy;
3630	macsec_offload(func: ops->mdo_dev_stop, ctx: &ctx);
3631	}
3632	}
3633
3634	dev_mc_unsync(to: real_dev, from: dev);
3635	dev_uc_unsync(to: real_dev, from: dev);
3636
3637	if (dev->flags & IFF_ALLMULTI)
3638	dev_set_allmulti(dev: real_dev, inc: -1);
3639
3640	if (dev->flags & IFF_PROMISC)
3641	dev_set_promiscuity(dev: real_dev, inc: -1);
3642
3643	dev_uc_del(dev: real_dev, addr: dev->dev_addr);
3644
3645	return 0;
3646	}
3647
3648	static void macsec_dev_change_rx_flags(struct net_device *dev, int change)
3649	{
3650	struct net_device *real_dev = macsec_priv(dev)->real_dev;
3651
3652	if (!(dev->flags & IFF_UP))
3653	return;
3654
3655	if (change & IFF_ALLMULTI)
3656	dev_set_allmulti(dev: real_dev, inc: dev->flags & IFF_ALLMULTI ? 1 : -1);
3657
3658	if (change & IFF_PROMISC)
3659	dev_set_promiscuity(dev: real_dev,
3660	inc: dev->flags & IFF_PROMISC ? 1 : -1);
3661	}
3662
3663	static void macsec_dev_set_rx_mode(struct net_device *dev)
3664	{
3665	struct net_device *real_dev = macsec_priv(dev)->real_dev;
3666
3667	dev_mc_sync(to: real_dev, from: dev);
3668	dev_uc_sync(to: real_dev, from: dev);
3669	}
3670
3671	static int macsec_set_mac_address(struct net_device *dev, void *p)
3672	{
3673	struct macsec_dev *macsec = macsec_priv(dev);
3674	struct net_device *real_dev = macsec->real_dev;
3675	struct sockaddr *addr = p;
3676	u8 old_addr[ETH_ALEN];
3677	int err;
3678
3679	if (!is_valid_ether_addr(addr: addr->sa_data))
3680	return -EADDRNOTAVAIL;
3681
3682	if (dev->flags & IFF_UP) {
3683	err = dev_uc_add(dev: real_dev, addr: addr->sa_data);
3684	if (err < 0)
3685	return err;
3686	}
3687
3688	ether_addr_copy(dst: old_addr, src: dev->dev_addr);
3689	eth_hw_addr_set(dev, addr: addr->sa_data);
3690
3691	/* If h/w offloading is available, propagate to the device */
3692	if (macsec_is_offloaded(macsec)) {
3693	const struct macsec_ops *ops;
3694	struct macsec_context ctx;
3695
3696	ops = macsec_get_ops(macsec, ctx: &ctx);
3697	if (!ops) {
3698	err = -EOPNOTSUPP;
3699	goto restore_old_addr;
3700	}
3701
3702	ctx.secy = &macsec->secy;
3703	err = macsec_offload(func: ops->mdo_upd_secy, ctx: &ctx);
3704	if (err)
3705	goto restore_old_addr;
3706	}
3707
3708	if (dev->flags & IFF_UP)
3709	dev_uc_del(dev: real_dev, addr: old_addr);
3710
3711	return 0;
3712
3713	restore_old_addr:
3714	if (dev->flags & IFF_UP)
3715	dev_uc_del(dev: real_dev, addr: addr->sa_data);
3716
3717	eth_hw_addr_set(dev, addr: old_addr);
3718
3719	return err;
3720	}
3721
3722	static int macsec_change_mtu(struct net_device *dev, int new_mtu)
3723	{
3724	struct macsec_dev *macsec = macsec_priv(dev);
3725	unsigned int extra = macsec->secy.icv_len + macsec_extra_len(sci_present: true);
3726
3727	if (macsec->real_dev->mtu - extra < new_mtu)
3728	return -ERANGE;
3729
3730	dev->mtu = new_mtu;
3731
3732	return 0;
3733	}
3734
3735	static void macsec_get_stats64(struct net_device *dev,
3736	struct rtnl_link_stats64 *s)
3737	{
3738	if (!dev->tstats)
3739	return;
3740
3741	dev_fetch_sw_netstats(s, netstats: dev->tstats);
3742
3743	s->rx_dropped = DEV_STATS_READ(dev, rx_dropped);
3744	s->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
3745	s->rx_errors = DEV_STATS_READ(dev, rx_errors);
3746	}
3747
3748	static int macsec_get_iflink(const struct net_device *dev)
3749	{
3750	return READ_ONCE(macsec_priv(dev)->real_dev->ifindex);
3751	}
3752
3753	static const struct net_device_ops macsec_netdev_ops = {
3754	.ndo_init = macsec_dev_init,
3755	.ndo_uninit = macsec_dev_uninit,
3756	.ndo_open = macsec_dev_open,
3757	.ndo_stop = macsec_dev_stop,
3758	.ndo_fix_features = macsec_fix_features,
3759	.ndo_change_mtu = macsec_change_mtu,
3760	.ndo_set_rx_mode = macsec_dev_set_rx_mode,
3761	.ndo_change_rx_flags = macsec_dev_change_rx_flags,
3762	.ndo_set_mac_address = macsec_set_mac_address,
3763	.ndo_start_xmit = macsec_start_xmit,
3764	.ndo_get_stats64 = macsec_get_stats64,
3765	.ndo_get_iflink = macsec_get_iflink,
3766	};
3767
3768	static const struct device_type macsec_type = {
3769	.name = "macsec",
3770	};
3771
3772	static const struct nla_policy macsec_rtnl_policy[IFLA_MACSEC_MAX + 1] = {
3773	[IFLA_MACSEC_SCI] = { .type = NLA_U64 },
3774	[IFLA_MACSEC_PORT] = { .type = NLA_U16 },
3775	[IFLA_MACSEC_ICV_LEN] = { .type = NLA_U8 },
3776	[IFLA_MACSEC_CIPHER_SUITE] = { .type = NLA_U64 },
3777	[IFLA_MACSEC_WINDOW] = { .type = NLA_U32 },
3778	[IFLA_MACSEC_ENCODING_SA] = { .type = NLA_U8 },
3779	[IFLA_MACSEC_ENCRYPT] = { .type = NLA_U8 },
3780	[IFLA_MACSEC_PROTECT] = { .type = NLA_U8 },
3781	[IFLA_MACSEC_INC_SCI] = { .type = NLA_U8 },
3782	[IFLA_MACSEC_ES] = { .type = NLA_U8 },
3783	[IFLA_MACSEC_SCB] = { .type = NLA_U8 },
3784	[IFLA_MACSEC_REPLAY_PROTECT] = { .type = NLA_U8 },
3785	[IFLA_MACSEC_VALIDATION] = { .type = NLA_U8 },
3786	[IFLA_MACSEC_OFFLOAD] = { .type = NLA_U8 },
3787	};
3788
3789	static void macsec_free_netdev(struct net_device *dev)
3790	{
3791	struct macsec_dev *macsec = macsec_priv(dev);
3792
3793	if (macsec->secy.tx_sc.md_dst)
3794	metadata_dst_free(macsec->secy.tx_sc.md_dst);
3795	free_percpu(pdata: macsec->stats);
3796	free_percpu(pdata: macsec->secy.tx_sc.stats);
3797
3798	/* Get rid of the macsec's reference to real_dev */
3799	netdev_put(dev: macsec->real_dev, tracker: &macsec->dev_tracker);
3800	}
3801
3802	static void macsec_setup(struct net_device *dev)
3803	{
3804	ether_setup(dev);
3805	dev->min_mtu = 0;
3806	dev->max_mtu = ETH_MAX_MTU;
3807	dev->priv_flags |= IFF_NO_QUEUE;
3808	dev->netdev_ops = &macsec_netdev_ops;
3809	dev->needs_free_netdev = true;
3810	dev->priv_destructor = macsec_free_netdev;
3811	SET_NETDEV_DEVTYPE(dev, &macsec_type);
3812
3813	eth_zero_addr(addr: dev->broadcast);
3814	}
3815
3816	static int macsec_changelink_common(struct net_device *dev,
3817	struct nlattr *data[])
3818	{
3819	struct macsec_secy *secy;
3820	struct macsec_tx_sc *tx_sc;
3821
3822	secy = &macsec_priv(dev)->secy;
3823	tx_sc = &secy->tx_sc;
3824
3825	if (data[IFLA_MACSEC_ENCODING_SA]) {
3826	struct macsec_tx_sa *tx_sa;
3827
3828	tx_sc->encoding_sa = nla_get_u8(nla: data[IFLA_MACSEC_ENCODING_SA]);
3829	tx_sa = rtnl_dereference(tx_sc->sa[tx_sc->encoding_sa]);
3830
3831	secy->operational = tx_sa && tx_sa->active;
3832	}
3833
3834	if (data[IFLA_MACSEC_ENCRYPT])
3835	tx_sc->encrypt = !!nla_get_u8(nla: data[IFLA_MACSEC_ENCRYPT]);
3836
3837	if (data[IFLA_MACSEC_PROTECT])
3838	secy->protect_frames = !!nla_get_u8(nla: data[IFLA_MACSEC_PROTECT]);
3839
3840	if (data[IFLA_MACSEC_INC_SCI])
3841	tx_sc->send_sci = !!nla_get_u8(nla: data[IFLA_MACSEC_INC_SCI]);
3842
3843	if (data[IFLA_MACSEC_ES])
3844	tx_sc->end_station = !!nla_get_u8(nla: data[IFLA_MACSEC_ES]);
3845
3846	if (data[IFLA_MACSEC_SCB])
3847	tx_sc->scb = !!nla_get_u8(nla: data[IFLA_MACSEC_SCB]);
3848
3849	if (data[IFLA_MACSEC_REPLAY_PROTECT])
3850	secy->replay_protect = !!nla_get_u8(nla: data[IFLA_MACSEC_REPLAY_PROTECT]);
3851
3852	if (data[IFLA_MACSEC_VALIDATION])
3853	secy->validate_frames = nla_get_u8(nla: data[IFLA_MACSEC_VALIDATION]);
3854
3855	if (data[IFLA_MACSEC_CIPHER_SUITE]) {
3856	switch (nla_get_u64(nla: data[IFLA_MACSEC_CIPHER_SUITE])) {
3857	case MACSEC_CIPHER_ID_GCM_AES_128:
3858	case MACSEC_DEFAULT_CIPHER_ID:
3859	secy->key_len = MACSEC_GCM_AES_128_SAK_LEN;
3860	secy->xpn = false;
3861	break;
3862	case MACSEC_CIPHER_ID_GCM_AES_256:
3863	secy->key_len = MACSEC_GCM_AES_256_SAK_LEN;
3864	secy->xpn = false;
3865	break;
3866	case MACSEC_CIPHER_ID_GCM_AES_XPN_128:
3867	secy->key_len = MACSEC_GCM_AES_128_SAK_LEN;
3868	secy->xpn = true;
3869	break;
3870	case MACSEC_CIPHER_ID_GCM_AES_XPN_256:
3871	secy->key_len = MACSEC_GCM_AES_256_SAK_LEN;
3872	secy->xpn = true;
3873	break;
3874	default:
3875	return -EINVAL;
3876	}
3877	}
3878
3879	if (data[IFLA_MACSEC_WINDOW]) {
3880	secy->replay_window = nla_get_u32(nla: data[IFLA_MACSEC_WINDOW]);
3881
3882	/* IEEE 802.1AEbw-2013 10.7.8 - maximum replay window
3883	* for XPN cipher suites */
3884	if (secy->xpn &&
3885	secy->replay_window > MACSEC_XPN_MAX_REPLAY_WINDOW)
3886	return -EINVAL;
3887	}
3888
3889	return 0;
3890	}
3891
3892	static int macsec_changelink(struct net_device *dev, struct nlattr *tb[],
3893	struct nlattr *data[],
3894	struct netlink_ext_ack *extack)
3895	{
3896	struct macsec_dev *macsec = macsec_priv(dev);
3897	bool macsec_offload_state_change = false;
3898	enum macsec_offload offload;
3899	struct macsec_tx_sc tx_sc;
3900	struct macsec_secy secy;
3901	int ret;
3902
3903	if (!data)
3904	return 0;
3905
3906	if (data[IFLA_MACSEC_CIPHER_SUITE] ||
3907	data[IFLA_MACSEC_ICV_LEN] ||
3908	data[IFLA_MACSEC_SCI] ||
3909	data[IFLA_MACSEC_PORT])
3910	return -EINVAL;
3911
3912	/* Keep a copy of unmodified secy and tx_sc, in case the offload
3913	* propagation fails, to revert macsec_changelink_common.
3914	*/
3915	memcpy(&secy, &macsec->secy, sizeof(secy));
3916	memcpy(&tx_sc, &macsec->secy.tx_sc, sizeof(tx_sc));
3917
3918	ret = macsec_changelink_common(dev, data);
3919	if (ret)
3920	goto cleanup;
3921
3922	if (data[IFLA_MACSEC_OFFLOAD]) {
3923	offload = nla_get_u8(nla: data[IFLA_MACSEC_OFFLOAD]);
3924	if (macsec->offload != offload) {
3925	macsec_offload_state_change = true;
3926	ret = macsec_update_offload(dev, offload);
3927	if (ret)
3928	goto cleanup;
3929	}
3930	}
3931
3932	/* If h/w offloading is available, propagate to the device */
3933	if (!macsec_offload_state_change && macsec_is_offloaded(macsec)) {
3934	const struct macsec_ops *ops;
3935	struct macsec_context ctx;
3936
3937	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
3938	if (!ops) {
3939	ret = -EOPNOTSUPP;
3940	goto cleanup;
3941	}
3942
3943	ctx.secy = &macsec->secy;
3944	ret = macsec_offload(func: ops->mdo_upd_secy, ctx: &ctx);
3945	if (ret)
3946	goto cleanup;
3947	}
3948
3949	return 0;
3950
3951	cleanup:
3952	memcpy(&macsec->secy.tx_sc, &tx_sc, sizeof(tx_sc));
3953	memcpy(&macsec->secy, &secy, sizeof(secy));
3954
3955	return ret;
3956	}
3957
3958	static void macsec_del_dev(struct macsec_dev *macsec)
3959	{
3960	int i;
3961
3962	while (macsec->secy.rx_sc) {
3963	struct macsec_rx_sc *rx_sc = rtnl_dereference(macsec->secy.rx_sc);
3964
3965	rcu_assign_pointer(macsec->secy.rx_sc, rx_sc->next);
3966	free_rx_sc(rx_sc);
3967	}
3968
3969	for (i = 0; i < MACSEC_NUM_AN; i++) {
3970	struct macsec_tx_sa *sa = rtnl_dereference(macsec->secy.tx_sc.sa[i]);
3971
3972	if (sa) {
3973	RCU_INIT_POINTER(macsec->secy.tx_sc.sa[i], NULL);
3974	clear_tx_sa(tx_sa: sa);
3975	}
3976	}
3977	}
3978
3979	static void macsec_common_dellink(struct net_device *dev, struct list_head *head)
3980	{
3981	struct macsec_dev *macsec = macsec_priv(dev);
3982	struct net_device *real_dev = macsec->real_dev;
3983
3984	/* If h/w offloading is available, propagate to the device */
3985	if (macsec_is_offloaded(macsec)) {
3986	const struct macsec_ops *ops;
3987	struct macsec_context ctx;
3988
3989	ops = macsec_get_ops(macsec: netdev_priv(dev), ctx: &ctx);
3990	if (ops) {
3991	ctx.secy = &macsec->secy;
3992	macsec_offload(func: ops->mdo_del_secy, ctx: &ctx);
3993	}
3994	}
3995
3996	unregister_netdevice_queue(dev, head);
3997	list_del_rcu(entry: &macsec->secys);
3998	macsec_del_dev(macsec);
3999	netdev_upper_dev_unlink(dev: real_dev, upper_dev: dev);
4000
4001	macsec_generation++;
4002	}
4003
4004	static void macsec_dellink(struct net_device *dev, struct list_head *head)
4005	{
4006	struct macsec_dev *macsec = macsec_priv(dev);
4007	struct net_device *real_dev = macsec->real_dev;
4008	struct macsec_rxh_data *rxd = macsec_data_rtnl(dev: real_dev);
4009
4010	macsec_common_dellink(dev, head);
4011
4012	if (list_empty(head: &rxd->secys)) {
4013	netdev_rx_handler_unregister(dev: real_dev);
4014	kfree(objp: rxd);
4015	}
4016	}
4017
4018	static int register_macsec_dev(struct net_device *real_dev,
4019	struct net_device *dev)
4020	{
4021	struct macsec_dev *macsec = macsec_priv(dev);
4022	struct macsec_rxh_data *rxd = macsec_data_rtnl(dev: real_dev);
4023
4024	if (!rxd) {
4025	int err;
4026
4027	rxd = kmalloc(size: sizeof(*rxd), GFP_KERNEL);
4028	if (!rxd)
4029	return -ENOMEM;
4030
4031	INIT_LIST_HEAD(list: &rxd->secys);
4032
4033	err = netdev_rx_handler_register(dev: real_dev, rx_handler: macsec_handle_frame,
4034	rx_handler_data: rxd);
4035	if (err < 0) {
4036	kfree(objp: rxd);
4037	return err;
4038	}
4039	}
4040
4041	list_add_tail_rcu(new: &macsec->secys, head: &rxd->secys);
4042	return 0;
4043	}
4044
4045	static bool sci_exists(struct net_device *dev, sci_t sci)
4046	{
4047	struct macsec_rxh_data *rxd = macsec_data_rtnl(dev);
4048	struct macsec_dev *macsec;
4049
4050	list_for_each_entry(macsec, &rxd->secys, secys) {
4051	if (macsec->secy.sci == sci)
4052	return true;
4053	}
4054
4055	return false;
4056	}
4057
4058	static sci_t dev_to_sci(struct net_device *dev, __be16 port)
4059	{
4060	return make_sci(addr: dev->dev_addr, port);
4061	}
4062
4063	static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len)
4064	{
4065	struct macsec_dev *macsec = macsec_priv(dev);
4066	struct macsec_secy *secy = &macsec->secy;
4067
4068	macsec->stats = netdev_alloc_pcpu_stats(struct pcpu_secy_stats);
4069	if (!macsec->stats)
4070	return -ENOMEM;
4071
4072	secy->tx_sc.stats = netdev_alloc_pcpu_stats(struct pcpu_tx_sc_stats);
4073	if (!secy->tx_sc.stats)
4074	return -ENOMEM;
4075
4076	secy->tx_sc.md_dst = metadata_dst_alloc(optslen: 0, type: METADATA_MACSEC, GFP_KERNEL);
4077	if (!secy->tx_sc.md_dst)
4078	/* macsec and secy percpu stats will be freed when unregistering
4079	* net_device in macsec_free_netdev()
4080	*/
4081	return -ENOMEM;
4082
4083	if (sci == MACSEC_UNDEF_SCI)
4084	sci = dev_to_sci(dev, MACSEC_PORT_ES);
4085
4086	secy->netdev = dev;
4087	secy->operational = true;
4088	secy->key_len = DEFAULT_SAK_LEN;
4089	secy->icv_len = icv_len;
4090	secy->validate_frames = MACSEC_VALIDATE_DEFAULT;
4091	secy->protect_frames = true;
4092	secy->replay_protect = false;
4093	secy->xpn = DEFAULT_XPN;
4094
4095	secy->sci = sci;
4096	secy->tx_sc.md_dst->u.macsec_info.sci = sci;
4097	secy->tx_sc.active = true;
4098	secy->tx_sc.encoding_sa = DEFAULT_ENCODING_SA;
4099	secy->tx_sc.encrypt = DEFAULT_ENCRYPT;
4100	secy->tx_sc.send_sci = DEFAULT_SEND_SCI;
4101	secy->tx_sc.end_station = false;
4102	secy->tx_sc.scb = false;
4103
4104	return 0;
4105	}
4106
4107	static struct lock_class_key macsec_netdev_addr_lock_key;
4108
4109	static int macsec_newlink(struct net *net, struct net_device *dev,
4110	struct nlattr *tb[], struct nlattr *data[],
4111	struct netlink_ext_ack *extack)
4112	{
4113	struct macsec_dev *macsec = macsec_priv(dev);
4114	rx_handler_func_t *rx_handler;
4115	u8 icv_len = MACSEC_DEFAULT_ICV_LEN;
4116	struct net_device *real_dev;
4117	int err, mtu;
4118	sci_t sci;
4119
4120	if (!tb[IFLA_LINK])
4121	return -EINVAL;
4122	real_dev = __dev_get_by_index(net, ifindex: nla_get_u32(nla: tb[IFLA_LINK]));
4123	if (!real_dev)
4124	return -ENODEV;
4125	if (real_dev->type != ARPHRD_ETHER)
4126	return -EINVAL;
4127
4128	dev->priv_flags |= IFF_MACSEC;
4129
4130	macsec->real_dev = real_dev;
4131
4132	if (data && data[IFLA_MACSEC_OFFLOAD])
4133	macsec->offload = nla_get_offload(nla: data[IFLA_MACSEC_OFFLOAD]);
4134	else
4135	/* MACsec offloading is off by default */
4136	macsec->offload = MACSEC_OFFLOAD_OFF;
4137
4138	/* Check if the offloading mode is supported by the underlying layers */
4139	if (macsec->offload != MACSEC_OFFLOAD_OFF &&
4140	!macsec_check_offload(offload: macsec->offload, macsec))
4141	return -EOPNOTSUPP;
4142
4143	/* send_sci must be set to true when transmit sci explicitly is set */
4144	if ((data && data[IFLA_MACSEC_SCI]) &&
4145	(data && data[IFLA_MACSEC_INC_SCI])) {
4146	u8 send_sci = !!nla_get_u8(nla: data[IFLA_MACSEC_INC_SCI]);
4147
4148	if (!send_sci)
4149	return -EINVAL;
4150	}
4151
4152	if (data && data[IFLA_MACSEC_ICV_LEN])
4153	icv_len = nla_get_u8(nla: data[IFLA_MACSEC_ICV_LEN]);
4154	mtu = real_dev->mtu - icv_len - macsec_extra_len(sci_present: true);
4155	if (mtu < 0)
4156	dev->mtu = 0;
4157	else
4158	dev->mtu = mtu;
4159
4160	rx_handler = rtnl_dereference(real_dev->rx_handler);
4161	if (rx_handler && rx_handler != macsec_handle_frame)
4162	return -EBUSY;
4163
4164	err = register_netdevice(dev);
4165	if (err < 0)
4166	return err;
4167
4168	netdev_lockdep_set_classes(dev);
4169	lockdep_set_class(&dev->addr_list_lock,
4170	&macsec_netdev_addr_lock_key);
4171
4172	err = netdev_upper_dev_link(dev: real_dev, upper_dev: dev, extack);
4173	if (err < 0)
4174	goto unregister;
4175
4176	/* need to be already registered so that ->init has run and
4177	* the MAC addr is set
4178	*/
4179	if (data && data[IFLA_MACSEC_SCI])
4180	sci = nla_get_sci(nla: data[IFLA_MACSEC_SCI]);
4181	else if (data && data[IFLA_MACSEC_PORT])
4182	sci = dev_to_sci(dev, port: nla_get_be16(nla: data[IFLA_MACSEC_PORT]));
4183	else
4184	sci = dev_to_sci(dev, MACSEC_PORT_ES);
4185
4186	if (rx_handler && sci_exists(dev: real_dev, sci)) {
4187	err = -EBUSY;
4188	goto unlink;
4189	}
4190
4191	err = macsec_add_dev(dev, sci, icv_len);
4192	if (err)
4193	goto unlink;
4194
4195	if (data) {
4196	err = macsec_changelink_common(dev, data);
4197	if (err)
4198	goto del_dev;
4199	}
4200
4201	/* If h/w offloading is available, propagate to the device */
4202	if (macsec_is_offloaded(macsec)) {
4203	const struct macsec_ops *ops;
4204	struct macsec_context ctx;
4205
4206	ops = macsec_get_ops(macsec, ctx: &ctx);
4207	if (ops) {
4208	ctx.secy = &macsec->secy;
4209	err = macsec_offload(func: ops->mdo_add_secy, ctx: &ctx);
4210	if (err)
4211	goto del_dev;
4212
4213	macsec->insert_tx_tag =
4214	macsec_needs_tx_tag(macsec, ops);
4215	}
4216	}
4217
4218	err = register_macsec_dev(real_dev, dev);
4219	if (err < 0)
4220	goto del_dev;
4221
4222	netif_stacked_transfer_operstate(rootdev: real_dev, dev);
4223	linkwatch_fire_event(dev);
4224
4225	macsec_generation++;
4226
4227	return 0;
4228
4229	del_dev:
4230	macsec_del_dev(macsec);
4231	unlink:
4232	netdev_upper_dev_unlink(dev: real_dev, upper_dev: dev);
4233	unregister:
4234	unregister_netdevice(dev);
4235	return err;
4236	}
4237
4238	static int macsec_validate_attr(struct nlattr *tb[], struct nlattr *data[],
4239	struct netlink_ext_ack *extack)
4240	{
4241	u64 csid = MACSEC_DEFAULT_CIPHER_ID;
4242	u8 icv_len = MACSEC_DEFAULT_ICV_LEN;
4243	int flag;
4244	bool es, scb, sci;
4245
4246	if (!data)
4247	return 0;
4248
4249	if (data[IFLA_MACSEC_CIPHER_SUITE])
4250	csid = nla_get_u64(nla: data[IFLA_MACSEC_CIPHER_SUITE]);
4251
4252	if (data[IFLA_MACSEC_ICV_LEN]) {
4253	icv_len = nla_get_u8(nla: data[IFLA_MACSEC_ICV_LEN]);
4254	if (icv_len != MACSEC_DEFAULT_ICV_LEN) {
4255	char dummy_key[DEFAULT_SAK_LEN] = { 0 };
4256	struct crypto_aead *dummy_tfm;
4257
4258	dummy_tfm = macsec_alloc_tfm(key: dummy_key,
4259	DEFAULT_SAK_LEN,
4260	icv_len);
4261	if (IS_ERR(ptr: dummy_tfm))
4262	return PTR_ERR(ptr: dummy_tfm);
4263	crypto_free_aead(tfm: dummy_tfm);
4264	}
4265	}
4266
4267	switch (csid) {
4268	case MACSEC_CIPHER_ID_GCM_AES_128:
4269	case MACSEC_CIPHER_ID_GCM_AES_256:
4270	case MACSEC_CIPHER_ID_GCM_AES_XPN_128:
4271	case MACSEC_CIPHER_ID_GCM_AES_XPN_256:
4272	case MACSEC_DEFAULT_CIPHER_ID:
4273	if (icv_len < MACSEC_MIN_ICV_LEN ||
4274	icv_len > MACSEC_STD_ICV_LEN)
4275	return -EINVAL;
4276	break;
4277	default:
4278	return -EINVAL;
4279	}
4280
4281	if (data[IFLA_MACSEC_ENCODING_SA]) {
4282	if (nla_get_u8(nla: data[IFLA_MACSEC_ENCODING_SA]) >= MACSEC_NUM_AN)
4283	return -EINVAL;
4284	}
4285
4286	for (flag = IFLA_MACSEC_ENCODING_SA + 1;
4287	flag < IFLA_MACSEC_VALIDATION;
4288	flag++) {
4289	if (data[flag]) {
4290	if (nla_get_u8(nla: data[flag]) > 1)
4291	return -EINVAL;
4292	}
4293	}
4294
4295	es = data[IFLA_MACSEC_ES] ? nla_get_u8(nla: data[IFLA_MACSEC_ES]) : false;
4296	sci = data[IFLA_MACSEC_INC_SCI] ? nla_get_u8(nla: data[IFLA_MACSEC_INC_SCI]) : false;
4297	scb = data[IFLA_MACSEC_SCB] ? nla_get_u8(nla: data[IFLA_MACSEC_SCB]) : false;
4298
4299	if ((sci && (scb || es)) || (scb && es))
4300	return -EINVAL;
4301
4302	if (data[IFLA_MACSEC_VALIDATION] &&
4303	nla_get_u8(nla: data[IFLA_MACSEC_VALIDATION]) > MACSEC_VALIDATE_MAX)
4304	return -EINVAL;
4305
4306	if ((data[IFLA_MACSEC_REPLAY_PROTECT] &&
4307	nla_get_u8(nla: data[IFLA_MACSEC_REPLAY_PROTECT])) &&
4308	!data[IFLA_MACSEC_WINDOW])
4309	return -EINVAL;
4310
4311	return 0;
4312	}
4313
4314	static struct net *macsec_get_link_net(const struct net_device *dev)
4315	{
4316	return dev_net(dev: macsec_priv(dev)->real_dev);
4317	}
4318
4319	struct net_device *macsec_get_real_dev(const struct net_device *dev)
4320	{
4321	return macsec_priv(dev)->real_dev;
4322	}
4323	EXPORT_SYMBOL_GPL(macsec_get_real_dev);
4324
4325	bool macsec_netdev_is_offloaded(struct net_device *dev)
4326	{
4327	return macsec_is_offloaded(macsec: macsec_priv(dev));
4328	}
4329	EXPORT_SYMBOL_GPL(macsec_netdev_is_offloaded);
4330
4331	static size_t macsec_get_size(const struct net_device *dev)
4332	{
4333	return nla_total_size_64bit(payload: 8) + /* IFLA_MACSEC_SCI */
4334	nla_total_size(payload: 1) + /* IFLA_MACSEC_ICV_LEN */
4335	nla_total_size_64bit(payload: 8) + /* IFLA_MACSEC_CIPHER_SUITE */
4336	nla_total_size(payload: 4) + /* IFLA_MACSEC_WINDOW */
4337	nla_total_size(payload: 1) + /* IFLA_MACSEC_ENCODING_SA */
4338	nla_total_size(payload: 1) + /* IFLA_MACSEC_ENCRYPT */
4339	nla_total_size(payload: 1) + /* IFLA_MACSEC_PROTECT */
4340	nla_total_size(payload: 1) + /* IFLA_MACSEC_INC_SCI */
4341	nla_total_size(payload: 1) + /* IFLA_MACSEC_ES */
4342	nla_total_size(payload: 1) + /* IFLA_MACSEC_SCB */
4343	nla_total_size(payload: 1) + /* IFLA_MACSEC_REPLAY_PROTECT */
4344	nla_total_size(payload: 1) + /* IFLA_MACSEC_VALIDATION */
4345	nla_total_size(payload: 1) + /* IFLA_MACSEC_OFFLOAD */
4346	0;
4347	}
4348
4349	static int macsec_fill_info(struct sk_buff *skb,
4350	const struct net_device *dev)
4351	{
4352	struct macsec_tx_sc *tx_sc;
4353	struct macsec_dev *macsec;
4354	struct macsec_secy *secy;
4355	u64 csid;
4356
4357	macsec = macsec_priv(dev);
4358	secy = &macsec->secy;
4359	tx_sc = &secy->tx_sc;
4360
4361	switch (secy->key_len) {
4362	case MACSEC_GCM_AES_128_SAK_LEN:
4363	csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_128 : MACSEC_DEFAULT_CIPHER_ID;
4364	break;
4365	case MACSEC_GCM_AES_256_SAK_LEN:
4366	csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_256 : MACSEC_CIPHER_ID_GCM_AES_256;
4367	break;
4368	default:
4369	goto nla_put_failure;
4370	}
4371
4372	if (nla_put_sci(skb, attrtype: IFLA_MACSEC_SCI, value: secy->sci,
4373	padattr: IFLA_MACSEC_PAD) ||
4374	nla_put_u8(skb, attrtype: IFLA_MACSEC_ICV_LEN, value: secy->icv_len) ||
4375	nla_put_u64_64bit(skb, attrtype: IFLA_MACSEC_CIPHER_SUITE,
4376	value: csid, padattr: IFLA_MACSEC_PAD) ||
4377	nla_put_u8(skb, attrtype: IFLA_MACSEC_ENCODING_SA, value: tx_sc->encoding_sa) ||
4378	nla_put_u8(skb, attrtype: IFLA_MACSEC_ENCRYPT, value: tx_sc->encrypt) ||
4379	nla_put_u8(skb, attrtype: IFLA_MACSEC_PROTECT, value: secy->protect_frames) ||
4380	nla_put_u8(skb, attrtype: IFLA_MACSEC_INC_SCI, value: tx_sc->send_sci) ||
4381	nla_put_u8(skb, attrtype: IFLA_MACSEC_ES, value: tx_sc->end_station) ||
4382	nla_put_u8(skb, attrtype: IFLA_MACSEC_SCB, value: tx_sc->scb) ||
4383	nla_put_u8(skb, attrtype: IFLA_MACSEC_REPLAY_PROTECT, value: secy->replay_protect) ||
4384	nla_put_u8(skb, attrtype: IFLA_MACSEC_VALIDATION, value: secy->validate_frames) ||
4385	nla_put_u8(skb, attrtype: IFLA_MACSEC_OFFLOAD, value: macsec->offload) ||
4386	0)
4387	goto nla_put_failure;
4388
4389	if (secy->replay_protect) {
4390	if (nla_put_u32(skb, attrtype: IFLA_MACSEC_WINDOW, value: secy->replay_window))
4391	goto nla_put_failure;
4392	}
4393
4394	return 0;
4395
4396	nla_put_failure:
4397	return -EMSGSIZE;
4398	}
4399
4400	static struct rtnl_link_ops macsec_link_ops __read_mostly = {
4401	.kind = "macsec",
4402	.priv_size = sizeof(struct macsec_dev),
4403	.maxtype = IFLA_MACSEC_MAX,
4404	.policy = macsec_rtnl_policy,
4405	.setup = macsec_setup,
4406	.validate = macsec_validate_attr,
4407	.newlink = macsec_newlink,
4408	.changelink = macsec_changelink,
4409	.dellink = macsec_dellink,
4410	.get_size = macsec_get_size,
4411	.fill_info = macsec_fill_info,
4412	.get_link_net = macsec_get_link_net,
4413	};
4414
4415	static bool is_macsec_master(struct net_device *dev)
4416	{
4417	return rcu_access_pointer(dev->rx_handler) == macsec_handle_frame;
4418	}
4419
4420	static int macsec_notify(struct notifier_block *this, unsigned long event,
4421	void *ptr)
4422	{
4423	struct net_device *real_dev = netdev_notifier_info_to_dev(info: ptr);
4424	LIST_HEAD(head);
4425
4426	if (!is_macsec_master(dev: real_dev))
4427	return NOTIFY_DONE;
4428
4429	switch (event) {
4430	case NETDEV_DOWN:
4431	case NETDEV_UP:
4432	case NETDEV_CHANGE: {
4433	struct macsec_dev *m, *n;
4434	struct macsec_rxh_data *rxd;
4435
4436	rxd = macsec_data_rtnl(dev: real_dev);
4437	list_for_each_entry_safe(m, n, &rxd->secys, secys) {
4438	struct net_device *dev = m->secy.netdev;
4439
4440	netif_stacked_transfer_operstate(rootdev: real_dev, dev);
4441	}
4442	break;
4443	}
4444	case NETDEV_UNREGISTER: {
4445	struct macsec_dev *m, *n;
4446	struct macsec_rxh_data *rxd;
4447
4448	rxd = macsec_data_rtnl(dev: real_dev);
4449	list_for_each_entry_safe(m, n, &rxd->secys, secys) {
4450	macsec_common_dellink(dev: m->secy.netdev, head: &head);
4451	}
4452
4453	netdev_rx_handler_unregister(dev: real_dev);
4454	kfree(objp: rxd);
4455
4456	unregister_netdevice_many(head: &head);
4457	break;
4458	}
4459	case NETDEV_CHANGEMTU: {
4460	struct macsec_dev *m;
4461	struct macsec_rxh_data *rxd;
4462
4463	rxd = macsec_data_rtnl(dev: real_dev);
4464	list_for_each_entry(m, &rxd->secys, secys) {
4465	struct net_device *dev = m->secy.netdev;
4466	unsigned int mtu = real_dev->mtu - (m->secy.icv_len +
4467	macsec_extra_len(sci_present: true));
4468
4469	if (dev->mtu > mtu)
4470	dev_set_mtu(dev, mtu);
4471	}
4472	}
4473	}
4474
4475	return NOTIFY_OK;
4476	}
4477
4478	static struct notifier_block macsec_notifier = {
4479	.notifier_call = macsec_notify,
4480	};
4481
4482	static int __init macsec_init(void)
4483	{
4484	int err;
4485
4486	pr_info("MACsec IEEE 802.1AE\n");
4487	err = register_netdevice_notifier(nb: &macsec_notifier);
4488	if (err)
4489	return err;
4490
4491	err = rtnl_link_register(ops: &macsec_link_ops);
4492	if (err)
4493	goto notifier;
4494
4495	err = genl_register_family(family: &macsec_fam);
4496	if (err)
4497	goto rtnl;
4498
4499	return 0;
4500
4501	rtnl:
4502	rtnl_link_unregister(ops: &macsec_link_ops);
4503	notifier:
4504	unregister_netdevice_notifier(nb: &macsec_notifier);
4505	return err;
4506	}
4507
4508	static void __exit macsec_exit(void)
4509	{
4510	genl_unregister_family(family: &macsec_fam);
4511	rtnl_link_unregister(ops: &macsec_link_ops);
4512	unregister_netdevice_notifier(nb: &macsec_notifier);
4513	rcu_barrier();
4514	}
4515
4516	module_init(macsec_init);
4517	module_exit(macsec_exit);
4518
4519	MODULE_ALIAS_RTNL_LINK("macsec");
4520	MODULE_ALIAS_GENL_FAMILY("macsec");
4521
4522	MODULE_DESCRIPTION("MACsec IEEE 802.1AE");
4523	MODULE_LICENSE("GPL v2");
4524