1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2017 Oracle and/or its affiliates. All rights reserved. */ |
3 | |
4 | #include "ixgbe.h" |
5 | #include <net/xfrm.h> |
6 | #include <crypto/aead.h> |
7 | #include <linux/if_bridge.h> |
8 | |
9 | #define IXGBE_IPSEC_KEY_BITS 160 |
10 | static const char aes_gcm_name[] = "rfc4106(gcm(aes))" ; |
11 | |
12 | static void ixgbe_ipsec_del_sa(struct xfrm_state *xs); |
13 | |
14 | /** |
15 | * ixgbe_ipsec_set_tx_sa - set the Tx SA registers |
16 | * @hw: hw specific details |
17 | * @idx: register index to write |
18 | * @key: key byte array |
19 | * @salt: salt bytes |
20 | **/ |
21 | static void ixgbe_ipsec_set_tx_sa(struct ixgbe_hw *hw, u16 idx, |
22 | u32 key[], u32 salt) |
23 | { |
24 | u32 reg; |
25 | int i; |
26 | |
27 | for (i = 0; i < 4; i++) |
28 | IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(i), |
29 | (__force u32)cpu_to_be32(key[3 - i])); |
30 | IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, (__force u32)cpu_to_be32(salt)); |
31 | IXGBE_WRITE_FLUSH(hw); |
32 | |
33 | reg = IXGBE_READ_REG(hw, IXGBE_IPSTXIDX); |
34 | reg &= IXGBE_RXTXIDX_IPS_EN; |
35 | reg |= idx << IXGBE_RXTXIDX_IDX_SHIFT | IXGBE_RXTXIDX_WRITE; |
36 | IXGBE_WRITE_REG(hw, IXGBE_IPSTXIDX, reg); |
37 | IXGBE_WRITE_FLUSH(hw); |
38 | } |
39 | |
40 | /** |
41 | * ixgbe_ipsec_set_rx_item - set an Rx table item |
42 | * @hw: hw specific details |
43 | * @idx: register index to write |
44 | * @tbl: table selector |
45 | * |
46 | * Trigger the device to store into a particular Rx table the |
47 | * data that has already been loaded into the input register |
48 | **/ |
49 | static void ixgbe_ipsec_set_rx_item(struct ixgbe_hw *hw, u16 idx, |
50 | enum ixgbe_ipsec_tbl_sel tbl) |
51 | { |
52 | u32 reg; |
53 | |
54 | reg = IXGBE_READ_REG(hw, IXGBE_IPSRXIDX); |
55 | reg &= IXGBE_RXTXIDX_IPS_EN; |
56 | reg |= tbl << IXGBE_RXIDX_TBL_SHIFT | |
57 | idx << IXGBE_RXTXIDX_IDX_SHIFT | |
58 | IXGBE_RXTXIDX_WRITE; |
59 | IXGBE_WRITE_REG(hw, IXGBE_IPSRXIDX, reg); |
60 | IXGBE_WRITE_FLUSH(hw); |
61 | } |
62 | |
63 | /** |
64 | * ixgbe_ipsec_set_rx_sa - set up the register bits to save SA info |
65 | * @hw: hw specific details |
66 | * @idx: register index to write |
67 | * @spi: security parameter index |
68 | * @key: key byte array |
69 | * @salt: salt bytes |
70 | * @mode: rx decrypt control bits |
71 | * @ip_idx: index into IP table for related IP address |
72 | **/ |
73 | static void ixgbe_ipsec_set_rx_sa(struct ixgbe_hw *hw, u16 idx, __be32 spi, |
74 | u32 key[], u32 salt, u32 mode, u32 ip_idx) |
75 | { |
76 | int i; |
77 | |
78 | /* store the SPI (in bigendian) and IPidx */ |
79 | IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, |
80 | (__force u32)cpu_to_le32((__force u32)spi)); |
81 | IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, ip_idx); |
82 | IXGBE_WRITE_FLUSH(hw); |
83 | |
84 | ixgbe_ipsec_set_rx_item(hw, idx, tbl: ips_rx_spi_tbl); |
85 | |
86 | /* store the key, salt, and mode */ |
87 | for (i = 0; i < 4; i++) |
88 | IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(i), |
89 | (__force u32)cpu_to_be32(key[3 - i])); |
90 | IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, (__force u32)cpu_to_be32(salt)); |
91 | IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, mode); |
92 | IXGBE_WRITE_FLUSH(hw); |
93 | |
94 | ixgbe_ipsec_set_rx_item(hw, idx, tbl: ips_rx_key_tbl); |
95 | } |
96 | |
97 | /** |
98 | * ixgbe_ipsec_set_rx_ip - set up the register bits to save SA IP addr info |
99 | * @hw: hw specific details |
100 | * @idx: register index to write |
101 | * @addr: IP address byte array |
102 | **/ |
103 | static void ixgbe_ipsec_set_rx_ip(struct ixgbe_hw *hw, u16 idx, __be32 addr[]) |
104 | { |
105 | int i; |
106 | |
107 | /* store the ip address */ |
108 | for (i = 0; i < 4; i++) |
109 | IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(i), |
110 | (__force u32)cpu_to_le32((__force u32)addr[i])); |
111 | IXGBE_WRITE_FLUSH(hw); |
112 | |
113 | ixgbe_ipsec_set_rx_item(hw, idx, tbl: ips_rx_ip_tbl); |
114 | } |
115 | |
116 | /** |
117 | * ixgbe_ipsec_clear_hw_tables - because some tables don't get cleared on reset |
118 | * @adapter: board private structure |
119 | **/ |
120 | static void ixgbe_ipsec_clear_hw_tables(struct ixgbe_adapter *adapter) |
121 | { |
122 | struct ixgbe_hw *hw = &adapter->hw; |
123 | u32 buf[4] = {0, 0, 0, 0}; |
124 | u16 idx; |
125 | |
126 | /* disable Rx and Tx SA lookup */ |
127 | IXGBE_WRITE_REG(hw, IXGBE_IPSRXIDX, 0); |
128 | IXGBE_WRITE_REG(hw, IXGBE_IPSTXIDX, 0); |
129 | |
130 | /* scrub the tables - split the loops for the max of the IP table */ |
131 | for (idx = 0; idx < IXGBE_IPSEC_MAX_RX_IP_COUNT; idx++) { |
132 | ixgbe_ipsec_set_tx_sa(hw, idx, key: buf, salt: 0); |
133 | ixgbe_ipsec_set_rx_sa(hw, idx, spi: 0, key: buf, salt: 0, mode: 0, ip_idx: 0); |
134 | ixgbe_ipsec_set_rx_ip(hw, idx, addr: (__be32 *)buf); |
135 | } |
136 | for (; idx < IXGBE_IPSEC_MAX_SA_COUNT; idx++) { |
137 | ixgbe_ipsec_set_tx_sa(hw, idx, key: buf, salt: 0); |
138 | ixgbe_ipsec_set_rx_sa(hw, idx, spi: 0, key: buf, salt: 0, mode: 0, ip_idx: 0); |
139 | } |
140 | } |
141 | |
142 | /** |
143 | * ixgbe_ipsec_stop_data |
144 | * @adapter: board private structure |
145 | **/ |
146 | static void ixgbe_ipsec_stop_data(struct ixgbe_adapter *adapter) |
147 | { |
148 | struct ixgbe_hw *hw = &adapter->hw; |
149 | bool link = adapter->link_up; |
150 | u32 t_rdy, r_rdy; |
151 | u32 limit; |
152 | u32 reg; |
153 | |
154 | /* halt data paths */ |
155 | reg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); |
156 | reg |= IXGBE_SECTXCTRL_TX_DIS; |
157 | IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, reg); |
158 | |
159 | reg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL); |
160 | reg |= IXGBE_SECRXCTRL_RX_DIS; |
161 | IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, reg); |
162 | |
163 | /* If both Tx and Rx are ready there are no packets |
164 | * that we need to flush so the loopback configuration |
165 | * below is not necessary. |
166 | */ |
167 | t_rdy = IXGBE_READ_REG(hw, IXGBE_SECTXSTAT) & |
168 | IXGBE_SECTXSTAT_SECTX_RDY; |
169 | r_rdy = IXGBE_READ_REG(hw, IXGBE_SECRXSTAT) & |
170 | IXGBE_SECRXSTAT_SECRX_RDY; |
171 | if (t_rdy && r_rdy) |
172 | return; |
173 | |
174 | /* If the tx fifo doesn't have link, but still has data, |
175 | * we can't clear the tx sec block. Set the MAC loopback |
176 | * before block clear |
177 | */ |
178 | if (!link) { |
179 | reg = IXGBE_READ_REG(hw, IXGBE_MACC); |
180 | reg |= IXGBE_MACC_FLU; |
181 | IXGBE_WRITE_REG(hw, IXGBE_MACC, reg); |
182 | |
183 | reg = IXGBE_READ_REG(hw, IXGBE_HLREG0); |
184 | reg |= IXGBE_HLREG0_LPBK; |
185 | IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg); |
186 | |
187 | IXGBE_WRITE_FLUSH(hw); |
188 | mdelay(3); |
189 | } |
190 | |
191 | /* wait for the paths to empty */ |
192 | limit = 20; |
193 | do { |
194 | mdelay(10); |
195 | t_rdy = IXGBE_READ_REG(hw, IXGBE_SECTXSTAT) & |
196 | IXGBE_SECTXSTAT_SECTX_RDY; |
197 | r_rdy = IXGBE_READ_REG(hw, IXGBE_SECRXSTAT) & |
198 | IXGBE_SECRXSTAT_SECRX_RDY; |
199 | } while (!(t_rdy && r_rdy) && limit--); |
200 | |
201 | /* undo loopback if we played with it earlier */ |
202 | if (!link) { |
203 | reg = IXGBE_READ_REG(hw, IXGBE_MACC); |
204 | reg &= ~IXGBE_MACC_FLU; |
205 | IXGBE_WRITE_REG(hw, IXGBE_MACC, reg); |
206 | |
207 | reg = IXGBE_READ_REG(hw, IXGBE_HLREG0); |
208 | reg &= ~IXGBE_HLREG0_LPBK; |
209 | IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg); |
210 | |
211 | IXGBE_WRITE_FLUSH(hw); |
212 | } |
213 | } |
214 | |
215 | /** |
216 | * ixgbe_ipsec_stop_engine |
217 | * @adapter: board private structure |
218 | **/ |
219 | static void ixgbe_ipsec_stop_engine(struct ixgbe_adapter *adapter) |
220 | { |
221 | struct ixgbe_hw *hw = &adapter->hw; |
222 | u32 reg; |
223 | |
224 | ixgbe_ipsec_stop_data(adapter); |
225 | |
226 | /* disable Rx and Tx SA lookup */ |
227 | IXGBE_WRITE_REG(hw, IXGBE_IPSTXIDX, 0); |
228 | IXGBE_WRITE_REG(hw, IXGBE_IPSRXIDX, 0); |
229 | |
230 | /* disable the Rx and Tx engines and full packet store-n-forward */ |
231 | reg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL); |
232 | reg |= IXGBE_SECTXCTRL_SECTX_DIS; |
233 | reg &= ~IXGBE_SECTXCTRL_STORE_FORWARD; |
234 | IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, reg); |
235 | |
236 | reg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL); |
237 | reg |= IXGBE_SECRXCTRL_SECRX_DIS; |
238 | IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, reg); |
239 | |
240 | /* restore the "tx security buffer almost full threshold" to 0x250 */ |
241 | IXGBE_WRITE_REG(hw, IXGBE_SECTXBUFFAF, 0x250); |
242 | |
243 | /* Set minimum IFG between packets back to the default 0x1 */ |
244 | reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG); |
245 | reg = (reg & 0xfffffff0) | 0x1; |
246 | IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg); |
247 | |
248 | /* final set for normal (no ipsec offload) processing */ |
249 | IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, IXGBE_SECTXCTRL_SECTX_DIS); |
250 | IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, IXGBE_SECRXCTRL_SECRX_DIS); |
251 | |
252 | IXGBE_WRITE_FLUSH(hw); |
253 | } |
254 | |
255 | /** |
256 | * ixgbe_ipsec_start_engine |
257 | * @adapter: board private structure |
258 | * |
259 | * NOTE: this increases power consumption whether being used or not |
260 | **/ |
261 | static void ixgbe_ipsec_start_engine(struct ixgbe_adapter *adapter) |
262 | { |
263 | struct ixgbe_hw *hw = &adapter->hw; |
264 | u32 reg; |
265 | |
266 | ixgbe_ipsec_stop_data(adapter); |
267 | |
268 | /* Set minimum IFG between packets to 3 */ |
269 | reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG); |
270 | reg = (reg & 0xfffffff0) | 0x3; |
271 | IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg); |
272 | |
273 | /* Set "tx security buffer almost full threshold" to 0x15 so that the |
274 | * almost full indication is generated only after buffer contains at |
275 | * least an entire jumbo packet. |
276 | */ |
277 | reg = IXGBE_READ_REG(hw, IXGBE_SECTXBUFFAF); |
278 | reg = (reg & 0xfffffc00) | 0x15; |
279 | IXGBE_WRITE_REG(hw, IXGBE_SECTXBUFFAF, reg); |
280 | |
281 | /* restart the data paths by clearing the DISABLE bits */ |
282 | IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, 0); |
283 | IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, IXGBE_SECTXCTRL_STORE_FORWARD); |
284 | |
285 | /* enable Rx and Tx SA lookup */ |
286 | IXGBE_WRITE_REG(hw, IXGBE_IPSTXIDX, IXGBE_RXTXIDX_IPS_EN); |
287 | IXGBE_WRITE_REG(hw, IXGBE_IPSRXIDX, IXGBE_RXTXIDX_IPS_EN); |
288 | |
289 | IXGBE_WRITE_FLUSH(hw); |
290 | } |
291 | |
292 | /** |
293 | * ixgbe_ipsec_restore - restore the ipsec HW settings after a reset |
294 | * @adapter: board private structure |
295 | * |
296 | * Reload the HW tables from the SW tables after they've been bashed |
297 | * by a chip reset. |
298 | * |
299 | * Any VF entries are removed from the SW and HW tables since either |
300 | * (a) the VF also gets reset on PF reset and will ask again for the |
301 | * offloads, or (b) the VF has been removed by a change in the num_vfs. |
302 | **/ |
303 | void ixgbe_ipsec_restore(struct ixgbe_adapter *adapter) |
304 | { |
305 | struct ixgbe_ipsec *ipsec = adapter->ipsec; |
306 | struct ixgbe_hw *hw = &adapter->hw; |
307 | int i; |
308 | |
309 | if (!(adapter->flags2 & IXGBE_FLAG2_IPSEC_ENABLED)) |
310 | return; |
311 | |
312 | /* clean up and restart the engine */ |
313 | ixgbe_ipsec_stop_engine(adapter); |
314 | ixgbe_ipsec_clear_hw_tables(adapter); |
315 | ixgbe_ipsec_start_engine(adapter); |
316 | |
317 | /* reload the Rx and Tx keys */ |
318 | for (i = 0; i < IXGBE_IPSEC_MAX_SA_COUNT; i++) { |
319 | struct rx_sa *r = &ipsec->rx_tbl[i]; |
320 | struct tx_sa *t = &ipsec->tx_tbl[i]; |
321 | |
322 | if (r->used) { |
323 | if (r->mode & IXGBE_RXTXMOD_VF) |
324 | ixgbe_ipsec_del_sa(xs: r->xs); |
325 | else |
326 | ixgbe_ipsec_set_rx_sa(hw, idx: i, spi: r->xs->id.spi, |
327 | key: r->key, salt: r->salt, |
328 | mode: r->mode, ip_idx: r->iptbl_ind); |
329 | } |
330 | |
331 | if (t->used) { |
332 | if (t->mode & IXGBE_RXTXMOD_VF) |
333 | ixgbe_ipsec_del_sa(xs: t->xs); |
334 | else |
335 | ixgbe_ipsec_set_tx_sa(hw, idx: i, key: t->key, salt: t->salt); |
336 | } |
337 | } |
338 | |
339 | /* reload the IP addrs */ |
340 | for (i = 0; i < IXGBE_IPSEC_MAX_RX_IP_COUNT; i++) { |
341 | struct rx_ip_sa *ipsa = &ipsec->ip_tbl[i]; |
342 | |
343 | if (ipsa->used) |
344 | ixgbe_ipsec_set_rx_ip(hw, idx: i, addr: ipsa->ipaddr); |
345 | } |
346 | } |
347 | |
348 | /** |
349 | * ixgbe_ipsec_find_empty_idx - find the first unused security parameter index |
350 | * @ipsec: pointer to ipsec struct |
351 | * @rxtable: true if we need to look in the Rx table |
352 | * |
353 | * Returns the first unused index in either the Rx or Tx SA table |
354 | **/ |
355 | static int ixgbe_ipsec_find_empty_idx(struct ixgbe_ipsec *ipsec, bool rxtable) |
356 | { |
357 | u32 i; |
358 | |
359 | if (rxtable) { |
360 | if (ipsec->num_rx_sa == IXGBE_IPSEC_MAX_SA_COUNT) |
361 | return -ENOSPC; |
362 | |
363 | /* search rx sa table */ |
364 | for (i = 0; i < IXGBE_IPSEC_MAX_SA_COUNT; i++) { |
365 | if (!ipsec->rx_tbl[i].used) |
366 | return i; |
367 | } |
368 | } else { |
369 | if (ipsec->num_tx_sa == IXGBE_IPSEC_MAX_SA_COUNT) |
370 | return -ENOSPC; |
371 | |
372 | /* search tx sa table */ |
373 | for (i = 0; i < IXGBE_IPSEC_MAX_SA_COUNT; i++) { |
374 | if (!ipsec->tx_tbl[i].used) |
375 | return i; |
376 | } |
377 | } |
378 | |
379 | return -ENOSPC; |
380 | } |
381 | |
382 | /** |
383 | * ixgbe_ipsec_find_rx_state - find the state that matches |
384 | * @ipsec: pointer to ipsec struct |
385 | * @daddr: inbound address to match |
386 | * @proto: protocol to match |
387 | * @spi: SPI to match |
388 | * @ip4: true if using an ipv4 address |
389 | * |
390 | * Returns a pointer to the matching SA state information |
391 | **/ |
392 | static struct xfrm_state *ixgbe_ipsec_find_rx_state(struct ixgbe_ipsec *ipsec, |
393 | __be32 *daddr, u8 proto, |
394 | __be32 spi, bool ip4) |
395 | { |
396 | struct rx_sa *rsa; |
397 | struct xfrm_state *ret = NULL; |
398 | |
399 | rcu_read_lock(); |
400 | hash_for_each_possible_rcu(ipsec->rx_sa_list, rsa, hlist, |
401 | (__force u32)spi) { |
402 | if (rsa->mode & IXGBE_RXTXMOD_VF) |
403 | continue; |
404 | if (spi == rsa->xs->id.spi && |
405 | ((ip4 && *daddr == rsa->xs->id.daddr.a4) || |
406 | (!ip4 && !memcmp(p: daddr, q: &rsa->xs->id.daddr.a6, |
407 | size: sizeof(rsa->xs->id.daddr.a6)))) && |
408 | proto == rsa->xs->id.proto) { |
409 | ret = rsa->xs; |
410 | xfrm_state_hold(x: ret); |
411 | break; |
412 | } |
413 | } |
414 | rcu_read_unlock(); |
415 | return ret; |
416 | } |
417 | |
418 | /** |
419 | * ixgbe_ipsec_parse_proto_keys - find the key and salt based on the protocol |
420 | * @xs: pointer to xfrm_state struct |
421 | * @mykey: pointer to key array to populate |
422 | * @mysalt: pointer to salt value to populate |
423 | * |
424 | * This copies the protocol keys and salt to our own data tables. The |
425 | * 82599 family only supports the one algorithm. |
426 | **/ |
427 | static int ixgbe_ipsec_parse_proto_keys(struct xfrm_state *xs, |
428 | u32 *mykey, u32 *mysalt) |
429 | { |
430 | struct net_device *dev = xs->xso.real_dev; |
431 | unsigned char *key_data; |
432 | char *alg_name = NULL; |
433 | int key_len; |
434 | |
435 | if (!xs->aead) { |
436 | netdev_err(dev, format: "Unsupported IPsec algorithm\n" ); |
437 | return -EINVAL; |
438 | } |
439 | |
440 | if (xs->aead->alg_icv_len != IXGBE_IPSEC_AUTH_BITS) { |
441 | netdev_err(dev, format: "IPsec offload requires %d bit authentication\n" , |
442 | IXGBE_IPSEC_AUTH_BITS); |
443 | return -EINVAL; |
444 | } |
445 | |
446 | key_data = &xs->aead->alg_key[0]; |
447 | key_len = xs->aead->alg_key_len; |
448 | alg_name = xs->aead->alg_name; |
449 | |
450 | if (strcmp(alg_name, aes_gcm_name)) { |
451 | netdev_err(dev, format: "Unsupported IPsec algorithm - please use %s\n" , |
452 | aes_gcm_name); |
453 | return -EINVAL; |
454 | } |
455 | |
456 | /* The key bytes come down in a bigendian array of bytes, so |
457 | * we don't need to do any byteswapping. |
458 | * 160 accounts for 16 byte key and 4 byte salt |
459 | */ |
460 | if (key_len == IXGBE_IPSEC_KEY_BITS) { |
461 | *mysalt = ((u32 *)key_data)[4]; |
462 | } else if (key_len != (IXGBE_IPSEC_KEY_BITS - (sizeof(*mysalt) * 8))) { |
463 | netdev_err(dev, format: "IPsec hw offload only supports keys up to 128 bits with a 32 bit salt\n" ); |
464 | return -EINVAL; |
465 | } else { |
466 | netdev_info(dev, format: "IPsec hw offload parameters missing 32 bit salt value\n" ); |
467 | *mysalt = 0; |
468 | } |
469 | memcpy(mykey, key_data, 16); |
470 | |
471 | return 0; |
472 | } |
473 | |
474 | /** |
475 | * ixgbe_ipsec_check_mgmt_ip - make sure there is no clash with mgmt IP filters |
476 | * @xs: pointer to transformer state struct |
477 | **/ |
478 | static int ixgbe_ipsec_check_mgmt_ip(struct xfrm_state *xs) |
479 | { |
480 | struct net_device *dev = xs->xso.real_dev; |
481 | struct ixgbe_adapter *adapter = netdev_priv(dev); |
482 | struct ixgbe_hw *hw = &adapter->hw; |
483 | u32 mfval, manc, reg; |
484 | int num_filters = 4; |
485 | bool manc_ipv4; |
486 | u32 bmcipval; |
487 | int i, j; |
488 | |
489 | #define MANC_EN_IPV4_FILTER BIT(24) |
490 | #define MFVAL_IPV4_FILTER_SHIFT 16 |
491 | #define MFVAL_IPV6_FILTER_SHIFT 24 |
492 | #define MIPAF_ARR(_m, _n) (IXGBE_MIPAF + ((_m) * 0x10) + ((_n) * 4)) |
493 | |
494 | #define IXGBE_BMCIP(_n) (0x5050 + ((_n) * 4)) |
495 | #define IXGBE_BMCIPVAL 0x5060 |
496 | #define BMCIP_V4 0x2 |
497 | #define BMCIP_V6 0x3 |
498 | #define BMCIP_MASK 0x3 |
499 | |
500 | manc = IXGBE_READ_REG(hw, IXGBE_MANC); |
501 | manc_ipv4 = !!(manc & MANC_EN_IPV4_FILTER); |
502 | mfval = IXGBE_READ_REG(hw, IXGBE_MFVAL); |
503 | bmcipval = IXGBE_READ_REG(hw, IXGBE_BMCIPVAL); |
504 | |
505 | if (xs->props.family == AF_INET) { |
506 | /* are there any IPv4 filters to check? */ |
507 | if (manc_ipv4) { |
508 | /* the 4 ipv4 filters are all in MIPAF(3, i) */ |
509 | for (i = 0; i < num_filters; i++) { |
510 | if (!(mfval & BIT(MFVAL_IPV4_FILTER_SHIFT + i))) |
511 | continue; |
512 | |
513 | reg = IXGBE_READ_REG(hw, MIPAF_ARR(3, i)); |
514 | if (reg == (__force u32)xs->id.daddr.a4) |
515 | return 1; |
516 | } |
517 | } |
518 | |
519 | if ((bmcipval & BMCIP_MASK) == BMCIP_V4) { |
520 | reg = IXGBE_READ_REG(hw, IXGBE_BMCIP(3)); |
521 | if (reg == (__force u32)xs->id.daddr.a4) |
522 | return 1; |
523 | } |
524 | |
525 | } else { |
526 | /* if there are ipv4 filters, they are in the last ipv6 slot */ |
527 | if (manc_ipv4) |
528 | num_filters = 3; |
529 | |
530 | for (i = 0; i < num_filters; i++) { |
531 | if (!(mfval & BIT(MFVAL_IPV6_FILTER_SHIFT + i))) |
532 | continue; |
533 | |
534 | for (j = 0; j < 4; j++) { |
535 | reg = IXGBE_READ_REG(hw, MIPAF_ARR(i, j)); |
536 | if (reg != (__force u32)xs->id.daddr.a6[j]) |
537 | break; |
538 | } |
539 | if (j == 4) /* did we match all 4 words? */ |
540 | return 1; |
541 | } |
542 | |
543 | if ((bmcipval & BMCIP_MASK) == BMCIP_V6) { |
544 | for (j = 0; j < 4; j++) { |
545 | reg = IXGBE_READ_REG(hw, IXGBE_BMCIP(j)); |
546 | if (reg != (__force u32)xs->id.daddr.a6[j]) |
547 | break; |
548 | } |
549 | if (j == 4) /* did we match all 4 words? */ |
550 | return 1; |
551 | } |
552 | } |
553 | |
554 | return 0; |
555 | } |
556 | |
557 | /** |
558 | * ixgbe_ipsec_add_sa - program device with a security association |
559 | * @xs: pointer to transformer state struct |
560 | * @extack: extack point to fill failure reason |
561 | **/ |
562 | static int ixgbe_ipsec_add_sa(struct xfrm_state *xs, |
563 | struct netlink_ext_ack *extack) |
564 | { |
565 | struct net_device *dev = xs->xso.real_dev; |
566 | struct ixgbe_adapter *adapter = netdev_priv(dev); |
567 | struct ixgbe_ipsec *ipsec = adapter->ipsec; |
568 | struct ixgbe_hw *hw = &adapter->hw; |
569 | int checked, match, first; |
570 | u16 sa_idx; |
571 | int ret; |
572 | int i; |
573 | |
574 | if (xs->id.proto != IPPROTO_ESP && xs->id.proto != IPPROTO_AH) { |
575 | NL_SET_ERR_MSG_MOD(extack, "Unsupported protocol for ipsec offload" ); |
576 | return -EINVAL; |
577 | } |
578 | |
579 | if (xs->props.mode != XFRM_MODE_TRANSPORT) { |
580 | NL_SET_ERR_MSG_MOD(extack, "Unsupported mode for ipsec offload" ); |
581 | return -EINVAL; |
582 | } |
583 | |
584 | if (ixgbe_ipsec_check_mgmt_ip(xs)) { |
585 | NL_SET_ERR_MSG_MOD(extack, "IPsec IP addr clash with mgmt filters" ); |
586 | return -EINVAL; |
587 | } |
588 | |
589 | if (xs->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { |
590 | NL_SET_ERR_MSG_MOD(extack, "Unsupported ipsec offload type" ); |
591 | return -EINVAL; |
592 | } |
593 | |
594 | if (xs->xso.dir == XFRM_DEV_OFFLOAD_IN) { |
595 | struct rx_sa rsa; |
596 | |
597 | if (xs->calg) { |
598 | NL_SET_ERR_MSG_MOD(extack, "Compression offload not supported" ); |
599 | return -EINVAL; |
600 | } |
601 | |
602 | /* find the first unused index */ |
603 | ret = ixgbe_ipsec_find_empty_idx(ipsec, rxtable: true); |
604 | if (ret < 0) { |
605 | NL_SET_ERR_MSG_MOD(extack, "No space for SA in Rx table!" ); |
606 | return ret; |
607 | } |
608 | sa_idx = (u16)ret; |
609 | |
610 | memset(&rsa, 0, sizeof(rsa)); |
611 | rsa.used = true; |
612 | rsa.xs = xs; |
613 | |
614 | if (rsa.xs->id.proto & IPPROTO_ESP) |
615 | rsa.decrypt = xs->ealg || xs->aead; |
616 | |
617 | /* get the key and salt */ |
618 | ret = ixgbe_ipsec_parse_proto_keys(xs, mykey: rsa.key, mysalt: &rsa.salt); |
619 | if (ret) { |
620 | NL_SET_ERR_MSG_MOD(extack, "Failed to get key data for Rx SA table" ); |
621 | return ret; |
622 | } |
623 | |
624 | /* get ip for rx sa table */ |
625 | if (xs->props.family == AF_INET6) |
626 | memcpy(rsa.ipaddr, &xs->id.daddr.a6, 16); |
627 | else |
628 | memcpy(&rsa.ipaddr[3], &xs->id.daddr.a4, 4); |
629 | |
630 | /* The HW does not have a 1:1 mapping from keys to IP addrs, so |
631 | * check for a matching IP addr entry in the table. If the addr |
632 | * already exists, use it; else find an unused slot and add the |
633 | * addr. If one does not exist and there are no unused table |
634 | * entries, fail the request. |
635 | */ |
636 | |
637 | /* Find an existing match or first not used, and stop looking |
638 | * after we've checked all we know we have. |
639 | */ |
640 | checked = 0; |
641 | match = -1; |
642 | first = -1; |
643 | for (i = 0; |
644 | i < IXGBE_IPSEC_MAX_RX_IP_COUNT && |
645 | (checked < ipsec->num_rx_sa || first < 0); |
646 | i++) { |
647 | if (ipsec->ip_tbl[i].used) { |
648 | if (!memcmp(p: ipsec->ip_tbl[i].ipaddr, |
649 | q: rsa.ipaddr, size: sizeof(rsa.ipaddr))) { |
650 | match = i; |
651 | break; |
652 | } |
653 | checked++; |
654 | } else if (first < 0) { |
655 | first = i; /* track the first empty seen */ |
656 | } |
657 | } |
658 | |
659 | if (ipsec->num_rx_sa == 0) |
660 | first = 0; |
661 | |
662 | if (match >= 0) { |
663 | /* addrs are the same, we should use this one */ |
664 | rsa.iptbl_ind = match; |
665 | ipsec->ip_tbl[match].ref_cnt++; |
666 | |
667 | } else if (first >= 0) { |
668 | /* no matches, but here's an empty slot */ |
669 | rsa.iptbl_ind = first; |
670 | |
671 | memcpy(ipsec->ip_tbl[first].ipaddr, |
672 | rsa.ipaddr, sizeof(rsa.ipaddr)); |
673 | ipsec->ip_tbl[first].ref_cnt = 1; |
674 | ipsec->ip_tbl[first].used = true; |
675 | |
676 | ixgbe_ipsec_set_rx_ip(hw, idx: rsa.iptbl_ind, addr: rsa.ipaddr); |
677 | |
678 | } else { |
679 | /* no match and no empty slot */ |
680 | NL_SET_ERR_MSG_MOD(extack, "No space for SA in Rx IP SA table" ); |
681 | memset(&rsa, 0, sizeof(rsa)); |
682 | return -ENOSPC; |
683 | } |
684 | |
685 | rsa.mode = IXGBE_RXMOD_VALID; |
686 | if (rsa.xs->id.proto & IPPROTO_ESP) |
687 | rsa.mode |= IXGBE_RXMOD_PROTO_ESP; |
688 | if (rsa.decrypt) |
689 | rsa.mode |= IXGBE_RXMOD_DECRYPT; |
690 | if (rsa.xs->props.family == AF_INET6) |
691 | rsa.mode |= IXGBE_RXMOD_IPV6; |
692 | |
693 | /* the preparations worked, so save the info */ |
694 | memcpy(&ipsec->rx_tbl[sa_idx], &rsa, sizeof(rsa)); |
695 | |
696 | ixgbe_ipsec_set_rx_sa(hw, idx: sa_idx, spi: rsa.xs->id.spi, key: rsa.key, |
697 | salt: rsa.salt, mode: rsa.mode, ip_idx: rsa.iptbl_ind); |
698 | xs->xso.offload_handle = sa_idx + IXGBE_IPSEC_BASE_RX_INDEX; |
699 | |
700 | ipsec->num_rx_sa++; |
701 | |
702 | /* hash the new entry for faster search in Rx path */ |
703 | hash_add_rcu(ipsec->rx_sa_list, &ipsec->rx_tbl[sa_idx].hlist, |
704 | (__force u32)rsa.xs->id.spi); |
705 | } else { |
706 | struct tx_sa tsa; |
707 | |
708 | if (adapter->num_vfs && |
709 | adapter->bridge_mode != BRIDGE_MODE_VEPA) |
710 | return -EOPNOTSUPP; |
711 | |
712 | /* find the first unused index */ |
713 | ret = ixgbe_ipsec_find_empty_idx(ipsec, rxtable: false); |
714 | if (ret < 0) { |
715 | NL_SET_ERR_MSG_MOD(extack, "No space for SA in Tx table" ); |
716 | return ret; |
717 | } |
718 | sa_idx = (u16)ret; |
719 | |
720 | memset(&tsa, 0, sizeof(tsa)); |
721 | tsa.used = true; |
722 | tsa.xs = xs; |
723 | |
724 | if (xs->id.proto & IPPROTO_ESP) |
725 | tsa.encrypt = xs->ealg || xs->aead; |
726 | |
727 | ret = ixgbe_ipsec_parse_proto_keys(xs, mykey: tsa.key, mysalt: &tsa.salt); |
728 | if (ret) { |
729 | NL_SET_ERR_MSG_MOD(extack, "Failed to get key data for Tx SA table" ); |
730 | memset(&tsa, 0, sizeof(tsa)); |
731 | return ret; |
732 | } |
733 | |
734 | /* the preparations worked, so save the info */ |
735 | memcpy(&ipsec->tx_tbl[sa_idx], &tsa, sizeof(tsa)); |
736 | |
737 | ixgbe_ipsec_set_tx_sa(hw, idx: sa_idx, key: tsa.key, salt: tsa.salt); |
738 | |
739 | xs->xso.offload_handle = sa_idx + IXGBE_IPSEC_BASE_TX_INDEX; |
740 | |
741 | ipsec->num_tx_sa++; |
742 | } |
743 | |
744 | /* enable the engine if not already warmed up */ |
745 | if (!(adapter->flags2 & IXGBE_FLAG2_IPSEC_ENABLED)) { |
746 | ixgbe_ipsec_start_engine(adapter); |
747 | adapter->flags2 |= IXGBE_FLAG2_IPSEC_ENABLED; |
748 | } |
749 | |
750 | return 0; |
751 | } |
752 | |
753 | /** |
754 | * ixgbe_ipsec_del_sa - clear out this specific SA |
755 | * @xs: pointer to transformer state struct |
756 | **/ |
757 | static void ixgbe_ipsec_del_sa(struct xfrm_state *xs) |
758 | { |
759 | struct net_device *dev = xs->xso.real_dev; |
760 | struct ixgbe_adapter *adapter = netdev_priv(dev); |
761 | struct ixgbe_ipsec *ipsec = adapter->ipsec; |
762 | struct ixgbe_hw *hw = &adapter->hw; |
763 | u32 zerobuf[4] = {0, 0, 0, 0}; |
764 | u16 sa_idx; |
765 | |
766 | if (xs->xso.dir == XFRM_DEV_OFFLOAD_IN) { |
767 | struct rx_sa *rsa; |
768 | u8 ipi; |
769 | |
770 | sa_idx = xs->xso.offload_handle - IXGBE_IPSEC_BASE_RX_INDEX; |
771 | rsa = &ipsec->rx_tbl[sa_idx]; |
772 | |
773 | if (!rsa->used) { |
774 | netdev_err(dev, format: "Invalid Rx SA selected sa_idx=%d offload_handle=%lu\n" , |
775 | sa_idx, xs->xso.offload_handle); |
776 | return; |
777 | } |
778 | |
779 | ixgbe_ipsec_set_rx_sa(hw, idx: sa_idx, spi: 0, key: zerobuf, salt: 0, mode: 0, ip_idx: 0); |
780 | hash_del_rcu(node: &rsa->hlist); |
781 | |
782 | /* if the IP table entry is referenced by only this SA, |
783 | * i.e. ref_cnt is only 1, clear the IP table entry as well |
784 | */ |
785 | ipi = rsa->iptbl_ind; |
786 | if (ipsec->ip_tbl[ipi].ref_cnt > 0) { |
787 | ipsec->ip_tbl[ipi].ref_cnt--; |
788 | |
789 | if (!ipsec->ip_tbl[ipi].ref_cnt) { |
790 | memset(&ipsec->ip_tbl[ipi], 0, |
791 | sizeof(struct rx_ip_sa)); |
792 | ixgbe_ipsec_set_rx_ip(hw, idx: ipi, |
793 | addr: (__force __be32 *)zerobuf); |
794 | } |
795 | } |
796 | |
797 | memset(rsa, 0, sizeof(struct rx_sa)); |
798 | ipsec->num_rx_sa--; |
799 | } else { |
800 | sa_idx = xs->xso.offload_handle - IXGBE_IPSEC_BASE_TX_INDEX; |
801 | |
802 | if (!ipsec->tx_tbl[sa_idx].used) { |
803 | netdev_err(dev, format: "Invalid Tx SA selected sa_idx=%d offload_handle=%lu\n" , |
804 | sa_idx, xs->xso.offload_handle); |
805 | return; |
806 | } |
807 | |
808 | ixgbe_ipsec_set_tx_sa(hw, idx: sa_idx, key: zerobuf, salt: 0); |
809 | memset(&ipsec->tx_tbl[sa_idx], 0, sizeof(struct tx_sa)); |
810 | ipsec->num_tx_sa--; |
811 | } |
812 | |
813 | /* if there are no SAs left, stop the engine to save energy */ |
814 | if (ipsec->num_rx_sa == 0 && ipsec->num_tx_sa == 0) { |
815 | adapter->flags2 &= ~IXGBE_FLAG2_IPSEC_ENABLED; |
816 | ixgbe_ipsec_stop_engine(adapter); |
817 | } |
818 | } |
819 | |
820 | /** |
821 | * ixgbe_ipsec_offload_ok - can this packet use the xfrm hw offload |
822 | * @skb: current data packet |
823 | * @xs: pointer to transformer state struct |
824 | **/ |
825 | static bool ixgbe_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *xs) |
826 | { |
827 | if (xs->props.family == AF_INET) { |
828 | /* Offload with IPv4 options is not supported yet */ |
829 | if (ip_hdr(skb)->ihl != 5) |
830 | return false; |
831 | } else { |
832 | /* Offload with IPv6 extension headers is not support yet */ |
833 | if (ipv6_ext_hdr(nexthdr: ipv6_hdr(skb)->nexthdr)) |
834 | return false; |
835 | } |
836 | |
837 | return true; |
838 | } |
839 | |
840 | static const struct xfrmdev_ops ixgbe_xfrmdev_ops = { |
841 | .xdo_dev_state_add = ixgbe_ipsec_add_sa, |
842 | .xdo_dev_state_delete = ixgbe_ipsec_del_sa, |
843 | .xdo_dev_offload_ok = ixgbe_ipsec_offload_ok, |
844 | }; |
845 | |
846 | /** |
847 | * ixgbe_ipsec_vf_clear - clear the tables of data for a VF |
848 | * @adapter: board private structure |
849 | * @vf: VF id to be removed |
850 | **/ |
851 | void ixgbe_ipsec_vf_clear(struct ixgbe_adapter *adapter, u32 vf) |
852 | { |
853 | struct ixgbe_ipsec *ipsec = adapter->ipsec; |
854 | int i; |
855 | |
856 | if (!ipsec) |
857 | return; |
858 | |
859 | /* search rx sa table */ |
860 | for (i = 0; i < IXGBE_IPSEC_MAX_SA_COUNT && ipsec->num_rx_sa; i++) { |
861 | if (!ipsec->rx_tbl[i].used) |
862 | continue; |
863 | if (ipsec->rx_tbl[i].mode & IXGBE_RXTXMOD_VF && |
864 | ipsec->rx_tbl[i].vf == vf) |
865 | ixgbe_ipsec_del_sa(xs: ipsec->rx_tbl[i].xs); |
866 | } |
867 | |
868 | /* search tx sa table */ |
869 | for (i = 0; i < IXGBE_IPSEC_MAX_SA_COUNT && ipsec->num_tx_sa; i++) { |
870 | if (!ipsec->tx_tbl[i].used) |
871 | continue; |
872 | if (ipsec->tx_tbl[i].mode & IXGBE_RXTXMOD_VF && |
873 | ipsec->tx_tbl[i].vf == vf) |
874 | ixgbe_ipsec_del_sa(xs: ipsec->tx_tbl[i].xs); |
875 | } |
876 | } |
877 | |
878 | /** |
879 | * ixgbe_ipsec_vf_add_sa - translate VF request to SA add |
880 | * @adapter: board private structure |
881 | * @msgbuf: The message buffer |
882 | * @vf: the VF index |
883 | * |
884 | * Make up a new xs and algorithm info from the data sent by the VF. |
885 | * We only need to sketch in just enough to set up the HW offload. |
886 | * Put the resulting offload_handle into the return message to the VF. |
887 | * |
888 | * Returns 0 or error value |
889 | **/ |
890 | int ixgbe_ipsec_vf_add_sa(struct ixgbe_adapter *adapter, u32 *msgbuf, u32 vf) |
891 | { |
892 | struct ixgbe_ipsec *ipsec = adapter->ipsec; |
893 | struct xfrm_algo_desc *algo; |
894 | struct sa_mbx_msg *sam; |
895 | struct xfrm_state *xs; |
896 | size_t aead_len; |
897 | u16 sa_idx; |
898 | u32 pfsa; |
899 | int err; |
900 | |
901 | sam = (struct sa_mbx_msg *)(&msgbuf[1]); |
902 | if (!adapter->vfinfo[vf].trusted || |
903 | !(adapter->flags2 & IXGBE_FLAG2_VF_IPSEC_ENABLED)) { |
904 | e_warn(drv, "VF %d attempted to add an IPsec SA\n" , vf); |
905 | err = -EACCES; |
906 | goto err_out; |
907 | } |
908 | |
909 | /* Tx IPsec offload doesn't seem to work on this |
910 | * device, so block these requests for now. |
911 | */ |
912 | if (sam->dir != XFRM_DEV_OFFLOAD_IN) { |
913 | err = -EOPNOTSUPP; |
914 | goto err_out; |
915 | } |
916 | |
917 | xs = kzalloc(size: sizeof(*xs), GFP_KERNEL); |
918 | if (unlikely(!xs)) { |
919 | err = -ENOMEM; |
920 | goto err_out; |
921 | } |
922 | |
923 | xs->xso.dir = sam->dir; |
924 | xs->id.spi = sam->spi; |
925 | xs->id.proto = sam->proto; |
926 | xs->props.family = sam->family; |
927 | if (xs->props.family == AF_INET6) |
928 | memcpy(&xs->id.daddr.a6, sam->addr, sizeof(xs->id.daddr.a6)); |
929 | else |
930 | memcpy(&xs->id.daddr.a4, sam->addr, sizeof(xs->id.daddr.a4)); |
931 | xs->xso.dev = adapter->netdev; |
932 | |
933 | algo = xfrm_aead_get_byname(name: aes_gcm_name, IXGBE_IPSEC_AUTH_BITS, probe: 1); |
934 | if (unlikely(!algo)) { |
935 | err = -ENOENT; |
936 | goto err_xs; |
937 | } |
938 | |
939 | aead_len = sizeof(*xs->aead) + IXGBE_IPSEC_KEY_BITS / 8; |
940 | xs->aead = kzalloc(size: aead_len, GFP_KERNEL); |
941 | if (unlikely(!xs->aead)) { |
942 | err = -ENOMEM; |
943 | goto err_xs; |
944 | } |
945 | |
946 | xs->props.ealgo = algo->desc.sadb_alg_id; |
947 | xs->geniv = algo->uinfo.aead.geniv; |
948 | xs->aead->alg_icv_len = IXGBE_IPSEC_AUTH_BITS; |
949 | xs->aead->alg_key_len = IXGBE_IPSEC_KEY_BITS; |
950 | memcpy(xs->aead->alg_key, sam->key, sizeof(sam->key)); |
951 | memcpy(xs->aead->alg_name, aes_gcm_name, sizeof(aes_gcm_name)); |
952 | |
953 | /* set up the HW offload */ |
954 | err = ixgbe_ipsec_add_sa(xs, NULL); |
955 | if (err) |
956 | goto err_aead; |
957 | |
958 | pfsa = xs->xso.offload_handle; |
959 | if (pfsa < IXGBE_IPSEC_BASE_TX_INDEX) { |
960 | sa_idx = pfsa - IXGBE_IPSEC_BASE_RX_INDEX; |
961 | ipsec->rx_tbl[sa_idx].vf = vf; |
962 | ipsec->rx_tbl[sa_idx].mode |= IXGBE_RXTXMOD_VF; |
963 | } else { |
964 | sa_idx = pfsa - IXGBE_IPSEC_BASE_TX_INDEX; |
965 | ipsec->tx_tbl[sa_idx].vf = vf; |
966 | ipsec->tx_tbl[sa_idx].mode |= IXGBE_RXTXMOD_VF; |
967 | } |
968 | |
969 | msgbuf[1] = xs->xso.offload_handle; |
970 | |
971 | return 0; |
972 | |
973 | err_aead: |
974 | kfree_sensitive(objp: xs->aead); |
975 | err_xs: |
976 | kfree_sensitive(objp: xs); |
977 | err_out: |
978 | msgbuf[1] = err; |
979 | return err; |
980 | } |
981 | |
982 | /** |
983 | * ixgbe_ipsec_vf_del_sa - translate VF request to SA delete |
984 | * @adapter: board private structure |
985 | * @msgbuf: The message buffer |
986 | * @vf: the VF index |
987 | * |
988 | * Given the offload_handle sent by the VF, look for the related SA table |
989 | * entry and use its xs field to call for a delete of the SA. |
990 | * |
991 | * Note: We silently ignore requests to delete entries that are already |
992 | * set to unused because when a VF is set to "DOWN", the PF first |
993 | * gets a reset and clears all the VF's entries; then the VF's |
994 | * XFRM stack sends individual deletes for each entry, which the |
995 | * reset already removed. In the future it might be good to try to |
996 | * optimize this so not so many unnecessary delete messages are sent. |
997 | * |
998 | * Returns 0 or error value |
999 | **/ |
1000 | int ixgbe_ipsec_vf_del_sa(struct ixgbe_adapter *adapter, u32 *msgbuf, u32 vf) |
1001 | { |
1002 | struct ixgbe_ipsec *ipsec = adapter->ipsec; |
1003 | struct xfrm_state *xs; |
1004 | u32 pfsa = msgbuf[1]; |
1005 | u16 sa_idx; |
1006 | |
1007 | if (!adapter->vfinfo[vf].trusted) { |
1008 | e_err(drv, "vf %d attempted to delete an SA\n" , vf); |
1009 | return -EPERM; |
1010 | } |
1011 | |
1012 | if (pfsa < IXGBE_IPSEC_BASE_TX_INDEX) { |
1013 | struct rx_sa *rsa; |
1014 | |
1015 | sa_idx = pfsa - IXGBE_IPSEC_BASE_RX_INDEX; |
1016 | if (sa_idx >= IXGBE_IPSEC_MAX_SA_COUNT) { |
1017 | e_err(drv, "vf %d SA index %d out of range\n" , |
1018 | vf, sa_idx); |
1019 | return -EINVAL; |
1020 | } |
1021 | |
1022 | rsa = &ipsec->rx_tbl[sa_idx]; |
1023 | |
1024 | if (!rsa->used) |
1025 | return 0; |
1026 | |
1027 | if (!(rsa->mode & IXGBE_RXTXMOD_VF) || |
1028 | rsa->vf != vf) { |
1029 | e_err(drv, "vf %d bad Rx SA index %d\n" , vf, sa_idx); |
1030 | return -ENOENT; |
1031 | } |
1032 | |
1033 | xs = ipsec->rx_tbl[sa_idx].xs; |
1034 | } else { |
1035 | struct tx_sa *tsa; |
1036 | |
1037 | sa_idx = pfsa - IXGBE_IPSEC_BASE_TX_INDEX; |
1038 | if (sa_idx >= IXGBE_IPSEC_MAX_SA_COUNT) { |
1039 | e_err(drv, "vf %d SA index %d out of range\n" , |
1040 | vf, sa_idx); |
1041 | return -EINVAL; |
1042 | } |
1043 | |
1044 | tsa = &ipsec->tx_tbl[sa_idx]; |
1045 | |
1046 | if (!tsa->used) |
1047 | return 0; |
1048 | |
1049 | if (!(tsa->mode & IXGBE_RXTXMOD_VF) || |
1050 | tsa->vf != vf) { |
1051 | e_err(drv, "vf %d bad Tx SA index %d\n" , vf, sa_idx); |
1052 | return -ENOENT; |
1053 | } |
1054 | |
1055 | xs = ipsec->tx_tbl[sa_idx].xs; |
1056 | } |
1057 | |
1058 | ixgbe_ipsec_del_sa(xs); |
1059 | |
1060 | /* remove the xs that was made-up in the add request */ |
1061 | kfree_sensitive(objp: xs); |
1062 | |
1063 | return 0; |
1064 | } |
1065 | |
1066 | /** |
1067 | * ixgbe_ipsec_tx - setup Tx flags for ipsec offload |
1068 | * @tx_ring: outgoing context |
1069 | * @first: current data packet |
1070 | * @itd: ipsec Tx data for later use in building context descriptor |
1071 | **/ |
1072 | int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring, |
1073 | struct ixgbe_tx_buffer *first, |
1074 | struct ixgbe_ipsec_tx_data *itd) |
1075 | { |
1076 | struct ixgbe_adapter *adapter = netdev_priv(dev: tx_ring->netdev); |
1077 | struct ixgbe_ipsec *ipsec = adapter->ipsec; |
1078 | struct xfrm_state *xs; |
1079 | struct sec_path *sp; |
1080 | struct tx_sa *tsa; |
1081 | |
1082 | sp = skb_sec_path(skb: first->skb); |
1083 | if (unlikely(!sp->len)) { |
1084 | netdev_err(dev: tx_ring->netdev, format: "%s: no xfrm state len = %d\n" , |
1085 | __func__, sp->len); |
1086 | return 0; |
1087 | } |
1088 | |
1089 | xs = xfrm_input_state(skb: first->skb); |
1090 | if (unlikely(!xs)) { |
1091 | netdev_err(dev: tx_ring->netdev, format: "%s: no xfrm_input_state() xs = %p\n" , |
1092 | __func__, xs); |
1093 | return 0; |
1094 | } |
1095 | |
1096 | itd->sa_idx = xs->xso.offload_handle - IXGBE_IPSEC_BASE_TX_INDEX; |
1097 | if (unlikely(itd->sa_idx >= IXGBE_IPSEC_MAX_SA_COUNT)) { |
1098 | netdev_err(dev: tx_ring->netdev, format: "%s: bad sa_idx=%d handle=%lu\n" , |
1099 | __func__, itd->sa_idx, xs->xso.offload_handle); |
1100 | return 0; |
1101 | } |
1102 | |
1103 | tsa = &ipsec->tx_tbl[itd->sa_idx]; |
1104 | if (unlikely(!tsa->used)) { |
1105 | netdev_err(dev: tx_ring->netdev, format: "%s: unused sa_idx=%d\n" , |
1106 | __func__, itd->sa_idx); |
1107 | return 0; |
1108 | } |
1109 | |
1110 | first->tx_flags |= IXGBE_TX_FLAGS_IPSEC | IXGBE_TX_FLAGS_CC; |
1111 | |
1112 | if (xs->id.proto == IPPROTO_ESP) { |
1113 | |
1114 | itd->flags |= IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP | |
1115 | IXGBE_ADVTXD_TUCMD_L4T_TCP; |
1116 | if (first->protocol == htons(ETH_P_IP)) |
1117 | itd->flags |= IXGBE_ADVTXD_TUCMD_IPV4; |
1118 | |
1119 | /* The actual trailer length is authlen (16 bytes) plus |
1120 | * 2 bytes for the proto and the padlen values, plus |
1121 | * padlen bytes of padding. This ends up not the same |
1122 | * as the static value found in xs->props.trailer_len (21). |
1123 | * |
1124 | * ... but if we're doing GSO, don't bother as the stack |
1125 | * doesn't add a trailer for those. |
1126 | */ |
1127 | if (!skb_is_gso(skb: first->skb)) { |
1128 | /* The "correct" way to get the auth length would be |
1129 | * to use |
1130 | * authlen = crypto_aead_authsize(xs->data); |
1131 | * but since we know we only have one size to worry |
1132 | * about * we can let the compiler use the constant |
1133 | * and save us a few CPU cycles. |
1134 | */ |
1135 | const int authlen = IXGBE_IPSEC_AUTH_BITS / 8; |
1136 | struct sk_buff *skb = first->skb; |
1137 | u8 padlen; |
1138 | int ret; |
1139 | |
1140 | ret = skb_copy_bits(skb, offset: skb->len - (authlen + 2), |
1141 | to: &padlen, len: 1); |
1142 | if (unlikely(ret)) |
1143 | return 0; |
1144 | itd->trailer_len = authlen + 2 + padlen; |
1145 | } |
1146 | } |
1147 | if (tsa->encrypt) |
1148 | itd->flags |= IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN; |
1149 | |
1150 | return 1; |
1151 | } |
1152 | |
1153 | /** |
1154 | * ixgbe_ipsec_rx - decode ipsec bits from Rx descriptor |
1155 | * @rx_ring: receiving ring |
1156 | * @rx_desc: receive data descriptor |
1157 | * @skb: current data packet |
1158 | * |
1159 | * Determine if there was an ipsec encapsulation noticed, and if so set up |
1160 | * the resulting status for later in the receive stack. |
1161 | **/ |
1162 | void ixgbe_ipsec_rx(struct ixgbe_ring *rx_ring, |
1163 | union ixgbe_adv_rx_desc *rx_desc, |
1164 | struct sk_buff *skb) |
1165 | { |
1166 | struct ixgbe_adapter *adapter = netdev_priv(dev: rx_ring->netdev); |
1167 | __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; |
1168 | __le16 ipsec_pkt_types = cpu_to_le16(IXGBE_RXDADV_PKTTYPE_IPSEC_AH | |
1169 | IXGBE_RXDADV_PKTTYPE_IPSEC_ESP); |
1170 | struct ixgbe_ipsec *ipsec = adapter->ipsec; |
1171 | struct xfrm_offload *xo = NULL; |
1172 | struct xfrm_state *xs = NULL; |
1173 | struct ipv6hdr *ip6 = NULL; |
1174 | struct iphdr *ip4 = NULL; |
1175 | struct sec_path *sp; |
1176 | void *daddr; |
1177 | __be32 spi; |
1178 | u8 *c_hdr; |
1179 | u8 proto; |
1180 | |
1181 | /* Find the ip and crypto headers in the data. |
1182 | * We can assume no vlan header in the way, b/c the |
1183 | * hw won't recognize the IPsec packet and anyway the |
1184 | * currently vlan device doesn't support xfrm offload. |
1185 | */ |
1186 | if (pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_IPV4)) { |
1187 | ip4 = (struct iphdr *)(skb->data + ETH_HLEN); |
1188 | daddr = &ip4->daddr; |
1189 | c_hdr = (u8 *)ip4 + ip4->ihl * 4; |
1190 | } else if (pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_IPV6)) { |
1191 | ip6 = (struct ipv6hdr *)(skb->data + ETH_HLEN); |
1192 | daddr = &ip6->daddr; |
1193 | c_hdr = (u8 *)ip6 + sizeof(struct ipv6hdr); |
1194 | } else { |
1195 | return; |
1196 | } |
1197 | |
1198 | switch (pkt_info & ipsec_pkt_types) { |
1199 | case cpu_to_le16(IXGBE_RXDADV_PKTTYPE_IPSEC_AH): |
1200 | spi = ((struct ip_auth_hdr *)c_hdr)->spi; |
1201 | proto = IPPROTO_AH; |
1202 | break; |
1203 | case cpu_to_le16(IXGBE_RXDADV_PKTTYPE_IPSEC_ESP): |
1204 | spi = ((struct ip_esp_hdr *)c_hdr)->spi; |
1205 | proto = IPPROTO_ESP; |
1206 | break; |
1207 | default: |
1208 | return; |
1209 | } |
1210 | |
1211 | xs = ixgbe_ipsec_find_rx_state(ipsec, daddr, proto, spi, ip4: !!ip4); |
1212 | if (unlikely(!xs)) |
1213 | return; |
1214 | |
1215 | sp = secpath_set(skb); |
1216 | if (unlikely(!sp)) |
1217 | return; |
1218 | |
1219 | sp->xvec[sp->len++] = xs; |
1220 | sp->olen++; |
1221 | xo = xfrm_offload(skb); |
1222 | xo->flags = CRYPTO_DONE; |
1223 | xo->status = CRYPTO_SUCCESS; |
1224 | |
1225 | adapter->rx_ipsec++; |
1226 | } |
1227 | |
1228 | /** |
1229 | * ixgbe_init_ipsec_offload - initialize security registers for IPSec operation |
1230 | * @adapter: board private structure |
1231 | **/ |
1232 | void ixgbe_init_ipsec_offload(struct ixgbe_adapter *adapter) |
1233 | { |
1234 | struct ixgbe_hw *hw = &adapter->hw; |
1235 | struct ixgbe_ipsec *ipsec; |
1236 | u32 t_dis, r_dis; |
1237 | size_t size; |
1238 | |
1239 | if (hw->mac.type == ixgbe_mac_82598EB) |
1240 | return; |
1241 | |
1242 | /* If there is no support for either Tx or Rx offload |
1243 | * we should not be advertising support for IPsec. |
1244 | */ |
1245 | t_dis = IXGBE_READ_REG(hw, IXGBE_SECTXSTAT) & |
1246 | IXGBE_SECTXSTAT_SECTX_OFF_DIS; |
1247 | r_dis = IXGBE_READ_REG(hw, IXGBE_SECRXSTAT) & |
1248 | IXGBE_SECRXSTAT_SECRX_OFF_DIS; |
1249 | if (t_dis || r_dis) |
1250 | return; |
1251 | |
1252 | ipsec = kzalloc(size: sizeof(*ipsec), GFP_KERNEL); |
1253 | if (!ipsec) |
1254 | goto err1; |
1255 | hash_init(ipsec->rx_sa_list); |
1256 | |
1257 | size = sizeof(struct rx_sa) * IXGBE_IPSEC_MAX_SA_COUNT; |
1258 | ipsec->rx_tbl = kzalloc(size, GFP_KERNEL); |
1259 | if (!ipsec->rx_tbl) |
1260 | goto err2; |
1261 | |
1262 | size = sizeof(struct tx_sa) * IXGBE_IPSEC_MAX_SA_COUNT; |
1263 | ipsec->tx_tbl = kzalloc(size, GFP_KERNEL); |
1264 | if (!ipsec->tx_tbl) |
1265 | goto err2; |
1266 | |
1267 | size = sizeof(struct rx_ip_sa) * IXGBE_IPSEC_MAX_RX_IP_COUNT; |
1268 | ipsec->ip_tbl = kzalloc(size, GFP_KERNEL); |
1269 | if (!ipsec->ip_tbl) |
1270 | goto err2; |
1271 | |
1272 | ipsec->num_rx_sa = 0; |
1273 | ipsec->num_tx_sa = 0; |
1274 | |
1275 | adapter->ipsec = ipsec; |
1276 | ixgbe_ipsec_stop_engine(adapter); |
1277 | ixgbe_ipsec_clear_hw_tables(adapter); |
1278 | |
1279 | adapter->netdev->xfrmdev_ops = &ixgbe_xfrmdev_ops; |
1280 | |
1281 | return; |
1282 | |
1283 | err2: |
1284 | kfree(objp: ipsec->ip_tbl); |
1285 | kfree(objp: ipsec->rx_tbl); |
1286 | kfree(objp: ipsec->tx_tbl); |
1287 | kfree(objp: ipsec); |
1288 | err1: |
1289 | netdev_err(dev: adapter->netdev, format: "Unable to allocate memory for SA tables" ); |
1290 | } |
1291 | |
1292 | /** |
1293 | * ixgbe_stop_ipsec_offload - tear down the ipsec offload |
1294 | * @adapter: board private structure |
1295 | **/ |
1296 | void ixgbe_stop_ipsec_offload(struct ixgbe_adapter *adapter) |
1297 | { |
1298 | struct ixgbe_ipsec *ipsec = adapter->ipsec; |
1299 | |
1300 | adapter->ipsec = NULL; |
1301 | if (ipsec) { |
1302 | kfree(objp: ipsec->ip_tbl); |
1303 | kfree(objp: ipsec->rx_tbl); |
1304 | kfree(objp: ipsec->tx_tbl); |
1305 | kfree(objp: ipsec); |
1306 | } |
1307 | } |
1308 | |