1 | /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ |
2 | /* |
3 | * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved. |
4 | */ |
5 | |
6 | #ifndef ENA_H |
7 | #define ENA_H |
8 | |
9 | #include <linux/bitops.h> |
10 | #include <linux/dim.h> |
11 | #include <linux/etherdevice.h> |
12 | #include <linux/if_vlan.h> |
13 | #include <linux/inetdevice.h> |
14 | #include <linux/interrupt.h> |
15 | #include <linux/netdevice.h> |
16 | #include <linux/skbuff.h> |
17 | #include <net/xdp.h> |
18 | #include <uapi/linux/bpf.h> |
19 | |
20 | #include "ena_com.h" |
21 | #include "ena_eth_com.h" |
22 | |
23 | #define DRV_MODULE_GEN_MAJOR 2 |
24 | #define DRV_MODULE_GEN_MINOR 1 |
25 | #define DRV_MODULE_GEN_SUBMINOR 0 |
26 | |
27 | #define DRV_MODULE_NAME "ena" |
28 | |
29 | #define DEVICE_NAME "Elastic Network Adapter (ENA)" |
30 | |
31 | /* 1 for AENQ + ADMIN */ |
32 | #define ENA_ADMIN_MSIX_VEC 1 |
33 | #define ENA_MAX_MSIX_VEC(io_queues) (ENA_ADMIN_MSIX_VEC + (io_queues)) |
34 | |
35 | /* The ENA buffer length fields is 16 bit long. So when PAGE_SIZE == 64kB the |
36 | * driver passes 0. |
37 | * Since the max packet size the ENA handles is ~9kB limit the buffer length to |
38 | * 16kB. |
39 | */ |
40 | #if PAGE_SIZE > SZ_16K |
41 | #define ENA_PAGE_SIZE (_AC(SZ_16K, UL)) |
42 | #else |
43 | #define ENA_PAGE_SIZE PAGE_SIZE |
44 | #endif |
45 | |
46 | #define ENA_MIN_MSIX_VEC 2 |
47 | |
48 | #define ENA_REG_BAR 0 |
49 | #define ENA_MEM_BAR 2 |
50 | #define ENA_BAR_MASK (BIT(ENA_REG_BAR) | BIT(ENA_MEM_BAR)) |
51 | |
52 | #define ENA_DEFAULT_RING_SIZE (1024) |
53 | #define ENA_MIN_RING_SIZE (256) |
54 | |
55 | #define ENA_MIN_RX_BUF_SIZE (2048) |
56 | |
57 | #define ENA_MIN_NUM_IO_QUEUES (1) |
58 | |
59 | #define ENA_TX_WAKEUP_THRESH (MAX_SKB_FRAGS + 2) |
60 | #define ENA_DEFAULT_RX_COPYBREAK (256 - NET_IP_ALIGN) |
61 | |
62 | #define ENA_MIN_MTU 128 |
63 | |
64 | #define ENA_NAME_MAX_LEN 20 |
65 | #define ENA_IRQNAME_SIZE 40 |
66 | |
67 | #define ENA_PKT_MAX_BUFS 19 |
68 | |
69 | #define 7 |
70 | #define (1 << ENA_RX_RSS_TABLE_LOG_SIZE) |
71 | |
72 | /* The number of tx packet completions that will be handled each NAPI poll |
73 | * cycle is ring_size / ENA_TX_POLL_BUDGET_DIVIDER. |
74 | */ |
75 | #define ENA_TX_POLL_BUDGET_DIVIDER 4 |
76 | |
77 | /* Refill Rx queue when number of required descriptors is above |
78 | * QUEUE_SIZE / ENA_RX_REFILL_THRESH_DIVIDER or ENA_RX_REFILL_THRESH_PACKET |
79 | */ |
80 | #define ENA_RX_REFILL_THRESH_DIVIDER 8 |
81 | #define ENA_RX_REFILL_THRESH_PACKET 256 |
82 | |
83 | /* Number of queues to check for missing queues per timer service */ |
84 | #define ENA_MONITORED_TX_QUEUES 4 |
85 | /* Max timeout packets before device reset */ |
86 | #define MAX_NUM_OF_TIMEOUTED_PACKETS 128 |
87 | |
88 | #define ENA_TX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1)) |
89 | |
90 | #define ENA_RX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1)) |
91 | #define ENA_RX_RING_IDX_ADD(idx, n, ring_size) \ |
92 | (((idx) + (n)) & ((ring_size) - 1)) |
93 | |
94 | #define ENA_IO_TXQ_IDX(q) (2 * (q)) |
95 | #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) |
96 | #define ENA_IO_TXQ_IDX_TO_COMBINED_IDX(q) ((q) / 2) |
97 | #define ENA_IO_RXQ_IDX_TO_COMBINED_IDX(q) (((q) - 1) / 2) |
98 | |
99 | #define ENA_MGMNT_IRQ_IDX 0 |
100 | #define ENA_IO_IRQ_FIRST_IDX 1 |
101 | #define ENA_IO_IRQ_IDX(q) (ENA_IO_IRQ_FIRST_IDX + (q)) |
102 | |
103 | #define ENA_ADMIN_POLL_DELAY_US 100 |
104 | |
105 | /* ENA device should send keep alive msg every 1 sec. |
106 | * We wait for 6 sec just to be on the safe side. |
107 | */ |
108 | #define ENA_DEVICE_KALIVE_TIMEOUT (6 * HZ) |
109 | #define ENA_MAX_NO_INTERRUPT_ITERATIONS 3 |
110 | |
111 | #define ENA_MMIO_DISABLE_REG_READ BIT(0) |
112 | |
113 | /* The max MTU size is configured to be the ethernet frame size without |
114 | * the overhead of the ethernet header, which can have a VLAN header, and |
115 | * a frame check sequence (FCS). |
116 | * The buffer size we share with the device is defined to be ENA_PAGE_SIZE |
117 | */ |
118 | |
119 | #define ENA_XDP_MAX_MTU (ENA_PAGE_SIZE - ETH_HLEN - ETH_FCS_LEN - \ |
120 | VLAN_HLEN - XDP_PACKET_HEADROOM - \ |
121 | SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) |
122 | |
123 | #define ENA_IS_XDP_INDEX(adapter, index) (((index) >= (adapter)->xdp_first_ring) && \ |
124 | ((index) < (adapter)->xdp_first_ring + (adapter)->xdp_num_queues)) |
125 | |
126 | struct ena_irq { |
127 | irq_handler_t handler; |
128 | void *data; |
129 | int cpu; |
130 | u32 vector; |
131 | cpumask_t affinity_hint_mask; |
132 | char name[ENA_IRQNAME_SIZE]; |
133 | }; |
134 | |
135 | struct ena_napi { |
136 | u8 first_interrupt ____cacheline_aligned; |
137 | u8 interrupts_masked; |
138 | struct napi_struct napi; |
139 | struct ena_ring *tx_ring; |
140 | struct ena_ring *rx_ring; |
141 | struct ena_ring *xdp_ring; |
142 | u32 qid; |
143 | struct dim dim; |
144 | }; |
145 | |
146 | struct ena_tx_buffer { |
147 | struct sk_buff *skb; |
148 | /* num of ena desc for this specific skb |
149 | * (includes data desc and metadata desc) |
150 | */ |
151 | u32 tx_descs; |
152 | /* num of buffers used by this skb */ |
153 | u32 num_of_bufs; |
154 | |
155 | /* XDP buffer structure which is used for sending packets in |
156 | * the xdp queues |
157 | */ |
158 | struct xdp_frame *xdpf; |
159 | |
160 | /* Indicate if bufs[0] map the linear data of the skb. */ |
161 | u8 map_linear_data; |
162 | |
163 | /* Used for detect missing tx packets to limit the number of prints */ |
164 | u32 print_once; |
165 | /* Save the last jiffies to detect missing tx packets |
166 | * |
167 | * sets to non zero value on ena_start_xmit and set to zero on |
168 | * napi and timer_Service_routine. |
169 | * |
170 | * while this value is not protected by lock, |
171 | * a given packet is not expected to be handled by ena_start_xmit |
172 | * and by napi/timer_service at the same time. |
173 | */ |
174 | unsigned long last_jiffies; |
175 | struct ena_com_buf bufs[ENA_PKT_MAX_BUFS]; |
176 | } ____cacheline_aligned; |
177 | |
178 | struct ena_rx_buffer { |
179 | struct sk_buff *skb; |
180 | struct page *page; |
181 | dma_addr_t dma_addr; |
182 | u32 page_offset; |
183 | u32 buf_offset; |
184 | struct ena_com_buf ena_buf; |
185 | } ____cacheline_aligned; |
186 | |
187 | struct ena_stats_tx { |
188 | u64 cnt; |
189 | u64 bytes; |
190 | u64 queue_stop; |
191 | u64 prepare_ctx_err; |
192 | u64 queue_wakeup; |
193 | u64 dma_mapping_err; |
194 | u64 linearize; |
195 | u64 linearize_failed; |
196 | u64 napi_comp; |
197 | u64 tx_poll; |
198 | u64 doorbells; |
199 | u64 bad_req_id; |
200 | u64 llq_buffer_copy; |
201 | u64 missed_tx; |
202 | u64 unmask_interrupt; |
203 | u64 last_napi_jiffies; |
204 | }; |
205 | |
206 | struct ena_stats_rx { |
207 | u64 cnt; |
208 | u64 bytes; |
209 | u64 rx_copybreak_pkt; |
210 | u64 csum_good; |
211 | u64 refil_partial; |
212 | u64 csum_bad; |
213 | u64 page_alloc_fail; |
214 | u64 skb_alloc_fail; |
215 | u64 dma_mapping_err; |
216 | u64 bad_desc_num; |
217 | u64 bad_req_id; |
218 | u64 empty_rx_ring; |
219 | u64 csum_unchecked; |
220 | u64 xdp_aborted; |
221 | u64 xdp_drop; |
222 | u64 xdp_pass; |
223 | u64 xdp_tx; |
224 | u64 xdp_invalid; |
225 | u64 xdp_redirect; |
226 | }; |
227 | |
228 | struct ena_ring { |
229 | /* Holds the empty requests for TX/RX |
230 | * out of order completions |
231 | */ |
232 | u16 *free_ids; |
233 | |
234 | union { |
235 | struct ena_tx_buffer *tx_buffer_info; |
236 | struct ena_rx_buffer *rx_buffer_info; |
237 | }; |
238 | |
239 | /* cache ptr to avoid using the adapter */ |
240 | struct device *dev; |
241 | struct pci_dev *pdev; |
242 | struct napi_struct *napi; |
243 | struct net_device *netdev; |
244 | struct ena_com_dev *ena_dev; |
245 | struct ena_adapter *adapter; |
246 | struct ena_com_io_cq *ena_com_io_cq; |
247 | struct ena_com_io_sq *ena_com_io_sq; |
248 | struct bpf_prog *xdp_bpf_prog; |
249 | struct xdp_rxq_info xdp_rxq; |
250 | spinlock_t xdp_tx_lock; /* synchronize XDP TX/Redirect traffic */ |
251 | /* Used for rx queues only to point to the xdp tx ring, to |
252 | * which traffic should be redirected from this rx ring. |
253 | */ |
254 | struct ena_ring *xdp_ring; |
255 | |
256 | u16 next_to_use; |
257 | u16 next_to_clean; |
258 | u16 rx_copybreak; |
259 | u16 rx_headroom; |
260 | u16 qid; |
261 | u16 mtu; |
262 | u16 sgl_size; |
263 | |
264 | /* The maximum header length the device can handle */ |
265 | u8 ; |
266 | |
267 | bool disable_meta_caching; |
268 | u16 no_interrupt_event_cnt; |
269 | |
270 | /* cpu and NUMA for TPH */ |
271 | int cpu; |
272 | int numa_node; |
273 | |
274 | /* number of tx/rx_buffer_info's entries */ |
275 | int ring_size; |
276 | |
277 | enum ena_admin_placement_policy_type tx_mem_queue_type; |
278 | |
279 | struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS]; |
280 | u32 smoothed_interval; |
281 | u32 per_napi_packets; |
282 | u16 non_empty_napi_events; |
283 | struct u64_stats_sync syncp; |
284 | union { |
285 | struct ena_stats_tx tx_stats; |
286 | struct ena_stats_rx rx_stats; |
287 | }; |
288 | |
289 | u8 *push_buf_intermediate_buf; |
290 | int empty_rx_queue; |
291 | } ____cacheline_aligned; |
292 | |
293 | struct ena_stats_dev { |
294 | u64 tx_timeout; |
295 | u64 suspend; |
296 | u64 resume; |
297 | u64 wd_expired; |
298 | u64 interface_up; |
299 | u64 interface_down; |
300 | u64 admin_q_pause; |
301 | u64 rx_drops; |
302 | u64 tx_drops; |
303 | }; |
304 | |
305 | enum ena_flags_t { |
306 | ENA_FLAG_DEVICE_RUNNING, |
307 | ENA_FLAG_DEV_UP, |
308 | ENA_FLAG_LINK_UP, |
309 | ENA_FLAG_MSIX_ENABLED, |
310 | ENA_FLAG_TRIGGER_RESET, |
311 | ENA_FLAG_ONGOING_RESET |
312 | }; |
313 | |
314 | /* adapter specific private data structure */ |
315 | struct ena_adapter { |
316 | struct ena_com_dev *ena_dev; |
317 | /* OS defined structs */ |
318 | struct net_device *netdev; |
319 | struct pci_dev *pdev; |
320 | |
321 | /* rx packets that shorter that this len will be copied to the skb |
322 | * header |
323 | */ |
324 | u32 rx_copybreak; |
325 | u32 max_mtu; |
326 | |
327 | u32 num_io_queues; |
328 | u32 max_num_io_queues; |
329 | |
330 | int msix_vecs; |
331 | |
332 | u32 missing_tx_completion_threshold; |
333 | |
334 | u32 requested_tx_ring_size; |
335 | u32 requested_rx_ring_size; |
336 | |
337 | u32 max_tx_ring_size; |
338 | u32 max_rx_ring_size; |
339 | |
340 | u32 msg_enable; |
341 | |
342 | /* large_llq_header_enabled is used for two purposes: |
343 | * 1. Indicates that large LLQ has been requested. |
344 | * 2. Indicates whether large LLQ is set or not after device |
345 | * initialization / configuration. |
346 | */ |
347 | bool ; |
348 | bool ; |
349 | |
350 | u16 max_tx_sgl_size; |
351 | u16 max_rx_sgl_size; |
352 | |
353 | u8 mac_addr[ETH_ALEN]; |
354 | |
355 | unsigned long keep_alive_timeout; |
356 | unsigned long missing_tx_completion_to; |
357 | |
358 | char name[ENA_NAME_MAX_LEN]; |
359 | |
360 | unsigned long flags; |
361 | /* TX */ |
362 | struct ena_ring tx_ring[ENA_MAX_NUM_IO_QUEUES] |
363 | ____cacheline_aligned_in_smp; |
364 | |
365 | /* RX */ |
366 | struct ena_ring rx_ring[ENA_MAX_NUM_IO_QUEUES] |
367 | ____cacheline_aligned_in_smp; |
368 | |
369 | struct ena_napi ena_napi[ENA_MAX_NUM_IO_QUEUES]; |
370 | |
371 | struct ena_irq irq_tbl[ENA_MAX_MSIX_VEC(ENA_MAX_NUM_IO_QUEUES)]; |
372 | |
373 | /* timer service */ |
374 | struct work_struct reset_task; |
375 | struct timer_list timer_service; |
376 | |
377 | bool wd_state; |
378 | bool dev_up_before_reset; |
379 | bool disable_meta_caching; |
380 | unsigned long last_keep_alive_jiffies; |
381 | |
382 | struct u64_stats_sync syncp; |
383 | struct ena_stats_dev dev_stats; |
384 | struct ena_admin_eni_stats eni_stats; |
385 | |
386 | /* last queue index that was checked for uncompleted tx packets */ |
387 | u32 last_monitored_tx_qid; |
388 | |
389 | enum ena_regs_reset_reason_types reset_reason; |
390 | |
391 | struct bpf_prog *xdp_bpf_prog; |
392 | u32 xdp_first_ring; |
393 | u32 xdp_num_queues; |
394 | }; |
395 | |
396 | void ena_set_ethtool_ops(struct net_device *netdev); |
397 | |
398 | void ena_dump_stats_to_dmesg(struct ena_adapter *adapter); |
399 | |
400 | void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf); |
401 | |
402 | int ena_update_hw_stats(struct ena_adapter *adapter); |
403 | |
404 | int ena_update_queue_params(struct ena_adapter *adapter, |
405 | u32 new_tx_size, |
406 | u32 new_rx_size, |
407 | u32 ); |
408 | |
409 | int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count); |
410 | |
411 | int ena_set_rx_copybreak(struct ena_adapter *adapter, u32 rx_copybreak); |
412 | |
413 | int ena_get_sset_count(struct net_device *netdev, int sset); |
414 | |
415 | static inline void ena_reset_device(struct ena_adapter *adapter, |
416 | enum ena_regs_reset_reason_types reset_reason) |
417 | { |
418 | adapter->reset_reason = reset_reason; |
419 | /* Make sure reset reason is set before triggering the reset */ |
420 | smp_mb__before_atomic(); |
421 | set_bit(nr: ENA_FLAG_TRIGGER_RESET, addr: &adapter->flags); |
422 | } |
423 | |
424 | enum ena_xdp_errors_t { |
425 | ENA_XDP_ALLOWED = 0, |
426 | ENA_XDP_CURRENT_MTU_TOO_LARGE, |
427 | ENA_XDP_NO_ENOUGH_QUEUES, |
428 | }; |
429 | |
430 | enum ENA_XDP_ACTIONS { |
431 | ENA_XDP_PASS = 0, |
432 | ENA_XDP_TX = BIT(0), |
433 | ENA_XDP_REDIRECT = BIT(1), |
434 | ENA_XDP_DROP = BIT(2) |
435 | }; |
436 | |
437 | #define ENA_XDP_FORWARDED (ENA_XDP_TX | ENA_XDP_REDIRECT) |
438 | |
439 | static inline bool ena_xdp_present(struct ena_adapter *adapter) |
440 | { |
441 | return !!adapter->xdp_bpf_prog; |
442 | } |
443 | |
444 | static inline bool ena_xdp_present_ring(struct ena_ring *ring) |
445 | { |
446 | return !!ring->xdp_bpf_prog; |
447 | } |
448 | |
449 | static inline bool ena_xdp_legal_queue_count(struct ena_adapter *adapter, |
450 | u32 queues) |
451 | { |
452 | return 2 * queues <= adapter->max_num_io_queues; |
453 | } |
454 | |
455 | static inline enum ena_xdp_errors_t ena_xdp_allowed(struct ena_adapter *adapter) |
456 | { |
457 | enum ena_xdp_errors_t rc = ENA_XDP_ALLOWED; |
458 | |
459 | if (adapter->netdev->mtu > ENA_XDP_MAX_MTU) |
460 | rc = ENA_XDP_CURRENT_MTU_TOO_LARGE; |
461 | else if (!ena_xdp_legal_queue_count(adapter, queues: adapter->num_io_queues)) |
462 | rc = ENA_XDP_NO_ENOUGH_QUEUES; |
463 | |
464 | return rc; |
465 | } |
466 | |
467 | #endif /* !(ENA_H) */ |
468 | |