1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* Texas Instruments ICSSG Ethernet driver |
3 | * |
4 | * Copyright (C) 2018-2022 Texas Instruments Incorporated - https://www.ti.com/ |
5 | * |
6 | */ |
7 | |
8 | #ifndef __NET_TI_ICSSG_PRUETH_H |
9 | #define __NET_TI_ICSSG_PRUETH_H |
10 | |
11 | #include <linux/bpf.h> |
12 | #include <linux/bpf_trace.h> |
13 | #include <linux/etherdevice.h> |
14 | #include <linux/genalloc.h> |
15 | #include <linux/if_vlan.h> |
16 | #include <linux/interrupt.h> |
17 | #include <linux/kernel.h> |
18 | #include <linux/mfd/syscon.h> |
19 | #include <linux/module.h> |
20 | #include <linux/mutex.h> |
21 | #include <linux/net_tstamp.h> |
22 | #include <linux/of.h> |
23 | #include <linux/of_irq.h> |
24 | #include <linux/of_mdio.h> |
25 | #include <linux/of_net.h> |
26 | #include <linux/of_platform.h> |
27 | #include <linux/phy.h> |
28 | #include <linux/remoteproc/pruss.h> |
29 | #include <linux/pruss_driver.h> |
30 | #include <linux/ptp_clock_kernel.h> |
31 | #include <linux/remoteproc.h> |
32 | |
33 | #include <linux/dma-mapping.h> |
34 | #include <linux/dma/ti-cppi5.h> |
35 | #include <linux/dma/k3-udma-glue.h> |
36 | |
37 | #include <net/devlink.h> |
38 | #include <net/xdp.h> |
39 | #include <net/page_pool/helpers.h> |
40 | |
41 | #include "icssg_config.h" |
42 | #include "icss_iep.h" |
43 | #include "icssg_switch_map.h" |
44 | |
45 | #define PRUETH_MAX_MTU (2000 - ETH_HLEN - ETH_FCS_LEN) |
46 | #define PRUETH_MIN_PKT_SIZE (VLAN_ETH_ZLEN) |
47 | #define PRUETH_MAX_PKT_SIZE (PRUETH_MAX_MTU + ETH_HLEN + ETH_FCS_LEN) |
48 | |
49 | #define ICSS_SLICE0 0 |
50 | #define ICSS_SLICE1 1 |
51 | |
52 | #define ICSS_FW_PRU 0 |
53 | #define ICSS_FW_RTU 1 |
54 | |
55 | #define ICSSG_MAX_RFLOWS 8 /* per slice */ |
56 | |
57 | #define ICSSG_NUM_PA_STATS 32 |
58 | #define ICSSG_NUM_MIIG_STATS 60 |
59 | /* Number of ICSSG related stats */ |
60 | #define ICSSG_NUM_STATS (ICSSG_NUM_MIIG_STATS + ICSSG_NUM_PA_STATS) |
61 | #define ICSSG_NUM_STANDARD_STATS 31 |
62 | #define ICSSG_NUM_ETHTOOL_STATS (ICSSG_NUM_STATS - ICSSG_NUM_STANDARD_STATS) |
63 | |
64 | #define IEP_DEFAULT_CYCLE_TIME_NS 1000000 /* 1 ms */ |
65 | |
66 | #define PRUETH_UNDIRECTED_PKT_DST_TAG 0 |
67 | #define PRUETH_UNDIRECTED_PKT_TAG_INS BIT(30) |
68 | |
69 | /* Firmware status codes */ |
70 | #define ICSS_HS_FW_READY 0x55555555 |
71 | #define ICSS_HS_FW_DEAD 0xDEAD0000 /* lower 16 bits contain error code */ |
72 | |
73 | /* Firmware command codes */ |
74 | #define ICSS_HS_CMD_BUSY 0x40000000 |
75 | #define ICSS_HS_CMD_DONE 0x80000000 |
76 | #define ICSS_HS_CMD_CANCEL 0x10000000 |
77 | |
78 | /* Firmware commands */ |
79 | #define ICSS_CMD_SPAD 0x20 |
80 | #define ICSS_CMD_RXTX 0x10 |
81 | #define ICSS_CMD_ADD_FDB 0x1 |
82 | #define ICSS_CMD_DEL_FDB 0x2 |
83 | #define ICSS_CMD_SET_RUN 0x4 |
84 | #define ICSS_CMD_GET_FDB_SLOT 0x5 |
85 | #define ICSS_CMD_ENABLE_VLAN 0x5 |
86 | #define ICSS_CMD_DISABLE_VLAN 0x6 |
87 | #define ICSS_CMD_ADD_FILTER 0x7 |
88 | #define ICSS_CMD_ADD_MAC 0x8 |
89 | |
90 | /* VLAN Filtering Related MACROs */ |
91 | #define PRUETH_DFLT_VLAN_HSR 1 |
92 | #define PRUETH_DFLT_VLAN_SW 1 |
93 | #define PRUETH_DFLT_VLAN_MAC 0 |
94 | #define MAX_VLAN_ID 256 |
95 | |
96 | /* In switch mode there are 3 real ports i.e. 3 mac addrs. |
97 | * however Linux sees only the host side port. The other 2 ports |
98 | * are the switch ports. |
99 | * In emac mode there are 2 real ports i.e. 2 mac addrs. |
100 | * Linux sees both the ports. |
101 | */ |
102 | enum prueth_port { |
103 | PRUETH_PORT_HOST = 0, /* host side port */ |
104 | PRUETH_PORT_MII0, /* physical port RG/SG MII 0 */ |
105 | PRUETH_PORT_MII1, /* physical port RG/SG MII 1 */ |
106 | PRUETH_PORT_INVALID, /* Invalid prueth port */ |
107 | }; |
108 | |
109 | enum prueth_mac { |
110 | PRUETH_MAC0 = 0, |
111 | PRUETH_MAC1, |
112 | PRUETH_NUM_MACS, |
113 | PRUETH_MAC_INVALID, |
114 | }; |
115 | |
116 | struct prueth_tx_chn { |
117 | struct device *dma_dev; |
118 | struct napi_struct napi_tx; |
119 | struct k3_cppi_desc_pool *desc_pool; |
120 | struct k3_udma_glue_tx_channel *tx_chn; |
121 | struct prueth_emac *emac; |
122 | u32 id; |
123 | u32 descs_num; |
124 | unsigned int irq; |
125 | char name[32]; |
126 | struct hrtimer tx_hrtimer; |
127 | unsigned long tx_pace_timeout_ns; |
128 | }; |
129 | |
130 | struct prueth_rx_chn { |
131 | struct device *dev; |
132 | struct device *dma_dev; |
133 | struct k3_cppi_desc_pool *desc_pool; |
134 | struct k3_udma_glue_rx_channel *rx_chn; |
135 | u32 descs_num; |
136 | unsigned int irq[ICSSG_MAX_RFLOWS]; /* separate irq per flow */ |
137 | char name[32]; |
138 | struct page_pool *pg_pool; |
139 | struct xdp_rxq_info xdp_rxq; |
140 | }; |
141 | |
142 | enum prueth_swdata_type { |
143 | PRUETH_SWDATA_INVALID = 0, |
144 | PRUETH_SWDATA_SKB, |
145 | PRUETH_SWDATA_PAGE, |
146 | PRUETH_SWDATA_CMD, |
147 | PRUETH_SWDATA_XDPF, |
148 | }; |
149 | |
150 | struct prueth_swdata { |
151 | enum prueth_swdata_type type; |
152 | union prueth_data { |
153 | struct sk_buff *skb; |
154 | struct page *page; |
155 | u32 cmd; |
156 | struct xdp_frame *xdpf; |
157 | } data; |
158 | }; |
159 | |
160 | /* There are 4 Tx DMA channels, but the highest priority is CH3 (thread 3) |
161 | * and lower three are lower priority channels or threads. |
162 | */ |
163 | #define PRUETH_MAX_TX_QUEUES 4 |
164 | |
165 | #define PRUETH_MAX_TX_TS_REQUESTS 50 /* Max simultaneous TX_TS requests */ |
166 | |
167 | /* XDP BPF state */ |
168 | #define ICSSG_XDP_PASS 0 |
169 | #define ICSSG_XDP_CONSUMED BIT(0) |
170 | #define ICSSG_XDP_TX BIT(1) |
171 | #define ICSSG_XDP_REDIR BIT(2) |
172 | |
173 | /* Minimum coalesce time in usecs for both Tx and Rx */ |
174 | #define ICSSG_MIN_COALESCE_USECS 20 |
175 | |
176 | /* data for each emac port */ |
177 | struct prueth_emac { |
178 | bool is_sr1; |
179 | struct prueth *prueth; |
180 | struct net_device *ndev; |
181 | u8 mac_addr[6]; |
182 | struct napi_struct napi_rx; |
183 | u32 msg_enable; |
184 | |
185 | int link; |
186 | int speed; |
187 | int duplex; |
188 | |
189 | const char *phy_id; |
190 | struct device_node *phy_node; |
191 | phy_interface_t phy_if; |
192 | enum prueth_port port_id; |
193 | struct icss_iep *iep; |
194 | unsigned int rx_ts_enabled : 1; |
195 | unsigned int tx_ts_enabled : 1; |
196 | unsigned int half_duplex : 1; |
197 | |
198 | /* DMA related */ |
199 | struct prueth_tx_chn tx_chns[PRUETH_MAX_TX_QUEUES]; |
200 | struct completion tdown_complete; |
201 | atomic_t tdown_cnt; |
202 | struct prueth_rx_chn rx_chns; |
203 | int rx_flow_id_base; |
204 | int tx_ch_num; |
205 | |
206 | /* SR1.0 Management channel */ |
207 | struct prueth_rx_chn rx_mgm_chn; |
208 | int rx_mgm_flow_id_base; |
209 | |
210 | spinlock_t lock; /* serialize access */ |
211 | |
212 | /* TX HW Timestamping */ |
213 | /* TX TS cookie will be index to the tx_ts_skb array */ |
214 | struct sk_buff *tx_ts_skb[PRUETH_MAX_TX_TS_REQUESTS]; |
215 | atomic_t tx_ts_pending; |
216 | int tx_ts_irq; |
217 | |
218 | u8 cmd_seq; |
219 | /* shutdown related */ |
220 | __le32 cmd_data[4]; |
221 | struct completion cmd_complete; |
222 | /* Mutex to serialize access to firmware command interface */ |
223 | struct mutex cmd_lock; |
224 | struct work_struct rx_mode_work; |
225 | struct workqueue_struct *cmd_wq; |
226 | |
227 | struct pruss_mem_region dram; |
228 | |
229 | bool offload_fwd_mark; |
230 | int port_vlan; |
231 | |
232 | struct delayed_work stats_work; |
233 | u64 stats[ICSSG_NUM_MIIG_STATS]; |
234 | u64 pa_stats[ICSSG_NUM_PA_STATS]; |
235 | |
236 | /* RX IRQ Coalescing Related */ |
237 | struct hrtimer rx_hrtimer; |
238 | unsigned long rx_pace_timeout_ns; |
239 | |
240 | struct netdev_hw_addr_list vlan_mcast_list[MAX_VLAN_ID]; |
241 | struct bpf_prog *xdp_prog; |
242 | struct xdp_attachment_info xdpi; |
243 | }; |
244 | |
245 | /* The buf includes headroom compatible with both skb and xdpf */ |
246 | #define PRUETH_HEADROOM_NA (max(XDP_PACKET_HEADROOM, NET_SKB_PAD) + NET_IP_ALIGN) |
247 | #define PRUETH_HEADROOM ALIGN(PRUETH_HEADROOM_NA, sizeof(long)) |
248 | |
249 | /** |
250 | * struct prueth_pdata - PRUeth platform data |
251 | * @fdqring_mode: Free desc queue mode |
252 | * @quirk_10m_link_issue: 10M link detect errata |
253 | * @switch_mode: switch firmware support |
254 | */ |
255 | struct prueth_pdata { |
256 | enum k3_ring_mode fdqring_mode; |
257 | u32 quirk_10m_link_issue:1; |
258 | u32 switch_mode:1; |
259 | }; |
260 | |
261 | struct icssg_firmwares { |
262 | char *pru; |
263 | char *rtu; |
264 | char *txpru; |
265 | }; |
266 | |
267 | /** |
268 | * struct prueth - PRUeth structure |
269 | * @dev: device |
270 | * @pruss: pruss handle |
271 | * @pru: rproc instances of PRUs |
272 | * @rtu: rproc instances of RTUs |
273 | * @txpru: rproc instances of TX_PRUs |
274 | * @shram: PRUSS shared RAM region |
275 | * @sram_pool: MSMC RAM pool for buffers |
276 | * @msmcram: MSMC RAM region |
277 | * @eth_node: DT node for the port |
278 | * @emac: private EMAC data structure |
279 | * @registered_netdevs: list of registered netdevs |
280 | * @miig_rt: regmap to mii_g_rt block |
281 | * @mii_rt: regmap to mii_rt block |
282 | * @pa_stats: regmap to pa_stats block |
283 | * @pru_id: ID for each of the PRUs |
284 | * @pdev: pointer to ICSSG platform device |
285 | * @pdata: pointer to platform data for ICSSG driver |
286 | * @icssg_hwcmdseq: seq counter or HWQ messages |
287 | * @emacs_initialized: num of EMACs/ext ports that are up/running |
288 | * @iep0: pointer to IEP0 device |
289 | * @iep1: pointer to IEP1 device |
290 | * @vlan_tbl: VLAN-FID table pointer |
291 | * @hw_bridge_dev: pointer to HW bridge net device |
292 | * @hsr_dev: pointer to the HSR net device |
293 | * @br_members: bitmask of bridge member ports |
294 | * @hsr_members: bitmask of hsr member ports |
295 | * @prueth_netdevice_nb: netdevice notifier block |
296 | * @prueth_switchdev_nb: switchdev notifier block |
297 | * @prueth_switchdev_bl_nb: switchdev blocking notifier block |
298 | * @is_switch_mode: flag to indicate if device is in Switch mode |
299 | * @is_hsr_offload_mode: flag to indicate if device is in hsr offload mode |
300 | * @is_switchmode_supported: indicates platform support for switch mode |
301 | * @switch_id: ID for mapping switch ports to bridge |
302 | * @default_vlan: Default VLAN for host |
303 | */ |
304 | struct prueth { |
305 | struct device *dev; |
306 | struct pruss *pruss; |
307 | struct rproc *pru[PRUSS_NUM_PRUS]; |
308 | struct rproc *rtu[PRUSS_NUM_PRUS]; |
309 | struct rproc *txpru[PRUSS_NUM_PRUS]; |
310 | struct pruss_mem_region shram; |
311 | struct gen_pool *sram_pool; |
312 | struct pruss_mem_region msmcram; |
313 | |
314 | struct device_node *eth_node[PRUETH_NUM_MACS]; |
315 | struct prueth_emac *emac[PRUETH_NUM_MACS]; |
316 | struct net_device *registered_netdevs[PRUETH_NUM_MACS]; |
317 | struct regmap *miig_rt; |
318 | struct regmap *mii_rt; |
319 | struct regmap *pa_stats; |
320 | |
321 | enum pruss_pru_id pru_id[PRUSS_NUM_PRUS]; |
322 | struct platform_device *pdev; |
323 | struct prueth_pdata pdata; |
324 | u8 icssg_hwcmdseq; |
325 | int emacs_initialized; |
326 | struct icss_iep *iep0; |
327 | struct icss_iep *iep1; |
328 | struct prueth_vlan_tbl *vlan_tbl; |
329 | |
330 | struct net_device *hw_bridge_dev; |
331 | struct net_device *hsr_dev; |
332 | u8 br_members; |
333 | u8 hsr_members; |
334 | struct notifier_block prueth_netdevice_nb; |
335 | struct notifier_block prueth_switchdev_nb; |
336 | struct notifier_block prueth_switchdev_bl_nb; |
337 | bool is_switch_mode; |
338 | bool is_hsr_offload_mode; |
339 | bool is_switchmode_supported; |
340 | unsigned char switch_id[MAX_PHYS_ITEM_ID_LEN]; |
341 | int default_vlan; |
342 | /** @vtbl_lock: Lock for vtbl in shared memory */ |
343 | spinlock_t vtbl_lock; |
344 | /** @stats_lock: Lock for reading icssg stats */ |
345 | spinlock_t stats_lock; |
346 | }; |
347 | |
348 | struct emac_tx_ts_response { |
349 | u32 reserved[2]; |
350 | u32 cookie; |
351 | u32 lo_ts; |
352 | u32 hi_ts; |
353 | }; |
354 | |
355 | struct emac_tx_ts_response_sr1 { |
356 | __le32 lo_ts; |
357 | __le32 hi_ts; |
358 | __le32 reserved; |
359 | __le32 cookie; |
360 | }; |
361 | |
362 | /* get PRUSS SLICE number from prueth_emac */ |
363 | static inline int prueth_emac_slice(struct prueth_emac *emac) |
364 | { |
365 | switch (emac->port_id) { |
366 | case PRUETH_PORT_MII0: |
367 | return ICSS_SLICE0; |
368 | case PRUETH_PORT_MII1: |
369 | return ICSS_SLICE1; |
370 | default: |
371 | return -EINVAL; |
372 | } |
373 | } |
374 | |
375 | extern const struct ethtool_ops icssg_ethtool_ops; |
376 | extern const struct dev_pm_ops prueth_dev_pm_ops; |
377 | |
378 | static inline u64 icssg_read_time(const void __iomem *addr) |
379 | { |
380 | u32 low, high; |
381 | |
382 | do { |
383 | high = readl(addr: addr + 4); |
384 | low = readl(addr); |
385 | } while (high != readl(addr: addr + 4)); |
386 | |
387 | return low + ((u64)high << 32); |
388 | } |
389 | |
390 | /* Classifier helpers */ |
391 | void icssg_class_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac); |
392 | void icssg_class_set_host_mac_addr(struct regmap *miig_rt, const u8 *mac); |
393 | void icssg_class_disable(struct regmap *miig_rt, int slice); |
394 | void icssg_class_default(struct regmap *miig_rt, int slice, bool allmulti, |
395 | bool is_sr1); |
396 | void icssg_class_promiscuous_sr1(struct regmap *miig_rt, int slice); |
397 | void icssg_class_add_mcast_sr1(struct regmap *miig_rt, int slice, |
398 | struct net_device *ndev); |
399 | void icssg_ft1_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac_addr); |
400 | |
401 | /* config helpers */ |
402 | void icssg_config_ipg(struct prueth_emac *emac); |
403 | int icssg_config(struct prueth *prueth, struct prueth_emac *emac, |
404 | int slice); |
405 | int icssg_set_port_state(struct prueth_emac *emac, |
406 | enum icssg_port_state_cmd state); |
407 | void icssg_config_set_speed(struct prueth_emac *emac); |
408 | void icssg_config_half_duplex(struct prueth_emac *emac); |
409 | void icssg_init_emac_mode(struct prueth *prueth); |
410 | void icssg_init_fw_offload_mode(struct prueth *prueth); |
411 | |
412 | /* Buffer queue helpers */ |
413 | int icssg_queue_pop(struct prueth *prueth, u8 queue); |
414 | void icssg_queue_push(struct prueth *prueth, int queue, u16 addr); |
415 | u32 icssg_queue_level(struct prueth *prueth, int queue); |
416 | |
417 | int icssg_send_fdb_msg(struct prueth_emac *emac, struct mgmt_cmd *cmd, |
418 | struct mgmt_cmd_rsp *rsp); |
419 | int icssg_fdb_add_del(struct prueth_emac *emac, const unsigned char *addr, |
420 | u8 vid, u8 fid_c2, bool add); |
421 | int icssg_fdb_lookup(struct prueth_emac *emac, const unsigned char *addr, |
422 | u8 vid); |
423 | void icssg_vtbl_modify(struct prueth_emac *emac, u8 vid, u8 port_mask, |
424 | u8 untag_mask, bool add); |
425 | u16 icssg_get_pvid(struct prueth_emac *emac); |
426 | void icssg_set_pvid(struct prueth *prueth, u8 vid, u8 port); |
427 | int emac_fdb_flow_id_updated(struct prueth_emac *emac); |
428 | #define prueth_napi_to_tx_chn(pnapi) \ |
429 | container_of(pnapi, struct prueth_tx_chn, napi_tx) |
430 | |
431 | void icssg_stats_work_handler(struct work_struct *work); |
432 | void emac_update_hardware_stats(struct prueth_emac *emac); |
433 | int emac_get_stat_by_name(struct prueth_emac *emac, char *stat_name); |
434 | |
435 | /* Common functions */ |
436 | void prueth_cleanup_rx_chns(struct prueth_emac *emac, |
437 | struct prueth_rx_chn *rx_chn, |
438 | int max_rflows); |
439 | void prueth_cleanup_tx_chns(struct prueth_emac *emac); |
440 | void prueth_ndev_del_tx_napi(struct prueth_emac *emac, int num); |
441 | void prueth_xmit_free(struct prueth_tx_chn *tx_chn, |
442 | struct cppi5_host_desc_t *desc); |
443 | int emac_tx_complete_packets(struct prueth_emac *emac, int chn, |
444 | int budget, bool *tdown); |
445 | int prueth_ndev_add_tx_napi(struct prueth_emac *emac); |
446 | int prueth_init_tx_chns(struct prueth_emac *emac); |
447 | int prueth_init_rx_chns(struct prueth_emac *emac, |
448 | struct prueth_rx_chn *rx_chn, |
449 | char *name, u32 max_rflows, |
450 | u32 max_desc_num); |
451 | int prueth_dma_rx_push_mapped(struct prueth_emac *emac, |
452 | struct prueth_rx_chn *rx_chn, |
453 | struct page *page, u32 buf_len); |
454 | unsigned int prueth_rxbuf_total_len(unsigned int len); |
455 | void emac_rx_timestamp(struct prueth_emac *emac, |
456 | struct sk_buff *skb, u32 *psdata); |
457 | enum netdev_tx icssg_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev); |
458 | irqreturn_t prueth_rx_irq(int irq, void *dev_id); |
459 | void prueth_cleanup_tx_ts(struct prueth_emac *emac); |
460 | int icssg_napi_rx_poll(struct napi_struct *napi_rx, int budget); |
461 | int prueth_prepare_rx_chan(struct prueth_emac *emac, |
462 | struct prueth_rx_chn *chn, |
463 | int buf_size); |
464 | void prueth_reset_tx_chan(struct prueth_emac *emac, int ch_num, |
465 | bool free_skb); |
466 | void prueth_reset_rx_chan(struct prueth_rx_chn *chn, |
467 | int num_flows, bool disable); |
468 | void icssg_ndo_tx_timeout(struct net_device *ndev, unsigned int txqueue); |
469 | int icssg_ndo_ioctl(struct net_device *ndev, struct ifreq *ifr, int cmd); |
470 | void icssg_ndo_get_stats64(struct net_device *ndev, |
471 | struct rtnl_link_stats64 *stats); |
472 | int icssg_ndo_get_phys_port_name(struct net_device *ndev, char *name, |
473 | size_t len); |
474 | int prueth_node_port(struct device_node *eth_node); |
475 | int prueth_node_mac(struct device_node *eth_node); |
476 | void prueth_netdev_exit(struct prueth *prueth, |
477 | struct device_node *eth_node); |
478 | int prueth_get_cores(struct prueth *prueth, int slice, bool is_sr1); |
479 | void prueth_put_cores(struct prueth *prueth, int slice); |
480 | |
481 | /* Revision specific helper */ |
482 | u64 icssg_ts_to_ns(u32 hi_sw, u32 hi, u32 lo, u32 cycle_time_ns); |
483 | u32 emac_xmit_xdp_frame(struct prueth_emac *emac, |
484 | struct xdp_frame *xdpf, |
485 | struct page *page, |
486 | unsigned int q_idx); |
487 | |
488 | #endif /* __NET_TI_ICSSG_PRUETH_H */ |
489 | |