1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */ |
3 | |
4 | #include <linux/kernel.h> |
5 | #include <linux/io.h> |
6 | #include <linux/iopoll.h> |
7 | #include <linux/etherdevice.h> |
8 | #include <linux/platform_device.h> |
9 | #include <linux/if_ether.h> |
10 | #include <linux/if_vlan.h> |
11 | #include <net/dst_metadata.h> |
12 | #include <net/dsa.h> |
13 | #include "mtk_eth_soc.h" |
14 | #include "mtk_ppe.h" |
15 | #include "mtk_ppe_regs.h" |
16 | |
17 | static DEFINE_SPINLOCK(ppe_lock); |
18 | |
19 | static const struct rhashtable_params mtk_flow_l2_ht_params = { |
20 | .head_offset = offsetof(struct mtk_flow_entry, l2_node), |
21 | .key_offset = offsetof(struct mtk_flow_entry, data.bridge), |
22 | .key_len = offsetof(struct mtk_foe_bridge, key_end), |
23 | .automatic_shrinking = true, |
24 | }; |
25 | |
26 | static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val) |
27 | { |
28 | writel(val, addr: ppe->base + reg); |
29 | } |
30 | |
31 | static u32 ppe_r32(struct mtk_ppe *ppe, u32 reg) |
32 | { |
33 | return readl(addr: ppe->base + reg); |
34 | } |
35 | |
36 | static u32 ppe_m32(struct mtk_ppe *ppe, u32 reg, u32 mask, u32 set) |
37 | { |
38 | u32 val; |
39 | |
40 | val = ppe_r32(ppe, reg); |
41 | val &= ~mask; |
42 | val |= set; |
43 | ppe_w32(ppe, reg, val); |
44 | |
45 | return val; |
46 | } |
47 | |
48 | static u32 ppe_set(struct mtk_ppe *ppe, u32 reg, u32 val) |
49 | { |
50 | return ppe_m32(ppe, reg, mask: 0, set: val); |
51 | } |
52 | |
53 | static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val) |
54 | { |
55 | return ppe_m32(ppe, reg, mask: val, set: 0); |
56 | } |
57 | |
58 | static u32 mtk_eth_timestamp(struct mtk_eth *eth) |
59 | { |
60 | return mtk_r32(eth, reg: 0x0010) & mtk_get_ib1_ts_mask(eth); |
61 | } |
62 | |
63 | static int mtk_ppe_wait_busy(struct mtk_ppe *ppe) |
64 | { |
65 | int ret; |
66 | u32 val; |
67 | |
68 | ret = readl_poll_timeout(ppe->base + MTK_PPE_GLO_CFG, val, |
69 | !(val & MTK_PPE_GLO_CFG_BUSY), |
70 | 20, MTK_PPE_WAIT_TIMEOUT_US); |
71 | |
72 | if (ret) |
73 | dev_err(ppe->dev, "PPE table busy" ); |
74 | |
75 | return ret; |
76 | } |
77 | |
78 | static int mtk_ppe_mib_wait_busy(struct mtk_ppe *ppe) |
79 | { |
80 | int ret; |
81 | u32 val; |
82 | |
83 | ret = readl_poll_timeout(ppe->base + MTK_PPE_MIB_SER_CR, val, |
84 | !(val & MTK_PPE_MIB_SER_CR_ST), |
85 | 20, MTK_PPE_WAIT_TIMEOUT_US); |
86 | |
87 | if (ret) |
88 | dev_err(ppe->dev, "MIB table busy" ); |
89 | |
90 | return ret; |
91 | } |
92 | |
93 | static int mtk_mib_entry_read(struct mtk_ppe *ppe, u16 index, u64 *bytes, u64 *packets) |
94 | { |
95 | u32 val, cnt_r0, cnt_r1, cnt_r2; |
96 | int ret; |
97 | |
98 | val = FIELD_PREP(MTK_PPE_MIB_SER_CR_ADDR, index) | MTK_PPE_MIB_SER_CR_ST; |
99 | ppe_w32(ppe, MTK_PPE_MIB_SER_CR, val); |
100 | |
101 | ret = mtk_ppe_mib_wait_busy(ppe); |
102 | if (ret) |
103 | return ret; |
104 | |
105 | cnt_r0 = readl(addr: ppe->base + MTK_PPE_MIB_SER_R0); |
106 | cnt_r1 = readl(addr: ppe->base + MTK_PPE_MIB_SER_R1); |
107 | cnt_r2 = readl(addr: ppe->base + MTK_PPE_MIB_SER_R2); |
108 | |
109 | if (mtk_is_netsys_v3_or_greater(eth: ppe->eth)) { |
110 | /* 64 bit for each counter */ |
111 | u32 cnt_r3 = readl(addr: ppe->base + MTK_PPE_MIB_SER_R3); |
112 | *bytes = ((u64)cnt_r1 << 32) | cnt_r0; |
113 | *packets = ((u64)cnt_r3 << 32) | cnt_r2; |
114 | } else { |
115 | /* 48 bit byte counter, 40 bit packet counter */ |
116 | u32 byte_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R0_BYTE_CNT_LOW, cnt_r0); |
117 | u32 byte_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R1_BYTE_CNT_HIGH, cnt_r1); |
118 | u32 pkt_cnt_low = FIELD_GET(MTK_PPE_MIB_SER_R1_PKT_CNT_LOW, cnt_r1); |
119 | u32 pkt_cnt_high = FIELD_GET(MTK_PPE_MIB_SER_R2_PKT_CNT_HIGH, cnt_r2); |
120 | *bytes = ((u64)byte_cnt_high << 32) | byte_cnt_low; |
121 | *packets = ((u64)pkt_cnt_high << 16) | pkt_cnt_low; |
122 | } |
123 | |
124 | return 0; |
125 | } |
126 | |
127 | static void mtk_ppe_cache_clear(struct mtk_ppe *ppe) |
128 | { |
129 | ppe_set(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR); |
130 | ppe_clear(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR); |
131 | } |
132 | |
133 | static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable) |
134 | { |
135 | mtk_ppe_cache_clear(ppe); |
136 | |
137 | ppe_m32(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_EN, |
138 | set: enable * MTK_PPE_CACHE_CTL_EN); |
139 | } |
140 | |
141 | static u32 mtk_ppe_hash_entry(struct mtk_eth *eth, struct mtk_foe_entry *e) |
142 | { |
143 | u32 hv1, hv2, hv3; |
144 | u32 hash; |
145 | |
146 | switch (mtk_get_ib1_pkt_type(eth, val: e->ib1)) { |
147 | case MTK_PPE_PKT_TYPE_IPV4_ROUTE: |
148 | case MTK_PPE_PKT_TYPE_IPV4_HNAPT: |
149 | hv1 = e->ipv4.orig.ports; |
150 | hv2 = e->ipv4.orig.dest_ip; |
151 | hv3 = e->ipv4.orig.src_ip; |
152 | break; |
153 | case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T: |
154 | case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T: |
155 | hv1 = e->ipv6.src_ip[3] ^ e->ipv6.dest_ip[3]; |
156 | hv1 ^= e->ipv6.ports; |
157 | |
158 | hv2 = e->ipv6.src_ip[2] ^ e->ipv6.dest_ip[2]; |
159 | hv2 ^= e->ipv6.dest_ip[0]; |
160 | |
161 | hv3 = e->ipv6.src_ip[1] ^ e->ipv6.dest_ip[1]; |
162 | hv3 ^= e->ipv6.src_ip[0]; |
163 | break; |
164 | case MTK_PPE_PKT_TYPE_IPV4_DSLITE: |
165 | case MTK_PPE_PKT_TYPE_IPV6_6RD: |
166 | default: |
167 | WARN_ON_ONCE(1); |
168 | return MTK_PPE_HASH_MASK; |
169 | } |
170 | |
171 | hash = (hv1 & hv2) | ((~hv1) & hv3); |
172 | hash = (hash >> 24) | ((hash & 0xffffff) << 8); |
173 | hash ^= hv1 ^ hv2 ^ hv3; |
174 | hash ^= hash >> 16; |
175 | hash <<= (ffs(eth->soc->hash_offset) - 1); |
176 | hash &= MTK_PPE_ENTRIES - 1; |
177 | |
178 | return hash; |
179 | } |
180 | |
181 | static inline struct mtk_foe_mac_info * |
182 | mtk_foe_entry_l2(struct mtk_eth *eth, struct mtk_foe_entry *entry) |
183 | { |
184 | int type = mtk_get_ib1_pkt_type(eth, val: entry->ib1); |
185 | |
186 | if (type == MTK_PPE_PKT_TYPE_BRIDGE) |
187 | return &entry->bridge.l2; |
188 | |
189 | if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) |
190 | return &entry->ipv6.l2; |
191 | |
192 | return &entry->ipv4.l2; |
193 | } |
194 | |
195 | static inline u32 * |
196 | mtk_foe_entry_ib2(struct mtk_eth *eth, struct mtk_foe_entry *entry) |
197 | { |
198 | int type = mtk_get_ib1_pkt_type(eth, val: entry->ib1); |
199 | |
200 | if (type == MTK_PPE_PKT_TYPE_BRIDGE) |
201 | return &entry->bridge.ib2; |
202 | |
203 | if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) |
204 | return &entry->ipv6.ib2; |
205 | |
206 | return &entry->ipv4.ib2; |
207 | } |
208 | |
209 | int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry, |
210 | int type, int l4proto, u8 pse_port, u8 *src_mac, |
211 | u8 *dest_mac) |
212 | { |
213 | struct mtk_foe_mac_info *l2; |
214 | u32 ports_pad, val; |
215 | |
216 | memset(entry, 0, sizeof(*entry)); |
217 | |
218 | if (mtk_is_netsys_v2_or_greater(eth)) { |
219 | val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | |
220 | FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE_V2, type) | |
221 | FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | |
222 | MTK_FOE_IB1_BIND_CACHE_V2 | MTK_FOE_IB1_BIND_TTL_V2; |
223 | entry->ib1 = val; |
224 | |
225 | val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, pse_port) | |
226 | FIELD_PREP(MTK_FOE_IB2_PORT_AG_V2, 0xf); |
227 | } else { |
228 | int port_mg = eth->soc->offload_version > 1 ? 0 : 0x3f; |
229 | |
230 | val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | |
231 | FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) | |
232 | FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | |
233 | MTK_FOE_IB1_BIND_CACHE | MTK_FOE_IB1_BIND_TTL; |
234 | entry->ib1 = val; |
235 | |
236 | val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port) | |
237 | FIELD_PREP(MTK_FOE_IB2_PORT_MG, port_mg) | |
238 | FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f); |
239 | } |
240 | |
241 | if (is_multicast_ether_addr(addr: dest_mac)) |
242 | val |= mtk_get_ib2_multicast_mask(eth); |
243 | |
244 | ports_pad = 0xa5a5a500 | (l4proto & 0xff); |
245 | if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE) |
246 | entry->ipv4.orig.ports = ports_pad; |
247 | if (type == MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T) |
248 | entry->ipv6.ports = ports_pad; |
249 | |
250 | if (type == MTK_PPE_PKT_TYPE_BRIDGE) { |
251 | ether_addr_copy(dst: entry->bridge.src_mac, src: src_mac); |
252 | ether_addr_copy(dst: entry->bridge.dest_mac, src: dest_mac); |
253 | entry->bridge.ib2 = val; |
254 | l2 = &entry->bridge.l2; |
255 | } else if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) { |
256 | entry->ipv6.ib2 = val; |
257 | l2 = &entry->ipv6.l2; |
258 | } else { |
259 | entry->ipv4.ib2 = val; |
260 | l2 = &entry->ipv4.l2; |
261 | } |
262 | |
263 | l2->dest_mac_hi = get_unaligned_be32(p: dest_mac); |
264 | l2->dest_mac_lo = get_unaligned_be16(p: dest_mac + 4); |
265 | l2->src_mac_hi = get_unaligned_be32(p: src_mac); |
266 | l2->src_mac_lo = get_unaligned_be16(p: src_mac + 4); |
267 | |
268 | if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T) |
269 | l2->etype = ETH_P_IPV6; |
270 | else |
271 | l2->etype = ETH_P_IP; |
272 | |
273 | return 0; |
274 | } |
275 | |
276 | int mtk_foe_entry_set_pse_port(struct mtk_eth *eth, |
277 | struct mtk_foe_entry *entry, u8 port) |
278 | { |
279 | u32 *ib2 = mtk_foe_entry_ib2(eth, entry); |
280 | u32 val = *ib2; |
281 | |
282 | if (mtk_is_netsys_v2_or_greater(eth)) { |
283 | val &= ~MTK_FOE_IB2_DEST_PORT_V2; |
284 | val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, port); |
285 | } else { |
286 | val &= ~MTK_FOE_IB2_DEST_PORT; |
287 | val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port); |
288 | } |
289 | *ib2 = val; |
290 | |
291 | return 0; |
292 | } |
293 | |
294 | int mtk_foe_entry_set_ipv4_tuple(struct mtk_eth *eth, |
295 | struct mtk_foe_entry *entry, bool egress, |
296 | __be32 src_addr, __be16 src_port, |
297 | __be32 dest_addr, __be16 dest_port) |
298 | { |
299 | int type = mtk_get_ib1_pkt_type(eth, val: entry->ib1); |
300 | struct mtk_ipv4_tuple *t; |
301 | |
302 | switch (type) { |
303 | case MTK_PPE_PKT_TYPE_IPV4_HNAPT: |
304 | if (egress) { |
305 | t = &entry->ipv4.new; |
306 | break; |
307 | } |
308 | fallthrough; |
309 | case MTK_PPE_PKT_TYPE_IPV4_DSLITE: |
310 | case MTK_PPE_PKT_TYPE_IPV4_ROUTE: |
311 | t = &entry->ipv4.orig; |
312 | break; |
313 | case MTK_PPE_PKT_TYPE_IPV6_6RD: |
314 | entry->ipv6_6rd.tunnel_src_ip = be32_to_cpu(src_addr); |
315 | entry->ipv6_6rd.tunnel_dest_ip = be32_to_cpu(dest_addr); |
316 | return 0; |
317 | default: |
318 | WARN_ON_ONCE(1); |
319 | return -EINVAL; |
320 | } |
321 | |
322 | t->src_ip = be32_to_cpu(src_addr); |
323 | t->dest_ip = be32_to_cpu(dest_addr); |
324 | |
325 | if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE) |
326 | return 0; |
327 | |
328 | t->src_port = be16_to_cpu(src_port); |
329 | t->dest_port = be16_to_cpu(dest_port); |
330 | |
331 | return 0; |
332 | } |
333 | |
334 | int mtk_foe_entry_set_ipv6_tuple(struct mtk_eth *eth, |
335 | struct mtk_foe_entry *entry, |
336 | __be32 *src_addr, __be16 src_port, |
337 | __be32 *dest_addr, __be16 dest_port) |
338 | { |
339 | int type = mtk_get_ib1_pkt_type(eth, val: entry->ib1); |
340 | u32 *src, *dest; |
341 | int i; |
342 | |
343 | switch (type) { |
344 | case MTK_PPE_PKT_TYPE_IPV4_DSLITE: |
345 | src = entry->dslite.tunnel_src_ip; |
346 | dest = entry->dslite.tunnel_dest_ip; |
347 | break; |
348 | case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T: |
349 | case MTK_PPE_PKT_TYPE_IPV6_6RD: |
350 | entry->ipv6.src_port = be16_to_cpu(src_port); |
351 | entry->ipv6.dest_port = be16_to_cpu(dest_port); |
352 | fallthrough; |
353 | case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T: |
354 | src = entry->ipv6.src_ip; |
355 | dest = entry->ipv6.dest_ip; |
356 | break; |
357 | default: |
358 | WARN_ON_ONCE(1); |
359 | return -EINVAL; |
360 | } |
361 | |
362 | for (i = 0; i < 4; i++) |
363 | src[i] = be32_to_cpu(src_addr[i]); |
364 | for (i = 0; i < 4; i++) |
365 | dest[i] = be32_to_cpu(dest_addr[i]); |
366 | |
367 | return 0; |
368 | } |
369 | |
370 | int mtk_foe_entry_set_dsa(struct mtk_eth *eth, struct mtk_foe_entry *entry, |
371 | int port) |
372 | { |
373 | struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); |
374 | |
375 | l2->etype = BIT(port); |
376 | |
377 | if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth))) |
378 | entry->ib1 |= mtk_prep_ib1_vlan_layer(eth, val: 1); |
379 | else |
380 | l2->etype |= BIT(8); |
381 | |
382 | entry->ib1 &= ~mtk_get_ib1_vlan_tag_mask(eth); |
383 | |
384 | return 0; |
385 | } |
386 | |
387 | int mtk_foe_entry_set_vlan(struct mtk_eth *eth, struct mtk_foe_entry *entry, |
388 | int vid) |
389 | { |
390 | struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); |
391 | |
392 | switch (mtk_get_ib1_vlan_layer(eth, val: entry->ib1)) { |
393 | case 0: |
394 | entry->ib1 |= mtk_get_ib1_vlan_tag_mask(eth) | |
395 | mtk_prep_ib1_vlan_layer(eth, val: 1); |
396 | l2->vlan1 = vid; |
397 | return 0; |
398 | case 1: |
399 | if (!(entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) { |
400 | l2->vlan1 = vid; |
401 | l2->etype |= BIT(8); |
402 | } else { |
403 | l2->vlan2 = vid; |
404 | entry->ib1 += mtk_prep_ib1_vlan_layer(eth, val: 1); |
405 | } |
406 | return 0; |
407 | default: |
408 | return -ENOSPC; |
409 | } |
410 | } |
411 | |
412 | int mtk_foe_entry_set_pppoe(struct mtk_eth *eth, struct mtk_foe_entry *entry, |
413 | int sid) |
414 | { |
415 | struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); |
416 | |
417 | if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)) || |
418 | (entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) |
419 | l2->etype = ETH_P_PPP_SES; |
420 | |
421 | entry->ib1 |= mtk_get_ib1_ppoe_mask(eth); |
422 | l2->pppoe_id = sid; |
423 | |
424 | return 0; |
425 | } |
426 | |
427 | int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry, |
428 | int wdma_idx, int txq, int bss, int wcid, |
429 | bool amsdu_en) |
430 | { |
431 | struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); |
432 | u32 *ib2 = mtk_foe_entry_ib2(eth, entry); |
433 | |
434 | switch (eth->soc->version) { |
435 | case 3: |
436 | *ib2 &= ~MTK_FOE_IB2_PORT_MG_V2; |
437 | *ib2 |= FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) | |
438 | MTK_FOE_IB2_WDMA_WINFO_V2; |
439 | l2->w3info = FIELD_PREP(MTK_FOE_WINFO_WCID_V3, wcid) | |
440 | FIELD_PREP(MTK_FOE_WINFO_BSS_V3, bss); |
441 | l2->amsdu = FIELD_PREP(MTK_FOE_WINFO_AMSDU_EN, amsdu_en); |
442 | break; |
443 | case 2: |
444 | *ib2 &= ~MTK_FOE_IB2_PORT_MG_V2; |
445 | *ib2 |= FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) | |
446 | MTK_FOE_IB2_WDMA_WINFO_V2; |
447 | l2->winfo = FIELD_PREP(MTK_FOE_WINFO_WCID, wcid) | |
448 | FIELD_PREP(MTK_FOE_WINFO_BSS, bss); |
449 | break; |
450 | default: |
451 | *ib2 &= ~MTK_FOE_IB2_PORT_MG; |
452 | *ib2 |= MTK_FOE_IB2_WDMA_WINFO; |
453 | if (wdma_idx) |
454 | *ib2 |= MTK_FOE_IB2_WDMA_DEVIDX; |
455 | l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) | |
456 | FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) | |
457 | FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq); |
458 | break; |
459 | } |
460 | |
461 | return 0; |
462 | } |
463 | |
464 | int mtk_foe_entry_set_queue(struct mtk_eth *eth, struct mtk_foe_entry *entry, |
465 | unsigned int queue) |
466 | { |
467 | u32 *ib2 = mtk_foe_entry_ib2(eth, entry); |
468 | |
469 | if (mtk_is_netsys_v2_or_greater(eth)) { |
470 | *ib2 &= ~MTK_FOE_IB2_QID_V2; |
471 | *ib2 |= FIELD_PREP(MTK_FOE_IB2_QID_V2, queue); |
472 | *ib2 |= MTK_FOE_IB2_PSE_QOS_V2; |
473 | } else { |
474 | *ib2 &= ~MTK_FOE_IB2_QID; |
475 | *ib2 |= FIELD_PREP(MTK_FOE_IB2_QID, queue); |
476 | *ib2 |= MTK_FOE_IB2_PSE_QOS; |
477 | } |
478 | |
479 | return 0; |
480 | } |
481 | |
482 | static bool |
483 | mtk_flow_entry_match(struct mtk_eth *eth, struct mtk_flow_entry *entry, |
484 | struct mtk_foe_entry *data) |
485 | { |
486 | int type, len; |
487 | |
488 | if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP) |
489 | return false; |
490 | |
491 | type = mtk_get_ib1_pkt_type(eth, val: entry->data.ib1); |
492 | if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE) |
493 | len = offsetof(struct mtk_foe_entry, ipv6._rsv); |
494 | else |
495 | len = offsetof(struct mtk_foe_entry, ipv4.ib2); |
496 | |
497 | return !memcmp(p: &entry->data.data, q: &data->data, size: len - 4); |
498 | } |
499 | |
500 | static void |
501 | __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) |
502 | { |
503 | struct hlist_head *head; |
504 | struct hlist_node *tmp; |
505 | |
506 | if (entry->type == MTK_FLOW_TYPE_L2) { |
507 | rhashtable_remove_fast(ht: &ppe->l2_flows, obj: &entry->l2_node, |
508 | params: mtk_flow_l2_ht_params); |
509 | |
510 | head = &entry->l2_flows; |
511 | hlist_for_each_entry_safe(entry, tmp, head, l2_data.list) |
512 | __mtk_foe_entry_clear(ppe, entry); |
513 | return; |
514 | } |
515 | |
516 | hlist_del_init(n: &entry->list); |
517 | if (entry->hash != 0xffff) { |
518 | struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash: entry->hash); |
519 | |
520 | hwe->ib1 &= ~MTK_FOE_IB1_STATE; |
521 | hwe->ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_INVALID); |
522 | dma_wmb(); |
523 | mtk_ppe_cache_clear(ppe); |
524 | |
525 | if (ppe->accounting) { |
526 | struct mtk_foe_accounting *acct; |
527 | |
528 | acct = ppe->acct_table + entry->hash * sizeof(*acct); |
529 | acct->packets = 0; |
530 | acct->bytes = 0; |
531 | } |
532 | } |
533 | entry->hash = 0xffff; |
534 | |
535 | if (entry->type != MTK_FLOW_TYPE_L2_SUBFLOW) |
536 | return; |
537 | |
538 | hlist_del_init(n: &entry->l2_data.list); |
539 | kfree(objp: entry); |
540 | } |
541 | |
542 | static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1) |
543 | { |
544 | u32 ib1_ts_mask = mtk_get_ib1_ts_mask(eth: ppe->eth); |
545 | u16 now = mtk_eth_timestamp(eth: ppe->eth); |
546 | u16 timestamp = ib1 & ib1_ts_mask; |
547 | |
548 | if (timestamp > now) |
549 | return ib1_ts_mask + 1 - timestamp + now; |
550 | else |
551 | return now - timestamp; |
552 | } |
553 | |
554 | static void |
555 | mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) |
556 | { |
557 | u32 ib1_ts_mask = mtk_get_ib1_ts_mask(eth: ppe->eth); |
558 | struct mtk_flow_entry *cur; |
559 | struct mtk_foe_entry *hwe; |
560 | struct hlist_node *tmp; |
561 | int idle; |
562 | |
563 | idle = __mtk_foe_entry_idle_time(ppe, ib1: entry->data.ib1); |
564 | hlist_for_each_entry_safe(cur, tmp, &entry->l2_flows, l2_data.list) { |
565 | int cur_idle; |
566 | u32 ib1; |
567 | |
568 | hwe = mtk_foe_get_entry(ppe, hash: cur->hash); |
569 | ib1 = READ_ONCE(hwe->ib1); |
570 | |
571 | if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) { |
572 | cur->hash = 0xffff; |
573 | __mtk_foe_entry_clear(ppe, entry: cur); |
574 | continue; |
575 | } |
576 | |
577 | cur_idle = __mtk_foe_entry_idle_time(ppe, ib1); |
578 | if (cur_idle >= idle) |
579 | continue; |
580 | |
581 | idle = cur_idle; |
582 | entry->data.ib1 &= ~ib1_ts_mask; |
583 | entry->data.ib1 |= hwe->ib1 & ib1_ts_mask; |
584 | } |
585 | } |
586 | |
587 | static void |
588 | mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) |
589 | { |
590 | struct mtk_foe_entry foe = {}; |
591 | struct mtk_foe_entry *hwe; |
592 | |
593 | spin_lock_bh(lock: &ppe_lock); |
594 | |
595 | if (entry->type == MTK_FLOW_TYPE_L2) { |
596 | mtk_flow_entry_update_l2(ppe, entry); |
597 | goto out; |
598 | } |
599 | |
600 | if (entry->hash == 0xffff) |
601 | goto out; |
602 | |
603 | hwe = mtk_foe_get_entry(ppe, hash: entry->hash); |
604 | memcpy(&foe, hwe, ppe->eth->soc->foe_entry_size); |
605 | if (!mtk_flow_entry_match(eth: ppe->eth, entry, data: &foe)) { |
606 | entry->hash = 0xffff; |
607 | goto out; |
608 | } |
609 | |
610 | entry->data.ib1 = foe.ib1; |
611 | |
612 | out: |
613 | spin_unlock_bh(lock: &ppe_lock); |
614 | } |
615 | |
616 | static void |
617 | __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, |
618 | u16 hash) |
619 | { |
620 | struct mtk_eth *eth = ppe->eth; |
621 | u16 timestamp = mtk_eth_timestamp(eth); |
622 | struct mtk_foe_entry *hwe; |
623 | u32 val; |
624 | |
625 | if (mtk_is_netsys_v2_or_greater(eth)) { |
626 | entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP_V2; |
627 | entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP_V2, |
628 | timestamp); |
629 | } else { |
630 | entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP; |
631 | entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, |
632 | timestamp); |
633 | } |
634 | |
635 | hwe = mtk_foe_get_entry(ppe, hash); |
636 | memcpy(&hwe->data, &entry->data, eth->soc->foe_entry_size - sizeof(hwe->ib1)); |
637 | wmb(); |
638 | hwe->ib1 = entry->ib1; |
639 | |
640 | if (ppe->accounting) { |
641 | if (mtk_is_netsys_v2_or_greater(eth)) |
642 | val = MTK_FOE_IB2_MIB_CNT_V2; |
643 | else |
644 | val = MTK_FOE_IB2_MIB_CNT; |
645 | *mtk_foe_entry_ib2(eth, entry: hwe) |= val; |
646 | } |
647 | |
648 | dma_wmb(); |
649 | |
650 | mtk_ppe_cache_clear(ppe); |
651 | } |
652 | |
653 | void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) |
654 | { |
655 | spin_lock_bh(lock: &ppe_lock); |
656 | __mtk_foe_entry_clear(ppe, entry); |
657 | spin_unlock_bh(lock: &ppe_lock); |
658 | } |
659 | |
660 | static int |
661 | mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) |
662 | { |
663 | struct mtk_flow_entry *prev; |
664 | |
665 | entry->type = MTK_FLOW_TYPE_L2; |
666 | |
667 | prev = rhashtable_lookup_get_insert_fast(ht: &ppe->l2_flows, obj: &entry->l2_node, |
668 | params: mtk_flow_l2_ht_params); |
669 | if (likely(!prev)) |
670 | return 0; |
671 | |
672 | if (IS_ERR(ptr: prev)) |
673 | return PTR_ERR(ptr: prev); |
674 | |
675 | return rhashtable_replace_fast(ht: &ppe->l2_flows, obj_old: &prev->l2_node, |
676 | obj_new: &entry->l2_node, params: mtk_flow_l2_ht_params); |
677 | } |
678 | |
679 | int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) |
680 | { |
681 | const struct mtk_soc_data *soc = ppe->eth->soc; |
682 | int type = mtk_get_ib1_pkt_type(eth: ppe->eth, val: entry->data.ib1); |
683 | u32 hash; |
684 | |
685 | if (type == MTK_PPE_PKT_TYPE_BRIDGE) |
686 | return mtk_foe_entry_commit_l2(ppe, entry); |
687 | |
688 | hash = mtk_ppe_hash_entry(eth: ppe->eth, e: &entry->data); |
689 | entry->hash = 0xffff; |
690 | spin_lock_bh(lock: &ppe_lock); |
691 | hlist_add_head(n: &entry->list, h: &ppe->foe_flow[hash / soc->hash_offset]); |
692 | spin_unlock_bh(lock: &ppe_lock); |
693 | |
694 | return 0; |
695 | } |
696 | |
697 | static void |
698 | mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry, |
699 | u16 hash) |
700 | { |
701 | const struct mtk_soc_data *soc = ppe->eth->soc; |
702 | struct mtk_flow_entry *flow_info; |
703 | struct mtk_foe_entry foe = {}, *hwe; |
704 | struct mtk_foe_mac_info *l2; |
705 | u32 ib1_mask = mtk_get_ib1_pkt_type_mask(eth: ppe->eth) | MTK_FOE_IB1_UDP; |
706 | int type; |
707 | |
708 | flow_info = kzalloc(size: sizeof(*flow_info), GFP_ATOMIC); |
709 | if (!flow_info) |
710 | return; |
711 | |
712 | flow_info->l2_data.base_flow = entry; |
713 | flow_info->type = MTK_FLOW_TYPE_L2_SUBFLOW; |
714 | flow_info->hash = hash; |
715 | hlist_add_head(n: &flow_info->list, |
716 | h: &ppe->foe_flow[hash / soc->hash_offset]); |
717 | hlist_add_head(n: &flow_info->l2_data.list, h: &entry->l2_flows); |
718 | |
719 | hwe = mtk_foe_get_entry(ppe, hash); |
720 | memcpy(&foe, hwe, soc->foe_entry_size); |
721 | foe.ib1 &= ib1_mask; |
722 | foe.ib1 |= entry->data.ib1 & ~ib1_mask; |
723 | |
724 | l2 = mtk_foe_entry_l2(eth: ppe->eth, entry: &foe); |
725 | memcpy(l2, &entry->data.bridge.l2, sizeof(*l2)); |
726 | |
727 | type = mtk_get_ib1_pkt_type(eth: ppe->eth, val: foe.ib1); |
728 | if (type == MTK_PPE_PKT_TYPE_IPV4_HNAPT) |
729 | memcpy(&foe.ipv4.new, &foe.ipv4.orig, sizeof(foe.ipv4.new)); |
730 | else if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T && l2->etype == ETH_P_IP) |
731 | l2->etype = ETH_P_IPV6; |
732 | |
733 | *mtk_foe_entry_ib2(eth: ppe->eth, entry: &foe) = entry->data.bridge.ib2; |
734 | |
735 | __mtk_foe_entry_commit(ppe, entry: &foe, hash); |
736 | } |
737 | |
738 | void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash) |
739 | { |
740 | const struct mtk_soc_data *soc = ppe->eth->soc; |
741 | struct hlist_head *head = &ppe->foe_flow[hash / soc->hash_offset]; |
742 | struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash); |
743 | struct mtk_flow_entry *entry; |
744 | struct mtk_foe_bridge key = {}; |
745 | struct hlist_node *n; |
746 | struct ethhdr *eh; |
747 | bool found = false; |
748 | u8 *tag; |
749 | |
750 | spin_lock_bh(lock: &ppe_lock); |
751 | |
752 | if (FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == MTK_FOE_STATE_BIND) |
753 | goto out; |
754 | |
755 | hlist_for_each_entry_safe(entry, n, head, list) { |
756 | if (entry->type == MTK_FLOW_TYPE_L2_SUBFLOW) { |
757 | if (unlikely(FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == |
758 | MTK_FOE_STATE_BIND)) |
759 | continue; |
760 | |
761 | entry->hash = 0xffff; |
762 | __mtk_foe_entry_clear(ppe, entry); |
763 | continue; |
764 | } |
765 | |
766 | if (found || !mtk_flow_entry_match(eth: ppe->eth, entry, data: hwe)) { |
767 | if (entry->hash != 0xffff) |
768 | entry->hash = 0xffff; |
769 | continue; |
770 | } |
771 | |
772 | entry->hash = hash; |
773 | __mtk_foe_entry_commit(ppe, entry: &entry->data, hash); |
774 | found = true; |
775 | } |
776 | |
777 | if (found) |
778 | goto out; |
779 | |
780 | eh = eth_hdr(skb); |
781 | ether_addr_copy(dst: key.dest_mac, src: eh->h_dest); |
782 | ether_addr_copy(dst: key.src_mac, src: eh->h_source); |
783 | tag = skb->data - 2; |
784 | key.vlan = 0; |
785 | switch (skb->protocol) { |
786 | #if IS_ENABLED(CONFIG_NET_DSA) |
787 | case htons(ETH_P_XDSA): |
788 | if (!netdev_uses_dsa(dev: skb->dev) || |
789 | skb->dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK) |
790 | goto out; |
791 | |
792 | if (!skb_metadata_dst(skb)) |
793 | tag += 4; |
794 | |
795 | if (get_unaligned_be16(p: tag) != ETH_P_8021Q) |
796 | break; |
797 | |
798 | fallthrough; |
799 | #endif |
800 | case htons(ETH_P_8021Q): |
801 | key.vlan = get_unaligned_be16(p: tag + 2) & VLAN_VID_MASK; |
802 | break; |
803 | default: |
804 | break; |
805 | } |
806 | |
807 | entry = rhashtable_lookup_fast(ht: &ppe->l2_flows, key: &key, params: mtk_flow_l2_ht_params); |
808 | if (!entry) |
809 | goto out; |
810 | |
811 | mtk_foe_entry_commit_subflow(ppe, entry, hash); |
812 | |
813 | out: |
814 | spin_unlock_bh(lock: &ppe_lock); |
815 | } |
816 | |
817 | int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) |
818 | { |
819 | mtk_flow_entry_update(ppe, entry); |
820 | |
821 | return __mtk_foe_entry_idle_time(ppe, ib1: entry->data.ib1); |
822 | } |
823 | |
824 | int mtk_ppe_prepare_reset(struct mtk_ppe *ppe) |
825 | { |
826 | if (!ppe) |
827 | return -EINVAL; |
828 | |
829 | /* disable KA */ |
830 | ppe_clear(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_KEEPALIVE); |
831 | ppe_clear(ppe, MTK_PPE_BIND_LMT1, MTK_PPE_NTU_KEEPALIVE); |
832 | ppe_w32(ppe, MTK_PPE_KEEPALIVE, val: 0); |
833 | usleep_range(min: 10000, max: 11000); |
834 | |
835 | /* set KA timer to maximum */ |
836 | ppe_set(ppe, MTK_PPE_BIND_LMT1, MTK_PPE_NTU_KEEPALIVE); |
837 | ppe_w32(ppe, MTK_PPE_KEEPALIVE, val: 0xffffffff); |
838 | |
839 | /* set KA tick select */ |
840 | ppe_set(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_TICK_SEL); |
841 | ppe_set(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_KEEPALIVE); |
842 | usleep_range(min: 10000, max: 11000); |
843 | |
844 | /* disable scan mode */ |
845 | ppe_clear(ppe, MTK_PPE_TB_CFG, MTK_PPE_TB_CFG_SCAN_MODE); |
846 | usleep_range(min: 10000, max: 11000); |
847 | |
848 | return mtk_ppe_wait_busy(ppe); |
849 | } |
850 | |
851 | struct mtk_foe_accounting *mtk_foe_entry_get_mib(struct mtk_ppe *ppe, u32 index, |
852 | struct mtk_foe_accounting *diff) |
853 | { |
854 | struct mtk_foe_accounting *acct; |
855 | int size = sizeof(struct mtk_foe_accounting); |
856 | u64 bytes, packets; |
857 | |
858 | if (!ppe->accounting) |
859 | return NULL; |
860 | |
861 | if (mtk_mib_entry_read(ppe, index, bytes: &bytes, packets: &packets)) |
862 | return NULL; |
863 | |
864 | acct = ppe->acct_table + index * size; |
865 | |
866 | acct->bytes += bytes; |
867 | acct->packets += packets; |
868 | |
869 | if (diff) { |
870 | diff->bytes = bytes; |
871 | diff->packets = packets; |
872 | } |
873 | |
874 | return acct; |
875 | } |
876 | |
877 | struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int index) |
878 | { |
879 | bool accounting = eth->soc->has_accounting; |
880 | const struct mtk_soc_data *soc = eth->soc; |
881 | struct mtk_foe_accounting *acct; |
882 | struct device *dev = eth->dev; |
883 | struct mtk_mib_entry *mib; |
884 | struct mtk_ppe *ppe; |
885 | u32 foe_flow_size; |
886 | void *foe; |
887 | |
888 | ppe = devm_kzalloc(dev, size: sizeof(*ppe), GFP_KERNEL); |
889 | if (!ppe) |
890 | return NULL; |
891 | |
892 | rhashtable_init(ht: &ppe->l2_flows, params: &mtk_flow_l2_ht_params); |
893 | |
894 | /* need to allocate a separate device, since it PPE DMA access is |
895 | * not coherent. |
896 | */ |
897 | ppe->base = base; |
898 | ppe->eth = eth; |
899 | ppe->dev = dev; |
900 | ppe->version = eth->soc->offload_version; |
901 | ppe->accounting = accounting; |
902 | |
903 | foe = dmam_alloc_coherent(dev: ppe->dev, |
904 | MTK_PPE_ENTRIES * soc->foe_entry_size, |
905 | dma_handle: &ppe->foe_phys, GFP_KERNEL); |
906 | if (!foe) |
907 | goto err_free_l2_flows; |
908 | |
909 | ppe->foe_table = foe; |
910 | |
911 | foe_flow_size = (MTK_PPE_ENTRIES / soc->hash_offset) * |
912 | sizeof(*ppe->foe_flow); |
913 | ppe->foe_flow = devm_kzalloc(dev, size: foe_flow_size, GFP_KERNEL); |
914 | if (!ppe->foe_flow) |
915 | goto err_free_l2_flows; |
916 | |
917 | if (accounting) { |
918 | mib = dmam_alloc_coherent(dev: ppe->dev, MTK_PPE_ENTRIES * sizeof(*mib), |
919 | dma_handle: &ppe->mib_phys, GFP_KERNEL); |
920 | if (!mib) |
921 | return NULL; |
922 | |
923 | ppe->mib_table = mib; |
924 | |
925 | acct = devm_kzalloc(dev, MTK_PPE_ENTRIES * sizeof(*acct), |
926 | GFP_KERNEL); |
927 | |
928 | if (!acct) |
929 | return NULL; |
930 | |
931 | ppe->acct_table = acct; |
932 | } |
933 | |
934 | mtk_ppe_debugfs_init(ppe, index); |
935 | |
936 | return ppe; |
937 | |
938 | err_free_l2_flows: |
939 | rhashtable_destroy(ht: &ppe->l2_flows); |
940 | return NULL; |
941 | } |
942 | |
943 | void mtk_ppe_deinit(struct mtk_eth *eth) |
944 | { |
945 | int i; |
946 | |
947 | for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) { |
948 | if (!eth->ppe[i]) |
949 | return; |
950 | rhashtable_destroy(ht: ð->ppe[i]->l2_flows); |
951 | } |
952 | } |
953 | |
954 | static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe) |
955 | { |
956 | static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 }; |
957 | int i, k; |
958 | |
959 | memset(ppe->foe_table, 0, |
960 | MTK_PPE_ENTRIES * ppe->eth->soc->foe_entry_size); |
961 | |
962 | if (!IS_ENABLED(CONFIG_SOC_MT7621)) |
963 | return; |
964 | |
965 | /* skip all entries that cross the 1024 byte boundary */ |
966 | for (i = 0; i < MTK_PPE_ENTRIES; i += 128) { |
967 | for (k = 0; k < ARRAY_SIZE(skip); k++) { |
968 | struct mtk_foe_entry *hwe; |
969 | |
970 | hwe = mtk_foe_get_entry(ppe, hash: i + skip[k]); |
971 | hwe->ib1 |= MTK_FOE_IB1_STATIC; |
972 | } |
973 | } |
974 | } |
975 | |
976 | void mtk_ppe_start(struct mtk_ppe *ppe) |
977 | { |
978 | u32 val; |
979 | |
980 | if (!ppe) |
981 | return; |
982 | |
983 | mtk_ppe_init_foe_table(ppe); |
984 | ppe_w32(ppe, MTK_PPE_TB_BASE, val: ppe->foe_phys); |
985 | |
986 | val = MTK_PPE_TB_CFG_AGE_NON_L4 | |
987 | MTK_PPE_TB_CFG_AGE_UNBIND | |
988 | MTK_PPE_TB_CFG_AGE_TCP | |
989 | MTK_PPE_TB_CFG_AGE_UDP | |
990 | MTK_PPE_TB_CFG_AGE_TCP_FIN | |
991 | FIELD_PREP(MTK_PPE_TB_CFG_SEARCH_MISS, |
992 | MTK_PPE_SEARCH_MISS_ACTION_FORWARD_BUILD) | |
993 | FIELD_PREP(MTK_PPE_TB_CFG_KEEPALIVE, |
994 | MTK_PPE_KEEPALIVE_DISABLE) | |
995 | FIELD_PREP(MTK_PPE_TB_CFG_HASH_MODE, 1) | |
996 | FIELD_PREP(MTK_PPE_TB_CFG_SCAN_MODE, |
997 | MTK_PPE_SCAN_MODE_CHECK_AGE) | |
998 | FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM, |
999 | MTK_PPE_ENTRIES_SHIFT); |
1000 | if (mtk_is_netsys_v2_or_greater(eth: ppe->eth)) |
1001 | val |= MTK_PPE_TB_CFG_INFO_SEL; |
1002 | if (!mtk_is_netsys_v3_or_greater(eth: ppe->eth)) |
1003 | val |= MTK_PPE_TB_CFG_ENTRY_80B; |
1004 | ppe_w32(ppe, MTK_PPE_TB_CFG, val); |
1005 | |
1006 | ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK, |
1007 | MTK_PPE_IP_PROTO_CHK_IPV4 | MTK_PPE_IP_PROTO_CHK_IPV6); |
1008 | |
1009 | mtk_ppe_cache_enable(ppe, enable: true); |
1010 | |
1011 | val = MTK_PPE_FLOW_CFG_IP6_3T_ROUTE | |
1012 | MTK_PPE_FLOW_CFG_IP6_5T_ROUTE | |
1013 | MTK_PPE_FLOW_CFG_IP6_6RD | |
1014 | MTK_PPE_FLOW_CFG_IP4_NAT | |
1015 | MTK_PPE_FLOW_CFG_IP4_NAPT | |
1016 | MTK_PPE_FLOW_CFG_IP4_DSLITE | |
1017 | MTK_PPE_FLOW_CFG_IP4_NAT_FRAG; |
1018 | if (mtk_is_netsys_v2_or_greater(eth: ppe->eth)) |
1019 | val |= MTK_PPE_MD_TOAP_BYP_CRSN0 | |
1020 | MTK_PPE_MD_TOAP_BYP_CRSN1 | |
1021 | MTK_PPE_MD_TOAP_BYP_CRSN2 | |
1022 | MTK_PPE_FLOW_CFG_IP4_HASH_GRE_KEY; |
1023 | else |
1024 | val |= MTK_PPE_FLOW_CFG_IP4_TCP_FRAG | |
1025 | MTK_PPE_FLOW_CFG_IP4_UDP_FRAG; |
1026 | ppe_w32(ppe, MTK_PPE_FLOW_CFG, val); |
1027 | |
1028 | val = FIELD_PREP(MTK_PPE_UNBIND_AGE_MIN_PACKETS, 1000) | |
1029 | FIELD_PREP(MTK_PPE_UNBIND_AGE_DELTA, 3); |
1030 | ppe_w32(ppe, MTK_PPE_UNBIND_AGE, val); |
1031 | |
1032 | val = FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_UDP, 12) | |
1033 | FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_NON_L4, 1); |
1034 | ppe_w32(ppe, MTK_PPE_BIND_AGE0, val); |
1035 | |
1036 | val = FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP_FIN, 1) | |
1037 | FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP, 7); |
1038 | ppe_w32(ppe, MTK_PPE_BIND_AGE1, val); |
1039 | |
1040 | val = MTK_PPE_BIND_LIMIT0_QUARTER | MTK_PPE_BIND_LIMIT0_HALF; |
1041 | ppe_w32(ppe, MTK_PPE_BIND_LIMIT0, val); |
1042 | |
1043 | val = MTK_PPE_BIND_LIMIT1_FULL | |
1044 | FIELD_PREP(MTK_PPE_BIND_LIMIT1_NON_L4, 1); |
1045 | ppe_w32(ppe, MTK_PPE_BIND_LIMIT1, val); |
1046 | |
1047 | val = FIELD_PREP(MTK_PPE_BIND_RATE_BIND, 30) | |
1048 | FIELD_PREP(MTK_PPE_BIND_RATE_PREBIND, 1); |
1049 | ppe_w32(ppe, MTK_PPE_BIND_RATE, val); |
1050 | |
1051 | /* enable PPE */ |
1052 | val = MTK_PPE_GLO_CFG_EN | |
1053 | MTK_PPE_GLO_CFG_IP4_L4_CS_DROP | |
1054 | MTK_PPE_GLO_CFG_IP4_CS_DROP | |
1055 | MTK_PPE_GLO_CFG_FLOW_DROP_UPDATE; |
1056 | ppe_w32(ppe, MTK_PPE_GLO_CFG, val); |
1057 | |
1058 | ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, val: 0); |
1059 | |
1060 | if (mtk_is_netsys_v2_or_greater(eth: ppe->eth)) { |
1061 | ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, val: 0xcb777); |
1062 | ppe_w32(ppe, MTK_PPE_SBW_CTRL, val: 0x7f); |
1063 | } |
1064 | |
1065 | if (ppe->accounting && ppe->mib_phys) { |
1066 | ppe_w32(ppe, MTK_PPE_MIB_TB_BASE, val: ppe->mib_phys); |
1067 | ppe_m32(ppe, MTK_PPE_MIB_CFG, MTK_PPE_MIB_CFG_EN, |
1068 | MTK_PPE_MIB_CFG_EN); |
1069 | ppe_m32(ppe, MTK_PPE_MIB_CFG, MTK_PPE_MIB_CFG_RD_CLR, |
1070 | MTK_PPE_MIB_CFG_RD_CLR); |
1071 | ppe_m32(ppe, MTK_PPE_MIB_CACHE_CTL, MTK_PPE_MIB_CACHE_CTL_EN, |
1072 | MTK_PPE_MIB_CFG_RD_CLR); |
1073 | } |
1074 | } |
1075 | |
1076 | int mtk_ppe_stop(struct mtk_ppe *ppe) |
1077 | { |
1078 | u32 val; |
1079 | int i; |
1080 | |
1081 | if (!ppe) |
1082 | return 0; |
1083 | |
1084 | for (i = 0; i < MTK_PPE_ENTRIES; i++) { |
1085 | struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash: i); |
1086 | |
1087 | hwe->ib1 = FIELD_PREP(MTK_FOE_IB1_STATE, |
1088 | MTK_FOE_STATE_INVALID); |
1089 | } |
1090 | |
1091 | mtk_ppe_cache_enable(ppe, enable: false); |
1092 | |
1093 | /* disable aging */ |
1094 | val = MTK_PPE_TB_CFG_AGE_NON_L4 | |
1095 | MTK_PPE_TB_CFG_AGE_UNBIND | |
1096 | MTK_PPE_TB_CFG_AGE_TCP | |
1097 | MTK_PPE_TB_CFG_AGE_UDP | |
1098 | MTK_PPE_TB_CFG_AGE_TCP_FIN | |
1099 | MTK_PPE_TB_CFG_SCAN_MODE; |
1100 | ppe_clear(ppe, MTK_PPE_TB_CFG, val); |
1101 | |
1102 | if (mtk_ppe_wait_busy(ppe)) |
1103 | return -ETIMEDOUT; |
1104 | |
1105 | /* disable offload engine */ |
1106 | ppe_clear(ppe, MTK_PPE_GLO_CFG, MTK_PPE_GLO_CFG_EN); |
1107 | ppe_w32(ppe, MTK_PPE_FLOW_CFG, val: 0); |
1108 | |
1109 | return 0; |
1110 | } |
1111 | |