1 | // SPDX-License-Identifier: GPL-2.0 |
2 | |
3 | /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. |
4 | * Copyright (C) 2018-2023 Linaro Ltd. |
5 | */ |
6 | |
7 | #include <linux/types.h> |
8 | #include <linux/kernel.h> |
9 | #include <linux/bits.h> |
10 | #include <linux/bitops.h> |
11 | #include <linux/bitfield.h> |
12 | #include <linux/io.h> |
13 | #include <linux/build_bug.h> |
14 | #include <linux/device.h> |
15 | #include <linux/dma-mapping.h> |
16 | |
17 | #include "ipa.h" |
18 | #include "ipa_version.h" |
19 | #include "ipa_endpoint.h" |
20 | #include "ipa_table.h" |
21 | #include "ipa_reg.h" |
22 | #include "ipa_mem.h" |
23 | #include "ipa_cmd.h" |
24 | #include "gsi.h" |
25 | #include "gsi_trans.h" |
26 | |
27 | /** |
28 | * DOC: IPA Filter and Route Tables |
29 | * |
30 | * The IPA has tables defined in its local (IPA-resident) memory that define |
31 | * filter and routing rules. An entry in either of these tables is a little |
32 | * endian 64-bit "slot" that holds the address of a rule definition. (The |
33 | * size of these slots is 64 bits regardless of the host DMA address size.) |
34 | * |
35 | * Separate tables (both filter and route) are used for IPv4 and IPv6. There |
36 | * is normally another set of "hashed" filter and route tables, which are |
37 | * used with a hash of message metadata. Hashed operation is not supported |
38 | * by all IPA hardware (IPA v4.2 doesn't support hashed tables). |
39 | * |
40 | * Rules can be in local memory or in DRAM (system memory). The offset of |
41 | * an object (such as a route or filter table) in IPA-resident memory must |
42 | * 128-byte aligned. An object in system memory (such as a route or filter |
43 | * rule) must be at an 8-byte aligned address. We currently only place |
44 | * route or filter rules in system memory. |
45 | * |
46 | * A rule consists of a contiguous block of 32-bit values terminated with |
47 | * 32 zero bits. A special "zero entry" rule consisting of 64 zero bits |
48 | * represents "no filtering" or "no routing," and is the reset value for |
49 | * filter or route table rules. |
50 | * |
51 | * Each filter rule is associated with an AP or modem TX endpoint, though |
52 | * not all TX endpoints support filtering. The first 64-bit slot in a |
53 | * filter table is a bitmap indicating which endpoints have entries in |
54 | * the table. Each set bit in this bitmap indicates the presence of the |
55 | * address of a filter rule in the memory following the bitmap. Until IPA |
56 | * v5.0, the low-order bit (bit 0) in this bitmap represents a special |
57 | * global filter, which applies to all traffic. Otherwise the position of |
58 | * each set bit represents an endpoint for which a filter rule is defined. |
59 | * |
60 | * The global rule is not used in current code, and support for it is |
61 | * removed starting at IPA v5.0. For IPA v5.0+, the endpoint bitmap |
62 | * position defines the endpoint ID--i.e. if bit 1 is set in the endpoint |
63 | * bitmap, endpoint 1 has a filter rule. Older versions of IPA represent |
64 | * the presence of a filter rule for endpoint X by bit (X + 1) being set. |
65 | * I.e., bit 1 set indicates the presence of a filter rule for endpoint 0, |
66 | * and bit 3 set means there is a filter rule present for endpoint 2. |
67 | * |
68 | * Each filter table entry has the address of a set of equations that |
69 | * implement a filter rule. So following the endpoint bitmap there |
70 | * will be such an address/entry for each endpoint with a set bit in |
71 | * the bitmap. |
72 | * |
73 | * The AP initializes all entries in a filter table to refer to a "zero" |
74 | * rule. Once initialized, the modem and AP update the entries for |
75 | * endpoints they "own" directly. Currently the AP does not use the IPA |
76 | * filtering functionality. |
77 | * |
78 | * This diagram shows an example of a filter table with an endpoint |
79 | * bitmap as defined prior to IPA v5.0. |
80 | * |
81 | * IPA Filter Table |
82 | * ---------------------- |
83 | * endpoint bitmap | 0x0000000000000048 | Bits 3 and 6 set (endpoints 2 and 5) |
84 | * |--------------------| |
85 | * 1st endpoint | 0x000123456789abc0 | DMA address for modem endpoint 2 rule |
86 | * |--------------------| |
87 | * 2nd endpoint | 0x000123456789abf0 | DMA address for AP endpoint 5 rule |
88 | * |--------------------| |
89 | * (unused) | | (Unused space in filter table) |
90 | * |--------------------| |
91 | * . . . |
92 | * |--------------------| |
93 | * (unused) | | (Unused space in filter table) |
94 | * ---------------------- |
95 | * |
96 | * The set of available route rules is divided about equally between the AP |
97 | * and modem. The AP initializes all entries in a route table to refer to |
98 | * a "zero entry". Once initialized, the modem and AP are responsible for |
99 | * updating their own entries. All entries in a route table are usable, |
100 | * though the AP currently does not use the IPA routing functionality. |
101 | * |
102 | * IPA Route Table |
103 | * ---------------------- |
104 | * 1st modem route | 0x0001234500001100 | DMA address for first route rule |
105 | * |--------------------| |
106 | * 2nd modem route | 0x0001234500001140 | DMA address for second route rule |
107 | * |--------------------| |
108 | * . . . |
109 | * |--------------------| |
110 | * Last modem route| 0x0001234500002280 | DMA address for Nth route rule |
111 | * |--------------------| |
112 | * 1st AP route | 0x0001234500001100 | DMA address for route rule (N+1) |
113 | * |--------------------| |
114 | * 2nd AP route | 0x0001234500001140 | DMA address for next route rule |
115 | * |--------------------| |
116 | * . . . |
117 | * |--------------------| |
118 | * Last AP route | 0x0001234500002280 | DMA address for last route rule |
119 | * ---------------------- |
120 | */ |
121 | |
122 | /* Filter or route rules consist of a set of 32-bit values followed by a |
123 | * 32-bit all-zero rule list terminator. The "zero rule" is simply an |
124 | * all-zero rule followed by the list terminator. |
125 | */ |
126 | #define IPA_ZERO_RULE_SIZE (2 * sizeof(__le32)) |
127 | |
128 | /* Check things that can be validated at build time. */ |
129 | static void ipa_table_validate_build(void) |
130 | { |
131 | /* Filter and route tables contain DMA addresses that refer |
132 | * to filter or route rules. But the size of a table entry |
133 | * is 64 bits regardless of what the size of an AP DMA address |
134 | * is. A fixed constant defines the size of an entry, and |
135 | * code in ipa_table_init() uses a pointer to __le64 to |
136 | * initialize tables. |
137 | */ |
138 | BUILD_BUG_ON(sizeof(dma_addr_t) > sizeof(__le64)); |
139 | |
140 | /* A "zero rule" is used to represent no filtering or no routing. |
141 | * It is a 64-bit block of zeroed memory. Code in ipa_table_init() |
142 | * assumes that it can be written using a pointer to __le64. |
143 | */ |
144 | BUILD_BUG_ON(IPA_ZERO_RULE_SIZE != sizeof(__le64)); |
145 | } |
146 | |
147 | static const struct ipa_mem * |
148 | ipa_table_mem(struct ipa *ipa, bool filter, bool hashed, bool ipv6) |
149 | { |
150 | enum ipa_mem_id mem_id; |
151 | |
152 | mem_id = filter ? hashed ? ipv6 ? IPA_MEM_V6_FILTER_HASHED |
153 | : IPA_MEM_V4_FILTER_HASHED |
154 | : ipv6 ? IPA_MEM_V6_FILTER |
155 | : IPA_MEM_V4_FILTER |
156 | : hashed ? ipv6 ? IPA_MEM_V6_ROUTE_HASHED |
157 | : IPA_MEM_V4_ROUTE_HASHED |
158 | : ipv6 ? IPA_MEM_V6_ROUTE |
159 | : IPA_MEM_V4_ROUTE; |
160 | |
161 | return ipa_mem_find(ipa, mem_id); |
162 | } |
163 | |
164 | bool ipa_filtered_valid(struct ipa *ipa, u64 filtered) |
165 | { |
166 | struct device *dev = ipa->dev; |
167 | u32 count; |
168 | |
169 | if (!filtered) { |
170 | dev_err(dev, "at least one filtering endpoint is required\n" ); |
171 | |
172 | return false; |
173 | } |
174 | |
175 | count = hweight64(filtered); |
176 | if (count > ipa->filter_count) { |
177 | dev_err(dev, "too many filtering endpoints (%u > %u)\n" , |
178 | count, ipa->filter_count); |
179 | |
180 | return false; |
181 | } |
182 | |
183 | return true; |
184 | } |
185 | |
186 | /* Zero entry count means no table, so just return a 0 address */ |
187 | static dma_addr_t ipa_table_addr(struct ipa *ipa, bool filter_mask, u16 count) |
188 | { |
189 | u32 skip; |
190 | |
191 | if (!count) |
192 | return 0; |
193 | |
194 | WARN_ON(count > max_t(u32, ipa->filter_count, ipa->route_count)); |
195 | |
196 | /* Skip over the zero rule and possibly the filter mask */ |
197 | skip = filter_mask ? 1 : 2; |
198 | |
199 | return ipa->table_addr + skip * sizeof(*ipa->table_virt); |
200 | } |
201 | |
202 | static void ipa_table_reset_add(struct gsi_trans *trans, bool filter, |
203 | bool hashed, bool ipv6, u16 first, u16 count) |
204 | { |
205 | struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi); |
206 | const struct ipa_mem *mem; |
207 | dma_addr_t addr; |
208 | u32 offset; |
209 | u16 size; |
210 | |
211 | /* Nothing to do if the memory region is doesn't exist or is empty */ |
212 | mem = ipa_table_mem(ipa, filter, hashed, ipv6); |
213 | if (!mem || !mem->size) |
214 | return; |
215 | |
216 | if (filter) |
217 | first++; /* skip over bitmap */ |
218 | |
219 | offset = mem->offset + first * sizeof(__le64); |
220 | size = count * sizeof(__le64); |
221 | addr = ipa_table_addr(ipa, filter_mask: false, count); |
222 | |
223 | ipa_cmd_dma_shared_mem_add(trans, offset, size, addr, toward_ipa: true); |
224 | } |
225 | |
226 | /* Reset entries in a single filter table belonging to either the AP or |
227 | * modem to refer to the zero entry. The memory region supplied will be |
228 | * for the IPv4 and IPv6 non-hashed and hashed filter tables. |
229 | */ |
230 | static int |
231 | ipa_filter_reset_table(struct ipa *ipa, bool hashed, bool ipv6, bool modem) |
232 | { |
233 | u64 ep_mask = ipa->filtered; |
234 | struct gsi_trans *trans; |
235 | enum gsi_ee_id ee_id; |
236 | |
237 | trans = ipa_cmd_trans_alloc(ipa, hweight64(ep_mask)); |
238 | if (!trans) { |
239 | dev_err(ipa->dev, "no transaction for %s filter reset\n" , |
240 | modem ? "modem" : "AP" ); |
241 | return -EBUSY; |
242 | } |
243 | |
244 | ee_id = modem ? GSI_EE_MODEM : GSI_EE_AP; |
245 | while (ep_mask) { |
246 | u32 endpoint_id = __ffs(ep_mask); |
247 | struct ipa_endpoint *endpoint; |
248 | |
249 | ep_mask ^= BIT(endpoint_id); |
250 | |
251 | endpoint = &ipa->endpoint[endpoint_id]; |
252 | if (endpoint->ee_id != ee_id) |
253 | continue; |
254 | |
255 | ipa_table_reset_add(trans, filter: true, hashed, ipv6, first: endpoint_id, count: 1); |
256 | } |
257 | |
258 | gsi_trans_commit_wait(trans); |
259 | |
260 | return 0; |
261 | } |
262 | |
263 | /* Theoretically, each filter table could have more filter slots to |
264 | * update than the maximum number of commands in a transaction. So |
265 | * we do each table separately. |
266 | */ |
267 | static int ipa_filter_reset(struct ipa *ipa, bool modem) |
268 | { |
269 | int ret; |
270 | |
271 | ret = ipa_filter_reset_table(ipa, hashed: false, ipv6: false, modem); |
272 | if (ret) |
273 | return ret; |
274 | |
275 | ret = ipa_filter_reset_table(ipa, hashed: false, ipv6: true, modem); |
276 | if (ret || !ipa_table_hash_support(ipa)) |
277 | return ret; |
278 | |
279 | ret = ipa_filter_reset_table(ipa, hashed: true, ipv6: false, modem); |
280 | if (ret) |
281 | return ret; |
282 | |
283 | return ipa_filter_reset_table(ipa, hashed: true, ipv6: true, modem); |
284 | } |
285 | |
286 | /* The AP routes and modem routes are each contiguous within the |
287 | * table. We can update each table with a single command, and we |
288 | * won't exceed the per-transaction command limit. |
289 | * */ |
290 | static int ipa_route_reset(struct ipa *ipa, bool modem) |
291 | { |
292 | bool hash_support = ipa_table_hash_support(ipa); |
293 | u32 modem_route_count = ipa->modem_route_count; |
294 | struct gsi_trans *trans; |
295 | u16 first; |
296 | u16 count; |
297 | |
298 | trans = ipa_cmd_trans_alloc(ipa, tre_count: hash_support ? 4 : 2); |
299 | if (!trans) { |
300 | dev_err(ipa->dev, "no transaction for %s route reset\n" , |
301 | modem ? "modem" : "AP" ); |
302 | return -EBUSY; |
303 | } |
304 | |
305 | if (modem) { |
306 | first = 0; |
307 | count = modem_route_count; |
308 | } else { |
309 | first = modem_route_count; |
310 | count = ipa->route_count - modem_route_count; |
311 | } |
312 | |
313 | ipa_table_reset_add(trans, filter: false, hashed: false, ipv6: false, first, count); |
314 | ipa_table_reset_add(trans, filter: false, hashed: false, ipv6: true, first, count); |
315 | |
316 | if (hash_support) { |
317 | ipa_table_reset_add(trans, filter: false, hashed: true, ipv6: false, first, count); |
318 | ipa_table_reset_add(trans, filter: false, hashed: true, ipv6: true, first, count); |
319 | } |
320 | |
321 | gsi_trans_commit_wait(trans); |
322 | |
323 | return 0; |
324 | } |
325 | |
326 | void ipa_table_reset(struct ipa *ipa, bool modem) |
327 | { |
328 | struct device *dev = ipa->dev; |
329 | const char *ee_name; |
330 | int ret; |
331 | |
332 | ee_name = modem ? "modem" : "AP" ; |
333 | |
334 | /* Report errors, but reset filter and route tables */ |
335 | ret = ipa_filter_reset(ipa, modem); |
336 | if (ret) |
337 | dev_err(dev, "error %d resetting filter table for %s\n" , |
338 | ret, ee_name); |
339 | |
340 | ret = ipa_route_reset(ipa, modem); |
341 | if (ret) |
342 | dev_err(dev, "error %d resetting route table for %s\n" , |
343 | ret, ee_name); |
344 | } |
345 | |
346 | int ipa_table_hash_flush(struct ipa *ipa) |
347 | { |
348 | struct gsi_trans *trans; |
349 | const struct reg *reg; |
350 | u32 val; |
351 | |
352 | if (!ipa_table_hash_support(ipa)) |
353 | return 0; |
354 | |
355 | trans = ipa_cmd_trans_alloc(ipa, tre_count: 1); |
356 | if (!trans) { |
357 | dev_err(ipa->dev, "no transaction for hash flush\n" ); |
358 | return -EBUSY; |
359 | } |
360 | |
361 | if (ipa->version < IPA_VERSION_5_0) { |
362 | reg = ipa_reg(ipa, reg_id: FILT_ROUT_HASH_FLUSH); |
363 | |
364 | val = reg_bit(reg, field_id: IPV6_ROUTER_HASH); |
365 | val |= reg_bit(reg, field_id: IPV6_FILTER_HASH); |
366 | val |= reg_bit(reg, field_id: IPV4_ROUTER_HASH); |
367 | val |= reg_bit(reg, field_id: IPV4_FILTER_HASH); |
368 | } else { |
369 | reg = ipa_reg(ipa, reg_id: FILT_ROUT_CACHE_FLUSH); |
370 | |
371 | /* IPA v5.0+ uses a unified cache (both IPv4 and IPv6) */ |
372 | val = reg_bit(reg, field_id: ROUTER_CACHE); |
373 | val |= reg_bit(reg, field_id: FILTER_CACHE); |
374 | } |
375 | |
376 | ipa_cmd_register_write_add(trans, offset: reg_offset(reg), value: val, mask: val, clear_full: false); |
377 | |
378 | gsi_trans_commit_wait(trans); |
379 | |
380 | return 0; |
381 | } |
382 | |
383 | static void ipa_table_init_add(struct gsi_trans *trans, bool filter, bool ipv6) |
384 | { |
385 | struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi); |
386 | const struct ipa_mem *hash_mem; |
387 | enum ipa_cmd_opcode opcode; |
388 | const struct ipa_mem *mem; |
389 | dma_addr_t hash_addr; |
390 | dma_addr_t addr; |
391 | u32 hash_offset; |
392 | u32 zero_offset; |
393 | u16 hash_count; |
394 | u32 zero_size; |
395 | u16 hash_size; |
396 | u16 count; |
397 | u16 size; |
398 | |
399 | opcode = filter ? ipv6 ? IPA_CMD_IP_V6_FILTER_INIT |
400 | : IPA_CMD_IP_V4_FILTER_INIT |
401 | : ipv6 ? IPA_CMD_IP_V6_ROUTING_INIT |
402 | : IPA_CMD_IP_V4_ROUTING_INIT; |
403 | |
404 | /* The non-hashed region will exist (see ipa_table_mem_valid()) */ |
405 | mem = ipa_table_mem(ipa, filter, hashed: false, ipv6); |
406 | hash_mem = ipa_table_mem(ipa, filter, hashed: true, ipv6); |
407 | hash_offset = hash_mem ? hash_mem->offset : 0; |
408 | |
409 | /* Compute the number of table entries to initialize */ |
410 | if (filter) { |
411 | /* The number of filtering endpoints determines number of |
412 | * entries in the filter table; we also add one more "slot" |
413 | * to hold the bitmap itself. The size of the hashed filter |
414 | * table is either the same as the non-hashed one, or zero. |
415 | */ |
416 | count = 1 + hweight64(ipa->filtered); |
417 | hash_count = hash_mem && hash_mem->size ? count : 0; |
418 | } else { |
419 | /* The size of a route table region determines the number |
420 | * of entries it has. |
421 | */ |
422 | count = mem->size / sizeof(__le64); |
423 | hash_count = hash_mem ? hash_mem->size / sizeof(__le64) : 0; |
424 | } |
425 | size = count * sizeof(__le64); |
426 | hash_size = hash_count * sizeof(__le64); |
427 | |
428 | addr = ipa_table_addr(ipa, filter_mask: filter, count); |
429 | hash_addr = ipa_table_addr(ipa, filter_mask: filter, count: hash_count); |
430 | |
431 | ipa_cmd_table_init_add(trans, opcode, size, offset: mem->offset, addr, |
432 | hash_size, hash_offset, hash_addr); |
433 | if (!filter) |
434 | return; |
435 | |
436 | /* Zero the unused space in the filter table */ |
437 | zero_offset = mem->offset + size; |
438 | zero_size = mem->size - size; |
439 | ipa_cmd_dma_shared_mem_add(trans, offset: zero_offset, size: zero_size, |
440 | addr: ipa->zero_addr, toward_ipa: true); |
441 | if (!hash_size) |
442 | return; |
443 | |
444 | /* Zero the unused space in the hashed filter table */ |
445 | zero_offset = hash_offset + hash_size; |
446 | zero_size = hash_mem->size - hash_size; |
447 | ipa_cmd_dma_shared_mem_add(trans, offset: zero_offset, size: zero_size, |
448 | addr: ipa->zero_addr, toward_ipa: true); |
449 | } |
450 | |
451 | int ipa_table_setup(struct ipa *ipa) |
452 | { |
453 | struct gsi_trans *trans; |
454 | |
455 | /* We will need at most 8 TREs: |
456 | * - IPv4: |
457 | * - One for route table initialization (non-hashed and hashed) |
458 | * - One for filter table initialization (non-hashed and hashed) |
459 | * - One to zero unused entries in the non-hashed filter table |
460 | * - One to zero unused entries in the hashed filter table |
461 | * - IPv6: |
462 | * - One for route table initialization (non-hashed and hashed) |
463 | * - One for filter table initialization (non-hashed and hashed) |
464 | * - One to zero unused entries in the non-hashed filter table |
465 | * - One to zero unused entries in the hashed filter table |
466 | * All platforms support at least 8 TREs in a transaction. |
467 | */ |
468 | trans = ipa_cmd_trans_alloc(ipa, tre_count: 8); |
469 | if (!trans) { |
470 | dev_err(ipa->dev, "no transaction for table setup\n" ); |
471 | return -EBUSY; |
472 | } |
473 | |
474 | ipa_table_init_add(trans, filter: false, ipv6: false); |
475 | ipa_table_init_add(trans, filter: false, ipv6: true); |
476 | ipa_table_init_add(trans, filter: true, ipv6: false); |
477 | ipa_table_init_add(trans, filter: true, ipv6: true); |
478 | |
479 | gsi_trans_commit_wait(trans); |
480 | |
481 | return 0; |
482 | } |
483 | |
484 | /** |
485 | * ipa_filter_tuple_zero() - Zero an endpoint's hashed filter tuple |
486 | * @endpoint: Endpoint whose filter hash tuple should be zeroed |
487 | * |
488 | * Endpoint must be for the AP (not modem) and support filtering. Updates |
489 | * the filter hash values without changing route ones. |
490 | */ |
491 | static void ipa_filter_tuple_zero(struct ipa_endpoint *endpoint) |
492 | { |
493 | u32 endpoint_id = endpoint->endpoint_id; |
494 | struct ipa *ipa = endpoint->ipa; |
495 | const struct reg *reg; |
496 | u32 offset; |
497 | u32 val; |
498 | |
499 | if (ipa->version < IPA_VERSION_5_0) { |
500 | reg = ipa_reg(ipa, reg_id: ENDP_FILTER_ROUTER_HSH_CFG); |
501 | |
502 | offset = reg_n_offset(reg, n: endpoint_id); |
503 | val = ioread32(endpoint->ipa->reg_virt + offset); |
504 | |
505 | /* Zero all filter-related fields, preserving the rest */ |
506 | val &= ~reg_fmask(reg, field_id: FILTER_HASH_MSK_ALL); |
507 | } else { |
508 | /* IPA v5.0 separates filter and router cache configuration */ |
509 | reg = ipa_reg(ipa, reg_id: ENDP_FILTER_CACHE_CFG); |
510 | offset = reg_n_offset(reg, n: endpoint_id); |
511 | |
512 | /* Zero all filter-related fields */ |
513 | val = 0; |
514 | } |
515 | |
516 | iowrite32(val, endpoint->ipa->reg_virt + offset); |
517 | } |
518 | |
519 | /* Configure a hashed filter table; there is no ipa_filter_deconfig() */ |
520 | static void ipa_filter_config(struct ipa *ipa, bool modem) |
521 | { |
522 | enum gsi_ee_id ee_id = modem ? GSI_EE_MODEM : GSI_EE_AP; |
523 | u64 ep_mask = ipa->filtered; |
524 | |
525 | if (!ipa_table_hash_support(ipa)) |
526 | return; |
527 | |
528 | while (ep_mask) { |
529 | u32 endpoint_id = __ffs(ep_mask); |
530 | struct ipa_endpoint *endpoint; |
531 | |
532 | ep_mask ^= BIT(endpoint_id); |
533 | |
534 | endpoint = &ipa->endpoint[endpoint_id]; |
535 | if (endpoint->ee_id == ee_id) |
536 | ipa_filter_tuple_zero(endpoint); |
537 | } |
538 | } |
539 | |
540 | static bool ipa_route_id_modem(struct ipa *ipa, u32 route_id) |
541 | { |
542 | return route_id < ipa->modem_route_count; |
543 | } |
544 | |
545 | /** |
546 | * ipa_route_tuple_zero() - Zero a hashed route table entry tuple |
547 | * @ipa: IPA pointer |
548 | * @route_id: Route table entry whose hash tuple should be zeroed |
549 | * |
550 | * Updates the route hash values without changing filter ones. |
551 | */ |
552 | static void ipa_route_tuple_zero(struct ipa *ipa, u32 route_id) |
553 | { |
554 | const struct reg *reg; |
555 | u32 offset; |
556 | u32 val; |
557 | |
558 | if (ipa->version < IPA_VERSION_5_0) { |
559 | reg = ipa_reg(ipa, reg_id: ENDP_FILTER_ROUTER_HSH_CFG); |
560 | offset = reg_n_offset(reg, n: route_id); |
561 | |
562 | val = ioread32(ipa->reg_virt + offset); |
563 | |
564 | /* Zero all route-related fields, preserving the rest */ |
565 | val &= ~reg_fmask(reg, field_id: ROUTER_HASH_MSK_ALL); |
566 | } else { |
567 | /* IPA v5.0 separates filter and router cache configuration */ |
568 | reg = ipa_reg(ipa, reg_id: ENDP_ROUTER_CACHE_CFG); |
569 | offset = reg_n_offset(reg, n: route_id); |
570 | |
571 | /* Zero all route-related fields */ |
572 | val = 0; |
573 | } |
574 | |
575 | iowrite32(val, ipa->reg_virt + offset); |
576 | } |
577 | |
578 | /* Configure a hashed route table; there is no ipa_route_deconfig() */ |
579 | static void ipa_route_config(struct ipa *ipa, bool modem) |
580 | { |
581 | u32 route_id; |
582 | |
583 | if (!ipa_table_hash_support(ipa)) |
584 | return; |
585 | |
586 | for (route_id = 0; route_id < ipa->route_count; route_id++) |
587 | if (ipa_route_id_modem(ipa, route_id) == modem) |
588 | ipa_route_tuple_zero(ipa, route_id); |
589 | } |
590 | |
591 | /* Configure a filter and route tables; there is no ipa_table_deconfig() */ |
592 | void ipa_table_config(struct ipa *ipa) |
593 | { |
594 | ipa_filter_config(ipa, modem: false); |
595 | ipa_filter_config(ipa, modem: true); |
596 | ipa_route_config(ipa, modem: false); |
597 | ipa_route_config(ipa, modem: true); |
598 | } |
599 | |
600 | /* Verify the sizes of all IPA table filter or routing table memory regions |
601 | * are valid. If valid, this records the size of the routing table. |
602 | */ |
603 | bool ipa_table_mem_valid(struct ipa *ipa, bool filter) |
604 | { |
605 | bool hash_support = ipa_table_hash_support(ipa); |
606 | const struct ipa_mem *mem_hashed; |
607 | const struct ipa_mem *mem_ipv4; |
608 | const struct ipa_mem *mem_ipv6; |
609 | u32 count; |
610 | |
611 | /* IPv4 and IPv6 non-hashed tables are expected to be defined and |
612 | * have the same size. Both must have at least two entries (and |
613 | * would normally have more than that). |
614 | */ |
615 | mem_ipv4 = ipa_table_mem(ipa, filter, hashed: false, ipv6: false); |
616 | if (!mem_ipv4) |
617 | return false; |
618 | |
619 | mem_ipv6 = ipa_table_mem(ipa, filter, hashed: false, ipv6: true); |
620 | if (!mem_ipv6) |
621 | return false; |
622 | |
623 | if (mem_ipv4->size != mem_ipv6->size) |
624 | return false; |
625 | |
626 | /* Compute and record the number of entries for each table type */ |
627 | count = mem_ipv4->size / sizeof(__le64); |
628 | if (count < 2) |
629 | return false; |
630 | if (filter) |
631 | ipa->filter_count = count - 1; /* Filter map in first entry */ |
632 | else |
633 | ipa->route_count = count; |
634 | |
635 | /* Table offset and size must fit in TABLE_INIT command fields */ |
636 | if (!ipa_cmd_table_init_valid(ipa, mem: mem_ipv4, route: !filter)) |
637 | return false; |
638 | |
639 | /* Make sure the regions are big enough */ |
640 | if (filter) { |
641 | /* Filter tables must able to hold the endpoint bitmap plus |
642 | * an entry for each endpoint that supports filtering |
643 | */ |
644 | if (count < 1 + hweight64(ipa->filtered)) |
645 | return false; |
646 | } else { |
647 | /* Routing tables must be able to hold all modem entries, |
648 | * plus at least one entry for the AP. |
649 | */ |
650 | if (count < ipa->modem_route_count + 1) |
651 | return false; |
652 | } |
653 | |
654 | /* If hashing is supported, hashed tables are expected to be defined, |
655 | * and have the same size as non-hashed tables. If hashing is not |
656 | * supported, hashed tables are expected to have zero size (or not |
657 | * be defined). |
658 | */ |
659 | mem_hashed = ipa_table_mem(ipa, filter, hashed: true, ipv6: false); |
660 | if (hash_support) { |
661 | if (!mem_hashed || mem_hashed->size != mem_ipv4->size) |
662 | return false; |
663 | } else { |
664 | if (mem_hashed && mem_hashed->size) |
665 | return false; |
666 | } |
667 | |
668 | /* Same check for IPv6 tables */ |
669 | mem_hashed = ipa_table_mem(ipa, filter, hashed: true, ipv6: true); |
670 | if (hash_support) { |
671 | if (!mem_hashed || mem_hashed->size != mem_ipv6->size) |
672 | return false; |
673 | } else { |
674 | if (mem_hashed && mem_hashed->size) |
675 | return false; |
676 | } |
677 | |
678 | return true; |
679 | } |
680 | |
681 | /* Initialize a coherent DMA allocation containing initialized filter and |
682 | * route table data. This is used when initializing or resetting the IPA |
683 | * filter or route table. |
684 | * |
685 | * The first entry in a filter table contains a bitmap indicating which |
686 | * endpoints contain entries in the table. In addition to that first entry, |
687 | * there is a fixed maximum number of entries that follow. Filter table |
688 | * entries are 64 bits wide, and (other than the bitmap) contain the DMA |
689 | * address of a filter rule. A "zero rule" indicates no filtering, and |
690 | * consists of 64 bits of zeroes. When a filter table is initialized (or |
691 | * reset) its entries are made to refer to the zero rule. |
692 | * |
693 | * Each entry in a route table is the DMA address of a routing rule. For |
694 | * routing there is also a 64-bit "zero rule" that means no routing, and |
695 | * when a route table is initialized or reset, its entries are made to refer |
696 | * to the zero rule. The zero rule is shared for route and filter tables. |
697 | * |
698 | * +-------------------+ |
699 | * --> | zero rule | |
700 | * / |-------------------| |
701 | * | | filter mask | |
702 | * |\ |-------------------| |
703 | * | ---- zero rule address | \ |
704 | * |\ |-------------------| | |
705 | * | ---- zero rule address | | Max IPA filter count |
706 | * | |-------------------| > or IPA route count, |
707 | * | ... | whichever is greater |
708 | * \ |-------------------| | |
709 | * ---- zero rule address | / |
710 | * +-------------------+ |
711 | */ |
712 | int ipa_table_init(struct ipa *ipa) |
713 | { |
714 | struct device *dev = ipa->dev; |
715 | dma_addr_t addr; |
716 | __le64 le_addr; |
717 | __le64 *virt; |
718 | size_t size; |
719 | u32 count; |
720 | |
721 | ipa_table_validate_build(); |
722 | |
723 | count = max_t(u32, ipa->filter_count, ipa->route_count); |
724 | |
725 | /* The IPA hardware requires route and filter table rules to be |
726 | * aligned on a 128-byte boundary. We put the "zero rule" at the |
727 | * base of the table area allocated here. The DMA address returned |
728 | * by dma_alloc_coherent() is guaranteed to be a power-of-2 number |
729 | * of pages, which satisfies the rule alignment requirement. |
730 | */ |
731 | size = IPA_ZERO_RULE_SIZE + (1 + count) * sizeof(__le64); |
732 | virt = dma_alloc_coherent(dev, size, dma_handle: &addr, GFP_KERNEL); |
733 | if (!virt) |
734 | return -ENOMEM; |
735 | |
736 | ipa->table_virt = virt; |
737 | ipa->table_addr = addr; |
738 | |
739 | /* First slot is the zero rule */ |
740 | *virt++ = 0; |
741 | |
742 | /* Next is the filter table bitmap. The "soft" bitmap value might |
743 | * need to be converted to the hardware representation by shifting |
744 | * it left one position. Prior to IPA v5.0, bit 0 repesents global |
745 | * filtering, which is possible but not used. IPA v5.0+ eliminated |
746 | * that option, so there's no shifting required. |
747 | */ |
748 | if (ipa->version < IPA_VERSION_5_0) |
749 | *virt++ = cpu_to_le64(ipa->filtered << 1); |
750 | else |
751 | *virt++ = cpu_to_le64(ipa->filtered); |
752 | |
753 | /* All the rest contain the DMA address of the zero rule */ |
754 | le_addr = cpu_to_le64(addr); |
755 | while (count--) |
756 | *virt++ = le_addr; |
757 | |
758 | return 0; |
759 | } |
760 | |
761 | void ipa_table_exit(struct ipa *ipa) |
762 | { |
763 | u32 count = max_t(u32, 1 + ipa->filter_count, ipa->route_count); |
764 | struct device *dev = ipa->dev; |
765 | size_t size; |
766 | |
767 | size = IPA_ZERO_RULE_SIZE + (1 + count) * sizeof(__le64); |
768 | |
769 | dma_free_coherent(dev, size, cpu_addr: ipa->table_virt, dma_handle: ipa->table_addr); |
770 | ipa->table_addr = 0; |
771 | ipa->table_virt = NULL; |
772 | } |
773 | |