1 | // SPDX-License-Identifier: GPL-2.0 |
2 | |
3 | /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. |
4 | * Copyright (C) 2019-2023 Linaro Ltd. |
5 | */ |
6 | |
7 | #include <linux/types.h> |
8 | #include <linux/device.h> |
9 | #include <linux/slab.h> |
10 | #include <linux/bitfield.h> |
11 | #include <linux/if_rmnet.h> |
12 | #include <linux/dma-direction.h> |
13 | |
14 | #include "gsi.h" |
15 | #include "gsi_trans.h" |
16 | #include "ipa.h" |
17 | #include "ipa_data.h" |
18 | #include "ipa_endpoint.h" |
19 | #include "ipa_cmd.h" |
20 | #include "ipa_mem.h" |
21 | #include "ipa_modem.h" |
22 | #include "ipa_table.h" |
23 | #include "ipa_gsi.h" |
24 | #include "ipa_power.h" |
25 | |
26 | /* Hardware is told about receive buffers once a "batch" has been queued */ |
27 | #define IPA_REPLENISH_BATCH 16 /* Must be non-zero */ |
28 | |
29 | /* The amount of RX buffer space consumed by standard skb overhead */ |
30 | #define IPA_RX_BUFFER_OVERHEAD (PAGE_SIZE - SKB_MAX_ORDER(NET_SKB_PAD, 0)) |
31 | |
32 | /* Where to find the QMAP mux_id for a packet within modem-supplied metadata */ |
33 | #define IPA_ENDPOINT_QMAP_METADATA_MASK 0x000000ff /* host byte order */ |
34 | |
35 | #define IPA_ENDPOINT_RESET_AGGR_RETRY_MAX 3 |
36 | |
37 | /** enum ipa_status_opcode - IPA status opcode field hardware values */ |
38 | enum ipa_status_opcode { /* *Not* a bitmask */ |
39 | IPA_STATUS_OPCODE_PACKET = 1, |
40 | IPA_STATUS_OPCODE_NEW_RULE_PACKET = 2, |
41 | IPA_STATUS_OPCODE_DROPPED_PACKET = 4, |
42 | IPA_STATUS_OPCODE_SUSPENDED_PACKET = 8, |
43 | IPA_STATUS_OPCODE_LOG = 16, |
44 | IPA_STATUS_OPCODE_DCMP = 32, |
45 | IPA_STATUS_OPCODE_PACKET_2ND_PASS = 64, |
46 | }; |
47 | |
48 | /** enum ipa_status_exception - IPA status exception field hardware values */ |
49 | enum ipa_status_exception { /* *Not* a bitmask */ |
50 | /* 0 means no exception */ |
51 | IPA_STATUS_EXCEPTION_DEAGGR = 1, |
52 | IPA_STATUS_EXCEPTION_IPTYPE = 4, |
53 | IPA_STATUS_EXCEPTION_PACKET_LENGTH = 8, |
54 | IPA_STATUS_EXCEPTION_FRAG_RULE_MISS = 16, |
55 | IPA_STATUS_EXCEPTION_SW_FILTER = 32, |
56 | IPA_STATUS_EXCEPTION_NAT = 64, /* IPv4 */ |
57 | IPA_STATUS_EXCEPTION_IPV6_CONN_TRACK = 64, /* IPv6 */ |
58 | IPA_STATUS_EXCEPTION_UC = 128, |
59 | IPA_STATUS_EXCEPTION_INVALID_ENDPOINT = 129, |
60 | = 136, |
61 | IPA_STATUS_EXCEPTION_CHEKCSUM = 229, |
62 | }; |
63 | |
64 | /** enum ipa_status_mask - IPA status mask field bitmask hardware values */ |
65 | enum ipa_status_mask { |
66 | IPA_STATUS_MASK_FRAG_PROCESS = BIT(0), |
67 | IPA_STATUS_MASK_FILT_PROCESS = BIT(1), |
68 | IPA_STATUS_MASK_NAT_PROCESS = BIT(2), |
69 | IPA_STATUS_MASK_ROUTE_PROCESS = BIT(3), |
70 | IPA_STATUS_MASK_TAG_VALID = BIT(4), |
71 | IPA_STATUS_MASK_FRAGMENT = BIT(5), |
72 | IPA_STATUS_MASK_FIRST_FRAGMENT = BIT(6), |
73 | IPA_STATUS_MASK_V4 = BIT(7), |
74 | IPA_STATUS_MASK_CKSUM_PROCESS = BIT(8), |
75 | IPA_STATUS_MASK_AGGR_PROCESS = BIT(9), |
76 | IPA_STATUS_MASK_DEST_EOT = BIT(10), |
77 | IPA_STATUS_MASK_DEAGGR_PROCESS = BIT(11), |
78 | IPA_STATUS_MASK_DEAGG_FIRST = BIT(12), |
79 | IPA_STATUS_MASK_SRC_EOT = BIT(13), |
80 | IPA_STATUS_MASK_PREV_EOT = BIT(14), |
81 | IPA_STATUS_MASK_BYTE_LIMIT = BIT(15), |
82 | }; |
83 | |
84 | /* Special IPA filter/router rule field value indicating "rule miss" */ |
85 | #define IPA_STATUS_RULE_MISS 0x3ff /* 10-bit filter/router rule fields */ |
86 | |
87 | /** The IPA status nat_type field uses enum ipa_nat_type hardware values */ |
88 | |
89 | /* enum ipa_status_field_id - IPA packet status structure field identifiers */ |
90 | enum ipa_status_field_id { |
91 | STATUS_OPCODE, /* enum ipa_status_opcode */ |
92 | STATUS_EXCEPTION, /* enum ipa_status_exception */ |
93 | STATUS_MASK, /* enum ipa_status_mask (bitmask) */ |
94 | STATUS_LENGTH, |
95 | STATUS_SRC_ENDPOINT, |
96 | STATUS_DST_ENDPOINT, |
97 | STATUS_METADATA, |
98 | STATUS_FILTER_LOCAL, /* Boolean */ |
99 | STATUS_FILTER_HASH, /* Boolean */ |
100 | STATUS_FILTER_GLOBAL, /* Boolean */ |
101 | STATUS_FILTER_RETAIN, /* Boolean */ |
102 | STATUS_FILTER_RULE_INDEX, |
103 | STATUS_ROUTER_LOCAL, /* Boolean */ |
104 | STATUS_ROUTER_HASH, /* Boolean */ |
105 | STATUS_UCP, /* Boolean */ |
106 | STATUS_ROUTER_TABLE, |
107 | STATUS_ROUTER_RULE_INDEX, |
108 | STATUS_NAT_HIT, /* Boolean */ |
109 | STATUS_NAT_INDEX, |
110 | STATUS_NAT_TYPE, /* enum ipa_nat_type */ |
111 | STATUS_TAG_LOW32, /* Low-order 32 bits of 48-bit tag */ |
112 | STATUS_TAG_HIGH16, /* High-order 16 bits of 48-bit tag */ |
113 | STATUS_SEQUENCE, |
114 | STATUS_TIME_OF_DAY, |
115 | , /* Boolean */ |
116 | , |
117 | STATUS_FRAG_HIT, /* Boolean */ |
118 | STATUS_FRAG_RULE_INDEX, |
119 | }; |
120 | |
121 | /* Size in bytes of an IPA packet status structure */ |
122 | #define IPA_STATUS_SIZE sizeof(__le32[8]) |
123 | |
124 | /* IPA status structure decoder; looks up field values for a structure */ |
125 | static u32 (struct ipa *ipa, const void *data, |
126 | enum ipa_status_field_id field) |
127 | { |
128 | enum ipa_version version = ipa->version; |
129 | const __le32 *word = data; |
130 | |
131 | switch (field) { |
132 | case STATUS_OPCODE: |
133 | return le32_get_bits(v: word[0], GENMASK(7, 0)); |
134 | case STATUS_EXCEPTION: |
135 | return le32_get_bits(v: word[0], GENMASK(15, 8)); |
136 | case STATUS_MASK: |
137 | return le32_get_bits(v: word[0], GENMASK(31, 16)); |
138 | case STATUS_LENGTH: |
139 | return le32_get_bits(v: word[1], GENMASK(15, 0)); |
140 | case STATUS_SRC_ENDPOINT: |
141 | if (version < IPA_VERSION_5_0) |
142 | return le32_get_bits(v: word[1], GENMASK(20, 16)); |
143 | return le32_get_bits(v: word[1], GENMASK(23, 16)); |
144 | /* Status word 1, bits 21-23 are reserved (not IPA v5.0+) */ |
145 | /* Status word 1, bits 24-26 are reserved (IPA v5.0+) */ |
146 | case STATUS_DST_ENDPOINT: |
147 | if (version < IPA_VERSION_5_0) |
148 | return le32_get_bits(v: word[1], GENMASK(28, 24)); |
149 | return le32_get_bits(v: word[7], GENMASK(23, 16)); |
150 | /* Status word 1, bits 29-31 are reserved */ |
151 | case STATUS_METADATA: |
152 | return le32_to_cpu(word[2]); |
153 | case STATUS_FILTER_LOCAL: |
154 | return le32_get_bits(v: word[3], GENMASK(0, 0)); |
155 | case STATUS_FILTER_HASH: |
156 | return le32_get_bits(v: word[3], GENMASK(1, 1)); |
157 | case STATUS_FILTER_GLOBAL: |
158 | return le32_get_bits(v: word[3], GENMASK(2, 2)); |
159 | case STATUS_FILTER_RETAIN: |
160 | return le32_get_bits(v: word[3], GENMASK(3, 3)); |
161 | case STATUS_FILTER_RULE_INDEX: |
162 | return le32_get_bits(v: word[3], GENMASK(13, 4)); |
163 | /* ROUTER_TABLE is in word 3, bits 14-21 (IPA v5.0+) */ |
164 | case STATUS_ROUTER_LOCAL: |
165 | if (version < IPA_VERSION_5_0) |
166 | return le32_get_bits(v: word[3], GENMASK(14, 14)); |
167 | return le32_get_bits(v: word[1], GENMASK(27, 27)); |
168 | case STATUS_ROUTER_HASH: |
169 | if (version < IPA_VERSION_5_0) |
170 | return le32_get_bits(v: word[3], GENMASK(15, 15)); |
171 | return le32_get_bits(v: word[1], GENMASK(28, 28)); |
172 | case STATUS_UCP: |
173 | if (version < IPA_VERSION_5_0) |
174 | return le32_get_bits(v: word[3], GENMASK(16, 16)); |
175 | return le32_get_bits(v: word[7], GENMASK(31, 31)); |
176 | case STATUS_ROUTER_TABLE: |
177 | if (version < IPA_VERSION_5_0) |
178 | return le32_get_bits(v: word[3], GENMASK(21, 17)); |
179 | return le32_get_bits(v: word[3], GENMASK(21, 14)); |
180 | case STATUS_ROUTER_RULE_INDEX: |
181 | return le32_get_bits(v: word[3], GENMASK(31, 22)); |
182 | case STATUS_NAT_HIT: |
183 | return le32_get_bits(v: word[4], GENMASK(0, 0)); |
184 | case STATUS_NAT_INDEX: |
185 | return le32_get_bits(v: word[4], GENMASK(13, 1)); |
186 | case STATUS_NAT_TYPE: |
187 | return le32_get_bits(v: word[4], GENMASK(15, 14)); |
188 | case STATUS_TAG_LOW32: |
189 | return le32_get_bits(v: word[4], GENMASK(31, 16)) | |
190 | (le32_get_bits(v: word[5], GENMASK(15, 0)) << 16); |
191 | case STATUS_TAG_HIGH16: |
192 | return le32_get_bits(v: word[5], GENMASK(31, 16)); |
193 | case STATUS_SEQUENCE: |
194 | return le32_get_bits(v: word[6], GENMASK(7, 0)); |
195 | case STATUS_TIME_OF_DAY: |
196 | return le32_get_bits(v: word[6], GENMASK(31, 8)); |
197 | case STATUS_HEADER_LOCAL: |
198 | return le32_get_bits(v: word[7], GENMASK(0, 0)); |
199 | case STATUS_HEADER_OFFSET: |
200 | return le32_get_bits(v: word[7], GENMASK(10, 1)); |
201 | case STATUS_FRAG_HIT: |
202 | return le32_get_bits(v: word[7], GENMASK(11, 11)); |
203 | case STATUS_FRAG_RULE_INDEX: |
204 | return le32_get_bits(v: word[7], GENMASK(15, 12)); |
205 | /* Status word 7, bits 16-30 are reserved */ |
206 | /* Status word 7, bit 31 is reserved (not IPA v5.0+) */ |
207 | default: |
208 | WARN(true, "%s: bad field_id %u\n" , __func__, field); |
209 | return 0; |
210 | } |
211 | } |
212 | |
213 | /* Compute the aggregation size value to use for a given buffer size */ |
214 | static u32 ipa_aggr_size_kb(u32 rx_buffer_size, bool aggr_hard_limit) |
215 | { |
216 | /* A hard aggregation limit will not be crossed; aggregation closes |
217 | * if saving incoming data would cross the hard byte limit boundary. |
218 | * |
219 | * With a soft limit, aggregation closes *after* the size boundary |
220 | * has been crossed. In that case the limit must leave enough space |
221 | * after that limit to receive a full MTU of data plus overhead. |
222 | */ |
223 | if (!aggr_hard_limit) |
224 | rx_buffer_size -= IPA_MTU + IPA_RX_BUFFER_OVERHEAD; |
225 | |
226 | /* The byte limit is encoded as a number of kilobytes */ |
227 | |
228 | return rx_buffer_size / SZ_1K; |
229 | } |
230 | |
231 | static bool ipa_endpoint_data_valid_one(struct ipa *ipa, u32 count, |
232 | const struct ipa_gsi_endpoint_data *all_data, |
233 | const struct ipa_gsi_endpoint_data *data) |
234 | { |
235 | const struct ipa_gsi_endpoint_data *other_data; |
236 | enum ipa_endpoint_name other_name; |
237 | struct device *dev = ipa->dev; |
238 | |
239 | if (ipa_gsi_endpoint_data_empty(data)) |
240 | return true; |
241 | |
242 | if (!data->toward_ipa) { |
243 | const struct ipa_endpoint_rx *rx_config; |
244 | const struct reg *reg; |
245 | u32 buffer_size; |
246 | u32 aggr_size; |
247 | u32 limit; |
248 | |
249 | if (data->endpoint.filter_support) { |
250 | dev_err(dev, "filtering not supported for " |
251 | "RX endpoint %u\n" , |
252 | data->endpoint_id); |
253 | return false; |
254 | } |
255 | |
256 | /* Nothing more to check for non-AP RX */ |
257 | if (data->ee_id != GSI_EE_AP) |
258 | return true; |
259 | |
260 | rx_config = &data->endpoint.config.rx; |
261 | |
262 | /* The buffer size must hold an MTU plus overhead */ |
263 | buffer_size = rx_config->buffer_size; |
264 | limit = IPA_MTU + IPA_RX_BUFFER_OVERHEAD; |
265 | if (buffer_size < limit) { |
266 | dev_err(dev, "RX buffer size too small for RX endpoint %u (%u < %u)\n" , |
267 | data->endpoint_id, buffer_size, limit); |
268 | return false; |
269 | } |
270 | |
271 | if (!data->endpoint.config.aggregation) { |
272 | bool result = true; |
273 | |
274 | /* No aggregation; check for bogus aggregation data */ |
275 | if (rx_config->aggr_time_limit) { |
276 | dev_err(dev, |
277 | "time limit with no aggregation for RX endpoint %u\n" , |
278 | data->endpoint_id); |
279 | result = false; |
280 | } |
281 | |
282 | if (rx_config->aggr_hard_limit) { |
283 | dev_err(dev, "hard limit with no aggregation for RX endpoint %u\n" , |
284 | data->endpoint_id); |
285 | result = false; |
286 | } |
287 | |
288 | if (rx_config->aggr_close_eof) { |
289 | dev_err(dev, "close EOF with no aggregation for RX endpoint %u\n" , |
290 | data->endpoint_id); |
291 | result = false; |
292 | } |
293 | |
294 | return result; /* Nothing more to check */ |
295 | } |
296 | |
297 | /* For an endpoint supporting receive aggregation, the byte |
298 | * limit defines the point at which aggregation closes. This |
299 | * check ensures the receive buffer size doesn't result in a |
300 | * limit that exceeds what's representable in the aggregation |
301 | * byte limit field. |
302 | */ |
303 | aggr_size = ipa_aggr_size_kb(rx_buffer_size: buffer_size - NET_SKB_PAD, |
304 | aggr_hard_limit: rx_config->aggr_hard_limit); |
305 | reg = ipa_reg(ipa, reg_id: ENDP_INIT_AGGR); |
306 | |
307 | limit = reg_field_max(reg, field_id: BYTE_LIMIT); |
308 | if (aggr_size > limit) { |
309 | dev_err(dev, "aggregated size too large for RX endpoint %u (%u KB > %u KB)\n" , |
310 | data->endpoint_id, aggr_size, limit); |
311 | |
312 | return false; |
313 | } |
314 | |
315 | return true; /* Nothing more to check for RX */ |
316 | } |
317 | |
318 | /* Starting with IPA v4.5 sequencer replication is obsolete */ |
319 | if (ipa->version >= IPA_VERSION_4_5) { |
320 | if (data->endpoint.config.tx.seq_rep_type) { |
321 | dev_err(dev, "no-zero seq_rep_type TX endpoint %u\n" , |
322 | data->endpoint_id); |
323 | return false; |
324 | } |
325 | } |
326 | |
327 | if (data->endpoint.config.status_enable) { |
328 | other_name = data->endpoint.config.tx.status_endpoint; |
329 | if (other_name >= count) { |
330 | dev_err(dev, "status endpoint name %u out of range " |
331 | "for endpoint %u\n" , |
332 | other_name, data->endpoint_id); |
333 | return false; |
334 | } |
335 | |
336 | /* Status endpoint must be defined... */ |
337 | other_data = &all_data[other_name]; |
338 | if (ipa_gsi_endpoint_data_empty(data: other_data)) { |
339 | dev_err(dev, "DMA endpoint name %u undefined " |
340 | "for endpoint %u\n" , |
341 | other_name, data->endpoint_id); |
342 | return false; |
343 | } |
344 | |
345 | /* ...and has to be an RX endpoint... */ |
346 | if (other_data->toward_ipa) { |
347 | dev_err(dev, |
348 | "status endpoint for endpoint %u not RX\n" , |
349 | data->endpoint_id); |
350 | return false; |
351 | } |
352 | |
353 | /* ...and if it's to be an AP endpoint... */ |
354 | if (other_data->ee_id == GSI_EE_AP) { |
355 | /* ...make sure it has status enabled. */ |
356 | if (!other_data->endpoint.config.status_enable) { |
357 | dev_err(dev, |
358 | "status not enabled for endpoint %u\n" , |
359 | other_data->endpoint_id); |
360 | return false; |
361 | } |
362 | } |
363 | } |
364 | |
365 | if (data->endpoint.config.dma_mode) { |
366 | other_name = data->endpoint.config.dma_endpoint; |
367 | if (other_name >= count) { |
368 | dev_err(dev, "DMA endpoint name %u out of range " |
369 | "for endpoint %u\n" , |
370 | other_name, data->endpoint_id); |
371 | return false; |
372 | } |
373 | |
374 | other_data = &all_data[other_name]; |
375 | if (ipa_gsi_endpoint_data_empty(data: other_data)) { |
376 | dev_err(dev, "DMA endpoint name %u undefined " |
377 | "for endpoint %u\n" , |
378 | other_name, data->endpoint_id); |
379 | return false; |
380 | } |
381 | } |
382 | |
383 | return true; |
384 | } |
385 | |
386 | /* Validate endpoint configuration data. Return max defined endpoint ID */ |
387 | static u32 ipa_endpoint_max(struct ipa *ipa, u32 count, |
388 | const struct ipa_gsi_endpoint_data *data) |
389 | { |
390 | const struct ipa_gsi_endpoint_data *dp = data; |
391 | struct device *dev = ipa->dev; |
392 | enum ipa_endpoint_name name; |
393 | u32 max; |
394 | |
395 | if (count > IPA_ENDPOINT_COUNT) { |
396 | dev_err(dev, "too many endpoints specified (%u > %u)\n" , |
397 | count, IPA_ENDPOINT_COUNT); |
398 | return 0; |
399 | } |
400 | |
401 | /* Make sure needed endpoints have defined data */ |
402 | if (ipa_gsi_endpoint_data_empty(data: &data[IPA_ENDPOINT_AP_COMMAND_TX])) { |
403 | dev_err(dev, "command TX endpoint not defined\n" ); |
404 | return 0; |
405 | } |
406 | if (ipa_gsi_endpoint_data_empty(data: &data[IPA_ENDPOINT_AP_LAN_RX])) { |
407 | dev_err(dev, "LAN RX endpoint not defined\n" ); |
408 | return 0; |
409 | } |
410 | if (ipa_gsi_endpoint_data_empty(data: &data[IPA_ENDPOINT_AP_MODEM_TX])) { |
411 | dev_err(dev, "AP->modem TX endpoint not defined\n" ); |
412 | return 0; |
413 | } |
414 | if (ipa_gsi_endpoint_data_empty(data: &data[IPA_ENDPOINT_AP_MODEM_RX])) { |
415 | dev_err(dev, "AP<-modem RX endpoint not defined\n" ); |
416 | return 0; |
417 | } |
418 | |
419 | max = 0; |
420 | for (name = 0; name < count; name++, dp++) { |
421 | if (!ipa_endpoint_data_valid_one(ipa, count, all_data: data, data: dp)) |
422 | return 0; |
423 | max = max_t(u32, max, dp->endpoint_id); |
424 | } |
425 | |
426 | return max; |
427 | } |
428 | |
429 | /* Allocate a transaction to use on a non-command endpoint */ |
430 | static struct gsi_trans *ipa_endpoint_trans_alloc(struct ipa_endpoint *endpoint, |
431 | u32 tre_count) |
432 | { |
433 | struct gsi *gsi = &endpoint->ipa->gsi; |
434 | u32 channel_id = endpoint->channel_id; |
435 | enum dma_data_direction direction; |
436 | |
437 | direction = endpoint->toward_ipa ? DMA_TO_DEVICE : DMA_FROM_DEVICE; |
438 | |
439 | return gsi_channel_trans_alloc(gsi, channel_id, tre_count, direction); |
440 | } |
441 | |
442 | /* suspend_delay represents suspend for RX, delay for TX endpoints. |
443 | * Note that suspend is not supported starting with IPA v4.0, and |
444 | * delay mode should not be used starting with IPA v4.2. |
445 | */ |
446 | static bool |
447 | ipa_endpoint_init_ctrl(struct ipa_endpoint *endpoint, bool suspend_delay) |
448 | { |
449 | struct ipa *ipa = endpoint->ipa; |
450 | const struct reg *reg; |
451 | u32 field_id; |
452 | u32 offset; |
453 | bool state; |
454 | u32 mask; |
455 | u32 val; |
456 | |
457 | if (endpoint->toward_ipa) |
458 | WARN_ON(ipa->version >= IPA_VERSION_4_2); |
459 | else |
460 | WARN_ON(ipa->version >= IPA_VERSION_4_0); |
461 | |
462 | reg = ipa_reg(ipa, reg_id: ENDP_INIT_CTRL); |
463 | offset = reg_n_offset(reg, n: endpoint->endpoint_id); |
464 | val = ioread32(ipa->reg_virt + offset); |
465 | |
466 | field_id = endpoint->toward_ipa ? ENDP_DELAY : ENDP_SUSPEND; |
467 | mask = reg_bit(reg, field_id); |
468 | |
469 | state = !!(val & mask); |
470 | |
471 | /* Don't bother if it's already in the requested state */ |
472 | if (suspend_delay != state) { |
473 | val ^= mask; |
474 | iowrite32(val, ipa->reg_virt + offset); |
475 | } |
476 | |
477 | return state; |
478 | } |
479 | |
480 | /* We don't care what the previous state was for delay mode */ |
481 | static void |
482 | ipa_endpoint_program_delay(struct ipa_endpoint *endpoint, bool enable) |
483 | { |
484 | /* Delay mode should not be used for IPA v4.2+ */ |
485 | WARN_ON(endpoint->ipa->version >= IPA_VERSION_4_2); |
486 | WARN_ON(!endpoint->toward_ipa); |
487 | |
488 | (void)ipa_endpoint_init_ctrl(endpoint, suspend_delay: enable); |
489 | } |
490 | |
491 | static bool ipa_endpoint_aggr_active(struct ipa_endpoint *endpoint) |
492 | { |
493 | u32 endpoint_id = endpoint->endpoint_id; |
494 | struct ipa *ipa = endpoint->ipa; |
495 | u32 unit = endpoint_id / 32; |
496 | const struct reg *reg; |
497 | u32 val; |
498 | |
499 | WARN_ON(!test_bit(endpoint_id, ipa->available)); |
500 | |
501 | reg = ipa_reg(ipa, reg_id: STATE_AGGR_ACTIVE); |
502 | val = ioread32(ipa->reg_virt + reg_n_offset(reg, n: unit)); |
503 | |
504 | return !!(val & BIT(endpoint_id % 32)); |
505 | } |
506 | |
507 | static void ipa_endpoint_force_close(struct ipa_endpoint *endpoint) |
508 | { |
509 | u32 endpoint_id = endpoint->endpoint_id; |
510 | u32 mask = BIT(endpoint_id % 32); |
511 | struct ipa *ipa = endpoint->ipa; |
512 | u32 unit = endpoint_id / 32; |
513 | const struct reg *reg; |
514 | |
515 | WARN_ON(!test_bit(endpoint_id, ipa->available)); |
516 | |
517 | reg = ipa_reg(ipa, reg_id: AGGR_FORCE_CLOSE); |
518 | iowrite32(mask, ipa->reg_virt + reg_n_offset(reg, n: unit)); |
519 | } |
520 | |
521 | /** |
522 | * ipa_endpoint_suspend_aggr() - Emulate suspend interrupt |
523 | * @endpoint: Endpoint on which to emulate a suspend |
524 | * |
525 | * Emulate suspend IPA interrupt to unsuspend an endpoint suspended |
526 | * with an open aggregation frame. This is to work around a hardware |
527 | * issue in IPA version 3.5.1 where the suspend interrupt will not be |
528 | * generated when it should be. |
529 | */ |
530 | static void ipa_endpoint_suspend_aggr(struct ipa_endpoint *endpoint) |
531 | { |
532 | struct ipa *ipa = endpoint->ipa; |
533 | |
534 | if (!endpoint->config.aggregation) |
535 | return; |
536 | |
537 | /* Nothing to do if the endpoint doesn't have aggregation open */ |
538 | if (!ipa_endpoint_aggr_active(endpoint)) |
539 | return; |
540 | |
541 | /* Force close aggregation */ |
542 | ipa_endpoint_force_close(endpoint); |
543 | |
544 | ipa_interrupt_simulate_suspend(interrupt: ipa->interrupt); |
545 | } |
546 | |
547 | /* Returns previous suspend state (true means suspend was enabled) */ |
548 | static bool |
549 | ipa_endpoint_program_suspend(struct ipa_endpoint *endpoint, bool enable) |
550 | { |
551 | bool suspended; |
552 | |
553 | if (endpoint->ipa->version >= IPA_VERSION_4_0) |
554 | return enable; /* For IPA v4.0+, no change made */ |
555 | |
556 | WARN_ON(endpoint->toward_ipa); |
557 | |
558 | suspended = ipa_endpoint_init_ctrl(endpoint, suspend_delay: enable); |
559 | |
560 | /* A client suspended with an open aggregation frame will not |
561 | * generate a SUSPEND IPA interrupt. If enabling suspend, have |
562 | * ipa_endpoint_suspend_aggr() handle this. |
563 | */ |
564 | if (enable && !suspended) |
565 | ipa_endpoint_suspend_aggr(endpoint); |
566 | |
567 | return suspended; |
568 | } |
569 | |
570 | /* Put all modem RX endpoints into suspend mode, and stop transmission |
571 | * on all modem TX endpoints. Prior to IPA v4.2, endpoint DELAY mode is |
572 | * used for TX endpoints; starting with IPA v4.2 we use GSI channel flow |
573 | * control instead. |
574 | */ |
575 | void ipa_endpoint_modem_pause_all(struct ipa *ipa, bool enable) |
576 | { |
577 | u32 endpoint_id = 0; |
578 | |
579 | while (endpoint_id < ipa->endpoint_count) { |
580 | struct ipa_endpoint *endpoint = &ipa->endpoint[endpoint_id++]; |
581 | |
582 | if (endpoint->ee_id != GSI_EE_MODEM) |
583 | continue; |
584 | |
585 | if (!endpoint->toward_ipa) |
586 | (void)ipa_endpoint_program_suspend(endpoint, enable); |
587 | else if (ipa->version < IPA_VERSION_4_2) |
588 | ipa_endpoint_program_delay(endpoint, enable); |
589 | else |
590 | gsi_modem_channel_flow_control(gsi: &ipa->gsi, |
591 | channel_id: endpoint->channel_id, |
592 | enable); |
593 | } |
594 | } |
595 | |
596 | /* Reset all modem endpoints to use the default exception endpoint */ |
597 | int ipa_endpoint_modem_exception_reset_all(struct ipa *ipa) |
598 | { |
599 | struct gsi_trans *trans; |
600 | u32 endpoint_id; |
601 | u32 count; |
602 | |
603 | /* We need one command per modem TX endpoint, plus the commands |
604 | * that clear the pipeline. |
605 | */ |
606 | count = ipa->modem_tx_count + ipa_cmd_pipeline_clear_count(); |
607 | trans = ipa_cmd_trans_alloc(ipa, tre_count: count); |
608 | if (!trans) { |
609 | dev_err(ipa->dev, |
610 | "no transaction to reset modem exception endpoints\n" ); |
611 | return -EBUSY; |
612 | } |
613 | |
614 | for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count) { |
615 | struct ipa_endpoint *endpoint; |
616 | const struct reg *reg; |
617 | u32 offset; |
618 | |
619 | /* We only reset modem TX endpoints */ |
620 | endpoint = &ipa->endpoint[endpoint_id]; |
621 | if (!(endpoint->ee_id == GSI_EE_MODEM && endpoint->toward_ipa)) |
622 | continue; |
623 | |
624 | reg = ipa_reg(ipa, reg_id: ENDP_STATUS); |
625 | offset = reg_n_offset(reg, n: endpoint_id); |
626 | |
627 | /* Value written is 0, and all bits are updated. That |
628 | * means status is disabled on the endpoint, and as a |
629 | * result all other fields in the register are ignored. |
630 | */ |
631 | ipa_cmd_register_write_add(trans, offset, value: 0, mask: ~0, clear_full: false); |
632 | } |
633 | |
634 | ipa_cmd_pipeline_clear_add(trans); |
635 | |
636 | gsi_trans_commit_wait(trans); |
637 | |
638 | ipa_cmd_pipeline_clear_wait(ipa); |
639 | |
640 | return 0; |
641 | } |
642 | |
643 | static void ipa_endpoint_init_cfg(struct ipa_endpoint *endpoint) |
644 | { |
645 | u32 endpoint_id = endpoint->endpoint_id; |
646 | struct ipa *ipa = endpoint->ipa; |
647 | enum ipa_cs_offload_en enabled; |
648 | const struct reg *reg; |
649 | u32 val = 0; |
650 | |
651 | reg = ipa_reg(ipa, reg_id: ENDP_INIT_CFG); |
652 | /* FRAG_OFFLOAD_EN is 0 */ |
653 | if (endpoint->config.checksum) { |
654 | enum ipa_version version = ipa->version; |
655 | |
656 | if (endpoint->toward_ipa) { |
657 | u32 off; |
658 | |
659 | /* Checksum header offset is in 4-byte units */ |
660 | off = sizeof(struct rmnet_map_header) / sizeof(u32); |
661 | val |= reg_encode(reg, field_id: CS_METADATA_HDR_OFFSET, val: off); |
662 | |
663 | enabled = version < IPA_VERSION_4_5 |
664 | ? IPA_CS_OFFLOAD_UL |
665 | : IPA_CS_OFFLOAD_INLINE; |
666 | } else { |
667 | enabled = version < IPA_VERSION_4_5 |
668 | ? IPA_CS_OFFLOAD_DL |
669 | : IPA_CS_OFFLOAD_INLINE; |
670 | } |
671 | } else { |
672 | enabled = IPA_CS_OFFLOAD_NONE; |
673 | } |
674 | val |= reg_encode(reg, field_id: CS_OFFLOAD_EN, val: enabled); |
675 | /* CS_GEN_QMB_MASTER_SEL is 0 */ |
676 | |
677 | iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id)); |
678 | } |
679 | |
680 | static void ipa_endpoint_init_nat(struct ipa_endpoint *endpoint) |
681 | { |
682 | u32 endpoint_id = endpoint->endpoint_id; |
683 | struct ipa *ipa = endpoint->ipa; |
684 | const struct reg *reg; |
685 | u32 val; |
686 | |
687 | if (!endpoint->toward_ipa) |
688 | return; |
689 | |
690 | reg = ipa_reg(ipa, reg_id: ENDP_INIT_NAT); |
691 | val = reg_encode(reg, field_id: NAT_EN, val: IPA_NAT_TYPE_BYPASS); |
692 | |
693 | iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id)); |
694 | } |
695 | |
696 | static u32 |
697 | (enum ipa_version version, struct ipa_endpoint *endpoint) |
698 | { |
699 | u32 = sizeof(struct rmnet_map_header); |
700 | |
701 | /* Without checksum offload, we just have the MAP header */ |
702 | if (!endpoint->config.checksum) |
703 | return header_size; |
704 | |
705 | if (version < IPA_VERSION_4_5) { |
706 | /* Checksum header inserted for AP TX endpoints only */ |
707 | if (endpoint->toward_ipa) |
708 | header_size += sizeof(struct rmnet_map_ul_csum_header); |
709 | } else { |
710 | /* Checksum header is used in both directions */ |
711 | header_size += sizeof(struct rmnet_map_v5_csum_header); |
712 | } |
713 | |
714 | return header_size; |
715 | } |
716 | |
717 | /* Encoded value for ENDP_INIT_HDR register HDR_LEN* field(s) */ |
718 | static u32 (enum ipa_version version, |
719 | const struct reg *reg, u32 ) |
720 | { |
721 | u32 field_max = reg_field_max(reg, field_id: HDR_LEN); |
722 | u32 val; |
723 | |
724 | /* We know field_max can be used as a mask (2^n - 1) */ |
725 | val = reg_encode(reg, field_id: HDR_LEN, val: header_size & field_max); |
726 | if (version < IPA_VERSION_4_5) { |
727 | WARN_ON(header_size > field_max); |
728 | return val; |
729 | } |
730 | |
731 | /* IPA v4.5 adds a few more most-significant bits */ |
732 | header_size >>= hweight32(field_max); |
733 | WARN_ON(header_size > reg_field_max(reg, HDR_LEN_MSB)); |
734 | val |= reg_encode(reg, field_id: HDR_LEN_MSB, val: header_size); |
735 | |
736 | return val; |
737 | } |
738 | |
739 | /* Encoded value for ENDP_INIT_HDR register OFST_METADATA* field(s) */ |
740 | static u32 ipa_metadata_offset_encode(enum ipa_version version, |
741 | const struct reg *reg, u32 offset) |
742 | { |
743 | u32 field_max = reg_field_max(reg, field_id: HDR_OFST_METADATA); |
744 | u32 val; |
745 | |
746 | /* We know field_max can be used as a mask (2^n - 1) */ |
747 | val = reg_encode(reg, field_id: HDR_OFST_METADATA, val: offset); |
748 | if (version < IPA_VERSION_4_5) { |
749 | WARN_ON(offset > field_max); |
750 | return val; |
751 | } |
752 | |
753 | /* IPA v4.5 adds a few more most-significant bits */ |
754 | offset >>= hweight32(field_max); |
755 | WARN_ON(offset > reg_field_max(reg, HDR_OFST_METADATA_MSB)); |
756 | val |= reg_encode(reg, field_id: HDR_OFST_METADATA_MSB, val: offset); |
757 | |
758 | return val; |
759 | } |
760 | |
761 | /** |
762 | * ipa_endpoint_init_hdr() - Initialize HDR endpoint configuration register |
763 | * @endpoint: Endpoint pointer |
764 | * |
765 | * We program QMAP endpoints so each packet received is preceded by a QMAP |
766 | * header structure. The QMAP header contains a 1-byte mux_id and 2-byte |
767 | * packet size field, and we have the IPA hardware populate both for each |
768 | * received packet. The header is configured (in the HDR_EXT register) |
769 | * to use big endian format. |
770 | * |
771 | * The packet size is written into the QMAP header's pkt_len field. That |
772 | * location is defined here using the HDR_OFST_PKT_SIZE field. |
773 | * |
774 | * The mux_id comes from a 4-byte metadata value supplied with each packet |
775 | * by the modem. It is *not* a QMAP header, but it does contain the mux_id |
776 | * value that we want, in its low-order byte. A bitmask defined in the |
777 | * endpoint's METADATA_MASK register defines which byte within the modem |
778 | * metadata contains the mux_id. And the OFST_METADATA field programmed |
779 | * here indicates where the extracted byte should be placed within the QMAP |
780 | * header. |
781 | */ |
782 | static void ipa_endpoint_init_hdr(struct ipa_endpoint *endpoint) |
783 | { |
784 | u32 endpoint_id = endpoint->endpoint_id; |
785 | struct ipa *ipa = endpoint->ipa; |
786 | const struct reg *reg; |
787 | u32 val = 0; |
788 | |
789 | reg = ipa_reg(ipa, reg_id: ENDP_INIT_HDR); |
790 | if (endpoint->config.qmap) { |
791 | enum ipa_version version = ipa->version; |
792 | size_t ; |
793 | |
794 | header_size = ipa_qmap_header_size(version, endpoint); |
795 | val = ipa_header_size_encode(version, reg, header_size); |
796 | |
797 | /* Define how to fill fields in a received QMAP header */ |
798 | if (!endpoint->toward_ipa) { |
799 | u32 off; /* Field offset within header */ |
800 | |
801 | /* Where IPA will write the metadata value */ |
802 | off = offsetof(struct rmnet_map_header, mux_id); |
803 | val |= ipa_metadata_offset_encode(version, reg, offset: off); |
804 | |
805 | /* Where IPA will write the length */ |
806 | off = offsetof(struct rmnet_map_header, pkt_len); |
807 | /* Upper bits are stored in HDR_EXT with IPA v4.5 */ |
808 | if (version >= IPA_VERSION_4_5) |
809 | off &= reg_field_max(reg, field_id: HDR_OFST_PKT_SIZE); |
810 | |
811 | val |= reg_bit(reg, field_id: HDR_OFST_PKT_SIZE_VALID); |
812 | val |= reg_encode(reg, field_id: HDR_OFST_PKT_SIZE, val: off); |
813 | } |
814 | /* For QMAP TX, metadata offset is 0 (modem assumes this) */ |
815 | val |= reg_bit(reg, field_id: HDR_OFST_METADATA_VALID); |
816 | |
817 | /* HDR_ADDITIONAL_CONST_LEN is 0; (RX only) */ |
818 | /* HDR_A5_MUX is 0 */ |
819 | /* HDR_LEN_INC_DEAGG_HDR is 0 */ |
820 | /* HDR_METADATA_REG_VALID is 0 (TX only, version < v4.5) */ |
821 | } |
822 | |
823 | iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id)); |
824 | } |
825 | |
826 | static void ipa_endpoint_init_hdr_ext(struct ipa_endpoint *endpoint) |
827 | { |
828 | u32 pad_align = endpoint->config.rx.pad_align; |
829 | u32 endpoint_id = endpoint->endpoint_id; |
830 | struct ipa *ipa = endpoint->ipa; |
831 | const struct reg *reg; |
832 | u32 val = 0; |
833 | |
834 | reg = ipa_reg(ipa, reg_id: ENDP_INIT_HDR_EXT); |
835 | if (endpoint->config.qmap) { |
836 | /* We have a header, so we must specify its endianness */ |
837 | val |= reg_bit(reg, field_id: HDR_ENDIANNESS); /* big endian */ |
838 | |
839 | /* A QMAP header contains a 6 bit pad field at offset 0. |
840 | * The RMNet driver assumes this field is meaningful in |
841 | * packets it receives, and assumes the header's payload |
842 | * length includes that padding. The RMNet driver does |
843 | * *not* pad packets it sends, however, so the pad field |
844 | * (although 0) should be ignored. |
845 | */ |
846 | if (!endpoint->toward_ipa) { |
847 | val |= reg_bit(reg, field_id: HDR_TOTAL_LEN_OR_PAD_VALID); |
848 | /* HDR_TOTAL_LEN_OR_PAD is 0 (pad, not total_len) */ |
849 | val |= reg_bit(reg, field_id: HDR_PAYLOAD_LEN_INC_PADDING); |
850 | /* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0 */ |
851 | } |
852 | } |
853 | |
854 | /* HDR_PAYLOAD_LEN_INC_PADDING is 0 */ |
855 | if (!endpoint->toward_ipa) |
856 | val |= reg_encode(reg, field_id: HDR_PAD_TO_ALIGNMENT, val: pad_align); |
857 | |
858 | /* IPA v4.5 adds some most-significant bits to a few fields, |
859 | * two of which are defined in the HDR (not HDR_EXT) register. |
860 | */ |
861 | if (ipa->version >= IPA_VERSION_4_5) { |
862 | /* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0, so MSB is 0 */ |
863 | if (endpoint->config.qmap && !endpoint->toward_ipa) { |
864 | u32 mask = reg_field_max(reg, field_id: HDR_OFST_PKT_SIZE); |
865 | u32 off; /* Field offset within header */ |
866 | |
867 | off = offsetof(struct rmnet_map_header, pkt_len); |
868 | /* Low bits are in the ENDP_INIT_HDR register */ |
869 | off >>= hweight32(mask); |
870 | val |= reg_encode(reg, field_id: HDR_OFST_PKT_SIZE_MSB, val: off); |
871 | /* HDR_ADDITIONAL_CONST_LEN is 0 so MSB is 0 */ |
872 | } |
873 | } |
874 | |
875 | iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id)); |
876 | } |
877 | |
878 | static void ipa_endpoint_init_hdr_metadata_mask(struct ipa_endpoint *endpoint) |
879 | { |
880 | u32 endpoint_id = endpoint->endpoint_id; |
881 | struct ipa *ipa = endpoint->ipa; |
882 | const struct reg *reg; |
883 | u32 val = 0; |
884 | u32 offset; |
885 | |
886 | if (endpoint->toward_ipa) |
887 | return; /* Register not valid for TX endpoints */ |
888 | |
889 | reg = ipa_reg(ipa, reg_id: ENDP_INIT_HDR_METADATA_MASK); |
890 | offset = reg_n_offset(reg, n: endpoint_id); |
891 | |
892 | /* Note that HDR_ENDIANNESS indicates big endian header fields */ |
893 | if (endpoint->config.qmap) |
894 | val = (__force u32)cpu_to_be32(IPA_ENDPOINT_QMAP_METADATA_MASK); |
895 | |
896 | iowrite32(val, ipa->reg_virt + offset); |
897 | } |
898 | |
899 | static void ipa_endpoint_init_mode(struct ipa_endpoint *endpoint) |
900 | { |
901 | struct ipa *ipa = endpoint->ipa; |
902 | const struct reg *reg; |
903 | u32 offset; |
904 | u32 val; |
905 | |
906 | if (!endpoint->toward_ipa) |
907 | return; /* Register not valid for RX endpoints */ |
908 | |
909 | reg = ipa_reg(ipa, reg_id: ENDP_INIT_MODE); |
910 | if (endpoint->config.dma_mode) { |
911 | enum ipa_endpoint_name name = endpoint->config.dma_endpoint; |
912 | u32 dma_endpoint_id = ipa->name_map[name]->endpoint_id; |
913 | |
914 | val = reg_encode(reg, field_id: ENDP_MODE, val: IPA_DMA); |
915 | val |= reg_encode(reg, field_id: DEST_PIPE_INDEX, val: dma_endpoint_id); |
916 | } else { |
917 | val = reg_encode(reg, field_id: ENDP_MODE, val: IPA_BASIC); |
918 | } |
919 | /* All other bits unspecified (and 0) */ |
920 | |
921 | offset = reg_n_offset(reg, n: endpoint->endpoint_id); |
922 | iowrite32(val, ipa->reg_virt + offset); |
923 | } |
924 | |
925 | /* For IPA v4.5+, times are expressed using Qtime. A time is represented |
926 | * at one of several available granularities, which are configured in |
927 | * ipa_qtime_config(). Three (or, starting with IPA v5.0, four) pulse |
928 | * generators are set up with different "tick" periods. A Qtime value |
929 | * encodes a tick count along with an indication of a pulse generator |
930 | * (which has a fixed tick period). Two pulse generators are always |
931 | * available to the AP; a third is available starting with IPA v5.0. |
932 | * This function determines which pulse generator most accurately |
933 | * represents the time period provided, and returns the tick count to |
934 | * use to represent that time. |
935 | */ |
936 | static u32 |
937 | ipa_qtime_val(struct ipa *ipa, u32 microseconds, u32 max, u32 *select) |
938 | { |
939 | u32 which = 0; |
940 | u32 ticks; |
941 | |
942 | /* Pulse generator 0 has 100 microsecond granularity */ |
943 | ticks = DIV_ROUND_CLOSEST(microseconds, 100); |
944 | if (ticks <= max) |
945 | goto out; |
946 | |
947 | /* Pulse generator 1 has millisecond granularity */ |
948 | which = 1; |
949 | ticks = DIV_ROUND_CLOSEST(microseconds, 1000); |
950 | if (ticks <= max) |
951 | goto out; |
952 | |
953 | if (ipa->version >= IPA_VERSION_5_0) { |
954 | /* Pulse generator 2 has 10 millisecond granularity */ |
955 | which = 2; |
956 | ticks = DIV_ROUND_CLOSEST(microseconds, 100); |
957 | } |
958 | WARN_ON(ticks > max); |
959 | out: |
960 | *select = which; |
961 | |
962 | return ticks; |
963 | } |
964 | |
965 | /* Encode the aggregation timer limit (microseconds) based on IPA version */ |
966 | static u32 aggr_time_limit_encode(struct ipa *ipa, const struct reg *reg, |
967 | u32 microseconds) |
968 | { |
969 | u32 ticks; |
970 | u32 max; |
971 | |
972 | if (!microseconds) |
973 | return 0; /* Nothing to compute if time limit is 0 */ |
974 | |
975 | max = reg_field_max(reg, field_id: TIME_LIMIT); |
976 | if (ipa->version >= IPA_VERSION_4_5) { |
977 | u32 select; |
978 | |
979 | ticks = ipa_qtime_val(ipa, microseconds, max, select: &select); |
980 | |
981 | return reg_encode(reg, field_id: AGGR_GRAN_SEL, val: select) | |
982 | reg_encode(reg, field_id: TIME_LIMIT, val: ticks); |
983 | } |
984 | |
985 | /* We program aggregation granularity in ipa_hardware_config() */ |
986 | ticks = DIV_ROUND_CLOSEST(microseconds, IPA_AGGR_GRANULARITY); |
987 | WARN(ticks > max, "aggr_time_limit too large (%u > %u usec)\n" , |
988 | microseconds, max * IPA_AGGR_GRANULARITY); |
989 | |
990 | return reg_encode(reg, field_id: TIME_LIMIT, val: ticks); |
991 | } |
992 | |
993 | static void ipa_endpoint_init_aggr(struct ipa_endpoint *endpoint) |
994 | { |
995 | u32 endpoint_id = endpoint->endpoint_id; |
996 | struct ipa *ipa = endpoint->ipa; |
997 | const struct reg *reg; |
998 | u32 val = 0; |
999 | |
1000 | reg = ipa_reg(ipa, reg_id: ENDP_INIT_AGGR); |
1001 | if (endpoint->config.aggregation) { |
1002 | if (!endpoint->toward_ipa) { |
1003 | const struct ipa_endpoint_rx *rx_config; |
1004 | u32 buffer_size; |
1005 | u32 limit; |
1006 | |
1007 | rx_config = &endpoint->config.rx; |
1008 | val |= reg_encode(reg, field_id: AGGR_EN, val: IPA_ENABLE_AGGR); |
1009 | val |= reg_encode(reg, field_id: AGGR_TYPE, val: IPA_GENERIC); |
1010 | |
1011 | buffer_size = rx_config->buffer_size; |
1012 | limit = ipa_aggr_size_kb(rx_buffer_size: buffer_size - NET_SKB_PAD, |
1013 | aggr_hard_limit: rx_config->aggr_hard_limit); |
1014 | val |= reg_encode(reg, field_id: BYTE_LIMIT, val: limit); |
1015 | |
1016 | limit = rx_config->aggr_time_limit; |
1017 | val |= aggr_time_limit_encode(ipa, reg, microseconds: limit); |
1018 | |
1019 | /* AGGR_PKT_LIMIT is 0 (unlimited) */ |
1020 | |
1021 | if (rx_config->aggr_close_eof) |
1022 | val |= reg_bit(reg, field_id: SW_EOF_ACTIVE); |
1023 | } else { |
1024 | val |= reg_encode(reg, field_id: AGGR_EN, val: IPA_ENABLE_DEAGGR); |
1025 | val |= reg_encode(reg, field_id: AGGR_TYPE, val: IPA_QCMAP); |
1026 | /* other fields ignored */ |
1027 | } |
1028 | /* AGGR_FORCE_CLOSE is 0 */ |
1029 | /* AGGR_GRAN_SEL is 0 for IPA v4.5 */ |
1030 | } else { |
1031 | val |= reg_encode(reg, field_id: AGGR_EN, val: IPA_BYPASS_AGGR); |
1032 | /* other fields ignored */ |
1033 | } |
1034 | |
1035 | iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id)); |
1036 | } |
1037 | |
1038 | /* The head-of-line blocking timer is defined as a tick count. For |
1039 | * IPA version 4.5 the tick count is based on the Qtimer, which is |
1040 | * derived from the 19.2 MHz SoC XO clock. For older IPA versions |
1041 | * each tick represents 128 cycles of the IPA core clock. |
1042 | * |
1043 | * Return the encoded value representing the timeout period provided |
1044 | * that should be written to the ENDP_INIT_HOL_BLOCK_TIMER register. |
1045 | */ |
1046 | static u32 hol_block_timer_encode(struct ipa *ipa, const struct reg *reg, |
1047 | u32 microseconds) |
1048 | { |
1049 | u32 width; |
1050 | u32 scale; |
1051 | u64 ticks; |
1052 | u64 rate; |
1053 | u32 high; |
1054 | u32 val; |
1055 | |
1056 | if (!microseconds) |
1057 | return 0; /* Nothing to compute if timer period is 0 */ |
1058 | |
1059 | if (ipa->version >= IPA_VERSION_4_5) { |
1060 | u32 max = reg_field_max(reg, field_id: TIMER_LIMIT); |
1061 | u32 select; |
1062 | u32 ticks; |
1063 | |
1064 | ticks = ipa_qtime_val(ipa, microseconds, max, select: &select); |
1065 | |
1066 | return reg_encode(reg, field_id: TIMER_GRAN_SEL, val: 1) | |
1067 | reg_encode(reg, field_id: TIMER_LIMIT, val: ticks); |
1068 | } |
1069 | |
1070 | /* Use 64 bit arithmetic to avoid overflow */ |
1071 | rate = ipa_core_clock_rate(ipa); |
1072 | ticks = DIV_ROUND_CLOSEST(microseconds * rate, 128 * USEC_PER_SEC); |
1073 | |
1074 | /* We still need the result to fit into the field */ |
1075 | WARN_ON(ticks > reg_field_max(reg, TIMER_BASE_VALUE)); |
1076 | |
1077 | /* IPA v3.5.1 through v4.1 just record the tick count */ |
1078 | if (ipa->version < IPA_VERSION_4_2) |
1079 | return reg_encode(reg, field_id: TIMER_BASE_VALUE, val: (u32)ticks); |
1080 | |
1081 | /* For IPA v4.2, the tick count is represented by base and |
1082 | * scale fields within the 32-bit timer register, where: |
1083 | * ticks = base << scale; |
1084 | * The best precision is achieved when the base value is as |
1085 | * large as possible. Find the highest set bit in the tick |
1086 | * count, and extract the number of bits in the base field |
1087 | * such that high bit is included. |
1088 | */ |
1089 | high = fls(x: ticks); /* 1..32 (or warning above) */ |
1090 | width = hweight32(reg_fmask(reg, TIMER_BASE_VALUE)); |
1091 | scale = high > width ? high - width : 0; |
1092 | if (scale) { |
1093 | /* If we're scaling, round up to get a closer result */ |
1094 | ticks += 1 << (scale - 1); |
1095 | /* High bit was set, so rounding might have affected it */ |
1096 | if (fls(x: ticks) != high) |
1097 | scale++; |
1098 | } |
1099 | |
1100 | val = reg_encode(reg, field_id: TIMER_SCALE, val: scale); |
1101 | val |= reg_encode(reg, field_id: TIMER_BASE_VALUE, val: (u32)ticks >> scale); |
1102 | |
1103 | return val; |
1104 | } |
1105 | |
1106 | /* If microseconds is 0, timeout is immediate */ |
1107 | static void ipa_endpoint_init_hol_block_timer(struct ipa_endpoint *endpoint, |
1108 | u32 microseconds) |
1109 | { |
1110 | u32 endpoint_id = endpoint->endpoint_id; |
1111 | struct ipa *ipa = endpoint->ipa; |
1112 | const struct reg *reg; |
1113 | u32 val; |
1114 | |
1115 | /* This should only be changed when HOL_BLOCK_EN is disabled */ |
1116 | reg = ipa_reg(ipa, reg_id: ENDP_INIT_HOL_BLOCK_TIMER); |
1117 | val = hol_block_timer_encode(ipa, reg, microseconds); |
1118 | |
1119 | iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id)); |
1120 | } |
1121 | |
1122 | static void |
1123 | ipa_endpoint_init_hol_block_en(struct ipa_endpoint *endpoint, bool enable) |
1124 | { |
1125 | u32 endpoint_id = endpoint->endpoint_id; |
1126 | struct ipa *ipa = endpoint->ipa; |
1127 | const struct reg *reg; |
1128 | u32 offset; |
1129 | u32 val; |
1130 | |
1131 | reg = ipa_reg(ipa, reg_id: ENDP_INIT_HOL_BLOCK_EN); |
1132 | offset = reg_n_offset(reg, n: endpoint_id); |
1133 | val = enable ? reg_bit(reg, field_id: HOL_BLOCK_EN) : 0; |
1134 | |
1135 | iowrite32(val, ipa->reg_virt + offset); |
1136 | |
1137 | /* When enabling, the register must be written twice for IPA v4.5+ */ |
1138 | if (enable && ipa->version >= IPA_VERSION_4_5) |
1139 | iowrite32(val, ipa->reg_virt + offset); |
1140 | } |
1141 | |
1142 | /* Assumes HOL_BLOCK is in disabled state */ |
1143 | static void ipa_endpoint_init_hol_block_enable(struct ipa_endpoint *endpoint, |
1144 | u32 microseconds) |
1145 | { |
1146 | ipa_endpoint_init_hol_block_timer(endpoint, microseconds); |
1147 | ipa_endpoint_init_hol_block_en(endpoint, enable: true); |
1148 | } |
1149 | |
1150 | static void ipa_endpoint_init_hol_block_disable(struct ipa_endpoint *endpoint) |
1151 | { |
1152 | ipa_endpoint_init_hol_block_en(endpoint, enable: false); |
1153 | } |
1154 | |
1155 | void ipa_endpoint_modem_hol_block_clear_all(struct ipa *ipa) |
1156 | { |
1157 | u32 endpoint_id = 0; |
1158 | |
1159 | while (endpoint_id < ipa->endpoint_count) { |
1160 | struct ipa_endpoint *endpoint = &ipa->endpoint[endpoint_id++]; |
1161 | |
1162 | if (endpoint->toward_ipa || endpoint->ee_id != GSI_EE_MODEM) |
1163 | continue; |
1164 | |
1165 | ipa_endpoint_init_hol_block_disable(endpoint); |
1166 | ipa_endpoint_init_hol_block_enable(endpoint, microseconds: 0); |
1167 | } |
1168 | } |
1169 | |
1170 | static void ipa_endpoint_init_deaggr(struct ipa_endpoint *endpoint) |
1171 | { |
1172 | u32 endpoint_id = endpoint->endpoint_id; |
1173 | struct ipa *ipa = endpoint->ipa; |
1174 | const struct reg *reg; |
1175 | u32 val = 0; |
1176 | |
1177 | if (!endpoint->toward_ipa) |
1178 | return; /* Register not valid for RX endpoints */ |
1179 | |
1180 | reg = ipa_reg(ipa, reg_id: ENDP_INIT_DEAGGR); |
1181 | /* DEAGGR_HDR_LEN is 0 */ |
1182 | /* PACKET_OFFSET_VALID is 0 */ |
1183 | /* PACKET_OFFSET_LOCATION is ignored (not valid) */ |
1184 | /* MAX_PACKET_LEN is 0 (not enforced) */ |
1185 | |
1186 | iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id)); |
1187 | } |
1188 | |
1189 | static void ipa_endpoint_init_rsrc_grp(struct ipa_endpoint *endpoint) |
1190 | { |
1191 | u32 resource_group = endpoint->config.resource_group; |
1192 | u32 endpoint_id = endpoint->endpoint_id; |
1193 | struct ipa *ipa = endpoint->ipa; |
1194 | const struct reg *reg; |
1195 | u32 val; |
1196 | |
1197 | reg = ipa_reg(ipa, reg_id: ENDP_INIT_RSRC_GRP); |
1198 | val = reg_encode(reg, field_id: ENDP_RSRC_GRP, val: resource_group); |
1199 | |
1200 | iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id)); |
1201 | } |
1202 | |
1203 | static void ipa_endpoint_init_seq(struct ipa_endpoint *endpoint) |
1204 | { |
1205 | u32 endpoint_id = endpoint->endpoint_id; |
1206 | struct ipa *ipa = endpoint->ipa; |
1207 | const struct reg *reg; |
1208 | u32 val; |
1209 | |
1210 | if (!endpoint->toward_ipa) |
1211 | return; /* Register not valid for RX endpoints */ |
1212 | |
1213 | reg = ipa_reg(ipa, reg_id: ENDP_INIT_SEQ); |
1214 | |
1215 | /* Low-order byte configures primary packet processing */ |
1216 | val = reg_encode(reg, field_id: SEQ_TYPE, val: endpoint->config.tx.seq_type); |
1217 | |
1218 | /* Second byte (if supported) configures replicated packet processing */ |
1219 | if (ipa->version < IPA_VERSION_4_5) |
1220 | val |= reg_encode(reg, field_id: SEQ_REP_TYPE, |
1221 | val: endpoint->config.tx.seq_rep_type); |
1222 | |
1223 | iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id)); |
1224 | } |
1225 | |
1226 | /** |
1227 | * ipa_endpoint_skb_tx() - Transmit a socket buffer |
1228 | * @endpoint: Endpoint pointer |
1229 | * @skb: Socket buffer to send |
1230 | * |
1231 | * Returns: 0 if successful, or a negative error code |
1232 | */ |
1233 | int ipa_endpoint_skb_tx(struct ipa_endpoint *endpoint, struct sk_buff *skb) |
1234 | { |
1235 | struct gsi_trans *trans; |
1236 | u32 nr_frags; |
1237 | int ret; |
1238 | |
1239 | /* Make sure source endpoint's TLV FIFO has enough entries to |
1240 | * hold the linear portion of the skb and all its fragments. |
1241 | * If not, see if we can linearize it before giving up. |
1242 | */ |
1243 | nr_frags = skb_shinfo(skb)->nr_frags; |
1244 | if (nr_frags > endpoint->skb_frag_max) { |
1245 | if (skb_linearize(skb)) |
1246 | return -E2BIG; |
1247 | nr_frags = 0; |
1248 | } |
1249 | |
1250 | trans = ipa_endpoint_trans_alloc(endpoint, tre_count: 1 + nr_frags); |
1251 | if (!trans) |
1252 | return -EBUSY; |
1253 | |
1254 | ret = gsi_trans_skb_add(trans, skb); |
1255 | if (ret) |
1256 | goto err_trans_free; |
1257 | trans->data = skb; /* transaction owns skb now */ |
1258 | |
1259 | gsi_trans_commit(trans, ring_db: !netdev_xmit_more()); |
1260 | |
1261 | return 0; |
1262 | |
1263 | err_trans_free: |
1264 | gsi_trans_free(trans); |
1265 | |
1266 | return -ENOMEM; |
1267 | } |
1268 | |
1269 | static void ipa_endpoint_status(struct ipa_endpoint *endpoint) |
1270 | { |
1271 | u32 endpoint_id = endpoint->endpoint_id; |
1272 | struct ipa *ipa = endpoint->ipa; |
1273 | const struct reg *reg; |
1274 | u32 val = 0; |
1275 | |
1276 | reg = ipa_reg(ipa, reg_id: ENDP_STATUS); |
1277 | if (endpoint->config.status_enable) { |
1278 | val |= reg_bit(reg, field_id: STATUS_EN); |
1279 | if (endpoint->toward_ipa) { |
1280 | enum ipa_endpoint_name name; |
1281 | u32 status_endpoint_id; |
1282 | |
1283 | name = endpoint->config.tx.status_endpoint; |
1284 | status_endpoint_id = ipa->name_map[name]->endpoint_id; |
1285 | |
1286 | val |= reg_encode(reg, field_id: STATUS_ENDP, val: status_endpoint_id); |
1287 | } |
1288 | /* STATUS_LOCATION is 0, meaning IPA packet status |
1289 | * precedes the packet (not present for IPA v4.5+) |
1290 | */ |
1291 | /* STATUS_PKT_SUPPRESS_FMASK is 0 (not present for v4.0+) */ |
1292 | } |
1293 | |
1294 | iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id)); |
1295 | } |
1296 | |
1297 | static int ipa_endpoint_replenish_one(struct ipa_endpoint *endpoint, |
1298 | struct gsi_trans *trans) |
1299 | { |
1300 | struct page *page; |
1301 | u32 buffer_size; |
1302 | u32 offset; |
1303 | u32 len; |
1304 | int ret; |
1305 | |
1306 | buffer_size = endpoint->config.rx.buffer_size; |
1307 | page = dev_alloc_pages(order: get_order(size: buffer_size)); |
1308 | if (!page) |
1309 | return -ENOMEM; |
1310 | |
1311 | /* Offset the buffer to make space for skb headroom */ |
1312 | offset = NET_SKB_PAD; |
1313 | len = buffer_size - offset; |
1314 | |
1315 | ret = gsi_trans_page_add(trans, page, size: len, offset); |
1316 | if (ret) |
1317 | put_page(page); |
1318 | else |
1319 | trans->data = page; /* transaction owns page now */ |
1320 | |
1321 | return ret; |
1322 | } |
1323 | |
1324 | /** |
1325 | * ipa_endpoint_replenish() - Replenish endpoint receive buffers |
1326 | * @endpoint: Endpoint to be replenished |
1327 | * |
1328 | * The IPA hardware can hold a fixed number of receive buffers for an RX |
1329 | * endpoint, based on the number of entries in the underlying channel ring |
1330 | * buffer. If an endpoint's "backlog" is non-zero, it indicates how many |
1331 | * more receive buffers can be supplied to the hardware. Replenishing for |
1332 | * an endpoint can be disabled, in which case buffers are not queued to |
1333 | * the hardware. |
1334 | */ |
1335 | static void ipa_endpoint_replenish(struct ipa_endpoint *endpoint) |
1336 | { |
1337 | struct gsi_trans *trans; |
1338 | |
1339 | if (!test_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags)) |
1340 | return; |
1341 | |
1342 | /* Skip it if it's already active */ |
1343 | if (test_and_set_bit(nr: IPA_REPLENISH_ACTIVE, addr: endpoint->replenish_flags)) |
1344 | return; |
1345 | |
1346 | while ((trans = ipa_endpoint_trans_alloc(endpoint, tre_count: 1))) { |
1347 | bool doorbell; |
1348 | |
1349 | if (ipa_endpoint_replenish_one(endpoint, trans)) |
1350 | goto try_again_later; |
1351 | |
1352 | |
1353 | /* Ring the doorbell if we've got a full batch */ |
1354 | doorbell = !(++endpoint->replenish_count % IPA_REPLENISH_BATCH); |
1355 | gsi_trans_commit(trans, ring_db: doorbell); |
1356 | } |
1357 | |
1358 | clear_bit(nr: IPA_REPLENISH_ACTIVE, addr: endpoint->replenish_flags); |
1359 | |
1360 | return; |
1361 | |
1362 | try_again_later: |
1363 | gsi_trans_free(trans); |
1364 | clear_bit(nr: IPA_REPLENISH_ACTIVE, addr: endpoint->replenish_flags); |
1365 | |
1366 | /* Whenever a receive buffer transaction completes we'll try to |
1367 | * replenish again. It's unlikely, but if we fail to supply even |
1368 | * one buffer, nothing will trigger another replenish attempt. |
1369 | * If the hardware has no receive buffers queued, schedule work to |
1370 | * try replenishing again. |
1371 | */ |
1372 | if (gsi_channel_trans_idle(gsi: &endpoint->ipa->gsi, channel_id: endpoint->channel_id)) |
1373 | schedule_delayed_work(dwork: &endpoint->replenish_work, |
1374 | delay: msecs_to_jiffies(m: 1)); |
1375 | } |
1376 | |
1377 | static void ipa_endpoint_replenish_enable(struct ipa_endpoint *endpoint) |
1378 | { |
1379 | set_bit(nr: IPA_REPLENISH_ENABLED, addr: endpoint->replenish_flags); |
1380 | |
1381 | /* Start replenishing if hardware currently has no buffers */ |
1382 | if (gsi_channel_trans_idle(gsi: &endpoint->ipa->gsi, channel_id: endpoint->channel_id)) |
1383 | ipa_endpoint_replenish(endpoint); |
1384 | } |
1385 | |
1386 | static void ipa_endpoint_replenish_disable(struct ipa_endpoint *endpoint) |
1387 | { |
1388 | clear_bit(nr: IPA_REPLENISH_ENABLED, addr: endpoint->replenish_flags); |
1389 | } |
1390 | |
1391 | static void ipa_endpoint_replenish_work(struct work_struct *work) |
1392 | { |
1393 | struct delayed_work *dwork = to_delayed_work(work); |
1394 | struct ipa_endpoint *endpoint; |
1395 | |
1396 | endpoint = container_of(dwork, struct ipa_endpoint, replenish_work); |
1397 | |
1398 | ipa_endpoint_replenish(endpoint); |
1399 | } |
1400 | |
1401 | static void ipa_endpoint_skb_copy(struct ipa_endpoint *endpoint, |
1402 | void *data, u32 len, u32 ) |
1403 | { |
1404 | struct sk_buff *skb; |
1405 | |
1406 | if (!endpoint->netdev) |
1407 | return; |
1408 | |
1409 | skb = __dev_alloc_skb(length: len, GFP_ATOMIC); |
1410 | if (skb) { |
1411 | /* Copy the data into the socket buffer and receive it */ |
1412 | skb_put(skb, len); |
1413 | memcpy(skb->data, data, len); |
1414 | skb->truesize += extra; |
1415 | } |
1416 | |
1417 | ipa_modem_skb_rx(netdev: endpoint->netdev, skb); |
1418 | } |
1419 | |
1420 | static bool ipa_endpoint_skb_build(struct ipa_endpoint *endpoint, |
1421 | struct page *page, u32 len) |
1422 | { |
1423 | u32 buffer_size = endpoint->config.rx.buffer_size; |
1424 | struct sk_buff *skb; |
1425 | |
1426 | /* Nothing to do if there's no netdev */ |
1427 | if (!endpoint->netdev) |
1428 | return false; |
1429 | |
1430 | WARN_ON(len > SKB_WITH_OVERHEAD(buffer_size - NET_SKB_PAD)); |
1431 | |
1432 | skb = build_skb(page_address(page), frag_size: buffer_size); |
1433 | if (skb) { |
1434 | /* Reserve the headroom and account for the data */ |
1435 | skb_reserve(skb, NET_SKB_PAD); |
1436 | skb_put(skb, len); |
1437 | } |
1438 | |
1439 | /* Receive the buffer (or record drop if unable to build it) */ |
1440 | ipa_modem_skb_rx(netdev: endpoint->netdev, skb); |
1441 | |
1442 | return skb != NULL; |
1443 | } |
1444 | |
1445 | /* The format of an IPA packet status structure is the same for several |
1446 | * status types (opcodes). Other types aren't currently supported. |
1447 | */ |
1448 | static bool ipa_status_format_packet(enum ipa_status_opcode opcode) |
1449 | { |
1450 | switch (opcode) { |
1451 | case IPA_STATUS_OPCODE_PACKET: |
1452 | case IPA_STATUS_OPCODE_DROPPED_PACKET: |
1453 | case IPA_STATUS_OPCODE_SUSPENDED_PACKET: |
1454 | case IPA_STATUS_OPCODE_PACKET_2ND_PASS: |
1455 | return true; |
1456 | default: |
1457 | return false; |
1458 | } |
1459 | } |
1460 | |
1461 | static bool |
1462 | ipa_endpoint_status_skip(struct ipa_endpoint *endpoint, const void *data) |
1463 | { |
1464 | struct ipa *ipa = endpoint->ipa; |
1465 | enum ipa_status_opcode opcode; |
1466 | u32 endpoint_id; |
1467 | |
1468 | opcode = ipa_status_extract(ipa, data, field: STATUS_OPCODE); |
1469 | if (!ipa_status_format_packet(opcode)) |
1470 | return true; |
1471 | |
1472 | endpoint_id = ipa_status_extract(ipa, data, field: STATUS_DST_ENDPOINT); |
1473 | if (endpoint_id != endpoint->endpoint_id) |
1474 | return true; |
1475 | |
1476 | return false; /* Don't skip this packet, process it */ |
1477 | } |
1478 | |
1479 | static bool |
1480 | ipa_endpoint_status_tag_valid(struct ipa_endpoint *endpoint, const void *data) |
1481 | { |
1482 | struct ipa_endpoint *command_endpoint; |
1483 | enum ipa_status_mask status_mask; |
1484 | struct ipa *ipa = endpoint->ipa; |
1485 | u32 endpoint_id; |
1486 | |
1487 | status_mask = ipa_status_extract(ipa, data, field: STATUS_MASK); |
1488 | if (!status_mask) |
1489 | return false; /* No valid tag */ |
1490 | |
1491 | /* The status contains a valid tag. We know the packet was sent to |
1492 | * this endpoint (already verified by ipa_endpoint_status_skip()). |
1493 | * If the packet came from the AP->command TX endpoint we know |
1494 | * this packet was sent as part of the pipeline clear process. |
1495 | */ |
1496 | endpoint_id = ipa_status_extract(ipa, data, field: STATUS_SRC_ENDPOINT); |
1497 | command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]; |
1498 | if (endpoint_id == command_endpoint->endpoint_id) { |
1499 | complete(&ipa->completion); |
1500 | } else { |
1501 | dev_err(ipa->dev, "unexpected tagged packet from endpoint %u\n" , |
1502 | endpoint_id); |
1503 | } |
1504 | |
1505 | return true; |
1506 | } |
1507 | |
1508 | /* Return whether the status indicates the packet should be dropped */ |
1509 | static bool |
1510 | ipa_endpoint_status_drop(struct ipa_endpoint *endpoint, const void *data) |
1511 | { |
1512 | enum ipa_status_exception exception; |
1513 | struct ipa *ipa = endpoint->ipa; |
1514 | u32 rule; |
1515 | |
1516 | /* If the status indicates a tagged transfer, we'll drop the packet */ |
1517 | if (ipa_endpoint_status_tag_valid(endpoint, data)) |
1518 | return true; |
1519 | |
1520 | /* Deaggregation exceptions we drop; all other types we consume */ |
1521 | exception = ipa_status_extract(ipa, data, field: STATUS_EXCEPTION); |
1522 | if (exception) |
1523 | return exception == IPA_STATUS_EXCEPTION_DEAGGR; |
1524 | |
1525 | /* Drop the packet if it fails to match a routing rule; otherwise no */ |
1526 | rule = ipa_status_extract(ipa, data, field: STATUS_ROUTER_RULE_INDEX); |
1527 | |
1528 | return rule == IPA_STATUS_RULE_MISS; |
1529 | } |
1530 | |
1531 | static void ipa_endpoint_status_parse(struct ipa_endpoint *endpoint, |
1532 | struct page *page, u32 total_len) |
1533 | { |
1534 | u32 buffer_size = endpoint->config.rx.buffer_size; |
1535 | void *data = page_address(page) + NET_SKB_PAD; |
1536 | u32 unused = buffer_size - total_len; |
1537 | struct ipa *ipa = endpoint->ipa; |
1538 | struct device *dev = ipa->dev; |
1539 | u32 resid = total_len; |
1540 | |
1541 | while (resid) { |
1542 | u32 length; |
1543 | u32 align; |
1544 | u32 len; |
1545 | |
1546 | if (resid < IPA_STATUS_SIZE) { |
1547 | dev_err(dev, |
1548 | "short message (%u bytes < %zu byte status)\n" , |
1549 | resid, IPA_STATUS_SIZE); |
1550 | break; |
1551 | } |
1552 | |
1553 | /* Skip over status packets that lack packet data */ |
1554 | length = ipa_status_extract(ipa, data, field: STATUS_LENGTH); |
1555 | if (!length || ipa_endpoint_status_skip(endpoint, data)) { |
1556 | data += IPA_STATUS_SIZE; |
1557 | resid -= IPA_STATUS_SIZE; |
1558 | continue; |
1559 | } |
1560 | |
1561 | /* Compute the amount of buffer space consumed by the packet, |
1562 | * including the status. If the hardware is configured to |
1563 | * pad packet data to an aligned boundary, account for that. |
1564 | * And if checksum offload is enabled a trailer containing |
1565 | * computed checksum information will be appended. |
1566 | */ |
1567 | align = endpoint->config.rx.pad_align ? : 1; |
1568 | len = IPA_STATUS_SIZE + ALIGN(length, align); |
1569 | if (endpoint->config.checksum) |
1570 | len += sizeof(struct rmnet_map_dl_csum_trailer); |
1571 | |
1572 | if (!ipa_endpoint_status_drop(endpoint, data)) { |
1573 | void *data2; |
1574 | u32 ; |
1575 | |
1576 | /* Client receives only packet data (no status) */ |
1577 | data2 = data + IPA_STATUS_SIZE; |
1578 | |
1579 | /* Have the true size reflect the extra unused space in |
1580 | * the original receive buffer. Distribute the "cost" |
1581 | * proportionately across all aggregated packets in the |
1582 | * buffer. |
1583 | */ |
1584 | extra = DIV_ROUND_CLOSEST(unused * len, total_len); |
1585 | ipa_endpoint_skb_copy(endpoint, data: data2, len: length, extra); |
1586 | } |
1587 | |
1588 | /* Consume status and the full packet it describes */ |
1589 | data += len; |
1590 | resid -= len; |
1591 | } |
1592 | } |
1593 | |
1594 | void ipa_endpoint_trans_complete(struct ipa_endpoint *endpoint, |
1595 | struct gsi_trans *trans) |
1596 | { |
1597 | struct page *page; |
1598 | |
1599 | if (endpoint->toward_ipa) |
1600 | return; |
1601 | |
1602 | if (trans->cancelled) |
1603 | goto done; |
1604 | |
1605 | /* Parse or build a socket buffer using the actual received length */ |
1606 | page = trans->data; |
1607 | if (endpoint->config.status_enable) |
1608 | ipa_endpoint_status_parse(endpoint, page, total_len: trans->len); |
1609 | else if (ipa_endpoint_skb_build(endpoint, page, len: trans->len)) |
1610 | trans->data = NULL; /* Pages have been consumed */ |
1611 | done: |
1612 | ipa_endpoint_replenish(endpoint); |
1613 | } |
1614 | |
1615 | void ipa_endpoint_trans_release(struct ipa_endpoint *endpoint, |
1616 | struct gsi_trans *trans) |
1617 | { |
1618 | if (endpoint->toward_ipa) { |
1619 | struct ipa *ipa = endpoint->ipa; |
1620 | |
1621 | /* Nothing to do for command transactions */ |
1622 | if (endpoint != ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]) { |
1623 | struct sk_buff *skb = trans->data; |
1624 | |
1625 | if (skb) |
1626 | dev_kfree_skb_any(skb); |
1627 | } |
1628 | } else { |
1629 | struct page *page = trans->data; |
1630 | |
1631 | if (page) |
1632 | put_page(page); |
1633 | } |
1634 | } |
1635 | |
1636 | void ipa_endpoint_default_route_set(struct ipa *ipa, u32 endpoint_id) |
1637 | { |
1638 | const struct reg *reg; |
1639 | u32 val; |
1640 | |
1641 | reg = ipa_reg(ipa, reg_id: ROUTE); |
1642 | /* ROUTE_DIS is 0 */ |
1643 | val = reg_encode(reg, field_id: ROUTE_DEF_PIPE, val: endpoint_id); |
1644 | val |= reg_bit(reg, field_id: ROUTE_DEF_HDR_TABLE); |
1645 | /* ROUTE_DEF_HDR_OFST is 0 */ |
1646 | val |= reg_encode(reg, field_id: ROUTE_FRAG_DEF_PIPE, val: endpoint_id); |
1647 | val |= reg_bit(reg, field_id: ROUTE_DEF_RETAIN_HDR); |
1648 | |
1649 | iowrite32(val, ipa->reg_virt + reg_offset(reg)); |
1650 | } |
1651 | |
1652 | void ipa_endpoint_default_route_clear(struct ipa *ipa) |
1653 | { |
1654 | ipa_endpoint_default_route_set(ipa, endpoint_id: 0); |
1655 | } |
1656 | |
1657 | /** |
1658 | * ipa_endpoint_reset_rx_aggr() - Reset RX endpoint with aggregation active |
1659 | * @endpoint: Endpoint to be reset |
1660 | * |
1661 | * If aggregation is active on an RX endpoint when a reset is performed |
1662 | * on its underlying GSI channel, a special sequence of actions must be |
1663 | * taken to ensure the IPA pipeline is properly cleared. |
1664 | * |
1665 | * Return: 0 if successful, or a negative error code |
1666 | */ |
1667 | static int ipa_endpoint_reset_rx_aggr(struct ipa_endpoint *endpoint) |
1668 | { |
1669 | struct ipa *ipa = endpoint->ipa; |
1670 | struct device *dev = ipa->dev; |
1671 | struct gsi *gsi = &ipa->gsi; |
1672 | bool suspended = false; |
1673 | dma_addr_t addr; |
1674 | u32 retries; |
1675 | u32 len = 1; |
1676 | void *virt; |
1677 | int ret; |
1678 | |
1679 | virt = kzalloc(size: len, GFP_KERNEL); |
1680 | if (!virt) |
1681 | return -ENOMEM; |
1682 | |
1683 | addr = dma_map_single(dev, virt, len, DMA_FROM_DEVICE); |
1684 | if (dma_mapping_error(dev, dma_addr: addr)) { |
1685 | ret = -ENOMEM; |
1686 | goto out_kfree; |
1687 | } |
1688 | |
1689 | /* Force close aggregation before issuing the reset */ |
1690 | ipa_endpoint_force_close(endpoint); |
1691 | |
1692 | /* Reset and reconfigure the channel with the doorbell engine |
1693 | * disabled. Then poll until we know aggregation is no longer |
1694 | * active. We'll re-enable the doorbell (if appropriate) when |
1695 | * we reset again below. |
1696 | */ |
1697 | gsi_channel_reset(gsi, channel_id: endpoint->channel_id, doorbell: false); |
1698 | |
1699 | /* Make sure the channel isn't suspended */ |
1700 | suspended = ipa_endpoint_program_suspend(endpoint, enable: false); |
1701 | |
1702 | /* Start channel and do a 1 byte read */ |
1703 | ret = gsi_channel_start(gsi, channel_id: endpoint->channel_id); |
1704 | if (ret) |
1705 | goto out_suspend_again; |
1706 | |
1707 | ret = gsi_trans_read_byte(gsi, channel_id: endpoint->channel_id, addr); |
1708 | if (ret) |
1709 | goto err_endpoint_stop; |
1710 | |
1711 | /* Wait for aggregation to be closed on the channel */ |
1712 | retries = IPA_ENDPOINT_RESET_AGGR_RETRY_MAX; |
1713 | do { |
1714 | if (!ipa_endpoint_aggr_active(endpoint)) |
1715 | break; |
1716 | usleep_range(USEC_PER_MSEC, max: 2 * USEC_PER_MSEC); |
1717 | } while (retries--); |
1718 | |
1719 | /* Check one last time */ |
1720 | if (ipa_endpoint_aggr_active(endpoint)) |
1721 | dev_err(dev, "endpoint %u still active during reset\n" , |
1722 | endpoint->endpoint_id); |
1723 | |
1724 | gsi_trans_read_byte_done(gsi, channel_id: endpoint->channel_id); |
1725 | |
1726 | ret = gsi_channel_stop(gsi, channel_id: endpoint->channel_id); |
1727 | if (ret) |
1728 | goto out_suspend_again; |
1729 | |
1730 | /* Finally, reset and reconfigure the channel again (re-enabling |
1731 | * the doorbell engine if appropriate). Sleep for 1 millisecond to |
1732 | * complete the channel reset sequence. Finish by suspending the |
1733 | * channel again (if necessary). |
1734 | */ |
1735 | gsi_channel_reset(gsi, channel_id: endpoint->channel_id, doorbell: true); |
1736 | |
1737 | usleep_range(USEC_PER_MSEC, max: 2 * USEC_PER_MSEC); |
1738 | |
1739 | goto out_suspend_again; |
1740 | |
1741 | err_endpoint_stop: |
1742 | (void)gsi_channel_stop(gsi, channel_id: endpoint->channel_id); |
1743 | out_suspend_again: |
1744 | if (suspended) |
1745 | (void)ipa_endpoint_program_suspend(endpoint, enable: true); |
1746 | dma_unmap_single(dev, addr, len, DMA_FROM_DEVICE); |
1747 | out_kfree: |
1748 | kfree(objp: virt); |
1749 | |
1750 | return ret; |
1751 | } |
1752 | |
1753 | static void ipa_endpoint_reset(struct ipa_endpoint *endpoint) |
1754 | { |
1755 | u32 channel_id = endpoint->channel_id; |
1756 | struct ipa *ipa = endpoint->ipa; |
1757 | bool special; |
1758 | int ret = 0; |
1759 | |
1760 | /* On IPA v3.5.1, if an RX endpoint is reset while aggregation |
1761 | * is active, we need to handle things specially to recover. |
1762 | * All other cases just need to reset the underlying GSI channel. |
1763 | */ |
1764 | special = ipa->version < IPA_VERSION_4_0 && !endpoint->toward_ipa && |
1765 | endpoint->config.aggregation; |
1766 | if (special && ipa_endpoint_aggr_active(endpoint)) |
1767 | ret = ipa_endpoint_reset_rx_aggr(endpoint); |
1768 | else |
1769 | gsi_channel_reset(gsi: &ipa->gsi, channel_id, doorbell: true); |
1770 | |
1771 | if (ret) |
1772 | dev_err(ipa->dev, |
1773 | "error %d resetting channel %u for endpoint %u\n" , |
1774 | ret, endpoint->channel_id, endpoint->endpoint_id); |
1775 | } |
1776 | |
1777 | static void ipa_endpoint_program(struct ipa_endpoint *endpoint) |
1778 | { |
1779 | if (endpoint->toward_ipa) { |
1780 | /* Newer versions of IPA use GSI channel flow control |
1781 | * instead of endpoint DELAY mode to prevent sending data. |
1782 | * Flow control is disabled for newly-allocated channels, |
1783 | * and we can assume flow control is not (ever) enabled |
1784 | * for AP TX channels. |
1785 | */ |
1786 | if (endpoint->ipa->version < IPA_VERSION_4_2) |
1787 | ipa_endpoint_program_delay(endpoint, enable: false); |
1788 | } else { |
1789 | /* Ensure suspend mode is off on all AP RX endpoints */ |
1790 | (void)ipa_endpoint_program_suspend(endpoint, enable: false); |
1791 | } |
1792 | ipa_endpoint_init_cfg(endpoint); |
1793 | ipa_endpoint_init_nat(endpoint); |
1794 | ipa_endpoint_init_hdr(endpoint); |
1795 | ipa_endpoint_init_hdr_ext(endpoint); |
1796 | ipa_endpoint_init_hdr_metadata_mask(endpoint); |
1797 | ipa_endpoint_init_mode(endpoint); |
1798 | ipa_endpoint_init_aggr(endpoint); |
1799 | if (!endpoint->toward_ipa) { |
1800 | if (endpoint->config.rx.holb_drop) |
1801 | ipa_endpoint_init_hol_block_enable(endpoint, microseconds: 0); |
1802 | else |
1803 | ipa_endpoint_init_hol_block_disable(endpoint); |
1804 | } |
1805 | ipa_endpoint_init_deaggr(endpoint); |
1806 | ipa_endpoint_init_rsrc_grp(endpoint); |
1807 | ipa_endpoint_init_seq(endpoint); |
1808 | ipa_endpoint_status(endpoint); |
1809 | } |
1810 | |
1811 | int ipa_endpoint_enable_one(struct ipa_endpoint *endpoint) |
1812 | { |
1813 | u32 endpoint_id = endpoint->endpoint_id; |
1814 | struct ipa *ipa = endpoint->ipa; |
1815 | struct gsi *gsi = &ipa->gsi; |
1816 | int ret; |
1817 | |
1818 | ret = gsi_channel_start(gsi, channel_id: endpoint->channel_id); |
1819 | if (ret) { |
1820 | dev_err(ipa->dev, |
1821 | "error %d starting %cX channel %u for endpoint %u\n" , |
1822 | ret, endpoint->toward_ipa ? 'T' : 'R', |
1823 | endpoint->channel_id, endpoint_id); |
1824 | return ret; |
1825 | } |
1826 | |
1827 | if (!endpoint->toward_ipa) { |
1828 | ipa_interrupt_suspend_enable(interrupt: ipa->interrupt, endpoint_id); |
1829 | ipa_endpoint_replenish_enable(endpoint); |
1830 | } |
1831 | |
1832 | __set_bit(endpoint_id, ipa->enabled); |
1833 | |
1834 | return 0; |
1835 | } |
1836 | |
1837 | void ipa_endpoint_disable_one(struct ipa_endpoint *endpoint) |
1838 | { |
1839 | u32 endpoint_id = endpoint->endpoint_id; |
1840 | struct ipa *ipa = endpoint->ipa; |
1841 | struct gsi *gsi = &ipa->gsi; |
1842 | int ret; |
1843 | |
1844 | if (!test_bit(endpoint_id, ipa->enabled)) |
1845 | return; |
1846 | |
1847 | __clear_bit(endpoint_id, endpoint->ipa->enabled); |
1848 | |
1849 | if (!endpoint->toward_ipa) { |
1850 | ipa_endpoint_replenish_disable(endpoint); |
1851 | ipa_interrupt_suspend_disable(interrupt: ipa->interrupt, endpoint_id); |
1852 | } |
1853 | |
1854 | /* Note that if stop fails, the channel's state is not well-defined */ |
1855 | ret = gsi_channel_stop(gsi, channel_id: endpoint->channel_id); |
1856 | if (ret) |
1857 | dev_err(ipa->dev, "error %d attempting to stop endpoint %u\n" , |
1858 | ret, endpoint_id); |
1859 | } |
1860 | |
1861 | void ipa_endpoint_suspend_one(struct ipa_endpoint *endpoint) |
1862 | { |
1863 | struct device *dev = endpoint->ipa->dev; |
1864 | struct gsi *gsi = &endpoint->ipa->gsi; |
1865 | int ret; |
1866 | |
1867 | if (!test_bit(endpoint->endpoint_id, endpoint->ipa->enabled)) |
1868 | return; |
1869 | |
1870 | if (!endpoint->toward_ipa) { |
1871 | ipa_endpoint_replenish_disable(endpoint); |
1872 | (void)ipa_endpoint_program_suspend(endpoint, enable: true); |
1873 | } |
1874 | |
1875 | ret = gsi_channel_suspend(gsi, channel_id: endpoint->channel_id); |
1876 | if (ret) |
1877 | dev_err(dev, "error %d suspending channel %u\n" , ret, |
1878 | endpoint->channel_id); |
1879 | } |
1880 | |
1881 | void ipa_endpoint_resume_one(struct ipa_endpoint *endpoint) |
1882 | { |
1883 | struct device *dev = endpoint->ipa->dev; |
1884 | struct gsi *gsi = &endpoint->ipa->gsi; |
1885 | int ret; |
1886 | |
1887 | if (!test_bit(endpoint->endpoint_id, endpoint->ipa->enabled)) |
1888 | return; |
1889 | |
1890 | if (!endpoint->toward_ipa) |
1891 | (void)ipa_endpoint_program_suspend(endpoint, enable: false); |
1892 | |
1893 | ret = gsi_channel_resume(gsi, channel_id: endpoint->channel_id); |
1894 | if (ret) |
1895 | dev_err(dev, "error %d resuming channel %u\n" , ret, |
1896 | endpoint->channel_id); |
1897 | else if (!endpoint->toward_ipa) |
1898 | ipa_endpoint_replenish_enable(endpoint); |
1899 | } |
1900 | |
1901 | void ipa_endpoint_suspend(struct ipa *ipa) |
1902 | { |
1903 | if (!ipa->setup_complete) |
1904 | return; |
1905 | |
1906 | if (ipa->modem_netdev) |
1907 | ipa_modem_suspend(netdev: ipa->modem_netdev); |
1908 | |
1909 | ipa_endpoint_suspend_one(endpoint: ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]); |
1910 | ipa_endpoint_suspend_one(endpoint: ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]); |
1911 | } |
1912 | |
1913 | void ipa_endpoint_resume(struct ipa *ipa) |
1914 | { |
1915 | if (!ipa->setup_complete) |
1916 | return; |
1917 | |
1918 | ipa_endpoint_resume_one(endpoint: ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]); |
1919 | ipa_endpoint_resume_one(endpoint: ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]); |
1920 | |
1921 | if (ipa->modem_netdev) |
1922 | ipa_modem_resume(netdev: ipa->modem_netdev); |
1923 | } |
1924 | |
1925 | static void ipa_endpoint_setup_one(struct ipa_endpoint *endpoint) |
1926 | { |
1927 | struct gsi *gsi = &endpoint->ipa->gsi; |
1928 | u32 channel_id = endpoint->channel_id; |
1929 | |
1930 | /* Only AP endpoints get set up */ |
1931 | if (endpoint->ee_id != GSI_EE_AP) |
1932 | return; |
1933 | |
1934 | endpoint->skb_frag_max = gsi->channel[channel_id].trans_tre_max - 1; |
1935 | if (!endpoint->toward_ipa) { |
1936 | /* RX transactions require a single TRE, so the maximum |
1937 | * backlog is the same as the maximum outstanding TREs. |
1938 | */ |
1939 | clear_bit(nr: IPA_REPLENISH_ENABLED, addr: endpoint->replenish_flags); |
1940 | clear_bit(nr: IPA_REPLENISH_ACTIVE, addr: endpoint->replenish_flags); |
1941 | INIT_DELAYED_WORK(&endpoint->replenish_work, |
1942 | ipa_endpoint_replenish_work); |
1943 | } |
1944 | |
1945 | ipa_endpoint_program(endpoint); |
1946 | |
1947 | __set_bit(endpoint->endpoint_id, endpoint->ipa->set_up); |
1948 | } |
1949 | |
1950 | static void ipa_endpoint_teardown_one(struct ipa_endpoint *endpoint) |
1951 | { |
1952 | __clear_bit(endpoint->endpoint_id, endpoint->ipa->set_up); |
1953 | |
1954 | if (!endpoint->toward_ipa) |
1955 | cancel_delayed_work_sync(dwork: &endpoint->replenish_work); |
1956 | |
1957 | ipa_endpoint_reset(endpoint); |
1958 | } |
1959 | |
1960 | void ipa_endpoint_setup(struct ipa *ipa) |
1961 | { |
1962 | u32 endpoint_id; |
1963 | |
1964 | for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count) |
1965 | ipa_endpoint_setup_one(endpoint: &ipa->endpoint[endpoint_id]); |
1966 | } |
1967 | |
1968 | void ipa_endpoint_teardown(struct ipa *ipa) |
1969 | { |
1970 | u32 endpoint_id; |
1971 | |
1972 | for_each_set_bit(endpoint_id, ipa->set_up, ipa->endpoint_count) |
1973 | ipa_endpoint_teardown_one(endpoint: &ipa->endpoint[endpoint_id]); |
1974 | } |
1975 | |
1976 | void ipa_endpoint_deconfig(struct ipa *ipa) |
1977 | { |
1978 | ipa->available_count = 0; |
1979 | bitmap_free(bitmap: ipa->available); |
1980 | ipa->available = NULL; |
1981 | } |
1982 | |
1983 | int ipa_endpoint_config(struct ipa *ipa) |
1984 | { |
1985 | struct device *dev = ipa->dev; |
1986 | const struct reg *reg; |
1987 | u32 endpoint_id; |
1988 | u32 hw_limit; |
1989 | u32 tx_count; |
1990 | u32 rx_count; |
1991 | u32 rx_base; |
1992 | u32 limit; |
1993 | u32 val; |
1994 | |
1995 | /* Prior to IPA v3.5, the FLAVOR_0 register was not supported. |
1996 | * Furthermore, the endpoints were not grouped such that TX |
1997 | * endpoint numbers started with 0 and RX endpoints had numbers |
1998 | * higher than all TX endpoints, so we can't do the simple |
1999 | * direction check used for newer hardware below. |
2000 | * |
2001 | * For hardware that doesn't support the FLAVOR_0 register, |
2002 | * just set the available mask to support any endpoint, and |
2003 | * assume the configuration is valid. |
2004 | */ |
2005 | if (ipa->version < IPA_VERSION_3_5) { |
2006 | ipa->available = bitmap_zalloc(IPA_ENDPOINT_MAX, GFP_KERNEL); |
2007 | if (!ipa->available) |
2008 | return -ENOMEM; |
2009 | ipa->available_count = IPA_ENDPOINT_MAX; |
2010 | |
2011 | bitmap_set(map: ipa->available, start: 0, IPA_ENDPOINT_MAX); |
2012 | |
2013 | return 0; |
2014 | } |
2015 | |
2016 | /* Find out about the endpoints supplied by the hardware, and ensure |
2017 | * the highest one doesn't exceed the number supported by software. |
2018 | */ |
2019 | reg = ipa_reg(ipa, reg_id: FLAVOR_0); |
2020 | val = ioread32(ipa->reg_virt + reg_offset(reg)); |
2021 | |
2022 | /* Our RX is an IPA producer; our TX is an IPA consumer. */ |
2023 | tx_count = reg_decode(reg, field_id: MAX_CONS_PIPES, val); |
2024 | rx_count = reg_decode(reg, field_id: MAX_PROD_PIPES, val); |
2025 | rx_base = reg_decode(reg, field_id: PROD_LOWEST, val); |
2026 | |
2027 | limit = rx_base + rx_count; |
2028 | if (limit > IPA_ENDPOINT_MAX) { |
2029 | dev_err(dev, "too many endpoints, %u > %u\n" , |
2030 | limit, IPA_ENDPOINT_MAX); |
2031 | return -EINVAL; |
2032 | } |
2033 | |
2034 | /* Until IPA v5.0, the max endpoint ID was 32 */ |
2035 | hw_limit = ipa->version < IPA_VERSION_5_0 ? 32 : U8_MAX + 1; |
2036 | if (limit > hw_limit) { |
2037 | dev_err(dev, "unexpected endpoint count, %u > %u\n" , |
2038 | limit, hw_limit); |
2039 | return -EINVAL; |
2040 | } |
2041 | |
2042 | /* Allocate and initialize the available endpoint bitmap */ |
2043 | ipa->available = bitmap_zalloc(nbits: limit, GFP_KERNEL); |
2044 | if (!ipa->available) |
2045 | return -ENOMEM; |
2046 | ipa->available_count = limit; |
2047 | |
2048 | /* Mark all supported RX and TX endpoints as available */ |
2049 | bitmap_set(map: ipa->available, start: 0, nbits: tx_count); |
2050 | bitmap_set(map: ipa->available, start: rx_base, nbits: rx_count); |
2051 | |
2052 | for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count) { |
2053 | struct ipa_endpoint *endpoint; |
2054 | |
2055 | if (endpoint_id >= limit) { |
2056 | dev_err(dev, "invalid endpoint id, %u > %u\n" , |
2057 | endpoint_id, limit - 1); |
2058 | goto err_free_bitmap; |
2059 | } |
2060 | |
2061 | if (!test_bit(endpoint_id, ipa->available)) { |
2062 | dev_err(dev, "unavailable endpoint id %u\n" , |
2063 | endpoint_id); |
2064 | goto err_free_bitmap; |
2065 | } |
2066 | |
2067 | /* Make sure it's pointing in the right direction */ |
2068 | endpoint = &ipa->endpoint[endpoint_id]; |
2069 | if (endpoint->toward_ipa) { |
2070 | if (endpoint_id < tx_count) |
2071 | continue; |
2072 | } else if (endpoint_id >= rx_base) { |
2073 | continue; |
2074 | } |
2075 | |
2076 | dev_err(dev, "endpoint id %u wrong direction\n" , endpoint_id); |
2077 | goto err_free_bitmap; |
2078 | } |
2079 | |
2080 | return 0; |
2081 | |
2082 | err_free_bitmap: |
2083 | ipa_endpoint_deconfig(ipa); |
2084 | |
2085 | return -EINVAL; |
2086 | } |
2087 | |
2088 | static void ipa_endpoint_init_one(struct ipa *ipa, enum ipa_endpoint_name name, |
2089 | const struct ipa_gsi_endpoint_data *data) |
2090 | { |
2091 | struct ipa_endpoint *endpoint; |
2092 | |
2093 | endpoint = &ipa->endpoint[data->endpoint_id]; |
2094 | |
2095 | if (data->ee_id == GSI_EE_AP) |
2096 | ipa->channel_map[data->channel_id] = endpoint; |
2097 | ipa->name_map[name] = endpoint; |
2098 | |
2099 | endpoint->ipa = ipa; |
2100 | endpoint->ee_id = data->ee_id; |
2101 | endpoint->channel_id = data->channel_id; |
2102 | endpoint->endpoint_id = data->endpoint_id; |
2103 | endpoint->toward_ipa = data->toward_ipa; |
2104 | endpoint->config = data->endpoint.config; |
2105 | |
2106 | __set_bit(endpoint->endpoint_id, ipa->defined); |
2107 | } |
2108 | |
2109 | static void ipa_endpoint_exit_one(struct ipa_endpoint *endpoint) |
2110 | { |
2111 | __clear_bit(endpoint->endpoint_id, endpoint->ipa->defined); |
2112 | |
2113 | memset(endpoint, 0, sizeof(*endpoint)); |
2114 | } |
2115 | |
2116 | void ipa_endpoint_exit(struct ipa *ipa) |
2117 | { |
2118 | u32 endpoint_id; |
2119 | |
2120 | ipa->filtered = 0; |
2121 | |
2122 | for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count) |
2123 | ipa_endpoint_exit_one(endpoint: &ipa->endpoint[endpoint_id]); |
2124 | |
2125 | bitmap_free(bitmap: ipa->enabled); |
2126 | ipa->enabled = NULL; |
2127 | bitmap_free(bitmap: ipa->set_up); |
2128 | ipa->set_up = NULL; |
2129 | bitmap_free(bitmap: ipa->defined); |
2130 | ipa->defined = NULL; |
2131 | |
2132 | memset(ipa->name_map, 0, sizeof(ipa->name_map)); |
2133 | memset(ipa->channel_map, 0, sizeof(ipa->channel_map)); |
2134 | } |
2135 | |
2136 | /* Returns a bitmask of endpoints that support filtering, or 0 on error */ |
2137 | int ipa_endpoint_init(struct ipa *ipa, u32 count, |
2138 | const struct ipa_gsi_endpoint_data *data) |
2139 | { |
2140 | enum ipa_endpoint_name name; |
2141 | u32 filtered; |
2142 | |
2143 | BUILD_BUG_ON(!IPA_REPLENISH_BATCH); |
2144 | |
2145 | /* Number of endpoints is one more than the maximum ID */ |
2146 | ipa->endpoint_count = ipa_endpoint_max(ipa, count, data) + 1; |
2147 | if (!ipa->endpoint_count) |
2148 | return -EINVAL; |
2149 | |
2150 | /* Initialize endpoint state bitmaps */ |
2151 | ipa->defined = bitmap_zalloc(nbits: ipa->endpoint_count, GFP_KERNEL); |
2152 | if (!ipa->defined) |
2153 | return -ENOMEM; |
2154 | |
2155 | ipa->set_up = bitmap_zalloc(nbits: ipa->endpoint_count, GFP_KERNEL); |
2156 | if (!ipa->set_up) |
2157 | goto err_free_defined; |
2158 | |
2159 | ipa->enabled = bitmap_zalloc(nbits: ipa->endpoint_count, GFP_KERNEL); |
2160 | if (!ipa->enabled) |
2161 | goto err_free_set_up; |
2162 | |
2163 | filtered = 0; |
2164 | for (name = 0; name < count; name++, data++) { |
2165 | if (ipa_gsi_endpoint_data_empty(data)) |
2166 | continue; /* Skip over empty slots */ |
2167 | |
2168 | ipa_endpoint_init_one(ipa, name, data); |
2169 | |
2170 | if (data->endpoint.filter_support) |
2171 | filtered |= BIT(data->endpoint_id); |
2172 | if (data->ee_id == GSI_EE_MODEM && data->toward_ipa) |
2173 | ipa->modem_tx_count++; |
2174 | } |
2175 | |
2176 | /* Make sure the set of filtered endpoints is valid */ |
2177 | if (!ipa_filtered_valid(ipa, filtered)) { |
2178 | ipa_endpoint_exit(ipa); |
2179 | |
2180 | return -EINVAL; |
2181 | } |
2182 | |
2183 | ipa->filtered = filtered; |
2184 | |
2185 | return 0; |
2186 | |
2187 | err_free_set_up: |
2188 | bitmap_free(bitmap: ipa->set_up); |
2189 | ipa->set_up = NULL; |
2190 | err_free_defined: |
2191 | bitmap_free(bitmap: ipa->defined); |
2192 | ipa->defined = NULL; |
2193 | |
2194 | return -ENOMEM; |
2195 | } |
2196 | |