1// SPDX-License-Identifier: GPL-2.0
2
3/* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2019-2024 Linaro Ltd.
5 */
6
7#include <linux/bitfield.h>
8#include <linux/bits.h>
9#include <linux/device.h>
10#include <linux/dma-direction.h>
11#include <linux/if_rmnet.h>
12#include <linux/types.h>
13
14#include "gsi.h"
15#include "gsi_trans.h"
16#include "ipa.h"
17#include "ipa_cmd.h"
18#include "ipa_data.h"
19#include "ipa_endpoint.h"
20#include "ipa_gsi.h"
21#include "ipa_interrupt.h"
22#include "ipa_mem.h"
23#include "ipa_modem.h"
24#include "ipa_power.h"
25#include "ipa_reg.h"
26#include "ipa_table.h"
27#include "ipa_version.h"
28
29/* Hardware is told about receive buffers once a "batch" has been queued */
30#define IPA_REPLENISH_BATCH 16 /* Must be non-zero */
31
32/* The amount of RX buffer space consumed by standard skb overhead */
33#define IPA_RX_BUFFER_OVERHEAD (PAGE_SIZE - SKB_MAX_ORDER(NET_SKB_PAD, 0))
34
35/* Where to find the QMAP mux_id for a packet within modem-supplied metadata */
36#define IPA_ENDPOINT_QMAP_METADATA_MASK 0x000000ff /* host byte order */
37
38#define IPA_ENDPOINT_RESET_AGGR_RETRY_MAX 3
39
40/** enum ipa_status_opcode - IPA status opcode field hardware values */
41enum ipa_status_opcode { /* *Not* a bitmask */
42 IPA_STATUS_OPCODE_PACKET = 1,
43 IPA_STATUS_OPCODE_NEW_RULE_PACKET = 2,
44 IPA_STATUS_OPCODE_DROPPED_PACKET = 4,
45 IPA_STATUS_OPCODE_SUSPENDED_PACKET = 8,
46 IPA_STATUS_OPCODE_LOG = 16,
47 IPA_STATUS_OPCODE_DCMP = 32,
48 IPA_STATUS_OPCODE_PACKET_2ND_PASS = 64,
49};
50
51/** enum ipa_status_exception - IPA status exception field hardware values */
52enum ipa_status_exception { /* *Not* a bitmask */
53 /* 0 means no exception */
54 IPA_STATUS_EXCEPTION_DEAGGR = 1,
55 IPA_STATUS_EXCEPTION_IPTYPE = 4,
56 IPA_STATUS_EXCEPTION_PACKET_LENGTH = 8,
57 IPA_STATUS_EXCEPTION_FRAG_RULE_MISS = 16,
58 IPA_STATUS_EXCEPTION_SW_FILTER = 32,
59 IPA_STATUS_EXCEPTION_NAT = 64, /* IPv4 */
60 IPA_STATUS_EXCEPTION_IPV6_CONN_TRACK = 64, /* IPv6 */
61 IPA_STATUS_EXCEPTION_UC = 128,
62 IPA_STATUS_EXCEPTION_INVALID_ENDPOINT = 129,
63 IPA_STATUS_EXCEPTION_HEADER_INSERT = 136,
64 IPA_STATUS_EXCEPTION_CHEKCSUM = 229,
65};
66
67/** enum ipa_status_mask - IPA status mask field bitmask hardware values */
68enum ipa_status_mask {
69 IPA_STATUS_MASK_FRAG_PROCESS = BIT(0),
70 IPA_STATUS_MASK_FILT_PROCESS = BIT(1),
71 IPA_STATUS_MASK_NAT_PROCESS = BIT(2),
72 IPA_STATUS_MASK_ROUTE_PROCESS = BIT(3),
73 IPA_STATUS_MASK_TAG_VALID = BIT(4),
74 IPA_STATUS_MASK_FRAGMENT = BIT(5),
75 IPA_STATUS_MASK_FIRST_FRAGMENT = BIT(6),
76 IPA_STATUS_MASK_V4 = BIT(7),
77 IPA_STATUS_MASK_CKSUM_PROCESS = BIT(8),
78 IPA_STATUS_MASK_AGGR_PROCESS = BIT(9),
79 IPA_STATUS_MASK_DEST_EOT = BIT(10),
80 IPA_STATUS_MASK_DEAGGR_PROCESS = BIT(11),
81 IPA_STATUS_MASK_DEAGG_FIRST = BIT(12),
82 IPA_STATUS_MASK_SRC_EOT = BIT(13),
83 IPA_STATUS_MASK_PREV_EOT = BIT(14),
84 IPA_STATUS_MASK_BYTE_LIMIT = BIT(15),
85};
86
87/* Special IPA filter/router rule field value indicating "rule miss" */
88#define IPA_STATUS_RULE_MISS 0x3ff /* 10-bit filter/router rule fields */
89
90/** The IPA status nat_type field uses enum ipa_nat_type hardware values */
91
92/* enum ipa_status_field_id - IPA packet status structure field identifiers */
93enum ipa_status_field_id {
94 STATUS_OPCODE, /* enum ipa_status_opcode */
95 STATUS_EXCEPTION, /* enum ipa_status_exception */
96 STATUS_MASK, /* enum ipa_status_mask (bitmask) */
97 STATUS_LENGTH,
98 STATUS_SRC_ENDPOINT,
99 STATUS_DST_ENDPOINT,
100 STATUS_METADATA,
101 STATUS_FILTER_LOCAL, /* Boolean */
102 STATUS_FILTER_HASH, /* Boolean */
103 STATUS_FILTER_GLOBAL, /* Boolean */
104 STATUS_FILTER_RETAIN, /* Boolean */
105 STATUS_FILTER_RULE_INDEX,
106 STATUS_ROUTER_LOCAL, /* Boolean */
107 STATUS_ROUTER_HASH, /* Boolean */
108 STATUS_UCP, /* Boolean */
109 STATUS_ROUTER_TABLE,
110 STATUS_ROUTER_RULE_INDEX,
111 STATUS_NAT_HIT, /* Boolean */
112 STATUS_NAT_INDEX,
113 STATUS_NAT_TYPE, /* enum ipa_nat_type */
114 STATUS_TAG_LOW32, /* Low-order 32 bits of 48-bit tag */
115 STATUS_TAG_HIGH16, /* High-order 16 bits of 48-bit tag */
116 STATUS_SEQUENCE,
117 STATUS_TIME_OF_DAY,
118 STATUS_HEADER_LOCAL, /* Boolean */
119 STATUS_HEADER_OFFSET,
120 STATUS_FRAG_HIT, /* Boolean */
121 STATUS_FRAG_RULE_INDEX,
122};
123
124/* Size in bytes of an IPA packet status structure */
125#define IPA_STATUS_SIZE sizeof(__le32[8])
126
127/* IPA status structure decoder; looks up field values for a structure */
128static u32 ipa_status_extract(struct ipa *ipa, const void *data,
129 enum ipa_status_field_id field)
130{
131 enum ipa_version version = ipa->version;
132 const __le32 *word = data;
133
134 switch (field) {
135 case STATUS_OPCODE:
136 return le32_get_bits(v: word[0], GENMASK(7, 0));
137 case STATUS_EXCEPTION:
138 return le32_get_bits(v: word[0], GENMASK(15, 8));
139 case STATUS_MASK:
140 return le32_get_bits(v: word[0], GENMASK(31, 16));
141 case STATUS_LENGTH:
142 return le32_get_bits(v: word[1], GENMASK(15, 0));
143 case STATUS_SRC_ENDPOINT:
144 if (version < IPA_VERSION_5_0)
145 return le32_get_bits(v: word[1], GENMASK(20, 16));
146 return le32_get_bits(v: word[1], GENMASK(23, 16));
147 /* Status word 1, bits 21-23 are reserved (not IPA v5.0+) */
148 /* Status word 1, bits 24-26 are reserved (IPA v5.0+) */
149 case STATUS_DST_ENDPOINT:
150 if (version < IPA_VERSION_5_0)
151 return le32_get_bits(v: word[1], GENMASK(28, 24));
152 return le32_get_bits(v: word[7], GENMASK(23, 16));
153 /* Status word 1, bits 29-31 are reserved */
154 case STATUS_METADATA:
155 return le32_to_cpu(word[2]);
156 case STATUS_FILTER_LOCAL:
157 return le32_get_bits(v: word[3], GENMASK(0, 0));
158 case STATUS_FILTER_HASH:
159 return le32_get_bits(v: word[3], GENMASK(1, 1));
160 case STATUS_FILTER_GLOBAL:
161 return le32_get_bits(v: word[3], GENMASK(2, 2));
162 case STATUS_FILTER_RETAIN:
163 return le32_get_bits(v: word[3], GENMASK(3, 3));
164 case STATUS_FILTER_RULE_INDEX:
165 return le32_get_bits(v: word[3], GENMASK(13, 4));
166 /* ROUTER_TABLE is in word 3, bits 14-21 (IPA v5.0+) */
167 case STATUS_ROUTER_LOCAL:
168 if (version < IPA_VERSION_5_0)
169 return le32_get_bits(v: word[3], GENMASK(14, 14));
170 return le32_get_bits(v: word[1], GENMASK(27, 27));
171 case STATUS_ROUTER_HASH:
172 if (version < IPA_VERSION_5_0)
173 return le32_get_bits(v: word[3], GENMASK(15, 15));
174 return le32_get_bits(v: word[1], GENMASK(28, 28));
175 case STATUS_UCP:
176 if (version < IPA_VERSION_5_0)
177 return le32_get_bits(v: word[3], GENMASK(16, 16));
178 return le32_get_bits(v: word[7], GENMASK(31, 31));
179 case STATUS_ROUTER_TABLE:
180 if (version < IPA_VERSION_5_0)
181 return le32_get_bits(v: word[3], GENMASK(21, 17));
182 return le32_get_bits(v: word[3], GENMASK(21, 14));
183 case STATUS_ROUTER_RULE_INDEX:
184 return le32_get_bits(v: word[3], GENMASK(31, 22));
185 case STATUS_NAT_HIT:
186 return le32_get_bits(v: word[4], GENMASK(0, 0));
187 case STATUS_NAT_INDEX:
188 return le32_get_bits(v: word[4], GENMASK(13, 1));
189 case STATUS_NAT_TYPE:
190 return le32_get_bits(v: word[4], GENMASK(15, 14));
191 case STATUS_TAG_LOW32:
192 return le32_get_bits(v: word[4], GENMASK(31, 16)) |
193 (le32_get_bits(v: word[5], GENMASK(15, 0)) << 16);
194 case STATUS_TAG_HIGH16:
195 return le32_get_bits(v: word[5], GENMASK(31, 16));
196 case STATUS_SEQUENCE:
197 return le32_get_bits(v: word[6], GENMASK(7, 0));
198 case STATUS_TIME_OF_DAY:
199 return le32_get_bits(v: word[6], GENMASK(31, 8));
200 case STATUS_HEADER_LOCAL:
201 return le32_get_bits(v: word[7], GENMASK(0, 0));
202 case STATUS_HEADER_OFFSET:
203 return le32_get_bits(v: word[7], GENMASK(10, 1));
204 case STATUS_FRAG_HIT:
205 return le32_get_bits(v: word[7], GENMASK(11, 11));
206 case STATUS_FRAG_RULE_INDEX:
207 return le32_get_bits(v: word[7], GENMASK(15, 12));
208 /* Status word 7, bits 16-30 are reserved */
209 /* Status word 7, bit 31 is reserved (not IPA v5.0+) */
210 default:
211 WARN(true, "%s: bad field_id %u\n", __func__, field);
212 return 0;
213 }
214}
215
216/* Compute the aggregation size value to use for a given buffer size */
217static u32 ipa_aggr_size_kb(u32 rx_buffer_size, bool aggr_hard_limit)
218{
219 /* A hard aggregation limit will not be crossed; aggregation closes
220 * if saving incoming data would cross the hard byte limit boundary.
221 *
222 * With a soft limit, aggregation closes *after* the size boundary
223 * has been crossed. In that case the limit must leave enough space
224 * after that limit to receive a full MTU of data plus overhead.
225 */
226 if (!aggr_hard_limit)
227 rx_buffer_size -= IPA_MTU + IPA_RX_BUFFER_OVERHEAD;
228
229 /* The byte limit is encoded as a number of kilobytes */
230
231 return rx_buffer_size / SZ_1K;
232}
233
234static bool ipa_endpoint_data_valid_one(struct ipa *ipa, u32 count,
235 const struct ipa_gsi_endpoint_data *all_data,
236 const struct ipa_gsi_endpoint_data *data)
237{
238 const struct ipa_gsi_endpoint_data *other_data;
239 enum ipa_endpoint_name other_name;
240 struct device *dev = ipa->dev;
241
242 if (ipa_gsi_endpoint_data_empty(data))
243 return true;
244
245 if (!data->toward_ipa) {
246 const struct ipa_endpoint_rx *rx_config;
247 const struct reg *reg;
248 u32 buffer_size;
249 u32 aggr_size;
250 u32 limit;
251
252 if (data->endpoint.filter_support) {
253 dev_err(dev, "filtering not supported for "
254 "RX endpoint %u\n",
255 data->endpoint_id);
256 return false;
257 }
258
259 /* Nothing more to check for non-AP RX */
260 if (data->ee_id != GSI_EE_AP)
261 return true;
262
263 rx_config = &data->endpoint.config.rx;
264
265 /* The buffer size must hold an MTU plus overhead */
266 buffer_size = rx_config->buffer_size;
267 limit = IPA_MTU + IPA_RX_BUFFER_OVERHEAD;
268 if (buffer_size < limit) {
269 dev_err(dev, "RX buffer size too small for RX endpoint %u (%u < %u)\n",
270 data->endpoint_id, buffer_size, limit);
271 return false;
272 }
273
274 if (!data->endpoint.config.aggregation) {
275 bool result = true;
276
277 /* No aggregation; check for bogus aggregation data */
278 if (rx_config->aggr_time_limit) {
279 dev_err(dev,
280 "time limit with no aggregation for RX endpoint %u\n",
281 data->endpoint_id);
282 result = false;
283 }
284
285 if (rx_config->aggr_hard_limit) {
286 dev_err(dev, "hard limit with no aggregation for RX endpoint %u\n",
287 data->endpoint_id);
288 result = false;
289 }
290
291 if (rx_config->aggr_close_eof) {
292 dev_err(dev, "close EOF with no aggregation for RX endpoint %u\n",
293 data->endpoint_id);
294 result = false;
295 }
296
297 return result; /* Nothing more to check */
298 }
299
300 /* For an endpoint supporting receive aggregation, the byte
301 * limit defines the point at which aggregation closes. This
302 * check ensures the receive buffer size doesn't result in a
303 * limit that exceeds what's representable in the aggregation
304 * byte limit field.
305 */
306 aggr_size = ipa_aggr_size_kb(rx_buffer_size: buffer_size - NET_SKB_PAD,
307 aggr_hard_limit: rx_config->aggr_hard_limit);
308 reg = ipa_reg(ipa, reg_id: ENDP_INIT_AGGR);
309
310 limit = reg_field_max(reg, field_id: BYTE_LIMIT);
311 if (aggr_size > limit) {
312 dev_err(dev, "aggregated size too large for RX endpoint %u (%u KB > %u KB)\n",
313 data->endpoint_id, aggr_size, limit);
314
315 return false;
316 }
317
318 return true; /* Nothing more to check for RX */
319 }
320
321 /* Starting with IPA v4.5 sequencer replication is obsolete */
322 if (ipa->version >= IPA_VERSION_4_5) {
323 if (data->endpoint.config.tx.seq_rep_type) {
324 dev_err(dev, "no-zero seq_rep_type TX endpoint %u\n",
325 data->endpoint_id);
326 return false;
327 }
328 }
329
330 if (data->endpoint.config.status_enable) {
331 other_name = data->endpoint.config.tx.status_endpoint;
332 if (other_name >= count) {
333 dev_err(dev, "status endpoint name %u out of range "
334 "for endpoint %u\n",
335 other_name, data->endpoint_id);
336 return false;
337 }
338
339 /* Status endpoint must be defined... */
340 other_data = &all_data[other_name];
341 if (ipa_gsi_endpoint_data_empty(data: other_data)) {
342 dev_err(dev, "DMA endpoint name %u undefined "
343 "for endpoint %u\n",
344 other_name, data->endpoint_id);
345 return false;
346 }
347
348 /* ...and has to be an RX endpoint... */
349 if (other_data->toward_ipa) {
350 dev_err(dev,
351 "status endpoint for endpoint %u not RX\n",
352 data->endpoint_id);
353 return false;
354 }
355
356 /* ...and if it's to be an AP endpoint... */
357 if (other_data->ee_id == GSI_EE_AP) {
358 /* ...make sure it has status enabled. */
359 if (!other_data->endpoint.config.status_enable) {
360 dev_err(dev,
361 "status not enabled for endpoint %u\n",
362 other_data->endpoint_id);
363 return false;
364 }
365 }
366 }
367
368 if (data->endpoint.config.dma_mode) {
369 other_name = data->endpoint.config.dma_endpoint;
370 if (other_name >= count) {
371 dev_err(dev, "DMA endpoint name %u out of range "
372 "for endpoint %u\n",
373 other_name, data->endpoint_id);
374 return false;
375 }
376
377 other_data = &all_data[other_name];
378 if (ipa_gsi_endpoint_data_empty(data: other_data)) {
379 dev_err(dev, "DMA endpoint name %u undefined "
380 "for endpoint %u\n",
381 other_name, data->endpoint_id);
382 return false;
383 }
384 }
385
386 return true;
387}
388
389/* Validate endpoint configuration data. Return max defined endpoint ID */
390static u32 ipa_endpoint_max(struct ipa *ipa, u32 count,
391 const struct ipa_gsi_endpoint_data *data)
392{
393 const struct ipa_gsi_endpoint_data *dp = data;
394 struct device *dev = ipa->dev;
395 enum ipa_endpoint_name name;
396 u32 max;
397
398 if (count > IPA_ENDPOINT_COUNT) {
399 dev_err(dev, "too many endpoints specified (%u > %u)\n",
400 count, IPA_ENDPOINT_COUNT);
401 return 0;
402 }
403
404 /* Make sure needed endpoints have defined data */
405 if (ipa_gsi_endpoint_data_empty(data: &data[IPA_ENDPOINT_AP_COMMAND_TX])) {
406 dev_err(dev, "command TX endpoint not defined\n");
407 return 0;
408 }
409 if (ipa_gsi_endpoint_data_empty(data: &data[IPA_ENDPOINT_AP_LAN_RX])) {
410 dev_err(dev, "LAN RX endpoint not defined\n");
411 return 0;
412 }
413 if (ipa_gsi_endpoint_data_empty(data: &data[IPA_ENDPOINT_AP_MODEM_TX])) {
414 dev_err(dev, "AP->modem TX endpoint not defined\n");
415 return 0;
416 }
417 if (ipa_gsi_endpoint_data_empty(data: &data[IPA_ENDPOINT_AP_MODEM_RX])) {
418 dev_err(dev, "AP<-modem RX endpoint not defined\n");
419 return 0;
420 }
421
422 max = 0;
423 for (name = 0; name < count; name++, dp++) {
424 if (!ipa_endpoint_data_valid_one(ipa, count, all_data: data, data: dp))
425 return 0;
426 max = max_t(u32, max, dp->endpoint_id);
427 }
428
429 return max;
430}
431
432/* Allocate a transaction to use on a non-command endpoint */
433static struct gsi_trans *ipa_endpoint_trans_alloc(struct ipa_endpoint *endpoint,
434 u32 tre_count)
435{
436 struct gsi *gsi = &endpoint->ipa->gsi;
437 u32 channel_id = endpoint->channel_id;
438 enum dma_data_direction direction;
439
440 direction = endpoint->toward_ipa ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
441
442 return gsi_channel_trans_alloc(gsi, channel_id, tre_count, direction);
443}
444
445/* suspend_delay represents suspend for RX, delay for TX endpoints.
446 * Note that suspend is not supported starting with IPA v4.0, and
447 * delay mode should not be used starting with IPA v4.2.
448 */
449static bool
450ipa_endpoint_init_ctrl(struct ipa_endpoint *endpoint, bool suspend_delay)
451{
452 struct ipa *ipa = endpoint->ipa;
453 const struct reg *reg;
454 u32 field_id;
455 u32 offset;
456 bool state;
457 u32 mask;
458 u32 val;
459
460 if (endpoint->toward_ipa)
461 WARN_ON(ipa->version >= IPA_VERSION_4_2);
462 else
463 WARN_ON(ipa->version >= IPA_VERSION_4_0);
464
465 reg = ipa_reg(ipa, reg_id: ENDP_INIT_CTRL);
466 offset = reg_n_offset(reg, n: endpoint->endpoint_id);
467 val = ioread32(ipa->reg_virt + offset);
468
469 field_id = endpoint->toward_ipa ? ENDP_DELAY : ENDP_SUSPEND;
470 mask = reg_bit(reg, field_id);
471
472 state = !!(val & mask);
473
474 /* Don't bother if it's already in the requested state */
475 if (suspend_delay != state) {
476 val ^= mask;
477 iowrite32(val, ipa->reg_virt + offset);
478 }
479
480 return state;
481}
482
483/* We don't care what the previous state was for delay mode */
484static void
485ipa_endpoint_program_delay(struct ipa_endpoint *endpoint, bool enable)
486{
487 /* Delay mode should not be used for IPA v4.2+ */
488 WARN_ON(endpoint->ipa->version >= IPA_VERSION_4_2);
489 WARN_ON(!endpoint->toward_ipa);
490
491 (void)ipa_endpoint_init_ctrl(endpoint, suspend_delay: enable);
492}
493
494static bool ipa_endpoint_aggr_active(struct ipa_endpoint *endpoint)
495{
496 u32 endpoint_id = endpoint->endpoint_id;
497 struct ipa *ipa = endpoint->ipa;
498 u32 unit = endpoint_id / 32;
499 const struct reg *reg;
500 u32 val;
501
502 WARN_ON(!test_bit(endpoint_id, ipa->available));
503
504 reg = ipa_reg(ipa, reg_id: STATE_AGGR_ACTIVE);
505 val = ioread32(ipa->reg_virt + reg_n_offset(reg, n: unit));
506
507 return !!(val & BIT(endpoint_id % 32));
508}
509
510static void ipa_endpoint_force_close(struct ipa_endpoint *endpoint)
511{
512 u32 endpoint_id = endpoint->endpoint_id;
513 u32 mask = BIT(endpoint_id % 32);
514 struct ipa *ipa = endpoint->ipa;
515 u32 unit = endpoint_id / 32;
516 const struct reg *reg;
517
518 WARN_ON(!test_bit(endpoint_id, ipa->available));
519
520 reg = ipa_reg(ipa, reg_id: AGGR_FORCE_CLOSE);
521 iowrite32(mask, ipa->reg_virt + reg_n_offset(reg, n: unit));
522}
523
524/**
525 * ipa_endpoint_suspend_aggr() - Emulate suspend interrupt
526 * @endpoint: Endpoint on which to emulate a suspend
527 *
528 * Emulate suspend IPA interrupt to unsuspend an endpoint suspended
529 * with an open aggregation frame. This is to work around a hardware
530 * issue in IPA version 3.5.1 where the suspend interrupt will not be
531 * generated when it should be.
532 */
533static void ipa_endpoint_suspend_aggr(struct ipa_endpoint *endpoint)
534{
535 struct ipa *ipa = endpoint->ipa;
536
537 if (!endpoint->config.aggregation)
538 return;
539
540 /* Nothing to do if the endpoint doesn't have aggregation open */
541 if (!ipa_endpoint_aggr_active(endpoint))
542 return;
543
544 /* Force close aggregation */
545 ipa_endpoint_force_close(endpoint);
546
547 ipa_interrupt_simulate_suspend(interrupt: ipa->interrupt);
548}
549
550/* Returns previous suspend state (true means suspend was enabled) */
551static bool
552ipa_endpoint_program_suspend(struct ipa_endpoint *endpoint, bool enable)
553{
554 bool suspended;
555
556 if (endpoint->ipa->version >= IPA_VERSION_4_0)
557 return enable; /* For IPA v4.0+, no change made */
558
559 WARN_ON(endpoint->toward_ipa);
560
561 suspended = ipa_endpoint_init_ctrl(endpoint, suspend_delay: enable);
562
563 /* A client suspended with an open aggregation frame will not
564 * generate a SUSPEND IPA interrupt. If enabling suspend, have
565 * ipa_endpoint_suspend_aggr() handle this.
566 */
567 if (enable && !suspended)
568 ipa_endpoint_suspend_aggr(endpoint);
569
570 return suspended;
571}
572
573/* Put all modem RX endpoints into suspend mode, and stop transmission
574 * on all modem TX endpoints. Prior to IPA v4.2, endpoint DELAY mode is
575 * used for TX endpoints; starting with IPA v4.2 we use GSI channel flow
576 * control instead.
577 */
578void ipa_endpoint_modem_pause_all(struct ipa *ipa, bool enable)
579{
580 u32 endpoint_id = 0;
581
582 while (endpoint_id < ipa->endpoint_count) {
583 struct ipa_endpoint *endpoint = &ipa->endpoint[endpoint_id++];
584
585 if (endpoint->ee_id != GSI_EE_MODEM)
586 continue;
587
588 if (!endpoint->toward_ipa)
589 (void)ipa_endpoint_program_suspend(endpoint, enable);
590 else if (ipa->version < IPA_VERSION_4_2)
591 ipa_endpoint_program_delay(endpoint, enable);
592 else
593 gsi_modem_channel_flow_control(gsi: &ipa->gsi,
594 channel_id: endpoint->channel_id,
595 enable);
596 }
597}
598
599/* Reset all modem endpoints to use the default exception endpoint */
600int ipa_endpoint_modem_exception_reset_all(struct ipa *ipa)
601{
602 struct gsi_trans *trans;
603 u32 endpoint_id;
604 u32 count;
605
606 /* We need one command per modem TX endpoint, plus the commands
607 * that clear the pipeline.
608 */
609 count = ipa->modem_tx_count + ipa_cmd_pipeline_clear_count();
610 trans = ipa_cmd_trans_alloc(ipa, tre_count: count);
611 if (!trans) {
612 dev_err(ipa->dev,
613 "no transaction to reset modem exception endpoints\n");
614 return -EBUSY;
615 }
616
617 for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count) {
618 struct ipa_endpoint *endpoint;
619 const struct reg *reg;
620 u32 offset;
621
622 /* We only reset modem TX endpoints */
623 endpoint = &ipa->endpoint[endpoint_id];
624 if (!(endpoint->ee_id == GSI_EE_MODEM && endpoint->toward_ipa))
625 continue;
626
627 reg = ipa_reg(ipa, reg_id: ENDP_STATUS);
628 offset = reg_n_offset(reg, n: endpoint_id);
629
630 /* Value written is 0, and all bits are updated. That
631 * means status is disabled on the endpoint, and as a
632 * result all other fields in the register are ignored.
633 */
634 ipa_cmd_register_write_add(trans, offset, value: 0, mask: ~0, clear_full: false);
635 }
636
637 ipa_cmd_pipeline_clear_add(trans);
638
639 gsi_trans_commit_wait(trans);
640
641 ipa_cmd_pipeline_clear_wait(ipa);
642
643 return 0;
644}
645
646static void ipa_endpoint_init_cfg(struct ipa_endpoint *endpoint)
647{
648 u32 endpoint_id = endpoint->endpoint_id;
649 struct ipa *ipa = endpoint->ipa;
650 enum ipa_cs_offload_en enabled;
651 const struct reg *reg;
652 u32 val = 0;
653
654 reg = ipa_reg(ipa, reg_id: ENDP_INIT_CFG);
655 /* FRAG_OFFLOAD_EN is 0 */
656 if (endpoint->config.checksum) {
657 enum ipa_version version = ipa->version;
658
659 if (endpoint->toward_ipa) {
660 u32 off;
661
662 /* Checksum header offset is in 4-byte units */
663 off = sizeof(struct rmnet_map_header) / sizeof(u32);
664 val |= reg_encode(reg, field_id: CS_METADATA_HDR_OFFSET, val: off);
665
666 enabled = version < IPA_VERSION_4_5
667 ? IPA_CS_OFFLOAD_UL
668 : IPA_CS_OFFLOAD_INLINE;
669 } else {
670 enabled = version < IPA_VERSION_4_5
671 ? IPA_CS_OFFLOAD_DL
672 : IPA_CS_OFFLOAD_INLINE;
673 }
674 } else {
675 enabled = IPA_CS_OFFLOAD_NONE;
676 }
677 val |= reg_encode(reg, field_id: CS_OFFLOAD_EN, val: enabled);
678 /* CS_GEN_QMB_MASTER_SEL is 0 */
679
680 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id));
681}
682
683static void ipa_endpoint_init_nat(struct ipa_endpoint *endpoint)
684{
685 u32 endpoint_id = endpoint->endpoint_id;
686 struct ipa *ipa = endpoint->ipa;
687 const struct reg *reg;
688 u32 val;
689
690 if (!endpoint->toward_ipa)
691 return;
692
693 reg = ipa_reg(ipa, reg_id: ENDP_INIT_NAT);
694 val = reg_encode(reg, field_id: NAT_EN, val: IPA_NAT_TYPE_BYPASS);
695
696 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id));
697}
698
699static u32
700ipa_qmap_header_size(enum ipa_version version, struct ipa_endpoint *endpoint)
701{
702 u32 header_size = sizeof(struct rmnet_map_header);
703
704 /* Without checksum offload, we just have the MAP header */
705 if (!endpoint->config.checksum)
706 return header_size;
707
708 if (version < IPA_VERSION_4_5) {
709 /* Checksum header inserted for AP TX endpoints only */
710 if (endpoint->toward_ipa)
711 header_size += sizeof(struct rmnet_map_ul_csum_header);
712 } else {
713 /* Checksum header is used in both directions */
714 header_size += sizeof(struct rmnet_map_v5_csum_header);
715 }
716
717 return header_size;
718}
719
720/* Encoded value for ENDP_INIT_HDR register HDR_LEN* field(s) */
721static u32 ipa_header_size_encode(enum ipa_version version,
722 const struct reg *reg, u32 header_size)
723{
724 u32 field_max = reg_field_max(reg, field_id: HDR_LEN);
725 u32 val;
726
727 /* We know field_max can be used as a mask (2^n - 1) */
728 val = reg_encode(reg, field_id: HDR_LEN, val: header_size & field_max);
729 if (version < IPA_VERSION_4_5) {
730 WARN_ON(header_size > field_max);
731 return val;
732 }
733
734 /* IPA v4.5 adds a few more most-significant bits */
735 header_size >>= hweight32(field_max);
736 WARN_ON(header_size > reg_field_max(reg, HDR_LEN_MSB));
737 val |= reg_encode(reg, field_id: HDR_LEN_MSB, val: header_size);
738
739 return val;
740}
741
742/* Encoded value for ENDP_INIT_HDR register OFST_METADATA* field(s) */
743static u32 ipa_metadata_offset_encode(enum ipa_version version,
744 const struct reg *reg, u32 offset)
745{
746 u32 field_max = reg_field_max(reg, field_id: HDR_OFST_METADATA);
747 u32 val;
748
749 /* We know field_max can be used as a mask (2^n - 1) */
750 val = reg_encode(reg, field_id: HDR_OFST_METADATA, val: offset);
751 if (version < IPA_VERSION_4_5) {
752 WARN_ON(offset > field_max);
753 return val;
754 }
755
756 /* IPA v4.5 adds a few more most-significant bits */
757 offset >>= hweight32(field_max);
758 WARN_ON(offset > reg_field_max(reg, HDR_OFST_METADATA_MSB));
759 val |= reg_encode(reg, field_id: HDR_OFST_METADATA_MSB, val: offset);
760
761 return val;
762}
763
764/**
765 * ipa_endpoint_init_hdr() - Initialize HDR endpoint configuration register
766 * @endpoint: Endpoint pointer
767 *
768 * We program QMAP endpoints so each packet received is preceded by a QMAP
769 * header structure. The QMAP header contains a 1-byte mux_id and 2-byte
770 * packet size field, and we have the IPA hardware populate both for each
771 * received packet. The header is configured (in the HDR_EXT register)
772 * to use big endian format.
773 *
774 * The packet size is written into the QMAP header's pkt_len field. That
775 * location is defined here using the HDR_OFST_PKT_SIZE field.
776 *
777 * The mux_id comes from a 4-byte metadata value supplied with each packet
778 * by the modem. It is *not* a QMAP header, but it does contain the mux_id
779 * value that we want, in its low-order byte. A bitmask defined in the
780 * endpoint's METADATA_MASK register defines which byte within the modem
781 * metadata contains the mux_id. And the OFST_METADATA field programmed
782 * here indicates where the extracted byte should be placed within the QMAP
783 * header.
784 */
785static void ipa_endpoint_init_hdr(struct ipa_endpoint *endpoint)
786{
787 u32 endpoint_id = endpoint->endpoint_id;
788 struct ipa *ipa = endpoint->ipa;
789 const struct reg *reg;
790 u32 val = 0;
791
792 reg = ipa_reg(ipa, reg_id: ENDP_INIT_HDR);
793 if (endpoint->config.qmap) {
794 enum ipa_version version = ipa->version;
795 size_t header_size;
796
797 header_size = ipa_qmap_header_size(version, endpoint);
798 val = ipa_header_size_encode(version, reg, header_size);
799
800 /* Define how to fill fields in a received QMAP header */
801 if (!endpoint->toward_ipa) {
802 u32 off; /* Field offset within header */
803
804 /* Where IPA will write the metadata value */
805 off = offsetof(struct rmnet_map_header, mux_id);
806 val |= ipa_metadata_offset_encode(version, reg, offset: off);
807
808 /* Where IPA will write the length */
809 off = offsetof(struct rmnet_map_header, pkt_len);
810 /* Upper bits are stored in HDR_EXT with IPA v4.5 */
811 if (version >= IPA_VERSION_4_5)
812 off &= reg_field_max(reg, field_id: HDR_OFST_PKT_SIZE);
813
814 val |= reg_bit(reg, field_id: HDR_OFST_PKT_SIZE_VALID);
815 val |= reg_encode(reg, field_id: HDR_OFST_PKT_SIZE, val: off);
816 }
817 /* For QMAP TX, metadata offset is 0 (modem assumes this) */
818 val |= reg_bit(reg, field_id: HDR_OFST_METADATA_VALID);
819
820 /* HDR_ADDITIONAL_CONST_LEN is 0; (RX only) */
821 /* HDR_A5_MUX is 0 */
822 /* HDR_LEN_INC_DEAGG_HDR is 0 */
823 /* HDR_METADATA_REG_VALID is 0 (TX only, version < v4.5) */
824 }
825
826 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id));
827}
828
829static void ipa_endpoint_init_hdr_ext(struct ipa_endpoint *endpoint)
830{
831 u32 pad_align = endpoint->config.rx.pad_align;
832 u32 endpoint_id = endpoint->endpoint_id;
833 struct ipa *ipa = endpoint->ipa;
834 const struct reg *reg;
835 u32 val = 0;
836
837 reg = ipa_reg(ipa, reg_id: ENDP_INIT_HDR_EXT);
838 if (endpoint->config.qmap) {
839 /* We have a header, so we must specify its endianness */
840 val |= reg_bit(reg, field_id: HDR_ENDIANNESS); /* big endian */
841
842 /* A QMAP header contains a 6 bit pad field at offset 0.
843 * The RMNet driver assumes this field is meaningful in
844 * packets it receives, and assumes the header's payload
845 * length includes that padding. The RMNet driver does
846 * *not* pad packets it sends, however, so the pad field
847 * (although 0) should be ignored.
848 */
849 if (!endpoint->toward_ipa) {
850 val |= reg_bit(reg, field_id: HDR_TOTAL_LEN_OR_PAD_VALID);
851 /* HDR_TOTAL_LEN_OR_PAD is 0 (pad, not total_len) */
852 val |= reg_bit(reg, field_id: HDR_PAYLOAD_LEN_INC_PADDING);
853 /* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0 */
854 }
855 }
856
857 /* HDR_PAYLOAD_LEN_INC_PADDING is 0 */
858 if (!endpoint->toward_ipa)
859 val |= reg_encode(reg, field_id: HDR_PAD_TO_ALIGNMENT, val: pad_align);
860
861 /* IPA v4.5 adds some most-significant bits to a few fields,
862 * two of which are defined in the HDR (not HDR_EXT) register.
863 */
864 if (ipa->version >= IPA_VERSION_4_5) {
865 /* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0, so MSB is 0 */
866 if (endpoint->config.qmap && !endpoint->toward_ipa) {
867 u32 mask = reg_field_max(reg, field_id: HDR_OFST_PKT_SIZE);
868 u32 off; /* Field offset within header */
869
870 off = offsetof(struct rmnet_map_header, pkt_len);
871 /* Low bits are in the ENDP_INIT_HDR register */
872 off >>= hweight32(mask);
873 val |= reg_encode(reg, field_id: HDR_OFST_PKT_SIZE_MSB, val: off);
874 /* HDR_ADDITIONAL_CONST_LEN is 0 so MSB is 0 */
875 }
876 }
877
878 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id));
879}
880
881static void ipa_endpoint_init_hdr_metadata_mask(struct ipa_endpoint *endpoint)
882{
883 u32 endpoint_id = endpoint->endpoint_id;
884 struct ipa *ipa = endpoint->ipa;
885 const struct reg *reg;
886 u32 val = 0;
887 u32 offset;
888
889 if (endpoint->toward_ipa)
890 return; /* Register not valid for TX endpoints */
891
892 reg = ipa_reg(ipa, reg_id: ENDP_INIT_HDR_METADATA_MASK);
893 offset = reg_n_offset(reg, n: endpoint_id);
894
895 /* Note that HDR_ENDIANNESS indicates big endian header fields */
896 if (endpoint->config.qmap)
897 val = (__force u32)cpu_to_be32(IPA_ENDPOINT_QMAP_METADATA_MASK);
898
899 iowrite32(val, ipa->reg_virt + offset);
900}
901
902static void ipa_endpoint_init_mode(struct ipa_endpoint *endpoint)
903{
904 struct ipa *ipa = endpoint->ipa;
905 const struct reg *reg;
906 u32 offset;
907 u32 val;
908
909 if (!endpoint->toward_ipa)
910 return; /* Register not valid for RX endpoints */
911
912 reg = ipa_reg(ipa, reg_id: ENDP_INIT_MODE);
913 if (endpoint->config.dma_mode) {
914 enum ipa_endpoint_name name = endpoint->config.dma_endpoint;
915 u32 dma_endpoint_id = ipa->name_map[name]->endpoint_id;
916
917 val = reg_encode(reg, field_id: ENDP_MODE, val: IPA_DMA);
918 val |= reg_encode(reg, field_id: DEST_PIPE_INDEX, val: dma_endpoint_id);
919 } else {
920 val = reg_encode(reg, field_id: ENDP_MODE, val: IPA_BASIC);
921 }
922 /* All other bits unspecified (and 0) */
923
924 offset = reg_n_offset(reg, n: endpoint->endpoint_id);
925 iowrite32(val, ipa->reg_virt + offset);
926}
927
928/* For IPA v4.5+, times are expressed using Qtime. A time is represented
929 * at one of several available granularities, which are configured in
930 * ipa_qtime_config(). Three (or, starting with IPA v5.0, four) pulse
931 * generators are set up with different "tick" periods. A Qtime value
932 * encodes a tick count along with an indication of a pulse generator
933 * (which has a fixed tick period). Two pulse generators are always
934 * available to the AP; a third is available starting with IPA v5.0.
935 * This function determines which pulse generator most accurately
936 * represents the time period provided, and returns the tick count to
937 * use to represent that time.
938 */
939static u32
940ipa_qtime_val(struct ipa *ipa, u32 microseconds, u32 max, u32 *select)
941{
942 u32 which = 0;
943 u32 ticks;
944
945 /* Pulse generator 0 has 100 microsecond granularity */
946 ticks = DIV_ROUND_CLOSEST(microseconds, 100);
947 if (ticks <= max)
948 goto out;
949
950 /* Pulse generator 1 has millisecond granularity */
951 which = 1;
952 ticks = DIV_ROUND_CLOSEST(microseconds, 1000);
953 if (ticks <= max)
954 goto out;
955
956 if (ipa->version >= IPA_VERSION_5_0) {
957 /* Pulse generator 2 has 10 millisecond granularity */
958 which = 2;
959 ticks = DIV_ROUND_CLOSEST(microseconds, 100);
960 }
961 WARN_ON(ticks > max);
962out:
963 *select = which;
964
965 return ticks;
966}
967
968/* Encode the aggregation timer limit (microseconds) based on IPA version */
969static u32 aggr_time_limit_encode(struct ipa *ipa, const struct reg *reg,
970 u32 microseconds)
971{
972 u32 ticks;
973 u32 max;
974
975 if (!microseconds)
976 return 0; /* Nothing to compute if time limit is 0 */
977
978 max = reg_field_max(reg, field_id: TIME_LIMIT);
979 if (ipa->version >= IPA_VERSION_4_5) {
980 u32 select;
981
982 ticks = ipa_qtime_val(ipa, microseconds, max, select: &select);
983
984 return reg_encode(reg, field_id: AGGR_GRAN_SEL, val: select) |
985 reg_encode(reg, field_id: TIME_LIMIT, val: ticks);
986 }
987
988 /* We program aggregation granularity in ipa_hardware_config() */
989 ticks = DIV_ROUND_CLOSEST(microseconds, IPA_AGGR_GRANULARITY);
990 WARN(ticks > max, "aggr_time_limit too large (%u > %u usec)\n",
991 microseconds, max * IPA_AGGR_GRANULARITY);
992
993 return reg_encode(reg, field_id: TIME_LIMIT, val: ticks);
994}
995
996static void ipa_endpoint_init_aggr(struct ipa_endpoint *endpoint)
997{
998 u32 endpoint_id = endpoint->endpoint_id;
999 struct ipa *ipa = endpoint->ipa;
1000 const struct reg *reg;
1001 u32 val = 0;
1002
1003 reg = ipa_reg(ipa, reg_id: ENDP_INIT_AGGR);
1004 if (endpoint->config.aggregation) {
1005 if (!endpoint->toward_ipa) {
1006 const struct ipa_endpoint_rx *rx_config;
1007 u32 buffer_size;
1008 u32 limit;
1009
1010 rx_config = &endpoint->config.rx;
1011 val |= reg_encode(reg, field_id: AGGR_EN, val: IPA_ENABLE_AGGR);
1012 val |= reg_encode(reg, field_id: AGGR_TYPE, val: IPA_GENERIC);
1013
1014 buffer_size = rx_config->buffer_size;
1015 limit = ipa_aggr_size_kb(rx_buffer_size: buffer_size - NET_SKB_PAD,
1016 aggr_hard_limit: rx_config->aggr_hard_limit);
1017 val |= reg_encode(reg, field_id: BYTE_LIMIT, val: limit);
1018
1019 limit = rx_config->aggr_time_limit;
1020 val |= aggr_time_limit_encode(ipa, reg, microseconds: limit);
1021
1022 /* AGGR_PKT_LIMIT is 0 (unlimited) */
1023
1024 if (rx_config->aggr_close_eof)
1025 val |= reg_bit(reg, field_id: SW_EOF_ACTIVE);
1026 } else {
1027 val |= reg_encode(reg, field_id: AGGR_EN, val: IPA_ENABLE_DEAGGR);
1028 val |= reg_encode(reg, field_id: AGGR_TYPE, val: IPA_QCMAP);
1029 /* other fields ignored */
1030 }
1031 /* AGGR_FORCE_CLOSE is 0 */
1032 /* AGGR_GRAN_SEL is 0 for IPA v4.5 */
1033 } else {
1034 val |= reg_encode(reg, field_id: AGGR_EN, val: IPA_BYPASS_AGGR);
1035 /* other fields ignored */
1036 }
1037
1038 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id));
1039}
1040
1041/* The head-of-line blocking timer is defined as a tick count. For
1042 * IPA version 4.5 the tick count is based on the Qtimer, which is
1043 * derived from the 19.2 MHz SoC XO clock. For older IPA versions
1044 * each tick represents 128 cycles of the IPA core clock.
1045 *
1046 * Return the encoded value representing the timeout period provided
1047 * that should be written to the ENDP_INIT_HOL_BLOCK_TIMER register.
1048 */
1049static u32 hol_block_timer_encode(struct ipa *ipa, const struct reg *reg,
1050 u32 microseconds)
1051{
1052 u32 width;
1053 u32 scale;
1054 u64 ticks;
1055 u64 rate;
1056 u32 high;
1057 u32 val;
1058
1059 if (!microseconds)
1060 return 0; /* Nothing to compute if timer period is 0 */
1061
1062 if (ipa->version >= IPA_VERSION_4_5) {
1063 u32 max = reg_field_max(reg, field_id: TIMER_LIMIT);
1064 u32 select;
1065 u32 ticks;
1066
1067 ticks = ipa_qtime_val(ipa, microseconds, max, select: &select);
1068
1069 return reg_encode(reg, field_id: TIMER_GRAN_SEL, val: 1) |
1070 reg_encode(reg, field_id: TIMER_LIMIT, val: ticks);
1071 }
1072
1073 /* Use 64 bit arithmetic to avoid overflow */
1074 rate = ipa_core_clock_rate(ipa);
1075 ticks = DIV_ROUND_CLOSEST(microseconds * rate, 128 * USEC_PER_SEC);
1076
1077 /* We still need the result to fit into the field */
1078 WARN_ON(ticks > reg_field_max(reg, TIMER_BASE_VALUE));
1079
1080 /* IPA v3.5.1 through v4.1 just record the tick count */
1081 if (ipa->version < IPA_VERSION_4_2)
1082 return reg_encode(reg, field_id: TIMER_BASE_VALUE, val: (u32)ticks);
1083
1084 /* For IPA v4.2, the tick count is represented by base and
1085 * scale fields within the 32-bit timer register, where:
1086 * ticks = base << scale;
1087 * The best precision is achieved when the base value is as
1088 * large as possible. Find the highest set bit in the tick
1089 * count, and extract the number of bits in the base field
1090 * such that high bit is included.
1091 */
1092 high = fls(x: ticks); /* 1..32 (or warning above) */
1093 width = hweight32(reg_fmask(reg, TIMER_BASE_VALUE));
1094 scale = high > width ? high - width : 0;
1095 if (scale) {
1096 /* If we're scaling, round up to get a closer result */
1097 ticks += 1 << (scale - 1);
1098 /* High bit was set, so rounding might have affected it */
1099 if (fls(x: ticks) != high)
1100 scale++;
1101 }
1102
1103 val = reg_encode(reg, field_id: TIMER_SCALE, val: scale);
1104 val |= reg_encode(reg, field_id: TIMER_BASE_VALUE, val: (u32)ticks >> scale);
1105
1106 return val;
1107}
1108
1109/* If microseconds is 0, timeout is immediate */
1110static void ipa_endpoint_init_hol_block_timer(struct ipa_endpoint *endpoint,
1111 u32 microseconds)
1112{
1113 u32 endpoint_id = endpoint->endpoint_id;
1114 struct ipa *ipa = endpoint->ipa;
1115 const struct reg *reg;
1116 u32 val;
1117
1118 /* This should only be changed when HOL_BLOCK_EN is disabled */
1119 reg = ipa_reg(ipa, reg_id: ENDP_INIT_HOL_BLOCK_TIMER);
1120 val = hol_block_timer_encode(ipa, reg, microseconds);
1121
1122 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id));
1123}
1124
1125static void
1126ipa_endpoint_init_hol_block_en(struct ipa_endpoint *endpoint, bool enable)
1127{
1128 u32 endpoint_id = endpoint->endpoint_id;
1129 struct ipa *ipa = endpoint->ipa;
1130 const struct reg *reg;
1131 u32 offset;
1132 u32 val;
1133
1134 reg = ipa_reg(ipa, reg_id: ENDP_INIT_HOL_BLOCK_EN);
1135 offset = reg_n_offset(reg, n: endpoint_id);
1136 val = enable ? reg_bit(reg, field_id: HOL_BLOCK_EN) : 0;
1137
1138 iowrite32(val, ipa->reg_virt + offset);
1139
1140 /* When enabling, the register must be written twice for IPA v4.5+ */
1141 if (enable && ipa->version >= IPA_VERSION_4_5)
1142 iowrite32(val, ipa->reg_virt + offset);
1143}
1144
1145/* Assumes HOL_BLOCK is in disabled state */
1146static void ipa_endpoint_init_hol_block_enable(struct ipa_endpoint *endpoint,
1147 u32 microseconds)
1148{
1149 ipa_endpoint_init_hol_block_timer(endpoint, microseconds);
1150 ipa_endpoint_init_hol_block_en(endpoint, enable: true);
1151}
1152
1153static void ipa_endpoint_init_hol_block_disable(struct ipa_endpoint *endpoint)
1154{
1155 ipa_endpoint_init_hol_block_en(endpoint, enable: false);
1156}
1157
1158void ipa_endpoint_modem_hol_block_clear_all(struct ipa *ipa)
1159{
1160 u32 endpoint_id = 0;
1161
1162 while (endpoint_id < ipa->endpoint_count) {
1163 struct ipa_endpoint *endpoint = &ipa->endpoint[endpoint_id++];
1164
1165 if (endpoint->toward_ipa || endpoint->ee_id != GSI_EE_MODEM)
1166 continue;
1167
1168 ipa_endpoint_init_hol_block_disable(endpoint);
1169 ipa_endpoint_init_hol_block_enable(endpoint, microseconds: 0);
1170 }
1171}
1172
1173static void ipa_endpoint_init_deaggr(struct ipa_endpoint *endpoint)
1174{
1175 u32 endpoint_id = endpoint->endpoint_id;
1176 struct ipa *ipa = endpoint->ipa;
1177 const struct reg *reg;
1178 u32 val = 0;
1179
1180 if (!endpoint->toward_ipa)
1181 return; /* Register not valid for RX endpoints */
1182
1183 reg = ipa_reg(ipa, reg_id: ENDP_INIT_DEAGGR);
1184 /* DEAGGR_HDR_LEN is 0 */
1185 /* PACKET_OFFSET_VALID is 0 */
1186 /* PACKET_OFFSET_LOCATION is ignored (not valid) */
1187 /* MAX_PACKET_LEN is 0 (not enforced) */
1188
1189 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id));
1190}
1191
1192static void ipa_endpoint_init_rsrc_grp(struct ipa_endpoint *endpoint)
1193{
1194 u32 resource_group = endpoint->config.resource_group;
1195 u32 endpoint_id = endpoint->endpoint_id;
1196 struct ipa *ipa = endpoint->ipa;
1197 const struct reg *reg;
1198 u32 val;
1199
1200 reg = ipa_reg(ipa, reg_id: ENDP_INIT_RSRC_GRP);
1201 val = reg_encode(reg, field_id: ENDP_RSRC_GRP, val: resource_group);
1202
1203 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id));
1204}
1205
1206static void ipa_endpoint_init_seq(struct ipa_endpoint *endpoint)
1207{
1208 u32 endpoint_id = endpoint->endpoint_id;
1209 struct ipa *ipa = endpoint->ipa;
1210 const struct reg *reg;
1211 u32 val;
1212
1213 if (!endpoint->toward_ipa)
1214 return; /* Register not valid for RX endpoints */
1215
1216 reg = ipa_reg(ipa, reg_id: ENDP_INIT_SEQ);
1217
1218 /* Low-order byte configures primary packet processing */
1219 val = reg_encode(reg, field_id: SEQ_TYPE, val: endpoint->config.tx.seq_type);
1220
1221 /* Second byte (if supported) configures replicated packet processing */
1222 if (ipa->version < IPA_VERSION_4_5)
1223 val |= reg_encode(reg, field_id: SEQ_REP_TYPE,
1224 val: endpoint->config.tx.seq_rep_type);
1225
1226 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id));
1227}
1228
1229/**
1230 * ipa_endpoint_skb_tx() - Transmit a socket buffer
1231 * @endpoint: Endpoint pointer
1232 * @skb: Socket buffer to send
1233 *
1234 * Returns: 0 if successful, or a negative error code
1235 */
1236int ipa_endpoint_skb_tx(struct ipa_endpoint *endpoint, struct sk_buff *skb)
1237{
1238 struct gsi_trans *trans;
1239 u32 nr_frags;
1240 int ret;
1241
1242 /* Make sure source endpoint's TLV FIFO has enough entries to
1243 * hold the linear portion of the skb and all its fragments.
1244 * If not, see if we can linearize it before giving up.
1245 */
1246 nr_frags = skb_shinfo(skb)->nr_frags;
1247 if (nr_frags > endpoint->skb_frag_max) {
1248 if (skb_linearize(skb))
1249 return -E2BIG;
1250 nr_frags = 0;
1251 }
1252
1253 trans = ipa_endpoint_trans_alloc(endpoint, tre_count: 1 + nr_frags);
1254 if (!trans)
1255 return -EBUSY;
1256
1257 ret = gsi_trans_skb_add(trans, skb);
1258 if (ret)
1259 goto err_trans_free;
1260 trans->data = skb; /* transaction owns skb now */
1261
1262 gsi_trans_commit(trans, ring_db: !netdev_xmit_more());
1263
1264 return 0;
1265
1266err_trans_free:
1267 gsi_trans_free(trans);
1268
1269 return -ENOMEM;
1270}
1271
1272static void ipa_endpoint_status(struct ipa_endpoint *endpoint)
1273{
1274 u32 endpoint_id = endpoint->endpoint_id;
1275 struct ipa *ipa = endpoint->ipa;
1276 const struct reg *reg;
1277 u32 val = 0;
1278
1279 reg = ipa_reg(ipa, reg_id: ENDP_STATUS);
1280 if (endpoint->config.status_enable) {
1281 val |= reg_bit(reg, field_id: STATUS_EN);
1282 if (endpoint->toward_ipa) {
1283 enum ipa_endpoint_name name;
1284 u32 status_endpoint_id;
1285
1286 name = endpoint->config.tx.status_endpoint;
1287 status_endpoint_id = ipa->name_map[name]->endpoint_id;
1288
1289 val |= reg_encode(reg, field_id: STATUS_ENDP, val: status_endpoint_id);
1290 }
1291 /* STATUS_LOCATION is 0, meaning IPA packet status
1292 * precedes the packet (not present for IPA v4.5+)
1293 */
1294 /* STATUS_PKT_SUPPRESS_FMASK is 0 (not present for v4.0+) */
1295 }
1296
1297 iowrite32(val, ipa->reg_virt + reg_n_offset(reg, n: endpoint_id));
1298}
1299
1300static int ipa_endpoint_replenish_one(struct ipa_endpoint *endpoint,
1301 struct gsi_trans *trans)
1302{
1303 struct page *page;
1304 u32 buffer_size;
1305 u32 offset;
1306 u32 len;
1307 int ret;
1308
1309 buffer_size = endpoint->config.rx.buffer_size;
1310 page = dev_alloc_pages(get_order(buffer_size));
1311 if (!page)
1312 return -ENOMEM;
1313
1314 /* Offset the buffer to make space for skb headroom */
1315 offset = NET_SKB_PAD;
1316 len = buffer_size - offset;
1317
1318 ret = gsi_trans_page_add(trans, page, size: len, offset);
1319 if (ret)
1320 put_page(page);
1321 else
1322 trans->data = page; /* transaction owns page now */
1323
1324 return ret;
1325}
1326
1327/**
1328 * ipa_endpoint_replenish() - Replenish endpoint receive buffers
1329 * @endpoint: Endpoint to be replenished
1330 *
1331 * The IPA hardware can hold a fixed number of receive buffers for an RX
1332 * endpoint, based on the number of entries in the underlying channel ring
1333 * buffer. If an endpoint's "backlog" is non-zero, it indicates how many
1334 * more receive buffers can be supplied to the hardware. Replenishing for
1335 * an endpoint can be disabled, in which case buffers are not queued to
1336 * the hardware.
1337 */
1338static void ipa_endpoint_replenish(struct ipa_endpoint *endpoint)
1339{
1340 struct gsi_trans *trans;
1341
1342 if (!test_bit(IPA_REPLENISH_ENABLED, endpoint->replenish_flags))
1343 return;
1344
1345 /* Skip it if it's already active */
1346 if (test_and_set_bit(nr: IPA_REPLENISH_ACTIVE, addr: endpoint->replenish_flags))
1347 return;
1348
1349 while ((trans = ipa_endpoint_trans_alloc(endpoint, tre_count: 1))) {
1350 bool doorbell;
1351
1352 if (ipa_endpoint_replenish_one(endpoint, trans))
1353 goto try_again_later;
1354
1355
1356 /* Ring the doorbell if we've got a full batch */
1357 doorbell = !(++endpoint->replenish_count % IPA_REPLENISH_BATCH);
1358 gsi_trans_commit(trans, ring_db: doorbell);
1359 }
1360
1361 clear_bit(nr: IPA_REPLENISH_ACTIVE, addr: endpoint->replenish_flags);
1362
1363 return;
1364
1365try_again_later:
1366 gsi_trans_free(trans);
1367 clear_bit(nr: IPA_REPLENISH_ACTIVE, addr: endpoint->replenish_flags);
1368
1369 /* Whenever a receive buffer transaction completes we'll try to
1370 * replenish again. It's unlikely, but if we fail to supply even
1371 * one buffer, nothing will trigger another replenish attempt.
1372 * If the hardware has no receive buffers queued, schedule work to
1373 * try replenishing again.
1374 */
1375 if (gsi_channel_trans_idle(gsi: &endpoint->ipa->gsi, channel_id: endpoint->channel_id))
1376 schedule_delayed_work(dwork: &endpoint->replenish_work,
1377 delay: msecs_to_jiffies(m: 1));
1378}
1379
1380static void ipa_endpoint_replenish_enable(struct ipa_endpoint *endpoint)
1381{
1382 set_bit(nr: IPA_REPLENISH_ENABLED, addr: endpoint->replenish_flags);
1383
1384 /* Start replenishing if hardware currently has no buffers */
1385 if (gsi_channel_trans_idle(gsi: &endpoint->ipa->gsi, channel_id: endpoint->channel_id))
1386 ipa_endpoint_replenish(endpoint);
1387}
1388
1389static void ipa_endpoint_replenish_disable(struct ipa_endpoint *endpoint)
1390{
1391 clear_bit(nr: IPA_REPLENISH_ENABLED, addr: endpoint->replenish_flags);
1392}
1393
1394static void ipa_endpoint_replenish_work(struct work_struct *work)
1395{
1396 struct delayed_work *dwork = to_delayed_work(work);
1397 struct ipa_endpoint *endpoint;
1398
1399 endpoint = container_of(dwork, struct ipa_endpoint, replenish_work);
1400
1401 ipa_endpoint_replenish(endpoint);
1402}
1403
1404static void ipa_endpoint_skb_copy(struct ipa_endpoint *endpoint,
1405 void *data, u32 len, u32 extra)
1406{
1407 struct sk_buff *skb;
1408
1409 if (!endpoint->netdev)
1410 return;
1411
1412 skb = __dev_alloc_skb(length: len, GFP_ATOMIC);
1413 if (skb) {
1414 /* Copy the data into the socket buffer and receive it */
1415 skb_put(skb, len);
1416 memcpy(skb->data, data, len);
1417 skb->truesize += extra;
1418 }
1419
1420 ipa_modem_skb_rx(netdev: endpoint->netdev, skb);
1421}
1422
1423static bool ipa_endpoint_skb_build(struct ipa_endpoint *endpoint,
1424 struct page *page, u32 len)
1425{
1426 u32 buffer_size = endpoint->config.rx.buffer_size;
1427 struct sk_buff *skb;
1428
1429 /* Nothing to do if there's no netdev */
1430 if (!endpoint->netdev)
1431 return false;
1432
1433 WARN_ON(len > SKB_WITH_OVERHEAD(buffer_size - NET_SKB_PAD));
1434
1435 skb = build_skb(page_address(page), frag_size: buffer_size);
1436 if (skb) {
1437 /* Reserve the headroom and account for the data */
1438 skb_reserve(skb, NET_SKB_PAD);
1439 skb_put(skb, len);
1440 }
1441
1442 /* Receive the buffer (or record drop if unable to build it) */
1443 ipa_modem_skb_rx(netdev: endpoint->netdev, skb);
1444
1445 return skb != NULL;
1446}
1447
1448 /* The format of an IPA packet status structure is the same for several
1449 * status types (opcodes). Other types aren't currently supported.
1450 */
1451static bool ipa_status_format_packet(enum ipa_status_opcode opcode)
1452{
1453 switch (opcode) {
1454 case IPA_STATUS_OPCODE_PACKET:
1455 case IPA_STATUS_OPCODE_DROPPED_PACKET:
1456 case IPA_STATUS_OPCODE_SUSPENDED_PACKET:
1457 case IPA_STATUS_OPCODE_PACKET_2ND_PASS:
1458 return true;
1459 default:
1460 return false;
1461 }
1462}
1463
1464static bool
1465ipa_endpoint_status_skip(struct ipa_endpoint *endpoint, const void *data)
1466{
1467 struct ipa *ipa = endpoint->ipa;
1468 enum ipa_status_opcode opcode;
1469 u32 endpoint_id;
1470
1471 opcode = ipa_status_extract(ipa, data, field: STATUS_OPCODE);
1472 if (!ipa_status_format_packet(opcode))
1473 return true;
1474
1475 endpoint_id = ipa_status_extract(ipa, data, field: STATUS_DST_ENDPOINT);
1476 if (endpoint_id != endpoint->endpoint_id)
1477 return true;
1478
1479 return false; /* Don't skip this packet, process it */
1480}
1481
1482static bool
1483ipa_endpoint_status_tag_valid(struct ipa_endpoint *endpoint, const void *data)
1484{
1485 struct ipa_endpoint *command_endpoint;
1486 enum ipa_status_mask status_mask;
1487 struct ipa *ipa = endpoint->ipa;
1488 u32 endpoint_id;
1489
1490 status_mask = ipa_status_extract(ipa, data, field: STATUS_MASK);
1491 if (!status_mask)
1492 return false; /* No valid tag */
1493
1494 /* The status contains a valid tag. We know the packet was sent to
1495 * this endpoint (already verified by ipa_endpoint_status_skip()).
1496 * If the packet came from the AP->command TX endpoint we know
1497 * this packet was sent as part of the pipeline clear process.
1498 */
1499 endpoint_id = ipa_status_extract(ipa, data, field: STATUS_SRC_ENDPOINT);
1500 command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX];
1501 if (endpoint_id == command_endpoint->endpoint_id) {
1502 complete(&ipa->completion);
1503 } else {
1504 dev_err(ipa->dev, "unexpected tagged packet from endpoint %u\n",
1505 endpoint_id);
1506 }
1507
1508 return true;
1509}
1510
1511/* Return whether the status indicates the packet should be dropped */
1512static bool
1513ipa_endpoint_status_drop(struct ipa_endpoint *endpoint, const void *data)
1514{
1515 enum ipa_status_exception exception;
1516 struct ipa *ipa = endpoint->ipa;
1517 u32 rule;
1518
1519 /* If the status indicates a tagged transfer, we'll drop the packet */
1520 if (ipa_endpoint_status_tag_valid(endpoint, data))
1521 return true;
1522
1523 /* Deaggregation exceptions we drop; all other types we consume */
1524 exception = ipa_status_extract(ipa, data, field: STATUS_EXCEPTION);
1525 if (exception)
1526 return exception == IPA_STATUS_EXCEPTION_DEAGGR;
1527
1528 /* Drop the packet if it fails to match a routing rule; otherwise no */
1529 rule = ipa_status_extract(ipa, data, field: STATUS_ROUTER_RULE_INDEX);
1530
1531 return rule == IPA_STATUS_RULE_MISS;
1532}
1533
1534static void ipa_endpoint_status_parse(struct ipa_endpoint *endpoint,
1535 struct page *page, u32 total_len)
1536{
1537 u32 buffer_size = endpoint->config.rx.buffer_size;
1538 void *data = page_address(page) + NET_SKB_PAD;
1539 u32 unused = buffer_size - total_len;
1540 struct ipa *ipa = endpoint->ipa;
1541 struct device *dev = ipa->dev;
1542 u32 resid = total_len;
1543
1544 while (resid) {
1545 u32 length;
1546 u32 align;
1547 u32 len;
1548
1549 if (resid < IPA_STATUS_SIZE) {
1550 dev_err(dev,
1551 "short message (%u bytes < %zu byte status)\n",
1552 resid, IPA_STATUS_SIZE);
1553 break;
1554 }
1555
1556 /* Skip over status packets that lack packet data */
1557 length = ipa_status_extract(ipa, data, field: STATUS_LENGTH);
1558 if (!length || ipa_endpoint_status_skip(endpoint, data)) {
1559 data += IPA_STATUS_SIZE;
1560 resid -= IPA_STATUS_SIZE;
1561 continue;
1562 }
1563
1564 /* Compute the amount of buffer space consumed by the packet,
1565 * including the status. If the hardware is configured to
1566 * pad packet data to an aligned boundary, account for that.
1567 * And if checksum offload is enabled a trailer containing
1568 * computed checksum information will be appended.
1569 */
1570 align = endpoint->config.rx.pad_align ? : 1;
1571 len = IPA_STATUS_SIZE + ALIGN(length, align);
1572 if (endpoint->config.checksum)
1573 len += sizeof(struct rmnet_map_dl_csum_trailer);
1574
1575 if (!ipa_endpoint_status_drop(endpoint, data)) {
1576 void *data2;
1577 u32 extra;
1578
1579 /* Client receives only packet data (no status) */
1580 data2 = data + IPA_STATUS_SIZE;
1581
1582 /* Have the true size reflect the extra unused space in
1583 * the original receive buffer. Distribute the "cost"
1584 * proportionately across all aggregated packets in the
1585 * buffer.
1586 */
1587 extra = DIV_ROUND_CLOSEST(unused * len, total_len);
1588 ipa_endpoint_skb_copy(endpoint, data: data2, len: length, extra);
1589 }
1590
1591 /* Consume status and the full packet it describes */
1592 data += len;
1593 resid -= len;
1594 }
1595}
1596
1597void ipa_endpoint_trans_complete(struct ipa_endpoint *endpoint,
1598 struct gsi_trans *trans)
1599{
1600 struct page *page;
1601
1602 if (endpoint->toward_ipa)
1603 return;
1604
1605 if (trans->cancelled)
1606 goto done;
1607
1608 /* Parse or build a socket buffer using the actual received length */
1609 page = trans->data;
1610 if (endpoint->config.status_enable)
1611 ipa_endpoint_status_parse(endpoint, page, total_len: trans->len);
1612 else if (ipa_endpoint_skb_build(endpoint, page, len: trans->len))
1613 trans->data = NULL; /* Pages have been consumed */
1614done:
1615 ipa_endpoint_replenish(endpoint);
1616}
1617
1618void ipa_endpoint_trans_release(struct ipa_endpoint *endpoint,
1619 struct gsi_trans *trans)
1620{
1621 if (endpoint->toward_ipa) {
1622 struct ipa *ipa = endpoint->ipa;
1623
1624 /* Nothing to do for command transactions */
1625 if (endpoint != ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]) {
1626 struct sk_buff *skb = trans->data;
1627
1628 if (skb)
1629 dev_kfree_skb_any(skb);
1630 }
1631 } else {
1632 struct page *page = trans->data;
1633
1634 if (page)
1635 put_page(page);
1636 }
1637}
1638
1639void ipa_endpoint_default_route_set(struct ipa *ipa, u32 endpoint_id)
1640{
1641 const struct reg *reg;
1642 u32 val;
1643
1644 reg = ipa_reg(ipa, reg_id: ROUTE);
1645 /* ROUTE_DIS is 0 */
1646 val = reg_encode(reg, field_id: ROUTE_DEF_PIPE, val: endpoint_id);
1647 val |= reg_bit(reg, field_id: ROUTE_DEF_HDR_TABLE);
1648 /* ROUTE_DEF_HDR_OFST is 0 */
1649 val |= reg_encode(reg, field_id: ROUTE_FRAG_DEF_PIPE, val: endpoint_id);
1650 val |= reg_bit(reg, field_id: ROUTE_DEF_RETAIN_HDR);
1651
1652 iowrite32(val, ipa->reg_virt + reg_offset(reg));
1653}
1654
1655void ipa_endpoint_default_route_clear(struct ipa *ipa)
1656{
1657 ipa_endpoint_default_route_set(ipa, endpoint_id: 0);
1658}
1659
1660/**
1661 * ipa_endpoint_reset_rx_aggr() - Reset RX endpoint with aggregation active
1662 * @endpoint: Endpoint to be reset
1663 *
1664 * If aggregation is active on an RX endpoint when a reset is performed
1665 * on its underlying GSI channel, a special sequence of actions must be
1666 * taken to ensure the IPA pipeline is properly cleared.
1667 *
1668 * Return: 0 if successful, or a negative error code
1669 */
1670static int ipa_endpoint_reset_rx_aggr(struct ipa_endpoint *endpoint)
1671{
1672 struct ipa *ipa = endpoint->ipa;
1673 struct device *dev = ipa->dev;
1674 struct gsi *gsi = &ipa->gsi;
1675 bool suspended = false;
1676 dma_addr_t addr;
1677 u32 retries;
1678 u32 len = 1;
1679 void *virt;
1680 int ret;
1681
1682 virt = kzalloc(len, GFP_KERNEL);
1683 if (!virt)
1684 return -ENOMEM;
1685
1686 addr = dma_map_single(dev, virt, len, DMA_FROM_DEVICE);
1687 if (dma_mapping_error(dev, dma_addr: addr)) {
1688 ret = -ENOMEM;
1689 goto out_kfree;
1690 }
1691
1692 /* Force close aggregation before issuing the reset */
1693 ipa_endpoint_force_close(endpoint);
1694
1695 /* Reset and reconfigure the channel with the doorbell engine
1696 * disabled. Then poll until we know aggregation is no longer
1697 * active. We'll re-enable the doorbell (if appropriate) when
1698 * we reset again below.
1699 */
1700 gsi_channel_reset(gsi, channel_id: endpoint->channel_id, doorbell: false);
1701
1702 /* Make sure the channel isn't suspended */
1703 suspended = ipa_endpoint_program_suspend(endpoint, enable: false);
1704
1705 /* Start channel and do a 1 byte read */
1706 ret = gsi_channel_start(gsi, channel_id: endpoint->channel_id);
1707 if (ret)
1708 goto out_suspend_again;
1709
1710 ret = gsi_trans_read_byte(gsi, channel_id: endpoint->channel_id, addr);
1711 if (ret)
1712 goto err_endpoint_stop;
1713
1714 /* Wait for aggregation to be closed on the channel */
1715 retries = IPA_ENDPOINT_RESET_AGGR_RETRY_MAX;
1716 do {
1717 if (!ipa_endpoint_aggr_active(endpoint))
1718 break;
1719 usleep_range(USEC_PER_MSEC, max: 2 * USEC_PER_MSEC);
1720 } while (retries--);
1721
1722 /* Check one last time */
1723 if (ipa_endpoint_aggr_active(endpoint))
1724 dev_err(dev, "endpoint %u still active during reset\n",
1725 endpoint->endpoint_id);
1726
1727 gsi_trans_read_byte_done(gsi, channel_id: endpoint->channel_id);
1728
1729 ret = gsi_channel_stop(gsi, channel_id: endpoint->channel_id);
1730 if (ret)
1731 goto out_suspend_again;
1732
1733 /* Finally, reset and reconfigure the channel again (re-enabling
1734 * the doorbell engine if appropriate). Sleep for 1 millisecond to
1735 * complete the channel reset sequence. Finish by suspending the
1736 * channel again (if necessary).
1737 */
1738 gsi_channel_reset(gsi, channel_id: endpoint->channel_id, doorbell: true);
1739
1740 usleep_range(USEC_PER_MSEC, max: 2 * USEC_PER_MSEC);
1741
1742 goto out_suspend_again;
1743
1744err_endpoint_stop:
1745 (void)gsi_channel_stop(gsi, channel_id: endpoint->channel_id);
1746out_suspend_again:
1747 if (suspended)
1748 (void)ipa_endpoint_program_suspend(endpoint, enable: true);
1749 dma_unmap_single(dev, addr, len, DMA_FROM_DEVICE);
1750out_kfree:
1751 kfree(objp: virt);
1752
1753 return ret;
1754}
1755
1756static void ipa_endpoint_reset(struct ipa_endpoint *endpoint)
1757{
1758 u32 channel_id = endpoint->channel_id;
1759 struct ipa *ipa = endpoint->ipa;
1760 bool special;
1761 int ret = 0;
1762
1763 /* On IPA v3.5.1, if an RX endpoint is reset while aggregation
1764 * is active, we need to handle things specially to recover.
1765 * All other cases just need to reset the underlying GSI channel.
1766 */
1767 special = ipa->version < IPA_VERSION_4_0 && !endpoint->toward_ipa &&
1768 endpoint->config.aggregation;
1769 if (special && ipa_endpoint_aggr_active(endpoint))
1770 ret = ipa_endpoint_reset_rx_aggr(endpoint);
1771 else
1772 gsi_channel_reset(gsi: &ipa->gsi, channel_id, doorbell: true);
1773
1774 if (ret)
1775 dev_err(ipa->dev,
1776 "error %d resetting channel %u for endpoint %u\n",
1777 ret, endpoint->channel_id, endpoint->endpoint_id);
1778}
1779
1780static void ipa_endpoint_program(struct ipa_endpoint *endpoint)
1781{
1782 if (endpoint->toward_ipa) {
1783 /* Newer versions of IPA use GSI channel flow control
1784 * instead of endpoint DELAY mode to prevent sending data.
1785 * Flow control is disabled for newly-allocated channels,
1786 * and we can assume flow control is not (ever) enabled
1787 * for AP TX channels.
1788 */
1789 if (endpoint->ipa->version < IPA_VERSION_4_2)
1790 ipa_endpoint_program_delay(endpoint, enable: false);
1791 } else {
1792 /* Ensure suspend mode is off on all AP RX endpoints */
1793 (void)ipa_endpoint_program_suspend(endpoint, enable: false);
1794 }
1795 ipa_endpoint_init_cfg(endpoint);
1796 ipa_endpoint_init_nat(endpoint);
1797 ipa_endpoint_init_hdr(endpoint);
1798 ipa_endpoint_init_hdr_ext(endpoint);
1799 ipa_endpoint_init_hdr_metadata_mask(endpoint);
1800 ipa_endpoint_init_mode(endpoint);
1801 ipa_endpoint_init_aggr(endpoint);
1802 if (!endpoint->toward_ipa) {
1803 if (endpoint->config.rx.holb_drop)
1804 ipa_endpoint_init_hol_block_enable(endpoint, microseconds: 0);
1805 else
1806 ipa_endpoint_init_hol_block_disable(endpoint);
1807 }
1808 ipa_endpoint_init_deaggr(endpoint);
1809 ipa_endpoint_init_rsrc_grp(endpoint);
1810 ipa_endpoint_init_seq(endpoint);
1811 ipa_endpoint_status(endpoint);
1812}
1813
1814int ipa_endpoint_enable_one(struct ipa_endpoint *endpoint)
1815{
1816 u32 endpoint_id = endpoint->endpoint_id;
1817 struct ipa *ipa = endpoint->ipa;
1818 struct gsi *gsi = &ipa->gsi;
1819 int ret;
1820
1821 ret = gsi_channel_start(gsi, channel_id: endpoint->channel_id);
1822 if (ret) {
1823 dev_err(ipa->dev,
1824 "error %d starting %cX channel %u for endpoint %u\n",
1825 ret, endpoint->toward_ipa ? 'T' : 'R',
1826 endpoint->channel_id, endpoint_id);
1827 return ret;
1828 }
1829
1830 if (!endpoint->toward_ipa) {
1831 ipa_interrupt_suspend_enable(interrupt: ipa->interrupt, endpoint_id);
1832 ipa_endpoint_replenish_enable(endpoint);
1833 }
1834
1835 __set_bit(endpoint_id, ipa->enabled);
1836
1837 return 0;
1838}
1839
1840void ipa_endpoint_disable_one(struct ipa_endpoint *endpoint)
1841{
1842 u32 endpoint_id = endpoint->endpoint_id;
1843 struct ipa *ipa = endpoint->ipa;
1844 struct gsi *gsi = &ipa->gsi;
1845 int ret;
1846
1847 if (!test_bit(endpoint_id, ipa->enabled))
1848 return;
1849
1850 __clear_bit(endpoint_id, endpoint->ipa->enabled);
1851
1852 if (!endpoint->toward_ipa) {
1853 ipa_endpoint_replenish_disable(endpoint);
1854 ipa_interrupt_suspend_disable(interrupt: ipa->interrupt, endpoint_id);
1855 }
1856
1857 /* Note that if stop fails, the channel's state is not well-defined */
1858 ret = gsi_channel_stop(gsi, channel_id: endpoint->channel_id);
1859 if (ret)
1860 dev_err(ipa->dev, "error %d attempting to stop endpoint %u\n",
1861 ret, endpoint_id);
1862}
1863
1864void ipa_endpoint_suspend_one(struct ipa_endpoint *endpoint)
1865{
1866 struct device *dev = endpoint->ipa->dev;
1867 struct gsi *gsi = &endpoint->ipa->gsi;
1868 int ret;
1869
1870 if (!test_bit(endpoint->endpoint_id, endpoint->ipa->enabled))
1871 return;
1872
1873 if (!endpoint->toward_ipa) {
1874 ipa_endpoint_replenish_disable(endpoint);
1875 (void)ipa_endpoint_program_suspend(endpoint, enable: true);
1876 }
1877
1878 ret = gsi_channel_suspend(gsi, channel_id: endpoint->channel_id);
1879 if (ret)
1880 dev_err(dev, "error %d suspending channel %u\n", ret,
1881 endpoint->channel_id);
1882}
1883
1884void ipa_endpoint_resume_one(struct ipa_endpoint *endpoint)
1885{
1886 struct device *dev = endpoint->ipa->dev;
1887 struct gsi *gsi = &endpoint->ipa->gsi;
1888 int ret;
1889
1890 if (!test_bit(endpoint->endpoint_id, endpoint->ipa->enabled))
1891 return;
1892
1893 if (!endpoint->toward_ipa)
1894 (void)ipa_endpoint_program_suspend(endpoint, enable: false);
1895
1896 ret = gsi_channel_resume(gsi, channel_id: endpoint->channel_id);
1897 if (ret)
1898 dev_err(dev, "error %d resuming channel %u\n", ret,
1899 endpoint->channel_id);
1900 else if (!endpoint->toward_ipa)
1901 ipa_endpoint_replenish_enable(endpoint);
1902}
1903
1904void ipa_endpoint_suspend(struct ipa *ipa)
1905{
1906 if (!ipa->setup_complete)
1907 return;
1908
1909 if (ipa->modem_netdev)
1910 ipa_modem_suspend(netdev: ipa->modem_netdev);
1911
1912 ipa_endpoint_suspend_one(endpoint: ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]);
1913 ipa_endpoint_suspend_one(endpoint: ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]);
1914}
1915
1916void ipa_endpoint_resume(struct ipa *ipa)
1917{
1918 if (!ipa->setup_complete)
1919 return;
1920
1921 ipa_endpoint_resume_one(endpoint: ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]);
1922 ipa_endpoint_resume_one(endpoint: ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]);
1923
1924 if (ipa->modem_netdev)
1925 ipa_modem_resume(netdev: ipa->modem_netdev);
1926}
1927
1928static void ipa_endpoint_setup_one(struct ipa_endpoint *endpoint)
1929{
1930 struct gsi *gsi = &endpoint->ipa->gsi;
1931 u32 channel_id = endpoint->channel_id;
1932
1933 /* Only AP endpoints get set up */
1934 if (endpoint->ee_id != GSI_EE_AP)
1935 return;
1936
1937 endpoint->skb_frag_max = gsi->channel[channel_id].trans_tre_max - 1;
1938 if (!endpoint->toward_ipa) {
1939 /* RX transactions require a single TRE, so the maximum
1940 * backlog is the same as the maximum outstanding TREs.
1941 */
1942 clear_bit(nr: IPA_REPLENISH_ENABLED, addr: endpoint->replenish_flags);
1943 clear_bit(nr: IPA_REPLENISH_ACTIVE, addr: endpoint->replenish_flags);
1944 INIT_DELAYED_WORK(&endpoint->replenish_work,
1945 ipa_endpoint_replenish_work);
1946 }
1947
1948 ipa_endpoint_program(endpoint);
1949
1950 __set_bit(endpoint->endpoint_id, endpoint->ipa->set_up);
1951}
1952
1953static void ipa_endpoint_teardown_one(struct ipa_endpoint *endpoint)
1954{
1955 __clear_bit(endpoint->endpoint_id, endpoint->ipa->set_up);
1956
1957 if (!endpoint->toward_ipa)
1958 cancel_delayed_work_sync(dwork: &endpoint->replenish_work);
1959
1960 ipa_endpoint_reset(endpoint);
1961}
1962
1963void ipa_endpoint_setup(struct ipa *ipa)
1964{
1965 u32 endpoint_id;
1966
1967 for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count)
1968 ipa_endpoint_setup_one(endpoint: &ipa->endpoint[endpoint_id]);
1969}
1970
1971void ipa_endpoint_teardown(struct ipa *ipa)
1972{
1973 u32 endpoint_id;
1974
1975 for_each_set_bit(endpoint_id, ipa->set_up, ipa->endpoint_count)
1976 ipa_endpoint_teardown_one(endpoint: &ipa->endpoint[endpoint_id]);
1977}
1978
1979void ipa_endpoint_deconfig(struct ipa *ipa)
1980{
1981 ipa->available_count = 0;
1982 bitmap_free(bitmap: ipa->available);
1983 ipa->available = NULL;
1984}
1985
1986int ipa_endpoint_config(struct ipa *ipa)
1987{
1988 struct device *dev = ipa->dev;
1989 const struct reg *reg;
1990 u32 endpoint_id;
1991 u32 hw_limit;
1992 u32 tx_count;
1993 u32 rx_count;
1994 u32 rx_base;
1995 u32 limit;
1996 u32 val;
1997
1998 /* Prior to IPA v3.5, the FLAVOR_0 register was not supported.
1999 * Furthermore, the endpoints were not grouped such that TX
2000 * endpoint numbers started with 0 and RX endpoints had numbers
2001 * higher than all TX endpoints, so we can't do the simple
2002 * direction check used for newer hardware below.
2003 *
2004 * For hardware that doesn't support the FLAVOR_0 register,
2005 * just set the available mask to support any endpoint, and
2006 * assume the configuration is valid.
2007 */
2008 if (ipa->version < IPA_VERSION_3_5) {
2009 ipa->available = bitmap_zalloc(IPA_ENDPOINT_MAX, GFP_KERNEL);
2010 if (!ipa->available)
2011 return -ENOMEM;
2012 ipa->available_count = IPA_ENDPOINT_MAX;
2013
2014 bitmap_set(map: ipa->available, start: 0, IPA_ENDPOINT_MAX);
2015
2016 return 0;
2017 }
2018
2019 /* Find out about the endpoints supplied by the hardware, and ensure
2020 * the highest one doesn't exceed the number supported by software.
2021 */
2022 reg = ipa_reg(ipa, reg_id: FLAVOR_0);
2023 val = ioread32(ipa->reg_virt + reg_offset(reg));
2024
2025 /* Our RX is an IPA producer; our TX is an IPA consumer. */
2026 tx_count = reg_decode(reg, field_id: MAX_CONS_PIPES, val);
2027 rx_count = reg_decode(reg, field_id: MAX_PROD_PIPES, val);
2028 rx_base = reg_decode(reg, field_id: PROD_LOWEST, val);
2029
2030 limit = rx_base + rx_count;
2031 if (limit > IPA_ENDPOINT_MAX) {
2032 dev_err(dev, "too many endpoints, %u > %u\n",
2033 limit, IPA_ENDPOINT_MAX);
2034 return -EINVAL;
2035 }
2036
2037 /* Until IPA v5.0, the max endpoint ID was 32 */
2038 hw_limit = ipa->version < IPA_VERSION_5_0 ? 32 : U8_MAX + 1;
2039 if (limit > hw_limit) {
2040 dev_err(dev, "unexpected endpoint count, %u > %u\n",
2041 limit, hw_limit);
2042 return -EINVAL;
2043 }
2044
2045 /* Allocate and initialize the available endpoint bitmap */
2046 ipa->available = bitmap_zalloc(nbits: limit, GFP_KERNEL);
2047 if (!ipa->available)
2048 return -ENOMEM;
2049 ipa->available_count = limit;
2050
2051 /* Mark all supported RX and TX endpoints as available */
2052 bitmap_set(map: ipa->available, start: 0, nbits: tx_count);
2053 bitmap_set(map: ipa->available, start: rx_base, nbits: rx_count);
2054
2055 for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count) {
2056 struct ipa_endpoint *endpoint;
2057
2058 if (endpoint_id >= limit) {
2059 dev_err(dev, "invalid endpoint id, %u > %u\n",
2060 endpoint_id, limit - 1);
2061 goto err_free_bitmap;
2062 }
2063
2064 if (!test_bit(endpoint_id, ipa->available)) {
2065 dev_err(dev, "unavailable endpoint id %u\n",
2066 endpoint_id);
2067 goto err_free_bitmap;
2068 }
2069
2070 /* Make sure it's pointing in the right direction */
2071 endpoint = &ipa->endpoint[endpoint_id];
2072 if (endpoint->toward_ipa) {
2073 if (endpoint_id < tx_count)
2074 continue;
2075 } else if (endpoint_id >= rx_base) {
2076 continue;
2077 }
2078
2079 dev_err(dev, "endpoint id %u wrong direction\n", endpoint_id);
2080 goto err_free_bitmap;
2081 }
2082
2083 return 0;
2084
2085err_free_bitmap:
2086 ipa_endpoint_deconfig(ipa);
2087
2088 return -EINVAL;
2089}
2090
2091static void ipa_endpoint_init_one(struct ipa *ipa, enum ipa_endpoint_name name,
2092 const struct ipa_gsi_endpoint_data *data)
2093{
2094 struct ipa_endpoint *endpoint;
2095
2096 endpoint = &ipa->endpoint[data->endpoint_id];
2097
2098 if (data->ee_id == GSI_EE_AP)
2099 ipa->channel_map[data->channel_id] = endpoint;
2100 ipa->name_map[name] = endpoint;
2101
2102 endpoint->ipa = ipa;
2103 endpoint->ee_id = data->ee_id;
2104 endpoint->channel_id = data->channel_id;
2105 endpoint->endpoint_id = data->endpoint_id;
2106 endpoint->toward_ipa = data->toward_ipa;
2107 endpoint->config = data->endpoint.config;
2108
2109 __set_bit(endpoint->endpoint_id, ipa->defined);
2110}
2111
2112static void ipa_endpoint_exit_one(struct ipa_endpoint *endpoint)
2113{
2114 __clear_bit(endpoint->endpoint_id, endpoint->ipa->defined);
2115
2116 memset(endpoint, 0, sizeof(*endpoint));
2117}
2118
2119void ipa_endpoint_exit(struct ipa *ipa)
2120{
2121 u32 endpoint_id;
2122
2123 ipa->filtered = 0;
2124
2125 for_each_set_bit(endpoint_id, ipa->defined, ipa->endpoint_count)
2126 ipa_endpoint_exit_one(endpoint: &ipa->endpoint[endpoint_id]);
2127
2128 bitmap_free(bitmap: ipa->enabled);
2129 ipa->enabled = NULL;
2130 bitmap_free(bitmap: ipa->set_up);
2131 ipa->set_up = NULL;
2132 bitmap_free(bitmap: ipa->defined);
2133 ipa->defined = NULL;
2134
2135 memset(ipa->name_map, 0, sizeof(ipa->name_map));
2136 memset(ipa->channel_map, 0, sizeof(ipa->channel_map));
2137}
2138
2139/* Returns a bitmask of endpoints that support filtering, or 0 on error */
2140int ipa_endpoint_init(struct ipa *ipa, u32 count,
2141 const struct ipa_gsi_endpoint_data *data)
2142{
2143 enum ipa_endpoint_name name;
2144 u32 filtered;
2145
2146 BUILD_BUG_ON(!IPA_REPLENISH_BATCH);
2147
2148 /* Number of endpoints is one more than the maximum ID */
2149 ipa->endpoint_count = ipa_endpoint_max(ipa, count, data) + 1;
2150 if (!ipa->endpoint_count)
2151 return -EINVAL;
2152
2153 /* Initialize endpoint state bitmaps */
2154 ipa->defined = bitmap_zalloc(nbits: ipa->endpoint_count, GFP_KERNEL);
2155 if (!ipa->defined)
2156 return -ENOMEM;
2157
2158 ipa->set_up = bitmap_zalloc(nbits: ipa->endpoint_count, GFP_KERNEL);
2159 if (!ipa->set_up)
2160 goto err_free_defined;
2161
2162 ipa->enabled = bitmap_zalloc(nbits: ipa->endpoint_count, GFP_KERNEL);
2163 if (!ipa->enabled)
2164 goto err_free_set_up;
2165
2166 filtered = 0;
2167 for (name = 0; name < count; name++, data++) {
2168 if (ipa_gsi_endpoint_data_empty(data))
2169 continue; /* Skip over empty slots */
2170
2171 ipa_endpoint_init_one(ipa, name, data);
2172
2173 if (data->endpoint.filter_support)
2174 filtered |= BIT(data->endpoint_id);
2175 if (data->ee_id == GSI_EE_MODEM && data->toward_ipa)
2176 ipa->modem_tx_count++;
2177 }
2178
2179 /* Make sure the set of filtered endpoints is valid */
2180 if (!ipa_filtered_valid(ipa, filtered)) {
2181 ipa_endpoint_exit(ipa);
2182
2183 return -EINVAL;
2184 }
2185
2186 ipa->filtered = filtered;
2187
2188 return 0;
2189
2190err_free_set_up:
2191 bitmap_free(bitmap: ipa->set_up);
2192 ipa->set_up = NULL;
2193err_free_defined:
2194 bitmap_free(bitmap: ipa->defined);
2195 ipa->defined = NULL;
2196
2197 return -ENOMEM;
2198}
2199

source code of linux/drivers/net/ipa/ipa_endpoint.c