1 | // SPDX-License-Identifier: GPL-2.0 |
2 | |
3 | /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. |
4 | * Copyright (C) 2018-2023 Linaro Ltd. |
5 | */ |
6 | |
7 | #include <linux/types.h> |
8 | #include <linux/atomic.h> |
9 | #include <linux/bitfield.h> |
10 | #include <linux/bug.h> |
11 | #include <linux/io.h> |
12 | #include <linux/firmware.h> |
13 | #include <linux/module.h> |
14 | #include <linux/of.h> |
15 | #include <linux/of_address.h> |
16 | #include <linux/platform_device.h> |
17 | #include <linux/pm_runtime.h> |
18 | #include <linux/firmware/qcom/qcom_scm.h> |
19 | #include <linux/soc/qcom/mdt_loader.h> |
20 | |
21 | #include "ipa.h" |
22 | #include "ipa_power.h" |
23 | #include "ipa_data.h" |
24 | #include "ipa_endpoint.h" |
25 | #include "ipa_resource.h" |
26 | #include "ipa_cmd.h" |
27 | #include "ipa_reg.h" |
28 | #include "ipa_mem.h" |
29 | #include "ipa_table.h" |
30 | #include "ipa_smp2p.h" |
31 | #include "ipa_modem.h" |
32 | #include "ipa_uc.h" |
33 | #include "ipa_interrupt.h" |
34 | #include "gsi_trans.h" |
35 | #include "ipa_sysfs.h" |
36 | |
37 | /** |
38 | * DOC: The IP Accelerator |
39 | * |
40 | * This driver supports the Qualcomm IP Accelerator (IPA), which is a |
41 | * networking component found in many Qualcomm SoCs. The IPA is connected |
42 | * to the application processor (AP), but is also connected (and partially |
43 | * controlled by) other "execution environments" (EEs), such as a modem. |
44 | * |
45 | * The IPA is the conduit between the AP and the modem that carries network |
46 | * traffic. This driver presents a network interface representing the |
47 | * connection of the modem to external (e.g. LTE) networks. |
48 | * |
49 | * The IPA provides protocol checksum calculation, offloading this work |
50 | * from the AP. The IPA offers additional functionality, including routing, |
51 | * filtering, and NAT support, but that more advanced functionality is not |
52 | * currently supported. Despite that, some resources--including routing |
53 | * tables and filter tables--are defined in this driver because they must |
54 | * be initialized even when the advanced hardware features are not used. |
55 | * |
56 | * There are two distinct layers that implement the IPA hardware, and this |
57 | * is reflected in the organization of the driver. The generic software |
58 | * interface (GSI) is an integral component of the IPA, providing a |
59 | * well-defined communication layer between the AP subsystem and the IPA |
60 | * core. The GSI implements a set of "channels" used for communication |
61 | * between the AP and the IPA. |
62 | * |
63 | * The IPA layer uses GSI channels to implement its "endpoints". And while |
64 | * a GSI channel carries data between the AP and the IPA, a pair of IPA |
65 | * endpoints is used to carry traffic between two EEs. Specifically, the main |
66 | * modem network interface is implemented by two pairs of endpoints: a TX |
67 | * endpoint on the AP coupled with an RX endpoint on the modem; and another |
68 | * RX endpoint on the AP receiving data from a TX endpoint on the modem. |
69 | */ |
70 | |
71 | /* The name of the GSI firmware file relative to /lib/firmware */ |
72 | #define IPA_FW_PATH_DEFAULT "ipa_fws.mdt" |
73 | #define IPA_PAS_ID 15 |
74 | |
75 | /* Shift of 19.2 MHz timestamp to achieve lower resolution timestamps */ |
76 | /* IPA v5.5+ does not specify Qtime timestamp config for DPL */ |
77 | #define DPL_TIMESTAMP_SHIFT 14 /* ~1.172 kHz, ~853 usec per tick */ |
78 | #define TAG_TIMESTAMP_SHIFT 14 |
79 | #define NAT_TIMESTAMP_SHIFT 24 /* ~1.144 Hz, ~874 msec per tick */ |
80 | |
81 | /* Divider for 19.2 MHz crystal oscillator clock to get common timer clock */ |
82 | #define IPA_XO_CLOCK_DIVIDER 192 /* 1 is subtracted where used */ |
83 | |
84 | /** |
85 | * enum ipa_firmware_loader: How GSI firmware gets loaded |
86 | * |
87 | * @IPA_LOADER_DEFER: System not ready; try again later |
88 | * @IPA_LOADER_SELF: AP loads GSI firmware |
89 | * @IPA_LOADER_MODEM: Modem loads GSI firmware, signals when done |
90 | * @IPA_LOADER_SKIP: Neither AP nor modem need to load GSI firmware |
91 | * @IPA_LOADER_INVALID: GSI firmware loader specification is invalid |
92 | */ |
93 | enum ipa_firmware_loader { |
94 | IPA_LOADER_DEFER, |
95 | IPA_LOADER_SELF, |
96 | IPA_LOADER_MODEM, |
97 | IPA_LOADER_SKIP, |
98 | IPA_LOADER_INVALID, |
99 | }; |
100 | |
101 | /** |
102 | * ipa_setup() - Set up IPA hardware |
103 | * @ipa: IPA pointer |
104 | * |
105 | * Perform initialization that requires issuing immediate commands on |
106 | * the command TX endpoint. If the modem is doing GSI firmware load |
107 | * and initialization, this function will be called when an SMP2P |
108 | * interrupt has been signaled by the modem. Otherwise it will be |
109 | * called from ipa_probe() after GSI firmware has been successfully |
110 | * loaded, authenticated, and started by Trust Zone. |
111 | */ |
112 | int ipa_setup(struct ipa *ipa) |
113 | { |
114 | struct ipa_endpoint *exception_endpoint; |
115 | struct ipa_endpoint *command_endpoint; |
116 | struct device *dev = ipa->dev; |
117 | int ret; |
118 | |
119 | ret = gsi_setup(gsi: &ipa->gsi); |
120 | if (ret) |
121 | return ret; |
122 | |
123 | ret = ipa_power_setup(ipa); |
124 | if (ret) |
125 | goto err_gsi_teardown; |
126 | |
127 | ipa_endpoint_setup(ipa); |
128 | |
129 | /* We need to use the AP command TX endpoint to perform other |
130 | * initialization, so we enable first. |
131 | */ |
132 | command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]; |
133 | ret = ipa_endpoint_enable_one(endpoint: command_endpoint); |
134 | if (ret) |
135 | goto err_endpoint_teardown; |
136 | |
137 | ret = ipa_mem_setup(ipa); /* No matching teardown required */ |
138 | if (ret) |
139 | goto err_command_disable; |
140 | |
141 | ret = ipa_table_setup(ipa); /* No matching teardown required */ |
142 | if (ret) |
143 | goto err_command_disable; |
144 | |
145 | /* Enable the exception handling endpoint, and tell the hardware |
146 | * to use it by default. |
147 | */ |
148 | exception_endpoint = ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]; |
149 | ret = ipa_endpoint_enable_one(endpoint: exception_endpoint); |
150 | if (ret) |
151 | goto err_command_disable; |
152 | |
153 | ipa_endpoint_default_route_set(ipa, endpoint_id: exception_endpoint->endpoint_id); |
154 | |
155 | /* We're all set. Now prepare for communication with the modem */ |
156 | ret = ipa_qmi_setup(ipa); |
157 | if (ret) |
158 | goto err_default_route_clear; |
159 | |
160 | ipa->setup_complete = true; |
161 | |
162 | dev_info(dev, "IPA driver setup completed successfully\n" ); |
163 | |
164 | return 0; |
165 | |
166 | err_default_route_clear: |
167 | ipa_endpoint_default_route_clear(ipa); |
168 | ipa_endpoint_disable_one(endpoint: exception_endpoint); |
169 | err_command_disable: |
170 | ipa_endpoint_disable_one(endpoint: command_endpoint); |
171 | err_endpoint_teardown: |
172 | ipa_endpoint_teardown(ipa); |
173 | ipa_power_teardown(ipa); |
174 | err_gsi_teardown: |
175 | gsi_teardown(gsi: &ipa->gsi); |
176 | |
177 | return ret; |
178 | } |
179 | |
180 | /** |
181 | * ipa_teardown() - Inverse of ipa_setup() |
182 | * @ipa: IPA pointer |
183 | */ |
184 | static void ipa_teardown(struct ipa *ipa) |
185 | { |
186 | struct ipa_endpoint *exception_endpoint; |
187 | struct ipa_endpoint *command_endpoint; |
188 | |
189 | /* We're going to tear everything down, as if setup never completed */ |
190 | ipa->setup_complete = false; |
191 | |
192 | ipa_qmi_teardown(ipa); |
193 | ipa_endpoint_default_route_clear(ipa); |
194 | exception_endpoint = ipa->name_map[IPA_ENDPOINT_AP_LAN_RX]; |
195 | ipa_endpoint_disable_one(endpoint: exception_endpoint); |
196 | command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX]; |
197 | ipa_endpoint_disable_one(endpoint: command_endpoint); |
198 | ipa_endpoint_teardown(ipa); |
199 | ipa_power_teardown(ipa); |
200 | gsi_teardown(gsi: &ipa->gsi); |
201 | } |
202 | |
203 | static void |
204 | ipa_hardware_config_bcr(struct ipa *ipa, const struct ipa_data *data) |
205 | { |
206 | const struct reg *reg; |
207 | u32 val; |
208 | |
209 | /* IPA v4.5+ has no backward compatibility register */ |
210 | if (ipa->version >= IPA_VERSION_4_5) |
211 | return; |
212 | |
213 | reg = ipa_reg(ipa, reg_id: IPA_BCR); |
214 | val = data->backward_compat; |
215 | iowrite32(val, ipa->reg_virt + reg_offset(reg)); |
216 | } |
217 | |
218 | static void ipa_hardware_config_tx(struct ipa *ipa) |
219 | { |
220 | enum ipa_version version = ipa->version; |
221 | const struct reg *reg; |
222 | u32 offset; |
223 | u32 val; |
224 | |
225 | if (version <= IPA_VERSION_4_0 || version >= IPA_VERSION_4_5) |
226 | return; |
227 | |
228 | /* Disable PA mask to allow HOLB drop */ |
229 | reg = ipa_reg(ipa, reg_id: IPA_TX_CFG); |
230 | offset = reg_offset(reg); |
231 | |
232 | val = ioread32(ipa->reg_virt + offset); |
233 | |
234 | val &= ~reg_bit(reg, field_id: PA_MASK_EN); |
235 | |
236 | iowrite32(val, ipa->reg_virt + offset); |
237 | } |
238 | |
239 | static void ipa_hardware_config_clkon(struct ipa *ipa) |
240 | { |
241 | enum ipa_version version = ipa->version; |
242 | const struct reg *reg; |
243 | u32 val; |
244 | |
245 | if (version >= IPA_VERSION_4_5) |
246 | return; |
247 | |
248 | if (version < IPA_VERSION_4_0 && version != IPA_VERSION_3_1) |
249 | return; |
250 | |
251 | /* Implement some hardware workarounds */ |
252 | reg = ipa_reg(ipa, reg_id: CLKON_CFG); |
253 | if (version == IPA_VERSION_3_1) { |
254 | /* Disable MISC clock gating */ |
255 | val = reg_bit(reg, field_id: CLKON_MISC); |
256 | } else { /* IPA v4.0+ */ |
257 | /* Enable open global clocks in the CLKON configuration */ |
258 | val = reg_bit(reg, field_id: CLKON_GLOBAL); |
259 | val |= reg_bit(reg, field_id: GLOBAL_2X_CLK); |
260 | } |
261 | |
262 | iowrite32(val, ipa->reg_virt + reg_offset(reg)); |
263 | } |
264 | |
265 | /* Configure bus access behavior for IPA components */ |
266 | static void ipa_hardware_config_comp(struct ipa *ipa) |
267 | { |
268 | const struct reg *reg; |
269 | u32 offset; |
270 | u32 val; |
271 | |
272 | /* Nothing to configure prior to IPA v4.0 */ |
273 | if (ipa->version < IPA_VERSION_4_0) |
274 | return; |
275 | |
276 | reg = ipa_reg(ipa, reg_id: COMP_CFG); |
277 | offset = reg_offset(reg); |
278 | |
279 | val = ioread32(ipa->reg_virt + offset); |
280 | |
281 | if (ipa->version == IPA_VERSION_4_0) { |
282 | val &= ~reg_bit(reg, field_id: IPA_QMB_SELECT_CONS_EN); |
283 | val &= ~reg_bit(reg, field_id: IPA_QMB_SELECT_PROD_EN); |
284 | val &= ~reg_bit(reg, field_id: IPA_QMB_SELECT_GLOBAL_EN); |
285 | } else if (ipa->version < IPA_VERSION_4_5) { |
286 | val |= reg_bit(reg, field_id: GSI_MULTI_AXI_MASTERS_DIS); |
287 | } else { |
288 | /* For IPA v4.5+ FULL_FLUSH_WAIT_RS_CLOSURE_EN is 0 */ |
289 | } |
290 | |
291 | val |= reg_bit(reg, field_id: GSI_MULTI_INORDER_RD_DIS); |
292 | val |= reg_bit(reg, field_id: GSI_MULTI_INORDER_WR_DIS); |
293 | |
294 | iowrite32(val, ipa->reg_virt + offset); |
295 | } |
296 | |
297 | /* Configure DDR and (possibly) PCIe max read/write QSB values */ |
298 | static void |
299 | ipa_hardware_config_qsb(struct ipa *ipa, const struct ipa_data *data) |
300 | { |
301 | const struct ipa_qsb_data *data0; |
302 | const struct ipa_qsb_data *data1; |
303 | const struct reg *reg; |
304 | u32 val; |
305 | |
306 | /* QMB 0 represents DDR; QMB 1 (if present) represents PCIe */ |
307 | data0 = &data->qsb_data[IPA_QSB_MASTER_DDR]; |
308 | if (data->qsb_count > 1) |
309 | data1 = &data->qsb_data[IPA_QSB_MASTER_PCIE]; |
310 | |
311 | /* Max outstanding write accesses for QSB masters */ |
312 | reg = ipa_reg(ipa, reg_id: QSB_MAX_WRITES); |
313 | |
314 | val = reg_encode(reg, field_id: GEN_QMB_0_MAX_WRITES, val: data0->max_writes); |
315 | if (data->qsb_count > 1) |
316 | val |= reg_encode(reg, field_id: GEN_QMB_1_MAX_WRITES, val: data1->max_writes); |
317 | |
318 | iowrite32(val, ipa->reg_virt + reg_offset(reg)); |
319 | |
320 | /* Max outstanding read accesses for QSB masters */ |
321 | reg = ipa_reg(ipa, reg_id: QSB_MAX_READS); |
322 | |
323 | val = reg_encode(reg, field_id: GEN_QMB_0_MAX_READS, val: data0->max_reads); |
324 | if (ipa->version >= IPA_VERSION_4_0) |
325 | val |= reg_encode(reg, field_id: GEN_QMB_0_MAX_READS_BEATS, |
326 | val: data0->max_reads_beats); |
327 | if (data->qsb_count > 1) { |
328 | val = reg_encode(reg, field_id: GEN_QMB_1_MAX_READS, val: data1->max_reads); |
329 | if (ipa->version >= IPA_VERSION_4_0) |
330 | val |= reg_encode(reg, field_id: GEN_QMB_1_MAX_READS_BEATS, |
331 | val: data1->max_reads_beats); |
332 | } |
333 | |
334 | iowrite32(val, ipa->reg_virt + reg_offset(reg)); |
335 | } |
336 | |
337 | /* The internal inactivity timer clock is used for the aggregation timer */ |
338 | #define TIMER_FREQUENCY 32000 /* 32 KHz inactivity timer clock */ |
339 | |
340 | /* Compute the value to use in the COUNTER_CFG register AGGR_GRANULARITY |
341 | * field to represent the given number of microseconds. The value is one |
342 | * less than the number of timer ticks in the requested period. 0 is not |
343 | * a valid granularity value (so for example @usec must be at least 16 for |
344 | * a TIMER_FREQUENCY of 32000). |
345 | */ |
346 | static __always_inline u32 ipa_aggr_granularity_val(u32 usec) |
347 | { |
348 | return DIV_ROUND_CLOSEST(usec * TIMER_FREQUENCY, USEC_PER_SEC) - 1; |
349 | } |
350 | |
351 | /* IPA uses unified Qtime starting at IPA v4.5, implementing various |
352 | * timestamps and timers independent of the IPA core clock rate. The |
353 | * Qtimer is based on a 56-bit timestamp incremented at each tick of |
354 | * a 19.2 MHz SoC crystal oscillator (XO clock). |
355 | * |
356 | * For IPA timestamps (tag, NAT, data path logging) a lower resolution |
357 | * timestamp is achieved by shifting the Qtimer timestamp value right |
358 | * some number of bits to produce the low-order bits of the coarser |
359 | * granularity timestamp. |
360 | * |
361 | * For timers, a common timer clock is derived from the XO clock using |
362 | * a divider (we use 192, to produce a 100kHz timer clock). From |
363 | * this common clock, three "pulse generators" are used to produce |
364 | * timer ticks at a configurable frequency. IPA timers (such as |
365 | * those used for aggregation or head-of-line block handling) now |
366 | * define their period based on one of these pulse generators. |
367 | */ |
368 | static void ipa_qtime_config(struct ipa *ipa) |
369 | { |
370 | const struct reg *reg; |
371 | u32 offset; |
372 | u32 val; |
373 | |
374 | /* Timer clock divider must be disabled when we change the rate */ |
375 | reg = ipa_reg(ipa, reg_id: TIMERS_XO_CLK_DIV_CFG); |
376 | iowrite32(0, ipa->reg_virt + reg_offset(reg)); |
377 | |
378 | reg = ipa_reg(ipa, reg_id: QTIME_TIMESTAMP_CFG); |
379 | if (ipa->version < IPA_VERSION_5_5) { |
380 | /* Set DPL time stamp resolution to use Qtime (not 1 msec) */ |
381 | val = reg_encode(reg, field_id: DPL_TIMESTAMP_LSB, DPL_TIMESTAMP_SHIFT); |
382 | val |= reg_bit(reg, field_id: DPL_TIMESTAMP_SEL); |
383 | } |
384 | /* Configure tag and NAT Qtime timestamp resolution as well */ |
385 | val = reg_encode(reg, field_id: TAG_TIMESTAMP_LSB, TAG_TIMESTAMP_SHIFT); |
386 | val = reg_encode(reg, field_id: NAT_TIMESTAMP_LSB, NAT_TIMESTAMP_SHIFT); |
387 | |
388 | iowrite32(val, ipa->reg_virt + reg_offset(reg)); |
389 | |
390 | /* Set granularity of pulse generators used for other timers */ |
391 | reg = ipa_reg(ipa, reg_id: TIMERS_PULSE_GRAN_CFG); |
392 | val = reg_encode(reg, field_id: PULSE_GRAN_0, val: IPA_GRAN_100_US); |
393 | val |= reg_encode(reg, field_id: PULSE_GRAN_1, val: IPA_GRAN_1_MS); |
394 | if (ipa->version >= IPA_VERSION_5_0) { |
395 | val |= reg_encode(reg, field_id: PULSE_GRAN_2, val: IPA_GRAN_10_MS); |
396 | val |= reg_encode(reg, field_id: PULSE_GRAN_3, val: IPA_GRAN_10_MS); |
397 | } else { |
398 | val |= reg_encode(reg, field_id: PULSE_GRAN_2, val: IPA_GRAN_1_MS); |
399 | } |
400 | |
401 | iowrite32(val, ipa->reg_virt + reg_offset(reg)); |
402 | |
403 | /* Actual divider is 1 more than value supplied here */ |
404 | reg = ipa_reg(ipa, reg_id: TIMERS_XO_CLK_DIV_CFG); |
405 | offset = reg_offset(reg); |
406 | |
407 | val = reg_encode(reg, field_id: DIV_VALUE, IPA_XO_CLOCK_DIVIDER - 1); |
408 | |
409 | iowrite32(val, ipa->reg_virt + offset); |
410 | |
411 | /* Divider value is set; re-enable the common timer clock divider */ |
412 | val |= reg_bit(reg, field_id: DIV_ENABLE); |
413 | |
414 | iowrite32(val, ipa->reg_virt + offset); |
415 | } |
416 | |
417 | /* Before IPA v4.5 timing is controlled by a counter register */ |
418 | static void ipa_hardware_config_counter(struct ipa *ipa) |
419 | { |
420 | u32 granularity = ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY); |
421 | const struct reg *reg; |
422 | u32 val; |
423 | |
424 | reg = ipa_reg(ipa, reg_id: COUNTER_CFG); |
425 | /* If defined, EOT_COAL_GRANULARITY is 0 */ |
426 | val = reg_encode(reg, field_id: AGGR_GRANULARITY, val: granularity); |
427 | iowrite32(val, ipa->reg_virt + reg_offset(reg)); |
428 | } |
429 | |
430 | static void ipa_hardware_config_timing(struct ipa *ipa) |
431 | { |
432 | if (ipa->version < IPA_VERSION_4_5) |
433 | ipa_hardware_config_counter(ipa); |
434 | else |
435 | ipa_qtime_config(ipa); |
436 | } |
437 | |
438 | static void ipa_hardware_config_hashing(struct ipa *ipa) |
439 | { |
440 | const struct reg *reg; |
441 | |
442 | /* Other than IPA v4.2, all versions enable "hashing". Starting |
443 | * with IPA v5.0, the filter and router tables are implemented |
444 | * differently, but the default configuration enables this feature |
445 | * (now referred to as "cacheing"), so there's nothing to do here. |
446 | */ |
447 | if (ipa->version != IPA_VERSION_4_2) |
448 | return; |
449 | |
450 | /* IPA v4.2 does not support hashed tables, so disable them */ |
451 | reg = ipa_reg(ipa, reg_id: FILT_ROUT_HASH_EN); |
452 | |
453 | /* IPV6_ROUTER_HASH, IPV6_FILTER_HASH, IPV4_ROUTER_HASH, |
454 | * IPV4_FILTER_HASH are all zero. |
455 | */ |
456 | iowrite32(0, ipa->reg_virt + reg_offset(reg)); |
457 | } |
458 | |
459 | static void ipa_idle_indication_cfg(struct ipa *ipa, |
460 | u32 enter_idle_debounce_thresh, |
461 | bool const_non_idle_enable) |
462 | { |
463 | const struct reg *reg; |
464 | u32 val; |
465 | |
466 | if (ipa->version < IPA_VERSION_3_5_1) |
467 | return; |
468 | |
469 | reg = ipa_reg(ipa, reg_id: IDLE_INDICATION_CFG); |
470 | val = reg_encode(reg, field_id: ENTER_IDLE_DEBOUNCE_THRESH, |
471 | val: enter_idle_debounce_thresh); |
472 | if (const_non_idle_enable) |
473 | val |= reg_bit(reg, field_id: CONST_NON_IDLE_ENABLE); |
474 | |
475 | iowrite32(val, ipa->reg_virt + reg_offset(reg)); |
476 | } |
477 | |
478 | /** |
479 | * ipa_hardware_dcd_config() - Enable dynamic clock division on IPA |
480 | * @ipa: IPA pointer |
481 | * |
482 | * Configures when the IPA signals it is idle to the global clock |
483 | * controller, which can respond by scaling down the clock to save |
484 | * power. |
485 | */ |
486 | static void ipa_hardware_dcd_config(struct ipa *ipa) |
487 | { |
488 | /* Recommended values for IPA 3.5 and later according to IPA HPG */ |
489 | ipa_idle_indication_cfg(ipa, enter_idle_debounce_thresh: 256, const_non_idle_enable: false); |
490 | } |
491 | |
492 | static void ipa_hardware_dcd_deconfig(struct ipa *ipa) |
493 | { |
494 | /* Power-on reset values */ |
495 | ipa_idle_indication_cfg(ipa, enter_idle_debounce_thresh: 0, const_non_idle_enable: true); |
496 | } |
497 | |
498 | /** |
499 | * ipa_hardware_config() - Primitive hardware initialization |
500 | * @ipa: IPA pointer |
501 | * @data: IPA configuration data |
502 | */ |
503 | static void ipa_hardware_config(struct ipa *ipa, const struct ipa_data *data) |
504 | { |
505 | ipa_hardware_config_bcr(ipa, data); |
506 | ipa_hardware_config_tx(ipa); |
507 | ipa_hardware_config_clkon(ipa); |
508 | ipa_hardware_config_comp(ipa); |
509 | ipa_hardware_config_qsb(ipa, data); |
510 | ipa_hardware_config_timing(ipa); |
511 | ipa_hardware_config_hashing(ipa); |
512 | ipa_hardware_dcd_config(ipa); |
513 | } |
514 | |
515 | /** |
516 | * ipa_hardware_deconfig() - Inverse of ipa_hardware_config() |
517 | * @ipa: IPA pointer |
518 | * |
519 | * This restores the power-on reset values (even if they aren't different) |
520 | */ |
521 | static void ipa_hardware_deconfig(struct ipa *ipa) |
522 | { |
523 | /* Mostly we just leave things as we set them. */ |
524 | ipa_hardware_dcd_deconfig(ipa); |
525 | } |
526 | |
527 | /** |
528 | * ipa_config() - Configure IPA hardware |
529 | * @ipa: IPA pointer |
530 | * @data: IPA configuration data |
531 | * |
532 | * Perform initialization requiring IPA power to be enabled. |
533 | */ |
534 | static int ipa_config(struct ipa *ipa, const struct ipa_data *data) |
535 | { |
536 | int ret; |
537 | |
538 | ipa_hardware_config(ipa, data); |
539 | |
540 | ret = ipa_mem_config(ipa); |
541 | if (ret) |
542 | goto err_hardware_deconfig; |
543 | |
544 | ret = ipa_interrupt_config(ipa); |
545 | if (ret) |
546 | goto err_mem_deconfig; |
547 | |
548 | ipa_uc_config(ipa); |
549 | |
550 | ret = ipa_endpoint_config(ipa); |
551 | if (ret) |
552 | goto err_uc_deconfig; |
553 | |
554 | ipa_table_config(ipa); /* No deconfig required */ |
555 | |
556 | /* Assign resource limitation to each group; no deconfig required */ |
557 | ret = ipa_resource_config(ipa, data: data->resource_data); |
558 | if (ret) |
559 | goto err_endpoint_deconfig; |
560 | |
561 | ret = ipa_modem_config(ipa); |
562 | if (ret) |
563 | goto err_endpoint_deconfig; |
564 | |
565 | return 0; |
566 | |
567 | err_endpoint_deconfig: |
568 | ipa_endpoint_deconfig(ipa); |
569 | err_uc_deconfig: |
570 | ipa_uc_deconfig(ipa); |
571 | ipa_interrupt_deconfig(ipa); |
572 | err_mem_deconfig: |
573 | ipa_mem_deconfig(ipa); |
574 | err_hardware_deconfig: |
575 | ipa_hardware_deconfig(ipa); |
576 | |
577 | return ret; |
578 | } |
579 | |
580 | /** |
581 | * ipa_deconfig() - Inverse of ipa_config() |
582 | * @ipa: IPA pointer |
583 | */ |
584 | static void ipa_deconfig(struct ipa *ipa) |
585 | { |
586 | ipa_modem_deconfig(ipa); |
587 | ipa_endpoint_deconfig(ipa); |
588 | ipa_uc_deconfig(ipa); |
589 | ipa_interrupt_deconfig(ipa); |
590 | ipa_mem_deconfig(ipa); |
591 | ipa_hardware_deconfig(ipa); |
592 | } |
593 | |
594 | static int ipa_firmware_load(struct device *dev) |
595 | { |
596 | const struct firmware *fw; |
597 | struct device_node *node; |
598 | struct resource res; |
599 | phys_addr_t phys; |
600 | const char *path; |
601 | ssize_t size; |
602 | void *virt; |
603 | int ret; |
604 | |
605 | node = of_parse_phandle(np: dev->of_node, phandle_name: "memory-region" , index: 0); |
606 | if (!node) { |
607 | dev_err(dev, "DT error getting \"memory-region\" property\n" ); |
608 | return -EINVAL; |
609 | } |
610 | |
611 | ret = of_address_to_resource(dev: node, index: 0, r: &res); |
612 | of_node_put(node); |
613 | if (ret) { |
614 | dev_err(dev, "error %d getting \"memory-region\" resource\n" , |
615 | ret); |
616 | return ret; |
617 | } |
618 | |
619 | /* Use name from DTB if specified; use default for *any* error */ |
620 | ret = of_property_read_string(np: dev->of_node, propname: "firmware-name" , out_string: &path); |
621 | if (ret) { |
622 | dev_dbg(dev, "error %d getting \"firmware-name\" resource\n" , |
623 | ret); |
624 | path = IPA_FW_PATH_DEFAULT; |
625 | } |
626 | |
627 | ret = request_firmware(fw: &fw, name: path, device: dev); |
628 | if (ret) { |
629 | dev_err(dev, "error %d requesting \"%s\"\n" , ret, path); |
630 | return ret; |
631 | } |
632 | |
633 | phys = res.start; |
634 | size = (size_t)resource_size(res: &res); |
635 | virt = memremap(offset: phys, size, flags: MEMREMAP_WC); |
636 | if (!virt) { |
637 | dev_err(dev, "unable to remap firmware memory\n" ); |
638 | ret = -ENOMEM; |
639 | goto out_release_firmware; |
640 | } |
641 | |
642 | ret = qcom_mdt_load(dev, fw, fw_name: path, IPA_PAS_ID, mem_region: virt, mem_phys: phys, mem_size: size, NULL); |
643 | if (ret) |
644 | dev_err(dev, "error %d loading \"%s\"\n" , ret, path); |
645 | else if ((ret = qcom_scm_pas_auth_and_reset(IPA_PAS_ID))) |
646 | dev_err(dev, "error %d authenticating \"%s\"\n" , ret, path); |
647 | |
648 | memunmap(addr: virt); |
649 | out_release_firmware: |
650 | release_firmware(fw); |
651 | |
652 | return ret; |
653 | } |
654 | |
655 | static const struct of_device_id ipa_match[] = { |
656 | { |
657 | .compatible = "qcom,msm8998-ipa" , |
658 | .data = &ipa_data_v3_1, |
659 | }, |
660 | { |
661 | .compatible = "qcom,sdm845-ipa" , |
662 | .data = &ipa_data_v3_5_1, |
663 | }, |
664 | { |
665 | .compatible = "qcom,sc7180-ipa" , |
666 | .data = &ipa_data_v4_2, |
667 | }, |
668 | { |
669 | .compatible = "qcom,sdx55-ipa" , |
670 | .data = &ipa_data_v4_5, |
671 | }, |
672 | { |
673 | .compatible = "qcom,sm6350-ipa" , |
674 | .data = &ipa_data_v4_7, |
675 | }, |
676 | { |
677 | .compatible = "qcom,sm8350-ipa" , |
678 | .data = &ipa_data_v4_9, |
679 | }, |
680 | { |
681 | .compatible = "qcom,sc7280-ipa" , |
682 | .data = &ipa_data_v4_11, |
683 | }, |
684 | { |
685 | .compatible = "qcom,sdx65-ipa" , |
686 | .data = &ipa_data_v5_0, |
687 | }, |
688 | { |
689 | .compatible = "qcom,sm8550-ipa" , |
690 | .data = &ipa_data_v5_5, |
691 | }, |
692 | { }, |
693 | }; |
694 | MODULE_DEVICE_TABLE(of, ipa_match); |
695 | |
696 | /* Check things that can be validated at build time. This just |
697 | * groups these things BUILD_BUG_ON() calls don't clutter the rest |
698 | * of the code. |
699 | * */ |
700 | static void ipa_validate_build(void) |
701 | { |
702 | /* At one time we assumed a 64-bit build, allowing some do_div() |
703 | * calls to be replaced by simple division or modulo operations. |
704 | * We currently only perform divide and modulo operations on u32, |
705 | * u16, or size_t objects, and of those only size_t has any chance |
706 | * of being a 64-bit value. (It should be guaranteed 32 bits wide |
707 | * on a 32-bit build, but there is no harm in verifying that.) |
708 | */ |
709 | BUILD_BUG_ON(!IS_ENABLED(CONFIG_64BIT) && sizeof(size_t) != 4); |
710 | |
711 | /* Code assumes the EE ID for the AP is 0 (zeroed structure field) */ |
712 | BUILD_BUG_ON(GSI_EE_AP != 0); |
713 | |
714 | /* There's no point if we have no channels or event rings */ |
715 | BUILD_BUG_ON(!GSI_CHANNEL_COUNT_MAX); |
716 | BUILD_BUG_ON(!GSI_EVT_RING_COUNT_MAX); |
717 | |
718 | /* GSI hardware design limits */ |
719 | BUILD_BUG_ON(GSI_CHANNEL_COUNT_MAX > 32); |
720 | BUILD_BUG_ON(GSI_EVT_RING_COUNT_MAX > 31); |
721 | |
722 | /* The number of TREs in a transaction is limited by the channel's |
723 | * TLV FIFO size. A transaction structure uses 8-bit fields |
724 | * to represents the number of TREs it has allocated and used. |
725 | */ |
726 | BUILD_BUG_ON(GSI_TLV_MAX > U8_MAX); |
727 | |
728 | /* This is used as a divisor */ |
729 | BUILD_BUG_ON(!IPA_AGGR_GRANULARITY); |
730 | |
731 | /* Aggregation granularity value can't be 0, and must fit */ |
732 | BUILD_BUG_ON(!ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY)); |
733 | } |
734 | |
735 | static enum ipa_firmware_loader ipa_firmware_loader(struct device *dev) |
736 | { |
737 | bool modem_init; |
738 | const char *str; |
739 | int ret; |
740 | |
741 | /* Look up the old and new properties by name */ |
742 | modem_init = of_property_read_bool(np: dev->of_node, propname: "modem-init" ); |
743 | ret = of_property_read_string(np: dev->of_node, propname: "qcom,gsi-loader" , out_string: &str); |
744 | |
745 | /* If the new property doesn't exist, it's legacy behavior */ |
746 | if (ret == -EINVAL) { |
747 | if (modem_init) |
748 | return IPA_LOADER_MODEM; |
749 | goto out_self; |
750 | } |
751 | |
752 | /* Any other error on the new property means it's poorly defined */ |
753 | if (ret) |
754 | return IPA_LOADER_INVALID; |
755 | |
756 | /* New property value exists; if old one does too, that's invalid */ |
757 | if (modem_init) |
758 | return IPA_LOADER_INVALID; |
759 | |
760 | /* Modem loads GSI firmware for "modem" */ |
761 | if (!strcmp(str, "modem" )) |
762 | return IPA_LOADER_MODEM; |
763 | |
764 | /* No GSI firmware load is needed for "skip" */ |
765 | if (!strcmp(str, "skip" )) |
766 | return IPA_LOADER_SKIP; |
767 | |
768 | /* Any value other than "self" is an error */ |
769 | if (strcmp(str, "self" )) |
770 | return IPA_LOADER_INVALID; |
771 | out_self: |
772 | /* We need Trust Zone to load firmware; make sure it's available */ |
773 | if (qcom_scm_is_available()) |
774 | return IPA_LOADER_SELF; |
775 | |
776 | return IPA_LOADER_DEFER; |
777 | } |
778 | |
779 | /** |
780 | * ipa_probe() - IPA platform driver probe function |
781 | * @pdev: Platform device pointer |
782 | * |
783 | * Return: 0 if successful, or a negative error code (possibly |
784 | * EPROBE_DEFER) |
785 | * |
786 | * This is the main entry point for the IPA driver. Initialization proceeds |
787 | * in several stages: |
788 | * - The "init" stage involves activities that can be initialized without |
789 | * access to the IPA hardware. |
790 | * - The "config" stage requires IPA power to be active so IPA registers |
791 | * can be accessed, but does not require the use of IPA immediate commands. |
792 | * - The "setup" stage uses IPA immediate commands, and so requires the GSI |
793 | * layer to be initialized. |
794 | * |
795 | * A Boolean Device Tree "modem-init" property determines whether GSI |
796 | * initialization will be performed by the AP (Trust Zone) or the modem. |
797 | * If the AP does GSI initialization, the setup phase is entered after |
798 | * this has completed successfully. Otherwise the modem initializes |
799 | * the GSI layer and signals it has finished by sending an SMP2P interrupt |
800 | * to the AP; this triggers the start if IPA setup. |
801 | */ |
802 | static int ipa_probe(struct platform_device *pdev) |
803 | { |
804 | struct device *dev = &pdev->dev; |
805 | struct ipa_interrupt *interrupt; |
806 | enum ipa_firmware_loader loader; |
807 | const struct ipa_data *data; |
808 | struct ipa_power *power; |
809 | struct ipa *ipa; |
810 | int ret; |
811 | |
812 | ipa_validate_build(); |
813 | |
814 | /* Get configuration data early; needed for power initialization */ |
815 | data = of_device_get_match_data(dev); |
816 | if (!data) { |
817 | dev_err(dev, "matched hardware not supported\n" ); |
818 | return -ENODEV; |
819 | } |
820 | |
821 | if (!ipa_version_supported(version: data->version)) { |
822 | dev_err(dev, "unsupported IPA version %u\n" , data->version); |
823 | return -EINVAL; |
824 | } |
825 | |
826 | if (!data->modem_route_count) { |
827 | dev_err(dev, "modem_route_count cannot be zero\n" ); |
828 | return -EINVAL; |
829 | } |
830 | |
831 | loader = ipa_firmware_loader(dev); |
832 | if (loader == IPA_LOADER_INVALID) |
833 | return -EINVAL; |
834 | if (loader == IPA_LOADER_DEFER) |
835 | return -EPROBE_DEFER; |
836 | |
837 | /* The IPA interrupt might not be ready when we're probed, so this |
838 | * might return -EPROBE_DEFER. |
839 | */ |
840 | interrupt = ipa_interrupt_init(pdev); |
841 | if (IS_ERR(ptr: interrupt)) |
842 | return PTR_ERR(ptr: interrupt); |
843 | |
844 | /* The clock and interconnects might not be ready when we're probed, |
845 | * so this might return -EPROBE_DEFER. |
846 | */ |
847 | power = ipa_power_init(dev, data: data->power_data); |
848 | if (IS_ERR(ptr: power)) { |
849 | ret = PTR_ERR(ptr: power); |
850 | goto err_interrupt_exit; |
851 | } |
852 | |
853 | /* No more EPROBE_DEFER. Allocate and initialize the IPA structure */ |
854 | ipa = kzalloc(size: sizeof(*ipa), GFP_KERNEL); |
855 | if (!ipa) { |
856 | ret = -ENOMEM; |
857 | goto err_power_exit; |
858 | } |
859 | |
860 | ipa->dev = dev; |
861 | dev_set_drvdata(dev, data: ipa); |
862 | ipa->interrupt = interrupt; |
863 | ipa->power = power; |
864 | ipa->version = data->version; |
865 | ipa->modem_route_count = data->modem_route_count; |
866 | init_completion(x: &ipa->completion); |
867 | |
868 | ret = ipa_reg_init(ipa, pdev); |
869 | if (ret) |
870 | goto err_kfree_ipa; |
871 | |
872 | ret = ipa_mem_init(ipa, pdev, mem_data: data->mem_data); |
873 | if (ret) |
874 | goto err_reg_exit; |
875 | |
876 | ret = gsi_init(gsi: &ipa->gsi, pdev, version: ipa->version, count: data->endpoint_count, |
877 | data: data->endpoint_data); |
878 | if (ret) |
879 | goto err_mem_exit; |
880 | |
881 | /* Result is a non-zero mask of endpoints that support filtering */ |
882 | ret = ipa_endpoint_init(ipa, count: data->endpoint_count, data: data->endpoint_data); |
883 | if (ret) |
884 | goto err_gsi_exit; |
885 | |
886 | ret = ipa_table_init(ipa); |
887 | if (ret) |
888 | goto err_endpoint_exit; |
889 | |
890 | ret = ipa_smp2p_init(ipa, pdev, modem_init: loader == IPA_LOADER_MODEM); |
891 | if (ret) |
892 | goto err_table_exit; |
893 | |
894 | /* Power needs to be active for config and setup */ |
895 | ret = pm_runtime_get_sync(dev); |
896 | if (WARN_ON(ret < 0)) |
897 | goto err_power_put; |
898 | |
899 | ret = ipa_config(ipa, data); |
900 | if (ret) |
901 | goto err_power_put; |
902 | |
903 | dev_info(dev, "IPA driver initialized" ); |
904 | |
905 | /* If the modem is loading GSI firmware, it will trigger a call to |
906 | * ipa_setup() when it has finished. In that case we're done here. |
907 | */ |
908 | if (loader == IPA_LOADER_MODEM) |
909 | goto done; |
910 | |
911 | if (loader == IPA_LOADER_SELF) { |
912 | /* The AP is loading GSI firmware; do so now */ |
913 | ret = ipa_firmware_load(dev); |
914 | if (ret) |
915 | goto err_deconfig; |
916 | } /* Otherwise loader == IPA_LOADER_SKIP */ |
917 | |
918 | /* GSI firmware is loaded; proceed to setup */ |
919 | ret = ipa_setup(ipa); |
920 | if (ret) |
921 | goto err_deconfig; |
922 | done: |
923 | pm_runtime_mark_last_busy(dev); |
924 | (void)pm_runtime_put_autosuspend(dev); |
925 | |
926 | return 0; |
927 | |
928 | err_deconfig: |
929 | ipa_deconfig(ipa); |
930 | err_power_put: |
931 | pm_runtime_put_noidle(dev); |
932 | ipa_smp2p_exit(ipa); |
933 | err_table_exit: |
934 | ipa_table_exit(ipa); |
935 | err_endpoint_exit: |
936 | ipa_endpoint_exit(ipa); |
937 | err_gsi_exit: |
938 | gsi_exit(gsi: &ipa->gsi); |
939 | err_mem_exit: |
940 | ipa_mem_exit(ipa); |
941 | err_reg_exit: |
942 | ipa_reg_exit(ipa); |
943 | err_kfree_ipa: |
944 | kfree(objp: ipa); |
945 | err_power_exit: |
946 | ipa_power_exit(power); |
947 | err_interrupt_exit: |
948 | ipa_interrupt_exit(interrupt); |
949 | |
950 | return ret; |
951 | } |
952 | |
953 | static void ipa_remove(struct platform_device *pdev) |
954 | { |
955 | struct ipa_interrupt *interrupt; |
956 | struct ipa_power *power; |
957 | struct device *dev; |
958 | struct ipa *ipa; |
959 | int ret; |
960 | |
961 | ipa = dev_get_drvdata(dev: &pdev->dev); |
962 | dev = ipa->dev; |
963 | WARN_ON(dev != &pdev->dev); |
964 | |
965 | power = ipa->power; |
966 | interrupt = ipa->interrupt; |
967 | |
968 | /* Prevent the modem from triggering a call to ipa_setup(). This |
969 | * also ensures a modem-initiated setup that's underway completes. |
970 | */ |
971 | ipa_smp2p_irq_disable_setup(ipa); |
972 | |
973 | ret = pm_runtime_get_sync(dev); |
974 | if (WARN_ON(ret < 0)) |
975 | goto out_power_put; |
976 | |
977 | if (ipa->setup_complete) { |
978 | ret = ipa_modem_stop(ipa); |
979 | /* If starting or stopping is in progress, try once more */ |
980 | if (ret == -EBUSY) { |
981 | usleep_range(USEC_PER_MSEC, max: 2 * USEC_PER_MSEC); |
982 | ret = ipa_modem_stop(ipa); |
983 | } |
984 | if (ret) { |
985 | /* |
986 | * Not cleaning up here properly might also yield a |
987 | * crash later on. As the device is still unregistered |
988 | * in this case, this might even yield a crash later on. |
989 | */ |
990 | dev_err(dev, "Failed to stop modem (%pe), leaking resources\n" , |
991 | ERR_PTR(ret)); |
992 | return; |
993 | } |
994 | |
995 | ipa_teardown(ipa); |
996 | } |
997 | |
998 | ipa_deconfig(ipa); |
999 | out_power_put: |
1000 | pm_runtime_put_noidle(dev); |
1001 | ipa_smp2p_exit(ipa); |
1002 | ipa_table_exit(ipa); |
1003 | ipa_endpoint_exit(ipa); |
1004 | gsi_exit(gsi: &ipa->gsi); |
1005 | ipa_mem_exit(ipa); |
1006 | ipa_reg_exit(ipa); |
1007 | kfree(objp: ipa); |
1008 | ipa_power_exit(power); |
1009 | ipa_interrupt_exit(interrupt); |
1010 | |
1011 | dev_info(dev, "IPA driver removed" ); |
1012 | } |
1013 | |
1014 | static const struct attribute_group *ipa_attribute_groups[] = { |
1015 | &ipa_attribute_group, |
1016 | &ipa_feature_attribute_group, |
1017 | &ipa_endpoint_id_attribute_group, |
1018 | &ipa_modem_attribute_group, |
1019 | NULL, |
1020 | }; |
1021 | |
1022 | static struct platform_driver ipa_driver = { |
1023 | .probe = ipa_probe, |
1024 | .remove_new = ipa_remove, |
1025 | .shutdown = ipa_remove, |
1026 | .driver = { |
1027 | .name = "ipa" , |
1028 | .pm = &ipa_pm_ops, |
1029 | .of_match_table = ipa_match, |
1030 | .dev_groups = ipa_attribute_groups, |
1031 | }, |
1032 | }; |
1033 | |
1034 | module_platform_driver(ipa_driver); |
1035 | |
1036 | MODULE_LICENSE("GPL v2" ); |
1037 | MODULE_DESCRIPTION("Qualcomm IP Accelerator device driver" ); |
1038 | |