1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. |
4 | */ |
5 | |
6 | #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME |
7 | |
8 | #include <linux/atomic.h> |
9 | #include <linux/cpu_pm.h> |
10 | #include <linux/delay.h> |
11 | #include <linux/interrupt.h> |
12 | #include <linux/io.h> |
13 | #include <linux/iopoll.h> |
14 | #include <linux/kernel.h> |
15 | #include <linux/ktime.h> |
16 | #include <linux/list.h> |
17 | #include <linux/module.h> |
18 | #include <linux/notifier.h> |
19 | #include <linux/of.h> |
20 | #include <linux/of_irq.h> |
21 | #include <linux/of_platform.h> |
22 | #include <linux/platform_device.h> |
23 | #include <linux/pm_domain.h> |
24 | #include <linux/pm_runtime.h> |
25 | #include <linux/slab.h> |
26 | #include <linux/spinlock.h> |
27 | #include <linux/wait.h> |
28 | |
29 | #include <clocksource/arm_arch_timer.h> |
30 | #include <soc/qcom/cmd-db.h> |
31 | #include <soc/qcom/tcs.h> |
32 | #include <dt-bindings/soc/qcom,rpmh-rsc.h> |
33 | |
34 | #include "rpmh-internal.h" |
35 | |
36 | #define CREATE_TRACE_POINTS |
37 | #include "trace-rpmh.h" |
38 | |
39 | |
40 | #define RSC_DRV_ID 0 |
41 | |
42 | #define MAJOR_VER_MASK 0xFF |
43 | #define MAJOR_VER_SHIFT 16 |
44 | #define MINOR_VER_MASK 0xFF |
45 | #define MINOR_VER_SHIFT 8 |
46 | |
47 | enum { |
48 | RSC_DRV_TCS_OFFSET, |
49 | RSC_DRV_CMD_OFFSET, |
50 | DRV_SOLVER_CONFIG, |
51 | DRV_PRNT_CHLD_CONFIG, |
52 | RSC_DRV_IRQ_ENABLE, |
53 | RSC_DRV_IRQ_STATUS, |
54 | RSC_DRV_IRQ_CLEAR, |
55 | RSC_DRV_CMD_WAIT_FOR_CMPL, |
56 | RSC_DRV_CONTROL, |
57 | RSC_DRV_STATUS, |
58 | RSC_DRV_CMD_ENABLE, |
59 | RSC_DRV_CMD_MSGID, |
60 | RSC_DRV_CMD_ADDR, |
61 | RSC_DRV_CMD_DATA, |
62 | RSC_DRV_CMD_STATUS, |
63 | RSC_DRV_CMD_RESP_DATA, |
64 | }; |
65 | |
66 | /* DRV HW Solver Configuration Information Register */ |
67 | #define DRV_HW_SOLVER_MASK 1 |
68 | #define DRV_HW_SOLVER_SHIFT 24 |
69 | |
70 | /* DRV TCS Configuration Information Register */ |
71 | #define DRV_NUM_TCS_MASK 0x3F |
72 | #define DRV_NUM_TCS_SHIFT 6 |
73 | #define DRV_NCPT_MASK 0x1F |
74 | #define DRV_NCPT_SHIFT 27 |
75 | |
76 | /* Offsets for CONTROL TCS Registers */ |
77 | #define RSC_DRV_CTL_TCS_DATA_HI 0x38 |
78 | #define RSC_DRV_CTL_TCS_DATA_HI_MASK 0xFFFFFF |
79 | #define RSC_DRV_CTL_TCS_DATA_HI_VALID BIT(31) |
80 | #define RSC_DRV_CTL_TCS_DATA_LO 0x40 |
81 | #define RSC_DRV_CTL_TCS_DATA_LO_MASK 0xFFFFFFFF |
82 | #define RSC_DRV_CTL_TCS_DATA_SIZE 32 |
83 | |
84 | #define TCS_AMC_MODE_ENABLE BIT(16) |
85 | #define TCS_AMC_MODE_TRIGGER BIT(24) |
86 | |
87 | /* TCS CMD register bit mask */ |
88 | #define CMD_MSGID_LEN 8 |
89 | #define CMD_MSGID_RESP_REQ BIT(8) |
90 | #define CMD_MSGID_WRITE BIT(16) |
91 | #define CMD_STATUS_ISSUED BIT(8) |
92 | #define CMD_STATUS_COMPL BIT(16) |
93 | |
94 | /* |
95 | * Here's a high level overview of how all the registers in RPMH work |
96 | * together: |
97 | * |
98 | * - The main rpmh-rsc address is the base of a register space that can |
99 | * be used to find overall configuration of the hardware |
100 | * (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register |
101 | * space are all the TCS blocks. The offset of the TCS blocks is |
102 | * specified in the device tree by "qcom,tcs-offset" and used to |
103 | * compute tcs_base. |
104 | * - TCS blocks come one after another. Type, count, and order are |
105 | * specified by the device tree as "qcom,tcs-config". |
106 | * - Each TCS block has some registers, then space for up to 16 commands. |
107 | * Note that though address space is reserved for 16 commands, fewer |
108 | * might be present. See ncpt (num cmds per TCS). |
109 | * |
110 | * Here's a picture: |
111 | * |
112 | * +---------------------------------------------------+ |
113 | * |RSC | |
114 | * | ctrl | |
115 | * | | |
116 | * | Drvs: | |
117 | * | +-----------------------------------------------+ | |
118 | * | |DRV0 | | |
119 | * | | ctrl/config | | |
120 | * | | IRQ | | |
121 | * | | | | |
122 | * | | TCSes: | | |
123 | * | | +------------------------------------------+ | | |
124 | * | | |TCS0 | | | | | | | | | | | | | | | |
125 | * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | |
126 | * | | | | | | | | | | | | | | | | | | |
127 | * | | +------------------------------------------+ | | |
128 | * | | +------------------------------------------+ | | |
129 | * | | |TCS1 | | | | | | | | | | | | | | | |
130 | * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | |
131 | * | | | | | | | | | | | | | | | | | | |
132 | * | | +------------------------------------------+ | | |
133 | * | | +------------------------------------------+ | | |
134 | * | | |TCS2 | | | | | | | | | | | | | | | |
135 | * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | |
136 | * | | | | | | | | | | | | | | | | | | |
137 | * | | +------------------------------------------+ | | |
138 | * | | ...... | | |
139 | * | +-----------------------------------------------+ | |
140 | * | +-----------------------------------------------+ | |
141 | * | |DRV1 | | |
142 | * | | (same as DRV0) | | |
143 | * | +-----------------------------------------------+ | |
144 | * | ...... | |
145 | * +---------------------------------------------------+ |
146 | */ |
147 | |
148 | #define USECS_TO_CYCLES(time_usecs) \ |
149 | xloops_to_cycles((time_usecs) * 0x10C7UL) |
150 | |
151 | static inline unsigned long xloops_to_cycles(u64 xloops) |
152 | { |
153 | return (xloops * loops_per_jiffy * HZ) >> 32; |
154 | } |
155 | |
156 | static u32 rpmh_rsc_reg_offset_ver_2_7[] = { |
157 | [RSC_DRV_TCS_OFFSET] = 672, |
158 | [RSC_DRV_CMD_OFFSET] = 20, |
159 | [DRV_SOLVER_CONFIG] = 0x04, |
160 | [DRV_PRNT_CHLD_CONFIG] = 0x0C, |
161 | [RSC_DRV_IRQ_ENABLE] = 0x00, |
162 | [RSC_DRV_IRQ_STATUS] = 0x04, |
163 | [RSC_DRV_IRQ_CLEAR] = 0x08, |
164 | [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x10, |
165 | [RSC_DRV_CONTROL] = 0x14, |
166 | [RSC_DRV_STATUS] = 0x18, |
167 | [RSC_DRV_CMD_ENABLE] = 0x1C, |
168 | [RSC_DRV_CMD_MSGID] = 0x30, |
169 | [RSC_DRV_CMD_ADDR] = 0x34, |
170 | [RSC_DRV_CMD_DATA] = 0x38, |
171 | [RSC_DRV_CMD_STATUS] = 0x3C, |
172 | [RSC_DRV_CMD_RESP_DATA] = 0x40, |
173 | }; |
174 | |
175 | static u32 rpmh_rsc_reg_offset_ver_3_0[] = { |
176 | [RSC_DRV_TCS_OFFSET] = 672, |
177 | [RSC_DRV_CMD_OFFSET] = 24, |
178 | [DRV_SOLVER_CONFIG] = 0x04, |
179 | [DRV_PRNT_CHLD_CONFIG] = 0x0C, |
180 | [RSC_DRV_IRQ_ENABLE] = 0x00, |
181 | [RSC_DRV_IRQ_STATUS] = 0x04, |
182 | [RSC_DRV_IRQ_CLEAR] = 0x08, |
183 | [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x20, |
184 | [RSC_DRV_CONTROL] = 0x24, |
185 | [RSC_DRV_STATUS] = 0x28, |
186 | [RSC_DRV_CMD_ENABLE] = 0x2C, |
187 | [RSC_DRV_CMD_MSGID] = 0x34, |
188 | [RSC_DRV_CMD_ADDR] = 0x38, |
189 | [RSC_DRV_CMD_DATA] = 0x3C, |
190 | [RSC_DRV_CMD_STATUS] = 0x40, |
191 | [RSC_DRV_CMD_RESP_DATA] = 0x44, |
192 | }; |
193 | |
194 | static inline void __iomem * |
195 | tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id) |
196 | { |
197 | return drv->tcs_base + drv->regs[RSC_DRV_TCS_OFFSET] * tcs_id + reg; |
198 | } |
199 | |
200 | static inline void __iomem * |
201 | tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) |
202 | { |
203 | return tcs_reg_addr(drv, reg, tcs_id) + drv->regs[RSC_DRV_CMD_OFFSET] * cmd_id; |
204 | } |
205 | |
206 | static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, |
207 | int cmd_id) |
208 | { |
209 | return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); |
210 | } |
211 | |
212 | static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id) |
213 | { |
214 | return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id)); |
215 | } |
216 | |
217 | static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, |
218 | int cmd_id, u32 data) |
219 | { |
220 | writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); |
221 | } |
222 | |
223 | static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id, |
224 | u32 data) |
225 | { |
226 | writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id)); |
227 | } |
228 | |
229 | static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id, |
230 | u32 data) |
231 | { |
232 | int i; |
233 | |
234 | writel(val: data, addr: tcs_reg_addr(drv, reg, tcs_id)); |
235 | |
236 | /* |
237 | * Wait until we read back the same value. Use a counter rather than |
238 | * ktime for timeout since this may be called after timekeeping stops. |
239 | */ |
240 | for (i = 0; i < USEC_PER_SEC; i++) { |
241 | if (readl(addr: tcs_reg_addr(drv, reg, tcs_id)) == data) |
242 | return; |
243 | udelay(1); |
244 | } |
245 | pr_err("%s: error writing %#x to %d:%#x\n" , drv->name, |
246 | data, tcs_id, reg); |
247 | } |
248 | |
249 | /** |
250 | * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake). |
251 | * @drv: The RSC controller. |
252 | * @type: SLEEP_TCS or WAKE_TCS |
253 | * |
254 | * This will clear the "slots" variable of the given tcs_group and also |
255 | * tell the hardware to forget about all entries. |
256 | * |
257 | * The caller must ensure that no other RPMH actions are happening when this |
258 | * function is called, since otherwise the device may immediately become |
259 | * used again even before this function exits. |
260 | */ |
261 | static void tcs_invalidate(struct rsc_drv *drv, int type) |
262 | { |
263 | int m; |
264 | struct tcs_group *tcs = &drv->tcs[type]; |
265 | |
266 | /* Caller ensures nobody else is running so no lock */ |
267 | if (bitmap_empty(src: tcs->slots, MAX_TCS_SLOTS)) |
268 | return; |
269 | |
270 | for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) |
271 | write_tcs_reg_sync(drv, reg: drv->regs[RSC_DRV_CMD_ENABLE], tcs_id: m, data: 0); |
272 | |
273 | bitmap_zero(dst: tcs->slots, MAX_TCS_SLOTS); |
274 | } |
275 | |
276 | /** |
277 | * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes. |
278 | * @drv: The RSC controller. |
279 | * |
280 | * The caller must ensure that no other RPMH actions are happening when this |
281 | * function is called, since otherwise the device may immediately become |
282 | * used again even before this function exits. |
283 | */ |
284 | void rpmh_rsc_invalidate(struct rsc_drv *drv) |
285 | { |
286 | tcs_invalidate(drv, SLEEP_TCS); |
287 | tcs_invalidate(drv, WAKE_TCS); |
288 | } |
289 | |
290 | /** |
291 | * get_tcs_for_msg() - Get the tcs_group used to send the given message. |
292 | * @drv: The RSC controller. |
293 | * @msg: The message we want to send. |
294 | * |
295 | * This is normally pretty straightforward except if we are trying to send |
296 | * an ACTIVE_ONLY message but don't have any active_only TCSes. |
297 | * |
298 | * Return: A pointer to a tcs_group or an ERR_PTR. |
299 | */ |
300 | static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, |
301 | const struct tcs_request *msg) |
302 | { |
303 | int type; |
304 | struct tcs_group *tcs; |
305 | |
306 | switch (msg->state) { |
307 | case RPMH_ACTIVE_ONLY_STATE: |
308 | type = ACTIVE_TCS; |
309 | break; |
310 | case RPMH_WAKE_ONLY_STATE: |
311 | type = WAKE_TCS; |
312 | break; |
313 | case RPMH_SLEEP_STATE: |
314 | type = SLEEP_TCS; |
315 | break; |
316 | default: |
317 | return ERR_PTR(error: -EINVAL); |
318 | } |
319 | |
320 | /* |
321 | * If we are making an active request on a RSC that does not have a |
322 | * dedicated TCS for active state use, then re-purpose a wake TCS to |
323 | * send active votes. This is safe because we ensure any active-only |
324 | * transfers have finished before we use it (maybe by running from |
325 | * the last CPU in PM code). |
326 | */ |
327 | tcs = &drv->tcs[type]; |
328 | if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) |
329 | tcs = &drv->tcs[WAKE_TCS]; |
330 | |
331 | return tcs; |
332 | } |
333 | |
334 | /** |
335 | * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS. |
336 | * @drv: The RSC controller. |
337 | * @tcs_id: The global ID of this TCS. |
338 | * |
339 | * For ACTIVE_ONLY transfers we want to call back into the client when the |
340 | * transfer finishes. To do this we need the "request" that the client |
341 | * originally provided us. This function grabs the request that we stashed |
342 | * when we started the transfer. |
343 | * |
344 | * This only makes sense for ACTIVE_ONLY transfers since those are the only |
345 | * ones we track sending (the only ones we enable interrupts for and the only |
346 | * ones we call back to the client for). |
347 | * |
348 | * Return: The stashed request. |
349 | */ |
350 | static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv, |
351 | int tcs_id) |
352 | { |
353 | struct tcs_group *tcs; |
354 | int i; |
355 | |
356 | for (i = 0; i < TCS_TYPE_NR; i++) { |
357 | tcs = &drv->tcs[i]; |
358 | if (tcs->mask & BIT(tcs_id)) |
359 | return tcs->req[tcs_id - tcs->offset]; |
360 | } |
361 | |
362 | return NULL; |
363 | } |
364 | |
365 | /** |
366 | * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS |
367 | * @drv: The controller. |
368 | * @tcs_id: The global ID of this TCS. |
369 | * @trigger: If true then untrigger/retrigger. If false then just untrigger. |
370 | * |
371 | * In the normal case we only ever call with "trigger=true" to start a |
372 | * transfer. That will un-trigger/disable the TCS from the last transfer |
373 | * then trigger/enable for this transfer. |
374 | * |
375 | * If we borrowed a wake TCS for an active-only transfer we'll also call |
376 | * this function with "trigger=false" to just do the un-trigger/disable |
377 | * before using the TCS for wake purposes again. |
378 | * |
379 | * Note that the AP is only in charge of triggering active-only transfers. |
380 | * The AP never triggers sleep/wake values using this function. |
381 | */ |
382 | static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger) |
383 | { |
384 | u32 enable; |
385 | u32 reg = drv->regs[RSC_DRV_CONTROL]; |
386 | |
387 | /* |
388 | * HW req: Clear the DRV_CONTROL and enable TCS again |
389 | * While clearing ensure that the AMC mode trigger is cleared |
390 | * and then the mode enable is cleared. |
391 | */ |
392 | enable = read_tcs_reg(drv, reg, tcs_id); |
393 | enable &= ~TCS_AMC_MODE_TRIGGER; |
394 | write_tcs_reg_sync(drv, reg, tcs_id, data: enable); |
395 | enable &= ~TCS_AMC_MODE_ENABLE; |
396 | write_tcs_reg_sync(drv, reg, tcs_id, data: enable); |
397 | |
398 | if (trigger) { |
399 | /* Enable the AMC mode on the TCS and then trigger the TCS */ |
400 | enable = TCS_AMC_MODE_ENABLE; |
401 | write_tcs_reg_sync(drv, reg, tcs_id, data: enable); |
402 | enable |= TCS_AMC_MODE_TRIGGER; |
403 | write_tcs_reg(drv, reg, tcs_id, data: enable); |
404 | } |
405 | } |
406 | |
407 | /** |
408 | * enable_tcs_irq() - Enable or disable interrupts on the given TCS. |
409 | * @drv: The controller. |
410 | * @tcs_id: The global ID of this TCS. |
411 | * @enable: If true then enable; if false then disable |
412 | * |
413 | * We only ever call this when we borrow a wake TCS for an active-only |
414 | * transfer. For active-only TCSes interrupts are always left enabled. |
415 | */ |
416 | static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) |
417 | { |
418 | u32 data; |
419 | u32 reg = drv->regs[RSC_DRV_IRQ_ENABLE]; |
420 | |
421 | data = readl_relaxed(drv->tcs_base + reg); |
422 | if (enable) |
423 | data |= BIT(tcs_id); |
424 | else |
425 | data &= ~BIT(tcs_id); |
426 | writel_relaxed(data, drv->tcs_base + reg); |
427 | } |
428 | |
429 | /** |
430 | * tcs_tx_done() - TX Done interrupt handler. |
431 | * @irq: The IRQ number (ignored). |
432 | * @p: Pointer to "struct rsc_drv". |
433 | * |
434 | * Called for ACTIVE_ONLY transfers (those are the only ones we enable the |
435 | * IRQ for) when a transfer is done. |
436 | * |
437 | * Return: IRQ_HANDLED |
438 | */ |
439 | static irqreturn_t tcs_tx_done(int irq, void *p) |
440 | { |
441 | struct rsc_drv *drv = p; |
442 | int i; |
443 | unsigned long irq_status; |
444 | const struct tcs_request *req; |
445 | |
446 | irq_status = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]); |
447 | |
448 | for_each_set_bit(i, &irq_status, BITS_PER_TYPE(u32)) { |
449 | req = get_req_from_tcs(drv, tcs_id: i); |
450 | if (WARN_ON(!req)) |
451 | goto skip; |
452 | |
453 | trace_rpmh_tx_done(d: drv, m: i, r: req); |
454 | |
455 | /* |
456 | * If wake tcs was re-purposed for sending active |
457 | * votes, clear AMC trigger & enable modes and |
458 | * disable interrupt for this TCS |
459 | */ |
460 | if (!drv->tcs[ACTIVE_TCS].num_tcs) |
461 | __tcs_set_trigger(drv, tcs_id: i, trigger: false); |
462 | skip: |
463 | /* Reclaim the TCS */ |
464 | write_tcs_reg(drv, reg: drv->regs[RSC_DRV_CMD_ENABLE], tcs_id: i, data: 0); |
465 | writel_relaxed(BIT(i), drv->tcs_base + drv->regs[RSC_DRV_IRQ_CLEAR]); |
466 | spin_lock(lock: &drv->lock); |
467 | clear_bit(nr: i, addr: drv->tcs_in_use); |
468 | /* |
469 | * Disable interrupt for WAKE TCS to avoid being |
470 | * spammed with interrupts coming when the solver |
471 | * sends its wake votes. |
472 | */ |
473 | if (!drv->tcs[ACTIVE_TCS].num_tcs) |
474 | enable_tcs_irq(drv, tcs_id: i, enable: false); |
475 | spin_unlock(lock: &drv->lock); |
476 | wake_up(&drv->tcs_wait); |
477 | if (req) |
478 | rpmh_tx_done(msg: req); |
479 | } |
480 | |
481 | return IRQ_HANDLED; |
482 | } |
483 | |
484 | /** |
485 | * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger. |
486 | * @drv: The controller. |
487 | * @tcs_id: The global ID of this TCS. |
488 | * @cmd_id: The index within the TCS to start writing. |
489 | * @msg: The message we want to send, which will contain several addr/data |
490 | * pairs to program (but few enough that they all fit in one TCS). |
491 | * |
492 | * This is used for all types of transfers (active, sleep, and wake). |
493 | */ |
494 | static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, |
495 | const struct tcs_request *msg) |
496 | { |
497 | u32 msgid; |
498 | u32 cmd_msgid = CMD_MSGID_LEN | CMD_MSGID_WRITE; |
499 | u32 cmd_enable = 0; |
500 | struct tcs_cmd *cmd; |
501 | int i, j; |
502 | |
503 | /* Convert all commands to RR when the request has wait_for_compl set */ |
504 | cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0; |
505 | |
506 | for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) { |
507 | cmd = &msg->cmds[i]; |
508 | cmd_enable |= BIT(j); |
509 | msgid = cmd_msgid; |
510 | /* |
511 | * Additionally, if the cmd->wait is set, make the command |
512 | * response reqd even if the overall request was fire-n-forget. |
513 | */ |
514 | msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0; |
515 | |
516 | write_tcs_cmd(drv, reg: drv->regs[RSC_DRV_CMD_MSGID], tcs_id, cmd_id: j, data: msgid); |
517 | write_tcs_cmd(drv, reg: drv->regs[RSC_DRV_CMD_ADDR], tcs_id, cmd_id: j, data: cmd->addr); |
518 | write_tcs_cmd(drv, reg: drv->regs[RSC_DRV_CMD_DATA], tcs_id, cmd_id: j, data: cmd->data); |
519 | trace_rpmh_send_msg(d: drv, m: tcs_id, state: msg->state, n: j, h: msgid, c: cmd); |
520 | } |
521 | |
522 | cmd_enable |= read_tcs_reg(drv, reg: drv->regs[RSC_DRV_CMD_ENABLE], tcs_id); |
523 | write_tcs_reg(drv, reg: drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, data: cmd_enable); |
524 | } |
525 | |
526 | /** |
527 | * check_for_req_inflight() - Look to see if conflicting cmds are in flight. |
528 | * @drv: The controller. |
529 | * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers. |
530 | * @msg: The message we want to send, which will contain several addr/data |
531 | * pairs to program (but few enough that they all fit in one TCS). |
532 | * |
533 | * This will walk through the TCSes in the group and check if any of them |
534 | * appear to be sending to addresses referenced in the message. If it finds |
535 | * one it'll return -EBUSY. |
536 | * |
537 | * Only for use for active-only transfers. |
538 | * |
539 | * Must be called with the drv->lock held since that protects tcs_in_use. |
540 | * |
541 | * Return: 0 if nothing in flight or -EBUSY if we should try again later. |
542 | * The caller must re-enable interrupts between tries since that's |
543 | * the only way tcs_in_use will ever be updated and the only way |
544 | * RSC_DRV_CMD_ENABLE will ever be cleared. |
545 | */ |
546 | static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, |
547 | const struct tcs_request *msg) |
548 | { |
549 | unsigned long curr_enabled; |
550 | u32 addr; |
551 | int j, k; |
552 | int i = tcs->offset; |
553 | |
554 | for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) { |
555 | curr_enabled = read_tcs_reg(drv, reg: drv->regs[RSC_DRV_CMD_ENABLE], tcs_id: i); |
556 | |
557 | for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) { |
558 | addr = read_tcs_cmd(drv, reg: drv->regs[RSC_DRV_CMD_ADDR], tcs_id: i, cmd_id: j); |
559 | for (k = 0; k < msg->num_cmds; k++) { |
560 | if (addr == msg->cmds[k].addr) |
561 | return -EBUSY; |
562 | } |
563 | } |
564 | } |
565 | |
566 | return 0; |
567 | } |
568 | |
569 | /** |
570 | * find_free_tcs() - Find free tcs in the given tcs_group; only for active. |
571 | * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if |
572 | * we borrowed it because there are zero active-only ones). |
573 | * |
574 | * Must be called with the drv->lock held since that protects tcs_in_use. |
575 | * |
576 | * Return: The first tcs that's free or -EBUSY if all in use. |
577 | */ |
578 | static int find_free_tcs(struct tcs_group *tcs) |
579 | { |
580 | const struct rsc_drv *drv = tcs->drv; |
581 | unsigned long i; |
582 | unsigned long max = tcs->offset + tcs->num_tcs; |
583 | |
584 | i = find_next_zero_bit(addr: drv->tcs_in_use, size: max, offset: tcs->offset); |
585 | if (i >= max) |
586 | return -EBUSY; |
587 | |
588 | return i; |
589 | } |
590 | |
591 | /** |
592 | * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active. |
593 | * @drv: The controller. |
594 | * @tcs: The tcs_group used for ACTIVE_ONLY transfers. |
595 | * @msg: The data to be sent. |
596 | * |
597 | * Claims a tcs in the given tcs_group while making sure that no existing cmd |
598 | * is in flight that would conflict with the one in @msg. |
599 | * |
600 | * Context: Must be called with the drv->lock held since that protects |
601 | * tcs_in_use. |
602 | * |
603 | * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight |
604 | * or the tcs_group is full. |
605 | */ |
606 | static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs, |
607 | const struct tcs_request *msg) |
608 | { |
609 | int ret; |
610 | |
611 | /* |
612 | * The h/w does not like if we send a request to the same address, |
613 | * when one is already in-flight or being processed. |
614 | */ |
615 | ret = check_for_req_inflight(drv, tcs, msg); |
616 | if (ret) |
617 | return ret; |
618 | |
619 | return find_free_tcs(tcs); |
620 | } |
621 | |
622 | /** |
623 | * rpmh_rsc_send_data() - Write / trigger active-only message. |
624 | * @drv: The controller. |
625 | * @msg: The data to be sent. |
626 | * |
627 | * NOTES: |
628 | * - This is only used for "ACTIVE_ONLY" since the limitations of this |
629 | * function don't make sense for sleep/wake cases. |
630 | * - To do the transfer, we will grab a whole TCS for ourselves--we don't |
631 | * try to share. If there are none available we'll wait indefinitely |
632 | * for a free one. |
633 | * - This function will not wait for the commands to be finished, only for |
634 | * data to be programmed into the RPMh. See rpmh_tx_done() which will |
635 | * be called when the transfer is fully complete. |
636 | * - This function must be called with interrupts enabled. If the hardware |
637 | * is busy doing someone else's transfer we need that transfer to fully |
638 | * finish so that we can have the hardware, and to fully finish it needs |
639 | * the interrupt handler to run. If the interrupts is set to run on the |
640 | * active CPU this can never happen if interrupts are disabled. |
641 | * |
642 | * Return: 0 on success, -EINVAL on error. |
643 | */ |
644 | int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg) |
645 | { |
646 | struct tcs_group *tcs; |
647 | int tcs_id; |
648 | unsigned long flags; |
649 | |
650 | tcs = get_tcs_for_msg(drv, msg); |
651 | if (IS_ERR(ptr: tcs)) |
652 | return PTR_ERR(ptr: tcs); |
653 | |
654 | spin_lock_irqsave(&drv->lock, flags); |
655 | |
656 | /* Wait forever for a free tcs. It better be there eventually! */ |
657 | wait_event_lock_irq(drv->tcs_wait, |
658 | (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0, |
659 | drv->lock); |
660 | |
661 | tcs->req[tcs_id - tcs->offset] = msg; |
662 | set_bit(nr: tcs_id, addr: drv->tcs_in_use); |
663 | if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) { |
664 | /* |
665 | * Clear previously programmed WAKE commands in selected |
666 | * repurposed TCS to avoid triggering them. tcs->slots will be |
667 | * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate() |
668 | */ |
669 | write_tcs_reg_sync(drv, reg: drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, data: 0); |
670 | enable_tcs_irq(drv, tcs_id, enable: true); |
671 | } |
672 | spin_unlock_irqrestore(lock: &drv->lock, flags); |
673 | |
674 | /* |
675 | * These two can be done after the lock is released because: |
676 | * - We marked "tcs_in_use" under lock. |
677 | * - Once "tcs_in_use" has been marked nobody else could be writing |
678 | * to these registers until the interrupt goes off. |
679 | * - The interrupt can't go off until we trigger w/ the last line |
680 | * of __tcs_set_trigger() below. |
681 | */ |
682 | __tcs_buffer_write(drv, tcs_id, cmd_id: 0, msg); |
683 | __tcs_set_trigger(drv, tcs_id, trigger: true); |
684 | |
685 | return 0; |
686 | } |
687 | |
688 | /** |
689 | * find_slots() - Find a place to write the given message. |
690 | * @tcs: The tcs group to search. |
691 | * @msg: The message we want to find room for. |
692 | * @tcs_id: If we return 0 from the function, we return the global ID of the |
693 | * TCS to write to here. |
694 | * @cmd_id: If we return 0 from the function, we return the index of |
695 | * the command array of the returned TCS where the client should |
696 | * start writing the message. |
697 | * |
698 | * Only for use on sleep/wake TCSes since those are the only ones we maintain |
699 | * tcs->slots for. |
700 | * |
701 | * Return: -ENOMEM if there was no room, else 0. |
702 | */ |
703 | static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, |
704 | int *tcs_id, int *cmd_id) |
705 | { |
706 | int slot, offset; |
707 | int i = 0; |
708 | |
709 | /* Do over, until we can fit the full payload in a single TCS */ |
710 | do { |
711 | slot = bitmap_find_next_zero_area(map: tcs->slots, MAX_TCS_SLOTS, |
712 | start: i, nr: msg->num_cmds, align_mask: 0); |
713 | if (slot >= tcs->num_tcs * tcs->ncpt) |
714 | return -ENOMEM; |
715 | i += tcs->ncpt; |
716 | } while (slot + msg->num_cmds - 1 >= i); |
717 | |
718 | bitmap_set(map: tcs->slots, start: slot, nbits: msg->num_cmds); |
719 | |
720 | offset = slot / tcs->ncpt; |
721 | *tcs_id = offset + tcs->offset; |
722 | *cmd_id = slot % tcs->ncpt; |
723 | |
724 | return 0; |
725 | } |
726 | |
727 | /** |
728 | * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger. |
729 | * @drv: The controller. |
730 | * @msg: The data to be written to the controller. |
731 | * |
732 | * This should only be called for sleep/wake state, never active-only |
733 | * state. |
734 | * |
735 | * The caller must ensure that no other RPMH actions are happening and the |
736 | * controller is idle when this function is called since it runs lockless. |
737 | * |
738 | * Return: 0 if no error; else -error. |
739 | */ |
740 | int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg) |
741 | { |
742 | struct tcs_group *tcs; |
743 | int tcs_id = 0, cmd_id = 0; |
744 | int ret; |
745 | |
746 | tcs = get_tcs_for_msg(drv, msg); |
747 | if (IS_ERR(ptr: tcs)) |
748 | return PTR_ERR(ptr: tcs); |
749 | |
750 | /* find the TCS id and the command in the TCS to write to */ |
751 | ret = find_slots(tcs, msg, tcs_id: &tcs_id, cmd_id: &cmd_id); |
752 | if (!ret) |
753 | __tcs_buffer_write(drv, tcs_id, cmd_id, msg); |
754 | |
755 | return ret; |
756 | } |
757 | |
758 | /** |
759 | * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy. |
760 | * @drv: The controller |
761 | * |
762 | * Checks if any of the AMCs are busy in handling ACTIVE sets. |
763 | * This is called from the last cpu powering down before flushing |
764 | * SLEEP and WAKE sets. If AMCs are busy, controller can not enter |
765 | * power collapse, so deny from the last cpu's pm notification. |
766 | * |
767 | * Context: Must be called with the drv->lock held. |
768 | * |
769 | * Return: |
770 | * * False - AMCs are idle |
771 | * * True - AMCs are busy |
772 | */ |
773 | static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv) |
774 | { |
775 | unsigned long set; |
776 | const struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS]; |
777 | unsigned long max; |
778 | |
779 | /* |
780 | * If we made an active request on a RSC that does not have a |
781 | * dedicated TCS for active state use, then re-purposed wake TCSes |
782 | * should be checked for not busy, because we used wake TCSes for |
783 | * active requests in this case. |
784 | */ |
785 | if (!tcs->num_tcs) |
786 | tcs = &drv->tcs[WAKE_TCS]; |
787 | |
788 | max = tcs->offset + tcs->num_tcs; |
789 | set = find_next_bit(addr: drv->tcs_in_use, size: max, offset: tcs->offset); |
790 | |
791 | return set < max; |
792 | } |
793 | |
794 | /** |
795 | * rpmh_rsc_write_next_wakeup() - Write next wakeup in CONTROL_TCS. |
796 | * @drv: The controller |
797 | * |
798 | * Writes maximum wakeup cycles when called from suspend. |
799 | * Writes earliest hrtimer wakeup when called from idle. |
800 | */ |
801 | void rpmh_rsc_write_next_wakeup(struct rsc_drv *drv) |
802 | { |
803 | ktime_t now, wakeup; |
804 | u64 wakeup_us, wakeup_cycles = ~0; |
805 | u32 lo, hi; |
806 | |
807 | if (!drv->tcs[CONTROL_TCS].num_tcs || !drv->genpd_nb.notifier_call) |
808 | return; |
809 | |
810 | /* Set highest time when system (timekeeping) is suspended */ |
811 | if (system_state == SYSTEM_SUSPEND) |
812 | goto exit; |
813 | |
814 | /* Find the earliest hrtimer wakeup from online cpus */ |
815 | wakeup = dev_pm_genpd_get_next_hrtimer(dev: drv->dev); |
816 | |
817 | /* Find the relative wakeup in kernel time scale */ |
818 | now = ktime_get(); |
819 | wakeup = ktime_sub(wakeup, now); |
820 | wakeup_us = ktime_to_us(kt: wakeup); |
821 | |
822 | /* Convert the wakeup to arch timer scale */ |
823 | wakeup_cycles = USECS_TO_CYCLES(wakeup_us); |
824 | wakeup_cycles += arch_timer_read_counter(); |
825 | |
826 | exit: |
827 | lo = wakeup_cycles & RSC_DRV_CTL_TCS_DATA_LO_MASK; |
828 | hi = wakeup_cycles >> RSC_DRV_CTL_TCS_DATA_SIZE; |
829 | hi &= RSC_DRV_CTL_TCS_DATA_HI_MASK; |
830 | hi |= RSC_DRV_CTL_TCS_DATA_HI_VALID; |
831 | |
832 | writel_relaxed(lo, drv->base + RSC_DRV_CTL_TCS_DATA_LO); |
833 | writel_relaxed(hi, drv->base + RSC_DRV_CTL_TCS_DATA_HI); |
834 | } |
835 | |
836 | /** |
837 | * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy. |
838 | * @nfb: Pointer to the notifier block in struct rsc_drv. |
839 | * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT. |
840 | * @v: Unused |
841 | * |
842 | * This function is given to cpu_pm_register_notifier so we can be informed |
843 | * about when CPUs go down. When all CPUs go down we know no more active |
844 | * transfers will be started so we write sleep/wake sets. This function gets |
845 | * called from cpuidle code paths and also at system suspend time. |
846 | * |
847 | * If its last CPU going down and AMCs are not busy then writes cached sleep |
848 | * and wake messages to TCSes. The firmware then takes care of triggering |
849 | * them when entering deepest low power modes. |
850 | * |
851 | * Return: See cpu_pm_register_notifier() |
852 | */ |
853 | static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb, |
854 | unsigned long action, void *v) |
855 | { |
856 | struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm); |
857 | int ret = NOTIFY_OK; |
858 | int cpus_in_pm; |
859 | |
860 | switch (action) { |
861 | case CPU_PM_ENTER: |
862 | cpus_in_pm = atomic_inc_return(v: &drv->cpus_in_pm); |
863 | /* |
864 | * NOTE: comments for num_online_cpus() point out that it's |
865 | * only a snapshot so we need to be careful. It should be OK |
866 | * for us to use, though. It's important for us not to miss |
867 | * if we're the last CPU going down so it would only be a |
868 | * problem if a CPU went offline right after we did the check |
869 | * AND that CPU was not idle AND that CPU was the last non-idle |
870 | * CPU. That can't happen. CPUs would have to come out of idle |
871 | * before the CPU could go offline. |
872 | */ |
873 | if (cpus_in_pm < num_online_cpus()) |
874 | return NOTIFY_OK; |
875 | break; |
876 | case CPU_PM_ENTER_FAILED: |
877 | case CPU_PM_EXIT: |
878 | atomic_dec(v: &drv->cpus_in_pm); |
879 | return NOTIFY_OK; |
880 | default: |
881 | return NOTIFY_DONE; |
882 | } |
883 | |
884 | /* |
885 | * It's likely we're on the last CPU. Grab the drv->lock and write |
886 | * out the sleep/wake commands to RPMH hardware. Grabbing the lock |
887 | * means that if we race with another CPU coming up we are still |
888 | * guaranteed to be safe. If another CPU came up just after we checked |
889 | * and has grabbed the lock or started an active transfer then we'll |
890 | * notice we're busy and abort. If another CPU comes up after we start |
891 | * flushing it will be blocked from starting an active transfer until |
892 | * we're done flushing. If another CPU starts an active transfer after |
893 | * we release the lock we're still OK because we're no longer the last |
894 | * CPU. |
895 | */ |
896 | if (spin_trylock(lock: &drv->lock)) { |
897 | if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(ctrlr: &drv->client)) |
898 | ret = NOTIFY_BAD; |
899 | spin_unlock(lock: &drv->lock); |
900 | } else { |
901 | /* Another CPU must be up */ |
902 | return NOTIFY_OK; |
903 | } |
904 | |
905 | if (ret == NOTIFY_BAD) { |
906 | /* Double-check if we're here because someone else is up */ |
907 | if (cpus_in_pm < num_online_cpus()) |
908 | ret = NOTIFY_OK; |
909 | else |
910 | /* We won't be called w/ CPU_PM_ENTER_FAILED */ |
911 | atomic_dec(v: &drv->cpus_in_pm); |
912 | } |
913 | |
914 | return ret; |
915 | } |
916 | |
917 | /** |
918 | * rpmh_rsc_pd_callback() - Check if any of the AMCs are busy. |
919 | * @nfb: Pointer to the genpd notifier block in struct rsc_drv. |
920 | * @action: GENPD_NOTIFY_PRE_OFF, GENPD_NOTIFY_OFF, GENPD_NOTIFY_PRE_ON or GENPD_NOTIFY_ON. |
921 | * @v: Unused |
922 | * |
923 | * This function is given to dev_pm_genpd_add_notifier() so we can be informed |
924 | * about when cluster-pd is going down. When cluster go down we know no more active |
925 | * transfers will be started so we write sleep/wake sets. This function gets |
926 | * called from cpuidle code paths and also at system suspend time. |
927 | * |
928 | * If AMCs are not busy then writes cached sleep and wake messages to TCSes. |
929 | * The firmware then takes care of triggering them when entering deepest low power modes. |
930 | * |
931 | * Return: |
932 | * * NOTIFY_OK - success |
933 | * * NOTIFY_BAD - failure |
934 | */ |
935 | static int rpmh_rsc_pd_callback(struct notifier_block *nfb, |
936 | unsigned long action, void *v) |
937 | { |
938 | struct rsc_drv *drv = container_of(nfb, struct rsc_drv, genpd_nb); |
939 | |
940 | /* We don't need to lock as genpd on/off are serialized */ |
941 | if ((action == GENPD_NOTIFY_PRE_OFF) && |
942 | (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(ctrlr: &drv->client))) |
943 | return NOTIFY_BAD; |
944 | |
945 | return NOTIFY_OK; |
946 | } |
947 | |
948 | static int rpmh_rsc_pd_attach(struct rsc_drv *drv, struct device *dev) |
949 | { |
950 | int ret; |
951 | |
952 | pm_runtime_enable(dev); |
953 | drv->genpd_nb.notifier_call = rpmh_rsc_pd_callback; |
954 | ret = dev_pm_genpd_add_notifier(dev, nb: &drv->genpd_nb); |
955 | if (ret) |
956 | pm_runtime_disable(dev); |
957 | |
958 | return ret; |
959 | } |
960 | |
961 | static int rpmh_probe_tcs_config(struct platform_device *pdev, struct rsc_drv *drv) |
962 | { |
963 | struct tcs_type_config { |
964 | u32 type; |
965 | u32 n; |
966 | } tcs_cfg[TCS_TYPE_NR] = { { 0 } }; |
967 | struct device_node *dn = pdev->dev.of_node; |
968 | u32 config, max_tcs, ncpt, offset; |
969 | int i, ret, n, st = 0; |
970 | struct tcs_group *tcs; |
971 | |
972 | ret = of_property_read_u32(np: dn, propname: "qcom,tcs-offset" , out_value: &offset); |
973 | if (ret) |
974 | return ret; |
975 | drv->tcs_base = drv->base + offset; |
976 | |
977 | config = readl_relaxed(drv->base + drv->regs[DRV_PRNT_CHLD_CONFIG]); |
978 | |
979 | max_tcs = config; |
980 | max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); |
981 | max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); |
982 | |
983 | ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); |
984 | ncpt = ncpt >> DRV_NCPT_SHIFT; |
985 | |
986 | n = of_property_count_u32_elems(np: dn, propname: "qcom,tcs-config" ); |
987 | if (n != 2 * TCS_TYPE_NR) |
988 | return -EINVAL; |
989 | |
990 | for (i = 0; i < TCS_TYPE_NR; i++) { |
991 | ret = of_property_read_u32_index(np: dn, propname: "qcom,tcs-config" , |
992 | index: i * 2, out_value: &tcs_cfg[i].type); |
993 | if (ret) |
994 | return ret; |
995 | if (tcs_cfg[i].type >= TCS_TYPE_NR) |
996 | return -EINVAL; |
997 | |
998 | ret = of_property_read_u32_index(np: dn, propname: "qcom,tcs-config" , |
999 | index: i * 2 + 1, out_value: &tcs_cfg[i].n); |
1000 | if (ret) |
1001 | return ret; |
1002 | if (tcs_cfg[i].n > MAX_TCS_PER_TYPE) |
1003 | return -EINVAL; |
1004 | } |
1005 | |
1006 | for (i = 0; i < TCS_TYPE_NR; i++) { |
1007 | tcs = &drv->tcs[tcs_cfg[i].type]; |
1008 | if (tcs->drv) |
1009 | return -EINVAL; |
1010 | tcs->drv = drv; |
1011 | tcs->type = tcs_cfg[i].type; |
1012 | tcs->num_tcs = tcs_cfg[i].n; |
1013 | tcs->ncpt = ncpt; |
1014 | |
1015 | if (!tcs->num_tcs || tcs->type == CONTROL_TCS) |
1016 | continue; |
1017 | |
1018 | if (st + tcs->num_tcs > max_tcs || |
1019 | st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask)) |
1020 | return -EINVAL; |
1021 | |
1022 | tcs->mask = ((1 << tcs->num_tcs) - 1) << st; |
1023 | tcs->offset = st; |
1024 | st += tcs->num_tcs; |
1025 | } |
1026 | |
1027 | drv->num_tcs = st; |
1028 | |
1029 | return 0; |
1030 | } |
1031 | |
1032 | static int rpmh_rsc_probe(struct platform_device *pdev) |
1033 | { |
1034 | struct device_node *dn = pdev->dev.of_node; |
1035 | struct rsc_drv *drv; |
1036 | char drv_id[10] = {0}; |
1037 | int ret, irq; |
1038 | u32 solver_config; |
1039 | u32 rsc_id; |
1040 | |
1041 | /* |
1042 | * Even though RPMh doesn't directly use cmd-db, all of its children |
1043 | * do. To avoid adding this check to our children we'll do it now. |
1044 | */ |
1045 | ret = cmd_db_ready(); |
1046 | if (ret) { |
1047 | if (ret != -EPROBE_DEFER) |
1048 | dev_err(&pdev->dev, "Command DB not available (%d)\n" , |
1049 | ret); |
1050 | return ret; |
1051 | } |
1052 | |
1053 | drv = devm_kzalloc(dev: &pdev->dev, size: sizeof(*drv), GFP_KERNEL); |
1054 | if (!drv) |
1055 | return -ENOMEM; |
1056 | |
1057 | ret = of_property_read_u32(np: dn, propname: "qcom,drv-id" , out_value: &drv->id); |
1058 | if (ret) |
1059 | return ret; |
1060 | |
1061 | drv->name = of_get_property(node: dn, name: "label" , NULL); |
1062 | if (!drv->name) |
1063 | drv->name = dev_name(dev: &pdev->dev); |
1064 | |
1065 | snprintf(buf: drv_id, ARRAY_SIZE(drv_id), fmt: "drv-%d" , drv->id); |
1066 | drv->base = devm_platform_ioremap_resource_byname(pdev, name: drv_id); |
1067 | if (IS_ERR(ptr: drv->base)) |
1068 | return PTR_ERR(ptr: drv->base); |
1069 | |
1070 | rsc_id = readl_relaxed(drv->base + RSC_DRV_ID); |
1071 | drv->ver.major = rsc_id & (MAJOR_VER_MASK << MAJOR_VER_SHIFT); |
1072 | drv->ver.major >>= MAJOR_VER_SHIFT; |
1073 | drv->ver.minor = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT); |
1074 | drv->ver.minor >>= MINOR_VER_SHIFT; |
1075 | |
1076 | if (drv->ver.major == 3) |
1077 | drv->regs = rpmh_rsc_reg_offset_ver_3_0; |
1078 | else |
1079 | drv->regs = rpmh_rsc_reg_offset_ver_2_7; |
1080 | |
1081 | ret = rpmh_probe_tcs_config(pdev, drv); |
1082 | if (ret) |
1083 | return ret; |
1084 | |
1085 | spin_lock_init(&drv->lock); |
1086 | init_waitqueue_head(&drv->tcs_wait); |
1087 | bitmap_zero(dst: drv->tcs_in_use, MAX_TCS_NR); |
1088 | |
1089 | irq = platform_get_irq(pdev, drv->id); |
1090 | if (irq < 0) |
1091 | return irq; |
1092 | |
1093 | ret = devm_request_irq(dev: &pdev->dev, irq, handler: tcs_tx_done, |
1094 | IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, |
1095 | devname: drv->name, dev_id: drv); |
1096 | if (ret) |
1097 | return ret; |
1098 | |
1099 | /* |
1100 | * CPU PM/genpd notification are not required for controllers that support |
1101 | * 'HW solver' mode where they can be in autonomous mode executing low |
1102 | * power mode to power down. |
1103 | */ |
1104 | solver_config = readl_relaxed(drv->base + drv->regs[DRV_SOLVER_CONFIG]); |
1105 | solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT; |
1106 | solver_config = solver_config >> DRV_HW_SOLVER_SHIFT; |
1107 | if (!solver_config) { |
1108 | if (pdev->dev.pm_domain) { |
1109 | ret = rpmh_rsc_pd_attach(drv, dev: &pdev->dev); |
1110 | if (ret) |
1111 | return ret; |
1112 | } else { |
1113 | drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; |
1114 | cpu_pm_register_notifier(nb: &drv->rsc_pm); |
1115 | } |
1116 | } |
1117 | |
1118 | /* Enable the active TCS to send requests immediately */ |
1119 | writel_relaxed(drv->tcs[ACTIVE_TCS].mask, |
1120 | drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); |
1121 | |
1122 | spin_lock_init(&drv->client.cache_lock); |
1123 | INIT_LIST_HEAD(list: &drv->client.cache); |
1124 | INIT_LIST_HEAD(list: &drv->client.batch_cache); |
1125 | |
1126 | dev_set_drvdata(dev: &pdev->dev, data: drv); |
1127 | drv->dev = &pdev->dev; |
1128 | |
1129 | ret = devm_of_platform_populate(dev: &pdev->dev); |
1130 | if (ret && pdev->dev.pm_domain) { |
1131 | dev_pm_genpd_remove_notifier(dev: &pdev->dev); |
1132 | pm_runtime_disable(dev: &pdev->dev); |
1133 | } |
1134 | |
1135 | return ret; |
1136 | } |
1137 | |
1138 | static const struct of_device_id rpmh_drv_match[] = { |
1139 | { .compatible = "qcom,rpmh-rsc" , }, |
1140 | { } |
1141 | }; |
1142 | MODULE_DEVICE_TABLE(of, rpmh_drv_match); |
1143 | |
1144 | static struct platform_driver rpmh_driver = { |
1145 | .probe = rpmh_rsc_probe, |
1146 | .driver = { |
1147 | .name = "rpmh" , |
1148 | .of_match_table = rpmh_drv_match, |
1149 | .suppress_bind_attrs = true, |
1150 | }, |
1151 | }; |
1152 | |
1153 | static int __init rpmh_driver_init(void) |
1154 | { |
1155 | return platform_driver_register(&rpmh_driver); |
1156 | } |
1157 | arch_initcall(rpmh_driver_init); |
1158 | |
1159 | MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Driver" ); |
1160 | MODULE_LICENSE("GPL v2" ); |
1161 | |