1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
4 */
5
6#define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME
7
8#include <linux/atomic.h>
9#include <linux/cpu_pm.h>
10#include <linux/delay.h>
11#include <linux/interrupt.h>
12#include <linux/io.h>
13#include <linux/iopoll.h>
14#include <linux/kernel.h>
15#include <linux/ktime.h>
16#include <linux/list.h>
17#include <linux/module.h>
18#include <linux/notifier.h>
19#include <linux/of.h>
20#include <linux/of_irq.h>
21#include <linux/of_platform.h>
22#include <linux/platform_device.h>
23#include <linux/pm_domain.h>
24#include <linux/pm_runtime.h>
25#include <linux/slab.h>
26#include <linux/spinlock.h>
27#include <linux/wait.h>
28
29#include <clocksource/arm_arch_timer.h>
30#include <soc/qcom/cmd-db.h>
31#include <soc/qcom/tcs.h>
32#include <dt-bindings/soc/qcom,rpmh-rsc.h>
33
34#include "rpmh-internal.h"
35
36#define CREATE_TRACE_POINTS
37#include "trace-rpmh.h"
38
39
40#define RSC_DRV_ID 0
41
42#define MAJOR_VER_MASK 0xFF
43#define MAJOR_VER_SHIFT 16
44#define MINOR_VER_MASK 0xFF
45#define MINOR_VER_SHIFT 8
46
47enum {
48 RSC_DRV_TCS_OFFSET,
49 RSC_DRV_CMD_OFFSET,
50 DRV_SOLVER_CONFIG,
51 DRV_PRNT_CHLD_CONFIG,
52 RSC_DRV_IRQ_ENABLE,
53 RSC_DRV_IRQ_STATUS,
54 RSC_DRV_IRQ_CLEAR,
55 RSC_DRV_CMD_WAIT_FOR_CMPL,
56 RSC_DRV_CONTROL,
57 RSC_DRV_STATUS,
58 RSC_DRV_CMD_ENABLE,
59 RSC_DRV_CMD_MSGID,
60 RSC_DRV_CMD_ADDR,
61 RSC_DRV_CMD_DATA,
62 RSC_DRV_CMD_STATUS,
63 RSC_DRV_CMD_RESP_DATA,
64};
65
66/* DRV HW Solver Configuration Information Register */
67#define DRV_HW_SOLVER_MASK 1
68#define DRV_HW_SOLVER_SHIFT 24
69
70/* DRV TCS Configuration Information Register */
71#define DRV_NUM_TCS_MASK 0x3F
72#define DRV_NUM_TCS_SHIFT 6
73#define DRV_NCPT_MASK 0x1F
74#define DRV_NCPT_SHIFT 27
75
76/* Offsets for CONTROL TCS Registers */
77#define RSC_DRV_CTL_TCS_DATA_HI 0x38
78#define RSC_DRV_CTL_TCS_DATA_HI_MASK 0xFFFFFF
79#define RSC_DRV_CTL_TCS_DATA_HI_VALID BIT(31)
80#define RSC_DRV_CTL_TCS_DATA_LO 0x40
81#define RSC_DRV_CTL_TCS_DATA_LO_MASK 0xFFFFFFFF
82#define RSC_DRV_CTL_TCS_DATA_SIZE 32
83
84#define TCS_AMC_MODE_ENABLE BIT(16)
85#define TCS_AMC_MODE_TRIGGER BIT(24)
86
87/* TCS CMD register bit mask */
88#define CMD_MSGID_LEN 8
89#define CMD_MSGID_RESP_REQ BIT(8)
90#define CMD_MSGID_WRITE BIT(16)
91#define CMD_STATUS_ISSUED BIT(8)
92#define CMD_STATUS_COMPL BIT(16)
93
94/*
95 * Here's a high level overview of how all the registers in RPMH work
96 * together:
97 *
98 * - The main rpmh-rsc address is the base of a register space that can
99 * be used to find overall configuration of the hardware
100 * (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register
101 * space are all the TCS blocks. The offset of the TCS blocks is
102 * specified in the device tree by "qcom,tcs-offset" and used to
103 * compute tcs_base.
104 * - TCS blocks come one after another. Type, count, and order are
105 * specified by the device tree as "qcom,tcs-config".
106 * - Each TCS block has some registers, then space for up to 16 commands.
107 * Note that though address space is reserved for 16 commands, fewer
108 * might be present. See ncpt (num cmds per TCS).
109 *
110 * Here's a picture:
111 *
112 * +---------------------------------------------------+
113 * |RSC |
114 * | ctrl |
115 * | |
116 * | Drvs: |
117 * | +-----------------------------------------------+ |
118 * | |DRV0 | |
119 * | | ctrl/config | |
120 * | | IRQ | |
121 * | | | |
122 * | | TCSes: | |
123 * | | +------------------------------------------+ | |
124 * | | |TCS0 | | | | | | | | | | | | | | |
125 * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | |
126 * | | | | | | | | | | | | | | | | | |
127 * | | +------------------------------------------+ | |
128 * | | +------------------------------------------+ | |
129 * | | |TCS1 | | | | | | | | | | | | | | |
130 * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | |
131 * | | | | | | | | | | | | | | | | | |
132 * | | +------------------------------------------+ | |
133 * | | +------------------------------------------+ | |
134 * | | |TCS2 | | | | | | | | | | | | | | |
135 * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | |
136 * | | | | | | | | | | | | | | | | | |
137 * | | +------------------------------------------+ | |
138 * | | ...... | |
139 * | +-----------------------------------------------+ |
140 * | +-----------------------------------------------+ |
141 * | |DRV1 | |
142 * | | (same as DRV0) | |
143 * | +-----------------------------------------------+ |
144 * | ...... |
145 * +---------------------------------------------------+
146 */
147
148#define USECS_TO_CYCLES(time_usecs) \
149 xloops_to_cycles((time_usecs) * 0x10C7UL)
150
151static inline unsigned long xloops_to_cycles(u64 xloops)
152{
153 return (xloops * loops_per_jiffy * HZ) >> 32;
154}
155
156static u32 rpmh_rsc_reg_offset_ver_2_7[] = {
157 [RSC_DRV_TCS_OFFSET] = 672,
158 [RSC_DRV_CMD_OFFSET] = 20,
159 [DRV_SOLVER_CONFIG] = 0x04,
160 [DRV_PRNT_CHLD_CONFIG] = 0x0C,
161 [RSC_DRV_IRQ_ENABLE] = 0x00,
162 [RSC_DRV_IRQ_STATUS] = 0x04,
163 [RSC_DRV_IRQ_CLEAR] = 0x08,
164 [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x10,
165 [RSC_DRV_CONTROL] = 0x14,
166 [RSC_DRV_STATUS] = 0x18,
167 [RSC_DRV_CMD_ENABLE] = 0x1C,
168 [RSC_DRV_CMD_MSGID] = 0x30,
169 [RSC_DRV_CMD_ADDR] = 0x34,
170 [RSC_DRV_CMD_DATA] = 0x38,
171 [RSC_DRV_CMD_STATUS] = 0x3C,
172 [RSC_DRV_CMD_RESP_DATA] = 0x40,
173};
174
175static u32 rpmh_rsc_reg_offset_ver_3_0[] = {
176 [RSC_DRV_TCS_OFFSET] = 672,
177 [RSC_DRV_CMD_OFFSET] = 24,
178 [DRV_SOLVER_CONFIG] = 0x04,
179 [DRV_PRNT_CHLD_CONFIG] = 0x0C,
180 [RSC_DRV_IRQ_ENABLE] = 0x00,
181 [RSC_DRV_IRQ_STATUS] = 0x04,
182 [RSC_DRV_IRQ_CLEAR] = 0x08,
183 [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x20,
184 [RSC_DRV_CONTROL] = 0x24,
185 [RSC_DRV_STATUS] = 0x28,
186 [RSC_DRV_CMD_ENABLE] = 0x2C,
187 [RSC_DRV_CMD_MSGID] = 0x34,
188 [RSC_DRV_CMD_ADDR] = 0x38,
189 [RSC_DRV_CMD_DATA] = 0x3C,
190 [RSC_DRV_CMD_STATUS] = 0x40,
191 [RSC_DRV_CMD_RESP_DATA] = 0x44,
192};
193
194static inline void __iomem *
195tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id)
196{
197 return drv->tcs_base + drv->regs[RSC_DRV_TCS_OFFSET] * tcs_id + reg;
198}
199
200static inline void __iomem *
201tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id)
202{
203 return tcs_reg_addr(drv, reg, tcs_id) + drv->regs[RSC_DRV_CMD_OFFSET] * cmd_id;
204}
205
206static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
207 int cmd_id)
208{
209 return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
210}
211
212static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id)
213{
214 return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id));
215}
216
217static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
218 int cmd_id, u32 data)
219{
220 writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
221}
222
223static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id,
224 u32 data)
225{
226 writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id));
227}
228
229static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id,
230 u32 data)
231{
232 int i;
233
234 writel(val: data, addr: tcs_reg_addr(drv, reg, tcs_id));
235
236 /*
237 * Wait until we read back the same value. Use a counter rather than
238 * ktime for timeout since this may be called after timekeeping stops.
239 */
240 for (i = 0; i < USEC_PER_SEC; i++) {
241 if (readl(addr: tcs_reg_addr(drv, reg, tcs_id)) == data)
242 return;
243 udelay(1);
244 }
245 pr_err("%s: error writing %#x to %d:%#x\n", drv->name,
246 data, tcs_id, reg);
247}
248
249/**
250 * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake).
251 * @drv: The RSC controller.
252 * @type: SLEEP_TCS or WAKE_TCS
253 *
254 * This will clear the "slots" variable of the given tcs_group and also
255 * tell the hardware to forget about all entries.
256 *
257 * The caller must ensure that no other RPMH actions are happening when this
258 * function is called, since otherwise the device may immediately become
259 * used again even before this function exits.
260 */
261static void tcs_invalidate(struct rsc_drv *drv, int type)
262{
263 int m;
264 struct tcs_group *tcs = &drv->tcs[type];
265
266 /* Caller ensures nobody else is running so no lock */
267 if (bitmap_empty(src: tcs->slots, MAX_TCS_SLOTS))
268 return;
269
270 for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++)
271 write_tcs_reg_sync(drv, reg: drv->regs[RSC_DRV_CMD_ENABLE], tcs_id: m, data: 0);
272
273 bitmap_zero(dst: tcs->slots, MAX_TCS_SLOTS);
274}
275
276/**
277 * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes.
278 * @drv: The RSC controller.
279 *
280 * The caller must ensure that no other RPMH actions are happening when this
281 * function is called, since otherwise the device may immediately become
282 * used again even before this function exits.
283 */
284void rpmh_rsc_invalidate(struct rsc_drv *drv)
285{
286 tcs_invalidate(drv, SLEEP_TCS);
287 tcs_invalidate(drv, WAKE_TCS);
288}
289
290/**
291 * get_tcs_for_msg() - Get the tcs_group used to send the given message.
292 * @drv: The RSC controller.
293 * @msg: The message we want to send.
294 *
295 * This is normally pretty straightforward except if we are trying to send
296 * an ACTIVE_ONLY message but don't have any active_only TCSes.
297 *
298 * Return: A pointer to a tcs_group or an ERR_PTR.
299 */
300static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv,
301 const struct tcs_request *msg)
302{
303 int type;
304 struct tcs_group *tcs;
305
306 switch (msg->state) {
307 case RPMH_ACTIVE_ONLY_STATE:
308 type = ACTIVE_TCS;
309 break;
310 case RPMH_WAKE_ONLY_STATE:
311 type = WAKE_TCS;
312 break;
313 case RPMH_SLEEP_STATE:
314 type = SLEEP_TCS;
315 break;
316 default:
317 return ERR_PTR(error: -EINVAL);
318 }
319
320 /*
321 * If we are making an active request on a RSC that does not have a
322 * dedicated TCS for active state use, then re-purpose a wake TCS to
323 * send active votes. This is safe because we ensure any active-only
324 * transfers have finished before we use it (maybe by running from
325 * the last CPU in PM code).
326 */
327 tcs = &drv->tcs[type];
328 if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs)
329 tcs = &drv->tcs[WAKE_TCS];
330
331 return tcs;
332}
333
334/**
335 * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS.
336 * @drv: The RSC controller.
337 * @tcs_id: The global ID of this TCS.
338 *
339 * For ACTIVE_ONLY transfers we want to call back into the client when the
340 * transfer finishes. To do this we need the "request" that the client
341 * originally provided us. This function grabs the request that we stashed
342 * when we started the transfer.
343 *
344 * This only makes sense for ACTIVE_ONLY transfers since those are the only
345 * ones we track sending (the only ones we enable interrupts for and the only
346 * ones we call back to the client for).
347 *
348 * Return: The stashed request.
349 */
350static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv,
351 int tcs_id)
352{
353 struct tcs_group *tcs;
354 int i;
355
356 for (i = 0; i < TCS_TYPE_NR; i++) {
357 tcs = &drv->tcs[i];
358 if (tcs->mask & BIT(tcs_id))
359 return tcs->req[tcs_id - tcs->offset];
360 }
361
362 return NULL;
363}
364
365/**
366 * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS
367 * @drv: The controller.
368 * @tcs_id: The global ID of this TCS.
369 * @trigger: If true then untrigger/retrigger. If false then just untrigger.
370 *
371 * In the normal case we only ever call with "trigger=true" to start a
372 * transfer. That will un-trigger/disable the TCS from the last transfer
373 * then trigger/enable for this transfer.
374 *
375 * If we borrowed a wake TCS for an active-only transfer we'll also call
376 * this function with "trigger=false" to just do the un-trigger/disable
377 * before using the TCS for wake purposes again.
378 *
379 * Note that the AP is only in charge of triggering active-only transfers.
380 * The AP never triggers sleep/wake values using this function.
381 */
382static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger)
383{
384 u32 enable;
385 u32 reg = drv->regs[RSC_DRV_CONTROL];
386
387 /*
388 * HW req: Clear the DRV_CONTROL and enable TCS again
389 * While clearing ensure that the AMC mode trigger is cleared
390 * and then the mode enable is cleared.
391 */
392 enable = read_tcs_reg(drv, reg, tcs_id);
393 enable &= ~TCS_AMC_MODE_TRIGGER;
394 write_tcs_reg_sync(drv, reg, tcs_id, data: enable);
395 enable &= ~TCS_AMC_MODE_ENABLE;
396 write_tcs_reg_sync(drv, reg, tcs_id, data: enable);
397
398 if (trigger) {
399 /* Enable the AMC mode on the TCS and then trigger the TCS */
400 enable = TCS_AMC_MODE_ENABLE;
401 write_tcs_reg_sync(drv, reg, tcs_id, data: enable);
402 enable |= TCS_AMC_MODE_TRIGGER;
403 write_tcs_reg(drv, reg, tcs_id, data: enable);
404 }
405}
406
407/**
408 * enable_tcs_irq() - Enable or disable interrupts on the given TCS.
409 * @drv: The controller.
410 * @tcs_id: The global ID of this TCS.
411 * @enable: If true then enable; if false then disable
412 *
413 * We only ever call this when we borrow a wake TCS for an active-only
414 * transfer. For active-only TCSes interrupts are always left enabled.
415 */
416static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable)
417{
418 u32 data;
419 u32 reg = drv->regs[RSC_DRV_IRQ_ENABLE];
420
421 data = readl_relaxed(drv->tcs_base + reg);
422 if (enable)
423 data |= BIT(tcs_id);
424 else
425 data &= ~BIT(tcs_id);
426 writel_relaxed(data, drv->tcs_base + reg);
427}
428
429/**
430 * tcs_tx_done() - TX Done interrupt handler.
431 * @irq: The IRQ number (ignored).
432 * @p: Pointer to "struct rsc_drv".
433 *
434 * Called for ACTIVE_ONLY transfers (those are the only ones we enable the
435 * IRQ for) when a transfer is done.
436 *
437 * Return: IRQ_HANDLED
438 */
439static irqreturn_t tcs_tx_done(int irq, void *p)
440{
441 struct rsc_drv *drv = p;
442 int i;
443 unsigned long irq_status;
444 const struct tcs_request *req;
445
446 irq_status = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]);
447
448 for_each_set_bit(i, &irq_status, BITS_PER_TYPE(u32)) {
449 req = get_req_from_tcs(drv, tcs_id: i);
450 if (WARN_ON(!req))
451 goto skip;
452
453 trace_rpmh_tx_done(d: drv, m: i, r: req);
454
455 /*
456 * If wake tcs was re-purposed for sending active
457 * votes, clear AMC trigger & enable modes and
458 * disable interrupt for this TCS
459 */
460 if (!drv->tcs[ACTIVE_TCS].num_tcs)
461 __tcs_set_trigger(drv, tcs_id: i, trigger: false);
462skip:
463 /* Reclaim the TCS */
464 write_tcs_reg(drv, reg: drv->regs[RSC_DRV_CMD_ENABLE], tcs_id: i, data: 0);
465 writel_relaxed(BIT(i), drv->tcs_base + drv->regs[RSC_DRV_IRQ_CLEAR]);
466 spin_lock(lock: &drv->lock);
467 clear_bit(nr: i, addr: drv->tcs_in_use);
468 /*
469 * Disable interrupt for WAKE TCS to avoid being
470 * spammed with interrupts coming when the solver
471 * sends its wake votes.
472 */
473 if (!drv->tcs[ACTIVE_TCS].num_tcs)
474 enable_tcs_irq(drv, tcs_id: i, enable: false);
475 spin_unlock(lock: &drv->lock);
476 wake_up(&drv->tcs_wait);
477 if (req)
478 rpmh_tx_done(msg: req);
479 }
480
481 return IRQ_HANDLED;
482}
483
484/**
485 * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger.
486 * @drv: The controller.
487 * @tcs_id: The global ID of this TCS.
488 * @cmd_id: The index within the TCS to start writing.
489 * @msg: The message we want to send, which will contain several addr/data
490 * pairs to program (but few enough that they all fit in one TCS).
491 *
492 * This is used for all types of transfers (active, sleep, and wake).
493 */
494static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id,
495 const struct tcs_request *msg)
496{
497 u32 msgid;
498 u32 cmd_msgid = CMD_MSGID_LEN | CMD_MSGID_WRITE;
499 u32 cmd_enable = 0;
500 struct tcs_cmd *cmd;
501 int i, j;
502
503 /* Convert all commands to RR when the request has wait_for_compl set */
504 cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0;
505
506 for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) {
507 cmd = &msg->cmds[i];
508 cmd_enable |= BIT(j);
509 msgid = cmd_msgid;
510 /*
511 * Additionally, if the cmd->wait is set, make the command
512 * response reqd even if the overall request was fire-n-forget.
513 */
514 msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0;
515
516 write_tcs_cmd(drv, reg: drv->regs[RSC_DRV_CMD_MSGID], tcs_id, cmd_id: j, data: msgid);
517 write_tcs_cmd(drv, reg: drv->regs[RSC_DRV_CMD_ADDR], tcs_id, cmd_id: j, data: cmd->addr);
518 write_tcs_cmd(drv, reg: drv->regs[RSC_DRV_CMD_DATA], tcs_id, cmd_id: j, data: cmd->data);
519 trace_rpmh_send_msg(d: drv, m: tcs_id, state: msg->state, n: j, h: msgid, c: cmd);
520 }
521
522 cmd_enable |= read_tcs_reg(drv, reg: drv->regs[RSC_DRV_CMD_ENABLE], tcs_id);
523 write_tcs_reg(drv, reg: drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, data: cmd_enable);
524}
525
526/**
527 * check_for_req_inflight() - Look to see if conflicting cmds are in flight.
528 * @drv: The controller.
529 * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers.
530 * @msg: The message we want to send, which will contain several addr/data
531 * pairs to program (but few enough that they all fit in one TCS).
532 *
533 * This will walk through the TCSes in the group and check if any of them
534 * appear to be sending to addresses referenced in the message. If it finds
535 * one it'll return -EBUSY.
536 *
537 * Only for use for active-only transfers.
538 *
539 * Must be called with the drv->lock held since that protects tcs_in_use.
540 *
541 * Return: 0 if nothing in flight or -EBUSY if we should try again later.
542 * The caller must re-enable interrupts between tries since that's
543 * the only way tcs_in_use will ever be updated and the only way
544 * RSC_DRV_CMD_ENABLE will ever be cleared.
545 */
546static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs,
547 const struct tcs_request *msg)
548{
549 unsigned long curr_enabled;
550 u32 addr;
551 int j, k;
552 int i = tcs->offset;
553
554 for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) {
555 curr_enabled = read_tcs_reg(drv, reg: drv->regs[RSC_DRV_CMD_ENABLE], tcs_id: i);
556
557 for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) {
558 addr = read_tcs_cmd(drv, reg: drv->regs[RSC_DRV_CMD_ADDR], tcs_id: i, cmd_id: j);
559 for (k = 0; k < msg->num_cmds; k++) {
560 if (addr == msg->cmds[k].addr)
561 return -EBUSY;
562 }
563 }
564 }
565
566 return 0;
567}
568
569/**
570 * find_free_tcs() - Find free tcs in the given tcs_group; only for active.
571 * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if
572 * we borrowed it because there are zero active-only ones).
573 *
574 * Must be called with the drv->lock held since that protects tcs_in_use.
575 *
576 * Return: The first tcs that's free or -EBUSY if all in use.
577 */
578static int find_free_tcs(struct tcs_group *tcs)
579{
580 const struct rsc_drv *drv = tcs->drv;
581 unsigned long i;
582 unsigned long max = tcs->offset + tcs->num_tcs;
583
584 i = find_next_zero_bit(addr: drv->tcs_in_use, size: max, offset: tcs->offset);
585 if (i >= max)
586 return -EBUSY;
587
588 return i;
589}
590
591/**
592 * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active.
593 * @drv: The controller.
594 * @tcs: The tcs_group used for ACTIVE_ONLY transfers.
595 * @msg: The data to be sent.
596 *
597 * Claims a tcs in the given tcs_group while making sure that no existing cmd
598 * is in flight that would conflict with the one in @msg.
599 *
600 * Context: Must be called with the drv->lock held since that protects
601 * tcs_in_use.
602 *
603 * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight
604 * or the tcs_group is full.
605 */
606static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs,
607 const struct tcs_request *msg)
608{
609 int ret;
610
611 /*
612 * The h/w does not like if we send a request to the same address,
613 * when one is already in-flight or being processed.
614 */
615 ret = check_for_req_inflight(drv, tcs, msg);
616 if (ret)
617 return ret;
618
619 return find_free_tcs(tcs);
620}
621
622/**
623 * rpmh_rsc_send_data() - Write / trigger active-only message.
624 * @drv: The controller.
625 * @msg: The data to be sent.
626 *
627 * NOTES:
628 * - This is only used for "ACTIVE_ONLY" since the limitations of this
629 * function don't make sense for sleep/wake cases.
630 * - To do the transfer, we will grab a whole TCS for ourselves--we don't
631 * try to share. If there are none available we'll wait indefinitely
632 * for a free one.
633 * - This function will not wait for the commands to be finished, only for
634 * data to be programmed into the RPMh. See rpmh_tx_done() which will
635 * be called when the transfer is fully complete.
636 * - This function must be called with interrupts enabled. If the hardware
637 * is busy doing someone else's transfer we need that transfer to fully
638 * finish so that we can have the hardware, and to fully finish it needs
639 * the interrupt handler to run. If the interrupts is set to run on the
640 * active CPU this can never happen if interrupts are disabled.
641 *
642 * Return: 0 on success, -EINVAL on error.
643 */
644int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg)
645{
646 struct tcs_group *tcs;
647 int tcs_id;
648 unsigned long flags;
649
650 tcs = get_tcs_for_msg(drv, msg);
651 if (IS_ERR(ptr: tcs))
652 return PTR_ERR(ptr: tcs);
653
654 spin_lock_irqsave(&drv->lock, flags);
655
656 /* Wait forever for a free tcs. It better be there eventually! */
657 wait_event_lock_irq(drv->tcs_wait,
658 (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0,
659 drv->lock);
660
661 tcs->req[tcs_id - tcs->offset] = msg;
662 set_bit(nr: tcs_id, addr: drv->tcs_in_use);
663 if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) {
664 /*
665 * Clear previously programmed WAKE commands in selected
666 * repurposed TCS to avoid triggering them. tcs->slots will be
667 * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate()
668 */
669 write_tcs_reg_sync(drv, reg: drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, data: 0);
670 enable_tcs_irq(drv, tcs_id, enable: true);
671 }
672 spin_unlock_irqrestore(lock: &drv->lock, flags);
673
674 /*
675 * These two can be done after the lock is released because:
676 * - We marked "tcs_in_use" under lock.
677 * - Once "tcs_in_use" has been marked nobody else could be writing
678 * to these registers until the interrupt goes off.
679 * - The interrupt can't go off until we trigger w/ the last line
680 * of __tcs_set_trigger() below.
681 */
682 __tcs_buffer_write(drv, tcs_id, cmd_id: 0, msg);
683 __tcs_set_trigger(drv, tcs_id, trigger: true);
684
685 return 0;
686}
687
688/**
689 * find_slots() - Find a place to write the given message.
690 * @tcs: The tcs group to search.
691 * @msg: The message we want to find room for.
692 * @tcs_id: If we return 0 from the function, we return the global ID of the
693 * TCS to write to here.
694 * @cmd_id: If we return 0 from the function, we return the index of
695 * the command array of the returned TCS where the client should
696 * start writing the message.
697 *
698 * Only for use on sleep/wake TCSes since those are the only ones we maintain
699 * tcs->slots for.
700 *
701 * Return: -ENOMEM if there was no room, else 0.
702 */
703static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg,
704 int *tcs_id, int *cmd_id)
705{
706 int slot, offset;
707 int i = 0;
708
709 /* Do over, until we can fit the full payload in a single TCS */
710 do {
711 slot = bitmap_find_next_zero_area(map: tcs->slots, MAX_TCS_SLOTS,
712 start: i, nr: msg->num_cmds, align_mask: 0);
713 if (slot >= tcs->num_tcs * tcs->ncpt)
714 return -ENOMEM;
715 i += tcs->ncpt;
716 } while (slot + msg->num_cmds - 1 >= i);
717
718 bitmap_set(map: tcs->slots, start: slot, nbits: msg->num_cmds);
719
720 offset = slot / tcs->ncpt;
721 *tcs_id = offset + tcs->offset;
722 *cmd_id = slot % tcs->ncpt;
723
724 return 0;
725}
726
727/**
728 * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger.
729 * @drv: The controller.
730 * @msg: The data to be written to the controller.
731 *
732 * This should only be called for sleep/wake state, never active-only
733 * state.
734 *
735 * The caller must ensure that no other RPMH actions are happening and the
736 * controller is idle when this function is called since it runs lockless.
737 *
738 * Return: 0 if no error; else -error.
739 */
740int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg)
741{
742 struct tcs_group *tcs;
743 int tcs_id = 0, cmd_id = 0;
744 int ret;
745
746 tcs = get_tcs_for_msg(drv, msg);
747 if (IS_ERR(ptr: tcs))
748 return PTR_ERR(ptr: tcs);
749
750 /* find the TCS id and the command in the TCS to write to */
751 ret = find_slots(tcs, msg, tcs_id: &tcs_id, cmd_id: &cmd_id);
752 if (!ret)
753 __tcs_buffer_write(drv, tcs_id, cmd_id, msg);
754
755 return ret;
756}
757
758/**
759 * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy.
760 * @drv: The controller
761 *
762 * Checks if any of the AMCs are busy in handling ACTIVE sets.
763 * This is called from the last cpu powering down before flushing
764 * SLEEP and WAKE sets. If AMCs are busy, controller can not enter
765 * power collapse, so deny from the last cpu's pm notification.
766 *
767 * Context: Must be called with the drv->lock held.
768 *
769 * Return:
770 * * False - AMCs are idle
771 * * True - AMCs are busy
772 */
773static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv)
774{
775 unsigned long set;
776 const struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS];
777 unsigned long max;
778
779 /*
780 * If we made an active request on a RSC that does not have a
781 * dedicated TCS for active state use, then re-purposed wake TCSes
782 * should be checked for not busy, because we used wake TCSes for
783 * active requests in this case.
784 */
785 if (!tcs->num_tcs)
786 tcs = &drv->tcs[WAKE_TCS];
787
788 max = tcs->offset + tcs->num_tcs;
789 set = find_next_bit(addr: drv->tcs_in_use, size: max, offset: tcs->offset);
790
791 return set < max;
792}
793
794/**
795 * rpmh_rsc_write_next_wakeup() - Write next wakeup in CONTROL_TCS.
796 * @drv: The controller
797 *
798 * Writes maximum wakeup cycles when called from suspend.
799 * Writes earliest hrtimer wakeup when called from idle.
800 */
801void rpmh_rsc_write_next_wakeup(struct rsc_drv *drv)
802{
803 ktime_t now, wakeup;
804 u64 wakeup_us, wakeup_cycles = ~0;
805 u32 lo, hi;
806
807 if (!drv->tcs[CONTROL_TCS].num_tcs || !drv->genpd_nb.notifier_call)
808 return;
809
810 /* Set highest time when system (timekeeping) is suspended */
811 if (system_state == SYSTEM_SUSPEND)
812 goto exit;
813
814 /* Find the earliest hrtimer wakeup from online cpus */
815 wakeup = dev_pm_genpd_get_next_hrtimer(dev: drv->dev);
816
817 /* Find the relative wakeup in kernel time scale */
818 now = ktime_get();
819 wakeup = ktime_sub(wakeup, now);
820 wakeup_us = ktime_to_us(kt: wakeup);
821
822 /* Convert the wakeup to arch timer scale */
823 wakeup_cycles = USECS_TO_CYCLES(wakeup_us);
824 wakeup_cycles += arch_timer_read_counter();
825
826exit:
827 lo = wakeup_cycles & RSC_DRV_CTL_TCS_DATA_LO_MASK;
828 hi = wakeup_cycles >> RSC_DRV_CTL_TCS_DATA_SIZE;
829 hi &= RSC_DRV_CTL_TCS_DATA_HI_MASK;
830 hi |= RSC_DRV_CTL_TCS_DATA_HI_VALID;
831
832 writel_relaxed(lo, drv->base + RSC_DRV_CTL_TCS_DATA_LO);
833 writel_relaxed(hi, drv->base + RSC_DRV_CTL_TCS_DATA_HI);
834}
835
836/**
837 * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy.
838 * @nfb: Pointer to the notifier block in struct rsc_drv.
839 * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT.
840 * @v: Unused
841 *
842 * This function is given to cpu_pm_register_notifier so we can be informed
843 * about when CPUs go down. When all CPUs go down we know no more active
844 * transfers will be started so we write sleep/wake sets. This function gets
845 * called from cpuidle code paths and also at system suspend time.
846 *
847 * If its last CPU going down and AMCs are not busy then writes cached sleep
848 * and wake messages to TCSes. The firmware then takes care of triggering
849 * them when entering deepest low power modes.
850 *
851 * Return: See cpu_pm_register_notifier()
852 */
853static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb,
854 unsigned long action, void *v)
855{
856 struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm);
857 int ret = NOTIFY_OK;
858 int cpus_in_pm;
859
860 switch (action) {
861 case CPU_PM_ENTER:
862 cpus_in_pm = atomic_inc_return(v: &drv->cpus_in_pm);
863 /*
864 * NOTE: comments for num_online_cpus() point out that it's
865 * only a snapshot so we need to be careful. It should be OK
866 * for us to use, though. It's important for us not to miss
867 * if we're the last CPU going down so it would only be a
868 * problem if a CPU went offline right after we did the check
869 * AND that CPU was not idle AND that CPU was the last non-idle
870 * CPU. That can't happen. CPUs would have to come out of idle
871 * before the CPU could go offline.
872 */
873 if (cpus_in_pm < num_online_cpus())
874 return NOTIFY_OK;
875 break;
876 case CPU_PM_ENTER_FAILED:
877 case CPU_PM_EXIT:
878 atomic_dec(v: &drv->cpus_in_pm);
879 return NOTIFY_OK;
880 default:
881 return NOTIFY_DONE;
882 }
883
884 /*
885 * It's likely we're on the last CPU. Grab the drv->lock and write
886 * out the sleep/wake commands to RPMH hardware. Grabbing the lock
887 * means that if we race with another CPU coming up we are still
888 * guaranteed to be safe. If another CPU came up just after we checked
889 * and has grabbed the lock or started an active transfer then we'll
890 * notice we're busy and abort. If another CPU comes up after we start
891 * flushing it will be blocked from starting an active transfer until
892 * we're done flushing. If another CPU starts an active transfer after
893 * we release the lock we're still OK because we're no longer the last
894 * CPU.
895 */
896 if (spin_trylock(lock: &drv->lock)) {
897 if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(ctrlr: &drv->client))
898 ret = NOTIFY_BAD;
899 spin_unlock(lock: &drv->lock);
900 } else {
901 /* Another CPU must be up */
902 return NOTIFY_OK;
903 }
904
905 if (ret == NOTIFY_BAD) {
906 /* Double-check if we're here because someone else is up */
907 if (cpus_in_pm < num_online_cpus())
908 ret = NOTIFY_OK;
909 else
910 /* We won't be called w/ CPU_PM_ENTER_FAILED */
911 atomic_dec(v: &drv->cpus_in_pm);
912 }
913
914 return ret;
915}
916
917/**
918 * rpmh_rsc_pd_callback() - Check if any of the AMCs are busy.
919 * @nfb: Pointer to the genpd notifier block in struct rsc_drv.
920 * @action: GENPD_NOTIFY_PRE_OFF, GENPD_NOTIFY_OFF, GENPD_NOTIFY_PRE_ON or GENPD_NOTIFY_ON.
921 * @v: Unused
922 *
923 * This function is given to dev_pm_genpd_add_notifier() so we can be informed
924 * about when cluster-pd is going down. When cluster go down we know no more active
925 * transfers will be started so we write sleep/wake sets. This function gets
926 * called from cpuidle code paths and also at system suspend time.
927 *
928 * If AMCs are not busy then writes cached sleep and wake messages to TCSes.
929 * The firmware then takes care of triggering them when entering deepest low power modes.
930 *
931 * Return:
932 * * NOTIFY_OK - success
933 * * NOTIFY_BAD - failure
934 */
935static int rpmh_rsc_pd_callback(struct notifier_block *nfb,
936 unsigned long action, void *v)
937{
938 struct rsc_drv *drv = container_of(nfb, struct rsc_drv, genpd_nb);
939
940 /* We don't need to lock as genpd on/off are serialized */
941 if ((action == GENPD_NOTIFY_PRE_OFF) &&
942 (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(ctrlr: &drv->client)))
943 return NOTIFY_BAD;
944
945 return NOTIFY_OK;
946}
947
948static int rpmh_rsc_pd_attach(struct rsc_drv *drv, struct device *dev)
949{
950 int ret;
951
952 pm_runtime_enable(dev);
953 drv->genpd_nb.notifier_call = rpmh_rsc_pd_callback;
954 ret = dev_pm_genpd_add_notifier(dev, nb: &drv->genpd_nb);
955 if (ret)
956 pm_runtime_disable(dev);
957
958 return ret;
959}
960
961static int rpmh_probe_tcs_config(struct platform_device *pdev, struct rsc_drv *drv)
962{
963 struct tcs_type_config {
964 u32 type;
965 u32 n;
966 } tcs_cfg[TCS_TYPE_NR] = { { 0 } };
967 struct device_node *dn = pdev->dev.of_node;
968 u32 config, max_tcs, ncpt, offset;
969 int i, ret, n, st = 0;
970 struct tcs_group *tcs;
971
972 ret = of_property_read_u32(np: dn, propname: "qcom,tcs-offset", out_value: &offset);
973 if (ret)
974 return ret;
975 drv->tcs_base = drv->base + offset;
976
977 config = readl_relaxed(drv->base + drv->regs[DRV_PRNT_CHLD_CONFIG]);
978
979 max_tcs = config;
980 max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id);
981 max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id);
982
983 ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT);
984 ncpt = ncpt >> DRV_NCPT_SHIFT;
985
986 n = of_property_count_u32_elems(np: dn, propname: "qcom,tcs-config");
987 if (n != 2 * TCS_TYPE_NR)
988 return -EINVAL;
989
990 for (i = 0; i < TCS_TYPE_NR; i++) {
991 ret = of_property_read_u32_index(np: dn, propname: "qcom,tcs-config",
992 index: i * 2, out_value: &tcs_cfg[i].type);
993 if (ret)
994 return ret;
995 if (tcs_cfg[i].type >= TCS_TYPE_NR)
996 return -EINVAL;
997
998 ret = of_property_read_u32_index(np: dn, propname: "qcom,tcs-config",
999 index: i * 2 + 1, out_value: &tcs_cfg[i].n);
1000 if (ret)
1001 return ret;
1002 if (tcs_cfg[i].n > MAX_TCS_PER_TYPE)
1003 return -EINVAL;
1004 }
1005
1006 for (i = 0; i < TCS_TYPE_NR; i++) {
1007 tcs = &drv->tcs[tcs_cfg[i].type];
1008 if (tcs->drv)
1009 return -EINVAL;
1010 tcs->drv = drv;
1011 tcs->type = tcs_cfg[i].type;
1012 tcs->num_tcs = tcs_cfg[i].n;
1013 tcs->ncpt = ncpt;
1014
1015 if (!tcs->num_tcs || tcs->type == CONTROL_TCS)
1016 continue;
1017
1018 if (st + tcs->num_tcs > max_tcs ||
1019 st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask))
1020 return -EINVAL;
1021
1022 tcs->mask = ((1 << tcs->num_tcs) - 1) << st;
1023 tcs->offset = st;
1024 st += tcs->num_tcs;
1025 }
1026
1027 drv->num_tcs = st;
1028
1029 return 0;
1030}
1031
1032static int rpmh_rsc_probe(struct platform_device *pdev)
1033{
1034 struct device_node *dn = pdev->dev.of_node;
1035 struct rsc_drv *drv;
1036 char drv_id[10] = {0};
1037 int ret, irq;
1038 u32 solver_config;
1039 u32 rsc_id;
1040
1041 /*
1042 * Even though RPMh doesn't directly use cmd-db, all of its children
1043 * do. To avoid adding this check to our children we'll do it now.
1044 */
1045 ret = cmd_db_ready();
1046 if (ret) {
1047 if (ret != -EPROBE_DEFER)
1048 dev_err(&pdev->dev, "Command DB not available (%d)\n",
1049 ret);
1050 return ret;
1051 }
1052
1053 drv = devm_kzalloc(dev: &pdev->dev, size: sizeof(*drv), GFP_KERNEL);
1054 if (!drv)
1055 return -ENOMEM;
1056
1057 ret = of_property_read_u32(np: dn, propname: "qcom,drv-id", out_value: &drv->id);
1058 if (ret)
1059 return ret;
1060
1061 drv->name = of_get_property(node: dn, name: "label", NULL);
1062 if (!drv->name)
1063 drv->name = dev_name(dev: &pdev->dev);
1064
1065 snprintf(buf: drv_id, ARRAY_SIZE(drv_id), fmt: "drv-%d", drv->id);
1066 drv->base = devm_platform_ioremap_resource_byname(pdev, name: drv_id);
1067 if (IS_ERR(ptr: drv->base))
1068 return PTR_ERR(ptr: drv->base);
1069
1070 rsc_id = readl_relaxed(drv->base + RSC_DRV_ID);
1071 drv->ver.major = rsc_id & (MAJOR_VER_MASK << MAJOR_VER_SHIFT);
1072 drv->ver.major >>= MAJOR_VER_SHIFT;
1073 drv->ver.minor = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT);
1074 drv->ver.minor >>= MINOR_VER_SHIFT;
1075
1076 if (drv->ver.major == 3)
1077 drv->regs = rpmh_rsc_reg_offset_ver_3_0;
1078 else
1079 drv->regs = rpmh_rsc_reg_offset_ver_2_7;
1080
1081 ret = rpmh_probe_tcs_config(pdev, drv);
1082 if (ret)
1083 return ret;
1084
1085 spin_lock_init(&drv->lock);
1086 init_waitqueue_head(&drv->tcs_wait);
1087 bitmap_zero(dst: drv->tcs_in_use, MAX_TCS_NR);
1088
1089 irq = platform_get_irq(pdev, drv->id);
1090 if (irq < 0)
1091 return irq;
1092
1093 ret = devm_request_irq(dev: &pdev->dev, irq, handler: tcs_tx_done,
1094 IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND,
1095 devname: drv->name, dev_id: drv);
1096 if (ret)
1097 return ret;
1098
1099 /*
1100 * CPU PM/genpd notification are not required for controllers that support
1101 * 'HW solver' mode where they can be in autonomous mode executing low
1102 * power mode to power down.
1103 */
1104 solver_config = readl_relaxed(drv->base + drv->regs[DRV_SOLVER_CONFIG]);
1105 solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT;
1106 solver_config = solver_config >> DRV_HW_SOLVER_SHIFT;
1107 if (!solver_config) {
1108 if (pdev->dev.pm_domain) {
1109 ret = rpmh_rsc_pd_attach(drv, dev: &pdev->dev);
1110 if (ret)
1111 return ret;
1112 } else {
1113 drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;
1114 cpu_pm_register_notifier(nb: &drv->rsc_pm);
1115 }
1116 }
1117
1118 /* Enable the active TCS to send requests immediately */
1119 writel_relaxed(drv->tcs[ACTIVE_TCS].mask,
1120 drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]);
1121
1122 spin_lock_init(&drv->client.cache_lock);
1123 INIT_LIST_HEAD(list: &drv->client.cache);
1124 INIT_LIST_HEAD(list: &drv->client.batch_cache);
1125
1126 dev_set_drvdata(dev: &pdev->dev, data: drv);
1127 drv->dev = &pdev->dev;
1128
1129 ret = devm_of_platform_populate(dev: &pdev->dev);
1130 if (ret && pdev->dev.pm_domain) {
1131 dev_pm_genpd_remove_notifier(dev: &pdev->dev);
1132 pm_runtime_disable(dev: &pdev->dev);
1133 }
1134
1135 return ret;
1136}
1137
1138static const struct of_device_id rpmh_drv_match[] = {
1139 { .compatible = "qcom,rpmh-rsc", },
1140 { }
1141};
1142MODULE_DEVICE_TABLE(of, rpmh_drv_match);
1143
1144static struct platform_driver rpmh_driver = {
1145 .probe = rpmh_rsc_probe,
1146 .driver = {
1147 .name = "rpmh",
1148 .of_match_table = rpmh_drv_match,
1149 .suppress_bind_attrs = true,
1150 },
1151};
1152
1153static int __init rpmh_driver_init(void)
1154{
1155 return platform_driver_register(&rpmh_driver);
1156}
1157arch_initcall(rpmh_driver_init);
1158
1159MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Driver");
1160MODULE_LICENSE("GPL v2");
1161

source code of linux/drivers/soc/qcom/rpmh-rsc.c