1// SPDX-License-Identifier: GPL-2.0
2/*
3 * System Control and Management Interface (SCMI) Performance Protocol
4 *
5 * Copyright (C) 2018-2023 ARM Ltd.
6 */
7
8#define pr_fmt(fmt) "SCMI Notifications PERF - " fmt
9
10#include <linux/bits.h>
11#include <linux/hashtable.h>
12#include <linux/io.h>
13#include <linux/log2.h>
14#include <linux/module.h>
15#include <linux/of.h>
16#include <linux/platform_device.h>
17#include <linux/pm_opp.h>
18#include <linux/scmi_protocol.h>
19#include <linux/sort.h>
20#include <linux/xarray.h>
21
22#include <trace/events/scmi.h>
23
24#include "protocols.h"
25#include "notify.h"
26
27/* Updated only after ALL the mandatory features for that version are merged */
28#define SCMI_PROTOCOL_SUPPORTED_VERSION 0x40000
29
30#define MAX_OPPS 32
31
32enum scmi_performance_protocol_cmd {
33 PERF_DOMAIN_ATTRIBUTES = 0x3,
34 PERF_DESCRIBE_LEVELS = 0x4,
35 PERF_LIMITS_SET = 0x5,
36 PERF_LIMITS_GET = 0x6,
37 PERF_LEVEL_SET = 0x7,
38 PERF_LEVEL_GET = 0x8,
39 PERF_NOTIFY_LIMITS = 0x9,
40 PERF_NOTIFY_LEVEL = 0xa,
41 PERF_DESCRIBE_FASTCHANNEL = 0xb,
42 PERF_DOMAIN_NAME_GET = 0xc,
43};
44
45enum {
46 PERF_FC_LEVEL,
47 PERF_FC_LIMIT,
48 PERF_FC_MAX,
49};
50
51struct scmi_opp {
52 u32 perf;
53 u32 power;
54 u32 trans_latency_us;
55 u32 indicative_freq;
56 u32 level_index;
57 struct hlist_node hash;
58};
59
60struct scmi_msg_resp_perf_attributes {
61 __le16 num_domains;
62 __le16 flags;
63#define POWER_SCALE_IN_MILLIWATT(x) ((x) & BIT(0))
64#define POWER_SCALE_IN_MICROWATT(x) ((x) & BIT(1))
65 __le32 stats_addr_low;
66 __le32 stats_addr_high;
67 __le32 stats_size;
68};
69
70struct scmi_msg_resp_perf_domain_attributes {
71 __le32 flags;
72#define SUPPORTS_SET_LIMITS(x) ((x) & BIT(31))
73#define SUPPORTS_SET_PERF_LVL(x) ((x) & BIT(30))
74#define SUPPORTS_PERF_LIMIT_NOTIFY(x) ((x) & BIT(29))
75#define SUPPORTS_PERF_LEVEL_NOTIFY(x) ((x) & BIT(28))
76#define SUPPORTS_PERF_FASTCHANNELS(x) ((x) & BIT(27))
77#define SUPPORTS_EXTENDED_NAMES(x) ((x) & BIT(26))
78#define SUPPORTS_LEVEL_INDEXING(x) ((x) & BIT(25))
79 __le32 rate_limit_us;
80 __le32 sustained_freq_khz;
81 __le32 sustained_perf_level;
82 u8 name[SCMI_SHORT_NAME_MAX_SIZE];
83};
84
85struct scmi_msg_perf_describe_levels {
86 __le32 domain;
87 __le32 level_index;
88};
89
90struct scmi_perf_set_limits {
91 __le32 domain;
92 __le32 max_level;
93 __le32 min_level;
94};
95
96struct scmi_perf_get_limits {
97 __le32 max_level;
98 __le32 min_level;
99};
100
101struct scmi_perf_set_level {
102 __le32 domain;
103 __le32 level;
104};
105
106struct scmi_perf_notify_level_or_limits {
107 __le32 domain;
108 __le32 notify_enable;
109};
110
111struct scmi_perf_limits_notify_payld {
112 __le32 agent_id;
113 __le32 domain_id;
114 __le32 range_max;
115 __le32 range_min;
116};
117
118struct scmi_perf_level_notify_payld {
119 __le32 agent_id;
120 __le32 domain_id;
121 __le32 performance_level;
122};
123
124struct scmi_msg_resp_perf_describe_levels {
125 __le16 num_returned;
126 __le16 num_remaining;
127 struct {
128 __le32 perf_val;
129 __le32 power;
130 __le16 transition_latency_us;
131 __le16 reserved;
132 } opp[];
133};
134
135struct scmi_msg_resp_perf_describe_levels_v4 {
136 __le16 num_returned;
137 __le16 num_remaining;
138 struct {
139 __le32 perf_val;
140 __le32 power;
141 __le16 transition_latency_us;
142 __le16 reserved;
143 __le32 indicative_freq;
144 __le32 level_index;
145 } opp[];
146};
147
148struct perf_dom_info {
149 u32 id;
150 bool set_limits;
151 bool perf_limit_notify;
152 bool perf_level_notify;
153 bool perf_fastchannels;
154 bool level_indexing_mode;
155 u32 opp_count;
156 u32 rate_limit_us;
157 u32 sustained_freq_khz;
158 u32 sustained_perf_level;
159 unsigned long mult_factor;
160 struct scmi_perf_domain_info info;
161 struct scmi_opp opp[MAX_OPPS];
162 struct scmi_fc_info *fc_info;
163 struct xarray opps_by_idx;
164 struct xarray opps_by_lvl;
165 DECLARE_HASHTABLE(opps_by_freq, ilog2(MAX_OPPS));
166};
167
168#define LOOKUP_BY_FREQ(__htp, __freq) \
169({ \
170 /* u32 cast is needed to pick right hash func */ \
171 u32 f_ = (u32)(__freq); \
172 struct scmi_opp *_opp; \
173 \
174 hash_for_each_possible((__htp), _opp, hash, f_) \
175 if (_opp->indicative_freq == f_) \
176 break; \
177 _opp; \
178})
179
180struct scmi_perf_info {
181 u32 version;
182 u16 num_domains;
183 enum scmi_power_scale power_scale;
184 u64 stats_addr;
185 u32 stats_size;
186 bool notify_lvl_cmd;
187 bool notify_lim_cmd;
188 struct perf_dom_info *dom_info;
189};
190
191static enum scmi_performance_protocol_cmd evt_2_cmd[] = {
192 PERF_NOTIFY_LIMITS,
193 PERF_NOTIFY_LEVEL,
194};
195
196static int scmi_perf_attributes_get(const struct scmi_protocol_handle *ph,
197 struct scmi_perf_info *pi)
198{
199 int ret;
200 struct scmi_xfer *t;
201 struct scmi_msg_resp_perf_attributes *attr;
202
203 ret = ph->xops->xfer_get_init(ph, PROTOCOL_ATTRIBUTES, 0,
204 sizeof(*attr), &t);
205 if (ret)
206 return ret;
207
208 attr = t->rx.buf;
209
210 ret = ph->xops->do_xfer(ph, t);
211 if (!ret) {
212 u16 flags = le16_to_cpu(attr->flags);
213
214 pi->num_domains = le16_to_cpu(attr->num_domains);
215
216 if (POWER_SCALE_IN_MILLIWATT(flags))
217 pi->power_scale = SCMI_POWER_MILLIWATTS;
218 if (PROTOCOL_REV_MAJOR(pi->version) >= 0x3)
219 if (POWER_SCALE_IN_MICROWATT(flags))
220 pi->power_scale = SCMI_POWER_MICROWATTS;
221
222 pi->stats_addr = le32_to_cpu(attr->stats_addr_low) |
223 (u64)le32_to_cpu(attr->stats_addr_high) << 32;
224 pi->stats_size = le32_to_cpu(attr->stats_size);
225 }
226
227 ph->xops->xfer_put(ph, t);
228
229 if (!ret) {
230 if (!ph->hops->protocol_msg_check(ph, PERF_NOTIFY_LEVEL, NULL))
231 pi->notify_lvl_cmd = true;
232
233 if (!ph->hops->protocol_msg_check(ph, PERF_NOTIFY_LIMITS, NULL))
234 pi->notify_lim_cmd = true;
235 }
236
237 return ret;
238}
239
240static void scmi_perf_xa_destroy(void *data)
241{
242 int domain;
243 struct scmi_perf_info *pinfo = data;
244
245 for (domain = 0; domain < pinfo->num_domains; domain++) {
246 xa_destroy(&((pinfo->dom_info + domain)->opps_by_idx));
247 xa_destroy(&((pinfo->dom_info + domain)->opps_by_lvl));
248 }
249}
250
251static int
252scmi_perf_domain_attributes_get(const struct scmi_protocol_handle *ph,
253 struct perf_dom_info *dom_info,
254 bool notify_lim_cmd, bool notify_lvl_cmd,
255 u32 version)
256{
257 int ret;
258 u32 flags;
259 struct scmi_xfer *t;
260 struct scmi_msg_resp_perf_domain_attributes *attr;
261
262 ret = ph->xops->xfer_get_init(ph, PERF_DOMAIN_ATTRIBUTES,
263 sizeof(dom_info->id), sizeof(*attr), &t);
264 if (ret)
265 return ret;
266
267 put_unaligned_le32(val: dom_info->id, p: t->tx.buf);
268 attr = t->rx.buf;
269
270 ret = ph->xops->do_xfer(ph, t);
271 if (!ret) {
272 flags = le32_to_cpu(attr->flags);
273
274 dom_info->set_limits = SUPPORTS_SET_LIMITS(flags);
275 dom_info->info.set_perf = SUPPORTS_SET_PERF_LVL(flags);
276 if (notify_lim_cmd)
277 dom_info->perf_limit_notify =
278 SUPPORTS_PERF_LIMIT_NOTIFY(flags);
279 if (notify_lvl_cmd)
280 dom_info->perf_level_notify =
281 SUPPORTS_PERF_LEVEL_NOTIFY(flags);
282 dom_info->perf_fastchannels = SUPPORTS_PERF_FASTCHANNELS(flags);
283 if (PROTOCOL_REV_MAJOR(version) >= 0x4)
284 dom_info->level_indexing_mode =
285 SUPPORTS_LEVEL_INDEXING(flags);
286 dom_info->rate_limit_us = le32_to_cpu(attr->rate_limit_us) &
287 GENMASK(19, 0);
288 dom_info->sustained_freq_khz =
289 le32_to_cpu(attr->sustained_freq_khz);
290 dom_info->sustained_perf_level =
291 le32_to_cpu(attr->sustained_perf_level);
292 /*
293 * sustained_freq_khz = mult_factor * sustained_perf_level
294 * mult_factor must be non zero positive integer(not fraction)
295 */
296 if (!dom_info->sustained_freq_khz ||
297 !dom_info->sustained_perf_level ||
298 dom_info->level_indexing_mode) {
299 /* CPUFreq converts to kHz, hence default 1000 */
300 dom_info->mult_factor = 1000;
301 } else {
302 dom_info->mult_factor =
303 (dom_info->sustained_freq_khz * 1000UL)
304 / dom_info->sustained_perf_level;
305 if ((dom_info->sustained_freq_khz * 1000UL) %
306 dom_info->sustained_perf_level)
307 dev_warn(ph->dev,
308 "multiplier for domain %d rounded\n",
309 dom_info->id);
310 }
311 if (!dom_info->mult_factor)
312 dev_warn(ph->dev,
313 "Wrong sustained perf/frequency(domain %d)\n",
314 dom_info->id);
315
316 strscpy(dom_info->info.name, attr->name,
317 SCMI_SHORT_NAME_MAX_SIZE);
318 }
319
320 ph->xops->xfer_put(ph, t);
321
322 /*
323 * If supported overwrite short name with the extended one;
324 * on error just carry on and use already provided short name.
325 */
326 if (!ret && PROTOCOL_REV_MAJOR(version) >= 0x3 &&
327 SUPPORTS_EXTENDED_NAMES(flags))
328 ph->hops->extended_name_get(ph, PERF_DOMAIN_NAME_GET,
329 dom_info->id, NULL, dom_info->info.name,
330 SCMI_MAX_STR_SIZE);
331
332 xa_init(xa: &dom_info->opps_by_lvl);
333 if (dom_info->level_indexing_mode) {
334 xa_init(xa: &dom_info->opps_by_idx);
335 hash_init(dom_info->opps_by_freq);
336 }
337
338 return ret;
339}
340
341static int opp_cmp_func(const void *opp1, const void *opp2)
342{
343 const struct scmi_opp *t1 = opp1, *t2 = opp2;
344
345 return t1->perf - t2->perf;
346}
347
348struct scmi_perf_ipriv {
349 u32 version;
350 struct perf_dom_info *perf_dom;
351};
352
353static void iter_perf_levels_prepare_message(void *message,
354 unsigned int desc_index,
355 const void *priv)
356{
357 struct scmi_msg_perf_describe_levels *msg = message;
358 const struct scmi_perf_ipriv *p = priv;
359
360 msg->domain = cpu_to_le32(p->perf_dom->id);
361 /* Set the number of OPPs to be skipped/already read */
362 msg->level_index = cpu_to_le32(desc_index);
363}
364
365static int iter_perf_levels_update_state(struct scmi_iterator_state *st,
366 const void *response, void *priv)
367{
368 const struct scmi_msg_resp_perf_describe_levels *r = response;
369
370 st->num_returned = le16_to_cpu(r->num_returned);
371 st->num_remaining = le16_to_cpu(r->num_remaining);
372
373 return 0;
374}
375
376static inline void
377process_response_opp(struct device *dev, struct perf_dom_info *dom,
378 struct scmi_opp *opp, unsigned int loop_idx,
379 const struct scmi_msg_resp_perf_describe_levels *r)
380{
381 int ret;
382
383 opp->perf = le32_to_cpu(r->opp[loop_idx].perf_val);
384 opp->power = le32_to_cpu(r->opp[loop_idx].power);
385 opp->trans_latency_us =
386 le16_to_cpu(r->opp[loop_idx].transition_latency_us);
387
388 ret = xa_insert(xa: &dom->opps_by_lvl, index: opp->perf, entry: opp, GFP_KERNEL);
389 if (ret)
390 dev_warn(dev, "Failed to add opps_by_lvl at %d - ret:%d\n",
391 opp->perf, ret);
392}
393
394static inline void
395process_response_opp_v4(struct device *dev, struct perf_dom_info *dom,
396 struct scmi_opp *opp, unsigned int loop_idx,
397 const struct scmi_msg_resp_perf_describe_levels_v4 *r)
398{
399 int ret;
400
401 opp->perf = le32_to_cpu(r->opp[loop_idx].perf_val);
402 opp->power = le32_to_cpu(r->opp[loop_idx].power);
403 opp->trans_latency_us =
404 le16_to_cpu(r->opp[loop_idx].transition_latency_us);
405
406 ret = xa_insert(xa: &dom->opps_by_lvl, index: opp->perf, entry: opp, GFP_KERNEL);
407 if (ret)
408 dev_warn(dev, "Failed to add opps_by_lvl at %d - ret:%d\n",
409 opp->perf, ret);
410
411 /* Note that PERF v4 reports always five 32-bit words */
412 opp->indicative_freq = le32_to_cpu(r->opp[loop_idx].indicative_freq);
413 if (dom->level_indexing_mode) {
414 opp->level_index = le32_to_cpu(r->opp[loop_idx].level_index);
415
416 ret = xa_insert(xa: &dom->opps_by_idx, index: opp->level_index, entry: opp,
417 GFP_KERNEL);
418 if (ret)
419 dev_warn(dev,
420 "Failed to add opps_by_idx at %d - ret:%d\n",
421 opp->level_index, ret);
422
423 hash_add(dom->opps_by_freq, &opp->hash, opp->indicative_freq);
424 }
425}
426
427static int
428iter_perf_levels_process_response(const struct scmi_protocol_handle *ph,
429 const void *response,
430 struct scmi_iterator_state *st, void *priv)
431{
432 struct scmi_opp *opp;
433 struct scmi_perf_ipriv *p = priv;
434
435 opp = &p->perf_dom->opp[st->desc_index + st->loop_idx];
436 if (PROTOCOL_REV_MAJOR(p->version) <= 0x3)
437 process_response_opp(dev: ph->dev, dom: p->perf_dom, opp, loop_idx: st->loop_idx,
438 r: response);
439 else
440 process_response_opp_v4(dev: ph->dev, dom: p->perf_dom, opp, loop_idx: st->loop_idx,
441 r: response);
442 p->perf_dom->opp_count++;
443
444 dev_dbg(ph->dev, "Level %d Power %d Latency %dus Ifreq %d Index %d\n",
445 opp->perf, opp->power, opp->trans_latency_us,
446 opp->indicative_freq, opp->level_index);
447
448 return 0;
449}
450
451static int
452scmi_perf_describe_levels_get(const struct scmi_protocol_handle *ph,
453 struct perf_dom_info *perf_dom, u32 version)
454{
455 int ret;
456 void *iter;
457 struct scmi_iterator_ops ops = {
458 .prepare_message = iter_perf_levels_prepare_message,
459 .update_state = iter_perf_levels_update_state,
460 .process_response = iter_perf_levels_process_response,
461 };
462 struct scmi_perf_ipriv ppriv = {
463 .version = version,
464 .perf_dom = perf_dom,
465 };
466
467 iter = ph->hops->iter_response_init(ph, &ops, MAX_OPPS,
468 PERF_DESCRIBE_LEVELS,
469 sizeof(struct scmi_msg_perf_describe_levels),
470 &ppriv);
471 if (IS_ERR(ptr: iter))
472 return PTR_ERR(ptr: iter);
473
474 ret = ph->hops->iter_response_run(iter);
475 if (ret)
476 return ret;
477
478 if (perf_dom->opp_count)
479 sort(base: perf_dom->opp, num: perf_dom->opp_count,
480 size: sizeof(struct scmi_opp), cmp_func: opp_cmp_func, NULL);
481
482 return ret;
483}
484
485static int scmi_perf_num_domains_get(const struct scmi_protocol_handle *ph)
486{
487 struct scmi_perf_info *pi = ph->get_priv(ph);
488
489 return pi->num_domains;
490}
491
492static inline struct perf_dom_info *
493scmi_perf_domain_lookup(const struct scmi_protocol_handle *ph, u32 domain)
494{
495 struct scmi_perf_info *pi = ph->get_priv(ph);
496
497 if (domain >= pi->num_domains)
498 return ERR_PTR(error: -EINVAL);
499
500 return pi->dom_info + domain;
501}
502
503static const struct scmi_perf_domain_info *
504scmi_perf_info_get(const struct scmi_protocol_handle *ph, u32 domain)
505{
506 struct perf_dom_info *dom;
507
508 dom = scmi_perf_domain_lookup(ph, domain);
509 if (IS_ERR(ptr: dom))
510 return ERR_PTR(error: -EINVAL);
511
512 return &dom->info;
513}
514
515static int scmi_perf_msg_limits_set(const struct scmi_protocol_handle *ph,
516 u32 domain, u32 max_perf, u32 min_perf)
517{
518 int ret;
519 struct scmi_xfer *t;
520 struct scmi_perf_set_limits *limits;
521
522 ret = ph->xops->xfer_get_init(ph, PERF_LIMITS_SET,
523 sizeof(*limits), 0, &t);
524 if (ret)
525 return ret;
526
527 limits = t->tx.buf;
528 limits->domain = cpu_to_le32(domain);
529 limits->max_level = cpu_to_le32(max_perf);
530 limits->min_level = cpu_to_le32(min_perf);
531
532 ret = ph->xops->do_xfer(ph, t);
533
534 ph->xops->xfer_put(ph, t);
535 return ret;
536}
537
538static int __scmi_perf_limits_set(const struct scmi_protocol_handle *ph,
539 struct perf_dom_info *dom, u32 max_perf,
540 u32 min_perf)
541{
542 if (dom->fc_info && dom->fc_info[PERF_FC_LIMIT].set_addr) {
543 struct scmi_fc_info *fci = &dom->fc_info[PERF_FC_LIMIT];
544
545 trace_scmi_fc_call(protocol_id: SCMI_PROTOCOL_PERF, msg_id: PERF_LIMITS_SET,
546 res_id: dom->id, val1: min_perf, val2: max_perf);
547 iowrite32(max_perf, fci->set_addr);
548 iowrite32(min_perf, fci->set_addr + 4);
549 ph->hops->fastchannel_db_ring(fci->set_db);
550 return 0;
551 }
552
553 return scmi_perf_msg_limits_set(ph, domain: dom->id, max_perf, min_perf);
554}
555
556static int scmi_perf_limits_set(const struct scmi_protocol_handle *ph,
557 u32 domain, u32 max_perf, u32 min_perf)
558{
559 struct scmi_perf_info *pi = ph->get_priv(ph);
560 struct perf_dom_info *dom;
561
562 dom = scmi_perf_domain_lookup(ph, domain);
563 if (IS_ERR(ptr: dom))
564 return PTR_ERR(ptr: dom);
565
566 if (!dom->set_limits)
567 return -EOPNOTSUPP;
568
569 if (PROTOCOL_REV_MAJOR(pi->version) >= 0x3 && !max_perf && !min_perf)
570 return -EINVAL;
571
572 if (dom->level_indexing_mode) {
573 struct scmi_opp *opp;
574
575 if (min_perf) {
576 opp = xa_load(&dom->opps_by_lvl, index: min_perf);
577 if (!opp)
578 return -EIO;
579
580 min_perf = opp->level_index;
581 }
582
583 if (max_perf) {
584 opp = xa_load(&dom->opps_by_lvl, index: max_perf);
585 if (!opp)
586 return -EIO;
587
588 max_perf = opp->level_index;
589 }
590 }
591
592 return __scmi_perf_limits_set(ph, dom, max_perf, min_perf);
593}
594
595static int scmi_perf_msg_limits_get(const struct scmi_protocol_handle *ph,
596 u32 domain, u32 *max_perf, u32 *min_perf)
597{
598 int ret;
599 struct scmi_xfer *t;
600 struct scmi_perf_get_limits *limits;
601
602 ret = ph->xops->xfer_get_init(ph, PERF_LIMITS_GET,
603 sizeof(__le32), 0, &t);
604 if (ret)
605 return ret;
606
607 put_unaligned_le32(val: domain, p: t->tx.buf);
608
609 ret = ph->xops->do_xfer(ph, t);
610 if (!ret) {
611 limits = t->rx.buf;
612
613 *max_perf = le32_to_cpu(limits->max_level);
614 *min_perf = le32_to_cpu(limits->min_level);
615 }
616
617 ph->xops->xfer_put(ph, t);
618 return ret;
619}
620
621static int __scmi_perf_limits_get(const struct scmi_protocol_handle *ph,
622 struct perf_dom_info *dom, u32 *max_perf,
623 u32 *min_perf)
624{
625 if (dom->fc_info && dom->fc_info[PERF_FC_LIMIT].get_addr) {
626 struct scmi_fc_info *fci = &dom->fc_info[PERF_FC_LIMIT];
627
628 *max_perf = ioread32(fci->get_addr);
629 *min_perf = ioread32(fci->get_addr + 4);
630 trace_scmi_fc_call(protocol_id: SCMI_PROTOCOL_PERF, msg_id: PERF_LIMITS_GET,
631 res_id: dom->id, val1: *min_perf, val2: *max_perf);
632 return 0;
633 }
634
635 return scmi_perf_msg_limits_get(ph, domain: dom->id, max_perf, min_perf);
636}
637
638static int scmi_perf_limits_get(const struct scmi_protocol_handle *ph,
639 u32 domain, u32 *max_perf, u32 *min_perf)
640{
641 int ret;
642 struct perf_dom_info *dom;
643
644 dom = scmi_perf_domain_lookup(ph, domain);
645 if (IS_ERR(ptr: dom))
646 return PTR_ERR(ptr: dom);
647
648 ret = __scmi_perf_limits_get(ph, dom, max_perf, min_perf);
649 if (ret)
650 return ret;
651
652 if (dom->level_indexing_mode) {
653 struct scmi_opp *opp;
654
655 opp = xa_load(&dom->opps_by_idx, index: *min_perf);
656 if (!opp)
657 return -EIO;
658
659 *min_perf = opp->perf;
660
661 opp = xa_load(&dom->opps_by_idx, index: *max_perf);
662 if (!opp)
663 return -EIO;
664
665 *max_perf = opp->perf;
666 }
667
668 return 0;
669}
670
671static int scmi_perf_msg_level_set(const struct scmi_protocol_handle *ph,
672 u32 domain, u32 level, bool poll)
673{
674 int ret;
675 struct scmi_xfer *t;
676 struct scmi_perf_set_level *lvl;
677
678 ret = ph->xops->xfer_get_init(ph, PERF_LEVEL_SET, sizeof(*lvl), 0, &t);
679 if (ret)
680 return ret;
681
682 t->hdr.poll_completion = poll;
683 lvl = t->tx.buf;
684 lvl->domain = cpu_to_le32(domain);
685 lvl->level = cpu_to_le32(level);
686
687 ret = ph->xops->do_xfer(ph, t);
688
689 ph->xops->xfer_put(ph, t);
690 return ret;
691}
692
693static int __scmi_perf_level_set(const struct scmi_protocol_handle *ph,
694 struct perf_dom_info *dom, u32 level,
695 bool poll)
696{
697 if (dom->fc_info && dom->fc_info[PERF_FC_LEVEL].set_addr) {
698 struct scmi_fc_info *fci = &dom->fc_info[PERF_FC_LEVEL];
699
700 trace_scmi_fc_call(protocol_id: SCMI_PROTOCOL_PERF, msg_id: PERF_LEVEL_SET,
701 res_id: dom->id, val1: level, val2: 0);
702 iowrite32(level, fci->set_addr);
703 ph->hops->fastchannel_db_ring(fci->set_db);
704 return 0;
705 }
706
707 return scmi_perf_msg_level_set(ph, domain: dom->id, level, poll);
708}
709
710static int scmi_perf_level_set(const struct scmi_protocol_handle *ph,
711 u32 domain, u32 level, bool poll)
712{
713 struct perf_dom_info *dom;
714
715 dom = scmi_perf_domain_lookup(ph, domain);
716 if (IS_ERR(ptr: dom))
717 return PTR_ERR(ptr: dom);
718
719 if (!dom->info.set_perf)
720 return -EOPNOTSUPP;
721
722 if (dom->level_indexing_mode) {
723 struct scmi_opp *opp;
724
725 opp = xa_load(&dom->opps_by_lvl, index: level);
726 if (!opp)
727 return -EIO;
728
729 level = opp->level_index;
730 }
731
732 return __scmi_perf_level_set(ph, dom, level, poll);
733}
734
735static int scmi_perf_msg_level_get(const struct scmi_protocol_handle *ph,
736 u32 domain, u32 *level, bool poll)
737{
738 int ret;
739 struct scmi_xfer *t;
740
741 ret = ph->xops->xfer_get_init(ph, PERF_LEVEL_GET,
742 sizeof(u32), sizeof(u32), &t);
743 if (ret)
744 return ret;
745
746 t->hdr.poll_completion = poll;
747 put_unaligned_le32(val: domain, p: t->tx.buf);
748
749 ret = ph->xops->do_xfer(ph, t);
750 if (!ret)
751 *level = get_unaligned_le32(p: t->rx.buf);
752
753 ph->xops->xfer_put(ph, t);
754 return ret;
755}
756
757static int __scmi_perf_level_get(const struct scmi_protocol_handle *ph,
758 struct perf_dom_info *dom, u32 *level,
759 bool poll)
760{
761 if (dom->fc_info && dom->fc_info[PERF_FC_LEVEL].get_addr) {
762 *level = ioread32(dom->fc_info[PERF_FC_LEVEL].get_addr);
763 trace_scmi_fc_call(protocol_id: SCMI_PROTOCOL_PERF, msg_id: PERF_LEVEL_GET,
764 res_id: dom->id, val1: *level, val2: 0);
765 return 0;
766 }
767
768 return scmi_perf_msg_level_get(ph, domain: dom->id, level, poll);
769}
770
771static int scmi_perf_level_get(const struct scmi_protocol_handle *ph,
772 u32 domain, u32 *level, bool poll)
773{
774 int ret;
775 struct perf_dom_info *dom;
776
777 dom = scmi_perf_domain_lookup(ph, domain);
778 if (IS_ERR(ptr: dom))
779 return PTR_ERR(ptr: dom);
780
781 ret = __scmi_perf_level_get(ph, dom, level, poll);
782 if (ret)
783 return ret;
784
785 if (dom->level_indexing_mode) {
786 struct scmi_opp *opp;
787
788 opp = xa_load(&dom->opps_by_idx, index: *level);
789 if (!opp)
790 return -EIO;
791
792 *level = opp->perf;
793 }
794
795 return 0;
796}
797
798static int scmi_perf_level_limits_notify(const struct scmi_protocol_handle *ph,
799 u32 domain, int message_id,
800 bool enable)
801{
802 int ret;
803 struct scmi_xfer *t;
804 struct scmi_perf_notify_level_or_limits *notify;
805
806 ret = ph->xops->xfer_get_init(ph, message_id, sizeof(*notify), 0, &t);
807 if (ret)
808 return ret;
809
810 notify = t->tx.buf;
811 notify->domain = cpu_to_le32(domain);
812 notify->notify_enable = enable ? cpu_to_le32(BIT(0)) : 0;
813
814 ret = ph->xops->do_xfer(ph, t);
815
816 ph->xops->xfer_put(ph, t);
817 return ret;
818}
819
820static void scmi_perf_domain_init_fc(const struct scmi_protocol_handle *ph,
821 struct perf_dom_info *dom)
822{
823 struct scmi_fc_info *fc;
824
825 fc = devm_kcalloc(dev: ph->dev, n: PERF_FC_MAX, size: sizeof(*fc), GFP_KERNEL);
826 if (!fc)
827 return;
828
829 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
830 PERF_LEVEL_GET, 4, dom->id,
831 &fc[PERF_FC_LEVEL].get_addr, NULL,
832 &fc[PERF_FC_LEVEL].rate_limit);
833
834 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
835 PERF_LIMITS_GET, 8, dom->id,
836 &fc[PERF_FC_LIMIT].get_addr, NULL,
837 &fc[PERF_FC_LIMIT].rate_limit);
838
839 if (dom->info.set_perf)
840 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
841 PERF_LEVEL_SET, 4, dom->id,
842 &fc[PERF_FC_LEVEL].set_addr,
843 &fc[PERF_FC_LEVEL].set_db,
844 &fc[PERF_FC_LEVEL].rate_limit);
845
846 if (dom->set_limits)
847 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
848 PERF_LIMITS_SET, 8, dom->id,
849 &fc[PERF_FC_LIMIT].set_addr,
850 &fc[PERF_FC_LIMIT].set_db,
851 &fc[PERF_FC_LIMIT].rate_limit);
852
853 dom->fc_info = fc;
854}
855
856static int scmi_dvfs_device_opps_add(const struct scmi_protocol_handle *ph,
857 struct device *dev, u32 domain)
858{
859 int idx, ret;
860 unsigned long freq;
861 struct dev_pm_opp_data data = {};
862 struct perf_dom_info *dom;
863
864 dom = scmi_perf_domain_lookup(ph, domain);
865 if (IS_ERR(ptr: dom))
866 return PTR_ERR(ptr: dom);
867
868 for (idx = 0; idx < dom->opp_count; idx++) {
869 if (!dom->level_indexing_mode)
870 freq = dom->opp[idx].perf * dom->mult_factor;
871 else
872 freq = dom->opp[idx].indicative_freq * dom->mult_factor;
873
874 /* All OPPs above the sustained frequency are treated as turbo */
875 data.turbo = freq > dom->sustained_freq_khz * 1000;
876
877 data.level = dom->opp[idx].perf;
878 data.freq = freq;
879
880 ret = dev_pm_opp_add_dynamic(dev, opp: &data);
881 if (ret) {
882 dev_warn(dev, "failed to add opp %luHz\n", freq);
883 dev_pm_opp_remove_all_dynamic(dev);
884 return ret;
885 }
886
887 dev_dbg(dev, "[%d][%s]:: Registered OPP[%d] %lu\n",
888 domain, dom->info.name, idx, freq);
889 }
890 return 0;
891}
892
893static int
894scmi_dvfs_transition_latency_get(const struct scmi_protocol_handle *ph,
895 u32 domain)
896{
897 struct perf_dom_info *dom;
898
899 dom = scmi_perf_domain_lookup(ph, domain);
900 if (IS_ERR(ptr: dom))
901 return PTR_ERR(ptr: dom);
902
903 /* uS to nS */
904 return dom->opp[dom->opp_count - 1].trans_latency_us * 1000;
905}
906
907static int
908scmi_dvfs_rate_limit_get(const struct scmi_protocol_handle *ph,
909 u32 domain, u32 *rate_limit)
910{
911 struct perf_dom_info *dom;
912
913 if (!rate_limit)
914 return -EINVAL;
915
916 dom = scmi_perf_domain_lookup(ph, domain);
917 if (IS_ERR(ptr: dom))
918 return PTR_ERR(ptr: dom);
919
920 *rate_limit = dom->rate_limit_us;
921 return 0;
922}
923
924static int scmi_dvfs_freq_set(const struct scmi_protocol_handle *ph, u32 domain,
925 unsigned long freq, bool poll)
926{
927 unsigned int level;
928 struct perf_dom_info *dom;
929
930 dom = scmi_perf_domain_lookup(ph, domain);
931 if (IS_ERR(ptr: dom))
932 return PTR_ERR(ptr: dom);
933
934 if (!dom->level_indexing_mode) {
935 level = freq / dom->mult_factor;
936 } else {
937 struct scmi_opp *opp;
938
939 opp = LOOKUP_BY_FREQ(dom->opps_by_freq,
940 freq / dom->mult_factor);
941 if (!opp)
942 return -EIO;
943
944 level = opp->level_index;
945 }
946
947 return __scmi_perf_level_set(ph, dom, level, poll);
948}
949
950static int scmi_dvfs_freq_get(const struct scmi_protocol_handle *ph, u32 domain,
951 unsigned long *freq, bool poll)
952{
953 int ret;
954 u32 level;
955 struct perf_dom_info *dom;
956
957 dom = scmi_perf_domain_lookup(ph, domain);
958 if (IS_ERR(ptr: dom))
959 return PTR_ERR(ptr: dom);
960
961 ret = __scmi_perf_level_get(ph, dom, level: &level, poll);
962 if (ret)
963 return ret;
964
965 if (!dom->level_indexing_mode) {
966 *freq = level * dom->mult_factor;
967 } else {
968 struct scmi_opp *opp;
969
970 opp = xa_load(&dom->opps_by_idx, index: level);
971 if (!opp)
972 return -EIO;
973
974 *freq = opp->indicative_freq * dom->mult_factor;
975 }
976
977 return ret;
978}
979
980static int scmi_dvfs_est_power_get(const struct scmi_protocol_handle *ph,
981 u32 domain, unsigned long *freq,
982 unsigned long *power)
983{
984 struct perf_dom_info *dom;
985 unsigned long opp_freq;
986 int idx, ret = -EINVAL;
987 struct scmi_opp *opp;
988
989 dom = scmi_perf_domain_lookup(ph, domain);
990 if (IS_ERR(ptr: dom))
991 return PTR_ERR(ptr: dom);
992
993 for (opp = dom->opp, idx = 0; idx < dom->opp_count; idx++, opp++) {
994 if (!dom->level_indexing_mode)
995 opp_freq = opp->perf * dom->mult_factor;
996 else
997 opp_freq = opp->indicative_freq * dom->mult_factor;
998
999 if (opp_freq < *freq)
1000 continue;
1001
1002 *freq = opp_freq;
1003 *power = opp->power;
1004 ret = 0;
1005 break;
1006 }
1007
1008 return ret;
1009}
1010
1011static bool scmi_fast_switch_possible(const struct scmi_protocol_handle *ph,
1012 u32 domain)
1013{
1014 struct perf_dom_info *dom;
1015
1016 dom = scmi_perf_domain_lookup(ph, domain);
1017 if (IS_ERR(ptr: dom))
1018 return false;
1019
1020 return dom->fc_info && dom->fc_info[PERF_FC_LEVEL].set_addr;
1021}
1022
1023static int scmi_fast_switch_rate_limit(const struct scmi_protocol_handle *ph,
1024 u32 domain, u32 *rate_limit)
1025{
1026 struct perf_dom_info *dom;
1027
1028 if (!rate_limit)
1029 return -EINVAL;
1030
1031 dom = scmi_perf_domain_lookup(ph, domain);
1032 if (IS_ERR(ptr: dom))
1033 return PTR_ERR(ptr: dom);
1034
1035 if (!dom->fc_info)
1036 return -EINVAL;
1037
1038 *rate_limit = dom->fc_info[PERF_FC_LEVEL].rate_limit;
1039 return 0;
1040}
1041
1042static enum scmi_power_scale
1043scmi_power_scale_get(const struct scmi_protocol_handle *ph)
1044{
1045 struct scmi_perf_info *pi = ph->get_priv(ph);
1046
1047 return pi->power_scale;
1048}
1049
1050static const struct scmi_perf_proto_ops perf_proto_ops = {
1051 .num_domains_get = scmi_perf_num_domains_get,
1052 .info_get = scmi_perf_info_get,
1053 .limits_set = scmi_perf_limits_set,
1054 .limits_get = scmi_perf_limits_get,
1055 .level_set = scmi_perf_level_set,
1056 .level_get = scmi_perf_level_get,
1057 .transition_latency_get = scmi_dvfs_transition_latency_get,
1058 .rate_limit_get = scmi_dvfs_rate_limit_get,
1059 .device_opps_add = scmi_dvfs_device_opps_add,
1060 .freq_set = scmi_dvfs_freq_set,
1061 .freq_get = scmi_dvfs_freq_get,
1062 .est_power_get = scmi_dvfs_est_power_get,
1063 .fast_switch_possible = scmi_fast_switch_possible,
1064 .fast_switch_rate_limit = scmi_fast_switch_rate_limit,
1065 .power_scale_get = scmi_power_scale_get,
1066};
1067
1068static bool scmi_perf_notify_supported(const struct scmi_protocol_handle *ph,
1069 u8 evt_id, u32 src_id)
1070{
1071 bool supported;
1072 struct perf_dom_info *dom;
1073
1074 if (evt_id >= ARRAY_SIZE(evt_2_cmd))
1075 return false;
1076
1077 dom = scmi_perf_domain_lookup(ph, domain: src_id);
1078 if (IS_ERR(ptr: dom))
1079 return false;
1080
1081 if (evt_id == SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED)
1082 supported = dom->perf_limit_notify;
1083 else
1084 supported = dom->perf_level_notify;
1085
1086 return supported;
1087}
1088
1089static int scmi_perf_set_notify_enabled(const struct scmi_protocol_handle *ph,
1090 u8 evt_id, u32 src_id, bool enable)
1091{
1092 int ret, cmd_id;
1093
1094 if (evt_id >= ARRAY_SIZE(evt_2_cmd))
1095 return -EINVAL;
1096
1097 cmd_id = evt_2_cmd[evt_id];
1098 ret = scmi_perf_level_limits_notify(ph, domain: src_id, message_id: cmd_id, enable);
1099 if (ret)
1100 pr_debug("FAIL_ENABLED - evt[%X] dom[%d] - ret:%d\n",
1101 evt_id, src_id, ret);
1102
1103 return ret;
1104}
1105
1106static int
1107scmi_perf_xlate_opp_to_freq(struct perf_dom_info *dom,
1108 unsigned int index, unsigned long *freq)
1109{
1110 struct scmi_opp *opp;
1111
1112 if (!dom || !freq)
1113 return -EINVAL;
1114
1115 if (!dom->level_indexing_mode) {
1116 opp = xa_load(&dom->opps_by_lvl, index);
1117 if (!opp)
1118 return -ENODEV;
1119
1120 *freq = opp->perf * dom->mult_factor;
1121 } else {
1122 opp = xa_load(&dom->opps_by_idx, index);
1123 if (!opp)
1124 return -ENODEV;
1125
1126 *freq = opp->indicative_freq * dom->mult_factor;
1127 }
1128
1129 return 0;
1130}
1131
1132static void *scmi_perf_fill_custom_report(const struct scmi_protocol_handle *ph,
1133 u8 evt_id, ktime_t timestamp,
1134 const void *payld, size_t payld_sz,
1135 void *report, u32 *src_id)
1136{
1137 int ret;
1138 void *rep = NULL;
1139 struct perf_dom_info *dom;
1140
1141 switch (evt_id) {
1142 case SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED:
1143 {
1144 const struct scmi_perf_limits_notify_payld *p = payld;
1145 struct scmi_perf_limits_report *r = report;
1146 unsigned long freq_min, freq_max;
1147
1148 if (sizeof(*p) != payld_sz)
1149 break;
1150
1151 r->timestamp = timestamp;
1152 r->agent_id = le32_to_cpu(p->agent_id);
1153 r->domain_id = le32_to_cpu(p->domain_id);
1154 r->range_max = le32_to_cpu(p->range_max);
1155 r->range_min = le32_to_cpu(p->range_min);
1156 /* Check if the reported domain exist at all */
1157 dom = scmi_perf_domain_lookup(ph, domain: r->domain_id);
1158 if (IS_ERR(ptr: dom))
1159 break;
1160 /*
1161 * Event will be reported from this point on...
1162 * ...even if, later, xlated frequencies were not retrieved.
1163 */
1164 *src_id = r->domain_id;
1165 rep = r;
1166
1167 ret = scmi_perf_xlate_opp_to_freq(dom, index: r->range_max, freq: &freq_max);
1168 if (ret)
1169 break;
1170
1171 ret = scmi_perf_xlate_opp_to_freq(dom, index: r->range_min, freq: &freq_min);
1172 if (ret)
1173 break;
1174
1175 /* Report translated freqs ONLY if both available */
1176 r->range_max_freq = freq_max;
1177 r->range_min_freq = freq_min;
1178
1179 break;
1180 }
1181 case SCMI_EVENT_PERFORMANCE_LEVEL_CHANGED:
1182 {
1183 const struct scmi_perf_level_notify_payld *p = payld;
1184 struct scmi_perf_level_report *r = report;
1185 unsigned long freq;
1186
1187 if (sizeof(*p) != payld_sz)
1188 break;
1189
1190 r->timestamp = timestamp;
1191 r->agent_id = le32_to_cpu(p->agent_id);
1192 r->domain_id = le32_to_cpu(p->domain_id);
1193 /* Report translated freqs ONLY if available */
1194 r->performance_level = le32_to_cpu(p->performance_level);
1195 /* Check if the reported domain exist at all */
1196 dom = scmi_perf_domain_lookup(ph, domain: r->domain_id);
1197 if (IS_ERR(ptr: dom))
1198 break;
1199 /*
1200 * Event will be reported from this point on...
1201 * ...even if, later, xlated frequencies were not retrieved.
1202 */
1203 *src_id = r->domain_id;
1204 rep = r;
1205
1206 /* Report translated freqs ONLY if available */
1207 ret = scmi_perf_xlate_opp_to_freq(dom, index: r->performance_level,
1208 freq: &freq);
1209 if (ret)
1210 break;
1211
1212 r->performance_level_freq = freq;
1213
1214 break;
1215 }
1216 default:
1217 break;
1218 }
1219
1220 return rep;
1221}
1222
1223static int scmi_perf_get_num_sources(const struct scmi_protocol_handle *ph)
1224{
1225 struct scmi_perf_info *pi = ph->get_priv(ph);
1226
1227 if (!pi)
1228 return -EINVAL;
1229
1230 return pi->num_domains;
1231}
1232
1233static const struct scmi_event perf_events[] = {
1234 {
1235 .id = SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED,
1236 .max_payld_sz = sizeof(struct scmi_perf_limits_notify_payld),
1237 .max_report_sz = sizeof(struct scmi_perf_limits_report),
1238 },
1239 {
1240 .id = SCMI_EVENT_PERFORMANCE_LEVEL_CHANGED,
1241 .max_payld_sz = sizeof(struct scmi_perf_level_notify_payld),
1242 .max_report_sz = sizeof(struct scmi_perf_level_report),
1243 },
1244};
1245
1246static const struct scmi_event_ops perf_event_ops = {
1247 .is_notify_supported = scmi_perf_notify_supported,
1248 .get_num_sources = scmi_perf_get_num_sources,
1249 .set_notify_enabled = scmi_perf_set_notify_enabled,
1250 .fill_custom_report = scmi_perf_fill_custom_report,
1251};
1252
1253static const struct scmi_protocol_events perf_protocol_events = {
1254 .queue_sz = SCMI_PROTO_QUEUE_SZ,
1255 .ops = &perf_event_ops,
1256 .evts = perf_events,
1257 .num_events = ARRAY_SIZE(perf_events),
1258};
1259
1260static int scmi_perf_protocol_init(const struct scmi_protocol_handle *ph)
1261{
1262 int domain, ret;
1263 u32 version;
1264 struct scmi_perf_info *pinfo;
1265
1266 ret = ph->xops->version_get(ph, &version);
1267 if (ret)
1268 return ret;
1269
1270 dev_dbg(ph->dev, "Performance Version %d.%d\n",
1271 PROTOCOL_REV_MAJOR(version), PROTOCOL_REV_MINOR(version));
1272
1273 pinfo = devm_kzalloc(dev: ph->dev, size: sizeof(*pinfo), GFP_KERNEL);
1274 if (!pinfo)
1275 return -ENOMEM;
1276
1277 pinfo->version = version;
1278
1279 ret = scmi_perf_attributes_get(ph, pi: pinfo);
1280 if (ret)
1281 return ret;
1282
1283 pinfo->dom_info = devm_kcalloc(dev: ph->dev, n: pinfo->num_domains,
1284 size: sizeof(*pinfo->dom_info), GFP_KERNEL);
1285 if (!pinfo->dom_info)
1286 return -ENOMEM;
1287
1288 for (domain = 0; domain < pinfo->num_domains; domain++) {
1289 struct perf_dom_info *dom = pinfo->dom_info + domain;
1290
1291 dom->id = domain;
1292 scmi_perf_domain_attributes_get(ph, dom_info: dom, notify_lim_cmd: pinfo->notify_lim_cmd,
1293 notify_lvl_cmd: pinfo->notify_lvl_cmd, version);
1294 scmi_perf_describe_levels_get(ph, perf_dom: dom, version);
1295
1296 if (dom->perf_fastchannels)
1297 scmi_perf_domain_init_fc(ph, dom);
1298 }
1299
1300 ret = devm_add_action_or_reset(ph->dev, scmi_perf_xa_destroy, pinfo);
1301 if (ret)
1302 return ret;
1303
1304 return ph->set_priv(ph, pinfo, version);
1305}
1306
1307static const struct scmi_protocol scmi_perf = {
1308 .id = SCMI_PROTOCOL_PERF,
1309 .owner = THIS_MODULE,
1310 .instance_init = &scmi_perf_protocol_init,
1311 .ops = &perf_proto_ops,
1312 .events = &perf_protocol_events,
1313 .supported_version = SCMI_PROTOCOL_SUPPORTED_VERSION,
1314};
1315
1316DEFINE_SCMI_PROTOCOL_REGISTER_UNREGISTER(perf, scmi_perf)
1317

source code of linux/drivers/firmware/arm_scmi/perf.c