1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Interconnect framework core driver |
4 | * |
5 | * Copyright (c) 2017-2019, Linaro Ltd. |
6 | * Author: Georgi Djakov <georgi.djakov@linaro.org> |
7 | */ |
8 | |
9 | #include <linux/debugfs.h> |
10 | #include <linux/device.h> |
11 | #include <linux/idr.h> |
12 | #include <linux/init.h> |
13 | #include <linux/interconnect.h> |
14 | #include <linux/interconnect-provider.h> |
15 | #include <linux/list.h> |
16 | #include <linux/mutex.h> |
17 | #include <linux/slab.h> |
18 | #include <linux/of.h> |
19 | #include <linux/overflow.h> |
20 | |
21 | #include "internal.h" |
22 | |
23 | #define CREATE_TRACE_POINTS |
24 | #include "trace.h" |
25 | |
26 | static DEFINE_IDR(icc_idr); |
27 | static LIST_HEAD(icc_providers); |
28 | static int providers_count; |
29 | static bool synced_state; |
30 | static DEFINE_MUTEX(icc_lock); |
31 | static DEFINE_MUTEX(icc_bw_lock); |
32 | static struct dentry *icc_debugfs_dir; |
33 | |
34 | static void icc_summary_show_one(struct seq_file *s, struct icc_node *n) |
35 | { |
36 | if (!n) |
37 | return; |
38 | |
39 | seq_printf(m: s, fmt: "%-42s %12u %12u\n" , |
40 | n->name, n->avg_bw, n->peak_bw); |
41 | } |
42 | |
43 | static int icc_summary_show(struct seq_file *s, void *data) |
44 | { |
45 | struct icc_provider *provider; |
46 | |
47 | seq_puts(m: s, s: " node tag avg peak\n" ); |
48 | seq_puts(m: s, s: "--------------------------------------------------------------------\n" ); |
49 | |
50 | mutex_lock(&icc_lock); |
51 | |
52 | list_for_each_entry(provider, &icc_providers, provider_list) { |
53 | struct icc_node *n; |
54 | |
55 | list_for_each_entry(n, &provider->nodes, node_list) { |
56 | struct icc_req *r; |
57 | |
58 | icc_summary_show_one(s, n); |
59 | hlist_for_each_entry(r, &n->req_list, req_node) { |
60 | u32 avg_bw = 0, peak_bw = 0; |
61 | |
62 | if (!r->dev) |
63 | continue; |
64 | |
65 | if (r->enabled) { |
66 | avg_bw = r->avg_bw; |
67 | peak_bw = r->peak_bw; |
68 | } |
69 | |
70 | seq_printf(m: s, fmt: " %-27s %12u %12u %12u\n" , |
71 | dev_name(dev: r->dev), r->tag, avg_bw, peak_bw); |
72 | } |
73 | } |
74 | } |
75 | |
76 | mutex_unlock(lock: &icc_lock); |
77 | |
78 | return 0; |
79 | } |
80 | DEFINE_SHOW_ATTRIBUTE(icc_summary); |
81 | |
82 | static void icc_graph_show_link(struct seq_file *s, int level, |
83 | struct icc_node *n, struct icc_node *m) |
84 | { |
85 | seq_printf(m: s, fmt: "%s\"%d:%s\" -> \"%d:%s\"\n" , |
86 | level == 2 ? "\t\t" : "\t" , |
87 | n->id, n->name, m->id, m->name); |
88 | } |
89 | |
90 | static void icc_graph_show_node(struct seq_file *s, struct icc_node *n) |
91 | { |
92 | seq_printf(m: s, fmt: "\t\t\"%d:%s\" [label=\"%d:%s" , |
93 | n->id, n->name, n->id, n->name); |
94 | seq_printf(m: s, fmt: "\n\t\t\t|avg_bw=%ukBps" , n->avg_bw); |
95 | seq_printf(m: s, fmt: "\n\t\t\t|peak_bw=%ukBps" , n->peak_bw); |
96 | seq_puts(m: s, s: "\"]\n" ); |
97 | } |
98 | |
99 | static int icc_graph_show(struct seq_file *s, void *data) |
100 | { |
101 | struct icc_provider *provider; |
102 | struct icc_node *n; |
103 | int cluster_index = 0; |
104 | int i; |
105 | |
106 | seq_puts(m: s, s: "digraph {\n\trankdir = LR\n\tnode [shape = record]\n" ); |
107 | mutex_lock(&icc_lock); |
108 | |
109 | /* draw providers as cluster subgraphs */ |
110 | cluster_index = 0; |
111 | list_for_each_entry(provider, &icc_providers, provider_list) { |
112 | seq_printf(m: s, fmt: "\tsubgraph cluster_%d {\n" , ++cluster_index); |
113 | if (provider->dev) |
114 | seq_printf(m: s, fmt: "\t\tlabel = \"%s\"\n" , |
115 | dev_name(dev: provider->dev)); |
116 | |
117 | /* draw nodes */ |
118 | list_for_each_entry(n, &provider->nodes, node_list) |
119 | icc_graph_show_node(s, n); |
120 | |
121 | /* draw internal links */ |
122 | list_for_each_entry(n, &provider->nodes, node_list) |
123 | for (i = 0; i < n->num_links; ++i) |
124 | if (n->provider == n->links[i]->provider) |
125 | icc_graph_show_link(s, level: 2, n, |
126 | m: n->links[i]); |
127 | |
128 | seq_puts(m: s, s: "\t}\n" ); |
129 | } |
130 | |
131 | /* draw external links */ |
132 | list_for_each_entry(provider, &icc_providers, provider_list) |
133 | list_for_each_entry(n, &provider->nodes, node_list) |
134 | for (i = 0; i < n->num_links; ++i) |
135 | if (n->provider != n->links[i]->provider) |
136 | icc_graph_show_link(s, level: 1, n, |
137 | m: n->links[i]); |
138 | |
139 | mutex_unlock(lock: &icc_lock); |
140 | seq_puts(m: s, s: "}" ); |
141 | |
142 | return 0; |
143 | } |
144 | DEFINE_SHOW_ATTRIBUTE(icc_graph); |
145 | |
146 | static struct icc_node *node_find(const int id) |
147 | { |
148 | return idr_find(&icc_idr, id); |
149 | } |
150 | |
151 | static struct icc_node *node_find_by_name(const char *name) |
152 | { |
153 | struct icc_provider *provider; |
154 | struct icc_node *n; |
155 | |
156 | list_for_each_entry(provider, &icc_providers, provider_list) { |
157 | list_for_each_entry(n, &provider->nodes, node_list) { |
158 | if (!strcmp(n->name, name)) |
159 | return n; |
160 | } |
161 | } |
162 | |
163 | return NULL; |
164 | } |
165 | |
166 | static struct icc_path *path_init(struct device *dev, struct icc_node *dst, |
167 | ssize_t num_nodes) |
168 | { |
169 | struct icc_node *node = dst; |
170 | struct icc_path *path; |
171 | int i; |
172 | |
173 | path = kzalloc(struct_size(path, reqs, num_nodes), GFP_KERNEL); |
174 | if (!path) |
175 | return ERR_PTR(error: -ENOMEM); |
176 | |
177 | path->num_nodes = num_nodes; |
178 | |
179 | mutex_lock(&icc_bw_lock); |
180 | |
181 | for (i = num_nodes - 1; i >= 0; i--) { |
182 | node->provider->users++; |
183 | hlist_add_head(n: &path->reqs[i].req_node, h: &node->req_list); |
184 | path->reqs[i].node = node; |
185 | path->reqs[i].dev = dev; |
186 | path->reqs[i].enabled = true; |
187 | /* reference to previous node was saved during path traversal */ |
188 | node = node->reverse; |
189 | } |
190 | |
191 | mutex_unlock(lock: &icc_bw_lock); |
192 | |
193 | return path; |
194 | } |
195 | |
196 | static struct icc_path *path_find(struct device *dev, struct icc_node *src, |
197 | struct icc_node *dst) |
198 | { |
199 | struct icc_path *path = ERR_PTR(error: -EPROBE_DEFER); |
200 | struct icc_node *n, *node = NULL; |
201 | struct list_head traverse_list; |
202 | struct list_head edge_list; |
203 | struct list_head visited_list; |
204 | size_t i, depth = 1; |
205 | bool found = false; |
206 | |
207 | INIT_LIST_HEAD(list: &traverse_list); |
208 | INIT_LIST_HEAD(list: &edge_list); |
209 | INIT_LIST_HEAD(list: &visited_list); |
210 | |
211 | list_add(new: &src->search_list, head: &traverse_list); |
212 | src->reverse = NULL; |
213 | |
214 | do { |
215 | list_for_each_entry_safe(node, n, &traverse_list, search_list) { |
216 | if (node == dst) { |
217 | found = true; |
218 | list_splice_init(list: &edge_list, head: &visited_list); |
219 | list_splice_init(list: &traverse_list, head: &visited_list); |
220 | break; |
221 | } |
222 | for (i = 0; i < node->num_links; i++) { |
223 | struct icc_node *tmp = node->links[i]; |
224 | |
225 | if (!tmp) { |
226 | path = ERR_PTR(error: -ENOENT); |
227 | goto out; |
228 | } |
229 | |
230 | if (tmp->is_traversed) |
231 | continue; |
232 | |
233 | tmp->is_traversed = true; |
234 | tmp->reverse = node; |
235 | list_add_tail(new: &tmp->search_list, head: &edge_list); |
236 | } |
237 | } |
238 | |
239 | if (found) |
240 | break; |
241 | |
242 | list_splice_init(list: &traverse_list, head: &visited_list); |
243 | list_splice_init(list: &edge_list, head: &traverse_list); |
244 | |
245 | /* count the hops including the source */ |
246 | depth++; |
247 | |
248 | } while (!list_empty(head: &traverse_list)); |
249 | |
250 | out: |
251 | |
252 | /* reset the traversed state */ |
253 | list_for_each_entry_reverse(n, &visited_list, search_list) |
254 | n->is_traversed = false; |
255 | |
256 | if (found) |
257 | path = path_init(dev, dst, num_nodes: depth); |
258 | |
259 | return path; |
260 | } |
261 | |
262 | /* |
263 | * We want the path to honor all bandwidth requests, so the average and peak |
264 | * bandwidth requirements from each consumer are aggregated at each node. |
265 | * The aggregation is platform specific, so each platform can customize it by |
266 | * implementing its own aggregate() function. |
267 | */ |
268 | |
269 | static int aggregate_requests(struct icc_node *node) |
270 | { |
271 | struct icc_provider *p = node->provider; |
272 | struct icc_req *r; |
273 | u32 avg_bw, peak_bw; |
274 | |
275 | node->avg_bw = 0; |
276 | node->peak_bw = 0; |
277 | |
278 | if (p->pre_aggregate) |
279 | p->pre_aggregate(node); |
280 | |
281 | hlist_for_each_entry(r, &node->req_list, req_node) { |
282 | if (r->enabled) { |
283 | avg_bw = r->avg_bw; |
284 | peak_bw = r->peak_bw; |
285 | } else { |
286 | avg_bw = 0; |
287 | peak_bw = 0; |
288 | } |
289 | p->aggregate(node, r->tag, avg_bw, peak_bw, |
290 | &node->avg_bw, &node->peak_bw); |
291 | |
292 | /* during boot use the initial bandwidth as a floor value */ |
293 | if (!synced_state) { |
294 | node->avg_bw = max(node->avg_bw, node->init_avg); |
295 | node->peak_bw = max(node->peak_bw, node->init_peak); |
296 | } |
297 | } |
298 | |
299 | return 0; |
300 | } |
301 | |
302 | static int apply_constraints(struct icc_path *path) |
303 | { |
304 | struct icc_node *next, *prev = NULL; |
305 | struct icc_provider *p; |
306 | int ret = -EINVAL; |
307 | int i; |
308 | |
309 | for (i = 0; i < path->num_nodes; i++) { |
310 | next = path->reqs[i].node; |
311 | p = next->provider; |
312 | |
313 | /* both endpoints should be valid master-slave pairs */ |
314 | if (!prev || (p != prev->provider && !p->inter_set)) { |
315 | prev = next; |
316 | continue; |
317 | } |
318 | |
319 | /* set the constraints */ |
320 | ret = p->set(prev, next); |
321 | if (ret) |
322 | goto out; |
323 | |
324 | prev = next; |
325 | } |
326 | out: |
327 | return ret; |
328 | } |
329 | |
330 | int icc_std_aggregate(struct icc_node *node, u32 tag, u32 avg_bw, |
331 | u32 peak_bw, u32 *agg_avg, u32 *agg_peak) |
332 | { |
333 | *agg_avg += avg_bw; |
334 | *agg_peak = max(*agg_peak, peak_bw); |
335 | |
336 | return 0; |
337 | } |
338 | EXPORT_SYMBOL_GPL(icc_std_aggregate); |
339 | |
340 | /* of_icc_xlate_onecell() - Translate function using a single index. |
341 | * @spec: OF phandle args to map into an interconnect node. |
342 | * @data: private data (pointer to struct icc_onecell_data) |
343 | * |
344 | * This is a generic translate function that can be used to model simple |
345 | * interconnect providers that have one device tree node and provide |
346 | * multiple interconnect nodes. A single cell is used as an index into |
347 | * an array of icc nodes specified in the icc_onecell_data struct when |
348 | * registering the provider. |
349 | */ |
350 | struct icc_node *of_icc_xlate_onecell(const struct of_phandle_args *spec, |
351 | void *data) |
352 | { |
353 | struct icc_onecell_data *icc_data = data; |
354 | unsigned int idx = spec->args[0]; |
355 | |
356 | if (idx >= icc_data->num_nodes) { |
357 | pr_err("%s: invalid index %u\n" , __func__, idx); |
358 | return ERR_PTR(error: -EINVAL); |
359 | } |
360 | |
361 | return icc_data->nodes[idx]; |
362 | } |
363 | EXPORT_SYMBOL_GPL(of_icc_xlate_onecell); |
364 | |
365 | /** |
366 | * of_icc_get_from_provider() - Look-up interconnect node |
367 | * @spec: OF phandle args to use for look-up |
368 | * |
369 | * Looks for interconnect provider under the node specified by @spec and if |
370 | * found, uses xlate function of the provider to map phandle args to node. |
371 | * |
372 | * Returns a valid pointer to struct icc_node_data on success or ERR_PTR() |
373 | * on failure. |
374 | */ |
375 | struct icc_node_data *of_icc_get_from_provider(const struct of_phandle_args *spec) |
376 | { |
377 | struct icc_node *node = ERR_PTR(error: -EPROBE_DEFER); |
378 | struct icc_node_data *data = NULL; |
379 | struct icc_provider *provider; |
380 | |
381 | if (!spec) |
382 | return ERR_PTR(error: -EINVAL); |
383 | |
384 | mutex_lock(&icc_lock); |
385 | list_for_each_entry(provider, &icc_providers, provider_list) { |
386 | if (provider->dev->of_node == spec->np) { |
387 | if (provider->xlate_extended) { |
388 | data = provider->xlate_extended(spec, provider->data); |
389 | if (!IS_ERR(ptr: data)) { |
390 | node = data->node; |
391 | break; |
392 | } |
393 | } else { |
394 | node = provider->xlate(spec, provider->data); |
395 | if (!IS_ERR(ptr: node)) |
396 | break; |
397 | } |
398 | } |
399 | } |
400 | mutex_unlock(lock: &icc_lock); |
401 | |
402 | if (!node) |
403 | return ERR_PTR(error: -EINVAL); |
404 | |
405 | if (IS_ERR(ptr: node)) |
406 | return ERR_CAST(ptr: node); |
407 | |
408 | if (!data) { |
409 | data = kzalloc(size: sizeof(*data), GFP_KERNEL); |
410 | if (!data) |
411 | return ERR_PTR(error: -ENOMEM); |
412 | data->node = node; |
413 | } |
414 | |
415 | return data; |
416 | } |
417 | EXPORT_SYMBOL_GPL(of_icc_get_from_provider); |
418 | |
419 | static void devm_icc_release(struct device *dev, void *res) |
420 | { |
421 | icc_put(path: *(struct icc_path **)res); |
422 | } |
423 | |
424 | struct icc_path *devm_of_icc_get(struct device *dev, const char *name) |
425 | { |
426 | struct icc_path **ptr, *path; |
427 | |
428 | ptr = devres_alloc(devm_icc_release, sizeof(*ptr), GFP_KERNEL); |
429 | if (!ptr) |
430 | return ERR_PTR(error: -ENOMEM); |
431 | |
432 | path = of_icc_get(dev, name); |
433 | if (!IS_ERR(ptr: path)) { |
434 | *ptr = path; |
435 | devres_add(dev, res: ptr); |
436 | } else { |
437 | devres_free(res: ptr); |
438 | } |
439 | |
440 | return path; |
441 | } |
442 | EXPORT_SYMBOL_GPL(devm_of_icc_get); |
443 | |
444 | /** |
445 | * of_icc_get_by_index() - get a path handle from a DT node based on index |
446 | * @dev: device pointer for the consumer device |
447 | * @idx: interconnect path index |
448 | * |
449 | * This function will search for a path between two endpoints and return an |
450 | * icc_path handle on success. Use icc_put() to release constraints when they |
451 | * are not needed anymore. |
452 | * If the interconnect API is disabled, NULL is returned and the consumer |
453 | * drivers will still build. Drivers are free to handle this specifically, |
454 | * but they don't have to. |
455 | * |
456 | * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned |
457 | * when the API is disabled or the "interconnects" DT property is missing. |
458 | */ |
459 | struct icc_path *of_icc_get_by_index(struct device *dev, int idx) |
460 | { |
461 | struct icc_path *path; |
462 | struct icc_node_data *src_data, *dst_data; |
463 | struct device_node *np; |
464 | struct of_phandle_args src_args, dst_args; |
465 | int ret; |
466 | |
467 | if (!dev || !dev->of_node) |
468 | return ERR_PTR(error: -ENODEV); |
469 | |
470 | np = dev->of_node; |
471 | |
472 | /* |
473 | * When the consumer DT node do not have "interconnects" property |
474 | * return a NULL path to skip setting constraints. |
475 | */ |
476 | if (!of_property_present(np, propname: "interconnects" )) |
477 | return NULL; |
478 | |
479 | /* |
480 | * We use a combination of phandle and specifier for endpoint. For now |
481 | * lets support only global ids and extend this in the future if needed |
482 | * without breaking DT compatibility. |
483 | */ |
484 | ret = of_parse_phandle_with_args(np, list_name: "interconnects" , |
485 | cells_name: "#interconnect-cells" , index: idx * 2, |
486 | out_args: &src_args); |
487 | if (ret) |
488 | return ERR_PTR(error: ret); |
489 | |
490 | of_node_put(node: src_args.np); |
491 | |
492 | ret = of_parse_phandle_with_args(np, list_name: "interconnects" , |
493 | cells_name: "#interconnect-cells" , index: idx * 2 + 1, |
494 | out_args: &dst_args); |
495 | if (ret) |
496 | return ERR_PTR(error: ret); |
497 | |
498 | of_node_put(node: dst_args.np); |
499 | |
500 | src_data = of_icc_get_from_provider(&src_args); |
501 | |
502 | if (IS_ERR(ptr: src_data)) { |
503 | dev_err_probe(dev, err: PTR_ERR(ptr: src_data), fmt: "error finding src node\n" ); |
504 | return ERR_CAST(ptr: src_data); |
505 | } |
506 | |
507 | dst_data = of_icc_get_from_provider(&dst_args); |
508 | |
509 | if (IS_ERR(ptr: dst_data)) { |
510 | dev_err_probe(dev, err: PTR_ERR(ptr: dst_data), fmt: "error finding dst node\n" ); |
511 | kfree(objp: src_data); |
512 | return ERR_CAST(ptr: dst_data); |
513 | } |
514 | |
515 | mutex_lock(&icc_lock); |
516 | path = path_find(dev, src: src_data->node, dst: dst_data->node); |
517 | mutex_unlock(lock: &icc_lock); |
518 | if (IS_ERR(ptr: path)) { |
519 | dev_err(dev, "%s: invalid path=%ld\n" , __func__, PTR_ERR(path)); |
520 | goto free_icc_data; |
521 | } |
522 | |
523 | if (src_data->tag && src_data->tag == dst_data->tag) |
524 | icc_set_tag(path, tag: src_data->tag); |
525 | |
526 | path->name = kasprintf(GFP_KERNEL, fmt: "%s-%s" , |
527 | src_data->node->name, dst_data->node->name); |
528 | if (!path->name) { |
529 | kfree(objp: path); |
530 | path = ERR_PTR(error: -ENOMEM); |
531 | } |
532 | |
533 | free_icc_data: |
534 | kfree(objp: src_data); |
535 | kfree(objp: dst_data); |
536 | return path; |
537 | } |
538 | EXPORT_SYMBOL_GPL(of_icc_get_by_index); |
539 | |
540 | /** |
541 | * of_icc_get() - get a path handle from a DT node based on name |
542 | * @dev: device pointer for the consumer device |
543 | * @name: interconnect path name |
544 | * |
545 | * This function will search for a path between two endpoints and return an |
546 | * icc_path handle on success. Use icc_put() to release constraints when they |
547 | * are not needed anymore. |
548 | * If the interconnect API is disabled, NULL is returned and the consumer |
549 | * drivers will still build. Drivers are free to handle this specifically, |
550 | * but they don't have to. |
551 | * |
552 | * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned |
553 | * when the API is disabled or the "interconnects" DT property is missing. |
554 | */ |
555 | struct icc_path *of_icc_get(struct device *dev, const char *name) |
556 | { |
557 | struct device_node *np; |
558 | int idx = 0; |
559 | |
560 | if (!dev || !dev->of_node) |
561 | return ERR_PTR(error: -ENODEV); |
562 | |
563 | np = dev->of_node; |
564 | |
565 | /* |
566 | * When the consumer DT node do not have "interconnects" property |
567 | * return a NULL path to skip setting constraints. |
568 | */ |
569 | if (!of_property_present(np, propname: "interconnects" )) |
570 | return NULL; |
571 | |
572 | /* |
573 | * We use a combination of phandle and specifier for endpoint. For now |
574 | * lets support only global ids and extend this in the future if needed |
575 | * without breaking DT compatibility. |
576 | */ |
577 | if (name) { |
578 | idx = of_property_match_string(np, propname: "interconnect-names" , string: name); |
579 | if (idx < 0) |
580 | return ERR_PTR(error: idx); |
581 | } |
582 | |
583 | return of_icc_get_by_index(dev, idx); |
584 | } |
585 | EXPORT_SYMBOL_GPL(of_icc_get); |
586 | |
587 | /** |
588 | * icc_get() - get a path handle between two endpoints |
589 | * @dev: device pointer for the consumer device |
590 | * @src: source node name |
591 | * @dst: destination node name |
592 | * |
593 | * This function will search for a path between two endpoints and return an |
594 | * icc_path handle on success. Use icc_put() to release constraints when they |
595 | * are not needed anymore. |
596 | * |
597 | * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned |
598 | * when the API is disabled. |
599 | */ |
600 | struct icc_path *icc_get(struct device *dev, const char *src, const char *dst) |
601 | { |
602 | struct icc_node *src_node, *dst_node; |
603 | struct icc_path *path = ERR_PTR(error: -EPROBE_DEFER); |
604 | |
605 | mutex_lock(&icc_lock); |
606 | |
607 | src_node = node_find_by_name(name: src); |
608 | if (!src_node) { |
609 | dev_err(dev, "%s: invalid src=%s\n" , __func__, src); |
610 | goto out; |
611 | } |
612 | |
613 | dst_node = node_find_by_name(name: dst); |
614 | if (!dst_node) { |
615 | dev_err(dev, "%s: invalid dst=%s\n" , __func__, dst); |
616 | goto out; |
617 | } |
618 | |
619 | path = path_find(dev, src: src_node, dst: dst_node); |
620 | if (IS_ERR(ptr: path)) { |
621 | dev_err(dev, "%s: invalid path=%ld\n" , __func__, PTR_ERR(path)); |
622 | goto out; |
623 | } |
624 | |
625 | path->name = kasprintf(GFP_KERNEL, fmt: "%s-%s" , src_node->name, dst_node->name); |
626 | if (!path->name) { |
627 | kfree(objp: path); |
628 | path = ERR_PTR(error: -ENOMEM); |
629 | } |
630 | out: |
631 | mutex_unlock(lock: &icc_lock); |
632 | return path; |
633 | } |
634 | |
635 | /** |
636 | * icc_set_tag() - set an optional tag on a path |
637 | * @path: the path we want to tag |
638 | * @tag: the tag value |
639 | * |
640 | * This function allows consumers to append a tag to the requests associated |
641 | * with a path, so that a different aggregation could be done based on this tag. |
642 | */ |
643 | void icc_set_tag(struct icc_path *path, u32 tag) |
644 | { |
645 | int i; |
646 | |
647 | if (!path) |
648 | return; |
649 | |
650 | mutex_lock(&icc_lock); |
651 | |
652 | for (i = 0; i < path->num_nodes; i++) |
653 | path->reqs[i].tag = tag; |
654 | |
655 | mutex_unlock(lock: &icc_lock); |
656 | } |
657 | EXPORT_SYMBOL_GPL(icc_set_tag); |
658 | |
659 | /** |
660 | * icc_get_name() - Get name of the icc path |
661 | * @path: interconnect path |
662 | * |
663 | * This function is used by an interconnect consumer to get the name of the icc |
664 | * path. |
665 | * |
666 | * Returns a valid pointer on success, or NULL otherwise. |
667 | */ |
668 | const char *icc_get_name(struct icc_path *path) |
669 | { |
670 | if (!path) |
671 | return NULL; |
672 | |
673 | return path->name; |
674 | } |
675 | EXPORT_SYMBOL_GPL(icc_get_name); |
676 | |
677 | /** |
678 | * icc_set_bw() - set bandwidth constraints on an interconnect path |
679 | * @path: interconnect path |
680 | * @avg_bw: average bandwidth in kilobytes per second |
681 | * @peak_bw: peak bandwidth in kilobytes per second |
682 | * |
683 | * This function is used by an interconnect consumer to express its own needs |
684 | * in terms of bandwidth for a previously requested path between two endpoints. |
685 | * The requests are aggregated and each node is updated accordingly. The entire |
686 | * path is locked by a mutex to ensure that the set() is completed. |
687 | * The @path can be NULL when the "interconnects" DT properties is missing, |
688 | * which will mean that no constraints will be set. |
689 | * |
690 | * Returns 0 on success, or an appropriate error code otherwise. |
691 | */ |
692 | int icc_set_bw(struct icc_path *path, u32 avg_bw, u32 peak_bw) |
693 | { |
694 | struct icc_node *node; |
695 | u32 old_avg, old_peak; |
696 | size_t i; |
697 | int ret; |
698 | |
699 | if (!path) |
700 | return 0; |
701 | |
702 | if (WARN_ON(IS_ERR(path) || !path->num_nodes)) |
703 | return -EINVAL; |
704 | |
705 | mutex_lock(&icc_bw_lock); |
706 | |
707 | old_avg = path->reqs[0].avg_bw; |
708 | old_peak = path->reqs[0].peak_bw; |
709 | |
710 | for (i = 0; i < path->num_nodes; i++) { |
711 | node = path->reqs[i].node; |
712 | |
713 | /* update the consumer request for this path */ |
714 | path->reqs[i].avg_bw = avg_bw; |
715 | path->reqs[i].peak_bw = peak_bw; |
716 | |
717 | /* aggregate requests for this node */ |
718 | aggregate_requests(node); |
719 | |
720 | trace_icc_set_bw(p: path, n: node, i, avg_bw, peak_bw); |
721 | } |
722 | |
723 | ret = apply_constraints(path); |
724 | if (ret) { |
725 | pr_debug("interconnect: error applying constraints (%d)\n" , |
726 | ret); |
727 | |
728 | for (i = 0; i < path->num_nodes; i++) { |
729 | node = path->reqs[i].node; |
730 | path->reqs[i].avg_bw = old_avg; |
731 | path->reqs[i].peak_bw = old_peak; |
732 | aggregate_requests(node); |
733 | } |
734 | apply_constraints(path); |
735 | } |
736 | |
737 | mutex_unlock(lock: &icc_bw_lock); |
738 | |
739 | trace_icc_set_bw_end(p: path, ret); |
740 | |
741 | return ret; |
742 | } |
743 | EXPORT_SYMBOL_GPL(icc_set_bw); |
744 | |
745 | static int __icc_enable(struct icc_path *path, bool enable) |
746 | { |
747 | int i; |
748 | |
749 | if (!path) |
750 | return 0; |
751 | |
752 | if (WARN_ON(IS_ERR(path) || !path->num_nodes)) |
753 | return -EINVAL; |
754 | |
755 | mutex_lock(&icc_lock); |
756 | |
757 | for (i = 0; i < path->num_nodes; i++) |
758 | path->reqs[i].enabled = enable; |
759 | |
760 | mutex_unlock(lock: &icc_lock); |
761 | |
762 | return icc_set_bw(path, path->reqs[0].avg_bw, |
763 | path->reqs[0].peak_bw); |
764 | } |
765 | |
766 | int icc_enable(struct icc_path *path) |
767 | { |
768 | return __icc_enable(path, enable: true); |
769 | } |
770 | EXPORT_SYMBOL_GPL(icc_enable); |
771 | |
772 | int icc_disable(struct icc_path *path) |
773 | { |
774 | return __icc_enable(path, enable: false); |
775 | } |
776 | EXPORT_SYMBOL_GPL(icc_disable); |
777 | |
778 | /** |
779 | * icc_put() - release the reference to the icc_path |
780 | * @path: interconnect path |
781 | * |
782 | * Use this function to release the constraints on a path when the path is |
783 | * no longer needed. The constraints will be re-aggregated. |
784 | */ |
785 | void icc_put(struct icc_path *path) |
786 | { |
787 | struct icc_node *node; |
788 | size_t i; |
789 | int ret; |
790 | |
791 | if (!path || WARN_ON(IS_ERR(path))) |
792 | return; |
793 | |
794 | ret = icc_set_bw(path, 0, 0); |
795 | if (ret) |
796 | pr_err("%s: error (%d)\n" , __func__, ret); |
797 | |
798 | mutex_lock(&icc_lock); |
799 | mutex_lock(&icc_bw_lock); |
800 | |
801 | for (i = 0; i < path->num_nodes; i++) { |
802 | node = path->reqs[i].node; |
803 | hlist_del(n: &path->reqs[i].req_node); |
804 | if (!WARN_ON(!node->provider->users)) |
805 | node->provider->users--; |
806 | } |
807 | |
808 | mutex_unlock(lock: &icc_bw_lock); |
809 | mutex_unlock(lock: &icc_lock); |
810 | |
811 | kfree_const(x: path->name); |
812 | kfree(objp: path); |
813 | } |
814 | EXPORT_SYMBOL_GPL(icc_put); |
815 | |
816 | static struct icc_node *icc_node_create_nolock(int id) |
817 | { |
818 | struct icc_node *node; |
819 | |
820 | /* check if node already exists */ |
821 | node = node_find(id); |
822 | if (node) |
823 | return node; |
824 | |
825 | node = kzalloc(size: sizeof(*node), GFP_KERNEL); |
826 | if (!node) |
827 | return ERR_PTR(error: -ENOMEM); |
828 | |
829 | id = idr_alloc(&icc_idr, ptr: node, start: id, end: id + 1, GFP_KERNEL); |
830 | if (id < 0) { |
831 | WARN(1, "%s: couldn't get idr\n" , __func__); |
832 | kfree(objp: node); |
833 | return ERR_PTR(error: id); |
834 | } |
835 | |
836 | node->id = id; |
837 | |
838 | return node; |
839 | } |
840 | |
841 | /** |
842 | * icc_node_create() - create a node |
843 | * @id: node id |
844 | * |
845 | * Return: icc_node pointer on success, or ERR_PTR() on error |
846 | */ |
847 | struct icc_node *icc_node_create(int id) |
848 | { |
849 | struct icc_node *node; |
850 | |
851 | mutex_lock(&icc_lock); |
852 | |
853 | node = icc_node_create_nolock(id); |
854 | |
855 | mutex_unlock(lock: &icc_lock); |
856 | |
857 | return node; |
858 | } |
859 | EXPORT_SYMBOL_GPL(icc_node_create); |
860 | |
861 | /** |
862 | * icc_node_destroy() - destroy a node |
863 | * @id: node id |
864 | */ |
865 | void icc_node_destroy(int id) |
866 | { |
867 | struct icc_node *node; |
868 | |
869 | mutex_lock(&icc_lock); |
870 | |
871 | node = node_find(id); |
872 | if (node) { |
873 | idr_remove(&icc_idr, id: node->id); |
874 | WARN_ON(!hlist_empty(&node->req_list)); |
875 | } |
876 | |
877 | mutex_unlock(lock: &icc_lock); |
878 | |
879 | if (!node) |
880 | return; |
881 | |
882 | kfree(objp: node->links); |
883 | kfree(objp: node); |
884 | } |
885 | EXPORT_SYMBOL_GPL(icc_node_destroy); |
886 | |
887 | /** |
888 | * icc_link_create() - create a link between two nodes |
889 | * @node: source node id |
890 | * @dst_id: destination node id |
891 | * |
892 | * Create a link between two nodes. The nodes might belong to different |
893 | * interconnect providers and the @dst_id node might not exist (if the |
894 | * provider driver has not probed yet). So just create the @dst_id node |
895 | * and when the actual provider driver is probed, the rest of the node |
896 | * data is filled. |
897 | * |
898 | * Return: 0 on success, or an error code otherwise |
899 | */ |
900 | int icc_link_create(struct icc_node *node, const int dst_id) |
901 | { |
902 | struct icc_node *dst; |
903 | struct icc_node **new; |
904 | int ret = 0; |
905 | |
906 | if (!node->provider) |
907 | return -EINVAL; |
908 | |
909 | mutex_lock(&icc_lock); |
910 | |
911 | dst = node_find(id: dst_id); |
912 | if (!dst) { |
913 | dst = icc_node_create_nolock(id: dst_id); |
914 | |
915 | if (IS_ERR(ptr: dst)) { |
916 | ret = PTR_ERR(ptr: dst); |
917 | goto out; |
918 | } |
919 | } |
920 | |
921 | new = krealloc(objp: node->links, |
922 | new_size: (node->num_links + 1) * sizeof(*node->links), |
923 | GFP_KERNEL); |
924 | if (!new) { |
925 | ret = -ENOMEM; |
926 | goto out; |
927 | } |
928 | |
929 | node->links = new; |
930 | node->links[node->num_links++] = dst; |
931 | |
932 | out: |
933 | mutex_unlock(lock: &icc_lock); |
934 | |
935 | return ret; |
936 | } |
937 | EXPORT_SYMBOL_GPL(icc_link_create); |
938 | |
939 | /** |
940 | * icc_node_add() - add interconnect node to interconnect provider |
941 | * @node: pointer to the interconnect node |
942 | * @provider: pointer to the interconnect provider |
943 | */ |
944 | void icc_node_add(struct icc_node *node, struct icc_provider *provider) |
945 | { |
946 | if (WARN_ON(node->provider)) |
947 | return; |
948 | |
949 | mutex_lock(&icc_lock); |
950 | mutex_lock(&icc_bw_lock); |
951 | |
952 | node->provider = provider; |
953 | list_add_tail(new: &node->node_list, head: &provider->nodes); |
954 | |
955 | /* get the initial bandwidth values and sync them with hardware */ |
956 | if (provider->get_bw) { |
957 | provider->get_bw(node, &node->init_avg, &node->init_peak); |
958 | } else { |
959 | node->init_avg = INT_MAX; |
960 | node->init_peak = INT_MAX; |
961 | } |
962 | node->avg_bw = node->init_avg; |
963 | node->peak_bw = node->init_peak; |
964 | |
965 | if (node->avg_bw || node->peak_bw) { |
966 | if (provider->pre_aggregate) |
967 | provider->pre_aggregate(node); |
968 | |
969 | if (provider->aggregate) |
970 | provider->aggregate(node, 0, node->init_avg, node->init_peak, |
971 | &node->avg_bw, &node->peak_bw); |
972 | if (provider->set) |
973 | provider->set(node, node); |
974 | } |
975 | |
976 | node->avg_bw = 0; |
977 | node->peak_bw = 0; |
978 | |
979 | mutex_unlock(lock: &icc_bw_lock); |
980 | mutex_unlock(lock: &icc_lock); |
981 | } |
982 | EXPORT_SYMBOL_GPL(icc_node_add); |
983 | |
984 | /** |
985 | * icc_node_del() - delete interconnect node from interconnect provider |
986 | * @node: pointer to the interconnect node |
987 | */ |
988 | void icc_node_del(struct icc_node *node) |
989 | { |
990 | mutex_lock(&icc_lock); |
991 | |
992 | list_del(entry: &node->node_list); |
993 | |
994 | mutex_unlock(lock: &icc_lock); |
995 | } |
996 | EXPORT_SYMBOL_GPL(icc_node_del); |
997 | |
998 | /** |
999 | * icc_nodes_remove() - remove all previously added nodes from provider |
1000 | * @provider: the interconnect provider we are removing nodes from |
1001 | * |
1002 | * Return: 0 on success, or an error code otherwise |
1003 | */ |
1004 | int icc_nodes_remove(struct icc_provider *provider) |
1005 | { |
1006 | struct icc_node *n, *tmp; |
1007 | |
1008 | if (WARN_ON(IS_ERR_OR_NULL(provider))) |
1009 | return -EINVAL; |
1010 | |
1011 | list_for_each_entry_safe_reverse(n, tmp, &provider->nodes, node_list) { |
1012 | icc_node_del(n); |
1013 | icc_node_destroy(n->id); |
1014 | } |
1015 | |
1016 | return 0; |
1017 | } |
1018 | EXPORT_SYMBOL_GPL(icc_nodes_remove); |
1019 | |
1020 | /** |
1021 | * icc_provider_init() - initialize a new interconnect provider |
1022 | * @provider: the interconnect provider to initialize |
1023 | * |
1024 | * Must be called before adding nodes to the provider. |
1025 | */ |
1026 | void icc_provider_init(struct icc_provider *provider) |
1027 | { |
1028 | WARN_ON(!provider->set); |
1029 | |
1030 | INIT_LIST_HEAD(list: &provider->nodes); |
1031 | } |
1032 | EXPORT_SYMBOL_GPL(icc_provider_init); |
1033 | |
1034 | /** |
1035 | * icc_provider_register() - register a new interconnect provider |
1036 | * @provider: the interconnect provider to register |
1037 | * |
1038 | * Return: 0 on success, or an error code otherwise |
1039 | */ |
1040 | int icc_provider_register(struct icc_provider *provider) |
1041 | { |
1042 | if (WARN_ON(!provider->xlate && !provider->xlate_extended)) |
1043 | return -EINVAL; |
1044 | |
1045 | mutex_lock(&icc_lock); |
1046 | list_add_tail(new: &provider->provider_list, head: &icc_providers); |
1047 | mutex_unlock(lock: &icc_lock); |
1048 | |
1049 | dev_dbg(provider->dev, "interconnect provider registered\n" ); |
1050 | |
1051 | return 0; |
1052 | } |
1053 | EXPORT_SYMBOL_GPL(icc_provider_register); |
1054 | |
1055 | /** |
1056 | * icc_provider_deregister() - deregister an interconnect provider |
1057 | * @provider: the interconnect provider to deregister |
1058 | */ |
1059 | void icc_provider_deregister(struct icc_provider *provider) |
1060 | { |
1061 | mutex_lock(&icc_lock); |
1062 | WARN_ON(provider->users); |
1063 | |
1064 | list_del(entry: &provider->provider_list); |
1065 | mutex_unlock(lock: &icc_lock); |
1066 | } |
1067 | EXPORT_SYMBOL_GPL(icc_provider_deregister); |
1068 | |
1069 | static const struct of_device_id __maybe_unused ignore_list[] = { |
1070 | { .compatible = "qcom,sc7180-ipa-virt" }, |
1071 | { .compatible = "qcom,sc8180x-ipa-virt" }, |
1072 | { .compatible = "qcom,sdx55-ipa-virt" }, |
1073 | { .compatible = "qcom,sm8150-ipa-virt" }, |
1074 | { .compatible = "qcom,sm8250-ipa-virt" }, |
1075 | {} |
1076 | }; |
1077 | |
1078 | static int of_count_icc_providers(struct device_node *np) |
1079 | { |
1080 | struct device_node *child; |
1081 | int count = 0; |
1082 | |
1083 | for_each_available_child_of_node(np, child) { |
1084 | if (of_property_read_bool(np: child, propname: "#interconnect-cells" ) && |
1085 | likely(!of_match_node(ignore_list, child))) |
1086 | count++; |
1087 | count += of_count_icc_providers(np: child); |
1088 | } |
1089 | |
1090 | return count; |
1091 | } |
1092 | |
1093 | void icc_sync_state(struct device *dev) |
1094 | { |
1095 | struct icc_provider *p; |
1096 | struct icc_node *n; |
1097 | static int count; |
1098 | |
1099 | count++; |
1100 | |
1101 | if (count < providers_count) |
1102 | return; |
1103 | |
1104 | mutex_lock(&icc_lock); |
1105 | mutex_lock(&icc_bw_lock); |
1106 | synced_state = true; |
1107 | list_for_each_entry(p, &icc_providers, provider_list) { |
1108 | dev_dbg(p->dev, "interconnect provider is in synced state\n" ); |
1109 | list_for_each_entry(n, &p->nodes, node_list) { |
1110 | if (n->init_avg || n->init_peak) { |
1111 | n->init_avg = 0; |
1112 | n->init_peak = 0; |
1113 | aggregate_requests(node: n); |
1114 | p->set(n, n); |
1115 | } |
1116 | } |
1117 | } |
1118 | mutex_unlock(lock: &icc_bw_lock); |
1119 | mutex_unlock(lock: &icc_lock); |
1120 | } |
1121 | EXPORT_SYMBOL_GPL(icc_sync_state); |
1122 | |
1123 | static int __init icc_init(void) |
1124 | { |
1125 | struct device_node *root; |
1126 | |
1127 | /* Teach lockdep about lock ordering wrt. shrinker: */ |
1128 | fs_reclaim_acquire(GFP_KERNEL); |
1129 | might_lock(&icc_bw_lock); |
1130 | fs_reclaim_release(GFP_KERNEL); |
1131 | |
1132 | root = of_find_node_by_path(path: "/" ); |
1133 | |
1134 | providers_count = of_count_icc_providers(np: root); |
1135 | of_node_put(node: root); |
1136 | |
1137 | icc_debugfs_dir = debugfs_create_dir(name: "interconnect" , NULL); |
1138 | debugfs_create_file(name: "interconnect_summary" , mode: 0444, |
1139 | parent: icc_debugfs_dir, NULL, fops: &icc_summary_fops); |
1140 | debugfs_create_file(name: "interconnect_graph" , mode: 0444, |
1141 | parent: icc_debugfs_dir, NULL, fops: &icc_graph_fops); |
1142 | |
1143 | icc_debugfs_client_init(icc_dir: icc_debugfs_dir); |
1144 | |
1145 | return 0; |
1146 | } |
1147 | |
1148 | device_initcall(icc_init); |
1149 | |