1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * The On Chip Memory (OCMEM) allocator allows various clients to allocate |
4 | * memory from OCMEM based on performance, latency and power requirements. |
5 | * This is typically used by the GPU, camera/video, and audio components on |
6 | * some Snapdragon SoCs. |
7 | * |
8 | * Copyright (C) 2019 Brian Masney <masneyb@onstation.org> |
9 | * Copyright (C) 2015 Red Hat. Author: Rob Clark <robdclark@gmail.com> |
10 | */ |
11 | |
12 | #include <linux/bitfield.h> |
13 | #include <linux/clk.h> |
14 | #include <linux/io.h> |
15 | #include <linux/kernel.h> |
16 | #include <linux/module.h> |
17 | #include <linux/of.h> |
18 | #include <linux/of_platform.h> |
19 | #include <linux/platform_device.h> |
20 | #include <linux/firmware/qcom/qcom_scm.h> |
21 | #include <linux/sizes.h> |
22 | #include <linux/slab.h> |
23 | #include <linux/types.h> |
24 | #include <soc/qcom/ocmem.h> |
25 | |
26 | enum region_mode { |
27 | WIDE_MODE = 0x0, |
28 | THIN_MODE, |
29 | MODE_DEFAULT = WIDE_MODE, |
30 | }; |
31 | |
32 | enum ocmem_macro_state { |
33 | PASSTHROUGH = 0, |
34 | PERI_ON = 1, |
35 | CORE_ON = 2, |
36 | CLK_OFF = 4, |
37 | }; |
38 | |
39 | struct ocmem_region { |
40 | bool interleaved; |
41 | enum region_mode mode; |
42 | unsigned int num_macros; |
43 | enum ocmem_macro_state macro_state[4]; |
44 | unsigned long macro_size; |
45 | unsigned long region_size; |
46 | }; |
47 | |
48 | struct ocmem_config { |
49 | uint8_t num_regions; |
50 | unsigned long macro_size; |
51 | }; |
52 | |
53 | struct ocmem { |
54 | struct device *dev; |
55 | const struct ocmem_config *config; |
56 | struct resource *memory; |
57 | void __iomem *mmio; |
58 | struct clk *core_clk; |
59 | struct clk *iface_clk; |
60 | unsigned int num_ports; |
61 | unsigned int num_macros; |
62 | bool interleaved; |
63 | struct ocmem_region *regions; |
64 | unsigned long active_allocations; |
65 | }; |
66 | |
67 | #define OCMEM_MIN_ALIGN SZ_64K |
68 | #define OCMEM_MIN_ALLOC SZ_64K |
69 | |
70 | #define OCMEM_REG_HW_VERSION 0x00000000 |
71 | #define OCMEM_REG_HW_PROFILE 0x00000004 |
72 | |
73 | #define OCMEM_REG_REGION_MODE_CTL 0x00001000 |
74 | #define OCMEM_REGION_MODE_CTL_REG0_THIN 0x00000001 |
75 | #define OCMEM_REGION_MODE_CTL_REG1_THIN 0x00000002 |
76 | #define OCMEM_REGION_MODE_CTL_REG2_THIN 0x00000004 |
77 | #define OCMEM_REGION_MODE_CTL_REG3_THIN 0x00000008 |
78 | |
79 | #define OCMEM_REG_GFX_MPU_START 0x00001004 |
80 | #define OCMEM_REG_GFX_MPU_END 0x00001008 |
81 | |
82 | #define OCMEM_HW_VERSION_MAJOR(val) FIELD_GET(GENMASK(31, 28), val) |
83 | #define OCMEM_HW_VERSION_MINOR(val) FIELD_GET(GENMASK(27, 16), val) |
84 | #define OCMEM_HW_VERSION_STEP(val) FIELD_GET(GENMASK(15, 0), val) |
85 | |
86 | #define OCMEM_HW_PROFILE_NUM_PORTS(val) FIELD_GET(0x0000000f, (val)) |
87 | #define OCMEM_HW_PROFILE_NUM_MACROS(val) FIELD_GET(0x00003f00, (val)) |
88 | |
89 | #define OCMEM_HW_PROFILE_LAST_REGN_HALFSIZE 0x00010000 |
90 | #define OCMEM_HW_PROFILE_INTERLEAVING 0x00020000 |
91 | #define OCMEM_REG_GEN_STATUS 0x0000000c |
92 | |
93 | #define OCMEM_REG_PSGSC_STATUS 0x00000038 |
94 | #define OCMEM_REG_PSGSC_CTL(i0) (0x0000003c + 0x1*(i0)) |
95 | |
96 | #define OCMEM_PSGSC_CTL_MACRO0_MODE(val) FIELD_PREP(0x00000007, (val)) |
97 | #define OCMEM_PSGSC_CTL_MACRO1_MODE(val) FIELD_PREP(0x00000070, (val)) |
98 | #define OCMEM_PSGSC_CTL_MACRO2_MODE(val) FIELD_PREP(0x00000700, (val)) |
99 | #define OCMEM_PSGSC_CTL_MACRO3_MODE(val) FIELD_PREP(0x00007000, (val)) |
100 | |
101 | static inline void ocmem_write(struct ocmem *ocmem, u32 reg, u32 data) |
102 | { |
103 | writel(val: data, addr: ocmem->mmio + reg); |
104 | } |
105 | |
106 | static inline u32 ocmem_read(struct ocmem *ocmem, u32 reg) |
107 | { |
108 | return readl(addr: ocmem->mmio + reg); |
109 | } |
110 | |
111 | static void update_ocmem(struct ocmem *ocmem) |
112 | { |
113 | uint32_t region_mode_ctrl = 0x0; |
114 | int i; |
115 | |
116 | if (!qcom_scm_ocmem_lock_available()) { |
117 | for (i = 0; i < ocmem->config->num_regions; i++) { |
118 | struct ocmem_region *region = &ocmem->regions[i]; |
119 | |
120 | if (region->mode == THIN_MODE) |
121 | region_mode_ctrl |= BIT(i); |
122 | } |
123 | |
124 | dev_dbg(ocmem->dev, "ocmem_region_mode_control %x\n" , |
125 | region_mode_ctrl); |
126 | ocmem_write(ocmem, OCMEM_REG_REGION_MODE_CTL, data: region_mode_ctrl); |
127 | } |
128 | |
129 | for (i = 0; i < ocmem->config->num_regions; i++) { |
130 | struct ocmem_region *region = &ocmem->regions[i]; |
131 | u32 data; |
132 | |
133 | data = OCMEM_PSGSC_CTL_MACRO0_MODE(region->macro_state[0]) | |
134 | OCMEM_PSGSC_CTL_MACRO1_MODE(region->macro_state[1]) | |
135 | OCMEM_PSGSC_CTL_MACRO2_MODE(region->macro_state[2]) | |
136 | OCMEM_PSGSC_CTL_MACRO3_MODE(region->macro_state[3]); |
137 | |
138 | ocmem_write(ocmem, OCMEM_REG_PSGSC_CTL(i), data); |
139 | } |
140 | } |
141 | |
142 | static unsigned long phys_to_offset(struct ocmem *ocmem, |
143 | unsigned long addr) |
144 | { |
145 | if (addr < ocmem->memory->start || addr >= ocmem->memory->end) |
146 | return 0; |
147 | |
148 | return addr - ocmem->memory->start; |
149 | } |
150 | |
151 | static unsigned long device_address(struct ocmem *ocmem, |
152 | enum ocmem_client client, |
153 | unsigned long addr) |
154 | { |
155 | WARN_ON(client != OCMEM_GRAPHICS); |
156 | |
157 | /* TODO: gpu uses phys_to_offset, but others do not.. */ |
158 | return phys_to_offset(ocmem, addr); |
159 | } |
160 | |
161 | static void update_range(struct ocmem *ocmem, struct ocmem_buf *buf, |
162 | enum ocmem_macro_state mstate, enum region_mode rmode) |
163 | { |
164 | unsigned long offset = 0; |
165 | int i, j; |
166 | |
167 | for (i = 0; i < ocmem->config->num_regions; i++) { |
168 | struct ocmem_region *region = &ocmem->regions[i]; |
169 | |
170 | if (buf->offset <= offset && offset < buf->offset + buf->len) |
171 | region->mode = rmode; |
172 | |
173 | for (j = 0; j < region->num_macros; j++) { |
174 | if (buf->offset <= offset && |
175 | offset < buf->offset + buf->len) |
176 | region->macro_state[j] = mstate; |
177 | |
178 | offset += region->macro_size; |
179 | } |
180 | } |
181 | |
182 | update_ocmem(ocmem); |
183 | } |
184 | |
185 | struct ocmem *of_get_ocmem(struct device *dev) |
186 | { |
187 | struct platform_device *pdev; |
188 | struct device_node *devnode; |
189 | struct ocmem *ocmem; |
190 | |
191 | devnode = of_parse_phandle(np: dev->of_node, phandle_name: "sram" , index: 0); |
192 | if (!devnode || !devnode->parent) { |
193 | dev_err(dev, "Cannot look up sram phandle\n" ); |
194 | of_node_put(node: devnode); |
195 | return ERR_PTR(error: -ENODEV); |
196 | } |
197 | |
198 | pdev = of_find_device_by_node(np: devnode->parent); |
199 | if (!pdev) { |
200 | dev_err(dev, "Cannot find device node %s\n" , devnode->name); |
201 | of_node_put(node: devnode); |
202 | return ERR_PTR(error: -EPROBE_DEFER); |
203 | } |
204 | of_node_put(node: devnode); |
205 | |
206 | ocmem = platform_get_drvdata(pdev); |
207 | if (!ocmem) { |
208 | dev_err(dev, "Cannot get ocmem\n" ); |
209 | put_device(dev: &pdev->dev); |
210 | return ERR_PTR(error: -ENODEV); |
211 | } |
212 | return ocmem; |
213 | } |
214 | EXPORT_SYMBOL_GPL(of_get_ocmem); |
215 | |
216 | struct ocmem_buf *ocmem_allocate(struct ocmem *ocmem, enum ocmem_client client, |
217 | unsigned long size) |
218 | { |
219 | struct ocmem_buf *buf; |
220 | int ret; |
221 | |
222 | /* TODO: add support for other clients... */ |
223 | if (WARN_ON(client != OCMEM_GRAPHICS)) |
224 | return ERR_PTR(error: -ENODEV); |
225 | |
226 | if (size < OCMEM_MIN_ALLOC || !IS_ALIGNED(size, OCMEM_MIN_ALIGN)) |
227 | return ERR_PTR(error: -EINVAL); |
228 | |
229 | if (test_and_set_bit_lock(BIT(client), addr: &ocmem->active_allocations)) |
230 | return ERR_PTR(error: -EBUSY); |
231 | |
232 | buf = kzalloc(size: sizeof(*buf), GFP_KERNEL); |
233 | if (!buf) { |
234 | ret = -ENOMEM; |
235 | goto err_unlock; |
236 | } |
237 | |
238 | buf->offset = 0; |
239 | buf->addr = device_address(ocmem, client, addr: buf->offset); |
240 | buf->len = size; |
241 | |
242 | update_range(ocmem, buf, mstate: CORE_ON, rmode: WIDE_MODE); |
243 | |
244 | if (qcom_scm_ocmem_lock_available()) { |
245 | ret = qcom_scm_ocmem_lock(id: QCOM_SCM_OCMEM_GRAPHICS_ID, |
246 | offset: buf->offset, size: buf->len, mode: WIDE_MODE); |
247 | if (ret) { |
248 | dev_err(ocmem->dev, "could not lock: %d\n" , ret); |
249 | ret = -EINVAL; |
250 | goto err_kfree; |
251 | } |
252 | } else { |
253 | ocmem_write(ocmem, OCMEM_REG_GFX_MPU_START, data: buf->offset); |
254 | ocmem_write(ocmem, OCMEM_REG_GFX_MPU_END, |
255 | data: buf->offset + buf->len); |
256 | } |
257 | |
258 | dev_dbg(ocmem->dev, "using %ldK of OCMEM at 0x%08lx for client %d\n" , |
259 | size / 1024, buf->addr, client); |
260 | |
261 | return buf; |
262 | |
263 | err_kfree: |
264 | kfree(objp: buf); |
265 | err_unlock: |
266 | clear_bit_unlock(BIT(client), addr: &ocmem->active_allocations); |
267 | |
268 | return ERR_PTR(error: ret); |
269 | } |
270 | EXPORT_SYMBOL_GPL(ocmem_allocate); |
271 | |
272 | void ocmem_free(struct ocmem *ocmem, enum ocmem_client client, |
273 | struct ocmem_buf *buf) |
274 | { |
275 | /* TODO: add support for other clients... */ |
276 | if (WARN_ON(client != OCMEM_GRAPHICS)) |
277 | return; |
278 | |
279 | update_range(ocmem, buf, mstate: CLK_OFF, rmode: MODE_DEFAULT); |
280 | |
281 | if (qcom_scm_ocmem_lock_available()) { |
282 | int ret; |
283 | |
284 | ret = qcom_scm_ocmem_unlock(id: QCOM_SCM_OCMEM_GRAPHICS_ID, |
285 | offset: buf->offset, size: buf->len); |
286 | if (ret) |
287 | dev_err(ocmem->dev, "could not unlock: %d\n" , ret); |
288 | } else { |
289 | ocmem_write(ocmem, OCMEM_REG_GFX_MPU_START, data: 0x0); |
290 | ocmem_write(ocmem, OCMEM_REG_GFX_MPU_END, data: 0x0); |
291 | } |
292 | |
293 | kfree(objp: buf); |
294 | |
295 | clear_bit_unlock(BIT(client), addr: &ocmem->active_allocations); |
296 | } |
297 | EXPORT_SYMBOL_GPL(ocmem_free); |
298 | |
299 | static int ocmem_dev_probe(struct platform_device *pdev) |
300 | { |
301 | struct device *dev = &pdev->dev; |
302 | unsigned long reg, region_size; |
303 | int i, j, ret, num_banks; |
304 | struct ocmem *ocmem; |
305 | |
306 | if (!qcom_scm_is_available()) |
307 | return -EPROBE_DEFER; |
308 | |
309 | ocmem = devm_kzalloc(dev, size: sizeof(*ocmem), GFP_KERNEL); |
310 | if (!ocmem) |
311 | return -ENOMEM; |
312 | |
313 | ocmem->dev = dev; |
314 | ocmem->config = device_get_match_data(dev); |
315 | |
316 | ocmem->core_clk = devm_clk_get(dev, id: "core" ); |
317 | if (IS_ERR(ptr: ocmem->core_clk)) |
318 | return dev_err_probe(dev, err: PTR_ERR(ptr: ocmem->core_clk), |
319 | fmt: "Unable to get core clock\n" ); |
320 | |
321 | ocmem->iface_clk = devm_clk_get_optional(dev, id: "iface" ); |
322 | if (IS_ERR(ptr: ocmem->iface_clk)) |
323 | return dev_err_probe(dev, err: PTR_ERR(ptr: ocmem->iface_clk), |
324 | fmt: "Unable to get iface clock\n" ); |
325 | |
326 | ocmem->mmio = devm_platform_ioremap_resource_byname(pdev, name: "ctrl" ); |
327 | if (IS_ERR(ptr: ocmem->mmio)) |
328 | return dev_err_probe(dev: &pdev->dev, err: PTR_ERR(ptr: ocmem->mmio), |
329 | fmt: "Failed to ioremap ocmem_ctrl resource\n" ); |
330 | |
331 | ocmem->memory = platform_get_resource_byname(pdev, IORESOURCE_MEM, |
332 | "mem" ); |
333 | if (!ocmem->memory) { |
334 | dev_err(dev, "Could not get mem region\n" ); |
335 | return -ENXIO; |
336 | } |
337 | |
338 | /* The core clock is synchronous with graphics */ |
339 | WARN_ON(clk_set_rate(ocmem->core_clk, 1000) < 0); |
340 | |
341 | ret = clk_prepare_enable(clk: ocmem->core_clk); |
342 | if (ret) |
343 | return dev_err_probe(dev: ocmem->dev, err: ret, fmt: "Failed to enable core clock\n" ); |
344 | |
345 | ret = clk_prepare_enable(clk: ocmem->iface_clk); |
346 | if (ret) { |
347 | clk_disable_unprepare(clk: ocmem->core_clk); |
348 | return dev_err_probe(dev: ocmem->dev, err: ret, fmt: "Failed to enable iface clock\n" ); |
349 | } |
350 | |
351 | if (qcom_scm_restore_sec_cfg_available()) { |
352 | dev_dbg(dev, "configuring scm\n" ); |
353 | ret = qcom_scm_restore_sec_cfg(device_id: QCOM_SCM_OCMEM_DEV_ID, spare: 0); |
354 | if (ret) { |
355 | dev_err_probe(dev, err: ret, fmt: "Could not enable secure configuration\n" ); |
356 | goto err_clk_disable; |
357 | } |
358 | } |
359 | |
360 | reg = ocmem_read(ocmem, OCMEM_REG_HW_VERSION); |
361 | dev_dbg(dev, "OCMEM hardware version: %lu.%lu.%lu\n" , |
362 | OCMEM_HW_VERSION_MAJOR(reg), |
363 | OCMEM_HW_VERSION_MINOR(reg), |
364 | OCMEM_HW_VERSION_STEP(reg)); |
365 | |
366 | reg = ocmem_read(ocmem, OCMEM_REG_HW_PROFILE); |
367 | ocmem->num_ports = OCMEM_HW_PROFILE_NUM_PORTS(reg); |
368 | ocmem->num_macros = OCMEM_HW_PROFILE_NUM_MACROS(reg); |
369 | ocmem->interleaved = !!(reg & OCMEM_HW_PROFILE_INTERLEAVING); |
370 | |
371 | num_banks = ocmem->num_ports / 2; |
372 | region_size = ocmem->config->macro_size * num_banks; |
373 | |
374 | dev_info(dev, "%u ports, %u regions, %u macros, %sinterleaved\n" , |
375 | ocmem->num_ports, ocmem->config->num_regions, |
376 | ocmem->num_macros, ocmem->interleaved ? "" : "not " ); |
377 | |
378 | ocmem->regions = devm_kcalloc(dev, n: ocmem->config->num_regions, |
379 | size: sizeof(struct ocmem_region), GFP_KERNEL); |
380 | if (!ocmem->regions) { |
381 | ret = -ENOMEM; |
382 | goto err_clk_disable; |
383 | } |
384 | |
385 | for (i = 0; i < ocmem->config->num_regions; i++) { |
386 | struct ocmem_region *region = &ocmem->regions[i]; |
387 | |
388 | if (WARN_ON(num_banks > ARRAY_SIZE(region->macro_state))) { |
389 | ret = -EINVAL; |
390 | goto err_clk_disable; |
391 | } |
392 | |
393 | region->mode = MODE_DEFAULT; |
394 | region->num_macros = num_banks; |
395 | |
396 | if (i == (ocmem->config->num_regions - 1) && |
397 | reg & OCMEM_HW_PROFILE_LAST_REGN_HALFSIZE) { |
398 | region->macro_size = ocmem->config->macro_size / 2; |
399 | region->region_size = region_size / 2; |
400 | } else { |
401 | region->macro_size = ocmem->config->macro_size; |
402 | region->region_size = region_size; |
403 | } |
404 | |
405 | for (j = 0; j < ARRAY_SIZE(region->macro_state); j++) |
406 | region->macro_state[j] = CLK_OFF; |
407 | } |
408 | |
409 | platform_set_drvdata(pdev, data: ocmem); |
410 | |
411 | return 0; |
412 | |
413 | err_clk_disable: |
414 | clk_disable_unprepare(clk: ocmem->core_clk); |
415 | clk_disable_unprepare(clk: ocmem->iface_clk); |
416 | return ret; |
417 | } |
418 | |
419 | static void ocmem_dev_remove(struct platform_device *pdev) |
420 | { |
421 | struct ocmem *ocmem = platform_get_drvdata(pdev); |
422 | |
423 | clk_disable_unprepare(clk: ocmem->core_clk); |
424 | clk_disable_unprepare(clk: ocmem->iface_clk); |
425 | } |
426 | |
427 | static const struct ocmem_config ocmem_8226_config = { |
428 | .num_regions = 1, |
429 | .macro_size = SZ_128K, |
430 | }; |
431 | |
432 | static const struct ocmem_config ocmem_8974_config = { |
433 | .num_regions = 3, |
434 | .macro_size = SZ_128K, |
435 | }; |
436 | |
437 | static const struct of_device_id ocmem_of_match[] = { |
438 | { .compatible = "qcom,msm8226-ocmem" , .data = &ocmem_8226_config }, |
439 | { .compatible = "qcom,msm8974-ocmem" , .data = &ocmem_8974_config }, |
440 | { } |
441 | }; |
442 | |
443 | MODULE_DEVICE_TABLE(of, ocmem_of_match); |
444 | |
445 | static struct platform_driver ocmem_driver = { |
446 | .probe = ocmem_dev_probe, |
447 | .remove_new = ocmem_dev_remove, |
448 | .driver = { |
449 | .name = "ocmem" , |
450 | .of_match_table = ocmem_of_match, |
451 | }, |
452 | }; |
453 | |
454 | module_platform_driver(ocmem_driver); |
455 | |
456 | MODULE_DESCRIPTION("On Chip Memory (OCMEM) allocator for some Snapdragon SoCs" ); |
457 | MODULE_LICENSE("GPL v2" ); |
458 | |