1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright(c) 2021 Intel Corporation. All rights reserved. */ |
3 | #include <linux/libnvdimm.h> |
4 | #include <asm/unaligned.h> |
5 | #include <linux/device.h> |
6 | #include <linux/module.h> |
7 | #include <linux/ndctl.h> |
8 | #include <linux/async.h> |
9 | #include <linux/slab.h> |
10 | #include <linux/nd.h> |
11 | #include "cxlmem.h" |
12 | #include "cxl.h" |
13 | |
14 | extern const struct nvdimm_security_ops *cxl_security_ops; |
15 | |
16 | static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX); |
17 | |
18 | static void clear_exclusive(void *mds) |
19 | { |
20 | clear_exclusive_cxl_commands(mds, cmds: exclusive_cmds); |
21 | } |
22 | |
23 | static void unregister_nvdimm(void *nvdimm) |
24 | { |
25 | nvdimm_delete(nvdimm); |
26 | } |
27 | |
28 | static ssize_t provider_show(struct device *dev, struct device_attribute *attr, char *buf) |
29 | { |
30 | struct nvdimm *nvdimm = to_nvdimm(dev); |
31 | struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); |
32 | |
33 | return sysfs_emit(buf, fmt: "%s\n" , dev_name(dev: &cxl_nvd->dev)); |
34 | } |
35 | static DEVICE_ATTR_RO(provider); |
36 | |
37 | static ssize_t id_show(struct device *dev, struct device_attribute *attr, char *buf) |
38 | { |
39 | struct nvdimm *nvdimm = to_nvdimm(dev); |
40 | struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); |
41 | struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds; |
42 | |
43 | return sysfs_emit(buf, fmt: "%lld\n" , cxlds->serial); |
44 | } |
45 | static DEVICE_ATTR_RO(id); |
46 | |
47 | static struct attribute *cxl_dimm_attributes[] = { |
48 | &dev_attr_id.attr, |
49 | &dev_attr_provider.attr, |
50 | NULL |
51 | }; |
52 | |
53 | static const struct attribute_group cxl_dimm_attribute_group = { |
54 | .name = "cxl" , |
55 | .attrs = cxl_dimm_attributes, |
56 | }; |
57 | |
58 | static const struct attribute_group *cxl_dimm_attribute_groups[] = { |
59 | &cxl_dimm_attribute_group, |
60 | NULL |
61 | }; |
62 | |
63 | static int cxl_nvdimm_probe(struct device *dev) |
64 | { |
65 | struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev); |
66 | struct cxl_memdev *cxlmd = cxl_nvd->cxlmd; |
67 | struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb; |
68 | struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds: cxlmd->cxlds); |
69 | unsigned long flags = 0, cmd_mask = 0; |
70 | struct nvdimm *nvdimm; |
71 | int rc; |
72 | |
73 | set_exclusive_cxl_commands(mds, cmds: exclusive_cmds); |
74 | rc = devm_add_action_or_reset(dev, clear_exclusive, mds); |
75 | if (rc) |
76 | return rc; |
77 | |
78 | set_bit(nr: NDD_LABELING, addr: &flags); |
79 | set_bit(nr: NDD_REGISTER_SYNC, addr: &flags); |
80 | set_bit(nr: ND_CMD_GET_CONFIG_SIZE, addr: &cmd_mask); |
81 | set_bit(nr: ND_CMD_GET_CONFIG_DATA, addr: &cmd_mask); |
82 | set_bit(nr: ND_CMD_SET_CONFIG_DATA, addr: &cmd_mask); |
83 | nvdimm = __nvdimm_create(nvdimm_bus: cxl_nvb->nvdimm_bus, provider_data: cxl_nvd, |
84 | groups: cxl_dimm_attribute_groups, flags, |
85 | cmd_mask, num_flush: 0, NULL, dimm_id: cxl_nvd->dev_id, |
86 | sec_ops: cxl_security_ops, NULL); |
87 | if (!nvdimm) |
88 | return -ENOMEM; |
89 | |
90 | dev_set_drvdata(dev, data: nvdimm); |
91 | return devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm); |
92 | } |
93 | |
94 | static struct cxl_driver cxl_nvdimm_driver = { |
95 | .name = "cxl_nvdimm" , |
96 | .probe = cxl_nvdimm_probe, |
97 | .id = CXL_DEVICE_NVDIMM, |
98 | .drv = { |
99 | .suppress_bind_attrs = true, |
100 | }, |
101 | }; |
102 | |
103 | static int cxl_pmem_get_config_size(struct cxl_memdev_state *mds, |
104 | struct nd_cmd_get_config_size *cmd, |
105 | unsigned int buf_len) |
106 | { |
107 | if (sizeof(*cmd) > buf_len) |
108 | return -EINVAL; |
109 | |
110 | *cmd = (struct nd_cmd_get_config_size){ |
111 | .config_size = mds->lsa_size, |
112 | .max_xfer = |
113 | mds->payload_size - sizeof(struct cxl_mbox_set_lsa), |
114 | }; |
115 | |
116 | return 0; |
117 | } |
118 | |
119 | static int cxl_pmem_get_config_data(struct cxl_memdev_state *mds, |
120 | struct nd_cmd_get_config_data_hdr *cmd, |
121 | unsigned int buf_len) |
122 | { |
123 | struct cxl_mbox_get_lsa get_lsa; |
124 | struct cxl_mbox_cmd mbox_cmd; |
125 | int rc; |
126 | |
127 | if (sizeof(*cmd) > buf_len) |
128 | return -EINVAL; |
129 | if (struct_size(cmd, out_buf, cmd->in_length) > buf_len) |
130 | return -EINVAL; |
131 | |
132 | get_lsa = (struct cxl_mbox_get_lsa) { |
133 | .offset = cpu_to_le32(cmd->in_offset), |
134 | .length = cpu_to_le32(cmd->in_length), |
135 | }; |
136 | mbox_cmd = (struct cxl_mbox_cmd) { |
137 | .opcode = CXL_MBOX_OP_GET_LSA, |
138 | .payload_in = &get_lsa, |
139 | .size_in = sizeof(get_lsa), |
140 | .size_out = cmd->in_length, |
141 | .payload_out = cmd->out_buf, |
142 | }; |
143 | |
144 | rc = cxl_internal_send_cmd(mds, cmd: &mbox_cmd); |
145 | cmd->status = 0; |
146 | |
147 | return rc; |
148 | } |
149 | |
150 | static int cxl_pmem_set_config_data(struct cxl_memdev_state *mds, |
151 | struct nd_cmd_set_config_hdr *cmd, |
152 | unsigned int buf_len) |
153 | { |
154 | struct cxl_mbox_set_lsa *set_lsa; |
155 | struct cxl_mbox_cmd mbox_cmd; |
156 | int rc; |
157 | |
158 | if (sizeof(*cmd) > buf_len) |
159 | return -EINVAL; |
160 | |
161 | /* 4-byte status follows the input data in the payload */ |
162 | if (size_add(struct_size(cmd, in_buf, cmd->in_length), addend2: 4) > buf_len) |
163 | return -EINVAL; |
164 | |
165 | set_lsa = |
166 | kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL); |
167 | if (!set_lsa) |
168 | return -ENOMEM; |
169 | |
170 | *set_lsa = (struct cxl_mbox_set_lsa) { |
171 | .offset = cpu_to_le32(cmd->in_offset), |
172 | }; |
173 | memcpy(set_lsa->data, cmd->in_buf, cmd->in_length); |
174 | mbox_cmd = (struct cxl_mbox_cmd) { |
175 | .opcode = CXL_MBOX_OP_SET_LSA, |
176 | .payload_in = set_lsa, |
177 | .size_in = struct_size(set_lsa, data, cmd->in_length), |
178 | }; |
179 | |
180 | rc = cxl_internal_send_cmd(mds, cmd: &mbox_cmd); |
181 | |
182 | /* |
183 | * Set "firmware" status (4-packed bytes at the end of the input |
184 | * payload. |
185 | */ |
186 | put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]); |
187 | kvfree(addr: set_lsa); |
188 | |
189 | return rc; |
190 | } |
191 | |
192 | static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd, |
193 | void *buf, unsigned int buf_len) |
194 | { |
195 | struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); |
196 | unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm); |
197 | struct cxl_memdev *cxlmd = cxl_nvd->cxlmd; |
198 | struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds: cxlmd->cxlds); |
199 | |
200 | if (!test_bit(cmd, &cmd_mask)) |
201 | return -ENOTTY; |
202 | |
203 | switch (cmd) { |
204 | case ND_CMD_GET_CONFIG_SIZE: |
205 | return cxl_pmem_get_config_size(mds, cmd: buf, buf_len); |
206 | case ND_CMD_GET_CONFIG_DATA: |
207 | return cxl_pmem_get_config_data(mds, cmd: buf, buf_len); |
208 | case ND_CMD_SET_CONFIG_DATA: |
209 | return cxl_pmem_set_config_data(mds, cmd: buf, buf_len); |
210 | default: |
211 | return -ENOTTY; |
212 | } |
213 | } |
214 | |
215 | static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc, |
216 | struct nvdimm *nvdimm, unsigned int cmd, void *buf, |
217 | unsigned int buf_len, int *cmd_rc) |
218 | { |
219 | /* |
220 | * No firmware response to translate, let the transport error |
221 | * code take precedence. |
222 | */ |
223 | *cmd_rc = 0; |
224 | |
225 | if (!nvdimm) |
226 | return -ENOTTY; |
227 | return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len); |
228 | } |
229 | |
230 | static int detach_nvdimm(struct device *dev, void *data) |
231 | { |
232 | struct cxl_nvdimm *cxl_nvd; |
233 | bool release = false; |
234 | |
235 | if (!is_cxl_nvdimm(dev)) |
236 | return 0; |
237 | |
238 | device_lock(dev); |
239 | if (!dev->driver) |
240 | goto out; |
241 | |
242 | cxl_nvd = to_cxl_nvdimm(dev); |
243 | if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data) |
244 | release = true; |
245 | out: |
246 | device_unlock(dev); |
247 | if (release) |
248 | device_release_driver(dev); |
249 | return 0; |
250 | } |
251 | |
252 | static void unregister_nvdimm_bus(void *_cxl_nvb) |
253 | { |
254 | struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb; |
255 | struct nvdimm_bus *nvdimm_bus = cxl_nvb->nvdimm_bus; |
256 | |
257 | bus_for_each_dev(bus: &cxl_bus_type, NULL, data: cxl_nvb, fn: detach_nvdimm); |
258 | |
259 | cxl_nvb->nvdimm_bus = NULL; |
260 | nvdimm_bus_unregister(nvdimm_bus); |
261 | } |
262 | |
263 | static int cxl_nvdimm_bridge_probe(struct device *dev) |
264 | { |
265 | struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev); |
266 | |
267 | cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) { |
268 | .provider_name = "CXL" , |
269 | .module = THIS_MODULE, |
270 | .ndctl = cxl_pmem_ctl, |
271 | }; |
272 | |
273 | cxl_nvb->nvdimm_bus = |
274 | nvdimm_bus_register(parent: &cxl_nvb->dev, nfit_desc: &cxl_nvb->nd_desc); |
275 | |
276 | if (!cxl_nvb->nvdimm_bus) |
277 | return -ENOMEM; |
278 | |
279 | return devm_add_action_or_reset(dev, unregister_nvdimm_bus, cxl_nvb); |
280 | } |
281 | |
282 | static struct cxl_driver cxl_nvdimm_bridge_driver = { |
283 | .name = "cxl_nvdimm_bridge" , |
284 | .probe = cxl_nvdimm_bridge_probe, |
285 | .id = CXL_DEVICE_NVDIMM_BRIDGE, |
286 | .drv = { |
287 | .suppress_bind_attrs = true, |
288 | }, |
289 | }; |
290 | |
291 | static void unregister_nvdimm_region(void *nd_region) |
292 | { |
293 | nvdimm_region_delete(nd_region); |
294 | } |
295 | |
296 | static void cxlr_pmem_remove_resource(void *res) |
297 | { |
298 | remove_resource(old: res); |
299 | } |
300 | |
301 | struct cxl_pmem_region_info { |
302 | u64 offset; |
303 | u64 serial; |
304 | }; |
305 | |
306 | static int cxl_pmem_region_probe(struct device *dev) |
307 | { |
308 | struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE]; |
309 | struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev); |
310 | struct cxl_region *cxlr = cxlr_pmem->cxlr; |
311 | struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; |
312 | struct cxl_pmem_region_info *info = NULL; |
313 | struct nd_interleave_set *nd_set; |
314 | struct nd_region_desc ndr_desc; |
315 | struct cxl_nvdimm *cxl_nvd; |
316 | struct nvdimm *nvdimm; |
317 | struct resource *res; |
318 | int rc, i = 0; |
319 | |
320 | memset(&mappings, 0, sizeof(mappings)); |
321 | memset(&ndr_desc, 0, sizeof(ndr_desc)); |
322 | |
323 | res = devm_kzalloc(dev, size: sizeof(*res), GFP_KERNEL); |
324 | if (!res) |
325 | return -ENOMEM; |
326 | |
327 | res->name = "Persistent Memory" ; |
328 | res->start = cxlr_pmem->hpa_range.start; |
329 | res->end = cxlr_pmem->hpa_range.end; |
330 | res->flags = IORESOURCE_MEM; |
331 | res->desc = IORES_DESC_PERSISTENT_MEMORY; |
332 | |
333 | rc = insert_resource(parent: &iomem_resource, new: res); |
334 | if (rc) |
335 | return rc; |
336 | |
337 | rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res); |
338 | if (rc) |
339 | return rc; |
340 | |
341 | ndr_desc.res = res; |
342 | ndr_desc.provider_data = cxlr_pmem; |
343 | |
344 | ndr_desc.numa_node = memory_add_physaddr_to_nid(start: res->start); |
345 | ndr_desc.target_node = phys_to_target_node(start: res->start); |
346 | if (ndr_desc.target_node == NUMA_NO_NODE) { |
347 | ndr_desc.target_node = ndr_desc.numa_node; |
348 | dev_dbg(&cxlr->dev, "changing target node from %d to %d" , |
349 | NUMA_NO_NODE, ndr_desc.target_node); |
350 | } |
351 | |
352 | nd_set = devm_kzalloc(dev, size: sizeof(*nd_set), GFP_KERNEL); |
353 | if (!nd_set) |
354 | return -ENOMEM; |
355 | |
356 | ndr_desc.memregion = cxlr->id; |
357 | set_bit(nr: ND_REGION_CXL, addr: &ndr_desc.flags); |
358 | set_bit(nr: ND_REGION_PERSIST_MEMCTRL, addr: &ndr_desc.flags); |
359 | |
360 | info = kmalloc_array(n: cxlr_pmem->nr_mappings, size: sizeof(*info), GFP_KERNEL); |
361 | if (!info) |
362 | return -ENOMEM; |
363 | |
364 | for (i = 0; i < cxlr_pmem->nr_mappings; i++) { |
365 | struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i]; |
366 | struct cxl_memdev *cxlmd = m->cxlmd; |
367 | struct cxl_dev_state *cxlds = cxlmd->cxlds; |
368 | |
369 | cxl_nvd = cxlmd->cxl_nvd; |
370 | nvdimm = dev_get_drvdata(dev: &cxl_nvd->dev); |
371 | if (!nvdimm) { |
372 | dev_dbg(dev, "[%d]: %s: no nvdimm found\n" , i, |
373 | dev_name(&cxlmd->dev)); |
374 | rc = -ENODEV; |
375 | goto out_nvd; |
376 | } |
377 | |
378 | m->cxl_nvd = cxl_nvd; |
379 | mappings[i] = (struct nd_mapping_desc) { |
380 | .nvdimm = nvdimm, |
381 | .start = m->start, |
382 | .size = m->size, |
383 | .position = i, |
384 | }; |
385 | info[i].offset = m->start; |
386 | info[i].serial = cxlds->serial; |
387 | } |
388 | ndr_desc.num_mappings = cxlr_pmem->nr_mappings; |
389 | ndr_desc.mapping = mappings; |
390 | |
391 | /* |
392 | * TODO enable CXL labels which skip the need for 'interleave-set cookie' |
393 | */ |
394 | nd_set->cookie1 = |
395 | nd_fletcher64(addr: info, len: sizeof(*info) * cxlr_pmem->nr_mappings, le: 0); |
396 | nd_set->cookie2 = nd_set->cookie1; |
397 | ndr_desc.nd_set = nd_set; |
398 | |
399 | cxlr_pmem->nd_region = |
400 | nvdimm_pmem_region_create(nvdimm_bus: cxl_nvb->nvdimm_bus, ndr_desc: &ndr_desc); |
401 | if (!cxlr_pmem->nd_region) { |
402 | rc = -ENOMEM; |
403 | goto out_nvd; |
404 | } |
405 | |
406 | rc = devm_add_action_or_reset(dev, unregister_nvdimm_region, |
407 | cxlr_pmem->nd_region); |
408 | out_nvd: |
409 | kfree(objp: info); |
410 | |
411 | return rc; |
412 | } |
413 | |
414 | static struct cxl_driver cxl_pmem_region_driver = { |
415 | .name = "cxl_pmem_region" , |
416 | .probe = cxl_pmem_region_probe, |
417 | .id = CXL_DEVICE_PMEM_REGION, |
418 | .drv = { |
419 | .suppress_bind_attrs = true, |
420 | }, |
421 | }; |
422 | |
423 | static __init int cxl_pmem_init(void) |
424 | { |
425 | int rc; |
426 | |
427 | set_bit(nr: CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, addr: exclusive_cmds); |
428 | set_bit(nr: CXL_MEM_COMMAND_ID_SET_LSA, addr: exclusive_cmds); |
429 | |
430 | rc = cxl_driver_register(&cxl_nvdimm_bridge_driver); |
431 | if (rc) |
432 | return rc; |
433 | |
434 | rc = cxl_driver_register(&cxl_nvdimm_driver); |
435 | if (rc) |
436 | goto err_nvdimm; |
437 | |
438 | rc = cxl_driver_register(&cxl_pmem_region_driver); |
439 | if (rc) |
440 | goto err_region; |
441 | |
442 | return 0; |
443 | |
444 | err_region: |
445 | cxl_driver_unregister(cxl_drv: &cxl_nvdimm_driver); |
446 | err_nvdimm: |
447 | cxl_driver_unregister(cxl_drv: &cxl_nvdimm_bridge_driver); |
448 | return rc; |
449 | } |
450 | |
451 | static __exit void cxl_pmem_exit(void) |
452 | { |
453 | cxl_driver_unregister(cxl_drv: &cxl_pmem_region_driver); |
454 | cxl_driver_unregister(cxl_drv: &cxl_nvdimm_driver); |
455 | cxl_driver_unregister(cxl_drv: &cxl_nvdimm_bridge_driver); |
456 | } |
457 | |
458 | MODULE_LICENSE("GPL v2" ); |
459 | module_init(cxl_pmem_init); |
460 | module_exit(cxl_pmem_exit); |
461 | MODULE_IMPORT_NS(CXL); |
462 | MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE); |
463 | MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM); |
464 | MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION); |
465 | |