1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (c) 2021, HiSilicon Ltd. |
4 | */ |
5 | |
6 | #include <linux/device.h> |
7 | #include <linux/eventfd.h> |
8 | #include <linux/file.h> |
9 | #include <linux/hisi_acc_qm.h> |
10 | #include <linux/interrupt.h> |
11 | #include <linux/module.h> |
12 | #include <linux/pci.h> |
13 | #include <linux/vfio.h> |
14 | #include <linux/vfio_pci_core.h> |
15 | #include <linux/anon_inodes.h> |
16 | |
17 | #include "hisi_acc_vfio_pci.h" |
18 | |
19 | /* Return 0 on VM acc device ready, -ETIMEDOUT hardware timeout */ |
20 | static int qm_wait_dev_not_ready(struct hisi_qm *qm) |
21 | { |
22 | u32 val; |
23 | |
24 | return readl_relaxed_poll_timeout(qm->io_base + QM_VF_STATE, |
25 | val, !(val & 0x1), MB_POLL_PERIOD_US, |
26 | MB_POLL_TIMEOUT_US); |
27 | } |
28 | |
29 | /* |
30 | * Each state Reg is checked 100 times, |
31 | * with a delay of 100 microseconds after each check |
32 | */ |
33 | static u32 qm_check_reg_state(struct hisi_qm *qm, u32 regs) |
34 | { |
35 | int check_times = 0; |
36 | u32 state; |
37 | |
38 | state = readl(addr: qm->io_base + regs); |
39 | while (state && check_times < ERROR_CHECK_TIMEOUT) { |
40 | udelay(CHECK_DELAY_TIME); |
41 | state = readl(addr: qm->io_base + regs); |
42 | check_times++; |
43 | } |
44 | |
45 | return state; |
46 | } |
47 | |
48 | static int qm_read_regs(struct hisi_qm *qm, u32 reg_addr, |
49 | u32 *data, u8 nums) |
50 | { |
51 | int i; |
52 | |
53 | if (nums < 1 || nums > QM_REGS_MAX_LEN) |
54 | return -EINVAL; |
55 | |
56 | for (i = 0; i < nums; i++) { |
57 | data[i] = readl(addr: qm->io_base + reg_addr); |
58 | reg_addr += QM_REG_ADDR_OFFSET; |
59 | } |
60 | |
61 | return 0; |
62 | } |
63 | |
64 | static int qm_write_regs(struct hisi_qm *qm, u32 reg, |
65 | u32 *data, u8 nums) |
66 | { |
67 | int i; |
68 | |
69 | if (nums < 1 || nums > QM_REGS_MAX_LEN) |
70 | return -EINVAL; |
71 | |
72 | for (i = 0; i < nums; i++) |
73 | writel(val: data[i], addr: qm->io_base + reg + i * QM_REG_ADDR_OFFSET); |
74 | |
75 | return 0; |
76 | } |
77 | |
78 | static int qm_get_vft(struct hisi_qm *qm, u32 *base) |
79 | { |
80 | u64 sqc_vft; |
81 | u32 qp_num; |
82 | int ret; |
83 | |
84 | ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, dma_addr: 0, queue: 0, op: 1); |
85 | if (ret) |
86 | return ret; |
87 | |
88 | sqc_vft = readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_L) | |
89 | ((u64)readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_H) << |
90 | QM_XQC_ADDR_OFFSET); |
91 | *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2); |
92 | qp_num = (QM_SQC_VFT_NUM_MASK_V2 & |
93 | (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1; |
94 | |
95 | return qp_num; |
96 | } |
97 | |
98 | static int qm_get_sqc(struct hisi_qm *qm, u64 *addr) |
99 | { |
100 | int ret; |
101 | |
102 | ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, dma_addr: 0, queue: 0, op: 1); |
103 | if (ret) |
104 | return ret; |
105 | |
106 | *addr = readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_L) | |
107 | ((u64)readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_H) << |
108 | QM_XQC_ADDR_OFFSET); |
109 | |
110 | return 0; |
111 | } |
112 | |
113 | static int qm_get_cqc(struct hisi_qm *qm, u64 *addr) |
114 | { |
115 | int ret; |
116 | |
117 | ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, dma_addr: 0, queue: 0, op: 1); |
118 | if (ret) |
119 | return ret; |
120 | |
121 | *addr = readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_L) | |
122 | ((u64)readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_H) << |
123 | QM_XQC_ADDR_OFFSET); |
124 | |
125 | return 0; |
126 | } |
127 | |
128 | static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data) |
129 | { |
130 | struct device *dev = &qm->pdev->dev; |
131 | int ret; |
132 | |
133 | ret = qm_read_regs(qm, QM_VF_AEQ_INT_MASK, data: &vf_data->aeq_int_mask, nums: 1); |
134 | if (ret) { |
135 | dev_err(dev, "failed to read QM_VF_AEQ_INT_MASK\n" ); |
136 | return ret; |
137 | } |
138 | |
139 | ret = qm_read_regs(qm, QM_VF_EQ_INT_MASK, data: &vf_data->eq_int_mask, nums: 1); |
140 | if (ret) { |
141 | dev_err(dev, "failed to read QM_VF_EQ_INT_MASK\n" ); |
142 | return ret; |
143 | } |
144 | |
145 | ret = qm_read_regs(qm, QM_IFC_INT_SOURCE_V, |
146 | data: &vf_data->ifc_int_source, nums: 1); |
147 | if (ret) { |
148 | dev_err(dev, "failed to read QM_IFC_INT_SOURCE_V\n" ); |
149 | return ret; |
150 | } |
151 | |
152 | ret = qm_read_regs(qm, QM_IFC_INT_MASK, data: &vf_data->ifc_int_mask, nums: 1); |
153 | if (ret) { |
154 | dev_err(dev, "failed to read QM_IFC_INT_MASK\n" ); |
155 | return ret; |
156 | } |
157 | |
158 | ret = qm_read_regs(qm, QM_IFC_INT_SET_V, data: &vf_data->ifc_int_set, nums: 1); |
159 | if (ret) { |
160 | dev_err(dev, "failed to read QM_IFC_INT_SET_V\n" ); |
161 | return ret; |
162 | } |
163 | |
164 | ret = qm_read_regs(qm, QM_PAGE_SIZE, data: &vf_data->page_size, nums: 1); |
165 | if (ret) { |
166 | dev_err(dev, "failed to read QM_PAGE_SIZE\n" ); |
167 | return ret; |
168 | } |
169 | |
170 | /* QM_EQC_DW has 7 regs */ |
171 | ret = qm_read_regs(qm, QM_EQC_DW0, data: vf_data->qm_eqc_dw, nums: 7); |
172 | if (ret) { |
173 | dev_err(dev, "failed to read QM_EQC_DW\n" ); |
174 | return ret; |
175 | } |
176 | |
177 | /* QM_AEQC_DW has 7 regs */ |
178 | ret = qm_read_regs(qm, QM_AEQC_DW0, data: vf_data->qm_aeqc_dw, nums: 7); |
179 | if (ret) { |
180 | dev_err(dev, "failed to read QM_AEQC_DW\n" ); |
181 | return ret; |
182 | } |
183 | |
184 | return 0; |
185 | } |
186 | |
187 | static int qm_set_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data) |
188 | { |
189 | struct device *dev = &qm->pdev->dev; |
190 | int ret; |
191 | |
192 | /* Check VF state */ |
193 | if (unlikely(hisi_qm_wait_mb_ready(qm))) { |
194 | dev_err(&qm->pdev->dev, "QM device is not ready to write\n" ); |
195 | return -EBUSY; |
196 | } |
197 | |
198 | ret = qm_write_regs(qm, QM_VF_AEQ_INT_MASK, data: &vf_data->aeq_int_mask, nums: 1); |
199 | if (ret) { |
200 | dev_err(dev, "failed to write QM_VF_AEQ_INT_MASK\n" ); |
201 | return ret; |
202 | } |
203 | |
204 | ret = qm_write_regs(qm, QM_VF_EQ_INT_MASK, data: &vf_data->eq_int_mask, nums: 1); |
205 | if (ret) { |
206 | dev_err(dev, "failed to write QM_VF_EQ_INT_MASK\n" ); |
207 | return ret; |
208 | } |
209 | |
210 | ret = qm_write_regs(qm, QM_IFC_INT_SOURCE_V, |
211 | data: &vf_data->ifc_int_source, nums: 1); |
212 | if (ret) { |
213 | dev_err(dev, "failed to write QM_IFC_INT_SOURCE_V\n" ); |
214 | return ret; |
215 | } |
216 | |
217 | ret = qm_write_regs(qm, QM_IFC_INT_MASK, data: &vf_data->ifc_int_mask, nums: 1); |
218 | if (ret) { |
219 | dev_err(dev, "failed to write QM_IFC_INT_MASK\n" ); |
220 | return ret; |
221 | } |
222 | |
223 | ret = qm_write_regs(qm, QM_IFC_INT_SET_V, data: &vf_data->ifc_int_set, nums: 1); |
224 | if (ret) { |
225 | dev_err(dev, "failed to write QM_IFC_INT_SET_V\n" ); |
226 | return ret; |
227 | } |
228 | |
229 | ret = qm_write_regs(qm, QM_QUE_ISO_CFG_V, data: &vf_data->que_iso_cfg, nums: 1); |
230 | if (ret) { |
231 | dev_err(dev, "failed to write QM_QUE_ISO_CFG_V\n" ); |
232 | return ret; |
233 | } |
234 | |
235 | ret = qm_write_regs(qm, QM_PAGE_SIZE, data: &vf_data->page_size, nums: 1); |
236 | if (ret) { |
237 | dev_err(dev, "failed to write QM_PAGE_SIZE\n" ); |
238 | return ret; |
239 | } |
240 | |
241 | /* QM_EQC_DW has 7 regs */ |
242 | ret = qm_write_regs(qm, QM_EQC_DW0, data: vf_data->qm_eqc_dw, nums: 7); |
243 | if (ret) { |
244 | dev_err(dev, "failed to write QM_EQC_DW\n" ); |
245 | return ret; |
246 | } |
247 | |
248 | /* QM_AEQC_DW has 7 regs */ |
249 | ret = qm_write_regs(qm, QM_AEQC_DW0, data: vf_data->qm_aeqc_dw, nums: 7); |
250 | if (ret) { |
251 | dev_err(dev, "failed to write QM_AEQC_DW\n" ); |
252 | return ret; |
253 | } |
254 | |
255 | return 0; |
256 | } |
257 | |
258 | static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, |
259 | u16 index, u8 priority) |
260 | { |
261 | u64 doorbell; |
262 | u64 dbase; |
263 | u16 randata = 0; |
264 | |
265 | if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ) |
266 | dbase = QM_DOORBELL_SQ_CQ_BASE_V2; |
267 | else |
268 | dbase = QM_DOORBELL_EQ_AEQ_BASE_V2; |
269 | |
270 | doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) | |
271 | ((u64)randata << QM_DB_RAND_SHIFT_V2) | |
272 | ((u64)index << QM_DB_INDEX_SHIFT_V2) | |
273 | ((u64)priority << QM_DB_PRIORITY_SHIFT_V2); |
274 | |
275 | writeq(val: doorbell, addr: qm->io_base + dbase); |
276 | } |
277 | |
278 | static int pf_qm_get_qp_num(struct hisi_qm *qm, int vf_id, u32 *rbase) |
279 | { |
280 | unsigned int val; |
281 | u64 sqc_vft; |
282 | u32 qp_num; |
283 | int ret; |
284 | |
285 | ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, |
286 | val & BIT(0), MB_POLL_PERIOD_US, |
287 | MB_POLL_TIMEOUT_US); |
288 | if (ret) |
289 | return ret; |
290 | |
291 | writel(val: 0x1, addr: qm->io_base + QM_VFT_CFG_OP_WR); |
292 | /* 0 mean SQC VFT */ |
293 | writel(val: 0x0, addr: qm->io_base + QM_VFT_CFG_TYPE); |
294 | writel(val: vf_id, addr: qm->io_base + QM_VFT_CFG); |
295 | |
296 | writel(val: 0x0, addr: qm->io_base + QM_VFT_CFG_RDY); |
297 | writel(val: 0x1, addr: qm->io_base + QM_VFT_CFG_OP_ENABLE); |
298 | |
299 | ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, |
300 | val & BIT(0), MB_POLL_PERIOD_US, |
301 | MB_POLL_TIMEOUT_US); |
302 | if (ret) |
303 | return ret; |
304 | |
305 | sqc_vft = readl(addr: qm->io_base + QM_VFT_CFG_DATA_L) | |
306 | ((u64)readl(addr: qm->io_base + QM_VFT_CFG_DATA_H) << |
307 | QM_XQC_ADDR_OFFSET); |
308 | *rbase = QM_SQC_VFT_BASE_MASK_V2 & |
309 | (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2); |
310 | qp_num = (QM_SQC_VFT_NUM_MASK_V2 & |
311 | (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1; |
312 | |
313 | return qp_num; |
314 | } |
315 | |
316 | static void qm_dev_cmd_init(struct hisi_qm *qm) |
317 | { |
318 | /* Clear VF communication status registers. */ |
319 | writel(val: 0x1, addr: qm->io_base + QM_IFC_INT_SOURCE_V); |
320 | |
321 | /* Enable pf and vf communication. */ |
322 | writel(val: 0x0, addr: qm->io_base + QM_IFC_INT_MASK); |
323 | } |
324 | |
325 | static int vf_qm_cache_wb(struct hisi_qm *qm) |
326 | { |
327 | unsigned int val; |
328 | |
329 | writel(val: 0x1, addr: qm->io_base + QM_CACHE_WB_START); |
330 | if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, |
331 | val, val & BIT(0), MB_POLL_PERIOD_US, |
332 | MB_POLL_TIMEOUT_US)) { |
333 | dev_err(&qm->pdev->dev, "vf QM writeback sqc cache fail\n" ); |
334 | return -EINVAL; |
335 | } |
336 | |
337 | return 0; |
338 | } |
339 | |
340 | static void vf_qm_fun_reset(struct hisi_qm *qm) |
341 | { |
342 | int i; |
343 | |
344 | for (i = 0; i < qm->qp_num; i++) |
345 | qm_db(qm, qn: i, QM_DOORBELL_CMD_SQ, index: 0, priority: 1); |
346 | } |
347 | |
348 | static int vf_qm_func_stop(struct hisi_qm *qm) |
349 | { |
350 | return hisi_qm_mb(qm, QM_MB_CMD_PAUSE_QM, dma_addr: 0, queue: 0, op: 0); |
351 | } |
352 | |
353 | static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, |
354 | struct hisi_acc_vf_migration_file *migf) |
355 | { |
356 | struct acc_vf_data *vf_data = &migf->vf_data; |
357 | struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; |
358 | struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm; |
359 | struct device *dev = &vf_qm->pdev->dev; |
360 | u32 que_iso_state; |
361 | int ret; |
362 | |
363 | if (migf->total_length < QM_MATCH_SIZE || hisi_acc_vdev->match_done) |
364 | return 0; |
365 | |
366 | if (vf_data->acc_magic != ACC_DEV_MAGIC) { |
367 | dev_err(dev, "failed to match ACC_DEV_MAGIC\n" ); |
368 | return -EINVAL; |
369 | } |
370 | |
371 | if (vf_data->dev_id != hisi_acc_vdev->vf_dev->device) { |
372 | dev_err(dev, "failed to match VF devices\n" ); |
373 | return -EINVAL; |
374 | } |
375 | |
376 | /* VF qp num check */ |
377 | ret = qm_get_vft(qm: vf_qm, base: &vf_qm->qp_base); |
378 | if (ret <= 0) { |
379 | dev_err(dev, "failed to get vft qp nums\n" ); |
380 | return -EINVAL; |
381 | } |
382 | |
383 | if (ret != vf_data->qp_num) { |
384 | dev_err(dev, "failed to match VF qp num\n" ); |
385 | return -EINVAL; |
386 | } |
387 | |
388 | vf_qm->qp_num = ret; |
389 | |
390 | /* VF isolation state check */ |
391 | ret = qm_read_regs(qm: pf_qm, QM_QUE_ISO_CFG_V, data: &que_iso_state, nums: 1); |
392 | if (ret) { |
393 | dev_err(dev, "failed to read QM_QUE_ISO_CFG_V\n" ); |
394 | return ret; |
395 | } |
396 | |
397 | if (vf_data->que_iso_cfg != que_iso_state) { |
398 | dev_err(dev, "failed to match isolation state\n" ); |
399 | return -EINVAL; |
400 | } |
401 | |
402 | ret = qm_write_regs(qm: vf_qm, QM_VF_STATE, data: &vf_data->vf_qm_state, nums: 1); |
403 | if (ret) { |
404 | dev_err(dev, "failed to write QM_VF_STATE\n" ); |
405 | return ret; |
406 | } |
407 | |
408 | hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; |
409 | hisi_acc_vdev->match_done = true; |
410 | return 0; |
411 | } |
412 | |
413 | static int vf_qm_get_match_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, |
414 | struct acc_vf_data *vf_data) |
415 | { |
416 | struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm; |
417 | struct device *dev = &pf_qm->pdev->dev; |
418 | int vf_id = hisi_acc_vdev->vf_id; |
419 | int ret; |
420 | |
421 | vf_data->acc_magic = ACC_DEV_MAGIC; |
422 | /* Save device id */ |
423 | vf_data->dev_id = hisi_acc_vdev->vf_dev->device; |
424 | |
425 | /* VF qp num save from PF */ |
426 | ret = pf_qm_get_qp_num(qm: pf_qm, vf_id, rbase: &vf_data->qp_base); |
427 | if (ret <= 0) { |
428 | dev_err(dev, "failed to get vft qp nums!\n" ); |
429 | return -EINVAL; |
430 | } |
431 | |
432 | vf_data->qp_num = ret; |
433 | |
434 | /* VF isolation state save from PF */ |
435 | ret = qm_read_regs(qm: pf_qm, QM_QUE_ISO_CFG_V, data: &vf_data->que_iso_cfg, nums: 1); |
436 | if (ret) { |
437 | dev_err(dev, "failed to read QM_QUE_ISO_CFG_V!\n" ); |
438 | return ret; |
439 | } |
440 | |
441 | return 0; |
442 | } |
443 | |
444 | static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, |
445 | struct hisi_acc_vf_migration_file *migf) |
446 | { |
447 | struct hisi_qm *qm = &hisi_acc_vdev->vf_qm; |
448 | struct device *dev = &qm->pdev->dev; |
449 | struct acc_vf_data *vf_data = &migf->vf_data; |
450 | int ret; |
451 | |
452 | /* Return if only match data was transferred */ |
453 | if (migf->total_length == QM_MATCH_SIZE) |
454 | return 0; |
455 | |
456 | if (migf->total_length < sizeof(struct acc_vf_data)) |
457 | return -EINVAL; |
458 | |
459 | qm->eqe_dma = vf_data->eqe_dma; |
460 | qm->aeqe_dma = vf_data->aeqe_dma; |
461 | qm->sqc_dma = vf_data->sqc_dma; |
462 | qm->cqc_dma = vf_data->cqc_dma; |
463 | |
464 | qm->qp_base = vf_data->qp_base; |
465 | qm->qp_num = vf_data->qp_num; |
466 | |
467 | ret = qm_set_regs(qm, vf_data); |
468 | if (ret) { |
469 | dev_err(dev, "set VF regs failed\n" ); |
470 | return ret; |
471 | } |
472 | |
473 | ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, dma_addr: qm->sqc_dma, queue: 0, op: 0); |
474 | if (ret) { |
475 | dev_err(dev, "set sqc failed\n" ); |
476 | return ret; |
477 | } |
478 | |
479 | ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, dma_addr: qm->cqc_dma, queue: 0, op: 0); |
480 | if (ret) { |
481 | dev_err(dev, "set cqc failed\n" ); |
482 | return ret; |
483 | } |
484 | |
485 | qm_dev_cmd_init(qm); |
486 | return 0; |
487 | } |
488 | |
489 | static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev, |
490 | struct hisi_acc_vf_migration_file *migf) |
491 | { |
492 | struct acc_vf_data *vf_data = &migf->vf_data; |
493 | struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; |
494 | struct device *dev = &vf_qm->pdev->dev; |
495 | int ret; |
496 | |
497 | if (unlikely(qm_wait_dev_not_ready(vf_qm))) { |
498 | /* Update state and return with match data */ |
499 | vf_data->vf_qm_state = QM_NOT_READY; |
500 | hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; |
501 | migf->total_length = QM_MATCH_SIZE; |
502 | return 0; |
503 | } |
504 | |
505 | vf_data->vf_qm_state = QM_READY; |
506 | hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; |
507 | |
508 | ret = vf_qm_cache_wb(qm: vf_qm); |
509 | if (ret) { |
510 | dev_err(dev, "failed to writeback QM Cache!\n" ); |
511 | return ret; |
512 | } |
513 | |
514 | ret = qm_get_regs(qm: vf_qm, vf_data); |
515 | if (ret) |
516 | return -EINVAL; |
517 | |
518 | /* Every reg is 32 bit, the dma address is 64 bit. */ |
519 | vf_data->eqe_dma = vf_data->qm_eqc_dw[1]; |
520 | vf_data->eqe_dma <<= QM_XQC_ADDR_OFFSET; |
521 | vf_data->eqe_dma |= vf_data->qm_eqc_dw[0]; |
522 | vf_data->aeqe_dma = vf_data->qm_aeqc_dw[1]; |
523 | vf_data->aeqe_dma <<= QM_XQC_ADDR_OFFSET; |
524 | vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[0]; |
525 | |
526 | /* Through SQC_BT/CQC_BT to get sqc and cqc address */ |
527 | ret = qm_get_sqc(qm: vf_qm, addr: &vf_data->sqc_dma); |
528 | if (ret) { |
529 | dev_err(dev, "failed to read SQC addr!\n" ); |
530 | return -EINVAL; |
531 | } |
532 | |
533 | ret = qm_get_cqc(qm: vf_qm, addr: &vf_data->cqc_dma); |
534 | if (ret) { |
535 | dev_err(dev, "failed to read CQC addr!\n" ); |
536 | return -EINVAL; |
537 | } |
538 | |
539 | migf->total_length = sizeof(struct acc_vf_data); |
540 | return 0; |
541 | } |
542 | |
543 | static struct hisi_acc_vf_core_device *hisi_acc_drvdata(struct pci_dev *pdev) |
544 | { |
545 | struct vfio_pci_core_device *core_device = dev_get_drvdata(dev: &pdev->dev); |
546 | |
547 | return container_of(core_device, struct hisi_acc_vf_core_device, |
548 | core_device); |
549 | } |
550 | |
551 | /* Check the PF's RAS state and Function INT state */ |
552 | static int |
553 | hisi_acc_check_int_state(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
554 | { |
555 | struct hisi_qm *vfqm = &hisi_acc_vdev->vf_qm; |
556 | struct hisi_qm *qm = hisi_acc_vdev->pf_qm; |
557 | struct pci_dev *vf_pdev = hisi_acc_vdev->vf_dev; |
558 | struct device *dev = &qm->pdev->dev; |
559 | u32 state; |
560 | |
561 | /* Check RAS state */ |
562 | state = qm_check_reg_state(qm, QM_ABNORMAL_INT_STATUS); |
563 | if (state) { |
564 | dev_err(dev, "failed to check QM RAS state!\n" ); |
565 | return -EBUSY; |
566 | } |
567 | |
568 | /* Check Function Communication state between PF and VF */ |
569 | state = qm_check_reg_state(qm: vfqm, QM_IFC_INT_STATUS); |
570 | if (state) { |
571 | dev_err(dev, "failed to check QM IFC INT state!\n" ); |
572 | return -EBUSY; |
573 | } |
574 | state = qm_check_reg_state(qm: vfqm, QM_IFC_INT_SET_V); |
575 | if (state) { |
576 | dev_err(dev, "failed to check QM IFC INT SET state!\n" ); |
577 | return -EBUSY; |
578 | } |
579 | |
580 | /* Check submodule task state */ |
581 | switch (vf_pdev->device) { |
582 | case PCI_DEVICE_ID_HUAWEI_SEC_VF: |
583 | state = qm_check_reg_state(qm, SEC_CORE_INT_STATUS); |
584 | if (state) { |
585 | dev_err(dev, "failed to check QM SEC Core INT state!\n" ); |
586 | return -EBUSY; |
587 | } |
588 | return 0; |
589 | case PCI_DEVICE_ID_HUAWEI_HPRE_VF: |
590 | state = qm_check_reg_state(qm, HPRE_HAC_INT_STATUS); |
591 | if (state) { |
592 | dev_err(dev, "failed to check QM HPRE HAC INT state!\n" ); |
593 | return -EBUSY; |
594 | } |
595 | return 0; |
596 | case PCI_DEVICE_ID_HUAWEI_ZIP_VF: |
597 | state = qm_check_reg_state(qm, HZIP_CORE_INT_STATUS); |
598 | if (state) { |
599 | dev_err(dev, "failed to check QM ZIP Core INT state!\n" ); |
600 | return -EBUSY; |
601 | } |
602 | return 0; |
603 | default: |
604 | dev_err(dev, "failed to detect acc module type!\n" ); |
605 | return -EINVAL; |
606 | } |
607 | } |
608 | |
609 | static void hisi_acc_vf_disable_fd(struct hisi_acc_vf_migration_file *migf) |
610 | { |
611 | mutex_lock(&migf->lock); |
612 | migf->disabled = true; |
613 | migf->total_length = 0; |
614 | migf->filp->f_pos = 0; |
615 | mutex_unlock(lock: &migf->lock); |
616 | } |
617 | |
618 | static void hisi_acc_vf_disable_fds(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
619 | { |
620 | if (hisi_acc_vdev->resuming_migf) { |
621 | hisi_acc_vf_disable_fd(migf: hisi_acc_vdev->resuming_migf); |
622 | fput(hisi_acc_vdev->resuming_migf->filp); |
623 | hisi_acc_vdev->resuming_migf = NULL; |
624 | } |
625 | |
626 | if (hisi_acc_vdev->saving_migf) { |
627 | hisi_acc_vf_disable_fd(migf: hisi_acc_vdev->saving_migf); |
628 | fput(hisi_acc_vdev->saving_migf->filp); |
629 | hisi_acc_vdev->saving_migf = NULL; |
630 | } |
631 | } |
632 | |
633 | /* |
634 | * This function is called in all state_mutex unlock cases to |
635 | * handle a 'deferred_reset' if exists. |
636 | */ |
637 | static void |
638 | hisi_acc_vf_state_mutex_unlock(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
639 | { |
640 | again: |
641 | spin_lock(lock: &hisi_acc_vdev->reset_lock); |
642 | if (hisi_acc_vdev->deferred_reset) { |
643 | hisi_acc_vdev->deferred_reset = false; |
644 | spin_unlock(lock: &hisi_acc_vdev->reset_lock); |
645 | hisi_acc_vdev->vf_qm_state = QM_NOT_READY; |
646 | hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; |
647 | hisi_acc_vf_disable_fds(hisi_acc_vdev); |
648 | goto again; |
649 | } |
650 | mutex_unlock(lock: &hisi_acc_vdev->state_mutex); |
651 | spin_unlock(lock: &hisi_acc_vdev->reset_lock); |
652 | } |
653 | |
654 | static void hisi_acc_vf_start_device(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
655 | { |
656 | struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; |
657 | |
658 | if (hisi_acc_vdev->vf_qm_state != QM_READY) |
659 | return; |
660 | |
661 | /* Make sure the device is enabled */ |
662 | qm_dev_cmd_init(qm: vf_qm); |
663 | |
664 | vf_qm_fun_reset(qm: vf_qm); |
665 | } |
666 | |
667 | static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
668 | { |
669 | struct device *dev = &hisi_acc_vdev->vf_dev->dev; |
670 | struct hisi_acc_vf_migration_file *migf = hisi_acc_vdev->resuming_migf; |
671 | int ret; |
672 | |
673 | /* Recover data to VF */ |
674 | ret = vf_qm_load_data(hisi_acc_vdev, migf); |
675 | if (ret) { |
676 | dev_err(dev, "failed to recover the VF!\n" ); |
677 | return ret; |
678 | } |
679 | |
680 | return 0; |
681 | } |
682 | |
683 | static int hisi_acc_vf_release_file(struct inode *inode, struct file *filp) |
684 | { |
685 | struct hisi_acc_vf_migration_file *migf = filp->private_data; |
686 | |
687 | hisi_acc_vf_disable_fd(migf); |
688 | mutex_destroy(lock: &migf->lock); |
689 | kfree(objp: migf); |
690 | return 0; |
691 | } |
692 | |
693 | static ssize_t hisi_acc_vf_resume_write(struct file *filp, const char __user *buf, |
694 | size_t len, loff_t *pos) |
695 | { |
696 | struct hisi_acc_vf_migration_file *migf = filp->private_data; |
697 | loff_t requested_length; |
698 | ssize_t done = 0; |
699 | int ret; |
700 | |
701 | if (pos) |
702 | return -ESPIPE; |
703 | pos = &filp->f_pos; |
704 | |
705 | if (*pos < 0 || |
706 | check_add_overflow((loff_t)len, *pos, &requested_length)) |
707 | return -EINVAL; |
708 | |
709 | if (requested_length > sizeof(struct acc_vf_data)) |
710 | return -ENOMEM; |
711 | |
712 | mutex_lock(&migf->lock); |
713 | if (migf->disabled) { |
714 | done = -ENODEV; |
715 | goto out_unlock; |
716 | } |
717 | |
718 | ret = copy_from_user(to: &migf->vf_data, from: buf, n: len); |
719 | if (ret) { |
720 | done = -EFAULT; |
721 | goto out_unlock; |
722 | } |
723 | *pos += len; |
724 | done = len; |
725 | migf->total_length += len; |
726 | |
727 | ret = vf_qm_check_match(hisi_acc_vdev: migf->hisi_acc_vdev, migf); |
728 | if (ret) |
729 | done = -EFAULT; |
730 | out_unlock: |
731 | mutex_unlock(lock: &migf->lock); |
732 | return done; |
733 | } |
734 | |
735 | static const struct file_operations hisi_acc_vf_resume_fops = { |
736 | .owner = THIS_MODULE, |
737 | .write = hisi_acc_vf_resume_write, |
738 | .release = hisi_acc_vf_release_file, |
739 | .llseek = no_llseek, |
740 | }; |
741 | |
742 | static struct hisi_acc_vf_migration_file * |
743 | hisi_acc_vf_pci_resume(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
744 | { |
745 | struct hisi_acc_vf_migration_file *migf; |
746 | |
747 | migf = kzalloc(size: sizeof(*migf), GFP_KERNEL_ACCOUNT); |
748 | if (!migf) |
749 | return ERR_PTR(error: -ENOMEM); |
750 | |
751 | migf->filp = anon_inode_getfile(name: "hisi_acc_vf_mig" , fops: &hisi_acc_vf_resume_fops, priv: migf, |
752 | O_WRONLY); |
753 | if (IS_ERR(ptr: migf->filp)) { |
754 | int err = PTR_ERR(ptr: migf->filp); |
755 | |
756 | kfree(objp: migf); |
757 | return ERR_PTR(error: err); |
758 | } |
759 | |
760 | stream_open(inode: migf->filp->f_inode, filp: migf->filp); |
761 | mutex_init(&migf->lock); |
762 | migf->hisi_acc_vdev = hisi_acc_vdev; |
763 | return migf; |
764 | } |
765 | |
766 | static long hisi_acc_vf_precopy_ioctl(struct file *filp, |
767 | unsigned int cmd, unsigned long arg) |
768 | { |
769 | struct hisi_acc_vf_migration_file *migf = filp->private_data; |
770 | struct hisi_acc_vf_core_device *hisi_acc_vdev = migf->hisi_acc_vdev; |
771 | loff_t *pos = &filp->f_pos; |
772 | struct vfio_precopy_info info; |
773 | unsigned long minsz; |
774 | int ret; |
775 | |
776 | if (cmd != VFIO_MIG_GET_PRECOPY_INFO) |
777 | return -ENOTTY; |
778 | |
779 | minsz = offsetofend(struct vfio_precopy_info, dirty_bytes); |
780 | |
781 | if (copy_from_user(to: &info, from: (void __user *)arg, n: minsz)) |
782 | return -EFAULT; |
783 | if (info.argsz < minsz) |
784 | return -EINVAL; |
785 | |
786 | mutex_lock(&hisi_acc_vdev->state_mutex); |
787 | if (hisi_acc_vdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY) { |
788 | mutex_unlock(lock: &hisi_acc_vdev->state_mutex); |
789 | return -EINVAL; |
790 | } |
791 | |
792 | mutex_lock(&migf->lock); |
793 | |
794 | if (migf->disabled) { |
795 | ret = -ENODEV; |
796 | goto out; |
797 | } |
798 | |
799 | if (*pos > migf->total_length) { |
800 | ret = -EINVAL; |
801 | goto out; |
802 | } |
803 | |
804 | info.dirty_bytes = 0; |
805 | info.initial_bytes = migf->total_length - *pos; |
806 | |
807 | ret = copy_to_user(to: (void __user *)arg, from: &info, n: minsz) ? -EFAULT : 0; |
808 | out: |
809 | mutex_unlock(lock: &migf->lock); |
810 | mutex_unlock(lock: &hisi_acc_vdev->state_mutex); |
811 | return ret; |
812 | } |
813 | |
814 | static ssize_t hisi_acc_vf_save_read(struct file *filp, char __user *buf, size_t len, |
815 | loff_t *pos) |
816 | { |
817 | struct hisi_acc_vf_migration_file *migf = filp->private_data; |
818 | ssize_t done = 0; |
819 | int ret; |
820 | |
821 | if (pos) |
822 | return -ESPIPE; |
823 | pos = &filp->f_pos; |
824 | |
825 | mutex_lock(&migf->lock); |
826 | if (*pos > migf->total_length) { |
827 | done = -EINVAL; |
828 | goto out_unlock; |
829 | } |
830 | |
831 | if (migf->disabled) { |
832 | done = -ENODEV; |
833 | goto out_unlock; |
834 | } |
835 | |
836 | len = min_t(size_t, migf->total_length - *pos, len); |
837 | if (len) { |
838 | ret = copy_to_user(to: buf, from: &migf->vf_data, n: len); |
839 | if (ret) { |
840 | done = -EFAULT; |
841 | goto out_unlock; |
842 | } |
843 | *pos += len; |
844 | done = len; |
845 | } |
846 | out_unlock: |
847 | mutex_unlock(lock: &migf->lock); |
848 | return done; |
849 | } |
850 | |
851 | static const struct file_operations hisi_acc_vf_save_fops = { |
852 | .owner = THIS_MODULE, |
853 | .read = hisi_acc_vf_save_read, |
854 | .unlocked_ioctl = hisi_acc_vf_precopy_ioctl, |
855 | .compat_ioctl = compat_ptr_ioctl, |
856 | .release = hisi_acc_vf_release_file, |
857 | .llseek = no_llseek, |
858 | }; |
859 | |
860 | static struct hisi_acc_vf_migration_file * |
861 | hisi_acc_open_saving_migf(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
862 | { |
863 | struct hisi_acc_vf_migration_file *migf; |
864 | int ret; |
865 | |
866 | migf = kzalloc(size: sizeof(*migf), GFP_KERNEL_ACCOUNT); |
867 | if (!migf) |
868 | return ERR_PTR(error: -ENOMEM); |
869 | |
870 | migf->filp = anon_inode_getfile(name: "hisi_acc_vf_mig" , fops: &hisi_acc_vf_save_fops, priv: migf, |
871 | O_RDONLY); |
872 | if (IS_ERR(ptr: migf->filp)) { |
873 | int err = PTR_ERR(ptr: migf->filp); |
874 | |
875 | kfree(objp: migf); |
876 | return ERR_PTR(error: err); |
877 | } |
878 | |
879 | stream_open(inode: migf->filp->f_inode, filp: migf->filp); |
880 | mutex_init(&migf->lock); |
881 | migf->hisi_acc_vdev = hisi_acc_vdev; |
882 | |
883 | ret = vf_qm_get_match_data(hisi_acc_vdev, vf_data: &migf->vf_data); |
884 | if (ret) { |
885 | fput(migf->filp); |
886 | return ERR_PTR(error: ret); |
887 | } |
888 | |
889 | return migf; |
890 | } |
891 | |
892 | static struct hisi_acc_vf_migration_file * |
893 | hisi_acc_vf_pre_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
894 | { |
895 | struct hisi_acc_vf_migration_file *migf; |
896 | |
897 | migf = hisi_acc_open_saving_migf(hisi_acc_vdev); |
898 | if (IS_ERR(ptr: migf)) |
899 | return migf; |
900 | |
901 | migf->total_length = QM_MATCH_SIZE; |
902 | return migf; |
903 | } |
904 | |
905 | static struct hisi_acc_vf_migration_file * |
906 | hisi_acc_vf_stop_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev, bool open) |
907 | { |
908 | int ret; |
909 | struct hisi_acc_vf_migration_file *migf = NULL; |
910 | |
911 | if (open) { |
912 | /* |
913 | * Userspace didn't use PRECOPY support. Hence saving_migf |
914 | * is not opened yet. |
915 | */ |
916 | migf = hisi_acc_open_saving_migf(hisi_acc_vdev); |
917 | if (IS_ERR(ptr: migf)) |
918 | return migf; |
919 | } else { |
920 | migf = hisi_acc_vdev->saving_migf; |
921 | } |
922 | |
923 | ret = vf_qm_state_save(hisi_acc_vdev, migf); |
924 | if (ret) |
925 | return ERR_PTR(error: ret); |
926 | |
927 | return open ? migf : NULL; |
928 | } |
929 | |
930 | static int hisi_acc_vf_stop_device(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
931 | { |
932 | struct device *dev = &hisi_acc_vdev->vf_dev->dev; |
933 | struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; |
934 | int ret; |
935 | |
936 | ret = vf_qm_func_stop(qm: vf_qm); |
937 | if (ret) { |
938 | dev_err(dev, "failed to stop QM VF function!\n" ); |
939 | return ret; |
940 | } |
941 | |
942 | ret = hisi_acc_check_int_state(hisi_acc_vdev); |
943 | if (ret) { |
944 | dev_err(dev, "failed to check QM INT state!\n" ); |
945 | return ret; |
946 | } |
947 | return 0; |
948 | } |
949 | |
950 | static struct file * |
951 | hisi_acc_vf_set_device_state(struct hisi_acc_vf_core_device *hisi_acc_vdev, |
952 | u32 new) |
953 | { |
954 | u32 cur = hisi_acc_vdev->mig_state; |
955 | int ret; |
956 | |
957 | if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_PRE_COPY) { |
958 | struct hisi_acc_vf_migration_file *migf; |
959 | |
960 | migf = hisi_acc_vf_pre_copy(hisi_acc_vdev); |
961 | if (IS_ERR(ptr: migf)) |
962 | return ERR_CAST(ptr: migf); |
963 | get_file(f: migf->filp); |
964 | hisi_acc_vdev->saving_migf = migf; |
965 | return migf->filp; |
966 | } |
967 | |
968 | if (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_STOP_COPY) { |
969 | struct hisi_acc_vf_migration_file *migf; |
970 | |
971 | ret = hisi_acc_vf_stop_device(hisi_acc_vdev); |
972 | if (ret) |
973 | return ERR_PTR(error: ret); |
974 | |
975 | migf = hisi_acc_vf_stop_copy(hisi_acc_vdev, open: false); |
976 | if (IS_ERR(ptr: migf)) |
977 | return ERR_CAST(ptr: migf); |
978 | |
979 | return NULL; |
980 | } |
981 | |
982 | if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_STOP) { |
983 | ret = hisi_acc_vf_stop_device(hisi_acc_vdev); |
984 | if (ret) |
985 | return ERR_PTR(error: ret); |
986 | return NULL; |
987 | } |
988 | |
989 | if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) { |
990 | struct hisi_acc_vf_migration_file *migf; |
991 | |
992 | migf = hisi_acc_vf_stop_copy(hisi_acc_vdev, open: true); |
993 | if (IS_ERR(ptr: migf)) |
994 | return ERR_CAST(ptr: migf); |
995 | get_file(f: migf->filp); |
996 | hisi_acc_vdev->saving_migf = migf; |
997 | return migf->filp; |
998 | } |
999 | |
1000 | if ((cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP)) { |
1001 | hisi_acc_vf_disable_fds(hisi_acc_vdev); |
1002 | return NULL; |
1003 | } |
1004 | |
1005 | if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) { |
1006 | struct hisi_acc_vf_migration_file *migf; |
1007 | |
1008 | migf = hisi_acc_vf_pci_resume(hisi_acc_vdev); |
1009 | if (IS_ERR(ptr: migf)) |
1010 | return ERR_CAST(ptr: migf); |
1011 | get_file(f: migf->filp); |
1012 | hisi_acc_vdev->resuming_migf = migf; |
1013 | return migf->filp; |
1014 | } |
1015 | |
1016 | if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) { |
1017 | ret = hisi_acc_vf_load_state(hisi_acc_vdev); |
1018 | if (ret) |
1019 | return ERR_PTR(error: ret); |
1020 | hisi_acc_vf_disable_fds(hisi_acc_vdev); |
1021 | return NULL; |
1022 | } |
1023 | |
1024 | if (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_RUNNING) { |
1025 | hisi_acc_vf_disable_fds(hisi_acc_vdev); |
1026 | return NULL; |
1027 | } |
1028 | |
1029 | if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING) { |
1030 | hisi_acc_vf_start_device(hisi_acc_vdev); |
1031 | return NULL; |
1032 | } |
1033 | |
1034 | /* |
1035 | * vfio_mig_get_next_state() does not use arcs other than the above |
1036 | */ |
1037 | WARN_ON(true); |
1038 | return ERR_PTR(error: -EINVAL); |
1039 | } |
1040 | |
1041 | static struct file * |
1042 | hisi_acc_vfio_pci_set_device_state(struct vfio_device *vdev, |
1043 | enum vfio_device_mig_state new_state) |
1044 | { |
1045 | struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(vdev, |
1046 | struct hisi_acc_vf_core_device, core_device.vdev); |
1047 | enum vfio_device_mig_state next_state; |
1048 | struct file *res = NULL; |
1049 | int ret; |
1050 | |
1051 | mutex_lock(&hisi_acc_vdev->state_mutex); |
1052 | while (new_state != hisi_acc_vdev->mig_state) { |
1053 | ret = vfio_mig_get_next_state(device: vdev, |
1054 | cur_fsm: hisi_acc_vdev->mig_state, |
1055 | new_fsm: new_state, next_fsm: &next_state); |
1056 | if (ret) { |
1057 | res = ERR_PTR(error: -EINVAL); |
1058 | break; |
1059 | } |
1060 | |
1061 | res = hisi_acc_vf_set_device_state(hisi_acc_vdev, new: next_state); |
1062 | if (IS_ERR(ptr: res)) |
1063 | break; |
1064 | hisi_acc_vdev->mig_state = next_state; |
1065 | if (WARN_ON(res && new_state != hisi_acc_vdev->mig_state)) { |
1066 | fput(res); |
1067 | res = ERR_PTR(error: -EINVAL); |
1068 | break; |
1069 | } |
1070 | } |
1071 | hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev); |
1072 | return res; |
1073 | } |
1074 | |
1075 | static int |
1076 | hisi_acc_vfio_pci_get_data_size(struct vfio_device *vdev, |
1077 | unsigned long *stop_copy_length) |
1078 | { |
1079 | *stop_copy_length = sizeof(struct acc_vf_data); |
1080 | return 0; |
1081 | } |
1082 | |
1083 | static int |
1084 | hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev, |
1085 | enum vfio_device_mig_state *curr_state) |
1086 | { |
1087 | struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(vdev, |
1088 | struct hisi_acc_vf_core_device, core_device.vdev); |
1089 | |
1090 | mutex_lock(&hisi_acc_vdev->state_mutex); |
1091 | *curr_state = hisi_acc_vdev->mig_state; |
1092 | hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev); |
1093 | return 0; |
1094 | } |
1095 | |
1096 | static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev) |
1097 | { |
1098 | struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); |
1099 | |
1100 | if (hisi_acc_vdev->core_device.vdev.migration_flags != |
1101 | VFIO_MIGRATION_STOP_COPY) |
1102 | return; |
1103 | |
1104 | /* |
1105 | * As the higher VFIO layers are holding locks across reset and using |
1106 | * those same locks with the mm_lock we need to prevent ABBA deadlock |
1107 | * with the state_mutex and mm_lock. |
1108 | * In case the state_mutex was taken already we defer the cleanup work |
1109 | * to the unlock flow of the other running context. |
1110 | */ |
1111 | spin_lock(lock: &hisi_acc_vdev->reset_lock); |
1112 | hisi_acc_vdev->deferred_reset = true; |
1113 | if (!mutex_trylock(lock: &hisi_acc_vdev->state_mutex)) { |
1114 | spin_unlock(lock: &hisi_acc_vdev->reset_lock); |
1115 | return; |
1116 | } |
1117 | spin_unlock(lock: &hisi_acc_vdev->reset_lock); |
1118 | hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev); |
1119 | } |
1120 | |
1121 | static int hisi_acc_vf_qm_init(struct hisi_acc_vf_core_device *hisi_acc_vdev) |
1122 | { |
1123 | struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device; |
1124 | struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; |
1125 | struct pci_dev *vf_dev = vdev->pdev; |
1126 | |
1127 | /* |
1128 | * ACC VF dev BAR2 region consists of both functional register space |
1129 | * and migration control register space. For migration to work, we |
1130 | * need access to both. Hence, we map the entire BAR2 region here. |
1131 | * But unnecessarily exposing the migration BAR region to the Guest |
1132 | * has the potential to prevent/corrupt the Guest migration. Hence, |
1133 | * we restrict access to the migration control space from |
1134 | * Guest(Please see mmap/ioctl/read/write override functions). |
1135 | * |
1136 | * Please note that it is OK to expose the entire VF BAR if migration |
1137 | * is not supported or required as this cannot affect the ACC PF |
1138 | * configurations. |
1139 | * |
1140 | * Also the HiSilicon ACC VF devices supported by this driver on |
1141 | * HiSilicon hardware platforms are integrated end point devices |
1142 | * and the platform lacks the capability to perform any PCIe P2P |
1143 | * between these devices. |
1144 | */ |
1145 | |
1146 | vf_qm->io_base = |
1147 | ioremap(pci_resource_start(vf_dev, VFIO_PCI_BAR2_REGION_INDEX), |
1148 | pci_resource_len(vf_dev, VFIO_PCI_BAR2_REGION_INDEX)); |
1149 | if (!vf_qm->io_base) |
1150 | return -EIO; |
1151 | |
1152 | vf_qm->fun_type = QM_HW_VF; |
1153 | vf_qm->pdev = vf_dev; |
1154 | mutex_init(&vf_qm->mailbox_lock); |
1155 | |
1156 | return 0; |
1157 | } |
1158 | |
1159 | static struct hisi_qm *hisi_acc_get_pf_qm(struct pci_dev *pdev) |
1160 | { |
1161 | struct hisi_qm *pf_qm; |
1162 | struct pci_driver *pf_driver; |
1163 | |
1164 | if (!pdev->is_virtfn) |
1165 | return NULL; |
1166 | |
1167 | switch (pdev->device) { |
1168 | case PCI_DEVICE_ID_HUAWEI_SEC_VF: |
1169 | pf_driver = hisi_sec_get_pf_driver(); |
1170 | break; |
1171 | case PCI_DEVICE_ID_HUAWEI_HPRE_VF: |
1172 | pf_driver = hisi_hpre_get_pf_driver(); |
1173 | break; |
1174 | case PCI_DEVICE_ID_HUAWEI_ZIP_VF: |
1175 | pf_driver = hisi_zip_get_pf_driver(); |
1176 | break; |
1177 | default: |
1178 | return NULL; |
1179 | } |
1180 | |
1181 | if (!pf_driver) |
1182 | return NULL; |
1183 | |
1184 | pf_qm = pci_iov_get_pf_drvdata(dev: pdev, pf_driver); |
1185 | |
1186 | return !IS_ERR(ptr: pf_qm) ? pf_qm : NULL; |
1187 | } |
1188 | |
1189 | static int hisi_acc_pci_rw_access_check(struct vfio_device *core_vdev, |
1190 | size_t count, loff_t *ppos, |
1191 | size_t *new_count) |
1192 | { |
1193 | unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); |
1194 | struct vfio_pci_core_device *vdev = |
1195 | container_of(core_vdev, struct vfio_pci_core_device, vdev); |
1196 | |
1197 | if (index == VFIO_PCI_BAR2_REGION_INDEX) { |
1198 | loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; |
1199 | resource_size_t end = pci_resource_len(vdev->pdev, index) / 2; |
1200 | |
1201 | /* Check if access is for migration control region */ |
1202 | if (pos >= end) |
1203 | return -EINVAL; |
1204 | |
1205 | *new_count = min(count, (size_t)(end - pos)); |
1206 | } |
1207 | |
1208 | return 0; |
1209 | } |
1210 | |
1211 | static int hisi_acc_vfio_pci_mmap(struct vfio_device *core_vdev, |
1212 | struct vm_area_struct *vma) |
1213 | { |
1214 | struct vfio_pci_core_device *vdev = |
1215 | container_of(core_vdev, struct vfio_pci_core_device, vdev); |
1216 | unsigned int index; |
1217 | |
1218 | index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); |
1219 | if (index == VFIO_PCI_BAR2_REGION_INDEX) { |
1220 | u64 req_len, pgoff, req_start; |
1221 | resource_size_t end = pci_resource_len(vdev->pdev, index) / 2; |
1222 | |
1223 | req_len = vma->vm_end - vma->vm_start; |
1224 | pgoff = vma->vm_pgoff & |
1225 | ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); |
1226 | req_start = pgoff << PAGE_SHIFT; |
1227 | |
1228 | if (req_start + req_len > end) |
1229 | return -EINVAL; |
1230 | } |
1231 | |
1232 | return vfio_pci_core_mmap(core_vdev, vma); |
1233 | } |
1234 | |
1235 | static ssize_t hisi_acc_vfio_pci_write(struct vfio_device *core_vdev, |
1236 | const char __user *buf, size_t count, |
1237 | loff_t *ppos) |
1238 | { |
1239 | size_t new_count = count; |
1240 | int ret; |
1241 | |
1242 | ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, new_count: &new_count); |
1243 | if (ret) |
1244 | return ret; |
1245 | |
1246 | return vfio_pci_core_write(core_vdev, buf, count: new_count, ppos); |
1247 | } |
1248 | |
1249 | static ssize_t hisi_acc_vfio_pci_read(struct vfio_device *core_vdev, |
1250 | char __user *buf, size_t count, |
1251 | loff_t *ppos) |
1252 | { |
1253 | size_t new_count = count; |
1254 | int ret; |
1255 | |
1256 | ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, new_count: &new_count); |
1257 | if (ret) |
1258 | return ret; |
1259 | |
1260 | return vfio_pci_core_read(core_vdev, buf, count: new_count, ppos); |
1261 | } |
1262 | |
1263 | static long hisi_acc_vfio_pci_ioctl(struct vfio_device *core_vdev, unsigned int cmd, |
1264 | unsigned long arg) |
1265 | { |
1266 | if (cmd == VFIO_DEVICE_GET_REGION_INFO) { |
1267 | struct vfio_pci_core_device *vdev = |
1268 | container_of(core_vdev, struct vfio_pci_core_device, vdev); |
1269 | struct pci_dev *pdev = vdev->pdev; |
1270 | struct vfio_region_info info; |
1271 | unsigned long minsz; |
1272 | |
1273 | minsz = offsetofend(struct vfio_region_info, offset); |
1274 | |
1275 | if (copy_from_user(to: &info, from: (void __user *)arg, n: minsz)) |
1276 | return -EFAULT; |
1277 | |
1278 | if (info.argsz < minsz) |
1279 | return -EINVAL; |
1280 | |
1281 | if (info.index == VFIO_PCI_BAR2_REGION_INDEX) { |
1282 | info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); |
1283 | |
1284 | /* |
1285 | * ACC VF dev BAR2 region consists of both functional |
1286 | * register space and migration control register space. |
1287 | * Report only the functional region to Guest. |
1288 | */ |
1289 | info.size = pci_resource_len(pdev, info.index) / 2; |
1290 | |
1291 | info.flags = VFIO_REGION_INFO_FLAG_READ | |
1292 | VFIO_REGION_INFO_FLAG_WRITE | |
1293 | VFIO_REGION_INFO_FLAG_MMAP; |
1294 | |
1295 | return copy_to_user(to: (void __user *)arg, from: &info, n: minsz) ? |
1296 | -EFAULT : 0; |
1297 | } |
1298 | } |
1299 | return vfio_pci_core_ioctl(core_vdev, cmd, arg); |
1300 | } |
1301 | |
1302 | static int hisi_acc_vfio_pci_open_device(struct vfio_device *core_vdev) |
1303 | { |
1304 | struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev, |
1305 | struct hisi_acc_vf_core_device, core_device.vdev); |
1306 | struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device; |
1307 | int ret; |
1308 | |
1309 | ret = vfio_pci_core_enable(vdev); |
1310 | if (ret) |
1311 | return ret; |
1312 | |
1313 | if (core_vdev->mig_ops) { |
1314 | ret = hisi_acc_vf_qm_init(hisi_acc_vdev); |
1315 | if (ret) { |
1316 | vfio_pci_core_disable(vdev); |
1317 | return ret; |
1318 | } |
1319 | hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; |
1320 | } |
1321 | |
1322 | vfio_pci_core_finish_enable(vdev); |
1323 | return 0; |
1324 | } |
1325 | |
1326 | static void hisi_acc_vfio_pci_close_device(struct vfio_device *core_vdev) |
1327 | { |
1328 | struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev, |
1329 | struct hisi_acc_vf_core_device, core_device.vdev); |
1330 | struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; |
1331 | |
1332 | iounmap(addr: vf_qm->io_base); |
1333 | vfio_pci_core_close_device(core_vdev); |
1334 | } |
1335 | |
1336 | static const struct vfio_migration_ops hisi_acc_vfio_pci_migrn_state_ops = { |
1337 | .migration_set_state = hisi_acc_vfio_pci_set_device_state, |
1338 | .migration_get_state = hisi_acc_vfio_pci_get_device_state, |
1339 | .migration_get_data_size = hisi_acc_vfio_pci_get_data_size, |
1340 | }; |
1341 | |
1342 | static int hisi_acc_vfio_pci_migrn_init_dev(struct vfio_device *core_vdev) |
1343 | { |
1344 | struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev, |
1345 | struct hisi_acc_vf_core_device, core_device.vdev); |
1346 | struct pci_dev *pdev = to_pci_dev(core_vdev->dev); |
1347 | struct hisi_qm *pf_qm = hisi_acc_get_pf_qm(pdev); |
1348 | |
1349 | hisi_acc_vdev->vf_id = pci_iov_vf_id(dev: pdev) + 1; |
1350 | hisi_acc_vdev->pf_qm = pf_qm; |
1351 | hisi_acc_vdev->vf_dev = pdev; |
1352 | mutex_init(&hisi_acc_vdev->state_mutex); |
1353 | |
1354 | core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY; |
1355 | core_vdev->mig_ops = &hisi_acc_vfio_pci_migrn_state_ops; |
1356 | |
1357 | return vfio_pci_core_init_dev(core_vdev); |
1358 | } |
1359 | |
1360 | static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = { |
1361 | .name = "hisi-acc-vfio-pci-migration" , |
1362 | .init = hisi_acc_vfio_pci_migrn_init_dev, |
1363 | .release = vfio_pci_core_release_dev, |
1364 | .open_device = hisi_acc_vfio_pci_open_device, |
1365 | .close_device = hisi_acc_vfio_pci_close_device, |
1366 | .ioctl = hisi_acc_vfio_pci_ioctl, |
1367 | .device_feature = vfio_pci_core_ioctl_feature, |
1368 | .read = hisi_acc_vfio_pci_read, |
1369 | .write = hisi_acc_vfio_pci_write, |
1370 | .mmap = hisi_acc_vfio_pci_mmap, |
1371 | .request = vfio_pci_core_request, |
1372 | .match = vfio_pci_core_match, |
1373 | .bind_iommufd = vfio_iommufd_physical_bind, |
1374 | .unbind_iommufd = vfio_iommufd_physical_unbind, |
1375 | .attach_ioas = vfio_iommufd_physical_attach_ioas, |
1376 | .detach_ioas = vfio_iommufd_physical_detach_ioas, |
1377 | }; |
1378 | |
1379 | static const struct vfio_device_ops hisi_acc_vfio_pci_ops = { |
1380 | .name = "hisi-acc-vfio-pci" , |
1381 | .init = vfio_pci_core_init_dev, |
1382 | .release = vfio_pci_core_release_dev, |
1383 | .open_device = hisi_acc_vfio_pci_open_device, |
1384 | .close_device = vfio_pci_core_close_device, |
1385 | .ioctl = vfio_pci_core_ioctl, |
1386 | .device_feature = vfio_pci_core_ioctl_feature, |
1387 | .read = vfio_pci_core_read, |
1388 | .write = vfio_pci_core_write, |
1389 | .mmap = vfio_pci_core_mmap, |
1390 | .request = vfio_pci_core_request, |
1391 | .match = vfio_pci_core_match, |
1392 | .bind_iommufd = vfio_iommufd_physical_bind, |
1393 | .unbind_iommufd = vfio_iommufd_physical_unbind, |
1394 | .attach_ioas = vfio_iommufd_physical_attach_ioas, |
1395 | .detach_ioas = vfio_iommufd_physical_detach_ioas, |
1396 | }; |
1397 | |
1398 | static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
1399 | { |
1400 | struct hisi_acc_vf_core_device *hisi_acc_vdev; |
1401 | const struct vfio_device_ops *ops = &hisi_acc_vfio_pci_ops; |
1402 | struct hisi_qm *pf_qm; |
1403 | int vf_id; |
1404 | int ret; |
1405 | |
1406 | pf_qm = hisi_acc_get_pf_qm(pdev); |
1407 | if (pf_qm && pf_qm->ver >= QM_HW_V3) { |
1408 | vf_id = pci_iov_vf_id(dev: pdev); |
1409 | if (vf_id >= 0) |
1410 | ops = &hisi_acc_vfio_pci_migrn_ops; |
1411 | else |
1412 | pci_warn(pdev, "migration support failed, continue with generic interface\n" ); |
1413 | } |
1414 | |
1415 | hisi_acc_vdev = vfio_alloc_device(hisi_acc_vf_core_device, |
1416 | core_device.vdev, &pdev->dev, ops); |
1417 | if (IS_ERR(ptr: hisi_acc_vdev)) |
1418 | return PTR_ERR(ptr: hisi_acc_vdev); |
1419 | |
1420 | dev_set_drvdata(dev: &pdev->dev, data: &hisi_acc_vdev->core_device); |
1421 | ret = vfio_pci_core_register_device(vdev: &hisi_acc_vdev->core_device); |
1422 | if (ret) |
1423 | goto out_put_vdev; |
1424 | return 0; |
1425 | |
1426 | out_put_vdev: |
1427 | vfio_put_device(device: &hisi_acc_vdev->core_device.vdev); |
1428 | return ret; |
1429 | } |
1430 | |
1431 | static void hisi_acc_vfio_pci_remove(struct pci_dev *pdev) |
1432 | { |
1433 | struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); |
1434 | |
1435 | vfio_pci_core_unregister_device(vdev: &hisi_acc_vdev->core_device); |
1436 | vfio_put_device(device: &hisi_acc_vdev->core_device.vdev); |
1437 | } |
1438 | |
1439 | static const struct pci_device_id hisi_acc_vfio_pci_table[] = { |
1440 | { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_VF) }, |
1441 | { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_HPRE_VF) }, |
1442 | { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_VF) }, |
1443 | { } |
1444 | }; |
1445 | |
1446 | MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table); |
1447 | |
1448 | static const struct pci_error_handlers hisi_acc_vf_err_handlers = { |
1449 | .reset_done = hisi_acc_vf_pci_aer_reset_done, |
1450 | .error_detected = vfio_pci_core_aer_err_detected, |
1451 | }; |
1452 | |
1453 | static struct pci_driver hisi_acc_vfio_pci_driver = { |
1454 | .name = KBUILD_MODNAME, |
1455 | .id_table = hisi_acc_vfio_pci_table, |
1456 | .probe = hisi_acc_vfio_pci_probe, |
1457 | .remove = hisi_acc_vfio_pci_remove, |
1458 | .err_handler = &hisi_acc_vf_err_handlers, |
1459 | .driver_managed_dma = true, |
1460 | }; |
1461 | |
1462 | module_pci_driver(hisi_acc_vfio_pci_driver); |
1463 | |
1464 | MODULE_LICENSE("GPL v2" ); |
1465 | MODULE_AUTHOR("Liu Longfang <liulongfang@huawei.com>" ); |
1466 | MODULE_AUTHOR("Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>" ); |
1467 | MODULE_DESCRIPTION("HiSilicon VFIO PCI - VFIO PCI driver with live migration support for HiSilicon ACC device family" ); |
1468 | |