1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2021, HiSilicon Ltd.
4 */
5
6#include <linux/device.h>
7#include <linux/eventfd.h>
8#include <linux/file.h>
9#include <linux/hisi_acc_qm.h>
10#include <linux/interrupt.h>
11#include <linux/module.h>
12#include <linux/pci.h>
13#include <linux/vfio.h>
14#include <linux/vfio_pci_core.h>
15#include <linux/anon_inodes.h>
16
17#include "hisi_acc_vfio_pci.h"
18
19/* Return 0 on VM acc device ready, -ETIMEDOUT hardware timeout */
20static int qm_wait_dev_not_ready(struct hisi_qm *qm)
21{
22 u32 val;
23
24 return readl_relaxed_poll_timeout(qm->io_base + QM_VF_STATE,
25 val, !(val & 0x1), MB_POLL_PERIOD_US,
26 MB_POLL_TIMEOUT_US);
27}
28
29/*
30 * Each state Reg is checked 100 times,
31 * with a delay of 100 microseconds after each check
32 */
33static u32 qm_check_reg_state(struct hisi_qm *qm, u32 regs)
34{
35 int check_times = 0;
36 u32 state;
37
38 state = readl(addr: qm->io_base + regs);
39 while (state && check_times < ERROR_CHECK_TIMEOUT) {
40 udelay(CHECK_DELAY_TIME);
41 state = readl(addr: qm->io_base + regs);
42 check_times++;
43 }
44
45 return state;
46}
47
48static int qm_read_regs(struct hisi_qm *qm, u32 reg_addr,
49 u32 *data, u8 nums)
50{
51 int i;
52
53 if (nums < 1 || nums > QM_REGS_MAX_LEN)
54 return -EINVAL;
55
56 for (i = 0; i < nums; i++) {
57 data[i] = readl(addr: qm->io_base + reg_addr);
58 reg_addr += QM_REG_ADDR_OFFSET;
59 }
60
61 return 0;
62}
63
64static int qm_write_regs(struct hisi_qm *qm, u32 reg,
65 u32 *data, u8 nums)
66{
67 int i;
68
69 if (nums < 1 || nums > QM_REGS_MAX_LEN)
70 return -EINVAL;
71
72 for (i = 0; i < nums; i++)
73 writel(val: data[i], addr: qm->io_base + reg + i * QM_REG_ADDR_OFFSET);
74
75 return 0;
76}
77
78static int qm_get_vft(struct hisi_qm *qm, u32 *base)
79{
80 u64 sqc_vft;
81 u32 qp_num;
82 int ret;
83
84 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, dma_addr: 0, queue: 0, op: 1);
85 if (ret)
86 return ret;
87
88 sqc_vft = readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
89 ((u64)readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
90 QM_XQC_ADDR_OFFSET);
91 *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
92 qp_num = (QM_SQC_VFT_NUM_MASK_V2 &
93 (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
94
95 return qp_num;
96}
97
98static int qm_get_sqc(struct hisi_qm *qm, u64 *addr)
99{
100 int ret;
101
102 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, dma_addr: 0, queue: 0, op: 1);
103 if (ret)
104 return ret;
105
106 *addr = readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
107 ((u64)readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
108 QM_XQC_ADDR_OFFSET);
109
110 return 0;
111}
112
113static int qm_get_cqc(struct hisi_qm *qm, u64 *addr)
114{
115 int ret;
116
117 ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, dma_addr: 0, queue: 0, op: 1);
118 if (ret)
119 return ret;
120
121 *addr = readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
122 ((u64)readl(addr: qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
123 QM_XQC_ADDR_OFFSET);
124
125 return 0;
126}
127
128static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
129{
130 struct device *dev = &qm->pdev->dev;
131 int ret;
132
133 ret = qm_read_regs(qm, QM_VF_AEQ_INT_MASK, data: &vf_data->aeq_int_mask, nums: 1);
134 if (ret) {
135 dev_err(dev, "failed to read QM_VF_AEQ_INT_MASK\n");
136 return ret;
137 }
138
139 ret = qm_read_regs(qm, QM_VF_EQ_INT_MASK, data: &vf_data->eq_int_mask, nums: 1);
140 if (ret) {
141 dev_err(dev, "failed to read QM_VF_EQ_INT_MASK\n");
142 return ret;
143 }
144
145 ret = qm_read_regs(qm, QM_IFC_INT_SOURCE_V,
146 data: &vf_data->ifc_int_source, nums: 1);
147 if (ret) {
148 dev_err(dev, "failed to read QM_IFC_INT_SOURCE_V\n");
149 return ret;
150 }
151
152 ret = qm_read_regs(qm, QM_IFC_INT_MASK, data: &vf_data->ifc_int_mask, nums: 1);
153 if (ret) {
154 dev_err(dev, "failed to read QM_IFC_INT_MASK\n");
155 return ret;
156 }
157
158 ret = qm_read_regs(qm, QM_IFC_INT_SET_V, data: &vf_data->ifc_int_set, nums: 1);
159 if (ret) {
160 dev_err(dev, "failed to read QM_IFC_INT_SET_V\n");
161 return ret;
162 }
163
164 ret = qm_read_regs(qm, QM_PAGE_SIZE, data: &vf_data->page_size, nums: 1);
165 if (ret) {
166 dev_err(dev, "failed to read QM_PAGE_SIZE\n");
167 return ret;
168 }
169
170 /* QM_EQC_DW has 7 regs */
171 ret = qm_read_regs(qm, QM_EQC_DW0, data: vf_data->qm_eqc_dw, nums: 7);
172 if (ret) {
173 dev_err(dev, "failed to read QM_EQC_DW\n");
174 return ret;
175 }
176
177 /* QM_AEQC_DW has 7 regs */
178 ret = qm_read_regs(qm, QM_AEQC_DW0, data: vf_data->qm_aeqc_dw, nums: 7);
179 if (ret) {
180 dev_err(dev, "failed to read QM_AEQC_DW\n");
181 return ret;
182 }
183
184 return 0;
185}
186
187static int qm_set_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)
188{
189 struct device *dev = &qm->pdev->dev;
190 int ret;
191
192 /* Check VF state */
193 if (unlikely(hisi_qm_wait_mb_ready(qm))) {
194 dev_err(&qm->pdev->dev, "QM device is not ready to write\n");
195 return -EBUSY;
196 }
197
198 ret = qm_write_regs(qm, QM_VF_AEQ_INT_MASK, data: &vf_data->aeq_int_mask, nums: 1);
199 if (ret) {
200 dev_err(dev, "failed to write QM_VF_AEQ_INT_MASK\n");
201 return ret;
202 }
203
204 ret = qm_write_regs(qm, QM_VF_EQ_INT_MASK, data: &vf_data->eq_int_mask, nums: 1);
205 if (ret) {
206 dev_err(dev, "failed to write QM_VF_EQ_INT_MASK\n");
207 return ret;
208 }
209
210 ret = qm_write_regs(qm, QM_IFC_INT_SOURCE_V,
211 data: &vf_data->ifc_int_source, nums: 1);
212 if (ret) {
213 dev_err(dev, "failed to write QM_IFC_INT_SOURCE_V\n");
214 return ret;
215 }
216
217 ret = qm_write_regs(qm, QM_IFC_INT_MASK, data: &vf_data->ifc_int_mask, nums: 1);
218 if (ret) {
219 dev_err(dev, "failed to write QM_IFC_INT_MASK\n");
220 return ret;
221 }
222
223 ret = qm_write_regs(qm, QM_IFC_INT_SET_V, data: &vf_data->ifc_int_set, nums: 1);
224 if (ret) {
225 dev_err(dev, "failed to write QM_IFC_INT_SET_V\n");
226 return ret;
227 }
228
229 ret = qm_write_regs(qm, QM_QUE_ISO_CFG_V, data: &vf_data->que_iso_cfg, nums: 1);
230 if (ret) {
231 dev_err(dev, "failed to write QM_QUE_ISO_CFG_V\n");
232 return ret;
233 }
234
235 ret = qm_write_regs(qm, QM_PAGE_SIZE, data: &vf_data->page_size, nums: 1);
236 if (ret) {
237 dev_err(dev, "failed to write QM_PAGE_SIZE\n");
238 return ret;
239 }
240
241 /* QM_EQC_DW has 7 regs */
242 ret = qm_write_regs(qm, QM_EQC_DW0, data: vf_data->qm_eqc_dw, nums: 7);
243 if (ret) {
244 dev_err(dev, "failed to write QM_EQC_DW\n");
245 return ret;
246 }
247
248 /* QM_AEQC_DW has 7 regs */
249 ret = qm_write_regs(qm, QM_AEQC_DW0, data: vf_data->qm_aeqc_dw, nums: 7);
250 if (ret) {
251 dev_err(dev, "failed to write QM_AEQC_DW\n");
252 return ret;
253 }
254
255 return 0;
256}
257
258static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd,
259 u16 index, u8 priority)
260{
261 u64 doorbell;
262 u64 dbase;
263 u16 randata = 0;
264
265 if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)
266 dbase = QM_DOORBELL_SQ_CQ_BASE_V2;
267 else
268 dbase = QM_DOORBELL_EQ_AEQ_BASE_V2;
269
270 doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |
271 ((u64)randata << QM_DB_RAND_SHIFT_V2) |
272 ((u64)index << QM_DB_INDEX_SHIFT_V2) |
273 ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);
274
275 writeq(val: doorbell, addr: qm->io_base + dbase);
276}
277
278static int pf_qm_get_qp_num(struct hisi_qm *qm, int vf_id, u32 *rbase)
279{
280 unsigned int val;
281 u64 sqc_vft;
282 u32 qp_num;
283 int ret;
284
285 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
286 val & BIT(0), MB_POLL_PERIOD_US,
287 MB_POLL_TIMEOUT_US);
288 if (ret)
289 return ret;
290
291 writel(val: 0x1, addr: qm->io_base + QM_VFT_CFG_OP_WR);
292 /* 0 mean SQC VFT */
293 writel(val: 0x0, addr: qm->io_base + QM_VFT_CFG_TYPE);
294 writel(val: vf_id, addr: qm->io_base + QM_VFT_CFG);
295
296 writel(val: 0x0, addr: qm->io_base + QM_VFT_CFG_RDY);
297 writel(val: 0x1, addr: qm->io_base + QM_VFT_CFG_OP_ENABLE);
298
299 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
300 val & BIT(0), MB_POLL_PERIOD_US,
301 MB_POLL_TIMEOUT_US);
302 if (ret)
303 return ret;
304
305 sqc_vft = readl(addr: qm->io_base + QM_VFT_CFG_DATA_L) |
306 ((u64)readl(addr: qm->io_base + QM_VFT_CFG_DATA_H) <<
307 QM_XQC_ADDR_OFFSET);
308 *rbase = QM_SQC_VFT_BASE_MASK_V2 &
309 (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
310 qp_num = (QM_SQC_VFT_NUM_MASK_V2 &
311 (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
312
313 return qp_num;
314}
315
316static void qm_dev_cmd_init(struct hisi_qm *qm)
317{
318 /* Clear VF communication status registers. */
319 writel(val: 0x1, addr: qm->io_base + QM_IFC_INT_SOURCE_V);
320
321 /* Enable pf and vf communication. */
322 writel(val: 0x0, addr: qm->io_base + QM_IFC_INT_MASK);
323}
324
325static int vf_qm_cache_wb(struct hisi_qm *qm)
326{
327 unsigned int val;
328
329 writel(val: 0x1, addr: qm->io_base + QM_CACHE_WB_START);
330 if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,
331 val, val & BIT(0), MB_POLL_PERIOD_US,
332 MB_POLL_TIMEOUT_US)) {
333 dev_err(&qm->pdev->dev, "vf QM writeback sqc cache fail\n");
334 return -EINVAL;
335 }
336
337 return 0;
338}
339
340static void vf_qm_fun_reset(struct hisi_qm *qm)
341{
342 int i;
343
344 for (i = 0; i < qm->qp_num; i++)
345 qm_db(qm, qn: i, QM_DOORBELL_CMD_SQ, index: 0, priority: 1);
346}
347
348static int vf_qm_func_stop(struct hisi_qm *qm)
349{
350 return hisi_qm_mb(qm, QM_MB_CMD_PAUSE_QM, dma_addr: 0, queue: 0, op: 0);
351}
352
353static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev,
354 struct hisi_acc_vf_migration_file *migf)
355{
356 struct acc_vf_data *vf_data = &migf->vf_data;
357 struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
358 struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;
359 struct device *dev = &vf_qm->pdev->dev;
360 u32 que_iso_state;
361 int ret;
362
363 if (migf->total_length < QM_MATCH_SIZE || hisi_acc_vdev->match_done)
364 return 0;
365
366 if (vf_data->acc_magic != ACC_DEV_MAGIC) {
367 dev_err(dev, "failed to match ACC_DEV_MAGIC\n");
368 return -EINVAL;
369 }
370
371 if (vf_data->dev_id != hisi_acc_vdev->vf_dev->device) {
372 dev_err(dev, "failed to match VF devices\n");
373 return -EINVAL;
374 }
375
376 /* VF qp num check */
377 ret = qm_get_vft(qm: vf_qm, base: &vf_qm->qp_base);
378 if (ret <= 0) {
379 dev_err(dev, "failed to get vft qp nums\n");
380 return -EINVAL;
381 }
382
383 if (ret != vf_data->qp_num) {
384 dev_err(dev, "failed to match VF qp num\n");
385 return -EINVAL;
386 }
387
388 vf_qm->qp_num = ret;
389
390 /* VF isolation state check */
391 ret = qm_read_regs(qm: pf_qm, QM_QUE_ISO_CFG_V, data: &que_iso_state, nums: 1);
392 if (ret) {
393 dev_err(dev, "failed to read QM_QUE_ISO_CFG_V\n");
394 return ret;
395 }
396
397 if (vf_data->que_iso_cfg != que_iso_state) {
398 dev_err(dev, "failed to match isolation state\n");
399 return -EINVAL;
400 }
401
402 ret = qm_write_regs(qm: vf_qm, QM_VF_STATE, data: &vf_data->vf_qm_state, nums: 1);
403 if (ret) {
404 dev_err(dev, "failed to write QM_VF_STATE\n");
405 return ret;
406 }
407
408 hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;
409 hisi_acc_vdev->match_done = true;
410 return 0;
411}
412
413static int vf_qm_get_match_data(struct hisi_acc_vf_core_device *hisi_acc_vdev,
414 struct acc_vf_data *vf_data)
415{
416 struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;
417 struct device *dev = &pf_qm->pdev->dev;
418 int vf_id = hisi_acc_vdev->vf_id;
419 int ret;
420
421 vf_data->acc_magic = ACC_DEV_MAGIC;
422 /* Save device id */
423 vf_data->dev_id = hisi_acc_vdev->vf_dev->device;
424
425 /* VF qp num save from PF */
426 ret = pf_qm_get_qp_num(qm: pf_qm, vf_id, rbase: &vf_data->qp_base);
427 if (ret <= 0) {
428 dev_err(dev, "failed to get vft qp nums!\n");
429 return -EINVAL;
430 }
431
432 vf_data->qp_num = ret;
433
434 /* VF isolation state save from PF */
435 ret = qm_read_regs(qm: pf_qm, QM_QUE_ISO_CFG_V, data: &vf_data->que_iso_cfg, nums: 1);
436 if (ret) {
437 dev_err(dev, "failed to read QM_QUE_ISO_CFG_V!\n");
438 return ret;
439 }
440
441 return 0;
442}
443
444static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev,
445 struct hisi_acc_vf_migration_file *migf)
446{
447 struct hisi_qm *qm = &hisi_acc_vdev->vf_qm;
448 struct device *dev = &qm->pdev->dev;
449 struct acc_vf_data *vf_data = &migf->vf_data;
450 int ret;
451
452 /* Return if only match data was transferred */
453 if (migf->total_length == QM_MATCH_SIZE)
454 return 0;
455
456 if (migf->total_length < sizeof(struct acc_vf_data))
457 return -EINVAL;
458
459 qm->eqe_dma = vf_data->eqe_dma;
460 qm->aeqe_dma = vf_data->aeqe_dma;
461 qm->sqc_dma = vf_data->sqc_dma;
462 qm->cqc_dma = vf_data->cqc_dma;
463
464 qm->qp_base = vf_data->qp_base;
465 qm->qp_num = vf_data->qp_num;
466
467 ret = qm_set_regs(qm, vf_data);
468 if (ret) {
469 dev_err(dev, "set VF regs failed\n");
470 return ret;
471 }
472
473 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, dma_addr: qm->sqc_dma, queue: 0, op: 0);
474 if (ret) {
475 dev_err(dev, "set sqc failed\n");
476 return ret;
477 }
478
479 ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, dma_addr: qm->cqc_dma, queue: 0, op: 0);
480 if (ret) {
481 dev_err(dev, "set cqc failed\n");
482 return ret;
483 }
484
485 qm_dev_cmd_init(qm);
486 return 0;
487}
488
489static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev,
490 struct hisi_acc_vf_migration_file *migf)
491{
492 struct acc_vf_data *vf_data = &migf->vf_data;
493 struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
494 struct device *dev = &vf_qm->pdev->dev;
495 int ret;
496
497 if (unlikely(qm_wait_dev_not_ready(vf_qm))) {
498 /* Update state and return with match data */
499 vf_data->vf_qm_state = QM_NOT_READY;
500 hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;
501 migf->total_length = QM_MATCH_SIZE;
502 return 0;
503 }
504
505 vf_data->vf_qm_state = QM_READY;
506 hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;
507
508 ret = vf_qm_cache_wb(qm: vf_qm);
509 if (ret) {
510 dev_err(dev, "failed to writeback QM Cache!\n");
511 return ret;
512 }
513
514 ret = qm_get_regs(qm: vf_qm, vf_data);
515 if (ret)
516 return -EINVAL;
517
518 /* Every reg is 32 bit, the dma address is 64 bit. */
519 vf_data->eqe_dma = vf_data->qm_eqc_dw[1];
520 vf_data->eqe_dma <<= QM_XQC_ADDR_OFFSET;
521 vf_data->eqe_dma |= vf_data->qm_eqc_dw[0];
522 vf_data->aeqe_dma = vf_data->qm_aeqc_dw[1];
523 vf_data->aeqe_dma <<= QM_XQC_ADDR_OFFSET;
524 vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[0];
525
526 /* Through SQC_BT/CQC_BT to get sqc and cqc address */
527 ret = qm_get_sqc(qm: vf_qm, addr: &vf_data->sqc_dma);
528 if (ret) {
529 dev_err(dev, "failed to read SQC addr!\n");
530 return -EINVAL;
531 }
532
533 ret = qm_get_cqc(qm: vf_qm, addr: &vf_data->cqc_dma);
534 if (ret) {
535 dev_err(dev, "failed to read CQC addr!\n");
536 return -EINVAL;
537 }
538
539 migf->total_length = sizeof(struct acc_vf_data);
540 return 0;
541}
542
543static struct hisi_acc_vf_core_device *hisi_acc_drvdata(struct pci_dev *pdev)
544{
545 struct vfio_pci_core_device *core_device = dev_get_drvdata(dev: &pdev->dev);
546
547 return container_of(core_device, struct hisi_acc_vf_core_device,
548 core_device);
549}
550
551/* Check the PF's RAS state and Function INT state */
552static int
553hisi_acc_check_int_state(struct hisi_acc_vf_core_device *hisi_acc_vdev)
554{
555 struct hisi_qm *vfqm = &hisi_acc_vdev->vf_qm;
556 struct hisi_qm *qm = hisi_acc_vdev->pf_qm;
557 struct pci_dev *vf_pdev = hisi_acc_vdev->vf_dev;
558 struct device *dev = &qm->pdev->dev;
559 u32 state;
560
561 /* Check RAS state */
562 state = qm_check_reg_state(qm, QM_ABNORMAL_INT_STATUS);
563 if (state) {
564 dev_err(dev, "failed to check QM RAS state!\n");
565 return -EBUSY;
566 }
567
568 /* Check Function Communication state between PF and VF */
569 state = qm_check_reg_state(qm: vfqm, QM_IFC_INT_STATUS);
570 if (state) {
571 dev_err(dev, "failed to check QM IFC INT state!\n");
572 return -EBUSY;
573 }
574 state = qm_check_reg_state(qm: vfqm, QM_IFC_INT_SET_V);
575 if (state) {
576 dev_err(dev, "failed to check QM IFC INT SET state!\n");
577 return -EBUSY;
578 }
579
580 /* Check submodule task state */
581 switch (vf_pdev->device) {
582 case PCI_DEVICE_ID_HUAWEI_SEC_VF:
583 state = qm_check_reg_state(qm, SEC_CORE_INT_STATUS);
584 if (state) {
585 dev_err(dev, "failed to check QM SEC Core INT state!\n");
586 return -EBUSY;
587 }
588 return 0;
589 case PCI_DEVICE_ID_HUAWEI_HPRE_VF:
590 state = qm_check_reg_state(qm, HPRE_HAC_INT_STATUS);
591 if (state) {
592 dev_err(dev, "failed to check QM HPRE HAC INT state!\n");
593 return -EBUSY;
594 }
595 return 0;
596 case PCI_DEVICE_ID_HUAWEI_ZIP_VF:
597 state = qm_check_reg_state(qm, HZIP_CORE_INT_STATUS);
598 if (state) {
599 dev_err(dev, "failed to check QM ZIP Core INT state!\n");
600 return -EBUSY;
601 }
602 return 0;
603 default:
604 dev_err(dev, "failed to detect acc module type!\n");
605 return -EINVAL;
606 }
607}
608
609static void hisi_acc_vf_disable_fd(struct hisi_acc_vf_migration_file *migf)
610{
611 mutex_lock(&migf->lock);
612 migf->disabled = true;
613 migf->total_length = 0;
614 migf->filp->f_pos = 0;
615 mutex_unlock(lock: &migf->lock);
616}
617
618static void hisi_acc_vf_disable_fds(struct hisi_acc_vf_core_device *hisi_acc_vdev)
619{
620 if (hisi_acc_vdev->resuming_migf) {
621 hisi_acc_vf_disable_fd(migf: hisi_acc_vdev->resuming_migf);
622 fput(hisi_acc_vdev->resuming_migf->filp);
623 hisi_acc_vdev->resuming_migf = NULL;
624 }
625
626 if (hisi_acc_vdev->saving_migf) {
627 hisi_acc_vf_disable_fd(migf: hisi_acc_vdev->saving_migf);
628 fput(hisi_acc_vdev->saving_migf->filp);
629 hisi_acc_vdev->saving_migf = NULL;
630 }
631}
632
633/*
634 * This function is called in all state_mutex unlock cases to
635 * handle a 'deferred_reset' if exists.
636 */
637static void
638hisi_acc_vf_state_mutex_unlock(struct hisi_acc_vf_core_device *hisi_acc_vdev)
639{
640again:
641 spin_lock(lock: &hisi_acc_vdev->reset_lock);
642 if (hisi_acc_vdev->deferred_reset) {
643 hisi_acc_vdev->deferred_reset = false;
644 spin_unlock(lock: &hisi_acc_vdev->reset_lock);
645 hisi_acc_vdev->vf_qm_state = QM_NOT_READY;
646 hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
647 hisi_acc_vf_disable_fds(hisi_acc_vdev);
648 goto again;
649 }
650 mutex_unlock(lock: &hisi_acc_vdev->state_mutex);
651 spin_unlock(lock: &hisi_acc_vdev->reset_lock);
652}
653
654static void hisi_acc_vf_start_device(struct hisi_acc_vf_core_device *hisi_acc_vdev)
655{
656 struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
657
658 if (hisi_acc_vdev->vf_qm_state != QM_READY)
659 return;
660
661 /* Make sure the device is enabled */
662 qm_dev_cmd_init(qm: vf_qm);
663
664 vf_qm_fun_reset(qm: vf_qm);
665}
666
667static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev)
668{
669 struct device *dev = &hisi_acc_vdev->vf_dev->dev;
670 struct hisi_acc_vf_migration_file *migf = hisi_acc_vdev->resuming_migf;
671 int ret;
672
673 /* Recover data to VF */
674 ret = vf_qm_load_data(hisi_acc_vdev, migf);
675 if (ret) {
676 dev_err(dev, "failed to recover the VF!\n");
677 return ret;
678 }
679
680 return 0;
681}
682
683static int hisi_acc_vf_release_file(struct inode *inode, struct file *filp)
684{
685 struct hisi_acc_vf_migration_file *migf = filp->private_data;
686
687 hisi_acc_vf_disable_fd(migf);
688 mutex_destroy(lock: &migf->lock);
689 kfree(objp: migf);
690 return 0;
691}
692
693static ssize_t hisi_acc_vf_resume_write(struct file *filp, const char __user *buf,
694 size_t len, loff_t *pos)
695{
696 struct hisi_acc_vf_migration_file *migf = filp->private_data;
697 loff_t requested_length;
698 ssize_t done = 0;
699 int ret;
700
701 if (pos)
702 return -ESPIPE;
703 pos = &filp->f_pos;
704
705 if (*pos < 0 ||
706 check_add_overflow((loff_t)len, *pos, &requested_length))
707 return -EINVAL;
708
709 if (requested_length > sizeof(struct acc_vf_data))
710 return -ENOMEM;
711
712 mutex_lock(&migf->lock);
713 if (migf->disabled) {
714 done = -ENODEV;
715 goto out_unlock;
716 }
717
718 ret = copy_from_user(to: &migf->vf_data, from: buf, n: len);
719 if (ret) {
720 done = -EFAULT;
721 goto out_unlock;
722 }
723 *pos += len;
724 done = len;
725 migf->total_length += len;
726
727 ret = vf_qm_check_match(hisi_acc_vdev: migf->hisi_acc_vdev, migf);
728 if (ret)
729 done = -EFAULT;
730out_unlock:
731 mutex_unlock(lock: &migf->lock);
732 return done;
733}
734
735static const struct file_operations hisi_acc_vf_resume_fops = {
736 .owner = THIS_MODULE,
737 .write = hisi_acc_vf_resume_write,
738 .release = hisi_acc_vf_release_file,
739 .llseek = no_llseek,
740};
741
742static struct hisi_acc_vf_migration_file *
743hisi_acc_vf_pci_resume(struct hisi_acc_vf_core_device *hisi_acc_vdev)
744{
745 struct hisi_acc_vf_migration_file *migf;
746
747 migf = kzalloc(size: sizeof(*migf), GFP_KERNEL_ACCOUNT);
748 if (!migf)
749 return ERR_PTR(error: -ENOMEM);
750
751 migf->filp = anon_inode_getfile(name: "hisi_acc_vf_mig", fops: &hisi_acc_vf_resume_fops, priv: migf,
752 O_WRONLY);
753 if (IS_ERR(ptr: migf->filp)) {
754 int err = PTR_ERR(ptr: migf->filp);
755
756 kfree(objp: migf);
757 return ERR_PTR(error: err);
758 }
759
760 stream_open(inode: migf->filp->f_inode, filp: migf->filp);
761 mutex_init(&migf->lock);
762 migf->hisi_acc_vdev = hisi_acc_vdev;
763 return migf;
764}
765
766static long hisi_acc_vf_precopy_ioctl(struct file *filp,
767 unsigned int cmd, unsigned long arg)
768{
769 struct hisi_acc_vf_migration_file *migf = filp->private_data;
770 struct hisi_acc_vf_core_device *hisi_acc_vdev = migf->hisi_acc_vdev;
771 loff_t *pos = &filp->f_pos;
772 struct vfio_precopy_info info;
773 unsigned long minsz;
774 int ret;
775
776 if (cmd != VFIO_MIG_GET_PRECOPY_INFO)
777 return -ENOTTY;
778
779 minsz = offsetofend(struct vfio_precopy_info, dirty_bytes);
780
781 if (copy_from_user(to: &info, from: (void __user *)arg, n: minsz))
782 return -EFAULT;
783 if (info.argsz < minsz)
784 return -EINVAL;
785
786 mutex_lock(&hisi_acc_vdev->state_mutex);
787 if (hisi_acc_vdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY) {
788 mutex_unlock(lock: &hisi_acc_vdev->state_mutex);
789 return -EINVAL;
790 }
791
792 mutex_lock(&migf->lock);
793
794 if (migf->disabled) {
795 ret = -ENODEV;
796 goto out;
797 }
798
799 if (*pos > migf->total_length) {
800 ret = -EINVAL;
801 goto out;
802 }
803
804 info.dirty_bytes = 0;
805 info.initial_bytes = migf->total_length - *pos;
806
807 ret = copy_to_user(to: (void __user *)arg, from: &info, n: minsz) ? -EFAULT : 0;
808out:
809 mutex_unlock(lock: &migf->lock);
810 mutex_unlock(lock: &hisi_acc_vdev->state_mutex);
811 return ret;
812}
813
814static ssize_t hisi_acc_vf_save_read(struct file *filp, char __user *buf, size_t len,
815 loff_t *pos)
816{
817 struct hisi_acc_vf_migration_file *migf = filp->private_data;
818 ssize_t done = 0;
819 int ret;
820
821 if (pos)
822 return -ESPIPE;
823 pos = &filp->f_pos;
824
825 mutex_lock(&migf->lock);
826 if (*pos > migf->total_length) {
827 done = -EINVAL;
828 goto out_unlock;
829 }
830
831 if (migf->disabled) {
832 done = -ENODEV;
833 goto out_unlock;
834 }
835
836 len = min_t(size_t, migf->total_length - *pos, len);
837 if (len) {
838 ret = copy_to_user(to: buf, from: &migf->vf_data, n: len);
839 if (ret) {
840 done = -EFAULT;
841 goto out_unlock;
842 }
843 *pos += len;
844 done = len;
845 }
846out_unlock:
847 mutex_unlock(lock: &migf->lock);
848 return done;
849}
850
851static const struct file_operations hisi_acc_vf_save_fops = {
852 .owner = THIS_MODULE,
853 .read = hisi_acc_vf_save_read,
854 .unlocked_ioctl = hisi_acc_vf_precopy_ioctl,
855 .compat_ioctl = compat_ptr_ioctl,
856 .release = hisi_acc_vf_release_file,
857 .llseek = no_llseek,
858};
859
860static struct hisi_acc_vf_migration_file *
861hisi_acc_open_saving_migf(struct hisi_acc_vf_core_device *hisi_acc_vdev)
862{
863 struct hisi_acc_vf_migration_file *migf;
864 int ret;
865
866 migf = kzalloc(size: sizeof(*migf), GFP_KERNEL_ACCOUNT);
867 if (!migf)
868 return ERR_PTR(error: -ENOMEM);
869
870 migf->filp = anon_inode_getfile(name: "hisi_acc_vf_mig", fops: &hisi_acc_vf_save_fops, priv: migf,
871 O_RDONLY);
872 if (IS_ERR(ptr: migf->filp)) {
873 int err = PTR_ERR(ptr: migf->filp);
874
875 kfree(objp: migf);
876 return ERR_PTR(error: err);
877 }
878
879 stream_open(inode: migf->filp->f_inode, filp: migf->filp);
880 mutex_init(&migf->lock);
881 migf->hisi_acc_vdev = hisi_acc_vdev;
882
883 ret = vf_qm_get_match_data(hisi_acc_vdev, vf_data: &migf->vf_data);
884 if (ret) {
885 fput(migf->filp);
886 return ERR_PTR(error: ret);
887 }
888
889 return migf;
890}
891
892static struct hisi_acc_vf_migration_file *
893hisi_acc_vf_pre_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev)
894{
895 struct hisi_acc_vf_migration_file *migf;
896
897 migf = hisi_acc_open_saving_migf(hisi_acc_vdev);
898 if (IS_ERR(ptr: migf))
899 return migf;
900
901 migf->total_length = QM_MATCH_SIZE;
902 return migf;
903}
904
905static struct hisi_acc_vf_migration_file *
906hisi_acc_vf_stop_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev, bool open)
907{
908 int ret;
909 struct hisi_acc_vf_migration_file *migf = NULL;
910
911 if (open) {
912 /*
913 * Userspace didn't use PRECOPY support. Hence saving_migf
914 * is not opened yet.
915 */
916 migf = hisi_acc_open_saving_migf(hisi_acc_vdev);
917 if (IS_ERR(ptr: migf))
918 return migf;
919 } else {
920 migf = hisi_acc_vdev->saving_migf;
921 }
922
923 ret = vf_qm_state_save(hisi_acc_vdev, migf);
924 if (ret)
925 return ERR_PTR(error: ret);
926
927 return open ? migf : NULL;
928}
929
930static int hisi_acc_vf_stop_device(struct hisi_acc_vf_core_device *hisi_acc_vdev)
931{
932 struct device *dev = &hisi_acc_vdev->vf_dev->dev;
933 struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
934 int ret;
935
936 ret = vf_qm_func_stop(qm: vf_qm);
937 if (ret) {
938 dev_err(dev, "failed to stop QM VF function!\n");
939 return ret;
940 }
941
942 ret = hisi_acc_check_int_state(hisi_acc_vdev);
943 if (ret) {
944 dev_err(dev, "failed to check QM INT state!\n");
945 return ret;
946 }
947 return 0;
948}
949
950static struct file *
951hisi_acc_vf_set_device_state(struct hisi_acc_vf_core_device *hisi_acc_vdev,
952 u32 new)
953{
954 u32 cur = hisi_acc_vdev->mig_state;
955 int ret;
956
957 if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_PRE_COPY) {
958 struct hisi_acc_vf_migration_file *migf;
959
960 migf = hisi_acc_vf_pre_copy(hisi_acc_vdev);
961 if (IS_ERR(ptr: migf))
962 return ERR_CAST(ptr: migf);
963 get_file(f: migf->filp);
964 hisi_acc_vdev->saving_migf = migf;
965 return migf->filp;
966 }
967
968 if (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_STOP_COPY) {
969 struct hisi_acc_vf_migration_file *migf;
970
971 ret = hisi_acc_vf_stop_device(hisi_acc_vdev);
972 if (ret)
973 return ERR_PTR(error: ret);
974
975 migf = hisi_acc_vf_stop_copy(hisi_acc_vdev, open: false);
976 if (IS_ERR(ptr: migf))
977 return ERR_CAST(ptr: migf);
978
979 return NULL;
980 }
981
982 if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_STOP) {
983 ret = hisi_acc_vf_stop_device(hisi_acc_vdev);
984 if (ret)
985 return ERR_PTR(error: ret);
986 return NULL;
987 }
988
989 if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) {
990 struct hisi_acc_vf_migration_file *migf;
991
992 migf = hisi_acc_vf_stop_copy(hisi_acc_vdev, open: true);
993 if (IS_ERR(ptr: migf))
994 return ERR_CAST(ptr: migf);
995 get_file(f: migf->filp);
996 hisi_acc_vdev->saving_migf = migf;
997 return migf->filp;
998 }
999
1000 if ((cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP)) {
1001 hisi_acc_vf_disable_fds(hisi_acc_vdev);
1002 return NULL;
1003 }
1004
1005 if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) {
1006 struct hisi_acc_vf_migration_file *migf;
1007
1008 migf = hisi_acc_vf_pci_resume(hisi_acc_vdev);
1009 if (IS_ERR(ptr: migf))
1010 return ERR_CAST(ptr: migf);
1011 get_file(f: migf->filp);
1012 hisi_acc_vdev->resuming_migf = migf;
1013 return migf->filp;
1014 }
1015
1016 if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) {
1017 ret = hisi_acc_vf_load_state(hisi_acc_vdev);
1018 if (ret)
1019 return ERR_PTR(error: ret);
1020 hisi_acc_vf_disable_fds(hisi_acc_vdev);
1021 return NULL;
1022 }
1023
1024 if (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_RUNNING) {
1025 hisi_acc_vf_disable_fds(hisi_acc_vdev);
1026 return NULL;
1027 }
1028
1029 if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING) {
1030 hisi_acc_vf_start_device(hisi_acc_vdev);
1031 return NULL;
1032 }
1033
1034 /*
1035 * vfio_mig_get_next_state() does not use arcs other than the above
1036 */
1037 WARN_ON(true);
1038 return ERR_PTR(error: -EINVAL);
1039}
1040
1041static struct file *
1042hisi_acc_vfio_pci_set_device_state(struct vfio_device *vdev,
1043 enum vfio_device_mig_state new_state)
1044{
1045 struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(vdev,
1046 struct hisi_acc_vf_core_device, core_device.vdev);
1047 enum vfio_device_mig_state next_state;
1048 struct file *res = NULL;
1049 int ret;
1050
1051 mutex_lock(&hisi_acc_vdev->state_mutex);
1052 while (new_state != hisi_acc_vdev->mig_state) {
1053 ret = vfio_mig_get_next_state(device: vdev,
1054 cur_fsm: hisi_acc_vdev->mig_state,
1055 new_fsm: new_state, next_fsm: &next_state);
1056 if (ret) {
1057 res = ERR_PTR(error: -EINVAL);
1058 break;
1059 }
1060
1061 res = hisi_acc_vf_set_device_state(hisi_acc_vdev, new: next_state);
1062 if (IS_ERR(ptr: res))
1063 break;
1064 hisi_acc_vdev->mig_state = next_state;
1065 if (WARN_ON(res && new_state != hisi_acc_vdev->mig_state)) {
1066 fput(res);
1067 res = ERR_PTR(error: -EINVAL);
1068 break;
1069 }
1070 }
1071 hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev);
1072 return res;
1073}
1074
1075static int
1076hisi_acc_vfio_pci_get_data_size(struct vfio_device *vdev,
1077 unsigned long *stop_copy_length)
1078{
1079 *stop_copy_length = sizeof(struct acc_vf_data);
1080 return 0;
1081}
1082
1083static int
1084hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev,
1085 enum vfio_device_mig_state *curr_state)
1086{
1087 struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(vdev,
1088 struct hisi_acc_vf_core_device, core_device.vdev);
1089
1090 mutex_lock(&hisi_acc_vdev->state_mutex);
1091 *curr_state = hisi_acc_vdev->mig_state;
1092 hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev);
1093 return 0;
1094}
1095
1096static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev)
1097{
1098 struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev);
1099
1100 if (hisi_acc_vdev->core_device.vdev.migration_flags !=
1101 VFIO_MIGRATION_STOP_COPY)
1102 return;
1103
1104 /*
1105 * As the higher VFIO layers are holding locks across reset and using
1106 * those same locks with the mm_lock we need to prevent ABBA deadlock
1107 * with the state_mutex and mm_lock.
1108 * In case the state_mutex was taken already we defer the cleanup work
1109 * to the unlock flow of the other running context.
1110 */
1111 spin_lock(lock: &hisi_acc_vdev->reset_lock);
1112 hisi_acc_vdev->deferred_reset = true;
1113 if (!mutex_trylock(lock: &hisi_acc_vdev->state_mutex)) {
1114 spin_unlock(lock: &hisi_acc_vdev->reset_lock);
1115 return;
1116 }
1117 spin_unlock(lock: &hisi_acc_vdev->reset_lock);
1118 hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev);
1119}
1120
1121static int hisi_acc_vf_qm_init(struct hisi_acc_vf_core_device *hisi_acc_vdev)
1122{
1123 struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;
1124 struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
1125 struct pci_dev *vf_dev = vdev->pdev;
1126
1127 /*
1128 * ACC VF dev BAR2 region consists of both functional register space
1129 * and migration control register space. For migration to work, we
1130 * need access to both. Hence, we map the entire BAR2 region here.
1131 * But unnecessarily exposing the migration BAR region to the Guest
1132 * has the potential to prevent/corrupt the Guest migration. Hence,
1133 * we restrict access to the migration control space from
1134 * Guest(Please see mmap/ioctl/read/write override functions).
1135 *
1136 * Please note that it is OK to expose the entire VF BAR if migration
1137 * is not supported or required as this cannot affect the ACC PF
1138 * configurations.
1139 *
1140 * Also the HiSilicon ACC VF devices supported by this driver on
1141 * HiSilicon hardware platforms are integrated end point devices
1142 * and the platform lacks the capability to perform any PCIe P2P
1143 * between these devices.
1144 */
1145
1146 vf_qm->io_base =
1147 ioremap(pci_resource_start(vf_dev, VFIO_PCI_BAR2_REGION_INDEX),
1148 pci_resource_len(vf_dev, VFIO_PCI_BAR2_REGION_INDEX));
1149 if (!vf_qm->io_base)
1150 return -EIO;
1151
1152 vf_qm->fun_type = QM_HW_VF;
1153 vf_qm->pdev = vf_dev;
1154 mutex_init(&vf_qm->mailbox_lock);
1155
1156 return 0;
1157}
1158
1159static struct hisi_qm *hisi_acc_get_pf_qm(struct pci_dev *pdev)
1160{
1161 struct hisi_qm *pf_qm;
1162 struct pci_driver *pf_driver;
1163
1164 if (!pdev->is_virtfn)
1165 return NULL;
1166
1167 switch (pdev->device) {
1168 case PCI_DEVICE_ID_HUAWEI_SEC_VF:
1169 pf_driver = hisi_sec_get_pf_driver();
1170 break;
1171 case PCI_DEVICE_ID_HUAWEI_HPRE_VF:
1172 pf_driver = hisi_hpre_get_pf_driver();
1173 break;
1174 case PCI_DEVICE_ID_HUAWEI_ZIP_VF:
1175 pf_driver = hisi_zip_get_pf_driver();
1176 break;
1177 default:
1178 return NULL;
1179 }
1180
1181 if (!pf_driver)
1182 return NULL;
1183
1184 pf_qm = pci_iov_get_pf_drvdata(dev: pdev, pf_driver);
1185
1186 return !IS_ERR(ptr: pf_qm) ? pf_qm : NULL;
1187}
1188
1189static int hisi_acc_pci_rw_access_check(struct vfio_device *core_vdev,
1190 size_t count, loff_t *ppos,
1191 size_t *new_count)
1192{
1193 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
1194 struct vfio_pci_core_device *vdev =
1195 container_of(core_vdev, struct vfio_pci_core_device, vdev);
1196
1197 if (index == VFIO_PCI_BAR2_REGION_INDEX) {
1198 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
1199 resource_size_t end = pci_resource_len(vdev->pdev, index) / 2;
1200
1201 /* Check if access is for migration control region */
1202 if (pos >= end)
1203 return -EINVAL;
1204
1205 *new_count = min(count, (size_t)(end - pos));
1206 }
1207
1208 return 0;
1209}
1210
1211static int hisi_acc_vfio_pci_mmap(struct vfio_device *core_vdev,
1212 struct vm_area_struct *vma)
1213{
1214 struct vfio_pci_core_device *vdev =
1215 container_of(core_vdev, struct vfio_pci_core_device, vdev);
1216 unsigned int index;
1217
1218 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1219 if (index == VFIO_PCI_BAR2_REGION_INDEX) {
1220 u64 req_len, pgoff, req_start;
1221 resource_size_t end = pci_resource_len(vdev->pdev, index) / 2;
1222
1223 req_len = vma->vm_end - vma->vm_start;
1224 pgoff = vma->vm_pgoff &
1225 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1226 req_start = pgoff << PAGE_SHIFT;
1227
1228 if (req_start + req_len > end)
1229 return -EINVAL;
1230 }
1231
1232 return vfio_pci_core_mmap(core_vdev, vma);
1233}
1234
1235static ssize_t hisi_acc_vfio_pci_write(struct vfio_device *core_vdev,
1236 const char __user *buf, size_t count,
1237 loff_t *ppos)
1238{
1239 size_t new_count = count;
1240 int ret;
1241
1242 ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, new_count: &new_count);
1243 if (ret)
1244 return ret;
1245
1246 return vfio_pci_core_write(core_vdev, buf, count: new_count, ppos);
1247}
1248
1249static ssize_t hisi_acc_vfio_pci_read(struct vfio_device *core_vdev,
1250 char __user *buf, size_t count,
1251 loff_t *ppos)
1252{
1253 size_t new_count = count;
1254 int ret;
1255
1256 ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, new_count: &new_count);
1257 if (ret)
1258 return ret;
1259
1260 return vfio_pci_core_read(core_vdev, buf, count: new_count, ppos);
1261}
1262
1263static long hisi_acc_vfio_pci_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
1264 unsigned long arg)
1265{
1266 if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
1267 struct vfio_pci_core_device *vdev =
1268 container_of(core_vdev, struct vfio_pci_core_device, vdev);
1269 struct pci_dev *pdev = vdev->pdev;
1270 struct vfio_region_info info;
1271 unsigned long minsz;
1272
1273 minsz = offsetofend(struct vfio_region_info, offset);
1274
1275 if (copy_from_user(to: &info, from: (void __user *)arg, n: minsz))
1276 return -EFAULT;
1277
1278 if (info.argsz < minsz)
1279 return -EINVAL;
1280
1281 if (info.index == VFIO_PCI_BAR2_REGION_INDEX) {
1282 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1283
1284 /*
1285 * ACC VF dev BAR2 region consists of both functional
1286 * register space and migration control register space.
1287 * Report only the functional region to Guest.
1288 */
1289 info.size = pci_resource_len(pdev, info.index) / 2;
1290
1291 info.flags = VFIO_REGION_INFO_FLAG_READ |
1292 VFIO_REGION_INFO_FLAG_WRITE |
1293 VFIO_REGION_INFO_FLAG_MMAP;
1294
1295 return copy_to_user(to: (void __user *)arg, from: &info, n: minsz) ?
1296 -EFAULT : 0;
1297 }
1298 }
1299 return vfio_pci_core_ioctl(core_vdev, cmd, arg);
1300}
1301
1302static int hisi_acc_vfio_pci_open_device(struct vfio_device *core_vdev)
1303{
1304 struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev,
1305 struct hisi_acc_vf_core_device, core_device.vdev);
1306 struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;
1307 int ret;
1308
1309 ret = vfio_pci_core_enable(vdev);
1310 if (ret)
1311 return ret;
1312
1313 if (core_vdev->mig_ops) {
1314 ret = hisi_acc_vf_qm_init(hisi_acc_vdev);
1315 if (ret) {
1316 vfio_pci_core_disable(vdev);
1317 return ret;
1318 }
1319 hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
1320 }
1321
1322 vfio_pci_core_finish_enable(vdev);
1323 return 0;
1324}
1325
1326static void hisi_acc_vfio_pci_close_device(struct vfio_device *core_vdev)
1327{
1328 struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev,
1329 struct hisi_acc_vf_core_device, core_device.vdev);
1330 struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;
1331
1332 iounmap(addr: vf_qm->io_base);
1333 vfio_pci_core_close_device(core_vdev);
1334}
1335
1336static const struct vfio_migration_ops hisi_acc_vfio_pci_migrn_state_ops = {
1337 .migration_set_state = hisi_acc_vfio_pci_set_device_state,
1338 .migration_get_state = hisi_acc_vfio_pci_get_device_state,
1339 .migration_get_data_size = hisi_acc_vfio_pci_get_data_size,
1340};
1341
1342static int hisi_acc_vfio_pci_migrn_init_dev(struct vfio_device *core_vdev)
1343{
1344 struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev,
1345 struct hisi_acc_vf_core_device, core_device.vdev);
1346 struct pci_dev *pdev = to_pci_dev(core_vdev->dev);
1347 struct hisi_qm *pf_qm = hisi_acc_get_pf_qm(pdev);
1348
1349 hisi_acc_vdev->vf_id = pci_iov_vf_id(dev: pdev) + 1;
1350 hisi_acc_vdev->pf_qm = pf_qm;
1351 hisi_acc_vdev->vf_dev = pdev;
1352 mutex_init(&hisi_acc_vdev->state_mutex);
1353
1354 core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY;
1355 core_vdev->mig_ops = &hisi_acc_vfio_pci_migrn_state_ops;
1356
1357 return vfio_pci_core_init_dev(core_vdev);
1358}
1359
1360static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = {
1361 .name = "hisi-acc-vfio-pci-migration",
1362 .init = hisi_acc_vfio_pci_migrn_init_dev,
1363 .release = vfio_pci_core_release_dev,
1364 .open_device = hisi_acc_vfio_pci_open_device,
1365 .close_device = hisi_acc_vfio_pci_close_device,
1366 .ioctl = hisi_acc_vfio_pci_ioctl,
1367 .device_feature = vfio_pci_core_ioctl_feature,
1368 .read = hisi_acc_vfio_pci_read,
1369 .write = hisi_acc_vfio_pci_write,
1370 .mmap = hisi_acc_vfio_pci_mmap,
1371 .request = vfio_pci_core_request,
1372 .match = vfio_pci_core_match,
1373 .bind_iommufd = vfio_iommufd_physical_bind,
1374 .unbind_iommufd = vfio_iommufd_physical_unbind,
1375 .attach_ioas = vfio_iommufd_physical_attach_ioas,
1376 .detach_ioas = vfio_iommufd_physical_detach_ioas,
1377};
1378
1379static const struct vfio_device_ops hisi_acc_vfio_pci_ops = {
1380 .name = "hisi-acc-vfio-pci",
1381 .init = vfio_pci_core_init_dev,
1382 .release = vfio_pci_core_release_dev,
1383 .open_device = hisi_acc_vfio_pci_open_device,
1384 .close_device = vfio_pci_core_close_device,
1385 .ioctl = vfio_pci_core_ioctl,
1386 .device_feature = vfio_pci_core_ioctl_feature,
1387 .read = vfio_pci_core_read,
1388 .write = vfio_pci_core_write,
1389 .mmap = vfio_pci_core_mmap,
1390 .request = vfio_pci_core_request,
1391 .match = vfio_pci_core_match,
1392 .bind_iommufd = vfio_iommufd_physical_bind,
1393 .unbind_iommufd = vfio_iommufd_physical_unbind,
1394 .attach_ioas = vfio_iommufd_physical_attach_ioas,
1395 .detach_ioas = vfio_iommufd_physical_detach_ioas,
1396};
1397
1398static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1399{
1400 struct hisi_acc_vf_core_device *hisi_acc_vdev;
1401 const struct vfio_device_ops *ops = &hisi_acc_vfio_pci_ops;
1402 struct hisi_qm *pf_qm;
1403 int vf_id;
1404 int ret;
1405
1406 pf_qm = hisi_acc_get_pf_qm(pdev);
1407 if (pf_qm && pf_qm->ver >= QM_HW_V3) {
1408 vf_id = pci_iov_vf_id(dev: pdev);
1409 if (vf_id >= 0)
1410 ops = &hisi_acc_vfio_pci_migrn_ops;
1411 else
1412 pci_warn(pdev, "migration support failed, continue with generic interface\n");
1413 }
1414
1415 hisi_acc_vdev = vfio_alloc_device(hisi_acc_vf_core_device,
1416 core_device.vdev, &pdev->dev, ops);
1417 if (IS_ERR(ptr: hisi_acc_vdev))
1418 return PTR_ERR(ptr: hisi_acc_vdev);
1419
1420 dev_set_drvdata(dev: &pdev->dev, data: &hisi_acc_vdev->core_device);
1421 ret = vfio_pci_core_register_device(vdev: &hisi_acc_vdev->core_device);
1422 if (ret)
1423 goto out_put_vdev;
1424 return 0;
1425
1426out_put_vdev:
1427 vfio_put_device(device: &hisi_acc_vdev->core_device.vdev);
1428 return ret;
1429}
1430
1431static void hisi_acc_vfio_pci_remove(struct pci_dev *pdev)
1432{
1433 struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev);
1434
1435 vfio_pci_core_unregister_device(vdev: &hisi_acc_vdev->core_device);
1436 vfio_put_device(device: &hisi_acc_vdev->core_device.vdev);
1437}
1438
1439static const struct pci_device_id hisi_acc_vfio_pci_table[] = {
1440 { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_VF) },
1441 { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_HPRE_VF) },
1442 { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_VF) },
1443 { }
1444};
1445
1446MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table);
1447
1448static const struct pci_error_handlers hisi_acc_vf_err_handlers = {
1449 .reset_done = hisi_acc_vf_pci_aer_reset_done,
1450 .error_detected = vfio_pci_core_aer_err_detected,
1451};
1452
1453static struct pci_driver hisi_acc_vfio_pci_driver = {
1454 .name = KBUILD_MODNAME,
1455 .id_table = hisi_acc_vfio_pci_table,
1456 .probe = hisi_acc_vfio_pci_probe,
1457 .remove = hisi_acc_vfio_pci_remove,
1458 .err_handler = &hisi_acc_vf_err_handlers,
1459 .driver_managed_dma = true,
1460};
1461
1462module_pci_driver(hisi_acc_vfio_pci_driver);
1463
1464MODULE_LICENSE("GPL v2");
1465MODULE_AUTHOR("Liu Longfang <liulongfang@huawei.com>");
1466MODULE_AUTHOR("Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>");
1467MODULE_DESCRIPTION("HiSilicon VFIO PCI - VFIO PCI driver with live migration support for HiSilicon ACC device family");
1468

source code of linux/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c