1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * SolidRun DPU driver for control plane |
4 | * |
5 | * Copyright (C) 2022-2023 SolidRun |
6 | * |
7 | * Author: Alvaro Karsz <alvaro.karsz@solid-run.com> |
8 | * |
9 | */ |
10 | |
11 | #include <linux/iopoll.h> |
12 | |
13 | #include "snet_vdpa.h" |
14 | |
15 | enum snet_ctrl_opcodes { |
16 | SNET_CTRL_OP_DESTROY = 1, |
17 | SNET_CTRL_OP_READ_VQ_STATE, |
18 | SNET_CTRL_OP_SUSPEND, |
19 | SNET_CTRL_OP_RESUME, |
20 | }; |
21 | |
22 | #define SNET_CTRL_TIMEOUT 2000000 |
23 | |
24 | #define SNET_CTRL_DATA_SIZE_MASK 0x0000FFFF |
25 | #define SNET_CTRL_IN_PROCESS_MASK 0x00010000 |
26 | #define SNET_CTRL_CHUNK_RDY_MASK 0x00020000 |
27 | #define SNET_CTRL_ERROR_MASK 0x0FFC0000 |
28 | |
29 | #define SNET_VAL_TO_ERR(val) (-(((val) & SNET_CTRL_ERROR_MASK) >> 18)) |
30 | #define SNET_EMPTY_CTRL(val) (((val) & SNET_CTRL_ERROR_MASK) || \ |
31 | !((val) & SNET_CTRL_IN_PROCESS_MASK)) |
32 | #define SNET_DATA_READY(val) ((val) & (SNET_CTRL_ERROR_MASK | SNET_CTRL_CHUNK_RDY_MASK)) |
33 | |
34 | /* Control register used to read data from the DPU */ |
35 | struct snet_ctrl_reg_ctrl { |
36 | /* Chunk size in 4B words */ |
37 | u16 data_size; |
38 | /* We are in the middle of a command */ |
39 | u16 in_process:1; |
40 | /* A data chunk is ready and can be consumed */ |
41 | u16 chunk_ready:1; |
42 | /* Error code */ |
43 | u16 error:10; |
44 | /* Saved for future usage */ |
45 | u16 rsvd:4; |
46 | }; |
47 | |
48 | /* Opcode register */ |
49 | struct snet_ctrl_reg_op { |
50 | u16 opcode; |
51 | /* Only if VQ index is relevant for the command */ |
52 | u16 vq_idx; |
53 | }; |
54 | |
55 | struct snet_ctrl_regs { |
56 | struct snet_ctrl_reg_op op; |
57 | struct snet_ctrl_reg_ctrl ctrl; |
58 | u32 rsvd; |
59 | u32 data[]; |
60 | }; |
61 | |
62 | static struct snet_ctrl_regs __iomem *snet_get_ctrl(struct snet *snet) |
63 | { |
64 | return snet->bar + snet->psnet->cfg.ctrl_off; |
65 | } |
66 | |
67 | static int snet_wait_for_empty_ctrl(struct snet_ctrl_regs __iomem *regs) |
68 | { |
69 | u32 val; |
70 | |
71 | return readx_poll_timeout(ioread32, ®s->ctrl, val, SNET_EMPTY_CTRL(val), 10, |
72 | SNET_CTRL_TIMEOUT); |
73 | } |
74 | |
75 | static int snet_wait_for_empty_op(struct snet_ctrl_regs __iomem *regs) |
76 | { |
77 | u32 val; |
78 | |
79 | return readx_poll_timeout(ioread32, ®s->op, val, !val, 10, SNET_CTRL_TIMEOUT); |
80 | } |
81 | |
82 | static int snet_wait_for_data(struct snet_ctrl_regs __iomem *regs) |
83 | { |
84 | u32 val; |
85 | |
86 | return readx_poll_timeout(ioread32, ®s->ctrl, val, SNET_DATA_READY(val), 10, |
87 | SNET_CTRL_TIMEOUT); |
88 | } |
89 | |
90 | static u32 snet_read32_word(struct snet_ctrl_regs __iomem *ctrl_regs, u16 word_idx) |
91 | { |
92 | return ioread32(&ctrl_regs->data[word_idx]); |
93 | } |
94 | |
95 | static u32 snet_read_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs) |
96 | { |
97 | return ioread32(&ctrl_regs->ctrl); |
98 | } |
99 | |
100 | static void snet_write_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val) |
101 | { |
102 | iowrite32(val, &ctrl_regs->ctrl); |
103 | } |
104 | |
105 | static void snet_write_op(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val) |
106 | { |
107 | iowrite32(val, &ctrl_regs->op); |
108 | } |
109 | |
110 | static int snet_wait_for_dpu_completion(struct snet_ctrl_regs __iomem *ctrl_regs) |
111 | { |
112 | /* Wait until the DPU finishes completely. |
113 | * It will clear the opcode register. |
114 | */ |
115 | return snet_wait_for_empty_op(regs: ctrl_regs); |
116 | } |
117 | |
118 | /* Reading ctrl from the DPU: |
119 | * buf_size must be 4B aligned |
120 | * |
121 | * Steps: |
122 | * |
123 | * (1) Verify that the DPU is not in the middle of another operation by |
124 | * reading the in_process and error bits in the control register. |
125 | * (2) Write the request opcode and the VQ idx in the opcode register |
126 | * and write the buffer size in the control register. |
127 | * (3) Start readind chunks of data, chunk_ready bit indicates that a |
128 | * data chunk is available, we signal that we read the data by clearing the bit. |
129 | * (4) Detect that the transfer is completed when the in_process bit |
130 | * in the control register is cleared or when the an error appears. |
131 | */ |
132 | static int snet_ctrl_read_from_dpu(struct snet *snet, u16 opcode, u16 vq_idx, void *buffer, |
133 | u32 buf_size) |
134 | { |
135 | struct pci_dev *pdev = snet->pdev; |
136 | struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet); |
137 | u32 *bfr_ptr = (u32 *)buffer; |
138 | u32 val; |
139 | u16 buf_words; |
140 | int ret; |
141 | u16 words, i, tot_words = 0; |
142 | |
143 | /* Supported for config 2+ */ |
144 | if (!SNET_CFG_VER(snet, 2)) |
145 | return -EOPNOTSUPP; |
146 | |
147 | if (!IS_ALIGNED(buf_size, 4)) |
148 | return -EINVAL; |
149 | |
150 | mutex_lock(&snet->ctrl_lock); |
151 | |
152 | buf_words = buf_size / 4; |
153 | |
154 | /* Make sure control register is empty */ |
155 | ret = snet_wait_for_empty_ctrl(regs); |
156 | if (ret) { |
157 | SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n" ); |
158 | goto exit; |
159 | } |
160 | |
161 | /* We need to write the buffer size in the control register, and the opcode + vq index in |
162 | * the opcode register. |
163 | * We use a spinlock to serialize the writes. |
164 | */ |
165 | spin_lock(lock: &snet->ctrl_spinlock); |
166 | |
167 | snet_write_ctrl(ctrl_regs: regs, val: buf_words); |
168 | snet_write_op(ctrl_regs: regs, val: opcode | (vq_idx << 16)); |
169 | |
170 | spin_unlock(lock: &snet->ctrl_spinlock); |
171 | |
172 | while (buf_words != tot_words) { |
173 | ret = snet_wait_for_data(regs); |
174 | if (ret) { |
175 | SNET_WARN(pdev, "Timeout waiting for control data\n" ); |
176 | goto exit; |
177 | } |
178 | |
179 | val = snet_read_ctrl(ctrl_regs: regs); |
180 | |
181 | /* Error? */ |
182 | if (val & SNET_CTRL_ERROR_MASK) { |
183 | ret = SNET_VAL_TO_ERR(val); |
184 | SNET_WARN(pdev, "Error while reading control data from DPU, err %d\n" , ret); |
185 | goto exit; |
186 | } |
187 | |
188 | words = min_t(u16, val & SNET_CTRL_DATA_SIZE_MASK, buf_words - tot_words); |
189 | |
190 | for (i = 0; i < words; i++) { |
191 | *bfr_ptr = snet_read32_word(ctrl_regs: regs, word_idx: i); |
192 | bfr_ptr++; |
193 | } |
194 | |
195 | tot_words += words; |
196 | |
197 | /* Is the job completed? */ |
198 | if (!(val & SNET_CTRL_IN_PROCESS_MASK)) |
199 | break; |
200 | |
201 | /* Clear the chunk ready bit and continue */ |
202 | val &= ~SNET_CTRL_CHUNK_RDY_MASK; |
203 | snet_write_ctrl(ctrl_regs: regs, val); |
204 | } |
205 | |
206 | ret = snet_wait_for_dpu_completion(ctrl_regs: regs); |
207 | if (ret) |
208 | SNET_WARN(pdev, "Timeout waiting for the DPU to complete a control command\n" ); |
209 | |
210 | exit: |
211 | mutex_unlock(lock: &snet->ctrl_lock); |
212 | return ret; |
213 | } |
214 | |
215 | /* Send a control message to the DPU using the old mechanism |
216 | * used with config version 1. |
217 | */ |
218 | static int snet_send_ctrl_msg_old(struct snet *snet, u32 opcode) |
219 | { |
220 | struct pci_dev *pdev = snet->pdev; |
221 | struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet); |
222 | int ret; |
223 | |
224 | mutex_lock(&snet->ctrl_lock); |
225 | |
226 | /* Old mechanism uses just 1 register, the opcode register. |
227 | * Make sure that the opcode register is empty, and that the DPU isn't |
228 | * processing an old message. |
229 | */ |
230 | ret = snet_wait_for_empty_op(regs); |
231 | if (ret) { |
232 | SNET_WARN(pdev, "Timeout waiting for previous control message to be ACKed\n" ); |
233 | goto exit; |
234 | } |
235 | |
236 | /* Write the message */ |
237 | snet_write_op(ctrl_regs: regs, val: opcode); |
238 | |
239 | /* DPU ACKs the message by clearing the opcode register */ |
240 | ret = snet_wait_for_empty_op(regs); |
241 | if (ret) |
242 | SNET_WARN(pdev, "Timeout waiting for a control message to be ACKed\n" ); |
243 | |
244 | exit: |
245 | mutex_unlock(lock: &snet->ctrl_lock); |
246 | return ret; |
247 | } |
248 | |
249 | /* Send a control message to the DPU. |
250 | * A control message is a message without payload. |
251 | */ |
252 | static int snet_send_ctrl_msg(struct snet *snet, u16 opcode, u16 vq_idx) |
253 | { |
254 | struct pci_dev *pdev = snet->pdev; |
255 | struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet); |
256 | u32 val; |
257 | int ret; |
258 | |
259 | /* If config version is not 2+, use the old mechanism */ |
260 | if (!SNET_CFG_VER(snet, 2)) |
261 | return snet_send_ctrl_msg_old(snet, opcode); |
262 | |
263 | mutex_lock(&snet->ctrl_lock); |
264 | |
265 | /* Make sure control register is empty */ |
266 | ret = snet_wait_for_empty_ctrl(regs); |
267 | if (ret) { |
268 | SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n" ); |
269 | goto exit; |
270 | } |
271 | |
272 | /* We need to clear the control register and write the opcode + vq index in the opcode |
273 | * register. |
274 | * We use a spinlock to serialize the writes. |
275 | */ |
276 | spin_lock(lock: &snet->ctrl_spinlock); |
277 | |
278 | snet_write_ctrl(ctrl_regs: regs, val: 0); |
279 | snet_write_op(ctrl_regs: regs, val: opcode | (vq_idx << 16)); |
280 | |
281 | spin_unlock(lock: &snet->ctrl_spinlock); |
282 | |
283 | /* The DPU ACKs control messages by setting the chunk ready bit |
284 | * without data. |
285 | */ |
286 | ret = snet_wait_for_data(regs); |
287 | if (ret) { |
288 | SNET_WARN(pdev, "Timeout waiting for control message to be ACKed\n" ); |
289 | goto exit; |
290 | } |
291 | |
292 | /* Check for errors */ |
293 | val = snet_read_ctrl(ctrl_regs: regs); |
294 | ret = SNET_VAL_TO_ERR(val); |
295 | |
296 | /* Clear the chunk ready bit */ |
297 | val &= ~SNET_CTRL_CHUNK_RDY_MASK; |
298 | snet_write_ctrl(ctrl_regs: regs, val); |
299 | |
300 | ret = snet_wait_for_dpu_completion(ctrl_regs: regs); |
301 | if (ret) |
302 | SNET_WARN(pdev, "Timeout waiting for DPU to complete a control command, err %d\n" , |
303 | ret); |
304 | |
305 | exit: |
306 | mutex_unlock(lock: &snet->ctrl_lock); |
307 | return ret; |
308 | } |
309 | |
310 | void snet_ctrl_clear(struct snet *snet) |
311 | { |
312 | struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet); |
313 | |
314 | snet_write_op(ctrl_regs: regs, val: 0); |
315 | } |
316 | |
317 | int snet_destroy_dev(struct snet *snet) |
318 | { |
319 | return snet_send_ctrl_msg(snet, opcode: SNET_CTRL_OP_DESTROY, vq_idx: 0); |
320 | } |
321 | |
322 | int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state) |
323 | { |
324 | return snet_ctrl_read_from_dpu(snet, opcode: SNET_CTRL_OP_READ_VQ_STATE, vq_idx: idx, buffer: state, |
325 | buf_size: sizeof(*state)); |
326 | } |
327 | |
328 | int snet_suspend_dev(struct snet *snet) |
329 | { |
330 | return snet_send_ctrl_msg(snet, opcode: SNET_CTRL_OP_SUSPEND, vq_idx: 0); |
331 | } |
332 | |
333 | int snet_resume_dev(struct snet *snet) |
334 | { |
335 | return snet_send_ctrl_msg(snet, opcode: SNET_CTRL_OP_RESUME, vq_idx: 0); |
336 | } |
337 | |