1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright (C) 2023, Intel Corporation. */ |
3 | |
4 | #include "ice.h" |
5 | #include "ice_lib.h" |
6 | #include "ice_irq.h" |
7 | |
8 | /** |
9 | * ice_init_irq_tracker - initialize interrupt tracker |
10 | * @pf: board private structure |
11 | * @max_vectors: maximum number of vectors that tracker can hold |
12 | * @num_static: number of preallocated interrupts |
13 | */ |
14 | static void |
15 | ice_init_irq_tracker(struct ice_pf *pf, unsigned int max_vectors, |
16 | unsigned int num_static) |
17 | { |
18 | pf->irq_tracker.num_entries = max_vectors; |
19 | pf->irq_tracker.num_static = num_static; |
20 | xa_init_flags(xa: &pf->irq_tracker.entries, XA_FLAGS_ALLOC); |
21 | } |
22 | |
23 | /** |
24 | * ice_deinit_irq_tracker - free xarray tracker |
25 | * @pf: board private structure |
26 | */ |
27 | static void ice_deinit_irq_tracker(struct ice_pf *pf) |
28 | { |
29 | xa_destroy(&pf->irq_tracker.entries); |
30 | } |
31 | |
32 | /** |
33 | * ice_free_irq_res - free a block of resources |
34 | * @pf: board private structure |
35 | * @index: starting index previously returned by ice_get_res |
36 | */ |
37 | static void ice_free_irq_res(struct ice_pf *pf, u16 index) |
38 | { |
39 | struct ice_irq_entry *entry; |
40 | |
41 | entry = xa_erase(&pf->irq_tracker.entries, index); |
42 | kfree(objp: entry); |
43 | } |
44 | |
45 | /** |
46 | * ice_get_irq_res - get an interrupt resource |
47 | * @pf: board private structure |
48 | * @dyn_only: force entry to be dynamically allocated |
49 | * |
50 | * Allocate new irq entry in the free slot of the tracker. Since xarray |
51 | * is used, always allocate new entry at the lowest possible index. Set |
52 | * proper allocation limit for maximum tracker entries. |
53 | * |
54 | * Returns allocated irq entry or NULL on failure. |
55 | */ |
56 | static struct ice_irq_entry *ice_get_irq_res(struct ice_pf *pf, bool dyn_only) |
57 | { |
58 | struct xa_limit limit = { .max = pf->irq_tracker.num_entries, |
59 | .min = 0 }; |
60 | unsigned int num_static = pf->irq_tracker.num_static; |
61 | struct ice_irq_entry *entry; |
62 | unsigned int index; |
63 | int ret; |
64 | |
65 | entry = kzalloc(size: sizeof(*entry), GFP_KERNEL); |
66 | if (!entry) |
67 | return NULL; |
68 | |
69 | /* skip preallocated entries if the caller says so */ |
70 | if (dyn_only) |
71 | limit.min = num_static; |
72 | |
73 | ret = xa_alloc(xa: &pf->irq_tracker.entries, id: &index, entry, limit, |
74 | GFP_KERNEL); |
75 | |
76 | if (ret) { |
77 | kfree(objp: entry); |
78 | entry = NULL; |
79 | } else { |
80 | entry->index = index; |
81 | entry->dynamic = index >= num_static; |
82 | } |
83 | |
84 | return entry; |
85 | } |
86 | |
87 | /** |
88 | * ice_reduce_msix_usage - Reduce usage of MSI-X vectors |
89 | * @pf: board private structure |
90 | * @v_remain: number of remaining MSI-X vectors to be distributed |
91 | * |
92 | * Reduce the usage of MSI-X vectors when entire request cannot be fulfilled. |
93 | * pf->num_lan_msix and pf->num_rdma_msix values are set based on number of |
94 | * remaining vectors. |
95 | */ |
96 | static void ice_reduce_msix_usage(struct ice_pf *pf, int v_remain) |
97 | { |
98 | int v_rdma; |
99 | |
100 | if (!ice_is_rdma_ena(pf)) { |
101 | pf->num_lan_msix = v_remain; |
102 | return; |
103 | } |
104 | |
105 | /* RDMA needs at least 1 interrupt in addition to AEQ MSIX */ |
106 | v_rdma = ICE_RDMA_NUM_AEQ_MSIX + 1; |
107 | |
108 | if (v_remain < ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_RDMA_MSIX) { |
109 | dev_warn(ice_pf_to_dev(pf), "Not enough MSI-X vectors to support RDMA.\n" ); |
110 | clear_bit(nr: ICE_FLAG_RDMA_ENA, addr: pf->flags); |
111 | |
112 | pf->num_rdma_msix = 0; |
113 | pf->num_lan_msix = ICE_MIN_LAN_TXRX_MSIX; |
114 | } else if ((v_remain < ICE_MIN_LAN_TXRX_MSIX + v_rdma) || |
115 | (v_remain - v_rdma < v_rdma)) { |
116 | /* Support minimum RDMA and give remaining vectors to LAN MSIX |
117 | */ |
118 | pf->num_rdma_msix = ICE_MIN_RDMA_MSIX; |
119 | pf->num_lan_msix = v_remain - ICE_MIN_RDMA_MSIX; |
120 | } else { |
121 | /* Split remaining MSIX with RDMA after accounting for AEQ MSIX |
122 | */ |
123 | pf->num_rdma_msix = (v_remain - ICE_RDMA_NUM_AEQ_MSIX) / 2 + |
124 | ICE_RDMA_NUM_AEQ_MSIX; |
125 | pf->num_lan_msix = v_remain - pf->num_rdma_msix; |
126 | } |
127 | } |
128 | |
129 | /** |
130 | * ice_ena_msix_range - Request a range of MSIX vectors from the OS |
131 | * @pf: board private structure |
132 | * |
133 | * Compute the number of MSIX vectors wanted and request from the OS. Adjust |
134 | * device usage if there are not enough vectors. Return the number of vectors |
135 | * reserved or negative on failure. |
136 | */ |
137 | static int ice_ena_msix_range(struct ice_pf *pf) |
138 | { |
139 | int num_cpus, hw_num_msix, v_other, v_wanted, v_actual; |
140 | struct device *dev = ice_pf_to_dev(pf); |
141 | int err; |
142 | |
143 | hw_num_msix = pf->hw.func_caps.common_cap.num_msix_vectors; |
144 | num_cpus = num_online_cpus(); |
145 | |
146 | /* LAN miscellaneous handler */ |
147 | v_other = ICE_MIN_LAN_OICR_MSIX; |
148 | |
149 | /* Flow Director */ |
150 | if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) |
151 | v_other += ICE_FDIR_MSIX; |
152 | |
153 | /* switchdev */ |
154 | v_other += ICE_ESWITCH_MSIX; |
155 | |
156 | v_wanted = v_other; |
157 | |
158 | /* LAN traffic */ |
159 | pf->num_lan_msix = num_cpus; |
160 | v_wanted += pf->num_lan_msix; |
161 | |
162 | /* RDMA auxiliary driver */ |
163 | if (ice_is_rdma_ena(pf)) { |
164 | pf->num_rdma_msix = num_cpus + ICE_RDMA_NUM_AEQ_MSIX; |
165 | v_wanted += pf->num_rdma_msix; |
166 | } |
167 | |
168 | if (v_wanted > hw_num_msix) { |
169 | int v_remain; |
170 | |
171 | dev_warn(dev, "not enough device MSI-X vectors. wanted = %d, available = %d\n" , |
172 | v_wanted, hw_num_msix); |
173 | |
174 | if (hw_num_msix < ICE_MIN_MSIX) { |
175 | err = -ERANGE; |
176 | goto exit_err; |
177 | } |
178 | |
179 | v_remain = hw_num_msix - v_other; |
180 | if (v_remain < ICE_MIN_LAN_TXRX_MSIX) { |
181 | v_other = ICE_MIN_MSIX - ICE_MIN_LAN_TXRX_MSIX; |
182 | v_remain = ICE_MIN_LAN_TXRX_MSIX; |
183 | } |
184 | |
185 | ice_reduce_msix_usage(pf, v_remain); |
186 | v_wanted = pf->num_lan_msix + pf->num_rdma_msix + v_other; |
187 | |
188 | dev_notice(dev, "Reducing request to %d MSI-X vectors for LAN traffic.\n" , |
189 | pf->num_lan_msix); |
190 | if (ice_is_rdma_ena(pf)) |
191 | dev_notice(dev, "Reducing request to %d MSI-X vectors for RDMA.\n" , |
192 | pf->num_rdma_msix); |
193 | } |
194 | |
195 | /* actually reserve the vectors */ |
196 | v_actual = pci_alloc_irq_vectors(dev: pf->pdev, ICE_MIN_MSIX, max_vecs: v_wanted, |
197 | PCI_IRQ_MSIX); |
198 | if (v_actual < 0) { |
199 | dev_err(dev, "unable to reserve MSI-X vectors\n" ); |
200 | err = v_actual; |
201 | goto exit_err; |
202 | } |
203 | |
204 | if (v_actual < v_wanted) { |
205 | dev_warn(dev, "not enough OS MSI-X vectors. requested = %d, obtained = %d\n" , |
206 | v_wanted, v_actual); |
207 | |
208 | if (v_actual < ICE_MIN_MSIX) { |
209 | /* error if we can't get minimum vectors */ |
210 | pci_free_irq_vectors(dev: pf->pdev); |
211 | err = -ERANGE; |
212 | goto exit_err; |
213 | } else { |
214 | int v_remain = v_actual - v_other; |
215 | |
216 | if (v_remain < ICE_MIN_LAN_TXRX_MSIX) |
217 | v_remain = ICE_MIN_LAN_TXRX_MSIX; |
218 | |
219 | ice_reduce_msix_usage(pf, v_remain); |
220 | |
221 | dev_notice(dev, "Enabled %d MSI-X vectors for LAN traffic.\n" , |
222 | pf->num_lan_msix); |
223 | |
224 | if (ice_is_rdma_ena(pf)) |
225 | dev_notice(dev, "Enabled %d MSI-X vectors for RDMA.\n" , |
226 | pf->num_rdma_msix); |
227 | } |
228 | } |
229 | |
230 | return v_actual; |
231 | |
232 | exit_err: |
233 | pf->num_rdma_msix = 0; |
234 | pf->num_lan_msix = 0; |
235 | return err; |
236 | } |
237 | |
238 | /** |
239 | * ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme |
240 | * @pf: board private structure |
241 | */ |
242 | void ice_clear_interrupt_scheme(struct ice_pf *pf) |
243 | { |
244 | pci_free_irq_vectors(dev: pf->pdev); |
245 | ice_deinit_irq_tracker(pf); |
246 | } |
247 | |
248 | /** |
249 | * ice_init_interrupt_scheme - Determine proper interrupt scheme |
250 | * @pf: board private structure to initialize |
251 | */ |
252 | int ice_init_interrupt_scheme(struct ice_pf *pf) |
253 | { |
254 | int total_vectors = pf->hw.func_caps.common_cap.num_msix_vectors; |
255 | int vectors, max_vectors; |
256 | |
257 | vectors = ice_ena_msix_range(pf); |
258 | |
259 | if (vectors < 0) |
260 | return -ENOMEM; |
261 | |
262 | if (pci_msix_can_alloc_dyn(dev: pf->pdev)) |
263 | max_vectors = total_vectors; |
264 | else |
265 | max_vectors = vectors; |
266 | |
267 | ice_init_irq_tracker(pf, max_vectors, num_static: vectors); |
268 | |
269 | return 0; |
270 | } |
271 | |
272 | /** |
273 | * ice_alloc_irq - Allocate new interrupt vector |
274 | * @pf: board private structure |
275 | * @dyn_only: force dynamic allocation of the interrupt |
276 | * |
277 | * Allocate new interrupt vector for a given owner id. |
278 | * return struct msi_map with interrupt details and track |
279 | * allocated interrupt appropriately. |
280 | * |
281 | * This function reserves new irq entry from the irq_tracker. |
282 | * if according to the tracker information all interrupts that |
283 | * were allocated with ice_pci_alloc_irq_vectors are already used |
284 | * and dynamically allocated interrupts are supported then new |
285 | * interrupt will be allocated with pci_msix_alloc_irq_at. |
286 | * |
287 | * Some callers may only support dynamically allocated interrupts. |
288 | * This is indicated with dyn_only flag. |
289 | * |
290 | * On failure, return map with negative .index. The caller |
291 | * is expected to check returned map index. |
292 | * |
293 | */ |
294 | struct msi_map ice_alloc_irq(struct ice_pf *pf, bool dyn_only) |
295 | { |
296 | int sriov_base_vector = pf->sriov_base_vector; |
297 | struct msi_map map = { .index = -ENOENT }; |
298 | struct device *dev = ice_pf_to_dev(pf); |
299 | struct ice_irq_entry *entry; |
300 | |
301 | entry = ice_get_irq_res(pf, dyn_only); |
302 | if (!entry) |
303 | return map; |
304 | |
305 | /* fail if we're about to violate SRIOV vectors space */ |
306 | if (sriov_base_vector && entry->index >= sriov_base_vector) |
307 | goto exit_free_res; |
308 | |
309 | if (pci_msix_can_alloc_dyn(dev: pf->pdev) && entry->dynamic) { |
310 | map = pci_msix_alloc_irq_at(dev: pf->pdev, index: entry->index, NULL); |
311 | if (map.index < 0) |
312 | goto exit_free_res; |
313 | dev_dbg(dev, "allocated new irq at index %d\n" , map.index); |
314 | } else { |
315 | map.index = entry->index; |
316 | map.virq = pci_irq_vector(dev: pf->pdev, nr: map.index); |
317 | } |
318 | |
319 | return map; |
320 | |
321 | exit_free_res: |
322 | dev_err(dev, "Could not allocate irq at idx %d\n" , entry->index); |
323 | ice_free_irq_res(pf, index: entry->index); |
324 | return map; |
325 | } |
326 | |
327 | /** |
328 | * ice_free_irq - Free interrupt vector |
329 | * @pf: board private structure |
330 | * @map: map with interrupt details |
331 | * |
332 | * Remove allocated interrupt from the interrupt tracker. If interrupt was |
333 | * allocated dynamically, free respective interrupt vector. |
334 | */ |
335 | void ice_free_irq(struct ice_pf *pf, struct msi_map map) |
336 | { |
337 | struct ice_irq_entry *entry; |
338 | |
339 | entry = xa_load(&pf->irq_tracker.entries, index: map.index); |
340 | |
341 | if (!entry) { |
342 | dev_err(ice_pf_to_dev(pf), "Failed to get MSIX interrupt entry at index %d" , |
343 | map.index); |
344 | return; |
345 | } |
346 | |
347 | dev_dbg(ice_pf_to_dev(pf), "Free irq at index %d\n" , map.index); |
348 | |
349 | if (entry->dynamic) |
350 | pci_msix_free_irq(pdev: pf->pdev, map); |
351 | |
352 | ice_free_irq_res(pf, index: map.index); |
353 | } |
354 | |
355 | /** |
356 | * ice_get_max_used_msix_vector - Get the max used interrupt vector |
357 | * @pf: board private structure |
358 | * |
359 | * Return index of maximum used interrupt vectors with respect to the |
360 | * beginning of the MSIX table. Take into account that some interrupts |
361 | * may have been dynamically allocated after MSIX was initially enabled. |
362 | */ |
363 | int ice_get_max_used_msix_vector(struct ice_pf *pf) |
364 | { |
365 | unsigned long start, index, max_idx; |
366 | void *entry; |
367 | |
368 | /* Treat all preallocated interrupts as used */ |
369 | start = pf->irq_tracker.num_static; |
370 | max_idx = start - 1; |
371 | |
372 | xa_for_each_start(&pf->irq_tracker.entries, index, entry, start) { |
373 | if (index > max_idx) |
374 | max_idx = index; |
375 | } |
376 | |
377 | return max_idx; |
378 | } |
379 | |