1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* Copyright (c) 2015 - 2021 Intel Corporation */ |
3 | #include "osdep.h" |
4 | #include "hmc.h" |
5 | #include "defs.h" |
6 | #include "type.h" |
7 | #include "protos.h" |
8 | #include "pble.h" |
9 | |
10 | static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc); |
11 | |
12 | /** |
13 | * irdma_destroy_pble_prm - destroy prm during module unload |
14 | * @pble_rsrc: pble resources |
15 | */ |
16 | void irdma_destroy_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc) |
17 | { |
18 | struct irdma_chunk *chunk; |
19 | struct irdma_pble_prm *pinfo = &pble_rsrc->pinfo; |
20 | |
21 | while (!list_empty(head: &pinfo->clist)) { |
22 | chunk = (struct irdma_chunk *) pinfo->clist.next; |
23 | list_del(entry: &chunk->list); |
24 | if (chunk->type == PBLE_SD_PAGED) |
25 | irdma_pble_free_paged_mem(chunk); |
26 | bitmap_free(bitmap: chunk->bitmapbuf); |
27 | kfree(objp: chunk->chunkmem.va); |
28 | } |
29 | } |
30 | |
31 | /** |
32 | * irdma_hmc_init_pble - Initialize pble resources during module load |
33 | * @dev: irdma_sc_dev struct |
34 | * @pble_rsrc: pble resources |
35 | */ |
36 | int irdma_hmc_init_pble(struct irdma_sc_dev *dev, |
37 | struct irdma_hmc_pble_rsrc *pble_rsrc) |
38 | { |
39 | struct irdma_hmc_info *hmc_info; |
40 | u32 fpm_idx = 0; |
41 | int status = 0; |
42 | |
43 | hmc_info = dev->hmc_info; |
44 | pble_rsrc->dev = dev; |
45 | pble_rsrc->fpm_base_addr = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].base; |
46 | /* Start pble' on 4k boundary */ |
47 | if (pble_rsrc->fpm_base_addr & 0xfff) |
48 | fpm_idx = (4096 - (pble_rsrc->fpm_base_addr & 0xfff)) >> 3; |
49 | pble_rsrc->unallocated_pble = |
50 | hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt - fpm_idx; |
51 | pble_rsrc->next_fpm_addr = pble_rsrc->fpm_base_addr + (fpm_idx << 3); |
52 | pble_rsrc->pinfo.pble_shift = PBLE_SHIFT; |
53 | |
54 | mutex_init(&pble_rsrc->pble_mutex_lock); |
55 | |
56 | spin_lock_init(&pble_rsrc->pinfo.prm_lock); |
57 | INIT_LIST_HEAD(list: &pble_rsrc->pinfo.clist); |
58 | if (add_pble_prm(pble_rsrc)) { |
59 | irdma_destroy_pble_prm(pble_rsrc); |
60 | status = -ENOMEM; |
61 | } |
62 | |
63 | return status; |
64 | } |
65 | |
66 | /** |
67 | * get_sd_pd_idx - Returns sd index, pd index and rel_pd_idx from fpm address |
68 | * @pble_rsrc: structure containing fpm address |
69 | * @idx: where to return indexes |
70 | */ |
71 | static void get_sd_pd_idx(struct irdma_hmc_pble_rsrc *pble_rsrc, |
72 | struct sd_pd_idx *idx) |
73 | { |
74 | idx->sd_idx = (u32)pble_rsrc->next_fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE; |
75 | idx->pd_idx = (u32)(pble_rsrc->next_fpm_addr / IRDMA_HMC_PAGED_BP_SIZE); |
76 | idx->rel_pd_idx = (idx->pd_idx % IRDMA_HMC_PD_CNT_IN_SD); |
77 | } |
78 | |
79 | /** |
80 | * add_sd_direct - add sd direct for pble |
81 | * @pble_rsrc: pble resource ptr |
82 | * @info: page info for sd |
83 | */ |
84 | static int add_sd_direct(struct irdma_hmc_pble_rsrc *pble_rsrc, |
85 | struct irdma_add_page_info *info) |
86 | { |
87 | struct irdma_sc_dev *dev = pble_rsrc->dev; |
88 | int ret_code = 0; |
89 | struct sd_pd_idx *idx = &info->idx; |
90 | struct irdma_chunk *chunk = info->chunk; |
91 | struct irdma_hmc_info *hmc_info = info->hmc_info; |
92 | struct irdma_hmc_sd_entry *sd_entry = info->sd_entry; |
93 | u32 offset = 0; |
94 | |
95 | if (!sd_entry->valid) { |
96 | ret_code = irdma_add_sd_table_entry(hw: dev->hw, hmc_info, |
97 | sd_index: info->idx.sd_idx, |
98 | type: IRDMA_SD_TYPE_DIRECT, |
99 | IRDMA_HMC_DIRECT_BP_SIZE); |
100 | if (ret_code) |
101 | return ret_code; |
102 | |
103 | chunk->type = PBLE_SD_CONTIGOUS; |
104 | } |
105 | |
106 | offset = idx->rel_pd_idx << HMC_PAGED_BP_SHIFT; |
107 | chunk->size = info->pages << HMC_PAGED_BP_SHIFT; |
108 | chunk->vaddr = sd_entry->u.bp.addr.va + offset; |
109 | chunk->fpm_addr = pble_rsrc->next_fpm_addr; |
110 | ibdev_dbg(to_ibdev(dev), |
111 | "PBLE: chunk_size[%lld] = 0x%llx vaddr=0x%pK fpm_addr = %llx\n" , |
112 | chunk->size, chunk->size, chunk->vaddr, chunk->fpm_addr); |
113 | |
114 | return 0; |
115 | } |
116 | |
117 | /** |
118 | * fpm_to_idx - given fpm address, get pble index |
119 | * @pble_rsrc: pble resource management |
120 | * @addr: fpm address for index |
121 | */ |
122 | static u32 fpm_to_idx(struct irdma_hmc_pble_rsrc *pble_rsrc, u64 addr) |
123 | { |
124 | u64 idx; |
125 | |
126 | idx = (addr - (pble_rsrc->fpm_base_addr)) >> 3; |
127 | |
128 | return (u32)idx; |
129 | } |
130 | |
131 | /** |
132 | * add_bp_pages - add backing pages for sd |
133 | * @pble_rsrc: pble resource management |
134 | * @info: page info for sd |
135 | */ |
136 | static int add_bp_pages(struct irdma_hmc_pble_rsrc *pble_rsrc, |
137 | struct irdma_add_page_info *info) |
138 | { |
139 | struct irdma_sc_dev *dev = pble_rsrc->dev; |
140 | u8 *addr; |
141 | struct irdma_dma_mem mem; |
142 | struct irdma_hmc_pd_entry *pd_entry; |
143 | struct irdma_hmc_sd_entry *sd_entry = info->sd_entry; |
144 | struct irdma_hmc_info *hmc_info = info->hmc_info; |
145 | struct irdma_chunk *chunk = info->chunk; |
146 | int status = 0; |
147 | u32 rel_pd_idx = info->idx.rel_pd_idx; |
148 | u32 pd_idx = info->idx.pd_idx; |
149 | u32 i; |
150 | |
151 | if (irdma_pble_get_paged_mem(chunk, pg_cnt: info->pages)) |
152 | return -ENOMEM; |
153 | |
154 | status = irdma_add_sd_table_entry(hw: dev->hw, hmc_info, sd_index: info->idx.sd_idx, |
155 | type: IRDMA_SD_TYPE_PAGED, |
156 | IRDMA_HMC_DIRECT_BP_SIZE); |
157 | if (status) |
158 | goto error; |
159 | |
160 | addr = chunk->vaddr; |
161 | for (i = 0; i < info->pages; i++) { |
162 | mem.pa = (u64)chunk->dmainfo.dmaaddrs[i]; |
163 | mem.size = 4096; |
164 | mem.va = addr; |
165 | pd_entry = &sd_entry->u.pd_table.pd_entry[rel_pd_idx++]; |
166 | if (!pd_entry->valid) { |
167 | status = irdma_add_pd_table_entry(dev, hmc_info, |
168 | pd_index: pd_idx++, rsrc_pg: &mem); |
169 | if (status) |
170 | goto error; |
171 | |
172 | addr += 4096; |
173 | } |
174 | } |
175 | |
176 | chunk->fpm_addr = pble_rsrc->next_fpm_addr; |
177 | return 0; |
178 | |
179 | error: |
180 | irdma_pble_free_paged_mem(chunk); |
181 | |
182 | return status; |
183 | } |
184 | |
185 | /** |
186 | * irdma_get_type - add a sd entry type for sd |
187 | * @dev: irdma_sc_dev struct |
188 | * @idx: index of sd |
189 | * @pages: pages in the sd |
190 | */ |
191 | static enum irdma_sd_entry_type irdma_get_type(struct irdma_sc_dev *dev, |
192 | struct sd_pd_idx *idx, u32 pages) |
193 | { |
194 | enum irdma_sd_entry_type sd_entry_type; |
195 | |
196 | sd_entry_type = !idx->rel_pd_idx && pages == IRDMA_HMC_PD_CNT_IN_SD ? |
197 | IRDMA_SD_TYPE_DIRECT : IRDMA_SD_TYPE_PAGED; |
198 | return sd_entry_type; |
199 | } |
200 | |
201 | /** |
202 | * add_pble_prm - add a sd entry for pble resoure |
203 | * @pble_rsrc: pble resource management |
204 | */ |
205 | static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc) |
206 | { |
207 | struct irdma_sc_dev *dev = pble_rsrc->dev; |
208 | struct irdma_hmc_sd_entry *sd_entry; |
209 | struct irdma_hmc_info *hmc_info; |
210 | struct irdma_chunk *chunk; |
211 | struct irdma_add_page_info info; |
212 | struct sd_pd_idx *idx = &info.idx; |
213 | int ret_code = 0; |
214 | enum irdma_sd_entry_type sd_entry_type; |
215 | u64 sd_reg_val = 0; |
216 | struct irdma_virt_mem chunkmem; |
217 | u32 pages; |
218 | |
219 | if (pble_rsrc->unallocated_pble < PBLE_PER_PAGE) |
220 | return -ENOMEM; |
221 | |
222 | if (pble_rsrc->next_fpm_addr & 0xfff) |
223 | return -EINVAL; |
224 | |
225 | chunkmem.size = sizeof(*chunk); |
226 | chunkmem.va = kzalloc(size: chunkmem.size, GFP_KERNEL); |
227 | if (!chunkmem.va) |
228 | return -ENOMEM; |
229 | |
230 | chunk = chunkmem.va; |
231 | chunk->chunkmem = chunkmem; |
232 | hmc_info = dev->hmc_info; |
233 | chunk->dev = dev; |
234 | chunk->fpm_addr = pble_rsrc->next_fpm_addr; |
235 | get_sd_pd_idx(pble_rsrc, idx); |
236 | sd_entry = &hmc_info->sd_table.sd_entry[idx->sd_idx]; |
237 | pages = (idx->rel_pd_idx) ? (IRDMA_HMC_PD_CNT_IN_SD - idx->rel_pd_idx) : |
238 | IRDMA_HMC_PD_CNT_IN_SD; |
239 | pages = min(pages, pble_rsrc->unallocated_pble >> PBLE_512_SHIFT); |
240 | info.chunk = chunk; |
241 | info.hmc_info = hmc_info; |
242 | info.pages = pages; |
243 | info.sd_entry = sd_entry; |
244 | if (!sd_entry->valid) |
245 | sd_entry_type = irdma_get_type(dev, idx, pages); |
246 | else |
247 | sd_entry_type = sd_entry->entry_type; |
248 | |
249 | ibdev_dbg(to_ibdev(dev), |
250 | "PBLE: pages = %d, unallocated_pble[%d] current_fpm_addr = %llx\n" , |
251 | pages, pble_rsrc->unallocated_pble, |
252 | pble_rsrc->next_fpm_addr); |
253 | ibdev_dbg(to_ibdev(dev), "PBLE: sd_entry_type = %d\n" , sd_entry_type); |
254 | if (sd_entry_type == IRDMA_SD_TYPE_DIRECT) |
255 | ret_code = add_sd_direct(pble_rsrc, info: &info); |
256 | |
257 | if (ret_code) |
258 | sd_entry_type = IRDMA_SD_TYPE_PAGED; |
259 | else |
260 | pble_rsrc->stats_direct_sds++; |
261 | |
262 | if (sd_entry_type == IRDMA_SD_TYPE_PAGED) { |
263 | ret_code = add_bp_pages(pble_rsrc, info: &info); |
264 | if (ret_code) |
265 | goto error; |
266 | else |
267 | pble_rsrc->stats_paged_sds++; |
268 | } |
269 | |
270 | ret_code = irdma_prm_add_pble_mem(pprm: &pble_rsrc->pinfo, pchunk: chunk); |
271 | if (ret_code) |
272 | goto error; |
273 | |
274 | pble_rsrc->next_fpm_addr += chunk->size; |
275 | ibdev_dbg(to_ibdev(dev), |
276 | "PBLE: next_fpm_addr = %llx chunk_size[%llu] = 0x%llx\n" , |
277 | pble_rsrc->next_fpm_addr, chunk->size, chunk->size); |
278 | pble_rsrc->unallocated_pble -= (u32)(chunk->size >> 3); |
279 | sd_reg_val = (sd_entry_type == IRDMA_SD_TYPE_PAGED) ? |
280 | sd_entry->u.pd_table.pd_page_addr.pa : |
281 | sd_entry->u.bp.addr.pa; |
282 | |
283 | if (!sd_entry->valid) { |
284 | ret_code = irdma_hmc_sd_one(dev, hmc_fn_id: hmc_info->hmc_fn_id, pa: sd_reg_val, |
285 | sd_idx: idx->sd_idx, type: sd_entry->entry_type, setsd: true); |
286 | if (ret_code) |
287 | goto error; |
288 | } |
289 | |
290 | list_add(new: &chunk->list, head: &pble_rsrc->pinfo.clist); |
291 | sd_entry->valid = true; |
292 | return 0; |
293 | |
294 | error: |
295 | bitmap_free(bitmap: chunk->bitmapbuf); |
296 | kfree(objp: chunk->chunkmem.va); |
297 | |
298 | return ret_code; |
299 | } |
300 | |
301 | /** |
302 | * free_lvl2 - fee level 2 pble |
303 | * @pble_rsrc: pble resource management |
304 | * @palloc: level 2 pble allocation |
305 | */ |
306 | static void free_lvl2(struct irdma_hmc_pble_rsrc *pble_rsrc, |
307 | struct irdma_pble_alloc *palloc) |
308 | { |
309 | u32 i; |
310 | struct irdma_pble_level2 *lvl2 = &palloc->level2; |
311 | struct irdma_pble_info *root = &lvl2->root; |
312 | struct irdma_pble_info *leaf = lvl2->leaf; |
313 | |
314 | for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { |
315 | if (leaf->addr) |
316 | irdma_prm_return_pbles(pprm: &pble_rsrc->pinfo, |
317 | chunkinfo: &leaf->chunkinfo); |
318 | else |
319 | break; |
320 | } |
321 | |
322 | if (root->addr) |
323 | irdma_prm_return_pbles(pprm: &pble_rsrc->pinfo, chunkinfo: &root->chunkinfo); |
324 | |
325 | kfree(objp: lvl2->leafmem.va); |
326 | lvl2->leaf = NULL; |
327 | } |
328 | |
329 | /** |
330 | * get_lvl2_pble - get level 2 pble resource |
331 | * @pble_rsrc: pble resource management |
332 | * @palloc: level 2 pble allocation |
333 | */ |
334 | static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, |
335 | struct irdma_pble_alloc *palloc) |
336 | { |
337 | u32 lf4k, lflast, total, i; |
338 | u32 pblcnt = PBLE_PER_PAGE; |
339 | u64 *addr; |
340 | struct irdma_pble_level2 *lvl2 = &palloc->level2; |
341 | struct irdma_pble_info *root = &lvl2->root; |
342 | struct irdma_pble_info *leaf; |
343 | int ret_code; |
344 | u64 fpm_addr; |
345 | |
346 | /* number of full 512 (4K) leafs) */ |
347 | lf4k = palloc->total_cnt >> 9; |
348 | lflast = palloc->total_cnt % PBLE_PER_PAGE; |
349 | total = (lflast == 0) ? lf4k : lf4k + 1; |
350 | lvl2->leaf_cnt = total; |
351 | |
352 | lvl2->leafmem.size = (sizeof(*leaf) * total); |
353 | lvl2->leafmem.va = kzalloc(size: lvl2->leafmem.size, GFP_KERNEL); |
354 | if (!lvl2->leafmem.va) |
355 | return -ENOMEM; |
356 | |
357 | lvl2->leaf = lvl2->leafmem.va; |
358 | leaf = lvl2->leaf; |
359 | ret_code = irdma_prm_get_pbles(pprm: &pble_rsrc->pinfo, chunkinfo: &root->chunkinfo, |
360 | mem_size: total << 3, vaddr: &root->addr, fpm_addr: &fpm_addr); |
361 | if (ret_code) { |
362 | kfree(objp: lvl2->leafmem.va); |
363 | lvl2->leaf = NULL; |
364 | return -ENOMEM; |
365 | } |
366 | |
367 | root->idx = fpm_to_idx(pble_rsrc, addr: fpm_addr); |
368 | root->cnt = total; |
369 | addr = root->addr; |
370 | for (i = 0; i < total; i++, leaf++) { |
371 | pblcnt = (lflast && ((i + 1) == total)) ? |
372 | lflast : PBLE_PER_PAGE; |
373 | ret_code = irdma_prm_get_pbles(pprm: &pble_rsrc->pinfo, |
374 | chunkinfo: &leaf->chunkinfo, mem_size: pblcnt << 3, |
375 | vaddr: &leaf->addr, fpm_addr: &fpm_addr); |
376 | if (ret_code) |
377 | goto error; |
378 | |
379 | leaf->idx = fpm_to_idx(pble_rsrc, addr: fpm_addr); |
380 | |
381 | leaf->cnt = pblcnt; |
382 | *addr = (u64)leaf->idx; |
383 | addr++; |
384 | } |
385 | |
386 | palloc->level = PBLE_LEVEL_2; |
387 | pble_rsrc->stats_lvl2++; |
388 | return 0; |
389 | |
390 | error: |
391 | free_lvl2(pble_rsrc, palloc); |
392 | |
393 | return -ENOMEM; |
394 | } |
395 | |
396 | /** |
397 | * get_lvl1_pble - get level 1 pble resource |
398 | * @pble_rsrc: pble resource management |
399 | * @palloc: level 1 pble allocation |
400 | */ |
401 | static int get_lvl1_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, |
402 | struct irdma_pble_alloc *palloc) |
403 | { |
404 | int ret_code; |
405 | u64 fpm_addr; |
406 | struct irdma_pble_info *lvl1 = &palloc->level1; |
407 | |
408 | ret_code = irdma_prm_get_pbles(pprm: &pble_rsrc->pinfo, chunkinfo: &lvl1->chunkinfo, |
409 | mem_size: palloc->total_cnt << 3, vaddr: &lvl1->addr, |
410 | fpm_addr: &fpm_addr); |
411 | if (ret_code) |
412 | return -ENOMEM; |
413 | |
414 | palloc->level = PBLE_LEVEL_1; |
415 | lvl1->idx = fpm_to_idx(pble_rsrc, addr: fpm_addr); |
416 | lvl1->cnt = palloc->total_cnt; |
417 | pble_rsrc->stats_lvl1++; |
418 | |
419 | return 0; |
420 | } |
421 | |
422 | /** |
423 | * get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine |
424 | * @pble_rsrc: pble resources |
425 | * @palloc: contains all inforamtion regarding pble (idx + pble addr) |
426 | * @lvl: Bitmask for requested pble level |
427 | */ |
428 | static int get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, |
429 | struct irdma_pble_alloc *palloc, u8 lvl) |
430 | { |
431 | int status = 0; |
432 | |
433 | status = get_lvl1_pble(pble_rsrc, palloc); |
434 | if (!status || lvl == PBLE_LEVEL_1 || palloc->total_cnt <= PBLE_PER_PAGE) |
435 | return status; |
436 | |
437 | status = get_lvl2_pble(pble_rsrc, palloc); |
438 | |
439 | return status; |
440 | } |
441 | |
442 | /** |
443 | * irdma_get_pble - allocate pbles from the prm |
444 | * @pble_rsrc: pble resources |
445 | * @palloc: contains all inforamtion regarding pble (idx + pble addr) |
446 | * @pble_cnt: #of pbles requested |
447 | * @lvl: requested pble level mask |
448 | */ |
449 | int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, |
450 | struct irdma_pble_alloc *palloc, u32 pble_cnt, |
451 | u8 lvl) |
452 | { |
453 | int status = 0; |
454 | int max_sds = 0; |
455 | int i; |
456 | |
457 | palloc->total_cnt = pble_cnt; |
458 | palloc->level = PBLE_LEVEL_0; |
459 | |
460 | mutex_lock(&pble_rsrc->pble_mutex_lock); |
461 | |
462 | /*check first to see if we can get pble's without acquiring |
463 | * additional sd's |
464 | */ |
465 | status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl); |
466 | if (!status) |
467 | goto exit; |
468 | |
469 | max_sds = (palloc->total_cnt >> 18) + 1; |
470 | for (i = 0; i < max_sds; i++) { |
471 | status = add_pble_prm(pble_rsrc); |
472 | if (status) |
473 | break; |
474 | |
475 | status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl); |
476 | /* if level1_only, only go through it once */ |
477 | if (!status || lvl) |
478 | break; |
479 | } |
480 | |
481 | exit: |
482 | if (!status) { |
483 | pble_rsrc->allocdpbles += pble_cnt; |
484 | pble_rsrc->stats_alloc_ok++; |
485 | } else { |
486 | pble_rsrc->stats_alloc_fail++; |
487 | } |
488 | mutex_unlock(lock: &pble_rsrc->pble_mutex_lock); |
489 | |
490 | return status; |
491 | } |
492 | |
493 | /** |
494 | * irdma_free_pble - put pbles back into prm |
495 | * @pble_rsrc: pble resources |
496 | * @palloc: contains all information regarding pble resource being freed |
497 | */ |
498 | void irdma_free_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, |
499 | struct irdma_pble_alloc *palloc) |
500 | { |
501 | pble_rsrc->freedpbles += palloc->total_cnt; |
502 | |
503 | if (palloc->level == PBLE_LEVEL_2) |
504 | free_lvl2(pble_rsrc, palloc); |
505 | else |
506 | irdma_prm_return_pbles(pprm: &pble_rsrc->pinfo, |
507 | chunkinfo: &palloc->level1.chunkinfo); |
508 | pble_rsrc->stats_alloc_freed++; |
509 | } |
510 | |