1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2015 - 2021 Intel Corporation */
3#include "osdep.h"
4#include "hmc.h"
5#include "defs.h"
6#include "type.h"
7#include "protos.h"
8#include "pble.h"
9
10static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc);
11
12/**
13 * irdma_destroy_pble_prm - destroy prm during module unload
14 * @pble_rsrc: pble resources
15 */
16void irdma_destroy_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
17{
18 struct irdma_chunk *chunk;
19 struct irdma_pble_prm *pinfo = &pble_rsrc->pinfo;
20
21 while (!list_empty(head: &pinfo->clist)) {
22 chunk = (struct irdma_chunk *) pinfo->clist.next;
23 list_del(entry: &chunk->list);
24 if (chunk->type == PBLE_SD_PAGED)
25 irdma_pble_free_paged_mem(chunk);
26 bitmap_free(bitmap: chunk->bitmapbuf);
27 kfree(objp: chunk->chunkmem.va);
28 }
29}
30
31/**
32 * irdma_hmc_init_pble - Initialize pble resources during module load
33 * @dev: irdma_sc_dev struct
34 * @pble_rsrc: pble resources
35 */
36int irdma_hmc_init_pble(struct irdma_sc_dev *dev,
37 struct irdma_hmc_pble_rsrc *pble_rsrc)
38{
39 struct irdma_hmc_info *hmc_info;
40 u32 fpm_idx = 0;
41 int status = 0;
42
43 hmc_info = dev->hmc_info;
44 pble_rsrc->dev = dev;
45 pble_rsrc->fpm_base_addr = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].base;
46 /* Start pble' on 4k boundary */
47 if (pble_rsrc->fpm_base_addr & 0xfff)
48 fpm_idx = (4096 - (pble_rsrc->fpm_base_addr & 0xfff)) >> 3;
49 pble_rsrc->unallocated_pble =
50 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt - fpm_idx;
51 pble_rsrc->next_fpm_addr = pble_rsrc->fpm_base_addr + (fpm_idx << 3);
52 pble_rsrc->pinfo.pble_shift = PBLE_SHIFT;
53
54 mutex_init(&pble_rsrc->pble_mutex_lock);
55
56 spin_lock_init(&pble_rsrc->pinfo.prm_lock);
57 INIT_LIST_HEAD(list: &pble_rsrc->pinfo.clist);
58 if (add_pble_prm(pble_rsrc)) {
59 irdma_destroy_pble_prm(pble_rsrc);
60 status = -ENOMEM;
61 }
62
63 return status;
64}
65
66/**
67 * get_sd_pd_idx - Returns sd index, pd index and rel_pd_idx from fpm address
68 * @pble_rsrc: structure containing fpm address
69 * @idx: where to return indexes
70 */
71static void get_sd_pd_idx(struct irdma_hmc_pble_rsrc *pble_rsrc,
72 struct sd_pd_idx *idx)
73{
74 idx->sd_idx = pble_rsrc->next_fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE;
75 idx->pd_idx = (u32)(pble_rsrc->next_fpm_addr / IRDMA_HMC_PAGED_BP_SIZE);
76 idx->rel_pd_idx = (idx->pd_idx % IRDMA_HMC_PD_CNT_IN_SD);
77}
78
79/**
80 * add_sd_direct - add sd direct for pble
81 * @pble_rsrc: pble resource ptr
82 * @info: page info for sd
83 */
84static int add_sd_direct(struct irdma_hmc_pble_rsrc *pble_rsrc,
85 struct irdma_add_page_info *info)
86{
87 struct irdma_sc_dev *dev = pble_rsrc->dev;
88 int ret_code = 0;
89 struct sd_pd_idx *idx = &info->idx;
90 struct irdma_chunk *chunk = info->chunk;
91 struct irdma_hmc_info *hmc_info = info->hmc_info;
92 struct irdma_hmc_sd_entry *sd_entry = info->sd_entry;
93 u32 offset = 0;
94
95 if (!sd_entry->valid) {
96 ret_code = irdma_add_sd_table_entry(hw: dev->hw, hmc_info,
97 sd_index: info->idx.sd_idx,
98 type: IRDMA_SD_TYPE_DIRECT,
99 IRDMA_HMC_DIRECT_BP_SIZE);
100 if (ret_code)
101 return ret_code;
102
103 chunk->type = PBLE_SD_CONTIGOUS;
104 }
105
106 offset = idx->rel_pd_idx << HMC_PAGED_BP_SHIFT;
107 chunk->size = info->pages << HMC_PAGED_BP_SHIFT;
108 chunk->vaddr = sd_entry->u.bp.addr.va + offset;
109 chunk->fpm_addr = pble_rsrc->next_fpm_addr;
110 ibdev_dbg(to_ibdev(dev),
111 "PBLE: chunk_size[%lld] = 0x%llx vaddr=0x%p fpm_addr = %llx\n",
112 chunk->size, chunk->size, chunk->vaddr, chunk->fpm_addr);
113
114 return 0;
115}
116
117/**
118 * fpm_to_idx - given fpm address, get pble index
119 * @pble_rsrc: pble resource management
120 * @addr: fpm address for index
121 */
122static u32 fpm_to_idx(struct irdma_hmc_pble_rsrc *pble_rsrc, u64 addr)
123{
124 u64 idx;
125
126 idx = (addr - (pble_rsrc->fpm_base_addr)) >> 3;
127
128 return (u32)idx;
129}
130
131/**
132 * add_bp_pages - add backing pages for sd
133 * @pble_rsrc: pble resource management
134 * @info: page info for sd
135 */
136static int add_bp_pages(struct irdma_hmc_pble_rsrc *pble_rsrc,
137 struct irdma_add_page_info *info)
138{
139 struct irdma_sc_dev *dev = pble_rsrc->dev;
140 u8 *addr;
141 struct irdma_dma_mem mem;
142 struct irdma_hmc_pd_entry *pd_entry;
143 struct irdma_hmc_sd_entry *sd_entry = info->sd_entry;
144 struct irdma_hmc_info *hmc_info = info->hmc_info;
145 struct irdma_chunk *chunk = info->chunk;
146 int status = 0;
147 u32 rel_pd_idx = info->idx.rel_pd_idx;
148 u32 pd_idx = info->idx.pd_idx;
149 u32 i;
150
151 if (irdma_pble_get_paged_mem(chunk, pg_cnt: info->pages))
152 return -ENOMEM;
153
154 status = irdma_add_sd_table_entry(hw: dev->hw, hmc_info, sd_index: info->idx.sd_idx,
155 type: IRDMA_SD_TYPE_PAGED,
156 IRDMA_HMC_DIRECT_BP_SIZE);
157 if (status)
158 goto error;
159
160 addr = chunk->vaddr;
161 for (i = 0; i < info->pages; i++) {
162 mem.pa = (u64)chunk->dmainfo.dmaaddrs[i];
163 mem.size = 4096;
164 mem.va = addr;
165 pd_entry = &sd_entry->u.pd_table.pd_entry[rel_pd_idx++];
166 if (!pd_entry->valid) {
167 status = irdma_add_pd_table_entry(dev, hmc_info,
168 pd_index: pd_idx++, rsrc_pg: &mem);
169 if (status)
170 goto error;
171
172 addr += 4096;
173 }
174 }
175
176 chunk->fpm_addr = pble_rsrc->next_fpm_addr;
177 return 0;
178
179error:
180 irdma_pble_free_paged_mem(chunk);
181
182 return status;
183}
184
185/**
186 * irdma_get_type - add a sd entry type for sd
187 * @dev: irdma_sc_dev struct
188 * @idx: index of sd
189 * @pages: pages in the sd
190 */
191static enum irdma_sd_entry_type irdma_get_type(struct irdma_sc_dev *dev,
192 struct sd_pd_idx *idx, u32 pages)
193{
194 enum irdma_sd_entry_type sd_entry_type;
195
196 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3)
197 sd_entry_type = (!idx->rel_pd_idx &&
198 pages == IRDMA_HMC_PD_CNT_IN_SD) ?
199 IRDMA_SD_TYPE_DIRECT : IRDMA_SD_TYPE_PAGED;
200 else
201 sd_entry_type = (!idx->rel_pd_idx &&
202 pages == IRDMA_HMC_PD_CNT_IN_SD &&
203 dev->privileged) ?
204 IRDMA_SD_TYPE_DIRECT : IRDMA_SD_TYPE_PAGED;
205 return sd_entry_type;
206}
207
208/**
209 * add_pble_prm - add a sd entry for pble resoure
210 * @pble_rsrc: pble resource management
211 */
212static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
213{
214 struct irdma_sc_dev *dev = pble_rsrc->dev;
215 struct irdma_hmc_sd_entry *sd_entry;
216 struct irdma_hmc_info *hmc_info;
217 struct irdma_chunk *chunk;
218 struct irdma_add_page_info info;
219 struct sd_pd_idx *idx = &info.idx;
220 int ret_code = 0;
221 enum irdma_sd_entry_type sd_entry_type;
222 u64 sd_reg_val = 0;
223 struct irdma_virt_mem chunkmem;
224 u32 pages;
225
226 if (pble_rsrc->unallocated_pble < PBLE_PER_PAGE)
227 return -ENOMEM;
228
229 if (pble_rsrc->next_fpm_addr & 0xfff)
230 return -EINVAL;
231
232 chunkmem.size = sizeof(*chunk);
233 chunkmem.va = kzalloc(chunkmem.size, GFP_KERNEL);
234 if (!chunkmem.va)
235 return -ENOMEM;
236
237 chunk = chunkmem.va;
238 chunk->chunkmem = chunkmem;
239 hmc_info = dev->hmc_info;
240 chunk->dev = dev;
241 chunk->fpm_addr = pble_rsrc->next_fpm_addr;
242 get_sd_pd_idx(pble_rsrc, idx);
243 sd_entry = &hmc_info->sd_table.sd_entry[idx->sd_idx];
244 pages = (idx->rel_pd_idx) ? (IRDMA_HMC_PD_CNT_IN_SD - idx->rel_pd_idx) :
245 IRDMA_HMC_PD_CNT_IN_SD;
246 pages = min(pages, pble_rsrc->unallocated_pble >> PBLE_512_SHIFT);
247 info.chunk = chunk;
248 info.hmc_info = hmc_info;
249 info.pages = pages;
250 info.sd_entry = sd_entry;
251 if (!sd_entry->valid)
252 sd_entry_type = irdma_get_type(dev, idx, pages);
253 else
254 sd_entry_type = sd_entry->entry_type;
255
256 ibdev_dbg(to_ibdev(dev),
257 "PBLE: pages = %d, unallocated_pble[%d] current_fpm_addr = %llx\n",
258 pages, pble_rsrc->unallocated_pble,
259 pble_rsrc->next_fpm_addr);
260 ibdev_dbg(to_ibdev(dev), "PBLE: sd_entry_type = %d\n", sd_entry_type);
261 if (sd_entry_type == IRDMA_SD_TYPE_DIRECT)
262 ret_code = add_sd_direct(pble_rsrc, info: &info);
263
264 if (ret_code)
265 sd_entry_type = IRDMA_SD_TYPE_PAGED;
266 else
267 pble_rsrc->stats_direct_sds++;
268
269 if (sd_entry_type == IRDMA_SD_TYPE_PAGED) {
270 ret_code = add_bp_pages(pble_rsrc, info: &info);
271 if (ret_code)
272 goto error;
273 else
274 pble_rsrc->stats_paged_sds++;
275 }
276
277 ret_code = irdma_prm_add_pble_mem(pprm: &pble_rsrc->pinfo, pchunk: chunk);
278 if (ret_code)
279 goto error;
280
281 pble_rsrc->next_fpm_addr += chunk->size;
282 ibdev_dbg(to_ibdev(dev),
283 "PBLE: next_fpm_addr = %llx chunk_size[%llu] = 0x%llx\n",
284 pble_rsrc->next_fpm_addr, chunk->size, chunk->size);
285 pble_rsrc->unallocated_pble -= (u32)(chunk->size >> 3);
286 sd_reg_val = (sd_entry_type == IRDMA_SD_TYPE_PAGED) ?
287 sd_entry->u.pd_table.pd_page_addr.pa :
288 sd_entry->u.bp.addr.pa;
289 if ((dev->privileged && !sd_entry->valid) ||
290 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3) {
291 ret_code = irdma_hmc_sd_one(dev, hmc_fn_id: hmc_info->hmc_fn_id,
292 pa: sd_reg_val, sd_idx: idx->sd_idx,
293 type: sd_entry->entry_type, setsd: true);
294 if (ret_code)
295 goto error;
296 }
297
298 list_add(new: &chunk->list, head: &pble_rsrc->pinfo.clist);
299 sd_entry->valid = true;
300 return 0;
301
302error:
303 bitmap_free(bitmap: chunk->bitmapbuf);
304 kfree(objp: chunk->chunkmem.va);
305
306 return ret_code;
307}
308
309/**
310 * free_lvl2 - fee level 2 pble
311 * @pble_rsrc: pble resource management
312 * @palloc: level 2 pble allocation
313 */
314static void free_lvl2(struct irdma_hmc_pble_rsrc *pble_rsrc,
315 struct irdma_pble_alloc *palloc)
316{
317 u32 i;
318 struct irdma_pble_level2 *lvl2 = &palloc->level2;
319 struct irdma_pble_info *root = &lvl2->root;
320 struct irdma_pble_info *leaf = lvl2->leaf;
321
322 for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
323 if (leaf->addr)
324 irdma_prm_return_pbles(pprm: &pble_rsrc->pinfo,
325 chunkinfo: &leaf->chunkinfo);
326 else
327 break;
328 }
329
330 if (root->addr)
331 irdma_prm_return_pbles(pprm: &pble_rsrc->pinfo, chunkinfo: &root->chunkinfo);
332
333 kfree(objp: lvl2->leafmem.va);
334 lvl2->leaf = NULL;
335}
336
337/**
338 * get_lvl2_pble - get level 2 pble resource
339 * @pble_rsrc: pble resource management
340 * @palloc: level 2 pble allocation
341 */
342static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
343 struct irdma_pble_alloc *palloc)
344{
345 u32 lf4k, lflast, total, i;
346 u32 pblcnt = PBLE_PER_PAGE;
347 u64 *addr;
348 struct irdma_pble_level2 *lvl2 = &palloc->level2;
349 struct irdma_pble_info *root = &lvl2->root;
350 struct irdma_pble_info *leaf;
351 int ret_code;
352 u64 fpm_addr;
353
354 /* number of full 512 (4K) leafs) */
355 lf4k = palloc->total_cnt >> 9;
356 lflast = palloc->total_cnt % PBLE_PER_PAGE;
357 total = (lflast == 0) ? lf4k : lf4k + 1;
358 lvl2->leaf_cnt = total;
359
360 lvl2->leafmem.size = (sizeof(*leaf) * total);
361 lvl2->leafmem.va = kzalloc(lvl2->leafmem.size, GFP_KERNEL);
362 if (!lvl2->leafmem.va)
363 return -ENOMEM;
364
365 lvl2->leaf = lvl2->leafmem.va;
366 leaf = lvl2->leaf;
367 ret_code = irdma_prm_get_pbles(pprm: &pble_rsrc->pinfo, chunkinfo: &root->chunkinfo,
368 mem_size: total << 3, vaddr: &root->addr, fpm_addr: &fpm_addr);
369 if (ret_code) {
370 kfree(objp: lvl2->leafmem.va);
371 lvl2->leaf = NULL;
372 return -ENOMEM;
373 }
374
375 root->idx = fpm_to_idx(pble_rsrc, addr: fpm_addr);
376 root->cnt = total;
377 addr = root->addr;
378 for (i = 0; i < total; i++, leaf++) {
379 pblcnt = (lflast && ((i + 1) == total)) ?
380 lflast : PBLE_PER_PAGE;
381 ret_code = irdma_prm_get_pbles(pprm: &pble_rsrc->pinfo,
382 chunkinfo: &leaf->chunkinfo, mem_size: pblcnt << 3,
383 vaddr: &leaf->addr, fpm_addr: &fpm_addr);
384 if (ret_code)
385 goto error;
386
387 leaf->idx = fpm_to_idx(pble_rsrc, addr: fpm_addr);
388
389 leaf->cnt = pblcnt;
390 *addr = (u64)leaf->idx;
391 addr++;
392 }
393
394 palloc->level = PBLE_LEVEL_2;
395 pble_rsrc->stats_lvl2++;
396 return 0;
397
398error:
399 free_lvl2(pble_rsrc, palloc);
400
401 return -ENOMEM;
402}
403
404/**
405 * get_lvl1_pble - get level 1 pble resource
406 * @pble_rsrc: pble resource management
407 * @palloc: level 1 pble allocation
408 */
409static int get_lvl1_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
410 struct irdma_pble_alloc *palloc)
411{
412 int ret_code;
413 u64 fpm_addr;
414 struct irdma_pble_info *lvl1 = &palloc->level1;
415
416 ret_code = irdma_prm_get_pbles(pprm: &pble_rsrc->pinfo, chunkinfo: &lvl1->chunkinfo,
417 mem_size: palloc->total_cnt << 3, vaddr: &lvl1->addr,
418 fpm_addr: &fpm_addr);
419 if (ret_code)
420 return -ENOMEM;
421
422 palloc->level = PBLE_LEVEL_1;
423 lvl1->idx = fpm_to_idx(pble_rsrc, addr: fpm_addr);
424 lvl1->cnt = palloc->total_cnt;
425 pble_rsrc->stats_lvl1++;
426
427 return 0;
428}
429
430/**
431 * get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine
432 * @pble_rsrc: pble resources
433 * @palloc: contains all inforamtion regarding pble (idx + pble addr)
434 * @lvl: Bitmask for requested pble level
435 */
436static int get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
437 struct irdma_pble_alloc *palloc, u8 lvl)
438{
439 int status = 0;
440
441 status = get_lvl1_pble(pble_rsrc, palloc);
442 if (!status || lvl == PBLE_LEVEL_1 || palloc->total_cnt <= PBLE_PER_PAGE)
443 return status;
444
445 status = get_lvl2_pble(pble_rsrc, palloc);
446
447 return status;
448}
449
450/**
451 * irdma_get_pble - allocate pbles from the prm
452 * @pble_rsrc: pble resources
453 * @palloc: contains all inforamtion regarding pble (idx + pble addr)
454 * @pble_cnt: #of pbles requested
455 * @lvl: requested pble level mask
456 */
457int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
458 struct irdma_pble_alloc *palloc, u32 pble_cnt,
459 u8 lvl)
460{
461 int status = 0;
462 int max_sds = 0;
463 int i;
464
465 palloc->total_cnt = pble_cnt;
466 palloc->level = PBLE_LEVEL_0;
467
468 mutex_lock(&pble_rsrc->pble_mutex_lock);
469
470 /*check first to see if we can get pble's without acquiring
471 * additional sd's
472 */
473 status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl);
474 if (!status)
475 goto exit;
476
477 max_sds = (palloc->total_cnt >> 18) + 1;
478 for (i = 0; i < max_sds; i++) {
479 status = add_pble_prm(pble_rsrc);
480 if (status)
481 break;
482
483 status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl);
484 /* if level1_only, only go through it once */
485 if (!status || lvl)
486 break;
487 }
488
489exit:
490 if (!status) {
491 pble_rsrc->allocdpbles += pble_cnt;
492 pble_rsrc->stats_alloc_ok++;
493 } else {
494 pble_rsrc->stats_alloc_fail++;
495 }
496 mutex_unlock(lock: &pble_rsrc->pble_mutex_lock);
497
498 return status;
499}
500
501/**
502 * irdma_free_pble - put pbles back into prm
503 * @pble_rsrc: pble resources
504 * @palloc: contains all information regarding pble resource being freed
505 */
506void irdma_free_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
507 struct irdma_pble_alloc *palloc)
508{
509 if (palloc->level == PBLE_LEVEL_2)
510 free_lvl2(pble_rsrc, palloc);
511 else
512 irdma_prm_return_pbles(pprm: &pble_rsrc->pinfo,
513 chunkinfo: &palloc->level1.chunkinfo);
514
515 mutex_lock(&pble_rsrc->pble_mutex_lock);
516 pble_rsrc->freedpbles += palloc->total_cnt;
517 pble_rsrc->stats_alloc_freed++;
518 mutex_unlock(lock: &pble_rsrc->pble_mutex_lock);
519}
520

source code of linux/drivers/infiniband/hw/irdma/pble.c