1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2013 - 2018 Intel Corporation. */ |
3 | |
4 | #include "i40e_alloc.h" |
5 | #include "i40e_debug.h" |
6 | #include "i40e_hmc.h" |
7 | #include "i40e_type.h" |
8 | |
9 | /** |
10 | * i40e_add_sd_table_entry - Adds a segment descriptor to the table |
11 | * @hw: pointer to our hw struct |
12 | * @hmc_info: pointer to the HMC configuration information struct |
13 | * @sd_index: segment descriptor index to manipulate |
14 | * @type: what type of segment descriptor we're manipulating |
15 | * @direct_mode_sz: size to alloc in direct mode |
16 | **/ |
17 | int i40e_add_sd_table_entry(struct i40e_hw *hw, |
18 | struct i40e_hmc_info *hmc_info, |
19 | u32 sd_index, |
20 | enum i40e_sd_entry_type type, |
21 | u64 direct_mode_sz) |
22 | { |
23 | struct i40e_hmc_sd_entry *sd_entry; |
24 | bool dma_mem_alloc_done = false; |
25 | struct i40e_dma_mem mem; |
26 | int ret_code = 0; |
27 | u64 alloc_len; |
28 | |
29 | if (NULL == hmc_info->sd_table.sd_entry) { |
30 | ret_code = -EINVAL; |
31 | hw_dbg(hw, "i40e_add_sd_table_entry: bad sd_entry\n" ); |
32 | goto exit; |
33 | } |
34 | |
35 | if (sd_index >= hmc_info->sd_table.sd_cnt) { |
36 | ret_code = -EINVAL; |
37 | hw_dbg(hw, "i40e_add_sd_table_entry: bad sd_index\n" ); |
38 | goto exit; |
39 | } |
40 | |
41 | sd_entry = &hmc_info->sd_table.sd_entry[sd_index]; |
42 | if (!sd_entry->valid) { |
43 | if (type == I40E_SD_TYPE_PAGED) |
44 | alloc_len = I40E_HMC_PAGED_BP_SIZE; |
45 | else |
46 | alloc_len = direct_mode_sz; |
47 | |
48 | /* allocate a 4K pd page or 2M backing page */ |
49 | ret_code = i40e_allocate_dma_mem(hw, mem: &mem, size: alloc_len, |
50 | I40E_HMC_PD_BP_BUF_ALIGNMENT); |
51 | if (ret_code) |
52 | goto exit; |
53 | dma_mem_alloc_done = true; |
54 | if (I40E_SD_TYPE_PAGED == type) { |
55 | ret_code = i40e_allocate_virt_mem(hw, |
56 | mem: &sd_entry->u.pd_table.pd_entry_virt_mem, |
57 | size: sizeof(struct i40e_hmc_pd_entry) * 512); |
58 | if (ret_code) |
59 | goto exit; |
60 | sd_entry->u.pd_table.pd_entry = |
61 | (struct i40e_hmc_pd_entry *) |
62 | sd_entry->u.pd_table.pd_entry_virt_mem.va; |
63 | sd_entry->u.pd_table.pd_page_addr = mem; |
64 | } else { |
65 | sd_entry->u.bp.addr = mem; |
66 | sd_entry->u.bp.sd_pd_index = sd_index; |
67 | } |
68 | /* initialize the sd entry */ |
69 | hmc_info->sd_table.sd_entry[sd_index].entry_type = type; |
70 | |
71 | /* increment the ref count */ |
72 | I40E_INC_SD_REFCNT(&hmc_info->sd_table); |
73 | } |
74 | /* Increment backing page reference count */ |
75 | if (I40E_SD_TYPE_DIRECT == sd_entry->entry_type) |
76 | I40E_INC_BP_REFCNT(&sd_entry->u.bp); |
77 | exit: |
78 | if (ret_code) |
79 | if (dma_mem_alloc_done) |
80 | i40e_free_dma_mem(hw, mem: &mem); |
81 | |
82 | return ret_code; |
83 | } |
84 | |
85 | /** |
86 | * i40e_add_pd_table_entry - Adds page descriptor to the specified table |
87 | * @hw: pointer to our HW structure |
88 | * @hmc_info: pointer to the HMC configuration information structure |
89 | * @pd_index: which page descriptor index to manipulate |
90 | * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one. |
91 | * |
92 | * This function: |
93 | * 1. Initializes the pd entry |
94 | * 2. Adds pd_entry in the pd_table |
95 | * 3. Mark the entry valid in i40e_hmc_pd_entry structure |
96 | * 4. Initializes the pd_entry's ref count to 1 |
97 | * assumptions: |
98 | * 1. The memory for pd should be pinned down, physically contiguous and |
99 | * aligned on 4K boundary and zeroed memory. |
100 | * 2. It should be 4K in size. |
101 | **/ |
102 | int i40e_add_pd_table_entry(struct i40e_hw *hw, |
103 | struct i40e_hmc_info *hmc_info, |
104 | u32 pd_index, |
105 | struct i40e_dma_mem *rsrc_pg) |
106 | { |
107 | struct i40e_hmc_pd_table *pd_table; |
108 | struct i40e_hmc_pd_entry *pd_entry; |
109 | struct i40e_dma_mem mem; |
110 | struct i40e_dma_mem *page = &mem; |
111 | u32 sd_idx, rel_pd_idx; |
112 | int ret_code = 0; |
113 | u64 page_desc; |
114 | u64 *pd_addr; |
115 | |
116 | if (pd_index / I40E_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt) { |
117 | ret_code = -EINVAL; |
118 | hw_dbg(hw, "i40e_add_pd_table_entry: bad pd_index\n" ); |
119 | goto exit; |
120 | } |
121 | |
122 | /* find corresponding sd */ |
123 | sd_idx = (pd_index / I40E_HMC_PD_CNT_IN_SD); |
124 | if (I40E_SD_TYPE_PAGED != |
125 | hmc_info->sd_table.sd_entry[sd_idx].entry_type) |
126 | goto exit; |
127 | |
128 | rel_pd_idx = (pd_index % I40E_HMC_PD_CNT_IN_SD); |
129 | pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table; |
130 | pd_entry = &pd_table->pd_entry[rel_pd_idx]; |
131 | if (!pd_entry->valid) { |
132 | if (rsrc_pg) { |
133 | pd_entry->rsrc_pg = true; |
134 | page = rsrc_pg; |
135 | } else { |
136 | /* allocate a 4K backing page */ |
137 | ret_code = i40e_allocate_dma_mem(hw, mem: page, |
138 | I40E_HMC_PAGED_BP_SIZE, |
139 | I40E_HMC_PD_BP_BUF_ALIGNMENT); |
140 | if (ret_code) |
141 | goto exit; |
142 | pd_entry->rsrc_pg = false; |
143 | } |
144 | |
145 | pd_entry->bp.addr = *page; |
146 | pd_entry->bp.sd_pd_index = pd_index; |
147 | pd_entry->bp.entry_type = I40E_SD_TYPE_PAGED; |
148 | /* Set page address and valid bit */ |
149 | page_desc = page->pa | 0x1; |
150 | |
151 | pd_addr = (u64 *)pd_table->pd_page_addr.va; |
152 | pd_addr += rel_pd_idx; |
153 | |
154 | /* Add the backing page physical address in the pd entry */ |
155 | memcpy(pd_addr, &page_desc, sizeof(u64)); |
156 | |
157 | pd_entry->sd_index = sd_idx; |
158 | pd_entry->valid = true; |
159 | I40E_INC_PD_REFCNT(pd_table); |
160 | } |
161 | I40E_INC_BP_REFCNT(&pd_entry->bp); |
162 | exit: |
163 | return ret_code; |
164 | } |
165 | |
166 | /** |
167 | * i40e_remove_pd_bp - remove a backing page from a page descriptor |
168 | * @hw: pointer to our HW structure |
169 | * @hmc_info: pointer to the HMC configuration information structure |
170 | * @idx: the page index |
171 | * |
172 | * This function: |
173 | * 1. Marks the entry in pd tabe (for paged address mode) or in sd table |
174 | * (for direct address mode) invalid. |
175 | * 2. Write to register PMPDINV to invalidate the backing page in FV cache |
176 | * 3. Decrement the ref count for the pd _entry |
177 | * assumptions: |
178 | * 1. Caller can deallocate the memory used by backing storage after this |
179 | * function returns. |
180 | **/ |
181 | int i40e_remove_pd_bp(struct i40e_hw *hw, |
182 | struct i40e_hmc_info *hmc_info, |
183 | u32 idx) |
184 | { |
185 | struct i40e_hmc_pd_entry *pd_entry; |
186 | struct i40e_hmc_pd_table *pd_table; |
187 | struct i40e_hmc_sd_entry *sd_entry; |
188 | u32 sd_idx, rel_pd_idx; |
189 | int ret_code = 0; |
190 | u64 *pd_addr; |
191 | |
192 | /* calculate index */ |
193 | sd_idx = idx / I40E_HMC_PD_CNT_IN_SD; |
194 | rel_pd_idx = idx % I40E_HMC_PD_CNT_IN_SD; |
195 | if (sd_idx >= hmc_info->sd_table.sd_cnt) { |
196 | ret_code = -EINVAL; |
197 | hw_dbg(hw, "i40e_remove_pd_bp: bad idx\n" ); |
198 | goto exit; |
199 | } |
200 | sd_entry = &hmc_info->sd_table.sd_entry[sd_idx]; |
201 | if (I40E_SD_TYPE_PAGED != sd_entry->entry_type) { |
202 | ret_code = -EINVAL; |
203 | hw_dbg(hw, "i40e_remove_pd_bp: wrong sd_entry type\n" ); |
204 | goto exit; |
205 | } |
206 | /* get the entry and decrease its ref counter */ |
207 | pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table; |
208 | pd_entry = &pd_table->pd_entry[rel_pd_idx]; |
209 | I40E_DEC_BP_REFCNT(&pd_entry->bp); |
210 | if (pd_entry->bp.ref_cnt) |
211 | goto exit; |
212 | |
213 | /* mark the entry invalid */ |
214 | pd_entry->valid = false; |
215 | I40E_DEC_PD_REFCNT(pd_table); |
216 | pd_addr = (u64 *)pd_table->pd_page_addr.va; |
217 | pd_addr += rel_pd_idx; |
218 | memset(pd_addr, 0, sizeof(u64)); |
219 | I40E_INVALIDATE_PF_HMC_PD(hw, sd_idx, idx); |
220 | |
221 | /* free memory here */ |
222 | if (!pd_entry->rsrc_pg) |
223 | ret_code = i40e_free_dma_mem(hw, mem: &pd_entry->bp.addr); |
224 | if (ret_code) |
225 | goto exit; |
226 | if (!pd_table->ref_cnt) |
227 | i40e_free_virt_mem(hw, mem: &pd_table->pd_entry_virt_mem); |
228 | exit: |
229 | return ret_code; |
230 | } |
231 | |
232 | /** |
233 | * i40e_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry |
234 | * @hmc_info: pointer to the HMC configuration information structure |
235 | * @idx: the page index |
236 | **/ |
237 | int i40e_prep_remove_sd_bp(struct i40e_hmc_info *hmc_info, |
238 | u32 idx) |
239 | { |
240 | struct i40e_hmc_sd_entry *sd_entry; |
241 | int ret_code = 0; |
242 | |
243 | /* get the entry and decrease its ref counter */ |
244 | sd_entry = &hmc_info->sd_table.sd_entry[idx]; |
245 | I40E_DEC_BP_REFCNT(&sd_entry->u.bp); |
246 | if (sd_entry->u.bp.ref_cnt) { |
247 | ret_code = -EBUSY; |
248 | goto exit; |
249 | } |
250 | I40E_DEC_SD_REFCNT(&hmc_info->sd_table); |
251 | |
252 | /* mark the entry invalid */ |
253 | sd_entry->valid = false; |
254 | exit: |
255 | return ret_code; |
256 | } |
257 | |
258 | /** |
259 | * i40e_remove_sd_bp_new - Removes a backing page from a segment descriptor |
260 | * @hw: pointer to our hw struct |
261 | * @hmc_info: pointer to the HMC configuration information structure |
262 | * @idx: the page index |
263 | * @is_pf: used to distinguish between VF and PF |
264 | **/ |
265 | int i40e_remove_sd_bp_new(struct i40e_hw *hw, |
266 | struct i40e_hmc_info *hmc_info, |
267 | u32 idx, bool is_pf) |
268 | { |
269 | struct i40e_hmc_sd_entry *sd_entry; |
270 | |
271 | if (!is_pf) |
272 | return -EOPNOTSUPP; |
273 | |
274 | /* get the entry and decrease its ref counter */ |
275 | sd_entry = &hmc_info->sd_table.sd_entry[idx]; |
276 | I40E_CLEAR_PF_SD_ENTRY(hw, idx, I40E_SD_TYPE_DIRECT); |
277 | |
278 | return i40e_free_dma_mem(hw, mem: &sd_entry->u.bp.addr); |
279 | } |
280 | |
281 | /** |
282 | * i40e_prep_remove_pd_page - Prepares to remove a PD page from sd entry. |
283 | * @hmc_info: pointer to the HMC configuration information structure |
284 | * @idx: segment descriptor index to find the relevant page descriptor |
285 | **/ |
286 | int i40e_prep_remove_pd_page(struct i40e_hmc_info *hmc_info, |
287 | u32 idx) |
288 | { |
289 | struct i40e_hmc_sd_entry *sd_entry; |
290 | int ret_code = 0; |
291 | |
292 | sd_entry = &hmc_info->sd_table.sd_entry[idx]; |
293 | |
294 | if (sd_entry->u.pd_table.ref_cnt) { |
295 | ret_code = -EBUSY; |
296 | goto exit; |
297 | } |
298 | |
299 | /* mark the entry invalid */ |
300 | sd_entry->valid = false; |
301 | |
302 | I40E_DEC_SD_REFCNT(&hmc_info->sd_table); |
303 | exit: |
304 | return ret_code; |
305 | } |
306 | |
307 | /** |
308 | * i40e_remove_pd_page_new - Removes a PD page from sd entry. |
309 | * @hw: pointer to our hw struct |
310 | * @hmc_info: pointer to the HMC configuration information structure |
311 | * @idx: segment descriptor index to find the relevant page descriptor |
312 | * @is_pf: used to distinguish between VF and PF |
313 | **/ |
314 | int i40e_remove_pd_page_new(struct i40e_hw *hw, |
315 | struct i40e_hmc_info *hmc_info, |
316 | u32 idx, bool is_pf) |
317 | { |
318 | struct i40e_hmc_sd_entry *sd_entry; |
319 | |
320 | if (!is_pf) |
321 | return -EOPNOTSUPP; |
322 | |
323 | sd_entry = &hmc_info->sd_table.sd_entry[idx]; |
324 | I40E_CLEAR_PF_SD_ENTRY(hw, idx, I40E_SD_TYPE_PAGED); |
325 | |
326 | return i40e_free_dma_mem(hw, mem: &sd_entry->u.pd_table.pd_page_addr); |
327 | } |
328 | |