1 | // SPDX-License-Identifier: GPL-2.0 |
2 | |
3 | /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. |
4 | * Copyright (C) 2019-2022 Linaro Ltd. |
5 | */ |
6 | |
7 | #include <linux/types.h> |
8 | #include <linux/bits.h> |
9 | #include <linux/bitfield.h> |
10 | #include <linux/refcount.h> |
11 | #include <linux/scatterlist.h> |
12 | #include <linux/dma-direction.h> |
13 | |
14 | #include "gsi.h" |
15 | #include "gsi_private.h" |
16 | #include "gsi_trans.h" |
17 | #include "ipa_gsi.h" |
18 | #include "ipa_data.h" |
19 | #include "ipa_cmd.h" |
20 | |
21 | /** |
22 | * DOC: GSI Transactions |
23 | * |
24 | * A GSI transaction abstracts the behavior of a GSI channel by representing |
25 | * everything about a related group of IPA operations in a single structure. |
26 | * (A "operation" in this sense is either a data transfer or an IPA immediate |
27 | * command.) Most details of interaction with the GSI hardware are managed |
28 | * by the GSI transaction core, allowing users to simply describe operations |
29 | * to be performed. When a transaction has completed a callback function |
30 | * (dependent on the type of endpoint associated with the channel) allows |
31 | * cleanup of resources associated with the transaction. |
32 | * |
33 | * To perform an operation (or set of them), a user of the GSI transaction |
34 | * interface allocates a transaction, indicating the number of TREs required |
35 | * (one per operation). If sufficient TREs are available, they are reserved |
36 | * for use in the transaction and the allocation succeeds. This way |
37 | * exhaustion of the available TREs in a channel ring is detected as early |
38 | * as possible. Any other resources that might be needed to complete a |
39 | * transaction are also allocated when the transaction is allocated. |
40 | * |
41 | * Operations performed as part of a transaction are represented in an array |
42 | * of Linux scatterlist structures, allocated with the transaction. These |
43 | * scatterlist structures are initialized by "adding" operations to the |
44 | * transaction. If a buffer in an operation must be mapped for DMA, this is |
45 | * done at the time it is added to the transaction. It is possible for a |
46 | * mapping error to occur when an operation is added. In this case the |
47 | * transaction should simply be freed; this correctly releases resources |
48 | * associated with the transaction. |
49 | * |
50 | * Once all operations have been successfully added to a transaction, the |
51 | * transaction is committed. Committing transfers ownership of the entire |
52 | * transaction to the GSI transaction core. The GSI transaction code |
53 | * formats the content of the scatterlist array into the channel ring |
54 | * buffer and informs the hardware that new TREs are available to process. |
55 | * |
56 | * The last TRE in each transaction is marked to interrupt the AP when the |
57 | * GSI hardware has completed it. Because transfers described by TREs are |
58 | * performed strictly in order, signaling the completion of just the last |
59 | * TRE in the transaction is sufficient to indicate the full transaction |
60 | * is complete. |
61 | * |
62 | * When a transaction is complete, ipa_gsi_trans_complete() is called by the |
63 | * GSI code into the IPA layer, allowing it to perform any final cleanup |
64 | * required before the transaction is freed. |
65 | */ |
66 | |
67 | /* Hardware values representing a transfer element type */ |
68 | enum gsi_tre_type { |
69 | GSI_RE_XFER = 0x2, |
70 | GSI_RE_IMMD_CMD = 0x3, |
71 | }; |
72 | |
73 | /* An entry in a channel ring */ |
74 | struct gsi_tre { |
75 | __le64 addr; /* DMA address */ |
76 | __le16 len_opcode; /* length in bytes or enum IPA_CMD_* */ |
77 | __le16 reserved; |
78 | __le32 flags; /* TRE_FLAGS_* */ |
79 | }; |
80 | |
81 | /* gsi_tre->flags mask values (in CPU byte order) */ |
82 | #define TRE_FLAGS_CHAIN_FMASK GENMASK(0, 0) |
83 | #define TRE_FLAGS_IEOT_FMASK GENMASK(9, 9) |
84 | #define TRE_FLAGS_BEI_FMASK GENMASK(10, 10) |
85 | #define TRE_FLAGS_TYPE_FMASK GENMASK(23, 16) |
86 | |
87 | int gsi_trans_pool_init(struct gsi_trans_pool *pool, size_t size, u32 count, |
88 | u32 max_alloc) |
89 | { |
90 | size_t alloc_size; |
91 | void *virt; |
92 | |
93 | if (!size) |
94 | return -EINVAL; |
95 | if (count < max_alloc) |
96 | return -EINVAL; |
97 | if (!max_alloc) |
98 | return -EINVAL; |
99 | |
100 | /* By allocating a few extra entries in our pool (one less |
101 | * than the maximum number that will be requested in a |
102 | * single allocation), we can always satisfy requests without |
103 | * ever worrying about straddling the end of the pool array. |
104 | * If there aren't enough entries starting at the free index, |
105 | * we just allocate free entries from the beginning of the pool. |
106 | */ |
107 | alloc_size = size_mul(factor1: count + max_alloc - 1, factor2: size); |
108 | alloc_size = kmalloc_size_roundup(size: alloc_size); |
109 | virt = kzalloc(size: alloc_size, GFP_KERNEL); |
110 | if (!virt) |
111 | return -ENOMEM; |
112 | |
113 | pool->base = virt; |
114 | /* If the allocator gave us any extra memory, use it */ |
115 | pool->count = alloc_size / size; |
116 | pool->free = 0; |
117 | pool->max_alloc = max_alloc; |
118 | pool->size = size; |
119 | pool->addr = 0; /* Only used for DMA pools */ |
120 | |
121 | return 0; |
122 | } |
123 | |
124 | void gsi_trans_pool_exit(struct gsi_trans_pool *pool) |
125 | { |
126 | kfree(objp: pool->base); |
127 | memset(pool, 0, sizeof(*pool)); |
128 | } |
129 | |
130 | /* Home-grown DMA pool. This way we can preallocate the pool, and guarantee |
131 | * allocations will succeed. The immediate commands in a transaction can |
132 | * require up to max_alloc elements from the pool. But we only allow |
133 | * allocation of a single element from a DMA pool at a time. |
134 | */ |
135 | int gsi_trans_pool_init_dma(struct device *dev, struct gsi_trans_pool *pool, |
136 | size_t size, u32 count, u32 max_alloc) |
137 | { |
138 | size_t total_size; |
139 | dma_addr_t addr; |
140 | void *virt; |
141 | |
142 | if (!size) |
143 | return -EINVAL; |
144 | if (count < max_alloc) |
145 | return -EINVAL; |
146 | if (!max_alloc) |
147 | return -EINVAL; |
148 | |
149 | /* Don't let allocations cross a power-of-two boundary */ |
150 | size = __roundup_pow_of_two(n: size); |
151 | total_size = (count + max_alloc - 1) * size; |
152 | |
153 | /* The allocator will give us a power-of-2 number of pages |
154 | * sufficient to satisfy our request. Round up our requested |
155 | * size to avoid any unused space in the allocation. This way |
156 | * gsi_trans_pool_exit_dma() can assume the total allocated |
157 | * size is exactly (count * size). |
158 | */ |
159 | total_size = PAGE_SIZE << get_order(size: total_size); |
160 | |
161 | virt = dma_alloc_coherent(dev, size: total_size, dma_handle: &addr, GFP_KERNEL); |
162 | if (!virt) |
163 | return -ENOMEM; |
164 | |
165 | pool->base = virt; |
166 | pool->count = total_size / size; |
167 | pool->free = 0; |
168 | pool->size = size; |
169 | pool->max_alloc = max_alloc; |
170 | pool->addr = addr; |
171 | |
172 | return 0; |
173 | } |
174 | |
175 | void gsi_trans_pool_exit_dma(struct device *dev, struct gsi_trans_pool *pool) |
176 | { |
177 | size_t total_size = pool->count * pool->size; |
178 | |
179 | dma_free_coherent(dev, size: total_size, cpu_addr: pool->base, dma_handle: pool->addr); |
180 | memset(pool, 0, sizeof(*pool)); |
181 | } |
182 | |
183 | /* Return the byte offset of the next free entry in the pool */ |
184 | static u32 gsi_trans_pool_alloc_common(struct gsi_trans_pool *pool, u32 count) |
185 | { |
186 | u32 offset; |
187 | |
188 | WARN_ON(!count); |
189 | WARN_ON(count > pool->max_alloc); |
190 | |
191 | /* Allocate from beginning if wrap would occur */ |
192 | if (count > pool->count - pool->free) |
193 | pool->free = 0; |
194 | |
195 | offset = pool->free * pool->size; |
196 | pool->free += count; |
197 | memset(pool->base + offset, 0, count * pool->size); |
198 | |
199 | return offset; |
200 | } |
201 | |
202 | /* Allocate a contiguous block of zeroed entries from a pool */ |
203 | void *gsi_trans_pool_alloc(struct gsi_trans_pool *pool, u32 count) |
204 | { |
205 | return pool->base + gsi_trans_pool_alloc_common(pool, count); |
206 | } |
207 | |
208 | /* Allocate a single zeroed entry from a DMA pool */ |
209 | void *gsi_trans_pool_alloc_dma(struct gsi_trans_pool *pool, dma_addr_t *addr) |
210 | { |
211 | u32 offset = gsi_trans_pool_alloc_common(pool, count: 1); |
212 | |
213 | *addr = pool->addr + offset; |
214 | |
215 | return pool->base + offset; |
216 | } |
217 | |
218 | /* Map a TRE ring entry index to the transaction it is associated with */ |
219 | static void gsi_trans_map(struct gsi_trans *trans, u32 index) |
220 | { |
221 | struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; |
222 | |
223 | /* The completion event will indicate the last TRE used */ |
224 | index += trans->used_count - 1; |
225 | |
226 | /* Note: index *must* be used modulo the ring count here */ |
227 | channel->trans_info.map[index % channel->tre_ring.count] = trans; |
228 | } |
229 | |
230 | /* Return the transaction mapped to a given ring entry */ |
231 | struct gsi_trans * |
232 | gsi_channel_trans_mapped(struct gsi_channel *channel, u32 index) |
233 | { |
234 | /* Note: index *must* be used modulo the ring count here */ |
235 | return channel->trans_info.map[index % channel->tre_ring.count]; |
236 | } |
237 | |
238 | /* Return the oldest completed transaction for a channel (or null) */ |
239 | struct gsi_trans *gsi_channel_trans_complete(struct gsi_channel *channel) |
240 | { |
241 | struct gsi_trans_info *trans_info = &channel->trans_info; |
242 | u16 trans_id = trans_info->completed_id; |
243 | |
244 | if (trans_id == trans_info->pending_id) { |
245 | gsi_channel_update(channel); |
246 | if (trans_id == trans_info->pending_id) |
247 | return NULL; |
248 | } |
249 | |
250 | return &trans_info->trans[trans_id %= channel->tre_count]; |
251 | } |
252 | |
253 | /* Move a transaction from allocated to committed state */ |
254 | static void gsi_trans_move_committed(struct gsi_trans *trans) |
255 | { |
256 | struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; |
257 | struct gsi_trans_info *trans_info = &channel->trans_info; |
258 | |
259 | /* This allocated transaction is now committed */ |
260 | trans_info->allocated_id++; |
261 | } |
262 | |
263 | /* Move committed transactions to pending state */ |
264 | static void gsi_trans_move_pending(struct gsi_trans *trans) |
265 | { |
266 | struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; |
267 | struct gsi_trans_info *trans_info = &channel->trans_info; |
268 | u16 trans_index = trans - &trans_info->trans[0]; |
269 | u16 delta; |
270 | |
271 | /* These committed transactions are now pending */ |
272 | delta = trans_index - trans_info->committed_id + 1; |
273 | trans_info->committed_id += delta % channel->tre_count; |
274 | } |
275 | |
276 | /* Move pending transactions to completed state */ |
277 | void gsi_trans_move_complete(struct gsi_trans *trans) |
278 | { |
279 | struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; |
280 | struct gsi_trans_info *trans_info = &channel->trans_info; |
281 | u16 trans_index = trans - trans_info->trans; |
282 | u16 delta; |
283 | |
284 | /* These pending transactions are now completed */ |
285 | delta = trans_index - trans_info->pending_id + 1; |
286 | delta %= channel->tre_count; |
287 | trans_info->pending_id += delta; |
288 | } |
289 | |
290 | /* Move a transaction from completed to polled state */ |
291 | void gsi_trans_move_polled(struct gsi_trans *trans) |
292 | { |
293 | struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; |
294 | struct gsi_trans_info *trans_info = &channel->trans_info; |
295 | |
296 | /* This completed transaction is now polled */ |
297 | trans_info->completed_id++; |
298 | } |
299 | |
300 | /* Reserve some number of TREs on a channel. Returns true if successful */ |
301 | static bool |
302 | gsi_trans_tre_reserve(struct gsi_trans_info *trans_info, u32 tre_count) |
303 | { |
304 | int avail = atomic_read(v: &trans_info->tre_avail); |
305 | int new; |
306 | |
307 | do { |
308 | new = avail - (int)tre_count; |
309 | if (unlikely(new < 0)) |
310 | return false; |
311 | } while (!atomic_try_cmpxchg(v: &trans_info->tre_avail, old: &avail, new)); |
312 | |
313 | return true; |
314 | } |
315 | |
316 | /* Release previously-reserved TRE entries to a channel */ |
317 | static void |
318 | gsi_trans_tre_release(struct gsi_trans_info *trans_info, u32 tre_count) |
319 | { |
320 | atomic_add(i: tre_count, v: &trans_info->tre_avail); |
321 | } |
322 | |
323 | /* Return true if no transactions are allocated, false otherwise */ |
324 | bool gsi_channel_trans_idle(struct gsi *gsi, u32 channel_id) |
325 | { |
326 | u32 tre_max = gsi_channel_tre_max(gsi, channel_id); |
327 | struct gsi_trans_info *trans_info; |
328 | |
329 | trans_info = &gsi->channel[channel_id].trans_info; |
330 | |
331 | return atomic_read(v: &trans_info->tre_avail) == tre_max; |
332 | } |
333 | |
334 | /* Allocate a GSI transaction on a channel */ |
335 | struct gsi_trans *gsi_channel_trans_alloc(struct gsi *gsi, u32 channel_id, |
336 | u32 tre_count, |
337 | enum dma_data_direction direction) |
338 | { |
339 | struct gsi_channel *channel = &gsi->channel[channel_id]; |
340 | struct gsi_trans_info *trans_info; |
341 | struct gsi_trans *trans; |
342 | u16 trans_index; |
343 | |
344 | if (WARN_ON(tre_count > channel->trans_tre_max)) |
345 | return NULL; |
346 | |
347 | trans_info = &channel->trans_info; |
348 | |
349 | /* If we can't reserve the TREs for the transaction, we're done */ |
350 | if (!gsi_trans_tre_reserve(trans_info, tre_count)) |
351 | return NULL; |
352 | |
353 | trans_index = trans_info->free_id % channel->tre_count; |
354 | trans = &trans_info->trans[trans_index]; |
355 | memset(trans, 0, sizeof(*trans)); |
356 | |
357 | /* Initialize non-zero fields in the transaction */ |
358 | trans->gsi = gsi; |
359 | trans->channel_id = channel_id; |
360 | trans->rsvd_count = tre_count; |
361 | init_completion(x: &trans->completion); |
362 | |
363 | /* Allocate the scatterlist */ |
364 | trans->sgl = gsi_trans_pool_alloc(pool: &trans_info->sg_pool, count: tre_count); |
365 | sg_init_marker(sgl: trans->sgl, nents: tre_count); |
366 | |
367 | trans->direction = direction; |
368 | refcount_set(r: &trans->refcount, n: 1); |
369 | |
370 | /* This free transaction is now allocated */ |
371 | trans_info->free_id++; |
372 | |
373 | return trans; |
374 | } |
375 | |
376 | /* Free a previously-allocated transaction */ |
377 | void gsi_trans_free(struct gsi_trans *trans) |
378 | { |
379 | struct gsi_trans_info *trans_info; |
380 | |
381 | if (!refcount_dec_and_test(r: &trans->refcount)) |
382 | return; |
383 | |
384 | /* Unused transactions are allocated but never committed, pending, |
385 | * completed, or polled. |
386 | */ |
387 | trans_info = &trans->gsi->channel[trans->channel_id].trans_info; |
388 | if (!trans->used_count) { |
389 | trans_info->allocated_id++; |
390 | trans_info->committed_id++; |
391 | trans_info->pending_id++; |
392 | trans_info->completed_id++; |
393 | } else { |
394 | ipa_gsi_trans_release(trans); |
395 | } |
396 | |
397 | /* This transaction is now free */ |
398 | trans_info->polled_id++; |
399 | |
400 | /* Releasing the reserved TREs implicitly frees the sgl[] and |
401 | * (if present) info[] arrays, plus the transaction itself. |
402 | */ |
403 | gsi_trans_tre_release(trans_info, tre_count: trans->rsvd_count); |
404 | } |
405 | |
406 | /* Add an immediate command to a transaction */ |
407 | void gsi_trans_cmd_add(struct gsi_trans *trans, void *buf, u32 size, |
408 | dma_addr_t addr, enum ipa_cmd_opcode opcode) |
409 | { |
410 | u32 which = trans->used_count++; |
411 | struct scatterlist *sg; |
412 | |
413 | WARN_ON(which >= trans->rsvd_count); |
414 | |
415 | /* Commands are quite different from data transfer requests. |
416 | * Their payloads come from a pool whose memory is allocated |
417 | * using dma_alloc_coherent(). We therefore do *not* map them |
418 | * for DMA (unlike what we do for pages and skbs). |
419 | * |
420 | * When a transaction completes, the SGL is normally unmapped. |
421 | * A command transaction has direction DMA_NONE, which tells |
422 | * gsi_trans_complete() to skip the unmapping step. |
423 | * |
424 | * The only things we use directly in a command scatter/gather |
425 | * entry are the DMA address and length. We still need the SG |
426 | * table flags to be maintained though, so assign a NULL page |
427 | * pointer for that purpose. |
428 | */ |
429 | sg = &trans->sgl[which]; |
430 | sg_assign_page(sg, NULL); |
431 | sg_dma_address(sg) = addr; |
432 | sg_dma_len(sg) = size; |
433 | |
434 | trans->cmd_opcode[which] = opcode; |
435 | } |
436 | |
437 | /* Add a page transfer to a transaction. It will fill the only TRE. */ |
438 | int gsi_trans_page_add(struct gsi_trans *trans, struct page *page, u32 size, |
439 | u32 offset) |
440 | { |
441 | struct scatterlist *sg = &trans->sgl[0]; |
442 | int ret; |
443 | |
444 | if (WARN_ON(trans->rsvd_count != 1)) |
445 | return -EINVAL; |
446 | if (WARN_ON(trans->used_count)) |
447 | return -EINVAL; |
448 | |
449 | sg_set_page(sg, page, len: size, offset); |
450 | ret = dma_map_sg(trans->gsi->dev, sg, 1, trans->direction); |
451 | if (!ret) |
452 | return -ENOMEM; |
453 | |
454 | trans->used_count++; /* Transaction now owns the (DMA mapped) page */ |
455 | |
456 | return 0; |
457 | } |
458 | |
459 | /* Add an SKB transfer to a transaction. No other TREs will be used. */ |
460 | int gsi_trans_skb_add(struct gsi_trans *trans, struct sk_buff *skb) |
461 | { |
462 | struct scatterlist *sg = &trans->sgl[0]; |
463 | u32 used_count; |
464 | int ret; |
465 | |
466 | if (WARN_ON(trans->rsvd_count != 1)) |
467 | return -EINVAL; |
468 | if (WARN_ON(trans->used_count)) |
469 | return -EINVAL; |
470 | |
471 | /* skb->len will not be 0 (checked early) */ |
472 | ret = skb_to_sgvec(skb, sg, offset: 0, len: skb->len); |
473 | if (ret < 0) |
474 | return ret; |
475 | used_count = ret; |
476 | |
477 | ret = dma_map_sg(trans->gsi->dev, sg, used_count, trans->direction); |
478 | if (!ret) |
479 | return -ENOMEM; |
480 | |
481 | /* Transaction now owns the (DMA mapped) skb */ |
482 | trans->used_count += used_count; |
483 | |
484 | return 0; |
485 | } |
486 | |
487 | /* Compute the length/opcode value to use for a TRE */ |
488 | static __le16 gsi_tre_len_opcode(enum ipa_cmd_opcode opcode, u32 len) |
489 | { |
490 | return opcode == IPA_CMD_NONE ? cpu_to_le16((u16)len) |
491 | : cpu_to_le16((u16)opcode); |
492 | } |
493 | |
494 | /* Compute the flags value to use for a given TRE */ |
495 | static __le32 gsi_tre_flags(bool last_tre, bool bei, enum ipa_cmd_opcode opcode) |
496 | { |
497 | enum gsi_tre_type tre_type; |
498 | u32 tre_flags; |
499 | |
500 | tre_type = opcode == IPA_CMD_NONE ? GSI_RE_XFER : GSI_RE_IMMD_CMD; |
501 | tre_flags = u32_encode_bits(v: tre_type, TRE_FLAGS_TYPE_FMASK); |
502 | |
503 | /* Last TRE contains interrupt flags */ |
504 | if (last_tre) { |
505 | /* All transactions end in a transfer completion interrupt */ |
506 | tre_flags |= TRE_FLAGS_IEOT_FMASK; |
507 | /* Don't interrupt when outbound commands are acknowledged */ |
508 | if (bei) |
509 | tre_flags |= TRE_FLAGS_BEI_FMASK; |
510 | } else { /* All others indicate there's more to come */ |
511 | tre_flags |= TRE_FLAGS_CHAIN_FMASK; |
512 | } |
513 | |
514 | return cpu_to_le32(tre_flags); |
515 | } |
516 | |
517 | static void gsi_trans_tre_fill(struct gsi_tre *dest_tre, dma_addr_t addr, |
518 | u32 len, bool last_tre, bool bei, |
519 | enum ipa_cmd_opcode opcode) |
520 | { |
521 | struct gsi_tre tre; |
522 | |
523 | tre.addr = cpu_to_le64(addr); |
524 | tre.len_opcode = gsi_tre_len_opcode(opcode, len); |
525 | tre.reserved = 0; |
526 | tre.flags = gsi_tre_flags(last_tre, bei, opcode); |
527 | |
528 | /* ARM64 can write 16 bytes as a unit with a single instruction. |
529 | * Doing the assignment this way is an attempt to make that happen. |
530 | */ |
531 | *dest_tre = tre; |
532 | } |
533 | |
534 | /** |
535 | * __gsi_trans_commit() - Common GSI transaction commit code |
536 | * @trans: Transaction to commit |
537 | * @ring_db: Whether to tell the hardware about these queued transfers |
538 | * |
539 | * Formats channel ring TRE entries based on the content of the scatterlist. |
540 | * Maps a transaction pointer to the last ring entry used for the transaction, |
541 | * so it can be recovered when it completes. Moves the transaction to |
542 | * pending state. Finally, updates the channel ring pointer and optionally |
543 | * rings the doorbell. |
544 | */ |
545 | static void __gsi_trans_commit(struct gsi_trans *trans, bool ring_db) |
546 | { |
547 | struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; |
548 | struct gsi_ring *tre_ring = &channel->tre_ring; |
549 | enum ipa_cmd_opcode opcode = IPA_CMD_NONE; |
550 | bool bei = channel->toward_ipa; |
551 | struct gsi_tre *dest_tre; |
552 | struct scatterlist *sg; |
553 | u32 byte_count = 0; |
554 | u8 *cmd_opcode; |
555 | u32 avail; |
556 | u32 i; |
557 | |
558 | WARN_ON(!trans->used_count); |
559 | |
560 | /* Consume the entries. If we cross the end of the ring while |
561 | * filling them we'll switch to the beginning to finish. |
562 | * If there is no info array we're doing a simple data |
563 | * transfer request, whose opcode is IPA_CMD_NONE. |
564 | */ |
565 | cmd_opcode = channel->command ? &trans->cmd_opcode[0] : NULL; |
566 | avail = tre_ring->count - tre_ring->index % tre_ring->count; |
567 | dest_tre = gsi_ring_virt(ring: tre_ring, index: tre_ring->index); |
568 | for_each_sg(trans->sgl, sg, trans->used_count, i) { |
569 | bool last_tre = i == trans->used_count - 1; |
570 | dma_addr_t addr = sg_dma_address(sg); |
571 | u32 len = sg_dma_len(sg); |
572 | |
573 | byte_count += len; |
574 | if (!avail--) |
575 | dest_tre = gsi_ring_virt(ring: tre_ring, index: 0); |
576 | if (cmd_opcode) |
577 | opcode = *cmd_opcode++; |
578 | |
579 | gsi_trans_tre_fill(dest_tre, addr, len, last_tre, bei, opcode); |
580 | dest_tre++; |
581 | } |
582 | /* Associate the TRE with the transaction */ |
583 | gsi_trans_map(trans, index: tre_ring->index); |
584 | |
585 | tre_ring->index += trans->used_count; |
586 | |
587 | trans->len = byte_count; |
588 | if (channel->toward_ipa) |
589 | gsi_trans_tx_committed(trans); |
590 | |
591 | gsi_trans_move_committed(trans); |
592 | |
593 | /* Ring doorbell if requested, or if all TREs are allocated */ |
594 | if (ring_db || !atomic_read(v: &channel->trans_info.tre_avail)) { |
595 | /* Report what we're handing off to hardware for TX channels */ |
596 | if (channel->toward_ipa) |
597 | gsi_trans_tx_queued(trans); |
598 | gsi_trans_move_pending(trans); |
599 | gsi_channel_doorbell(channel); |
600 | } |
601 | } |
602 | |
603 | /* Commit a GSI transaction */ |
604 | void gsi_trans_commit(struct gsi_trans *trans, bool ring_db) |
605 | { |
606 | if (trans->used_count) |
607 | __gsi_trans_commit(trans, ring_db); |
608 | else |
609 | gsi_trans_free(trans); |
610 | } |
611 | |
612 | /* Commit a GSI transaction and wait for it to complete */ |
613 | void gsi_trans_commit_wait(struct gsi_trans *trans) |
614 | { |
615 | if (!trans->used_count) |
616 | goto out_trans_free; |
617 | |
618 | refcount_inc(r: &trans->refcount); |
619 | |
620 | __gsi_trans_commit(trans, ring_db: true); |
621 | |
622 | wait_for_completion(&trans->completion); |
623 | |
624 | out_trans_free: |
625 | gsi_trans_free(trans); |
626 | } |
627 | |
628 | /* Process the completion of a transaction; called while polling */ |
629 | void gsi_trans_complete(struct gsi_trans *trans) |
630 | { |
631 | /* If the entire SGL was mapped when added, unmap it now */ |
632 | if (trans->direction != DMA_NONE) |
633 | dma_unmap_sg(trans->gsi->dev, trans->sgl, trans->used_count, |
634 | trans->direction); |
635 | |
636 | ipa_gsi_trans_complete(trans); |
637 | |
638 | complete(&trans->completion); |
639 | |
640 | gsi_trans_free(trans); |
641 | } |
642 | |
643 | /* Cancel a channel's pending transactions */ |
644 | void gsi_channel_trans_cancel_pending(struct gsi_channel *channel) |
645 | { |
646 | struct gsi_trans_info *trans_info = &channel->trans_info; |
647 | u16 trans_id = trans_info->pending_id; |
648 | |
649 | /* channel->gsi->mutex is held by caller */ |
650 | |
651 | /* If there are no pending transactions, we're done */ |
652 | if (trans_id == trans_info->committed_id) |
653 | return; |
654 | |
655 | /* Mark all pending transactions cancelled */ |
656 | do { |
657 | struct gsi_trans *trans; |
658 | |
659 | trans = &trans_info->trans[trans_id % channel->tre_count]; |
660 | trans->cancelled = true; |
661 | } while (++trans_id != trans_info->committed_id); |
662 | |
663 | /* All pending transactions are now completed */ |
664 | trans_info->pending_id = trans_info->committed_id; |
665 | |
666 | /* Schedule NAPI polling to complete the cancelled transactions */ |
667 | napi_schedule(n: &channel->napi); |
668 | } |
669 | |
670 | /* Issue a command to read a single byte from a channel */ |
671 | int gsi_trans_read_byte(struct gsi *gsi, u32 channel_id, dma_addr_t addr) |
672 | { |
673 | struct gsi_channel *channel = &gsi->channel[channel_id]; |
674 | struct gsi_ring *tre_ring = &channel->tre_ring; |
675 | struct gsi_trans_info *trans_info; |
676 | struct gsi_tre *dest_tre; |
677 | |
678 | trans_info = &channel->trans_info; |
679 | |
680 | /* First reserve the TRE, if possible */ |
681 | if (!gsi_trans_tre_reserve(trans_info, tre_count: 1)) |
682 | return -EBUSY; |
683 | |
684 | /* Now fill the reserved TRE and tell the hardware */ |
685 | |
686 | dest_tre = gsi_ring_virt(ring: tre_ring, index: tre_ring->index); |
687 | gsi_trans_tre_fill(dest_tre, addr, len: 1, last_tre: true, bei: false, opcode: IPA_CMD_NONE); |
688 | |
689 | tre_ring->index++; |
690 | gsi_channel_doorbell(channel); |
691 | |
692 | return 0; |
693 | } |
694 | |
695 | /* Mark a gsi_trans_read_byte() request done */ |
696 | void gsi_trans_read_byte_done(struct gsi *gsi, u32 channel_id) |
697 | { |
698 | struct gsi_channel *channel = &gsi->channel[channel_id]; |
699 | |
700 | gsi_trans_tre_release(trans_info: &channel->trans_info, tre_count: 1); |
701 | } |
702 | |
703 | /* Initialize a channel's GSI transaction info */ |
704 | int gsi_channel_trans_init(struct gsi *gsi, u32 channel_id) |
705 | { |
706 | struct gsi_channel *channel = &gsi->channel[channel_id]; |
707 | u32 tre_count = channel->tre_count; |
708 | struct gsi_trans_info *trans_info; |
709 | u32 tre_max; |
710 | int ret; |
711 | |
712 | /* Ensure the size of a channel element is what's expected */ |
713 | BUILD_BUG_ON(sizeof(struct gsi_tre) != GSI_RING_ELEMENT_SIZE); |
714 | |
715 | trans_info = &channel->trans_info; |
716 | |
717 | /* The tre_avail field is what ultimately limits the number of |
718 | * outstanding transactions and their resources. A transaction |
719 | * allocation succeeds only if the TREs available are sufficient |
720 | * for what the transaction might need. |
721 | */ |
722 | tre_max = gsi_channel_tre_max(gsi: channel->gsi, channel_id); |
723 | atomic_set(v: &trans_info->tre_avail, i: tre_max); |
724 | |
725 | /* We can't use more TREs than the number available in the ring. |
726 | * This limits the number of transactions that can be outstanding. |
727 | * Worst case is one TRE per transaction (but we actually limit |
728 | * it to something a little less than that). By allocating a |
729 | * power-of-two number of transactions we can use an index |
730 | * modulo that number to determine the next one that's free. |
731 | * Transactions are allocated one at a time. |
732 | */ |
733 | trans_info->trans = kcalloc(n: tre_count, size: sizeof(*trans_info->trans), |
734 | GFP_KERNEL); |
735 | if (!trans_info->trans) |
736 | return -ENOMEM; |
737 | trans_info->free_id = 0; /* all modulo channel->tre_count */ |
738 | trans_info->allocated_id = 0; |
739 | trans_info->committed_id = 0; |
740 | trans_info->pending_id = 0; |
741 | trans_info->completed_id = 0; |
742 | trans_info->polled_id = 0; |
743 | |
744 | /* A completion event contains a pointer to the TRE that caused |
745 | * the event (which will be the last one used by the transaction). |
746 | * Each entry in this map records the transaction associated |
747 | * with a corresponding completed TRE. |
748 | */ |
749 | trans_info->map = kcalloc(n: tre_count, size: sizeof(*trans_info->map), |
750 | GFP_KERNEL); |
751 | if (!trans_info->map) { |
752 | ret = -ENOMEM; |
753 | goto err_trans_free; |
754 | } |
755 | |
756 | /* A transaction uses a scatterlist array to represent the data |
757 | * transfers implemented by the transaction. Each scatterlist |
758 | * element is used to fill a single TRE when the transaction is |
759 | * committed. So we need as many scatterlist elements as the |
760 | * maximum number of TREs that can be outstanding. |
761 | */ |
762 | ret = gsi_trans_pool_init(pool: &trans_info->sg_pool, |
763 | size: sizeof(struct scatterlist), |
764 | count: tre_max, max_alloc: channel->trans_tre_max); |
765 | if (ret) |
766 | goto err_map_free; |
767 | |
768 | |
769 | return 0; |
770 | |
771 | err_map_free: |
772 | kfree(objp: trans_info->map); |
773 | err_trans_free: |
774 | kfree(objp: trans_info->trans); |
775 | |
776 | dev_err(gsi->dev, "error %d initializing channel %u transactions\n" , |
777 | ret, channel_id); |
778 | |
779 | return ret; |
780 | } |
781 | |
782 | /* Inverse of gsi_channel_trans_init() */ |
783 | void gsi_channel_trans_exit(struct gsi_channel *channel) |
784 | { |
785 | struct gsi_trans_info *trans_info = &channel->trans_info; |
786 | |
787 | gsi_trans_pool_exit(pool: &trans_info->sg_pool); |
788 | kfree(objp: trans_info->trans); |
789 | kfree(objp: trans_info->map); |
790 | } |
791 | |