1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | #include <linux/init.h> |
3 | #include <linux/scatterlist.h> |
4 | #include <linux/mempool.h> |
5 | #include <linux/slab.h> |
6 | |
7 | #define SG_MEMPOOL_NR ARRAY_SIZE(sg_pools) |
8 | #define SG_MEMPOOL_SIZE 2 |
9 | |
10 | struct sg_pool { |
11 | size_t size; |
12 | char *name; |
13 | struct kmem_cache *slab; |
14 | mempool_t *pool; |
15 | }; |
16 | |
17 | #define SP(x) { .size = x, "sgpool-" __stringify(x) } |
18 | #if (SG_CHUNK_SIZE < 32) |
19 | #error SG_CHUNK_SIZE is too small (must be 32 or greater) |
20 | #endif |
21 | static struct sg_pool sg_pools[] = { |
22 | SP(8), |
23 | SP(16), |
24 | #if (SG_CHUNK_SIZE > 32) |
25 | SP(32), |
26 | #if (SG_CHUNK_SIZE > 64) |
27 | SP(64), |
28 | #if (SG_CHUNK_SIZE > 128) |
29 | SP(128), |
30 | #if (SG_CHUNK_SIZE > 256) |
31 | #error SG_CHUNK_SIZE is too large (256 MAX) |
32 | #endif |
33 | #endif |
34 | #endif |
35 | #endif |
36 | SP(SG_CHUNK_SIZE) |
37 | }; |
38 | #undef SP |
39 | |
40 | static inline unsigned int sg_pool_index(unsigned short nents) |
41 | { |
42 | unsigned int index; |
43 | |
44 | BUG_ON(nents > SG_CHUNK_SIZE); |
45 | |
46 | if (nents <= 8) |
47 | index = 0; |
48 | else |
49 | index = get_count_order(count: nents) - 3; |
50 | |
51 | return index; |
52 | } |
53 | |
54 | static void sg_pool_free(struct scatterlist *sgl, unsigned int nents) |
55 | { |
56 | struct sg_pool *sgp; |
57 | |
58 | sgp = sg_pools + sg_pool_index(nents); |
59 | mempool_free(element: sgl, pool: sgp->pool); |
60 | } |
61 | |
62 | static struct scatterlist *sg_pool_alloc(unsigned int nents, gfp_t gfp_mask) |
63 | { |
64 | struct sg_pool *sgp; |
65 | |
66 | sgp = sg_pools + sg_pool_index(nents); |
67 | return mempool_alloc(pool: sgp->pool, gfp_mask); |
68 | } |
69 | |
70 | /** |
71 | * sg_free_table_chained - Free a previously mapped sg table |
72 | * @table: The sg table header to use |
73 | * @nents_first_chunk: size of the first_chunk SGL passed to |
74 | * sg_alloc_table_chained |
75 | * |
76 | * Description: |
77 | * Free an sg table previously allocated and setup with |
78 | * sg_alloc_table_chained(). |
79 | * |
80 | * @nents_first_chunk has to be same with that same parameter passed |
81 | * to sg_alloc_table_chained(). |
82 | * |
83 | **/ |
84 | void sg_free_table_chained(struct sg_table *table, |
85 | unsigned nents_first_chunk) |
86 | { |
87 | if (table->orig_nents <= nents_first_chunk) |
88 | return; |
89 | |
90 | if (nents_first_chunk == 1) |
91 | nents_first_chunk = 0; |
92 | |
93 | __sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free, |
94 | table->orig_nents); |
95 | } |
96 | EXPORT_SYMBOL_GPL(sg_free_table_chained); |
97 | |
98 | /** |
99 | * sg_alloc_table_chained - Allocate and chain SGLs in an sg table |
100 | * @table: The sg table header to use |
101 | * @nents: Number of entries in sg list |
102 | * @first_chunk: first SGL |
103 | * @nents_first_chunk: number of the SGL of @first_chunk |
104 | * |
105 | * Description: |
106 | * Allocate and chain SGLs in an sg table. If @nents@ is larger than |
107 | * @nents_first_chunk a chained sg table will be setup. @first_chunk is |
108 | * ignored if nents_first_chunk <= 1 because user expects the SGL points |
109 | * non-chain SGL. |
110 | * |
111 | **/ |
112 | int sg_alloc_table_chained(struct sg_table *table, int nents, |
113 | struct scatterlist *first_chunk, unsigned nents_first_chunk) |
114 | { |
115 | int ret; |
116 | |
117 | BUG_ON(!nents); |
118 | |
119 | if (first_chunk && nents_first_chunk) { |
120 | if (nents <= nents_first_chunk) { |
121 | table->nents = table->orig_nents = nents; |
122 | sg_init_table(table->sgl, nents); |
123 | return 0; |
124 | } |
125 | } |
126 | |
127 | /* User supposes that the 1st SGL includes real entry */ |
128 | if (nents_first_chunk <= 1) { |
129 | first_chunk = NULL; |
130 | nents_first_chunk = 0; |
131 | } |
132 | |
133 | ret = __sg_alloc_table(table, nents, SG_CHUNK_SIZE, |
134 | first_chunk, nents_first_chunk, |
135 | GFP_ATOMIC, sg_pool_alloc); |
136 | if (unlikely(ret)) |
137 | sg_free_table_chained(table, nents_first_chunk); |
138 | return ret; |
139 | } |
140 | EXPORT_SYMBOL_GPL(sg_alloc_table_chained); |
141 | |
142 | static __init int sg_pool_init(void) |
143 | { |
144 | int i; |
145 | |
146 | for (i = 0; i < SG_MEMPOOL_NR; i++) { |
147 | struct sg_pool *sgp = sg_pools + i; |
148 | int size = sgp->size * sizeof(struct scatterlist); |
149 | |
150 | sgp->slab = kmem_cache_create(name: sgp->name, size, align: 0, |
151 | SLAB_HWCACHE_ALIGN, NULL); |
152 | if (!sgp->slab) { |
153 | printk(KERN_ERR "SG_POOL: can't init sg slab %s\n" , |
154 | sgp->name); |
155 | goto cleanup_sdb; |
156 | } |
157 | |
158 | sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE, |
159 | kc: sgp->slab); |
160 | if (!sgp->pool) { |
161 | printk(KERN_ERR "SG_POOL: can't init sg mempool %s\n" , |
162 | sgp->name); |
163 | goto cleanup_sdb; |
164 | } |
165 | } |
166 | |
167 | return 0; |
168 | |
169 | cleanup_sdb: |
170 | for (i = 0; i < SG_MEMPOOL_NR; i++) { |
171 | struct sg_pool *sgp = sg_pools + i; |
172 | |
173 | mempool_destroy(pool: sgp->pool); |
174 | kmem_cache_destroy(s: sgp->slab); |
175 | } |
176 | |
177 | return -ENOMEM; |
178 | } |
179 | |
180 | subsys_initcall(sg_pool_init); |
181 | |