1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* Copyright (c) 2018 Mellanox Technologies */ |
3 | |
4 | #include <linux/jhash.h> |
5 | #include <linux/slab.h> |
6 | #include <linux/xarray.h> |
7 | #include <linux/hashtable.h> |
8 | #include <linux/refcount.h> |
9 | |
10 | #include "mapping.h" |
11 | |
12 | #define MAPPING_GRACE_PERIOD 2000 |
13 | |
14 | static LIST_HEAD(shared_ctx_list); |
15 | static DEFINE_MUTEX(shared_ctx_lock); |
16 | |
17 | struct mapping_ctx { |
18 | struct xarray xarray; |
19 | DECLARE_HASHTABLE(ht, 8); |
20 | struct mutex lock; /* Guards hashtable and xarray */ |
21 | unsigned long max_id; |
22 | size_t data_size; |
23 | bool delayed_removal; |
24 | struct delayed_work dwork; |
25 | struct list_head pending_list; |
26 | spinlock_t pending_list_lock; /* Guards pending list */ |
27 | u64 id; |
28 | u8 type; |
29 | struct list_head list; |
30 | refcount_t refcount; |
31 | }; |
32 | |
33 | struct mapping_item { |
34 | struct rcu_head rcu; |
35 | struct list_head list; |
36 | unsigned long timeout; |
37 | struct hlist_node node; |
38 | int cnt; |
39 | u32 id; |
40 | char data[]; |
41 | }; |
42 | |
43 | int mapping_add(struct mapping_ctx *ctx, void *data, u32 *id) |
44 | { |
45 | struct mapping_item *mi; |
46 | int err = -ENOMEM; |
47 | u32 hash_key; |
48 | |
49 | mutex_lock(&ctx->lock); |
50 | |
51 | hash_key = jhash(key: data, length: ctx->data_size, initval: 0); |
52 | hash_for_each_possible(ctx->ht, mi, node, hash_key) { |
53 | if (!memcmp(p: data, q: mi->data, size: ctx->data_size)) |
54 | goto attach; |
55 | } |
56 | |
57 | mi = kzalloc(size: sizeof(*mi) + ctx->data_size, GFP_KERNEL); |
58 | if (!mi) |
59 | goto err_alloc; |
60 | |
61 | memcpy(mi->data, data, ctx->data_size); |
62 | hash_add(ctx->ht, &mi->node, hash_key); |
63 | |
64 | err = xa_alloc(xa: &ctx->xarray, id: &mi->id, entry: mi, XA_LIMIT(1, ctx->max_id), |
65 | GFP_KERNEL); |
66 | if (err) |
67 | goto err_assign; |
68 | attach: |
69 | ++mi->cnt; |
70 | *id = mi->id; |
71 | |
72 | mutex_unlock(lock: &ctx->lock); |
73 | |
74 | return 0; |
75 | |
76 | err_assign: |
77 | hash_del(node: &mi->node); |
78 | kfree(objp: mi); |
79 | err_alloc: |
80 | mutex_unlock(lock: &ctx->lock); |
81 | |
82 | return err; |
83 | } |
84 | |
85 | static void mapping_remove_and_free(struct mapping_ctx *ctx, |
86 | struct mapping_item *mi) |
87 | { |
88 | xa_erase(&ctx->xarray, index: mi->id); |
89 | kfree_rcu(mi, rcu); |
90 | } |
91 | |
92 | static void mapping_free_item(struct mapping_ctx *ctx, |
93 | struct mapping_item *mi) |
94 | { |
95 | if (!ctx->delayed_removal) { |
96 | mapping_remove_and_free(ctx, mi); |
97 | return; |
98 | } |
99 | |
100 | mi->timeout = jiffies + msecs_to_jiffies(MAPPING_GRACE_PERIOD); |
101 | |
102 | spin_lock(lock: &ctx->pending_list_lock); |
103 | list_add_tail(new: &mi->list, head: &ctx->pending_list); |
104 | spin_unlock(lock: &ctx->pending_list_lock); |
105 | |
106 | schedule_delayed_work(dwork: &ctx->dwork, MAPPING_GRACE_PERIOD); |
107 | } |
108 | |
109 | int mapping_remove(struct mapping_ctx *ctx, u32 id) |
110 | { |
111 | unsigned long index = id; |
112 | struct mapping_item *mi; |
113 | int err = -ENOENT; |
114 | |
115 | mutex_lock(&ctx->lock); |
116 | mi = xa_load(&ctx->xarray, index); |
117 | if (!mi) |
118 | goto out; |
119 | err = 0; |
120 | |
121 | if (--mi->cnt > 0) |
122 | goto out; |
123 | |
124 | hash_del(node: &mi->node); |
125 | mapping_free_item(ctx, mi); |
126 | out: |
127 | mutex_unlock(lock: &ctx->lock); |
128 | |
129 | return err; |
130 | } |
131 | |
132 | int mapping_find(struct mapping_ctx *ctx, u32 id, void *data) |
133 | { |
134 | unsigned long index = id; |
135 | struct mapping_item *mi; |
136 | int err = -ENOENT; |
137 | |
138 | rcu_read_lock(); |
139 | mi = xa_load(&ctx->xarray, index); |
140 | if (!mi) |
141 | goto err_find; |
142 | |
143 | memcpy(data, mi->data, ctx->data_size); |
144 | err = 0; |
145 | |
146 | err_find: |
147 | rcu_read_unlock(); |
148 | return err; |
149 | } |
150 | |
151 | static void |
152 | mapping_remove_and_free_list(struct mapping_ctx *ctx, struct list_head *list) |
153 | { |
154 | struct mapping_item *mi; |
155 | |
156 | list_for_each_entry(mi, list, list) |
157 | mapping_remove_and_free(ctx, mi); |
158 | } |
159 | |
160 | static void mapping_work_handler(struct work_struct *work) |
161 | { |
162 | unsigned long min_timeout = 0, now = jiffies; |
163 | struct mapping_item *mi, *next; |
164 | LIST_HEAD(pending_items); |
165 | struct mapping_ctx *ctx; |
166 | |
167 | ctx = container_of(work, struct mapping_ctx, dwork.work); |
168 | |
169 | spin_lock(lock: &ctx->pending_list_lock); |
170 | list_for_each_entry_safe(mi, next, &ctx->pending_list, list) { |
171 | if (time_after(now, mi->timeout)) |
172 | list_move(list: &mi->list, head: &pending_items); |
173 | else if (!min_timeout || |
174 | time_before(mi->timeout, min_timeout)) |
175 | min_timeout = mi->timeout; |
176 | } |
177 | spin_unlock(lock: &ctx->pending_list_lock); |
178 | |
179 | mapping_remove_and_free_list(ctx, list: &pending_items); |
180 | |
181 | if (min_timeout) |
182 | schedule_delayed_work(dwork: &ctx->dwork, abs(min_timeout - now)); |
183 | } |
184 | |
185 | static void mapping_flush_work(struct mapping_ctx *ctx) |
186 | { |
187 | if (!ctx->delayed_removal) |
188 | return; |
189 | |
190 | cancel_delayed_work_sync(dwork: &ctx->dwork); |
191 | mapping_remove_and_free_list(ctx, list: &ctx->pending_list); |
192 | } |
193 | |
194 | struct mapping_ctx * |
195 | mapping_create(size_t data_size, u32 max_id, bool delayed_removal) |
196 | { |
197 | struct mapping_ctx *ctx; |
198 | |
199 | ctx = kzalloc(size: sizeof(*ctx), GFP_KERNEL); |
200 | if (!ctx) |
201 | return ERR_PTR(error: -ENOMEM); |
202 | |
203 | ctx->max_id = max_id ? max_id : UINT_MAX; |
204 | ctx->data_size = data_size; |
205 | |
206 | if (delayed_removal) { |
207 | INIT_DELAYED_WORK(&ctx->dwork, mapping_work_handler); |
208 | INIT_LIST_HEAD(list: &ctx->pending_list); |
209 | spin_lock_init(&ctx->pending_list_lock); |
210 | ctx->delayed_removal = true; |
211 | } |
212 | |
213 | mutex_init(&ctx->lock); |
214 | xa_init_flags(xa: &ctx->xarray, XA_FLAGS_ALLOC1); |
215 | |
216 | refcount_set(r: &ctx->refcount, n: 1); |
217 | INIT_LIST_HEAD(list: &ctx->list); |
218 | |
219 | return ctx; |
220 | } |
221 | |
222 | struct mapping_ctx * |
223 | mapping_create_for_id(u64 id, u8 type, size_t data_size, u32 max_id, bool delayed_removal) |
224 | { |
225 | struct mapping_ctx *ctx; |
226 | |
227 | mutex_lock(&shared_ctx_lock); |
228 | list_for_each_entry(ctx, &shared_ctx_list, list) { |
229 | if (ctx->id == id && ctx->type == type) { |
230 | if (refcount_inc_not_zero(r: &ctx->refcount)) |
231 | goto unlock; |
232 | break; |
233 | } |
234 | } |
235 | |
236 | ctx = mapping_create(data_size, max_id, delayed_removal); |
237 | if (IS_ERR(ptr: ctx)) |
238 | goto unlock; |
239 | |
240 | ctx->id = id; |
241 | ctx->type = type; |
242 | list_add(new: &ctx->list, head: &shared_ctx_list); |
243 | |
244 | unlock: |
245 | mutex_unlock(lock: &shared_ctx_lock); |
246 | return ctx; |
247 | } |
248 | |
249 | void mapping_destroy(struct mapping_ctx *ctx) |
250 | { |
251 | if (!refcount_dec_and_test(r: &ctx->refcount)) |
252 | return; |
253 | |
254 | mutex_lock(&shared_ctx_lock); |
255 | list_del(entry: &ctx->list); |
256 | mutex_unlock(lock: &shared_ctx_lock); |
257 | |
258 | mapping_flush_work(ctx); |
259 | xa_destroy(&ctx->xarray); |
260 | mutex_destroy(lock: &ctx->lock); |
261 | |
262 | kfree(objp: ctx); |
263 | } |
264 | |