1/* Copyright (C) 2002-2024 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3
4 The GNU C Library is free software; you can redistribute it and/or
5 modify it under the terms of the GNU Lesser General Public
6 License as published by the Free Software Foundation; either
7 version 2.1 of the License, or (at your option) any later version.
8
9 The GNU C Library is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 Lesser General Public License for more details.
13
14 You should have received a copy of the GNU Lesser General Public
15 License along with the GNU C Library; if not, see
16 <https://www.gnu.org/licenses/>. */
17
18#include <assert.h>
19#include <errno.h>
20#include <stdlib.h>
21#include "pthreadP.h"
22#include <lowlevellock.h>
23#include <futex-internal.h>
24
25int
26___pthread_mutex_trylock (pthread_mutex_t *mutex)
27{
28 int oldval;
29 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
30
31 /* See concurrency notes regarding mutex type which is loaded from __kind
32 in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h. */
33 switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex),
34 PTHREAD_MUTEX_TIMED_NP))
35 {
36 /* Recursive mutex. */
37 case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP:
38 case PTHREAD_MUTEX_RECURSIVE_NP:
39 /* Check whether we already hold the mutex. */
40 if (mutex->__data.__owner == id)
41 {
42 /* Just bump the counter. */
43 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
44 /* Overflow of the counter. */
45 return EAGAIN;
46
47 ++mutex->__data.__count;
48 return 0;
49 }
50
51 if (atomic_load_relaxed (&(mutex->__data.__lock)) == 0
52 && lll_trylock (mutex->__data.__lock) == 0)
53 {
54 /* Record the ownership. */
55 mutex->__data.__owner = id;
56 mutex->__data.__count = 1;
57 ++mutex->__data.__nusers;
58 return 0;
59 }
60 break;
61
62 case PTHREAD_MUTEX_TIMED_ELISION_NP:
63 elision: __attribute__((unused))
64 if (lll_trylock_elision (mutex->__data.__lock,
65 mutex->__data.__elision) != 0)
66 break;
67 /* Don't record the ownership. */
68 return 0;
69
70 case PTHREAD_MUTEX_TIMED_NP:
71 FORCE_ELISION (mutex, goto elision);
72 /*FALL THROUGH*/
73 case PTHREAD_MUTEX_ADAPTIVE_NP:
74 case PTHREAD_MUTEX_ERRORCHECK_NP:
75 /* Mutex type is already loaded, lock check overhead should
76 be minimal. */
77 if (atomic_load_relaxed (&(mutex->__data.__lock)) != 0
78 || lll_trylock (mutex->__data.__lock) != 0)
79 break;
80
81 /* Record the ownership. */
82 mutex->__data.__owner = id;
83 ++mutex->__data.__nusers;
84
85 return 0;
86
87 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
88 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
89 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
90 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
91 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
92 &mutex->__data.__list.__next);
93 /* We need to set op_pending before starting the operation. Also
94 see comments at ENQUEUE_MUTEX. */
95 __asm ("" ::: "memory");
96
97 oldval = mutex->__data.__lock;
98 do
99 {
100 again:
101 if ((oldval & FUTEX_OWNER_DIED) != 0)
102 {
103 /* The previous owner died. Try locking the mutex. */
104 int newval = id | (oldval & FUTEX_WAITERS);
105
106 newval
107 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
108 newval, oldval);
109
110 if (newval != oldval)
111 {
112 oldval = newval;
113 goto again;
114 }
115
116 /* We got the mutex. */
117 mutex->__data.__count = 1;
118 /* But it is inconsistent unless marked otherwise. */
119 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
120
121 /* We must not enqueue the mutex before we have acquired it.
122 Also see comments at ENQUEUE_MUTEX. */
123 __asm ("" ::: "memory");
124 ENQUEUE_MUTEX (mutex);
125 /* We need to clear op_pending after we enqueue the mutex. */
126 __asm ("" ::: "memory");
127 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
128
129 /* Note that we deliberately exit here. If we fall
130 through to the end of the function __nusers would be
131 incremented which is not correct because the old
132 owner has to be discounted. */
133 return EOWNERDEAD;
134 }
135
136 /* Check whether we already hold the mutex. */
137 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
138 {
139 int kind = PTHREAD_MUTEX_TYPE (mutex);
140 if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
141 {
142 /* We do not need to ensure ordering wrt another memory
143 access. Also see comments at ENQUEUE_MUTEX. */
144 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
145 NULL);
146 return EDEADLK;
147 }
148
149 if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
150 {
151 /* We do not need to ensure ordering wrt another memory
152 access. */
153 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
154 NULL);
155
156 /* Just bump the counter. */
157 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
158 /* Overflow of the counter. */
159 return EAGAIN;
160
161 ++mutex->__data.__count;
162
163 return 0;
164 }
165 }
166
167 oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
168 id, 0);
169 if (oldval != 0 && (oldval & FUTEX_OWNER_DIED) == 0)
170 {
171 /* We haven't acquired the lock as it is already acquired by
172 another owner. We do not need to ensure ordering wrt another
173 memory access. */
174 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
175
176 return EBUSY;
177 }
178
179 if (__builtin_expect (mutex->__data.__owner
180 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
181 {
182 /* This mutex is now not recoverable. */
183 mutex->__data.__count = 0;
184 if (oldval == id)
185 lll_unlock (mutex->__data.__lock,
186 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
187 /* FIXME This violates the mutex destruction requirements. See
188 __pthread_mutex_unlock_full. */
189 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
190 return ENOTRECOVERABLE;
191 }
192 }
193 while ((oldval & FUTEX_OWNER_DIED) != 0);
194
195 /* We must not enqueue the mutex before we have acquired it.
196 Also see comments at ENQUEUE_MUTEX. */
197 __asm ("" ::: "memory");
198 ENQUEUE_MUTEX (mutex);
199 /* We need to clear op_pending after we enqueue the mutex. */
200 __asm ("" ::: "memory");
201 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
202
203 mutex->__data.__owner = id;
204 ++mutex->__data.__nusers;
205 mutex->__data.__count = 1;
206
207 return 0;
208
209 /* The PI support requires the Linux futex system call. If that's not
210 available, pthread_mutex_init should never have allowed the type to
211 be set. So it will get the default case for an invalid type. */
212#ifdef __NR_futex
213 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
214 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
215 case PTHREAD_MUTEX_PI_NORMAL_NP:
216 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
217 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
218 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
219 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
220 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
221 {
222 int kind, robust;
223 {
224 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
225 in sysdeps/nptl/bits/thread-shared-types.h. */
226 int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind));
227 kind = mutex_kind & PTHREAD_MUTEX_KIND_MASK_NP;
228 robust = mutex_kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
229 }
230
231 if (robust)
232 {
233 /* Note: robust PI futexes are signaled by setting bit 0. */
234 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
235 (void *) (((uintptr_t) &mutex->__data.__list.__next)
236 | 1));
237 /* We need to set op_pending before starting the operation. Also
238 see comments at ENQUEUE_MUTEX. */
239 __asm ("" ::: "memory");
240 }
241
242 oldval = mutex->__data.__lock;
243
244 /* Check whether we already hold the mutex. */
245 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
246 {
247 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
248 {
249 /* We do not need to ensure ordering wrt another memory
250 access. */
251 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
252 return EDEADLK;
253 }
254
255 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
256 {
257 /* We do not need to ensure ordering wrt another memory
258 access. */
259 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
260
261 /* Just bump the counter. */
262 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
263 /* Overflow of the counter. */
264 return EAGAIN;
265
266 ++mutex->__data.__count;
267
268 return 0;
269 }
270 }
271
272 oldval
273 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
274 id, 0);
275
276 if (oldval != 0)
277 {
278 if ((oldval & FUTEX_OWNER_DIED) == 0)
279 {
280 /* We haven't acquired the lock as it is already acquired by
281 another owner. We do not need to ensure ordering wrt another
282 memory access. */
283 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
284
285 return EBUSY;
286 }
287
288 assert (robust);
289
290 /* The mutex owner died. The kernel will now take care of
291 everything. */
292 int private = (robust
293 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
294 : PTHREAD_MUTEX_PSHARED (mutex));
295 int e = INTERNAL_SYSCALL_CALL (futex, &mutex->__data.__lock,
296 __lll_private_flag (FUTEX_TRYLOCK_PI,
297 private), 0, 0);
298
299 if (INTERNAL_SYSCALL_ERROR_P (e)
300 && INTERNAL_SYSCALL_ERRNO (e) == EWOULDBLOCK)
301 {
302 /* The kernel has not yet finished the mutex owner death.
303 We do not need to ensure ordering wrt another memory
304 access. */
305 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
306
307 return EBUSY;
308 }
309
310 oldval = mutex->__data.__lock;
311 }
312
313 if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
314 {
315 atomic_fetch_and_acquire (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
316
317 /* We got the mutex. */
318 mutex->__data.__count = 1;
319 /* But it is inconsistent unless marked otherwise. */
320 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
321
322 /* We must not enqueue the mutex before we have acquired it.
323 Also see comments at ENQUEUE_MUTEX. */
324 __asm ("" ::: "memory");
325 ENQUEUE_MUTEX (mutex);
326 /* We need to clear op_pending after we enqueue the mutex. */
327 __asm ("" ::: "memory");
328 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
329
330 /* Note that we deliberately exit here. If we fall
331 through to the end of the function __nusers would be
332 incremented which is not correct because the old owner
333 has to be discounted. */
334 return EOWNERDEAD;
335 }
336
337 if (robust
338 && __builtin_expect (mutex->__data.__owner
339 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
340 {
341 /* This mutex is now not recoverable. */
342 mutex->__data.__count = 0;
343
344 futex_unlock_pi (futex_word: (unsigned int *) &mutex->__data.__lock,
345 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
346
347 /* To the kernel, this will be visible after the kernel has
348 acquired the mutex in the syscall. */
349 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
350 return ENOTRECOVERABLE;
351 }
352
353 if (robust)
354 {
355 /* We must not enqueue the mutex before we have acquired it.
356 Also see comments at ENQUEUE_MUTEX. */
357 __asm ("" ::: "memory");
358 ENQUEUE_MUTEX_PI (mutex);
359 /* We need to clear op_pending after we enqueue the mutex. */
360 __asm ("" ::: "memory");
361 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
362 }
363
364 mutex->__data.__owner = id;
365 ++mutex->__data.__nusers;
366 mutex->__data.__count = 1;
367
368 return 0;
369 }
370#endif /* __NR_futex. */
371
372 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
373 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
374 case PTHREAD_MUTEX_PP_NORMAL_NP:
375 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
376 {
377 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
378 in sysdeps/nptl/bits/thread-shared-types.h. */
379 int kind = atomic_load_relaxed (&(mutex->__data.__kind))
380 & PTHREAD_MUTEX_KIND_MASK_NP;
381
382 oldval = mutex->__data.__lock;
383
384 /* Check whether we already hold the mutex. */
385 if (mutex->__data.__owner == id)
386 {
387 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
388 return EDEADLK;
389
390 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
391 {
392 /* Just bump the counter. */
393 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
394 /* Overflow of the counter. */
395 return EAGAIN;
396
397 ++mutex->__data.__count;
398
399 return 0;
400 }
401 }
402
403 int oldprio = -1, ceilval;
404 do
405 {
406 int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
407 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
408
409 if (__pthread_current_priority () > ceiling)
410 {
411 if (oldprio != -1)
412 __pthread_tpp_change_priority (oldprio, -1);
413 return EINVAL;
414 }
415
416 int retval = __pthread_tpp_change_priority (oldprio, ceiling);
417 if (retval)
418 return retval;
419
420 ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
421 oldprio = ceiling;
422
423 oldval
424 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
425 ceilval | 1, ceilval);
426
427 if (oldval == ceilval)
428 break;
429 }
430 while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
431
432 if (oldval != ceilval)
433 {
434 __pthread_tpp_change_priority (oldprio, -1);
435 break;
436 }
437
438 assert (mutex->__data.__owner == 0);
439 /* Record the ownership. */
440 mutex->__data.__owner = id;
441 ++mutex->__data.__nusers;
442 mutex->__data.__count = 1;
443
444 return 0;
445 }
446 break;
447
448 default:
449 /* Correct code cannot set any other type. */
450 return EINVAL;
451 }
452
453 return EBUSY;
454}
455versioned_symbol (libc, ___pthread_mutex_trylock,
456 pthread_mutex_trylock, GLIBC_2_34);
457libc_hidden_ver (___pthread_mutex_trylock, __pthread_mutex_trylock)
458#ifndef SHARED
459strong_alias (___pthread_mutex_trylock, __pthread_mutex_trylock)
460#endif
461
462#if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_0, GLIBC_2_34)
463compat_symbol (libpthread, ___pthread_mutex_trylock,
464 pthread_mutex_trylock, GLIBC_2_0);
465compat_symbol (libpthread, ___pthread_mutex_trylock,
466 __pthread_mutex_trylock, GLIBC_2_0);
467#endif
468

source code of glibc/nptl/pthread_mutex_trylock.c