1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | #ifndef _LINUX_RCUREF_H |
3 | #define _LINUX_RCUREF_H |
4 | |
5 | #include <linux/atomic.h> |
6 | #include <linux/bug.h> |
7 | #include <linux/limits.h> |
8 | #include <linux/lockdep.h> |
9 | #include <linux/preempt.h> |
10 | #include <linux/rcupdate.h> |
11 | |
12 | #define RCUREF_ONEREF 0x00000000U |
13 | #define RCUREF_MAXREF 0x7FFFFFFFU |
14 | #define RCUREF_SATURATED 0xA0000000U |
15 | #define RCUREF_RELEASED 0xC0000000U |
16 | #define RCUREF_DEAD 0xE0000000U |
17 | #define RCUREF_NOREF 0xFFFFFFFFU |
18 | |
19 | /** |
20 | * rcuref_init - Initialize a rcuref reference count with the given reference count |
21 | * @ref: Pointer to the reference count |
22 | * @cnt: The initial reference count typically '1' |
23 | */ |
24 | static inline void rcuref_init(rcuref_t *ref, unsigned int cnt) |
25 | { |
26 | atomic_set(v: &ref->refcnt, i: cnt - 1); |
27 | } |
28 | |
29 | /** |
30 | * rcuref_read - Read the number of held reference counts of a rcuref |
31 | * @ref: Pointer to the reference count |
32 | * |
33 | * Return: The number of held references (0 ... N) |
34 | */ |
35 | static inline unsigned int rcuref_read(rcuref_t *ref) |
36 | { |
37 | unsigned int c = atomic_read(v: &ref->refcnt); |
38 | |
39 | /* Return 0 if within the DEAD zone. */ |
40 | return c >= RCUREF_RELEASED ? 0 : c + 1; |
41 | } |
42 | |
43 | extern __must_check bool rcuref_get_slowpath(rcuref_t *ref); |
44 | |
45 | /** |
46 | * rcuref_get - Acquire one reference on a rcuref reference count |
47 | * @ref: Pointer to the reference count |
48 | * |
49 | * Similar to atomic_inc_not_zero() but saturates at RCUREF_MAXREF. |
50 | * |
51 | * Provides no memory ordering, it is assumed the caller has guaranteed the |
52 | * object memory to be stable (RCU, etc.). It does provide a control dependency |
53 | * and thereby orders future stores. See documentation in lib/rcuref.c |
54 | * |
55 | * Return: |
56 | * False if the attempt to acquire a reference failed. This happens |
57 | * when the last reference has been put already |
58 | * |
59 | * True if a reference was successfully acquired |
60 | */ |
61 | static inline __must_check bool rcuref_get(rcuref_t *ref) |
62 | { |
63 | /* |
64 | * Unconditionally increase the reference count. The saturation and |
65 | * dead zones provide enough tolerance for this. |
66 | */ |
67 | if (likely(!atomic_add_negative_relaxed(1, &ref->refcnt))) |
68 | return true; |
69 | |
70 | /* Handle the cases inside the saturation and dead zones */ |
71 | return rcuref_get_slowpath(ref); |
72 | } |
73 | |
74 | extern __must_check bool rcuref_put_slowpath(rcuref_t *ref); |
75 | |
76 | /* |
77 | * Internal helper. Do not invoke directly. |
78 | */ |
79 | static __always_inline __must_check bool __rcuref_put(rcuref_t *ref) |
80 | { |
81 | RCU_LOCKDEP_WARN(!rcu_read_lock_held() && preemptible(), |
82 | "suspicious rcuref_put_rcusafe() usage" ); |
83 | /* |
84 | * Unconditionally decrease the reference count. The saturation and |
85 | * dead zones provide enough tolerance for this. |
86 | */ |
87 | if (likely(!atomic_add_negative_release(-1, &ref->refcnt))) |
88 | return false; |
89 | |
90 | /* |
91 | * Handle the last reference drop and cases inside the saturation |
92 | * and dead zones. |
93 | */ |
94 | return rcuref_put_slowpath(ref); |
95 | } |
96 | |
97 | /** |
98 | * rcuref_put_rcusafe -- Release one reference for a rcuref reference count RCU safe |
99 | * @ref: Pointer to the reference count |
100 | * |
101 | * Provides release memory ordering, such that prior loads and stores are done |
102 | * before, and provides an acquire ordering on success such that free() |
103 | * must come after. |
104 | * |
105 | * Can be invoked from contexts, which guarantee that no grace period can |
106 | * happen which would free the object concurrently if the decrement drops |
107 | * the last reference and the slowpath races against a concurrent get() and |
108 | * put() pair. rcu_read_lock()'ed and atomic contexts qualify. |
109 | * |
110 | * Return: |
111 | * True if this was the last reference with no future references |
112 | * possible. This signals the caller that it can safely release the |
113 | * object which is protected by the reference counter. |
114 | * |
115 | * False if there are still active references or the put() raced |
116 | * with a concurrent get()/put() pair. Caller is not allowed to |
117 | * release the protected object. |
118 | */ |
119 | static inline __must_check bool rcuref_put_rcusafe(rcuref_t *ref) |
120 | { |
121 | return __rcuref_put(ref); |
122 | } |
123 | |
124 | /** |
125 | * rcuref_put -- Release one reference for a rcuref reference count |
126 | * @ref: Pointer to the reference count |
127 | * |
128 | * Can be invoked from any context. |
129 | * |
130 | * Provides release memory ordering, such that prior loads and stores are done |
131 | * before, and provides an acquire ordering on success such that free() |
132 | * must come after. |
133 | * |
134 | * Return: |
135 | * |
136 | * True if this was the last reference with no future references |
137 | * possible. This signals the caller that it can safely schedule the |
138 | * object, which is protected by the reference counter, for |
139 | * deconstruction. |
140 | * |
141 | * False if there are still active references or the put() raced |
142 | * with a concurrent get()/put() pair. Caller is not allowed to |
143 | * deconstruct the protected object. |
144 | */ |
145 | static inline __must_check bool rcuref_put(rcuref_t *ref) |
146 | { |
147 | bool released; |
148 | |
149 | preempt_disable(); |
150 | released = __rcuref_put(ref); |
151 | preempt_enable(); |
152 | return released; |
153 | } |
154 | |
155 | #endif |
156 | |