1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _LINUX_PERCPU_RWSEM_H |
3 | #define _LINUX_PERCPU_RWSEM_H |
4 | |
5 | #include <linux/atomic.h> |
6 | #include <linux/percpu.h> |
7 | #include <linux/rcuwait.h> |
8 | #include <linux/wait.h> |
9 | #include <linux/rcu_sync.h> |
10 | #include <linux/lockdep.h> |
11 | |
12 | struct percpu_rw_semaphore { |
13 | struct rcu_sync ; |
14 | unsigned int __percpu *read_count; |
15 | struct rcuwait writer; |
16 | wait_queue_head_t waiters; |
17 | atomic_t block; |
18 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
19 | struct lockdep_map dep_map; |
20 | #endif |
21 | }; |
22 | |
23 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
24 | #define __PERCPU_RWSEM_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname }, |
25 | #else |
26 | #define __PERCPU_RWSEM_DEP_MAP_INIT(lockname) |
27 | #endif |
28 | |
29 | #define __DEFINE_PERCPU_RWSEM(name, is_static) \ |
30 | static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_rc_##name); \ |
31 | is_static struct percpu_rw_semaphore name = { \ |
32 | .rss = __RCU_SYNC_INITIALIZER(name.rss), \ |
33 | .read_count = &__percpu_rwsem_rc_##name, \ |
34 | .writer = __RCUWAIT_INITIALIZER(name.writer), \ |
35 | .waiters = __WAIT_QUEUE_HEAD_INITIALIZER(name.waiters), \ |
36 | .block = ATOMIC_INIT(0), \ |
37 | __PERCPU_RWSEM_DEP_MAP_INIT(name) \ |
38 | } |
39 | |
40 | #define DEFINE_PERCPU_RWSEM(name) \ |
41 | __DEFINE_PERCPU_RWSEM(name, /* not static */) |
42 | #define DEFINE_STATIC_PERCPU_RWSEM(name) \ |
43 | __DEFINE_PERCPU_RWSEM(name, static) |
44 | |
45 | extern bool __percpu_down_read(struct percpu_rw_semaphore *, bool); |
46 | |
47 | static inline void percpu_down_read(struct percpu_rw_semaphore *sem) |
48 | { |
49 | might_sleep(); |
50 | |
51 | rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); |
52 | |
53 | preempt_disable(); |
54 | /* |
55 | * We are in an RCU-sched read-side critical section, so the writer |
56 | * cannot both change sem->state from readers_fast and start checking |
57 | * counters while we are here. So if we see !sem->state, we know that |
58 | * the writer won't be checking until we're past the preempt_enable() |
59 | * and that once the synchronize_rcu() is done, the writer will see |
60 | * anything we did within this RCU-sched read-size critical section. |
61 | */ |
62 | if (likely(rcu_sync_is_idle(&sem->rss))) |
63 | this_cpu_inc(*sem->read_count); |
64 | else |
65 | __percpu_down_read(sem, false); /* Unconditional memory barrier */ |
66 | /* |
67 | * The preempt_enable() prevents the compiler from |
68 | * bleeding the critical section out. |
69 | */ |
70 | preempt_enable(); |
71 | } |
72 | |
73 | static inline bool percpu_down_read_trylock(struct percpu_rw_semaphore *sem) |
74 | { |
75 | bool ret = true; |
76 | |
77 | preempt_disable(); |
78 | /* |
79 | * Same as in percpu_down_read(). |
80 | */ |
81 | if (likely(rcu_sync_is_idle(&sem->rss))) |
82 | this_cpu_inc(*sem->read_count); |
83 | else |
84 | ret = __percpu_down_read(sem, true); /* Unconditional memory barrier */ |
85 | preempt_enable(); |
86 | /* |
87 | * The barrier() from preempt_enable() prevents the compiler from |
88 | * bleeding the critical section out. |
89 | */ |
90 | |
91 | if (ret) |
92 | rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_); |
93 | |
94 | return ret; |
95 | } |
96 | |
97 | static inline void percpu_up_read(struct percpu_rw_semaphore *sem) |
98 | { |
99 | rwsem_release(&sem->dep_map, _RET_IP_); |
100 | |
101 | preempt_disable(); |
102 | /* |
103 | * Same as in percpu_down_read(). |
104 | */ |
105 | if (likely(rcu_sync_is_idle(&sem->rss))) { |
106 | this_cpu_dec(*sem->read_count); |
107 | } else { |
108 | /* |
109 | * slowpath; reader will only ever wake a single blocked |
110 | * writer. |
111 | */ |
112 | smp_mb(); /* B matches C */ |
113 | /* |
114 | * In other words, if they see our decrement (presumably to |
115 | * aggregate zero, as that is the only time it matters) they |
116 | * will also see our critical section. |
117 | */ |
118 | this_cpu_dec(*sem->read_count); |
119 | rcuwait_wake_up(w: &sem->writer); |
120 | } |
121 | preempt_enable(); |
122 | } |
123 | |
124 | extern bool percpu_is_read_locked(struct percpu_rw_semaphore *); |
125 | extern void percpu_down_write(struct percpu_rw_semaphore *); |
126 | extern void percpu_up_write(struct percpu_rw_semaphore *); |
127 | |
128 | static inline bool percpu_is_write_locked(struct percpu_rw_semaphore *sem) |
129 | { |
130 | return atomic_read(v: &sem->block); |
131 | } |
132 | |
133 | extern int __percpu_init_rwsem(struct percpu_rw_semaphore *, |
134 | const char *, struct lock_class_key *); |
135 | |
136 | extern void percpu_free_rwsem(struct percpu_rw_semaphore *); |
137 | |
138 | #define percpu_init_rwsem(sem) \ |
139 | ({ \ |
140 | static struct lock_class_key rwsem_key; \ |
141 | __percpu_init_rwsem(sem, #sem, &rwsem_key); \ |
142 | }) |
143 | |
144 | #define percpu_rwsem_is_held(sem) lockdep_is_held(sem) |
145 | #define percpu_rwsem_assert_held(sem) lockdep_assert_held(sem) |
146 | |
147 | static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem, |
148 | bool read, unsigned long ip) |
149 | { |
150 | lock_release(lock: &sem->dep_map, ip); |
151 | } |
152 | |
153 | static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem, |
154 | bool read, unsigned long ip) |
155 | { |
156 | lock_acquire(lock: &sem->dep_map, subclass: 0, trylock: 1, read, check: 1, NULL, ip); |
157 | } |
158 | |
159 | #endif |
160 | |