1 | /* SPDX-License-Identifier: GPL-2.0+ */ |
2 | /* |
3 | * RCU segmented callback lists, internal-to-rcu header file |
4 | * |
5 | * Copyright IBM Corporation, 2017 |
6 | * |
7 | * Authors: Paul E. McKenney <paulmck@linux.ibm.com> |
8 | */ |
9 | |
10 | #include <linux/rcu_segcblist.h> |
11 | |
12 | /* Return number of callbacks in the specified callback list. */ |
13 | static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp) |
14 | { |
15 | return READ_ONCE(rclp->len); |
16 | } |
17 | |
18 | long rcu_segcblist_get_seglen(struct rcu_segcblist *rsclp, int seg); |
19 | |
20 | /* Return number of callbacks in segmented callback list by summing seglen. */ |
21 | long rcu_segcblist_n_segment_cbs(struct rcu_segcblist *rsclp); |
22 | |
23 | void rcu_cblist_init(struct rcu_cblist *rclp); |
24 | void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp); |
25 | void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, |
26 | struct rcu_cblist *srclp, |
27 | struct rcu_head *rhp); |
28 | struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp); |
29 | |
30 | /* |
31 | * Is the specified rcu_segcblist structure empty? |
32 | * |
33 | * But careful! The fact that the ->head field is NULL does not |
34 | * necessarily imply that there are no callbacks associated with |
35 | * this structure. When callbacks are being invoked, they are |
36 | * removed as a group. If callback invocation must be preempted, |
37 | * the remaining callbacks will be added back to the list. Either |
38 | * way, the counts are updated later. |
39 | * |
40 | * So it is often the case that rcu_segcblist_n_cbs() should be used |
41 | * instead. |
42 | */ |
43 | static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp) |
44 | { |
45 | return !READ_ONCE(rsclp->head); |
46 | } |
47 | |
48 | /* Return number of callbacks in segmented callback list. */ |
49 | static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp) |
50 | { |
51 | #ifdef CONFIG_RCU_NOCB_CPU |
52 | return atomic_long_read(v: &rsclp->len); |
53 | #else |
54 | return READ_ONCE(rsclp->len); |
55 | #endif |
56 | } |
57 | |
58 | static inline void rcu_segcblist_set_flags(struct rcu_segcblist *rsclp, |
59 | int flags) |
60 | { |
61 | WRITE_ONCE(rsclp->flags, rsclp->flags | flags); |
62 | } |
63 | |
64 | static inline void rcu_segcblist_clear_flags(struct rcu_segcblist *rsclp, |
65 | int flags) |
66 | { |
67 | WRITE_ONCE(rsclp->flags, rsclp->flags & ~flags); |
68 | } |
69 | |
70 | static inline bool rcu_segcblist_test_flags(struct rcu_segcblist *rsclp, |
71 | int flags) |
72 | { |
73 | return READ_ONCE(rsclp->flags) & flags; |
74 | } |
75 | |
76 | /* |
77 | * Is the specified rcu_segcblist enabled, for example, not corresponding |
78 | * to an offline CPU? |
79 | */ |
80 | static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp) |
81 | { |
82 | return rcu_segcblist_test_flags(rsclp, SEGCBLIST_ENABLED); |
83 | } |
84 | |
85 | /* |
86 | * Is the specified rcu_segcblist NOCB offloaded (or in the middle of the |
87 | * [de]offloading process)? |
88 | */ |
89 | static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) |
90 | { |
91 | if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) && |
92 | rcu_segcblist_test_flags(rsclp, SEGCBLIST_LOCKING)) |
93 | return true; |
94 | |
95 | return false; |
96 | } |
97 | |
98 | static inline bool rcu_segcblist_completely_offloaded(struct rcu_segcblist *rsclp) |
99 | { |
100 | if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) && |
101 | !rcu_segcblist_test_flags(rsclp, SEGCBLIST_RCU_CORE)) |
102 | return true; |
103 | |
104 | return false; |
105 | } |
106 | |
107 | /* |
108 | * Are all segments following the specified segment of the specified |
109 | * rcu_segcblist structure empty of callbacks? (The specified |
110 | * segment might well contain callbacks.) |
111 | */ |
112 | static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg) |
113 | { |
114 | return !READ_ONCE(*READ_ONCE(rsclp->tails[seg])); |
115 | } |
116 | |
117 | /* |
118 | * Is the specified segment of the specified rcu_segcblist structure |
119 | * empty of callbacks? |
120 | */ |
121 | static inline bool rcu_segcblist_segempty(struct rcu_segcblist *rsclp, int seg) |
122 | { |
123 | if (seg == RCU_DONE_TAIL) |
124 | return &rsclp->head == rsclp->tails[RCU_DONE_TAIL]; |
125 | return rsclp->tails[seg - 1] == rsclp->tails[seg]; |
126 | } |
127 | |
128 | void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp); |
129 | void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v); |
130 | void rcu_segcblist_init(struct rcu_segcblist *rsclp); |
131 | void rcu_segcblist_disable(struct rcu_segcblist *rsclp); |
132 | void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload); |
133 | bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp); |
134 | bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp); |
135 | struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp); |
136 | struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp); |
137 | bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp); |
138 | void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, |
139 | struct rcu_head *rhp); |
140 | bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, |
141 | struct rcu_head *rhp); |
142 | void (struct rcu_segcblist *rsclp, |
143 | struct rcu_cblist *rclp); |
144 | void (struct rcu_segcblist *rsclp, |
145 | struct rcu_cblist *rclp); |
146 | void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp, |
147 | struct rcu_cblist *rclp); |
148 | void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, |
149 | struct rcu_cblist *rclp); |
150 | void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, |
151 | struct rcu_cblist *rclp); |
152 | void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); |
153 | bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); |
154 | void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, |
155 | struct rcu_segcblist *src_rsclp); |
156 | |