1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | #define _GNU_SOURCE /* for program_invocation_short_name */ |
3 | #include <errno.h> |
4 | #include <fcntl.h> |
5 | #include <pthread.h> |
6 | #include <sched.h> |
7 | #include <stdio.h> |
8 | #include <stdlib.h> |
9 | #include <string.h> |
10 | #include <signal.h> |
11 | #include <syscall.h> |
12 | #include <sys/ioctl.h> |
13 | #include <sys/sysinfo.h> |
14 | #include <asm/barrier.h> |
15 | #include <linux/atomic.h> |
16 | #include <linux/rseq.h> |
17 | #include <linux/unistd.h> |
18 | |
19 | #include "kvm_util.h" |
20 | #include "processor.h" |
21 | #include "test_util.h" |
22 | |
23 | #include "../rseq/rseq.c" |
24 | |
25 | /* |
26 | * Any bug related to task migration is likely to be timing-dependent; perform |
27 | * a large number of migrations to reduce the odds of a false negative. |
28 | */ |
29 | #define NR_TASK_MIGRATIONS 100000 |
30 | |
31 | static pthread_t migration_thread; |
32 | static cpu_set_t possible_mask; |
33 | static int min_cpu, max_cpu; |
34 | static bool done; |
35 | |
36 | static atomic_t seq_cnt; |
37 | |
38 | static void guest_code(void) |
39 | { |
40 | for (;;) |
41 | GUEST_SYNC(0); |
42 | } |
43 | |
44 | static int next_cpu(int cpu) |
45 | { |
46 | /* |
47 | * Advance to the next CPU, skipping those that weren't in the original |
48 | * affinity set. Sadly, there is no CPU_SET_FOR_EACH, and cpu_set_t's |
49 | * data storage is considered as opaque. Note, if this task is pinned |
50 | * to a small set of discontigous CPUs, e.g. 2 and 1023, this loop will |
51 | * burn a lot cycles and the test will take longer than normal to |
52 | * complete. |
53 | */ |
54 | do { |
55 | cpu++; |
56 | if (cpu > max_cpu) { |
57 | cpu = min_cpu; |
58 | TEST_ASSERT(CPU_ISSET(cpu, &possible_mask), |
59 | "Min CPU = %d must always be usable" , cpu); |
60 | break; |
61 | } |
62 | } while (!CPU_ISSET(cpu, &possible_mask)); |
63 | |
64 | return cpu; |
65 | } |
66 | |
67 | static void *migration_worker(void *__rseq_tid) |
68 | { |
69 | pid_t rseq_tid = (pid_t)(unsigned long)__rseq_tid; |
70 | cpu_set_t allowed_mask; |
71 | int r, i, cpu; |
72 | |
73 | CPU_ZERO(&allowed_mask); |
74 | |
75 | for (i = 0, cpu = min_cpu; i < NR_TASK_MIGRATIONS; i++, cpu = next_cpu(cpu)) { |
76 | CPU_SET(cpu, &allowed_mask); |
77 | |
78 | /* |
79 | * Bump the sequence count twice to allow the reader to detect |
80 | * that a migration may have occurred in between rseq and sched |
81 | * CPU ID reads. An odd sequence count indicates a migration |
82 | * is in-progress, while a completely different count indicates |
83 | * a migration occurred since the count was last read. |
84 | */ |
85 | atomic_inc(v: &seq_cnt); |
86 | |
87 | /* |
88 | * Ensure the odd count is visible while getcpu() isn't |
89 | * stable, i.e. while changing affinity is in-progress. |
90 | */ |
91 | smp_wmb(); |
92 | r = sched_setaffinity(rseq_tid, sizeof(allowed_mask), &allowed_mask); |
93 | TEST_ASSERT(!r, "sched_setaffinity failed, errno = %d (%s)" , |
94 | errno, strerror(errno)); |
95 | smp_wmb(); |
96 | atomic_inc(v: &seq_cnt); |
97 | |
98 | CPU_CLR(cpu, &allowed_mask); |
99 | |
100 | /* |
101 | * Wait 1-10us before proceeding to the next iteration and more |
102 | * specifically, before bumping seq_cnt again. A delay is |
103 | * needed on three fronts: |
104 | * |
105 | * 1. To allow sched_setaffinity() to prompt migration before |
106 | * ioctl(KVM_RUN) enters the guest so that TIF_NOTIFY_RESUME |
107 | * (or TIF_NEED_RESCHED, which indirectly leads to handling |
108 | * NOTIFY_RESUME) is handled in KVM context. |
109 | * |
110 | * If NOTIFY_RESUME/NEED_RESCHED is set after KVM enters |
111 | * the guest, the guest will trigger a IO/MMIO exit all the |
112 | * way to userspace and the TIF flags will be handled by |
113 | * the generic "exit to userspace" logic, not by KVM. The |
114 | * exit to userspace is necessary to give the test a chance |
115 | * to check the rseq CPU ID (see #2). |
116 | * |
117 | * Alternatively, guest_code() could include an instruction |
118 | * to trigger an exit that is handled by KVM, but any such |
119 | * exit requires architecture specific code. |
120 | * |
121 | * 2. To let ioctl(KVM_RUN) make its way back to the test |
122 | * before the next round of migration. The test's check on |
123 | * the rseq CPU ID must wait for migration to complete in |
124 | * order to avoid false positive, thus any kernel rseq bug |
125 | * will be missed if the next migration starts before the |
126 | * check completes. |
127 | * |
128 | * 3. To ensure the read-side makes efficient forward progress, |
129 | * e.g. if getcpu() involves a syscall. Stalling the read-side |
130 | * means the test will spend more time waiting for getcpu() |
131 | * to stabilize and less time trying to hit the timing-dependent |
132 | * bug. |
133 | * |
134 | * Because any bug in this area is likely to be timing-dependent, |
135 | * run with a range of delays at 1us intervals from 1us to 10us |
136 | * as a best effort to avoid tuning the test to the point where |
137 | * it can hit _only_ the original bug and not detect future |
138 | * regressions. |
139 | * |
140 | * The original bug can reproduce with a delay up to ~500us on |
141 | * x86-64, but starts to require more iterations to reproduce |
142 | * as the delay creeps above ~10us, and the average runtime of |
143 | * each iteration obviously increases as well. Cap the delay |
144 | * at 10us to keep test runtime reasonable while minimizing |
145 | * potential coverage loss. |
146 | * |
147 | * The lower bound for reproducing the bug is likely below 1us, |
148 | * e.g. failures occur on x86-64 with nanosleep(0), but at that |
149 | * point the overhead of the syscall likely dominates the delay. |
150 | * Use usleep() for simplicity and to avoid unnecessary kernel |
151 | * dependencies. |
152 | */ |
153 | usleep((i % 10) + 1); |
154 | } |
155 | done = true; |
156 | return NULL; |
157 | } |
158 | |
159 | static void calc_min_max_cpu(void) |
160 | { |
161 | int i, cnt, nproc; |
162 | |
163 | TEST_REQUIRE(CPU_COUNT(&possible_mask) >= 2); |
164 | |
165 | /* |
166 | * CPU_SET doesn't provide a FOR_EACH helper, get the min/max CPU that |
167 | * this task is affined to in order to reduce the time spent querying |
168 | * unusable CPUs, e.g. if this task is pinned to a small percentage of |
169 | * total CPUs. |
170 | */ |
171 | nproc = get_nprocs_conf(); |
172 | min_cpu = -1; |
173 | max_cpu = -1; |
174 | cnt = 0; |
175 | |
176 | for (i = 0; i < nproc; i++) { |
177 | if (!CPU_ISSET(i, &possible_mask)) |
178 | continue; |
179 | if (min_cpu == -1) |
180 | min_cpu = i; |
181 | max_cpu = i; |
182 | cnt++; |
183 | } |
184 | |
185 | __TEST_REQUIRE(cnt >= 2, |
186 | "Only one usable CPU, task migration not possible" ); |
187 | } |
188 | |
189 | int main(int argc, char *argv[]) |
190 | { |
191 | int r, i, snapshot; |
192 | struct kvm_vm *vm; |
193 | struct kvm_vcpu *vcpu; |
194 | u32 cpu, rseq_cpu; |
195 | |
196 | r = sched_getaffinity(pid: 0, mask: sizeof(possible_mask), &possible_mask); |
197 | TEST_ASSERT(!r, "sched_getaffinity failed, errno = %d (%s)" , errno, |
198 | strerror(errno)); |
199 | |
200 | calc_min_max_cpu(); |
201 | |
202 | r = rseq_register_current_thread(); |
203 | TEST_ASSERT(!r, "rseq_register_current_thread failed, errno = %d (%s)" , |
204 | errno, strerror(errno)); |
205 | |
206 | /* |
207 | * Create and run a dummy VM that immediately exits to userspace via |
208 | * GUEST_SYNC, while concurrently migrating the process by setting its |
209 | * CPU affinity. |
210 | */ |
211 | vm = vm_create_with_one_vcpu(&vcpu, guest_code); |
212 | |
213 | pthread_create(&migration_thread, NULL, migration_worker, |
214 | (void *)(unsigned long)syscall(SYS_gettid)); |
215 | |
216 | for (i = 0; !done; i++) { |
217 | vcpu_run(vcpu); |
218 | TEST_ASSERT(get_ucall(vcpu, NULL) == UCALL_SYNC, |
219 | "Guest failed?" ); |
220 | |
221 | /* |
222 | * Verify rseq's CPU matches sched's CPU. Ensure migration |
223 | * doesn't occur between getcpu() and reading the rseq cpu_id |
224 | * by rereading both if the sequence count changes, or if the |
225 | * count is odd (migration in-progress). |
226 | */ |
227 | do { |
228 | /* |
229 | * Drop bit 0 to force a mismatch if the count is odd, |
230 | * i.e. if a migration is in-progress. |
231 | */ |
232 | snapshot = atomic_read(v: &seq_cnt) & ~1; |
233 | |
234 | /* |
235 | * Ensure calling getcpu() and reading rseq.cpu_id complete |
236 | * in a single "no migration" window, i.e. are not reordered |
237 | * across the seq_cnt reads. |
238 | */ |
239 | smp_rmb(); |
240 | r = sys_getcpu(cpu: &cpu, NULL); |
241 | TEST_ASSERT(!r, "getcpu failed, errno = %d (%s)" , |
242 | errno, strerror(errno)); |
243 | rseq_cpu = rseq_current_cpu_raw(); |
244 | smp_rmb(); |
245 | } while (snapshot != atomic_read(v: &seq_cnt)); |
246 | |
247 | TEST_ASSERT(rseq_cpu == cpu, |
248 | "rseq CPU = %d, sched CPU = %d" , rseq_cpu, cpu); |
249 | } |
250 | |
251 | /* |
252 | * Sanity check that the test was able to enter the guest a reasonable |
253 | * number of times, e.g. didn't get stalled too often/long waiting for |
254 | * getcpu() to stabilize. A 2:1 migration:KVM_RUN ratio is a fairly |
255 | * conservative ratio on x86-64, which can do _more_ KVM_RUNs than |
256 | * migrations given the 1us+ delay in the migration task. |
257 | */ |
258 | TEST_ASSERT(i > (NR_TASK_MIGRATIONS / 2), |
259 | "Only performed %d KVM_RUNs, task stalled too much?" , i); |
260 | |
261 | pthread_join(migration_thread, NULL); |
262 | |
263 | kvm_vm_free(vm); |
264 | |
265 | rseq_unregister_current_thread(); |
266 | |
267 | return 0; |
268 | } |
269 | |