1 | //===- unittests/DirectoryWatcher/DirectoryWatcherTest.cpp ----------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "clang/DirectoryWatcher/DirectoryWatcher.h" |
10 | #include "llvm/Support/FileSystem.h" |
11 | #include "llvm/Support/Path.h" |
12 | #include "llvm/Support/raw_ostream.h" |
13 | #include "llvm/Testing/Support/Error.h" |
14 | #include "gtest/gtest.h" |
15 | #include <condition_variable> |
16 | #include <future> |
17 | #include <mutex> |
18 | #include <optional> |
19 | #include <thread> |
20 | |
21 | using namespace llvm; |
22 | using namespace llvm::sys; |
23 | using namespace llvm::sys::fs; |
24 | using namespace clang; |
25 | |
26 | namespace clang { |
27 | static bool operator==(const DirectoryWatcher::Event &lhs, |
28 | const DirectoryWatcher::Event &rhs) { |
29 | return lhs.Filename == rhs.Filename && |
30 | static_cast<int>(lhs.Kind) == static_cast<int>(rhs.Kind); |
31 | } |
32 | } // namespace clang |
33 | |
34 | namespace { |
35 | |
36 | typedef DirectoryWatcher::Event::EventKind EventKind; |
37 | |
38 | // We've observed this test being significantly flaky when running on a heavily |
39 | // loaded machine (e.g. when it's being run as part of the full check-clang |
40 | // suite). Set a high timeout value to avoid this flakiness. The 60s timeout |
41 | // value was determined empirically. It's a timeout value, not a sleep value, |
42 | // and the test should require much less time in practice the vast majority of |
43 | // instances. The cases where we do come close to (or still end up hitting) the |
44 | // longer timeout are most likely to occur when other tests are also running at |
45 | // the same time (e.g. as part of the full check-clang suite), in which case the |
46 | // latency of the timeout will be masked by the latency of the other tests. |
47 | constexpr std::chrono::seconds EventualResultTimeout(60); |
48 | |
49 | struct DirectoryWatcherTestFixture { |
50 | std::string TestRootDir; |
51 | std::string TestWatchedDir; |
52 | |
53 | DirectoryWatcherTestFixture() { |
54 | SmallString<128> pathBuf; |
55 | #ifndef NDEBUG |
56 | std::error_code UniqDirRes = |
57 | #endif |
58 | createUniqueDirectory(Prefix: "dirwatcher" , ResultPath&: pathBuf); |
59 | assert(!UniqDirRes); |
60 | TestRootDir = std::string(pathBuf.str()); |
61 | path::append(path&: pathBuf, a: "watch" ); |
62 | TestWatchedDir = std::string(pathBuf.str()); |
63 | #ifndef NDEBUG |
64 | std::error_code CreateDirRes = |
65 | #endif |
66 | create_directory(path: TestWatchedDir, IgnoreExisting: false); |
67 | assert(!CreateDirRes); |
68 | } |
69 | |
70 | ~DirectoryWatcherTestFixture() { remove_directories(path: TestRootDir); } |
71 | |
72 | SmallString<128> getPathInWatched(const std::string &testFile) { |
73 | SmallString<128> pathBuf; |
74 | pathBuf = TestWatchedDir; |
75 | path::append(path&: pathBuf, a: testFile); |
76 | return pathBuf; |
77 | } |
78 | |
79 | void addFile(const std::string &testFile) { |
80 | Expected<file_t> ft = openNativeFileForWrite(Name: getPathInWatched(testFile), |
81 | Disp: CD_CreateNew, Flags: OF_None); |
82 | if (ft) { |
83 | closeFile(F&: *ft); |
84 | } else { |
85 | llvm::errs() << llvm::toString(E: ft.takeError()) << "\n" ; |
86 | llvm::errs() << getPathInWatched(testFile) << "\n" ; |
87 | llvm_unreachable("Couldn't create test file." ); |
88 | } |
89 | } |
90 | |
91 | void deleteFile(const std::string &testFile) { |
92 | std::error_code EC = |
93 | remove(path: getPathInWatched(testFile), /*IgnoreNonExisting=*/false); |
94 | ASSERT_FALSE(EC); |
95 | } |
96 | }; |
97 | |
98 | std::string eventKindToString(const EventKind K) { |
99 | switch (K) { |
100 | case EventKind::Removed: |
101 | return "Removed" ; |
102 | case EventKind::Modified: |
103 | return "Modified" ; |
104 | case EventKind::WatchedDirRemoved: |
105 | return "WatchedDirRemoved" ; |
106 | case EventKind::WatcherGotInvalidated: |
107 | return "WatcherGotInvalidated" ; |
108 | } |
109 | llvm_unreachable("unknown event kind" ); |
110 | } |
111 | |
112 | struct VerifyingConsumer { |
113 | std::vector<DirectoryWatcher::Event> ExpectedInitial; |
114 | const std::vector<DirectoryWatcher::Event> ExpectedInitialCopy; |
115 | std::vector<DirectoryWatcher::Event> ExpectedNonInitial; |
116 | const std::vector<DirectoryWatcher::Event> ExpectedNonInitialCopy; |
117 | std::vector<DirectoryWatcher::Event> OptionalNonInitial; |
118 | std::vector<DirectoryWatcher::Event> UnexpectedInitial; |
119 | std::vector<DirectoryWatcher::Event> UnexpectedNonInitial; |
120 | std::mutex Mtx; |
121 | std::condition_variable ResultIsReady; |
122 | |
123 | VerifyingConsumer( |
124 | const std::vector<DirectoryWatcher::Event> &ExpectedInitial, |
125 | const std::vector<DirectoryWatcher::Event> &ExpectedNonInitial, |
126 | const std::vector<DirectoryWatcher::Event> &OptionalNonInitial = {}) |
127 | : ExpectedInitial(ExpectedInitial), ExpectedInitialCopy(ExpectedInitial), |
128 | ExpectedNonInitial(ExpectedNonInitial), ExpectedNonInitialCopy(ExpectedNonInitial), |
129 | OptionalNonInitial(OptionalNonInitial) {} |
130 | |
131 | // This method is used by DirectoryWatcher. |
132 | void consume(DirectoryWatcher::Event E, bool IsInitial) { |
133 | if (IsInitial) |
134 | consumeInitial(E); |
135 | else |
136 | consumeNonInitial(E); |
137 | } |
138 | |
139 | void consumeInitial(DirectoryWatcher::Event E) { |
140 | std::unique_lock<std::mutex> L(Mtx); |
141 | auto It = llvm::find(Range&: ExpectedInitial, Val: E); |
142 | if (It == ExpectedInitial.end()) { |
143 | UnexpectedInitial.push_back(x: E); |
144 | } else { |
145 | ExpectedInitial.erase(position: It); |
146 | } |
147 | if (result()) { |
148 | L.unlock(); |
149 | ResultIsReady.notify_one(); |
150 | } |
151 | } |
152 | |
153 | void consumeNonInitial(DirectoryWatcher::Event E) { |
154 | std::unique_lock<std::mutex> L(Mtx); |
155 | auto It = llvm::find(Range&: ExpectedNonInitial, Val: E); |
156 | if (It == ExpectedNonInitial.end()) { |
157 | auto OptIt = llvm::find(Range&: OptionalNonInitial, Val: E); |
158 | if (OptIt != OptionalNonInitial.end()) { |
159 | OptionalNonInitial.erase(position: OptIt); |
160 | } else { |
161 | UnexpectedNonInitial.push_back(x: E); |
162 | } |
163 | } else { |
164 | ExpectedNonInitial.erase(position: It); |
165 | } |
166 | if (result()) { |
167 | L.unlock(); |
168 | ResultIsReady.notify_one(); |
169 | } |
170 | } |
171 | |
172 | // This method is used by DirectoryWatcher. |
173 | void consume(llvm::ArrayRef<DirectoryWatcher::Event> Es, bool IsInitial) { |
174 | for (const auto &E : Es) |
175 | consume(E, IsInitial); |
176 | } |
177 | |
178 | // Not locking - caller has to lock Mtx. |
179 | std::optional<bool> result() const { |
180 | if (ExpectedInitial.empty() && ExpectedNonInitial.empty() && |
181 | UnexpectedInitial.empty() && UnexpectedNonInitial.empty()) |
182 | return true; |
183 | if (!UnexpectedInitial.empty() || !UnexpectedNonInitial.empty()) |
184 | return false; |
185 | return std::nullopt; |
186 | } |
187 | |
188 | // This method is used by tests. |
189 | // \returns true on success |
190 | bool blockUntilResult() { |
191 | std::unique_lock<std::mutex> L(Mtx); |
192 | while (true) { |
193 | if (result()) |
194 | return *result(); |
195 | |
196 | ResultIsReady.wait(lock&: L, p: [this]() { return result().has_value(); }); |
197 | } |
198 | return false; // Just to make compiler happy. |
199 | } |
200 | |
201 | void printUnmetExpectations(llvm::raw_ostream &OS) { |
202 | // If there was any issue, print the expected state |
203 | if ( |
204 | !ExpectedInitial.empty() |
205 | || |
206 | !ExpectedNonInitial.empty() |
207 | || |
208 | !UnexpectedInitial.empty() |
209 | || |
210 | !UnexpectedNonInitial.empty() |
211 | ) { |
212 | OS << "Expected initial events: \n" ; |
213 | for (const auto &E : ExpectedInitialCopy) { |
214 | OS << eventKindToString(K: E.Kind) << " " << E.Filename << "\n" ; |
215 | } |
216 | OS << "Expected non-initial events: \n" ; |
217 | for (const auto &E : ExpectedNonInitialCopy) { |
218 | OS << eventKindToString(K: E.Kind) << " " << E.Filename << "\n" ; |
219 | } |
220 | } |
221 | |
222 | if (!ExpectedInitial.empty()) { |
223 | OS << "Expected but not seen initial events: \n" ; |
224 | for (const auto &E : ExpectedInitial) { |
225 | OS << eventKindToString(K: E.Kind) << " " << E.Filename << "\n" ; |
226 | } |
227 | } |
228 | if (!ExpectedNonInitial.empty()) { |
229 | OS << "Expected but not seen non-initial events: \n" ; |
230 | for (const auto &E : ExpectedNonInitial) { |
231 | OS << eventKindToString(K: E.Kind) << " " << E.Filename << "\n" ; |
232 | } |
233 | } |
234 | if (!UnexpectedInitial.empty()) { |
235 | OS << "Unexpected initial events seen: \n" ; |
236 | for (const auto &E : UnexpectedInitial) { |
237 | OS << eventKindToString(K: E.Kind) << " " << E.Filename << "\n" ; |
238 | } |
239 | } |
240 | if (!UnexpectedNonInitial.empty()) { |
241 | OS << "Unexpected non-initial events seen: \n" ; |
242 | for (const auto &E : UnexpectedNonInitial) { |
243 | OS << eventKindToString(K: E.Kind) << " " << E.Filename << "\n" ; |
244 | } |
245 | } |
246 | } |
247 | }; |
248 | |
249 | void checkEventualResultWithTimeout(VerifyingConsumer &TestConsumer) { |
250 | std::packaged_task<int(void)> task( |
251 | [&TestConsumer]() { return TestConsumer.blockUntilResult(); }); |
252 | std::future<int> WaitForExpectedStateResult = task.get_future(); |
253 | std::thread worker(std::move(task)); |
254 | worker.detach(); |
255 | |
256 | EXPECT_TRUE(WaitForExpectedStateResult.wait_for(EventualResultTimeout) == |
257 | std::future_status::ready) |
258 | << "The expected result state wasn't reached before the time-out." ; |
259 | std::unique_lock<std::mutex> L(TestConsumer.Mtx); |
260 | EXPECT_TRUE(TestConsumer.result().has_value()); |
261 | if (TestConsumer.result()) { |
262 | EXPECT_TRUE(*TestConsumer.result()); |
263 | } |
264 | if ((TestConsumer.result() && !*TestConsumer.result()) || |
265 | !TestConsumer.result()) |
266 | TestConsumer.printUnmetExpectations(OS&: llvm::outs()); |
267 | } |
268 | } // namespace |
269 | |
270 | TEST(DirectoryWatcherTest, InitialScanSync) { |
271 | DirectoryWatcherTestFixture fixture; |
272 | |
273 | fixture.addFile(testFile: "a" ); |
274 | fixture.addFile(testFile: "b" ); |
275 | fixture.addFile(testFile: "c" ); |
276 | |
277 | VerifyingConsumer TestConsumer{ |
278 | {{EventKind::Modified, "a" }, |
279 | {EventKind::Modified, "b" }, |
280 | {EventKind::Modified, "c" }}, |
281 | {}, |
282 | // We have to ignore these as it's a race between the test process |
283 | // which is scanning the directory and kernel which is sending |
284 | // notification. |
285 | {{EventKind::Modified, "a" }, |
286 | {EventKind::Modified, "b" }, |
287 | {EventKind::Modified, "c" }} |
288 | }; |
289 | |
290 | llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW = |
291 | DirectoryWatcher::create( |
292 | Path: fixture.TestWatchedDir, |
293 | Receiver: [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events, |
294 | bool IsInitial) { |
295 | TestConsumer.consume(Es: Events, IsInitial); |
296 | }, |
297 | /*waitForInitialSync=*/WaitForInitialSync: true); |
298 | ASSERT_THAT_ERROR(DW.takeError(), Succeeded()); |
299 | |
300 | checkEventualResultWithTimeout(TestConsumer); |
301 | } |
302 | |
303 | TEST(DirectoryWatcherTest, InitialScanAsync) { |
304 | DirectoryWatcherTestFixture fixture; |
305 | |
306 | fixture.addFile(testFile: "a" ); |
307 | fixture.addFile(testFile: "b" ); |
308 | fixture.addFile(testFile: "c" ); |
309 | |
310 | VerifyingConsumer TestConsumer{ |
311 | {{EventKind::Modified, "a" }, |
312 | {EventKind::Modified, "b" }, |
313 | {EventKind::Modified, "c" }}, |
314 | {}, |
315 | // We have to ignore these as it's a race between the test process |
316 | // which is scanning the directory and kernel which is sending |
317 | // notification. |
318 | {{EventKind::Modified, "a" }, |
319 | {EventKind::Modified, "b" }, |
320 | {EventKind::Modified, "c" }} |
321 | }; |
322 | |
323 | llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW = |
324 | DirectoryWatcher::create( |
325 | Path: fixture.TestWatchedDir, |
326 | Receiver: [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events, |
327 | bool IsInitial) { |
328 | TestConsumer.consume(Es: Events, IsInitial); |
329 | }, |
330 | /*waitForInitialSync=*/WaitForInitialSync: false); |
331 | ASSERT_THAT_ERROR(DW.takeError(), Succeeded()); |
332 | |
333 | checkEventualResultWithTimeout(TestConsumer); |
334 | } |
335 | |
336 | TEST(DirectoryWatcherTest, AddFiles) { |
337 | DirectoryWatcherTestFixture fixture; |
338 | |
339 | VerifyingConsumer TestConsumer{ |
340 | {}, |
341 | {{EventKind::Modified, "a" }, |
342 | {EventKind::Modified, "b" }, |
343 | {EventKind::Modified, "c" }}}; |
344 | |
345 | llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW = |
346 | DirectoryWatcher::create( |
347 | Path: fixture.TestWatchedDir, |
348 | Receiver: [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events, |
349 | bool IsInitial) { |
350 | TestConsumer.consume(Es: Events, IsInitial); |
351 | }, |
352 | /*waitForInitialSync=*/WaitForInitialSync: true); |
353 | ASSERT_THAT_ERROR(DW.takeError(), Succeeded()); |
354 | |
355 | fixture.addFile(testFile: "a" ); |
356 | fixture.addFile(testFile: "b" ); |
357 | fixture.addFile(testFile: "c" ); |
358 | |
359 | checkEventualResultWithTimeout(TestConsumer); |
360 | } |
361 | |
362 | TEST(DirectoryWatcherTest, ModifyFile) { |
363 | DirectoryWatcherTestFixture fixture; |
364 | |
365 | fixture.addFile(testFile: "a" ); |
366 | |
367 | VerifyingConsumer TestConsumer{ |
368 | {{EventKind::Modified, "a" }}, |
369 | {{EventKind::Modified, "a" }}, |
370 | {{EventKind::Modified, "a" }}}; |
371 | |
372 | llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW = |
373 | DirectoryWatcher::create( |
374 | Path: fixture.TestWatchedDir, |
375 | Receiver: [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events, |
376 | bool IsInitial) { |
377 | TestConsumer.consume(Es: Events, IsInitial); |
378 | }, |
379 | /*waitForInitialSync=*/WaitForInitialSync: true); |
380 | ASSERT_THAT_ERROR(DW.takeError(), Succeeded()); |
381 | |
382 | // modify the file |
383 | { |
384 | std::error_code error; |
385 | llvm::raw_fd_ostream bStream(fixture.getPathInWatched(testFile: "a" ), error, |
386 | CD_OpenExisting); |
387 | assert(!error); |
388 | bStream << "foo" ; |
389 | } |
390 | |
391 | checkEventualResultWithTimeout(TestConsumer); |
392 | } |
393 | |
394 | TEST(DirectoryWatcherTest, DeleteFile) { |
395 | DirectoryWatcherTestFixture fixture; |
396 | |
397 | fixture.addFile(testFile: "a" ); |
398 | |
399 | VerifyingConsumer TestConsumer{ |
400 | {{EventKind::Modified, "a" }}, |
401 | {{EventKind::Removed, "a" }}, |
402 | {{EventKind::Modified, "a" }, {EventKind::Removed, "a" }}}; |
403 | |
404 | llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW = |
405 | DirectoryWatcher::create( |
406 | Path: fixture.TestWatchedDir, |
407 | Receiver: [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events, |
408 | bool IsInitial) { |
409 | TestConsumer.consume(Es: Events, IsInitial); |
410 | }, |
411 | /*waitForInitialSync=*/WaitForInitialSync: true); |
412 | ASSERT_THAT_ERROR(DW.takeError(), Succeeded()); |
413 | |
414 | fixture.deleteFile(testFile: "a" ); |
415 | |
416 | checkEventualResultWithTimeout(TestConsumer); |
417 | } |
418 | |
419 | TEST(DirectoryWatcherTest, DeleteWatchedDir) { |
420 | DirectoryWatcherTestFixture fixture; |
421 | |
422 | VerifyingConsumer TestConsumer{ |
423 | {}, |
424 | {{EventKind::WatchedDirRemoved, "" }, |
425 | {EventKind::WatcherGotInvalidated, "" }}}; |
426 | |
427 | llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW = |
428 | DirectoryWatcher::create( |
429 | Path: fixture.TestWatchedDir, |
430 | Receiver: [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events, |
431 | bool IsInitial) { |
432 | TestConsumer.consume(Es: Events, IsInitial); |
433 | }, |
434 | /*waitForInitialSync=*/WaitForInitialSync: true); |
435 | ASSERT_THAT_ERROR(DW.takeError(), Succeeded()); |
436 | |
437 | remove_directories(path: fixture.TestWatchedDir); |
438 | |
439 | checkEventualResultWithTimeout(TestConsumer); |
440 | } |
441 | |
442 | TEST(DirectoryWatcherTest, InvalidatedWatcher) { |
443 | DirectoryWatcherTestFixture fixture; |
444 | |
445 | VerifyingConsumer TestConsumer{ |
446 | {}, {{EventKind::WatcherGotInvalidated, "" }}}; |
447 | |
448 | { |
449 | llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW = |
450 | DirectoryWatcher::create( |
451 | Path: fixture.TestWatchedDir, |
452 | Receiver: [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events, |
453 | bool IsInitial) { |
454 | TestConsumer.consume(Es: Events, IsInitial); |
455 | }, |
456 | /*waitForInitialSync=*/WaitForInitialSync: true); |
457 | ASSERT_THAT_ERROR(DW.takeError(), Succeeded()); |
458 | } // DW is destructed here. |
459 | |
460 | checkEventualResultWithTimeout(TestConsumer); |
461 | } |
462 | |
463 | TEST(DirectoryWatcherTest, InvalidatedWatcherAsync) { |
464 | DirectoryWatcherTestFixture fixture; |
465 | fixture.addFile(testFile: "a" ); |
466 | |
467 | // This test is checking that we get the initial notification for 'a' before |
468 | // the final 'invalidated'. Strictly speaking, we do not care whether 'a' is |
469 | // processed or not, only that it is neither racing with, nor after |
470 | // 'invalidated'. In practice, it is always processed in our implementations. |
471 | VerifyingConsumer TestConsumer{ |
472 | {{EventKind::Modified, "a" }}, |
473 | {{EventKind::WatcherGotInvalidated, "" }}, |
474 | // We have to ignore these as it's a race between the test process |
475 | // which is scanning the directory and kernel which is sending |
476 | // notification. |
477 | {{EventKind::Modified, "a" }}, |
478 | }; |
479 | |
480 | // A counter that can help detect data races on the event receiver, |
481 | // particularly if used with TSan. Expected count will be 2 or 3 depending on |
482 | // whether we get the kernel event or not (see comment above). |
483 | unsigned Count = 0; |
484 | { |
485 | llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW = |
486 | DirectoryWatcher::create( |
487 | Path: fixture.TestWatchedDir, |
488 | Receiver: [&TestConsumer, |
489 | &Count](llvm::ArrayRef<DirectoryWatcher::Event> Events, |
490 | bool IsInitial) { |
491 | Count += 1; |
492 | TestConsumer.consume(Es: Events, IsInitial); |
493 | }, |
494 | /*waitForInitialSync=*/WaitForInitialSync: false); |
495 | ASSERT_THAT_ERROR(DW.takeError(), Succeeded()); |
496 | } // DW is destructed here. |
497 | |
498 | checkEventualResultWithTimeout(TestConsumer); |
499 | ASSERT_TRUE(Count == 2u || Count == 3u); |
500 | } |
501 | |