1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9// UNSUPPORTED: c++03, c++11, c++14, c++17
10
11#include "atomic_wait_helper.h"
12
13#include <atomic>
14#include <cstdint>
15#include <numeric>
16#include <stop_token>
17#include <pthread.h>
18#include <sched.h>
19#include <thread>
20#include <chrono>
21#include <array>
22
23#include "benchmark/benchmark.h"
24#include "make_test_thread.h"
25
26using namespace std::chrono_literals;
27
28template <class NotifyPolicy, class NumberOfAtomics, class NumPrioTasks>
29void BM_N_atomics_N_waiter_N_notifier(benchmark::State& state) {
30 [[maybe_unused]] std::array<HighPrioTask, NumPrioTasks::value> tasks{};
31 const std::uint64_t total_loop_test_param = state.range(0);
32 constexpr std::uint64_t num_atomics = NumberOfAtomics::value;
33 std::vector<std::atomic<std::uint64_t>> atomics(num_atomics);
34
35 auto notify_func = [&](std::stop_token st, size_t idx) {
36 while (!st.stop_requested()) {
37 NotifyPolicy::notify(atomics[idx], st);
38 }
39 };
40
41 std::atomic<std::uint64_t> start_flag = 0;
42 std::atomic<std::uint64_t> done_count = 0;
43
44 auto wait_func = [&, total_loop_test_param](std::stop_token st, size_t idx) {
45 auto old_start = 0;
46 while (!st.stop_requested()) {
47 start_flag.wait(old_start);
48 old_start = start_flag.load();
49 for (std::uint64_t i = 0; i < total_loop_test_param; ++i) {
50 auto old = atomics[idx].load(std::memory_order_relaxed);
51 atomics[idx].wait(old);
52 }
53 done_count.fetch_add(i: 1);
54 }
55 };
56
57 std::vector<std::jthread> notify_threads;
58 notify_threads.reserve(num_atomics);
59
60 std::vector<std::jthread> wait_threads;
61 wait_threads.reserve(num_atomics);
62
63 for (size_t i = 0; i < num_atomics; ++i) {
64 notify_threads.emplace_back(support::make_test_jthread(notify_func, i));
65 }
66
67 for (size_t i = 0; i < num_atomics; ++i) {
68 wait_threads.emplace_back(support::make_test_jthread(wait_func, i));
69 }
70
71 for (auto _ : state) {
72 done_count = 0;
73 start_flag.fetch_add(1);
74 start_flag.notify_all();
75 while (done_count < num_atomics) {
76 std::this_thread::yield();
77 }
78 }
79 for (auto& t : wait_threads) {
80 t.request_stop();
81 }
82 start_flag.fetch_add(i: 1);
83 start_flag.notify_all();
84 for (auto& t : wait_threads) {
85 t.join();
86 }
87}
88
89BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<2>, NumHighPrioTasks<0>>)
90 ->RangeMultiplier(2)
91 ->Range(1 << 12, 1 << 14);
92BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<3>, NumHighPrioTasks<0>>)
93 ->RangeMultiplier(2)
94 ->Range(1 << 10, 1 << 12);
95BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<5>, NumHighPrioTasks<0>>)
96 ->RangeMultiplier(2)
97 ->Range(1 << 10, 1 << 12);
98BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<7>, NumHighPrioTasks<0>>)
99 ->RangeMultiplier(2)
100 ->Range(1 << 8, 1 << 10);
101
102BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<2>, NumHighPrioTasks<0>>)
103 ->RangeMultiplier(2)
104 ->Range(1 << 10, 1 << 12);
105BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<3>, NumHighPrioTasks<0>>)
106 ->RangeMultiplier(2)
107 ->Range(1 << 8, 1 << 10);
108BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<5>, NumHighPrioTasks<0>>)
109 ->RangeMultiplier(2)
110 ->Range(1 << 8, 1 << 10);
111BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<7>, NumHighPrioTasks<0>>)
112 ->RangeMultiplier(2)
113 ->Range(1 << 6, 1 << 8);
114
115BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<2>, NumHighPrioTasks<0>>)
116 ->RangeMultiplier(2)
117 ->Range(1 << 8, 1 << 10);
118BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<3>, NumHighPrioTasks<0>>)
119 ->RangeMultiplier(2)
120 ->Range(1 << 8, 1 << 10);
121BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<5>, NumHighPrioTasks<0>>)
122 ->RangeMultiplier(2)
123 ->Range(1 << 7, 1 << 9);
124BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<7>, NumHighPrioTasks<0>>)
125 ->RangeMultiplier(2)
126 ->Range(1 << 6, 1 << 8);
127
128BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<2>, NumHighPrioTasks<4>>)
129 ->RangeMultiplier(2)
130 ->Range(1 << 7, 1 << 9);
131BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<3>, NumHighPrioTasks<4>>)
132 ->RangeMultiplier(2)
133 ->Range(1 << 7, 1 << 9);
134BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<5>, NumHighPrioTasks<4>>)
135 ->RangeMultiplier(2)
136 ->Range(1 << 6, 1 << 8);
137BENCHMARK(BM_N_atomics_N_waiter_N_notifier<KeepNotifying, NumberOfAtomics<7>, NumHighPrioTasks<4>>)
138 ->RangeMultiplier(2)
139 ->Range(1 << 4, 1 << 6);
140
141BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<2>, NumHighPrioTasks<4>>)
142 ->RangeMultiplier(2)
143 ->Range(1 << 7, 1 << 9);
144BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<3>, NumHighPrioTasks<4>>)
145 ->RangeMultiplier(2)
146 ->Range(1 << 7, 1 << 9);
147BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<5>, NumHighPrioTasks<4>>)
148 ->RangeMultiplier(2)
149 ->Range(1 << 5, 1 << 7);
150BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<50>, NumberOfAtomics<7>, NumHighPrioTasks<4>>)
151 ->RangeMultiplier(2)
152 ->Range(1 << 3, 1 << 5);
153
154BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<2>, NumHighPrioTasks<4>>)
155 ->RangeMultiplier(2)
156 ->Range(1 << 6, 1 << 8);
157BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<3>, NumHighPrioTasks<4>>)
158 ->RangeMultiplier(2)
159 ->Range(1 << 6, 1 << 8);
160BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<5>, NumHighPrioTasks<4>>)
161 ->RangeMultiplier(2)
162 ->Range(1 << 5, 1 << 7);
163BENCHMARK(BM_N_atomics_N_waiter_N_notifier<NotifyEveryNus<100>, NumberOfAtomics<7>, NumHighPrioTasks<4>>)
164 ->RangeMultiplier(2)
165 ->Range(1 << 3, 1 << 5);
166
167BENCHMARK_MAIN();
168

source code of libcxx/test/benchmarks/atomic_wait_N_waiter_N_notifier.bench.cpp