1// Copyright (C) 2025 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qrtaudioengine_p.h"
5
6#include <QtCore/qcoreapplication.h>
7#include <QtCore/qdebug.h>
8#include <QtCore/qmutex.h>
9#include <QtCore/qthread.h>
10
11#include <QtMultimedia/private/qaudio_rtsan_support_p.h>
12#include <QtMultimedia/private/qaudiosystem_p.h>
13#include <QtMultimedia/private/qmemory_resource_tlsf_p.h>
14
15#include <QtCore/q20map.h>
16#include <mutex>
17
18#ifdef Q_CC_MINGW
19// mingw-13.1 seems to have a false positive when using std::function inside a std::variant
20QT_WARNING_PUSH
21QT_WARNING_DISABLE_GCC("-Wmaybe-uninitialized")
22#endif
23
24QT_BEGIN_NAMESPACE
25
26namespace QtMultimediaPrivate {
27
28using namespace QtPrivate;
29using namespace std::chrono_literals;
30
31///////////////////////////////////////////////////////////////////////////////////////////////////
32
33namespace {
34struct AudioDeviceFormatLess
35{
36 bool operator()(const std::pair<QAudioDevice, QAudioFormat> &lhs,
37 const std::pair<QAudioDevice, QAudioFormat> &rhs) const
38 {
39 auto cmp = qCompareThreeWay(lhs: lhs.first.id(), rhs: rhs.first.id());
40 if (cmp == Qt::strong_ordering::less)
41 return true;
42 if (cmp == Qt::strong_ordering::greater)
43 return false;
44
45 return std::tuple(lhs.second.sampleRate(), lhs.second.sampleFormat(),
46 lhs.second.channelCount())
47 < std::tuple(rhs.second.sampleRate(), rhs.second.sampleFormat(),
48 rhs.second.channelCount());
49 }
50};
51} // namespace
52
53std::shared_ptr<QRtAudioEngine>
54QRtAudioEngine::getEngineFor(const QAudioDevice &device, const QAudioFormat &format)
55{
56 if (device.isNull()) {
57 qWarning() << "QRtAudioEngine needs to be called with a valid device";
58 return nullptr;
59 }
60
61 if (format.sampleFormat() != QAudioFormat::Float) {
62 qWarning() << "QRtAudioEngine requires floating point samples";
63 return nullptr;
64 }
65
66 if (!device.isFormatSupported(format)) {
67 qWarning() << "QRtAudioEngine needs to be called with a supported fromat";
68 return nullptr;
69 }
70
71 static QMutex s_playerRegistryMutex;
72 static std::map<std::pair<QAudioDevice, QAudioFormat>, std::weak_ptr<QRtAudioEngine>,
73 AudioDeviceFormatLess>
74 s_playerRegistry;
75
76 auto guard = std::lock_guard{ s_playerRegistryMutex };
77
78 auto key = std::pair{ device, format };
79 auto found = s_playerRegistry.find(x: key);
80 if (found != s_playerRegistry.end()) {
81 auto player = found->second.lock();
82 if (player)
83 return player;
84 }
85
86 // lazy clean up
87 q20::erase_if(c&: s_playerRegistry, p: [](auto &&keyValuePair) {
88 return keyValuePair.second.expired();
89 });
90
91 auto player = std::shared_ptr<QRtAudioEngine>(new QRtAudioEngine{ device, format },
92 [](QRtAudioEngine *engine) {
93 engine->deleteLater();
94 });
95 s_playerRegistry.emplace(args&: key, args&: player);
96
97 return player;
98}
99
100QRtAudioEngine::QRtAudioEngine(const QAudioDevice &device, const QAudioFormat &format)
101 : m_sink{
102 device,
103 format,
104 },
105 m_rtMemoryPool {
106 std::make_unique<QTlsfMemoryResource>(args: poolSize)
107 }
108{
109 m_notificationEvent.callOnActivated(functor: [this] {
110 runNonRtNotifications();
111 });
112
113 if (!QThread::isMainThread()) {
114 QThread *appThread = qApp->thread();
115 moveToThread(thread: appThread);
116 m_sink.moveToThread(thread: appThread);
117 m_notificationEvent.moveToThread(thread: appThread);
118 m_pendingCommandsTimer.moveToThread(thread: appThread);
119 }
120
121 m_pendingCommandsTimer.setInterval(10ms);
122 m_pendingCommandsTimer.setTimerType(Qt::CoarseTimer);
123 m_pendingCommandsTimer.callOnTimeout(args: &m_pendingCommandsTimer, args: [this] {
124 auto lock = std::lock_guard{ m_mutex };
125 sendPendingRtCommands();
126 if (m_appToRtOverflowBuffer.empty())
127 m_pendingCommandsTimer.stop();
128 });
129
130 QPlatformAudioSink *platformSink = QPlatformAudioSink::get(m_sink);
131
132 platformSink->start([this](QSpan<float> outputBuffer) {
133 audioCallback(outputBuffer);
134 });
135
136 // we start suspended
137 platformSink->suspend();
138}
139
140QRtAudioEngine::~QRtAudioEngine()
141{
142 m_sink.reset();
143
144 // consume the ringbuffers
145 m_appToRt.consumeAll(consumer: [](auto) {
146 });
147 m_rtToApp.consumeAll(consumer: [](auto) {
148 });
149}
150
151void QRtAudioEngine::play(SharedVoice voice)
152{
153 auto lock = std::lock_guard{ m_mutex };
154
155 // TODO: where do we expect reampling to happen?
156 Q_ASSERT(voice->format() == m_sink.format());
157
158 if (m_voices.empty())
159 m_sink.resume();
160
161 m_voices.insert(x: voice);
162
163 sendAppToRtCommand(cmd: PlayCommand{
164 .voice: std::move(voice),
165 });
166}
167
168void QRtAudioEngine::stop(const SharedVoice &voice)
169{
170 stop(voice->voiceId());
171}
172
173void QRtAudioEngine::stop(VoiceId voiceId)
174{
175 auto lock = std::lock_guard{ m_mutex };
176 sendAppToRtCommand(cmd: StopCommand{ .voiceId: voiceId });
177}
178
179void QRtAudioEngine::visitVoiceRt(VoiceId voiceId, RtVoiceVisitor fn, bool visitorIsTrivial)
180{
181 auto lock = std::lock_guard{ m_mutex };
182
183 if (visitorIsTrivial) {
184 sendAppToRtCommand(cmd: VisitCommandTrivial{
185 .voiceId: voiceId,
186 .callback: std::move(fn),
187 });
188
189 } else {
190 sendAppToRtCommand(cmd: VisitCommand{
191 .voiceId: voiceId,
192 .callback: std::move(fn),
193 });
194 }
195}
196
197VoiceId QRtAudioEngine::allocateVoiceId()
198{
199 static std::atomic_uint64_t allocator{ 0 };
200 return VoiceId{ allocator.fetch_add(i: 1, m: std::memory_order_relaxed) };
201}
202
203void QRtAudioEngine::audioCallback(QSpan<float> outputBuffer) noexcept QT_MM_NONBLOCKING
204{
205 runRtCommands();
206 bool sendNotification = sendPendingAppNotifications();
207
208 std::fill(first: outputBuffer.begin(), last: outputBuffer.end(), value: 0.f);
209
210 std::vector<SharedVoice, pmr::polymorphic_allocator<SharedVoice>> finishedVoices{
211 m_rtMemoryPool.get(),
212 };
213
214 for (const SharedVoice &voice : m_rtVoiceRegistry) {
215 Q_ASSERT(voice.use_count() >= 2); // voice in both m_rtVoiceRegistry and m_voices
216
217 VoicePlayResult playResult = voice->play(outputBuffer);
218 if (playResult == VoicePlayResult::Finished)
219 finishedVoices.push_back(x: voice);
220 }
221
222 if (!finishedVoices.empty()) {
223 withRTSanDisabled(f: [&] {
224 for (const SharedVoice &voice : finishedVoices) {
225 m_rtVoiceRegistry.erase(x: voice);
226 bool stopSent = sendRtToAppNotification(cmd: StopNotification{ .voice: voice });
227 if (stopSent)
228 sendNotification = true;
229 }
230 });
231 }
232
233 // TODO: we should probably (soft)clip the output buffer
234
235 cleanupRetiredVoices();
236 if (sendNotification)
237 m_notificationEvent.set();
238}
239
240void QRtAudioEngine::cleanupRetiredVoices() noexcept QT_MM_NONBLOCKING
241{
242 bool notifyApp = false;
243
244#if __cpp_lib_erase_if >= 202002L
245 using std::erase_if;
246#else
247 auto erase_if = [](auto &c, auto &&pred) {
248 auto old_size = c.size();
249 for (auto first = c.begin(), last = c.end(); first != last;) {
250 if (pred(*first))
251 first = c.erase(first);
252 else
253 ++first;
254 }
255 return old_size - c.size();
256 };
257#endif
258 withRTSanDisabled(f: [&] {
259 erase_if(m_rtVoiceRegistry, [&](const SharedVoice &voice) {
260 bool voiceIsActive = voice->isActive();
261 if (!voiceIsActive)
262 notifyApp = sendRtToAppNotification(cmd: StopNotification{ .voice: voice });
263
264 return false;
265 });
266 });
267
268 if (notifyApp)
269 m_notificationEvent.set();
270}
271
272void QRtAudioEngine::runRtCommands() noexcept QT_MM_NONBLOCKING
273{
274 m_appToRt.consumeAll(consumer: [&](QSpan<RtCommand> commands) {
275 for (RtCommand &cmd : commands) {
276 std::visit(visitor: [&](auto cmd) {
277 runRtCommand(std::move(cmd));
278 }, variants: std::move(cmd));
279 }
280 });
281}
282
283void QRtAudioEngine::runRtCommand(PlayCommand cmd) noexcept QT_MM_NONBLOCKING
284{
285 withRTSanDisabled(f: [&] {
286 m_rtVoiceRegistry.insert(x: cmd.voice);
287 });
288}
289
290void QRtAudioEngine::runRtCommand(StopCommand cmd) noexcept QT_MM_NONBLOCKING
291{
292 auto it = m_rtVoiceRegistry.find(x: cmd.voiceId);
293 Q_ASSERT(it != m_rtVoiceRegistry.end());
294
295 SharedVoice voice = *it;
296 m_rtVoiceRegistry.erase(position: it);
297
298 bool emitNotify = sendRtToAppNotification(cmd: StopNotification{
299 .voice: std::move(voice),
300 });
301 if (emitNotify)
302 m_notificationEvent.set();
303}
304
305void QRtAudioEngine::runRtCommand(VisitCommand cmd) noexcept QT_MM_NONBLOCKING
306{
307 auto it = m_rtVoiceRegistry.find(x: cmd.voiceId);
308 Q_ASSERT(it != m_rtVoiceRegistry.end());
309
310 cmd.callback(**it);
311
312 // send callback back to application for destruction
313 bool emitNotify = sendRtToAppNotification(cmd: VisitReply{
314 .callback: std::move(cmd.callback),
315 });
316 if (emitNotify)
317 m_notificationEvent.set();
318}
319
320void QRtAudioEngine::runRtCommand(VisitCommandTrivial cmd) noexcept QT_MM_NONBLOCKING
321{
322 auto it = m_rtVoiceRegistry.find(x: cmd.voiceId);
323 Q_ASSERT(it != m_rtVoiceRegistry.end());
324
325 cmd.callback(**it);
326}
327
328void QRtAudioEngine::runNonRtNotifications()
329{
330 std::vector<VoiceId> finishedVoices;
331 {
332 auto lock = std::lock_guard{ m_mutex };
333 m_rtToApp.consumeAll(consumer: [&](QSpan<Notification> notifications) {
334 for (Notification &notification : notifications) {
335 std::visit(visitor: [&](auto notification) {
336 runNonRtNotification(std::move(notification));
337 }, variants: std::move(notification));
338 }
339 });
340
341 finishedVoices = std::move(m_finishedVoices);
342 m_finishedVoices.clear();
343 }
344
345 // emit voiceFinished outside of the lock
346 for (VoiceId voiceId : finishedVoices)
347 emit voiceFinished(voiceId);
348}
349
350void QRtAudioEngine::runNonRtNotification(StopNotification notification)
351{
352 m_voices.erase(x: notification.voice);
353 if (m_voices.empty())
354 m_sink.suspend();
355 m_finishedVoices.push_back(x: notification.voice->voiceId());
356}
357
358void QRtAudioEngine::runNonRtNotification(VisitReply)
359{
360 // nop (just making sure to delete on the application thread);
361}
362
363void QRtAudioEngine::sendAppToRtCommand(RtCommand cmd)
364{
365 // first write all pending commands from overflow buffer
366 sendPendingRtCommands();
367
368 bool written = m_appToRt.produceOne(producer: [&] {
369 return std::move(cmd);
370 });
371
372 if (written)
373 return;
374
375 m_appToRtOverflowBuffer.emplace_back(args: std::move(cmd));
376
377 QMetaObject::invokeMethod(object: &m_pendingCommandsTimer, function: [this] {
378 if (!m_pendingCommandsTimer.isActive())
379 m_pendingCommandsTimer.start();
380 });
381}
382
383bool QRtAudioEngine::sendRtToAppNotification(Notification cmd)
384{
385 // first write all pending commands from overflow buffer
386 bool emitNotification = sendPendingAppNotifications();
387
388 bool written = m_rtToApp.produceOne(producer: [&] {
389 return std::move(cmd);
390 });
391
392 if (written)
393 return true;
394
395 m_rtToAppOverflowBuffer.emplace_back(args: std::move(cmd));
396
397 return emitNotification;
398}
399
400void QRtAudioEngine::sendPendingRtCommands()
401{
402 while (!m_appToRtOverflowBuffer.empty()) {
403 Q_UNLIKELY_BRANCH;
404 bool written = m_appToRt.produceOne(producer: [&] {
405 return std::move(m_appToRtOverflowBuffer.front());
406 });
407 if (!written)
408 return;
409
410 m_appToRtOverflowBuffer.pop_front();
411 }
412}
413
414bool QRtAudioEngine::sendPendingAppNotifications()
415{
416 bool emitNotification = false;
417 while (!m_rtToAppOverflowBuffer.empty()) {
418 Q_UNLIKELY_BRANCH;
419
420 // first write all pending commands from overflow buffer
421 bool written = m_rtToApp.produceOne(producer: [&] {
422 return std::move(m_rtToAppOverflowBuffer.front());
423 });
424 if (!written)
425 break;
426
427 m_rtToAppOverflowBuffer.pop_front();
428 emitNotification = true;
429 }
430
431 return emitNotification;
432}
433
434} // namespace QtMultimediaPrivate
435
436QT_END_NAMESPACE
437
438#ifdef Q_CC_MINGW
439QT_WARNING_POP
440#endif
441
442#include "moc_qrtaudioengine_p.cpp"
443

source code of qtmultimedia/src/multimedia/audio/qrtaudioengine.cpp