1// Copyright (C) 2022 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only
3
4#include "qaudioengine_p.h"
5
6#include <QtCore/qiodevice.h>
7#include <QtCore/qdebug.h>
8#include <QtCore/qelapsedtimer.h>
9
10#include <QtMultimedia/qaudiodecoder.h>
11#include <QtMultimedia/qmediadevices.h>
12#include <QtMultimedia/qaudiosink.h>
13#ifdef Q_OS_WIN
14# include <QtMultimedia/private/qwindows_wasapi_warmup_client_p.h>
15#endif
16
17#include <QtSpatialAudio/private/qambientsound_p.h>
18#include <QtSpatialAudio/private/qspatialsound_p.h>
19#include <QtSpatialAudio/private/qaudioroom_p.h>
20#include <QtSpatialAudio/private/qambisonicdecoder_p.h>
21#include <QtSpatialAudio/qambientsound.h>
22#include <QtSpatialAudio/qaudiolistener.h>
23
24#include <resonance_audio.h>
25
26QT_BEGIN_NAMESPACE
27
28// We'd like to have short buffer times, so the sound adjusts itself to changes
29// quickly, but times below 100ms seem to give stuttering on macOS.
30// It might be possible to set this value lower on other OSes.
31const int bufferTimeMs = 100;
32
33// This class lives in the audioThread, but pulls data from QAudioEnginePrivate
34// which lives in the mainThread.
35class QAudioOutputStream : public QIODevice
36{
37public:
38 explicit QAudioOutputStream(QAudioEnginePrivate *d)
39 : d(d)
40 {
41 open(mode: QIODevice::ReadOnly);
42 }
43 ~QAudioOutputStream() override;
44
45 qint64 readData(char *data, qint64 len) override;
46
47 qint64 writeData(const char *, qint64) override;
48
49 qint64 size() const override { return 0; }
50 qint64 bytesAvailable() const override {
51 return std::numeric_limits<qint64>::max();
52 }
53 bool isSequential() const override {
54 return true;
55 }
56 bool atEnd() const override {
57 return false;
58 }
59 qint64 pos() const override {
60 return m_pos;
61 }
62
63 void startOutput()
64 {
65 d->mutex.lock();
66 Q_ASSERT(!sink);
67 QAudioFormat format;
68 auto channelConfig = d->outputMode == QAudioEngine::Surround ?
69 d->device.channelConfiguration() : QAudioFormat::ChannelConfigStereo;
70 if (channelConfig != QAudioFormat::ChannelConfigUnknown)
71 format.setChannelConfig(channelConfig);
72 else
73 format.setChannelCount(d->device.preferredFormat().channelCount());
74 format.setSampleRate(d->sampleRate);
75 format.setSampleFormat(QAudioFormat::Int16);
76 ambisonicDecoder.reset(p: new QAmbisonicDecoder(QAmbisonicDecoder::HighQuality, format));
77 sink.reset(p: new QAudioSink(d->device, format));
78 const qsizetype bufferSize = format.bytesForDuration(microseconds: bufferTimeMs * 1000);
79 sink->setBufferSize(bufferSize);
80 d->mutex.unlock();
81 // It is important to unlock the mutex before starting the sink, as the sink will
82 // call readData() in the audio thread, which will try to lock the mutex (again)
83 sink->start(device: this);
84
85#ifdef Q_OS_WIN
86 QtMultimediaPrivate::refreshWarmupClient();
87#endif
88 }
89
90 void stopOutput()
91 {
92 if (!sink)
93 return;
94 sink->stop();
95 sink.reset();
96 ambisonicDecoder.reset();
97 }
98
99 void restartOutput()
100 {
101 stopOutput();
102 startOutput();
103 }
104
105 void setPaused(bool paused) {
106 if (paused)
107 sink->suspend();
108 else
109 sink->resume();
110 }
111
112private:
113 qint64 m_pos = 0;
114 QAudioEnginePrivate *d = nullptr;
115 std::unique_ptr<QAudioSink> sink;
116 std::unique_ptr<QAmbisonicDecoder> ambisonicDecoder;
117};
118
119
120QAudioOutputStream::~QAudioOutputStream()
121{
122}
123
124qint64 QAudioOutputStream::writeData(const char *, qint64)
125{
126 return 0;
127}
128
129qint64 QAudioOutputStream::readData(char *data, qint64 len)
130{
131 if (d->paused.loadRelaxed())
132 return 0;
133
134 QMutexLocker l(&d->mutex);
135 d->updateRooms();
136
137 int nChannels = ambisonicDecoder ? ambisonicDecoder->nOutputChannels() : 2;
138 if (len < nChannels*int(sizeof(float))*QAudioEnginePrivate::bufferSize)
139 return 0;
140
141 short *fd = (short *)data;
142 qint64 frames = len / nChannels / sizeof(short);
143 bool ok = true;
144 while (frames >= qint64(QAudioEnginePrivate::bufferSize)) {
145 // Fill input buffers
146 for (auto *source : std::as_const(t&: d->sources)) {
147 auto *sp = QSpatialSoundPrivate::get(soundSource: source);
148 if (!sp)
149 continue;
150 float buf[QAudioEnginePrivate::bufferSize];
151 sp->getBuffer(buf, frames: QAudioEnginePrivate::bufferSize, channels: 1);
152 d->resonanceAudio->api->SetInterleavedBuffer(source_id: sp->sourceId, audio_buffer_ptr: buf, num_channels: 1, num_frames: QAudioEnginePrivate::bufferSize);
153 }
154 for (auto *source : std::as_const(t&: d->stereoSources)) {
155 auto *sp = QAmbientSoundPrivate::get(soundSource: source);
156 if (!sp)
157 continue;
158 float buf[2*QAudioEnginePrivate::bufferSize];
159 sp->getBuffer(buf, frames: QAudioEnginePrivate::bufferSize, channels: 2);
160 d->resonanceAudio->api->SetInterleavedBuffer(source_id: sp->sourceId, audio_buffer_ptr: buf, num_channels: 2, num_frames: QAudioEnginePrivate::bufferSize);
161 }
162
163 if (ambisonicDecoder && d->outputMode == QAudioEngine::Surround) {
164 const float *channels[QAmbisonicDecoder::maxAmbisonicChannels];
165 const float *reverbBuffers[2];
166 int nSamples = d->resonanceAudio->getAmbisonicOutput(buffers: channels, reverb: reverbBuffers, nChannels: ambisonicDecoder->nInputChannels());
167 Q_ASSERT(ambisonicDecoder->nOutputChannels() <= 8);
168 ambisonicDecoder->processBufferWithReverb(input: channels, reverb: reverbBuffers, output: fd, nSamples);
169 } else {
170 ok = d->resonanceAudio->api->FillInterleavedOutputBuffer(num_channels: 2, num_frames: QAudioEnginePrivate::bufferSize, buffer_ptr: fd);
171 if (!ok) {
172 // If we get here, it means that resonanceAudio did not actually fill the buffer.
173 // Sometimes this is expected, for example if resonanceAudio does not have any sources.
174 // In this case we just fill the buffer with silence.
175 if (d->sources.isEmpty() && d->stereoSources.isEmpty()) {
176 memset(s: fd, c: 0, n: nChannels * QAudioEnginePrivate::bufferSize * sizeof(short));
177 } else {
178 // If we get here, it means that something unexpected happened, so bail.
179 qWarning() << " Reading failed!";
180 break;
181 }
182 }
183 }
184 fd += nChannels*QAudioEnginePrivate::bufferSize;
185 frames -= QAudioEnginePrivate::bufferSize;
186 }
187 const int bytesProcessed = ((char *)fd - data);
188 m_pos += bytesProcessed;
189 return bytesProcessed;
190}
191
192QAudioEnginePrivate::QAudioEnginePrivate(QAudioEngine *q) : q(q)
193{
194 audioThread.setObjectName(u"QAudioThread");
195 device = QMediaDevices::defaultAudioOutput();
196}
197
198QAudioEnginePrivate::~QAudioEnginePrivate()
199{
200 resonanceAudio = {};
201}
202
203void QAudioEnginePrivate::start()
204{
205 if (outputStream)
206 // already started
207 return;
208
209 resonanceAudio->api->SetStereoSpeakerMode(outputMode != QAudioEngine::Headphone);
210 resonanceAudio->api->SetMasterVolume(masterVolume);
211
212 outputStream = std::make_unique<QAudioOutputStream>(args: this);
213 outputStream->moveToThread(thread: &audioThread);
214 audioThread.start(QThread::TimeCriticalPriority);
215
216 QMetaObject::invokeMethod(object: outputStream.get(), function: &QAudioOutputStream::startOutput);
217}
218
219void QAudioEnginePrivate::stop()
220{
221 QMetaObject::invokeMethod(object: outputStream.get(), function: &QAudioOutputStream::stopOutput,
222 type: Qt::BlockingQueuedConnection);
223 outputStream.reset();
224 audioThread.exit(retcode: 0);
225 audioThread.wait();
226}
227
228void QAudioEnginePrivate::setPaused(bool paused)
229{
230 bool old = this->paused.fetchAndStoreRelaxed(newValue: paused);
231 if (old != paused) {
232 if (outputStream)
233 outputStream->setPaused(paused);
234 emit q->pausedChanged();
235 }
236}
237
238void QAudioEnginePrivate::setOutputDevice(const QAudioDevice &device)
239{
240 if (this->device == device)
241 return;
242 if (resonanceAudio->api) {
243 qWarning() << "Changing device on a running engine not implemented";
244 return;
245 }
246 this->device = device;
247 emit q->outputDeviceChanged();
248}
249
250void QAudioEnginePrivate::setOutputMode(QAudioEngine::OutputMode mode)
251{
252 if (outputMode == mode)
253 return;
254 outputMode = mode;
255 if (resonanceAudio->api)
256 resonanceAudio->api->SetStereoSpeakerMode(mode != QAudioEngine::Headphone);
257
258 QMetaObject::invokeMethod(object: outputStream.get(), function: &QAudioOutputStream::restartOutput,
259 type: Qt::BlockingQueuedConnection);
260
261 emit q->outputModeChanged();
262}
263
264void QAudioEnginePrivate::addSpatialSound(QSpatialSound *sound)
265{
266 QMutexLocker l(&mutex);
267 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
268
269 sd->sourceId = resonanceAudio->api->CreateSoundObjectSource(rendering_mode: vraudio::kBinauralHighQuality);
270 sources.append(t: sound);
271}
272
273void QAudioEnginePrivate::removeSpatialSound(QSpatialSound *sound)
274{
275 QMutexLocker l(&mutex);
276 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
277
278 resonanceAudio->api->DestroySource(id: sd->sourceId);
279 sd->sourceId = vraudio::ResonanceAudioApi::kInvalidSourceId;
280 sources.removeOne(t: sound);
281}
282
283void QAudioEnginePrivate::addStereoSound(QAmbientSound *sound)
284{
285 QMutexLocker l(&mutex);
286 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
287
288 sd->sourceId = resonanceAudio->api->CreateStereoSource(num_channels: 2);
289 stereoSources.append(t: sound);
290}
291
292void QAudioEnginePrivate::removeStereoSound(QAmbientSound *sound)
293{
294 QMutexLocker l(&mutex);
295 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
296
297 resonanceAudio->api->DestroySource(id: sd->sourceId);
298 sd->sourceId = vraudio::ResonanceAudioApi::kInvalidSourceId;
299 stereoSources.removeOne(t: sound);
300}
301
302void QAudioEnginePrivate::addRoom(QAudioRoom *room)
303{
304 QMutexLocker l(&mutex);
305 rooms.append(t: room);
306}
307
308void QAudioEnginePrivate::removeRoom(QAudioRoom *room)
309{
310 QMutexLocker l(&mutex);
311 rooms.removeOne(t: room);
312}
313
314// This method is called from the audio thread
315void QAudioEnginePrivate::updateRooms()
316{
317 if (!roomEffectsEnabled)
318 return;
319
320 bool needUpdate = listenerPositionDirty;
321 listenerPositionDirty = false;
322
323 bool roomDirty = false;
324 for (const auto &room : std::as_const(t&: rooms)) {
325 auto *rd = QAudioRoomPrivate::get(r: room);
326 if (rd->dirty) {
327 roomDirty = true;
328 rd->update();
329 needUpdate = true;
330 }
331 }
332
333 if (!needUpdate)
334 return;
335
336 QVector3D listenerPos = listenerPosition();
337 float roomVolume = float(qInf());
338 QAudioRoom *room = nullptr;
339 // Find the smallest room that contains the listener and apply its room effects
340 for (auto *r : std::as_const(t&: rooms)) {
341 QVector3D dim2 = r->dimensions()/2.;
342 float vol = dim2.x()*dim2.y()*dim2.z();
343 if (vol > roomVolume)
344 continue;
345 QVector3D dist = r->position() - listenerPos;
346 // transform into room coordinates
347 dist = r->rotation().rotatedVector(vector: dist);
348 if (qAbs(t: dist.x()) <= dim2.x() &&
349 qAbs(t: dist.y()) <= dim2.y() &&
350 qAbs(t: dist.z()) <= dim2.z()) {
351 room = r;
352 roomVolume = vol;
353 }
354 }
355 if (room != currentRoom)
356 roomDirty = true;
357 const bool previousRoom = currentRoom;
358 currentRoom = room;
359
360 if (!roomDirty)
361 return;
362
363 // apply room to engine
364 if (!currentRoom) {
365 resonanceAudio->api->EnableRoomEffects(enable: false);
366 return;
367 }
368 if (!previousRoom)
369 resonanceAudio->api->EnableRoomEffects(enable: true);
370
371 QAudioRoomPrivate *rp = QAudioRoomPrivate::get(r: room);
372 resonanceAudio->api->SetReflectionProperties(rp->reflections);
373 resonanceAudio->api->SetReverbProperties(rp->reverb);
374
375 // update room effects for all sound sources
376 for (auto *s : std::as_const(t&: sources)) {
377 auto *sp = QSpatialSoundPrivate::get(soundSource: s);
378 if (!sp)
379 continue;
380 sp->updateRoomEffects();
381 }
382}
383
384QVector3D QAudioEnginePrivate::listenerPosition() const
385{
386 return listener ? listener->position() : QVector3D();
387}
388
389/*!
390 \class QAudioEngine
391 \inmodule QtSpatialAudio
392 \ingroup spatialaudio
393 \ingroup multimedia_audio
394
395 \brief QAudioEngine manages a three dimensional sound field.
396
397 You can use an instance of QAudioEngine to manage a sound field in
398 three dimensions. A sound field is defined by several QSpatialSound
399 objects that define a sound at a specified location in 3D space. You can also
400 add stereo overlays using QAmbientSound.
401
402 You can use QAudioListener to define the position of the person listening
403 to the sound field relative to the sound sources. Sound sources will be less audible
404 if the listener is further away from source. They will also get mapped to the corresponding
405 loudspeakers depending on the direction between listener and source.
406
407 QAudioEngine offers two output modes. The first mode renders the sound field to a set of
408 speakers, either a stereo speaker pair or a surround configuration. The second mode provides
409 an immersive 3D sound experience when using headphones.
410
411 Perception of sound localization is driven mainly by two factors. The first factor is timing
412 differences of the sound waves between left and right ear. The second factor comes from various
413 ways how sounds coming from different direcations create different types of reflections from our
414 ears and heads. See https://en.wikipedia.org/wiki/Sound_localization for more details.
415
416 The spatial audio engine emulates those timing differences and reflections through
417 Head related transfer functions (HRTF, see
418 https://en.wikipedia.org/wiki/Head-related_transfer_function). The functions used emulates those
419 effects for an average persons ears and head. It provides a good and immersive 3D sound localization
420 experience for most persons when using headphones.
421
422 The engine is rather versatile allowing you to define room properties and reverb settings to emulate
423 different types of rooms.
424
425 Sound sources can also be occluded dampening the sound coming from those sources.
426
427 The audio engine uses a coordinate system that is in centimeters by default. The axes are aligned with the
428 typical coordinate system used in 3D. Positive x points to the right, positive y points up and positive z points
429 backwards.
430
431*/
432
433/*!
434 \fn QAudioEngine::QAudioEngine()
435 \fn QAudioEngine::QAudioEngine(QObject *parent)
436 \fn QAudioEngine::QAudioEngine(int sampleRate, QObject *parent = nullptr)
437
438 Constructs a spatial audio engine with \a parent, if any.
439
440 The engine will operate with a sample rate given by \a sampleRate. The
441 default sample rate, if none is provided, is 44100 (44.1kHz).
442
443 Sound content that is not provided at that sample rate will automatically
444 get resampled to \a sampleRate when being processed by the engine. The
445 default sample rate is fine in most cases, but you can define a different
446 rate if most of your sound files are sampled with a different rate, and
447 avoid some CPU overhead for resampling.
448 */
449QAudioEngine::QAudioEngine(int sampleRate, QObject *parent)
450 : QObject(parent), d(new QAudioEnginePrivate(this))
451{
452 d->sampleRate = sampleRate;
453 d->resonanceAudio = std::make_unique<vraudio::ResonanceAudio>(
454 args: 2, args: QAudioEnginePrivate::bufferSize, args&: d->sampleRate);
455}
456
457/*!
458 Destroys the spatial audio engine.
459 */
460QAudioEngine::~QAudioEngine()
461{
462 stop();
463 delete d;
464}
465
466/*! \enum QAudioEngine::OutputMode
467 \value Surround Map the sounds to the loudspeaker configuration of the output device.
468 This is normally a stereo or surround speaker setup.
469 \value Stereo Map the sounds to the stereo loudspeaker configuration of the output device.
470 This will ignore any additional speakers and only use the left and right channels
471 to create a stero rendering of the sound field.
472 \value Headphone Use Headphone spatialization to create a 3D audio effect when listening
473 to the sound field through headphones
474*/
475
476/*!
477 \property QAudioEngine::outputMode
478
479 Sets or retrieves the current output mode of the engine.
480
481 \sa QAudioEngine::OutputMode
482 */
483void QAudioEngine::setOutputMode(OutputMode mode)
484{
485 d->setOutputMode(mode);
486}
487
488QAudioEngine::OutputMode QAudioEngine::outputMode() const
489{
490 return d->outputMode;
491}
492
493/*!
494 Returns the sample rate the engine has been configured with.
495 */
496int QAudioEngine::sampleRate() const
497{
498 return d->sampleRate;
499}
500
501/*!
502 \property QAudioEngine::outputDevice
503
504 Sets or returns the device that is being used for playing the sound field.
505 */
506void QAudioEngine::setOutputDevice(const QAudioDevice &device)
507{
508 d->setOutputDevice(device);
509}
510
511QAudioDevice QAudioEngine::outputDevice() const
512{
513 return d->device;
514}
515
516/*!
517 \property QAudioEngine::masterVolume
518
519 Sets or returns volume being used to render the sound field.
520 */
521void QAudioEngine::setMasterVolume(float volume)
522{
523 if (d->masterVolume == volume)
524 return;
525 d->masterVolume = volume;
526 d->resonanceAudio->api->SetMasterVolume(volume);
527 emit masterVolumeChanged();
528}
529
530float QAudioEngine::masterVolume() const
531{
532 return d->masterVolume;
533}
534
535/*!
536 Starts the engine.
537 */
538void QAudioEngine::start()
539{
540 d->start();
541}
542
543/*!
544 Stops the engine.
545 */
546void QAudioEngine::stop()
547{
548 d->stop();
549}
550
551/*!
552 \property QAudioEngine::paused
553
554 Pauses the spatial audio engine.
555 */
556void QAudioEngine::setPaused(bool paused)
557{
558 d->setPaused(paused);
559}
560
561bool QAudioEngine::paused() const
562{
563 return d->paused.loadRelaxed();
564}
565
566/*!
567 Enables room effects such as echos and reverb.
568
569 Enables room effects if \a enabled is true.
570 Room effects will only apply if you create one or more \l QAudioRoom objects
571 and the listener is inside at least one of the rooms. If the listener is inside
572 multiple rooms, the room with the smallest volume will be used.
573 */
574void QAudioEngine::setRoomEffectsEnabled(bool enabled)
575{
576 if (d->roomEffectsEnabled == enabled)
577 return;
578 d->roomEffectsEnabled = enabled;
579 d->resonanceAudio->roomEffectsEnabled = enabled;
580}
581
582/*!
583 Returns true if room effects are enabled.
584 */
585bool QAudioEngine::roomEffectsEnabled() const
586{
587 return d->roomEffectsEnabled;
588}
589
590/*!
591 \property QAudioEngine::distanceScale
592
593 Defines the scale of the coordinate system being used by the spatial audio engine.
594 By default, all units are in centimeters, in line with the default units being
595 used by Qt Quick 3D.
596
597 Set the distance scale to QAudioEngine::DistanceScaleMeter to get units in meters.
598*/
599void QAudioEngine::setDistanceScale(float scale)
600{
601 // multiply with 100, to get the conversion to meters that resonance audio uses
602 scale /= 100.f;
603 if (scale <= 0.0f) {
604 qWarning() << "QAudioEngine: Invalid distance scale.";
605 return;
606 }
607 if (scale == d->distanceScale)
608 return;
609 d->distanceScale = scale;
610 emit distanceScaleChanged();
611}
612
613float QAudioEngine::distanceScale() const
614{
615 return d->distanceScale*100.f;
616}
617
618/*!
619 \fn void QAudioEngine::pause()
620
621 Pauses playback.
622*/
623/*!
624 \fn void QAudioEngine::resume()
625
626 Resumes playback.
627*/
628/*!
629 \variable QAudioEngine::DistanceScaleCentimeter
630 \internal
631*/
632/*!
633 \variable QAudioEngine::DistanceScaleMeter
634 \internal
635*/
636
637QT_END_NAMESPACE
638
639#include "moc_qaudioengine.cpp"
640

source code of qtmultimedia/src/spatialaudio/qaudioengine.cpp