1// Copyright (C) 2022 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only
3#include <qaudioengine_p.h>
4#include <qambientsound_p.h>
5#include <qspatialsound_p.h>
6#include <qambientsound.h>
7#include <qaudioroom_p.h>
8#include <qaudiolistener.h>
9#include <resonance_audio.h>
10#include <qambisonicdecoder_p.h>
11#include <qaudiodecoder.h>
12#include <qmediadevices.h>
13#include <qiodevice.h>
14#include <qaudiosink.h>
15#include <qdebug.h>
16#include <qelapsedtimer.h>
17
18#include <QFile>
19
20QT_BEGIN_NAMESPACE
21
22// We'd like to have short buffer times, so the sound adjusts itself to changes
23// quickly, but times below 100ms seem to give stuttering on macOS.
24// It might be possible to set this value lower on other OSes.
25const int bufferTimeMs = 100;
26
27// This class lives in the audioThread, but pulls data from QAudioEnginePrivate
28// which lives in the mainThread.
29class QAudioOutputStream : public QIODevice
30{
31 Q_OBJECT
32public:
33 explicit QAudioOutputStream(QAudioEnginePrivate *d)
34 : d(d)
35 {
36 open(mode: QIODevice::ReadOnly);
37 }
38 ~QAudioOutputStream() override;
39
40 qint64 readData(char *data, qint64 len) override;
41
42 qint64 writeData(const char *, qint64) override;
43
44 qint64 size() const override { return 0; }
45 qint64 bytesAvailable() const override {
46 return std::numeric_limits<qint64>::max();
47 }
48 bool isSequential() const override {
49 return true;
50 }
51 bool atEnd() const override {
52 return false;
53 }
54 qint64 pos() const override {
55 return m_pos;
56 }
57
58 Q_INVOKABLE void startOutput() {
59 d->mutex.lock();
60 Q_ASSERT(!sink);
61 QAudioFormat format;
62 auto channelConfig = d->outputMode == QAudioEngine::Surround ?
63 d->device.channelConfiguration() : QAudioFormat::ChannelConfigStereo;
64 if (channelConfig != QAudioFormat::ChannelConfigUnknown)
65 format.setChannelConfig(channelConfig);
66 else
67 format.setChannelCount(d->device.preferredFormat().channelCount());
68 format.setSampleRate(d->sampleRate);
69 format.setSampleFormat(QAudioFormat::Int16);
70 ambisonicDecoder.reset(p: new QAmbisonicDecoder(QAmbisonicDecoder::HighQuality, format));
71 sink.reset(p: new QAudioSink(d->device, format));
72 const qsizetype bufferSize = format.bytesForDuration(microseconds: bufferTimeMs * 1000);
73 sink->setBufferSize(bufferSize);
74 d->mutex.unlock();
75 // It is important to unlock the mutex before starting the sink, as the sink will
76 // call readData() in the audio thread, which will try to lock the mutex (again)
77 sink->start(device: this);
78 }
79
80 Q_INVOKABLE void stopOutput() {
81 sink->stop();
82 sink.reset();
83 ambisonicDecoder.reset();
84 }
85
86 Q_INVOKABLE void restartOutput() {
87 stopOutput();
88 startOutput();
89 }
90
91 void setPaused(bool paused) {
92 if (paused)
93 sink->suspend();
94 else
95 sink->resume();
96 }
97
98private:
99 qint64 m_pos = 0;
100 QAudioEnginePrivate *d = nullptr;
101 std::unique_ptr<QAudioSink> sink;
102 std::unique_ptr<QAmbisonicDecoder> ambisonicDecoder;
103};
104
105
106QAudioOutputStream::~QAudioOutputStream()
107{
108}
109
110qint64 QAudioOutputStream::writeData(const char *, qint64)
111{
112 return 0;
113}
114
115qint64 QAudioOutputStream::readData(char *data, qint64 len)
116{
117 if (d->paused.loadRelaxed())
118 return 0;
119
120 QMutexLocker l(&d->mutex);
121 d->updateRooms();
122
123 int nChannels = ambisonicDecoder ? ambisonicDecoder->nOutputChannels() : 2;
124 if (len < nChannels*int(sizeof(float))*QAudioEnginePrivate::bufferSize)
125 return 0;
126
127 short *fd = (short *)data;
128 qint64 frames = len / nChannels / sizeof(short);
129 bool ok = true;
130 while (frames >= qint64(QAudioEnginePrivate::bufferSize)) {
131 // Fill input buffers
132 for (auto *source : std::as_const(t&: d->sources)) {
133 auto *sp = QSpatialSoundPrivate::get(soundSource: source);
134 if (!sp)
135 continue;
136 float buf[QAudioEnginePrivate::bufferSize];
137 sp->getBuffer(buf, frames: QAudioEnginePrivate::bufferSize, channels: 1);
138 d->resonanceAudio->api->SetInterleavedBuffer(source_id: sp->sourceId, audio_buffer_ptr: buf, num_channels: 1, num_frames: QAudioEnginePrivate::bufferSize);
139 }
140 for (auto *source : std::as_const(t&: d->stereoSources)) {
141 auto *sp = QAmbientSoundPrivate::get(soundSource: source);
142 if (!sp)
143 continue;
144 float buf[2*QAudioEnginePrivate::bufferSize];
145 sp->getBuffer(buf, frames: QAudioEnginePrivate::bufferSize, channels: 2);
146 d->resonanceAudio->api->SetInterleavedBuffer(source_id: sp->sourceId, audio_buffer_ptr: buf, num_channels: 2, num_frames: QAudioEnginePrivate::bufferSize);
147 }
148
149 if (ambisonicDecoder && d->outputMode == QAudioEngine::Surround) {
150 const float *channels[QAmbisonicDecoder::maxAmbisonicChannels];
151 const float *reverbBuffers[2];
152 int nSamples = d->resonanceAudio->getAmbisonicOutput(buffers: channels, reverb: reverbBuffers, nChannels: ambisonicDecoder->nInputChannels());
153 Q_ASSERT(ambisonicDecoder->nOutputChannels() <= 8);
154 ambisonicDecoder->processBufferWithReverb(input: channels, reverb: reverbBuffers, output: fd, nSamples);
155 } else {
156 ok = d->resonanceAudio->api->FillInterleavedOutputBuffer(num_channels: 2, num_frames: QAudioEnginePrivate::bufferSize, buffer_ptr: fd);
157 if (!ok) {
158 // If we get here, it means that resonanceAudio did not actually fill the buffer.
159 // Sometimes this is expected, for example if resonanceAudio does not have any sources.
160 // In this case we just fill the buffer with silence.
161 if (d->sources.isEmpty() && d->stereoSources.isEmpty()) {
162 memset(s: fd, c: 0, n: nChannels * QAudioEnginePrivate::bufferSize * sizeof(short));
163 } else {
164 // If we get here, it means that something unexpected happened, so bail.
165 qWarning() << " Reading failed!";
166 break;
167 }
168 }
169 }
170 fd += nChannels*QAudioEnginePrivate::bufferSize;
171 frames -= QAudioEnginePrivate::bufferSize;
172 }
173 const int bytesProcessed = ((char *)fd - data);
174 m_pos += bytesProcessed;
175 return bytesProcessed;
176}
177
178
179QAudioEnginePrivate::QAudioEnginePrivate()
180{
181 audioThread.setObjectName(u"QAudioThread");
182 device = QMediaDevices::defaultAudioOutput();
183}
184
185QAudioEnginePrivate::~QAudioEnginePrivate()
186{
187 delete resonanceAudio;
188}
189
190void QAudioEnginePrivate::addSpatialSound(QSpatialSound *sound)
191{
192 QMutexLocker l(&mutex);
193 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
194
195 sd->sourceId = resonanceAudio->api->CreateSoundObjectSource(rendering_mode: vraudio::kBinauralHighQuality);
196 sources.append(t: sound);
197}
198
199void QAudioEnginePrivate::removeSpatialSound(QSpatialSound *sound)
200{
201 QMutexLocker l(&mutex);
202 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
203
204 resonanceAudio->api->DestroySource(id: sd->sourceId);
205 sd->sourceId = vraudio::ResonanceAudioApi::kInvalidSourceId;
206 sources.removeOne(t: sound);
207}
208
209void QAudioEnginePrivate::addStereoSound(QAmbientSound *sound)
210{
211 QMutexLocker l(&mutex);
212 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
213
214 sd->sourceId = resonanceAudio->api->CreateStereoSource(num_channels: 2);
215 stereoSources.append(t: sound);
216}
217
218void QAudioEnginePrivate::removeStereoSound(QAmbientSound *sound)
219{
220 QMutexLocker l(&mutex);
221 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
222
223 resonanceAudio->api->DestroySource(id: sd->sourceId);
224 sd->sourceId = vraudio::ResonanceAudioApi::kInvalidSourceId;
225 stereoSources.removeOne(t: sound);
226}
227
228void QAudioEnginePrivate::addRoom(QAudioRoom *room)
229{
230 QMutexLocker l(&mutex);
231 rooms.append(t: room);
232}
233
234void QAudioEnginePrivate::removeRoom(QAudioRoom *room)
235{
236 QMutexLocker l(&mutex);
237 rooms.removeOne(t: room);
238}
239
240// This method is called from the audio thread
241void QAudioEnginePrivate::updateRooms()
242{
243 if (!roomEffectsEnabled)
244 return;
245
246 bool needUpdate = listenerPositionDirty;
247 listenerPositionDirty = false;
248
249 bool roomDirty = false;
250 for (const auto &room : std::as_const(t&: rooms)) {
251 auto *rd = QAudioRoomPrivate::get(r: room);
252 if (rd->dirty) {
253 roomDirty = true;
254 rd->update();
255 needUpdate = true;
256 }
257 }
258
259 if (!needUpdate)
260 return;
261
262 QVector3D listenerPos = listenerPosition();
263 float roomVolume = float(qInf());
264 QAudioRoom *room = nullptr;
265 // Find the smallest room that contains the listener and apply its room effects
266 for (auto *r : std::as_const(t&: rooms)) {
267 QVector3D dim2 = r->dimensions()/2.;
268 float vol = dim2.x()*dim2.y()*dim2.z();
269 if (vol > roomVolume)
270 continue;
271 QVector3D dist = r->position() - listenerPos;
272 // transform into room coordinates
273 dist = r->rotation().rotatedVector(vector: dist);
274 if (qAbs(t: dist.x()) <= dim2.x() &&
275 qAbs(t: dist.y()) <= dim2.y() &&
276 qAbs(t: dist.z()) <= dim2.z()) {
277 room = r;
278 roomVolume = vol;
279 }
280 }
281 if (room != currentRoom)
282 roomDirty = true;
283 const bool previousRoom = currentRoom;
284 currentRoom = room;
285
286 if (!roomDirty)
287 return;
288
289 // apply room to engine
290 if (!currentRoom) {
291 resonanceAudio->api->EnableRoomEffects(enable: false);
292 return;
293 }
294 if (!previousRoom)
295 resonanceAudio->api->EnableRoomEffects(enable: true);
296
297 QAudioRoomPrivate *rp = QAudioRoomPrivate::get(r: room);
298 resonanceAudio->api->SetReflectionProperties(rp->reflections);
299 resonanceAudio->api->SetReverbProperties(rp->reverb);
300
301 // update room effects for all sound sources
302 for (auto *s : std::as_const(t&: sources)) {
303 auto *sp = QSpatialSoundPrivate::get(soundSource: s);
304 if (!sp)
305 continue;
306 sp->updateRoomEffects();
307 }
308}
309
310QVector3D QAudioEnginePrivate::listenerPosition() const
311{
312 return listener ? listener->position() : QVector3D();
313}
314
315
316/*!
317 \class QAudioEngine
318 \inmodule QtSpatialAudio
319 \ingroup spatialaudio
320 \ingroup multimedia_audio
321
322 \brief QAudioEngine manages a three dimensional sound field.
323
324 You can use an instance of QAudioEngine to manage a sound field in
325 three dimensions. A sound field is defined by several QSpatialSound
326 objects that define a sound at a specified location in 3D space. You can also
327 add stereo overlays using QAmbientSound.
328
329 You can use QAudioListener to define the position of the person listening
330 to the sound field relative to the sound sources. Sound sources will be less audible
331 if the listener is further away from source. They will also get mapped to the corresponding
332 loudspeakers depending on the direction between listener and source.
333
334 QAudioEngine offers two output modes. The first mode renders the sound field to a set of
335 speakers, either a stereo speaker pair or a surround configuration. The second mode provides
336 an immersive 3D sound experience when using headphones.
337
338 Perception of sound localization is driven mainly by two factors. The first factor is timing
339 differences of the sound waves between left and right ear. The second factor comes from various
340 ways how sounds coming from different direcations create different types of reflections from our
341 ears and heads. See https://en.wikipedia.org/wiki/Sound_localization for more details.
342
343 The spatial audio engine emulates those timing differences and reflections through
344 Head related transfer functions (HRTF, see
345 https://en.wikipedia.org/wiki/Head-related_transfer_function). The functions used emulates those
346 effects for an average persons ears and head. It provides a good and immersive 3D sound localization
347 experience for most persons when using headphones.
348
349 The engine is rather versatile allowing you to define room properties and reverb settings to emulate
350 different types of rooms.
351
352 Sound sources can also be occluded dampening the sound coming from those sources.
353
354 The audio engine uses a coordinate system that is in centimeters by default. The axes are aligned with the
355 typical coordinate system used in 3D. Positive x points to the right, positive y points up and positive z points
356 backwards.
357
358*/
359
360/*!
361 \fn QAudioEngine::QAudioEngine()
362 \fn QAudioEngine::QAudioEngine(QObject *parent)
363 \fn QAudioEngine::QAudioEngine(int sampleRate, QObject *parent = nullptr)
364
365 Constructs a spatial audio engine with \a parent, if any.
366
367 The engine will operate with a sample rate given by \a sampleRate. The
368 default sample rate, if none is provided, is 44100 (44.1kHz).
369
370 Sound content that is not provided at that sample rate will automatically
371 get resampled to \a sampleRate when being processed by the engine. The
372 default sample rate is fine in most cases, but you can define a different
373 rate if most of your sound files are sampled with a different rate, and
374 avoid some CPU overhead for resampling.
375 */
376QAudioEngine::QAudioEngine(int sampleRate, QObject *parent)
377 : QObject(parent)
378 , d(new QAudioEnginePrivate)
379{
380 d->sampleRate = sampleRate;
381 d->resonanceAudio = new vraudio::ResonanceAudio(2, QAudioEnginePrivate::bufferSize, d->sampleRate);
382}
383
384/*!
385 Destroys the spatial audio engine.
386 */
387QAudioEngine::~QAudioEngine()
388{
389 stop();
390 delete d;
391}
392
393/*! \enum QAudioEngine::OutputMode
394 \value Surround Map the sounds to the loudspeaker configuration of the output device.
395 This is normally a stereo or surround speaker setup.
396 \value Stereo Map the sounds to the stereo loudspeaker configuration of the output device.
397 This will ignore any additional speakers and only use the left and right channels
398 to create a stero rendering of the sound field.
399 \value Headphone Use Headphone spatialization to create a 3D audio effect when listening
400 to the sound field through headphones
401*/
402
403/*!
404 \property QAudioEngine::outputMode
405
406 Sets or retrieves the current output mode of the engine.
407
408 \sa QAudioEngine::OutputMode
409 */
410void QAudioEngine::setOutputMode(OutputMode mode)
411{
412 if (d->outputMode == mode)
413 return;
414 d->outputMode = mode;
415 if (d->resonanceAudio->api)
416 d->resonanceAudio->api->SetStereoSpeakerMode(mode != Headphone);
417
418 QMetaObject::invokeMethod(obj: d->outputStream.get(), member: "restartOutput", c: Qt::BlockingQueuedConnection);
419
420 emit outputModeChanged();
421}
422
423QAudioEngine::OutputMode QAudioEngine::outputMode() const
424{
425 return d->outputMode;
426}
427
428/*!
429 Returns the sample rate the engine has been configured with.
430 */
431int QAudioEngine::sampleRate() const
432{
433 return d->sampleRate;
434}
435
436/*!
437 \property QAudioEngine::outputDevice
438
439 Sets or returns the device that is being used for playing the sound field.
440 */
441void QAudioEngine::setOutputDevice(const QAudioDevice &device)
442{
443 if (d->device == device)
444 return;
445 if (d->resonanceAudio->api) {
446 qWarning() << "Changing device on a running engine not implemented";
447 return;
448 }
449 d->device = device;
450 emit outputDeviceChanged();
451}
452
453QAudioDevice QAudioEngine::outputDevice() const
454{
455 return d->device;
456}
457
458/*!
459 \property QAudioEngine::masterVolume
460
461 Sets or returns volume being used to render the sound field.
462 */
463void QAudioEngine::setMasterVolume(float volume)
464{
465 if (d->masterVolume == volume)
466 return;
467 d->masterVolume = volume;
468 d->resonanceAudio->api->SetMasterVolume(volume);
469 emit masterVolumeChanged();
470}
471
472float QAudioEngine::masterVolume() const
473{
474 return d->masterVolume;
475}
476
477/*!
478 Starts the engine.
479 */
480void QAudioEngine::start()
481{
482 if (d->outputStream)
483 // already started
484 return;
485
486 d->resonanceAudio->api->SetStereoSpeakerMode(d->outputMode != Headphone);
487 d->resonanceAudio->api->SetMasterVolume(d->masterVolume);
488
489 d->outputStream.reset(p: new QAudioOutputStream(d));
490 d->outputStream->moveToThread(thread: &d->audioThread);
491 d->audioThread.start(QThread::TimeCriticalPriority);
492
493 QMetaObject::invokeMethod(obj: d->outputStream.get(), member: "startOutput");
494}
495
496/*!
497 Stops the engine.
498 */
499void QAudioEngine::stop()
500{
501 QMetaObject::invokeMethod(obj: d->outputStream.get(), member: "stopOutput", c: Qt::BlockingQueuedConnection);
502 d->outputStream.reset();
503 d->audioThread.exit(retcode: 0);
504 d->audioThread.wait();
505 delete d->resonanceAudio->api;
506 d->resonanceAudio->api = nullptr;
507}
508
509/*!
510 \property QAudioEngine::paused
511
512 Pauses the spatial audio engine.
513 */
514void QAudioEngine::setPaused(bool paused)
515{
516 bool old = d->paused.fetchAndStoreRelaxed(newValue: paused);
517 if (old != paused) {
518 if (d->outputStream)
519 d->outputStream->setPaused(paused);
520 emit pausedChanged();
521 }
522}
523
524bool QAudioEngine::paused() const
525{
526 return d->paused.loadRelaxed();
527}
528
529/*!
530 Enables room effects such as echos and reverb.
531
532 Enables room effects if \a enabled is true.
533 Room effects will only apply if you create one or more \l QAudioRoom objects
534 and the listener is inside at least one of the rooms. If the listener is inside
535 multiple rooms, the room with the smallest volume will be used.
536 */
537void QAudioEngine::setRoomEffectsEnabled(bool enabled)
538{
539 if (d->roomEffectsEnabled == enabled)
540 return;
541 d->roomEffectsEnabled = enabled;
542 d->resonanceAudio->roomEffectsEnabled = enabled;
543}
544
545/*!
546 Returns true if room effects are enabled.
547 */
548bool QAudioEngine::roomEffectsEnabled() const
549{
550 return d->roomEffectsEnabled;
551}
552
553/*!
554 \property QAudioEngine::distanceScale
555
556 Defines the scale of the coordinate system being used by the spatial audio engine.
557 By default, all units are in centimeters, in line with the default units being
558 used by Qt Quick 3D.
559
560 Set the distance scale to QAudioEngine::DistanceScaleMeter to get units in meters.
561*/
562void QAudioEngine::setDistanceScale(float scale)
563{
564 // multiply with 100, to get the conversion to meters that resonance audio uses
565 scale /= 100.f;
566 if (scale <= 0.0f) {
567 qWarning() << "QAudioEngine: Invalid distance scale.";
568 return;
569 }
570 if (scale == d->distanceScale)
571 return;
572 d->distanceScale = scale;
573 emit distanceScaleChanged();
574}
575
576float QAudioEngine::distanceScale() const
577{
578 return d->distanceScale*100.f;
579}
580
581/*!
582 \fn void QAudioEngine::pause()
583
584 Pauses playback.
585*/
586/*!
587 \fn void QAudioEngine::resume()
588
589 Resumes playback.
590*/
591/*!
592 \variable QAudioEngine::DistanceScaleCentimeter
593 \internal
594*/
595/*!
596 \variable QAudioEngine::DistanceScaleMeter
597 \internal
598*/
599
600QT_END_NAMESPACE
601
602#include "moc_qaudioengine.cpp"
603#include "qaudioengine.moc"
604

source code of qtmultimedia/src/spatialaudio/qaudioengine.cpp