1// Copyright (C) 2022 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only
3#include <qaudioengine_p.h>
4#include <qspatialsound_p.h>
5#include <qambientsound.h>
6#include <qaudioroom_p.h>
7#include <qaudiolistener.h>
8#include <resonance_audio.h>
9#include <qambisonicdecoder_p.h>
10#include <qaudiodecoder.h>
11#include <qmediadevices.h>
12#include <qiodevice.h>
13#include <qaudiosink.h>
14#include <qdebug.h>
15#include <qelapsedtimer.h>
16
17#include <QFile>
18
19QT_BEGIN_NAMESPACE
20
21// We'd like to have short buffer times, so the sound adjusts itself to changes
22// quickly, but times below 100ms seem to give stuttering on macOS.
23// It might be possible to set this value lower on other OSes.
24const int bufferTimeMs = 100;
25
26class QAudioOutputStream : public QIODevice
27{
28 Q_OBJECT
29public:
30 explicit QAudioOutputStream(QAudioEnginePrivate *d)
31 : d(d)
32 {
33 open(mode: QIODevice::ReadOnly);
34 }
35 ~QAudioOutputStream();
36
37 qint64 readData(char *data, qint64 len) override;
38
39 qint64 writeData(const char *, qint64) override;
40
41 qint64 size() const override { return 0; }
42 qint64 bytesAvailable() const override {
43 return std::numeric_limits<qint64>::max();
44 }
45 bool isSequential() const override {
46 return true;
47 }
48 bool atEnd() const override {
49 return false;
50 }
51 qint64 pos() const override {
52 return m_pos;
53 }
54
55 Q_INVOKABLE void startOutput() {
56 QMutexLocker l(&d->mutex);
57 Q_ASSERT(!sink);
58 QAudioFormat format;
59 format.setChannelConfig(d->outputMode == QAudioEngine::Surround ?
60 d->device.channelConfiguration() : QAudioFormat::ChannelConfigStereo);
61 format.setSampleRate(d->sampleRate);
62 format.setSampleFormat(QAudioFormat::Int16);
63 d->ambisonicDecoder.reset(p: new QAmbisonicDecoder(QAmbisonicDecoder::HighQuality, format));
64 sink.reset(p: new QAudioSink(d->device, format));
65 sink->setBufferSize(d->sampleRate*bufferTimeMs/1000*sizeof(qint16)*format.channelCount());
66 sink->start(device: this);
67 }
68
69 Q_INVOKABLE void stopOutput() {
70 sink->stop();
71 sink.reset();
72 d->ambisonicDecoder.reset();
73 }
74
75 Q_INVOKABLE void restartOutput() {
76 stopOutput();
77 startOutput();
78 }
79
80 void setPaused(bool paused) {
81 if (paused)
82 sink->suspend();
83 else
84 sink->resume();
85 }
86
87private:
88 qint64 m_pos = 0;
89 QAudioEnginePrivate *d = nullptr;
90 std::unique_ptr<QAudioSink> sink;
91};
92
93
94QAudioOutputStream::~QAudioOutputStream()
95{
96}
97
98qint64 QAudioOutputStream::writeData(const char *, qint64)
99{
100 return 0;
101}
102
103qint64 QAudioOutputStream::readData(char *data, qint64 len)
104{
105 if (d->paused.loadRelaxed())
106 return 0;
107
108 d->updateRooms();
109
110 int nChannels = d->ambisonicDecoder ? d->ambisonicDecoder->nOutputChannels() : 2;
111 if (len < nChannels*int(sizeof(float))*QAudioEnginePrivate::bufferSize)
112 return 0;
113
114 short *fd = (short *)data;
115 qint64 frames = len / nChannels / sizeof(short);
116 bool ok = true;
117 while (frames >= qint64(QAudioEnginePrivate::bufferSize)) {
118 // Fill input buffers
119 for (auto *source : std::as_const(t&: d->sources)) {
120 auto *sp = QSpatialSoundPrivate::get(soundSource: source);
121 float buf[QAudioEnginePrivate::bufferSize];
122 sp->getBuffer(buf, frames: QAudioEnginePrivate::bufferSize, channels: 1);
123 d->resonanceAudio->api->SetInterleavedBuffer(source_id: sp->sourceId, audio_buffer_ptr: buf, num_channels: 1, num_frames: QAudioEnginePrivate::bufferSize);
124 }
125 for (auto *source : std::as_const(t&: d->stereoSources)) {
126 auto *sp = QAmbientSoundPrivate::get(soundSource: source);
127 float buf[2*QAudioEnginePrivate::bufferSize];
128 sp->getBuffer(buf, frames: QAudioEnginePrivate::bufferSize, channels: 2);
129 d->resonanceAudio->api->SetInterleavedBuffer(source_id: sp->sourceId, audio_buffer_ptr: buf, num_channels: 2, num_frames: QAudioEnginePrivate::bufferSize);
130 }
131
132 if (d->ambisonicDecoder && d->outputMode == QAudioEngine::Surround) {
133 const float *channels[QAmbisonicDecoder::maxAmbisonicChannels];
134 const float *reverbBuffers[2];
135 int nSamples = d->resonanceAudio->getAmbisonicOutput(buffers: channels, reverb: reverbBuffers, nChannels: d->ambisonicDecoder->nInputChannels());
136 Q_ASSERT(d->ambisonicDecoder->nOutputChannels() <= 8);
137 d->ambisonicDecoder->processBufferWithReverb(input: channels, reverb: reverbBuffers, output: fd, nSamples);
138 } else {
139 ok = d->resonanceAudio->api->FillInterleavedOutputBuffer(num_channels: 2, num_frames: QAudioEnginePrivate::bufferSize, buffer_ptr: fd);
140 if (!ok) {
141 qWarning() << " Reading failed!";
142 break;
143 }
144 }
145 fd += nChannels*QAudioEnginePrivate::bufferSize;
146 frames -= QAudioEnginePrivate::bufferSize;
147 }
148 const int bytesProcessed = ((char *)fd - data);
149 m_pos += bytesProcessed;
150 return bytesProcessed;
151}
152
153
154QAudioEnginePrivate::QAudioEnginePrivate()
155{
156 device = QMediaDevices::defaultAudioOutput();
157 audioThread.setPriority(QThread::TimeCriticalPriority);
158}
159
160QAudioEnginePrivate::~QAudioEnginePrivate()
161{
162 delete resonanceAudio;
163}
164
165void QAudioEnginePrivate::addSpatialSound(QSpatialSound *sound)
166{
167 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
168
169 sd->sourceId = resonanceAudio->api->CreateSoundObjectSource(rendering_mode: vraudio::kBinauralHighQuality);
170 sources.append(t: sound);
171}
172
173void QAudioEnginePrivate::removeSpatialSound(QSpatialSound *sound)
174{
175 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
176
177 resonanceAudio->api->DestroySource(id: sd->sourceId);
178 sd->sourceId = vraudio::ResonanceAudioApi::kInvalidSourceId;
179 sources.removeOne(t: sound);
180}
181
182void QAudioEnginePrivate::addStereoSound(QAmbientSound *sound)
183{
184 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
185
186 sd->sourceId = resonanceAudio->api->CreateStereoSource(num_channels: 2);
187 stereoSources.append(t: sound);
188}
189
190void QAudioEnginePrivate::removeStereoSound(QAmbientSound *sound)
191{
192 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
193
194 resonanceAudio->api->DestroySource(id: sd->sourceId);
195 sd->sourceId = vraudio::ResonanceAudioApi::kInvalidSourceId;
196 stereoSources.removeOne(t: sound);
197}
198
199void QAudioEnginePrivate::addRoom(QAudioRoom *room)
200{
201 rooms.append(t: room);
202}
203
204void QAudioEnginePrivate::removeRoom(QAudioRoom *room)
205{
206 rooms.removeOne(t: room);
207}
208
209void QAudioEnginePrivate::updateRooms()
210{
211 if (!roomEffectsEnabled)
212 return;
213
214 bool needUpdate = listenerPositionDirty;
215 listenerPositionDirty = false;
216
217 bool roomDirty = false;
218 for (const auto &room : rooms) {
219 auto *rd = QAudioRoomPrivate::get(r: room);
220 if (rd->dirty) {
221 roomDirty = true;
222 rd->update();
223 needUpdate = true;
224 }
225 }
226
227 if (!needUpdate)
228 return;
229
230 QVector3D listenerPos = listenerPosition();
231 float roomVolume = float(qInf());
232 QAudioRoom *room = nullptr;
233 // Find the smallest room that contains the listener and apply its room effects
234 for (auto *r : std::as_const(t&: rooms)) {
235 QVector3D dim2 = r->dimensions()/2.;
236 float vol = dim2.x()*dim2.y()*dim2.z();
237 if (vol > roomVolume)
238 continue;
239 QVector3D dist = r->position() - listenerPos;
240 // transform into room coordinates
241 dist = r->rotation().rotatedVector(vector: dist);
242 if (qAbs(t: dist.x()) <= dim2.x() &&
243 qAbs(t: dist.y()) <= dim2.y() &&
244 qAbs(t: dist.z()) <= dim2.z()) {
245 room = r;
246 roomVolume = vol;
247 }
248 }
249 if (room != currentRoom)
250 roomDirty = true;
251 const bool previousRoom = currentRoom;
252 currentRoom = room;
253
254 if (!roomDirty)
255 return;
256
257 // apply room to engine
258 if (!currentRoom) {
259 resonanceAudio->api->EnableRoomEffects(enable: false);
260 return;
261 }
262 if (!previousRoom)
263 resonanceAudio->api->EnableRoomEffects(enable: true);
264
265 QAudioRoomPrivate *rp = QAudioRoomPrivate::get(r: room);
266 resonanceAudio->api->SetReflectionProperties(rp->reflections);
267 resonanceAudio->api->SetReverbProperties(rp->reverb);
268
269 // update room effects for all sound sources
270 for (auto *s : std::as_const(t&: sources)) {
271 auto *sp = QSpatialSoundPrivate::get(soundSource: s);
272 sp->updateRoomEffects();
273 }
274}
275
276QVector3D QAudioEnginePrivate::listenerPosition() const
277{
278 return listener ? listener->position() : QVector3D();
279}
280
281
282/*!
283 \class QAudioEngine
284 \inmodule QtSpatialAudio
285 \ingroup spatialaudio
286 \ingroup multimedia_audio
287
288 \brief QAudioEngine manages a three dimensional sound field.
289
290 You can use an instance of QAudioEngine to manage a sound field in
291 three dimensions. A sound field is defined by several QSpatialSound
292 objects that define a sound at a specified location in 3D space. You can also
293 add stereo overlays using QAmbientSound.
294
295 You can use QAudioListener to define the position of the person listening
296 to the sound field relative to the sound sources. Sound sources will be less audible
297 if the listener is further away from source. They will also get mapped to the corresponding
298 loudspeakers depending on the direction between listener and source.
299
300 QAudioEngine offers two output modes. The first mode renders the sound field to a set of
301 speakers, either a stereo speaker pair or a surround configuration. The second mode provides
302 an immersive 3D sound experience when using headphones.
303
304 Perception of sound localization is driven mainly by two factors. The first factor is timing
305 differences of the sound waves between left and right ear. The second factor comes from various
306 ways how sounds coming from different direcations create different types of reflections from our
307 ears and heads. See https://en.wikipedia.org/wiki/Sound_localization for more details.
308
309 The spatial audio engine emulates those timing differences and reflections through
310 Head related transfer functions (HRTF, see
311 https://en.wikipedia.org/wiki/Head-related_transfer_function). The functions used emulates those
312 effects for an average persons ears and head. It provides a good and immersive 3D sound localization
313 experience for most persons when using headphones.
314
315 The engine is rather versatile allowing you to define room properties and reverb settings to emulate
316 different types of rooms.
317
318 Sound sources can also be occluded dampening the sound coming from those sources.
319
320 The audio engine uses a coordinate system that is in centimeters by default. The axes are aligned with the
321 typical coordinate system used in 3D. Positive x points to the right, positive y points up and positive z points
322 backwards.
323
324*/
325
326/*!
327 \fn QAudioEngine::QAudioEngine()
328 \fn QAudioEngine::QAudioEngine(QObject *parent)
329 \fn QAudioEngine::QAudioEngine(int sampleRate, QObject *parent = nullptr)
330
331 Constructs a spatial audio engine with \a parent, if any.
332
333 The engine will operate with a sample rate given by \a sampleRate. The
334 default sample rate, if none is provided, is 44100 (44.1kHz).
335
336 Sound content that is not provided at that sample rate will automatically
337 get resampled to \a sampleRate when being processed by the engine. The
338 default sample rate is fine in most cases, but you can define a different
339 rate if most of your sound files are sampled with a different rate, and
340 avoid some CPU overhead for resampling.
341 */
342QAudioEngine::QAudioEngine(int sampleRate, QObject *parent)
343 : QObject(parent)
344 , d(new QAudioEnginePrivate)
345{
346 d->sampleRate = sampleRate;
347 d->resonanceAudio = new vraudio::ResonanceAudio(2, QAudioEnginePrivate::bufferSize, d->sampleRate);
348}
349
350/*!
351 Destroys the spatial audio engine.
352 */
353QAudioEngine::~QAudioEngine()
354{
355 stop();
356 delete d;
357}
358
359/*! \enum QAudioEngine::OutputMode
360 \value Surround Map the sounds to the loudspeaker configuration of the output device.
361 This is normally a stereo or surround speaker setup.
362 \value Stereo Map the sounds to the stereo loudspeaker configuration of the output device.
363 This will ignore any additional speakers and only use the left and right channels
364 to create a stero rendering of the sound field.
365 \value Headphone Use Headphone spatialization to create a 3D audio effect when listening
366 to the sound field through headphones
367*/
368
369/*!
370 \property QAudioEngine::outputMode
371
372 Sets or retrieves the current output mode of the engine.
373
374 \sa QAudioEngine::OutputMode
375 */
376void QAudioEngine::setOutputMode(OutputMode mode)
377{
378 if (d->outputMode == mode)
379 return;
380 d->outputMode = mode;
381 if (d->resonanceAudio->api)
382 d->resonanceAudio->api->SetStereoSpeakerMode(mode != Headphone);
383
384 QMetaObject::invokeMethod(obj: d->outputStream.get(), member: "restartOutput", c: Qt::BlockingQueuedConnection);
385
386 emit outputModeChanged();
387}
388
389QAudioEngine::OutputMode QAudioEngine::outputMode() const
390{
391 return d->outputMode;
392}
393
394/*!
395 Returns the sample rate the engine has been configured with.
396 */
397int QAudioEngine::sampleRate() const
398{
399 return d->sampleRate;
400}
401
402/*!
403 \property QAudioEngine::outputDevice
404
405 Sets or returns the device that is being used for playing the sound field.
406 */
407void QAudioEngine::setOutputDevice(const QAudioDevice &device)
408{
409 if (d->device == device)
410 return;
411 if (d->resonanceAudio->api) {
412 qWarning() << "Changing device on a running engine not implemented";
413 return;
414 }
415 d->device = device;
416 emit outputDeviceChanged();
417}
418
419QAudioDevice QAudioEngine::outputDevice() const
420{
421 return d->device;
422}
423
424/*!
425 \property QAudioEngine::masterVolume
426
427 Sets or returns volume being used to render the sound field.
428 */
429void QAudioEngine::setMasterVolume(float volume)
430{
431 if (d->masterVolume == volume)
432 return;
433 d->masterVolume = volume;
434 d->resonanceAudio->api->SetMasterVolume(volume);
435 emit masterVolumeChanged();
436}
437
438float QAudioEngine::masterVolume() const
439{
440 return d->masterVolume;
441}
442
443/*!
444 Starts the engine.
445 */
446void QAudioEngine::start()
447{
448 if (d->outputStream)
449 // already started
450 return;
451
452 d->resonanceAudio->api->SetStereoSpeakerMode(d->outputMode != Headphone);
453 d->resonanceAudio->api->SetMasterVolume(d->masterVolume);
454
455 d->outputStream.reset(p: new QAudioOutputStream(d));
456 d->outputStream->moveToThread(thread: &d->audioThread);
457 d->audioThread.start();
458
459 QMetaObject::invokeMethod(obj: d->outputStream.get(), member: "startOutput");
460}
461
462/*!
463 Stops the engine.
464 */
465void QAudioEngine::stop()
466{
467 QMetaObject::invokeMethod(obj: d->outputStream.get(), member: "stopOutput", c: Qt::BlockingQueuedConnection);
468 d->outputStream.reset();
469 d->audioThread.exit(retcode: 0);
470 d->audioThread.wait();
471 delete d->resonanceAudio->api;
472 d->resonanceAudio->api = nullptr;
473}
474
475/*!
476 \property QAudioEngine::paused
477
478 Pauses the spatial audio engine.
479 */
480void QAudioEngine::setPaused(bool paused)
481{
482 bool old = d->paused.fetchAndStoreRelaxed(newValue: paused);
483 if (old != paused) {
484 if (d->outputStream)
485 d->outputStream->setPaused(paused);
486 emit pausedChanged();
487 }
488}
489
490bool QAudioEngine::paused() const
491{
492 return d->paused.loadRelaxed();
493}
494
495/*!
496 Enables room effects such as echos and reverb.
497
498 Enables room effects if \a enabled is true.
499 Room effects will only apply if you create one or more \l QAudioRoom objects
500 and the listener is inside at least one of the rooms. If the listener is inside
501 multiple rooms, the room with the smallest volume will be used.
502 */
503void QAudioEngine::setRoomEffectsEnabled(bool enabled)
504{
505 if (d->roomEffectsEnabled == enabled)
506 return;
507 d->roomEffectsEnabled = enabled;
508 d->resonanceAudio->roomEffectsEnabled = enabled;
509}
510
511/*!
512 Returns true if room effects are enabled.
513 */
514bool QAudioEngine::roomEffectsEnabled() const
515{
516 return d->roomEffectsEnabled;
517}
518
519/*!
520 \property QAudioEngine::distanceScale
521
522 Defines the scale of the coordinate system being used by the spatial audio engine.
523 By default, all units are in centimeters, in line with the default units being
524 used by Qt Quick 3D.
525
526 Set the distance scale to QAudioEngine::DistanceScaleMeter to get units in meters.
527*/
528void QAudioEngine::setDistanceScale(float scale)
529{
530 // multiply with 100, to get the conversion to meters that resonance audio uses
531 scale /= 100.f;
532 if (scale <= 0.0f) {
533 qWarning() << "QAudioEngine: Invalid distance scale.";
534 return;
535 }
536 if (scale == d->distanceScale)
537 return;
538 d->distanceScale = scale;
539 emit distanceScaleChanged();
540}
541
542float QAudioEngine::distanceScale() const
543{
544 return d->distanceScale*100.f;
545}
546
547
548void QAmbientSoundPrivate::load()
549{
550 decoder.reset(p: new QAudioDecoder);
551 buffers.clear();
552 currentBuffer = 0;
553 sourceDeviceFile.reset(p: nullptr);
554 bufPos = 0;
555 m_playing = false;
556 m_loading = true;
557 auto *ep = QAudioEnginePrivate::get(engine);
558 QAudioFormat f;
559 f.setSampleFormat(QAudioFormat::Float);
560 f.setSampleRate(ep->sampleRate);
561 f.setChannelConfig(nchannels == 2 ? QAudioFormat::ChannelConfigStereo : QAudioFormat::ChannelConfigMono);
562 decoder->setAudioFormat(f);
563 if (url.scheme().compare(s: u"qrc", cs: Qt::CaseInsensitive) == 0) {
564 auto qrcFile = std::make_unique<QFile>(args: u':' + url.path());
565 if (!qrcFile->open(flags: QFile::ReadOnly))
566 return;
567 sourceDeviceFile = std::move(qrcFile);
568 decoder->setSourceDevice(sourceDeviceFile.get());
569 } else {
570 decoder->setSource(url);
571 }
572 connect(sender: decoder.get(), signal: &QAudioDecoder::bufferReady, context: this, slot: &QAmbientSoundPrivate::bufferReady);
573 connect(sender: decoder.get(), signal: &QAudioDecoder::finished, context: this, slot: &QAmbientSoundPrivate::finished);
574 decoder->start();
575}
576
577void QAmbientSoundPrivate::getBuffer(float *buf, int nframes, int channels)
578{
579 Q_ASSERT(channels == nchannels);
580 QMutexLocker l(&mutex);
581 if (!m_playing || currentBuffer >= buffers.size()) {
582 memset(s: buf, c: 0, n: channels * nframes * sizeof(float));
583 } else {
584 int frames = nframes;
585 float *ff = buf;
586 while (frames) {
587 if (currentBuffer < buffers.size()) {
588 const QAudioBuffer &b = buffers.at(i: currentBuffer);
589// qDebug() << s << b.format().sampleRate() << b.format().channelCount() << b.format().sampleFormat();
590 auto *f = b.constData<float>() + bufPos*nchannels;
591 int toCopy = qMin(a: b.frameCount() - bufPos, b: frames);
592 memcpy(dest: ff, src: f, n: toCopy*sizeof(float)*nchannels);
593 ff += toCopy*nchannels;
594 frames -= toCopy;
595 bufPos += toCopy;
596 Q_ASSERT(bufPos <= b.frameCount());
597 if (bufPos == b.frameCount()) {
598 ++currentBuffer;
599 bufPos = 0;
600 }
601 } else {
602 // no more data available
603 if (m_loading)
604 qDebug() << "underrun" << frames << "frames when loading" << url;
605 memset(s: ff, c: 0, n: frames * channels * sizeof(float));
606 ff += frames * channels;
607 frames = 0;
608 }
609 if (!m_loading) {
610 if (currentBuffer == buffers.size()) {
611 currentBuffer = 0;
612 ++m_currentLoop;
613 }
614 if (m_loops > 0 && m_currentLoop >= m_loops) {
615 m_playing = false;
616 m_currentLoop = 0;
617 }
618 }
619 }
620 Q_ASSERT(ff - buf == channels*nframes);
621 }
622}
623
624void QAmbientSoundPrivate::bufferReady()
625{
626 QMutexLocker l(&mutex);
627 auto b = decoder->read();
628// qDebug() << "read buffer" << b.format() << b.startTime() << b.duration();
629 buffers.append(t: b);
630 if (m_autoPlay)
631 m_playing = true;
632}
633
634void QAmbientSoundPrivate::finished()
635{
636 m_loading = false;
637}
638
639/*!
640 \fn void QAudioEngine::pause()
641
642 Pauses playback.
643*/
644/*!
645 \fn void QAudioEngine::resume()
646
647 Resumes playback.
648*/
649/*!
650 \variable QAudioEngine::DistanceScaleCentimeter
651 \internal
652*/
653/*!
654 \variable QAudioEngine::DistanceScaleMeter
655 \internal
656*/
657
658QT_END_NAMESPACE
659
660#include "moc_qaudioengine.cpp"
661#include "qaudioengine.moc"
662

source code of qtmultimedia/src/spatialaudio/qaudioengine.cpp