1// Copyright (C) 2022 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only
3#include <qaudioengine_p.h>
4#include <qambientsound_p.h>
5#include <qspatialsound_p.h>
6#include <qambientsound.h>
7#include <qaudioroom_p.h>
8#include <qaudiolistener.h>
9#include <resonance_audio.h>
10#include <qambisonicdecoder_p.h>
11#include <qaudiodecoder.h>
12#include <qmediadevices.h>
13#include <qiodevice.h>
14#include <qaudiosink.h>
15#include <qdebug.h>
16#include <qelapsedtimer.h>
17
18#include <QFile>
19
20QT_BEGIN_NAMESPACE
21
22// We'd like to have short buffer times, so the sound adjusts itself to changes
23// quickly, but times below 100ms seem to give stuttering on macOS.
24// It might be possible to set this value lower on other OSes.
25const int bufferTimeMs = 100;
26
27// This class lives in the audioThread, but pulls data from QAudioEnginePrivate
28// which lives in the mainThread.
29class QAudioOutputStream : public QIODevice
30{
31 Q_OBJECT
32public:
33 explicit QAudioOutputStream(QAudioEnginePrivate *d)
34 : d(d)
35 {
36 open(mode: QIODevice::ReadOnly);
37 }
38 ~QAudioOutputStream();
39
40 qint64 readData(char *data, qint64 len) override;
41
42 qint64 writeData(const char *, qint64) override;
43
44 qint64 size() const override { return 0; }
45 qint64 bytesAvailable() const override {
46 return std::numeric_limits<qint64>::max();
47 }
48 bool isSequential() const override {
49 return true;
50 }
51 bool atEnd() const override {
52 return false;
53 }
54 qint64 pos() const override {
55 return m_pos;
56 }
57
58 Q_INVOKABLE void startOutput() {
59 d->mutex.lock();
60 Q_ASSERT(!sink);
61 QAudioFormat format;
62 auto channelConfig = d->outputMode == QAudioEngine::Surround ?
63 d->device.channelConfiguration() : QAudioFormat::ChannelConfigStereo;
64 if (channelConfig != QAudioFormat::ChannelConfigUnknown)
65 format.setChannelConfig(channelConfig);
66 else
67 format.setChannelCount(d->device.preferredFormat().channelCount());
68 format.setSampleRate(d->sampleRate);
69 format.setSampleFormat(QAudioFormat::Int16);
70 ambisonicDecoder.reset(p: new QAmbisonicDecoder(QAmbisonicDecoder::HighQuality, format));
71 sink.reset(p: new QAudioSink(d->device, format));
72 const qsizetype bufferSize = format.bytesForDuration(microseconds: bufferTimeMs * 1000);
73 sink->setBufferSize(bufferSize);
74 d->mutex.unlock();
75 // It is important to unlock the mutex before starting the sink, as the sink will
76 // call readData() in the audio thread, which will try to lock the mutex (again)
77 sink->start(device: this);
78 }
79
80 Q_INVOKABLE void stopOutput() {
81 sink->stop();
82 sink.reset();
83 ambisonicDecoder.reset();
84 }
85
86 Q_INVOKABLE void restartOutput() {
87 stopOutput();
88 startOutput();
89 }
90
91 void setPaused(bool paused) {
92 if (paused)
93 sink->suspend();
94 else
95 sink->resume();
96 }
97
98private:
99 qint64 m_pos = 0;
100 QAudioEnginePrivate *d = nullptr;
101 std::unique_ptr<QAudioSink> sink;
102 std::unique_ptr<QAmbisonicDecoder> ambisonicDecoder;
103};
104
105
106QAudioOutputStream::~QAudioOutputStream()
107{
108}
109
110qint64 QAudioOutputStream::writeData(const char *, qint64)
111{
112 return 0;
113}
114
115qint64 QAudioOutputStream::readData(char *data, qint64 len)
116{
117 if (d->paused.loadRelaxed())
118 return 0;
119
120 QMutexLocker l(&d->mutex);
121 d->updateRooms();
122
123 int nChannels = ambisonicDecoder ? ambisonicDecoder->nOutputChannels() : 2;
124 if (len < nChannels*int(sizeof(float))*QAudioEnginePrivate::bufferSize)
125 return 0;
126
127 short *fd = (short *)data;
128 qint64 frames = len / nChannels / sizeof(short);
129 bool ok = true;
130 while (frames >= qint64(QAudioEnginePrivate::bufferSize)) {
131 // Fill input buffers
132 for (auto *source : std::as_const(t&: d->sources)) {
133 auto *sp = QSpatialSoundPrivate::get(soundSource: source);
134 if (!sp)
135 continue;
136 float buf[QAudioEnginePrivate::bufferSize];
137 sp->getBuffer(buf, frames: QAudioEnginePrivate::bufferSize, channels: 1);
138 d->resonanceAudio->api->SetInterleavedBuffer(source_id: sp->sourceId, audio_buffer_ptr: buf, num_channels: 1, num_frames: QAudioEnginePrivate::bufferSize);
139 }
140 for (auto *source : std::as_const(t&: d->stereoSources)) {
141 auto *sp = QAmbientSoundPrivate::get(soundSource: source);
142 if (!sp)
143 continue;
144 float buf[2*QAudioEnginePrivate::bufferSize];
145 sp->getBuffer(buf, frames: QAudioEnginePrivate::bufferSize, channels: 2);
146 d->resonanceAudio->api->SetInterleavedBuffer(source_id: sp->sourceId, audio_buffer_ptr: buf, num_channels: 2, num_frames: QAudioEnginePrivate::bufferSize);
147 }
148
149 if (ambisonicDecoder && d->outputMode == QAudioEngine::Surround) {
150 const float *channels[QAmbisonicDecoder::maxAmbisonicChannels];
151 const float *reverbBuffers[2];
152 int nSamples = d->resonanceAudio->getAmbisonicOutput(buffers: channels, reverb: reverbBuffers, nChannels: ambisonicDecoder->nInputChannels());
153 Q_ASSERT(ambisonicDecoder->nOutputChannels() <= 8);
154 ambisonicDecoder->processBufferWithReverb(input: channels, reverb: reverbBuffers, output: fd, nSamples);
155 } else {
156 ok = d->resonanceAudio->api->FillInterleavedOutputBuffer(num_channels: 2, num_frames: QAudioEnginePrivate::bufferSize, buffer_ptr: fd);
157 if (!ok) {
158 // If we get here, it means that resonanceAudio did not actually fill the buffer.
159 // Sometimes this is expected, for example if resonanceAudio does not have any sources.
160 // In this case we just fill the buffer with silence.
161 if (d->sources.isEmpty() && d->stereoSources.isEmpty()) {
162 memset(s: fd, c: 0, n: nChannels * QAudioEnginePrivate::bufferSize * sizeof(short));
163 } else {
164 // If we get here, it means that something unexpected happened, so bail.
165 qWarning() << " Reading failed!";
166 break;
167 }
168 }
169 }
170 fd += nChannels*QAudioEnginePrivate::bufferSize;
171 frames -= QAudioEnginePrivate::bufferSize;
172 }
173 const int bytesProcessed = ((char *)fd - data);
174 m_pos += bytesProcessed;
175 return bytesProcessed;
176}
177
178
179QAudioEnginePrivate::QAudioEnginePrivate()
180{
181 device = QMediaDevices::defaultAudioOutput();
182}
183
184QAudioEnginePrivate::~QAudioEnginePrivate()
185{
186 delete resonanceAudio;
187}
188
189void QAudioEnginePrivate::addSpatialSound(QSpatialSound *sound)
190{
191 QMutexLocker l(&mutex);
192 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
193
194 sd->sourceId = resonanceAudio->api->CreateSoundObjectSource(rendering_mode: vraudio::kBinauralHighQuality);
195 sources.append(t: sound);
196}
197
198void QAudioEnginePrivate::removeSpatialSound(QSpatialSound *sound)
199{
200 QMutexLocker l(&mutex);
201 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
202
203 resonanceAudio->api->DestroySource(id: sd->sourceId);
204 sd->sourceId = vraudio::ResonanceAudioApi::kInvalidSourceId;
205 sources.removeOne(t: sound);
206}
207
208void QAudioEnginePrivate::addStereoSound(QAmbientSound *sound)
209{
210 QMutexLocker l(&mutex);
211 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
212
213 sd->sourceId = resonanceAudio->api->CreateStereoSource(num_channels: 2);
214 stereoSources.append(t: sound);
215}
216
217void QAudioEnginePrivate::removeStereoSound(QAmbientSound *sound)
218{
219 QMutexLocker l(&mutex);
220 QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(soundSource: sound);
221
222 resonanceAudio->api->DestroySource(id: sd->sourceId);
223 sd->sourceId = vraudio::ResonanceAudioApi::kInvalidSourceId;
224 stereoSources.removeOne(t: sound);
225}
226
227void QAudioEnginePrivate::addRoom(QAudioRoom *room)
228{
229 QMutexLocker l(&mutex);
230 rooms.append(t: room);
231}
232
233void QAudioEnginePrivate::removeRoom(QAudioRoom *room)
234{
235 QMutexLocker l(&mutex);
236 rooms.removeOne(t: room);
237}
238
239// This method is called from the audio thread
240void QAudioEnginePrivate::updateRooms()
241{
242 if (!roomEffectsEnabled)
243 return;
244
245 bool needUpdate = listenerPositionDirty;
246 listenerPositionDirty = false;
247
248 bool roomDirty = false;
249 for (const auto &room : rooms) {
250 auto *rd = QAudioRoomPrivate::get(r: room);
251 if (rd->dirty) {
252 roomDirty = true;
253 rd->update();
254 needUpdate = true;
255 }
256 }
257
258 if (!needUpdate)
259 return;
260
261 QVector3D listenerPos = listenerPosition();
262 float roomVolume = float(qInf());
263 QAudioRoom *room = nullptr;
264 // Find the smallest room that contains the listener and apply its room effects
265 for (auto *r : std::as_const(t&: rooms)) {
266 QVector3D dim2 = r->dimensions()/2.;
267 float vol = dim2.x()*dim2.y()*dim2.z();
268 if (vol > roomVolume)
269 continue;
270 QVector3D dist = r->position() - listenerPos;
271 // transform into room coordinates
272 dist = r->rotation().rotatedVector(vector: dist);
273 if (qAbs(t: dist.x()) <= dim2.x() &&
274 qAbs(t: dist.y()) <= dim2.y() &&
275 qAbs(t: dist.z()) <= dim2.z()) {
276 room = r;
277 roomVolume = vol;
278 }
279 }
280 if (room != currentRoom)
281 roomDirty = true;
282 const bool previousRoom = currentRoom;
283 currentRoom = room;
284
285 if (!roomDirty)
286 return;
287
288 // apply room to engine
289 if (!currentRoom) {
290 resonanceAudio->api->EnableRoomEffects(enable: false);
291 return;
292 }
293 if (!previousRoom)
294 resonanceAudio->api->EnableRoomEffects(enable: true);
295
296 QAudioRoomPrivate *rp = QAudioRoomPrivate::get(r: room);
297 resonanceAudio->api->SetReflectionProperties(rp->reflections);
298 resonanceAudio->api->SetReverbProperties(rp->reverb);
299
300 // update room effects for all sound sources
301 for (auto *s : std::as_const(t&: sources)) {
302 auto *sp = QSpatialSoundPrivate::get(soundSource: s);
303 if (!sp)
304 continue;
305 sp->updateRoomEffects();
306 }
307}
308
309QVector3D QAudioEnginePrivate::listenerPosition() const
310{
311 return listener ? listener->position() : QVector3D();
312}
313
314
315/*!
316 \class QAudioEngine
317 \inmodule QtSpatialAudio
318 \ingroup spatialaudio
319 \ingroup multimedia_audio
320
321 \brief QAudioEngine manages a three dimensional sound field.
322
323 You can use an instance of QAudioEngine to manage a sound field in
324 three dimensions. A sound field is defined by several QSpatialSound
325 objects that define a sound at a specified location in 3D space. You can also
326 add stereo overlays using QAmbientSound.
327
328 You can use QAudioListener to define the position of the person listening
329 to the sound field relative to the sound sources. Sound sources will be less audible
330 if the listener is further away from source. They will also get mapped to the corresponding
331 loudspeakers depending on the direction between listener and source.
332
333 QAudioEngine offers two output modes. The first mode renders the sound field to a set of
334 speakers, either a stereo speaker pair or a surround configuration. The second mode provides
335 an immersive 3D sound experience when using headphones.
336
337 Perception of sound localization is driven mainly by two factors. The first factor is timing
338 differences of the sound waves between left and right ear. The second factor comes from various
339 ways how sounds coming from different direcations create different types of reflections from our
340 ears and heads. See https://en.wikipedia.org/wiki/Sound_localization for more details.
341
342 The spatial audio engine emulates those timing differences and reflections through
343 Head related transfer functions (HRTF, see
344 https://en.wikipedia.org/wiki/Head-related_transfer_function). The functions used emulates those
345 effects for an average persons ears and head. It provides a good and immersive 3D sound localization
346 experience for most persons when using headphones.
347
348 The engine is rather versatile allowing you to define room properties and reverb settings to emulate
349 different types of rooms.
350
351 Sound sources can also be occluded dampening the sound coming from those sources.
352
353 The audio engine uses a coordinate system that is in centimeters by default. The axes are aligned with the
354 typical coordinate system used in 3D. Positive x points to the right, positive y points up and positive z points
355 backwards.
356
357*/
358
359/*!
360 \fn QAudioEngine::QAudioEngine()
361 \fn QAudioEngine::QAudioEngine(QObject *parent)
362 \fn QAudioEngine::QAudioEngine(int sampleRate, QObject *parent = nullptr)
363
364 Constructs a spatial audio engine with \a parent, if any.
365
366 The engine will operate with a sample rate given by \a sampleRate. The
367 default sample rate, if none is provided, is 44100 (44.1kHz).
368
369 Sound content that is not provided at that sample rate will automatically
370 get resampled to \a sampleRate when being processed by the engine. The
371 default sample rate is fine in most cases, but you can define a different
372 rate if most of your sound files are sampled with a different rate, and
373 avoid some CPU overhead for resampling.
374 */
375QAudioEngine::QAudioEngine(int sampleRate, QObject *parent)
376 : QObject(parent)
377 , d(new QAudioEnginePrivate)
378{
379 d->sampleRate = sampleRate;
380 d->resonanceAudio = new vraudio::ResonanceAudio(2, QAudioEnginePrivate::bufferSize, d->sampleRate);
381}
382
383/*!
384 Destroys the spatial audio engine.
385 */
386QAudioEngine::~QAudioEngine()
387{
388 stop();
389 delete d;
390}
391
392/*! \enum QAudioEngine::OutputMode
393 \value Surround Map the sounds to the loudspeaker configuration of the output device.
394 This is normally a stereo or surround speaker setup.
395 \value Stereo Map the sounds to the stereo loudspeaker configuration of the output device.
396 This will ignore any additional speakers and only use the left and right channels
397 to create a stero rendering of the sound field.
398 \value Headphone Use Headphone spatialization to create a 3D audio effect when listening
399 to the sound field through headphones
400*/
401
402/*!
403 \property QAudioEngine::outputMode
404
405 Sets or retrieves the current output mode of the engine.
406
407 \sa QAudioEngine::OutputMode
408 */
409void QAudioEngine::setOutputMode(OutputMode mode)
410{
411 if (d->outputMode == mode)
412 return;
413 d->outputMode = mode;
414 if (d->resonanceAudio->api)
415 d->resonanceAudio->api->SetStereoSpeakerMode(mode != Headphone);
416
417 QMetaObject::invokeMethod(obj: d->outputStream.get(), member: "restartOutput", c: Qt::BlockingQueuedConnection);
418
419 emit outputModeChanged();
420}
421
422QAudioEngine::OutputMode QAudioEngine::outputMode() const
423{
424 return d->outputMode;
425}
426
427/*!
428 Returns the sample rate the engine has been configured with.
429 */
430int QAudioEngine::sampleRate() const
431{
432 return d->sampleRate;
433}
434
435/*!
436 \property QAudioEngine::outputDevice
437
438 Sets or returns the device that is being used for playing the sound field.
439 */
440void QAudioEngine::setOutputDevice(const QAudioDevice &device)
441{
442 if (d->device == device)
443 return;
444 if (d->resonanceAudio->api) {
445 qWarning() << "Changing device on a running engine not implemented";
446 return;
447 }
448 d->device = device;
449 emit outputDeviceChanged();
450}
451
452QAudioDevice QAudioEngine::outputDevice() const
453{
454 return d->device;
455}
456
457/*!
458 \property QAudioEngine::masterVolume
459
460 Sets or returns volume being used to render the sound field.
461 */
462void QAudioEngine::setMasterVolume(float volume)
463{
464 if (d->masterVolume == volume)
465 return;
466 d->masterVolume = volume;
467 d->resonanceAudio->api->SetMasterVolume(volume);
468 emit masterVolumeChanged();
469}
470
471float QAudioEngine::masterVolume() const
472{
473 return d->masterVolume;
474}
475
476/*!
477 Starts the engine.
478 */
479void QAudioEngine::start()
480{
481 if (d->outputStream)
482 // already started
483 return;
484
485 d->resonanceAudio->api->SetStereoSpeakerMode(d->outputMode != Headphone);
486 d->resonanceAudio->api->SetMasterVolume(d->masterVolume);
487
488 d->outputStream.reset(p: new QAudioOutputStream(d));
489 d->outputStream->moveToThread(thread: &d->audioThread);
490 d->audioThread.start(QThread::TimeCriticalPriority);
491
492 QMetaObject::invokeMethod(obj: d->outputStream.get(), member: "startOutput");
493}
494
495/*!
496 Stops the engine.
497 */
498void QAudioEngine::stop()
499{
500 QMetaObject::invokeMethod(obj: d->outputStream.get(), member: "stopOutput", c: Qt::BlockingQueuedConnection);
501 d->outputStream.reset();
502 d->audioThread.exit(retcode: 0);
503 d->audioThread.wait();
504 delete d->resonanceAudio->api;
505 d->resonanceAudio->api = nullptr;
506}
507
508/*!
509 \property QAudioEngine::paused
510
511 Pauses the spatial audio engine.
512 */
513void QAudioEngine::setPaused(bool paused)
514{
515 bool old = d->paused.fetchAndStoreRelaxed(newValue: paused);
516 if (old != paused) {
517 if (d->outputStream)
518 d->outputStream->setPaused(paused);
519 emit pausedChanged();
520 }
521}
522
523bool QAudioEngine::paused() const
524{
525 return d->paused.loadRelaxed();
526}
527
528/*!
529 Enables room effects such as echos and reverb.
530
531 Enables room effects if \a enabled is true.
532 Room effects will only apply if you create one or more \l QAudioRoom objects
533 and the listener is inside at least one of the rooms. If the listener is inside
534 multiple rooms, the room with the smallest volume will be used.
535 */
536void QAudioEngine::setRoomEffectsEnabled(bool enabled)
537{
538 if (d->roomEffectsEnabled == enabled)
539 return;
540 d->roomEffectsEnabled = enabled;
541 d->resonanceAudio->roomEffectsEnabled = enabled;
542}
543
544/*!
545 Returns true if room effects are enabled.
546 */
547bool QAudioEngine::roomEffectsEnabled() const
548{
549 return d->roomEffectsEnabled;
550}
551
552/*!
553 \property QAudioEngine::distanceScale
554
555 Defines the scale of the coordinate system being used by the spatial audio engine.
556 By default, all units are in centimeters, in line with the default units being
557 used by Qt Quick 3D.
558
559 Set the distance scale to QAudioEngine::DistanceScaleMeter to get units in meters.
560*/
561void QAudioEngine::setDistanceScale(float scale)
562{
563 // multiply with 100, to get the conversion to meters that resonance audio uses
564 scale /= 100.f;
565 if (scale <= 0.0f) {
566 qWarning() << "QAudioEngine: Invalid distance scale.";
567 return;
568 }
569 if (scale == d->distanceScale)
570 return;
571 d->distanceScale = scale;
572 emit distanceScaleChanged();
573}
574
575float QAudioEngine::distanceScale() const
576{
577 return d->distanceScale*100.f;
578}
579
580/*!
581 \fn void QAudioEngine::pause()
582
583 Pauses playback.
584*/
585/*!
586 \fn void QAudioEngine::resume()
587
588 Resumes playback.
589*/
590/*!
591 \variable QAudioEngine::DistanceScaleCentimeter
592 \internal
593*/
594/*!
595 \variable QAudioEngine::DistanceScaleMeter
596 \internal
597*/
598
599QT_END_NAMESPACE
600
601#include "moc_qaudioengine.cpp"
602#include "qaudioengine.moc"
603

Provided by KDAB

Privacy Policy
Start learning QML with our Intro Training
Find out more

source code of qtmultimedia/src/spatialaudio/qaudioengine.cpp