1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "playbackengine/qffmpegaudiorenderer_p.h"
5#include "qaudiosink.h"
6#include "qaudiooutput.h"
7#include "qaudiobufferoutput.h"
8#include "private/qplatformaudiooutput_p.h"
9#include <QtCore/qloggingcategory.h>
10
11#include "qffmpegresampler_p.h"
12#include "qffmpegmediaformatinfo_p.h"
13
14QT_BEGIN_NAMESPACE
15
16static Q_LOGGING_CATEGORY(qLcAudioRenderer, "qt.multimedia.ffmpeg.audiorenderer");
17
18namespace QFFmpeg {
19
20using namespace std::chrono_literals;
21using namespace std::chrono;
22
23namespace {
24constexpr auto DesiredBufferTime = 110000us;
25constexpr auto MinDesiredBufferTime = 22000us;
26constexpr auto MaxDesiredBufferTime = 64000us;
27constexpr auto MinDesiredFreeBufferTime = 10000us;
28
29// It might be changed with #ifdef, as on Linux, QPulseAudioSink has quite unstable timings,
30// and it needs much more time to make sure that the buffer is overloaded.
31constexpr auto BufferLoadingMeasureTime = 400ms;
32
33constexpr auto DurationBias = 2ms; // avoids extra timer events
34
35qreal sampleRateFactor() {
36 // Test purposes:
37 //
38 // The env var describes a factor for the sample rate of
39 // audio data that we feed to the audio sink.
40 //
41 // In some cases audio sink might consume data slightly slower or faster than expected;
42 // even though the synchronization in the audio renderer is supposed to handle it,
43 // it makes sense to experiment with QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR != 1.
44 //
45 // Set QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR > 1 (e.g. 1.01 - 1.1) to test high buffer loading
46 // or try compensating too fast data consumption by the audio sink.
47 // Set QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR < 1 to test low buffer loading
48 // or try compensating too slow data consumption by the audio sink.
49
50
51 static const qreal result = []() {
52 const auto sampleRateFactorStr = qEnvironmentVariable(varName: "QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR");
53 bool ok = false;
54 const auto result = sampleRateFactorStr.toDouble(ok: &ok);
55 return ok ? result : 1.;
56 }();
57
58 return result;
59}
60
61QAudioFormat audioFormatFromFrame(const Frame &frame)
62{
63 return QFFmpegMediaFormatInfo::audioFormatFromCodecParameters(
64 codecPar: frame.codec()->stream()->codecpar);
65}
66
67std::unique_ptr<QFFmpegResampler> createResampler(const Frame &frame,
68 const QAudioFormat &outputFormat)
69{
70 return std::make_unique<QFFmpegResampler>(args: frame.codec(), args: outputFormat, args: frame.pts());
71}
72
73} // namespace
74
75AudioRenderer::AudioRenderer(const TimeController &tc, QAudioOutput *output,
76 QAudioBufferOutput *bufferOutput)
77 : Renderer(tc), m_output(output), m_bufferOutput(bufferOutput)
78{
79 if (output) {
80 // TODO: implement the signals in QPlatformAudioOutput and connect to them, QTBUG-112294
81 connect(sender: output, signal: &QAudioOutput::deviceChanged, context: this, slot: &AudioRenderer::onDeviceChanged);
82 connect(sender: output, signal: &QAudioOutput::volumeChanged, context: this, slot: &AudioRenderer::updateVolume);
83 connect(sender: output, signal: &QAudioOutput::mutedChanged, context: this, slot: &AudioRenderer::updateVolume);
84 }
85}
86
87void AudioRenderer::setOutput(QAudioOutput *output)
88{
89 setOutputInternal(actual&: m_output, desired: output, changeHandler: [this](QAudioOutput *) { onDeviceChanged(); });
90}
91
92void AudioRenderer::setOutput(QAudioBufferOutput *bufferOutput)
93{
94 setOutputInternal(actual&: m_bufferOutput, desired: bufferOutput,
95 changeHandler: [this](QAudioBufferOutput *) { m_bufferOutputChanged = true; });
96}
97
98AudioRenderer::~AudioRenderer()
99{
100 freeOutput();
101}
102
103void AudioRenderer::updateVolume()
104{
105 if (m_sink)
106 m_sink->setVolume(m_output->isMuted() ? 0.f : m_output->volume());
107}
108
109void AudioRenderer::onDeviceChanged()
110{
111 m_deviceChanged = true;
112}
113
114Renderer::RenderingResult AudioRenderer::renderInternal(Frame frame)
115{
116 if (frame.isValid())
117 updateOutputs(frame);
118
119 // push to sink first in order not to waste time on resampling
120 // for QAudioBufferOutput
121 const RenderingResult result = pushFrameToOutput(frame);
122
123 if (m_lastFramePushDone)
124 pushFrameToBufferOutput(frame);
125 // else // skip pushing the same data to QAudioBufferOutput
126
127 m_lastFramePushDone = result.done;
128
129 return result;
130}
131
132AudioRenderer::RenderingResult AudioRenderer::pushFrameToOutput(const Frame &frame)
133{
134 if (!m_ioDevice || !m_resampler)
135 return {};
136
137 Q_ASSERT(m_sink);
138
139 auto firstFrameFlagGuard = qScopeGuard(f: [&]() { m_firstFrameToSink = false; });
140
141 const SynchronizationStamp syncStamp{ .audioSinkState: m_sink->state(), .audioSinkBytesFree: m_sink->bytesFree(),
142 .bufferBytesWritten: m_bufferedData.offset, .timePoint: Clock::now() };
143
144 if (!m_bufferedData.isValid()) {
145 if (!frame.isValid()) {
146 if (std::exchange(obj&: m_drained, new_val: true))
147 return {};
148
149 const auto time = bufferLoadingTime(syncStamp);
150
151 qCDebug(qLcAudioRenderer) << "Draining AudioRenderer, time:" << time;
152
153 return { .done: time.count() == 0, .recheckInterval: time };
154 }
155
156 m_bufferedData = { .buffer: m_resampler->resample(frame: frame.avFrame()) };
157 }
158
159 if (m_bufferedData.isValid()) {
160 // synchronize after "QIODevice::write" to deliver audio data to the sink ASAP.
161 auto syncGuard = qScopeGuard(f: [&]() { updateSynchronization(stamp: syncStamp, frame); });
162
163 const auto bytesWritten = m_ioDevice->write(data: m_bufferedData.data(), len: m_bufferedData.size());
164
165 m_bufferedData.offset += bytesWritten;
166
167 if (m_bufferedData.size() <= 0) {
168 m_bufferedData = {};
169
170 return {};
171 }
172
173 const auto remainingDuration = durationForBytes(bytes: m_bufferedData.size());
174
175 return { .done: false,
176 .recheckInterval: std::min(a: remainingDuration + DurationBias, b: m_timings.actualBufferDuration / 2) };
177 }
178
179 return {};
180}
181
182void AudioRenderer::pushFrameToBufferOutput(const Frame &frame)
183{
184 if (!m_bufferOutput)
185 return;
186
187 Q_ASSERT(m_bufferOutputResampler);
188
189 if (frame.isValid()) {
190 // TODO: get buffer from m_bufferedData if resample formats are equal
191 QAudioBuffer buffer = m_bufferOutputResampler->resample(frame: frame.avFrame());
192 emit m_bufferOutput->audioBufferReceived(buffer);
193 } else {
194 emit m_bufferOutput->audioBufferReceived(buffer: {});
195 }
196}
197
198void AudioRenderer::onPlaybackRateChanged()
199{
200 m_resampler.reset();
201}
202
203int AudioRenderer::timerInterval() const
204{
205 constexpr auto MaxFixableInterval = 50; // ms
206
207 const auto interval = Renderer::timerInterval();
208
209 if (m_firstFrameToSink || !m_sink || m_sink->state() != QAudio::IdleState
210 || interval > MaxFixableInterval)
211 return interval;
212
213 return 0;
214}
215
216void AudioRenderer::onPauseChanged()
217{
218 m_firstFrameToSink = true;
219 Renderer::onPauseChanged();
220}
221
222void AudioRenderer::initResempler(const Frame &frame)
223{
224 // We recreate resampler whenever format is changed
225
226 auto resamplerFormat = m_sinkFormat;
227 resamplerFormat.setSampleRate(
228 qRound(d: m_sinkFormat.sampleRate() / playbackRate() * sampleRateFactor()));
229 m_resampler = createResampler(frame, outputFormat: resamplerFormat);
230}
231
232void AudioRenderer::freeOutput()
233{
234 qCDebug(qLcAudioRenderer) << "Free audio output";
235 if (m_sink) {
236 m_sink->reset();
237
238 // TODO: inestigate if it's enough to reset the sink without deleting
239 m_sink.reset();
240 }
241
242 m_ioDevice = nullptr;
243
244 m_bufferedData = {};
245 m_deviceChanged = false;
246 m_sinkFormat = {};
247 m_timings = {};
248 m_bufferLoadingInfo = {};
249}
250
251void AudioRenderer::updateOutputs(const Frame &frame)
252{
253 if (m_deviceChanged) {
254 freeOutput();
255 m_resampler.reset();
256 }
257
258 if (m_bufferOutput) {
259 if (m_bufferOutputChanged) {
260 m_bufferOutputChanged = false;
261 m_bufferOutputResampler.reset();
262 }
263
264 if (!m_bufferOutputResampler) {
265 QAudioFormat outputFormat = m_bufferOutput->format();
266 if (!outputFormat.isValid())
267 outputFormat = audioFormatFromFrame(frame);
268 m_bufferOutputResampler = createResampler(frame, outputFormat);
269 }
270 }
271
272 if (!m_output)
273 return;
274
275 if (!m_sinkFormat.isValid()) {
276 m_sinkFormat = audioFormatFromFrame(frame);
277 m_sinkFormat.setChannelConfig(m_output->device().channelConfiguration());
278 }
279
280 if (!m_sink) {
281 // Insert a delay here to test time offset synchronization, e.g. QThread::sleep(1)
282 m_sink = std::make_unique<QAudioSink>(args: m_output->device(), args&: m_sinkFormat);
283 updateVolume();
284 m_sink->setBufferSize(m_sinkFormat.bytesForDuration(microseconds: DesiredBufferTime.count()));
285 m_ioDevice = m_sink->start();
286 m_firstFrameToSink = true;
287
288 connect(sender: m_sink.get(), signal: &QAudioSink::stateChanged, context: this,
289 slot: &AudioRenderer::onAudioSinkStateChanged);
290
291 m_timings.actualBufferDuration = durationForBytes(bytes: m_sink->bufferSize());
292 m_timings.maxSoundDelay = qMin(a: MaxDesiredBufferTime,
293 b: m_timings.actualBufferDuration - MinDesiredFreeBufferTime);
294 m_timings.minSoundDelay = MinDesiredBufferTime;
295
296 Q_ASSERT(DurationBias < m_timings.minSoundDelay
297 && m_timings.maxSoundDelay < m_timings.actualBufferDuration);
298 }
299
300 if (!m_resampler)
301 initResempler(frame);
302}
303
304void AudioRenderer::updateSynchronization(const SynchronizationStamp &stamp, const Frame &frame)
305{
306 if (!frame.isValid())
307 return;
308
309 Q_ASSERT(m_sink);
310
311 const auto bufferLoadingTime = this->bufferLoadingTime(syncStamp: stamp);
312 const auto currentFrameDelay = frameDelay(frame, timePoint: stamp.timePoint);
313 const auto writtenTime = durationForBytes(bytes: stamp.bufferBytesWritten);
314 const auto soundDelay = currentFrameDelay + bufferLoadingTime - writtenTime;
315
316 auto synchronize = [&](microseconds fixedDelay, microseconds targetSoundDelay) {
317 // TODO: investigate if we need sample compensation here
318
319 changeRendererTime(offset: fixedDelay - targetSoundDelay);
320 if (qLcAudioRenderer().isDebugEnabled()) {
321 // clang-format off
322 qCDebug(qLcAudioRenderer)
323 << "Change rendering time:"
324 << "\n First frame:" << m_firstFrameToSink
325 << "\n Delay (frame+buffer-written):" << currentFrameDelay << "+"
326 << bufferLoadingTime << "-"
327 << writtenTime << "="
328 << soundDelay
329 << "\n Fixed delay:" << fixedDelay
330 << "\n Target delay:" << targetSoundDelay
331 << "\n Buffer durations (min/max/limit):" << m_timings.minSoundDelay
332 << m_timings.maxSoundDelay
333 << m_timings.actualBufferDuration
334 << "\n Audio sink state:" << stamp.audioSinkState;
335 // clang-format on
336 }
337 };
338
339 const auto loadingType = soundDelay > m_timings.maxSoundDelay ? BufferLoadingInfo::High
340 : soundDelay < m_timings.minSoundDelay ? BufferLoadingInfo::Low
341 : BufferLoadingInfo::Moderate;
342
343 if (loadingType != m_bufferLoadingInfo.type) {
344 // qCDebug(qLcAudioRenderer) << "Change buffer loading type:" <<
345 // m_bufferLoadingInfo.type
346 // << "->" << loadingType << "soundDelay:" << soundDelay;
347 m_bufferLoadingInfo = { .type: loadingType, .timePoint: stamp.timePoint, .delay: soundDelay };
348 }
349
350 if (loadingType != BufferLoadingInfo::Moderate) {
351 const auto isHigh = loadingType == BufferLoadingInfo::High;
352 const auto shouldHandleIdle = stamp.audioSinkState == QAudio::IdleState && !isHigh;
353
354 auto &fixedDelay = m_bufferLoadingInfo.delay;
355
356 fixedDelay = shouldHandleIdle ? soundDelay
357 : isHigh ? qMin(a: soundDelay, b: fixedDelay)
358 : qMax(a: soundDelay, b: fixedDelay);
359
360 if (stamp.timePoint - m_bufferLoadingInfo.timePoint > BufferLoadingMeasureTime
361 || (m_firstFrameToSink && isHigh) || shouldHandleIdle) {
362 const auto targetDelay = isHigh
363 ? (m_timings.maxSoundDelay + m_timings.minSoundDelay) / 2
364 : m_timings.minSoundDelay + DurationBias;
365
366 synchronize(fixedDelay, targetDelay);
367 m_bufferLoadingInfo = { .type: BufferLoadingInfo::Moderate, .timePoint: stamp.timePoint, .delay: targetDelay };
368 }
369 }
370}
371
372microseconds AudioRenderer::bufferLoadingTime(const SynchronizationStamp &syncStamp) const
373{
374 Q_ASSERT(m_sink);
375
376 if (syncStamp.audioSinkState == QAudio::IdleState)
377 return microseconds(0);
378
379 const auto bytes = qMax(a: m_sink->bufferSize() - syncStamp.audioSinkBytesFree, b: 0);
380
381#ifdef Q_OS_ANDROID
382 // The hack has been added due to QAndroidAudioSink issues (QTBUG-118609).
383 // The method QAndroidAudioSink::bytesFree returns 0 or bufferSize, intermediate values are not
384 // available now; to be fixed.
385 if (bytes == 0)
386 return m_timings.minSoundDelay + MinDesiredBufferTime;
387#endif
388
389 return durationForBytes(bytes);
390}
391
392void AudioRenderer::onAudioSinkStateChanged(QAudio::State state)
393{
394 if (state == QAudio::IdleState && !m_firstFrameToSink && !m_deviceChanged)
395 scheduleNextStep();
396}
397
398microseconds AudioRenderer::durationForBytes(qsizetype bytes) const
399{
400 return microseconds(m_sinkFormat.durationForBytes(byteCount: static_cast<qint32>(bytes)));
401}
402
403} // namespace QFFmpeg
404
405QT_END_NAMESPACE
406
407#include "moc_qffmpegaudiorenderer_p.cpp"
408

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp