1// Copyright (C) 2021 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qffmpegaudioframeconverter_p.h"
5
6#include <QtMultimedia/private/qaudiobuffer_support_p.h>
7#include <QtFFmpegMediaPluginImpl/private/qffmpegframe_p.h>
8#include <QtFFmpegMediaPluginImpl/private/qffmpegresampler_p.h>
9#include <QtFFmpegMediaPluginImpl/private/qffmpegmediaformatinfo_p.h>
10
11#if defined(Q_CC_MSVC) && defined(QT_MM_OPTIMIZE_DEBUG)
12# pragma optimize("s", on)
13#endif
14
15// TODO: namespace 3p library to prevent odr violations
16#include <signalsmith-stretch.h>
17
18QT_BEGIN_NAMESPACE
19
20namespace QFFmpeg {
21
22namespace {
23
24qreal sampleRateFactor()
25{
26 // Test purposes:
27 //
28 // The env var describes a factor for the sample rate of
29 // audio data that we feed to the audio sink.
30 //
31 // In some cases audio sink might consume data slightly slower or faster than expected;
32 // even though the synchronization in the audio renderer is supposed to handle it,
33 // it makes sense to experiment with QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR != 1.
34 //
35 // Set QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR > 1 (e.g. 1.01 - 1.1) to test high buffer
36 // loading
37 // or try compensating too fast data consumption by the audio sink.
38 // Set QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR < 1 to test low buffer loading
39 // or try compensating too slow data consumption by the audio sink.
40
41 static const qreal result = []() {
42 const auto sampleRateFactorStr =
43 qEnvironmentVariable(varName: "QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR");
44 bool ok = false;
45 const auto result = sampleRateFactorStr.toDouble(ok: &ok);
46 return ok ? result : 1.;
47 }();
48
49 return result;
50}
51
52struct TrivialAudioFrameConverter : AbstractAudioFrameConverter
53{
54 explicit TrivialAudioFrameConverter(const Frame &frame, QAudioFormat outputFormat,
55 float playbackRate)
56 {
57 int sampleRate = qRound(d: outputFormat.sampleRate() / playbackRate * sampleRateFactor());
58 outputFormat.setSampleRate(sampleRate);
59 m_converter = createResampler(frame, outputFormat);
60 }
61
62 QAudioBuffer convert(AVFrame *frame) override { return m_converter->resample(frame); }
63
64private:
65 std::unique_ptr<QFFmpegResampler> m_converter;
66};
67
68struct PitchShiftingAudioFrameConverter : AbstractAudioFrameConverter
69{
70 explicit PitchShiftingAudioFrameConverter(const Frame &frame, QAudioFormat outputFormat,
71 float playbackRate)
72 : m_playbackRate{ playbackRate }
73 {
74 const QAudioFormat mediaFormat = QFFmpegMediaFormatInfo::audioFormatFromCodecParameters(
75 codecPar: *frame.codecContext()->stream()->codecpar);
76
77 const QAudioFormat floatFormat = [&] {
78 QAudioFormat ret = mediaFormat;
79 ret.setSampleFormat(QAudioFormat::SampleFormat::Float);
80 return ret;
81 }();
82
83 m_toPCMDecoder = createResampler(frame, outputFormat: floatFormat); // this is *not* a resampler
84 m_stretcher.reset();
85 m_pendingFractionalFrames = 0.f;
86 m_stretcher.presetDefault(nChannels: mediaFormat.channelCount(), sampleRate: outputFormat.sampleRate());
87
88 const QAudioFormat pitchCompensatedFormat = [&] {
89 QAudioFormat ret = floatFormat;
90 ret.setSampleRate(outputFormat.sampleRate());
91 return ret;
92 }();
93 m_toOutputFormatConverter = QFFmpegResampler::createFromInputFormat(input: pitchCompensatedFormat, output: outputFormat,
94 startTime: frame.startTime().get());
95 }
96
97 QAudioBuffer convert(AVFrame *frame) override
98 {
99 using namespace QtPrivate;
100
101 // convert to pcm buffer
102 QAudioBuffer wordConverted = m_toPCMDecoder->resample(frame);
103
104 // compute stretch amount
105 int mediaFrameCount = wordConverted.frameCount();
106 float expectedNumberOfFrames = mediaFrameCount / m_playbackRate + m_pendingFractionalFrames;
107 int numberOfFullExpectedFrames = qFloor(v: expectedNumberOfFrames);
108 m_pendingFractionalFrames = expectedNumberOfFrames - numberOfFullExpectedFrames;
109
110 auto timeStretcherOutput = QAudioBuffer{
111 numberOfFullExpectedFrames,
112 wordConverted.format(),
113 };
114
115 // stretch
116 m_stretcher.process(
117 inputs: QAudioBufferDeinterleaveAdaptor<const float>{
118 wordConverted,
119 },
120 inputSamples: mediaFrameCount,
121 outputs: QAudioBufferDeinterleaveAdaptor<float>{
122 timeStretcherOutput,
123 },
124 outputSamples: numberOfFullExpectedFrames);
125
126 // convert to audio output format
127 QAudioBuffer outputBuffer = m_toOutputFormatConverter->resample(
128 data: timeStretcherOutput.constData<const char>(), size: timeStretcherOutput.byteCount());
129
130 return outputBuffer;
131 }
132
133private:
134 std::unique_ptr<QFFmpegResampler> m_toPCMDecoder;
135 signalsmith::stretch::SignalsmithStretch<float> m_stretcher;
136 std::unique_ptr<QFFmpegResampler> m_toOutputFormatConverter;
137 float m_playbackRate;
138 float m_pendingFractionalFrames = 0.f;
139};
140
141} // namespace
142
143AbstractAudioFrameConverter::~AbstractAudioFrameConverter() = default;
144
145std::unique_ptr<QFFmpegResampler> createResampler(const Frame &frame,
146 const QAudioFormat &outputFormat)
147{
148 return QFFmpegResampler::createFromCodecContext(frame.codecContext(), output: outputFormat, startTime: frame.startTime().get());
149}
150
151std::unique_ptr<AbstractAudioFrameConverter>
152makeTrivialAudioFrameConverter(const Frame &frame, QAudioFormat outputFormat, float playbackRate)
153{
154 return std::make_unique<TrivialAudioFrameConverter>(args: frame, args&: outputFormat, args&: playbackRate);
155}
156
157std::unique_ptr<AbstractAudioFrameConverter>
158makePitchShiftingAudioFrameConverter(const Frame &frame, QAudioFormat outputFormat,
159 float playbackRate)
160{
161 return std::make_unique<PitchShiftingAudioFrameConverter>(args: frame, args&: outputFormat, args&: playbackRate);
162}
163
164} // namespace QFFmpeg
165
166QT_END_NAMESPACE
167

source code of qtmultimedia/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudioframeconverter.cpp